@mux/ai 0.26.0 → 0.27.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{index-C5zhbQHK.d.ts → index-Zwh0Bwq3.d.ts} +3 -1
- package/dist/index.d.ts +2 -28
- package/dist/index.js +266 -95
- package/dist/index.js.map +1 -1
- package/dist/primitives/index.js +64 -5
- package/dist/primitives/index.js.map +1 -1
- package/dist/workflows/index.d.ts +1 -1
- package/dist/workflows/index.js +265 -94
- package/dist/workflows/index.js.map +1 -1
- package/package.json +1 -1
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export { A as AskQuestionsOptions, j as AskQuestionsResult, k as AskQuestionsType, l as AudioTranslationOptions, m as AudioTranslationResult, n as AutoCensorProfanityOptions, B as BurnedInCaptionsAnalysis, o as BurnedInCaptionsOptions, p as BurnedInCaptionsPromptOverrides, q as BurnedInCaptionsPromptSections, r as BurnedInCaptionsResult, C as CaptionReplacement, s as CensorMode, u as Chapter, v as ChapterSystemPromptSections, w as ChaptersOptions, x as ChaptersPromptOverrides, y as ChaptersPromptSections, z as ChaptersResult, D as ChaptersType, E as DEFAULT_DESCRIPTION_LENGTH, F as DEFAULT_SUMMARY_KEYWORD_LIMIT, G as DEFAULT_TITLE_LENGTH, H as EditCaptionsOptions, J as EditCaptionsResult, K as EmbeddingsOptions, M as EmbeddingsResult, N as EngagementInsightsOptions, O as EngagementInsightsResult, P as GOOGLE_VISION_LIKELIHOOD_TO_SCORE, Q as HIVE_SEXUAL_CATEGORIES, R as HIVE_VIOLENCE_CATEGORIES, T as HiveModerationOutput, U as HiveModerationSource, V as ModerationOptions, W as ModerationProvider, X as ModerationResult, Y as MomentInsight, Z as OverallInsight, _ as ProfanityDetectionPayload, $ as Question, a0 as QuestionAnswer, a1 as QuestionAnswerType, a2 as ReplacementRecord, a3 as SummarizationOptions, a4 as SummarizationPromptOverrides, a5 as SummarizationPromptSections, a6 as SummaryAndTagsResult, a7 as SummaryType, a8 as ThumbnailModerationScore, a9 as TranslationChunkingOptions, aa as TranslationOptions, ab as TranslationPayload, ac as TranslationResult, ad as aggregateTokenUsage, ae as applyOverrideLists, af as applyReplacements, ag as askQuestions, ah as buildReplacementRegex, ai as burnedInCaptionsSchema, aj as censorVttContent, ak as chapterSchema, al as chaptersSchema, am as createReplacer, an as editCaptions, ao as extractTranscriptSegmentsForHotspots, ap as generateChapters, aq as generateEmbeddings, ar as generateEngagementInsights, as as generateVideoEmbeddings, at as getModerationScores, au as getSummaryAndTags, av as hasBurnedInCaptions, aw as mapShotsToHotspots, ax as normalizeTranslatedVtt, ay as profanityDetectionSchema, az as questionAnswerSchema, aA as shouldSplitChunkTranslationError, aB as summarySchema, aC as transformCueText, aD as translateAudio, aE as translateCaptions, aF as translationSchema } from '../index-
|
|
1
|
+
export { A as AskQuestionsOptions, j as AskQuestionsResult, k as AskQuestionsType, l as AudioTranslationOptions, m as AudioTranslationResult, n as AutoCensorProfanityOptions, B as BurnedInCaptionsAnalysis, o as BurnedInCaptionsOptions, p as BurnedInCaptionsPromptOverrides, q as BurnedInCaptionsPromptSections, r as BurnedInCaptionsResult, C as CaptionReplacement, s as CensorMode, u as Chapter, v as ChapterSystemPromptSections, w as ChaptersOptions, x as ChaptersPromptOverrides, y as ChaptersPromptSections, z as ChaptersResult, D as ChaptersType, E as DEFAULT_DESCRIPTION_LENGTH, F as DEFAULT_SUMMARY_KEYWORD_LIMIT, G as DEFAULT_TITLE_LENGTH, H as EditCaptionsOptions, J as EditCaptionsResult, K as EmbeddingsOptions, M as EmbeddingsResult, N as EngagementInsightsOptions, O as EngagementInsightsResult, P as GOOGLE_VISION_LIKELIHOOD_TO_SCORE, Q as HIVE_SEXUAL_CATEGORIES, R as HIVE_VIOLENCE_CATEGORIES, T as HiveModerationOutput, U as HiveModerationSource, V as ModerationOptions, W as ModerationProvider, X as ModerationResult, Y as MomentInsight, Z as OverallInsight, _ as ProfanityDetectionPayload, $ as Question, a0 as QuestionAnswer, a1 as QuestionAnswerType, a2 as ReplacementRecord, a3 as SummarizationOptions, a4 as SummarizationPromptOverrides, a5 as SummarizationPromptSections, a6 as SummaryAndTagsResult, a7 as SummaryType, a8 as ThumbnailModerationScore, a9 as TranslationChunkingOptions, aa as TranslationOptions, ab as TranslationPayload, ac as TranslationResult, ad as aggregateTokenUsage, ae as applyOverrideLists, af as applyReplacements, ag as askQuestions, ah as buildReplacementRegex, ai as burnedInCaptionsSchema, aj as censorVttContent, ak as chapterSchema, al as chaptersSchema, am as createReplacer, an as editCaptions, ao as extractTranscriptSegmentsForHotspots, ap as generateChapters, aq as generateEmbeddings, ar as generateEngagementInsights, as as generateVideoEmbeddings, at as getModerationScores, au as getSummaryAndTags, av as hasBurnedInCaptions, aw as mapShotsToHotspots, ax as normalizeTranslatedVtt, ay as profanityDetectionSchema, az as questionAnswerSchema, aA as shouldSplitChunkTranslationError, aB as summarySchema, aC as transformCueText, aD as translateAudio, aE as translateCaptions, aF as translationSchema } from '../index-Zwh0Bwq3.js';
|
|
2
2
|
import 'zod';
|
|
3
3
|
import '@ai-sdk/anthropic';
|
|
4
4
|
import '@ai-sdk/google';
|
package/dist/workflows/index.js
CHANGED
|
@@ -492,6 +492,68 @@ function createSafetyReporter() {
|
|
|
492
492
|
};
|
|
493
493
|
}
|
|
494
494
|
|
|
495
|
+
// src/lib/token-usage.ts
|
|
496
|
+
var TOKEN_USAGE_FIELDS = [
|
|
497
|
+
"inputTokens",
|
|
498
|
+
"outputTokens",
|
|
499
|
+
"totalTokens",
|
|
500
|
+
"reasoningTokens",
|
|
501
|
+
"cachedInputTokens"
|
|
502
|
+
];
|
|
503
|
+
function isDefinedTokenUsageValue(value) {
|
|
504
|
+
return typeof value === "number";
|
|
505
|
+
}
|
|
506
|
+
function aggregateTokenUsage(usages) {
|
|
507
|
+
return TOKEN_USAGE_FIELDS.reduce((aggregate, field) => {
|
|
508
|
+
const values = usages.map((usage) => usage[field]).filter(isDefinedTokenUsageValue);
|
|
509
|
+
if (values.length > 0) {
|
|
510
|
+
aggregate[field] = values.reduce((total, value) => total + value, 0);
|
|
511
|
+
}
|
|
512
|
+
return aggregate;
|
|
513
|
+
}, {});
|
|
514
|
+
}
|
|
515
|
+
function getErrorTokenUsage(error) {
|
|
516
|
+
if (typeof error !== "object" || error === null) {
|
|
517
|
+
return void 0;
|
|
518
|
+
}
|
|
519
|
+
const usage = error.usage;
|
|
520
|
+
if (typeof usage !== "object" || usage === null) {
|
|
521
|
+
return void 0;
|
|
522
|
+
}
|
|
523
|
+
const source = usage;
|
|
524
|
+
const extracted = {};
|
|
525
|
+
for (const field of TOKEN_USAGE_FIELDS) {
|
|
526
|
+
const value = source[field];
|
|
527
|
+
if (typeof value === "number") {
|
|
528
|
+
extracted[field] = value;
|
|
529
|
+
}
|
|
530
|
+
}
|
|
531
|
+
if (extracted.reasoningTokens === void 0) {
|
|
532
|
+
const reasoningTokens = source.outputTokenDetails?.reasoningTokens;
|
|
533
|
+
if (typeof reasoningTokens === "number") {
|
|
534
|
+
extracted.reasoningTokens = reasoningTokens;
|
|
535
|
+
}
|
|
536
|
+
}
|
|
537
|
+
if (extracted.cachedInputTokens === void 0) {
|
|
538
|
+
const cacheReadTokens = source.inputTokenDetails?.cacheReadTokens;
|
|
539
|
+
if (typeof cacheReadTokens === "number") {
|
|
540
|
+
extracted.cachedInputTokens = cacheReadTokens;
|
|
541
|
+
}
|
|
542
|
+
}
|
|
543
|
+
return Object.keys(extracted).length > 0 ? extracted : void 0;
|
|
544
|
+
}
|
|
545
|
+
function rethrowWithTokenUsage(error, collectedUsage) {
|
|
546
|
+
const usages = [...collectedUsage];
|
|
547
|
+
const errorUsage = getErrorTokenUsage(error);
|
|
548
|
+
if (errorUsage) {
|
|
549
|
+
usages.push(errorUsage);
|
|
550
|
+
}
|
|
551
|
+
if (usages.length > 0 && typeof error === "object" && error !== null) {
|
|
552
|
+
error.usage = aggregateTokenUsage(usages);
|
|
553
|
+
}
|
|
554
|
+
throw error;
|
|
555
|
+
}
|
|
556
|
+
|
|
495
557
|
// src/lib/mux-ai-error.ts
|
|
496
558
|
var MuxAiError = class extends Error {
|
|
497
559
|
constructor(message, opts) {
|
|
@@ -504,18 +566,38 @@ var MuxAiError = class extends Error {
|
|
|
504
566
|
this.retryable = opts?.retryable ?? false;
|
|
505
567
|
}
|
|
506
568
|
};
|
|
569
|
+
function extractErrorDetail(error) {
|
|
570
|
+
if (error instanceof Error && error.message) {
|
|
571
|
+
return error.message;
|
|
572
|
+
}
|
|
573
|
+
if (typeof error === "string" && error.length > 0) {
|
|
574
|
+
return error;
|
|
575
|
+
}
|
|
576
|
+
if (typeof error === "object" && error !== null) {
|
|
577
|
+
const message = error.message;
|
|
578
|
+
if (typeof message === "string" && message.length > 0) {
|
|
579
|
+
return message;
|
|
580
|
+
}
|
|
581
|
+
}
|
|
582
|
+
return "Unknown error";
|
|
583
|
+
}
|
|
507
584
|
function wrapError(error, message) {
|
|
508
585
|
if (error instanceof MuxAiError) {
|
|
509
586
|
throw error;
|
|
510
587
|
}
|
|
511
|
-
const rawDetail = error
|
|
588
|
+
const rawDetail = extractErrorDetail(error);
|
|
512
589
|
const leakReason = detectLeakReason(rawDetail);
|
|
513
590
|
const shouldSuppress = leakReason === "canary" || leakReason === "prompt_tag";
|
|
514
591
|
const detail = shouldSuppress ? "Upstream error details suppressed by safety filter" : rawDetail;
|
|
515
592
|
if (shouldSuppress) {
|
|
516
593
|
console.warn(`[@mux/ai] Suppressed suspected prompt leak in wrapped error (context: ${message}, reason: ${leakReason}).`);
|
|
517
594
|
}
|
|
518
|
-
|
|
595
|
+
const wrapped = new Error(`${message}: ${detail}`);
|
|
596
|
+
const usage = getErrorTokenUsage(error);
|
|
597
|
+
if (usage) {
|
|
598
|
+
wrapped.usage = usage;
|
|
599
|
+
}
|
|
600
|
+
throw wrapped;
|
|
519
601
|
}
|
|
520
602
|
|
|
521
603
|
// src/lib/providers.ts
|
|
@@ -865,7 +947,7 @@ var DEFAULT_EMBEDDING_MODELS = {
|
|
|
865
947
|
var LANGUAGE_MODELS = {
|
|
866
948
|
openai: ["gpt-5.1", "gpt-5-mini"],
|
|
867
949
|
anthropic: ["claude-sonnet-4-5"],
|
|
868
|
-
google: ["gemini-3-flash-preview", "gemini-3.1-flash-lite", "gemini-
|
|
950
|
+
google: ["gemini-3-flash-preview", "gemini-3.1-flash-lite", "gemini-2.5-flash"]
|
|
869
951
|
};
|
|
870
952
|
var LANGUAGE_MODEL_DEPRECATIONS = [
|
|
871
953
|
{
|
|
@@ -881,10 +963,10 @@ var LANGUAGE_MODEL_DEPRECATIONS = [
|
|
|
881
963
|
provider: "google",
|
|
882
964
|
modelId: "gemini-3.1-flash-lite-preview",
|
|
883
965
|
replacementModelId: "gemini-3.1-flash-lite",
|
|
884
|
-
phase: "
|
|
966
|
+
phase: "blocked",
|
|
885
967
|
deprecatedOn: "2026-05-12",
|
|
886
968
|
sunsetOn: "2026-05-25",
|
|
887
|
-
reason: "
|
|
969
|
+
reason: "Google shut down Gemini 3.1 Flash-Lite Preview on 2026-05-25."
|
|
888
970
|
}
|
|
889
971
|
];
|
|
890
972
|
var warnedDeprecatedLanguageModels = /* @__PURE__ */ new Set();
|
|
@@ -2331,6 +2413,14 @@ async function analyzeQuestions({
|
|
|
2331
2413
|
}
|
|
2332
2414
|
async function askQuestions(assetId, questions, options) {
|
|
2333
2415
|
"use workflow";
|
|
2416
|
+
const collectedUsage = [];
|
|
2417
|
+
try {
|
|
2418
|
+
return await askQuestionsInternal(assetId, questions, options, collectedUsage);
|
|
2419
|
+
} catch (error) {
|
|
2420
|
+
rethrowWithTokenUsage(error, collectedUsage);
|
|
2421
|
+
}
|
|
2422
|
+
}
|
|
2423
|
+
async function askQuestionsInternal(assetId, questions, options, collectedUsage) {
|
|
2334
2424
|
if (!questions || questions.length === 0) {
|
|
2335
2425
|
throw new MuxAiError("At least one question must be provided.", { type: "validation_error" });
|
|
2336
2426
|
}
|
|
@@ -2479,6 +2569,7 @@ async function askQuestions(assetId, questions, options) {
|
|
|
2479
2569
|
const contentType = isAudioOnly ? "audio" : "video";
|
|
2480
2570
|
wrapError(error, `Failed to analyze ${contentType} questions with ${provider}`);
|
|
2481
2571
|
}
|
|
2572
|
+
collectedUsage.push(analysisResponse.usage);
|
|
2482
2573
|
if (!analysisResponse.result?.answers) {
|
|
2483
2574
|
throw new MuxAiError(`Failed to generate answers for asset ${assetId}.`);
|
|
2484
2575
|
}
|
|
@@ -2712,6 +2803,14 @@ async function analyzeStoryboard({
|
|
|
2712
2803
|
}
|
|
2713
2804
|
async function hasBurnedInCaptions(assetId, options = {}) {
|
|
2714
2805
|
"use workflow";
|
|
2806
|
+
const collectedUsage = [];
|
|
2807
|
+
try {
|
|
2808
|
+
return await hasBurnedInCaptionsInternal(assetId, options, collectedUsage);
|
|
2809
|
+
} catch (error) {
|
|
2810
|
+
rethrowWithTokenUsage(error, collectedUsage);
|
|
2811
|
+
}
|
|
2812
|
+
}
|
|
2813
|
+
async function hasBurnedInCaptionsInternal(assetId, options, collectedUsage) {
|
|
2715
2814
|
const {
|
|
2716
2815
|
provider = DEFAULT_PROVIDER,
|
|
2717
2816
|
model,
|
|
@@ -2751,6 +2850,7 @@ async function hasBurnedInCaptions(assetId, options = {}) {
|
|
|
2751
2850
|
credentials
|
|
2752
2851
|
});
|
|
2753
2852
|
}
|
|
2853
|
+
collectedUsage.push(analysisResponse.usage);
|
|
2754
2854
|
if (!analysisResponse.result) {
|
|
2755
2855
|
throw new Error("No analysis result received from AI provider");
|
|
2756
2856
|
}
|
|
@@ -2952,6 +3052,14 @@ function buildUserPrompt3({
|
|
|
2952
3052
|
}
|
|
2953
3053
|
async function generateChapters(assetId, options = {}) {
|
|
2954
3054
|
"use workflow";
|
|
3055
|
+
const collectedUsage = [];
|
|
3056
|
+
try {
|
|
3057
|
+
return await generateChaptersInternal(assetId, options, collectedUsage);
|
|
3058
|
+
} catch (error) {
|
|
3059
|
+
rethrowWithTokenUsage(error, collectedUsage);
|
|
3060
|
+
}
|
|
3061
|
+
}
|
|
3062
|
+
async function generateChaptersInternal(assetId, options, collectedUsage) {
|
|
2955
3063
|
const {
|
|
2956
3064
|
languageCode,
|
|
2957
3065
|
provider = "openai",
|
|
@@ -3019,6 +3127,9 @@ async function generateChapters(assetId, options = {}) {
|
|
|
3019
3127
|
} catch (error) {
|
|
3020
3128
|
wrapError(error, `Failed to generate chapters with ${provider}`);
|
|
3021
3129
|
}
|
|
3130
|
+
if (chaptersData) {
|
|
3131
|
+
collectedUsage.push(chaptersData.usage);
|
|
3132
|
+
}
|
|
3022
3133
|
if (!chaptersData || !chaptersData.chapters) {
|
|
3023
3134
|
throw new MuxAiError(`Failed to generate chapters for asset ${assetId}.`);
|
|
3024
3135
|
}
|
|
@@ -3605,6 +3716,14 @@ async function deleteTrackOnMux(assetId, trackId, credentials) {
|
|
|
3605
3716
|
}
|
|
3606
3717
|
async function editCaptions(assetId, trackId, options) {
|
|
3607
3718
|
"use workflow";
|
|
3719
|
+
const collectedUsage = [];
|
|
3720
|
+
try {
|
|
3721
|
+
return await editCaptionsInternal(assetId, trackId, options, collectedUsage);
|
|
3722
|
+
} catch (error) {
|
|
3723
|
+
rethrowWithTokenUsage(error, collectedUsage);
|
|
3724
|
+
}
|
|
3725
|
+
}
|
|
3726
|
+
async function editCaptionsInternal(assetId, trackId, options, collectedUsage) {
|
|
3608
3727
|
const {
|
|
3609
3728
|
provider,
|
|
3610
3729
|
model,
|
|
@@ -3693,6 +3812,7 @@ async function editCaptions(assetId, trackId, options) {
|
|
|
3693
3812
|
});
|
|
3694
3813
|
detectedProfanity = result.profanity;
|
|
3695
3814
|
usage = result.usage;
|
|
3815
|
+
collectedUsage.push(result.usage);
|
|
3696
3816
|
if (result.unexpectedKeys.length > 0) {
|
|
3697
3817
|
console.warn(
|
|
3698
3818
|
`[@mux/ai] Model emitted unexpected keys in profanity detection (stripped): ${result.unexpectedKeys.join(", ")}.`
|
|
@@ -4053,16 +4173,19 @@ async function generateSingleChunkEmbedding({
|
|
|
4053
4173
|
})
|
|
4054
4174
|
);
|
|
4055
4175
|
return {
|
|
4056
|
-
|
|
4057
|
-
|
|
4058
|
-
|
|
4059
|
-
|
|
4060
|
-
|
|
4061
|
-
|
|
4062
|
-
|
|
4176
|
+
chunk: {
|
|
4177
|
+
chunkId: chunk.id,
|
|
4178
|
+
embedding: response.embedding,
|
|
4179
|
+
metadata: {
|
|
4180
|
+
startTime: chunk.startTime,
|
|
4181
|
+
endTime: chunk.endTime,
|
|
4182
|
+
tokenCount: chunk.tokenCount
|
|
4183
|
+
}
|
|
4184
|
+
},
|
|
4185
|
+
usage: typeof response.usage?.tokens === "number" ? { inputTokens: response.usage.tokens, totalTokens: response.usage.tokens } : void 0
|
|
4063
4186
|
};
|
|
4064
4187
|
}
|
|
4065
|
-
async function generateEmbeddingsInternal(assetId, options
|
|
4188
|
+
async function generateEmbeddingsInternal(assetId, options, collectedUsage) {
|
|
4066
4189
|
const {
|
|
4067
4190
|
provider = "openai",
|
|
4068
4191
|
model,
|
|
@@ -4101,7 +4224,7 @@ async function generateEmbeddingsInternal(assetId, options = {}) {
|
|
|
4101
4224
|
try {
|
|
4102
4225
|
for (let i = 0; i < chunks.length; i += batchSize) {
|
|
4103
4226
|
const batch = chunks.slice(i, i + batchSize);
|
|
4104
|
-
const
|
|
4227
|
+
const batchOutcomes = await Promise.allSettled(
|
|
4105
4228
|
batch.map(
|
|
4106
4229
|
(chunk) => generateSingleChunkEmbedding({
|
|
4107
4230
|
chunk,
|
|
@@ -4111,7 +4234,24 @@ async function generateEmbeddingsInternal(assetId, options = {}) {
|
|
|
4111
4234
|
})
|
|
4112
4235
|
)
|
|
4113
4236
|
);
|
|
4114
|
-
|
|
4237
|
+
const failures = [];
|
|
4238
|
+
for (const outcome of batchOutcomes) {
|
|
4239
|
+
if (outcome.status === "fulfilled") {
|
|
4240
|
+
if (outcome.value.usage) {
|
|
4241
|
+
collectedUsage.push(outcome.value.usage);
|
|
4242
|
+
}
|
|
4243
|
+
chunkEmbeddings.push(outcome.value.chunk);
|
|
4244
|
+
} else {
|
|
4245
|
+
const failureUsage = getErrorTokenUsage(outcome.reason);
|
|
4246
|
+
if (failureUsage) {
|
|
4247
|
+
collectedUsage.push(failureUsage);
|
|
4248
|
+
}
|
|
4249
|
+
failures.push(outcome.reason);
|
|
4250
|
+
}
|
|
4251
|
+
}
|
|
4252
|
+
if (failures.length > 0) {
|
|
4253
|
+
throw failures[0];
|
|
4254
|
+
}
|
|
4115
4255
|
}
|
|
4116
4256
|
} catch (error) {
|
|
4117
4257
|
throw new Error(
|
|
@@ -4145,12 +4285,22 @@ async function generateEmbeddingsInternal(assetId, options = {}) {
|
|
|
4145
4285
|
}
|
|
4146
4286
|
async function generateEmbeddings(assetId, options = {}) {
|
|
4147
4287
|
"use workflow";
|
|
4148
|
-
|
|
4288
|
+
const collectedUsage = [];
|
|
4289
|
+
try {
|
|
4290
|
+
return await generateEmbeddingsInternal(assetId, options, collectedUsage);
|
|
4291
|
+
} catch (error) {
|
|
4292
|
+
rethrowWithTokenUsage(error, collectedUsage);
|
|
4293
|
+
}
|
|
4149
4294
|
}
|
|
4150
4295
|
async function generateVideoEmbeddings(assetId, options = {}) {
|
|
4151
4296
|
"use workflow";
|
|
4152
4297
|
console.warn("generateVideoEmbeddings is deprecated. Use generateEmbeddings instead.");
|
|
4153
|
-
|
|
4298
|
+
const collectedUsage = [];
|
|
4299
|
+
try {
|
|
4300
|
+
return await generateEmbeddingsInternal(assetId, options, collectedUsage);
|
|
4301
|
+
} catch (error) {
|
|
4302
|
+
rethrowWithTokenUsage(error, collectedUsage);
|
|
4303
|
+
}
|
|
4154
4304
|
}
|
|
4155
4305
|
|
|
4156
4306
|
// src/workflows/engagement-insights.ts
|
|
@@ -4635,6 +4785,14 @@ async function generateInsightsWithAI(provider, modelId, systemPrompt, userPromp
|
|
|
4635
4785
|
}
|
|
4636
4786
|
async function generateEngagementInsights(assetId, options = {}) {
|
|
4637
4787
|
"use workflow";
|
|
4788
|
+
const collectedUsage = [];
|
|
4789
|
+
try {
|
|
4790
|
+
return await generateEngagementInsightsInternal(assetId, options, collectedUsage);
|
|
4791
|
+
} catch (error) {
|
|
4792
|
+
rethrowWithTokenUsage(error, collectedUsage);
|
|
4793
|
+
}
|
|
4794
|
+
}
|
|
4795
|
+
async function generateEngagementInsightsInternal(assetId, options, collectedUsage) {
|
|
4638
4796
|
const {
|
|
4639
4797
|
provider = "openai",
|
|
4640
4798
|
model,
|
|
@@ -4761,6 +4919,7 @@ async function generateEngagementInsights(assetId, options = {}) {
|
|
|
4761
4919
|
credentials
|
|
4762
4920
|
);
|
|
4763
4921
|
const { result: aiInsights, usage } = aiStepResult;
|
|
4922
|
+
collectedUsage.push(usage);
|
|
4764
4923
|
if (!aiInsights.momentInsights || aiInsights.momentInsights.length === 0) {
|
|
4765
4924
|
throw new MuxAiError(`Failed to generate insights for asset ${assetId}.`);
|
|
4766
4925
|
}
|
|
@@ -5945,6 +6104,14 @@ function normalizeKeywords(keywords, limit = DEFAULT_SUMMARY_KEYWORD_LIMIT) {
|
|
|
5945
6104
|
}
|
|
5946
6105
|
async function getSummaryAndTags(assetId, options) {
|
|
5947
6106
|
"use workflow";
|
|
6107
|
+
const collectedUsage = [];
|
|
6108
|
+
try {
|
|
6109
|
+
return await getSummaryAndTagsInternal(assetId, options, collectedUsage);
|
|
6110
|
+
} catch (error) {
|
|
6111
|
+
rethrowWithTokenUsage(error, collectedUsage);
|
|
6112
|
+
}
|
|
6113
|
+
}
|
|
6114
|
+
async function getSummaryAndTagsInternal(assetId, options, collectedUsage) {
|
|
5948
6115
|
const {
|
|
5949
6116
|
provider = "openai",
|
|
5950
6117
|
model,
|
|
@@ -6054,6 +6221,7 @@ async function getSummaryAndTags(assetId, options) {
|
|
|
6054
6221
|
const contentType = isAudioOnly ? "audio" : "video";
|
|
6055
6222
|
wrapError(error, `Failed to analyze ${contentType} content with ${provider}`);
|
|
6056
6223
|
}
|
|
6224
|
+
collectedUsage.push(analysisResponse.usage);
|
|
6057
6225
|
if (!analysisResponse.result) {
|
|
6058
6226
|
const contentType = isAudioOnly ? "audio" : "video";
|
|
6059
6227
|
throw new MuxAiError(`Failed to analyze ${contentType} content for asset ${assetId}.`);
|
|
@@ -6096,12 +6264,9 @@ async function getSummaryAndTags(assetId, options) {
|
|
|
6096
6264
|
}
|
|
6097
6265
|
|
|
6098
6266
|
// src/workflows/translate-audio.ts
|
|
6267
|
+
import { sleep } from "workflow";
|
|
6099
6268
|
var STATIC_RENDITION_POLL_INTERVAL_MS = 5e3;
|
|
6100
6269
|
var STATIC_RENDITION_MAX_ATTEMPTS = 36;
|
|
6101
|
-
async function sleep(ms) {
|
|
6102
|
-
"use step";
|
|
6103
|
-
await new Promise((resolve) => setTimeout(resolve, ms));
|
|
6104
|
-
}
|
|
6105
6270
|
function getReadyAudioStaticRendition(asset) {
|
|
6106
6271
|
const files = asset.static_renditions?.files;
|
|
6107
6272
|
if (!files || files.length === 0) {
|
|
@@ -6142,14 +6307,17 @@ async function requestStaticRenditionCreation(assetId, credentials) {
|
|
|
6142
6307
|
wrapError(error, "Failed to request static rendition from Mux");
|
|
6143
6308
|
}
|
|
6144
6309
|
}
|
|
6310
|
+
async function retrieveAsset(assetId, credentials) {
|
|
6311
|
+
"use step";
|
|
6312
|
+
const muxClient = await resolveMuxClient(credentials);
|
|
6313
|
+
const mux = await muxClient.createClient();
|
|
6314
|
+
return mux.video.assets.retrieve(assetId);
|
|
6315
|
+
}
|
|
6145
6316
|
async function waitForAudioStaticRendition({
|
|
6146
6317
|
assetId,
|
|
6147
6318
|
initialAsset,
|
|
6148
6319
|
credentials
|
|
6149
6320
|
}) {
|
|
6150
|
-
"use step";
|
|
6151
|
-
const muxClient = await resolveMuxClient(credentials);
|
|
6152
|
-
const mux = await muxClient.createClient();
|
|
6153
6321
|
let currentAsset = initialAsset;
|
|
6154
6322
|
if (hasReadyAudioStaticRendition(currentAsset)) {
|
|
6155
6323
|
return currentAsset;
|
|
@@ -6164,7 +6332,7 @@ async function waitForAudioStaticRendition({
|
|
|
6164
6332
|
}
|
|
6165
6333
|
for (let attempt = 1; attempt <= STATIC_RENDITION_MAX_ATTEMPTS; attempt++) {
|
|
6166
6334
|
await sleep(STATIC_RENDITION_POLL_INTERVAL_MS);
|
|
6167
|
-
currentAsset = await
|
|
6335
|
+
currentAsset = await retrieveAsset(assetId, credentials);
|
|
6168
6336
|
if (hasReadyAudioStaticRendition(currentAsset)) {
|
|
6169
6337
|
return currentAsset;
|
|
6170
6338
|
}
|
|
@@ -6183,16 +6351,8 @@ async function waitForAudioStaticRendition({
|
|
|
6183
6351
|
{ type: "timeout_error", retryable: true }
|
|
6184
6352
|
);
|
|
6185
6353
|
}
|
|
6186
|
-
async function fetchAudioFromMux(audioUrl) {
|
|
6187
|
-
"use step";
|
|
6188
|
-
const audioResponse = await fetch(audioUrl);
|
|
6189
|
-
if (!audioResponse.ok) {
|
|
6190
|
-
throw new Error(`Failed to fetch audio file: ${audioResponse.statusText}`);
|
|
6191
|
-
}
|
|
6192
|
-
return audioResponse.arrayBuffer();
|
|
6193
|
-
}
|
|
6194
6354
|
async function createElevenLabsDubbingJob({
|
|
6195
|
-
|
|
6355
|
+
sourceUrl,
|
|
6196
6356
|
assetId,
|
|
6197
6357
|
elevenLabsLangCode,
|
|
6198
6358
|
elevenLabsSourceLangCode,
|
|
@@ -6201,9 +6361,8 @@ async function createElevenLabsDubbingJob({
|
|
|
6201
6361
|
}) {
|
|
6202
6362
|
"use step";
|
|
6203
6363
|
const elevenLabsApiKey = await getApiKeyFromEnv("elevenlabs", credentials);
|
|
6204
|
-
const audioBlob = new Blob([audioBuffer], { type: "audio/mp4" });
|
|
6205
6364
|
const formData = new FormData();
|
|
6206
|
-
formData.append("
|
|
6365
|
+
formData.append("source_url", sourceUrl);
|
|
6207
6366
|
formData.append("target_lang", elevenLabsLangCode);
|
|
6208
6367
|
if (elevenLabsSourceLangCode) {
|
|
6209
6368
|
formData.append("source_lang", elevenLabsSourceLangCode);
|
|
@@ -6368,13 +6527,6 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
|
|
|
6368
6527
|
if (policy === "signed") {
|
|
6369
6528
|
audioUrl = await signUrl(audioUrl, playbackId, "video", void 0, credentials);
|
|
6370
6529
|
}
|
|
6371
|
-
console.warn("\u{1F399}\uFE0F Fetching audio from Mux...");
|
|
6372
|
-
let audioBuffer;
|
|
6373
|
-
try {
|
|
6374
|
-
audioBuffer = await fetchAudioFromMux(audioUrl);
|
|
6375
|
-
} catch (error) {
|
|
6376
|
-
wrapError(error, "Failed to fetch audio from Mux");
|
|
6377
|
-
}
|
|
6378
6530
|
console.warn("\u{1F399}\uFE0F Creating dubbing job in ElevenLabs...");
|
|
6379
6531
|
const elevenLabsLangCode = toISO639_3(toLanguageCode);
|
|
6380
6532
|
const normalizedFromLanguageCode = fromLanguageCode?.trim();
|
|
@@ -6385,7 +6537,7 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
|
|
|
6385
6537
|
let dubbingId;
|
|
6386
6538
|
try {
|
|
6387
6539
|
dubbingId = await createElevenLabsDubbingJob({
|
|
6388
|
-
|
|
6540
|
+
sourceUrl: audioUrl,
|
|
6389
6541
|
assetId,
|
|
6390
6542
|
elevenLabsLangCode,
|
|
6391
6543
|
elevenLabsSourceLangCode,
|
|
@@ -6401,7 +6553,7 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
|
|
|
6401
6553
|
let pollAttempts = 0;
|
|
6402
6554
|
const maxPollAttempts = 180;
|
|
6403
6555
|
let targetLanguages = [];
|
|
6404
|
-
while (dubbingStatus
|
|
6556
|
+
while (dubbingStatus !== "dubbed" && pollAttempts < maxPollAttempts) {
|
|
6405
6557
|
await sleep(1e4);
|
|
6406
6558
|
pollAttempts++;
|
|
6407
6559
|
try {
|
|
@@ -6411,12 +6563,12 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
|
|
|
6411
6563
|
});
|
|
6412
6564
|
dubbingStatus = statusResult.status;
|
|
6413
6565
|
targetLanguages = statusResult.targetLanguages;
|
|
6414
|
-
if (dubbingStatus === "failed") {
|
|
6415
|
-
throw new Error("ElevenLabs dubbing job failed");
|
|
6416
|
-
}
|
|
6417
6566
|
} catch (error) {
|
|
6418
6567
|
wrapError(error, "Failed to check dubbing status");
|
|
6419
6568
|
}
|
|
6569
|
+
if (dubbingStatus === "failed") {
|
|
6570
|
+
throw new MuxAiError("ElevenLabs reported that the dubbing job failed.", { type: "processing_error" });
|
|
6571
|
+
}
|
|
6420
6572
|
}
|
|
6421
6573
|
if (dubbingStatus !== "dubbed") {
|
|
6422
6574
|
throw new MuxAiError("Audio translation timed out or failed. Please try again.", { type: "timeout_error", retryable: true });
|
|
@@ -6563,13 +6715,6 @@ var DEFAULT_TRANSLATION_CHUNKING = {
|
|
|
6563
6715
|
maxCuesPerChunk: 80,
|
|
6564
6716
|
maxCueTextTokensPerChunk: 2e3
|
|
6565
6717
|
};
|
|
6566
|
-
var TOKEN_USAGE_FIELDS = [
|
|
6567
|
-
"inputTokens",
|
|
6568
|
-
"outputTokens",
|
|
6569
|
-
"totalTokens",
|
|
6570
|
-
"reasoningTokens",
|
|
6571
|
-
"cachedInputTokens"
|
|
6572
|
-
];
|
|
6573
6718
|
var TranslationChunkValidationError = class extends Error {
|
|
6574
6719
|
constructor(message) {
|
|
6575
6720
|
super(message);
|
|
@@ -6600,9 +6745,6 @@ function shouldSplitChunkTranslationError(error) {
|
|
|
6600
6745
|
}
|
|
6601
6746
|
return NoObjectGeneratedError.isInstance(error) || TypeValidationError.isInstance(error) || isTranslationChunkValidationError(error);
|
|
6602
6747
|
}
|
|
6603
|
-
function isDefinedTokenUsageValue(value) {
|
|
6604
|
-
return typeof value === "number";
|
|
6605
|
-
}
|
|
6606
6748
|
function resolveTranslationChunkingOptions(options) {
|
|
6607
6749
|
const targetChunkDurationSeconds = Math.max(
|
|
6608
6750
|
1,
|
|
@@ -6629,14 +6771,27 @@ function resolveTranslationChunkingOptions(options) {
|
|
|
6629
6771
|
)
|
|
6630
6772
|
};
|
|
6631
6773
|
}
|
|
6632
|
-
function
|
|
6633
|
-
|
|
6634
|
-
|
|
6635
|
-
|
|
6636
|
-
|
|
6774
|
+
function collectSettledTranslationsOrRethrow(outcomes, priorUsage) {
|
|
6775
|
+
const results = [];
|
|
6776
|
+
const failures = [];
|
|
6777
|
+
for (const outcome of outcomes) {
|
|
6778
|
+
if (outcome.status === "fulfilled") {
|
|
6779
|
+
results.push(outcome.value);
|
|
6780
|
+
} else {
|
|
6781
|
+
failures.push(outcome.reason);
|
|
6637
6782
|
}
|
|
6638
|
-
|
|
6639
|
-
|
|
6783
|
+
}
|
|
6784
|
+
if (failures.length > 0) {
|
|
6785
|
+
const partialUsage = [...priorUsage, ...results.map((result) => result.usage)];
|
|
6786
|
+
for (const failure of failures.slice(1)) {
|
|
6787
|
+
const usage = getErrorTokenUsage(failure);
|
|
6788
|
+
if (usage) {
|
|
6789
|
+
partialUsage.push(usage);
|
|
6790
|
+
}
|
|
6791
|
+
}
|
|
6792
|
+
rethrowWithTokenUsage(failures[0], partialUsage);
|
|
6793
|
+
}
|
|
6794
|
+
return results;
|
|
6640
6795
|
}
|
|
6641
6796
|
function createTranslationChunkRequest(id, cues, cueBlocks) {
|
|
6642
6797
|
return {
|
|
@@ -6912,24 +7067,28 @@ async function translateChunkWithFallback({
|
|
|
6912
7067
|
wrapError(error, `Chunk ${chunk.id} failed for ${Math.round(chunk.startTime)}s-${Math.round(chunk.endTime)}s`);
|
|
6913
7068
|
}
|
|
6914
7069
|
const [leftChunk, rightChunk] = splitTranslationChunkAtMidpoint(chunk);
|
|
6915
|
-
const
|
|
6916
|
-
|
|
6917
|
-
|
|
6918
|
-
|
|
6919
|
-
|
|
6920
|
-
|
|
6921
|
-
|
|
6922
|
-
|
|
6923
|
-
|
|
6924
|
-
|
|
6925
|
-
|
|
6926
|
-
|
|
6927
|
-
|
|
6928
|
-
|
|
6929
|
-
|
|
6930
|
-
|
|
6931
|
-
|
|
6932
|
-
|
|
7070
|
+
const failedAttemptUsage = getErrorTokenUsage(error);
|
|
7071
|
+
const [leftResult, rightResult] = collectSettledTranslationsOrRethrow(
|
|
7072
|
+
await Promise.allSettled([
|
|
7073
|
+
translateChunkWithFallback({
|
|
7074
|
+
chunk: leftChunk,
|
|
7075
|
+
fromLanguageCode,
|
|
7076
|
+
toLanguageCode,
|
|
7077
|
+
provider,
|
|
7078
|
+
modelId,
|
|
7079
|
+
credentials
|
|
7080
|
+
}),
|
|
7081
|
+
translateChunkWithFallback({
|
|
7082
|
+
chunk: rightChunk,
|
|
7083
|
+
fromLanguageCode,
|
|
7084
|
+
toLanguageCode,
|
|
7085
|
+
provider,
|
|
7086
|
+
modelId,
|
|
7087
|
+
credentials
|
|
7088
|
+
})
|
|
7089
|
+
]),
|
|
7090
|
+
failedAttemptUsage ? [failedAttemptUsage] : []
|
|
7091
|
+
);
|
|
6933
7092
|
return {
|
|
6934
7093
|
translatedVtt: concatenateVttSegments([leftResult.translatedVtt, rightResult.translatedVtt]),
|
|
6935
7094
|
usage: aggregateTokenUsage([leftResult.usage, rightResult.usage]),
|
|
@@ -6970,17 +7129,20 @@ async function translateCaptionTrack({
|
|
|
6970
7129
|
let totalUnexpectedKeyCount = 0;
|
|
6971
7130
|
for (let index = 0; index < chunkPlan.chunks.length; index += resolvedChunking.maxConcurrentTranslations) {
|
|
6972
7131
|
const batch = chunkPlan.chunks.slice(index, index + resolvedChunking.maxConcurrentTranslations);
|
|
6973
|
-
const batchResults =
|
|
6974
|
-
|
|
6975
|
-
(
|
|
6976
|
-
chunk
|
|
6977
|
-
|
|
6978
|
-
|
|
6979
|
-
|
|
6980
|
-
|
|
6981
|
-
|
|
6982
|
-
|
|
6983
|
-
|
|
7132
|
+
const batchResults = collectSettledTranslationsOrRethrow(
|
|
7133
|
+
await Promise.allSettled(
|
|
7134
|
+
batch.map(
|
|
7135
|
+
(chunk) => translateChunkWithFallback({
|
|
7136
|
+
chunk,
|
|
7137
|
+
fromLanguageCode,
|
|
7138
|
+
toLanguageCode,
|
|
7139
|
+
provider,
|
|
7140
|
+
modelId,
|
|
7141
|
+
credentials
|
|
7142
|
+
})
|
|
7143
|
+
)
|
|
7144
|
+
),
|
|
7145
|
+
usageByChunk
|
|
6984
7146
|
);
|
|
6985
7147
|
translatedSegments.push(...batchResults.map((result) => result.translatedVtt));
|
|
6986
7148
|
usageByChunk.push(...batchResults.map((result) => result.usage));
|
|
@@ -7033,6 +7195,14 @@ async function uploadVttToS3({
|
|
|
7033
7195
|
}
|
|
7034
7196
|
async function translateCaptions(assetId, trackId, toLanguageCode, options) {
|
|
7035
7197
|
"use workflow";
|
|
7198
|
+
const collectedUsage = [];
|
|
7199
|
+
try {
|
|
7200
|
+
return await translateCaptionsInternal(assetId, trackId, toLanguageCode, options, collectedUsage);
|
|
7201
|
+
} catch (error) {
|
|
7202
|
+
rethrowWithTokenUsage(error, collectedUsage);
|
|
7203
|
+
}
|
|
7204
|
+
}
|
|
7205
|
+
async function translateCaptionsInternal(assetId, trackId, toLanguageCode, options, collectedUsage) {
|
|
7036
7206
|
const {
|
|
7037
7207
|
provider = "openai",
|
|
7038
7208
|
model,
|
|
@@ -7111,6 +7281,7 @@ async function translateCaptions(assetId, trackId, toLanguageCode, options) {
|
|
|
7111
7281
|
});
|
|
7112
7282
|
translatedVtt = result.translatedVtt;
|
|
7113
7283
|
usage = result.usage;
|
|
7284
|
+
collectedUsage.push(result.usage);
|
|
7114
7285
|
scrubbedCueCounts = result.scrubbedCueCounts;
|
|
7115
7286
|
unexpectedKeyCount = result.unexpectedKeyCount;
|
|
7116
7287
|
} catch (error) {
|