@mux/ai 0.25.0 → 0.27.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{index-C5zhbQHK.d.ts → index-Zwh0Bwq3.d.ts} +3 -1
- package/dist/index.d.ts +2 -28
- package/dist/index.js +277 -96
- package/dist/index.js.map +1 -1
- package/dist/primitives/index.js +64 -5
- package/dist/primitives/index.js.map +1 -1
- package/dist/workflows/index.d.ts +1 -1
- package/dist/workflows/index.js +276 -95
- package/dist/workflows/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -5,7 +5,7 @@ var __export = (target, all) => {
|
|
|
5
5
|
};
|
|
6
6
|
|
|
7
7
|
// package.json
|
|
8
|
-
var version = "0.
|
|
8
|
+
var version = "0.27.1";
|
|
9
9
|
|
|
10
10
|
// src/lib/language-codes.ts
|
|
11
11
|
var ISO639_1_TO_3 = {
|
|
@@ -560,6 +560,68 @@ function createSafetyReporter() {
|
|
|
560
560
|
};
|
|
561
561
|
}
|
|
562
562
|
|
|
563
|
+
// src/lib/token-usage.ts
|
|
564
|
+
var TOKEN_USAGE_FIELDS = [
|
|
565
|
+
"inputTokens",
|
|
566
|
+
"outputTokens",
|
|
567
|
+
"totalTokens",
|
|
568
|
+
"reasoningTokens",
|
|
569
|
+
"cachedInputTokens"
|
|
570
|
+
];
|
|
571
|
+
function isDefinedTokenUsageValue(value) {
|
|
572
|
+
return typeof value === "number";
|
|
573
|
+
}
|
|
574
|
+
function aggregateTokenUsage(usages) {
|
|
575
|
+
return TOKEN_USAGE_FIELDS.reduce((aggregate, field) => {
|
|
576
|
+
const values = usages.map((usage) => usage[field]).filter(isDefinedTokenUsageValue);
|
|
577
|
+
if (values.length > 0) {
|
|
578
|
+
aggregate[field] = values.reduce((total, value) => total + value, 0);
|
|
579
|
+
}
|
|
580
|
+
return aggregate;
|
|
581
|
+
}, {});
|
|
582
|
+
}
|
|
583
|
+
function getErrorTokenUsage(error) {
|
|
584
|
+
if (typeof error !== "object" || error === null) {
|
|
585
|
+
return void 0;
|
|
586
|
+
}
|
|
587
|
+
const usage = error.usage;
|
|
588
|
+
if (typeof usage !== "object" || usage === null) {
|
|
589
|
+
return void 0;
|
|
590
|
+
}
|
|
591
|
+
const source = usage;
|
|
592
|
+
const extracted = {};
|
|
593
|
+
for (const field of TOKEN_USAGE_FIELDS) {
|
|
594
|
+
const value = source[field];
|
|
595
|
+
if (typeof value === "number") {
|
|
596
|
+
extracted[field] = value;
|
|
597
|
+
}
|
|
598
|
+
}
|
|
599
|
+
if (extracted.reasoningTokens === void 0) {
|
|
600
|
+
const reasoningTokens = source.outputTokenDetails?.reasoningTokens;
|
|
601
|
+
if (typeof reasoningTokens === "number") {
|
|
602
|
+
extracted.reasoningTokens = reasoningTokens;
|
|
603
|
+
}
|
|
604
|
+
}
|
|
605
|
+
if (extracted.cachedInputTokens === void 0) {
|
|
606
|
+
const cacheReadTokens = source.inputTokenDetails?.cacheReadTokens;
|
|
607
|
+
if (typeof cacheReadTokens === "number") {
|
|
608
|
+
extracted.cachedInputTokens = cacheReadTokens;
|
|
609
|
+
}
|
|
610
|
+
}
|
|
611
|
+
return Object.keys(extracted).length > 0 ? extracted : void 0;
|
|
612
|
+
}
|
|
613
|
+
function rethrowWithTokenUsage(error, collectedUsage) {
|
|
614
|
+
const usages = [...collectedUsage];
|
|
615
|
+
const errorUsage = getErrorTokenUsage(error);
|
|
616
|
+
if (errorUsage) {
|
|
617
|
+
usages.push(errorUsage);
|
|
618
|
+
}
|
|
619
|
+
if (usages.length > 0 && typeof error === "object" && error !== null) {
|
|
620
|
+
error.usage = aggregateTokenUsage(usages);
|
|
621
|
+
}
|
|
622
|
+
throw error;
|
|
623
|
+
}
|
|
624
|
+
|
|
563
625
|
// src/lib/mux-ai-error.ts
|
|
564
626
|
var MuxAiError = class extends Error {
|
|
565
627
|
constructor(message, opts) {
|
|
@@ -572,18 +634,38 @@ var MuxAiError = class extends Error {
|
|
|
572
634
|
this.retryable = opts?.retryable ?? false;
|
|
573
635
|
}
|
|
574
636
|
};
|
|
637
|
+
function extractErrorDetail(error) {
|
|
638
|
+
if (error instanceof Error && error.message) {
|
|
639
|
+
return error.message;
|
|
640
|
+
}
|
|
641
|
+
if (typeof error === "string" && error.length > 0) {
|
|
642
|
+
return error;
|
|
643
|
+
}
|
|
644
|
+
if (typeof error === "object" && error !== null) {
|
|
645
|
+
const message = error.message;
|
|
646
|
+
if (typeof message === "string" && message.length > 0) {
|
|
647
|
+
return message;
|
|
648
|
+
}
|
|
649
|
+
}
|
|
650
|
+
return "Unknown error";
|
|
651
|
+
}
|
|
575
652
|
function wrapError(error, message) {
|
|
576
653
|
if (error instanceof MuxAiError) {
|
|
577
654
|
throw error;
|
|
578
655
|
}
|
|
579
|
-
const rawDetail = error
|
|
656
|
+
const rawDetail = extractErrorDetail(error);
|
|
580
657
|
const leakReason = detectLeakReason(rawDetail);
|
|
581
658
|
const shouldSuppress = leakReason === "canary" || leakReason === "prompt_tag";
|
|
582
659
|
const detail = shouldSuppress ? "Upstream error details suppressed by safety filter" : rawDetail;
|
|
583
660
|
if (shouldSuppress) {
|
|
584
661
|
console.warn(`[@mux/ai] Suppressed suspected prompt leak in wrapped error (context: ${message}, reason: ${leakReason}).`);
|
|
585
662
|
}
|
|
586
|
-
|
|
663
|
+
const wrapped = new Error(`${message}: ${detail}`);
|
|
664
|
+
const usage = getErrorTokenUsage(error);
|
|
665
|
+
if (usage) {
|
|
666
|
+
wrapped.usage = usage;
|
|
667
|
+
}
|
|
668
|
+
throw wrapped;
|
|
587
669
|
}
|
|
588
670
|
|
|
589
671
|
// src/lib/workflow-crypto.ts
|
|
@@ -1329,7 +1411,7 @@ var DEFAULT_EMBEDDING_MODELS = {
|
|
|
1329
1411
|
var LANGUAGE_MODELS = {
|
|
1330
1412
|
openai: ["gpt-5.1", "gpt-5-mini"],
|
|
1331
1413
|
anthropic: ["claude-sonnet-4-5"],
|
|
1332
|
-
google: ["gemini-3-flash-preview", "gemini-3.1-flash-lite", "gemini-
|
|
1414
|
+
google: ["gemini-3-flash-preview", "gemini-3.1-flash-lite", "gemini-2.5-flash"]
|
|
1333
1415
|
};
|
|
1334
1416
|
var LANGUAGE_MODEL_DEPRECATIONS = [
|
|
1335
1417
|
{
|
|
@@ -1345,10 +1427,10 @@ var LANGUAGE_MODEL_DEPRECATIONS = [
|
|
|
1345
1427
|
provider: "google",
|
|
1346
1428
|
modelId: "gemini-3.1-flash-lite-preview",
|
|
1347
1429
|
replacementModelId: "gemini-3.1-flash-lite",
|
|
1348
|
-
phase: "
|
|
1430
|
+
phase: "blocked",
|
|
1349
1431
|
deprecatedOn: "2026-05-12",
|
|
1350
1432
|
sunsetOn: "2026-05-25",
|
|
1351
|
-
reason: "
|
|
1433
|
+
reason: "Google shut down Gemini 3.1 Flash-Lite Preview on 2026-05-25."
|
|
1352
1434
|
}
|
|
1353
1435
|
];
|
|
1354
1436
|
var warnedDeprecatedLanguageModels = /* @__PURE__ */ new Set();
|
|
@@ -3286,6 +3368,14 @@ async function analyzeQuestions({
|
|
|
3286
3368
|
}
|
|
3287
3369
|
async function askQuestions(assetId, questions, options) {
|
|
3288
3370
|
"use workflow";
|
|
3371
|
+
const collectedUsage = [];
|
|
3372
|
+
try {
|
|
3373
|
+
return await askQuestionsInternal(assetId, questions, options, collectedUsage);
|
|
3374
|
+
} catch (error) {
|
|
3375
|
+
rethrowWithTokenUsage(error, collectedUsage);
|
|
3376
|
+
}
|
|
3377
|
+
}
|
|
3378
|
+
async function askQuestionsInternal(assetId, questions, options, collectedUsage) {
|
|
3289
3379
|
if (!questions || questions.length === 0) {
|
|
3290
3380
|
throw new MuxAiError("At least one question must be provided.", { type: "validation_error" });
|
|
3291
3381
|
}
|
|
@@ -3434,6 +3524,7 @@ async function askQuestions(assetId, questions, options) {
|
|
|
3434
3524
|
const contentType = isAudioOnly ? "audio" : "video";
|
|
3435
3525
|
wrapError(error, `Failed to analyze ${contentType} questions with ${provider}`);
|
|
3436
3526
|
}
|
|
3527
|
+
collectedUsage.push(analysisResponse.usage);
|
|
3437
3528
|
if (!analysisResponse.result?.answers) {
|
|
3438
3529
|
throw new MuxAiError(`Failed to generate answers for asset ${assetId}.`);
|
|
3439
3530
|
}
|
|
@@ -3667,6 +3758,14 @@ async function analyzeStoryboard({
|
|
|
3667
3758
|
}
|
|
3668
3759
|
async function hasBurnedInCaptions(assetId, options = {}) {
|
|
3669
3760
|
"use workflow";
|
|
3761
|
+
const collectedUsage = [];
|
|
3762
|
+
try {
|
|
3763
|
+
return await hasBurnedInCaptionsInternal(assetId, options, collectedUsage);
|
|
3764
|
+
} catch (error) {
|
|
3765
|
+
rethrowWithTokenUsage(error, collectedUsage);
|
|
3766
|
+
}
|
|
3767
|
+
}
|
|
3768
|
+
async function hasBurnedInCaptionsInternal(assetId, options, collectedUsage) {
|
|
3670
3769
|
const {
|
|
3671
3770
|
provider = DEFAULT_PROVIDER,
|
|
3672
3771
|
model,
|
|
@@ -3706,6 +3805,7 @@ async function hasBurnedInCaptions(assetId, options = {}) {
|
|
|
3706
3805
|
credentials
|
|
3707
3806
|
});
|
|
3708
3807
|
}
|
|
3808
|
+
collectedUsage.push(analysisResponse.usage);
|
|
3709
3809
|
if (!analysisResponse.result) {
|
|
3710
3810
|
throw new Error("No analysis result received from AI provider");
|
|
3711
3811
|
}
|
|
@@ -3907,6 +4007,14 @@ function buildUserPrompt3({
|
|
|
3907
4007
|
}
|
|
3908
4008
|
async function generateChapters(assetId, options = {}) {
|
|
3909
4009
|
"use workflow";
|
|
4010
|
+
const collectedUsage = [];
|
|
4011
|
+
try {
|
|
4012
|
+
return await generateChaptersInternal(assetId, options, collectedUsage);
|
|
4013
|
+
} catch (error) {
|
|
4014
|
+
rethrowWithTokenUsage(error, collectedUsage);
|
|
4015
|
+
}
|
|
4016
|
+
}
|
|
4017
|
+
async function generateChaptersInternal(assetId, options, collectedUsage) {
|
|
3910
4018
|
const {
|
|
3911
4019
|
languageCode,
|
|
3912
4020
|
provider = "openai",
|
|
@@ -3974,6 +4082,9 @@ async function generateChapters(assetId, options = {}) {
|
|
|
3974
4082
|
} catch (error) {
|
|
3975
4083
|
wrapError(error, `Failed to generate chapters with ${provider}`);
|
|
3976
4084
|
}
|
|
4085
|
+
if (chaptersData) {
|
|
4086
|
+
collectedUsage.push(chaptersData.usage);
|
|
4087
|
+
}
|
|
3977
4088
|
if (!chaptersData || !chaptersData.chapters) {
|
|
3978
4089
|
throw new MuxAiError(`Failed to generate chapters for asset ${assetId}.`);
|
|
3979
4090
|
}
|
|
@@ -4348,6 +4459,14 @@ async function deleteTrackOnMux(assetId, trackId, credentials) {
|
|
|
4348
4459
|
}
|
|
4349
4460
|
async function editCaptions(assetId, trackId, options) {
|
|
4350
4461
|
"use workflow";
|
|
4462
|
+
const collectedUsage = [];
|
|
4463
|
+
try {
|
|
4464
|
+
return await editCaptionsInternal(assetId, trackId, options, collectedUsage);
|
|
4465
|
+
} catch (error) {
|
|
4466
|
+
rethrowWithTokenUsage(error, collectedUsage);
|
|
4467
|
+
}
|
|
4468
|
+
}
|
|
4469
|
+
async function editCaptionsInternal(assetId, trackId, options, collectedUsage) {
|
|
4351
4470
|
const {
|
|
4352
4471
|
provider,
|
|
4353
4472
|
model,
|
|
@@ -4436,6 +4555,7 @@ async function editCaptions(assetId, trackId, options) {
|
|
|
4436
4555
|
});
|
|
4437
4556
|
detectedProfanity = result.profanity;
|
|
4438
4557
|
usage = result.usage;
|
|
4558
|
+
collectedUsage.push(result.usage);
|
|
4439
4559
|
if (result.unexpectedKeys.length > 0) {
|
|
4440
4560
|
console.warn(
|
|
4441
4561
|
`[@mux/ai] Model emitted unexpected keys in profanity detection (stripped): ${result.unexpectedKeys.join(", ")}.`
|
|
@@ -4556,16 +4676,19 @@ async function generateSingleChunkEmbedding({
|
|
|
4556
4676
|
})
|
|
4557
4677
|
);
|
|
4558
4678
|
return {
|
|
4559
|
-
|
|
4560
|
-
|
|
4561
|
-
|
|
4562
|
-
|
|
4563
|
-
|
|
4564
|
-
|
|
4565
|
-
|
|
4679
|
+
chunk: {
|
|
4680
|
+
chunkId: chunk.id,
|
|
4681
|
+
embedding: response.embedding,
|
|
4682
|
+
metadata: {
|
|
4683
|
+
startTime: chunk.startTime,
|
|
4684
|
+
endTime: chunk.endTime,
|
|
4685
|
+
tokenCount: chunk.tokenCount
|
|
4686
|
+
}
|
|
4687
|
+
},
|
|
4688
|
+
usage: typeof response.usage?.tokens === "number" ? { inputTokens: response.usage.tokens, totalTokens: response.usage.tokens } : void 0
|
|
4566
4689
|
};
|
|
4567
4690
|
}
|
|
4568
|
-
async function generateEmbeddingsInternal(assetId, options
|
|
4691
|
+
async function generateEmbeddingsInternal(assetId, options, collectedUsage) {
|
|
4569
4692
|
const {
|
|
4570
4693
|
provider = "openai",
|
|
4571
4694
|
model,
|
|
@@ -4604,7 +4727,7 @@ async function generateEmbeddingsInternal(assetId, options = {}) {
|
|
|
4604
4727
|
try {
|
|
4605
4728
|
for (let i = 0; i < chunks.length; i += batchSize) {
|
|
4606
4729
|
const batch = chunks.slice(i, i + batchSize);
|
|
4607
|
-
const
|
|
4730
|
+
const batchOutcomes = await Promise.allSettled(
|
|
4608
4731
|
batch.map(
|
|
4609
4732
|
(chunk) => generateSingleChunkEmbedding({
|
|
4610
4733
|
chunk,
|
|
@@ -4614,7 +4737,24 @@ async function generateEmbeddingsInternal(assetId, options = {}) {
|
|
|
4614
4737
|
})
|
|
4615
4738
|
)
|
|
4616
4739
|
);
|
|
4617
|
-
|
|
4740
|
+
const failures = [];
|
|
4741
|
+
for (const outcome of batchOutcomes) {
|
|
4742
|
+
if (outcome.status === "fulfilled") {
|
|
4743
|
+
if (outcome.value.usage) {
|
|
4744
|
+
collectedUsage.push(outcome.value.usage);
|
|
4745
|
+
}
|
|
4746
|
+
chunkEmbeddings.push(outcome.value.chunk);
|
|
4747
|
+
} else {
|
|
4748
|
+
const failureUsage = getErrorTokenUsage(outcome.reason);
|
|
4749
|
+
if (failureUsage) {
|
|
4750
|
+
collectedUsage.push(failureUsage);
|
|
4751
|
+
}
|
|
4752
|
+
failures.push(outcome.reason);
|
|
4753
|
+
}
|
|
4754
|
+
}
|
|
4755
|
+
if (failures.length > 0) {
|
|
4756
|
+
throw failures[0];
|
|
4757
|
+
}
|
|
4618
4758
|
}
|
|
4619
4759
|
} catch (error) {
|
|
4620
4760
|
throw new Error(
|
|
@@ -4648,12 +4788,22 @@ async function generateEmbeddingsInternal(assetId, options = {}) {
|
|
|
4648
4788
|
}
|
|
4649
4789
|
async function generateEmbeddings(assetId, options = {}) {
|
|
4650
4790
|
"use workflow";
|
|
4651
|
-
|
|
4791
|
+
const collectedUsage = [];
|
|
4792
|
+
try {
|
|
4793
|
+
return await generateEmbeddingsInternal(assetId, options, collectedUsage);
|
|
4794
|
+
} catch (error) {
|
|
4795
|
+
rethrowWithTokenUsage(error, collectedUsage);
|
|
4796
|
+
}
|
|
4652
4797
|
}
|
|
4653
4798
|
async function generateVideoEmbeddings(assetId, options = {}) {
|
|
4654
4799
|
"use workflow";
|
|
4655
4800
|
console.warn("generateVideoEmbeddings is deprecated. Use generateEmbeddings instead.");
|
|
4656
|
-
|
|
4801
|
+
const collectedUsage = [];
|
|
4802
|
+
try {
|
|
4803
|
+
return await generateEmbeddingsInternal(assetId, options, collectedUsage);
|
|
4804
|
+
} catch (error) {
|
|
4805
|
+
rethrowWithTokenUsage(error, collectedUsage);
|
|
4806
|
+
}
|
|
4657
4807
|
}
|
|
4658
4808
|
|
|
4659
4809
|
// src/workflows/engagement-insights.ts
|
|
@@ -4992,6 +5142,14 @@ async function generateInsightsWithAI(provider, modelId, systemPrompt, userPromp
|
|
|
4992
5142
|
}
|
|
4993
5143
|
async function generateEngagementInsights(assetId, options = {}) {
|
|
4994
5144
|
"use workflow";
|
|
5145
|
+
const collectedUsage = [];
|
|
5146
|
+
try {
|
|
5147
|
+
return await generateEngagementInsightsInternal(assetId, options, collectedUsage);
|
|
5148
|
+
} catch (error) {
|
|
5149
|
+
rethrowWithTokenUsage(error, collectedUsage);
|
|
5150
|
+
}
|
|
5151
|
+
}
|
|
5152
|
+
async function generateEngagementInsightsInternal(assetId, options, collectedUsage) {
|
|
4995
5153
|
const {
|
|
4996
5154
|
provider = "openai",
|
|
4997
5155
|
model,
|
|
@@ -5118,6 +5276,7 @@ async function generateEngagementInsights(assetId, options = {}) {
|
|
|
5118
5276
|
credentials
|
|
5119
5277
|
);
|
|
5120
5278
|
const { result: aiInsights, usage } = aiStepResult;
|
|
5279
|
+
collectedUsage.push(usage);
|
|
5121
5280
|
if (!aiInsights.momentInsights || aiInsights.momentInsights.length === 0) {
|
|
5122
5281
|
throw new MuxAiError(`Failed to generate insights for asset ${assetId}.`);
|
|
5123
5282
|
}
|
|
@@ -5861,6 +6020,15 @@ async function getModerationScores(assetId, options = {}) {
|
|
|
5861
6020
|
import { generateText as generateText6, Output as Output6 } from "ai";
|
|
5862
6021
|
import dedent6 from "dedent";
|
|
5863
6022
|
import { z as z7 } from "zod";
|
|
6023
|
+
|
|
6024
|
+
// src/lib/markdown-normalization.ts
|
|
6025
|
+
var ESCAPED_LINE_BREAK_REGEX = /\\r\\n|\\n|\\r/g;
|
|
6026
|
+
var MULTIPLE_BLANK_LINES_REGEX = /\n{3,}/g;
|
|
6027
|
+
function normalizeMarkdownDescription(description) {
|
|
6028
|
+
return description.replace(ESCAPED_LINE_BREAK_REGEX, "\n").replace(/([^\n]) {2,}([*-]) (?=\S)/g, "$1\n$2 ").replace(/([.!?:;]) ([*-]) (?=\S)/g, "$1\n$2 ").replace(MULTIPLE_BLANK_LINES_REGEX, "\n\n");
|
|
6029
|
+
}
|
|
6030
|
+
|
|
6031
|
+
// src/workflows/summarization.ts
|
|
5864
6032
|
var DEFAULT_SUMMARY_KEYWORD_LIMIT = 10;
|
|
5865
6033
|
var DEFAULT_TITLE_LENGTH = 10;
|
|
5866
6034
|
var DEFAULT_DESCRIPTION_LENGTH = 50;
|
|
@@ -6258,6 +6426,14 @@ function normalizeKeywords(keywords, limit = DEFAULT_SUMMARY_KEYWORD_LIMIT) {
|
|
|
6258
6426
|
}
|
|
6259
6427
|
async function getSummaryAndTags(assetId, options) {
|
|
6260
6428
|
"use workflow";
|
|
6429
|
+
const collectedUsage = [];
|
|
6430
|
+
try {
|
|
6431
|
+
return await getSummaryAndTagsInternal(assetId, options, collectedUsage);
|
|
6432
|
+
} catch (error) {
|
|
6433
|
+
rethrowWithTokenUsage(error, collectedUsage);
|
|
6434
|
+
}
|
|
6435
|
+
}
|
|
6436
|
+
async function getSummaryAndTagsInternal(assetId, options, collectedUsage) {
|
|
6261
6437
|
const {
|
|
6262
6438
|
provider = "openai",
|
|
6263
6439
|
model,
|
|
@@ -6367,6 +6543,7 @@ async function getSummaryAndTags(assetId, options) {
|
|
|
6367
6543
|
const contentType = isAudioOnly ? "audio" : "video";
|
|
6368
6544
|
wrapError(error, `Failed to analyze ${contentType} content with ${provider}`);
|
|
6369
6545
|
}
|
|
6546
|
+
collectedUsage.push(analysisResponse.usage);
|
|
6370
6547
|
if (!analysisResponse.result) {
|
|
6371
6548
|
const contentType = isAudioOnly ? "audio" : "video";
|
|
6372
6549
|
throw new MuxAiError(`Failed to analyze ${contentType} content for asset ${assetId}.`);
|
|
@@ -6387,7 +6564,8 @@ async function getSummaryAndTags(assetId, options) {
|
|
|
6387
6564
|
}
|
|
6388
6565
|
}
|
|
6389
6566
|
const scrubbedTitle = safety.scrub(analysisResponse.result.title, "title");
|
|
6390
|
-
const
|
|
6567
|
+
const normalizedDescription = normalizeMarkdownDescription(analysisResponse.result.description);
|
|
6568
|
+
const scrubbedDescription = safety.scrub(normalizedDescription, "description");
|
|
6391
6569
|
const scrubbedKeywords = (analysisResponse.result.keywords ?? []).map((kw, i) => safety.scrubDetailed(kw, `keywords[${i}]`)).filter((result) => !result.leaked).map((result) => result.text);
|
|
6392
6570
|
return {
|
|
6393
6571
|
assetId,
|
|
@@ -6408,12 +6586,9 @@ async function getSummaryAndTags(assetId, options) {
|
|
|
6408
6586
|
}
|
|
6409
6587
|
|
|
6410
6588
|
// src/workflows/translate-audio.ts
|
|
6589
|
+
import { sleep as sleep2 } from "workflow";
|
|
6411
6590
|
var STATIC_RENDITION_POLL_INTERVAL_MS = 5e3;
|
|
6412
6591
|
var STATIC_RENDITION_MAX_ATTEMPTS = 36;
|
|
6413
|
-
async function sleep2(ms) {
|
|
6414
|
-
"use step";
|
|
6415
|
-
await new Promise((resolve) => setTimeout(resolve, ms));
|
|
6416
|
-
}
|
|
6417
6592
|
function getReadyAudioStaticRendition(asset) {
|
|
6418
6593
|
const files = asset.static_renditions?.files;
|
|
6419
6594
|
if (!files || files.length === 0) {
|
|
@@ -6454,14 +6629,17 @@ async function requestStaticRenditionCreation(assetId, credentials) {
|
|
|
6454
6629
|
wrapError(error, "Failed to request static rendition from Mux");
|
|
6455
6630
|
}
|
|
6456
6631
|
}
|
|
6632
|
+
async function retrieveAsset(assetId, credentials) {
|
|
6633
|
+
"use step";
|
|
6634
|
+
const muxClient = await resolveMuxClient(credentials);
|
|
6635
|
+
const mux = await muxClient.createClient();
|
|
6636
|
+
return mux.video.assets.retrieve(assetId);
|
|
6637
|
+
}
|
|
6457
6638
|
async function waitForAudioStaticRendition({
|
|
6458
6639
|
assetId,
|
|
6459
6640
|
initialAsset,
|
|
6460
6641
|
credentials
|
|
6461
6642
|
}) {
|
|
6462
|
-
"use step";
|
|
6463
|
-
const muxClient = await resolveMuxClient(credentials);
|
|
6464
|
-
const mux = await muxClient.createClient();
|
|
6465
6643
|
let currentAsset = initialAsset;
|
|
6466
6644
|
if (hasReadyAudioStaticRendition(currentAsset)) {
|
|
6467
6645
|
return currentAsset;
|
|
@@ -6476,7 +6654,7 @@ async function waitForAudioStaticRendition({
|
|
|
6476
6654
|
}
|
|
6477
6655
|
for (let attempt = 1; attempt <= STATIC_RENDITION_MAX_ATTEMPTS; attempt++) {
|
|
6478
6656
|
await sleep2(STATIC_RENDITION_POLL_INTERVAL_MS);
|
|
6479
|
-
currentAsset = await
|
|
6657
|
+
currentAsset = await retrieveAsset(assetId, credentials);
|
|
6480
6658
|
if (hasReadyAudioStaticRendition(currentAsset)) {
|
|
6481
6659
|
return currentAsset;
|
|
6482
6660
|
}
|
|
@@ -6495,16 +6673,8 @@ async function waitForAudioStaticRendition({
|
|
|
6495
6673
|
{ type: "timeout_error", retryable: true }
|
|
6496
6674
|
);
|
|
6497
6675
|
}
|
|
6498
|
-
async function fetchAudioFromMux(audioUrl) {
|
|
6499
|
-
"use step";
|
|
6500
|
-
const audioResponse = await fetch(audioUrl);
|
|
6501
|
-
if (!audioResponse.ok) {
|
|
6502
|
-
throw new Error(`Failed to fetch audio file: ${audioResponse.statusText}`);
|
|
6503
|
-
}
|
|
6504
|
-
return audioResponse.arrayBuffer();
|
|
6505
|
-
}
|
|
6506
6676
|
async function createElevenLabsDubbingJob({
|
|
6507
|
-
|
|
6677
|
+
sourceUrl,
|
|
6508
6678
|
assetId,
|
|
6509
6679
|
elevenLabsLangCode,
|
|
6510
6680
|
elevenLabsSourceLangCode,
|
|
@@ -6513,9 +6683,8 @@ async function createElevenLabsDubbingJob({
|
|
|
6513
6683
|
}) {
|
|
6514
6684
|
"use step";
|
|
6515
6685
|
const elevenLabsApiKey = await getApiKeyFromEnv("elevenlabs", credentials);
|
|
6516
|
-
const audioBlob = new Blob([audioBuffer], { type: "audio/mp4" });
|
|
6517
6686
|
const formData = new FormData();
|
|
6518
|
-
formData.append("
|
|
6687
|
+
formData.append("source_url", sourceUrl);
|
|
6519
6688
|
formData.append("target_lang", elevenLabsLangCode);
|
|
6520
6689
|
if (elevenLabsSourceLangCode) {
|
|
6521
6690
|
formData.append("source_lang", elevenLabsSourceLangCode);
|
|
@@ -6680,13 +6849,6 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
|
|
|
6680
6849
|
if (policy === "signed") {
|
|
6681
6850
|
audioUrl = await signUrl(audioUrl, playbackId, "video", void 0, credentials);
|
|
6682
6851
|
}
|
|
6683
|
-
console.warn("\u{1F399}\uFE0F Fetching audio from Mux...");
|
|
6684
|
-
let audioBuffer;
|
|
6685
|
-
try {
|
|
6686
|
-
audioBuffer = await fetchAudioFromMux(audioUrl);
|
|
6687
|
-
} catch (error) {
|
|
6688
|
-
wrapError(error, "Failed to fetch audio from Mux");
|
|
6689
|
-
}
|
|
6690
6852
|
console.warn("\u{1F399}\uFE0F Creating dubbing job in ElevenLabs...");
|
|
6691
6853
|
const elevenLabsLangCode = toISO639_3(toLanguageCode);
|
|
6692
6854
|
const normalizedFromLanguageCode = fromLanguageCode?.trim();
|
|
@@ -6697,7 +6859,7 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
|
|
|
6697
6859
|
let dubbingId;
|
|
6698
6860
|
try {
|
|
6699
6861
|
dubbingId = await createElevenLabsDubbingJob({
|
|
6700
|
-
|
|
6862
|
+
sourceUrl: audioUrl,
|
|
6701
6863
|
assetId,
|
|
6702
6864
|
elevenLabsLangCode,
|
|
6703
6865
|
elevenLabsSourceLangCode,
|
|
@@ -6713,7 +6875,7 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
|
|
|
6713
6875
|
let pollAttempts = 0;
|
|
6714
6876
|
const maxPollAttempts = 180;
|
|
6715
6877
|
let targetLanguages = [];
|
|
6716
|
-
while (dubbingStatus
|
|
6878
|
+
while (dubbingStatus !== "dubbed" && pollAttempts < maxPollAttempts) {
|
|
6717
6879
|
await sleep2(1e4);
|
|
6718
6880
|
pollAttempts++;
|
|
6719
6881
|
try {
|
|
@@ -6723,12 +6885,12 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
|
|
|
6723
6885
|
});
|
|
6724
6886
|
dubbingStatus = statusResult.status;
|
|
6725
6887
|
targetLanguages = statusResult.targetLanguages;
|
|
6726
|
-
if (dubbingStatus === "failed") {
|
|
6727
|
-
throw new Error("ElevenLabs dubbing job failed");
|
|
6728
|
-
}
|
|
6729
6888
|
} catch (error) {
|
|
6730
6889
|
wrapError(error, "Failed to check dubbing status");
|
|
6731
6890
|
}
|
|
6891
|
+
if (dubbingStatus === "failed") {
|
|
6892
|
+
throw new MuxAiError("ElevenLabs reported that the dubbing job failed.", { type: "processing_error" });
|
|
6893
|
+
}
|
|
6732
6894
|
}
|
|
6733
6895
|
if (dubbingStatus !== "dubbed") {
|
|
6734
6896
|
throw new MuxAiError("Audio translation timed out or failed. Please try again.", { type: "timeout_error", retryable: true });
|
|
@@ -6875,13 +7037,6 @@ var DEFAULT_TRANSLATION_CHUNKING = {
|
|
|
6875
7037
|
maxCuesPerChunk: 80,
|
|
6876
7038
|
maxCueTextTokensPerChunk: 2e3
|
|
6877
7039
|
};
|
|
6878
|
-
var TOKEN_USAGE_FIELDS = [
|
|
6879
|
-
"inputTokens",
|
|
6880
|
-
"outputTokens",
|
|
6881
|
-
"totalTokens",
|
|
6882
|
-
"reasoningTokens",
|
|
6883
|
-
"cachedInputTokens"
|
|
6884
|
-
];
|
|
6885
7040
|
var TranslationChunkValidationError = class extends Error {
|
|
6886
7041
|
constructor(message) {
|
|
6887
7042
|
super(message);
|
|
@@ -6912,9 +7067,6 @@ function shouldSplitChunkTranslationError(error) {
|
|
|
6912
7067
|
}
|
|
6913
7068
|
return NoObjectGeneratedError.isInstance(error) || TypeValidationError.isInstance(error) || isTranslationChunkValidationError(error);
|
|
6914
7069
|
}
|
|
6915
|
-
function isDefinedTokenUsageValue(value) {
|
|
6916
|
-
return typeof value === "number";
|
|
6917
|
-
}
|
|
6918
7070
|
function resolveTranslationChunkingOptions(options) {
|
|
6919
7071
|
const targetChunkDurationSeconds = Math.max(
|
|
6920
7072
|
1,
|
|
@@ -6941,14 +7093,27 @@ function resolveTranslationChunkingOptions(options) {
|
|
|
6941
7093
|
)
|
|
6942
7094
|
};
|
|
6943
7095
|
}
|
|
6944
|
-
function
|
|
6945
|
-
|
|
6946
|
-
|
|
6947
|
-
|
|
6948
|
-
|
|
7096
|
+
function collectSettledTranslationsOrRethrow(outcomes, priorUsage) {
|
|
7097
|
+
const results = [];
|
|
7098
|
+
const failures = [];
|
|
7099
|
+
for (const outcome of outcomes) {
|
|
7100
|
+
if (outcome.status === "fulfilled") {
|
|
7101
|
+
results.push(outcome.value);
|
|
7102
|
+
} else {
|
|
7103
|
+
failures.push(outcome.reason);
|
|
6949
7104
|
}
|
|
6950
|
-
|
|
6951
|
-
|
|
7105
|
+
}
|
|
7106
|
+
if (failures.length > 0) {
|
|
7107
|
+
const partialUsage = [...priorUsage, ...results.map((result) => result.usage)];
|
|
7108
|
+
for (const failure of failures.slice(1)) {
|
|
7109
|
+
const usage = getErrorTokenUsage(failure);
|
|
7110
|
+
if (usage) {
|
|
7111
|
+
partialUsage.push(usage);
|
|
7112
|
+
}
|
|
7113
|
+
}
|
|
7114
|
+
rethrowWithTokenUsage(failures[0], partialUsage);
|
|
7115
|
+
}
|
|
7116
|
+
return results;
|
|
6952
7117
|
}
|
|
6953
7118
|
function createTranslationChunkRequest(id, cues, cueBlocks) {
|
|
6954
7119
|
return {
|
|
@@ -7224,24 +7389,28 @@ async function translateChunkWithFallback({
|
|
|
7224
7389
|
wrapError(error, `Chunk ${chunk.id} failed for ${Math.round(chunk.startTime)}s-${Math.round(chunk.endTime)}s`);
|
|
7225
7390
|
}
|
|
7226
7391
|
const [leftChunk, rightChunk] = splitTranslationChunkAtMidpoint(chunk);
|
|
7227
|
-
const
|
|
7228
|
-
|
|
7229
|
-
|
|
7230
|
-
|
|
7231
|
-
|
|
7232
|
-
|
|
7233
|
-
|
|
7234
|
-
|
|
7235
|
-
|
|
7236
|
-
|
|
7237
|
-
|
|
7238
|
-
|
|
7239
|
-
|
|
7240
|
-
|
|
7241
|
-
|
|
7242
|
-
|
|
7243
|
-
|
|
7244
|
-
|
|
7392
|
+
const failedAttemptUsage = getErrorTokenUsage(error);
|
|
7393
|
+
const [leftResult, rightResult] = collectSettledTranslationsOrRethrow(
|
|
7394
|
+
await Promise.allSettled([
|
|
7395
|
+
translateChunkWithFallback({
|
|
7396
|
+
chunk: leftChunk,
|
|
7397
|
+
fromLanguageCode,
|
|
7398
|
+
toLanguageCode,
|
|
7399
|
+
provider,
|
|
7400
|
+
modelId,
|
|
7401
|
+
credentials
|
|
7402
|
+
}),
|
|
7403
|
+
translateChunkWithFallback({
|
|
7404
|
+
chunk: rightChunk,
|
|
7405
|
+
fromLanguageCode,
|
|
7406
|
+
toLanguageCode,
|
|
7407
|
+
provider,
|
|
7408
|
+
modelId,
|
|
7409
|
+
credentials
|
|
7410
|
+
})
|
|
7411
|
+
]),
|
|
7412
|
+
failedAttemptUsage ? [failedAttemptUsage] : []
|
|
7413
|
+
);
|
|
7245
7414
|
return {
|
|
7246
7415
|
translatedVtt: concatenateVttSegments([leftResult.translatedVtt, rightResult.translatedVtt]),
|
|
7247
7416
|
usage: aggregateTokenUsage([leftResult.usage, rightResult.usage]),
|
|
@@ -7282,17 +7451,20 @@ async function translateCaptionTrack({
|
|
|
7282
7451
|
let totalUnexpectedKeyCount = 0;
|
|
7283
7452
|
for (let index = 0; index < chunkPlan.chunks.length; index += resolvedChunking.maxConcurrentTranslations) {
|
|
7284
7453
|
const batch = chunkPlan.chunks.slice(index, index + resolvedChunking.maxConcurrentTranslations);
|
|
7285
|
-
const batchResults =
|
|
7286
|
-
|
|
7287
|
-
(
|
|
7288
|
-
chunk
|
|
7289
|
-
|
|
7290
|
-
|
|
7291
|
-
|
|
7292
|
-
|
|
7293
|
-
|
|
7294
|
-
|
|
7295
|
-
|
|
7454
|
+
const batchResults = collectSettledTranslationsOrRethrow(
|
|
7455
|
+
await Promise.allSettled(
|
|
7456
|
+
batch.map(
|
|
7457
|
+
(chunk) => translateChunkWithFallback({
|
|
7458
|
+
chunk,
|
|
7459
|
+
fromLanguageCode,
|
|
7460
|
+
toLanguageCode,
|
|
7461
|
+
provider,
|
|
7462
|
+
modelId,
|
|
7463
|
+
credentials
|
|
7464
|
+
})
|
|
7465
|
+
)
|
|
7466
|
+
),
|
|
7467
|
+
usageByChunk
|
|
7296
7468
|
);
|
|
7297
7469
|
translatedSegments.push(...batchResults.map((result) => result.translatedVtt));
|
|
7298
7470
|
usageByChunk.push(...batchResults.map((result) => result.usage));
|
|
@@ -7345,6 +7517,14 @@ async function uploadVttToS3({
|
|
|
7345
7517
|
}
|
|
7346
7518
|
async function translateCaptions(assetId, trackId, toLanguageCode, options) {
|
|
7347
7519
|
"use workflow";
|
|
7520
|
+
const collectedUsage = [];
|
|
7521
|
+
try {
|
|
7522
|
+
return await translateCaptionsInternal(assetId, trackId, toLanguageCode, options, collectedUsage);
|
|
7523
|
+
} catch (error) {
|
|
7524
|
+
rethrowWithTokenUsage(error, collectedUsage);
|
|
7525
|
+
}
|
|
7526
|
+
}
|
|
7527
|
+
async function translateCaptionsInternal(assetId, trackId, toLanguageCode, options, collectedUsage) {
|
|
7348
7528
|
const {
|
|
7349
7529
|
provider = "openai",
|
|
7350
7530
|
model,
|
|
@@ -7423,6 +7603,7 @@ async function translateCaptions(assetId, trackId, toLanguageCode, options) {
|
|
|
7423
7603
|
});
|
|
7424
7604
|
translatedVtt = result.translatedVtt;
|
|
7425
7605
|
usage = result.usage;
|
|
7606
|
+
collectedUsage.push(result.usage);
|
|
7426
7607
|
scrubbedCueCounts = result.scrubbedCueCounts;
|
|
7427
7608
|
unexpectedKeyCount = result.unexpectedKeyCount;
|
|
7428
7609
|
} catch (error) {
|