@mux/ai 0.26.0 → 0.28.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{index-C5zhbQHK.d.ts → index-DUVBpnj5.d.ts} +5 -1
- package/dist/index.d.ts +2 -28
- package/dist/index.js +271 -97
- package/dist/index.js.map +1 -1
- package/dist/primitives/index.js +64 -5
- package/dist/primitives/index.js.map +1 -1
- package/dist/workflows/index.d.ts +1 -1
- package/dist/workflows/index.js +270 -96
- package/dist/workflows/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -5,7 +5,7 @@ var __export = (target, all) => {
|
|
|
5
5
|
};
|
|
6
6
|
|
|
7
7
|
// package.json
|
|
8
|
-
var version = "0.
|
|
8
|
+
var version = "0.28.0";
|
|
9
9
|
|
|
10
10
|
// src/lib/language-codes.ts
|
|
11
11
|
var ISO639_1_TO_3 = {
|
|
@@ -560,6 +560,68 @@ function createSafetyReporter() {
|
|
|
560
560
|
};
|
|
561
561
|
}
|
|
562
562
|
|
|
563
|
+
// src/lib/token-usage.ts
|
|
564
|
+
var TOKEN_USAGE_FIELDS = [
|
|
565
|
+
"inputTokens",
|
|
566
|
+
"outputTokens",
|
|
567
|
+
"totalTokens",
|
|
568
|
+
"reasoningTokens",
|
|
569
|
+
"cachedInputTokens"
|
|
570
|
+
];
|
|
571
|
+
function isDefinedTokenUsageValue(value) {
|
|
572
|
+
return typeof value === "number";
|
|
573
|
+
}
|
|
574
|
+
function aggregateTokenUsage(usages) {
|
|
575
|
+
return TOKEN_USAGE_FIELDS.reduce((aggregate, field) => {
|
|
576
|
+
const values = usages.map((usage) => usage[field]).filter(isDefinedTokenUsageValue);
|
|
577
|
+
if (values.length > 0) {
|
|
578
|
+
aggregate[field] = values.reduce((total, value) => total + value, 0);
|
|
579
|
+
}
|
|
580
|
+
return aggregate;
|
|
581
|
+
}, {});
|
|
582
|
+
}
|
|
583
|
+
function getErrorTokenUsage(error) {
|
|
584
|
+
if (typeof error !== "object" || error === null) {
|
|
585
|
+
return void 0;
|
|
586
|
+
}
|
|
587
|
+
const usage = error.usage;
|
|
588
|
+
if (typeof usage !== "object" || usage === null) {
|
|
589
|
+
return void 0;
|
|
590
|
+
}
|
|
591
|
+
const source = usage;
|
|
592
|
+
const extracted = {};
|
|
593
|
+
for (const field of TOKEN_USAGE_FIELDS) {
|
|
594
|
+
const value = source[field];
|
|
595
|
+
if (typeof value === "number") {
|
|
596
|
+
extracted[field] = value;
|
|
597
|
+
}
|
|
598
|
+
}
|
|
599
|
+
if (extracted.reasoningTokens === void 0) {
|
|
600
|
+
const reasoningTokens = source.outputTokenDetails?.reasoningTokens;
|
|
601
|
+
if (typeof reasoningTokens === "number") {
|
|
602
|
+
extracted.reasoningTokens = reasoningTokens;
|
|
603
|
+
}
|
|
604
|
+
}
|
|
605
|
+
if (extracted.cachedInputTokens === void 0) {
|
|
606
|
+
const cacheReadTokens = source.inputTokenDetails?.cacheReadTokens;
|
|
607
|
+
if (typeof cacheReadTokens === "number") {
|
|
608
|
+
extracted.cachedInputTokens = cacheReadTokens;
|
|
609
|
+
}
|
|
610
|
+
}
|
|
611
|
+
return Object.keys(extracted).length > 0 ? extracted : void 0;
|
|
612
|
+
}
|
|
613
|
+
function rethrowWithTokenUsage(error, collectedUsage) {
|
|
614
|
+
const usages = [...collectedUsage];
|
|
615
|
+
const errorUsage = getErrorTokenUsage(error);
|
|
616
|
+
if (errorUsage) {
|
|
617
|
+
usages.push(errorUsage);
|
|
618
|
+
}
|
|
619
|
+
if (usages.length > 0 && typeof error === "object" && error !== null) {
|
|
620
|
+
error.usage = aggregateTokenUsage(usages);
|
|
621
|
+
}
|
|
622
|
+
throw error;
|
|
623
|
+
}
|
|
624
|
+
|
|
563
625
|
// src/lib/mux-ai-error.ts
|
|
564
626
|
var MuxAiError = class extends Error {
|
|
565
627
|
constructor(message, opts) {
|
|
@@ -572,18 +634,38 @@ var MuxAiError = class extends Error {
|
|
|
572
634
|
this.retryable = opts?.retryable ?? false;
|
|
573
635
|
}
|
|
574
636
|
};
|
|
637
|
+
function extractErrorDetail(error) {
|
|
638
|
+
if (error instanceof Error && error.message) {
|
|
639
|
+
return error.message;
|
|
640
|
+
}
|
|
641
|
+
if (typeof error === "string" && error.length > 0) {
|
|
642
|
+
return error;
|
|
643
|
+
}
|
|
644
|
+
if (typeof error === "object" && error !== null) {
|
|
645
|
+
const message = error.message;
|
|
646
|
+
if (typeof message === "string" && message.length > 0) {
|
|
647
|
+
return message;
|
|
648
|
+
}
|
|
649
|
+
}
|
|
650
|
+
return "Unknown error";
|
|
651
|
+
}
|
|
575
652
|
function wrapError(error, message) {
|
|
576
653
|
if (error instanceof MuxAiError) {
|
|
577
654
|
throw error;
|
|
578
655
|
}
|
|
579
|
-
const rawDetail = error
|
|
656
|
+
const rawDetail = extractErrorDetail(error);
|
|
580
657
|
const leakReason = detectLeakReason(rawDetail);
|
|
581
658
|
const shouldSuppress = leakReason === "canary" || leakReason === "prompt_tag";
|
|
582
659
|
const detail = shouldSuppress ? "Upstream error details suppressed by safety filter" : rawDetail;
|
|
583
660
|
if (shouldSuppress) {
|
|
584
661
|
console.warn(`[@mux/ai] Suppressed suspected prompt leak in wrapped error (context: ${message}, reason: ${leakReason}).`);
|
|
585
662
|
}
|
|
586
|
-
|
|
663
|
+
const wrapped = new Error(`${message}: ${detail}`);
|
|
664
|
+
const usage = getErrorTokenUsage(error);
|
|
665
|
+
if (usage) {
|
|
666
|
+
wrapped.usage = usage;
|
|
667
|
+
}
|
|
668
|
+
throw wrapped;
|
|
587
669
|
}
|
|
588
670
|
|
|
589
671
|
// src/lib/workflow-crypto.ts
|
|
@@ -1329,7 +1411,7 @@ var DEFAULT_EMBEDDING_MODELS = {
|
|
|
1329
1411
|
var LANGUAGE_MODELS = {
|
|
1330
1412
|
openai: ["gpt-5.1", "gpt-5-mini"],
|
|
1331
1413
|
anthropic: ["claude-sonnet-4-5"],
|
|
1332
|
-
google: ["gemini-3-flash-preview", "gemini-3.1-flash-lite", "gemini-
|
|
1414
|
+
google: ["gemini-3-flash-preview", "gemini-3.1-flash-lite", "gemini-2.5-flash"]
|
|
1333
1415
|
};
|
|
1334
1416
|
var LANGUAGE_MODEL_DEPRECATIONS = [
|
|
1335
1417
|
{
|
|
@@ -1345,10 +1427,10 @@ var LANGUAGE_MODEL_DEPRECATIONS = [
|
|
|
1345
1427
|
provider: "google",
|
|
1346
1428
|
modelId: "gemini-3.1-flash-lite-preview",
|
|
1347
1429
|
replacementModelId: "gemini-3.1-flash-lite",
|
|
1348
|
-
phase: "
|
|
1430
|
+
phase: "blocked",
|
|
1349
1431
|
deprecatedOn: "2026-05-12",
|
|
1350
1432
|
sunsetOn: "2026-05-25",
|
|
1351
|
-
reason: "
|
|
1433
|
+
reason: "Google shut down Gemini 3.1 Flash-Lite Preview on 2026-05-25."
|
|
1352
1434
|
}
|
|
1353
1435
|
];
|
|
1354
1436
|
var warnedDeprecatedLanguageModels = /* @__PURE__ */ new Set();
|
|
@@ -3286,6 +3368,14 @@ async function analyzeQuestions({
|
|
|
3286
3368
|
}
|
|
3287
3369
|
async function askQuestions(assetId, questions, options) {
|
|
3288
3370
|
"use workflow";
|
|
3371
|
+
const collectedUsage = [];
|
|
3372
|
+
try {
|
|
3373
|
+
return await askQuestionsInternal(assetId, questions, options, collectedUsage);
|
|
3374
|
+
} catch (error) {
|
|
3375
|
+
rethrowWithTokenUsage(error, collectedUsage);
|
|
3376
|
+
}
|
|
3377
|
+
}
|
|
3378
|
+
async function askQuestionsInternal(assetId, questions, options, collectedUsage) {
|
|
3289
3379
|
if (!questions || questions.length === 0) {
|
|
3290
3380
|
throw new MuxAiError("At least one question must be provided.", { type: "validation_error" });
|
|
3291
3381
|
}
|
|
@@ -3434,6 +3524,7 @@ async function askQuestions(assetId, questions, options) {
|
|
|
3434
3524
|
const contentType = isAudioOnly ? "audio" : "video";
|
|
3435
3525
|
wrapError(error, `Failed to analyze ${contentType} questions with ${provider}`);
|
|
3436
3526
|
}
|
|
3527
|
+
collectedUsage.push(analysisResponse.usage);
|
|
3437
3528
|
if (!analysisResponse.result?.answers) {
|
|
3438
3529
|
throw new MuxAiError(`Failed to generate answers for asset ${assetId}.`);
|
|
3439
3530
|
}
|
|
@@ -3667,6 +3758,14 @@ async function analyzeStoryboard({
|
|
|
3667
3758
|
}
|
|
3668
3759
|
async function hasBurnedInCaptions(assetId, options = {}) {
|
|
3669
3760
|
"use workflow";
|
|
3761
|
+
const collectedUsage = [];
|
|
3762
|
+
try {
|
|
3763
|
+
return await hasBurnedInCaptionsInternal(assetId, options, collectedUsage);
|
|
3764
|
+
} catch (error) {
|
|
3765
|
+
rethrowWithTokenUsage(error, collectedUsage);
|
|
3766
|
+
}
|
|
3767
|
+
}
|
|
3768
|
+
async function hasBurnedInCaptionsInternal(assetId, options, collectedUsage) {
|
|
3670
3769
|
const {
|
|
3671
3770
|
provider = DEFAULT_PROVIDER,
|
|
3672
3771
|
model,
|
|
@@ -3706,6 +3805,7 @@ async function hasBurnedInCaptions(assetId, options = {}) {
|
|
|
3706
3805
|
credentials
|
|
3707
3806
|
});
|
|
3708
3807
|
}
|
|
3808
|
+
collectedUsage.push(analysisResponse.usage);
|
|
3709
3809
|
if (!analysisResponse.result) {
|
|
3710
3810
|
throw new Error("No analysis result received from AI provider");
|
|
3711
3811
|
}
|
|
@@ -3907,6 +4007,14 @@ function buildUserPrompt3({
|
|
|
3907
4007
|
}
|
|
3908
4008
|
async function generateChapters(assetId, options = {}) {
|
|
3909
4009
|
"use workflow";
|
|
4010
|
+
const collectedUsage = [];
|
|
4011
|
+
try {
|
|
4012
|
+
return await generateChaptersInternal(assetId, options, collectedUsage);
|
|
4013
|
+
} catch (error) {
|
|
4014
|
+
rethrowWithTokenUsage(error, collectedUsage);
|
|
4015
|
+
}
|
|
4016
|
+
}
|
|
4017
|
+
async function generateChaptersInternal(assetId, options, collectedUsage) {
|
|
3910
4018
|
const {
|
|
3911
4019
|
languageCode,
|
|
3912
4020
|
provider = "openai",
|
|
@@ -3974,6 +4082,9 @@ async function generateChapters(assetId, options = {}) {
|
|
|
3974
4082
|
} catch (error) {
|
|
3975
4083
|
wrapError(error, `Failed to generate chapters with ${provider}`);
|
|
3976
4084
|
}
|
|
4085
|
+
if (chaptersData) {
|
|
4086
|
+
collectedUsage.push(chaptersData.usage);
|
|
4087
|
+
}
|
|
3977
4088
|
if (!chaptersData || !chaptersData.chapters) {
|
|
3978
4089
|
throw new MuxAiError(`Failed to generate chapters for asset ${assetId}.`);
|
|
3979
4090
|
}
|
|
@@ -4348,6 +4459,14 @@ async function deleteTrackOnMux(assetId, trackId, credentials) {
|
|
|
4348
4459
|
}
|
|
4349
4460
|
async function editCaptions(assetId, trackId, options) {
|
|
4350
4461
|
"use workflow";
|
|
4462
|
+
const collectedUsage = [];
|
|
4463
|
+
try {
|
|
4464
|
+
return await editCaptionsInternal(assetId, trackId, options, collectedUsage);
|
|
4465
|
+
} catch (error) {
|
|
4466
|
+
rethrowWithTokenUsage(error, collectedUsage);
|
|
4467
|
+
}
|
|
4468
|
+
}
|
|
4469
|
+
async function editCaptionsInternal(assetId, trackId, options, collectedUsage) {
|
|
4351
4470
|
const {
|
|
4352
4471
|
provider,
|
|
4353
4472
|
model,
|
|
@@ -4436,6 +4555,7 @@ async function editCaptions(assetId, trackId, options) {
|
|
|
4436
4555
|
});
|
|
4437
4556
|
detectedProfanity = result.profanity;
|
|
4438
4557
|
usage = result.usage;
|
|
4558
|
+
collectedUsage.push(result.usage);
|
|
4439
4559
|
if (result.unexpectedKeys.length > 0) {
|
|
4440
4560
|
console.warn(
|
|
4441
4561
|
`[@mux/ai] Model emitted unexpected keys in profanity detection (stripped): ${result.unexpectedKeys.join(", ")}.`
|
|
@@ -4556,16 +4676,19 @@ async function generateSingleChunkEmbedding({
|
|
|
4556
4676
|
})
|
|
4557
4677
|
);
|
|
4558
4678
|
return {
|
|
4559
|
-
|
|
4560
|
-
|
|
4561
|
-
|
|
4562
|
-
|
|
4563
|
-
|
|
4564
|
-
|
|
4565
|
-
|
|
4679
|
+
chunk: {
|
|
4680
|
+
chunkId: chunk.id,
|
|
4681
|
+
embedding: response.embedding,
|
|
4682
|
+
metadata: {
|
|
4683
|
+
startTime: chunk.startTime,
|
|
4684
|
+
endTime: chunk.endTime,
|
|
4685
|
+
tokenCount: chunk.tokenCount
|
|
4686
|
+
}
|
|
4687
|
+
},
|
|
4688
|
+
usage: typeof response.usage?.tokens === "number" ? { inputTokens: response.usage.tokens, totalTokens: response.usage.tokens } : void 0
|
|
4566
4689
|
};
|
|
4567
4690
|
}
|
|
4568
|
-
async function generateEmbeddingsInternal(assetId, options
|
|
4691
|
+
async function generateEmbeddingsInternal(assetId, options, collectedUsage) {
|
|
4569
4692
|
const {
|
|
4570
4693
|
provider = "openai",
|
|
4571
4694
|
model,
|
|
@@ -4604,7 +4727,7 @@ async function generateEmbeddingsInternal(assetId, options = {}) {
|
|
|
4604
4727
|
try {
|
|
4605
4728
|
for (let i = 0; i < chunks.length; i += batchSize) {
|
|
4606
4729
|
const batch = chunks.slice(i, i + batchSize);
|
|
4607
|
-
const
|
|
4730
|
+
const batchOutcomes = await Promise.allSettled(
|
|
4608
4731
|
batch.map(
|
|
4609
4732
|
(chunk) => generateSingleChunkEmbedding({
|
|
4610
4733
|
chunk,
|
|
@@ -4614,7 +4737,24 @@ async function generateEmbeddingsInternal(assetId, options = {}) {
|
|
|
4614
4737
|
})
|
|
4615
4738
|
)
|
|
4616
4739
|
);
|
|
4617
|
-
|
|
4740
|
+
const failures = [];
|
|
4741
|
+
for (const outcome of batchOutcomes) {
|
|
4742
|
+
if (outcome.status === "fulfilled") {
|
|
4743
|
+
if (outcome.value.usage) {
|
|
4744
|
+
collectedUsage.push(outcome.value.usage);
|
|
4745
|
+
}
|
|
4746
|
+
chunkEmbeddings.push(outcome.value.chunk);
|
|
4747
|
+
} else {
|
|
4748
|
+
const failureUsage = getErrorTokenUsage(outcome.reason);
|
|
4749
|
+
if (failureUsage) {
|
|
4750
|
+
collectedUsage.push(failureUsage);
|
|
4751
|
+
}
|
|
4752
|
+
failures.push(outcome.reason);
|
|
4753
|
+
}
|
|
4754
|
+
}
|
|
4755
|
+
if (failures.length > 0) {
|
|
4756
|
+
throw failures[0];
|
|
4757
|
+
}
|
|
4618
4758
|
}
|
|
4619
4759
|
} catch (error) {
|
|
4620
4760
|
throw new Error(
|
|
@@ -4648,12 +4788,22 @@ async function generateEmbeddingsInternal(assetId, options = {}) {
|
|
|
4648
4788
|
}
|
|
4649
4789
|
async function generateEmbeddings(assetId, options = {}) {
|
|
4650
4790
|
"use workflow";
|
|
4651
|
-
|
|
4791
|
+
const collectedUsage = [];
|
|
4792
|
+
try {
|
|
4793
|
+
return await generateEmbeddingsInternal(assetId, options, collectedUsage);
|
|
4794
|
+
} catch (error) {
|
|
4795
|
+
rethrowWithTokenUsage(error, collectedUsage);
|
|
4796
|
+
}
|
|
4652
4797
|
}
|
|
4653
4798
|
async function generateVideoEmbeddings(assetId, options = {}) {
|
|
4654
4799
|
"use workflow";
|
|
4655
4800
|
console.warn("generateVideoEmbeddings is deprecated. Use generateEmbeddings instead.");
|
|
4656
|
-
|
|
4801
|
+
const collectedUsage = [];
|
|
4802
|
+
try {
|
|
4803
|
+
return await generateEmbeddingsInternal(assetId, options, collectedUsage);
|
|
4804
|
+
} catch (error) {
|
|
4805
|
+
rethrowWithTokenUsage(error, collectedUsage);
|
|
4806
|
+
}
|
|
4657
4807
|
}
|
|
4658
4808
|
|
|
4659
4809
|
// src/workflows/engagement-insights.ts
|
|
@@ -4992,6 +5142,14 @@ async function generateInsightsWithAI(provider, modelId, systemPrompt, userPromp
|
|
|
4992
5142
|
}
|
|
4993
5143
|
async function generateEngagementInsights(assetId, options = {}) {
|
|
4994
5144
|
"use workflow";
|
|
5145
|
+
const collectedUsage = [];
|
|
5146
|
+
try {
|
|
5147
|
+
return await generateEngagementInsightsInternal(assetId, options, collectedUsage);
|
|
5148
|
+
} catch (error) {
|
|
5149
|
+
rethrowWithTokenUsage(error, collectedUsage);
|
|
5150
|
+
}
|
|
5151
|
+
}
|
|
5152
|
+
async function generateEngagementInsightsInternal(assetId, options, collectedUsage) {
|
|
4995
5153
|
const {
|
|
4996
5154
|
provider = "openai",
|
|
4997
5155
|
model,
|
|
@@ -5118,6 +5276,7 @@ async function generateEngagementInsights(assetId, options = {}) {
|
|
|
5118
5276
|
credentials
|
|
5119
5277
|
);
|
|
5120
5278
|
const { result: aiInsights, usage } = aiStepResult;
|
|
5279
|
+
collectedUsage.push(usage);
|
|
5121
5280
|
if (!aiInsights.momentInsights || aiInsights.momentInsights.length === 0) {
|
|
5122
5281
|
throw new MuxAiError(`Failed to generate insights for asset ${assetId}.`);
|
|
5123
5282
|
}
|
|
@@ -6267,6 +6426,14 @@ function normalizeKeywords(keywords, limit = DEFAULT_SUMMARY_KEYWORD_LIMIT) {
|
|
|
6267
6426
|
}
|
|
6268
6427
|
async function getSummaryAndTags(assetId, options) {
|
|
6269
6428
|
"use workflow";
|
|
6429
|
+
const collectedUsage = [];
|
|
6430
|
+
try {
|
|
6431
|
+
return await getSummaryAndTagsInternal(assetId, options, collectedUsage);
|
|
6432
|
+
} catch (error) {
|
|
6433
|
+
rethrowWithTokenUsage(error, collectedUsage);
|
|
6434
|
+
}
|
|
6435
|
+
}
|
|
6436
|
+
async function getSummaryAndTagsInternal(assetId, options, collectedUsage) {
|
|
6270
6437
|
const {
|
|
6271
6438
|
provider = "openai",
|
|
6272
6439
|
model,
|
|
@@ -6376,6 +6543,7 @@ async function getSummaryAndTags(assetId, options) {
|
|
|
6376
6543
|
const contentType = isAudioOnly ? "audio" : "video";
|
|
6377
6544
|
wrapError(error, `Failed to analyze ${contentType} content with ${provider}`);
|
|
6378
6545
|
}
|
|
6546
|
+
collectedUsage.push(analysisResponse.usage);
|
|
6379
6547
|
if (!analysisResponse.result) {
|
|
6380
6548
|
const contentType = isAudioOnly ? "audio" : "video";
|
|
6381
6549
|
throw new MuxAiError(`Failed to analyze ${contentType} content for asset ${assetId}.`);
|
|
@@ -6418,12 +6586,11 @@ async function getSummaryAndTags(assetId, options) {
|
|
|
6418
6586
|
}
|
|
6419
6587
|
|
|
6420
6588
|
// src/workflows/translate-audio.ts
|
|
6589
|
+
import { sleep as sleep2 } from "workflow";
|
|
6421
6590
|
var STATIC_RENDITION_POLL_INTERVAL_MS = 5e3;
|
|
6422
6591
|
var STATIC_RENDITION_MAX_ATTEMPTS = 36;
|
|
6423
|
-
|
|
6424
|
-
|
|
6425
|
-
await new Promise((resolve) => setTimeout(resolve, ms));
|
|
6426
|
-
}
|
|
6592
|
+
var DUBBING_POLL_INTERVAL_MS = 1e4;
|
|
6593
|
+
var DEFAULT_DUBBING_POLL_TIMEOUT_SECONDS = 7200;
|
|
6427
6594
|
function getReadyAudioStaticRendition(asset) {
|
|
6428
6595
|
const files = asset.static_renditions?.files;
|
|
6429
6596
|
if (!files || files.length === 0) {
|
|
@@ -6464,14 +6631,17 @@ async function requestStaticRenditionCreation(assetId, credentials) {
|
|
|
6464
6631
|
wrapError(error, "Failed to request static rendition from Mux");
|
|
6465
6632
|
}
|
|
6466
6633
|
}
|
|
6634
|
+
async function retrieveAsset(assetId, credentials) {
|
|
6635
|
+
"use step";
|
|
6636
|
+
const muxClient = await resolveMuxClient(credentials);
|
|
6637
|
+
const mux = await muxClient.createClient();
|
|
6638
|
+
return mux.video.assets.retrieve(assetId);
|
|
6639
|
+
}
|
|
6467
6640
|
async function waitForAudioStaticRendition({
|
|
6468
6641
|
assetId,
|
|
6469
6642
|
initialAsset,
|
|
6470
6643
|
credentials
|
|
6471
6644
|
}) {
|
|
6472
|
-
"use step";
|
|
6473
|
-
const muxClient = await resolveMuxClient(credentials);
|
|
6474
|
-
const mux = await muxClient.createClient();
|
|
6475
6645
|
let currentAsset = initialAsset;
|
|
6476
6646
|
if (hasReadyAudioStaticRendition(currentAsset)) {
|
|
6477
6647
|
return currentAsset;
|
|
@@ -6486,7 +6656,7 @@ async function waitForAudioStaticRendition({
|
|
|
6486
6656
|
}
|
|
6487
6657
|
for (let attempt = 1; attempt <= STATIC_RENDITION_MAX_ATTEMPTS; attempt++) {
|
|
6488
6658
|
await sleep2(STATIC_RENDITION_POLL_INTERVAL_MS);
|
|
6489
|
-
currentAsset = await
|
|
6659
|
+
currentAsset = await retrieveAsset(assetId, credentials);
|
|
6490
6660
|
if (hasReadyAudioStaticRendition(currentAsset)) {
|
|
6491
6661
|
return currentAsset;
|
|
6492
6662
|
}
|
|
@@ -6505,16 +6675,8 @@ async function waitForAudioStaticRendition({
|
|
|
6505
6675
|
{ type: "timeout_error", retryable: true }
|
|
6506
6676
|
);
|
|
6507
6677
|
}
|
|
6508
|
-
async function fetchAudioFromMux(audioUrl) {
|
|
6509
|
-
"use step";
|
|
6510
|
-
const audioResponse = await fetch(audioUrl);
|
|
6511
|
-
if (!audioResponse.ok) {
|
|
6512
|
-
throw new Error(`Failed to fetch audio file: ${audioResponse.statusText}`);
|
|
6513
|
-
}
|
|
6514
|
-
return audioResponse.arrayBuffer();
|
|
6515
|
-
}
|
|
6516
6678
|
async function createElevenLabsDubbingJob({
|
|
6517
|
-
|
|
6679
|
+
sourceUrl,
|
|
6518
6680
|
assetId,
|
|
6519
6681
|
elevenLabsLangCode,
|
|
6520
6682
|
elevenLabsSourceLangCode,
|
|
@@ -6523,9 +6685,8 @@ async function createElevenLabsDubbingJob({
|
|
|
6523
6685
|
}) {
|
|
6524
6686
|
"use step";
|
|
6525
6687
|
const elevenLabsApiKey = await getApiKeyFromEnv("elevenlabs", credentials);
|
|
6526
|
-
const audioBlob = new Blob([audioBuffer], { type: "audio/mp4" });
|
|
6527
6688
|
const formData = new FormData();
|
|
6528
|
-
formData.append("
|
|
6689
|
+
formData.append("source_url", sourceUrl);
|
|
6529
6690
|
formData.append("target_lang", elevenLabsLangCode);
|
|
6530
6691
|
if (elevenLabsSourceLangCode) {
|
|
6531
6692
|
formData.append("source_lang", elevenLabsSourceLangCode);
|
|
@@ -6690,13 +6851,6 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
|
|
|
6690
6851
|
if (policy === "signed") {
|
|
6691
6852
|
audioUrl = await signUrl(audioUrl, playbackId, "video", void 0, credentials);
|
|
6692
6853
|
}
|
|
6693
|
-
console.warn("\u{1F399}\uFE0F Fetching audio from Mux...");
|
|
6694
|
-
let audioBuffer;
|
|
6695
|
-
try {
|
|
6696
|
-
audioBuffer = await fetchAudioFromMux(audioUrl);
|
|
6697
|
-
} catch (error) {
|
|
6698
|
-
wrapError(error, "Failed to fetch audio from Mux");
|
|
6699
|
-
}
|
|
6700
6854
|
console.warn("\u{1F399}\uFE0F Creating dubbing job in ElevenLabs...");
|
|
6701
6855
|
const elevenLabsLangCode = toISO639_3(toLanguageCode);
|
|
6702
6856
|
const normalizedFromLanguageCode = fromLanguageCode?.trim();
|
|
@@ -6707,7 +6861,7 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
|
|
|
6707
6861
|
let dubbingId;
|
|
6708
6862
|
try {
|
|
6709
6863
|
dubbingId = await createElevenLabsDubbingJob({
|
|
6710
|
-
|
|
6864
|
+
sourceUrl: audioUrl,
|
|
6711
6865
|
assetId,
|
|
6712
6866
|
elevenLabsLangCode,
|
|
6713
6867
|
elevenLabsSourceLangCode,
|
|
@@ -6719,12 +6873,13 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
|
|
|
6719
6873
|
wrapError(error, "Failed to create ElevenLabs dubbing job");
|
|
6720
6874
|
}
|
|
6721
6875
|
console.warn("\u23F3 Waiting for dubbing to complete...");
|
|
6876
|
+
const dubbingPollTimeoutSeconds = options.dubbingPollTimeoutSeconds ?? DEFAULT_DUBBING_POLL_TIMEOUT_SECONDS;
|
|
6877
|
+
const maxPollAttempts = Math.max(1, Math.ceil(dubbingPollTimeoutSeconds * 1e3 / DUBBING_POLL_INTERVAL_MS));
|
|
6722
6878
|
let dubbingStatus = "dubbing";
|
|
6723
6879
|
let pollAttempts = 0;
|
|
6724
|
-
const maxPollAttempts = 180;
|
|
6725
6880
|
let targetLanguages = [];
|
|
6726
|
-
while (dubbingStatus
|
|
6727
|
-
await sleep2(
|
|
6881
|
+
while (dubbingStatus !== "dubbed" && pollAttempts < maxPollAttempts) {
|
|
6882
|
+
await sleep2(DUBBING_POLL_INTERVAL_MS);
|
|
6728
6883
|
pollAttempts++;
|
|
6729
6884
|
try {
|
|
6730
6885
|
const statusResult = await checkElevenLabsDubbingStatus({
|
|
@@ -6733,12 +6888,12 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
|
|
|
6733
6888
|
});
|
|
6734
6889
|
dubbingStatus = statusResult.status;
|
|
6735
6890
|
targetLanguages = statusResult.targetLanguages;
|
|
6736
|
-
if (dubbingStatus === "failed") {
|
|
6737
|
-
throw new Error("ElevenLabs dubbing job failed");
|
|
6738
|
-
}
|
|
6739
6891
|
} catch (error) {
|
|
6740
6892
|
wrapError(error, "Failed to check dubbing status");
|
|
6741
6893
|
}
|
|
6894
|
+
if (dubbingStatus === "failed") {
|
|
6895
|
+
throw new MuxAiError("ElevenLabs reported that the dubbing job failed.", { type: "processing_error" });
|
|
6896
|
+
}
|
|
6742
6897
|
}
|
|
6743
6898
|
if (dubbingStatus !== "dubbed") {
|
|
6744
6899
|
throw new MuxAiError("Audio translation timed out or failed. Please try again.", { type: "timeout_error", retryable: true });
|
|
@@ -6885,13 +7040,6 @@ var DEFAULT_TRANSLATION_CHUNKING = {
|
|
|
6885
7040
|
maxCuesPerChunk: 80,
|
|
6886
7041
|
maxCueTextTokensPerChunk: 2e3
|
|
6887
7042
|
};
|
|
6888
|
-
var TOKEN_USAGE_FIELDS = [
|
|
6889
|
-
"inputTokens",
|
|
6890
|
-
"outputTokens",
|
|
6891
|
-
"totalTokens",
|
|
6892
|
-
"reasoningTokens",
|
|
6893
|
-
"cachedInputTokens"
|
|
6894
|
-
];
|
|
6895
7043
|
var TranslationChunkValidationError = class extends Error {
|
|
6896
7044
|
constructor(message) {
|
|
6897
7045
|
super(message);
|
|
@@ -6922,9 +7070,6 @@ function shouldSplitChunkTranslationError(error) {
|
|
|
6922
7070
|
}
|
|
6923
7071
|
return NoObjectGeneratedError.isInstance(error) || TypeValidationError.isInstance(error) || isTranslationChunkValidationError(error);
|
|
6924
7072
|
}
|
|
6925
|
-
function isDefinedTokenUsageValue(value) {
|
|
6926
|
-
return typeof value === "number";
|
|
6927
|
-
}
|
|
6928
7073
|
function resolveTranslationChunkingOptions(options) {
|
|
6929
7074
|
const targetChunkDurationSeconds = Math.max(
|
|
6930
7075
|
1,
|
|
@@ -6951,14 +7096,27 @@ function resolveTranslationChunkingOptions(options) {
|
|
|
6951
7096
|
)
|
|
6952
7097
|
};
|
|
6953
7098
|
}
|
|
6954
|
-
function
|
|
6955
|
-
|
|
6956
|
-
|
|
6957
|
-
|
|
6958
|
-
|
|
7099
|
+
function collectSettledTranslationsOrRethrow(outcomes, priorUsage) {
|
|
7100
|
+
const results = [];
|
|
7101
|
+
const failures = [];
|
|
7102
|
+
for (const outcome of outcomes) {
|
|
7103
|
+
if (outcome.status === "fulfilled") {
|
|
7104
|
+
results.push(outcome.value);
|
|
7105
|
+
} else {
|
|
7106
|
+
failures.push(outcome.reason);
|
|
6959
7107
|
}
|
|
6960
|
-
|
|
6961
|
-
|
|
7108
|
+
}
|
|
7109
|
+
if (failures.length > 0) {
|
|
7110
|
+
const partialUsage = [...priorUsage, ...results.map((result) => result.usage)];
|
|
7111
|
+
for (const failure of failures.slice(1)) {
|
|
7112
|
+
const usage = getErrorTokenUsage(failure);
|
|
7113
|
+
if (usage) {
|
|
7114
|
+
partialUsage.push(usage);
|
|
7115
|
+
}
|
|
7116
|
+
}
|
|
7117
|
+
rethrowWithTokenUsage(failures[0], partialUsage);
|
|
7118
|
+
}
|
|
7119
|
+
return results;
|
|
6962
7120
|
}
|
|
6963
7121
|
function createTranslationChunkRequest(id, cues, cueBlocks) {
|
|
6964
7122
|
return {
|
|
@@ -7234,24 +7392,28 @@ async function translateChunkWithFallback({
|
|
|
7234
7392
|
wrapError(error, `Chunk ${chunk.id} failed for ${Math.round(chunk.startTime)}s-${Math.round(chunk.endTime)}s`);
|
|
7235
7393
|
}
|
|
7236
7394
|
const [leftChunk, rightChunk] = splitTranslationChunkAtMidpoint(chunk);
|
|
7237
|
-
const
|
|
7238
|
-
|
|
7239
|
-
|
|
7240
|
-
|
|
7241
|
-
|
|
7242
|
-
|
|
7243
|
-
|
|
7244
|
-
|
|
7245
|
-
|
|
7246
|
-
|
|
7247
|
-
|
|
7248
|
-
|
|
7249
|
-
|
|
7250
|
-
|
|
7251
|
-
|
|
7252
|
-
|
|
7253
|
-
|
|
7254
|
-
|
|
7395
|
+
const failedAttemptUsage = getErrorTokenUsage(error);
|
|
7396
|
+
const [leftResult, rightResult] = collectSettledTranslationsOrRethrow(
|
|
7397
|
+
await Promise.allSettled([
|
|
7398
|
+
translateChunkWithFallback({
|
|
7399
|
+
chunk: leftChunk,
|
|
7400
|
+
fromLanguageCode,
|
|
7401
|
+
toLanguageCode,
|
|
7402
|
+
provider,
|
|
7403
|
+
modelId,
|
|
7404
|
+
credentials
|
|
7405
|
+
}),
|
|
7406
|
+
translateChunkWithFallback({
|
|
7407
|
+
chunk: rightChunk,
|
|
7408
|
+
fromLanguageCode,
|
|
7409
|
+
toLanguageCode,
|
|
7410
|
+
provider,
|
|
7411
|
+
modelId,
|
|
7412
|
+
credentials
|
|
7413
|
+
})
|
|
7414
|
+
]),
|
|
7415
|
+
failedAttemptUsage ? [failedAttemptUsage] : []
|
|
7416
|
+
);
|
|
7255
7417
|
return {
|
|
7256
7418
|
translatedVtt: concatenateVttSegments([leftResult.translatedVtt, rightResult.translatedVtt]),
|
|
7257
7419
|
usage: aggregateTokenUsage([leftResult.usage, rightResult.usage]),
|
|
@@ -7292,17 +7454,20 @@ async function translateCaptionTrack({
|
|
|
7292
7454
|
let totalUnexpectedKeyCount = 0;
|
|
7293
7455
|
for (let index = 0; index < chunkPlan.chunks.length; index += resolvedChunking.maxConcurrentTranslations) {
|
|
7294
7456
|
const batch = chunkPlan.chunks.slice(index, index + resolvedChunking.maxConcurrentTranslations);
|
|
7295
|
-
const batchResults =
|
|
7296
|
-
|
|
7297
|
-
(
|
|
7298
|
-
chunk
|
|
7299
|
-
|
|
7300
|
-
|
|
7301
|
-
|
|
7302
|
-
|
|
7303
|
-
|
|
7304
|
-
|
|
7305
|
-
|
|
7457
|
+
const batchResults = collectSettledTranslationsOrRethrow(
|
|
7458
|
+
await Promise.allSettled(
|
|
7459
|
+
batch.map(
|
|
7460
|
+
(chunk) => translateChunkWithFallback({
|
|
7461
|
+
chunk,
|
|
7462
|
+
fromLanguageCode,
|
|
7463
|
+
toLanguageCode,
|
|
7464
|
+
provider,
|
|
7465
|
+
modelId,
|
|
7466
|
+
credentials
|
|
7467
|
+
})
|
|
7468
|
+
)
|
|
7469
|
+
),
|
|
7470
|
+
usageByChunk
|
|
7306
7471
|
);
|
|
7307
7472
|
translatedSegments.push(...batchResults.map((result) => result.translatedVtt));
|
|
7308
7473
|
usageByChunk.push(...batchResults.map((result) => result.usage));
|
|
@@ -7355,6 +7520,14 @@ async function uploadVttToS3({
|
|
|
7355
7520
|
}
|
|
7356
7521
|
async function translateCaptions(assetId, trackId, toLanguageCode, options) {
|
|
7357
7522
|
"use workflow";
|
|
7523
|
+
const collectedUsage = [];
|
|
7524
|
+
try {
|
|
7525
|
+
return await translateCaptionsInternal(assetId, trackId, toLanguageCode, options, collectedUsage);
|
|
7526
|
+
} catch (error) {
|
|
7527
|
+
rethrowWithTokenUsage(error, collectedUsage);
|
|
7528
|
+
}
|
|
7529
|
+
}
|
|
7530
|
+
async function translateCaptionsInternal(assetId, trackId, toLanguageCode, options, collectedUsage) {
|
|
7358
7531
|
const {
|
|
7359
7532
|
provider = "openai",
|
|
7360
7533
|
model,
|
|
@@ -7433,6 +7606,7 @@ async function translateCaptions(assetId, trackId, toLanguageCode, options) {
|
|
|
7433
7606
|
});
|
|
7434
7607
|
translatedVtt = result.translatedVtt;
|
|
7435
7608
|
usage = result.usage;
|
|
7609
|
+
collectedUsage.push(result.usage);
|
|
7436
7610
|
scrubbedCueCounts = result.scrubbedCueCounts;
|
|
7437
7611
|
unexpectedKeyCount = result.unexpectedKeyCount;
|
|
7438
7612
|
} catch (error) {
|