@mux/ai 0.28.1 → 0.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/{index-bjQi1ZCj.d.ts → index-CJtQiGj7.d.ts} +18 -4
- package/dist/{index-DUVBpnj5.d.ts → index-D3cB2ltG.d.ts} +47 -15
- package/dist/index.d.ts +7 -6
- package/dist/index.js +571 -189
- package/dist/index.js.map +1 -1
- package/dist/primitives/index.d.ts +2 -2
- package/dist/primitives/index.js +122 -21
- package/dist/primitives/index.js.map +1 -1
- package/dist/{shots-mGJfYGiO.d.ts → shots-Bh0GrvLx.d.ts} +21 -1
- package/dist/workflows/index.d.ts +2 -2
- package/dist/workflows/index.js +678 -295
- package/dist/workflows/index.js.map +1 -1
- package/package.json +7 -1
package/dist/workflows/index.js
CHANGED
|
@@ -1,114 +1,11 @@
|
|
|
1
1
|
// src/workflows/ask-questions.ts
|
|
2
2
|
import { generateText, Output } from "ai";
|
|
3
3
|
import dedent2 from "dedent";
|
|
4
|
-
import { z as
|
|
4
|
+
import { z as z3 } from "zod";
|
|
5
5
|
|
|
6
|
-
// src/lib/
|
|
7
|
-
import
|
|
8
|
-
|
|
9
|
-
function bytesToBase64(bytes) {
|
|
10
|
-
if (bytes.length === 0) {
|
|
11
|
-
return "";
|
|
12
|
-
}
|
|
13
|
-
let binary = "";
|
|
14
|
-
for (let i = 0; i < bytes.length; i += BASE64_CHUNK_SIZE) {
|
|
15
|
-
const chunk = bytes.subarray(i, i + BASE64_CHUNK_SIZE);
|
|
16
|
-
binary += String.fromCharCode(...chunk);
|
|
17
|
-
}
|
|
18
|
-
if (typeof globalThis.btoa !== "function") {
|
|
19
|
-
throw new TypeError("Base64 encoder is not available in this environment.");
|
|
20
|
-
}
|
|
21
|
-
return globalThis.btoa(binary);
|
|
22
|
-
}
|
|
23
|
-
var DEFAULT_OPTIONS = {
|
|
24
|
-
timeout: 1e4,
|
|
25
|
-
retries: 3,
|
|
26
|
-
retryDelay: 1e3,
|
|
27
|
-
maxRetryDelay: 1e4,
|
|
28
|
-
exponentialBackoff: true
|
|
29
|
-
};
|
|
30
|
-
async function downloadImageAsBase64(url, options = {}) {
|
|
31
|
-
"use step";
|
|
32
|
-
const opts = { ...DEFAULT_OPTIONS, ...options };
|
|
33
|
-
let attemptCount = 0;
|
|
34
|
-
return pRetry(
|
|
35
|
-
async () => {
|
|
36
|
-
attemptCount++;
|
|
37
|
-
const controller = new AbortController();
|
|
38
|
-
const timeoutId = setTimeout(() => controller.abort(), opts.timeout);
|
|
39
|
-
try {
|
|
40
|
-
const response = await fetch(url, {
|
|
41
|
-
signal: controller.signal,
|
|
42
|
-
headers: {
|
|
43
|
-
"User-Agent": "@mux/ai image downloader"
|
|
44
|
-
}
|
|
45
|
-
});
|
|
46
|
-
clearTimeout(timeoutId);
|
|
47
|
-
if (!response.ok) {
|
|
48
|
-
if (response.status >= 400 && response.status < 500 && response.status !== 429) {
|
|
49
|
-
throw new AbortError(`HTTP ${response.status}: ${response.statusText}`);
|
|
50
|
-
}
|
|
51
|
-
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
|
52
|
-
}
|
|
53
|
-
const contentType = response.headers.get("content-type");
|
|
54
|
-
if (!contentType?.startsWith("image/")) {
|
|
55
|
-
throw new AbortError(`Invalid content type: ${contentType}. Expected image/*`);
|
|
56
|
-
}
|
|
57
|
-
const arrayBuffer = await response.arrayBuffer();
|
|
58
|
-
const buffer = new Uint8Array(arrayBuffer);
|
|
59
|
-
if (buffer.length === 0) {
|
|
60
|
-
throw new AbortError("Downloaded image is empty");
|
|
61
|
-
}
|
|
62
|
-
const base64Data = `data:${contentType};base64,${bytesToBase64(buffer)}`;
|
|
63
|
-
return {
|
|
64
|
-
base64Data,
|
|
65
|
-
buffer,
|
|
66
|
-
url,
|
|
67
|
-
contentType,
|
|
68
|
-
sizeBytes: buffer.length,
|
|
69
|
-
attempts: attemptCount
|
|
70
|
-
};
|
|
71
|
-
} catch (error) {
|
|
72
|
-
clearTimeout(timeoutId);
|
|
73
|
-
if (error instanceof AbortError) {
|
|
74
|
-
throw error;
|
|
75
|
-
}
|
|
76
|
-
if (error instanceof Error) {
|
|
77
|
-
if (error.name === "AbortError") {
|
|
78
|
-
throw new Error(`Request timeout after ${opts.timeout}ms`);
|
|
79
|
-
}
|
|
80
|
-
throw new Error(`Download failed: ${error.message}`);
|
|
81
|
-
}
|
|
82
|
-
throw new Error("Unknown download error");
|
|
83
|
-
}
|
|
84
|
-
},
|
|
85
|
-
{
|
|
86
|
-
retries: opts.retries,
|
|
87
|
-
minTimeout: opts.retryDelay,
|
|
88
|
-
maxTimeout: opts.maxRetryDelay,
|
|
89
|
-
factor: opts.exponentialBackoff ? 2 : 1,
|
|
90
|
-
randomize: true,
|
|
91
|
-
// Add jitter to prevent thundering herd
|
|
92
|
-
onFailedAttempt: (error) => {
|
|
93
|
-
console.warn(`Image download attempt ${error.attemptNumber} failed for ${url}`);
|
|
94
|
-
if (error.retriesLeft > 0) {
|
|
95
|
-
console.warn(`Retrying... (${error.retriesLeft} attempts left)`);
|
|
96
|
-
}
|
|
97
|
-
}
|
|
98
|
-
}
|
|
99
|
-
);
|
|
100
|
-
}
|
|
101
|
-
async function downloadImagesAsBase64(urls, options = {}, maxConcurrent = 5) {
|
|
102
|
-
"use step";
|
|
103
|
-
const results = [];
|
|
104
|
-
for (let i = 0; i < urls.length; i += maxConcurrent) {
|
|
105
|
-
const batch = urls.slice(i, i + maxConcurrent);
|
|
106
|
-
const batchPromises = batch.map((url) => downloadImageAsBase64(url, options));
|
|
107
|
-
const batchResults = await Promise.all(batchPromises);
|
|
108
|
-
results.push(...batchResults);
|
|
109
|
-
}
|
|
110
|
-
return results;
|
|
111
|
-
}
|
|
6
|
+
// src/lib/content-policy-error.ts
|
|
7
|
+
import { APICallError, NoObjectGeneratedError, RetryError } from "ai";
|
|
8
|
+
import { z as z2 } from "zod";
|
|
112
9
|
|
|
113
10
|
// src/lib/prompt-fragments.ts
|
|
114
11
|
import dedent from "dedent";
|
|
@@ -179,7 +76,7 @@ var EnvSchema = z.object({
|
|
|
179
76
|
MUX_TEST_ASSET_ID_VIOLENT_AUDIO_ONLY: optionalString("Mux test asset ID for audio-only assets with violent content.", "Mux test asset id for audio-only assets with violent content for testing"),
|
|
180
77
|
// Eval config
|
|
181
78
|
MUX_AI_EVAL_MODEL_SET: optionalString("Eval model selection mode.", "Choose between 'default' (provider defaults only) or 'all' (all configured models)"),
|
|
182
|
-
MUX_AI_EVAL_MODELS: optionalString("Comma-separated eval model pairs.", "Comma-separated provider:model pairs (e.g. 'openai:gpt-5.
|
|
79
|
+
MUX_AI_EVAL_MODELS: optionalString("Comma-separated eval model pairs.", "Comma-separated provider:model pairs (e.g. 'openai:gpt-5.6-luna,anthropic:claude-sonnet-4-5,google:gemini-3-flash-preview')"),
|
|
183
80
|
// AI Providers
|
|
184
81
|
OPENAI_API_KEY: optionalString("OpenAI API key for OpenAI-backed workflows.", "OpenAI API key"),
|
|
185
82
|
ANTHROPIC_API_KEY: optionalString("Anthropic API key for Claude-backed workflows.", "Anthropic API key"),
|
|
@@ -498,7 +395,8 @@ var TOKEN_USAGE_FIELDS = [
|
|
|
498
395
|
"outputTokens",
|
|
499
396
|
"totalTokens",
|
|
500
397
|
"reasoningTokens",
|
|
501
|
-
"cachedInputTokens"
|
|
398
|
+
"cachedInputTokens",
|
|
399
|
+
"cacheWriteTokens"
|
|
502
400
|
];
|
|
503
401
|
function isDefinedTokenUsageValue(value) {
|
|
504
402
|
return typeof value === "number";
|
|
@@ -540,6 +438,12 @@ function getErrorTokenUsage(error) {
|
|
|
540
438
|
extracted.cachedInputTokens = cacheReadTokens;
|
|
541
439
|
}
|
|
542
440
|
}
|
|
441
|
+
if (extracted.cacheWriteTokens === void 0) {
|
|
442
|
+
const cacheWriteTokens = source.inputTokenDetails?.cacheWriteTokens;
|
|
443
|
+
if (typeof cacheWriteTokens === "number") {
|
|
444
|
+
extracted.cacheWriteTokens = cacheWriteTokens;
|
|
445
|
+
}
|
|
446
|
+
}
|
|
543
447
|
return Object.keys(extracted).length > 0 ? extracted : void 0;
|
|
544
448
|
}
|
|
545
449
|
function rethrowWithTokenUsage(error, collectedUsage) {
|
|
@@ -565,6 +469,9 @@ var MuxAiError = class extends Error {
|
|
|
565
469
|
this.publicMessage = message;
|
|
566
470
|
this.retryable = opts?.retryable ?? false;
|
|
567
471
|
}
|
|
472
|
+
static is(value) {
|
|
473
|
+
return typeof value === "object" && value !== null && "__robots_error" in value && value.__robots_error === true;
|
|
474
|
+
}
|
|
568
475
|
};
|
|
569
476
|
function extractErrorDetail(error) {
|
|
570
477
|
if (error instanceof Error && error.message) {
|
|
@@ -582,7 +489,7 @@ function extractErrorDetail(error) {
|
|
|
582
489
|
return "Unknown error";
|
|
583
490
|
}
|
|
584
491
|
function wrapError(error, message) {
|
|
585
|
-
if (error
|
|
492
|
+
if (MuxAiError.is(error)) {
|
|
586
493
|
throw error;
|
|
587
494
|
}
|
|
588
495
|
const rawDetail = extractErrorDetail(error);
|
|
@@ -600,6 +507,260 @@ function wrapError(error, message) {
|
|
|
600
507
|
throw wrapped;
|
|
601
508
|
}
|
|
602
509
|
|
|
510
|
+
// src/lib/content-policy-error.ts
|
|
511
|
+
var PolicyTokenSchema = z2.string().trim().min(1).max(64).regex(/^[A-Z0-9_]+$/);
|
|
512
|
+
var SafetyRatingSchema = z2.object({
|
|
513
|
+
category: PolicyTokenSchema.nullish(),
|
|
514
|
+
blocked: z2.boolean().nullish()
|
|
515
|
+
});
|
|
516
|
+
var ContentPolicyResponseSchema = z2.object({
|
|
517
|
+
promptFeedback: z2.object({
|
|
518
|
+
blockReason: PolicyTokenSchema.nullish(),
|
|
519
|
+
safetyRatings: z2.array(SafetyRatingSchema).max(32).nullish()
|
|
520
|
+
}).nullish(),
|
|
521
|
+
candidates: z2.array(z2.object({
|
|
522
|
+
finishReason: PolicyTokenSchema.nullish(),
|
|
523
|
+
safetyRatings: z2.array(SafetyRatingSchema).max(32).nullish()
|
|
524
|
+
})).max(8).nullish()
|
|
525
|
+
});
|
|
526
|
+
var ContentPolicyUsageResponseSchema = z2.object({
|
|
527
|
+
usageMetadata: z2.object({
|
|
528
|
+
promptTokenCount: z2.number().finite().nonnegative().nullish(),
|
|
529
|
+
candidatesTokenCount: z2.number().finite().nonnegative().nullish(),
|
|
530
|
+
totalTokenCount: z2.number().finite().nonnegative().nullish(),
|
|
531
|
+
cachedContentTokenCount: z2.number().finite().nonnegative().nullish(),
|
|
532
|
+
thoughtsTokenCount: z2.number().finite().nonnegative().nullish()
|
|
533
|
+
}).nullish()
|
|
534
|
+
});
|
|
535
|
+
var CONTENT_POLICY_REASONS = /* @__PURE__ */ new Set([
|
|
536
|
+
"BLOCKLIST",
|
|
537
|
+
"ESCALATION",
|
|
538
|
+
"IMAGE_PROHIBITED_CONTENT",
|
|
539
|
+
"IMAGE_RECITATION",
|
|
540
|
+
"IMAGE_SAFETY",
|
|
541
|
+
"JAILBREAK",
|
|
542
|
+
"MODEL_ARMOR",
|
|
543
|
+
"PROHIBITED_CONTENT",
|
|
544
|
+
"RECITATION",
|
|
545
|
+
"SAFETY",
|
|
546
|
+
"SPII"
|
|
547
|
+
]);
|
|
548
|
+
async function withContentPolicyErrorHandling(operation) {
|
|
549
|
+
try {
|
|
550
|
+
return await operation();
|
|
551
|
+
} catch (error) {
|
|
552
|
+
rethrowContentPolicyError(error);
|
|
553
|
+
}
|
|
554
|
+
}
|
|
555
|
+
function rethrowContentPolicyError(error) {
|
|
556
|
+
const block = extractContentPolicyBlock(error);
|
|
557
|
+
if (!block) {
|
|
558
|
+
throw error;
|
|
559
|
+
}
|
|
560
|
+
throwContentPolicyError(block, getContentPolicyTokenUsage(error));
|
|
561
|
+
}
|
|
562
|
+
function getGeneratedOutputWithContentPolicyHandling(response) {
|
|
563
|
+
if (response.finishReason === "content-filter") {
|
|
564
|
+
const rawReason = PolicyTokenSchema.safeParse(response.rawFinishReason);
|
|
565
|
+
const reason = rawReason.success && isContentPolicyReason(rawReason.data) ? rawReason.data : "CONTENT_FILTER";
|
|
566
|
+
throwContentPolicyError({ reason }, getErrorTokenUsage(response));
|
|
567
|
+
}
|
|
568
|
+
return response.output;
|
|
569
|
+
}
|
|
570
|
+
function throwContentPolicyError(block, usage) {
|
|
571
|
+
const category = block.category ? `; category: ${block.category}` : "";
|
|
572
|
+
const contentPolicyError = new MuxAiError(
|
|
573
|
+
`The supplied content was blocked by a content policy (reason: ${block.reason}${category}).`,
|
|
574
|
+
{
|
|
575
|
+
type: "content_policy_error",
|
|
576
|
+
retryable: false
|
|
577
|
+
}
|
|
578
|
+
);
|
|
579
|
+
if (usage) {
|
|
580
|
+
contentPolicyError.usage = usage;
|
|
581
|
+
}
|
|
582
|
+
throw contentPolicyError;
|
|
583
|
+
}
|
|
584
|
+
function extractContentPolicyBlock(error) {
|
|
585
|
+
if (RetryError.isInstance(error)) {
|
|
586
|
+
return extractContentPolicyBlock(error.lastError);
|
|
587
|
+
}
|
|
588
|
+
if (NoObjectGeneratedError.isInstance(error) && error.finishReason === "content-filter") {
|
|
589
|
+
return { reason: "CONTENT_FILTER" };
|
|
590
|
+
}
|
|
591
|
+
if (!APICallError.isInstance(error) || !error.responseBody) {
|
|
592
|
+
return void 0;
|
|
593
|
+
}
|
|
594
|
+
try {
|
|
595
|
+
const parsed = ContentPolicyResponseSchema.safeParse(JSON.parse(error.responseBody));
|
|
596
|
+
if (!parsed.success) {
|
|
597
|
+
return void 0;
|
|
598
|
+
}
|
|
599
|
+
const promptReason = parsed.data.promptFeedback?.blockReason;
|
|
600
|
+
if (isContentPolicyReason(promptReason)) {
|
|
601
|
+
return {
|
|
602
|
+
reason: promptReason,
|
|
603
|
+
category: findBlockedCategory(parsed.data.promptFeedback?.safetyRatings)
|
|
604
|
+
};
|
|
605
|
+
}
|
|
606
|
+
for (const candidate of parsed.data.candidates ?? []) {
|
|
607
|
+
if (isContentPolicyReason(candidate.finishReason)) {
|
|
608
|
+
return {
|
|
609
|
+
reason: candidate.finishReason,
|
|
610
|
+
category: findBlockedCategory(candidate.safetyRatings)
|
|
611
|
+
};
|
|
612
|
+
}
|
|
613
|
+
}
|
|
614
|
+
} catch {
|
|
615
|
+
return void 0;
|
|
616
|
+
}
|
|
617
|
+
return void 0;
|
|
618
|
+
}
|
|
619
|
+
function isContentPolicyReason(reason) {
|
|
620
|
+
return Boolean(reason && CONTENT_POLICY_REASONS.has(reason));
|
|
621
|
+
}
|
|
622
|
+
function getContentPolicyTokenUsage(error) {
|
|
623
|
+
const errorUsage = getErrorTokenUsage(error);
|
|
624
|
+
if (errorUsage) {
|
|
625
|
+
return errorUsage;
|
|
626
|
+
}
|
|
627
|
+
if (RetryError.isInstance(error)) {
|
|
628
|
+
return getContentPolicyTokenUsage(error.lastError);
|
|
629
|
+
}
|
|
630
|
+
if (!APICallError.isInstance(error) || !error.responseBody) {
|
|
631
|
+
return void 0;
|
|
632
|
+
}
|
|
633
|
+
try {
|
|
634
|
+
const parsed = ContentPolicyUsageResponseSchema.safeParse(JSON.parse(error.responseBody));
|
|
635
|
+
const usage = parsed.success ? parsed.data.usageMetadata : void 0;
|
|
636
|
+
if (!usage) {
|
|
637
|
+
return void 0;
|
|
638
|
+
}
|
|
639
|
+
const inputTokens = usage.promptTokenCount ?? 0;
|
|
640
|
+
const reasoningTokens = usage.thoughtsTokenCount ?? 0;
|
|
641
|
+
const outputTokens = (usage.candidatesTokenCount ?? 0) + reasoningTokens;
|
|
642
|
+
return {
|
|
643
|
+
inputTokens,
|
|
644
|
+
outputTokens,
|
|
645
|
+
totalTokens: usage.totalTokenCount ?? inputTokens + outputTokens,
|
|
646
|
+
reasoningTokens,
|
|
647
|
+
cachedInputTokens: usage.cachedContentTokenCount ?? 0
|
|
648
|
+
};
|
|
649
|
+
} catch {
|
|
650
|
+
return void 0;
|
|
651
|
+
}
|
|
652
|
+
}
|
|
653
|
+
function findBlockedCategory(ratings) {
|
|
654
|
+
return ratings?.find((rating) => rating.blocked === true)?.category ?? void 0;
|
|
655
|
+
}
|
|
656
|
+
|
|
657
|
+
// src/lib/image-download.ts
|
|
658
|
+
import pRetry, { AbortError } from "p-retry";
|
|
659
|
+
var BASE64_CHUNK_SIZE = 32768;
|
|
660
|
+
function bytesToBase64(bytes) {
|
|
661
|
+
if (bytes.length === 0) {
|
|
662
|
+
return "";
|
|
663
|
+
}
|
|
664
|
+
let binary = "";
|
|
665
|
+
for (let i = 0; i < bytes.length; i += BASE64_CHUNK_SIZE) {
|
|
666
|
+
const chunk = bytes.subarray(i, i + BASE64_CHUNK_SIZE);
|
|
667
|
+
binary += String.fromCharCode(...chunk);
|
|
668
|
+
}
|
|
669
|
+
if (typeof globalThis.btoa !== "function") {
|
|
670
|
+
throw new TypeError("Base64 encoder is not available in this environment.");
|
|
671
|
+
}
|
|
672
|
+
return globalThis.btoa(binary);
|
|
673
|
+
}
|
|
674
|
+
var DEFAULT_OPTIONS = {
|
|
675
|
+
timeout: 1e4,
|
|
676
|
+
retries: 3,
|
|
677
|
+
retryDelay: 1e3,
|
|
678
|
+
maxRetryDelay: 1e4,
|
|
679
|
+
exponentialBackoff: true
|
|
680
|
+
};
|
|
681
|
+
async function downloadImageAsBase64(url, options = {}) {
|
|
682
|
+
"use step";
|
|
683
|
+
const opts = { ...DEFAULT_OPTIONS, ...options };
|
|
684
|
+
let attemptCount = 0;
|
|
685
|
+
return pRetry(
|
|
686
|
+
async () => {
|
|
687
|
+
attemptCount++;
|
|
688
|
+
const controller = new AbortController();
|
|
689
|
+
const timeoutId = setTimeout(() => controller.abort(), opts.timeout);
|
|
690
|
+
try {
|
|
691
|
+
const response = await fetch(url, {
|
|
692
|
+
signal: controller.signal,
|
|
693
|
+
headers: {
|
|
694
|
+
"User-Agent": "@mux/ai image downloader"
|
|
695
|
+
}
|
|
696
|
+
});
|
|
697
|
+
clearTimeout(timeoutId);
|
|
698
|
+
if (!response.ok) {
|
|
699
|
+
if (response.status >= 400 && response.status < 500 && response.status !== 429) {
|
|
700
|
+
throw new AbortError(`HTTP ${response.status}: ${response.statusText}`);
|
|
701
|
+
}
|
|
702
|
+
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
|
703
|
+
}
|
|
704
|
+
const contentType = response.headers.get("content-type");
|
|
705
|
+
if (!contentType?.startsWith("image/")) {
|
|
706
|
+
throw new AbortError(`Invalid content type: ${contentType}. Expected image/*`);
|
|
707
|
+
}
|
|
708
|
+
const arrayBuffer = await response.arrayBuffer();
|
|
709
|
+
const buffer = new Uint8Array(arrayBuffer);
|
|
710
|
+
if (buffer.length === 0) {
|
|
711
|
+
throw new AbortError("Downloaded image is empty");
|
|
712
|
+
}
|
|
713
|
+
const base64Data = `data:${contentType};base64,${bytesToBase64(buffer)}`;
|
|
714
|
+
return {
|
|
715
|
+
base64Data,
|
|
716
|
+
buffer,
|
|
717
|
+
url,
|
|
718
|
+
contentType,
|
|
719
|
+
sizeBytes: buffer.length,
|
|
720
|
+
attempts: attemptCount
|
|
721
|
+
};
|
|
722
|
+
} catch (error) {
|
|
723
|
+
clearTimeout(timeoutId);
|
|
724
|
+
if (error instanceof AbortError) {
|
|
725
|
+
throw error;
|
|
726
|
+
}
|
|
727
|
+
if (error instanceof Error) {
|
|
728
|
+
if (error.name === "AbortError") {
|
|
729
|
+
throw new Error(`Request timeout after ${opts.timeout}ms`);
|
|
730
|
+
}
|
|
731
|
+
throw new Error(`Download failed: ${error.message}`);
|
|
732
|
+
}
|
|
733
|
+
throw new Error("Unknown download error");
|
|
734
|
+
}
|
|
735
|
+
},
|
|
736
|
+
{
|
|
737
|
+
retries: opts.retries,
|
|
738
|
+
minTimeout: opts.retryDelay,
|
|
739
|
+
maxTimeout: opts.maxRetryDelay,
|
|
740
|
+
factor: opts.exponentialBackoff ? 2 : 1,
|
|
741
|
+
randomize: true,
|
|
742
|
+
// Add jitter to prevent thundering herd
|
|
743
|
+
onFailedAttempt: (error) => {
|
|
744
|
+
console.warn(`Image download attempt ${error.attemptNumber} failed for ${url}`);
|
|
745
|
+
if (error.retriesLeft > 0) {
|
|
746
|
+
console.warn(`Retrying... (${error.retriesLeft} attempts left)`);
|
|
747
|
+
}
|
|
748
|
+
}
|
|
749
|
+
}
|
|
750
|
+
);
|
|
751
|
+
}
|
|
752
|
+
async function downloadImagesAsBase64(urls, options = {}, maxConcurrent = 5) {
|
|
753
|
+
"use step";
|
|
754
|
+
const results = [];
|
|
755
|
+
for (let i = 0; i < urls.length; i += maxConcurrent) {
|
|
756
|
+
const batch = urls.slice(i, i + maxConcurrent);
|
|
757
|
+
const batchPromises = batch.map((url) => downloadImageAsBase64(url, options));
|
|
758
|
+
const batchResults = await Promise.all(batchPromises);
|
|
759
|
+
results.push(...batchResults);
|
|
760
|
+
}
|
|
761
|
+
return results;
|
|
762
|
+
}
|
|
763
|
+
|
|
603
764
|
// src/lib/providers.ts
|
|
604
765
|
import { createAnthropic } from "@ai-sdk/anthropic";
|
|
605
766
|
import { createGoogleGenerativeAI } from "@ai-sdk/google";
|
|
@@ -936,7 +1097,8 @@ async function resolveMuxSigningContext(credentials) {
|
|
|
936
1097
|
|
|
937
1098
|
// src/lib/providers.ts
|
|
938
1099
|
var DEFAULT_LANGUAGE_MODELS = {
|
|
939
|
-
|
|
1100
|
+
// GPT-5.6 models default to medium reasoning when no effort is specified.
|
|
1101
|
+
openai: "gpt-5.6-luna",
|
|
940
1102
|
anthropic: "claude-sonnet-4-5",
|
|
941
1103
|
google: "gemini-3-flash-preview"
|
|
942
1104
|
};
|
|
@@ -945,11 +1107,20 @@ var DEFAULT_EMBEDDING_MODELS = {
|
|
|
945
1107
|
google: "gemini-embedding-001"
|
|
946
1108
|
};
|
|
947
1109
|
var LANGUAGE_MODELS = {
|
|
948
|
-
openai: ["gpt-5.1", "gpt-5-mini"],
|
|
1110
|
+
openai: ["gpt-5.6-luna", "gpt-5.1", "gpt-5-mini"],
|
|
949
1111
|
anthropic: ["claude-sonnet-4-5"],
|
|
950
1112
|
google: ["gemini-3-flash-preview", "gemini-3.1-flash-lite", "gemini-2.5-flash"]
|
|
951
1113
|
};
|
|
952
1114
|
var LANGUAGE_MODEL_DEPRECATIONS = [
|
|
1115
|
+
{
|
|
1116
|
+
provider: "openai",
|
|
1117
|
+
modelId: "gpt-5.1",
|
|
1118
|
+
replacementModelId: "gpt-5.6-luna",
|
|
1119
|
+
phase: "warn",
|
|
1120
|
+
deprecatedOn: "2026-08-01",
|
|
1121
|
+
sunsetOn: "2026-10-01",
|
|
1122
|
+
reason: "GPT-5.6 Luna with its default medium reasoning effort is the new OpenAI default."
|
|
1123
|
+
},
|
|
953
1124
|
{
|
|
954
1125
|
provider: "google",
|
|
955
1126
|
modelId: "gemini-2.5-flash",
|
|
@@ -1010,7 +1181,7 @@ function parseEvalModelPair(value) {
|
|
|
1010
1181
|
const modelId = modelIdRaw?.trim();
|
|
1011
1182
|
if (!provider || !modelId) {
|
|
1012
1183
|
throw new Error(
|
|
1013
|
-
`Invalid eval model pair "${value}". Use "provider:model" (example: "openai:gpt-5.
|
|
1184
|
+
`Invalid eval model pair "${value}". Use "provider:model" (example: "openai:gpt-5.6-luna").`
|
|
1014
1185
|
);
|
|
1015
1186
|
}
|
|
1016
1187
|
if (!isSupportedProvider(provider)) {
|
|
@@ -1283,19 +1454,82 @@ async function withRetry(fn, {
|
|
|
1283
1454
|
try {
|
|
1284
1455
|
return await fn();
|
|
1285
1456
|
} catch (error) {
|
|
1286
|
-
lastError = error
|
|
1457
|
+
lastError = error;
|
|
1458
|
+
const retryError = error instanceof Error ? error : new Error(String(error));
|
|
1287
1459
|
const isLastAttempt = attempt === maxRetries;
|
|
1288
|
-
if (isLastAttempt || !shouldRetry(
|
|
1289
|
-
throw
|
|
1460
|
+
if (isLastAttempt || !shouldRetry(retryError, attempt + 1)) {
|
|
1461
|
+
throw error;
|
|
1290
1462
|
}
|
|
1291
1463
|
const delay = calculateDelay(attempt + 1, baseDelay, maxDelay);
|
|
1292
1464
|
console.warn(
|
|
1293
|
-
`Attempt ${attempt + 1} failed: ${
|
|
1465
|
+
`Attempt ${attempt + 1} failed: ${retryError.message}. Retrying in ${Math.round(delay)}ms...`
|
|
1294
1466
|
);
|
|
1295
1467
|
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
1296
1468
|
}
|
|
1297
1469
|
}
|
|
1298
|
-
throw lastError
|
|
1470
|
+
throw lastError ?? new Error("Retry failed with unknown error");
|
|
1471
|
+
}
|
|
1472
|
+
|
|
1473
|
+
// src/lib/workflow-scope.ts
|
|
1474
|
+
function hasWorkflowScopeBoundaries(scope) {
|
|
1475
|
+
return scope?.startTime !== void 0 || scope?.endTime !== void 0;
|
|
1476
|
+
}
|
|
1477
|
+
function resolveWorkflowScope(scope, assetDurationSeconds) {
|
|
1478
|
+
if (assetDurationSeconds === void 0 || !Number.isFinite(assetDurationSeconds) || assetDurationSeconds <= 0) {
|
|
1479
|
+
throw new MuxAiError("Asset has no valid duration.", { type: "validation_error" });
|
|
1480
|
+
}
|
|
1481
|
+
const startTime = scope?.startTime ?? 0;
|
|
1482
|
+
const endTime = scope?.endTime ?? assetDurationSeconds;
|
|
1483
|
+
if (!Number.isFinite(startTime) || startTime < 0) {
|
|
1484
|
+
throw new MuxAiError(
|
|
1485
|
+
`scope.startTime must be a finite, non-negative number (received ${startTime}).`,
|
|
1486
|
+
{ type: "validation_error" }
|
|
1487
|
+
);
|
|
1488
|
+
}
|
|
1489
|
+
if (!Number.isFinite(endTime) || endTime < 0) {
|
|
1490
|
+
throw new MuxAiError(
|
|
1491
|
+
`scope.endTime must be a finite, non-negative number (received ${endTime}).`,
|
|
1492
|
+
{ type: "validation_error" }
|
|
1493
|
+
);
|
|
1494
|
+
}
|
|
1495
|
+
if (endTime > assetDurationSeconds) {
|
|
1496
|
+
throw new MuxAiError(
|
|
1497
|
+
`scope.endTime (${endTime}) cannot exceed the asset duration (${assetDurationSeconds}).`,
|
|
1498
|
+
{ type: "validation_error" }
|
|
1499
|
+
);
|
|
1500
|
+
}
|
|
1501
|
+
if (startTime >= endTime) {
|
|
1502
|
+
throw new MuxAiError(
|
|
1503
|
+
`scope.startTime (${startTime}) must be less than scope.endTime (${endTime}).`,
|
|
1504
|
+
{ type: "validation_error" }
|
|
1505
|
+
);
|
|
1506
|
+
}
|
|
1507
|
+
return { startTime, endTime };
|
|
1508
|
+
}
|
|
1509
|
+
function resolveRenderableVideoScope(scope, assetDurationSeconds, videoTrackDurationSeconds) {
|
|
1510
|
+
if (!hasWorkflowScopeBoundaries(scope)) {
|
|
1511
|
+
return void 0;
|
|
1512
|
+
}
|
|
1513
|
+
const resolvedScope = resolveWorkflowScope(scope, assetDurationSeconds);
|
|
1514
|
+
let renderableEndTime = resolvedScope.endTime;
|
|
1515
|
+
if (videoTrackDurationSeconds !== void 0 && Number.isFinite(videoTrackDurationSeconds) && videoTrackDurationSeconds >= 0) {
|
|
1516
|
+
renderableEndTime = Math.min(videoTrackDurationSeconds, resolvedScope.endTime);
|
|
1517
|
+
}
|
|
1518
|
+
const renderableScope = {
|
|
1519
|
+
startTime: Math.min(resolvedScope.startTime, renderableEndTime),
|
|
1520
|
+
endTime: renderableEndTime
|
|
1521
|
+
};
|
|
1522
|
+
if (renderableScope.startTime >= renderableScope.endTime) {
|
|
1523
|
+
throw new MuxAiError("The requested scope does not include any renderable video.", {
|
|
1524
|
+
type: "validation_error"
|
|
1525
|
+
});
|
|
1526
|
+
}
|
|
1527
|
+
return renderableScope;
|
|
1528
|
+
}
|
|
1529
|
+
function timeRangesOverlap(firstStart, firstEnd, second) {
|
|
1530
|
+
const secondStart = second.startTime ?? 0;
|
|
1531
|
+
const secondEnd = second.endTime ?? Number.POSITIVE_INFINITY;
|
|
1532
|
+
return firstStart < secondEnd && firstEnd > secondStart;
|
|
1299
1533
|
}
|
|
1300
1534
|
|
|
1301
1535
|
// src/lib/mux-url.ts
|
|
@@ -1384,13 +1618,22 @@ async function signUrl(url, playbackId, type = "video", params, credentials) {
|
|
|
1384
1618
|
|
|
1385
1619
|
// src/primitives/storyboards.ts
|
|
1386
1620
|
var DEFAULT_STORYBOARD_WIDTH = 640;
|
|
1387
|
-
async function getStoryboardUrl(playbackId, width = DEFAULT_STORYBOARD_WIDTH, shouldSign = false, credentials) {
|
|
1621
|
+
async function getStoryboardUrl(playbackId, width = DEFAULT_STORYBOARD_WIDTH, shouldSign = false, credentials, scope) {
|
|
1388
1622
|
"use step";
|
|
1389
1623
|
const baseUrl = getMuxStoryboardBaseUrl(playbackId);
|
|
1624
|
+
const params = { width };
|
|
1625
|
+
if (scope?.startTime !== void 0) {
|
|
1626
|
+
params.asset_start_time = scope.startTime;
|
|
1627
|
+
}
|
|
1628
|
+
if (scope?.endTime !== void 0) {
|
|
1629
|
+
params.asset_end_time = scope.endTime;
|
|
1630
|
+
}
|
|
1390
1631
|
if (shouldSign) {
|
|
1391
|
-
return signUrl(baseUrl, playbackId, "storyboard",
|
|
1632
|
+
return signUrl(baseUrl, playbackId, "storyboard", params, credentials);
|
|
1392
1633
|
}
|
|
1393
|
-
return `${baseUrl}
|
|
1634
|
+
return `${baseUrl}?${new URLSearchParams(
|
|
1635
|
+
Object.entries(params).map(([key, value]) => [key, String(value)])
|
|
1636
|
+
).toString()}`;
|
|
1394
1637
|
}
|
|
1395
1638
|
|
|
1396
1639
|
// src/lib/language-codes.ts
|
|
@@ -1829,6 +2072,23 @@ function parseVTTCues(vttContent) {
|
|
|
1829
2072
|
}
|
|
1830
2073
|
return cues;
|
|
1831
2074
|
}
|
|
2075
|
+
function filterVttByScope(vttContent, scope) {
|
|
2076
|
+
if (!scope || scope.startTime === void 0 && scope.endTime === void 0) {
|
|
2077
|
+
return vttContent;
|
|
2078
|
+
}
|
|
2079
|
+
const { preamble, cueBlocks } = splitVttPreambleAndCueBlocks(vttContent);
|
|
2080
|
+
const scopedCueBlocks = cueBlocks.filter((cueBlock) => {
|
|
2081
|
+
const cue = parseVTTCues(cueBlock)[0];
|
|
2082
|
+
return cue ? timeRangesOverlap(cue.startTime, cue.endTime, scope) : false;
|
|
2083
|
+
}).map((cueBlock, index) => {
|
|
2084
|
+
const lines = cueBlock.split("\n");
|
|
2085
|
+
if (/^\d+$/.test(lines[0]?.trim()) && lines[1]?.includes("-->")) {
|
|
2086
|
+
lines[0] = String(index + 1);
|
|
2087
|
+
}
|
|
2088
|
+
return lines.join("\n");
|
|
2089
|
+
});
|
|
2090
|
+
return buildVttFromCueBlocks(scopedCueBlocks, preamble);
|
|
2091
|
+
}
|
|
1832
2092
|
function splitVttPreambleAndCueBlocks(vttContent) {
|
|
1833
2093
|
const normalizedContent = normalizeLineEndings(vttContent).trim();
|
|
1834
2094
|
if (!normalizedContent) {
|
|
@@ -1933,6 +2193,7 @@ async function fetchTranscriptForAsset(asset, playbackId, options = {}) {
|
|
|
1933
2193
|
const {
|
|
1934
2194
|
languageCode,
|
|
1935
2195
|
cleanTranscript = true,
|
|
2196
|
+
scope,
|
|
1936
2197
|
shouldSign,
|
|
1937
2198
|
credentials,
|
|
1938
2199
|
required = false
|
|
@@ -1964,9 +2225,15 @@ async function fetchTranscriptForAsset(asset, playbackId, options = {}) {
|
|
|
1964
2225
|
return { transcriptText: "", transcriptUrl, track };
|
|
1965
2226
|
}
|
|
1966
2227
|
const rawVtt = await response.text();
|
|
1967
|
-
const
|
|
1968
|
-
|
|
1969
|
-
|
|
2228
|
+
const effectiveScope = hasWorkflowScopeBoundaries(scope) ? scope : void 0;
|
|
2229
|
+
const scopedVtt = filterVttByScope(rawVtt, effectiveScope);
|
|
2230
|
+
const transcriptText = cleanTranscript ? extractTextFromVTT(scopedVtt) : scopedVtt;
|
|
2231
|
+
const hasScopedCues = !effectiveScope || parseVTTCues(scopedVtt).length > 0;
|
|
2232
|
+
if (required && (!transcriptText.trim() || !hasScopedCues)) {
|
|
2233
|
+
throw new MuxAiError(
|
|
2234
|
+
effectiveScope ? "Transcript has no cues in the requested scope." : "Transcript is empty.",
|
|
2235
|
+
{ type: "validation_error" }
|
|
2236
|
+
);
|
|
1970
2237
|
}
|
|
1971
2238
|
return { transcriptText, transcriptUrl, track };
|
|
1972
2239
|
} catch (error) {
|
|
@@ -1979,17 +2246,20 @@ async function fetchTranscriptForAsset(asset, playbackId, options = {}) {
|
|
|
1979
2246
|
}
|
|
1980
2247
|
|
|
1981
2248
|
// src/workflows/ask-questions.ts
|
|
1982
|
-
var
|
|
1983
|
-
|
|
1984
|
-
|
|
1985
|
-
|
|
1986
|
-
|
|
1987
|
-
|
|
2249
|
+
var ASK_QUESTIONS_MAX_QUESTION_LENGTH = 600;
|
|
2250
|
+
var ASK_QUESTIONS_DEFAULT_MAX_ANSWER_OPTION_LENGTH = 150;
|
|
2251
|
+
var ASK_QUESTIONS_DEFAULT_MAX_FREE_FORM_ANSWER_LENGTH = 500;
|
|
2252
|
+
var questionAnswerSchema = z3.object({
|
|
2253
|
+
question: z3.string().describe("The full text of the original question"),
|
|
2254
|
+
answer: z3.string().nullable(),
|
|
2255
|
+
confidence: z3.number(),
|
|
2256
|
+
reasoning: z3.string().max(1e3),
|
|
2257
|
+
skipped: z3.boolean()
|
|
1988
2258
|
});
|
|
1989
2259
|
var SKIP_SENTINEL = "__SKIPPED__";
|
|
1990
2260
|
function createAskQuestionsSchema(answerSchema) {
|
|
1991
|
-
return
|
|
1992
|
-
answers:
|
|
2261
|
+
return z3.object({
|
|
2262
|
+
answers: z3.array(
|
|
1993
2263
|
questionAnswerSchema.extend({
|
|
1994
2264
|
answer: answerSchema
|
|
1995
2265
|
})
|
|
@@ -2339,12 +2609,12 @@ function buildResponseSchemaForQuestions(normalizedQuestions, maxFreeFormAnswerL
|
|
|
2339
2609
|
SKIP_SENTINEL.length,
|
|
2340
2610
|
longestConstrained
|
|
2341
2611
|
);
|
|
2342
|
-
return createAskQuestionsSchema(
|
|
2612
|
+
return createAskQuestionsSchema(z3.string().min(1).max(schemaMax));
|
|
2343
2613
|
}
|
|
2344
2614
|
const allowed = Array.from(
|
|
2345
2615
|
new Set(normalizedQuestions.flatMap(allowedAnswersForQuestion))
|
|
2346
2616
|
);
|
|
2347
|
-
return createAskQuestionsSchema(
|
|
2617
|
+
return createAskQuestionsSchema(z3.enum([...allowed, SKIP_SENTINEL]));
|
|
2348
2618
|
}
|
|
2349
2619
|
async function analyzeQuestions({
|
|
2350
2620
|
provider,
|
|
@@ -2362,7 +2632,7 @@ async function analyzeQuestions({
|
|
|
2362
2632
|
normalizedQuestions,
|
|
2363
2633
|
maxFreeFormAnswerLength
|
|
2364
2634
|
);
|
|
2365
|
-
const response = await generateText({
|
|
2635
|
+
const response = await withContentPolicyErrorHandling(() => generateText({
|
|
2366
2636
|
model,
|
|
2367
2637
|
output: Output.object({ schema: responseSchema }),
|
|
2368
2638
|
experimental_telemetry: { isEnabled: true },
|
|
@@ -2379,11 +2649,12 @@ async function analyzeQuestions({
|
|
|
2379
2649
|
] : userPrompt
|
|
2380
2650
|
}
|
|
2381
2651
|
]
|
|
2382
|
-
});
|
|
2383
|
-
|
|
2652
|
+
}));
|
|
2653
|
+
const output = getGeneratedOutputWithContentPolicyHandling(response);
|
|
2654
|
+
if (!output) {
|
|
2384
2655
|
throw new Error("Ask-questions output missing");
|
|
2385
2656
|
}
|
|
2386
|
-
const parsed = responseSchema.parse(
|
|
2657
|
+
const parsed = responseSchema.parse(output);
|
|
2387
2658
|
const unexpectedRootKeys = detectUnexpectedKeysFromRawText(
|
|
2388
2659
|
response.text,
|
|
2389
2660
|
responseSchema.keyof().options
|
|
@@ -2405,7 +2676,8 @@ async function analyzeQuestions({
|
|
|
2405
2676
|
outputTokens: response.usage.outputTokens,
|
|
2406
2677
|
totalTokens: response.usage.totalTokens,
|
|
2407
2678
|
reasoningTokens: response.usage.reasoningTokens,
|
|
2408
|
-
cachedInputTokens: response.usage.cachedInputTokens
|
|
2679
|
+
cachedInputTokens: response.usage.cachedInputTokens,
|
|
2680
|
+
cacheWriteTokens: response.usage.inputTokenDetails?.cacheWriteTokens
|
|
2409
2681
|
},
|
|
2410
2682
|
unexpectedRootKeys,
|
|
2411
2683
|
unexpectedAnswerKeys
|
|
@@ -2424,7 +2696,6 @@ async function askQuestionsInternal(assetId, questions, options, collectedUsage)
|
|
|
2424
2696
|
if (!questions || questions.length === 0) {
|
|
2425
2697
|
throw new MuxAiError("At least one question must be provided.", { type: "validation_error" });
|
|
2426
2698
|
}
|
|
2427
|
-
const MAX_QUESTION_LENGTH = 500;
|
|
2428
2699
|
questions.forEach((q, idx) => {
|
|
2429
2700
|
if (!q.question || typeof q.question !== "string" || !q.question.trim()) {
|
|
2430
2701
|
throw new MuxAiError(
|
|
@@ -2432,15 +2703,14 @@ async function askQuestionsInternal(assetId, questions, options, collectedUsage)
|
|
|
2432
2703
|
{ type: "validation_error" }
|
|
2433
2704
|
);
|
|
2434
2705
|
}
|
|
2435
|
-
if (q.question.length >
|
|
2706
|
+
if (q.question.length > ASK_QUESTIONS_MAX_QUESTION_LENGTH) {
|
|
2436
2707
|
throw new MuxAiError(
|
|
2437
|
-
`Question at index ${idx} exceeds the ${
|
|
2708
|
+
`Question at index ${idx} exceeds the ${ASK_QUESTIONS_MAX_QUESTION_LENGTH}-character limit (received ${q.question.length}).`,
|
|
2438
2709
|
{ type: "validation_error" }
|
|
2439
2710
|
);
|
|
2440
2711
|
}
|
|
2441
2712
|
});
|
|
2442
|
-
const
|
|
2443
|
-
const maxAnswerOptionLength = options?.maxAnswerOptionLength ?? DEFAULT_MAX_ANSWER_OPTION_LENGTH;
|
|
2713
|
+
const maxAnswerOptionLength = options?.maxAnswerOptionLength ?? ASK_QUESTIONS_DEFAULT_MAX_ANSWER_OPTION_LENGTH;
|
|
2444
2714
|
if (!Number.isFinite(maxAnswerOptionLength) || maxAnswerOptionLength <= 0) {
|
|
2445
2715
|
throw new MuxAiError(
|
|
2446
2716
|
`maxAnswerOptionLength must be a positive number (received ${maxAnswerOptionLength}).`,
|
|
@@ -2459,8 +2729,7 @@ async function askQuestionsInternal(assetId, questions, options, collectedUsage)
|
|
|
2459
2729
|
}
|
|
2460
2730
|
}
|
|
2461
2731
|
});
|
|
2462
|
-
const
|
|
2463
|
-
const maxFreeFormAnswerLength = options?.maxFreeFormAnswerLength ?? DEFAULT_MAX_FREE_FORM_ANSWER_LENGTH;
|
|
2732
|
+
const maxFreeFormAnswerLength = options?.maxFreeFormAnswerLength ?? ASK_QUESTIONS_DEFAULT_MAX_FREE_FORM_ANSWER_LENGTH;
|
|
2464
2733
|
if (!Number.isFinite(maxFreeFormAnswerLength) || maxFreeFormAnswerLength <= 0) {
|
|
2465
2734
|
throw new MuxAiError(
|
|
2466
2735
|
`maxFreeFormAnswerLength must be a positive number (received ${maxFreeFormAnswerLength}).`,
|
|
@@ -2476,7 +2745,8 @@ async function askQuestionsInternal(assetId, questions, options, collectedUsage)
|
|
|
2476
2745
|
imageSubmissionMode = "url",
|
|
2477
2746
|
imageDownloadOptions,
|
|
2478
2747
|
storyboardWidth = 640,
|
|
2479
|
-
credentials
|
|
2748
|
+
credentials,
|
|
2749
|
+
scope
|
|
2480
2750
|
} = options ?? {};
|
|
2481
2751
|
const normalizedQuestions = questions.map((q, idx) => normalizeQuestion(q, idx));
|
|
2482
2752
|
const hasFreeForm = normalizedQuestions.some((q) => q.mode.kind === "freeForm");
|
|
@@ -2488,6 +2758,15 @@ async function askQuestionsInternal(assetId, questions, options, collectedUsage)
|
|
|
2488
2758
|
const { asset: assetData, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials);
|
|
2489
2759
|
const assetDurationSeconds = getAssetDurationSecondsFromAsset(assetData);
|
|
2490
2760
|
const isAudioOnly = isAudioOnlyAsset(assetData);
|
|
2761
|
+
const effectiveScope = hasWorkflowScopeBoundaries(scope) ? scope : void 0;
|
|
2762
|
+
const storyboardScope = isAudioOnly ? void 0 : resolveRenderableVideoScope(
|
|
2763
|
+
effectiveScope,
|
|
2764
|
+
assetDurationSeconds,
|
|
2765
|
+
getVideoTrackDurationSecondsFromAsset(assetData)
|
|
2766
|
+
);
|
|
2767
|
+
if (isAudioOnly && effectiveScope) {
|
|
2768
|
+
resolveWorkflowScope(effectiveScope, assetDurationSeconds);
|
|
2769
|
+
}
|
|
2491
2770
|
if (isAudioOnly && !includeTranscript) {
|
|
2492
2771
|
throw new MuxAiError(
|
|
2493
2772
|
"Audio-only assets require a transcript. Set includeTranscript: true and ensure the asset has a ready text track (captions/subtitles).",
|
|
@@ -2506,7 +2785,8 @@ async function askQuestionsInternal(assetId, questions, options, collectedUsage)
|
|
|
2506
2785
|
cleanTranscript,
|
|
2507
2786
|
shouldSign: policy === "signed",
|
|
2508
2787
|
credentials,
|
|
2509
|
-
required: isAudioOnly
|
|
2788
|
+
required: isAudioOnly,
|
|
2789
|
+
scope: effectiveScope
|
|
2510
2790
|
}) : void 0;
|
|
2511
2791
|
const transcriptText = transcriptResult?.transcriptText ?? "";
|
|
2512
2792
|
const userPrompt = buildUserPrompt({
|
|
@@ -2535,7 +2815,8 @@ async function askQuestionsInternal(assetId, questions, options, collectedUsage)
|
|
|
2535
2815
|
playbackId,
|
|
2536
2816
|
storyboardWidth,
|
|
2537
2817
|
policy === "signed",
|
|
2538
|
-
credentials
|
|
2818
|
+
credentials,
|
|
2819
|
+
storyboardScope
|
|
2539
2820
|
);
|
|
2540
2821
|
imageUrl = storyboardUrl;
|
|
2541
2822
|
if (imageSubmissionMode === "base64") {
|
|
@@ -2650,11 +2931,11 @@ async function askQuestionsInternal(assetId, questions, options, collectedUsage)
|
|
|
2650
2931
|
// src/workflows/burned-in-captions.ts
|
|
2651
2932
|
import { generateText as generateText2, Output as Output2 } from "ai";
|
|
2652
2933
|
import dedent3 from "dedent";
|
|
2653
|
-
import { z as
|
|
2654
|
-
var burnedInCaptionsSchema =
|
|
2655
|
-
hasBurnedInCaptions:
|
|
2656
|
-
confidence:
|
|
2657
|
-
detectedLanguage:
|
|
2934
|
+
import { z as z4 } from "zod";
|
|
2935
|
+
var burnedInCaptionsSchema = z4.object({
|
|
2936
|
+
hasBurnedInCaptions: z4.boolean(),
|
|
2937
|
+
confidence: z4.number(),
|
|
2938
|
+
detectedLanguage: z4.string().max(100).nullable()
|
|
2658
2939
|
});
|
|
2659
2940
|
var SYSTEM_PROMPT2 = promptDedent`
|
|
2660
2941
|
<role>
|
|
@@ -2764,7 +3045,7 @@ async function analyzeStoryboard({
|
|
|
2764
3045
|
}) {
|
|
2765
3046
|
"use step";
|
|
2766
3047
|
const model = await createLanguageModelFromConfig(provider, modelId, credentials);
|
|
2767
|
-
const response = await generateText2({
|
|
3048
|
+
const response = await withContentPolicyErrorHandling(() => generateText2({
|
|
2768
3049
|
model,
|
|
2769
3050
|
output: Output2.object({ schema: burnedInCaptionsSchema }),
|
|
2770
3051
|
experimental_telemetry: { isEnabled: true },
|
|
@@ -2781,22 +3062,24 @@ async function analyzeStoryboard({
|
|
|
2781
3062
|
]
|
|
2782
3063
|
}
|
|
2783
3064
|
]
|
|
2784
|
-
});
|
|
3065
|
+
}));
|
|
3066
|
+
const output = getGeneratedOutputWithContentPolicyHandling(response);
|
|
2785
3067
|
const unexpectedKeys = detectUnexpectedKeysFromRawText(
|
|
2786
3068
|
response.text,
|
|
2787
3069
|
burnedInCaptionsSchema.keyof().options
|
|
2788
3070
|
);
|
|
2789
3071
|
return {
|
|
2790
3072
|
result: {
|
|
2791
|
-
...
|
|
2792
|
-
confidence: Math.min(1, Math.max(0,
|
|
3073
|
+
...output,
|
|
3074
|
+
confidence: Math.min(1, Math.max(0, output.confidence))
|
|
2793
3075
|
},
|
|
2794
3076
|
usage: {
|
|
2795
3077
|
inputTokens: response.usage.inputTokens,
|
|
2796
3078
|
outputTokens: response.usage.outputTokens,
|
|
2797
3079
|
totalTokens: response.usage.totalTokens,
|
|
2798
3080
|
reasoningTokens: response.usage.reasoningTokens,
|
|
2799
|
-
cachedInputTokens: response.usage.cachedInputTokens
|
|
3081
|
+
cachedInputTokens: response.usage.cachedInputTokens,
|
|
3082
|
+
cacheWriteTokens: response.usage.inputTokenDetails?.cacheWriteTokens
|
|
2800
3083
|
},
|
|
2801
3084
|
unexpectedKeys
|
|
2802
3085
|
};
|
|
@@ -2818,6 +3101,7 @@ async function hasBurnedInCaptionsInternal(assetId, options, collectedUsage) {
|
|
|
2818
3101
|
imageDownloadOptions,
|
|
2819
3102
|
promptOverrides,
|
|
2820
3103
|
credentials,
|
|
3104
|
+
scope,
|
|
2821
3105
|
...config
|
|
2822
3106
|
} = options;
|
|
2823
3107
|
const userPrompt = buildUserPrompt2(promptOverrides);
|
|
@@ -2828,7 +3112,18 @@ async function hasBurnedInCaptionsInternal(assetId, options, collectedUsage) {
|
|
|
2828
3112
|
});
|
|
2829
3113
|
const { asset: assetData, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials);
|
|
2830
3114
|
const assetDurationSeconds = getAssetDurationSecondsFromAsset(assetData);
|
|
2831
|
-
const
|
|
3115
|
+
const storyboardScope = resolveRenderableVideoScope(
|
|
3116
|
+
scope,
|
|
3117
|
+
assetDurationSeconds,
|
|
3118
|
+
getVideoTrackDurationSecondsFromAsset(assetData)
|
|
3119
|
+
);
|
|
3120
|
+
const imageUrl = await getStoryboardUrl(
|
|
3121
|
+
playbackId,
|
|
3122
|
+
640,
|
|
3123
|
+
policy === "signed",
|
|
3124
|
+
credentials,
|
|
3125
|
+
storyboardScope
|
|
3126
|
+
);
|
|
2832
3127
|
let analysisResponse;
|
|
2833
3128
|
if (imageSubmissionMode === "base64") {
|
|
2834
3129
|
const base64Data = await fetchImageAsBase642(imageUrl, imageDownloadOptions);
|
|
@@ -2882,13 +3177,13 @@ async function hasBurnedInCaptionsInternal(assetId, options, collectedUsage) {
|
|
|
2882
3177
|
// src/workflows/chapters.ts
|
|
2883
3178
|
import { generateText as generateText3, Output as Output3 } from "ai";
|
|
2884
3179
|
import dedent4 from "dedent";
|
|
2885
|
-
import { z as
|
|
2886
|
-
var chapterSchema =
|
|
2887
|
-
startTime:
|
|
2888
|
-
title:
|
|
3180
|
+
import { z as z5 } from "zod";
|
|
3181
|
+
var chapterSchema = z5.object({
|
|
3182
|
+
startTime: z5.number(),
|
|
3183
|
+
title: z5.string().max(300)
|
|
2889
3184
|
});
|
|
2890
|
-
var chaptersSchema =
|
|
2891
|
-
chapters:
|
|
3185
|
+
var chaptersSchema = z5.object({
|
|
3186
|
+
chapters: z5.array(chapterSchema)
|
|
2892
3187
|
});
|
|
2893
3188
|
async function generateChaptersWithAI({
|
|
2894
3189
|
provider,
|
|
@@ -2900,21 +3195,24 @@ async function generateChaptersWithAI({
|
|
|
2900
3195
|
"use step";
|
|
2901
3196
|
const model = await createLanguageModelFromConfig(provider, modelId, credentials);
|
|
2902
3197
|
const response = await withRetry(
|
|
2903
|
-
() =>
|
|
2904
|
-
|
|
2905
|
-
|
|
2906
|
-
|
|
2907
|
-
|
|
2908
|
-
|
|
2909
|
-
|
|
2910
|
-
|
|
2911
|
-
|
|
2912
|
-
|
|
2913
|
-
|
|
2914
|
-
|
|
2915
|
-
|
|
2916
|
-
|
|
3198
|
+
() => withContentPolicyErrorHandling(
|
|
3199
|
+
() => generateText3({
|
|
3200
|
+
model,
|
|
3201
|
+
output: Output3.object({ schema: chaptersSchema }),
|
|
3202
|
+
messages: [
|
|
3203
|
+
{
|
|
3204
|
+
role: "system",
|
|
3205
|
+
content: systemPrompt
|
|
3206
|
+
},
|
|
3207
|
+
{
|
|
3208
|
+
role: "user",
|
|
3209
|
+
content: userPrompt
|
|
3210
|
+
}
|
|
3211
|
+
]
|
|
3212
|
+
})
|
|
3213
|
+
)
|
|
2917
3214
|
);
|
|
3215
|
+
const output = getGeneratedOutputWithContentPolicyHandling(response);
|
|
2918
3216
|
const unexpectedRootKeys = detectUnexpectedKeysFromRawText(
|
|
2919
3217
|
response.text,
|
|
2920
3218
|
chaptersSchema.keyof().options
|
|
@@ -2930,13 +3228,14 @@ async function generateChaptersWithAI({
|
|
|
2930
3228
|
} catch {
|
|
2931
3229
|
}
|
|
2932
3230
|
return {
|
|
2933
|
-
chapters:
|
|
3231
|
+
chapters: output,
|
|
2934
3232
|
usage: {
|
|
2935
3233
|
inputTokens: response.usage.inputTokens,
|
|
2936
3234
|
outputTokens: response.usage.outputTokens,
|
|
2937
3235
|
totalTokens: response.usage.totalTokens,
|
|
2938
3236
|
reasoningTokens: response.usage.reasoningTokens,
|
|
2939
|
-
cachedInputTokens: response.usage.cachedInputTokens
|
|
3237
|
+
cachedInputTokens: response.usage.cachedInputTokens,
|
|
3238
|
+
cacheWriteTokens: response.usage.inputTokenDetails?.cacheWriteTokens
|
|
2940
3239
|
},
|
|
2941
3240
|
unexpectedRootKeys,
|
|
2942
3241
|
unexpectedChapterKeys
|
|
@@ -3027,7 +3326,8 @@ function buildUserPrompt3({
|
|
|
3027
3326
|
promptOverrides,
|
|
3028
3327
|
minChaptersPerHour = 3,
|
|
3029
3328
|
maxChaptersPerHour = 8,
|
|
3030
|
-
languageName
|
|
3329
|
+
languageName,
|
|
3330
|
+
scope
|
|
3031
3331
|
}) {
|
|
3032
3332
|
const contextSections = [
|
|
3033
3333
|
{
|
|
@@ -3042,6 +3342,8 @@ function buildUserPrompt3({
|
|
|
3042
3342
|
const dynamicChapterGuidelines = dedent4`
|
|
3043
3343
|
- Create at least ${minChaptersPerHour} and at most ${maxChaptersPerHour} chapters per hour of content
|
|
3044
3344
|
- Use start times in seconds (not HH:MM:SS)
|
|
3345
|
+
${scope ? `- Analyze only the range from ${scope.startTime}s (inclusive) to ${scope.endTime}s (exclusive)
|
|
3346
|
+
- The first chapter must start at ${scope.startTime}s` : ""}
|
|
3045
3347
|
- Chapter start times should be non-decreasing
|
|
3046
3348
|
- Do not include text before or after the JSON`;
|
|
3047
3349
|
const mergedOverrides = {
|
|
@@ -3068,7 +3370,8 @@ async function generateChaptersInternal(assetId, options, collectedUsage) {
|
|
|
3068
3370
|
minChaptersPerHour,
|
|
3069
3371
|
maxChaptersPerHour,
|
|
3070
3372
|
credentials,
|
|
3071
|
-
outputLanguageCode
|
|
3373
|
+
outputLanguageCode,
|
|
3374
|
+
scope
|
|
3072
3375
|
} = options;
|
|
3073
3376
|
const modelConfig = resolveLanguageModelConfig({
|
|
3074
3377
|
...options,
|
|
@@ -3077,6 +3380,8 @@ async function generateChaptersInternal(assetId, options, collectedUsage) {
|
|
|
3077
3380
|
});
|
|
3078
3381
|
const { asset: assetData, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials);
|
|
3079
3382
|
const assetDurationSeconds = getAssetDurationSecondsFromAsset(assetData);
|
|
3383
|
+
const effectiveScope = hasWorkflowScopeBoundaries(scope) ? scope : void 0;
|
|
3384
|
+
const resolvedScope = effectiveScope ? resolveWorkflowScope(effectiveScope, assetDurationSeconds) : void 0;
|
|
3080
3385
|
const isAudioOnly = isAudioOnlyAsset(assetData);
|
|
3081
3386
|
const signingContext = await resolveMuxSigningContext(credentials);
|
|
3082
3387
|
if (policy === "signed" && !signingContext) {
|
|
@@ -3091,7 +3396,8 @@ async function generateChaptersInternal(assetId, options, collectedUsage) {
|
|
|
3091
3396
|
cleanTranscript: false,
|
|
3092
3397
|
// keep timestamps for chapter segmentation
|
|
3093
3398
|
shouldSign: policy === "signed",
|
|
3094
|
-
credentials
|
|
3399
|
+
credentials,
|
|
3400
|
+
scope: effectiveScope
|
|
3095
3401
|
});
|
|
3096
3402
|
if (!transcriptResult.track || !transcriptResult.transcriptText) {
|
|
3097
3403
|
const availableLanguages = readyTextTracks.map((t) => t.language_code).filter(Boolean).join(", ");
|
|
@@ -3112,7 +3418,8 @@ async function generateChaptersInternal(assetId, options, collectedUsage) {
|
|
|
3112
3418
|
promptOverrides,
|
|
3113
3419
|
minChaptersPerHour,
|
|
3114
3420
|
maxChaptersPerHour,
|
|
3115
|
-
languageName
|
|
3421
|
+
languageName,
|
|
3422
|
+
scope: resolvedScope
|
|
3116
3423
|
});
|
|
3117
3424
|
let chaptersData = null;
|
|
3118
3425
|
try {
|
|
@@ -3134,7 +3441,9 @@ async function generateChaptersInternal(assetId, options, collectedUsage) {
|
|
|
3134
3441
|
throw new MuxAiError(`Failed to generate chapters for asset ${assetId}.`);
|
|
3135
3442
|
}
|
|
3136
3443
|
const { chapters: chaptersPayload, usage } = chaptersData;
|
|
3137
|
-
const validChapters = chaptersPayload.chapters.filter((chapter) => typeof chapter.startTime === "number" && typeof chapter.title === "string").
|
|
3444
|
+
const validChapters = chaptersPayload.chapters.filter((chapter) => typeof chapter.startTime === "number" && typeof chapter.title === "string").filter(
|
|
3445
|
+
(chapter) => resolvedScope ? chapter.startTime >= resolvedScope.startTime && chapter.startTime < resolvedScope.endTime : true
|
|
3446
|
+
).sort((a, b) => a.startTime - b.startTime);
|
|
3138
3447
|
if (validChapters.length === 0) {
|
|
3139
3448
|
throw new MuxAiError(`Failed to generate valid chapters for asset ${assetId}.`);
|
|
3140
3449
|
}
|
|
@@ -3160,8 +3469,9 @@ async function generateChaptersInternal(assetId, options, collectedUsage) {
|
|
|
3160
3469
|
if (scrubbedChapters.length === 0) {
|
|
3161
3470
|
throw new MuxAiError(`Failed to generate valid chapters for asset ${assetId}.`);
|
|
3162
3471
|
}
|
|
3163
|
-
|
|
3164
|
-
|
|
3472
|
+
const firstChapterStartTime = resolvedScope?.startTime ?? 0;
|
|
3473
|
+
if (scrubbedChapters[0].startTime !== firstChapterStartTime) {
|
|
3474
|
+
scrubbedChapters[0].startTime = firstChapterStartTime;
|
|
3165
3475
|
}
|
|
3166
3476
|
const usageWithMetadata = {
|
|
3167
3477
|
...usage,
|
|
@@ -3181,7 +3491,7 @@ async function generateChaptersInternal(assetId, options, collectedUsage) {
|
|
|
3181
3491
|
|
|
3182
3492
|
// src/workflows/edit-captions.ts
|
|
3183
3493
|
import { generateText as generateText4, Output as Output4 } from "ai";
|
|
3184
|
-
import { z as
|
|
3494
|
+
import { z as z6 } from "zod";
|
|
3185
3495
|
|
|
3186
3496
|
// src/lib/mux-tracks.ts
|
|
3187
3497
|
async function fetchVttFromMux(vttUrl) {
|
|
@@ -3464,8 +3774,8 @@ async function createPresignedGetUrlWithStorageAdapter(input, adapter) {
|
|
|
3464
3774
|
}
|
|
3465
3775
|
|
|
3466
3776
|
// src/workflows/edit-captions.ts
|
|
3467
|
-
var profanityDetectionSchema =
|
|
3468
|
-
profanity:
|
|
3777
|
+
var profanityDetectionSchema = z6.object({
|
|
3778
|
+
profanity: z6.array(z6.string().max(100)).describe(
|
|
3469
3779
|
"Unique profane words or short phrases exactly as they appear in the transcript text. Include each distinct form only once (e.g., if 'fuck' and 'fucking' both appear, list both)."
|
|
3470
3780
|
)
|
|
3471
3781
|
});
|
|
@@ -3642,7 +3952,7 @@ async function identifyProfanityWithAI({
|
|
|
3642
3952
|
tag: "transcript",
|
|
3643
3953
|
content: plainText
|
|
3644
3954
|
});
|
|
3645
|
-
const response = await generateText4({
|
|
3955
|
+
const response = await withContentPolicyErrorHandling(() => generateText4({
|
|
3646
3956
|
model,
|
|
3647
3957
|
output: Output4.object({ schema: profanityDetectionSchema }),
|
|
3648
3958
|
messages: [
|
|
@@ -3657,19 +3967,21 @@ async function identifyProfanityWithAI({
|
|
|
3657
3967
|
${transcriptSection}`
|
|
3658
3968
|
}
|
|
3659
3969
|
]
|
|
3660
|
-
});
|
|
3970
|
+
}));
|
|
3971
|
+
const output = getGeneratedOutputWithContentPolicyHandling(response);
|
|
3661
3972
|
const unexpectedKeys = detectUnexpectedKeysFromRawText(
|
|
3662
3973
|
response.text,
|
|
3663
3974
|
profanityDetectionSchema.keyof().options
|
|
3664
3975
|
);
|
|
3665
3976
|
return {
|
|
3666
|
-
profanity:
|
|
3977
|
+
profanity: output.profanity,
|
|
3667
3978
|
usage: {
|
|
3668
3979
|
inputTokens: response.usage.inputTokens,
|
|
3669
3980
|
outputTokens: response.usage.outputTokens,
|
|
3670
3981
|
totalTokens: response.usage.totalTokens,
|
|
3671
3982
|
reasoningTokens: response.usage.reasoningTokens,
|
|
3672
|
-
cachedInputTokens: response.usage.cachedInputTokens
|
|
3983
|
+
cachedInputTokens: response.usage.cachedInputTokens,
|
|
3984
|
+
cacheWriteTokens: response.usage.inputTokenDetails?.cacheWriteTokens
|
|
3673
3985
|
},
|
|
3674
3986
|
unexpectedKeys
|
|
3675
3987
|
};
|
|
@@ -4192,11 +4504,16 @@ async function generateEmbeddingsInternal(assetId, options, collectedUsage) {
|
|
|
4192
4504
|
languageCode,
|
|
4193
4505
|
chunkingStrategy = { type: "token", maxTokens: 500, overlap: 100 },
|
|
4194
4506
|
batchSize = 5,
|
|
4195
|
-
credentials
|
|
4507
|
+
credentials,
|
|
4508
|
+
scope
|
|
4196
4509
|
} = options;
|
|
4197
4510
|
const embeddingModel = resolveEmbeddingModelConfig({ ...options, provider, model });
|
|
4198
4511
|
const { asset: assetData, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials);
|
|
4199
4512
|
const assetDurationSeconds = getAssetDurationSecondsFromAsset(assetData);
|
|
4513
|
+
const effectiveScope = hasWorkflowScopeBoundaries(scope) ? scope : void 0;
|
|
4514
|
+
if (effectiveScope) {
|
|
4515
|
+
resolveWorkflowScope(effectiveScope, assetDurationSeconds);
|
|
4516
|
+
}
|
|
4200
4517
|
const signingContext = await resolveMuxSigningContext(credentials);
|
|
4201
4518
|
if (policy === "signed" && !signingContext) {
|
|
4202
4519
|
throw new Error(
|
|
@@ -4209,7 +4526,8 @@ async function generateEmbeddingsInternal(assetId, options, collectedUsage) {
|
|
|
4209
4526
|
cleanTranscript: !useVttChunking,
|
|
4210
4527
|
shouldSign: policy === "signed",
|
|
4211
4528
|
credentials,
|
|
4212
|
-
required: true
|
|
4529
|
+
required: true,
|
|
4530
|
+
scope: effectiveScope
|
|
4213
4531
|
});
|
|
4214
4532
|
const transcriptText = transcriptResult.transcriptText;
|
|
4215
4533
|
const chunks = useVttChunking ? chunkVTTCues(
|
|
@@ -4306,7 +4624,7 @@ async function generateVideoEmbeddings(assetId, options = {}) {
|
|
|
4306
4624
|
// src/workflows/engagement-insights.ts
|
|
4307
4625
|
import { generateText as generateText5, Output as Output5 } from "ai";
|
|
4308
4626
|
import dedent5 from "dedent";
|
|
4309
|
-
import { z as
|
|
4627
|
+
import { z as z7 } from "zod";
|
|
4310
4628
|
|
|
4311
4629
|
// src/primitives/heatmap.ts
|
|
4312
4630
|
async function getHeatmapForAsset(assetId, options = {}) {
|
|
@@ -4453,20 +4771,20 @@ async function getShotsForAsset(assetId, options = {}) {
|
|
|
4453
4771
|
}
|
|
4454
4772
|
|
|
4455
4773
|
// src/workflows/engagement-insights.ts
|
|
4456
|
-
var aiMomentInsightSchema =
|
|
4457
|
-
hotspotIndex:
|
|
4458
|
-
startMs:
|
|
4459
|
-
endMs:
|
|
4460
|
-
timestamp:
|
|
4461
|
-
engagementScore:
|
|
4462
|
-
insight:
|
|
4774
|
+
var aiMomentInsightSchema = z7.object({
|
|
4775
|
+
hotspotIndex: z7.number(),
|
|
4776
|
+
startMs: z7.number(),
|
|
4777
|
+
endMs: z7.number(),
|
|
4778
|
+
timestamp: z7.string(),
|
|
4779
|
+
engagementScore: z7.number(),
|
|
4780
|
+
insight: z7.string().max(1e3)
|
|
4463
4781
|
});
|
|
4464
|
-
var aiOverallInsightSchema =
|
|
4465
|
-
summary:
|
|
4466
|
-
trends:
|
|
4782
|
+
var aiOverallInsightSchema = z7.object({
|
|
4783
|
+
summary: z7.string().max(2e3),
|
|
4784
|
+
trends: z7.array(z7.string().max(500))
|
|
4467
4785
|
});
|
|
4468
|
-
var engagementInsightsSchema =
|
|
4469
|
-
momentInsights:
|
|
4786
|
+
var engagementInsightsSchema = z7.object({
|
|
4787
|
+
momentInsights: z7.array(aiMomentInsightSchema),
|
|
4470
4788
|
overallInsight: aiOverallInsightSchema
|
|
4471
4789
|
});
|
|
4472
4790
|
var SYSTEM_PROMPT4 = promptDedent`
|
|
@@ -4728,26 +5046,29 @@ async function generateInsightsWithAI(provider, modelId, systemPrompt, userPromp
|
|
|
4728
5046
|
"use step";
|
|
4729
5047
|
const model = await createLanguageModelFromConfig(provider, modelId, credentials);
|
|
4730
5048
|
const response = await withRetry(
|
|
4731
|
-
() =>
|
|
4732
|
-
|
|
4733
|
-
|
|
4734
|
-
|
|
4735
|
-
|
|
4736
|
-
|
|
4737
|
-
|
|
4738
|
-
|
|
4739
|
-
|
|
4740
|
-
|
|
4741
|
-
|
|
4742
|
-
|
|
4743
|
-
|
|
4744
|
-
|
|
4745
|
-
|
|
4746
|
-
|
|
4747
|
-
|
|
4748
|
-
|
|
5049
|
+
() => withContentPolicyErrorHandling(
|
|
5050
|
+
() => generateText5({
|
|
5051
|
+
model,
|
|
5052
|
+
output: Output5.object({ schema: engagementInsightsSchema }),
|
|
5053
|
+
experimental_telemetry: { isEnabled: true },
|
|
5054
|
+
messages: [
|
|
5055
|
+
{
|
|
5056
|
+
role: "system",
|
|
5057
|
+
content: systemPrompt
|
|
5058
|
+
},
|
|
5059
|
+
{
|
|
5060
|
+
role: "user",
|
|
5061
|
+
content: [
|
|
5062
|
+
{ type: "text", text: userPrompt },
|
|
5063
|
+
...imageUrls.map((url) => ({ type: "image", image: url }))
|
|
5064
|
+
]
|
|
5065
|
+
}
|
|
5066
|
+
]
|
|
5067
|
+
})
|
|
5068
|
+
)
|
|
4749
5069
|
);
|
|
4750
|
-
|
|
5070
|
+
const output = getGeneratedOutputWithContentPolicyHandling(response);
|
|
5071
|
+
if (!output) {
|
|
4751
5072
|
throw new Error("AI returned empty or unparseable response");
|
|
4752
5073
|
}
|
|
4753
5074
|
const unexpectedRootKeys = detectUnexpectedKeysFromRawText(
|
|
@@ -4770,13 +5091,14 @@ async function generateInsightsWithAI(provider, modelId, systemPrompt, userPromp
|
|
|
4770
5091
|
} catch {
|
|
4771
5092
|
}
|
|
4772
5093
|
return {
|
|
4773
|
-
result:
|
|
5094
|
+
result: output,
|
|
4774
5095
|
usage: {
|
|
4775
5096
|
inputTokens: response.usage.inputTokens,
|
|
4776
5097
|
outputTokens: response.usage.outputTokens,
|
|
4777
5098
|
totalTokens: response.usage.totalTokens,
|
|
4778
5099
|
reasoningTokens: response.usage.reasoningTokens,
|
|
4779
|
-
cachedInputTokens: response.usage.cachedInputTokens
|
|
5100
|
+
cachedInputTokens: response.usage.cachedInputTokens,
|
|
5101
|
+
cacheWriteTokens: response.usage.inputTokenDetails?.cacheWriteTokens
|
|
4780
5102
|
},
|
|
4781
5103
|
unexpectedRootKeys,
|
|
4782
5104
|
unexpectedMomentKeys,
|
|
@@ -5048,27 +5370,32 @@ function planSamplingTimestamps(options) {
|
|
|
5048
5370
|
// src/primitives/thumbnails.ts
|
|
5049
5371
|
async function getThumbnailUrls(playbackId, duration, options = {}) {
|
|
5050
5372
|
"use step";
|
|
5051
|
-
const { interval = 10, width = 640, shouldSign = false, maxSamples, credentials } = options;
|
|
5373
|
+
const { interval = 10, width = 640, shouldSign = false, maxSamples, credentials, scope } = options;
|
|
5374
|
+
const effectiveScope = hasWorkflowScopeBoundaries(scope) ? scope : void 0;
|
|
5375
|
+
const resolvedScope = effectiveScope ? resolveWorkflowScope(effectiveScope, duration) : { startTime: 0, endTime: duration };
|
|
5376
|
+
const rangeDuration = resolvedScope.endTime - resolvedScope.startTime;
|
|
5052
5377
|
let timestamps = [];
|
|
5053
|
-
if (
|
|
5054
|
-
const spacing =
|
|
5378
|
+
if (rangeDuration <= 50) {
|
|
5379
|
+
const spacing = rangeDuration / 6;
|
|
5055
5380
|
for (let i = 1; i <= 5; i++) {
|
|
5056
|
-
|
|
5381
|
+
const time = resolvedScope.startTime + i * spacing;
|
|
5382
|
+
timestamps.push(effectiveScope ? Number(time.toFixed(3)) : Math.round(time));
|
|
5057
5383
|
}
|
|
5058
5384
|
} else {
|
|
5059
|
-
for (let time =
|
|
5385
|
+
for (let time = resolvedScope.startTime; time < resolvedScope.endTime; time += interval) {
|
|
5060
5386
|
timestamps.push(time);
|
|
5061
5387
|
}
|
|
5062
5388
|
}
|
|
5063
5389
|
if (maxSamples !== void 0 && timestamps.length > maxSamples) {
|
|
5064
5390
|
const newTimestamps = [];
|
|
5065
|
-
newTimestamps.push(
|
|
5391
|
+
newTimestamps.push(resolvedScope.startTime);
|
|
5066
5392
|
if (maxSamples >= 2) {
|
|
5067
|
-
const
|
|
5393
|
+
const lastTime = effectiveScope ? Math.max(resolvedScope.startTime, resolvedScope.endTime - 1e-3) : resolvedScope.endTime;
|
|
5394
|
+
const spacing = (lastTime - resolvedScope.startTime) / (maxSamples - 1);
|
|
5068
5395
|
for (let i = 1; i < maxSamples - 1; i++) {
|
|
5069
|
-
newTimestamps.push(spacing * i);
|
|
5396
|
+
newTimestamps.push(resolvedScope.startTime + spacing * i);
|
|
5070
5397
|
}
|
|
5071
|
-
newTimestamps.push(
|
|
5398
|
+
newTimestamps.push(lastTime);
|
|
5072
5399
|
}
|
|
5073
5400
|
timestamps = newTimestamps;
|
|
5074
5401
|
}
|
|
@@ -5521,12 +5848,24 @@ async function requestGoogleVisionModeration(images, maxConcurrent = 5, submissi
|
|
|
5521
5848
|
}));
|
|
5522
5849
|
return processConcurrently(targets, moderateImageWithGoogleVision, maxConcurrent);
|
|
5523
5850
|
}
|
|
5851
|
+
function toThumbnailTimeSeconds(timestampMs) {
|
|
5852
|
+
return Number((timestampMs / 1e3).toFixed(2));
|
|
5853
|
+
}
|
|
5854
|
+
function filterTimestampsWithinExclusiveScope(timestampsMs, scope) {
|
|
5855
|
+
if (!scope) {
|
|
5856
|
+
return timestampsMs;
|
|
5857
|
+
}
|
|
5858
|
+
return timestampsMs.filter((timestampMs) => {
|
|
5859
|
+
const time = toThumbnailTimeSeconds(timestampMs);
|
|
5860
|
+
return time >= scope.startTime && time < scope.endTime;
|
|
5861
|
+
});
|
|
5862
|
+
}
|
|
5524
5863
|
async function getThumbnailUrlsFromTimestamps(playbackId, timestampsMs, options) {
|
|
5525
5864
|
"use step";
|
|
5526
5865
|
const { width, shouldSign, credentials } = options;
|
|
5527
5866
|
const baseUrl = getMuxThumbnailBaseUrl(playbackId);
|
|
5528
5867
|
const urlPromises = timestampsMs.map(async (tsMs) => {
|
|
5529
|
-
const time =
|
|
5868
|
+
const time = toThumbnailTimeSeconds(tsMs);
|
|
5530
5869
|
const url = shouldSign ? await signUrl(baseUrl, playbackId, "thumbnail", { time, width }, credentials) : `${baseUrl}?time=${time}&width=${width}`;
|
|
5531
5870
|
return { url, time };
|
|
5532
5871
|
});
|
|
@@ -5545,7 +5884,8 @@ async function getModerationScores(assetId, options = {}) {
|
|
|
5545
5884
|
maxConcurrent = 5,
|
|
5546
5885
|
imageSubmissionMode = "url",
|
|
5547
5886
|
imageDownloadOptions,
|
|
5548
|
-
credentials: providedCredentials
|
|
5887
|
+
credentials: providedCredentials,
|
|
5888
|
+
scope
|
|
5549
5889
|
} = options;
|
|
5550
5890
|
const credentials = providedCredentials;
|
|
5551
5891
|
const { asset, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials);
|
|
@@ -5556,7 +5896,16 @@ async function getModerationScores(assetId, options = {}) {
|
|
|
5556
5896
|
(d) => d != null
|
|
5557
5897
|
);
|
|
5558
5898
|
const duration = candidateDurations.length > 0 ? Math.min(...candidateDurations) : 0;
|
|
5899
|
+
const effectiveScope = hasWorkflowScopeBoundaries(scope) ? scope : void 0;
|
|
5900
|
+
const resolvedScope = effectiveScope ? resolveWorkflowScope(effectiveScope, assetDurationSeconds) : void 0;
|
|
5559
5901
|
const isAudioOnly = isAudioOnlyAsset(asset);
|
|
5902
|
+
const renderableScope = resolvedScope ? {
|
|
5903
|
+
startTime: Math.min(resolvedScope.startTime, duration),
|
|
5904
|
+
endTime: Math.min(resolvedScope.endTime, duration)
|
|
5905
|
+
} : void 0;
|
|
5906
|
+
if (!isAudioOnly && renderableScope && renderableScope.startTime >= renderableScope.endTime) {
|
|
5907
|
+
throw new Error("The requested scope does not include any renderable video.");
|
|
5908
|
+
}
|
|
5560
5909
|
const signingContext = await resolveMuxSigningContext(credentials);
|
|
5561
5910
|
if (policy === "signed" && !signingContext) {
|
|
5562
5911
|
throw new Error(
|
|
@@ -5573,7 +5922,8 @@ async function getModerationScores(assetId, options = {}) {
|
|
|
5573
5922
|
cleanTranscript: true,
|
|
5574
5923
|
shouldSign: policy === "signed",
|
|
5575
5924
|
credentials,
|
|
5576
|
-
required: true
|
|
5925
|
+
required: true,
|
|
5926
|
+
scope: effectiveScope
|
|
5577
5927
|
});
|
|
5578
5928
|
if (provider === "openai") {
|
|
5579
5929
|
thumbnailScores = await requestOpenAITranscriptModeration(
|
|
@@ -5591,17 +5941,21 @@ async function getModerationScores(assetId, options = {}) {
|
|
|
5591
5941
|
throw new Error(`Unsupported moderation provider: ${exhaustiveCheck}`);
|
|
5592
5942
|
}
|
|
5593
5943
|
} else {
|
|
5594
|
-
const
|
|
5944
|
+
const scopedDuration = renderableScope ? renderableScope.endTime - renderableScope.startTime : duration;
|
|
5945
|
+
const estimatedIntervalCount = scopedDuration <= 50 ? 5 : Math.ceil(scopedDuration / thumbnailInterval);
|
|
5595
5946
|
const thumbnailUrls = maxSamples !== void 0 && estimatedIntervalCount > maxSamples ? await getThumbnailUrlsFromTimestamps(
|
|
5596
5947
|
playbackId,
|
|
5597
|
-
|
|
5598
|
-
|
|
5599
|
-
|
|
5600
|
-
|
|
5601
|
-
|
|
5602
|
-
|
|
5603
|
-
|
|
5604
|
-
|
|
5948
|
+
filterTimestampsWithinExclusiveScope(
|
|
5949
|
+
planSamplingTimestamps({
|
|
5950
|
+
duration_sec: duration,
|
|
5951
|
+
max_candidates: maxSamples,
|
|
5952
|
+
trim_start_sec: renderableScope?.startTime ?? (duration > 2 ? Math.min(5, Math.max(1, duration / 6)) : 0),
|
|
5953
|
+
trim_end_sec: renderableScope ? duration - renderableScope.endTime : duration > 2 ? Math.min(5, Math.max(1, duration / 6)) : 0,
|
|
5954
|
+
fps: videoTrackFps,
|
|
5955
|
+
base_cadence_hz: thumbnailInterval > 0 ? 1 / thumbnailInterval : void 0
|
|
5956
|
+
}),
|
|
5957
|
+
renderableScope
|
|
5958
|
+
),
|
|
5605
5959
|
{
|
|
5606
5960
|
width: thumbnailWidth,
|
|
5607
5961
|
shouldSign: policy === "signed",
|
|
@@ -5611,7 +5965,8 @@ async function getModerationScores(assetId, options = {}) {
|
|
|
5611
5965
|
interval: thumbnailInterval,
|
|
5612
5966
|
width: thumbnailWidth,
|
|
5613
5967
|
shouldSign: policy === "signed",
|
|
5614
|
-
credentials
|
|
5968
|
+
credentials,
|
|
5969
|
+
scope: renderableScope
|
|
5615
5970
|
});
|
|
5616
5971
|
thumbnailCount = thumbnailUrls.length;
|
|
5617
5972
|
if (provider === "openai") {
|
|
@@ -5697,7 +6052,7 @@ async function getModerationScores(assetId, options = {}) {
|
|
|
5697
6052
|
// src/workflows/summarization.ts
|
|
5698
6053
|
import { generateText as generateText6, Output as Output6 } from "ai";
|
|
5699
6054
|
import dedent6 from "dedent";
|
|
5700
|
-
import { z as
|
|
6055
|
+
import { z as z8 } from "zod";
|
|
5701
6056
|
|
|
5702
6057
|
// src/lib/markdown-normalization.ts
|
|
5703
6058
|
var ESCAPED_LINE_BREAK_REGEX = /\\r\\n|\\n|\\r/g;
|
|
@@ -5710,17 +6065,17 @@ function normalizeMarkdownDescription(description) {
|
|
|
5710
6065
|
var DEFAULT_SUMMARY_KEYWORD_LIMIT = 10;
|
|
5711
6066
|
var DEFAULT_TITLE_LENGTH = 10;
|
|
5712
6067
|
var DEFAULT_DESCRIPTION_LENGTH = 50;
|
|
5713
|
-
var summarySchema =
|
|
5714
|
-
keywords:
|
|
5715
|
-
title:
|
|
5716
|
-
description:
|
|
6068
|
+
var summarySchema = z8.object({
|
|
6069
|
+
keywords: z8.array(z8.string().max(200)),
|
|
6070
|
+
title: z8.string().max(500),
|
|
6071
|
+
description: z8.string().max(1e4)
|
|
5717
6072
|
});
|
|
5718
6073
|
function buildSummarySchema(descriptionLength) {
|
|
5719
6074
|
const descriptionMaxChars = Math.max(2e3, descriptionLength * 15);
|
|
5720
|
-
return
|
|
5721
|
-
keywords:
|
|
5722
|
-
title:
|
|
5723
|
-
description:
|
|
6075
|
+
return z8.object({
|
|
6076
|
+
keywords: z8.array(z8.string().max(200)),
|
|
6077
|
+
title: z8.string().max(500),
|
|
6078
|
+
description: z8.string().max(descriptionMaxChars)
|
|
5724
6079
|
});
|
|
5725
6080
|
}
|
|
5726
6081
|
var VALID_TONES = ["neutral", "playful", "professional"];
|
|
@@ -5996,7 +6351,7 @@ async function analyzeStoryboard2(imageDataUrl, provider, modelId, userPrompt, s
|
|
|
5996
6351
|
"use step";
|
|
5997
6352
|
const model = await createLanguageModelFromConfig(provider, modelId, credentials);
|
|
5998
6353
|
const schema = buildSummarySchema(descriptionLength);
|
|
5999
|
-
const response = await generateText6({
|
|
6354
|
+
const response = await withContentPolicyErrorHandling(() => generateText6({
|
|
6000
6355
|
model,
|
|
6001
6356
|
output: Output6.object({
|
|
6002
6357
|
name: "summary_metadata",
|
|
@@ -6016,11 +6371,12 @@ async function analyzeStoryboard2(imageDataUrl, provider, modelId, userPrompt, s
|
|
|
6016
6371
|
]
|
|
6017
6372
|
}
|
|
6018
6373
|
]
|
|
6019
|
-
});
|
|
6020
|
-
|
|
6374
|
+
}));
|
|
6375
|
+
const output = getGeneratedOutputWithContentPolicyHandling(response);
|
|
6376
|
+
if (!output) {
|
|
6021
6377
|
throw new Error("Summarization output missing");
|
|
6022
6378
|
}
|
|
6023
|
-
const parsed = schema.parse(
|
|
6379
|
+
const parsed = schema.parse(output);
|
|
6024
6380
|
const unexpectedKeys = detectUnexpectedKeysFromRawText(
|
|
6025
6381
|
response.text,
|
|
6026
6382
|
schema.keyof().options
|
|
@@ -6032,7 +6388,8 @@ async function analyzeStoryboard2(imageDataUrl, provider, modelId, userPrompt, s
|
|
|
6032
6388
|
outputTokens: response.usage.outputTokens,
|
|
6033
6389
|
totalTokens: response.usage.totalTokens,
|
|
6034
6390
|
reasoningTokens: response.usage.reasoningTokens,
|
|
6035
|
-
cachedInputTokens: response.usage.cachedInputTokens
|
|
6391
|
+
cachedInputTokens: response.usage.cachedInputTokens,
|
|
6392
|
+
cacheWriteTokens: response.usage.inputTokenDetails?.cacheWriteTokens
|
|
6036
6393
|
},
|
|
6037
6394
|
unexpectedKeys
|
|
6038
6395
|
};
|
|
@@ -6041,7 +6398,7 @@ async function analyzeAudioOnly(provider, modelId, userPrompt, systemPrompt, des
|
|
|
6041
6398
|
"use step";
|
|
6042
6399
|
const model = await createLanguageModelFromConfig(provider, modelId, credentials);
|
|
6043
6400
|
const schema = buildSummarySchema(descriptionLength);
|
|
6044
|
-
const response = await generateText6({
|
|
6401
|
+
const response = await withContentPolicyErrorHandling(() => generateText6({
|
|
6045
6402
|
model,
|
|
6046
6403
|
output: Output6.object({
|
|
6047
6404
|
name: "summary_metadata",
|
|
@@ -6058,11 +6415,12 @@ async function analyzeAudioOnly(provider, modelId, userPrompt, systemPrompt, des
|
|
|
6058
6415
|
content: userPrompt
|
|
6059
6416
|
}
|
|
6060
6417
|
]
|
|
6061
|
-
});
|
|
6062
|
-
|
|
6418
|
+
}));
|
|
6419
|
+
const output = getGeneratedOutputWithContentPolicyHandling(response);
|
|
6420
|
+
if (!output) {
|
|
6063
6421
|
throw new Error("Summarization output missing");
|
|
6064
6422
|
}
|
|
6065
|
-
const parsed = schema.parse(
|
|
6423
|
+
const parsed = schema.parse(output);
|
|
6066
6424
|
const unexpectedKeys = detectUnexpectedKeysFromRawText(
|
|
6067
6425
|
response.text,
|
|
6068
6426
|
schema.keyof().options
|
|
@@ -6074,7 +6432,8 @@ async function analyzeAudioOnly(provider, modelId, userPrompt, systemPrompt, des
|
|
|
6074
6432
|
outputTokens: response.usage.outputTokens,
|
|
6075
6433
|
totalTokens: response.usage.totalTokens,
|
|
6076
6434
|
reasoningTokens: response.usage.reasoningTokens,
|
|
6077
|
-
cachedInputTokens: response.usage.cachedInputTokens
|
|
6435
|
+
cachedInputTokens: response.usage.cachedInputTokens,
|
|
6436
|
+
cacheWriteTokens: response.usage.inputTokenDetails?.cacheWriteTokens
|
|
6078
6437
|
},
|
|
6079
6438
|
unexpectedKeys
|
|
6080
6439
|
};
|
|
@@ -6126,7 +6485,8 @@ async function getSummaryAndTagsInternal(assetId, options, collectedUsage) {
|
|
|
6126
6485
|
titleLength,
|
|
6127
6486
|
descriptionLength,
|
|
6128
6487
|
tagCount,
|
|
6129
|
-
outputLanguageCode
|
|
6488
|
+
outputLanguageCode,
|
|
6489
|
+
scope
|
|
6130
6490
|
} = options ?? {};
|
|
6131
6491
|
if (!VALID_TONES.includes(tone)) {
|
|
6132
6492
|
throw new MuxAiError(
|
|
@@ -6143,6 +6503,15 @@ async function getSummaryAndTagsInternal(assetId, options, collectedUsage) {
|
|
|
6143
6503
|
const { asset: assetData, playbackId, policy } = await getPlaybackIdForAsset(assetId, workflowCredentials);
|
|
6144
6504
|
const assetDurationSeconds = getAssetDurationSecondsFromAsset(assetData);
|
|
6145
6505
|
const isAudioOnly = isAudioOnlyAsset(assetData);
|
|
6506
|
+
const effectiveScope = hasWorkflowScopeBoundaries(scope) ? scope : void 0;
|
|
6507
|
+
const storyboardScope = isAudioOnly ? void 0 : resolveRenderableVideoScope(
|
|
6508
|
+
effectiveScope,
|
|
6509
|
+
assetDurationSeconds,
|
|
6510
|
+
getVideoTrackDurationSecondsFromAsset(assetData)
|
|
6511
|
+
);
|
|
6512
|
+
if (isAudioOnly && effectiveScope) {
|
|
6513
|
+
resolveWorkflowScope(effectiveScope, assetDurationSeconds);
|
|
6514
|
+
}
|
|
6146
6515
|
if (isAudioOnly && !includeTranscript) {
|
|
6147
6516
|
throw new MuxAiError(
|
|
6148
6517
|
"Audio-only assets require a transcript. Set includeTranscript: true and ensure the asset has a ready text track (captions/subtitles).",
|
|
@@ -6161,7 +6530,8 @@ async function getSummaryAndTagsInternal(assetId, options, collectedUsage) {
|
|
|
6161
6530
|
cleanTranscript,
|
|
6162
6531
|
shouldSign: policy === "signed",
|
|
6163
6532
|
credentials: workflowCredentials,
|
|
6164
|
-
required: isAudioOnly
|
|
6533
|
+
required: isAudioOnly,
|
|
6534
|
+
scope: effectiveScope
|
|
6165
6535
|
}) : void 0;
|
|
6166
6536
|
const transcriptText = transcriptResult?.transcriptText ?? "";
|
|
6167
6537
|
const resolvedLanguageCode = outputLanguageCode && outputLanguageCode !== "auto" ? outputLanguageCode : getReliableLanguageCode(transcriptResult?.track) ?? getReliableLanguageCode(getReadyTextTracks(assetData)[0]);
|
|
@@ -6192,7 +6562,13 @@ async function getSummaryAndTagsInternal(assetId, options, collectedUsage) {
|
|
|
6192
6562
|
workflowCredentials
|
|
6193
6563
|
);
|
|
6194
6564
|
} else {
|
|
6195
|
-
const storyboardUrl = await getStoryboardUrl(
|
|
6565
|
+
const storyboardUrl = await getStoryboardUrl(
|
|
6566
|
+
playbackId,
|
|
6567
|
+
640,
|
|
6568
|
+
policy === "signed",
|
|
6569
|
+
workflowCredentials,
|
|
6570
|
+
storyboardScope
|
|
6571
|
+
);
|
|
6196
6572
|
imageUrl = storyboardUrl;
|
|
6197
6573
|
if (imageSubmissionMode === "base64") {
|
|
6198
6574
|
const downloadResult = await downloadImageAsBase64(storyboardUrl, imageDownloadOptions);
|
|
@@ -6628,16 +7004,16 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
|
|
|
6628
7004
|
|
|
6629
7005
|
// src/workflows/translate-captions.ts
|
|
6630
7006
|
import {
|
|
6631
|
-
APICallError,
|
|
7007
|
+
APICallError as APICallError2,
|
|
6632
7008
|
generateText as generateText7,
|
|
6633
|
-
NoObjectGeneratedError,
|
|
7009
|
+
NoObjectGeneratedError as NoObjectGeneratedError2,
|
|
6634
7010
|
Output as Output7,
|
|
6635
|
-
RetryError,
|
|
7011
|
+
RetryError as RetryError2,
|
|
6636
7012
|
TypeValidationError
|
|
6637
7013
|
} from "ai";
|
|
6638
|
-
import { z as
|
|
6639
|
-
var translationSchema =
|
|
6640
|
-
translation:
|
|
7014
|
+
import { z as z9 } from "zod";
|
|
7015
|
+
var translationSchema = z9.object({
|
|
7016
|
+
translation: z9.string()
|
|
6641
7017
|
});
|
|
6642
7018
|
var SYSTEM_PROMPT6 = promptDedent`
|
|
6643
7019
|
You are a subtitle translation expert. Translate VTT subtitle files to the target language specified by the user.
|
|
@@ -6704,10 +7080,10 @@ function isProviderServiceError(error) {
|
|
|
6704
7080
|
if (!error) {
|
|
6705
7081
|
return false;
|
|
6706
7082
|
}
|
|
6707
|
-
if (
|
|
7083
|
+
if (RetryError2.isInstance(error)) {
|
|
6708
7084
|
return isProviderServiceError(error.lastError);
|
|
6709
7085
|
}
|
|
6710
|
-
if (
|
|
7086
|
+
if (APICallError2.isInstance(error)) {
|
|
6711
7087
|
return true;
|
|
6712
7088
|
}
|
|
6713
7089
|
if (error instanceof Error && "cause" in error) {
|
|
@@ -6719,7 +7095,7 @@ function shouldSplitChunkTranslationError(error) {
|
|
|
6719
7095
|
if (isProviderServiceError(error)) {
|
|
6720
7096
|
return false;
|
|
6721
7097
|
}
|
|
6722
|
-
return
|
|
7098
|
+
return NoObjectGeneratedError2.isInstance(error) || TypeValidationError.isInstance(error) || isTranslationChunkValidationError(error);
|
|
6723
7099
|
}
|
|
6724
7100
|
function resolveTranslationChunkingOptions(options) {
|
|
6725
7101
|
const targetChunkDurationSeconds = Math.max(
|
|
@@ -6880,7 +7256,7 @@ async function translateVttWithAI({
|
|
|
6880
7256
|
"use step";
|
|
6881
7257
|
const model = await createLanguageModelFromConfig(provider, modelId, credentials);
|
|
6882
7258
|
const sanitisedVttContent = stripVttMetadataBlocks(vttContent);
|
|
6883
|
-
const response = await generateText7({
|
|
7259
|
+
const response = await withContentPolicyErrorHandling(() => generateText7({
|
|
6884
7260
|
model,
|
|
6885
7261
|
output: Output7.object({ schema: translationSchema }),
|
|
6886
7262
|
messages: [
|
|
@@ -6895,8 +7271,9 @@ async function translateVttWithAI({
|
|
|
6895
7271
|
${sanitisedVttContent}`
|
|
6896
7272
|
}
|
|
6897
7273
|
]
|
|
6898
|
-
});
|
|
6899
|
-
const
|
|
7274
|
+
}));
|
|
7275
|
+
const output = getGeneratedOutputWithContentPolicyHandling(response);
|
|
7276
|
+
const translated = normalizeTranslatedVtt(output.translation);
|
|
6900
7277
|
const leakReason = detectLeakReason(translated);
|
|
6901
7278
|
const safeTranslated = leakReason !== null ? vttContent : translated;
|
|
6902
7279
|
if (leakReason !== null) {
|
|
@@ -6921,7 +7298,8 @@ ${sanitisedVttContent}`
|
|
|
6921
7298
|
outputTokens: response.usage.outputTokens,
|
|
6922
7299
|
totalTokens: response.usage.totalTokens,
|
|
6923
7300
|
reasoningTokens: response.usage.reasoningTokens,
|
|
6924
|
-
cachedInputTokens: response.usage.cachedInputTokens
|
|
7301
|
+
cachedInputTokens: response.usage.cachedInputTokens,
|
|
7302
|
+
cacheWriteTokens: response.usage.inputTokenDetails?.cacheWriteTokens
|
|
6925
7303
|
}
|
|
6926
7304
|
};
|
|
6927
7305
|
}
|
|
@@ -6935,8 +7313,8 @@ async function translateCueChunkWithAI({
|
|
|
6935
7313
|
}) {
|
|
6936
7314
|
"use step";
|
|
6937
7315
|
const model = await createLanguageModelFromConfig(provider, modelId, credentials);
|
|
6938
|
-
const schema =
|
|
6939
|
-
translations:
|
|
7316
|
+
const schema = z9.object({
|
|
7317
|
+
translations: z9.array(z9.string().min(1).max(2e3)).length(cues.length)
|
|
6940
7318
|
});
|
|
6941
7319
|
const cuePayload = cues.map((cue, index) => ({
|
|
6942
7320
|
index,
|
|
@@ -6944,7 +7322,7 @@ async function translateCueChunkWithAI({
|
|
|
6944
7322
|
endTime: cue.endTime,
|
|
6945
7323
|
text: cue.text
|
|
6946
7324
|
}));
|
|
6947
|
-
const response = await generateText7({
|
|
7325
|
+
const response = await withContentPolicyErrorHandling(() => generateText7({
|
|
6948
7326
|
model,
|
|
6949
7327
|
output: Output7.object({ schema }),
|
|
6950
7328
|
messages: [
|
|
@@ -6960,7 +7338,8 @@ Return exactly ${cues.length} translated cues in the same order as the input.
|
|
|
6960
7338
|
${JSON.stringify(cuePayload, null, 2)}`
|
|
6961
7339
|
}
|
|
6962
7340
|
]
|
|
6963
|
-
});
|
|
7341
|
+
}));
|
|
7342
|
+
const output = getGeneratedOutputWithContentPolicyHandling(response);
|
|
6964
7343
|
const unexpectedKeys = detectUnexpectedKeysFromRawText(
|
|
6965
7344
|
response.text,
|
|
6966
7345
|
schema.keyof().options
|
|
@@ -6971,13 +7350,14 @@ ${JSON.stringify(cuePayload, null, 2)}`
|
|
|
6971
7350
|
);
|
|
6972
7351
|
}
|
|
6973
7352
|
return {
|
|
6974
|
-
translations:
|
|
7353
|
+
translations: output.translations,
|
|
6975
7354
|
usage: {
|
|
6976
7355
|
inputTokens: response.usage.inputTokens,
|
|
6977
7356
|
outputTokens: response.usage.outputTokens,
|
|
6978
7357
|
totalTokens: response.usage.totalTokens,
|
|
6979
7358
|
reasoningTokens: response.usage.reasoningTokens,
|
|
6980
|
-
cachedInputTokens: response.usage.cachedInputTokens
|
|
7359
|
+
cachedInputTokens: response.usage.cachedInputTokens,
|
|
7360
|
+
cacheWriteTokens: response.usage.inputTokenDetails?.cacheWriteTokens
|
|
6981
7361
|
},
|
|
6982
7362
|
unexpectedKeyCount: unexpectedKeys.length
|
|
6983
7363
|
};
|
|
@@ -7340,6 +7720,9 @@ async function translateCaptionsInternal(assetId, trackId, toLanguageCode, optio
|
|
|
7340
7720
|
};
|
|
7341
7721
|
}
|
|
7342
7722
|
export {
|
|
7723
|
+
ASK_QUESTIONS_DEFAULT_MAX_ANSWER_OPTION_LENGTH,
|
|
7724
|
+
ASK_QUESTIONS_DEFAULT_MAX_FREE_FORM_ANSWER_LENGTH,
|
|
7725
|
+
ASK_QUESTIONS_MAX_QUESTION_LENGTH,
|
|
7343
7726
|
DEFAULT_DESCRIPTION_LENGTH,
|
|
7344
7727
|
DEFAULT_SUMMARY_KEYWORD_LIMIT,
|
|
7345
7728
|
DEFAULT_TITLE_LENGTH,
|