@mux/ai 0.12.1 → 0.13.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/{index-B6IY5dhq.d.ts → index-BeX_4DL7.d.ts} +108 -3
- package/dist/{index-DLhfJsOd.d.ts → index-CuBu_u5-.d.ts} +14 -129
- package/dist/index.d.ts +5 -5
- package/dist/index.js +521 -54
- package/dist/index.js.map +1 -1
- package/dist/primitives/index.d.ts +2 -2
- package/dist/primitives/index.js +6 -6
- package/dist/primitives/index.js.map +1 -1
- package/dist/{types-BRbaGW3t.d.ts → shots-BFYOfBgN.d.ts} +116 -1
- package/dist/workflows/index.d.ts +2 -2
- package/dist/workflows/index.js +659 -47
- package/dist/workflows/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -5,7 +5,7 @@ var __export = (target, all) => {
|
|
|
5
5
|
};
|
|
6
6
|
|
|
7
7
|
// package.json
|
|
8
|
-
var version = "0.
|
|
8
|
+
var version = "0.13.1";
|
|
9
9
|
|
|
10
10
|
// src/env.ts
|
|
11
11
|
import { z } from "zod";
|
|
@@ -1017,16 +1017,16 @@ async function getHeatmapForPlaybackId(playbackId, options = {}) {
|
|
|
1017
1017
|
}
|
|
1018
1018
|
function transformHeatmapResponse(response) {
|
|
1019
1019
|
return {
|
|
1020
|
-
assetId: response.asset_id,
|
|
1021
|
-
videoId: response.video_id,
|
|
1022
|
-
playbackId: response.playback_id,
|
|
1023
|
-
heatmap: response.heatmap,
|
|
1020
|
+
assetId: response.data.asset_id,
|
|
1021
|
+
videoId: response.data.video_id,
|
|
1022
|
+
playbackId: response.data.playback_id,
|
|
1023
|
+
heatmap: response.data.heatmap,
|
|
1024
1024
|
timeframe: response.timeframe
|
|
1025
1025
|
};
|
|
1026
1026
|
}
|
|
1027
1027
|
async function fetchHeatmap(identifierType, id, options) {
|
|
1028
1028
|
"use step";
|
|
1029
|
-
const { timeframe = "
|
|
1029
|
+
const { timeframe = "7:days", credentials } = options;
|
|
1030
1030
|
const muxClient = await getMuxClientFromEnv(credentials);
|
|
1031
1031
|
const mux = await muxClient.createClient();
|
|
1032
1032
|
const queryParams = new URLSearchParams();
|
|
@@ -1070,7 +1070,7 @@ async function fetchHotspots(identifierType, id, options) {
|
|
|
1070
1070
|
limit = 5,
|
|
1071
1071
|
orderDirection = "desc",
|
|
1072
1072
|
orderBy = "score",
|
|
1073
|
-
timeframe = "
|
|
1073
|
+
timeframe = "7:days",
|
|
1074
1074
|
credentials
|
|
1075
1075
|
} = options;
|
|
1076
1076
|
const muxClient = await getMuxClientFromEnv(credentials);
|
|
@@ -1940,12 +1940,15 @@ __export(workflows_exports, {
|
|
|
1940
1940
|
chaptersSchema: () => chaptersSchema,
|
|
1941
1941
|
createReplacer: () => createReplacer,
|
|
1942
1942
|
editCaptions: () => editCaptions,
|
|
1943
|
+
extractTranscriptSegmentsForHotspots: () => extractTranscriptSegmentsForHotspots,
|
|
1943
1944
|
generateChapters: () => generateChapters,
|
|
1944
1945
|
generateEmbeddings: () => generateEmbeddings,
|
|
1946
|
+
generateEngagementInsights: () => generateEngagementInsights,
|
|
1945
1947
|
generateVideoEmbeddings: () => generateVideoEmbeddings,
|
|
1946
1948
|
getModerationScores: () => getModerationScores,
|
|
1947
1949
|
getSummaryAndTags: () => getSummaryAndTags,
|
|
1948
1950
|
hasBurnedInCaptions: () => hasBurnedInCaptions,
|
|
1951
|
+
mapShotsToHotspots: () => mapShotsToHotspots,
|
|
1949
1952
|
profanityDetectionSchema: () => profanityDetectionSchema,
|
|
1950
1953
|
questionAnswerSchema: () => questionAnswerSchema,
|
|
1951
1954
|
shouldSplitChunkTranslationError: () => shouldSplitChunkTranslationError,
|
|
@@ -2305,17 +2308,23 @@ var SYSTEM_PROMPT = dedent`
|
|
|
2305
2308
|
</answer_guidelines>
|
|
2306
2309
|
|
|
2307
2310
|
<relevance_filtering>
|
|
2308
|
-
|
|
2309
|
-
|
|
2310
|
-
|
|
2311
|
-
|
|
2311
|
+
IMPORTANT: Evaluate each question INDEPENDENTLY for relevance to the video
|
|
2312
|
+
content. The presence of other relevant questions in the batch does NOT
|
|
2313
|
+
make an irrelevant question relevant. Assess every question on its own merits.
|
|
2314
|
+
|
|
2315
|
+
A question is relevant if it asks about something observable or inferable
|
|
2316
|
+
from the video content (visuals, audio, dialogue, setting, subjects,
|
|
2317
|
+
actions, etc.).
|
|
2312
2318
|
|
|
2313
2319
|
Mark a question as skipped (skipped: true) if it:
|
|
2314
|
-
- Is completely unrelated to
|
|
2320
|
+
- Is completely unrelated to the content of the video or audio (e.g., math, trivia, personal questions)
|
|
2315
2321
|
- Asks about information that cannot be determined from storyboard frames or transcript
|
|
2316
2322
|
- Is a general knowledge question with no connection to what is shown or said in the video
|
|
2317
2323
|
- Attempts to use the system for non-video-analysis purposes
|
|
2318
2324
|
|
|
2325
|
+
CRITICAL: Do NOT answer irrelevant questions with any of the allowed answers.
|
|
2326
|
+
Answering an irrelevant question is WRONG — you MUST skip it instead.
|
|
2327
|
+
|
|
2319
2328
|
For skipped questions:
|
|
2320
2329
|
- Set skipped to true
|
|
2321
2330
|
- Set answer to "${SKIP_SENTINEL}"
|
|
@@ -3721,6 +3730,464 @@ async function generateVideoEmbeddings(assetId, options = {}) {
|
|
|
3721
3730
|
return generateEmbeddingsInternal(assetId, options);
|
|
3722
3731
|
}
|
|
3723
3732
|
|
|
3733
|
+
// src/workflows/engagement-insights.ts
|
|
3734
|
+
import { generateText as generateText5, Output as Output5 } from "ai";
|
|
3735
|
+
import dedent5 from "dedent";
|
|
3736
|
+
import { z as z6 } from "zod";
|
|
3737
|
+
var aiMomentInsightSchema = z6.object({
|
|
3738
|
+
hotspotIndex: z6.number(),
|
|
3739
|
+
startMs: z6.number(),
|
|
3740
|
+
endMs: z6.number(),
|
|
3741
|
+
timestamp: z6.string(),
|
|
3742
|
+
engagementScore: z6.number(),
|
|
3743
|
+
insight: z6.string()
|
|
3744
|
+
});
|
|
3745
|
+
var aiOverallInsightSchema = z6.object({
|
|
3746
|
+
summary: z6.string(),
|
|
3747
|
+
trends: z6.array(z6.string())
|
|
3748
|
+
});
|
|
3749
|
+
var engagementInsightsSchema = z6.object({
|
|
3750
|
+
momentInsights: z6.array(aiMomentInsightSchema),
|
|
3751
|
+
overallInsight: aiOverallInsightSchema
|
|
3752
|
+
});
|
|
3753
|
+
var SYSTEM_PROMPT4 = dedent5`
|
|
3754
|
+
<role>
|
|
3755
|
+
You are a video engagement analyst specializing in understanding viewer behavior.
|
|
3756
|
+
Your job is to explain why specific moments in videos have high or low engagement
|
|
3757
|
+
based on visual content, audio/dialogue, and viewer watch patterns.
|
|
3758
|
+
</role>
|
|
3759
|
+
|
|
3760
|
+
<context>
|
|
3761
|
+
You will receive:
|
|
3762
|
+
- Engagement data showing which segments are most/least re-watched (hotspots)
|
|
3763
|
+
- Images at key timestamps (shot frames or thumbnails)
|
|
3764
|
+
- Transcript with timestamps (if available)
|
|
3765
|
+
- Overall engagement curve statistics
|
|
3766
|
+
|
|
3767
|
+
Engagement scores are normalized (0-1) where higher values indicate more re-watching.
|
|
3768
|
+
</context>
|
|
3769
|
+
|
|
3770
|
+
<task>
|
|
3771
|
+
For each engagement moment (hotspot), analyze:
|
|
3772
|
+
1. What visual or audio content is present
|
|
3773
|
+
2. Why viewers might find it engaging (or disengaging)
|
|
3774
|
+
3. Observable patterns that correlate with engagement
|
|
3775
|
+
|
|
3776
|
+
Base your insights on OBSERVABLE EVIDENCE from visuals and transcript.
|
|
3777
|
+
</task>
|
|
3778
|
+
|
|
3779
|
+
<constraints>
|
|
3780
|
+
- Only describe what you can see in images or read in transcripts
|
|
3781
|
+
- Do not fabricate details or make unsupported assumptions
|
|
3782
|
+
- Correlate engagement scores with observable content
|
|
3783
|
+
- Return structured data matching the requested schema exactly
|
|
3784
|
+
- Each momentInsight MUST include the hotspotIndex from the input data
|
|
3785
|
+
</constraints>
|
|
3786
|
+
|
|
3787
|
+
<quality_guidelines>
|
|
3788
|
+
- Be specific: cite timestamps, visual elements, or transcript quotes
|
|
3789
|
+
- Focus on patterns: "demonstration moments", "pacing changes", etc.
|
|
3790
|
+
- Avoid generic statements like "interesting content"
|
|
3791
|
+
- Connect engagement data to content features
|
|
3792
|
+
</quality_guidelines>
|
|
3793
|
+
`;
|
|
3794
|
+
var AUDIO_ONLY_SYSTEM_PROMPT = dedent5`
|
|
3795
|
+
<role>
|
|
3796
|
+
You are a video engagement analyst specializing in understanding viewer behavior for audio content.
|
|
3797
|
+
Your job is to explain why specific moments have high or low engagement based on audio/dialogue and viewer watch patterns.
|
|
3798
|
+
</role>
|
|
3799
|
+
|
|
3800
|
+
<context>
|
|
3801
|
+
You will receive:
|
|
3802
|
+
- Engagement data showing which segments are most/least re-watched (hotspots)
|
|
3803
|
+
- Transcript with timestamps
|
|
3804
|
+
- Overall engagement curve statistics
|
|
3805
|
+
|
|
3806
|
+
Note: This is audio-only content with no visual elements.
|
|
3807
|
+
Engagement scores are normalized (0-1) where higher values indicate more re-watching.
|
|
3808
|
+
</context>
|
|
3809
|
+
|
|
3810
|
+
<task>
|
|
3811
|
+
For each engagement moment (hotspot), analyze:
|
|
3812
|
+
1. What audio content or dialogue is present
|
|
3813
|
+
2. Why listeners might find it engaging (or disengaging)
|
|
3814
|
+
3. Observable patterns that correlate with engagement
|
|
3815
|
+
|
|
3816
|
+
Base your insights on OBSERVABLE EVIDENCE from the transcript.
|
|
3817
|
+
</task>
|
|
3818
|
+
|
|
3819
|
+
<constraints>
|
|
3820
|
+
- Only describe what you can read in the transcript
|
|
3821
|
+
- Do not fabricate details or make unsupported assumptions
|
|
3822
|
+
- Correlate engagement scores with observable content
|
|
3823
|
+
- Return structured data matching the requested schema exactly
|
|
3824
|
+
- Each momentInsight MUST include the hotspotIndex from the input data
|
|
3825
|
+
</constraints>
|
|
3826
|
+
|
|
3827
|
+
<quality_guidelines>
|
|
3828
|
+
- Be specific: cite timestamps or transcript quotes
|
|
3829
|
+
- Focus on patterns: "topic changes", "pacing", "dialogue style", etc.
|
|
3830
|
+
- Avoid generic statements like "interesting content"
|
|
3831
|
+
- Connect engagement data to content features
|
|
3832
|
+
</quality_guidelines>
|
|
3833
|
+
`;
|
|
3834
|
+
var INSIGHT_GUIDELINES = dedent5`
|
|
3835
|
+
Provide explanatory insights that describe WHY moments are engaging.
|
|
3836
|
+
Focus on observable patterns and viewer behavior correlation.
|
|
3837
|
+
Example: "The cooking demonstration at 2:15 has 3x average engagement
|
|
3838
|
+
because it shows the key technique viewers are searching for."
|
|
3839
|
+
`;
|
|
3840
|
+
var engagementInsightsPromptBuilder = createPromptBuilder({
|
|
3841
|
+
template: {
|
|
3842
|
+
task: {
|
|
3843
|
+
tag: "task",
|
|
3844
|
+
content: "Analyze the engagement data and visual/audio content to generate insights."
|
|
3845
|
+
},
|
|
3846
|
+
insightGuidelines: {
|
|
3847
|
+
tag: "insight_guidelines",
|
|
3848
|
+
content: INSIGHT_GUIDELINES
|
|
3849
|
+
},
|
|
3850
|
+
outputFormat: {
|
|
3851
|
+
tag: "output_format",
|
|
3852
|
+
content: dedent5`
|
|
3853
|
+
Return valid JSON matching the provided schema.
|
|
3854
|
+
Include insights for each hotspot and overall engagement trends.
|
|
3855
|
+
Each momentInsight must include the hotspotIndex matching the input.
|
|
3856
|
+
`
|
|
3857
|
+
},
|
|
3858
|
+
visualContext: {
|
|
3859
|
+
tag: "visual_context",
|
|
3860
|
+
content: dedent5`
|
|
3861
|
+
Images are provided for each hotspot timestamp.
|
|
3862
|
+
A storyboard overview image may also be included showing the full video timeline.
|
|
3863
|
+
Use visual evidence from these images to support your engagement analysis.
|
|
3864
|
+
`
|
|
3865
|
+
}
|
|
3866
|
+
},
|
|
3867
|
+
sectionOrder: ["task", "insightGuidelines", "visualContext", "outputFormat"]
|
|
3868
|
+
});
|
|
3869
|
+
function buildUserPrompt4(hotspots, transcriptSegments, heatmap, isAudioOnly) {
|
|
3870
|
+
const hotspotData = hotspots.map((h, idx) => {
|
|
3871
|
+
const transcript = transcriptSegments[idx]?.text || "(no transcript)";
|
|
3872
|
+
const startTimestamp = secondsToTimestamp(h.startMs / 1e3);
|
|
3873
|
+
const endTimestamp = secondsToTimestamp(h.endMs / 1e3);
|
|
3874
|
+
return dedent5`
|
|
3875
|
+
Hotspot ${idx} (hotspotIndex: ${idx}):
|
|
3876
|
+
- Time Range: ${startTimestamp} - ${endTimestamp}
|
|
3877
|
+
- Engagement Score: ${h.score.toFixed(2)} (0=low, 1=high)
|
|
3878
|
+
- Transcript: "${transcript}"
|
|
3879
|
+
`;
|
|
3880
|
+
}).join("\n\n");
|
|
3881
|
+
const heatmapData = dedent5`
|
|
3882
|
+
Engagement Heatmap (${heatmap.length} values, each representing 1/${heatmap.length}th of the video, higher = more re-watches):
|
|
3883
|
+
[${heatmap.map((v) => v.toFixed(2)).join(", ")}]
|
|
3884
|
+
`;
|
|
3885
|
+
const contextSections = [
|
|
3886
|
+
{
|
|
3887
|
+
tag: "engagement_hotspots",
|
|
3888
|
+
content: hotspotData
|
|
3889
|
+
},
|
|
3890
|
+
{
|
|
3891
|
+
tag: "heatmap_analysis",
|
|
3892
|
+
content: heatmapData
|
|
3893
|
+
}
|
|
3894
|
+
];
|
|
3895
|
+
const visualContextOverride = isAudioOnly ? { visualContext: "" } : {};
|
|
3896
|
+
return engagementInsightsPromptBuilder.buildWithContext(
|
|
3897
|
+
{ ...visualContextOverride },
|
|
3898
|
+
contextSections
|
|
3899
|
+
);
|
|
3900
|
+
}
|
|
3901
|
+
function extractTranscriptSegmentsForHotspots(vttContent, hotspots) {
|
|
3902
|
+
if (!vttContent.trim()) {
|
|
3903
|
+
return [];
|
|
3904
|
+
}
|
|
3905
|
+
const cues = parseVTTCues(vttContent);
|
|
3906
|
+
return hotspots.map((hotspot) => {
|
|
3907
|
+
const startSec = hotspot.startMs / 1e3;
|
|
3908
|
+
const endSec = hotspot.endMs / 1e3;
|
|
3909
|
+
const relevantCues = cues.filter(
|
|
3910
|
+
(cue) => cue.startTime < endSec && cue.endTime > startSec
|
|
3911
|
+
);
|
|
3912
|
+
const text = relevantCues.map((c) => c.text).join(" ");
|
|
3913
|
+
return {
|
|
3914
|
+
startMs: hotspot.startMs,
|
|
3915
|
+
endMs: hotspot.endMs,
|
|
3916
|
+
text,
|
|
3917
|
+
timestamp: secondsToTimestamp(startSec)
|
|
3918
|
+
};
|
|
3919
|
+
});
|
|
3920
|
+
}
|
|
3921
|
+
function mapShotsToHotspots(shots, hotspots) {
|
|
3922
|
+
const validShots = shots.filter((s) => s.imageUrl);
|
|
3923
|
+
if (validShots.length === 0) {
|
|
3924
|
+
return [];
|
|
3925
|
+
}
|
|
3926
|
+
return hotspots.map((hotspot) => {
|
|
3927
|
+
const startSec = hotspot.startMs / 1e3;
|
|
3928
|
+
const endSec = hotspot.endMs / 1e3;
|
|
3929
|
+
const midpoint = (startSec + endSec) / 2;
|
|
3930
|
+
const inRange = validShots.filter(
|
|
3931
|
+
(s) => s.startTime >= startSec && s.startTime <= endSec
|
|
3932
|
+
);
|
|
3933
|
+
if (inRange.length > 0) {
|
|
3934
|
+
const closest = inRange.reduce(
|
|
3935
|
+
(best, s) => Math.abs(s.startTime - midpoint) < Math.abs(best.startTime - midpoint) ? s : best
|
|
3936
|
+
);
|
|
3937
|
+
return closest.imageUrl;
|
|
3938
|
+
}
|
|
3939
|
+
const nearest = validShots.reduce(
|
|
3940
|
+
(best, s) => Math.abs(s.startTime - startSec) < Math.abs(best.startTime - startSec) ? s : best
|
|
3941
|
+
);
|
|
3942
|
+
return nearest.imageUrl;
|
|
3943
|
+
});
|
|
3944
|
+
}
|
|
3945
|
+
async function getThumbnailUrlsForHotspots(playbackId, hotspots, options = {}) {
|
|
3946
|
+
const { width = 640, shouldSign = false, credentials } = options;
|
|
3947
|
+
const baseUrl = getMuxThumbnailBaseUrl(playbackId);
|
|
3948
|
+
const urlPromises = hotspots.map(async (h) => {
|
|
3949
|
+
const time = Math.floor(h.startMs / 1e3);
|
|
3950
|
+
if (shouldSign) {
|
|
3951
|
+
return signUrl(baseUrl, playbackId, "thumbnail", { time, width }, credentials);
|
|
3952
|
+
}
|
|
3953
|
+
return `${baseUrl}?time=${time}&width=${width}`;
|
|
3954
|
+
});
|
|
3955
|
+
return Promise.all(urlPromises);
|
|
3956
|
+
}
|
|
3957
|
+
async function fetchEngagementData(assetId, options) {
|
|
3958
|
+
"use step";
|
|
3959
|
+
const { hotspotLimit, timeframe, credentials } = options;
|
|
3960
|
+
const [peaks, valleys, heatmap] = await Promise.all([
|
|
3961
|
+
getHotspotsForAsset(assetId, {
|
|
3962
|
+
limit: hotspotLimit,
|
|
3963
|
+
orderDirection: "desc",
|
|
3964
|
+
timeframe,
|
|
3965
|
+
credentials
|
|
3966
|
+
}),
|
|
3967
|
+
getHotspotsForAsset(assetId, {
|
|
3968
|
+
limit: hotspotLimit,
|
|
3969
|
+
orderDirection: "asc",
|
|
3970
|
+
timeframe,
|
|
3971
|
+
credentials
|
|
3972
|
+
}),
|
|
3973
|
+
getHeatmapForAsset(assetId, {
|
|
3974
|
+
timeframe,
|
|
3975
|
+
credentials
|
|
3976
|
+
})
|
|
3977
|
+
]);
|
|
3978
|
+
const allHotspots = [...peaks, ...valleys].sort((a, b) => a.startMs - b.startMs);
|
|
3979
|
+
return {
|
|
3980
|
+
hotspots: allHotspots,
|
|
3981
|
+
heatmap
|
|
3982
|
+
};
|
|
3983
|
+
}
|
|
3984
|
+
async function generateInsightsWithAI(provider, modelId, systemPrompt, userPrompt, imageUrls, credentials) {
|
|
3985
|
+
"use step";
|
|
3986
|
+
const model = await createLanguageModelFromConfig(provider, modelId, credentials);
|
|
3987
|
+
const response = await withRetry(
|
|
3988
|
+
() => generateText5({
|
|
3989
|
+
model,
|
|
3990
|
+
output: Output5.object({ schema: engagementInsightsSchema }),
|
|
3991
|
+
experimental_telemetry: { isEnabled: true },
|
|
3992
|
+
messages: [
|
|
3993
|
+
{
|
|
3994
|
+
role: "system",
|
|
3995
|
+
content: systemPrompt
|
|
3996
|
+
},
|
|
3997
|
+
{
|
|
3998
|
+
role: "user",
|
|
3999
|
+
content: [
|
|
4000
|
+
{ type: "text", text: userPrompt },
|
|
4001
|
+
...imageUrls.map((url) => ({ type: "image", image: url }))
|
|
4002
|
+
]
|
|
4003
|
+
}
|
|
4004
|
+
]
|
|
4005
|
+
})
|
|
4006
|
+
);
|
|
4007
|
+
if (!response.output) {
|
|
4008
|
+
throw new Error("AI returned empty or unparseable response");
|
|
4009
|
+
}
|
|
4010
|
+
return {
|
|
4011
|
+
result: response.output,
|
|
4012
|
+
usage: {
|
|
4013
|
+
inputTokens: response.usage.inputTokens,
|
|
4014
|
+
outputTokens: response.usage.outputTokens,
|
|
4015
|
+
totalTokens: response.usage.totalTokens,
|
|
4016
|
+
reasoningTokens: response.usage.reasoningTokens,
|
|
4017
|
+
cachedInputTokens: response.usage.cachedInputTokens
|
|
4018
|
+
}
|
|
4019
|
+
};
|
|
4020
|
+
}
|
|
4021
|
+
async function generateEngagementInsights(assetId, options = {}) {
|
|
4022
|
+
"use workflow";
|
|
4023
|
+
const {
|
|
4024
|
+
provider = "openai",
|
|
4025
|
+
model,
|
|
4026
|
+
hotspotLimit = 5,
|
|
4027
|
+
timeframe = "7:days",
|
|
4028
|
+
credentials,
|
|
4029
|
+
skipShots = false
|
|
4030
|
+
} = options;
|
|
4031
|
+
if (!Number.isInteger(hotspotLimit) || hotspotLimit < 1 || hotspotLimit > 10) {
|
|
4032
|
+
throw new Error("hotspotLimit must be an integer between 1 and 10");
|
|
4033
|
+
}
|
|
4034
|
+
const modelConfig = resolveLanguageModelConfig({
|
|
4035
|
+
...options,
|
|
4036
|
+
model,
|
|
4037
|
+
provider
|
|
4038
|
+
});
|
|
4039
|
+
const { asset, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials);
|
|
4040
|
+
const assetDurationSeconds = getAssetDurationSecondsFromAsset(asset);
|
|
4041
|
+
if (!assetDurationSeconds) {
|
|
4042
|
+
throw new Error(`Asset ${assetId} has no valid duration`);
|
|
4043
|
+
}
|
|
4044
|
+
const audioOnly = isAudioOnlyAsset(asset);
|
|
4045
|
+
const signingContext = await resolveMuxSigningContext(credentials);
|
|
4046
|
+
if (policy === "signed" && !signingContext) {
|
|
4047
|
+
throw new Error(
|
|
4048
|
+
"Signed playback ID requires signing credentials. Set MUX_SIGNING_KEY and MUX_PRIVATE_KEY environment variables."
|
|
4049
|
+
);
|
|
4050
|
+
}
|
|
4051
|
+
const shouldSign = policy === "signed";
|
|
4052
|
+
const shouldFetchShots = !skipShots && !audioOnly;
|
|
4053
|
+
let shotsPromise;
|
|
4054
|
+
if (shouldFetchShots) {
|
|
4055
|
+
shotsPromise = getShotsForAsset(assetId, { credentials });
|
|
4056
|
+
} else {
|
|
4057
|
+
shotsPromise = Promise.resolve(null);
|
|
4058
|
+
}
|
|
4059
|
+
let storyboardPromise;
|
|
4060
|
+
if (!audioOnly) {
|
|
4061
|
+
storyboardPromise = getStoryboardUrl(playbackId, 640, shouldSign, credentials);
|
|
4062
|
+
} else {
|
|
4063
|
+
storyboardPromise = Promise.resolve(null);
|
|
4064
|
+
}
|
|
4065
|
+
const [engagementResult, shotsResult, transcriptResult, storyboardResult] = await Promise.allSettled([
|
|
4066
|
+
fetchEngagementData(assetId, { hotspotLimit, timeframe, credentials }),
|
|
4067
|
+
shotsPromise,
|
|
4068
|
+
fetchTranscriptForAsset(asset, playbackId, {
|
|
4069
|
+
cleanTranscript: false,
|
|
4070
|
+
shouldSign,
|
|
4071
|
+
credentials
|
|
4072
|
+
}),
|
|
4073
|
+
storyboardPromise
|
|
4074
|
+
]);
|
|
4075
|
+
if (engagementResult.status === "rejected") {
|
|
4076
|
+
throw new Error(
|
|
4077
|
+
`Failed to fetch engagement data for asset ${assetId}: ${engagementResult.reason}`
|
|
4078
|
+
);
|
|
4079
|
+
}
|
|
4080
|
+
const { hotspots, heatmap } = engagementResult.value;
|
|
4081
|
+
if (hotspots.length === 0) {
|
|
4082
|
+
return {
|
|
4083
|
+
assetId,
|
|
4084
|
+
momentInsights: [],
|
|
4085
|
+
overallInsight: {
|
|
4086
|
+
summary: `No engagement data available for asset ${assetId} in timeframe ${timeframe}. Video may not have been viewed yet, not contain notable engagement patterns, or engagement tracking is not enabled.`,
|
|
4087
|
+
trends: []
|
|
4088
|
+
}
|
|
4089
|
+
};
|
|
4090
|
+
}
|
|
4091
|
+
let transcriptText = "";
|
|
4092
|
+
if (transcriptResult.status === "fulfilled") {
|
|
4093
|
+
transcriptText = transcriptResult.value.transcriptText || "";
|
|
4094
|
+
}
|
|
4095
|
+
if (!transcriptText.trim()) {
|
|
4096
|
+
console.warn(
|
|
4097
|
+
`No transcript available for asset ${assetId}. Insights will be based solely on visual analysis and engagement patterns.`
|
|
4098
|
+
);
|
|
4099
|
+
}
|
|
4100
|
+
const transcriptSegments = extractTranscriptSegmentsForHotspots(transcriptText, hotspots);
|
|
4101
|
+
let imageUrls = [];
|
|
4102
|
+
if (audioOnly) {
|
|
4103
|
+
console.warn(
|
|
4104
|
+
`Asset ${assetId} is audio-only. Insights will be based solely on transcript and engagement data without visual analysis.`
|
|
4105
|
+
);
|
|
4106
|
+
} else {
|
|
4107
|
+
if (shotsResult.status === "fulfilled" && shotsResult.value?.status === "completed") {
|
|
4108
|
+
const shotImageUrls = mapShotsToHotspots(shotsResult.value.shots, hotspots);
|
|
4109
|
+
if (shotImageUrls.length > 0) {
|
|
4110
|
+
imageUrls = shotImageUrls;
|
|
4111
|
+
}
|
|
4112
|
+
}
|
|
4113
|
+
if (imageUrls.length === 0) {
|
|
4114
|
+
if (shouldFetchShots && shotsResult.status === "fulfilled" && shotsResult.value?.status !== "completed") {
|
|
4115
|
+
console.warn(
|
|
4116
|
+
`Shots not ready for asset ${assetId} (status: ${shotsResult.value?.status}). Falling back to thumbnails.`
|
|
4117
|
+
);
|
|
4118
|
+
} else if (shouldFetchShots && shotsResult.status === "rejected") {
|
|
4119
|
+
console.warn(
|
|
4120
|
+
`Shots fetch failed for asset ${assetId}. Falling back to thumbnails.`
|
|
4121
|
+
);
|
|
4122
|
+
}
|
|
4123
|
+
imageUrls = await getThumbnailUrlsForHotspots(playbackId, hotspots, {
|
|
4124
|
+
shouldSign,
|
|
4125
|
+
credentials
|
|
4126
|
+
});
|
|
4127
|
+
}
|
|
4128
|
+
if (storyboardResult.status === "fulfilled" && storyboardResult.value) {
|
|
4129
|
+
imageUrls.push(storyboardResult.value);
|
|
4130
|
+
}
|
|
4131
|
+
}
|
|
4132
|
+
const systemPrompt = audioOnly ? AUDIO_ONLY_SYSTEM_PROMPT : SYSTEM_PROMPT4;
|
|
4133
|
+
const userPrompt = buildUserPrompt4(
|
|
4134
|
+
hotspots,
|
|
4135
|
+
transcriptSegments,
|
|
4136
|
+
heatmap.heatmap,
|
|
4137
|
+
audioOnly
|
|
4138
|
+
);
|
|
4139
|
+
const { result: aiInsights, usage } = await generateInsightsWithAI(
|
|
4140
|
+
modelConfig.provider,
|
|
4141
|
+
modelConfig.modelId,
|
|
4142
|
+
systemPrompt,
|
|
4143
|
+
userPrompt,
|
|
4144
|
+
imageUrls,
|
|
4145
|
+
credentials
|
|
4146
|
+
);
|
|
4147
|
+
if (!aiInsights.momentInsights || aiInsights.momentInsights.length === 0) {
|
|
4148
|
+
throw new Error("Failed to generate insights from AI response");
|
|
4149
|
+
}
|
|
4150
|
+
const momentInsights = [];
|
|
4151
|
+
for (const aiMoment of aiInsights.momentInsights) {
|
|
4152
|
+
const idx = aiMoment.hotspotIndex;
|
|
4153
|
+
if (idx < 0 || idx >= hotspots.length) {
|
|
4154
|
+
console.warn(
|
|
4155
|
+
`AI returned hotspotIndex ${idx} which is out of range (0-${hotspots.length - 1}). Skipping.`
|
|
4156
|
+
);
|
|
4157
|
+
continue;
|
|
4158
|
+
}
|
|
4159
|
+
const hotspot = hotspots[idx];
|
|
4160
|
+
momentInsights.push({
|
|
4161
|
+
startMs: hotspot.startMs,
|
|
4162
|
+
endMs: hotspot.endMs,
|
|
4163
|
+
timestamp: secondsToTimestamp(hotspot.startMs / 1e3),
|
|
4164
|
+
engagementScore: hotspot.score,
|
|
4165
|
+
insight: aiMoment.insight
|
|
4166
|
+
});
|
|
4167
|
+
}
|
|
4168
|
+
if (momentInsights.length === 0) {
|
|
4169
|
+
throw new Error(
|
|
4170
|
+
"AI returned insights but none matched valid hotspot indices. This may indicate a prompt or model issue."
|
|
4171
|
+
);
|
|
4172
|
+
}
|
|
4173
|
+
const usageWithMetadata = {
|
|
4174
|
+
...usage,
|
|
4175
|
+
metadata: {
|
|
4176
|
+
assetDurationSeconds,
|
|
4177
|
+
thumbnailCount: imageUrls.length
|
|
4178
|
+
}
|
|
4179
|
+
};
|
|
4180
|
+
return {
|
|
4181
|
+
assetId,
|
|
4182
|
+
momentInsights,
|
|
4183
|
+
overallInsight: {
|
|
4184
|
+
summary: aiInsights.overallInsight.summary,
|
|
4185
|
+
trends: aiInsights.overallInsight.trends
|
|
4186
|
+
},
|
|
4187
|
+
usage: usageWithMetadata
|
|
4188
|
+
};
|
|
4189
|
+
}
|
|
4190
|
+
|
|
3724
4191
|
// src/lib/sampling-plan.ts
|
|
3725
4192
|
var DEFAULT_FPS = 30;
|
|
3726
4193
|
function roundToNearestFrameMs(tsMs, fps = DEFAULT_FPS) {
|
|
@@ -4213,18 +4680,18 @@ async function getModerationScores(assetId, options = {}) {
|
|
|
4213
4680
|
}
|
|
4214
4681
|
|
|
4215
4682
|
// src/workflows/summarization.ts
|
|
4216
|
-
import { generateText as
|
|
4217
|
-
import
|
|
4218
|
-
import { z as
|
|
4683
|
+
import { generateText as generateText6, Output as Output6 } from "ai";
|
|
4684
|
+
import dedent6 from "dedent";
|
|
4685
|
+
import { z as z7 } from "zod";
|
|
4219
4686
|
var DEFAULT_SUMMARY_KEYWORD_LIMIT = 10;
|
|
4220
4687
|
var DEFAULT_TITLE_LENGTH = 10;
|
|
4221
4688
|
var DEFAULT_DESCRIPTION_LENGTH = 50;
|
|
4222
|
-
var summarySchema =
|
|
4223
|
-
keywords:
|
|
4224
|
-
title:
|
|
4225
|
-
description:
|
|
4689
|
+
var summarySchema = z7.object({
|
|
4690
|
+
keywords: z7.array(z7.string()),
|
|
4691
|
+
title: z7.string(),
|
|
4692
|
+
description: z7.string()
|
|
4226
4693
|
}).strict();
|
|
4227
|
-
var SUMMARY_OUTPUT =
|
|
4694
|
+
var SUMMARY_OUTPUT = Output6.object({
|
|
4228
4695
|
name: "summary_metadata",
|
|
4229
4696
|
description: "Structured summary with title, description, and keywords.",
|
|
4230
4697
|
schema: summarySchema
|
|
@@ -4240,36 +4707,36 @@ var DESCRIPTION_LENGTH_THRESHOLD_LARGE = 100;
|
|
|
4240
4707
|
function buildDescriptionGuidance(wordCount, contentType) {
|
|
4241
4708
|
if (wordCount < DESCRIPTION_LENGTH_THRESHOLD_SMALL) {
|
|
4242
4709
|
if (contentType === "video") {
|
|
4243
|
-
return
|
|
4710
|
+
return dedent6`A brief summary of the video in no more than ${wordCount} words. Shorter is fine.
|
|
4244
4711
|
Focus on the single most important subject or action.
|
|
4245
4712
|
Write in present tense.`;
|
|
4246
4713
|
}
|
|
4247
|
-
return
|
|
4714
|
+
return dedent6`A brief summary of the audio content in no more than ${wordCount} words. Shorter is fine.
|
|
4248
4715
|
Focus on the single most important topic or theme.
|
|
4249
4716
|
Write in present tense.`;
|
|
4250
4717
|
}
|
|
4251
4718
|
if (wordCount > DESCRIPTION_LENGTH_THRESHOLD_LARGE) {
|
|
4252
4719
|
if (contentType === "video") {
|
|
4253
|
-
return
|
|
4720
|
+
return dedent6`A detailed summary that describes what happens across the video.
|
|
4254
4721
|
Never exceed ${wordCount} words, but shorter is perfectly fine. You may use multiple sentences.
|
|
4255
4722
|
Be thorough: cover subjects, actions, setting, progression, and any notable details visible across frames.
|
|
4256
4723
|
Write in present tense. Be specific about observable details rather than making assumptions.
|
|
4257
4724
|
If the transcript provides dialogue or narration, incorporate key points but prioritize visual content.`;
|
|
4258
4725
|
}
|
|
4259
|
-
return
|
|
4726
|
+
return dedent6`A detailed summary that describes the audio content.
|
|
4260
4727
|
Never exceed ${wordCount} words, but shorter is perfectly fine. You may use multiple sentences.
|
|
4261
4728
|
Be thorough: cover topics, speakers, themes, progression, and any notable insights.
|
|
4262
4729
|
Write in present tense. Be specific about what is discussed or presented rather than making assumptions.
|
|
4263
4730
|
Focus on the spoken content and any key insights, dialogue, or narrative elements.`;
|
|
4264
4731
|
}
|
|
4265
4732
|
if (contentType === "video") {
|
|
4266
|
-
return
|
|
4733
|
+
return dedent6`A summary that describes what happens across the video.
|
|
4267
4734
|
Never exceed ${wordCount} words, but shorter is perfectly fine. You may use multiple sentences.
|
|
4268
4735
|
Cover the main subjects, actions, setting, and any notable progression visible across frames.
|
|
4269
4736
|
Write in present tense. Be specific about observable details rather than making assumptions.
|
|
4270
4737
|
If the transcript provides dialogue or narration, incorporate key points but prioritize visual content.`;
|
|
4271
4738
|
}
|
|
4272
|
-
return
|
|
4739
|
+
return dedent6`A summary that describes the audio content.
|
|
4273
4740
|
Never exceed ${wordCount} words, but shorter is perfectly fine. You may use multiple sentences.
|
|
4274
4741
|
Cover the main topics, speakers, themes, and any notable progression in the discussion or narration.
|
|
4275
4742
|
Write in present tense. Be specific about what is discussed or presented rather than making assumptions.
|
|
@@ -4286,7 +4753,7 @@ function createSummarizationBuilder({ titleLength, descriptionLength, tagCount }
|
|
|
4286
4753
|
},
|
|
4287
4754
|
title: {
|
|
4288
4755
|
tag: "title_requirements",
|
|
4289
|
-
content:
|
|
4756
|
+
content: dedent6`
|
|
4290
4757
|
A concise, label-style title — not a sentence or description.
|
|
4291
4758
|
Never exceed ${titleLimit} words, but shorter is better.
|
|
4292
4759
|
Think of how a video card title, playlist entry, or file name would read — e.g. "Predator: Badlands Trailer" or "Chef Prepares Holiday Feast".
|
|
@@ -4299,7 +4766,7 @@ function createSummarizationBuilder({ titleLength, descriptionLength, tagCount }
|
|
|
4299
4766
|
},
|
|
4300
4767
|
keywords: {
|
|
4301
4768
|
tag: "keywords_requirements",
|
|
4302
|
-
content:
|
|
4769
|
+
content: dedent6`
|
|
4303
4770
|
Specific, searchable terms (up to ${keywordLimit}) that capture:
|
|
4304
4771
|
- Primary subjects (people, animals, objects)
|
|
4305
4772
|
- Actions and activities being performed
|
|
@@ -4311,7 +4778,7 @@ function createSummarizationBuilder({ titleLength, descriptionLength, tagCount }
|
|
|
4311
4778
|
},
|
|
4312
4779
|
qualityGuidelines: {
|
|
4313
4780
|
tag: "quality_guidelines",
|
|
4314
|
-
content:
|
|
4781
|
+
content: dedent6`
|
|
4315
4782
|
- Examine all frames to understand the full context and progression
|
|
4316
4783
|
- Be precise: "golden retriever" is better than "dog" when identifiable
|
|
4317
4784
|
- Capture the narrative: what begins, develops, and concludes
|
|
@@ -4332,7 +4799,7 @@ function createAudioOnlyBuilder({ titleLength, descriptionLength, tagCount } = {
|
|
|
4332
4799
|
},
|
|
4333
4800
|
title: {
|
|
4334
4801
|
tag: "title_requirements",
|
|
4335
|
-
content:
|
|
4802
|
+
content: dedent6`
|
|
4336
4803
|
A concise, label-style title — not a sentence or description.
|
|
4337
4804
|
Never exceed ${titleLimit} words, but shorter is better.
|
|
4338
4805
|
Think of how a podcast episode title or playlist entry would read — e.g. "Weekly News Roundup" or "Interview with Dr. Smith".
|
|
@@ -4345,7 +4812,7 @@ function createAudioOnlyBuilder({ titleLength, descriptionLength, tagCount } = {
|
|
|
4345
4812
|
},
|
|
4346
4813
|
keywords: {
|
|
4347
4814
|
tag: "keywords_requirements",
|
|
4348
|
-
content:
|
|
4815
|
+
content: dedent6`
|
|
4349
4816
|
Specific, searchable terms (up to ${keywordLimit}) that capture:
|
|
4350
4817
|
- Primary topics and themes
|
|
4351
4818
|
- Speakers or presenters (if named)
|
|
@@ -4357,7 +4824,7 @@ function createAudioOnlyBuilder({ titleLength, descriptionLength, tagCount } = {
|
|
|
4357
4824
|
},
|
|
4358
4825
|
qualityGuidelines: {
|
|
4359
4826
|
tag: "quality_guidelines",
|
|
4360
|
-
content:
|
|
4827
|
+
content: dedent6`
|
|
4361
4828
|
- Analyze the full transcript to understand context and themes
|
|
4362
4829
|
- Be precise: use specific terminology when mentioned
|
|
4363
4830
|
- Capture the narrative: what is introduced, discussed, and concluded
|
|
@@ -4367,7 +4834,7 @@ function createAudioOnlyBuilder({ titleLength, descriptionLength, tagCount } = {
|
|
|
4367
4834
|
sectionOrder: ["task", "title", "description", "keywords", "qualityGuidelines"]
|
|
4368
4835
|
});
|
|
4369
4836
|
}
|
|
4370
|
-
var
|
|
4837
|
+
var SYSTEM_PROMPT5 = dedent6`
|
|
4371
4838
|
<role>
|
|
4372
4839
|
You are a video content analyst specializing in storyboard interpretation and multimodal analysis.
|
|
4373
4840
|
</role>
|
|
@@ -4424,7 +4891,7 @@ var SYSTEM_PROMPT4 = dedent5`
|
|
|
4424
4891
|
|
|
4425
4892
|
Write as if describing reality, not describing a recording of reality.
|
|
4426
4893
|
</language_guidelines>`;
|
|
4427
|
-
var
|
|
4894
|
+
var AUDIO_ONLY_SYSTEM_PROMPT2 = dedent6`
|
|
4428
4895
|
<role>
|
|
4429
4896
|
You are an audio content analyst specializing in transcript analysis and metadata generation.
|
|
4430
4897
|
</role>
|
|
@@ -4478,7 +4945,7 @@ var AUDIO_ONLY_SYSTEM_PROMPT = dedent5`
|
|
|
4478
4945
|
|
|
4479
4946
|
Write as if describing reality, not describing a recording of reality.
|
|
4480
4947
|
</language_guidelines>`;
|
|
4481
|
-
function
|
|
4948
|
+
function buildUserPrompt5({
|
|
4482
4949
|
tone,
|
|
4483
4950
|
transcriptText,
|
|
4484
4951
|
isCleanTranscript = true,
|
|
@@ -4509,7 +4976,7 @@ function buildUserPrompt4({
|
|
|
4509
4976
|
async function analyzeStoryboard2(imageDataUrl, provider, modelId, userPrompt, systemPrompt, credentials) {
|
|
4510
4977
|
"use step";
|
|
4511
4978
|
const model = await createLanguageModelFromConfig(provider, modelId, credentials);
|
|
4512
|
-
const response = await
|
|
4979
|
+
const response = await generateText6({
|
|
4513
4980
|
model,
|
|
4514
4981
|
output: SUMMARY_OUTPUT,
|
|
4515
4982
|
messages: [
|
|
@@ -4544,7 +5011,7 @@ async function analyzeStoryboard2(imageDataUrl, provider, modelId, userPrompt, s
|
|
|
4544
5011
|
async function analyzeAudioOnly(provider, modelId, userPrompt, systemPrompt, credentials) {
|
|
4545
5012
|
"use step";
|
|
4546
5013
|
const model = await createLanguageModelFromConfig(provider, modelId, credentials);
|
|
4547
|
-
const response = await
|
|
5014
|
+
const response = await generateText6({
|
|
4548
5015
|
model,
|
|
4549
5016
|
output: SUMMARY_OUTPUT,
|
|
4550
5017
|
messages: [
|
|
@@ -4647,7 +5114,7 @@ async function getSummaryAndTags(assetId, options) {
|
|
|
4647
5114
|
const transcriptText = transcriptResult?.transcriptText ?? "";
|
|
4648
5115
|
const resolvedLanguageCode = outputLanguageCode && outputLanguageCode !== "auto" ? outputLanguageCode : transcriptResult?.track?.language_code ?? getReadyTextTracks(assetData)[0]?.language_code;
|
|
4649
5116
|
const languageName = resolvedLanguageCode ? getLanguageName(resolvedLanguageCode) : void 0;
|
|
4650
|
-
const userPrompt =
|
|
5117
|
+
const userPrompt = buildUserPrompt5({
|
|
4651
5118
|
tone,
|
|
4652
5119
|
transcriptText,
|
|
4653
5120
|
isCleanTranscript: cleanTranscript,
|
|
@@ -4660,7 +5127,7 @@ async function getSummaryAndTags(assetId, options) {
|
|
|
4660
5127
|
});
|
|
4661
5128
|
let analysisResponse;
|
|
4662
5129
|
let imageUrl;
|
|
4663
|
-
const systemPrompt = isAudioOnly ?
|
|
5130
|
+
const systemPrompt = isAudioOnly ? AUDIO_ONLY_SYSTEM_PROMPT2 : SYSTEM_PROMPT5;
|
|
4664
5131
|
try {
|
|
4665
5132
|
if (isAudioOnly) {
|
|
4666
5133
|
analysisResponse = await analyzeAudioOnly(
|
|
@@ -5133,24 +5600,24 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
|
|
|
5133
5600
|
// src/workflows/translate-captions.ts
|
|
5134
5601
|
import {
|
|
5135
5602
|
APICallError,
|
|
5136
|
-
generateText as
|
|
5603
|
+
generateText as generateText7,
|
|
5137
5604
|
NoObjectGeneratedError,
|
|
5138
|
-
Output as
|
|
5605
|
+
Output as Output7,
|
|
5139
5606
|
RetryError,
|
|
5140
5607
|
TypeValidationError
|
|
5141
5608
|
} from "ai";
|
|
5142
|
-
import
|
|
5143
|
-
import { z as
|
|
5144
|
-
var translationSchema =
|
|
5145
|
-
translation:
|
|
5609
|
+
import dedent7 from "dedent";
|
|
5610
|
+
import { z as z8 } from "zod";
|
|
5611
|
+
var translationSchema = z8.object({
|
|
5612
|
+
translation: z8.string()
|
|
5146
5613
|
});
|
|
5147
|
-
var
|
|
5614
|
+
var SYSTEM_PROMPT6 = dedent7`
|
|
5148
5615
|
You are a subtitle translation expert. Translate VTT subtitle files to the target language specified by the user.
|
|
5149
5616
|
You may receive either a full VTT file or a chunk from a larger VTT.
|
|
5150
5617
|
Preserve all timestamps, cue ordering, and VTT formatting exactly as they appear.
|
|
5151
5618
|
Return JSON with a single key "translation" containing the translated VTT content.
|
|
5152
5619
|
`;
|
|
5153
|
-
var CUE_TRANSLATION_SYSTEM_PROMPT =
|
|
5620
|
+
var CUE_TRANSLATION_SYSTEM_PROMPT = dedent7`
|
|
5154
5621
|
You are a subtitle translation expert.
|
|
5155
5622
|
You will receive a sequence of subtitle cues extracted from a VTT file.
|
|
5156
5623
|
Translate the cues to the requested target language while preserving their original order.
|
|
@@ -5322,13 +5789,13 @@ async function translateVttWithAI({
|
|
|
5322
5789
|
}) {
|
|
5323
5790
|
"use step";
|
|
5324
5791
|
const model = await createLanguageModelFromConfig(provider, modelId, credentials);
|
|
5325
|
-
const response = await
|
|
5792
|
+
const response = await generateText7({
|
|
5326
5793
|
model,
|
|
5327
|
-
output:
|
|
5794
|
+
output: Output7.object({ schema: translationSchema }),
|
|
5328
5795
|
messages: [
|
|
5329
5796
|
{
|
|
5330
5797
|
role: "system",
|
|
5331
|
-
content:
|
|
5798
|
+
content: SYSTEM_PROMPT6
|
|
5332
5799
|
},
|
|
5333
5800
|
{
|
|
5334
5801
|
role: "user",
|
|
@@ -5359,8 +5826,8 @@ async function translateCueChunkWithAI({
|
|
|
5359
5826
|
}) {
|
|
5360
5827
|
"use step";
|
|
5361
5828
|
const model = await createLanguageModelFromConfig(provider, modelId, credentials);
|
|
5362
|
-
const schema =
|
|
5363
|
-
translations:
|
|
5829
|
+
const schema = z8.object({
|
|
5830
|
+
translations: z8.array(z8.string().min(1)).length(cues.length)
|
|
5364
5831
|
});
|
|
5365
5832
|
const cuePayload = cues.map((cue, index) => ({
|
|
5366
5833
|
index,
|
|
@@ -5368,9 +5835,9 @@ async function translateCueChunkWithAI({
|
|
|
5368
5835
|
endTime: cue.endTime,
|
|
5369
5836
|
text: cue.text
|
|
5370
5837
|
}));
|
|
5371
|
-
const response = await
|
|
5838
|
+
const response = await generateText7({
|
|
5372
5839
|
model,
|
|
5373
|
-
output:
|
|
5840
|
+
output: Output7.object({ schema }),
|
|
5374
5841
|
messages: [
|
|
5375
5842
|
{
|
|
5376
5843
|
role: "system",
|