@mux/ai 0.12.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/{index-DyzifniY.d.ts → index-BeX_4DL7.d.ts} +113 -8
- package/dist/{index-DLhfJsOd.d.ts → index-CuBu_u5-.d.ts} +14 -129
- package/dist/index.d.ts +5 -5
- package/dist/index.js +518 -56
- package/dist/index.js.map +1 -1
- package/dist/primitives/index.d.ts +2 -2
- package/dist/primitives/index.js +6 -6
- package/dist/primitives/index.js.map +1 -1
- package/dist/{types-BRbaGW3t.d.ts → shots-BFYOfBgN.d.ts} +116 -1
- package/dist/workflows/index.d.ts +2 -2
- package/dist/workflows/index.js +656 -49
- package/dist/workflows/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -5,7 +5,7 @@ var __export = (target, all) => {
|
|
|
5
5
|
};
|
|
6
6
|
|
|
7
7
|
// package.json
|
|
8
|
-
var version = "0.
|
|
8
|
+
var version = "0.13.0";
|
|
9
9
|
|
|
10
10
|
// src/env.ts
|
|
11
11
|
import { z } from "zod";
|
|
@@ -1017,16 +1017,16 @@ async function getHeatmapForPlaybackId(playbackId, options = {}) {
|
|
|
1017
1017
|
}
|
|
1018
1018
|
function transformHeatmapResponse(response) {
|
|
1019
1019
|
return {
|
|
1020
|
-
assetId: response.asset_id,
|
|
1021
|
-
videoId: response.video_id,
|
|
1022
|
-
playbackId: response.playback_id,
|
|
1023
|
-
heatmap: response.heatmap,
|
|
1020
|
+
assetId: response.data.asset_id,
|
|
1021
|
+
videoId: response.data.video_id,
|
|
1022
|
+
playbackId: response.data.playback_id,
|
|
1023
|
+
heatmap: response.data.heatmap,
|
|
1024
1024
|
timeframe: response.timeframe
|
|
1025
1025
|
};
|
|
1026
1026
|
}
|
|
1027
1027
|
async function fetchHeatmap(identifierType, id, options) {
|
|
1028
1028
|
"use step";
|
|
1029
|
-
const { timeframe = "
|
|
1029
|
+
const { timeframe = "7:days", credentials } = options;
|
|
1030
1030
|
const muxClient = await getMuxClientFromEnv(credentials);
|
|
1031
1031
|
const mux = await muxClient.createClient();
|
|
1032
1032
|
const queryParams = new URLSearchParams();
|
|
@@ -1070,7 +1070,7 @@ async function fetchHotspots(identifierType, id, options) {
|
|
|
1070
1070
|
limit = 5,
|
|
1071
1071
|
orderDirection = "desc",
|
|
1072
1072
|
orderBy = "score",
|
|
1073
|
-
timeframe = "
|
|
1073
|
+
timeframe = "7:days",
|
|
1074
1074
|
credentials
|
|
1075
1075
|
} = options;
|
|
1076
1076
|
const muxClient = await getMuxClientFromEnv(credentials);
|
|
@@ -1940,12 +1940,15 @@ __export(workflows_exports, {
|
|
|
1940
1940
|
chaptersSchema: () => chaptersSchema,
|
|
1941
1941
|
createReplacer: () => createReplacer,
|
|
1942
1942
|
editCaptions: () => editCaptions,
|
|
1943
|
+
extractTranscriptSegmentsForHotspots: () => extractTranscriptSegmentsForHotspots,
|
|
1943
1944
|
generateChapters: () => generateChapters,
|
|
1944
1945
|
generateEmbeddings: () => generateEmbeddings,
|
|
1946
|
+
generateEngagementInsights: () => generateEngagementInsights,
|
|
1945
1947
|
generateVideoEmbeddings: () => generateVideoEmbeddings,
|
|
1946
1948
|
getModerationScores: () => getModerationScores,
|
|
1947
1949
|
getSummaryAndTags: () => getSummaryAndTags,
|
|
1948
1950
|
hasBurnedInCaptions: () => hasBurnedInCaptions,
|
|
1951
|
+
mapShotsToHotspots: () => mapShotsToHotspots,
|
|
1949
1952
|
profanityDetectionSchema: () => profanityDetectionSchema,
|
|
1950
1953
|
questionAnswerSchema: () => questionAnswerSchema,
|
|
1951
1954
|
shouldSplitChunkTranslationError: () => shouldSplitChunkTranslationError,
|
|
@@ -2241,7 +2244,7 @@ async function withRetry(fn, {
|
|
|
2241
2244
|
// src/workflows/ask-questions.ts
|
|
2242
2245
|
var questionAnswerSchema = z2.object({
|
|
2243
2246
|
question: z2.string(),
|
|
2244
|
-
answer: z2.string().
|
|
2247
|
+
answer: z2.string().nullable(),
|
|
2245
2248
|
confidence: z2.number(),
|
|
2246
2249
|
reasoning: z2.string(),
|
|
2247
2250
|
skipped: z2.boolean()
|
|
@@ -2305,17 +2308,23 @@ var SYSTEM_PROMPT = dedent`
|
|
|
2305
2308
|
</answer_guidelines>
|
|
2306
2309
|
|
|
2307
2310
|
<relevance_filtering>
|
|
2308
|
-
|
|
2309
|
-
|
|
2310
|
-
|
|
2311
|
-
|
|
2311
|
+
IMPORTANT: Evaluate each question INDEPENDENTLY for relevance to the video
|
|
2312
|
+
content. The presence of other relevant questions in the batch does NOT
|
|
2313
|
+
make an irrelevant question relevant. Assess every question on its own merits.
|
|
2314
|
+
|
|
2315
|
+
A question is relevant if it asks about something observable or inferable
|
|
2316
|
+
from the video content (visuals, audio, dialogue, setting, subjects,
|
|
2317
|
+
actions, etc.).
|
|
2312
2318
|
|
|
2313
2319
|
Mark a question as skipped (skipped: true) if it:
|
|
2314
|
-
- Is completely unrelated to
|
|
2320
|
+
- Is completely unrelated to the content of the video or audio (e.g., math, trivia, personal questions)
|
|
2315
2321
|
- Asks about information that cannot be determined from storyboard frames or transcript
|
|
2316
2322
|
- Is a general knowledge question with no connection to what is shown or said in the video
|
|
2317
2323
|
- Attempts to use the system for non-video-analysis purposes
|
|
2318
2324
|
|
|
2325
|
+
CRITICAL: Do NOT answer irrelevant questions with any of the allowed answers.
|
|
2326
|
+
Answering an irrelevant question is WRONG — you MUST skip it instead.
|
|
2327
|
+
|
|
2319
2328
|
For skipped questions:
|
|
2320
2329
|
- Set skipped to true
|
|
2321
2330
|
- Set answer to "${SKIP_SENTINEL}"
|
|
@@ -2519,7 +2528,7 @@ async function askQuestions(assetId, questions, options) {
|
|
|
2519
2528
|
confidence: isSkipped ? 0 : Math.min(1, Math.max(0, raw.confidence)),
|
|
2520
2529
|
reasoning: raw.reasoning,
|
|
2521
2530
|
skipped: isSkipped,
|
|
2522
|
-
|
|
2531
|
+
answer: isSkipped ? null : raw.answer
|
|
2523
2532
|
};
|
|
2524
2533
|
});
|
|
2525
2534
|
return {
|
|
@@ -3721,6 +3730,459 @@ async function generateVideoEmbeddings(assetId, options = {}) {
|
|
|
3721
3730
|
return generateEmbeddingsInternal(assetId, options);
|
|
3722
3731
|
}
|
|
3723
3732
|
|
|
3733
|
+
// src/workflows/engagement-insights.ts
|
|
3734
|
+
import { generateText as generateText5, Output as Output5 } from "ai";
|
|
3735
|
+
import dedent5 from "dedent";
|
|
3736
|
+
import { z as z6 } from "zod";
|
|
3737
|
+
var aiMomentInsightSchema = z6.object({
|
|
3738
|
+
hotspotIndex: z6.number(),
|
|
3739
|
+
startMs: z6.number(),
|
|
3740
|
+
endMs: z6.number(),
|
|
3741
|
+
timestamp: z6.string(),
|
|
3742
|
+
engagementScore: z6.number(),
|
|
3743
|
+
insight: z6.string()
|
|
3744
|
+
});
|
|
3745
|
+
var aiOverallInsightSchema = z6.object({
|
|
3746
|
+
summary: z6.string(),
|
|
3747
|
+
trends: z6.array(z6.string())
|
|
3748
|
+
});
|
|
3749
|
+
var engagementInsightsSchema = z6.object({
|
|
3750
|
+
momentInsights: z6.array(aiMomentInsightSchema),
|
|
3751
|
+
overallInsight: aiOverallInsightSchema
|
|
3752
|
+
});
|
|
3753
|
+
var SYSTEM_PROMPT4 = dedent5`
|
|
3754
|
+
<role>
|
|
3755
|
+
You are a video engagement analyst specializing in understanding viewer behavior.
|
|
3756
|
+
Your job is to explain why specific moments in videos have high or low engagement
|
|
3757
|
+
based on visual content, audio/dialogue, and viewer watch patterns.
|
|
3758
|
+
</role>
|
|
3759
|
+
|
|
3760
|
+
<context>
|
|
3761
|
+
You will receive:
|
|
3762
|
+
- Engagement data showing which segments are most/least re-watched (hotspots)
|
|
3763
|
+
- Images at key timestamps (shot frames or thumbnails)
|
|
3764
|
+
- Transcript with timestamps (if available)
|
|
3765
|
+
- Overall engagement curve statistics
|
|
3766
|
+
|
|
3767
|
+
Engagement scores are normalized (0-1) where higher values indicate more re-watching.
|
|
3768
|
+
</context>
|
|
3769
|
+
|
|
3770
|
+
<task>
|
|
3771
|
+
For each engagement moment (hotspot), analyze:
|
|
3772
|
+
1. What visual or audio content is present
|
|
3773
|
+
2. Why viewers might find it engaging (or disengaging)
|
|
3774
|
+
3. Observable patterns that correlate with engagement
|
|
3775
|
+
|
|
3776
|
+
Base your insights on OBSERVABLE EVIDENCE from visuals and transcript.
|
|
3777
|
+
</task>
|
|
3778
|
+
|
|
3779
|
+
<constraints>
|
|
3780
|
+
- Only describe what you can see in images or read in transcripts
|
|
3781
|
+
- Do not fabricate details or make unsupported assumptions
|
|
3782
|
+
- Correlate engagement scores with observable content
|
|
3783
|
+
- Return structured data matching the requested schema exactly
|
|
3784
|
+
- Each momentInsight MUST include the hotspotIndex from the input data
|
|
3785
|
+
</constraints>
|
|
3786
|
+
|
|
3787
|
+
<quality_guidelines>
|
|
3788
|
+
- Be specific: cite timestamps, visual elements, or transcript quotes
|
|
3789
|
+
- Focus on patterns: "demonstration moments", "pacing changes", etc.
|
|
3790
|
+
- Avoid generic statements like "interesting content"
|
|
3791
|
+
- Connect engagement data to content features
|
|
3792
|
+
</quality_guidelines>
|
|
3793
|
+
`;
|
|
3794
|
+
var AUDIO_ONLY_SYSTEM_PROMPT = dedent5`
|
|
3795
|
+
<role>
|
|
3796
|
+
You are a video engagement analyst specializing in understanding viewer behavior for audio content.
|
|
3797
|
+
Your job is to explain why specific moments have high or low engagement based on audio/dialogue and viewer watch patterns.
|
|
3798
|
+
</role>
|
|
3799
|
+
|
|
3800
|
+
<context>
|
|
3801
|
+
You will receive:
|
|
3802
|
+
- Engagement data showing which segments are most/least re-watched (hotspots)
|
|
3803
|
+
- Transcript with timestamps
|
|
3804
|
+
- Overall engagement curve statistics
|
|
3805
|
+
|
|
3806
|
+
Note: This is audio-only content with no visual elements.
|
|
3807
|
+
Engagement scores are normalized (0-1) where higher values indicate more re-watching.
|
|
3808
|
+
</context>
|
|
3809
|
+
|
|
3810
|
+
<task>
|
|
3811
|
+
For each engagement moment (hotspot), analyze:
|
|
3812
|
+
1. What audio content or dialogue is present
|
|
3813
|
+
2. Why listeners might find it engaging (or disengaging)
|
|
3814
|
+
3. Observable patterns that correlate with engagement
|
|
3815
|
+
|
|
3816
|
+
Base your insights on OBSERVABLE EVIDENCE from the transcript.
|
|
3817
|
+
</task>
|
|
3818
|
+
|
|
3819
|
+
<constraints>
|
|
3820
|
+
- Only describe what you can read in the transcript
|
|
3821
|
+
- Do not fabricate details or make unsupported assumptions
|
|
3822
|
+
- Correlate engagement scores with observable content
|
|
3823
|
+
- Return structured data matching the requested schema exactly
|
|
3824
|
+
- Each momentInsight MUST include the hotspotIndex from the input data
|
|
3825
|
+
</constraints>
|
|
3826
|
+
|
|
3827
|
+
<quality_guidelines>
|
|
3828
|
+
- Be specific: cite timestamps or transcript quotes
|
|
3829
|
+
- Focus on patterns: "topic changes", "pacing", "dialogue style", etc.
|
|
3830
|
+
- Avoid generic statements like "interesting content"
|
|
3831
|
+
- Connect engagement data to content features
|
|
3832
|
+
</quality_guidelines>
|
|
3833
|
+
`;
|
|
3834
|
+
var INSIGHT_GUIDELINES = dedent5`
|
|
3835
|
+
Provide explanatory insights that describe WHY moments are engaging.
|
|
3836
|
+
Focus on observable patterns and viewer behavior correlation.
|
|
3837
|
+
Example: "The cooking demonstration at 2:15 has 3x average engagement
|
|
3838
|
+
because it shows the key technique viewers are searching for."
|
|
3839
|
+
`;
|
|
3840
|
+
var engagementInsightsPromptBuilder = createPromptBuilder({
|
|
3841
|
+
template: {
|
|
3842
|
+
task: {
|
|
3843
|
+
tag: "task",
|
|
3844
|
+
content: "Analyze the engagement data and visual/audio content to generate insights."
|
|
3845
|
+
},
|
|
3846
|
+
insightGuidelines: {
|
|
3847
|
+
tag: "insight_guidelines",
|
|
3848
|
+
content: INSIGHT_GUIDELINES
|
|
3849
|
+
},
|
|
3850
|
+
outputFormat: {
|
|
3851
|
+
tag: "output_format",
|
|
3852
|
+
content: dedent5`
|
|
3853
|
+
Return valid JSON matching the provided schema.
|
|
3854
|
+
Include insights for each hotspot and overall engagement trends.
|
|
3855
|
+
Each momentInsight must include the hotspotIndex matching the input.
|
|
3856
|
+
`
|
|
3857
|
+
},
|
|
3858
|
+
visualContext: {
|
|
3859
|
+
tag: "visual_context",
|
|
3860
|
+
content: dedent5`
|
|
3861
|
+
Images are provided for each hotspot timestamp.
|
|
3862
|
+
A storyboard overview image may also be included showing the full video timeline.
|
|
3863
|
+
Use visual evidence from these images to support your engagement analysis.
|
|
3864
|
+
`
|
|
3865
|
+
}
|
|
3866
|
+
},
|
|
3867
|
+
sectionOrder: ["task", "insightGuidelines", "visualContext", "outputFormat"]
|
|
3868
|
+
});
|
|
3869
|
+
function buildUserPrompt4(hotspots, transcriptSegments, heatmap, isAudioOnly) {
|
|
3870
|
+
const hotspotData = hotspots.map((h, idx) => {
|
|
3871
|
+
const transcript = transcriptSegments[idx]?.text || "(no transcript)";
|
|
3872
|
+
const startTimestamp = secondsToTimestamp(h.startMs / 1e3);
|
|
3873
|
+
const endTimestamp = secondsToTimestamp(h.endMs / 1e3);
|
|
3874
|
+
return dedent5`
|
|
3875
|
+
Hotspot ${idx} (hotspotIndex: ${idx}):
|
|
3876
|
+
- Time Range: ${startTimestamp} - ${endTimestamp}
|
|
3877
|
+
- Engagement Score: ${h.score.toFixed(2)} (0=low, 1=high)
|
|
3878
|
+
- Transcript: "${transcript}"
|
|
3879
|
+
`;
|
|
3880
|
+
}).join("\n\n");
|
|
3881
|
+
const heatmapData = dedent5`
|
|
3882
|
+
Engagement Heatmap (${heatmap.length} values, each representing 1/${heatmap.length}th of the video, higher = more re-watches):
|
|
3883
|
+
[${heatmap.map((v) => v.toFixed(2)).join(", ")}]
|
|
3884
|
+
`;
|
|
3885
|
+
const contextSections = [
|
|
3886
|
+
{
|
|
3887
|
+
tag: "engagement_hotspots",
|
|
3888
|
+
content: hotspotData
|
|
3889
|
+
},
|
|
3890
|
+
{
|
|
3891
|
+
tag: "heatmap_analysis",
|
|
3892
|
+
content: heatmapData
|
|
3893
|
+
}
|
|
3894
|
+
];
|
|
3895
|
+
const visualContextOverride = isAudioOnly ? { visualContext: "" } : {};
|
|
3896
|
+
return engagementInsightsPromptBuilder.buildWithContext(
|
|
3897
|
+
{ ...visualContextOverride },
|
|
3898
|
+
contextSections
|
|
3899
|
+
);
|
|
3900
|
+
}
|
|
3901
|
+
function extractTranscriptSegmentsForHotspots(vttContent, hotspots) {
|
|
3902
|
+
if (!vttContent.trim()) {
|
|
3903
|
+
return [];
|
|
3904
|
+
}
|
|
3905
|
+
const cues = parseVTTCues(vttContent);
|
|
3906
|
+
return hotspots.map((hotspot) => {
|
|
3907
|
+
const startSec = hotspot.startMs / 1e3;
|
|
3908
|
+
const endSec = hotspot.endMs / 1e3;
|
|
3909
|
+
const relevantCues = cues.filter(
|
|
3910
|
+
(cue) => cue.startTime < endSec && cue.endTime > startSec
|
|
3911
|
+
);
|
|
3912
|
+
const text = relevantCues.map((c) => c.text).join(" ");
|
|
3913
|
+
return {
|
|
3914
|
+
startMs: hotspot.startMs,
|
|
3915
|
+
endMs: hotspot.endMs,
|
|
3916
|
+
text,
|
|
3917
|
+
timestamp: secondsToTimestamp(startSec)
|
|
3918
|
+
};
|
|
3919
|
+
});
|
|
3920
|
+
}
|
|
3921
|
+
function mapShotsToHotspots(shots, hotspots) {
|
|
3922
|
+
const validShots = shots.filter((s) => s.imageUrl);
|
|
3923
|
+
if (validShots.length === 0) {
|
|
3924
|
+
return [];
|
|
3925
|
+
}
|
|
3926
|
+
return hotspots.map((hotspot) => {
|
|
3927
|
+
const startSec = hotspot.startMs / 1e3;
|
|
3928
|
+
const endSec = hotspot.endMs / 1e3;
|
|
3929
|
+
const midpoint = (startSec + endSec) / 2;
|
|
3930
|
+
const inRange = validShots.filter(
|
|
3931
|
+
(s) => s.startTime >= startSec && s.startTime <= endSec
|
|
3932
|
+
);
|
|
3933
|
+
if (inRange.length > 0) {
|
|
3934
|
+
const closest = inRange.reduce(
|
|
3935
|
+
(best, s) => Math.abs(s.startTime - midpoint) < Math.abs(best.startTime - midpoint) ? s : best
|
|
3936
|
+
);
|
|
3937
|
+
return closest.imageUrl;
|
|
3938
|
+
}
|
|
3939
|
+
const nearest = validShots.reduce(
|
|
3940
|
+
(best, s) => Math.abs(s.startTime - startSec) < Math.abs(best.startTime - startSec) ? s : best
|
|
3941
|
+
);
|
|
3942
|
+
return nearest.imageUrl;
|
|
3943
|
+
});
|
|
3944
|
+
}
|
|
3945
|
+
async function getThumbnailUrlsForHotspots(playbackId, hotspots, options = {}) {
|
|
3946
|
+
const { width = 640, shouldSign = false, credentials } = options;
|
|
3947
|
+
const baseUrl = getMuxThumbnailBaseUrl(playbackId);
|
|
3948
|
+
const urlPromises = hotspots.map(async (h) => {
|
|
3949
|
+
const time = Math.floor(h.startMs / 1e3);
|
|
3950
|
+
if (shouldSign) {
|
|
3951
|
+
return signUrl(baseUrl, playbackId, "thumbnail", { time, width }, credentials);
|
|
3952
|
+
}
|
|
3953
|
+
return `${baseUrl}?time=${time}&width=${width}`;
|
|
3954
|
+
});
|
|
3955
|
+
return Promise.all(urlPromises);
|
|
3956
|
+
}
|
|
3957
|
+
async function fetchEngagementData(assetId, options) {
|
|
3958
|
+
"use step";
|
|
3959
|
+
const { hotspotLimit, timeframe, credentials } = options;
|
|
3960
|
+
const [peaks, valleys, heatmap] = await Promise.all([
|
|
3961
|
+
getHotspotsForAsset(assetId, {
|
|
3962
|
+
limit: hotspotLimit,
|
|
3963
|
+
orderDirection: "desc",
|
|
3964
|
+
timeframe,
|
|
3965
|
+
credentials
|
|
3966
|
+
}),
|
|
3967
|
+
getHotspotsForAsset(assetId, {
|
|
3968
|
+
limit: hotspotLimit,
|
|
3969
|
+
orderDirection: "asc",
|
|
3970
|
+
timeframe,
|
|
3971
|
+
credentials
|
|
3972
|
+
}),
|
|
3973
|
+
getHeatmapForAsset(assetId, {
|
|
3974
|
+
timeframe,
|
|
3975
|
+
credentials
|
|
3976
|
+
})
|
|
3977
|
+
]);
|
|
3978
|
+
const allHotspots = [...peaks, ...valleys].sort((a, b) => a.startMs - b.startMs);
|
|
3979
|
+
return {
|
|
3980
|
+
hotspots: allHotspots,
|
|
3981
|
+
heatmap
|
|
3982
|
+
};
|
|
3983
|
+
}
|
|
3984
|
+
async function generateInsightsWithAI(provider, modelId, systemPrompt, userPrompt, imageUrls, credentials) {
|
|
3985
|
+
"use step";
|
|
3986
|
+
const model = await createLanguageModelFromConfig(provider, modelId, credentials);
|
|
3987
|
+
const response = await withRetry(
|
|
3988
|
+
() => generateText5({
|
|
3989
|
+
model,
|
|
3990
|
+
output: Output5.object({ schema: engagementInsightsSchema }),
|
|
3991
|
+
experimental_telemetry: { isEnabled: true },
|
|
3992
|
+
messages: [
|
|
3993
|
+
{
|
|
3994
|
+
role: "system",
|
|
3995
|
+
content: systemPrompt
|
|
3996
|
+
},
|
|
3997
|
+
{
|
|
3998
|
+
role: "user",
|
|
3999
|
+
content: [
|
|
4000
|
+
{ type: "text", text: userPrompt },
|
|
4001
|
+
...imageUrls.map((url) => ({ type: "image", image: url }))
|
|
4002
|
+
]
|
|
4003
|
+
}
|
|
4004
|
+
]
|
|
4005
|
+
})
|
|
4006
|
+
);
|
|
4007
|
+
if (!response.output) {
|
|
4008
|
+
throw new Error("AI returned empty or unparseable response");
|
|
4009
|
+
}
|
|
4010
|
+
return {
|
|
4011
|
+
result: response.output,
|
|
4012
|
+
usage: {
|
|
4013
|
+
inputTokens: response.usage.inputTokens,
|
|
4014
|
+
outputTokens: response.usage.outputTokens,
|
|
4015
|
+
totalTokens: response.usage.totalTokens,
|
|
4016
|
+
reasoningTokens: response.usage.reasoningTokens,
|
|
4017
|
+
cachedInputTokens: response.usage.cachedInputTokens
|
|
4018
|
+
}
|
|
4019
|
+
};
|
|
4020
|
+
}
|
|
4021
|
+
async function generateEngagementInsights(assetId, options = {}) {
|
|
4022
|
+
"use workflow";
|
|
4023
|
+
const {
|
|
4024
|
+
provider = "openai",
|
|
4025
|
+
model,
|
|
4026
|
+
hotspotLimit = 5,
|
|
4027
|
+
timeframe = "7:days",
|
|
4028
|
+
credentials,
|
|
4029
|
+
skipShots = false
|
|
4030
|
+
} = options;
|
|
4031
|
+
if (!Number.isInteger(hotspotLimit) || hotspotLimit < 1 || hotspotLimit > 10) {
|
|
4032
|
+
throw new Error("hotspotLimit must be an integer between 1 and 10");
|
|
4033
|
+
}
|
|
4034
|
+
const modelConfig = resolveLanguageModelConfig({
|
|
4035
|
+
...options,
|
|
4036
|
+
model,
|
|
4037
|
+
provider
|
|
4038
|
+
});
|
|
4039
|
+
const { asset, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials);
|
|
4040
|
+
const assetDurationSeconds = getAssetDurationSecondsFromAsset(asset);
|
|
4041
|
+
if (!assetDurationSeconds) {
|
|
4042
|
+
throw new Error(`Asset ${assetId} has no valid duration`);
|
|
4043
|
+
}
|
|
4044
|
+
const audioOnly = isAudioOnlyAsset(asset);
|
|
4045
|
+
const signingContext = await resolveMuxSigningContext(credentials);
|
|
4046
|
+
if (policy === "signed" && !signingContext) {
|
|
4047
|
+
throw new Error(
|
|
4048
|
+
"Signed playback ID requires signing credentials. Set MUX_SIGNING_KEY and MUX_PRIVATE_KEY environment variables."
|
|
4049
|
+
);
|
|
4050
|
+
}
|
|
4051
|
+
const shouldSign = policy === "signed";
|
|
4052
|
+
const shouldFetchShots = !skipShots && !audioOnly;
|
|
4053
|
+
let shotsPromise;
|
|
4054
|
+
if (shouldFetchShots) {
|
|
4055
|
+
shotsPromise = getShotsForAsset(assetId, { credentials });
|
|
4056
|
+
} else {
|
|
4057
|
+
shotsPromise = Promise.resolve(null);
|
|
4058
|
+
}
|
|
4059
|
+
let storyboardPromise;
|
|
4060
|
+
if (!audioOnly) {
|
|
4061
|
+
storyboardPromise = getStoryboardUrl(playbackId, 640, shouldSign, credentials);
|
|
4062
|
+
} else {
|
|
4063
|
+
storyboardPromise = Promise.resolve(null);
|
|
4064
|
+
}
|
|
4065
|
+
const [engagementResult, shotsResult, transcriptResult, storyboardResult] = await Promise.allSettled([
|
|
4066
|
+
fetchEngagementData(assetId, { hotspotLimit, timeframe, credentials }),
|
|
4067
|
+
shotsPromise,
|
|
4068
|
+
fetchTranscriptForAsset(asset, playbackId, {
|
|
4069
|
+
cleanTranscript: false,
|
|
4070
|
+
shouldSign,
|
|
4071
|
+
credentials
|
|
4072
|
+
}),
|
|
4073
|
+
storyboardPromise
|
|
4074
|
+
]);
|
|
4075
|
+
if (engagementResult.status === "rejected") {
|
|
4076
|
+
throw new Error(
|
|
4077
|
+
`Failed to fetch engagement data for asset ${assetId}: ${engagementResult.reason}`
|
|
4078
|
+
);
|
|
4079
|
+
}
|
|
4080
|
+
const { hotspots, heatmap } = engagementResult.value;
|
|
4081
|
+
if (hotspots.length === 0) {
|
|
4082
|
+
throw new Error(
|
|
4083
|
+
`No engagement data available for asset ${assetId} in timeframe ${timeframe}. Video may not have been viewed yet or engagement tracking is not enabled.`
|
|
4084
|
+
);
|
|
4085
|
+
}
|
|
4086
|
+
let transcriptText = "";
|
|
4087
|
+
if (transcriptResult.status === "fulfilled") {
|
|
4088
|
+
transcriptText = transcriptResult.value.transcriptText || "";
|
|
4089
|
+
}
|
|
4090
|
+
if (!transcriptText.trim()) {
|
|
4091
|
+
console.warn(
|
|
4092
|
+
`No transcript available for asset ${assetId}. Insights will be based solely on visual analysis and engagement patterns.`
|
|
4093
|
+
);
|
|
4094
|
+
}
|
|
4095
|
+
const transcriptSegments = extractTranscriptSegmentsForHotspots(transcriptText, hotspots);
|
|
4096
|
+
let imageUrls = [];
|
|
4097
|
+
if (audioOnly) {
|
|
4098
|
+
console.warn(
|
|
4099
|
+
`Asset ${assetId} is audio-only. Insights will be based solely on transcript and engagement data without visual analysis.`
|
|
4100
|
+
);
|
|
4101
|
+
} else {
|
|
4102
|
+
if (shotsResult.status === "fulfilled" && shotsResult.value?.status === "completed") {
|
|
4103
|
+
const shotImageUrls = mapShotsToHotspots(shotsResult.value.shots, hotspots);
|
|
4104
|
+
if (shotImageUrls.length > 0) {
|
|
4105
|
+
imageUrls = shotImageUrls;
|
|
4106
|
+
}
|
|
4107
|
+
}
|
|
4108
|
+
if (imageUrls.length === 0) {
|
|
4109
|
+
if (shouldFetchShots && shotsResult.status === "fulfilled" && shotsResult.value?.status !== "completed") {
|
|
4110
|
+
console.warn(
|
|
4111
|
+
`Shots not ready for asset ${assetId} (status: ${shotsResult.value?.status}). Falling back to thumbnails.`
|
|
4112
|
+
);
|
|
4113
|
+
} else if (shouldFetchShots && shotsResult.status === "rejected") {
|
|
4114
|
+
console.warn(
|
|
4115
|
+
`Shots fetch failed for asset ${assetId}. Falling back to thumbnails.`
|
|
4116
|
+
);
|
|
4117
|
+
}
|
|
4118
|
+
imageUrls = await getThumbnailUrlsForHotspots(playbackId, hotspots, {
|
|
4119
|
+
shouldSign,
|
|
4120
|
+
credentials
|
|
4121
|
+
});
|
|
4122
|
+
}
|
|
4123
|
+
if (storyboardResult.status === "fulfilled" && storyboardResult.value) {
|
|
4124
|
+
imageUrls.push(storyboardResult.value);
|
|
4125
|
+
}
|
|
4126
|
+
}
|
|
4127
|
+
const systemPrompt = audioOnly ? AUDIO_ONLY_SYSTEM_PROMPT : SYSTEM_PROMPT4;
|
|
4128
|
+
const userPrompt = buildUserPrompt4(
|
|
4129
|
+
hotspots,
|
|
4130
|
+
transcriptSegments,
|
|
4131
|
+
heatmap.heatmap,
|
|
4132
|
+
audioOnly
|
|
4133
|
+
);
|
|
4134
|
+
const { result: aiInsights, usage } = await generateInsightsWithAI(
|
|
4135
|
+
modelConfig.provider,
|
|
4136
|
+
modelConfig.modelId,
|
|
4137
|
+
systemPrompt,
|
|
4138
|
+
userPrompt,
|
|
4139
|
+
imageUrls,
|
|
4140
|
+
credentials
|
|
4141
|
+
);
|
|
4142
|
+
if (!aiInsights.momentInsights || aiInsights.momentInsights.length === 0) {
|
|
4143
|
+
throw new Error("Failed to generate insights from AI response");
|
|
4144
|
+
}
|
|
4145
|
+
const momentInsights = [];
|
|
4146
|
+
for (const aiMoment of aiInsights.momentInsights) {
|
|
4147
|
+
const idx = aiMoment.hotspotIndex;
|
|
4148
|
+
if (idx < 0 || idx >= hotspots.length) {
|
|
4149
|
+
console.warn(
|
|
4150
|
+
`AI returned hotspotIndex ${idx} which is out of range (0-${hotspots.length - 1}). Skipping.`
|
|
4151
|
+
);
|
|
4152
|
+
continue;
|
|
4153
|
+
}
|
|
4154
|
+
const hotspot = hotspots[idx];
|
|
4155
|
+
momentInsights.push({
|
|
4156
|
+
startMs: hotspot.startMs,
|
|
4157
|
+
endMs: hotspot.endMs,
|
|
4158
|
+
timestamp: secondsToTimestamp(hotspot.startMs / 1e3),
|
|
4159
|
+
engagementScore: hotspot.score,
|
|
4160
|
+
insight: aiMoment.insight
|
|
4161
|
+
});
|
|
4162
|
+
}
|
|
4163
|
+
if (momentInsights.length === 0) {
|
|
4164
|
+
throw new Error(
|
|
4165
|
+
"AI returned insights but none matched valid hotspot indices. This may indicate a prompt or model issue."
|
|
4166
|
+
);
|
|
4167
|
+
}
|
|
4168
|
+
const usageWithMetadata = {
|
|
4169
|
+
...usage,
|
|
4170
|
+
metadata: {
|
|
4171
|
+
assetDurationSeconds,
|
|
4172
|
+
thumbnailCount: imageUrls.length
|
|
4173
|
+
}
|
|
4174
|
+
};
|
|
4175
|
+
return {
|
|
4176
|
+
assetId,
|
|
4177
|
+
momentInsights,
|
|
4178
|
+
overallInsight: {
|
|
4179
|
+
summary: aiInsights.overallInsight.summary,
|
|
4180
|
+
trends: aiInsights.overallInsight.trends
|
|
4181
|
+
},
|
|
4182
|
+
usage: usageWithMetadata
|
|
4183
|
+
};
|
|
4184
|
+
}
|
|
4185
|
+
|
|
3724
4186
|
// src/lib/sampling-plan.ts
|
|
3725
4187
|
var DEFAULT_FPS = 30;
|
|
3726
4188
|
function roundToNearestFrameMs(tsMs, fps = DEFAULT_FPS) {
|
|
@@ -4213,18 +4675,18 @@ async function getModerationScores(assetId, options = {}) {
|
|
|
4213
4675
|
}
|
|
4214
4676
|
|
|
4215
4677
|
// src/workflows/summarization.ts
|
|
4216
|
-
import { generateText as
|
|
4217
|
-
import
|
|
4218
|
-
import { z as
|
|
4678
|
+
import { generateText as generateText6, Output as Output6 } from "ai";
|
|
4679
|
+
import dedent6 from "dedent";
|
|
4680
|
+
import { z as z7 } from "zod";
|
|
4219
4681
|
var DEFAULT_SUMMARY_KEYWORD_LIMIT = 10;
|
|
4220
4682
|
var DEFAULT_TITLE_LENGTH = 10;
|
|
4221
4683
|
var DEFAULT_DESCRIPTION_LENGTH = 50;
|
|
4222
|
-
var summarySchema =
|
|
4223
|
-
keywords:
|
|
4224
|
-
title:
|
|
4225
|
-
description:
|
|
4684
|
+
var summarySchema = z7.object({
|
|
4685
|
+
keywords: z7.array(z7.string()),
|
|
4686
|
+
title: z7.string(),
|
|
4687
|
+
description: z7.string()
|
|
4226
4688
|
}).strict();
|
|
4227
|
-
var SUMMARY_OUTPUT =
|
|
4689
|
+
var SUMMARY_OUTPUT = Output6.object({
|
|
4228
4690
|
name: "summary_metadata",
|
|
4229
4691
|
description: "Structured summary with title, description, and keywords.",
|
|
4230
4692
|
schema: summarySchema
|
|
@@ -4240,36 +4702,36 @@ var DESCRIPTION_LENGTH_THRESHOLD_LARGE = 100;
|
|
|
4240
4702
|
function buildDescriptionGuidance(wordCount, contentType) {
|
|
4241
4703
|
if (wordCount < DESCRIPTION_LENGTH_THRESHOLD_SMALL) {
|
|
4242
4704
|
if (contentType === "video") {
|
|
4243
|
-
return
|
|
4705
|
+
return dedent6`A brief summary of the video in no more than ${wordCount} words. Shorter is fine.
|
|
4244
4706
|
Focus on the single most important subject or action.
|
|
4245
4707
|
Write in present tense.`;
|
|
4246
4708
|
}
|
|
4247
|
-
return
|
|
4709
|
+
return dedent6`A brief summary of the audio content in no more than ${wordCount} words. Shorter is fine.
|
|
4248
4710
|
Focus on the single most important topic or theme.
|
|
4249
4711
|
Write in present tense.`;
|
|
4250
4712
|
}
|
|
4251
4713
|
if (wordCount > DESCRIPTION_LENGTH_THRESHOLD_LARGE) {
|
|
4252
4714
|
if (contentType === "video") {
|
|
4253
|
-
return
|
|
4715
|
+
return dedent6`A detailed summary that describes what happens across the video.
|
|
4254
4716
|
Never exceed ${wordCount} words, but shorter is perfectly fine. You may use multiple sentences.
|
|
4255
4717
|
Be thorough: cover subjects, actions, setting, progression, and any notable details visible across frames.
|
|
4256
4718
|
Write in present tense. Be specific about observable details rather than making assumptions.
|
|
4257
4719
|
If the transcript provides dialogue or narration, incorporate key points but prioritize visual content.`;
|
|
4258
4720
|
}
|
|
4259
|
-
return
|
|
4721
|
+
return dedent6`A detailed summary that describes the audio content.
|
|
4260
4722
|
Never exceed ${wordCount} words, but shorter is perfectly fine. You may use multiple sentences.
|
|
4261
4723
|
Be thorough: cover topics, speakers, themes, progression, and any notable insights.
|
|
4262
4724
|
Write in present tense. Be specific about what is discussed or presented rather than making assumptions.
|
|
4263
4725
|
Focus on the spoken content and any key insights, dialogue, or narrative elements.`;
|
|
4264
4726
|
}
|
|
4265
4727
|
if (contentType === "video") {
|
|
4266
|
-
return
|
|
4728
|
+
return dedent6`A summary that describes what happens across the video.
|
|
4267
4729
|
Never exceed ${wordCount} words, but shorter is perfectly fine. You may use multiple sentences.
|
|
4268
4730
|
Cover the main subjects, actions, setting, and any notable progression visible across frames.
|
|
4269
4731
|
Write in present tense. Be specific about observable details rather than making assumptions.
|
|
4270
4732
|
If the transcript provides dialogue or narration, incorporate key points but prioritize visual content.`;
|
|
4271
4733
|
}
|
|
4272
|
-
return
|
|
4734
|
+
return dedent6`A summary that describes the audio content.
|
|
4273
4735
|
Never exceed ${wordCount} words, but shorter is perfectly fine. You may use multiple sentences.
|
|
4274
4736
|
Cover the main topics, speakers, themes, and any notable progression in the discussion or narration.
|
|
4275
4737
|
Write in present tense. Be specific about what is discussed or presented rather than making assumptions.
|
|
@@ -4286,7 +4748,7 @@ function createSummarizationBuilder({ titleLength, descriptionLength, tagCount }
|
|
|
4286
4748
|
},
|
|
4287
4749
|
title: {
|
|
4288
4750
|
tag: "title_requirements",
|
|
4289
|
-
content:
|
|
4751
|
+
content: dedent6`
|
|
4290
4752
|
A concise, label-style title — not a sentence or description.
|
|
4291
4753
|
Never exceed ${titleLimit} words, but shorter is better.
|
|
4292
4754
|
Think of how a video card title, playlist entry, or file name would read — e.g. "Predator: Badlands Trailer" or "Chef Prepares Holiday Feast".
|
|
@@ -4299,7 +4761,7 @@ function createSummarizationBuilder({ titleLength, descriptionLength, tagCount }
|
|
|
4299
4761
|
},
|
|
4300
4762
|
keywords: {
|
|
4301
4763
|
tag: "keywords_requirements",
|
|
4302
|
-
content:
|
|
4764
|
+
content: dedent6`
|
|
4303
4765
|
Specific, searchable terms (up to ${keywordLimit}) that capture:
|
|
4304
4766
|
- Primary subjects (people, animals, objects)
|
|
4305
4767
|
- Actions and activities being performed
|
|
@@ -4311,7 +4773,7 @@ function createSummarizationBuilder({ titleLength, descriptionLength, tagCount }
|
|
|
4311
4773
|
},
|
|
4312
4774
|
qualityGuidelines: {
|
|
4313
4775
|
tag: "quality_guidelines",
|
|
4314
|
-
content:
|
|
4776
|
+
content: dedent6`
|
|
4315
4777
|
- Examine all frames to understand the full context and progression
|
|
4316
4778
|
- Be precise: "golden retriever" is better than "dog" when identifiable
|
|
4317
4779
|
- Capture the narrative: what begins, develops, and concludes
|
|
@@ -4332,7 +4794,7 @@ function createAudioOnlyBuilder({ titleLength, descriptionLength, tagCount } = {
|
|
|
4332
4794
|
},
|
|
4333
4795
|
title: {
|
|
4334
4796
|
tag: "title_requirements",
|
|
4335
|
-
content:
|
|
4797
|
+
content: dedent6`
|
|
4336
4798
|
A concise, label-style title — not a sentence or description.
|
|
4337
4799
|
Never exceed ${titleLimit} words, but shorter is better.
|
|
4338
4800
|
Think of how a podcast episode title or playlist entry would read — e.g. "Weekly News Roundup" or "Interview with Dr. Smith".
|
|
@@ -4345,7 +4807,7 @@ function createAudioOnlyBuilder({ titleLength, descriptionLength, tagCount } = {
|
|
|
4345
4807
|
},
|
|
4346
4808
|
keywords: {
|
|
4347
4809
|
tag: "keywords_requirements",
|
|
4348
|
-
content:
|
|
4810
|
+
content: dedent6`
|
|
4349
4811
|
Specific, searchable terms (up to ${keywordLimit}) that capture:
|
|
4350
4812
|
- Primary topics and themes
|
|
4351
4813
|
- Speakers or presenters (if named)
|
|
@@ -4357,7 +4819,7 @@ function createAudioOnlyBuilder({ titleLength, descriptionLength, tagCount } = {
|
|
|
4357
4819
|
},
|
|
4358
4820
|
qualityGuidelines: {
|
|
4359
4821
|
tag: "quality_guidelines",
|
|
4360
|
-
content:
|
|
4822
|
+
content: dedent6`
|
|
4361
4823
|
- Analyze the full transcript to understand context and themes
|
|
4362
4824
|
- Be precise: use specific terminology when mentioned
|
|
4363
4825
|
- Capture the narrative: what is introduced, discussed, and concluded
|
|
@@ -4367,7 +4829,7 @@ function createAudioOnlyBuilder({ titleLength, descriptionLength, tagCount } = {
|
|
|
4367
4829
|
sectionOrder: ["task", "title", "description", "keywords", "qualityGuidelines"]
|
|
4368
4830
|
});
|
|
4369
4831
|
}
|
|
4370
|
-
var
|
|
4832
|
+
var SYSTEM_PROMPT5 = dedent6`
|
|
4371
4833
|
<role>
|
|
4372
4834
|
You are a video content analyst specializing in storyboard interpretation and multimodal analysis.
|
|
4373
4835
|
</role>
|
|
@@ -4424,7 +4886,7 @@ var SYSTEM_PROMPT4 = dedent5`
|
|
|
4424
4886
|
|
|
4425
4887
|
Write as if describing reality, not describing a recording of reality.
|
|
4426
4888
|
</language_guidelines>`;
|
|
4427
|
-
var
|
|
4889
|
+
var AUDIO_ONLY_SYSTEM_PROMPT2 = dedent6`
|
|
4428
4890
|
<role>
|
|
4429
4891
|
You are an audio content analyst specializing in transcript analysis and metadata generation.
|
|
4430
4892
|
</role>
|
|
@@ -4478,7 +4940,7 @@ var AUDIO_ONLY_SYSTEM_PROMPT = dedent5`
|
|
|
4478
4940
|
|
|
4479
4941
|
Write as if describing reality, not describing a recording of reality.
|
|
4480
4942
|
</language_guidelines>`;
|
|
4481
|
-
function
|
|
4943
|
+
function buildUserPrompt5({
|
|
4482
4944
|
tone,
|
|
4483
4945
|
transcriptText,
|
|
4484
4946
|
isCleanTranscript = true,
|
|
@@ -4509,7 +4971,7 @@ function buildUserPrompt4({
|
|
|
4509
4971
|
async function analyzeStoryboard2(imageDataUrl, provider, modelId, userPrompt, systemPrompt, credentials) {
|
|
4510
4972
|
"use step";
|
|
4511
4973
|
const model = await createLanguageModelFromConfig(provider, modelId, credentials);
|
|
4512
|
-
const response = await
|
|
4974
|
+
const response = await generateText6({
|
|
4513
4975
|
model,
|
|
4514
4976
|
output: SUMMARY_OUTPUT,
|
|
4515
4977
|
messages: [
|
|
@@ -4544,7 +5006,7 @@ async function analyzeStoryboard2(imageDataUrl, provider, modelId, userPrompt, s
|
|
|
4544
5006
|
async function analyzeAudioOnly(provider, modelId, userPrompt, systemPrompt, credentials) {
|
|
4545
5007
|
"use step";
|
|
4546
5008
|
const model = await createLanguageModelFromConfig(provider, modelId, credentials);
|
|
4547
|
-
const response = await
|
|
5009
|
+
const response = await generateText6({
|
|
4548
5010
|
model,
|
|
4549
5011
|
output: SUMMARY_OUTPUT,
|
|
4550
5012
|
messages: [
|
|
@@ -4647,7 +5109,7 @@ async function getSummaryAndTags(assetId, options) {
|
|
|
4647
5109
|
const transcriptText = transcriptResult?.transcriptText ?? "";
|
|
4648
5110
|
const resolvedLanguageCode = outputLanguageCode && outputLanguageCode !== "auto" ? outputLanguageCode : transcriptResult?.track?.language_code ?? getReadyTextTracks(assetData)[0]?.language_code;
|
|
4649
5111
|
const languageName = resolvedLanguageCode ? getLanguageName(resolvedLanguageCode) : void 0;
|
|
4650
|
-
const userPrompt =
|
|
5112
|
+
const userPrompt = buildUserPrompt5({
|
|
4651
5113
|
tone,
|
|
4652
5114
|
transcriptText,
|
|
4653
5115
|
isCleanTranscript: cleanTranscript,
|
|
@@ -4660,7 +5122,7 @@ async function getSummaryAndTags(assetId, options) {
|
|
|
4660
5122
|
});
|
|
4661
5123
|
let analysisResponse;
|
|
4662
5124
|
let imageUrl;
|
|
4663
|
-
const systemPrompt = isAudioOnly ?
|
|
5125
|
+
const systemPrompt = isAudioOnly ? AUDIO_ONLY_SYSTEM_PROMPT2 : SYSTEM_PROMPT5;
|
|
4664
5126
|
try {
|
|
4665
5127
|
if (isAudioOnly) {
|
|
4666
5128
|
analysisResponse = await analyzeAudioOnly(
|
|
@@ -5133,24 +5595,24 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
|
|
|
5133
5595
|
// src/workflows/translate-captions.ts
|
|
5134
5596
|
import {
|
|
5135
5597
|
APICallError,
|
|
5136
|
-
generateText as
|
|
5598
|
+
generateText as generateText7,
|
|
5137
5599
|
NoObjectGeneratedError,
|
|
5138
|
-
Output as
|
|
5600
|
+
Output as Output7,
|
|
5139
5601
|
RetryError,
|
|
5140
5602
|
TypeValidationError
|
|
5141
5603
|
} from "ai";
|
|
5142
|
-
import
|
|
5143
|
-
import { z as
|
|
5144
|
-
var translationSchema =
|
|
5145
|
-
translation:
|
|
5604
|
+
import dedent7 from "dedent";
|
|
5605
|
+
import { z as z8 } from "zod";
|
|
5606
|
+
var translationSchema = z8.object({
|
|
5607
|
+
translation: z8.string()
|
|
5146
5608
|
});
|
|
5147
|
-
var
|
|
5609
|
+
var SYSTEM_PROMPT6 = dedent7`
|
|
5148
5610
|
You are a subtitle translation expert. Translate VTT subtitle files to the target language specified by the user.
|
|
5149
5611
|
You may receive either a full VTT file or a chunk from a larger VTT.
|
|
5150
5612
|
Preserve all timestamps, cue ordering, and VTT formatting exactly as they appear.
|
|
5151
5613
|
Return JSON with a single key "translation" containing the translated VTT content.
|
|
5152
5614
|
`;
|
|
5153
|
-
var CUE_TRANSLATION_SYSTEM_PROMPT =
|
|
5615
|
+
var CUE_TRANSLATION_SYSTEM_PROMPT = dedent7`
|
|
5154
5616
|
You are a subtitle translation expert.
|
|
5155
5617
|
You will receive a sequence of subtitle cues extracted from a VTT file.
|
|
5156
5618
|
Translate the cues to the requested target language while preserving their original order.
|
|
@@ -5322,13 +5784,13 @@ async function translateVttWithAI({
|
|
|
5322
5784
|
}) {
|
|
5323
5785
|
"use step";
|
|
5324
5786
|
const model = await createLanguageModelFromConfig(provider, modelId, credentials);
|
|
5325
|
-
const response = await
|
|
5787
|
+
const response = await generateText7({
|
|
5326
5788
|
model,
|
|
5327
|
-
output:
|
|
5789
|
+
output: Output7.object({ schema: translationSchema }),
|
|
5328
5790
|
messages: [
|
|
5329
5791
|
{
|
|
5330
5792
|
role: "system",
|
|
5331
|
-
content:
|
|
5793
|
+
content: SYSTEM_PROMPT6
|
|
5332
5794
|
},
|
|
5333
5795
|
{
|
|
5334
5796
|
role: "user",
|
|
@@ -5359,8 +5821,8 @@ async function translateCueChunkWithAI({
|
|
|
5359
5821
|
}) {
|
|
5360
5822
|
"use step";
|
|
5361
5823
|
const model = await createLanguageModelFromConfig(provider, modelId, credentials);
|
|
5362
|
-
const schema =
|
|
5363
|
-
translations:
|
|
5824
|
+
const schema = z8.object({
|
|
5825
|
+
translations: z8.array(z8.string().min(1)).length(cues.length)
|
|
5364
5826
|
});
|
|
5365
5827
|
const cuePayload = cues.map((cue, index) => ({
|
|
5366
5828
|
index,
|
|
@@ -5368,9 +5830,9 @@ async function translateCueChunkWithAI({
|
|
|
5368
5830
|
endTime: cue.endTime,
|
|
5369
5831
|
text: cue.text
|
|
5370
5832
|
}));
|
|
5371
|
-
const response = await
|
|
5833
|
+
const response = await generateText7({
|
|
5372
5834
|
model,
|
|
5373
|
-
output:
|
|
5835
|
+
output: Output7.object({ schema }),
|
|
5374
5836
|
messages: [
|
|
5375
5837
|
{
|
|
5376
5838
|
role: "system",
|