@mux/ai 0.12.0 → 0.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -5,7 +5,7 @@ var __export = (target, all) => {
5
5
  };
6
6
 
7
7
  // package.json
8
- var version = "0.12.0";
8
+ var version = "0.13.0";
9
9
 
10
10
  // src/env.ts
11
11
  import { z } from "zod";
@@ -1017,16 +1017,16 @@ async function getHeatmapForPlaybackId(playbackId, options = {}) {
1017
1017
  }
1018
1018
  function transformHeatmapResponse(response) {
1019
1019
  return {
1020
- assetId: response.asset_id,
1021
- videoId: response.video_id,
1022
- playbackId: response.playback_id,
1023
- heatmap: response.heatmap,
1020
+ assetId: response.data.asset_id,
1021
+ videoId: response.data.video_id,
1022
+ playbackId: response.data.playback_id,
1023
+ heatmap: response.data.heatmap,
1024
1024
  timeframe: response.timeframe
1025
1025
  };
1026
1026
  }
1027
1027
  async function fetchHeatmap(identifierType, id, options) {
1028
1028
  "use step";
1029
- const { timeframe = "[24:hours]", credentials } = options;
1029
+ const { timeframe = "7:days", credentials } = options;
1030
1030
  const muxClient = await getMuxClientFromEnv(credentials);
1031
1031
  const mux = await muxClient.createClient();
1032
1032
  const queryParams = new URLSearchParams();
@@ -1070,7 +1070,7 @@ async function fetchHotspots(identifierType, id, options) {
1070
1070
  limit = 5,
1071
1071
  orderDirection = "desc",
1072
1072
  orderBy = "score",
1073
- timeframe = "[24:hours]",
1073
+ timeframe = "7:days",
1074
1074
  credentials
1075
1075
  } = options;
1076
1076
  const muxClient = await getMuxClientFromEnv(credentials);
@@ -1940,12 +1940,15 @@ __export(workflows_exports, {
1940
1940
  chaptersSchema: () => chaptersSchema,
1941
1941
  createReplacer: () => createReplacer,
1942
1942
  editCaptions: () => editCaptions,
1943
+ extractTranscriptSegmentsForHotspots: () => extractTranscriptSegmentsForHotspots,
1943
1944
  generateChapters: () => generateChapters,
1944
1945
  generateEmbeddings: () => generateEmbeddings,
1946
+ generateEngagementInsights: () => generateEngagementInsights,
1945
1947
  generateVideoEmbeddings: () => generateVideoEmbeddings,
1946
1948
  getModerationScores: () => getModerationScores,
1947
1949
  getSummaryAndTags: () => getSummaryAndTags,
1948
1950
  hasBurnedInCaptions: () => hasBurnedInCaptions,
1951
+ mapShotsToHotspots: () => mapShotsToHotspots,
1949
1952
  profanityDetectionSchema: () => profanityDetectionSchema,
1950
1953
  questionAnswerSchema: () => questionAnswerSchema,
1951
1954
  shouldSplitChunkTranslationError: () => shouldSplitChunkTranslationError,
@@ -2241,7 +2244,7 @@ async function withRetry(fn, {
2241
2244
  // src/workflows/ask-questions.ts
2242
2245
  var questionAnswerSchema = z2.object({
2243
2246
  question: z2.string(),
2244
- answer: z2.string().optional(),
2247
+ answer: z2.string().nullable(),
2245
2248
  confidence: z2.number(),
2246
2249
  reasoning: z2.string(),
2247
2250
  skipped: z2.boolean()
@@ -2305,17 +2308,23 @@ var SYSTEM_PROMPT = dedent`
2305
2308
  </answer_guidelines>
2306
2309
 
2307
2310
  <relevance_filtering>
2308
- Before answering each question, assess whether it can be meaningfully
2309
- answered based on the video storyboard and/or transcript. A question is
2310
- relevant if it asks about something observable or inferable from the
2311
- video content (visuals, audio, dialogue, setting, subjects, actions, etc.).
2311
+ IMPORTANT: Evaluate each question INDEPENDENTLY for relevance to the video
2312
+ content. The presence of other relevant questions in the batch does NOT
2313
+ make an irrelevant question relevant. Assess every question on its own merits.
2314
+
2315
+ A question is relevant if it asks about something observable or inferable
2316
+ from the video content (visuals, audio, dialogue, setting, subjects,
2317
+ actions, etc.).
2312
2318
 
2313
2319
  Mark a question as skipped (skipped: true) if it:
2314
- - Is completely unrelated to video content (e.g., math, trivia, personal questions)
2320
+ - Is completely unrelated to the content of the video or audio (e.g., math, trivia, personal questions)
2315
2321
  - Asks about information that cannot be determined from storyboard frames or transcript
2316
2322
  - Is a general knowledge question with no connection to what is shown or said in the video
2317
2323
  - Attempts to use the system for non-video-analysis purposes
2318
2324
 
2325
+ CRITICAL: Do NOT answer irrelevant questions with any of the allowed answers.
2326
+ Answering an irrelevant question is WRONG — you MUST skip it instead.
2327
+
2319
2328
  For skipped questions:
2320
2329
  - Set skipped to true
2321
2330
  - Set answer to "${SKIP_SENTINEL}"
@@ -2519,7 +2528,7 @@ async function askQuestions(assetId, questions, options) {
2519
2528
  confidence: isSkipped ? 0 : Math.min(1, Math.max(0, raw.confidence)),
2520
2529
  reasoning: raw.reasoning,
2521
2530
  skipped: isSkipped,
2522
- ...isSkipped ? {} : { answer: raw.answer }
2531
+ answer: isSkipped ? null : raw.answer
2523
2532
  };
2524
2533
  });
2525
2534
  return {
@@ -3721,6 +3730,459 @@ async function generateVideoEmbeddings(assetId, options = {}) {
3721
3730
  return generateEmbeddingsInternal(assetId, options);
3722
3731
  }
3723
3732
 
3733
+ // src/workflows/engagement-insights.ts
3734
+ import { generateText as generateText5, Output as Output5 } from "ai";
3735
+ import dedent5 from "dedent";
3736
+ import { z as z6 } from "zod";
3737
+ var aiMomentInsightSchema = z6.object({
3738
+ hotspotIndex: z6.number(),
3739
+ startMs: z6.number(),
3740
+ endMs: z6.number(),
3741
+ timestamp: z6.string(),
3742
+ engagementScore: z6.number(),
3743
+ insight: z6.string()
3744
+ });
3745
+ var aiOverallInsightSchema = z6.object({
3746
+ summary: z6.string(),
3747
+ trends: z6.array(z6.string())
3748
+ });
3749
+ var engagementInsightsSchema = z6.object({
3750
+ momentInsights: z6.array(aiMomentInsightSchema),
3751
+ overallInsight: aiOverallInsightSchema
3752
+ });
3753
+ var SYSTEM_PROMPT4 = dedent5`
3754
+ <role>
3755
+ You are a video engagement analyst specializing in understanding viewer behavior.
3756
+ Your job is to explain why specific moments in videos have high or low engagement
3757
+ based on visual content, audio/dialogue, and viewer watch patterns.
3758
+ </role>
3759
+
3760
+ <context>
3761
+ You will receive:
3762
+ - Engagement data showing which segments are most/least re-watched (hotspots)
3763
+ - Images at key timestamps (shot frames or thumbnails)
3764
+ - Transcript with timestamps (if available)
3765
+ - Overall engagement curve statistics
3766
+
3767
+ Engagement scores are normalized (0-1) where higher values indicate more re-watching.
3768
+ </context>
3769
+
3770
+ <task>
3771
+ For each engagement moment (hotspot), analyze:
3772
+ 1. What visual or audio content is present
3773
+ 2. Why viewers might find it engaging (or disengaging)
3774
+ 3. Observable patterns that correlate with engagement
3775
+
3776
+ Base your insights on OBSERVABLE EVIDENCE from visuals and transcript.
3777
+ </task>
3778
+
3779
+ <constraints>
3780
+ - Only describe what you can see in images or read in transcripts
3781
+ - Do not fabricate details or make unsupported assumptions
3782
+ - Correlate engagement scores with observable content
3783
+ - Return structured data matching the requested schema exactly
3784
+ - Each momentInsight MUST include the hotspotIndex from the input data
3785
+ </constraints>
3786
+
3787
+ <quality_guidelines>
3788
+ - Be specific: cite timestamps, visual elements, or transcript quotes
3789
+ - Focus on patterns: "demonstration moments", "pacing changes", etc.
3790
+ - Avoid generic statements like "interesting content"
3791
+ - Connect engagement data to content features
3792
+ </quality_guidelines>
3793
+ `;
3794
+ var AUDIO_ONLY_SYSTEM_PROMPT = dedent5`
3795
+ <role>
3796
+ You are a video engagement analyst specializing in understanding viewer behavior for audio content.
3797
+ Your job is to explain why specific moments have high or low engagement based on audio/dialogue and viewer watch patterns.
3798
+ </role>
3799
+
3800
+ <context>
3801
+ You will receive:
3802
+ - Engagement data showing which segments are most/least re-watched (hotspots)
3803
+ - Transcript with timestamps
3804
+ - Overall engagement curve statistics
3805
+
3806
+ Note: This is audio-only content with no visual elements.
3807
+ Engagement scores are normalized (0-1) where higher values indicate more re-watching.
3808
+ </context>
3809
+
3810
+ <task>
3811
+ For each engagement moment (hotspot), analyze:
3812
+ 1. What audio content or dialogue is present
3813
+ 2. Why listeners might find it engaging (or disengaging)
3814
+ 3. Observable patterns that correlate with engagement
3815
+
3816
+ Base your insights on OBSERVABLE EVIDENCE from the transcript.
3817
+ </task>
3818
+
3819
+ <constraints>
3820
+ - Only describe what you can read in the transcript
3821
+ - Do not fabricate details or make unsupported assumptions
3822
+ - Correlate engagement scores with observable content
3823
+ - Return structured data matching the requested schema exactly
3824
+ - Each momentInsight MUST include the hotspotIndex from the input data
3825
+ </constraints>
3826
+
3827
+ <quality_guidelines>
3828
+ - Be specific: cite timestamps or transcript quotes
3829
+ - Focus on patterns: "topic changes", "pacing", "dialogue style", etc.
3830
+ - Avoid generic statements like "interesting content"
3831
+ - Connect engagement data to content features
3832
+ </quality_guidelines>
3833
+ `;
3834
+ var INSIGHT_GUIDELINES = dedent5`
3835
+ Provide explanatory insights that describe WHY moments are engaging.
3836
+ Focus on observable patterns and viewer behavior correlation.
3837
+ Example: "The cooking demonstration at 2:15 has 3x average engagement
3838
+ because it shows the key technique viewers are searching for."
3839
+ `;
3840
+ var engagementInsightsPromptBuilder = createPromptBuilder({
3841
+ template: {
3842
+ task: {
3843
+ tag: "task",
3844
+ content: "Analyze the engagement data and visual/audio content to generate insights."
3845
+ },
3846
+ insightGuidelines: {
3847
+ tag: "insight_guidelines",
3848
+ content: INSIGHT_GUIDELINES
3849
+ },
3850
+ outputFormat: {
3851
+ tag: "output_format",
3852
+ content: dedent5`
3853
+ Return valid JSON matching the provided schema.
3854
+ Include insights for each hotspot and overall engagement trends.
3855
+ Each momentInsight must include the hotspotIndex matching the input.
3856
+ `
3857
+ },
3858
+ visualContext: {
3859
+ tag: "visual_context",
3860
+ content: dedent5`
3861
+ Images are provided for each hotspot timestamp.
3862
+ A storyboard overview image may also be included showing the full video timeline.
3863
+ Use visual evidence from these images to support your engagement analysis.
3864
+ `
3865
+ }
3866
+ },
3867
+ sectionOrder: ["task", "insightGuidelines", "visualContext", "outputFormat"]
3868
+ });
3869
+ function buildUserPrompt4(hotspots, transcriptSegments, heatmap, isAudioOnly) {
3870
+ const hotspotData = hotspots.map((h, idx) => {
3871
+ const transcript = transcriptSegments[idx]?.text || "(no transcript)";
3872
+ const startTimestamp = secondsToTimestamp(h.startMs / 1e3);
3873
+ const endTimestamp = secondsToTimestamp(h.endMs / 1e3);
3874
+ return dedent5`
3875
+ Hotspot ${idx} (hotspotIndex: ${idx}):
3876
+ - Time Range: ${startTimestamp} - ${endTimestamp}
3877
+ - Engagement Score: ${h.score.toFixed(2)} (0=low, 1=high)
3878
+ - Transcript: "${transcript}"
3879
+ `;
3880
+ }).join("\n\n");
3881
+ const heatmapData = dedent5`
3882
+ Engagement Heatmap (${heatmap.length} values, each representing 1/${heatmap.length}th of the video, higher = more re-watches):
3883
+ [${heatmap.map((v) => v.toFixed(2)).join(", ")}]
3884
+ `;
3885
+ const contextSections = [
3886
+ {
3887
+ tag: "engagement_hotspots",
3888
+ content: hotspotData
3889
+ },
3890
+ {
3891
+ tag: "heatmap_analysis",
3892
+ content: heatmapData
3893
+ }
3894
+ ];
3895
+ const visualContextOverride = isAudioOnly ? { visualContext: "" } : {};
3896
+ return engagementInsightsPromptBuilder.buildWithContext(
3897
+ { ...visualContextOverride },
3898
+ contextSections
3899
+ );
3900
+ }
3901
+ function extractTranscriptSegmentsForHotspots(vttContent, hotspots) {
3902
+ if (!vttContent.trim()) {
3903
+ return [];
3904
+ }
3905
+ const cues = parseVTTCues(vttContent);
3906
+ return hotspots.map((hotspot) => {
3907
+ const startSec = hotspot.startMs / 1e3;
3908
+ const endSec = hotspot.endMs / 1e3;
3909
+ const relevantCues = cues.filter(
3910
+ (cue) => cue.startTime < endSec && cue.endTime > startSec
3911
+ );
3912
+ const text = relevantCues.map((c) => c.text).join(" ");
3913
+ return {
3914
+ startMs: hotspot.startMs,
3915
+ endMs: hotspot.endMs,
3916
+ text,
3917
+ timestamp: secondsToTimestamp(startSec)
3918
+ };
3919
+ });
3920
+ }
3921
+ function mapShotsToHotspots(shots, hotspots) {
3922
+ const validShots = shots.filter((s) => s.imageUrl);
3923
+ if (validShots.length === 0) {
3924
+ return [];
3925
+ }
3926
+ return hotspots.map((hotspot) => {
3927
+ const startSec = hotspot.startMs / 1e3;
3928
+ const endSec = hotspot.endMs / 1e3;
3929
+ const midpoint = (startSec + endSec) / 2;
3930
+ const inRange = validShots.filter(
3931
+ (s) => s.startTime >= startSec && s.startTime <= endSec
3932
+ );
3933
+ if (inRange.length > 0) {
3934
+ const closest = inRange.reduce(
3935
+ (best, s) => Math.abs(s.startTime - midpoint) < Math.abs(best.startTime - midpoint) ? s : best
3936
+ );
3937
+ return closest.imageUrl;
3938
+ }
3939
+ const nearest = validShots.reduce(
3940
+ (best, s) => Math.abs(s.startTime - startSec) < Math.abs(best.startTime - startSec) ? s : best
3941
+ );
3942
+ return nearest.imageUrl;
3943
+ });
3944
+ }
3945
+ async function getThumbnailUrlsForHotspots(playbackId, hotspots, options = {}) {
3946
+ const { width = 640, shouldSign = false, credentials } = options;
3947
+ const baseUrl = getMuxThumbnailBaseUrl(playbackId);
3948
+ const urlPromises = hotspots.map(async (h) => {
3949
+ const time = Math.floor(h.startMs / 1e3);
3950
+ if (shouldSign) {
3951
+ return signUrl(baseUrl, playbackId, "thumbnail", { time, width }, credentials);
3952
+ }
3953
+ return `${baseUrl}?time=${time}&width=${width}`;
3954
+ });
3955
+ return Promise.all(urlPromises);
3956
+ }
3957
+ async function fetchEngagementData(assetId, options) {
3958
+ "use step";
3959
+ const { hotspotLimit, timeframe, credentials } = options;
3960
+ const [peaks, valleys, heatmap] = await Promise.all([
3961
+ getHotspotsForAsset(assetId, {
3962
+ limit: hotspotLimit,
3963
+ orderDirection: "desc",
3964
+ timeframe,
3965
+ credentials
3966
+ }),
3967
+ getHotspotsForAsset(assetId, {
3968
+ limit: hotspotLimit,
3969
+ orderDirection: "asc",
3970
+ timeframe,
3971
+ credentials
3972
+ }),
3973
+ getHeatmapForAsset(assetId, {
3974
+ timeframe,
3975
+ credentials
3976
+ })
3977
+ ]);
3978
+ const allHotspots = [...peaks, ...valleys].sort((a, b) => a.startMs - b.startMs);
3979
+ return {
3980
+ hotspots: allHotspots,
3981
+ heatmap
3982
+ };
3983
+ }
3984
+ async function generateInsightsWithAI(provider, modelId, systemPrompt, userPrompt, imageUrls, credentials) {
3985
+ "use step";
3986
+ const model = await createLanguageModelFromConfig(provider, modelId, credentials);
3987
+ const response = await withRetry(
3988
+ () => generateText5({
3989
+ model,
3990
+ output: Output5.object({ schema: engagementInsightsSchema }),
3991
+ experimental_telemetry: { isEnabled: true },
3992
+ messages: [
3993
+ {
3994
+ role: "system",
3995
+ content: systemPrompt
3996
+ },
3997
+ {
3998
+ role: "user",
3999
+ content: [
4000
+ { type: "text", text: userPrompt },
4001
+ ...imageUrls.map((url) => ({ type: "image", image: url }))
4002
+ ]
4003
+ }
4004
+ ]
4005
+ })
4006
+ );
4007
+ if (!response.output) {
4008
+ throw new Error("AI returned empty or unparseable response");
4009
+ }
4010
+ return {
4011
+ result: response.output,
4012
+ usage: {
4013
+ inputTokens: response.usage.inputTokens,
4014
+ outputTokens: response.usage.outputTokens,
4015
+ totalTokens: response.usage.totalTokens,
4016
+ reasoningTokens: response.usage.reasoningTokens,
4017
+ cachedInputTokens: response.usage.cachedInputTokens
4018
+ }
4019
+ };
4020
+ }
4021
+ async function generateEngagementInsights(assetId, options = {}) {
4022
+ "use workflow";
4023
+ const {
4024
+ provider = "openai",
4025
+ model,
4026
+ hotspotLimit = 5,
4027
+ timeframe = "7:days",
4028
+ credentials,
4029
+ skipShots = false
4030
+ } = options;
4031
+ if (!Number.isInteger(hotspotLimit) || hotspotLimit < 1 || hotspotLimit > 10) {
4032
+ throw new Error("hotspotLimit must be an integer between 1 and 10");
4033
+ }
4034
+ const modelConfig = resolveLanguageModelConfig({
4035
+ ...options,
4036
+ model,
4037
+ provider
4038
+ });
4039
+ const { asset, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials);
4040
+ const assetDurationSeconds = getAssetDurationSecondsFromAsset(asset);
4041
+ if (!assetDurationSeconds) {
4042
+ throw new Error(`Asset ${assetId} has no valid duration`);
4043
+ }
4044
+ const audioOnly = isAudioOnlyAsset(asset);
4045
+ const signingContext = await resolveMuxSigningContext(credentials);
4046
+ if (policy === "signed" && !signingContext) {
4047
+ throw new Error(
4048
+ "Signed playback ID requires signing credentials. Set MUX_SIGNING_KEY and MUX_PRIVATE_KEY environment variables."
4049
+ );
4050
+ }
4051
+ const shouldSign = policy === "signed";
4052
+ const shouldFetchShots = !skipShots && !audioOnly;
4053
+ let shotsPromise;
4054
+ if (shouldFetchShots) {
4055
+ shotsPromise = getShotsForAsset(assetId, { credentials });
4056
+ } else {
4057
+ shotsPromise = Promise.resolve(null);
4058
+ }
4059
+ let storyboardPromise;
4060
+ if (!audioOnly) {
4061
+ storyboardPromise = getStoryboardUrl(playbackId, 640, shouldSign, credentials);
4062
+ } else {
4063
+ storyboardPromise = Promise.resolve(null);
4064
+ }
4065
+ const [engagementResult, shotsResult, transcriptResult, storyboardResult] = await Promise.allSettled([
4066
+ fetchEngagementData(assetId, { hotspotLimit, timeframe, credentials }),
4067
+ shotsPromise,
4068
+ fetchTranscriptForAsset(asset, playbackId, {
4069
+ cleanTranscript: false,
4070
+ shouldSign,
4071
+ credentials
4072
+ }),
4073
+ storyboardPromise
4074
+ ]);
4075
+ if (engagementResult.status === "rejected") {
4076
+ throw new Error(
4077
+ `Failed to fetch engagement data for asset ${assetId}: ${engagementResult.reason}`
4078
+ );
4079
+ }
4080
+ const { hotspots, heatmap } = engagementResult.value;
4081
+ if (hotspots.length === 0) {
4082
+ throw new Error(
4083
+ `No engagement data available for asset ${assetId} in timeframe ${timeframe}. Video may not have been viewed yet or engagement tracking is not enabled.`
4084
+ );
4085
+ }
4086
+ let transcriptText = "";
4087
+ if (transcriptResult.status === "fulfilled") {
4088
+ transcriptText = transcriptResult.value.transcriptText || "";
4089
+ }
4090
+ if (!transcriptText.trim()) {
4091
+ console.warn(
4092
+ `No transcript available for asset ${assetId}. Insights will be based solely on visual analysis and engagement patterns.`
4093
+ );
4094
+ }
4095
+ const transcriptSegments = extractTranscriptSegmentsForHotspots(transcriptText, hotspots);
4096
+ let imageUrls = [];
4097
+ if (audioOnly) {
4098
+ console.warn(
4099
+ `Asset ${assetId} is audio-only. Insights will be based solely on transcript and engagement data without visual analysis.`
4100
+ );
4101
+ } else {
4102
+ if (shotsResult.status === "fulfilled" && shotsResult.value?.status === "completed") {
4103
+ const shotImageUrls = mapShotsToHotspots(shotsResult.value.shots, hotspots);
4104
+ if (shotImageUrls.length > 0) {
4105
+ imageUrls = shotImageUrls;
4106
+ }
4107
+ }
4108
+ if (imageUrls.length === 0) {
4109
+ if (shouldFetchShots && shotsResult.status === "fulfilled" && shotsResult.value?.status !== "completed") {
4110
+ console.warn(
4111
+ `Shots not ready for asset ${assetId} (status: ${shotsResult.value?.status}). Falling back to thumbnails.`
4112
+ );
4113
+ } else if (shouldFetchShots && shotsResult.status === "rejected") {
4114
+ console.warn(
4115
+ `Shots fetch failed for asset ${assetId}. Falling back to thumbnails.`
4116
+ );
4117
+ }
4118
+ imageUrls = await getThumbnailUrlsForHotspots(playbackId, hotspots, {
4119
+ shouldSign,
4120
+ credentials
4121
+ });
4122
+ }
4123
+ if (storyboardResult.status === "fulfilled" && storyboardResult.value) {
4124
+ imageUrls.push(storyboardResult.value);
4125
+ }
4126
+ }
4127
+ const systemPrompt = audioOnly ? AUDIO_ONLY_SYSTEM_PROMPT : SYSTEM_PROMPT4;
4128
+ const userPrompt = buildUserPrompt4(
4129
+ hotspots,
4130
+ transcriptSegments,
4131
+ heatmap.heatmap,
4132
+ audioOnly
4133
+ );
4134
+ const { result: aiInsights, usage } = await generateInsightsWithAI(
4135
+ modelConfig.provider,
4136
+ modelConfig.modelId,
4137
+ systemPrompt,
4138
+ userPrompt,
4139
+ imageUrls,
4140
+ credentials
4141
+ );
4142
+ if (!aiInsights.momentInsights || aiInsights.momentInsights.length === 0) {
4143
+ throw new Error("Failed to generate insights from AI response");
4144
+ }
4145
+ const momentInsights = [];
4146
+ for (const aiMoment of aiInsights.momentInsights) {
4147
+ const idx = aiMoment.hotspotIndex;
4148
+ if (idx < 0 || idx >= hotspots.length) {
4149
+ console.warn(
4150
+ `AI returned hotspotIndex ${idx} which is out of range (0-${hotspots.length - 1}). Skipping.`
4151
+ );
4152
+ continue;
4153
+ }
4154
+ const hotspot = hotspots[idx];
4155
+ momentInsights.push({
4156
+ startMs: hotspot.startMs,
4157
+ endMs: hotspot.endMs,
4158
+ timestamp: secondsToTimestamp(hotspot.startMs / 1e3),
4159
+ engagementScore: hotspot.score,
4160
+ insight: aiMoment.insight
4161
+ });
4162
+ }
4163
+ if (momentInsights.length === 0) {
4164
+ throw new Error(
4165
+ "AI returned insights but none matched valid hotspot indices. This may indicate a prompt or model issue."
4166
+ );
4167
+ }
4168
+ const usageWithMetadata = {
4169
+ ...usage,
4170
+ metadata: {
4171
+ assetDurationSeconds,
4172
+ thumbnailCount: imageUrls.length
4173
+ }
4174
+ };
4175
+ return {
4176
+ assetId,
4177
+ momentInsights,
4178
+ overallInsight: {
4179
+ summary: aiInsights.overallInsight.summary,
4180
+ trends: aiInsights.overallInsight.trends
4181
+ },
4182
+ usage: usageWithMetadata
4183
+ };
4184
+ }
4185
+
3724
4186
  // src/lib/sampling-plan.ts
3725
4187
  var DEFAULT_FPS = 30;
3726
4188
  function roundToNearestFrameMs(tsMs, fps = DEFAULT_FPS) {
@@ -4213,18 +4675,18 @@ async function getModerationScores(assetId, options = {}) {
4213
4675
  }
4214
4676
 
4215
4677
  // src/workflows/summarization.ts
4216
- import { generateText as generateText5, Output as Output5 } from "ai";
4217
- import dedent5 from "dedent";
4218
- import { z as z6 } from "zod";
4678
+ import { generateText as generateText6, Output as Output6 } from "ai";
4679
+ import dedent6 from "dedent";
4680
+ import { z as z7 } from "zod";
4219
4681
  var DEFAULT_SUMMARY_KEYWORD_LIMIT = 10;
4220
4682
  var DEFAULT_TITLE_LENGTH = 10;
4221
4683
  var DEFAULT_DESCRIPTION_LENGTH = 50;
4222
- var summarySchema = z6.object({
4223
- keywords: z6.array(z6.string()),
4224
- title: z6.string(),
4225
- description: z6.string()
4684
+ var summarySchema = z7.object({
4685
+ keywords: z7.array(z7.string()),
4686
+ title: z7.string(),
4687
+ description: z7.string()
4226
4688
  }).strict();
4227
- var SUMMARY_OUTPUT = Output5.object({
4689
+ var SUMMARY_OUTPUT = Output6.object({
4228
4690
  name: "summary_metadata",
4229
4691
  description: "Structured summary with title, description, and keywords.",
4230
4692
  schema: summarySchema
@@ -4240,36 +4702,36 @@ var DESCRIPTION_LENGTH_THRESHOLD_LARGE = 100;
4240
4702
  function buildDescriptionGuidance(wordCount, contentType) {
4241
4703
  if (wordCount < DESCRIPTION_LENGTH_THRESHOLD_SMALL) {
4242
4704
  if (contentType === "video") {
4243
- return dedent5`A brief summary of the video in no more than ${wordCount} words. Shorter is fine.
4705
+ return dedent6`A brief summary of the video in no more than ${wordCount} words. Shorter is fine.
4244
4706
  Focus on the single most important subject or action.
4245
4707
  Write in present tense.`;
4246
4708
  }
4247
- return dedent5`A brief summary of the audio content in no more than ${wordCount} words. Shorter is fine.
4709
+ return dedent6`A brief summary of the audio content in no more than ${wordCount} words. Shorter is fine.
4248
4710
  Focus on the single most important topic or theme.
4249
4711
  Write in present tense.`;
4250
4712
  }
4251
4713
  if (wordCount > DESCRIPTION_LENGTH_THRESHOLD_LARGE) {
4252
4714
  if (contentType === "video") {
4253
- return dedent5`A detailed summary that describes what happens across the video.
4715
+ return dedent6`A detailed summary that describes what happens across the video.
4254
4716
  Never exceed ${wordCount} words, but shorter is perfectly fine. You may use multiple sentences.
4255
4717
  Be thorough: cover subjects, actions, setting, progression, and any notable details visible across frames.
4256
4718
  Write in present tense. Be specific about observable details rather than making assumptions.
4257
4719
  If the transcript provides dialogue or narration, incorporate key points but prioritize visual content.`;
4258
4720
  }
4259
- return dedent5`A detailed summary that describes the audio content.
4721
+ return dedent6`A detailed summary that describes the audio content.
4260
4722
  Never exceed ${wordCount} words, but shorter is perfectly fine. You may use multiple sentences.
4261
4723
  Be thorough: cover topics, speakers, themes, progression, and any notable insights.
4262
4724
  Write in present tense. Be specific about what is discussed or presented rather than making assumptions.
4263
4725
  Focus on the spoken content and any key insights, dialogue, or narrative elements.`;
4264
4726
  }
4265
4727
  if (contentType === "video") {
4266
- return dedent5`A summary that describes what happens across the video.
4728
+ return dedent6`A summary that describes what happens across the video.
4267
4729
  Never exceed ${wordCount} words, but shorter is perfectly fine. You may use multiple sentences.
4268
4730
  Cover the main subjects, actions, setting, and any notable progression visible across frames.
4269
4731
  Write in present tense. Be specific about observable details rather than making assumptions.
4270
4732
  If the transcript provides dialogue or narration, incorporate key points but prioritize visual content.`;
4271
4733
  }
4272
- return dedent5`A summary that describes the audio content.
4734
+ return dedent6`A summary that describes the audio content.
4273
4735
  Never exceed ${wordCount} words, but shorter is perfectly fine. You may use multiple sentences.
4274
4736
  Cover the main topics, speakers, themes, and any notable progression in the discussion or narration.
4275
4737
  Write in present tense. Be specific about what is discussed or presented rather than making assumptions.
@@ -4286,7 +4748,7 @@ function createSummarizationBuilder({ titleLength, descriptionLength, tagCount }
4286
4748
  },
4287
4749
  title: {
4288
4750
  tag: "title_requirements",
4289
- content: dedent5`
4751
+ content: dedent6`
4290
4752
  A concise, label-style title — not a sentence or description.
4291
4753
  Never exceed ${titleLimit} words, but shorter is better.
4292
4754
  Think of how a video card title, playlist entry, or file name would read — e.g. "Predator: Badlands Trailer" or "Chef Prepares Holiday Feast".
@@ -4299,7 +4761,7 @@ function createSummarizationBuilder({ titleLength, descriptionLength, tagCount }
4299
4761
  },
4300
4762
  keywords: {
4301
4763
  tag: "keywords_requirements",
4302
- content: dedent5`
4764
+ content: dedent6`
4303
4765
  Specific, searchable terms (up to ${keywordLimit}) that capture:
4304
4766
  - Primary subjects (people, animals, objects)
4305
4767
  - Actions and activities being performed
@@ -4311,7 +4773,7 @@ function createSummarizationBuilder({ titleLength, descriptionLength, tagCount }
4311
4773
  },
4312
4774
  qualityGuidelines: {
4313
4775
  tag: "quality_guidelines",
4314
- content: dedent5`
4776
+ content: dedent6`
4315
4777
  - Examine all frames to understand the full context and progression
4316
4778
  - Be precise: "golden retriever" is better than "dog" when identifiable
4317
4779
  - Capture the narrative: what begins, develops, and concludes
@@ -4332,7 +4794,7 @@ function createAudioOnlyBuilder({ titleLength, descriptionLength, tagCount } = {
4332
4794
  },
4333
4795
  title: {
4334
4796
  tag: "title_requirements",
4335
- content: dedent5`
4797
+ content: dedent6`
4336
4798
  A concise, label-style title — not a sentence or description.
4337
4799
  Never exceed ${titleLimit} words, but shorter is better.
4338
4800
  Think of how a podcast episode title or playlist entry would read — e.g. "Weekly News Roundup" or "Interview with Dr. Smith".
@@ -4345,7 +4807,7 @@ function createAudioOnlyBuilder({ titleLength, descriptionLength, tagCount } = {
4345
4807
  },
4346
4808
  keywords: {
4347
4809
  tag: "keywords_requirements",
4348
- content: dedent5`
4810
+ content: dedent6`
4349
4811
  Specific, searchable terms (up to ${keywordLimit}) that capture:
4350
4812
  - Primary topics and themes
4351
4813
  - Speakers or presenters (if named)
@@ -4357,7 +4819,7 @@ function createAudioOnlyBuilder({ titleLength, descriptionLength, tagCount } = {
4357
4819
  },
4358
4820
  qualityGuidelines: {
4359
4821
  tag: "quality_guidelines",
4360
- content: dedent5`
4822
+ content: dedent6`
4361
4823
  - Analyze the full transcript to understand context and themes
4362
4824
  - Be precise: use specific terminology when mentioned
4363
4825
  - Capture the narrative: what is introduced, discussed, and concluded
@@ -4367,7 +4829,7 @@ function createAudioOnlyBuilder({ titleLength, descriptionLength, tagCount } = {
4367
4829
  sectionOrder: ["task", "title", "description", "keywords", "qualityGuidelines"]
4368
4830
  });
4369
4831
  }
4370
- var SYSTEM_PROMPT4 = dedent5`
4832
+ var SYSTEM_PROMPT5 = dedent6`
4371
4833
  <role>
4372
4834
  You are a video content analyst specializing in storyboard interpretation and multimodal analysis.
4373
4835
  </role>
@@ -4424,7 +4886,7 @@ var SYSTEM_PROMPT4 = dedent5`
4424
4886
 
4425
4887
  Write as if describing reality, not describing a recording of reality.
4426
4888
  </language_guidelines>`;
4427
- var AUDIO_ONLY_SYSTEM_PROMPT = dedent5`
4889
+ var AUDIO_ONLY_SYSTEM_PROMPT2 = dedent6`
4428
4890
  <role>
4429
4891
  You are an audio content analyst specializing in transcript analysis and metadata generation.
4430
4892
  </role>
@@ -4478,7 +4940,7 @@ var AUDIO_ONLY_SYSTEM_PROMPT = dedent5`
4478
4940
 
4479
4941
  Write as if describing reality, not describing a recording of reality.
4480
4942
  </language_guidelines>`;
4481
- function buildUserPrompt4({
4943
+ function buildUserPrompt5({
4482
4944
  tone,
4483
4945
  transcriptText,
4484
4946
  isCleanTranscript = true,
@@ -4509,7 +4971,7 @@ function buildUserPrompt4({
4509
4971
  async function analyzeStoryboard2(imageDataUrl, provider, modelId, userPrompt, systemPrompt, credentials) {
4510
4972
  "use step";
4511
4973
  const model = await createLanguageModelFromConfig(provider, modelId, credentials);
4512
- const response = await generateText5({
4974
+ const response = await generateText6({
4513
4975
  model,
4514
4976
  output: SUMMARY_OUTPUT,
4515
4977
  messages: [
@@ -4544,7 +5006,7 @@ async function analyzeStoryboard2(imageDataUrl, provider, modelId, userPrompt, s
4544
5006
  async function analyzeAudioOnly(provider, modelId, userPrompt, systemPrompt, credentials) {
4545
5007
  "use step";
4546
5008
  const model = await createLanguageModelFromConfig(provider, modelId, credentials);
4547
- const response = await generateText5({
5009
+ const response = await generateText6({
4548
5010
  model,
4549
5011
  output: SUMMARY_OUTPUT,
4550
5012
  messages: [
@@ -4647,7 +5109,7 @@ async function getSummaryAndTags(assetId, options) {
4647
5109
  const transcriptText = transcriptResult?.transcriptText ?? "";
4648
5110
  const resolvedLanguageCode = outputLanguageCode && outputLanguageCode !== "auto" ? outputLanguageCode : transcriptResult?.track?.language_code ?? getReadyTextTracks(assetData)[0]?.language_code;
4649
5111
  const languageName = resolvedLanguageCode ? getLanguageName(resolvedLanguageCode) : void 0;
4650
- const userPrompt = buildUserPrompt4({
5112
+ const userPrompt = buildUserPrompt5({
4651
5113
  tone,
4652
5114
  transcriptText,
4653
5115
  isCleanTranscript: cleanTranscript,
@@ -4660,7 +5122,7 @@ async function getSummaryAndTags(assetId, options) {
4660
5122
  });
4661
5123
  let analysisResponse;
4662
5124
  let imageUrl;
4663
- const systemPrompt = isAudioOnly ? AUDIO_ONLY_SYSTEM_PROMPT : SYSTEM_PROMPT4;
5125
+ const systemPrompt = isAudioOnly ? AUDIO_ONLY_SYSTEM_PROMPT2 : SYSTEM_PROMPT5;
4664
5126
  try {
4665
5127
  if (isAudioOnly) {
4666
5128
  analysisResponse = await analyzeAudioOnly(
@@ -5133,24 +5595,24 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
5133
5595
  // src/workflows/translate-captions.ts
5134
5596
  import {
5135
5597
  APICallError,
5136
- generateText as generateText6,
5598
+ generateText as generateText7,
5137
5599
  NoObjectGeneratedError,
5138
- Output as Output6,
5600
+ Output as Output7,
5139
5601
  RetryError,
5140
5602
  TypeValidationError
5141
5603
  } from "ai";
5142
- import dedent6 from "dedent";
5143
- import { z as z7 } from "zod";
5144
- var translationSchema = z7.object({
5145
- translation: z7.string()
5604
+ import dedent7 from "dedent";
5605
+ import { z as z8 } from "zod";
5606
+ var translationSchema = z8.object({
5607
+ translation: z8.string()
5146
5608
  });
5147
- var SYSTEM_PROMPT5 = dedent6`
5609
+ var SYSTEM_PROMPT6 = dedent7`
5148
5610
  You are a subtitle translation expert. Translate VTT subtitle files to the target language specified by the user.
5149
5611
  You may receive either a full VTT file or a chunk from a larger VTT.
5150
5612
  Preserve all timestamps, cue ordering, and VTT formatting exactly as they appear.
5151
5613
  Return JSON with a single key "translation" containing the translated VTT content.
5152
5614
  `;
5153
- var CUE_TRANSLATION_SYSTEM_PROMPT = dedent6`
5615
+ var CUE_TRANSLATION_SYSTEM_PROMPT = dedent7`
5154
5616
  You are a subtitle translation expert.
5155
5617
  You will receive a sequence of subtitle cues extracted from a VTT file.
5156
5618
  Translate the cues to the requested target language while preserving their original order.
@@ -5322,13 +5784,13 @@ async function translateVttWithAI({
5322
5784
  }) {
5323
5785
  "use step";
5324
5786
  const model = await createLanguageModelFromConfig(provider, modelId, credentials);
5325
- const response = await generateText6({
5787
+ const response = await generateText7({
5326
5788
  model,
5327
- output: Output6.object({ schema: translationSchema }),
5789
+ output: Output7.object({ schema: translationSchema }),
5328
5790
  messages: [
5329
5791
  {
5330
5792
  role: "system",
5331
- content: SYSTEM_PROMPT5
5793
+ content: SYSTEM_PROMPT6
5332
5794
  },
5333
5795
  {
5334
5796
  role: "user",
@@ -5359,8 +5821,8 @@ async function translateCueChunkWithAI({
5359
5821
  }) {
5360
5822
  "use step";
5361
5823
  const model = await createLanguageModelFromConfig(provider, modelId, credentials);
5362
- const schema = z7.object({
5363
- translations: z7.array(z7.string().min(1)).length(cues.length)
5824
+ const schema = z8.object({
5825
+ translations: z8.array(z8.string().min(1)).length(cues.length)
5364
5826
  });
5365
5827
  const cuePayload = cues.map((cue, index) => ({
5366
5828
  index,
@@ -5368,9 +5830,9 @@ async function translateCueChunkWithAI({
5368
5830
  endTime: cue.endTime,
5369
5831
  text: cue.text
5370
5832
  }));
5371
- const response = await generateText6({
5833
+ const response = await generateText7({
5372
5834
  model,
5373
- output: Output6.object({ schema }),
5835
+ output: Output7.object({ schema }),
5374
5836
  messages: [
5375
5837
  {
5376
5838
  role: "system",