@mux/ai 0.41.0 → 0.42.0

package/dist/workflows/index.d.ts

@@ -1,5 +1,6 @@
-export { E as ASK_QUESTIONS_DEFAULT_MAX_ANSWER_OPTION_LENGTH, F as ASK_QUESTIONS_DEFAULT_MAX_FREE_FORM_ANSWER_LENGTH, G as ASK_QUESTIONS_MAX_QUESTIONS_PER_CALL, H as ASK_QUESTIONS_MAX_QUESTION_LENGTH, J as AskQuestionsOptions, K as AskQuestionsResult, N as AskQuestionsType, O as AudioTranslationOptions, P as AudioTranslationResult, Q as AutoCensorProfanityOptions, U as BurnedInCaptionsAnalysis, V as BurnedInCaptionsOptions, W as BurnedInCaptionsPromptOverrides, X as BurnedInCaptionsPromptSections, Y as BurnedInCaptionsResult, Z as CaptionReplacement, _ as CensorMode, $ as Chapter, a0 as ChapterSystemPromptSections, a1 as ChaptersOptions, a2 as ChaptersPromptOverrides, a3 as ChaptersPromptSections, a4 as ChaptersResult, a5 as ChaptersType, a6 as DEFAULT_DESCRIPTION_LENGTH, a7 as DEFAULT_SUMMARY_KEYWORD_LIMIT, a8 as DEFAULT_TITLE_LENGTH, a9 as EditCaptionsOptions, aa as EditCaptionsResult, ab as EmbeddingsOptions, ac as EmbeddingsResult, ad as EngagementInsightsOptions, ae as EngagementInsightsResult, af as GOOGLE_VISION_LIKELIHOOD_TO_SCORE, ag as HIVE_SEXUAL_CATEGORIES, ah as HIVE_VIOLENCE_CATEGORIES, ai as HiveModerationOutput, aj as HiveModerationSource, ak as ModerationOptions, al as ModerationProvider, am as ModerationResult, an as MomentInsight, ao as OverallInsight, ap as ProfanityDetectionPayload, aq as Question, ar as QuestionAnswer, as as QuestionAnswerType, at as ReplacementRecord, au as SpeakerReplacement, av as StaticRenditionCleanupOutcome, aw as SummarizationOptions, ax as SummarizationPromptOverrides, ay as SummarizationPromptSections, az as SummaryAndTagsResult, aA as SummaryType, aB as ThumbnailModerationScore, aC as TranslationChunkingOptions, aD as TranslationOptions, aE as TranslationPayload, aF as TranslationResult, aG as aggregateTokenUsage, aH as applyOverrideLists, aI as applyReplacements, aJ as applySpeakerReplacements, aK as askQuestions, aL as buildReplacementRegex, aM as burnedInCaptionsSchema, aN as censorVttContent, aO as chapterSchema, aP as chaptersSchema, aQ as createReplacer, aR as editCaptions, aS as extractTranscriptSegmentsForHotspots, aT as generateChapters, aU as generateEmbeddings, aV as generateEngagementInsights, aW as generateVideoEmbeddings, aX as getModerationScores, aY as getSummaryAndTags, aZ as hasBurnedInCaptions, a_ as mapShotsToHotspots, a$ as normalizeTranslatedVtt, b0 as profanityDetectionSchema, b1 as questionAnswerSchema, b2 as shouldSplitChunkTranslationError, b3 as summarySchema, b4 as transformCueText, b5 as translateAudio, b6 as translateCaptions, b7 as translationSchema, b8 as validateNeverTranslateTerms, b9 as verifyNeverTranslateTerms } from '../index-C3PW73qH.js';
+export { E as ASK_QUESTIONS_DEFAULT_MAX_ANSWER_OPTION_LENGTH, F as ASK_QUESTIONS_DEFAULT_MAX_FREE_FORM_ANSWER_LENGTH, G as ASK_QUESTIONS_MAX_QUESTIONS_PER_CALL, H as ASK_QUESTIONS_MAX_QUESTION_LENGTH, J as AskQuestionsOptions, K as AskQuestionsResult, N as AskQuestionsType, O as AudioTranslationOptions, P as AudioTranslationResult, Q as AutoCensorProfanityOptions, U as BurnedInCaptionsAnalysis, V as BurnedInCaptionsOptions, W as BurnedInCaptionsPromptOverrides, X as BurnedInCaptionsPromptSections, Y as BurnedInCaptionsResult, Z as CaptionReplacement, _ as CensorMode, $ as Chapter, a0 as ChapterSystemPromptSections, a1 as ChaptersOptions, a2 as ChaptersPromptOverrides, a3 as ChaptersPromptSections, a4 as ChaptersResult, a5 as ChaptersType, a6 as DEFAULT_DESCRIPTION_LENGTH, a7 as DEFAULT_SUMMARY_KEYWORD_LIMIT, a8 as DEFAULT_TITLE_LENGTH, a9 as EditCaptionsOptions, aa as EditCaptionsResult, ab as EmbeddingsOptions, ac as EmbeddingsResult, ad as EngagementInsightsOptions, ae as EngagementInsightsResult, af as GOOGLE_VISION_LIKELIHOOD_TO_SCORE, ag as HIVE_SEXUAL_CATEGORIES, ah as HIVE_VIOLENCE_CATEGORIES, ai as HiveModerationOutput, aj as HiveModerationSource, ak as ModerationOptions, al as ModerationProvider, am as ModerationResult, an as MomentInsight, ao as OverallInsight, ap as ProfanityDetectionPayload, aq as Question, ar as QuestionAnswer, as as QuestionAnswerType, at as ReplacementRecord, au as SpeakerReplacement, av as StaticRenditionCleanupOutcome, aw as SummarizationOptions, ax as SummarizationPromptOverrides, ay as SummarizationPromptSections, az as SummaryAndTagsResult, aA as SummaryType, aB as ThumbnailModerationScore, aC as TranslationChunkingOptions, aD as TranslationOptions, aE as TranslationPayload, aF as TranslationResult, aG as aggregateTokenUsage, aH as applyOverrideLists, aI as applyReplacements, aJ as applySpeakerReplacements, aK as askQuestions, aL as buildReplacementRegex, aM as burnedInCaptionsSchema, aN as censorVttContent, aO as chapterSchema, aP as chaptersSchema, aQ as createReplacer, aR as editCaptions, aS as extractTranscriptSegmentsForHotspots, aT as generateChapters, aU as generateEmbeddings, aV as generateEngagementInsights, aW as generateVideoEmbeddings, aX as getModerationScores, aY as getSummaryAndTags, aZ as hasBurnedInCaptions, a_ as mapShotsToHotspots, a$ as normalizeTranslatedVtt, b0 as profanityDetectionSchema, b1 as questionAnswerSchema, b2 as shouldSplitChunkTranslationError, b3 as summarySchema, b4 as transformCueText, b5 as translateAudio, b6 as translateCaptions, b7 as translationSchema, b8 as validateNeverTranslateTerms, b9 as verifyNeverTranslateTerms } from '../index-NkSOF92U.js';
 import 'zod';
+import '../prompts/index.js';
 import '@ai-sdk/anthropic';
 import '@ai-sdk/google';
 import '@ai-sdk/openai';

package/dist/workflows/index.js

@@ -275,6 +275,7 @@ var PROMPT_TAG_NAMES = [
   "answer_guidelines",
   "relevance_filtering",
   "transcript_guidance",
+  "scene_guidance",
   "capabilities",
   "language_guidelines",
   "tone_guidance",
@@ -3571,6 +3572,17 @@ var AUDIO_ONLY_SYSTEM_PROMPT_OVERRIDES = {
     You receive a timestamped transcript from audio-only content with lines in the form "[12s] Transcript text".
     Use those timestamps as anchors to determine chapter start times in seconds.`
 };
+var SCENE_CONTEXT_SYSTEM_PROMPT_OVERRIDES = {
+  context: dedent4`
+    You receive a timestamped transcript with lines in the form "[12s] Caption text" and an ordered scene context.
+    Transcript timestamps are in seconds. Scene start and end timestamps are in milliseconds.
+    Use the scene progression to choose logical chapter groupings and scene starts as chapter boundaries.`,
+  constraints: dedent4`
+    - Only use information present in the transcript and scene context
+    - Return structured data that matches the requested JSON schema
+    - Do not add commentary or extra text outside the JSON
+    - When a <language> section is provided, all chapter titles MUST be written in that language`
+};
 var chaptersPromptBuilder = createPromptBuilder({
   template: {
     task: {
@@ -3608,6 +3620,7 @@ var chaptersPromptBuilder = createPromptBuilder({
 });
 function buildUserPrompt3({
   timestampedTranscript,
+  sceneContext,
   promptOverrides,
   minChaptersPerHour = 3,
   maxChaptersPerHour = 8,
@@ -3621,14 +3634,31 @@ function buildUserPrompt3({
       attributes: { format: "seconds" }
     }
   ];
+  if (sceneContext?.length) {
+    contextSections.push({
+      tag: "scene_context",
+      content: JSON.stringify(sceneContext, null, 2),
+      attributes: { timestamps: "milliseconds", trust: "untrusted_evidence" }
+    });
+  }
   if (languageName) {
     contextSections.push(createLanguageSection(languageName));
   }
+  const sceneBoundarySeconds = getSceneBoundarySeconds(sceneContext).filter(
+    (boundary) => scope ? boundary >= scope.startTime && boundary < scope.endTime : true
+  );
+  let firstChapterGuideline = "";
+  if (sceneBoundarySeconds.length > 0) {
+    firstChapterGuideline = `- The first chapter must start at ${sceneBoundarySeconds[0]}s, the first scene boundary inside the analyzed range`;
+  } else if (scope) {
+    firstChapterGuideline = `- The first chapter must start at ${scope.startTime}s`;
+  }
   const dynamicChapterGuidelines = dedent4`
     - Create at least ${minChaptersPerHour} and at most ${maxChaptersPerHour} chapters per hour of content
     - Use start times in seconds (not HH:MM:SS)
-    ${scope ? `- Analyze only the range from ${scope.startTime}s (inclusive) to ${scope.endTime}s (exclusive)
-- The first chapter must start at ${scope.startTime}s` : ""}
+    ${scope ? `- Analyze only the range from ${scope.startTime}s (inclusive) to ${scope.endTime}s (exclusive)` : ""}
+    ${firstChapterGuideline}
+    ${sceneContext?.length ? "- Use the ordered scene context to identify chapter-sized runs of consecutive scenes\n- Every chapter startTime must exactly equal a scene start_ms divided by 1000; never split a scene" : ""}
     - Chapter start times should be non-decreasing
     - Do not include text before or after the JSON`;
   const mergedOverrides = {
@@ -3637,6 +3667,32 @@ function buildUserPrompt3({
   };
   return chaptersPromptBuilder.buildWithContext(mergedOverrides, contextSections);
 }
+function getSceneBoundarySeconds(sceneContext) {
+  if (!sceneContext?.length) {
+    return [];
+  }
+  return [...new Set(sceneContext.map((scene) => scene.start_ms / 1e3))].sort((a, b) => a - b);
+}
+function alignChaptersToSceneBoundaries(chapters, sceneBoundarySeconds) {
+  if (sceneBoundarySeconds.length === 0) {
+    return chapters;
+  }
+  const chapterByBoundary = /* @__PURE__ */ new Map();
+  for (const chapter of chapters) {
+    const startTime = sceneBoundarySeconds.reduce(
+      (closest, boundary) => Math.abs(boundary - chapter.startTime) < Math.abs(closest - chapter.startTime) ? boundary : closest
+    );
+    const distance = Math.abs(startTime - chapter.startTime);
+    const existing = chapterByBoundary.get(startTime);
+    if (!existing || distance === 0 && existing.distance !== 0) {
+      chapterByBoundary.set(startTime, {
+        chapter: { ...chapter, startTime },
+        distance
+      });
+    }
+  }
+  return [...chapterByBoundary.values()].map(({ chapter }) => chapter).sort((a, b) => a.startTime - b.startTime);
+}
 async function generateChapters(assetId, options = {}) {
   "use workflow";
   const collectedUsage = [];
@@ -3656,7 +3712,8 @@ async function generateChaptersInternal(assetId, options, collectedUsage) {
     maxChaptersPerHour,
     credentials,
     outputLanguageCode,
-    scope
+    scope,
+    sceneContext
   } = options;
   const modelConfig = resolveLanguageModelConfig({
     ...options,
@@ -3698,8 +3755,18 @@ async function generateChaptersInternal(assetId, options, collectedUsage) {
   }
   const resolvedLanguageCode = outputLanguageCode && outputLanguageCode !== "auto" ? outputLanguageCode : getReliableLanguageCode(transcriptResult.track) ?? languageCode;
   const languageName = resolvedLanguageCode ? getLanguageName(resolvedLanguageCode) : void 0;
+  const sceneBoundarySeconds = getSceneBoundarySeconds(sceneContext).filter(
+    (boundary) => resolvedScope ? boundary >= resolvedScope.startTime && boundary < resolvedScope.endTime : true
+  );
+  if (sceneContext?.length && sceneBoundarySeconds.length === 0) {
+    throw new MuxAiError(
+      "Scene context has no chapter boundaries within the requested scope.",
+      { type: "validation_error" }
+    );
+  }
   const userPrompt = buildUserPrompt3({
     timestampedTranscript,
+    sceneContext,
     promptOverrides,
     minChaptersPerHour,
     maxChaptersPerHour,
@@ -3708,7 +3775,11 @@ async function generateChaptersInternal(assetId, options, collectedUsage) {
   });
   let chaptersData = null;
   try {
-    const systemPrompt = isAudioOnly ? chapterSystemPromptBuilder.build(AUDIO_ONLY_SYSTEM_PROMPT_OVERRIDES) : chapterSystemPromptBuilder.build();
+    const systemPromptOverrides = {
+      ...isAudioOnly ? AUDIO_ONLY_SYSTEM_PROMPT_OVERRIDES : {},
+      ...sceneContext?.length ? SCENE_CONTEXT_SYSTEM_PROMPT_OVERRIDES : {}
+    };
+    const systemPrompt = chapterSystemPromptBuilder.build(systemPromptOverrides);
     chaptersData = await generateChaptersWithAI({
       provider: modelConfig.provider,
       modelId: modelConfig.modelId,
@@ -3726,9 +3797,13 @@ async function generateChaptersInternal(assetId, options, collectedUsage) {
     throw new MuxAiError(`Failed to generate chapters for asset ${assetId}.`);
   }
   const { chapters: chaptersPayload, usage } = chaptersData;
-  const validChapters = chaptersPayload.chapters.filter((chapter) => typeof chapter.startTime === "number" && typeof chapter.title === "string").filter(
+  const scopedChapters = chaptersPayload.chapters.filter((chapter) => typeof chapter.startTime === "number" && typeof chapter.title === "string").filter(
     (chapter) => resolvedScope ? chapter.startTime >= resolvedScope.startTime && chapter.startTime < resolvedScope.endTime : true
   ).sort((a, b) => a.startTime - b.startTime);
+  const validChapters = alignChaptersToSceneBoundaries(
+    scopedChapters,
+    sceneBoundarySeconds
+  );
   if (validChapters.length === 0) {
     throw new MuxAiError(`Failed to generate valid chapters for asset ${assetId}.`);
   }
@@ -3754,9 +3829,11 @@ async function generateChaptersInternal(assetId, options, collectedUsage) {
   if (scrubbedChapters.length === 0) {
     throw new MuxAiError(`Failed to generate valid chapters for asset ${assetId}.`);
   }
-  const firstChapterStartTime = resolvedScope?.startTime ?? 0;
-  if (scrubbedChapters[0].startTime !== firstChapterStartTime) {
-    scrubbedChapters[0].startTime = firstChapterStartTime;
+  if (sceneBoundarySeconds.length === 0) {
+    const firstChapterStartTime = resolvedScope?.startTime ?? 0;
+    if (scrubbedChapters[0].startTime !== firstChapterStartTime) {
+      scrubbedChapters[0].startTime = firstChapterStartTime;
+    }
   }
   const usageWithMetadata = {
     ...usage,
@@ -6730,14 +6807,44 @@ function buildDescriptionGuidance(wordCount, contentType) {
     Write in present tense. Be specific about what is discussed or presented rather than making assumptions.
     Focus on the spoken content and any key insights, dialogue, or narrative elements.`;
 }
-function createSummarizationBuilder({ titleLength, descriptionLength, tagCount } = {}) {
+function buildSceneAwareDescriptionGuidance(wordCount) {
+  if (wordCount < DESCRIPTION_LENGTH_THRESHOLD_SMALL) {
+    return dedent6`A brief summary of the video in no more than ${wordCount} words. Shorter is fine.
+      Focus on the single most important subject or action across the ordered scenes.
+      Use the storyboard as direct visual evidence and the scene context to preserve progression.
+      Write in present tense.`;
+  }
+  if (wordCount > DESCRIPTION_LENGTH_THRESHOLD_LARGE) {
+    return dedent6`A detailed summary that describes what happens across the ordered scenes.
+      Never exceed ${wordCount} words, but shorter is perfectly fine. You may use multiple sentences.
+      Be thorough: cover subjects, actions, setting, progression, and notable details supported across scenes.
+      Use the storyboard as direct visual evidence and the scene context to preserve progression.
+      Write in present tense. If the transcript provides dialogue or narration, incorporate key points while preserving scene progression.`;
+  }
+  return dedent6`A summary that describes what happens across the ordered scenes.
+    Never exceed ${wordCount} words, but shorter is perfectly fine. You may use multiple sentences.
+    Cover the main subjects, actions, setting, and notable progression supported across scenes.
+    Use the storyboard as direct visual evidence and the scene context to preserve progression.
+    Write in present tense. If the transcript provides dialogue or narration, incorporate key points while preserving scene progression.`;
+}
+function createSummarizationBuilder({ titleLength, descriptionLength, tagCount } = {}, { hasSceneContext = false } = {}) {
   const titleLimit = titleLength ?? DEFAULT_TITLE_LENGTH;
   const keywordLimit = tagCount ?? DEFAULT_SUMMARY_KEYWORD_LIMIT;
+  const task = hasSceneContext ? "Analyze the storyboard frames and ordered scene context together, then generate metadata that captures the full progression of the video content." : "Analyze the storyboard frames and generate metadata that captures the essence of the video content.";
+  const qualityGuidelines = hasSceneContext ? dedent6`
+      - Follow scene_index order to understand what begins, develops, and concludes
+      - Use scene narratives and concepts as grounded evidence, preserving consistent terminology across the output
+      - Use the storyboard as direct visual evidence for whole-asset details
+      - Balance brevity with informativeness` : dedent6`
+      - Examine all frames to understand the full context and progression
+      - Be precise: "golden retriever" is better than "dog" when identifiable
+      - Capture the narrative: what begins, develops, and concludes
+      - Balance brevity with informativeness`;
   return createPromptBuilder({
     template: {
       task: {
         tag: "task",
-        content: "Analyze the storyboard frames and generate metadata that captures the essence of the video content."
+        content: task
       },
       title: {
         tag: "title_requirements",
@@ -6750,7 +6857,7 @@ function createSummarizationBuilder({ titleLength, descriptionLength, tagCount }
       },
       description: {
         tag: "description_requirements",
-        content: buildDescriptionGuidance(descriptionLength ?? DEFAULT_DESCRIPTION_LENGTH, "video")
+        content: hasSceneContext ? buildSceneAwareDescriptionGuidance(descriptionLength ?? DEFAULT_DESCRIPTION_LENGTH) : buildDescriptionGuidance(descriptionLength ?? DEFAULT_DESCRIPTION_LENGTH, "video")
       },
       keywords: {
         tag: "keywords_requirements",
@@ -6766,11 +6873,7 @@ function createSummarizationBuilder({ titleLength, descriptionLength, tagCount }
       },
       qualityGuidelines: {
         tag: "quality_guidelines",
-        content: dedent6`
-          - Examine all frames to understand the full context and progression
-          - Be precise: "golden retriever" is better than "dog" when identifiable
-          - Capture the narrative: what begins, develops, and concludes
-          - Balance brevity with informativeness`
+        content: qualityGuidelines
       }
     },
     sectionOrder: ["task", "title", "description", "keywords", "qualityGuidelines"]
@@ -6822,59 +6925,83 @@ function createAudioOnlyBuilder({ titleLength, descriptionLength, tagCount } = {
     sectionOrder: ["task", "title", "description", "keywords", "qualityGuidelines"]
   });
 }
-var SYSTEM_PROMPT5 = promptDedent`
-  <role>
-    You are a video content analyst specializing in storyboard interpretation and multimodal analysis.
-  </role>
-
-  <context>
-    You receive storyboard images containing multiple sequential frames extracted from a video.
-    ${STORYBOARD_FRAME_INSTRUCTIONS}
-  </context>
-
-  <transcript_guidance>
-    When a transcript is provided alongside the storyboard:
-    - Use it to understand spoken content, dialogue, narration, and audio context
-    - Correlate transcript content with visual frames to build a complete picture
-    - Extract key terminology, names, and specific language used by speakers
-    - Let the transcript inform keyword selection, especially for topics not visually obvious
-    - Prioritize visual content for the description, but enrich it with transcript insights
-    - If transcript and visuals conflict, trust the visual evidence
-  </transcript_guidance>
-
-  <capabilities>
-    - Extract meaning from visual sequences
-    - Identify subjects, actions, settings, and narrative arcs
-    - Generate accurate, searchable metadata
-    - Synthesize visual and transcript information when provided
-  </capabilities>
-
-  <security>
-    ${NON_DISCLOSURE_CONSTRAINT}
-
-    ${UNTRUSTED_USER_INPUT_NOTICE}
-
-    ${VISUAL_TEXT_AS_CONTENT}
-
-    ${CANARY_TRIPWIRE}
-  </security>
-
-  <constraints>
-    - Only describe what is clearly observable in the frames or explicitly stated in the transcript
-    - ${NO_FABRICATION_CONSTRAINT}
-    - ${METADATA_BOUNDARY_WARNING}
-    - ${STRUCTURED_DATA_CONSTRAINT}
-    - Output only the JSON object; no markdown or extra text
-    - When a <language> section is provided, all output text MUST be written in that language
-  </constraints>
-
-  <tone_guidance>
-    ${TONE_GUIDANCE}
-  </tone_guidance>
-
-  <language_guidelines>
-    ${createLanguageGuidelines("video")}
-  </language_guidelines>`;
+function createVideoSystemPrompt(hasSceneContext) {
+  let sceneContext = "";
+  let sceneGuidance = "";
+  if (hasSceneContext) {
+    sceneContext = promptDedent`
+      Ordered scene context is also provided as structured evidence of the video's progression.
+      Entries use stable, zero-based scene_index values and absolute millisecond timestamps.
+      Read scenes in scene_index order to understand what begins, develops, and concludes.`;
+    sceneGuidance = promptDedent`
+      <scene_guidance>
+        - Use scene titles, narratives, and concepts to understand temporal progression
+        - Use storyboard frames as direct visual evidence for subjects, actions, settings, and details
+        - Preserve consistent terminology from the scene context across the title, description, and keywords
+        - Treat optional scene fields as evidence only when present; do not infer omitted details
+        - Consider all provided evidence together; if derived scene context conflicts with storyboard visuals, trust the visuals
+      </scene_guidance>`;
+  }
+  const transcriptEvidence = hasSceneContext ? "storyboard frames and ordered scenes" : "visual frames";
+  const descriptionPriority = hasSceneContext ? "Use the scene context for temporal structure and the storyboard for direct visual details, enriching both with transcript insights" : "Prioritize visual content for the description, but enrich it with transcript insights";
+  const groundingConstraint = hasSceneContext ? "Use all provided evidence together. Ground each claim in the storyboard, scene context, or transcript" : "Only describe what is clearly observable in the frames or explicitly stated in the transcript";
+  return promptDedent`
+    <role>
+      You are a video content analyst specializing in storyboard interpretation and multimodal analysis.
+    </role>
+
+    <context>
+      You receive storyboard images containing multiple sequential frames extracted from a video.
+      ${STORYBOARD_FRAME_INSTRUCTIONS}
+      ${sceneContext}
+    </context>
+
+    ${sceneGuidance}
+
+    <transcript_guidance>
+      When a transcript is provided alongside the storyboard:
+      - Use it to understand spoken content, dialogue, narration, and audio context
+      - Correlate transcript content with ${transcriptEvidence} to build a complete picture
+      - Extract key terminology, names, and specific language used by speakers
+      - Let the transcript inform keyword selection, especially for topics not visually obvious
+      - ${descriptionPriority}
+      - If transcript and visuals conflict, trust the visual evidence
+    </transcript_guidance>
+
+    <capabilities>
+      - Extract meaning from visual sequences
+      - Identify subjects, actions, settings, and narrative arcs
+      - Generate accurate, searchable metadata
+      - Synthesize visual and transcript information when provided
+    </capabilities>
+
+    <security>
+      ${NON_DISCLOSURE_CONSTRAINT}
+
+      ${UNTRUSTED_USER_INPUT_NOTICE}
+
+      ${VISUAL_TEXT_AS_CONTENT}
+
+      ${CANARY_TRIPWIRE}
+    </security>
+
+    <constraints>
+      - ${groundingConstraint}
+      - ${NO_FABRICATION_CONSTRAINT}
+      - ${METADATA_BOUNDARY_WARNING}
+      - ${STRUCTURED_DATA_CONSTRAINT}
+      - Output only the JSON object; no markdown or extra text
+      - When a <language> section is provided, all output text MUST be written in that language
+    </constraints>
+
+    <tone_guidance>
+      ${TONE_GUIDANCE}
+    </tone_guidance>
+
+    <language_guidelines>
+      ${createLanguageGuidelines("video")}
+    </language_guidelines>`;
+}
 var AUDIO_ONLY_SYSTEM_PROMPT3 = promptDedent`
   <role>
     You are an audio content analyst specializing in transcript analysis and metadata generation.
@@ -6925,6 +7052,16 @@ var AUDIO_ONLY_SYSTEM_PROMPT3 = promptDedent`
   <language_guidelines>
     ${createLanguageGuidelines("audio")}
   </language_guidelines>`;
+function createSceneContextSection(sceneContext) {
+  return {
+    tag: "scene_context",
+    content: JSON.stringify(sceneContext),
+    attributes: {
+      format: "json",
+      order: "scene_index_ascending"
+    }
+  };
+}
 function buildUserPrompt5({
   tone,
   transcriptText,
@@ -6934,7 +7071,8 @@ function buildUserPrompt5({
   titleLength,
   descriptionLength,
   tagCount,
-  languageName
+  languageName,
+  sceneContext
 }) {
   const contextSections = [createToneSection(TONE_INSTRUCTIONS[tone])];
   if (languageName) {
@@ -6949,11 +7087,15 @@ function buildUserPrompt5({
     const format = isCleanTranscript ? "plain text" : "WebVTT";
     contextSections.push(createTranscriptSection(transcriptText, format));
   }
+  const hasSceneContext = !isAudioOnly && Boolean(sceneContext?.length);
+  if (hasSceneContext && sceneContext) {
+    contextSections.push(createSceneContextSection(sceneContext));
+  }
   const constraints = { titleLength, descriptionLength, tagCount };
-  const promptBuilder = isAudioOnly ? createAudioOnlyBuilder(constraints) : createSummarizationBuilder(constraints);
+  const promptBuilder = isAudioOnly ? createAudioOnlyBuilder(constraints) : createSummarizationBuilder(constraints, { hasSceneContext });
   return promptBuilder.buildWithContext(promptOverrides, contextSections);
 }
-async function analyzeStoryboard2(imageDataUrl, provider, modelId, userPrompt, systemPrompt, descriptionLength, credentials) {
+async function analyzeVideo(imageDataUrl, provider, modelId, userPrompt, systemPrompt, descriptionLength, credentials) {
   "use step";
   const model = await createLanguageModelFromConfig(provider, modelId, credentials);
   const schema = buildSummarySchema(descriptionLength);
@@ -6969,10 +7111,10 @@ async function analyzeStoryboard2(imageDataUrl, provider, modelId, userPrompt, s
     messages: [
       {
         role: "user",
-        content: [
+        content: imageDataUrl ? [
           { type: "text", text: userPrompt },
           { type: "image", image: imageDataUrl }
-        ]
+        ] : userPrompt
       }
     ]
   }));
@@ -7082,6 +7224,7 @@ async function getSummaryAndTagsInternal(assetId, options, collectedUsage) {
     cleanTranscript = true,
     imageSubmissionMode = "url",
     imageDownloadOptions,
+    sceneContext,
     promptOverrides,
     credentials,
     titleLength,
@@ -7105,6 +7248,7 @@ async function getSummaryAndTagsInternal(assetId, options, collectedUsage) {
   const { asset: assetData, playbackId, policy } = await getPlaybackIdForAsset(assetId, workflowCredentials, options?.assetSnapshot);
   const assetDurationSeconds = getAssetDurationSecondsFromAsset(assetData);
   const isAudioOnly = isAudioOnlyAsset(assetData);
+  const hasSceneContext = !isAudioOnly && Boolean(sceneContext?.length);
   const effectiveScope = hasWorkflowScopeBoundaries(scope) ? scope : void 0;
   const storyboardScope = isAudioOnly ? void 0 : resolveRenderableVideoScope(
     effectiveScope,
@@ -7121,7 +7265,8 @@ async function getSummaryAndTagsInternal(assetId, options, collectedUsage) {
     );
   }
   const signingContext = await resolveMuxSigningContext(workflowCredentials);
-  if (policy === "signed" && !signingContext) {
+  const requiresSignedPlaybackAccess = includeTranscript || !isAudioOnly;
+  if (policy === "signed" && requiresSignedPlaybackAccess && !signingContext) {
     throw new MuxAiError(
       "Signed playback ID requires signing credentials. Set MUX_SIGNING_KEY and MUX_PRIVATE_KEY environment variables.",
       { type: "validation_error" }
@@ -7147,11 +7292,15 @@ async function getSummaryAndTagsInternal(assetId, options, collectedUsage) {
     titleLength,
     descriptionLength,
     tagCount,
-    languageName
+    languageName,
+    sceneContext: hasSceneContext ? sceneContext : void 0
   });
   let analysisResponse;
   let imageUrl;
-  const systemPrompt = isAudioOnly ? AUDIO_ONLY_SYSTEM_PROMPT3 : SYSTEM_PROMPT5;
+  let systemPrompt = AUDIO_ONLY_SYSTEM_PROMPT3;
+  if (!isAudioOnly) {
+    systemPrompt = createVideoSystemPrompt(hasSceneContext);
+  }
   const effectiveDescriptionLength = descriptionLength ?? DEFAULT_DESCRIPTION_LENGTH;
   try {
     if (isAudioOnly) {
@@ -7174,7 +7323,7 @@ async function getSummaryAndTagsInternal(assetId, options, collectedUsage) {
       imageUrl = storyboardUrl;
       if (imageSubmissionMode === "base64") {
         const downloadResult = await downloadImageAsBase64(storyboardUrl, imageDownloadOptions);
-        analysisResponse = await analyzeStoryboard2(
+        analysisResponse = await analyzeVideo(
           downloadResult.base64Data,
           modelConfig.provider,
           modelConfig.modelId,
@@ -7184,7 +7333,7 @@ async function getSummaryAndTagsInternal(assetId, options, collectedUsage) {
           workflowCredentials
         );
       } else {
-        analysisResponse = await analyzeStoryboard2(
+        analysisResponse = await analyzeVideo(
           storyboardUrl,
           modelConfig.provider,
           modelConfig.modelId,
@@ -7769,7 +7918,7 @@ import { z as z9 } from "zod";
 var translationSchema = z9.object({
   translation: z9.string()
 });
-var SYSTEM_PROMPT6 = promptDedent`
+var SYSTEM_PROMPT5 = promptDedent`
   You are a subtitle translation expert. Translate VTT subtitle files to the target language specified by the user.
   You may receive either a full VTT file or a chunk from a larger VTT.
   Preserve all timestamps, cue ordering, and VTT formatting exactly as they appear.
@@ -8094,7 +8243,7 @@ async function translateVttWithAI({
     model,
     maxRetries: 0,
     output: Output7.object({ schema: translationSchema }),
-    system: SYSTEM_PROMPT6,
+    system: SYSTEM_PROMPT5,
     messages: [
       {
         role: "user",