@mux/ai 0.15.1 → 0.16.1

package/dist/workflows/index.d.ts

@@ -1,4 +1,4 @@
-export { A as AskQuestionsOptions, a as AskQuestionsResult, b as AskQuestionsType, c as AudioTranslationOptions, d as AudioTranslationResult, e as AutoCensorProfanityOptions, B as BurnedInCaptionsAnalysis, f as BurnedInCaptionsOptions, g as BurnedInCaptionsPromptOverrides, h as BurnedInCaptionsPromptSections, j as BurnedInCaptionsResult, C as CaptionReplacement, k as CensorMode, l as Chapter, m as ChapterSystemPromptSections, n as ChaptersOptions, o as ChaptersPromptOverrides, p as ChaptersPromptSections, q as ChaptersResult, r as ChaptersType, D as DEFAULT_DESCRIPTION_LENGTH, s as DEFAULT_SUMMARY_KEYWORD_LIMIT, t as DEFAULT_TITLE_LENGTH, E as EditCaptionsOptions, u as EditCaptionsResult, v as EmbeddingsOptions, w as EmbeddingsResult, x as EngagementInsightsOptions, y as EngagementInsightsResult, H as HIVE_SEXUAL_CATEGORIES, z as HIVE_VIOLENCE_CATEGORIES, F as HiveModerationOutput, G as HiveModerationSource, M as ModerationOptions, I as ModerationProvider, J as ModerationResult, K as MomentInsight, O as OverallInsight, P as ProfanityDetectionPayload, Q as Question, L as QuestionAnswer, N as QuestionAnswerType, R as ReplacementRecord, S as SummarizationOptions, T as SummarizationPromptOverrides, U as SummarizationPromptSections, V as SummaryAndTagsResult, W as SummaryType, X as ThumbnailModerationScore, Y as TranslationChunkingOptions, Z as TranslationOptions, _ as TranslationPayload, $ as TranslationResult, a0 as aggregateTokenUsage, a1 as applyOverrideLists, a2 as applyReplacements, a3 as askQuestions, a4 as buildReplacementRegex, a5 as burnedInCaptionsSchema, a6 as censorVttContent, a7 as chapterSchema, a8 as chaptersSchema, a9 as createReplacer, aa as editCaptions, ab as extractTranscriptSegmentsForHotspots, ac as generateChapters, ad as generateEmbeddings, ae as generateEngagementInsights, af as generateVideoEmbeddings, ag as getModerationScores, ah as getSummaryAndTags, ai as hasBurnedInCaptions, aj as mapShotsToHotspots, ak as profanityDetectionSchema, al as questionAnswerSchema, am as shouldSplitChunkTranslationError, an as summarySchema, ao as transformCueText, ap as translateAudio, aq as translateCaptions, ar as translationSchema } from '../index-yn3fUDgv.js';
+export { A as AskQuestionsOptions, a as AskQuestionsResult, b as AskQuestionsType, c as AudioTranslationOptions, d as AudioTranslationResult, e as AutoCensorProfanityOptions, B as BurnedInCaptionsAnalysis, f as BurnedInCaptionsOptions, g as BurnedInCaptionsPromptOverrides, h as BurnedInCaptionsPromptSections, j as BurnedInCaptionsResult, C as CaptionReplacement, k as CensorMode, l as Chapter, m as ChapterSystemPromptSections, n as ChaptersOptions, o as ChaptersPromptOverrides, p as ChaptersPromptSections, q as ChaptersResult, r as ChaptersType, D as DEFAULT_DESCRIPTION_LENGTH, s as DEFAULT_SUMMARY_KEYWORD_LIMIT, t as DEFAULT_TITLE_LENGTH, E as EditCaptionsOptions, u as EditCaptionsResult, v as EmbeddingsOptions, w as EmbeddingsResult, x as EngagementInsightsOptions, y as EngagementInsightsResult, H as HIVE_SEXUAL_CATEGORIES, z as HIVE_VIOLENCE_CATEGORIES, F as HiveModerationOutput, G as HiveModerationSource, M as ModerationOptions, I as ModerationProvider, J as ModerationResult, K as MomentInsight, O as OverallInsight, P as ProfanityDetectionPayload, Q as Question, L as QuestionAnswer, N as QuestionAnswerType, R as ReplacementRecord, S as SummarizationOptions, T as SummarizationPromptOverrides, U as SummarizationPromptSections, V as SummaryAndTagsResult, W as SummaryType, X as ThumbnailModerationScore, Y as TranslationChunkingOptions, Z as TranslationOptions, _ as TranslationPayload, $ as TranslationResult, a0 as aggregateTokenUsage, a1 as applyOverrideLists, a2 as applyReplacements, a3 as askQuestions, a4 as buildReplacementRegex, a5 as burnedInCaptionsSchema, a6 as censorVttContent, a7 as chapterSchema, a8 as chaptersSchema, a9 as createReplacer, aa as editCaptions, ab as extractTranscriptSegmentsForHotspots, ac as generateChapters, ad as generateEmbeddings, ae as generateEngagementInsights, af as generateVideoEmbeddings, ag as getModerationScores, ah as getSummaryAndTags, ai as hasBurnedInCaptions, aj as mapShotsToHotspots, ak as profanityDetectionSchema, al as questionAnswerSchema, am as shouldSplitChunkTranslationError, an as summarySchema, ao as transformCueText, ap as translateAudio, aq as translateCaptions, ar as translationSchema } from '../index-BfsJvx3r.js';
 import 'zod';
 import '@ai-sdk/anthropic';
 import '@ai-sdk/google';

package/dist/workflows/index.js

@@ -116,7 +116,7 @@ var MuxAiError = class extends Error {
     super(message);
     /** Structural brand for duck-type identification. */
     this.__robots_error = true;
-    this.name = "MuxAiError";
+    this.name = "FatalError";
     this.publicType = opts?.type ?? "processing_error";
     this.publicMessage = message;
     this.retryable = opts?.retryable ?? false;
@@ -794,7 +794,7 @@ function renderSection(section) {
       throw new Error(`Invalid XML ${context} name: "${name}"`);
     }
   };
-  const escapeXmlText = (value) => value.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;/");
+  const escapeXmlText = (value) => value.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
   const escapeXmlAttribute = (value) => escapeXmlText(value).replace(/"/g, "&quot;");
   if (!content.trim()) {
     return "";
@@ -1342,7 +1342,7 @@ async function fetchTranscriptForAsset(asset, playbackId, options = {}) {
 
 // src/workflows/ask-questions.ts
 var questionAnswerSchema = z2.object({
-  question: z2.string(),
+  question: z2.string().describe("The full text of the original question"),
   answer: z2.string().nullable(),
   confidence: z2.number(),
   reasoning: z2.string(),
@@ -1388,14 +1388,17 @@ var SYSTEM_PROMPT = dedent`
   <task>
     For each question provided, you must:
     1. Analyze the storyboard frames and transcript (if provided)
-    2. Answer with ONLY the allowed response options - no other values are acceptable
+    2. Answer with ONLY the allowed response options listed for that question - no other values are acceptable
     3. Provide a confidence score between 0 and 1 reflecting your certainty
     4. Explain your reasoning based on observable evidence
   </task>
 
   <answer_guidelines>
-    - Choose the affirmative option only if you have clear evidence supporting it
-    - Choose the negative/contradicting option if evidence contradicts or if insufficient evidence exists
+    - Each question is presented as its own <question> block with a <text> element (the question) and an <allowed_answers> element (the permitted response values)
+    - Choose only from the values listed in that question's <allowed_answers> element
+    - Questions may have any set of allowed answers. Always read the <allowed_answers> for each question and select the best matching option based on the evidence
+    - Select the answer best supported by observable evidence from the content
+    - When evidence is ambiguous but some signal exists, select the most conservative option and use a low confidence score. If the question cannot be answered at all from the content, skip it per the relevance_filtering rules
     - Confidence should reflect the clarity and strength of evidence:
       * 0.9-1.0: Clear, unambiguous evidence
       * 0.7-0.9: Strong evidence with minor ambiguity
@@ -1441,7 +1444,7 @@ var SYSTEM_PROMPT = dedent`
   </relevance_filtering>
 
   <constraints>
-    - You MUST answer every relevant question with one of the allowed response options
+    - You MUST answer every relevant question with one of its own listed allowed response options
     - Skip irrelevant questions as described in relevance_filtering
     - Only describe observable evidence from frames or transcript
     - Do not fabricate details or make unsupported assumptions
@@ -1476,14 +1479,17 @@ var AUDIO_ONLY_SYSTEM_PROMPT = dedent`
   <task>
     For each question provided, you must:
     1. Analyze the transcript
-    2. Answer with ONLY the allowed response options - no other values are acceptable
+    2. Answer with ONLY the allowed response options listed for that question - no other values are acceptable
     3. Provide a confidence score between 0 and 1 reflecting your certainty
     4. Explain your reasoning based on observable evidence
   </task>
 
   <answer_guidelines>
-    - Choose the affirmative option only if you have clear evidence supporting it
-    - Choose the negative/contradicting option if evidence contradicts or if insufficient evidence exists
+    - Each question is presented as its own <question> block with a <text> element (the question) and an <allowed_answers> element (the permitted response values)
+    - Choose only from the values listed in that question's <allowed_answers> element
+    - Questions may have any set of allowed answers. Always read the <allowed_answers> for each question and select the best matching option based on the evidence
+    - Select the answer best supported by observable evidence from the content
+    - When evidence is ambiguous but some signal exists, select the most conservative option and use a low confidence score. If the question cannot be answered at all from the content, skip it per the relevance_filtering rules
     - Confidence should reflect the clarity and strength of evidence:
       * 0.9-1.0: Clear, unambiguous evidence
       * 0.7-0.9: Strong evidence with minor ambiguity
@@ -1529,7 +1535,7 @@ var AUDIO_ONLY_SYSTEM_PROMPT = dedent`
   </relevance_filtering>
 
   <constraints>
-    - You MUST answer every relevant question with one of the allowed response options
+    - You MUST answer every relevant question with one of its own listed allowed response options
     - Skip irrelevant questions as described in relevance_filtering
     - Only describe observable evidence from transcript content
     - Do not fabricate details or make unsupported assumptions
@@ -1544,46 +1550,43 @@ var AUDIO_ONLY_SYSTEM_PROMPT = dedent`
     - GOOD: "The speaker describes running through a park"
     - Be specific and evidence-based
   </language_guidelines>`;
-var askQuestionsPromptBuilder = createPromptBuilder({
-  template: {
-    questions: {
-      tag: "questions",
-      content: "Please answer the following questions about this content:"
-    }
-  },
-  sectionOrder: ["questions"]
-});
 function buildUserPrompt({
   questions,
-  allowedAnswers,
   transcriptText,
   isCleanTranscript = true,
   isAudioOnly = false
 }) {
-  const questionsList = questions.map((q, idx) => `${idx + 1}. ${q.question}`).join("\n");
-  const questionsIntro = isAudioOnly ? "Please answer the following questions about this audio content using transcript evidence:" : "Please answer the following questions about this content:";
-  const questionsContent = dedent`
-    ${questionsIntro}
-
-    ${questionsList}`;
-  const answerList = allowedAnswers.map((answer) => `"${answer}"`).join(", ");
-  const responseOptions = dedent`
-    <response_options>
-      Allowed answers: ${answerList}
-    </response_options>`;
-  const questionsSection = askQuestionsPromptBuilder.build({ questions: questionsContent });
-  if (!transcriptText) {
-    return `${questionsSection}
-
-${responseOptions}`;
+  const contentDescriptor = isAudioOnly ? "audio content using transcript evidence" : "content";
+  const taskContent = dedent`
+    Answer each question in the <questions> block below about the ${contentDescriptor}.
+    Use only the values listed inside each question's own <allowed_answers> element.
+    Return one answer per question, in the order the questions appear.
+    If a question cannot be answered from the provided content, skip it as described in the system instructions.`;
+  const taskSection = `<task>
+${taskContent}
+</task>`;
+  const questionBlocks = questions.map((q, idx) => {
+    const textTag = renderSection({ tag: "text", content: q.question });
+    const answersTag = renderSection({
+      tag: "allowed_answers",
+      content: q.answerOptions.map((a) => `"${a}"`).join(", ")
+    });
+    return `<question number="${idx + 1}">
+${textTag}
+${answersTag}
+</question>`;
+  }).join("\n\n");
+  const questionsSection = `<questions>
+${questionBlocks}
+</questions>`;
+  const sections = [];
+  if (transcriptText) {
+    const format = isCleanTranscript ? "plain text" : "WebVTT";
+    sections.push(renderSection(createTranscriptSection(transcriptText, format)));
   }
-  const format = isCleanTranscript ? "plain text" : "WebVTT";
-  const transcriptSection = renderSection(createTranscriptSection(transcriptText, format));
-  return `${transcriptSection}
-
-${questionsSection}
-
-${responseOptions}`;
+  sections.push(taskSection);
+  sections.push(questionsSection);
+  return sections.join("\n\n");
 }
 async function fetchImageAsBase64(imageUrl, imageDownloadOptions) {
   "use step";
@@ -1652,7 +1655,6 @@ async function askQuestions(assetId, questions, options) {
     provider = "openai",
     model,
     languageCode,
-    answerOptions,
     includeTranscript = true,
     cleanTranscript = true,
     imageSubmissionMode = "url",
@@ -1660,15 +1662,23 @@ async function askQuestions(assetId, questions, options) {
     storyboardWidth = 640,
     credentials
   } = options ?? {};
-  const normalizedAnswerOptions = Array.from(
-    new Set(
-      (answerOptions?.length ? answerOptions : ["yes", "no"]).map((option) => option.trim()).filter(Boolean)
-    )
+  const normalizedQuestions = questions.map((q, idx) => {
+    const options2 = Array.from(
+      new Set(
+        (q.answerOptions?.length ? q.answerOptions : ["yes", "no"]).map((o) => o.trim()).filter(Boolean)
+      )
+    );
+    if (!options2.length) {
+      throw new MuxAiError(
+        `Question at index ${idx} has invalid answerOptions: must include at least one non-empty value.`,
+        { type: "validation_error" }
+      );
+    }
+    return { ...q, answerOptions: options2 };
+  });
+  const allowedAnswers = Array.from(
+    new Set(normalizedQuestions.flatMap((q) => q.answerOptions))
   );
-  if (!normalizedAnswerOptions.length) {
-    throw new MuxAiError("answerOptions must include at least one non-empty value.", { type: "validation_error" });
-  }
-  const allowedAnswers = normalizedAnswerOptions;
   const modelConfig = resolveLanguageModelConfig({
     ...options,
     model,
@@ -1699,8 +1709,7 @@ async function askQuestions(assetId, questions, options) {
   }) : void 0;
   const transcriptText = transcriptResult?.transcriptText ?? "";
   const userPrompt = buildUserPrompt({
-    questions,
-    allowedAnswers,
+    questions: normalizedQuestions,
     transcriptText,
     isCleanTranscript: cleanTranscript,
     isAudioOnly
@@ -1763,10 +1772,15 @@ async function askQuestions(assetId, questions, options) {
       `Failed to generate answers for all questions for asset ${assetId}.`
     );
   }
-  const answers = analysisResponse.result.answers.map((raw) => {
+  const answers = analysisResponse.result.answers.map((raw, idx) => {
     const isSkipped = raw.skipped || raw.answer === SKIP_SENTINEL;
+    if (!isSkipped && !normalizedQuestions[idx].answerOptions.includes(raw.answer)) {
+      throw new MuxAiError(
+        `Answer "${raw.answer}" for question ${idx} is not in allowed options: ${normalizedQuestions[idx].answerOptions.join(", ")}`
+      );
+    }
     return {
-      // Strip numbering prefix (e.g., "1. " or "2. ") from questions
+      // Strip numbering prefix (e.g. "1. ") if the LLM prepends one
       question: raw.question.replace(/^\d+\.\s*/, ""),
       confidence: isSkipped ? 0 : Math.min(1, Math.max(0, raw.confidence)),
       reasoning: raw.reasoning,