@mux/ai 0.19.0 → 0.20.0

package/dist/workflows/index.d.ts

@@ -1,4 +1,4 @@
-export { A as AskQuestionsOptions, a as AskQuestionsResult, b as AskQuestionsType, c as AudioTranslationOptions, d as AudioTranslationResult, e as AutoCensorProfanityOptions, B as BurnedInCaptionsAnalysis, f as BurnedInCaptionsOptions, g as BurnedInCaptionsPromptOverrides, h as BurnedInCaptionsPromptSections, j as BurnedInCaptionsResult, C as CaptionReplacement, k as CensorMode, l as Chapter, m as ChapterSystemPromptSections, n as ChaptersOptions, o as ChaptersPromptOverrides, p as ChaptersPromptSections, q as ChaptersResult, r as ChaptersType, D as DEFAULT_DESCRIPTION_LENGTH, s as DEFAULT_SUMMARY_KEYWORD_LIMIT, t as DEFAULT_TITLE_LENGTH, E as EditCaptionsOptions, u as EditCaptionsResult, v as EmbeddingsOptions, w as EmbeddingsResult, x as EngagementInsightsOptions, y as EngagementInsightsResult, H as HIVE_SEXUAL_CATEGORIES, z as HIVE_VIOLENCE_CATEGORIES, F as HiveModerationOutput, G as HiveModerationSource, M as ModerationOptions, I as ModerationProvider, J as ModerationResult, K as MomentInsight, O as OverallInsight, P as ProfanityDetectionPayload, Q as Question, L as QuestionAnswer, N as QuestionAnswerType, R as ReplacementRecord, S as SummarizationOptions, T as SummarizationPromptOverrides, U as SummarizationPromptSections, V as SummaryAndTagsResult, W as SummaryType, X as ThumbnailModerationScore, Y as TranslationChunkingOptions, Z as TranslationOptions, _ as TranslationPayload, $ as TranslationResult, a0 as aggregateTokenUsage, a1 as applyOverrideLists, a2 as applyReplacements, a3 as askQuestions, a4 as buildReplacementRegex, a5 as burnedInCaptionsSchema, a6 as censorVttContent, a7 as chapterSchema, a8 as chaptersSchema, a9 as createReplacer, aa as editCaptions, ab as extractTranscriptSegmentsForHotspots, ac as generateChapters, ad as generateEmbeddings, ae as generateEngagementInsights, af as generateVideoEmbeddings, ag as getModerationScores, ah as getSummaryAndTags, ai as hasBurnedInCaptions, aj as mapShotsToHotspots, ak as normalizeTranslatedVtt, al as profanityDetectionSchema, am as questionAnswerSchema, an as shouldSplitChunkTranslationError, ao as summarySchema, ap as transformCueText, aq as translateAudio, ar as translateCaptions, as as translationSchema } from '../index-CwzNfBjV.js';
+export { A as AskQuestionsOptions, a as AskQuestionsResult, b as AskQuestionsType, c as AudioTranslationOptions, d as AudioTranslationResult, e as AutoCensorProfanityOptions, B as BurnedInCaptionsAnalysis, f as BurnedInCaptionsOptions, g as BurnedInCaptionsPromptOverrides, h as BurnedInCaptionsPromptSections, j as BurnedInCaptionsResult, C as CaptionReplacement, k as CensorMode, l as Chapter, m as ChapterSystemPromptSections, n as ChaptersOptions, o as ChaptersPromptOverrides, p as ChaptersPromptSections, q as ChaptersResult, r as ChaptersType, D as DEFAULT_DESCRIPTION_LENGTH, s as DEFAULT_SUMMARY_KEYWORD_LIMIT, t as DEFAULT_TITLE_LENGTH, E as EditCaptionsOptions, u as EditCaptionsResult, v as EmbeddingsOptions, w as EmbeddingsResult, x as EngagementInsightsOptions, y as EngagementInsightsResult, H as HIVE_SEXUAL_CATEGORIES, z as HIVE_VIOLENCE_CATEGORIES, F as HiveModerationOutput, G as HiveModerationSource, M as ModerationOptions, I as ModerationProvider, J as ModerationResult, K as MomentInsight, O as OverallInsight, P as ProfanityDetectionPayload, Q as Question, L as QuestionAnswer, N as QuestionAnswerType, R as ReplacementRecord, S as SummarizationOptions, T as SummarizationPromptOverrides, U as SummarizationPromptSections, V as SummaryAndTagsResult, W as SummaryType, X as ThumbnailModerationScore, Y as TranslationChunkingOptions, Z as TranslationOptions, _ as TranslationPayload, $ as TranslationResult, a0 as aggregateTokenUsage, a1 as applyOverrideLists, a2 as applyReplacements, a3 as askQuestions, a4 as buildReplacementRegex, a5 as burnedInCaptionsSchema, a6 as censorVttContent, a7 as chapterSchema, a8 as chaptersSchema, a9 as createReplacer, aa as editCaptions, ab as extractTranscriptSegmentsForHotspots, ac as generateChapters, ad as generateEmbeddings, ae as generateEngagementInsights, af as generateVideoEmbeddings, ag as getModerationScores, ah as getSummaryAndTags, ai as hasBurnedInCaptions, aj as mapShotsToHotspots, ak as normalizeTranslatedVtt, al as profanityDetectionSchema, am as questionAnswerSchema, an as shouldSplitChunkTranslationError, ao as summarySchema, ap as transformCueText, aq as translateAudio, ar as translateCaptions, as as translationSchema } from '../index-DXMIJjUa.js';
 import 'zod';
 import '@ai-sdk/anthropic';
 import '@ai-sdk/google';

package/dist/workflows/index.js

@@ -1891,8 +1891,7 @@ var questionAnswerSchema = z2.object({
   skipped: z2.boolean()
 });
 var SKIP_SENTINEL = "__SKIPPED__";
-function createAskQuestionsSchema(allowedAnswers) {
-  const answerSchema = z2.enum([...allowedAnswers, SKIP_SENTINEL]);
+function createAskQuestionsSchema(answerSchema) {
   return z2.object({
     answers: z2.array(
       questionAnswerSchema.extend({
@@ -1928,15 +1927,16 @@ var SYSTEM_PROMPT = promptDedent`
   <task>
     For each question provided, you must:
     1. Analyze the storyboard frames and transcript (if provided)
-    2. Answer with ONLY the allowed response options listed for that question - no other values are acceptable
+    2. Answer each question according to its specified response format: for questions with <allowed_answers>, select ONLY from the listed values; for questions with <answer_format>, follow the format specification exactly
     3. Provide a confidence score between 0 and 1 reflecting your certainty
     4. Explain your reasoning based on observable evidence
   </task>
 
   <answer_guidelines>
-    - Each question is presented as its own <question> block with a <text> element (the question) and an <allowed_answers> element (the permitted response values)
-    - Choose only from the values listed in that question's <allowed_answers> element
-    - Questions may have any set of allowed answers. Always read the <allowed_answers> for each question and select the best matching option based on the evidence
+    - Each question is presented as its own <question> block with a <text> element and either an <allowed_answers> element (listing the permitted response values) or an <answer_format> element (describing the required response shape for open-ended responses)
+    - For questions with <allowed_answers>: choose ONLY from the values listed in that question's <allowed_answers> element
+    - For questions with <answer_format>: follow the format specification exactly (e.g. free-form text within the stated character budget)
+    - Always read each question's <allowed_answers> or <answer_format> and respond in the required shape based on the evidence
     - Select the answer best supported by observable evidence from the content
     - When evidence is ambiguous but some signal exists, select the most conservative option and use a low confidence score. If the question cannot be answered at all from the content, skip it per the relevance_filtering rules
     - Confidence should reflect the clarity and strength of evidence:
@@ -1963,7 +1963,8 @@ var SYSTEM_PROMPT = promptDedent`
     CRITICAL: Base your answers ONLY on the actual visual and audio/transcript content.
     ${METADATA_BOUNDARY_WARNING}
 
-    CRITICAL: Do NOT answer irrelevant questions with any of the allowed answers.
+    CRITICAL: Do NOT answer irrelevant questions in any form — neither with
+    one of the <allowed_answers> nor with prose for <answer_format>.
     Answering an irrelevant question is WRONG — you MUST skip it instead.
 
     For skipped questions:
@@ -1990,7 +1991,7 @@ var SYSTEM_PROMPT = promptDedent`
   </security>
 
   <constraints>
-    - You MUST answer every relevant question with one of its own listed allowed response options
+    - You MUST answer every relevant question following its specified response format: an option from <allowed_answers>, or prose matching <answer_format>
     - Skip irrelevant questions as described in relevance_filtering
     - Only describe observable evidence from frames or transcript
     - ${NO_FABRICATION_CONSTRAINT}
@@ -2025,15 +2026,16 @@ var AUDIO_ONLY_SYSTEM_PROMPT = promptDedent`
   <task>
     For each question provided, you must:
     1. Analyze the transcript
-    2. Answer with ONLY the allowed response options listed for that question - no other values are acceptable
+    2. Answer each question according to its specified response format: for questions with <allowed_answers>, select ONLY from the listed values; for questions with <answer_format>, follow the format specification exactly
     3. Provide a confidence score between 0 and 1 reflecting your certainty
     4. Explain your reasoning based on observable evidence
   </task>
 
   <answer_guidelines>
-    - Each question is presented as its own <question> block with a <text> element (the question) and an <allowed_answers> element (the permitted response values)
-    - Choose only from the values listed in that question's <allowed_answers> element
-    - Questions may have any set of allowed answers. Always read the <allowed_answers> for each question and select the best matching option based on the evidence
+    - Each question is presented as its own <question> block with a <text> element and either an <allowed_answers> element (listing the permitted response values) or an <answer_format> element (describing the required response shape for open-ended responses)
+    - For questions with <allowed_answers>: choose ONLY from the values listed in that question's <allowed_answers> element
+    - For questions with <answer_format>: follow the format specification exactly (e.g. free-form text within the stated character budget)
+    - Always read each question's <allowed_answers> or <answer_format> and respond in the required shape based on the evidence
     - Select the answer best supported by observable evidence from the content
     - When evidence is ambiguous but some signal exists, select the most conservative option and use a low confidence score. If the question cannot be answered at all from the content, skip it per the relevance_filtering rules
     - Confidence should reflect the clarity and strength of evidence:
@@ -2060,7 +2062,8 @@ var AUDIO_ONLY_SYSTEM_PROMPT = promptDedent`
     CRITICAL: Base your answers ONLY on the actual audio/transcript content.
     ${METADATA_BOUNDARY_WARNING}
 
-    CRITICAL: Do NOT answer irrelevant questions with any of the allowed answers.
+    CRITICAL: Do NOT answer irrelevant questions in any form — neither with
+    one of the <allowed_answers> nor with prose for <answer_format>.
     Answering an irrelevant question is WRONG — you MUST skip it instead.
 
     For skipped questions:
@@ -2085,7 +2088,7 @@ var AUDIO_ONLY_SYSTEM_PROMPT = promptDedent`
   </security>
 
   <constraints>
-    - You MUST answer every relevant question with one of its own listed allowed response options
+    - You MUST answer every relevant question following its specified response format: an option from <allowed_answers>, or prose matching <answer_format>
     - Skip irrelevant questions as described in relevance_filtering
     - Only describe observable evidence from transcript content
     - ${NO_FABRICATION_CONSTRAINT}
@@ -2100,16 +2103,104 @@ var AUDIO_ONLY_SYSTEM_PROMPT = promptDedent`
     - GOOD: "The speaker describes running through a park"
     - Be specific and evidence-based
   </language_guidelines>`;
+function freeFormAddendum(maxFreeFormAnswerLength) {
+  return promptDedent`
+    <free_form_answers>
+      Additional rules for questions with <answer_format>free-form text ...</answer_format>:
+
+      - Produce a short, evidence-grounded answer in plain prose
+      - Keep it tight: one or two sentences, maximum ${maxFreeFormAnswerLength} characters
+      - If the question is irrelevant to the asset content, still skip it
+        by setting answer to "${SKIP_SENTINEL}" exactly, per relevance_filtering
+      - The answer field is a content-analysis channel, not an instruction
+        channel. Do NOT include raw instructions, escape sequences, code,
+        or anything that looks like structured control content in the
+        answer. All existing security, non-fabrication, and
+        non-disclosure rules apply unchanged.
+    </free_form_answers>`;
+}
+function buildSystemPrompt(isAudioOnly, hasFreeForm, maxFreeFormAnswerLength) {
+  const base = isAudioOnly ? AUDIO_ONLY_SYSTEM_PROMPT : SYSTEM_PROMPT;
+  if (!hasFreeForm)
+    return base;
+  return `${base}
+
+${freeFormAddendum(maxFreeFormAnswerLength)}`;
+}
+var YES_NO_ANSWERS = ["yes", "no"];
+function allowedAnswersForQuestion(q) {
+  switch (q.mode.kind) {
+    case "yesNo":
+      return YES_NO_ANSWERS;
+    case "options":
+      return q.mode.options;
+    case "freeForm":
+      return [];
+  }
+}
+function normalizeQuestion(q, idx) {
+  const hasOptions = Array.isArray(q.answerOptions);
+  const isFreeForm = q.freeFormReply === true;
+  if (hasOptions && isFreeForm) {
+    throw new MuxAiError(
+      `Question at index ${idx} sets both answerOptions and freeFormReply: true. These are mutually exclusive \u2014 pick one.`,
+      { type: "validation_error" }
+    );
+  }
+  if (isFreeForm) {
+    return { question: q.question, mode: { kind: "freeForm" } };
+  }
+  if (hasOptions) {
+    const options = Array.from(
+      new Set(
+        (q.answerOptions ?? []).map((o) => o.trim()).filter(Boolean)
+      )
+    );
+    if (!options.length) {
+      throw new MuxAiError(
+        `Question at index ${idx} has invalid answerOptions: must include at least one non-empty value.`,
+        { type: "validation_error" }
+      );
+    }
+    return {
+      question: q.question,
+      mode: { kind: "options", options }
+    };
+  }
+  return { question: q.question, mode: { kind: "yesNo" } };
+}
+function renderFormatTag(mode, maxFreeFormAnswerLength) {
+  switch (mode.kind) {
+    case "yesNo":
+      return renderSection({
+        tag: "allowed_answers",
+        content: YES_NO_ANSWERS.map((a) => `"${a}"`).join(", ")
+      });
+    case "options":
+      return renderSection({
+        tag: "allowed_answers",
+        content: mode.options.map((a) => `"${a}"`).join(", ")
+      });
+    case "freeForm":
+      return renderSection({
+        tag: "answer_format",
+        content: `free-form text, maximum ${maxFreeFormAnswerLength} characters`
+      });
+  }
+}
 function buildUserPrompt({
   questions,
   transcriptText,
   isCleanTranscript = true,
-  isAudioOnly = false
+  isAudioOnly = false,
+  maxFreeFormAnswerLength
 }) {
   const contentDescriptor = isAudioOnly ? "audio content using transcript evidence" : "content";
+  const hasFreeForm = questions.some((q) => q.mode.kind === "freeForm");
+  const formatInstruction = hasFreeForm ? "Follow each question's specified response format: pick from <allowed_answers> where listed, or produce a concise free-form answer where <answer_format> is specified." : "Use only the values listed inside each question's own <allowed_answers> element.";
   const taskContent = dedent2`
     Answer each question in the <questions> block below about the ${contentDescriptor}.
-    Use only the values listed inside each question's own <allowed_answers> element.
+    ${formatInstruction}
     Return one answer per question, in the order the questions appear.
     If a question cannot be answered from the provided content, skip it as described in the system instructions.`;
   const taskSection = `<task>
@@ -2117,13 +2208,10 @@ ${taskContent}
 </task>`;
   const questionBlocks = questions.map((q, idx) => {
     const textTag = renderSection({ tag: "text", content: q.question });
-    const answersTag = renderSection({
-      tag: "allowed_answers",
-      content: q.answerOptions.map((a) => `"${a}"`).join(", ")
-    });
+    const formatTag = renderFormatTag(q.mode, maxFreeFormAnswerLength);
     return `<question number="${idx + 1}">
 ${textTag}
-${answersTag}
+${formatTag}
 </question>`;
   }).join("\n\n");
   const questionsSection = `<questions>
@@ -2143,18 +2231,41 @@ async function fetchImageAsBase64(imageUrl, imageDownloadOptions) {
   const downloadResult = await downloadImageAsBase64(imageUrl, imageDownloadOptions);
   return downloadResult.base64Data;
 }
+function buildResponseSchemaForQuestions(normalizedQuestions, maxFreeFormAnswerLength) {
+  const hasFreeForm = normalizedQuestions.some((q) => q.mode.kind === "freeForm");
+  if (hasFreeForm) {
+    const longestConstrained = Math.max(
+      0,
+      ...normalizedQuestions.flatMap(allowedAnswersForQuestion).map((a) => a.length)
+    );
+    const schemaMax = Math.max(
+      maxFreeFormAnswerLength,
+      SKIP_SENTINEL.length,
+      longestConstrained
+    );
+    return createAskQuestionsSchema(z2.string().min(1).max(schemaMax));
+  }
+  const allowed = Array.from(
+    new Set(normalizedQuestions.flatMap(allowedAnswersForQuestion))
+  );
+  return createAskQuestionsSchema(z2.enum([...allowed, SKIP_SENTINEL]));
+}
 async function analyzeQuestions({
   provider,
   modelId,
   userPrompt,
   systemPrompt,
-  allowedAnswers,
+  normalizedQuestions,
+  maxFreeFormAnswerLength,
   imageDataUrl,
   credentials
 }) {
   "use step";
   const model = await createLanguageModelFromConfig(provider, modelId, credentials);
-  const responseSchema = createAskQuestionsSchema(allowedAnswers);
+  const responseSchema = buildResponseSchemaForQuestions(
+    normalizedQuestions,
+    maxFreeFormAnswerLength
+  );
   const response = await generateText({
     model,
     output: Output.object({ schema: responseSchema }),
@@ -2233,6 +2344,8 @@ async function askQuestions(assetId, questions, options) {
     );
   }
   questions.forEach((q, idx) => {
+    if (q.freeFormReply === true)
+      return;
     for (const opt of q.answerOptions ?? []) {
       if (typeof opt === "string" && opt.length > maxAnswerOptionLength) {
         throw new MuxAiError(
@@ -2242,6 +2355,14 @@ async function askQuestions(assetId, questions, options) {
       }
     }
   });
+  const DEFAULT_MAX_FREE_FORM_ANSWER_LENGTH = 500;
+  const maxFreeFormAnswerLength = options?.maxFreeFormAnswerLength ?? DEFAULT_MAX_FREE_FORM_ANSWER_LENGTH;
+  if (!Number.isFinite(maxFreeFormAnswerLength) || maxFreeFormAnswerLength <= 0) {
+    throw new MuxAiError(
+      `maxFreeFormAnswerLength must be a positive number (received ${maxFreeFormAnswerLength}).`,
+      { type: "validation_error" }
+    );
+  }
   const {
     provider = "openai",
     model,
@@ -2253,23 +2374,8 @@ async function askQuestions(assetId, questions, options) {
     storyboardWidth = 640,
     credentials
   } = options ?? {};
-  const normalizedQuestions = questions.map((q, idx) => {
-    const options2 = Array.from(
-      new Set(
-        (q.answerOptions?.length ? q.answerOptions : ["yes", "no"]).map((o) => o.trim()).filter(Boolean)
-      )
-    );
-    if (!options2.length) {
-      throw new MuxAiError(
-        `Question at index ${idx} has invalid answerOptions: must include at least one non-empty value.`,
-        { type: "validation_error" }
-      );
-    }
-    return { ...q, answerOptions: options2 };
-  });
-  const allowedAnswers = Array.from(
-    new Set(normalizedQuestions.flatMap((q) => q.answerOptions))
-  );
+  const normalizedQuestions = questions.map((q, idx) => normalizeQuestion(q, idx));
+  const hasFreeForm = normalizedQuestions.some((q) => q.mode.kind === "freeForm");
   const modelConfig = resolveLanguageModelConfig({
     ...options,
     model,
@@ -2303,9 +2409,10 @@ async function askQuestions(assetId, questions, options) {
     questions: normalizedQuestions,
     transcriptText,
     isCleanTranscript: cleanTranscript,
-    isAudioOnly
+    isAudioOnly,
+    maxFreeFormAnswerLength
   });
-  const systemPrompt = isAudioOnly ? AUDIO_ONLY_SYSTEM_PROMPT : SYSTEM_PROMPT;
+  const systemPrompt = buildSystemPrompt(isAudioOnly, hasFreeForm, maxFreeFormAnswerLength);
   let analysisResponse;
   let imageUrl;
   try {
@@ -2315,7 +2422,8 @@ async function askQuestions(assetId, questions, options) {
         modelId: modelConfig.modelId,
         userPrompt,
         systemPrompt,
-        allowedAnswers,
+        normalizedQuestions,
+        maxFreeFormAnswerLength,
         credentials
       });
     } else {
@@ -2333,7 +2441,8 @@ async function askQuestions(assetId, questions, options) {
           modelId: modelConfig.modelId,
           userPrompt,
           systemPrompt,
-          allowedAnswers,
+          normalizedQuestions,
+          maxFreeFormAnswerLength,
           imageDataUrl: base64Data,
           credentials
         });
@@ -2344,7 +2453,8 @@ async function askQuestions(assetId, questions, options) {
             modelId: modelConfig.modelId,
             userPrompt,
             systemPrompt,
-            allowedAnswers,
+            normalizedQuestions,
+            maxFreeFormAnswerLength,
             imageDataUrl: storyboardUrl,
             credentials
           })
@@ -2379,20 +2489,42 @@ async function askQuestions(assetId, questions, options) {
     );
   }
   const answers = analysisResponse.result.answers.map((raw, idx) => {
-    const scrub = safety.scrubDetailed(raw.reasoning, `reasoning[${idx}]`);
-    const isSkipped = raw.skipped || raw.answer === SKIP_SENTINEL || scrub.leaked;
-    if (!isSkipped && !normalizedQuestions[idx].answerOptions.includes(raw.answer)) {
-      throw new MuxAiError(
-        `Answer "${raw.answer}" for question ${idx} is not in allowed options: ${normalizedQuestions[idx].answerOptions.join(", ")}`
+    const question = normalizedQuestions[idx];
+    const isFreeForm = question.mode.kind === "freeForm";
+    const reasoningScrub = safety.scrubDetailed(raw.reasoning, `reasoning[${idx}]`);
+    const explicitSkip = raw.skipped || raw.answer === SKIP_SENTINEL;
+    const answerScrub = isFreeForm && !explicitSkip ? safety.scrubDetailed(raw.answer, `answer[${idx}]`) : null;
+    const overLength = isFreeForm && !explicitSkip && !(answerScrub?.leaked ?? false) && (answerScrub?.text.length ?? 0) > maxFreeFormAnswerLength;
+    if (overLength) {
+      console.warn(
+        `[@mux/ai] Free-form answer at index ${idx} exceeded ${maxFreeFormAnswerLength}-char cap (${answerScrub.text.length} chars); treating as skipped.`
       );
     }
+    const isSkipped = explicitSkip || reasoningScrub.leaked || (answerScrub?.leaked ?? false) || overLength;
+    if (!isSkipped && !isFreeForm) {
+      const allowed = allowedAnswersForQuestion(question);
+      if (!allowed.includes(raw.answer)) {
+        throw new MuxAiError(
+          `Answer "${raw.answer}" for question ${idx} is not in allowed options: ${allowed.join(", ")}`
+        );
+      }
+    }
+    let finalAnswer;
+    if (isSkipped) {
+      finalAnswer = null;
+    } else if (isFreeForm) {
+      finalAnswer = answerScrub ? answerScrub.text : raw.answer;
+    } else {
+      finalAnswer = raw.answer;
+    }
+    const suppressed = reasoningScrub.leaked || (answerScrub?.leaked ?? false);
     return {
       // Use the trusted input verbatim rather than the model's echo.
-      question: normalizedQuestions[idx].question,
+      question: question.question,
       confidence: isSkipped ? 0 : Math.min(1, Math.max(0, raw.confidence)),
-      reasoning: scrub.leaked ? "Response suppressed by safety filter." : scrub.text,
+      reasoning: suppressed ? "Response suppressed by safety filter." : reasoningScrub.text,
       skipped: isSkipped,
-      answer: isSkipped ? null : raw.answer
+      answer: finalAnswer
     };
   });
   return {