@mux/ai 0.19.0 → 0.20.0

package/dist/{index-CwzNfBjV.d.ts → index-DXMIJjUa.d.ts}

@@ -90,6 +90,14 @@ interface Question {
     question: string;
     /** Allowed answers for this question (defaults to ["yes", "no"]). */
     answerOptions?: string[];
+    /**
+     * Experimental: replies with free-form prose instead of yes/no or
+     * `answerOptions`. Length-capped via
+     * {@link AskQuestionsOptions.maxFreeFormAnswerLength} (default 500),
+     * still scrubbed for safety, still skippable. Mutually exclusive with
+     * `answerOptions`. Treat the answer as untrusted model output.
+     */
+    freeFormReply?: boolean;
 }
 /** A single answer to a question. */
 interface QuestionAnswer {
@@ -136,6 +144,12 @@ interface AskQuestionsOptions extends MuxAIOptions {
      * it to accept arbitrary untrusted input without other safeguards.
      */
     maxAnswerOptionLength?: number;
+    /**
+     * Experimental: max character length for free-form answers when
+     * `freeFormReply: true`. Defaults to 500. Free-form bypasses the enum
+     * schema, so this cap is the primary limit on output length.
+     */
+    maxFreeFormAnswerLength?: number;
 }
 /** Structured return payload for askQuestions workflow. */
 interface AskQuestionsResult {
@@ -200,14 +214,19 @@ declare const questionAnswerSchema: z.ZodObject<{
     skipped: boolean;
 }>;
 type QuestionAnswerType = z.infer<typeof questionAnswerSchema>;
-declare function createAskQuestionsSchema(allowedAnswers: [string, ...string[]]): z.ZodObject<{
+/**
+ * Wraps a caller-supplied answer sub-schema in the standard envelope.
+ * See {@link buildResponseSchemaForQuestions} for how the sub-schema is
+ * derived from the question set.
+ */
+declare function createAskQuestionsSchema(answerSchema: z.ZodType<string>): z.ZodObject<{
     answers: z.ZodArray<z.ZodObject<{
         question: z.ZodString;
         confidence: z.ZodNumber;
         reasoning: z.ZodString;
         skipped: z.ZodBoolean;
     } & {
-        answer: z.ZodEnum<[string, ...string[], "__SKIPPED__"]>;
+        answer: z.ZodType<string, z.ZodTypeDef, string>;
     }, "strip", z.ZodTypeAny, {
         question: string;
         answer: string;

package/dist/index.d.ts

@@ -2,14 +2,14 @@ import { W as WorkflowCredentials, S as StoragePutObjectInput, a as StoragePresi
 export { A as AssetTextTrack, C as ChunkEmbedding, b as ChunkingStrategy, E as Encrypted, c as EncryptedPayload, I as ImageSubmissionMode, M as MuxAIOptions, d as MuxAsset, P as PlaybackAsset, e as PlaybackPolicy, f as StorageAdapter, T as TextChunk, g as TokenChunkingConfig, h as TokenUsage, i as ToneType, U as UsageMetadata, V as VTTChunkingConfig, j as VideoEmbeddingsResult, k as WorkflowCredentialsInput, l as WorkflowMuxClient, m as decryptFromWorkflow, n as encryptForWorkflow } from './shots-BFYOfBgN.js';
 import { WORKFLOW_SERIALIZE, WORKFLOW_DESERIALIZE } from '@workflow/serde';
 export { i as primitives } from './index-Drj9Cy_k.js';
-export { i as workflows } from './index-CwzNfBjV.js';
+export { i as workflows } from './index-DXMIJjUa.js';
 import '@mux/mux-node';
 import 'zod';
 import '@ai-sdk/anthropic';
 import '@ai-sdk/google';
 import '@ai-sdk/openai';
 
-var version = "0.19.0";
+var version = "0.20.0";
 
 type MuxAiErrorType = "validation_error" | "processing_error" | "timeout_error";
 /**

package/dist/index.js

@@ -5,7 +5,7 @@ var __export = (target, all) => {
 };
 
 // package.json
-var version = "0.19.0";
+var version = "0.20.0";
 
 // src/lib/prompt-fragments.ts
 import dedent from "dedent";
@@ -2828,8 +2828,7 @@ var questionAnswerSchema = z2.object({
   skipped: z2.boolean()
 });
 var SKIP_SENTINEL = "__SKIPPED__";
-function createAskQuestionsSchema(allowedAnswers) {
-  const answerSchema = z2.enum([...allowedAnswers, SKIP_SENTINEL]);
+function createAskQuestionsSchema(answerSchema) {
   return z2.object({
     answers: z2.array(
       questionAnswerSchema.extend({
@@ -2865,15 +2864,16 @@ var SYSTEM_PROMPT = promptDedent`
   <task>
     For each question provided, you must:
     1. Analyze the storyboard frames and transcript (if provided)
-    2. Answer with ONLY the allowed response options listed for that question - no other values are acceptable
+    2. Answer each question according to its specified response format: for questions with <allowed_answers>, select ONLY from the listed values; for questions with <answer_format>, follow the format specification exactly
     3. Provide a confidence score between 0 and 1 reflecting your certainty
     4. Explain your reasoning based on observable evidence
   </task>
 
   <answer_guidelines>
-    - Each question is presented as its own <question> block with a <text> element (the question) and an <allowed_answers> element (the permitted response values)
-    - Choose only from the values listed in that question's <allowed_answers> element
-    - Questions may have any set of allowed answers. Always read the <allowed_answers> for each question and select the best matching option based on the evidence
+    - Each question is presented as its own <question> block with a <text> element and either an <allowed_answers> element (listing the permitted response values) or an <answer_format> element (describing the required response shape for open-ended responses)
+    - For questions with <allowed_answers>: choose ONLY from the values listed in that question's <allowed_answers> element
+    - For questions with <answer_format>: follow the format specification exactly (e.g. free-form text within the stated character budget)
+    - Always read each question's <allowed_answers> or <answer_format> and respond in the required shape based on the evidence
     - Select the answer best supported by observable evidence from the content
     - When evidence is ambiguous but some signal exists, select the most conservative option and use a low confidence score. If the question cannot be answered at all from the content, skip it per the relevance_filtering rules
     - Confidence should reflect the clarity and strength of evidence:
@@ -2900,7 +2900,8 @@ var SYSTEM_PROMPT = promptDedent`
     CRITICAL: Base your answers ONLY on the actual visual and audio/transcript content.
     ${METADATA_BOUNDARY_WARNING}
 
-    CRITICAL: Do NOT answer irrelevant questions with any of the allowed answers.
+    CRITICAL: Do NOT answer irrelevant questions in any form — neither with
+    one of the <allowed_answers> nor with prose for <answer_format>.
     Answering an irrelevant question is WRONG — you MUST skip it instead.
 
     For skipped questions:
@@ -2927,7 +2928,7 @@ var SYSTEM_PROMPT = promptDedent`
   </security>
 
   <constraints>
-    - You MUST answer every relevant question with one of its own listed allowed response options
+    - You MUST answer every relevant question following its specified response format: an option from <allowed_answers>, or prose matching <answer_format>
     - Skip irrelevant questions as described in relevance_filtering
     - Only describe observable evidence from frames or transcript
     - ${NO_FABRICATION_CONSTRAINT}
@@ -2962,15 +2963,16 @@ var AUDIO_ONLY_SYSTEM_PROMPT = promptDedent`
   <task>
     For each question provided, you must:
     1. Analyze the transcript
-    2. Answer with ONLY the allowed response options listed for that question - no other values are acceptable
+    2. Answer each question according to its specified response format: for questions with <allowed_answers>, select ONLY from the listed values; for questions with <answer_format>, follow the format specification exactly
     3. Provide a confidence score between 0 and 1 reflecting your certainty
     4. Explain your reasoning based on observable evidence
   </task>
 
   <answer_guidelines>
-    - Each question is presented as its own <question> block with a <text> element (the question) and an <allowed_answers> element (the permitted response values)
-    - Choose only from the values listed in that question's <allowed_answers> element
-    - Questions may have any set of allowed answers. Always read the <allowed_answers> for each question and select the best matching option based on the evidence
+    - Each question is presented as its own <question> block with a <text> element and either an <allowed_answers> element (listing the permitted response values) or an <answer_format> element (describing the required response shape for open-ended responses)
+    - For questions with <allowed_answers>: choose ONLY from the values listed in that question's <allowed_answers> element
+    - For questions with <answer_format>: follow the format specification exactly (e.g. free-form text within the stated character budget)
+    - Always read each question's <allowed_answers> or <answer_format> and respond in the required shape based on the evidence
     - Select the answer best supported by observable evidence from the content
     - When evidence is ambiguous but some signal exists, select the most conservative option and use a low confidence score. If the question cannot be answered at all from the content, skip it per the relevance_filtering rules
     - Confidence should reflect the clarity and strength of evidence:
@@ -2997,7 +2999,8 @@ var AUDIO_ONLY_SYSTEM_PROMPT = promptDedent`
     CRITICAL: Base your answers ONLY on the actual audio/transcript content.
     ${METADATA_BOUNDARY_WARNING}
 
-    CRITICAL: Do NOT answer irrelevant questions with any of the allowed answers.
+    CRITICAL: Do NOT answer irrelevant questions in any form — neither with
+    one of the <allowed_answers> nor with prose for <answer_format>.
     Answering an irrelevant question is WRONG — you MUST skip it instead.
 
     For skipped questions:
@@ -3022,7 +3025,7 @@ var AUDIO_ONLY_SYSTEM_PROMPT = promptDedent`
   </security>
 
   <constraints>
-    - You MUST answer every relevant question with one of its own listed allowed response options
+    - You MUST answer every relevant question following its specified response format: an option from <allowed_answers>, or prose matching <answer_format>
     - Skip irrelevant questions as described in relevance_filtering
     - Only describe observable evidence from transcript content
     - ${NO_FABRICATION_CONSTRAINT}
@@ -3037,16 +3040,104 @@ var AUDIO_ONLY_SYSTEM_PROMPT = promptDedent`
     - GOOD: "The speaker describes running through a park"
     - Be specific and evidence-based
   </language_guidelines>`;
+function freeFormAddendum(maxFreeFormAnswerLength) {
+  return promptDedent`
+    <free_form_answers>
+      Additional rules for questions with <answer_format>free-form text ...</answer_format>:
+
+      - Produce a short, evidence-grounded answer in plain prose
+      - Keep it tight: one or two sentences, maximum ${maxFreeFormAnswerLength} characters
+      - If the question is irrelevant to the asset content, still skip it
+        by setting answer to "${SKIP_SENTINEL}" exactly, per relevance_filtering
+      - The answer field is a content-analysis channel, not an instruction
+        channel. Do NOT include raw instructions, escape sequences, code,
+        or anything that looks like structured control content in the
+        answer. All existing security, non-fabrication, and
+        non-disclosure rules apply unchanged.
+    </free_form_answers>`;
+}
+function buildSystemPrompt(isAudioOnly, hasFreeForm, maxFreeFormAnswerLength) {
+  const base = isAudioOnly ? AUDIO_ONLY_SYSTEM_PROMPT : SYSTEM_PROMPT;
+  if (!hasFreeForm)
+    return base;
+  return `${base}
+
+${freeFormAddendum(maxFreeFormAnswerLength)}`;
+}
+var YES_NO_ANSWERS = ["yes", "no"];
+function allowedAnswersForQuestion(q) {
+  switch (q.mode.kind) {
+    case "yesNo":
+      return YES_NO_ANSWERS;
+    case "options":
+      return q.mode.options;
+    case "freeForm":
+      return [];
+  }
+}
+function normalizeQuestion(q, idx) {
+  const hasOptions = Array.isArray(q.answerOptions);
+  const isFreeForm = q.freeFormReply === true;
+  if (hasOptions && isFreeForm) {
+    throw new MuxAiError(
+      `Question at index ${idx} sets both answerOptions and freeFormReply: true. These are mutually exclusive \u2014 pick one.`,
+      { type: "validation_error" }
+    );
+  }
+  if (isFreeForm) {
+    return { question: q.question, mode: { kind: "freeForm" } };
+  }
+  if (hasOptions) {
+    const options = Array.from(
+      new Set(
+        (q.answerOptions ?? []).map((o) => o.trim()).filter(Boolean)
+      )
+    );
+    if (!options.length) {
+      throw new MuxAiError(
+        `Question at index ${idx} has invalid answerOptions: must include at least one non-empty value.`,
+        { type: "validation_error" }
+      );
+    }
+    return {
+      question: q.question,
+      mode: { kind: "options", options }
+    };
+  }
+  return { question: q.question, mode: { kind: "yesNo" } };
+}
+function renderFormatTag(mode, maxFreeFormAnswerLength) {
+  switch (mode.kind) {
+    case "yesNo":
+      return renderSection({
+        tag: "allowed_answers",
+        content: YES_NO_ANSWERS.map((a) => `"${a}"`).join(", ")
+      });
+    case "options":
+      return renderSection({
+        tag: "allowed_answers",
+        content: mode.options.map((a) => `"${a}"`).join(", ")
+      });
+    case "freeForm":
+      return renderSection({
+        tag: "answer_format",
+        content: `free-form text, maximum ${maxFreeFormAnswerLength} characters`
+      });
+  }
+}
 function buildUserPrompt({
   questions,
   transcriptText,
   isCleanTranscript = true,
-  isAudioOnly = false
+  isAudioOnly = false,
+  maxFreeFormAnswerLength
 }) {
   const contentDescriptor = isAudioOnly ? "audio content using transcript evidence" : "content";
+  const hasFreeForm = questions.some((q) => q.mode.kind === "freeForm");
+  const formatInstruction = hasFreeForm ? "Follow each question's specified response format: pick from <allowed_answers> where listed, or produce a concise free-form answer where <answer_format> is specified." : "Use only the values listed inside each question's own <allowed_answers> element.";
   const taskContent = dedent2`
     Answer each question in the <questions> block below about the ${contentDescriptor}.
-    Use only the values listed inside each question's own <allowed_answers> element.
+    ${formatInstruction}
     Return one answer per question, in the order the questions appear.
     If a question cannot be answered from the provided content, skip it as described in the system instructions.`;
   const taskSection = `<task>
@@ -3054,13 +3145,10 @@ ${taskContent}
 </task>`;
   const questionBlocks = questions.map((q, idx) => {
     const textTag = renderSection({ tag: "text", content: q.question });
-    const answersTag = renderSection({
-      tag: "allowed_answers",
-      content: q.answerOptions.map((a) => `"${a}"`).join(", ")
-    });
+    const formatTag = renderFormatTag(q.mode, maxFreeFormAnswerLength);
     return `<question number="${idx + 1}">
 ${textTag}
-${answersTag}
+${formatTag}
 </question>`;
   }).join("\n\n");
   const questionsSection = `<questions>
@@ -3080,18 +3168,41 @@ async function fetchImageAsBase64(imageUrl, imageDownloadOptions) {
   const downloadResult = await downloadImageAsBase64(imageUrl, imageDownloadOptions);
   return downloadResult.base64Data;
 }
+function buildResponseSchemaForQuestions(normalizedQuestions, maxFreeFormAnswerLength) {
+  const hasFreeForm = normalizedQuestions.some((q) => q.mode.kind === "freeForm");
+  if (hasFreeForm) {
+    const longestConstrained = Math.max(
+      0,
+      ...normalizedQuestions.flatMap(allowedAnswersForQuestion).map((a) => a.length)
+    );
+    const schemaMax = Math.max(
+      maxFreeFormAnswerLength,
+      SKIP_SENTINEL.length,
+      longestConstrained
+    );
+    return createAskQuestionsSchema(z2.string().min(1).max(schemaMax));
+  }
+  const allowed = Array.from(
+    new Set(normalizedQuestions.flatMap(allowedAnswersForQuestion))
+  );
+  return createAskQuestionsSchema(z2.enum([...allowed, SKIP_SENTINEL]));
+}
 async function analyzeQuestions({
   provider,
   modelId,
   userPrompt,
   systemPrompt,
-  allowedAnswers,
+  normalizedQuestions,
+  maxFreeFormAnswerLength,
   imageDataUrl,
   credentials
 }) {
   "use step";
   const model = await createLanguageModelFromConfig(provider, modelId, credentials);
-  const responseSchema = createAskQuestionsSchema(allowedAnswers);
+  const responseSchema = buildResponseSchemaForQuestions(
+    normalizedQuestions,
+    maxFreeFormAnswerLength
+  );
   const response = await generateText({
     model,
     output: Output.object({ schema: responseSchema }),
@@ -3170,6 +3281,8 @@ async function askQuestions(assetId, questions, options) {
     );
   }
   questions.forEach((q, idx) => {
+    if (q.freeFormReply === true)
+      return;
     for (const opt of q.answerOptions ?? []) {
       if (typeof opt === "string" && opt.length > maxAnswerOptionLength) {
         throw new MuxAiError(
@@ -3179,6 +3292,14 @@ async function askQuestions(assetId, questions, options) {
       }
     }
   });
+  const DEFAULT_MAX_FREE_FORM_ANSWER_LENGTH = 500;
+  const maxFreeFormAnswerLength = options?.maxFreeFormAnswerLength ?? DEFAULT_MAX_FREE_FORM_ANSWER_LENGTH;
+  if (!Number.isFinite(maxFreeFormAnswerLength) || maxFreeFormAnswerLength <= 0) {
+    throw new MuxAiError(
+      `maxFreeFormAnswerLength must be a positive number (received ${maxFreeFormAnswerLength}).`,
+      { type: "validation_error" }
+    );
+  }
   const {
     provider = "openai",
     model,
@@ -3190,23 +3311,8 @@ async function askQuestions(assetId, questions, options) {
     storyboardWidth = 640,
     credentials
   } = options ?? {};
-  const normalizedQuestions = questions.map((q, idx) => {
-    const options2 = Array.from(
-      new Set(
-        (q.answerOptions?.length ? q.answerOptions : ["yes", "no"]).map((o) => o.trim()).filter(Boolean)
-      )
-    );
-    if (!options2.length) {
-      throw new MuxAiError(
-        `Question at index ${idx} has invalid answerOptions: must include at least one non-empty value.`,
-        { type: "validation_error" }
-      );
-    }
-    return { ...q, answerOptions: options2 };
-  });
-  const allowedAnswers = Array.from(
-    new Set(normalizedQuestions.flatMap((q) => q.answerOptions))
-  );
+  const normalizedQuestions = questions.map((q, idx) => normalizeQuestion(q, idx));
+  const hasFreeForm = normalizedQuestions.some((q) => q.mode.kind === "freeForm");
   const modelConfig = resolveLanguageModelConfig({
     ...options,
     model,
@@ -3240,9 +3346,10 @@ async function askQuestions(assetId, questions, options) {
     questions: normalizedQuestions,
     transcriptText,
     isCleanTranscript: cleanTranscript,
-    isAudioOnly
+    isAudioOnly,
+    maxFreeFormAnswerLength
   });
-  const systemPrompt = isAudioOnly ? AUDIO_ONLY_SYSTEM_PROMPT : SYSTEM_PROMPT;
+  const systemPrompt = buildSystemPrompt(isAudioOnly, hasFreeForm, maxFreeFormAnswerLength);
   let analysisResponse;
   let imageUrl;
   try {
@@ -3252,7 +3359,8 @@ async function askQuestions(assetId, questions, options) {
         modelId: modelConfig.modelId,
         userPrompt,
         systemPrompt,
-        allowedAnswers,
+        normalizedQuestions,
+        maxFreeFormAnswerLength,
         credentials
       });
     } else {
@@ -3270,7 +3378,8 @@ async function askQuestions(assetId, questions, options) {
           modelId: modelConfig.modelId,
           userPrompt,
           systemPrompt,
-          allowedAnswers,
+          normalizedQuestions,
+          maxFreeFormAnswerLength,
           imageDataUrl: base64Data,
           credentials
         });
@@ -3281,7 +3390,8 @@ async function askQuestions(assetId, questions, options) {
             modelId: modelConfig.modelId,
             userPrompt,
             systemPrompt,
-            allowedAnswers,
+            normalizedQuestions,
+            maxFreeFormAnswerLength,
             imageDataUrl: storyboardUrl,
             credentials
           })
@@ -3316,20 +3426,42 @@ async function askQuestions(assetId, questions, options) {
     );
   }
   const answers = analysisResponse.result.answers.map((raw, idx) => {
-    const scrub = safety.scrubDetailed(raw.reasoning, `reasoning[${idx}]`);
-    const isSkipped = raw.skipped || raw.answer === SKIP_SENTINEL || scrub.leaked;
-    if (!isSkipped && !normalizedQuestions[idx].answerOptions.includes(raw.answer)) {
-      throw new MuxAiError(
-        `Answer "${raw.answer}" for question ${idx} is not in allowed options: ${normalizedQuestions[idx].answerOptions.join(", ")}`
+    const question = normalizedQuestions[idx];
+    const isFreeForm = question.mode.kind === "freeForm";
+    const reasoningScrub = safety.scrubDetailed(raw.reasoning, `reasoning[${idx}]`);
+    const explicitSkip = raw.skipped || raw.answer === SKIP_SENTINEL;
+    const answerScrub = isFreeForm && !explicitSkip ? safety.scrubDetailed(raw.answer, `answer[${idx}]`) : null;
+    const overLength = isFreeForm && !explicitSkip && !(answerScrub?.leaked ?? false) && (answerScrub?.text.length ?? 0) > maxFreeFormAnswerLength;
+    if (overLength) {
+      console.warn(
+        `[@mux/ai] Free-form answer at index ${idx} exceeded ${maxFreeFormAnswerLength}-char cap (${answerScrub.text.length} chars); treating as skipped.`
       );
     }
+    const isSkipped = explicitSkip || reasoningScrub.leaked || (answerScrub?.leaked ?? false) || overLength;
+    if (!isSkipped && !isFreeForm) {
+      const allowed = allowedAnswersForQuestion(question);
+      if (!allowed.includes(raw.answer)) {
+        throw new MuxAiError(
+          `Answer "${raw.answer}" for question ${idx} is not in allowed options: ${allowed.join(", ")}`
+        );
+      }
+    }
+    let finalAnswer;
+    if (isSkipped) {
+      finalAnswer = null;
+    } else if (isFreeForm) {
+      finalAnswer = answerScrub ? answerScrub.text : raw.answer;
+    } else {
+      finalAnswer = raw.answer;
+    }
+    const suppressed = reasoningScrub.leaked || (answerScrub?.leaked ?? false);
     return {
       // Use the trusted input verbatim rather than the model's echo.
-      question: normalizedQuestions[idx].question,
+      question: question.question,
       confidence: isSkipped ? 0 : Math.min(1, Math.max(0, raw.confidence)),
-      reasoning: scrub.leaked ? "Response suppressed by safety filter." : scrub.text,
+      reasoning: suppressed ? "Response suppressed by safety filter." : reasoningScrub.text,
       skipped: isSkipped,
-      answer: isSkipped ? null : raw.answer
+      answer: finalAnswer
     };
   });
   return {