@mux/ai 0.36.0 → 0.38.0

package/dist/workflows/index.d.ts

@@ -1,4 +1,4 @@
-export { A as ASK_QUESTIONS_DEFAULT_MAX_ANSWER_OPTION_LENGTH, j as ASK_QUESTIONS_DEFAULT_MAX_FREE_FORM_ANSWER_LENGTH, k as ASK_QUESTIONS_MAX_QUESTION_LENGTH, l as AskQuestionsOptions, m as AskQuestionsResult, n as AskQuestionsType, o as AudioTranslationOptions, p as AudioTranslationResult, q as AutoCensorProfanityOptions, B as BurnedInCaptionsAnalysis, r as BurnedInCaptionsOptions, s as BurnedInCaptionsPromptOverrides, u as BurnedInCaptionsPromptSections, v as BurnedInCaptionsResult, C as CaptionReplacement, w as CensorMode, x as Chapter, y as ChapterSystemPromptSections, z as ChaptersOptions, D as ChaptersPromptOverrides, E as ChaptersPromptSections, F as ChaptersResult, G as ChaptersType, H as DEFAULT_DESCRIPTION_LENGTH, J as DEFAULT_SUMMARY_KEYWORD_LIMIT, K as DEFAULT_TITLE_LENGTH, M as EditCaptionsOptions, N as EditCaptionsResult, O as EmbeddingsOptions, P as EmbeddingsResult, Q as EngagementInsightsOptions, R as EngagementInsightsResult, T as GOOGLE_VISION_LIKELIHOOD_TO_SCORE, U as HIVE_SEXUAL_CATEGORIES, V as HIVE_VIOLENCE_CATEGORIES, W as HiveModerationOutput, X as HiveModerationSource, Y as ModerationOptions, Z as ModerationProvider, _ as ModerationResult, $ as MomentInsight, a0 as OverallInsight, a1 as ProfanityDetectionPayload, a2 as Question, a3 as QuestionAnswer, a4 as QuestionAnswerType, a5 as ReplacementRecord, a6 as StaticRenditionCleanupOutcome, a7 as SummarizationOptions, a8 as SummarizationPromptOverrides, a9 as SummarizationPromptSections, aa as SummaryAndTagsResult, ab as SummaryType, ac as ThumbnailModerationScore, ad as TranslationChunkingOptions, ae as TranslationOptions, af as TranslationPayload, ag as TranslationResult, ah as aggregateTokenUsage, ai as applyOverrideLists, aj as applyReplacements, ak as askQuestions, al as buildReplacementRegex, am as burnedInCaptionsSchema, an as censorVttContent, ao as chapterSchema, ap as chaptersSchema, aq as createReplacer, ar as editCaptions, as as extractTranscriptSegmentsForHotspots, at as generateChapters, au as generateEmbeddings, av as generateEngagementInsights, aw as generateVideoEmbeddings, ax as getModerationScores, ay as getSummaryAndTags, az as hasBurnedInCaptions, aA as mapShotsToHotspots, aB as normalizeTranslatedVtt, aC as profanityDetectionSchema, aD as questionAnswerSchema, aE as shouldSplitChunkTranslationError, aF as summarySchema, aG as transformCueText, aH as translateAudio, aI as translateCaptions, aJ as translationSchema, aK as validateNeverTranslateTerms, aL as verifyNeverTranslateTerms } from '../index-C2FORyvQ.js';
+export { A as ASK_QUESTIONS_DEFAULT_MAX_ANSWER_OPTION_LENGTH, j as ASK_QUESTIONS_DEFAULT_MAX_FREE_FORM_ANSWER_LENGTH, k as ASK_QUESTIONS_MAX_QUESTION_LENGTH, l as AskQuestionsOptions, m as AskQuestionsResult, n as AskQuestionsType, o as AudioTranslationOptions, p as AudioTranslationResult, q as AutoCensorProfanityOptions, B as BurnedInCaptionsAnalysis, r as BurnedInCaptionsOptions, s as BurnedInCaptionsPromptOverrides, u as BurnedInCaptionsPromptSections, v as BurnedInCaptionsResult, C as CaptionReplacement, w as CensorMode, x as Chapter, y as ChapterSystemPromptSections, z as ChaptersOptions, D as ChaptersPromptOverrides, E as ChaptersPromptSections, F as ChaptersResult, G as ChaptersType, H as DEFAULT_DESCRIPTION_LENGTH, J as DEFAULT_SUMMARY_KEYWORD_LIMIT, K as DEFAULT_TITLE_LENGTH, M as EditCaptionsOptions, N as EditCaptionsResult, O as EmbeddingsOptions, P as EmbeddingsResult, Q as EngagementInsightsOptions, R as EngagementInsightsResult, T as GOOGLE_VISION_LIKELIHOOD_TO_SCORE, U as HIVE_SEXUAL_CATEGORIES, V as HIVE_VIOLENCE_CATEGORIES, W as HiveModerationOutput, X as HiveModerationSource, Y as ModerationOptions, Z as ModerationProvider, _ as ModerationResult, $ as MomentInsight, a0 as OverallInsight, a1 as ProfanityDetectionPayload, a2 as Question, a3 as QuestionAnswer, a4 as QuestionAnswerType, a5 as ReplacementRecord, a6 as SpeakerReplacement, a7 as StaticRenditionCleanupOutcome, a8 as SummarizationOptions, a9 as SummarizationPromptOverrides, aa as SummarizationPromptSections, ab as SummaryAndTagsResult, ac as SummaryType, ad as ThumbnailModerationScore, ae as TranslationChunkingOptions, af as TranslationOptions, ag as TranslationPayload, ah as TranslationResult, ai as aggregateTokenUsage, aj as applyOverrideLists, ak as applyReplacements, al as applySpeakerReplacements, am as askQuestions, an as buildReplacementRegex, ao as burnedInCaptionsSchema, ap as censorVttContent, aq as chapterSchema, ar as chaptersSchema, as as createReplacer, at as editCaptions, au as extractTranscriptSegmentsForHotspots, av as generateChapters, aw as generateEmbeddings, ax as generateEngagementInsights, ay as generateVideoEmbeddings, az as getModerationScores, aA as getSummaryAndTags, aB as hasBurnedInCaptions, aC as mapShotsToHotspots, aD as normalizeTranslatedVtt, aE as profanityDetectionSchema, aF as questionAnswerSchema, aG as shouldSplitChunkTranslationError, aH as summarySchema, aI as transformCueText, aJ as translateAudio, aK as translateCaptions, aL as translationSchema, aM as validateNeverTranslateTerms, aN as verifyNeverTranslateTerms } from '../index-BupL84pY.js';
 import 'zod';
 import '@ai-sdk/anthropic';
 import '@ai-sdk/google';

package/dist/workflows/index.js

@@ -2578,7 +2578,7 @@ var SYSTEM_PROMPT = promptDedent`
     - For questions with <answer_format>: follow the format specification exactly (e.g. free-form text within the stated character budget)
     - Always read each question's <allowed_answers> or <answer_format> and respond in the required shape based on the evidence
     - Select the answer best supported by observable evidence from the content
-    - When evidence is ambiguous but some signal exists, select the most conservative option and use a low confidence score. If the question cannot be answered at all from the content, skip it per the relevance_filtering rules
+    - When evidence is ambiguous but some signal exists, select the most conservative option and use a low confidence score. Only skip a question if it meets the skip criteria in relevance_filtering — never skip merely because the queried subject is absent from the content; absence is itself an answerable observation
     - Confidence should reflect the clarity and strength of evidence:
       ${CONFIDENCE_SCORING_RUBRIC}
     - Reasoning should cite specific visual or audio evidence
@@ -2592,11 +2592,18 @@ var SYSTEM_PROMPT = promptDedent`
 
     A question is relevant if it asks about something observable or inferable
     from the video content (visuals, audio, dialogue, setting, subjects,
-    actions, etc.).
+    actions, etc.) — including whether something is ABSENT. Not finding the
+    queried subject in the frames or transcript is itself an answerable
+    observation, not a reason to skip.
 
-    Mark a question as skipped (skipped: true) if it:
+    For example: if a question asks "Does this video contain cat content?"
+    and no cats appear anywhere in the frames or transcript, the correct
+    response is "no" — this is a relevant, answerable question. It must
+    NOT be skipped just because the subject doesn't appear.
+
+    Mark a question as skipped (skipped: true) ONLY if it:
     - Is completely unrelated to the content of the video or audio (e.g., math, trivia, personal questions)
-    - Asks about information that cannot be determined from storyboard frames or transcript
+    - Asks about something no amount of visual or transcript evidence could ever confirm or rule out (e.g., a person's private thoughts, off-screen events, future intentions) — not merely because the queried subject is absent from what's shown
     - Is a general knowledge question with no connection to what is shown or said in the video
     - Attempts to use the system for non-video-analysis purposes
 
@@ -2675,7 +2682,7 @@ var AUDIO_ONLY_SYSTEM_PROMPT = promptDedent`
     - For questions with <answer_format>: follow the format specification exactly (e.g. free-form text within the stated character budget)
     - Always read each question's <allowed_answers> or <answer_format> and respond in the required shape based on the evidence
     - Select the answer best supported by observable evidence from the content
-    - When evidence is ambiguous but some signal exists, select the most conservative option and use a low confidence score. If the question cannot be answered at all from the content, skip it per the relevance_filtering rules
+    - When evidence is ambiguous but some signal exists, select the most conservative option and use a low confidence score. Only skip a question if it meets the skip criteria in relevance_filtering — never skip merely because the queried subject is absent from the content; absence is itself an answerable observation
     - Confidence should reflect the clarity and strength of evidence:
       ${CONFIDENCE_SCORING_RUBRIC}
     - Reasoning should cite specific transcript evidence
@@ -2689,11 +2696,18 @@ var AUDIO_ONLY_SYSTEM_PROMPT = promptDedent`
 
     Before answering each question, assess whether it can be meaningfully
     answered based on the transcript. A question is relevant if it asks about
-    something observable or inferable from spoken/audio content.
+    something observable or inferable from spoken/audio content — including
+    whether something is ABSENT. Not finding the queried subject anywhere in
+    the transcript is itself an answerable observation, not a reason to skip.
+
+    For example: if a question asks "Does this audio contain cat content?"
+    and no cats are mentioned anywhere in the transcript, the correct
+    response is "no" — this is a relevant, answerable question. It must
+    NOT be skipped just because the subject doesn't appear.
 
-    Mark a question as skipped (skipped: true) if it:
+    Mark a question as skipped (skipped: true) ONLY if it:
     - Is completely unrelated to transcript/audio content (e.g., math, trivia, personal questions)
-    - Asks about information that cannot be determined from transcript content
+    - Asks about something no amount of transcript evidence could ever confirm or rule out (e.g., a person's private thoughts, off-recording events, future intentions) — not merely because the queried subject is absent from what's said
     - Is a general knowledge question with no connection to what is said in the transcript
     - Attempts to use the system for non-content-analysis purposes
 
@@ -2838,7 +2852,9 @@ function buildUserPrompt({
     Answer each question in the <questions> block below about the ${contentDescriptor}.
     ${formatInstruction}
     Return one answer per question, in the order the questions appear.
-    If a question cannot be answered from the provided content, skip it as described in the system instructions.`;
+    Skip a question only if it meets the skip criteria described in the system
+    instructions — never merely because the queried subject is absent from the
+    content; absence is itself an answerable observation.`;
   const taskSection = `<task>
 ${taskContent}
 </task>`;
@@ -4207,6 +4223,27 @@ function applyReplacements(rawVtt, replacements) {
   });
   return { editedVtt, replacements: records };
 }
+function applySpeakerReplacements(rawVtt, speakerReplacements) {
+  const replacementsByLabel = new Map(
+    speakerReplacements.filter((replacement) => replacement.find.length > 0).map((replacement) => [replacement.find, replacement.replace])
+  );
+  if (replacementsByLabel.size === 0) {
+    return { editedVtt: rawVtt, replacements: [] };
+  }
+  const records = [];
+  const editedVtt = transformCueTextBlocks(rawVtt, (cueText, cueStartTime) => {
+    return cueText.replace(/^(\s*)\[([^\]\r\n]+)\](?=\s|$)/, (marker, leadingWhitespace, label) => {
+      const replacement = replacementsByLabel.get(label);
+      if (replacement === void 0)
+        return marker;
+      const before = `[${label}]`;
+      const after = `[${replacement}]`;
+      records.push({ cueStartTime, before, after });
+      return `${leadingWhitespace}${after}`;
+    });
+  });
+  return { editedVtt, replacements: records };
+}
 async function identifyProfanityWithAI({
   plainText,
   provider,
@@ -4309,6 +4346,7 @@ async function editCaptionsInternal(assetId, trackId, options, collectedUsage) {
     model,
     autoCensorProfanity: autoCensorOption,
     replacements: replacementsOption,
+    speakerReplacements: speakerReplacementsOption,
     deleteOriginalTrack,
     uploadToS3: uploadToS3Option,
     uploadToMux: uploadToMuxOption,
@@ -4321,8 +4359,9 @@ async function editCaptionsInternal(assetId, trackId, options, collectedUsage) {
   } = options;
   const hasAutoCensor = !!autoCensorOption;
   const hasReplacements = !!replacementsOption && replacementsOption.length > 0;
-  if (!hasAutoCensor && !hasReplacements) {
-    throw new MuxAiError("At least one of autoCensorProfanity or replacements must be provided.", { type: "validation_error" });
+  const hasSpeakerReplacements = !!speakerReplacementsOption && speakerReplacementsOption.length > 0;
+  if (!hasAutoCensor && !hasReplacements && !hasSpeakerReplacements) {
+    throw new MuxAiError("At least one of autoCensorProfanity, replacements, or speakerReplacements must be provided.", { type: "validation_error" });
   }
   if (autoCensorOption && !provider) {
     throw new MuxAiError("provider is required when using autoCensorProfanity.", { type: "validation_error" });
@@ -4330,6 +4369,16 @@ async function editCaptionsInternal(assetId, trackId, options, collectedUsage) {
   const deleteOriginal = deleteOriginalTrack !== false;
   const uploadToMux = uploadToMuxOption !== false;
   const uploadToS3 = uploadToS3Option || uploadToMux;
+  for (const replacement of speakerReplacementsOption ?? []) {
+    const hasInvalidFindCharacter = replacement.find.includes("[") || replacement.find.includes("]") || /[\r\n]/.test(replacement.find);
+    const hasInvalidReplaceCharacter = replacement.replace.includes("[") || replacement.replace.includes("]") || /[\r\n]/.test(replacement.replace);
+    if (!replacement.find || !replacement.replace || hasInvalidFindCharacter || hasInvalidReplaceCharacter) {
+      throw new MuxAiError(
+        "Speaker replacement labels must be non-empty and must not contain square brackets or newlines.",
+        { type: "validation_error" }
+      );
+    }
+  }
   const s3Endpoint = providedS3Endpoint ?? env_default.S3_ENDPOINT;
   const s3Region = providedS3Region ?? env_default.S3_REGION ?? "auto";
   const s3Bucket = providedS3Bucket ?? env_default.S3_BUCKET;
@@ -4411,6 +4460,16 @@ async function editCaptionsInternal(assetId, trackId, options, collectedUsage) {
     totalReplacementCount += censorReplacements.length;
     autoCensorResult = { replacements: censorReplacements };
   }
+  let speakerReplacementsResult;
+  if (speakerReplacementsOption && speakerReplacementsOption.length > 0) {
+    const { editedVtt: afterSpeakerReplacements, replacements: speakerReplacements } = applySpeakerReplacements(
+      editedVtt,
+      speakerReplacementsOption
+    );
+    editedVtt = afterSpeakerReplacements;
+    totalReplacementCount += speakerReplacements.length;
+    speakerReplacementsResult = { replacements: speakerReplacements };
+  }
   let replacementsResult;
   if (replacementsOption && replacementsOption.length > 0) {
     const { editedVtt: afterReplacements, replacements: staticReplacements } = applyReplacements(editedVtt, replacementsOption);
@@ -4473,6 +4532,7 @@ async function editCaptionsInternal(assetId, trackId, options, collectedUsage) {
     totalReplacementCount,
     autoCensorProfanity: autoCensorResult,
     replacements: replacementsResult,
+    speakerReplacements: speakerReplacementsResult,
     uploadedTrackId,
     presignedUrl,
     usage: usageWithMetadata,
@@ -8239,6 +8299,7 @@ export {
   aggregateTokenUsage,
   applyOverrideLists,
   applyReplacements,
+  applySpeakerReplacements,
   askQuestions,
   buildReplacementRegex,
   burnedInCaptionsSchema,