@mux/ai 0.3.1 → 0.4.2

package/README.md

@@ -83,12 +83,12 @@ S3_SECRET_ACCESS_KEY=your-secret-key
 
 | Workflow                                                                 | Description                                                       | Providers                 | Default Models                                                     | Mux Asset Requirements | Cloud Infrastructure Requirements |
 | ------------------------------------------------------------------------ | ----------------------------------------------------------------- | ------------------------- | ------------------------------------------------------------------ | ---------------------- | --------------------------------- |
-| [`getSummaryAndTags`](./docs/WORKFLOWS.md#video-summarization)<br/>[API](./docs/API.md#getsummaryandtagsassetid-options) · [Source](./src/workflows/summarization.ts) | Generate titles, descriptions, and tags for an asset              | OpenAI, Anthropic, Google | `gpt-5.1` (OpenAI), `claude-sonnet-4-5` (Anthropic), `gemini-2.5-flash` (Google) | Video (required), Captions (optional) | None |
+| [`getSummaryAndTags`](./docs/WORKFLOWS.md#video-summarization)<br/>[API](./docs/API.md#getsummaryandtagsassetid-options) · [Source](./src/workflows/summarization.ts) | Generate titles, descriptions, and tags for an asset              | OpenAI, Anthropic, Google | `gpt-5.1` (OpenAI), `claude-sonnet-4-5` (Anthropic), `gemini-3-flash-preview` (Google) | Video (required), Captions (optional) | None |
 | [`getModerationScores`](./docs/WORKFLOWS.md#content-moderation)<br/>[API](./docs/API.md#getmoderationscoresassetid-options) · [Source](./src/workflows/moderation.ts) | Detect inappropriate (sexual or violent) content in an asset      | OpenAI, Hive              | `omni-moderation-latest` (OpenAI) or Hive visual moderation task   | Video (required) | None |
-| [`hasBurnedInCaptions`](./docs/WORKFLOWS.md#burned-in-caption-detection)<br/>[API](./docs/API.md#hasburnedincaptionsassetid-options) · [Source](./src/workflows/burned-in-captions.ts) | Detect burned-in captions (hardcoded subtitles) in an asset       | OpenAI, Anthropic, Google | `gpt-5.1` (OpenAI), `claude-sonnet-4-5` (Anthropic), `gemini-2.5-flash` (Google) | Video (required) | None |
-| [`generateChapters`](./docs/WORKFLOWS.md#chapter-generation)<br/>[API](./docs/API.md#generatechaptersassetid-languagecode-options) · [Source](./src/workflows/chapters.ts) | Generate chapter markers for an asset using the transcript        | OpenAI, Anthropic, Google | `gpt-5.1` (OpenAI), `claude-sonnet-4-5` (Anthropic), `gemini-2.5-flash` (Google) | Video (required), Captions (required) | None |
+| [`hasBurnedInCaptions`](./docs/WORKFLOWS.md#burned-in-caption-detection)<br/>[API](./docs/API.md#hasburnedincaptionsassetid-options) · [Source](./src/workflows/burned-in-captions.ts) | Detect burned-in captions (hardcoded subtitles) in an asset       | OpenAI, Anthropic, Google | `gpt-5.1` (OpenAI), `claude-sonnet-4-5` (Anthropic), `gemini-3-flash-preview` (Google) | Video (required) | None |
+| [`generateChapters`](./docs/WORKFLOWS.md#chapter-generation)<br/>[API](./docs/API.md#generatechaptersassetid-languagecode-options) · [Source](./src/workflows/chapters.ts) | Generate chapter markers for an asset using the transcript        | OpenAI, Anthropic, Google | `gpt-5.1` (OpenAI), `claude-sonnet-4-5` (Anthropic), `gemini-3-flash-preview` (Google) | Video (required), Captions (required) | None |
 | [`generateVideoEmbeddings`](./docs/WORKFLOWS.md#video-embeddings)<br/>[API](./docs/API.md#generatevideoembeddingsassetid-options) · [Source](./src/workflows/embeddings.ts) | Generate vector embeddings for an asset's transcript chunks       | OpenAI, Google            | `text-embedding-3-small` (OpenAI), `gemini-embedding-001` (Google) | Video (required), Captions (required) | None |
-| [`translateCaptions`](./docs/WORKFLOWS.md#caption-translation)<br/>[API](./docs/API.md#translatecaptionsassetid-fromlanguagecode-tolanguagecode-options) · [Source](./src/workflows/translate-captions.ts) | Translate an asset's captions into different languages            | OpenAI, Anthropic, Google | `gpt-5.1` (OpenAI), `claude-sonnet-4-5` (Anthropic), `gemini-2.5-flash` (Google) | Video (required), Captions (required) | AWS S3 (if `uploadToMux=true`) |
+| [`translateCaptions`](./docs/WORKFLOWS.md#caption-translation)<br/>[API](./docs/API.md#translatecaptionsassetid-fromlanguagecode-tolanguagecode-options) · [Source](./src/workflows/translate-captions.ts) | Translate an asset's captions into different languages            | OpenAI, Anthropic, Google | `gpt-5.1` (OpenAI), `claude-sonnet-4-5` (Anthropic), `gemini-3-flash-preview` (Google) | Video (required), Captions (required) | AWS S3 (if `uploadToMux=true`) |
 | [`translateAudio`](./docs/WORKFLOWS.md#audio-dubbing)<br/>[API](./docs/API.md#translateaudioassetid-tolanguagecode-options) · [Source](./src/workflows/translate-audio.ts) | Create AI-dubbed audio tracks in different languages for an asset | ElevenLabs only           | ElevenLabs Dubbing API                                             | Video (required), Audio (required) | AWS S3 (if `uploadToMux=true`) |
 
 ## Compatability with Workflow DevKit
@@ -233,7 +233,7 @@ for (const chunk of result.chunks) {
 
 # Key Features
 
-- **Cost-Effective by Default**: Uses affordable frontier models like `gpt-5.1`, `claude-sonnet-4-5`, and `gemini-2.5-flash` to keep analysis costs low while maintaining high quality results
+- **Cost-Effective by Default**: Uses affordable frontier models like `gpt-5.1`, `claude-sonnet-4-5`, and `gemini-3-flash-preview` to keep analysis costs low while maintaining high quality results
 - **Multi-modal Analysis**: Combines storyboard images with video transcripts for richer understanding
 - **Tone Control**: Choose between neutral, playful, or professional analysis styles for summarization
 - **Prompt Customization**: Override specific prompt sections to tune workflows to your exact use case

package/dist/index.js

@@ -404,7 +404,7 @@ import { createOpenAI } from "@ai-sdk/openai";
 var DEFAULT_LANGUAGE_MODELS = {
   openai: "gpt-5.1",
   anthropic: "claude-sonnet-4-5",
-  google: "gemini-2.5-flash"
+  google: "gemini-3-flash-preview"
 };
 var DEFAULT_EMBEDDING_MODELS = {
   openai: "text-embedding-3-small",
@@ -1338,57 +1338,57 @@ async function processConcurrently(items, processor, maxConcurrent = 5) {
   }
   return results;
 }
+async function moderateImageWithOpenAI(entry) {
+  "use step";
+  const apiKey = getApiKeyFromEnv("openai");
+  try {
+    const res = await fetch("https://api.openai.com/v1/moderations", {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        "Authorization": `Bearer ${apiKey}`
+      },
+      body: JSON.stringify({
+        model: entry.model,
+        input: [
+          {
+            type: "image_url",
+            image_url: {
+              url: entry.image
+            }
+          }
+        ]
+      })
+    });
+    const json = await res.json();
+    if (!res.ok) {
+      throw new Error(
+        `OpenAI moderation error: ${res.status} ${res.statusText} - ${JSON.stringify(json)}`
+      );
+    }
+    const categoryScores = json.results?.[0]?.category_scores || {};
+    return {
+      url: entry.url,
+      sexual: categoryScores.sexual || 0,
+      violence: categoryScores.violence || 0,
+      error: false
+    };
+  } catch (error) {
+    console.error("OpenAI moderation failed:", error);
+    return {
+      url: entry.url,
+      sexual: 0,
+      violence: 0,
+      error: true
+    };
+  }
+}
 async function requestOpenAIModeration(imageUrls, model, maxConcurrent = 5, submissionMode = "url", downloadOptions) {
   "use step";
   const targetUrls = submissionMode === "base64" ? (await downloadImagesAsBase64(imageUrls, downloadOptions, maxConcurrent)).map(
     (img) => ({ url: img.url, image: img.base64Data, model })
   ) : imageUrls.map((url) => ({ url, image: url, model }));
-  const moderate = async (entry) => {
-    "use step";
-    const apiKey = getApiKeyFromEnv("openai");
-    try {
-      const res = await fetch("https://api.openai.com/v1/moderations", {
-        method: "POST",
-        headers: {
-          "Content-Type": "application/json",
-          "Authorization": `Bearer ${apiKey}`
-        },
-        body: JSON.stringify({
-          model: entry.model,
-          input: [
-            {
-              type: "image_url",
-              image_url: {
-                url: entry.image
-              }
-            }
-          ]
-        })
-      });
-      const json = await res.json();
-      if (!res.ok) {
-        throw new Error(
-          `OpenAI moderation error: ${res.status} ${res.statusText} - ${JSON.stringify(json)}`
-        );
-      }
-      const categoryScores = json.results?.[0]?.category_scores || {};
-      return {
-        url: entry.url,
-        sexual: categoryScores.sexual || 0,
-        violence: categoryScores.violence || 0,
-        error: false
-      };
-    } catch (error) {
-      console.error("OpenAI moderation failed:", error);
-      return {
-        url: entry.url,
-        sexual: 0,
-        violence: 0,
-        error: true
-      };
-    }
-  };
-  return processConcurrently(targetUrls, moderate, maxConcurrent);
+  return processConcurrently(targetUrls, moderateImageWithOpenAI, maxConcurrent);
 }
 function getHiveCategoryScores(classes, categoryNames) {
   const scoreMap = Object.fromEntries(
@@ -1397,6 +1397,51 @@ function getHiveCategoryScores(classes, categoryNames) {
   const scores = categoryNames.map((category) => scoreMap[category] || 0);
   return Math.max(...scores, 0);
 }
+async function moderateImageWithHive(entry) {
+  "use step";
+  const apiKey = getApiKeyFromEnv("hive");
+  try {
+    const formData = new FormData();
+    if (entry.source.kind === "url") {
+      formData.append("url", entry.source.value);
+    } else {
+      const extension = entry.source.contentType.split("/")[1] || "jpg";
+      const blob = new Blob([entry.source.buffer], {
+        type: entry.source.contentType
+      });
+      formData.append("media", blob, `thumbnail.${extension}`);
+    }
+    const res = await fetch(HIVE_ENDPOINT, {
+      method: "POST",
+      headers: {
+        Accept: "application/json",
+        Authorization: `Token ${apiKey}`
+      },
+      body: formData
+    });
+    const json = await res.json().catch(() => void 0);
+    if (!res.ok) {
+      throw new Error(
+        `Hive moderation error: ${res.status} ${res.statusText} - ${JSON.stringify(json)}`
+      );
+    }
+    const classes = json?.status?.[0]?.response?.output?.[0]?.classes || [];
+    return {
+      url: entry.url,
+      sexual: getHiveCategoryScores(classes, HIVE_SEXUAL_CATEGORIES),
+      violence: getHiveCategoryScores(classes, HIVE_VIOLENCE_CATEGORIES),
+      error: false
+    };
+  } catch (error) {
+    console.error("Hive moderation failed:", error);
+    return {
+      url: entry.url,
+      sexual: 0,
+      violence: 0,
+      error: true
+    };
+  }
+}
 async function requestHiveModeration(imageUrls, maxConcurrent = 5, submissionMode = "url", downloadOptions) {
   "use step";
   const targets = submissionMode === "base64" ? (await downloadImagesAsBase64(imageUrls, downloadOptions, maxConcurrent)).map((img) => ({
@@ -1410,52 +1455,7 @@ async function requestHiveModeration(imageUrls, maxConcurrent = 5, submissionMod
     url,
     source: { kind: "url", value: url }
   }));
-  const moderate = async (entry) => {
-    "use step";
-    const apiKey = getApiKeyFromEnv("hive");
-    try {
-      const formData = new FormData();
-      if (entry.source.kind === "url") {
-        formData.append("url", entry.source.value);
-      } else {
-        const extension = entry.source.contentType.split("/")[1] || "jpg";
-        const blob = new Blob([entry.source.buffer], {
-          type: entry.source.contentType
-        });
-        formData.append("media", blob, `thumbnail.${extension}`);
-      }
-      const res = await fetch(HIVE_ENDPOINT, {
-        method: "POST",
-        headers: {
-          Accept: "application/json",
-          Authorization: `Token ${apiKey}`
-        },
-        body: formData
-      });
-      const json = await res.json().catch(() => void 0);
-      if (!res.ok) {
-        throw new Error(
-          `Hive moderation error: ${res.status} ${res.statusText} - ${JSON.stringify(json)}`
-        );
-      }
-      const classes = json?.status?.[0]?.response?.output?.[0]?.classes || [];
-      return {
-        url: entry.url,
-        sexual: getHiveCategoryScores(classes, HIVE_SEXUAL_CATEGORIES),
-        violence: getHiveCategoryScores(classes, HIVE_VIOLENCE_CATEGORIES),
-        error: false
-      };
-    } catch (error) {
-      console.error("Hive moderation failed:", error);
-      return {
-        url: entry.url,
-        sexual: 0,
-        violence: 0,
-        error: true
-      };
-    }
-  };
-  return processConcurrently(targets, moderate, maxConcurrent);
+  return processConcurrently(targets, moderateImageWithHive, maxConcurrent);
 }
 async function getModerationScores(assetId, options = {}) {
   "use workflow";