@mux/ai 0.1.6 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +346 -86
- package/dist/{index-DyTSka2R.d.ts → index-BcNDGOI6.d.ts} +12 -24
- package/dist/{index-Bnv7tv90.d.ts → index-D3fZHu0h.d.ts} +124 -13
- package/dist/index.d.ts +3 -3
- package/dist/index.js +989 -669
- package/dist/index.js.map +1 -1
- package/dist/primitives/index.d.ts +2 -2
- package/dist/primitives/index.js +37 -79
- package/dist/primitives/index.js.map +1 -1
- package/dist/{types-ktXDZ93V.d.mts → types-DzOQNn9R.d.ts} +3 -25
- package/dist/workflows/index.d.ts +2 -2
- package/dist/workflows/index.js +987 -683
- package/dist/workflows/index.js.map +1 -1
- package/package.json +34 -36
- package/dist/index-BNnz9P_5.d.mts +0 -144
- package/dist/index-vJ5r2FNm.d.mts +0 -477
- package/dist/index.d.mts +0 -13
- package/dist/index.mjs +0 -2205
- package/dist/index.mjs.map +0 -1
- package/dist/primitives/index.d.mts +0 -3
- package/dist/primitives/index.mjs +0 -358
- package/dist/primitives/index.mjs.map +0 -1
- package/dist/types-ktXDZ93V.d.ts +0 -137
- package/dist/workflows/index.d.mts +0 -8
- package/dist/workflows/index.mjs +0 -2168
- package/dist/workflows/index.mjs.map +0 -1
package/README.md
CHANGED
|
@@ -1,36 +1,46 @@
|
|
|
1
|
-
#
|
|
1
|
+
# `@mux/ai` 📼 🤝 🤖
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
[](https://www.npmjs.com/package/@mux/ai)
|
|
4
|
+
[](https://opensource.org/licenses/Apache-2.0)
|
|
4
5
|
|
|
5
|
-
|
|
6
|
+
> **A TypeScript toolkit for building AI-driven video workflows on the server, powered by [Mux](https://www.mux.com)!**
|
|
6
7
|
|
|
7
|
-
|
|
8
|
+
`@mux/ai` does this by providing:
|
|
8
9
|
|
|
9
|
-
|
|
10
|
+
Easy to use, purpose-driven, cost effective, configurable **_workflow functions_** that integrate with a variety of popular AI/LLM providers (OpenAI, Anthropic, Google).
|
|
11
|
+
- **Examples:** [`getSummaryAndTags`](#video-summarization), [`getModerationScores`](#content-moderation), [`hasBurnedInCaptions`](#burned-in-caption-detection), [`generateChapters`](#chapter-generation), [`generateVideoEmbeddings`](#video-search-with-embeddings), [`translateCaptions`](#caption-translation), [`translateAudio`](#audio-dubbing)
|
|
12
|
+
- Workflows automatically ship with `"use workflow"` [compatability with Workflow DevKit](#compatability-with-workflow-devkit)
|
|
10
13
|
|
|
11
|
-
|
|
14
|
+
Convenient, parameterized, commonly needed **_primitive functions_** backed by [Mux Video](https://www.mux.com/video-api) for building your own media-based AI workflows and integrations.
|
|
15
|
+
- **Examples:** `getStoryboardUrl`, `chunkVTTCues`, `fetchTranscriptForAsset`
|
|
16
|
+
|
|
17
|
+
# Usage
|
|
18
|
+
|
|
19
|
+
```ts
|
|
20
|
+
import { getSummaryAndTags } from "@mux/ai/workflows";
|
|
21
|
+
|
|
22
|
+
const result = await getSummaryAndTags("your-asset-id", {
|
|
23
|
+
provider: "openai",
|
|
24
|
+
tone: "professional",
|
|
25
|
+
includeTranscript: true
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
console.log(result.title); // "Getting Started with TypeScript"
|
|
29
|
+
console.log(result.description); // "A comprehensive guide to..."
|
|
30
|
+
console.log(result.tags); // ["typescript", "tutorial", "programming"]
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
> **⚠️ Important:** Many workflows rely on video transcripts for best results. Consider enabling [auto-generated captions](https://www.mux.com/docs/guides/add-autogenerated-captions-and-use-transcripts) on your Mux assets to unlock the full potential of transcript-based workflows like summarization, chapters, and embeddings.
|
|
34
|
+
|
|
35
|
+
# Quick Start
|
|
36
|
+
|
|
37
|
+
## Prerequisites
|
|
38
|
+
|
|
39
|
+
- [Node.js](https://nodejs.org/en/download) (≥ 21.0.0)
|
|
40
|
+
- A Mux account and necessary [credentials](#credentials---mux) for your environment (sign up [here](https://dashboard.mux.com/signup) for free!)
|
|
41
|
+
- Accounts and [credentials](#credentials---ai-providers) for any AI providers you intend to use for your workflows
|
|
42
|
+
- (For some workflows only) AWS S3 and [other credentials](#credentials---other)
|
|
12
43
|
|
|
13
|
-
| Workflow | Description | Providers | Default Models |
|
|
14
|
-
| ------------------------------------------------------------------------ | ----------------------------------------------------------------- | ------------------------- | ------------------------------------------------------------------ |
|
|
15
|
-
| [`getSummaryAndTags`](./docs/WORKFLOWS.md#video-summarization) | Generate titles, descriptions, and tags for an asset | OpenAI, Anthropic, Google | `gpt-5-mini`, `claude-sonnet-4-5`, `gemini-2.5-flash` |
|
|
16
|
-
| [`getModerationScores`](./docs/WORKFLOWS.md#content-moderation) | Detect inappropriate (sexual or violent) content in an asset | OpenAI, Hive | `omni-moderation-latest` (OpenAI) or Hive visual moderation task |
|
|
17
|
-
| [`hasBurnedInCaptions`](./docs/WORKFLOWS.md#burned-in-caption-detection) | Detect burned-in captions (hardcoded subtitles) in an asset | OpenAI, Anthropic, Google | `gpt-5-mini`, `claude-sonnet-4-5`, `gemini-2.5-flash` |
|
|
18
|
-
| [`generateChapters`](./docs/WORKFLOWS.md#chapter-generation) | Generate chapter markers for an asset using the transcript | OpenAI, Anthropic, Google | `gpt-5-mini`, `claude-sonnet-4-5`, `gemini-2.5-flash` |
|
|
19
|
-
| [`generateVideoEmbeddings`](./docs/WORKFLOWS.md#video-embeddings) | Generate vector embeddings for an asset's transcript chunks | OpenAI, Google | `text-embedding-3-small` (OpenAI), `gemini-embedding-001` (Google) |
|
|
20
|
-
| [`translateCaptions`](./docs/WORKFLOWS.md#caption-translation) | Translate an asset's captions into different languages | OpenAI, Anthropic, Google | `gpt-5-mini`, `claude-sonnet-4-5`, `gemini-2.5-flash` |
|
|
21
|
-
| [`translateAudio`](./docs/WORKFLOWS.md#audio-dubbing) | Create AI-dubbed audio tracks in different languages for an asset | ElevenLabs only | ElevenLabs Dubbing API |
|
|
22
|
-
|
|
23
|
-
## Features
|
|
24
|
-
|
|
25
|
-
- **Cost-Effective by Default**: Uses affordable frontier models like `gpt-5-mini`, `claude-sonnet-4-5`, and `gemini-2.5-flash` to keep analysis costs low while maintaining high quality results
|
|
26
|
-
- **Multi-modal Analysis**: Combines storyboard images with video transcripts
|
|
27
|
-
- **Tone Control**: Normal, sassy, or professional analysis styles
|
|
28
|
-
- **Prompt Customization**: Override specific prompt sections to tune workflows to your use case
|
|
29
|
-
- **Configurable Thresholds**: Custom sensitivity levels for content moderation
|
|
30
|
-
- **TypeScript**: Fully typed for excellent developer experience
|
|
31
|
-
- **Provider Choice**: Switch between OpenAI, Anthropic, and Google for different perspectives
|
|
32
|
-
- **Composable Building Blocks**: Import primitives to fetch transcripts, thumbnails, and storyboards to build bespoke flows
|
|
33
|
-
- **Universal Language Support**: Automatic language name detection using `Intl.DisplayNames` for all ISO 639-1 codes
|
|
34
44
|
|
|
35
45
|
## Installation
|
|
36
46
|
|
|
@@ -40,7 +50,7 @@ npm install @mux/ai
|
|
|
40
50
|
|
|
41
51
|
## Configuration
|
|
42
52
|
|
|
43
|
-
|
|
53
|
+
We support [dotenv](https://www.npmjs.com/package/dotenv), so you can simply add the following environment variables to your `.env` file:
|
|
44
54
|
|
|
45
55
|
```bash
|
|
46
56
|
# Required
|
|
@@ -51,12 +61,10 @@ MUX_TOKEN_SECRET=your_mux_token_secret
|
|
|
51
61
|
MUX_SIGNING_KEY=your_signing_key_id
|
|
52
62
|
MUX_PRIVATE_KEY=your_base64_encoded_private_key
|
|
53
63
|
|
|
54
|
-
# You only need to configure API keys for the AI platforms you're using
|
|
64
|
+
# You only need to configure API keys for the AI platforms and workflows you're using
|
|
55
65
|
OPENAI_API_KEY=your_openai_api_key
|
|
56
66
|
ANTHROPIC_API_KEY=your_anthropic_api_key
|
|
57
67
|
GOOGLE_GENERATIVE_AI_API_KEY=your_google_api_key
|
|
58
|
-
|
|
59
|
-
# Needed for audio dubbing workflow
|
|
60
68
|
ELEVENLABS_API_KEY=your_elevenlabs_api_key
|
|
61
69
|
|
|
62
70
|
# S3-Compatible Storage (required for translation & audio dubbing)
|
|
@@ -67,116 +75,368 @@ S3_ACCESS_KEY_ID=your-access-key
|
|
|
67
75
|
S3_SECRET_ACCESS_KEY=your-secret-key
|
|
68
76
|
```
|
|
69
77
|
|
|
70
|
-
|
|
78
|
+
> **💡 Tip:** If you're using `.env` in a repository or version tracking system, make sure you add this file to your `.gitignore` or equivalent to avoid unintentionally committing secure credentials.
|
|
71
79
|
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
80
|
+
# Workflows
|
|
81
|
+
|
|
82
|
+
## Available pre-built workflows
|
|
83
|
+
|
|
84
|
+
| Workflow | Description | Providers | Default Models | Mux Asset Requirements | Cloud Infrastructure Requirements |
|
|
85
|
+
| ------------------------------------------------------------------------ | ----------------------------------------------------------------- | ------------------------- | ------------------------------------------------------------------ | ---------------------- | --------------------------------- |
|
|
86
|
+
| [`getSummaryAndTags`](./docs/WORKFLOWS.md#video-summarization)<br/>[API](./docs/API.md#getsummaryandtagsassetid-options) · [Source](./src/workflows/summarization.ts) | Generate titles, descriptions, and tags for an asset | OpenAI, Anthropic, Google | `gpt-5.1` (OpenAI), `claude-sonnet-4-5` (Anthropic), `gemini-2.5-flash` (Google) | Video (required), Captions (optional) | None |
|
|
87
|
+
| [`getModerationScores`](./docs/WORKFLOWS.md#content-moderation)<br/>[API](./docs/API.md#getmoderationscoresassetid-options) · [Source](./src/workflows/moderation.ts) | Detect inappropriate (sexual or violent) content in an asset | OpenAI, Hive | `omni-moderation-latest` (OpenAI) or Hive visual moderation task | Video (required) | None |
|
|
88
|
+
| [`hasBurnedInCaptions`](./docs/WORKFLOWS.md#burned-in-caption-detection)<br/>[API](./docs/API.md#hasburnedincaptionsassetid-options) · [Source](./src/workflows/burned-in-captions.ts) | Detect burned-in captions (hardcoded subtitles) in an asset | OpenAI, Anthropic, Google | `gpt-5.1` (OpenAI), `claude-sonnet-4-5` (Anthropic), `gemini-2.5-flash` (Google) | Video (required) | None |
|
|
89
|
+
| [`generateChapters`](./docs/WORKFLOWS.md#chapter-generation)<br/>[API](./docs/API.md#generatechaptersassetid-languagecode-options) · [Source](./src/workflows/chapters.ts) | Generate chapter markers for an asset using the transcript | OpenAI, Anthropic, Google | `gpt-5.1` (OpenAI), `claude-sonnet-4-5` (Anthropic), `gemini-2.5-flash` (Google) | Video (required), Captions (required) | None |
|
|
90
|
+
| [`generateVideoEmbeddings`](./docs/WORKFLOWS.md#video-embeddings)<br/>[API](./docs/API.md#generatevideoembeddingsassetid-options) · [Source](./src/workflows/embeddings.ts) | Generate vector embeddings for an asset's transcript chunks | OpenAI, Google | `text-embedding-3-small` (OpenAI), `gemini-embedding-001` (Google) | Video (required), Captions (required) | None |
|
|
91
|
+
| [`translateCaptions`](./docs/WORKFLOWS.md#caption-translation)<br/>[API](./docs/API.md#translatecaptionsassetid-fromlanguagecode-tolanguagecode-options) · [Source](./src/workflows/translate-captions.ts) | Translate an asset's captions into different languages | OpenAI, Anthropic, Google | `gpt-5.1` (OpenAI), `claude-sonnet-4-5` (Anthropic), `gemini-2.5-flash` (Google) | Video (required), Captions (required) | AWS S3 (if `uploadToMux=true`) |
|
|
92
|
+
| [`translateAudio`](./docs/WORKFLOWS.md#audio-dubbing)<br/>[API](./docs/API.md#translateaudioassetid-tolanguagecode-options) · [Source](./src/workflows/translate-audio.ts) | Create AI-dubbed audio tracks in different languages for an asset | ElevenLabs only | ElevenLabs Dubbing API | Video (required), Audio (required) | AWS S3 (if `uploadToMux=true`) |
|
|
93
|
+
|
|
94
|
+
## Compatability with Workflow DevKit
|
|
95
|
+
|
|
96
|
+
All workflows are compatible with [Workflow DevKit](https://useworkflow.dev). The workflows in this SDK are exported with `"use workflow"` directives and `"use step"` directives in the code.
|
|
97
|
+
|
|
98
|
+
If you are using Workflow DevKit in your project, then you must call workflow functions like this:
|
|
99
|
+
|
|
100
|
+
```ts
|
|
101
|
+
import { start } from 'workflow/api';
|
|
102
|
+
import { getSummaryAndTags } from '@mux/ai/workflows';
|
|
103
|
+
|
|
104
|
+
const assetId = 'YOUR_ASSET_ID';
|
|
105
|
+
const run = await start(getSummaryAndTags, [assetId]);
|
|
106
|
+
|
|
107
|
+
// optionally, wait for the workflow run return value:
|
|
108
|
+
// const result = await run.returnValue
|
|
78
109
|
```
|
|
79
110
|
|
|
80
|
-
|
|
111
|
+
### Features of Workflow DevKit
|
|
112
|
+
|
|
113
|
+
- [Observability Dashboard](https://useworkflow.dev/docs/observability)
|
|
114
|
+
- [Control Flow Patterns](https://useworkflow.dev/docs/foundations/control-flow-patterns) like Parallel Execution.
|
|
115
|
+
- [Errors and Retrying](https://useworkflow.dev/docs/foundations/errors-and-retries)
|
|
116
|
+
- [Hooks and Webhooks](https://useworkflow.dev/docs/foundations/hooks)
|
|
117
|
+
- Patterns for building Agents with [Human in the Loop](https://useworkflow.dev/docs/ai/human-in-the-loop)
|
|
118
|
+
|
|
119
|
+
**Workflows can be nested**
|
|
81
120
|
|
|
82
|
-
|
|
121
|
+
```ts
|
|
122
|
+
import { start } from "workflow/api";
|
|
123
|
+
import { getSummaryAndTags } from '@mux/ai/workflows';
|
|
124
|
+
|
|
125
|
+
async function processVideoSummary (assetId: string) {
|
|
126
|
+
'use workflow'
|
|
127
|
+
|
|
128
|
+
const summary = await getSummaryAndTags(assetId);
|
|
129
|
+
const emailResp = await emailSummaryToAdmins(summary: summary);
|
|
130
|
+
|
|
131
|
+
return { assetId, summary, emailResp }
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
async function emailSummaryToAdmins (assetId: string) {
|
|
135
|
+
'use step';
|
|
136
|
+
return { sent: true }
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
//
|
|
140
|
+
// this will call the processVideoSummary workflow that is defined above
|
|
141
|
+
// in that workflow, it calls `getSummaryAndTags()` workflow
|
|
142
|
+
//
|
|
143
|
+
const run = await start(processVideoSummary, [assetId]);
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
## Example Workflows
|
|
83
147
|
|
|
84
148
|
### Video Summarization
|
|
85
149
|
|
|
150
|
+
Generate SEO-friendly titles, descriptions, and tags from your video content:
|
|
151
|
+
|
|
86
152
|
```typescript
|
|
87
153
|
import { getSummaryAndTags } from "@mux/ai/workflows";
|
|
88
154
|
|
|
89
|
-
const result = await getSummaryAndTags("your-
|
|
90
|
-
|
|
155
|
+
const result = await getSummaryAndTags("your-asset-id", {
|
|
156
|
+
provider: "openai",
|
|
157
|
+
tone: "professional",
|
|
158
|
+
includeTranscript: true
|
|
91
159
|
});
|
|
92
160
|
|
|
93
|
-
console.log(result.title);
|
|
94
|
-
console.log(result.description);
|
|
95
|
-
console.log(result.tags);
|
|
161
|
+
console.log(result.title); // "Getting Started with TypeScript"
|
|
162
|
+
console.log(result.description); // "A comprehensive guide to..."
|
|
163
|
+
console.log(result.tags); // ["typescript", "tutorial", "programming"]
|
|
96
164
|
```
|
|
97
165
|
|
|
98
166
|
### Content Moderation
|
|
99
167
|
|
|
168
|
+
Automatically detect inappropriate content in videos:
|
|
169
|
+
|
|
100
170
|
```typescript
|
|
101
171
|
import { getModerationScores } from "@mux/ai/workflows";
|
|
102
172
|
|
|
103
|
-
const result = await getModerationScores("your-
|
|
173
|
+
const result = await getModerationScores("your-asset-id", {
|
|
174
|
+
provider: "openai",
|
|
104
175
|
thresholds: { sexual: 0.7, violence: 0.8 }
|
|
105
176
|
});
|
|
106
177
|
|
|
107
|
-
|
|
178
|
+
if (result.exceedsThreshold) {
|
|
179
|
+
console.log("Content flagged for review");
|
|
180
|
+
console.log(`Max scores: ${result.maxScores}`);
|
|
181
|
+
}
|
|
108
182
|
```
|
|
109
183
|
|
|
110
|
-
###
|
|
184
|
+
### Chapter Generation
|
|
185
|
+
|
|
186
|
+
Create automatic chapter markers for better video navigation:
|
|
111
187
|
|
|
112
188
|
```typescript
|
|
113
189
|
import { generateChapters } from "@mux/ai/workflows";
|
|
114
190
|
|
|
115
|
-
const result = await generateChapters("your-
|
|
191
|
+
const result = await generateChapters("your-asset-id", "en", {
|
|
192
|
+
provider: "anthropic"
|
|
193
|
+
});
|
|
116
194
|
|
|
117
195
|
// Use with Mux Player
|
|
118
196
|
player.addChapters(result.chapters);
|
|
197
|
+
// [
|
|
198
|
+
// { startTime: 0, title: "Introduction" },
|
|
199
|
+
// { startTime: 45, title: "Main Content" },
|
|
200
|
+
// { startTime: 120, title: "Conclusion" }
|
|
201
|
+
// ]
|
|
119
202
|
```
|
|
120
203
|
|
|
121
|
-
###
|
|
204
|
+
### Video Search with Embeddings
|
|
205
|
+
|
|
206
|
+
Generate embeddings for semantic video search:
|
|
122
207
|
|
|
123
208
|
```typescript
|
|
124
|
-
import {
|
|
209
|
+
import { generateVideoEmbeddings } from "@mux/ai/workflows";
|
|
210
|
+
|
|
211
|
+
const result = await generateVideoEmbeddings("your-asset-id", {
|
|
212
|
+
provider: "openai",
|
|
213
|
+
languageCode: "en",
|
|
214
|
+
chunkingStrategy: {
|
|
215
|
+
type: "token",
|
|
216
|
+
maxTokens: 500,
|
|
217
|
+
overlap: 100
|
|
218
|
+
}
|
|
219
|
+
});
|
|
125
220
|
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
221
|
+
// Store embeddings in your vector database
|
|
222
|
+
for (const chunk of result.chunks) {
|
|
223
|
+
await vectorDB.insert({
|
|
224
|
+
embedding: chunk.embedding,
|
|
225
|
+
metadata: {
|
|
226
|
+
assetId: result.assetId,
|
|
227
|
+
startTime: chunk.metadata.startTime,
|
|
228
|
+
endTime: chunk.metadata.endTime
|
|
229
|
+
}
|
|
230
|
+
});
|
|
231
|
+
}
|
|
232
|
+
```
|
|
233
|
+
|
|
234
|
+
# Key Features
|
|
235
|
+
|
|
236
|
+
- **Cost-Effective by Default**: Uses affordable frontier models like `gpt-5.1`, `claude-sonnet-4-5`, and `gemini-2.5-flash` to keep analysis costs low while maintaining high quality results
|
|
237
|
+
- **Multi-modal Analysis**: Combines storyboard images with video transcripts for richer understanding
|
|
238
|
+
- **Tone Control**: Choose between neutral, playful, or professional analysis styles for summarization
|
|
239
|
+
- **Prompt Customization**: Override specific prompt sections to tune workflows to your exact use case
|
|
240
|
+
- **Configurable Thresholds**: Set custom sensitivity levels for content moderation
|
|
241
|
+
- **Full TypeScript Support**: Comprehensive types for excellent developer experience and IDE autocomplete
|
|
242
|
+
- **Provider Flexibility**: Switch between OpenAI, Anthropic, Google, and other providers based on your needs
|
|
243
|
+
- **Composable Building Blocks**: Use primitives to fetch transcripts, thumbnails, and storyboards for custom workflows
|
|
244
|
+
- **Universal Language Support**: Automatic language name detection using `Intl.DisplayNames` for all ISO 639-1 codes
|
|
245
|
+
- **Production Ready**: Built-in retry logic, error handling, and edge case management
|
|
132
246
|
|
|
133
|
-
|
|
247
|
+
# Core Concepts
|
|
248
|
+
|
|
249
|
+
`@mux/ai` is built around two complementary abstractions:
|
|
250
|
+
|
|
251
|
+
## Workflows
|
|
252
|
+
|
|
253
|
+
**Workflows** are functions that handle complete video AI tasks end-to-end. Each workflow orchestrates the entire process: fetching video data from Mux (transcripts, thumbnails, storyboards), formatting it for AI providers, and returning structured results.
|
|
254
|
+
|
|
255
|
+
```typescript
|
|
256
|
+
import { getSummaryAndTags } from "@mux/ai/workflows";
|
|
257
|
+
|
|
258
|
+
const result = await getSummaryAndTags("asset-id", { provider: "openai" });
|
|
134
259
|
```
|
|
135
260
|
|
|
136
|
-
|
|
261
|
+
Use workflows when you need battle-tested solutions for common tasks like summarization, content moderation, chapter generation, or translation.
|
|
137
262
|
|
|
138
|
-
|
|
263
|
+
## Primitives
|
|
139
264
|
|
|
140
|
-
-
|
|
141
|
-
- **`@mux/ai/primitives`** – Low-level building blocks like `fetchTranscriptForAsset` and `getStoryboardUrl`
|
|
142
|
-
- **`@mux/ai`** – Main entry point that re-exports both namespaces plus shared types
|
|
265
|
+
**Primitives** are low-level building blocks that give you direct access to Mux video data and utilities. They provide functions for fetching transcripts, storyboards, thumbnails, and processing text—perfect for building custom workflows.
|
|
143
266
|
|
|
144
267
|
```typescript
|
|
145
|
-
// Or import everything
|
|
146
|
-
import { primitives, workflows } from "@mux/ai";
|
|
147
|
-
// Low-level primitives for custom workflows
|
|
148
268
|
import { fetchTranscriptForAsset, getStoryboardUrl } from "@mux/ai/primitives";
|
|
149
|
-
|
|
150
|
-
|
|
269
|
+
|
|
270
|
+
const transcript = await fetchTranscriptForAsset("asset-id", "en");
|
|
271
|
+
const storyboard = getStoryboardUrl("playback-id", { width: 640 });
|
|
151
272
|
```
|
|
152
273
|
|
|
153
|
-
|
|
274
|
+
Use primitives when you need complete control over your AI prompts or want to build custom workflows not covered by the pre-built options.
|
|
154
275
|
|
|
155
|
-
##
|
|
276
|
+
## Package Structure
|
|
156
277
|
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
278
|
+
```typescript
|
|
279
|
+
// Import workflows
|
|
280
|
+
import { generateChapters } from "@mux/ai/workflows";
|
|
281
|
+
|
|
282
|
+
// Import primitives
|
|
283
|
+
import { fetchTranscriptForAsset } from "@mux/ai/primitives";
|
|
284
|
+
|
|
285
|
+
// Or import everything
|
|
286
|
+
import { workflows, primitives } from "@mux/ai";
|
|
287
|
+
```
|
|
288
|
+
|
|
289
|
+
# Credentials
|
|
290
|
+
|
|
291
|
+
You'll need to set up credentials for Mux as well as any AI provider you want to use for a particular workflow. In addition, some workflows will need other cloud-hosted access (e.g. cloud storage via AWS S3).
|
|
292
|
+
|
|
293
|
+
## Credentials - Mux
|
|
294
|
+
|
|
295
|
+
### Access Token (required)
|
|
296
|
+
|
|
297
|
+
All workflows require a Mux API access token to interact with your video assets. If you're already logged into the dashboard, you can [create a new access token here](https://dashboard.mux.com/settings/access-tokens).
|
|
298
|
+
|
|
299
|
+
**Required Permissions:**
|
|
300
|
+
- **Mux Video**: Read + Write access
|
|
301
|
+
- **Mux Data**: Read access
|
|
161
302
|
|
|
162
|
-
|
|
303
|
+
These permissions cover all current workflows. You can set these when creating your token in the dashboard.
|
|
304
|
+
|
|
305
|
+
> **💡 Tip:** For security reasons, consider creating a dedicated access token specifically for your AI workflows rather than reusing existing tokens.
|
|
306
|
+
|
|
307
|
+
### Signing Key (conditionally required)
|
|
308
|
+
|
|
309
|
+
If your Mux assets use [signed playback URLs](https://docs.mux.com/guides/secure-video-playback) for security, you'll need to provide signing credentials so `@mux/ai` can access the video data.
|
|
310
|
+
|
|
311
|
+
**When needed:** Only if your assets have signed playback policies enabled and no public playback ID.
|
|
312
|
+
|
|
313
|
+
**How to get:**
|
|
314
|
+
1. Go to [Settings > Signing Keys](https://dashboard.mux.com/settings/signing-keys) in your Mux dashboard
|
|
315
|
+
2. Create a new signing key or use an existing one
|
|
316
|
+
3. Save both the **Signing Key ID** and the **Base64-encoded Private Key**
|
|
317
|
+
|
|
318
|
+
**Configuration:**
|
|
319
|
+
```bash
|
|
320
|
+
MUX_SIGNING_KEY=your_signing_key_id
|
|
321
|
+
MUX_PRIVATE_KEY=your_base64_encoded_private_key
|
|
322
|
+
```
|
|
323
|
+
|
|
324
|
+
## Credentials - AI Providers
|
|
325
|
+
|
|
326
|
+
Different workflows support various AI providers. You only need to configure API keys for the providers you plan to use.
|
|
327
|
+
|
|
328
|
+
### OpenAI
|
|
329
|
+
|
|
330
|
+
**Used by:** `getSummaryAndTags`, `getModerationScores`, `hasBurnedInCaptions`, `generateChapters`, `generateVideoEmbeddings`, `translateCaptions`
|
|
331
|
+
|
|
332
|
+
**Get your API key:** [OpenAI API Keys](https://platform.openai.com/api-keys)
|
|
333
|
+
|
|
334
|
+
```bash
|
|
335
|
+
OPENAI_API_KEY=your_openai_api_key
|
|
336
|
+
```
|
|
337
|
+
|
|
338
|
+
### Anthropic
|
|
339
|
+
|
|
340
|
+
**Used by:** `getSummaryAndTags`, `hasBurnedInCaptions`, `generateChapters`, `translateCaptions`
|
|
341
|
+
|
|
342
|
+
**Get your API key:** [Anthropic Console](https://console.anthropic.com/)
|
|
343
|
+
|
|
344
|
+
```bash
|
|
345
|
+
ANTHROPIC_API_KEY=your_anthropic_api_key
|
|
346
|
+
```
|
|
347
|
+
|
|
348
|
+
### Google Generative AI
|
|
349
|
+
|
|
350
|
+
**Used by:** `getSummaryAndTags`, `hasBurnedInCaptions`, `generateChapters`, `generateVideoEmbeddings`, `translateCaptions`
|
|
351
|
+
|
|
352
|
+
**Get your API key:** [Google AI Studio](https://aistudio.google.com/app/apikey)
|
|
353
|
+
|
|
354
|
+
```bash
|
|
355
|
+
GOOGLE_GENERATIVE_AI_API_KEY=your_google_api_key
|
|
356
|
+
```
|
|
357
|
+
|
|
358
|
+
### ElevenLabs
|
|
359
|
+
|
|
360
|
+
**Used by:** `translateAudio` (audio dubbing)
|
|
361
|
+
|
|
362
|
+
**Get your API key:** [ElevenLabs API Keys](https://elevenlabs.io/app/settings/api-keys)
|
|
363
|
+
|
|
364
|
+
**Note:** Requires a Creator plan or higher for dubbing features.
|
|
163
365
|
|
|
164
366
|
```bash
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
cd mux-ai
|
|
168
|
-
npm install # Automatically sets up git hooks
|
|
367
|
+
ELEVENLABS_API_KEY=your_elevenlabs_api_key
|
|
368
|
+
```
|
|
169
369
|
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
370
|
+
### Hive
|
|
371
|
+
|
|
372
|
+
**Used by:** `getModerationScores` (alternative to OpenAI moderation)
|
|
373
|
+
|
|
374
|
+
**Get your API key:** [Hive Console](https://thehive.ai/)
|
|
375
|
+
|
|
376
|
+
```bash
|
|
377
|
+
HIVE_API_KEY=your_hive_api_key
|
|
378
|
+
```
|
|
379
|
+
|
|
380
|
+
## Credentials - Cloud Infrastructure
|
|
381
|
+
|
|
382
|
+
### AWS S3 (or S3-compatible storage)
|
|
383
|
+
|
|
384
|
+
**Required for:** `translateCaptions`, `translateAudio` (only if `uploadToMux` is true, which is the default)
|
|
385
|
+
|
|
386
|
+
Translation workflows need temporary storage to upload translated files before attaching them to your Mux assets. Any S3-compatible storage service works (AWS S3, Cloudflare R2, DigitalOcean Spaces, etc.).
|
|
387
|
+
|
|
388
|
+
**AWS S3 Setup:**
|
|
389
|
+
1. [Create an S3 bucket](https://s3.console.aws.amazon.com/s3/home)
|
|
390
|
+
2. [Create an IAM user](https://console.aws.amazon.com/iam/) with programmatic access
|
|
391
|
+
3. Attach a policy with `s3:PutObject`, `s3:GetObject`, and `s3:PutObjectAcl` permissions for your bucket
|
|
392
|
+
|
|
393
|
+
**Configuration:**
|
|
394
|
+
```bash
|
|
395
|
+
S3_ENDPOINT=https://s3.amazonaws.com # Or your S3-compatible endpoint
|
|
396
|
+
S3_REGION=us-east-1 # Your bucket region
|
|
397
|
+
S3_BUCKET=your-bucket-name
|
|
398
|
+
S3_ACCESS_KEY_ID=your-access-key
|
|
399
|
+
S3_SECRET_ACCESS_KEY=your-secret-key
|
|
400
|
+
```
|
|
174
401
|
|
|
175
|
-
|
|
176
|
-
|
|
402
|
+
**Cloudflare R2 Example:**
|
|
403
|
+
```bash
|
|
404
|
+
S3_ENDPOINT=https://your-account-id.r2.cloudflarestorage.com
|
|
405
|
+
S3_REGION=auto
|
|
406
|
+
S3_BUCKET=your-bucket-name
|
|
407
|
+
S3_ACCESS_KEY_ID=your-r2-access-key
|
|
408
|
+
S3_SECRET_ACCESS_KEY=your-r2-secret-key
|
|
177
409
|
```
|
|
178
410
|
|
|
179
|
-
|
|
411
|
+
# Documentation
|
|
412
|
+
|
|
413
|
+
## Full Documentation
|
|
414
|
+
|
|
415
|
+
- **[Workflows Guide](./docs/WORKFLOWS.md)** - Detailed guide to each pre-built workflow with examples
|
|
416
|
+
- **[API Reference](./docs/API.md)** - Complete API documentation for all functions, parameters, and return types
|
|
417
|
+
- **[Primitives Guide](./docs/PRIMITIVES.md)** - Low-level building blocks for custom workflows
|
|
418
|
+
- **[Examples](./docs/EXAMPLES.md)** - Running examples from the repository
|
|
419
|
+
|
|
420
|
+
## Additional Resources
|
|
421
|
+
|
|
422
|
+
- **[Mux Video API Docs](https://docs.mux.com/guides/video)** - Learn about Mux Video features
|
|
423
|
+
- **[Auto-generated Captions](https://www.mux.com/docs/guides/add-autogenerated-captions-and-use-transcripts)** - Enable transcripts for your assets
|
|
424
|
+
- **[GitHub Repository](https://github.com/muxinc/ai)** - Source code, issues, and contributions
|
|
425
|
+
- **[npm Package](https://www.npmjs.com/package/@mux/ai)** - Package page and version history
|
|
426
|
+
|
|
427
|
+
# Contributing
|
|
428
|
+
|
|
429
|
+
We welcome contributions! Whether you're fixing bugs, adding features, or improving documentation, we'd love your help.
|
|
430
|
+
|
|
431
|
+
Please see our **[Contributing Guide](./CONTRIBUTING.md)** for details on:
|
|
432
|
+
|
|
433
|
+
- Setting up your development environment
|
|
434
|
+
- Running examples and tests
|
|
435
|
+
- Code style and conventions
|
|
436
|
+
- Submitting pull requests
|
|
437
|
+
- Reporting issues
|
|
438
|
+
|
|
439
|
+
For questions or discussions, feel free to [open an issue](https://github.com/muxinc/ai/issues).
|
|
180
440
|
|
|
181
441
|
## License
|
|
182
442
|
|
|
@@ -1,28 +1,16 @@
|
|
|
1
|
-
import { A as AssetTextTrack,
|
|
2
|
-
|
|
3
|
-
/**
|
|
4
|
-
* Context required to sign URLs for signed playback IDs.
|
|
5
|
-
*/
|
|
6
|
-
interface SigningContext {
|
|
7
|
-
/** The signing key ID from Mux dashboard. */
|
|
8
|
-
keyId: string;
|
|
9
|
-
/** The base64-encoded private key from Mux dashboard. */
|
|
10
|
-
keySecret: string;
|
|
11
|
-
/** Token expiration time (e.g. '1h', '1d'). Defaults to '1h'. */
|
|
12
|
-
expiration?: string;
|
|
13
|
-
}
|
|
1
|
+
import { A as AssetTextTrack, a as MuxAsset, d as TextChunk, C as ChunkingStrategy } from './types-DzOQNn9R.js';
|
|
14
2
|
|
|
15
3
|
declare const DEFAULT_STORYBOARD_WIDTH = 640;
|
|
16
4
|
/**
|
|
17
5
|
* Generates a storyboard URL for the given playback ID.
|
|
18
|
-
* If
|
|
6
|
+
* If shouldSign is true, the URL will be signed with a token using credentials from environment variables.
|
|
19
7
|
*
|
|
20
8
|
* @param playbackId - The Mux playback ID
|
|
21
9
|
* @param width - Width of the storyboard in pixels (default: 640)
|
|
22
|
-
* @param
|
|
23
|
-
* @returns Storyboard URL (signed if
|
|
10
|
+
* @param shouldSign - Flag for whether or not to use signed playback IDs (default: false)
|
|
11
|
+
* @returns Storyboard URL (signed if shouldSign is true)
|
|
24
12
|
*/
|
|
25
|
-
declare function getStoryboardUrl(playbackId: string, width?: number,
|
|
13
|
+
declare function getStoryboardUrl(playbackId: string, width?: number, shouldSign?: boolean): Promise<string>;
|
|
26
14
|
|
|
27
15
|
/** A single cue from a VTT file with timing info. */
|
|
28
16
|
interface VTTCue {
|
|
@@ -34,7 +22,7 @@ interface TranscriptFetchOptions {
|
|
|
34
22
|
languageCode?: string;
|
|
35
23
|
cleanTranscript?: boolean;
|
|
36
24
|
/** Optional signing context for signed playback IDs */
|
|
37
|
-
|
|
25
|
+
shouldSign?: boolean;
|
|
38
26
|
}
|
|
39
27
|
interface TranscriptResult {
|
|
40
28
|
transcriptText: string;
|
|
@@ -59,10 +47,10 @@ declare function parseVTTCues(vttContent: string): VTTCue[];
|
|
|
59
47
|
*
|
|
60
48
|
* @param playbackId - The Mux playback ID
|
|
61
49
|
* @param trackId - The text track ID
|
|
62
|
-
* @param
|
|
50
|
+
* @param shouldSign - Flag for whether or not to use signed playback IDs
|
|
63
51
|
* @returns Transcript URL (signed if context provided)
|
|
64
52
|
*/
|
|
65
|
-
declare function buildTranscriptUrl(playbackId: string, trackId: string,
|
|
53
|
+
declare function buildTranscriptUrl(playbackId: string, trackId: string, shouldSign?: boolean): Promise<string>;
|
|
66
54
|
declare function fetchTranscriptForAsset(asset: MuxAsset, playbackId: string, options?: TranscriptFetchOptions): Promise<TranscriptResult>;
|
|
67
55
|
|
|
68
56
|
/**
|
|
@@ -104,17 +92,17 @@ interface ThumbnailOptions {
|
|
|
104
92
|
interval?: number;
|
|
105
93
|
/** Width of the thumbnail in pixels (default: 640) */
|
|
106
94
|
width?: number;
|
|
107
|
-
/**
|
|
108
|
-
|
|
95
|
+
/** Flag for whether or not to use signed playback IDs (default: false) */
|
|
96
|
+
shouldSign?: boolean;
|
|
109
97
|
}
|
|
110
98
|
/**
|
|
111
99
|
* Generates thumbnail URLs at regular intervals based on video duration.
|
|
112
|
-
* If
|
|
100
|
+
* If shouldSign is true, the URLs will be signed with tokens using credentials from environment variables.
|
|
113
101
|
*
|
|
114
102
|
* @param playbackId - The Mux playback ID
|
|
115
103
|
* @param duration - Video duration in seconds
|
|
116
104
|
* @param options - Thumbnail generation options
|
|
117
|
-
* @returns Array of thumbnail URLs (signed if
|
|
105
|
+
* @returns Array of thumbnail URLs (signed if shouldSign is true)
|
|
118
106
|
*/
|
|
119
107
|
declare function getThumbnailUrls(playbackId: string, duration: number, options?: ThumbnailOptions): Promise<string[]>;
|
|
120
108
|
|