@mux/ai 0.1.2 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +10 -18
- package/README.md +493 -171
- package/dist/index-BNnz9P_5.d.mts +144 -0
- package/dist/index-Bnv7tv90.d.ts +477 -0
- package/dist/index-DyTSka2R.d.ts +144 -0
- package/dist/index-vJ5r2FNm.d.mts +477 -0
- package/dist/index.d.mts +13 -0
- package/dist/index.d.ts +10 -161
- package/dist/index.js +1784 -1304
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +2205 -0
- package/dist/index.mjs.map +1 -0
- package/dist/primitives/index.d.mts +3 -0
- package/dist/primitives/index.d.ts +3 -0
- package/dist/primitives/index.js +409 -0
- package/dist/primitives/index.js.map +1 -0
- package/dist/primitives/index.mjs +358 -0
- package/dist/primitives/index.mjs.map +1 -0
- package/dist/types-ktXDZ93V.d.mts +137 -0
- package/dist/types-ktXDZ93V.d.ts +137 -0
- package/dist/workflows/index.d.mts +8 -0
- package/dist/workflows/index.d.ts +8 -0
- package/dist/workflows/index.js +2217 -0
- package/dist/workflows/index.js.map +1 -0
- package/dist/workflows/index.mjs +2168 -0
- package/dist/workflows/index.mjs.map +1 -0
- package/package.json +102 -30
- package/dist/index.cjs +0 -1773
- package/dist/index.cjs.map +0 -1
- package/dist/index.d.cts +0 -164
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
import { A as AssetTextTrack, b as MuxAsset, e as TextChunk, C as ChunkingStrategy } from './types-ktXDZ93V.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Context required to sign URLs for signed playback IDs.
|
|
5
|
+
*/
|
|
6
|
+
interface SigningContext {
|
|
7
|
+
/** The signing key ID from Mux dashboard. */
|
|
8
|
+
keyId: string;
|
|
9
|
+
/** The base64-encoded private key from Mux dashboard. */
|
|
10
|
+
keySecret: string;
|
|
11
|
+
/** Token expiration time (e.g. '1h', '1d'). Defaults to '1h'. */
|
|
12
|
+
expiration?: string;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
declare const DEFAULT_STORYBOARD_WIDTH = 640;
|
|
16
|
+
/**
|
|
17
|
+
* Generates a storyboard URL for the given playback ID.
|
|
18
|
+
* If a signing context is provided, the URL will be signed with a token.
|
|
19
|
+
*
|
|
20
|
+
* @param playbackId - The Mux playback ID
|
|
21
|
+
* @param width - Width of the storyboard in pixels (default: 640)
|
|
22
|
+
* @param signingContext - Optional signing context for signed playback IDs
|
|
23
|
+
* @returns Storyboard URL (signed if context provided)
|
|
24
|
+
*/
|
|
25
|
+
declare function getStoryboardUrl(playbackId: string, width?: number, signingContext?: SigningContext): Promise<string>;
|
|
26
|
+
|
|
27
|
+
/** A single cue from a VTT file with timing info. */
|
|
28
|
+
interface VTTCue {
|
|
29
|
+
startTime: number;
|
|
30
|
+
endTime: number;
|
|
31
|
+
text: string;
|
|
32
|
+
}
|
|
33
|
+
interface TranscriptFetchOptions {
|
|
34
|
+
languageCode?: string;
|
|
35
|
+
cleanTranscript?: boolean;
|
|
36
|
+
/** Optional signing context for signed playback IDs */
|
|
37
|
+
signingContext?: SigningContext;
|
|
38
|
+
}
|
|
39
|
+
interface TranscriptResult {
|
|
40
|
+
transcriptText: string;
|
|
41
|
+
transcriptUrl?: string;
|
|
42
|
+
track?: AssetTextTrack;
|
|
43
|
+
}
|
|
44
|
+
declare function getReadyTextTracks(asset: MuxAsset): AssetTextTrack[];
|
|
45
|
+
declare function findCaptionTrack(asset: MuxAsset, languageCode?: string): AssetTextTrack | undefined;
|
|
46
|
+
declare function extractTextFromVTT(vttContent: string): string;
|
|
47
|
+
declare function vttTimestampToSeconds(timestamp: string): number;
|
|
48
|
+
declare function extractTimestampedTranscript(vttContent: string): string;
|
|
49
|
+
/**
|
|
50
|
+
* Parses VTT content into structured cues with timing.
|
|
51
|
+
*
|
|
52
|
+
* @param vttContent - Raw VTT file content
|
|
53
|
+
* @returns Array of VTT cues with start/end times and text
|
|
54
|
+
*/
|
|
55
|
+
declare function parseVTTCues(vttContent: string): VTTCue[];
|
|
56
|
+
/**
|
|
57
|
+
* Builds a transcript URL for the given playback ID and track ID.
|
|
58
|
+
* If a signing context is provided, the URL will be signed with a token.
|
|
59
|
+
*
|
|
60
|
+
* @param playbackId - The Mux playback ID
|
|
61
|
+
* @param trackId - The text track ID
|
|
62
|
+
* @param signingContext - Optional signing context for signed playback IDs
|
|
63
|
+
* @returns Transcript URL (signed if context provided)
|
|
64
|
+
*/
|
|
65
|
+
declare function buildTranscriptUrl(playbackId: string, trackId: string, signingContext?: SigningContext): Promise<string>;
|
|
66
|
+
declare function fetchTranscriptForAsset(asset: MuxAsset, playbackId: string, options?: TranscriptFetchOptions): Promise<TranscriptResult>;
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Simple token counter that approximates tokens by word count.
|
|
70
|
+
* For production use with OpenAI, consider using a proper tokenizer like tiktoken.
|
|
71
|
+
* This approximation is generally close enough for chunking purposes (1 token ≈ 0.75 words).
|
|
72
|
+
*/
|
|
73
|
+
declare function estimateTokenCount(text: string): number;
|
|
74
|
+
/**
|
|
75
|
+
* Chunks text into overlapping segments based on token count.
|
|
76
|
+
*
|
|
77
|
+
* @param text - The text to chunk
|
|
78
|
+
* @param maxTokens - Maximum tokens per chunk
|
|
79
|
+
* @param overlapTokens - Number of tokens to overlap between chunks
|
|
80
|
+
* @returns Array of text chunks with metadata
|
|
81
|
+
*/
|
|
82
|
+
declare function chunkByTokens(text: string, maxTokens: number, overlapTokens?: number): TextChunk[];
|
|
83
|
+
/**
|
|
84
|
+
* Chunks VTT cues into groups that respect natural cue boundaries.
|
|
85
|
+
* Splits at cue boundaries rather than mid-sentence, preserving accurate timestamps.
|
|
86
|
+
*
|
|
87
|
+
* @param cues - Array of VTT cues to chunk
|
|
88
|
+
* @param maxTokens - Maximum tokens per chunk
|
|
89
|
+
* @param overlapCues - Number of cues to overlap between chunks (default: 2)
|
|
90
|
+
* @returns Array of text chunks with accurate start/end times
|
|
91
|
+
*/
|
|
92
|
+
declare function chunkVTTCues(cues: VTTCue[], maxTokens: number, overlapCues?: number): TextChunk[];
|
|
93
|
+
/**
|
|
94
|
+
* Chunks text according to the specified strategy.
|
|
95
|
+
*
|
|
96
|
+
* @param text - The text to chunk
|
|
97
|
+
* @param strategy - The chunking strategy to use
|
|
98
|
+
* @returns Array of text chunks
|
|
99
|
+
*/
|
|
100
|
+
declare function chunkText(text: string, strategy: ChunkingStrategy): TextChunk[];
|
|
101
|
+
|
|
102
|
+
interface ThumbnailOptions {
|
|
103
|
+
/** Interval between thumbnails in seconds (default: 10) */
|
|
104
|
+
interval?: number;
|
|
105
|
+
/** Width of the thumbnail in pixels (default: 640) */
|
|
106
|
+
width?: number;
|
|
107
|
+
/** Optional signing context for signed playback IDs */
|
|
108
|
+
signingContext?: SigningContext;
|
|
109
|
+
}
|
|
110
|
+
/**
|
|
111
|
+
* Generates thumbnail URLs at regular intervals based on video duration.
|
|
112
|
+
* If a signing context is provided, the URLs will be signed with tokens.
|
|
113
|
+
*
|
|
114
|
+
* @param playbackId - The Mux playback ID
|
|
115
|
+
* @param duration - Video duration in seconds
|
|
116
|
+
* @param options - Thumbnail generation options
|
|
117
|
+
* @returns Array of thumbnail URLs (signed if context provided)
|
|
118
|
+
*/
|
|
119
|
+
declare function getThumbnailUrls(playbackId: string, duration: number, options?: ThumbnailOptions): Promise<string[]>;
|
|
120
|
+
|
|
121
|
+
declare const index_DEFAULT_STORYBOARD_WIDTH: typeof DEFAULT_STORYBOARD_WIDTH;
|
|
122
|
+
type index_ThumbnailOptions = ThumbnailOptions;
|
|
123
|
+
type index_TranscriptFetchOptions = TranscriptFetchOptions;
|
|
124
|
+
type index_TranscriptResult = TranscriptResult;
|
|
125
|
+
type index_VTTCue = VTTCue;
|
|
126
|
+
declare const index_buildTranscriptUrl: typeof buildTranscriptUrl;
|
|
127
|
+
declare const index_chunkByTokens: typeof chunkByTokens;
|
|
128
|
+
declare const index_chunkText: typeof chunkText;
|
|
129
|
+
declare const index_chunkVTTCues: typeof chunkVTTCues;
|
|
130
|
+
declare const index_estimateTokenCount: typeof estimateTokenCount;
|
|
131
|
+
declare const index_extractTextFromVTT: typeof extractTextFromVTT;
|
|
132
|
+
declare const index_extractTimestampedTranscript: typeof extractTimestampedTranscript;
|
|
133
|
+
declare const index_fetchTranscriptForAsset: typeof fetchTranscriptForAsset;
|
|
134
|
+
declare const index_findCaptionTrack: typeof findCaptionTrack;
|
|
135
|
+
declare const index_getReadyTextTracks: typeof getReadyTextTracks;
|
|
136
|
+
declare const index_getStoryboardUrl: typeof getStoryboardUrl;
|
|
137
|
+
declare const index_getThumbnailUrls: typeof getThumbnailUrls;
|
|
138
|
+
declare const index_parseVTTCues: typeof parseVTTCues;
|
|
139
|
+
declare const index_vttTimestampToSeconds: typeof vttTimestampToSeconds;
|
|
140
|
+
declare namespace index {
|
|
141
|
+
export { index_DEFAULT_STORYBOARD_WIDTH as DEFAULT_STORYBOARD_WIDTH, type index_ThumbnailOptions as ThumbnailOptions, type index_TranscriptFetchOptions as TranscriptFetchOptions, type index_TranscriptResult as TranscriptResult, type index_VTTCue as VTTCue, index_buildTranscriptUrl as buildTranscriptUrl, index_chunkByTokens as chunkByTokens, index_chunkText as chunkText, index_chunkVTTCues as chunkVTTCues, index_estimateTokenCount as estimateTokenCount, index_extractTextFromVTT as extractTextFromVTT, index_extractTimestampedTranscript as extractTimestampedTranscript, index_fetchTranscriptForAsset as fetchTranscriptForAsset, index_findCaptionTrack as findCaptionTrack, index_getReadyTextTracks as getReadyTextTracks, index_getStoryboardUrl as getStoryboardUrl, index_getThumbnailUrls as getThumbnailUrls, index_parseVTTCues as parseVTTCues, index_vttTimestampToSeconds as vttTimestampToSeconds };
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
export { DEFAULT_STORYBOARD_WIDTH as D, type ThumbnailOptions as T, type VTTCue as V, chunkVTTCues as a, chunkText as b, chunkByTokens as c, getThumbnailUrls as d, estimateTokenCount as e, type TranscriptFetchOptions as f, getStoryboardUrl as g, type TranscriptResult as h, index as i, getReadyTextTracks as j, findCaptionTrack as k, extractTextFromVTT as l, extractTimestampedTranscript as m, buildTranscriptUrl as n, fetchTranscriptForAsset as o, parseVTTCues as p, vttTimestampToSeconds as v };
|
|
@@ -0,0 +1,477 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
import { createAnthropic } from '@ai-sdk/anthropic';
|
|
3
|
+
import { createGoogleGenerativeAI } from '@ai-sdk/google';
|
|
4
|
+
import { createOpenAI } from '@ai-sdk/openai';
|
|
5
|
+
import { h as TokenUsage, a as MuxAIOptions, I as ImageSubmissionMode, C as ChunkingStrategy, g as VideoEmbeddingsResult, T as ToneType } from './types-ktXDZ93V.mjs';
|
|
6
|
+
import { Buffer } from 'node:buffer';
|
|
7
|
+
|
|
8
|
+
interface ImageDownloadOptions {
|
|
9
|
+
/** Request timeout in milliseconds (default: 10000) */
|
|
10
|
+
timeout?: number;
|
|
11
|
+
/** Maximum number of retry attempts (default: 3) */
|
|
12
|
+
retries?: number;
|
|
13
|
+
/** Base delay between retries in milliseconds (default: 1000) */
|
|
14
|
+
retryDelay?: number;
|
|
15
|
+
/** Maximum delay between retries in milliseconds (default: 10000) */
|
|
16
|
+
maxRetryDelay?: number;
|
|
17
|
+
/** Whether to use exponential backoff (default: true) */
|
|
18
|
+
exponentialBackoff?: boolean;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* A single section of a prompt, rendered as an XML-like tag.
|
|
23
|
+
*/
|
|
24
|
+
interface PromptSection {
|
|
25
|
+
/** The XML tag name for this section */
|
|
26
|
+
tag: string;
|
|
27
|
+
/** The content inside the tag */
|
|
28
|
+
content: string;
|
|
29
|
+
/** Optional attributes to add to the tag (e.g., { format: "plain text" }) */
|
|
30
|
+
attributes?: Record<string, string>;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Configuration for building a prompt section.
|
|
34
|
+
* Can be a full PromptSection object, just a string (content only), or undefined to use default.
|
|
35
|
+
*/
|
|
36
|
+
type SectionOverride = string | PromptSection | undefined;
|
|
37
|
+
/**
|
|
38
|
+
* User-provided overrides for prompt sections.
|
|
39
|
+
* Each key can override the corresponding section's content or full definition.
|
|
40
|
+
*/
|
|
41
|
+
type PromptOverrides<TSections extends string> = Partial<Record<TSections, SectionOverride>>;
|
|
42
|
+
|
|
43
|
+
type SupportedProvider = "openai" | "anthropic" | "google";
|
|
44
|
+
type SupportedEmbeddingProvider = "openai" | "google";
|
|
45
|
+
type OpenAIModelId = Parameters<ReturnType<typeof createOpenAI>["chat"]>[0];
|
|
46
|
+
type AnthropicModelId = Parameters<ReturnType<typeof createAnthropic>["chat"]>[0];
|
|
47
|
+
type GoogleModelId = Parameters<ReturnType<typeof createGoogleGenerativeAI>["chat"]>[0];
|
|
48
|
+
type OpenAIEmbeddingModelId = Parameters<ReturnType<typeof createOpenAI>["embedding"]>[0];
|
|
49
|
+
type GoogleEmbeddingModelId = Parameters<ReturnType<typeof createGoogleGenerativeAI>["textEmbeddingModel"]>[0];
|
|
50
|
+
interface ModelIdByProvider {
|
|
51
|
+
openai: OpenAIModelId;
|
|
52
|
+
anthropic: AnthropicModelId;
|
|
53
|
+
google: GoogleModelId;
|
|
54
|
+
}
|
|
55
|
+
interface EmbeddingModelIdByProvider {
|
|
56
|
+
openai: OpenAIEmbeddingModelId;
|
|
57
|
+
google: GoogleEmbeddingModelId;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** Structured payload returned from `hasBurnedInCaptions`. */
|
|
61
|
+
interface BurnedInCaptionsResult {
|
|
62
|
+
assetId: string;
|
|
63
|
+
hasBurnedInCaptions: boolean;
|
|
64
|
+
confidence: number;
|
|
65
|
+
detectedLanguage: string | null;
|
|
66
|
+
storyboardUrl: string;
|
|
67
|
+
/** Token usage from the AI provider (for efficiency/cost analysis). */
|
|
68
|
+
usage?: TokenUsage;
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Sections of the burned-in captions user prompt that can be overridden.
|
|
72
|
+
* Use these to customize the AI's behavior for your specific use case.
|
|
73
|
+
*/
|
|
74
|
+
type BurnedInCaptionsPromptSections = "task" | "analysisSteps" | "positiveIndicators" | "negativeIndicators";
|
|
75
|
+
/**
|
|
76
|
+
* Override specific sections of the burned-in captions prompt.
|
|
77
|
+
* Each key corresponds to a section that can be customized.
|
|
78
|
+
*
|
|
79
|
+
* @example
|
|
80
|
+
* ```typescript
|
|
81
|
+
* const result = await hasBurnedInCaptions(assetId, {
|
|
82
|
+
* promptOverrides: {
|
|
83
|
+
* task: 'Detect any text overlays in the video frames.',
|
|
84
|
+
* positiveIndicators: 'Classify as captions if text appears consistently.',
|
|
85
|
+
* },
|
|
86
|
+
* });
|
|
87
|
+
* ```
|
|
88
|
+
*/
|
|
89
|
+
type BurnedInCaptionsPromptOverrides = PromptOverrides<BurnedInCaptionsPromptSections>;
|
|
90
|
+
/** Configuration accepted by `hasBurnedInCaptions`. */
|
|
91
|
+
interface BurnedInCaptionsOptions extends MuxAIOptions {
|
|
92
|
+
/** AI provider used for storyboard inspection (defaults to 'openai'). */
|
|
93
|
+
provider?: SupportedProvider;
|
|
94
|
+
/** Provider-specific model identifier. */
|
|
95
|
+
model?: ModelIdByProvider[SupportedProvider];
|
|
96
|
+
/** Transport used for storyboard submission (defaults to 'url'). */
|
|
97
|
+
imageSubmissionMode?: ImageSubmissionMode;
|
|
98
|
+
/** Download tuning used when `imageSubmissionMode` === 'base64'. */
|
|
99
|
+
imageDownloadOptions?: ImageDownloadOptions;
|
|
100
|
+
/**
|
|
101
|
+
* Override specific sections of the user prompt.
|
|
102
|
+
* Useful for customizing the AI's detection criteria for specific use cases.
|
|
103
|
+
*/
|
|
104
|
+
promptOverrides?: BurnedInCaptionsPromptOverrides;
|
|
105
|
+
}
|
|
106
|
+
/** Schema used to validate burned-in captions analysis responses. */
|
|
107
|
+
declare const burnedInCaptionsSchema: z.ZodObject<{
|
|
108
|
+
hasBurnedInCaptions: z.ZodBoolean;
|
|
109
|
+
confidence: z.ZodNumber;
|
|
110
|
+
detectedLanguage: z.ZodNullable<z.ZodString>;
|
|
111
|
+
}, "strip", z.ZodTypeAny, {
|
|
112
|
+
hasBurnedInCaptions: boolean;
|
|
113
|
+
confidence: number;
|
|
114
|
+
detectedLanguage: string | null;
|
|
115
|
+
}, {
|
|
116
|
+
hasBurnedInCaptions: boolean;
|
|
117
|
+
confidence: number;
|
|
118
|
+
detectedLanguage: string | null;
|
|
119
|
+
}>;
|
|
120
|
+
/** Inferred shape returned from the burned-in captions schema. */
|
|
121
|
+
type BurnedInCaptionsAnalysis = z.infer<typeof burnedInCaptionsSchema>;
|
|
122
|
+
declare function hasBurnedInCaptions(assetId: string, options?: BurnedInCaptionsOptions): Promise<BurnedInCaptionsResult>;
|
|
123
|
+
|
|
124
|
+
declare const chapterSchema: z.ZodObject<{
|
|
125
|
+
startTime: z.ZodNumber;
|
|
126
|
+
title: z.ZodString;
|
|
127
|
+
}, "strip", z.ZodTypeAny, {
|
|
128
|
+
startTime: number;
|
|
129
|
+
title: string;
|
|
130
|
+
}, {
|
|
131
|
+
startTime: number;
|
|
132
|
+
title: string;
|
|
133
|
+
}>;
|
|
134
|
+
type Chapter = z.infer<typeof chapterSchema>;
|
|
135
|
+
declare const chaptersSchema: z.ZodObject<{
|
|
136
|
+
chapters: z.ZodArray<z.ZodObject<{
|
|
137
|
+
startTime: z.ZodNumber;
|
|
138
|
+
title: z.ZodString;
|
|
139
|
+
}, "strip", z.ZodTypeAny, {
|
|
140
|
+
startTime: number;
|
|
141
|
+
title: string;
|
|
142
|
+
}, {
|
|
143
|
+
startTime: number;
|
|
144
|
+
title: string;
|
|
145
|
+
}>, "many">;
|
|
146
|
+
}, "strip", z.ZodTypeAny, {
|
|
147
|
+
chapters: {
|
|
148
|
+
startTime: number;
|
|
149
|
+
title: string;
|
|
150
|
+
}[];
|
|
151
|
+
}, {
|
|
152
|
+
chapters: {
|
|
153
|
+
startTime: number;
|
|
154
|
+
title: string;
|
|
155
|
+
}[];
|
|
156
|
+
}>;
|
|
157
|
+
type ChaptersType = z.infer<typeof chaptersSchema>;
|
|
158
|
+
/** Structured return payload from `generateChapters`. */
|
|
159
|
+
interface ChaptersResult {
|
|
160
|
+
assetId: string;
|
|
161
|
+
languageCode: string;
|
|
162
|
+
chapters: Chapter[];
|
|
163
|
+
}
|
|
164
|
+
/** Configuration accepted by `generateChapters`. */
|
|
165
|
+
interface ChaptersOptions extends MuxAIOptions {
|
|
166
|
+
/** AI provider used to interpret the transcript (defaults to 'openai'). */
|
|
167
|
+
provider?: SupportedProvider;
|
|
168
|
+
/** Provider-specific model identifier. */
|
|
169
|
+
model?: ModelIdByProvider[SupportedProvider];
|
|
170
|
+
}
|
|
171
|
+
declare function generateChapters(assetId: string, languageCode: string, options?: ChaptersOptions): Promise<ChaptersResult>;
|
|
172
|
+
|
|
173
|
+
/** Configuration accepted by `generateVideoEmbeddings`. */
|
|
174
|
+
interface EmbeddingsOptions extends MuxAIOptions {
|
|
175
|
+
/** AI provider used to generate embeddings (defaults to 'openai'). */
|
|
176
|
+
provider?: SupportedEmbeddingProvider;
|
|
177
|
+
/** Provider-specific model identifier (defaults to text-embedding-3-small for OpenAI). */
|
|
178
|
+
model?: EmbeddingModelIdByProvider[SupportedEmbeddingProvider];
|
|
179
|
+
/** Language code for transcript selection (defaults to first available). */
|
|
180
|
+
languageCode?: string;
|
|
181
|
+
/** Chunking strategy configuration (defaults to token-based with 500 tokens, 100 overlap). */
|
|
182
|
+
chunkingStrategy?: ChunkingStrategy;
|
|
183
|
+
/** Maximum number of chunks to process concurrently (defaults to 5). */
|
|
184
|
+
batchSize?: number;
|
|
185
|
+
}
|
|
186
|
+
/**
|
|
187
|
+
* Generates vector embeddings for a video asset's transcript.
|
|
188
|
+
*
|
|
189
|
+
* This function:
|
|
190
|
+
* 1. Fetches the video transcript from Mux
|
|
191
|
+
* 2. Chunks the transcript according to the specified strategy
|
|
192
|
+
* 3. Generates embeddings for each chunk using the specified AI provider
|
|
193
|
+
* 4. Returns both individual chunk embeddings and an averaged embedding
|
|
194
|
+
*
|
|
195
|
+
* @param assetId - Mux asset ID
|
|
196
|
+
* @param options - Configuration options
|
|
197
|
+
* @returns Video embeddings result with chunks and averaged embedding
|
|
198
|
+
*
|
|
199
|
+
* @example
|
|
200
|
+
* ```typescript
|
|
201
|
+
* const embeddings = await generateVideoEmbeddings("asset-id", {
|
|
202
|
+
* provider: "openai",
|
|
203
|
+
* chunkingStrategy: { type: "token", maxTokens: 500, overlap: 100 },
|
|
204
|
+
* });
|
|
205
|
+
*
|
|
206
|
+
* // Store in vector database
|
|
207
|
+
* for (const chunk of embeddings.chunks) {
|
|
208
|
+
* await db.insert({
|
|
209
|
+
* assetId: embeddings.assetId,
|
|
210
|
+
* chunkId: chunk.chunkId,
|
|
211
|
+
* embedding: chunk.embedding,
|
|
212
|
+
* metadata: chunk.metadata,
|
|
213
|
+
* });
|
|
214
|
+
* }
|
|
215
|
+
* ```
|
|
216
|
+
*/
|
|
217
|
+
declare function generateVideoEmbeddings(assetId: string, options?: EmbeddingsOptions): Promise<VideoEmbeddingsResult>;
|
|
218
|
+
|
|
219
|
+
/** Per-thumbnail moderation result returned from `getModerationScores`. */
|
|
220
|
+
interface ThumbnailModerationScore {
|
|
221
|
+
url: string;
|
|
222
|
+
sexual: number;
|
|
223
|
+
violence: number;
|
|
224
|
+
error: boolean;
|
|
225
|
+
}
|
|
226
|
+
/** Aggregated moderation payload returned from `getModerationScores`. */
|
|
227
|
+
interface ModerationResult {
|
|
228
|
+
assetId: string;
|
|
229
|
+
thumbnailScores: ThumbnailModerationScore[];
|
|
230
|
+
maxScores: {
|
|
231
|
+
sexual: number;
|
|
232
|
+
violence: number;
|
|
233
|
+
};
|
|
234
|
+
exceedsThreshold: boolean;
|
|
235
|
+
thresholds: {
|
|
236
|
+
sexual: number;
|
|
237
|
+
violence: number;
|
|
238
|
+
};
|
|
239
|
+
}
|
|
240
|
+
/** Provider list accepted by `getModerationScores`. */
|
|
241
|
+
type ModerationProvider = "openai" | "hive";
|
|
242
|
+
type HiveModerationSource = {
|
|
243
|
+
kind: "url";
|
|
244
|
+
value: string;
|
|
245
|
+
} | {
|
|
246
|
+
kind: "file";
|
|
247
|
+
buffer: Buffer;
|
|
248
|
+
contentType: string;
|
|
249
|
+
};
|
|
250
|
+
interface HiveModerationOutput {
|
|
251
|
+
classes?: Array<{
|
|
252
|
+
class: string;
|
|
253
|
+
score: number;
|
|
254
|
+
}>;
|
|
255
|
+
}
|
|
256
|
+
/** Configuration accepted by `getModerationScores`. */
|
|
257
|
+
interface ModerationOptions extends MuxAIOptions {
|
|
258
|
+
/** Provider used for moderation (defaults to 'openai'). */
|
|
259
|
+
provider?: ModerationProvider;
|
|
260
|
+
/** OpenAI moderation model identifier (defaults to 'omni-moderation-latest'). */
|
|
261
|
+
model?: string;
|
|
262
|
+
/** Override the default sexual/violence thresholds (0-1). */
|
|
263
|
+
thresholds?: {
|
|
264
|
+
sexual?: number;
|
|
265
|
+
violence?: number;
|
|
266
|
+
};
|
|
267
|
+
/** Interval between storyboard thumbnails in seconds (defaults to 10). */
|
|
268
|
+
thumbnailInterval?: number;
|
|
269
|
+
/** Width of storyboard thumbnails in pixels (defaults to 640). */
|
|
270
|
+
thumbnailWidth?: number;
|
|
271
|
+
/** Max concurrent moderation requests (defaults to 5). */
|
|
272
|
+
maxConcurrent?: number;
|
|
273
|
+
/** Transport used for thumbnails (defaults to 'url'). */
|
|
274
|
+
imageSubmissionMode?: ImageSubmissionMode;
|
|
275
|
+
/** Download tuning used when `imageSubmissionMode` === 'base64'. */
|
|
276
|
+
imageDownloadOptions?: ImageDownloadOptions;
|
|
277
|
+
}
|
|
278
|
+
/**
|
|
279
|
+
* Moderate a Mux asset's thumbnails.
|
|
280
|
+
* - provider 'openai' uses OpenAI's hosted moderation endpoint (requires OPENAI_API_KEY)
|
|
281
|
+
*/
|
|
282
|
+
declare function getModerationScores(assetId: string, options?: ModerationOptions): Promise<ModerationResult>;
|
|
283
|
+
|
|
284
|
+
declare const SUMMARY_KEYWORD_LIMIT = 10;
|
|
285
|
+
declare const summarySchema: z.ZodObject<{
|
|
286
|
+
keywords: z.ZodArray<z.ZodString, "many">;
|
|
287
|
+
title: z.ZodString;
|
|
288
|
+
description: z.ZodString;
|
|
289
|
+
}, "strip", z.ZodTypeAny, {
|
|
290
|
+
title: string;
|
|
291
|
+
keywords: string[];
|
|
292
|
+
description: string;
|
|
293
|
+
}, {
|
|
294
|
+
title: string;
|
|
295
|
+
keywords: string[];
|
|
296
|
+
description: string;
|
|
297
|
+
}>;
|
|
298
|
+
type SummaryType = z.infer<typeof summarySchema>;
|
|
299
|
+
/** Structured return payload for `getSummaryAndTags`. */
|
|
300
|
+
interface SummaryAndTagsResult {
|
|
301
|
+
/** Asset ID passed into the workflow. */
|
|
302
|
+
assetId: string;
|
|
303
|
+
/** Short headline generated from the storyboard. */
|
|
304
|
+
title: string;
|
|
305
|
+
/** Longer description of the detected content. */
|
|
306
|
+
description: string;
|
|
307
|
+
/** Up to 10 keywords extracted by the model. */
|
|
308
|
+
tags: string[];
|
|
309
|
+
/** Storyboard image URL that was analyzed. */
|
|
310
|
+
storyboardUrl: string;
|
|
311
|
+
}
|
|
312
|
+
/**
|
|
313
|
+
* Sections of the summarization user prompt that can be overridden.
|
|
314
|
+
* Use these to customize the AI's behavior for your specific use case.
|
|
315
|
+
*/
|
|
316
|
+
type SummarizationPromptSections = "task" | "title" | "description" | "keywords" | "qualityGuidelines";
|
|
317
|
+
/**
|
|
318
|
+
* Override specific sections of the summarization prompt.
|
|
319
|
+
* Each key corresponds to a section that can be customized.
|
|
320
|
+
*
|
|
321
|
+
* @example
|
|
322
|
+
* ```typescript
|
|
323
|
+
* const result = await getSummaryAndTags(assetId, {
|
|
324
|
+
* promptOverrides: {
|
|
325
|
+
* task: 'Generate SEO-optimized metadata for this product video.',
|
|
326
|
+
* title: 'Create a click-worthy title under 60 characters for YouTube.',
|
|
327
|
+
* },
|
|
328
|
+
* });
|
|
329
|
+
* ```
|
|
330
|
+
*/
|
|
331
|
+
type SummarizationPromptOverrides = PromptOverrides<SummarizationPromptSections>;
|
|
332
|
+
/** Configuration accepted by `getSummaryAndTags`. */
|
|
333
|
+
interface SummarizationOptions extends MuxAIOptions {
|
|
334
|
+
/** AI provider to run (defaults to 'openai'). */
|
|
335
|
+
provider?: SupportedProvider;
|
|
336
|
+
/** Provider-specific chat model identifier. */
|
|
337
|
+
model?: ModelIdByProvider[SupportedProvider];
|
|
338
|
+
/** Prompt tone shim applied to the system instruction (defaults to 'normal'). */
|
|
339
|
+
tone?: ToneType;
|
|
340
|
+
/** Fetch the transcript and send it alongside the storyboard (defaults to true). */
|
|
341
|
+
includeTranscript?: boolean;
|
|
342
|
+
/** Strip timestamps/markup from transcripts before including them (defaults to true). */
|
|
343
|
+
cleanTranscript?: boolean;
|
|
344
|
+
/** How storyboard frames should be delivered to the provider (defaults to 'url'). */
|
|
345
|
+
imageSubmissionMode?: ImageSubmissionMode;
|
|
346
|
+
/** Fine-tune storyboard downloads when `imageSubmissionMode` === 'base64'. */
|
|
347
|
+
imageDownloadOptions?: ImageDownloadOptions;
|
|
348
|
+
/**
|
|
349
|
+
* Override specific sections of the user prompt.
|
|
350
|
+
* Useful for customizing the AI's output for specific use cases (SEO, social media, etc.)
|
|
351
|
+
*/
|
|
352
|
+
promptOverrides?: SummarizationPromptOverrides;
|
|
353
|
+
}
|
|
354
|
+
declare function getSummaryAndTags(assetId: string, options?: SummarizationOptions): Promise<SummaryAndTagsResult>;
|
|
355
|
+
|
|
356
|
+
/** Output returned from `translateAudio`. */
|
|
357
|
+
interface AudioTranslationResult {
|
|
358
|
+
assetId: string;
|
|
359
|
+
targetLanguageCode: string;
|
|
360
|
+
dubbingId: string;
|
|
361
|
+
uploadedTrackId?: string;
|
|
362
|
+
presignedUrl?: string;
|
|
363
|
+
}
|
|
364
|
+
/** Configuration accepted by `translateAudio`. */
|
|
365
|
+
interface AudioTranslationOptions extends MuxAIOptions {
|
|
366
|
+
/** Audio dubbing provider (currently ElevenLabs only). */
|
|
367
|
+
provider?: "elevenlabs";
|
|
368
|
+
/** Number of speakers supplied to ElevenLabs (0 = auto-detect, default). */
|
|
369
|
+
numSpeakers?: number;
|
|
370
|
+
/** Optional override for the S3-compatible endpoint used for uploads. */
|
|
371
|
+
s3Endpoint?: string;
|
|
372
|
+
/** S3 region (defaults to env.S3_REGION or 'auto'). */
|
|
373
|
+
s3Region?: string;
|
|
374
|
+
/** Bucket that will store dubbed audio files. */
|
|
375
|
+
s3Bucket?: string;
|
|
376
|
+
/** Access key ID used for uploads. */
|
|
377
|
+
s3AccessKeyId?: string;
|
|
378
|
+
/** Secret access key used for uploads. */
|
|
379
|
+
s3SecretAccessKey?: string;
|
|
380
|
+
/**
|
|
381
|
+
* When true (default) the dubbed audio file is uploaded to the configured
|
|
382
|
+
* bucket and attached to the Mux asset.
|
|
383
|
+
*/
|
|
384
|
+
uploadToMux?: boolean;
|
|
385
|
+
/** Override for env.ELEVENLABS_API_KEY. */
|
|
386
|
+
elevenLabsApiKey?: string;
|
|
387
|
+
}
|
|
388
|
+
declare function translateAudio(assetId: string, toLanguageCode: string, options?: AudioTranslationOptions): Promise<AudioTranslationResult>;
|
|
389
|
+
|
|
390
|
+
/** Output returned from `translateCaptions`. */
|
|
391
|
+
interface TranslationResult {
|
|
392
|
+
assetId: string;
|
|
393
|
+
sourceLanguageCode: string;
|
|
394
|
+
targetLanguageCode: string;
|
|
395
|
+
originalVtt: string;
|
|
396
|
+
translatedVtt: string;
|
|
397
|
+
uploadedTrackId?: string;
|
|
398
|
+
presignedUrl?: string;
|
|
399
|
+
}
|
|
400
|
+
/** Configuration accepted by `translateCaptions`. */
|
|
401
|
+
interface TranslationOptions<P extends SupportedProvider = SupportedProvider> extends MuxAIOptions {
|
|
402
|
+
/** Provider responsible for the translation. */
|
|
403
|
+
provider: P;
|
|
404
|
+
/** Provider-specific chat model identifier. */
|
|
405
|
+
model?: ModelIdByProvider[P];
|
|
406
|
+
/** Optional override for the S3-compatible endpoint used for uploads. */
|
|
407
|
+
s3Endpoint?: string;
|
|
408
|
+
/** S3 region (defaults to env.S3_REGION or 'auto'). */
|
|
409
|
+
s3Region?: string;
|
|
410
|
+
/** Bucket that will store translated VTT files. */
|
|
411
|
+
s3Bucket?: string;
|
|
412
|
+
/** Access key ID used for uploads. */
|
|
413
|
+
s3AccessKeyId?: string;
|
|
414
|
+
/** Secret access key used for uploads. */
|
|
415
|
+
s3SecretAccessKey?: string;
|
|
416
|
+
/**
|
|
417
|
+
* When true (default) the translated VTT is uploaded to the configured
|
|
418
|
+
* bucket and attached to the Mux asset.
|
|
419
|
+
*/
|
|
420
|
+
uploadToMux?: boolean;
|
|
421
|
+
}
|
|
422
|
+
/** Schema used when requesting caption translation from a language model. */
|
|
423
|
+
declare const translationSchema: z.ZodObject<{
|
|
424
|
+
translation: z.ZodString;
|
|
425
|
+
}, "strip", z.ZodTypeAny, {
|
|
426
|
+
translation: string;
|
|
427
|
+
}, {
|
|
428
|
+
translation: string;
|
|
429
|
+
}>;
|
|
430
|
+
/** Inferred shape returned by `translationSchema`. */
|
|
431
|
+
type TranslationPayload = z.infer<typeof translationSchema>;
|
|
432
|
+
declare function translateCaptions<P extends SupportedProvider = SupportedProvider>(assetId: string, fromLanguageCode: string, toLanguageCode: string, options: TranslationOptions<P>): Promise<TranslationResult>;
|
|
433
|
+
|
|
434
|
+
type index_AudioTranslationOptions = AudioTranslationOptions;
|
|
435
|
+
type index_AudioTranslationResult = AudioTranslationResult;
|
|
436
|
+
type index_BurnedInCaptionsAnalysis = BurnedInCaptionsAnalysis;
|
|
437
|
+
type index_BurnedInCaptionsOptions = BurnedInCaptionsOptions;
|
|
438
|
+
type index_BurnedInCaptionsPromptOverrides = BurnedInCaptionsPromptOverrides;
|
|
439
|
+
type index_BurnedInCaptionsPromptSections = BurnedInCaptionsPromptSections;
|
|
440
|
+
type index_BurnedInCaptionsResult = BurnedInCaptionsResult;
|
|
441
|
+
type index_Chapter = Chapter;
|
|
442
|
+
type index_ChaptersOptions = ChaptersOptions;
|
|
443
|
+
type index_ChaptersResult = ChaptersResult;
|
|
444
|
+
type index_ChaptersType = ChaptersType;
|
|
445
|
+
type index_EmbeddingsOptions = EmbeddingsOptions;
|
|
446
|
+
type index_HiveModerationOutput = HiveModerationOutput;
|
|
447
|
+
type index_HiveModerationSource = HiveModerationSource;
|
|
448
|
+
type index_ModerationOptions = ModerationOptions;
|
|
449
|
+
type index_ModerationProvider = ModerationProvider;
|
|
450
|
+
type index_ModerationResult = ModerationResult;
|
|
451
|
+
declare const index_SUMMARY_KEYWORD_LIMIT: typeof SUMMARY_KEYWORD_LIMIT;
|
|
452
|
+
type index_SummarizationOptions = SummarizationOptions;
|
|
453
|
+
type index_SummarizationPromptOverrides = SummarizationPromptOverrides;
|
|
454
|
+
type index_SummarizationPromptSections = SummarizationPromptSections;
|
|
455
|
+
type index_SummaryAndTagsResult = SummaryAndTagsResult;
|
|
456
|
+
type index_SummaryType = SummaryType;
|
|
457
|
+
type index_ThumbnailModerationScore = ThumbnailModerationScore;
|
|
458
|
+
type index_TranslationOptions<P extends SupportedProvider = SupportedProvider> = TranslationOptions<P>;
|
|
459
|
+
type index_TranslationPayload = TranslationPayload;
|
|
460
|
+
type index_TranslationResult = TranslationResult;
|
|
461
|
+
declare const index_burnedInCaptionsSchema: typeof burnedInCaptionsSchema;
|
|
462
|
+
declare const index_chapterSchema: typeof chapterSchema;
|
|
463
|
+
declare const index_chaptersSchema: typeof chaptersSchema;
|
|
464
|
+
declare const index_generateChapters: typeof generateChapters;
|
|
465
|
+
declare const index_generateVideoEmbeddings: typeof generateVideoEmbeddings;
|
|
466
|
+
declare const index_getModerationScores: typeof getModerationScores;
|
|
467
|
+
declare const index_getSummaryAndTags: typeof getSummaryAndTags;
|
|
468
|
+
declare const index_hasBurnedInCaptions: typeof hasBurnedInCaptions;
|
|
469
|
+
declare const index_summarySchema: typeof summarySchema;
|
|
470
|
+
declare const index_translateAudio: typeof translateAudio;
|
|
471
|
+
declare const index_translateCaptions: typeof translateCaptions;
|
|
472
|
+
declare const index_translationSchema: typeof translationSchema;
|
|
473
|
+
declare namespace index {
|
|
474
|
+
export { type index_AudioTranslationOptions as AudioTranslationOptions, type index_AudioTranslationResult as AudioTranslationResult, type index_BurnedInCaptionsAnalysis as BurnedInCaptionsAnalysis, type index_BurnedInCaptionsOptions as BurnedInCaptionsOptions, type index_BurnedInCaptionsPromptOverrides as BurnedInCaptionsPromptOverrides, type index_BurnedInCaptionsPromptSections as BurnedInCaptionsPromptSections, type index_BurnedInCaptionsResult as BurnedInCaptionsResult, type index_Chapter as Chapter, type index_ChaptersOptions as ChaptersOptions, type index_ChaptersResult as ChaptersResult, type index_ChaptersType as ChaptersType, type index_EmbeddingsOptions as EmbeddingsOptions, type index_HiveModerationOutput as HiveModerationOutput, type index_HiveModerationSource as HiveModerationSource, type index_ModerationOptions as ModerationOptions, type index_ModerationProvider as ModerationProvider, type index_ModerationResult as ModerationResult, index_SUMMARY_KEYWORD_LIMIT as SUMMARY_KEYWORD_LIMIT, type index_SummarizationOptions as SummarizationOptions, type index_SummarizationPromptOverrides as SummarizationPromptOverrides, type index_SummarizationPromptSections as SummarizationPromptSections, type index_SummaryAndTagsResult as SummaryAndTagsResult, type index_SummaryType as SummaryType, type index_ThumbnailModerationScore as ThumbnailModerationScore, type index_TranslationOptions as TranslationOptions, type index_TranslationPayload as TranslationPayload, type index_TranslationResult as TranslationResult, index_burnedInCaptionsSchema as burnedInCaptionsSchema, index_chapterSchema as chapterSchema, index_chaptersSchema as chaptersSchema, index_generateChapters as generateChapters, index_generateVideoEmbeddings as generateVideoEmbeddings, index_getModerationScores as getModerationScores, index_getSummaryAndTags as getSummaryAndTags, index_hasBurnedInCaptions as hasBurnedInCaptions, index_summarySchema as summarySchema, index_translateAudio as translateAudio, index_translateCaptions as translateCaptions, index_translationSchema as translationSchema };
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
export { type AudioTranslationResult as A, type BurnedInCaptionsResult as B, type Chapter as C, translateAudio as D, type EmbeddingsOptions as E, type TranslationResult as F, type TranslationOptions as G, type HiveModerationSource as H, translationSchema as I, type TranslationPayload as J, translateCaptions as K, type ModerationResult as M, SUMMARY_KEYWORD_LIMIT as S, type ThumbnailModerationScore as T, type BurnedInCaptionsPromptSections as a, type BurnedInCaptionsPromptOverrides as b, type BurnedInCaptionsOptions as c, burnedInCaptionsSchema as d, type BurnedInCaptionsAnalysis as e, chapterSchema as f, chaptersSchema as g, hasBurnedInCaptions as h, index as i, type ChaptersType as j, type ChaptersResult as k, type ChaptersOptions as l, generateChapters as m, generateVideoEmbeddings as n, type ModerationProvider as o, type HiveModerationOutput as p, type ModerationOptions as q, getModerationScores as r, summarySchema as s, type SummaryType as t, type SummaryAndTagsResult as u, type SummarizationPromptSections as v, type SummarizationPromptOverrides as w, type SummarizationOptions as x, getSummaryAndTags as y, type AudioTranslationOptions as z };
|
package/dist/index.d.mts
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
export { i as primitives } from './index-BNnz9P_5.mjs';
|
|
2
|
+
export { A as AssetTextTrack, f as ChunkEmbedding, C as ChunkingStrategy, I as ImageSubmissionMode, M as MuxAIConfig, a as MuxAIOptions, b as MuxAsset, c as PlaybackAsset, P as PlaybackPolicy, e as TextChunk, d as TokenChunkingConfig, h as TokenUsage, T as ToneType, V as VTTChunkingConfig, g as VideoEmbeddingsResult } from './types-ktXDZ93V.mjs';
|
|
3
|
+
export { i as workflows } from './index-vJ5r2FNm.mjs';
|
|
4
|
+
import '@mux/mux-node';
|
|
5
|
+
import 'zod';
|
|
6
|
+
import '@ai-sdk/anthropic';
|
|
7
|
+
import '@ai-sdk/google';
|
|
8
|
+
import '@ai-sdk/openai';
|
|
9
|
+
import 'node:buffer';
|
|
10
|
+
|
|
11
|
+
declare const version = "0.1.0";
|
|
12
|
+
|
|
13
|
+
export { version };
|