nexus-ai-pro 2.20.2 → 2.22.0

package/dist/embeddings/batch-google.d.ts

@@ -4,7 +4,10 @@ import type { EmbeddingBatchOutputItem, EmbeddingBatchProvider, EmbeddingBatchPr
 export interface GoogleEmbeddingBatchProviderOptions {
     /** Gemini API key, sent as `x-goog-api-key`. */
     apiKey: string;
-    /** API base URL. Defaults to `https://generativelanguage.googleapis.com/v1beta`. */
+    /**
+     * API base URL. Defaults to `https://generativelanguage.googleapis.com/v1beta`. A file is uploaded to,
+     * and results downloaded from, the same URL with `/upload` and `/download` before its version.
+     */
     baseUrl?: string;
     /** Headers added to every request. */
     headers?: Record<string, string>;
@@ -12,10 +15,11 @@ export interface GoogleEmbeddingBatchProviderOptions {
     fetch?: typeof fetch;
 }
 /**
- * Gemini's Batch API for embeddings (`asyncBatchEmbedContent`): half price, inline requests under
- * 20 MB, and results inline, in the order submitted. It takes text, images, audio, and PDFs on a model
- * that does, such as gemini-embedding-2. The wire shape follows Google's API reference and its own
- * clients; it graduates with a recorded live run.
+ * Gemini's Batch API for embeddings (`asyncBatchEmbedContent`): half price. Requests under 20 MB go
+ * inline; a larger batch, up to 2 GB, is uploaded as a JSONL file through the Files API, a line per
+ * input with its id as the key, and its results are read from the file the batch writes. It takes
+ * text, images, audio, video, and PDFs on a model that does, such as gemini-embedding-2. The wire shape
+ * follows Google's API reference and its own clients; it graduates with a recorded live run.
  */
 export declare class GoogleEmbeddingBatchProvider implements EmbeddingBatchProvider {
     private readonly options;
@@ -23,18 +27,33 @@ export declare class GoogleEmbeddingBatchProvider implements EmbeddingBatchProvi
     readonly info: EmbeddingBatchProviderInfo;
     constructor(options: GoogleEmbeddingBatchProviderOptions);
     /**
-     * Sends every input inline as one batch. Results come back in the same order, so the ref keeps the
-     * ids in that order.
+     * Sends every input as one batch: inline under 20 MB, or else as a JSONL file uploaded first. Results
+     * come back in the same order, so the ref keeps the ids in that order.
      */
     submit(request: EmbeddingBatchProviderRequest, context: BatchProviderCallContext): Promise<BatchJobRef>;
     /** Reads a batch's state and counts. */
     poll(ref: BatchJobRef, context: BatchProviderCallContext): Promise<BatchJobState>;
-    /** Every input's vector or error, matched to its id by position. */
+    /**
+     * Every input's vector or error: inline, or read from the file the batch wrote. An answer that names
+     * its key is matched by it, and one that does not, by its position.
+     */
     results(ref: BatchJobRef, context: BatchProviderCallContext): Promise<EmbeddingBatchOutputItem[]>;
     /** Cancels a batch. */
     cancel(ref: BatchJobRef, context: BatchProviderCallContext): Promise<BatchJobState>;
+    /**
+     * Uploads an input file through the Files API's resumable protocol, a start and then the bytes, and
+     * returns its name, such as `files/abc`. Google keeps it 48 hours.
+     */
+    private upload;
+    /** A results file's text, from the Files API's download URL. */
+    private download;
+    /** The URL beside the API's own for uploads or downloads: `/upload` or `/download` before the version. */
+    private alongside;
     /** The batch resource, whether Google answers with it or with an operation that holds it. */
     private read;
     private state;
     private request;
+    private base;
+    /** A call with the key, refused unless it succeeds. */
+    private send;
 }

package/dist/embeddings/batch-google.js

@@ -1,6 +1,7 @@
 import { BatchCapabilityError, BatchProviderResponseError } from '../batch/errors.js';
 import { GOOGLE_TASK_TYPES, googleParts, usesTaskPrefix } from './google-wire.js';
 const INLINE_BYTES = 20 * 1024 * 1024;
+const FILE_BYTES = 2 * 1024 * 1024 * 1024;
 const STATES = {
     PENDING: 'validating',
     RUNNING: 'in_progress',
@@ -14,10 +15,10 @@ export class GoogleEmbeddingBatchProvider {
     info = {
         name: 'google',
         capabilities: {
-            maxBytes: INLINE_BYTES,
+            maxBytes: FILE_BYTES,
             supportsCancel: true,
             discount: 0.5,
-            inputModalities: ['text', 'image', 'audio', 'document'],
+            inputModalities: ['text', 'image', 'audio', 'video', 'document'],
         },
     };
     constructor(options) {
@@ -26,33 +27,44 @@ export class GoogleEmbeddingBatchProvider {
     async submit(request, context) {
         const qualified = request.model.startsWith('models/') ? request.model : `models/${request.model}`;
         const prefixed = usesTaskPrefix(request.model);
-        const body = JSON.stringify({
-            batch: {
-                displayName: request.metadata?.key ?? 'nexus-embeddings',
-                inputConfig: {
-                    requests: {
-                        requests: request.items.map((item) => ({
-                            request: {
-                                model: qualified,
-                                content: {
-                                    parts: googleParts(item.contents ?? [{ type: 'text', text: item.input }], prefixed ? request.inputType : undefined),
-                                },
-                                ...(request.inputType && !prefixed ? { taskType: GOOGLE_TASK_TYPES[request.inputType] } : {}),
-                                ...(request.dimensions !== undefined ? { outputDimensionality: request.dimensions } : {}),
-                            },
-                        })),
-                    },
-                },
+        const displayName = request.metadata?.key ?? 'nexus-embeddings';
+        const encoded = request.items.map((item) => JSON.stringify({
+            model: qualified,
+            content: {
+                parts: googleParts(item.contents ?? [{ type: 'text', text: item.input }], prefixed ? request.inputType : undefined),
             },
-        });
-        if (body.length > INLINE_BYTES) {
-            throw new BatchCapabilityError('google', 'maxBytes', `An inline Gemini batch holds at most 20 MB; this one is ${Math.ceil(body.length / 1024 / 1024)} MB, so split it`);
+            ...(request.inputType && !prefixed ? { taskType: GOOGLE_TASK_TYPES[request.inputType] } : {}),
+            ...(request.dimensions !== undefined ? { outputDimensionality: request.dimensions } : {}),
+        }));
+        const opening = `{"batch":{"displayName":${JSON.stringify(displayName)},"inputConfig":{"requests":{"requests":[`;
+        const closing = ']}}}}';
+        const inlineSize = utf8Length(opening) +
+            encoded.reduce((total, line) => total + utf8Length(line) + '{"request":},'.length, 0) +
+            closing.length;
+        let body;
+        let inputFile;
+        if (inlineSize <= INLINE_BYTES) {
+            body = `${opening}${encoded.map((line) => `{"request":${line}}`).join(',')}${closing}`;
+        }
+        else {
+            const lines = request.items.map((item, index) => `{"key":${JSON.stringify(item.customId)},"request":${encoded[index]}}\n`);
+            const size = lines.reduce((total, line) => total + utf8Length(line), 0);
+            const most = this.info.capabilities.maxBytes ?? FILE_BYTES;
+            if (size > most) {
+                throw new BatchCapabilityError('google', 'maxBytes', `A Gemini batch file holds at most ${Math.floor(most / 1024 / 1024)} MB; this one is ${Math.ceil(size / 1024 / 1024)} MB, so split it`);
+            }
+            inputFile = await this.upload(new Blob(lines, { type: 'application/jsonl' }), displayName, context);
+            body = JSON.stringify({ batch: { displayName, inputConfig: { fileName: inputFile } } });
         }
         const created = (await this.request(`/${qualified}:asyncBatchEmbedContent`, { method: 'POST', body }, context));
         const name = created.name ?? created.metadata?.name;
         if (!name)
             throw new BatchProviderResponseError('google', 'the batch response returned no name');
-        return { id: name, provider: 'google', metadata: { customIds: request.items.map((item) => item.customId) } };
+        return {
+            id: name,
+            provider: 'google',
+            metadata: { customIds: request.items.map((item) => item.customId), ...(inputFile ? { inputFile } : {}) },
+        };
     }
     async poll(ref, context) {
         return this.state(ref, await this.read(ref, context));
@@ -60,14 +72,18 @@ export class GoogleEmbeddingBatchProvider {
     async results(ref, context) {
         const batch = await this.read(ref, context);
         const output = batch.output;
-        if (output?.responsesFile) {
-            throw new BatchProviderResponseError('google', 'the batch wrote its results to a file; this adapter reads inline results');
-        }
         const ids = (ref.metadata?.customIds ?? []);
-        const responses = output?.inlinedEmbedContentResponses?.inlinedResponses ?? [];
+        const responses = output?.responsesFile
+            ? (await this.download(output.responsesFile, context))
+                .split('\n')
+                .filter((line) => line.trim() !== '')
+                .map((line) => JSON.parse(line))
+            : (output?.inlinedResponses?.inlinedResponses ?? output?.inlinedEmbedContentResponses?.inlinedResponses ?? []);
+        const known = new Set(ids);
         return responses.map((entry, index) => {
-            const customId = ids[index] ?? String(index);
-            const values = entry.response?.embedding?.values;
+            const key = entry.key ?? entry.metadata?.key;
+            const customId = key !== undefined && known.has(key) ? key : (ids[index] ?? String(index));
+            const values = (entry.response ?? entry).embedding?.values;
             if (Array.isArray(values))
                 return { customId, vector: values };
             return {
@@ -83,6 +99,43 @@ export class GoogleEmbeddingBatchProvider {
         await this.request(`/${ref.id}:cancel`, { method: 'POST' }, context);
         return this.poll(ref, context);
     }
+    async upload(file, displayName, context) {
+        const started = await this.send(this.alongside('upload', '/files'), {
+            method: 'POST',
+            body: JSON.stringify({ file: { displayName } }),
+            headers: {
+                'content-type': 'application/json',
+                'x-goog-upload-protocol': 'resumable',
+                'x-goog-upload-command': 'start',
+                'x-goog-upload-header-content-length': String(file.size),
+                'x-goog-upload-header-content-type': 'application/jsonl',
+            },
+        }, context);
+        const target = started.headers.get('x-goog-upload-url');
+        if (!target)
+            throw new BatchProviderResponseError('google', 'the upload was not given a URL to send the file to');
+        const finished = await this.send(target, {
+            method: 'POST',
+            body: file,
+            headers: { 'x-goog-upload-offset': '0', 'x-goog-upload-command': 'upload, finalize' },
+        }, context);
+        const name = JSON.parse(await finished.text()).file?.name;
+        if (!name)
+            throw new BatchProviderResponseError('google', 'the upload returned no file name');
+        return name;
+    }
+    async download(file, context) {
+        const response = await this.send(this.alongside('download', `/${file}:download?alt=media`), {}, context);
+        return response.text();
+    }
+    alongside(kind, path) {
+        const base = this.base();
+        const moved = base.replace(/\/(v1[^/]*)$/, `/${kind}/$1`);
+        if (moved === base) {
+            throw new BatchProviderResponseError('google', `the base URL ${base} names no API version to ${kind} beside`);
+        }
+        return `${moved}${path}`;
+    }
     async read(ref, context) {
         const answer = (await this.request(`/${ref.id}`, {}, context));
         return { ...answer, ...answer.metadata };
@@ -112,23 +165,46 @@ export class GoogleEmbeddingBatchProvider {
         };
     }
     async request(path, init, context) {
-        const fetchImpl = this.options.fetch ?? fetch;
-        const base = (this.options.baseUrl ?? 'https://generativelanguage.googleapis.com/v1beta').replace(/\/+$/, '');
+        const response = await this.send(`${this.base()}${path}`, { ...init, headers: { 'content-type': 'application/json' } }, context);
+        const text = await response.text();
+        return text ? JSON.parse(text) : {};
+    }
+    base() {
+        return (this.options.baseUrl ?? 'https://generativelanguage.googleapis.com/v1beta').replace(/\/+$/, '');
+    }
+    async send(url, init, context) {
         const credential = await context.credential?.(this.info.name);
-        const response = await fetchImpl(`${base}${path}`, {
+        const response = await (this.options.fetch ?? fetch)(url, {
             ...init,
             headers: {
-                'content-type': 'application/json',
+                ...init.headers,
                 'x-goog-api-key': credential?.apiKey ?? this.options.apiKey,
                 ...this.options.headers,
                 ...credential?.headers,
             },
             signal: context.signal,
         });
-        const text = await response.text();
         if (!response.ok) {
-            throw new BatchProviderResponseError('google', `${path} failed: ${response.status} ${text.slice(0, 500)}`);
+            const path = new URL(url).pathname;
+            throw new BatchProviderResponseError('google', `${path} failed: ${response.status} ${(await response.text()).slice(0, 500)}`);
         }
-        return text ? JSON.parse(text) : {};
+        return response;
+    }
+}
+function utf8Length(text) {
+    let bytes = 0;
+    for (let index = 0; index < text.length; index += 1) {
+        const code = text.charCodeAt(index);
+        if (code < 0x80)
+            bytes += 1;
+        else if (code < 0x800)
+            bytes += 2;
+        else if (code >= 0xd800 && code <= 0xdbff) {
+            bytes += 4;
+            index += 1;
+        }
+        else
+            bytes += 3;
     }
+    return bytes;
 }

package/dist/embeddings/batch-mock.js

@@ -12,7 +12,7 @@ export class MockEmbeddingBatchProvider {
                 maxItems: 50_000,
                 supportsCancel: true,
                 discount: 0.5,
-                inputModalities: ['text', 'image', 'audio', 'document'],
+                inputModalities: ['text', 'image', 'audio', 'video', 'document'],
                 ...options.capabilities,
             },
         };

package/dist/embeddings/batch.js

@@ -3,7 +3,7 @@ import { pollUntilSettled } from '../batch/poll.js';
 import { continuing } from '../lifecycle/continue.js';
 import { generateRequestId } from '../utils/ids.js';
 import { Tokenizer } from '../utils/tokenizer.js';
-import { acceptedModalities, normalizeInputs } from './inputs.js';
+import { acceptedModalities, assertLengths, mediaCost, normalizeInputs, totalUnits, } from './inputs.js';
 import { priceEmbeddingUsage, resolveEmbeddingModel } from './models.js';
 const DEFAULT_POLL_INTERVAL_MS = 30_000;
 const DEFAULT_MAX_POLL_INTERVAL_MS = 300_000;
@@ -150,6 +150,12 @@ export class EmbeddingBatchManager {
         if (refused.length) {
             throw new BatchCapabilityError(providerName, 'input', `Embedding model "${request.model}" in the "${providerName}" batch tier takes ${accepted.join(', ')} input, not ${refused.join(' or ')}`);
         }
+        try {
+            assertLengths(inputs, request.model, model);
+        }
+        catch (error) {
+            throw new BatchCapabilityError(providerName, 'input', error.message);
+        }
         const parts = inputs.parts;
         let estimatedInputTokens = 0;
         for (const text of inputs.texts)
@@ -170,7 +176,7 @@ export class EmbeddingBatchManager {
             notes: {
                 model: request.model,
                 estimatedInputTokens,
-                images: inputs.images.reduce((total, count) => total + count, 0),
+                media: totalUnits(inputs.units),
                 items: request.items.length,
                 ...(request.normalize ? { normalize: true } : {}),
                 ...(request.tenantId !== undefined ? { tenant: request.tenantId } : {}),
@@ -228,8 +234,8 @@ export class EmbeddingBatchManager {
         if (!notes.model)
             return { amount: 0, currency: 'USD', basis: 'estimated' };
         const cost = priceEmbeddingUsage(notes.model, usage, this.config.models);
-        const perImage = resolveEmbeddingModel(notes.model, this.config.models).capabilities?.costPerImage ?? 0;
-        const amount = (cost.amount + (notes.images ?? 0) * perImage) * discount;
+        const media = notes.media ?? { images: notes.images ?? 0, audio: [], video: [], unmeasured: 0 };
+        const amount = (cost.amount + mediaCost(media, resolveEmbeddingModel(notes.model, this.config.models).capabilities)) * discount;
         return { ...cost, amount, input: amount, output: 0 };
     }
     resolveProvider(preferred) {

package/dist/embeddings/engine.js

@@ -1,7 +1,7 @@
 import { MemoryCache, createCacheKey } from '../cache/memory-cache.js';
 import { RateLimiter } from '../ops/rate-limiter.js';
 import { OperationLifecycle } from '../core/lifecycle.js';
-import { acceptedModalities, inputIdentities, normalizeInputs } from './inputs.js';
+import { acceptedModalities, assertLengths, inputIdentities, mediaCost, normalizeInputs, pricesByLength, totalUnits, } from './inputs.js';
 import { NexusProviderError, createTimeoutProviderError, isAbortError } from '../providers/errors.js';
 import { assertWithinCostBudget, formatCost } from '../optimizer/cost-budget.js';
 import { Tokenizer } from '../utils/tokenizer.js';
@@ -55,6 +55,7 @@ export class EmbeddingEngine {
                 route = this.resolveRoute(request);
                 assertRequestedOptions(route, request);
                 assertModalities(route.providerName, route.resolved.model, route.resolved.capabilities, route.provider, inputs);
+                assertLengths(inputs, route.resolved.model, route.resolved.capabilities);
                 estimate = this.enforceCostBudget(inputs, route.resolved.model);
                 return { provider: route.providerName, model: route.resolved.model };
             },
@@ -119,7 +120,7 @@ export class EmbeddingEngine {
         let truncated = false;
         let providerUsed = route.providerName;
         let modelUsed = model;
-        let images = 0;
+        const embedded = [];
         if (unique.length > 0) {
             const batchSize = resolveMaxBatchSize(route.resolved.capabilities, route.provider);
             const chunks = chunkInputs(unique, batchSize);
@@ -146,7 +147,7 @@ export class EmbeddingEngine {
                 usage = estimateUsage(unique.map((index) => inputs.texts[index]), this.tokenizer);
             }
             for (const index of unique)
-                images += inputs.images[index] ?? 0;
+                embedded.push(inputs.units[index]);
             const finished = flatVectors.map((values) => this.postProcess(values, route, request));
             for (let position = 0; position < pending.length; position += 1) {
                 const index = pending[position];
@@ -172,6 +173,7 @@ export class EmbeddingEngine {
                 truncated: truncated || undefined,
             };
         });
+        const media = totalUnits(embedded);
         const meta = {
             requestId,
             providerUsed,
@@ -181,7 +183,8 @@ export class EmbeddingEngine {
             latencyMs: Date.now() - startedAt,
             batches,
             usage,
-            cost: withImageCost(priceEmbeddingUsage(model, usage, this.config.models), images, route.resolved.capabilities),
+            cost: withMediaCost(priceEmbeddingUsage(model, usage, this.config.models), media, route.resolved.capabilities),
+            ...(media.unmeasured && pricesByLength(route.resolved.capabilities) ? { unpricedMedia: media.unmeasured } : {}),
             cacheHit: cachedInputs === count && count > 0,
             cachedInputs: cachedInputs || undefined,
             deduplicatedInputs: deduplicatedInputs || undefined,
@@ -407,11 +410,10 @@ export class EmbeddingEngine {
         for (const text of inputs.texts)
             inputTokens += this.tokenizer.estimateTextTokens(text);
         const estimate = estimateEmbeddingCost({ model, inputTokens, config: this.config.models });
-        const images = inputs.images.reduce((total, count) => total + count, 0);
-        const perImage = images ? resolveEmbeddingModel(model, this.config.models).capabilities?.costPerImage : undefined;
-        if (!perImage)
+        const extra = mediaCost(totalUnits(inputs.units), resolveEmbeddingModel(model, this.config.models).capabilities);
+        if (!extra)
             return estimate;
-        const totalCost = estimate.totalCost + images * perImage;
+        const totalCost = estimate.totalCost + extra;
         return { ...estimate, totalCost, formatted: formatCost(totalCost) };
     }
     cacheContext(route, request) {
@@ -529,12 +531,11 @@ function assertModalities(providerName, modelName, model, provider, inputs) {
     const refused = [...inputs.media].filter((kind) => !accepted.includes(kind));
     throw new EmbeddingCapabilityError(providerName, 'input', refused.join(', '), `Embedding model "${modelName}" takes ${accepted.join(', ')} input, not ${refused.join(' or ')}; use a model whose inputModalities include it, such as gemini-embedding-2`);
 }
-function withImageCost(cost, images, model) {
-    const perImage = model?.costPerImage;
-    if (!images || !perImage)
+function withMediaCost(cost, media, model) {
+    const extra = mediaCost(media, model);
+    if (!extra)
         return cost;
-    const imageCost = images * perImage;
-    return { ...cost, amount: cost.amount + imageCost, input: (cost.input ?? 0) + imageCost };
+    return { ...cost, amount: cost.amount + extra, input: (cost.input ?? 0) + extra };
 }
 function addUsage(usage, inputTokens, totalTokens) {
     const input = usage.inputTokens + (inputTokens ?? 0);

package/dist/embeddings/index.d.ts

@@ -5,4 +5,4 @@ export { MockEmbeddingProvider, type MockEmbeddingProviderOptions } from './mock
 export { createConfiguredEmbeddingProviders } from './register.js';
 export { EMBEDDING_MODEL_ALIASES, EMBEDDING_REGISTRY_PROVENANCE, KNOWN_EMBEDDING_MODELS, embeddingDimensions, estimateEmbeddingCost, getEmbeddingModelAliases, getEmbeddingModelCapabilities, getEmbeddingModelRegistry, listEmbeddingModels, listEmbeddingModelsForProvider, priceEmbeddingUsage, resolveEmbeddingModel, resolveMaxBatchSize, type EmbeddingCostEstimateInput, type ResolvedEmbeddingModel, } from './models.js';
 export { createCohereEmbeddingProvider, createGeminiEmbeddingProvider, createOpenAIEmbeddingProvider, toEmbeddingFunction, type CohereEmbeddingOptions, type EmbeddingSource, type GeminiEmbeddingOptions, type OpenAIEmbeddingOptions, } from './providers.js';
-export type { Embedding, EmbeddingCacheConfig, EmbeddingConfig, EmbeddingCostBudgetConfig, EmbeddingEncodingFormat, EmbeddingInput, EmbeddingInputType, EmbeddingMeta, EmbeddingModality, EmbeddingModelCapabilities, EmbeddingModelRegistryConfig, EmbeddingPart, EmbeddingParts, EmbeddingProviderCallContext, EmbeddingProviderCapabilities, EmbeddingProviderInfo, EmbeddingProviderMediaPart, EmbeddingProviderPart, EmbeddingProviderRequest, EmbeddingProviderTextPart, EmbeddingProviderResult, EmbeddingProviderUsage, EmbeddingRequest, EmbeddingResponse, EmbeddingTruncateMode, EmbeddingsProvider, } from '../types/embeddings.js';
+export type { Embedding, EmbeddingCacheConfig, EmbeddingConfig, EmbeddingCostBudgetConfig, EmbeddingEncodingFormat, EmbeddingInput, EmbeddingInputType, EmbeddingMeta, EmbeddingModality, EmbeddingModelCapabilities, EmbeddingModelRegistryConfig, EmbeddingPart, EmbeddingParts, EmbeddingProviderCallContext, EmbeddingProviderCapabilities, EmbeddingProviderInfo, EmbeddingProviderMediaPart, EmbeddingProviderPart, EmbeddingProviderRequest, EmbeddingProviderTextPart, EmbeddingProviderResult, EmbeddingProviderUsage, EmbeddingRequest, EmbeddingResponse, EmbeddingTruncateMode, EmbeddingVideoPart, EmbeddingsProvider, } from '../types/embeddings.js';

package/dist/embeddings/inputs.d.ts

@@ -1,4 +1,4 @@
-import type { EmbeddingInput, EmbeddingModality, EmbeddingProviderPart } from '../types/embeddings.js';
+import type { EmbeddingInput, EmbeddingModality, EmbeddingModelCapabilities, EmbeddingProviderPart } from '../types/embeddings.js';
 /** A media kind an embedding input can hold. */
 export type EmbeddingMediaKind = Exclude<EmbeddingModality, 'text'>;
 /** A request's inputs, checked and ready for routing, the cache, and the provider. */
@@ -9,11 +9,34 @@ export interface EmbeddingInputs {
     parts?: EmbeddingProviderPart[][];
     /** The media kinds the inputs hold. */
     media: Set<EmbeddingMediaKind>;
-    /** How many images each input holds, for pricing. */
-    images: number[];
+    /** What media each input holds, for pricing and for a model's limits. */
+    units: MediaUnits[];
+}
+/**
+ * What an input's media comes to: its images, each audio part's and each video part's length in
+ * seconds, read from its own headers, and the audio and video parts whose length could not be read.
+ */
+export interface MediaUnits {
+    images: number;
+    audio: number[];
+    video: number[];
+    unmeasured: number;
 }
 /** Checks a request's input and resolves every media part to base64 data or a URL. */
 export declare function normalizeInputs(input: EmbeddingInput): EmbeddingInputs;
+/** Every input's media together. */
+export declare function totalUnits(units: readonly MediaUnits[]): MediaUnits;
+/** How many frames a model takes from a video of this length, by its `videoFrames`. */
+export declare function framesOf(seconds: number, model: EmbeddingModelCapabilities): number;
+/**
+ * What a model charges for media, in US dollars: each image, each second of audio, counted up, and
+ * each frame it takes from a video. A model with no media prices charges nothing for it.
+ */
+export declare function mediaCost(units: MediaUnits, model: EmbeddingModelCapabilities | undefined): number;
+/** Whether a model prices a medium by its length, so a part whose length is unknown is unpriced. */
+export declare function pricesByLength(model: EmbeddingModelCapabilities | undefined): boolean;
+/** Refuses audio or video longer than a model takes, naming the input, before anything is sent. */
+export declare function assertLengths(inputs: EmbeddingInputs, modelName: string, model: EmbeddingModelCapabilities | undefined): void;
 /**
  * Each input's identity, which repeats and the cache are keyed by: a text itself, or its parts with
  * every medium's content hashed, so a megabyte image is never a key. Loaded only for media.

package/dist/embeddings/inputs.js

@@ -1,4 +1,6 @@
 import { wireAudio, wireDocument, wireImage } from '../providers/content.js';
+import { decodeBase64, encodeBase64 } from '../utils/base64.js';
+import { readMediaDuration } from '../utils/media-duration.js';
 import { EmbeddingValidationError } from './errors.js';
 export function normalizeInputs(input) {
     const items = typeof input === 'string' || !Array.isArray(input) ? [input] : input;
@@ -8,14 +10,14 @@ export function normalizeInputs(input) {
     const texts = [];
     const resolved = [];
     const media = new Set();
-    const images = [];
+    const units = [];
     for (const item of items) {
         if (typeof item === 'string') {
             if (!item)
                 throw new EmbeddingValidationError('Every embedding input must be a non-empty string');
             texts.push(item);
             resolved.push([{ type: 'text', text: item }]);
-            images.push(0);
+            units.push({ images: 0, audio: [], video: [], unmeasured: 0 });
             continue;
         }
         if (!item || typeof item !== 'object' || !Array.isArray(item.parts) || item.parts.length === 0) {
@@ -30,9 +32,60 @@ export function normalizeInputs(input) {
             .map((part) => part.text)
             .join('\n'));
         resolved.push(parts);
-        images.push(parts.filter((part) => part.type === 'image').length);
+        units.push(measure(parts));
     }
-    return { texts, ...(media.size ? { parts: resolved } : {}), media, images };
+    return { texts, ...(media.size ? { parts: resolved } : {}), media, units };
+}
+function measure(parts) {
+    const units = { images: 0, audio: [], video: [], unmeasured: 0 };
+    for (const part of parts) {
+        if (part.type === 'image')
+            units.images += 1;
+        if (part.type !== 'audio' && part.type !== 'video')
+            continue;
+        const read = part.data === undefined ? undefined : readMediaDuration(decodeBase64(part.data));
+        if (read)
+            units[part.type].push(read.durationMs / 1000);
+        else
+            units.unmeasured += 1;
+    }
+    return units;
+}
+export function totalUnits(units) {
+    return {
+        images: units.reduce((total, unit) => total + unit.images, 0),
+        audio: units.flatMap((unit) => unit.audio),
+        video: units.flatMap((unit) => unit.video),
+        unmeasured: units.reduce((total, unit) => total + unit.unmeasured, 0),
+    };
+}
+export function framesOf(seconds, model) {
+    const sampling = model.videoFrames ?? { perSecond: 1 };
+    const frames = Math.ceil(seconds * sampling.perSecond);
+    return sampling.max === undefined ? frames : Math.min(sampling.max, frames);
+}
+export function mediaCost(units, model) {
+    if (!model)
+        return 0;
+    const seconds = units.audio.reduce((total, length) => total + Math.ceil(length), 0);
+    const frames = units.video.reduce((total, length) => total + framesOf(length, model), 0);
+    return (units.images * (model.costPerImage ?? 0) +
+        seconds * (model.costPerAudioSecond ?? 0) +
+        frames * (model.costPerVideoFrame ?? 0));
+}
+export function pricesByLength(model) {
+    return model?.costPerAudioSecond !== undefined || model?.costPerVideoFrame !== undefined;
+}
+export function assertLengths(inputs, modelName, model) {
+    inputs.units.forEach((unit, index) => {
+        for (const kind of ['audio', 'video']) {
+            const most = kind === 'audio' ? model?.maxAudioSeconds : model?.maxVideoSeconds;
+            const longest = Math.max(0, ...unit[kind]);
+            if (most !== undefined && longest > most) {
+                throw new EmbeddingValidationError(`Input ${index} holds ${longest.toFixed(1)} s of ${kind}, and ${modelName} takes at most ${most} s; split it`);
+            }
+        }
+    });
 }
 function toProviderPart(part) {
     switch (part?.type) {
@@ -63,10 +116,36 @@ function toProviderPart(part) {
                 ? { type: 'document', mimeType: wire.mimeType, url: wire.url }
                 : { type: 'document', mimeType: wire.mimeType, data: wire.data };
         }
+        case 'video':
+            return videoPart(part);
         default:
-            throw new EmbeddingValidationError(`An embedding part is text, image, audio, or document, not ${JSON.stringify(part?.type)}`);
+            throw new EmbeddingValidationError(`An embedding part is text, image, audio, video, or document, not ${JSON.stringify(part?.type)}`);
     }
 }
+function videoPart(part) {
+    const source = (part.source ?? {});
+    const format = typeof source.format === 'string' ? source.format.toLowerCase() : undefined;
+    const named = format ? (VIDEO_TYPES[format] ?? `video/${format}`) : undefined;
+    if (typeof source.url === 'string' && source.url) {
+        const extension = /\.([A-Za-z0-9]+)(?:[?#]|$)/.exec(source.url)?.[1]?.toLowerCase();
+        return { type: 'video', mimeType: named ?? (extension && VIDEO_TYPES[extension]) ?? 'video/mp4', url: source.url };
+    }
+    const data = typeof source.base64 === 'string'
+        ? source.base64
+        : source.buffer && typeof source.buffer.length === 'number'
+            ? encodeBase64(source.buffer)
+            : undefined;
+    if (!data) {
+        throw new EmbeddingValidationError('An embedding video part must hold its content, as bytes, base64, or a URL; read a file on disk first');
+    }
+    return { type: 'video', mimeType: named ?? 'video/mp4', data };
+}
+const VIDEO_TYPES = {
+    mp4: 'video/mp4',
+    mov: 'video/quicktime',
+    webm: 'video/webm',
+    mpeg: 'video/mpeg',
+};
 function unreadable(kind) {
     return new EmbeddingValidationError(`An embedding ${kind} part must hold its content, as bytes, base64, or a URL; read a file or a stream first`);
 }

package/dist/embeddings/mock.js

@@ -16,7 +16,7 @@ export class MockEmbeddingProvider {
                 dimensions: true,
                 encodingFormats: ['float'],
                 inputTypes: ['document', 'query', 'classification', 'clustering'],
-                inputModalities: ['text', 'image', 'audio', 'document'],
+                inputModalities: ['text', 'image', 'audio', 'video', 'document'],
                 truncate: true,
                 ...options.capabilities,
             },