nexus-ai-pro 2.20.2 → 2.22.0

package/dist/embeddings/models.js

@@ -60,13 +60,18 @@ export const KNOWN_EMBEDDING_MODELS = {
         maxBatchSize: 100,
         costPer1kInput: 0.0002,
         costPerImage: 0.00012,
+        costPerAudioSecond: 0.00016,
+        costPerVideoFrame: 0.00079,
+        videoFrames: { perSecond: 1, max: 32 },
+        maxAudioSeconds: 180,
+        maxVideoSeconds: 120,
         normalized: true,
         inputTypes: ['document', 'query', 'classification', 'clustering'],
-        inputModalities: ['text', 'image', 'audio', 'document'],
+        inputModalities: ['text', 'image', 'audio', 'video', 'document'],
         status: 'stable',
-        verifiedAt: '2026-10-13',
-        source: 'https://ai.google.dev/gemini-api/docs/embeddings',
-        notes: 'One vector for text, images (up to 6), audio (up to 180 s), and a PDF (up to 6 pages) together. Reads its task from a text prefix, not a task type. Audio and video are billed per second and frame, which is not priced here.',
+        verifiedAt: '2026-10-28',
+        source: 'https://ai.google.dev/gemini-api/docs/pricing',
+        notes: 'One vector for text, images (up to 6, PNG or JPEG), audio (up to 180 s, MP3 or WAV), video (up to 120 s, MP4 or MOV, its sound not heard), and a PDF (up to 6 pages) together. Reads its task from a text prefix, not a task type. Audio is billed by the second, video by the frame it takes: one a second, at most 32.',
     }),
     'text-embedding-004': embeddingModel({
         provider: 'google',
@@ -93,6 +98,36 @@ export const KNOWN_EMBEDDING_MODELS = {
         status: 'stable',
         notes: 'Takes images, and text with images, as data URIs: JPEG, PNG, WebP, or GIF, 20 MB a request in total.',
     }),
+    'embed-v5.0-pro': embeddingModel({
+        provider: 'cohere',
+        family: 'embed-v5',
+        dimensions: 2048,
+        supportedDimensions: [256, 512, 768, 1024, 1536, 2048],
+        maxInputTokens: 128000,
+        maxBatchSize: 96,
+        costPer1kInput: 0,
+        inputTypes: ['document', 'query', 'classification', 'clustering'],
+        inputModalities: ['text', 'image'],
+        status: 'stable',
+        verifiedAt: '2026-10-29',
+        source: 'https://docs.cohere.com/docs/cohere-embed',
+        notes: 'The quality model, for indexing. Shares one vector space with embed-v5.0-fast, so a corpus indexed with Pro is searched with Fast. Cohere publishes no per-token price for it, so its cost reads as zero: give one through embeddings.models.registry.',
+    }),
+    'embed-v5.0-fast': embeddingModel({
+        provider: 'cohere',
+        family: 'embed-v5',
+        dimensions: 2048,
+        supportedDimensions: [256, 512, 768, 1024, 1536, 2048],
+        maxInputTokens: 128000,
+        maxBatchSize: 96,
+        costPer1kInput: 0,
+        inputTypes: ['document', 'query', 'classification', 'clustering'],
+        inputModalities: ['text', 'image'],
+        status: 'stable',
+        verifiedAt: '2026-10-29',
+        source: 'https://docs.cohere.com/docs/cohere-embed',
+        notes: 'The fast model, for queries and agent loops. Shares one vector space with embed-v5.0-pro, so a corpus indexed with Pro is searched with Fast. Cohere publishes no per-token price for it, so its cost reads as zero: give one through embeddings.models.registry.',
+    }),
     'embed-english-v3.0': embeddingModel({
         provider: 'cohere',
         family: 'embed-v3',

package/dist/loaders/csv.d.ts

@@ -5,6 +5,27 @@ export interface CsvParseOptions {
     /** The field separator. Defaults to a comma; use `'\t'` for TSV. */
     delimiter?: string;
 }
+/**
+ * CSV parsed as it arrives: `push()` takes the next piece of text, of any length, and returns the rows
+ * it completed; `end()` returns the last. A quote, or a CRLF, split between two pieces is read as one.
+ * `parseCsv()` is this, given the whole text at once.
+ */
+export declare class CsvRowParser {
+    private readonly delimiter;
+    private row;
+    private field;
+    private quoted;
+    /** A quote inside a quoted field, waiting for the next character: a second quote, or the field's end. */
+    private quote;
+    /** A carriage return that ended a row, whose line feed, if one follows, belongs to it. */
+    private carriage;
+    private started;
+    constructor(options?: CsvParseOptions);
+    /** Reads the next piece of text, and returns the rows it completed. */
+    push(text: string): string[][];
+    /** Ends the text, and returns its last row, unless it ended with a line break. */
+    end(): string[][];
+}
 /**
  * Parses CSV into rows of fields, following RFC 4180: quoted fields may hold the delimiter, line
  * breaks, and doubled quotes; lines may end in CRLF or LF; and a trailing line break adds no row.
@@ -24,6 +45,7 @@ export interface CsvLoaderOptions extends FileLoaderOptions, CsvParseOptions {
 }
 /**
  * CSV files, one document per row. The first row names the columns. Each row's text lists its columns
- * by name, so a model reading a chunk knows what each value is.
+ * by name, so a model reading a chunk knows what each value is. Rows are read as the file arrives, so
+ * a file larger than memory streams through.
  */
 export declare function loadCsv(inputs: FileInput | Iterable<FileInput>, options?: CsvLoaderOptions): AsyncGenerator<DocumentSource>;

package/dist/loaders/csv.js

@@ -1,87 +1,140 @@
-import { inputsOf, readText } from './input.js';
-export function parseCsv(text, options = {}) {
-    const delimiter = options.delimiter ?? ',';
-    if (delimiter.length !== 1 || delimiter === '"' || delimiter === '\n' || delimiter === '\r') {
-        throw new RangeError('delimiter must be one character other than a quote or a line break');
+import { inputsOf, sourceOf, textChunks } from './input.js';
+export class CsvRowParser {
+    delimiter;
+    row = [];
+    field = '';
+    quoted = false;
+    quote = false;
+    carriage = false;
+    started = false;
+    constructor(options = {}) {
+        const delimiter = options.delimiter ?? ',';
+        if (delimiter.length !== 1 || delimiter === '"' || delimiter === '\n' || delimiter === '\r') {
+            throw new RangeError('delimiter must be one character other than a quote or a line break');
+        }
+        this.delimiter = delimiter;
     }
-    const rows = [];
-    let row = [];
-    let field = '';
-    let quoted = false;
-    let index = text.charCodeAt(0) === 0xfeff ? 1 : 0;
-    for (; index < text.length; index++) {
-        const char = text[index];
-        if (quoted) {
-            if (char !== '"')
-                field += char;
-            else if (text[index + 1] === '"') {
-                field += '"';
-                index++;
+    push(text) {
+        const rows = [];
+        let index = 0;
+        if (!this.started && text.length > 0) {
+            this.started = true;
+            if (text.charCodeAt(0) === 0xfeff)
+                index = 1;
+        }
+        for (; index < text.length; index++) {
+            const char = text[index];
+            if (this.carriage) {
+                this.carriage = false;
+                if (char === '\n')
+                    continue;
+            }
+            if (this.quote) {
+                this.quote = false;
+                if (char === '"') {
+                    this.field += '"';
+                    continue;
+                }
+                this.quoted = false;
+            }
+            if (this.quoted) {
+                if (char === '"')
+                    this.quote = true;
+                else
+                    this.field += char;
+            }
+            else if (char === '"' && this.field === '')
+                this.quoted = true;
+            else if (char === this.delimiter) {
+                this.row.push(this.field);
+                this.field = '';
+            }
+            else if (char === '\n' || char === '\r') {
+                this.carriage = char === '\r';
+                this.row.push(this.field);
+                rows.push(this.row);
+                this.row = [];
+                this.field = '';
             }
             else
-                quoted = false;
+                this.field += char;
         }
-        else if (char === '"' && field === '')
-            quoted = true;
-        else if (char === delimiter) {
-            row.push(field);
-            field = '';
+        return rows;
+    }
+    end() {
+        if (this.quote) {
+            this.quote = false;
+            this.quoted = false;
         }
-        else if (char === '\n' || char === '\r') {
-            if (char === '\r' && text[index + 1] === '\n')
-                index++;
-            row.push(field);
-            rows.push(row);
-            row = [];
-            field = '';
+        if (this.quoted)
+            throw new SyntaxError('The CSV ends inside a quoted field');
+        const rows = [];
+        if (this.field !== '' || this.row.length > 0) {
+            this.row.push(this.field);
+            rows.push(this.row);
         }
-        else
-            field += char;
-    }
-    if (quoted)
-        throw new SyntaxError('The CSV ends inside a quoted field');
-    if (field !== '' || row.length > 0) {
-        row.push(field);
-        rows.push(row);
+        this.row = [];
+        this.field = '';
+        return rows;
     }
-    return rows;
+}
+export function parseCsv(text, options = {}) {
+    const parser = new CsvRowParser(options);
+    return [...parser.push(text), ...parser.end()];
 }
 export async function* loadCsv(inputs, options = {}) {
     for (const input of inputsOf(inputs)) {
-        const { source, text } = await readText(input);
-        const [header, ...rows] = parseCsv(text, options);
-        if (!header)
-            continue;
-        const column = (name) => {
-            const index = header.indexOf(name);
-            if (index < 0)
-                throw new RangeError(`${source} has no column "${name}"`);
-            return index;
+        const source = sourceOf(input);
+        const parser = new CsvRowParser(options);
+        let reader;
+        const rows = async function* () {
+            for await (const chunk of textChunks(input))
+                yield* parser.push(chunk);
+            yield* parser.end();
         };
-        const idIndex = options.idColumn === undefined ? -1 : column(options.idColumn);
-        const metadataIndexes = (options.metadataColumns ?? []).map((name) => [name, column(name)]);
-        const textIndexes = options.textColumns
-            ? options.textColumns.map(column)
-            : header
-                .map((_, index) => index)
-                .filter((index) => index !== idIndex && !metadataIndexes.some(([, metadata]) => metadata === index));
-        for (const [rowIndex, fields] of rows.entries()) {
-            if (fields.length === 1 && fields[0] === '')
+        for await (const fields of rows()) {
+            if (!reader) {
+                reader = rowReader(source, fields, options);
                 continue;
-            const key = idIndex >= 0 ? fields[idIndex] : String(rowIndex + 1);
-            yield {
-                id: `${source}#${key}`,
-                text: textIndexes
-                    .map((index) => `${header[index]}: ${fields[index] ?? ''}`)
-                    .filter((line) => !line.endsWith(': '))
-                    .join('\n'),
-                source,
-                metadata: {
-                    ...options.metadata,
-                    ...Object.fromEntries(metadataIndexes.map(([name, index]) => [name, fields[index] ?? ''])),
-                    row: rowIndex + 1,
-                },
-            };
+            }
+            const document = reader(fields);
+            if (document)
+                yield document;
         }
     }
 }
+function rowReader(source, header, options) {
+    const column = (name) => {
+        const index = header.indexOf(name);
+        if (index < 0)
+            throw new RangeError(`${source} has no column "${name}"`);
+        return index;
+    };
+    const idIndex = options.idColumn === undefined ? -1 : column(options.idColumn);
+    const metadataIndexes = (options.metadataColumns ?? []).map((name) => [name, column(name)]);
+    const textIndexes = options.textColumns
+        ? options.textColumns.map(column)
+        : header
+            .map((_, index) => index)
+            .filter((index) => index !== idIndex && !metadataIndexes.some(([, metadata]) => metadata === index));
+    let rowIndex = 0;
+    return (fields) => {
+        rowIndex += 1;
+        if (fields.length === 1 && fields[0] === '')
+            return undefined;
+        const key = idIndex >= 0 ? fields[idIndex] : String(rowIndex);
+        return {
+            id: `${source}#${key}`,
+            text: textIndexes
+                .map((index) => `${header[index]}: ${fields[index] ?? ''}`)
+                .filter((line) => !line.endsWith(': '))
+                .join('\n'),
+            source,
+            metadata: {
+                ...options.metadata,
+                ...Object.fromEntries(metadataIndexes.map(([name, index]) => [name, fields[index] ?? ''])),
+                row: rowIndex,
+            },
+        };
+    };
+}

package/dist/loaders/index.d.ts

@@ -7,12 +7,17 @@ export type { DocumentSource } from '../rag/ingestion.js';
  */
 export type DocumentLoader = AsyncIterable<DocumentSource> | Iterable<DocumentSource>;
 /**
- * A file to load: a path, a `file:` URL, or content already in memory with a name for it. In-memory
- * content is how a loader reads an upload, a database blob, or a file on a runtime without a disk.
+ * A file to load: a path, a `file:` URL, content already in memory with a name for it, or a stream of
+ * it, such as a response's body or an object store's download. In-memory content is how a loader reads
+ * an upload, a database blob, or a file on a runtime without a disk; a stream, how the CSV and JSON
+ * loaders read a file larger than memory, as it arrives.
  */
 export type FileInput = string | URL | {
     source: string;
     content: string | Uint8Array;
+} | {
+    source: string;
+    stream: AsyncIterable<Uint8Array | string> | Iterable<Uint8Array | string>;
 };
 /** Reads every document from one or more loaders, in order, into an array. */
 export declare function collectDocuments(...loaders: DocumentLoader[]): Promise<DocumentSource[]>;

package/dist/loaders/input.d.ts

@@ -18,3 +18,9 @@ export declare function readText(input: FileInput): Promise<{
     source: string;
     text: string;
 }>;
+/**
+ * An input's text a piece at a time, so a large file is never held whole: a path or file URL read from
+ * disk in chunks of 64 KiB, a stream as it arrives, and content in memory as it is. A character split
+ * between two chunks is decoded whole, and a byte-order mark is dropped.
+ */
+export declare function textChunks(input: FileInput): AsyncGenerator<string>;

package/dist/loaders/input.js

@@ -14,6 +14,19 @@ export function sourceOf(input) {
 }
 export async function readInput(input) {
     if (typeof input === 'object' && !(input instanceof URL)) {
+        if ('stream' in input) {
+            const pieces = [];
+            for await (const piece of input.stream) {
+                pieces.push(typeof piece === 'string' ? new TextEncoder().encode(piece) : piece);
+            }
+            const bytes = new Uint8Array(pieces.reduce((total, piece) => total + piece.byteLength, 0));
+            let offset = 0;
+            for (const piece of pieces) {
+                bytes.set(piece, offset);
+                offset += piece.byteLength;
+            }
+            return { source: input.source, bytes };
+        }
         const bytes = typeof input.content === 'string' ? new TextEncoder().encode(input.content) : input.content;
         return { source: input.source, bytes };
     }
@@ -21,12 +34,40 @@ export async function readInput(input) {
     return { source: sourceOf(input), bytes: new Uint8Array(await readFile(input)) };
 }
 export async function readText(input) {
-    if (typeof input === 'object' && !(input instanceof URL) && typeof input.content === 'string') {
+    if (typeof input === 'object' && !(input instanceof URL) && 'content' in input && typeof input.content === 'string') {
         return { source: input.source, text: stripBom(input.content) };
     }
     const { source, bytes } = await readInput(input);
     return { source, text: stripBom(new TextDecoder().decode(bytes)) };
 }
+export async function* textChunks(input) {
+    const decoder = new TextDecoder();
+    let first = true;
+    for await (const piece of piecesOf(input)) {
+        let text = typeof piece === 'string' ? piece : decoder.decode(piece, { stream: true });
+        if (first && text) {
+            first = false;
+            text = stripBom(text);
+        }
+        if (text)
+            yield text;
+    }
+    const rest = decoder.decode();
+    if (rest)
+        yield rest;
+}
+async function* piecesOf(input) {
+    if (typeof input === 'object' && !(input instanceof URL)) {
+        if ('stream' in input)
+            yield* input.stream;
+        else
+            yield input.content;
+        return;
+    }
+    const { createReadStream } = await import('node:fs');
+    for await (const chunk of createReadStream(input, { highWaterMark: 64 * 1024 }))
+        yield chunk;
+}
 function stripBom(text) {
     return text.charCodeAt(0) === 0xfeff ? text.slice(1) : text;
 }

package/dist/loaders/json.d.ts

@@ -3,10 +3,17 @@ import type { FileLoaderOptions } from './text.js';
 /** Options for `loadJson()`. */
 export interface JsonLoaderOptions extends FileLoaderOptions {
     /**
-     * Picks the records out of the parsed file. Defaults to the items of a top-level array, or else the
-     * value itself as one record.
+     * Picks the records out of the parsed file, which is then read whole. Defaults to the items of the
+     * array at `recordsAt`, or of a top-level array, read as the file arrives; or else the value itself as
+     * one record.
      */
     records?: (value: unknown) => Iterable<unknown>;
+    /**
+     * Where the array of records is: object keys from the top, dotted, such as `data.items`. Its items
+     * are read one at a time as the file arrives, so a file larger than memory streams through. A file
+     * with no array there is refused.
+     */
+    recordsAt?: string;
     /**
      * A record's text: a field, by dotted path such as `body.text`, or a function. Defaults to a string
      * record itself, or else the record as indented JSON.
@@ -22,5 +29,9 @@ export interface JsonLoaderOptions extends FileLoaderOptions {
      */
     lines?: boolean;
 }
-/** JSON and JSON Lines files, one document per record. */
+/**
+ * JSON and JSON Lines files, one document per record. A JSON Lines file, and the array of records in a
+ * JSON one, are read as the file arrives, a record at a time, so a file larger than memory streams
+ * through; a `records` function reads the file whole.
+ */
 export declare function loadJson(inputs: FileInput | Iterable<FileInput>, options?: JsonLoaderOptions): AsyncGenerator<DocumentSource>;

package/dist/loaders/json.js

@@ -1,28 +1,15 @@
-import { inputsOf, readText } from './input.js';
+import { inputsOf, readText, sourceOf, textChunks } from './input.js';
 export async function* loadJson(inputs, options = {}) {
     for (const input of inputsOf(inputs)) {
-        const { source, text } = await readText(input);
+        const source = sourceOf(input);
         const lines = options.lines ?? /\.(jsonl|ndjson)$/i.test(source);
-        const parsed = lines
-            ? text
-                .split(/\r?\n/)
-                .filter((line) => line.trim())
-                .map((line, index) => {
-                try {
-                    return JSON.parse(line);
-                }
-                catch (error) {
-                    throw new SyntaxError(`${source} line ${index + 1}: ${error.message}`);
-                }
-            })
-            : JSON.parse(text);
-        const records = options.records ? options.records(parsed) : Array.isArray(parsed) ? parsed : [parsed];
         let index = 0;
-        for (const record of records) {
+        const documentOf = (record) => {
             const key = typeof options.id === 'function' ? options.id(record, index) : pick(record, options.id);
             const value = typeof options.text === 'function' ? options.text(record) : pick(record, options.text, record);
-            yield {
-                id: `${source}#${key === undefined || key === null ? index + 1 : String(key)}`,
+            index++;
+            return {
+                id: `${source}#${key === undefined || key === null ? index : String(key)}`,
                 text: typeof value === 'string' ? value : JSON.stringify(value, null, 2),
                 source,
                 metadata: {
@@ -32,8 +19,172 @@ export async function* loadJson(inputs, options = {}) {
                         : Object.fromEntries((options.metadataFields ?? []).map((field) => [field, pick(record, field)]))),
                 },
             };
-            index++;
+        };
+        if (options.records) {
+            const { text } = await readText(input);
+            const values = [];
+            if (lines)
+                for await (const value of jsonLines([text], source))
+                    values.push(value);
+            const parsed = lines ? values : JSON.parse(text);
+            for (const record of options.records(parsed))
+                yield documentOf(record);
+        }
+        else if (lines) {
+            for await (const record of jsonLines(textChunks(input), source))
+                yield documentOf(record);
+        }
+        else {
+            const scanner = new JsonRecordScanner(options.recordsAt?.split('.') ?? []);
+            for await (const chunk of textChunks(input)) {
+                for (const record of scanner.push(chunk))
+                    yield documentOf(record);
+                if (scanner.finished)
+                    break;
+            }
+            for (const record of scanner.end(source))
+                yield documentOf(record);
+        }
+    }
+}
+async function* jsonLines(chunks, source) {
+    let rest = '';
+    let number = 0;
+    const parse = (line) => {
+        number += 1;
+        if (!line.trim())
+            return undefined;
+        try {
+            return { value: JSON.parse(line) };
+        }
+        catch (error) {
+            throw new SyntaxError(`${source} line ${number}: ${error.message}`);
+        }
+    };
+    for await (const chunk of chunks) {
+        const pieces = (rest + chunk).split('\n');
+        rest = pieces.pop() ?? '';
+        for (const line of pieces) {
+            const parsed = parse(line);
+            if (parsed)
+                yield parsed.value;
+        }
+    }
+    const last = parse(rest);
+    if (last)
+        yield last.value;
+}
+class JsonRecordScanner {
+    path;
+    stack = [];
+    inString = false;
+    escaped = false;
+    keyText;
+    target = -1;
+    found = false;
+    done = false;
+    record;
+    whole;
+    started = false;
+    explicit;
+    constructor(path = []) {
+        this.path = path;
+        this.explicit = path.length > 0;
+    }
+    get finished() {
+        return this.done;
+    }
+    push(text) {
+        const records = [];
+        if (this.whole) {
+            this.whole.push(text);
+            return records;
         }
+        let from = this.record ? 0 : -1;
+        for (let index = 0; index < text.length; index++) {
+            const char = text[index];
+            if (!this.started) {
+                if (/\s/.test(char))
+                    continue;
+                this.started = true;
+                if (char !== '[' && !this.explicit) {
+                    this.whole = [text.slice(index)];
+                    return records;
+                }
+            }
+            if (this.inString) {
+                if (this.keyText)
+                    this.keyText.push(char);
+                if (this.escaped)
+                    this.escaped = false;
+                else if (char === '\\')
+                    this.escaped = true;
+                else if (char === '"') {
+                    this.inString = false;
+                    if (this.keyText) {
+                        const frame = this.stack.at(-1);
+                        frame.key = JSON.parse(`"${this.keyText.slice(0, -1).join('')}"`);
+                        frame.expectKey = false;
+                        this.keyText = undefined;
+                    }
+                }
+                continue;
+            }
+            const atTarget = this.target >= 0 && this.stack.length === this.target;
+            if (atTarget && (char === ',' || char === ']')) {
+                if (this.record) {
+                    this.record.push(text.slice(from, index));
+                    records.push(JSON.parse(this.record.join('')));
+                    this.record = undefined;
+                }
+                if (char === ']') {
+                    this.done = true;
+                    return records;
+                }
+                continue;
+            }
+            if (atTarget && !this.record && !/\s/.test(char)) {
+                this.record = [];
+                from = index;
+            }
+            const frame = this.stack.at(-1);
+            if (char === '"') {
+                this.inString = true;
+                if (frame?.kind === 'object' && frame.expectKey && !this.record)
+                    this.keyText = [];
+            }
+            else if (char === '{' || char === '[') {
+                const opened = char === '{' ? { kind: 'object', expectKey: true } : { kind: 'array' };
+                if (char === '[' && this.at(this.path)) {
+                    this.found = true;
+                    this.target = this.stack.length + 1;
+                }
+                this.stack.push(opened);
+            }
+            else if (char === '}' || char === ']') {
+                this.stack.pop();
+            }
+            else if (char === ',' && frame?.kind === 'object')
+                frame.expectKey = true;
+        }
+        if (this.record)
+            this.record.push(text.slice(Math.max(from, 0)));
+        return records;
+    }
+    end(source) {
+        if (this.whole)
+            return [JSON.parse(this.whole.join(''))];
+        if (!this.found) {
+            throw new SyntaxError(this.explicit ? `${source} has no array at "${this.path.join('.')}"` : `${source} holds no JSON value`);
+        }
+        if (!this.done)
+            throw new SyntaxError(`${source} ends inside its array of records`);
+        return [];
+    }
+    at(path) {
+        if (this.stack.length !== path.length)
+            return false;
+        return this.stack.every((frame, depth) => frame.key === path[depth]);
     }
 }
 function pick(record, path, fallback) {