nexus-ai-pro 2.20.2 → 2.22.0

package/dist-cjs/voice/providers/google.js

@@ -0,0 +1,252 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.GoogleVoiceProvider = void 0;
+const base64_js_1 = require("../../utils/base64.js");
+const media_duration_js_1 = require("../../utils/media-duration.js");
+const errors_js_1 = require("../errors.js");
+const DEFAULT_TRANSCRIPTION_MODEL = 'gemini-3.8-flash';
+const DEFAULT_SPEECH_MODEL = 'gemini-3.8-flash-tts';
+const DEFAULT_VOICE = 'Kore';
+const SPEECH_RATE = 24_000;
+const INLINE_AUDIO_BYTES = 14 * 1024 * 1024;
+class GoogleVoiceProvider {
+    config;
+    info = {
+        name: 'google',
+        isLocal: false,
+        supports: { transcription: true, speech: true, realtime: false },
+    };
+    constructor(config) {
+        this.config = config;
+    }
+    async transcribe(request, context) {
+        if (request.timestampGranularities?.length ||
+            (request.responseFormat !== undefined && request.responseFormat !== 'json' && request.responseFormat !== 'text')) {
+            throw new errors_js_1.VoiceProviderError('Gemini transcribes to text alone, without timings or subtitles; ask for json or text', 'google');
+        }
+        const model = request.model || this.config.transcriptionModel || DEFAULT_TRANSCRIPTION_MODEL;
+        const signal = context?.signal ?? request.signal;
+        const { part, bytes } = await this.audioPart(request.audio, signal);
+        const instruction = [
+            'Transcribe the speech in this audio word for word, in the language it is spoken in.',
+            request.language ? `It is in ${request.language}.` : '',
+            request.prompt ? `Names and spellings it may hold: ${request.prompt}` : '',
+            'Answer with the transcript alone.',
+        ]
+            .filter(Boolean)
+            .join(' ');
+        const answer = await this.generate(model, {
+            contents: [{ role: 'user', parts: [{ text: instruction }, part] }],
+            generationConfig: { temperature: request.temperature ?? 0 },
+        }, signal, context);
+        const text = (answer.candidates?.[0]?.content?.parts ?? [])
+            .map((item) => item.text ?? '')
+            .join('')
+            .trim();
+        const duration = bytes ? (0, media_duration_js_1.readMediaDuration)(bytes) : undefined;
+        return {
+            text,
+            providerUsed: 'google',
+            modelUsed: answer.modelVersion ?? model,
+            ...(request.language ? { language: request.language } : {}),
+            ...(duration ? { durationSeconds: duration.durationMs / 1000 } : {}),
+            raw: answer,
+        };
+    }
+    async speak(request, context) {
+        const format = request.format ?? 'wav';
+        if (format !== 'wav' && format !== 'pcm') {
+            throw new errors_js_1.VoiceProviderError(`Gemini speaks WAV or raw PCM, not ${format}`, 'google');
+        }
+        const model = request.model || this.config.speechModel || DEFAULT_SPEECH_MODEL;
+        const voice = request.voice || this.config.defaultVoice || DEFAULT_VOICE;
+        const direction = [
+            request.instructions,
+            request.speed !== undefined && request.speed !== 1 ? `Speak at ${request.speed} times the usual pace.` : '',
+        ]
+            .filter(Boolean)
+            .join(' ');
+        const answer = await this.generate(model, {
+            contents: [{ role: 'user', parts: [{ text: request.text }] }],
+            ...(direction ? { systemInstruction: { parts: [{ text: direction }] } } : {}),
+            generationConfig: {
+                responseModalities: ['AUDIO'],
+                speechConfig: { voiceConfig: { prebuiltVoiceConfig: { voiceName: voice } } },
+            },
+        }, context?.signal ?? request.signal, context);
+        const inline = (answer.candidates?.[0]?.content?.parts ?? []).find((item) => item.inlineData?.data)?.inlineData;
+        if (!inline?.data)
+            throw new errors_js_1.VoiceProviderError('Gemini answered with no audio', 'google');
+        const received = (0, base64_js_1.decodeBase64)(inline.data);
+        const wav = isWav(received);
+        const rate = Number(/rate=(\d+)/i.exec(inline.mimeType ?? '')?.[1] ?? SPEECH_RATE);
+        const data = format === 'wav' ? (wav ? received : wavOf(received, rate)) : wav ? samplesOf(received) : received;
+        const mimeType = format === 'wav' ? 'audio/wav' : `audio/L16;rate=${wav ? rateOf(received) : rate}`;
+        return {
+            audio: { data, format, mimeType },
+            providerUsed: 'google',
+            modelUsed: answer.modelVersion ?? model,
+            voice,
+            format,
+            mimeType,
+            raw: answer,
+        };
+    }
+    async audioPart(input, signal) {
+        if ('url' in input && input.url.startsWith('https://generativelanguage.googleapis.com/')) {
+            return {
+                part: { fileData: { mimeType: input.mimeType ?? mimeTypeOf(input.url) ?? 'audio/wav', fileUri: input.url } },
+            };
+        }
+        const bytes = await this.read(input, signal);
+        if (bytes.byteLength > INLINE_AUDIO_BYTES) {
+            throw new errors_js_1.VoiceProviderError(`The audio is ${Math.ceil(bytes.byteLength / 1024 / 1024)} MB; Gemini takes at most 14 MB inline, so upload it to the Files API and pass its URI as the url`, 'google');
+        }
+        const mimeType = input.mimeType ??
+            ('filename' in input && input.filename ? mimeTypeOf(input.filename) : undefined) ??
+            ('path' in input ? mimeTypeOf(input.path) : undefined) ??
+            (0, media_duration_js_1.readMediaDuration)(bytes)?.mimeType ??
+            'audio/wav';
+        return { part: { inlineData: { mimeType, data: (0, base64_js_1.encodeBase64)(bytes) } }, bytes };
+    }
+    async read(input, signal) {
+        if ('url' in input) {
+            const response = await (this.config.fetch ?? fetch)(input.url, signal ? { signal } : {});
+            if (!response.ok)
+                throw new errors_js_1.VoiceProviderError(`The audio at ${input.url} answered ${response.status}`, 'google');
+            return new Uint8Array(await response.arrayBuffer());
+        }
+        if ('path' in input) {
+            const { readFile } = await Promise.resolve().then(() => __importStar(require('node:fs/promises')));
+            return new Uint8Array(await readFile(input.path));
+        }
+        if ('base64' in input)
+            return (0, base64_js_1.decodeBase64)(input.base64);
+        if ('buffer' in input) {
+            return input.buffer instanceof ArrayBuffer ? new Uint8Array(input.buffer) : new Uint8Array(input.buffer);
+        }
+        const pieces = [];
+        for await (const piece of input.stream) {
+            pieces.push(typeof piece === 'string' ? new TextEncoder().encode(piece) : new Uint8Array(piece));
+        }
+        const bytes = new Uint8Array(pieces.reduce((total, piece) => total + piece.byteLength, 0));
+        let offset = 0;
+        for (const piece of pieces) {
+            bytes.set(piece, offset);
+            offset += piece.byteLength;
+        }
+        return bytes;
+    }
+    async generate(model, body, signal, context) {
+        const base = (this.config.baseUrl ?? 'https://generativelanguage.googleapis.com/v1beta').replace(/\/+$/, '');
+        const qualified = model.startsWith('models/') ? model : `models/${model}`;
+        const credential = await context?.credential?.(this.info.name);
+        const response = await (this.config.fetch ?? fetch)(`${base}/${qualified}:generateContent`, {
+            method: 'POST',
+            headers: {
+                'content-type': 'application/json',
+                'x-goog-api-key': credential?.apiKey ?? this.config.apiKey,
+                ...this.config.headers,
+                ...credential?.headers,
+            },
+            body: JSON.stringify(body),
+            ...(signal ? { signal } : {}),
+        });
+        if (!response.ok) {
+            const message = await response.text().catch(() => response.statusText);
+            throw new errors_js_1.VoiceProviderError(`Gemini voice request failed: ${response.status} ${message.slice(0, 500)}`, 'google');
+        }
+        return (await response.json());
+    }
+}
+exports.GoogleVoiceProvider = GoogleVoiceProvider;
+const AUDIO_TYPES = {
+    wav: 'audio/wav',
+    mp3: 'audio/mp3',
+    aiff: 'audio/aiff',
+    aac: 'audio/aac',
+    ogg: 'audio/ogg',
+    flac: 'audio/flac',
+};
+function mimeTypeOf(name) {
+    const extension = /\.([A-Za-z0-9]+)(?:[?#]|$)/.exec(name)?.[1]?.toLowerCase();
+    return extension ? AUDIO_TYPES[extension] : undefined;
+}
+function isWav(bytes) {
+    return ascii(bytes, 0, 4) === 'RIFF' && ascii(bytes, 8, 4) === 'WAVE';
+}
+function ascii(bytes, at, length) {
+    return String.fromCharCode(...bytes.subarray(at, at + length));
+}
+function rateOf(wav) {
+    const view = new DataView(wav.buffer, wav.byteOffset, wav.byteLength);
+    for (let at = 12; at + 8 <= wav.length; at += 8 + view.getUint32(at + 4, true)) {
+        if (ascii(wav, at, 4) === 'fmt ')
+            return view.getUint32(at + 12, true);
+    }
+    return SPEECH_RATE;
+}
+function samplesOf(wav) {
+    const view = new DataView(wav.buffer, wav.byteOffset, wav.byteLength);
+    for (let at = 12; at + 8 <= wav.length; at += 8 + view.getUint32(at + 4, true)) {
+        if (ascii(wav, at, 4) === 'data')
+            return wav.subarray(at + 8, at + 8 + view.getUint32(at + 4, true));
+    }
+    return wav;
+}
+function wavOf(samples, rate) {
+    const wav = new Uint8Array(44 + samples.byteLength);
+    const view = new DataView(wav.buffer);
+    const text = (at, value) => {
+        for (let index = 0; index < value.length; index++)
+            wav[at + index] = value.charCodeAt(index);
+    };
+    text(0, 'RIFF');
+    view.setUint32(4, 36 + samples.byteLength, true);
+    text(8, 'WAVE');
+    text(12, 'fmt ');
+    view.setUint32(16, 16, true);
+    view.setUint16(20, 1, true);
+    view.setUint16(22, 1, true);
+    view.setUint32(24, rate, true);
+    view.setUint32(28, rate * 2, true);
+    view.setUint16(32, 2, true);
+    view.setUint16(34, 16, true);
+    text(36, 'data');
+    view.setUint32(40, samples.byteLength, true);
+    wav.set(samples, 44);
+    return wav;
+}

package/llms.txt

@@ -5,7 +5,7 @@
 Import only what you use. The root import is the client and nothing else, and every other capability has its own entry point with a size budget CI enforces; [the packaging guide](https://raw.githubusercontent.com/mkhitar-abrahamyan/nexus-ai/main/docs/packaging.md) publishes what each one costs. The package has no required dependencies: a graph-only application loads 81 KB, and installs nothing but this package. The client, the provider adapters, graphs, agents, and the agent protocols run on Node.js, Deno, Bun, edge runtimes, and browsers; [the runtimes guide](https://raw.githubusercontent.com/mkhitar-abrahamyan/nexus-ai/main/docs/runtimes.md) lists the entry points that need Node.js, and why.
 
 - Install with `npm install nexus-ai-pro`. It needs Node.js 22 or newer, and ships ESM and CommonJS builds with shared types. License: MIT.
-- It has 199 entry points. `nexus-ai-pro` itself is the client, and every other capability has its own subpath, such as `nexus-ai-pro/graph`.
+- It has 200 entry points. `nexus-ai-pro` itself is the client, and every other capability has its own subpath, such as `nexus-ai-pro/graph`.
 - Nothing is a required dependency. Install only the optional peers you use: `@anthropic-ai/sdk`, `@types/node`, `ajv`, `ajv-formats`, `ollama`, `openai`, `zod`.
 - Each guide explains every export of its entry points, then lists them in a reference generated from the doc comments.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "nexus-ai-pro",
-  "version": "2.20.2",
+  "version": "2.22.0",
   "description": "Typed AI framework for TypeScript: provider routing and failover, durable agent graphs and workflows, retrieval over any vector store, MCP, evals, guardrails, and cost control",
   "type": "module",
   "main": "dist-cjs/index.js",
@@ -270,6 +270,16 @@
         "default": "./dist-cjs/voice/providers/openai.js"
       }
     },
+    "./voice/google": {
+      "import": {
+        "types": "./dist/voice/providers/google.d.ts",
+        "default": "./dist/voice/providers/google.js"
+      },
+      "require": {
+        "types": "./dist-cjs/voice/providers/google.d.ts",
+        "default": "./dist-cjs/voice/providers/google.js"
+      }
+    },
     "./voice/session": {
       "import": {
         "types": "./dist/voice/session.d.ts",
@@ -2063,6 +2073,8 @@
     "conformance:images:record": "tsx tests/conformance.images.ts --record",
     "test:portable": "node scripts/check-portability.mjs && node tests/portable/run.mjs node && node tests/portable/edge.mjs",
     "test:browser": "node tests/browser.mjs",
+    "test:browser:firefox": "node tests/browser.mjs firefox",
+    "test:browser:webkit": "node tests/browser.mjs webkit",
     "test:browser:realtime": "node tests/browser-realtime.mjs",
     "bench:surfaces": "node --expose-gc --no-warnings scripts/bench-surfaces.mjs",
     "bench:streams": "node scripts/bench-asset-streams.mjs",
@@ -2197,6 +2209,7 @@
     "nexus-ai-pro-2-7": "npm:nexus-ai-pro@^2.7.0",
     "ollama": "^0.5.6",
     "openai": "^4.52.0",
+    "playwright-core": "^1.64.1",
     "tsx": "^4.16.0",
     "typescript": "^5.5.0",
     "wasmoon": "^1.16.0",
@@ -2311,6 +2324,9 @@
       "voice/openai": [
         "dist-cjs/voice/providers/openai.d.ts"
       ],
+      "voice/google": [
+        "dist-cjs/voice/providers/google.d.ts"
+      ],
       "voice/session": [
         "dist-cjs/voice/session.d.ts"
       ],