@xiaozhiclaw/provider-core 0.1.0

package/dist/transports/gemini-lyria-realtime.d.ts

@@ -0,0 +1,117 @@
+/**
+ * GeminiLyriaRealtimeSession 鈥?WebSocket-based real-time streaming music generation.
+ *
+ * Uses Lyria RealTime (`lyria-realtime-exp`) via the Gemini Live API WebSocket.
+ * Provides both a full interactive session API and a one-shot convenience method.
+ *
+ * Protocol:
+ *   - WebSocket URL: wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContent
+ *   - Auth: API key as query parameter
+ *   - Client 鈫?Server: setup, musicInput (weightedPrompts, musicGenerationConfig, playbackControl)
+ *   - Server 鈫?Client: serverContent.audioChunks (base64 PCM s16le, 48kHz, stereo)
+ *
+ * Docs: https://ai.google.dev/gemini-api/docs/realtime-music-generation
+ */
+export interface WeightedPrompt {
+    text: string;
+    weight: number;
+}
+export type MusicScale = "C_MAJOR_A_MINOR" | "D_FLAT_MAJOR_B_FLAT_MINOR" | "D_MAJOR_B_MINOR" | "E_FLAT_MAJOR_C_MINOR" | "E_MAJOR_D_FLAT_MINOR" | "F_MAJOR_D_MINOR" | "G_FLAT_MAJOR_E_FLAT_MINOR" | "G_MAJOR_E_MINOR" | "A_FLAT_MAJOR_F_MINOR" | "A_MAJOR_G_FLAT_MINOR" | "B_FLAT_MAJOR_G_MINOR" | "B_MAJOR_A_FLAT_MINOR" | "SCALE_UNSPECIFIED";
+export type MusicGenerationMode = "QUALITY" | "DIVERSITY" | "VOCALIZATION";
+export interface MusicGenerationConfig {
+    bpm?: number;
+    density?: number;
+    brightness?: number;
+    guidance?: number;
+    scale?: MusicScale;
+    temperature?: number;
+    topK?: number;
+    seed?: number;
+    muteBass?: boolean;
+    muteDrums?: boolean;
+    onlyBassAndDrums?: boolean;
+    musicGenerationMode?: MusicGenerationMode;
+    audioFormat?: string;
+    sampleRateHz?: number;
+}
+export interface LyriaRealtimeConfig {
+    /** Base URL (REST), e.g. "https://generativelanguage.googleapis.com/v1beta" */
+    baseUrl: string;
+}
+export interface LyriaRealtimeSessionOptions {
+    model?: string;
+    prompts: WeightedPrompt[];
+    config?: MusicGenerationConfig;
+}
+/** Audio chunk received from the server. */
+export interface AudioChunk {
+    /** Raw PCM s16le data (48kHz, stereo) */
+    data: Buffer;
+}
+/**
+ * Interactive Lyria RealTime session over WebSocket.
+ *
+ * Usage:
+ *   const session = new GeminiLyriaRealtimeSession({ baseUrl: "..." });
+ *   await session.connect(apiKey, { prompts: [{ text: "jazz", weight: 1 }] });
+ *   session.onAudioChunk = (chunk) => { ... };
+ *   await session.play();
+ *   // ... later
+ *   await session.stop();
+ *   session.close();
+ */
+export declare class GeminiLyriaRealtimeSession {
+    private ws;
+    private wsUrl;
+    private model;
+    /** Called for each audio chunk received from the server. */
+    onAudioChunk: ((chunk: AudioChunk) => void) | null;
+    /** Called when the server reports a filtered prompt. */
+    onFilteredPrompt: ((reason: string) => void) | null;
+    /** Called on WebSocket error. */
+    onError: ((error: Error) => void) | null;
+    /** Called when the WebSocket connection closes. */
+    onClose: (() => void) | null;
+    constructor(config: LyriaRealtimeConfig);
+    /**
+     * Connect to the Lyria RealTime WebSocket and send setup + initial config.
+     */
+    connect(apiKey: string, options: LyriaRealtimeSessionOptions): Promise<void>;
+    /** Set or update weighted prompts (smooth transition). */
+    setWeightedPrompts(prompts: WeightedPrompt[]): Promise<void>;
+    /** Set or update music generation config. */
+    setMusicGenerationConfig(config: MusicGenerationConfig): Promise<void>;
+    /** Start streaming music. */
+    play(): Promise<void>;
+    /** Pause music streaming (can resume with play). */
+    pause(): Promise<void>;
+    /** Stop music streaming (terminates the current piece). */
+    stop(): Promise<void>;
+    /** Reset model context (for BPM/scale changes). */
+    resetContext(): Promise<void>;
+    /** Close the WebSocket connection. */
+    close(): void;
+    /** Whether the session is connected. */
+    get connected(): boolean;
+    private send;
+    private handleMessage;
+}
+/**
+ * Generate a fixed-duration music clip using Lyria RealTime.
+ *
+ * Connects, plays for the specified duration (default 30s), collects all
+ * audio chunks, assembles into a WAV file, and returns the file path.
+ *
+ * Output: 48kHz, stereo, 16-bit PCM wrapped in WAV.
+ */
+export declare function generateRealtimeMusic(apiKey: string, config: LyriaRealtimeConfig, options: {
+    prompts: WeightedPrompt[];
+    durationSeconds?: number;
+    musicConfig?: MusicGenerationConfig;
+    model?: string;
+    signal?: AbortSignal;
+    onProgress?: (percent: number, status: string) => void;
+}): Promise<{
+    filePath: string;
+    durationMs: number;
+}>;

package/dist/transports/gemini-lyria-realtime.js

@@ -0,0 +1,295 @@
+/**
+ * GeminiLyriaRealtimeSession 鈥?WebSocket-based real-time streaming music generation.
+ *
+ * Uses Lyria RealTime (`lyria-realtime-exp`) via the Gemini Live API WebSocket.
+ * Provides both a full interactive session API and a one-shot convenience method.
+ *
+ * Protocol:
+ *   - WebSocket URL: wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContent
+ *   - Auth: API key as query parameter
+ *   - Client 鈫?Server: setup, musicInput (weightedPrompts, musicGenerationConfig, playbackControl)
+ *   - Server 鈫?Client: serverContent.audioChunks (base64 PCM s16le, 48kHz, stereo)
+ *
+ * Docs: https://ai.google.dev/gemini-api/docs/realtime-music-generation
+ */
+import { writeFileSync, mkdirSync } from "node:fs";
+import { join } from "node:path";
+import { randomUUID } from "node:crypto";
+import { getUserCacheDir } from "../paths.js";
+// 鈹€鈹€ Session 鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€
+/**
+ * Interactive Lyria RealTime session over WebSocket.
+ *
+ * Usage:
+ *   const session = new GeminiLyriaRealtimeSession({ baseUrl: "..." });
+ *   await session.connect(apiKey, { prompts: [{ text: "jazz", weight: 1 }] });
+ *   session.onAudioChunk = (chunk) => { ... };
+ *   await session.play();
+ *   // ... later
+ *   await session.stop();
+ *   session.close();
+ */
+export class GeminiLyriaRealtimeSession {
+    ws = null;
+    wsUrl;
+    model = "models/lyria-realtime-exp";
+    /** Called for each audio chunk received from the server. */
+    onAudioChunk = null;
+    /** Called when the server reports a filtered prompt. */
+    onFilteredPrompt = null;
+    /** Called on WebSocket error. */
+    onError = null;
+    /** Called when the WebSocket connection closes. */
+    onClose = null;
+    constructor(config) {
+        // Derive WebSocket URL from REST base URL
+        // "https://generativelanguage.googleapis.com/v1beta"
+        // 鈫?"wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContent"
+        const restBase = config.baseUrl.replace(/\/+$/, "");
+        const origin = new URL(restBase).origin.replace(/^http/, "ws");
+        this.wsUrl = `${origin}/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContent`;
+    }
+    /**
+     * Connect to the Lyria RealTime WebSocket and send setup + initial config.
+     */
+    async connect(apiKey, options) {
+        if (options.model) {
+            this.model = options.model.startsWith("models/")
+                ? options.model
+                : `models/${options.model}`;
+        }
+        const url = `${this.wsUrl}?key=${encodeURIComponent(apiKey)}`;
+        const ws = new WebSocket(url);
+        this.ws = ws;
+        await new Promise((resolve, reject) => {
+            const onOpen = () => {
+                ws.removeEventListener("error", onInitError);
+                resolve();
+            };
+            const onInitError = () => {
+                ws.removeEventListener("open", onOpen);
+                reject(new Error("Lyria RealTime WebSocket connection failed"));
+            };
+            ws.addEventListener("open", onOpen, { once: true });
+            ws.addEventListener("error", onInitError, { once: true });
+        });
+        // Wire message handling
+        ws.addEventListener("message", (ev) => this.handleMessage(ev));
+        ws.addEventListener("error", () => {
+            this.onError?.(new Error("Lyria RealTime WebSocket error"));
+        });
+        ws.addEventListener("close", () => {
+            this.ws = null;
+            this.onClose?.();
+        });
+        // Send setup message
+        this.send({
+            setup: { model: this.model },
+        });
+        // Wait a tick for setup acknowledgement
+        await new Promise(r => setTimeout(r, 100));
+        // Send initial prompts
+        await this.setWeightedPrompts(options.prompts);
+        // Send initial config
+        if (options.config) {
+            await this.setMusicGenerationConfig(options.config);
+        }
+    }
+    /** Set or update weighted prompts (smooth transition). */
+    async setWeightedPrompts(prompts) {
+        this.send({
+            musicInput: {
+                weightedPrompts: prompts.map(p => ({
+                    text: p.text,
+                    weight: p.weight,
+                })),
+            },
+        });
+    }
+    /** Set or update music generation config. */
+    async setMusicGenerationConfig(config) {
+        const mc = {};
+        if (config.bpm !== undefined)
+            mc.bpm = config.bpm;
+        if (config.density !== undefined)
+            mc.density = config.density;
+        if (config.brightness !== undefined)
+            mc.brightness = config.brightness;
+        if (config.guidance !== undefined)
+            mc.guidance = config.guidance;
+        if (config.scale !== undefined)
+            mc.scale = config.scale;
+        if (config.temperature !== undefined)
+            mc.temperature = config.temperature;
+        if (config.topK !== undefined)
+            mc.top_k = config.topK;
+        if (config.seed !== undefined)
+            mc.seed = config.seed;
+        if (config.muteBass !== undefined)
+            mc.mute_bass = config.muteBass;
+        if (config.muteDrums !== undefined)
+            mc.mute_drums = config.muteDrums;
+        if (config.onlyBassAndDrums !== undefined)
+            mc.only_bass_and_drums = config.onlyBassAndDrums;
+        if (config.musicGenerationMode !== undefined)
+            mc.music_generation_mode = config.musicGenerationMode;
+        if (config.audioFormat !== undefined)
+            mc.audio_format = config.audioFormat;
+        if (config.sampleRateHz !== undefined)
+            mc.sample_rate_hz = config.sampleRateHz;
+        this.send({
+            musicInput: { musicGenerationConfig: mc },
+        });
+    }
+    /** Start streaming music. */
+    async play() {
+        this.send({ musicInput: { playbackControl: "PLAY" } });
+    }
+    /** Pause music streaming (can resume with play). */
+    async pause() {
+        this.send({ musicInput: { playbackControl: "PAUSE" } });
+    }
+    /** Stop music streaming (terminates the current piece). */
+    async stop() {
+        this.send({ musicInput: { playbackControl: "STOP" } });
+    }
+    /** Reset model context (for BPM/scale changes). */
+    async resetContext() {
+        this.send({ musicInput: { playbackControl: "RESET_CONTEXT" } });
+    }
+    /** Close the WebSocket connection. */
+    close() {
+        if (this.ws && this.ws.readyState !== WebSocket.CLOSED) {
+            this.ws.close();
+        }
+        this.ws = null;
+    }
+    /** Whether the session is connected. */
+    get connected() {
+        return this.ws !== null && this.ws.readyState === WebSocket.OPEN;
+    }
+    // 鈹€鈹€ Private 鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€
+    send(msg) {
+        if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
+            throw new Error("Lyria RealTime session not connected");
+        }
+        this.ws.send(JSON.stringify(msg));
+    }
+    handleMessage(ev) {
+        let data;
+        try {
+            data = JSON.parse(typeof ev.data === "string" ? ev.data : ev.data.toString());
+        }
+        catch {
+            return;
+        }
+        const serverContent = data.serverContent;
+        if (!serverContent)
+            return;
+        // Audio chunks
+        const audioChunks = serverContent.audioChunks;
+        if (audioChunks && this.onAudioChunk) {
+            for (const chunk of audioChunks) {
+                if (chunk.data) {
+                    this.onAudioChunk({ data: Buffer.from(chunk.data, "base64") });
+                }
+            }
+        }
+        // Filtered prompt (safety)
+        const filtered = serverContent.filteredPrompt;
+        if (filtered) {
+            this.onFilteredPrompt?.(filtered);
+        }
+    }
+}
+// 鈹€鈹€ One-shot convenience: generate a fixed-duration music clip 鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€
+/**
+ * Generate a fixed-duration music clip using Lyria RealTime.
+ *
+ * Connects, plays for the specified duration (default 30s), collects all
+ * audio chunks, assembles into a WAV file, and returns the file path.
+ *
+ * Output: 48kHz, stereo, 16-bit PCM wrapped in WAV.
+ */
+export async function generateRealtimeMusic(apiKey, config, options) {
+    const durationSec = options.durationSeconds ?? 30;
+    const start = Date.now();
+    const session = new GeminiLyriaRealtimeSession(config);
+    // Collect PCM chunks
+    const pcmChunks = [];
+    session.onAudioChunk = (chunk) => {
+        pcmChunks.push(chunk.data);
+    };
+    let sessionError = null;
+    session.onError = (err) => {
+        sessionError = err;
+    };
+    try {
+        await session.connect(apiKey, {
+            model: options.model ?? "lyria-realtime-exp",
+            prompts: options.prompts,
+            config: options.musicConfig,
+        });
+        options.onProgress?.(5, "connected");
+        // Start playback
+        await session.play();
+        options.onProgress?.(10, "streaming");
+        // Collect audio for the specified duration
+        const deadline = Date.now() + durationSec * 1000;
+        const pollInterval = 500;
+        while (Date.now() < deadline) {
+            options.signal?.throwIfAborted();
+            if (sessionError)
+                throw sessionError;
+            if (!session.connected)
+                break;
+            const elapsed = Date.now() - start;
+            const totalExpected = durationSec * 1000;
+            const pct = Math.min(95, Math.round(10 + (elapsed / totalExpected) * 85));
+            options.onProgress?.(pct, "streaming");
+            await new Promise(r => setTimeout(r, pollInterval));
+        }
+        // Stop playback
+        if (session.connected) {
+            await session.stop();
+        }
+        options.onProgress?.(98, "assembling");
+        // Assemble PCM 鈫?WAV
+        const pcm = Buffer.concat(pcmChunks);
+        const wav = wrapPcmAsWav(pcm, 48000, 2, 16);
+        const cacheDir = join(getUserCacheDir(), "gemini-music-realtime");
+        mkdirSync(cacheDir, { recursive: true });
+        const filename = `lyria-rt-${randomUUID()}.wav`;
+        const filePath = join(cacheDir, filename);
+        writeFileSync(filePath, wav);
+        options.onProgress?.(100, "completed");
+        return {
+            filePath: `file://${filePath}`,
+            durationMs: Date.now() - start,
+        };
+    }
+    finally {
+        session.close();
+    }
+}
+// 鈹€鈹€ WAV helper 鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€
+function wrapPcmAsWav(pcm, sampleRate, channels, bitsPerSample) {
+    const byteRate = sampleRate * channels * (bitsPerSample / 8);
+    const blockAlign = channels * (bitsPerSample / 8);
+    const dataSize = pcm.length;
+    const header = Buffer.alloc(44);
+    header.write("RIFF", 0);
+    header.writeUInt32LE(36 + dataSize, 4);
+    header.write("WAVE", 8);
+    header.write("fmt ", 12);
+    header.writeUInt32LE(16, 16);
+    header.writeUInt16LE(1, 20);
+    header.writeUInt16LE(channels, 22);
+    header.writeUInt32LE(sampleRate, 24);
+    header.writeUInt32LE(byteRate, 28);
+    header.writeUInt16LE(blockAlign, 30);
+    header.writeUInt16LE(bitsPerSample, 32);
+    header.write("data", 36);
+    header.writeUInt32LE(dataSize, 40);
+    return Buffer.concat([header, pcm]);
+}

package/dist/transports/gemini-media.d.ts

@@ -0,0 +1,53 @@
+/**
+ * Gemini Media Transport 鈥?unified media generation for all Gemini media APIs.
+ *
+ * Supported media types and endpoints:
+ *   image          鈥?POST /models/{model}:generateContent  (responseModalities: ["TEXT","IMAGE"])
+ *   video          鈥?POST /models/{model}:predictLongRunning 鈫?poll operations 鈫?download URI
+ *   music          鈥?POST /models/{model}:generateContent  (Lyria 3 鈥?inlineData audio)
+ *   music_realtime 鈥?WebSocket session (Lyria RealTime 鈥?streaming PCM 鈫?WAV)
+ *   tts            鈥?POST /models/{model}:generateContent  (speechConfig 鈥?inlineData PCM)
+ *   embedding      鈥?POST /models/{model}:embedContent   (float vector)
+ *
+ * Auth: x-goog-api-key header for all endpoints.
+ */
+import type { AsyncMediaTransport, MediaRequest, MediaResult, MediaType } from "../media-transport.js";
+export interface GeminiMediaConfig {
+    /** Base URL, e.g. "https://generativelanguage.googleapis.com/v1beta" */
+    baseUrl: string;
+    timeoutMs?: number;
+}
+export declare class GeminiMediaTransport implements AsyncMediaTransport {
+    readonly supportedTypes: readonly MediaType[];
+    private apiBase;
+    private timeoutMs;
+    constructor(config: GeminiMediaConfig);
+    generate(request: MediaRequest, apiKey: string, signal?: AbortSignal): Promise<MediaResult>;
+    deleteVideoTask(_taskId: string, _apiKey: string, _signal?: AbortSignal): Promise<void>;
+    listVideoTasks(_apiKey: string, _options?: {
+        after?: string;
+        limit?: number;
+        status?: string;
+    }, _signal?: AbortSignal): Promise<Record<string, unknown>>;
+    getTaskStatus(taskId: string, apiKey: string, signal?: AbortSignal): Promise<{
+        status: string;
+        task: Record<string, unknown>;
+    }>;
+    private generateImage;
+    private generateVideo;
+    private generateMusic;
+    private generateMusicRealtime;
+    private generateTTS;
+    private generateEmbedding;
+    private postJson;
+    private pollOperation;
+    /**
+     * Resolve an image URL to inline data for the Veo API.
+     * Supports file:// paths and https:// URLs.
+     */
+    private resolveImageData;
+    /** Extract base64 image data from generateContent response 鈫?persist to cache files. */
+    private extractInlineImages;
+    /** Extract base64 audio data from generateContent response 鈫?persist to cache files. */
+    private extractInlineAudio;
+}