@open-gitagent/voice 1.0.0

package/dist/gemini-live.js

@@ -0,0 +1,279 @@
+import WebSocket from "ws";
+import { DEFAULT_VOICE_INSTRUCTIONS, } from "@open-gitagent/gitagent";
+const dim = (s) => `\x1b[2m${s}\x1b[0m`;
+/**
+ * Downsample 24kHz PCM (Int16LE) to 16kHz by linear interpolation (2 of every 3 samples).
+ * Input: base64-encoded 24kHz Int16LE. Output: base64-encoded 16kHz Int16LE.
+ */
+function downsample24kTo16k(base64_24k) {
+    const binary = Buffer.from(base64_24k, "base64");
+    const samples24 = new Int16Array(binary.buffer, binary.byteOffset, binary.byteLength / 2);
+    const outLength = Math.floor(samples24.length * 2 / 3);
+    const samples16 = new Int16Array(outLength);
+    for (let i = 0; i < outLength; i++) {
+        // Map output index to fractional input index
+        const srcIdx = i * 1.5;
+        const lo = Math.floor(srcIdx);
+        const frac = srcIdx - lo;
+        const hi = Math.min(lo + 1, samples24.length - 1);
+        samples16[i] = Math.round(samples24[lo] * (1 - frac) + samples24[hi] * frac);
+    }
+    return Buffer.from(samples16.buffer).toString("base64");
+}
+/**
+ * Upsample 16kHz PCM (Int16LE) to 24kHz by linear interpolation.
+ * Input: base64-encoded 16kHz Int16LE. Output: base64-encoded 24kHz Int16LE.
+ */
+function upsample16kTo24k(base64_16k) {
+    const binary = Buffer.from(base64_16k, "base64");
+    const samples16 = new Int16Array(binary.buffer, binary.byteOffset, binary.byteLength / 2);
+    const outLength = Math.floor(samples16.length * 3 / 2);
+    const samples24 = new Int16Array(outLength);
+    for (let i = 0; i < outLength; i++) {
+        const srcIdx = i * (2 / 3);
+        const lo = Math.floor(srcIdx);
+        const frac = srcIdx - lo;
+        const hi = Math.min(lo + 1, samples16.length - 1);
+        samples24[i] = Math.round(samples16[lo] * (1 - frac) + samples16[hi] * frac);
+    }
+    return Buffer.from(samples24.buffer).toString("base64");
+}
+export class GeminiLiveAdapter {
+    ws = null;
+    config;
+    onMessage = null;
+    toolHandler = null;
+    setupDone = false;
+    constructor(config) {
+        this.config = config;
+    }
+    async connect(opts) {
+        this.onMessage = opts.onMessage;
+        this.toolHandler = opts.toolHandler;
+        this.setupDone = false;
+        const model = this.config.model || "models/gemini-2.5-flash-native-audio-preview";
+        const url = `wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent?key=${this.config.apiKey}`;
+        return new Promise((resolve, reject) => {
+            this.ws = new WebSocket(url);
+            this.ws.on("open", () => {
+                console.log(dim("[voice] Connected to Gemini Multimodal Live"));
+                this.sendSetup(model);
+            });
+            this.ws.on("error", (err) => {
+                console.error(dim(`[voice] Gemini WS error: ${err.message}`));
+                if (!this.setupDone) {
+                    reject(err);
+                }
+                else {
+                    this.emit({ type: "error", message: err.message });
+                }
+            });
+            this.ws.on("close", () => {
+                console.log(dim("[voice] Gemini WS closed"));
+            });
+            this.ws.on("message", (data) => {
+                try {
+                    const msg = JSON.parse(data.toString());
+                    this.handleGeminiMessage(msg);
+                    // Resolve after setup acknowledgment
+                    if (!this.setupDone && msg.setupComplete) {
+                        this.setupDone = true;
+                        console.log(dim("[voice] Gemini session ready"));
+                        resolve();
+                    }
+                }
+                catch (err) {
+                    console.error(dim(`[voice] Gemini parse error: ${err.message}`));
+                }
+            });
+        });
+    }
+    send(msg) {
+        switch (msg.type) {
+            case "audio":
+                // Browser sends 24kHz, Gemini expects 16kHz
+                this.sendRaw({
+                    realtimeInput: {
+                        mediaChunks: [{
+                                mimeType: "audio/pcm;rate=16000",
+                                data: downsample24kTo16k(msg.audio),
+                            }],
+                    },
+                });
+                break;
+            case "video_frame":
+                // Gemini supports continuous video streaming natively
+                this.sendRaw({
+                    realtimeInput: {
+                        mediaChunks: [{
+                                mimeType: msg.mimeType,
+                                data: msg.frame,
+                            }],
+                    },
+                });
+                break;
+            case "text":
+                this.sendRaw({
+                    clientContent: {
+                        turns: [{
+                                role: "user",
+                                parts: [{ text: msg.text }],
+                            }],
+                        turnComplete: true,
+                    },
+                });
+                break;
+            case "file": {
+                const parts = [];
+                if (msg.mimeType.startsWith("image/")) {
+                    parts.push({ inlineData: { mimeType: msg.mimeType, data: msg.data } });
+                    parts.push({ text: msg.text || `[User attached image: ${msg.name}]` });
+                }
+                else {
+                    const decoded = Buffer.from(msg.data, "base64").toString("utf-8");
+                    const label = msg.text ? `${msg.text}\n\n` : "";
+                    parts.push({ text: `${label}[File: ${msg.name}]\n\`\`\`\n${decoded}\n\`\`\`` });
+                }
+                this.sendRaw({
+                    clientContent: {
+                        turns: [{ role: "user", parts }],
+                        turnComplete: true,
+                    },
+                });
+                break;
+            }
+        }
+    }
+    async disconnect() {
+        if (this.ws) {
+            this.ws.close();
+            this.ws = null;
+        }
+    }
+    emit(msg) {
+        this.onMessage?.(msg);
+    }
+    sendSetup(model) {
+        const instructions = this.config.instructions || DEFAULT_VOICE_INSTRUCTIONS;
+        const voiceName = this.config.voice || "Aoede";
+        this.sendRaw({
+            setup: {
+                model,
+                generationConfig: {
+                    responseModalities: ["AUDIO", "TEXT"],
+                    speechConfig: {
+                        voiceConfig: {
+                            prebuiltVoiceConfig: { voiceName },
+                        },
+                    },
+                },
+                tools: [{
+                        functionDeclarations: [{
+                                name: "run_agent",
+                                description: "Execute any request through the gitagent agent. It has full access to the terminal (can run any shell command, open apps, install packages), file system (read/write/create files), git operations, and persistent memory. Use this for ALL actionable requests. IMPORTANT: If the user uploaded a file, always include the file path (from the '[File saved to: ...]' annotation) in the query.",
+                                parameters: {
+                                    type: "OBJECT",
+                                    properties: {
+                                        query: {
+                                            type: "STRING",
+                                            description: "The user's request. MUST include file paths when referencing uploaded files (e.g. 'make a game using the image at workspace/lobster.png').",
+                                        },
+                                    },
+                                    required: ["query"],
+                                },
+                            }],
+                    }],
+                systemInstruction: {
+                    parts: [{ text: instructions }],
+                },
+                contextWindowCompression: {
+                    triggerTokens: 25000,
+                    slidingWindow: { targetTokens: 12500 },
+                },
+            },
+        });
+    }
+    handleGeminiMessage(msg) {
+        // Tool calls
+        if (msg.toolCall) {
+            this.handleToolCall(msg.toolCall);
+            return;
+        }
+        // Server content (audio/text responses)
+        if (msg.serverContent) {
+            const sc = msg.serverContent;
+            // Model turn parts
+            if (sc.modelTurn?.parts) {
+                for (const part of sc.modelTurn.parts) {
+                    if (part.inlineData) {
+                        const mimeType = part.inlineData.mimeType || "";
+                        if (mimeType.startsWith("audio/")) {
+                            // Gemini outputs 16kHz, browser expects 24kHz
+                            const audio24k = upsample16kTo24k(part.inlineData.data);
+                            this.emit({ type: "audio_delta", audio: audio24k });
+                        }
+                    }
+                    if (part.text) {
+                        this.emit({
+                            type: "transcript",
+                            role: "assistant",
+                            text: part.text,
+                            partial: !sc.turnComplete,
+                        });
+                    }
+                }
+            }
+            // Turn complete marker
+            if (sc.turnComplete && sc.modelTurn?.parts) {
+                const textParts = sc.modelTurn.parts.filter((p) => p.text).map((p) => p.text);
+                if (textParts.length > 0) {
+                    this.emit({ type: "transcript", role: "assistant", text: textParts.join("") });
+                }
+            }
+            // Input transcription
+            if (sc.inputTranscription?.text) {
+                console.log(dim(`[voice] User: ${sc.inputTranscription.text}`));
+                this.emit({ type: "transcript", role: "user", text: sc.inputTranscription.text });
+            }
+        }
+    }
+    async handleToolCall(toolCall) {
+        if (!this.toolHandler)
+            return;
+        const functionCalls = toolCall.functionCalls || [];
+        const responses = [];
+        for (const fc of functionCalls) {
+            if (fc.name !== "run_agent") {
+                console.error(dim(`[voice] Unknown Gemini function call: ${fc.name}`));
+                responses.push({ id: fc.id, name: fc.name, response: { error: `Unknown function: ${fc.name}` } });
+                continue;
+            }
+            const queryArg = fc.args?.query;
+            if (!queryArg) {
+                responses.push({ id: fc.id, name: fc.name, response: { error: "Missing query argument" } });
+                continue;
+            }
+            console.log(dim(`[voice] Agent query: ${queryArg}`));
+            this.emit({ type: "agent_working", query: queryArg });
+            try {
+                const result = await this.toolHandler(queryArg);
+                console.log(dim(`[voice] Agent response: ${result.slice(0, 200)}${result.length > 200 ? "..." : ""}`));
+                responses.push({ id: fc.id, name: fc.name, response: { result } });
+                this.emit({ type: "agent_done", result: result.slice(0, 500) });
+            }
+            catch (err) {
+                console.error(dim(`[voice] Agent error: ${err.message}`));
+                responses.push({ id: fc.id, name: fc.name, response: { error: err.message } });
+                this.emit({ type: "error", message: err.message });
+            }
+        }
+        this.sendRaw({
+            toolResponse: { functionResponses: responses },
+        });
+    }
+    sendRaw(msg) {
+        if (this.ws && this.ws.readyState === WebSocket.OPEN) {
+            this.ws.send(JSON.stringify(msg));
+        }
+    }
+}

package/dist/index.d.ts

@@ -0,0 +1,4 @@
+export type { VoiceAdapter, VoiceAdapterConfig, VoiceServerOptions, MultimodalAdapter, MultimodalAdapterConfig, AdapterBackend, ClientMessage, ServerMessage, } from "@open-gitagent/gitagent";
+export { OpenAIRealtimeAdapter } from "./openai-realtime.js";
+export { GeminiLiveAdapter } from "./gemini-live.js";
+export { startVoiceServer } from "./server.js";

package/dist/index.js

@@ -0,0 +1,3 @@
+export { OpenAIRealtimeAdapter } from "./openai-realtime.js";
+export { GeminiLiveAdapter } from "./gemini-live.js";
+export { startVoiceServer } from "./server.js";

package/dist/openai-realtime.d.ts

@@ -0,0 +1,40 @@
+import { type MultimodalAdapter, type MultimodalAdapterConfig, type ClientMessage, type ServerMessage } from "@open-gitagent/gitagent";
+export declare class OpenAIRealtimeAdapter implements MultimodalAdapter {
+    private ws;
+    private config;
+    private latestVideoFrame;
+    private latestScreenFrame;
+    private onMessage;
+    private toolHandler;
+    private interrupted;
+    private refreshTimer;
+    private refreshing;
+    private disposed;
+    private static readonly REFRESH_AFTER_MS;
+    constructor(config: MultimodalAdapterConfig);
+    connect(opts: {
+        toolHandler: (query: string) => Promise<string>;
+        onMessage: (msg: ServerMessage) => void;
+    }): Promise<void>;
+    private connectWs;
+    /** Send session.update on a specific ws instance (before this.ws is set). */
+    private sendSessionUpdateOn;
+    send(msg: ClientMessage): void;
+    disconnect(): Promise<void>;
+    /**
+     * Tear down and reopen the Realtime WS before (or right after) OpenAI's
+     * 60-minute hard cap expires. Re-sends the stored session.update so the
+     * agent picks up where it left off without the user noticing.
+     */
+    private refreshSession;
+    private emit;
+    /**
+     * Inject the latest video frame as a conversation item so the model
+     * can see it when generating the next response (e.g. after a voice turn).
+     */
+    private injectVideoFrame;
+    private sendSessionUpdate;
+    private handleEvent;
+    private handleFunctionCall;
+    private sendRaw;
+}