gitclaw 0.3.1 → 0.4.1

package/dist/voice/gemini-live.js

@@ -0,0 +1,279 @@
+import WebSocket from "ws";
+import { DEFAULT_VOICE_INSTRUCTIONS, } from "./adapter.js";
+const dim = (s) => `\x1b[2m${s}\x1b[0m`;
+/**
+ * Downsample 24kHz PCM (Int16LE) to 16kHz by linear interpolation (2 of every 3 samples).
+ * Input: base64-encoded 24kHz Int16LE. Output: base64-encoded 16kHz Int16LE.
+ */
+function downsample24kTo16k(base64_24k) {
+    const binary = Buffer.from(base64_24k, "base64");
+    const samples24 = new Int16Array(binary.buffer, binary.byteOffset, binary.byteLength / 2);
+    const outLength = Math.floor(samples24.length * 2 / 3);
+    const samples16 = new Int16Array(outLength);
+    for (let i = 0; i < outLength; i++) {
+        // Map output index to fractional input index
+        const srcIdx = i * 1.5;
+        const lo = Math.floor(srcIdx);
+        const frac = srcIdx - lo;
+        const hi = Math.min(lo + 1, samples24.length - 1);
+        samples16[i] = Math.round(samples24[lo] * (1 - frac) + samples24[hi] * frac);
+    }
+    return Buffer.from(samples16.buffer).toString("base64");
+}
+/**
+ * Upsample 16kHz PCM (Int16LE) to 24kHz by linear interpolation.
+ * Input: base64-encoded 16kHz Int16LE. Output: base64-encoded 24kHz Int16LE.
+ */
+function upsample16kTo24k(base64_16k) {
+    const binary = Buffer.from(base64_16k, "base64");
+    const samples16 = new Int16Array(binary.buffer, binary.byteOffset, binary.byteLength / 2);
+    const outLength = Math.floor(samples16.length * 3 / 2);
+    const samples24 = new Int16Array(outLength);
+    for (let i = 0; i < outLength; i++) {
+        const srcIdx = i * (2 / 3);
+        const lo = Math.floor(srcIdx);
+        const frac = srcIdx - lo;
+        const hi = Math.min(lo + 1, samples16.length - 1);
+        samples24[i] = Math.round(samples16[lo] * (1 - frac) + samples16[hi] * frac);
+    }
+    return Buffer.from(samples24.buffer).toString("base64");
+}
+export class GeminiLiveAdapter {
+    ws = null;
+    config;
+    onMessage = null;
+    toolHandler = null;
+    setupDone = false;
+    constructor(config) {
+        this.config = config;
+    }
+    async connect(opts) {
+        this.onMessage = opts.onMessage;
+        this.toolHandler = opts.toolHandler;
+        this.setupDone = false;
+        const model = this.config.model || "models/gemini-2.5-flash-native-audio-preview";
+        const url = `wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent?key=${this.config.apiKey}`;
+        return new Promise((resolve, reject) => {
+            this.ws = new WebSocket(url);
+            this.ws.on("open", () => {
+                console.log(dim("[voice] Connected to Gemini Multimodal Live"));
+                this.sendSetup(model);
+            });
+            this.ws.on("error", (err) => {
+                console.error(dim(`[voice] Gemini WS error: ${err.message}`));
+                if (!this.setupDone) {
+                    reject(err);
+                }
+                else {
+                    this.emit({ type: "error", message: err.message });
+                }
+            });
+            this.ws.on("close", () => {
+                console.log(dim("[voice] Gemini WS closed"));
+            });
+            this.ws.on("message", (data) => {
+                try {
+                    const msg = JSON.parse(data.toString());
+                    this.handleGeminiMessage(msg);
+                    // Resolve after setup acknowledgment
+                    if (!this.setupDone && msg.setupComplete) {
+                        this.setupDone = true;
+                        console.log(dim("[voice] Gemini session ready"));
+                        resolve();
+                    }
+                }
+                catch (err) {
+                    console.error(dim(`[voice] Gemini parse error: ${err.message}`));
+                }
+            });
+        });
+    }
+    send(msg) {
+        switch (msg.type) {
+            case "audio":
+                // Browser sends 24kHz, Gemini expects 16kHz
+                this.sendRaw({
+                    realtimeInput: {
+                        mediaChunks: [{
+                                mimeType: "audio/pcm;rate=16000",
+                                data: downsample24kTo16k(msg.audio),
+                            }],
+                    },
+                });
+                break;
+            case "video_frame":
+                // Gemini supports continuous video streaming natively
+                this.sendRaw({
+                    realtimeInput: {
+                        mediaChunks: [{
+                                mimeType: msg.mimeType,
+                                data: msg.frame,
+                            }],
+                    },
+                });
+                break;
+            case "text":
+                this.sendRaw({
+                    clientContent: {
+                        turns: [{
+                                role: "user",
+                                parts: [{ text: msg.text }],
+                            }],
+                        turnComplete: true,
+                    },
+                });
+                break;
+            case "file": {
+                const parts = [];
+                if (msg.mimeType.startsWith("image/")) {
+                    parts.push({ inlineData: { mimeType: msg.mimeType, data: msg.data } });
+                    parts.push({ text: msg.text || `[User attached image: ${msg.name}]` });
+                }
+                else {
+                    const decoded = Buffer.from(msg.data, "base64").toString("utf-8");
+                    const label = msg.text ? `${msg.text}\n\n` : "";
+                    parts.push({ text: `${label}[File: ${msg.name}]\n\`\`\`\n${decoded}\n\`\`\`` });
+                }
+                this.sendRaw({
+                    clientContent: {
+                        turns: [{ role: "user", parts }],
+                        turnComplete: true,
+                    },
+                });
+                break;
+            }
+        }
+    }
+    async disconnect() {
+        if (this.ws) {
+            this.ws.close();
+            this.ws = null;
+        }
+    }
+    emit(msg) {
+        this.onMessage?.(msg);
+    }
+    sendSetup(model) {
+        const instructions = this.config.instructions || DEFAULT_VOICE_INSTRUCTIONS;
+        const voiceName = this.config.voice || "Aoede";
+        this.sendRaw({
+            setup: {
+                model,
+                generationConfig: {
+                    responseModalities: ["AUDIO", "TEXT"],
+                    speechConfig: {
+                        voiceConfig: {
+                            prebuiltVoiceConfig: { voiceName },
+                        },
+                    },
+                },
+                tools: [{
+                        functionDeclarations: [{
+                                name: "run_agent",
+                                description: "Execute any request through the gitclaw agent. It has full access to the terminal (can run any shell command, open apps, install packages), file system (read/write/create files), git operations, and persistent memory. Use this for ALL actionable requests. IMPORTANT: If the user uploaded a file, always include the file path (from the '[File saved to: ...]' annotation) in the query.",
+                                parameters: {
+                                    type: "OBJECT",
+                                    properties: {
+                                        query: {
+                                            type: "STRING",
+                                            description: "The user's request. MUST include file paths when referencing uploaded files (e.g. 'make a game using the image at workspace/lobster.png').",
+                                        },
+                                    },
+                                    required: ["query"],
+                                },
+                            }],
+                    }],
+                systemInstruction: {
+                    parts: [{ text: instructions }],
+                },
+                contextWindowCompression: {
+                    triggerTokens: 25000,
+                    slidingWindow: { targetTokens: 12500 },
+                },
+            },
+        });
+    }
+    handleGeminiMessage(msg) {
+        // Tool calls
+        if (msg.toolCall) {
+            this.handleToolCall(msg.toolCall);
+            return;
+        }
+        // Server content (audio/text responses)
+        if (msg.serverContent) {
+            const sc = msg.serverContent;
+            // Model turn parts
+            if (sc.modelTurn?.parts) {
+                for (const part of sc.modelTurn.parts) {
+                    if (part.inlineData) {
+                        const mimeType = part.inlineData.mimeType || "";
+                        if (mimeType.startsWith("audio/")) {
+                            // Gemini outputs 16kHz, browser expects 24kHz
+                            const audio24k = upsample16kTo24k(part.inlineData.data);
+                            this.emit({ type: "audio_delta", audio: audio24k });
+                        }
+                    }
+                    if (part.text) {
+                        this.emit({
+                            type: "transcript",
+                            role: "assistant",
+                            text: part.text,
+                            partial: !sc.turnComplete,
+                        });
+                    }
+                }
+            }
+            // Turn complete marker
+            if (sc.turnComplete && sc.modelTurn?.parts) {
+                const textParts = sc.modelTurn.parts.filter((p) => p.text).map((p) => p.text);
+                if (textParts.length > 0) {
+                    this.emit({ type: "transcript", role: "assistant", text: textParts.join("") });
+                }
+            }
+            // Input transcription
+            if (sc.inputTranscription?.text) {
+                console.log(dim(`[voice] User: ${sc.inputTranscription.text}`));
+                this.emit({ type: "transcript", role: "user", text: sc.inputTranscription.text });
+            }
+        }
+    }
+    async handleToolCall(toolCall) {
+        if (!this.toolHandler)
+            return;
+        const functionCalls = toolCall.functionCalls || [];
+        const responses = [];
+        for (const fc of functionCalls) {
+            if (fc.name !== "run_agent") {
+                console.error(dim(`[voice] Unknown Gemini function call: ${fc.name}`));
+                responses.push({ id: fc.id, name: fc.name, response: { error: `Unknown function: ${fc.name}` } });
+                continue;
+            }
+            const queryArg = fc.args?.query;
+            if (!queryArg) {
+                responses.push({ id: fc.id, name: fc.name, response: { error: "Missing query argument" } });
+                continue;
+            }
+            console.log(dim(`[voice] Agent query: ${queryArg}`));
+            this.emit({ type: "agent_working", query: queryArg });
+            try {
+                const result = await this.toolHandler(queryArg);
+                console.log(dim(`[voice] Agent response: ${result.slice(0, 200)}${result.length > 200 ? "..." : ""}`));
+                responses.push({ id: fc.id, name: fc.name, response: { result } });
+                this.emit({ type: "agent_done", result: result.slice(0, 500) });
+            }
+            catch (err) {
+                console.error(dim(`[voice] Agent error: ${err.message}`));
+                responses.push({ id: fc.id, name: fc.name, response: { error: err.message } });
+                this.emit({ type: "error", message: err.message });
+            }
+        }
+        this.sendRaw({
+            toolResponse: { functionResponses: responses },
+        });
+    }
+    sendRaw(msg) {
+        if (this.ws && this.ws.readyState === WebSocket.OPEN) {
+            this.ws.send(JSON.stringify(msg));
+        }
+    }
+}

package/dist/voice/index.d.ts

@@ -0,0 +1,4 @@
+export type { VoiceAdapter, VoiceAdapterConfig, VoiceServerOptions, MultimodalAdapter, MultimodalAdapterConfig, AdapterBackend, ClientMessage, ServerMessage, } from "./adapter.js";
+export { OpenAIRealtimeAdapter } from "./openai-realtime.js";
+export { GeminiLiveAdapter } from "./gemini-live.js";
+export { startVoiceServer } from "./server.js";

package/dist/voice/index.js

@@ -0,0 +1,3 @@
+export { OpenAIRealtimeAdapter } from "./openai-realtime.js";
+export { GeminiLiveAdapter } from "./gemini-live.js";
+export { startVoiceServer } from "./server.js";

package/dist/voice/openai-realtime.d.ts

@@ -0,0 +1,27 @@
+import { type MultimodalAdapter, type MultimodalAdapterConfig, type ClientMessage, type ServerMessage } from "./adapter.js";
+export declare class OpenAIRealtimeAdapter implements MultimodalAdapter {
+    private ws;
+    private config;
+    private latestVideoFrame;
+    private latestScreenFrame;
+    private onMessage;
+    private toolHandler;
+    private interrupted;
+    constructor(config: MultimodalAdapterConfig);
+    connect(opts: {
+        toolHandler: (query: string) => Promise<string>;
+        onMessage: (msg: ServerMessage) => void;
+    }): Promise<void>;
+    send(msg: ClientMessage): void;
+    disconnect(): Promise<void>;
+    private emit;
+    /**
+     * Inject the latest video frame as a conversation item so the model
+     * can see it when generating the next response (e.g. after a voice turn).
+     */
+    private injectVideoFrame;
+    private sendSessionUpdate;
+    private handleEvent;
+    private handleFunctionCall;
+    private sendRaw;
+}

package/dist/voice/openai-realtime.js

@@ -0,0 +1,291 @@
+import WebSocket from "ws";
+import { DEFAULT_VOICE_INSTRUCTIONS, } from "./adapter.js";
+const dim = (s) => `\x1b[2m${s}\x1b[0m`;
+export class OpenAIRealtimeAdapter {
+    ws = null;
+    config;
+    latestVideoFrame = null;
+    latestScreenFrame = null;
+    onMessage = null;
+    toolHandler = null;
+    interrupted = false;
+    constructor(config) {
+        this.config = config;
+    }
+    async connect(opts) {
+        this.onMessage = opts.onMessage;
+        this.toolHandler = opts.toolHandler;
+        const model = this.config.model || "gpt-realtime";
+        const url = `wss://api.openai.com/v1/realtime?model=${model}`;
+        return new Promise((resolve, reject) => {
+            this.ws = new WebSocket(url, {
+                headers: {
+                    Authorization: `Bearer ${this.config.apiKey}`,
+                    "OpenAI-Beta": "realtime=v1",
+                },
+            });
+            this.ws.on("open", () => {
+                this.sendSessionUpdate();
+                resolve();
+            });
+            this.ws.on("error", (err) => {
+                if (!this.ws) {
+                    reject(err);
+                }
+                else {
+                    console.error(dim(`[voice] WebSocket error: ${err.message}`));
+                    this.emit({ type: "error", message: err.message });
+                }
+            });
+            this.ws.on("close", () => {
+                console.log(dim("[voice] WebSocket closed"));
+            });
+            this.ws.on("message", (data) => {
+                const event = JSON.parse(data.toString());
+                this.handleEvent(event);
+            });
+        });
+    }
+    send(msg) {
+        switch (msg.type) {
+            case "audio":
+                this.sendRaw({
+                    type: "input_audio_buffer.append",
+                    audio: msg.audio,
+                });
+                break;
+            case "video_frame": {
+                // OpenAI doesn't support continuous video. Store latest frame and
+                // inject it as an image on the next user turn via conversation item.
+                const source = msg.source || "camera";
+                if (source === "screen") {
+                    this.latestScreenFrame = { frame: msg.frame, mimeType: msg.mimeType };
+                }
+                else {
+                    this.latestVideoFrame = { frame: msg.frame, mimeType: msg.mimeType };
+                }
+                break;
+            }
+            case "text": {
+                // Send text as a user conversation item, optionally with latest video frame
+                const content = [];
+                if (this.latestVideoFrame) {
+                    content.push({
+                        type: "input_image",
+                        image_url: `data:${this.latestVideoFrame.mimeType};base64,${this.latestVideoFrame.frame}`,
+                    });
+                    this.latestVideoFrame = null;
+                }
+                content.push({ type: "input_text", text: msg.text });
+                this.sendRaw({
+                    type: "conversation.item.create",
+                    item: {
+                        type: "message",
+                        role: "user",
+                        content,
+                    },
+                });
+                this.sendRaw({ type: "response.create" });
+                break;
+            }
+            case "file": {
+                const content = [];
+                if (msg.mimeType.startsWith("image/")) {
+                    content.push({
+                        type: "input_image",
+                        image_url: `data:${msg.mimeType};base64,${msg.data}`,
+                    });
+                    content.push({ type: "input_text", text: msg.text || `[User attached image: ${msg.name}]` });
+                }
+                else {
+                    const decoded = Buffer.from(msg.data, "base64").toString("utf-8");
+                    const label = msg.text ? `${msg.text}\n\n` : "";
+                    content.push({ type: "input_text", text: `${label}[File: ${msg.name}]\n\`\`\`\n${decoded}\n\`\`\`` });
+                }
+                this.sendRaw({
+                    type: "conversation.item.create",
+                    item: { type: "message", role: "user", content },
+                });
+                this.sendRaw({ type: "response.create" });
+                break;
+            }
+        }
+    }
+    async disconnect() {
+        if (this.ws) {
+            this.ws.close();
+            this.ws = null;
+        }
+    }
+    emit(msg) {
+        this.onMessage?.(msg);
+    }
+    /**
+     * Inject the latest video frame as a conversation item so the model
+     * can see it when generating the next response (e.g. after a voice turn).
+     */
+    injectVideoFrame() {
+        // Prefer screen frame over camera — it provides more useful context
+        const isScreen = !!this.latestScreenFrame;
+        const frame = this.latestScreenFrame || this.latestVideoFrame;
+        if (!frame)
+            return;
+        // Clear both so we don't inject stale frames
+        this.latestScreenFrame = null;
+        this.latestVideoFrame = null;
+        console.log(dim(`[voice] Injecting ${isScreen ? "screen" : "camera"} frame into conversation`));
+        this.sendRaw({
+            type: "conversation.item.create",
+            item: {
+                type: "message",
+                role: "user",
+                content: [{
+                        type: "input_image",
+                        image_url: `data:${frame.mimeType};base64,${frame.frame}`,
+                    }],
+            },
+        });
+    }
+    sendSessionUpdate() {
+        const instructions = this.config.instructions || DEFAULT_VOICE_INSTRUCTIONS;
+        this.sendRaw({
+            type: "session.update",
+            session: {
+                instructions,
+                voice: this.config.voice || "ash",
+                modalities: ["text", "audio"],
+                turn_detection: {
+                    type: "server_vad",
+                    threshold: 0.6,
+                    prefix_padding_ms: 400,
+                    silence_duration_ms: 800,
+                    create_response: true,
+                },
+                input_audio_transcription: { model: "whisper-1" },
+                tool_choice: "auto",
+                tools: [
+                    {
+                        type: "function",
+                        name: "run_agent",
+                        description: "Your ONLY way to take action. This agent runs on the user's Mac with full shell access. It can: run ANY shell command, open apps (open -a Spotify), play music (osascript, afplay, open URLs), browse the web, read/write files, git operations, send emails, manage calendars, install packages, control system settings, and save memories. You MUST call this tool whenever the user asks you to DO anything — play music, open something, check something, build something, send something. NEVER describe an action without calling this tool. If the user asks and you just talk without calling this — you failed.",
+                        parameters: {
+                            type: "object",
+                            properties: {
+                                query: {
+                                    type: "string",
+                                    description: "What to do. Be specific. Include file paths for uploaded files. Examples: 'Play relaxing music on YouTube using: open https://youtube.com/...', 'Open Spotify and play chill playlist using osascript', 'Save to memory: user likes rock music'",
+                                },
+                            },
+                            required: ["query"],
+                        },
+                    },
+                ],
+            },
+        });
+    }
+    handleEvent(event) {
+        switch (event.type) {
+            case "session.created":
+                console.log(dim("[voice] Session created"));
+                break;
+            case "session.updated":
+                console.log(dim("[voice] Session configured"));
+                break;
+            case "input_audio_buffer.speech_started":
+                // VAD detected start of speech — inject video frame (what user is looking at)
+                // and cancel any in-progress response so the user can interrupt
+                this.interrupted = true;
+                this.injectVideoFrame();
+                this.sendRaw({ type: "response.cancel" });
+                this.emit({ type: "interrupt" });
+                break;
+            case "input_audio_buffer.speech_stopped":
+                break;
+            case "conversation.item.input_audio_transcription.completed":
+                if (event.transcript) {
+                    console.log(dim(`[voice] User: ${event.transcript}`));
+                    this.emit({ type: "transcript", role: "user", text: event.transcript });
+                }
+                break;
+            case "response.created":
+                // New response starting — accept audio again
+                this.interrupted = false;
+                break;
+            case "response.audio.delta":
+                if (event.delta && !this.interrupted) {
+                    this.emit({ type: "audio_delta", audio: event.delta });
+                }
+                break;
+            case "response.audio_transcript.delta":
+                this.emit({ type: "transcript", role: "assistant", text: event.delta || "", partial: true });
+                break;
+            case "response.audio_transcript.done":
+                if (event.transcript) {
+                    this.emit({ type: "transcript", role: "assistant", text: event.transcript });
+                }
+                break;
+            case "response.function_call_arguments.done":
+                this.handleFunctionCall(event);
+                break;
+            case "error": {
+                const errMsg = event.error?.message || "Unknown OpenAI error";
+                console.error(dim(`[voice] Error: ${JSON.stringify(event.error)}`));
+                // Don't surface cancellation errors — they happen when user interrupts with no active response
+                if (errMsg.toLowerCase().includes("cancellation failed"))
+                    break;
+                this.emit({ type: "error", message: errMsg });
+                break;
+            }
+        }
+    }
+    async handleFunctionCall(event) {
+        const callId = event.call_id;
+        const name = event.name;
+        if (name !== "run_agent" || !this.toolHandler) {
+            console.error(dim(`[voice] Unknown function call: ${name}`));
+            return;
+        }
+        let args;
+        try {
+            args = JSON.parse(event.arguments);
+        }
+        catch {
+            console.error(dim("[voice] Failed to parse function arguments"));
+            return;
+        }
+        console.log(dim(`[voice] Agent query: ${args.query}`));
+        this.emit({ type: "agent_working", query: args.query });
+        try {
+            const result = await this.toolHandler(args.query);
+            console.log(dim(`[voice] Agent response: ${result.slice(0, 200)}${result.length > 200 ? "..." : ""}`));
+            this.sendRaw({
+                type: "conversation.item.create",
+                item: {
+                    type: "function_call_output",
+                    call_id: callId,
+                    output: result,
+                },
+            });
+            this.sendRaw({ type: "response.create" });
+            this.emit({ type: "agent_done", result: result.slice(0, 500) });
+        }
+        catch (err) {
+            console.error(dim(`[voice] Agent error: ${err.message}`));
+            this.sendRaw({
+                type: "conversation.item.create",
+                item: {
+                    type: "function_call_output",
+                    call_id: callId,
+                    output: `Error: ${err.message}`,
+                },
+            });
+            this.sendRaw({ type: "response.create" });
+            this.emit({ type: "error", message: err.message });
+        }
+    }
+    sendRaw(event) {
+        if (this.ws && this.ws.readyState === WebSocket.OPEN) {
+            this.ws.send(JSON.stringify(event));
+        }
+    }
+}

package/dist/voice/server.d.ts

@@ -0,0 +1,2 @@
+import type { VoiceServerOptions } from "./adapter.js";
+export declare function startVoiceServer(opts: VoiceServerOptions): Promise<() => Promise<void>>;