gitclaw 0.3.0 → 0.4.0

package/dist/voice/openai-realtime.js

@@ -0,0 +1,291 @@
+import WebSocket from "ws";
+import { DEFAULT_VOICE_INSTRUCTIONS, } from "./adapter.js";
+const dim = (s) => `\x1b[2m${s}\x1b[0m`;
+export class OpenAIRealtimeAdapter {
+    ws = null;
+    config;
+    latestVideoFrame = null;
+    latestScreenFrame = null;
+    onMessage = null;
+    toolHandler = null;
+    interrupted = false;
+    constructor(config) {
+        this.config = config;
+    }
+    async connect(opts) {
+        this.onMessage = opts.onMessage;
+        this.toolHandler = opts.toolHandler;
+        const model = this.config.model || "gpt-realtime";
+        const url = `wss://api.openai.com/v1/realtime?model=${model}`;
+        return new Promise((resolve, reject) => {
+            this.ws = new WebSocket(url, {
+                headers: {
+                    Authorization: `Bearer ${this.config.apiKey}`,
+                    "OpenAI-Beta": "realtime=v1",
+                },
+            });
+            this.ws.on("open", () => {
+                this.sendSessionUpdate();
+                resolve();
+            });
+            this.ws.on("error", (err) => {
+                if (!this.ws) {
+                    reject(err);
+                }
+                else {
+                    console.error(dim(`[voice] WebSocket error: ${err.message}`));
+                    this.emit({ type: "error", message: err.message });
+                }
+            });
+            this.ws.on("close", () => {
+                console.log(dim("[voice] WebSocket closed"));
+            });
+            this.ws.on("message", (data) => {
+                const event = JSON.parse(data.toString());
+                this.handleEvent(event);
+            });
+        });
+    }
+    send(msg) {
+        switch (msg.type) {
+            case "audio":
+                this.sendRaw({
+                    type: "input_audio_buffer.append",
+                    audio: msg.audio,
+                });
+                break;
+            case "video_frame": {
+                // OpenAI doesn't support continuous video. Store latest frame and
+                // inject it as an image on the next user turn via conversation item.
+                const source = msg.source || "camera";
+                if (source === "screen") {
+                    this.latestScreenFrame = { frame: msg.frame, mimeType: msg.mimeType };
+                }
+                else {
+                    this.latestVideoFrame = { frame: msg.frame, mimeType: msg.mimeType };
+                }
+                break;
+            }
+            case "text": {
+                // Send text as a user conversation item, optionally with latest video frame
+                const content = [];
+                if (this.latestVideoFrame) {
+                    content.push({
+                        type: "input_image",
+                        image_url: `data:${this.latestVideoFrame.mimeType};base64,${this.latestVideoFrame.frame}`,
+                    });
+                    this.latestVideoFrame = null;
+                }
+                content.push({ type: "input_text", text: msg.text });
+                this.sendRaw({
+                    type: "conversation.item.create",
+                    item: {
+                        type: "message",
+                        role: "user",
+                        content,
+                    },
+                });
+                this.sendRaw({ type: "response.create" });
+                break;
+            }
+            case "file": {
+                const content = [];
+                if (msg.mimeType.startsWith("image/")) {
+                    content.push({
+                        type: "input_image",
+                        image_url: `data:${msg.mimeType};base64,${msg.data}`,
+                    });
+                    content.push({ type: "input_text", text: msg.text || `[User attached image: ${msg.name}]` });
+                }
+                else {
+                    const decoded = Buffer.from(msg.data, "base64").toString("utf-8");
+                    const label = msg.text ? `${msg.text}\n\n` : "";
+                    content.push({ type: "input_text", text: `${label}[File: ${msg.name}]\n\`\`\`\n${decoded}\n\`\`\`` });
+                }
+                this.sendRaw({
+                    type: "conversation.item.create",
+                    item: { type: "message", role: "user", content },
+                });
+                this.sendRaw({ type: "response.create" });
+                break;
+            }
+        }
+    }
+    async disconnect() {
+        if (this.ws) {
+            this.ws.close();
+            this.ws = null;
+        }
+    }
+    emit(msg) {
+        this.onMessage?.(msg);
+    }
+    /**
+     * Inject the latest video frame as a conversation item so the model
+     * can see it when generating the next response (e.g. after a voice turn).
+     */
+    injectVideoFrame() {
+        // Prefer screen frame over camera — it provides more useful context
+        const isScreen = !!this.latestScreenFrame;
+        const frame = this.latestScreenFrame || this.latestVideoFrame;
+        if (!frame)
+            return;
+        // Clear both so we don't inject stale frames
+        this.latestScreenFrame = null;
+        this.latestVideoFrame = null;
+        console.log(dim(`[voice] Injecting ${isScreen ? "screen" : "camera"} frame into conversation`));
+        this.sendRaw({
+            type: "conversation.item.create",
+            item: {
+                type: "message",
+                role: "user",
+                content: [{
+                        type: "input_image",
+                        image_url: `data:${frame.mimeType};base64,${frame.frame}`,
+                    }],
+            },
+        });
+    }
+    sendSessionUpdate() {
+        const instructions = this.config.instructions || DEFAULT_VOICE_INSTRUCTIONS;
+        this.sendRaw({
+            type: "session.update",
+            session: {
+                instructions,
+                voice: this.config.voice || "ash",
+                modalities: ["text", "audio"],
+                turn_detection: {
+                    type: "server_vad",
+                    threshold: 0.6,
+                    prefix_padding_ms: 400,
+                    silence_duration_ms: 800,
+                    create_response: true,
+                },
+                input_audio_transcription: { model: "whisper-1" },
+                tool_choice: "auto",
+                tools: [
+                    {
+                        type: "function",
+                        name: "run_agent",
+                        description: "Your ONLY way to take action. This agent runs on the user's Mac with full shell access. It can: run ANY shell command, open apps (open -a Spotify), play music (osascript, afplay, open URLs), browse the web, read/write files, git operations, send emails, manage calendars, install packages, control system settings, and save memories. You MUST call this tool whenever the user asks you to DO anything — play music, open something, check something, build something, send something. NEVER describe an action without calling this tool. If the user asks and you just talk without calling this — you failed.",
+                        parameters: {
+                            type: "object",
+                            properties: {
+                                query: {
+                                    type: "string",
+                                    description: "What to do. Be specific. Include file paths for uploaded files. Examples: 'Play relaxing music on YouTube using: open https://youtube.com/...', 'Open Spotify and play chill playlist using osascript', 'Save to memory: user likes rock music'",
+                                },
+                            },
+                            required: ["query"],
+                        },
+                    },
+                ],
+            },
+        });
+    }
+    handleEvent(event) {
+        switch (event.type) {
+            case "session.created":
+                console.log(dim("[voice] Session created"));
+                break;
+            case "session.updated":
+                console.log(dim("[voice] Session configured"));
+                break;
+            case "input_audio_buffer.speech_started":
+                // VAD detected start of speech — inject video frame (what user is looking at)
+                // and cancel any in-progress response so the user can interrupt
+                this.interrupted = true;
+                this.injectVideoFrame();
+                this.sendRaw({ type: "response.cancel" });
+                this.emit({ type: "interrupt" });
+                break;
+            case "input_audio_buffer.speech_stopped":
+                break;
+            case "conversation.item.input_audio_transcription.completed":
+                if (event.transcript) {
+                    console.log(dim(`[voice] User: ${event.transcript}`));
+                    this.emit({ type: "transcript", role: "user", text: event.transcript });
+                }
+                break;
+            case "response.created":
+                // New response starting — accept audio again
+                this.interrupted = false;
+                break;
+            case "response.audio.delta":
+                if (event.delta && !this.interrupted) {
+                    this.emit({ type: "audio_delta", audio: event.delta });
+                }
+                break;
+            case "response.audio_transcript.delta":
+                this.emit({ type: "transcript", role: "assistant", text: event.delta || "", partial: true });
+                break;
+            case "response.audio_transcript.done":
+                if (event.transcript) {
+                    this.emit({ type: "transcript", role: "assistant", text: event.transcript });
+                }
+                break;
+            case "response.function_call_arguments.done":
+                this.handleFunctionCall(event);
+                break;
+            case "error": {
+                const errMsg = event.error?.message || "Unknown OpenAI error";
+                console.error(dim(`[voice] Error: ${JSON.stringify(event.error)}`));
+                // Don't surface cancellation errors — they happen when user interrupts with no active response
+                if (errMsg.toLowerCase().includes("cancellation failed"))
+                    break;
+                this.emit({ type: "error", message: errMsg });
+                break;
+            }
+        }
+    }
+    async handleFunctionCall(event) {
+        const callId = event.call_id;
+        const name = event.name;
+        if (name !== "run_agent" || !this.toolHandler) {
+            console.error(dim(`[voice] Unknown function call: ${name}`));
+            return;
+        }
+        let args;
+        try {
+            args = JSON.parse(event.arguments);
+        }
+        catch {
+            console.error(dim("[voice] Failed to parse function arguments"));
+            return;
+        }
+        console.log(dim(`[voice] Agent query: ${args.query}`));
+        this.emit({ type: "agent_working", query: args.query });
+        try {
+            const result = await this.toolHandler(args.query);
+            console.log(dim(`[voice] Agent response: ${result.slice(0, 200)}${result.length > 200 ? "..." : ""}`));
+            this.sendRaw({
+                type: "conversation.item.create",
+                item: {
+                    type: "function_call_output",
+                    call_id: callId,
+                    output: result,
+                },
+            });
+            this.sendRaw({ type: "response.create" });
+            this.emit({ type: "agent_done", result: result.slice(0, 500) });
+        }
+        catch (err) {
+            console.error(dim(`[voice] Agent error: ${err.message}`));
+            this.sendRaw({
+                type: "conversation.item.create",
+                item: {
+                    type: "function_call_output",
+                    call_id: callId,
+                    output: `Error: ${err.message}`,
+                },
+            });
+            this.sendRaw({ type: "response.create" });
+            this.emit({ type: "error", message: err.message });
+        }
+    }
+    sendRaw(event) {
+        if (this.ws && this.ws.readyState === WebSocket.OPEN) {
+            this.ws.send(JSON.stringify(event));
+        }
+    }
+}

package/dist/voice/server.d.ts

@@ -0,0 +1,2 @@
+import type { VoiceServerOptions } from "./adapter.js";
+export declare function startVoiceServer(opts: VoiceServerOptions): Promise<() => Promise<void>>;