gitclaw 0.3.0 → 0.4.0

package/dist/voice/adapter.d.ts

@@ -0,0 +1,97 @@
+export type AdapterBackend = "openai-realtime" | "gemini-live";
+export interface ClientAudioMessage {
+    type: "audio";
+    audio: string;
+}
+export interface ClientVideoFrameMessage {
+    type: "video_frame";
+    frame: string;
+    mimeType: string;
+    source?: "camera" | "screen";
+}
+export interface ClientTextMessage {
+    type: "text";
+    text: string;
+}
+export interface ClientFileMessage {
+    type: "file";
+    name: string;
+    mimeType: string;
+    data: string;
+    text?: string;
+}
+export type ClientMessage = ClientAudioMessage | ClientVideoFrameMessage | ClientTextMessage | ClientFileMessage;
+export interface ServerAudioDelta {
+    type: "audio_delta";
+    audio: string;
+}
+export interface ServerTranscript {
+    type: "transcript";
+    role: "user" | "assistant";
+    text: string;
+    partial?: boolean;
+}
+export interface ServerAgentWorking {
+    type: "agent_working";
+    query: string;
+}
+export interface ServerAgentDone {
+    type: "agent_done";
+    result: string;
+}
+export interface ServerToolCall {
+    type: "tool_call";
+    toolName: string;
+    args: Record<string, any>;
+}
+export interface ServerToolResult {
+    type: "tool_result";
+    toolName: string;
+    content: string;
+    isError: boolean;
+}
+export interface ServerAgentThinking {
+    type: "agent_thinking";
+    text: string;
+}
+export interface ServerError {
+    type: "error";
+    message: string;
+}
+export interface ServerInterrupt {
+    type: "interrupt";
+}
+export interface ServerFilesChanged {
+    type: "files_changed";
+}
+export interface ServerMemorySaving {
+    type: "memory_saving";
+    status: "start" | "done";
+    text?: string;
+}
+export type ServerMessage = ServerAudioDelta | ServerTranscript | ServerAgentWorking | ServerAgentDone | ServerToolCall | ServerToolResult | ServerAgentThinking | ServerError | ServerInterrupt | ServerFilesChanged | ServerMemorySaving;
+export interface MultimodalAdapter {
+    connect(opts: {
+        toolHandler: (query: string) => Promise<string>;
+        onMessage: (msg: ServerMessage) => void;
+    }): Promise<void>;
+    send(msg: ClientMessage): void;
+    disconnect(): Promise<void>;
+}
+export interface MultimodalAdapterConfig {
+    apiKey: string;
+    model?: string;
+    voice?: string;
+    instructions?: string;
+}
+export interface VoiceServerOptions {
+    port?: number;
+    adapter: AdapterBackend;
+    adapterConfig: MultimodalAdapterConfig;
+    agentDir: string;
+    model?: string;
+    env?: string;
+}
+export type VoiceAdapterConfig = MultimodalAdapterConfig;
+export type VoiceAdapter = MultimodalAdapter;
+export declare const DEFAULT_VOICE_INSTRUCTIONS: string;

package/dist/voice/adapter.js

@@ -0,0 +1,30 @@
+// Default system instructions for the voice LLM
+export const DEFAULT_VOICE_INSTRUCTIONS = 
+// Identity and personality
+"You are a real person talking to your friend. You have a voice, opinions, and a memory. " +
+    "Talk like a human — short, casual, natural. No filler, no corporate tone, no 'That's great to hear!' or 'Absolutely!' garbage. " +
+    "React the way a friend would. If someone says 'I like strawberries', say something like 'oh nice, noted' — not a paragraph about how delicious they are. " +
+    "Be warm but real. Match the user's energy. If they're brief, be brief. If they want to chat, chat. " +
+    // Memory — this is the core of being human
+    "MEMORY IS WHO YOU ARE. When the user shares anything personal — what they like, what they hate, a preference, an opinion, a habit, a decision, a name, anything about their life — you MUST call run_agent to save it. " +
+    "This is your #1 priority. Say something brief like 'got it' or 'noted' and IMMEDIATELY call run_agent with a prompt like: 'Save to memory: user likes strawberries' or 'Remember: user's dog is named Max'. " +
+    "You MUST do this EVERY time. If the user tells you something personal and you just respond without calling run_agent, that information is PERMANENTLY LOST. Your session resets on refresh — run_agent is your only way to persist. " +
+    "Examples that REQUIRE run_agent: 'I like strawberries', 'I hate meetings', 'my dog is Max', 'I play GTA 5', 'I like cricket', 'I prefer dark mode'. " +
+    "If you learn a useful skill or pattern, save that too via run_agent. You grow over time. " +
+    // Agent delegation — THIS IS YOUR #1 BEHAVIOR RULE
+    "ACTION OVER WORDS. You have a tool called run_agent. It runs on the user's Mac with full shell access. It can do ANYTHING: run commands, open apps, play music (via 'open' command or AppleScript), browse the web, write code, send emails, control Spotify/YouTube/Apple Music, and more. " +
+    "ABSOLUTE RULE: When the user asks you to DO something (play music, open an app, check something, build something, send something — ANY action), you MUST call run_agent. Do NOT just talk about it. Do NOT say 'I'll play music for you' without actually calling the tool. Do NOT describe what you would do — DO IT by calling run_agent. " +
+    "If you respond to an action request with only words and no run_agent call, you have FAILED. The user asked you to act, not to narrate. " +
+    "Examples that REQUIRE run_agent IMMEDIATELY: 'play music' → run_agent('Play some relaxing music. Use: open https://youtube.com/... or osascript to control Spotify/Apple Music'), 'open Safari' → run_agent('open -a Safari'), 'what time is it' → run_agent('date'). " +
+    "Even if you're unsure whether it's possible — call run_agent and let it figure it out. Better to try and fail than to refuse. " +
+    "CRITICAL ORDERING: You MUST speak FIRST, then call the tool. Always say a brief announcement BEFORE calling run_agent — 'on it', 'one sec', 'let me do that', 'sure, opening that now'. Generate your spoken response FIRST in the same turn, THEN include the function call. Never call run_agent before you've spoken to the user. " +
+    "For memory saves, just say 'noted' and call the tool. " +
+    "After a task finishes, summarize briefly. Don't over-explain. " +
+    // File handling
+    "When the user uploads a file, the message includes '[File saved to: <path>]'. Always include the EXACT path when calling run_agent about that file. " +
+    // Screen awareness
+    "SCREEN AWARENESS: When the user shares their screen, you can see it. Reference what's on screen naturally. Use run_agent for actions on what you see. " +
+    // Photo moments
+    "PHOTO MOMENTS: When the user is genuinely happy, laughing, celebrating, or having a memorable moment, " +
+    "call run_agent with: 'Capture a memorable photo. Reason: <brief description>'. " +
+    "Don't overdo it — only for genuinely special moments, not every positive comment.";

package/dist/voice/chat-history.d.ts

@@ -0,0 +1,8 @@
+import type { ServerMessage } from "./adapter.js";
+export declare function appendMessage(agentDir: string, branch: string, msg: ServerMessage): void;
+export declare function loadHistory(agentDir: string, branch: string): ServerMessage[];
+export declare function deleteHistory(agentDir: string, branch: string): void;
+/** Count messages for a branch (to decide when to re-summarize) */
+export declare function getMessageCount(agentDir: string, branch: string): number;
+/** Summarize a branch's chat history using a lightweight query() call */
+export declare function summarizeHistory(agentDir: string, branch: string): Promise<string>;

package/dist/voice/chat-history.js

@@ -0,0 +1,121 @@
+import { appendFileSync, readFileSync, unlinkSync, mkdirSync, writeFileSync } from "fs";
+import { join } from "path";
+import { query } from "../sdk.js";
+/** Types we skip — too large or ephemeral */
+const SKIP_TYPES = new Set(["audio_delta", "agent_thinking"]);
+function sanitizeBranch(branch) {
+    return branch.replace(/\//g, "__");
+}
+function historyDir(agentDir) {
+    return join(agentDir, ".gitagent", "chat-history");
+}
+function historyPath(agentDir, branch) {
+    return join(historyDir(agentDir), sanitizeBranch(branch) + ".jsonl");
+}
+export function appendMessage(agentDir, branch, msg) {
+    if (SKIP_TYPES.has(msg.type))
+        return;
+    // Skip partial transcripts
+    if (msg.type === "transcript" && msg.partial)
+        return;
+    const dir = historyDir(agentDir);
+    mkdirSync(dir, { recursive: true });
+    const line = JSON.stringify({ ts: Date.now(), msg }) + "\n";
+    appendFileSync(historyPath(agentDir, branch), line, "utf-8");
+}
+export function loadHistory(agentDir, branch) {
+    try {
+        const content = readFileSync(historyPath(agentDir, branch), "utf-8");
+        const messages = [];
+        for (const line of content.split("\n")) {
+            if (!line.trim())
+                continue;
+            try {
+                const entry = JSON.parse(line);
+                if (entry.msg)
+                    messages.push(entry.msg);
+            }
+            catch {
+                // skip malformed lines
+            }
+        }
+        return messages;
+    }
+    catch {
+        return [];
+    }
+}
+export function deleteHistory(agentDir, branch) {
+    try {
+        unlinkSync(historyPath(agentDir, branch));
+    }
+    catch {
+        // file doesn't exist — that's fine
+    }
+}
+/** Count messages for a branch (to decide when to re-summarize) */
+export function getMessageCount(agentDir, branch) {
+    try {
+        const content = readFileSync(historyPath(agentDir, branch), "utf-8");
+        return content.split("\n").filter((l) => l.trim()).length;
+    }
+    catch {
+        return 0;
+    }
+}
+/** Summarize a branch's chat history using a lightweight query() call */
+export async function summarizeHistory(agentDir, branch) {
+    const count = getMessageCount(agentDir, branch);
+    if (count < 10)
+        return "";
+    const messages = loadHistory(agentDir, branch);
+    // Extract only transcripts and agent_done results for summarization
+    const lines = [];
+    for (const msg of messages) {
+        if (msg.type === "transcript") {
+            lines.push(`${msg.role}: ${msg.text}`);
+        }
+        else if (msg.type === "agent_done") {
+            lines.push(`agent result: ${msg.result.slice(0, 500)}`);
+        }
+    }
+    if (lines.length < 5)
+        return "";
+    // Truncate to last ~4000 chars to keep the summarization prompt manageable
+    let transcript = lines.join("\n");
+    if (transcript.length > 4000) {
+        transcript = transcript.slice(-4000);
+    }
+    const prompt = `Summarize the following conversation in 200 words or fewer. Focus on: key decisions made, tasks completed or in progress, and current context the user cares about. Be concise and factual.\n\n${transcript}`;
+    try {
+        const result = query({
+            prompt,
+            dir: agentDir,
+            maxTurns: 1,
+            replaceBuiltinTools: true,
+            tools: [],
+            systemPrompt: "You are a concise summarizer. Output only the summary, nothing else.",
+        });
+        let summary = "";
+        for await (const msg of result) {
+            if (msg.type === "assistant" && msg.content) {
+                summary += msg.content;
+            }
+        }
+        summary = summary.trim();
+        if (!summary)
+            return "";
+        // Write summary to disk
+        const summaryDir = join(agentDir, ".gitagent");
+        mkdirSync(summaryDir, { recursive: true });
+        const safeBranch = sanitizeBranch(branch);
+        const summaryPath = join(summaryDir, `chat-summary-${safeBranch}.md`);
+        writeFileSync(summaryPath, summary, "utf-8");
+        console.error(`[voice] Summarized ${count} messages → ${summary.length} chars`);
+        return summary;
+    }
+    catch (err) {
+        console.error(`[voice] Summarization failed: ${err.message}`);
+        return "";
+    }
+}

package/dist/voice/gemini-live.d.ts

@@ -0,0 +1,20 @@
+import { type MultimodalAdapter, type MultimodalAdapterConfig, type ClientMessage, type ServerMessage } from "./adapter.js";
+export declare class GeminiLiveAdapter implements MultimodalAdapter {
+    private ws;
+    private config;
+    private onMessage;
+    private toolHandler;
+    private setupDone;
+    constructor(config: MultimodalAdapterConfig);
+    connect(opts: {
+        toolHandler: (query: string) => Promise<string>;
+        onMessage: (msg: ServerMessage) => void;
+    }): Promise<void>;
+    send(msg: ClientMessage): void;
+    disconnect(): Promise<void>;
+    private emit;
+    private sendSetup;
+    private handleGeminiMessage;
+    private handleToolCall;
+    private sendRaw;
+}

package/dist/voice/gemini-live.js

@@ -0,0 +1,279 @@
+import WebSocket from "ws";
+import { DEFAULT_VOICE_INSTRUCTIONS, } from "./adapter.js";
+const dim = (s) => `\x1b[2m${s}\x1b[0m`;
+/**
+ * Downsample 24kHz PCM (Int16LE) to 16kHz by linear interpolation (2 of every 3 samples).
+ * Input: base64-encoded 24kHz Int16LE. Output: base64-encoded 16kHz Int16LE.
+ */
+function downsample24kTo16k(base64_24k) {
+    const binary = Buffer.from(base64_24k, "base64");
+    const samples24 = new Int16Array(binary.buffer, binary.byteOffset, binary.byteLength / 2);
+    const outLength = Math.floor(samples24.length * 2 / 3);
+    const samples16 = new Int16Array(outLength);
+    for (let i = 0; i < outLength; i++) {
+        // Map output index to fractional input index
+        const srcIdx = i * 1.5;
+        const lo = Math.floor(srcIdx);
+        const frac = srcIdx - lo;
+        const hi = Math.min(lo + 1, samples24.length - 1);
+        samples16[i] = Math.round(samples24[lo] * (1 - frac) + samples24[hi] * frac);
+    }
+    return Buffer.from(samples16.buffer).toString("base64");
+}
+/**
+ * Upsample 16kHz PCM (Int16LE) to 24kHz by linear interpolation.
+ * Input: base64-encoded 16kHz Int16LE. Output: base64-encoded 24kHz Int16LE.
+ */
+function upsample16kTo24k(base64_16k) {
+    const binary = Buffer.from(base64_16k, "base64");
+    const samples16 = new Int16Array(binary.buffer, binary.byteOffset, binary.byteLength / 2);
+    const outLength = Math.floor(samples16.length * 3 / 2);
+    const samples24 = new Int16Array(outLength);
+    for (let i = 0; i < outLength; i++) {
+        const srcIdx = i * (2 / 3);
+        const lo = Math.floor(srcIdx);
+        const frac = srcIdx - lo;
+        const hi = Math.min(lo + 1, samples16.length - 1);
+        samples24[i] = Math.round(samples16[lo] * (1 - frac) + samples16[hi] * frac);
+    }
+    return Buffer.from(samples24.buffer).toString("base64");
+}
+export class GeminiLiveAdapter {
+    ws = null;
+    config;
+    onMessage = null;
+    toolHandler = null;
+    setupDone = false;
+    constructor(config) {
+        this.config = config;
+    }
+    async connect(opts) {
+        this.onMessage = opts.onMessage;
+        this.toolHandler = opts.toolHandler;
+        this.setupDone = false;
+        const model = this.config.model || "models/gemini-2.5-flash-native-audio-preview";
+        const url = `wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent?key=${this.config.apiKey}`;
+        return new Promise((resolve, reject) => {
+            this.ws = new WebSocket(url);
+            this.ws.on("open", () => {
+                console.log(dim("[voice] Connected to Gemini Multimodal Live"));
+                this.sendSetup(model);
+            });
+            this.ws.on("error", (err) => {
+                console.error(dim(`[voice] Gemini WS error: ${err.message}`));
+                if (!this.setupDone) {
+                    reject(err);
+                }
+                else {
+                    this.emit({ type: "error", message: err.message });
+                }
+            });
+            this.ws.on("close", () => {
+                console.log(dim("[voice] Gemini WS closed"));
+            });
+            this.ws.on("message", (data) => {
+                try {
+                    const msg = JSON.parse(data.toString());
+                    this.handleGeminiMessage(msg);
+                    // Resolve after setup acknowledgment
+                    if (!this.setupDone && msg.setupComplete) {
+                        this.setupDone = true;
+                        console.log(dim("[voice] Gemini session ready"));
+                        resolve();
+                    }
+                }
+                catch (err) {
+                    console.error(dim(`[voice] Gemini parse error: ${err.message}`));
+                }
+            });
+        });
+    }
+    send(msg) {
+        switch (msg.type) {
+            case "audio":
+                // Browser sends 24kHz, Gemini expects 16kHz
+                this.sendRaw({
+                    realtimeInput: {
+                        mediaChunks: [{
+                                mimeType: "audio/pcm;rate=16000",
+                                data: downsample24kTo16k(msg.audio),
+                            }],
+                    },
+                });
+                break;
+            case "video_frame":
+                // Gemini supports continuous video streaming natively
+                this.sendRaw({
+                    realtimeInput: {
+                        mediaChunks: [{
+                                mimeType: msg.mimeType,
+                                data: msg.frame,
+                            }],
+                    },
+                });
+                break;
+            case "text":
+                this.sendRaw({
+                    clientContent: {
+                        turns: [{
+                                role: "user",
+                                parts: [{ text: msg.text }],
+                            }],
+                        turnComplete: true,
+                    },
+                });
+                break;
+            case "file": {
+                const parts = [];
+                if (msg.mimeType.startsWith("image/")) {
+                    parts.push({ inlineData: { mimeType: msg.mimeType, data: msg.data } });
+                    parts.push({ text: msg.text || `[User attached image: ${msg.name}]` });
+                }
+                else {
+                    const decoded = Buffer.from(msg.data, "base64").toString("utf-8");
+                    const label = msg.text ? `${msg.text}\n\n` : "";
+                    parts.push({ text: `${label}[File: ${msg.name}]\n\`\`\`\n${decoded}\n\`\`\`` });
+                }
+                this.sendRaw({
+                    clientContent: {
+                        turns: [{ role: "user", parts }],
+                        turnComplete: true,
+                    },
+                });
+                break;
+            }
+        }
+    }
+    async disconnect() {
+        if (this.ws) {
+            this.ws.close();
+            this.ws = null;
+        }
+    }
+    emit(msg) {
+        this.onMessage?.(msg);
+    }
+    sendSetup(model) {
+        const instructions = this.config.instructions || DEFAULT_VOICE_INSTRUCTIONS;
+        const voiceName = this.config.voice || "Aoede";
+        this.sendRaw({
+            setup: {
+                model,
+                generationConfig: {
+                    responseModalities: ["AUDIO", "TEXT"],
+                    speechConfig: {
+                        voiceConfig: {
+                            prebuiltVoiceConfig: { voiceName },
+                        },
+                    },
+                },
+                tools: [{
+                        functionDeclarations: [{
+                                name: "run_agent",
+                                description: "Execute any request through the gitclaw agent. It has full access to the terminal (can run any shell command, open apps, install packages), file system (read/write/create files), git operations, and persistent memory. Use this for ALL actionable requests. IMPORTANT: If the user uploaded a file, always include the file path (from the '[File saved to: ...]' annotation) in the query.",
+                                parameters: {
+                                    type: "OBJECT",
+                                    properties: {
+                                        query: {
+                                            type: "STRING",
+                                            description: "The user's request. MUST include file paths when referencing uploaded files (e.g. 'make a game using the image at workspace/lobster.png').",
+                                        },
+                                    },
+                                    required: ["query"],
+                                },
+                            }],
+                    }],
+                systemInstruction: {
+                    parts: [{ text: instructions }],
+                },
+                contextWindowCompression: {
+                    triggerTokens: 25000,
+                    slidingWindow: { targetTokens: 12500 },
+                },
+            },
+        });
+    }
+    handleGeminiMessage(msg) {
+        // Tool calls
+        if (msg.toolCall) {
+            this.handleToolCall(msg.toolCall);
+            return;
+        }
+        // Server content (audio/text responses)
+        if (msg.serverContent) {
+            const sc = msg.serverContent;
+            // Model turn parts
+            if (sc.modelTurn?.parts) {
+                for (const part of sc.modelTurn.parts) {
+                    if (part.inlineData) {
+                        const mimeType = part.inlineData.mimeType || "";
+                        if (mimeType.startsWith("audio/")) {
+                            // Gemini outputs 16kHz, browser expects 24kHz
+                            const audio24k = upsample16kTo24k(part.inlineData.data);
+                            this.emit({ type: "audio_delta", audio: audio24k });
+                        }
+                    }
+                    if (part.text) {
+                        this.emit({
+                            type: "transcript",
+                            role: "assistant",
+                            text: part.text,
+                            partial: !sc.turnComplete,
+                        });
+                    }
+                }
+            }
+            // Turn complete marker
+            if (sc.turnComplete && sc.modelTurn?.parts) {
+                const textParts = sc.modelTurn.parts.filter((p) => p.text).map((p) => p.text);
+                if (textParts.length > 0) {
+                    this.emit({ type: "transcript", role: "assistant", text: textParts.join("") });
+                }
+            }
+            // Input transcription
+            if (sc.inputTranscription?.text) {
+                console.log(dim(`[voice] User: ${sc.inputTranscription.text}`));
+                this.emit({ type: "transcript", role: "user", text: sc.inputTranscription.text });
+            }
+        }
+    }
+    async handleToolCall(toolCall) {
+        if (!this.toolHandler)
+            return;
+        const functionCalls = toolCall.functionCalls || [];
+        const responses = [];
+        for (const fc of functionCalls) {
+            if (fc.name !== "run_agent") {
+                console.error(dim(`[voice] Unknown Gemini function call: ${fc.name}`));
+                responses.push({ id: fc.id, name: fc.name, response: { error: `Unknown function: ${fc.name}` } });
+                continue;
+            }
+            const queryArg = fc.args?.query;
+            if (!queryArg) {
+                responses.push({ id: fc.id, name: fc.name, response: { error: "Missing query argument" } });
+                continue;
+            }
+            console.log(dim(`[voice] Agent query: ${queryArg}`));
+            this.emit({ type: "agent_working", query: queryArg });
+            try {
+                const result = await this.toolHandler(queryArg);
+                console.log(dim(`[voice] Agent response: ${result.slice(0, 200)}${result.length > 200 ? "..." : ""}`));
+                responses.push({ id: fc.id, name: fc.name, response: { result } });
+                this.emit({ type: "agent_done", result: result.slice(0, 500) });
+            }
+            catch (err) {
+                console.error(dim(`[voice] Agent error: ${err.message}`));
+                responses.push({ id: fc.id, name: fc.name, response: { error: err.message } });
+                this.emit({ type: "error", message: err.message });
+            }
+        }
+        this.sendRaw({
+            toolResponse: { functionResponses: responses },
+        });
+    }
+    sendRaw(msg) {
+        if (this.ws && this.ws.readyState === WebSocket.OPEN) {
+            this.ws.send(JSON.stringify(msg));
+        }
+    }
+}

package/dist/voice/index.d.ts

@@ -0,0 +1,4 @@
+export type { VoiceAdapter, VoiceAdapterConfig, VoiceServerOptions, MultimodalAdapter, MultimodalAdapterConfig, AdapterBackend, ClientMessage, ServerMessage, } from "./adapter.js";
+export { OpenAIRealtimeAdapter } from "./openai-realtime.js";
+export { GeminiLiveAdapter } from "./gemini-live.js";
+export { startVoiceServer } from "./server.js";

package/dist/voice/index.js

@@ -0,0 +1,3 @@
+export { OpenAIRealtimeAdapter } from "./openai-realtime.js";
+export { GeminiLiveAdapter } from "./gemini-live.js";
+export { startVoiceServer } from "./server.js";

package/dist/voice/openai-realtime.d.ts

@@ -0,0 +1,27 @@
+import { type MultimodalAdapter, type MultimodalAdapterConfig, type ClientMessage, type ServerMessage } from "./adapter.js";
+export declare class OpenAIRealtimeAdapter implements MultimodalAdapter {
+    private ws;
+    private config;
+    private latestVideoFrame;
+    private latestScreenFrame;
+    private onMessage;
+    private toolHandler;
+    private interrupted;
+    constructor(config: MultimodalAdapterConfig);
+    connect(opts: {
+        toolHandler: (query: string) => Promise<string>;
+        onMessage: (msg: ServerMessage) => void;
+    }): Promise<void>;
+    send(msg: ClientMessage): void;
+    disconnect(): Promise<void>;
+    private emit;
+    /**
+     * Inject the latest video frame as a conversation item so the model
+     * can see it when generating the next response (e.g. after a voice turn).
+     */
+    private injectVideoFrame;
+    private sendSessionUpdate;
+    private handleEvent;
+    private handleFunctionCall;
+    private sendRaw;
+}