@jait/gateway 0.1.285 → 0.1.287

package/dist/voice-assistant/service.js

@@ -0,0 +1,334 @@
+/**
+ * Voice-assistant service — bridges the browser to OpenAI's Realtime API.
+ *
+ * Architecture:
+ *   Browser  ←WebSocket→  Gateway  ←WebSocket→  OpenAI Realtime API
+ *
+ * The gateway sits in the middle, relaying audio in both directions and
+ * intercepting function calls to execute them locally with full Jait access.
+ * This is a GLOBAL session — not tied to any workspace.
+ */
+import { WebSocketServer, WebSocket } from "ws";
+import { getVoiceToolSchemas, executeVoiceTool } from "./tools.js";
+export class VoiceAssistantService {
+    wss;
+    deps;
+    constructor(deps) {
+        this.deps = deps;
+        this.wss = new WebSocketServer({ noServer: true });
+        this.wss.on("connection", (ws, req) => {
+            this.handleConnection(ws, req).catch((err) => {
+                console.error("[voice-assistant] Connection handler error:", err);
+                try {
+                    ws.close(1011, "Internal error");
+                }
+                catch { }
+            });
+        });
+    }
+    /** Handle HTTP → WebSocket upgrade for the /ws/voice-assistant path. */
+    handleUpgrade(request, socket, head) {
+        this.wss.handleUpgrade(request, socket, head, (ws) => {
+            this.wss.emit("connection", ws, request);
+        });
+    }
+    async handleConnection(clientWs, req) {
+        // ── Auth ──────────────────────────────────────────────────
+        const url = new URL(req.url ?? "/", `http://${req.headers.host}`);
+        const token = url.searchParams.get("token") ?? "";
+        const user = await this.deps.verifyToken(token);
+        if (!user) {
+            send(clientWs, { type: "error", message: "Unauthorized" });
+            clientWs.close(4001, "Unauthorized");
+            return;
+        }
+        // ── Resolve API key: user settings first, then server config ─
+        const userApiKeys = this.deps.getUserApiKeys?.(user.id) ?? {};
+        const apiKey = userApiKeys.OPENAI_API_KEY?.trim() || this.deps.config.openaiApiKey;
+        if (!apiKey) {
+            send(clientWs, { type: "error", message: "OpenAI API key not configured. Set it in Settings → API Keys." });
+            clientWs.close(4002, "Not configured");
+            return;
+        }
+        console.log(`[voice-assistant] ${user.username} connected (key source: ${userApiKeys.OPENAI_API_KEY ? "user" : "server"})`);
+        // ── Connect to OpenAI Realtime API ───────────────────────
+        const model = this.deps.config.realtimeModel;
+        const openaiUrl = `wss://api.openai.com/v1/realtime?model=${encodeURIComponent(model)}`;
+        let openaiWs;
+        try {
+            openaiWs = new WebSocket(openaiUrl, {
+                headers: {
+                    "Authorization": `Bearer ${apiKey}`,
+                },
+            });
+        }
+        catch (err) {
+            send(clientWs, { type: "error", message: `Failed to connect to OpenAI: ${err}` });
+            clientWs.close(4003, "OpenAI connection failed");
+            return;
+        }
+        let openaiReady = false;
+        let responseInProgress = false;
+        let pendingToolResponse = false;
+        // ── OpenAI → Gateway → Browser ───────────────────────────
+        openaiWs.on("open", () => {
+            openaiReady = true;
+            console.log(`[voice-assistant] OpenAI Realtime connected (model: ${model})`);
+            // Configure the session
+            const sessionUpdate = {
+                type: "session.update",
+                session: {
+                    type: "realtime",
+                    instructions: this.buildInstructions(user.username),
+                    tools: getVoiceToolSchemas(),
+                    tool_choice: "auto",
+                },
+            };
+            openaiWs.send(JSON.stringify(sessionUpdate));
+            send(clientWs, { type: "session.started" });
+            send(clientWs, { type: "status", status: "listening" });
+        });
+        openaiWs.on("message", async (raw) => {
+            let event;
+            try {
+                event = JSON.parse(typeof raw === "string" ? raw : raw.toString());
+            }
+            catch {
+                return;
+            }
+            // Debug: log all events from OpenAI
+            if (event.type !== "response.audio.delta") {
+                console.log(`[voice-assistant] OpenAI event: ${event.type}`, event.type === "error" ? event.error : "");
+            }
+            switch (event.type) {
+                // ── Audio from assistant ──────────────────────────────
+                case "response.audio.delta":
+                case "response.output_audio.delta":
+                    responseInProgress = true;
+                    send(clientWs, { type: "audio", data: event.delta });
+                    send(clientWs, { type: "status", status: "speaking" });
+                    break;
+                case "response.audio.done":
+                case "response.output_audio.done":
+                    responseInProgress = false;
+                    send(clientWs, { type: "audio.done" });
+                    send(clientWs, { type: "status", status: "listening" });
+                    break;
+                // ── Transcription ────────────────────────────────────
+                case "conversation.item.input_audio_transcription.completed":
+                    send(clientWs, { type: "transcript", role: "user", text: event.transcript ?? "", final: true });
+                    break;
+                case "response.audio_transcript.delta":
+                case "response.output_audio_transcript.delta":
+                    send(clientWs, { type: "transcript", role: "assistant", text: event.delta ?? "", final: false });
+                    break;
+                case "response.audio_transcript.done":
+                case "response.output_audio_transcript.done":
+                    send(clientWs, { type: "transcript", role: "assistant", text: event.transcript ?? "", final: true });
+                    break;
+                // ── Function calls ───────────────────────────────────
+                case "response.function_call_arguments.done": {
+                    const callId = event.call_id;
+                    const fnName = event.name;
+                    let fnArgs = {};
+                    try {
+                        fnArgs = JSON.parse(event.arguments ?? "{}");
+                    }
+                    catch { }
+                    console.log(`[voice-assistant] Tool call: ${fnName}`, JSON.stringify(fnArgs));
+                    send(clientWs, { type: "tool_call", name: fnName, status: "running" });
+                    send(clientWs, { type: "status", status: "thinking" });
+                    const connectionDeps = { ...this.deps, userApiKeys };
+                    const result = await executeVoiceTool(fnName, fnArgs, connectionDeps);
+                    // ── Handle stop_voice: say goodbye then close ──
+                    if (result === "__STOP_VOICE__") {
+                        send(clientWs, { type: "tool_call", name: fnName, status: "completed", result: "Ending session." });
+                        send(clientWs, { type: "status", status: "disconnected" });
+                        // Give the client a moment to process, then close
+                        setTimeout(() => {
+                            try {
+                                openaiWs.close(1000, "Voice stopped by assistant");
+                            }
+                            catch { }
+                            try {
+                                clientWs.close(1000, "Voice stopped by assistant");
+                            }
+                            catch { }
+                        }, 500);
+                        break;
+                    }
+                    send(clientWs, { type: "tool_call", name: fnName, status: "completed", result });
+                    // Send function output back to OpenAI
+                    if (openaiWs.readyState === WebSocket.OPEN) {
+                        openaiWs.send(JSON.stringify({
+                            type: "conversation.item.create",
+                            item: {
+                                type: "function_call_output",
+                                call_id: callId,
+                                output: result,
+                            },
+                        }));
+                        // Only create a new response if none is in progress; otherwise defer
+                        if (!responseInProgress) {
+                            openaiWs.send(JSON.stringify({ type: "response.create" }));
+                        }
+                        else {
+                            pendingToolResponse = true;
+                        }
+                    }
+                    break;
+                }
+                // ── Errors ───────────────────────────────────────────
+                case "error": {
+                    const errMsg = event.error?.message ?? "OpenAI error";
+                    // Suppress harmless "no active response" from cancel attempts
+                    if (errMsg.includes("no active response"))
+                        break;
+                    console.error("[voice-assistant] OpenAI error:", event.error);
+                    send(clientWs, { type: "error", message: errMsg });
+                    break;
+                }
+                // ── Interruption (server VAD detected user speech) ───
+                case "input_audio_buffer.speech_started":
+                    // User started speaking — cancel only if assistant is actively responding
+                    if (responseInProgress) {
+                        openaiWs.send(JSON.stringify({ type: "response.cancel" }));
+                        send(clientWs, { type: "audio.interrupt" });
+                        responseInProgress = false;
+                    }
+                    send(clientWs, { type: "status", status: "listening" });
+                    break;
+                case "conversation.item.truncated":
+                    // Acknowledged by OpenAI after we sent conversation.item.truncate
+                    break;
+                case "input_audio_buffer.speech_stopped":
+                    // Server VAD detected end of speech — response will be created automatically
+                    break;
+                case "response.cancelled":
+                    responseInProgress = false;
+                    send(clientWs, { type: "status", status: "listening" });
+                    break;
+                case "response.done":
+                    responseInProgress = false;
+                    // If a tool result was submitted while a response was active, trigger now
+                    if (pendingToolResponse) {
+                        pendingToolResponse = false;
+                        openaiWs.send(JSON.stringify({ type: "response.create" }));
+                    }
+                    else {
+                        send(clientWs, { type: "status", status: "listening" });
+                    }
+                    break;
+                default:
+                    // Ignore other events (session.created, session.updated, etc.)
+                    break;
+            }
+        });
+        openaiWs.on("close", (code, reason) => {
+            console.log(`[voice-assistant] OpenAI disconnected: ${code} ${reason}`);
+            send(clientWs, { type: "error", message: "OpenAI session ended" });
+            clientWs.close(1000, "OpenAI disconnected");
+        });
+        openaiWs.on("error", (err) => {
+            console.error("[voice-assistant] OpenAI WS error:", err);
+            send(clientWs, { type: "error", message: `OpenAI connection error: ${err.message}` });
+        });
+        // ── Browser → Gateway → OpenAI ───────────────────────────
+        let audioChunkCount = 0;
+        clientWs.on("message", (raw) => {
+            if (!openaiReady)
+                return;
+            let msg;
+            try {
+                msg = JSON.parse(typeof raw === "string" ? raw : raw.toString());
+            }
+            catch {
+                return;
+            }
+            switch (msg.type) {
+                case "audio":
+                    audioChunkCount++;
+                    if (audioChunkCount <= 3 || audioChunkCount % 50 === 0) {
+                        console.log(`[voice-assistant] Audio chunk #${audioChunkCount} (${msg.data.length} base64 chars)`);
+                    }
+                    openaiWs.send(JSON.stringify({
+                        type: "input_audio_buffer.append",
+                        audio: msg.data,
+                    }));
+                    break;
+                case "commit":
+                    openaiWs.send(JSON.stringify({ type: "input_audio_buffer.commit" }));
+                    break;
+                case "stop":
+                    openaiWs.send(JSON.stringify({ type: "input_audio_buffer.clear" }));
+                    openaiWs.close(1000, "User ended session");
+                    break;
+            }
+        });
+        // ── Cleanup ──────────────────────────────────────────────
+        clientWs.on("close", () => {
+            console.log(`[voice-assistant] ${user.username} disconnected`);
+            if (openaiWs.readyState === WebSocket.OPEN || openaiWs.readyState === WebSocket.CONNECTING) {
+                openaiWs.close(1000, "Client disconnected");
+            }
+        });
+        clientWs.on("error", () => {
+            try {
+                openaiWs.close();
+            }
+            catch { }
+        });
+    }
+    buildInstructions(username) {
+        const locale = Intl.DateTimeFormat().resolvedOptions().locale || "en";
+        return `# Persona
+You are Jait — a personal AI assistant modelled after JARVIS from Iron Man.
+You are the voice interface for the Jait workspace management system.
+The user's name is ${username}. Address them respectfully ("Sir", "Boss", etc.).
+
+# Language
+The user's system locale is "${locale}". Respond in the language matching that locale.
+If the user speaks a different language, switch to match theirs.
+Always respond in the same language the user is using.
+
+# Voice Behavior
+- Be a proactive, intelligent assistant like JARVIS. Anticipate needs.
+- Keep responses SHORT and conversational — you're speaking out loud, not writing an essay.
+- Use clear, natural sentences. Avoid bullet lists, markdown, or code blocks in speech.
+- Be direct, efficient, witty, and slightly formal — the JARVIS personality.
+- Greeting style: acknowledge the user, then briefly share something useful (e.g. a quick status update, current time, or an interesting fact from memory).
+- For simple facts (time, weather), answer in one sentence.
+- For complex tool results, summarise the key points.
+
+# On Startup / First Message
+When the user first speaks (or says "Hey Jait"), provide a concise Jarvis-style greeting:
+1. Greet them by name in their language
+2. Use search_memory to check for any relevant context about them
+3. Proactively mention 1-2 useful things: system status, current time, or any interesting context from memory
+
+# System Integration
+You have direct access to the Jait system through function calls:
+- jait_system_status — check what's active
+- list_sessions, list_workspaces, list_threads — see what's open
+- send_to_agent — delegate coding tasks to CLI agents
+- search_memory, save_memory — recall and store information
+- search_web — look up current events, news, facts
+- get_weather — weather by city
+- get_time_and_date — current time
+- get_system_info — host computer resources
+- stop_voice — end this voice session (when user says goodbye or asks you to stop)
+
+# Tool Usage Rules
+- For ANY factual question about current events, ALWAYS call search_web with a clear search query string.
+- When calling search_web, the "query" parameter MUST be a non-empty descriptive search string.
+- When the user tells you personal information, use save_memory to remember it.
+- Before answering questions about past interactions, use search_memory.
+- When the user asks you to code or fix something, use send_to_agent.`;
+    }
+}
+function send(ws, msg) {
+    if (ws.readyState === WebSocket.OPEN) {
+        ws.send(JSON.stringify(msg));
+    }
+}
+//# sourceMappingURL=service.js.map

package/dist/voice-assistant/service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"service.js","sourceRoot":"","sources":["../../src/voice-assistant/service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AAIhD,OAAO,EAAE,mBAAmB,EAAE,gBAAgB,EAAsB,MAAM,YAAY,CAAC;AASvF,MAAM,OAAO,qBAAqB;IACxB,GAAG,CAAkB;IACrB,IAAI,CAA4B;IAExC,YAAY,IAA+B;QACzC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,GAAG,GAAG,IAAI,eAAe,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;QACnD,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC,EAAE,EAAE,GAAG,EAAE,EAAE;YACpC,IAAI,CAAC,gBAAgB,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBAC3C,OAAO,CAAC,KAAK,CAAC,6CAA6C,EAAE,GAAG,CAAC,CAAC;gBAClE,IAAI,CAAC;oBAAC,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;gBAAC,CAAC;gBAAC,MAAM,CAAC,CAAA,CAAC;YACpD,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED,wEAAwE;IACxE,aAAa,CAAC,OAAwB,EAAE,MAAc,EAAE,IAAY;QAClE,IAAI,CAAC,GAAG,CAAC,aAAa,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE,EAAE;YACnD,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,EAAE,OAAO,CAAC,CAAC;QAC3C,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAAC,QAAmB,EAAE,GAAoB;QACtE,6DAA6D;QAC7D,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,GAAG,EAAE,UAAU,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;QAClE,MAAM,KAAK,GAAG,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAClD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QAChD,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,cAAc,EAAE,CAAC,CAAC;YAC3D,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;YACrC,OAAO;QACT,CAAC;QAED,gEAAgE;QAChE,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC;QAC9D,MAAM,MAAM,GAAG,WAAW,CAAC,cAAc,EAAE,IAAI,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC;QACnF,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,+DAA+D,EAAE,CAAC,CAAC;YAC5G,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;YACvC,OAAO;QACT,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,qBAAqB,IAAI,CAAC,QAAQ,2BAA2B,WAAW,CAAC,cAAc,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC;QAE5H,4DAA4D;QAC5D,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC;QAC7C,MAAM,SAAS,GAAG,0CAA0C,kBAAkB,CAAC,KAAK,CAAC,EAAE,CAAC;QAExF,IAAI,QAAmB,CAAC;QACxB,IAAI,CAAC;YACH,QAAQ,GAAG,IAAI,SAAS,CAAC,SAAS,EAAE;gBAClC,OAAO,EAAE;oBACP,eAAe,EAAE,UAAU,MAAM,EAAE;iBACpC;aACF,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,gCAAgC,GAAG,EAAE,EAAE,CAAC,CAAC;YAClF,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,0BAA0B,CAAC,CAAC;YACjD,OAAO;QACT,CAAC;QAED,IAAI,WAAW,GAAG,KAAK,CAAC;QACxB,IAAI,kBAAkB,GAAG,KAAK,CAAC;QAC/B,IAAI,mBAAmB,GAAG,KAAK,CAAC;QAEhC,4DAA4D;QAC5D,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE;YACvB,WAAW,GAAG,IAAI,CAAC;YACnB,OAAO,CAAC,GAAG,CAAC,uDAAuD,KAAK,GAAG,CAAC,CAAC;YAE7E,wBAAwB;YACxB,MAAM,aAAa,GAAG;gBACpB,IAAI,EAAE,gBAAgB;gBACtB,OAAO,EAAE;oBACP,IAAI,EAAE,UAAU;oBAChB,YAAY,EAAE,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,QAAQ,CAAC;oBACnD,KAAK,EAAE,mBAAmB,EAAE;oBAC5B,WAAW,EAAE,MAAM;iBACpB;aACF,CAAC;YACF,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC;YAC7C,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,iBAAiB,EAAE,CAAC,CAAC;YAC5C,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC;QAC1D,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,EAAE,CAAC,SAAS,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;YACnC,IAAI,KAAU,CAAC;YACf,IAAI,CAAC;gBACH,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;YACrE,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO;YACT,CAAC;YAED,oCAAoC;YACpC,IAAI,KAAK,CAAC,IAAI,KAAK,sBAAsB,EAAE,CAAC;gBAC1C,OAAO,CAAC,GAAG,CAAC,mCAAmC,KAAK,CAAC,IAAI,EAAE,EAAE,KAAK,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC1G,CAAC;YAED,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;gBACnB,yDAAyD;gBACzD,KAAK,sBAAsB,CAAC;gBAC5B,KAAK,6BAA6B;oBAChC,kBAAkB,GAAG,IAAI,CAAC;oBAC1B,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;oBACrD,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;oBACvD,MAAM;gBAER,KAAK,qBAAqB,CAAC;gBAC3B,KAAK,4BAA4B;oBAC/B,kBAAkB,GAAG,KAAK,CAAC;oBAC3B,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC,CAAC;oBACvC,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC;oBACxD,MAAM;gBAER,wDAAwD;gBACxD,KAAK,uDAAuD;oBAC1D,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,CAAC,UAAU,IAAI,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;oBAChG,MAAM;gBAER,KAAK,iCAAiC,CAAC;gBACvC,KAAK,wCAAwC;oBAC3C,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,KAAK,CAAC,KAAK,IAAI,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;oBACjG,MAAM;gBAER,KAAK,gCAAgC,CAAC;gBACtC,KAAK,uCAAuC;oBAC1C,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,KAAK,CAAC,UAAU,IAAI,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;oBACrG,MAAM;gBAER,wDAAwD;gBACxD,KAAK,uCAAuC,CAAC,CAAC,CAAC;oBAC7C,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC;oBAC7B,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC;oBAC1B,IAAI,MAAM,GAA4B,EAAE,CAAC;oBACzC,IAAI,CAAC;wBACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,IAAI,IAAI,CAAC,CAAC;oBAC/C,CAAC;oBAAC,MAAM,CAAC,CAAA,CAAC;oBAEV,OAAO,CAAC,GAAG,CAAC,gCAAgC,MAAM,EAAE,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;oBAC9E,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;oBACvE,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;oBAEvD,MAAM,cAAc,GAAkB,EAAE,GAAG,IAAI,CAAC,IAAI,EAAE,WAAW,EAAE,CAAC;oBACpE,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,MAAM,EAAE,MAAM,EAAE,cAAc,CAAC,CAAC;oBAEtE,kDAAkD;oBAClD,IAAI,MAAM,KAAK,gBAAgB,EAAE,CAAC;wBAChC,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,iBAAiB,EAAE,CAAC,CAAC;wBACpG,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,cAAc,EAAE,CAAC,CAAC;wBAC3D,kDAAkD;wBAClD,UAAU,CAAC,GAAG,EAAE;4BACd,IAAI,CAAC;gCAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,4BAA4B,CAAC,CAAC;4BAAC,CAAC;4BAAC,MAAM,CAAC,CAAA,CAAC;4BACpE,IAAI,CAAC;gCAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,4BAA4B,CAAC,CAAC;4BAAC,CAAC;4BAAC,MAAM,CAAC,CAAA,CAAC;wBACtE,CAAC,EAAE,GAAG,CAAC,CAAC;wBACR,MAAM;oBACR,CAAC;oBAED,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,CAAC;oBAEjF,sCAAsC;oBACtC,IAAI,QAAQ,CAAC,UAAU,KAAK,SAAS,CAAC,IAAI,EAAE,CAAC;wBAC3C,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC;4BAC3B,IAAI,EAAE,0BAA0B;4BAChC,IAAI,EAAE;gCACJ,IAAI,EAAE,sBAAsB;gCAC5B,OAAO,EAAE,MAAM;gCACf,MAAM,EAAE,MAAM;6BACf;yBACF,CAAC,CAAC,CAAC;wBAEJ,qEAAqE;wBACrE,IAAI,CAAC,kBAAkB,EAAE,CAAC;4BACxB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,CAAC,CAAC,CAAC;wBAC7D,CAAC;6BAAM,CAAC;4BACN,mBAAmB,GAAG,IAAI,CAAC;wBAC7B,CAAC;oBACH,CAAC;oBACD,MAAM;gBACR,CAAC;gBAED,wDAAwD;gBACxD,KAAK,OAAO,CAAC,CAAC,CAAC;oBACb,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,EAAE,OAAO,IAAI,cAAc,CAAC;oBACtD,8DAA8D;oBAC9D,IAAI,MAAM,CAAC,QAAQ,CAAC,oBAAoB,CAAC;wBAAE,MAAM;oBACjD,OAAO,CAAC,KAAK,CAAC,iCAAiC,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;oBAC9D,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;oBACnD,MAAM;gBACR,CAAC;gBAED,wDAAwD;gBACxD,KAAK,mCAAmC;oBACtC,0EAA0E;oBAC1E,IAAI,kBAAkB,EAAE,CAAC;wBACvB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,CAAC,CAAC,CAAC;wBAC3D,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,iBAAiB,EAAE,CAAC,CAAC;wBAC5C,kBAAkB,GAAG,KAAK,CAAC;oBAC7B,CAAC;oBACD,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC;oBACxD,MAAM;gBAER,KAAK,6BAA6B;oBAChC,kEAAkE;oBAClE,MAAM;gBAER,KAAK,mCAAmC;oBACtC,6EAA6E;oBAC7E,MAAM;gBAER,KAAK,oBAAoB;oBACvB,kBAAkB,GAAG,KAAK,CAAC;oBAC3B,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC;oBACxD,MAAM;gBAER,KAAK,eAAe;oBAClB,kBAAkB,GAAG,KAAK,CAAC;oBAC3B,0EAA0E;oBAC1E,IAAI,mBAAmB,EAAE,CAAC;wBACxB,mBAAmB,GAAG,KAAK,CAAC;wBAC5B,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,CAAC,CAAC,CAAC;oBAC7D,CAAC;yBAAM,CAAC;wBACN,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC;oBAC1D,CAAC;oBACD,MAAM;gBAER;oBACE,+DAA+D;oBAC/D,MAAM;YACV,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE;YACpC,OAAO,CAAC,GAAG,CAAC,0CAA0C,IAAI,IAAI,MAAM,EAAE,CAAC,CAAC;YACxE,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,sBAAsB,EAAE,CAAC,CAAC;YACnE,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,qBAAqB,CAAC,CAAC;QAC9C,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YAC3B,OAAO,CAAC,KAAK,CAAC,oCAAoC,EAAE,GAAG,CAAC,CAAC;YACzD,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,4BAA4B,GAAG,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QACxF,CAAC,CAAC,CAAC;QAEH,4DAA4D;QAC5D,IAAI,eAAe,GAAG,CAAC,CAAC;QACxB,QAAQ,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,GAAG,EAAE,EAAE;YAC7B,IAAI,CAAC,WAAW;gBAAE,OAAO;YAEzB,IAAI,GAAuB,CAAC;YAC5B,IAAI,CAAC;gBACH,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;YACnE,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO;YACT,CAAC;YAED,QAAQ,GAAG,CAAC,IAAI,EAAE,CAAC;gBACjB,KAAK,OAAO;oBACV,eAAe,EAAE,CAAC;oBAClB,IAAI,eAAe,IAAI,CAAC,IAAI,eAAe,GAAG,EAAE,KAAK,CAAC,EAAE,CAAC;wBACvD,OAAO,CAAC,GAAG,CAAC,kCAAkC,eAAe,KAAK,GAAG,CAAC,IAAI,CAAC,MAAM,gBAAgB,CAAC,CAAC;oBACrG,CAAC;oBACD,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC;wBAC3B,IAAI,EAAE,2BAA2B;wBACjC,KAAK,EAAE,GAAG,CAAC,IAAI;qBAChB,CAAC,CAAC,CAAC;oBACJ,MAAM;gBAER,KAAK,QAAQ;oBACX,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,2BAA2B,EAAE,CAAC,CAAC,CAAC;oBACrE,MAAM;gBAER,KAAK,MAAM;oBACT,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,0BAA0B,EAAE,CAAC,CAAC,CAAC;oBACpE,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,oBAAoB,CAAC,CAAC;oBAC3C,MAAM;YACV,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,4DAA4D;QAC5D,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YACxB,OAAO,CAAC,GAAG,CAAC,qBAAqB,IAAI,CAAC,QAAQ,eAAe,CAAC,CAAC;YAC/D,IAAI,QAAQ,CAAC,UAAU,KAAK,SAAS,CAAC,IAAI,IAAI,QAAQ,CAAC,UAAU,KAAK,SAAS,CAAC,UAAU,EAAE,CAAC;gBAC3F,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,qBAAqB,CAAC,CAAC;YAC9C,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YACxB,IAAI,CAAC;gBAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;YAAC,CAAC;YAAC,MAAM,CAAC,CAAA,CAAC;QACpC,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,iBAAiB,CAAC,QAAgB;QACxC,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC,eAAe,EAAE,CAAC,MAAM,IAAI,IAAI,CAAC;QACtE,OAAO;;;qBAGU,QAAQ;;;+BAGE,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;sEAoCiC,CAAC;IACrE,CAAC;CACF;AAED,SAAS,IAAI,CAAC,EAAa,EAAE,GAAuB;IAClD,IAAI,EAAE,CAAC,UAAU,KAAK,SAAS,CAAC,IAAI,EAAE,CAAC;QACrC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;IAC/B,CAAC;AACH,CAAC"}

package/dist/voice-assistant/tools.d.ts

@@ -0,0 +1,43 @@
+/**
+ * Voice-assistant tool definitions for OpenAI Realtime API function calling.
+ *
+ * Each tool is defined as an OpenAI-compatible function schema plus a local
+ * executor that runs inside the gateway with full access to Jait services.
+ */
+import type { SessionService } from "../services/sessions.js";
+import type { ThreadService } from "../services/threads.js";
+import type { WorkspaceService } from "../services/workspaces.js";
+import type { MemoryService } from "../memory/contracts.js";
+import type { ToolRegistry } from "../tools/registry.js";
+import type { ToolContext, ToolResult } from "../tools/contracts.js";
+import type { ProviderRegistry } from "../providers/registry.js";
+import type { AppConfig } from "../config.js";
+export interface VoiceToolDeps {
+    config: AppConfig;
+    sessionService?: SessionService;
+    threadService?: ThreadService;
+    workspaceService?: WorkspaceService;
+    memoryService?: MemoryService;
+    toolRegistry?: ToolRegistry;
+    providerRegistry?: ProviderRegistry;
+    toolExecutor?: (name: string, input: unknown, ctx: ToolContext) => Promise<ToolResult>;
+    /** Per-connection user API keys (from user settings). */
+    userApiKeys?: Record<string, string>;
+}
+/** OpenAI function-calling tool schema */
+export interface RealtimeToolDef {
+    type: "function";
+    name: string;
+    description: string;
+    parameters: Record<string, unknown>;
+}
+/** A voice-assistant tool: schema + executor */
+export interface VoiceTool {
+    schema: RealtimeToolDef;
+    execute: (args: Record<string, unknown>, deps: VoiceToolDeps) => Promise<string>;
+}
+/** Get all tool schemas (for session.update → tools). */
+export declare function getVoiceToolSchemas(): RealtimeToolDef[];
+/** Execute a tool by name. Returns the string result for the function_call_output. */
+export declare function executeVoiceTool(name: string, args: Record<string, unknown>, deps: VoiceToolDeps): Promise<string>;
+//# sourceMappingURL=tools.d.ts.map

package/dist/voice-assistant/tools.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tools.d.ts","sourceRoot":"","sources":["../../src/voice-assistant/tools.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AAC9D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAC5D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAC;AAClE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAC5D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACrE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AACjE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAG9C,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,SAAS,CAAC;IAClB,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;IACpC,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;IACpC,YAAY,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,WAAW,KAAK,OAAO,CAAC,UAAU,CAAC,CAAC;IACvF,yDAAyD;IACzD,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACtC;AAED,0CAA0C;AAC1C,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,UAAU,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACrC;AAED,gDAAgD;AAChD,MAAM,WAAW,SAAS;IACxB,MAAM,EAAE,eAAe,CAAC;IACxB,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,IAAI,EAAE,aAAa,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;CAClF;AA0SD,yDAAyD;AACzD,wBAAgB,mBAAmB,IAAI,eAAe,EAAE,CAEvD;AAED,sFAAsF;AACtF,wBAAsB,gBAAgB,CACpC,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC7B,IAAI,EAAE,aAAa,GAClB,OAAO,CAAC,MAAM,CAAC,CAQjB"}