@yak-io/javascript 0.7.0 → 0.8.0

package/dist/voice-session.js

@@ -0,0 +1,530 @@
+import { logger } from "./logger.js";
+import { extractPageContext } from "./page-context.js";
+import { generateToolId } from "./tool-name.js";
+import { handleRealtimeMessage, INITIAL_VOICE_MACHINE, voiceReducer, } from "./voice-machine.js";
+const DEFAULT_REALTIME_MODEL = "gpt-realtime";
+const REALTIME_CALLS_URL = "https://api.openai.com/v1/realtime/calls";
+function getApiOrigin() {
+    if (typeof window !== "undefined") {
+        const hostname = window.location.hostname;
+        if (hostname.endsWith(".yak.supply") || hostname === "yak.supply") {
+            return "https://chat.yak.supply";
+        }
+        if ((hostname === "localhost" || hostname === "127.0.0.1") &&
+            typeof window.__YAK_INTERNAL_DEV__ !== "undefined") {
+            return "http://localhost:3001";
+        }
+    }
+    return "https://chat.yak.io";
+}
+const EMPTY_RESOURCES = {
+    pc: null,
+    dataChannel: null,
+    micStream: null,
+    audioElement: null,
+    voiceSessionId: null,
+};
+function emptyUsage() {
+    return {
+        inputTokens: 0,
+        cachedInputTokens: 0,
+        outputTokens: 0,
+        audioInputTokens: 0,
+        audioOutputTokens: 0,
+        textInputTokens: 0,
+        textOutputTokens: 0,
+        responseCount: 0,
+    };
+}
+export class YakVoiceSession {
+    config;
+    machine = INITIAL_VOICE_MACHINE;
+    resources = EMPTY_RESOURCES;
+    dispatchedCallIds = new Set();
+    listeners = new Set();
+    pageHideHandler = null;
+    /** Per-session token totals, accumulated from each `response.done` event. */
+    usage = emptyUsage();
+    /**
+     * Reverse map: hashed tool id (what OpenAI calls back with) → original host
+     * tool name (what `onToolCall` expects). Populated on every `start()` from
+     * the resolved chat config.
+     */
+    toolNameById = new Map();
+    constructor(config) {
+        this.config = config;
+        this.attachPageHide();
+    }
+    /**
+     * Resolve the API origin lazily on each call. The internal-dev flag
+     * (`window.__YAK_INTERNAL_DEV__`) is often set in a `useEffect` that fires
+     * after this session is constructed, so resolving at construction would
+     * bake in the production URL.
+     */
+    get apiOrigin() {
+        return this.config.apiOrigin ?? getApiOrigin();
+    }
+    /** Update mutable config fields (handlers, getConfig). */
+    updateConfig(patch) {
+        this.config = { ...this.config, ...patch };
+    }
+    getState() {
+        return this.machine;
+    }
+    /**
+     * The current API origin (`chat.yak.io`, `chat.yak.supply`, or
+     * `http://localhost:3001` when `__YAK_INTERNAL_DEV__` is set). Useful for
+     * building URLs to static assets like the brand logo.
+     */
+    getApiOrigin() {
+        return this.apiOrigin;
+    }
+    onStateChange(listener) {
+        this.listeners.add(listener);
+        return () => {
+            this.listeners.delete(listener);
+        };
+    }
+    /**
+     * Begin a voice session. Should be invoked from a user gesture (button
+     * click) so `getUserMedia` and audio playback both have transient activation.
+     */
+    async start() {
+        if (this.machine.state !== "idle")
+            return;
+        logger.debug("Voice: start() called");
+        this.usage = emptyUsage();
+        this.dispatch({ type: "start" });
+        let chatConfig = this.config.chatConfig;
+        if (this.config.getConfig) {
+            try {
+                chatConfig = await this.config.getConfig();
+                logger.debug("Voice: getConfig() resolved", {
+                    toolCount: chatConfig?.tools?.tools.length ?? 0,
+                    routeCount: chatConfig?.routes?.routes.length ?? 0,
+                    hasSchemas: !!chatConfig?.schemaSources?.length,
+                });
+            }
+            catch (err) {
+                logger.warn("Voice: getConfig() failed", err);
+            }
+        }
+        else if (chatConfig) {
+            logger.debug("Voice: using static chatConfig", {
+                toolCount: chatConfig.tools?.tools.length ?? 0,
+                routeCount: chatConfig.routes?.routes.length ?? 0,
+                hasSchemas: !!chatConfig.schemaSources?.length,
+            });
+        }
+        else {
+            logger.debug("Voice: no chatConfig or getConfig — only built-in tools will be available");
+        }
+        // Decorate host tools with hash ids and build the reverse lookup so we
+        // can map id-named tool calls back to the original host name when the
+        // model invokes them. Mirrors the chat-ui iframe's decoration step.
+        const decoratedManifest = this.buildDecoratedManifest(chatConfig);
+        logger.debug("Voice: decorated tools", {
+            ids: decoratedManifest.tools.map((t) => `${t.id}=${t.name}`),
+        });
+        const pageContext = this.safeExtractPageContext();
+        logger.debug("Voice: page context extracted", {
+            url: pageContext?.url,
+            title: pageContext?.title,
+            textLength: pageContext?.text?.length ?? 0,
+        });
+        let mint;
+        try {
+            logger.debug("Voice: requesting ephemeral token from mint endpoint");
+            mint = await this.mintToken(chatConfig, decoratedManifest, pageContext);
+            logger.debug("Voice: mint succeeded", {
+                voiceSessionId: mint.voiceSessionId,
+                expiresAt: mint.expiresAt,
+            });
+        }
+        catch (err) {
+            await this.failWith(err instanceof Error ? err.message : "Failed to start voice session");
+            return;
+        }
+        let micStream;
+        try {
+            logger.debug("Voice: requesting microphone access");
+            micStream = await navigator.mediaDevices.getUserMedia({ audio: true });
+            logger.debug("Voice: microphone access granted");
+        }
+        catch (err) {
+            const name = err instanceof Error ? err.name : "";
+            const message = name === "NotAllowedError" || name === "PermissionDeniedError"
+                ? "Microphone permission was denied. Enable microphone access in your browser settings to use voice mode."
+                : "Could not access microphone.";
+            await this.failWith(message);
+            return;
+        }
+        const pc = new RTCPeerConnection();
+        const audioElement = document.createElement("audio");
+        audioElement.autoplay = true;
+        audioElement.style.display = "none";
+        document.body.appendChild(audioElement);
+        pc.ontrack = (event) => {
+            logger.debug("Voice: pc.ontrack received remote audio stream");
+            if (event.streams[0]) {
+                audioElement.srcObject = event.streams[0];
+            }
+        };
+        pc.oniceconnectionstatechange = () => {
+            const s = pc.iceConnectionState;
+            logger.debug("Voice: ICE connection state →", s);
+            if (s === "failed" || s === "disconnected") {
+                void this.failWith(`WebRTC connection ${s}`);
+            }
+        };
+        pc.onconnectionstatechange = () => {
+            logger.debug("Voice: peer connection state →", pc.connectionState);
+            if (pc.connectionState === "failed") {
+                void this.failWith("WebRTC connection failed");
+            }
+        };
+        for (const track of micStream.getAudioTracks()) {
+            pc.addTrack(track, micStream);
+        }
+        const dataChannel = pc.createDataChannel("oai-events");
+        dataChannel.onmessage = (event) => {
+            const raw = typeof event.data === "string" ? event.data : "";
+            if (!raw)
+                return;
+            logger.debug("Voice: ← data channel message", raw.slice(0, 200));
+            void handleRealtimeMessage(raw, this.buildMessageContext());
+        };
+        dataChannel.onopen = () => {
+            logger.debug("Voice: data channel opened");
+            this.dispatch({ type: "connected" });
+        };
+        dataChannel.onclose = () => {
+            logger.debug("Voice: data channel closed");
+        };
+        this.resources = {
+            pc,
+            dataChannel,
+            micStream,
+            audioElement,
+            voiceSessionId: mint.voiceSessionId,
+        };
+        try {
+            logger.debug("Voice: creating WebRTC offer");
+            const offer = await pc.createOffer();
+            await pc.setLocalDescription(offer);
+            logger.debug("Voice: exchanging SDP with OpenAI Realtime");
+            const answerSdp = await this.exchangeSdp(offer, mint.clientSecret);
+            await pc.setRemoteDescription({ type: "answer", sdp: answerSdp });
+            logger.debug("Voice: WebRTC negotiation complete");
+        }
+        catch (err) {
+            await this.failWith(err instanceof Error ? err.message : "Failed to negotiate voice connection");
+            return;
+        }
+        void this.postSessionEvent("start", mint.voiceSessionId, pageContext);
+    }
+    /** Stop the session and tear down all resources. */
+    async stop() {
+        logger.debug("Voice: stop() called");
+        this.dispatch({ type: "stop" });
+        await this.teardown();
+    }
+    /** Tear down everything and remove listeners. Call once before discarding the instance. */
+    destroy() {
+        void this.teardown();
+        if (this.pageHideHandler) {
+            window.removeEventListener("pagehide", this.pageHideHandler);
+            this.pageHideHandler = null;
+        }
+        this.listeners.clear();
+    }
+    // ── Internals ───────────────────────────────────────────────────────────
+    buildMessageContext() {
+        return {
+            send: (event) => this.dispatch(event),
+            sendData: (payload) => this.sendOverDataChannel(payload),
+            dispatchToolCall: (name, args) => this.routeToolCall(name, args),
+            isDispatched: (id) => this.dispatchedCallIds.has(id),
+            markDispatched: (id) => {
+                this.dispatchedCallIds.add(id);
+            },
+            recordUsage: (usage) => this.accumulateUsage(usage),
+        };
+    }
+    accumulateUsage(usage) {
+        this.usage.responseCount += 1;
+        if (typeof usage.inputTokens === "number")
+            this.usage.inputTokens += usage.inputTokens;
+        if (typeof usage.cachedInputTokens === "number") {
+            this.usage.cachedInputTokens += usage.cachedInputTokens;
+        }
+        if (typeof usage.outputTokens === "number")
+            this.usage.outputTokens += usage.outputTokens;
+        if (typeof usage.audioInputTokens === "number") {
+            this.usage.audioInputTokens += usage.audioInputTokens;
+        }
+        if (typeof usage.audioOutputTokens === "number") {
+            this.usage.audioOutputTokens += usage.audioOutputTokens;
+        }
+        if (typeof usage.textInputTokens === "number") {
+            this.usage.textInputTokens += usage.textInputTokens;
+        }
+        if (typeof usage.textOutputTokens === "number") {
+            this.usage.textOutputTokens += usage.textOutputTokens;
+        }
+    }
+    sendOverDataChannel(payload) {
+        const channel = this.resources.dataChannel;
+        if (!channel || channel.readyState !== "open") {
+            logger.warn("Voice data channel not ready; dropping payload");
+            return;
+        }
+        try {
+            const serialized = JSON.stringify(payload);
+            logger.debug("Voice: → data channel send", serialized.slice(0, 200));
+            channel.send(serialized);
+        }
+        catch (err) {
+            logger.warn("Failed to send on voice data channel", err);
+        }
+    }
+    async routeToolCall(idOrName, args) {
+        // The model calls us back using the decorated id (e.g. yt_abc12345).
+        // Resolve it to the original host tool name; fall back to the raw value
+        // (it might be `redirect` or some non-decorated name).
+        const name = this.toolNameById.get(idOrName) ?? idOrName;
+        logger.debug("Voice: tool call dispatched", { id: idOrName, name, args });
+        // MCP tools execute server-side (the org token never reaches the browser).
+        // The model calls back with the `mcp__…` name minted by the server; relay
+        // it to the exec endpoint and feed the result back over the data channel.
+        if (name.startsWith("mcp__")) {
+            return await this.execMcpTool(name, args);
+        }
+        if (name === "redirect") {
+            const path = args?.path;
+            if (typeof path !== "string") {
+                throw new Error("redirect tool requires a string `path` argument");
+            }
+            if (this.config.onRedirect) {
+                this.config.onRedirect(path);
+            }
+            else if (typeof window !== "undefined") {
+                window.location.assign(path);
+            }
+            return { success: true, redirected: true, path };
+        }
+        if (name.startsWith("graphql_") && this.config.onGraphQLSchemaCall) {
+            const schemaName = name.slice("graphql_".length);
+            return await this.config.onGraphQLSchemaCall(schemaName, args);
+        }
+        if (name.startsWith("rest_") && this.config.onRESTSchemaCall) {
+            const schemaName = name.slice("rest_".length);
+            return await this.config.onRESTSchemaCall(schemaName, args);
+        }
+        if (this.config.onToolCall) {
+            return await this.config.onToolCall(name, args);
+        }
+        throw new Error(`No handler configured for tool: ${name}`);
+    }
+    /**
+     * Relay an MCP tool call to the server, which holds the org's credentials
+     * and executes against the remote MCP server. The browser only ever passes
+     * through the tool name, args, and the opaque result.
+     */
+    async execMcpTool(toolName, args) {
+        try {
+            const res = await fetch(`${this.apiOrigin}/api/voice/mcp-exec`, {
+                method: "POST",
+                headers: { "Content-Type": "application/json" },
+                body: JSON.stringify({
+                    appId: this.config.appId,
+                    toolName,
+                    args: args ?? {},
+                    pageContext: this.safeExtractPageContext(),
+                }),
+            });
+            if (!res.ok) {
+                const body = (await res.json().catch(() => ({})));
+                return { error: body.error ?? `MCP tool failed (${res.status})` };
+            }
+            const body = (await res.json());
+            return body.result ?? {};
+        }
+        catch (err) {
+            logger.warn("Voice: MCP tool relay failed", err);
+            return { error: "The integration could not complete this request." };
+        }
+    }
+    async mintToken(chatConfig, decoratedManifest, pageContext) {
+        const res = await fetch(`${this.apiOrigin}/api/voice/realtime-token`, {
+            method: "POST",
+            headers: { "Content-Type": "application/json" },
+            body: JSON.stringify({
+                appId: this.config.appId,
+                pageContext,
+                toolManifest: decoratedManifest,
+                routeManifest: chatConfig?.routes,
+                schemaSources: chatConfig?.schemaSources,
+            }),
+        });
+        if (!res.ok) {
+            const body = (await res.json().catch(() => ({})));
+            throw new Error(body.error ?? `Mint failed (${res.status})`);
+        }
+        return (await res.json());
+    }
+    /**
+     * Decorate the host's tool manifest + schema sources with hashed ids and
+     * populate `this.toolNameById` for reverse lookup. Mirrors the decoration
+     * the chat-ui iframe applies before sending tools to `/api/chat`.
+     */
+    buildDecoratedManifest(chatConfig) {
+        this.toolNameById.clear();
+        const decoratedHostTools = (chatConfig?.tools?.tools ?? []).map((t) => {
+            const id = generateToolId(t.name);
+            this.toolNameById.set(id, t.name);
+            return { ...t, id };
+        });
+        const decoratedSchemaTools = (chatConfig?.schemaSources ?? []).map((source) => {
+            const name = source.type === "graphql" ? `graphql_${source.name}` : `rest_${source.name}`;
+            const id = generateToolId(name);
+            this.toolNameById.set(id, name);
+            return {
+                id,
+                name,
+                description: source.type === "graphql"
+                    ? `Execute GraphQL operations against ${source.name}`
+                    : `Make REST API calls to ${source.name}`,
+            };
+        });
+        return { tools: [...decoratedHostTools, ...decoratedSchemaTools] };
+    }
+    async exchangeSdp(offer, clientSecret) {
+        const sdpResponse = await fetch(`${REALTIME_CALLS_URL}?model=${DEFAULT_REALTIME_MODEL}`, {
+            method: "POST",
+            headers: {
+                Authorization: `Bearer ${clientSecret}`,
+                "Content-Type": "application/sdp",
+            },
+            body: offer.sdp,
+        });
+        if (!sdpResponse.ok) {
+            const body = await sdpResponse.text().catch(() => "");
+            throw new Error(`SDP exchange failed (${sdpResponse.status}): ${body}`);
+        }
+        return await sdpResponse.text();
+    }
+    buildStopEventBody(voiceSessionId, pageContext) {
+        return {
+            appId: this.config.appId,
+            voiceSessionId,
+            event: "stop",
+            clientTimestamp: Date.now(),
+            pageContext,
+            usage: { ...this.usage },
+        };
+    }
+    async postSessionEvent(event, voiceSessionId, pageContext) {
+        try {
+            const body = event === "stop"
+                ? this.buildStopEventBody(voiceSessionId, pageContext)
+                : {
+                    appId: this.config.appId,
+                    voiceSessionId,
+                    event,
+                    clientTimestamp: Date.now(),
+                    pageContext,
+                };
+            await fetch(`${this.apiOrigin}/api/voice/session-event`, {
+                method: "POST",
+                headers: { "Content-Type": "application/json" },
+                body: JSON.stringify(body),
+                keepalive: event === "stop",
+            });
+        }
+        catch (err) {
+            logger.warn(`Failed to post voice.session.${event}`, err);
+        }
+    }
+    async teardown() {
+        const r = this.resources;
+        this.resources = EMPTY_RESOURCES;
+        this.dispatchedCallIds = new Set();
+        try {
+            r.dataChannel?.close();
+        }
+        catch (err) {
+            logger.warn("Error closing data channel", err);
+        }
+        try {
+            for (const track of r.micStream?.getTracks() ?? []) {
+                track.stop();
+            }
+        }
+        catch (err) {
+            logger.warn("Error stopping mic tracks", err);
+        }
+        try {
+            for (const sender of r.pc?.getSenders() ?? []) {
+                sender.track?.stop();
+            }
+            r.pc?.close();
+        }
+        catch (err) {
+            logger.warn("Error closing peer connection", err);
+        }
+        if (r.audioElement) {
+            try {
+                r.audioElement.srcObject = null;
+                r.audioElement.remove();
+            }
+            catch (err) {
+                logger.warn("Error removing audio element", err);
+            }
+        }
+        if (r.voiceSessionId) {
+            await this.postSessionEvent("stop", r.voiceSessionId, this.safeExtractPageContext());
+        }
+    }
+    async failWith(message) {
+        logger.warn("Voice session error:", message);
+        this.dispatch({ type: "error", message });
+        await this.teardown();
+    }
+    dispatch(event) {
+        const next = voiceReducer(this.machine, event);
+        if (next === this.machine)
+            return;
+        this.machine = next;
+        for (const listener of this.listeners) {
+            try {
+                listener(next);
+            }
+            catch (err) {
+                logger.warn("Voice state listener threw", err);
+            }
+        }
+    }
+    safeExtractPageContext() {
+        try {
+            return extractPageContext();
+        }
+        catch {
+            return undefined;
+        }
+    }
+    attachPageHide() {
+        if (typeof window === "undefined")
+            return;
+        this.pageHideHandler = () => {
+            const r = this.resources;
+            if (!r.voiceSessionId)
+                return;
+            const body = JSON.stringify(this.buildStopEventBody(r.voiceSessionId, undefined));
+            if (navigator.sendBeacon) {
+                navigator.sendBeacon(`${this.apiOrigin}/api/voice/session-event`, new Blob([body], { type: "application/json" }));
+            }
+        };
+        window.addEventListener("pagehide", this.pageHideHandler);
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yak-io/javascript",
-  "version": "0.7.0",
+  "version": "0.8.0",
   "description": "Core JavaScript SDK for embedding yak chatbot",
   "type": "module",
   "license": "SEE LICENSE IN LICENSE",
@@ -56,8 +56,10 @@
     }
   },
   "devDependencies": {
-    "@types/node": "^24.12.0",
+    "@types/node": "^24.12.4",
+    "jsdom": "^28.1.0",
     "typescript": "^5.3.0",
+    "vitest": "^4.1.6",
     "@repo/typescript-config": "0.0.0"
   },
   "scripts": {