@yak-io/javascript 0.10.1 → 0.11.0

package/dist/voice-machine.js

@@ -1,168 +0,0 @@
-/**
- * Pure state machine for a single voice session.
- *
- * The reducer has no DOM or WebRTC dependencies — it can be unit-tested by
- * driving events through `voiceReducer` and checking the resulting state.
- *
- * The companion `handleRealtimeMessage` parses an OpenAI Realtime data-channel
- * message and dispatches reducer events plus side effects (tool dispatch,
- * sending follow-up events back over the data channel). Side effects are
- * delegated to the injected `RealtimeMessageContext` so the function is
- * testable with a plain in-memory mock.
- */
-export const INITIAL_VOICE_MACHINE = { state: "idle" };
-export function voiceReducer(machine, event) {
-    switch (event.type) {
-        case "start":
-            return machine.state === "idle" ? { state: "connecting" } : machine;
-        case "connected":
-            return machine.state === "connecting" ? { state: "listening" } : machine;
-        case "response_requested":
-            // The assistant is generating an unprompted turn (the opening greeting).
-            // Move to `thinking` so the subsequent `audio_delta` lands on `speaking`,
-            // matching a normal turn; no-op if we're not idling in `listening`.
-            return machine.state === "listening" ? { state: "thinking" } : machine;
-        case "speech_started":
-            if (machine.state === "idle" || machine.state === "error")
-                return machine;
-            return { state: "listening" };
-        case "speech_stopped":
-            return machine.state === "listening" ? { state: "thinking" } : machine;
-        case "audio_delta":
-            if (machine.state === "thinking" || machine.state === "speaking") {
-                return { state: "speaking" };
-            }
-            return machine;
-        case "audio_stopped":
-            return machine.state === "speaking" ? { state: "listening" } : machine;
-        case "stop":
-            return { state: "idle" };
-        case "error":
-            return { state: "error", errorMessage: event.message };
-        default: {
-            const _exhaustive = event;
-            void _exhaustive;
-            return machine;
-        }
-    }
-}
-function isFunctionCall(item) {
-    return item.type === "function_call";
-}
-function parseToolArgs(raw) {
-    if (!raw)
-        return {};
-    try {
-        return JSON.parse(raw);
-    }
-    catch {
-        return {};
-    }
-}
-async function dispatchFunctionCall(call, ctx) {
-    const callId = call.call_id;
-    const name = call.name;
-    if (!callId || !name)
-        return;
-    if (ctx.isDispatched(callId))
-        return;
-    ctx.markDispatched(callId);
-    const args = parseToolArgs(call.arguments);
-    let output;
-    try {
-        const result = await ctx.dispatchToolCall(name, args);
-        output = JSON.stringify(result ?? null);
-    }
-    catch (error) {
-        output = JSON.stringify({
-            error: error instanceof Error ? error.message : "Tool execution failed",
-        });
-    }
-    ctx.sendData({
-        type: "conversation.item.create",
-        item: { type: "function_call_output", call_id: callId, output },
-    });
-    ctx.sendData({ type: "response.create" });
-}
-function extractUsage(raw) {
-    if (!raw)
-        return null;
-    const usage = {};
-    if (typeof raw.input_tokens === "number")
-        usage.inputTokens = raw.input_tokens;
-    if (typeof raw.output_tokens === "number")
-        usage.outputTokens = raw.output_tokens;
-    const inDetails = raw.input_token_details;
-    if (inDetails) {
-        if (typeof inDetails.cached_tokens === "number") {
-            usage.cachedInputTokens = inDetails.cached_tokens;
-        }
-        if (typeof inDetails.audio_tokens === "number") {
-            usage.audioInputTokens = inDetails.audio_tokens;
-        }
-        if (typeof inDetails.text_tokens === "number") {
-            usage.textInputTokens = inDetails.text_tokens;
-        }
-    }
-    const outDetails = raw.output_token_details;
-    if (outDetails) {
-        if (typeof outDetails.audio_tokens === "number") {
-            usage.audioOutputTokens = outDetails.audio_tokens;
-        }
-        if (typeof outDetails.text_tokens === "number") {
-            usage.textOutputTokens = outDetails.text_tokens;
-        }
-    }
-    return Object.keys(usage).length > 0 ? usage : null;
-}
-async function handleResponseDone(response, ctx) {
-    const usage = extractUsage(response?.usage);
-    if (usage && ctx.recordUsage) {
-        try {
-            ctx.recordUsage(usage);
-        }
-        catch {
-            // recordUsage is best-effort; never let it break the session loop.
-        }
-    }
-    const calls = (response?.output ?? []).filter(isFunctionCall);
-    for (const call of calls) {
-        await dispatchFunctionCall(call, ctx);
-    }
-}
-export async function handleRealtimeMessage(raw, ctx) {
-    let message;
-    try {
-        message = JSON.parse(raw);
-    }
-    catch {
-        return;
-    }
-    switch (message.type) {
-        case "input_audio_buffer.speech_started":
-            ctx.send({ type: "speech_started" });
-            return;
-        case "input_audio_buffer.speech_stopped":
-            ctx.send({ type: "speech_stopped" });
-            return;
-        case "response.output_audio_transcript.delta":
-        case "response.audio_transcript.delta":
-            ctx.send({ type: "audio_delta" });
-            return;
-        case "output_audio_buffer.stopped":
-        case "response.output_audio_buffer.stopped":
-            ctx.send({ type: "audio_stopped" });
-            return;
-        case "response.done":
-            await handleResponseDone(message.response, ctx);
-            return;
-        case "error":
-            ctx.send({
-                type: "error",
-                message: message.error?.message ?? "Voice session error",
-            });
-            return;
-        default:
-            return;
-    }
-}

package/dist/voice-session.js

@@ -1,520 +0,0 @@
-import { logger } from "./logger.js";
-import { extractPageContext } from "./page-context.js";
-import { uniqueToolId } from "./tool-name.js";
-import { handleRealtimeMessage, INITIAL_VOICE_MACHINE, voiceReducer, } from "./voice-machine.js";
-const DEFAULT_REALTIME_MODEL = "gpt-realtime";
-const REALTIME_CALLS_URL = "https://api.openai.com/v1/realtime/calls";
-const DEFAULT_API_ORIGIN = "https://chat.yak.io";
-/**
- * Resolves the API origin when no explicit `apiOrigin` is configured. Points at
- * a local chat UI during yak's own local development; production otherwise.
- */
-function getDefaultApiOrigin() {
-    if (typeof window !== "undefined" &&
-        (window.location.hostname === "localhost" || window.location.hostname === "127.0.0.1") &&
-        typeof window.__YAK_INTERNAL_DEV__ !== "undefined") {
-        return "http://localhost:3001";
-    }
-    return DEFAULT_API_ORIGIN;
-}
-const EMPTY_RESOURCES = {
-    pc: null,
-    dataChannel: null,
-    micStream: null,
-    audioElement: null,
-    voiceSessionId: null,
-};
-function emptyUsage() {
-    return {
-        inputTokens: 0,
-        cachedInputTokens: 0,
-        outputTokens: 0,
-        audioInputTokens: 0,
-        audioOutputTokens: 0,
-        textInputTokens: 0,
-        textOutputTokens: 0,
-        responseCount: 0,
-    };
-}
-export class YakVoiceSession {
-    config;
-    machine = INITIAL_VOICE_MACHINE;
-    resources = EMPTY_RESOURCES;
-    dispatchedCallIds = new Set();
-    listeners = new Set();
-    pageHideHandler = null;
-    /** Per-session token totals, accumulated from each `response.done` event. */
-    usage = emptyUsage();
-    /**
-     * Reverse map: hashed tool id (what OpenAI calls back with) → original host
-     * tool name (what `onToolCall` expects). Populated on every `start()` from
-     * the resolved chat config.
-     */
-    toolNameById = new Map();
-    constructor(config) {
-        this.config = config;
-        this.attachPageHide();
-    }
-    /**
-     * Resolve the API origin lazily on each call. Environment-dependent defaults
-     * (e.g. a local chat UI) may not be ready at construction time, so resolving
-     * eagerly would risk baking in the production URL.
-     */
-    get apiOrigin() {
-        return this.config.apiOrigin ?? getDefaultApiOrigin();
-    }
-    /** Update mutable config fields (handlers, getConfig). */
-    updateConfig(patch) {
-        this.config = { ...this.config, ...patch };
-    }
-    getState() {
-        return this.machine;
-    }
-    /**
-     * The current API origin (defaults to `https://chat.yak.io`). Useful for
-     * building URLs to static assets like the brand logo.
-     */
-    getApiOrigin() {
-        return this.apiOrigin;
-    }
-    onStateChange(listener) {
-        this.listeners.add(listener);
-        return () => {
-            this.listeners.delete(listener);
-        };
-    }
-    /**
-     * Begin a voice session. Should be invoked from a user gesture (button
-     * click) so `getUserMedia` and audio playback both have transient activation.
-     */
-    async start() {
-        if (this.machine.state !== "idle")
-            return;
-        logger.debug("Voice: start() called");
-        this.usage = emptyUsage();
-        this.dispatch({ type: "start" });
-        let chatConfig = this.config.chatConfig;
-        if (this.config.getConfig) {
-            try {
-                chatConfig = await this.config.getConfig();
-                logger.debug("Voice: getConfig() resolved", {
-                    toolCount: chatConfig?.tools?.tools.length ?? 0,
-                    routeCount: chatConfig?.routes?.routes.length ?? 0,
-                });
-            }
-            catch (err) {
-                logger.warn("Voice: getConfig() failed", err);
-            }
-        }
-        else if (chatConfig) {
-            logger.debug("Voice: using static chatConfig", {
-                toolCount: chatConfig.tools?.tools.length ?? 0,
-                routeCount: chatConfig.routes?.routes.length ?? 0,
-            });
-        }
-        else {
-            logger.debug("Voice: no chatConfig or getConfig — only built-in tools will be available");
-        }
-        // Decorate host tools with hash ids and build the reverse lookup so we
-        // can map id-named tool calls back to the original host name when the
-        // model invokes them. Mirrors the chat-ui iframe's decoration step.
-        const decoratedManifest = this.buildDecoratedManifest(chatConfig);
-        logger.debug("Voice: decorated tools", {
-            ids: decoratedManifest.tools.map((t) => `${t.id}=${t.name}`),
-        });
-        const pageContext = this.safeExtractPageContext();
-        logger.debug("Voice: page context extracted", {
-            url: pageContext?.url,
-            title: pageContext?.title,
-            textLength: pageContext?.text?.length ?? 0,
-        });
-        let mint;
-        try {
-            logger.debug("Voice: requesting ephemeral token from mint endpoint");
-            mint = await this.mintToken(chatConfig, decoratedManifest, pageContext);
-            logger.debug("Voice: mint succeeded", {
-                voiceSessionId: mint.voiceSessionId,
-                expiresAt: mint.expiresAt,
-            });
-        }
-        catch (err) {
-            await this.failWith(err instanceof Error ? err.message : "Failed to start voice session");
-            return;
-        }
-        let micStream;
-        try {
-            logger.debug("Voice: requesting microphone access");
-            micStream = await navigator.mediaDevices.getUserMedia({ audio: true });
-            logger.debug("Voice: microphone access granted");
-        }
-        catch (err) {
-            const name = err instanceof Error ? err.name : "";
-            const message = name === "NotAllowedError" || name === "PermissionDeniedError"
-                ? "Microphone permission was denied. Enable microphone access in your browser settings to use voice mode."
-                : "Could not access microphone.";
-            await this.failWith(message);
-            return;
-        }
-        const pc = new RTCPeerConnection();
-        const audioElement = document.createElement("audio");
-        audioElement.autoplay = true;
-        audioElement.style.display = "none";
-        document.body.appendChild(audioElement);
-        pc.ontrack = (event) => {
-            logger.debug("Voice: pc.ontrack received remote audio stream");
-            if (event.streams[0]) {
-                audioElement.srcObject = event.streams[0];
-            }
-        };
-        pc.oniceconnectionstatechange = () => {
-            const s = pc.iceConnectionState;
-            logger.debug("Voice: ICE connection state →", s);
-            if (s === "failed" || s === "disconnected") {
-                void this.failWith(`WebRTC connection ${s}`);
-            }
-        };
-        pc.onconnectionstatechange = () => {
-            logger.debug("Voice: peer connection state →", pc.connectionState);
-            if (pc.connectionState === "failed") {
-                void this.failWith("WebRTC connection failed");
-            }
-        };
-        for (const track of micStream.getAudioTracks()) {
-            pc.addTrack(track, micStream);
-        }
-        const dataChannel = pc.createDataChannel("oai-events");
-        dataChannel.onmessage = (event) => {
-            const raw = typeof event.data === "string" ? event.data : "";
-            if (!raw)
-                return;
-            logger.debug("Voice: ← data channel message", raw.slice(0, 200));
-            void handleRealtimeMessage(raw, this.buildMessageContext());
-        };
-        dataChannel.onopen = () => {
-            logger.debug("Voice: data channel opened");
-            this.dispatch({ type: "connected" });
-            // Kick off an opening greeting so the assistant speaks first instead of
-            // waiting silently for the user. The minted session already carries the
-            // full instructions (persona, governance, language, "first turn" rule),
-            // so a bare `response.create` is enough — do NOT attach response-level
-            // instructions here, which would replace the session instructions.
-            // Skip it when the app's voice intro is "none" (`autoGreet === false`);
-            // a missing flag defaults to greeting for back-compat.
-            if (mint.autoGreet !== false) {
-                this.dispatch({ type: "response_requested" });
-                this.sendOverDataChannel({ type: "response.create" });
-            }
-        };
-        dataChannel.onclose = () => {
-            logger.debug("Voice: data channel closed");
-        };
-        this.resources = {
-            pc,
-            dataChannel,
-            micStream,
-            audioElement,
-            voiceSessionId: mint.voiceSessionId,
-        };
-        try {
-            logger.debug("Voice: creating WebRTC offer");
-            const offer = await pc.createOffer();
-            await pc.setLocalDescription(offer);
-            logger.debug("Voice: exchanging SDP with OpenAI Realtime");
-            const answerSdp = await this.exchangeSdp(offer, mint.clientSecret);
-            await pc.setRemoteDescription({ type: "answer", sdp: answerSdp });
-            logger.debug("Voice: WebRTC negotiation complete");
-        }
-        catch (err) {
-            await this.failWith(err instanceof Error ? err.message : "Failed to negotiate voice connection");
-            return;
-        }
-        void this.postSessionEvent("start", mint.voiceSessionId, pageContext);
-    }
-    /** Stop the session and tear down all resources. */
-    async stop() {
-        logger.debug("Voice: stop() called");
-        this.dispatch({ type: "stop" });
-        await this.teardown();
-    }
-    /** Tear down everything and remove listeners. Call once before discarding the instance. */
-    destroy() {
-        void this.teardown();
-        if (this.pageHideHandler) {
-            window.removeEventListener("pagehide", this.pageHideHandler);
-            this.pageHideHandler = null;
-        }
-        this.listeners.clear();
-    }
-    // ── Internals ───────────────────────────────────────────────────────────
-    buildMessageContext() {
-        return {
-            send: (event) => this.dispatch(event),
-            sendData: (payload) => this.sendOverDataChannel(payload),
-            dispatchToolCall: (name, args) => this.routeToolCall(name, args),
-            isDispatched: (id) => this.dispatchedCallIds.has(id),
-            markDispatched: (id) => {
-                this.dispatchedCallIds.add(id);
-            },
-            recordUsage: (usage) => this.accumulateUsage(usage),
-        };
-    }
-    accumulateUsage(usage) {
-        this.usage.responseCount += 1;
-        if (typeof usage.inputTokens === "number")
-            this.usage.inputTokens += usage.inputTokens;
-        if (typeof usage.cachedInputTokens === "number") {
-            this.usage.cachedInputTokens += usage.cachedInputTokens;
-        }
-        if (typeof usage.outputTokens === "number")
-            this.usage.outputTokens += usage.outputTokens;
-        if (typeof usage.audioInputTokens === "number") {
-            this.usage.audioInputTokens += usage.audioInputTokens;
-        }
-        if (typeof usage.audioOutputTokens === "number") {
-            this.usage.audioOutputTokens += usage.audioOutputTokens;
-        }
-        if (typeof usage.textInputTokens === "number") {
-            this.usage.textInputTokens += usage.textInputTokens;
-        }
-        if (typeof usage.textOutputTokens === "number") {
-            this.usage.textOutputTokens += usage.textOutputTokens;
-        }
-    }
-    sendOverDataChannel(payload) {
-        const channel = this.resources.dataChannel;
-        if (!channel || channel.readyState !== "open") {
-            logger.warn("Voice data channel not ready; dropping payload");
-            return;
-        }
-        try {
-            const serialized = JSON.stringify(payload);
-            logger.debug("Voice: → data channel send", serialized.slice(0, 200));
-            channel.send(serialized);
-        }
-        catch (err) {
-            logger.warn("Failed to send on voice data channel", err);
-        }
-    }
-    async routeToolCall(idOrName, args) {
-        // The model calls us back using the decorated id (e.g. orders_list).
-        // Resolve it to the original host tool name; fall back to the raw value
-        // (it might be `redirect` or some non-decorated name).
-        const name = this.toolNameById.get(idOrName) ?? idOrName;
-        logger.debug("Voice: tool call dispatched", { id: idOrName, name, args });
-        // MCP tools execute server-side (the org token never reaches the browser).
-        // The model calls back with the `mcp__…` name minted by the server; relay
-        // it to the exec endpoint and feed the result back over the data channel.
-        if (name.startsWith("mcp__")) {
-            return await this.execMcpTool(name, args);
-        }
-        if (name === "redirect") {
-            const path = args?.path;
-            if (typeof path !== "string") {
-                throw new Error("redirect tool requires a string `path` argument");
-            }
-            if (this.config.onRedirect) {
-                this.config.onRedirect(path);
-            }
-            else if (typeof window !== "undefined") {
-                window.location.assign(path);
-            }
-            return { success: true, redirected: true, path };
-        }
-        if (this.config.onToolCall) {
-            return await this.config.onToolCall(name, args);
-        }
-        throw new Error(`No handler configured for tool: ${name}`);
-    }
-    /**
-     * Relay an MCP tool call to the server, which holds the org's credentials
-     * and executes against the remote MCP server. The browser only ever passes
-     * through the tool name, args, and the opaque result.
-     */
-    async execMcpTool(toolName, args) {
-        try {
-            const res = await fetch(`${this.apiOrigin}/api/voice/mcp-exec`, {
-                method: "POST",
-                headers: { "Content-Type": "application/json" },
-                body: JSON.stringify({
-                    appId: this.config.appId,
-                    toolName,
-                    args: args ?? {},
-                    pageContext: this.safeExtractPageContext(),
-                }),
-            });
-            if (!res.ok) {
-                const body = (await res.json().catch(() => ({})));
-                return { error: body.error ?? `MCP tool failed (${res.status})` };
-            }
-            const body = (await res.json());
-            return body.result ?? {};
-        }
-        catch (err) {
-            logger.warn("Voice: MCP tool relay failed", err);
-            return { error: "The integration could not complete this request." };
-        }
-    }
-    async mintToken(chatConfig, decoratedManifest, pageContext) {
-        const res = await fetch(`${this.apiOrigin}/api/voice/realtime-token`, {
-            method: "POST",
-            headers: { "Content-Type": "application/json" },
-            body: JSON.stringify({
-                appId: this.config.appId,
-                pageContext,
-                toolManifest: decoratedManifest,
-                routeManifest: chatConfig?.routes,
-            }),
-        });
-        if (!res.ok) {
-            const body = (await res.json().catch(() => ({})));
-            throw new Error(body.error ?? `Mint failed (${res.status})`);
-        }
-        return (await res.json());
-    }
-    /**
-     * Decorate the host's tool manifest with readable, collision-free model-facing ids
-     * and populate `this.toolNameById` for reverse lookup. Mirrors the decoration the
-     * chat-ui iframe applies before sending tools to `/api/chat`. GraphQL/REST tools are
-     * ordinary manifest entries here (contributed by their adapters), so no special-casing
-     * is needed.
-     */
-    buildDecoratedManifest(chatConfig) {
-        this.toolNameById.clear();
-        const used = new Set(["redirect"]);
-        const decoratedHostTools = (chatConfig?.tools?.tools ?? []).map((t) => {
-            const id = uniqueToolId(t.name, used);
-            used.add(id);
-            this.toolNameById.set(id, t.name);
-            return { ...t, id };
-        });
-        return { tools: decoratedHostTools };
-    }
-    async exchangeSdp(offer, clientSecret) {
-        const sdpResponse = await fetch(`${REALTIME_CALLS_URL}?model=${DEFAULT_REALTIME_MODEL}`, {
-            method: "POST",
-            headers: {
-                Authorization: `Bearer ${clientSecret}`,
-                "Content-Type": "application/sdp",
-            },
-            body: offer.sdp,
-        });
-        if (!sdpResponse.ok) {
-            const body = await sdpResponse.text().catch(() => "");
-            throw new Error(`SDP exchange failed (${sdpResponse.status}): ${body}`);
-        }
-        return await sdpResponse.text();
-    }
-    buildStopEventBody(voiceSessionId, pageContext) {
-        return {
-            appId: this.config.appId,
-            voiceSessionId,
-            event: "stop",
-            clientTimestamp: Date.now(),
-            pageContext,
-            usage: { ...this.usage },
-        };
-    }
-    async postSessionEvent(event, voiceSessionId, pageContext) {
-        try {
-            const body = event === "stop"
-                ? this.buildStopEventBody(voiceSessionId, pageContext)
-                : {
-                    appId: this.config.appId,
-                    voiceSessionId,
-                    event,
-                    clientTimestamp: Date.now(),
-                    pageContext,
-                };
-            await fetch(`${this.apiOrigin}/api/voice/session-event`, {
-                method: "POST",
-                headers: { "Content-Type": "application/json" },
-                body: JSON.stringify(body),
-                keepalive: event === "stop",
-            });
-        }
-        catch (err) {
-            logger.warn(`Failed to post voice.session.${event}`, err);
-        }
-    }
-    async teardown() {
-        const r = this.resources;
-        this.resources = EMPTY_RESOURCES;
-        this.dispatchedCallIds = new Set();
-        try {
-            r.dataChannel?.close();
-        }
-        catch (err) {
-            logger.warn("Error closing data channel", err);
-        }
-        try {
-            for (const track of r.micStream?.getTracks() ?? []) {
-                track.stop();
-            }
-        }
-        catch (err) {
-            logger.warn("Error stopping mic tracks", err);
-        }
-        try {
-            for (const sender of r.pc?.getSenders() ?? []) {
-                sender.track?.stop();
-            }
-            r.pc?.close();
-        }
-        catch (err) {
-            logger.warn("Error closing peer connection", err);
-        }
-        if (r.audioElement) {
-            try {
-                r.audioElement.srcObject = null;
-                r.audioElement.remove();
-            }
-            catch (err) {
-                logger.warn("Error removing audio element", err);
-            }
-        }
-        if (r.voiceSessionId) {
-            await this.postSessionEvent("stop", r.voiceSessionId, this.safeExtractPageContext());
-        }
-    }
-    async failWith(message) {
-        logger.warn("Voice session error:", message);
-        this.dispatch({ type: "error", message });
-        await this.teardown();
-    }
-    dispatch(event) {
-        const next = voiceReducer(this.machine, event);
-        if (next === this.machine)
-            return;
-        this.machine = next;
-        for (const listener of this.listeners) {
-            try {
-                listener(next);
-            }
-            catch (err) {
-                logger.warn("Voice state listener threw", err);
-            }
-        }
-    }
-    safeExtractPageContext() {
-        try {
-            return extractPageContext();
-        }
-        catch {
-            return undefined;
-        }
-    }
-    attachPageHide() {
-        if (typeof window === "undefined")
-            return;
-        this.pageHideHandler = () => {
-            const r = this.resources;
-            if (!r.voiceSessionId)
-                return;
-            const body = JSON.stringify(this.buildStopEventBody(r.voiceSessionId, undefined));
-            if (navigator.sendBeacon) {
-                navigator.sendBeacon(`${this.apiOrigin}/api/voice/session-event`, new Blob([body], { type: "application/json" }));
-            }
-        };
-        window.addEventListener("pagehide", this.pageHideHandler);
-    }
-}