@cairnvibe/sdk 0.2.8 → 0.2.9

package/dist/index.js

@@ -106,7 +106,18 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
     const rtMicMutedRef = (0, react_1.useRef)(false);
     const rtSpeakerMutedRef = (0, react_1.useRef)(false);
     const rtStartingRef = (0, react_1.useRef)(false); // closes the click-to-first-state-update gap so a rapid double-click can't open two sessions
-    const activeAudioRef = (0, react_1.useRef)(null);
+    // Progressive PCM playback for the buffered (non-realtime) speak endpoint
+    // — the same gapless AudioBufferSourceNode scheduling the realtime path
+    // uses for its audio_chunk messages (see rtPlaybackCtxRef below), just fed
+    // by a fetch() ReadableStream instead of WebSocket messages. This exists
+    // because res.blob()/res.arrayBuffer() always wait for the whole response
+    // body in every browser no matter how the server sent it — streaming the
+    // wire alone (speak-server.ts) doesn't help unless playback also starts
+    // before the full reply has arrived.
+    const typedPlaybackCtxRef = (0, react_1.useRef)(null);
+    const typedPlaybackGainRef = (0, react_1.useRef)(null);
+    const typedNextPlayTimeRef = (0, react_1.useRef)(0);
+    const typedScheduledSourcesRef = (0, react_1.useRef)([]);
     // Watchdog for the "rt-thinking" state: started on every "final" transcript,
     // cleared the moment the server responds with anything for that turn
     // (verb/speaking_start/speaking_end/turn_complete/error). If it ever
@@ -433,30 +444,105 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
             { role: "assistant", text: "(gave up after too many steps)" },
         ].slice(-MAX_HISTORY_TURNS);
     }
+    function ensureTypedPlaybackGraph() {
+        if (!typedPlaybackCtxRef.current) {
+            const ctx = new AudioContext();
+            const gain = ctx.createGain();
+            gain.connect(ctx.destination);
+            typedPlaybackCtxRef.current = ctx;
+            typedPlaybackGainRef.current = gain;
+        }
+        return { ctx: typedPlaybackCtxRef.current, gain: typedPlaybackGainRef.current };
+    }
+    /** Stops whatever's currently playing on the typed/mic path's playback
+     * graph, so two responses (e.g. a rapid double-click, or two answers
+     * resolved close together) can never be heard overlapping. */
+    function stopTypedPlayback() {
+        for (const source of typedScheduledSourcesRef.current) {
+            source.onended = null;
+            try {
+                source.stop();
+            }
+            catch {
+                // may already have finished naturally
+            }
+        }
+        typedScheduledSourcesRef.current = [];
+        typedNextPlayTimeRef.current = typedPlaybackCtxRef.current?.currentTime ?? 0;
+    }
+    function concatBytes(a, b) {
+        const out = new Uint8Array(a.length + b.length);
+        out.set(a, 0);
+        out.set(b, a.length);
+        return out;
+    }
     /**
-     * The one place that starts audio playback for a spoken response — stops
-     * whatever's currently playing first, so two responses (e.g. a rapid
-     * double-click on "start conversation", or two utterances resolved close
-     * together) can never be heard overlapping. Used by both the typed/mic
-     * path and the realtime path.
+     * Reads a raw linear16 PCM stream (mono, 24kHz — matches speak-server.ts)
+     * and schedules it gapless-appended into the Web Audio graph as chunks
+     * arrive — the same technique the realtime path uses for its audio_chunk
+     * messages, just driven by a fetch() reader instead of WebSocket frames.
+     * Resolves once every scheduled chunk has actually finished *playing*,
+     * not just finished arriving.
      */
-    function playResponseAudio(blob) {
-        if (activeAudioRef.current) {
-            activeAudioRef.current.pause();
-            activeAudioRef.current.currentTime = 0;
-        }
-        const url = URL.createObjectURL(blob);
-        const audio = new Audio(url);
-        activeAudioRef.current = audio;
+    function playPcmStream(stream) {
+        stopTypedPlayback();
+        const { ctx, gain } = ensureTypedPlaybackGraph();
+        void ctx.resume().catch(() => { });
         return new Promise((resolve) => {
-            const clear = () => {
-                URL.revokeObjectURL(url);
-                if (activeAudioRef.current === audio)
-                    activeAudioRef.current = null;
-                resolve();
+            let doneArriving = false;
+            let leftover = new Uint8Array(0);
+            const maybeResolve = () => {
+                if (doneArriving && typedScheduledSourcesRef.current.length === 0)
+                    resolve();
+            };
+            const scheduleChunk = (bytes) => {
+                const sampleCount = Math.floor(bytes.length / 2);
+                if (sampleCount === 0)
+                    return;
+                const float32 = new Float32Array(sampleCount);
+                const view = new DataView(bytes.buffer, bytes.byteOffset, sampleCount * 2);
+                for (let i = 0; i < sampleCount; i++)
+                    float32[i] = view.getInt16(i * 2, true) / 32768;
+                const buffer = ctx.createBuffer(1, sampleCount, 24000);
+                buffer.copyToChannel(float32, 0);
+                const source = ctx.createBufferSource();
+                source.buffer = buffer;
+                source.connect(gain);
+                const startAt = Math.max(ctx.currentTime, typedNextPlayTimeRef.current);
+                source.start(startAt);
+                typedNextPlayTimeRef.current = startAt + buffer.duration;
+                typedScheduledSourcesRef.current.push(source);
+                source.onended = () => {
+                    typedScheduledSourcesRef.current = typedScheduledSourcesRef.current.filter((s) => s !== source);
+                    maybeResolve();
+                };
             };
-            audio.onended = clear;
-            audio.play().catch(clear);
+            (async () => {
+                const reader = stream.getReader();
+                try {
+                    for (;;) {
+                        const { done, value } = await reader.read();
+                        if (done)
+                            break;
+                        if (!value || value.length === 0)
+                            continue;
+                        // PCM16 samples are 2 bytes each — a chunk boundary can split a
+                        // sample in half, so carry any odd trailing byte into the next
+                        // read instead of corrupting one sample at every chunk seam.
+                        const combined = concatBytes(leftover, value);
+                        const usableLen = combined.length - (combined.length % 2);
+                        scheduleChunk(combined.subarray(0, usableLen));
+                        leftover = combined.subarray(usableLen);
+                    }
+                }
+                catch {
+                    // Best-effort — never let a stream read failure hang the caller forever.
+                }
+                finally {
+                    doneArriving = true;
+                    maybeResolve();
+                }
+            })();
         });
     }
     async function speak(text) {
@@ -468,9 +554,9 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
                 headers: { "content-type": "application/json" },
                 body: JSON.stringify({ text }),
             });
-            if (!res.ok)
+            if (!res.ok || !res.body)
                 return;
-            void playResponseAudio(await res.blob());
+            void playPcmStream(res.body);
         }
         catch {
             // Best-effort — never let speech playback break the widget.
@@ -488,9 +574,9 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
                 headers: { "content-type": "application/json" },
                 body: JSON.stringify({ text }),
             });
-            if (!res.ok)
+            if (!res.ok || !res.body)
                 return;
-            await playResponseAudio(await res.blob());
+            await playPcmStream(res.body);
         }
         catch {
             // Best-effort — never let a synthesis failure hang the tour forever.
@@ -884,8 +970,7 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
             rtThinkingWatchdogRef.current = null;
         }
         rtStartingRef.current = false;
-        activeAudioRef.current?.pause();
-        activeAudioRef.current = null;
+        stopTypedPlayback();
         rtSocketRef.current?.close();
         rtSocketRef.current = null;
         rtCleanupRef.current?.();

package/dist/realtime-server.js

@@ -39,6 +39,12 @@ const tts_stream_1 = require("./tts-stream");
 const DEEPGRAM_LIVE_URL = "wss://api.deepgram.com/v1/listen";
 const DEFAULT_STT_MODEL = "nova-2";
 const DEFAULT_TTS_VOICE = "aura-2-thalia-en";
+// The Talker half of a Talker/Reasoner split (see finalizeTurn): spoken the
+// instant a turn turns out to need more than one step, so the user hears
+// something within about a second instead of dead air while the real
+// multi-step work runs. A short rotating set, not one fixed line, so it
+// doesn't read as a canned bot phrase on every multi-step question.
+const ACK_PHRASES = ["Let me check that for you.", "One moment, let me look into that.", "Give me a second to check.", "Let me take a look."];
 // Not constrained by any telephony 8kHz requirement — this is just "what
 // quality does Deepgram render at" for browser playback, and the Web Audio
 // API resamples an AudioBuffer at any declared rate transparently.
@@ -348,6 +354,14 @@ async function finalizeTurn(turnState, client, deps, getContext, speakStreamed,
     const myGeneration = getGeneration();
     safeSend(client, { type: "final", text: transcript });
     let loopHistory = history;
+    // The Talker: set once, the first time a turn turns out to need more
+    // than one step (see the loop below) — a real, in-flight speakStreamed()
+    // call, never awaited until we're actually ready to speak the real
+    // answer. Deliberately not re-triggered per step: the Speak connection
+    // (speakStreamed) only ever handles one utterance at a time, so a second
+    // ack mid-loop would race the first one's own audio_chunk/Flushed
+    // handling instead of queuing cleanly.
+    let ackPromise = null;
     try {
         for (let i = 0; i < MAX_LOOP_ITERATIONS; i++) {
             const { route, visible, liveElements, webMcpTools } = getContext();
@@ -367,9 +381,19 @@ async function finalizeTurn(turnState, client, deps, getContext, speakStreamed,
             // speak, not after.
             safeSend(client, { type: "verb", verb });
             if (!core_1.TERMINAL_VERBS.has(verb.verb)) {
-                // A continuing step — no speech for it (keeps the loop fast;
-                // the client still shows it visually) — wait for its real result
-                // and go around again instead of ending the turn.
+                if (i === 0) {
+                    // This turn just revealed it needs more than one step — speak a
+                    // quick, cheap acknowledgment *now*, in parallel with the rest
+                    // of the loop's own real work below (not awaited here), so the
+                    // user hears something within about a second instead of dead
+                    // air for however long the real multi-step answer takes.
+                    // Single-step turns (the common case) never reach this branch
+                    // at all, so they keep today's latency exactly as it is.
+                    ackPromise = speakStreamed(ACK_PHRASES[Math.floor(Math.random() * ACK_PHRASES.length)]);
+                }
+                // A continuing step itself stays silent (keeps the loop fast; the
+                // client still shows it visually) — wait for its real result and
+                // go around again instead of ending the turn.
                 const observation = await waitForToolResult();
                 if (myGeneration !== getGeneration())
                     return;
@@ -381,6 +405,17 @@ async function finalizeTurn(turnState, client, deps, getContext, speakStreamed,
             }
             history.push({ role: "user", text: transcript }, { role: "assistant", text: summarizeVerbForHistory(verb) });
             history.splice(0, Math.max(0, history.length - MAX_HISTORY_TURNS));
+            if (ackPromise) {
+                // Never start a second speakStreamed call before the first (the
+                // ack) has actually finished — same single Speak connection, one
+                // utterance at a time. In the common multi-step case the real
+                // work below already took about as long as the ack itself did, so
+                // this rarely adds a real wait.
+                await ackPromise;
+                ackPromise = null;
+                if (myGeneration !== getGeneration())
+                    return; // a barge-in could have landed during the ack itself
+            }
             // A verb with no spoken text (highlight/navigate/do often have none)
             // still needs to unstick the client's "thinking" state and let the mic
             // resume — turn_complete covers that with no audio path involved.
@@ -397,6 +432,11 @@ async function finalizeTurn(turnState, client, deps, getContext, speakStreamed,
         history.push({ role: "user", text: transcript }, { role: "assistant", text: "(gave up after too many steps)" });
         history.splice(0, Math.max(0, history.length - MAX_HISTORY_TURNS));
         safeSend(client, { type: "verb", verb: { verb: "explain", text: "I wasn't able to finish that — try asking again or breaking it into smaller steps." } });
+        if (ackPromise) {
+            await ackPromise;
+            if (myGeneration !== getGeneration())
+                return;
+        }
         await speakStreamed("I wasn't able to finish that — try asking again or breaking it into smaller steps.");
     }
     catch (err) {

package/dist/server.d.ts

@@ -93,6 +93,7 @@ export declare class GroqVerbLLM implements VerbLLM {
     private clientFactory;
     constructor(keys: KeyRotator, model: string, toolSchema: Record<string, unknown>, clientFactory?: (apiKey: string) => GroqLikeClient);
     respond(systemPrompt: string, userMessage: string): Promise<unknown>;
+    private attemptRespond;
 }
 /**
  * A compact route directory — NOT every element on every page. Found live

package/dist/server.js

@@ -210,6 +210,28 @@ class GroqVerbLLM {
         this.clientFactory = clientFactory;
     }
     async respond(systemPrompt, userMessage) {
+        try {
+            return await this.attemptRespond(systemPrompt, userMessage);
+        }
+        catch (err) {
+            // Real, live bug, not theoretical: openai/gpt-oss-120b (a reasoning-
+            // capable open model) occasionally "thinks out loud" in plain prose
+            // instead of emitting the forced tool call — Groq's own server-side
+            // validation rejects that outright, a 400 with code
+            // "output_parse_failed", before this code ever sees a real response
+            // to work with. Non-deterministic (found live re-asking the exact
+            // same question a moment later succeeded cleanly), so one retry —
+            // not exponential backoff, this is a latency-sensitive voice/chat
+            // path — genuinely helps rather than just delaying the same
+            // failure. Anything else still propagates to resolveVerb's own
+            // catch, unchanged.
+            if (isOutputParseFailure(err)) {
+                return await this.attemptRespond(systemPrompt, userMessage);
+            }
+            throw err;
+        }
+    }
+    async attemptRespond(systemPrompt, userMessage) {
         const client = this.clientFactory(this.keys.take());
         const completion = await client.chat.completions.create({
             model: this.model,
@@ -241,6 +263,19 @@ class GroqVerbLLM {
     }
 }
 exports.GroqVerbLLM = GroqVerbLLM;
+/** Groq's SDK doesn't export a stable error shape to import and check
+ * against, so this checks defensively across the ways the real error has
+ * actually been observed to surface — a thrown APIError with a nested
+ * `.error.code`, a plain `.code`, or just the code string showing up
+ * somewhere in the message — rather than relying on exactly one of them. */
+function isOutputParseFailure(err) {
+    if (!err || typeof err !== "object")
+        return false;
+    const e = err;
+    if (e.code === "output_parse_failed" || e.error?.code === "output_parse_failed")
+        return true;
+    return typeof e.message === "string" && e.message.includes("output_parse_failed");
+}
 // ---------------------------------------------------------------------------
 // Shared tool schema / system prompt
 // ---------------------------------------------------------------------------

package/dist/speak-server.d.ts

@@ -1,16 +1,26 @@
+import { DeepgramSpeakStream, type DeepgramSpeakStreamOptions, type SpeakChunkCallback } from "./tts-stream";
 export interface CreateSpeakHandlerOptions {
     apiKey: string;
     model?: string;
 }
 export interface SpeakResult {
     status: number;
-    /** `audio` is raw MP3 bytes on success. */
+    /** `stream` yields raw linear16 PCM chunks (mono, 24kHz) as Deepgram
+     * renders them — forward it directly, unbuffered; do not await it into a
+     * Blob/ArrayBuffer or the whole point of streaming is lost. */
     body: {
-        audio: ArrayBuffer;
+        stream: ReadableStream<Uint8Array>;
         contentType: string;
     } | {
         error: string;
     };
 }
 export type SpeakHandler = (text: string) => Promise<SpeakResult>;
-export declare function createSpeakHandler(options: CreateSpeakHandlerOptions): SpeakHandler;
+/** Test-only seam: lets tests inject a fake stream instead of opening a real
+ * Deepgram WebSocket. Not part of CreateSpeakHandlerOptions on purpose — real
+ * call sites (the scaffolded route templates) never pass this. */
+export type SpeakStreamFactory = (opts: DeepgramSpeakStreamOptions, onAudioChunk: SpeakChunkCallback, handlers?: {
+    onFlushed?: (sequenceId: number) => void;
+    onError?: (err: Error) => void;
+}) => DeepgramSpeakStream;
+export declare function createSpeakHandler(options: CreateSpeakHandlerOptions, streamFactory?: SpeakStreamFactory): SpeakHandler;

package/dist/speak-server.js

@@ -1,41 +1,97 @@
 "use strict";
-// Server-side text-to-speech for the Copilot widget's spoken answers (see
-// `speakEndpoint` in index.tsx). The Deepgram key must never reach the
-// client, so this is a plain fetch to Deepgram's /v1/speak REST endpoint —
-// no SDK dependency needed for one request shape.
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.createSpeakHandler = createSpeakHandler;
-const DEEPGRAM_SPEAK_URL = "https://api.deepgram.com/v1/speak";
-// Verified against Deepgram's docs while building this — re-check if this
-// starts erroring, voice model names retire over time.
+// Server-side text-to-speech for the Copilot widget's spoken answers (see
+// `speakEndpoint` in index.tsx). The Deepgram key must never reach the
+// client.
+//
+// This used to be one fetch to Deepgram's /v1/speak REST endpoint, buffered
+// into an ArrayBuffer with `await response.arrayBuffer()` before returning
+// anything. That's the exact same bug the realtime path already fixed once
+// (see tts-stream.ts's own comment): nothing plays until Deepgram renders
+// AND the network delivers the *entire* reply, which measured 5-8s for a
+// normal explain answer in real production logs. It also hit Deepgram's
+// REST-only 2000-character cap on longer replies with no handling at all.
+//
+// Fixed the same way the realtime path was: open the streaming Speak
+// WebSocket (tts-stream.ts's DeepgramSpeakStream, the same class the
+// realtime server already uses) and forward audio chunks to the caller as
+// they arrive, via a ReadableStream — not buffered. The route handler
+// forwards that stream straight through as the HTTP response body, and the
+// client (index.tsx) reads it progressively instead of awaiting a full
+// Blob, so playback can start on the first chunk. Splitting the text into
+// sentence-sized `Speak` messages before one `Flush` sidesteps the old
+// 2000-char REST limit entirely (it doesn't apply to the WS protocol) and
+// lets Deepgram start rendering the first sentence sooner.
+const tts_stream_1 = require("./tts-stream");
 const DEEPGRAM_DEFAULT_VOICE = "aura-2-thalia-en";
-function createSpeakHandler(options) {
+// Matches the realtime path's own playback sample rate (index.tsx's
+// audio_chunk handling defaults to 24000 too) — keeping them identical lets
+// both paths share one raw-PCM16 decode/schedule routine on the client.
+const SAMPLE_RATE = 24000;
+// Not a protocol limit (the WS Speak protocol has none like REST's 2000
+// chars) — just keeps each queued chunk sentence-sized so Deepgram can start
+// rendering the first one quickly instead of parsing one giant message.
+const MAX_CHUNK_CHARS = 300;
+function splitIntoChunks(text, maxChars) {
+    const sentences = text.match(/[^.!?]+[.!?]*\s*/g) ?? [text];
+    const chunks = [];
+    let current = "";
+    for (const sentence of sentences) {
+        if (current && current.length + sentence.length > maxChars) {
+            chunks.push(current);
+            current = "";
+        }
+        current += sentence;
+    }
+    if (current)
+        chunks.push(current);
+    return chunks;
+}
+function createSpeakHandler(options, streamFactory = (opts, onAudioChunk, handlers) => new tts_stream_1.DeepgramSpeakStream(opts, onAudioChunk, handlers)) {
     const model = options.model ?? process.env.DEEPGRAM_VOICE ?? DEEPGRAM_DEFAULT_VOICE;
     return async function handleSpeak(text) {
         if (!text || !text.trim()) {
             return { status: 400, body: { error: "no text provided" } };
         }
-        let response;
+        let enqueue = null;
+        let closeOut = null;
+        let failOut = null;
+        const stream = new ReadableStream({
+            start(controller) {
+                enqueue = (chunk) => controller.enqueue(chunk);
+                closeOut = () => controller.close();
+                failOut = (err) => controller.error(err);
+            },
+        });
+        // True once connect() below resolves — an onError before that point is
+        // already reported through connect()'s own rejection, so it's ignored
+        // here to avoid double-handling the same failure.
+        let connected = false;
+        const speakStream = streamFactory({ apiKey: options.apiKey, model, encoding: "linear16", sampleRate: SAMPLE_RATE }, (chunk) => enqueue?.(new Uint8Array(chunk)), {
+            onFlushed: () => {
+                closeOut?.();
+                speakStream.close();
+            },
+            onError: (err) => {
+                if (!connected)
+                    return;
+                console.error("[cairn] speak stream error:", err);
+                failOut?.(err);
+            },
+        });
         try {
-            response = await fetch(`${DEEPGRAM_SPEAK_URL}?model=${encodeURIComponent(model)}`, {
-                method: "POST",
-                headers: {
-                    Authorization: `Token ${options.apiKey}`,
-                    "content-type": "application/json",
-                },
-                body: JSON.stringify({ text }),
-            });
+            await speakStream.connect();
         }
         catch (err) {
             console.error("[cairn] speak request failed:", err);
             return { status: 200, body: { error: "speech service unreachable" } };
         }
-        if (!response.ok) {
-            const detail = await response.text().catch(() => "");
-            console.error("[cairn] Deepgram speak returned an error:", response.status, detail);
-            return { status: 200, body: { error: "speech synthesis failed" } };
+        connected = true;
+        for (const chunk of splitIntoChunks(text, MAX_CHUNK_CHARS)) {
+            speakStream.sendText(chunk);
         }
-        const audio = await response.arrayBuffer();
-        return { status: 200, body: { audio, contentType: response.headers.get("content-type") ?? "audio/mpeg" } };
+        speakStream.flush();
+        return { status: 200, body: { stream, contentType: `audio/L16;rate=${SAMPLE_RATE}` } };
     };
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cairnvibe/sdk",
-  "version": "0.2.8",
+  "version": "0.2.9",
   "description": "In-app AI copilot — <Copilot/> for React/Next.js, <cairn-widget> for any framework — plus the server handlers and realtime voice relay behind them.",
   "license": "MIT",
   "publishConfig": { "access": "public" },

package/src/index.tsx

@@ -161,7 +161,18 @@ export function Copilot({
   const rtMicMutedRef = useRef(false);
   const rtSpeakerMutedRef = useRef(false);
   const rtStartingRef = useRef(false); // closes the click-to-first-state-update gap so a rapid double-click can't open two sessions
-  const activeAudioRef = useRef<HTMLAudioElement | null>(null);
+  // Progressive PCM playback for the buffered (non-realtime) speak endpoint
+  // — the same gapless AudioBufferSourceNode scheduling the realtime path
+  // uses for its audio_chunk messages (see rtPlaybackCtxRef below), just fed
+  // by a fetch() ReadableStream instead of WebSocket messages. This exists
+  // because res.blob()/res.arrayBuffer() always wait for the whole response
+  // body in every browser no matter how the server sent it — streaming the
+  // wire alone (speak-server.ts) doesn't help unless playback also starts
+  // before the full reply has arrived.
+  const typedPlaybackCtxRef = useRef<AudioContext | null>(null);
+  const typedPlaybackGainRef = useRef<GainNode | null>(null);
+  const typedNextPlayTimeRef = useRef(0);
+  const typedScheduledSourcesRef = useRef<AudioBufferSourceNode[]>([]);
   // Watchdog for the "rt-thinking" state: started on every "final" transcript,
   // cleared the moment the server responds with anything for that turn
   // (verb/speaking_start/speaking_end/turn_complete/error). If it ever
@@ -497,29 +508,108 @@ export function Copilot({
     ].slice(-MAX_HISTORY_TURNS);
   }
 
+  function ensureTypedPlaybackGraph(): { ctx: AudioContext; gain: GainNode } {
+    if (!typedPlaybackCtxRef.current) {
+      const ctx = new AudioContext();
+      const gain = ctx.createGain();
+      gain.connect(ctx.destination);
+      typedPlaybackCtxRef.current = ctx;
+      typedPlaybackGainRef.current = gain;
+    }
+    return { ctx: typedPlaybackCtxRef.current, gain: typedPlaybackGainRef.current! };
+  }
+
+  /** Stops whatever's currently playing on the typed/mic path's playback
+   * graph, so two responses (e.g. a rapid double-click, or two answers
+   * resolved close together) can never be heard overlapping. */
+  function stopTypedPlayback() {
+    for (const source of typedScheduledSourcesRef.current) {
+      source.onended = null;
+      try {
+        source.stop();
+      } catch {
+        // may already have finished naturally
+      }
+    }
+    typedScheduledSourcesRef.current = [];
+    typedNextPlayTimeRef.current = typedPlaybackCtxRef.current?.currentTime ?? 0;
+  }
+
+  function concatBytes(a: Uint8Array, b: Uint8Array): Uint8Array {
+    const out = new Uint8Array(a.length + b.length);
+    out.set(a, 0);
+    out.set(b, a.length);
+    return out;
+  }
+
   /**
-   * The one place that starts audio playback for a spoken response — stops
-   * whatever's currently playing first, so two responses (e.g. a rapid
-   * double-click on "start conversation", or two utterances resolved close
-   * together) can never be heard overlapping. Used by both the typed/mic
-   * path and the realtime path.
+   * Reads a raw linear16 PCM stream (mono, 24kHz — matches speak-server.ts)
+   * and schedules it gapless-appended into the Web Audio graph as chunks
+   * arrive — the same technique the realtime path uses for its audio_chunk
+   * messages, just driven by a fetch() reader instead of WebSocket frames.
+   * Resolves once every scheduled chunk has actually finished *playing*,
+   * not just finished arriving.
    */
-  function playResponseAudio(blob: Blob): Promise<void> {
-    if (activeAudioRef.current) {
-      activeAudioRef.current.pause();
-      activeAudioRef.current.currentTime = 0;
-    }
-    const url = URL.createObjectURL(blob);
-    const audio = new Audio(url);
-    activeAudioRef.current = audio;
+  function playPcmStream(stream: ReadableStream<Uint8Array>): Promise<void> {
+    stopTypedPlayback();
+    const { ctx, gain } = ensureTypedPlaybackGraph();
+    void ctx.resume().catch(() => {});
+
     return new Promise((resolve) => {
-      const clear = () => {
-        URL.revokeObjectURL(url);
-        if (activeAudioRef.current === audio) activeAudioRef.current = null;
-        resolve();
+      let doneArriving = false;
+      let leftover: Uint8Array<ArrayBufferLike> = new Uint8Array(0);
+
+      const maybeResolve = () => {
+        if (doneArriving && typedScheduledSourcesRef.current.length === 0) resolve();
       };
-      audio.onended = clear;
-      audio.play().catch(clear);
+
+      const scheduleChunk = (bytes: Uint8Array) => {
+        const sampleCount = Math.floor(bytes.length / 2);
+        if (sampleCount === 0) return;
+        const float32 = new Float32Array(sampleCount);
+        const view = new DataView(bytes.buffer, bytes.byteOffset, sampleCount * 2);
+        for (let i = 0; i < sampleCount; i++) float32[i] = view.getInt16(i * 2, true) / 32768;
+
+        const buffer = ctx.createBuffer(1, sampleCount, 24000);
+        buffer.copyToChannel(float32, 0);
+
+        const source = ctx.createBufferSource();
+        source.buffer = buffer;
+        source.connect(gain);
+
+        const startAt = Math.max(ctx.currentTime, typedNextPlayTimeRef.current);
+        source.start(startAt);
+        typedNextPlayTimeRef.current = startAt + buffer.duration;
+
+        typedScheduledSourcesRef.current.push(source);
+        source.onended = () => {
+          typedScheduledSourcesRef.current = typedScheduledSourcesRef.current.filter((s) => s !== source);
+          maybeResolve();
+        };
+      };
+
+      (async () => {
+        const reader = stream.getReader();
+        try {
+          for (;;) {
+            const { done, value } = await reader.read();
+            if (done) break;
+            if (!value || value.length === 0) continue;
+            // PCM16 samples are 2 bytes each — a chunk boundary can split a
+            // sample in half, so carry any odd trailing byte into the next
+            // read instead of corrupting one sample at every chunk seam.
+            const combined = concatBytes(leftover, value);
+            const usableLen = combined.length - (combined.length % 2);
+            scheduleChunk(combined.subarray(0, usableLen));
+            leftover = combined.subarray(usableLen);
+          }
+        } catch {
+          // Best-effort — never let a stream read failure hang the caller forever.
+        } finally {
+          doneArriving = true;
+          maybeResolve();
+        }
+      })();
     });
   }
 
@@ -531,8 +621,8 @@ export function Copilot({
         headers: { "content-type": "application/json" },
         body: JSON.stringify({ text }),
       });
-      if (!res.ok) return;
-      void playResponseAudio(await res.blob());
+      if (!res.ok || !res.body) return;
+      void playPcmStream(res.body);
     } catch {
       // Best-effort — never let speech playback break the widget.
     }
@@ -549,8 +639,8 @@ export function Copilot({
         headers: { "content-type": "application/json" },
         body: JSON.stringify({ text }),
       });
-      if (!res.ok) return;
-      await playResponseAudio(await res.blob());
+      if (!res.ok || !res.body) return;
+      await playPcmStream(res.body);
     } catch {
       // Best-effort — never let a synthesis failure hang the tour forever.
     }
@@ -938,8 +1028,7 @@ export function Copilot({
       rtThinkingWatchdogRef.current = null;
     }
     rtStartingRef.current = false;
-    activeAudioRef.current?.pause();
-    activeAudioRef.current = null;
+    stopTypedPlayback();
     rtSocketRef.current?.close();
     rtSocketRef.current = null;
     rtCleanupRef.current?.();

package/src/realtime-server.ts

@@ -34,6 +34,12 @@ import { DeepgramSpeakStream } from "./tts-stream";
 const DEEPGRAM_LIVE_URL = "wss://api.deepgram.com/v1/listen";
 const DEFAULT_STT_MODEL = "nova-2";
 const DEFAULT_TTS_VOICE = "aura-2-thalia-en";
+// The Talker half of a Talker/Reasoner split (see finalizeTurn): spoken the
+// instant a turn turns out to need more than one step, so the user hears
+// something within about a second instead of dead air while the real
+// multi-step work runs. A short rotating set, not one fixed line, so it
+// doesn't read as a canned bot phrase on every multi-step question.
+const ACK_PHRASES = ["Let me check that for you.", "One moment, let me look into that.", "Give me a second to check.", "Let me take a look."];
 // Not constrained by any telephony 8kHz requirement — this is just "what
 // quality does Deepgram render at" for browser playback, and the Web Audio
 // API resamples an AudioBuffer at any declared rate transparently.
@@ -413,6 +419,14 @@ async function finalizeTurn(
   safeSend(client, { type: "final", text: transcript });
 
   let loopHistory = history;
+  // The Talker: set once, the first time a turn turns out to need more
+  // than one step (see the loop below) — a real, in-flight speakStreamed()
+  // call, never awaited until we're actually ready to speak the real
+  // answer. Deliberately not re-triggered per step: the Speak connection
+  // (speakStreamed) only ever handles one utterance at a time, so a second
+  // ack mid-loop would race the first one's own audio_chunk/Flushed
+  // handling instead of queuing cleanly.
+  let ackPromise: Promise<void> | null = null;
 
   try {
     for (let i = 0; i < MAX_LOOP_ITERATIONS; i++) {
@@ -435,9 +449,19 @@ async function finalizeTurn(
       safeSend(client, { type: "verb", verb });
 
       if (!TERMINAL_VERBS.has(verb.verb)) {
-        // A continuing step — no speech for it (keeps the loop fast;
-        // the client still shows it visually) — wait for its real result
-        // and go around again instead of ending the turn.
+        if (i === 0) {
+          // This turn just revealed it needs more than one step — speak a
+          // quick, cheap acknowledgment *now*, in parallel with the rest
+          // of the loop's own real work below (not awaited here), so the
+          // user hears something within about a second instead of dead
+          // air for however long the real multi-step answer takes.
+          // Single-step turns (the common case) never reach this branch
+          // at all, so they keep today's latency exactly as it is.
+          ackPromise = speakStreamed(ACK_PHRASES[Math.floor(Math.random() * ACK_PHRASES.length)]);
+        }
+        // A continuing step itself stays silent (keeps the loop fast; the
+        // client still shows it visually) — wait for its real result and
+        // go around again instead of ending the turn.
         const observation = await waitForToolResult();
         if (myGeneration !== getGeneration()) return;
         loopHistory = [
@@ -450,6 +474,17 @@ async function finalizeTurn(
       history.push({ role: "user", text: transcript }, { role: "assistant", text: summarizeVerbForHistory(verb) });
       history.splice(0, Math.max(0, history.length - MAX_HISTORY_TURNS));
 
+      if (ackPromise) {
+        // Never start a second speakStreamed call before the first (the
+        // ack) has actually finished — same single Speak connection, one
+        // utterance at a time. In the common multi-step case the real
+        // work below already took about as long as the ack itself did, so
+        // this rarely adds a real wait.
+        await ackPromise;
+        ackPromise = null;
+        if (myGeneration !== getGeneration()) return; // a barge-in could have landed during the ack itself
+      }
+
       // A verb with no spoken text (highlight/navigate/do often have none)
       // still needs to unstick the client's "thinking" state and let the mic
       // resume — turn_complete covers that with no audio path involved.
@@ -466,6 +501,10 @@ async function finalizeTurn(
     history.push({ role: "user", text: transcript }, { role: "assistant", text: "(gave up after too many steps)" });
     history.splice(0, Math.max(0, history.length - MAX_HISTORY_TURNS));
     safeSend(client, { type: "verb", verb: { verb: "explain", text: "I wasn't able to finish that — try asking again or breaking it into smaller steps." } });
+    if (ackPromise) {
+      await ackPromise;
+      if (myGeneration !== getGeneration()) return;
+    }
     await speakStreamed("I wasn't able to finish that — try asking again or breaking it into smaller steps.");
   } catch (err) {
     console.error("[cairn realtime] failed to resolve/speak this turn:", err);

package/src/server.ts

@@ -303,6 +303,28 @@ export class GroqVerbLLM implements VerbLLM {
   ) {}
 
   async respond(systemPrompt: string, userMessage: string): Promise<unknown> {
+    try {
+      return await this.attemptRespond(systemPrompt, userMessage);
+    } catch (err) {
+      // Real, live bug, not theoretical: openai/gpt-oss-120b (a reasoning-
+      // capable open model) occasionally "thinks out loud" in plain prose
+      // instead of emitting the forced tool call — Groq's own server-side
+      // validation rejects that outright, a 400 with code
+      // "output_parse_failed", before this code ever sees a real response
+      // to work with. Non-deterministic (found live re-asking the exact
+      // same question a moment later succeeded cleanly), so one retry —
+      // not exponential backoff, this is a latency-sensitive voice/chat
+      // path — genuinely helps rather than just delaying the same
+      // failure. Anything else still propagates to resolveVerb's own
+      // catch, unchanged.
+      if (isOutputParseFailure(err)) {
+        return await this.attemptRespond(systemPrompt, userMessage);
+      }
+      throw err;
+    }
+  }
+
+  private async attemptRespond(systemPrompt: string, userMessage: string): Promise<unknown> {
     const client = this.clientFactory(this.keys.take());
     const completion = await client.chat.completions.create({
       model: this.model,
@@ -333,6 +355,18 @@ export class GroqVerbLLM implements VerbLLM {
   }
 }
 
+/** Groq's SDK doesn't export a stable error shape to import and check
+ * against, so this checks defensively across the ways the real error has
+ * actually been observed to surface — a thrown APIError with a nested
+ * `.error.code`, a plain `.code`, or just the code string showing up
+ * somewhere in the message — rather than relying on exactly one of them. */
+function isOutputParseFailure(err: unknown): boolean {
+  if (!err || typeof err !== "object") return false;
+  const e = err as { code?: unknown; error?: { code?: unknown }; message?: unknown };
+  if (e.code === "output_parse_failed" || e.error?.code === "output_parse_failed") return true;
+  return typeof e.message === "string" && e.message.includes("output_parse_failed");
+}
+
 // ---------------------------------------------------------------------------
 // Shared tool schema / system prompt
 // ---------------------------------------------------------------------------

package/src/speak-server.ts

@@ -1,12 +1,36 @@
 // Server-side text-to-speech for the Copilot widget's spoken answers (see
 // `speakEndpoint` in index.tsx). The Deepgram key must never reach the
-// client, so this is a plain fetch to Deepgram's /v1/speak REST endpoint —
-// no SDK dependency needed for one request shape.
+// client.
+//
+// This used to be one fetch to Deepgram's /v1/speak REST endpoint, buffered
+// into an ArrayBuffer with `await response.arrayBuffer()` before returning
+// anything. That's the exact same bug the realtime path already fixed once
+// (see tts-stream.ts's own comment): nothing plays until Deepgram renders
+// AND the network delivers the *entire* reply, which measured 5-8s for a
+// normal explain answer in real production logs. It also hit Deepgram's
+// REST-only 2000-character cap on longer replies with no handling at all.
+//
+// Fixed the same way the realtime path was: open the streaming Speak
+// WebSocket (tts-stream.ts's DeepgramSpeakStream, the same class the
+// realtime server already uses) and forward audio chunks to the caller as
+// they arrive, via a ReadableStream — not buffered. The route handler
+// forwards that stream straight through as the HTTP response body, and the
+// client (index.tsx) reads it progressively instead of awaiting a full
+// Blob, so playback can start on the first chunk. Splitting the text into
+// sentence-sized `Speak` messages before one `Flush` sidesteps the old
+// 2000-char REST limit entirely (it doesn't apply to the WS protocol) and
+// lets Deepgram start rendering the first sentence sooner.
+import { DeepgramSpeakStream, type DeepgramSpeakStreamOptions, type SpeakChunkCallback } from "./tts-stream";
 
-const DEEPGRAM_SPEAK_URL = "https://api.deepgram.com/v1/speak";
-// Verified against Deepgram's docs while building this — re-check if this
-// starts erroring, voice model names retire over time.
 const DEEPGRAM_DEFAULT_VOICE = "aura-2-thalia-en";
+// Matches the realtime path's own playback sample rate (index.tsx's
+// audio_chunk handling defaults to 24000 too) — keeping them identical lets
+// both paths share one raw-PCM16 decode/schedule routine on the client.
+const SAMPLE_RATE = 24000;
+// Not a protocol limit (the WS Speak protocol has none like REST's 2000
+// chars) — just keeps each queued chunk sentence-sized so Deepgram can start
+// rendering the first one quickly instead of parsing one giant message.
+const MAX_CHUNK_CHARS = 300;
 
 export interface CreateSpeakHandlerOptions {
   apiKey: string;
@@ -15,13 +39,42 @@ export interface CreateSpeakHandlerOptions {
 
 export interface SpeakResult {
   status: number;
-  /** `audio` is raw MP3 bytes on success. */
-  body: { audio: ArrayBuffer; contentType: string } | { error: string };
+  /** `stream` yields raw linear16 PCM chunks (mono, 24kHz) as Deepgram
+   * renders them — forward it directly, unbuffered; do not await it into a
+   * Blob/ArrayBuffer or the whole point of streaming is lost. */
+  body: { stream: ReadableStream<Uint8Array>; contentType: string } | { error: string };
 }
 
 export type SpeakHandler = (text: string) => Promise<SpeakResult>;
 
-export function createSpeakHandler(options: CreateSpeakHandlerOptions): SpeakHandler {
+/** Test-only seam: lets tests inject a fake stream instead of opening a real
+ * Deepgram WebSocket. Not part of CreateSpeakHandlerOptions on purpose — real
+ * call sites (the scaffolded route templates) never pass this. */
+export type SpeakStreamFactory = (
+  opts: DeepgramSpeakStreamOptions,
+  onAudioChunk: SpeakChunkCallback,
+  handlers?: { onFlushed?: (sequenceId: number) => void; onError?: (err: Error) => void },
+) => DeepgramSpeakStream;
+
+function splitIntoChunks(text: string, maxChars: number): string[] {
+  const sentences = text.match(/[^.!?]+[.!?]*\s*/g) ?? [text];
+  const chunks: string[] = [];
+  let current = "";
+  for (const sentence of sentences) {
+    if (current && current.length + sentence.length > maxChars) {
+      chunks.push(current);
+      current = "";
+    }
+    current += sentence;
+  }
+  if (current) chunks.push(current);
+  return chunks;
+}
+
+export function createSpeakHandler(
+  options: CreateSpeakHandlerOptions,
+  streamFactory: SpeakStreamFactory = (opts, onAudioChunk, handlers) => new DeepgramSpeakStream(opts, onAudioChunk, handlers),
+): SpeakHandler {
   const model = options.model ?? process.env.DEEPGRAM_VOICE ?? DEEPGRAM_DEFAULT_VOICE;
 
   return async function handleSpeak(text: string) {
@@ -29,28 +82,51 @@ export function createSpeakHandler(options: CreateSpeakHandlerOptions): SpeakHan
       return { status: 400, body: { error: "no text provided" } };
     }
 
-    let response: Response;
-    try {
-      response = await fetch(`${DEEPGRAM_SPEAK_URL}?model=${encodeURIComponent(model)}`, {
-        method: "POST",
-        headers: {
-          Authorization: `Token ${options.apiKey}`,
-          "content-type": "application/json",
+    let enqueue: ((chunk: Uint8Array) => void) | null = null;
+    let closeOut: (() => void) | null = null;
+    let failOut: ((err: Error) => void) | null = null;
+    const stream = new ReadableStream<Uint8Array>({
+      start(controller) {
+        enqueue = (chunk) => controller.enqueue(chunk);
+        closeOut = () => controller.close();
+        failOut = (err) => controller.error(err);
+      },
+    });
+
+    // True once connect() below resolves — an onError before that point is
+    // already reported through connect()'s own rejection, so it's ignored
+    // here to avoid double-handling the same failure.
+    let connected = false;
+
+    const speakStream = streamFactory(
+      { apiKey: options.apiKey, model, encoding: "linear16", sampleRate: SAMPLE_RATE },
+      (chunk) => enqueue?.(new Uint8Array(chunk)),
+      {
+        onFlushed: () => {
+          closeOut?.();
+          speakStream.close();
+        },
+        onError: (err) => {
+          if (!connected) return;
+          console.error("[cairn] speak stream error:", err);
+          failOut?.(err);
         },
-        body: JSON.stringify({ text }),
-      });
+      },
+    );
+
+    try {
+      await speakStream.connect();
     } catch (err) {
       console.error("[cairn] speak request failed:", err);
       return { status: 200, body: { error: "speech service unreachable" } };
     }
+    connected = true;
 
-    if (!response.ok) {
-      const detail = await response.text().catch(() => "");
-      console.error("[cairn] Deepgram speak returned an error:", response.status, detail);
-      return { status: 200, body: { error: "speech synthesis failed" } };
+    for (const chunk of splitIntoChunks(text, MAX_CHUNK_CHARS)) {
+      speakStream.sendText(chunk);
     }
+    speakStream.flush();
 
-    const audio = await response.arrayBuffer();
-    return { status: 200, body: { audio, contentType: response.headers.get("content-type") ?? "audio/mpeg" } };
+    return { status: 200, body: { stream, contentType: `audio/L16;rate=${SAMPLE_RATE}` } };
   };
 }