@inetafrica/open-claudia 3.4.13 → 3.4.15

package/channels/kazee/call/session.js

@@ -4,7 +4,8 @@ const { MediaStreamTrack, RTCPeerConnection, RtpHeader, RtpPacket } = require("w
 const { JanusClient } = require("./janus-client");
 const { projectAssignedIce, supportedIceServers } = require("./access");
 const { AdaptiveTurnDetector, OpusAudioCodec, WEBRTC_SAMPLES, downsample48To16, upsampleTo48 } = require("./audio");
-const { SpeechStream, synthesizePcm, synthesizeStream: synthesizeStreamPcm } = require("./speech-client");
+const { SpeechStream, synthesizePcm, synthesizeStream: synthesizeStreamPcm, detectTurn, selectAcknowledgement, CUE_PHRASES } = require("./speech-client");
+const { SpeechTurnGate } = require("./turn-gate");
 const ttsVoice = require("../../../core/tts-voice");
 
 const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
@@ -42,40 +43,18 @@ const WORD_RESCUE_MIN_MS = 1500;
 const WORD_RESCUE_NO_CADENCE_MS = 4000;
 const WORD_WAIT_MAX_MS = 4000;
 const FINAL_WAIT_MS = 2500;
+const SEGMENT_FINAL_WAIT_MS = 12000;
 const MIN_DISPATCH_WORDS = 2;
 const MAX_DURATION_GOODBYE = "We've hit the call time limit, so I'll hang up now. Message me any time.";
 const IDLE_GOODBYE = "I haven't heard anything for a while, so I'll hang up now. Message me any time.";
 
-// Audible state cues: a short ack chime when the caller finishes speaking, then
-// a soft periodic tick while the turn is being worked on — dead air otherwise
-// reads as a dropped call.
-const THINK_TICK_INTERVAL_MS = 1600;
-const THINKING_MAX_MS = 5 * 60_000;
-
-function tonePcm48(segments, gain = 0.08) {
-  const rate = 48000;
-  const fade = Math.floor(rate * 0.012);
-  const buffers = segments.map(({ freq, ms }) => {
-    const samples = Math.floor((rate * ms) / 1000);
-    const buf = Buffer.alloc(samples * 2);
-    for (let i = 0; i < samples; i += 1) {
-      let a = gain;
-      if (i < fade) a *= i / fade;
-      if (i > samples - fade) a *= Math.max(0, (samples - i) / fade);
-      buf.writeInt16LE(Math.round(Math.sin((2 * Math.PI * freq * i) / rate) * a * 32767), i * 2);
-    }
-    return buf;
-  });
-  return Buffer.concat(buffers);
-}
-
-// Rising two-tone "heard you", and a quieter single "still here" pulse.
-const ACK_CHIME_PCM48 = tonePcm48([{ freq: 660, ms: 90 }, { freq: 990, ms: 110 }], 0.09);
-const THINK_TICK_PCM48 = tonePcm48([{ freq: 520, ms: 90 }], 0.045);
+// One optional spoken cue after a delay, never periodic beeps.
+const ACK_DELAY_MS = 2800;
 
 class KazeeCallSession {
   constructor({ roomId, chatId, callerId, botUserId, janusUrl, janusToken, janusTokenExpiresAt, roomToken, roomTokenExpiresAt, managedRoom = false, refreshRoomAccess, iceServers = [], iceCredentialsExpiresAt, onTranscript, onEnded, onMediaReady,
     maxCallMs, idleMs, finalWaitMs, synthesize = synthesizePcm, synthesizeStream = null, asCaller = false, greeting = null,
+    turnDecision = detectTurn, acknowledgement = selectAcknowledgement, onSpeechStart,
     JanusClientImpl = JanusClient, PeerConnectionImpl = RTCPeerConnection, SpeechStreamImpl = SpeechStream } = {}) {
     this.roomId = Number(roomId);
     this.chatId = String(chatId);
@@ -103,6 +82,11 @@ class KazeeCallSession {
     this._transportIceExpiry = new Map();
     this._transportExpiryTimer = null;
     this.onTranscript = onTranscript || (() => {});
+    this.onSpeechStart = onSpeechStart || (() => {});
+    this.turnDecision = turnDecision;
+    this.acknowledgement = acknowledgement;
+    this._cueController = null;
+    this._cueRevision = 0;
     this.onEnded = onEnded || (() => {});
     this.onMediaReady = onMediaReady || (() => {});
     this._sentRtp = false;
@@ -256,8 +240,8 @@ class KazeeCallSession {
     }
     for (const publisher of data.publishers || []) {
       const feedId = Number(publisher.id);
-      if (Number.isSafeInteger(feedId) && feedId > 0 && feedId !== this.publisherId && !this.subscribers.has(feedId) && !String(publisher.display || "").startsWith("open-claudia:")) {
-        await this._subscribe(feedId, publisher.display);
+      if (Number.isSafeInteger(feedId) && feedId > 0 && feedId !== this.publisherId && !this.subscribers.has(feedId)) {
+        await this._subscribe(feedId, publisher.display, publisher.metadata);
       }
     }
     if (data.videoroom === "joined") {
@@ -357,8 +341,21 @@ class KazeeCallSession {
     this.speak(this.greeting).catch((error) => console.error("Kazee call greeting:", error.message));
   }
 
-  async _subscribe(feedId, display) {
-    if (this.subscribers.has(feedId)) return;
+  async _subscribe(feedId, display, metadata) {
+    if (this.closed || feedId === this.publisherId || this.subscribers.has(feedId)) return;
+    if (this.managedRoom) {
+      // Only the enforced gateway owns this metadata. Display remains an email
+      // for presentation, and cannot tell us whether a publisher is another bot.
+      // Missing or malformed classification must not start STT on an unknown feed.
+      const principal = metadata?.kazee;
+      if (!principal || Array.isArray(principal) || principal.version !== 1 ||
+        typeof principal.userId !== "string" || !/^[a-f\d]{24}$/i.test(principal.userId) ||
+        !["User", "Bot"].includes(principal.principalKind) || principal.principalKind === "Bot" ||
+        principal.userId.toLowerCase() === this.botUserId.toLowerCase()) return;
+    } else if (String(display || "").startsWith("open-claudia:")) {
+      // Legacy/direct rooms allow client metadata, so it is not a trusted source.
+      return;
+    }
     // Reserve the slot before the first await: the join response and the async
     // publishers event both list the same feed, and while attach() is in
     // flight the second frame passes the caller's has() check and subscribes
@@ -379,8 +376,9 @@ class KazeeCallSession {
         codec: new OpusAudioCodec(),
         vad: new AdaptiveTurnDetector(),
         stt: new this.SpeechStreamImpl({
-          onTranscript: (text) => this._onFeedFinal(feedId, text),
-          onPartial: (text) => this._onFeedPartial(feedId, text),
+          onTranscript: (text, meta) => this._onFeedFinal(feedId, text, meta),
+          onPartial: (text, meta) => this._onFeedPartial(feedId, text, meta),
+          onDisconnect: () => this._resetRecognition(feedId),
         }),
         prebuffer: [],
         pending: [],
@@ -394,7 +392,9 @@ class KazeeCallSession {
         noiseCommit: false,
         fallbackUsed: false,
         finalTimer: null,
+        segments: new Map(),
       };
+      sub.turnGate = new SpeechTurnGate({ decide: this.turnDecision, commit: reason => this._commitTurn(feedId, reason) });
       this.subscribers.set(feedId, sub);
       pc.onTrack.subscribe((track) => {
         if (track.kind === "audio") track.onReceiveRtp.subscribe((packet) => this._onFeedRtp(feedId, packet));
@@ -439,9 +439,26 @@ class KazeeCallSession {
     }
   }
 
-  _onFeedPartial(feedId, text) {
+  _resetRecognition(feedId) {
+    const sub = this.subscribers.get(feedId);
+    if (!sub) return;
+    for (const segment of sub.segments.values()) clearTimeout(segment.timer);
+    sub.segments.clear();
+    if (sub.finalTimer) clearTimeout(sub.finalTimer);
+    sub.finalTimer = null;
+    sub.lastPartial = sub.partialAtCommit = "";
+    sub.wordCount = sub.lastWordAt = sub.awaitingWordsSince = 0;
+    sub.wordGaps = [];
+    sub.noiseCommit = sub.fallbackUsed = false;
+    sub.vad.reset();
+    sub.turnGate?.reset();
+    this._stopThinking();
+  }
+
+  _onFeedPartial(feedId, text, meta = {}) {
     const sub = this.subscribers.get(feedId);
     if (!sub || this.closed) return;
+    if (meta.protocol === 2 && (meta.streamId !== sub.stt.streamId || meta.segmentId !== sub.stt.segmentId)) return;
     const partial = String(text || "");
     const prevWords = countWords(sub.lastPartial);
     const words = countWords(partial);
@@ -456,7 +473,7 @@ class KazeeCallSession {
     }
     sub.wordCount += words - prevWords;
     sub.lastWordAt = now;
-    if (sub.awaitingWordsSince) this._commitTurn(feedId, "words-arrived");
+    if (sub.awaitingWordsSince && sub.stt.protocol !== 2) this._commitTurn(feedId, "words-arrived");
   }
 
   // Interims arrive in bursts whose cadence tracks recogniser health (jig
@@ -472,23 +489,49 @@ class KazeeCallSession {
     const sub = this.subscribers.get(feedId);
     if (!sub || this.closed) return;
     sub.awaitingWordsSince = 0;
+    sub.turnGate?.reset();
     if (sub.vad.active) sub.vad.reset();
     this._lastHeard = Date.now();
     clog(`Kazee call turn commit: feed=${feedId} reason=${reason} words=${sub.wordCount}`);
-    sub.stt.commit().catch((error) => console.error("Kazee call STT commit:", error.message));
-    if (!sub.noiseCommit) {
-      this._playCue(ACK_CHIME_PCM48).catch(() => {});
-      // STT finalization takes seconds; start ticking at commit so the
-      // chime→first-tick stretch doesn't read as a dead line.
-      this._startThinking();
-    }
-    // Finals routinely never arrive; after finalWaitMs the interim transcript
-    // is dispatched in their place.
+    const versioned = sub.stt.protocol === 2 && typeof sub.stt.commitTurn === "function";
+    if (versioned) {
+      if (sub.segments.size >= 4) {
+        this.close("speech-backlog").catch(() => {});
+        return;
+      }
+      const commit = sub.stt.commitTurn();
+      const key = `${commit.streamId}:${commit.segmentId}`;
+      const segment = { noiseCommit: sub.noiseCommit, committedAt: Date.now(), timer: null };
+      sub.segments.set(key, segment);
+      segment.timer = setTimeout(() => {
+        if (!sub.segments.delete(key)) return;
+        clog(`Kazee call final timeout: feed=${feedId} segment=${key}`);
+        this._stopThinking();
+        // Never substitute an old or unfinished transcript. Ask for a repeat
+        // only when the caller is quiet, so recovery cannot interrupt them.
+        if (!this.closed && !segment.noiseCommit && !this._speakActive &&
+            ![...this.subscribers.values()].some(peer => peer?.vad?.active)) {
+          this.speak("I missed that. Could you repeat it?").catch(() => {});
+        }
+      }, SEGMENT_FINAL_WAIT_MS);
+      segment.timer.unref?.();
+      commit.pending.catch(() => {
+        clearTimeout(segment.timer);
+        sub.segments.delete(key);
+        clog(`Kazee call STT commit failed: feed=${feedId} segment=${key}`);
+      });
+    } else {
+      sub.stt.commit().catch((error) => console.error("Kazee call STT commit:", error.message));
+    }
+    // Recognition must finish before any acknowledgement claims a heard turn.
+    // The legacy fallback is retained only for older speech deployments.
     sub.partialAtCommit = sub.lastPartial;
+    sub.lastPartial = "";
     sub.fallbackUsed = false;
     if (sub.finalTimer) clearTimeout(sub.finalTimer);
-    sub.finalTimer = setTimeout(() => this._finalFallback(feedId), this.finalWaitMs);
-    if (typeof sub.finalTimer.unref === "function") sub.finalTimer.unref();
+    sub.finalTimer = versioned ? null : setTimeout(() => this._finalFallback(feedId), this.finalWaitMs);
+    sub.finalTimer?.unref?.();
+    if (versioned) sub.noiseCommit = false;
     sub.wordCount = 0;
     sub.lastWordAt = 0;
     sub.wordGaps = [];
@@ -498,19 +541,28 @@ class KazeeCallSession {
     const sub = this.subscribers.get(feedId);
     if (!sub || this.closed) return;
     sub.finalTimer = null;
-    // Partials keep growing after commit while the decoder catches up; accept
-    // the grown text only if it extends the committed snapshot (anything else
-    // belongs to the next turn).
-    const current = sub.lastPartial || "";
+    // Legacy services cannot label late partials. Use only the committed
+    // snapshot; newer text could belong to a different utterance.
     const snapshot = sub.partialAtCommit || "";
-    const text = current.length > snapshot.length && current.startsWith(snapshot) ? current : snapshot;
+    const text = snapshot;
     sub.fallbackUsed = true;
     clog(`Kazee call final fallback: feed=${feedId} chars=${text.trim().length}`);
     this._dispatchTranscript(feedId, text);
   }
 
-  _onFeedFinal(feedId, text) {
+  _onFeedFinal(feedId, text, meta = {}) {
     const sub = this.subscribers.get(feedId);
+    if (!sub || this.closed) return;
+    if (meta.protocol === 2) {
+      const key = `${meta.streamId}:${meta.segmentId}`;
+      const segment = sub.segments.get(key);
+      if (!segment || meta.streamId !== sub.stt.streamId) return;
+      sub.segments.delete(key);
+      clearTimeout(segment.timer);
+      clog(`Kazee call final ready: feed=${feedId} segment=${key} wait=${Date.now() - segment.committedAt}ms`);
+      this._dispatchTranscript(feedId, text, { noiseCommit: segment.noiseCommit, utteranceId: key });
+      return;
+    }
     if (sub?.finalTimer) {
       clearTimeout(sub.finalTimer);
       sub.finalTimer = null;
@@ -523,11 +575,11 @@ class KazeeCallSession {
     this._dispatchTranscript(feedId, text);
   }
 
-  _dispatchTranscript(feedId, text) {
+  _dispatchTranscript(feedId, text, options = {}) {
     const sub = this.subscribers.get(feedId);
     const trimmed = String(text || "").trim();
-    if (sub?.noiseCommit) {
-      sub.noiseCommit = false;
+    if (options.noiseCommit ?? sub?.noiseCommit) {
+      if (options.noiseCommit === undefined) sub.noiseCommit = false;
       // A turn the recogniser never put words in was background noise; babble
       // occasionally leaks a single stray word into the transcript too.
       if (countWords(trimmed) < MIN_DISPATCH_WORDS) {
@@ -537,14 +589,16 @@ class KazeeCallSession {
       }
     }
     clog(`Kazee call transcript: feed=${feedId} chars=${trimmed.length}${trimmed.length ? "" : " (noise commit)"}`);
-    if (trimmed) this._startThinking();
+    if (trimmed) this._startThinking(trimmed);
     // Empty finalization (noise commit): nothing dispatches, no reply is
     // coming — don't tick into the void.
     else if (!this._speakActive && !this._speakQueue.length) this._stopThinking();
+    if (!trimmed) return;
     this.onTranscript(text, {
       feedId,
       display: sub?.display || "",
       participants: this.subscribers.size,
+      ...(options.utteranceId ? { utteranceId: options.utteranceId } : {}),
     });
   }
 
@@ -567,32 +621,34 @@ class KazeeCallSession {
     this._reportMediaReady();
   }
 
-  // Writes a short cue into the outbound track. Yields to real speech: the
-  // moment an utterance is active (or starting), remaining frames are dropped.
-  async _playCue(pcm48) {
-    if (this.closed || this.speaking || this._speakActive || !this.publisher) return;
-    for (let offset = 0; offset < pcm48.length; offset += WEBRTC_SAMPLES * 2) {
-      if (this.closed || this.speaking || this._speakActive) return;
-      this._writeFrame(pcm48.subarray(offset, offset + WEBRTC_SAMPLES * 2));
-      await wait(20);
-    }
-  }
-
-  _startThinking() {
-    if (this.closed || this._thinkTimer) return;
-    this._thinkingSince = Date.now();
-    this._thinkTimer = setInterval(() => {
-      if (this.closed || Date.now() - this._thinkingSince > THINKING_MAX_MS) return this._stopThinking();
-      if (this.speaking || this._speakActive) return;
-      this._playCue(THINK_TICK_PCM48).catch(() => {});
-    }, THINK_TICK_INTERVAL_MS);
-    if (typeof this._thinkTimer.unref === "function") this._thinkTimer.unref();
+  _startThinking(text = "", state = "waiting") {
+    this._stopThinking();
+    if (this.closed || !text) return;
+    const revision = this._cueRevision;
+    this._thinkTimer = setTimeout(async () => {
+      this._thinkTimer = null;
+      if (this.closed || this.speaking || this._speakActive ||
+          [...this.subscribers.values()].some(sub => sub?.vad?.active || sub?.turnGate?.pending)) return;
+      const controller = new AbortController();
+      this._cueController = controller;
+      try {
+        const cue = await this.acknowledgement(text, state, { signal: controller.signal });
+        if (cue && revision === this._cueRevision && !this.closed && !this._speakActive &&
+            ![...this.subscribers.values()].some(sub => sub?.vad?.active || sub?.turnGate?.pending)) {
+          await this.speak(cue);
+        }
+      } catch (_) { /* A missing cue never delays an answer. */ }
+      finally { if (this._cueController === controller) this._cueController = null; }
+    }, ACK_DELAY_MS);
+    this._thinkTimer.unref?.();
   }
 
   _stopThinking() {
-    if (!this._thinkTimer) return;
-    clearInterval(this._thinkTimer);
+    this._cueRevision += 1;
+    if (this._thinkTimer) clearTimeout(this._thinkTimer);
     this._thinkTimer = null;
+    this._cueController?.abort();
+    this._cueController = null;
   }
 
   _onFeedRtp(feedId, packet) {
@@ -613,6 +669,28 @@ class KazeeCallSession {
     sub.vad.boostTo = this.speaking ? SPEAKING_THRESHOLD_DB : null;
     const event = sub.vad.push(pcm);
     const pcm16 = downsample48To16(pcm);
+    if (sub.stt.protocol === 2 && sub.turnGate) {
+      if (event?.type === "start") {
+        sub.turnGate.resume();
+        this._lastHeard = Date.now();
+        this.cancelSpeech();
+        this._stopThinking();
+        this.onSpeechStart({ feedId });
+        for (const frame of sub.prebuffer.splice(0)) {
+          sub.turnGate.audio(frame);
+          this._feedAudio(sub, frame);
+        }
+      }
+      if (sub.vad.active || event || sub.turnGate.pending) {
+        sub.turnGate.audio(pcm16);
+        this._feedAudio(sub, pcm16);
+        if (event?.type === "commit") sub.turnGate.candidate(event.reason);
+      } else {
+        sub.prebuffer.push(pcm16);
+        if (sub.prebuffer.length > PREBUFFER_FRAMES) sub.prebuffer.shift();
+      }
+      return;
+    }
     if (sub.awaitingWordsSince) {
       // Energy closed the turn but the recogniser hasn't produced a word yet:
       // hold the commit and keep streaming audio so a slow decode still lands.
@@ -621,7 +699,8 @@ class KazeeCallSession {
         // Caller resumed before any words arrived — same turn continues.
         sub.awaitingWordsSince = 0;
         this._lastHeard = Date.now();
-        if (this.speaking) this.cancelSpeech();
+        this.cancelSpeech();
+        this.onSpeechStart({ feedId });
         this._stopThinking();
         return;
       }
@@ -640,7 +719,8 @@ class KazeeCallSession {
       this._lastHeard = Date.now();
       clog(`Kazee call VAD start: feed=${feedId}`);
       // Barge-in: the human wins — stop talking and listen.
-      if (this.speaking) this.cancelSpeech();
+      this.cancelSpeech();
+      this.onSpeechStart({ feedId });
       this._stopThinking();
       for (const frame of sub.prebuffer.splice(0)) this._feedAudio(sub, frame);
     }
@@ -681,6 +761,7 @@ class KazeeCallSession {
 
   cancelSpeech() {
     this._speakGen += 1;
+    this._synthController?.abort();
     for (const item of this._speakQueue.splice(0)) item.resolve(false);
   }
 
@@ -691,8 +772,11 @@ class KazeeCallSession {
       try {
         while (this._speakQueue.length && !this.closed) {
           const item = this._speakQueue.shift();
+          const controller = new AbortController();
+          this._synthController = controller;
           try { item.resolve(await this._playUtterance(item.text, this._speakGen)); }
-          catch (error) { item.reject(error); }
+          catch (error) { if (controller.signal.aborted) item.resolve(false); else item.reject(error); }
+          finally { if (this._synthController === controller) this._synthController = null; }
         }
       } finally {
         this._speakActive = false;
@@ -724,7 +808,7 @@ class KazeeCallSession {
     // Cache only audio rendered by the real service: the key is voice+text, so
     // PCM from an injected synthesize would leak across sessions.
     const usingService = this.synthesizeStream !== null || this.synthesize === synthesizePcm;
-    const cacheable = usingService && (text === this.greeting || text === MAX_DURATION_GOODBYE || text === IDLE_GOODBYE);
+    const cacheable = usingService && (text === this.greeting || text === MAX_DURATION_GOODBYE || text === IDLE_GOODBYE || Object.values(CUE_PHRASES).includes(text));
     const cacheKey = `${ttsVoice.getVoice()}::${text}`;
     const synthStart = Date.now();
     if (cacheable && SYNTH_CACHE.has(cacheKey)) {
@@ -739,7 +823,7 @@ class KazeeCallSession {
         clog(`Kazee call stream synth failed (${error.message}); buffering`);
       }
     }
-    const synthesized = await this.synthesize(text);
+    const synthesized = await this.synthesize(text, { signal: this._synthController?.signal });
     if (this.closed || gen !== this._speakGen) return false;
     const pcm48 = upsampleTo48(synthesized.pcm, synthesized.sampleRate);
     clog(`Kazee call speak: synth=${Date.now() - synthStart}ms chars=${text.length}`);
@@ -750,7 +834,7 @@ class KazeeCallSession {
   // Plays TTS as it downloads: the service streams PCM, but buffering the full
   // response cost ~3s of silence before the first word on every reply.
   async _playStreaming(text, gen, { cacheable, cacheKey, synthStart }) {
-    const { sampleRate, chunks } = await this.synthesizeStream(text);
+    const { sampleRate, chunks } = await this.synthesizeStream(text, { signal: this._synthController?.signal });
     if (this.closed || gen !== this._speakGen) return false;
     let rawCarry = Buffer.alloc(0); // an odd byte straddling a chunk boundary splits a sample
     let frameCarry = Buffer.alloc(0);
@@ -831,6 +915,9 @@ class KazeeCallSession {
     this._transportIceExpiry.delete(sub.pc);
     this._armTransportExpiry();
     if (sub.finalTimer) clearTimeout(sub.finalTimer);
+    for (const segment of sub.segments.values()) clearTimeout(segment.timer);
+    sub.segments.clear();
+    sub.turnGate?.close();
     try { sub.stt.close(); } catch (_) {}
     try { sub.codec.close(); } catch (_) {}
     try { await sub.pc.close(); } catch (_) {}

package/channels/kazee/call/speech-client.js

@@ -6,58 +6,104 @@ const { speechBearerToken } = require("../../../core/media");
 const ttsVoice = require("../../../core/tts-voice");
 
 class SpeechStream {
-  constructor({ baseUrl = SPEECH_SERVICE_URL, token = speechBearerToken(), WebSocketImpl = WebSocket, onTranscript, onPartial } = {}) {
+  constructor({ baseUrl = SPEECH_SERVICE_URL, token = speechBearerToken(), WebSocketImpl = WebSocket, onTranscript, onPartial, onDisconnect } = {}) {
     this.baseUrl = String(baseUrl || "").replace(/\/$/, "");
     this.token = token;
     this.WebSocketImpl = WebSocketImpl;
     this.onTranscript = onTranscript || (() => {});
     this.onPartial = onPartial || (() => {});
+    this.onDisconnect = onDisconnect || (() => {});
     this.socket = null;
     this.ready = null;
+    this.protocol = 1;
+    this._protocolCommitted = false;
+    this.segmentId = 1;
+    this.streamId = 0;
+    this.closed = false;
   }
 
   async connect() {
+    if (this.closed) throw new Error("Speech stream is closed");
     if (this.ready) return this.ready;
     if (!this.baseUrl || !this.token) throw new Error("Hosted speech service is not configured");
     const url = this.baseUrl.replace(/^http/, "ws") + "/v1/audio/transcriptions/stream";
     this.ready = new Promise((resolve, reject) => {
       const ws = new this.WebSocketImpl(url, { headers: { Authorization: `Bearer ${this.token}` } });
       this.socket = ws;
+      this.segmentId = 1;
+      this.protocol = 1;
+      this._protocolCommitted = false;
+      const streamId = ++this.streamId;
       ws.once("open", () => {
-        ws.send(JSON.stringify({ sample_rate: 16000, language: "en" }));
+        if (this.closed || this.socket !== ws) return reject(new Error("Speech stream is closed"));
+        ws.send(JSON.stringify({ sample_rate: 16000, language: "en", protocol: 2 }));
         resolve();
       });
       ws.once("error", reject);
-      ws.on("message", (raw) => this._onMessage(raw));
-      ws.on("close", () => { this.socket = null; this.ready = null; });
+      ws.on("message", (raw) => { if (this.socket === ws && !this.closed) this._onMessage(raw, streamId); });
+      ws.on("close", () => {
+        reject(new Error("Speech stream disconnected"));
+        if (this.socket !== ws) return;
+        this.socket = null;
+        this.ready = null;
+        this.protocol = 1;
+        this._protocolCommitted = false;
+        this.onDisconnect({ streamId });
+      });
     });
     return this.ready;
   }
 
   async audio(pcm16k) {
-    await this.connect();
-    if (this.socket?.readyState === this.WebSocketImpl.OPEN) this.socket.send(pcm16k);
+    const pending = this.connect();
+    const socket = this.socket;
+    await pending;
+    if (this.closed || this.socket !== socket) return;
+    if (socket?.readyState === this.WebSocketImpl.OPEN) socket.send(pcm16k);
   }
 
-  async commit() {
-    await this.connect();
-    this.socket.send(JSON.stringify({ type: "commit" }));
+  commitTurn() {
+    const connected = this.connect();
+    const streamId = this.streamId;
+    const segmentId = this.segmentId++;
+    // The session has already selected bookkeeping for this protocol. Pin it
+    // for this socket before any asynchronous send or delayed ready frame.
+    // V2 servers accept V1 commits too, so an early turn can safely retain
+    // legacy behavior until reconnect; it cannot switch final-routing mid-turn.
+    this._protocolCommitted = true;
+    const protocol = this.protocol;
+    const pending = connected.then(() => {
+      if (this.closed || streamId !== this.streamId || !this.socket) throw new Error("Speech stream disconnected before commit");
+      this.socket.send(JSON.stringify({ type: "commit", ...(protocol === 2 ? { segment_id: segmentId } : {}) }));
+    });
+    return { streamId, segmentId, pending };
   }
 
-  _onMessage(raw) {
+  commit() { return this.commitTurn().pending; }
+
+  _onMessage(raw, streamId = this.streamId) {
     let message;
     try { message = JSON.parse(raw.toString()); } catch (_) { return; }
+    if (!message || typeof message !== "object" || Array.isArray(message)) return;
+    if (message.type === "ready" && message.protocol === 2 && message.segment_id === 1) {
+      if (!this._protocolCommitted) this.protocol = 2;
+      return;
+    }
+    const meta = { streamId, segmentId: message.segment_id, protocol: this.protocol };
+    if (this.protocol === 2 && (!Number.isSafeInteger(meta.segmentId) || meta.segmentId < 1)) return;
     const text = String(message.text || message.transcript || "").trim();
-    if (!text) return;
     if (message.final === true || message.type === "final" || message.type === "transcription") {
-      Promise.resolve(this.onTranscript(text)).catch((error) => console.error("Kazee call transcript:", error.message));
-    } else {
+      // Empty finals are meaningful: they close that segment without reusing
+      // an earlier partial. The session owns exactly-once dispatch.
+      Promise.resolve(this.onTranscript(text, meta)).catch((error) => console.error("Kazee call transcript:", error.message));
+    } else if (text && message.type === "partial") {
       // Interims drive word-based turn-end detection; they were previously discarded.
-      Promise.resolve(this.onPartial(text)).catch((error) => console.error("Kazee call partial:", error.message));
+      Promise.resolve(this.onPartial(text, meta)).catch((error) => console.error("Kazee call partial:", error.message));
     }
   }
 
   close() {
+    this.closed = true;
     const ws = this.socket;
     this.socket = null;
     this.ready = null;
@@ -66,13 +112,13 @@ class SpeechStream {
   }
 }
 
-async function synthesizePcm(text, { baseUrl = SPEECH_SERVICE_URL, token = speechBearerToken(), fetchImpl = fetch } = {}) {
+async function synthesizePcm(text, { baseUrl = SPEECH_SERVICE_URL, token = speechBearerToken(), fetchImpl = fetch, signal } = {}) {
   if (!baseUrl || !token) throw new Error("Hosted speech service is not configured");
   const response = await fetchImpl(`${String(baseUrl).replace(/\/$/, "")}/v1/audio/speech`, {
     method: "POST",
     headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json" },
     body: JSON.stringify({ input: text, voice: ttsVoice.getVoice(), response_format: "pcm" }),
-    signal: AbortSignal.timeout(60000),
+    signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(60000)]) : AbortSignal.timeout(60000),
   });
   if (!response.ok) throw new Error(`Hosted TTS failed: HTTP ${response.status}`);
   const contentType = response.headers?.get?.("content-type") || "";
@@ -83,13 +129,13 @@ async function synthesizePcm(text, { baseUrl = SPEECH_SERVICE_URL, token = speec
 
 // Same endpoint as synthesizePcm, but yields PCM chunks as they arrive so
 // playback can start on the first byte instead of after the full download.
-async function synthesizeStream(text, { baseUrl = SPEECH_SERVICE_URL, token = speechBearerToken(), fetchImpl = fetch } = {}) {
+async function synthesizeStream(text, { baseUrl = SPEECH_SERVICE_URL, token = speechBearerToken(), fetchImpl = fetch, signal } = {}) {
   if (!baseUrl || !token) throw new Error("Hosted speech service is not configured");
   const response = await fetchImpl(`${String(baseUrl).replace(/\/$/, "")}/v1/audio/speech`, {
     method: "POST",
     headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json" },
     body: JSON.stringify({ input: text, voice: ttsVoice.getVoice(), response_format: "pcm" }),
-    signal: AbortSignal.timeout(90000),
+    signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(90000)]) : AbortSignal.timeout(90000),
   });
   if (!response.ok) throw new Error(`Hosted TTS failed: HTTP ${response.status}`);
   const contentType = response.headers?.get?.("content-type") || "";
@@ -111,4 +157,35 @@ async function synthesizeStream(text, { baseUrl = SPEECH_SERVICE_URL, token = sp
   return { sampleRate: rate, chunks: chunks() };
 }
 
-module.exports = { SpeechStream, synthesizePcm, synthesizeStream };
+const CUE_PHRASES = Object.freeze({ none: "", acknowledge: "Mhm.", wait: "One moment.", working: "I'm checking that." });
+
+async function controllerRequest(path, body, { baseUrl = SPEECH_SERVICE_URL, token = speechBearerToken(), fetchImpl = fetch, signal, binary = false } = {}) {
+  if (!baseUrl || !token) throw new Error("Hosted speech controller is not configured");
+  const response = await fetchImpl(`${String(baseUrl).replace(/\/$/, "")}/v1/audio/${path}`, {
+    method: "POST",
+    headers: { Authorization: `Bearer ${token}`, "Content-Type": binary ? "application/octet-stream" : "application/json" },
+    body: binary ? body : JSON.stringify(body),
+    signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(1800)]) : AbortSignal.timeout(1800),
+  });
+  if (!response.ok) throw new Error("Hosted speech controller unavailable");
+  const result = await response.json();
+  if (!result || typeof result !== "object" || Array.isArray(result)) throw new Error("Invalid controller response");
+  return result;
+}
+
+async function detectTurn(pcm16, options = {}) {
+  const result = await controllerRequest("turn", pcm16, { ...options, binary: true });
+  if (typeof result.complete !== "boolean" || typeof result.probability !== "number" ||
+      !Number.isFinite(result.probability) || result.probability < 0 || result.probability > 1) throw new Error("Invalid turn decision");
+  return { complete: result.complete, probability: result.probability };
+}
+
+async function selectAcknowledgement(text, state = "waiting", options = {}) {
+  const allowed = { listening: ["none", "acknowledge"], waiting: ["none", "wait"], working: ["none", "working"] }[state];
+  if (!allowed) return "";
+  const result = await controllerRequest("acknowledgement", { text: String(text).slice(-1000), state }, options);
+  // Never speak remote free text. A cue is selected from our fixed dictionary.
+  return allowed.includes(result.choice) ? CUE_PHRASES[result.choice] : "";
+}
+
+module.exports = { SpeechStream, synthesizePcm, synthesizeStream, detectTurn, selectAcknowledgement, CUE_PHRASES };