@inetafrica/open-claudia 3.2.1 → 3.3.1

package/channels/kazee/call/session.js

@@ -2,10 +2,17 @@
 
 const { MediaStreamTrack, RTCPeerConnection, RtpHeader, RtpPacket } = require("werift");
 const { JanusClient } = require("./janus-client");
-const { OpusAudioCodec, TurnDetector, WEBRTC_SAMPLES, downsample48To16, upsampleTo48 } = require("./audio");
-const { SpeechStream, synthesizePcm } = require("./speech-client");
+const { AdaptiveTurnDetector, OpusAudioCodec, WEBRTC_SAMPLES, downsample48To16, upsampleTo48 } = require("./audio");
+const { SpeechStream, synthesizePcm, synthesizeStream: synthesizeStreamPcm } = require("./speech-client");
+const ttsVoice = require("../../../core/tts-voice");
 
 const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+const countWords = (text) => String(text || "").trim().split(/\s+/).filter(Boolean).length;
+
+// Fixed lines (greeting, goodbyes) re-synthesize identically on every call;
+// keep the rendered PCM per voice. Only those constants are cached, so the map
+// stays tiny.
+const SYNTH_CACHE = new Map();
 
 // Per-stage call logging. The media leg (ICE/RTP) can fail with zero thrown
 // errors — a caller then hears silence while the log shows a clean
@@ -13,7 +20,6 @@ const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 // double as the STT/TTS timing measurements chat-side estimates lacked.
 const clog = (msg) => console.log(`${new Date().toISOString()} ${msg}`);
 
-const BASE_THRESHOLD_DB = -38;
 // While TTS plays, the caller's device can leak bot audio back through its mic.
 // Require a much louder signal before treating incoming audio as human speech.
 const SPEAKING_THRESHOLD_DB = -26;
@@ -23,6 +29,19 @@ const DEFAULT_MAX_CALL_MS = 30 * 60_000;
 const DEFAULT_IDLE_MS = 2 * 60_000;
 const DEFAULT_JOIN_RETRIES = 10;
 const DEFAULT_JOIN_RETRY_DELAY_MS = 500;
+// Inbound defer-publish window: how long to wait for the caller's own
+// publisher before giving up and publishing anyway (see _maybePublish).
+const PUBLISH_DEFER_MS = 4000;
+// Hybrid turn-end (jig 2026-08-16): energy alone never closes a turn below
+// ~15dB SNR of café noise, STT finals routinely never arrive at all, and
+// babble can open a turn containing zero real words — so the recogniser's word
+// timeline guards every energy close and the interim transcript stands in for
+// missing finals.
+const WORD_RESCUE_MIN_MS = 1500;
+const WORD_RESCUE_NO_CADENCE_MS = 4000;
+const WORD_WAIT_MAX_MS = 4000;
+const FINAL_WAIT_MS = 2500;
+const MIN_DISPATCH_WORDS = 2;
 const MAX_DURATION_GOODBYE = "We've hit the call time limit, so I'll hang up now. Message me any time.";
 const IDLE_GOODBYE = "I haven't heard anything for a while, so I'll hang up now. Message me any time.";
 
@@ -55,7 +74,7 @@ const THINK_TICK_PCM48 = tonePcm48([{ freq: 520, ms: 90 }], 0.045);
 
 class KazeeCallSession {
   constructor({ roomId, chatId, callerId, botUserId, janusUrl, iceServers = [], onTranscript, onEnded,
-    maxCallMs, idleMs, synthesize = synthesizePcm, asCaller = false, greeting = null,
+    maxCallMs, idleMs, finalWaitMs, synthesize = synthesizePcm, synthesizeStream = null, asCaller = false, greeting = null,
     JanusClientImpl = JanusClient, PeerConnectionImpl = RTCPeerConnection, SpeechStreamImpl = SpeechStream } = {}) {
     this.roomId = Number(roomId);
     this.chatId = String(chatId);
@@ -64,7 +83,10 @@ class KazeeCallSession {
     this.asCaller = !!asCaller;
     this.greeting = greeting ? String(greeting) : null;
     this._greeted = false;
+    this._publisherUp = false;
     this._createdRoom = false;
+    this._joined = false;
+    this._publishTimer = null;
     this.janus = new JanusClientImpl({ url: janusUrl });
     this.PeerConnectionImpl = PeerConnectionImpl;
     this.SpeechStreamImpl = SpeechStreamImpl;
@@ -72,8 +94,12 @@ class KazeeCallSession {
     this.onTranscript = onTranscript || (() => {});
     this.onEnded = onEnded || (() => {});
     this.synthesize = synthesize;
+    // Tests and the jig inject a custom synthesize; only pair the streaming
+    // path with the service client it belongs to.
+    this.synthesizeStream = synthesizeStream ?? (synthesize === synthesizePcm ? synthesizeStreamPcm : null);
     this.maxCallMs = Number(maxCallMs ?? process.env.KAZEE_CALL_MAX_MS) || DEFAULT_MAX_CALL_MS;
     this.idleMs = Number(idleMs ?? process.env.KAZEE_CALL_IDLE_MS) || DEFAULT_IDLE_MS;
+    this.finalWaitMs = Number(finalWaitMs) || FINAL_WAIT_MS;
     this.joinRetries = DEFAULT_JOIN_RETRIES;
     this.joinRetryDelayMs = DEFAULT_JOIN_RETRY_DELAY_MS;
     this.codec = new OpusAudioCodec();
@@ -107,6 +133,9 @@ class KazeeCallSession {
       const created = await this.janus.message(this.publisherHandle, {
         request: "create", room: this.roomId, publishers: 6, audiocodec: "opus",
         description: `Call room ${this.roomId}`, permanent: false,
+        // Hidden from videoroom "list": the control plane is internet-reachable,
+        // so a listable room is eavesdroppable. Join-by-roomId is unaffected.
+        is_private: true,
       });
       const cdata = created?.plugindata?.data || {};
       const exists = Number(cdata.error_code) === 427 || /already exists/i.test(String(cdata.error || ""));
@@ -139,6 +168,7 @@ class KazeeCallSession {
   }
 
   async _onPublisherEvent(frame) {
+    if (frame?.janus === "trickle") return this._onTrickle(this.publisher, frame);
     try { await this._consumePublisherFrame(frame); }
     catch (error) { console.error("Kazee call publisher:", error.message); await this.close("janus-error"); }
   }
@@ -157,30 +187,100 @@ class KazeeCallSession {
       }
     }
     if (data.videoroom === "joined") {
+      this._joined = true;
       clog(`Kazee call room joined: room=${this.roomId} publishers=${(data.publishers || []).length}`);
-      if (!this.publisher) await this._publish();
     }
+    if (this._joined && !this.publisher) await this._maybePublish();
     const gone = Number(data.leaving || data.unpublished || 0);
     if (gone && this.subscribers.has(gone)) await this._removeSubscriber(gone);
     if (data.videoroom === "destroyed") await this.close("room-destroyed");
   }
 
+  // Inbound, the bot answers faster than the caller's client finishes joining:
+  // publishing at join fires our publishers event into a tab that may not be
+  // listening yet, and a caller who misses it never subscribes to us — no
+  // remote audio, UI stuck pre-connected (web callers, 2026-08-16). Wait until
+  // the caller's own publisher is visible (proof their handlers are live, and
+  // subscribing to us flips their UI to connected), with a fallback so a
+  // caller who never publishes still gets our media leg. Outbound (asCaller)
+  // publishes at join as before: the callee joins after us and receives our
+  // feed in their own join response, so there is no race.
+  async _maybePublish() {
+    if (this.closed || this.publisher) return;
+    if (this.asCaller || this.subscribers.size > 0) {
+      this._clearPublishTimer();
+      return this._publish();
+    }
+    if (this._publishTimer) return;
+    this._publishTimer = setTimeout(() => {
+      this._publishTimer = null;
+      if (this.closed || this.publisher) return;
+      clog(`Kazee call publish fallback: room=${this.roomId} no remote publisher after ${PUBLISH_DEFER_MS}ms`);
+      this._publish().catch((error) => {
+        console.error("Kazee call publish:", error.message);
+        this.close("publish-failed").catch(() => {});
+      });
+    }, PUBLISH_DEFER_MS);
+    if (typeof this._publishTimer.unref === "function") this._publishTimer.unref();
+  }
+
+  _clearPublishTimer() {
+    if (this._publishTimer) { clearTimeout(this._publishTimer); this._publishTimer = null; }
+  }
+
   async _publish() {
     if (this.publisher) return;
     const pc = new this.PeerConnectionImpl({ iceServers: this.iceServers });
-    pc.iceConnectionStateChange?.subscribe?.((state) => clog(`Kazee call publisher ICE: ${state} room=${this.roomId}`));
+    pc.iceConnectionStateChange?.subscribe?.((state) => {
+      clog(`Kazee call publisher ICE: ${state} room=${this.roomId}`);
+      if (state === "connected" || state === "completed") {
+        this._publisherUp = true;
+        this._maybeGreet();
+      }
+    });
     const track = new MediaStreamTrack({ kind: "audio" });
     const sender = pc.addTrack(track);
-    this.publisher = { pc, track, sender };
+    this.publisher = { pc, track, sender, pending: [], remoteSet: false };
     const offer = await pc.createOffer();
     await pc.setLocalDescription(offer);
     const configured = await this.janus.message(this.publisherHandle, { request: "configure", audio: true, video: false }, pc.localDescription);
     const answer = configured?.jsep;
     if (!answer) throw new Error("Janus publisher configure did not return SDP");
     await pc.setRemoteDescription(answer);
+    this._flushTrickle(this.publisher);
     clog(`Kazee call publisher configured: room=${this.roomId}`);
   }
 
+  // Janus answers candidate-free and trickles ICE afterwards; dropping those
+  // frames leaves both legs stuck in "checking" and the call silently dead.
+  _onTrickle(target, frame) {
+    const candidate = frame?.candidate;
+    if (!target || !candidate || candidate.completed || !candidate.candidate) return;
+    if (!target.remoteSet) {
+      (target.pending ||= []).push(candidate);
+      return;
+    }
+    target.pc.addIceCandidate(candidate).catch((error) => console.error("Kazee call trickle:", error.message));
+  }
+
+  _flushTrickle(target) {
+    if (!target) return;
+    target.remoteSet = true;
+    for (const candidate of (target.pending || []).splice(0)) {
+      target.pc.addIceCandidate(candidate).catch((error) => console.error("Kazee call trickle:", error.message));
+    }
+  }
+
+  // Greeting on STT-connect alone used to speak into dead air while the media
+  // plane was still ICE-checking; wait for the publisher leg to actually come
+  // up AND a remote feed to exist, whichever lands second.
+  _maybeGreet() {
+    if (!this.greeting || this._greeted || this.closed) return;
+    if (!this._publisherUp || this.subscribers.size === 0) return;
+    this._greeted = true;
+    this.speak(this.greeting).catch((error) => console.error("Kazee call greeting:", error.message));
+  }
+
   async _subscribe(feedId, display) {
     if (this.subscribers.has(feedId)) return;
     // Reserve the slot before the first await: the join response and the async
@@ -199,9 +299,23 @@ class KazeeCallSession {
         display: String(display || ""),
         // Opus decoders are stateful, so each feed needs its own.
         codec: new OpusAudioCodec(),
-        vad: new TurnDetector({ thresholdDb: BASE_THRESHOLD_DB }),
-        stt: new this.SpeechStreamImpl({ onTranscript: (text) => this._onFeedTranscript(feedId, text) }),
+        vad: new AdaptiveTurnDetector(),
+        stt: new this.SpeechStreamImpl({
+          onTranscript: (text) => this._onFeedFinal(feedId, text),
+          onPartial: (text) => this._onFeedPartial(feedId, text),
+        }),
         prebuffer: [],
+        pending: [],
+        remoteSet: false,
+        wordCount: 0,
+        lastWordAt: 0,
+        wordGaps: [],
+        lastPartial: "",
+        partialAtCommit: "",
+        awaitingWordsSince: 0,
+        noiseCommit: false,
+        fallbackUsed: false,
+        finalTimer: null,
       };
       this.subscribers.set(feedId, sub);
       pc.onTrack.subscribe((track) => {
@@ -214,11 +328,7 @@ class KazeeCallSession {
         () => clog(`Kazee call STT connected: feed=${feedId}`),
         (error) => console.error("Kazee call STT connect:", error.message),
       );
-      // First remote feed up = both media legs established; greet exactly once.
-      if (this.greeting && !this._greeted) {
-        this._greeted = true;
-        this.speak(this.greeting).catch((error) => console.error("Kazee call greeting:", error.message));
-      }
+      this._maybeGreet();
     } catch (error) {
       if (this.subscribers.get(feedId) === null) this.subscribers.delete(feedId);
       throw error;
@@ -226,6 +336,7 @@ class KazeeCallSession {
   }
 
   async _onSubscriberEvent(feedId, frame) {
+    if (frame?.janus === "trickle") return this._onTrickle(this.subscribers.get(feedId), frame);
     try { await this._consumeSubscriberFrame(feedId, frame); }
     catch (error) { console.error("Kazee call subscriber:", error.message); await this._removeSubscriber(feedId); }
   }
@@ -242,6 +353,7 @@ class KazeeCallSession {
     if (frame.jsep && !sub.started) {
       sub.started = true;
       await sub.pc.setRemoteDescription(frame.jsep);
+      this._flushTrickle(sub);
       const answer = await sub.pc.createAnswer();
       await sub.pc.setLocalDescription(answer);
       await this.janus.message(sub.handle, { request: "start", room: this.roomId }, sub.pc.localDescription);
@@ -249,11 +361,105 @@ class KazeeCallSession {
     }
   }
 
-  _onFeedTranscript(feedId, text) {
+  _onFeedPartial(feedId, text) {
+    const sub = this.subscribers.get(feedId);
+    if (!sub || this.closed) return;
+    const partial = String(text || "");
+    const prevWords = countWords(sub.lastPartial);
+    const words = countWords(partial);
+    sub.lastPartial = partial;
+    // Growth only: a shrunk partial is a recogniser revision or a post-final
+    // buffer reset, not new words.
+    if (words <= prevWords) return;
+    const now = Date.now();
+    if (sub.lastWordAt) {
+      sub.wordGaps.push(now - sub.lastWordAt);
+      if (sub.wordGaps.length > 8) sub.wordGaps.shift();
+    }
+    sub.wordCount += words - prevWords;
+    sub.lastWordAt = now;
+    if (sub.awaitingWordsSince) this._commitTurn(feedId, "words-arrived");
+  }
+
+  // Interims arrive in bursts whose cadence tracks recogniser health (jig
+  // measured 1–20s between new words on a struggling service); a fixed stall
+  // gate would cut callers off mid-sentence on a slow day.
+  _wordRescueGapMs(sub) {
+    if (!sub.wordGaps.length) return WORD_RESCUE_NO_CADENCE_MS;
+    const sorted = [...sub.wordGaps].sort((a, b) => a - b);
+    return Math.max(WORD_RESCUE_MIN_MS, 2 * sorted[Math.floor(sorted.length / 2)]);
+  }
+
+  _commitTurn(feedId, reason) {
+    const sub = this.subscribers.get(feedId);
+    if (!sub || this.closed) return;
+    sub.awaitingWordsSince = 0;
+    if (sub.vad.active) sub.vad.reset();
+    this._lastHeard = Date.now();
+    clog(`Kazee call turn commit: feed=${feedId} reason=${reason} words=${sub.wordCount}`);
+    sub.stt.commit().catch((error) => console.error("Kazee call STT commit:", error.message));
+    if (!sub.noiseCommit) {
+      this._playCue(ACK_CHIME_PCM48).catch(() => {});
+      // STT finalization takes seconds; start ticking at commit so the
+      // chime→first-tick stretch doesn't read as a dead line.
+      this._startThinking();
+    }
+    // Finals routinely never arrive; after finalWaitMs the interim transcript
+    // is dispatched in their place.
+    sub.partialAtCommit = sub.lastPartial;
+    sub.fallbackUsed = false;
+    if (sub.finalTimer) clearTimeout(sub.finalTimer);
+    sub.finalTimer = setTimeout(() => this._finalFallback(feedId), this.finalWaitMs);
+    if (typeof sub.finalTimer.unref === "function") sub.finalTimer.unref();
+    sub.wordCount = 0;
+    sub.lastWordAt = 0;
+    sub.wordGaps = [];
+  }
+
+  _finalFallback(feedId) {
+    const sub = this.subscribers.get(feedId);
+    if (!sub || this.closed) return;
+    sub.finalTimer = null;
+    // Partials keep growing after commit while the decoder catches up; accept
+    // the grown text only if it extends the committed snapshot (anything else
+    // belongs to the next turn).
+    const current = sub.lastPartial || "";
+    const snapshot = sub.partialAtCommit || "";
+    const text = current.length > snapshot.length && current.startsWith(snapshot) ? current : snapshot;
+    sub.fallbackUsed = true;
+    clog(`Kazee call final fallback: feed=${feedId} chars=${text.trim().length}`);
+    this._dispatchTranscript(feedId, text);
+  }
+
+  _onFeedFinal(feedId, text) {
+    const sub = this.subscribers.get(feedId);
+    if (sub?.finalTimer) {
+      clearTimeout(sub.finalTimer);
+      sub.finalTimer = null;
+    } else if (sub?.fallbackUsed) {
+      // The interim already went to the model; a late final would double-answer.
+      sub.fallbackUsed = false;
+      clog(`Kazee call late final dropped: feed=${feedId}`);
+      return;
+    }
+    this._dispatchTranscript(feedId, text);
+  }
+
+  _dispatchTranscript(feedId, text) {
     const sub = this.subscribers.get(feedId);
-    const len = String(text || "").trim().length;
-    clog(`Kazee call transcript: feed=${feedId} chars=${len}${len ? "" : " (noise commit)"}`);
-    if (String(text || "").trim()) this._startThinking();
+    const trimmed = String(text || "").trim();
+    if (sub?.noiseCommit) {
+      sub.noiseCommit = false;
+      // A turn the recogniser never put words in was background noise; babble
+      // occasionally leaks a single stray word into the transcript too.
+      if (countWords(trimmed) < MIN_DISPATCH_WORDS) {
+        clog(`Kazee call noise turn dropped: feed=${feedId} words=${countWords(trimmed)}`);
+        if (!this._speakActive && !this._speakQueue.length) this._stopThinking();
+        return;
+      }
+    }
+    clog(`Kazee call transcript: feed=${feedId} chars=${trimmed.length}${trimmed.length ? "" : " (noise commit)"}`);
+    if (trimmed) this._startThinking();
     // Empty finalization (noise commit): nothing dispatches, no reply is
     // coming — don't tick into the void.
     else if (!this._speakActive && !this._speakQueue.length) this._stopThinking();
@@ -264,20 +470,23 @@ class KazeeCallSession {
     });
   }
 
+  _writeFrame(frame) {
+    if (frame.length < WEBRTC_SAMPLES * 2) frame = Buffer.concat([frame, Buffer.alloc(WEBRTC_SAMPLES * 2 - frame.length)]);
+    const opus = this.codec.encode(frame);
+    const payloadType = this.publisher?.sender?.codec?.payloadType || 111;
+    this.publisher?.track?.writeRtp(new RtpPacket(new RtpHeader({
+      payloadType, sequenceNumber: this.sequence++ & 0xffff, timestamp: this.timestamp >>> 0, ssrc: this.ssrc,
+    }), opus));
+    this.timestamp = (this.timestamp + WEBRTC_SAMPLES) >>> 0;
+  }
+
   // Writes a short cue into the outbound track. Yields to real speech: the
   // moment an utterance is active (or starting), remaining frames are dropped.
   async _playCue(pcm48) {
     if (this.closed || this.speaking || this._speakActive || !this.publisher) return;
     for (let offset = 0; offset < pcm48.length; offset += WEBRTC_SAMPLES * 2) {
       if (this.closed || this.speaking || this._speakActive) return;
-      let frame = pcm48.subarray(offset, offset + WEBRTC_SAMPLES * 2);
-      if (frame.length < WEBRTC_SAMPLES * 2) frame = Buffer.concat([frame, Buffer.alloc(WEBRTC_SAMPLES * 2 - frame.length)]);
-      const opus = this.codec.encode(frame);
-      const payloadType = this.publisher?.sender?.codec?.payloadType || 111;
-      this.publisher?.track?.writeRtp(new RtpPacket(new RtpHeader({
-        payloadType, sequenceNumber: this.sequence++ & 0xffff, timestamp: this.timestamp >>> 0, ssrc: this.ssrc,
-      }), opus));
-      this.timestamp = (this.timestamp + WEBRTC_SAMPLES) >>> 0;
+      this._writeFrame(pcm48.subarray(offset, offset + WEBRTC_SAMPLES * 2));
       await wait(20);
     }
   }
@@ -309,9 +518,30 @@ class KazeeCallSession {
     }
     let pcm;
     try { pcm = sub.codec.decode(packet.payload); } catch (_) { return; }
-    sub.vad.thresholdDb = this.speaking ? SPEAKING_THRESHOLD_DB : BASE_THRESHOLD_DB;
+    // Echo guard: raise the effective threshold while the bot speaks; the
+    // adaptive floor keeps sampling the un-boosted base so echo never pollutes
+    // the noise-floor estimate.
+    sub.vad.boostTo = this.speaking ? SPEAKING_THRESHOLD_DB : null;
     const event = sub.vad.push(pcm);
     const pcm16 = downsample48To16(pcm);
+    if (sub.awaitingWordsSince) {
+      // Energy closed the turn but the recogniser hasn't produced a word yet:
+      // hold the commit and keep streaming audio so a slow decode still lands.
+      this._feedAudio(sub, pcm16);
+      if (event?.type === "start") {
+        // Caller resumed before any words arrived — same turn continues.
+        sub.awaitingWordsSince = 0;
+        this._lastHeard = Date.now();
+        if (this.speaking) this.cancelSpeech();
+        this._stopThinking();
+        return;
+      }
+      if (Date.now() - sub.awaitingWordsSince >= WORD_WAIT_MAX_MS) {
+        sub.noiseCommit = true;
+        this._commitTurn(feedId, "no-words");
+      }
+      return;
+    }
     if (!sub.vad.active && !event) {
       sub.prebuffer.push(pcm16);
       if (sub.prebuffer.length > PREBUFFER_FRAMES) sub.prebuffer.shift();
@@ -327,13 +557,22 @@ class KazeeCallSession {
     }
     this._feedAudio(sub, pcm16);
     if (event?.type === "commit") {
-      this._lastHeard = Date.now();
-      clog(`Kazee call VAD commit: feed=${feedId}`);
-      sub.stt.commit().catch((error) => console.error("Kazee call STT commit:", error.message));
-      this._playCue(ACK_CHIME_PCM48).catch(() => {});
-      // STT finalization takes seconds; start ticking at commit so the
-      // chime→first-tick stretch doesn't read as a dead line.
-      this._startThinking();
+      if (sub.wordCount > 0 || event.reason === "max-duration") {
+        if (sub.wordCount === 0) sub.noiseCommit = true;
+        this._commitTurn(feedId, event.reason);
+      } else {
+        sub.awaitingWordsSince = Date.now();
+        clog(`Kazee call commit held for words: feed=${feedId}`);
+      }
+    } else if (sub.vad.active && sub.wordCount >= MIN_DISPATCH_WORDS && sub.lastWordAt) {
+      // Steady background can keep every frame "loud" so the energy close
+      // never fires; when the word timeline has stalled well past its own
+      // cadence, the caller has finished — commit anyway.
+      const gap = this._wordRescueGapMs(sub);
+      if (Date.now() - sub.lastWordAt >= gap) {
+        clog(`Kazee call word-stall commit: feed=${feedId} gap=${gap}ms`);
+        this._commitTurn(feedId, "word-stall");
+      }
     }
   }
 
@@ -373,30 +612,100 @@ class KazeeCallSession {
     })();
   }
 
+  async _playPcm48(pcm48, gen) {
+    this._stopThinking();
+    this.speaking = true;
+    try {
+      // Drift-corrected pacing: a naive wait(20) accumulates encode time and
+      // timer slop, audibly stretching long utterances.
+      const startAt = Date.now();
+      for (let offset = 0, i = 1; offset < pcm48.length; offset += WEBRTC_SAMPLES * 2, i += 1) {
+        if (this.closed || gen !== this._speakGen) return false;
+        this._writeFrame(pcm48.subarray(offset, offset + WEBRTC_SAMPLES * 2));
+        const delay = startAt + i * 20 - Date.now();
+        if (delay > 0) await wait(delay);
+      }
+      return true;
+    } finally {
+      this._lastSpokeAt = Date.now();
+    }
+  }
+
   async _playUtterance(text, gen) {
+    // Cache only audio rendered by the real service: the key is voice+text, so
+    // PCM from an injected synthesize would leak across sessions.
+    const usingService = this.synthesizeStream !== null || this.synthesize === synthesizePcm;
+    const cacheable = usingService && (text === this.greeting || text === MAX_DURATION_GOODBYE || text === IDLE_GOODBYE);
+    const cacheKey = `${ttsVoice.getVoice()}::${text}`;
     const synthStart = Date.now();
+    if (cacheable && SYNTH_CACHE.has(cacheKey)) {
+      clog(`Kazee call speak: cached chars=${text.length}`);
+      return this._playPcm48(SYNTH_CACHE.get(cacheKey), gen);
+    }
+    if (this.synthesizeStream) {
+      try {
+        return await this._playStreaming(text, gen, { cacheable, cacheKey, synthStart });
+      } catch (error) {
+        if (this.closed || gen !== this._speakGen) return false;
+        clog(`Kazee call stream synth failed (${error.message}); buffering`);
+      }
+    }
     const synthesized = await this.synthesize(text);
     if (this.closed || gen !== this._speakGen) return false;
     const pcm48 = upsampleTo48(synthesized.pcm, synthesized.sampleRate);
     clog(`Kazee call speak: synth=${Date.now() - synthStart}ms chars=${text.length}`);
-    this._stopThinking();
-    this.speaking = true;
+    if (cacheable) SYNTH_CACHE.set(cacheKey, pcm48);
+    return this._playPcm48(pcm48, gen);
+  }
+
+  // Plays TTS as it downloads: the service streams PCM, but buffering the full
+  // response cost ~3s of silence before the first word on every reply.
+  async _playStreaming(text, gen, { cacheable, cacheKey, synthStart }) {
+    const { sampleRate, chunks } = await this.synthesizeStream(text);
+    if (this.closed || gen !== this._speakGen) return false;
+    let rawCarry = Buffer.alloc(0); // an odd byte straddling a chunk boundary splits a sample
+    let frameCarry = Buffer.alloc(0);
+    const parts = cacheable ? [] : null;
+    let started = false;
+    let startAt = 0;
+    let i = 0;
     try {
-      for (let offset = 0; offset < pcm48.length; offset += WEBRTC_SAMPLES * 2) {
+      for await (const chunk of chunks) {
         if (this.closed || gen !== this._speakGen) return false;
-        let frame = pcm48.subarray(offset, offset + WEBRTC_SAMPLES * 2);
-        if (frame.length < WEBRTC_SAMPLES * 2) frame = Buffer.concat([frame, Buffer.alloc(WEBRTC_SAMPLES * 2 - frame.length)]);
-        const opus = this.codec.encode(frame);
-        const payloadType = this.publisher?.sender?.codec?.payloadType || 111;
-        this.publisher?.track?.writeRtp(new RtpPacket(new RtpHeader({
-          payloadType, sequenceNumber: this.sequence++ & 0xffff, timestamp: this.timestamp >>> 0, ssrc: this.ssrc,
-        }), opus));
-        this.timestamp = (this.timestamp + WEBRTC_SAMPLES) >>> 0;
-        await wait(20);
+        const raw = rawCarry.length ? Buffer.concat([rawCarry, chunk]) : chunk;
+        const usable = raw.length & ~1;
+        rawCarry = raw.subarray(usable);
+        if (!usable) continue;
+        const pcm48 = upsampleTo48(raw.subarray(0, usable), sampleRate);
+        if (parts) parts.push(pcm48);
+        if (!started) {
+          started = true;
+          clog(`Kazee call speak: first-audio=${Date.now() - synthStart}ms chars=${text.length}`);
+          // Only now is the caller actually hearing us — until first audio the
+          // thinking ticks and barge-in semantics should stay in "waiting" mode.
+          this._stopThinking();
+          this.speaking = true;
+          startAt = Date.now();
+        }
+        frameCarry = frameCarry.length ? Buffer.concat([frameCarry, pcm48]) : pcm48;
+        let offset = 0;
+        for (; offset + WEBRTC_SAMPLES * 2 <= frameCarry.length; offset += WEBRTC_SAMPLES * 2) {
+          if (this.closed || gen !== this._speakGen) return false;
+          this._writeFrame(frameCarry.subarray(offset, offset + WEBRTC_SAMPLES * 2));
+          i += 1;
+          const delay = startAt + i * 20 - Date.now();
+          if (delay > 0) await wait(delay);
+          // The service stalled mid-utterance; re-anchor so playback resumes
+          // at normal pace instead of bursting the backlog.
+          else if (delay < -200) startAt = Date.now() - i * 20;
+        }
+        frameCarry = frameCarry.subarray(offset);
       }
-      return true;
+      if (frameCarry.length && started && !this.closed && gen === this._speakGen) this._writeFrame(frameCarry);
+      if (parts && started) SYNTH_CACHE.set(cacheKey, Buffer.concat(parts));
+      return started;
     } finally {
-      this._lastSpokeAt = Date.now();
+      if (started) this._lastSpokeAt = Date.now();
     }
   }
 
@@ -424,6 +733,7 @@ class KazeeCallSession {
     const sub = this.subscribers.get(feedId);
     if (!sub) return;
     this.subscribers.delete(feedId);
+    if (sub.finalTimer) clearTimeout(sub.finalTimer);
     try { sub.stt.close(); } catch (_) {}
     try { sub.codec.close(); } catch (_) {}
     try { await sub.pc.close(); } catch (_) {}
@@ -435,6 +745,7 @@ class KazeeCallSession {
     this.closed = true;
     console.log(`Kazee call session closed: room=${this.roomId} reason=${reason}`);
     if (this._limitTimer) { clearInterval(this._limitTimer); this._limitTimer = null; }
+    this._clearPublishTimer();
     this._stopThinking();
     this.cancelSpeech();
     for (const feed of [...this.subscribers.keys()]) await this._removeSubscriber(feed);

package/channels/kazee/call/speech-client.js

@@ -6,11 +6,12 @@ const { speechBearerToken } = require("../../../core/media");
 const ttsVoice = require("../../../core/tts-voice");
 
 class SpeechStream {
-  constructor({ baseUrl = SPEECH_SERVICE_URL, token = speechBearerToken(), WebSocketImpl = WebSocket, onTranscript } = {}) {
+  constructor({ baseUrl = SPEECH_SERVICE_URL, token = speechBearerToken(), WebSocketImpl = WebSocket, onTranscript, onPartial } = {}) {
     this.baseUrl = String(baseUrl || "").replace(/\/$/, "");
     this.token = token;
     this.WebSocketImpl = WebSocketImpl;
     this.onTranscript = onTranscript || (() => {});
+    this.onPartial = onPartial || (() => {});
     this.socket = null;
     this.ready = null;
   }
@@ -47,8 +48,12 @@ class SpeechStream {
     let message;
     try { message = JSON.parse(raw.toString()); } catch (_) { return; }
     const text = String(message.text || message.transcript || "").trim();
-    if (text && (message.final === true || message.type === "final" || message.type === "transcription")) {
+    if (!text) return;
+    if (message.final === true || message.type === "final" || message.type === "transcription") {
       Promise.resolve(this.onTranscript(text)).catch((error) => console.error("Kazee call transcript:", error.message));
+    } else {
+      // Interims drive word-based turn-end detection; they were previously discarded.
+      Promise.resolve(this.onPartial(text)).catch((error) => console.error("Kazee call partial:", error.message));
     }
   }
 
@@ -76,4 +81,34 @@ async function synthesizePcm(text, { baseUrl = SPEECH_SERVICE_URL, token = speec
   return { pcm: Buffer.from(await response.arrayBuffer()), sampleRate: rate };
 }
 
-module.exports = { SpeechStream, synthesizePcm };
+// Same endpoint as synthesizePcm, but yields PCM chunks as they arrive so
+// playback can start on the first byte instead of after the full download.
+async function synthesizeStream(text, { baseUrl = SPEECH_SERVICE_URL, token = speechBearerToken(), fetchImpl = fetch } = {}) {
+  if (!baseUrl || !token) throw new Error("Hosted speech service is not configured");
+  const response = await fetchImpl(`${String(baseUrl).replace(/\/$/, "")}/v1/audio/speech`, {
+    method: "POST",
+    headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json" },
+    body: JSON.stringify({ input: text, voice: ttsVoice.getVoice(), response_format: "pcm" }),
+    signal: AbortSignal.timeout(90000),
+  });
+  if (!response.ok) throw new Error(`Hosted TTS failed: HTTP ${response.status}`);
+  const contentType = response.headers?.get?.("content-type") || "";
+  const rate = Number(contentType.match(/(?:^|;)\s*rate=(\d+)/i)?.[1] || 24000);
+  if (!Number.isFinite(rate) || rate < 8000 || rate > 48000) throw new Error(`Hosted TTS returned invalid PCM rate: ${rate}`);
+  const reader = response.body.getReader();
+  async function* chunks() {
+    try {
+      for (;;) {
+        const { done, value } = await reader.read();
+        if (done) return;
+        if (value?.byteLength) yield Buffer.from(value);
+      }
+    } finally {
+      // Barge-in abandons the iterator mid-stream; stop the download too.
+      reader.cancel?.().catch?.(() => {});
+    }
+  }
+  return { sampleRate: rate, chunks: chunks() };
+}
+
+module.exports = { SpeechStream, synthesizePcm, synthesizeStream };