@inetafrica/open-claudia 3.1.12 → 3.1.14

package/.env.example

@@ -13,6 +13,8 @@ KAZEE_BOT_USER_ID=
 KAZEE_OWNER_USER_ID=
 # Set to 1 to log every inbound Kazee socket event (debug aid; noisy).
 KAZEE_DEBUG_EVENTS=
+# Optional override; Kazee calls default to the production Janus WebSocket.
+KAZEE_JANUS_URL=wss://janus.kazee.africa/ws
 
 WORKSPACE=/path/to/your/workspace
 # Both provider paths are optional. Leave blank to discover the CLI on PATH.

package/CHANGELOG.md

@@ -1,5 +1,21 @@
 # Changelog
 
+## v3.1.14 — Call bridge production hardening: serialized speech, barge-in, group attribution, limits
+
+- **TTS is serialized through a queue with cancellation.** Concurrent `speak()` calls used to interleave two 20ms-frame RTP loops over the same sequence/timestamp counters — garbled audio whenever the provider emitted more than one message. Utterances now play strictly one at a time; `cancelSpeech()` (generation-based) stops the current utterance mid-frame and resolves everything queued, and `close()` cancels so no pending `speak()` can hang the adapter.
+- **Barge-in + echo gate.** While the bot is speaking, the VAD threshold rises from −38 to −26 dBFS so the caller's device echoing the bot's own TTS can't trigger STT; genuine speech over the gate cancels playback immediately — you can interrupt her mid-answer like a real call.
+- **Group calls get per-feed pipelines and speaker attribution.** Each subscribed feed now has its own Opus decoder (they're stateful — one shared decoder produced garbage on 2+ feeds), its own VAD, and its own STT stream. Transcripts carry `{feedId, display, participants}`; with more than one participant in the room the provider sees `alice@…: text`, so group calls stop reading as one merged anonymous voice. STT send/commit failures now log and drop the frame instead of killing the whole call (the stream self-heals on the next send).
+- **Calls end on their own when they should.** 30-minute duration cap (`KAZEE_CALL_MAX_MS`) and 2-minute idle timeout (`KAZEE_CALL_IDLE_MS`), both announced with a spoken goodbye; if the caller barges in during the idle goodbye the call stays up. A leaked session can no longer hold Janus/STT resources forever.
+- **First syllables reach STT.** A 240ms pre-onset ring buffer flushes into the STT stream when a turn starts, so VAD onset latency stops clipping the start of every utterance.
+- The five existing call tests plus new `test-kazee-call-session.js` (serialization, cancellation, echo gate, barge-in, prebuffer, attribution, idle/max limits, close semantics) are wired into the main `npm test` chain — call regressions now block every release, not just `test:kazee-call` runs.
+
+## v3.1.13 — Owner-only Kazee calls reach Open Claudia
+
+- Accepts incoming audio calls from the configured Kazee owner only; external, video, malformed, and concurrent calls are rejected before any media session is created.
+- Joins the existing Janus VideoRoom as a headless publisher/subscriber using pure-TypeScript WebRTC, decodes inbound Opus, commits VAD-delimited 16 kHz PCM to hosted streaming STT, and routes final utterances through the owner's normal Kazee identity and conversation.
+- Speaks only the settled, guardrail-vetted answer through hosted Pocket TTS PCM and Opus RTP. Provider narration and transient progress previews are suppressed in calls; audio is never persisted.
+- Adds deterministic audio, Janus transaction, WebRTC RTP, speech framing, owner gate, lifecycle, and AI-routing tests. Updates `ws` to the patched 8.21 line across direct and Socket.IO use.
+
 ## v3.1.10 — `say` synthesizes in the bot process, not the stripped subprocess
 
 - v3.1.9 carried this exact fix but its build never published: the version pin in `test-provider-language.js` still read `3.1.8`, so the guard failed and the docker stage was skipped. Re-cut as v3.1.10 per "never retag a failed version."

package/channels/kazee/adapter.js

@@ -13,6 +13,7 @@ const { canonicalForChannel } = require("../../core/identity");
 const { inlineKeyboardToPortable } = require("../types");
 const { createKazeeSocket } = require("./socket");
 const { normalize } = require("./format");
+const { KazeeCallManager } = require("./call/manager");
 
 class KazeeAdapter {
   constructor({ id = "kazee", url, token, ownerUserId, botUserId }) {
@@ -27,6 +28,7 @@ class KazeeAdapter {
     this.botUserId = botUserId || null;
     this._listeners = { message: new Set(), action: new Set() };
     this._socket = null;
+    this._callManager = new KazeeCallManager({ adapter: this });
   }
 
   on(event, fn) {
@@ -67,11 +69,13 @@ class KazeeAdapter {
   }
 
   async stop() {
+    await this._callManager.stop();
     try { this._socket?.disconnect?.(); } catch (e) {}
     this._socket = null;
   }
 
   _wireSocket(socket) {
+    this._callManager.wire(socket);
     if (process.env.KAZEE_DEBUG_EVENTS) {
       socket.onAny((event, ...args) => {
         try { console.log(`[kazee event] ${event}`, JSON.stringify(args).slice(0, 400)); }

package/channels/kazee/call/audio.js

@@ -0,0 +1,133 @@
+"use strict";
+
+const OpusScript = require("opusscript");
+
+const WEBRTC_RATE = 48000;
+const STT_RATE = 16000;
+const FRAME_MS = 20;
+const WEBRTC_SAMPLES = WEBRTC_RATE * FRAME_MS / 1000;
+
+class OpusAudioCodec {
+  constructor({ channels = 1 } = {}) {
+    this.channels = channels;
+    this.encoder = new OpusScript(WEBRTC_RATE, channels, OpusScript.Application.VOIP);
+    this.decoder = new OpusScript(WEBRTC_RATE, channels, OpusScript.Application.VOIP);
+  }
+
+  encode(pcm48k) {
+    if (!Buffer.isBuffer(pcm48k)) pcm48k = Buffer.from(pcm48k);
+    const samples = pcm48k.length / 2 / this.channels;
+    if (samples !== WEBRTC_SAMPLES) {
+      throw new Error(`Opus input must be exactly ${WEBRTC_SAMPLES} samples per channel (20ms)`);
+    }
+    return Buffer.from(this.encoder.encode(pcm48k, samples));
+  }
+
+  decode(opus) {
+    return Buffer.from(this.decoder.decode(Buffer.from(opus)));
+  }
+
+  close() {
+    this.encoder?.delete();
+    this.decoder?.delete();
+    this.encoder = null;
+    this.decoder = null;
+  }
+}
+
+// WebRTC Opus is 48 kHz. Faster Whisper's streaming endpoint accepts PCM at
+// any declared rate, but 16 kHz cuts bandwidth and inference conversion work.
+function downsample48To16(pcm48k) {
+  if (!Buffer.isBuffer(pcm48k)) pcm48k = Buffer.from(pcm48k);
+  const input = new Int16Array(pcm48k.buffer, pcm48k.byteOffset, Math.floor(pcm48k.length / 2));
+  const output = Buffer.allocUnsafe(Math.floor(input.length / 3) * 2);
+  const samples = new Int16Array(output.buffer, output.byteOffset, output.length / 2);
+  for (let i = 0, j = 0; j < samples.length; i += 3, j += 1) {
+    samples[j] = Math.round((input[i] + input[i + 1] + input[i + 2]) / 3);
+  }
+  return output;
+}
+
+function upsampleTo48(pcm, sourceRate) {
+  if (!Buffer.isBuffer(pcm)) pcm = Buffer.from(pcm);
+  if (sourceRate === WEBRTC_RATE) return Buffer.from(pcm);
+  if (!Number.isFinite(sourceRate) || sourceRate < 8000 || sourceRate > WEBRTC_RATE) {
+    throw new Error(`Unsupported PCM source rate: ${sourceRate}`);
+  }
+  const input = new Int16Array(pcm.buffer, pcm.byteOffset, Math.floor(pcm.length / 2));
+  const outputLength = Math.max(0, Math.round(input.length * WEBRTC_RATE / sourceRate));
+  const output = Buffer.allocUnsafe(outputLength * 2);
+  const samples = new Int16Array(output.buffer, output.byteOffset, outputLength);
+  const scale = sourceRate / WEBRTC_RATE;
+  for (let i = 0; i < outputLength; i += 1) {
+    const position = i * scale;
+    const left = Math.floor(position);
+    const right = Math.min(left + 1, input.length - 1);
+    const fraction = position - left;
+    samples[i] = Math.round(input[left] + (input[right] - input[left]) * fraction);
+  }
+  return output;
+}
+
+function rmsDbfs(pcm) {
+  if (!pcm?.length) return -Infinity;
+  const samples = new Int16Array(pcm.buffer, pcm.byteOffset, Math.floor(pcm.length / 2));
+  let sum = 0;
+  for (const sample of samples) {
+    const normalized = sample / 32768;
+    sum += normalized * normalized;
+  }
+  if (!sum) return -Infinity;
+  return 20 * Math.log10(Math.sqrt(sum / samples.length));
+}
+
+class TurnDetector {
+  constructor({ thresholdDb = -38, onsetFrames = 3, silenceMs = 700, maxTurnMs = 30000 } = {}) {
+    this.thresholdDb = thresholdDb;
+    this.onsetFrames = onsetFrames;
+    this.silenceFrames = Math.ceil(silenceMs / FRAME_MS);
+    this.maxFrames = Math.ceil(maxTurnMs / FRAME_MS);
+    this.reset();
+  }
+
+  reset() {
+    this.active = false;
+    this.loudFrames = 0;
+    this.quietFrames = 0;
+    this.turnFrames = 0;
+  }
+
+  push(pcm48k) {
+    const loud = rmsDbfs(pcm48k) >= this.thresholdDb;
+    if (!this.active) {
+      this.loudFrames = loud ? this.loudFrames + 1 : 0;
+      if (this.loudFrames >= this.onsetFrames) {
+        this.active = true;
+        this.quietFrames = 0;
+        this.turnFrames = this.loudFrames;
+        return { type: "start" };
+      }
+      return null;
+    }
+    this.turnFrames += 1;
+    this.quietFrames = loud ? 0 : this.quietFrames + 1;
+    if (this.quietFrames >= this.silenceFrames || this.turnFrames >= this.maxFrames) {
+      const reason = this.turnFrames >= this.maxFrames ? "max-duration" : "silence";
+      this.reset();
+      return { type: "commit", reason };
+    }
+    return null;
+  }
+}
+
+module.exports = {
+  FRAME_MS,
+  STT_RATE,
+  WEBRTC_RATE,
+  WEBRTC_SAMPLES,
+  OpusAudioCodec,
+  TurnDetector,
+  downsample48To16,
+  rmsDbfs,
+  upsampleTo48,
+};

package/channels/kazee/call/janus-client.js

@@ -0,0 +1,136 @@
+"use strict";
+
+const crypto = require("crypto");
+const WebSocket = require("ws");
+
+function transactionId() {
+  return crypto.randomBytes(8).toString("hex");
+}
+
+class JanusClient {
+  constructor({ url, WebSocketImpl = WebSocket, requestTimeoutMs = 10000, keepaliveMs = 25000 } = {}) {
+    if (!url) throw new Error("Janus URL is required");
+    this.url = url;
+    this.WebSocketImpl = WebSocketImpl;
+    this.requestTimeoutMs = requestTimeoutMs;
+    this.keepaliveMs = keepaliveMs;
+    this.socket = null;
+    this.sessionId = null;
+    this.pending = new Map();
+    this.handleListeners = new Map();
+    this.keepalive = null;
+  }
+
+  async connect() {
+    if (this.socket) return;
+    const socket = new this.WebSocketImpl(this.url, "janus-protocol");
+    this.socket = socket;
+    socket.on("message", (data) => this._onMessage(data));
+    socket.on("close", () => this._onClose(new Error("Janus socket closed")));
+    socket.on("error", (error) => this._onClose(error));
+    await new Promise((resolve, reject) => {
+      socket.once("open", resolve);
+      socket.once("error", reject);
+    });
+    const created = await this.request({ janus: "create" }, { withSession: false });
+    this.sessionId = created?.data?.id;
+    if (!this.sessionId) throw new Error("Janus create response did not include a session id");
+    this.keepalive = setInterval(() => {
+      this.request({ janus: "keepalive" }).catch((error) => console.error("Janus keepalive:", error.message));
+    }, this.keepaliveMs);
+    this.keepalive.unref?.();
+  }
+
+  async attach(plugin = "janus.plugin.videoroom") {
+    const response = await this.request({ janus: "attach", plugin });
+    const handleId = response?.data?.id;
+    if (!handleId) throw new Error("Janus attach response did not include a handle id");
+    return handleId;
+  }
+
+  onHandle(handleId, listener) {
+    const id = String(handleId);
+    if (!this.handleListeners.has(id)) this.handleListeners.set(id, new Set());
+    this.handleListeners.get(id).add(listener);
+    return () => this.handleListeners.get(id)?.delete(listener);
+  }
+
+  message(handleId, body, jsep) {
+    return this.request({ janus: "message", handle_id: handleId, body, ...(jsep ? { jsep } : {}) });
+  }
+
+  async detach(handleId) {
+    if (!handleId || !this.sessionId) return;
+    try { await this.request({ janus: "detach", handle_id: handleId }); }
+    finally { this.handleListeners.delete(String(handleId)); }
+  }
+
+  request(payload, { withSession = true } = {}) {
+    if (!this.socket || this.socket.readyState !== this.WebSocketImpl.OPEN) {
+      return Promise.reject(new Error("Janus socket is not open"));
+    }
+    const transaction = transactionId();
+    const frame = {
+      ...payload,
+      ...(withSession && this.sessionId ? { session_id: this.sessionId } : {}),
+      transaction,
+    };
+    return new Promise((resolve, reject) => {
+      const timer = setTimeout(() => {
+        this.pending.delete(transaction);
+        reject(new Error(`Janus ${payload.janus} timed out`));
+      }, this.requestTimeoutMs);
+      this.pending.set(transaction, { resolve, reject, timer });
+      this.socket.send(JSON.stringify(frame), (error) => {
+        if (!error) return;
+        clearTimeout(timer);
+        this.pending.delete(transaction);
+        reject(error);
+      });
+    });
+  }
+
+  async close() {
+    if (this.keepalive) clearInterval(this.keepalive);
+    this.keepalive = null;
+    if (this.sessionId && this.socket?.readyState === this.WebSocketImpl.OPEN) {
+      try { await this.request({ janus: "destroy" }); } catch (_) {}
+    }
+    this.sessionId = null;
+    const socket = this.socket;
+    this.socket = null;
+    if (socket && socket.readyState < this.WebSocketImpl.CLOSING) socket.close();
+    this._onClose(new Error("Janus client closed"));
+  }
+
+  _onMessage(raw) {
+    let frame;
+    try { frame = JSON.parse(raw.toString()); }
+    catch (_) { return; }
+    const pending = frame.transaction && this.pending.get(frame.transaction);
+    if (pending && frame.janus !== "ack") {
+      clearTimeout(pending.timer);
+      this.pending.delete(frame.transaction);
+      if (frame.janus === "error") pending.reject(new Error(frame.error?.reason || "Janus error"));
+      else pending.resolve(frame);
+    }
+    const sender = frame.sender != null ? String(frame.sender) : null;
+    if (sender) {
+      for (const listener of this.handleListeners.get(sender) || []) {
+        try { listener(frame); } catch (error) { console.error("Janus handle listener:", error.message); }
+      }
+    }
+  }
+
+  _onClose(error) {
+    if (this.keepalive) clearInterval(this.keepalive);
+    this.keepalive = null;
+    for (const pending of this.pending.values()) {
+      clearTimeout(pending.timer);
+      pending.reject(error);
+    }
+    this.pending.clear();
+  }
+}
+
+module.exports = { JanusClient };

package/channels/kazee/call/manager.js

@@ -0,0 +1,112 @@
+"use strict";
+
+const { canonicalForChannel } = require("../../../core/identity");
+const { KazeeCallSession } = require("./session");
+
+const DEFAULT_JANUS_URL = "wss://janus.kazee.africa/ws";
+const DEFAULT_ICE_SERVERS = [
+  { urls: "stun:coturn.kazee.africa:3478" },
+  { urls: "stun:stun.l.google.com:19302" },
+];
+
+class KazeeCallManager {
+  constructor({ adapter, SessionImpl = KazeeCallSession, janusUrl = process.env.KAZEE_JANUS_URL || DEFAULT_JANUS_URL,
+    iceServers = DEFAULT_ICE_SERVERS } = {}) {
+    this.adapter = adapter;
+    this.SessionImpl = SessionImpl;
+    this.janusUrl = janusUrl;
+    this.iceServers = iceServers;
+    this.session = null;
+  }
+
+  wire(socket) {
+    socket.on("call:incoming", (payload) => this.incoming(payload));
+    socket.on("call:ended", (payload) => this.ended(payload));
+    socket.on("call:rejected", (payload) => this.ended(payload));
+  }
+
+  async incoming(payload = {}) {
+    const callerId = String(payload.fromUserId || "");
+    const chatId = String(payload.chatId || "");
+    const roomId = Number(payload.roomId);
+    const callType = String(payload.callType || "");
+    const owner = String(this.adapter.ownerUserId || "");
+    const reject = (reason) => this.adapter._socket?.emit("call:reject", {
+      toUserId: callerId, callType, chatId, roomId, reason,
+    });
+
+    if (!callerId || !chatId || !Number.isSafeInteger(roomId)) return reject("invalid_call");
+    if (callerId !== owner) return reject("not_allowed");
+    if (callType !== "audio") return reject("unsupported_media");
+    if (this.session) return reject("busy");
+
+    const session = new this.SessionImpl({
+      roomId, chatId, callerId, botUserId: this.adapter.botUserId,
+      janusUrl: this.janusUrl, iceServers: this.iceServers,
+      onTranscript: (text, meta) => this._transcript(text, payload, meta),
+      onEnded: () => { if (this.session === session) this.session = null; },
+    });
+    this.session = session;
+    try {
+      await session.start();
+      this.adapter._socket?.emit("call:accept", {
+        toUserId: callerId, callType: "audio", chatId, roomId,
+      });
+    } catch (error) {
+      console.error("Kazee call connect:", error.message);
+      this.session = null;
+      await session.close("connect-failed").catch(() => {});
+      reject("connection_failed");
+    }
+  }
+
+  _transcript(text, incoming, meta = {}) {
+    const session = this.session;
+    if (!session || !text) return;
+    // In a group room, tell the provider who is talking; 1:1 stays unprefixed.
+    const speaker = String(meta.display || "").trim();
+    const attributed = (meta.participants || 0) > 1 && speaker ? `${speaker}: ${text}` : text;
+    const callAdapter = Object.create(this.adapter);
+    // Provider narration/previews are useful in chat but must never be spoken
+    // into a call. Only the settled, guarded final answer reaches send().
+    callAdapter.suppressStreamPreview = true;
+    callAdapter.send = async (_channelId, response) => {
+      await session.speak(response);
+      return true;
+    };
+    callAdapter.edit = async () => true;
+    callAdapter.typing = async () => {};
+    callAdapter.sendVoice = async (_channelId, voicePath) => {
+      console.warn(`Kazee call ignored file voice response ${voicePath}; streamed TTS is active`);
+      return true;
+    };
+    this.adapter._emit("message", {
+      adapter: callAdapter,
+      channelId: String(incoming.chatId),
+      canonicalUserId: canonicalForChannel("kazee", String(incoming.fromUserId)),
+      userId: String(incoming.fromUserId),
+      type: "text",
+      text: attributed,
+      messageId: `call-${incoming.roomId}-${Date.now()}`,
+      from: { id: String(incoming.fromUserId), name: incoming.fromUserName || "" },
+      raw: { ...incoming, source: "kazee-call", ephemeralAudio: true, speaker: speaker || undefined, feedId: meta.feedId },
+    });
+  }
+
+  async ended(payload = {}) {
+    if (!this.session) return;
+    if (payload.roomId != null && Number(payload.roomId) !== this.session.roomId) return;
+    const session = this.session;
+    this.session = null;
+    await session.close("remote-ended");
+  }
+
+  async stop() {
+    if (!this.session) return;
+    const session = this.session;
+    this.session = null;
+    await session.close("adapter-stopped");
+  }
+}
+
+module.exports = { KazeeCallManager, DEFAULT_ICE_SERVERS, DEFAULT_JANUS_URL };