@profullstack/qryptchat 0.8.1 → 0.9.0

package/dist/{chunk-wtbsdxe7.js → chunk-e39k2cyf.js}

@@ -1251,6 +1251,10 @@ Usage:
   qc                      open the chat client (signs you in first if needed)
   qc fonts install|status|remove   our OpenEmoji colour font as the terminal's emoji font
   qc agent join <link>    join as an AI agent from an invite (makes its own keys) [--name --username]
+  qc agent call --brain "<cmd>" [--voice orion] [--post] [--no-greet]
+                          as an agent, join voice calls in your chats: hear people, ask <cmd> what
+                          to say (JSON on stdin; it prints PASS or SAY: text), speak it
+                          (needs: npm i -g @livekit/rtc-node)
   qc login [--oob]        sign in through your browser; --oob to paste a code (SSH)
   qc logout               forget this terminal's session and keys
   qc update [--check]     update qc to the latest release (--check only says whether there is one)
@@ -1292,7 +1296,7 @@ function parseArgs2(argv) {
       const [k, v] = a.slice(2).split("=", 2);
       if (v !== undefined)
         args.flags[k] = v;
-      else if (["url", "n", "limit", "reply", "emoji", "pronouns", "website", "bio", "name", "username"].includes(k) && argv[i + 1] !== undefined && !argv[i + 1].startsWith("--"))
+      else if (["url", "n", "limit", "reply", "emoji", "pronouns", "website", "bio", "name", "username", "brain", "voice"].includes(k) && argv[i + 1] !== undefined && !argv[i + 1].startsWith("--"))
         args.flags[k] = argv[++i];
       else
         args.flags[k] = true;
@@ -1364,8 +1368,18 @@ async function main2(argv, { version = "0.0.0" } = {}) {
       return runTui(c, { initialChat: rest[0] });
     }
     case "agent": {
+      if (rest[0] === "call") {
+        const { runVoiceAgent } = await import("./chunk-s094t64g.js");
+        return runVoiceAgent(await client(flags), {
+          brain: typeof flags.brain === "string" ? flags.brain : undefined,
+          voice: typeof flags.voice === "string" ? flags.voice : undefined,
+          post: Boolean(flags.post),
+          greet: !flags["no-greet"],
+          hearAgents: Boolean(flags["hear-agents"])
+        });
+      }
       if (rest[0] !== "join" || !rest[1])
-        throw new Error("Usage: qc agent join <invite link> [--name NAME] [--username NAME]");
+        throw new Error('Usage: qc agent join <invite link> [--name NAME] [--username NAME]  |  qc agent call --brain "<cmd>"');
       const { session, conversationId, operator } = await joinAsAgent(rest[1], {
         name: typeof flags.name === "string" ? flags.name : undefined,
         username: typeof flags.username === "string" ? flags.username : undefined
@@ -1623,4 +1637,4 @@ async function voiceCommand(argv, { out, json }) {
   }
 }
 
-export { exitWhenFlushed2, errorLine2, HELP2, parseArgs2, findChat2, main2 };
+export { CALL_TYPE, parseCall, exitWhenFlushed2, errorLine2, HELP2, parseArgs2, findChat2, main2 };

package/dist/chunk-s094t64g.js

@@ -0,0 +1,440 @@
+import {
+  CALL_TYPE,
+  parseCall
+} from "./chunk-e39k2cyf.js";
+
+// src/cli/voice-agent.js
+import { spawn } from "node:child_process";
+
+// src/lib/chat/voice-recorder.js
+function encodeWav(samples, sampleRate) {
+  const buf = new ArrayBuffer(44 + samples.length * 2);
+  const v = new DataView(buf);
+  const str = (o, t) => [...t].forEach((c, i) => v.setUint8(o + i, c.charCodeAt(0)));
+  str(0, "RIFF");
+  v.setUint32(4, 36 + samples.length * 2, true);
+  str(8, "WAVE");
+  str(12, "fmt ");
+  v.setUint32(16, 16, true);
+  v.setUint16(20, 1, true);
+  v.setUint16(22, 1, true);
+  v.setUint32(24, sampleRate, true);
+  v.setUint32(28, sampleRate * 2, true);
+  v.setUint16(32, 2, true);
+  v.setUint16(34, 16, true);
+  str(36, "data");
+  v.setUint32(40, samples.length * 2, true);
+  for (let i = 0;i < samples.length; i++) {
+    const x = Math.max(-1, Math.min(1, samples[i]));
+    v.setInt16(44 + i * 2, x < 0 ? x * 32768 : x * 32767, true);
+  }
+  return new Uint8Array(buf);
+}
+function createEndpointer({ stepMs = 50, silenceMs = 1100, noSpeechMs = 8000, maxMs = 60000, minLevel = 0.012, maxThreshold = 0.05, speechMs = 150 } = {}) {
+  let t = 0;
+  let floor = null;
+  let loud = 0;
+  let quietFor = 0;
+  let spoke = false;
+  return {
+    get spoke() {
+      return spoke;
+    },
+    push(rms) {
+      t += stepMs;
+      if (t <= 300) {
+        floor = floor === null ? rms : Math.min(floor, rms);
+        return null;
+      }
+      const threshold = Math.max(minLevel, Math.min(maxThreshold, (floor ?? 0) * 3));
+      if (rms > threshold) {
+        loud += 1;
+        quietFor = 0;
+        if (!spoke && loud * stepMs >= speechMs) {
+          spoke = true;
+          return "speech-start";
+        }
+      } else {
+        loud = 0;
+        quietFor += stepMs;
+        if (!spoke)
+          floor = floor === null ? rms : floor * 0.95 + rms * 0.05;
+      }
+      if (spoke && quietFor >= silenceMs)
+        return "end";
+      if (!spoke && t >= noSpeechMs)
+        return "no-speech";
+      if (t >= maxMs)
+        return "max";
+      return null;
+    }
+  };
+}
+
+// src/lib/chat/voice-mode.js
+var DICTATED = "\uD83D\uDDE3️";
+var hostOf = (url) => {
+  try {
+    return new URL(url).hostname.replace(/^www\./, "");
+  } catch {
+    return "a link";
+  }
+};
+function speakable(text, max = 1200) {
+  let s = String(text ?? "");
+  if (s.startsWith(DICTATED))
+    s = s.slice(DICTATED.length);
+  s = s.replace(/```[\s\S]*?(```|$)/g, " (code in the chat) ").replace(/`([^`]+)`/g, "$1").replace(/!\[([^\]]*)\]\([^)]*\)/g, "$1").replace(/\[([^\]]+)\]\((https?:[^)\s]+)\)/g, "$1").replace(/https?:\/\/[^\s)>\]]+/g, (u) => `link to ${hostOf(u)}`).replace(/^\s{0,3}#{1,6}\s+/gm, "").replace(/^\s*(?:[-*+]|\d+[.)])\s+/gm, "").replace(/^\s*>\s?/gm, "").replace(/^\s*\|?(?:\s*:?-{3,}:?\s*\|)+\s*$/gm, "").replace(/\|/g, ", ").replace(/(\*\*|__|~~)(.+?)\1/g, "$2").replace(/(^|\W)[*_](\S[^*_]*?)[*_](?=\W|$)/g, "$1$2").replace(/\s*\n+\s*/g, ". ").replace(/([.!?:;,])(?:\s*\.)+/g, "$1").replace(/\s{2,}/g, " ").trim();
+  if (s.length <= max)
+    return s;
+  const cut = s.slice(0, max);
+  const end = Math.max(cut.lastIndexOf(". "), cut.lastIndexOf("! "), cut.lastIndexOf("? "));
+  return `${end > max / 3 ? cut.slice(0, end + 1) : cut.replace(/\s+\S*$/, "…")} The rest is in the chat.`;
+}
+
+// src/cli/voice-agent.js
+var HKDF = 1;
+var e2eeOptions = (key) => ({
+  keyProviderOptions: {
+    sharedKey: new Uint8Array(key),
+    ratchetSalt: new TextEncoder().encode("LKFrameEncryptionKey"),
+    ratchetWindowSize: 16,
+    failureTolerance: -1,
+    keyRingSize: 16,
+    keyDerivationFunction: HKDF
+  }
+});
+var JOIN_RECENT_MS = 30 * 60 * 1000;
+var ALONE_LEAVE_MS = 30 * 1000;
+var MAX_CALL_MS = 3 * 60 * 60 * 1000;
+var RATE = 16000;
+var SPEAK_RATE = 24000;
+var SPOKEN_PREFIX = "\uD83D\uDD0A";
+var userIdOf = (identity) => String(identity || "").split(":").pop();
+function splitSentences(text, max = 300) {
+  const out = [];
+  let cur = "";
+  for (const s of String(text).match(/[^.!?]+[.!?]+["')\]]*\s*|[^.!?]+$/g) || []) {
+    if (cur && (cur + s).length > max) {
+      out.push(cur.trim());
+      cur = "";
+    }
+    cur += s;
+  }
+  if (cur.trim())
+    out.push(cur.trim());
+  return out;
+}
+function parseBrain(out) {
+  const s = String(out || "");
+  const i = s.search(/^\s*SAY:/m);
+  return i < 0 ? "" : s.slice(i).replace(/^\s*SAY:\s*/, "").trim();
+}
+function wavSamples(bytes) {
+  const b = bytes instanceof Uint8Array ? bytes : new Uint8Array(bytes);
+  const v = new DataView(b.buffer, b.byteOffset, b.byteLength);
+  let o = 12;
+  while (o + 8 <= b.length) {
+    const id = String.fromCharCode(b[o], b[o + 1], b[o + 2], b[o + 3]);
+    let size = v.getUint32(o + 4, true);
+    if (id === "data") {
+      if (!size || size === 4294967295 || o + 8 + size > b.length)
+        size = b.length - o - 8;
+      const n = Math.floor(size / 2);
+      const out = new Int16Array(n);
+      for (let i = 0;i < n; i++)
+        out[i] = v.getInt16(o + 8 + i * 2, true);
+      return out;
+    }
+    o += 8 + size + size % 2;
+  }
+  return new Int16Array(0);
+}
+var rmsOf = (int16) => {
+  let s = 0;
+  for (let i = 0;i < int16.length; i++)
+    s += (int16[i] / 32768) ** 2;
+  return Math.sqrt(s / Math.max(1, int16.length));
+};
+function askBrain(cmd, input, { timeoutMs = 90000, env = process.env } = {}) {
+  return new Promise((resolve) => {
+    const child = spawn(cmd, { shell: true, env, stdio: ["pipe", "pipe", "pipe"] });
+    let out = "";
+    let err = "";
+    const timer = setTimeout(() => child.kill("SIGKILL"), timeoutMs);
+    child.stdout.on("data", (d) => out += d);
+    child.stderr.on("data", (d) => err += d);
+    child.on("error", (e) => {
+      clearTimeout(timer);
+      resolve({ say: "", error: e.message });
+    });
+    child.on("close", (code) => {
+      clearTimeout(timer);
+      resolve({ say: parseBrain(out), error: code ? `brain exited ${code}: ${err.trim().slice(-200)}` : "" });
+    });
+    child.stdin.end(JSON.stringify(input));
+  });
+}
+
+class CallSession {
+  constructor({ client, rtc, conversation, call, brain, voice, post, log, isHuman }) {
+    Object.assign(this, { client, rtc, conversation, call, brain, voice, post, log, isHuman });
+    this.transcript = [];
+    this.queue = Promise.resolve();
+    this.speakToken = 0;
+    this.speaking = false;
+    this.closed = false;
+    this.started = Date.now();
+    this.lastHuman = Date.now();
+  }
+  get id() {
+    return this.conversation.id;
+  }
+  async join({ greet = true } = {}) {
+    const { Room, RoomEvent, AudioSource, LocalAudioTrack, TrackPublishOptions, TrackSource, TrackKind } = this.rtc;
+    const { token, url } = await this.client.request("/api/calls/token", { method: "POST", body: { conversationId: this.id } });
+    const room = new Room;
+    this.room = room;
+    room.on(RoomEvent.TrackSubscribed, (track, _pub, participant) => {
+      if (track.kind === TrackKind.KIND_AUDIO && this.isHuman(userIdOf(participant.identity)))
+        this.listen(track, participant);
+    });
+    room.on(RoomEvent.Disconnected, () => this.close("disconnected"));
+    await room.connect(url, token, { autoSubscribe: true, e2ee: e2eeOptions(this.call.key) });
+    this.source = new AudioSource(SPEAK_RATE, 1);
+    const track = LocalAudioTrack.createAudioTrack(`${this.client.me?.username || "agent"}-voice`, this.source);
+    const opts = new TrackPublishOptions;
+    opts.source = TrackSource.SOURCE_MICROPHONE;
+    await room.localParticipant.publishTrack(track, opts);
+    this.log(`joined the call in ${this.conversation.title}`);
+    this.watchdog = setInterval(() => this.checkAlone(), 5000);
+    if (greet)
+      this.say(`${this.client.me?.display_name || this.client.me?.username || "Agent"} here.`);
+  }
+  humansPresent() {
+    return [...this.room?.remoteParticipants?.values() ?? []].some((p) => this.isHuman(userIdOf(p.identity)));
+  }
+  checkAlone() {
+    if (this.humansPresent())
+      this.lastHuman = Date.now();
+    if (Date.now() - this.lastHuman > ALONE_LEAVE_MS)
+      return this.close("everyone left");
+    if (Date.now() - this.started > MAX_CALL_MS)
+      return this.close("call too long");
+  }
+  async listen(track, participant) {
+    const { AudioStream } = this.rtc;
+    const name = participant.name || "someone";
+    const speakerId = userIdOf(participant.identity);
+    const stream = new AudioStream(track, { sampleRate: RATE, numChannels: 1 });
+    let ep = createEndpointer({ stepMs: 10, silenceMs: 900, noSpeechMs: Infinity, maxMs: 30000 });
+    let frames = [];
+    try {
+      for await (const frame of stream) {
+        if (this.closed)
+          break;
+        const pcm = Int16Array.from(frame.data);
+        frames.push(pcm);
+        const ev = ep.push(rmsOf(pcm));
+        if (!ep.spoke && frames.length > 50)
+          frames = frames.slice(-50);
+        if (ev === "speech-start")
+          this.bargeIn();
+        if (ev === "end" || ev === "max") {
+          const all = new Float32Array(frames.reduce((n, f) => n + f.length, 0));
+          let o = 0;
+          for (const f of frames)
+            for (let i = 0;i < f.length; i++)
+              all[o++] = f[i] / 32768;
+          frames = [];
+          ep = createEndpointer({ stepMs: 10, silenceMs: 900, noSpeechMs: Infinity, maxMs: 30000 });
+          this.enqueue(() => this.heard(all, name, speakerId));
+        }
+      }
+    } catch (e) {
+      if (!this.closed)
+        this.log(`listening to ${name} stopped: ${e.message}`);
+    }
+  }
+  enqueue(job) {
+    this.queue = this.queue.then(job).catch((e) => this.log(`error: ${e.message}`));
+    return this.queue;
+  }
+  async heard(samples, name, speakerId) {
+    const wav = encodeWav(samples, RATE);
+    const res = await this.client.fetch(`${this.client.base}/api/voice/transcribe`, {
+      method: "POST",
+      headers: { Authorization: `Bearer ${await this.client.token()}`, "Content-Type": "audio/wav", "X-Voice-Lang": "en", "X-Voice-Hint": encodeURIComponent(this.hints().join(",")) },
+      body: wav
+    });
+    const data = await res.json().catch(() => ({}));
+    if (!res.ok)
+      throw new Error(`transcribe ${res.status}: ${data.error || ""}`);
+    const text = String(data.text || "").trim();
+    if (!text)
+      return;
+    this.log(`${name}: ${text}`);
+    this.transcript.push({ who: name, text });
+    const { say, error } = await askBrain(this.brain, {
+      agent: this.client.me?.username,
+      conversationId: this.id,
+      title: this.conversation.title,
+      speaker: name,
+      speakerId,
+      text,
+      at: new Date().toISOString(),
+      transcript: this.transcript.slice(-30)
+    });
+    if (error)
+      this.log(error);
+    if (!say)
+      return;
+    this.transcript.push({ who: `${this.client.me?.username} (you)`, text: say });
+    if (this.post)
+      this.client.send(this.id, `${SPOKEN_PREFIX} ${say}`).catch((e) => this.log(`post failed: ${e.message}`));
+    await this.say(say);
+  }
+  hints() {
+    return [...new Set((this.conversation.participants || []).flatMap((p) => [p.user?.username, p.user?.display_name]).filter(Boolean))];
+  }
+  async say(text) {
+    const clean = speakable(text, 1500);
+    if (!clean || !this.source)
+      return;
+    const token = ++this.speakToken;
+    const sentences = splitSentences(clean, 120);
+    let next = this.tts(sentences[0]);
+    this.speaking = true;
+    try {
+      for (let i = 0;i < sentences.length; i++) {
+        const pcm = await next;
+        if (i + 1 < sentences.length)
+          next = this.tts(sentences[i + 1]);
+        if (token !== this.speakToken || this.closed)
+          return;
+        const { AudioFrame } = this.rtc;
+        const step = SPEAK_RATE / 100;
+        for (let o = 0;o + step <= pcm.length; o += step) {
+          if (token !== this.speakToken || this.closed)
+            return;
+          await this.source.captureFrame(new AudioFrame(pcm.slice(o, o + step), SPEAK_RATE, 1, step));
+        }
+      }
+      await this.source.waitForPlayout?.();
+    } finally {
+      if (token === this.speakToken)
+        this.speaking = false;
+    }
+  }
+  async tts(sentence) {
+    const res = await this.client.fetch(`${this.client.base}/api/voice/speak`, {
+      method: "POST",
+      headers: { Authorization: `Bearer ${await this.client.token()}`, "Content-Type": "application/json" },
+      body: JSON.stringify({ text: sentence, voice: this.voice })
+    });
+    if (!res.ok)
+      throw new Error(`speak ${res.status}: ${(await res.json().catch(() => ({}))).error || ""}`);
+    return wavSamples(new Uint8Array(await res.arrayBuffer()));
+  }
+  bargeIn() {
+    if (!this.speaking)
+      return;
+    this.speakToken++;
+    this.speaking = false;
+    this.source?.clearQueue();
+  }
+  async close(why) {
+    if (this.closed)
+      return;
+    this.closed = true;
+    clearInterval(this.watchdog);
+    this.speakToken++;
+    this.log(`left the call in ${this.conversation.title} (${why})`);
+    await this.room?.disconnect().catch(() => {});
+    this.onClose?.();
+  }
+}
+async function loadRtc() {
+  try {
+    const name = "@livekit/rtc-node";
+    return await import(name);
+  } catch {
+    throw new Error("qc agent call needs @livekit/rtc-node: npm i -g @livekit/rtc-node (or install it next to qc).");
+  }
+}
+async function latestCall(client, conversationId) {
+  const { messages = [] } = await client.request("/api/messages/load", { method: "POST", body: { conversationId, limit: 20 } });
+  for (const m of [...messages].reverse()) {
+    if (m.message_type !== CALL_TYPE || !m.encrypted_content)
+      continue;
+    const call = parseCall(await client.ring.decrypt(m.encrypted_content));
+    if (call)
+      return { ...call, messageId: m.id, at: call.at || m.created_at, senderId: m.sender_id };
+  }
+  return null;
+}
+async function runVoiceAgent(client, { brain, voice = "orion", post = false, greet = true, hearAgents = false, log = (s) => process.stderr.write(`qc agent call: ${s}
+`), rtc, signal } = {}) {
+  if (!brain)
+    throw new Error('Usage: qc agent call --brain "<command>" [--voice orion] [--post] [--no-greet]');
+  rtc ??= await loadRtc();
+  const sessions = new Map;
+  let chats = new Map;
+  const loadChats = async () => {
+    chats = new Map((await client.conversations()).map((c) => [c.id, c]));
+    return chats;
+  };
+  const agentIds = () => new Set([...chats.values()].flatMap((c) => (c.participants || []).filter((p) => p.user?.account_type === "agent").map((p) => p.user_id ?? p.user?.id)));
+  const isHuman = (userId) => userId !== client.me?.id && (hearAgents || !agentIds().has(userId));
+  const join = async (conversation, call) => {
+    if (sessions.has(conversation.id) || Date.now() - Date.parse(call.at) > MAX_CALL_MS)
+      return;
+    const session = new CallSession({ client, rtc, conversation, call, brain, voice, post, log, isHuman });
+    sessions.set(conversation.id, session);
+    session.onClose = () => sessions.delete(conversation.id);
+    try {
+      await session.join({ greet });
+    } catch (e) {
+      log(`could not join the call in ${conversation.title}: ${e.message}`);
+      await session.close("join failed");
+    }
+  };
+  await loadChats();
+  for (const c of chats.values()) {
+    const call = await latestCall(client, c.id).catch(() => null);
+    if (call && Date.now() - Date.parse(call.at) < JOIN_RECENT_MS)
+      await join(c, call);
+  }
+  log(`listening for calls in ${chats.size} chat(s)`);
+  await client.events(async ({ type, data }) => {
+    const m = data?.message;
+    if (type !== "NEW_MESSAGE" || !m?.conversation_id)
+      return;
+    if (!chats.has(m.conversation_id))
+      await loadChats();
+    const conversation = chats.get(m.conversation_id);
+    if (!conversation)
+      return;
+    if (m.message_type === CALL_TYPE) {
+      const call = await latestCall(client, conversation.id);
+      if (call && call.messageId === m.id) {
+        await sessions.get(conversation.id)?.close("a new call started");
+        await join(conversation, call);
+      }
+      return;
+    }
+    const session = sessions.get(conversation.id);
+    if (session && m.sender_id === client.me?.id && (m.message_type ?? "text") === "text") {
+      const { messages = [] } = await client.request("/api/messages/load", { method: "POST", body: { conversationId: conversation.id, limit: 5 } });
+      const raw = messages.find((x) => x.id === m.id);
+      const text = raw?.encrypted_content ? await client.ring.decrypt(raw.encrypted_content) : "";
+      if (text && !text.startsWith(SPOKEN_PREFIX) && !text.startsWith("\uD83D\uDDE3️"))
+        session.enqueue(() => session.say(text));
+    }
+  }, { signal, onStatus: (s) => log(`events ${s}`) });
+  for (const s of sessions.values())
+    await s.close("stopped");
+}
+export {
+  runVoiceAgent
+};

package/dist/commands.js

@@ -5,7 +5,7 @@ import {
   parseArgs2,
   findChat2,
   main2
-} from "./chunk-wtbsdxe7.js";
+} from "./chunk-e39k2cyf.js";
 import"./chunk-8gpysrd6.js";
 export {
   HELP2 as HELP,

package/dist/mcp.js

@@ -1,7 +1,7 @@
 import"./chunk-8gpysrd6.js";
 import {
   findChat2
-} from "./chunk-wtbsdxe7.js";
+} from "./chunk-e39k2cyf.js";
 
 // src/cli/mcp.js
 import { createInterface } from "node:readline";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@profullstack/qryptchat",
-  "version": "0.8.1",
+  "version": "0.9.0",
   "description": "qc: qrypt.chat in your terminal. A full-screen end-to-end encrypted chat client (ML-KEM-1024), plus a scriptable CLI and an MCP server.",
   "type": "module",
   "bin": {
@@ -38,5 +38,13 @@
     "directory": "packages/qryptchat"
   },
   "license": "MIT",
-  "author": "Profullstack, Inc."
+  "author": "Profullstack, Inc.",
+  "peerDependencies": {
+    "@livekit/rtc-node": ">=1.2.1"
+  },
+  "peerDependenciesMeta": {
+    "@livekit/rtc-node": {
+      "optional": true
+    }
+  }
 }