friend-assistant 0.1.40 → 0.1.42

package/dist/BUILD.json

@@ -1,5 +1,5 @@
 {
   "name": "friend-assistant",
-  "version": "0.1.40",
-  "builtAt": "2026-10-13T02:23:53.362Z"
+  "version": "0.1.42",
+  "builtAt": "2026-10-13T05:30:33.387Z"
 }

package/dist/friend.mjs

@@ -159,6 +159,14 @@ function resolveVoice(input) {
   if (PLAIN.test(t) && PIPER_NAME.test(t)) return { id: t, label: t, listed: false };
   return null;
 }
+var LISTENING_MODELS = [
+  { id: "tiny", name: "Tiny", mb: 75, note: "fastest, mishears a lot" },
+  { id: "base", name: "Base", mb: 145, note: "fast, struggles with Hindi, Marathi and distance" },
+  { id: "small", name: "Small", mb: 480, note: "recommended: much better, still quick" },
+  { id: "medium", name: "Medium", mb: 1500, note: "better again, slow on a PC without a graphics card" },
+  { id: "large-v3-turbo", name: "Large (turbo)", mb: 1600, note: "most accurate for Marathi; needs a fast PC" }
+];
+var downloadSize = (mb) => `about ${mb >= 1e3 ? `${(mb / 1e3).toFixed(1)} GB` : `${mb} MB`}`;
 
 // ../protocol/src/themes.ts
 var THEMES = [
@@ -3337,7 +3345,7 @@ function sanitizeLook(raw) {
 var DEFAULT_OLLAMA = { url: "http://127.0.0.1:11434", model: "qwen2.5:7b", contextTokens: 8192, vision: false };
 var DEFAULT_VOICE = {
   enabled: true,
-  sttModel: "base",
+  sttModel: "small",
   ttsVoice: "en_US-lessac-medium",
   language: null,
   speakTyped: true,
@@ -11035,7 +11043,8 @@ function createGateway(deps) {
       const session = token ? auth.resolveSession(token) : null;
       if (!session) return reply.code(401).send({ error: "Unauthorized" });
       try {
-        return await deps.onVoiceUtterance(session, body, String(req.headers["content-type"] ?? "audio/webm"), req.ip);
+        const wake = req.query.wake === "1";
+        return await deps.onVoiceUtterance(session, body, String(req.headers["content-type"] ?? "audio/webm"), req.ip, wake);
       } catch (e) {
         if (e instanceof HttpError) return reply.code(e.status).send({ error: e.message });
         audit.log("owner", "voice.utterance_failed", { message: e.message });
@@ -12969,7 +12978,7 @@ function refuseForAgent(raw) {
   if ("command" in cc) throw new SettingsError("The Claude Code command can only be changed by the owner on the settings screen.");
 }
 var PROVIDER_KEYS = /* @__PURE__ */ new Set(["provider", "model", "apiKey", "ollama", "openai", "claudeCode"]);
-var RESTART_VOICE = ["enabled", "sttModel"];
+var RESTART_VOICE = ["enabled"];
 function applySettings(config, raw) {
   const parsed = SettingsPatch.safeParse(raw);
   if (!parsed.success) {
@@ -13761,6 +13770,39 @@ var Enrollment = class {
   }
 };
 
+// ../server/src/voice/wake.ts
+var GREETINGS = /* @__PURE__ */ new Set(["hey", "hay", "hi", "hello", "hellow", "hallo", "hullo", "ok", "okay", "yo", "he", "\u0939\u0947", "\u0939\u093E\u092F", "\u0939\u0945\u0932\u094B", "\u0939\u0947\u0932\u094B", "\u0939\u0948\u0932\u094B", "\u0939\u0932\u094B", "\u0913\u0915\u0947"]);
+var WORD = /[\p{L}\p{M}\p{N}'’]+/gu;
+var GAP = /^[\s,.:;!?…\-–—]+/u;
+function distance(a, b) {
+  const row = Array.from({ length: b.length + 1 }, (_, i) => i);
+  for (let i = 1; i <= a.length; i++) {
+    let prev = row[0];
+    row[0] = i;
+    for (let j = 1; j <= b.length; j++) {
+      const keep = row[j];
+      row[j] = Math.min(row[j] + 1, row[j - 1] + 1, prev + (a[i - 1] === b[j - 1] ? 0 : 1));
+      prev = keep;
+    }
+  }
+  return row[b.length];
+}
+var sameWord = (heard, name) => heard === name || name.length >= 4 && distance(heard, name) <= 1;
+function matchWake(text3, name) {
+  const no = { matched: false, rest: "" };
+  const wanted = [...name.normalize("NFC").toLowerCase().matchAll(WORD)].map((m) => m[0]);
+  if (wanted.length === 0) return no;
+  const words = [...text3.normalize("NFC").matchAll(WORD)].map((m) => ({ low: m[0].toLowerCase(), end: m.index + m[0].length }));
+  let at = 0;
+  if (words[0] && GREETINGS.has(words[0].low)) at = 1;
+  for (let i = 0; i < wanted.length; i++) {
+    const heard = words[at + i];
+    if (!heard || !sameWord(heard.low, wanted[i])) return no;
+  }
+  const last = words[at + wanted.length - 1];
+  return { matched: true, rest: text3.normalize("NFC").slice(last.end).replace(GAP, "").trim() };
+}
+
 // ../server/src/voice/service.ts
 var VoiceError = class extends Error {
 };
@@ -13818,7 +13860,11 @@ var HttpVoiceService = class {
     });
     return Buffer.from(await res.arrayBuffer());
   }
-  async request(path3, init) {
+  /** Loading a model the first time downloads it (up to about 1.6 GB), so this waits far longer than a normal request. */
+  async useListeningModel(name) {
+    await this.request("/stt-model", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ model: name }) }, 30 * 6e4);
+  }
+  async request(path3, init, timeoutMs) {
     const base = this.opts.baseUrl();
     if (!base) throw new VoiceError("the voice sidecar is not running");
     let res;
@@ -13826,7 +13872,7 @@ var HttpVoiceService = class {
       res = await fetch(base + path3, {
         ...init,
         headers: { ...init.headers, authorization: `Bearer ${this.opts.token}` },
-        signal: AbortSignal.timeout(this.opts.timeoutMs ?? 6e4)
+        signal: AbortSignal.timeout(timeoutMs ?? this.opts.timeoutMs ?? 6e4)
       });
     } catch (e) {
       if (e.name === "TimeoutError") throw new VoiceError("the voice sidecar timed out");
@@ -14345,6 +14391,7 @@ async function createApp(opts) {
   function commitSettings(patch, how) {
     let applied;
     let voiceBefore = config.voice.ttsVoice;
+    let sttBefore = config.voice.sttModel;
     try {
       const saved = loadConfig(opts.dataDir);
       const check3 = applySettings(saved, patch);
@@ -14352,6 +14399,7 @@ async function createApp(opts) {
         throw new HttpError(409, "Friend is busy with a request. Change the model when it is done.");
       }
       voiceBefore = config.voice.ttsVoice;
+      sttBefore = config.voice.sttModel;
       applied = applySettings(config, patch);
       saveConfig(saved);
     } catch (e) {
@@ -14377,6 +14425,7 @@ async function createApp(opts) {
       else swap();
     }
     if (applied.changed.includes("voice.ttsVoice")) void warmVoice(config.voice.ttsVoice, voiceBefore);
+    if (applied.changed.includes("voice.sttModel")) void warmListening(config.voice.sttModel, sttBefore);
     if (applied.changed.includes("voice.hindiVoice")) void warmLanguageVoice(config.voice.hindiVoice);
     if (applied.changed.includes("voice.marathiVoice")) void warmLanguageVoice(config.voice.marathiVoice);
     if (applied.changed.some((k) => k === "view" || k === "ownerName" || k.startsWith("look.") || k.startsWith("hud") || k.startsWith("music"))) {
@@ -14403,6 +14452,25 @@ async function createApp(opts) {
       broadcast({ type: "alert", level: "warn", text: `Could not load the voice ${wanted} (it needs internet the first time). Using ${voiceLabel(previous)} again.` });
     }
   }
+  async function warmListening(wanted, previous) {
+    if (!voice2?.useListeningModel || !voiceAvailable()) return;
+    const known = LISTENING_MODELS.find((m) => m.id === wanted);
+    const download = known ? ` (${downloadSize(known.mb)} the first time it is downloaded)` : ". The first time it downloads";
+    broadcast({ type: "alert", level: "info", text: `Getting the listening model ${wanted}${download}. Friend keeps listening with the old one meanwhile.` });
+    try {
+      await voice2.useListeningModel(wanted);
+      broadcast({ type: "alert", level: "info", text: `The listening model ${wanted} is ready.` });
+    } catch (e) {
+      audit.log("system", "voice.model_failed", { model: wanted, message: e.message });
+      if (config.voice.sttModel === wanted) {
+        config.voice.sttModel = previous;
+        const saved = loadConfig(opts.dataDir);
+        saved.voice.sttModel = previous;
+        saveConfig(saved);
+      }
+      broadcast({ type: "alert", level: "warn", text: `Could not load the listening model ${wanted} (it needs internet the first time). Using ${previous} again.` });
+    }
+  }
   async function warmLanguageVoice(wanted) {
     if (!wanted || !voice2 || !voiceAvailable()) return;
     broadcast({ type: "alert", level: "info", text: `Loading the voice ${voiceLabel(wanted)}. The first time it downloads about 64 MB.` });
@@ -14817,7 +14885,7 @@ async function createApp(opts) {
     audit,
     webDir: opts.webDir,
     voiceAvailable,
-    onVoiceUtterance: async (session, audio, mime, ip) => {
+    onVoiceUtterance: async (session, audio, mime, ip, wake = false) => {
       if (!voice2) throw new HttpError(503, "Voice is not available");
       if (agent.isBusy()) throw new HttpError(409, "Friend is busy with the last request. Try again in a moment.");
       const epoch = lockEpoch;
@@ -14835,7 +14903,13 @@ async function createApp(opts) {
         throw e;
       }
       if (epoch !== lockEpoch || !auth.sessionActive(session.id)) throw new HttpError(401, "Session ended");
-      if (!/[\p{L}\p{N}]/u.test(heard)) return { text: "" };
+      if (!/[\p{L}\p{N}]/u.test(heard)) return wake ? { text: "", wake: false } : { text: "" };
+      if (wake) {
+        const m = matchWake(heard, auth.assistantName());
+        if (!m.matched) return { text: "", wake: false };
+        heard = m.rest;
+        if (!/[\p{L}\p{N}]/u.test(heard)) return { text: "", wake: true };
+      }
       const crash = (e) => audit.log("system", "voice.turn_crash", { message: e.message });
       const who = enrolled ? judge(embedding) : { trust: "unverified" };
       const trust = who.trust;
@@ -14857,7 +14931,7 @@ async function createApp(opts) {
         broadcast({ type: "voice.heard", text: heard, trust: trust === "owner" ? "owner" : "unverified" });
         void runVoiceTurn(heard, ip, epoch, "owner", void 0, spoken).catch(crash);
       }
-      return { text: heard };
+      return wake ? { text: heard, wake: true } : { text: heard };
     },
     voiceEnroll: enrollment && voice2 ? {
       state: () => enrollment.state(),
@@ -15635,7 +15709,7 @@ var KEY_HELP = {
   allowedHosts: "extra host names that may reach Friend (for example my-pc), separated by commas",
   trustProxy: "true when Friend sits behind your own reverse proxy",
   "voice.enabled": "true or false: use speech at all",
-  "voice.sttModel": "listening model: tiny, base, small, \u2026 (bigger is more accurate and slower)",
+  "voice.sttModel": "listening model: tiny, base, small, large-v3-turbo \u2026 (bigger is more accurate and slower; Marathi and far-from-mic speech do best with small or large-v3-turbo)",
   "voice.ttsVoice": "speaking voice (easier: friend voice setup, or Settings)",
   "voice.language": "force a spoken language such as en, hi or mr, or empty to detect it (empty is best when you mix languages)",
   "voice.hindiVoice": "the voice that reads Hindi (Devanagari) text: hi_IN-priyamvada-medium, hi_IN-pratham-medium or hi_IN-rohan-medium; empty uses the main voice",
@@ -17804,7 +17878,7 @@ async function daemonCommand(args, io2, deps = realDeps) {
 import crossSpawn5 from "cross-spawn";
 var PACKAGE = "friend-assistant";
 var real = {
-  current: () => true ? "0.1.40" : "development",
+  current: () => true ? "0.1.42" : "development",
   latest: () => new Promise((resolve13) => {
     const child = crossSpawn5("npm", ["view", PACKAGE, "version", "--prefer-online"], { stdio: ["ignore", "pipe", "ignore"] });
     let out = "";
@@ -18092,7 +18166,7 @@ async function run(argv, io2) {
       case "version":
       case "--version":
       case "-v":
-        io2.out(true ? "0.1.40" : "development");
+        io2.out(true ? "0.1.42" : "development");
         return 0;
       case "lan":
         return await lanCommand(rest, io2);

package/dist/voice/friend_voice/__main__.py

@@ -18,7 +18,7 @@ def main() -> int:
     ap.add_argument("--port", type=int, default=0, help="0 picks a free port")
     ap.add_argument("--fake", action="store_true", help="use stub engines (tests)")
     ap.add_argument("--data-dir", default=None, help="folder holding downloaded models")
-    ap.add_argument("--stt-model", default="base")
+    ap.add_argument("--stt-model", default="small")
     ap.add_argument("--tts-voice", default="en_US-lessac-medium")
     args = ap.parse_args()
     token = args.token or os.environ.get("FRIEND_VOICE_TOKEN")
@@ -32,7 +32,13 @@ def main() -> int:
 
         engines = load_engines(args.data_dir, args.stt_model, args.tts_voice)
 
-    server, port = make_server(engines, token, port=args.port)
+    if args.fake:
+        loader = lambda name: FakeEngines().stt  # noqa: E731
+    else:
+        from .models import load_stt
+
+        loader = lambda name: load_stt(args.data_dir, name)  # noqa: E731
+    server, port = make_server(engines, token, port=args.port, load_stt=loader)
     print(json.dumps({"ready": True, "port": port}), flush=True)
 
     # When the parent process goes away its end of our stdin closes: shut down instead of lingering.

package/dist/voice/friend_voice/models.py

@@ -46,7 +46,13 @@ def ensure_speaker_model(directory: Path, url: str = SPEAKER_URL, sha256: str =
     return target
 
 
-def load_engines(data_dir: Optional[str], stt_model: str = "base", tts_voice: str = "en_US-lessac-medium") -> Engines:
+def load_stt(data_dir: Optional[str], name: str) -> WhisperSTT:
+    """One listening model, downloaded into the models folder when it is not there yet."""
+    root = Path(data_dir) if data_dir else Path.home() / ".friend" / "voice" / "models"
+    return WhisperSTT(name, str(root / "whisper"))
+
+
+def load_engines(data_dir: Optional[str], stt_model: str = "small", tts_voice: str = "en_US-lessac-medium") -> Engines:
     root = Path(data_dir) if data_dir else Path.home() / ".friend" / "voice" / "models"
     root.mkdir(parents=True, exist_ok=True)
     speaker = None

package/dist/voice/friend_voice/server.py

@@ -4,20 +4,33 @@ from __future__ import annotations
 
 import hmac
 import json
+import re
+import threading
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
-from typing import Tuple
+from typing import Callable, Optional, Tuple
 from urllib.parse import parse_qs, urlparse
 
 from .engines import Engines
 from .tts import check_voice_name
 
 MAX_TTS_CHARS = 2000
+#: What a listening model is called: tiny, base, small, large-v3-turbo, or a Hugging Face folder such as Systran/faster-whisper-small.
+MODEL_NAME = re.compile(r"^[A-Za-z0-9_./-]{1,80}$")
 
 
 def make_server(
-    engines: Engines, token: str, port: int = 0, max_body: int = 4_000_000, io_timeout: float = 30
+    engines: Engines,
+    token: str,
+    port: int = 0,
+    max_body: int = 4_000_000,
+    io_timeout: float = 30,
+    load_stt: Optional[Callable[[str], object]] = None,
 ) -> Tuple[ThreadingHTTPServer, int]:
-    """Create (but do not start) a server bound to 127.0.0.1. Returns (server, actual_port)."""
+    """Create (but do not start) a server bound to 127.0.0.1. Returns (server, actual_port).
+
+    `load_stt(name)` builds (and, the first time, downloads) a listening model; POST /stt-model swaps it in while running.
+    """
+    swapping = threading.Lock()  # one model change at a time
 
     class Handler(BaseHTTPRequestHandler):
         protocol_version = "HTTP/1.1"
@@ -85,6 +98,8 @@ def make_server(
                 return self._tts()
             if url.path == "/embed":
                 return self._embed()
+            if url.path == "/stt-model":
+                return self._stt_model()
             self._drain(64 * 1024)
             self._json(404, {"error": "not found"}, close=True)
 
@@ -118,6 +133,26 @@ def make_server(
             except Exception as e:  # engine failures are reported, never crash the server
                 self._json(500, {"error": str(e)})
 
+        def _stt_model(self) -> None:
+            raw = self._read_body()
+            if raw is None:
+                return
+            if load_stt is None:
+                return self._json(501, {"error": "this speech engine cannot change its listening model"})
+            try:
+                name = json.loads(raw)["model"]
+            except (ValueError, KeyError, TypeError):
+                return self._json(400, {"error": "expected JSON with a model field"})
+            if not isinstance(name, str) or not MODEL_NAME.match(name) or ".." in name:
+                return self._json(400, {"error": "that is not a listening model name"})
+            with swapping:
+                try:
+                    fresh = load_stt(name)  # may take minutes the first time (download); the old model keeps listening meanwhile
+                except Exception as e:
+                    return self._json(500, {"error": str(e) or "the listening model could not be loaded"})
+                engines.stt = fresh
+            self._json(200, {"ok": True, "model": name})
+
         def _embed(self) -> None:
             audio = self._read_body()
             if audio is None:

package/dist/voice/friend_voice/stt.py

@@ -10,6 +10,17 @@ from typing import Optional
 MAX_SECONDS = 90.0
 SAMPLE_RATE = 16000
 
+#: Speech quieter than this (RMS) is brought up toward it; a speaker far from the microphone arrives faint.
+TARGET_RMS = 0.06
+#: Below this the room is silent, so there is nothing to bring up (it would only make hiss louder).
+SILENCE_RMS = 0.001
+MAX_GAIN = 20.0
+#: A segment is dropped when the model thinks it is probably not speech AND is not sure of its words (Whisper's own rule),
+#: or when it is very unsure of its words whatever the speech chance: noise turns into invented words in other scripts.
+NO_SPEECH_PROB = 0.6
+MIN_LOGPROB = -1.0
+HOPELESS_LOGPROB = -2.0
+
 #: A sentence in the style wanted, given to the recognizer as a hint: Hindi and Marathi are written in Devanagari,
 #: and the English words that people mix in stay in Latin letters.
 HINT_PROMPTS = {
@@ -18,6 +29,29 @@ HINT_PROMPTS = {
 }
 
 
+def level(samples):
+    """Bring faint speech up to a usable volume without clipping; leave loud enough speech and silence alone."""
+    import numpy as np
+
+    if samples.size == 0:
+        return samples
+    loudness = float(np.sqrt(np.mean(np.square(samples))))
+    if loudness >= TARGET_RMS or loudness < SILENCE_RMS:
+        return samples
+    peak = float(np.max(np.abs(samples)))
+    gain = min(TARGET_RMS / loudness, MAX_GAIN, 0.99 / peak if peak > 0 else MAX_GAIN)
+    return (samples * gain).astype(np.float32)
+
+
+def believed(segment) -> bool:
+    """Whether to keep what the model heard in this segment."""
+    no_speech = getattr(segment, "no_speech_prob", 0.0)
+    logprob = getattr(segment, "avg_logprob", 0.0)
+    if no_speech > NO_SPEECH_PROB and logprob < MIN_LOGPROB:
+        return False
+    return logprob >= HOPELESS_LOGPROB
+
+
 def decode_audio(data: bytes, max_seconds: float = MAX_SECONDS):
     """Decode any container PyAV understands (webm/opus, mp4/aac, wav, ...) to mono float32 at 16 kHz.
 
@@ -49,7 +83,7 @@ def decode_audio(data: bytes, max_seconds: float = MAX_SECONDS):
         take(resampler.resample(None))  # flush
     if not pieces:
         return np.zeros(0, dtype=np.float32)
-    return np.concatenate(pieces).astype(np.float32) / 32768.0
+    return level(np.concatenate(pieces).astype(np.float32) / 32768.0)
 
 
 class WhisperSTT:
@@ -64,12 +98,15 @@ class WhisperSTT:
         segments, info = self._model.transcribe(
             samples,
             language=language,
-            beam_size=1,
+            beam_size=5,  # a wider search is slower but much better on faint or accented speech
             vad_filter=True,  # drop silence so quiet clips return an empty transcript
             condition_on_previous_text=False,
+            no_speech_threshold=NO_SPEECH_PROB,
+            log_prob_threshold=MIN_LOGPROB,
+            compression_ratio_threshold=2.4,  # repeated gibberish compresses well: the model tries again instead of keeping it
             **extra,
         )
-        return " ".join(s.text.strip() for s in segments).strip(), info
+        return " ".join(s.text.strip() for s in segments if believed(s)).strip(), info
 
     def transcribe(self, audio: bytes, language: Optional[str]) -> dict:
         samples = decode_audio(audio)  # bounded decode: hostile or huge input is refused here