mac-voice-mcp 0.3.0 → 0.4.0

package/dist/pcm.js

@@ -0,0 +1,49 @@
+/** Small PCM helpers for the Kokoro voice (pure, unit-tested). */
+/** Float samples (-1…1) → 16-bit little-endian PCM, clipped. */
+export function floatToPcm16(samples) {
+    const buf = Buffer.alloc(samples.length * 2);
+    for (let i = 0; i < samples.length; i++) {
+        const s = Math.max(-1, Math.min(1, samples[i]));
+        buf.writeInt16LE(Math.round(s < 0 ? s * 0x8000 : s * 0x7fff), i * 2);
+    }
+    return buf;
+}
+/**
+ * Resample 16-bit mono PCM by linear interpolation (24 kHz → 16 kHz for whisper). Good enough
+ * for speech recognition, which is all it's used for; playback always uses the original rate.
+ */
+export function resamplePcm16(pcm, fromRate, toRate) {
+    const inCount = Math.floor(pcm.length / 2);
+    if (fromRate === toRate || inCount === 0)
+        return Buffer.from(pcm.subarray(0, inCount * 2));
+    const outCount = Math.floor((inCount * toRate) / fromRate);
+    const out = Buffer.alloc(outCount * 2);
+    const step = fromRate / toRate;
+    for (let i = 0; i < outCount; i++) {
+        const x = i * step;
+        const i0 = Math.floor(x);
+        const i1 = Math.min(i0 + 1, inCount - 1);
+        const frac = x - i0;
+        const v = pcm.readInt16LE(i0 * 2) * (1 - frac) + pcm.readInt16LE(i1 * 2) * frac;
+        out.writeInt16LE(Math.round(v), i * 2);
+    }
+    return out;
+}
+/** A 44-byte WAV header for 16-bit mono PCM. */
+export function wavHeaderFor(dataBytes, sampleRate) {
+    const h = Buffer.alloc(44);
+    h.write("RIFF", 0);
+    h.writeUInt32LE(36 + dataBytes, 4);
+    h.write("WAVE", 8);
+    h.write("fmt ", 12);
+    h.writeUInt32LE(16, 16);
+    h.writeUInt16LE(1, 20); // PCM
+    h.writeUInt16LE(1, 22); // mono
+    h.writeUInt32LE(sampleRate, 24);
+    h.writeUInt32LE(sampleRate * 2, 28);
+    h.writeUInt16LE(2, 32);
+    h.writeUInt16LE(16, 34);
+    h.write("data", 36);
+    h.writeUInt32LE(dataBytes, 40);
+    return h;
+}

package/dist/proc.js

@@ -64,6 +64,7 @@ export function run(cmd, args, opts = {}) {
         const child = spawn(cmd, args, {
             stdio: [opts.input !== undefined ? "pipe" : "ignore", "pipe", "pipe"],
             windowsHide: true,
+            cwd: opts.cwd,
             env: opts.env ? { ...process.env, ...opts.env } : process.env,
         });
         activeChildren.add(child);

package/dist/server.js

@@ -3,6 +3,7 @@ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
 import { z } from "zod";
 import { CONFIG, DEFAULT_LISTEN_SECONDS, log, MAX_LISTEN_SECONDS, PKG } from "./config.js";
+import { stopKokoro } from "./kokoro.js";
 import { locateModel } from "./model.js";
 import { activeChildren, SetupError } from "./proc.js";
 import { runSetupFlow } from "./setup.js";
@@ -74,11 +75,16 @@ export function createServer() {
                 .boolean()
                 .optional()
                 .describe("false (default): only check. true: brew install missing packages and download the model. Ask the user first."),
+            kokoro: z
+                .boolean()
+                .optional()
+                .describe("true: install the optional Kokoro voice — a natural-sounding neural voice that runs on this Mac " +
+                "(kokoro-js via npm and a one-time model download, about 1 GB on disk). Ask the user first."),
         },
         annotations: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: true },
-    }, async ({ install }, extra) => {
+    }, async ({ install, kokoro }, extra) => {
         try {
-            const outcome = await exclusive(() => runSetupFlow(install === true, progressReporter(extra)));
+            const outcome = await exclusive(() => runSetupFlow({ install: install === true, kokoro: kokoro === true, onProgress: progressReporter(extra) }));
             return { content: [{ type: "text", text: outcome.report }] };
         }
         catch (err) {
@@ -110,6 +116,7 @@ export async function startServer() {
             return;
         shuttingDown = true;
         stopWhisperServer();
+        stopKokoro();
         for (const child of activeChildren)
             child.kill("SIGTERM");
         try {

package/dist/setup.js

@@ -9,6 +9,7 @@
 import { statSync } from "node:fs";
 import { chooseVoice, describeRecorder, findRecorder, findTts, resetVoiceChoice, VOICE_UPGRADE_HINT } from "./audio.js";
 import { CONFIG, IS_MAC, IS_WIN, log } from "./config.js";
+import { installKokoro, isKokoroInstalled, isKokoroModelPresent, KOKORO_SIZES_MB, kokoroVoiceLabel } from "./kokoro.js";
 import { ensureModel, isModelDownloading, locateModel, modelSource, MODEL_SIZES_MB } from "./model.js";
 import { isExecutable, resetWhichCache, run, sleep, tail, which } from "./proc.js";
 import { describeStt, findWhisperCli, findWhisperServer } from "./stt.js";
@@ -21,6 +22,9 @@ export function findBrew() {
 }
 let brewJob = null;
 let modelJob = null;
+let kokoroJob = null;
+/** The Kokoro install's latest progress line. */
+let kokoroProgress = "";
 /** Results of finished jobs not yet shown to the user. */
 const finishedMessages = [];
 function startJob(label, work) {
@@ -58,6 +62,10 @@ function startModelDownload() {
         return `Downloaded the ${CONFIG.modelName} speech model.`;
     });
 }
+function startKokoroInstall() {
+    kokoroProgress = "starting";
+    return startJob("Installing the Kokoro voice", () => installKokoro((m) => (kokoroProgress = m)));
+}
 const running = (j) => !!j && !j.done;
 const elapsed = (j) => `${Math.round((Date.now() - j.startedAt) / 1000)}s`;
 // --- Checks -------------------------------------------------------------------------------
@@ -68,17 +76,34 @@ async function checkRequirements() {
     checks.push(tts
         ? { label: "Text-to-speech", status: "ok", detail: IS_MAC ? "macOS `say` (built in)" : tts }
         : { label: "Text-to-speech", status: "missing", detail: "no engine found — install espeak-ng (`sudo apt install espeak-ng`)" });
+    const kokoro = kokoroCheck();
     if (tts && IS_MAC) {
         resetVoiceChoice(); // pick up a voice downloaded since the last check
         const choice = await chooseVoice();
-        checks.push(choice.canUpgrade
-            ? { label: "Voice", status: "optional", detail: `${choice.label} — ${VOICE_UPGRADE_HINT}` }
-            : { label: "Voice", status: "ok", detail: choice.label });
+        checks.push(kokoro?.status === "ok"
+            ? { label: "Voice", status: "ok", detail: `${choice.label} (used if the Kokoro voice ever fails)` }
+            : choice.canUpgrade
+                ? { label: "Voice", status: "optional", detail: `${choice.label} — ${VOICE_UPGRADE_HINT}` }
+                : { label: "Voice", status: "ok", detail: choice.label });
     }
+    if (kokoro)
+        checks.push(kokoro);
     const rec = findRecorder();
-    checks.push(rec
-        ? { label: "Recorder", status: "ok", detail: describeRecorder(rec) }
-        : { label: "Recorder", status: brewing ? "installing" : "missing", detail: "SoX is not installed", brew: IS_WIN ? undefined : "sox" });
+    if (!rec) {
+        checks.push({ label: "Recorder", status: brewing ? "installing" : "missing", detail: "SoX is not installed", brew: IS_WIN ? undefined : "sox" });
+    }
+    else if (rec.kind === "ffmpeg" && CONFIG.recorder === "auto") {
+        // Works, but it's the fragile path: recommend SoX, which is what the turn-taking is tuned on.
+        checks.push({
+            label: "Recorder",
+            status: brewing ? "installing" : "optional",
+            detail: `${describeRecorder(rec)}, as a fallback. SoX is recommended: it's more reliable with Bluetooth headsets and other audio devices`,
+            brew: "sox",
+        });
+    }
+    else {
+        checks.push({ label: "Recorder", status: "ok", detail: describeRecorder(rec) });
+    }
     const cli = findWhisperCli();
     const server = findWhisperServer();
     if (cli || server) {
@@ -125,52 +150,91 @@ async function checkRequirements() {
     }
     return checks;
 }
+/** The optional Kokoro voice: shown once it's installed or asked for (VOICE_MCP_TTS=kokoro), or while installing. */
+function kokoroCheck() {
+    const label = "Kokoro voice";
+    if (running(kokoroJob))
+        return { label, status: "installing", detail: `${kokoroProgress} (${elapsed(kokoroJob)} so far)` };
+    if (CONFIG.tts === "say")
+        return null;
+    const installed = isKokoroInstalled();
+    const size = `~${KOKORO_SIZES_MB[CONFIG.kokoroDtype]} MB`;
+    if (installed && isKokoroModelPresent()) {
+        return { label, status: "ok", detail: `${kokoroVoiceLabel()} — the natural on-device voice, used for speaking (VOICE_MCP_TTS=say turns it off)` };
+    }
+    if (installed) {
+        return { label, status: "optional", detail: `installed, but its model isn't downloaded yet (${size}) — until then the built-in voice speaks`, kokoro: true };
+    }
+    if (CONFIG.tts === "kokoro") {
+        return {
+            label,
+            status: "optional",
+            detail: `VOICE_MCP_TTS=kokoro, but it isn't installed — a one-time install (kokoro-js with npm, and a ${size} model; about 1 GB on disk) into ${CONFIG.kokoroDir}. Until then the built-in voice speaks`,
+            kokoro: true,
+        };
+    }
+    return null;
+}
 // --- The flow ------------------------------------------------------------------------------
-/**
- * @param install    true only after the user agreed: brew-install missing formulae, download the model if absent.
- * @param onProgress optional heartbeat while waiting (used for MCP progress notifications).
- */
-export async function runSetupFlow(install, onProgress) {
+export async function runSetupFlow(opts = {}) {
+    const { install = false, onProgress } = opts;
+    resetWhichCache(); // see anything installed since the last check (e.g. `brew install sox` in a terminal)
     let checks = await checkRequirements();
     if (install) {
-        const formulae = [...new Set(checks.filter((c) => c.status === "missing" && c.brew).map((c) => c.brew))];
+        // Missing pieces, plus recommended ones (SoX when only ffmpeg is there): the user agreed to install.
+        const formulae = [...new Set(checks.filter((c) => (c.status === "missing" || c.status === "optional") && c.brew).map((c) => c.brew))];
         const brew = findBrew();
         if (formulae.length && brew && !running(brewJob))
             brewJob = startBrewInstall(brew, formulae);
         if (checks.some((c) => c.model && c.status === "missing") && !running(modelJob))
             modelJob = startModelDownload();
     }
+    if (opts.kokoro && !running(kokoroJob) && !(isKokoroInstalled() && isKokoroModelPresent()))
+        kokoroJob = startKokoroInstall();
     // Wait (bounded) for anything in flight, with a heartbeat.
-    const jobs = [brewJob, modelJob].filter(running);
+    const jobs = [brewJob, modelJob, kokoroJob].filter(running);
     if (jobs.length) {
         const deadline = Date.now() + INSTALL_WAIT_MS;
         while (jobs.some((j) => !j.done) && Date.now() < deadline) {
-            onProgress?.(jobs.filter((j) => !j.done).map((j) => `${j.label} (${elapsed(j)})`).join("; "));
+            onProgress?.(jobs
+                .filter((j) => !j.done)
+                .map((j) => `${j.label} (${j === kokoroJob ? `${kokoroProgress}, ` : ""}${elapsed(j)})`)
+                .join("; "));
             await Promise.race([Promise.all(jobs.map((j) => j.promise)), sleep(5000)]);
         }
         checks = await checkRequirements();
     }
-    const installing = running(brewJob) || running(modelJob);
+    const installing = running(brewJob) || running(modelJob) || running(kokoroJob);
     const ready = checks.every((c) => c.status === "ok" || c.status === "optional");
     const icon = { ok: "✔", missing: "✘", optional: "•", installing: "…" };
     const done = finishedMessages.splice(0);
     const lines = [
         `voice-mcp setup — ${ready ? "READY" : installing ? "INSTALLING" : "NOT READY"}`,
         "",
-        ...checks.map((c) => `${icon[c.status]} ${c.label}: ${c.detail}${c.brew && c.status === "missing" ? ` → brew install ${c.brew}` : ""}`),
+        ...checks.map((c) => `${icon[c.status]} ${c.label}: ${c.detail}${c.brew && c.status !== "ok" && c.status !== "installing" ? ` → brew install ${c.brew}` : ""}`),
         IS_MAC ? "• Microphone: macOS asks for permission the first time speak_and_listen listens — click Allow." : "",
     ];
     if (done.length)
         lines.push("", "What was done:", ...done.map((d) => `- ${d}`));
-    if (install && !done.length && !installing && ready)
+    if ((install || opts.kokoro) && !done.length && !installing && ready)
         lines.push("", "Nothing to install — everything was already present.");
     const toInstall = [...new Set(checks.filter((c) => c.status === "missing" && c.brew).map((c) => c.brew))];
     const needsModel = checks.some((c) => c.model && c.status === "missing");
     lines.push("");
     if (ready) {
         lines.push("Next: everything is in place — speak_and_listen is ready to use.");
+        const recommended = [...new Set(checks.filter((c) => c.status === "optional" && c.brew).map((c) => c.brew))];
+        if (recommended.length && !install && findBrew()) {
+            lines.push(`Recommended: ask the user whether to brew install ${recommended.join(" ")} (see • above). ` +
+                "Only if they agree, call voice_setup with install=true.");
+        }
+        if (checks.some((c) => c.kokoro && c.status === "optional") && !opts.kokoro) {
+            lines.push(`Optional: ask the user whether to install the Kokoro voice (a one-time install, about 1 GB on disk). ` +
+                "Only if they agree, call voice_setup with kokoro=true.");
+        }
         if (checks.some((c) => c.label === "Voice" && c.status === "optional")) {
-            lines.push("Mention the optional voice tip (•) to the user once — it makes the voice sound far more natural. Nothing else to do.");
+            lines.push("Mention the optional voice tip (•) to the user once — it makes the voice sound far more natural. " +
+                "If they'd rather not download a macOS voice, the Kokoro voice (voice_setup with kokoro=true, about 1 GB on disk) is the other option. Nothing else to do.");
         }
     }
     else if (installing) {

package/dist/texts.js

@@ -62,6 +62,8 @@ export const SETUP_TOOL_DESCRIPTION = [
     "Call it with install=false (the default) first and tell the user what is missing.",
     "Only call it with install=true after the user agrees: it runs `brew install` for the missing packages",
     "and downloads the speech model once. It never uninstalls or changes anything else.",
+    "kokoro=true (also only after the user agrees) installs the optional Kokoro voice: more natural than the built-in one,",
+    "runs on this Mac, about 1 GB on disk. Once installed it's used automatically; the built-in voice stays as the fallback.",
     'If it reports INSTALLING, the install continues in the background — wait a minute and call it again to check.',
 ].join("\n");
 export const SETUP_PROMPT = [

package/dist/voice.js

@@ -6,7 +6,7 @@ import { chime, findRecorder, listenForTurn, MIC_PERMISSION_HINT, RECORDER_MISSI
 import { CONFIG, debug, DEFAULT_LISTEN_SECONDS, MAX_LISTEN_SECONDS, MAX_SPEAK_CHARS } from "./config.js";
 import { acquireMicLock, MicBusyError } from "./lock.js";
 import { ensureModel } from "./model.js";
-import { SetupError } from "./proc.js";
+import { resetWhichCache, SetupError } from "./proc.js";
 import { prepareSpeech } from "./speech-text.js";
 import { findWhisperCli, findWhisperServer, prewarm, STT_MISSING, transcribe } from "./stt.js";
 /** Everything speak_and_listen needs, checked before saying a word. Never installs anything. */
@@ -51,11 +51,14 @@ async function turn(textToSpeak, seconds, model, signal, onPhase) {
     try {
         const t0 = Date.now();
         onPhase?.("speaking");
-        await speak(speech.text, signal);
+        const spoken = await speak(speech.text, signal);
         const spoke = Date.now() - t0;
+        const fallback = fallbackNote(spoken);
+        if (fallback)
+            notes.push(fallback);
         if (!model) {
             const text = "(Spoken. The microphone was not opened, because listen was false.)";
-            return { ok: true, text, notes: [...notes, `voice-mcp timing: spoke ${secs(spoke)}`] };
+            return { ok: true, text, notes: [...notes, timingNote({ spokeMs: spoke, spoken })] };
         }
         await chime("start");
         onPhase?.("listening");
@@ -64,11 +67,18 @@ async function turn(textToSpeak, seconds, model, signal, onPhase) {
         void chime("stop");
         const t1 = Date.now();
         debug("listen:", heard);
-        const timing = (transcribedMs) => timingNote({ spokeMs: spoke, listenedMs: t1 - tListen, talkedSeconds: heard.speechSeconds, transcribedMs });
+        const timing = (transcribedMs) => timingNote({ spokeMs: spoke, spoken, listenedMs: t1 - tListen, talkedSeconds: heard.speechSeconds, transcribedMs });
         if (heard.digitalSilence) {
+            resetWhichCache(); // if the user fixes it by installing SoX, the next turn picks it up
+            const viaFfmpeg = findRecorder()?.kind === "ffmpeg";
             return {
                 ok: false,
-                text: `The microphone returned pure digital silence, which usually means microphone access is blocked. ${MIC_PERMISSION_HINT}`,
+                text: viaFfmpeg
+                    ? "The microphone returned pure digital silence. voice-mcp is recording with ffmpeg because SoX isn't installed, " +
+                        "and ffmpeg may be using a silent or wrong input device (for example after connecting Bluetooth headphones). " +
+                        "Suggest `brew install sox` (or call voice_setup), then try again. If that doesn't help: " +
+                        MIC_PERMISSION_HINT
+                    : `The microphone returned pure digital silence, which usually means microphone access is blocked. ${MIC_PERMISSION_HINT}`,
                 notes,
             };
         }
@@ -98,12 +108,23 @@ async function turn(textToSpeak, seconds, model, signal, onPhase) {
     }
 }
 const secs = (ms) => `${(ms / 1000).toFixed(1)} s`;
+/** Kokoro failures already reported in this session: say each once, not on every turn. */
+const reportedFallbacks = new Set();
+function fallbackNote(spoken) {
+    if (!spoken.fallback || reportedFallbacks.has(spoken.fallback))
+        return null;
+    reportedFallbacks.add(spoken.fallback);
+    return (`voice-mcp note: the Kokoro voice didn't work (${spoken.fallback}), so the built-in voice spoke instead. ` +
+        "Mention it to the user once; voice_setup shows what's wrong.");
+}
 /** One compact line per turn, so "why did that feel slow?" has an answer. */
 export function timingNote(t) {
-    const parts = [
-        `spoke ${secs(t.spokeMs)}`,
-        `listened ${secs(t.listenedMs)}` + (t.talkedSeconds > 0 ? ` (user talked ${t.talkedSeconds.toFixed(1)} s)` : " (no speech)"),
-    ];
+    const k = t.spoken?.engine === "kokoro" && t.spoken.firstAudioMs !== undefined ? t.spoken : null;
+    const parts = [`spoke ${secs(t.spokeMs)}` + (k ? ` (Kokoro, first sound after ${secs(k.firstAudioMs)}${k.coldStart ? ", voice loaded" : ""})` : "")];
+    if (t.listenedMs !== undefined) {
+        const talked = t.talkedSeconds ?? 0;
+        parts.push(`listened ${secs(t.listenedMs)}` + (talked > 0 ? ` (user talked ${talked.toFixed(1)} s)` : " (no speech)"));
+    }
     if (t.transcribedMs !== undefined)
         parts.push(`transcribed ${secs(t.transcribedMs)}`);
     return `voice-mcp timing: ${parts.join(" · ")}`;

package/dist/wer.js

@@ -0,0 +1,41 @@
+/**
+ * Word error rate: how far a transcript is from the words that were actually said.
+ * Used by `doctor` and the round-trip tests, so "it heard me right" is a number, not a feeling.
+ */
+const ONES = ["zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten", "eleven", "twelve", "thirteen", "fourteen", "fifteen", "sixteen", "seventeen", "eighteen", "nineteen"];
+const TENS = ["", "", "twenty", "thirty", "forty", "fifty", "sixty", "seventy", "eighty", "ninety"];
+/** 0–999 as words ("42" → "forty two"), so "60" and "sixty" count as the same word. */
+export function numberToWords(n) {
+    if (n < 20)
+        return ONES[n];
+    if (n < 100)
+        return TENS[Math.floor(n / 10)] + (n % 10 ? ` ${ONES[n % 10]}` : "");
+    return `${ONES[Math.floor(n / 100)]} hundred` + (n % 100 ? ` ${numberToWords(n % 100)}` : "");
+}
+/** Lower-case words without punctuation; "index.ts" → "index ts", "60" → "sixty". */
+export function normalizeWords(text) {
+    return text
+        .toLowerCase()
+        .replace(/\[[^\]]*\]|\([^)]*\)/g, " ") // whisper's [BLANK_AUDIO], (music) …
+        .replace(/(\d)[,.](?=\d{3}\b)/g, "$1") // 1,000 → 1000
+        .replace(/[^\p{L}\p{N}']+/gu, " ")
+        .replace(/'/g, "")
+        .split(/\s+/)
+        .filter(Boolean)
+        .flatMap((w) => (/^\d{1,3}$/.test(w) ? numberToWords(Number(w)).split(" ") : [w]));
+}
+/** Word-level edit distance between what was said (reference) and what was heard (hypothesis). */
+export function wordErrorRate(reference, hypothesis) {
+    const ref = normalizeWords(reference);
+    const hyp = normalizeWords(hypothesis);
+    let prev = Array.from({ length: hyp.length + 1 }, (_, j) => j);
+    for (let i = 1; i <= ref.length; i++) {
+        const cur = [i];
+        for (let j = 1; j <= hyp.length; j++) {
+            cur[j] = Math.min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + (ref[i - 1] === hyp[j - 1] ? 0 : 1));
+        }
+        prev = cur;
+    }
+    const errors = prev[hyp.length];
+    return { wer: ref.length ? errors / ref.length : hyp.length ? 1 : 0, errors, words: ref.length };
+}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "mac-voice-mcp",
   "mcpName": "io.github.jeet0007/mac-voice-mcp",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "description": "Talk with Claude out loud on your Mac: speaks with macOS `say`, listens for one natural conversational turn, and transcribes on-device with whisper.cpp. An MCP server.",
   "type": "module",
   "bin": {
@@ -23,6 +23,8 @@
     "test": "npm run build && node --test test/*.test.mjs",
     "setup": "node dist/index.js setup",
     "test:voice": "node dist/index.js test",
+    "doctor": "node dist/index.js doctor",
+    "dev:plugin": "node scripts/dev-plugin.mjs",
     "inspect": "npx -y @modelcontextprotocol/inspector node dist/index.js",
     "prepare": "npm run build",
     "prepublishOnly": "npm test",