mac-voice-mcp 0.1.0

package/dist/texts.js

@@ -0,0 +1,90 @@
+/**
+ * Everything the model reads: speaking rules, server instructions, tool
+ * descriptions and prompts. The rules are shared so every client sees the same
+ * guidance through whichever channel it actually surfaces (Claude Desktop, for
+ * instance, ignores server instructions but always shows tool descriptions).
+ */
+export const SPEECH_RULES = [
+    "Write text_to_speak for the ear, not the screen:",
+    "- 1–3 short sentences, under ~40 words; lead with the outcome, then one question.",
+    "- Plain words only: no markdown, bullets, emoji, code, file paths, URLs, stack traces or tables.",
+    '- Describe code instead of reading it ("I added a retry to the upload function"), say file names',
+    '  not paths ("in index.ts"), round numbers ("about two hundred ms"), spell out symbols.',
+    '- Ask one question at a time, answerable in a few words ("Should I deploy it — yes or no?").',
+    "- Put the details (diffs, logs, links) in your normal on-screen reply, and say so out loud.",
+].join("\n");
+/** Claude Code reads these (truncated at 2 KB) — keep under that. */
+export const SERVER_INSTRUCTIONS = [
+    "voice-mcp speaks to the user through their computer's speakers and returns a local transcript of",
+    "their spoken reply (tool: speak_and_listen). Use it when the user asked to be kept in the loop by",
+    "voice, is in a voice/hands-free session, or you need a quick decision while they may be away from",
+    "the keyboard. Do not use it for routine output the user can read on screen.",
+    "",
+    SPEECH_RULES,
+    "",
+    "Handling replies:",
+    "- The result is a speech-to-text transcript: expect homophones and mangled jargon; interpret",
+    "  generously, and confirm by voice before anything destructive or irreversible.",
+    '- "(No speech detected …)" means no answer: never treat silence as consent. Ask once more or',
+    "  continue with safe work and report on screen.",
+    "- Once the user is talking with you by voice, stay in voice: answer every turn with speak_and_listen,",
+    "  not a text reply, until they say stop or start typing.",
+    "- Listening ends on its own when the user stops talking; no need to set listen_seconds.",
+    '- If the result includes a "voice-mcp note", follow it.',
+    "",
+    "Setup: if speak_and_listen says voice-mcp is not set up, call voice_setup (check only), tell the user",
+    "what is missing, and call voice_setup with install=true only after they agree.",
+].join("\n");
+export const SPEAK_TOOL_DESCRIPTION = [
+    "Say something out loud to the user and hear their spoken reply.",
+    "Speaks text_to_speak through the computer's speakers, then listens like a conversation turn — it waits for",
+    "the user to start talking and stops when they finish — and returns an on-device transcript of what they said.",
+    "",
+    SPEECH_RULES,
+    "",
+    'Good: "The build passed and all tests are green. Want me to open the pull request?"',
+    'Bad: "## Results\\n- `npm test` ✅ 42/42\\n- see /Users/x/repo/src/index.ts:120"',
+    "",
+    "Once the user is talking with you by voice, keep the conversation in voice: answer each transcript with",
+    "another speak_and_listen call (not a text reply) until they say stop or start typing.",
+    'A reply of "(No speech detected …)" means the user did not answer — never treat it as consent.',
+    "If it reports that voice-mcp is not set up, call voice_setup.",
+].join("\n");
+export const SETUP_TOOL_DESCRIPTION = [
+    "Check whether this computer has everything speak_and_listen needs — text-to-speech, a microphone recorder (SoX),",
+    "whisper.cpp speech-to-text and the speech model — and optionally install what's missing.",
+    "Anything already installed is reused (an existing model elsewhere on disk is symlinked, not re-downloaded).",
+    "Call it with install=false (the default) first and tell the user what is missing.",
+    "Only call it with install=true after the user agrees: it runs `brew install` for the missing packages",
+    "and downloads the speech model once. It never uninstalls or changes anything else.",
+    'If it reports INSTALLING, the install continues in the background — wait a minute and call it again to check.',
+].join("\n");
+export const SETUP_PROMPT = [
+    "Set up voice-mcp on this computer.",
+    "1. Call voice_setup (check only) and give me a short summary of what's already installed and what's missing.",
+    "2. If something is missing, ask me before installing. Only if I agree, call voice_setup with install=true.",
+    "   If it reports INSTALLING, wait a minute and check again until it's READY.",
+    "3. When it reports READY, test it: call speak_and_listen with a one-sentence greeting that asks me to say something back,",
+    "   then tell me what you heard. If the result says the microphone returned silence, walk me through",
+    "   allowing microphone access in System Settings → Privacy & Security → Microphone.",
+].join("\n");
+export const VOICE_MODE_PROMPT = (task) => [
+    "Let's work in voice mode. I may be away from the screen, so keep me in the loop through the",
+    "speak_and_listen tool instead of waiting for me to type.",
+    "",
+    "- This is a spoken conversation. Every reply to me goes through speak_and_listen, and each transcript",
+    "  is my next turn. Don't fall back to text replies until I say \"stop voice mode\" or \"I'm back\".",
+    "- While you work on something, say briefly what you're about to do, do it, then report back by voice.",
+    "  Don't narrate every small action.",
+    "- If a transcript is unclear, ask again by voice.",
+    "- Confirm by voice before anything destructive, irreversible, or that costs money.",
+    "- If I don't answer, don't assume yes: carry on with safe work or pause, and summarize on screen.",
+    "- Keep writing full details (code, diffs, links) on screen as usual; the voice line is the headline.",
+    '- Stop using voice when I say "stop voice mode", "I\'m back", or start typing again.',
+    "",
+    SPEECH_RULES,
+    "",
+    task
+        ? `Start by briefly telling me out loud what you're about to do for: ${task}. Then ask if I want to change anything.`
+        : "Start by calling speak_and_listen to say voice mode is on and ask what I'd like to work on.",
+].join("\n");

package/dist/voice.js

@@ -0,0 +1,74 @@
+/** The round trip: speak → listen for one turn → transcribe. */
+import { mkdtemp, rm } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { chime, findRecorder, listenForTurn, MIC_PERMISSION_HINT, RECORDER_MISSING, speak } from "./audio.js";
+import { CONFIG, debug, DEFAULT_LISTEN_SECONDS, MAX_LISTEN_SECONDS, MAX_SPEAK_CHARS } from "./config.js";
+import { ensureModel } from "./model.js";
+import { SetupError } from "./proc.js";
+import { prepareSpeech } from "./speech-text.js";
+import { findWhisperCli, findWhisperServer, prewarm, STT_MISSING, transcribe } from "./stt.js";
+/** Everything speak_and_listen needs, checked before saying a word. Never installs anything. */
+async function preflight() {
+    if (!findRecorder())
+        throw new SetupError(RECORDER_MISSING);
+    if (!findWhisperCli() && !findWhisperServer())
+        throw new SetupError(STT_MISSING);
+    return { model: await ensureModel({ download: false }) };
+}
+export async function speakAndListen(textToSpeak, listenSeconds, signal, onPhase) {
+    const seconds = Math.min(MAX_LISTEN_SECONDS, Math.max(1, Number.isFinite(listenSeconds) ? listenSeconds : DEFAULT_LISTEN_SECONDS));
+    const { model } = await preflight();
+    // Load the model into the warm server while we talk, so it's ready when the user finishes.
+    prewarm(model);
+    const speech = prepareSpeech(textToSpeak, { maxWords: CONFIG.maxSpeakWords, maxChars: MAX_SPEAK_CHARS });
+    const notes = [...speech.notes];
+    const tmpDir = await mkdtemp(path.join(os.tmpdir(), "voice-mcp-"));
+    const wav = path.join(tmpDir, "reply.wav");
+    try {
+        const t0 = Date.now();
+        onPhase?.("speaking");
+        await speak(speech.text, signal);
+        await chime("start");
+        onPhase?.("listening");
+        const heard = await listenForTurn(seconds, wav, signal);
+        void chime("stop");
+        const t1 = Date.now();
+        debug("listen:", heard);
+        if (heard.digitalSilence) {
+            return {
+                ok: false,
+                text: `The microphone returned pure digital silence, which usually means microphone access is blocked. ${MIC_PERMISSION_HINT}`,
+                notes,
+            };
+        }
+        const waited = Math.min(CONFIG.startTimeoutSeconds, seconds);
+        const noSpeech = `(No speech detected — the user did not reply within ${waited} seconds.)`;
+        if (heard.reason === "no-speech")
+            return { ok: true, text: noSpeech, notes };
+        onPhase?.("transcribing");
+        const transcript = await transcribe(wav, model, signal);
+        debug(`timing: speak+listen ${t1 - t0} ms, transcribe ${Date.now() - t1} ms`);
+        if (!transcript)
+            return { ok: true, text: noSpeech, notes };
+        if (heard.reason === "max-duration") {
+            const l = heard.levels;
+            const f = (n) => (Number.isFinite(n) ? n.toFixed(0) : "?");
+            notes.push(`voice-mcp note: listening stopped at the ${seconds}-second limit while the user was still talking, ` +
+                "so the reply may be cut off. Ask them to continue if it seems unfinished. " +
+                `(Audio levels, for tuning: background ${f(l.floorDb)} dB, voice ${f(l.speechDb)} dB, ` +
+                `last second ${f(l.recentDb)} ± ${l.recentSpreadDb.toFixed(1)} dB.)`);
+        }
+        return { ok: true, text: transcript, notes };
+    }
+    finally {
+        await rm(tmpDir, { recursive: true, force: true });
+    }
+}
+/** Serialize calls: there is one speaker and one microphone. */
+let queue = Promise.resolve();
+export function exclusive(fn) {
+    const next = queue.then(fn, fn);
+    queue = next.catch(() => { });
+    return next;
+}

package/examples/CLAUDE.md

@@ -0,0 +1,22 @@
+## Voice check-ins (voice-mcp)
+
+I have a `speak_and_listen` tool that talks to me through my Mac's speakers and returns a
+transcript of my spoken reply. Use it when I ask to be kept in the loop by voice, when I start
+voice mode, or when you need a quick decision and I may be away from the keyboard. Don't use it
+for routine output I can read on screen.
+
+When you call it, write `text_to_speak` for the ear, not the screen:
+- 1–3 short sentences, under ~40 words. Lead with the outcome, then ask one question.
+- Plain words only: no markdown, bullets, emoji, code, file paths, URLs, stack traces or tables.
+- Describe code instead of reading it ("I added a retry to the upload function"). Say file names,
+  not paths ("in index.ts"). Round numbers ("about two hundred milliseconds").
+- Ask questions I can answer in a few words ("Should I deploy it — yes or no?").
+- Keep the details (diffs, logs, links) in your normal on-screen reply, and tell me they're there.
+
+Treating my replies:
+- Once I'm talking with you by voice, stay in voice: answer every turn with `speak_and_listen`,
+  not a text reply, until I say stop or start typing.
+- They are speech-to-text transcripts. Expect misheard words and interpret them generously.
+- Confirm by voice before anything destructive, irreversible, or that costs money.
+- "(No speech detected …)" means I didn't answer. Never treat silence as a yes.
+- Listening stops on its own when I finish talking, so leave `listen_seconds` at its default.

package/examples/voice-mcp.mdc

@@ -0,0 +1,27 @@
+---
+description: How to use the voice-mcp speak_and_listen tool with speakable phrasing
+alwaysApply: true
+---
+
+## Voice check-ins (voice-mcp)
+
+I have a `speak_and_listen` tool that talks to me through my Mac's speakers and returns a
+transcript of my spoken reply. Use it when I ask to be kept in the loop by voice, when I start
+voice mode, or when you need a quick decision and I may be away from the keyboard. Don't use it
+for routine output I can read on screen.
+
+When you call it, write `text_to_speak` for the ear, not the screen:
+- 1–3 short sentences, under ~40 words. Lead with the outcome, then ask one question.
+- Plain words only: no markdown, bullets, emoji, code, file paths, URLs, stack traces or tables.
+- Describe code instead of reading it ("I added a retry to the upload function"). Say file names,
+  not paths ("in index.ts"). Round numbers ("about two hundred milliseconds").
+- Ask questions I can answer in a few words ("Should I deploy it — yes or no?").
+- Keep the details (diffs, logs, links) in your normal on-screen reply, and tell me they're there.
+
+Treating my replies:
+- Once I'm talking with you by voice, stay in voice: answer every turn with `speak_and_listen`,
+  not a text reply, until I say stop or start typing.
+- They are speech-to-text transcripts. Expect misheard words and interpret them generously.
+- Confirm by voice before anything destructive, irreversible, or that costs money.
+- "(No speech detected …)" means I didn't answer. Never treat silence as a yes.
+- Listening stops on its own when I finish talking, so leave `listen_seconds` at its default.

package/package.json

@@ -0,0 +1,67 @@
+{
+  "name": "mac-voice-mcp",
+  "mcpName": "io.github.jeet0007/mac-voice-mcp",
+  "version": "0.1.0",
+  "description": "Talk with Claude out loud on your Mac: speaks with macOS `say`, listens for one natural conversational turn, and transcribes on-device with whisper.cpp. An MCP server.",
+  "type": "module",
+  "bin": {
+    "mac-voice-mcp": "dist/index.js",
+    "voice-mcp": "dist/index.js"
+  },
+  "main": "dist/index.js",
+  "files": [
+    "dist",
+    "examples",
+    "README.md",
+    "LICENSE"
+  ],
+  "scripts": {
+    "build": "tsc && node -e \"require('fs').chmodSync('dist/index.js', 0o755)\"",
+    "dev": "tsc --watch",
+    "start": "node dist/index.js",
+    "test": "npm run build && node --test test/*.test.mjs",
+    "setup": "node dist/index.js setup",
+    "test:voice": "node dist/index.js test",
+    "inspect": "npx -y @modelcontextprotocol/inspector node dist/index.js",
+    "prepare": "npm run build",
+    "prepublishOnly": "npm test",
+    "audit": "npm audit --omit=dev --audit-level=high && npm audit signatures"
+  },
+  "dependencies": {
+    "@modelcontextprotocol/sdk": "^1.30.0",
+    "zod": "^4.1.0"
+  },
+  "devDependencies": {
+    "@types/node": "^22.0.0",
+    "typescript": "^7.0.2"
+  },
+  "engines": {
+    "node": ">=18.17"
+  },
+  "keywords": [
+    "mcp",
+    "mcp-server",
+    "model-context-protocol",
+    "voice",
+    "speech",
+    "tts",
+    "stt",
+    "whisper",
+    "whisper.cpp",
+    "macos",
+    "apple-silicon",
+    "claude",
+    "claude-code",
+    "vibe-coded"
+  ],
+  "author": "Jeet",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/jeet0007/mac-voice-mcp.git"
+  },
+  "homepage": "https://github.com/jeet0007/mac-voice-mcp#readme",
+  "bugs": {
+    "url": "https://github.com/jeet0007/mac-voice-mcp/issues"
+  }
+}