@bojackduy/opencode-voice 0.1.0

package/index.js

@@ -0,0 +1,124 @@
+// opencode-voice: Speech-to-text and text-to-speech for OpenCode.
+//
+// STT: Record voice via sox, transcribe with whisper-cpp, normalize with
+//      an OpenAI-compatible LLM, append to the TUI prompt.
+//
+// TTS: Auto-speak assistant responses (or read on demand) via Piper,
+//      with LLM normalization for natural speech.
+//
+// Prerequisites:
+//   STT: brew install whisper-cpp sox
+//   TTS: Piper binary on PATH, voice models at ~/.local/share/piper-voices/
+//
+// Configuration via tui.json plugin options:
+//   ["opencode-voice", { "endpoint": "...", "model": "...", "apiKeyEnv": "..." }]
+//
+// Runtime state (model, mic, voice, tts mode) persisted via api.kv.
+//
+// Commands (palette + slash; default shortcuts use rare leader combos to avoid collisions):
+//   /stt-record          - start/stop recording + transcribe  (default: <leader>[ = ctrl+x, [)
+//   /stt-submit          - stop recording + transcribe + submit (palette-only)
+//   /stt-stop            - cancel recording (palette-only)
+//   /stt-model           - select whisper model
+//   /stt-language        - select transcription language
+//   /stt-mic             - select microphone
+//   /tts-speak           - read last response aloud        (default: <leader>] = ctrl+x, ])
+//   /tts-mode            - toggle auto TTS on/off (palette-only)
+//   /tts-stop            - stop playback                   (default: <leader>; , also palette)
+//   /tts-voice           - select TTS voice
+//   /voice-conversation  - toggle hands-free voice conversation (default: <leader>v = ctrl+x, v)
+//   /voice-conversation-stop - exit voice conversation mode (palette-only)
+//   /voice-notes-start   - start continuous live-notes recording (palette-only)
+//   /voice-notes-stop    - stop live notes, flush, and save (palette-only)
+//   /voice-notes-cancel  - stop live notes immediately, save in background (palette-only)
+//   /voice-notes-status  - show live-notes recording status (palette-only)
+//   All also palette-accessible via Ctrl+P or /slash. Override via plugin options `keybinds`:
+//   { "keybinds": { "stt.record": "ctrl+r", "tts.speak-last": "none", "voice.conversation": "none" } }
+//   Weird keys [ ] ; were chosen because opencode doesn't use them and shift variants were ignored in terminals.
+//   <leader>v is free in opencode defaults (c/e/s/m/a/y/u/r/h etc. are taken) and mnemonic for voice.
+
+import fs from "node:fs";
+import os from "node:os";
+import { registerSTT } from "./lib/stt.js";
+import { registerTTS } from "./lib/tts.js";
+import { registerConversation } from "./lib/conversation.js";
+import { registerLiveNotes } from "./lib/live-notes.js";
+import { registerVoiceModel, resolveVoiceProviderModel } from "./lib/voice-model.js";
+import { createClient } from "./lib/llm-client.js";
+import { createLogger } from "./lib/logger.js";
+
+function loadPromptFile(filePath, logger, name) {
+  if (!filePath) return null;
+  const resolved = filePath.replace(/^~(?=\/|$)/, os.homedir());
+  try {
+    const prompt = fs.readFileSync(resolved, "utf-8").trim() || null;
+    logger?.log(
+      "plugin",
+      prompt ? `Loaded ${name} prompt: ${resolved}` : `Ignored empty ${name} prompt: ${resolved}`,
+      "debug",
+    );
+    return prompt;
+  } catch (err) {
+    logger?.log("Plugin", `Failed to load ${name} prompt ${resolved}: ${err.message}`, "warn");
+    return null;
+  }
+}
+
+export default {
+  id: "opencode-voice",
+  tui: async (api, options) => {
+    const { kv } = api;
+    const logger = createLogger(api.client);
+    logger.log("plugin", "Initializing", "debug");
+    // Session-scoped gateways (opencode.ai/zen/go) require x-opencode-session.
+    // Read the live route at call time so normalize works in any session.
+    // The voice-selected provider (/voice-model) fills endpoint/model only
+    // when the explicit options leave them out.
+    const { complete } = createClient(
+      options,
+      logger,
+      () => {
+        const route = api?.route?.current;
+        return route?.name === "session" ? route?.params?.sessionID : undefined;
+      },
+      () => resolveVoiceProviderModel(api, kv),
+    );
+
+    const prompts = {
+      stt: loadPromptFile(options?.sttPrompt, logger, "STT"),
+      ttsAuto: loadPromptFile(options?.ttsAutoPrompt, logger, "TTS auto"),
+      ttsManual: loadPromptFile(options?.ttsManualPrompt, logger, "TTS manual"),
+    };
+
+    const shared = {};
+    const stt = registerSTT(api, kv, complete, prompts, options, logger, {
+      isConversationActive: () => shared.conversation?.isActive() === true,
+      onConversationKey: (source) => shared.conversation?.onKey(source),
+      isLiveNotesActive: () => shared.liveNotes?.isActive() === true,
+    });
+    const tts = registerTTS(api, kv, complete, prompts, options, logger, {
+      isConversationActive: () => shared.conversation?.isActive() === true,
+      onConversationStopKey: () => shared.conversation?.onTtsStop() === true,
+    });
+    const conversation = registerConversation(api, options, logger, {
+      stt: stt.controller,
+      tts: tts.controller,
+      isLiveNotesActive: () => shared.liveNotes?.isActive() === true,
+    });
+    shared.conversation = conversation.controller;
+    const liveNotes = registerLiveNotes(api, kv, complete, options, logger, {
+      tts: tts.controller,
+      isConversationActive: () => shared.conversation?.isActive() === true,
+    });
+    shared.liveNotes = liveNotes.controller;
+    const voiceModel = registerVoiceModel(api, kv, options, logger);
+
+    api.command.register(() => [
+      ...stt.commands,
+      ...tts.commands,
+      ...conversation.commands,
+      ...liveNotes.commands,
+      ...voiceModel.commands,
+    ]);
+  },
+};

package/lib/audio-chunker.js

@@ -0,0 +1,231 @@
+// Continuous-capture audio chunker for live voice notes.
+//
+// Consumes a continuous stream of raw PCM16LE mono samples (as produced by
+// `sox ... -t raw -`) and splits it into self-contained chunks the
+// transcription lane can process independently, without ever pausing
+// capture. Two triggers close a chunk:
+//
+//   - natural: enough speech has accumulated and a silence gap follows (a
+//     real pause in the conversation - the ideal cut point)
+//   - forced: the chunk hit its max duration regardless of silence (bounds
+//     worst-case processing latency during continuous speech)
+//
+// Forced cuts carry a short audio overlap into the next chunk so a word is
+// never fully lost mid-cut; the caller (notes-writer) dedupes the
+// overlapping words from the transcribed text.
+
+const SAMPLE_RATE = 16000;
+const BYTES_PER_SAMPLE = 2;
+const FRAME_MS = 20;
+const FRAME_BYTES = (SAMPLE_RATE / 1000) * FRAME_MS * BYTES_PER_SAMPLE; // 640
+
+export const CHUNKER_FRAME_MS = FRAME_MS;
+export const CHUNKER_FRAME_BYTES = FRAME_BYTES;
+
+const DEFAULTS = {
+  sampleRate: SAMPLE_RATE,
+  minChunkMs: 3000,
+  maxChunkMs: 20000,
+  silenceMs: 700,
+  overlapMs: 400,
+  silenceRmsThreshold: 0.02,
+  minSpeechMsToKeep: 300,
+  // Auto-learn the room noise floor from the first `calibrationMs` of audio
+  // (usually mic hiss / HVAC before anyone speaks) instead of assuming the
+  // fixed threshold fits every room. 0 disables. An explicit
+  // silenceRmsThreshold always wins and skips calibration.
+  calibrationMs: 0,
+};
+
+// Threshold = 2.5x the room floor, clamped so a very quiet room never drops
+// into the mic's own noise and a loud room never eats quiet speech.
+export const CALIBRATION_MULTIPLIER = 2.5;
+export const CALIBRATION_MIN_THRESHOLD = 0.008;
+export const CALIBRATION_MAX_THRESHOLD = 0.03;
+
+export function calibrateSilenceThreshold(noiseFloorRms) {
+  const t = (noiseFloorRms || 0) * CALIBRATION_MULTIPLIER;
+  return Math.min(CALIBRATION_MAX_THRESHOLD, Math.max(CALIBRATION_MIN_THRESHOLD, t));
+}
+
+/**
+ * RMS (0..1) of a buffer of PCM16LE mono samples. Returns 0 for an empty or
+ * odd-length buffer.
+ */
+export function pcm16Rms(buffer) {
+  const usable = buffer.length - (buffer.length % 2);
+  if (usable <= 0) return 0;
+  let sum = 0;
+  const samples = usable / 2;
+  for (let i = 0; i < usable; i += 2) {
+    const n = buffer.readInt16LE(i) / 32768;
+    sum += n * n;
+  }
+  return Math.sqrt(sum / samples);
+}
+
+/**
+ * Build a standard 44-byte PCM WAV header for the given data length.
+ */
+export function buildWavHeader(dataLength, options = {}) {
+  const sampleRate = options.sampleRate ?? SAMPLE_RATE;
+  const channels = options.channels ?? 1;
+  const bitsPerSample = options.bitsPerSample ?? 16;
+  const blockAlign = channels * (bitsPerSample / 8);
+  const byteRate = sampleRate * blockAlign;
+
+  const header = Buffer.alloc(44);
+  header.write("RIFF", 0, "ascii");
+  header.writeUInt32LE(36 + dataLength, 4);
+  header.write("WAVE", 8, "ascii");
+  header.write("fmt ", 12, "ascii");
+  header.writeUInt32LE(16, 16);
+  header.writeUInt16LE(1, 20); // PCM
+  header.writeUInt16LE(channels, 22);
+  header.writeUInt32LE(sampleRate, 24);
+  header.writeUInt32LE(byteRate, 28);
+  header.writeUInt16LE(blockAlign, 32);
+  header.writeUInt16LE(bitsPerSample, 34);
+  header.write("data", 36, "ascii");
+  header.writeUInt32LE(dataLength, 40);
+  return header;
+}
+
+/**
+ * Wrap raw PCM16LE mono samples into a self-contained WAV buffer.
+ */
+export function wrapPcmAsWav(pcmBuffer, options = {}) {
+  return Buffer.concat([buildWavHeader(pcmBuffer.length, options), pcmBuffer]);
+}
+
+/**
+ * Create a stateful chunker. Feed it raw PCM16LE mono bytes via `push()` as
+ * they arrive from the capture process; it returns any chunks that became
+ * ready to transcribe. Call `flush()` once when capture stops to get the
+ * final partial chunk.
+ *
+ * Chunk shape: { seq, pcm, startMs, endMs, durationMs, forced, hasSpeech }
+ */
+export function createPcmChunker(options = {}) {
+  const cfg = { ...DEFAULTS, ...options };
+  const frameBytes = ((cfg.sampleRate / 1000) * FRAME_MS * BYTES_PER_SAMPLE) | 0;
+  const overlapFrameCount = Math.max(0, Math.round(cfg.overlapMs / FRAME_MS));
+  // Explicit threshold wins; otherwise learn the room for calibrationMs.
+  const autoCalibrate = !(options.silenceRmsThreshold > 0) && cfg.calibrationMs > 0;
+
+  let frames = [];
+  let leftover = Buffer.alloc(0);
+  let accMs = 0;
+  let speechMs = 0;
+  let silenceRun = 0;
+  let startMs = 0;
+  let totalElapsedMs = 0;
+  let seq = 0;
+  let calibrated = !autoCalibrate;
+  let calibrationRms = [];
+  let noiseFloorRms = null;
+
+  function finalizeChunk(forced) {
+    const pcm = Buffer.concat(frames);
+    const endMs = totalElapsedMs;
+    const chunk = {
+      seq: seq++,
+      pcm,
+      startMs,
+      endMs,
+      durationMs: endMs - startMs,
+      forced,
+      hasSpeech: speechMs >= cfg.minSpeechMsToKeep,
+    };
+
+    const keepFrames = forced ? frames.slice(-overlapFrameCount) : [];
+    frames = keepFrames.map((f) => Buffer.from(f));
+    accMs = frames.length * FRAME_MS;
+    speechMs = 0;
+    silenceRun = 0;
+    startMs = endMs - accMs;
+
+    return chunk;
+  }
+
+  function push(buffer) {
+    const ready = [];
+    let combined = leftover.length > 0 ? Buffer.concat([leftover, buffer]) : buffer;
+    const frameCount = Math.floor(combined.length / frameBytes);
+    leftover = combined.subarray(frameCount * frameBytes);
+
+    for (let i = 0; i < frameCount; i++) {
+      const frame = combined.subarray(i * frameBytes, (i + 1) * frameBytes);
+      const frameRms = pcm16Rms(frame);
+
+      // While calibrating, collect room-tone levels but don't cut on them -
+      // the fixed default threshold is meaningless before we know the room.
+      if (!calibrated) {
+        calibrationRms.push(frameRms);
+        frames.push(frame);
+        accMs += FRAME_MS;
+        totalElapsedMs += FRAME_MS;
+        if (totalElapsedMs >= cfg.calibrationMs) {
+          const sorted = [...calibrationRms].sort((a, b) => a - b);
+          noiseFloorRms = sorted[Math.floor(sorted.length / 2)] ?? 0;
+          cfg.silenceRmsThreshold = calibrateSilenceThreshold(noiseFloorRms);
+          calibrationRms = [];
+          calibrated = true;
+        }
+        continue;
+      }
+
+      const isSilence = frameRms < cfg.silenceRmsThreshold;
+
+      frames.push(frame);
+      accMs += FRAME_MS;
+      totalElapsedMs += FRAME_MS;
+      if (isSilence) silenceRun += FRAME_MS;
+      else {
+        silenceRun = 0;
+        speechMs += FRAME_MS;
+      }
+
+      const forcedReady = accMs >= cfg.maxChunkMs;
+      const naturalReady =
+        !forcedReady &&
+        accMs >= cfg.minChunkMs &&
+        speechMs >= cfg.minSpeechMsToKeep &&
+        silenceRun >= cfg.silenceMs;
+
+      if (forcedReady || naturalReady) {
+        ready.push(finalizeChunk(forcedReady));
+      }
+    }
+
+    return ready;
+  }
+
+  function flush() {
+    if (accMs <= 0) return null;
+    return finalizeChunk(false);
+  }
+
+  function reset() {
+    frames = [];
+    leftover = Buffer.alloc(0);
+    accMs = 0;
+    speechMs = 0;
+    silenceRun = 0;
+    startMs = 0;
+    totalElapsedMs = 0;
+    seq = 0;
+    calibrated = !autoCalibrate;
+    calibrationRms = [];
+    noiseFloorRms = null;
+  }
+
+  return {
+    push,
+    flush,
+    reset,
+    getSilenceThreshold: () => cfg.silenceRmsThreshold,
+    getNoiseFloor: () => noiseFloorRms,
+    isCalibrated: () => calibrated,
+  };
+}

package/lib/audio-enhance.js

@@ -0,0 +1,172 @@
+// Voice-only preprocessing for live notes (and anything else that feeds
+// quiet far-field audio to whisper).
+//
+// A classroom professor 3-5m from a laptop mic produces RMS ~0.01-0.02 -
+// right at the chunker silence threshold and far below what whisper.cpp was
+// trained on. Human ears compensate with automatic gain control; whisper
+// does not, so it hallucinates (Vietnamese YouTube outros) or mistranscribes
+// instead. This module applies the missing AGC in software before each chunk
+// reaches whisper:
+//
+//   1. measure the chunk (RMS + peak, same math as audio-chunker.js)
+//   2. compute the gain needed to bring speech up to a healthy target level
+//   3. run sox: highpass (room rumble/HVAC out) + gain with limiter (speech
+//      up, loud chunks untouched, never clips)
+//
+// Enhancement is per-chunk (not in the live capture chain) so the mic path
+// stays untouched and a failed enhance can never lose audio - on any sox
+// error the original file is kept as-is.
+
+import fs from "node:fs";
+import { spawn } from "node:child_process";
+import { pcm16Rms } from "./audio-chunker.js";
+
+export const DEFAULT_ENHANCE_TARGET_RMS = 0.1;
+export const DEFAULT_ENHANCE_MAX_GAIN_DB = 24;
+export const DEFAULT_ENHANCE_HIGHPASS_HZ = 80;
+// Chunks quieter than this are essentially silence - gaining them up only
+// amplifies noise into hallucination fuel, so they pass through untouched.
+export const DEFAULT_ENHANCE_MIN_RMS = 0.004;
+
+/**
+ * RMS + peak (both 0..1) of a PCM16LE mono buffer. Pure.
+ */
+export function measurePcmStats(pcmBuffer) {
+  const rms = pcm16Rms(pcmBuffer);
+  let peak = 0;
+  const usable = pcmBuffer.length - (pcmBuffer.length % 2);
+  for (let i = 0; i < usable; i += 2) {
+    const a = Math.abs(pcmBuffer.readInt16LE(i) / 32768);
+    if (a > peak) peak = a;
+  }
+  return { rms, peak };
+}
+
+export function rmsToDb(rms) {
+  if (!rms || rms <= 0) return -Infinity;
+  return 20 * Math.log10(rms);
+}
+
+/**
+ * Gain in dB needed to bring `rms` up to `targetRms`. Never negative
+ * (loud chunks stay as they are) and never above `maxGainDb`. Pure.
+ */
+export function computeGainDb(rms, options = {}) {
+  const targetRms = options.targetRms ?? DEFAULT_ENHANCE_TARGET_RMS;
+  const maxGainDb = options.maxGainDb ?? DEFAULT_ENHANCE_MAX_GAIN_DB;
+  if (!rms || rms <= 0 || rms >= targetRms) return 0;
+  return Math.min(maxGainDb, 20 * Math.log10(targetRms / rms));
+}
+
+/**
+ * sox effect chain: strip sub-voice rumble, then adaptive gain with the
+ * limiter engaged so an underestimated peak can never clip. Pure.
+ */
+export function buildEnhanceArgs(gainDb, options = {}) {
+  const highpassHz = options.highpassHz ?? DEFAULT_ENHANCE_HIGHPASS_HZ;
+  return ["highpass", String(highpassHz), "gain", "-l", gainDb.toFixed(1)];
+}
+
+/**
+ * Run sox without blocking the event loop. Enhancement runs on the
+ * processing-lane stage (not the capture lane), but a synchronous 15s spawn
+ * would still stall every timer/callback sharing the loop — including the
+ * stdout drain of a live capture — so the pipe can fill and stall SoX.
+ */
+function runSoxAsync(args, timeoutMs) {
+  return new Promise((resolve) => {
+    let proc;
+    try {
+      proc = spawn("sox", args);
+    } catch (err) {
+      resolve({ error: err, status: null, stderr: "" });
+      return;
+    }
+    let stderr = "";
+    const timer = setTimeout(() => {
+      try {
+        proc.kill("SIGKILL");
+      } catch {}
+    }, timeoutMs);
+    proc.stderr?.on("data", (chunk) => {
+      stderr += chunk.toString();
+    });
+    proc.on("error", (err) => {
+      clearTimeout(timer);
+      resolve({ error: err, status: null, stderr });
+    });
+    proc.on("close", (code) => {
+      clearTimeout(timer);
+      resolve({ error: null, status: code, stderr });
+    });
+  });
+}
+
+/**
+ * Enhance one chunk WAV file in place for whisper. `pcm` is the raw chunk
+ * samples (used for measurement so we don't have to re-read the file).
+ * Returns stats for the JSONL sidecar; `enhanced:false` means the original
+ * file was kept untouched (quiet-as-silence, already loud, or sox failed).
+ * Async (never blocks the event loop); on any sox failure the original file
+ * is kept as-is so enhancement can never lose audio.
+ */
+export async function enhanceWavFile(wavPath, pcm, options = {}, logger) {
+  const targetRms = options.targetRms ?? DEFAULT_ENHANCE_TARGET_RMS;
+  const maxGainDb = options.maxGainDb ?? DEFAULT_ENHANCE_MAX_GAIN_DB;
+  const minRms = options.minRms ?? DEFAULT_ENHANCE_MIN_RMS;
+
+  const before = measurePcmStats(pcm);
+  const noEnhance = (reason) => ({
+    enhanced: false,
+    reason,
+    rmsBefore: before.rms,
+    peakBefore: before.peak,
+    gainDb: 0,
+  });
+
+  if (before.rms < minRms) return noEnhance("too-quiet");
+  const gainDb = computeGainDb(before.rms, { targetRms, maxGainDb });
+  if (gainDb <= 0) return noEnhance("already-loud");
+
+  const tmpPath = `${wavPath}.enh.wav`;
+  const args = [wavPath, "-b", "16", tmpPath, ...buildEnhanceArgs(gainDb, options)];
+  let res;
+  try {
+    res = await runSoxAsync(args, 15000);
+  } catch (err) {
+    logger?.log("VOICE", `Enhance spawn failed, keeping original: ${err.message}`, "warn");
+    return noEnhance("sox-spawn-failed");
+  }
+  if (res.error || res.status !== 0) {
+    try {
+      fs.unlinkSync(tmpPath);
+    } catch {}
+    logger?.log(
+      "VOICE",
+      `Enhance failed, keeping original: ${(res.error?.message || (res.stderr || "").trim() || `exit ${res.status}`).slice(0, 200)}`,
+      "warn",
+    );
+    return noEnhance(res.error ? "sox-spawn-failed" : "sox-failed");
+  }
+
+  let after;
+  try {
+    const out = fs.readFileSync(tmpPath);
+    after = measurePcmStats(out.subarray(Math.min(44, out.length)));
+    fs.renameSync(tmpPath, wavPath);
+  } catch (err) {
+    try {
+      fs.unlinkSync(tmpPath);
+    } catch {}
+    logger?.log("VOICE", `Enhance replace failed, keeping original: ${err.message}`, "warn");
+    return noEnhance("replace-failed");
+  }
+  return {
+    enhanced: true,
+    gainDb,
+    rmsBefore: before.rms,
+    peakBefore: before.peak,
+    rmsAfter: after.rms,
+    peakAfter: after.peak,
+  };
+}