opencode-voice-dictation 0.6.2 → 0.7.0

package/src/gui/window.tsx

@@ -56,9 +56,28 @@ export const setup: Plugin.Setup<typeof definition> = (ctx) => {
   const prefs = ctx.stores.prefs
   const engine = ctx.stores.engine
   const whisper = ctx.stores.whisper
+  const whisperCpp = ctx.stores.whispercpp
   const [windowsActive, setWindowsActive] = createSignal(false)
   const [whisperState, setWhisperState] = createSignal<"idle" | "recording" | "transcribing">("idle")
 
+  // The server owns the OS: it decides which engines exist here. On Linux this
+  // switches the default to whisper.cpp and drops the Windows-only engines.
+  void ctx.client
+    .rpc(VoiceRpc)
+    .capabilities({})
+    .then((value) => {
+      const caps = value as { platform: string; engines: string[] }
+      if (!caps.engines.includes(engine.value.mode) && caps.engines.length > 0) {
+        engine.update((draft) => {
+          draft.mode = caps.engines[0] ?? "windows"
+        })
+      }
+    })
+    .catch(() => {})
+
+  const localMode = () =>
+    engine.value.mode === "whisper" || engine.value.mode === "whispercpp" ? engine.value.mode : undefined
+
   let targetSession: string | undefined
   let recording: Recording | undefined
 
@@ -122,7 +141,9 @@ export const setup: Plugin.Setup<typeof definition> = (ctx) => {
       })
   }
 
-  const startWhisper = async () => {
+  const localError = () => ctx.t(localMode() === "whispercpp" ? "error.whispercpp" : "error.whisper")
+
+  const startLocal = async () => {
     if (whisperState() !== "idle") return
     speaker.cancel()
 
@@ -133,19 +154,19 @@ export const setup: Plugin.Setup<typeof definition> = (ctx) => {
       ctx.toast.show({
         variant: "error",
         title: ctx.t("pill.title"),
-        description: cause instanceof Error ? cause.message : ctx.t("error.whisper"),
+        description: cause instanceof Error ? cause.message : localError(),
       })
     }
   }
 
-  const cancelWhisper = () => {
+  const cancelLocal = () => {
     const current = recording
     recording = undefined
     if (current) void current.stop().catch(() => {})
     setWhisperState("idle")
   }
 
-  const stopWhisper = async () => {
+  const stopLocal = async () => {
     const current = recording
     recording = undefined
 
@@ -155,22 +176,34 @@ export const setup: Plugin.Setup<typeof definition> = (ctx) => {
     }
 
     setWhisperState("transcribing")
+    const mode = localMode()
 
     try {
       const blob = await current.stop()
       const audio = await blobToBase64(blob)
-      const result = (await ctx.client.rpc(VoiceRpc).whisperTranscribe({
-        audio,
-        mime: blob.type,
-        language: whisper.value.language,
-        python: whisper.value.python,
-        model: whisper.value.model,
-      })) as { text: string; language: string }
+      const result = (mode === "whispercpp"
+        ? await ctx.client.rpc(VoiceRpc).whisperCppTranscribe({
+            audio,
+            mime: blob.type,
+            language: whisperCpp.value.language,
+            model: whisperCpp.value.model,
+          })
+        : await ctx.client.rpc(VoiceRpc).whisperTranscribe({
+            audio,
+            mime: blob.type,
+            language: whisper.value.language,
+            python: whisper.value.python,
+            model: whisper.value.model,
+          })) as { text: string; language: string }
 
       const text = result.text.trim()
 
       if (!text) {
-        ctx.toast.show({ variant: "default", title: ctx.t("pill.title"), description: ctx.t("whisper.empty") })
+        ctx.toast.show({
+          variant: "default",
+          title: ctx.t("pill.title"),
+          description: ctx.t(mode === "whispercpp" ? "whispercpp.empty" : "whisper.empty"),
+        })
         return
       }
 
@@ -185,7 +218,7 @@ export const setup: Plugin.Setup<typeof definition> = (ctx) => {
       ctx.toast.show({
         variant: "error",
         title: ctx.t("pill.title"),
-        description: cause instanceof Error ? cause.message : ctx.t("error.whisper"),
+        description: cause instanceof Error ? cause.message : localError(),
       })
     } finally {
       setWhisperState("idle")
@@ -193,9 +226,9 @@ export const setup: Plugin.Setup<typeof definition> = (ctx) => {
   }
 
   const toggle = () => {
-    if (engine.value.mode === "whisper") {
-      if (whisperState() === "recording") void stopWhisper()
-      else if (whisperState() === "idle") void startWhisper()
+    if (localMode()) {
+      if (whisperState() === "recording") void stopLocal()
+      else if (whisperState() === "idle") void startLocal()
       return
     }
 
@@ -210,7 +243,7 @@ export const setup: Plugin.Setup<typeof definition> = (ctx) => {
   }
 
   const read = () => {
-    if (whisperState() === "recording") cancelWhisper()
+    if (whisperState() === "recording") cancelLocal()
 
     if (speaker.speaking()) {
       speaker.cancel()
@@ -318,7 +351,9 @@ export const setup: Plugin.Setup<typeof definition> = (ctx) => {
           }
     }
 
-    if (engine.value.mode === "whisper") {
+    if (engine.value.mode === "whisper" || engine.value.mode === "whispercpp") {
+      const title = ctx.t(engine.value.mode === "whispercpp" ? "pill.whispercpp" : "pill.whisper")
+
       if (whisperState() === "recording") {
         return {
           label: ctx.t("pill.recording"),
@@ -333,14 +368,14 @@ export const setup: Plugin.Setup<typeof definition> = (ctx) => {
         return {
           label: ctx.t("pill.transcribing"),
           icon: "outline-audio",
-          title: ctx.t("pill.whisper"),
+          title,
         }
       }
 
       return {
         label: ctx.t("pill.idle"),
         icon: "outline-wave-audio",
-        title: ctx.t("pill.whisper"),
+        title,
         run: toggle,
       }
     }
@@ -464,6 +499,10 @@ export const setup: Plugin.Setup<typeof definition> = (ctx) => {
         { id: "whisper-model", title: ctx.t("whisper.model"), keywords: "whisper model modelo" },
         { id: "whisper-language", title: ctx.t("whisper.language"), keywords: "whisper language idioma" },
         { id: "whisper-test", title: ctx.t("whisper.test"), keywords: "whisper check comprobar" },
+        { id: "whispercpp-model", title: ctx.t("whispercpp.model"), keywords: "whisper.cpp model modelo" },
+        { id: "whispercpp-language", title: ctx.t("whispercpp.language"), keywords: "whisper.cpp language idioma" },
+        { id: "whispercpp-install", title: ctx.t("whispercpp.install"), keywords: "whisper.cpp download descargar instalar" },
+        { id: "whispercpp-test", title: ctx.t("whispercpp.test"), keywords: "whisper.cpp check comprobar" },
         { id: "rate", title: ctx.t("speech.rate"), keywords: "read aloud speed velocidad lectura" },
         { id: "skip-code", title: ctx.t("speech.skipCode"), keywords: "read aloud code codigo lectura" },
         { id: "preview", title: ctx.t("speech.preview"), keywords: "read aloud test voice probar voz" },

package/src/index.ts

@@ -2,20 +2,31 @@ import { Plugin } from "@opencode/plugin"
 import { listEdgeVoices, synthesizeEdge } from "./edge.js"
 import { VoiceRpc } from "./rpc.js"
 import { WhisperEngine } from "./whisper/server.js"
+import { installWhisperCpp, WhisperCppEngine } from "./whispercpp/server.js"
 
 /**
  * Server part of the plugin.
  *
- * It registers the Edge TTS and Whisper RPCs. Running them here (instead of
- * in the Electron main process) lets their dependencies resolve from the
- * plugin's own node_modules.
+ * It registers the Edge TTS, WhisperX and whisper.cpp RPCs. Running them here
+ * (instead of in the Electron main process) lets their dependencies resolve
+ * from the plugin's own node_modules.
  */
 export default Plugin.define({
   id: "voice-dictation",
   async setup(ctx) {
     const whisper = new WhisperEngine()
+    const whisperCpp = new WhisperCppEngine()
 
     await ctx.rpc.register(VoiceRpc, {
+      capabilities: async () => ({
+        platform: process.platform,
+        engines:
+          process.platform === "linux"
+            ? ["whispercpp"]
+            : process.platform === "win32"
+              ? ["windows", "system", "whisper"]
+              : [],
+      }),
       edgeVoices: async () => listEdgeVoices(),
       edgeSpeak: async (input) => {
         const { text, voice, rate } = input as { text: string; voice: string; rate: number }
@@ -42,9 +53,28 @@ export default Plugin.define({
           }
         }
       },
+      whisperCppInstall: async (input) => {
+        const { model } = input as { model: string }
+        return installWhisperCpp(model)
+      },
+      whisperCppStatus: async (input) => {
+        const { model } = input as { model: string }
+        return whisperCpp.status(model)
+      },
+      whisperCppTranscribe: async (input) => {
+        return whisperCpp.transcribe(input as {
+          audio: string
+          mime: string
+          language: string
+          model: string
+        })
+      },
     })
 
-    return () => whisper.stop()
+    return () => {
+      whisper.stop()
+      whisperCpp.stop()
+    }
   },
 })
 

package/src/rpc.ts

@@ -85,6 +85,86 @@ export const VoiceRpc = {
         additionalProperties: false,
       },
     },
+    /**
+     * Tells the window which OS the server runs on and which dictation engines
+     * exist there. The window uses it to pick the default engine and to hide
+     * engines that do not apply (Win+H on Linux, whisper.cpp on Windows).
+     */
+    capabilities: {
+      input: { type: "object", additionalProperties: false },
+      output: {
+        type: "object",
+        properties: {
+          platform: { type: "string" },
+          engines: { type: "array", items: { type: "string" } },
+        },
+        required: ["platform", "engines"],
+        additionalProperties: false,
+      },
+    },
+    whisperCppInstall: {
+      input: {
+        type: "object",
+        properties: {
+          model: { type: "string" },
+        },
+        required: ["model"],
+        additionalProperties: false,
+      },
+      output: {
+        type: "object",
+        properties: {
+          ok: { type: "boolean" },
+          model: { type: "string" },
+          error: { type: "string" },
+        },
+        required: ["ok", "model", "error"],
+        additionalProperties: false,
+      },
+    },
+    whisperCppStatus: {
+      input: {
+        type: "object",
+        properties: {
+          model: { type: "string" },
+        },
+        required: ["model"],
+        additionalProperties: false,
+      },
+      output: {
+        type: "object",
+        properties: {
+          installed: { type: "boolean" },
+          ready: { type: "boolean" },
+          model: { type: "string" },
+          error: { type: "string" },
+        },
+        required: ["installed", "ready", "model", "error"],
+        additionalProperties: false,
+      },
+    },
+    whisperCppTranscribe: {
+      input: {
+        type: "object",
+        properties: {
+          audio: { type: "string" },
+          mime: { type: "string" },
+          language: { type: "string" },
+          model: { type: "string" },
+        },
+        required: ["audio", "mime", "language", "model"],
+        additionalProperties: false,
+      },
+      output: {
+        type: "object",
+        properties: {
+          text: { type: "string" },
+          language: { type: "string" },
+        },
+        required: ["text", "language"],
+        additionalProperties: false,
+      },
+    },
   },
   events: {},
 } as const satisfies Rpc.PortableDefinition

package/src/whispercpp/server.ts

@@ -0,0 +1,277 @@
+import { spawn, type ChildProcess } from "node:child_process"
+import { randomUUID } from "node:crypto"
+import { chmodSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"
+import { connect } from "node:net"
+import { homedir, tmpdir } from "node:os"
+import { join } from "node:path"
+
+const RELEASE = "v1.9.2"
+const X64_TARBALL = `https://github.com/ggml-org/whisper.cpp/releases/download/${RELEASE}/whisper-bin-ubuntu-x64.tar.gz`
+const ARM64_TARBALL = `https://github.com/ggml-org/whisper.cpp/releases/download/${RELEASE}/whisper-bin-ubuntu-arm64.tar.gz`
+const MODEL_BASE = "https://huggingface.co/ggerganov/whisper.cpp/resolve/main"
+
+const MODELS = {
+  "base-q5_1": "ggml-base-q5_1.bin",
+  "small-q5_1": "ggml-small-q5_1.bin",
+} as const
+
+type ModelKey = keyof typeof MODELS
+
+export interface WhisperCppStatus {
+  installed: boolean
+  ready: boolean
+  model: string
+  error: string
+}
+
+export interface WhisperCppResult {
+  text: string
+  language: string
+}
+
+function rootDir(): string {
+  return join(homedir(), ".local", "share", "opencode-voice-dictation", "whisper.cpp")
+}
+
+function packageDir(): string {
+  return join(rootDir(), process.arch === "arm64" ? "whisper-bin-ubuntu-arm64" : "whisper-bin-ubuntu-x64")
+}
+
+function binaryPath(): string {
+  return join(packageDir(), "whisper-server")
+}
+
+function modelFile(model: string): ModelKey {
+  return model in MODELS ? (model as ModelKey) : "base-q5_1"
+}
+
+function modelPath(model: string): string {
+  return join(rootDir(), MODELS[modelFile(model)])
+}
+
+async function download(url: string, destination: string): Promise<void> {
+  const response = await fetch(url)
+
+  if (!response.ok) {
+    throw new Error(`Download failed (${response.status}): ${url}`)
+  }
+
+  mkdirSync(rootDir(), { recursive: true })
+  writeFileSync(destination, Buffer.from(await response.arrayBuffer()))
+}
+
+function run(command: string, args: string[]): Promise<string> {
+  return new Promise((resolve, reject) => {
+    const child = spawn(command, args, { windowsHide: true })
+    let stderr = ""
+
+    child.stderr?.setEncoding("utf8")
+    child.stderr?.on("data", (chunk: string) => {
+      stderr = (stderr + chunk).slice(-2000)
+    })
+
+    child.on("error", reject)
+    child.on("close", (code) => {
+      if (code === 0) resolve(stderr)
+      else reject(new Error(stderr.trim() || `${command} exited with code ${code ?? "unknown"}`))
+    })
+  })
+}
+
+export async function installWhisperCpp(model: string): Promise<{ ok: boolean; model: string; error: string }> {
+  if (process.platform !== "linux") {
+    return { ok: false, model: modelFile(model), error: "Whisper.cpp native is only implemented for Linux" }
+  }
+
+  try {
+    mkdirSync(rootDir(), { recursive: true })
+
+    if (!existsSync(binaryPath())) {
+      const tarball = join(rootDir(), "whisper-bin.tar.gz")
+      await download(process.arch === "arm64" ? ARM64_TARBALL : X64_TARBALL, tarball)
+      await run("tar", ["-xzf", tarball, "-C", rootDir()])
+      rmSync(tarball, { force: true })
+      chmodSync(binaryPath(), 0o755)
+    }
+
+    const file = modelFile(model)
+    if (!existsSync(modelPath(file))) {
+      await download(`${MODEL_BASE}/${MODELS[file]}`, modelPath(file))
+    }
+
+    return { ok: true, model: file, error: "" }
+  } catch (cause) {
+    return { ok: false, model: modelFile(model), error: cause instanceof Error ? cause.message : String(cause) }
+  }
+}
+
+interface StartState {
+  child: ChildProcess
+  port: number
+  model: string
+  language: string
+  stderr: string
+}
+
+export class WhisperCppEngine {
+  private state: StartState | undefined
+  private starting: Promise<number> | undefined
+  private error = ""
+
+  private waitForPort(port: number, timeout: number): Promise<number> {
+    const started = Date.now()
+
+    return new Promise((resolve, reject) => {
+      const attempt = () => {
+        const socket = connect(port, "127.0.0.1")
+
+        socket.once("connect", () => {
+          socket.destroy()
+          resolve(port)
+        })
+
+        socket.once("error", () => {
+          socket.destroy()
+          if (Date.now() - started > timeout) {
+            reject(new Error("whisper-server did not start in time"))
+            return
+          }
+          setTimeout(attempt, 250)
+        })
+      }
+
+      attempt()
+    })
+  }
+
+  private async ensure(model: string, language: string): Promise<number> {
+    const file = modelFile(model)
+    const lang = language || "auto"
+
+    if (this.state && this.state.model === file && this.state.language === lang) {
+      return this.state.port
+    }
+
+    if (this.starting) return this.starting
+
+    this.stop()
+    this.error = ""
+
+    for (const path of [binaryPath(), modelPath(file)]) {
+      if (!existsSync(path)) {
+        throw new Error("Whisper.cpp is not installed. Open Settings and download it first.")
+      }
+    }
+
+    const port = 18081 + Math.floor(Math.random() * 500)
+    const child = spawn(
+      binaryPath(),
+      ["-m", modelPath(file), "-l", lang, "-t", "4", "--host", "127.0.0.1", "--port", String(port)],
+      {
+        cwd: packageDir(),
+        env: {
+          ...process.env,
+          LD_LIBRARY_PATH: [packageDir(), process.env.LD_LIBRARY_PATH].filter(Boolean).join(":"),
+        },
+        windowsHide: true,
+        stdio: ["ignore", "ignore", "pipe"],
+      },
+    )
+
+    const state: StartState = { child, port, model: file, language: lang, stderr: "" }
+    this.state = state
+
+    child.stderr?.setEncoding("utf8")
+    child.stderr?.on("data", (chunk: string) => {
+      state.stderr = (state.stderr + chunk).slice(-2000)
+    })
+
+    const exited = new Promise<never>((_, reject) => {
+      child.once("error", (cause) => reject(cause))
+      child.once("close", (code) => {
+        if (this.state === state) this.state = undefined
+        reject(new Error(state.stderr.trim() || `whisper-server exited with code ${code ?? "unknown"}`))
+      })
+    })
+    exited.catch(() => {})
+
+    this.starting = Promise.race([this.waitForPort(port, 120000), exited])
+      .then((value) => {
+        this.starting = undefined
+        return value
+      })
+      .catch((cause) => {
+        this.starting = undefined
+        this.error = cause instanceof Error ? cause.message : String(cause)
+        this.stop()
+        throw cause
+      })
+
+    return this.starting
+  }
+
+  async transcribe(input: {
+    audio: string
+    mime: string
+    language: string
+    model: string
+  }): Promise<WhisperCppResult> {
+    const port = await this.ensure(input.model, input.language)
+
+    const dir = join(tmpdir(), "opencode-voice-dictation")
+    mkdirSync(dir, { recursive: true })
+
+    const extension = input.mime.includes("webm") ? "webm" : "wav"
+    const source = join(dir, `whispercpp-${randomUUID()}.${extension}`)
+    const wav = `${source}.wav`
+
+    writeFileSync(source, Buffer.from(input.audio, "base64"))
+
+    try {
+      await run("ffmpeg", ["-y", "-loglevel", "error", "-i", source, "-ar", "16000", "-ac", "1", "-c:a", "pcm_s16le", wav])
+
+      const form = new FormData()
+      form.append("file", new Blob([new Uint8Array(readFileSync(wav))], { type: "audio/wav" }), "audio.wav")
+
+      const response = await fetch(`http://127.0.0.1:${port}/inference`, { method: "POST", body: form })
+
+      if (!response.ok) {
+        throw new Error(`whisper-server HTTP ${response.status}`)
+      }
+
+      const data = (await response.json()) as { text?: string }
+
+      return {
+        text: (data.text ?? "").trim(),
+        language: input.language || "",
+      }
+    } finally {
+      rmSync(source, { force: true })
+      rmSync(wav, { force: true })
+    }
+  }
+
+  status(model: string): WhisperCppStatus {
+    const file = modelFile(model)
+    const installed = existsSync(binaryPath()) && existsSync(modelPath(file))
+
+    return {
+      installed,
+      ready: this.state !== undefined && this.state.model === file,
+      model: file,
+      error: this.error,
+    }
+  }
+
+  stop() {
+    if (this.state) {
+      const child = this.state.child
+      this.state = undefined
+      child.kill()
+      if (child.pid) {
+        spawn("pkill", ["-P", String(child.pid)], { windowsHide: true, stdio: "ignore" })
+      }
+    }
+    this.starting = undefined
+  }
+}