opencode-voice-dictation 0.4.2 → 0.6.0

package/src/gui/settings.tsx

@@ -3,6 +3,7 @@ import { usePlugin } from "@opencode/plugin/gui"
 import { Dictation, VoiceTyping } from "./contract"
 import type { definition } from "./definition"
 import type { Speaker } from "./speak"
+import { VoiceRpc } from "../rpc"
 
 const FALLBACK_LOCALES = [
   "es-ES",
@@ -29,11 +30,13 @@ export function VoiceSettings(props: { speaker: Speaker }) {
   const speechPrefs = ctx.stores.speech
   const ttsPrefs = ctx.stores.tts
   const enginePrefs = ctx.stores.engine
+  const whisperPrefs = ctx.stores.whisper
   const dictation = ctx.main(Dictation)
   const voiceTyping = ctx.main(VoiceTyping)
 
   const [locales, setLocales] = createSignal<readonly string[]>([])
   const [busy, setBusy] = createSignal(false)
+  const [whisperBusy, setWhisperBusy] = createSignal(false)
 
   onMount(() => {
     props.speaker.setConfig({ engine: ttsPrefs.value.engine, voice: ttsPrefs.value.voice })
@@ -96,6 +99,48 @@ export function VoiceSettings(props: { speaker: Speaker }) {
     })
   }
 
+  const patchWhisper = (change: { python?: string; model?: string; language?: string }) => {
+    whisperPrefs.update((draft) => {
+      if (change.python !== undefined) draft.python = change.python
+      if (change.model !== undefined) draft.model = change.model
+      if (change.language !== undefined) draft.language = change.language
+    })
+  }
+
+  const testWhisper = async () => {
+    setWhisperBusy(true)
+    try {
+      const status = (await ctx.client.rpc(VoiceRpc).whisperStatus({
+        python: whisperPrefs.value.python,
+        model: whisperPrefs.value.model,
+      })) as { ready: boolean; model: string; device: string; error: string }
+
+      if (status.ready) {
+        ctx.toast.show({
+          variant: "success",
+          title: ctx.t("pill.title"),
+          description: ctx.t("whisper.test.ready", { model: status.model, device: status.device }),
+        })
+      } else {
+        ctx.toast.show({
+          variant: "error",
+          title: ctx.t("pill.title"),
+          description: ctx.t("whisper.test.error", { error: status.error || "?" }),
+        })
+      }
+    } catch (cause) {
+      ctx.toast.show({
+        variant: "error",
+        title: ctx.t("pill.title"),
+        description: ctx.t("whisper.test.error", {
+          error: cause instanceof Error ? cause.message : String(cause),
+        }),
+      })
+    } finally {
+      setWhisperBusy(false)
+    }
+  }
+
   const patchTts = (engine: string) => {
     ttsPrefs.update((draft) => {
       draft.engine = engine
@@ -190,6 +235,7 @@ export function VoiceSettings(props: { speaker: Speaker }) {
             options={[
               { value: "windows", label: ctx.t("engine.windows") },
               { value: "system", label: ctx.t("engine.system") },
+              { value: "whisper", label: ctx.t("engine.whisper") },
             ]}
             onChange={(value) => patchEngine(value)}
           />
@@ -205,6 +251,65 @@ export function VoiceSettings(props: { speaker: Speaker }) {
           </ui.SettingsRow>
         </Show>
 
+        <Show when={enginePrefs.value.mode === "whisper"}>
+          <ui.SettingsRow
+            id="whisper-python"
+            title={ctx.t("whisper.python")}
+            description={ctx.t("whisper.python.description")}
+          >
+            <ui.TextInput
+              label={ctx.t("whisper.python")}
+              value={whisperPrefs.value.python}
+              onInput={(value) => patchWhisper({ python: value })}
+            />
+          </ui.SettingsRow>
+
+          <ui.SettingsRow
+            id="whisper-model"
+            title={ctx.t("whisper.model")}
+            description={ctx.t("whisper.model.description")}
+          >
+            <ui.Select
+              label={ctx.t("whisper.model")}
+              value={whisperPrefs.value.model}
+              options={[
+                { value: "large-v3-turbo", label: "large-v3-turbo" },
+                { value: "large-v3", label: "large-v3" },
+                { value: "medium", label: "medium" },
+              ]}
+              onChange={(value) => patchWhisper({ model: value })}
+            />
+          </ui.SettingsRow>
+
+          <ui.SettingsRow
+            id="whisper-language"
+            title={ctx.t("whisper.language")}
+            description={ctx.t("whisper.language.description")}
+          >
+            <ui.Select
+              label={ctx.t("whisper.language")}
+              value={whisperPrefs.value.language}
+              options={[
+                { value: "", label: ctx.t("whisper.language.auto") },
+                { value: "es", label: "Español" },
+                { value: "en", label: "English" },
+                { value: "pt", label: "Português" },
+              ]}
+              onChange={(value) => patchWhisper({ language: value })}
+            />
+          </ui.SettingsRow>
+
+          <ui.SettingsRow
+            id="whisper-test"
+            title={ctx.t("whisper.test")}
+            description={ctx.t("whisper.test.description")}
+          >
+            <ui.Button busy={whisperBusy()} onClick={() => void testWhisper()}>
+              {ctx.t("whisper.test.run")}
+            </ui.Button>
+          </ui.SettingsRow>
+        </Show>
+
         <Show when={enginePrefs.value.mode === "system"}>
           <ui.SettingsRow
             id="language"

package/src/gui/speak.ts

@@ -1,5 +1,6 @@
 import { createSignal } from "solid-js"
 import type { IpcClient, SetupContext } from "@opencode/plugin/gui"
+import { VoiceRpc } from "../rpc"
 import { Speech } from "./contract"
 import type { definition } from "./definition"
 
@@ -153,7 +154,10 @@ export function createSpeaker(
   const refreshVoices = async () => {
     if (engine() === "edge") {
       try {
-        const list = await speech.edgeVoices()
+        const list = (await ctx.client.rpc(VoiceRpc).edgeVoices({})) as unknown as Array<{
+          id: string
+          label: string
+        }>
         if (list.length > 0) {
           setVoices(list.map((voice) => ({ id: voice.id, label: voice.label })))
           setError("")
@@ -279,7 +283,11 @@ export function createSpeaker(
 
         try {
           if (!encoded) {
-            encoded = await speech.edgeSpeak({ text: blocks[index] ?? "", voice: selected, rate })
+            encoded = (await ctx.client.rpc(VoiceRpc).edgeSpeak({
+              text: blocks[index] ?? "",
+              voice: selected,
+              rate,
+            })) as string
             cached[index] = encoded
           }
         } catch {

package/src/gui/window.tsx

@@ -13,6 +13,8 @@ import { Dictation, Speech, VoiceTyping } from "./contract"
 import type { definition } from "./definition"
 import { createSpeaker, type Speaker } from "./speak"
 import { VoiceSettings } from "./settings"
+import { blobToBase64, startRecording, type Recording } from "./recorder"
+import { VoiceRpc } from "../rpc"
 
 interface ReadAction {
   id: string
@@ -53,9 +55,12 @@ export const setup: Plugin.Setup<typeof definition> = (ctx) => {
   const speaker = createSpeaker(ctx, speech)
   const prefs = ctx.stores.prefs
   const engine = ctx.stores.engine
+  const whisper = ctx.stores.whisper
   const [windowsActive, setWindowsActive] = createSignal(false)
+  const [whisperState, setWhisperState] = createSignal<"idle" | "recording" | "transcribing">("idle")
 
   let targetSession: string | undefined
+  let recording: Recording | undefined
 
   const reason = (code: string, detail: string, available: readonly string[]) => {
     switch (code) {
@@ -117,7 +122,83 @@ export const setup: Plugin.Setup<typeof definition> = (ctx) => {
       })
   }
 
+  const startWhisper = async () => {
+    if (whisperState() !== "idle") return
+    speaker.cancel()
+
+    try {
+      recording = await startRecording()
+      setWhisperState("recording")
+    } catch (cause) {
+      ctx.toast.show({
+        variant: "error",
+        title: ctx.t("pill.title"),
+        description: cause instanceof Error ? cause.message : ctx.t("error.whisper"),
+      })
+    }
+  }
+
+  const cancelWhisper = () => {
+    const current = recording
+    recording = undefined
+    if (current) void current.stop().catch(() => {})
+    setWhisperState("idle")
+  }
+
+  const stopWhisper = async () => {
+    const current = recording
+    recording = undefined
+
+    if (!current) {
+      setWhisperState("idle")
+      return
+    }
+
+    setWhisperState("transcribing")
+
+    try {
+      const blob = await current.stop()
+      const audio = await blobToBase64(blob)
+      const result = (await ctx.client.rpc(VoiceRpc).whisperTranscribe({
+        audio,
+        mime: blob.type,
+        language: whisper.value.language,
+        python: whisper.value.python,
+        model: whisper.value.model,
+      })) as { text: string; language: string }
+
+      const text = result.text.trim()
+
+      if (!text) {
+        ctx.toast.show({ variant: "default", title: ctx.t("pill.title"), description: ctx.t("whisper.empty") })
+        return
+      }
+
+      const composer = ctx.screen.current()?.composer
+      if (composer) {
+        const existing = composer.text()
+        const separator = existing.length > 0 && !/\s$/.test(existing) ? " " : ""
+        composer.insert(separator + text)
+        composer.focus()
+      }
+    } catch (cause) {
+      ctx.toast.show({
+        variant: "error",
+        title: ctx.t("pill.title"),
+        description: cause instanceof Error ? cause.message : ctx.t("error.whisper"),
+      })
+    } finally {
+      setWhisperState("idle")
+    }
+  }
+
   const toggle = () => {
+    if (engine.value.mode === "whisper") {
+      if (whisperState() === "recording") void stopWhisper()
+      else if (whisperState() === "idle") void startWhisper()
+      return
+    }
+
     if (engine.value.mode === "windows") {
       toggleWindows()
       return
@@ -129,6 +210,8 @@ export const setup: Plugin.Setup<typeof definition> = (ctx) => {
   }
 
   const read = () => {
+    if (whisperState() === "recording") cancelWhisper()
+
     if (speaker.speaking()) {
       speaker.cancel()
       return
@@ -235,6 +318,33 @@ export const setup: Plugin.Setup<typeof definition> = (ctx) => {
           }
     }
 
+    if (engine.value.mode === "whisper") {
+      if (whisperState() === "recording") {
+        return {
+          label: ctx.t("pill.recording"),
+          icon: "outline-wave-audio",
+          tone: "danger",
+          title: ctx.t("pill.stop"),
+          run: toggle,
+        }
+      }
+
+      if (whisperState() === "transcribing") {
+        return {
+          label: ctx.t("pill.transcribing"),
+          icon: "outline-audio",
+          title: ctx.t("pill.whisper"),
+        }
+      }
+
+      return {
+        label: ctx.t("pill.idle"),
+        icon: "outline-wave-audio",
+        title: ctx.t("pill.whisper"),
+        run: toggle,
+      }
+    }
+
     const status = dictation.state()?.status ?? "idle"
 
     if (status === "starting") {
@@ -350,6 +460,10 @@ export const setup: Plugin.Setup<typeof definition> = (ctx) => {
         { id: "test", title: ctx.t("settings.test"), keywords: "voice microphone microphone test microfono" },
         { id: "voice", title: ctx.t("speech.voice"), keywords: "read aloud voice voz lectura altavoz" },
         { id: "tts-engine", title: ctx.t("speech.engine"), keywords: "voice engine motor edge neural sistema" },
+        { id: "whisper-python", title: ctx.t("whisper.python"), keywords: "whisper python venv entorno" },
+        { id: "whisper-model", title: ctx.t("whisper.model"), keywords: "whisper model modelo" },
+        { id: "whisper-language", title: ctx.t("whisper.language"), keywords: "whisper language idioma" },
+        { id: "whisper-test", title: ctx.t("whisper.test"), keywords: "whisper check comprobar" },
         { id: "rate", title: ctx.t("speech.rate"), keywords: "read aloud speed velocidad lectura" },
         { id: "skip-code", title: ctx.t("speech.skipCode"), keywords: "read aloud code codigo lectura" },
         { id: "preview", title: ctx.t("speech.preview"), keywords: "read aloud test voice probar voz" },

package/src/index.ts

@@ -1,19 +1,50 @@
-import type { Plugin } from "@opencode/plugin"
+import { Plugin } from "@opencode/plugin"
+import { listEdgeVoices, synthesizeEdge } from "./edge.js"
+import { VoiceRpc } from "./rpc.js"
+import { WhisperEngine } from "./whisper/server.js"
 
 /**
  * Server part of the plugin.
  *
- * It does nothing on its own: it exists so the desktop app assigns the plugin
- * the id `voice-dictation` (used for commands, settings pages and the Ipc
- * contract) and so the GUI part is loaded from the global `opencode.json`.
- *
- * `@opencode/plugin` is a type-only import on purpose: the server loads this
- * file directly, and a runtime import would make the plugin depend on a
- * `node_modules` tree next to it.
+ * It registers the Edge TTS and Whisper RPCs. Running them here (instead of
+ * in the Electron main process) lets their dependencies resolve from the
+ * plugin's own node_modules.
  */
-const plugin: Plugin.Plugin = {
+export default Plugin.define({
   id: "voice-dictation",
-  setup() {},
-}
+  async setup(ctx) {
+    const whisper = new WhisperEngine()
+
+    await ctx.rpc.register(VoiceRpc, {
+      edgeVoices: async () => listEdgeVoices(),
+      edgeSpeak: async (input) => {
+        const { text, voice, rate } = input as { text: string; voice: string; rate: number }
+        return synthesizeEdge({ text, voice, rate })
+      },
+      whisperTranscribe: async (input) => {
+        return whisper.transcribe(input as {
+          audio: string
+          mime: string
+          language: string
+          python: string
+          model: string
+        })
+      },
+      whisperStatus: async (input) => {
+        const { python, model } = input as { python: string; model: string }
+        try {
+          return await whisper.warm(python, model)
+        } catch (cause) {
+          return {
+            ...whisper.statusOf(),
+            ready: false,
+            error: cause instanceof Error ? cause.message : String(cause),
+          }
+        }
+      },
+    })
+
+    return () => whisper.stop()
+  },
+})
 
-export default plugin

package/src/rpc.ts

@@ -0,0 +1,90 @@
+import type { Rpc } from "@opencode/plugin"
+
+/**
+ * Edge TTS runs in the OpenCode server, where the plugin's own `node_modules`
+ * resolves. The GUI only calls these methods and plays the returned MP3.
+ *
+ * The definition is a plain object (Rpc.define is the identity function), so
+ * the GUI bundle does not need to load the RPC module at runtime.
+ */
+export const VoiceRpc = {
+  id: "voice-dictation.voice",
+  methods: {
+    edgeVoices: {
+      input: { type: "object", additionalProperties: false },
+      output: {
+        type: "array",
+        items: {
+          type: "object",
+          properties: {
+            id: { type: "string" },
+            label: { type: "string" },
+            locale: { type: "string" },
+            gender: { type: "string" },
+          },
+          required: ["id", "label", "locale", "gender"],
+          additionalProperties: false,
+        },
+      },
+    },
+    edgeSpeak: {
+      input: {
+        type: "object",
+        properties: {
+          text: { type: "string" },
+          voice: { type: "string" },
+          rate: { type: "number" },
+        },
+        required: ["text", "voice", "rate"],
+        additionalProperties: false,
+      },
+      output: { type: "string" },
+    },
+    whisperTranscribe: {
+      input: {
+        type: "object",
+        properties: {
+          audio: { type: "string" },
+          mime: { type: "string" },
+          language: { type: "string" },
+          python: { type: "string" },
+          model: { type: "string" },
+        },
+        required: ["audio", "mime", "language", "python", "model"],
+        additionalProperties: false,
+      },
+      output: {
+        type: "object",
+        properties: {
+          text: { type: "string" },
+          language: { type: "string" },
+        },
+        required: ["text", "language"],
+        additionalProperties: false,
+      },
+    },
+    whisperStatus: {
+      input: {
+        type: "object",
+        properties: {
+          python: { type: "string" },
+          model: { type: "string" },
+        },
+        required: ["python", "model"],
+        additionalProperties: false,
+      },
+      output: {
+        type: "object",
+        properties: {
+          ready: { type: "boolean" },
+          model: { type: "string" },
+          device: { type: "string" },
+          error: { type: "string" },
+        },
+        required: ["ready", "model", "device", "error"],
+        additionalProperties: false,
+      },
+    },
+  },
+  events: {},
+} as const satisfies Rpc.PortableDefinition