@goodandready/dsh-voice 0.9.14 → 0.9.15

package/README.md

@@ -151,12 +151,13 @@ Quality batch after the v0.8.18 review (Gitea #79–#87, PR #88):
 
 | Provider Key | Service Backend | Default Model | Credential Ref | Features & Notes |
 |---|---|---|---|---|
+| `browser-webgpu` | In-Browser WebGPU/WASM Whisper | `onnx-community/whisper-tiny` | *None* | 100% private offline client-side GPU transcription; falls back to server chain |
 | `browser` | Web Speech API | Native Browser | *None* | Zero latency, floating live captions in Chrome (vendor server-based; not offline) |
 | `deepgram` | Deepgram API | `nova-2` | `DEEPGRAM_API_KEY` | Ultra-fast cloud transcription |
 | `groq` | Groq Whisper | `whisper-large-v3-turbo` | `GROQ_API_KEY` | Near-instant inference speed |
 | `hf` | HuggingFace Inference | `openai/whisper-large-v3` | `HF_TOKEN` | High-accuracy open Whisper |
 | `local-whisper` | Local whisper.cpp | Server defined | *None* | 100% private, offline, no internet needed |
-| `sensevoice` | SenseVoice-ONNX / Sherpa-ONNX | `SenseVoiceSmall` | *None* | Ultra-fast (~50ms) local non-autoregressive STT |
+| `sensevoice` | SenseVoice-ONNX / Sherpa-ONNX | `SenseVoiceSmall` | *None* | Ultra-fast (~50ms) local STT; supports Rockchip RK3588 NPU (rknpu), OpenVINO, CUDA |
 
 ### 🚀 Ready-Made Presets (Plug & Play)
 

package/README.ru.md

@@ -151,12 +151,13 @@ Quality-батч после ревью v0.8.18 (Gitea #79–#87, PR #88):
 
 | Ключ | Сервис | Модель по умолчанию | Переменная секрета | Особенности |
 |---|---|---|---|---|
+| `browser-webgpu` | WebGPU/WASM Whisper в браузере | `onnx-community/whisper-tiny` | *Не требуется* | 100% приватное офлайн распознавание на GPU клиента; авто-фоллбек на серверную цепочку |
 | `browser` | Web Speech API | Нативная в браузере | *Не требуется* | Нулевая задержка, живые субтитры в Chrome (обработка на серверах вендора; не является офлайн) |
 | `deepgram` | Deepgram API | `nova-2` | `DEEPGRAM_API_KEY` | Сверхбыстрая облачная транскрипция |
 | `groq` | Groq Whisper | `whisper-large-v3-turbo` | `GROQ_API_KEY` | Мгновенная скорость генерации |
 | `hf` | HuggingFace Inference | `openai/whisper-large-v3` | `HF_TOKEN` | Высокоточный облачный Whisper |
 | `local-whisper` | Локальный whisper.cpp | из параметров сервера | *Не требуется* | 100% приватность, оффлайн, без интернета |
-| `sensevoice` | SenseVoice-ONNX / Sherpa-ONNX | `SenseVoiceSmall` | *Не требуется* | Сверхбыстрый (~50мс) локальный неавторегрессивный STT |
+| `sensevoice` | SenseVoice-ONNX / Sherpa-ONNX | `SenseVoiceSmall` | *Не требуется* | Сверхбыстрый (~50мс) локальный STT; ускорение на Rockchip RK3588 NPU (rknpu), OpenVINO, CUDA |
 
 ### 🚀 Готовые пресеты (Plug & Play)
 

package/lib/client.js

@@ -63,6 +63,14 @@ window.__ModuleLoader__.load({
       'title': 'Voice',
       'recordSlot': 'Voice recording',
       'browserHint': 'Web Speech recognition (uses page/browser language when auto, Chrome sends audio to Google)',
+      'browserWebGpu': 'Browser WebGPU Whisper (Offline)',
+      'browserWebGpuHint': 'Runs Whisper model directly in browser GPU via WebGPU/WASM: offline, no host or cloud upload.',
+      'npuProvider': 'Hardware Acceleration (SBC NPU / Host)',
+      'npuProviderHint': 'Execution provider for SenseVoice / sherpa-onnx: CPU, Rockchip RK3588 NPU (rknpu), OpenVINO, or CUDA.',
+      'npuThreads': 'Threads (CPU/SBC)',
+      'webgpuStatus': 'Browser WebGPU Status',
+      'webgpuAvailable': 'WebGPU Available (hardware acceleration ready)',
+      'webgpuNotAvailable': 'WebGPU Not Detected (will fallback to server)',
       'openaiHint': 'whisper-1 · key OPENAI_API_KEY',
       'siliconflowHint': 'FunAudioLLM/SenseVoiceSmall · key SILICONFLOW_API_KEY',
       'deepinfraHint': 'openai/whisper-large-v3-turbo · key DEEPINFRA_API_KEY',
@@ -270,6 +278,14 @@ window.__ModuleLoader__.load({
       'title': '语音',
       'recordSlot': '语音录制',
       'browserHint': 'Web Speech 语音识别 (设为 auto 时使用页面或浏览器语言，Chrome 会上传音频至 Google)',
+      'browserWebGpu': '浏览器 WebGPU Whisper (离线)',
+      'browserWebGpuHint': '通过 WebGPU/WASM 在浏览器本地运行 Whisper 模型：完全离线，音频无需上传到主机。',
+      'npuProvider': '硬件加速 (SBC NPU / 主机)',
+      'npuProviderHint': 'SenseVoice / sherpa-onnx 执行加速引擎：CPU、Rockchip RK3588 NPU (rknpu)、OpenVINO 或 CUDA。',
+      'npuThreads': '计算线程 (CPU/SBC)',
+      'webgpuStatus': '浏览器 WebGPU 状态',
+      'webgpuAvailable': 'WebGPU 可用 (硬件加速就绪)',
+      'webgpuNotAvailable': '未检测到 WebGPU (将自动回退到服务端)',
       'openaiHint': 'whisper-1 · 密钥 OPENAI_API_KEY',
       'siliconflowHint': 'FunAudioLLM/SenseVoiceSmall · 密钥 SILICONFLOW_API_KEY',
       'deepinfraHint': 'openai/whisper-large-v3-turbo · 密钥 DEEPINFRA_API_KEY',
@@ -717,6 +733,23 @@ window.__ModuleLoader__.load({
     }
 
     async function sendAudio(blob, mime, mode) {
+      const activeChain = (voice._activeChain || [])
+      if (activeChain.includes('browser-webgpu')) {
+        if (typeof isWebGpuSupported === 'function' && isWebGpuSupported()) {
+          try {
+            const out = await transcribeWithWebGpu(blob, mime, {
+              language: voice._activeLang || (voice.settings && voice.settings.dictation && voice.settings.dictation.language) || '',
+              model: voice.settings && voice.settings.webgpuModel,
+            })
+            if (out && typeof out.text === 'string') {
+              return { text: tidyPhrase(out.text), command: null, action: null, provider: 'browser-webgpu' }
+            }
+          } catch (webGpuErr) {
+            // Graceful fallback to server chain
+            console.warn('[dsh-voice] WebGPU Whisper failed, falling back to server chain:', webGpuErr)
+          }
+        }
+      }
       const dataBase64 = await blobToBase64(blob)
       const payload = { dataBase64, mimeType: mime, mode }
       const contextWords = extractContextKeywords()
@@ -1046,6 +1079,78 @@ window.__ModuleLoader__.load({
     // Fetch the mode chain from the host once. Only needed to decide whether
 
     voice._core = { tidyPhrase, applyVoiceCommands, VOICE_COMMANDS, undoLastInsert, insertHistory, insertDraftText, appendDraft, resolveBrowserRecognitionLang }
+// lib/client-src/35-webgpu.js — Client-side WebGPU Whisper offline recognition
+// Detects WebGPU availability in browser and provides offline transcription with graceful fallback.
+
+function isWebGpuSupported() {
+  if (typeof navigator === 'undefined' || !navigator.gpu) return false
+  return true
+}
+
+let _webGpuPipeline = null
+let _webGpuLoading = false
+
+async function loadWebGpuWhisper(modelName) {
+  if (_webGpuPipeline) return _webGpuPipeline
+  if (_webGpuLoading) {
+    while (_webGpuLoading) {
+      await new Promise((r) => setTimeout(r, 80))
+    }
+    if (_webGpuPipeline) return _webGpuPipeline
+  }
+  _webGpuLoading = true
+  try {
+    if (!isWebGpuSupported()) {
+      throw new Error('WebGPU is not supported in this browser environment')
+    }
+    // Dynamic import of transformers.js from ESM CDN
+    const mod = await import('https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.3.3')
+    const pipeline = mod.pipeline || (mod.default && mod.default.pipeline)
+    if (!pipeline) throw new Error('Transformers pipeline not found in loaded bundle')
+    const targetModel = modelName || (voice.settings && voice.settings.webgpuModel) || 'onnx-community/whisper-tiny'
+    _webGpuPipeline = await pipeline('automatic-speech-recognition', targetModel, {
+      device: 'webgpu',
+      dtype: 'fp32',
+    })
+    return _webGpuPipeline
+  } finally {
+    _webGpuLoading = false
+  }
+}
+
+async function audioBlobToFloat32Array(blob) {
+  const arrayBuffer = await blob.arrayBuffer()
+  const AC = typeof AudioContext !== 'undefined' ? AudioContext
+    : (typeof webkitAudioContext !== 'undefined' ? webkitAudioContext : null)
+  if (!AC) throw new Error('AudioContext unavailable for audio decoding')
+  const ctx = new AC({ sampleRate: 16000 })
+  try {
+    const audioBuffer = await ctx.decodeAudioData(arrayBuffer)
+    return audioBuffer.getChannelData(0)
+  } finally {
+    try { await ctx.close() } catch (_) {}
+  }
+}
+
+async function transcribeWithWebGpu(blob, mime, options) {
+  if (!isWebGpuSupported()) {
+    throw new Error('WebGPU unavailable in browser')
+  }
+  const transcriber = await loadWebGpuWhisper(options && options.model)
+  const audioData = await audioBlobToFloat32Array(blob)
+  const generateArgs = {}
+  if (options && options.language && options.language !== 'auto') {
+    generateArgs.language = options.language
+  }
+  const result = await transcriber(audioData, generateArgs)
+  const text = (result && typeof result.text === 'string') ? result.text.trim() : ''
+  return { text, provider: 'browser-webgpu' }
+}
+
+voice.webGpu = {
+  isSupported: isWebGpuSupported,
+  transcribe: transcribeWithWebGpu,
+}
     voice._chainsPromise = null
     function modeChain(mode) {
       if (!voice._chainsPromise) {
@@ -1276,6 +1381,8 @@ window.__ModuleLoader__.load({
       dictationQueue = Promise.resolve()
       voice.set({ phase: 'recording', mode, error: '', levels: [], caption: '', interimTranscript: '' })
       modeChain(mode).then((info) => {
+        voice._activeChain = info.chain || []
+        voice._activeLang = info.language || ''
         if (info.chain[0] === 'browser' && startBrowserLeg(mode, info.language)) return
         openMic(mode)
           .then((rec) => {
@@ -2133,7 +2240,7 @@ window.__ModuleLoader__.load({
 
     // ------------------------------------------------------- settings page
     const BUILTIN = [
-      'browser', 'deepgram', 'groq', 'hf', 'local-whisper', 'sensevoice',
+      'browser', 'browser-webgpu', 'deepgram', 'groq', 'hf', 'local-whisper', 'sensevoice',
       // Presets: address and model are filled on the host; only a key is needed.
       'openai', 'siliconflow', 'deepinfra', 'fireworks', 'mistral', 'openrouter',
     ]
@@ -2141,6 +2248,7 @@ window.__ModuleLoader__.load({
     // Resolve at render time: t() must not be captured before locale bind.
     const MODEL_HINT_KEYS = {
       browser: 'browserHint',
+      'browser-webgpu': 'browserWebGpuHint',
       openai: 'openaiHint',
       siliconflow: 'siliconflowHint',
       deepinfra: 'deepinfraHint',
@@ -2151,6 +2259,7 @@ window.__ModuleLoader__.load({
       sensevoice: 'sensevoiceHint',
     }
     const MODEL_HINT_STATIC = {
+      'browser-webgpu': 'onnx-community/whisper-tiny',
       deepgram: 'nova-2',
       groq: 'whisper-large-v3-turbo',
       hf: 'openai/whisper-large-v3',
@@ -2589,6 +2698,51 @@ window.__ModuleLoader__.load({
             React.createElement('option', { value: 'off' }, t('visOff')),
           ),
         ),
+        // Hardware Acceleration (SBC NPU / Host)
+        React.createElement('div', { className: 'cb-field' },
+          React.createElement('label', null, t('npuProvider')),
+          React.createElement('div', { className: 'cb-row' },
+            React.createElement('select', {
+              value: String((draft && draft.sensevoiceProvider) || 'cpu'),
+              disabled: !writable,
+              onChange: (e) => setTop('sensevoiceProvider', e.target.value),
+              style: { flex: 1 },
+            },
+              React.createElement('option', { value: 'cpu' }, 'CPU (Standard)'),
+              React.createElement('option', { value: 'rknpu' }, 'Rockchip RK3588 NPU (rknpu)'),
+              React.createElement('option', { value: 'openvino' }, 'Intel OpenVINO'),
+              React.createElement('option', { value: 'cuda' }, 'NVIDIA CUDA'),
+            ),
+            React.createElement('input', {
+              type: 'number',
+              min: 1,
+              max: 32,
+              value: Number((draft && draft.sensevoiceThreads) || 4),
+              disabled: !writable,
+              title: t('npuThreads'),
+              style: { width: '80px' },
+              onChange: (e) => setTop('sensevoiceThreads', Math.max(1, Math.min(32, Number(e.target.value) || 4))),
+            }),
+          ),
+          React.createElement('span', { className: 'dvs-sub' }, t('npuProviderHint')),
+        ),
+        // WebGPU Status Indicator
+        React.createElement('div', { className: 'cb-field' },
+          React.createElement('label', null, t('webgpuStatus')),
+          React.createElement('div', {
+            style: {
+              padding: '6px 10px',
+              borderRadius: '6px',
+              fontSize: '12px',
+              backgroundColor: 'var(--dsw-alias-fill-quaternary)',
+              color: (typeof isWebGpuSupported === 'function' && isWebGpuSupported())
+                ? 'var(--dsw-alias-state-success)'
+                : 'var(--dsw-alias-label-secondary)'
+            }
+          }, (typeof isWebGpuSupported === 'function' && isWebGpuSupported())
+              ? ('✓ ' + t('webgpuAvailable'))
+              : ('ℹ ' + t('webgpuNotAvailable'))),
+        ),
         // Vocabulary textarea
         React.createElement('div', { className: 'cb-field' },
           React.createElement('label', null, t('vocabulary')),

package/lib/index.js

@@ -25,7 +25,7 @@ import { createPolishText } from "./polish.js"
 import { createUserMessage } from "@deepseek-ai/dsh-llm"
 import { buildProviderOrder, sessionCommand } from "./transcribe-core.js"
 import { registerPluginUpdater } from "./updater.js"
-import { registerSensevoiceInstaller, findTokensFile } from "./sensevoice-installer.js"
+import { registerSensevoiceInstaller, buildSensevoiceArgs } from "./sensevoice-installer.js"
 import { createConfigReader } from "./config-compat.js"
 import { registerConfigRoutes } from "./config-routes.js"
 import { isAutoLang } from "./provider-http.js"
@@ -202,25 +202,10 @@ export function apply(ctx, baseConfig) {
       }
       let port = '6006'
       try { port = new URL(cfg.sensevoiceUrl).port || '6006' } catch { /* invalid URL */ }
-      const tokensPath = findTokensFile(path.dirname(cfg.sensevoiceModel))
-      const tokensArg = tokensPath ? ` --tokens=${JSON.stringify(tokensPath)}` : ''
-      const spec = typeof ctx.shell?.resolve === 'function'
-        ? ctx.shell.resolve({
-            command: `${JSON.stringify(cfg.sensevoiceBin)}`
-              + ` --sense-voice-model=${JSON.stringify(cfg.sensevoiceModel)}`
-              + tokensArg
-              + ` --port=${port} --num-threads=4`,
-            timeoutMs: 0,
-            stdoutMaxBytes: 4 * 1024 * 1024,
-          })
-        : {
-            command: `${JSON.stringify(cfg.sensevoiceBin)}`
-              + ` --sense-voice-model=${JSON.stringify(cfg.sensevoiceModel)}`
-              + tokensArg
-              + ` --port=${port} --num-threads=4`,
-            timeoutMs: 0,
-            stdoutMaxBytes: 4 * 1024 * 1024,
-          }
+      const cmdArgs = buildSensevoiceArgs(cfg, port)
+      const commandStr = `${JSON.stringify(cfg.sensevoiceBin)}${cmdArgs}`
+      const specOpts = { command: commandStr, timeoutMs: 0, stdoutMaxBytes: 4 * 1024 * 1024 }
+      const spec = typeof ctx.shell?.resolve === 'function' ? ctx.shell.resolve(specOpts) : specOpts
       sensevoiceChild = typeof ctx.shell?.execute === 'function'
         ? ctx.shell.execute(spec)
         : typeof ctx.shell?.start === 'function'
@@ -399,6 +384,9 @@ export function apply(ctx, baseConfig) {
         contextGlossary: cfg.contextGlossary !== false,
         sensevoiceUrl: cfg.sensevoiceUrl,
         sensevoiceAutostart: !!cfg.sensevoiceAutostart,
+        sensevoiceProvider: cfg.sensevoiceProvider || 'cpu',
+        sensevoiceThreads: cfg.sensevoiceThreads || 4,
+        webgpuModel: cfg.webgpuModel || 'onnx-community/whisper-tiny',
         deepgramBaseUrl: cfg.deepgramBaseUrl || 'https://api.deepgram.com',
         visualizerStyle: cfg.visualizerStyle || 'liquid-wave',
         gatedTurnTaking: cfg.gatedTurnTaking !== false,

package/lib/providers.js

@@ -5,7 +5,7 @@
 // 'browser' is listed for chain validation only: recognition runs in the
 // page and audio never reaches the host. Without this key a ['browser','groq']
 // chain would break on the host instead of falling through to groq.
-export const PROVIDER_KEYS = ['browser', 'deepgram', 'groq', 'hf', 'local-whisper', 'sensevoice']
+export const PROVIDER_KEYS = ['browser', 'browser-webgpu', 'deepgram', 'groq', 'hf', 'local-whisper', 'sensevoice']
 
 // Custom providers use one of two templates because OpenAI-compatible APIs
 // diverged: OpenRouter has no /audio/transcriptions and recognizes via chat.
@@ -342,8 +342,8 @@ export function makeProviders(deps, req) {
     }
   }
 
-  // If audio still reached the host with 'browser' in the chain, the browser
-  // leg failed: fail clearly and move on.
+  // If audio still reached the host with 'browser' or 'browser-webgpu' in the chain,
+  // the client-side leg failed: fail clearly and move on.
   async function browser() {
     return {
       ok: false,
@@ -352,7 +352,15 @@ export function makeProviders(deps, req) {
     }
   }
 
-  const out = { browser, deepgram, groq, hf, 'local-whisper': localWhisper, sensevoice }
+  async function browserWebGpu() {
+    return {
+      ok: false,
+      provider: 'browser-webgpu',
+      reason: 'browser-webgpu: recognition runs in the page via WebGPU/WASM, the host has no browser engine',
+    }
+  }
+
+  const out = { browser, 'browser-webgpu': browserWebGpu, deepgram, groq, hf, 'local-whisper': localWhisper, sensevoice }
 
   // Presets: custom providers with the form pre-filled.
   for (const key of PRESET_KEYS) {

package/lib/schema.js

@@ -7,8 +7,9 @@ export const ChainEntry = z.object({
     .description(`Provider key. Built in: ${PROVIDER_KEYS.join(', ')}. `
       + `Ready-made, just add the key: ${PRESET_KEYS.join(', ')}. `
       + 'Or the name of an entry from customProviders. '
-      + '"browser" recognises speech in the page itself — no key, no upload to this host, '
-      + 'text appears while you speak; put a normal provider after it as a fallback.'),
+      + '"browser" recognises speech via Web Speech API; '
+      + '"browser-webgpu" runs offline Whisper directly in the page via WebGPU/WASM; '
+      + 'put a normal host provider after them as a fallback.'),
   model: z.string().default('')
     .description('Model override. Empty means the provider default.'),
 })
@@ -132,6 +133,12 @@ export const BaseConfig = z.object({
     .description('SenseVoice / sherpa-onnx model path or model identifier.'),
   sensevoiceAutostart: z.boolean().default(false)
     .description('Start the local SenseVoice / sherpa-onnx server process automatically on startup.'),
+  sensevoiceProvider: z.string().default('cpu')
+    .description('Execution provider for sherpa-onnx: "cpu", "rknpu" (Rockchip RK3588/RK3568 NPU on ARM SBC), "openvino", "cuda".'),
+  sensevoiceThreads: z.number().default(4)
+    .description('Execution thread count for local SenseVoice / sherpa-onnx on CPU/SBC.'),
+  webgpuModel: z.string().default('onnx-community/whisper-tiny')
+    .description('In-browser offline WebGPU Whisper model identifier (e.g. "onnx-community/whisper-tiny").'),
   visualizerStyle: z.union(['liquid-wave', 'dynamic-orb', 'bars', 'off']).default('liquid-wave')
     .description('Audio visualizer animation style in the recording pill: "liquid-wave", "dynamic-orb", "bars", or "off".'),
   gatedTurnTaking: z.boolean().default(true)

package/lib/sensevoice-installer.js

@@ -230,3 +230,17 @@ export function registerSensevoiceInstaller(ctx, { liveConfig, isAlive, updateCo
     },
   })
 }
+
+export function buildSensevoiceArgs(cfg, port) {
+  const modelDir = cfg?.sensevoiceModel ? path.dirname(cfg.sensevoiceModel) : getDefaultModelDir()
+  const tokensPath = findTokensFile(modelDir)
+  const tokensArg = tokensPath ? ` --tokens=${JSON.stringify(tokensPath)}` : ''
+  const provider = (cfg?.sensevoiceProvider && cfg.sensevoiceProvider !== 'cpu')
+    ? ` --provider=${cfg.sensevoiceProvider}`
+    : ''
+  const threads = Number(cfg?.sensevoiceThreads) || 4
+  return ` --sense-voice-model=${JSON.stringify(cfg?.sensevoiceModel || '')}`
+    + tokensArg
+    + ` --port=${port} --num-threads=${threads}`
+    + provider
+}

package/lib/transcribe-core.js

@@ -11,7 +11,7 @@
  * @returns {{order: string[], models: Record<string, string>}}
  */
 export function buildProviderOrder({ localOnly, chain, customKeys, knownKeys, defaultModels }) {
-  const localEngines = new Set(['local-whisper', 'sensevoice'])
+  const localEngines = new Set(['local-whisper', 'sensevoice', 'browser-webgpu'])
   const source = localOnly
     ? (chain || []).filter((e) => e && localEngines.has(e.provider))
     : (chain || [])

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@goodandready/dsh-voice",
-  "version": "0.9.14",
+  "version": "0.9.15",
   "description": "Voice input for DeepSeek Harness: dictation chunked by pauses and voice messages, each with its own provider fallback chain (Deepgram, Groq, HuggingFace, local whisper.cpp, plus any OpenAI-compatible API of your own).",
   "license": "MIT",
   "type": "module",