@goodandready/dsh-voice 0.8.9 → 0.8.11

package/lib/index.js

@@ -21,6 +21,7 @@ import { runChain } from './chain.js'
 import { makeProviders, PROVIDER_KEYS, PRESET_KEYS, KNOWN_KEYS, DEFAULT_MODELS, CUSTOM_TEMPLATES } from './providers.js'
 import { toWav16k } from './wav.js'
 import { normalizePhrase } from './normalize.js'
+import { createStatsTracker, mergeContextVocabulary } from './stats.js'
 import { createUserMessage } from '@deepseek-ai/dsh-llm'
 
 function isAutoLang(lang) {
@@ -123,6 +124,10 @@ export const Config = z.object({
     .description('How many recent dictation inserts to keep for undo in the browser. 0 disables history.'),
   vocabulary: z.array(z.string()).default([])
     .description('Custom words (names, terms) hinted to providers so they recognize them correctly.'),
+  contextGlossary: z.boolean().default(true)
+    .description('Automatically extract code words and terms from context and hint them to STT providers.'),
+  noiseSuppression: z.boolean().default(true)
+    .description('Enable hardware noise suppression, echo cancellation, and auto gain control.'),
   voiceCommands: z.boolean().default(false)
     .description('During dictation, spoken edit commands ("new line", "paragraph") become real line breaks instead of words.'),
   wakeWord: z.string().default('')
@@ -139,6 +144,20 @@ export const Config = z.object({
     .description('Harness polish: provider id. Empty uses the agent default model selection.'),
   polishModelId: z.string().default('')
     .description('Harness polish: model id. Empty uses the agent default model selection.'),
+  sensevoiceUrl: z.string().default('http://127.0.0.1:6006/api/v1/asr')
+    .description('SenseVoice-ONNX / Sherpa-ONNX endpoint: POST /api/v1/asr or OpenAI-compatible /v1/audio/transcriptions.'),
+  sensevoiceBin: z.string().default('sherpa-onnx-offline-http-server')
+    .description('SenseVoice / sherpa-onnx executable binary used when autostart is enabled.'),
+  sensevoiceModel: z.string().default('')
+    .description('SenseVoice / sherpa-onnx model path or model identifier.'),
+  sensevoiceAutostart: z.boolean().default(false)
+    .description('Start the local SenseVoice / sherpa-onnx server process automatically on startup.'),
+  realtimeStreaming: z.boolean().default(false)
+    .description('Enable low-latency streaming recognition (OpenAI Realtime API or Sherpa-ONNX).'),
+  realtimeProvider: z.string().default('openai')
+    .description('Realtime streaming provider: "openai" (OpenAI Realtime API) or "sherpa-onnx" (local streaming).'),
+  realtimeModel: z.string().default('gpt-4o-realtime-preview')
+    .description('Realtime model identifier.'),
 })
 
 const MIME_BY_EXT = {
@@ -227,8 +246,52 @@ export function apply(ctx, baseConfig) {
 
   startWhisper().catch(() => {})
 
+  // SenseVoice-ONNX / Sherpa-ONNX: автозапуск и проверка здоровья.
+  let sensevoiceChild = null
+
+  async function sensevoiceAlive() {
+    try {
+      const cfg = live()
+      const base = String(cfg.sensevoiceUrl || '').replace(/\/api\/.*$/, '').replace(/\/v1\/.*$/, '').replace(/\/+$/, '')
+      if (!base) return false
+      const controller = new AbortController()
+      const t = setTimeout(() => controller.abort(), 2000)
+      const res = await fetch(base + '/', { signal: controller.signal })
+      clearTimeout(t)
+      return res.ok || res.status === 404 // sherpa-onnx отвечает 404 на /, но живой
+    } catch { return false }
+  }
+
+  async function startSensevoice() {
+    const cfg = live()
+    if (!cfg.sensevoiceAutostart) return false
+    if (!cfg.sensevoiceModel) return false
+    if (await sensevoiceAlive()) return true
+    try {
+      const port = new URL(cfg.sensevoiceUrl).port || '6006'
+      const spec = ctx.shell.resolve({
+        command: `${JSON.stringify(cfg.sensevoiceBin)}`
+          + ` --sense-voice-model=${JSON.stringify(cfg.sensevoiceModel)}`
+          + ` --port=${port} --num-threads=4`,
+        timeoutMs: 0,
+        stdoutMaxBytes: 4 * 1024 * 1024,
+      })
+      sensevoiceChild = ctx.shell.start(spec)
+      for (let i = 0; i < 20; i++) {
+        await new Promise((r) => setTimeout(r, 500))
+        if (await sensevoiceAlive()) return true
+      }
+      return false
+    } catch { return false }
+  }
+
+  startSensevoice().catch(() => {})
+
+  // Статистика здоровья и задержек провайдеров (Latency & Health Dashboard).
+  const statsTracker = createStatsTracker()
+
   // Общий путь распознавания: собрать провайдеров по цепочке режима и пройти её.
-  async function transcribe(modeCfg, bytes, mime, signal) {
+  async function transcribe(modeCfg, bytes, mime, signal, contextWords) {
     const cfg = live()
     const customKeys = (Array.isArray(cfg.customProviders) ? cfg.customProviders : [])
       .map((c) => String(c && c.key || '').trim()).filter(Boolean)
@@ -248,11 +311,16 @@ export function apply(ctx, baseConfig) {
     if (cfg.localOnly && order.length === 0) {
       throw new Error('localOnly mode is on, but local-whisper is not in the chain')
     }
+
+    const vocab = cfg.contextGlossary
+      ? mergeContextVocabulary(cfg.vocabulary, contextWords)
+      : (Array.isArray(cfg.vocabulary) ? cfg.vocabulary : [])
+
     const providers = makeProviders(
       { resolveKey, fetchImpl: fetch, cfg, toWav: (b) => toWav16k(b, cfg.ffmpegBin) },
-      { bytes, mime, lang: modeCfg.language, signal, models, vocabulary: cfg.vocabulary },
+      { bytes, mime, lang: modeCfg.language, signal, models, vocabulary: vocab },
     )
-    return runChain(order, providers)
+    return runChain(order, providers, statsTracker.record)
   }
 
   // Полировка транскрипта (#35) с поддержкой локального LLM (#47).
@@ -364,6 +432,14 @@ export function apply(ctx, baseConfig) {
         wakeWord: String(cfg.wakeWord || ''),
         bargeIn: !!cfg.bargeIn,
         polishBaseUrl: String(cfg.polishBaseUrl || ''),
+        noiseSuppression: cfg.noiseSuppression !== false,
+        contextGlossary: cfg.contextGlossary !== false,
+        sensevoiceUrl: cfg.sensevoiceUrl,
+        sensevoiceAutostart: !!cfg.sensevoiceAutostart,
+        realtimeStreaming: !!cfg.realtimeStreaming,
+        realtimeProvider: cfg.realtimeProvider || 'openai',
+        realtimeModel: cfg.realtimeModel || 'gpt-4o-realtime-preview',
+        providerStats: getProviderStats(),
       })
     },
   }), 'dsh-voice: /status route')
@@ -425,11 +501,15 @@ export function apply(ctx, baseConfig) {
       if ((modeCfg.chain || []).some((e) => e.provider === 'local-whisper') && !(await whisperAlive())) {
         await startWhisper()
       }
+      // Аналогично для SenseVoice-ONNX / Sherpa-ONNX.
+      if ((modeCfg.chain || []).some((e) => e.provider === 'sensevoice') && !(await sensevoiceAlive())) {
+        await startSensevoice()
+      }
 
       const controller = new AbortController()
       const timer = setTimeout(() => controller.abort(), cfg.timeoutMs)
       try {
-        const out = await transcribe(modeCfg, bytes, mime, controller.signal)
+        const out = await transcribe(modeCfg, bytes, mime, controller.signal, payload.contextWords)
         // Голосовые команды сессии (#48): чистые «отправь/отмени/стоп/продолжи»
         // не становятся текстом, а возвращаются командой для браузера.
         if (payload.mode === 'message' && modeCfg.sessionCommands === true) {
@@ -499,7 +579,226 @@ export function apply(ctx, baseConfig) {
     }),
   )
 
+  // ─── Realtime WebSocket Bridge ───────────────────────────────────────
+  // Клиент открывает WS на /dsh-voice/realtime, шлёт бинарные аудио-чанки.
+  // Хост открывает WS к провайдеру (OpenAI Realtime API или совместимому),
+  // пересылает аудио и получает текстовые дельты обратно клиенту.
+  // API-ключи никогда не покидают хост.
+  const realtimeSessions = new Set()
+
+  ctx.effect(() => ctx.webServer.register({
+    kind: 'exact',
+    path: '/dsh-voice/realtime',
+    handler: async (req, res) => {
+      const cfg = live()
+      if (!cfg.realtimeStreaming) {
+        writeJson(res, 403, { ok: false, error: { code: 'disabled', message: 'realtime streaming is disabled' } })
+        return
+      }
+      // Только WebSocket upgrade
+      if (!req.headers.upgrade || req.headers.upgrade.toLowerCase() !== 'websocket') {
+        writeJson(res, 426, { ok: false, error: { code: 'upgrade', message: 'WebSocket upgrade required' } })
+        return
+      }
+
+      // Получаем API-ключ для upstream-провайдера
+      const provider = cfg.realtimeProvider || 'openai'
+      let apiKey = ''
+      if (provider === 'openai') {
+        apiKey = await resolveKey('OPENAI_API_KEY')
+      }
+
+      // Определяем upstream WebSocket URL
+      let upstreamUrl
+      const model = cfg.realtimeModel || 'gpt-4o-realtime-preview'
+      if (provider === 'openai') {
+        upstreamUrl = `wss://api.openai.com/v1/realtime?model=${encodeURIComponent(model)}`
+      } else if (provider === 'sherpa-onnx') {
+        // Локальный sherpa-onnx WebSocket endpoint
+        const base = String(cfg.sensevoiceUrl || '').replace(/^http/, 'ws').replace(/\/api\/.*$/, '').replace(/\/v1\/.*$/, '')
+        upstreamUrl = base + '/ws/transcribe'
+      } else {
+        writeJson(res, 400, { ok: false, error: { code: 'provider', message: `unknown realtime provider: ${provider}` } })
+        return
+      }
+
+      // Upgrade клиентский запрос в WebSocket через ручной handshake
+      // Node.js http.Server не имеет встроенного WS — используем raw socket
+      const crypto = await import('node:crypto')
+      const wsKey = req.headers['sec-websocket-key']
+      if (!wsKey) {
+        res.writeHead(400); res.end('Missing Sec-WebSocket-Key'); return
+      }
+      const accept = crypto.createHash('sha1')
+        .update(wsKey + '258EAFA5-E914-47DA-95CA-5AB5ADF35F20')
+        .digest('base64')
+
+      res.writeHead(101, {
+        Upgrade: 'websocket',
+        Connection: 'Upgrade',
+        'Sec-WebSocket-Accept': accept,
+      })
+
+      const clientSocket = req.socket
+
+      // Простой WS frame parser/writer для бинарных и текстовых сообщений
+      function writeWsFrame(socket, data, opcode = 0x01) {
+        const payload = typeof data === 'string' ? Buffer.from(data, 'utf8') : data
+        const len = payload.length
+        let header
+        if (len < 126) {
+          header = Buffer.alloc(2)
+          header[0] = 0x80 | opcode
+          header[1] = len
+        } else if (len < 65536) {
+          header = Buffer.alloc(4)
+          header[0] = 0x80 | opcode
+          header[1] = 126
+          header.writeUInt16BE(len, 2)
+        } else {
+          header = Buffer.alloc(10)
+          header[0] = 0x80 | opcode
+          header[1] = 127
+          header.writeBigUInt64BE(BigInt(len), 2)
+        }
+        socket.write(Buffer.concat([header, payload]))
+      }
+
+      // Открываем upstream WebSocket через глобальный WebSocket (Node 22+)
+      let upstream = null
+      try {
+        const headers = {}
+        if (apiKey) headers['Authorization'] = `Bearer ${apiKey}`
+        if (provider === 'openai') headers['OpenAI-Beta'] = 'realtime=v1'
+        upstream = new WebSocket(upstreamUrl, { headers })
+      } catch (e) {
+        writeWsFrame(clientSocket, JSON.stringify({ type: 'error', message: String(e && e.message || e) }))
+        clientSocket.end()
+        return
+      }
+
+      const sessionId = crypto.randomUUID()
+      realtimeSessions.add(sessionId)
+
+      upstream.addEventListener('open', () => {
+        writeWsFrame(clientSocket, JSON.stringify({ type: 'session.created', sessionId }))
+        // Для OpenAI Realtime — отправить session.update с конфигурацией
+        if (provider === 'openai') {
+          upstream.send(JSON.stringify({
+            type: 'session.update',
+            session: {
+              modalities: ['text'],
+              input_audio_format: 'pcm16',
+              input_audio_transcription: { model: 'whisper-1' },
+              turn_detection: { type: 'server_vad' },
+            },
+          }))
+        }
+      })
+
+      upstream.addEventListener('message', (event) => {
+        // Пересылаем текстовые сообщения от upstream к клиенту
+        const msg = typeof event.data === 'string' ? event.data : event.data.toString()
+        try {
+          const parsed = JSON.parse(msg)
+          // Фильтруем и пересылаем только полезные события
+          if (parsed.type === 'conversation.item.input_audio_transcription.completed'
+            || parsed.type === 'response.audio_transcript.delta'
+            || parsed.type === 'response.audio_transcript.done'
+            || parsed.type === 'input_audio_buffer.speech_started'
+            || parsed.type === 'input_audio_buffer.speech_stopped'
+            || parsed.type === 'error') {
+            writeWsFrame(clientSocket, msg)
+          }
+        } catch {
+          // Не JSON — пересылаем как есть
+          writeWsFrame(clientSocket, msg)
+        }
+      })
+
+      upstream.addEventListener('error', () => {
+        writeWsFrame(clientSocket, JSON.stringify({ type: 'error', message: 'upstream connection error' }))
+      })
+
+      upstream.addEventListener('close', () => {
+        writeWsFrame(clientSocket, JSON.stringify({ type: 'session.closed' }))
+        try { clientSocket.end() } catch { /* socket уже закрыт */ }
+        realtimeSessions.delete(sessionId)
+      })
+
+      // Парсим входящие WS фреймы от клиента
+      let frameBuf = Buffer.alloc(0)
+      clientSocket.on('data', (chunk) => {
+        frameBuf = Buffer.concat([frameBuf, chunk])
+        while (frameBuf.length >= 2) {
+          const opcode = frameBuf[0] & 0x0f
+          const masked = !!(frameBuf[1] & 0x80)
+          let payloadLen = frameBuf[1] & 0x7f
+          let offset = 2
+          if (payloadLen === 126) {
+            if (frameBuf.length < 4) return
+            payloadLen = frameBuf.readUInt16BE(2)
+            offset = 4
+          } else if (payloadLen === 127) {
+            if (frameBuf.length < 10) return
+            payloadLen = Number(frameBuf.readBigUInt64BE(2))
+            offset = 10
+          }
+          if (masked) offset += 4
+          if (frameBuf.length < offset + payloadLen) return
+
+          let payload = frameBuf.subarray(offset, offset + payloadLen)
+          if (masked) {
+            const mask = frameBuf.subarray(offset - 4, offset)
+            payload = Buffer.from(payload)
+            for (let i = 0; i < payload.length; i++) payload[i] ^= mask[i & 3]
+          }
+          frameBuf = frameBuf.subarray(offset + payloadLen)
+
+          if (opcode === 0x08) {
+            // Close frame
+            upstream.close()
+            try { clientSocket.end() } catch { /* ok */ }
+            realtimeSessions.delete(sessionId)
+            return
+          }
+          if (opcode === 0x09) {
+            // Ping → Pong
+            writeWsFrame(clientSocket, payload, 0x0a)
+            continue
+          }
+          if (opcode === 0x01) {
+            // Текст: JSON команда от клиента → upstream
+            try { upstream.send(payload.toString('utf8')) } catch { /* upstream закрыт */ }
+          } else if (opcode === 0x02) {
+            // Бинарные данные: аудио-чанк → upstream как input_audio_buffer.append
+            if (provider === 'openai') {
+              upstream.send(JSON.stringify({
+                type: 'input_audio_buffer.append',
+                audio: payload.toString('base64'),
+              }))
+            } else {
+              // Для sherpa-onnx — отправляем бинарные данные напрямую
+              try { upstream.send(payload) } catch { /* upstream закрыт */ }
+            }
+          }
+        }
+      })
+
+      clientSocket.on('close', () => {
+        try { upstream.close() } catch { /* ok */ }
+        realtimeSessions.delete(sessionId)
+      })
+      clientSocket.on('error', () => {
+        try { upstream.close() } catch { /* ok */ }
+        realtimeSessions.delete(sessionId)
+      })
+    },
+  }), 'dsh-voice: /realtime route')
+
   ctx.effect(() => () => {
     if (child) { try { child.kill && child.kill() } catch { /* уже мёртв */ } }
-  }, 'dsh-voice: stop whisper child')
+    if (sensevoiceChild) { try { sensevoiceChild.kill && sensevoiceChild.kill() } catch { /* уже мёртв */ } }
+    for (const sid of realtimeSessions) realtimeSessions.delete(sid)
+  }, 'dsh-voice: stop child processes')
 }

package/lib/providers.js

@@ -6,7 +6,7 @@
 // браузер, до хоста звук не доходит. Ключ нужен, чтобы такую цепочку принимал
 // и валидатор настроек, и перебор ниже — иначе цепочка ['browser', 'groq'] на
 // хосте оборвалась бы на первом же шаге вместо перехода к groq.
-export const PROVIDER_KEYS = ['browser', 'deepgram', 'groq', 'hf', 'local-whisper']
+export const PROVIDER_KEYS = ['browser', 'deepgram', 'groq', 'hf', 'local-whisper', 'sensevoice']
 
 // Свой провайдер описывается одним из двух шаблонов, потому что
 // OpenAI-совместимые API разошлись: у OpenRouter, например, нет
@@ -84,6 +84,7 @@ export const DEFAULT_MODELS = {
   groq: 'whisper-large-v3-turbo',
   hf: 'openai/whisper-large-v3',
   'local-whisper': '',
+  sensevoice: 'SenseVoiceSmall',
 }
 
 function pickModel(models, key) {
@@ -199,6 +200,65 @@ export function makeProviders(deps, req) {
     return { ok: text.length > 0, provider: 'local-whisper', text, reason: text ? '' : 'empty transcript' }
   }
 
+  // SenseVoice-ONNX / Sherpa-ONNX: сверхбыстрое локальное распознавание (~50-100ms).
+  // Поддерживает эндпоинты sherpa-onnx (/api/v1/asr) и OpenAI-совместимые (/audio/transcriptions).
+  async function sensevoice() {
+    const url = String(cfg.sensevoiceUrl || 'http://127.0.0.1:6006/api/v1/asr').trim()
+    let sendBytes = bytes
+    let sendMime = mime
+    if (!mime.includes('wav')) {
+      if (typeof deps.toWav !== 'function') {
+        return { ok: false, provider: 'sensevoice', reason: 'sensevoice needs WAV, no converter configured' }
+      }
+      try {
+        sendBytes = await deps.toWav(bytes)
+        sendMime = 'audio/wav'
+      } catch (e) {
+        return { ok: false, provider: 'sensevoice', reason: `sensevoice: ${String(e && e.message || e)}` }
+      }
+    }
+    const form = new FormData()
+    form.append('file', new Blob([sendBytes], { type: sendMime }), fileName(sendMime))
+    const isOpenAI = url.includes('/transcriptions')
+    if (isOpenAI) {
+      const model = pickModel(models, 'sensevoice') || 'SenseVoiceSmall'
+      form.append('model', model)
+      form.append('response_format', 'json')
+    }
+    if (!isAutoLang(lang)) form.append('language', lang)
+    if (vocab) form.append('prompt', vocab)
+
+    let res
+    try {
+      res = await fetchImpl(url, { method: 'POST', body: form, signal })
+    } catch (e) {
+      return { ok: false, provider: 'sensevoice', reason: `sensevoice: ${String(e && e.message || e)}` }
+    }
+
+    if (!res.ok) {
+      let detail = `HTTP ${res.status}`
+      try { const e = await res.json(); if (e?.error) detail = e.error } catch { /* тело не json */ }
+      return { ok: false, provider: 'sensevoice', reason: `sensevoice: ${detail}` }
+    }
+
+    let data
+    try {
+      data = await res.json()
+    } catch {
+      return { ok: false, provider: 'sensevoice', reason: 'sensevoice: invalid json response' }
+    }
+
+    let text = String(data?.text || data?.transcript || '').trim()
+    // Очищаем служебные теги эмоций и звуковых событий SenseVoice (<|NEUTRAL|>, <|HAPPY|>, <|Speech|> и др.)
+    text = text.replace(/<\|[^|>]+\|>/g, '').trim()
+    return {
+      ok: text.length > 0,
+      provider: 'sensevoice',
+      text,
+      reason: text ? '' : 'empty transcript',
+    }
+  }
+
   // Свой провайдер. Ключ в цепочке — его имя, поэтому в остальном коде он
   // ничем не отличается от встроенного.
   function customProvider(spec) {
@@ -289,7 +349,7 @@ export function makeProviders(deps, req) {
     }
   }
 
-  const out = { browser, deepgram, groq, hf, 'local-whisper': localWhisper }
+  const out = { browser, deepgram, groq, hf, 'local-whisper': localWhisper, sensevoice }
 
   // Заготовки: те же свои провайдеры, только анкета заполнена заранее.
   for (const key of PRESET_KEYS) {

package/lib/stats.js

@@ -0,0 +1,72 @@
+// dsh-voice — чистые утилиты метрик провайдеров и объединения контекстного словаря.
+// Без cordis и без сети — тестируются юнит-тестами.
+
+export function createStatsTracker() {
+  const stats = new Map()
+
+  function record(key, res = {}) {
+    if (!key) return
+    const cur = stats.get(key) || {
+      attempts: 0,
+      successes: 0,
+      failures: 0,
+      totalTookMs: 0,
+      avgTookMs: 0,
+      lastTookMs: 0,
+      lastError: '',
+      lastSuccessAt: 0,
+      lastErrorAt: 0,
+    }
+    cur.attempts += 1
+    cur.lastTookMs = res.tookMs || 0
+    if (res.ok) {
+      cur.successes += 1
+      cur.totalTookMs += res.tookMs || 0
+      cur.avgTookMs = Math.round(cur.totalTookMs / cur.successes)
+      cur.lastSuccessAt = Date.now()
+    } else {
+      cur.failures += 1
+      cur.lastError = String(res.reason || 'unknown').slice(0, 150)
+      cur.lastErrorAt = Date.now()
+    }
+    stats.set(key, cur)
+  }
+
+  function get() {
+    const out = {}
+    for (const [k, v] of stats.entries()) {
+      out[k] = {
+        attempts: v.attempts,
+        successes: v.successes,
+        failures: v.failures,
+        avgTookMs: v.avgTookMs || v.lastTookMs || 0,
+        lastTookMs: v.lastTookMs || 0,
+        lastError: v.lastError,
+        lastSuccessAt: v.lastSuccessAt,
+        lastErrorAt: v.lastErrorAt,
+      }
+    }
+    return out
+  }
+
+  function clear() {
+    stats.clear()
+  }
+
+  return { record, get, clear }
+}
+
+export function mergeContextVocabulary(baseVocab = [], contextWords = [], maxWords = 50) {
+  const vocab = Array.isArray(baseVocab) ? [...baseVocab] : []
+  if (!Array.isArray(contextWords) || contextWords.length === 0) return vocab
+  const seen = new Set(vocab.map((w) => String(w).toLowerCase()))
+  for (const word of contextWords) {
+    const clean = String(word || '').trim()
+    if (clean && clean.length >= 2 && !seen.has(clean.toLowerCase())) {
+      seen.add(clean.toLowerCase())
+      vocab.push(clean)
+      if (vocab.length >= maxWords) break
+    }
+  }
+  return vocab
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@goodandready/dsh-voice",
-  "version": "0.8.9",
+  "version": "0.8.11",
   "description": "Voice input for DeepSeek Harness: dictation chunked by pauses and voice messages, each with its own provider fallback chain (Deepgram, Groq, HuggingFace, local whisper.cpp, plus any OpenAI-compatible API of your own).",
   "license": "MIT",
   "type": "module",
@@ -61,4 +61,4 @@
     "@deepseek-ai/dsh-llm": "^0.1.0-rc.6",
     "@deepseek-ai/schemastery": "^3.18.1"
   }
-}
+}