@goodandready/dsh-voice 0.9.47 → 0.9.48
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/local-daemon.js +22 -0
- package/lib/provider-registry.js +502 -0
- package/lib/providers.js +37 -326
- package/lib/supervisor.js +154 -0
- package/package.json +1 -1
package/lib/local-daemon.js
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
// lib/local-daemon.js — Local background process daemon management for whisper and sensevoice
|
|
2
2
|
import { isAutoLang } from './provider-http.js'
|
|
3
|
+
import { ProcessSupervisor, DAEMON_STATES } from './supervisor.js'
|
|
4
|
+
export { ProcessSupervisor, DAEMON_STATES }
|
|
3
5
|
import { buildSensevoiceArgs } from './sensevoice-installer.js'
|
|
4
6
|
|
|
5
7
|
export function createLocalDaemons({ live, ctx }) {
|
|
@@ -7,6 +9,11 @@ export function createLocalDaemons({ live, ctx }) {
|
|
|
7
9
|
let whisperError = null
|
|
8
10
|
let startingWhisper = false
|
|
9
11
|
let disposed = false
|
|
12
|
+
const whisperSupervisor = new ProcessSupervisor({ name: 'whisper', healthcheck: () => whisperAlive() })
|
|
13
|
+
const sensevoiceSupervisor = new ProcessSupervisor({ name: 'sensevoice', healthcheck: () => sensevoiceAlive() })
|
|
14
|
+
if (ctx && typeof ctx.on === 'function') {
|
|
15
|
+
try { ctx.on('dispose', () => dispose()) } catch { /* ignore */ }
|
|
16
|
+
}
|
|
10
17
|
|
|
11
18
|
async function whisperAlive() {
|
|
12
19
|
const rawUrl = String(live().whisperUrl || '').trim()
|
|
@@ -29,8 +36,10 @@ export function createLocalDaemons({ live, ctx }) {
|
|
|
29
36
|
if (!cfg.whisperModel) return false
|
|
30
37
|
if (await whisperAlive()) {
|
|
31
38
|
whisperError = null
|
|
39
|
+
whisperSupervisor.transition(DAEMON_STATES.RUNNING, 'already alive')
|
|
32
40
|
return true
|
|
33
41
|
}
|
|
42
|
+
whisperSupervisor.transition(DAEMON_STATES.STARTING, 'spawning whisper')
|
|
34
43
|
if (child && typeof child.kill === 'function') {
|
|
35
44
|
try { child.kill() } catch { /* ignore */ }
|
|
36
45
|
child = null
|
|
@@ -119,6 +128,8 @@ export function createLocalDaemons({ live, ctx }) {
|
|
|
119
128
|
if (disposed) return false
|
|
120
129
|
if (await whisperAlive()) {
|
|
121
130
|
whisperError = null
|
|
131
|
+
whisperSupervisor.attachHandle(child)
|
|
132
|
+
whisperSupervisor.transition(DAEMON_STATES.RUNNING, 'healthcheck passed')
|
|
122
133
|
return true
|
|
123
134
|
}
|
|
124
135
|
if (child) {
|
|
@@ -208,8 +219,10 @@ export function createLocalDaemons({ live, ctx }) {
|
|
|
208
219
|
if (!cfg.sensevoiceModel) return false
|
|
209
220
|
if (await sensevoiceAlive()) {
|
|
210
221
|
sensevoiceError = null
|
|
222
|
+
sensevoiceSupervisor.transition(DAEMON_STATES.RUNNING, 'already alive')
|
|
211
223
|
return true
|
|
212
224
|
}
|
|
225
|
+
sensevoiceSupervisor.transition(DAEMON_STATES.STARTING, 'spawning sensevoice')
|
|
213
226
|
if (sensevoiceChild && typeof sensevoiceChild.kill === 'function') {
|
|
214
227
|
try { sensevoiceChild.kill() } catch { /* ignore */ }
|
|
215
228
|
sensevoiceChild = null
|
|
@@ -283,6 +296,8 @@ export function createLocalDaemons({ live, ctx }) {
|
|
|
283
296
|
if (disposed) return false
|
|
284
297
|
if (await sensevoiceAlive()) {
|
|
285
298
|
sensevoiceError = null
|
|
299
|
+
sensevoiceSupervisor.attachHandle(sensevoiceChild)
|
|
300
|
+
sensevoiceSupervisor.transition(DAEMON_STATES.RUNNING, 'healthcheck passed')
|
|
286
301
|
return true
|
|
287
302
|
}
|
|
288
303
|
if (sensevoiceChild) {
|
|
@@ -338,6 +353,7 @@ export function createLocalDaemons({ live, ctx }) {
|
|
|
338
353
|
}
|
|
339
354
|
}
|
|
340
355
|
sensevoiceError = 'sensevoice process started but failed health check'
|
|
356
|
+
sensevoiceSupervisor.transition(DAEMON_STATES.DEGRADED, sensevoiceError)
|
|
341
357
|
return false
|
|
342
358
|
} catch (err) {
|
|
343
359
|
sensevoiceError = String(err && err.message ? err.message : err)
|
|
@@ -358,6 +374,8 @@ export function createLocalDaemons({ live, ctx }) {
|
|
|
358
374
|
|
|
359
375
|
function dispose() {
|
|
360
376
|
disposed = true
|
|
377
|
+
whisperSupervisor.dispose()
|
|
378
|
+
sensevoiceSupervisor.dispose()
|
|
361
379
|
if (child && typeof child.kill === 'function') {
|
|
362
380
|
try { child.kill() } catch { /* ignore */ }
|
|
363
381
|
child = null
|
|
@@ -372,9 +390,13 @@ export function createLocalDaemons({ live, ctx }) {
|
|
|
372
390
|
whisperAlive,
|
|
373
391
|
startWhisper,
|
|
374
392
|
getWhisperError: () => whisperError,
|
|
393
|
+
getWhisperState: () => (whisperError ? DAEMON_STATES.CRASHED : whisperSupervisor.state),
|
|
394
|
+
whisperSupervisor,
|
|
375
395
|
sensevoiceAlive,
|
|
376
396
|
startSensevoice,
|
|
377
397
|
getSensevoiceError: () => sensevoiceError,
|
|
398
|
+
getSensevoiceState: () => (sensevoiceError ? DAEMON_STATES.CRASHED : sensevoiceSupervisor.state),
|
|
399
|
+
sensevoiceSupervisor,
|
|
378
400
|
triggerAutostart,
|
|
379
401
|
dispose,
|
|
380
402
|
}
|
|
@@ -0,0 +1,502 @@
|
|
|
1
|
+
// lib/provider-registry.js — Strategy pattern and extensible registry for STT providers (#306)
|
|
2
|
+
import { readHttpError, safeJson, isAutoLang, fileName, chatAudioFormat } from './provider-http.js'
|
|
3
|
+
|
|
4
|
+
const CHAT_AUDIO_PROMPT =
|
|
5
|
+
'Transcribe the audio verbatim. Reply with the transcript text only, '
|
|
6
|
+
+ 'without comments, quotes or formatting.'
|
|
7
|
+
|
|
8
|
+
function bindNamedRunner(name, fn) {
|
|
9
|
+
Object.defineProperty(fn, 'name', { value: name, configurable: true })
|
|
10
|
+
return fn
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Abstract base strategy for Speech-to-Text providers.
|
|
15
|
+
*/
|
|
16
|
+
export class SpeechToTextProvider {
|
|
17
|
+
/**
|
|
18
|
+
* @param {{ key: string, kind?: 'browser' | 'cloud' | 'local' | 'custom', defaultModel?: string, requiresWav?: boolean }} meta
|
|
19
|
+
*/
|
|
20
|
+
constructor(meta) {
|
|
21
|
+
if (!meta || !meta.key) throw new Error('SpeechToTextProvider requires a non-empty key')
|
|
22
|
+
this.key = String(meta.key)
|
|
23
|
+
this.kind = meta.kind || 'cloud'
|
|
24
|
+
this.defaultModel = meta.defaultModel || ''
|
|
25
|
+
this.requiresWav = Boolean(meta.requiresWav)
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
pickModel(models) {
|
|
29
|
+
const chosen = models && typeof models[this.key] === 'string' ? models[this.key].trim() : ''
|
|
30
|
+
return chosen || this.defaultModel
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Validate configuration for this provider.
|
|
35
|
+
* @param {Record<string, any>} _cfg
|
|
36
|
+
* @returns {{ valid: boolean, reason?: string }}
|
|
37
|
+
*/
|
|
38
|
+
validateConfig(_cfg) {
|
|
39
|
+
return { valid: true }
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Check provider readiness/health.
|
|
44
|
+
* @param {Record<string, any>} _deps
|
|
45
|
+
* @returns {Promise<{ ok: boolean, reason?: string }>}
|
|
46
|
+
*/
|
|
47
|
+
async healthcheck(_deps) {
|
|
48
|
+
return { ok: true }
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Execute speech recognition strategy.
|
|
53
|
+
* @param {Record<string, any>} _deps
|
|
54
|
+
* @param {Record<string, any>} _req
|
|
55
|
+
* @returns {Promise<{ ok: boolean, provider: string, text?: string, reason?: string }>}
|
|
56
|
+
*/
|
|
57
|
+
async transcribe(_deps, _req) {
|
|
58
|
+
return { ok: false, provider: this.key, reason: `${this.key}: not implemented` }
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Create a zero-arg chain runner bound to deps and request context.
|
|
63
|
+
*/
|
|
64
|
+
createRunner(deps, req) {
|
|
65
|
+
return bindNamedRunner(this.key, async () => this.transcribe(deps, req))
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export class BrowserStubProvider extends SpeechToTextProvider {
|
|
70
|
+
constructor(key = 'browser', reasonSuffix = 'recognition runs in the page, the host has no browser engine') {
|
|
71
|
+
super({ key, kind: 'browser' })
|
|
72
|
+
this.reasonSuffix = reasonSuffix
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
async transcribe() {
|
|
76
|
+
return {
|
|
77
|
+
ok: false,
|
|
78
|
+
provider: this.key,
|
|
79
|
+
reason: `${this.key}: ${this.reasonSuffix}`,
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
export class DeepgramProvider extends SpeechToTextProvider {
|
|
85
|
+
constructor(defaultModel = 'nova-2') {
|
|
86
|
+
super({ key: 'deepgram', kind: 'cloud', defaultModel })
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
validateConfig(cfg) {
|
|
90
|
+
if (!cfg?.deepgramKeyEnv) return { valid: false, reason: 'missing deepgramKeyEnv' }
|
|
91
|
+
return { valid: true }
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
async healthcheck({ resolveKey, cfg }) {
|
|
95
|
+
const key = await resolveKey?.(cfg?.deepgramKeyEnv)
|
|
96
|
+
return key ? { ok: true } : { ok: false, reason: `no ${cfg?.deepgramKeyEnv || 'DEEPGRAM_API_KEY'}` }
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
async transcribe(deps, req) {
|
|
100
|
+
const { resolveKey, fetchImpl, cfg } = deps
|
|
101
|
+
const { bytes, mime, lang, signal, models, effectiveKeywords } = req
|
|
102
|
+
const key = await resolveKey(cfg.deepgramKeyEnv)
|
|
103
|
+
if (!key) return { ok: false, provider: 'deepgram', reason: `no ${cfg.deepgramKeyEnv}` }
|
|
104
|
+
const model = this.pickModel(models)
|
|
105
|
+
const base = String(cfg.deepgramBaseUrl || 'https://api.deepgram.com').replace(/\/+$/, '')
|
|
106
|
+
const url = `${base}/v1/listen?model=${encodeURIComponent(model)}`
|
|
107
|
+
+ (!isAutoLang(lang) ? `&language=${encodeURIComponent(lang)}` : '')
|
|
108
|
+
+ (effectiveKeywords ? `&keywords=${encodeURIComponent(effectiveKeywords)}` : '')
|
|
109
|
+
+ '&smart_format=true'
|
|
110
|
+
const res = await fetchImpl(url, {
|
|
111
|
+
method: 'POST',
|
|
112
|
+
headers: { authorization: `Token ${key}`, 'content-type': mime },
|
|
113
|
+
body: bytes,
|
|
114
|
+
signal,
|
|
115
|
+
keepalive: true,
|
|
116
|
+
})
|
|
117
|
+
if (!res.ok) throw await readHttpError(res, 'Deepgram', 'deepgram')
|
|
118
|
+
let data
|
|
119
|
+
try { data = await safeJson(res, 'Deepgram') } catch (e) {
|
|
120
|
+
return { ok: false, provider: 'deepgram', reason: e.message }
|
|
121
|
+
}
|
|
122
|
+
const text = (data?.results?.channels?.[0]?.alternatives?.[0]?.transcript || '').trim()
|
|
123
|
+
return { ok: text.length > 0, provider: 'deepgram', text, reason: text ? '' : 'empty transcript' }
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
export class GroqProvider extends SpeechToTextProvider {
|
|
128
|
+
constructor(defaultModel = 'whisper-large-v3-turbo') {
|
|
129
|
+
super({ key: 'groq', kind: 'cloud', defaultModel })
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
validateConfig(cfg) {
|
|
133
|
+
if (!cfg?.groqKeyEnv) return { valid: false, reason: 'missing groqKeyEnv' }
|
|
134
|
+
return { valid: true }
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
async healthcheck({ resolveKey, cfg }) {
|
|
138
|
+
const key = await resolveKey?.(cfg?.groqKeyEnv)
|
|
139
|
+
return key ? { ok: true } : { ok: false, reason: `no ${cfg?.groqKeyEnv || 'GROQ_API_KEY'}` }
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
async transcribe(deps, req) {
|
|
143
|
+
const { resolveKey, fetchImpl, cfg } = deps
|
|
144
|
+
const { bytes, mime, lang, signal, models, effectivePrompt } = req
|
|
145
|
+
const key = await resolveKey(cfg.groqKeyEnv)
|
|
146
|
+
if (!key) return { ok: false, provider: 'groq', reason: `no ${cfg.groqKeyEnv}` }
|
|
147
|
+
const form = new FormData()
|
|
148
|
+
form.append('file', new Blob([bytes], { type: mime }), fileName(mime))
|
|
149
|
+
form.append('model', this.pickModel(models))
|
|
150
|
+
if (!isAutoLang(lang)) form.append('language', lang)
|
|
151
|
+
if (effectivePrompt) form.append('prompt', effectivePrompt)
|
|
152
|
+
form.append('response_format', 'json')
|
|
153
|
+
const res = await fetchImpl('https://api.groq.com/openai/v1/audio/transcriptions', {
|
|
154
|
+
method: 'POST',
|
|
155
|
+
headers: { authorization: `Bearer ${key}` },
|
|
156
|
+
body: form,
|
|
157
|
+
signal,
|
|
158
|
+
keepalive: true,
|
|
159
|
+
})
|
|
160
|
+
if (!res.ok) throw await readHttpError(res, 'Groq', 'groq')
|
|
161
|
+
let data
|
|
162
|
+
try { data = await safeJson(res, 'Groq') } catch (e) {
|
|
163
|
+
return { ok: false, provider: 'groq', reason: e.message }
|
|
164
|
+
}
|
|
165
|
+
const text = (data?.text || '').trim()
|
|
166
|
+
return { ok: text.length > 0, provider: 'groq', text, reason: text ? '' : 'empty transcript' }
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
export class HuggingFaceProvider extends SpeechToTextProvider {
|
|
171
|
+
constructor(defaultModel = 'openai/whisper-large-v3') {
|
|
172
|
+
super({ key: 'hf', kind: 'cloud', defaultModel })
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
validateConfig(cfg) {
|
|
176
|
+
if (!cfg?.hfTokenEnv) return { valid: false, reason: 'missing hfTokenEnv' }
|
|
177
|
+
return { valid: true }
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
async healthcheck({ resolveKey, cfg }) {
|
|
181
|
+
const token = await resolveKey?.(cfg?.hfTokenEnv)
|
|
182
|
+
return token ? { ok: true } : { ok: false, reason: `no ${cfg?.hfTokenEnv || 'HF_TOKEN'}` }
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
async transcribe(deps, req) {
|
|
186
|
+
const { resolveKey, fetchImpl, cfg } = deps
|
|
187
|
+
const { bytes, mime, signal, models } = req
|
|
188
|
+
const token = await resolveKey(cfg.hfTokenEnv)
|
|
189
|
+
if (!token) return { ok: false, provider: 'hf', reason: `no ${cfg.hfTokenEnv}` }
|
|
190
|
+
const model = this.pickModel(models)
|
|
191
|
+
const res = await fetchImpl(`https://router.huggingface.co/hf-inference/models/${model}`, {
|
|
192
|
+
method: 'POST',
|
|
193
|
+
headers: { authorization: `Bearer ${token}`, 'content-type': mime, 'x-wait-for-model': 'true' },
|
|
194
|
+
body: bytes,
|
|
195
|
+
signal,
|
|
196
|
+
keepalive: true,
|
|
197
|
+
})
|
|
198
|
+
if (!res.ok) throw await readHttpError(res, 'HF', 'hf')
|
|
199
|
+
let data
|
|
200
|
+
try { data = await safeJson(res, 'HF') } catch (e) {
|
|
201
|
+
return { ok: false, provider: 'hf', reason: e.message }
|
|
202
|
+
}
|
|
203
|
+
const text = (data?.text || '').trim()
|
|
204
|
+
return { ok: text.length > 0, provider: 'hf', text, reason: text ? '' : 'empty transcript' }
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
export class LocalWhisperProvider extends SpeechToTextProvider {
|
|
209
|
+
constructor() {
|
|
210
|
+
super({ key: 'local-whisper', kind: 'local', defaultModel: '', requiresWav: true })
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
validateConfig(cfg) {
|
|
214
|
+
if (!cfg?.whisperUrl) return { valid: false, reason: 'missing whisperUrl' }
|
|
215
|
+
return { valid: true }
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
async transcribe(deps, req) {
|
|
219
|
+
const { fetchImpl, cfg } = deps
|
|
220
|
+
const { bytes, mime, lang, signal, effectivePrompt } = req
|
|
221
|
+
let sendBytes = bytes
|
|
222
|
+
let sendMime = mime
|
|
223
|
+
if (!mime.includes('wav')) {
|
|
224
|
+
if (typeof deps.toWav !== 'function') {
|
|
225
|
+
return { ok: false, provider: 'local-whisper', reason: 'local whisper needs WAV, no converter configured' }
|
|
226
|
+
}
|
|
227
|
+
try {
|
|
228
|
+
sendBytes = await deps.toWav(bytes, signal)
|
|
229
|
+
sendMime = 'audio/wav'
|
|
230
|
+
} catch (e) {
|
|
231
|
+
return { ok: false, provider: 'local-whisper', reason: `local whisper: ${String(e && e.message || e)}` }
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
const form = new FormData()
|
|
235
|
+
form.append('file', new Blob([sendBytes], { type: sendMime }), fileName(sendMime))
|
|
236
|
+
form.append('language', isAutoLang(lang) ? 'auto' : lang)
|
|
237
|
+
form.append('translate', 'false')
|
|
238
|
+
if (effectivePrompt) form.append('prompt', effectivePrompt)
|
|
239
|
+
form.append('response_format', 'json')
|
|
240
|
+
const res = await fetchImpl(cfg.whisperUrl, { method: 'POST', body: form, signal })
|
|
241
|
+
if (!res.ok) {
|
|
242
|
+
let detail = `HTTP ${res.status}`
|
|
243
|
+
try { const e = await res.json(); if (e?.error) detail = e.error } catch { /* body is not json */ }
|
|
244
|
+
return { ok: false, provider: 'local-whisper', reason: `local whisper: ${detail}` }
|
|
245
|
+
}
|
|
246
|
+
let data
|
|
247
|
+
try { data = await safeJson(res, 'local whisper') } catch (e) {
|
|
248
|
+
return { ok: false, provider: 'local-whisper', reason: e.message }
|
|
249
|
+
}
|
|
250
|
+
const text = (data?.text || '').trim()
|
|
251
|
+
return { ok: text.length > 0, provider: 'local-whisper', text, reason: text ? '' : 'empty transcript' }
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
export class SenseVoiceProvider extends SpeechToTextProvider {
|
|
256
|
+
constructor(defaultModel = 'SenseVoiceSmall') {
|
|
257
|
+
super({ key: 'sensevoice', kind: 'local', defaultModel, requiresWav: true })
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
validateConfig(cfg) {
|
|
261
|
+
if (!cfg?.sensevoiceUrl) return { valid: false, reason: 'missing sensevoiceUrl' }
|
|
262
|
+
return { valid: true }
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
async transcribe(deps, req) {
|
|
266
|
+
const { fetchImpl, cfg } = deps
|
|
267
|
+
const { bytes, mime, lang, signal, models, effectivePrompt } = req
|
|
268
|
+
const url = String(cfg.sensevoiceUrl || 'http://127.0.0.1:6006/api/v1/asr').trim()
|
|
269
|
+
let sendBytes = bytes
|
|
270
|
+
let sendMime = mime
|
|
271
|
+
if (!mime.includes('wav')) {
|
|
272
|
+
if (typeof deps.toWav !== 'function') {
|
|
273
|
+
return { ok: false, provider: 'sensevoice', reason: 'sensevoice needs WAV, no converter configured' }
|
|
274
|
+
}
|
|
275
|
+
try {
|
|
276
|
+
sendBytes = await deps.toWav(bytes, signal)
|
|
277
|
+
sendMime = 'audio/wav'
|
|
278
|
+
} catch (e) {
|
|
279
|
+
return { ok: false, provider: 'sensevoice', reason: `sensevoice: ${String(e && e.message || e)}` }
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
const form = new FormData()
|
|
283
|
+
form.append('file', new Blob([sendBytes], { type: sendMime }), fileName(sendMime))
|
|
284
|
+
const isOpenAI = url.includes('/transcriptions')
|
|
285
|
+
if (isOpenAI) {
|
|
286
|
+
form.append('model', this.pickModel(models))
|
|
287
|
+
if (!isAutoLang(lang)) form.append('language', lang)
|
|
288
|
+
if (effectivePrompt) form.append('prompt', effectivePrompt)
|
|
289
|
+
form.append('response_format', 'json')
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
let res
|
|
293
|
+
try {
|
|
294
|
+
res = await fetchImpl(url, { method: 'POST', body: form, signal })
|
|
295
|
+
} catch (e) {
|
|
296
|
+
return { ok: false, provider: 'sensevoice', reason: `sensevoice: ${String(e && e.message || e)}` }
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
if (!res.ok) {
|
|
300
|
+
let detail = `HTTP ${res.status}`
|
|
301
|
+
try { const e = await res.json(); if (e?.error) detail = e.error } catch { /* body is not json */ }
|
|
302
|
+
return { ok: false, provider: 'sensevoice', reason: `sensevoice: ${detail}` }
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
let data
|
|
306
|
+
try {
|
|
307
|
+
data = await res.json()
|
|
308
|
+
} catch {
|
|
309
|
+
return { ok: false, provider: 'sensevoice', reason: 'sensevoice: invalid json response' }
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
let text = String(data?.text || data?.transcript || '').trim()
|
|
313
|
+
text = text.replace(/<\|[^|>]+\|>/g, '').trim()
|
|
314
|
+
return {
|
|
315
|
+
ok: text.length > 0,
|
|
316
|
+
provider: 'sensevoice',
|
|
317
|
+
text,
|
|
318
|
+
reason: text ? '' : 'empty transcript',
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
export class OpenAICompatibleProvider extends SpeechToTextProvider {
|
|
324
|
+
/**
|
|
325
|
+
* @param {{ key: string, template?: string, baseURL?: string, model?: string, keyEnv?: string, prompt?: string }} spec
|
|
326
|
+
*/
|
|
327
|
+
constructor(spec) {
|
|
328
|
+
super({
|
|
329
|
+
key: spec.key,
|
|
330
|
+
kind: 'custom',
|
|
331
|
+
defaultModel: spec.model || '',
|
|
332
|
+
})
|
|
333
|
+
this.spec = spec
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
validateConfig() {
|
|
337
|
+
if (!this.spec?.baseURL) return { valid: false, reason: `${this.key}: no baseURL` }
|
|
338
|
+
if (!this.spec?.model) return { valid: false, reason: `${this.key}: no model` }
|
|
339
|
+
return { valid: true }
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
async transcribe(deps, req) {
|
|
343
|
+
const { resolveKey, fetchImpl } = deps
|
|
344
|
+
const { bytes, mime, lang, signal, models, effectivePrompt } = req
|
|
345
|
+
const label = this.key
|
|
346
|
+
const base = String(this.spec.baseURL || '').replace(/\/+$/, '')
|
|
347
|
+
const chosen = models && typeof models[label] === 'string' ? models[label].trim() : ''
|
|
348
|
+
const model = chosen || this.spec.model
|
|
349
|
+
|
|
350
|
+
if (!base) return { ok: false, provider: label, reason: `${label}: no baseURL` }
|
|
351
|
+
if (!model) return { ok: false, provider: label, reason: `${label}: no model` }
|
|
352
|
+
|
|
353
|
+
let headers = {}
|
|
354
|
+
if (this.spec.keyEnv) {
|
|
355
|
+
const key = await resolveKey(this.spec.keyEnv)
|
|
356
|
+
if (!key) return { ok: false, provider: label, reason: `no ${this.spec.keyEnv}` }
|
|
357
|
+
headers = { authorization: `Bearer ${key}` }
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
const viaTranscriptions = async () => {
|
|
361
|
+
const form = new FormData()
|
|
362
|
+
form.append('file', new Blob([bytes], { type: mime }), fileName(mime))
|
|
363
|
+
form.append('model', model)
|
|
364
|
+
if (lang && lang !== 'auto') form.append('language', lang)
|
|
365
|
+
if (effectivePrompt) form.append('prompt', effectivePrompt)
|
|
366
|
+
form.append('response_format', 'json')
|
|
367
|
+
const res = await fetchImpl(`${base}/audio/transcriptions`, {
|
|
368
|
+
method: 'POST', headers, body: form, signal,
|
|
369
|
+
})
|
|
370
|
+
if (!res.ok) throw await readHttpError(res, label, label)
|
|
371
|
+
const data = await safeJson(res, label)
|
|
372
|
+
return (data?.text || '').trim()
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
const viaChatAudio = async () => {
|
|
376
|
+
let sendBytes = bytes
|
|
377
|
+
let format = chatAudioFormat(mime)
|
|
378
|
+
if (!format) {
|
|
379
|
+
if (typeof deps.toWav !== 'function') {
|
|
380
|
+
throw new Error(`${label}: chat template needs wav or mp3, unsupported format ${mime}`)
|
|
381
|
+
}
|
|
382
|
+
sendBytes = await deps.toWav(bytes, signal)
|
|
383
|
+
format = 'wav'
|
|
384
|
+
}
|
|
385
|
+
const ask = (this.spec.prompt || CHAT_AUDIO_PROMPT)
|
|
386
|
+
+ (effectivePrompt ? ` Vocabulary hints (spell these correctly): ${effectivePrompt}.` : '')
|
|
387
|
+
+ (lang && lang !== 'auto' ? ` The audio language is ${lang}.` : '')
|
|
388
|
+
const res = await fetchImpl(`${base}/chat/completions`, {
|
|
389
|
+
method: 'POST',
|
|
390
|
+
headers: { ...headers, 'content-type': 'application/json' },
|
|
391
|
+
body: JSON.stringify({
|
|
392
|
+
model,
|
|
393
|
+
messages: [{
|
|
394
|
+
role: 'user',
|
|
395
|
+
content: [
|
|
396
|
+
{ type: 'text', text: ask },
|
|
397
|
+
{ type: 'input_audio', input_audio: { data: Buffer.from(sendBytes).toString('base64'), format } },
|
|
398
|
+
],
|
|
399
|
+
}],
|
|
400
|
+
}),
|
|
401
|
+
signal,
|
|
402
|
+
})
|
|
403
|
+
if (!res.ok) throw await readHttpError(res, label, label)
|
|
404
|
+
const data = await safeJson(res, label)
|
|
405
|
+
return String(data?.choices?.[0]?.message?.content || '').trim()
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
let text
|
|
409
|
+
try {
|
|
410
|
+
text = this.spec.template === 'openai-chat-audio'
|
|
411
|
+
? await viaChatAudio()
|
|
412
|
+
: await viaTranscriptions()
|
|
413
|
+
} catch (e) {
|
|
414
|
+
return { ok: false, provider: label, reason: `${label}: ${String(e && e.message || e)}` }
|
|
415
|
+
}
|
|
416
|
+
return { ok: text.length > 0, provider: label, text, reason: text ? '' : 'empty transcript' }
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
/**
|
|
421
|
+
* Registry managing STT provider strategies and instantiating chain runners.
|
|
422
|
+
*/
|
|
423
|
+
export class ProviderRegistry {
|
|
424
|
+
constructor() {
|
|
425
|
+
/** @type {Map<string, SpeechToTextProvider>} */
|
|
426
|
+
this.providers = new Map()
|
|
427
|
+
/** @type {Set<string>} */
|
|
428
|
+
this.builtinKeys = new Set()
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
/**
|
|
432
|
+
* Register a provider strategy.
|
|
433
|
+
* @param {SpeechToTextProvider} strategy
|
|
434
|
+
* @param {{ builtin?: boolean }} [options]
|
|
435
|
+
*/
|
|
436
|
+
register(strategy, options = {}) {
|
|
437
|
+
if (!(strategy instanceof SpeechToTextProvider)) {
|
|
438
|
+
throw new TypeError('ProviderRegistry.register requires a SpeechToTextProvider instance')
|
|
439
|
+
}
|
|
440
|
+
this.providers.set(strategy.key, strategy)
|
|
441
|
+
if (options.builtin) {
|
|
442
|
+
this.builtinKeys.add(strategy.key)
|
|
443
|
+
}
|
|
444
|
+
return this
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
get(key) {
|
|
448
|
+
return this.providers.get(key)
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
has(key) {
|
|
452
|
+
return this.providers.has(key)
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
keys() {
|
|
456
|
+
return Array.from(this.providers.keys())
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
/**
|
|
460
|
+
* Build execution runners for runChain(), applying presets and customProviders.
|
|
461
|
+
*/
|
|
462
|
+
buildRunners(deps, req) {
|
|
463
|
+
const vocab = Array.isArray(req.vocabulary)
|
|
464
|
+
? req.vocabulary.map((w) => String(w || '').trim()).filter(Boolean).join(', ')
|
|
465
|
+
: ''
|
|
466
|
+
const effectivePrompt = (typeof req.promptText === 'string' && req.promptText.trim()) ? req.promptText.trim() : vocab
|
|
467
|
+
const effectiveKeywords = (typeof req.keywords === 'string' && req.keywords.trim()) ? req.keywords.trim() : vocab
|
|
468
|
+
const enrichedReq = { ...req, effectivePrompt, effectiveKeywords }
|
|
469
|
+
|
|
470
|
+
const out = {}
|
|
471
|
+
for (const [key, strategy] of this.providers.entries()) {
|
|
472
|
+
out[key] = strategy.createRunner(deps, enrichedReq)
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
const customList = Array.isArray(deps?.cfg?.customProviders) ? deps.cfg.customProviders : []
|
|
476
|
+
for (const spec of customList) {
|
|
477
|
+
const key = String(spec && spec.key || '').trim()
|
|
478
|
+
if (!key || this.builtinKeys.has(key)) continue
|
|
479
|
+
const customStrategy = new OpenAICompatibleProvider({ ...spec, key })
|
|
480
|
+
out[key] = customStrategy.createRunner(deps, enrichedReq)
|
|
481
|
+
}
|
|
482
|
+
return out
|
|
483
|
+
}
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
export function createDefaultProviderRegistry(presetProviders = {}) {
|
|
487
|
+
const registry = new ProviderRegistry()
|
|
488
|
+
registry.register(new BrowserStubProvider('browser', 'recognition runs in the page, the host has no browser engine'), { builtin: true })
|
|
489
|
+
registry.register(new BrowserStubProvider('browser-webgpu', 'recognition runs in the page via WebGPU/WASM, the host has no browser engine'), { builtin: true })
|
|
490
|
+
registry.register(new DeepgramProvider(), { builtin: true })
|
|
491
|
+
registry.register(new GroqProvider(), { builtin: true })
|
|
492
|
+
registry.register(new HuggingFaceProvider(), { builtin: true })
|
|
493
|
+
registry.register(new LocalWhisperProvider(), { builtin: true })
|
|
494
|
+
registry.register(new SenseVoiceProvider(), { builtin: true })
|
|
495
|
+
|
|
496
|
+
for (const [key, spec] of Object.entries(presetProviders)) {
|
|
497
|
+
if (!registry.has(key)) {
|
|
498
|
+
registry.register(new OpenAICompatibleProvider({ ...spec, key }), { builtin: false })
|
|
499
|
+
}
|
|
500
|
+
}
|
|
501
|
+
return registry
|
|
502
|
+
}
|
package/lib/providers.js
CHANGED
|
@@ -1,24 +1,42 @@
|
|
|
1
|
-
// Speech recognition providers: built-ins plus custom ones from settings.
|
|
2
|
-
//
|
|
3
|
-
// everything is testable without real requests.
|
|
1
|
+
// Speech recognition providers: built-ins plus custom ones from settings (#306).
|
|
2
|
+
// Uses Strategy pattern and ProviderRegistry in lib/provider-registry.js.
|
|
4
3
|
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
4
|
+
import { readErrorDetail, readHttpError, safeJson, isAutoLang, fileName, chatAudioFormat } from './provider-http.js'
|
|
5
|
+
import {
|
|
6
|
+
SpeechToTextProvider,
|
|
7
|
+
BrowserStubProvider,
|
|
8
|
+
DeepgramProvider,
|
|
9
|
+
GroqProvider,
|
|
10
|
+
HuggingFaceProvider,
|
|
11
|
+
LocalWhisperProvider,
|
|
12
|
+
SenseVoiceProvider,
|
|
13
|
+
OpenAICompatibleProvider,
|
|
14
|
+
ProviderRegistry,
|
|
15
|
+
createDefaultProviderRegistry,
|
|
16
|
+
} from './provider-registry.js'
|
|
17
|
+
|
|
18
|
+
export {
|
|
19
|
+
readErrorDetail,
|
|
20
|
+
readHttpError,
|
|
21
|
+
safeJson,
|
|
22
|
+
isAutoLang,
|
|
23
|
+
fileName,
|
|
24
|
+
chatAudioFormat,
|
|
25
|
+
SpeechToTextProvider,
|
|
26
|
+
BrowserStubProvider,
|
|
27
|
+
DeepgramProvider,
|
|
28
|
+
GroqProvider,
|
|
29
|
+
HuggingFaceProvider,
|
|
30
|
+
LocalWhisperProvider,
|
|
31
|
+
SenseVoiceProvider,
|
|
32
|
+
OpenAICompatibleProvider,
|
|
33
|
+
ProviderRegistry,
|
|
34
|
+
createDefaultProviderRegistry,
|
|
35
|
+
}
|
|
9
36
|
|
|
10
|
-
|
|
11
|
-
// diverged: OpenRouter has no /audio/transcriptions and recognizes via chat.
|
|
37
|
+
export const PROVIDER_KEYS = ['browser', 'browser-webgpu', 'deepgram', 'groq', 'hf', 'local-whisper', 'sensevoice']
|
|
12
38
|
export const CUSTOM_TEMPLATES = ['openai-transcriptions', 'openai-chat-audio']
|
|
13
39
|
|
|
14
|
-
// Presets: the same as a custom provider with address, model and key name
|
|
15
|
-
// already filled in. Only the key itself is required.
|
|
16
|
-
//
|
|
17
|
-
// Each URL answers 401 without a key, so the path exists. The model is a
|
|
18
|
-
// reasonable starting point and can be overridden per chain row.
|
|
19
|
-
//
|
|
20
|
-
// A custom provider with the same name fully replaces a preset: presets are
|
|
21
|
-
// prefilled forms, not special engines.
|
|
22
40
|
export const PRESET_PROVIDERS = {
|
|
23
41
|
openai: {
|
|
24
42
|
template: 'openai-transcriptions',
|
|
@@ -50,7 +68,6 @@ export const PRESET_PROVIDERS = {
|
|
|
50
68
|
model: 'voxtral-mini-latest',
|
|
51
69
|
keyEnv: 'MISTRAL_API_KEY',
|
|
52
70
|
},
|
|
53
|
-
// OpenRouter has no /audio/transcriptions at all — chat only.
|
|
54
71
|
openrouter: {
|
|
55
72
|
template: 'openai-chat-audio',
|
|
56
73
|
baseURL: 'https://openrouter.ai/api/v1',
|
|
@@ -60,17 +77,8 @@ export const PRESET_PROVIDERS = {
|
|
|
60
77
|
}
|
|
61
78
|
|
|
62
79
|
export const PRESET_KEYS = Object.keys(PRESET_PROVIDERS)
|
|
63
|
-
|
|
64
|
-
/** All names usable in a chain without declaring a custom provider. */
|
|
65
80
|
export const KNOWN_KEYS = PROVIDER_KEYS.concat(PRESET_KEYS)
|
|
66
81
|
|
|
67
|
-
const CHAT_AUDIO_PROMPT =
|
|
68
|
-
'Transcribe the audio verbatim. Reply with the transcript text only, '
|
|
69
|
-
+ 'without comments, quotes or formatting.'
|
|
70
|
-
|
|
71
|
-
import { readErrorDetail, readHttpError, safeJson, isAutoLang, fileName, chatAudioFormat } from './provider-http.js'
|
|
72
|
-
export { readErrorDetail, readHttpError, safeJson, isAutoLang, fileName, chatAudioFormat }
|
|
73
|
-
|
|
74
82
|
export const DEFAULT_MODELS = {
|
|
75
83
|
deepgram: 'nova-2',
|
|
76
84
|
groq: 'whisper-large-v3-turbo',
|
|
@@ -79,305 +87,8 @@ export const DEFAULT_MODELS = {
|
|
|
79
87
|
sensevoice: 'SenseVoiceSmall',
|
|
80
88
|
}
|
|
81
89
|
|
|
82
|
-
|
|
83
|
-
const chosen = models && typeof models[key] === 'string' ? models[key].trim() : ''
|
|
84
|
-
return chosen || DEFAULT_MODELS[key]
|
|
85
|
-
}
|
|
86
|
-
|
|
90
|
+
export const defaultProviderRegistry = createDefaultProviderRegistry(PRESET_PROVIDERS)
|
|
87
91
|
|
|
88
92
|
export function makeProviders(deps, req) {
|
|
89
|
-
|
|
90
|
-
const { bytes, mime, lang, signal, models, promptText, keywords } = req
|
|
91
|
-
const vocab = Array.isArray(req.vocabulary)
|
|
92
|
-
? req.vocabulary.map((w) => String(w || '').trim()).filter(Boolean).join(', ')
|
|
93
|
-
: ''
|
|
94
|
-
const effectivePrompt = (typeof promptText === 'string' && promptText.trim()) ? promptText.trim() : vocab
|
|
95
|
-
const effectiveKeywords = (typeof keywords === 'string' && keywords.trim()) ? keywords.trim() : vocab
|
|
96
|
-
|
|
97
|
-
async function deepgram() {
|
|
98
|
-
const key = await resolveKey(cfg.deepgramKeyEnv)
|
|
99
|
-
if (!key) return { ok: false, provider: 'deepgram', reason: `no ${cfg.deepgramKeyEnv}` }
|
|
100
|
-
const model = pickModel(models, 'deepgram')
|
|
101
|
-
const base = String(cfg.deepgramBaseUrl || 'https://api.deepgram.com').replace(/\/+$/, '')
|
|
102
|
-
const url = `${base}/v1/listen?model=${encodeURIComponent(model)}`
|
|
103
|
-
+ (!isAutoLang(lang) ? `&language=${encodeURIComponent(lang)}` : '')
|
|
104
|
-
+ (effectiveKeywords ? `&keywords=${encodeURIComponent(effectiveKeywords)}` : '')
|
|
105
|
-
+ '&smart_format=true'
|
|
106
|
-
const res = await fetchImpl(url, {
|
|
107
|
-
method: 'POST',
|
|
108
|
-
headers: { authorization: `Token ${key}`, 'content-type': mime },
|
|
109
|
-
body: bytes,
|
|
110
|
-
signal,
|
|
111
|
-
keepalive: true,
|
|
112
|
-
})
|
|
113
|
-
if (!res.ok) throw await readHttpError(res, 'Deepgram', 'deepgram')
|
|
114
|
-
let data
|
|
115
|
-
try { data = await safeJson(res, 'Deepgram') } catch (e) {
|
|
116
|
-
return { ok: false, provider: 'deepgram', reason: e.message }
|
|
117
|
-
}
|
|
118
|
-
const text = (data?.results?.channels?.[0]?.alternatives?.[0]?.transcript || '').trim()
|
|
119
|
-
return { ok: text.length > 0, provider: 'deepgram', text, reason: text ? '' : 'empty transcript' }
|
|
120
|
-
}
|
|
121
|
-
|
|
122
|
-
async function groq() {
|
|
123
|
-
const key = await resolveKey(cfg.groqKeyEnv)
|
|
124
|
-
if (!key) return { ok: false, provider: 'groq', reason: `no ${cfg.groqKeyEnv}` }
|
|
125
|
-
const form = new FormData()
|
|
126
|
-
form.append('file', new Blob([bytes], { type: mime }), fileName(mime))
|
|
127
|
-
form.append('model', pickModel(models, 'groq'))
|
|
128
|
-
if (!isAutoLang(lang)) form.append('language', lang)
|
|
129
|
-
if (effectivePrompt) form.append('prompt', effectivePrompt)
|
|
130
|
-
form.append('response_format', 'json')
|
|
131
|
-
const res = await fetchImpl('https://api.groq.com/openai/v1/audio/transcriptions', {
|
|
132
|
-
method: 'POST',
|
|
133
|
-
headers: { authorization: `Bearer ${key}` },
|
|
134
|
-
body: form,
|
|
135
|
-
signal,
|
|
136
|
-
keepalive: true,
|
|
137
|
-
})
|
|
138
|
-
if (!res.ok) throw await readHttpError(res, 'Groq', 'groq')
|
|
139
|
-
let data
|
|
140
|
-
try { data = await safeJson(res, 'Groq') } catch (e) {
|
|
141
|
-
return { ok: false, provider: 'groq', reason: e.message }
|
|
142
|
-
}
|
|
143
|
-
const text = (data?.text || '').trim()
|
|
144
|
-
return { ok: text.length > 0, provider: 'groq', text, reason: text ? '' : 'empty transcript' }
|
|
145
|
-
}
|
|
146
|
-
|
|
147
|
-
async function hf() {
|
|
148
|
-
const token = await resolveKey(cfg.hfTokenEnv)
|
|
149
|
-
if (!token) return { ok: false, provider: 'hf', reason: `no ${cfg.hfTokenEnv}` }
|
|
150
|
-
const model = pickModel(models, 'hf')
|
|
151
|
-
const res = await fetchImpl(`https://router.huggingface.co/hf-inference/models/${model}`, {
|
|
152
|
-
method: 'POST',
|
|
153
|
-
headers: { authorization: `Bearer ${token}`, 'content-type': mime, 'x-wait-for-model': 'true' },
|
|
154
|
-
body: bytes,
|
|
155
|
-
signal,
|
|
156
|
-
keepalive: true,
|
|
157
|
-
})
|
|
158
|
-
if (!res.ok) throw await readHttpError(res, 'HF', 'hf')
|
|
159
|
-
let data
|
|
160
|
-
try { data = await safeJson(res, 'HF') } catch (e) {
|
|
161
|
-
return { ok: false, provider: 'hf', reason: e.message }
|
|
162
|
-
}
|
|
163
|
-
const text = (data?.text || '').trim()
|
|
164
|
-
return { ok: text.length > 0, provider: 'hf', text, reason: text ? '' : 'empty transcript' }
|
|
165
|
-
}
|
|
166
|
-
|
|
167
|
-
// whisper.cpp accepts WAV only — browser webm/opus gets "Invalid request".
|
|
168
|
-
// Convert when the format is not WAV.
|
|
169
|
-
async function localWhisper() {
|
|
170
|
-
let sendBytes = bytes
|
|
171
|
-
let sendMime = mime
|
|
172
|
-
if (!mime.includes('wav')) {
|
|
173
|
-
if (typeof deps.toWav !== 'function') {
|
|
174
|
-
return { ok: false, provider: 'local-whisper', reason: 'local whisper needs WAV, no converter configured' }
|
|
175
|
-
}
|
|
176
|
-
try {
|
|
177
|
-
sendBytes = await deps.toWav(bytes, signal)
|
|
178
|
-
sendMime = 'audio/wav'
|
|
179
|
-
} catch (e) {
|
|
180
|
-
return { ok: false, provider: 'local-whisper', reason: `local whisper: ${String(e && e.message || e)}` }
|
|
181
|
-
}
|
|
182
|
-
}
|
|
183
|
-
const form = new FormData()
|
|
184
|
-
form.append('file', new Blob([sendBytes], { type: sendMime }), fileName(sendMime))
|
|
185
|
-
// Unlike OpenAI-compatible APIs, whisper.cpp inherits its server's default
|
|
186
|
-
// language when the form omits this field (normally `en`). Send `auto`
|
|
187
|
-
// explicitly so every request detects the spoken language.
|
|
188
|
-
form.append('language', isAutoLang(lang) ? 'auto' : lang)
|
|
189
|
-
// Also override a server started with --translate: this plugin always needs
|
|
190
|
-
// a transcript in the spoken language, never Whisper's English translation.
|
|
191
|
-
form.append('translate', 'false')
|
|
192
|
-
if (effectivePrompt) form.append('prompt', effectivePrompt)
|
|
193
|
-
form.append('response_format', 'json')
|
|
194
|
-
const res = await fetchImpl(cfg.whisperUrl, { method: 'POST', body: form, signal })
|
|
195
|
-
// whisper.cpp returns 400 with a JSON body on internal failures (e.g. decode
|
|
196
|
-
// errors) — read the reason instead of throwing.
|
|
197
|
-
if (!res.ok) {
|
|
198
|
-
let detail = `HTTP ${res.status}`
|
|
199
|
-
try { const e = await res.json(); if (e?.error) detail = e.error } catch { /* body is not json */ }
|
|
200
|
-
return { ok: false, provider: 'local-whisper', reason: `local whisper: ${detail}` }
|
|
201
|
-
}
|
|
202
|
-
let data
|
|
203
|
-
try { data = await safeJson(res, 'local whisper') } catch (e) {
|
|
204
|
-
return { ok: false, provider: 'local-whisper', reason: e.message }
|
|
205
|
-
}
|
|
206
|
-
const text = (data?.text || '').trim()
|
|
207
|
-
return { ok: text.length > 0, provider: 'local-whisper', text, reason: text ? '' : 'empty transcript' }
|
|
208
|
-
}
|
|
209
|
-
|
|
210
|
-
// SenseVoice-ONNX / Sherpa-ONNX: ultra-fast local recognition (~50-100ms).
|
|
211
|
-
// Supports sherpa-onnx (/api/v1/asr) and OpenAI-compatible endpoints.
|
|
212
|
-
async function sensevoice() {
|
|
213
|
-
const url = String(cfg.sensevoiceUrl || 'http://127.0.0.1:6006/api/v1/asr').trim()
|
|
214
|
-
let sendBytes = bytes
|
|
215
|
-
let sendMime = mime
|
|
216
|
-
if (!mime.includes('wav')) {
|
|
217
|
-
if (typeof deps.toWav !== 'function') {
|
|
218
|
-
return { ok: false, provider: 'sensevoice', reason: 'sensevoice needs WAV, no converter configured' }
|
|
219
|
-
}
|
|
220
|
-
try {
|
|
221
|
-
sendBytes = await deps.toWav(bytes, signal)
|
|
222
|
-
sendMime = 'audio/wav'
|
|
223
|
-
} catch (e) {
|
|
224
|
-
return { ok: false, provider: 'sensevoice', reason: `sensevoice: ${String(e && e.message || e)}` }
|
|
225
|
-
}
|
|
226
|
-
}
|
|
227
|
-
const form = new FormData()
|
|
228
|
-
form.append('file', new Blob([sendBytes], { type: sendMime }), fileName(sendMime))
|
|
229
|
-
const isOpenAI = url.includes('/transcriptions')
|
|
230
|
-
if (isOpenAI) {
|
|
231
|
-
form.append('model', pickModel(models, 'sensevoice'))
|
|
232
|
-
if (!isAutoLang(lang)) form.append('language', lang)
|
|
233
|
-
if (effectivePrompt) form.append('prompt', effectivePrompt)
|
|
234
|
-
form.append('response_format', 'json')
|
|
235
|
-
}
|
|
236
|
-
|
|
237
|
-
let res
|
|
238
|
-
try {
|
|
239
|
-
res = await fetchImpl(url, { method: 'POST', body: form, signal })
|
|
240
|
-
} catch (e) {
|
|
241
|
-
return { ok: false, provider: 'sensevoice', reason: `sensevoice: ${String(e && e.message || e)}` }
|
|
242
|
-
}
|
|
243
|
-
|
|
244
|
-
if (!res.ok) {
|
|
245
|
-
let detail = `HTTP ${res.status}`
|
|
246
|
-
try { const e = await res.json(); if (e?.error) detail = e.error } catch { /* body is not json */ }
|
|
247
|
-
return { ok: false, provider: 'sensevoice', reason: `sensevoice: ${detail}` }
|
|
248
|
-
}
|
|
249
|
-
|
|
250
|
-
let data
|
|
251
|
-
try {
|
|
252
|
-
data = await res.json()
|
|
253
|
-
} catch {
|
|
254
|
-
return { ok: false, provider: 'sensevoice', reason: 'sensevoice: invalid json response' }
|
|
255
|
-
}
|
|
256
|
-
|
|
257
|
-
let text = String(data?.text || data?.transcript || '').trim()
|
|
258
|
-
// Strip SenseVoice emotion/event tags (<|NEUTRAL|>, <|HAPPY|>, <|Speech|>, ...).
|
|
259
|
-
text = text.replace(/<\|[^|>]+\|>/g, '').trim()
|
|
260
|
-
return {
|
|
261
|
-
ok: text.length > 0,
|
|
262
|
-
provider: 'sensevoice',
|
|
263
|
-
text,
|
|
264
|
-
reason: text ? '' : 'empty transcript',
|
|
265
|
-
}
|
|
266
|
-
}
|
|
267
|
-
|
|
268
|
-
// Custom provider. The chain key is its name, so the rest of the code
|
|
269
|
-
// treats it like a built-in.
|
|
270
|
-
function customProvider(spec) {
|
|
271
|
-
const label = spec.key
|
|
272
|
-
const base = String(spec.baseURL || '').replace(/\/+$/, '')
|
|
273
|
-
const model = pickModel(models, label) || spec.model
|
|
274
|
-
|
|
275
|
-
async function auth() {
|
|
276
|
-
if (!spec.keyEnv) return {}
|
|
277
|
-
const key = await resolveKey(spec.keyEnv)
|
|
278
|
-
if (!key) return null
|
|
279
|
-
return { authorization: `Bearer ${key}` }
|
|
280
|
-
}
|
|
281
|
-
|
|
282
|
-
async function viaTranscriptions(headers) {
|
|
283
|
-
const form = new FormData()
|
|
284
|
-
form.append('file', new Blob([bytes], { type: mime }), fileName(mime))
|
|
285
|
-
form.append('model', model)
|
|
286
|
-
if (lang && lang !== 'auto') form.append('language', lang)
|
|
287
|
-
if (effectivePrompt) form.append('prompt', effectivePrompt)
|
|
288
|
-
form.append('response_format', 'json')
|
|
289
|
-
const res = await fetchImpl(`${base}/audio/transcriptions`, {
|
|
290
|
-
method: 'POST', headers, body: form, signal,
|
|
291
|
-
})
|
|
292
|
-
if (!res.ok) throw await readHttpError(res, label, spec.key)
|
|
293
|
-
const data = await safeJson(res, label)
|
|
294
|
-
return (data?.text || '').trim()
|
|
295
|
-
}
|
|
296
|
-
|
|
297
|
-
async function viaChatAudio(headers) {
|
|
298
|
-
let sendBytes = bytes
|
|
299
|
-
let format = chatAudioFormat(mime)
|
|
300
|
-
if (!format) {
|
|
301
|
-
if (typeof deps.toWav !== 'function') {
|
|
302
|
-
throw new Error(`${label}: chat template needs wav or mp3, unsupported format ${mime}`)
|
|
303
|
-
}
|
|
304
|
-
sendBytes = await deps.toWav(bytes, signal)
|
|
305
|
-
format = 'wav'
|
|
306
|
-
}
|
|
307
|
-
const ask = (spec.prompt || CHAT_AUDIO_PROMPT)
|
|
308
|
-
+ (effectivePrompt ? ` Vocabulary hints (spell these correctly): ${effectivePrompt}.` : '')
|
|
309
|
-
+ (lang && lang !== 'auto' ? ` The audio language is ${lang}.` : '')
|
|
310
|
-
const res = await fetchImpl(`${base}/chat/completions`, {
|
|
311
|
-
method: 'POST',
|
|
312
|
-
headers: { ...headers, 'content-type': 'application/json' },
|
|
313
|
-
body: JSON.stringify({
|
|
314
|
-
model,
|
|
315
|
-
messages: [{
|
|
316
|
-
role: 'user',
|
|
317
|
-
content: [
|
|
318
|
-
{ type: 'text', text: ask },
|
|
319
|
-
{ type: 'input_audio', input_audio: { data: Buffer.from(sendBytes).toString('base64'), format } },
|
|
320
|
-
],
|
|
321
|
-
}],
|
|
322
|
-
}),
|
|
323
|
-
signal,
|
|
324
|
-
})
|
|
325
|
-
if (!res.ok) throw await readHttpError(res, label, spec.key)
|
|
326
|
-
const data = await safeJson(res, label)
|
|
327
|
-
return String(data?.choices?.[0]?.message?.content || '').trim()
|
|
328
|
-
}
|
|
329
|
-
|
|
330
|
-
return async function run() {
|
|
331
|
-
if (!base) return { ok: false, provider: label, reason: `${label}: no baseURL` }
|
|
332
|
-
if (!model) return { ok: false, provider: label, reason: `${label}: no model` }
|
|
333
|
-
const headers = await auth()
|
|
334
|
-
if (headers === null) return { ok: false, provider: label, reason: `no ${spec.keyEnv}` }
|
|
335
|
-
let text
|
|
336
|
-
try {
|
|
337
|
-
text = spec.template === 'openai-chat-audio'
|
|
338
|
-
? await viaChatAudio(headers)
|
|
339
|
-
: await viaTranscriptions(headers)
|
|
340
|
-
} catch (e) {
|
|
341
|
-
// One provider failure must not abort the chain — the chain decides
|
|
342
|
-
// whether to continue or give up.
|
|
343
|
-
return { ok: false, provider: label, reason: `${label}: ${String(e && e.message || e)}` }
|
|
344
|
-
}
|
|
345
|
-
return { ok: text.length > 0, provider: label, text, reason: text ? '' : 'empty transcript' }
|
|
346
|
-
}
|
|
347
|
-
}
|
|
348
|
-
|
|
349
|
-
// If audio still reached the host with 'browser' or 'browser-webgpu' in the chain,
|
|
350
|
-
// the client-side leg failed: fail clearly and move on.
|
|
351
|
-
async function browser() {
|
|
352
|
-
return {
|
|
353
|
-
ok: false,
|
|
354
|
-
provider: 'browser',
|
|
355
|
-
reason: 'browser: recognition runs in the page, the host has no browser engine',
|
|
356
|
-
}
|
|
357
|
-
}
|
|
358
|
-
|
|
359
|
-
async function browserWebGpu() {
|
|
360
|
-
return {
|
|
361
|
-
ok: false,
|
|
362
|
-
provider: 'browser-webgpu',
|
|
363
|
-
reason: 'browser-webgpu: recognition runs in the page via WebGPU/WASM, the host has no browser engine',
|
|
364
|
-
}
|
|
365
|
-
}
|
|
366
|
-
|
|
367
|
-
const out = { browser, 'browser-webgpu': browserWebGpu, deepgram, groq, hf, 'local-whisper': localWhisper, sensevoice }
|
|
368
|
-
|
|
369
|
-
// Presets: custom providers with the form pre-filled.
|
|
370
|
-
for (const key of PRESET_KEYS) {
|
|
371
|
-
if (out[key]) continue
|
|
372
|
-
out[key] = customProvider({ ...PRESET_PROVIDERS[key], key })
|
|
373
|
-
}
|
|
374
|
-
|
|
375
|
-
for (const spec of Array.isArray(cfg.customProviders) ? cfg.customProviders : []) {
|
|
376
|
-
const key = String(spec && spec.key || '').trim()
|
|
377
|
-
// Do not override built-in engines: a typo would silently replace a working
|
|
378
|
-
// provider in a chain. Presets may be overridden — that is their purpose.
|
|
379
|
-
if (!key || PROVIDER_KEYS.includes(key)) continue
|
|
380
|
-
out[key] = customProvider({ ...spec, key })
|
|
381
|
-
}
|
|
382
|
-
return out
|
|
93
|
+
return defaultProviderRegistry.buildRunners(deps, req)
|
|
383
94
|
}
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
// lib/supervisor.js — Process Supervisor and Lifecycle Finite State Machine (#310)
|
|
2
|
+
|
|
3
|
+
export const DAEMON_STATES = Object.freeze({
|
|
4
|
+
STOPPED: 'STOPPED',
|
|
5
|
+
STARTING: 'STARTING',
|
|
6
|
+
RUNNING: 'RUNNING',
|
|
7
|
+
DEGRADED: 'DEGRADED',
|
|
8
|
+
CRASHED: 'CRASHED',
|
|
9
|
+
RESTARTING: 'RESTARTING',
|
|
10
|
+
})
|
|
11
|
+
|
|
12
|
+
const VALID_TRANSITIONS = {
|
|
13
|
+
STOPPED: new Set(['STARTING', 'RUNNING', 'STOPPED']),
|
|
14
|
+
STARTING: new Set(['RUNNING', 'CRASHED', 'DEGRADED', 'STOPPED']),
|
|
15
|
+
RUNNING: new Set(['DEGRADED', 'CRASHED', 'RESTARTING', 'STOPPED', 'RUNNING']),
|
|
16
|
+
DEGRADED: new Set(['RUNNING', 'CRASHED', 'RESTARTING', 'STOPPED']),
|
|
17
|
+
CRASHED: new Set(['STARTING', 'RESTARTING', 'RUNNING', 'STOPPED']),
|
|
18
|
+
RESTARTING: new Set(['STARTING', 'RUNNING', 'CRASHED', 'STOPPED']),
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export class ProcessSupervisor {
|
|
22
|
+
/**
|
|
23
|
+
* @param {{
|
|
24
|
+
* name: string,
|
|
25
|
+
* healthcheck?: () => Promise<boolean>,
|
|
26
|
+
* heartbeatMs?: number
|
|
27
|
+
* }} options
|
|
28
|
+
*/
|
|
29
|
+
constructor(options = {}) {
|
|
30
|
+
this.name = options.name || 'daemon'
|
|
31
|
+
this.state = DAEMON_STATES.STOPPED
|
|
32
|
+
this.lastError = null
|
|
33
|
+
this.crashCount = 0
|
|
34
|
+
this.startedAt = null
|
|
35
|
+
this.updatedAt = Date.now()
|
|
36
|
+
this.handle = null
|
|
37
|
+
this.disposed = false
|
|
38
|
+
this.healthcheck = typeof options.healthcheck === 'function' ? options.healthcheck : null
|
|
39
|
+
this.heartbeatMs = typeof options.heartbeatMs === 'number' && options.heartbeatMs > 0 ? options.heartbeatMs : 0
|
|
40
|
+
this._timer = null
|
|
41
|
+
this._listeners = new Set()
|
|
42
|
+
this.history = [{ state: this.state, reason: 'init', at: this.updatedAt }]
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
canTransition(nextState) {
|
|
46
|
+
const allowed = VALID_TRANSITIONS[this.state]
|
|
47
|
+
return allowed ? allowed.has(nextState) : false
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
transition(nextState, reason = '') {
|
|
51
|
+
if (!DAEMON_STATES[nextState]) {
|
|
52
|
+
throw new Error(`Invalid daemon state: ${nextState}`)
|
|
53
|
+
}
|
|
54
|
+
const prev = this.state
|
|
55
|
+
this.state = nextState
|
|
56
|
+
this.updatedAt = Date.now()
|
|
57
|
+
if (nextState === DAEMON_STATES.RUNNING) {
|
|
58
|
+
this.lastError = null
|
|
59
|
+
if (!this.startedAt) this.startedAt = this.updatedAt
|
|
60
|
+
} else if (nextState === DAEMON_STATES.CRASHED || nextState === DAEMON_STATES.DEGRADED) {
|
|
61
|
+
if (reason) this.lastError = reason
|
|
62
|
+
if (nextState === DAEMON_STATES.CRASHED) this.crashCount++
|
|
63
|
+
} else if (nextState === DAEMON_STATES.STOPPED) {
|
|
64
|
+
this.startedAt = null
|
|
65
|
+
}
|
|
66
|
+
const entry = { from: prev, state: nextState, reason: reason || '', at: this.updatedAt }
|
|
67
|
+
this.history.push(entry)
|
|
68
|
+
if (this.history.length > 30) this.history.shift()
|
|
69
|
+
for (const fn of this._listeners) {
|
|
70
|
+
try { fn(entry) } catch { /* ignore listener errors */ }
|
|
71
|
+
}
|
|
72
|
+
return this.state
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
onStateChange(listener) {
|
|
76
|
+
if (typeof listener !== 'function') return () => {}
|
|
77
|
+
this._listeners.add(listener)
|
|
78
|
+
return () => this._listeners.delete(listener)
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
attachHandle(handle) {
|
|
82
|
+
this.handle = handle
|
|
83
|
+
if (handle && typeof handle.on === 'function') {
|
|
84
|
+
handle.on('error', (err) => {
|
|
85
|
+
if (!this.disposed) {
|
|
86
|
+
this.transition(DAEMON_STATES.CRASHED, err?.message || String(err))
|
|
87
|
+
}
|
|
88
|
+
})
|
|
89
|
+
handle.on('exit', (code, signal) => {
|
|
90
|
+
if (!this.disposed && code !== 0 && code !== null) {
|
|
91
|
+
this.transition(DAEMON_STATES.CRASHED, `${this.name} process exited (${code ?? signal})`)
|
|
92
|
+
} else if (!this.disposed && this.state === DAEMON_STATES.RUNNING) {
|
|
93
|
+
this.transition(DAEMON_STATES.STOPPED, 'process exited cleanly')
|
|
94
|
+
}
|
|
95
|
+
})
|
|
96
|
+
}
|
|
97
|
+
return handle
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
startHeartbeat() {
|
|
101
|
+
this.stopHeartbeat()
|
|
102
|
+
if (!this.healthcheck || this.heartbeatMs <= 0 || this.disposed) return
|
|
103
|
+
this._timer = setInterval(async () => {
|
|
104
|
+
if (this.disposed || (this.state !== DAEMON_STATES.RUNNING && this.state !== DAEMON_STATES.DEGRADED)) return
|
|
105
|
+
try {
|
|
106
|
+
const ok = await this.healthcheck()
|
|
107
|
+
if (!ok && this.state === DAEMON_STATES.RUNNING) {
|
|
108
|
+
this.transition(DAEMON_STATES.DEGRADED, `${this.name} healthcheck failed`)
|
|
109
|
+
} else if (ok && this.state === DAEMON_STATES.DEGRADED) {
|
|
110
|
+
this.transition(DAEMON_STATES.RUNNING, `${this.name} healthcheck recovered`)
|
|
111
|
+
}
|
|
112
|
+
} catch (err) {
|
|
113
|
+
if (this.state === DAEMON_STATES.RUNNING) {
|
|
114
|
+
this.transition(DAEMON_STATES.DEGRADED, err?.message || String(err))
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
}, this.heartbeatMs)
|
|
118
|
+
if (this._timer && typeof this._timer.unref === 'function') {
|
|
119
|
+
this._timer.unref()
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
stopHeartbeat() {
|
|
124
|
+
if (this._timer) {
|
|
125
|
+
clearInterval(this._timer)
|
|
126
|
+
this._timer = null
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
killHandle() {
|
|
131
|
+
if (this.handle && typeof this.handle.kill === 'function') {
|
|
132
|
+
try { this.handle.kill() } catch { /* ignore */ }
|
|
133
|
+
}
|
|
134
|
+
this.handle = null
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
dispose() {
|
|
138
|
+
this.disposed = true
|
|
139
|
+
this.stopHeartbeat()
|
|
140
|
+
this.killHandle()
|
|
141
|
+
this.transition(DAEMON_STATES.STOPPED, 'disposed')
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
snapshot() {
|
|
145
|
+
return {
|
|
146
|
+
name: this.name,
|
|
147
|
+
state: this.state,
|
|
148
|
+
lastError: this.lastError,
|
|
149
|
+
crashCount: this.crashCount,
|
|
150
|
+
startedAt: this.startedAt,
|
|
151
|
+
updatedAt: this.updatedAt,
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@goodandready/dsh-voice",
|
|
3
|
-
"version": "0.9.
|
|
3
|
+
"version": "0.9.48",
|
|
4
4
|
"description": "Voice input for DeepSeek Harness: dictation chunked by pauses and voice messages, each with its own provider fallback chain (Deepgram, Groq, HuggingFace, local whisper.cpp, plus any OpenAI-compatible API of your own).",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|