@goodandready/dsh-voice 0.3.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +42 -1
- package/lib/client.js +120 -9
- package/lib/index.js +72 -24
- package/lib/providers.js +108 -4
- package/package.json +5 -3
package/README.md
CHANGED
|
@@ -39,15 +39,56 @@ Keys are read through the DSH credentials service (Settings → Credentials, or
|
|
|
39
39
|
`$DSH_HOME/.credentials.yaml`), falling back to the process environment. A
|
|
40
40
|
provider without a key is skipped, not fatal.
|
|
41
41
|
|
|
42
|
+
### Your own providers
|
|
43
|
+
|
|
44
|
+
Any OpenAI-compatible API can be added as a provider and used in the chains
|
|
45
|
+
next to the built-in ones. Two templates, because those APIs disagree on how
|
|
46
|
+
audio is sent:
|
|
47
|
+
|
|
48
|
+
| Template | Endpoint | Request | Transcript read from |
|
|
49
|
+
|---|---|---|---|
|
|
50
|
+
| `openai-transcriptions` | `{baseURL}/audio/transcriptions` | multipart: file, model, language | `text` |
|
|
51
|
+
| `openai-chat-audio` | `{baseURL}/chat/completions` | JSON with `input_audio`: base64 and format | `choices[0].message.content` |
|
|
52
|
+
|
|
53
|
+
OpenRouter has no `/audio/transcriptions` endpoint at all — use the chat
|
|
54
|
+
template there:
|
|
55
|
+
|
|
56
|
+
```yaml
|
|
57
|
+
- id: dsh-voice
|
|
58
|
+
config:
|
|
59
|
+
customProviders:
|
|
60
|
+
- key: openrouter
|
|
61
|
+
template: openai-chat-audio
|
|
62
|
+
baseURL: https://openrouter.ai/api/v1
|
|
63
|
+
model: google/gemini-2.5-flash
|
|
64
|
+
keyEnv: OPENROUTER_API_KEY
|
|
65
|
+
message:
|
|
66
|
+
chain:
|
|
67
|
+
- provider: openrouter
|
|
68
|
+
- provider: local-whisper
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
Fields: `key` is the name the chains refer to (it cannot shadow a built-in
|
|
72
|
+
one), `keyEnv` names the credential holding the API key (empty means no
|
|
73
|
+
authorization header), and `prompt` overrides the instruction sent with the
|
|
74
|
+
audio in the chat template. A row in a chain may still override `model`.
|
|
75
|
+
|
|
76
|
+
The chat template accepts WAV and MP3 only, while the browser records
|
|
77
|
+
webm/opus — the plugin converts with ffmpeg, the same way the local whisper
|
|
78
|
+
provider does, so **ffmpeg is required for `openai-chat-audio`**.
|
|
79
|
+
|
|
42
80
|
## Configure (Web GUI)
|
|
43
81
|
|
|
44
|
-
Settings → **Голос** (Voice) has
|
|
82
|
+
Settings → **Голос** (Voice) has four blocks:
|
|
45
83
|
|
|
46
84
|
- **Dictation** — fallback chain (provider + optional model per row, order is
|
|
47
85
|
the order of attempts), language, and the silence threshold that ends a
|
|
48
86
|
phrase (`vadSilenceMs`, default 700 ms).
|
|
49
87
|
- **Voice message** — its own independent chain, language, and the cancel
|
|
50
88
|
window before the message is sent (`autoSendMs`, default 4000 ms).
|
|
89
|
+
- **Your own providers** — an OpenAI-compatible API per card: name, template,
|
|
90
|
+
base URL, model, credential name. The name becomes selectable in both chains
|
|
91
|
+
as soon as it is filled in.
|
|
51
92
|
- **General** — local whisper endpoint, binary, model, autostart.
|
|
52
93
|
|
|
53
94
|
Speed matters for dictation and accuracy for messages, which is why the chains
|
package/lib/client.js
CHANGED
|
@@ -403,7 +403,8 @@ window.__ModuleLoader__.load({
|
|
|
403
403
|
}
|
|
404
404
|
|
|
405
405
|
// ------------------------------------------------------- settings page
|
|
406
|
-
const
|
|
406
|
+
const BUILTIN = ['deepgram', 'groq', 'hf', 'local-whisper']
|
|
407
|
+
const TEMPLATES = ['openai-transcriptions', 'openai-chat-audio']
|
|
407
408
|
const MODEL_HINT = {
|
|
408
409
|
deepgram: 'nova-2', groq: 'whisper-large-v3-turbo',
|
|
409
410
|
hf: 'openai/whisper-large-v3', 'local-whisper': 'задаётся при запуске сервера',
|
|
@@ -422,6 +423,9 @@ window.__ModuleLoader__.load({
|
|
|
422
423
|
'.dvs-field input,.dvs-field select{background:var(--dsw-alias-bg-layer-1);border:1px solid var(--dsw-alias-border-l1);color:var(--dsw-alias-label-primary);border-radius:6px;padding:6px 8px;font-size:13px}' +
|
|
423
424
|
'.dvs-mini{border:1px solid var(--dsw-alias-border-l2);background:transparent;color:var(--dsw-alias-label-primary);border-radius:6px;width:28px;height:28px;cursor:pointer;flex:none}' +
|
|
424
425
|
'.dvs-save{background:var(--dsw-alias-brand-primary);color:#fff;border:none;border-radius:6px;padding:7px 14px;font-size:13px;cursor:pointer}' +
|
|
426
|
+
'.dvs-card{display:flex;flex-direction:column;gap:6px;border:1px solid var(--dsw-alias-border-l2);border-radius:8px;padding:10px}' +
|
|
427
|
+
'.dvs-card input,.dvs-card select{background:var(--dsw-alias-bg-layer-1);border:1px solid var(--dsw-alias-border-l1);color:var(--dsw-alias-label-primary);border-radius:6px;padding:6px 8px;font-size:13px}' +
|
|
428
|
+
'.dvs-wait{font-size:13px;color:var(--dsw-alias-label-secondary);line-height:1.5;max-width:520px}' +
|
|
425
429
|
'.dvs-ok{font-size:12px;color:var(--dsw-alias-state-success-primary)}' +
|
|
426
430
|
'.dvs-bad{font-size:12px;color:var(--dsw-alias-state-error-primary)}'
|
|
427
431
|
const setCssId = 'dsh-voice/settings.module.css'
|
|
@@ -449,13 +453,15 @@ window.__ModuleLoader__.load({
|
|
|
449
453
|
}
|
|
450
454
|
const remove = (i) => props.onChange(rows.filter((_, k) => k !== i))
|
|
451
455
|
const add = () => props.onChange(rows.concat([{ provider: 'local-whisper', model: '' }]))
|
|
456
|
+
const options = Array.isArray(props.options) && props.options.length ? props.options : BUILTIN
|
|
452
457
|
|
|
453
458
|
return React.createElement('div', { className: 'dvs-block' },
|
|
454
459
|
rows.map((row, i) => React.createElement('div', { className: 'dvs-row', key: i },
|
|
455
460
|
React.createElement('select', {
|
|
456
461
|
value: row.provider, disabled: !props.writable,
|
|
457
462
|
onChange: (e) => change(i, { provider: e.target.value }),
|
|
458
|
-
},
|
|
463
|
+
}, (options.indexOf(row.provider) < 0 ? options.concat([row.provider]) : options)
|
|
464
|
+
.map((p) => React.createElement('option', { key: p, value: p }, p))),
|
|
459
465
|
React.createElement('input', {
|
|
460
466
|
className: 'dvs-model', value: row.model || '', disabled: !props.writable,
|
|
461
467
|
placeholder: MODEL_HINT[row.provider] || '', onChange: (e) => change(i, { model: e.target.value }),
|
|
@@ -471,6 +477,55 @@ window.__ModuleLoader__.load({
|
|
|
471
477
|
)
|
|
472
478
|
}
|
|
473
479
|
|
|
480
|
+
// Свои провайдеры: имя, шаблон API, куда ходить и чем авторизоваться.
|
|
481
|
+
function CustomEditor(props) {
|
|
482
|
+
const rows = Array.isArray(props.value) ? props.value : []
|
|
483
|
+
const change = (i, patch) => props.onChange(rows.map((r, k) => (k === i ? Object.assign({}, r, patch) : r)))
|
|
484
|
+
const remove = (i) => props.onChange(rows.filter((_, k) => k !== i))
|
|
485
|
+
const add = () => props.onChange(rows.concat([
|
|
486
|
+
{ key: '', template: 'openai-transcriptions', baseURL: '', model: '', keyEnv: '', prompt: '' },
|
|
487
|
+
]))
|
|
488
|
+
const field = (i, row, name, placeholder, wide) => React.createElement('input', {
|
|
489
|
+
className: wide ? 'dvs-model' : '', value: row[name] || '', placeholder: placeholder,
|
|
490
|
+
disabled: !props.writable, onChange: (e) => change(i, { [name]: e.target.value }),
|
|
491
|
+
})
|
|
492
|
+
|
|
493
|
+
return React.createElement('div', { className: 'dvs-block' },
|
|
494
|
+
rows.map((row, i) => React.createElement('div', { className: 'dvs-card', key: i },
|
|
495
|
+
React.createElement('div', { className: 'dvs-row' },
|
|
496
|
+
field(i, row, 'key', 'имя для цепочки'),
|
|
497
|
+
React.createElement('select', {
|
|
498
|
+
value: row.template || 'openai-transcriptions', disabled: !props.writable,
|
|
499
|
+
onChange: (e) => change(i, { template: e.target.value }),
|
|
500
|
+
}, TEMPLATES.map((t) => React.createElement('option', { key: t, value: t }, t))),
|
|
501
|
+
React.createElement('button', {
|
|
502
|
+
type: 'button', className: 'dvs-mini', title: 'Убрать',
|
|
503
|
+
disabled: !props.writable, onClick: () => remove(i),
|
|
504
|
+
}, '\u00d7'),
|
|
505
|
+
),
|
|
506
|
+
React.createElement('div', { className: 'dvs-row' },
|
|
507
|
+
field(i, row, 'baseURL', 'https://openrouter.ai/api/v1', true),
|
|
508
|
+
),
|
|
509
|
+
React.createElement('div', { className: 'dvs-row' },
|
|
510
|
+
field(i, row, 'model', 'модель', true),
|
|
511
|
+
field(i, row, 'keyEnv', 'имя ключа'),
|
|
512
|
+
),
|
|
513
|
+
row.template === 'openai-chat-audio'
|
|
514
|
+
? React.createElement('div', { className: 'dvs-row' },
|
|
515
|
+
field(i, row, 'prompt', 'указание модели (пусто — встроенное)', true))
|
|
516
|
+
: null,
|
|
517
|
+
)),
|
|
518
|
+
React.createElement('div', { className: 'dvs-row' },
|
|
519
|
+
React.createElement('button', {
|
|
520
|
+
type: 'button', className: 'dvs-mini', title: 'Добавить своего провайдера',
|
|
521
|
+
disabled: !props.writable, onClick: add,
|
|
522
|
+
}, '+'),
|
|
523
|
+
React.createElement('span', { className: 'dvs-sub' },
|
|
524
|
+
'У OpenRouter нет /audio/transcriptions \u2014 там нужен шаблон openai-chat-audio'),
|
|
525
|
+
),
|
|
526
|
+
)
|
|
527
|
+
}
|
|
528
|
+
|
|
474
529
|
function VoiceSection(props) {
|
|
475
530
|
const ctx = props.ctx
|
|
476
531
|
const scope = ctx.settingsScope.bind({ namespace: NS })
|
|
@@ -487,19 +542,56 @@ window.__ModuleLoader__.load({
|
|
|
487
542
|
return () => { alive = false; off() }
|
|
488
543
|
}, [])
|
|
489
544
|
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
545
|
+
// Снимок приходит со статусом, и он важнее самого значения.
|
|
546
|
+
// loading — ответа хоста ещё нет;
|
|
547
|
+
// unavailable — хост ответил, но наш namespace ему пока неизвестен:
|
|
548
|
+
// так бывает, если страницу открыли в первые секунды
|
|
549
|
+
// после старта, пока плагины ещё регистрируются;
|
|
550
|
+
// ready — значения на месте.
|
|
551
|
+
// При unavailable поле writable берётся из документа и остаётся true,
|
|
552
|
+
// поэтому без проверки статуса карточка рисует пустую, но с виду рабочую
|
|
553
|
+
// форму — именно так этот баг и выглядел.
|
|
554
|
+
const ready = !!snap && snap.status === 'ready'
|
|
555
|
+
const value = ready && snap.value ? snap.value : {}
|
|
556
|
+
const writable = ready && snap.writable !== false
|
|
557
|
+
|
|
558
|
+
// Зеркало настроек в браузере перечитывается только на запись значения и
|
|
559
|
+
// на переподключение; появление namespace таким сигналом не считается.
|
|
560
|
+
// Значит само оно не починится — просим перечитать, пока не готово.
|
|
561
|
+
// Зеркало общее, так что это чинит и остальные разделы настроек.
|
|
562
|
+
React.useEffect(() => {
|
|
563
|
+
if (ready) return undefined
|
|
564
|
+
let tries = 0
|
|
565
|
+
const timer = setInterval(() => {
|
|
566
|
+
if (tries >= 15) { clearInterval(timer); return }
|
|
567
|
+
tries += 1
|
|
568
|
+
try { ctx.settingsScope.describe().load() } catch (e) { /* сервис ещё не поднялся */ }
|
|
569
|
+
}, 1000)
|
|
570
|
+
return () => clearInterval(timer)
|
|
571
|
+
}, [ready])
|
|
572
|
+
|
|
573
|
+
// Черновик сеем только из готового снимка. Раньше он заполнялся из
|
|
574
|
+
// первого пришедшего, и пустой замораживался навсегда: следующий снимок
|
|
575
|
+
// уже не пересевал его, и карточка оставалась пустой до перезагрузки.
|
|
576
|
+
React.useEffect(() => { if (ready && draft === null) setDraft(JSON.parse(JSON.stringify(value))) }, [ready, draft, value])
|
|
493
577
|
// Клиент должен знать порог VAD и окно отмены — они живут в тех же настройках.
|
|
494
578
|
React.useEffect(() => {
|
|
495
|
-
if (!
|
|
579
|
+
if (!ready) return
|
|
496
580
|
voice.settings = {
|
|
497
581
|
vadSilenceMs: Number(value && value.dictation && value.dictation.vadSilenceMs) || 700,
|
|
498
582
|
autoSendMs: Number(value && value.message && value.message.autoSendMs) || 4000,
|
|
499
583
|
}
|
|
500
|
-
}, [
|
|
501
|
-
|
|
502
|
-
if (!
|
|
584
|
+
}, [ready, value])
|
|
585
|
+
|
|
586
|
+
if (!ready) {
|
|
587
|
+
const waiting = !snap || snap.status === 'loading'
|
|
588
|
+
return React.createElement('div', { className: 'dvs-wrap' },
|
|
589
|
+
React.createElement('div', { className: 'dvs-wait' }, waiting
|
|
590
|
+
? 'Загрузка настроек…'
|
|
591
|
+
: 'Харнесс ещё не объявил настройки плагина. Если он только что перезапустился, '
|
|
592
|
+
+ 'раздел появится сам через несколько секунд.'),
|
|
593
|
+
)
|
|
594
|
+
}
|
|
503
595
|
|
|
504
596
|
const setIn = (mode, key, v) => setDraft((d) => {
|
|
505
597
|
const next = JSON.parse(JSON.stringify(d || {}))
|
|
@@ -527,6 +619,14 @@ window.__ModuleLoader__.load({
|
|
|
527
619
|
return m && m[key] !== undefined ? m[key] : fallback
|
|
528
620
|
}
|
|
529
621
|
|
|
622
|
+
// Имена своих провайдеров берём из черновика, чтобы только что
|
|
623
|
+
// добавленный сразу появлялся в списках цепочек.
|
|
624
|
+
const chainOptions = BUILTIN.concat(
|
|
625
|
+
(draft && Array.isArray(draft.customProviders) ? draft.customProviders : [])
|
|
626
|
+
.map((c) => String(c && c.key || '').trim())
|
|
627
|
+
.filter((k) => k && BUILTIN.indexOf(k) < 0),
|
|
628
|
+
)
|
|
629
|
+
|
|
530
630
|
const langField = (mode) => React.createElement('label', { className: 'dvs-field' }, 'Язык',
|
|
531
631
|
React.createElement('select', {
|
|
532
632
|
value: modeVal(mode, 'language', 'ru'), disabled: !writable,
|
|
@@ -553,6 +653,7 @@ window.__ModuleLoader__.load({
|
|
|
553
653
|
React.createElement('div', { className: 'dvs-sub' }, 'Речь режется по паузам, текст дописывается в строку ввода.'),
|
|
554
654
|
React.createElement(ChainEditor, {
|
|
555
655
|
value: draft && draft.dictation ? draft.dictation.chain : [], writable: writable,
|
|
656
|
+
options: chainOptions,
|
|
556
657
|
onChange: (v) => setIn('dictation', 'chain', v),
|
|
557
658
|
}),
|
|
558
659
|
langField('dictation'),
|
|
@@ -563,11 +664,21 @@ window.__ModuleLoader__.load({
|
|
|
563
664
|
React.createElement('div', { className: 'dvs-sub' }, 'Одна запись целиком, после распознавания уходит агенту.'),
|
|
564
665
|
React.createElement(ChainEditor, {
|
|
565
666
|
value: draft && draft.message ? draft.message.chain : [], writable: writable,
|
|
667
|
+
options: chainOptions,
|
|
566
668
|
onChange: (v) => setIn('message', 'chain', v),
|
|
567
669
|
}),
|
|
568
670
|
langField('message'),
|
|
569
671
|
numField('message', 'autoSendMs', 'Окно отмены, мс', 'Сколько времени можно отменить автоматическую отправку'),
|
|
570
672
|
),
|
|
673
|
+
React.createElement('div', { className: 'dvs-block' },
|
|
674
|
+
React.createElement('div', { className: 'dvs-h' }, 'Свои провайдеры'),
|
|
675
|
+
React.createElement('div', { className: 'dvs-sub' },
|
|
676
|
+
'Любой OpenAI-совместимый API. Имя становится доступным в цепочках выше.'),
|
|
677
|
+
React.createElement(CustomEditor, {
|
|
678
|
+
value: draft && draft.customProviders ? draft.customProviders : [], writable: writable,
|
|
679
|
+
onChange: (v) => setTop('customProviders', v),
|
|
680
|
+
}),
|
|
681
|
+
),
|
|
571
682
|
React.createElement('div', { className: 'dvs-block' },
|
|
572
683
|
React.createElement('div', { className: 'dvs-h' }, 'Общее'),
|
|
573
684
|
textField('whisperUrl', 'Локальный whisper: endpoint', 'POST /inference сервера whisper.cpp'),
|
package/lib/index.js
CHANGED
|
@@ -18,19 +18,37 @@ import { credentialRef } from '@deepseek-ai/dsh-credentials'
|
|
|
18
18
|
import { readFile, stat } from 'node:fs/promises'
|
|
19
19
|
import path from 'node:path'
|
|
20
20
|
import { runChain } from './chain.js'
|
|
21
|
-
import { makeProviders, PROVIDER_KEYS, DEFAULT_MODELS } from './providers.js'
|
|
21
|
+
import { makeProviders, PROVIDER_KEYS, DEFAULT_MODELS, CUSTOM_TEMPLATES } from './providers.js'
|
|
22
22
|
import { toWav16k } from './wav.js'
|
|
23
23
|
|
|
24
24
|
export const name = 'dsh-voice'
|
|
25
|
-
export const inject = ['tools', 'credentials', 'webServer', 'shell']
|
|
25
|
+
export const inject = ['tools', 'credentials', 'webServer', 'shell', 'settings']
|
|
26
26
|
|
|
27
27
|
const ChainEntry = z.object({
|
|
28
28
|
provider: z.string().default('local-whisper')
|
|
29
|
-
.description(`Provider key
|
|
29
|
+
.description(`Provider key: one of ${PROVIDER_KEYS.join(', ')}, `
|
|
30
|
+
+ 'or the name of an entry from customProviders.'),
|
|
30
31
|
model: z.string().default('')
|
|
31
32
|
.description('Model override. Empty means the provider default.'),
|
|
32
33
|
})
|
|
33
34
|
|
|
35
|
+
// Свой провайдер: всё, что нужно, чтобы сходить в чужой OpenAI-совместимый API.
|
|
36
|
+
const CustomProvider = z.object({
|
|
37
|
+
key: z.string().default('')
|
|
38
|
+
.description('Name used in the chains above. Must differ from the built-in keys.'),
|
|
39
|
+
template: z.string().default('openai-transcriptions')
|
|
40
|
+
.description(`API shape: ${CUSTOM_TEMPLATES.join(' or ')}. `
|
|
41
|
+
+ 'OpenRouter has no /audio/transcriptions, use openai-chat-audio there.'),
|
|
42
|
+
baseURL: z.string().default('')
|
|
43
|
+
.description('API root without a trailing slash, e.g. https://openrouter.ai/api/v1'),
|
|
44
|
+
model: z.string().default(''),
|
|
45
|
+
keyEnv: z.string().default('')
|
|
46
|
+
.description('Credential name holding the API key. Empty means no authorization header.'),
|
|
47
|
+
prompt: z.string().default('')
|
|
48
|
+
.description('openai-chat-audio only: instruction sent along with the audio. '
|
|
49
|
+
+ 'Empty means the built-in one.'),
|
|
50
|
+
})
|
|
51
|
+
|
|
34
52
|
export const Config = z.object({
|
|
35
53
|
dictation: z.object({
|
|
36
54
|
chain: z.array(ChainEntry)
|
|
@@ -48,6 +66,8 @@ export const Config = z.object({
|
|
|
48
66
|
autoSendMs: z.number().default(4000)
|
|
49
67
|
.description('Cancel window before the recognized text is sent to the agent.'),
|
|
50
68
|
}).default({}),
|
|
69
|
+
customProviders: z.array(CustomProvider).default([])
|
|
70
|
+
.description('Own recognition providers, usable in both chains next to the built-in ones.'),
|
|
51
71
|
deepgramKeyEnv: z.string().default('DEEPGRAM_API_KEY'),
|
|
52
72
|
groqKeyEnv: z.string().default('GROQ_API_KEY'),
|
|
53
73
|
hfTokenEnv: z.string().default('HF_TOKEN'),
|
|
@@ -92,9 +112,23 @@ function readBody(req, maxBytes) {
|
|
|
92
112
|
})
|
|
93
113
|
}
|
|
94
114
|
|
|
95
|
-
export function apply(ctx,
|
|
115
|
+
export function apply(ctx, baseConfig) {
|
|
96
116
|
let child = null
|
|
97
117
|
|
|
118
|
+
// Карточка настроек правит namespace с именем плагина. Пока хост его не
|
|
119
|
+
// объявил через settings.register, снимок приходит пустым и нередактируемым:
|
|
120
|
+
// поля серые, цепочки пустые, сохранять некуда. Чтение через live() заодно
|
|
121
|
+
// означает, что правка применяется к следующему запросу, а не после
|
|
122
|
+
// перезапуска процесса.
|
|
123
|
+
let getConfig = () => baseConfig
|
|
124
|
+
const live = () => Config(structuredClone(getConfig() ?? {})) ?? baseConfig
|
|
125
|
+
|
|
126
|
+
ctx.inject(['settings'], (sctx) => {
|
|
127
|
+
const scope = sctx.settings.register(name, Config, { base: baseConfig })
|
|
128
|
+
getConfig = () => scope.get() ?? baseConfig
|
|
129
|
+
sctx.effect(() => () => { getConfig = () => baseConfig })
|
|
130
|
+
})
|
|
131
|
+
|
|
98
132
|
async function resolveKey(ref) {
|
|
99
133
|
try {
|
|
100
134
|
const resolved = await ctx.credentials.resolve(credentialRef(ref))
|
|
@@ -107,22 +141,23 @@ export function apply(ctx, config) {
|
|
|
107
141
|
try {
|
|
108
142
|
const controller = new AbortController()
|
|
109
143
|
const t = setTimeout(() => controller.abort(), 2000)
|
|
110
|
-
const res = await fetch(
|
|
144
|
+
const res = await fetch(live().whisperUrl.split('/inference')[0] + '/', { signal: controller.signal })
|
|
111
145
|
clearTimeout(t)
|
|
112
146
|
return res.ok
|
|
113
147
|
} catch { return false }
|
|
114
148
|
}
|
|
115
149
|
|
|
116
150
|
async function startWhisper() {
|
|
117
|
-
|
|
151
|
+
const cfg = live()
|
|
152
|
+
if (!cfg.autoStart) return false
|
|
118
153
|
// Без пути к модели запускать нечего: пакет не знает, где она лежит у
|
|
119
154
|
// конкретного пользователя, и молча стартовать чужой бинарь не должен.
|
|
120
|
-
if (!
|
|
155
|
+
if (!cfg.whisperModel) return false
|
|
121
156
|
if (await whisperAlive()) return true
|
|
122
157
|
try {
|
|
123
158
|
const spec = ctx.shell.resolve({
|
|
124
|
-
command: `${JSON.stringify(
|
|
125
|
-
+ ` --host 127.0.0.1 --port 8001 -t 8 -p 1 -l ${
|
|
159
|
+
command: `${JSON.stringify(cfg.whisperBin)} -m ${JSON.stringify(cfg.whisperModel)}`
|
|
160
|
+
+ ` --host 127.0.0.1 --port 8001 -t 8 -p 1 -l ${cfg.dictation.language}`,
|
|
126
161
|
timeoutMs: 0,
|
|
127
162
|
stdoutMaxBytes: 4 * 1024 * 1024,
|
|
128
163
|
})
|
|
@@ -139,15 +174,21 @@ export function apply(ctx, config) {
|
|
|
139
174
|
|
|
140
175
|
// Общий путь распознавания: собрать провайдеров по цепочке режима и пройти её.
|
|
141
176
|
async function transcribe(modeCfg, bytes, mime, signal) {
|
|
177
|
+
const cfg = live()
|
|
178
|
+
const customKeys = (Array.isArray(cfg.customProviders) ? cfg.customProviders : [])
|
|
179
|
+
.map((c) => String(c && c.key || '').trim())
|
|
180
|
+
.filter(Boolean)
|
|
142
181
|
const models = {}
|
|
143
182
|
const order = []
|
|
144
183
|
for (const entry of Array.isArray(modeCfg.chain) ? modeCfg.chain : []) {
|
|
145
|
-
if (!PROVIDER_KEYS.includes(entry.provider)) continue
|
|
184
|
+
if (!PROVIDER_KEYS.includes(entry.provider) && !customKeys.includes(entry.provider)) continue
|
|
146
185
|
order.push(entry.provider)
|
|
147
|
-
|
|
186
|
+
// Для своего провайдера модель по умолчанию живёт в его описании,
|
|
187
|
+
// подставит makeProviders — здесь пусто означает «бери оттуда».
|
|
188
|
+
models[entry.provider] = entry.model || DEFAULT_MODELS[entry.provider] || ''
|
|
148
189
|
}
|
|
149
190
|
const providers = makeProviders(
|
|
150
|
-
{ resolveKey, fetchImpl: fetch, cfg
|
|
191
|
+
{ resolveKey, fetchImpl: fetch, cfg, toWav: (b) => toWav16k(b, cfg.ffmpegBin) },
|
|
151
192
|
{ bytes, mime, lang: modeCfg.language, signal, models },
|
|
152
193
|
)
|
|
153
194
|
return runChain(order, providers)
|
|
@@ -158,12 +199,17 @@ export function apply(ctx, config) {
|
|
|
158
199
|
path: '/dsh-voice/status',
|
|
159
200
|
handler: async (req, res) => {
|
|
160
201
|
if (req.method !== 'GET') { writeJson(res, 405, { ok: false, error: { code: 'method', message: 'GET only' } }); return }
|
|
202
|
+
const cfg = live()
|
|
161
203
|
writeJson(res, 200, {
|
|
162
204
|
ok: true,
|
|
163
205
|
whisperRunning: await whisperAlive(),
|
|
206
|
+
providers: PROVIDER_KEYS.concat(
|
|
207
|
+
(Array.isArray(cfg.customProviders) ? cfg.customProviders : [])
|
|
208
|
+
.map((c) => String(c && c.key || '').trim()).filter(Boolean),
|
|
209
|
+
),
|
|
164
210
|
modes: {
|
|
165
|
-
dictation: { chain:
|
|
166
|
-
message: { chain:
|
|
211
|
+
dictation: { chain: cfg.dictation.chain, language: cfg.dictation.language, vadSilenceMs: cfg.dictation.vadSilenceMs },
|
|
212
|
+
message: { chain: cfg.message.chain, language: cfg.message.language, autoSendMs: cfg.message.autoSendMs },
|
|
167
213
|
},
|
|
168
214
|
})
|
|
169
215
|
},
|
|
@@ -174,9 +220,10 @@ export function apply(ctx, config) {
|
|
|
174
220
|
path: '/dsh-voice/transcribe',
|
|
175
221
|
handler: async (req, res) => {
|
|
176
222
|
if (req.method !== 'POST') { writeJson(res, 405, { ok: false, error: { code: 'method', message: 'POST only' } }); return }
|
|
223
|
+
const cfg = live()
|
|
177
224
|
let raw
|
|
178
225
|
try {
|
|
179
|
-
raw = await readBody(req,
|
|
226
|
+
raw = await readBody(req, cfg.maxFileBytes + 1024 * 1024)
|
|
180
227
|
} catch (e) {
|
|
181
228
|
writeJson(res, 400, { ok: false, error: { code: 'body', message: e.message } }); return
|
|
182
229
|
}
|
|
@@ -186,15 +233,15 @@ export function apply(ctx, config) {
|
|
|
186
233
|
const dataBase64 = typeof payload.dataBase64 === 'string' ? payload.dataBase64 : ''
|
|
187
234
|
if (!dataBase64) { writeJson(res, 400, { ok: false, error: { code: 'no-audio', message: 'no audio data' } }); return }
|
|
188
235
|
const mime = typeof payload.mimeType === 'string' && payload.mimeType ? payload.mimeType : 'audio/webm'
|
|
189
|
-
const modeCfg = payload.mode === 'message' ?
|
|
236
|
+
const modeCfg = payload.mode === 'message' ? cfg.message : cfg.dictation
|
|
190
237
|
|
|
191
238
|
let bytes
|
|
192
239
|
try { bytes = Buffer.from(dataBase64, 'base64') } catch { bytes = null }
|
|
193
240
|
if (!bytes || bytes.length === 0) {
|
|
194
241
|
writeJson(res, 400, { ok: false, error: { code: 'decode', message: 'failed to decode audio' } }); return
|
|
195
242
|
}
|
|
196
|
-
if (bytes.length >
|
|
197
|
-
writeJson(res, 413, { ok: false, error: { code: 'too-large', message: `audio is ${bytes.length} bytes, max ${
|
|
243
|
+
if (bytes.length > cfg.maxFileBytes) {
|
|
244
|
+
writeJson(res, 413, { ok: false, error: { code: 'too-large', message: `audio is ${bytes.length} bytes, max ${cfg.maxFileBytes}` } }); return
|
|
198
245
|
}
|
|
199
246
|
|
|
200
247
|
// Локальный whisper в цепочке — поднимаем сервер заранее, иначе первый
|
|
@@ -204,7 +251,7 @@ export function apply(ctx, config) {
|
|
|
204
251
|
}
|
|
205
252
|
|
|
206
253
|
const controller = new AbortController()
|
|
207
|
-
const timer = setTimeout(() => controller.abort(),
|
|
254
|
+
const timer = setTimeout(() => controller.abort(), cfg.timeoutMs)
|
|
208
255
|
try {
|
|
209
256
|
const out = await transcribe(modeCfg, bytes, mime, controller.signal)
|
|
210
257
|
writeJson(res, 200, { ok: true, text: out.text, provider: out.provider, tookMs: out.tookMs })
|
|
@@ -226,7 +273,7 @@ export function apply(ctx, config) {
|
|
|
226
273
|
+ 'Use for voice messages, recordings, interviews.',
|
|
227
274
|
parameters: {
|
|
228
275
|
file_path: { type: 'string', required: true, description: 'Absolute path to the audio file (wav, mp3, m4a, ogg, flac, webm).' },
|
|
229
|
-
language: { type: 'string', description: `Recognition language code. Default: ${
|
|
276
|
+
language: { type: 'string', description: `Recognition language code. Default: ${baseConfig.message.language}.` },
|
|
230
277
|
},
|
|
231
278
|
output: {
|
|
232
279
|
schema: {
|
|
@@ -242,19 +289,20 @@ export function apply(ctx, config) {
|
|
|
242
289
|
},
|
|
243
290
|
},
|
|
244
291
|
isConcurrencySafe: () => false,
|
|
245
|
-
timeoutMs:
|
|
292
|
+
timeoutMs: baseConfig.timeoutMs * 3 + 5000,
|
|
246
293
|
async execute(args, exec) {
|
|
294
|
+
const cfg = live()
|
|
247
295
|
const filePath = String(args.file_path || '').trim()
|
|
248
296
|
if (!filePath) throw new Error('transcribe_audio: file_path is required')
|
|
249
297
|
const info = await stat(filePath).catch(() => null)
|
|
250
298
|
if (!info) throw new Error(`transcribe_audio: file not found: ${filePath}`)
|
|
251
|
-
if (info.size >
|
|
252
|
-
throw new Error(`transcribe_audio: file too large (${info.size} bytes, max ${
|
|
299
|
+
if (info.size > cfg.maxFileBytes) {
|
|
300
|
+
throw new Error(`transcribe_audio: file too large (${info.size} bytes, max ${cfg.maxFileBytes})`)
|
|
253
301
|
}
|
|
254
302
|
if (info.size < 100) throw new Error('transcribe_audio: file is empty or too small')
|
|
255
303
|
const mime = MIME_BY_EXT[path.extname(filePath).toLowerCase()] || 'audio/wav'
|
|
256
304
|
const bytes = await readFile(filePath)
|
|
257
|
-
const modeCfg = { ...
|
|
305
|
+
const modeCfg = { ...cfg.message, language: String(args.language || cfg.message.language) }
|
|
258
306
|
return transcribe(modeCfg, bytes, mime, exec.signal)
|
|
259
307
|
},
|
|
260
308
|
}),
|
package/lib/providers.js
CHANGED
|
@@ -1,9 +1,26 @@
|
|
|
1
|
-
//
|
|
2
|
-
// параметром (fetchImpl), ключи —
|
|
3
|
-
// без реальных запросов.
|
|
1
|
+
// Провайдеры распознавания речи: четыре встроенных плюс любые свои, объявленные
|
|
2
|
+
// в настройках. Чистые функции: сеть приходит параметром (fetchImpl), ключи —
|
|
3
|
+
// через resolveKey, поэтому всё проверяется без реальных запросов.
|
|
4
4
|
|
|
5
5
|
export const PROVIDER_KEYS = ['deepgram', 'groq', 'hf', 'local-whisper']
|
|
6
6
|
|
|
7
|
+
// Свой провайдер описывается одним из двух шаблонов, потому что
|
|
8
|
+
// OpenAI-совместимые API разошлись: у OpenRouter, например, нет
|
|
9
|
+
// /audio/transcriptions вовсе, и распознавание там идёт через чат.
|
|
10
|
+
export const CUSTOM_TEMPLATES = ['openai-transcriptions', 'openai-chat-audio']
|
|
11
|
+
|
|
12
|
+
const CHAT_AUDIO_PROMPT =
|
|
13
|
+
'Transcribe the audio verbatim. Reply with the transcript text only, '
|
|
14
|
+
+ 'without comments, quotes or formatting.'
|
|
15
|
+
|
|
16
|
+
// Форматы, которые чат-шаблон принимает в input_audio. Всё остальное —
|
|
17
|
+
// включая webm/opus, который пишет браузер, — сначала перегоняем в WAV.
|
|
18
|
+
function chatAudioFormat(mime) {
|
|
19
|
+
if (mime.includes('wav')) return 'wav'
|
|
20
|
+
if (mime.includes('mpeg') || mime.includes('mp3')) return 'mp3'
|
|
21
|
+
return ''
|
|
22
|
+
}
|
|
23
|
+
|
|
7
24
|
export const DEFAULT_MODELS = {
|
|
8
25
|
deepgram: 'nova-2',
|
|
9
26
|
groq: 'whisper-large-v3-turbo',
|
|
@@ -114,5 +131,92 @@ export function makeProviders(deps, req) {
|
|
|
114
131
|
return { ok: text.length > 0, provider: 'local-whisper', text, reason: text ? '' : 'empty transcript' }
|
|
115
132
|
}
|
|
116
133
|
|
|
117
|
-
|
|
134
|
+
// Свой провайдер. Ключ в цепочке — его имя, поэтому в остальном коде он
|
|
135
|
+
// ничем не отличается от встроенного.
|
|
136
|
+
function customProvider(spec) {
|
|
137
|
+
const label = spec.key
|
|
138
|
+
const base = String(spec.baseURL || '').replace(/\/+$/, '')
|
|
139
|
+
const model = pickModel(models, label) || spec.model
|
|
140
|
+
|
|
141
|
+
async function auth() {
|
|
142
|
+
if (!spec.keyEnv) return {}
|
|
143
|
+
const key = await resolveKey(spec.keyEnv)
|
|
144
|
+
if (!key) return null
|
|
145
|
+
return { authorization: `Bearer ${key}` }
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
async function viaTranscriptions(headers) {
|
|
149
|
+
const form = new FormData()
|
|
150
|
+
form.append('file', new Blob([bytes], { type: mime }), fileName(mime))
|
|
151
|
+
form.append('model', model)
|
|
152
|
+
if (lang && lang !== 'auto') form.append('language', lang)
|
|
153
|
+
form.append('response_format', 'json')
|
|
154
|
+
const res = await fetchImpl(`${base}/audio/transcriptions`, {
|
|
155
|
+
method: 'POST', headers, body: form, signal,
|
|
156
|
+
})
|
|
157
|
+
if (!res.ok) throw new Error(`${label} HTTP ${res.status}`)
|
|
158
|
+
const data = await res.json()
|
|
159
|
+
return (data?.text || '').trim()
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
async function viaChatAudio(headers) {
|
|
163
|
+
let sendBytes = bytes
|
|
164
|
+
let format = chatAudioFormat(mime)
|
|
165
|
+
if (!format) {
|
|
166
|
+
if (typeof deps.toWav !== 'function') {
|
|
167
|
+
throw new Error(`${label} needs wav or mp3, no converter configured`)
|
|
168
|
+
}
|
|
169
|
+
sendBytes = await deps.toWav(bytes)
|
|
170
|
+
format = 'wav'
|
|
171
|
+
}
|
|
172
|
+
const ask = (spec.prompt || CHAT_AUDIO_PROMPT)
|
|
173
|
+
+ (lang && lang !== 'auto' ? ` The audio language is ${lang}.` : '')
|
|
174
|
+
const res = await fetchImpl(`${base}/chat/completions`, {
|
|
175
|
+
method: 'POST',
|
|
176
|
+
headers: { ...headers, 'content-type': 'application/json' },
|
|
177
|
+
body: JSON.stringify({
|
|
178
|
+
model,
|
|
179
|
+
messages: [{
|
|
180
|
+
role: 'user',
|
|
181
|
+
content: [
|
|
182
|
+
{ type: 'text', text: ask },
|
|
183
|
+
{ type: 'input_audio', input_audio: { data: Buffer.from(sendBytes).toString('base64'), format } },
|
|
184
|
+
],
|
|
185
|
+
}],
|
|
186
|
+
}),
|
|
187
|
+
signal,
|
|
188
|
+
})
|
|
189
|
+
if (!res.ok) throw new Error(`${label} HTTP ${res.status}`)
|
|
190
|
+
const data = await res.json()
|
|
191
|
+
return String(data?.choices?.[0]?.message?.content || '').trim()
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
return async function run() {
|
|
195
|
+
if (!base) return { ok: false, provider: label, reason: `${label}: no baseURL` }
|
|
196
|
+
if (!model) return { ok: false, provider: label, reason: `${label}: no model` }
|
|
197
|
+
const headers = await auth()
|
|
198
|
+
if (headers === null) return { ok: false, provider: label, reason: `no ${spec.keyEnv}` }
|
|
199
|
+
let text
|
|
200
|
+
try {
|
|
201
|
+
text = spec.template === 'openai-chat-audio'
|
|
202
|
+
? await viaChatAudio(headers)
|
|
203
|
+
: await viaTranscriptions(headers)
|
|
204
|
+
} catch (e) {
|
|
205
|
+
// Отказ одного провайдера не должен ронять цепочку — она сама решит,
|
|
206
|
+
// идти дальше или сдаться.
|
|
207
|
+
return { ok: false, provider: label, reason: `${label}: ${String(e && e.message || e)}` }
|
|
208
|
+
}
|
|
209
|
+
return { ok: text.length > 0, provider: label, text, reason: text ? '' : 'empty transcript' }
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
const out = { deepgram, groq, hf, 'local-whisper': localWhisper }
|
|
214
|
+
for (const spec of Array.isArray(cfg.customProviders) ? cfg.customProviders : []) {
|
|
215
|
+
const key = String(spec && spec.key || '').trim()
|
|
216
|
+
// Встроенные не перекрываем: иначе опечатка в имени тихо подменит рабочего
|
|
217
|
+
// провайдера в чужой цепочке.
|
|
218
|
+
if (!key || out[key]) continue
|
|
219
|
+
out[key] = customProvider({ ...spec, key })
|
|
220
|
+
}
|
|
221
|
+
return out
|
|
118
222
|
}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@goodandready/dsh-voice",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "Voice input for DeepSeek Harness: dictation chunked by pauses and voice messages, each with its own provider fallback chain (Deepgram, Groq, HuggingFace, local whisper.cpp).",
|
|
3
|
+
"version": "0.4.1",
|
|
4
|
+
"description": "Voice input for DeepSeek Harness: dictation chunked by pauses and voice messages, each with its own provider fallback chain (Deepgram, Groq, HuggingFace, local whisper.cpp, plus any OpenAI-compatible API of your own).",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"main": "./lib/index.js",
|
|
@@ -26,7 +26,9 @@
|
|
|
26
26
|
"dictation",
|
|
27
27
|
"whisper",
|
|
28
28
|
"deepgram",
|
|
29
|
-
"groq"
|
|
29
|
+
"groq",
|
|
30
|
+
"openrouter",
|
|
31
|
+
"openai-compatible"
|
|
30
32
|
],
|
|
31
33
|
"repository": {
|
|
32
34
|
"type": "git",
|