@goodandready/dsh-tts 0.4.26 → 0.4.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +40 -1
- package/README.md +9 -5
- package/lib/cache.js +70 -40
- package/lib/client.js +257 -160
- package/lib/engines/manager.js +103 -69
- package/lib/index.js +277 -54
- package/lib/local.js +1 -1
- package/lib/providers/cloud.js +36 -5
- package/lib/providers/constants.js +4 -4
- package/lib/providers/local.js +1 -1
- package/lib/providers.js +11 -2
- package/lib/routes.js +102 -36
- package/lib/stream-hub.js +51 -7
- package/lib/text.js +18 -1
- package/package.json +1 -1
package/lib/index.js
CHANGED
|
@@ -87,7 +87,7 @@ export const Config = z.object({
|
|
|
87
87
|
to: z.string().default(''),
|
|
88
88
|
whole: z.boolean().default(false),
|
|
89
89
|
lang: z.string().default(''),
|
|
90
|
-
})).default([])
|
|
90
|
+
})).volatile().default([])
|
|
91
91
|
.description('Say this instead of that. Applied top-down; /regex/ form allowed in from.'),
|
|
92
92
|
enableItDictionary: z.boolean().volatile().default(true)
|
|
93
93
|
.description('Auto-correct common IT terminology pronunciation (SQL, Nginx, K8s, etc.).'),
|
|
@@ -111,43 +111,55 @@ export const Config = z.object({
|
|
|
111
111
|
.description('Send synthesized voice notes to messengers via @goodandready/dsh-messenger-gateway.'),
|
|
112
112
|
autoDetectSubagent: z.boolean().volatile().default(true)
|
|
113
113
|
.description('Auto-detect subagent roles in multi-agent sessions and route to dedicated voice profiles.'),
|
|
114
|
-
language: z.string().volatile().default('
|
|
114
|
+
language: z.string().volatile().default('en'),
|
|
115
115
|
speakAsItGoes: z
|
|
116
116
|
.boolean()
|
|
117
|
+
.volatile()
|
|
117
118
|
.description('Speak each reply as it lands instead of waiting for the whole turn to finish. '
|
|
118
119
|
+ 'Long answers start sounding almost at once because they are synthesized sentence by sentence.')
|
|
119
120
|
.default(true),
|
|
120
121
|
sentenceChars: z
|
|
121
122
|
.number()
|
|
123
|
+
.step(1)
|
|
124
|
+
.min(10)
|
|
125
|
+
.volatile()
|
|
122
126
|
.description('Upper bound of one spoken piece when speaking as it goes.')
|
|
123
127
|
.default(320),
|
|
124
128
|
rate: z
|
|
125
129
|
.number()
|
|
130
|
+
.min(0.1)
|
|
131
|
+
.max(5)
|
|
132
|
+
.volatile()
|
|
126
133
|
.description('Playback speed, 0.5 to 2. Synthesis is untouched; the browser plays faster or slower.')
|
|
127
134
|
.default(1),
|
|
128
135
|
bargeIn: z
|
|
129
136
|
.boolean()
|
|
137
|
+
.volatile()
|
|
130
138
|
.description('Fall silent the moment the microphone opens. Listening to a reply and talking over it '
|
|
131
139
|
+ 'at the same time does not work, and the reply would be recorded along with the voice.')
|
|
132
140
|
.default(true),
|
|
133
141
|
announceApproval: z
|
|
134
142
|
.boolean()
|
|
143
|
+
.volatile()
|
|
135
144
|
.description('Say out loud when the agent stops and waits for an approval, and play a short chime. '
|
|
136
145
|
+ 'Useful when you walk away from a long run.')
|
|
137
146
|
.default(true),
|
|
138
147
|
approvalText: z
|
|
139
148
|
.string()
|
|
149
|
+
.volatile()
|
|
140
150
|
.description('What to say when an approval is asked for. The tool name is appended.')
|
|
141
151
|
.default('Approval required'),
|
|
142
152
|
questionText: z
|
|
143
153
|
.string()
|
|
154
|
+
.volatile()
|
|
144
155
|
.description('What to say when the agent asks a question. Empty disables it.')
|
|
145
156
|
.default('Agent asked a question'),
|
|
146
157
|
chime: z
|
|
147
158
|
.string()
|
|
159
|
+
.volatile()
|
|
148
160
|
.description('Short sound before an announcement: ding, beep or none.')
|
|
149
161
|
.default('ding'),
|
|
150
|
-
maxChars: z.number().volatile().default(4000)
|
|
162
|
+
maxChars: z.number().step(1).min(0).volatile().default(4000)
|
|
151
163
|
.description('Longer replies are truncated before synthesis.'),
|
|
152
164
|
chain: z.array(ChainEntry).volatile()
|
|
153
165
|
.default([
|
|
@@ -182,8 +194,8 @@ export const Config = z.object({
|
|
|
182
194
|
piperModel: z.string().volatile().default('')
|
|
183
195
|
.description('Path to a Piper ONNX model. The piper provider is skipped while this is empty.'),
|
|
184
196
|
espeakBin: z.string().volatile().default('espeak-ng'),
|
|
185
|
-
timeoutMs: z.number().volatile().default(60000),
|
|
186
|
-
maxQueue: z.number().volatile().default(8),
|
|
197
|
+
timeoutMs: z.number().step(1).min(1000).volatile().default(60000),
|
|
198
|
+
maxQueue: z.number().step(1).min(0).volatile().default(8),
|
|
187
199
|
})
|
|
188
200
|
|
|
189
201
|
let utteranceSeq = 0
|
|
@@ -191,23 +203,48 @@ let utteranceSeq = 0
|
|
|
191
203
|
export function apply(ctx, config) {
|
|
192
204
|
const providerBreaker = createProviderBreaker()
|
|
193
205
|
|
|
194
|
-
let
|
|
195
|
-
|
|
196
|
-
|
|
206
|
+
let savedConfig = null
|
|
207
|
+
const live = () => {
|
|
208
|
+
const fromBoxes = plainConfig(config)
|
|
209
|
+
const merged = savedConfig ? { ...fromBoxes, ...savedConfig } : fromBoxes
|
|
210
|
+
return plainConfig(Config(merged))
|
|
211
|
+
}
|
|
197
212
|
|
|
198
213
|
const settingsApi = {
|
|
199
214
|
get: () => live(),
|
|
200
|
-
replace: async (next) => {
|
|
215
|
+
replace: async (next, expectedRevision) => {
|
|
201
216
|
const plain = plainConfig(next)
|
|
202
|
-
|
|
203
|
-
const svc = ctx?.get?.('settings')
|
|
204
|
-
if (svc) {
|
|
205
|
-
|
|
217
|
+
const prevSaved = savedConfig
|
|
218
|
+
const svc = ctx?.get?.('settings') ?? ctx?.settings ?? ctx?.get?.('settingsForms') ?? ctx?.settingsForms
|
|
219
|
+
if (!svc) {
|
|
220
|
+
throw new Error('Settings service is unavailable')
|
|
221
|
+
}
|
|
222
|
+
let rev = expectedRevision
|
|
223
|
+
if (rev === undefined && typeof svc.describe === 'function') {
|
|
224
|
+
try {
|
|
225
|
+
const desc = await svc.describe(NS)
|
|
226
|
+
if (desc && typeof desc.revision === 'number') {
|
|
227
|
+
rev = desc.revision
|
|
228
|
+
} else if (desc && Array.isArray(desc.namespaces)) {
|
|
229
|
+
const entry = desc.namespaces.find((n) => n.ns === NS)
|
|
230
|
+
if (entry && typeof entry.revision === 'number') rev = entry.revision
|
|
231
|
+
}
|
|
232
|
+
} catch {
|
|
233
|
+
// ignore describe error
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
try {
|
|
206
237
|
if (typeof svc.replace === 'function') {
|
|
207
238
|
await svc.replace(NS, plain, rev)
|
|
208
239
|
} else if (typeof svc.update === 'function') {
|
|
209
240
|
await svc.update(NS, plain, rev)
|
|
241
|
+
} else {
|
|
242
|
+
throw new Error('Settings service does not support replace or update')
|
|
210
243
|
}
|
|
244
|
+
savedConfig = plain
|
|
245
|
+
} catch (err) {
|
|
246
|
+
savedConfig = prevSaved
|
|
247
|
+
throw err
|
|
211
248
|
}
|
|
212
249
|
return plain
|
|
213
250
|
},
|
|
@@ -321,15 +358,60 @@ export function apply(ctx, config) {
|
|
|
321
358
|
function cleanOpts(cfg) {
|
|
322
359
|
return { skipCode: cfg.skipCode !== false, phrases: speechPhrases(cfg.language) }
|
|
323
360
|
}
|
|
324
|
-
function cleanText(raw, cfg) {
|
|
361
|
+
function cleanText(raw, cfg, maxChars = 0) {
|
|
325
362
|
return applyPronunciation(
|
|
326
|
-
applyNarrationFilters(stripForSpeech(raw,
|
|
363
|
+
applyNarrationFilters(stripForSpeech(raw, maxChars, cleanOpts(cfg)), cfg),
|
|
327
364
|
cfg.pronunciation,
|
|
328
365
|
cfg.language,
|
|
329
366
|
cfg.enableItDictionary !== false,
|
|
330
367
|
)
|
|
331
368
|
}
|
|
332
369
|
|
|
370
|
+
// Canonical text preparation (#156, #157): clean once without premature truncation,
|
|
371
|
+
// then apply longReply strategy (full / summarize / truncate).
|
|
372
|
+
async function prepareSpeechText(raw, cfg, signal) {
|
|
373
|
+
let text = cleanText(raw, cfg, 0)
|
|
374
|
+
if (!text) return ''
|
|
375
|
+
const maxChars = Number(cfg.maxChars) > 0 ? Number(cfg.maxChars) : 0
|
|
376
|
+
const overLimit = maxChars > 0 && text.length > maxChars
|
|
377
|
+
if (overLimit && cfg.longReply !== 'full') {
|
|
378
|
+
if (cfg.longReply === 'summarize') {
|
|
379
|
+
try {
|
|
380
|
+
const retold = await summarizeReply(text.slice(0, 8000), cfg, signal)
|
|
381
|
+
if (retold) {
|
|
382
|
+
return speechPhrases(cfg.language).summaryIntro + ' ' + retold
|
|
383
|
+
}
|
|
384
|
+
} catch { /* quiet fallback to truncation */ }
|
|
385
|
+
}
|
|
386
|
+
return text.slice(0, maxChars)
|
|
387
|
+
}
|
|
388
|
+
return text
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
function synthesisFingerprint(text, provider, model, voice, role, ssmlStyle, cfg, lang) {
|
|
392
|
+
const parts = [text, provider, model || '', voice || '', role || '', ssmlStyle || '', lang || '']
|
|
393
|
+
if (provider === 'custom') parts.push(cfg.customBaseUrl || '')
|
|
394
|
+
else if (provider === 'openai') parts.push(cfg.openaiBaseUrl || '')
|
|
395
|
+
else if (provider === 'mimo') parts.push(cfg.mimoBaseUrl || '', cfg.mimoFormat || '')
|
|
396
|
+
else if (provider === 'azure') parts.push(cfg.azureRegion || '')
|
|
397
|
+
else if (provider === 'piper') parts.push(cfg.piperModel || '', cfg.piperBin || '')
|
|
398
|
+
else if (provider === 'edge') parts.push(cfg.edgeBin || '')
|
|
399
|
+
else if (provider === 'espeak') parts.push(cfg.espeakBin || '')
|
|
400
|
+
return cacheKey(parts)
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
function flightFingerprint(text, order, models, voices, role, ssmlStyle, cfg, lang) {
|
|
404
|
+
const details = order.map((p) => {
|
|
405
|
+
const extra = p === 'custom' ? (cfg.customBaseUrl || '') :
|
|
406
|
+
p === 'openai' ? (cfg.openaiBaseUrl || '') :
|
|
407
|
+
p === 'mimo' ? `${cfg.mimoBaseUrl || ''}:${cfg.mimoFormat || ''}` :
|
|
408
|
+
p === 'azure' ? (cfg.azureRegion || '') :
|
|
409
|
+
p === 'piper' ? `${cfg.piperModel || ''}:${cfg.piperBin || ''}` : ''
|
|
410
|
+
return `${p}:${models[p] || ''}:${voices[p] || ''}:${extra}`
|
|
411
|
+
}).join(';')
|
|
412
|
+
return cacheKey([text, details, role, ssmlStyle, lang])
|
|
413
|
+
}
|
|
414
|
+
|
|
333
415
|
// Fast-model summary. Any failure returns an empty string and the caller
|
|
334
416
|
// silently falls back to truncation.
|
|
335
417
|
async function summarizeReply(text, cfg, signal) {
|
|
@@ -353,21 +435,10 @@ export function apply(ctx, config) {
|
|
|
353
435
|
return parts.join('').trim()
|
|
354
436
|
}
|
|
355
437
|
|
|
356
|
-
async function synthesize(rawText, cfg, signal, role = 'reply') {
|
|
357
|
-
let text = cleanText(rawText, cfg)
|
|
438
|
+
async function synthesize(rawText, cfg, signal, role = 'reply', { skipClean = false } = {}) {
|
|
439
|
+
let text = skipClean ? String(rawText || '') : cleanText(rawText, cfg, 0)
|
|
358
440
|
if (!text) throw new Error('nothing to speak')
|
|
359
|
-
const
|
|
360
|
-
const overLimit = maxChars > 0 && text.length > maxChars
|
|
361
|
-
if (overLimit && cfg.longReply !== 'full') {
|
|
362
|
-
let cut = text.slice(0, maxChars)
|
|
363
|
-
if (cfg.longReply === 'summarize') {
|
|
364
|
-
try {
|
|
365
|
-
const retold = await summarizeReply(text.slice(0, 8000), cfg, signal)
|
|
366
|
-
if (retold) cut = speechPhrases(cfg.language).summaryIntro + ' ' + retold
|
|
367
|
-
} catch (llmDown) { /* quiet fallback to truncation */ }
|
|
368
|
-
}
|
|
369
|
-
text = cut
|
|
370
|
-
}
|
|
441
|
+
const effectiveLang = cfg.autoDetect ? detectLang(text) : cfg.language
|
|
371
442
|
const models = {}
|
|
372
443
|
const voices = {}
|
|
373
444
|
const order = []
|
|
@@ -389,7 +460,8 @@ export function apply(ctx, config) {
|
|
|
389
460
|
const cacheAllowed = cfg.cache !== false && text.length <= CACHE_MAX_TEXT
|
|
390
461
|
if (cacheAllowed) {
|
|
391
462
|
for (const provider of order) {
|
|
392
|
-
const
|
|
463
|
+
const fp = synthesisFingerprint(text, provider, models[provider], voices[provider], role, ov.ssmlStyle, cfg, effectiveLang)
|
|
464
|
+
const hit = await speechCache.get(fp)
|
|
393
465
|
if (hit) {
|
|
394
466
|
stats.total++
|
|
395
467
|
stats.cacheHits++
|
|
@@ -399,7 +471,7 @@ export function apply(ctx, config) {
|
|
|
399
471
|
}
|
|
400
472
|
}
|
|
401
473
|
}
|
|
402
|
-
const flightKey =
|
|
474
|
+
const flightKey = flightFingerprint(text, order, models, voices, role, ov.ssmlStyle, cfg, effectiveLang)
|
|
403
475
|
if (inFlightSyntheses.has(flightKey)) {
|
|
404
476
|
return inFlightSyntheses.get(flightKey)
|
|
405
477
|
}
|
|
@@ -408,7 +480,7 @@ export function apply(ctx, config) {
|
|
|
408
480
|
try {
|
|
409
481
|
const baseProviders = makeProviders(
|
|
410
482
|
{ resolveKey, fetchImpl: fetch, cfg, modelManager },
|
|
411
|
-
{ text, lang:
|
|
483
|
+
{ text, lang: effectiveLang, signal, models, voices, role },
|
|
412
484
|
)
|
|
413
485
|
const providers = {}
|
|
414
486
|
for (const key of Object.keys(baseProviders)) {
|
|
@@ -435,7 +507,8 @@ export function apply(ctx, config) {
|
|
|
435
507
|
ps.ms += out.tookMs || 0
|
|
436
508
|
if (cacheAllowed) {
|
|
437
509
|
const p = out.provider
|
|
438
|
-
|
|
510
|
+
const fp = synthesisFingerprint(text, p, models[p], voices[p], role, ov.ssmlStyle, cfg, effectiveLang)
|
|
511
|
+
speechCache.put(fp, out.mime, out.audio).catch(() => {})
|
|
439
512
|
}
|
|
440
513
|
return out
|
|
441
514
|
} finally {
|
|
@@ -447,16 +520,93 @@ export function apply(ctx, config) {
|
|
|
447
520
|
return synthPromise
|
|
448
521
|
}
|
|
449
522
|
|
|
523
|
+
// Active reservations map and concurrency throttler (#174)
|
|
524
|
+
const activeReservations = new Map() // id -> { controller, timer, dropped }
|
|
525
|
+
let activeSyntheses = 0
|
|
526
|
+
const synthesisQueue = []
|
|
527
|
+
|
|
528
|
+
function getEffectiveConcurrency(cfg) {
|
|
529
|
+
const maxQ = typeof cfg.maxQueue === 'number' && cfg.maxQueue > 0 ? cfg.maxQueue : 8
|
|
530
|
+
return Math.max(1, Math.min(maxQ, 4))
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
function runThrottledSynthesis(id, signal, fn) {
|
|
534
|
+
if (signal && signal.aborted) {
|
|
535
|
+
return Promise.reject(new Error('aborted'))
|
|
536
|
+
}
|
|
537
|
+
return new Promise((resolve, reject) => {
|
|
538
|
+
let isQueued = true
|
|
539
|
+
const onAbort = () => {
|
|
540
|
+
if (isQueued) {
|
|
541
|
+
const idx = synthesisQueue.findIndex((item) => item.id === id)
|
|
542
|
+
if (idx >= 0) synthesisQueue.splice(idx, 1)
|
|
543
|
+
reject(new Error('aborted'))
|
|
544
|
+
}
|
|
545
|
+
}
|
|
546
|
+
if (signal) {
|
|
547
|
+
signal.addEventListener('abort', onAbort, { once: true })
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
const execute = async () => {
|
|
551
|
+
isQueued = false
|
|
552
|
+
if (signal) signal.removeEventListener('abort', onAbort)
|
|
553
|
+
if (signal && signal.aborted) {
|
|
554
|
+
reject(new Error('aborted'))
|
|
555
|
+
pumpSynthesisQueue()
|
|
556
|
+
return
|
|
557
|
+
}
|
|
558
|
+
const resv = activeReservations.get(id)
|
|
559
|
+
if (resv && resv.dropped) {
|
|
560
|
+
reject(new Error('dropped from queue'))
|
|
561
|
+
pumpSynthesisQueue()
|
|
562
|
+
return
|
|
563
|
+
}
|
|
564
|
+
activeSyntheses++
|
|
565
|
+
try {
|
|
566
|
+
const result = await fn()
|
|
567
|
+
resolve(result)
|
|
568
|
+
} catch (err) {
|
|
569
|
+
reject(err)
|
|
570
|
+
} finally {
|
|
571
|
+
activeSyntheses--
|
|
572
|
+
pumpSynthesisQueue()
|
|
573
|
+
}
|
|
574
|
+
}
|
|
575
|
+
|
|
576
|
+
const maxLimit = getEffectiveConcurrency(live())
|
|
577
|
+
if (activeSyntheses < maxLimit) {
|
|
578
|
+
execute()
|
|
579
|
+
} else {
|
|
580
|
+
synthesisQueue.push({ id, run: execute })
|
|
581
|
+
}
|
|
582
|
+
})
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
function pumpSynthesisQueue() {
|
|
586
|
+
const maxLimit = getEffectiveConcurrency(live())
|
|
587
|
+
while (activeSyntheses < maxLimit && synthesisQueue.length > 0) {
|
|
588
|
+
const next = synthesisQueue.shift()
|
|
589
|
+
if (next) next.run()
|
|
590
|
+
}
|
|
591
|
+
}
|
|
592
|
+
|
|
450
593
|
// Synthesize one phrase with queue reservation. The id is issued immediately,
|
|
451
594
|
// not when the provider answers, so short phrases cannot overtake long ones.
|
|
452
595
|
function speakPiece(sid, text, cfg, kind) {
|
|
453
596
|
if (!text) return
|
|
597
|
+
const maxQ = Math.max(0, typeof cfg.maxQueue === 'number' ? cfg.maxQueue : 32)
|
|
598
|
+
if (maxQ === 0) return
|
|
599
|
+
|
|
454
600
|
const role = kind === 'notice' ? 'approval' : kind === 'error' ? 'error' : (kind && kind !== 'speech' ? kind : 'reply')
|
|
455
601
|
const id = `u${++utteranceSeq}`
|
|
456
602
|
const controller = new AbortController()
|
|
457
603
|
const timer = setTimeout(() => controller.abort(), cfg.timeoutMs)
|
|
604
|
+
activeReservations.set(id, { controller, timer, dropped: false })
|
|
458
605
|
reserve(id)
|
|
459
|
-
|
|
606
|
+
|
|
607
|
+
runThrottledSynthesis(id, controller.signal, () =>
|
|
608
|
+
synthesize(text, cfg, controller.signal, role, { skipClean: true })
|
|
609
|
+
)
|
|
460
610
|
.then((out) => settle(id, {
|
|
461
611
|
id,
|
|
462
612
|
sessionId: sid,
|
|
@@ -469,10 +619,18 @@ export function apply(ctx, config) {
|
|
|
469
619
|
tookMs: out.tookMs,
|
|
470
620
|
}))
|
|
471
621
|
.catch((err) => {
|
|
622
|
+
const resv = activeReservations.get(id)
|
|
623
|
+
if (resv && resv.dropped) {
|
|
624
|
+
activeReservations.delete(id)
|
|
625
|
+
return
|
|
626
|
+
}
|
|
472
627
|
stats.errors++
|
|
473
628
|
settle(id, { id, sessionId: sid, kind: 'error', text, error: String(err && err.message || err) })
|
|
474
629
|
})
|
|
475
|
-
.finally(() =>
|
|
630
|
+
.finally(() => {
|
|
631
|
+
clearTimeout(timer)
|
|
632
|
+
activeReservations.delete(id)
|
|
633
|
+
})
|
|
476
634
|
}
|
|
477
635
|
|
|
478
636
|
// Announcement: short chime then phrase. The chime is drawn by the browser,
|
|
@@ -492,16 +650,35 @@ export function apply(ctx, config) {
|
|
|
492
650
|
|
|
493
651
|
function enqueue(item) {
|
|
494
652
|
pending.push(item)
|
|
495
|
-
const max = live().maxQueue
|
|
496
|
-
|
|
653
|
+
const max = Math.max(0, typeof live().maxQueue === 'number' ? live().maxQueue : 32)
|
|
654
|
+
if (max === 0) {
|
|
655
|
+
for (const p of pending) {
|
|
656
|
+
if (p && p.id && activeReservations.has(p.id)) {
|
|
657
|
+
const resv = activeReservations.get(p.id)
|
|
658
|
+
resv.dropped = true
|
|
659
|
+
try { resv.controller.abort() } catch {}
|
|
660
|
+
clearTimeout(resv.timer)
|
|
661
|
+
activeReservations.delete(p.id)
|
|
662
|
+
}
|
|
663
|
+
}
|
|
664
|
+
pending.length = 0
|
|
665
|
+
return
|
|
666
|
+
}
|
|
667
|
+
while (pending.length > max && pending.length > 0) {
|
|
497
668
|
const dropIndex = pending.findIndex((x) => x.kind !== 'reserved')
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
669
|
+
const dropped = dropIndex >= 0 ? pending.splice(dropIndex, 1)[0] : pending.shift()
|
|
670
|
+
if (dropped && dropped.id && activeReservations.has(dropped.id)) {
|
|
671
|
+
const resv = activeReservations.get(dropped.id)
|
|
672
|
+
if (resv) {
|
|
673
|
+
resv.dropped = true
|
|
674
|
+
try { resv.controller.abort() } catch {}
|
|
675
|
+
clearTimeout(resv.timer)
|
|
676
|
+
activeReservations.delete(dropped.id)
|
|
677
|
+
}
|
|
502
678
|
}
|
|
503
679
|
}
|
|
504
680
|
if (live().streamingEnabled && item.kind === 'chime') {
|
|
681
|
+
item._streamEmitted = true
|
|
505
682
|
broadcastStream('chime', item)
|
|
506
683
|
}
|
|
507
684
|
}
|
|
@@ -512,13 +689,30 @@ export function apply(ctx, config) {
|
|
|
512
689
|
enqueue({ id, kind: 'reserved' })
|
|
513
690
|
}
|
|
514
691
|
|
|
692
|
+
function flushReadyStream() {
|
|
693
|
+
if (!live().streamingEnabled) return
|
|
694
|
+
for (const item of pending) {
|
|
695
|
+
if (item.kind === 'reserved') {
|
|
696
|
+
break // Stop at the first unsettled reservation to preserve sequence order
|
|
697
|
+
}
|
|
698
|
+
if (!item._streamEmitted) {
|
|
699
|
+
item._streamEmitted = true
|
|
700
|
+
if (item.audioBase64) {
|
|
701
|
+
broadcastStream('utterance', item)
|
|
702
|
+
}
|
|
703
|
+
}
|
|
704
|
+
}
|
|
705
|
+
}
|
|
706
|
+
|
|
515
707
|
function settle(id, item) {
|
|
708
|
+
activeReservations.delete(id)
|
|
516
709
|
const at = pending.findIndex((row) => row.id === id)
|
|
517
|
-
if (at === -1) {
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
broadcastStream('utterance', item)
|
|
710
|
+
if (at === -1) {
|
|
711
|
+
// Dropped reservation: do NOT re-enqueue (#174)
|
|
712
|
+
return
|
|
521
713
|
}
|
|
714
|
+
pending[at] = item
|
|
715
|
+
flushReadyStream()
|
|
522
716
|
}
|
|
523
717
|
|
|
524
718
|
ctx.effect(() => ctx.on('session/event', (session, event) => {
|
|
@@ -526,6 +720,8 @@ export function apply(ctx, config) {
|
|
|
526
720
|
if (!cfg.speakReplies) return
|
|
527
721
|
const sid = session && session.id
|
|
528
722
|
if (!sid) return
|
|
723
|
+
const isMessenger = session && (session.kind === 'im' || session.kind === 'messenger' || session.channel === 'messenger' || session.type === 'im')
|
|
724
|
+
if (isMessenger && cfg.messengerTtsEnabled === false) return
|
|
529
725
|
// The agent is waiting for human approval — worth speaking aloud
|
|
530
726
|
// so the user notices without watching the screen.
|
|
531
727
|
if (event.type === 'approval/asked') {
|
|
@@ -547,13 +743,14 @@ export function apply(ctx, config) {
|
|
|
547
743
|
event.data.agent ||
|
|
548
744
|
(event.data.message && (event.data.message.subagent || event.data.message.agent || event.data.message.sender || (event.data.message.author && event.data.message.author.name)))
|
|
549
745
|
)) || (session && (session.subagent || session.agent)) || ''
|
|
550
|
-
const
|
|
746
|
+
const allowSubagent = cfg.autoDetectSubagent !== false
|
|
747
|
+
const activeRole = (allowSubagent && agentName && cfg.roles && cfg.roles[agentName] && (cfg.roles[agentName].provider || cfg.roles[agentName].voice)) ? agentName : 'reply'
|
|
551
748
|
|
|
552
749
|
// Speak as it goes: each incoming chunk is spoken immediately
|
|
553
750
|
// in sentence pieces; waiting for turn end means silence while the agent works.
|
|
554
751
|
|
|
555
752
|
if (cfg.speakAsItGoes) {
|
|
556
|
-
const ready = cleanText(text, cfg)
|
|
753
|
+
const ready = cleanText(text, cfg, 0)
|
|
557
754
|
for (const piece of splitSentences(ready, cfg.sentenceChars)) speakPiece(sid, piece, cfg, activeRole)
|
|
558
755
|
return
|
|
559
756
|
}
|
|
@@ -569,13 +766,14 @@ export function apply(ctx, config) {
|
|
|
569
766
|
const cur = collectors.get(sid)
|
|
570
767
|
collectors.delete(sid)
|
|
571
768
|
const raw = cur && Array.isArray(cur.parts) ? cur.parts.join('\n') : ''
|
|
572
|
-
const text = cleanText(raw, cfg)
|
|
573
|
-
if (!text) return
|
|
574
|
-
const pieces = splitSentences(text, cfg.sentenceChars)
|
|
575
769
|
const activeRole = (cur && cur.role) || 'reply'
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
770
|
+
prepareSpeechText(raw, cfg).then((text) => {
|
|
771
|
+
if (!text) return
|
|
772
|
+
const pieces = splitSentences(text, cfg.sentenceChars)
|
|
773
|
+
for (const piece of pieces) {
|
|
774
|
+
speakPiece(sid, piece, cfg, activeRole)
|
|
775
|
+
}
|
|
776
|
+
}).catch(() => {})
|
|
579
777
|
}), 'dsh-tts: speak finished replies')
|
|
580
778
|
|
|
581
779
|
|
|
@@ -587,6 +785,7 @@ export function apply(ctx, config) {
|
|
|
587
785
|
stats,
|
|
588
786
|
speechCache,
|
|
589
787
|
cleanText,
|
|
788
|
+
prepareSpeechText,
|
|
590
789
|
synthesize,
|
|
591
790
|
credentialsView,
|
|
592
791
|
configResponse,
|
|
@@ -633,16 +832,40 @@ export function apply(ctx, config) {
|
|
|
633
832
|
},
|
|
634
833
|
execute: async (args, exec) => {
|
|
635
834
|
const cfg = live()
|
|
835
|
+
const rawText = String(args.text || '').trim()
|
|
836
|
+
if (!rawText) return { ok: false, error: 'empty text' }
|
|
837
|
+
const sid = exec?.session?.id || 'tool'
|
|
838
|
+
const id = `u${++utteranceSeq}`
|
|
839
|
+
reserve(id)
|
|
636
840
|
try {
|
|
637
|
-
const
|
|
841
|
+
const text = await prepareSpeechText(rawText, cfg, exec.signal)
|
|
842
|
+
if (!text) {
|
|
843
|
+
settle(id, { id, sessionId: sid, kind: 'error', text: rawText, error: 'nothing to speak' })
|
|
844
|
+
return { ok: false, error: 'nothing to speak' }
|
|
845
|
+
}
|
|
846
|
+
const out = await synthesize(text, cfg, exec.signal, 'reply', { skipClean: true })
|
|
847
|
+
const item = {
|
|
848
|
+
id,
|
|
849
|
+
sessionId: sid,
|
|
850
|
+
kind: 'speech',
|
|
851
|
+
role: 'reply',
|
|
852
|
+
text,
|
|
853
|
+
provider: out.provider,
|
|
854
|
+
mime: out.mime,
|
|
855
|
+
audioBase64: Buffer.from(out.audio).toString('base64'),
|
|
856
|
+
tookMs: out.tookMs,
|
|
857
|
+
}
|
|
858
|
+
settle(id, item)
|
|
638
859
|
return {
|
|
639
860
|
ok: true,
|
|
861
|
+
id,
|
|
640
862
|
provider: out.provider,
|
|
641
863
|
mime: out.mime,
|
|
642
864
|
bytes: out.audio.length,
|
|
643
865
|
tookMs: out.tookMs,
|
|
644
866
|
}
|
|
645
867
|
} catch (e) {
|
|
868
|
+
settle(id, { id, sessionId: sid, kind: 'error', text: rawText, error: String(e && e.message || e) })
|
|
646
869
|
return { ok: false, error: String(e && e.message || e) }
|
|
647
870
|
}
|
|
648
871
|
},
|
package/lib/local.js
CHANGED
|
@@ -39,7 +39,7 @@ function spawnCollect(bin, args, { stdinText, timeoutMs } = {}) {
|
|
|
39
39
|
|
|
40
40
|
export async function espeakSpeak(text, { bin, voice, timeoutMs }) {
|
|
41
41
|
const bytes = await spawnCollect(bin || 'espeak-ng', [
|
|
42
|
-
'-v', voice || '
|
|
42
|
+
'-v', voice || 'en',
|
|
43
43
|
'--stdout',
|
|
44
44
|
'-s', '165',
|
|
45
45
|
], { stdinText: text, timeoutMs })
|
package/lib/providers/cloud.js
CHANGED
|
@@ -18,6 +18,29 @@ function escapeXml(s) {
|
|
|
18
18
|
.replace(/"/g, '"')
|
|
19
19
|
}
|
|
20
20
|
|
|
21
|
+
function pcmToWav(pcmBuffer, sampleRate = 24000, channels = 1) {
|
|
22
|
+
if (pcmBuffer.length >= 4 && pcmBuffer.toString('ascii', 0, 4) === 'RIFF') {
|
|
23
|
+
return pcmBuffer
|
|
24
|
+
}
|
|
25
|
+
const header = Buffer.alloc(44)
|
|
26
|
+
const byteRate = sampleRate * channels * 2
|
|
27
|
+
const blockAlign = channels * 2
|
|
28
|
+
header.write('RIFF', 0)
|
|
29
|
+
header.writeUInt32LE(36 + pcmBuffer.length, 4)
|
|
30
|
+
header.write('WAVE', 8)
|
|
31
|
+
header.write('fmt ', 12)
|
|
32
|
+
header.writeUInt32LE(16, 16)
|
|
33
|
+
header.writeUInt16LE(1, 20)
|
|
34
|
+
header.writeUInt16LE(channels, 22)
|
|
35
|
+
header.writeUInt32LE(sampleRate, 24)
|
|
36
|
+
header.writeUInt32LE(byteRate, 28)
|
|
37
|
+
header.writeUInt16LE(blockAlign, 32)
|
|
38
|
+
header.writeUInt16LE(16, 34)
|
|
39
|
+
header.write('data', 36)
|
|
40
|
+
header.writeUInt32LE(pcmBuffer.length, 40)
|
|
41
|
+
return Buffer.concat([header, pcmBuffer])
|
|
42
|
+
}
|
|
43
|
+
|
|
21
44
|
export function makeCloudProviders(deps, req, withCloudTimeout) {
|
|
22
45
|
const { resolveKey, fetchImpl, cfg } = deps
|
|
23
46
|
const { text, lang, models, voices, role } = req
|
|
@@ -161,9 +184,15 @@ export function makeCloudProviders(deps, req, withCloudTimeout) {
|
|
|
161
184
|
)
|
|
162
185
|
if (!res.ok) throw new Error(`Google HTTP ${res.status}`)
|
|
163
186
|
const data = await res.json()
|
|
164
|
-
const
|
|
165
|
-
if (!
|
|
166
|
-
const
|
|
187
|
+
const inline = data?.candidates?.[0]?.content?.parts?.find((p) => p.inlineData)?.inlineData
|
|
188
|
+
if (!inline?.data) return { ok: false, provider: 'google', reason: 'no inline audio' }
|
|
189
|
+
const raw = Buffer.from(inline.data, 'base64')
|
|
190
|
+
const mimeType = String(inline.mimeType || '')
|
|
191
|
+
let sampleRate = 24000
|
|
192
|
+
const m = mimeType.match(/rate=(\d+)/i)
|
|
193
|
+
if (m) sampleRate = parseInt(m[1], 10)
|
|
194
|
+
const isWavAlready = raw.length >= 4 && raw.toString('ascii', 0, 4) === 'RIFF'
|
|
195
|
+
const audio = isWavAlready ? raw : pcmToWav(raw, sampleRate, 1)
|
|
167
196
|
return { ok: audio.length > 0, provider: 'google', audio, mime: 'audio/wav', reason: audio.length ? '' : 'empty audio' }
|
|
168
197
|
} finally {
|
|
169
198
|
cleanup()
|
|
@@ -181,8 +210,10 @@ export function makeCloudProviders(deps, req, withCloudTimeout) {
|
|
|
181
210
|
const inner = safeStyle
|
|
182
211
|
? `<mstts:express-as style="${safeStyle}">${escapeXml(text)}</mstts:express-as>`
|
|
183
212
|
: escapeXml(text)
|
|
184
|
-
const
|
|
185
|
-
|
|
213
|
+
const safeLang = escapeXml(lang || 'en-US')
|
|
214
|
+
const safeVoice = escapeXml(voice)
|
|
215
|
+
const ssml = `<speak version="1.0" xmlns="http://www.w3.org/2001/10/synthesis" xmlns:mstts="https://www.w3.org/2001/mstts" xml:lang="${safeLang}">`
|
|
216
|
+
+ `<voice xml:lang="${safeLang}" name="${safeVoice}">`
|
|
186
217
|
+ `${inner}</voice></speak>`
|
|
187
218
|
const { signal, cleanup } = withCloudTimeout()
|
|
188
219
|
try {
|
|
@@ -31,9 +31,9 @@ export const DEFAULT_MODELS = {
|
|
|
31
31
|
deepgram: 'aura-asteria-en',
|
|
32
32
|
openrouter: 'openai/gpt-4o-mini-tts-2025-12-15',
|
|
33
33
|
custom: '',
|
|
34
|
-
edge: '
|
|
34
|
+
edge: 'en-US-AriaNeural',
|
|
35
35
|
piper: '',
|
|
36
|
-
espeak: '
|
|
36
|
+
espeak: 'en',
|
|
37
37
|
}
|
|
38
38
|
|
|
39
39
|
export const DEFAULT_VOICES = {
|
|
@@ -53,9 +53,9 @@ export const DEFAULT_VOICES = {
|
|
|
53
53
|
deepgram: '',
|
|
54
54
|
openrouter: 'alloy',
|
|
55
55
|
custom: '',
|
|
56
|
-
edge: '
|
|
56
|
+
edge: 'en-US-AriaNeural',
|
|
57
57
|
piper: '',
|
|
58
|
-
espeak: '
|
|
58
|
+
espeak: 'en',
|
|
59
59
|
}
|
|
60
60
|
|
|
61
61
|
|
package/lib/providers/local.js
CHANGED
|
@@ -32,7 +32,7 @@ export function makeLocalProviders(deps, req) {
|
|
|
32
32
|
}
|
|
33
33
|
async function espeak() {
|
|
34
34
|
try {
|
|
35
|
-
const voice = pick(DEFAULT_VOICES, voices, 'espeak') || lang || '
|
|
35
|
+
const voice = pick(DEFAULT_VOICES, voices, 'espeak') || lang || 'en'
|
|
36
36
|
const out = await espeakSpeak(text, { bin: cfg.espeakBin, voice, timeoutMs: cfg.timeoutMs })
|
|
37
37
|
return { ok: true, provider: 'espeak', audio: out.audio, mime: out.mime }
|
|
38
38
|
} catch (e) {
|
package/lib/providers.js
CHANGED
|
@@ -16,13 +16,22 @@ export function makeProviders(deps, req) {
|
|
|
16
16
|
timedOut = true
|
|
17
17
|
controller.abort(new Error())
|
|
18
18
|
}, cloudTimeoutMs)
|
|
19
|
+
let onParentAbort = null
|
|
19
20
|
if (parentSignal) {
|
|
20
21
|
if (parentSignal.aborted) controller.abort(parentSignal.reason)
|
|
21
|
-
else
|
|
22
|
+
else {
|
|
23
|
+
onParentAbort = () => controller.abort(parentSignal.reason)
|
|
24
|
+
parentSignal.addEventListener('abort', onParentAbort, { once: true })
|
|
25
|
+
}
|
|
22
26
|
}
|
|
23
27
|
return {
|
|
24
28
|
signal: controller.signal,
|
|
25
|
-
cleanup: () =>
|
|
29
|
+
cleanup: () => {
|
|
30
|
+
clearTimeout(timer)
|
|
31
|
+
if (parentSignal && onParentAbort) {
|
|
32
|
+
try { parentSignal.removeEventListener('abort', onParentAbort) } catch {}
|
|
33
|
+
}
|
|
34
|
+
},
|
|
26
35
|
}
|
|
27
36
|
}
|
|
28
37
|
|