iobroker.assistant 0.1.6 → 1.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +61 -4
- package/admin/custom/.vite/manifest.json +122 -77
- package/admin/custom/assets/Components-7BDDNrGP.js +1 -0
- package/admin/custom/assets/_virtual_mf-localSharedImportMap___mfe_internal__ConfigCustomAssistant__mf_owner__1-DcAGnE2z.js +1 -0
- package/admin/custom/assets/_virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__198531368674860__loadShare___mf_0_emotion_mf_1_react__loadShare__.js-CzBMxGFr.js +1 -0
- package/admin/custom/assets/_virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__198531368674860__loadShare___mf_0_emotion_mf_1_styled__loadShare__.js-1XsLCK2M.js +1 -0
- package/admin/custom/assets/_virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__198531368674860__loadShare___mf_0_iobroker_mf_1_gui_mf_2_components__loadShare__.js-CMv8bcyA.js +1217 -0
- package/admin/custom/assets/_virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__198531368674860__loadShare__shared-3mzaJWDF.js +2 -0
- package/admin/custom/assets/_virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__198531368674860__prebuild___mf_0_iobroker_mf_1_json_mf_2_config__prebuild__-DS8oaQQt.js +122 -0
- package/admin/custom/assets/_virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__198531368674860__prebuild__react_mf_1_jsx_mf_2_runtime__prebuild__-B_v57Qfr.js +1 -0
- package/admin/custom/assets/dist-DiU1y0R9.js +1 -0
- package/admin/custom/assets/helpers-Dot7vgQQ.js +1 -0
- package/admin/custom/assets/hostInit-MshDicd1.js +1 -0
- package/admin/custom/assets/iconLibraryData-B5fiyl1C.js +1 -0
- package/admin/custom/assets/index-P2Xu9kJm.js +1 -0
- package/admin/custom/assets/lib-DjwHKtRi.js +3 -0
- package/admin/custom/assets/{mf-entry-bootstrap-0-87f15505.js → mf-entry-bootstrap-0-7c1ddf47.js} +14 -2
- package/admin/custom/assets/pendingShares-CaJDCKLB.js +2 -0
- package/admin/custom/assets/rolldown-runtime-Dd_uD5pT.js +1 -0
- package/admin/custom/assets/utils-towTPKqG.js +4 -0
- package/admin/custom/assets/vite-preload-helper-BbbETgeG.js +1 -0
- package/admin/custom/customComponents.js +2 -2
- package/admin/custom/i18n/de.json +64 -57
- package/admin/custom/i18n/en.json +8 -1
- package/admin/custom/i18n/es.json +64 -57
- package/admin/custom/i18n/fr.json +64 -57
- package/admin/custom/i18n/it.json +64 -57
- package/admin/custom/i18n/nl.json +64 -57
- package/admin/custom/i18n/pl.json +64 -57
- package/admin/custom/i18n/pt.json +64 -57
- package/admin/custom/i18n/ru.json +64 -57
- package/admin/custom/i18n/uk.json +64 -57
- package/admin/custom/i18n/zh-cn.json +64 -57
- package/admin/custom/index.html +2 -2
- package/admin/custom/mf-manifest.json +1 -1
- package/admin/custom/mf-stats.json +1 -1
- package/admin/i18n/de.json +59 -3
- package/admin/i18n/en.json +59 -3
- package/admin/i18n/es.json +59 -3
- package/admin/i18n/fr.json +59 -3
- package/admin/i18n/it.json +59 -3
- package/admin/i18n/nl.json +59 -3
- package/admin/i18n/pl.json +59 -3
- package/admin/i18n/pt.json +59 -3
- package/admin/i18n/ru.json +59 -3
- package/admin/i18n/uk.json +59 -3
- package/admin/i18n/zh-cn.json +59 -3
- package/admin/jsonConfig.json +589 -75
- package/build/lib/ask.js +130 -0
- package/build/lib/ask.js.map +1 -0
- package/build/lib/llm.js +135 -13
- package/build/lib/llm.js.map +1 -1
- package/build/lib/nlu.js +74 -1
- package/build/lib/nlu.js.map +1 -1
- package/build/lib/notifications.js +112 -0
- package/build/lib/notifications.js.map +1 -0
- package/build/lib/presence.js +188 -0
- package/build/lib/presence.js.map +1 -0
- package/build/lib/routines.js +115 -0
- package/build/lib/routines.js.map +1 -0
- package/build/lib/targets.js +96 -0
- package/build/lib/targets.js.map +1 -0
- package/build/lib/triggers.js +632 -0
- package/build/lib/triggers.js.map +1 -0
- package/build/lib/voice/aws.js +9 -1
- package/build/lib/voice/aws.js.map +1 -1
- package/build/lib/voice/azure.js +9 -2
- package/build/lib/voice/azure.js.map +1 -1
- package/build/lib/voice/engines.js +82 -0
- package/build/lib/voice/engines.js.map +1 -1
- package/build/lib/voice/esphome.js +732 -0
- package/build/lib/voice/esphome.js.map +1 -0
- package/build/lib/voice/esphomeEntities.js +303 -0
- package/build/lib/voice/esphomeEntities.js.map +1 -0
- package/build/lib/voice/esphomeProto.js +181 -0
- package/build/lib/voice/esphomeProto.js.map +1 -0
- package/build/lib/voice/fallback.js +85 -0
- package/build/lib/voice/fallback.js.map +1 -0
- package/build/lib/voice/mediaServer.js +151 -0
- package/build/lib/voice/mediaServer.js.map +1 -0
- package/build/lib/voice/protocol.js.map +1 -1
- package/build/lib/voice/tone.js +32 -0
- package/build/lib/voice/tone.js.map +1 -0
- package/build/lib/voice/tts.js.map +1 -1
- package/build/lib/voice/ttsCache.js +274 -0
- package/build/lib/voice/ttsCache.js.map +1 -0
- package/build/lib/voice/vad.js +173 -0
- package/build/lib/voice/vad.js.map +1 -0
- package/build/lib/voice/voiceServer.js +13 -0
- package/build/lib/voice/voiceServer.js.map +1 -1
- package/build/main.js +1721 -67
- package/build/main.js.map +1 -1
- package/io-package.json +241 -3
- package/package.json +13 -12
- package/admin/custom/assets/Components-xWdtqvTH.js +0 -1
- package/admin/custom/assets/_virtual_mf-localSharedImportMap___mfe_internal__ConfigCustomAssistant__mf_owner__1-dybYZgsu.js +0 -1
- package/admin/custom/assets/_virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__1__loadShare___mf_0_emotion_mf_1_react__loadShare__.js-BnSUsPHa.js +0 -1
- package/admin/custom/assets/_virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__1__loadShare___mf_0_emotion_mf_1_styled__loadShare__.js-DlSVT49A.js +0 -1
- package/admin/custom/assets/_virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__1__loadShare___mf_0_iobroker_mf_1_gui_mf_2_components__loadShare__.js-Cfsgderv.js +0 -1243
- package/admin/custom/assets/_virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__1__loadShare___mf_0_iobroker_mf_1_json_mf_2_config__loadShare__.js-ByJf4zv_.js +0 -1494
- package/admin/custom/assets/hostInit-CZawMH9B.js +0 -1
- package/admin/custom/assets/index-JB7g1jgx.js +0 -1
- package/admin/custom/assets/lib-bB-LHRhZ.js +0 -3
- package/admin/custom/assets/rolldown-runtime-C0FnF6B9.js +0 -1
- package/admin/custom/assets/virtualExposes-brKKZvi0.js +0 -1
- package/admin/custom/assets/virtual_mf-REMOTE_ENTRY_ID___mfe_internal__ConfigCustomAssistant__customComponents_js-CRhwj12f.js +0 -1
- package/admin/custom/assets/virtual_mf-exposes___mfe_internal__ConfigCustomAssistant__customComponents_js-C5XHUc1J.js +0 -2
- package/admin/custom/assets/vite-preload-helper-B7qeedMF.js +0 -1
|
@@ -0,0 +1,274 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.CachedTts = exports.DEFAULT_CACHE_BYTES = exports.MAX_CACHE_CHARS = exports.MAX_TTS_CHARS = void 0;
|
|
37
|
+
exports.isSsml = isSsml;
|
|
38
|
+
exports.stripSsml = stripSsml;
|
|
39
|
+
exports.truncateForTts = truncateForTts;
|
|
40
|
+
/**
|
|
41
|
+
* The plumbing every TTS engine benefits from: a disk cache, a length limit, and SSML handling.
|
|
42
|
+
*
|
|
43
|
+
* Wrapped around any {@link TtsEngine}, so each engine stays a thin provider binding:
|
|
44
|
+
* - **cache** — the assistant says the same short things over and over ("Ok.", "Timer abgelaufen",
|
|
45
|
+
* a trigger's announcement). Synthesising those again on every repeat costs a cloud round-trip and
|
|
46
|
+
* a few hundred milliseconds of latency for audio that is byte-for-byte identical. Only short texts
|
|
47
|
+
* are cached: a long one-off answer will never be asked for twice, so it would only fill the disk.
|
|
48
|
+
* - **length limit** — an LLM can answer with three paragraphs. Spoken at a speaker that is a minute
|
|
49
|
+
* of monologue nobody can interrupt, so the text is cut at the last sentence end that fits.
|
|
50
|
+
* - **SSML** — Azure and Polly can speak `<speak>…</speak>` (pauses, emphasis); OpenAI and the local
|
|
51
|
+
* engines cannot, so there the tags are stripped rather than read out loud.
|
|
52
|
+
*
|
|
53
|
+
* All three come from the Python original (`C:\iot\Hannah`, `core/hannah/tts.py`: `_TTSCache`,
|
|
54
|
+
* `_truncate_for_tts`, `_strip_ssml_tags`), including the detail that the sample rate belongs in the
|
|
55
|
+
* cache entry — serving 24 kHz audio as 16 kHz plays it back at the wrong pitch.
|
|
56
|
+
*/
|
|
57
|
+
const crypto = __importStar(require("node:crypto"));
|
|
58
|
+
const fs = __importStar(require("node:fs"));
|
|
59
|
+
const path = __importStar(require("node:path"));
|
|
60
|
+
/** Hard limit for a spoken text. Hannah's 400 characters ≈ 25 seconds of speech. */
|
|
61
|
+
exports.MAX_TTS_CHARS = 400;
|
|
62
|
+
/** Texts up to this length are worth caching; a longer one is a one-off answer. */
|
|
63
|
+
exports.MAX_CACHE_CHARS = 200;
|
|
64
|
+
/** Default budget for the cache directory. ~1 MB per 20 s of 24 kHz 16-bit audio. */
|
|
65
|
+
exports.DEFAULT_CACHE_BYTES = 64 * 1024 * 1024;
|
|
66
|
+
/** Does this text ask for SSML? */
|
|
67
|
+
function isSsml(text) {
|
|
68
|
+
return /^\s*<speak[\s>]/i.test(text || '');
|
|
69
|
+
}
|
|
70
|
+
/** Drop the XML tags, for an engine that cannot speak SSML (reading them aloud would be worse). */
|
|
71
|
+
function stripSsml(text) {
|
|
72
|
+
return (text || '')
|
|
73
|
+
.replace(/<[^>]+>/g, ' ')
|
|
74
|
+
.replace(/\s+/g, ' ')
|
|
75
|
+
.trim();
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Cut `text` to `maxChars`, preferring the last sentence end in the second half so the result does not
|
|
79
|
+
* stop mid-thought; failing that, the last word boundary plus an ellipsis. SSML is left alone — cutting
|
|
80
|
+
* it would break the markup, and it is written by us, not by a model.
|
|
81
|
+
*/
|
|
82
|
+
function truncateForTts(text, maxChars = exports.MAX_TTS_CHARS) {
|
|
83
|
+
const t = (text || '').trim();
|
|
84
|
+
if (t.length <= maxChars || isSsml(t)) {
|
|
85
|
+
return t;
|
|
86
|
+
}
|
|
87
|
+
const chunk = t.slice(0, maxChars);
|
|
88
|
+
for (const sep of ['.', '!', '?', '…']) {
|
|
89
|
+
const idx = chunk.lastIndexOf(sep);
|
|
90
|
+
if (idx > maxChars / 2) {
|
|
91
|
+
return chunk.slice(0, idx + 1);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
const lastSpace = chunk.lastIndexOf(' ');
|
|
95
|
+
return `${(lastSpace > 0 ? chunk.slice(0, lastSpace) : chunk).trim()} …`;
|
|
96
|
+
}
|
|
97
|
+
/** A cache entry on disk: the sample rate first, so an entry can never be replayed at the wrong pitch. */
|
|
98
|
+
function encodeEntry(result) {
|
|
99
|
+
const head = Buffer.alloc(4);
|
|
100
|
+
head.writeUInt32LE(result.sampleRate, 0);
|
|
101
|
+
return Buffer.concat([head, result.pcm]);
|
|
102
|
+
}
|
|
103
|
+
function decodeEntry(data) {
|
|
104
|
+
if (data.length < 8) {
|
|
105
|
+
return null;
|
|
106
|
+
}
|
|
107
|
+
const sampleRate = data.readUInt32LE(0);
|
|
108
|
+
// A plausible speech rate; anything else means the file is not one of ours (or is truncated).
|
|
109
|
+
if (sampleRate < 8000 || sampleRate > 48000) {
|
|
110
|
+
return null;
|
|
111
|
+
}
|
|
112
|
+
return { pcm: data.subarray(4), sampleRate };
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* A {@link TtsEngine} wrapper that truncates, serves repeats from disk and resolves SSML against what the
|
|
116
|
+
* wrapped engine actually supports. Every cache operation is best-effort: a broken or read-only cache
|
|
117
|
+
* directory must never stop the assistant from speaking.
|
|
118
|
+
*/
|
|
119
|
+
class CachedTts {
|
|
120
|
+
engine;
|
|
121
|
+
opts;
|
|
122
|
+
maxChars;
|
|
123
|
+
maxCacheChars;
|
|
124
|
+
maxBytes;
|
|
125
|
+
ready = false;
|
|
126
|
+
constructor(engine, opts) {
|
|
127
|
+
this.engine = engine;
|
|
128
|
+
this.opts = opts;
|
|
129
|
+
this.maxChars = opts.maxChars ?? exports.MAX_TTS_CHARS;
|
|
130
|
+
this.maxCacheChars = opts.maxCacheChars ?? exports.MAX_CACHE_CHARS;
|
|
131
|
+
this.maxBytes = opts.maxBytes ?? exports.DEFAULT_CACHE_BYTES;
|
|
132
|
+
}
|
|
133
|
+
prepare(lang) {
|
|
134
|
+
return this.engine.prepare ? this.engine.prepare(lang) : Promise.resolve();
|
|
135
|
+
}
|
|
136
|
+
async synthesize(text, lang) {
|
|
137
|
+
const wanted = this.maxChars > 0 ? truncateForTts(text, this.maxChars) : (text || '').trim();
|
|
138
|
+
if (wanted.length < (text || '').trim().length) {
|
|
139
|
+
this.opts.log.warn(`TTS text shortened from ${text.trim().length} to ${wanted.length} characters.`);
|
|
140
|
+
}
|
|
141
|
+
const cacheable = this.maxCacheChars === 0 || wanted.length <= this.maxCacheChars;
|
|
142
|
+
const file = cacheable ? this.entryPath(wanted, lang) : '';
|
|
143
|
+
if (file) {
|
|
144
|
+
const hit = this.read(file);
|
|
145
|
+
if (hit) {
|
|
146
|
+
this.opts.log.debug(`TTS cache hit: "${wanted.slice(0, 40)}"`);
|
|
147
|
+
return hit;
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
const result = await this.speak(wanted, lang);
|
|
151
|
+
if (file && result.pcm.length) {
|
|
152
|
+
this.write(file, result);
|
|
153
|
+
}
|
|
154
|
+
return result;
|
|
155
|
+
}
|
|
156
|
+
/** Hand SSML to the engine only if it can speak it; otherwise speak the text without the markup. */
|
|
157
|
+
speak(text, lang) {
|
|
158
|
+
if (!isSsml(text)) {
|
|
159
|
+
return this.engine.synthesize(text, lang);
|
|
160
|
+
}
|
|
161
|
+
if (this.engine.synthesizeSsml) {
|
|
162
|
+
return this.engine.synthesizeSsml(text, lang);
|
|
163
|
+
}
|
|
164
|
+
this.opts.log.debug('TTS engine cannot speak SSML — using the plain text.');
|
|
165
|
+
return this.engine.synthesize(stripSsml(text), lang);
|
|
166
|
+
}
|
|
167
|
+
/** Is this text already on disk? Used by the warm-up to skip what it has. */
|
|
168
|
+
has(text, lang) {
|
|
169
|
+
const wanted = this.maxChars > 0 ? truncateForTts(text, this.maxChars) : (text || '').trim();
|
|
170
|
+
const file = this.entryPath(wanted, lang);
|
|
171
|
+
try {
|
|
172
|
+
return fs.existsSync(file);
|
|
173
|
+
}
|
|
174
|
+
catch {
|
|
175
|
+
return false;
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
/** Pre-synthesise frequently spoken phrases, so the first "Ok." of the day is not a cloud call. */
|
|
179
|
+
async warm(phrases, lang) {
|
|
180
|
+
let added = 0;
|
|
181
|
+
for (const phrase of phrases) {
|
|
182
|
+
const text = (phrase || '').trim();
|
|
183
|
+
if (!text || this.has(text, lang)) {
|
|
184
|
+
continue;
|
|
185
|
+
}
|
|
186
|
+
try {
|
|
187
|
+
await this.synthesize(text, lang);
|
|
188
|
+
added++;
|
|
189
|
+
}
|
|
190
|
+
catch (e) {
|
|
191
|
+
this.opts.log.debug(`TTS warm-up failed for "${text.slice(0, 30)}": ${e.message}`);
|
|
192
|
+
break; // the engine is unavailable — no point trying the rest
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
if (added) {
|
|
196
|
+
this.opts.log.info(`TTS cache warmed with ${added} phrase(s).`);
|
|
197
|
+
}
|
|
198
|
+
return added;
|
|
199
|
+
}
|
|
200
|
+
/** The language is part of the key: the same words are spoken differently in another language. */
|
|
201
|
+
entryPath(text, lang) {
|
|
202
|
+
const key = crypto.createHash('sha256').update(`${lang}\u0000${text}`).digest('hex').slice(0, 20);
|
|
203
|
+
return path.join(this.opts.dir, `${key}.pcm`);
|
|
204
|
+
}
|
|
205
|
+
read(file) {
|
|
206
|
+
try {
|
|
207
|
+
if (!fs.existsSync(file)) {
|
|
208
|
+
return null;
|
|
209
|
+
}
|
|
210
|
+
const entry = decodeEntry(fs.readFileSync(file));
|
|
211
|
+
if (entry) {
|
|
212
|
+
// Touch it, so pruning drops what nobody asks for rather than what is merely old.
|
|
213
|
+
const now = new Date();
|
|
214
|
+
try {
|
|
215
|
+
fs.utimesSync(file, now, now);
|
|
216
|
+
}
|
|
217
|
+
catch {
|
|
218
|
+
/* a read-only cache is still a usable cache */
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
return entry;
|
|
222
|
+
}
|
|
223
|
+
catch (e) {
|
|
224
|
+
this.opts.log.debug(`TTS cache read failed: ${e.message}`);
|
|
225
|
+
return null;
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
write(file, result) {
|
|
229
|
+
try {
|
|
230
|
+
if (!this.ready) {
|
|
231
|
+
fs.mkdirSync(this.opts.dir, { recursive: true });
|
|
232
|
+
this.ready = true;
|
|
233
|
+
}
|
|
234
|
+
fs.writeFileSync(file, encodeEntry(result));
|
|
235
|
+
this.prune();
|
|
236
|
+
}
|
|
237
|
+
catch (e) {
|
|
238
|
+
this.opts.log.debug(`TTS cache write failed: ${e.message}`);
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
/** Keep the directory under its budget by dropping the least recently used entries. */
|
|
242
|
+
prune() {
|
|
243
|
+
if (this.maxBytes <= 0) {
|
|
244
|
+
return;
|
|
245
|
+
}
|
|
246
|
+
try {
|
|
247
|
+
const entries = fs
|
|
248
|
+
.readdirSync(this.opts.dir)
|
|
249
|
+
.filter(name => name.endsWith('.pcm'))
|
|
250
|
+
.map(name => {
|
|
251
|
+
const full = path.join(this.opts.dir, name);
|
|
252
|
+
const stat = fs.statSync(full);
|
|
253
|
+
return { full, size: stat.size, used: stat.mtimeMs };
|
|
254
|
+
});
|
|
255
|
+
let total = entries.reduce((sum, e) => sum + e.size, 0);
|
|
256
|
+
if (total <= this.maxBytes) {
|
|
257
|
+
return;
|
|
258
|
+
}
|
|
259
|
+
for (const entry of entries.sort((a, b) => a.used - b.used)) {
|
|
260
|
+
if (total <= this.maxBytes) {
|
|
261
|
+
break;
|
|
262
|
+
}
|
|
263
|
+
fs.unlinkSync(entry.full);
|
|
264
|
+
total -= entry.size;
|
|
265
|
+
}
|
|
266
|
+
this.opts.log.debug(`TTS cache pruned to ${Math.round(total / 1024)} KiB.`);
|
|
267
|
+
}
|
|
268
|
+
catch (e) {
|
|
269
|
+
this.opts.log.debug(`TTS cache prune failed: ${e.message}`);
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
exports.CachedTts = CachedTts;
|
|
274
|
+
//# sourceMappingURL=ttsCache.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ttsCache.js","sourceRoot":"","sources":["../../../src/lib/voice/ttsCache.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAiCA,wBAEC;AAGD,8BAKC;AAOD,wCAcC;AAhED;;;;;;;;;;;;;;;;GAgBG;AACH,oDAAsC;AACtC,4CAA8B;AAC9B,gDAAkC;AAIlC,oFAAoF;AACvE,QAAA,aAAa,GAAG,GAAG,CAAC;AAEjC,mFAAmF;AACtE,QAAA,eAAe,GAAG,GAAG,CAAC;AAEnC,qFAAqF;AACxE,QAAA,mBAAmB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;AAEpD,mCAAmC;AACnC,SAAgB,MAAM,CAAC,IAAY;IAC/B,OAAO,kBAAkB,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;AAC/C,CAAC;AAED,mGAAmG;AACnG,SAAgB,SAAS,CAAC,IAAY;IAClC,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC;SACd,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC;SACxB,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,IAAI,EAAE,CAAC;AAChB,CAAC;AAED;;;;GAIG;AACH,SAAgB,cAAc,CAAC,IAAY,EAAE,QAAQ,GAAG,qBAAa;IACjE,MAAM,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAC9B,IAAI,CAAC,CAAC,MAAM,IAAI,QAAQ,IAAI,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;QACpC,OAAO,CAAC,CAAC;IACb,CAAC;IACD,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IACnC,KAAK,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC;QACrC,MAAM,GAAG,GAAG,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QACnC,IAAI,GAAG,GAAG,QAAQ,GAAG,CAAC,EAAE,CAAC;YACrB,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC;QACnC,CAAC;IACL,CAAC;IACD,MAAM,SAAS,GAAG,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IACzC,OAAO,GAAG,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC;AAC7E,CAAC;AAED,0GAA0G;AAC1G,SAAS,WAAW,CAAC,MAAiB;IAClC,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAC7B,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;IACzC,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;AAC7C,CAAC;AAED,SAAS,WAAW,CAAC,IAAY;IAC7B,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClB,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,MAAM,UAAU,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;IACxC,8FAA8F;IAC9F,IAAI,UAAU,GAAG,IAAI,IAAI,UAAU,GAAG,KAAK,EAAE,CAAC;QAC1C,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC;AACjD,CAAC;AAcD;;;;GAIG;AACH,MAAa,SAAS;IAOG;IACA;IAPJ,QAAQ,CAAS;IACjB,aAAa,CAAS;IACtB,QAAQ,CAAS;IAC1B,KAAK,GAAG,KAAK,CAAC;IAEtB,YACqB,MAAiB,EACjB,IAAqB;QADrB,WAAM,GAAN,MAAM,CAAW;QACjB,SAAI,GAAJ,IAAI,CAAiB;QAEtC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,qBAAa,CAAC;QAC/C,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,aAAa,IAAI,uBAAe,CAAC;QAC3D,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,2BAAmB,CAAC;IACzD,CAAC;IAED,OAAO,CAAC,IAAY;QAChB,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;IAC/E,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,IAAY,EAAE,IAAY;QACvC,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAC7F,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC;YAC7C,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,2BAA2B,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,OAAO,MAAM,CAAC,MAAM,cAAc,CAAC,CAAC;QACxG,CAAC;QACD,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,KAAK,CAAC,IAAI,MAAM,CAAC,MAAM,IAAI,IAAI,CAAC,aAAa,CAAC;QAClF,MAAM,IAAI,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3D,IAAI,IAAI,EAAE,CAAC;YACP,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC5B,IAAI,GAAG,EAAE,CAAC;gBACN,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,mBAAmB,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC;gBAC/D,OAAO,GAAG,CAAC;YACf,CAAC;QACL,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAC9C,IAAI,IAAI,IAAI,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC;YAC5B,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAC7B,CAAC;QACD,OAAO,MAAM,CAAC;IAClB,CAAC;IAED,oGAAoG;IAC5F,KAAK,CAAC,IAAY,EAAE,IAAY;QACpC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;YAChB,OAAO,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAC9C,CAAC;QACD,IAAI,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC;YAC7B,OAAO,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAClD,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,sDAAsD,CAAC,CAAC;QAC5E,OAAO,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,CAAC;IACzD,CAAC;IAED,6EAA6E;IAC7E,GAAG,CAAC,IAAY,EAAE,IAAY;QAC1B,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAC7F,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAC1C,IAAI,CAAC;YACD,OAAO,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QAC/B,CAAC;QAAC,MAAM,CAAC;YACL,OAAO,KAAK,CAAC;QACjB,CAAC;IACL,CAAC;IAED,mGAAmG;IACnG,KAAK,CAAC,IAAI,CAAC,OAAiB,EAAE,IAAY;QACtC,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC3B,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;YACnC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,CAAC;gBAChC,SAAS;YACb,CAAC;YACD,IAAI,CAAC;gBACD,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;gBAClC,KAAK,EAAE,CAAC;YACZ,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACT,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,2BAA2B,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,MAAO,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;gBAC9F,MAAM,CAAC,uDAAuD;YAClE,CAAC;QACL,CAAC;QACD,IAAI,KAAK,EAAE,CAAC;YACR,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,yBAAyB,KAAK,aAAa,CAAC,CAAC;QACpE,CAAC;QACD,OAAO,KAAK,CAAC;IACjB,CAAC;IAED,kGAAkG;IAC1F,SAAS,CAAC,IAAY,EAAE,IAAY;QACxC,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,IAAI,SAAS,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAClG,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,GAAG,MAAM,CAAC,CAAC;IAClD,CAAC;IAEO,IAAI,CAAC,IAAY;QACrB,IAAI,CAAC;YACD,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;gBACvB,OAAO,IAAI,CAAC;YAChB,CAAC;YACD,MAAM,KAAK,GAAG,WAAW,CAAC,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC;YACjD,IAAI,KAAK,EAAE,CAAC;gBACR,kFAAkF;gBAClF,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,IAAI,CAAC;oBACD,EAAE,CAAC,UAAU,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;gBAClC,CAAC;gBAAC,MAAM,CAAC;oBACL,+CAA+C;gBACnD,CAAC;YACL,CAAC;YACD,OAAO,KAAK,CAAC;QACjB,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,0BAA2B,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;YACtE,OAAO,IAAI,CAAC;QAChB,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,IAAY,EAAE,MAAiB;QACzC,IAAI,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;gBACd,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;gBACjD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;YACtB,CAAC;YACD,EAAE,CAAC,aAAa,CAAC,IAAI,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC;YAC5C,IAAI,CAAC,KAAK,EAAE,CAAC;QACjB,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,2BAA4B,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;QAC3E,CAAC;IACL,CAAC;IAED,uFAAuF;IAC/E,KAAK;QACT,IAAI,IAAI,CAAC,QAAQ,IAAI,CAAC,EAAE,CAAC;YACrB,OAAO;QACX,CAAC;QACD,IAAI,CAAC;YACD,MAAM,OAAO,GAAG,EAAE;iBACb,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;iBAC1B,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;iBACrC,GAAG,CAAC,IAAI,CAAC,EAAE;gBACR,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;gBAC5C,MAAM,IAAI,GAAG,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;gBAC/B,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC;YACzD,CAAC,CAAC,CAAC;YACP,IAAI,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YACxD,IAAI,KAAK,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACzB,OAAO;YACX,CAAC;YACD,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC1D,IAAI,KAAK,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACzB,MAAM;gBACV,CAAC;gBACD,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAC1B,KAAK,IAAI,KAAK,CAAC,IAAI,CAAC;YACxB,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,uBAAuB,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC;QAChF,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,2BAA4B,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;QAC3E,CAAC;IACL,CAAC;CACJ;AA5JD,8BA4JC","sourcesContent":["/**\n * The plumbing every TTS engine benefits from: a disk cache, a length limit, and SSML handling.\n *\n * Wrapped around any {@link TtsEngine}, so each engine stays a thin provider binding:\n * - **cache** — the assistant says the same short things over and over (\"Ok.\", \"Timer abgelaufen\",\n * a trigger's announcement). Synthesising those again on every repeat costs a cloud round-trip and\n * a few hundred milliseconds of latency for audio that is byte-for-byte identical. Only short texts\n * are cached: a long one-off answer will never be asked for twice, so it would only fill the disk.\n * - **length limit** — an LLM can answer with three paragraphs. Spoken at a speaker that is a minute\n * of monologue nobody can interrupt, so the text is cut at the last sentence end that fits.\n * - **SSML** — Azure and Polly can speak `<speak>…</speak>` (pauses, emphasis); OpenAI and the local\n * engines cannot, so there the tags are stripped rather than read out loud.\n *\n * All three come from the Python original (`C:\\iot\\Hannah`, `core/hannah/tts.py`: `_TTSCache`,\n * `_truncate_for_tts`, `_strip_ssml_tags`), including the detail that the sample rate belongs in the\n * cache entry — serving 24 kHz audio as 16 kHz plays it back at the wrong pitch.\n */\nimport * as crypto from 'node:crypto';\nimport * as fs from 'node:fs';\nimport * as path from 'node:path';\nimport type { TtsEngine, TtsResult } from './tts';\nimport type { VoiceLogger } from './download';\n\n/** Hard limit for a spoken text. Hannah's 400 characters ≈ 25 seconds of speech. */\nexport const MAX_TTS_CHARS = 400;\n\n/** Texts up to this length are worth caching; a longer one is a one-off answer. */\nexport const MAX_CACHE_CHARS = 200;\n\n/** Default budget for the cache directory. ~1 MB per 20 s of 24 kHz 16-bit audio. */\nexport const DEFAULT_CACHE_BYTES = 64 * 1024 * 1024;\n\n/** Does this text ask for SSML? */\nexport function isSsml(text: string): boolean {\n return /^\\s*<speak[\\s>]/i.test(text || '');\n}\n\n/** Drop the XML tags, for an engine that cannot speak SSML (reading them aloud would be worse). */\nexport function stripSsml(text: string): string {\n return (text || '')\n .replace(/<[^>]+>/g, ' ')\n .replace(/\\s+/g, ' ')\n .trim();\n}\n\n/**\n * Cut `text` to `maxChars`, preferring the last sentence end in the second half so the result does not\n * stop mid-thought; failing that, the last word boundary plus an ellipsis. SSML is left alone — cutting\n * it would break the markup, and it is written by us, not by a model.\n */\nexport function truncateForTts(text: string, maxChars = MAX_TTS_CHARS): string {\n const t = (text || '').trim();\n if (t.length <= maxChars || isSsml(t)) {\n return t;\n }\n const chunk = t.slice(0, maxChars);\n for (const sep of ['.', '!', '?', '…']) {\n const idx = chunk.lastIndexOf(sep);\n if (idx > maxChars / 2) {\n return chunk.slice(0, idx + 1);\n }\n }\n const lastSpace = chunk.lastIndexOf(' ');\n return `${(lastSpace > 0 ? chunk.slice(0, lastSpace) : chunk).trim()} …`;\n}\n\n/** A cache entry on disk: the sample rate first, so an entry can never be replayed at the wrong pitch. */\nfunction encodeEntry(result: TtsResult): Buffer {\n const head = Buffer.alloc(4);\n head.writeUInt32LE(result.sampleRate, 0);\n return Buffer.concat([head, result.pcm]);\n}\n\nfunction decodeEntry(data: Buffer): TtsResult | null {\n if (data.length < 8) {\n return null;\n }\n const sampleRate = data.readUInt32LE(0);\n // A plausible speech rate; anything else means the file is not one of ours (or is truncated).\n if (sampleRate < 8000 || sampleRate > 48000) {\n return null;\n }\n return { pcm: data.subarray(4), sampleRate };\n}\n\nexport interface TtsCacheOptions {\n /** Directory for this engine's entries (one per provider+voice, so a voice change cannot serve stale audio). */\n dir: string;\n log: VoiceLogger;\n /** Cut texts longer than this before synthesising (0 = no limit). */\n maxChars?: number;\n /** Only cache texts up to this length (0 = cache everything). */\n maxCacheChars?: number;\n /** Prune the oldest entries once the directory exceeds this (0 = no limit). */\n maxBytes?: number;\n}\n\n/**\n * A {@link TtsEngine} wrapper that truncates, serves repeats from disk and resolves SSML against what the\n * wrapped engine actually supports. Every cache operation is best-effort: a broken or read-only cache\n * directory must never stop the assistant from speaking.\n */\nexport class CachedTts implements TtsEngine {\n private readonly maxChars: number;\n private readonly maxCacheChars: number;\n private readonly maxBytes: number;\n private ready = false;\n\n constructor(\n private readonly engine: TtsEngine,\n private readonly opts: TtsCacheOptions,\n ) {\n this.maxChars = opts.maxChars ?? MAX_TTS_CHARS;\n this.maxCacheChars = opts.maxCacheChars ?? MAX_CACHE_CHARS;\n this.maxBytes = opts.maxBytes ?? DEFAULT_CACHE_BYTES;\n }\n\n prepare(lang: string): Promise<void> {\n return this.engine.prepare ? this.engine.prepare(lang) : Promise.resolve();\n }\n\n async synthesize(text: string, lang: string): Promise<TtsResult> {\n const wanted = this.maxChars > 0 ? truncateForTts(text, this.maxChars) : (text || '').trim();\n if (wanted.length < (text || '').trim().length) {\n this.opts.log.warn(`TTS text shortened from ${text.trim().length} to ${wanted.length} characters.`);\n }\n const cacheable = this.maxCacheChars === 0 || wanted.length <= this.maxCacheChars;\n const file = cacheable ? this.entryPath(wanted, lang) : '';\n if (file) {\n const hit = this.read(file);\n if (hit) {\n this.opts.log.debug(`TTS cache hit: \"${wanted.slice(0, 40)}\"`);\n return hit;\n }\n }\n const result = await this.speak(wanted, lang);\n if (file && result.pcm.length) {\n this.write(file, result);\n }\n return result;\n }\n\n /** Hand SSML to the engine only if it can speak it; otherwise speak the text without the markup. */\n private speak(text: string, lang: string): Promise<TtsResult> {\n if (!isSsml(text)) {\n return this.engine.synthesize(text, lang);\n }\n if (this.engine.synthesizeSsml) {\n return this.engine.synthesizeSsml(text, lang);\n }\n this.opts.log.debug('TTS engine cannot speak SSML — using the plain text.');\n return this.engine.synthesize(stripSsml(text), lang);\n }\n\n /** Is this text already on disk? Used by the warm-up to skip what it has. */\n has(text: string, lang: string): boolean {\n const wanted = this.maxChars > 0 ? truncateForTts(text, this.maxChars) : (text || '').trim();\n const file = this.entryPath(wanted, lang);\n try {\n return fs.existsSync(file);\n } catch {\n return false;\n }\n }\n\n /** Pre-synthesise frequently spoken phrases, so the first \"Ok.\" of the day is not a cloud call. */\n async warm(phrases: string[], lang: string): Promise<number> {\n let added = 0;\n for (const phrase of phrases) {\n const text = (phrase || '').trim();\n if (!text || this.has(text, lang)) {\n continue;\n }\n try {\n await this.synthesize(text, lang);\n added++;\n } catch (e) {\n this.opts.log.debug(`TTS warm-up failed for \"${text.slice(0, 30)}\": ${(e as Error).message}`);\n break; // the engine is unavailable — no point trying the rest\n }\n }\n if (added) {\n this.opts.log.info(`TTS cache warmed with ${added} phrase(s).`);\n }\n return added;\n }\n\n /** The language is part of the key: the same words are spoken differently in another language. */\n private entryPath(text: string, lang: string): string {\n const key = crypto.createHash('sha256').update(`${lang}\\u0000${text}`).digest('hex').slice(0, 20);\n return path.join(this.opts.dir, `${key}.pcm`);\n }\n\n private read(file: string): TtsResult | null {\n try {\n if (!fs.existsSync(file)) {\n return null;\n }\n const entry = decodeEntry(fs.readFileSync(file));\n if (entry) {\n // Touch it, so pruning drops what nobody asks for rather than what is merely old.\n const now = new Date();\n try {\n fs.utimesSync(file, now, now);\n } catch {\n /* a read-only cache is still a usable cache */\n }\n }\n return entry;\n } catch (e) {\n this.opts.log.debug(`TTS cache read failed: ${(e as Error).message}`);\n return null;\n }\n }\n\n private write(file: string, result: TtsResult): void {\n try {\n if (!this.ready) {\n fs.mkdirSync(this.opts.dir, { recursive: true });\n this.ready = true;\n }\n fs.writeFileSync(file, encodeEntry(result));\n this.prune();\n } catch (e) {\n this.opts.log.debug(`TTS cache write failed: ${(e as Error).message}`);\n }\n }\n\n /** Keep the directory under its budget by dropping the least recently used entries. */\n private prune(): void {\n if (this.maxBytes <= 0) {\n return;\n }\n try {\n const entries = fs\n .readdirSync(this.opts.dir)\n .filter(name => name.endsWith('.pcm'))\n .map(name => {\n const full = path.join(this.opts.dir, name);\n const stat = fs.statSync(full);\n return { full, size: stat.size, used: stat.mtimeMs };\n });\n let total = entries.reduce((sum, e) => sum + e.size, 0);\n if (total <= this.maxBytes) {\n return;\n }\n for (const entry of entries.sort((a, b) => a.used - b.used)) {\n if (total <= this.maxBytes) {\n break;\n }\n fs.unlinkSync(entry.full);\n total -= entry.size;\n }\n this.opts.log.debug(`TTS cache pruned to ${Math.round(total / 1024)} KiB.`);\n } catch (e) {\n this.opts.log.debug(`TTS cache prune failed: ${(e as Error).message}`);\n }\n }\n}\n"]}
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* End-of-speech detection for satellites that stream microphone audio until they are told to stop.
|
|
4
|
+
*
|
|
5
|
+
* The Hannah/UDP protocol has the satellite decide when an utterance ended (`audio_end`), and Wyoming
|
|
6
|
+
* clients send `audio-stop` themselves. ESPHome voice satellites do neither: they keep streaming until
|
|
7
|
+
* the server sends `STT_VAD_END`/`STT_END`, so the decision has to happen here.
|
|
8
|
+
*
|
|
9
|
+
* This is a plain energy gate, not a neural VAD — good enough to close an utterance, and it costs
|
|
10
|
+
* nothing per frame. Speech must first rise above `startLevel`; after that, `silenceMs` worth of frames
|
|
11
|
+
* below `endLevel` ends it. `maxMs` is the hard stop for rooms that are simply loud.
|
|
12
|
+
*
|
|
13
|
+
* Fixed thresholds only work when the microphone level is known, and on a real satellite it is not.
|
|
14
|
+
* Measured on a ThirdReality Voice & Music Assistant: the device plays a wake-acknowledgement chirp
|
|
15
|
+
* into its own microphone that peaks at full scale (RMS ~16000, clipped), while the speech that follows
|
|
16
|
+
* arrives at RMS 358–469 — below the 700 default for `startLevel` and right on the 400 for `endLevel`.
|
|
17
|
+
* That breaks the gate twice over: `sawSpeech` is satisfied by the chirp rather than by any voice, so
|
|
18
|
+
* the "nothing but speech-free audio" guard never fires, and half the real speech frames count as
|
|
19
|
+
* silence, so the utterance can be cut off mid-sentence.
|
|
20
|
+
*
|
|
21
|
+
* Hence two opt-in knobs, both enabled by the ESPHome transport:
|
|
22
|
+
*
|
|
23
|
+
* - `skipMs` drops the leading chirp from the *analysis* (never from the audio, which still goes to STT
|
|
24
|
+
* in full — a fast speaker may already be talking).
|
|
25
|
+
* - `adaptive` derives the thresholds from a tracked noise floor instead of fixed levels. The floor
|
|
26
|
+
* follows the quietest frame seen (instant down, very slow up), so it settles on room noise even when
|
|
27
|
+
* someone starts talking immediately — which a mean or median over a calibration window would not.
|
|
28
|
+
*/
|
|
29
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
30
|
+
exports.SpeechDetector = void 0;
|
|
31
|
+
exports.rms = rms;
|
|
32
|
+
/** Root-mean-square amplitude (0…32767) of a mono 16-bit signed LE PCM buffer. */
|
|
33
|
+
function rms(pcm) {
|
|
34
|
+
const samples = Math.floor(pcm.length / 2);
|
|
35
|
+
if (!samples) {
|
|
36
|
+
return 0;
|
|
37
|
+
}
|
|
38
|
+
let sum = 0;
|
|
39
|
+
for (let i = 0; i < samples; i++) {
|
|
40
|
+
const value = pcm.readInt16LE(i * 2);
|
|
41
|
+
sum += value * value;
|
|
42
|
+
}
|
|
43
|
+
return Math.sqrt(sum / samples);
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* How fast the tracked noise floor may follow a *rising* ambient level, per analysed frame (30 ms).
|
|
47
|
+
* Fast enough that a genuinely noisy room is matched within about a second, slow enough that a voice
|
|
48
|
+
* cannot pull the floor up to its own level before it has been recognised as speech.
|
|
49
|
+
*/
|
|
50
|
+
const NOISE_RISE = 0.02;
|
|
51
|
+
class SpeechDetector {
|
|
52
|
+
frameBytes;
|
|
53
|
+
frameMs;
|
|
54
|
+
fixedStart;
|
|
55
|
+
fixedEnd;
|
|
56
|
+
silenceMs;
|
|
57
|
+
maxMs;
|
|
58
|
+
skipMs;
|
|
59
|
+
adaptive;
|
|
60
|
+
minStartLevel;
|
|
61
|
+
minEndLevel;
|
|
62
|
+
startFactor;
|
|
63
|
+
endFactor;
|
|
64
|
+
warmupMs;
|
|
65
|
+
pending = Buffer.alloc(0);
|
|
66
|
+
speechSeen = false;
|
|
67
|
+
silentMs = 0;
|
|
68
|
+
/** Tracked room level in adaptive mode; null until the first analysed frame. */
|
|
69
|
+
floor = null;
|
|
70
|
+
/** Audio seen so far *including* the skipped lead-in, in milliseconds. */
|
|
71
|
+
elapsedMs = 0;
|
|
72
|
+
/** Audio analysed so far, in milliseconds (exposed for logging). */
|
|
73
|
+
totalMs = 0;
|
|
74
|
+
/** Loudest frame seen so far (exposed for logging — the usual clue when a mic is mis-configured). */
|
|
75
|
+
peak = 0;
|
|
76
|
+
constructor({ sampleRate = 16000, frameMs = 30, startLevel = 700, endLevel = 400, silenceMs = 900, maxMs = 12000, skipMs = 0, adaptive = false, minStartLevel = 250, minEndLevel = 150, startFactor = 3.5, endFactor = 2, warmupMs = 150, } = {}) {
|
|
77
|
+
this.frameBytes = Math.round((sampleRate * frameMs) / 1000) * 2;
|
|
78
|
+
this.frameMs = frameMs;
|
|
79
|
+
this.fixedStart = startLevel;
|
|
80
|
+
this.fixedEnd = endLevel;
|
|
81
|
+
this.silenceMs = silenceMs;
|
|
82
|
+
this.maxMs = maxMs;
|
|
83
|
+
this.skipMs = skipMs;
|
|
84
|
+
this.adaptive = adaptive;
|
|
85
|
+
this.minStartLevel = minStartLevel;
|
|
86
|
+
this.minEndLevel = minEndLevel;
|
|
87
|
+
this.startFactor = startFactor;
|
|
88
|
+
this.endFactor = endFactor;
|
|
89
|
+
this.warmupMs = warmupMs;
|
|
90
|
+
}
|
|
91
|
+
/** True once any frame crossed {@link startLevel} — an utterance that never did is room noise. */
|
|
92
|
+
get sawSpeech() {
|
|
93
|
+
return this.speechSeen;
|
|
94
|
+
}
|
|
95
|
+
/** Tracked room level (adaptive mode), for logging. 0 before the first analysed frame. */
|
|
96
|
+
get noiseFloor() {
|
|
97
|
+
return this.floor ?? 0;
|
|
98
|
+
}
|
|
99
|
+
/** Current speech threshold — fixed, or derived from the noise floor in adaptive mode. */
|
|
100
|
+
get startLevel() {
|
|
101
|
+
return this.adaptive ? Math.max(this.minStartLevel, (this.floor ?? 0) * this.startFactor) : this.fixedStart;
|
|
102
|
+
}
|
|
103
|
+
/** Current silence threshold. Never above {@link startLevel}, so the gate keeps its hysteresis. */
|
|
104
|
+
get endLevel() {
|
|
105
|
+
if (!this.adaptive) {
|
|
106
|
+
return this.fixedEnd;
|
|
107
|
+
}
|
|
108
|
+
return Math.min(this.startLevel, Math.max(this.minEndLevel, (this.floor ?? 0) * this.endFactor));
|
|
109
|
+
}
|
|
110
|
+
/** Feed one audio chunk (any size). Returns why the utterance ended, or null to keep listening. */
|
|
111
|
+
push(chunk) {
|
|
112
|
+
this.pending = this.pending.length ? Buffer.concat([this.pending, chunk]) : chunk;
|
|
113
|
+
let verdict = null;
|
|
114
|
+
while (this.pending.length >= this.frameBytes) {
|
|
115
|
+
const frame = this.pending.subarray(0, this.frameBytes);
|
|
116
|
+
this.pending = this.pending.subarray(this.frameBytes);
|
|
117
|
+
this.elapsedMs += this.frameMs;
|
|
118
|
+
const level = rms(frame);
|
|
119
|
+
if (level > this.peak) {
|
|
120
|
+
this.peak = level;
|
|
121
|
+
}
|
|
122
|
+
// The device's own wake chirp: loud, clipped, and not speech. Leaving it out of the
|
|
123
|
+
// analysis is what makes `sawSpeech` mean anything, and keeps the noise floor honest.
|
|
124
|
+
if (this.elapsedMs <= this.skipMs) {
|
|
125
|
+
continue;
|
|
126
|
+
}
|
|
127
|
+
this.totalMs += this.frameMs;
|
|
128
|
+
if (this.adaptive) {
|
|
129
|
+
// Three rules, each there to defeat a specific way of getting this wrong:
|
|
130
|
+
// - start at zero, i.e. maximally sensitive. Seeding from the first frame instead
|
|
131
|
+
// lets someone who talks straight after the wake word install their own voice as
|
|
132
|
+
// the room's noise floor, which shuts the gate for good — the same trap an average
|
|
133
|
+
// or median over a calibration window falls into.
|
|
134
|
+
// - fall instantly, so a room that went quiet is believed at once.
|
|
135
|
+
// - rise slowly, and not at all once speech has been recognised: otherwise a long
|
|
136
|
+
// utterance drags the floor, and with it `endLevel`, up to its own level and cuts
|
|
137
|
+
// itself off mid-sentence. Falling stays on, so the pauses between words still count.
|
|
138
|
+
this.floor =
|
|
139
|
+
this.floor === null
|
|
140
|
+
? 0
|
|
141
|
+
: level < this.floor
|
|
142
|
+
? level
|
|
143
|
+
: this.speechSeen
|
|
144
|
+
? this.floor
|
|
145
|
+
: this.floor + (level - this.floor) * NOISE_RISE;
|
|
146
|
+
if (this.totalMs < this.warmupMs) {
|
|
147
|
+
continue; // let the floor settle before anything may count as speech
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
if (!this.speechSeen) {
|
|
151
|
+
if (level >= this.startLevel) {
|
|
152
|
+
this.speechSeen = true;
|
|
153
|
+
this.silentMs = 0;
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
else if (level < this.endLevel) {
|
|
157
|
+
this.silentMs += this.frameMs;
|
|
158
|
+
if (this.silentMs >= this.silenceMs) {
|
|
159
|
+
verdict ||= 'speech-end';
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
else {
|
|
163
|
+
this.silentMs = 0;
|
|
164
|
+
}
|
|
165
|
+
if (this.totalMs >= this.maxMs) {
|
|
166
|
+
verdict ||= 'timeout';
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
return verdict;
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
exports.SpeechDetector = SpeechDetector;
|
|
173
|
+
//# sourceMappingURL=vad.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"vad.js","sourceRoot":"","sources":["../../../src/lib/voice/vad.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;;;AASH,kBAWC;AAZD,kFAAkF;AAClF,SAAgB,GAAG,CAAC,GAAU;IAC1B,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC3C,IAAI,CAAC,OAAO,EAAE,CAAC;QACX,OAAO,CAAC,CAAC;IACb,CAAC;IACD,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,GAAG,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACrC,GAAG,IAAI,KAAK,GAAG,KAAK,CAAC;IACzB,CAAC;IACD,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,CAAC;AACpC,CAAC;AAsCD;;;;GAIG;AACH,MAAM,UAAU,GAAG,IAAI,CAAC;AAExB,MAAa,cAAc;IACN,UAAU,CAAS;IACnB,OAAO,CAAS;IAChB,UAAU,CAAS;IACnB,QAAQ,CAAS;IACjB,SAAS,CAAS;IAClB,KAAK,CAAS;IACd,MAAM,CAAS;IACf,QAAQ,CAAU;IAClB,aAAa,CAAS;IACtB,WAAW,CAAS;IACpB,WAAW,CAAS;IACpB,SAAS,CAAS;IAClB,QAAQ,CAAS;IAE1B,OAAO,GAAU,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACjC,UAAU,GAAG,KAAK,CAAC;IACnB,QAAQ,GAAG,CAAC,CAAC;IACrB,gFAAgF;IACxE,KAAK,GAAkB,IAAI,CAAC;IACpC,0EAA0E;IAClE,SAAS,GAAG,CAAC,CAAC;IACtB,oEAAoE;IACpE,OAAO,GAAG,CAAC,CAAC;IACZ,qGAAqG;IACrG,IAAI,GAAG,CAAC,CAAC;IAET,YAAY,EACR,UAAU,GAAG,KAAK,EAClB,OAAO,GAAG,EAAE,EACZ,UAAU,GAAG,GAAG,EAChB,QAAQ,GAAG,GAAG,EACd,SAAS,GAAG,GAAG,EACf,KAAK,GAAG,KAAK,EACb,MAAM,GAAG,CAAC,EACV,QAAQ,GAAG,KAAK,EAChB,aAAa,GAAG,GAAG,EACnB,WAAW,GAAG,GAAG,EACjB,WAAW,GAAG,GAAG,EACjB,SAAS,GAAG,CAAC,EACb,QAAQ,GAAG,GAAG,MACS,EAAE;QACzB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,UAAU,GAAG,OAAO,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;QAChE,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC7B,CAAC;IAED,kGAAkG;IAClG,IAAI,SAAS;QACT,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;IAED,0FAA0F;IAC1F,IAAI,UAAU;QACV,OAAO,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;IAC3B,CAAC;IAED,0FAA0F;IAC1F,IAAI,UAAU;QACV,OAAO,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC;IAChH,CAAC;IAED,mGAAmG;IACnG,IAAI,QAAQ;QACR,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACjB,OAAO,IAAI,CAAC,QAAQ,CAAC;QACzB,CAAC;QACD,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;IACrG,CAAC;IAED,mGAAmG;IACnG,IAAI,CAAC,KAAY;QACb,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QAClF,IAAI,OAAO,GAAqB,IAAI,CAAC;QAErC,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YAC5C,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;YACxD,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACtD,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC;YAE/B,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC;YACzB,IAAI,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;gBACpB,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC;YACtB,CAAC;YAED,oFAAoF;YACpF,sFAAsF;YACtF,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBAChC,SAAS;YACb,CAAC;YACD,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,CAAC;YAE7B,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAChB,0EAA0E;gBAC1E,mFAAmF;gBACnF,oFAAoF;gBACpF,sFAAsF;gBACtF,qDAAqD;gBACrD,oEAAoE;gBACpE,mFAAmF;gBACnF,qFAAqF;gBACrF,yFAAyF;gBACzF,IAAI,CAAC,KAAK;oBACN,IAAI,CAAC,KAAK,KAAK,IAAI;wBACf,CAAC,CAAC,CAAC;wBACH,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK;4BAClB,CAAC,CAAC,KAAK;4BACP,CAAC,CAAC,IAAI,CAAC,UAAU;gCACf,CAAC,CAAC,IAAI,CAAC,KAAK;gCACZ,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,UAAU,CAAC;gBAC7D,IAAI,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;oBAC/B,SAAS,CAAC,2DAA2D;gBACzE,CAAC;YACL,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;gBACnB,IAAI,KAAK,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;oBAC3B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;oBACvB,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;gBACtB,CAAC;YACL,CAAC;iBAAM,IAAI,KAAK,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAC/B,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,OAAO,CAAC;gBAC9B,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;oBAClC,OAAO,KAAK,YAAY,CAAC;gBAC7B,CAAC;YACL,CAAC;iBAAM,CAAC;gBACJ,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtB,CAAC;YAED,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;gBAC7B,OAAO,KAAK,SAAS,CAAC;YAC1B,CAAC;QACL,CAAC;QACD,OAAO,OAAO,CAAC;IACnB,CAAC;CACJ;AAjJD,wCAiJC","sourcesContent":["/**\n * End-of-speech detection for satellites that stream microphone audio until they are told to stop.\n *\n * The Hannah/UDP protocol has the satellite decide when an utterance ended (`audio_end`), and Wyoming\n * clients send `audio-stop` themselves. ESPHome voice satellites do neither: they keep streaming until\n * the server sends `STT_VAD_END`/`STT_END`, so the decision has to happen here.\n *\n * This is a plain energy gate, not a neural VAD — good enough to close an utterance, and it costs\n * nothing per frame. Speech must first rise above `startLevel`; after that, `silenceMs` worth of frames\n * below `endLevel` ends it. `maxMs` is the hard stop for rooms that are simply loud.\n *\n * Fixed thresholds only work when the microphone level is known, and on a real satellite it is not.\n * Measured on a ThirdReality Voice & Music Assistant: the device plays a wake-acknowledgement chirp\n * into its own microphone that peaks at full scale (RMS ~16000, clipped), while the speech that follows\n * arrives at RMS 358–469 — below the 700 default for `startLevel` and right on the 400 for `endLevel`.\n * That breaks the gate twice over: `sawSpeech` is satisfied by the chirp rather than by any voice, so\n * the \"nothing but speech-free audio\" guard never fires, and half the real speech frames count as\n * silence, so the utterance can be cut off mid-sentence.\n *\n * Hence two opt-in knobs, both enabled by the ESPHome transport:\n *\n * - `skipMs` drops the leading chirp from the *analysis* (never from the audio, which still goes to STT\n * in full — a fast speaker may already be talking).\n * - `adaptive` derives the thresholds from a tracked noise floor instead of fixed levels. The floor\n * follows the quietest frame seen (instant down, very slow up), so it settles on room noise even when\n * someone starts talking immediately — which a mean or median over a calibration window would not.\n */\n\n/**\n * A buffer that may be a view onto any `ArrayBufferLike` — what `Buffer.subarray()` and\n * `Buffer.concat()` hand back. Plain `Buffer` (= `Buffer<ArrayBuffer>`) assigns to it, not the reverse.\n */\nexport type Bytes = Buffer<ArrayBufferLike>;\n\n/** Root-mean-square amplitude (0…32767) of a mono 16-bit signed LE PCM buffer. */\nexport function rms(pcm: Bytes): number {\n const samples = Math.floor(pcm.length / 2);\n if (!samples) {\n return 0;\n }\n let sum = 0;\n for (let i = 0; i < samples; i++) {\n const value = pcm.readInt16LE(i * 2);\n sum += value * value;\n }\n return Math.sqrt(sum / samples);\n}\n\nexport interface SpeechDetectorOptions {\n /** Sample rate of the incoming PCM (ESPHome satellites always capture at 16 kHz). */\n sampleRate?: number;\n /** Analysis window; 30 ms is the usual compromise between latency and stability. */\n frameMs?: number;\n /** RMS above which we consider speech to have started. Ignored when `adaptive` is on. */\n startLevel?: number;\n /** RMS below which a frame counts as silence. Ignored when `adaptive` is on. */\n endLevel?: number;\n /** Silence after speech that ends the utterance. */\n silenceMs?: number;\n /** Hard cap on one utterance, in case the room never falls quiet. */\n maxMs?: number;\n /**\n * Leading audio to leave out of the analysis — for devices that chirp into their own microphone\n * when the wake word fires. The audio itself is untouched; this only stops the chirp from being\n * mistaken for speech and from poisoning the noise floor.\n */\n skipMs?: number;\n /** Derive the thresholds from the tracked noise floor instead of using fixed levels. */\n adaptive?: boolean;\n /** Adaptive mode: `startLevel` is at least this, however quiet the room is. */\n minStartLevel?: number;\n /** Adaptive mode: `endLevel` is at least this. */\n minEndLevel?: number;\n /** Adaptive mode: speech threshold as a multiple of the noise floor. */\n startFactor?: number;\n /** Adaptive mode: silence threshold as a multiple of the noise floor. */\n endFactor?: number;\n /** Adaptive mode: settle the noise floor this long before speech may be declared. */\n warmupMs?: number;\n}\n\n/** Why an utterance ended: the speaker stopped, or we hit the hard cap. */\nexport type SpeechEnd = 'speech-end' | 'timeout';\n\n/**\n * How fast the tracked noise floor may follow a *rising* ambient level, per analysed frame (30 ms).\n * Fast enough that a genuinely noisy room is matched within about a second, slow enough that a voice\n * cannot pull the floor up to its own level before it has been recognised as speech.\n */\nconst NOISE_RISE = 0.02;\n\nexport class SpeechDetector {\n private readonly frameBytes: number;\n private readonly frameMs: number;\n private readonly fixedStart: number;\n private readonly fixedEnd: number;\n private readonly silenceMs: number;\n private readonly maxMs: number;\n private readonly skipMs: number;\n private readonly adaptive: boolean;\n private readonly minStartLevel: number;\n private readonly minEndLevel: number;\n private readonly startFactor: number;\n private readonly endFactor: number;\n private readonly warmupMs: number;\n\n private pending: Bytes = Buffer.alloc(0);\n private speechSeen = false;\n private silentMs = 0;\n /** Tracked room level in adaptive mode; null until the first analysed frame. */\n private floor: number | null = null;\n /** Audio seen so far *including* the skipped lead-in, in milliseconds. */\n private elapsedMs = 0;\n /** Audio analysed so far, in milliseconds (exposed for logging). */\n totalMs = 0;\n /** Loudest frame seen so far (exposed for logging — the usual clue when a mic is mis-configured). */\n peak = 0;\n\n constructor({\n sampleRate = 16000,\n frameMs = 30,\n startLevel = 700,\n endLevel = 400,\n silenceMs = 900,\n maxMs = 12000,\n skipMs = 0,\n adaptive = false,\n minStartLevel = 250,\n minEndLevel = 150,\n startFactor = 3.5,\n endFactor = 2,\n warmupMs = 150,\n }: SpeechDetectorOptions = {}) {\n this.frameBytes = Math.round((sampleRate * frameMs) / 1000) * 2;\n this.frameMs = frameMs;\n this.fixedStart = startLevel;\n this.fixedEnd = endLevel;\n this.silenceMs = silenceMs;\n this.maxMs = maxMs;\n this.skipMs = skipMs;\n this.adaptive = adaptive;\n this.minStartLevel = minStartLevel;\n this.minEndLevel = minEndLevel;\n this.startFactor = startFactor;\n this.endFactor = endFactor;\n this.warmupMs = warmupMs;\n }\n\n /** True once any frame crossed {@link startLevel} — an utterance that never did is room noise. */\n get sawSpeech(): boolean {\n return this.speechSeen;\n }\n\n /** Tracked room level (adaptive mode), for logging. 0 before the first analysed frame. */\n get noiseFloor(): number {\n return this.floor ?? 0;\n }\n\n /** Current speech threshold — fixed, or derived from the noise floor in adaptive mode. */\n get startLevel(): number {\n return this.adaptive ? Math.max(this.minStartLevel, (this.floor ?? 0) * this.startFactor) : this.fixedStart;\n }\n\n /** Current silence threshold. Never above {@link startLevel}, so the gate keeps its hysteresis. */\n get endLevel(): number {\n if (!this.adaptive) {\n return this.fixedEnd;\n }\n return Math.min(this.startLevel, Math.max(this.minEndLevel, (this.floor ?? 0) * this.endFactor));\n }\n\n /** Feed one audio chunk (any size). Returns why the utterance ended, or null to keep listening. */\n push(chunk: Bytes): SpeechEnd | null {\n this.pending = this.pending.length ? Buffer.concat([this.pending, chunk]) : chunk;\n let verdict: SpeechEnd | null = null;\n\n while (this.pending.length >= this.frameBytes) {\n const frame = this.pending.subarray(0, this.frameBytes);\n this.pending = this.pending.subarray(this.frameBytes);\n this.elapsedMs += this.frameMs;\n\n const level = rms(frame);\n if (level > this.peak) {\n this.peak = level;\n }\n\n // The device's own wake chirp: loud, clipped, and not speech. Leaving it out of the\n // analysis is what makes `sawSpeech` mean anything, and keeps the noise floor honest.\n if (this.elapsedMs <= this.skipMs) {\n continue;\n }\n this.totalMs += this.frameMs;\n\n if (this.adaptive) {\n // Three rules, each there to defeat a specific way of getting this wrong:\n // - start at zero, i.e. maximally sensitive. Seeding from the first frame instead\n // lets someone who talks straight after the wake word install their own voice as\n // the room's noise floor, which shuts the gate for good — the same trap an average\n // or median over a calibration window falls into.\n // - fall instantly, so a room that went quiet is believed at once.\n // - rise slowly, and not at all once speech has been recognised: otherwise a long\n // utterance drags the floor, and with it `endLevel`, up to its own level and cuts\n // itself off mid-sentence. Falling stays on, so the pauses between words still count.\n this.floor =\n this.floor === null\n ? 0\n : level < this.floor\n ? level\n : this.speechSeen\n ? this.floor\n : this.floor + (level - this.floor) * NOISE_RISE;\n if (this.totalMs < this.warmupMs) {\n continue; // let the floor settle before anything may count as speech\n }\n }\n\n if (!this.speechSeen) {\n if (level >= this.startLevel) {\n this.speechSeen = true;\n this.silentMs = 0;\n }\n } else if (level < this.endLevel) {\n this.silentMs += this.frameMs;\n if (this.silentMs >= this.silenceMs) {\n verdict ||= 'speech-end';\n }\n } else {\n this.silentMs = 0;\n }\n\n if (this.totalMs >= this.maxMs) {\n verdict ||= 'timeout';\n }\n }\n return verdict;\n }\n}\n"]}
|
|
@@ -216,6 +216,19 @@ class VoiceServer {
|
|
|
216
216
|
}
|
|
217
217
|
}));
|
|
218
218
|
}
|
|
219
|
+
/**
|
|
220
|
+
* Re-open the microphone on one satellite (by device name) or all of them, so the user can answer a
|
|
221
|
+
* question the assistant just asked without saying the wake word again. Returns how many satellites
|
|
222
|
+
* were told to listen — whether they can is up to their firmware (the message is additive).
|
|
223
|
+
*/
|
|
224
|
+
listen(device) {
|
|
225
|
+
const targets = [...this.sats.values()].filter(s => !device || s.device === device);
|
|
226
|
+
for (const sat of targets) {
|
|
227
|
+
void this.send((0, protocol_1.encodeControl)({ type: 'listen' }), sat);
|
|
228
|
+
this.setStatus(sat, 'listening');
|
|
229
|
+
}
|
|
230
|
+
return targets.length;
|
|
231
|
+
}
|
|
219
232
|
/** Device names of the currently registered satellites. */
|
|
220
233
|
devices() {
|
|
221
234
|
return [...this.sats.values()].map(s => s.device);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"voiceServer.js","sourceRoot":"","sources":["../../../src/lib/voice/voiceServer.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;;;;;;GAMG;AACH,kDAAoC;AACpC,yCASoB;AA+BpB,+FAA+F;AAC/F,MAAM,eAAe,GAAG,IAAI,CAAC;AAC7B,+FAA+F;AAC/F,MAAM,QAAQ,GAAG,MAAM,CAAC;AAExB,MAAa,WAAW;IAMS;IALrB,IAAI,GAAwB,IAAI,CAAC;IACzC,6FAA6F;IAC5E,IAAI,GAAG,IAAI,GAAG,EAAqB,CAAC;IAC7C,UAAU,GAA0B,IAAI,CAAC;IAEjD,YAA6B,IAAwB;QAAxB,SAAI,GAAJ,IAAI,CAAoB;IAAG,CAAC;IAEzD,KAAK;QACD,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACnC,MAAM,IAAI,GAAG,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;YACxC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;YACjB,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;YACjE,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YAC5E,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YAC3B,sFAAsF;YACtF,MAAM,IAAI,GACN,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,IAAI,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC;YACrG,MAAM,OAAO,GAAG,GAAS,EAAE;gBACvB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,iCAAiC,IAAI,IAAI,SAAS,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC3F,IAAI,CAAC,UAAU,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,MAAM,CAAC,CAAC;gBAC/D,OAAO,EAAE,CAAC;YACd,CAAC,CAAC;YACF,IAAI,IAAI,EAAE,CAAC;gBACP,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;YAC7C,CAAC;iBAAM,CAAC;gBACJ,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YACvC,CAAC;QACL,CAAC,CAAC,CAAC;IACP,CAAC;IAED,KAAK,CAAC,IAAI;QACN,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YAClB,aAAa,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC/B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QAC3B,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;QACvB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;QAClB,IAAI,IAAI,EAAE,CAAC;YACP,MAAM,IAAI,OAAO,CAAO,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QAC5D,CAAC;IACL,CAAC;IAEO,MAAM,CAAC,OAAO,CAAC,OAAe,EAAE,IAAY;QAChD,OAAO,GAAG,OAAO,IAAI,IAAI,EAAE,CAAC;IAChC,CAAC;IAEO,SAAS,CAAC,IAAY,EAAE,KAAuB;QACnD,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACf,OAAO;QACX,CAAC;QACD,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,IAAA,uBAAY,EAAC,IAAI,CAAC,CAAC;QAC7C,IAAI,IAAI,KAAK,qBAAU,EAAE,CAAC;YACtB,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;YAC1E,IAAI,GAAG,EAAE,CAAC;gBACN,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBAC1B,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,oDAAoD;YAC/F,CAAC;YACD,OAAO;QACX,CAAC;QACD,IAAI,IAAI,KAAK,uBAAY,EAAE,CAAC;YACxB,IAAI,GAAgB,CAAC;YACrB,IAAI,CAAC;gBACD,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAgB,CAAC;YAC9D,CAAC;YAAC,MAAM,CAAC;gBACL,OAAO;YACX,CAAC;YACD,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,wBAAyB,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QAC9G,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,SAAS,CAAC,GAAgB,EAAE,KAAuB;QAC7D,MAAM,GAAG,GAAG,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QAE3D,IAAI,GAAG,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YAC1B,MAAM,GAAG,GAAc;gBACnB,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,GAAG;gBACzB,IAAI,EAAE,GAAG,CAAC,IAAI,IAAI,EAAE;gBACpB,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE;gBACpB,MAAM,EAAE,EAAE;aACb,CAAC;YACF,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;YACxB,KAAK,IAAI,CAAC,IAAI,CAAC,IAAA,wBAAa,EAAC,EAAE,IAAI,EAAE,YAAY,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,CAAC;YACrE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,yBAAyB,GAAG,CAAC,MAAM,WAAW,GAAG,CAAC,IAAI,IAAI,GAAG,OAAO,GAAG,EAAE,CAAC,CAAC;YAC9F,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YAC5B,OAAO;QACX,CAAC;QAED,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC/B,IAAI,CAAC,GAAG,EAAE,CAAC;YACP,6EAA6E;YAC7E,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAA,wBAAa,EAAC,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;YAClF,OAAO;QACX,CAAC;QACD,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE1B,IAAI,GAAG,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YAC3B,KAAK,IAAI,CAAC,IAAI,CAAC,IAAA,wBAAa,EAAC,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC,EAAE,GAAG,CAAC,CAAC;YAC9D,OAAO;QACX,CAAC;QACD,IAAI,GAAG,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YAC3B,MAAM,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;QACpC,CAAC;IACL,CAAC;IAED,6FAA6F;IACrF,KAAK,CAAC,eAAe,CAAC,GAAc;QACxC,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACtC,GAAG,CAAC,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC;YACd,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YAC5B,OAAO;QACX,CAAC;QAED,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;QAClC,IAAI,CAAC;YACD,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;YAC1E,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,EAAE,4BAAiB,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;YAC/F,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,GAAG,CAAC,MAAM,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC,CAAC;YACpE,IAAI,CAAC,IAAI,EAAE,CAAC;gBACR,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;gBAC5B,OAAO;YACX,CAAC;YAED,MAAM,MAAM,GAAG,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC7F,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,GAAG,CAAC,MAAM,MAAM,MAAM,EAAE,CAAC,CAAC;YACzD,IAAI,CAAC,MAAM,EAAE,CAAC;gBACV,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;gBAC5B,OAAO;YACX,CAAC;YAED,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;YAChC,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC/F,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;QAClD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,0BAA0B,GAAG,CAAC,MAAM,MAAO,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;QAC1F,CAAC;gBAAS,CAAC;YACP,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QAChC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,SAAS,CAAC,GAAc,EAAE,GAAW,EAAE,UAAkB;QACnE,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,GAAG,GAAG,GAAG,CAAC,MAAM,EAAE,GAAG,IAAI,eAAe,EAAE,CAAC,EAAE,EAAE,CAAC;YACrE,MAAM,IAAI,CAAC,IAAI,CAAC,IAAA,oBAAS,EAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,eAAe,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YAC1E,kGAAkG;YAClG,IAAI,CAAC,CAAC,GAAG,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC;gBAClB,MAAM,IAAI,OAAO,CAAO,GAAG,CAAC,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC;YACtD,CAAC;QACL,CAAC;QACD,MAAM,IAAI,CAAC,IAAI,CAAC,IAAA,wBAAa,EAAC,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,UAAU,EAAE,CAAC,EAAE,GAAG,CAAC,CAAC;IACtF,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ,CAAC,MAAqB,EAAE,GAAW,EAAE,UAAkB;QACjE,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC;YACd,OAAO;QACX,CAAC;QACD,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC;QACpF,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;YAClB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,gBAAgB,MAAM,CAAC,CAAC,CAAC,cAAc,MAAM,GAAG,CAAC,CAAC,CAAC,YAAY,aAAa,CAAC,CAAC;YACjG,OAAO;QACX,CAAC;QACD,MAAM,OAAO,CAAC,GAAG,CACb,OAAO,CAAC,GAAG,CAAC,KAAK,EAAC,GAAG,EAAC,EAAE;YACpB,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;YAChC,IAAI,CAAC;gBACD,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,GAAG,EAAE,UAAU,CAAC,CAAC;YAC/C,CAAC;oBAAS,CAAC;gBACP,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YAChC,CAAC;QACL,CAAC,CAAC,CACL,CAAC;IACN,CAAC;IAED,2DAA2D;IAC3D,OAAO;QACH,OAAO,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IACtD,CAAC;IAEO,SAAS,CAAC,GAAc,EAAE,KAAqB;QACnD,KAAK,IAAI,CAAC,IAAI,CAAC,IAAA,wBAAa,EAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,CAAC,CAAC;QAC9D,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACtD,CAAC;IAEO,IAAI,CAAC,IAAY,EAAE,GAAc;QACrC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACnC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;YACvB,IAAI,CAAC,IAAI,EAAE,CAAC;gBACR,OAAO,EAAE,CAAC;gBACV,OAAO;YACX,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QACnF,CAAC,CAAC,CAAC;IACP,CAAC;IAEO,UAAU;QACd,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACjC,IAAI,GAAG,GAAG,GAAG,CAAC,QAAQ,GAAG,QAAQ,EAAE,CAAC;gBAChC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBACtB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,gCAAgC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;gBACjE,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;YAC1D,CAAC;QACL,CAAC;IACL,CAAC;CACJ;AAnND,kCAmNC","sourcesContent":["/**\n * UDP voice server — the \"brain\" side of the satellite protocol.\n *\n * Receives streamed microphone audio from satellites, runs STT → answer callback → TTS,\n * and streams the spoken reply back. Registration + heartbeat keep the satellite registry\n * live. Protocol is Hannah-compatible (see `protocol.ts`).\n */\nimport * as dgram from 'node:dgram';\nimport {\n AUDIO_SAMPLE_RATE,\n TYPE_AUDIO,\n TYPE_CONTROL,\n decodePacket,\n encodeControl,\n encodeTts,\n type SatToServer,\n type SatelliteState,\n} from './protocol';\nimport type { SttEngine } from './stt';\nimport type { TtsEngine } from './tts';\n\ninterface Satellite {\n device: string;\n room: string;\n address: string;\n port: number;\n lastSeen: number;\n /** Buffered audio chunks of the utterance currently being spoken. */\n chunks: Buffer[];\n}\n\nexport interface VoiceServerOptions {\n port: number;\n /** Interface to bind to ('0.0.0.0'/'' = all interfaces, or a specific IP). */\n bindAddress?: string;\n /** ISO-639-1 language hint for STT/TTS ('' = auto/provider default). */\n language: string;\n stt: SttEngine;\n tts: TtsEngine;\n /** Produce the assistant reply for a transcribed utterance. */\n answer: (question: string, ctx: { device: string; room: string }) => Promise<string>;\n /** Optional STT vocabulary hints (device/room names) to bias recognition; re-read per utterance. */\n getHints?: () => string[] | Promise<string[]>;\n log: ioBroker.Logger;\n /** Notified on every satellite state transition (idle/listening/processing/speaking/offline). */\n onStatus?: (device: string, room: string, state: SatelliteState | 'offline') => void;\n}\n\n/** Payload bytes per TTS datagram — well under the typical MTU-fragmentation-free UDP size. */\nconst TTS_CHUNK_BYTES = 8192;\n/** Drop a satellite after this long without any packet (3× the satellite's 10 s heartbeat). */\nconst STALE_MS = 30_000;\n\nexport class VoiceServer {\n private sock: dgram.Socket | null = null;\n /** Keyed by `${address}:${port}` — audio packets carry no device name, only their source. */\n private readonly sats = new Map<string, Satellite>();\n private sweepTimer: NodeJS.Timeout | null = null;\n\n constructor(private readonly opts: VoiceServerOptions) {}\n\n start(): Promise<void> {\n return new Promise((resolve, reject) => {\n const sock = dgram.createSocket('udp4');\n this.sock = sock;\n sock.on('message', (data, rinfo) => this.onMessage(data, rinfo));\n sock.on('error', e => this.opts.log.error(`voice UDP error: ${e.message}`));\n sock.once('error', reject);\n // '0.0.0.0'/'' → bind all interfaces (omit the address); otherwise bind the given IP.\n const addr =\n this.opts.bindAddress && this.opts.bindAddress !== '0.0.0.0' ? this.opts.bindAddress : undefined;\n const onBound = (): void => {\n this.opts.log.info(`Voice server listening on UDP ${addr || '0.0.0.0'}:${this.opts.port}`);\n this.sweepTimer = setInterval(() => this.sweepStale(), 15_000);\n resolve();\n };\n if (addr) {\n sock.bind(this.opts.port, addr, onBound);\n } else {\n sock.bind(this.opts.port, onBound);\n }\n });\n }\n\n async stop(): Promise<void> {\n if (this.sweepTimer) {\n clearInterval(this.sweepTimer);\n this.sweepTimer = null;\n }\n const sock = this.sock;\n this.sock = null;\n this.sats.clear();\n if (sock) {\n await new Promise<void>(res => sock.close(() => res()));\n }\n }\n\n private static addrKey(address: string, port: number): string {\n return `${address}:${port}`;\n }\n\n private onMessage(data: Buffer, rinfo: dgram.RemoteInfo): void {\n if (!data.length) {\n return;\n }\n const { type, payload } = decodePacket(data);\n if (type === TYPE_AUDIO) {\n const sat = this.sats.get(VoiceServer.addrKey(rinfo.address, rinfo.port));\n if (sat) {\n sat.lastSeen = Date.now();\n sat.chunks.push(Buffer.from(payload)); // copy: payload is a view onto a reused recv buffer\n }\n return;\n }\n if (type === TYPE_CONTROL) {\n let msg: SatToServer;\n try {\n msg = JSON.parse(payload.toString('utf8')) as SatToServer;\n } catch {\n return;\n }\n this.onControl(msg, rinfo).catch(e => this.opts.log.warn(`voice control error: ${(e as Error).message}`));\n }\n }\n\n private async onControl(msg: SatToServer, rinfo: dgram.RemoteInfo): Promise<void> {\n const key = VoiceServer.addrKey(rinfo.address, rinfo.port);\n\n if (msg.type === 'register') {\n const sat: Satellite = {\n device: msg.device || key,\n room: msg.room || '',\n address: rinfo.address,\n port: rinfo.port,\n lastSeen: Date.now(),\n chunks: [],\n };\n this.sats.set(key, sat);\n void this.send(encodeControl({ type: 'registered', ok: true }), sat);\n this.opts.log.info(`Satellite registered: ${sat.device} (room: ${sat.room || '-'}) @ ${key}`);\n this.setStatus(sat, 'idle');\n return;\n }\n\n const sat = this.sats.get(key);\n if (!sat) {\n // Unknown source (server restarted?) — tell the satellite to register again.\n this.sock?.send(encodeControl({ type: 'reregister' }), rinfo.port, rinfo.address);\n return;\n }\n sat.lastSeen = Date.now();\n\n if (msg.type === 'heartbeat') {\n void this.send(encodeControl({ type: 'heartbeat_ack' }), sat);\n return;\n }\n if (msg.type === 'audio_end') {\n await this.handleUtterance(sat);\n }\n }\n\n /** STT → answer → TTS for one completed utterance, with status transitions along the way. */\n private async handleUtterance(sat: Satellite): Promise<void> {\n const pcm = Buffer.concat(sat.chunks);\n sat.chunks = [];\n if (!pcm.length) {\n this.setStatus(sat, 'idle');\n return;\n }\n\n this.setStatus(sat, 'processing');\n try {\n const hints = this.opts.getHints ? await this.opts.getHints() : undefined;\n const text = await this.opts.stt.transcribe(pcm, AUDIO_SAMPLE_RATE, this.opts.language, hints);\n this.opts.log.info(`Voice Q (${sat.device}): ${text || '(empty)'}`);\n if (!text) {\n this.setStatus(sat, 'idle');\n return;\n }\n\n const answer = (await this.opts.answer(text, { device: sat.device, room: sat.room })).trim();\n this.opts.log.info(`Voice A (${sat.device}): ${answer}`);\n if (!answer) {\n this.setStatus(sat, 'idle');\n return;\n }\n\n this.setStatus(sat, 'speaking');\n const { pcm: ttsPcm, sampleRate } = await this.opts.tts.synthesize(answer, this.opts.language);\n await this.streamTts(sat, ttsPcm, sampleRate);\n } catch (e) {\n this.opts.log.error(`Voice pipeline failed (${sat.device}): ${(e as Error).message}`);\n } finally {\n this.setStatus(sat, 'idle');\n }\n }\n\n private async streamTts(sat: Satellite, pcm: Buffer, sampleRate: number): Promise<void> {\n for (let off = 0, i = 0; off < pcm.length; off += TTS_CHUNK_BYTES, i++) {\n await this.send(encodeTts(pcm.subarray(off, off + TTS_CHUNK_BYTES)), sat);\n // Yield every 16 packets so a long reply doesn't burst the OS send buffer / the satellite's recv.\n if ((i & 15) === 15) {\n await new Promise<void>(res => setImmediate(res));\n }\n }\n await this.send(encodeControl({ type: 'tts_end', sample_rate: sampleRate }), sat);\n }\n\n /**\n * Play arbitrary PCM on one satellite (by device name) or all registered satellites (device=null).\n * Used for announcements (`tts.text` / per-satellite `tts`), independent of the wake-word flow.\n */\n async announce(device: string | null, pcm: Buffer, sampleRate: number): Promise<void> {\n if (!pcm.length) {\n return;\n }\n const targets = [...this.sats.values()].filter(s => !device || s.device === device);\n if (!targets.length) {\n this.opts.log.warn(`announce: no ${device ? `satellite '${device}'` : 'satellites'} registered`);\n return;\n }\n await Promise.all(\n targets.map(async sat => {\n this.setStatus(sat, 'speaking');\n try {\n await this.streamTts(sat, pcm, sampleRate);\n } finally {\n this.setStatus(sat, 'idle');\n }\n }),\n );\n }\n\n /** Device names of the currently registered satellites. */\n devices(): string[] {\n return [...this.sats.values()].map(s => s.device);\n }\n\n private setStatus(sat: Satellite, state: SatelliteState): void {\n void this.send(encodeControl({ type: 'status', state }), sat);\n this.opts.onStatus?.(sat.device, sat.room, state);\n }\n\n private send(data: Buffer, sat: Satellite): Promise<void> {\n return new Promise((resolve, reject) => {\n const sock = this.sock;\n if (!sock) {\n resolve();\n return;\n }\n sock.send(data, sat.port, sat.address, err => (err ? reject(err) : resolve()));\n });\n }\n\n private sweepStale(): void {\n const now = Date.now();\n for (const [key, sat] of this.sats) {\n if (now - sat.lastSeen > STALE_MS) {\n this.sats.delete(key);\n this.opts.log.info(`Satellite offline (timeout): ${sat.device}`);\n this.opts.onStatus?.(sat.device, sat.room, 'offline');\n }\n }\n }\n}\n"]}
|
|
1
|
+
{"version":3,"file":"voiceServer.js","sourceRoot":"","sources":["../../../src/lib/voice/voiceServer.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;;;;;;GAMG;AACH,kDAAoC;AACpC,yCASoB;AA+BpB,+FAA+F;AAC/F,MAAM,eAAe,GAAG,IAAI,CAAC;AAC7B,+FAA+F;AAC/F,MAAM,QAAQ,GAAG,MAAM,CAAC;AAExB,MAAa,WAAW;IAMS;IALrB,IAAI,GAAwB,IAAI,CAAC;IACzC,6FAA6F;IAC5E,IAAI,GAAG,IAAI,GAAG,EAAqB,CAAC;IAC7C,UAAU,GAA0B,IAAI,CAAC;IAEjD,YAA6B,IAAwB;QAAxB,SAAI,GAAJ,IAAI,CAAoB;IAAG,CAAC;IAEzD,KAAK;QACD,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACnC,MAAM,IAAI,GAAG,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;YACxC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;YACjB,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;YACjE,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YAC5E,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YAC3B,sFAAsF;YACtF,MAAM,IAAI,GACN,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,IAAI,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC;YACrG,MAAM,OAAO,GAAG,GAAS,EAAE;gBACvB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,iCAAiC,IAAI,IAAI,SAAS,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC3F,IAAI,CAAC,UAAU,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,MAAM,CAAC,CAAC;gBAC/D,OAAO,EAAE,CAAC;YACd,CAAC,CAAC;YACF,IAAI,IAAI,EAAE,CAAC;gBACP,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;YAC7C,CAAC;iBAAM,CAAC;gBACJ,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YACvC,CAAC;QACL,CAAC,CAAC,CAAC;IACP,CAAC;IAED,KAAK,CAAC,IAAI;QACN,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YAClB,aAAa,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC/B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QAC3B,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;QACvB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;QAClB,IAAI,IAAI,EAAE,CAAC;YACP,MAAM,IAAI,OAAO,CAAO,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QAC5D,CAAC;IACL,CAAC;IAEO,MAAM,CAAC,OAAO,CAAC,OAAe,EAAE,IAAY;QAChD,OAAO,GAAG,OAAO,IAAI,IAAI,EAAE,CAAC;IAChC,CAAC;IAEO,SAAS,CAAC,IAAY,EAAE,KAAuB;QACnD,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACf,OAAO;QACX,CAAC;QACD,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,IAAA,uBAAY,EAAC,IAAI,CAAC,CAAC;QAC7C,IAAI,IAAI,KAAK,qBAAU,EAAE,CAAC;YACtB,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;YAC1E,IAAI,GAAG,EAAE,CAAC;gBACN,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBAC1B,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,oDAAoD;YAC/F,CAAC;YACD,OAAO;QACX,CAAC;QACD,IAAI,IAAI,KAAK,uBAAY,EAAE,CAAC;YACxB,IAAI,GAAgB,CAAC;YACrB,IAAI,CAAC;gBACD,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAgB,CAAC;YAC9D,CAAC;YAAC,MAAM,CAAC;gBACL,OAAO;YACX,CAAC;YACD,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,wBAAyB,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QAC9G,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,SAAS,CAAC,GAAgB,EAAE,KAAuB;QAC7D,MAAM,GAAG,GAAG,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QAE3D,IAAI,GAAG,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YAC1B,MAAM,GAAG,GAAc;gBACnB,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,GAAG;gBACzB,IAAI,EAAE,GAAG,CAAC,IAAI,IAAI,EAAE;gBACpB,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE;gBACpB,MAAM,EAAE,EAAE;aACb,CAAC;YACF,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;YACxB,KAAK,IAAI,CAAC,IAAI,CAAC,IAAA,wBAAa,EAAC,EAAE,IAAI,EAAE,YAAY,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,CAAC;YACrE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,yBAAyB,GAAG,CAAC,MAAM,WAAW,GAAG,CAAC,IAAI,IAAI,GAAG,OAAO,GAAG,EAAE,CAAC,CAAC;YAC9F,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YAC5B,OAAO;QACX,CAAC;QAED,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC/B,IAAI,CAAC,GAAG,EAAE,CAAC;YACP,6EAA6E;YAC7E,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAA,wBAAa,EAAC,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;YAClF,OAAO;QACX,CAAC;QACD,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE1B,IAAI,GAAG,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YAC3B,KAAK,IAAI,CAAC,IAAI,CAAC,IAAA,wBAAa,EAAC,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC,EAAE,GAAG,CAAC,CAAC;YAC9D,OAAO;QACX,CAAC;QACD,IAAI,GAAG,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YAC3B,MAAM,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;QACpC,CAAC;IACL,CAAC;IAED,6FAA6F;IACrF,KAAK,CAAC,eAAe,CAAC,GAAc;QACxC,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACtC,GAAG,CAAC,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC;YACd,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YAC5B,OAAO;QACX,CAAC;QAED,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;QAClC,IAAI,CAAC;YACD,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;YAC1E,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,EAAE,4BAAiB,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;YAC/F,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,GAAG,CAAC,MAAM,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC,CAAC;YACpE,IAAI,CAAC,IAAI,EAAE,CAAC;gBACR,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;gBAC5B,OAAO;YACX,CAAC;YAED,MAAM,MAAM,GAAG,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC7F,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,GAAG,CAAC,MAAM,MAAM,MAAM,EAAE,CAAC,CAAC;YACzD,IAAI,CAAC,MAAM,EAAE,CAAC;gBACV,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;gBAC5B,OAAO;YACX,CAAC;YAED,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;YAChC,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC/F,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;QAClD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,0BAA0B,GAAG,CAAC,MAAM,MAAO,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;QAC1F,CAAC;gBAAS,CAAC;YACP,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QAChC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,SAAS,CAAC,GAAc,EAAE,GAAW,EAAE,UAAkB;QACnE,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,GAAG,GAAG,GAAG,CAAC,MAAM,EAAE,GAAG,IAAI,eAAe,EAAE,CAAC,EAAE,EAAE,CAAC;YACrE,MAAM,IAAI,CAAC,IAAI,CAAC,IAAA,oBAAS,EAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,eAAe,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YAC1E,kGAAkG;YAClG,IAAI,CAAC,CAAC,GAAG,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC;gBAClB,MAAM,IAAI,OAAO,CAAO,GAAG,CAAC,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC;YACtD,CAAC;QACL,CAAC;QACD,MAAM,IAAI,CAAC,IAAI,CAAC,IAAA,wBAAa,EAAC,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,UAAU,EAAE,CAAC,EAAE,GAAG,CAAC,CAAC;IACtF,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ,CAAC,MAAqB,EAAE,GAAW,EAAE,UAAkB;QACjE,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC;YACd,OAAO;QACX,CAAC;QACD,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC;QACpF,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;YAClB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,gBAAgB,MAAM,CAAC,CAAC,CAAC,cAAc,MAAM,GAAG,CAAC,CAAC,CAAC,YAAY,aAAa,CAAC,CAAC;YACjG,OAAO;QACX,CAAC;QACD,MAAM,OAAO,CAAC,GAAG,CACb,OAAO,CAAC,GAAG,CAAC,KAAK,EAAC,GAAG,EAAC,EAAE;YACpB,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;YAChC,IAAI,CAAC;gBACD,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,GAAG,EAAE,UAAU,CAAC,CAAC;YAC/C,CAAC;oBAAS,CAAC;gBACP,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YAChC,CAAC;QACL,CAAC,CAAC,CACL,CAAC;IACN,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,MAAqB;QACxB,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC;QACpF,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;YACxB,KAAK,IAAI,CAAC,IAAI,CAAC,IAAA,wBAAa,EAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,EAAE,GAAG,CAAC,CAAC;YACvD,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;QACrC,CAAC;QACD,OAAO,OAAO,CAAC,MAAM,CAAC;IAC1B,CAAC;IAED,2DAA2D;IAC3D,OAAO;QACH,OAAO,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IACtD,CAAC;IAEO,SAAS,CAAC,GAAc,EAAE,KAAqB;QACnD,KAAK,IAAI,CAAC,IAAI,CAAC,IAAA,wBAAa,EAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,CAAC,CAAC;QAC9D,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACtD,CAAC;IAEO,IAAI,CAAC,IAAY,EAAE,GAAc;QACrC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACnC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;YACvB,IAAI,CAAC,IAAI,EAAE,CAAC;gBACR,OAAO,EAAE,CAAC;gBACV,OAAO;YACX,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QACnF,CAAC,CAAC,CAAC;IACP,CAAC;IAEO,UAAU;QACd,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACjC,IAAI,GAAG,GAAG,GAAG,CAAC,QAAQ,GAAG,QAAQ,EAAE,CAAC;gBAChC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBACtB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,gCAAgC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;gBACjE,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;YAC1D,CAAC;QACL,CAAC;IACL,CAAC;CACJ;AAjOD,kCAiOC","sourcesContent":["/**\n * UDP voice server — the \"brain\" side of the satellite protocol.\n *\n * Receives streamed microphone audio from satellites, runs STT → answer callback → TTS,\n * and streams the spoken reply back. Registration + heartbeat keep the satellite registry\n * live. Protocol is Hannah-compatible (see `protocol.ts`).\n */\nimport * as dgram from 'node:dgram';\nimport {\n AUDIO_SAMPLE_RATE,\n TYPE_AUDIO,\n TYPE_CONTROL,\n decodePacket,\n encodeControl,\n encodeTts,\n type SatToServer,\n type SatelliteState,\n} from './protocol';\nimport type { SttEngine } from './stt';\nimport type { TtsEngine } from './tts';\n\ninterface Satellite {\n device: string;\n room: string;\n address: string;\n port: number;\n lastSeen: number;\n /** Buffered audio chunks of the utterance currently being spoken. */\n chunks: Buffer[];\n}\n\nexport interface VoiceServerOptions {\n port: number;\n /** Interface to bind to ('0.0.0.0'/'' = all interfaces, or a specific IP). */\n bindAddress?: string;\n /** ISO-639-1 language hint for STT/TTS ('' = auto/provider default). */\n language: string;\n stt: SttEngine;\n tts: TtsEngine;\n /** Produce the assistant reply for a transcribed utterance. */\n answer: (question: string, ctx: { device: string; room: string }) => Promise<string>;\n /** Optional STT vocabulary hints (device/room names) to bias recognition; re-read per utterance. */\n getHints?: () => string[] | Promise<string[]>;\n log: ioBroker.Logger;\n /** Notified on every satellite state transition (idle/listening/processing/speaking/offline). */\n onStatus?: (device: string, room: string, state: SatelliteState | 'offline') => void;\n}\n\n/** Payload bytes per TTS datagram — well under the typical MTU-fragmentation-free UDP size. */\nconst TTS_CHUNK_BYTES = 8192;\n/** Drop a satellite after this long without any packet (3× the satellite's 10 s heartbeat). */\nconst STALE_MS = 30_000;\n\nexport class VoiceServer {\n private sock: dgram.Socket | null = null;\n /** Keyed by `${address}:${port}` — audio packets carry no device name, only their source. */\n private readonly sats = new Map<string, Satellite>();\n private sweepTimer: NodeJS.Timeout | null = null;\n\n constructor(private readonly opts: VoiceServerOptions) {}\n\n start(): Promise<void> {\n return new Promise((resolve, reject) => {\n const sock = dgram.createSocket('udp4');\n this.sock = sock;\n sock.on('message', (data, rinfo) => this.onMessage(data, rinfo));\n sock.on('error', e => this.opts.log.error(`voice UDP error: ${e.message}`));\n sock.once('error', reject);\n // '0.0.0.0'/'' → bind all interfaces (omit the address); otherwise bind the given IP.\n const addr =\n this.opts.bindAddress && this.opts.bindAddress !== '0.0.0.0' ? this.opts.bindAddress : undefined;\n const onBound = (): void => {\n this.opts.log.info(`Voice server listening on UDP ${addr || '0.0.0.0'}:${this.opts.port}`);\n this.sweepTimer = setInterval(() => this.sweepStale(), 15_000);\n resolve();\n };\n if (addr) {\n sock.bind(this.opts.port, addr, onBound);\n } else {\n sock.bind(this.opts.port, onBound);\n }\n });\n }\n\n async stop(): Promise<void> {\n if (this.sweepTimer) {\n clearInterval(this.sweepTimer);\n this.sweepTimer = null;\n }\n const sock = this.sock;\n this.sock = null;\n this.sats.clear();\n if (sock) {\n await new Promise<void>(res => sock.close(() => res()));\n }\n }\n\n private static addrKey(address: string, port: number): string {\n return `${address}:${port}`;\n }\n\n private onMessage(data: Buffer, rinfo: dgram.RemoteInfo): void {\n if (!data.length) {\n return;\n }\n const { type, payload } = decodePacket(data);\n if (type === TYPE_AUDIO) {\n const sat = this.sats.get(VoiceServer.addrKey(rinfo.address, rinfo.port));\n if (sat) {\n sat.lastSeen = Date.now();\n sat.chunks.push(Buffer.from(payload)); // copy: payload is a view onto a reused recv buffer\n }\n return;\n }\n if (type === TYPE_CONTROL) {\n let msg: SatToServer;\n try {\n msg = JSON.parse(payload.toString('utf8')) as SatToServer;\n } catch {\n return;\n }\n this.onControl(msg, rinfo).catch(e => this.opts.log.warn(`voice control error: ${(e as Error).message}`));\n }\n }\n\n private async onControl(msg: SatToServer, rinfo: dgram.RemoteInfo): Promise<void> {\n const key = VoiceServer.addrKey(rinfo.address, rinfo.port);\n\n if (msg.type === 'register') {\n const sat: Satellite = {\n device: msg.device || key,\n room: msg.room || '',\n address: rinfo.address,\n port: rinfo.port,\n lastSeen: Date.now(),\n chunks: [],\n };\n this.sats.set(key, sat);\n void this.send(encodeControl({ type: 'registered', ok: true }), sat);\n this.opts.log.info(`Satellite registered: ${sat.device} (room: ${sat.room || '-'}) @ ${key}`);\n this.setStatus(sat, 'idle');\n return;\n }\n\n const sat = this.sats.get(key);\n if (!sat) {\n // Unknown source (server restarted?) — tell the satellite to register again.\n this.sock?.send(encodeControl({ type: 'reregister' }), rinfo.port, rinfo.address);\n return;\n }\n sat.lastSeen = Date.now();\n\n if (msg.type === 'heartbeat') {\n void this.send(encodeControl({ type: 'heartbeat_ack' }), sat);\n return;\n }\n if (msg.type === 'audio_end') {\n await this.handleUtterance(sat);\n }\n }\n\n /** STT → answer → TTS for one completed utterance, with status transitions along the way. */\n private async handleUtterance(sat: Satellite): Promise<void> {\n const pcm = Buffer.concat(sat.chunks);\n sat.chunks = [];\n if (!pcm.length) {\n this.setStatus(sat, 'idle');\n return;\n }\n\n this.setStatus(sat, 'processing');\n try {\n const hints = this.opts.getHints ? await this.opts.getHints() : undefined;\n const text = await this.opts.stt.transcribe(pcm, AUDIO_SAMPLE_RATE, this.opts.language, hints);\n this.opts.log.info(`Voice Q (${sat.device}): ${text || '(empty)'}`);\n if (!text) {\n this.setStatus(sat, 'idle');\n return;\n }\n\n const answer = (await this.opts.answer(text, { device: sat.device, room: sat.room })).trim();\n this.opts.log.info(`Voice A (${sat.device}): ${answer}`);\n if (!answer) {\n this.setStatus(sat, 'idle');\n return;\n }\n\n this.setStatus(sat, 'speaking');\n const { pcm: ttsPcm, sampleRate } = await this.opts.tts.synthesize(answer, this.opts.language);\n await this.streamTts(sat, ttsPcm, sampleRate);\n } catch (e) {\n this.opts.log.error(`Voice pipeline failed (${sat.device}): ${(e as Error).message}`);\n } finally {\n this.setStatus(sat, 'idle');\n }\n }\n\n private async streamTts(sat: Satellite, pcm: Buffer, sampleRate: number): Promise<void> {\n for (let off = 0, i = 0; off < pcm.length; off += TTS_CHUNK_BYTES, i++) {\n await this.send(encodeTts(pcm.subarray(off, off + TTS_CHUNK_BYTES)), sat);\n // Yield every 16 packets so a long reply doesn't burst the OS send buffer / the satellite's recv.\n if ((i & 15) === 15) {\n await new Promise<void>(res => setImmediate(res));\n }\n }\n await this.send(encodeControl({ type: 'tts_end', sample_rate: sampleRate }), sat);\n }\n\n /**\n * Play arbitrary PCM on one satellite (by device name) or all registered satellites (device=null).\n * Used for announcements (`tts.text` / per-satellite `tts`), independent of the wake-word flow.\n */\n async announce(device: string | null, pcm: Buffer, sampleRate: number): Promise<void> {\n if (!pcm.length) {\n return;\n }\n const targets = [...this.sats.values()].filter(s => !device || s.device === device);\n if (!targets.length) {\n this.opts.log.warn(`announce: no ${device ? `satellite '${device}'` : 'satellites'} registered`);\n return;\n }\n await Promise.all(\n targets.map(async sat => {\n this.setStatus(sat, 'speaking');\n try {\n await this.streamTts(sat, pcm, sampleRate);\n } finally {\n this.setStatus(sat, 'idle');\n }\n }),\n );\n }\n\n /**\n * Re-open the microphone on one satellite (by device name) or all of them, so the user can answer a\n * question the assistant just asked without saying the wake word again. Returns how many satellites\n * were told to listen — whether they can is up to their firmware (the message is additive).\n */\n listen(device: string | null): number {\n const targets = [...this.sats.values()].filter(s => !device || s.device === device);\n for (const sat of targets) {\n void this.send(encodeControl({ type: 'listen' }), sat);\n this.setStatus(sat, 'listening');\n }\n return targets.length;\n }\n\n /** Device names of the currently registered satellites. */\n devices(): string[] {\n return [...this.sats.values()].map(s => s.device);\n }\n\n private setStatus(sat: Satellite, state: SatelliteState): void {\n void this.send(encodeControl({ type: 'status', state }), sat);\n this.opts.onStatus?.(sat.device, sat.room, state);\n }\n\n private send(data: Buffer, sat: Satellite): Promise<void> {\n return new Promise((resolve, reject) => {\n const sock = this.sock;\n if (!sock) {\n resolve();\n return;\n }\n sock.send(data, sat.port, sat.address, err => (err ? reject(err) : resolve()));\n });\n }\n\n private sweepStale(): void {\n const now = Date.now();\n for (const [key, sat] of this.sats) {\n if (now - sat.lastSeen > STALE_MS) {\n this.sats.delete(key);\n this.opts.log.info(`Satellite offline (timeout): ${sat.device}`);\n this.opts.onStatus?.(sat.device, sat.room, 'offline');\n }\n }\n }\n}\n"]}
|