iobroker.assistant 1.1.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +74 -1
- package/admin/custom/.vite/manifest.json +62 -62
- package/admin/custom/assets/Components-x2Dfv8qE.js +2 -0
- package/admin/custom/assets/{_virtual_mf-localSharedImportMap___mfe_internal__ConfigCustomAssistant__mf_owner__1-A4gakomR.js → _virtual_mf-localSharedImportMap___mfe_internal__ConfigCustomAssistant__mf_owner__1-B4YbqhBb.js} +1 -1
- package/admin/custom/assets/{_virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__201375093093590__loadShare___mf_0_emotion_mf_1_styled__loadShare__.js-CwqmfMvn.js → _virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__163539842885036__loadShare___mf_0_emotion_mf_1_styled__loadShare__.js-IaosaymG.js} +1 -1
- package/admin/custom/assets/{_virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__201375093093590__loadShare___mf_0_iobroker_mf_1_gui_mf_2_components__loadShare__.js-BMuLYe6f.js → _virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__163539842885036__loadShare___mf_0_iobroker_mf_1_gui_mf_2_components__loadShare__.js-o4EU_Ut5.js} +39 -39
- package/admin/custom/assets/{_virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__201375093093590__loadShare__shared-BxjZmnpc.js → _virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__163539842885036__loadShare__shared--be2hWbV.js} +1 -1
- package/admin/custom/assets/_virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__163539842885036__prebuild___mf_0_iobroker_mf_1_json_mf_2_config__prebuild__-BsTz_EWE.js +122 -0
- package/admin/custom/assets/{hostInit-4roC4hEj.js → hostInit-Blh7a-Qe.js} +1 -1
- package/admin/custom/assets/{lib-CchzgTUs.js → lib-DBbNtolG.js} +1 -1
- package/admin/custom/assets/{mf-entry-bootstrap-0-4806bbe4.js → mf-entry-bootstrap-0-7abd30d3.js} +2 -2
- package/admin/custom/assets/{pendingShares-DLlv8yFY.js → pendingShares-U0D1MYK4.js} +1 -1
- package/admin/custom/customComponents.js +2 -2
- package/admin/custom/i18n/de.json +56 -0
- package/admin/custom/i18n/en.json +56 -0
- package/admin/custom/i18n/es.json +56 -0
- package/admin/custom/i18n/fr.json +56 -0
- package/admin/custom/i18n/it.json +56 -0
- package/admin/custom/i18n/nl.json +56 -0
- package/admin/custom/i18n/pl.json +56 -0
- package/admin/custom/i18n/pt.json +56 -0
- package/admin/custom/i18n/ru.json +56 -0
- package/admin/custom/i18n/uk.json +56 -0
- package/admin/custom/i18n/zh-cn.json +56 -0
- package/admin/custom/index.html +2 -2
- package/admin/custom/mf-manifest.json +1 -1
- package/admin/custom/mf-stats.json +1 -1
- package/admin/i18n/de.json +31 -4
- package/admin/i18n/en.json +31 -4
- package/admin/i18n/es.json +31 -4
- package/admin/i18n/fr.json +31 -4
- package/admin/i18n/it.json +31 -4
- package/admin/i18n/nl.json +31 -4
- package/admin/i18n/pl.json +31 -4
- package/admin/i18n/pt.json +31 -4
- package/admin/i18n/ru.json +31 -4
- package/admin/i18n/uk.json +31 -4
- package/admin/i18n/zh-cn.json +31 -4
- package/admin/jsonConfig.json +226 -15
- package/build/lib/credentials.js +11 -5
- package/build/lib/credentials.js.map +1 -1
- package/build/lib/journal.js +298 -2
- package/build/lib/journal.js.map +1 -1
- package/build/lib/llm.js +122 -7
- package/build/lib/llm.js.map +1 -1
- package/build/lib/localLlm.js +47 -10
- package/build/lib/localLlm.js.map +1 -1
- package/build/lib/nlu.js +372 -13
- package/build/lib/nlu.js.map +1 -1
- package/build/lib/piAgent.js +89 -0
- package/build/lib/piAgent.js.map +1 -0
- package/build/lib/systemAi.js +124 -0
- package/build/lib/systemAi.js.map +1 -0
- package/build/lib/tools.js +3 -0
- package/build/lib/tools.js.map +1 -1
- package/build/lib/voice/engines.js +31 -3
- package/build/lib/voice/engines.js.map +1 -1
- package/build/lib/voice/esphome.js +81 -7
- package/build/lib/voice/esphome.js.map +1 -1
- package/build/lib/voice/esphomeProto.js.map +1 -1
- package/build/lib/voice/mediaServer.js +29 -1
- package/build/lib/voice/mediaServer.js.map +1 -1
- package/build/lib/voice/openrouter.js +220 -0
- package/build/lib/voice/openrouter.js.map +1 -0
- package/build/lib/voice/protocol.js.map +1 -1
- package/build/lib/voice/voiceServer.js +81 -7
- package/build/lib/voice/voiceServer.js.map +1 -1
- package/build/lib/voice/wakeWords.js +284 -0
- package/build/lib/voice/wakeWords.js.map +1 -0
- package/build/lib/voice/wyoming.js +12 -4
- package/build/lib/voice/wyoming.js.map +1 -1
- package/build/lib/weather.js +160 -0
- package/build/lib/weather.js.map +1 -1
- package/build/main.js +1282 -126
- package/build/main.js.map +1 -1
- package/io-package.json +53 -41
- package/package.json +5 -5
- package/admin/custom/assets/Components-C6qcpaRC.js +0 -1
- package/admin/custom/assets/_virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__201375093093590__prebuild___mf_0_iobroker_mf_1_json_mf_2_config__prebuild__-TM4l0OYm.js +0 -122
- /package/admin/custom/assets/{_virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__201375093093590__loadShare___mf_0_emotion_mf_1_react__loadShare__.js-BkH7fzDw.js → _virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__163539842885036__loadShare___mf_0_emotion_mf_1_react__loadShare__.js-BkH7fzDw.js} +0 -0
- /package/admin/custom/assets/{_virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__201375093093590__prebuild__react_mf_1_jsx_mf_2_runtime__prebuild__-M61rjtmp.js → _virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__163539842885036__prebuild__react_mf_1_jsx_mf_2_runtime__prebuild__-M61rjtmp.js} +0 -0
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.OpenRouterTts = exports.OpenRouterStt = exports.DEFAULT_OPENROUTER_TTS_MODEL = exports.DEFAULT_OPENROUTER_STT_MODEL = void 0;
|
|
37
|
+
exports.listOpenRouterSpeechModels = listOpenRouterSpeechModels;
|
|
38
|
+
exports.voicesForLanguage = voicesForLanguage;
|
|
39
|
+
exports.pcmFromResponse = pcmFromResponse;
|
|
40
|
+
/**
|
|
41
|
+
* Speech over OpenRouter: one key for the speech-to-text and text-to-speech models of many vendors (OpenAI,
|
|
42
|
+
* ElevenLabs, Google, Deepgram, Mistral …) behind the OpenAI audio API at https://openrouter.ai/api/v1.
|
|
43
|
+
* Model ids carry the vendor (`openai/whisper-large-v3-turbo`), and the voices depend on the model — the public
|
|
44
|
+
* model list names them (`supported_voices`), so the settings offer exactly the voices of the chosen model.
|
|
45
|
+
*
|
|
46
|
+
* Differences to OpenAI itself: transcription ignores `prompt` (no vocabulary hints), and speech comes back as
|
|
47
|
+
* `pcm` (most models, the default) or `mp3` (MiniMax only). The PCM rate is not part of the API — OpenRouter
|
|
48
|
+
* documents 24 kHz, the rate of OpenAI, Gemini and Azure — so a rate in the content type wins over that default.
|
|
49
|
+
*/
|
|
50
|
+
const openai_1 = __importStar(require("openai"));
|
|
51
|
+
const llm_1 = require("../llm");
|
|
52
|
+
const stt_1 = require("./stt");
|
|
53
|
+
/** Speech-to-text model when none is configured: Whisper large v3 turbo, multilingual and the cheapest Whisper */
|
|
54
|
+
exports.DEFAULT_OPENROUTER_STT_MODEL = 'openai/whisper-large-v3-turbo';
|
|
55
|
+
/**
|
|
56
|
+
* Text-to-speech model when none is configured: multilingual, PCM (no decoder needed) and fast — in a test about
|
|
57
|
+
* 0.8 s for a sentence where Gemini TTS took 2.5 s, which a spoken answer waits for.
|
|
58
|
+
*/
|
|
59
|
+
exports.DEFAULT_OPENROUTER_TTS_MODEL = 'elevenlabs/eleven-flash-v2.5';
|
|
60
|
+
/** Rate of the PCM OpenRouter returns when the content type does not say otherwise */
|
|
61
|
+
const PCM_RATE = 24000;
|
|
62
|
+
/** Vendors whose speech models only return mp3 (they answer `pcm` with a 400) */
|
|
63
|
+
const MP3_ONLY = ['minimax/'];
|
|
64
|
+
/** How long the public model list is kept */
|
|
65
|
+
const MODEL_LIST_TTL_MS = 60 * 60 * 1000;
|
|
66
|
+
function client(apiKey) {
|
|
67
|
+
return new openai_1.default({ apiKey, baseURL: llm_1.PROVIDER_PRESETS.openrouter.baseUrl, defaultHeaders: llm_1.OPENROUTER_HEADERS });
|
|
68
|
+
}
|
|
69
|
+
const modelLists = new Map();
|
|
70
|
+
/**
|
|
71
|
+
* The speech models of OpenRouter. The list is public (no key), cached for an hour; a failed request is not kept.
|
|
72
|
+
*
|
|
73
|
+
* @param kind `speech` = text-to-speech, `transcription` = speech-to-text
|
|
74
|
+
* @param fetchImpl fetch (tests)
|
|
75
|
+
*/
|
|
76
|
+
function listOpenRouterSpeechModels(kind, fetchImpl = fetch) {
|
|
77
|
+
const cached = modelLists.get(kind);
|
|
78
|
+
if (cached && Date.now() - cached.at < MODEL_LIST_TTL_MS) {
|
|
79
|
+
return cached.models;
|
|
80
|
+
}
|
|
81
|
+
const models = (async () => {
|
|
82
|
+
const res = await fetchImpl(`${llm_1.PROVIDER_PRESETS.openrouter.baseUrl}/models?output_modalities=${kind}`, {
|
|
83
|
+
headers: llm_1.OPENROUTER_HEADERS,
|
|
84
|
+
signal: AbortSignal.timeout(15000),
|
|
85
|
+
});
|
|
86
|
+
if (!res.ok) {
|
|
87
|
+
throw new Error(`OpenRouter model list: HTTP ${res.status}`);
|
|
88
|
+
}
|
|
89
|
+
const body = (await res.json());
|
|
90
|
+
return (body.data || [])
|
|
91
|
+
.filter(m => typeof m.id === 'string' && m.id)
|
|
92
|
+
.map(m => ({
|
|
93
|
+
id: m.id,
|
|
94
|
+
name: typeof m.name === 'string' ? m.name : m.id,
|
|
95
|
+
voices: Array.isArray(m.supported_voices)
|
|
96
|
+
? m.supported_voices.filter((v) => typeof v === 'string')
|
|
97
|
+
: [],
|
|
98
|
+
}));
|
|
99
|
+
})();
|
|
100
|
+
modelLists.set(kind, { at: Date.now(), models });
|
|
101
|
+
models.catch(() => modelLists.delete(kind));
|
|
102
|
+
return models;
|
|
103
|
+
}
|
|
104
|
+
/**
|
|
105
|
+
* The voices of a text-to-speech model. Voices named after a locale (`de-DE-Klaus:MAI-Voice-2`) are narrowed to
|
|
106
|
+
* the language, if that leaves any; other names (`Kore`, `george`) say nothing about the language.
|
|
107
|
+
*
|
|
108
|
+
* @param voices the voices of the model
|
|
109
|
+
* @param language ISO-639-1 language ('' = all)
|
|
110
|
+
*/
|
|
111
|
+
function voicesForLanguage(voices, language) {
|
|
112
|
+
const lang = (language || '').split('-')[0].toLowerCase();
|
|
113
|
+
if (!lang) {
|
|
114
|
+
return voices;
|
|
115
|
+
}
|
|
116
|
+
const matching = voices.filter(v => new RegExp(`^${lang}[-_][A-Z]{2}\\b`).test(v));
|
|
117
|
+
return matching.length ? matching : voices;
|
|
118
|
+
}
|
|
119
|
+
/**
|
|
120
|
+
* Raw PCM out of a speech response: WAV is unwrapped, the rate comes from the content type (`audio/pcm;rate=…`)
|
|
121
|
+
* or is the documented 24 kHz.
|
|
122
|
+
*
|
|
123
|
+
* @param audio the response body
|
|
124
|
+
* @param contentType the content type of the response
|
|
125
|
+
*/
|
|
126
|
+
function pcmFromResponse(audio, contentType) {
|
|
127
|
+
if (audio.length > 44 && audio.toString('latin1', 0, 4) === 'RIFF' && audio.toString('latin1', 8, 12) === 'WAVE') {
|
|
128
|
+
const sampleRate = audio.readUInt32LE(24);
|
|
129
|
+
// the data chunk is not always right after the 16-byte fmt chunk
|
|
130
|
+
let offset = 12;
|
|
131
|
+
while (offset + 8 <= audio.length) {
|
|
132
|
+
const id = audio.toString('latin1', offset, offset + 4);
|
|
133
|
+
const size = audio.readUInt32LE(offset + 4);
|
|
134
|
+
if (id === 'data') {
|
|
135
|
+
return { pcm: audio.subarray(offset + 8, Math.min(audio.length, offset + 8 + size)), sampleRate };
|
|
136
|
+
}
|
|
137
|
+
offset += 8 + size + (size % 2);
|
|
138
|
+
}
|
|
139
|
+
return { pcm: audio.subarray(44), sampleRate };
|
|
140
|
+
}
|
|
141
|
+
const rate = /(?:^|[;\s])(?:rate|sample_?rate)=(\d+)/i.exec(contentType || '');
|
|
142
|
+
return { pcm: audio, sampleRate: rate ? Number(rate[1]) : PCM_RATE };
|
|
143
|
+
}
|
|
144
|
+
/** Speech-to-text over OpenRouter (OpenAI-compatible multipart upload) */
|
|
145
|
+
class OpenRouterStt {
|
|
146
|
+
client;
|
|
147
|
+
model;
|
|
148
|
+
constructor(apiKey, model = '') {
|
|
149
|
+
this.client = client(apiKey);
|
|
150
|
+
this.model = model || exports.DEFAULT_OPENROUTER_STT_MODEL;
|
|
151
|
+
}
|
|
152
|
+
async transcribe(pcm, sampleRate, lang) {
|
|
153
|
+
const file = await (0, openai_1.toFile)((0, stt_1.pcmToWav)(pcm, sampleRate), 'audio.wav', { type: 'audio/wav' });
|
|
154
|
+
// no vocabulary hints: OpenRouter accepts `prompt`, but ignores it
|
|
155
|
+
const res = await this.client.audio.transcriptions.create({
|
|
156
|
+
file,
|
|
157
|
+
model: this.model,
|
|
158
|
+
language: (lang || '').split('-')[0] || undefined,
|
|
159
|
+
});
|
|
160
|
+
return (res.text || '').trim();
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
exports.OpenRouterStt = OpenRouterStt;
|
|
164
|
+
/** Text-to-speech over OpenRouter */
|
|
165
|
+
class OpenRouterTts {
|
|
166
|
+
client;
|
|
167
|
+
model;
|
|
168
|
+
voice;
|
|
169
|
+
decode;
|
|
170
|
+
fetchImpl;
|
|
171
|
+
/**
|
|
172
|
+
* @param apiKey OpenRouter key
|
|
173
|
+
* @param model model id ('' = {@link DEFAULT_OPENROUTER_TTS_MODEL})
|
|
174
|
+
* @param voice voice of the model ('' = the first voice of the model for the language)
|
|
175
|
+
* @param decode decoder for the models that only return mp3
|
|
176
|
+
* @param fetchImpl fetch for the model list (tests)
|
|
177
|
+
*/
|
|
178
|
+
constructor(apiKey, model = '', voice = '', decode, fetchImpl) {
|
|
179
|
+
this.client = client(apiKey);
|
|
180
|
+
this.model = model || exports.DEFAULT_OPENROUTER_TTS_MODEL;
|
|
181
|
+
this.voice = voice;
|
|
182
|
+
this.decode = decode;
|
|
183
|
+
this.fetchImpl = fetchImpl;
|
|
184
|
+
}
|
|
185
|
+
/** The configured voice, or the first one the model offers for the language — most models need a voice */
|
|
186
|
+
async voiceFor(lang) {
|
|
187
|
+
if (this.voice) {
|
|
188
|
+
return this.voice;
|
|
189
|
+
}
|
|
190
|
+
try {
|
|
191
|
+
const models = await listOpenRouterSpeechModels('speech', this.fetchImpl);
|
|
192
|
+
const voices = models.find(m => m.id === this.model)?.voices || [];
|
|
193
|
+
return voicesForLanguage(voices, lang)[0] || '';
|
|
194
|
+
}
|
|
195
|
+
catch {
|
|
196
|
+
return '';
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
async synthesize(text, lang) {
|
|
200
|
+
const mp3 = MP3_ONLY.some(prefix => this.model.startsWith(prefix));
|
|
201
|
+
if (mp3 && !this.decode) {
|
|
202
|
+
throw new Error(`${this.model} only returns mp3, which needs ffmpeg`);
|
|
203
|
+
}
|
|
204
|
+
const voice = await this.voiceFor(lang);
|
|
205
|
+
const res = await this.client.audio.speech.create({
|
|
206
|
+
model: this.model,
|
|
207
|
+
input: text,
|
|
208
|
+
// `voice` is required by the OpenAI types; an empty one is left out (a model with a default voice)
|
|
209
|
+
...(voice ? { voice } : {}),
|
|
210
|
+
response_format: mp3 ? 'mp3' : 'pcm',
|
|
211
|
+
});
|
|
212
|
+
const audio = Buffer.from(await res.arrayBuffer());
|
|
213
|
+
if (mp3) {
|
|
214
|
+
return this.decode(audio);
|
|
215
|
+
}
|
|
216
|
+
return pcmFromResponse(audio, res.headers.get('content-type') || '');
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
exports.OpenRouterTts = OpenRouterTts;
|
|
220
|
+
//# sourceMappingURL=openrouter.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"openrouter.js","sourceRoot":"","sources":["../../../src/lib/voice/openrouter.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAuDA,gEA8BC;AASD,8CAOC;AASD,0CAiBC;AA/HD;;;;;;;;;GASG;AACH,iDAAwC;AACxC,gCAA8D;AAC9D,+BAAiD;AAGjD,kHAAkH;AACrG,QAAA,4BAA4B,GAAG,+BAA+B,CAAC;AAC5E;;;GAGG;AACU,QAAA,4BAA4B,GAAG,8BAA8B,CAAC;AAE3E,sFAAsF;AACtF,MAAM,QAAQ,GAAG,KAAK,CAAC;AAEvB,iFAAiF;AACjF,MAAM,QAAQ,GAAG,CAAC,UAAU,CAAC,CAAC;AAE9B,6CAA6C;AAC7C,MAAM,iBAAiB,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAazC,SAAS,MAAM,CAAC,MAAc;IAC1B,OAAO,IAAI,gBAAM,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,sBAAgB,CAAC,UAAU,CAAC,OAAO,EAAE,cAAc,EAAE,wBAAkB,EAAE,CAAC,CAAC;AACpH,CAAC;AAED,MAAM,UAAU,GAAG,IAAI,GAAG,EAAoE,CAAC;AAE/F;;;;;GAKG;AACH,SAAgB,0BAA0B,CACtC,IAAgC,EAChC,YAA0B,KAAK;IAE/B,MAAM,MAAM,GAAG,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACpC,IAAI,MAAM,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,EAAE,GAAG,iBAAiB,EAAE,CAAC;QACvD,OAAO,MAAM,CAAC,MAAM,CAAC;IACzB,CAAC;IACD,MAAM,MAAM,GAAG,CAAC,KAAK,IAAI,EAAE;QACvB,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,GAAG,sBAAgB,CAAC,UAAU,CAAC,OAAO,6BAA6B,IAAI,EAAE,EAAE;YACnG,OAAO,EAAE,wBAAkB;YAC3B,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC;SACrC,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACV,MAAM,IAAI,KAAK,CAAC,+BAA+B,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;QACjE,CAAC;QACD,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAA8E,CAAC;QAC7G,OAAO,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;aACnB,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC,EAAE,KAAK,QAAQ,IAAI,CAAC,CAAC,EAAE,CAAC;aAC7C,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACP,EAAE,EAAE,CAAC,CAAC,EAAY;YAClB,IAAI,EAAE,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC,CAAC,EAAa;YAC5D,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,gBAAgB,CAAC;gBACrC,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC;gBACtE,CAAC,CAAC,EAAE;SACX,CAAC,CAAC,CAAC;IACZ,CAAC,CAAC,EAAE,CAAC;IACL,UAAU,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;IACjD,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IAC5C,OAAO,MAAM,CAAC;AAClB,CAAC;AAED;;;;;;GAMG;AACH,SAAgB,iBAAiB,CAAC,MAAgB,EAAE,QAAgB;IAChE,MAAM,IAAI,GAAG,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;IAC1D,IAAI,CAAC,IAAI,EAAE,CAAC;QACR,OAAO,MAAM,CAAC;IAClB,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,IAAI,IAAI,iBAAiB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IACnF,OAAO,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC;AAC/C,CAAC;AAED;;;;;;GAMG;AACH,SAAgB,eAAe,CAAC,KAAa,EAAE,WAAmB;IAC9D,IAAI,KAAK,CAAC,MAAM,GAAG,EAAE,IAAI,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,KAAK,MAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC,KAAK,MAAM,EAAE,CAAC;QAC/G,MAAM,UAAU,GAAG,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;QAC1C,iEAAiE;QACjE,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,OAAO,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;YAChC,MAAM,EAAE,GAAG,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC;YACxD,MAAM,IAAI,GAAG,KAAK,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAC5C,IAAI,EAAE,KAAK,MAAM,EAAE,CAAC;gBAChB,OAAO,EAAE,GAAG,EAAE,KAAK,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC;YACtG,CAAC;YACD,MAAM,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC;QACpC,CAAC;QACD,OAAO,EAAE,GAAG,EAAE,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC;IACnD,CAAC;IACD,MAAM,IAAI,GAAG,yCAAyC,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC;IAC/E,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;AACzE,CAAC;AAED,0EAA0E;AAC1E,MAAa,aAAa;IACL,MAAM,CAAS;IACf,KAAK,CAAS;IAE/B,YAAY,MAAc,EAAE,KAAK,GAAG,EAAE;QAClC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;QAC7B,IAAI,CAAC,KAAK,GAAG,KAAK,IAAI,oCAA4B,CAAC;IACvD,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,GAAW,EAAE,UAAkB,EAAE,IAAY;QAC1D,MAAM,IAAI,GAAG,MAAM,IAAA,eAAM,EAAC,IAAA,cAAQ,EAAC,GAAG,EAAE,UAAU,CAAC,EAAE,WAAW,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;QACzF,mEAAmE;QACnE,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,MAAM,CAAC;YACtD,IAAI;YACJ,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,QAAQ,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,SAAS;SACpD,CAAC,CAAC;QACH,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IACnC,CAAC;CACJ;AAnBD,sCAmBC;AAED,qCAAqC;AACrC,MAAa,aAAa;IACL,MAAM,CAAS;IACf,KAAK,CAAS;IACd,KAAK,CAAS;IACd,MAAM,CAAgB;IACtB,SAAS,CAAgB;IAE1C;;;;;;OAMG;IACH,YAAY,MAAc,EAAE,KAAK,GAAG,EAAE,EAAE,KAAK,GAAG,EAAE,EAAE,MAAqB,EAAE,SAAwB;QAC/F,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;QAC7B,IAAI,CAAC,KAAK,GAAG,KAAK,IAAI,oCAA4B,CAAC;QACnD,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;IAC/B,CAAC;IAED,0GAA0G;IAClG,KAAK,CAAC,QAAQ,CAAC,IAAY;QAC/B,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACb,OAAO,IAAI,CAAC,KAAK,CAAC;QACtB,CAAC;QACD,IAAI,CAAC;YACD,MAAM,MAAM,GAAG,MAAM,0BAA0B,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;YAC1E,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC,KAAK,CAAC,EAAE,MAAM,IAAI,EAAE,CAAC;YACnE,OAAO,iBAAiB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACpD,CAAC;QAAC,MAAM,CAAC;YACL,OAAO,EAAE,CAAC;QACd,CAAC;IACL,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,IAAY,EAAE,IAAY;QACvC,MAAM,GAAG,GAAG,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC;QACnE,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,CAAC,KAAK,uCAAuC,CAAC,CAAC;QAC1E,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACxC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;YAC9C,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,KAAK,EAAE,IAAI;YACX,mGAAmG;YACnG,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3B,eAAe,EAAE,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK;SACJ,CAAC,CAAC;QACtC,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC;QACnD,IAAI,GAAG,EAAE,CAAC;YACN,OAAO,IAAI,CAAC,MAAO,CAAC,KAAK,CAAC,CAAC;QAC/B,CAAC;QACD,OAAO,eAAe,CAAC,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC;IACzE,CAAC;CACJ;AAvDD,sCAuDC","sourcesContent":["/**\n * Speech over OpenRouter: one key for the speech-to-text and text-to-speech models of many vendors (OpenAI,\n * ElevenLabs, Google, Deepgram, Mistral …) behind the OpenAI audio API at https://openrouter.ai/api/v1.\n * Model ids carry the vendor (`openai/whisper-large-v3-turbo`), and the voices depend on the model — the public\n * model list names them (`supported_voices`), so the settings offer exactly the voices of the chosen model.\n *\n * Differences to OpenAI itself: transcription ignores `prompt` (no vocabulary hints), and speech comes back as\n * `pcm` (most models, the default) or `mp3` (MiniMax only). The PCM rate is not part of the API — OpenRouter\n * documents 24 kHz, the rate of OpenAI, Gemini and Azure — so a rate in the content type wins over that default.\n */\nimport OpenAI, { toFile } from 'openai';\nimport { OPENROUTER_HEADERS, PROVIDER_PRESETS } from '../llm';\nimport { pcmToWav, type SttEngine } from './stt';\nimport type { TtsEngine, TtsResult } from './tts';\n\n/** Speech-to-text model when none is configured: Whisper large v3 turbo, multilingual and the cheapest Whisper */\nexport const DEFAULT_OPENROUTER_STT_MODEL = 'openai/whisper-large-v3-turbo';\n/**\n * Text-to-speech model when none is configured: multilingual, PCM (no decoder needed) and fast — in a test about\n * 0.8 s for a sentence where Gemini TTS took 2.5 s, which a spoken answer waits for.\n */\nexport const DEFAULT_OPENROUTER_TTS_MODEL = 'elevenlabs/eleven-flash-v2.5';\n\n/** Rate of the PCM OpenRouter returns when the content type does not say otherwise */\nconst PCM_RATE = 24000;\n\n/** Vendors whose speech models only return mp3 (they answer `pcm` with a 400) */\nconst MP3_ONLY = ['minimax/'];\n\n/** How long the public model list is kept */\nconst MODEL_LIST_TTL_MS = 60 * 60 * 1000;\n\n/** Decode compressed audio (mp3) to mono 16-bit PCM — ffmpeg in the adapter */\nexport type AudioDecoder = (audio: Buffer) => Promise<TtsResult>;\n\n/** A speech model of OpenRouter */\nexport interface OpenRouterSpeechModel {\n id: string;\n name: string;\n /** Voices of a text-to-speech model, empty when the model names none */\n voices: string[];\n}\n\nfunction client(apiKey: string): OpenAI {\n return new OpenAI({ apiKey, baseURL: PROVIDER_PRESETS.openrouter.baseUrl, defaultHeaders: OPENROUTER_HEADERS });\n}\n\nconst modelLists = new Map<string, { at: number; models: Promise<OpenRouterSpeechModel[]> }>();\n\n/**\n * The speech models of OpenRouter. The list is public (no key), cached for an hour; a failed request is not kept.\n *\n * @param kind `speech` = text-to-speech, `transcription` = speech-to-text\n * @param fetchImpl fetch (tests)\n */\nexport function listOpenRouterSpeechModels(\n kind: 'speech' | 'transcription',\n fetchImpl: typeof fetch = fetch,\n): Promise<OpenRouterSpeechModel[]> {\n const cached = modelLists.get(kind);\n if (cached && Date.now() - cached.at < MODEL_LIST_TTL_MS) {\n return cached.models;\n }\n const models = (async () => {\n const res = await fetchImpl(`${PROVIDER_PRESETS.openrouter.baseUrl}/models?output_modalities=${kind}`, {\n headers: OPENROUTER_HEADERS,\n signal: AbortSignal.timeout(15000),\n });\n if (!res.ok) {\n throw new Error(`OpenRouter model list: HTTP ${res.status}`);\n }\n const body = (await res.json()) as { data?: { id?: unknown; name?: unknown; supported_voices?: unknown }[] };\n return (body.data || [])\n .filter(m => typeof m.id === 'string' && m.id)\n .map(m => ({\n id: m.id as string,\n name: typeof m.name === 'string' ? m.name : (m.id as string),\n voices: Array.isArray(m.supported_voices)\n ? m.supported_voices.filter((v): v is string => typeof v === 'string')\n : [],\n }));\n })();\n modelLists.set(kind, { at: Date.now(), models });\n models.catch(() => modelLists.delete(kind));\n return models;\n}\n\n/**\n * The voices of a text-to-speech model. Voices named after a locale (`de-DE-Klaus:MAI-Voice-2`) are narrowed to\n * the language, if that leaves any; other names (`Kore`, `george`) say nothing about the language.\n *\n * @param voices the voices of the model\n * @param language ISO-639-1 language ('' = all)\n */\nexport function voicesForLanguage(voices: string[], language: string): string[] {\n const lang = (language || '').split('-')[0].toLowerCase();\n if (!lang) {\n return voices;\n }\n const matching = voices.filter(v => new RegExp(`^${lang}[-_][A-Z]{2}\\\\b`).test(v));\n return matching.length ? matching : voices;\n}\n\n/**\n * Raw PCM out of a speech response: WAV is unwrapped, the rate comes from the content type (`audio/pcm;rate=…`)\n * or is the documented 24 kHz.\n *\n * @param audio the response body\n * @param contentType the content type of the response\n */\nexport function pcmFromResponse(audio: Buffer, contentType: string): TtsResult {\n if (audio.length > 44 && audio.toString('latin1', 0, 4) === 'RIFF' && audio.toString('latin1', 8, 12) === 'WAVE') {\n const sampleRate = audio.readUInt32LE(24);\n // the data chunk is not always right after the 16-byte fmt chunk\n let offset = 12;\n while (offset + 8 <= audio.length) {\n const id = audio.toString('latin1', offset, offset + 4);\n const size = audio.readUInt32LE(offset + 4);\n if (id === 'data') {\n return { pcm: audio.subarray(offset + 8, Math.min(audio.length, offset + 8 + size)), sampleRate };\n }\n offset += 8 + size + (size % 2);\n }\n return { pcm: audio.subarray(44), sampleRate };\n }\n const rate = /(?:^|[;\\s])(?:rate|sample_?rate)=(\\d+)/i.exec(contentType || '');\n return { pcm: audio, sampleRate: rate ? Number(rate[1]) : PCM_RATE };\n}\n\n/** Speech-to-text over OpenRouter (OpenAI-compatible multipart upload) */\nexport class OpenRouterStt implements SttEngine {\n private readonly client: OpenAI;\n private readonly model: string;\n\n constructor(apiKey: string, model = '') {\n this.client = client(apiKey);\n this.model = model || DEFAULT_OPENROUTER_STT_MODEL;\n }\n\n async transcribe(pcm: Buffer, sampleRate: number, lang: string): Promise<string> {\n const file = await toFile(pcmToWav(pcm, sampleRate), 'audio.wav', { type: 'audio/wav' });\n // no vocabulary hints: OpenRouter accepts `prompt`, but ignores it\n const res = await this.client.audio.transcriptions.create({\n file,\n model: this.model,\n language: (lang || '').split('-')[0] || undefined,\n });\n return (res.text || '').trim();\n }\n}\n\n/** Text-to-speech over OpenRouter */\nexport class OpenRouterTts implements TtsEngine {\n private readonly client: OpenAI;\n private readonly model: string;\n private readonly voice: string;\n private readonly decode?: AudioDecoder;\n private readonly fetchImpl?: typeof fetch;\n\n /**\n * @param apiKey OpenRouter key\n * @param model model id ('' = {@link DEFAULT_OPENROUTER_TTS_MODEL})\n * @param voice voice of the model ('' = the first voice of the model for the language)\n * @param decode decoder for the models that only return mp3\n * @param fetchImpl fetch for the model list (tests)\n */\n constructor(apiKey: string, model = '', voice = '', decode?: AudioDecoder, fetchImpl?: typeof fetch) {\n this.client = client(apiKey);\n this.model = model || DEFAULT_OPENROUTER_TTS_MODEL;\n this.voice = voice;\n this.decode = decode;\n this.fetchImpl = fetchImpl;\n }\n\n /** The configured voice, or the first one the model offers for the language — most models need a voice */\n private async voiceFor(lang: string): Promise<string> {\n if (this.voice) {\n return this.voice;\n }\n try {\n const models = await listOpenRouterSpeechModels('speech', this.fetchImpl);\n const voices = models.find(m => m.id === this.model)?.voices || [];\n return voicesForLanguage(voices, lang)[0] || '';\n } catch {\n return '';\n }\n }\n\n async synthesize(text: string, lang: string): Promise<TtsResult> {\n const mp3 = MP3_ONLY.some(prefix => this.model.startsWith(prefix));\n if (mp3 && !this.decode) {\n throw new Error(`${this.model} only returns mp3, which needs ffmpeg`);\n }\n const voice = await this.voiceFor(lang);\n const res = await this.client.audio.speech.create({\n model: this.model,\n input: text,\n // `voice` is required by the OpenAI types; an empty one is left out (a model with a default voice)\n ...(voice ? { voice } : {}),\n response_format: mp3 ? 'mp3' : 'pcm',\n } as OpenAI.Audio.SpeechCreateParams);\n const audio = Buffer.from(await res.arrayBuffer());\n if (mp3) {\n return this.decode!(audio);\n }\n return pcmFromResponse(audio, res.headers.get('content-type') || '');\n }\n}\n"]}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"protocol.js","sourceRoot":"","sources":["../../../src/lib/voice/protocol.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;GAUG;;;
|
|
1
|
+
{"version":3,"file":"protocol.js","sourceRoot":"","sources":["../../../src/lib/voice/protocol.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;GAUG;;;AAyCH,sCAEC;AAGD,8BAEC;AAGD,oCAEC;AAnDY,QAAA,YAAY,GAAG,IAAI,CAAC;AACpB,QAAA,UAAU,GAAG,IAAI,CAAC;AAClB,QAAA,QAAQ,GAAG,IAAI,CAAC;AAE7B,oGAAoG;AACvF,QAAA,iBAAiB,GAAG,KAAK,CAAC;AAiCvC,gEAAgE;AAChE,SAAgB,aAAa,CAAC,GAAgB;IAC1C,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,oBAAY,CAAC,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;AAClG,CAAC;AAED,qDAAqD;AACrD,SAAgB,SAAS,CAAC,GAAW;IACjC,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,gBAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;AACzD,CAAC;AAED,mGAAmG;AACnG,SAAgB,YAAY,CAAC,IAAY;IACrC,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;AACxD,CAAC","sourcesContent":["/**\n * Voice-satellite UDP wire protocol — compatible with the Hannah satellite\n * (`C:\\iot\\Hannah\\satellite-pi\\satellite.py`) so existing ESP/Python satellites keep working.\n *\n * One UDP socket, first byte is the packet type:\n * 0x01 CONTROL — UTF-8 JSON message (both directions)\n * 0x02 AUDIO — raw 16 kHz mono 16-bit signed LE PCM (satellite → server)\n * 0x03 TTS — raw mono 16-bit signed LE PCM (server → satellite)\n *\n * ⚠️ KEEP IN SYNC with the copy in the standalone `@iobroker/assistant-satellite` repo.\n */\n\nexport const TYPE_CONTROL = 0x01;\nexport const TYPE_AUDIO = 0x02;\nexport const TYPE_TTS = 0x03;\n\n/** Audio on the wire from the satellite is always 16 kHz, mono, 16-bit signed little-endian PCM. */\nexport const AUDIO_SAMPLE_RATE = 16000;\n\n/** Control messages sent by a satellite to the server. */\nexport type SatToServer =\n | { type: 'register'; device: string; room?: string; listen_port?: number }\n | { type: 'heartbeat'; device: string }\n | { type: 'audio_end'; device: string }\n /** The satellite's playback volume (0–100) after a `volume` command, or whenever it changed locally. */\n | { type: 'volume'; device: string; level: number };\n\n/** Satellite lifecycle/status as reflected to the satellite (drives its LED) and to ioBroker states. */\nexport type SatelliteState = 'idle' | 'listening' | 'processing' | 'speaking';\n\n/**\n * Control messages sent by the server to a satellite.\n *\n * `listen` re-opens the microphone without a wake word, for the answer to a question the assistant just\n * asked (see `PendingQuestions`). The same request rides on `tts_end.listen` when the reply itself is the\n * question, so it cannot overtake the audio. `volume` sets the playback volume, absolute (`level`, 0–100)\n * or relative (`delta`, ±percentage points); the satellite answers with its resulting level.\n *\n * All of these are additive: a satellite built before they existed ignores the unknown type or field and\n * simply waits for its wake word, or keeps its volume.\n */\nexport type ServerToSat =\n | { type: 'registered'; ok: boolean }\n | { type: 'heartbeat_ack' }\n | { type: 'reregister' }\n | { type: 'tts_end'; sample_rate: number; listen?: boolean }\n | { type: 'status'; state: SatelliteState }\n | { type: 'listen' }\n | { type: 'volume'; level?: number; delta?: number };\n\n/** Prefix a control JSON message with the CONTROL type byte. */\nexport function encodeControl(msg: ServerToSat): Buffer {\n return Buffer.concat([Buffer.from([TYPE_CONTROL]), Buffer.from(JSON.stringify(msg), 'utf8')]);\n}\n\n/** Prefix a raw PCM chunk with the TTS type byte. */\nexport function encodeTts(pcm: Buffer): Buffer {\n return Buffer.concat([Buffer.from([TYPE_TTS]), pcm]);\n}\n\n/** Split an incoming datagram into its type byte and payload (a view onto the original buffer). */\nexport function decodePacket(data: Buffer): { type: number; payload: Buffer } {\n return { type: data[0], payload: data.subarray(1) };\n}\n"]}
|
|
@@ -47,6 +47,11 @@ const protocol_1 = require("./protocol");
|
|
|
47
47
|
const TTS_CHUNK_BYTES = 8192;
|
|
48
48
|
/** Drop a satellite after this long without any packet (3× the satellite's 10 s heartbeat). */
|
|
49
49
|
const STALE_MS = 30_000;
|
|
50
|
+
/**
|
|
51
|
+
* How long a satellite stays `listening` after a mic-on once its playback is over. A little longer than the
|
|
52
|
+
* satellite's own follow-up window (6 s by default), which we cannot see from here.
|
|
53
|
+
*/
|
|
54
|
+
const LISTEN_WINDOW_MS = 10_000;
|
|
50
55
|
class VoiceServer {
|
|
51
56
|
opts;
|
|
52
57
|
sock = null;
|
|
@@ -85,6 +90,7 @@ class VoiceServer {
|
|
|
85
90
|
}
|
|
86
91
|
const sock = this.sock;
|
|
87
92
|
this.sock = null;
|
|
93
|
+
this.sats.forEach(sat => this.clearListenTimer(sat));
|
|
88
94
|
this.sats.clear();
|
|
89
95
|
if (sock) {
|
|
90
96
|
await new Promise(res => sock.close(() => res()));
|
|
@@ -102,6 +108,8 @@ class VoiceServer {
|
|
|
102
108
|
const sat = this.sats.get(VoiceServer.addrKey(rinfo.address, rinfo.port));
|
|
103
109
|
if (sat) {
|
|
104
110
|
sat.lastSeen = Date.now();
|
|
111
|
+
// The user answered within the mic-on window — don't let the timer flip the LED to idle.
|
|
112
|
+
this.clearListenTimer(sat);
|
|
105
113
|
sat.chunks.push(Buffer.from(payload)); // copy: payload is a view onto a reused recv buffer
|
|
106
114
|
}
|
|
107
115
|
return;
|
|
@@ -128,6 +136,10 @@ class VoiceServer {
|
|
|
128
136
|
lastSeen: Date.now(),
|
|
129
137
|
chunks: [],
|
|
130
138
|
};
|
|
139
|
+
const previous = this.sats.get(key);
|
|
140
|
+
if (previous) {
|
|
141
|
+
this.clearListenTimer(previous);
|
|
142
|
+
}
|
|
131
143
|
this.sats.set(key, sat);
|
|
132
144
|
void this.send((0, protocol_1.encodeControl)({ type: 'registered', ok: true }), sat);
|
|
133
145
|
this.opts.log.info(`Satellite registered: ${sat.device} (room: ${sat.room || '-'}) @ ${key}`);
|
|
@@ -147,6 +159,13 @@ class VoiceServer {
|
|
|
147
159
|
}
|
|
148
160
|
if (msg.type === 'audio_end') {
|
|
149
161
|
await this.handleUtterance(sat);
|
|
162
|
+
return;
|
|
163
|
+
}
|
|
164
|
+
if (msg.type === 'volume') {
|
|
165
|
+
const level = Math.round(Number(msg.level));
|
|
166
|
+
if (Number.isFinite(level)) {
|
|
167
|
+
this.opts.onVolume?.(sat.device, sat.room, Math.min(100, Math.max(0, level)));
|
|
168
|
+
}
|
|
150
169
|
}
|
|
151
170
|
}
|
|
152
171
|
/** STT → answer → TTS for one completed utterance, with status transitions along the way. */
|
|
@@ -158,6 +177,8 @@ class VoiceServer {
|
|
|
158
177
|
return;
|
|
159
178
|
}
|
|
160
179
|
this.setStatus(sat, 'processing');
|
|
180
|
+
let listen = false;
|
|
181
|
+
let playMs = 0;
|
|
161
182
|
try {
|
|
162
183
|
const hints = this.opts.getHints ? await this.opts.getHints() : undefined;
|
|
163
184
|
const sttStart = Date.now();
|
|
@@ -168,24 +189,39 @@ class VoiceServer {
|
|
|
168
189
|
this.setStatus(sat, 'idle');
|
|
169
190
|
return;
|
|
170
191
|
}
|
|
171
|
-
const
|
|
192
|
+
const audioMs = Math.round((pcm.length / 2 / protocol_1.AUDIO_SAMPLE_RATE) * 1000);
|
|
193
|
+
const answer = (await this.opts.answer(text, { device: sat.device, room: sat.room, sttMs, audioMs })).trim();
|
|
172
194
|
this.opts.log.info(`Voice A (${sat.device}): ${answer}`);
|
|
173
195
|
if (!answer) {
|
|
174
196
|
this.setStatus(sat, 'idle');
|
|
175
197
|
return;
|
|
176
198
|
}
|
|
177
199
|
this.setStatus(sat, 'speaking');
|
|
200
|
+
const ttsStart = Date.now();
|
|
178
201
|
const { pcm: ttsPcm, sampleRate } = await this.opts.tts.synthesize(answer, this.opts.language);
|
|
179
|
-
|
|
202
|
+
playMs = (ttsPcm.length / 2 / sampleRate) * 1000;
|
|
203
|
+
this.opts.onSpoken?.(sat.device, { ms: Date.now() - ttsStart, audioMs: playMs });
|
|
204
|
+
listen = this.opts.expectsFollowUp?.(answer, sat.device) === true;
|
|
205
|
+
await this.streamTts(sat, ttsPcm, sampleRate, listen);
|
|
180
206
|
}
|
|
181
207
|
catch (e) {
|
|
182
208
|
this.opts.log.error(`Voice pipeline failed (${sat.device}): ${e.message}`);
|
|
209
|
+
listen = false;
|
|
183
210
|
}
|
|
184
211
|
finally {
|
|
185
|
-
|
|
212
|
+
if (listen) {
|
|
213
|
+
this.markListening(sat, playMs);
|
|
214
|
+
}
|
|
215
|
+
else {
|
|
216
|
+
this.setStatus(sat, 'idle');
|
|
217
|
+
}
|
|
186
218
|
}
|
|
187
219
|
}
|
|
188
|
-
|
|
220
|
+
/**
|
|
221
|
+
* Stream one reply. `listen` travels inside `tts_end` rather than as a message of its own: the satellite
|
|
222
|
+
* then knows before playback starts that it has to open the mic afterwards, and nothing can overtake it.
|
|
223
|
+
*/
|
|
224
|
+
async streamTts(sat, pcm, sampleRate, listen = false) {
|
|
189
225
|
for (let off = 0, i = 0; off < pcm.length; off += TTS_CHUNK_BYTES, i++) {
|
|
190
226
|
await this.send((0, protocol_1.encodeTts)(pcm.subarray(off, off + TTS_CHUNK_BYTES)), sat);
|
|
191
227
|
// Yield every 16 packets so a long reply doesn't burst the OS send buffer / the satellite's recv.
|
|
@@ -193,7 +229,9 @@ class VoiceServer {
|
|
|
193
229
|
await new Promise(res => setImmediate(res));
|
|
194
230
|
}
|
|
195
231
|
}
|
|
196
|
-
await this.send((0, protocol_1.encodeControl)(
|
|
232
|
+
await this.send((0, protocol_1.encodeControl)(listen
|
|
233
|
+
? { type: 'tts_end', sample_rate: sampleRate, listen }
|
|
234
|
+
: { type: 'tts_end', sample_rate: sampleRate }), sat);
|
|
197
235
|
}
|
|
198
236
|
/**
|
|
199
237
|
* Play arbitrary PCM on one satellite (by device name) or all registered satellites (device=null).
|
|
@@ -223,19 +261,54 @@ class VoiceServer {
|
|
|
223
261
|
* question the assistant just asked without saying the wake word again. Returns how many satellites
|
|
224
262
|
* were told to listen — whether they can is up to their firmware (the message is additive).
|
|
225
263
|
*/
|
|
226
|
-
listen(device) {
|
|
264
|
+
listen(device, playMs = 0) {
|
|
227
265
|
const targets = [...this.sats.values()].filter(s => !device || s.device === device);
|
|
228
266
|
for (const sat of targets) {
|
|
229
267
|
void this.send((0, protocol_1.encodeControl)({ type: 'listen' }), sat);
|
|
230
|
-
this.
|
|
268
|
+
this.markListening(sat, playMs);
|
|
231
269
|
}
|
|
232
270
|
return targets.length;
|
|
233
271
|
}
|
|
272
|
+
/**
|
|
273
|
+
* Change the playback volume of one satellite: absolute (`level`, 0–100) or relative (`delta`). The
|
|
274
|
+
* satellite answers with the level it ended up at (→ `onVolume`); one built before this message
|
|
275
|
+
* existed ignores it. Returns false when the device is not registered.
|
|
276
|
+
*/
|
|
277
|
+
volume(device, change) {
|
|
278
|
+
const sat = [...this.sats.values()].find(s => s.device === device);
|
|
279
|
+
if (!sat) {
|
|
280
|
+
return false;
|
|
281
|
+
}
|
|
282
|
+
const msg = change.level !== undefined
|
|
283
|
+
? { type: 'volume', level: Math.min(100, Math.max(0, Math.round(change.level))) }
|
|
284
|
+
: { type: 'volume', delta: Math.round(change.delta || 0) };
|
|
285
|
+
void this.send((0, protocol_1.encodeControl)(msg), sat);
|
|
286
|
+
return true;
|
|
287
|
+
}
|
|
288
|
+
/** Whether a satellite with this device name is registered. */
|
|
289
|
+
has(device) {
|
|
290
|
+
return [...this.sats.values()].some(s => s.device === device);
|
|
291
|
+
}
|
|
292
|
+
/** Show `listening` while the satellite waits for an answer, and fall back to `idle` if none comes. */
|
|
293
|
+
markListening(sat, playMs) {
|
|
294
|
+
this.setStatus(sat, 'listening');
|
|
295
|
+
sat.listenTimer = setTimeout(() => {
|
|
296
|
+
sat.listenTimer = undefined;
|
|
297
|
+
this.setStatus(sat, 'idle');
|
|
298
|
+
}, playMs + LISTEN_WINDOW_MS);
|
|
299
|
+
}
|
|
300
|
+
clearListenTimer(sat) {
|
|
301
|
+
if (sat.listenTimer) {
|
|
302
|
+
clearTimeout(sat.listenTimer);
|
|
303
|
+
sat.listenTimer = undefined;
|
|
304
|
+
}
|
|
305
|
+
}
|
|
234
306
|
/** Device names of the currently registered satellites. */
|
|
235
307
|
devices() {
|
|
236
308
|
return [...this.sats.values()].map(s => s.device);
|
|
237
309
|
}
|
|
238
310
|
setStatus(sat, state) {
|
|
311
|
+
this.clearListenTimer(sat);
|
|
239
312
|
void this.send((0, protocol_1.encodeControl)({ type: 'status', state }), sat);
|
|
240
313
|
this.opts.onStatus?.(sat.device, sat.room, state);
|
|
241
314
|
}
|
|
@@ -253,6 +326,7 @@ class VoiceServer {
|
|
|
253
326
|
const now = Date.now();
|
|
254
327
|
for (const [key, sat] of this.sats) {
|
|
255
328
|
if (now - sat.lastSeen > STALE_MS) {
|
|
329
|
+
this.clearListenTimer(sat);
|
|
256
330
|
this.sats.delete(key);
|
|
257
331
|
this.opts.log.info(`Satellite offline (timeout): ${sat.device}`);
|
|
258
332
|
this.opts.onStatus?.(sat.device, sat.room, 'offline');
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"voiceServer.js","sourceRoot":"","sources":["../../../src/lib/voice/voiceServer.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;;;;;;GAMG;AACH,kDAAoC;AACpC,yCASoB;AA+BpB,+FAA+F;AAC/F,MAAM,eAAe,GAAG,IAAI,CAAC;AAC7B,+FAA+F;AAC/F,MAAM,QAAQ,GAAG,MAAM,CAAC;AAExB,MAAa,WAAW;IAMS;IALrB,IAAI,GAAwB,IAAI,CAAC;IACzC,6FAA6F;IAC5E,IAAI,GAAG,IAAI,GAAG,EAAqB,CAAC;IAC7C,UAAU,GAA0B,IAAI,CAAC;IAEjD,YAA6B,IAAwB;QAAxB,SAAI,GAAJ,IAAI,CAAoB;IAAG,CAAC;IAEzD,KAAK;QACD,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACnC,MAAM,IAAI,GAAG,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;YACxC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;YACjB,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;YACjE,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YAC5E,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YAC3B,sFAAsF;YACtF,MAAM,IAAI,GACN,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,IAAI,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC;YACrG,MAAM,OAAO,GAAG,GAAS,EAAE;gBACvB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,iCAAiC,IAAI,IAAI,SAAS,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC3F,IAAI,CAAC,UAAU,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,MAAM,CAAC,CAAC;gBAC/D,OAAO,EAAE,CAAC;YACd,CAAC,CAAC;YACF,IAAI,IAAI,EAAE,CAAC;gBACP,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;YAC7C,CAAC;iBAAM,CAAC;gBACJ,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YACvC,CAAC;QACL,CAAC,CAAC,CAAC;IACP,CAAC;IAED,KAAK,CAAC,IAAI;QACN,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YAClB,aAAa,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC/B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QAC3B,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;QACvB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;QAClB,IAAI,IAAI,EAAE,CAAC;YACP,MAAM,IAAI,OAAO,CAAO,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QAC5D,CAAC;IACL,CAAC;IAEO,MAAM,CAAC,OAAO,CAAC,OAAe,EAAE,IAAY;QAChD,OAAO,GAAG,OAAO,IAAI,IAAI,EAAE,CAAC;IAChC,CAAC;IAEO,SAAS,CAAC,IAAY,EAAE,KAAuB;QACnD,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACf,OAAO;QACX,CAAC;QACD,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,IAAA,uBAAY,EAAC,IAAI,CAAC,CAAC;QAC7C,IAAI,IAAI,KAAK,qBAAU,EAAE,CAAC;YACtB,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;YAC1E,IAAI,GAAG,EAAE,CAAC;gBACN,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBAC1B,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,oDAAoD;YAC/F,CAAC;YACD,OAAO;QACX,CAAC;QACD,IAAI,IAAI,KAAK,uBAAY,EAAE,CAAC;YACxB,IAAI,GAAgB,CAAC;YACrB,IAAI,CAAC;gBACD,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAgB,CAAC;YAC9D,CAAC;YAAC,MAAM,CAAC;gBACL,OAAO;YACX,CAAC;YACD,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,wBAAyB,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QAC9G,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,SAAS,CAAC,GAAgB,EAAE,KAAuB;QAC7D,MAAM,GAAG,GAAG,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QAE3D,IAAI,GAAG,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YAC1B,MAAM,GAAG,GAAc;gBACnB,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,GAAG;gBACzB,IAAI,EAAE,GAAG,CAAC,IAAI,IAAI,EAAE;gBACpB,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE;gBACpB,MAAM,EAAE,EAAE;aACb,CAAC;YACF,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;YACxB,KAAK,IAAI,CAAC,IAAI,CAAC,IAAA,wBAAa,EAAC,EAAE,IAAI,EAAE,YAAY,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,CAAC;YACrE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,yBAAyB,GAAG,CAAC,MAAM,WAAW,GAAG,CAAC,IAAI,IAAI,GAAG,OAAO,GAAG,EAAE,CAAC,CAAC;YAC9F,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YAC5B,OAAO;QACX,CAAC;QAED,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC/B,IAAI,CAAC,GAAG,EAAE,CAAC;YACP,6EAA6E;YAC7E,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAA,wBAAa,EAAC,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;YAClF,OAAO;QACX,CAAC;QACD,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE1B,IAAI,GAAG,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YAC3B,KAAK,IAAI,CAAC,IAAI,CAAC,IAAA,wBAAa,EAAC,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC,EAAE,GAAG,CAAC,CAAC;YAC9D,OAAO;QACX,CAAC;QACD,IAAI,GAAG,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YAC3B,MAAM,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;QACpC,CAAC;IACL,CAAC;IAED,6FAA6F;IACrF,KAAK,CAAC,eAAe,CAAC,GAAc;QACxC,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACtC,GAAG,CAAC,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC;YACd,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YAC5B,OAAO;QACX,CAAC;QAED,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;QAClC,IAAI,CAAC;YACD,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;YAC1E,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC5B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,EAAE,4BAAiB,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;YAC/F,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,CAAC;YACpC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,GAAG,CAAC,MAAM,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC,CAAC;YACpE,IAAI,CAAC,IAAI,EAAE,CAAC;gBACR,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;gBAC5B,OAAO;YACX,CAAC;YAED,MAAM,MAAM,GAAG,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YACpG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,GAAG,CAAC,MAAM,MAAM,MAAM,EAAE,CAAC,CAAC;YACzD,IAAI,CAAC,MAAM,EAAE,CAAC;gBACV,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;gBAC5B,OAAO;YACX,CAAC;YAED,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;YAChC,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC/F,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;QAClD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,0BAA0B,GAAG,CAAC,MAAM,MAAO,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;QAC1F,CAAC;gBAAS,CAAC;YACP,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QAChC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,SAAS,CAAC,GAAc,EAAE,GAAW,EAAE,UAAkB;QACnE,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,GAAG,GAAG,GAAG,CAAC,MAAM,EAAE,GAAG,IAAI,eAAe,EAAE,CAAC,EAAE,EAAE,CAAC;YACrE,MAAM,IAAI,CAAC,IAAI,CAAC,IAAA,oBAAS,EAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,eAAe,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YAC1E,kGAAkG;YAClG,IAAI,CAAC,CAAC,GAAG,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC;gBAClB,MAAM,IAAI,OAAO,CAAO,GAAG,CAAC,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC;YACtD,CAAC;QACL,CAAC;QACD,MAAM,IAAI,CAAC,IAAI,CAAC,IAAA,wBAAa,EAAC,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,UAAU,EAAE,CAAC,EAAE,GAAG,CAAC,CAAC;IACtF,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ,CAAC,MAAqB,EAAE,GAAW,EAAE,UAAkB;QACjE,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC;YACd,OAAO;QACX,CAAC;QACD,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC;QACpF,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;YAClB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,gBAAgB,MAAM,CAAC,CAAC,CAAC,cAAc,MAAM,GAAG,CAAC,CAAC,CAAC,YAAY,aAAa,CAAC,CAAC;YACjG,OAAO;QACX,CAAC;QACD,MAAM,OAAO,CAAC,GAAG,CACb,OAAO,CAAC,GAAG,CAAC,KAAK,EAAC,GAAG,EAAC,EAAE;YACpB,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;YAChC,IAAI,CAAC;gBACD,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,GAAG,EAAE,UAAU,CAAC,CAAC;YAC/C,CAAC;oBAAS,CAAC;gBACP,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YAChC,CAAC;QACL,CAAC,CAAC,CACL,CAAC;IACN,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,MAAqB;QACxB,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC;QACpF,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;YACxB,KAAK,IAAI,CAAC,IAAI,CAAC,IAAA,wBAAa,EAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,EAAE,GAAG,CAAC,CAAC;YACvD,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;QACrC,CAAC;QACD,OAAO,OAAO,CAAC,MAAM,CAAC;IAC1B,CAAC;IAED,2DAA2D;IAC3D,OAAO;QACH,OAAO,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IACtD,CAAC;IAEO,SAAS,CAAC,GAAc,EAAE,KAAqB;QACnD,KAAK,IAAI,CAAC,IAAI,CAAC,IAAA,wBAAa,EAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,CAAC,CAAC;QAC9D,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACtD,CAAC;IAEO,IAAI,CAAC,IAAY,EAAE,GAAc;QACrC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACnC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;YACvB,IAAI,CAAC,IAAI,EAAE,CAAC;gBACR,OAAO,EAAE,CAAC;gBACV,OAAO;YACX,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QACnF,CAAC,CAAC,CAAC;IACP,CAAC;IAEO,UAAU;QACd,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACjC,IAAI,GAAG,GAAG,GAAG,CAAC,QAAQ,GAAG,QAAQ,EAAE,CAAC;gBAChC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBACtB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,gCAAgC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;gBACjE,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;YAC1D,CAAC;QACL,CAAC;IACL,CAAC;CACJ;AAnOD,kCAmOC","sourcesContent":["/**\n * UDP voice server — the \"brain\" side of the satellite protocol.\n *\n * Receives streamed microphone audio from satellites, runs STT → answer callback → TTS,\n * and streams the spoken reply back. Registration + heartbeat keep the satellite registry\n * live. Protocol is Hannah-compatible (see `protocol.ts`).\n */\nimport * as dgram from 'node:dgram';\nimport {\n AUDIO_SAMPLE_RATE,\n TYPE_AUDIO,\n TYPE_CONTROL,\n decodePacket,\n encodeControl,\n encodeTts,\n type SatToServer,\n type SatelliteState,\n} from './protocol';\nimport type { SttEngine } from './stt';\nimport type { TtsEngine } from './tts';\n\ninterface Satellite {\n device: string;\n room: string;\n address: string;\n port: number;\n lastSeen: number;\n /** Buffered audio chunks of the utterance currently being spoken. */\n chunks: Buffer[];\n}\n\nexport interface VoiceServerOptions {\n port: number;\n /** Interface to bind to ('0.0.0.0'/'' = all interfaces, or a specific IP). */\n bindAddress?: string;\n /** ISO-639-1 language hint for STT/TTS ('' = auto/provider default). */\n language: string;\n stt: SttEngine;\n tts: TtsEngine;\n /** Produce the assistant reply for a transcribed utterance. */\n answer: (question: string, ctx: { device: string; room: string; sttMs: number }) => Promise<string>;\n /** Optional STT vocabulary hints (device/room names) to bias recognition; re-read per utterance. */\n getHints?: () => string[] | Promise<string[]>;\n log: ioBroker.Logger;\n /** Notified on every satellite state transition (idle/listening/processing/speaking/offline). */\n onStatus?: (device: string, room: string, state: SatelliteState | 'offline') => void;\n}\n\n/** Payload bytes per TTS datagram — well under the typical MTU-fragmentation-free UDP size. */\nconst TTS_CHUNK_BYTES = 8192;\n/** Drop a satellite after this long without any packet (3× the satellite's 10 s heartbeat). */\nconst STALE_MS = 30_000;\n\nexport class VoiceServer {\n private sock: dgram.Socket | null = null;\n /** Keyed by `${address}:${port}` — audio packets carry no device name, only their source. */\n private readonly sats = new Map<string, Satellite>();\n private sweepTimer: NodeJS.Timeout | null = null;\n\n constructor(private readonly opts: VoiceServerOptions) {}\n\n start(): Promise<void> {\n return new Promise((resolve, reject) => {\n const sock = dgram.createSocket('udp4');\n this.sock = sock;\n sock.on('message', (data, rinfo) => this.onMessage(data, rinfo));\n sock.on('error', e => this.opts.log.error(`voice UDP error: ${e.message}`));\n sock.once('error', reject);\n // '0.0.0.0'/'' → bind all interfaces (omit the address); otherwise bind the given IP.\n const addr =\n this.opts.bindAddress && this.opts.bindAddress !== '0.0.0.0' ? this.opts.bindAddress : undefined;\n const onBound = (): void => {\n this.opts.log.info(`Voice server listening on UDP ${addr || '0.0.0.0'}:${this.opts.port}`);\n this.sweepTimer = setInterval(() => this.sweepStale(), 15_000);\n resolve();\n };\n if (addr) {\n sock.bind(this.opts.port, addr, onBound);\n } else {\n sock.bind(this.opts.port, onBound);\n }\n });\n }\n\n async stop(): Promise<void> {\n if (this.sweepTimer) {\n clearInterval(this.sweepTimer);\n this.sweepTimer = null;\n }\n const sock = this.sock;\n this.sock = null;\n this.sats.clear();\n if (sock) {\n await new Promise<void>(res => sock.close(() => res()));\n }\n }\n\n private static addrKey(address: string, port: number): string {\n return `${address}:${port}`;\n }\n\n private onMessage(data: Buffer, rinfo: dgram.RemoteInfo): void {\n if (!data.length) {\n return;\n }\n const { type, payload } = decodePacket(data);\n if (type === TYPE_AUDIO) {\n const sat = this.sats.get(VoiceServer.addrKey(rinfo.address, rinfo.port));\n if (sat) {\n sat.lastSeen = Date.now();\n sat.chunks.push(Buffer.from(payload)); // copy: payload is a view onto a reused recv buffer\n }\n return;\n }\n if (type === TYPE_CONTROL) {\n let msg: SatToServer;\n try {\n msg = JSON.parse(payload.toString('utf8')) as SatToServer;\n } catch {\n return;\n }\n this.onControl(msg, rinfo).catch(e => this.opts.log.warn(`voice control error: ${(e as Error).message}`));\n }\n }\n\n private async onControl(msg: SatToServer, rinfo: dgram.RemoteInfo): Promise<void> {\n const key = VoiceServer.addrKey(rinfo.address, rinfo.port);\n\n if (msg.type === 'register') {\n const sat: Satellite = {\n device: msg.device || key,\n room: msg.room || '',\n address: rinfo.address,\n port: rinfo.port,\n lastSeen: Date.now(),\n chunks: [],\n };\n this.sats.set(key, sat);\n void this.send(encodeControl({ type: 'registered', ok: true }), sat);\n this.opts.log.info(`Satellite registered: ${sat.device} (room: ${sat.room || '-'}) @ ${key}`);\n this.setStatus(sat, 'idle');\n return;\n }\n\n const sat = this.sats.get(key);\n if (!sat) {\n // Unknown source (server restarted?) — tell the satellite to register again.\n this.sock?.send(encodeControl({ type: 'reregister' }), rinfo.port, rinfo.address);\n return;\n }\n sat.lastSeen = Date.now();\n\n if (msg.type === 'heartbeat') {\n void this.send(encodeControl({ type: 'heartbeat_ack' }), sat);\n return;\n }\n if (msg.type === 'audio_end') {\n await this.handleUtterance(sat);\n }\n }\n\n /** STT → answer → TTS for one completed utterance, with status transitions along the way. */\n private async handleUtterance(sat: Satellite): Promise<void> {\n const pcm = Buffer.concat(sat.chunks);\n sat.chunks = [];\n if (!pcm.length) {\n this.setStatus(sat, 'idle');\n return;\n }\n\n this.setStatus(sat, 'processing');\n try {\n const hints = this.opts.getHints ? await this.opts.getHints() : undefined;\n const sttStart = Date.now();\n const text = await this.opts.stt.transcribe(pcm, AUDIO_SAMPLE_RATE, this.opts.language, hints);\n const sttMs = Date.now() - sttStart;\n this.opts.log.info(`Voice Q (${sat.device}): ${text || '(empty)'}`);\n if (!text) {\n this.setStatus(sat, 'idle');\n return;\n }\n\n const answer = (await this.opts.answer(text, { device: sat.device, room: sat.room, sttMs })).trim();\n this.opts.log.info(`Voice A (${sat.device}): ${answer}`);\n if (!answer) {\n this.setStatus(sat, 'idle');\n return;\n }\n\n this.setStatus(sat, 'speaking');\n const { pcm: ttsPcm, sampleRate } = await this.opts.tts.synthesize(answer, this.opts.language);\n await this.streamTts(sat, ttsPcm, sampleRate);\n } catch (e) {\n this.opts.log.error(`Voice pipeline failed (${sat.device}): ${(e as Error).message}`);\n } finally {\n this.setStatus(sat, 'idle');\n }\n }\n\n private async streamTts(sat: Satellite, pcm: Buffer, sampleRate: number): Promise<void> {\n for (let off = 0, i = 0; off < pcm.length; off += TTS_CHUNK_BYTES, i++) {\n await this.send(encodeTts(pcm.subarray(off, off + TTS_CHUNK_BYTES)), sat);\n // Yield every 16 packets so a long reply doesn't burst the OS send buffer / the satellite's recv.\n if ((i & 15) === 15) {\n await new Promise<void>(res => setImmediate(res));\n }\n }\n await this.send(encodeControl({ type: 'tts_end', sample_rate: sampleRate }), sat);\n }\n\n /**\n * Play arbitrary PCM on one satellite (by device name) or all registered satellites (device=null).\n * Used for announcements (`tts.text` / per-satellite `tts`), independent of the wake-word flow.\n */\n async announce(device: string | null, pcm: Buffer, sampleRate: number): Promise<void> {\n if (!pcm.length) {\n return;\n }\n const targets = [...this.sats.values()].filter(s => !device || s.device === device);\n if (!targets.length) {\n this.opts.log.warn(`announce: no ${device ? `satellite '${device}'` : 'satellites'} registered`);\n return;\n }\n await Promise.all(\n targets.map(async sat => {\n this.setStatus(sat, 'speaking');\n try {\n await this.streamTts(sat, pcm, sampleRate);\n } finally {\n this.setStatus(sat, 'idle');\n }\n }),\n );\n }\n\n /**\n * Re-open the microphone on one satellite (by device name) or all of them, so the user can answer a\n * question the assistant just asked without saying the wake word again. Returns how many satellites\n * were told to listen — whether they can is up to their firmware (the message is additive).\n */\n listen(device: string | null): number {\n const targets = [...this.sats.values()].filter(s => !device || s.device === device);\n for (const sat of targets) {\n void this.send(encodeControl({ type: 'listen' }), sat);\n this.setStatus(sat, 'listening');\n }\n return targets.length;\n }\n\n /** Device names of the currently registered satellites. */\n devices(): string[] {\n return [...this.sats.values()].map(s => s.device);\n }\n\n private setStatus(sat: Satellite, state: SatelliteState): void {\n void this.send(encodeControl({ type: 'status', state }), sat);\n this.opts.onStatus?.(sat.device, sat.room, state);\n }\n\n private send(data: Buffer, sat: Satellite): Promise<void> {\n return new Promise((resolve, reject) => {\n const sock = this.sock;\n if (!sock) {\n resolve();\n return;\n }\n sock.send(data, sat.port, sat.address, err => (err ? reject(err) : resolve()));\n });\n }\n\n private sweepStale(): void {\n const now = Date.now();\n for (const [key, sat] of this.sats) {\n if (now - sat.lastSeen > STALE_MS) {\n this.sats.delete(key);\n this.opts.log.info(`Satellite offline (timeout): ${sat.device}`);\n this.opts.onStatus?.(sat.device, sat.room, 'offline');\n }\n }\n }\n}\n"]}
|
|
1
|
+
{"version":3,"file":"voiceServer.js","sourceRoot":"","sources":["../../../src/lib/voice/voiceServer.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;;;;;;GAMG;AACH,kDAAoC;AACpC,yCASoB;AA0CpB,+FAA+F;AAC/F,MAAM,eAAe,GAAG,IAAI,CAAC;AAC7B,+FAA+F;AAC/F,MAAM,QAAQ,GAAG,MAAM,CAAC;AACxB;;;GAGG;AACH,MAAM,gBAAgB,GAAG,MAAM,CAAC;AAEhC,MAAa,WAAW;IAMS;IALrB,IAAI,GAAwB,IAAI,CAAC;IACzC,6FAA6F;IAC5E,IAAI,GAAG,IAAI,GAAG,EAAqB,CAAC;IAC7C,UAAU,GAA0B,IAAI,CAAC;IAEjD,YAA6B,IAAwB;QAAxB,SAAI,GAAJ,IAAI,CAAoB;IAAG,CAAC;IAEzD,KAAK;QACD,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACnC,MAAM,IAAI,GAAG,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;YACxC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;YACjB,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;YACjE,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YAC5E,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YAC3B,sFAAsF;YACtF,MAAM,IAAI,GACN,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,IAAI,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC;YACrG,MAAM,OAAO,GAAG,GAAS,EAAE;gBACvB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,iCAAiC,IAAI,IAAI,SAAS,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC3F,IAAI,CAAC,UAAU,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,MAAM,CAAC,CAAC;gBAC/D,OAAO,EAAE,CAAC;YACd,CAAC,CAAC;YACF,IAAI,IAAI,EAAE,CAAC;gBACP,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;YAC7C,CAAC;iBAAM,CAAC;gBACJ,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YACvC,CAAC;QACL,CAAC,CAAC,CAAC;IACP,CAAC;IAED,KAAK,CAAC,IAAI;QACN,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YAClB,aAAa,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC/B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QAC3B,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;QACvB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC;QACrD,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;QAClB,IAAI,IAAI,EAAE,CAAC;YACP,MAAM,IAAI,OAAO,CAAO,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QAC5D,CAAC;IACL,CAAC;IAEO,MAAM,CAAC,OAAO,CAAC,OAAe,EAAE,IAAY;QAChD,OAAO,GAAG,OAAO,IAAI,IAAI,EAAE,CAAC;IAChC,CAAC;IAEO,SAAS,CAAC,IAAY,EAAE,KAAuB;QACnD,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACf,OAAO;QACX,CAAC;QACD,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,IAAA,uBAAY,EAAC,IAAI,CAAC,CAAC;QAC7C,IAAI,IAAI,KAAK,qBAAU,EAAE,CAAC;YACtB,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;YAC1E,IAAI,GAAG,EAAE,CAAC;gBACN,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBAC1B,yFAAyF;gBACzF,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;gBAC3B,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,oDAAoD;YAC/F,CAAC;YACD,OAAO;QACX,CAAC;QACD,IAAI,IAAI,KAAK,uBAAY,EAAE,CAAC;YACxB,IAAI,GAAgB,CAAC;YACrB,IAAI,CAAC;gBACD,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAgB,CAAC;YAC9D,CAAC;YAAC,MAAM,CAAC;gBACL,OAAO;YACX,CAAC;YACD,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,wBAAyB,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QAC9G,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,SAAS,CAAC,GAAgB,EAAE,KAAuB;QAC7D,MAAM,GAAG,GAAG,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QAE3D,IAAI,GAAG,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YAC1B,MAAM,GAAG,GAAc;gBACnB,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,GAAG;gBACzB,IAAI,EAAE,GAAG,CAAC,IAAI,IAAI,EAAE;gBACpB,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE;gBACpB,MAAM,EAAE,EAAE;aACb,CAAC;YACF,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACpC,IAAI,QAAQ,EAAE,CAAC;gBACX,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;YACpC,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;YACxB,KAAK,IAAI,CAAC,IAAI,CAAC,IAAA,wBAAa,EAAC,EAAE,IAAI,EAAE,YAAY,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,CAAC;YACrE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,yBAAyB,GAAG,CAAC,MAAM,WAAW,GAAG,CAAC,IAAI,IAAI,GAAG,OAAO,GAAG,EAAE,CAAC,CAAC;YAC9F,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YAC5B,OAAO;QACX,CAAC;QAED,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC/B,IAAI,CAAC,GAAG,EAAE,CAAC;YACP,6EAA6E;YAC7E,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAA,wBAAa,EAAC,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;YAClF,OAAO;QACX,CAAC;QACD,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE1B,IAAI,GAAG,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YAC3B,KAAK,IAAI,CAAC,IAAI,CAAC,IAAA,wBAAa,EAAC,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC,EAAE,GAAG,CAAC,CAAC;YAC9D,OAAO;QACX,CAAC;QACD,IAAI,GAAG,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YAC3B,MAAM,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;YAChC,OAAO;QACX,CAAC;QACD,IAAI,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACxB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;YAC5C,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;gBACzB,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;YAClF,CAAC;QACL,CAAC;IACL,CAAC;IAED,6FAA6F;IACrF,KAAK,CAAC,eAAe,CAAC,GAAc;QACxC,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACtC,GAAG,CAAC,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC;YACd,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YAC5B,OAAO;QACX,CAAC;QAED,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;QAClC,IAAI,MAAM,GAAG,KAAK,CAAC;QACnB,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,CAAC;YACD,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;YAC1E,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC5B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,EAAE,4BAAiB,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;YAC/F,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,CAAC;YACpC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,GAAG,CAAC,MAAM,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC,CAAC;YACpE,IAAI,CAAC,IAAI,EAAE,CAAC;gBACR,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;gBAC5B,OAAO;YACX,CAAC;YAED,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,GAAG,4BAAiB,CAAC,GAAG,IAAI,CAAC,CAAC;YACxE,MAAM,MAAM,GAAG,CACX,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CACvF,CAAC,IAAI,EAAE,CAAC;YACT,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,GAAG,CAAC,MAAM,MAAM,MAAM,EAAE,CAAC,CAAC;YACzD,IAAI,CAAC,MAAM,EAAE,CAAC;gBACV,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;gBAC5B,OAAO;YACX,CAAC;YAED,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;YAChC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC5B,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC/F,MAAM,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,GAAG,UAAU,CAAC,GAAG,IAAI,CAAC;YACjD,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,GAAG,CAAC,MAAM,EAAE,EAAE,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;YACjF,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC;YAClE,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;QAC1D,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,0BAA0B,GAAG,CAAC,MAAM,MAAO,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;YACtF,MAAM,GAAG,KAAK,CAAC;QACnB,CAAC;gBAAS,CAAC;YACP,IAAI,MAAM,EAAE,CAAC;gBACT,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YACpC,CAAC;iBAAM,CAAC;gBACJ,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YAChC,CAAC;QACL,CAAC;IACL,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,SAAS,CAAC,GAAc,EAAE,GAAW,EAAE,UAAkB,EAAE,MAAM,GAAG,KAAK;QACnF,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,GAAG,GAAG,GAAG,CAAC,MAAM,EAAE,GAAG,IAAI,eAAe,EAAE,CAAC,EAAE,EAAE,CAAC;YACrE,MAAM,IAAI,CAAC,IAAI,CAAC,IAAA,oBAAS,EAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,eAAe,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YAC1E,kGAAkG;YAClG,IAAI,CAAC,CAAC,GAAG,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC;gBAClB,MAAM,IAAI,OAAO,CAAO,GAAG,CAAC,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC;YACtD,CAAC;QACL,CAAC;QACD,MAAM,IAAI,CAAC,IAAI,CACX,IAAA,wBAAa,EACT,MAAM;YACF,CAAC,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,EAAE;YACtD,CAAC,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,UAAU,EAAE,CACrD,EACD,GAAG,CACN,CAAC;IACN,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ,CAAC,MAAqB,EAAE,GAAW,EAAE,UAAkB;QACjE,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC;YACd,OAAO;QACX,CAAC;QACD,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC;QACpF,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;YAClB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,gBAAgB,MAAM,CAAC,CAAC,CAAC,cAAc,MAAM,GAAG,CAAC,CAAC,CAAC,YAAY,aAAa,CAAC,CAAC;YACjG,OAAO;QACX,CAAC;QACD,MAAM,OAAO,CAAC,GAAG,CACb,OAAO,CAAC,GAAG,CAAC,KAAK,EAAC,GAAG,EAAC,EAAE;YACpB,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;YAChC,IAAI,CAAC;gBACD,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,GAAG,EAAE,UAAU,CAAC,CAAC;YAC/C,CAAC;oBAAS,CAAC;gBACP,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YAChC,CAAC;QACL,CAAC,CAAC,CACL,CAAC;IACN,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,MAAqB,EAAE,MAAM,GAAG,CAAC;QACpC,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC;QACpF,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;YACxB,KAAK,IAAI,CAAC,IAAI,CAAC,IAAA,wBAAa,EAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,EAAE,GAAG,CAAC,CAAC;YACvD,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QACpC,CAAC;QACD,OAAO,OAAO,CAAC,MAAM,CAAC;IAC1B,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,MAAc,EAAE,MAA0C;QAC7D,MAAM,GAAG,GAAG,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC;QACnE,IAAI,CAAC,GAAG,EAAE,CAAC;YACP,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,MAAM,GAAG,GACL,MAAM,CAAC,KAAK,KAAK,SAAS;YACtB,CAAC,CAAC,EAAE,IAAI,EAAE,QAAiB,EAAE,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;YAC1F,CAAC,CAAC,EAAE,IAAI,EAAE,QAAiB,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC,CAAC,EAAE,CAAC;QAC5E,KAAK,IAAI,CAAC,IAAI,CAAC,IAAA,wBAAa,EAAC,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC;QACxC,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,+DAA+D;IAC/D,GAAG,CAAC,MAAc;QACd,OAAO,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC;IAClE,CAAC;IAED,uGAAuG;IAC/F,aAAa,CAAC,GAAc,EAAE,MAAc;QAChD,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;QACjC,GAAG,CAAC,WAAW,GAAG,UAAU,CAAC,GAAG,EAAE;YAC9B,GAAG,CAAC,WAAW,GAAG,SAAS,CAAC;YAC5B,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QAChC,CAAC,EAAE,MAAM,GAAG,gBAAgB,CAAC,CAAC;IAClC,CAAC;IAEO,gBAAgB,CAAC,GAAc;QACnC,IAAI,GAAG,CAAC,WAAW,EAAE,CAAC;YAClB,YAAY,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;YAC9B,GAAG,CAAC,WAAW,GAAG,SAAS,CAAC;QAChC,CAAC;IACL,CAAC;IAED,2DAA2D;IAC3D,OAAO;QACH,OAAO,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IACtD,CAAC;IAEO,SAAS,CAAC,GAAc,EAAE,KAAqB;QACnD,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;QAC3B,KAAK,IAAI,CAAC,IAAI,CAAC,IAAA,wBAAa,EAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,CAAC,CAAC;QAC9D,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACtD,CAAC;IAEO,IAAI,CAAC,IAAY,EAAE,GAAc;QACrC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACnC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;YACvB,IAAI,CAAC,IAAI,EAAE,CAAC;gBACR,OAAO,EAAE,CAAC;gBACV,OAAO;YACX,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QACnF,CAAC,CAAC,CAAC;IACP,CAAC;IAEO,UAAU;QACd,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACjC,IAAI,GAAG,GAAG,GAAG,CAAC,QAAQ,GAAG,QAAQ,EAAE,CAAC;gBAChC,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;gBAC3B,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBACtB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,gCAAgC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;gBACjE,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;YAC1D,CAAC;QACL,CAAC;IACL,CAAC;CACJ;AAnTD,kCAmTC","sourcesContent":["/**\n * UDP voice server — the \"brain\" side of the satellite protocol.\n *\n * Receives streamed microphone audio from satellites, runs STT → answer callback → TTS,\n * and streams the spoken reply back. Registration + heartbeat keep the satellite registry\n * live. Protocol is Hannah-compatible (see `protocol.ts`).\n */\nimport * as dgram from 'node:dgram';\nimport {\n AUDIO_SAMPLE_RATE,\n TYPE_AUDIO,\n TYPE_CONTROL,\n decodePacket,\n encodeControl,\n encodeTts,\n type SatToServer,\n type SatelliteState,\n} from './protocol';\nimport type { SttEngine } from './stt';\nimport type { TtsEngine } from './tts';\n\ninterface Satellite {\n device: string;\n room: string;\n address: string;\n port: number;\n lastSeen: number;\n /** Buffered audio chunks of the utterance currently being spoken. */\n chunks: Buffer[];\n /** Puts the satellite back to `idle` once a mic-on window passed unanswered. */\n listenTimer?: NodeJS.Timeout;\n}\n\nexport interface VoiceServerOptions {\n port: number;\n /** Interface to bind to ('0.0.0.0'/'' = all interfaces, or a specific IP). */\n bindAddress?: string;\n /** ISO-639-1 language hint for STT/TTS ('' = auto/provider default). */\n language: string;\n stt: SttEngine;\n tts: TtsEngine;\n /** Produce the assistant reply for a transcribed utterance (`audioMs` = length of the recording). */\n answer: (\n question: string,\n ctx: { device: string; room: string; sttMs: number; audioMs: number },\n ) => Promise<string>;\n /** The reply was synthesised: how long that took and how long it plays (for the request journal). */\n onSpoken?: (device: string, info: { ms: number; audioMs: number }) => void;\n /** Optional STT vocabulary hints (device/room names) to bias recognition; re-read per utterance. */\n getHints?: () => string[] | Promise<string[]>;\n log: ioBroker.Logger;\n /** Notified on every satellite state transition (idle/listening/processing/speaking/offline). */\n onStatus?: (device: string, room: string, state: SatelliteState | 'offline') => void;\n /** True when the reply asks something back, so the satellite should re-open its mic without a wake word. */\n expectsFollowUp?: (answer: string, device: string) => boolean;\n /** The playback volume (0–100) a satellite reported, after a `volume` command or a local change. */\n onVolume?: (device: string, room: string, level: number) => void;\n}\n\n/** Payload bytes per TTS datagram — well under the typical MTU-fragmentation-free UDP size. */\nconst TTS_CHUNK_BYTES = 8192;\n/** Drop a satellite after this long without any packet (3× the satellite's 10 s heartbeat). */\nconst STALE_MS = 30_000;\n/**\n * How long a satellite stays `listening` after a mic-on once its playback is over. A little longer than the\n * satellite's own follow-up window (6 s by default), which we cannot see from here.\n */\nconst LISTEN_WINDOW_MS = 10_000;\n\nexport class VoiceServer {\n private sock: dgram.Socket | null = null;\n /** Keyed by `${address}:${port}` — audio packets carry no device name, only their source. */\n private readonly sats = new Map<string, Satellite>();\n private sweepTimer: NodeJS.Timeout | null = null;\n\n constructor(private readonly opts: VoiceServerOptions) {}\n\n start(): Promise<void> {\n return new Promise((resolve, reject) => {\n const sock = dgram.createSocket('udp4');\n this.sock = sock;\n sock.on('message', (data, rinfo) => this.onMessage(data, rinfo));\n sock.on('error', e => this.opts.log.error(`voice UDP error: ${e.message}`));\n sock.once('error', reject);\n // '0.0.0.0'/'' → bind all interfaces (omit the address); otherwise bind the given IP.\n const addr =\n this.opts.bindAddress && this.opts.bindAddress !== '0.0.0.0' ? this.opts.bindAddress : undefined;\n const onBound = (): void => {\n this.opts.log.info(`Voice server listening on UDP ${addr || '0.0.0.0'}:${this.opts.port}`);\n this.sweepTimer = setInterval(() => this.sweepStale(), 15_000);\n resolve();\n };\n if (addr) {\n sock.bind(this.opts.port, addr, onBound);\n } else {\n sock.bind(this.opts.port, onBound);\n }\n });\n }\n\n async stop(): Promise<void> {\n if (this.sweepTimer) {\n clearInterval(this.sweepTimer);\n this.sweepTimer = null;\n }\n const sock = this.sock;\n this.sock = null;\n this.sats.forEach(sat => this.clearListenTimer(sat));\n this.sats.clear();\n if (sock) {\n await new Promise<void>(res => sock.close(() => res()));\n }\n }\n\n private static addrKey(address: string, port: number): string {\n return `${address}:${port}`;\n }\n\n private onMessage(data: Buffer, rinfo: dgram.RemoteInfo): void {\n if (!data.length) {\n return;\n }\n const { type, payload } = decodePacket(data);\n if (type === TYPE_AUDIO) {\n const sat = this.sats.get(VoiceServer.addrKey(rinfo.address, rinfo.port));\n if (sat) {\n sat.lastSeen = Date.now();\n // The user answered within the mic-on window — don't let the timer flip the LED to idle.\n this.clearListenTimer(sat);\n sat.chunks.push(Buffer.from(payload)); // copy: payload is a view onto a reused recv buffer\n }\n return;\n }\n if (type === TYPE_CONTROL) {\n let msg: SatToServer;\n try {\n msg = JSON.parse(payload.toString('utf8')) as SatToServer;\n } catch {\n return;\n }\n this.onControl(msg, rinfo).catch(e => this.opts.log.warn(`voice control error: ${(e as Error).message}`));\n }\n }\n\n private async onControl(msg: SatToServer, rinfo: dgram.RemoteInfo): Promise<void> {\n const key = VoiceServer.addrKey(rinfo.address, rinfo.port);\n\n if (msg.type === 'register') {\n const sat: Satellite = {\n device: msg.device || key,\n room: msg.room || '',\n address: rinfo.address,\n port: rinfo.port,\n lastSeen: Date.now(),\n chunks: [],\n };\n const previous = this.sats.get(key);\n if (previous) {\n this.clearListenTimer(previous);\n }\n this.sats.set(key, sat);\n void this.send(encodeControl({ type: 'registered', ok: true }), sat);\n this.opts.log.info(`Satellite registered: ${sat.device} (room: ${sat.room || '-'}) @ ${key}`);\n this.setStatus(sat, 'idle');\n return;\n }\n\n const sat = this.sats.get(key);\n if (!sat) {\n // Unknown source (server restarted?) — tell the satellite to register again.\n this.sock?.send(encodeControl({ type: 'reregister' }), rinfo.port, rinfo.address);\n return;\n }\n sat.lastSeen = Date.now();\n\n if (msg.type === 'heartbeat') {\n void this.send(encodeControl({ type: 'heartbeat_ack' }), sat);\n return;\n }\n if (msg.type === 'audio_end') {\n await this.handleUtterance(sat);\n return;\n }\n if (msg.type === 'volume') {\n const level = Math.round(Number(msg.level));\n if (Number.isFinite(level)) {\n this.opts.onVolume?.(sat.device, sat.room, Math.min(100, Math.max(0, level)));\n }\n }\n }\n\n /** STT → answer → TTS for one completed utterance, with status transitions along the way. */\n private async handleUtterance(sat: Satellite): Promise<void> {\n const pcm = Buffer.concat(sat.chunks);\n sat.chunks = [];\n if (!pcm.length) {\n this.setStatus(sat, 'idle');\n return;\n }\n\n this.setStatus(sat, 'processing');\n let listen = false;\n let playMs = 0;\n try {\n const hints = this.opts.getHints ? await this.opts.getHints() : undefined;\n const sttStart = Date.now();\n const text = await this.opts.stt.transcribe(pcm, AUDIO_SAMPLE_RATE, this.opts.language, hints);\n const sttMs = Date.now() - sttStart;\n this.opts.log.info(`Voice Q (${sat.device}): ${text || '(empty)'}`);\n if (!text) {\n this.setStatus(sat, 'idle');\n return;\n }\n\n const audioMs = Math.round((pcm.length / 2 / AUDIO_SAMPLE_RATE) * 1000);\n const answer = (\n await this.opts.answer(text, { device: sat.device, room: sat.room, sttMs, audioMs })\n ).trim();\n this.opts.log.info(`Voice A (${sat.device}): ${answer}`);\n if (!answer) {\n this.setStatus(sat, 'idle');\n return;\n }\n\n this.setStatus(sat, 'speaking');\n const ttsStart = Date.now();\n const { pcm: ttsPcm, sampleRate } = await this.opts.tts.synthesize(answer, this.opts.language);\n playMs = (ttsPcm.length / 2 / sampleRate) * 1000;\n this.opts.onSpoken?.(sat.device, { ms: Date.now() - ttsStart, audioMs: playMs });\n listen = this.opts.expectsFollowUp?.(answer, sat.device) === true;\n await this.streamTts(sat, ttsPcm, sampleRate, listen);\n } catch (e) {\n this.opts.log.error(`Voice pipeline failed (${sat.device}): ${(e as Error).message}`);\n listen = false;\n } finally {\n if (listen) {\n this.markListening(sat, playMs);\n } else {\n this.setStatus(sat, 'idle');\n }\n }\n }\n\n /**\n * Stream one reply. `listen` travels inside `tts_end` rather than as a message of its own: the satellite\n * then knows before playback starts that it has to open the mic afterwards, and nothing can overtake it.\n */\n private async streamTts(sat: Satellite, pcm: Buffer, sampleRate: number, listen = false): Promise<void> {\n for (let off = 0, i = 0; off < pcm.length; off += TTS_CHUNK_BYTES, i++) {\n await this.send(encodeTts(pcm.subarray(off, off + TTS_CHUNK_BYTES)), sat);\n // Yield every 16 packets so a long reply doesn't burst the OS send buffer / the satellite's recv.\n if ((i & 15) === 15) {\n await new Promise<void>(res => setImmediate(res));\n }\n }\n await this.send(\n encodeControl(\n listen\n ? { type: 'tts_end', sample_rate: sampleRate, listen }\n : { type: 'tts_end', sample_rate: sampleRate },\n ),\n sat,\n );\n }\n\n /**\n * Play arbitrary PCM on one satellite (by device name) or all registered satellites (device=null).\n * Used for announcements (`tts.text` / per-satellite `tts`), independent of the wake-word flow.\n */\n async announce(device: string | null, pcm: Buffer, sampleRate: number): Promise<void> {\n if (!pcm.length) {\n return;\n }\n const targets = [...this.sats.values()].filter(s => !device || s.device === device);\n if (!targets.length) {\n this.opts.log.warn(`announce: no ${device ? `satellite '${device}'` : 'satellites'} registered`);\n return;\n }\n await Promise.all(\n targets.map(async sat => {\n this.setStatus(sat, 'speaking');\n try {\n await this.streamTts(sat, pcm, sampleRate);\n } finally {\n this.setStatus(sat, 'idle');\n }\n }),\n );\n }\n\n /**\n * Re-open the microphone on one satellite (by device name) or all of them, so the user can answer a\n * question the assistant just asked without saying the wake word again. Returns how many satellites\n * were told to listen — whether they can is up to their firmware (the message is additive).\n */\n listen(device: string | null, playMs = 0): number {\n const targets = [...this.sats.values()].filter(s => !device || s.device === device);\n for (const sat of targets) {\n void this.send(encodeControl({ type: 'listen' }), sat);\n this.markListening(sat, playMs);\n }\n return targets.length;\n }\n\n /**\n * Change the playback volume of one satellite: absolute (`level`, 0–100) or relative (`delta`). The\n * satellite answers with the level it ended up at (→ `onVolume`); one built before this message\n * existed ignores it. Returns false when the device is not registered.\n */\n volume(device: string, change: { level?: number; delta?: number }): boolean {\n const sat = [...this.sats.values()].find(s => s.device === device);\n if (!sat) {\n return false;\n }\n const msg =\n change.level !== undefined\n ? { type: 'volume' as const, level: Math.min(100, Math.max(0, Math.round(change.level))) }\n : { type: 'volume' as const, delta: Math.round(change.delta || 0) };\n void this.send(encodeControl(msg), sat);\n return true;\n }\n\n /** Whether a satellite with this device name is registered. */\n has(device: string): boolean {\n return [...this.sats.values()].some(s => s.device === device);\n }\n\n /** Show `listening` while the satellite waits for an answer, and fall back to `idle` if none comes. */\n private markListening(sat: Satellite, playMs: number): void {\n this.setStatus(sat, 'listening');\n sat.listenTimer = setTimeout(() => {\n sat.listenTimer = undefined;\n this.setStatus(sat, 'idle');\n }, playMs + LISTEN_WINDOW_MS);\n }\n\n private clearListenTimer(sat: Satellite): void {\n if (sat.listenTimer) {\n clearTimeout(sat.listenTimer);\n sat.listenTimer = undefined;\n }\n }\n\n /** Device names of the currently registered satellites. */\n devices(): string[] {\n return [...this.sats.values()].map(s => s.device);\n }\n\n private setStatus(sat: Satellite, state: SatelliteState): void {\n this.clearListenTimer(sat);\n void this.send(encodeControl({ type: 'status', state }), sat);\n this.opts.onStatus?.(sat.device, sat.room, state);\n }\n\n private send(data: Buffer, sat: Satellite): Promise<void> {\n return new Promise((resolve, reject) => {\n const sock = this.sock;\n if (!sock) {\n resolve();\n return;\n }\n sock.send(data, sat.port, sat.address, err => (err ? reject(err) : resolve()));\n });\n }\n\n private sweepStale(): void {\n const now = Date.now();\n for (const [key, sat] of this.sats) {\n if (now - sat.lastSeen > STALE_MS) {\n this.clearListenTimer(sat);\n this.sats.delete(key);\n this.opts.log.info(`Satellite offline (timeout): ${sat.device}`);\n this.opts.onStatus?.(sat.device, sat.room, 'offline');\n }\n }\n }\n}\n"]}
|