iobroker.assistant 1.1.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +74 -1
- package/admin/custom/.vite/manifest.json +62 -62
- package/admin/custom/assets/Components-x2Dfv8qE.js +2 -0
- package/admin/custom/assets/{_virtual_mf-localSharedImportMap___mfe_internal__ConfigCustomAssistant__mf_owner__1-A4gakomR.js → _virtual_mf-localSharedImportMap___mfe_internal__ConfigCustomAssistant__mf_owner__1-B4YbqhBb.js} +1 -1
- package/admin/custom/assets/{_virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__201375093093590__loadShare___mf_0_emotion_mf_1_styled__loadShare__.js-CwqmfMvn.js → _virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__163539842885036__loadShare___mf_0_emotion_mf_1_styled__loadShare__.js-IaosaymG.js} +1 -1
- package/admin/custom/assets/{_virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__201375093093590__loadShare___mf_0_iobroker_mf_1_gui_mf_2_components__loadShare__.js-BMuLYe6f.js → _virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__163539842885036__loadShare___mf_0_iobroker_mf_1_gui_mf_2_components__loadShare__.js-o4EU_Ut5.js} +39 -39
- package/admin/custom/assets/{_virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__201375093093590__loadShare__shared-BxjZmnpc.js → _virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__163539842885036__loadShare__shared--be2hWbV.js} +1 -1
- package/admin/custom/assets/_virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__163539842885036__prebuild___mf_0_iobroker_mf_1_json_mf_2_config__prebuild__-BsTz_EWE.js +122 -0
- package/admin/custom/assets/{hostInit-4roC4hEj.js → hostInit-Blh7a-Qe.js} +1 -1
- package/admin/custom/assets/{lib-CchzgTUs.js → lib-DBbNtolG.js} +1 -1
- package/admin/custom/assets/{mf-entry-bootstrap-0-4806bbe4.js → mf-entry-bootstrap-0-7abd30d3.js} +2 -2
- package/admin/custom/assets/{pendingShares-DLlv8yFY.js → pendingShares-U0D1MYK4.js} +1 -1
- package/admin/custom/customComponents.js +2 -2
- package/admin/custom/i18n/de.json +56 -0
- package/admin/custom/i18n/en.json +56 -0
- package/admin/custom/i18n/es.json +56 -0
- package/admin/custom/i18n/fr.json +56 -0
- package/admin/custom/i18n/it.json +56 -0
- package/admin/custom/i18n/nl.json +56 -0
- package/admin/custom/i18n/pl.json +56 -0
- package/admin/custom/i18n/pt.json +56 -0
- package/admin/custom/i18n/ru.json +56 -0
- package/admin/custom/i18n/uk.json +56 -0
- package/admin/custom/i18n/zh-cn.json +56 -0
- package/admin/custom/index.html +2 -2
- package/admin/custom/mf-manifest.json +1 -1
- package/admin/custom/mf-stats.json +1 -1
- package/admin/i18n/de.json +31 -4
- package/admin/i18n/en.json +31 -4
- package/admin/i18n/es.json +31 -4
- package/admin/i18n/fr.json +31 -4
- package/admin/i18n/it.json +31 -4
- package/admin/i18n/nl.json +31 -4
- package/admin/i18n/pl.json +31 -4
- package/admin/i18n/pt.json +31 -4
- package/admin/i18n/ru.json +31 -4
- package/admin/i18n/uk.json +31 -4
- package/admin/i18n/zh-cn.json +31 -4
- package/admin/jsonConfig.json +226 -15
- package/build/lib/credentials.js +11 -5
- package/build/lib/credentials.js.map +1 -1
- package/build/lib/journal.js +298 -2
- package/build/lib/journal.js.map +1 -1
- package/build/lib/llm.js +122 -7
- package/build/lib/llm.js.map +1 -1
- package/build/lib/localLlm.js +47 -10
- package/build/lib/localLlm.js.map +1 -1
- package/build/lib/nlu.js +372 -13
- package/build/lib/nlu.js.map +1 -1
- package/build/lib/piAgent.js +89 -0
- package/build/lib/piAgent.js.map +1 -0
- package/build/lib/systemAi.js +124 -0
- package/build/lib/systemAi.js.map +1 -0
- package/build/lib/tools.js +3 -0
- package/build/lib/tools.js.map +1 -1
- package/build/lib/voice/engines.js +31 -3
- package/build/lib/voice/engines.js.map +1 -1
- package/build/lib/voice/esphome.js +81 -7
- package/build/lib/voice/esphome.js.map +1 -1
- package/build/lib/voice/esphomeProto.js.map +1 -1
- package/build/lib/voice/mediaServer.js +29 -1
- package/build/lib/voice/mediaServer.js.map +1 -1
- package/build/lib/voice/openrouter.js +220 -0
- package/build/lib/voice/openrouter.js.map +1 -0
- package/build/lib/voice/protocol.js.map +1 -1
- package/build/lib/voice/voiceServer.js +81 -7
- package/build/lib/voice/voiceServer.js.map +1 -1
- package/build/lib/voice/wakeWords.js +284 -0
- package/build/lib/voice/wakeWords.js.map +1 -0
- package/build/lib/voice/wyoming.js +12 -4
- package/build/lib/voice/wyoming.js.map +1 -1
- package/build/lib/weather.js +160 -0
- package/build/lib/weather.js.map +1 -1
- package/build/main.js +1282 -126
- package/build/main.js.map +1 -1
- package/io-package.json +53 -41
- package/package.json +5 -5
- package/admin/custom/assets/Components-C6qcpaRC.js +0 -1
- package/admin/custom/assets/_virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__201375093093590__prebuild___mf_0_iobroker_mf_1_json_mf_2_config__prebuild__-TM4l0OYm.js +0 -122
- /package/admin/custom/assets/{_virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__201375093093590__loadShare___mf_0_emotion_mf_1_react__loadShare__.js-BkH7fzDw.js → _virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__163539842885036__loadShare___mf_0_emotion_mf_1_react__loadShare__.js-BkH7fzDw.js} +0 -0
- /package/admin/custom/assets/{_virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__201375093093590__prebuild__react_mf_1_jsx_mf_2_runtime__prebuild__-M61rjtmp.js → _virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__163539842885036__prebuild__react_mf_1_jsx_mf_2_runtime__prebuild__-M61rjtmp.js} +0 -0
package/build/main.js
CHANGED
|
@@ -63,17 +63,65 @@ const wyoming_1 = require("./lib/voice/wyoming");
|
|
|
63
63
|
const esphome_1 = require("./lib/voice/esphome");
|
|
64
64
|
const esphomeEntities_1 = require("./lib/voice/esphomeEntities");
|
|
65
65
|
const mediaServer_1 = require("./lib/voice/mediaServer");
|
|
66
|
+
const wakeWords_1 = require("./lib/voice/wakeWords");
|
|
66
67
|
const tone_1 = require("./lib/voice/tone");
|
|
67
68
|
const engines_1 = require("./lib/voice/engines");
|
|
69
|
+
const openrouter_1 = require("./lib/voice/openrouter");
|
|
70
|
+
const piAgent_1 = require("./lib/piAgent");
|
|
71
|
+
const systemAi_1 = require("./lib/systemAi");
|
|
68
72
|
/**
|
|
69
73
|
* How long a rendered weather context line stays valid. Weather adapters refresh every few minutes at best,
|
|
70
74
|
* so re-reading their whole state tree on every single request would be pure overhead.
|
|
71
75
|
*/
|
|
72
76
|
const WEATHER_CTX_TTL = 5 * 60 * 1000;
|
|
77
|
+
/**
|
|
78
|
+
* How long the NLU device model stays valid. Building it costs a `list_devices` plus a name lookup per
|
|
79
|
+
* device, and one spoken command needs it up to three times (STT hints, NLU, the cloud device list).
|
|
80
|
+
* Enum and name changes drop it at once (see {@link ListCache.onClear}); the TTL only catches devices
|
|
81
|
+
* that were added or renamed elsewhere.
|
|
82
|
+
*/
|
|
83
|
+
const NLU_DEVICES_TTL_MS = 60_000;
|
|
84
|
+
/** Default for `localLlmTimeoutSec`: after this the local model is stopped and the next tier answers. */
|
|
85
|
+
const LOCAL_LLM_TIMEOUT_SEC = 15;
|
|
86
|
+
/**
|
|
87
|
+
* Answer length cap of the local model. It answers small talk and general questions, mostly by voice, and
|
|
88
|
+
* on a Raspberry Pi every token costs about a fifth of a second — 512 tokens would be a minute and a half.
|
|
89
|
+
*/
|
|
90
|
+
const LOCAL_LLM_MAX_TOKENS = 256;
|
|
73
91
|
/**
|
|
74
92
|
* The replies the assistant speaks verbatim, whatever the question was. They are pre-synthesised into
|
|
75
93
|
* the TTS cache at startup; everything else is cached on first use.
|
|
76
94
|
*/
|
|
95
|
+
/** Fields of the Voice tab the voice test takes from the unsaved form (everything an STT or TTS engine is built from). */
|
|
96
|
+
const VOICE_FORM_FIELDS = [
|
|
97
|
+
'voiceLanguage',
|
|
98
|
+
'sttProvider',
|
|
99
|
+
'sttModel',
|
|
100
|
+
'openrouterSttModel',
|
|
101
|
+
'voskModel',
|
|
102
|
+
'ttsProvider',
|
|
103
|
+
'ttsVoice',
|
|
104
|
+
'ttsModel',
|
|
105
|
+
'voiceCredentialType',
|
|
106
|
+
'voiceApiKey',
|
|
107
|
+
'voiceCredentialId',
|
|
108
|
+
'openrouterSpeechKey',
|
|
109
|
+
'openrouterCredentialId',
|
|
110
|
+
'openrouterTtsModel',
|
|
111
|
+
'openrouterVoice',
|
|
112
|
+
'azureSpeechKey',
|
|
113
|
+
'azureSpeechRegion',
|
|
114
|
+
'azureCredentialId',
|
|
115
|
+
'azureVoice',
|
|
116
|
+
'awsAccessKeyId',
|
|
117
|
+
'awsSecretAccessKey',
|
|
118
|
+
'awsRegion',
|
|
119
|
+
'awsCredentialId',
|
|
120
|
+
'awsVoice',
|
|
121
|
+
'piperVoice',
|
|
122
|
+
];
|
|
123
|
+
/** Of those, the protected ones: the browser never gets them, so an empty value there means "keep the saved one" */
|
|
124
|
+
const VOICE_FORM_SECRETS = ['voiceApiKey', 'openrouterSpeechKey', 'azureSpeechKey', 'awsSecretAccessKey'];
|
|
77
125
|
/** How long to wait for a device to confirm a write, and how often to look. */
|
|
78
126
|
const ACK_TIMEOUT_MS = 2000;
|
|
79
127
|
const ACK_POLL_MS = 150;
|
|
@@ -163,7 +211,37 @@ function iaqLabel(value, unit, lang) {
|
|
|
163
211
|
return lang === 'ru' ? row[3] : lang === 'de' ? row[1] : row[2];
|
|
164
212
|
}
|
|
165
213
|
/** NLU actions that only switch something; their success is what the confirmation tone replaces. */
|
|
166
|
-
const CONTROL_ACTIONS = new Set(['on', 'off', 'level', 'color']);
|
|
214
|
+
const CONTROL_ACTIONS = new Set(['on', 'off', 'level', 'color', 'volumeSet']);
|
|
215
|
+
/** An intent in one line for the request trace: "level Rollo (Büro) = 30", "timerSet 300 s", "alarmSet 7:05". */
|
|
216
|
+
function describeIntent(intent) {
|
|
217
|
+
let text = intent.action;
|
|
218
|
+
if (intent.device) {
|
|
219
|
+
text += ` ${intent.device.name}${intent.device.room ? ` (${intent.device.room})` : ''}`;
|
|
220
|
+
}
|
|
221
|
+
else if (intent.room) {
|
|
222
|
+
text += ` (${intent.room})`;
|
|
223
|
+
}
|
|
224
|
+
if (intent.value !== undefined) {
|
|
225
|
+
text += ` = ${JSON.stringify(intent.value)}`;
|
|
226
|
+
}
|
|
227
|
+
else if (intent.durationSec !== undefined) {
|
|
228
|
+
text += ` ${intent.durationSec} s`;
|
|
229
|
+
}
|
|
230
|
+
else if (intent.hour !== undefined) {
|
|
231
|
+
text += ` ${intent.hour}:${String(intent.minute ?? 0).padStart(2, '0')}`;
|
|
232
|
+
}
|
|
233
|
+
if (intent.delaySec !== undefined) {
|
|
234
|
+
text += ` in ${intent.delaySec} s`;
|
|
235
|
+
}
|
|
236
|
+
else if (intent.at) {
|
|
237
|
+
text += ` at ${intent.at.day ? `+${intent.at.day}d ` : ''}${intent.at.hour}:${String(intent.at.minute).padStart(2, '0')}`;
|
|
238
|
+
}
|
|
239
|
+
return text;
|
|
240
|
+
}
|
|
241
|
+
/** A device command for later ("in 2 Stunden", "um 22 Uhr") — goes into the calendar, not to the device */
|
|
242
|
+
function isDeferred(intent) {
|
|
243
|
+
return intent.delaySec !== undefined || !!intent.at;
|
|
244
|
+
}
|
|
167
245
|
const FIXED_REPLIES = {
|
|
168
246
|
de: ['Ok.', 'Erledigt.'],
|
|
169
247
|
en: ['Okay.', 'Done.'],
|
|
@@ -193,6 +271,12 @@ function languageLabel(lang) {
|
|
|
193
271
|
class Assistant extends adapter_core_1.Adapter {
|
|
194
272
|
/** Cloud LLM; `null` with provider "none" (local tiers only) or before `onReady` finished. */
|
|
195
273
|
agent = null;
|
|
274
|
+
/** Provider "pi-agent": the cloud tier is a pi-agent instance instead of {@link agent}. */
|
|
275
|
+
piAgent = null;
|
|
276
|
+
/** The cloud LLM of the instance (own or system-wide settings); `null` without one. */
|
|
277
|
+
llm = null;
|
|
278
|
+
/** What of `system.ai` this instance depends on, while it takes the system-wide AI settings. */
|
|
279
|
+
systemAiWatch = null;
|
|
196
280
|
/** Set once `onReady` initialised the pipeline — with or without a cloud {@link agent}. */
|
|
197
281
|
ready = false;
|
|
198
282
|
mcp = null;
|
|
@@ -200,6 +284,8 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
200
284
|
journal = null;
|
|
201
285
|
/** Short-TTL cache for the expensive device/room/function listings; invalidated on enum changes. */
|
|
202
286
|
listCache = null;
|
|
287
|
+
/** The NLU device model of {@link getNluDevices}, shared by all callers (and concurrent requests). */
|
|
288
|
+
nluDevices = null;
|
|
203
289
|
/** Cache of a `<base>.SET` level state id → its writable boolean sibling switch (`<base>.ON_SET`/`.ON`), or ''. */
|
|
204
290
|
siblingSwitch = new Map();
|
|
205
291
|
/** Tier-1a local LLM (node-llama-cpp), lazily loaded when enabled + installed; null otherwise. */
|
|
@@ -216,6 +302,10 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
216
302
|
esphome = null;
|
|
217
303
|
/** Serves spoken replies to ESPHome satellites over HTTP; null unless those are enabled. */
|
|
218
304
|
media = null;
|
|
305
|
+
/** Uploaded wake words (`assistant.0/wakewords/`), offered to every ESPHome satellite. */
|
|
306
|
+
customWakeWords = new Map();
|
|
307
|
+
/** Loads {@link customWakeWords} once from the file storage; later calls wait for the same load. */
|
|
308
|
+
customWakeWordsLoaded = null;
|
|
219
309
|
/** Short-term per-source conversation memory (in-memory, TTL) for follow-up questions. */
|
|
220
310
|
context = new context_1.ConversationStore();
|
|
221
311
|
/** Questions the assistant asked and is waiting for an answer to (`askUser`), keyed by source. */
|
|
@@ -230,6 +320,8 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
230
320
|
mirroredTimers = new Map();
|
|
231
321
|
/** Per satellite: resolves once its control objects exist, so state writes can wait for them. */
|
|
232
322
|
satControlsReady = new Map();
|
|
323
|
+
/** Satellites whose `volume` state exists and is subscribed (see updateSatelliteVolume). */
|
|
324
|
+
satVolumeEnsured = new Set();
|
|
233
325
|
/** Native (ioBroker) satellite state-id → sender instance id, so we can push announcements back to it. */
|
|
234
326
|
nativeSatFrom = new Map();
|
|
235
327
|
/** Countdown timers / reminders (roadmap #2); mirrored into `timers.*` states. Null until onReady. */
|
|
@@ -287,15 +379,30 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
287
379
|
// alarms, the local model) answer. The ioBroker tools are still built: the NLU switches through them.
|
|
288
380
|
const provider = cfg.provider || 'openai';
|
|
289
381
|
const localOnly = provider === 'none';
|
|
290
|
-
|
|
382
|
+
// Provider "pi-agent": the adapter pi-agent is the cloud tier, with its own model, key and tools.
|
|
383
|
+
const harness = provider === 'pi-agent';
|
|
384
|
+
// Provider "system" (or no key of its own): provider, model and key of the system-wide AI settings.
|
|
385
|
+
if (!localOnly && !harness && (0, systemAi_1.usesSystemAi)(cfg)) {
|
|
386
|
+
await this.watchSystemAi();
|
|
387
|
+
}
|
|
388
|
+
const resolved = localOnly || harness ? null : await (0, systemAi_1.resolveLlm)(this, cfg);
|
|
389
|
+
this.llm = resolved && !('error' in resolved) ? resolved : null;
|
|
390
|
+
// speech of the same vendor reuses the key of the LLM
|
|
391
|
+
const llmKey = this.llm ? { provider: this.llm.provider, apiKey: this.llm.apiKey } : null;
|
|
291
392
|
await this.setupJournal();
|
|
292
|
-
if (
|
|
293
|
-
this.log.warn(
|
|
294
|
-
|
|
295
|
-
|
|
393
|
+
if (harness && !(cfg.piAgentInstance || '').trim()) {
|
|
394
|
+
this.log.warn('Provider "pi-agent" without an instance — choose the pi-agent instance in the adapter settings.');
|
|
395
|
+
await this.setStateAsync('info.connection', { val: false, ack: true });
|
|
396
|
+
return;
|
|
397
|
+
}
|
|
398
|
+
if (resolved && 'error' in resolved) {
|
|
399
|
+
this.log.warn(resolved.error);
|
|
296
400
|
await this.setStateAsync('info.connection', { val: false, ack: true });
|
|
297
401
|
return;
|
|
298
402
|
}
|
|
403
|
+
if (this.llm?.system) {
|
|
404
|
+
this.log.info(`LLM from the system-wide AI settings: ${this.llm.provider} · ${this.llm.model || 'auto'}`);
|
|
405
|
+
}
|
|
299
406
|
try {
|
|
300
407
|
this.mcp = this.journalMcp(await (0, mcp_server_1.createInProcessMcp)({
|
|
301
408
|
adapter: this,
|
|
@@ -304,6 +411,9 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
304
411
|
allowObjectChange: cfg.allowObjectChange,
|
|
305
412
|
}));
|
|
306
413
|
this.listCache = new tools_1.ListCache();
|
|
414
|
+
this.listCache.onClear = () => {
|
|
415
|
+
this.nluDevices = null;
|
|
416
|
+
};
|
|
307
417
|
const { tools, denied } = await (0, tools_1.buildMcpTools)(this.mcp, cfg, this.listCache, (id, fb) => this.resolveDeviceName(id, fb));
|
|
308
418
|
// Let the cloud LLM handle timers/alarms too (for phrasings the rule-based NLU misses).
|
|
309
419
|
tools.push(...this.buildTimerTools(), ...this.buildAlarmTools());
|
|
@@ -313,6 +423,9 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
313
423
|
if ((cfg.weatherInstance || '').trim()) {
|
|
314
424
|
tools.push(this.buildWeatherTool());
|
|
315
425
|
}
|
|
426
|
+
if ((cfg.calendarInstance || '').trim()) {
|
|
427
|
+
tools.push(...this.buildCalendarTools());
|
|
428
|
+
}
|
|
316
429
|
if (cfg.voiceEnabled) {
|
|
317
430
|
// Only useful with satellites: without voice there is nothing to speak on.
|
|
318
431
|
tools.push(this.buildAnnounceTool());
|
|
@@ -336,17 +449,20 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
336
449
|
// Bust the device/room cache when room/function memberships change (new device shows within TTL anyway).
|
|
337
450
|
this.subscribeForeignObjects('enum.rooms.*');
|
|
338
451
|
this.subscribeForeignObjects('enum.functions.*');
|
|
339
|
-
if (
|
|
340
|
-
|
|
452
|
+
if (harness) {
|
|
453
|
+
this.piAgent = new piAgent_1.PiAgentHarness(this, cfg.piAgentInstance.trim(), Math.max(5, Number(cfg.piAgentTimeoutSec) || piAgent_1.PI_AGENT_TIMEOUT_SEC) * 1000);
|
|
454
|
+
}
|
|
455
|
+
else if (this.llm) {
|
|
341
456
|
this.agent = new llm_1.LlmAgent({
|
|
342
|
-
provider,
|
|
343
|
-
apiKey,
|
|
344
|
-
model:
|
|
345
|
-
baseUrl:
|
|
457
|
+
provider: this.llm.provider,
|
|
458
|
+
apiKey: this.llm.apiKey,
|
|
459
|
+
model: this.llm.model,
|
|
460
|
+
baseUrl: this.llm.baseUrl,
|
|
346
461
|
systemPrompt: cfg.systemPrompt || '',
|
|
347
462
|
maxTokens: cfg.maxTokens || 1024,
|
|
348
463
|
tools: tools.map(t => this.journalTool(t)),
|
|
349
464
|
log: this.log,
|
|
465
|
+
trace: step => this.journal?.step(step),
|
|
350
466
|
});
|
|
351
467
|
}
|
|
352
468
|
}
|
|
@@ -391,8 +507,10 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
391
507
|
await this.setupPresence();
|
|
392
508
|
await this.setupTriggers();
|
|
393
509
|
this.log.info(this.agent
|
|
394
|
-
? `Assistant ready (provider=${
|
|
395
|
-
:
|
|
510
|
+
? `Assistant ready (provider=${this.llm?.provider}, model=${this.agent.model || 'auto'}).`
|
|
511
|
+
: this.piAgent
|
|
512
|
+
? `Assistant ready (cloud tier: ${this.piAgent.instance}).`
|
|
513
|
+
: 'Assistant ready without an LLM provider — answering locally only.');
|
|
396
514
|
if (cfg.useLocalLlm) {
|
|
397
515
|
void this.ensureLocalLlm(); // background: model load/download must not block onReady
|
|
398
516
|
}
|
|
@@ -401,20 +519,20 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
401
519
|
// Subscribe regardless of the UDP server so writes are always handled (and logged if unroutable).
|
|
402
520
|
this.subscribeStates('tts.text');
|
|
403
521
|
if (cfg.udpServerEnabled) {
|
|
404
|
-
await this.startVoiceServer(
|
|
522
|
+
await this.startVoiceServer(llmKey);
|
|
405
523
|
}
|
|
406
524
|
}
|
|
407
525
|
if (cfg.wyomingEnabled) {
|
|
408
|
-
await this.startWyoming(
|
|
526
|
+
await this.startWyoming(llmKey);
|
|
409
527
|
}
|
|
410
528
|
if (cfg.esphomeEnabled) {
|
|
411
|
-
await this.startEsphome(
|
|
529
|
+
await this.startEsphome(llmKey);
|
|
412
530
|
}
|
|
413
531
|
}
|
|
414
532
|
/** Start the Wyoming TCP endpoint, bridging to the same STT/answer/TTS pipeline as the UDP server. */
|
|
415
|
-
async startWyoming(
|
|
533
|
+
async startWyoming(llmKey) {
|
|
416
534
|
const cfg = this.config;
|
|
417
|
-
const creds = await (0, credentials_1.resolveVoiceCredentials)(this, cfg,
|
|
535
|
+
const creds = await (0, credentials_1.resolveVoiceCredentials)(this, cfg, llmKey);
|
|
418
536
|
const ctx = this.voiceContext(cfg, creds);
|
|
419
537
|
const language = cfg.voiceLanguage || this.language || '';
|
|
420
538
|
let stt;
|
|
@@ -434,7 +552,8 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
434
552
|
language,
|
|
435
553
|
stt,
|
|
436
554
|
tts,
|
|
437
|
-
answer: (question, sttMs) => this.answer(question, 'wyoming', { transport: 'wyoming', sttMs }),
|
|
555
|
+
answer: (question, sttMs, audioMs) => this.answer(question, 'wyoming', { transport: 'wyoming', sttMs, audioMs }),
|
|
556
|
+
onSpoken: info => this.reportSpoken('wyoming', info),
|
|
438
557
|
getHints: () => this.buildSttHints(),
|
|
439
558
|
log: this.log,
|
|
440
559
|
});
|
|
@@ -454,14 +573,14 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
454
573
|
* linux-voice-assistant). The adapter is the client here — the devices listen on TCP 6053 — and the
|
|
455
574
|
* spoken reply is handed over as a URL, so the little media server comes up together with them.
|
|
456
575
|
*/
|
|
457
|
-
async startEsphome(
|
|
576
|
+
async startEsphome(llmKey) {
|
|
458
577
|
const cfg = this.config;
|
|
459
578
|
const devices = (cfg.esphomeDevices || []).filter(d => d?.ip?.trim());
|
|
460
579
|
if (!devices.length) {
|
|
461
580
|
this.log.warn('ESPHome satellites are enabled, but no device is configured.');
|
|
462
581
|
return;
|
|
463
582
|
}
|
|
464
|
-
const creds = await (0, credentials_1.resolveVoiceCredentials)(this, cfg,
|
|
583
|
+
const creds = await (0, credentials_1.resolveVoiceCredentials)(this, cfg, llmKey);
|
|
465
584
|
const ctx = this.voiceContext(cfg, creds);
|
|
466
585
|
const language = cfg.voiceLanguage || this.language || '';
|
|
467
586
|
let stt;
|
|
@@ -489,6 +608,10 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
489
608
|
this.media = null;
|
|
490
609
|
return;
|
|
491
610
|
}
|
|
611
|
+
await this.loadCustomWakeWords();
|
|
612
|
+
for (const word of this.customWakeWords.values()) {
|
|
613
|
+
this.serveWakeWord(word);
|
|
614
|
+
}
|
|
492
615
|
this.esphome = new esphome_1.EsphomeSatellites({
|
|
493
616
|
devices,
|
|
494
617
|
language,
|
|
@@ -499,7 +622,10 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
499
622
|
room: origin.room,
|
|
500
623
|
wakeWord: origin.wakeWord,
|
|
501
624
|
sttMs: origin.sttMs,
|
|
625
|
+
audioMs: origin.audioMs,
|
|
626
|
+
vad: origin.vad,
|
|
502
627
|
}),
|
|
628
|
+
onSpoken: (device, info) => this.reportSpoken(device, info),
|
|
503
629
|
getHints: () => this.buildSttHints(),
|
|
504
630
|
media: this.media,
|
|
505
631
|
mediaHost: (cfg.esphomeMediaHost || '').trim(),
|
|
@@ -517,7 +643,15 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
517
643
|
onEntityState: (device, room, entity) => {
|
|
518
644
|
this.updateSatelliteControl(device, room, entity).catch(e => this.log.debug(`satellite control update failed: ${e.message}`));
|
|
519
645
|
},
|
|
520
|
-
expectsFollowUp: answer => this.
|
|
646
|
+
expectsFollowUp: (answer, device) => this.followUpFor(device, answer),
|
|
647
|
+
externalWakeWords: () => [...this.customWakeWords.values()].map(w => ({
|
|
648
|
+
id: w.id,
|
|
649
|
+
phrase: w.phrase,
|
|
650
|
+
languages: w.languages,
|
|
651
|
+
size: w.size,
|
|
652
|
+
sha256: w.sha256,
|
|
653
|
+
path: `${wakeWords_1.WAKE_WORD_URL_PREFIX}${w.id}.json`,
|
|
654
|
+
})),
|
|
521
655
|
});
|
|
522
656
|
await this.esphome.start();
|
|
523
657
|
this.log.info(`ESPHome satellites: ${devices.length} device(s), STT=${cfg.sttProvider || 'openai'}, TTS=${cfg.ttsProvider || 'openai'}.`);
|
|
@@ -534,6 +668,7 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
534
668
|
azure: (cfg.azureVoice || '').trim(),
|
|
535
669
|
aws: (cfg.awsVoice || '').trim(),
|
|
536
670
|
piper: (cfg.piperVoice || '').trim(),
|
|
671
|
+
openrouter: (cfg.openrouterVoice || '').trim(),
|
|
537
672
|
},
|
|
538
673
|
sttFallback: cfg.sttFallback || '',
|
|
539
674
|
ttsFallback: cfg.ttsFallback || '',
|
|
@@ -542,6 +677,9 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
542
677
|
voskModel: (cfg.voskModel || '').trim(),
|
|
543
678
|
sttModel: (cfg.sttModel || '').trim(),
|
|
544
679
|
ttsModel: (cfg.ttsModel || '').trim(),
|
|
680
|
+
openrouterSttModel: (cfg.openrouterSttModel || '').trim(),
|
|
681
|
+
openrouterTtsModel: (cfg.openrouterTtsModel || '').trim(),
|
|
682
|
+
decodeAudio: audio => this.decodeAudioBufferToPcm(audio),
|
|
545
683
|
};
|
|
546
684
|
}
|
|
547
685
|
/**
|
|
@@ -549,9 +687,9 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
549
687
|
* AWS cloud, or Vosk / Piper locally); credentials come from the encrypted config fields or the
|
|
550
688
|
* central credential store (voiceCredentialType).
|
|
551
689
|
*/
|
|
552
|
-
async startVoiceServer(
|
|
690
|
+
async startVoiceServer(llmKey) {
|
|
553
691
|
const cfg = this.config;
|
|
554
|
-
const creds = await (0, credentials_1.resolveVoiceCredentials)(this, cfg,
|
|
692
|
+
const creds = await (0, credentials_1.resolveVoiceCredentials)(this, cfg, llmKey);
|
|
555
693
|
const ctx = this.voiceContext(cfg, creds);
|
|
556
694
|
// Raw language (e.g. 'de' or 'zh-cn') — each engine normalises it (ISO for cloud, model per lang for local).
|
|
557
695
|
const language = cfg.voiceLanguage || this.language || '';
|
|
@@ -572,7 +710,17 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
572
710
|
language,
|
|
573
711
|
stt,
|
|
574
712
|
tts,
|
|
575
|
-
answer: (question, ctx) => this.answerVoice(question, ctx.device, {
|
|
713
|
+
answer: (question, ctx) => this.answerVoice(question, ctx.device, {
|
|
714
|
+
transport: 'udp',
|
|
715
|
+
room: ctx.room,
|
|
716
|
+
sttMs: ctx.sttMs,
|
|
717
|
+
audioMs: ctx.audioMs,
|
|
718
|
+
}),
|
|
719
|
+
onSpoken: (device, info) => this.reportSpoken(device, info),
|
|
720
|
+
expectsFollowUp: (answer, device) => this.followUpFor(device, answer),
|
|
721
|
+
onVolume: (device, room, level) => {
|
|
722
|
+
this.updateSatelliteVolume(device, room, level).catch(e => this.log.debug(`satellite volume update failed: ${e.message}`));
|
|
723
|
+
},
|
|
576
724
|
getHints: () => this.buildSttHints(),
|
|
577
725
|
log: this.log,
|
|
578
726
|
onStatus: (device, room, state) => {
|
|
@@ -639,6 +787,24 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
639
787
|
expectsFollowUp(answer) {
|
|
640
788
|
return /[??]/.test(answer || '');
|
|
641
789
|
}
|
|
790
|
+
/** A transport synthesised the reply to `source`'s latest request — the journal shows it with that request. */
|
|
791
|
+
reportSpoken(source, info) {
|
|
792
|
+
const cfg = this.config;
|
|
793
|
+
const provider = cfg.ttsProvider || 'openai';
|
|
794
|
+
const voices = {
|
|
795
|
+
openai: cfg.ttsVoice || 'alloy',
|
|
796
|
+
azure: cfg.azureVoice,
|
|
797
|
+
aws: cfg.awsVoice,
|
|
798
|
+
piper: cfg.piperVoice,
|
|
799
|
+
openrouter: cfg.openrouterVoice,
|
|
800
|
+
};
|
|
801
|
+
this.journal?.spoken(source, {
|
|
802
|
+
...info,
|
|
803
|
+
provider,
|
|
804
|
+
fallback: cfg.ttsFallback || undefined,
|
|
805
|
+
voice: (voices[provider] || '').trim() || undefined,
|
|
806
|
+
});
|
|
807
|
+
}
|
|
642
808
|
/** Record the origin of the current text.request/response ('' = state write, 'chat' = message, else satellite). */
|
|
643
809
|
setQuerySource(source) {
|
|
644
810
|
this.setStateAsync('text.querySource', { val: source, ack: true }).catch(() => { });
|
|
@@ -650,8 +816,7 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
650
816
|
return this.speechEngines;
|
|
651
817
|
}
|
|
652
818
|
const cfg = this.config;
|
|
653
|
-
const
|
|
654
|
-
const creds = await (0, credentials_1.resolveVoiceCredentials)(this, cfg, mainKey);
|
|
819
|
+
const creds = await (0, credentials_1.resolveVoiceCredentials)(this, cfg, await this.speechLlmKey());
|
|
655
820
|
const ctx = this.voiceContext(cfg, creds);
|
|
656
821
|
const language = cfg.voiceLanguage || this.language || '';
|
|
657
822
|
try {
|
|
@@ -705,17 +870,18 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
705
870
|
room,
|
|
706
871
|
wakeWord: typeof msg.wakeWord === 'string' ? msg.wakeWord : undefined,
|
|
707
872
|
sttMs: Date.now() - sttStart,
|
|
873
|
+
audioMs: Math.round((pcm.length / 2 / rate) * 1000),
|
|
708
874
|
});
|
|
709
875
|
if (!answer) {
|
|
710
876
|
return { text, answer: '' };
|
|
711
877
|
}
|
|
878
|
+
const ttsStart = Date.now();
|
|
712
879
|
const reply = await engines.tts.synthesize(answer, language);
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
}
|
|
880
|
+
this.reportSpoken(source, {
|
|
881
|
+
ms: Date.now() - ttsStart,
|
|
882
|
+
audioMs: (reply.pcm.length / 2 / reply.sampleRate) * 1000,
|
|
883
|
+
});
|
|
884
|
+
const listen = this.followUpFor(source, answer);
|
|
719
885
|
return { text, answer, audio: reply.pcm.toString('base64'), sampleRate: reply.sampleRate, listen };
|
|
720
886
|
}
|
|
721
887
|
catch (e) {
|
|
@@ -738,8 +904,7 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
738
904
|
const cfg = { ...this.config, ...overrides };
|
|
739
905
|
const provider = msg.ttsProvider || cfg.ttsProvider || 'openai';
|
|
740
906
|
const language = msg.language || cfg.voiceLanguage || this.language || '';
|
|
741
|
-
const
|
|
742
|
-
const creds = await (0, credentials_1.resolveVoiceCredentials)(this, cfg, mainKey);
|
|
907
|
+
const creds = await (0, credentials_1.resolveVoiceCredentials)(this, cfg, await this.speechLlmKey());
|
|
743
908
|
return await (0, engines_1.listVoices)(provider, this.voiceContext(cfg, creds), language);
|
|
744
909
|
}
|
|
745
910
|
catch (e) {
|
|
@@ -778,6 +943,33 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
778
943
|
await this.setStateAsync(`${base}.room`, { val: room, ack: true });
|
|
779
944
|
}
|
|
780
945
|
}
|
|
946
|
+
/**
|
|
947
|
+
* Mirror the playback volume a UDP satellite reported into `satellites.<id>.volume`. The state only
|
|
948
|
+
* appears for satellites that speak the `volume` message — an older one would never answer a write.
|
|
949
|
+
*/
|
|
950
|
+
async updateSatelliteVolume(device, room, level) {
|
|
951
|
+
const id = await this.ensureSatelliteObjects(device, room);
|
|
952
|
+
const stateId = `satellites.${id}.volume`;
|
|
953
|
+
if (!this.satVolumeEnsured.has(id)) {
|
|
954
|
+
await this.setObjectNotExistsAsync(stateId, {
|
|
955
|
+
type: 'state',
|
|
956
|
+
common: {
|
|
957
|
+
name: 'Playback volume',
|
|
958
|
+
type: 'number',
|
|
959
|
+
role: 'level.volume',
|
|
960
|
+
unit: '%',
|
|
961
|
+
min: 0,
|
|
962
|
+
max: 100,
|
|
963
|
+
read: true,
|
|
964
|
+
write: true,
|
|
965
|
+
},
|
|
966
|
+
native: {},
|
|
967
|
+
});
|
|
968
|
+
this.subscribeStates(stateId);
|
|
969
|
+
this.satVolumeEnsured.add(id);
|
|
970
|
+
}
|
|
971
|
+
await this.setStateAsync(stateId, { val: level, ack: true });
|
|
972
|
+
}
|
|
781
973
|
/**
|
|
782
974
|
* Create the object tree for one satellite and return its (sanitised) state id, without touching
|
|
783
975
|
* any value. Callers that only need somewhere to hang a child object use this rather than
|
|
@@ -913,11 +1105,111 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
913
1105
|
this.satWakeWordStatesEnsured.add(id);
|
|
914
1106
|
}
|
|
915
1107
|
await this.setStateAsync(`${base}.wakeWords`, { val: config.active.join(','), ack: true });
|
|
1108
|
+
// `custom` marks the uploaded ones, so the settings dialog can offer to delete them.
|
|
1109
|
+
const words = config.available.map(w => (this.customWakeWords.has(w.id) ? { ...w, custom: true } : w));
|
|
916
1110
|
await this.setStateAsync(`${base}.availableWakeWords`, {
|
|
917
|
-
val: JSON.stringify({ max: config.max, words
|
|
1111
|
+
val: JSON.stringify({ max: config.max, words }),
|
|
918
1112
|
ack: true,
|
|
919
1113
|
});
|
|
920
1114
|
}
|
|
1115
|
+
/** Read the uploaded wake words (`<instance>/wakewords/<id>.json` + `.tflite`) once. */
|
|
1116
|
+
loadCustomWakeWords() {
|
|
1117
|
+
this.customWakeWordsLoaded ||= (async () => {
|
|
1118
|
+
let entries = [];
|
|
1119
|
+
try {
|
|
1120
|
+
entries = await this.readDirAsync(this.namespace, wakeWords_1.WAKE_WORD_DIR);
|
|
1121
|
+
}
|
|
1122
|
+
catch {
|
|
1123
|
+
return; // nothing uploaded yet
|
|
1124
|
+
}
|
|
1125
|
+
for (const entry of entries) {
|
|
1126
|
+
const id = entry.file.replace(/\.json$/, '');
|
|
1127
|
+
if (entry.isDir || !entry.file.endsWith('.json') || !wakeWords_1.WAKE_WORD_ID_RE.test(id)) {
|
|
1128
|
+
continue;
|
|
1129
|
+
}
|
|
1130
|
+
try {
|
|
1131
|
+
const config = await this.readStoredFile(`${wakeWords_1.WAKE_WORD_DIR}/${id}.json`, 'utf8');
|
|
1132
|
+
const model = await this.readStoredFile(`${wakeWords_1.WAKE_WORD_DIR}/${id}.tflite`, 'binary');
|
|
1133
|
+
const word = (0, wakeWords_1.loadWakeWord)(id, config, model);
|
|
1134
|
+
this.customWakeWords.set(id, word);
|
|
1135
|
+
// Uploaded before the upload checked the model: keep it, so it can be deleted in the dialog.
|
|
1136
|
+
const problem = (0, wakeWords_1.microModelProblem)(model);
|
|
1137
|
+
if (problem) {
|
|
1138
|
+
this.log.warn(`Custom wake word ${word.phrase} (${id}) cannot run on the device: ${problem}. Delete it in the satellite settings.`);
|
|
1139
|
+
}
|
|
1140
|
+
}
|
|
1141
|
+
catch (e) {
|
|
1142
|
+
this.log.warn(`Custom wake word ${id} skipped: ${e.message}`);
|
|
1143
|
+
}
|
|
1144
|
+
}
|
|
1145
|
+
if (this.customWakeWords.size) {
|
|
1146
|
+
this.log.info(`Custom wake words: ${[...this.customWakeWords.values()].map(w => `${w.phrase} (${w.id})`).join(', ')}`);
|
|
1147
|
+
}
|
|
1148
|
+
})();
|
|
1149
|
+
return this.customWakeWordsLoaded;
|
|
1150
|
+
}
|
|
1151
|
+
/**
|
|
1152
|
+
* Read a file from this instance's file storage as a Buffer. Depending on the backend and the file's
|
|
1153
|
+
* type it comes back as a Buffer or a string — `encoding` says how to read the latter.
|
|
1154
|
+
*/
|
|
1155
|
+
async readStoredFile(name, encoding) {
|
|
1156
|
+
const res = (await this.readFileAsync(this.namespace, name));
|
|
1157
|
+
const f = Buffer.isBuffer(res) || typeof res === 'string' ? res : res.file;
|
|
1158
|
+
if (f == null) {
|
|
1159
|
+
throw new Error(`${name} not found`);
|
|
1160
|
+
}
|
|
1161
|
+
return Buffer.isBuffer(f) ? f : Buffer.from(f, encoding);
|
|
1162
|
+
}
|
|
1163
|
+
serveWakeWord(word) {
|
|
1164
|
+
this.media?.serve(`${wakeWords_1.WAKE_WORD_URL_PREFIX}${word.id}.json`, word.config, 'application/json');
|
|
1165
|
+
this.media?.serve(`${wakeWords_1.WAKE_WORD_URL_PREFIX}${word.id}.tflite`, word.model, 'application/octet-stream');
|
|
1166
|
+
}
|
|
1167
|
+
/**
|
|
1168
|
+
* Store an uploaded wake word and offer it to the connected satellites. It shows up in their wake
|
|
1169
|
+
* word list right away; a device downloads the model only when it is selected there.
|
|
1170
|
+
*/
|
|
1171
|
+
async addCustomWakeWord(message) {
|
|
1172
|
+
await this.loadCustomWakeWords();
|
|
1173
|
+
const text = (value) => (typeof value === 'string' ? value : undefined);
|
|
1174
|
+
const languages = (Array.isArray(message.languages)
|
|
1175
|
+
? message.languages.map(String)
|
|
1176
|
+
: (text(message.languages) ?? '').split(/[,;\s]+/)).filter(Boolean);
|
|
1177
|
+
const word = (0, wakeWords_1.prepareWakeWord)({
|
|
1178
|
+
model: Buffer.from(text(message.model) ?? '', 'base64'),
|
|
1179
|
+
fileName: text(message.fileName),
|
|
1180
|
+
config: message.config && typeof message.config === 'object'
|
|
1181
|
+
? message.config
|
|
1182
|
+
: text(message.config),
|
|
1183
|
+
phrase: text(message.phrase),
|
|
1184
|
+
languages,
|
|
1185
|
+
}, this.config.voiceLanguage || this.language || '');
|
|
1186
|
+
if (!this.customWakeWords.has(word.id)) {
|
|
1187
|
+
await this.writeFileAsync(this.namespace, `${wakeWords_1.WAKE_WORD_DIR}/${word.id}.tflite`, word.model);
|
|
1188
|
+
await this.writeFileAsync(this.namespace, `${wakeWords_1.WAKE_WORD_DIR}/${word.id}.json`, word.config);
|
|
1189
|
+
this.customWakeWords.set(word.id, word);
|
|
1190
|
+
this.serveWakeWord(word);
|
|
1191
|
+
this.log.info(`Custom wake word uploaded: ${word.phrase} (${word.id}, ${word.size} bytes)`);
|
|
1192
|
+
}
|
|
1193
|
+
const devices = this.esphome?.refreshWakeWords() ?? 0;
|
|
1194
|
+
return { id: word.id, phrase: word.phrase, languages: word.languages, size: word.size, devices };
|
|
1195
|
+
}
|
|
1196
|
+
/**
|
|
1197
|
+
* Remove an uploaded wake word: it is no longer offered or served. A device that already downloaded
|
|
1198
|
+
* it keeps its copy — the ESPHome API cannot delete a model from a device.
|
|
1199
|
+
*/
|
|
1200
|
+
async deleteCustomWakeWord(id) {
|
|
1201
|
+
await this.loadCustomWakeWords();
|
|
1202
|
+
if (!wakeWords_1.WAKE_WORD_ID_RE.test(id) || !this.customWakeWords.has(id)) {
|
|
1203
|
+
throw new Error(`unknown wake word ${id}`);
|
|
1204
|
+
}
|
|
1205
|
+
for (const ext of ['json', 'tflite']) {
|
|
1206
|
+
await this.delFileAsync(this.namespace, `${wakeWords_1.WAKE_WORD_DIR}/${id}.${ext}`).catch(() => undefined);
|
|
1207
|
+
this.media?.unserve(`${wakeWords_1.WAKE_WORD_URL_PREFIX}${id}.${ext}`);
|
|
1208
|
+
}
|
|
1209
|
+
this.customWakeWords.delete(id);
|
|
1210
|
+
this.esphome?.refreshWakeWords();
|
|
1211
|
+
this.log.info(`Custom wake word deleted: ${id}`);
|
|
1212
|
+
}
|
|
921
1213
|
/**
|
|
922
1214
|
* Mirror everything an ESPHome satellite exposes besides the voice pipeline into
|
|
923
1215
|
* `satellites.<id>.controls.*` — microphone gain and volume, noise suppression, wake-word and
|
|
@@ -1281,7 +1573,8 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
1281
1573
|
await Promise.all(allowed.map(d => this.voice.announce(d, pcm, sampleRate)));
|
|
1282
1574
|
delivered += allowed.length;
|
|
1283
1575
|
if (listen) {
|
|
1284
|
-
|
|
1576
|
+
const playMs = (pcm.length / 2 / sampleRate) * 1000;
|
|
1577
|
+
allowed.forEach(d => this.voice.listen(d, playMs));
|
|
1285
1578
|
}
|
|
1286
1579
|
}
|
|
1287
1580
|
catch (e) {
|
|
@@ -1526,7 +1819,36 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
1526
1819
|
});
|
|
1527
1820
|
}
|
|
1528
1821
|
/** Invalidate the cached device/room/function listings when a room/function enum changes. */
|
|
1529
|
-
|
|
1822
|
+
/** Vendor and key of the cloud LLM: speech of the same vendor (OpenAI, OpenRouter) reuses the key. */
|
|
1823
|
+
async speechLlmKey() {
|
|
1824
|
+
const provider = this.config.provider || 'openai';
|
|
1825
|
+
const llm = this.llm ?? (provider === 'none' || provider === 'pi-agent' ? null : await (0, systemAi_1.resolveLlm)(this, this.config));
|
|
1826
|
+
return llm && !('error' in llm) ? { provider: llm.provider, apiKey: llm.apiKey } : null;
|
|
1827
|
+
}
|
|
1828
|
+
/**
|
|
1829
|
+
* Restart when the system-wide AI settings this instance takes change — also when they are still empty, so
|
|
1830
|
+
* setting them up in admin starts the instance. The credential is watched too: a new key needs no new id.
|
|
1831
|
+
*/
|
|
1832
|
+
async watchSystemAi() {
|
|
1833
|
+
const native = await (0, systemAi_1.readSystemAi)(this);
|
|
1834
|
+
const credentialId = (0, systemAi_1.systemAiSelection)(native)?.credentialId || '';
|
|
1835
|
+
this.systemAiWatch = { fingerprint: (0, systemAi_1.systemAiFingerprint)(native), credentialId };
|
|
1836
|
+
await this.subscribeForeignObjectsAsync(systemAi_1.SYSTEM_AI_ID);
|
|
1837
|
+
if (credentialId) {
|
|
1838
|
+
await this.subscribeForeignObjectsAsync(credentialId);
|
|
1839
|
+
}
|
|
1840
|
+
}
|
|
1841
|
+
onObjectChange(id, obj) {
|
|
1842
|
+
if (this.systemAiWatch && (id === systemAi_1.SYSTEM_AI_ID || id === this.systemAiWatch.credentialId)) {
|
|
1843
|
+
// A credential has no fingerprint: a change of it is a new key.
|
|
1844
|
+
if (id === systemAi_1.SYSTEM_AI_ID && (0, systemAi_1.systemAiFingerprint)(obj?.native) === this.systemAiWatch.fingerprint) {
|
|
1845
|
+
return;
|
|
1846
|
+
}
|
|
1847
|
+
this.log.info(`The system-wide AI settings changed (${id}) — restarting.`);
|
|
1848
|
+
this.systemAiWatch = null;
|
|
1849
|
+
this.restart();
|
|
1850
|
+
return;
|
|
1851
|
+
}
|
|
1530
1852
|
if (this.listCache && (id.startsWith('enum.rooms.') || id.startsWith('enum.functions.'))) {
|
|
1531
1853
|
this.listCache.clear();
|
|
1532
1854
|
this.log.debug(`list cache cleared (object changed: ${id})`);
|
|
@@ -1630,6 +1952,15 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
1630
1952
|
await this.setStateAsync('dnd', { val: this.globalDnd, ack: true });
|
|
1631
1953
|
return;
|
|
1632
1954
|
}
|
|
1955
|
+
// Playback volume of a UDP satellite — not acked here: the satellite reports the level it took.
|
|
1956
|
+
const satVolume = id.match(/\.satellites\.([^.]+)\.volume$/);
|
|
1957
|
+
if (satVolume) {
|
|
1958
|
+
const device = this.satDeviceById.get(satVolume[1]) || satVolume[1];
|
|
1959
|
+
if (!this.voice?.volume(device, { level: Number(state.val) })) {
|
|
1960
|
+
this.log.warn(`Cannot set the volume of '${device}': it is not a registered UDP satellite.`);
|
|
1961
|
+
}
|
|
1962
|
+
return;
|
|
1963
|
+
}
|
|
1633
1964
|
const satDnd = id.match(/\.satellites\.([^.]+)\.dnd$/);
|
|
1634
1965
|
if (satDnd) {
|
|
1635
1966
|
this.dndById.set(satDnd[1], !!state.val);
|
|
@@ -1775,14 +2106,23 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
1775
2106
|
}
|
|
1776
2107
|
return;
|
|
1777
2108
|
}
|
|
1778
|
-
// STT model
|
|
1779
|
-
if (obj.command === 'getSttModels') {
|
|
2109
|
+
// STT/TTS model dropdowns (Vosk, OpenRouter): suggest model names for the selected provider.
|
|
2110
|
+
if (obj.command === 'getSttModels' || obj.command === 'getTtsModels') {
|
|
1780
2111
|
const msg = (obj.message || {});
|
|
1781
2112
|
const cfg = this.config;
|
|
1782
|
-
const provider = msg.sttProvider || cfg.sttProvider || 'openai';
|
|
1783
2113
|
const language = msg.language || cfg.voiceLanguage || this.language || '';
|
|
2114
|
+
let models = [];
|
|
2115
|
+
try {
|
|
2116
|
+
models =
|
|
2117
|
+
obj.command === 'getSttModels'
|
|
2118
|
+
? await (0, engines_1.listSttModels)(msg.sttProvider || cfg.sttProvider || 'openai', language)
|
|
2119
|
+
: await (0, engines_1.listTtsModels)(msg.ttsProvider || cfg.ttsProvider || 'openai');
|
|
2120
|
+
}
|
|
2121
|
+
catch (e) {
|
|
2122
|
+
this.log.warn(`${obj.command} failed: ${e.message}`);
|
|
2123
|
+
}
|
|
1784
2124
|
if (obj.callback) {
|
|
1785
|
-
this.sendTo(obj.from, obj.command,
|
|
2125
|
+
this.sendTo(obj.from, obj.command, models, obj.callback);
|
|
1786
2126
|
}
|
|
1787
2127
|
return;
|
|
1788
2128
|
}
|
|
@@ -2014,6 +2354,41 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
2014
2354
|
}
|
|
2015
2355
|
return;
|
|
2016
2356
|
}
|
|
2357
|
+
// Settings dialog / scripts: own wake words for the ESPHome satellites (microWakeWord models).
|
|
2358
|
+
// uploadWakeWord: { model: base64 .tflite, fileName?, config?: JSON text, phrase?, languages? }
|
|
2359
|
+
// → { ok, id, phrase, languages, size, devices } | { error }
|
|
2360
|
+
// deleteWakeWord: { id } → { ok } | { error }; listWakeWords → { wakeWords: [{id, phrase, …}] }
|
|
2361
|
+
if (obj.command === 'uploadWakeWord' || obj.command === 'deleteWakeWord' || obj.command === 'listWakeWords') {
|
|
2362
|
+
const m = (obj.message || {});
|
|
2363
|
+
let payload;
|
|
2364
|
+
try {
|
|
2365
|
+
if (obj.command === 'uploadWakeWord') {
|
|
2366
|
+
payload = { ok: true, ...(await this.addCustomWakeWord(m)) };
|
|
2367
|
+
}
|
|
2368
|
+
else if (obj.command === 'deleteWakeWord') {
|
|
2369
|
+
await this.deleteCustomWakeWord(typeof m.id === 'string' ? m.id : '');
|
|
2370
|
+
payload = { ok: true };
|
|
2371
|
+
}
|
|
2372
|
+
else {
|
|
2373
|
+
await this.loadCustomWakeWords();
|
|
2374
|
+
payload = {
|
|
2375
|
+
wakeWords: [...this.customWakeWords.values()].map(w => ({
|
|
2376
|
+
id: w.id,
|
|
2377
|
+
phrase: w.phrase,
|
|
2378
|
+
languages: w.languages,
|
|
2379
|
+
size: w.size,
|
|
2380
|
+
})),
|
|
2381
|
+
};
|
|
2382
|
+
}
|
|
2383
|
+
}
|
|
2384
|
+
catch (e) {
|
|
2385
|
+
payload = { error: e.message };
|
|
2386
|
+
}
|
|
2387
|
+
if (obj.callback) {
|
|
2388
|
+
this.sendTo(obj.from, obj.command, payload, obj.callback);
|
|
2389
|
+
}
|
|
2390
|
+
return;
|
|
2391
|
+
}
|
|
2017
2392
|
// Scripts: everything an ESPHome satellite exposes besides the voice pipeline.
|
|
2018
2393
|
// message = { device? }; without a device, every connected one is listed.
|
|
2019
2394
|
if (obj.command === 'getControls') {
|
|
@@ -2122,10 +2497,20 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
2122
2497
|
}
|
|
2123
2498
|
return;
|
|
2124
2499
|
}
|
|
2125
|
-
// Chat: synthesize
|
|
2500
|
+
// Chat and the voice test of the settings: synthesize to speech (WAV) via the configured TTS engine,
|
|
2501
|
+
// or via the unsaved settings of the Voice tab (`config`).
|
|
2126
2502
|
if (obj.command === 'tts') {
|
|
2127
|
-
const { text, language } = (obj.message || {});
|
|
2128
|
-
const
|
|
2503
|
+
const { text, language, config } = (obj.message || {});
|
|
2504
|
+
const cfg = config ? this.withVoiceForm(config) : this.config;
|
|
2505
|
+
const result = await this.synthesizeToWav(String(text ?? ''), language, cfg);
|
|
2506
|
+
if (obj.callback) {
|
|
2507
|
+
this.sendTo(obj.from, obj.command, result, obj.callback);
|
|
2508
|
+
}
|
|
2509
|
+
return;
|
|
2510
|
+
}
|
|
2511
|
+
// Voice test of the settings: recognize a recording with the (unsaved) speech-to-text settings.
|
|
2512
|
+
if (obj.command === 'stt') {
|
|
2513
|
+
const result = await this.transcribeForTest((obj.message || {}));
|
|
2129
2514
|
if (obj.callback) {
|
|
2130
2515
|
this.sendTo(obj.from, obj.command, result, obj.callback);
|
|
2131
2516
|
}
|
|
@@ -2220,6 +2605,18 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
2220
2605
|
}
|
|
2221
2606
|
return;
|
|
2222
2607
|
}
|
|
2608
|
+
// Admin chat debug view: one request with every step of its processing (message = { id }).
|
|
2609
|
+
if (obj.command === 'getJournalTrace') {
|
|
2610
|
+
const id = String(obj.message?.id || '');
|
|
2611
|
+
const entry = this.journal?.get(id);
|
|
2612
|
+
const result = entry
|
|
2613
|
+
? { entry, steps: await this.journal.trace(id) }
|
|
2614
|
+
: { error: this.journal?.enabled ? 'unknown request' : 'journal disabled' };
|
|
2615
|
+
if (obj.callback) {
|
|
2616
|
+
this.sendTo(obj.from, obj.command, result, obj.callback);
|
|
2617
|
+
}
|
|
2618
|
+
return;
|
|
2619
|
+
}
|
|
2223
2620
|
if (obj.command === 'clearJournal') {
|
|
2224
2621
|
this.journal?.clear();
|
|
2225
2622
|
if (obj.callback) {
|
|
@@ -2271,28 +2668,126 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
2271
2668
|
*/
|
|
2272
2669
|
answer(question, source = '', origin = {}) {
|
|
2273
2670
|
return this.requestOrigin.run({ source, room: origin.room || '' }, () => this.journal
|
|
2274
|
-
? this.journal.run(question, source, origin, () =>
|
|
2671
|
+
? this.journal.run(question, source, origin, () => {
|
|
2672
|
+
this.traceSpeechInput(question, origin);
|
|
2673
|
+
return this.answerTracked(question, source);
|
|
2674
|
+
})
|
|
2275
2675
|
: this.answerTracked(question, source));
|
|
2276
2676
|
}
|
|
2677
|
+
/** Speech recognition happened before the request: put it at the head of its trace. */
|
|
2678
|
+
traceSpeechInput(question, origin) {
|
|
2679
|
+
if (origin.sttMs === undefined) {
|
|
2680
|
+
return;
|
|
2681
|
+
}
|
|
2682
|
+
const cfg = this.config;
|
|
2683
|
+
const provider = cfg.sttProvider || 'openai';
|
|
2684
|
+
const audio = origin.audioMs !== undefined ? `, ${(origin.audioMs / 1000).toFixed(1)} s of audio` : '';
|
|
2685
|
+
this.journal?.step({
|
|
2686
|
+
stage: 'stt',
|
|
2687
|
+
title: `Speech recognition: ${provider}${audio}`,
|
|
2688
|
+
ms: origin.sttMs,
|
|
2689
|
+
data: {
|
|
2690
|
+
provider,
|
|
2691
|
+
fallback: cfg.sttFallback || undefined,
|
|
2692
|
+
model: (provider === 'openrouter' ? cfg.openrouterSttModel : cfg.sttModel || cfg.voskModel || '').trim() ||
|
|
2693
|
+
undefined,
|
|
2694
|
+
language: cfg.voiceLanguage || this.language || '',
|
|
2695
|
+
audioMs: origin.audioMs,
|
|
2696
|
+
wakeWord: origin.wakeWord,
|
|
2697
|
+
vad: origin.vad,
|
|
2698
|
+
transcript: question,
|
|
2699
|
+
},
|
|
2700
|
+
});
|
|
2701
|
+
}
|
|
2277
2702
|
async answerTracked(question, source) {
|
|
2278
2703
|
const useCtx = this.config.useConversationContext !== false;
|
|
2279
2704
|
const history = useCtx ? this.context.get(source) : [];
|
|
2705
|
+
if (history.length) {
|
|
2706
|
+
this.journal?.step({
|
|
2707
|
+
stage: 'context',
|
|
2708
|
+
title: `Conversation context: ${history.length} earlier turn(s) of '${source}'`,
|
|
2709
|
+
data: history,
|
|
2710
|
+
});
|
|
2711
|
+
}
|
|
2280
2712
|
const raw = await this.produceAnswer(question, history, source);
|
|
2281
2713
|
// The LLM signals "I'm waiting for the user's reply" by appending the [[LISTEN]] control marker
|
|
2282
|
-
// (see followUpHint). Strip it here so it's neither stored, shown nor spoken, and
|
|
2283
|
-
//
|
|
2284
|
-
|
|
2285
|
-
|
|
2714
|
+
// (see followUpHint). Strip it here so it's neither stored, shown nor spoken, and keep it per source:
|
|
2715
|
+
// two satellites answered at the same time must not hand each other their mic-on.
|
|
2716
|
+
const followUp = /\[\[LISTEN\]\]/i.test(raw);
|
|
2717
|
+
this.followUps.set(source, followUp);
|
|
2718
|
+
if (followUp) {
|
|
2286
2719
|
this.journal?.note({ followUp: true });
|
|
2287
2720
|
}
|
|
2288
|
-
|
|
2289
|
-
|
|
2290
|
-
|
|
2721
|
+
let result = raw.replace(/\s*\[\[(?:LISTEN|DONE)\]\]\s*/gi, ' ').trim();
|
|
2722
|
+
// The LLM answers a switch command that worked with [[DONE]] alone (see doneHint): a beep on the satellite
|
|
2723
|
+
// instead of a sentence. Where nothing can beep (chat, Telegram), a short word. With more text, that is said.
|
|
2724
|
+
let toned = false;
|
|
2725
|
+
if (!result && /\[\[DONE\]\]/i.test(raw)) {
|
|
2726
|
+
toned = await this.playConfirmationTone(source);
|
|
2727
|
+
if (!toned) {
|
|
2728
|
+
result = this.doneText();
|
|
2729
|
+
}
|
|
2730
|
+
}
|
|
2731
|
+
this.journal?.step({
|
|
2732
|
+
stage: 'answer',
|
|
2733
|
+
title: toned
|
|
2734
|
+
? 'Confirmed with a tone instead of words'
|
|
2735
|
+
: result
|
|
2736
|
+
? `Answer: ${result.length} characters${followUp ? ', waits for a reply' : ''}`
|
|
2737
|
+
: 'No spoken answer',
|
|
2738
|
+
data: { answer: result, raw: raw !== result ? raw : undefined, followUp, tone: toned || undefined },
|
|
2739
|
+
});
|
|
2740
|
+
if (useCtx && (result || toned)) {
|
|
2741
|
+
this.context.add(source, question, result || this.doneText());
|
|
2291
2742
|
}
|
|
2292
2743
|
return result;
|
|
2293
2744
|
}
|
|
2294
|
-
/**
|
|
2295
|
-
|
|
2745
|
+
/**
|
|
2746
|
+
* System-prompt instruction for `confirmWithTone`: a switch command that worked is answered with [[DONE]]
|
|
2747
|
+
* alone, which becomes a beep on the satellite. Empty when the option (or voice) is off.
|
|
2748
|
+
*/
|
|
2749
|
+
doneHint() {
|
|
2750
|
+
if (!this.config.confirmWithTone || !this.config.voiceEnabled) {
|
|
2751
|
+
return '';
|
|
2752
|
+
}
|
|
2753
|
+
const lang = String(this.config.voiceLanguage || this.language || 'en');
|
|
2754
|
+
if (lang === 'ru') {
|
|
2755
|
+
return 'Если запрос был только командой включить, выключить или установить устройства на значение и всё получилось, ответь только управляющим маркером [[DONE]] (вместо фразы прозвучит короткий сигнал). На вопросы, при ошибках или если нужно что-то сообщить, отвечай как обычно.';
|
|
2756
|
+
}
|
|
2757
|
+
if (lang === 'de') {
|
|
2758
|
+
return 'Wenn die Anfrage nur ein Befehl war, Geräte ein- oder auszuschalten oder auf einen Wert zu stellen, und alles geklappt hat, antworte ausschließlich mit dem Steuer-Marker [[DONE]] (statt eines Satzes ertönt ein kurzer Bestätigungston). Bei Fragen, bei Fehlern oder wenn du etwas mitteilen musst, antworte normal.';
|
|
2759
|
+
}
|
|
2760
|
+
return 'If the request was only a command to switch devices on or off or to set them to a value, and everything worked, answer with the control marker [[DONE]] alone (a short beep plays instead of a sentence). For questions, errors or when you have something to tell, answer normally.';
|
|
2761
|
+
}
|
|
2762
|
+
/** What [[DONE]] says where no beep can be played */
|
|
2763
|
+
doneText() {
|
|
2764
|
+
const lang = String(this.config.voiceLanguage || this.language || 'en');
|
|
2765
|
+
return lang === 'ru' ? 'Готово.' : lang === 'de' ? 'Erledigt.' : 'Done.';
|
|
2766
|
+
}
|
|
2767
|
+
/** Per source: whether its last {@link answer} asked the user something (LLM-signalled via [[LISTEN]]). */
|
|
2768
|
+
followUps = new Map();
|
|
2769
|
+
/**
|
|
2770
|
+
* Should the satellite behind `source` re-open its mic after this reply? Primarily the LLM's own
|
|
2771
|
+
* [[LISTEN]] signal, with a question mark in the reply as the fallback (the NLU and the local tiers
|
|
2772
|
+
* cannot emit the marker, but they do ask "Meinst du Büro oder Wohnzimmer?").
|
|
2773
|
+
*/
|
|
2774
|
+
wantsFollowUp(source, answer) {
|
|
2775
|
+
if (!answer) {
|
|
2776
|
+
return null;
|
|
2777
|
+
}
|
|
2778
|
+
if (this.followUps.get(source)) {
|
|
2779
|
+
return 'llm';
|
|
2780
|
+
}
|
|
2781
|
+
return this.expectsFollowUp(answer) ? 'question' : null;
|
|
2782
|
+
}
|
|
2783
|
+
/** {@link wantsFollowUp} for a voice transport, logged, as the `listen` flag of the reply. */
|
|
2784
|
+
followUpFor(source, answer) {
|
|
2785
|
+
const why = this.wantsFollowUp(source, answer);
|
|
2786
|
+
if (why) {
|
|
2787
|
+
this.log.info(`→ mic-on to satellite '${source}' (${why === 'llm' ? 'LLM signalled a follow-up' : 'reply is a question'}).`);
|
|
2788
|
+
}
|
|
2789
|
+
return !!why;
|
|
2790
|
+
}
|
|
2296
2791
|
/**
|
|
2297
2792
|
* System-prompt instruction letting the LLM explicitly signal a follow-up: append [[LISTEN]] when it
|
|
2298
2793
|
* asks the user something and waits for an answer. More reliable than a "?" heuristic and lets a
|
|
@@ -2369,6 +2864,7 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
2369
2864
|
const n = this.stopRinging();
|
|
2370
2865
|
this.log.info(`Ring stopped by voice (${n} session(s)).`);
|
|
2371
2866
|
this.journal?.note({ tier: 'stop' });
|
|
2867
|
+
this.journal?.step({ stage: 'stop', title: `Stop word: silenced ${n} ringing timer/alarm(s)` });
|
|
2372
2868
|
const lang = String(this.config.voiceLanguage || this.language || 'en');
|
|
2373
2869
|
return lang === 'ru' ? 'Хорошо.' : lang === 'de' ? 'Ok.' : 'Okay.';
|
|
2374
2870
|
}
|
|
@@ -2380,6 +2876,11 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
2380
2876
|
if (this.pending.deliver(source, question)) {
|
|
2381
2877
|
this.log.info(`Answer to "${asked}" from source '${source || 'any'}': ${question}`);
|
|
2382
2878
|
this.journal?.note({ tier: 'askReply', detail: asked || undefined });
|
|
2879
|
+
this.journal?.step({
|
|
2880
|
+
stage: 'askReply',
|
|
2881
|
+
title: `Reply to the open question "${asked}"`,
|
|
2882
|
+
data: { question: asked, reply: question },
|
|
2883
|
+
});
|
|
2383
2884
|
return '';
|
|
2384
2885
|
}
|
|
2385
2886
|
}
|
|
@@ -2390,9 +2891,17 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
2390
2891
|
if (routine) {
|
|
2391
2892
|
this.log.info(`Routine '${routine.name}' triggered (source='${source}').`);
|
|
2392
2893
|
this.journal?.note({ tier: 'routine', detail: routine.name });
|
|
2894
|
+
this.journal?.step({ stage: 'routine', title: `Routine "${routine.name}" matched`, data: routine });
|
|
2393
2895
|
await this.runRoutine(routine, source);
|
|
2394
2896
|
return routine.reply;
|
|
2395
2897
|
}
|
|
2898
|
+
if (this.routines.length) {
|
|
2899
|
+
this.journal?.step({
|
|
2900
|
+
stage: 'routine',
|
|
2901
|
+
title: `No routine matched (${this.routines.length} configured)`,
|
|
2902
|
+
data: { phrases: this.routines.map(r => ({ name: r.name, phrases: r.phrases })) },
|
|
2903
|
+
});
|
|
2904
|
+
}
|
|
2396
2905
|
// Tier 0: rule-based NLU (device commands) — fastest, offline, free.
|
|
2397
2906
|
if (this.config.useLocalNlu) {
|
|
2398
2907
|
// "Spiele den Kanal X ab" first: the device NLU would find the TV in "… auf dem Fernseher" and
|
|
@@ -2401,9 +2910,19 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
2401
2910
|
if (yt) {
|
|
2402
2911
|
this.log.info(`YouTube request understood locally: ${yt.kind} "${yt.query}" (source='${source}').`);
|
|
2403
2912
|
this.journal?.note({ tier: 'nlu', detail: `youtube ${yt.kind}` });
|
|
2913
|
+
const start = Date.now();
|
|
2404
2914
|
const res = await this.playYouTube(yt.query, yt.kind, yt.player);
|
|
2915
|
+
this.journal?.step({
|
|
2916
|
+
stage: 'youtube',
|
|
2917
|
+
title: `YouTube ${yt.kind} "${yt.query}"${yt.player ? ` on "${yt.player}"` : ''}`,
|
|
2918
|
+
ms: Date.now() - start,
|
|
2919
|
+
data: { parsed: yt, result: res },
|
|
2920
|
+
});
|
|
2405
2921
|
return res.text;
|
|
2406
2922
|
}
|
|
2923
|
+
if (this.youtubePlayers.length) {
|
|
2924
|
+
this.journal?.step({ stage: 'youtube', title: 'Not a YouTube command' });
|
|
2925
|
+
}
|
|
2407
2926
|
try {
|
|
2408
2927
|
const handled = await this.tryLocalNlu(question, source);
|
|
2409
2928
|
if (handled !== null) {
|
|
@@ -2413,61 +2932,205 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
2413
2932
|
}
|
|
2414
2933
|
catch (e) {
|
|
2415
2934
|
this.log.debug(`NLU skipped: ${e.message}`);
|
|
2935
|
+
this.journal?.step({
|
|
2936
|
+
stage: 'nlu',
|
|
2937
|
+
level: 'warn',
|
|
2938
|
+
title: `NLU gave up, the request goes on: ${e.message}`,
|
|
2939
|
+
data: (0, journal_1.errorInfo)(e),
|
|
2940
|
+
});
|
|
2416
2941
|
}
|
|
2417
2942
|
}
|
|
2943
|
+
else {
|
|
2944
|
+
this.journal?.step({ stage: 'nlu', title: 'Rule-based NLU is switched off (useLocalNlu)' });
|
|
2945
|
+
}
|
|
2418
2946
|
// Current weather from the configured adapter — prepended to the user turn of BOTH LLM tiers so
|
|
2419
2947
|
// "how's the weather?" is answered from real data (the local model has no tools and would otherwise
|
|
2420
2948
|
// invent a forecast; the cloud model saves a get_weather round-trip). Cached, see buildWeatherContext.
|
|
2421
2949
|
const weather = await this.buildWeatherContext();
|
|
2422
2950
|
// Tier 1a: local LLM — answers general questions offline; emits HANDOFF for anything needing tools.
|
|
2951
|
+
// With a cloud LLM it only goes first when asked to (`localLlmFirst`): on a Raspberry Pi a small model
|
|
2952
|
+
// needs 10–25 s, often only to hand off, and the cloud then starts from zero. By default it is the
|
|
2953
|
+
// fallback for when the cloud fails (see below).
|
|
2423
2954
|
// Skipped inside an ongoing conversation (history present): the tool-free local model gets no history,
|
|
2424
2955
|
// so it would answer a follow-up like "yes" context-free and swallow it — let the cloud LLM (which has
|
|
2425
2956
|
// the thread and the tools) handle follow-ups instead. Without a cloud LLM there is nobody to hand
|
|
2426
2957
|
// follow-ups to, so the local model takes them too.
|
|
2427
|
-
|
|
2428
|
-
|
|
2429
|
-
|
|
2430
|
-
|
|
2431
|
-
|
|
2432
|
-
|
|
2433
|
-
|
|
2434
|
-
this.log.info(`Answered by local LLM (source='${source}').`);
|
|
2435
|
-
this.journal?.note({ tier: 'localLlm', detail: this.localLlmName() });
|
|
2436
|
-
return ans;
|
|
2437
|
-
}
|
|
2438
|
-
this.log.debug('Local LLM handed off to the cloud LLM.');
|
|
2439
|
-
}
|
|
2440
|
-
catch (e) {
|
|
2441
|
-
this.log.debug(`Local LLM error: ${e.message}`);
|
|
2958
|
+
const useLocal = !!(this.config.useLocalLlm && this.localLlm);
|
|
2959
|
+
const cloud = !!(this.agent || this.piAgent);
|
|
2960
|
+
const localFirst = useLocal && (!cloud || !!this.config.localLlmFirst);
|
|
2961
|
+
if (localFirst && (!history.length || !cloud)) {
|
|
2962
|
+
const ans = await this.askLocalLlm(question, weather, source);
|
|
2963
|
+
if (ans) {
|
|
2964
|
+
return ans;
|
|
2442
2965
|
}
|
|
2443
2966
|
}
|
|
2444
|
-
else if (
|
|
2967
|
+
else if (localFirst) {
|
|
2445
2968
|
this.log.debug('Local LLM skipped (follow-up in an ongoing conversation → cloud LLM).');
|
|
2969
|
+
this.journal?.step({
|
|
2970
|
+
stage: 'localLlm',
|
|
2971
|
+
title: 'Local LLM skipped: follow-up in an ongoing conversation goes to the cloud LLM',
|
|
2972
|
+
});
|
|
2973
|
+
}
|
|
2974
|
+
else if (useLocal) {
|
|
2975
|
+
this.journal?.step({
|
|
2976
|
+
stage: 'localLlm',
|
|
2977
|
+
title: 'Local LLM not asked first: it is the fallback for when the cloud LLM fails (localLlmFirst)',
|
|
2978
|
+
});
|
|
2446
2979
|
}
|
|
2447
2980
|
// Tier 2: cloud LLM with full tool access. Inject a compact device list into the system prompt so
|
|
2448
2981
|
// the model can act without a first `list_devices` round-trip (kept cached via prompt caching).
|
|
2982
|
+
if (this.piAgent) {
|
|
2983
|
+
return this.askPiAgent(this.piAgent, question, weather, source, useLocal && !localFirst);
|
|
2984
|
+
}
|
|
2449
2985
|
if (!this.agent) {
|
|
2450
2986
|
// Provider "none": no tier further down. Say so instead of failing — over voice an error is
|
|
2451
2987
|
// read out as a technical message.
|
|
2452
2988
|
this.log.info(`No local tier could answer and no LLM provider is configured (source='${source}').`);
|
|
2453
2989
|
this.journal?.note({ tier: 'none' });
|
|
2990
|
+
this.journal?.step({
|
|
2991
|
+
stage: 'none',
|
|
2992
|
+
level: 'warn',
|
|
2993
|
+
title: 'Nothing local understood the request and no LLM provider is configured',
|
|
2994
|
+
});
|
|
2454
2995
|
return this.localOnlyFallback();
|
|
2455
2996
|
}
|
|
2456
2997
|
const ctx = await this.buildDeviceContext();
|
|
2457
2998
|
const mem = this.buildMemoryContext();
|
|
2458
2999
|
// Only ask the model to emit the follow-up marker when satellites can actually re-open the mic.
|
|
2459
3000
|
const followUp = this.config.voiceEnabled ? this.followUpHint() : '';
|
|
2460
|
-
const
|
|
3001
|
+
const done = this.doneHint();
|
|
3002
|
+
const parts = [this.config.systemPrompt || '', followUp, done, mem, ctx].filter(Boolean);
|
|
2461
3003
|
const sys = parts.length ? parts.join('\n\n') : undefined;
|
|
2462
3004
|
this.log.info(`Cloud LLM [${this.agent.model || 'auto'}] (source='${source}', context=${history.length} turn(s)${mem ? ', memory' : ''}${weather ? ', weather' : ''}${this.presence?.configured ? ', presence' : ''}) — running tool loop…`);
|
|
2463
3005
|
// Prepend the current date/time (and the weather line, if a source is configured) to the user turn —
|
|
2464
3006
|
// cache-safe, see buildTimeContext. The stored history keeps the untouched question.
|
|
2465
|
-
const
|
|
3007
|
+
const time = this.buildTimeContext();
|
|
3008
|
+
const questionForLlm = [time, weather, question].filter(Boolean).join('\n\n');
|
|
2466
3009
|
this.journal?.note({
|
|
2467
3010
|
tier: 'cloud',
|
|
2468
|
-
detail: `${this.config.provider || 'openai'} · ${this.agent.model || 'auto'}`,
|
|
3011
|
+
detail: `${this.llm?.provider || this.config.provider || 'openai'} · ${this.agent.model || 'auto'}`,
|
|
3012
|
+
});
|
|
3013
|
+
// What went into the prompt, piece by piece. The device list is only counted: the model's opening
|
|
3014
|
+
// step carries the whole system prompt, and with it the list.
|
|
3015
|
+
const deviceLines = ctx ? ctx.split('\n').length - 1 : 0;
|
|
3016
|
+
this.journal?.step({
|
|
3017
|
+
stage: 'context',
|
|
3018
|
+
title: `Prompt: ${deviceLines} devices${mem ? ', memory' : ''}${weather ? ', weather' : ''}${followUp ? ', follow-up marker' : ''}`,
|
|
3019
|
+
data: {
|
|
3020
|
+
systemPrompt: this.config.systemPrompt || '',
|
|
3021
|
+
followUpHint: followUp || undefined,
|
|
3022
|
+
doneHint: done || undefined,
|
|
3023
|
+
memory: mem || undefined,
|
|
3024
|
+
devices: deviceLines,
|
|
3025
|
+
time,
|
|
3026
|
+
weather: weather || undefined,
|
|
3027
|
+
},
|
|
2469
3028
|
});
|
|
2470
|
-
|
|
3029
|
+
try {
|
|
3030
|
+
return await this.agent.ask(questionForLlm, sys, history);
|
|
3031
|
+
}
|
|
3032
|
+
catch (e) {
|
|
3033
|
+
// Offline, quota, outage: the local model is the fallback — unless it was already asked above.
|
|
3034
|
+
if (useLocal && !localFirst) {
|
|
3035
|
+
this.log.warn(`Cloud LLM failed (${e.message}) — asking the local LLM instead.`);
|
|
3036
|
+
this.journal?.step({
|
|
3037
|
+
stage: 'localLlm',
|
|
3038
|
+
level: 'warn',
|
|
3039
|
+
title: `Cloud LLM failed (${e.message}) — asking the local LLM instead`,
|
|
3040
|
+
});
|
|
3041
|
+
const ans = await this.askLocalLlm(question, weather, source);
|
|
3042
|
+
if (ans) {
|
|
3043
|
+
return ans;
|
|
3044
|
+
}
|
|
3045
|
+
}
|
|
3046
|
+
throw e;
|
|
3047
|
+
}
|
|
3048
|
+
}
|
|
3049
|
+
/**
|
|
3050
|
+
* Cloud tier through the adapter pi-agent: one pi-agent conversation per source. The style, the
|
|
3051
|
+
* follow-up marker, the memory and the device list are the instructions of that conversation (pi-agent
|
|
3052
|
+
* re-configures it only when they change); time, weather and presence change every turn and go in front
|
|
3053
|
+
* of the question. The history stays in pi-agent, so nothing of {@link ConversationStore} is sent.
|
|
3054
|
+
*/
|
|
3055
|
+
async askPiAgent(harness, question, weather, source, localFallback) {
|
|
3056
|
+
const followUp = this.config.voiceEnabled ? this.followUpHint() : '';
|
|
3057
|
+
// The device list maps names to state ids, so pi-agent can act without searching first. It is the
|
|
3058
|
+
// first to go when the instructions get too long — pi-agent can still search.
|
|
3059
|
+
const { text: instructions, dropped } = (0, piAgent_1.composeInstructions)([this.config.systemPrompt || '', followUp, this.doneHint()], [this.buildMemoryContext(), await this.buildDeviceContext()]);
|
|
3060
|
+
if (dropped) {
|
|
3061
|
+
this.log.debug(`pi-agent: ${dropped} context block(s) left out, the instructions would be too long.`);
|
|
3062
|
+
}
|
|
3063
|
+
const text = [this.buildTimeContext(), weather, this.buildPresenceContext(), question]
|
|
3064
|
+
.filter(Boolean)
|
|
3065
|
+
.join('\n\n');
|
|
3066
|
+
this.log.info(`Cloud tier ${harness.instance} (source='${source}') — asking pi-agent…`);
|
|
3067
|
+
this.journal?.note({ tier: 'cloud', detail: harness.instance });
|
|
3068
|
+
try {
|
|
3069
|
+
return await harness.ask({ text, conversation: source, instructions });
|
|
3070
|
+
}
|
|
3071
|
+
catch (e) {
|
|
3072
|
+
if (localFallback) {
|
|
3073
|
+
this.log.warn(`pi-agent failed (${e.message}) — asking the local LLM instead.`);
|
|
3074
|
+
const ans = await this.askLocalLlm(question, weather, source);
|
|
3075
|
+
if (ans) {
|
|
3076
|
+
return ans;
|
|
3077
|
+
}
|
|
3078
|
+
}
|
|
3079
|
+
throw e;
|
|
3080
|
+
}
|
|
3081
|
+
}
|
|
3082
|
+
/**
|
|
3083
|
+
* Ask the local model, bounded by `localLlmTimeoutSec`. Returns its answer, or null when it handed off,
|
|
3084
|
+
* timed out or failed — the caller then goes on to the next tier.
|
|
3085
|
+
*/
|
|
3086
|
+
async askLocalLlm(question, weather, source) {
|
|
3087
|
+
if (!this.localLlm) {
|
|
3088
|
+
return null;
|
|
3089
|
+
}
|
|
3090
|
+
const timeoutMs = Math.max(1, Number(this.config.localLlmTimeoutSec) || LOCAL_LLM_TIMEOUT_SEC) * 1000;
|
|
3091
|
+
const name = this.localLlmName();
|
|
3092
|
+
// The local model has no tools: without these lines it would invent the weather and guess who is
|
|
3093
|
+
// at home.
|
|
3094
|
+
const facts = [weather, this.buildPresenceContext()].filter(Boolean).join('\n\n');
|
|
3095
|
+
const prompt = facts ? `${facts}\n\n${question}` : question;
|
|
3096
|
+
const busy = this.localLlm.running;
|
|
3097
|
+
const data = { model: name, timeoutMs, system: this.localLlm.systemPrompt(), prompt };
|
|
3098
|
+
const start = Date.now();
|
|
3099
|
+
try {
|
|
3100
|
+
const ans = await this.localLlm.ask(prompt, timeoutMs);
|
|
3101
|
+
const ms = Date.now() - start;
|
|
3102
|
+
if (ans && !(0, localLlm_1.isHandoff)(ans)) {
|
|
3103
|
+
this.log.info(`Answered by local LLM (source='${source}').`);
|
|
3104
|
+
this.journal?.note({ tier: 'localLlm', detail: name });
|
|
3105
|
+
this.journal?.step({
|
|
3106
|
+
stage: 'localLlm',
|
|
3107
|
+
title: `Local LLM ${name}: answered`,
|
|
3108
|
+
ms,
|
|
3109
|
+
data: { ...data, answer: ans },
|
|
3110
|
+
});
|
|
3111
|
+
return ans;
|
|
3112
|
+
}
|
|
3113
|
+
this.log.debug('Local LLM handed off.');
|
|
3114
|
+
this.journal?.step({
|
|
3115
|
+
stage: 'localLlm',
|
|
3116
|
+
title: busy
|
|
3117
|
+
? `Local LLM ${name}: busy with another question → handed off`
|
|
3118
|
+
: `Local LLM ${name}: handed off`,
|
|
3119
|
+
ms,
|
|
3120
|
+
data: { ...data, answer: ans },
|
|
3121
|
+
});
|
|
3122
|
+
}
|
|
3123
|
+
catch (e) {
|
|
3124
|
+
this.log.info(`Local LLM gave no answer: ${e.message}`);
|
|
3125
|
+
this.journal?.step({
|
|
3126
|
+
stage: 'localLlm',
|
|
3127
|
+
level: 'warn',
|
|
3128
|
+
title: `Local LLM ${name}: ${e.message}`,
|
|
3129
|
+
ms: Date.now() - start,
|
|
3130
|
+
data: { ...data, error: (0, journal_1.errorInfo)(e) },
|
|
3131
|
+
});
|
|
3132
|
+
}
|
|
3133
|
+
return null;
|
|
2471
3134
|
}
|
|
2472
3135
|
/** Reply when nothing local understood the request and there is no cloud LLM (provider "none"). */
|
|
2473
3136
|
localOnlyFallback() {
|
|
@@ -2506,10 +3169,11 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
2506
3169
|
const room = d.room ? `${d.room}, ` : '';
|
|
2507
3170
|
lines.push(`- ${d.name} (${room}${d.type}): ${ids.join(', ')}`);
|
|
2508
3171
|
}
|
|
3172
|
+
const lang = String(this.config.voiceLanguage || this.language || 'en');
|
|
3173
|
+
lines.push(...this.satelliteVolumeLines(lang));
|
|
2509
3174
|
if (!lines.length) {
|
|
2510
3175
|
return '';
|
|
2511
3176
|
}
|
|
2512
|
-
const lang = String(this.config.voiceLanguage || this.language || 'en');
|
|
2513
3177
|
const header = lang === 'ru'
|
|
2514
3178
|
? 'Известные устройства — «имя (комната, тип): stateId». Используй эти stateId напрямую с set_state/get_states; вызывай list_devices только если нужного устройства здесь нет.'
|
|
2515
3179
|
: lang === 'de'
|
|
@@ -2517,6 +3181,41 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
2517
3181
|
: 'Known devices — "name (room, type): stateId". Use these stateIds directly with set_state/get_states; only call list_devices if a needed device is missing here.';
|
|
2518
3182
|
return `${header}\n${lines.join('\n')}`;
|
|
2519
3183
|
}
|
|
3184
|
+
/**
|
|
3185
|
+
* The playback volume of every voice satellite as a line of the device list. Without it, "Lautstärke auf 50
|
|
3186
|
+
* Prozent" that the rules miss reaches the LLM (or pi-agent) with only the media devices of the room in view, and
|
|
3187
|
+
* the model turns those up instead of the speaker that heard the request. The scales differ per transport.
|
|
3188
|
+
*/
|
|
3189
|
+
satelliteVolumeLines(lang) {
|
|
3190
|
+
const label = (satId) => lang === 'ru'
|
|
3191
|
+
? `Громкость голосового спутника ${satId}`
|
|
3192
|
+
: lang === 'de'
|
|
3193
|
+
? `Lautstärke des Sprachsatelliten ${satId}`
|
|
3194
|
+
: `Volume of the voice satellite ${satId}`;
|
|
3195
|
+
const lines = [];
|
|
3196
|
+
const seen = new Set();
|
|
3197
|
+
for (const [satId, instance] of this.nativeSatFrom) {
|
|
3198
|
+
seen.add(satId);
|
|
3199
|
+
lines.push(`- ${label(satId)} (${satId}, level.volume 0-100): ${instance}.volume`);
|
|
3200
|
+
}
|
|
3201
|
+
for (const [satId, device] of this.satDeviceById) {
|
|
3202
|
+
if (seen.has(satId)) {
|
|
3203
|
+
continue;
|
|
3204
|
+
}
|
|
3205
|
+
const player = this.esphome?.entities(device).find(e => e.kind === 'mediaPlayer');
|
|
3206
|
+
if (player) {
|
|
3207
|
+
seen.add(satId);
|
|
3208
|
+
const id = `${this.namespace}.satellites.${satId}.controls.${player.objectId}.volume`;
|
|
3209
|
+
lines.push(`- ${label(satId)} (${satId}, level.volume 0-1): ${id}`);
|
|
3210
|
+
}
|
|
3211
|
+
}
|
|
3212
|
+
for (const satId of this.satVolumeEnsured) {
|
|
3213
|
+
if (!seen.has(satId)) {
|
|
3214
|
+
lines.push(`- ${label(satId)} (${satId}, level.volume 0-100): ${this.namespace}.satellites.${satId}.volume`);
|
|
3215
|
+
}
|
|
3216
|
+
}
|
|
3217
|
+
return lines;
|
|
3218
|
+
}
|
|
2520
3219
|
/**
|
|
2521
3220
|
* List a port we listen on in the host's used-resources registry (js-controller ≥ 8), so the admin shows it
|
|
2522
3221
|
* as occupied when someone configures another instance. `common.declareUsedResources` is set because the
|
|
@@ -2578,35 +3277,27 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
2578
3277
|
else {
|
|
2579
3278
|
// Switched off: do not keep what was recorded while it was on.
|
|
2580
3279
|
await fs.promises.rm(file, { force: true }).catch(() => { });
|
|
3280
|
+
await fs.promises.rm((0, journal_1.traceDirFor)(file), { recursive: true, force: true }).catch(() => { });
|
|
2581
3281
|
}
|
|
2582
3282
|
}
|
|
2583
|
-
/**
|
|
3283
|
+
/**
|
|
3284
|
+
* Wrap the MCP connection so every state the pipeline reads or writes lands in the journal entry — and a
|
|
3285
|
+
* call the NLU makes directly, with its result, in the request's trace.
|
|
3286
|
+
*/
|
|
2584
3287
|
journalMcp(mcp) {
|
|
2585
3288
|
return {
|
|
2586
3289
|
listTools: () => mcp.listTools(),
|
|
2587
|
-
callTool: (name, args) =>
|
|
2588
|
-
this.journal
|
|
2589
|
-
|
|
2590
|
-
},
|
|
3290
|
+
callTool: (name, args) => this.journal
|
|
3291
|
+
? this.journal.stateCall(name, args, () => mcp.callTool(name, args))
|
|
3292
|
+
: mcp.callTool(name, args),
|
|
2591
3293
|
close: () => mcp.close(),
|
|
2592
3294
|
};
|
|
2593
3295
|
}
|
|
2594
|
-
/** Wrap an LLM tool so its calls (arguments, result, duration) land in the journal entry. */
|
|
3296
|
+
/** Wrap an LLM tool so its calls (arguments, result, duration) land in the journal entry and its trace. */
|
|
2595
3297
|
journalTool(tool) {
|
|
2596
3298
|
return {
|
|
2597
3299
|
...tool,
|
|
2598
|
-
run:
|
|
2599
|
-
const start = Date.now();
|
|
2600
|
-
try {
|
|
2601
|
-
const result = await tool.run(args);
|
|
2602
|
-
this.journal?.tool(tool.name, args, Date.now() - start, result);
|
|
2603
|
-
return result;
|
|
2604
|
-
}
|
|
2605
|
-
catch (e) {
|
|
2606
|
-
this.journal?.tool(tool.name, args, Date.now() - start, undefined, e.message);
|
|
2607
|
-
throw e;
|
|
2608
|
-
}
|
|
2609
|
-
},
|
|
3300
|
+
run: args => (this.journal ? this.journal.toolCall(tool.name, args, () => tool.run(args)) : tool.run(args)),
|
|
2610
3301
|
};
|
|
2611
3302
|
}
|
|
2612
3303
|
/** Short name of the local model for the journal ("qwen2.5-1.5b-instruct-q4_k_m"). */
|
|
@@ -2704,7 +3395,7 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
2704
3395
|
dataDir,
|
|
2705
3396
|
modelUrl: (this.config.localLlmModelUrl || '').trim() || localLlm_1.DEFAULT_LOCAL_MODEL_URL,
|
|
2706
3397
|
systemPrompt: this.config.systemPrompt || '',
|
|
2707
|
-
maxTokens: this.config.maxTokens ||
|
|
3398
|
+
maxTokens: Math.min(this.config.maxTokens || LOCAL_LLM_MAX_TOKENS, LOCAL_LLM_MAX_TOKENS),
|
|
2708
3399
|
log: this.log,
|
|
2709
3400
|
});
|
|
2710
3401
|
this.lastLlmPct = -1;
|
|
@@ -2737,20 +3428,56 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
2737
3428
|
}
|
|
2738
3429
|
// Timer intents match device-independently (parse() checks them first), so build the NLU even
|
|
2739
3430
|
// when no devices are known and always try timers before bailing out.
|
|
3431
|
+
const start = Date.now();
|
|
2740
3432
|
const { rooms, devices } = await this.getNluDevices();
|
|
2741
|
-
const
|
|
2742
|
-
|
|
3433
|
+
const aliases = this.getNluAliases();
|
|
3434
|
+
const intents = new nlu_1.Nlu(rooms, devices, aliases, {
|
|
3435
|
+
weather: !!(this.config.weatherInstance || '').trim(),
|
|
3436
|
+
}).parseAll(question);
|
|
3437
|
+
const blockers = intents.map(i => this.nluBlocker(i, devices.length));
|
|
3438
|
+
this.journal?.step({
|
|
3439
|
+
stage: 'nlu',
|
|
3440
|
+
title: !intents.length
|
|
3441
|
+
? `NLU: nothing recognised (${devices.length} devices, ${rooms.length} rooms known)`
|
|
3442
|
+
: blockers.some(Boolean)
|
|
3443
|
+
? `NLU: ${intents.map(i => i.action).join(' + ')} — cannot run here: ${blockers.filter(Boolean).join('; ')}`
|
|
3444
|
+
: `NLU: ${intents.map(describeIntent).join(' + ')}`,
|
|
3445
|
+
level: intents.length && blockers.some(Boolean) ? 'warn' : undefined,
|
|
3446
|
+
ms: Date.now() - start,
|
|
3447
|
+
data: {
|
|
3448
|
+
devices: devices.length,
|
|
3449
|
+
rooms,
|
|
3450
|
+
aliases: aliases.length ? aliases : undefined,
|
|
3451
|
+
intents: intents.map((intent, i) => ({ ...intent, blockedBecause: blockers[i] || undefined })),
|
|
3452
|
+
},
|
|
3453
|
+
});
|
|
3454
|
+
if (!intents.length || blockers.some(Boolean)) {
|
|
2743
3455
|
return null; // nothing matched, or one part cannot run here → let the LLM answer the whole thing
|
|
2744
3456
|
}
|
|
2745
3457
|
this.journal?.note({ tier: 'nlu', detail: intents.map(i => i.action).join(' + ') });
|
|
2746
3458
|
const answers = [];
|
|
2747
3459
|
let failed = false;
|
|
2748
3460
|
for (const intent of intents) {
|
|
3461
|
+
const started = Date.now();
|
|
2749
3462
|
try {
|
|
2750
|
-
|
|
3463
|
+
const answer = await this.executeNluIntent(intent, source);
|
|
3464
|
+
answers.push(answer);
|
|
3465
|
+
this.journal?.step({
|
|
3466
|
+
stage: 'nlu',
|
|
3467
|
+
title: `Executed ${describeIntent(intent)}`,
|
|
3468
|
+
ms: Date.now() - started,
|
|
3469
|
+
data: { answer },
|
|
3470
|
+
});
|
|
2751
3471
|
}
|
|
2752
3472
|
catch (e) {
|
|
2753
3473
|
failed = true;
|
|
3474
|
+
this.journal?.step({
|
|
3475
|
+
stage: 'nlu',
|
|
3476
|
+
level: 'error',
|
|
3477
|
+
title: `${describeIntent(intent)} failed: ${e.message}${answers.length ? '' : ' → the LLM takes over'}`,
|
|
3478
|
+
ms: Date.now() - started,
|
|
3479
|
+
data: (0, journal_1.errorInfo)(e),
|
|
3480
|
+
});
|
|
2754
3481
|
// Nothing executed yet → hand the whole utterance to the LLM. Afterwards a state has already
|
|
2755
3482
|
// been written and the LLM would repeat it, so report the failed part instead of bailing out.
|
|
2756
3483
|
if (!answers.length) {
|
|
@@ -2770,38 +3497,51 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
2770
3497
|
if (this.config.confirmWithTone &&
|
|
2771
3498
|
answer &&
|
|
2772
3499
|
!failed &&
|
|
2773
|
-
|
|
3500
|
+
// a command for later says when it will run — a beep would hide that
|
|
3501
|
+
intents.every(i => CONTROL_ACTIONS.has(i.action) && !isDeferred(i)) &&
|
|
2774
3502
|
(await this.playConfirmationTone(source))) {
|
|
2775
3503
|
this.log.debug(`NLU confirmed with a tone instead of "${answer}".`);
|
|
3504
|
+
this.journal?.step({ stage: 'nlu', title: 'Confirmed with a tone instead of words', data: { answer } });
|
|
2776
3505
|
return '';
|
|
2777
3506
|
}
|
|
2778
3507
|
return answer;
|
|
2779
3508
|
}
|
|
2780
3509
|
/**
|
|
2781
|
-
*
|
|
2782
|
-
* is executed: a missing manager, no known devices or disabled writes send the whole
|
|
2783
|
-
* LLM (which can explain why) instead of silently doing nothing — or only half of it.
|
|
3510
|
+
* Why this intent cannot run locally ('' = it can). Checked for **every** part of a combined command
|
|
3511
|
+
* before anything is executed: a missing manager, no known devices or disabled writes send the whole
|
|
3512
|
+
* utterance to the LLM (which can explain why) instead of silently doing nothing — or only half of it.
|
|
2784
3513
|
*/
|
|
2785
|
-
|
|
3514
|
+
nluBlocker(intent, deviceCount) {
|
|
2786
3515
|
switch (intent.action) {
|
|
2787
3516
|
case 'timerSet':
|
|
2788
3517
|
case 'timerQuery':
|
|
2789
3518
|
case 'timerCancel':
|
|
2790
|
-
return
|
|
3519
|
+
return this.timers ? '' : 'timers are not running';
|
|
2791
3520
|
case 'alarmSet':
|
|
2792
3521
|
case 'alarmQuery':
|
|
2793
3522
|
case 'alarmCancel':
|
|
2794
|
-
return
|
|
3523
|
+
return this.alarms ? '' : 'alarms are not running';
|
|
2795
3524
|
case 'timeQuery':
|
|
2796
3525
|
case 'dateQuery':
|
|
2797
|
-
return
|
|
3526
|
+
return '';
|
|
3527
|
+
case 'weatherQuery':
|
|
3528
|
+
return (this.config.weatherInstance || '').trim() ? '' : 'no weather instance configured';
|
|
3529
|
+
case 'volumeSet':
|
|
3530
|
+
return this.voice || this.esphome || this.nativeSatFrom.size ? '' : 'no satellites';
|
|
2798
3531
|
default:
|
|
2799
3532
|
// Device intents; writes additionally require the coarse "allow control" toggle.
|
|
2800
|
-
|
|
2801
|
-
|
|
3533
|
+
if (!deviceCount) {
|
|
3534
|
+
return 'no devices known';
|
|
3535
|
+
}
|
|
3536
|
+
if (isDeferred(intent) && !(this.config.calendarInstance || '').trim()) {
|
|
3537
|
+
return 'a command for later needs a calendar (fullcalendar) instance in the settings';
|
|
3538
|
+
}
|
|
3539
|
+
return ['on', 'off', 'level', 'color'].includes(intent.action) && !this.config.allowWriteStates
|
|
3540
|
+
? 'writing states is not allowed (allowWriteStates)'
|
|
3541
|
+
: '';
|
|
2802
3542
|
}
|
|
2803
3543
|
}
|
|
2804
|
-
/** Execute one intent that {@link
|
|
3544
|
+
/** Execute one intent that {@link nluBlocker} accepted, and return its spoken answer. */
|
|
2805
3545
|
async executeNluIntent(intent, source) {
|
|
2806
3546
|
switch (intent.action) {
|
|
2807
3547
|
case 'timerSet':
|
|
@@ -2815,9 +3555,314 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
2815
3555
|
case 'timeQuery':
|
|
2816
3556
|
case 'dateQuery':
|
|
2817
3557
|
return this.executeTimeIntent(intent);
|
|
3558
|
+
case 'weatherQuery':
|
|
3559
|
+
return this.executeWeatherIntent(intent);
|
|
3560
|
+
case 'volumeSet':
|
|
3561
|
+
return this.executeVolumeIntent(intent, source);
|
|
2818
3562
|
default:
|
|
2819
|
-
return this.executeIntent(intent);
|
|
3563
|
+
return isDeferred(intent) ? this.executeDeferredIntent(intent) : this.executeIntent(intent);
|
|
3564
|
+
}
|
|
3565
|
+
}
|
|
3566
|
+
/**
|
|
3567
|
+
* Send a message to the calendar instance (fullcalendar). Its answer `{error}` becomes an exception.
|
|
3568
|
+
*
|
|
3569
|
+
* @param command `addEvent`, `listEvents`, `deleteEvent`, …
|
|
3570
|
+
* @param message the parameters
|
|
3571
|
+
*/
|
|
3572
|
+
async calendarCall(command, message) {
|
|
3573
|
+
const instance = (this.config.calendarInstance || '').trim().replace(/^system\.adapter\./, '');
|
|
3574
|
+
if (!instance) {
|
|
3575
|
+
throw new Error('No calendar instance configured in the assistant settings');
|
|
3576
|
+
}
|
|
3577
|
+
const alive = await this.getForeignStateAsync(`system.adapter.${instance}.alive`);
|
|
3578
|
+
if (!alive?.val) {
|
|
3579
|
+
throw new Error(`The calendar ${instance} is not running`);
|
|
3580
|
+
}
|
|
3581
|
+
const res = (await this.sendToAsync(instance, command, message, { timeout: 10000 }));
|
|
3582
|
+
if (!res) {
|
|
3583
|
+
throw new Error(`The calendar ${instance} did not answer`);
|
|
3584
|
+
}
|
|
3585
|
+
if (res.error) {
|
|
3586
|
+
throw new Error(res.error);
|
|
3587
|
+
}
|
|
3588
|
+
return res;
|
|
3589
|
+
}
|
|
3590
|
+
/**
|
|
3591
|
+
* Whether the per-device ACL forbids writing this state: an explicit entry, else the default of the device
|
|
3592
|
+
* type (locks are read-only). States outside the device list are not restricted (like `set_state`).
|
|
3593
|
+
*/
|
|
3594
|
+
async writeForbidden(stateId) {
|
|
3595
|
+
const { devices } = await this.getNluDevices();
|
|
3596
|
+
const device = devices.find(d => Object.values(d.controls).includes(stateId));
|
|
3597
|
+
if (!device) {
|
|
3598
|
+
return false;
|
|
3599
|
+
}
|
|
3600
|
+
const entry = this.config.deviceAcl?.[(0, tools_1.deviceKey)(Object.values(device.controls))];
|
|
3601
|
+
return entry ? entry.write === false : device.type === 'lock';
|
|
3602
|
+
}
|
|
3603
|
+
/** "22:30", "morgen um 22:30", "30.10. um 7:00" — when a planned command runs, for the spoken answer */
|
|
3604
|
+
describeWhen(time, lang) {
|
|
3605
|
+
const pad = (n) => String(n).padStart(2, '0');
|
|
3606
|
+
const clock = `${time.getHours()}:${pad(time.getMinutes())}`;
|
|
3607
|
+
const now = new Date();
|
|
3608
|
+
const days = Math.round((new Date(time.getFullYear(), time.getMonth(), time.getDate()).getTime() -
|
|
3609
|
+
new Date(now.getFullYear(), now.getMonth(), now.getDate()).getTime()) /
|
|
3610
|
+
86_400_000);
|
|
3611
|
+
const date = `${pad(time.getDate())}.${pad(time.getMonth() + 1)}.`;
|
|
3612
|
+
if (lang === 'ru') {
|
|
3613
|
+
return days === 0 ? `в ${clock}` : days === 1 ? `завтра в ${clock}` : `${date} в ${clock}`;
|
|
3614
|
+
}
|
|
3615
|
+
if (lang === 'de') {
|
|
3616
|
+
return days === 0 ? `um ${clock}` : days === 1 ? `morgen um ${clock}` : `am ${date} um ${clock}`;
|
|
3617
|
+
}
|
|
3618
|
+
return days === 0 ? `at ${clock}` : days === 1 ? `tomorrow at ${clock}` : `on ${date} at ${clock}`;
|
|
3619
|
+
}
|
|
3620
|
+
/**
|
|
3621
|
+
* A device command for later ("Licht in 2 Stunden aus", "выключи свет через 2 часа"): one event in the calendar
|
|
3622
|
+
* (fullcalendar) per write — the calendar shows it, survives restarts and can be cancelled there.
|
|
3623
|
+
*/
|
|
3624
|
+
async executeDeferredIntent(intent) {
|
|
3625
|
+
const device = intent.device;
|
|
3626
|
+
if (!device || !intent.stateId) {
|
|
3627
|
+
throw new Error('deferred intent without a device');
|
|
3628
|
+
}
|
|
3629
|
+
const lang = String(this.config.voiceLanguage || this.language || 'en');
|
|
3630
|
+
const pick = (rus, ger, eng) => lang === 'ru' ? rus : lang === 'de' ? ger : eng;
|
|
3631
|
+
const room = intent.room || device.room;
|
|
3632
|
+
const where = room ? ` (${room})` : '';
|
|
3633
|
+
const dev = device.name;
|
|
3634
|
+
if (await this.writeForbidden(intent.stateId)) {
|
|
3635
|
+
return pick(`${dev}${where} только для чтения.`, `${dev}${where} ist schreibgeschützt.`, `${dev}${where} is read-only.`);
|
|
3636
|
+
}
|
|
3637
|
+
let time;
|
|
3638
|
+
if (intent.delaySec !== undefined) {
|
|
3639
|
+
time = new Date(Date.now() + intent.delaySec * 1000);
|
|
3640
|
+
}
|
|
3641
|
+
else {
|
|
3642
|
+
const { hour, minute, day } = intent.at;
|
|
3643
|
+
const now = new Date();
|
|
3644
|
+
time = new Date(now.getFullYear(), now.getMonth(), now.getDate() + (day ?? 0), hour, minute, 0, 0);
|
|
3645
|
+
if (time.getTime() <= now.getTime()) {
|
|
3646
|
+
if (day !== undefined) {
|
|
3647
|
+
return pick(`${this.describeWhen(time, lang)} уже прошло.`, `${this.describeWhen(time, lang).replace(/^um /, '')} ist schon vorbei.`, `${this.describeWhen(time, lang).replace(/^at /, '')} has already passed.`);
|
|
3648
|
+
}
|
|
3649
|
+
time.setDate(time.getDate() + 1); // "um 7 Uhr" after 7 o'clock: tomorrow
|
|
3650
|
+
}
|
|
3651
|
+
}
|
|
3652
|
+
const name = `${dev}${where}`;
|
|
3653
|
+
await this.calendarCall('addEvent', { oid: intent.stateId, value: intent.value, start: time.getTime(), name });
|
|
3654
|
+
// a dimmer with its own switch: the switch goes into the calendar, too
|
|
3655
|
+
if (intent.also) {
|
|
3656
|
+
await this.calendarCall('addEvent', {
|
|
3657
|
+
oid: intent.also.stateId,
|
|
3658
|
+
value: intent.also.value,
|
|
3659
|
+
start: time.getTime(),
|
|
3660
|
+
name,
|
|
3661
|
+
});
|
|
3662
|
+
}
|
|
3663
|
+
this.log.info(`Planned in the calendar: ${intent.stateId} = ${JSON.stringify(intent.value)} at ${time.toString()}`);
|
|
3664
|
+
const when = this.describeWhen(time, lang);
|
|
3665
|
+
switch (intent.action) {
|
|
3666
|
+
case 'on':
|
|
3667
|
+
return pick(`Хорошо, ${when} включу: ${dev}${where}.`, `Okay, ${dev}${where} wird ${when} eingeschaltet.`, `Okay, ${dev}${where} will be turned on ${when}.`);
|
|
3668
|
+
case 'off':
|
|
3669
|
+
return pick(`Хорошо, ${when} выключу: ${dev}${where}.`, `Okay, ${dev}${where} wird ${when} ausgeschaltet.`, `Okay, ${dev}${where} will be turned off ${when}.`);
|
|
3670
|
+
case 'level':
|
|
3671
|
+
return pick(`Хорошо, ${when} поставлю ${dev}${where} на ${String(intent.value)} %.`, `Okay, ${dev}${where} wird ${when} auf ${String(intent.value)} % gestellt.`, `Okay, ${dev}${where} will be set to ${String(intent.value)} % ${when}.`);
|
|
3672
|
+
default:
|
|
3673
|
+
return pick(`Хорошо, запланировано ${when}: ${dev}${where}.`, `Okay, für ${dev}${where} ${when} eingeplant.`, `Okay, planned for ${dev}${where} ${when}.`);
|
|
3674
|
+
}
|
|
3675
|
+
}
|
|
3676
|
+
/**
|
|
3677
|
+
* LLM tools for the calendar (fullcalendar): plan a state change for later, list what is planned, cancel it.
|
|
3678
|
+
* Planning respects the same write rules as `set_state` (the coarse switch and the per-device ACL).
|
|
3679
|
+
*/
|
|
3680
|
+
buildCalendarTools() {
|
|
3681
|
+
const fail = (e) => JSON.stringify({ ok: false, error: e.message });
|
|
3682
|
+
const tools = [
|
|
3683
|
+
{
|
|
3684
|
+
name: 'list_scheduled',
|
|
3685
|
+
description: 'List the planned events of the calendar (state changes at a time, also recurring ones), in local time of the house. Optional "days" ahead (default 1 = the next 24 hours) or a "from"/"to" range as local time "YYYY-MM-DDTHH:mm".',
|
|
3686
|
+
parameters: {
|
|
3687
|
+
type: 'object',
|
|
3688
|
+
properties: {
|
|
3689
|
+
days: { type: 'number', description: 'How many days ahead, default 1' },
|
|
3690
|
+
from: { type: 'string', description: 'Start, local time YYYY-MM-DDTHH:mm' },
|
|
3691
|
+
to: { type: 'string', description: 'End, local time YYYY-MM-DDTHH:mm' },
|
|
3692
|
+
},
|
|
3693
|
+
additionalProperties: false,
|
|
3694
|
+
},
|
|
3695
|
+
run: async (args) => {
|
|
3696
|
+
try {
|
|
3697
|
+
const res = await this.calendarCall('listEvents', {
|
|
3698
|
+
days: typeof args.days === 'number' ? args.days : 1,
|
|
3699
|
+
from: args.from,
|
|
3700
|
+
to: args.to,
|
|
3701
|
+
max: 50,
|
|
3702
|
+
});
|
|
3703
|
+
return JSON.stringify({ ok: true, events: res.events });
|
|
3704
|
+
}
|
|
3705
|
+
catch (e) {
|
|
3706
|
+
return fail(e);
|
|
3707
|
+
}
|
|
3708
|
+
},
|
|
3709
|
+
},
|
|
3710
|
+
];
|
|
3711
|
+
if (!this.config.allowWriteStates) {
|
|
3712
|
+
return tools;
|
|
2820
3713
|
}
|
|
3714
|
+
tools.push({
|
|
3715
|
+
name: 'schedule_state',
|
|
3716
|
+
description: 'Plan a state change for later in the calendar: write "value" to state "id" after "in_minutes" or at "at" (local time of the house, "YYYY-MM-DDTHH:mm"). Use it for "in 2 hours", "at 10 pm", "tomorrow morning" — never set_state for something meant for later. With "duration_minutes" the state gets "end_value" after that time (for a boolean the opposite by default). Tell the user the planned time.',
|
|
3717
|
+
parameters: {
|
|
3718
|
+
type: 'object',
|
|
3719
|
+
properties: {
|
|
3720
|
+
id: { type: 'string', description: 'State ID' },
|
|
3721
|
+
value: { type: ['boolean', 'number', 'string'], description: 'Value to write' },
|
|
3722
|
+
in_minutes: { type: 'number', description: 'Delay in minutes from now' },
|
|
3723
|
+
at: { type: 'string', description: 'Local time YYYY-MM-DDTHH:mm' },
|
|
3724
|
+
duration_minutes: { type: 'number', description: 'Optional: write end_value this much later' },
|
|
3725
|
+
end_value: { type: ['boolean', 'number', 'string'], description: 'Value after the duration' },
|
|
3726
|
+
name: {
|
|
3727
|
+
type: 'string',
|
|
3728
|
+
description: 'Short title for the calendar, e.g. "Licht Wohnzimmer aus"',
|
|
3729
|
+
},
|
|
3730
|
+
},
|
|
3731
|
+
required: ['id', 'value'],
|
|
3732
|
+
additionalProperties: false,
|
|
3733
|
+
},
|
|
3734
|
+
run: async (args) => {
|
|
3735
|
+
try {
|
|
3736
|
+
const id = typeof args.id === 'string' ? args.id : '';
|
|
3737
|
+
if (await this.writeForbidden(id)) {
|
|
3738
|
+
return JSON.stringify({
|
|
3739
|
+
ok: false,
|
|
3740
|
+
error: `Writing "${id}" is not allowed: the device is read-only in the assistant settings.`,
|
|
3741
|
+
});
|
|
3742
|
+
}
|
|
3743
|
+
if (args.in_minutes === undefined && !args.at) {
|
|
3744
|
+
return JSON.stringify({ ok: false, error: 'Give "in_minutes" or "at"' });
|
|
3745
|
+
}
|
|
3746
|
+
const res = await this.calendarCall('addEvent', {
|
|
3747
|
+
oid: id,
|
|
3748
|
+
value: args.value,
|
|
3749
|
+
in: typeof args.in_minutes === 'number' ? Math.round(args.in_minutes * 60) : undefined,
|
|
3750
|
+
start: args.in_minutes === undefined ? args.at : undefined,
|
|
3751
|
+
duration: typeof args.duration_minutes === 'number'
|
|
3752
|
+
? Math.round(args.duration_minutes * 60)
|
|
3753
|
+
: undefined,
|
|
3754
|
+
endValue: args.end_value,
|
|
3755
|
+
name: typeof args.name === 'string' ? args.name : undefined,
|
|
3756
|
+
});
|
|
3757
|
+
return JSON.stringify({ ok: true, eventId: res.id, start: res.start });
|
|
3758
|
+
}
|
|
3759
|
+
catch (e) {
|
|
3760
|
+
return fail(e);
|
|
3761
|
+
}
|
|
3762
|
+
},
|
|
3763
|
+
}, {
|
|
3764
|
+
name: 'cancel_scheduled',
|
|
3765
|
+
description: 'Delete a planned event from the calendar, by its "id" from list_scheduled or schedule_state.',
|
|
3766
|
+
parameters: {
|
|
3767
|
+
type: 'object',
|
|
3768
|
+
properties: { id: { type: 'string', description: 'Event ID' } },
|
|
3769
|
+
required: ['id'],
|
|
3770
|
+
additionalProperties: false,
|
|
3771
|
+
},
|
|
3772
|
+
run: async (args) => {
|
|
3773
|
+
try {
|
|
3774
|
+
await this.calendarCall('deleteEvent', { id: typeof args.id === 'string' ? args.id : '' });
|
|
3775
|
+
return JSON.stringify({ ok: true });
|
|
3776
|
+
}
|
|
3777
|
+
catch (e) {
|
|
3778
|
+
return fail(e);
|
|
3779
|
+
}
|
|
3780
|
+
},
|
|
3781
|
+
});
|
|
3782
|
+
return tools;
|
|
3783
|
+
}
|
|
3784
|
+
/**
|
|
3785
|
+
* Speak the weather straight from the configured adapter. Throws when the report has nothing for the
|
|
3786
|
+
* asked day (or the adapter has no mapper), so the question goes on to the LLM and its tool instead.
|
|
3787
|
+
*/
|
|
3788
|
+
async executeWeatherIntent(intent) {
|
|
3789
|
+
const res = await this.readWeather(this.config.weatherInstance || '');
|
|
3790
|
+
if (!res.report) {
|
|
3791
|
+
throw new Error(res.error || 'no weather report');
|
|
3792
|
+
}
|
|
3793
|
+
const lang = String(this.config.voiceLanguage || this.language || 'en');
|
|
3794
|
+
const text = (0, weather_1.buildWeatherAnswer)(res.report, lang, intent.when);
|
|
3795
|
+
if (!text) {
|
|
3796
|
+
throw new Error(`the weather report has no data for '${intent.when || 'now'}'`);
|
|
3797
|
+
}
|
|
3798
|
+
return text;
|
|
3799
|
+
}
|
|
3800
|
+
/**
|
|
3801
|
+
* Change the playback volume of a satellite: the one in the named room, else the one that heard the
|
|
3802
|
+
* command. Each transport has its own knob — the native satellite's `volume` state, the media player
|
|
3803
|
+
* of an ESPHome device, the `volume` message of a UDP satellite.
|
|
3804
|
+
*/
|
|
3805
|
+
async executeVolumeIntent(intent, source) {
|
|
3806
|
+
const lang = String(this.config.voiceLanguage || this.language || 'en');
|
|
3807
|
+
const satId = intent.room ? this.resolveSatelliteId(intent.room) : this.announceTargetForSource(source);
|
|
3808
|
+
const absolute = typeof intent.value === 'number' ? intent.value : null;
|
|
3809
|
+
const delta = intent.delta || 0;
|
|
3810
|
+
const next = (current) => Math.min(100, Math.max(0, Math.round(absolute ?? current + delta)));
|
|
3811
|
+
// 1 процент, 2 процента, 5 процентов (11–14 take the last form).
|
|
3812
|
+
const ruPercent = (n) => n % 100 > 10 && n % 100 < 20
|
|
3813
|
+
? 'процентов'
|
|
3814
|
+
: n % 10 === 1
|
|
3815
|
+
? 'процент'
|
|
3816
|
+
: n % 10 >= 2 && n % 10 <= 4
|
|
3817
|
+
? 'процента'
|
|
3818
|
+
: 'процентов';
|
|
3819
|
+
const confirm = (level) => lang === 'ru'
|
|
3820
|
+
? `Громкость ${level} ${ruPercent(level)}.`
|
|
3821
|
+
: lang === 'de'
|
|
3822
|
+
? `Lautstärke ${level} Prozent.`
|
|
3823
|
+
: `Volume ${level} percent.`;
|
|
3824
|
+
if (satId) {
|
|
3825
|
+
const device = this.satDeviceById.get(satId) || satId;
|
|
3826
|
+
const instance = this.nativeSatFrom.get(satId);
|
|
3827
|
+
if (instance) {
|
|
3828
|
+
const stateId = `${instance}.volume`;
|
|
3829
|
+
const current = Number((await this.getForeignStateAsync(stateId))?.val ?? 50);
|
|
3830
|
+
const level = next(Number.isFinite(current) ? current : 50);
|
|
3831
|
+
await this.setForeignStateAsync(stateId, level);
|
|
3832
|
+
return confirm(level);
|
|
3833
|
+
}
|
|
3834
|
+
const player = this.esphome?.entities(device).find(e => e.kind === 'mediaPlayer');
|
|
3835
|
+
if (player) {
|
|
3836
|
+
const current = (player.value?.volume ?? 0.5) * 100;
|
|
3837
|
+
const level = next(current);
|
|
3838
|
+
if (this.esphome.setEntity(device, player.objectId, level / 100)) {
|
|
3839
|
+
return confirm(level);
|
|
3840
|
+
}
|
|
3841
|
+
}
|
|
3842
|
+
// A UDP satellite works out a relative step itself (it knows its level) and reports the result.
|
|
3843
|
+
if (this.voice?.has(device)) {
|
|
3844
|
+
this.voice.volume(device, absolute !== null ? { level: absolute } : { delta });
|
|
3845
|
+
if (absolute !== null) {
|
|
3846
|
+
return confirm(next(0));
|
|
3847
|
+
}
|
|
3848
|
+
return lang === 'ru'
|
|
3849
|
+
? delta > 0
|
|
3850
|
+
? 'Сделала громче.'
|
|
3851
|
+
: 'Сделала тише.'
|
|
3852
|
+
: lang === 'de'
|
|
3853
|
+
? delta > 0
|
|
3854
|
+
? 'Etwas lauter.'
|
|
3855
|
+
: 'Etwas leiser.'
|
|
3856
|
+
: delta > 0
|
|
3857
|
+
? 'A bit louder.'
|
|
3858
|
+
: 'A bit quieter.';
|
|
3859
|
+
}
|
|
3860
|
+
}
|
|
3861
|
+
return lang === 'ru'
|
|
3862
|
+
? 'Я не знаю, громкость какой колонки изменить.'
|
|
3863
|
+
: lang === 'de'
|
|
3864
|
+
? 'Ich weiß nicht, bei welchem Lautsprecher ich die Lautstärke ändern soll.'
|
|
3865
|
+
: "I don't know which speaker's volume to change.";
|
|
2821
3866
|
}
|
|
2822
3867
|
/** Localized "this part did not work" sentence for a failed part of a combined command. */
|
|
2823
3868
|
nluFailureText(intent) {
|
|
@@ -2868,14 +3913,36 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
2868
3913
|
return [];
|
|
2869
3914
|
}
|
|
2870
3915
|
}
|
|
2871
|
-
/**
|
|
2872
|
-
|
|
3916
|
+
/**
|
|
3917
|
+
* Device model for the NLU: friendly name, room, type and controls (controlType → state id). Cached
|
|
3918
|
+
* for {@link NLU_DEVICES_TTL_MS}; requests arriving while it is being built wait for the same build.
|
|
3919
|
+
*/
|
|
3920
|
+
getNluDevices() {
|
|
2873
3921
|
if (!this.mcp) {
|
|
2874
|
-
return { rooms: [], devices: [] };
|
|
3922
|
+
return Promise.resolve({ rooms: [], devices: [] });
|
|
2875
3923
|
}
|
|
2876
3924
|
// Match names in the assistant's interaction language (voice language, else system) so a
|
|
2877
3925
|
// Russian command matches Russian smartNames.
|
|
2878
3926
|
const nluLang = (this.config.voiceLanguage || this.language || 'en');
|
|
3927
|
+
const now = Date.now();
|
|
3928
|
+
const hit = this.nluDevices;
|
|
3929
|
+
if (hit && hit.key === nluLang && now - hit.at < NLU_DEVICES_TTL_MS) {
|
|
3930
|
+
return hit.result;
|
|
3931
|
+
}
|
|
3932
|
+
const entry = { key: nluLang, at: now, result: this.buildNluDevices(nluLang) };
|
|
3933
|
+
this.nluDevices = entry;
|
|
3934
|
+
// An empty model is usually a failed build (the manager not ready yet) — don't keep it for a minute.
|
|
3935
|
+
void entry.result.then(r => {
|
|
3936
|
+
if (!r.devices.length && this.nluDevices === entry) {
|
|
3937
|
+
this.nluDevices = null;
|
|
3938
|
+
}
|
|
3939
|
+
});
|
|
3940
|
+
return entry.result;
|
|
3941
|
+
}
|
|
3942
|
+
async buildNluDevices(nluLang) {
|
|
3943
|
+
if (!this.mcp) {
|
|
3944
|
+
return { rooms: [], devices: [] };
|
|
3945
|
+
}
|
|
2879
3946
|
try {
|
|
2880
3947
|
const res = await this.mcp.callTool('list_devices', { language: nluLang });
|
|
2881
3948
|
const parsed = JSON.parse(res.text);
|
|
@@ -3507,16 +4574,30 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
3507
4574
|
if (!this.config.verifyWrites || !stateId) {
|
|
3508
4575
|
return true;
|
|
3509
4576
|
}
|
|
3510
|
-
const
|
|
4577
|
+
const start = Date.now();
|
|
4578
|
+
const deadline = start + ACK_TIMEOUT_MS;
|
|
3511
4579
|
let last;
|
|
3512
4580
|
while (Date.now() < deadline) {
|
|
3513
4581
|
await this.delay(ACK_POLL_MS);
|
|
3514
4582
|
last = await this.getForeignStateAsync(stateId).catch(() => null);
|
|
3515
4583
|
if (last?.ack && valuesMatch(last.val, expected)) {
|
|
3516
4584
|
this.log.debug(`write to ${stateId} confirmed by the device.`);
|
|
4585
|
+
this.journal?.step({
|
|
4586
|
+
stage: 'state',
|
|
4587
|
+
title: `${stateId} confirmed by the device`,
|
|
4588
|
+
ms: Date.now() - start,
|
|
4589
|
+
data: { expected, state: last },
|
|
4590
|
+
});
|
|
3517
4591
|
return true;
|
|
3518
4592
|
}
|
|
3519
4593
|
}
|
|
4594
|
+
this.journal?.step({
|
|
4595
|
+
stage: 'state',
|
|
4596
|
+
level: 'warn',
|
|
4597
|
+
title: `${stateId}: no confirmation within ${ACK_TIMEOUT_MS} ms`,
|
|
4598
|
+
ms: Date.now() - start,
|
|
4599
|
+
data: { expected, state: last },
|
|
4600
|
+
});
|
|
3520
4601
|
this.log.warn(`No confirmation for ${stateId} within ${ACK_TIMEOUT_MS} ms ` +
|
|
3521
4602
|
`(last: ${JSON.stringify(last?.val)}, ack=${String(last?.ack)}). ` +
|
|
3522
4603
|
'If this device never acknowledges, switch "Verify device feedback" off.');
|
|
@@ -4729,12 +5810,26 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
4729
5810
|
*/
|
|
4730
5811
|
async runRoutine(routine, source) {
|
|
4731
5812
|
const origin = this.announceTargetForSource(source);
|
|
4732
|
-
for (const action of routine.actions) {
|
|
5813
|
+
for (const [i, action] of routine.actions.entries()) {
|
|
5814
|
+
const start = Date.now();
|
|
4733
5815
|
try {
|
|
4734
5816
|
await this.runTriggerAction({ id: `routine:${routine.name}`, when: [] }, action, origin);
|
|
5817
|
+
this.journal?.step({
|
|
5818
|
+
stage: 'routine',
|
|
5819
|
+
title: `Routine action ${i + 1}/${routine.actions.length}`,
|
|
5820
|
+
ms: Date.now() - start,
|
|
5821
|
+
data: action,
|
|
5822
|
+
});
|
|
4735
5823
|
}
|
|
4736
5824
|
catch (e) {
|
|
4737
5825
|
this.log.warn(`Routine '${routine.name}': an action failed — ${e.message}`);
|
|
5826
|
+
this.journal?.step({
|
|
5827
|
+
stage: 'routine',
|
|
5828
|
+
level: 'error',
|
|
5829
|
+
title: `Routine action ${i + 1}/${routine.actions.length} failed: ${e.message}`,
|
|
5830
|
+
ms: Date.now() - start,
|
|
5831
|
+
data: { action, error: (0, journal_1.errorInfo)(e) },
|
|
5832
|
+
});
|
|
4738
5833
|
}
|
|
4739
5834
|
}
|
|
4740
5835
|
}
|
|
@@ -5274,13 +6369,52 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
5274
6369
|
return { ok: false, error: e.message };
|
|
5275
6370
|
}
|
|
5276
6371
|
}
|
|
5277
|
-
/**
|
|
5278
|
-
|
|
5279
|
-
|
|
5280
|
-
|
|
5281
|
-
|
|
6372
|
+
/**
|
|
6373
|
+
* The saved configuration with the speech settings of an unsaved form on top. Only the speech fields count,
|
|
6374
|
+
* and a key only when one was typed: protected fields never reach the browser, so they are empty there.
|
|
6375
|
+
* Without fallbacks: a test must show whether the chosen engine works, not whether the reserve does.
|
|
6376
|
+
*/
|
|
6377
|
+
withVoiceForm(form) {
|
|
6378
|
+
const cfg = { ...this.config, sttFallback: '', ttsFallback: '' };
|
|
6379
|
+
for (const key of VOICE_FORM_FIELDS) {
|
|
6380
|
+
const value = form[key];
|
|
6381
|
+
if (typeof value !== 'string' || (VOICE_FORM_SECRETS.includes(key) && !value.trim())) {
|
|
6382
|
+
continue;
|
|
6383
|
+
}
|
|
6384
|
+
cfg[key] = value;
|
|
6385
|
+
}
|
|
6386
|
+
return cfg;
|
|
6387
|
+
}
|
|
6388
|
+
/** Build the TTS engine of a configuration (throws when no voice key is available). */
|
|
6389
|
+
async buildTtsEngine(cfg = this.config) {
|
|
6390
|
+
const creds = await (0, credentials_1.resolveVoiceCredentials)(this, cfg, await this.speechLlmKey());
|
|
5282
6391
|
return (0, engines_1.createTtsEngine)(cfg.ttsProvider || 'openai', this.voiceContext(cfg, creds));
|
|
5283
6392
|
}
|
|
6393
|
+
/**
|
|
6394
|
+
* Speech recognition for the voice test of the settings: a recording of the browser, an uploaded file or the
|
|
6395
|
+
* test sentence the TTS just spoke. WAV is read directly, anything else goes through ffmpeg.
|
|
6396
|
+
*/
|
|
6397
|
+
async transcribeForTest(msg) {
|
|
6398
|
+
try {
|
|
6399
|
+
if (!msg.audio) {
|
|
6400
|
+
return { error: 'no audio' };
|
|
6401
|
+
}
|
|
6402
|
+
const cfg = msg.config ? this.withVoiceForm(msg.config) : this.config;
|
|
6403
|
+
const data = Buffer.from(msg.audio, 'base64');
|
|
6404
|
+
const { pcm, sampleRate } = (msg.format || 'wav') === 'wav' && data.toString('latin1', 0, 4) === 'RIFF'
|
|
6405
|
+
? (0, openrouter_1.pcmFromResponse)(data, '')
|
|
6406
|
+
: await this.decodeAudioBufferToPcm(data);
|
|
6407
|
+
const creds = await (0, credentials_1.resolveVoiceCredentials)(this, cfg, await this.speechLlmKey());
|
|
6408
|
+
const stt = (0, engines_1.createSttEngine)(cfg.sttProvider || 'openai', this.voiceContext(cfg, creds));
|
|
6409
|
+
const language = msg.language || cfg.voiceLanguage || this.language || '';
|
|
6410
|
+
const start = Date.now();
|
|
6411
|
+
const text = await stt.transcribe(pcm, sampleRate, language, await this.buildSttHints());
|
|
6412
|
+
return { text, ms: Date.now() - start, audioMs: Math.round((pcm.length / 2 / sampleRate) * 1000) };
|
|
6413
|
+
}
|
|
6414
|
+
catch (e) {
|
|
6415
|
+
return { error: e.message };
|
|
6416
|
+
}
|
|
6417
|
+
}
|
|
5284
6418
|
/** True if backend TTS can run (a voice key is configured) — drives the Chat play button. */
|
|
5285
6419
|
async isTtsAvailable() {
|
|
5286
6420
|
try {
|
|
@@ -5292,15 +6426,23 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
5292
6426
|
}
|
|
5293
6427
|
}
|
|
5294
6428
|
/** Synthesize `text` to a base64 WAV via the configured TTS engine (used by the Chat play button). */
|
|
5295
|
-
async synthesizeToWav(text, language) {
|
|
6429
|
+
async synthesizeToWav(text, language, cfg = this.config) {
|
|
5296
6430
|
if (!text.trim()) {
|
|
5297
6431
|
return { error: 'empty text' };
|
|
5298
6432
|
}
|
|
5299
6433
|
try {
|
|
5300
|
-
const tts = await this.buildTtsEngine();
|
|
5301
|
-
const lang = language ||
|
|
6434
|
+
const tts = await this.buildTtsEngine(cfg);
|
|
6435
|
+
const lang = language || cfg.voiceLanguage || this.language || '';
|
|
6436
|
+
const start = Date.now();
|
|
5302
6437
|
const { pcm, sampleRate } = await tts.synthesize(text, lang);
|
|
5303
|
-
return {
|
|
6438
|
+
return {
|
|
6439
|
+
audio: this.pcmToWav(pcm, sampleRate).toString('base64'),
|
|
6440
|
+
mime: 'audio/wav',
|
|
6441
|
+
sampleRate,
|
|
6442
|
+
// a wrong rate shows up as an implausible length of the spoken text
|
|
6443
|
+
audioMs: Math.round((pcm.length / 2 / sampleRate) * 1000),
|
|
6444
|
+
ms: Date.now() - start,
|
|
6445
|
+
};
|
|
5304
6446
|
}
|
|
5305
6447
|
catch (e) {
|
|
5306
6448
|
return { error: e.message };
|
|
@@ -5332,7 +6474,8 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
5332
6474
|
async getModels(msg) {
|
|
5333
6475
|
const cfg = this.config;
|
|
5334
6476
|
const provider = msg.provider || cfg.provider || 'openai';
|
|
5335
|
-
|
|
6477
|
+
// `system`: the model is chosen in the system settings of admin
|
|
6478
|
+
if (provider === 'none' || provider === 'pi-agent' || provider === 'system') {
|
|
5336
6479
|
return [];
|
|
5337
6480
|
}
|
|
5338
6481
|
const apiKey = (0, llm_1.effectiveApiKey)(provider, await (0, credentials_1.resolveApiKey)(this, cfg, {
|
|
@@ -5367,6 +6510,19 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
5367
6510
|
if (provider === 'none') {
|
|
5368
6511
|
return { error: 'No LLM provider selected.' };
|
|
5369
6512
|
}
|
|
6513
|
+
if (provider === 'pi-agent') {
|
|
6514
|
+
return { error: 'pi-agent has its own connection test in its instance settings.' };
|
|
6515
|
+
}
|
|
6516
|
+
if (provider === 'system') {
|
|
6517
|
+
const llm = await (0, systemAi_1.resolveSystemLlm)(this, await (0, systemAi_1.readSystemAi)(this), true);
|
|
6518
|
+
if ('error' in llm) {
|
|
6519
|
+
return { error: llm.error };
|
|
6520
|
+
}
|
|
6521
|
+
const res = await new llm_1.LlmAgent({ ...llm, maxTokens: 16, tools: [], log: this.log }).testConnection();
|
|
6522
|
+
return res.ok
|
|
6523
|
+
? { result: `Connection OK (${llm.provider} · ${llm.model || 'auto'})` }
|
|
6524
|
+
: { error: res.error || 'unknown error' };
|
|
6525
|
+
}
|
|
5370
6526
|
const apiKey = (0, llm_1.effectiveApiKey)(provider, await (0, credentials_1.resolveApiKey)(this, cfg, {
|
|
5371
6527
|
credentialType: msg.credentialType,
|
|
5372
6528
|
apiKey: msg.apiKey,
|