iobroker.assistant 1.0.3 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +65 -0
- package/admin/custom/.vite/manifest.json +62 -62
- package/admin/custom/assets/Components-BSewCBfS.js +2 -0
- package/admin/custom/assets/{_virtual_mf-localSharedImportMap___mfe_internal__ConfigCustomAssistant__mf_owner__1-A4gakomR.js → _virtual_mf-localSharedImportMap___mfe_internal__ConfigCustomAssistant__mf_owner__1-C5NPM0Of.js} +1 -1
- package/admin/custom/assets/{_virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__201375093093590__loadShare___mf_0_emotion_mf_1_styled__loadShare__.js-CwqmfMvn.js → _virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__163539842885036__loadShare___mf_0_emotion_mf_1_styled__loadShare__.js-IaosaymG.js} +1 -1
- package/admin/custom/assets/{_virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__201375093093590__loadShare___mf_0_iobroker_mf_1_gui_mf_2_components__loadShare__.js-BMuLYe6f.js → _virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__163539842885036__loadShare___mf_0_iobroker_mf_1_gui_mf_2_components__loadShare__.js-Cdyjt5KT.js} +39 -39
- package/admin/custom/assets/{_virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__201375093093590__loadShare__shared-BxjZmnpc.js → _virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__163539842885036__loadShare__shared-Dsw5zWS8.js} +1 -1
- package/admin/custom/assets/_virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__163539842885036__prebuild___mf_0_iobroker_mf_1_json_mf_2_config__prebuild__-SgSagTOv.js +122 -0
- package/admin/custom/assets/{hostInit-4roC4hEj.js → hostInit-jJe8bIfV.js} +1 -1
- package/admin/custom/assets/{lib-CchzgTUs.js → lib-CL8mXvI6.js} +1 -1
- package/admin/custom/assets/{mf-entry-bootstrap-0-4806bbe4.js → mf-entry-bootstrap-0-ab227aa6.js} +2 -2
- package/admin/custom/assets/{pendingShares-DLlv8yFY.js → pendingShares-wrGtfliQ.js} +1 -1
- package/admin/custom/customComponents.js +2 -2
- package/admin/custom/i18n/de.json +40 -0
- package/admin/custom/i18n/en.json +40 -0
- package/admin/custom/i18n/es.json +40 -0
- package/admin/custom/i18n/fr.json +40 -0
- package/admin/custom/i18n/it.json +40 -0
- package/admin/custom/i18n/nl.json +40 -0
- package/admin/custom/i18n/pl.json +40 -0
- package/admin/custom/i18n/pt.json +40 -0
- package/admin/custom/i18n/ru.json +40 -0
- package/admin/custom/i18n/uk.json +40 -0
- package/admin/custom/i18n/zh-cn.json +40 -0
- package/admin/custom/index.html +2 -2
- package/admin/custom/mf-manifest.json +1 -1
- package/admin/custom/mf-stats.json +1 -1
- package/admin/i18n/de.json +13 -3
- package/admin/i18n/en.json +13 -3
- package/admin/i18n/es.json +13 -3
- package/admin/i18n/fr.json +13 -3
- package/admin/i18n/it.json +13 -3
- package/admin/i18n/nl.json +13 -3
- package/admin/i18n/pl.json +13 -3
- package/admin/i18n/pt.json +13 -3
- package/admin/i18n/ru.json +13 -3
- package/admin/i18n/uk.json +13 -3
- package/admin/i18n/zh-cn.json +13 -3
- package/admin/jsonConfig.json +61 -2
- package/build/lib/journal.js +298 -2
- package/build/lib/journal.js.map +1 -1
- package/build/lib/llm.js +105 -4
- package/build/lib/llm.js.map +1 -1
- package/build/lib/localLlm.js +47 -10
- package/build/lib/localLlm.js.map +1 -1
- package/build/lib/nlu.js +124 -5
- package/build/lib/nlu.js.map +1 -1
- package/build/lib/tools.js +3 -0
- package/build/lib/tools.js.map +1 -1
- package/build/lib/voice/esphome.js +81 -7
- package/build/lib/voice/esphome.js.map +1 -1
- package/build/lib/voice/esphomeProto.js.map +1 -1
- package/build/lib/voice/mediaServer.js +29 -1
- package/build/lib/voice/mediaServer.js.map +1 -1
- package/build/lib/voice/protocol.js.map +1 -1
- package/build/lib/voice/voiceServer.js +81 -7
- package/build/lib/voice/voiceServer.js.map +1 -1
- package/build/lib/voice/wakeWords.js +284 -0
- package/build/lib/voice/wakeWords.js.map +1 -0
- package/build/lib/voice/wyoming.js +12 -4
- package/build/lib/voice/wyoming.js.map +1 -1
- package/build/lib/weather.js +160 -0
- package/build/lib/weather.js.map +1 -1
- package/build/lib/youtube.js +625 -0
- package/build/lib/youtube.js.map +1 -0
- package/build/main.js +1152 -76
- package/build/main.js.map +1 -1
- package/io-package.json +30 -14
- package/package.json +6 -6
- package/admin/custom/assets/Components-C6qcpaRC.js +0 -1
- package/admin/custom/assets/_virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__201375093093590__prebuild___mf_0_iobroker_mf_1_json_mf_2_config__prebuild__-TM4l0OYm.js +0 -122
- /package/admin/custom/assets/{_virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__201375093093590__loadShare___mf_0_emotion_mf_1_react__loadShare__.js-BkH7fzDw.js → _virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__163539842885036__loadShare___mf_0_emotion_mf_1_react__loadShare__.js-BkH7fzDw.js} +0 -0
- /package/admin/custom/assets/{_virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__201375093093590__prebuild__react_mf_1_jsx_mf_2_runtime__prebuild__-M61rjtmp.js → _virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__163539842885036__prebuild__react_mf_1_jsx_mf_2_runtime__prebuild__-M61rjtmp.js} +0 -0
package/build/main.js
CHANGED
|
@@ -36,6 +36,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
36
36
|
const os = __importStar(require("node:os"));
|
|
37
37
|
const fs = __importStar(require("node:fs"));
|
|
38
38
|
const node_child_process_1 = require("node:child_process");
|
|
39
|
+
const node_async_hooks_1 = require("node:async_hooks");
|
|
39
40
|
const adapter_core_1 = require("@iobroker/adapter-core");
|
|
40
41
|
const path = __importStar(require("node:path"));
|
|
41
42
|
const mcp_server_1 = require("@iobroker/mcp-server");
|
|
@@ -54,6 +55,7 @@ const ask_1 = require("./lib/ask");
|
|
|
54
55
|
const presence_1 = require("./lib/presence");
|
|
55
56
|
const targets_1 = require("./lib/targets");
|
|
56
57
|
const routines_1 = require("./lib/routines");
|
|
58
|
+
const youtube_1 = require("./lib/youtube");
|
|
57
59
|
const notifications_1 = require("./lib/notifications");
|
|
58
60
|
const triggers_1 = require("./lib/triggers");
|
|
59
61
|
const voiceServer_1 = require("./lib/voice/voiceServer");
|
|
@@ -61,6 +63,7 @@ const wyoming_1 = require("./lib/voice/wyoming");
|
|
|
61
63
|
const esphome_1 = require("./lib/voice/esphome");
|
|
62
64
|
const esphomeEntities_1 = require("./lib/voice/esphomeEntities");
|
|
63
65
|
const mediaServer_1 = require("./lib/voice/mediaServer");
|
|
66
|
+
const wakeWords_1 = require("./lib/voice/wakeWords");
|
|
64
67
|
const tone_1 = require("./lib/voice/tone");
|
|
65
68
|
const engines_1 = require("./lib/voice/engines");
|
|
66
69
|
/**
|
|
@@ -68,6 +71,20 @@ const engines_1 = require("./lib/voice/engines");
|
|
|
68
71
|
* so re-reading their whole state tree on every single request would be pure overhead.
|
|
69
72
|
*/
|
|
70
73
|
const WEATHER_CTX_TTL = 5 * 60 * 1000;
|
|
74
|
+
/**
|
|
75
|
+
* How long the NLU device model stays valid. Building it costs a `list_devices` plus a name lookup per
|
|
76
|
+
* device, and one spoken command needs it up to three times (STT hints, NLU, the cloud device list).
|
|
77
|
+
* Enum and name changes drop it at once (see {@link ListCache.onClear}); the TTL only catches devices
|
|
78
|
+
* that were added or renamed elsewhere.
|
|
79
|
+
*/
|
|
80
|
+
const NLU_DEVICES_TTL_MS = 60_000;
|
|
81
|
+
/** Default for `localLlmTimeoutSec`: after this the local model is stopped and the next tier answers. */
|
|
82
|
+
const LOCAL_LLM_TIMEOUT_SEC = 15;
|
|
83
|
+
/**
|
|
84
|
+
* Answer length cap of the local model. It answers small talk and general questions, mostly by voice, and
|
|
85
|
+
* on a Raspberry Pi every token costs about a fifth of a second — 512 tokens would be a minute and a half.
|
|
86
|
+
*/
|
|
87
|
+
const LOCAL_LLM_MAX_TOKENS = 256;
|
|
71
88
|
/**
|
|
72
89
|
* The replies the assistant speaks verbatim, whatever the question was. They are pre-synthesised into
|
|
73
90
|
* the TTS cache at startup; everything else is cached on first use.
|
|
@@ -75,6 +92,12 @@ const WEATHER_CTX_TTL = 5 * 60 * 1000;
|
|
|
75
92
|
/** How long to wait for a device to confirm a write, and how often to look. */
|
|
76
93
|
const ACK_TIMEOUT_MS = 2000;
|
|
77
94
|
const ACK_POLL_MS = 150;
|
|
95
|
+
/** How long a TV may take to report "on" after it was switched on for YouTube. */
|
|
96
|
+
const TV_POWER_TIMEOUT_MS = 30_000;
|
|
97
|
+
/** How long youtube-remote may take to acknowledge a play command (it goes through Google's lounge). */
|
|
98
|
+
const YOUTUBE_ACK_TIMEOUT_MS = 8000;
|
|
99
|
+
/** Pause before the play command is repeated once on a TV that just booted. */
|
|
100
|
+
const YOUTUBE_RETRY_MS = 5000;
|
|
78
101
|
/**
|
|
79
102
|
* Did the device end up with the value we asked for? Compared loosely: a dimmer told `30` may report
|
|
80
103
|
* `30.0`, a switch told `true` may report `1`, and a lamp told 100 % may settle at 99 % — insisting on
|
|
@@ -155,7 +178,27 @@ function iaqLabel(value, unit, lang) {
|
|
|
155
178
|
return lang === 'ru' ? row[3] : lang === 'de' ? row[1] : row[2];
|
|
156
179
|
}
|
|
157
180
|
/** NLU actions that only switch something; their success is what the confirmation tone replaces. */
|
|
158
|
-
const CONTROL_ACTIONS = new Set(['on', 'off', 'level', 'color']);
|
|
181
|
+
const CONTROL_ACTIONS = new Set(['on', 'off', 'level', 'color', 'volumeSet']);
|
|
182
|
+
/** An intent in one line for the request trace: "level Rollo (Büro) = 30", "timerSet 300 s", "alarmSet 7:05". */
|
|
183
|
+
function describeIntent(intent) {
|
|
184
|
+
let text = intent.action;
|
|
185
|
+
if (intent.device) {
|
|
186
|
+
text += ` ${intent.device.name}${intent.device.room ? ` (${intent.device.room})` : ''}`;
|
|
187
|
+
}
|
|
188
|
+
else if (intent.room) {
|
|
189
|
+
text += ` (${intent.room})`;
|
|
190
|
+
}
|
|
191
|
+
if (intent.value !== undefined) {
|
|
192
|
+
text += ` = ${JSON.stringify(intent.value)}`;
|
|
193
|
+
}
|
|
194
|
+
else if (intent.durationSec !== undefined) {
|
|
195
|
+
text += ` ${intent.durationSec} s`;
|
|
196
|
+
}
|
|
197
|
+
else if (intent.hour !== undefined) {
|
|
198
|
+
text += ` ${intent.hour}:${String(intent.minute ?? 0).padStart(2, '0')}`;
|
|
199
|
+
}
|
|
200
|
+
return text;
|
|
201
|
+
}
|
|
159
202
|
const FIXED_REPLIES = {
|
|
160
203
|
de: ['Ok.', 'Erledigt.'],
|
|
161
204
|
en: ['Okay.', 'Done.'],
|
|
@@ -192,6 +235,8 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
192
235
|
journal = null;
|
|
193
236
|
/** Short-TTL cache for the expensive device/room/function listings; invalidated on enum changes. */
|
|
194
237
|
listCache = null;
|
|
238
|
+
/** The NLU device model of {@link getNluDevices}, shared by all callers (and concurrent requests). */
|
|
239
|
+
nluDevices = null;
|
|
195
240
|
/** Cache of a `<base>.SET` level state id → its writable boolean sibling switch (`<base>.ON_SET`/`.ON`), or ''. */
|
|
196
241
|
siblingSwitch = new Map();
|
|
197
242
|
/** Tier-1a local LLM (node-llama-cpp), lazily loaded when enabled + installed; null otherwise. */
|
|
@@ -208,6 +253,10 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
208
253
|
esphome = null;
|
|
209
254
|
/** Serves spoken replies to ESPHome satellites over HTTP; null unless those are enabled. */
|
|
210
255
|
media = null;
|
|
256
|
+
/** Uploaded wake words (`assistant.0/wakewords/`), offered to every ESPHome satellite. */
|
|
257
|
+
customWakeWords = new Map();
|
|
258
|
+
/** Loads {@link customWakeWords} once from the file storage; later calls wait for the same load. */
|
|
259
|
+
customWakeWordsLoaded = null;
|
|
211
260
|
/** Short-term per-source conversation memory (in-memory, TTL) for follow-up questions. */
|
|
212
261
|
context = new context_1.ConversationStore();
|
|
213
262
|
/** Questions the assistant asked and is waiting for an answer to (`askUser`), keyed by source. */
|
|
@@ -222,6 +271,8 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
222
271
|
mirroredTimers = new Map();
|
|
223
272
|
/** Per satellite: resolves once its control objects exist, so state writes can wait for them. */
|
|
224
273
|
satControlsReady = new Map();
|
|
274
|
+
/** Satellites whose `volume` state exists and is subscribed (see updateSatelliteVolume). */
|
|
275
|
+
satVolumeEnsured = new Set();
|
|
225
276
|
/** Native (ioBroker) satellite state-id → sender instance id, so we can push announcements back to it. */
|
|
226
277
|
nativeSatFrom = new Map();
|
|
227
278
|
/** Countdown timers / reminders (roadmap #2); mirrored into `timers.*` states. Null until onReady. */
|
|
@@ -248,6 +299,13 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
248
299
|
routines = [];
|
|
249
300
|
/** Named announcement targets (roadmap B2): groups of rooms and people, from the configuration. */
|
|
250
301
|
announceTargets = [];
|
|
302
|
+
/** YouTube players from the configuration; the first one is the default. */
|
|
303
|
+
youtubePlayers = [];
|
|
304
|
+
/**
|
|
305
|
+
* Where the request being answered came from, for the LLM tools: a tool only gets the model's
|
|
306
|
+
* arguments, but "play X" from the bedroom satellite should go to the bedroom TV.
|
|
307
|
+
*/
|
|
308
|
+
requestOrigin = new node_async_hooks_1.AsyncLocalStorage();
|
|
251
309
|
/** Who is at home (roadmap B1); mirrored into `presence.*`. Null until onReady. */
|
|
252
310
|
presence = null;
|
|
253
311
|
/** Satellites currently set to Do-Not-Disturb, by state id (mirrors `satellites.<id>.dnd`). */
|
|
@@ -289,6 +347,9 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
289
347
|
allowObjectChange: cfg.allowObjectChange,
|
|
290
348
|
}));
|
|
291
349
|
this.listCache = new tools_1.ListCache();
|
|
350
|
+
this.listCache.onClear = () => {
|
|
351
|
+
this.nluDevices = null;
|
|
352
|
+
};
|
|
292
353
|
const { tools, denied } = await (0, tools_1.buildMcpTools)(this.mcp, cfg, this.listCache, (id, fb) => this.resolveDeviceName(id, fb));
|
|
293
354
|
// Let the cloud LLM handle timers/alarms too (for phrasings the rule-based NLU misses).
|
|
294
355
|
tools.push(...this.buildTimerTools(), ...this.buildAlarmTools());
|
|
@@ -302,6 +363,18 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
302
363
|
// Only useful with satellites: without voice there is nothing to speak on.
|
|
303
364
|
tools.push(this.buildAnnounceTool());
|
|
304
365
|
}
|
|
366
|
+
this.youtubePlayers = (0, youtube_1.parsePlayerRows)(cfg.youtubePlayers);
|
|
367
|
+
if (this.youtubePlayers.length) {
|
|
368
|
+
// Before the tools are built: their descriptions carry the player names, which an instance
|
|
369
|
+
// row only gets from its device.
|
|
370
|
+
for (const player of this.youtubePlayers) {
|
|
371
|
+
if (!(await this.resolveYouTubePlayer(player))) {
|
|
372
|
+
this.log.warn(`YouTube: ${player.instance} has no paired TV yet — pair one in its settings.`);
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
tools.push(...this.buildYouTubeTools());
|
|
376
|
+
this.log.info(`YouTube players: ${this.youtubePlayers.map(p => p.name).join(', ')}.`);
|
|
377
|
+
}
|
|
305
378
|
this.log.info(`ioBroker tools enabled (${tools.length}): ${tools.map(t => t.name).join(', ')}`);
|
|
306
379
|
if (denied.length) {
|
|
307
380
|
this.log.debug(`Tools denied by access settings: ${denied.join(', ')}`);
|
|
@@ -320,6 +393,7 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
320
393
|
maxTokens: cfg.maxTokens || 1024,
|
|
321
394
|
tools: tools.map(t => this.journalTool(t)),
|
|
322
395
|
log: this.log,
|
|
396
|
+
trace: step => this.journal?.step(step),
|
|
323
397
|
});
|
|
324
398
|
}
|
|
325
399
|
}
|
|
@@ -407,7 +481,8 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
407
481
|
language,
|
|
408
482
|
stt,
|
|
409
483
|
tts,
|
|
410
|
-
answer: (question, sttMs) => this.answer(question, 'wyoming', { transport: 'wyoming', sttMs }),
|
|
484
|
+
answer: (question, sttMs, audioMs) => this.answer(question, 'wyoming', { transport: 'wyoming', sttMs, audioMs }),
|
|
485
|
+
onSpoken: info => this.reportSpoken('wyoming', info),
|
|
411
486
|
getHints: () => this.buildSttHints(),
|
|
412
487
|
log: this.log,
|
|
413
488
|
});
|
|
@@ -462,6 +537,10 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
462
537
|
this.media = null;
|
|
463
538
|
return;
|
|
464
539
|
}
|
|
540
|
+
await this.loadCustomWakeWords();
|
|
541
|
+
for (const word of this.customWakeWords.values()) {
|
|
542
|
+
this.serveWakeWord(word);
|
|
543
|
+
}
|
|
465
544
|
this.esphome = new esphome_1.EsphomeSatellites({
|
|
466
545
|
devices,
|
|
467
546
|
language,
|
|
@@ -472,7 +551,10 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
472
551
|
room: origin.room,
|
|
473
552
|
wakeWord: origin.wakeWord,
|
|
474
553
|
sttMs: origin.sttMs,
|
|
554
|
+
audioMs: origin.audioMs,
|
|
555
|
+
vad: origin.vad,
|
|
475
556
|
}),
|
|
557
|
+
onSpoken: (device, info) => this.reportSpoken(device, info),
|
|
476
558
|
getHints: () => this.buildSttHints(),
|
|
477
559
|
media: this.media,
|
|
478
560
|
mediaHost: (cfg.esphomeMediaHost || '').trim(),
|
|
@@ -490,7 +572,15 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
490
572
|
onEntityState: (device, room, entity) => {
|
|
491
573
|
this.updateSatelliteControl(device, room, entity).catch(e => this.log.debug(`satellite control update failed: ${e.message}`));
|
|
492
574
|
},
|
|
493
|
-
expectsFollowUp: answer => this.
|
|
575
|
+
expectsFollowUp: (answer, device) => this.followUpFor(device, answer),
|
|
576
|
+
externalWakeWords: () => [...this.customWakeWords.values()].map(w => ({
|
|
577
|
+
id: w.id,
|
|
578
|
+
phrase: w.phrase,
|
|
579
|
+
languages: w.languages,
|
|
580
|
+
size: w.size,
|
|
581
|
+
sha256: w.sha256,
|
|
582
|
+
path: `${wakeWords_1.WAKE_WORD_URL_PREFIX}${w.id}.json`,
|
|
583
|
+
})),
|
|
494
584
|
});
|
|
495
585
|
await this.esphome.start();
|
|
496
586
|
this.log.info(`ESPHome satellites: ${devices.length} device(s), STT=${cfg.sttProvider || 'openai'}, TTS=${cfg.ttsProvider || 'openai'}.`);
|
|
@@ -545,7 +635,17 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
545
635
|
language,
|
|
546
636
|
stt,
|
|
547
637
|
tts,
|
|
548
|
-
answer: (question, ctx) => this.answerVoice(question, ctx.device, {
|
|
638
|
+
answer: (question, ctx) => this.answerVoice(question, ctx.device, {
|
|
639
|
+
transport: 'udp',
|
|
640
|
+
room: ctx.room,
|
|
641
|
+
sttMs: ctx.sttMs,
|
|
642
|
+
audioMs: ctx.audioMs,
|
|
643
|
+
}),
|
|
644
|
+
onSpoken: (device, info) => this.reportSpoken(device, info),
|
|
645
|
+
expectsFollowUp: (answer, device) => this.followUpFor(device, answer),
|
|
646
|
+
onVolume: (device, room, level) => {
|
|
647
|
+
this.updateSatelliteVolume(device, room, level).catch(e => this.log.debug(`satellite volume update failed: ${e.message}`));
|
|
648
|
+
},
|
|
549
649
|
getHints: () => this.buildSttHints(),
|
|
550
650
|
log: this.log,
|
|
551
651
|
onStatus: (device, room, state) => {
|
|
@@ -612,6 +712,23 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
612
712
|
expectsFollowUp(answer) {
|
|
613
713
|
return /[??]/.test(answer || '');
|
|
614
714
|
}
|
|
715
|
+
/** A transport synthesised the reply to `source`'s latest request — the journal shows it with that request. */
|
|
716
|
+
reportSpoken(source, info) {
|
|
717
|
+
const cfg = this.config;
|
|
718
|
+
const provider = cfg.ttsProvider || 'openai';
|
|
719
|
+
const voices = {
|
|
720
|
+
openai: cfg.ttsVoice || 'alloy',
|
|
721
|
+
azure: cfg.azureVoice,
|
|
722
|
+
aws: cfg.awsVoice,
|
|
723
|
+
piper: cfg.piperVoice,
|
|
724
|
+
};
|
|
725
|
+
this.journal?.spoken(source, {
|
|
726
|
+
...info,
|
|
727
|
+
provider,
|
|
728
|
+
fallback: cfg.ttsFallback || undefined,
|
|
729
|
+
voice: (voices[provider] || '').trim() || undefined,
|
|
730
|
+
});
|
|
731
|
+
}
|
|
615
732
|
/** Record the origin of the current text.request/response ('' = state write, 'chat' = message, else satellite). */
|
|
616
733
|
setQuerySource(source) {
|
|
617
734
|
this.setStateAsync('text.querySource', { val: source, ack: true }).catch(() => { });
|
|
@@ -678,17 +795,18 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
678
795
|
room,
|
|
679
796
|
wakeWord: typeof msg.wakeWord === 'string' ? msg.wakeWord : undefined,
|
|
680
797
|
sttMs: Date.now() - sttStart,
|
|
798
|
+
audioMs: Math.round((pcm.length / 2 / rate) * 1000),
|
|
681
799
|
});
|
|
682
800
|
if (!answer) {
|
|
683
801
|
return { text, answer: '' };
|
|
684
802
|
}
|
|
803
|
+
const ttsStart = Date.now();
|
|
685
804
|
const reply = await engines.tts.synthesize(answer, language);
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
}
|
|
805
|
+
this.reportSpoken(source, {
|
|
806
|
+
ms: Date.now() - ttsStart,
|
|
807
|
+
audioMs: (reply.pcm.length / 2 / reply.sampleRate) * 1000,
|
|
808
|
+
});
|
|
809
|
+
const listen = this.followUpFor(source, answer);
|
|
692
810
|
return { text, answer, audio: reply.pcm.toString('base64'), sampleRate: reply.sampleRate, listen };
|
|
693
811
|
}
|
|
694
812
|
catch (e) {
|
|
@@ -751,6 +869,33 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
751
869
|
await this.setStateAsync(`${base}.room`, { val: room, ack: true });
|
|
752
870
|
}
|
|
753
871
|
}
|
|
872
|
+
/**
|
|
873
|
+
* Mirror the playback volume a UDP satellite reported into `satellites.<id>.volume`. The state only
|
|
874
|
+
* appears for satellites that speak the `volume` message — an older one would never answer a write.
|
|
875
|
+
*/
|
|
876
|
+
async updateSatelliteVolume(device, room, level) {
|
|
877
|
+
const id = await this.ensureSatelliteObjects(device, room);
|
|
878
|
+
const stateId = `satellites.${id}.volume`;
|
|
879
|
+
if (!this.satVolumeEnsured.has(id)) {
|
|
880
|
+
await this.setObjectNotExistsAsync(stateId, {
|
|
881
|
+
type: 'state',
|
|
882
|
+
common: {
|
|
883
|
+
name: 'Playback volume',
|
|
884
|
+
type: 'number',
|
|
885
|
+
role: 'level.volume',
|
|
886
|
+
unit: '%',
|
|
887
|
+
min: 0,
|
|
888
|
+
max: 100,
|
|
889
|
+
read: true,
|
|
890
|
+
write: true,
|
|
891
|
+
},
|
|
892
|
+
native: {},
|
|
893
|
+
});
|
|
894
|
+
this.subscribeStates(stateId);
|
|
895
|
+
this.satVolumeEnsured.add(id);
|
|
896
|
+
}
|
|
897
|
+
await this.setStateAsync(stateId, { val: level, ack: true });
|
|
898
|
+
}
|
|
754
899
|
/**
|
|
755
900
|
* Create the object tree for one satellite and return its (sanitised) state id, without touching
|
|
756
901
|
* any value. Callers that only need somewhere to hang a child object use this rather than
|
|
@@ -886,11 +1031,111 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
886
1031
|
this.satWakeWordStatesEnsured.add(id);
|
|
887
1032
|
}
|
|
888
1033
|
await this.setStateAsync(`${base}.wakeWords`, { val: config.active.join(','), ack: true });
|
|
1034
|
+
// `custom` marks the uploaded ones, so the settings dialog can offer to delete them.
|
|
1035
|
+
const words = config.available.map(w => (this.customWakeWords.has(w.id) ? { ...w, custom: true } : w));
|
|
889
1036
|
await this.setStateAsync(`${base}.availableWakeWords`, {
|
|
890
|
-
val: JSON.stringify({ max: config.max, words
|
|
1037
|
+
val: JSON.stringify({ max: config.max, words }),
|
|
891
1038
|
ack: true,
|
|
892
1039
|
});
|
|
893
1040
|
}
|
|
1041
|
+
/** Read the uploaded wake words (`<instance>/wakewords/<id>.json` + `.tflite`) once. */
|
|
1042
|
+
loadCustomWakeWords() {
|
|
1043
|
+
this.customWakeWordsLoaded ||= (async () => {
|
|
1044
|
+
let entries = [];
|
|
1045
|
+
try {
|
|
1046
|
+
entries = await this.readDirAsync(this.namespace, wakeWords_1.WAKE_WORD_DIR);
|
|
1047
|
+
}
|
|
1048
|
+
catch {
|
|
1049
|
+
return; // nothing uploaded yet
|
|
1050
|
+
}
|
|
1051
|
+
for (const entry of entries) {
|
|
1052
|
+
const id = entry.file.replace(/\.json$/, '');
|
|
1053
|
+
if (entry.isDir || !entry.file.endsWith('.json') || !wakeWords_1.WAKE_WORD_ID_RE.test(id)) {
|
|
1054
|
+
continue;
|
|
1055
|
+
}
|
|
1056
|
+
try {
|
|
1057
|
+
const config = await this.readStoredFile(`${wakeWords_1.WAKE_WORD_DIR}/${id}.json`, 'utf8');
|
|
1058
|
+
const model = await this.readStoredFile(`${wakeWords_1.WAKE_WORD_DIR}/${id}.tflite`, 'binary');
|
|
1059
|
+
const word = (0, wakeWords_1.loadWakeWord)(id, config, model);
|
|
1060
|
+
this.customWakeWords.set(id, word);
|
|
1061
|
+
// Uploaded before the upload checked the model: keep it, so it can be deleted in the dialog.
|
|
1062
|
+
const problem = (0, wakeWords_1.microModelProblem)(model);
|
|
1063
|
+
if (problem) {
|
|
1064
|
+
this.log.warn(`Custom wake word ${word.phrase} (${id}) cannot run on the device: ${problem}. Delete it in the satellite settings.`);
|
|
1065
|
+
}
|
|
1066
|
+
}
|
|
1067
|
+
catch (e) {
|
|
1068
|
+
this.log.warn(`Custom wake word ${id} skipped: ${e.message}`);
|
|
1069
|
+
}
|
|
1070
|
+
}
|
|
1071
|
+
if (this.customWakeWords.size) {
|
|
1072
|
+
this.log.info(`Custom wake words: ${[...this.customWakeWords.values()].map(w => `${w.phrase} (${w.id})`).join(', ')}`);
|
|
1073
|
+
}
|
|
1074
|
+
})();
|
|
1075
|
+
return this.customWakeWordsLoaded;
|
|
1076
|
+
}
|
|
1077
|
+
/**
|
|
1078
|
+
* Read a file from this instance's file storage as a Buffer. Depending on the backend and the file's
|
|
1079
|
+
* type it comes back as a Buffer or a string — `encoding` says how to read the latter.
|
|
1080
|
+
*/
|
|
1081
|
+
async readStoredFile(name, encoding) {
|
|
1082
|
+
const res = (await this.readFileAsync(this.namespace, name));
|
|
1083
|
+
const f = Buffer.isBuffer(res) || typeof res === 'string' ? res : res.file;
|
|
1084
|
+
if (f == null) {
|
|
1085
|
+
throw new Error(`${name} not found`);
|
|
1086
|
+
}
|
|
1087
|
+
return Buffer.isBuffer(f) ? f : Buffer.from(f, encoding);
|
|
1088
|
+
}
|
|
1089
|
+
serveWakeWord(word) {
|
|
1090
|
+
this.media?.serve(`${wakeWords_1.WAKE_WORD_URL_PREFIX}${word.id}.json`, word.config, 'application/json');
|
|
1091
|
+
this.media?.serve(`${wakeWords_1.WAKE_WORD_URL_PREFIX}${word.id}.tflite`, word.model, 'application/octet-stream');
|
|
1092
|
+
}
|
|
1093
|
+
/**
|
|
1094
|
+
* Store an uploaded wake word and offer it to the connected satellites. It shows up in their wake
|
|
1095
|
+
* word list right away; a device downloads the model only when it is selected there.
|
|
1096
|
+
*/
|
|
1097
|
+
async addCustomWakeWord(message) {
|
|
1098
|
+
await this.loadCustomWakeWords();
|
|
1099
|
+
const text = (value) => (typeof value === 'string' ? value : undefined);
|
|
1100
|
+
const languages = (Array.isArray(message.languages)
|
|
1101
|
+
? message.languages.map(String)
|
|
1102
|
+
: (text(message.languages) ?? '').split(/[,;\s]+/)).filter(Boolean);
|
|
1103
|
+
const word = (0, wakeWords_1.prepareWakeWord)({
|
|
1104
|
+
model: Buffer.from(text(message.model) ?? '', 'base64'),
|
|
1105
|
+
fileName: text(message.fileName),
|
|
1106
|
+
config: message.config && typeof message.config === 'object'
|
|
1107
|
+
? message.config
|
|
1108
|
+
: text(message.config),
|
|
1109
|
+
phrase: text(message.phrase),
|
|
1110
|
+
languages,
|
|
1111
|
+
}, this.config.voiceLanguage || this.language || '');
|
|
1112
|
+
if (!this.customWakeWords.has(word.id)) {
|
|
1113
|
+
await this.writeFileAsync(this.namespace, `${wakeWords_1.WAKE_WORD_DIR}/${word.id}.tflite`, word.model);
|
|
1114
|
+
await this.writeFileAsync(this.namespace, `${wakeWords_1.WAKE_WORD_DIR}/${word.id}.json`, word.config);
|
|
1115
|
+
this.customWakeWords.set(word.id, word);
|
|
1116
|
+
this.serveWakeWord(word);
|
|
1117
|
+
this.log.info(`Custom wake word uploaded: ${word.phrase} (${word.id}, ${word.size} bytes)`);
|
|
1118
|
+
}
|
|
1119
|
+
const devices = this.esphome?.refreshWakeWords() ?? 0;
|
|
1120
|
+
return { id: word.id, phrase: word.phrase, languages: word.languages, size: word.size, devices };
|
|
1121
|
+
}
|
|
1122
|
+
/**
|
|
1123
|
+
* Remove an uploaded wake word: it is no longer offered or served. A device that already downloaded
|
|
1124
|
+
* it keeps its copy — the ESPHome API cannot delete a model from a device.
|
|
1125
|
+
*/
|
|
1126
|
+
async deleteCustomWakeWord(id) {
|
|
1127
|
+
await this.loadCustomWakeWords();
|
|
1128
|
+
if (!wakeWords_1.WAKE_WORD_ID_RE.test(id) || !this.customWakeWords.has(id)) {
|
|
1129
|
+
throw new Error(`unknown wake word ${id}`);
|
|
1130
|
+
}
|
|
1131
|
+
for (const ext of ['json', 'tflite']) {
|
|
1132
|
+
await this.delFileAsync(this.namespace, `${wakeWords_1.WAKE_WORD_DIR}/${id}.${ext}`).catch(() => undefined);
|
|
1133
|
+
this.media?.unserve(`${wakeWords_1.WAKE_WORD_URL_PREFIX}${id}.${ext}`);
|
|
1134
|
+
}
|
|
1135
|
+
this.customWakeWords.delete(id);
|
|
1136
|
+
this.esphome?.refreshWakeWords();
|
|
1137
|
+
this.log.info(`Custom wake word deleted: ${id}`);
|
|
1138
|
+
}
|
|
894
1139
|
/**
|
|
895
1140
|
* Mirror everything an ESPHome satellite exposes besides the voice pipeline into
|
|
896
1141
|
* `satellites.<id>.controls.*` — microphone gain and volume, noise suppression, wake-word and
|
|
@@ -1254,7 +1499,8 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
1254
1499
|
await Promise.all(allowed.map(d => this.voice.announce(d, pcm, sampleRate)));
|
|
1255
1500
|
delivered += allowed.length;
|
|
1256
1501
|
if (listen) {
|
|
1257
|
-
|
|
1502
|
+
const playMs = (pcm.length / 2 / sampleRate) * 1000;
|
|
1503
|
+
allowed.forEach(d => this.voice.listen(d, playMs));
|
|
1258
1504
|
}
|
|
1259
1505
|
}
|
|
1260
1506
|
catch (e) {
|
|
@@ -1603,6 +1849,15 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
1603
1849
|
await this.setStateAsync('dnd', { val: this.globalDnd, ack: true });
|
|
1604
1850
|
return;
|
|
1605
1851
|
}
|
|
1852
|
+
// Playback volume of a UDP satellite — not acked here: the satellite reports the level it took.
|
|
1853
|
+
const satVolume = id.match(/\.satellites\.([^.]+)\.volume$/);
|
|
1854
|
+
if (satVolume) {
|
|
1855
|
+
const device = this.satDeviceById.get(satVolume[1]) || satVolume[1];
|
|
1856
|
+
if (!this.voice?.volume(device, { level: Number(state.val) })) {
|
|
1857
|
+
this.log.warn(`Cannot set the volume of '${device}': it is not a registered UDP satellite.`);
|
|
1858
|
+
}
|
|
1859
|
+
return;
|
|
1860
|
+
}
|
|
1606
1861
|
const satDnd = id.match(/\.satellites\.([^.]+)\.dnd$/);
|
|
1607
1862
|
if (satDnd) {
|
|
1608
1863
|
this.dndById.set(satDnd[1], !!state.val);
|
|
@@ -1929,6 +2184,27 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
1929
2184
|
}
|
|
1930
2185
|
return;
|
|
1931
2186
|
}
|
|
2187
|
+
// Scripts: find and play something on a configured YouTube player, the TV switched on first.
|
|
2188
|
+
// message = { query, kind?: 'channel'|'video'|'playlist', player? } → { ok, text, title?, id?, player? }
|
|
2189
|
+
// Settings: the "Player" column of the YouTube table (selectSendTo).
|
|
2190
|
+
if (obj.command === 'getYouTubeInstances') {
|
|
2191
|
+
const list = await this.getYouTubeInstances();
|
|
2192
|
+
if (obj.callback) {
|
|
2193
|
+
this.sendTo(obj.from, obj.command, list, obj.callback);
|
|
2194
|
+
}
|
|
2195
|
+
return;
|
|
2196
|
+
}
|
|
2197
|
+
if (obj.command === 'playYouTube') {
|
|
2198
|
+
const m = (obj.message || {});
|
|
2199
|
+
const kind = m.kind === 'channel' || m.kind === 'playlist' || m.kind === 'video' ? m.kind : 'auto';
|
|
2200
|
+
const res = String(m.query || '').trim()
|
|
2201
|
+
? await this.playYouTube(String(m.query).trim(), kind, m.player)
|
|
2202
|
+
: { ok: false, text: 'no query given' };
|
|
2203
|
+
if (obj.callback) {
|
|
2204
|
+
this.sendTo(obj.from, obj.command, this.youtubeResultData(res), obj.callback);
|
|
2205
|
+
}
|
|
2206
|
+
return;
|
|
2207
|
+
}
|
|
1932
2208
|
// Scripts / settings: which wake words an ESPHome satellite offers and which are listening.
|
|
1933
2209
|
// message = { device? } — omitted means every connected one.
|
|
1934
2210
|
if (obj.command === 'getWakeWords') {
|
|
@@ -1966,6 +2242,41 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
1966
2242
|
}
|
|
1967
2243
|
return;
|
|
1968
2244
|
}
|
|
2245
|
+
// Settings dialog / scripts: own wake words for the ESPHome satellites (microWakeWord models).
|
|
2246
|
+
// uploadWakeWord: { model: base64 .tflite, fileName?, config?: JSON text, phrase?, languages? }
|
|
2247
|
+
// → { ok, id, phrase, languages, size, devices } | { error }
|
|
2248
|
+
// deleteWakeWord: { id } → { ok } | { error }; listWakeWords → { wakeWords: [{id, phrase, …}] }
|
|
2249
|
+
if (obj.command === 'uploadWakeWord' || obj.command === 'deleteWakeWord' || obj.command === 'listWakeWords') {
|
|
2250
|
+
const m = (obj.message || {});
|
|
2251
|
+
let payload;
|
|
2252
|
+
try {
|
|
2253
|
+
if (obj.command === 'uploadWakeWord') {
|
|
2254
|
+
payload = { ok: true, ...(await this.addCustomWakeWord(m)) };
|
|
2255
|
+
}
|
|
2256
|
+
else if (obj.command === 'deleteWakeWord') {
|
|
2257
|
+
await this.deleteCustomWakeWord(typeof m.id === 'string' ? m.id : '');
|
|
2258
|
+
payload = { ok: true };
|
|
2259
|
+
}
|
|
2260
|
+
else {
|
|
2261
|
+
await this.loadCustomWakeWords();
|
|
2262
|
+
payload = {
|
|
2263
|
+
wakeWords: [...this.customWakeWords.values()].map(w => ({
|
|
2264
|
+
id: w.id,
|
|
2265
|
+
phrase: w.phrase,
|
|
2266
|
+
languages: w.languages,
|
|
2267
|
+
size: w.size,
|
|
2268
|
+
})),
|
|
2269
|
+
};
|
|
2270
|
+
}
|
|
2271
|
+
}
|
|
2272
|
+
catch (e) {
|
|
2273
|
+
payload = { error: e.message };
|
|
2274
|
+
}
|
|
2275
|
+
if (obj.callback) {
|
|
2276
|
+
this.sendTo(obj.from, obj.command, payload, obj.callback);
|
|
2277
|
+
}
|
|
2278
|
+
return;
|
|
2279
|
+
}
|
|
1969
2280
|
// Scripts: everything an ESPHome satellite exposes besides the voice pipeline.
|
|
1970
2281
|
// message = { device? }; without a device, every connected one is listed.
|
|
1971
2282
|
if (obj.command === 'getControls') {
|
|
@@ -2172,6 +2483,18 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
2172
2483
|
}
|
|
2173
2484
|
return;
|
|
2174
2485
|
}
|
|
2486
|
+
// Admin chat debug view: one request with every step of its processing (message = { id }).
|
|
2487
|
+
if (obj.command === 'getJournalTrace') {
|
|
2488
|
+
const id = String(obj.message?.id || '');
|
|
2489
|
+
const entry = this.journal?.get(id);
|
|
2490
|
+
const result = entry
|
|
2491
|
+
? { entry, steps: await this.journal.trace(id) }
|
|
2492
|
+
: { error: this.journal?.enabled ? 'unknown request' : 'journal disabled' };
|
|
2493
|
+
if (obj.callback) {
|
|
2494
|
+
this.sendTo(obj.from, obj.command, result, obj.callback);
|
|
2495
|
+
}
|
|
2496
|
+
return;
|
|
2497
|
+
}
|
|
2175
2498
|
if (obj.command === 'clearJournal') {
|
|
2176
2499
|
this.journal?.clear();
|
|
2177
2500
|
if (obj.callback) {
|
|
@@ -2222,29 +2545,93 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
2222
2545
|
* later turns have the full thread.
|
|
2223
2546
|
*/
|
|
2224
2547
|
answer(question, source = '', origin = {}) {
|
|
2225
|
-
return this.journal
|
|
2226
|
-
? this.journal.run(question, source, origin, () =>
|
|
2227
|
-
|
|
2548
|
+
return this.requestOrigin.run({ source, room: origin.room || '' }, () => this.journal
|
|
2549
|
+
? this.journal.run(question, source, origin, () => {
|
|
2550
|
+
this.traceSpeechInput(question, origin);
|
|
2551
|
+
return this.answerTracked(question, source);
|
|
2552
|
+
})
|
|
2553
|
+
: this.answerTracked(question, source));
|
|
2554
|
+
}
|
|
2555
|
+
/** Speech recognition happened before the request: put it at the head of its trace. */
|
|
2556
|
+
traceSpeechInput(question, origin) {
|
|
2557
|
+
if (origin.sttMs === undefined) {
|
|
2558
|
+
return;
|
|
2559
|
+
}
|
|
2560
|
+
const cfg = this.config;
|
|
2561
|
+
const provider = cfg.sttProvider || 'openai';
|
|
2562
|
+
const audio = origin.audioMs !== undefined ? `, ${(origin.audioMs / 1000).toFixed(1)} s of audio` : '';
|
|
2563
|
+
this.journal?.step({
|
|
2564
|
+
stage: 'stt',
|
|
2565
|
+
title: `Speech recognition: ${provider}${audio}`,
|
|
2566
|
+
ms: origin.sttMs,
|
|
2567
|
+
data: {
|
|
2568
|
+
provider,
|
|
2569
|
+
fallback: cfg.sttFallback || undefined,
|
|
2570
|
+
model: (cfg.sttModel || cfg.voskModel || '').trim() || undefined,
|
|
2571
|
+
language: cfg.voiceLanguage || this.language || '',
|
|
2572
|
+
audioMs: origin.audioMs,
|
|
2573
|
+
wakeWord: origin.wakeWord,
|
|
2574
|
+
vad: origin.vad,
|
|
2575
|
+
transcript: question,
|
|
2576
|
+
},
|
|
2577
|
+
});
|
|
2228
2578
|
}
|
|
2229
2579
|
async answerTracked(question, source) {
|
|
2230
2580
|
const useCtx = this.config.useConversationContext !== false;
|
|
2231
2581
|
const history = useCtx ? this.context.get(source) : [];
|
|
2582
|
+
if (history.length) {
|
|
2583
|
+
this.journal?.step({
|
|
2584
|
+
stage: 'context',
|
|
2585
|
+
title: `Conversation context: ${history.length} earlier turn(s) of '${source}'`,
|
|
2586
|
+
data: history,
|
|
2587
|
+
});
|
|
2588
|
+
}
|
|
2232
2589
|
const raw = await this.produceAnswer(question, history, source);
|
|
2233
2590
|
// The LLM signals "I'm waiting for the user's reply" by appending the [[LISTEN]] control marker
|
|
2234
|
-
// (see followUpHint). Strip it here so it's neither stored, shown nor spoken, and
|
|
2235
|
-
//
|
|
2236
|
-
|
|
2237
|
-
|
|
2591
|
+
// (see followUpHint). Strip it here so it's neither stored, shown nor spoken, and keep it per source:
|
|
2592
|
+
// two satellites answered at the same time must not hand each other their mic-on.
|
|
2593
|
+
const followUp = /\[\[LISTEN\]\]/i.test(raw);
|
|
2594
|
+
this.followUps.set(source, followUp);
|
|
2595
|
+
if (followUp) {
|
|
2238
2596
|
this.journal?.note({ followUp: true });
|
|
2239
2597
|
}
|
|
2240
2598
|
const result = raw.replace(/\s*\[\[LISTEN\]\]\s*/gi, ' ').trim();
|
|
2599
|
+
this.journal?.step({
|
|
2600
|
+
stage: 'answer',
|
|
2601
|
+
title: result
|
|
2602
|
+
? `Answer: ${result.length} characters${followUp ? ', waits for a reply' : ''}`
|
|
2603
|
+
: 'No spoken answer',
|
|
2604
|
+
data: { answer: result, raw: raw !== result ? raw : undefined, followUp },
|
|
2605
|
+
});
|
|
2241
2606
|
if (useCtx && result) {
|
|
2242
2607
|
this.context.add(source, question, result);
|
|
2243
2608
|
}
|
|
2244
2609
|
return result;
|
|
2245
2610
|
}
|
|
2246
|
-
/**
|
|
2247
|
-
|
|
2611
|
+
/** Per source: whether its last {@link answer} asked the user something (LLM-signalled via [[LISTEN]]). */
|
|
2612
|
+
followUps = new Map();
|
|
2613
|
+
/**
|
|
2614
|
+
* Should the satellite behind `source` re-open its mic after this reply? Primarily the LLM's own
|
|
2615
|
+
* [[LISTEN]] signal, with a question mark in the reply as the fallback (the NLU and the local tiers
|
|
2616
|
+
* cannot emit the marker, but they do ask "Meinst du Büro oder Wohnzimmer?").
|
|
2617
|
+
*/
|
|
2618
|
+
wantsFollowUp(source, answer) {
|
|
2619
|
+
if (!answer) {
|
|
2620
|
+
return null;
|
|
2621
|
+
}
|
|
2622
|
+
if (this.followUps.get(source)) {
|
|
2623
|
+
return 'llm';
|
|
2624
|
+
}
|
|
2625
|
+
return this.expectsFollowUp(answer) ? 'question' : null;
|
|
2626
|
+
}
|
|
2627
|
+
/** {@link wantsFollowUp} for a voice transport, logged, as the `listen` flag of the reply. */
|
|
2628
|
+
followUpFor(source, answer) {
|
|
2629
|
+
const why = this.wantsFollowUp(source, answer);
|
|
2630
|
+
if (why) {
|
|
2631
|
+
this.log.info(`→ mic-on to satellite '${source}' (${why === 'llm' ? 'LLM signalled a follow-up' : 'reply is a question'}).`);
|
|
2632
|
+
}
|
|
2633
|
+
return !!why;
|
|
2634
|
+
}
|
|
2248
2635
|
/**
|
|
2249
2636
|
* System-prompt instruction letting the LLM explicitly signal a follow-up: append [[LISTEN]] when it
|
|
2250
2637
|
* asks the user something and waits for an answer. More reliable than a "?" heuristic and lets a
|
|
@@ -2321,6 +2708,7 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
2321
2708
|
const n = this.stopRinging();
|
|
2322
2709
|
this.log.info(`Ring stopped by voice (${n} session(s)).`);
|
|
2323
2710
|
this.journal?.note({ tier: 'stop' });
|
|
2711
|
+
this.journal?.step({ stage: 'stop', title: `Stop word: silenced ${n} ringing timer/alarm(s)` });
|
|
2324
2712
|
const lang = String(this.config.voiceLanguage || this.language || 'en');
|
|
2325
2713
|
return lang === 'ru' ? 'Хорошо.' : lang === 'de' ? 'Ok.' : 'Okay.';
|
|
2326
2714
|
}
|
|
@@ -2332,6 +2720,11 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
2332
2720
|
if (this.pending.deliver(source, question)) {
|
|
2333
2721
|
this.log.info(`Answer to "${asked}" from source '${source || 'any'}': ${question}`);
|
|
2334
2722
|
this.journal?.note({ tier: 'askReply', detail: asked || undefined });
|
|
2723
|
+
this.journal?.step({
|
|
2724
|
+
stage: 'askReply',
|
|
2725
|
+
title: `Reply to the open question "${asked}"`,
|
|
2726
|
+
data: { question: asked, reply: question },
|
|
2727
|
+
});
|
|
2335
2728
|
return '';
|
|
2336
2729
|
}
|
|
2337
2730
|
}
|
|
@@ -2342,11 +2735,38 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
2342
2735
|
if (routine) {
|
|
2343
2736
|
this.log.info(`Routine '${routine.name}' triggered (source='${source}').`);
|
|
2344
2737
|
this.journal?.note({ tier: 'routine', detail: routine.name });
|
|
2738
|
+
this.journal?.step({ stage: 'routine', title: `Routine "${routine.name}" matched`, data: routine });
|
|
2345
2739
|
await this.runRoutine(routine, source);
|
|
2346
2740
|
return routine.reply;
|
|
2347
2741
|
}
|
|
2742
|
+
if (this.routines.length) {
|
|
2743
|
+
this.journal?.step({
|
|
2744
|
+
stage: 'routine',
|
|
2745
|
+
title: `No routine matched (${this.routines.length} configured)`,
|
|
2746
|
+
data: { phrases: this.routines.map(r => ({ name: r.name, phrases: r.phrases })) },
|
|
2747
|
+
});
|
|
2748
|
+
}
|
|
2348
2749
|
// Tier 0: rule-based NLU (device commands) — fastest, offline, free.
|
|
2349
2750
|
if (this.config.useLocalNlu) {
|
|
2751
|
+
// "Spiele den Kanal X ab" first: the device NLU would find the TV in "… auf dem Fernseher" and
|
|
2752
|
+
// answer with its power state.
|
|
2753
|
+
const yt = this.youtubePlayers.length ? (0, youtube_1.parseYouTubeCommand)(question, this.youtubePlayers) : null;
|
|
2754
|
+
if (yt) {
|
|
2755
|
+
this.log.info(`YouTube request understood locally: ${yt.kind} "${yt.query}" (source='${source}').`);
|
|
2756
|
+
this.journal?.note({ tier: 'nlu', detail: `youtube ${yt.kind}` });
|
|
2757
|
+
const start = Date.now();
|
|
2758
|
+
const res = await this.playYouTube(yt.query, yt.kind, yt.player);
|
|
2759
|
+
this.journal?.step({
|
|
2760
|
+
stage: 'youtube',
|
|
2761
|
+
title: `YouTube ${yt.kind} "${yt.query}"${yt.player ? ` on "${yt.player}"` : ''}`,
|
|
2762
|
+
ms: Date.now() - start,
|
|
2763
|
+
data: { parsed: yt, result: res },
|
|
2764
|
+
});
|
|
2765
|
+
return res.text;
|
|
2766
|
+
}
|
|
2767
|
+
if (this.youtubePlayers.length) {
|
|
2768
|
+
this.journal?.step({ stage: 'youtube', title: 'Not a YouTube command' });
|
|
2769
|
+
}
|
|
2350
2770
|
try {
|
|
2351
2771
|
const handled = await this.tryLocalNlu(question, source);
|
|
2352
2772
|
if (handled !== null) {
|
|
@@ -2356,36 +2776,49 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
2356
2776
|
}
|
|
2357
2777
|
catch (e) {
|
|
2358
2778
|
this.log.debug(`NLU skipped: ${e.message}`);
|
|
2779
|
+
this.journal?.step({
|
|
2780
|
+
stage: 'nlu',
|
|
2781
|
+
level: 'warn',
|
|
2782
|
+
title: `NLU gave up, the request goes on: ${e.message}`,
|
|
2783
|
+
data: (0, journal_1.errorInfo)(e),
|
|
2784
|
+
});
|
|
2359
2785
|
}
|
|
2360
2786
|
}
|
|
2787
|
+
else {
|
|
2788
|
+
this.journal?.step({ stage: 'nlu', title: 'Rule-based NLU is switched off (useLocalNlu)' });
|
|
2789
|
+
}
|
|
2361
2790
|
// Current weather from the configured adapter — prepended to the user turn of BOTH LLM tiers so
|
|
2362
2791
|
// "how's the weather?" is answered from real data (the local model has no tools and would otherwise
|
|
2363
2792
|
// invent a forecast; the cloud model saves a get_weather round-trip). Cached, see buildWeatherContext.
|
|
2364
2793
|
const weather = await this.buildWeatherContext();
|
|
2365
2794
|
// Tier 1a: local LLM — answers general questions offline; emits HANDOFF for anything needing tools.
|
|
2795
|
+
// With a cloud LLM it only goes first when asked to (`localLlmFirst`): on a Raspberry Pi a small model
|
|
2796
|
+
// needs 10–25 s, often only to hand off, and the cloud then starts from zero. By default it is the
|
|
2797
|
+
// fallback for when the cloud fails (see below).
|
|
2366
2798
|
// Skipped inside an ongoing conversation (history present): the tool-free local model gets no history,
|
|
2367
2799
|
// so it would answer a follow-up like "yes" context-free and swallow it — let the cloud LLM (which has
|
|
2368
2800
|
// the thread and the tools) handle follow-ups instead. Without a cloud LLM there is nobody to hand
|
|
2369
2801
|
// follow-ups to, so the local model takes them too.
|
|
2370
|
-
|
|
2371
|
-
|
|
2372
|
-
|
|
2373
|
-
|
|
2374
|
-
|
|
2375
|
-
|
|
2376
|
-
if (ans && !(0, localLlm_1.isHandoff)(ans)) {
|
|
2377
|
-
this.log.info(`Answered by local LLM (source='${source}').`);
|
|
2378
|
-
this.journal?.note({ tier: 'localLlm', detail: this.localLlmName() });
|
|
2379
|
-
return ans;
|
|
2380
|
-
}
|
|
2381
|
-
this.log.debug('Local LLM handed off to the cloud LLM.');
|
|
2382
|
-
}
|
|
2383
|
-
catch (e) {
|
|
2384
|
-
this.log.debug(`Local LLM error: ${e.message}`);
|
|
2802
|
+
const useLocal = !!(this.config.useLocalLlm && this.localLlm);
|
|
2803
|
+
const localFirst = useLocal && (!this.agent || !!this.config.localLlmFirst);
|
|
2804
|
+
if (localFirst && (!history.length || !this.agent)) {
|
|
2805
|
+
const ans = await this.askLocalLlm(question, weather, source);
|
|
2806
|
+
if (ans) {
|
|
2807
|
+
return ans;
|
|
2385
2808
|
}
|
|
2386
2809
|
}
|
|
2387
|
-
else if (
|
|
2810
|
+
else if (localFirst) {
|
|
2388
2811
|
this.log.debug('Local LLM skipped (follow-up in an ongoing conversation → cloud LLM).');
|
|
2812
|
+
this.journal?.step({
|
|
2813
|
+
stage: 'localLlm',
|
|
2814
|
+
title: 'Local LLM skipped: follow-up in an ongoing conversation goes to the cloud LLM',
|
|
2815
|
+
});
|
|
2816
|
+
}
|
|
2817
|
+
else if (useLocal) {
|
|
2818
|
+
this.journal?.step({
|
|
2819
|
+
stage: 'localLlm',
|
|
2820
|
+
title: 'Local LLM not asked first: it is the fallback for when the cloud LLM fails (localLlmFirst)',
|
|
2821
|
+
});
|
|
2389
2822
|
}
|
|
2390
2823
|
// Tier 2: cloud LLM with full tool access. Inject a compact device list into the system prompt so
|
|
2391
2824
|
// the model can act without a first `list_devices` round-trip (kept cached via prompt caching).
|
|
@@ -2394,6 +2827,11 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
2394
2827
|
// read out as a technical message.
|
|
2395
2828
|
this.log.info(`No local tier could answer and no LLM provider is configured (source='${source}').`);
|
|
2396
2829
|
this.journal?.note({ tier: 'none' });
|
|
2830
|
+
this.journal?.step({
|
|
2831
|
+
stage: 'none',
|
|
2832
|
+
level: 'warn',
|
|
2833
|
+
title: 'Nothing local understood the request and no LLM provider is configured',
|
|
2834
|
+
});
|
|
2397
2835
|
return this.localOnlyFallback();
|
|
2398
2836
|
}
|
|
2399
2837
|
const ctx = await this.buildDeviceContext();
|
|
@@ -2405,12 +2843,99 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
2405
2843
|
this.log.info(`Cloud LLM [${this.agent.model || 'auto'}] (source='${source}', context=${history.length} turn(s)${mem ? ', memory' : ''}${weather ? ', weather' : ''}${this.presence?.configured ? ', presence' : ''}) — running tool loop…`);
|
|
2406
2844
|
// Prepend the current date/time (and the weather line, if a source is configured) to the user turn —
|
|
2407
2845
|
// cache-safe, see buildTimeContext. The stored history keeps the untouched question.
|
|
2408
|
-
const
|
|
2846
|
+
const time = this.buildTimeContext();
|
|
2847
|
+
const questionForLlm = [time, weather, question].filter(Boolean).join('\n\n');
|
|
2409
2848
|
this.journal?.note({
|
|
2410
2849
|
tier: 'cloud',
|
|
2411
2850
|
detail: `${this.config.provider || 'openai'} · ${this.agent.model || 'auto'}`,
|
|
2412
2851
|
});
|
|
2413
|
-
|
|
2852
|
+
// What went into the prompt, piece by piece. The device list is only counted: the model's opening
|
|
2853
|
+
// step carries the whole system prompt, and with it the list.
|
|
2854
|
+
const deviceLines = ctx ? ctx.split('\n').length - 1 : 0;
|
|
2855
|
+
this.journal?.step({
|
|
2856
|
+
stage: 'context',
|
|
2857
|
+
title: `Prompt: ${deviceLines} devices${mem ? ', memory' : ''}${weather ? ', weather' : ''}${followUp ? ', follow-up marker' : ''}`,
|
|
2858
|
+
data: {
|
|
2859
|
+
systemPrompt: this.config.systemPrompt || '',
|
|
2860
|
+
followUpHint: followUp || undefined,
|
|
2861
|
+
memory: mem || undefined,
|
|
2862
|
+
devices: deviceLines,
|
|
2863
|
+
time,
|
|
2864
|
+
weather: weather || undefined,
|
|
2865
|
+
},
|
|
2866
|
+
});
|
|
2867
|
+
try {
|
|
2868
|
+
return await this.agent.ask(questionForLlm, sys, history);
|
|
2869
|
+
}
|
|
2870
|
+
catch (e) {
|
|
2871
|
+
// Offline, quota, outage: the local model is the fallback — unless it was already asked above.
|
|
2872
|
+
if (useLocal && !localFirst) {
|
|
2873
|
+
this.log.warn(`Cloud LLM failed (${e.message}) — asking the local LLM instead.`);
|
|
2874
|
+
this.journal?.step({
|
|
2875
|
+
stage: 'localLlm',
|
|
2876
|
+
level: 'warn',
|
|
2877
|
+
title: `Cloud LLM failed (${e.message}) — asking the local LLM instead`,
|
|
2878
|
+
});
|
|
2879
|
+
const ans = await this.askLocalLlm(question, weather, source);
|
|
2880
|
+
if (ans) {
|
|
2881
|
+
return ans;
|
|
2882
|
+
}
|
|
2883
|
+
}
|
|
2884
|
+
throw e;
|
|
2885
|
+
}
|
|
2886
|
+
}
|
|
2887
|
+
/**
|
|
2888
|
+
* Ask the local model, bounded by `localLlmTimeoutSec`. Returns its answer, or null when it handed off,
|
|
2889
|
+
* timed out or failed — the caller then goes on to the next tier.
|
|
2890
|
+
*/
|
|
2891
|
+
async askLocalLlm(question, weather, source) {
|
|
2892
|
+
if (!this.localLlm) {
|
|
2893
|
+
return null;
|
|
2894
|
+
}
|
|
2895
|
+
const timeoutMs = Math.max(1, Number(this.config.localLlmTimeoutSec) || LOCAL_LLM_TIMEOUT_SEC) * 1000;
|
|
2896
|
+
const name = this.localLlmName();
|
|
2897
|
+
// The local model has no tools: without these lines it would invent the weather and guess who is
|
|
2898
|
+
// at home.
|
|
2899
|
+
const facts = [weather, this.buildPresenceContext()].filter(Boolean).join('\n\n');
|
|
2900
|
+
const prompt = facts ? `${facts}\n\n${question}` : question;
|
|
2901
|
+
const busy = this.localLlm.running;
|
|
2902
|
+
const data = { model: name, timeoutMs, system: this.localLlm.systemPrompt(), prompt };
|
|
2903
|
+
const start = Date.now();
|
|
2904
|
+
try {
|
|
2905
|
+
const ans = await this.localLlm.ask(prompt, timeoutMs);
|
|
2906
|
+
const ms = Date.now() - start;
|
|
2907
|
+
if (ans && !(0, localLlm_1.isHandoff)(ans)) {
|
|
2908
|
+
this.log.info(`Answered by local LLM (source='${source}').`);
|
|
2909
|
+
this.journal?.note({ tier: 'localLlm', detail: name });
|
|
2910
|
+
this.journal?.step({
|
|
2911
|
+
stage: 'localLlm',
|
|
2912
|
+
title: `Local LLM ${name}: answered`,
|
|
2913
|
+
ms,
|
|
2914
|
+
data: { ...data, answer: ans },
|
|
2915
|
+
});
|
|
2916
|
+
return ans;
|
|
2917
|
+
}
|
|
2918
|
+
this.log.debug('Local LLM handed off.');
|
|
2919
|
+
this.journal?.step({
|
|
2920
|
+
stage: 'localLlm',
|
|
2921
|
+
title: busy
|
|
2922
|
+
? `Local LLM ${name}: busy with another question → handed off`
|
|
2923
|
+
: `Local LLM ${name}: handed off`,
|
|
2924
|
+
ms,
|
|
2925
|
+
data: { ...data, answer: ans },
|
|
2926
|
+
});
|
|
2927
|
+
}
|
|
2928
|
+
catch (e) {
|
|
2929
|
+
this.log.info(`Local LLM gave no answer: ${e.message}`);
|
|
2930
|
+
this.journal?.step({
|
|
2931
|
+
stage: 'localLlm',
|
|
2932
|
+
level: 'warn',
|
|
2933
|
+
title: `Local LLM ${name}: ${e.message}`,
|
|
2934
|
+
ms: Date.now() - start,
|
|
2935
|
+
data: { ...data, error: (0, journal_1.errorInfo)(e) },
|
|
2936
|
+
});
|
|
2937
|
+
}
|
|
2938
|
+
return null;
|
|
2414
2939
|
}
|
|
2415
2940
|
/** Reply when nothing local understood the request and there is no cloud LLM (provider "none"). */
|
|
2416
2941
|
localOnlyFallback() {
|
|
@@ -2521,35 +3046,27 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
2521
3046
|
else {
|
|
2522
3047
|
// Switched off: do not keep what was recorded while it was on.
|
|
2523
3048
|
await fs.promises.rm(file, { force: true }).catch(() => { });
|
|
3049
|
+
await fs.promises.rm((0, journal_1.traceDirFor)(file), { recursive: true, force: true }).catch(() => { });
|
|
2524
3050
|
}
|
|
2525
3051
|
}
|
|
2526
|
-
/**
|
|
3052
|
+
/**
|
|
3053
|
+
* Wrap the MCP connection so every state the pipeline reads or writes lands in the journal entry — and a
|
|
3054
|
+
* call the NLU makes directly, with its result, in the request's trace.
|
|
3055
|
+
*/
|
|
2527
3056
|
journalMcp(mcp) {
|
|
2528
3057
|
return {
|
|
2529
3058
|
listTools: () => mcp.listTools(),
|
|
2530
|
-
callTool: (name, args) =>
|
|
2531
|
-
this.journal
|
|
2532
|
-
|
|
2533
|
-
},
|
|
3059
|
+
callTool: (name, args) => this.journal
|
|
3060
|
+
? this.journal.stateCall(name, args, () => mcp.callTool(name, args))
|
|
3061
|
+
: mcp.callTool(name, args),
|
|
2534
3062
|
close: () => mcp.close(),
|
|
2535
3063
|
};
|
|
2536
3064
|
}
|
|
2537
|
-
/** Wrap an LLM tool so its calls (arguments, result, duration) land in the journal entry. */
|
|
3065
|
+
/** Wrap an LLM tool so its calls (arguments, result, duration) land in the journal entry and its trace. */
|
|
2538
3066
|
journalTool(tool) {
|
|
2539
3067
|
return {
|
|
2540
3068
|
...tool,
|
|
2541
|
-
run:
|
|
2542
|
-
const start = Date.now();
|
|
2543
|
-
try {
|
|
2544
|
-
const result = await tool.run(args);
|
|
2545
|
-
this.journal?.tool(tool.name, args, Date.now() - start, result);
|
|
2546
|
-
return result;
|
|
2547
|
-
}
|
|
2548
|
-
catch (e) {
|
|
2549
|
-
this.journal?.tool(tool.name, args, Date.now() - start, undefined, e.message);
|
|
2550
|
-
throw e;
|
|
2551
|
-
}
|
|
2552
|
-
},
|
|
3069
|
+
run: args => (this.journal ? this.journal.toolCall(tool.name, args, () => tool.run(args)) : tool.run(args)),
|
|
2553
3070
|
};
|
|
2554
3071
|
}
|
|
2555
3072
|
/** Short name of the local model for the journal ("qwen2.5-1.5b-instruct-q4_k_m"). */
|
|
@@ -2647,7 +3164,7 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
2647
3164
|
dataDir,
|
|
2648
3165
|
modelUrl: (this.config.localLlmModelUrl || '').trim() || localLlm_1.DEFAULT_LOCAL_MODEL_URL,
|
|
2649
3166
|
systemPrompt: this.config.systemPrompt || '',
|
|
2650
|
-
maxTokens: this.config.maxTokens ||
|
|
3167
|
+
maxTokens: Math.min(this.config.maxTokens || LOCAL_LLM_MAX_TOKENS, LOCAL_LLM_MAX_TOKENS),
|
|
2651
3168
|
log: this.log,
|
|
2652
3169
|
});
|
|
2653
3170
|
this.lastLlmPct = -1;
|
|
@@ -2680,20 +3197,56 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
2680
3197
|
}
|
|
2681
3198
|
// Timer intents match device-independently (parse() checks them first), so build the NLU even
|
|
2682
3199
|
// when no devices are known and always try timers before bailing out.
|
|
3200
|
+
const start = Date.now();
|
|
2683
3201
|
const { rooms, devices } = await this.getNluDevices();
|
|
2684
|
-
const
|
|
2685
|
-
|
|
3202
|
+
const aliases = this.getNluAliases();
|
|
3203
|
+
const intents = new nlu_1.Nlu(rooms, devices, aliases, {
|
|
3204
|
+
weather: !!(this.config.weatherInstance || '').trim(),
|
|
3205
|
+
}).parseAll(question);
|
|
3206
|
+
const blockers = intents.map(i => this.nluBlocker(i, devices.length));
|
|
3207
|
+
this.journal?.step({
|
|
3208
|
+
stage: 'nlu',
|
|
3209
|
+
title: !intents.length
|
|
3210
|
+
? `NLU: nothing recognised (${devices.length} devices, ${rooms.length} rooms known)`
|
|
3211
|
+
: blockers.some(Boolean)
|
|
3212
|
+
? `NLU: ${intents.map(i => i.action).join(' + ')} — cannot run here: ${blockers.filter(Boolean).join('; ')}`
|
|
3213
|
+
: `NLU: ${intents.map(describeIntent).join(' + ')}`,
|
|
3214
|
+
level: intents.length && blockers.some(Boolean) ? 'warn' : undefined,
|
|
3215
|
+
ms: Date.now() - start,
|
|
3216
|
+
data: {
|
|
3217
|
+
devices: devices.length,
|
|
3218
|
+
rooms,
|
|
3219
|
+
aliases: aliases.length ? aliases : undefined,
|
|
3220
|
+
intents: intents.map((intent, i) => ({ ...intent, blockedBecause: blockers[i] || undefined })),
|
|
3221
|
+
},
|
|
3222
|
+
});
|
|
3223
|
+
if (!intents.length || blockers.some(Boolean)) {
|
|
2686
3224
|
return null; // nothing matched, or one part cannot run here → let the LLM answer the whole thing
|
|
2687
3225
|
}
|
|
2688
3226
|
this.journal?.note({ tier: 'nlu', detail: intents.map(i => i.action).join(' + ') });
|
|
2689
3227
|
const answers = [];
|
|
2690
3228
|
let failed = false;
|
|
2691
3229
|
for (const intent of intents) {
|
|
3230
|
+
const started = Date.now();
|
|
2692
3231
|
try {
|
|
2693
|
-
|
|
3232
|
+
const answer = await this.executeNluIntent(intent, source);
|
|
3233
|
+
answers.push(answer);
|
|
3234
|
+
this.journal?.step({
|
|
3235
|
+
stage: 'nlu',
|
|
3236
|
+
title: `Executed ${describeIntent(intent)}`,
|
|
3237
|
+
ms: Date.now() - started,
|
|
3238
|
+
data: { answer },
|
|
3239
|
+
});
|
|
2694
3240
|
}
|
|
2695
3241
|
catch (e) {
|
|
2696
3242
|
failed = true;
|
|
3243
|
+
this.journal?.step({
|
|
3244
|
+
stage: 'nlu',
|
|
3245
|
+
level: 'error',
|
|
3246
|
+
title: `${describeIntent(intent)} failed: ${e.message}${answers.length ? '' : ' → the LLM takes over'}`,
|
|
3247
|
+
ms: Date.now() - started,
|
|
3248
|
+
data: (0, journal_1.errorInfo)(e),
|
|
3249
|
+
});
|
|
2697
3250
|
// Nothing executed yet → hand the whole utterance to the LLM. Afterwards a state has already
|
|
2698
3251
|
// been written and the LLM would repeat it, so report the failed part instead of bailing out.
|
|
2699
3252
|
if (!answers.length) {
|
|
@@ -2716,35 +3269,44 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
2716
3269
|
intents.every(i => CONTROL_ACTIONS.has(i.action)) &&
|
|
2717
3270
|
(await this.playConfirmationTone(source))) {
|
|
2718
3271
|
this.log.debug(`NLU confirmed with a tone instead of "${answer}".`);
|
|
3272
|
+
this.journal?.step({ stage: 'nlu', title: 'Confirmed with a tone instead of words', data: { answer } });
|
|
2719
3273
|
return '';
|
|
2720
3274
|
}
|
|
2721
3275
|
return answer;
|
|
2722
3276
|
}
|
|
2723
3277
|
/**
|
|
2724
|
-
*
|
|
2725
|
-
* is executed: a missing manager, no known devices or disabled writes send the whole
|
|
2726
|
-
* LLM (which can explain why) instead of silently doing nothing — or only half of it.
|
|
3278
|
+
* Why this intent cannot run locally ('' = it can). Checked for **every** part of a combined command
|
|
3279
|
+
* before anything is executed: a missing manager, no known devices or disabled writes send the whole
|
|
3280
|
+
* utterance to the LLM (which can explain why) instead of silently doing nothing — or only half of it.
|
|
2727
3281
|
*/
|
|
2728
|
-
|
|
3282
|
+
nluBlocker(intent, deviceCount) {
|
|
2729
3283
|
switch (intent.action) {
|
|
2730
3284
|
case 'timerSet':
|
|
2731
3285
|
case 'timerQuery':
|
|
2732
3286
|
case 'timerCancel':
|
|
2733
|
-
return
|
|
3287
|
+
return this.timers ? '' : 'timers are not running';
|
|
2734
3288
|
case 'alarmSet':
|
|
2735
3289
|
case 'alarmQuery':
|
|
2736
3290
|
case 'alarmCancel':
|
|
2737
|
-
return
|
|
3291
|
+
return this.alarms ? '' : 'alarms are not running';
|
|
2738
3292
|
case 'timeQuery':
|
|
2739
3293
|
case 'dateQuery':
|
|
2740
|
-
return
|
|
3294
|
+
return '';
|
|
3295
|
+
case 'weatherQuery':
|
|
3296
|
+
return (this.config.weatherInstance || '').trim() ? '' : 'no weather instance configured';
|
|
3297
|
+
case 'volumeSet':
|
|
3298
|
+
return this.voice || this.esphome || this.nativeSatFrom.size ? '' : 'no satellites';
|
|
2741
3299
|
default:
|
|
2742
3300
|
// Device intents; writes additionally require the coarse "allow control" toggle.
|
|
2743
|
-
|
|
2744
|
-
|
|
3301
|
+
if (!deviceCount) {
|
|
3302
|
+
return 'no devices known';
|
|
3303
|
+
}
|
|
3304
|
+
return ['on', 'off', 'level', 'color'].includes(intent.action) && !this.config.allowWriteStates
|
|
3305
|
+
? 'writing states is not allowed (allowWriteStates)'
|
|
3306
|
+
: '';
|
|
2745
3307
|
}
|
|
2746
3308
|
}
|
|
2747
|
-
/** Execute one intent that {@link
|
|
3309
|
+
/** Execute one intent that {@link nluBlocker} accepted, and return its spoken answer. */
|
|
2748
3310
|
async executeNluIntent(intent, source) {
|
|
2749
3311
|
switch (intent.action) {
|
|
2750
3312
|
case 'timerSet':
|
|
@@ -2758,10 +3320,97 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
2758
3320
|
case 'timeQuery':
|
|
2759
3321
|
case 'dateQuery':
|
|
2760
3322
|
return this.executeTimeIntent(intent);
|
|
3323
|
+
case 'weatherQuery':
|
|
3324
|
+
return this.executeWeatherIntent(intent);
|
|
3325
|
+
case 'volumeSet':
|
|
3326
|
+
return this.executeVolumeIntent(intent, source);
|
|
2761
3327
|
default:
|
|
2762
3328
|
return this.executeIntent(intent);
|
|
2763
3329
|
}
|
|
2764
3330
|
}
|
|
3331
|
+
/**
|
|
3332
|
+
* Speak the weather straight from the configured adapter. Throws when the report has nothing for the
|
|
3333
|
+
* asked day (or the adapter has no mapper), so the question goes on to the LLM and its tool instead.
|
|
3334
|
+
*/
|
|
3335
|
+
async executeWeatherIntent(intent) {
|
|
3336
|
+
const res = await this.readWeather(this.config.weatherInstance || '');
|
|
3337
|
+
if (!res.report) {
|
|
3338
|
+
throw new Error(res.error || 'no weather report');
|
|
3339
|
+
}
|
|
3340
|
+
const lang = String(this.config.voiceLanguage || this.language || 'en');
|
|
3341
|
+
const text = (0, weather_1.buildWeatherAnswer)(res.report, lang, intent.when);
|
|
3342
|
+
if (!text) {
|
|
3343
|
+
throw new Error(`the weather report has no data for '${intent.when || 'now'}'`);
|
|
3344
|
+
}
|
|
3345
|
+
return text;
|
|
3346
|
+
}
|
|
3347
|
+
/**
|
|
3348
|
+
* Change the playback volume of a satellite: the one in the named room, else the one that heard the
|
|
3349
|
+
* command. Each transport has its own knob — the native satellite's `volume` state, the media player
|
|
3350
|
+
* of an ESPHome device, the `volume` message of a UDP satellite.
|
|
3351
|
+
*/
|
|
3352
|
+
async executeVolumeIntent(intent, source) {
|
|
3353
|
+
const lang = String(this.config.voiceLanguage || this.language || 'en');
|
|
3354
|
+
const satId = intent.room ? this.resolveSatelliteId(intent.room) : this.announceTargetForSource(source);
|
|
3355
|
+
const absolute = typeof intent.value === 'number' ? intent.value : null;
|
|
3356
|
+
const delta = intent.delta || 0;
|
|
3357
|
+
const next = (current) => Math.min(100, Math.max(0, Math.round(absolute ?? current + delta)));
|
|
3358
|
+
// 1 процент, 2 процента, 5 процентов (11–14 take the last form).
|
|
3359
|
+
const ruPercent = (n) => n % 100 > 10 && n % 100 < 20
|
|
3360
|
+
? 'процентов'
|
|
3361
|
+
: n % 10 === 1
|
|
3362
|
+
? 'процент'
|
|
3363
|
+
: n % 10 >= 2 && n % 10 <= 4
|
|
3364
|
+
? 'процента'
|
|
3365
|
+
: 'процентов';
|
|
3366
|
+
const confirm = (level) => lang === 'ru'
|
|
3367
|
+
? `Громкость ${level} ${ruPercent(level)}.`
|
|
3368
|
+
: lang === 'de'
|
|
3369
|
+
? `Lautstärke ${level} Prozent.`
|
|
3370
|
+
: `Volume ${level} percent.`;
|
|
3371
|
+
if (satId) {
|
|
3372
|
+
const device = this.satDeviceById.get(satId) || satId;
|
|
3373
|
+
const instance = this.nativeSatFrom.get(satId);
|
|
3374
|
+
if (instance) {
|
|
3375
|
+
const stateId = `${instance}.volume`;
|
|
3376
|
+
const current = Number((await this.getForeignStateAsync(stateId))?.val ?? 50);
|
|
3377
|
+
const level = next(Number.isFinite(current) ? current : 50);
|
|
3378
|
+
await this.setForeignStateAsync(stateId, level);
|
|
3379
|
+
return confirm(level);
|
|
3380
|
+
}
|
|
3381
|
+
const player = this.esphome?.entities(device).find(e => e.kind === 'mediaPlayer');
|
|
3382
|
+
if (player) {
|
|
3383
|
+
const current = (player.value?.volume ?? 0.5) * 100;
|
|
3384
|
+
const level = next(current);
|
|
3385
|
+
if (this.esphome.setEntity(device, player.objectId, level / 100)) {
|
|
3386
|
+
return confirm(level);
|
|
3387
|
+
}
|
|
3388
|
+
}
|
|
3389
|
+
// A UDP satellite works out a relative step itself (it knows its level) and reports the result.
|
|
3390
|
+
if (this.voice?.has(device)) {
|
|
3391
|
+
this.voice.volume(device, absolute !== null ? { level: absolute } : { delta });
|
|
3392
|
+
if (absolute !== null) {
|
|
3393
|
+
return confirm(next(0));
|
|
3394
|
+
}
|
|
3395
|
+
return lang === 'ru'
|
|
3396
|
+
? delta > 0
|
|
3397
|
+
? 'Сделала громче.'
|
|
3398
|
+
: 'Сделала тише.'
|
|
3399
|
+
: lang === 'de'
|
|
3400
|
+
? delta > 0
|
|
3401
|
+
? 'Etwas lauter.'
|
|
3402
|
+
: 'Etwas leiser.'
|
|
3403
|
+
: delta > 0
|
|
3404
|
+
? 'A bit louder.'
|
|
3405
|
+
: 'A bit quieter.';
|
|
3406
|
+
}
|
|
3407
|
+
}
|
|
3408
|
+
return lang === 'ru'
|
|
3409
|
+
? 'Я не знаю, громкость какой колонки изменить.'
|
|
3410
|
+
: lang === 'de'
|
|
3411
|
+
? 'Ich weiß nicht, bei welchem Lautsprecher ich die Lautstärke ändern soll.'
|
|
3412
|
+
: "I don't know which speaker's volume to change.";
|
|
3413
|
+
}
|
|
2765
3414
|
/** Localized "this part did not work" sentence for a failed part of a combined command. */
|
|
2766
3415
|
nluFailureText(intent) {
|
|
2767
3416
|
const lang = String(this.config.voiceLanguage || this.language || 'en');
|
|
@@ -2811,14 +3460,36 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
2811
3460
|
return [];
|
|
2812
3461
|
}
|
|
2813
3462
|
}
|
|
2814
|
-
/**
|
|
2815
|
-
|
|
3463
|
+
/**
|
|
3464
|
+
* Device model for the NLU: friendly name, room, type and controls (controlType → state id). Cached
|
|
3465
|
+
* for {@link NLU_DEVICES_TTL_MS}; requests arriving while it is being built wait for the same build.
|
|
3466
|
+
*/
|
|
3467
|
+
getNluDevices() {
|
|
2816
3468
|
if (!this.mcp) {
|
|
2817
|
-
return { rooms: [], devices: [] };
|
|
3469
|
+
return Promise.resolve({ rooms: [], devices: [] });
|
|
2818
3470
|
}
|
|
2819
3471
|
// Match names in the assistant's interaction language (voice language, else system) so a
|
|
2820
3472
|
// Russian command matches Russian smartNames.
|
|
2821
3473
|
const nluLang = (this.config.voiceLanguage || this.language || 'en');
|
|
3474
|
+
const now = Date.now();
|
|
3475
|
+
const hit = this.nluDevices;
|
|
3476
|
+
if (hit && hit.key === nluLang && now - hit.at < NLU_DEVICES_TTL_MS) {
|
|
3477
|
+
return hit.result;
|
|
3478
|
+
}
|
|
3479
|
+
const entry = { key: nluLang, at: now, result: this.buildNluDevices(nluLang) };
|
|
3480
|
+
this.nluDevices = entry;
|
|
3481
|
+
// An empty model is usually a failed build (the manager not ready yet) — don't keep it for a minute.
|
|
3482
|
+
void entry.result.then(r => {
|
|
3483
|
+
if (!r.devices.length && this.nluDevices === entry) {
|
|
3484
|
+
this.nluDevices = null;
|
|
3485
|
+
}
|
|
3486
|
+
});
|
|
3487
|
+
return entry.result;
|
|
3488
|
+
}
|
|
3489
|
+
async buildNluDevices(nluLang) {
|
|
3490
|
+
if (!this.mcp) {
|
|
3491
|
+
return { rooms: [], devices: [] };
|
|
3492
|
+
}
|
|
2822
3493
|
try {
|
|
2823
3494
|
const res = await this.mcp.callTool('list_devices', { language: nluLang });
|
|
2824
3495
|
const parsed = JSON.parse(res.text);
|
|
@@ -3450,16 +4121,30 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
3450
4121
|
if (!this.config.verifyWrites || !stateId) {
|
|
3451
4122
|
return true;
|
|
3452
4123
|
}
|
|
3453
|
-
const
|
|
4124
|
+
const start = Date.now();
|
|
4125
|
+
const deadline = start + ACK_TIMEOUT_MS;
|
|
3454
4126
|
let last;
|
|
3455
4127
|
while (Date.now() < deadline) {
|
|
3456
4128
|
await this.delay(ACK_POLL_MS);
|
|
3457
4129
|
last = await this.getForeignStateAsync(stateId).catch(() => null);
|
|
3458
4130
|
if (last?.ack && valuesMatch(last.val, expected)) {
|
|
3459
4131
|
this.log.debug(`write to ${stateId} confirmed by the device.`);
|
|
4132
|
+
this.journal?.step({
|
|
4133
|
+
stage: 'state',
|
|
4134
|
+
title: `${stateId} confirmed by the device`,
|
|
4135
|
+
ms: Date.now() - start,
|
|
4136
|
+
data: { expected, state: last },
|
|
4137
|
+
});
|
|
3460
4138
|
return true;
|
|
3461
4139
|
}
|
|
3462
4140
|
}
|
|
4141
|
+
this.journal?.step({
|
|
4142
|
+
stage: 'state',
|
|
4143
|
+
level: 'warn',
|
|
4144
|
+
title: `${stateId}: no confirmation within ${ACK_TIMEOUT_MS} ms`,
|
|
4145
|
+
ms: Date.now() - start,
|
|
4146
|
+
data: { expected, state: last },
|
|
4147
|
+
});
|
|
3463
4148
|
this.log.warn(`No confirmation for ${stateId} within ${ACK_TIMEOUT_MS} ms ` +
|
|
3464
4149
|
`(last: ${JSON.stringify(last?.val)}, ack=${String(last?.ack)}). ` +
|
|
3465
4150
|
'If this device never acknowledges, switch "Verify device feedback" off.');
|
|
@@ -3938,6 +4623,383 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
3938
4623
|
},
|
|
3939
4624
|
};
|
|
3940
4625
|
}
|
|
4626
|
+
// ── YouTube on the TV (youtube-remote players) ──────────────────────────
|
|
4627
|
+
/** The player a request means: the named one, else the one in the requesting room, else the first. */
|
|
4628
|
+
youtubePlayerFor(name) {
|
|
4629
|
+
return (0, youtube_1.findPlayer)(this.youtubePlayers, name, this.requestOrigin.getStore()?.room || '');
|
|
4630
|
+
}
|
|
4631
|
+
/**
|
|
4632
|
+
* Find the device of a row that names a youtube-remote instance. Done again before every command: the
|
|
4633
|
+
* device id comes from the TV's screen id, so pairing the TV again replaces the device while we run.
|
|
4634
|
+
* false = nothing is paired with that instance.
|
|
4635
|
+
*/
|
|
4636
|
+
async resolveYouTubePlayer(player) {
|
|
4637
|
+
if (!player.instance) {
|
|
4638
|
+
return true;
|
|
4639
|
+
}
|
|
4640
|
+
let devices = [];
|
|
4641
|
+
try {
|
|
4642
|
+
const objs = await this.getForeignObjectsAsync(`${player.instance}.*`, 'device');
|
|
4643
|
+
devices = Object.values(objs).map(o => ({ id: o._id, name: this.youtubeDeviceName(o) }));
|
|
4644
|
+
}
|
|
4645
|
+
catch (e) {
|
|
4646
|
+
this.log.debug(`YouTube: cannot read the devices of ${player.instance}: ${e.message}`);
|
|
4647
|
+
}
|
|
4648
|
+
const device = (0, youtube_1.pickDevice)(devices, player);
|
|
4649
|
+
player.base = device?.id || '';
|
|
4650
|
+
player.target = device ? `${device.id}.youtube` : '';
|
|
4651
|
+
if (device && !player.name) {
|
|
4652
|
+
player.name = device.name || player.instance;
|
|
4653
|
+
}
|
|
4654
|
+
return !!device;
|
|
4655
|
+
}
|
|
4656
|
+
/** Display name of an object, in the voice language if it is translated. */
|
|
4657
|
+
youtubeDeviceName(obj) {
|
|
4658
|
+
const name = obj.common?.name;
|
|
4659
|
+
if (typeof name === 'string') {
|
|
4660
|
+
return name;
|
|
4661
|
+
}
|
|
4662
|
+
const lang = String(this.config.voiceLanguage || this.language || 'en');
|
|
4663
|
+
return name?.[lang] || name?.en || obj._id.split('.').pop() || obj._id;
|
|
4664
|
+
}
|
|
4665
|
+
/**
|
|
4666
|
+
* Options of the "Player" column: every youtube-remote instance (it finds its TV itself), and for an
|
|
4667
|
+
* instance with several paired TVs also each of them, to pin a row to one.
|
|
4668
|
+
*/
|
|
4669
|
+
async getYouTubeInstances() {
|
|
4670
|
+
const options = [];
|
|
4671
|
+
let instances = [];
|
|
4672
|
+
try {
|
|
4673
|
+
const view = await this.getObjectViewAsync('system', 'instance', {
|
|
4674
|
+
startkey: 'system.adapter.youtube-remote.',
|
|
4675
|
+
endkey: 'system.adapter.youtube-remote.香',
|
|
4676
|
+
});
|
|
4677
|
+
instances = (view?.rows || []).map(r => r.id.replace(/^system\.adapter\./, '')).sort();
|
|
4678
|
+
}
|
|
4679
|
+
catch (e) {
|
|
4680
|
+
this.log.warn(`getYouTubeInstances failed: ${e.message}`);
|
|
4681
|
+
}
|
|
4682
|
+
for (const instance of instances) {
|
|
4683
|
+
const objs = await this.getForeignObjectsAsync(`${instance}.*`, 'device').catch(() => ({}));
|
|
4684
|
+
const devices = Object.values(objs)
|
|
4685
|
+
.map(o => ({ id: o._id, name: this.youtubeDeviceName(o) }))
|
|
4686
|
+
.sort((a, b) => a.id.localeCompare(b.id));
|
|
4687
|
+
const names = devices.map(d => d.name).join(', ');
|
|
4688
|
+
options.push({ label: names ? `${instance} (${names})` : `${instance} (no TV paired)`, value: instance });
|
|
4689
|
+
if (devices.length > 1) {
|
|
4690
|
+
for (const d of devices) {
|
|
4691
|
+
options.push({ label: `${instance} › ${d.name}`, value: d.id });
|
|
4692
|
+
}
|
|
4693
|
+
}
|
|
4694
|
+
}
|
|
4695
|
+
return options;
|
|
4696
|
+
}
|
|
4697
|
+
/**
|
|
4698
|
+
* Find `query` on YouTube and play it on a player, switching its TV on first. The search runs before
|
|
4699
|
+
* anything is switched: a TV woken for a channel that does not exist would be the worst outcome.
|
|
4700
|
+
*
|
|
4701
|
+
* A TV that has to boot takes 10–30 s, so in that case the reply comes right away and the rest runs
|
|
4702
|
+
* in the background ({@link playWhenTvIsOn}) — over voice, half a minute of silence reads as "did not
|
|
4703
|
+
* hear me". An already running TV is answered only after the player acknowledged the command.
|
|
4704
|
+
*/
|
|
4705
|
+
async playYouTube(query, kind, playerName) {
|
|
4706
|
+
const lang = String(this.config.voiceLanguage || this.language || 'en');
|
|
4707
|
+
const player = this.youtubePlayerFor(playerName);
|
|
4708
|
+
if (!player) {
|
|
4709
|
+
const names = this.youtubePlayers.map(p => p.name).join(', ');
|
|
4710
|
+
return { ok: false, text: (0, youtube_1.youtubeReply)(lang, 'unknownPlayer', { query: playerName, names }) };
|
|
4711
|
+
}
|
|
4712
|
+
if (!(await this.resolveYouTubePlayer(player))) {
|
|
4713
|
+
return {
|
|
4714
|
+
ok: false,
|
|
4715
|
+
player: player.name,
|
|
4716
|
+
text: (0, youtube_1.youtubeReply)(lang, 'noDevice', { instance: player.instance }),
|
|
4717
|
+
};
|
|
4718
|
+
}
|
|
4719
|
+
if (!this.config.allowWriteStates) {
|
|
4720
|
+
return { ok: false, player: player.name, text: (0, youtube_1.youtubeReply)(lang, 'noControl') };
|
|
4721
|
+
}
|
|
4722
|
+
let hit = null;
|
|
4723
|
+
let alternatives = [];
|
|
4724
|
+
// Unsaid kind: what is named like a channel is the channel ("spiele Sailing OceanAir auf YouTube"),
|
|
4725
|
+
// anything else a video search — a video search for a channel name finds some video about it.
|
|
4726
|
+
const searchKind = kind === 'auto' ? 'video' : kind;
|
|
4727
|
+
const direct = (0, youtube_1.directYouTubeTarget)(query, searchKind);
|
|
4728
|
+
if (direct) {
|
|
4729
|
+
hit = { kind: searchKind, id: direct, title: query };
|
|
4730
|
+
}
|
|
4731
|
+
else {
|
|
4732
|
+
try {
|
|
4733
|
+
if (kind === 'auto') {
|
|
4734
|
+
const channels = await (0, youtube_1.searchYouTube)(query, 'channel', { language: lang });
|
|
4735
|
+
hit = channels.find(h => (0, youtube_1.isNamedChannel)(h, query)) || null;
|
|
4736
|
+
alternatives = channels.filter(h => h !== hit).slice(0, 3);
|
|
4737
|
+
}
|
|
4738
|
+
if (!hit) {
|
|
4739
|
+
const hits = await (0, youtube_1.searchYouTube)(query, searchKind, { language: lang });
|
|
4740
|
+
hit = (0, youtube_1.pickBest)(hits, query);
|
|
4741
|
+
alternatives = hits.filter(h => h !== hit).slice(0, 3);
|
|
4742
|
+
}
|
|
4743
|
+
}
|
|
4744
|
+
catch (e) {
|
|
4745
|
+
this.log.warn(`YouTube search for "${query}" failed: ${e.message}`);
|
|
4746
|
+
return { ok: false, player: player.name, text: (0, youtube_1.youtubeReply)(lang, 'searchFailed') };
|
|
4747
|
+
}
|
|
4748
|
+
if (!hit) {
|
|
4749
|
+
return {
|
|
4750
|
+
ok: false,
|
|
4751
|
+
player: player.name,
|
|
4752
|
+
text: (0, youtube_1.youtubeReply)(lang, 'notFound', { kind: searchKind, query }),
|
|
4753
|
+
};
|
|
4754
|
+
}
|
|
4755
|
+
}
|
|
4756
|
+
const title = (0, youtube_1.spokenTitle)(hit);
|
|
4757
|
+
this.log.info(`YouTube: ${kind} "${query}" → ${hit.kind} ${hit.id} (${hit.title}) on ${player.name}.`);
|
|
4758
|
+
const reply = { kind: hit.kind, title, player: player.name };
|
|
4759
|
+
if ((await this.tvPower(player)) === 'off') {
|
|
4760
|
+
try {
|
|
4761
|
+
await this.switchTvOn(player);
|
|
4762
|
+
}
|
|
4763
|
+
catch (e) {
|
|
4764
|
+
this.log.warn(`YouTube: cannot switch ${player.power} on: ${e.message}`);
|
|
4765
|
+
}
|
|
4766
|
+
void this.playWhenTvIsOn(player, hit);
|
|
4767
|
+
return {
|
|
4768
|
+
ok: true,
|
|
4769
|
+
hit,
|
|
4770
|
+
alternatives,
|
|
4771
|
+
player: player.name,
|
|
4772
|
+
poweringOn: true,
|
|
4773
|
+
text: (0, youtube_1.youtubeReply)(lang, 'poweringOn', reply),
|
|
4774
|
+
};
|
|
4775
|
+
}
|
|
4776
|
+
if (await this.sendToYouTubePlayer(player, hit.id)) {
|
|
4777
|
+
return { ok: true, hit, alternatives, player: player.name, text: (0, youtube_1.youtubeReply)(lang, 'playing', reply) };
|
|
4778
|
+
}
|
|
4779
|
+
const instance = player.base.split('.').slice(0, 2).join('.');
|
|
4780
|
+
const alive = (await this.getForeignStateAsync(`system.adapter.${instance}.alive`).catch(() => null))?.val;
|
|
4781
|
+
return {
|
|
4782
|
+
ok: false,
|
|
4783
|
+
hit,
|
|
4784
|
+
player: player.name,
|
|
4785
|
+
text: alive ? (0, youtube_1.youtubeReply)(lang, 'noConfirm', reply) : (0, youtube_1.youtubeReply)(lang, 'adapterDown', { instance }),
|
|
4786
|
+
};
|
|
4787
|
+
}
|
|
4788
|
+
/**
|
|
4789
|
+
* Whether the TV of a player is on. `unknown` when there is nothing to tell it by: no power state, or
|
|
4790
|
+
* one that is a button — pressing a toggle button on a TV that is already on would switch it off.
|
|
4791
|
+
*/
|
|
4792
|
+
async tvPower(player) {
|
|
4793
|
+
if (!player.power) {
|
|
4794
|
+
return 'unknown';
|
|
4795
|
+
}
|
|
4796
|
+
const obj = await this.getForeignObjectAsync(player.power).catch(() => null);
|
|
4797
|
+
if (!obj) {
|
|
4798
|
+
this.log.warn(`YouTube player ${player.name}: power state ${player.power} does not exist.`);
|
|
4799
|
+
return 'unknown';
|
|
4800
|
+
}
|
|
4801
|
+
const common = obj.common;
|
|
4802
|
+
if (common.read === false || String(common.role || '').startsWith('button')) {
|
|
4803
|
+
this.log.debug(`YouTube player ${player.name}: ${player.power} is a button, the TV state is unknown.`);
|
|
4804
|
+
return 'unknown';
|
|
4805
|
+
}
|
|
4806
|
+
const state = await this.getForeignStateAsync(player.power).catch(() => null);
|
|
4807
|
+
return (0, youtube_1.isPowerOn)(state?.val) ? 'on' : 'off';
|
|
4808
|
+
}
|
|
4809
|
+
/** Write "on" to the power state, in the type the state has. */
|
|
4810
|
+
async switchTvOn(player) {
|
|
4811
|
+
const obj = await this.getForeignObjectAsync(player.power);
|
|
4812
|
+
const type = obj?.common?.type;
|
|
4813
|
+
const val = type === 'number' ? 1 : type === 'string' ? 'on' : true;
|
|
4814
|
+
await this.setForeignStateAsync(player.power, { val, ack: false });
|
|
4815
|
+
this.journal?.stateAccess('set_state', { id: player.power, value: val });
|
|
4816
|
+
this.log.info(`YouTube: switching ${player.name} on (${player.power} = ${JSON.stringify(val)}).`);
|
|
4817
|
+
}
|
|
4818
|
+
/**
|
|
4819
|
+
* After the TV was switched on: wait until its power state confirms (an `ack` from the TV adapter — our
|
|
4820
|
+
* own write already says `true`), give the TV its boot time, then play. The YouTube app is often not
|
|
4821
|
+
* reachable over the lounge right after boot, so a command that is not acknowledged is sent once more.
|
|
4822
|
+
*/
|
|
4823
|
+
async playWhenTvIsOn(player, hit) {
|
|
4824
|
+
try {
|
|
4825
|
+
const until = Date.now() + TV_POWER_TIMEOUT_MS;
|
|
4826
|
+
let on = false;
|
|
4827
|
+
while (!on && Date.now() < until) {
|
|
4828
|
+
await this.delay(1000);
|
|
4829
|
+
const st = await this.getForeignStateAsync(player.power).catch(() => null);
|
|
4830
|
+
on = !!st?.ack && (0, youtube_1.isPowerOn)(st.val);
|
|
4831
|
+
}
|
|
4832
|
+
if (!on) {
|
|
4833
|
+
this.log.warn(`YouTube: ${player.power} did not report "on" within ${TV_POWER_TIMEOUT_MS / 1000} s — trying to play anyway.`);
|
|
4834
|
+
}
|
|
4835
|
+
await this.delay(player.delaySec * 1000);
|
|
4836
|
+
let ok = await this.sendToYouTubePlayer(player, hit.id);
|
|
4837
|
+
if (!ok) {
|
|
4838
|
+
await this.delay(YOUTUBE_RETRY_MS);
|
|
4839
|
+
ok = await this.sendToYouTubePlayer(player, hit.id);
|
|
4840
|
+
}
|
|
4841
|
+
if (ok) {
|
|
4842
|
+
this.log.info(`YouTube: playing ${hit.id} (${hit.title}) on ${player.name}.`);
|
|
4843
|
+
}
|
|
4844
|
+
else {
|
|
4845
|
+
this.log.warn(`YouTube: ${player.name} did not take ${hit.id} after power-on. If the TV needs longer to start, raise its start delay.`);
|
|
4846
|
+
}
|
|
4847
|
+
}
|
|
4848
|
+
catch (e) {
|
|
4849
|
+
this.log.warn(`YouTube: playing on ${player.name} failed: ${e.message}`);
|
|
4850
|
+
}
|
|
4851
|
+
}
|
|
4852
|
+
/**
|
|
4853
|
+
* Hand a video/playlist/channel id to the player and wait for its acknowledgement: youtube-remote acks
|
|
4854
|
+
* `youtube` (back to '') only once the TV took the command, and leaves it unacknowledged on failure.
|
|
4855
|
+
*/
|
|
4856
|
+
async sendToYouTubePlayer(player, id) {
|
|
4857
|
+
await this.setForeignStateAsync(player.target, { val: id, ack: false });
|
|
4858
|
+
this.journal?.stateAccess('set_state', { id: player.target, value: id });
|
|
4859
|
+
const until = Date.now() + YOUTUBE_ACK_TIMEOUT_MS;
|
|
4860
|
+
while (Date.now() < until) {
|
|
4861
|
+
await this.delay(250);
|
|
4862
|
+
if ((await this.getForeignStateAsync(player.target).catch(() => null))?.ack) {
|
|
4863
|
+
return true;
|
|
4864
|
+
}
|
|
4865
|
+
}
|
|
4866
|
+
return false;
|
|
4867
|
+
}
|
|
4868
|
+
/** A play result as tool / sendTo data: without the internal objects, with the alternatives as titles. */
|
|
4869
|
+
youtubeResultData(res) {
|
|
4870
|
+
return {
|
|
4871
|
+
ok: res.ok,
|
|
4872
|
+
text: res.text,
|
|
4873
|
+
...(res.player ? { player: res.player } : {}),
|
|
4874
|
+
...(res.hit ? { kind: res.hit.kind, id: res.hit.id, title: res.hit.title } : {}),
|
|
4875
|
+
...(res.hit?.channel ? { channel: res.hit.channel } : {}),
|
|
4876
|
+
...(res.poweringOn ? { poweringOn: true } : {}),
|
|
4877
|
+
...(res.alternatives?.length
|
|
4878
|
+
? { otherResults: res.alternatives.map(h => ({ id: h.id, title: h.title })) }
|
|
4879
|
+
: {}),
|
|
4880
|
+
};
|
|
4881
|
+
}
|
|
4882
|
+
/**
|
|
4883
|
+
* LLM tools for the YouTube players: `play_youtube` (search + power-on + play) and `youtube_player`
|
|
4884
|
+
* (pause, skip, volume, what is playing). The configured names go into the descriptions, the model
|
|
4885
|
+
* cannot discover them otherwise.
|
|
4886
|
+
*/
|
|
4887
|
+
buildYouTubeTools() {
|
|
4888
|
+
const names = this.youtubePlayers.map((p, i) => (i === 0 ? `${p.name} (default)` : p.name)).join(', ');
|
|
4889
|
+
const playerParam = {
|
|
4890
|
+
type: 'string',
|
|
4891
|
+
description: `Player name or room: ${names}. Leave it out for the player in the room the user is in, else the default.`,
|
|
4892
|
+
};
|
|
4893
|
+
const play = {
|
|
4894
|
+
name: 'play_youtube',
|
|
4895
|
+
description: 'Play something from YouTube on the TV: finds the channel, video or playlist by name, switches ' +
|
|
4896
|
+
'the TV on if it is off and starts playing. Use this for every "play … on YouTube", ' +
|
|
4897
|
+
'"spiele den Kanal …", "zeig das Video …" — never write the player states yourself. kind ' +
|
|
4898
|
+
'"channel" plays the newest uploads of a channel. The reply text of the result can be said as is; ' +
|
|
4899
|
+
'otherResults are the next search hits, offer them if the user says it was the wrong one ' +
|
|
4900
|
+
'(pass the id as query to play one).',
|
|
4901
|
+
parameters: {
|
|
4902
|
+
type: 'object',
|
|
4903
|
+
properties: {
|
|
4904
|
+
query: {
|
|
4905
|
+
type: 'string',
|
|
4906
|
+
description: 'Name of the channel, title of the video/playlist, a search phrase, or an id/URL',
|
|
4907
|
+
},
|
|
4908
|
+
kind: {
|
|
4909
|
+
type: 'string',
|
|
4910
|
+
enum: ['channel', 'video', 'playlist'],
|
|
4911
|
+
description: 'Leave it out if the user did not say: a channel of that name wins, else videos',
|
|
4912
|
+
},
|
|
4913
|
+
player: playerParam,
|
|
4914
|
+
},
|
|
4915
|
+
required: ['query'],
|
|
4916
|
+
additionalProperties: false,
|
|
4917
|
+
},
|
|
4918
|
+
run: async (args) => {
|
|
4919
|
+
const query = typeof args.query === 'string' ? args.query.trim() : '';
|
|
4920
|
+
if (!query) {
|
|
4921
|
+
return JSON.stringify({ ok: false, error: 'no query given' });
|
|
4922
|
+
}
|
|
4923
|
+
const kind = args.kind === 'channel' || args.kind === 'playlist' || args.kind === 'video' ? args.kind : 'auto';
|
|
4924
|
+
const player = typeof args.player === 'string' ? args.player : undefined;
|
|
4925
|
+
return JSON.stringify(this.youtubeResultData(await this.playYouTube(query, kind, player)));
|
|
4926
|
+
},
|
|
4927
|
+
};
|
|
4928
|
+
const control = {
|
|
4929
|
+
name: 'youtube_player',
|
|
4930
|
+
description: 'Control a YouTube player or ask what it plays: action play (resume), pause, next, previous, ' +
|
|
4931
|
+
'volume (with "volume" 0-100), mute, unmute, or status (title, channel, play state, TV power).',
|
|
4932
|
+
parameters: {
|
|
4933
|
+
type: 'object',
|
|
4934
|
+
properties: {
|
|
4935
|
+
action: {
|
|
4936
|
+
type: 'string',
|
|
4937
|
+
enum: ['play', 'pause', 'next', 'previous', 'volume', 'mute', 'unmute', 'status'],
|
|
4938
|
+
},
|
|
4939
|
+
volume: { type: 'number', description: 'Volume 0-100, only for action volume' },
|
|
4940
|
+
player: playerParam,
|
|
4941
|
+
},
|
|
4942
|
+
required: ['action'],
|
|
4943
|
+
additionalProperties: false,
|
|
4944
|
+
},
|
|
4945
|
+
run: async (args) => {
|
|
4946
|
+
const player = this.youtubePlayerFor(typeof args.player === 'string' ? args.player : undefined);
|
|
4947
|
+
if (!player) {
|
|
4948
|
+
return JSON.stringify({ ok: false, error: `unknown player, known are: ${names}` });
|
|
4949
|
+
}
|
|
4950
|
+
if (!(await this.resolveYouTubePlayer(player))) {
|
|
4951
|
+
return JSON.stringify({ ok: false, error: `no TV is paired with ${player.instance}` });
|
|
4952
|
+
}
|
|
4953
|
+
const action = typeof args.action === 'string' ? args.action : '';
|
|
4954
|
+
if (action === 'status') {
|
|
4955
|
+
const ids = ['title', 'artist', 'state', 'connected', 'volume'].map(s => `${player.base}.${s}`);
|
|
4956
|
+
const states = await this.getForeignStatesAsync(`${player.base}.*`).catch(() => ({}));
|
|
4957
|
+
const val = (i) => states[ids[i]]?.val ?? null;
|
|
4958
|
+
const power = await this.tvPower(player);
|
|
4959
|
+
return JSON.stringify({
|
|
4960
|
+
ok: true,
|
|
4961
|
+
data: {
|
|
4962
|
+
player: player.name,
|
|
4963
|
+
title: val(0),
|
|
4964
|
+
channel: val(1),
|
|
4965
|
+
state: { 0: 'paused', 1: 'playing', 2: 'stopped' }[String(val(2))],
|
|
4966
|
+
connected: val(3),
|
|
4967
|
+
volume: val(4),
|
|
4968
|
+
...(power !== 'unknown' ? { tv: power } : {}),
|
|
4969
|
+
},
|
|
4970
|
+
});
|
|
4971
|
+
}
|
|
4972
|
+
if (!this.config.allowWriteStates) {
|
|
4973
|
+
return JSON.stringify({ ok: false, error: 'device control is disabled in the settings' });
|
|
4974
|
+
}
|
|
4975
|
+
let id;
|
|
4976
|
+
let value;
|
|
4977
|
+
if (action === 'volume') {
|
|
4978
|
+
const v = Number(args.volume);
|
|
4979
|
+
if (!Number.isFinite(v)) {
|
|
4980
|
+
return JSON.stringify({ ok: false, error: 'volume 0-100 needed' });
|
|
4981
|
+
}
|
|
4982
|
+
id = `${player.base}.volume`;
|
|
4983
|
+
value = Math.min(100, Math.max(0, Math.round(v)));
|
|
4984
|
+
}
|
|
4985
|
+
else if (action === 'mute' || action === 'unmute') {
|
|
4986
|
+
id = `${player.base}.muted`;
|
|
4987
|
+
value = action === 'mute';
|
|
4988
|
+
}
|
|
4989
|
+
else if (['play', 'pause', 'next', 'previous'].includes(action)) {
|
|
4990
|
+
id = `${player.base}.${action === 'previous' ? 'prev' : action}`;
|
|
4991
|
+
value = true;
|
|
4992
|
+
}
|
|
4993
|
+
else {
|
|
4994
|
+
return JSON.stringify({ ok: false, error: `unknown action '${action}'` });
|
|
4995
|
+
}
|
|
4996
|
+
await this.setForeignStateAsync(id, { val: value, ack: false });
|
|
4997
|
+
this.journal?.stateAccess('set_state', { id, value });
|
|
4998
|
+
return JSON.stringify({ ok: true, data: { player: player.name, [action]: value } });
|
|
4999
|
+
},
|
|
5000
|
+
};
|
|
5001
|
+
return [play, control];
|
|
5002
|
+
}
|
|
3941
5003
|
// ── Presence: who is at home (roadmap B1) ───────────────────────────────
|
|
3942
5004
|
/**
|
|
3943
5005
|
* Build the presence tracker from the configured states, subscribe to them and mirror the result into
|
|
@@ -4295,12 +5357,26 @@ class Assistant extends adapter_core_1.Adapter {
|
|
|
4295
5357
|
*/
|
|
4296
5358
|
async runRoutine(routine, source) {
|
|
4297
5359
|
const origin = this.announceTargetForSource(source);
|
|
4298
|
-
for (const action of routine.actions) {
|
|
5360
|
+
for (const [i, action] of routine.actions.entries()) {
|
|
5361
|
+
const start = Date.now();
|
|
4299
5362
|
try {
|
|
4300
5363
|
await this.runTriggerAction({ id: `routine:${routine.name}`, when: [] }, action, origin);
|
|
5364
|
+
this.journal?.step({
|
|
5365
|
+
stage: 'routine',
|
|
5366
|
+
title: `Routine action ${i + 1}/${routine.actions.length}`,
|
|
5367
|
+
ms: Date.now() - start,
|
|
5368
|
+
data: action,
|
|
5369
|
+
});
|
|
4301
5370
|
}
|
|
4302
5371
|
catch (e) {
|
|
4303
5372
|
this.log.warn(`Routine '${routine.name}': an action failed — ${e.message}`);
|
|
5373
|
+
this.journal?.step({
|
|
5374
|
+
stage: 'routine',
|
|
5375
|
+
level: 'error',
|
|
5376
|
+
title: `Routine action ${i + 1}/${routine.actions.length} failed: ${e.message}`,
|
|
5377
|
+
ms: Date.now() - start,
|
|
5378
|
+
data: { action, error: (0, journal_1.errorInfo)(e) },
|
|
5379
|
+
});
|
|
4304
5380
|
}
|
|
4305
5381
|
}
|
|
4306
5382
|
}
|