iobroker.assistant 1.0.3 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/README.md +65 -0
  2. package/admin/custom/.vite/manifest.json +62 -62
  3. package/admin/custom/assets/Components-BSewCBfS.js +2 -0
  4. package/admin/custom/assets/{_virtual_mf-localSharedImportMap___mfe_internal__ConfigCustomAssistant__mf_owner__1-A4gakomR.js → _virtual_mf-localSharedImportMap___mfe_internal__ConfigCustomAssistant__mf_owner__1-C5NPM0Of.js} +1 -1
  5. package/admin/custom/assets/{_virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__201375093093590__loadShare___mf_0_emotion_mf_1_styled__loadShare__.js-CwqmfMvn.js → _virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__163539842885036__loadShare___mf_0_emotion_mf_1_styled__loadShare__.js-IaosaymG.js} +1 -1
  6. package/admin/custom/assets/{_virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__201375093093590__loadShare___mf_0_iobroker_mf_1_gui_mf_2_components__loadShare__.js-BMuLYe6f.js → _virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__163539842885036__loadShare___mf_0_iobroker_mf_1_gui_mf_2_components__loadShare__.js-Cdyjt5KT.js} +39 -39
  7. package/admin/custom/assets/{_virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__201375093093590__loadShare__shared-BxjZmnpc.js → _virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__163539842885036__loadShare__shared-Dsw5zWS8.js} +1 -1
  8. package/admin/custom/assets/_virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__163539842885036__prebuild___mf_0_iobroker_mf_1_json_mf_2_config__prebuild__-SgSagTOv.js +122 -0
  9. package/admin/custom/assets/{hostInit-4roC4hEj.js → hostInit-jJe8bIfV.js} +1 -1
  10. package/admin/custom/assets/{lib-CchzgTUs.js → lib-CL8mXvI6.js} +1 -1
  11. package/admin/custom/assets/{mf-entry-bootstrap-0-4806bbe4.js → mf-entry-bootstrap-0-ab227aa6.js} +2 -2
  12. package/admin/custom/assets/{pendingShares-DLlv8yFY.js → pendingShares-wrGtfliQ.js} +1 -1
  13. package/admin/custom/customComponents.js +2 -2
  14. package/admin/custom/i18n/de.json +40 -0
  15. package/admin/custom/i18n/en.json +40 -0
  16. package/admin/custom/i18n/es.json +40 -0
  17. package/admin/custom/i18n/fr.json +40 -0
  18. package/admin/custom/i18n/it.json +40 -0
  19. package/admin/custom/i18n/nl.json +40 -0
  20. package/admin/custom/i18n/pl.json +40 -0
  21. package/admin/custom/i18n/pt.json +40 -0
  22. package/admin/custom/i18n/ru.json +40 -0
  23. package/admin/custom/i18n/uk.json +40 -0
  24. package/admin/custom/i18n/zh-cn.json +40 -0
  25. package/admin/custom/index.html +2 -2
  26. package/admin/custom/mf-manifest.json +1 -1
  27. package/admin/custom/mf-stats.json +1 -1
  28. package/admin/i18n/de.json +13 -3
  29. package/admin/i18n/en.json +13 -3
  30. package/admin/i18n/es.json +13 -3
  31. package/admin/i18n/fr.json +13 -3
  32. package/admin/i18n/it.json +13 -3
  33. package/admin/i18n/nl.json +13 -3
  34. package/admin/i18n/pl.json +13 -3
  35. package/admin/i18n/pt.json +13 -3
  36. package/admin/i18n/ru.json +13 -3
  37. package/admin/i18n/uk.json +13 -3
  38. package/admin/i18n/zh-cn.json +13 -3
  39. package/admin/jsonConfig.json +61 -2
  40. package/build/lib/journal.js +298 -2
  41. package/build/lib/journal.js.map +1 -1
  42. package/build/lib/llm.js +105 -4
  43. package/build/lib/llm.js.map +1 -1
  44. package/build/lib/localLlm.js +47 -10
  45. package/build/lib/localLlm.js.map +1 -1
  46. package/build/lib/nlu.js +124 -5
  47. package/build/lib/nlu.js.map +1 -1
  48. package/build/lib/tools.js +3 -0
  49. package/build/lib/tools.js.map +1 -1
  50. package/build/lib/voice/esphome.js +81 -7
  51. package/build/lib/voice/esphome.js.map +1 -1
  52. package/build/lib/voice/esphomeProto.js.map +1 -1
  53. package/build/lib/voice/mediaServer.js +29 -1
  54. package/build/lib/voice/mediaServer.js.map +1 -1
  55. package/build/lib/voice/protocol.js.map +1 -1
  56. package/build/lib/voice/voiceServer.js +81 -7
  57. package/build/lib/voice/voiceServer.js.map +1 -1
  58. package/build/lib/voice/wakeWords.js +284 -0
  59. package/build/lib/voice/wakeWords.js.map +1 -0
  60. package/build/lib/voice/wyoming.js +12 -4
  61. package/build/lib/voice/wyoming.js.map +1 -1
  62. package/build/lib/weather.js +160 -0
  63. package/build/lib/weather.js.map +1 -1
  64. package/build/lib/youtube.js +625 -0
  65. package/build/lib/youtube.js.map +1 -0
  66. package/build/main.js +1152 -76
  67. package/build/main.js.map +1 -1
  68. package/io-package.json +30 -14
  69. package/package.json +6 -6
  70. package/admin/custom/assets/Components-C6qcpaRC.js +0 -1
  71. package/admin/custom/assets/_virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__201375093093590__prebuild___mf_0_iobroker_mf_1_json_mf_2_config__prebuild__-TM4l0OYm.js +0 -122
  72. /package/admin/custom/assets/{_virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__201375093093590__loadShare___mf_0_emotion_mf_1_react__loadShare__.js-BkH7fzDw.js → _virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__163539842885036__loadShare___mf_0_emotion_mf_1_react__loadShare__.js-BkH7fzDw.js} +0 -0
  73. /package/admin/custom/assets/{_virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__201375093093590__prebuild__react_mf_1_jsx_mf_2_runtime__prebuild__-M61rjtmp.js → _virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__163539842885036__prebuild__react_mf_1_jsx_mf_2_runtime__prebuild__-M61rjtmp.js} +0 -0
package/build/main.js CHANGED
@@ -36,6 +36,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
36
36
  const os = __importStar(require("node:os"));
37
37
  const fs = __importStar(require("node:fs"));
38
38
  const node_child_process_1 = require("node:child_process");
39
+ const node_async_hooks_1 = require("node:async_hooks");
39
40
  const adapter_core_1 = require("@iobroker/adapter-core");
40
41
  const path = __importStar(require("node:path"));
41
42
  const mcp_server_1 = require("@iobroker/mcp-server");
@@ -54,6 +55,7 @@ const ask_1 = require("./lib/ask");
54
55
  const presence_1 = require("./lib/presence");
55
56
  const targets_1 = require("./lib/targets");
56
57
  const routines_1 = require("./lib/routines");
58
+ const youtube_1 = require("./lib/youtube");
57
59
  const notifications_1 = require("./lib/notifications");
58
60
  const triggers_1 = require("./lib/triggers");
59
61
  const voiceServer_1 = require("./lib/voice/voiceServer");
@@ -61,6 +63,7 @@ const wyoming_1 = require("./lib/voice/wyoming");
61
63
  const esphome_1 = require("./lib/voice/esphome");
62
64
  const esphomeEntities_1 = require("./lib/voice/esphomeEntities");
63
65
  const mediaServer_1 = require("./lib/voice/mediaServer");
66
+ const wakeWords_1 = require("./lib/voice/wakeWords");
64
67
  const tone_1 = require("./lib/voice/tone");
65
68
  const engines_1 = require("./lib/voice/engines");
66
69
  /**
@@ -68,6 +71,20 @@ const engines_1 = require("./lib/voice/engines");
68
71
  * so re-reading their whole state tree on every single request would be pure overhead.
69
72
  */
70
73
  const WEATHER_CTX_TTL = 5 * 60 * 1000;
74
+ /**
75
+ * How long the NLU device model stays valid. Building it costs a `list_devices` plus a name lookup per
76
+ * device, and one spoken command needs it up to three times (STT hints, NLU, the cloud device list).
77
+ * Enum and name changes drop it at once (see {@link ListCache.onClear}); the TTL only catches devices
78
+ * that were added or renamed elsewhere.
79
+ */
80
+ const NLU_DEVICES_TTL_MS = 60_000;
81
+ /** Default for `localLlmTimeoutSec`: after this the local model is stopped and the next tier answers. */
82
+ const LOCAL_LLM_TIMEOUT_SEC = 15;
83
+ /**
84
+ * Answer length cap of the local model. It answers small talk and general questions, mostly by voice, and
85
+ * on a Raspberry Pi every token costs about a fifth of a second — 512 tokens would be a minute and a half.
86
+ */
87
+ const LOCAL_LLM_MAX_TOKENS = 256;
71
88
  /**
72
89
  * The replies the assistant speaks verbatim, whatever the question was. They are pre-synthesised into
73
90
  * the TTS cache at startup; everything else is cached on first use.
@@ -75,6 +92,12 @@ const WEATHER_CTX_TTL = 5 * 60 * 1000;
75
92
  /** How long to wait for a device to confirm a write, and how often to look. */
76
93
  const ACK_TIMEOUT_MS = 2000;
77
94
  const ACK_POLL_MS = 150;
95
+ /** How long a TV may take to report "on" after it was switched on for YouTube. */
96
+ const TV_POWER_TIMEOUT_MS = 30_000;
97
+ /** How long youtube-remote may take to acknowledge a play command (it goes through Google's lounge). */
98
+ const YOUTUBE_ACK_TIMEOUT_MS = 8000;
99
+ /** Pause before the play command is repeated once on a TV that just booted. */
100
+ const YOUTUBE_RETRY_MS = 5000;
78
101
  /**
79
102
  * Did the device end up with the value we asked for? Compared loosely: a dimmer told `30` may report
80
103
  * `30.0`, a switch told `true` may report `1`, and a lamp told 100 % may settle at 99 % — insisting on
@@ -155,7 +178,27 @@ function iaqLabel(value, unit, lang) {
155
178
  return lang === 'ru' ? row[3] : lang === 'de' ? row[1] : row[2];
156
179
  }
157
180
  /** NLU actions that only switch something; their success is what the confirmation tone replaces. */
158
- const CONTROL_ACTIONS = new Set(['on', 'off', 'level', 'color']);
181
+ const CONTROL_ACTIONS = new Set(['on', 'off', 'level', 'color', 'volumeSet']);
182
+ /** An intent in one line for the request trace: "level Rollo (Büro) = 30", "timerSet 300 s", "alarmSet 7:05". */
183
+ function describeIntent(intent) {
184
+ let text = intent.action;
185
+ if (intent.device) {
186
+ text += ` ${intent.device.name}${intent.device.room ? ` (${intent.device.room})` : ''}`;
187
+ }
188
+ else if (intent.room) {
189
+ text += ` (${intent.room})`;
190
+ }
191
+ if (intent.value !== undefined) {
192
+ text += ` = ${JSON.stringify(intent.value)}`;
193
+ }
194
+ else if (intent.durationSec !== undefined) {
195
+ text += ` ${intent.durationSec} s`;
196
+ }
197
+ else if (intent.hour !== undefined) {
198
+ text += ` ${intent.hour}:${String(intent.minute ?? 0).padStart(2, '0')}`;
199
+ }
200
+ return text;
201
+ }
159
202
  const FIXED_REPLIES = {
160
203
  de: ['Ok.', 'Erledigt.'],
161
204
  en: ['Okay.', 'Done.'],
@@ -192,6 +235,8 @@ class Assistant extends adapter_core_1.Adapter {
192
235
  journal = null;
193
236
  /** Short-TTL cache for the expensive device/room/function listings; invalidated on enum changes. */
194
237
  listCache = null;
238
+ /** The NLU device model of {@link getNluDevices}, shared by all callers (and concurrent requests). */
239
+ nluDevices = null;
195
240
  /** Cache of a `<base>.SET` level state id → its writable boolean sibling switch (`<base>.ON_SET`/`.ON`), or ''. */
196
241
  siblingSwitch = new Map();
197
242
  /** Tier-1a local LLM (node-llama-cpp), lazily loaded when enabled + installed; null otherwise. */
@@ -208,6 +253,10 @@ class Assistant extends adapter_core_1.Adapter {
208
253
  esphome = null;
209
254
  /** Serves spoken replies to ESPHome satellites over HTTP; null unless those are enabled. */
210
255
  media = null;
256
+ /** Uploaded wake words (`assistant.0/wakewords/`), offered to every ESPHome satellite. */
257
+ customWakeWords = new Map();
258
+ /** Loads {@link customWakeWords} once from the file storage; later calls wait for the same load. */
259
+ customWakeWordsLoaded = null;
211
260
  /** Short-term per-source conversation memory (in-memory, TTL) for follow-up questions. */
212
261
  context = new context_1.ConversationStore();
213
262
  /** Questions the assistant asked and is waiting for an answer to (`askUser`), keyed by source. */
@@ -222,6 +271,8 @@ class Assistant extends adapter_core_1.Adapter {
222
271
  mirroredTimers = new Map();
223
272
  /** Per satellite: resolves once its control objects exist, so state writes can wait for them. */
224
273
  satControlsReady = new Map();
274
+ /** Satellites whose `volume` state exists and is subscribed (see updateSatelliteVolume). */
275
+ satVolumeEnsured = new Set();
225
276
  /** Native (ioBroker) satellite state-id → sender instance id, so we can push announcements back to it. */
226
277
  nativeSatFrom = new Map();
227
278
  /** Countdown timers / reminders (roadmap #2); mirrored into `timers.*` states. Null until onReady. */
@@ -248,6 +299,13 @@ class Assistant extends adapter_core_1.Adapter {
248
299
  routines = [];
249
300
  /** Named announcement targets (roadmap B2): groups of rooms and people, from the configuration. */
250
301
  announceTargets = [];
302
+ /** YouTube players from the configuration; the first one is the default. */
303
+ youtubePlayers = [];
304
+ /**
305
+ * Where the request being answered came from, for the LLM tools: a tool only gets the model's
306
+ * arguments, but "play X" from the bedroom satellite should go to the bedroom TV.
307
+ */
308
+ requestOrigin = new node_async_hooks_1.AsyncLocalStorage();
251
309
  /** Who is at home (roadmap B1); mirrored into `presence.*`. Null until onReady. */
252
310
  presence = null;
253
311
  /** Satellites currently set to Do-Not-Disturb, by state id (mirrors `satellites.<id>.dnd`). */
@@ -289,6 +347,9 @@ class Assistant extends adapter_core_1.Adapter {
289
347
  allowObjectChange: cfg.allowObjectChange,
290
348
  }));
291
349
  this.listCache = new tools_1.ListCache();
350
+ this.listCache.onClear = () => {
351
+ this.nluDevices = null;
352
+ };
292
353
  const { tools, denied } = await (0, tools_1.buildMcpTools)(this.mcp, cfg, this.listCache, (id, fb) => this.resolveDeviceName(id, fb));
293
354
  // Let the cloud LLM handle timers/alarms too (for phrasings the rule-based NLU misses).
294
355
  tools.push(...this.buildTimerTools(), ...this.buildAlarmTools());
@@ -302,6 +363,18 @@ class Assistant extends adapter_core_1.Adapter {
302
363
  // Only useful with satellites: without voice there is nothing to speak on.
303
364
  tools.push(this.buildAnnounceTool());
304
365
  }
366
+ this.youtubePlayers = (0, youtube_1.parsePlayerRows)(cfg.youtubePlayers);
367
+ if (this.youtubePlayers.length) {
368
+ // Before the tools are built: their descriptions carry the player names, which an instance
369
+ // row only gets from its device.
370
+ for (const player of this.youtubePlayers) {
371
+ if (!(await this.resolveYouTubePlayer(player))) {
372
+ this.log.warn(`YouTube: ${player.instance} has no paired TV yet — pair one in its settings.`);
373
+ }
374
+ }
375
+ tools.push(...this.buildYouTubeTools());
376
+ this.log.info(`YouTube players: ${this.youtubePlayers.map(p => p.name).join(', ')}.`);
377
+ }
305
378
  this.log.info(`ioBroker tools enabled (${tools.length}): ${tools.map(t => t.name).join(', ')}`);
306
379
  if (denied.length) {
307
380
  this.log.debug(`Tools denied by access settings: ${denied.join(', ')}`);
@@ -320,6 +393,7 @@ class Assistant extends adapter_core_1.Adapter {
320
393
  maxTokens: cfg.maxTokens || 1024,
321
394
  tools: tools.map(t => this.journalTool(t)),
322
395
  log: this.log,
396
+ trace: step => this.journal?.step(step),
323
397
  });
324
398
  }
325
399
  }
@@ -407,7 +481,8 @@ class Assistant extends adapter_core_1.Adapter {
407
481
  language,
408
482
  stt,
409
483
  tts,
410
- answer: (question, sttMs) => this.answer(question, 'wyoming', { transport: 'wyoming', sttMs }),
484
+ answer: (question, sttMs, audioMs) => this.answer(question, 'wyoming', { transport: 'wyoming', sttMs, audioMs }),
485
+ onSpoken: info => this.reportSpoken('wyoming', info),
411
486
  getHints: () => this.buildSttHints(),
412
487
  log: this.log,
413
488
  });
@@ -462,6 +537,10 @@ class Assistant extends adapter_core_1.Adapter {
462
537
  this.media = null;
463
538
  return;
464
539
  }
540
+ await this.loadCustomWakeWords();
541
+ for (const word of this.customWakeWords.values()) {
542
+ this.serveWakeWord(word);
543
+ }
465
544
  this.esphome = new esphome_1.EsphomeSatellites({
466
545
  devices,
467
546
  language,
@@ -472,7 +551,10 @@ class Assistant extends adapter_core_1.Adapter {
472
551
  room: origin.room,
473
552
  wakeWord: origin.wakeWord,
474
553
  sttMs: origin.sttMs,
554
+ audioMs: origin.audioMs,
555
+ vad: origin.vad,
475
556
  }),
557
+ onSpoken: (device, info) => this.reportSpoken(device, info),
476
558
  getHints: () => this.buildSttHints(),
477
559
  media: this.media,
478
560
  mediaHost: (cfg.esphomeMediaHost || '').trim(),
@@ -490,7 +572,15 @@ class Assistant extends adapter_core_1.Adapter {
490
572
  onEntityState: (device, room, entity) => {
491
573
  this.updateSatelliteControl(device, room, entity).catch(e => this.log.debug(`satellite control update failed: ${e.message}`));
492
574
  },
493
- expectsFollowUp: answer => this.expectsFollowUp(answer),
575
+ expectsFollowUp: (answer, device) => this.followUpFor(device, answer),
576
+ externalWakeWords: () => [...this.customWakeWords.values()].map(w => ({
577
+ id: w.id,
578
+ phrase: w.phrase,
579
+ languages: w.languages,
580
+ size: w.size,
581
+ sha256: w.sha256,
582
+ path: `${wakeWords_1.WAKE_WORD_URL_PREFIX}${w.id}.json`,
583
+ })),
494
584
  });
495
585
  await this.esphome.start();
496
586
  this.log.info(`ESPHome satellites: ${devices.length} device(s), STT=${cfg.sttProvider || 'openai'}, TTS=${cfg.ttsProvider || 'openai'}.`);
@@ -545,7 +635,17 @@ class Assistant extends adapter_core_1.Adapter {
545
635
  language,
546
636
  stt,
547
637
  tts,
548
- answer: (question, ctx) => this.answerVoice(question, ctx.device, { transport: 'udp', room: ctx.room, sttMs: ctx.sttMs }),
638
+ answer: (question, ctx) => this.answerVoice(question, ctx.device, {
639
+ transport: 'udp',
640
+ room: ctx.room,
641
+ sttMs: ctx.sttMs,
642
+ audioMs: ctx.audioMs,
643
+ }),
644
+ onSpoken: (device, info) => this.reportSpoken(device, info),
645
+ expectsFollowUp: (answer, device) => this.followUpFor(device, answer),
646
+ onVolume: (device, room, level) => {
647
+ this.updateSatelliteVolume(device, room, level).catch(e => this.log.debug(`satellite volume update failed: ${e.message}`));
648
+ },
549
649
  getHints: () => this.buildSttHints(),
550
650
  log: this.log,
551
651
  onStatus: (device, room, state) => {
@@ -612,6 +712,23 @@ class Assistant extends adapter_core_1.Adapter {
612
712
  expectsFollowUp(answer) {
613
713
  return /[??]/.test(answer || '');
614
714
  }
715
+ /** A transport synthesised the reply to `source`'s latest request — the journal shows it with that request. */
716
+ reportSpoken(source, info) {
717
+ const cfg = this.config;
718
+ const provider = cfg.ttsProvider || 'openai';
719
+ const voices = {
720
+ openai: cfg.ttsVoice || 'alloy',
721
+ azure: cfg.azureVoice,
722
+ aws: cfg.awsVoice,
723
+ piper: cfg.piperVoice,
724
+ };
725
+ this.journal?.spoken(source, {
726
+ ...info,
727
+ provider,
728
+ fallback: cfg.ttsFallback || undefined,
729
+ voice: (voices[provider] || '').trim() || undefined,
730
+ });
731
+ }
615
732
  /** Record the origin of the current text.request/response ('' = state write, 'chat' = message, else satellite). */
616
733
  setQuerySource(source) {
617
734
  this.setStateAsync('text.querySource', { val: source, ack: true }).catch(() => { });
@@ -678,17 +795,18 @@ class Assistant extends adapter_core_1.Adapter {
678
795
  room,
679
796
  wakeWord: typeof msg.wakeWord === 'string' ? msg.wakeWord : undefined,
680
797
  sttMs: Date.now() - sttStart,
798
+ audioMs: Math.round((pcm.length / 2 / rate) * 1000),
681
799
  });
682
800
  if (!answer) {
683
801
  return { text, answer: '' };
684
802
  }
803
+ const ttsStart = Date.now();
685
804
  const reply = await engines.tts.synthesize(answer, language);
686
- // Follow-up: primarily the LLM's own signal ([[LISTEN]] → lastFollowUp); "?" heuristic as fallback.
687
- const llmSignalled = this.lastFollowUp;
688
- const listen = llmSignalled || this.expectsFollowUp(answer);
689
- if (listen) {
690
- this.log.info(`→ mic-on to satellite '${source}' (${llmSignalled ? 'LLM signalled a follow-up' : 'reply is a question (fallback)'}).`);
691
- }
805
+ this.reportSpoken(source, {
806
+ ms: Date.now() - ttsStart,
807
+ audioMs: (reply.pcm.length / 2 / reply.sampleRate) * 1000,
808
+ });
809
+ const listen = this.followUpFor(source, answer);
692
810
  return { text, answer, audio: reply.pcm.toString('base64'), sampleRate: reply.sampleRate, listen };
693
811
  }
694
812
  catch (e) {
@@ -751,6 +869,33 @@ class Assistant extends adapter_core_1.Adapter {
751
869
  await this.setStateAsync(`${base}.room`, { val: room, ack: true });
752
870
  }
753
871
  }
872
+ /**
873
+ * Mirror the playback volume a UDP satellite reported into `satellites.<id>.volume`. The state only
874
+ * appears for satellites that speak the `volume` message — an older one would never answer a write.
875
+ */
876
+ async updateSatelliteVolume(device, room, level) {
877
+ const id = await this.ensureSatelliteObjects(device, room);
878
+ const stateId = `satellites.${id}.volume`;
879
+ if (!this.satVolumeEnsured.has(id)) {
880
+ await this.setObjectNotExistsAsync(stateId, {
881
+ type: 'state',
882
+ common: {
883
+ name: 'Playback volume',
884
+ type: 'number',
885
+ role: 'level.volume',
886
+ unit: '%',
887
+ min: 0,
888
+ max: 100,
889
+ read: true,
890
+ write: true,
891
+ },
892
+ native: {},
893
+ });
894
+ this.subscribeStates(stateId);
895
+ this.satVolumeEnsured.add(id);
896
+ }
897
+ await this.setStateAsync(stateId, { val: level, ack: true });
898
+ }
754
899
  /**
755
900
  * Create the object tree for one satellite and return its (sanitised) state id, without touching
756
901
  * any value. Callers that only need somewhere to hang a child object use this rather than
@@ -886,11 +1031,111 @@ class Assistant extends adapter_core_1.Adapter {
886
1031
  this.satWakeWordStatesEnsured.add(id);
887
1032
  }
888
1033
  await this.setStateAsync(`${base}.wakeWords`, { val: config.active.join(','), ack: true });
1034
+ // `custom` marks the uploaded ones, so the settings dialog can offer to delete them.
1035
+ const words = config.available.map(w => (this.customWakeWords.has(w.id) ? { ...w, custom: true } : w));
889
1036
  await this.setStateAsync(`${base}.availableWakeWords`, {
890
- val: JSON.stringify({ max: config.max, words: config.available }),
1037
+ val: JSON.stringify({ max: config.max, words }),
891
1038
  ack: true,
892
1039
  });
893
1040
  }
1041
+ /** Read the uploaded wake words (`<instance>/wakewords/<id>.json` + `.tflite`) once. */
1042
+ loadCustomWakeWords() {
1043
+ this.customWakeWordsLoaded ||= (async () => {
1044
+ let entries = [];
1045
+ try {
1046
+ entries = await this.readDirAsync(this.namespace, wakeWords_1.WAKE_WORD_DIR);
1047
+ }
1048
+ catch {
1049
+ return; // nothing uploaded yet
1050
+ }
1051
+ for (const entry of entries) {
1052
+ const id = entry.file.replace(/\.json$/, '');
1053
+ if (entry.isDir || !entry.file.endsWith('.json') || !wakeWords_1.WAKE_WORD_ID_RE.test(id)) {
1054
+ continue;
1055
+ }
1056
+ try {
1057
+ const config = await this.readStoredFile(`${wakeWords_1.WAKE_WORD_DIR}/${id}.json`, 'utf8');
1058
+ const model = await this.readStoredFile(`${wakeWords_1.WAKE_WORD_DIR}/${id}.tflite`, 'binary');
1059
+ const word = (0, wakeWords_1.loadWakeWord)(id, config, model);
1060
+ this.customWakeWords.set(id, word);
1061
+ // Uploaded before the upload checked the model: keep it, so it can be deleted in the dialog.
1062
+ const problem = (0, wakeWords_1.microModelProblem)(model);
1063
+ if (problem) {
1064
+ this.log.warn(`Custom wake word ${word.phrase} (${id}) cannot run on the device: ${problem}. Delete it in the satellite settings.`);
1065
+ }
1066
+ }
1067
+ catch (e) {
1068
+ this.log.warn(`Custom wake word ${id} skipped: ${e.message}`);
1069
+ }
1070
+ }
1071
+ if (this.customWakeWords.size) {
1072
+ this.log.info(`Custom wake words: ${[...this.customWakeWords.values()].map(w => `${w.phrase} (${w.id})`).join(', ')}`);
1073
+ }
1074
+ })();
1075
+ return this.customWakeWordsLoaded;
1076
+ }
1077
+ /**
1078
+ * Read a file from this instance's file storage as a Buffer. Depending on the backend and the file's
1079
+ * type it comes back as a Buffer or a string — `encoding` says how to read the latter.
1080
+ */
1081
+ async readStoredFile(name, encoding) {
1082
+ const res = (await this.readFileAsync(this.namespace, name));
1083
+ const f = Buffer.isBuffer(res) || typeof res === 'string' ? res : res.file;
1084
+ if (f == null) {
1085
+ throw new Error(`${name} not found`);
1086
+ }
1087
+ return Buffer.isBuffer(f) ? f : Buffer.from(f, encoding);
1088
+ }
1089
+ serveWakeWord(word) {
1090
+ this.media?.serve(`${wakeWords_1.WAKE_WORD_URL_PREFIX}${word.id}.json`, word.config, 'application/json');
1091
+ this.media?.serve(`${wakeWords_1.WAKE_WORD_URL_PREFIX}${word.id}.tflite`, word.model, 'application/octet-stream');
1092
+ }
1093
+ /**
1094
+ * Store an uploaded wake word and offer it to the connected satellites. It shows up in their wake
1095
+ * word list right away; a device downloads the model only when it is selected there.
1096
+ */
1097
+ async addCustomWakeWord(message) {
1098
+ await this.loadCustomWakeWords();
1099
+ const text = (value) => (typeof value === 'string' ? value : undefined);
1100
+ const languages = (Array.isArray(message.languages)
1101
+ ? message.languages.map(String)
1102
+ : (text(message.languages) ?? '').split(/[,;\s]+/)).filter(Boolean);
1103
+ const word = (0, wakeWords_1.prepareWakeWord)({
1104
+ model: Buffer.from(text(message.model) ?? '', 'base64'),
1105
+ fileName: text(message.fileName),
1106
+ config: message.config && typeof message.config === 'object'
1107
+ ? message.config
1108
+ : text(message.config),
1109
+ phrase: text(message.phrase),
1110
+ languages,
1111
+ }, this.config.voiceLanguage || this.language || '');
1112
+ if (!this.customWakeWords.has(word.id)) {
1113
+ await this.writeFileAsync(this.namespace, `${wakeWords_1.WAKE_WORD_DIR}/${word.id}.tflite`, word.model);
1114
+ await this.writeFileAsync(this.namespace, `${wakeWords_1.WAKE_WORD_DIR}/${word.id}.json`, word.config);
1115
+ this.customWakeWords.set(word.id, word);
1116
+ this.serveWakeWord(word);
1117
+ this.log.info(`Custom wake word uploaded: ${word.phrase} (${word.id}, ${word.size} bytes)`);
1118
+ }
1119
+ const devices = this.esphome?.refreshWakeWords() ?? 0;
1120
+ return { id: word.id, phrase: word.phrase, languages: word.languages, size: word.size, devices };
1121
+ }
1122
+ /**
1123
+ * Remove an uploaded wake word: it is no longer offered or served. A device that already downloaded
1124
+ * it keeps its copy — the ESPHome API cannot delete a model from a device.
1125
+ */
1126
+ async deleteCustomWakeWord(id) {
1127
+ await this.loadCustomWakeWords();
1128
+ if (!wakeWords_1.WAKE_WORD_ID_RE.test(id) || !this.customWakeWords.has(id)) {
1129
+ throw new Error(`unknown wake word ${id}`);
1130
+ }
1131
+ for (const ext of ['json', 'tflite']) {
1132
+ await this.delFileAsync(this.namespace, `${wakeWords_1.WAKE_WORD_DIR}/${id}.${ext}`).catch(() => undefined);
1133
+ this.media?.unserve(`${wakeWords_1.WAKE_WORD_URL_PREFIX}${id}.${ext}`);
1134
+ }
1135
+ this.customWakeWords.delete(id);
1136
+ this.esphome?.refreshWakeWords();
1137
+ this.log.info(`Custom wake word deleted: ${id}`);
1138
+ }
894
1139
  /**
895
1140
  * Mirror everything an ESPHome satellite exposes besides the voice pipeline into
896
1141
  * `satellites.<id>.controls.*` — microphone gain and volume, noise suppression, wake-word and
@@ -1254,7 +1499,8 @@ class Assistant extends adapter_core_1.Adapter {
1254
1499
  await Promise.all(allowed.map(d => this.voice.announce(d, pcm, sampleRate)));
1255
1500
  delivered += allowed.length;
1256
1501
  if (listen) {
1257
- allowed.forEach(d => this.voice.listen(d));
1502
+ const playMs = (pcm.length / 2 / sampleRate) * 1000;
1503
+ allowed.forEach(d => this.voice.listen(d, playMs));
1258
1504
  }
1259
1505
  }
1260
1506
  catch (e) {
@@ -1603,6 +1849,15 @@ class Assistant extends adapter_core_1.Adapter {
1603
1849
  await this.setStateAsync('dnd', { val: this.globalDnd, ack: true });
1604
1850
  return;
1605
1851
  }
1852
+ // Playback volume of a UDP satellite — not acked here: the satellite reports the level it took.
1853
+ const satVolume = id.match(/\.satellites\.([^.]+)\.volume$/);
1854
+ if (satVolume) {
1855
+ const device = this.satDeviceById.get(satVolume[1]) || satVolume[1];
1856
+ if (!this.voice?.volume(device, { level: Number(state.val) })) {
1857
+ this.log.warn(`Cannot set the volume of '${device}': it is not a registered UDP satellite.`);
1858
+ }
1859
+ return;
1860
+ }
1606
1861
  const satDnd = id.match(/\.satellites\.([^.]+)\.dnd$/);
1607
1862
  if (satDnd) {
1608
1863
  this.dndById.set(satDnd[1], !!state.val);
@@ -1929,6 +2184,27 @@ class Assistant extends adapter_core_1.Adapter {
1929
2184
  }
1930
2185
  return;
1931
2186
  }
2187
+ // Scripts: find and play something on a configured YouTube player, the TV switched on first.
2188
+ // message = { query, kind?: 'channel'|'video'|'playlist', player? } → { ok, text, title?, id?, player? }
2189
+ // Settings: the "Player" column of the YouTube table (selectSendTo).
2190
+ if (obj.command === 'getYouTubeInstances') {
2191
+ const list = await this.getYouTubeInstances();
2192
+ if (obj.callback) {
2193
+ this.sendTo(obj.from, obj.command, list, obj.callback);
2194
+ }
2195
+ return;
2196
+ }
2197
+ if (obj.command === 'playYouTube') {
2198
+ const m = (obj.message || {});
2199
+ const kind = m.kind === 'channel' || m.kind === 'playlist' || m.kind === 'video' ? m.kind : 'auto';
2200
+ const res = String(m.query || '').trim()
2201
+ ? await this.playYouTube(String(m.query).trim(), kind, m.player)
2202
+ : { ok: false, text: 'no query given' };
2203
+ if (obj.callback) {
2204
+ this.sendTo(obj.from, obj.command, this.youtubeResultData(res), obj.callback);
2205
+ }
2206
+ return;
2207
+ }
1932
2208
  // Scripts / settings: which wake words an ESPHome satellite offers and which are listening.
1933
2209
  // message = { device? } — omitted means every connected one.
1934
2210
  if (obj.command === 'getWakeWords') {
@@ -1966,6 +2242,41 @@ class Assistant extends adapter_core_1.Adapter {
1966
2242
  }
1967
2243
  return;
1968
2244
  }
2245
+ // Settings dialog / scripts: own wake words for the ESPHome satellites (microWakeWord models).
2246
+ // uploadWakeWord: { model: base64 .tflite, fileName?, config?: JSON text, phrase?, languages? }
2247
+ // → { ok, id, phrase, languages, size, devices } | { error }
2248
+ // deleteWakeWord: { id } → { ok } | { error }; listWakeWords → { wakeWords: [{id, phrase, …}] }
2249
+ if (obj.command === 'uploadWakeWord' || obj.command === 'deleteWakeWord' || obj.command === 'listWakeWords') {
2250
+ const m = (obj.message || {});
2251
+ let payload;
2252
+ try {
2253
+ if (obj.command === 'uploadWakeWord') {
2254
+ payload = { ok: true, ...(await this.addCustomWakeWord(m)) };
2255
+ }
2256
+ else if (obj.command === 'deleteWakeWord') {
2257
+ await this.deleteCustomWakeWord(typeof m.id === 'string' ? m.id : '');
2258
+ payload = { ok: true };
2259
+ }
2260
+ else {
2261
+ await this.loadCustomWakeWords();
2262
+ payload = {
2263
+ wakeWords: [...this.customWakeWords.values()].map(w => ({
2264
+ id: w.id,
2265
+ phrase: w.phrase,
2266
+ languages: w.languages,
2267
+ size: w.size,
2268
+ })),
2269
+ };
2270
+ }
2271
+ }
2272
+ catch (e) {
2273
+ payload = { error: e.message };
2274
+ }
2275
+ if (obj.callback) {
2276
+ this.sendTo(obj.from, obj.command, payload, obj.callback);
2277
+ }
2278
+ return;
2279
+ }
1969
2280
  // Scripts: everything an ESPHome satellite exposes besides the voice pipeline.
1970
2281
  // message = { device? }; without a device, every connected one is listed.
1971
2282
  if (obj.command === 'getControls') {
@@ -2172,6 +2483,18 @@ class Assistant extends adapter_core_1.Adapter {
2172
2483
  }
2173
2484
  return;
2174
2485
  }
2486
+ // Admin chat debug view: one request with every step of its processing (message = { id }).
2487
+ if (obj.command === 'getJournalTrace') {
2488
+ const id = String(obj.message?.id || '');
2489
+ const entry = this.journal?.get(id);
2490
+ const result = entry
2491
+ ? { entry, steps: await this.journal.trace(id) }
2492
+ : { error: this.journal?.enabled ? 'unknown request' : 'journal disabled' };
2493
+ if (obj.callback) {
2494
+ this.sendTo(obj.from, obj.command, result, obj.callback);
2495
+ }
2496
+ return;
2497
+ }
2175
2498
  if (obj.command === 'clearJournal') {
2176
2499
  this.journal?.clear();
2177
2500
  if (obj.callback) {
@@ -2222,29 +2545,93 @@ class Assistant extends adapter_core_1.Adapter {
2222
2545
  * later turns have the full thread.
2223
2546
  */
2224
2547
  answer(question, source = '', origin = {}) {
2225
- return this.journal
2226
- ? this.journal.run(question, source, origin, () => this.answerTracked(question, source))
2227
- : this.answerTracked(question, source);
2548
+ return this.requestOrigin.run({ source, room: origin.room || '' }, () => this.journal
2549
+ ? this.journal.run(question, source, origin, () => {
2550
+ this.traceSpeechInput(question, origin);
2551
+ return this.answerTracked(question, source);
2552
+ })
2553
+ : this.answerTracked(question, source));
2554
+ }
2555
+ /** Speech recognition happened before the request: put it at the head of its trace. */
2556
+ traceSpeechInput(question, origin) {
2557
+ if (origin.sttMs === undefined) {
2558
+ return;
2559
+ }
2560
+ const cfg = this.config;
2561
+ const provider = cfg.sttProvider || 'openai';
2562
+ const audio = origin.audioMs !== undefined ? `, ${(origin.audioMs / 1000).toFixed(1)} s of audio` : '';
2563
+ this.journal?.step({
2564
+ stage: 'stt',
2565
+ title: `Speech recognition: ${provider}${audio}`,
2566
+ ms: origin.sttMs,
2567
+ data: {
2568
+ provider,
2569
+ fallback: cfg.sttFallback || undefined,
2570
+ model: (cfg.sttModel || cfg.voskModel || '').trim() || undefined,
2571
+ language: cfg.voiceLanguage || this.language || '',
2572
+ audioMs: origin.audioMs,
2573
+ wakeWord: origin.wakeWord,
2574
+ vad: origin.vad,
2575
+ transcript: question,
2576
+ },
2577
+ });
2228
2578
  }
2229
2579
  async answerTracked(question, source) {
2230
2580
  const useCtx = this.config.useConversationContext !== false;
2231
2581
  const history = useCtx ? this.context.get(source) : [];
2582
+ if (history.length) {
2583
+ this.journal?.step({
2584
+ stage: 'context',
2585
+ title: `Conversation context: ${history.length} earlier turn(s) of '${source}'`,
2586
+ data: history,
2587
+ });
2588
+ }
2232
2589
  const raw = await this.produceAnswer(question, history, source);
2233
2590
  // The LLM signals "I'm waiting for the user's reply" by appending the [[LISTEN]] control marker
2234
- // (see followUpHint). Strip it here so it's neither stored, shown nor spoken, and expose it as the
2235
- // follow-up flag. Single-user voice → a shared field is race-free enough (read right after await).
2236
- this.lastFollowUp = /\[\[LISTEN\]\]/i.test(raw);
2237
- if (this.lastFollowUp) {
2591
+ // (see followUpHint). Strip it here so it's neither stored, shown nor spoken, and keep it per source:
2592
+ // two satellites answered at the same time must not hand each other their mic-on.
2593
+ const followUp = /\[\[LISTEN\]\]/i.test(raw);
2594
+ this.followUps.set(source, followUp);
2595
+ if (followUp) {
2238
2596
  this.journal?.note({ followUp: true });
2239
2597
  }
2240
2598
  const result = raw.replace(/\s*\[\[LISTEN\]\]\s*/gi, ' ').trim();
2599
+ this.journal?.step({
2600
+ stage: 'answer',
2601
+ title: result
2602
+ ? `Answer: ${result.length} characters${followUp ? ', waits for a reply' : ''}`
2603
+ : 'No spoken answer',
2604
+ data: { answer: result, raw: raw !== result ? raw : undefined, followUp },
2605
+ });
2241
2606
  if (useCtx && result) {
2242
2607
  this.context.add(source, question, result);
2243
2608
  }
2244
2609
  return result;
2245
2610
  }
2246
- /** Whether the LAST {@link answer} asked the user something (LLM-signalled via [[LISTEN]]). */
2247
- lastFollowUp = false;
2611
+ /** Per source: whether its last {@link answer} asked the user something (LLM-signalled via [[LISTEN]]). */
2612
+ followUps = new Map();
2613
+ /**
2614
+ * Should the satellite behind `source` re-open its mic after this reply? Primarily the LLM's own
2615
+ * [[LISTEN]] signal, with a question mark in the reply as the fallback (the NLU and the local tiers
2616
+ * cannot emit the marker, but they do ask "Meinst du Büro oder Wohnzimmer?").
2617
+ */
2618
+ wantsFollowUp(source, answer) {
2619
+ if (!answer) {
2620
+ return null;
2621
+ }
2622
+ if (this.followUps.get(source)) {
2623
+ return 'llm';
2624
+ }
2625
+ return this.expectsFollowUp(answer) ? 'question' : null;
2626
+ }
2627
+ /** {@link wantsFollowUp} for a voice transport, logged, as the `listen` flag of the reply. */
2628
+ followUpFor(source, answer) {
2629
+ const why = this.wantsFollowUp(source, answer);
2630
+ if (why) {
2631
+ this.log.info(`→ mic-on to satellite '${source}' (${why === 'llm' ? 'LLM signalled a follow-up' : 'reply is a question'}).`);
2632
+ }
2633
+ return !!why;
2634
+ }
2248
2635
  /**
2249
2636
  * System-prompt instruction letting the LLM explicitly signal a follow-up: append [[LISTEN]] when it
2250
2637
  * asks the user something and waits for an answer. More reliable than a "?" heuristic and lets a
@@ -2321,6 +2708,7 @@ class Assistant extends adapter_core_1.Adapter {
2321
2708
  const n = this.stopRinging();
2322
2709
  this.log.info(`Ring stopped by voice (${n} session(s)).`);
2323
2710
  this.journal?.note({ tier: 'stop' });
2711
+ this.journal?.step({ stage: 'stop', title: `Stop word: silenced ${n} ringing timer/alarm(s)` });
2324
2712
  const lang = String(this.config.voiceLanguage || this.language || 'en');
2325
2713
  return lang === 'ru' ? 'Хорошо.' : lang === 'de' ? 'Ok.' : 'Okay.';
2326
2714
  }
@@ -2332,6 +2720,11 @@ class Assistant extends adapter_core_1.Adapter {
2332
2720
  if (this.pending.deliver(source, question)) {
2333
2721
  this.log.info(`Answer to "${asked}" from source '${source || 'any'}': ${question}`);
2334
2722
  this.journal?.note({ tier: 'askReply', detail: asked || undefined });
2723
+ this.journal?.step({
2724
+ stage: 'askReply',
2725
+ title: `Reply to the open question "${asked}"`,
2726
+ data: { question: asked, reply: question },
2727
+ });
2335
2728
  return '';
2336
2729
  }
2337
2730
  }
@@ -2342,11 +2735,38 @@ class Assistant extends adapter_core_1.Adapter {
2342
2735
  if (routine) {
2343
2736
  this.log.info(`Routine '${routine.name}' triggered (source='${source}').`);
2344
2737
  this.journal?.note({ tier: 'routine', detail: routine.name });
2738
+ this.journal?.step({ stage: 'routine', title: `Routine "${routine.name}" matched`, data: routine });
2345
2739
  await this.runRoutine(routine, source);
2346
2740
  return routine.reply;
2347
2741
  }
2742
+ if (this.routines.length) {
2743
+ this.journal?.step({
2744
+ stage: 'routine',
2745
+ title: `No routine matched (${this.routines.length} configured)`,
2746
+ data: { phrases: this.routines.map(r => ({ name: r.name, phrases: r.phrases })) },
2747
+ });
2748
+ }
2348
2749
  // Tier 0: rule-based NLU (device commands) — fastest, offline, free.
2349
2750
  if (this.config.useLocalNlu) {
2751
+ // "Spiele den Kanal X ab" first: the device NLU would find the TV in "… auf dem Fernseher" and
2752
+ // answer with its power state.
2753
+ const yt = this.youtubePlayers.length ? (0, youtube_1.parseYouTubeCommand)(question, this.youtubePlayers) : null;
2754
+ if (yt) {
2755
+ this.log.info(`YouTube request understood locally: ${yt.kind} "${yt.query}" (source='${source}').`);
2756
+ this.journal?.note({ tier: 'nlu', detail: `youtube ${yt.kind}` });
2757
+ const start = Date.now();
2758
+ const res = await this.playYouTube(yt.query, yt.kind, yt.player);
2759
+ this.journal?.step({
2760
+ stage: 'youtube',
2761
+ title: `YouTube ${yt.kind} "${yt.query}"${yt.player ? ` on "${yt.player}"` : ''}`,
2762
+ ms: Date.now() - start,
2763
+ data: { parsed: yt, result: res },
2764
+ });
2765
+ return res.text;
2766
+ }
2767
+ if (this.youtubePlayers.length) {
2768
+ this.journal?.step({ stage: 'youtube', title: 'Not a YouTube command' });
2769
+ }
2350
2770
  try {
2351
2771
  const handled = await this.tryLocalNlu(question, source);
2352
2772
  if (handled !== null) {
@@ -2356,36 +2776,49 @@ class Assistant extends adapter_core_1.Adapter {
2356
2776
  }
2357
2777
  catch (e) {
2358
2778
  this.log.debug(`NLU skipped: ${e.message}`);
2779
+ this.journal?.step({
2780
+ stage: 'nlu',
2781
+ level: 'warn',
2782
+ title: `NLU gave up, the request goes on: ${e.message}`,
2783
+ data: (0, journal_1.errorInfo)(e),
2784
+ });
2359
2785
  }
2360
2786
  }
2787
+ else {
2788
+ this.journal?.step({ stage: 'nlu', title: 'Rule-based NLU is switched off (useLocalNlu)' });
2789
+ }
2361
2790
  // Current weather from the configured adapter — prepended to the user turn of BOTH LLM tiers so
2362
2791
  // "how's the weather?" is answered from real data (the local model has no tools and would otherwise
2363
2792
  // invent a forecast; the cloud model saves a get_weather round-trip). Cached, see buildWeatherContext.
2364
2793
  const weather = await this.buildWeatherContext();
2365
2794
  // Tier 1a: local LLM — answers general questions offline; emits HANDOFF for anything needing tools.
2795
+ // With a cloud LLM it only goes first when asked to (`localLlmFirst`): on a Raspberry Pi a small model
2796
+ // needs 10–25 s, often only to hand off, and the cloud then starts from zero. By default it is the
2797
+ // fallback for when the cloud fails (see below).
2366
2798
  // Skipped inside an ongoing conversation (history present): the tool-free local model gets no history,
2367
2799
  // so it would answer a follow-up like "yes" context-free and swallow it — let the cloud LLM (which has
2368
2800
  // the thread and the tools) handle follow-ups instead. Without a cloud LLM there is nobody to hand
2369
2801
  // follow-ups to, so the local model takes them too.
2370
- if (this.config.useLocalLlm && this.localLlm && (!history.length || !this.agent)) {
2371
- try {
2372
- // The local model has no tools: without these lines it would invent the weather and guess
2373
- // who is at home.
2374
- const facts = [weather, this.buildPresenceContext()].filter(Boolean).join('\n\n');
2375
- const ans = await this.localLlm.ask(facts ? `${facts}\n\n${question}` : question);
2376
- if (ans && !(0, localLlm_1.isHandoff)(ans)) {
2377
- this.log.info(`Answered by local LLM (source='${source}').`);
2378
- this.journal?.note({ tier: 'localLlm', detail: this.localLlmName() });
2379
- return ans;
2380
- }
2381
- this.log.debug('Local LLM handed off to the cloud LLM.');
2382
- }
2383
- catch (e) {
2384
- this.log.debug(`Local LLM error: ${e.message}`);
2802
+ const useLocal = !!(this.config.useLocalLlm && this.localLlm);
2803
+ const localFirst = useLocal && (!this.agent || !!this.config.localLlmFirst);
2804
+ if (localFirst && (!history.length || !this.agent)) {
2805
+ const ans = await this.askLocalLlm(question, weather, source);
2806
+ if (ans) {
2807
+ return ans;
2385
2808
  }
2386
2809
  }
2387
- else if (this.config.useLocalLlm && this.localLlm) {
2810
+ else if (localFirst) {
2388
2811
  this.log.debug('Local LLM skipped (follow-up in an ongoing conversation → cloud LLM).');
2812
+ this.journal?.step({
2813
+ stage: 'localLlm',
2814
+ title: 'Local LLM skipped: follow-up in an ongoing conversation goes to the cloud LLM',
2815
+ });
2816
+ }
2817
+ else if (useLocal) {
2818
+ this.journal?.step({
2819
+ stage: 'localLlm',
2820
+ title: 'Local LLM not asked first: it is the fallback for when the cloud LLM fails (localLlmFirst)',
2821
+ });
2389
2822
  }
2390
2823
  // Tier 2: cloud LLM with full tool access. Inject a compact device list into the system prompt so
2391
2824
  // the model can act without a first `list_devices` round-trip (kept cached via prompt caching).
@@ -2394,6 +2827,11 @@ class Assistant extends adapter_core_1.Adapter {
2394
2827
  // read out as a technical message.
2395
2828
  this.log.info(`No local tier could answer and no LLM provider is configured (source='${source}').`);
2396
2829
  this.journal?.note({ tier: 'none' });
2830
+ this.journal?.step({
2831
+ stage: 'none',
2832
+ level: 'warn',
2833
+ title: 'Nothing local understood the request and no LLM provider is configured',
2834
+ });
2397
2835
  return this.localOnlyFallback();
2398
2836
  }
2399
2837
  const ctx = await this.buildDeviceContext();
@@ -2405,12 +2843,99 @@ class Assistant extends adapter_core_1.Adapter {
2405
2843
  this.log.info(`Cloud LLM [${this.agent.model || 'auto'}] (source='${source}', context=${history.length} turn(s)${mem ? ', memory' : ''}${weather ? ', weather' : ''}${this.presence?.configured ? ', presence' : ''}) — running tool loop…`);
2406
2844
  // Prepend the current date/time (and the weather line, if a source is configured) to the user turn —
2407
2845
  // cache-safe, see buildTimeContext. The stored history keeps the untouched question.
2408
- const questionForLlm = [this.buildTimeContext(), weather, question].filter(Boolean).join('\n\n');
2846
+ const time = this.buildTimeContext();
2847
+ const questionForLlm = [time, weather, question].filter(Boolean).join('\n\n');
2409
2848
  this.journal?.note({
2410
2849
  tier: 'cloud',
2411
2850
  detail: `${this.config.provider || 'openai'} · ${this.agent.model || 'auto'}`,
2412
2851
  });
2413
- return this.agent.ask(questionForLlm, sys, history);
2852
+ // What went into the prompt, piece by piece. The device list is only counted: the model's opening
2853
+ // step carries the whole system prompt, and with it the list.
2854
+ const deviceLines = ctx ? ctx.split('\n').length - 1 : 0;
2855
+ this.journal?.step({
2856
+ stage: 'context',
2857
+ title: `Prompt: ${deviceLines} devices${mem ? ', memory' : ''}${weather ? ', weather' : ''}${followUp ? ', follow-up marker' : ''}`,
2858
+ data: {
2859
+ systemPrompt: this.config.systemPrompt || '',
2860
+ followUpHint: followUp || undefined,
2861
+ memory: mem || undefined,
2862
+ devices: deviceLines,
2863
+ time,
2864
+ weather: weather || undefined,
2865
+ },
2866
+ });
2867
+ try {
2868
+ return await this.agent.ask(questionForLlm, sys, history);
2869
+ }
2870
+ catch (e) {
2871
+ // Offline, quota, outage: the local model is the fallback — unless it was already asked above.
2872
+ if (useLocal && !localFirst) {
2873
+ this.log.warn(`Cloud LLM failed (${e.message}) — asking the local LLM instead.`);
2874
+ this.journal?.step({
2875
+ stage: 'localLlm',
2876
+ level: 'warn',
2877
+ title: `Cloud LLM failed (${e.message}) — asking the local LLM instead`,
2878
+ });
2879
+ const ans = await this.askLocalLlm(question, weather, source);
2880
+ if (ans) {
2881
+ return ans;
2882
+ }
2883
+ }
2884
+ throw e;
2885
+ }
2886
+ }
2887
+ /**
2888
+ * Ask the local model, bounded by `localLlmTimeoutSec`. Returns its answer, or null when it handed off,
2889
+ * timed out or failed — the caller then goes on to the next tier.
2890
+ */
2891
+ async askLocalLlm(question, weather, source) {
2892
+ if (!this.localLlm) {
2893
+ return null;
2894
+ }
2895
+ const timeoutMs = Math.max(1, Number(this.config.localLlmTimeoutSec) || LOCAL_LLM_TIMEOUT_SEC) * 1000;
2896
+ const name = this.localLlmName();
2897
+ // The local model has no tools: without these lines it would invent the weather and guess who is
2898
+ // at home.
2899
+ const facts = [weather, this.buildPresenceContext()].filter(Boolean).join('\n\n');
2900
+ const prompt = facts ? `${facts}\n\n${question}` : question;
2901
+ const busy = this.localLlm.running;
2902
+ const data = { model: name, timeoutMs, system: this.localLlm.systemPrompt(), prompt };
2903
+ const start = Date.now();
2904
+ try {
2905
+ const ans = await this.localLlm.ask(prompt, timeoutMs);
2906
+ const ms = Date.now() - start;
2907
+ if (ans && !(0, localLlm_1.isHandoff)(ans)) {
2908
+ this.log.info(`Answered by local LLM (source='${source}').`);
2909
+ this.journal?.note({ tier: 'localLlm', detail: name });
2910
+ this.journal?.step({
2911
+ stage: 'localLlm',
2912
+ title: `Local LLM ${name}: answered`,
2913
+ ms,
2914
+ data: { ...data, answer: ans },
2915
+ });
2916
+ return ans;
2917
+ }
2918
+ this.log.debug('Local LLM handed off.');
2919
+ this.journal?.step({
2920
+ stage: 'localLlm',
2921
+ title: busy
2922
+ ? `Local LLM ${name}: busy with another question → handed off`
2923
+ : `Local LLM ${name}: handed off`,
2924
+ ms,
2925
+ data: { ...data, answer: ans },
2926
+ });
2927
+ }
2928
+ catch (e) {
2929
+ this.log.info(`Local LLM gave no answer: ${e.message}`);
2930
+ this.journal?.step({
2931
+ stage: 'localLlm',
2932
+ level: 'warn',
2933
+ title: `Local LLM ${name}: ${e.message}`,
2934
+ ms: Date.now() - start,
2935
+ data: { ...data, error: (0, journal_1.errorInfo)(e) },
2936
+ });
2937
+ }
2938
+ return null;
2414
2939
  }
2415
2940
  /** Reply when nothing local understood the request and there is no cloud LLM (provider "none"). */
2416
2941
  localOnlyFallback() {
@@ -2521,35 +3046,27 @@ class Assistant extends adapter_core_1.Adapter {
2521
3046
  else {
2522
3047
  // Switched off: do not keep what was recorded while it was on.
2523
3048
  await fs.promises.rm(file, { force: true }).catch(() => { });
3049
+ await fs.promises.rm((0, journal_1.traceDirFor)(file), { recursive: true, force: true }).catch(() => { });
2524
3050
  }
2525
3051
  }
2526
- /** Wrap the MCP connection so every state the pipeline reads or writes lands in the journal entry. */
3052
+ /**
3053
+ * Wrap the MCP connection so every state the pipeline reads or writes lands in the journal entry — and a
3054
+ * call the NLU makes directly, with its result, in the request's trace.
3055
+ */
2527
3056
  journalMcp(mcp) {
2528
3057
  return {
2529
3058
  listTools: () => mcp.listTools(),
2530
- callTool: (name, args) => {
2531
- this.journal?.stateAccess(name, args);
2532
- return mcp.callTool(name, args);
2533
- },
3059
+ callTool: (name, args) => this.journal
3060
+ ? this.journal.stateCall(name, args, () => mcp.callTool(name, args))
3061
+ : mcp.callTool(name, args),
2534
3062
  close: () => mcp.close(),
2535
3063
  };
2536
3064
  }
2537
- /** Wrap an LLM tool so its calls (arguments, result, duration) land in the journal entry. */
3065
+ /** Wrap an LLM tool so its calls (arguments, result, duration) land in the journal entry and its trace. */
2538
3066
  journalTool(tool) {
2539
3067
  return {
2540
3068
  ...tool,
2541
- run: async (args) => {
2542
- const start = Date.now();
2543
- try {
2544
- const result = await tool.run(args);
2545
- this.journal?.tool(tool.name, args, Date.now() - start, result);
2546
- return result;
2547
- }
2548
- catch (e) {
2549
- this.journal?.tool(tool.name, args, Date.now() - start, undefined, e.message);
2550
- throw e;
2551
- }
2552
- },
3069
+ run: args => (this.journal ? this.journal.toolCall(tool.name, args, () => tool.run(args)) : tool.run(args)),
2553
3070
  };
2554
3071
  }
2555
3072
  /** Short name of the local model for the journal ("qwen2.5-1.5b-instruct-q4_k_m"). */
@@ -2647,7 +3164,7 @@ class Assistant extends adapter_core_1.Adapter {
2647
3164
  dataDir,
2648
3165
  modelUrl: (this.config.localLlmModelUrl || '').trim() || localLlm_1.DEFAULT_LOCAL_MODEL_URL,
2649
3166
  systemPrompt: this.config.systemPrompt || '',
2650
- maxTokens: this.config.maxTokens || 512,
3167
+ maxTokens: Math.min(this.config.maxTokens || LOCAL_LLM_MAX_TOKENS, LOCAL_LLM_MAX_TOKENS),
2651
3168
  log: this.log,
2652
3169
  });
2653
3170
  this.lastLlmPct = -1;
@@ -2680,20 +3197,56 @@ class Assistant extends adapter_core_1.Adapter {
2680
3197
  }
2681
3198
  // Timer intents match device-independently (parse() checks them first), so build the NLU even
2682
3199
  // when no devices are known and always try timers before bailing out.
3200
+ const start = Date.now();
2683
3201
  const { rooms, devices } = await this.getNluDevices();
2684
- const intents = new nlu_1.Nlu(rooms, devices, this.getNluAliases()).parseAll(question);
2685
- if (!intents.length || !intents.every(i => this.canExecuteNlu(i, devices.length))) {
3202
+ const aliases = this.getNluAliases();
3203
+ const intents = new nlu_1.Nlu(rooms, devices, aliases, {
3204
+ weather: !!(this.config.weatherInstance || '').trim(),
3205
+ }).parseAll(question);
3206
+ const blockers = intents.map(i => this.nluBlocker(i, devices.length));
3207
+ this.journal?.step({
3208
+ stage: 'nlu',
3209
+ title: !intents.length
3210
+ ? `NLU: nothing recognised (${devices.length} devices, ${rooms.length} rooms known)`
3211
+ : blockers.some(Boolean)
3212
+ ? `NLU: ${intents.map(i => i.action).join(' + ')} — cannot run here: ${blockers.filter(Boolean).join('; ')}`
3213
+ : `NLU: ${intents.map(describeIntent).join(' + ')}`,
3214
+ level: intents.length && blockers.some(Boolean) ? 'warn' : undefined,
3215
+ ms: Date.now() - start,
3216
+ data: {
3217
+ devices: devices.length,
3218
+ rooms,
3219
+ aliases: aliases.length ? aliases : undefined,
3220
+ intents: intents.map((intent, i) => ({ ...intent, blockedBecause: blockers[i] || undefined })),
3221
+ },
3222
+ });
3223
+ if (!intents.length || blockers.some(Boolean)) {
2686
3224
  return null; // nothing matched, or one part cannot run here → let the LLM answer the whole thing
2687
3225
  }
2688
3226
  this.journal?.note({ tier: 'nlu', detail: intents.map(i => i.action).join(' + ') });
2689
3227
  const answers = [];
2690
3228
  let failed = false;
2691
3229
  for (const intent of intents) {
3230
+ const started = Date.now();
2692
3231
  try {
2693
- answers.push(await this.executeNluIntent(intent, source));
3232
+ const answer = await this.executeNluIntent(intent, source);
3233
+ answers.push(answer);
3234
+ this.journal?.step({
3235
+ stage: 'nlu',
3236
+ title: `Executed ${describeIntent(intent)}`,
3237
+ ms: Date.now() - started,
3238
+ data: { answer },
3239
+ });
2694
3240
  }
2695
3241
  catch (e) {
2696
3242
  failed = true;
3243
+ this.journal?.step({
3244
+ stage: 'nlu',
3245
+ level: 'error',
3246
+ title: `${describeIntent(intent)} failed: ${e.message}${answers.length ? '' : ' → the LLM takes over'}`,
3247
+ ms: Date.now() - started,
3248
+ data: (0, journal_1.errorInfo)(e),
3249
+ });
2697
3250
  // Nothing executed yet → hand the whole utterance to the LLM. Afterwards a state has already
2698
3251
  // been written and the LLM would repeat it, so report the failed part instead of bailing out.
2699
3252
  if (!answers.length) {
@@ -2716,35 +3269,44 @@ class Assistant extends adapter_core_1.Adapter {
2716
3269
  intents.every(i => CONTROL_ACTIONS.has(i.action)) &&
2717
3270
  (await this.playConfirmationTone(source))) {
2718
3271
  this.log.debug(`NLU confirmed with a tone instead of "${answer}".`);
3272
+ this.journal?.step({ stage: 'nlu', title: 'Confirmed with a tone instead of words', data: { answer } });
2719
3273
  return '';
2720
3274
  }
2721
3275
  return answer;
2722
3276
  }
2723
3277
  /**
2724
- * Can this intent run locally at all? Checked for **every** part of a combined command before anything
2725
- * is executed: a missing manager, no known devices or disabled writes send the whole utterance to the
2726
- * LLM (which can explain why) instead of silently doing nothing — or only half of it.
3278
+ * Why this intent cannot run locally ('' = it can). Checked for **every** part of a combined command
3279
+ * before anything is executed: a missing manager, no known devices or disabled writes send the whole
3280
+ * utterance to the LLM (which can explain why) instead of silently doing nothing — or only half of it.
2727
3281
  */
2728
- canExecuteNlu(intent, deviceCount) {
3282
+ nluBlocker(intent, deviceCount) {
2729
3283
  switch (intent.action) {
2730
3284
  case 'timerSet':
2731
3285
  case 'timerQuery':
2732
3286
  case 'timerCancel':
2733
- return !!this.timers;
3287
+ return this.timers ? '' : 'timers are not running';
2734
3288
  case 'alarmSet':
2735
3289
  case 'alarmQuery':
2736
3290
  case 'alarmCancel':
2737
- return !!this.alarms;
3291
+ return this.alarms ? '' : 'alarms are not running';
2738
3292
  case 'timeQuery':
2739
3293
  case 'dateQuery':
2740
- return true;
3294
+ return '';
3295
+ case 'weatherQuery':
3296
+ return (this.config.weatherInstance || '').trim() ? '' : 'no weather instance configured';
3297
+ case 'volumeSet':
3298
+ return this.voice || this.esphome || this.nativeSatFrom.size ? '' : 'no satellites';
2741
3299
  default:
2742
3300
  // Device intents; writes additionally require the coarse "allow control" toggle.
2743
- return (deviceCount > 0 &&
2744
- (!['on', 'off', 'level', 'color'].includes(intent.action) || !!this.config.allowWriteStates));
3301
+ if (!deviceCount) {
3302
+ return 'no devices known';
3303
+ }
3304
+ return ['on', 'off', 'level', 'color'].includes(intent.action) && !this.config.allowWriteStates
3305
+ ? 'writing states is not allowed (allowWriteStates)'
3306
+ : '';
2745
3307
  }
2746
3308
  }
2747
- /** Execute one intent that {@link canExecuteNlu} accepted, and return its spoken answer. */
3309
+ /** Execute one intent that {@link nluBlocker} accepted, and return its spoken answer. */
2748
3310
  async executeNluIntent(intent, source) {
2749
3311
  switch (intent.action) {
2750
3312
  case 'timerSet':
@@ -2758,10 +3320,97 @@ class Assistant extends adapter_core_1.Adapter {
2758
3320
  case 'timeQuery':
2759
3321
  case 'dateQuery':
2760
3322
  return this.executeTimeIntent(intent);
3323
+ case 'weatherQuery':
3324
+ return this.executeWeatherIntent(intent);
3325
+ case 'volumeSet':
3326
+ return this.executeVolumeIntent(intent, source);
2761
3327
  default:
2762
3328
  return this.executeIntent(intent);
2763
3329
  }
2764
3330
  }
3331
+ /**
3332
+ * Speak the weather straight from the configured adapter. Throws when the report has nothing for the
3333
+ * asked day (or the adapter has no mapper), so the question goes on to the LLM and its tool instead.
3334
+ */
3335
+ async executeWeatherIntent(intent) {
3336
+ const res = await this.readWeather(this.config.weatherInstance || '');
3337
+ if (!res.report) {
3338
+ throw new Error(res.error || 'no weather report');
3339
+ }
3340
+ const lang = String(this.config.voiceLanguage || this.language || 'en');
3341
+ const text = (0, weather_1.buildWeatherAnswer)(res.report, lang, intent.when);
3342
+ if (!text) {
3343
+ throw new Error(`the weather report has no data for '${intent.when || 'now'}'`);
3344
+ }
3345
+ return text;
3346
+ }
3347
+ /**
3348
+ * Change the playback volume of a satellite: the one in the named room, else the one that heard the
3349
+ * command. Each transport has its own knob — the native satellite's `volume` state, the media player
3350
+ * of an ESPHome device, the `volume` message of a UDP satellite.
3351
+ */
3352
+ async executeVolumeIntent(intent, source) {
3353
+ const lang = String(this.config.voiceLanguage || this.language || 'en');
3354
+ const satId = intent.room ? this.resolveSatelliteId(intent.room) : this.announceTargetForSource(source);
3355
+ const absolute = typeof intent.value === 'number' ? intent.value : null;
3356
+ const delta = intent.delta || 0;
3357
+ const next = (current) => Math.min(100, Math.max(0, Math.round(absolute ?? current + delta)));
3358
+ // 1 процент, 2 процента, 5 процентов (11–14 take the last form).
3359
+ const ruPercent = (n) => n % 100 > 10 && n % 100 < 20
3360
+ ? 'процентов'
3361
+ : n % 10 === 1
3362
+ ? 'процент'
3363
+ : n % 10 >= 2 && n % 10 <= 4
3364
+ ? 'процента'
3365
+ : 'процентов';
3366
+ const confirm = (level) => lang === 'ru'
3367
+ ? `Громкость ${level} ${ruPercent(level)}.`
3368
+ : lang === 'de'
3369
+ ? `Lautstärke ${level} Prozent.`
3370
+ : `Volume ${level} percent.`;
3371
+ if (satId) {
3372
+ const device = this.satDeviceById.get(satId) || satId;
3373
+ const instance = this.nativeSatFrom.get(satId);
3374
+ if (instance) {
3375
+ const stateId = `${instance}.volume`;
3376
+ const current = Number((await this.getForeignStateAsync(stateId))?.val ?? 50);
3377
+ const level = next(Number.isFinite(current) ? current : 50);
3378
+ await this.setForeignStateAsync(stateId, level);
3379
+ return confirm(level);
3380
+ }
3381
+ const player = this.esphome?.entities(device).find(e => e.kind === 'mediaPlayer');
3382
+ if (player) {
3383
+ const current = (player.value?.volume ?? 0.5) * 100;
3384
+ const level = next(current);
3385
+ if (this.esphome.setEntity(device, player.objectId, level / 100)) {
3386
+ return confirm(level);
3387
+ }
3388
+ }
3389
+ // A UDP satellite works out a relative step itself (it knows its level) and reports the result.
3390
+ if (this.voice?.has(device)) {
3391
+ this.voice.volume(device, absolute !== null ? { level: absolute } : { delta });
3392
+ if (absolute !== null) {
3393
+ return confirm(next(0));
3394
+ }
3395
+ return lang === 'ru'
3396
+ ? delta > 0
3397
+ ? 'Сделала громче.'
3398
+ : 'Сделала тише.'
3399
+ : lang === 'de'
3400
+ ? delta > 0
3401
+ ? 'Etwas lauter.'
3402
+ : 'Etwas leiser.'
3403
+ : delta > 0
3404
+ ? 'A bit louder.'
3405
+ : 'A bit quieter.';
3406
+ }
3407
+ }
3408
+ return lang === 'ru'
3409
+ ? 'Я не знаю, громкость какой колонки изменить.'
3410
+ : lang === 'de'
3411
+ ? 'Ich weiß nicht, bei welchem Lautsprecher ich die Lautstärke ändern soll.'
3412
+ : "I don't know which speaker's volume to change.";
3413
+ }
2765
3414
  /** Localized "this part did not work" sentence for a failed part of a combined command. */
2766
3415
  nluFailureText(intent) {
2767
3416
  const lang = String(this.config.voiceLanguage || this.language || 'en');
@@ -2811,14 +3460,36 @@ class Assistant extends adapter_core_1.Adapter {
2811
3460
  return [];
2812
3461
  }
2813
3462
  }
2814
- /** Device model for the NLU: friendly name, room, type and controls (controlType → state id). */
2815
- async getNluDevices() {
3463
+ /**
3464
+ * Device model for the NLU: friendly name, room, type and controls (controlType → state id). Cached
3465
+ * for {@link NLU_DEVICES_TTL_MS}; requests arriving while it is being built wait for the same build.
3466
+ */
3467
+ getNluDevices() {
2816
3468
  if (!this.mcp) {
2817
- return { rooms: [], devices: [] };
3469
+ return Promise.resolve({ rooms: [], devices: [] });
2818
3470
  }
2819
3471
  // Match names in the assistant's interaction language (voice language, else system) so a
2820
3472
  // Russian command matches Russian smartNames.
2821
3473
  const nluLang = (this.config.voiceLanguage || this.language || 'en');
3474
+ const now = Date.now();
3475
+ const hit = this.nluDevices;
3476
+ if (hit && hit.key === nluLang && now - hit.at < NLU_DEVICES_TTL_MS) {
3477
+ return hit.result;
3478
+ }
3479
+ const entry = { key: nluLang, at: now, result: this.buildNluDevices(nluLang) };
3480
+ this.nluDevices = entry;
3481
+ // An empty model is usually a failed build (the manager not ready yet) — don't keep it for a minute.
3482
+ void entry.result.then(r => {
3483
+ if (!r.devices.length && this.nluDevices === entry) {
3484
+ this.nluDevices = null;
3485
+ }
3486
+ });
3487
+ return entry.result;
3488
+ }
3489
+ async buildNluDevices(nluLang) {
3490
+ if (!this.mcp) {
3491
+ return { rooms: [], devices: [] };
3492
+ }
2822
3493
  try {
2823
3494
  const res = await this.mcp.callTool('list_devices', { language: nluLang });
2824
3495
  const parsed = JSON.parse(res.text);
@@ -3450,16 +4121,30 @@ class Assistant extends adapter_core_1.Adapter {
3450
4121
  if (!this.config.verifyWrites || !stateId) {
3451
4122
  return true;
3452
4123
  }
3453
- const deadline = Date.now() + ACK_TIMEOUT_MS;
4124
+ const start = Date.now();
4125
+ const deadline = start + ACK_TIMEOUT_MS;
3454
4126
  let last;
3455
4127
  while (Date.now() < deadline) {
3456
4128
  await this.delay(ACK_POLL_MS);
3457
4129
  last = await this.getForeignStateAsync(stateId).catch(() => null);
3458
4130
  if (last?.ack && valuesMatch(last.val, expected)) {
3459
4131
  this.log.debug(`write to ${stateId} confirmed by the device.`);
4132
+ this.journal?.step({
4133
+ stage: 'state',
4134
+ title: `${stateId} confirmed by the device`,
4135
+ ms: Date.now() - start,
4136
+ data: { expected, state: last },
4137
+ });
3460
4138
  return true;
3461
4139
  }
3462
4140
  }
4141
+ this.journal?.step({
4142
+ stage: 'state',
4143
+ level: 'warn',
4144
+ title: `${stateId}: no confirmation within ${ACK_TIMEOUT_MS} ms`,
4145
+ ms: Date.now() - start,
4146
+ data: { expected, state: last },
4147
+ });
3463
4148
  this.log.warn(`No confirmation for ${stateId} within ${ACK_TIMEOUT_MS} ms ` +
3464
4149
  `(last: ${JSON.stringify(last?.val)}, ack=${String(last?.ack)}). ` +
3465
4150
  'If this device never acknowledges, switch "Verify device feedback" off.');
@@ -3938,6 +4623,383 @@ class Assistant extends adapter_core_1.Adapter {
3938
4623
  },
3939
4624
  };
3940
4625
  }
4626
+ // ── YouTube on the TV (youtube-remote players) ──────────────────────────
4627
+ /** The player a request means: the named one, else the one in the requesting room, else the first. */
4628
+ youtubePlayerFor(name) {
4629
+ return (0, youtube_1.findPlayer)(this.youtubePlayers, name, this.requestOrigin.getStore()?.room || '');
4630
+ }
4631
+ /**
4632
+ * Find the device of a row that names a youtube-remote instance. Done again before every command: the
4633
+ * device id comes from the TV's screen id, so pairing the TV again replaces the device while we run.
4634
+ * false = nothing is paired with that instance.
4635
+ */
4636
+ async resolveYouTubePlayer(player) {
4637
+ if (!player.instance) {
4638
+ return true;
4639
+ }
4640
+ let devices = [];
4641
+ try {
4642
+ const objs = await this.getForeignObjectsAsync(`${player.instance}.*`, 'device');
4643
+ devices = Object.values(objs).map(o => ({ id: o._id, name: this.youtubeDeviceName(o) }));
4644
+ }
4645
+ catch (e) {
4646
+ this.log.debug(`YouTube: cannot read the devices of ${player.instance}: ${e.message}`);
4647
+ }
4648
+ const device = (0, youtube_1.pickDevice)(devices, player);
4649
+ player.base = device?.id || '';
4650
+ player.target = device ? `${device.id}.youtube` : '';
4651
+ if (device && !player.name) {
4652
+ player.name = device.name || player.instance;
4653
+ }
4654
+ return !!device;
4655
+ }
4656
+ /** Display name of an object, in the voice language if it is translated. */
4657
+ youtubeDeviceName(obj) {
4658
+ const name = obj.common?.name;
4659
+ if (typeof name === 'string') {
4660
+ return name;
4661
+ }
4662
+ const lang = String(this.config.voiceLanguage || this.language || 'en');
4663
+ return name?.[lang] || name?.en || obj._id.split('.').pop() || obj._id;
4664
+ }
4665
+ /**
4666
+ * Options of the "Player" column: every youtube-remote instance (it finds its TV itself), and for an
4667
+ * instance with several paired TVs also each of them, to pin a row to one.
4668
+ */
4669
+ async getYouTubeInstances() {
4670
+ const options = [];
4671
+ let instances = [];
4672
+ try {
4673
+ const view = await this.getObjectViewAsync('system', 'instance', {
4674
+ startkey: 'system.adapter.youtube-remote.',
4675
+ endkey: 'system.adapter.youtube-remote.香',
4676
+ });
4677
+ instances = (view?.rows || []).map(r => r.id.replace(/^system\.adapter\./, '')).sort();
4678
+ }
4679
+ catch (e) {
4680
+ this.log.warn(`getYouTubeInstances failed: ${e.message}`);
4681
+ }
4682
+ for (const instance of instances) {
4683
+ const objs = await this.getForeignObjectsAsync(`${instance}.*`, 'device').catch(() => ({}));
4684
+ const devices = Object.values(objs)
4685
+ .map(o => ({ id: o._id, name: this.youtubeDeviceName(o) }))
4686
+ .sort((a, b) => a.id.localeCompare(b.id));
4687
+ const names = devices.map(d => d.name).join(', ');
4688
+ options.push({ label: names ? `${instance} (${names})` : `${instance} (no TV paired)`, value: instance });
4689
+ if (devices.length > 1) {
4690
+ for (const d of devices) {
4691
+ options.push({ label: `${instance} › ${d.name}`, value: d.id });
4692
+ }
4693
+ }
4694
+ }
4695
+ return options;
4696
+ }
4697
+ /**
4698
+ * Find `query` on YouTube and play it on a player, switching its TV on first. The search runs before
4699
+ * anything is switched: a TV woken for a channel that does not exist would be the worst outcome.
4700
+ *
4701
+ * A TV that has to boot takes 10–30 s, so in that case the reply comes right away and the rest runs
4702
+ * in the background ({@link playWhenTvIsOn}) — over voice, half a minute of silence reads as "did not
4703
+ * hear me". An already running TV is answered only after the player acknowledged the command.
4704
+ */
4705
+ async playYouTube(query, kind, playerName) {
4706
+ const lang = String(this.config.voiceLanguage || this.language || 'en');
4707
+ const player = this.youtubePlayerFor(playerName);
4708
+ if (!player) {
4709
+ const names = this.youtubePlayers.map(p => p.name).join(', ');
4710
+ return { ok: false, text: (0, youtube_1.youtubeReply)(lang, 'unknownPlayer', { query: playerName, names }) };
4711
+ }
4712
+ if (!(await this.resolveYouTubePlayer(player))) {
4713
+ return {
4714
+ ok: false,
4715
+ player: player.name,
4716
+ text: (0, youtube_1.youtubeReply)(lang, 'noDevice', { instance: player.instance }),
4717
+ };
4718
+ }
4719
+ if (!this.config.allowWriteStates) {
4720
+ return { ok: false, player: player.name, text: (0, youtube_1.youtubeReply)(lang, 'noControl') };
4721
+ }
4722
+ let hit = null;
4723
+ let alternatives = [];
4724
+ // Unsaid kind: what is named like a channel is the channel ("spiele Sailing OceanAir auf YouTube"),
4725
+ // anything else a video search — a video search for a channel name finds some video about it.
4726
+ const searchKind = kind === 'auto' ? 'video' : kind;
4727
+ const direct = (0, youtube_1.directYouTubeTarget)(query, searchKind);
4728
+ if (direct) {
4729
+ hit = { kind: searchKind, id: direct, title: query };
4730
+ }
4731
+ else {
4732
+ try {
4733
+ if (kind === 'auto') {
4734
+ const channels = await (0, youtube_1.searchYouTube)(query, 'channel', { language: lang });
4735
+ hit = channels.find(h => (0, youtube_1.isNamedChannel)(h, query)) || null;
4736
+ alternatives = channels.filter(h => h !== hit).slice(0, 3);
4737
+ }
4738
+ if (!hit) {
4739
+ const hits = await (0, youtube_1.searchYouTube)(query, searchKind, { language: lang });
4740
+ hit = (0, youtube_1.pickBest)(hits, query);
4741
+ alternatives = hits.filter(h => h !== hit).slice(0, 3);
4742
+ }
4743
+ }
4744
+ catch (e) {
4745
+ this.log.warn(`YouTube search for "${query}" failed: ${e.message}`);
4746
+ return { ok: false, player: player.name, text: (0, youtube_1.youtubeReply)(lang, 'searchFailed') };
4747
+ }
4748
+ if (!hit) {
4749
+ return {
4750
+ ok: false,
4751
+ player: player.name,
4752
+ text: (0, youtube_1.youtubeReply)(lang, 'notFound', { kind: searchKind, query }),
4753
+ };
4754
+ }
4755
+ }
4756
+ const title = (0, youtube_1.spokenTitle)(hit);
4757
+ this.log.info(`YouTube: ${kind} "${query}" → ${hit.kind} ${hit.id} (${hit.title}) on ${player.name}.`);
4758
+ const reply = { kind: hit.kind, title, player: player.name };
4759
+ if ((await this.tvPower(player)) === 'off') {
4760
+ try {
4761
+ await this.switchTvOn(player);
4762
+ }
4763
+ catch (e) {
4764
+ this.log.warn(`YouTube: cannot switch ${player.power} on: ${e.message}`);
4765
+ }
4766
+ void this.playWhenTvIsOn(player, hit);
4767
+ return {
4768
+ ok: true,
4769
+ hit,
4770
+ alternatives,
4771
+ player: player.name,
4772
+ poweringOn: true,
4773
+ text: (0, youtube_1.youtubeReply)(lang, 'poweringOn', reply),
4774
+ };
4775
+ }
4776
+ if (await this.sendToYouTubePlayer(player, hit.id)) {
4777
+ return { ok: true, hit, alternatives, player: player.name, text: (0, youtube_1.youtubeReply)(lang, 'playing', reply) };
4778
+ }
4779
+ const instance = player.base.split('.').slice(0, 2).join('.');
4780
+ const alive = (await this.getForeignStateAsync(`system.adapter.${instance}.alive`).catch(() => null))?.val;
4781
+ return {
4782
+ ok: false,
4783
+ hit,
4784
+ player: player.name,
4785
+ text: alive ? (0, youtube_1.youtubeReply)(lang, 'noConfirm', reply) : (0, youtube_1.youtubeReply)(lang, 'adapterDown', { instance }),
4786
+ };
4787
+ }
4788
+ /**
4789
+ * Whether the TV of a player is on. `unknown` when there is nothing to tell it by: no power state, or
4790
+ * one that is a button — pressing a toggle button on a TV that is already on would switch it off.
4791
+ */
4792
+ async tvPower(player) {
4793
+ if (!player.power) {
4794
+ return 'unknown';
4795
+ }
4796
+ const obj = await this.getForeignObjectAsync(player.power).catch(() => null);
4797
+ if (!obj) {
4798
+ this.log.warn(`YouTube player ${player.name}: power state ${player.power} does not exist.`);
4799
+ return 'unknown';
4800
+ }
4801
+ const common = obj.common;
4802
+ if (common.read === false || String(common.role || '').startsWith('button')) {
4803
+ this.log.debug(`YouTube player ${player.name}: ${player.power} is a button, the TV state is unknown.`);
4804
+ return 'unknown';
4805
+ }
4806
+ const state = await this.getForeignStateAsync(player.power).catch(() => null);
4807
+ return (0, youtube_1.isPowerOn)(state?.val) ? 'on' : 'off';
4808
+ }
4809
+ /** Write "on" to the power state, in the type the state has. */
4810
+ async switchTvOn(player) {
4811
+ const obj = await this.getForeignObjectAsync(player.power);
4812
+ const type = obj?.common?.type;
4813
+ const val = type === 'number' ? 1 : type === 'string' ? 'on' : true;
4814
+ await this.setForeignStateAsync(player.power, { val, ack: false });
4815
+ this.journal?.stateAccess('set_state', { id: player.power, value: val });
4816
+ this.log.info(`YouTube: switching ${player.name} on (${player.power} = ${JSON.stringify(val)}).`);
4817
+ }
4818
+ /**
4819
+ * After the TV was switched on: wait until its power state confirms (an `ack` from the TV adapter — our
4820
+ * own write already says `true`), give the TV its boot time, then play. The YouTube app is often not
4821
+ * reachable over the lounge right after boot, so a command that is not acknowledged is sent once more.
4822
+ */
4823
+ async playWhenTvIsOn(player, hit) {
4824
+ try {
4825
+ const until = Date.now() + TV_POWER_TIMEOUT_MS;
4826
+ let on = false;
4827
+ while (!on && Date.now() < until) {
4828
+ await this.delay(1000);
4829
+ const st = await this.getForeignStateAsync(player.power).catch(() => null);
4830
+ on = !!st?.ack && (0, youtube_1.isPowerOn)(st.val);
4831
+ }
4832
+ if (!on) {
4833
+ this.log.warn(`YouTube: ${player.power} did not report "on" within ${TV_POWER_TIMEOUT_MS / 1000} s — trying to play anyway.`);
4834
+ }
4835
+ await this.delay(player.delaySec * 1000);
4836
+ let ok = await this.sendToYouTubePlayer(player, hit.id);
4837
+ if (!ok) {
4838
+ await this.delay(YOUTUBE_RETRY_MS);
4839
+ ok = await this.sendToYouTubePlayer(player, hit.id);
4840
+ }
4841
+ if (ok) {
4842
+ this.log.info(`YouTube: playing ${hit.id} (${hit.title}) on ${player.name}.`);
4843
+ }
4844
+ else {
4845
+ this.log.warn(`YouTube: ${player.name} did not take ${hit.id} after power-on. If the TV needs longer to start, raise its start delay.`);
4846
+ }
4847
+ }
4848
+ catch (e) {
4849
+ this.log.warn(`YouTube: playing on ${player.name} failed: ${e.message}`);
4850
+ }
4851
+ }
4852
+ /**
4853
+ * Hand a video/playlist/channel id to the player and wait for its acknowledgement: youtube-remote acks
4854
+ * `youtube` (back to '') only once the TV took the command, and leaves it unacknowledged on failure.
4855
+ */
4856
+ async sendToYouTubePlayer(player, id) {
4857
+ await this.setForeignStateAsync(player.target, { val: id, ack: false });
4858
+ this.journal?.stateAccess('set_state', { id: player.target, value: id });
4859
+ const until = Date.now() + YOUTUBE_ACK_TIMEOUT_MS;
4860
+ while (Date.now() < until) {
4861
+ await this.delay(250);
4862
+ if ((await this.getForeignStateAsync(player.target).catch(() => null))?.ack) {
4863
+ return true;
4864
+ }
4865
+ }
4866
+ return false;
4867
+ }
4868
+ /** A play result as tool / sendTo data: without the internal objects, with the alternatives as titles. */
4869
+ youtubeResultData(res) {
4870
+ return {
4871
+ ok: res.ok,
4872
+ text: res.text,
4873
+ ...(res.player ? { player: res.player } : {}),
4874
+ ...(res.hit ? { kind: res.hit.kind, id: res.hit.id, title: res.hit.title } : {}),
4875
+ ...(res.hit?.channel ? { channel: res.hit.channel } : {}),
4876
+ ...(res.poweringOn ? { poweringOn: true } : {}),
4877
+ ...(res.alternatives?.length
4878
+ ? { otherResults: res.alternatives.map(h => ({ id: h.id, title: h.title })) }
4879
+ : {}),
4880
+ };
4881
+ }
4882
+ /**
4883
+ * LLM tools for the YouTube players: `play_youtube` (search + power-on + play) and `youtube_player`
4884
+ * (pause, skip, volume, what is playing). The configured names go into the descriptions, the model
4885
+ * cannot discover them otherwise.
4886
+ */
4887
+ buildYouTubeTools() {
4888
+ const names = this.youtubePlayers.map((p, i) => (i === 0 ? `${p.name} (default)` : p.name)).join(', ');
4889
+ const playerParam = {
4890
+ type: 'string',
4891
+ description: `Player name or room: ${names}. Leave it out for the player in the room the user is in, else the default.`,
4892
+ };
4893
+ const play = {
4894
+ name: 'play_youtube',
4895
+ description: 'Play something from YouTube on the TV: finds the channel, video or playlist by name, switches ' +
4896
+ 'the TV on if it is off and starts playing. Use this for every "play … on YouTube", ' +
4897
+ '"spiele den Kanal …", "zeig das Video …" — never write the player states yourself. kind ' +
4898
+ '"channel" plays the newest uploads of a channel. The reply text of the result can be said as is; ' +
4899
+ 'otherResults are the next search hits, offer them if the user says it was the wrong one ' +
4900
+ '(pass the id as query to play one).',
4901
+ parameters: {
4902
+ type: 'object',
4903
+ properties: {
4904
+ query: {
4905
+ type: 'string',
4906
+ description: 'Name of the channel, title of the video/playlist, a search phrase, or an id/URL',
4907
+ },
4908
+ kind: {
4909
+ type: 'string',
4910
+ enum: ['channel', 'video', 'playlist'],
4911
+ description: 'Leave it out if the user did not say: a channel of that name wins, else videos',
4912
+ },
4913
+ player: playerParam,
4914
+ },
4915
+ required: ['query'],
4916
+ additionalProperties: false,
4917
+ },
4918
+ run: async (args) => {
4919
+ const query = typeof args.query === 'string' ? args.query.trim() : '';
4920
+ if (!query) {
4921
+ return JSON.stringify({ ok: false, error: 'no query given' });
4922
+ }
4923
+ const kind = args.kind === 'channel' || args.kind === 'playlist' || args.kind === 'video' ? args.kind : 'auto';
4924
+ const player = typeof args.player === 'string' ? args.player : undefined;
4925
+ return JSON.stringify(this.youtubeResultData(await this.playYouTube(query, kind, player)));
4926
+ },
4927
+ };
4928
+ const control = {
4929
+ name: 'youtube_player',
4930
+ description: 'Control a YouTube player or ask what it plays: action play (resume), pause, next, previous, ' +
4931
+ 'volume (with "volume" 0-100), mute, unmute, or status (title, channel, play state, TV power).',
4932
+ parameters: {
4933
+ type: 'object',
4934
+ properties: {
4935
+ action: {
4936
+ type: 'string',
4937
+ enum: ['play', 'pause', 'next', 'previous', 'volume', 'mute', 'unmute', 'status'],
4938
+ },
4939
+ volume: { type: 'number', description: 'Volume 0-100, only for action volume' },
4940
+ player: playerParam,
4941
+ },
4942
+ required: ['action'],
4943
+ additionalProperties: false,
4944
+ },
4945
+ run: async (args) => {
4946
+ const player = this.youtubePlayerFor(typeof args.player === 'string' ? args.player : undefined);
4947
+ if (!player) {
4948
+ return JSON.stringify({ ok: false, error: `unknown player, known are: ${names}` });
4949
+ }
4950
+ if (!(await this.resolveYouTubePlayer(player))) {
4951
+ return JSON.stringify({ ok: false, error: `no TV is paired with ${player.instance}` });
4952
+ }
4953
+ const action = typeof args.action === 'string' ? args.action : '';
4954
+ if (action === 'status') {
4955
+ const ids = ['title', 'artist', 'state', 'connected', 'volume'].map(s => `${player.base}.${s}`);
4956
+ const states = await this.getForeignStatesAsync(`${player.base}.*`).catch(() => ({}));
4957
+ const val = (i) => states[ids[i]]?.val ?? null;
4958
+ const power = await this.tvPower(player);
4959
+ return JSON.stringify({
4960
+ ok: true,
4961
+ data: {
4962
+ player: player.name,
4963
+ title: val(0),
4964
+ channel: val(1),
4965
+ state: { 0: 'paused', 1: 'playing', 2: 'stopped' }[String(val(2))],
4966
+ connected: val(3),
4967
+ volume: val(4),
4968
+ ...(power !== 'unknown' ? { tv: power } : {}),
4969
+ },
4970
+ });
4971
+ }
4972
+ if (!this.config.allowWriteStates) {
4973
+ return JSON.stringify({ ok: false, error: 'device control is disabled in the settings' });
4974
+ }
4975
+ let id;
4976
+ let value;
4977
+ if (action === 'volume') {
4978
+ const v = Number(args.volume);
4979
+ if (!Number.isFinite(v)) {
4980
+ return JSON.stringify({ ok: false, error: 'volume 0-100 needed' });
4981
+ }
4982
+ id = `${player.base}.volume`;
4983
+ value = Math.min(100, Math.max(0, Math.round(v)));
4984
+ }
4985
+ else if (action === 'mute' || action === 'unmute') {
4986
+ id = `${player.base}.muted`;
4987
+ value = action === 'mute';
4988
+ }
4989
+ else if (['play', 'pause', 'next', 'previous'].includes(action)) {
4990
+ id = `${player.base}.${action === 'previous' ? 'prev' : action}`;
4991
+ value = true;
4992
+ }
4993
+ else {
4994
+ return JSON.stringify({ ok: false, error: `unknown action '${action}'` });
4995
+ }
4996
+ await this.setForeignStateAsync(id, { val: value, ack: false });
4997
+ this.journal?.stateAccess('set_state', { id, value });
4998
+ return JSON.stringify({ ok: true, data: { player: player.name, [action]: value } });
4999
+ },
5000
+ };
5001
+ return [play, control];
5002
+ }
3941
5003
  // ── Presence: who is at home (roadmap B1) ───────────────────────────────
3942
5004
  /**
3943
5005
  * Build the presence tracker from the configured states, subscribe to them and mirror the result into
@@ -4295,12 +5357,26 @@ class Assistant extends adapter_core_1.Adapter {
4295
5357
  */
4296
5358
  async runRoutine(routine, source) {
4297
5359
  const origin = this.announceTargetForSource(source);
4298
- for (const action of routine.actions) {
5360
+ for (const [i, action] of routine.actions.entries()) {
5361
+ const start = Date.now();
4299
5362
  try {
4300
5363
  await this.runTriggerAction({ id: `routine:${routine.name}`, when: [] }, action, origin);
5364
+ this.journal?.step({
5365
+ stage: 'routine',
5366
+ title: `Routine action ${i + 1}/${routine.actions.length}`,
5367
+ ms: Date.now() - start,
5368
+ data: action,
5369
+ });
4301
5370
  }
4302
5371
  catch (e) {
4303
5372
  this.log.warn(`Routine '${routine.name}': an action failed — ${e.message}`);
5373
+ this.journal?.step({
5374
+ stage: 'routine',
5375
+ level: 'error',
5376
+ title: `Routine action ${i + 1}/${routine.actions.length} failed: ${e.message}`,
5377
+ ms: Date.now() - start,
5378
+ data: { action, error: (0, journal_1.errorInfo)(e) },
5379
+ });
4304
5380
  }
4305
5381
  }
4306
5382
  }