iobroker.assistant 1.1.0 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. package/README.md +74 -1
  2. package/admin/custom/.vite/manifest.json +62 -62
  3. package/admin/custom/assets/Components-x2Dfv8qE.js +2 -0
  4. package/admin/custom/assets/{_virtual_mf-localSharedImportMap___mfe_internal__ConfigCustomAssistant__mf_owner__1-A4gakomR.js → _virtual_mf-localSharedImportMap___mfe_internal__ConfigCustomAssistant__mf_owner__1-B4YbqhBb.js} +1 -1
  5. package/admin/custom/assets/{_virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__201375093093590__loadShare___mf_0_emotion_mf_1_styled__loadShare__.js-CwqmfMvn.js → _virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__163539842885036__loadShare___mf_0_emotion_mf_1_styled__loadShare__.js-IaosaymG.js} +1 -1
  6. package/admin/custom/assets/{_virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__201375093093590__loadShare___mf_0_iobroker_mf_1_gui_mf_2_components__loadShare__.js-BMuLYe6f.js → _virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__163539842885036__loadShare___mf_0_iobroker_mf_1_gui_mf_2_components__loadShare__.js-o4EU_Ut5.js} +39 -39
  7. package/admin/custom/assets/{_virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__201375093093590__loadShare__shared-BxjZmnpc.js → _virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__163539842885036__loadShare__shared--be2hWbV.js} +1 -1
  8. package/admin/custom/assets/_virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__163539842885036__prebuild___mf_0_iobroker_mf_1_json_mf_2_config__prebuild__-BsTz_EWE.js +122 -0
  9. package/admin/custom/assets/{hostInit-4roC4hEj.js → hostInit-Blh7a-Qe.js} +1 -1
  10. package/admin/custom/assets/{lib-CchzgTUs.js → lib-DBbNtolG.js} +1 -1
  11. package/admin/custom/assets/{mf-entry-bootstrap-0-4806bbe4.js → mf-entry-bootstrap-0-7abd30d3.js} +2 -2
  12. package/admin/custom/assets/{pendingShares-DLlv8yFY.js → pendingShares-U0D1MYK4.js} +1 -1
  13. package/admin/custom/customComponents.js +2 -2
  14. package/admin/custom/i18n/de.json +56 -0
  15. package/admin/custom/i18n/en.json +56 -0
  16. package/admin/custom/i18n/es.json +56 -0
  17. package/admin/custom/i18n/fr.json +56 -0
  18. package/admin/custom/i18n/it.json +56 -0
  19. package/admin/custom/i18n/nl.json +56 -0
  20. package/admin/custom/i18n/pl.json +56 -0
  21. package/admin/custom/i18n/pt.json +56 -0
  22. package/admin/custom/i18n/ru.json +56 -0
  23. package/admin/custom/i18n/uk.json +56 -0
  24. package/admin/custom/i18n/zh-cn.json +56 -0
  25. package/admin/custom/index.html +2 -2
  26. package/admin/custom/mf-manifest.json +1 -1
  27. package/admin/custom/mf-stats.json +1 -1
  28. package/admin/i18n/de.json +31 -4
  29. package/admin/i18n/en.json +31 -4
  30. package/admin/i18n/es.json +31 -4
  31. package/admin/i18n/fr.json +31 -4
  32. package/admin/i18n/it.json +31 -4
  33. package/admin/i18n/nl.json +31 -4
  34. package/admin/i18n/pl.json +31 -4
  35. package/admin/i18n/pt.json +31 -4
  36. package/admin/i18n/ru.json +31 -4
  37. package/admin/i18n/uk.json +31 -4
  38. package/admin/i18n/zh-cn.json +31 -4
  39. package/admin/jsonConfig.json +226 -15
  40. package/build/lib/credentials.js +11 -5
  41. package/build/lib/credentials.js.map +1 -1
  42. package/build/lib/journal.js +298 -2
  43. package/build/lib/journal.js.map +1 -1
  44. package/build/lib/llm.js +122 -7
  45. package/build/lib/llm.js.map +1 -1
  46. package/build/lib/localLlm.js +47 -10
  47. package/build/lib/localLlm.js.map +1 -1
  48. package/build/lib/nlu.js +372 -13
  49. package/build/lib/nlu.js.map +1 -1
  50. package/build/lib/piAgent.js +89 -0
  51. package/build/lib/piAgent.js.map +1 -0
  52. package/build/lib/systemAi.js +124 -0
  53. package/build/lib/systemAi.js.map +1 -0
  54. package/build/lib/tools.js +3 -0
  55. package/build/lib/tools.js.map +1 -1
  56. package/build/lib/voice/engines.js +31 -3
  57. package/build/lib/voice/engines.js.map +1 -1
  58. package/build/lib/voice/esphome.js +81 -7
  59. package/build/lib/voice/esphome.js.map +1 -1
  60. package/build/lib/voice/esphomeProto.js.map +1 -1
  61. package/build/lib/voice/mediaServer.js +29 -1
  62. package/build/lib/voice/mediaServer.js.map +1 -1
  63. package/build/lib/voice/openrouter.js +220 -0
  64. package/build/lib/voice/openrouter.js.map +1 -0
  65. package/build/lib/voice/protocol.js.map +1 -1
  66. package/build/lib/voice/voiceServer.js +81 -7
  67. package/build/lib/voice/voiceServer.js.map +1 -1
  68. package/build/lib/voice/wakeWords.js +284 -0
  69. package/build/lib/voice/wakeWords.js.map +1 -0
  70. package/build/lib/voice/wyoming.js +12 -4
  71. package/build/lib/voice/wyoming.js.map +1 -1
  72. package/build/lib/weather.js +160 -0
  73. package/build/lib/weather.js.map +1 -1
  74. package/build/main.js +1282 -126
  75. package/build/main.js.map +1 -1
  76. package/io-package.json +53 -41
  77. package/package.json +5 -5
  78. package/admin/custom/assets/Components-C6qcpaRC.js +0 -1
  79. package/admin/custom/assets/_virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__201375093093590__prebuild___mf_0_iobroker_mf_1_json_mf_2_config__prebuild__-TM4l0OYm.js +0 -122
  80. /package/admin/custom/assets/{_virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__201375093093590__loadShare___mf_0_emotion_mf_1_react__loadShare__.js-BkH7fzDw.js → _virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__163539842885036__loadShare___mf_0_emotion_mf_1_react__loadShare__.js-BkH7fzDw.js} +0 -0
  81. /package/admin/custom/assets/{_virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__201375093093590__prebuild__react_mf_1_jsx_mf_2_runtime__prebuild__-M61rjtmp.js → _virtual_mf___mfe_internal__ConfigCustomAssistant__mf_owner__163539842885036__prebuild__react_mf_1_jsx_mf_2_runtime__prebuild__-M61rjtmp.js} +0 -0
package/build/main.js CHANGED
@@ -63,17 +63,65 @@ const wyoming_1 = require("./lib/voice/wyoming");
63
63
  const esphome_1 = require("./lib/voice/esphome");
64
64
  const esphomeEntities_1 = require("./lib/voice/esphomeEntities");
65
65
  const mediaServer_1 = require("./lib/voice/mediaServer");
66
+ const wakeWords_1 = require("./lib/voice/wakeWords");
66
67
  const tone_1 = require("./lib/voice/tone");
67
68
  const engines_1 = require("./lib/voice/engines");
69
+ const openrouter_1 = require("./lib/voice/openrouter");
70
+ const piAgent_1 = require("./lib/piAgent");
71
+ const systemAi_1 = require("./lib/systemAi");
68
72
  /**
69
73
  * How long a rendered weather context line stays valid. Weather adapters refresh every few minutes at best,
70
74
  * so re-reading their whole state tree on every single request would be pure overhead.
71
75
  */
72
76
  const WEATHER_CTX_TTL = 5 * 60 * 1000;
77
+ /**
78
+ * How long the NLU device model stays valid. Building it costs a `list_devices` plus a name lookup per
79
+ * device, and one spoken command needs it up to three times (STT hints, NLU, the cloud device list).
80
+ * Enum and name changes drop it at once (see {@link ListCache.onClear}); the TTL only catches devices
81
+ * that were added or renamed elsewhere.
82
+ */
83
+ const NLU_DEVICES_TTL_MS = 60_000;
84
+ /** Default for `localLlmTimeoutSec`: after this the local model is stopped and the next tier answers. */
85
+ const LOCAL_LLM_TIMEOUT_SEC = 15;
86
+ /**
87
+ * Answer length cap of the local model. It answers small talk and general questions, mostly by voice, and
88
+ * on a Raspberry Pi every token costs about a fifth of a second — 512 tokens would be a minute and a half.
89
+ */
90
+ const LOCAL_LLM_MAX_TOKENS = 256;
73
91
  /**
74
92
  * The replies the assistant speaks verbatim, whatever the question was. They are pre-synthesised into
75
93
  * the TTS cache at startup; everything else is cached on first use.
76
94
  */
95
+ /** Fields of the Voice tab the voice test takes from the unsaved form (everything an STT or TTS engine is built from). */
96
+ const VOICE_FORM_FIELDS = [
97
+ 'voiceLanguage',
98
+ 'sttProvider',
99
+ 'sttModel',
100
+ 'openrouterSttModel',
101
+ 'voskModel',
102
+ 'ttsProvider',
103
+ 'ttsVoice',
104
+ 'ttsModel',
105
+ 'voiceCredentialType',
106
+ 'voiceApiKey',
107
+ 'voiceCredentialId',
108
+ 'openrouterSpeechKey',
109
+ 'openrouterCredentialId',
110
+ 'openrouterTtsModel',
111
+ 'openrouterVoice',
112
+ 'azureSpeechKey',
113
+ 'azureSpeechRegion',
114
+ 'azureCredentialId',
115
+ 'azureVoice',
116
+ 'awsAccessKeyId',
117
+ 'awsSecretAccessKey',
118
+ 'awsRegion',
119
+ 'awsCredentialId',
120
+ 'awsVoice',
121
+ 'piperVoice',
122
+ ];
123
+ /** Of those, the protected ones: the browser never gets them, so an empty value there means "keep the saved one" */
124
+ const VOICE_FORM_SECRETS = ['voiceApiKey', 'openrouterSpeechKey', 'azureSpeechKey', 'awsSecretAccessKey'];
77
125
  /** How long to wait for a device to confirm a write, and how often to look. */
78
126
  const ACK_TIMEOUT_MS = 2000;
79
127
  const ACK_POLL_MS = 150;
@@ -163,7 +211,37 @@ function iaqLabel(value, unit, lang) {
163
211
  return lang === 'ru' ? row[3] : lang === 'de' ? row[1] : row[2];
164
212
  }
165
213
  /** NLU actions that only switch something; their success is what the confirmation tone replaces. */
166
- const CONTROL_ACTIONS = new Set(['on', 'off', 'level', 'color']);
214
+ const CONTROL_ACTIONS = new Set(['on', 'off', 'level', 'color', 'volumeSet']);
215
+ /** An intent in one line for the request trace: "level Rollo (Büro) = 30", "timerSet 300 s", "alarmSet 7:05". */
216
+ function describeIntent(intent) {
217
+ let text = intent.action;
218
+ if (intent.device) {
219
+ text += ` ${intent.device.name}${intent.device.room ? ` (${intent.device.room})` : ''}`;
220
+ }
221
+ else if (intent.room) {
222
+ text += ` (${intent.room})`;
223
+ }
224
+ if (intent.value !== undefined) {
225
+ text += ` = ${JSON.stringify(intent.value)}`;
226
+ }
227
+ else if (intent.durationSec !== undefined) {
228
+ text += ` ${intent.durationSec} s`;
229
+ }
230
+ else if (intent.hour !== undefined) {
231
+ text += ` ${intent.hour}:${String(intent.minute ?? 0).padStart(2, '0')}`;
232
+ }
233
+ if (intent.delaySec !== undefined) {
234
+ text += ` in ${intent.delaySec} s`;
235
+ }
236
+ else if (intent.at) {
237
+ text += ` at ${intent.at.day ? `+${intent.at.day}d ` : ''}${intent.at.hour}:${String(intent.at.minute).padStart(2, '0')}`;
238
+ }
239
+ return text;
240
+ }
241
+ /** A device command for later ("in 2 Stunden", "um 22 Uhr") — goes into the calendar, not to the device */
242
+ function isDeferred(intent) {
243
+ return intent.delaySec !== undefined || !!intent.at;
244
+ }
167
245
  const FIXED_REPLIES = {
168
246
  de: ['Ok.', 'Erledigt.'],
169
247
  en: ['Okay.', 'Done.'],
@@ -193,6 +271,12 @@ function languageLabel(lang) {
193
271
  class Assistant extends adapter_core_1.Adapter {
194
272
  /** Cloud LLM; `null` with provider "none" (local tiers only) or before `onReady` finished. */
195
273
  agent = null;
274
+ /** Provider "pi-agent": the cloud tier is a pi-agent instance instead of {@link agent}. */
275
+ piAgent = null;
276
+ /** The cloud LLM of the instance (own or system-wide settings); `null` without one. */
277
+ llm = null;
278
+ /** What of `system.ai` this instance depends on, while it takes the system-wide AI settings. */
279
+ systemAiWatch = null;
196
280
  /** Set once `onReady` initialised the pipeline — with or without a cloud {@link agent}. */
197
281
  ready = false;
198
282
  mcp = null;
@@ -200,6 +284,8 @@ class Assistant extends adapter_core_1.Adapter {
200
284
  journal = null;
201
285
  /** Short-TTL cache for the expensive device/room/function listings; invalidated on enum changes. */
202
286
  listCache = null;
287
+ /** The NLU device model of {@link getNluDevices}, shared by all callers (and concurrent requests). */
288
+ nluDevices = null;
203
289
  /** Cache of a `<base>.SET` level state id → its writable boolean sibling switch (`<base>.ON_SET`/`.ON`), or ''. */
204
290
  siblingSwitch = new Map();
205
291
  /** Tier-1a local LLM (node-llama-cpp), lazily loaded when enabled + installed; null otherwise. */
@@ -216,6 +302,10 @@ class Assistant extends adapter_core_1.Adapter {
216
302
  esphome = null;
217
303
  /** Serves spoken replies to ESPHome satellites over HTTP; null unless those are enabled. */
218
304
  media = null;
305
+ /** Uploaded wake words (`assistant.0/wakewords/`), offered to every ESPHome satellite. */
306
+ customWakeWords = new Map();
307
+ /** Loads {@link customWakeWords} once from the file storage; later calls wait for the same load. */
308
+ customWakeWordsLoaded = null;
219
309
  /** Short-term per-source conversation memory (in-memory, TTL) for follow-up questions. */
220
310
  context = new context_1.ConversationStore();
221
311
  /** Questions the assistant asked and is waiting for an answer to (`askUser`), keyed by source. */
@@ -230,6 +320,8 @@ class Assistant extends adapter_core_1.Adapter {
230
320
  mirroredTimers = new Map();
231
321
  /** Per satellite: resolves once its control objects exist, so state writes can wait for them. */
232
322
  satControlsReady = new Map();
323
+ /** Satellites whose `volume` state exists and is subscribed (see updateSatelliteVolume). */
324
+ satVolumeEnsured = new Set();
233
325
  /** Native (ioBroker) satellite state-id → sender instance id, so we can push announcements back to it. */
234
326
  nativeSatFrom = new Map();
235
327
  /** Countdown timers / reminders (roadmap #2); mirrored into `timers.*` states. Null until onReady. */
@@ -287,15 +379,30 @@ class Assistant extends adapter_core_1.Adapter {
287
379
  // alarms, the local model) answer. The ioBroker tools are still built: the NLU switches through them.
288
380
  const provider = cfg.provider || 'openai';
289
381
  const localOnly = provider === 'none';
290
- const apiKey = localOnly ? '' : (0, llm_1.effectiveApiKey)(cfg.provider, await (0, credentials_1.resolveApiKey)(this, cfg));
382
+ // Provider "pi-agent": the adapter pi-agent is the cloud tier, with its own model, key and tools.
383
+ const harness = provider === 'pi-agent';
384
+ // Provider "system" (or no key of its own): provider, model and key of the system-wide AI settings.
385
+ if (!localOnly && !harness && (0, systemAi_1.usesSystemAi)(cfg)) {
386
+ await this.watchSystemAi();
387
+ }
388
+ const resolved = localOnly || harness ? null : await (0, systemAi_1.resolveLlm)(this, cfg);
389
+ this.llm = resolved && !('error' in resolved) ? resolved : null;
390
+ // speech of the same vendor reuses the key of the LLM
391
+ const llmKey = this.llm ? { provider: this.llm.provider, apiKey: this.llm.apiKey } : null;
291
392
  await this.setupJournal();
292
- if (!localOnly && !apiKey) {
293
- this.log.warn(cfg.credentialType === 'manager'
294
- ? 'No credential selected (manager mode) — pick an API-key credential in the adapter settings.'
295
- : 'No API key configured — open the adapter settings and enter your LLM API key.');
393
+ if (harness && !(cfg.piAgentInstance || '').trim()) {
394
+ this.log.warn('Provider "pi-agent" without an instance — choose the pi-agent instance in the adapter settings.');
395
+ await this.setStateAsync('info.connection', { val: false, ack: true });
396
+ return;
397
+ }
398
+ if (resolved && 'error' in resolved) {
399
+ this.log.warn(resolved.error);
296
400
  await this.setStateAsync('info.connection', { val: false, ack: true });
297
401
  return;
298
402
  }
403
+ if (this.llm?.system) {
404
+ this.log.info(`LLM from the system-wide AI settings: ${this.llm.provider} · ${this.llm.model || 'auto'}`);
405
+ }
299
406
  try {
300
407
  this.mcp = this.journalMcp(await (0, mcp_server_1.createInProcessMcp)({
301
408
  adapter: this,
@@ -304,6 +411,9 @@ class Assistant extends adapter_core_1.Adapter {
304
411
  allowObjectChange: cfg.allowObjectChange,
305
412
  }));
306
413
  this.listCache = new tools_1.ListCache();
414
+ this.listCache.onClear = () => {
415
+ this.nluDevices = null;
416
+ };
307
417
  const { tools, denied } = await (0, tools_1.buildMcpTools)(this.mcp, cfg, this.listCache, (id, fb) => this.resolveDeviceName(id, fb));
308
418
  // Let the cloud LLM handle timers/alarms too (for phrasings the rule-based NLU misses).
309
419
  tools.push(...this.buildTimerTools(), ...this.buildAlarmTools());
@@ -313,6 +423,9 @@ class Assistant extends adapter_core_1.Adapter {
313
423
  if ((cfg.weatherInstance || '').trim()) {
314
424
  tools.push(this.buildWeatherTool());
315
425
  }
426
+ if ((cfg.calendarInstance || '').trim()) {
427
+ tools.push(...this.buildCalendarTools());
428
+ }
316
429
  if (cfg.voiceEnabled) {
317
430
  // Only useful with satellites: without voice there is nothing to speak on.
318
431
  tools.push(this.buildAnnounceTool());
@@ -336,17 +449,20 @@ class Assistant extends adapter_core_1.Adapter {
336
449
  // Bust the device/room cache when room/function memberships change (new device shows within TTL anyway).
337
450
  this.subscribeForeignObjects('enum.rooms.*');
338
451
  this.subscribeForeignObjects('enum.functions.*');
339
- if (provider !== 'none') {
340
- const prov = (0, llm_1.resolveProvider)(provider, cfg.baseUrl);
452
+ if (harness) {
453
+ this.piAgent = new piAgent_1.PiAgentHarness(this, cfg.piAgentInstance.trim(), Math.max(5, Number(cfg.piAgentTimeoutSec) || piAgent_1.PI_AGENT_TIMEOUT_SEC) * 1000);
454
+ }
455
+ else if (this.llm) {
341
456
  this.agent = new llm_1.LlmAgent({
342
- provider,
343
- apiKey,
344
- model: cfg.model || prov.defaultModel,
345
- baseUrl: prov.baseUrl,
457
+ provider: this.llm.provider,
458
+ apiKey: this.llm.apiKey,
459
+ model: this.llm.model,
460
+ baseUrl: this.llm.baseUrl,
346
461
  systemPrompt: cfg.systemPrompt || '',
347
462
  maxTokens: cfg.maxTokens || 1024,
348
463
  tools: tools.map(t => this.journalTool(t)),
349
464
  log: this.log,
465
+ trace: step => this.journal?.step(step),
350
466
  });
351
467
  }
352
468
  }
@@ -391,8 +507,10 @@ class Assistant extends adapter_core_1.Adapter {
391
507
  await this.setupPresence();
392
508
  await this.setupTriggers();
393
509
  this.log.info(this.agent
394
- ? `Assistant ready (provider=${cfg.provider}, model=${this.agent.model || 'auto'}).`
395
- : 'Assistant ready without an LLM provider — answering locally only.');
510
+ ? `Assistant ready (provider=${this.llm?.provider}, model=${this.agent.model || 'auto'}).`
511
+ : this.piAgent
512
+ ? `Assistant ready (cloud tier: ${this.piAgent.instance}).`
513
+ : 'Assistant ready without an LLM provider — answering locally only.');
396
514
  if (cfg.useLocalLlm) {
397
515
  void this.ensureLocalLlm(); // background: model load/download must not block onReady
398
516
  }
@@ -401,20 +519,20 @@ class Assistant extends adapter_core_1.Adapter {
401
519
  // Subscribe regardless of the UDP server so writes are always handled (and logged if unroutable).
402
520
  this.subscribeStates('tts.text');
403
521
  if (cfg.udpServerEnabled) {
404
- await this.startVoiceServer(apiKey);
522
+ await this.startVoiceServer(llmKey);
405
523
  }
406
524
  }
407
525
  if (cfg.wyomingEnabled) {
408
- await this.startWyoming(apiKey);
526
+ await this.startWyoming(llmKey);
409
527
  }
410
528
  if (cfg.esphomeEnabled) {
411
- await this.startEsphome(apiKey);
529
+ await this.startEsphome(llmKey);
412
530
  }
413
531
  }
414
532
  /** Start the Wyoming TCP endpoint, bridging to the same STT/answer/TTS pipeline as the UDP server. */
415
- async startWyoming(mainApiKey) {
533
+ async startWyoming(llmKey) {
416
534
  const cfg = this.config;
417
- const creds = await (0, credentials_1.resolveVoiceCredentials)(this, cfg, mainApiKey);
535
+ const creds = await (0, credentials_1.resolveVoiceCredentials)(this, cfg, llmKey);
418
536
  const ctx = this.voiceContext(cfg, creds);
419
537
  const language = cfg.voiceLanguage || this.language || '';
420
538
  let stt;
@@ -434,7 +552,8 @@ class Assistant extends adapter_core_1.Adapter {
434
552
  language,
435
553
  stt,
436
554
  tts,
437
- answer: (question, sttMs) => this.answer(question, 'wyoming', { transport: 'wyoming', sttMs }),
555
+ answer: (question, sttMs, audioMs) => this.answer(question, 'wyoming', { transport: 'wyoming', sttMs, audioMs }),
556
+ onSpoken: info => this.reportSpoken('wyoming', info),
438
557
  getHints: () => this.buildSttHints(),
439
558
  log: this.log,
440
559
  });
@@ -454,14 +573,14 @@ class Assistant extends adapter_core_1.Adapter {
454
573
  * linux-voice-assistant). The adapter is the client here — the devices listen on TCP 6053 — and the
455
574
  * spoken reply is handed over as a URL, so the little media server comes up together with them.
456
575
  */
457
- async startEsphome(mainApiKey) {
576
+ async startEsphome(llmKey) {
458
577
  const cfg = this.config;
459
578
  const devices = (cfg.esphomeDevices || []).filter(d => d?.ip?.trim());
460
579
  if (!devices.length) {
461
580
  this.log.warn('ESPHome satellites are enabled, but no device is configured.');
462
581
  return;
463
582
  }
464
- const creds = await (0, credentials_1.resolveVoiceCredentials)(this, cfg, mainApiKey);
583
+ const creds = await (0, credentials_1.resolveVoiceCredentials)(this, cfg, llmKey);
465
584
  const ctx = this.voiceContext(cfg, creds);
466
585
  const language = cfg.voiceLanguage || this.language || '';
467
586
  let stt;
@@ -489,6 +608,10 @@ class Assistant extends adapter_core_1.Adapter {
489
608
  this.media = null;
490
609
  return;
491
610
  }
611
+ await this.loadCustomWakeWords();
612
+ for (const word of this.customWakeWords.values()) {
613
+ this.serveWakeWord(word);
614
+ }
492
615
  this.esphome = new esphome_1.EsphomeSatellites({
493
616
  devices,
494
617
  language,
@@ -499,7 +622,10 @@ class Assistant extends adapter_core_1.Adapter {
499
622
  room: origin.room,
500
623
  wakeWord: origin.wakeWord,
501
624
  sttMs: origin.sttMs,
625
+ audioMs: origin.audioMs,
626
+ vad: origin.vad,
502
627
  }),
628
+ onSpoken: (device, info) => this.reportSpoken(device, info),
503
629
  getHints: () => this.buildSttHints(),
504
630
  media: this.media,
505
631
  mediaHost: (cfg.esphomeMediaHost || '').trim(),
@@ -517,7 +643,15 @@ class Assistant extends adapter_core_1.Adapter {
517
643
  onEntityState: (device, room, entity) => {
518
644
  this.updateSatelliteControl(device, room, entity).catch(e => this.log.debug(`satellite control update failed: ${e.message}`));
519
645
  },
520
- expectsFollowUp: answer => this.expectsFollowUp(answer),
646
+ expectsFollowUp: (answer, device) => this.followUpFor(device, answer),
647
+ externalWakeWords: () => [...this.customWakeWords.values()].map(w => ({
648
+ id: w.id,
649
+ phrase: w.phrase,
650
+ languages: w.languages,
651
+ size: w.size,
652
+ sha256: w.sha256,
653
+ path: `${wakeWords_1.WAKE_WORD_URL_PREFIX}${w.id}.json`,
654
+ })),
521
655
  });
522
656
  await this.esphome.start();
523
657
  this.log.info(`ESPHome satellites: ${devices.length} device(s), STT=${cfg.sttProvider || 'openai'}, TTS=${cfg.ttsProvider || 'openai'}.`);
@@ -534,6 +668,7 @@ class Assistant extends adapter_core_1.Adapter {
534
668
  azure: (cfg.azureVoice || '').trim(),
535
669
  aws: (cfg.awsVoice || '').trim(),
536
670
  piper: (cfg.piperVoice || '').trim(),
671
+ openrouter: (cfg.openrouterVoice || '').trim(),
537
672
  },
538
673
  sttFallback: cfg.sttFallback || '',
539
674
  ttsFallback: cfg.ttsFallback || '',
@@ -542,6 +677,9 @@ class Assistant extends adapter_core_1.Adapter {
542
677
  voskModel: (cfg.voskModel || '').trim(),
543
678
  sttModel: (cfg.sttModel || '').trim(),
544
679
  ttsModel: (cfg.ttsModel || '').trim(),
680
+ openrouterSttModel: (cfg.openrouterSttModel || '').trim(),
681
+ openrouterTtsModel: (cfg.openrouterTtsModel || '').trim(),
682
+ decodeAudio: audio => this.decodeAudioBufferToPcm(audio),
545
683
  };
546
684
  }
547
685
  /**
@@ -549,9 +687,9 @@ class Assistant extends adapter_core_1.Adapter {
549
687
  * AWS cloud, or Vosk / Piper locally); credentials come from the encrypted config fields or the
550
688
  * central credential store (voiceCredentialType).
551
689
  */
552
- async startVoiceServer(mainApiKey) {
690
+ async startVoiceServer(llmKey) {
553
691
  const cfg = this.config;
554
- const creds = await (0, credentials_1.resolveVoiceCredentials)(this, cfg, mainApiKey);
692
+ const creds = await (0, credentials_1.resolveVoiceCredentials)(this, cfg, llmKey);
555
693
  const ctx = this.voiceContext(cfg, creds);
556
694
  // Raw language (e.g. 'de' or 'zh-cn') — each engine normalises it (ISO for cloud, model per lang for local).
557
695
  const language = cfg.voiceLanguage || this.language || '';
@@ -572,7 +710,17 @@ class Assistant extends adapter_core_1.Adapter {
572
710
  language,
573
711
  stt,
574
712
  tts,
575
- answer: (question, ctx) => this.answerVoice(question, ctx.device, { transport: 'udp', room: ctx.room, sttMs: ctx.sttMs }),
713
+ answer: (question, ctx) => this.answerVoice(question, ctx.device, {
714
+ transport: 'udp',
715
+ room: ctx.room,
716
+ sttMs: ctx.sttMs,
717
+ audioMs: ctx.audioMs,
718
+ }),
719
+ onSpoken: (device, info) => this.reportSpoken(device, info),
720
+ expectsFollowUp: (answer, device) => this.followUpFor(device, answer),
721
+ onVolume: (device, room, level) => {
722
+ this.updateSatelliteVolume(device, room, level).catch(e => this.log.debug(`satellite volume update failed: ${e.message}`));
723
+ },
576
724
  getHints: () => this.buildSttHints(),
577
725
  log: this.log,
578
726
  onStatus: (device, room, state) => {
@@ -639,6 +787,24 @@ class Assistant extends adapter_core_1.Adapter {
639
787
  expectsFollowUp(answer) {
640
788
  return /[??]/.test(answer || '');
641
789
  }
790
+ /** A transport synthesised the reply to `source`'s latest request — the journal shows it with that request. */
791
+ reportSpoken(source, info) {
792
+ const cfg = this.config;
793
+ const provider = cfg.ttsProvider || 'openai';
794
+ const voices = {
795
+ openai: cfg.ttsVoice || 'alloy',
796
+ azure: cfg.azureVoice,
797
+ aws: cfg.awsVoice,
798
+ piper: cfg.piperVoice,
799
+ openrouter: cfg.openrouterVoice,
800
+ };
801
+ this.journal?.spoken(source, {
802
+ ...info,
803
+ provider,
804
+ fallback: cfg.ttsFallback || undefined,
805
+ voice: (voices[provider] || '').trim() || undefined,
806
+ });
807
+ }
642
808
  /** Record the origin of the current text.request/response ('' = state write, 'chat' = message, else satellite). */
643
809
  setQuerySource(source) {
644
810
  this.setStateAsync('text.querySource', { val: source, ack: true }).catch(() => { });
@@ -650,8 +816,7 @@ class Assistant extends adapter_core_1.Adapter {
650
816
  return this.speechEngines;
651
817
  }
652
818
  const cfg = this.config;
653
- const mainKey = await (0, credentials_1.resolveApiKey)(this, cfg);
654
- const creds = await (0, credentials_1.resolveVoiceCredentials)(this, cfg, mainKey);
819
+ const creds = await (0, credentials_1.resolveVoiceCredentials)(this, cfg, await this.speechLlmKey());
655
820
  const ctx = this.voiceContext(cfg, creds);
656
821
  const language = cfg.voiceLanguage || this.language || '';
657
822
  try {
@@ -705,17 +870,18 @@ class Assistant extends adapter_core_1.Adapter {
705
870
  room,
706
871
  wakeWord: typeof msg.wakeWord === 'string' ? msg.wakeWord : undefined,
707
872
  sttMs: Date.now() - sttStart,
873
+ audioMs: Math.round((pcm.length / 2 / rate) * 1000),
708
874
  });
709
875
  if (!answer) {
710
876
  return { text, answer: '' };
711
877
  }
878
+ const ttsStart = Date.now();
712
879
  const reply = await engines.tts.synthesize(answer, language);
713
- // Follow-up: primarily the LLM's own signal ([[LISTEN]] → lastFollowUp); "?" heuristic as fallback.
714
- const llmSignalled = this.lastFollowUp;
715
- const listen = llmSignalled || this.expectsFollowUp(answer);
716
- if (listen) {
717
- this.log.info(`→ mic-on to satellite '${source}' (${llmSignalled ? 'LLM signalled a follow-up' : 'reply is a question (fallback)'}).`);
718
- }
880
+ this.reportSpoken(source, {
881
+ ms: Date.now() - ttsStart,
882
+ audioMs: (reply.pcm.length / 2 / reply.sampleRate) * 1000,
883
+ });
884
+ const listen = this.followUpFor(source, answer);
719
885
  return { text, answer, audio: reply.pcm.toString('base64'), sampleRate: reply.sampleRate, listen };
720
886
  }
721
887
  catch (e) {
@@ -738,8 +904,7 @@ class Assistant extends adapter_core_1.Adapter {
738
904
  const cfg = { ...this.config, ...overrides };
739
905
  const provider = msg.ttsProvider || cfg.ttsProvider || 'openai';
740
906
  const language = msg.language || cfg.voiceLanguage || this.language || '';
741
- const mainKey = await (0, credentials_1.resolveApiKey)(this, this.config);
742
- const creds = await (0, credentials_1.resolveVoiceCredentials)(this, cfg, mainKey);
907
+ const creds = await (0, credentials_1.resolveVoiceCredentials)(this, cfg, await this.speechLlmKey());
743
908
  return await (0, engines_1.listVoices)(provider, this.voiceContext(cfg, creds), language);
744
909
  }
745
910
  catch (e) {
@@ -778,6 +943,33 @@ class Assistant extends adapter_core_1.Adapter {
778
943
  await this.setStateAsync(`${base}.room`, { val: room, ack: true });
779
944
  }
780
945
  }
946
+ /**
947
+ * Mirror the playback volume a UDP satellite reported into `satellites.<id>.volume`. The state only
948
+ * appears for satellites that speak the `volume` message — an older one would never answer a write.
949
+ */
950
+ async updateSatelliteVolume(device, room, level) {
951
+ const id = await this.ensureSatelliteObjects(device, room);
952
+ const stateId = `satellites.${id}.volume`;
953
+ if (!this.satVolumeEnsured.has(id)) {
954
+ await this.setObjectNotExistsAsync(stateId, {
955
+ type: 'state',
956
+ common: {
957
+ name: 'Playback volume',
958
+ type: 'number',
959
+ role: 'level.volume',
960
+ unit: '%',
961
+ min: 0,
962
+ max: 100,
963
+ read: true,
964
+ write: true,
965
+ },
966
+ native: {},
967
+ });
968
+ this.subscribeStates(stateId);
969
+ this.satVolumeEnsured.add(id);
970
+ }
971
+ await this.setStateAsync(stateId, { val: level, ack: true });
972
+ }
781
973
  /**
782
974
  * Create the object tree for one satellite and return its (sanitised) state id, without touching
783
975
  * any value. Callers that only need somewhere to hang a child object use this rather than
@@ -913,11 +1105,111 @@ class Assistant extends adapter_core_1.Adapter {
913
1105
  this.satWakeWordStatesEnsured.add(id);
914
1106
  }
915
1107
  await this.setStateAsync(`${base}.wakeWords`, { val: config.active.join(','), ack: true });
1108
+ // `custom` marks the uploaded ones, so the settings dialog can offer to delete them.
1109
+ const words = config.available.map(w => (this.customWakeWords.has(w.id) ? { ...w, custom: true } : w));
916
1110
  await this.setStateAsync(`${base}.availableWakeWords`, {
917
- val: JSON.stringify({ max: config.max, words: config.available }),
1111
+ val: JSON.stringify({ max: config.max, words }),
918
1112
  ack: true,
919
1113
  });
920
1114
  }
1115
+ /** Read the uploaded wake words (`<instance>/wakewords/<id>.json` + `.tflite`) once. */
1116
+ loadCustomWakeWords() {
1117
+ this.customWakeWordsLoaded ||= (async () => {
1118
+ let entries = [];
1119
+ try {
1120
+ entries = await this.readDirAsync(this.namespace, wakeWords_1.WAKE_WORD_DIR);
1121
+ }
1122
+ catch {
1123
+ return; // nothing uploaded yet
1124
+ }
1125
+ for (const entry of entries) {
1126
+ const id = entry.file.replace(/\.json$/, '');
1127
+ if (entry.isDir || !entry.file.endsWith('.json') || !wakeWords_1.WAKE_WORD_ID_RE.test(id)) {
1128
+ continue;
1129
+ }
1130
+ try {
1131
+ const config = await this.readStoredFile(`${wakeWords_1.WAKE_WORD_DIR}/${id}.json`, 'utf8');
1132
+ const model = await this.readStoredFile(`${wakeWords_1.WAKE_WORD_DIR}/${id}.tflite`, 'binary');
1133
+ const word = (0, wakeWords_1.loadWakeWord)(id, config, model);
1134
+ this.customWakeWords.set(id, word);
1135
+ // Uploaded before the upload checked the model: keep it, so it can be deleted in the dialog.
1136
+ const problem = (0, wakeWords_1.microModelProblem)(model);
1137
+ if (problem) {
1138
+ this.log.warn(`Custom wake word ${word.phrase} (${id}) cannot run on the device: ${problem}. Delete it in the satellite settings.`);
1139
+ }
1140
+ }
1141
+ catch (e) {
1142
+ this.log.warn(`Custom wake word ${id} skipped: ${e.message}`);
1143
+ }
1144
+ }
1145
+ if (this.customWakeWords.size) {
1146
+ this.log.info(`Custom wake words: ${[...this.customWakeWords.values()].map(w => `${w.phrase} (${w.id})`).join(', ')}`);
1147
+ }
1148
+ })();
1149
+ return this.customWakeWordsLoaded;
1150
+ }
1151
+ /**
1152
+ * Read a file from this instance's file storage as a Buffer. Depending on the backend and the file's
1153
+ * type it comes back as a Buffer or a string — `encoding` says how to read the latter.
1154
+ */
1155
+ async readStoredFile(name, encoding) {
1156
+ const res = (await this.readFileAsync(this.namespace, name));
1157
+ const f = Buffer.isBuffer(res) || typeof res === 'string' ? res : res.file;
1158
+ if (f == null) {
1159
+ throw new Error(`${name} not found`);
1160
+ }
1161
+ return Buffer.isBuffer(f) ? f : Buffer.from(f, encoding);
1162
+ }
1163
+ serveWakeWord(word) {
1164
+ this.media?.serve(`${wakeWords_1.WAKE_WORD_URL_PREFIX}${word.id}.json`, word.config, 'application/json');
1165
+ this.media?.serve(`${wakeWords_1.WAKE_WORD_URL_PREFIX}${word.id}.tflite`, word.model, 'application/octet-stream');
1166
+ }
1167
+ /**
1168
+ * Store an uploaded wake word and offer it to the connected satellites. It shows up in their wake
1169
+ * word list right away; a device downloads the model only when it is selected there.
1170
+ */
1171
+ async addCustomWakeWord(message) {
1172
+ await this.loadCustomWakeWords();
1173
+ const text = (value) => (typeof value === 'string' ? value : undefined);
1174
+ const languages = (Array.isArray(message.languages)
1175
+ ? message.languages.map(String)
1176
+ : (text(message.languages) ?? '').split(/[,;\s]+/)).filter(Boolean);
1177
+ const word = (0, wakeWords_1.prepareWakeWord)({
1178
+ model: Buffer.from(text(message.model) ?? '', 'base64'),
1179
+ fileName: text(message.fileName),
1180
+ config: message.config && typeof message.config === 'object'
1181
+ ? message.config
1182
+ : text(message.config),
1183
+ phrase: text(message.phrase),
1184
+ languages,
1185
+ }, this.config.voiceLanguage || this.language || '');
1186
+ if (!this.customWakeWords.has(word.id)) {
1187
+ await this.writeFileAsync(this.namespace, `${wakeWords_1.WAKE_WORD_DIR}/${word.id}.tflite`, word.model);
1188
+ await this.writeFileAsync(this.namespace, `${wakeWords_1.WAKE_WORD_DIR}/${word.id}.json`, word.config);
1189
+ this.customWakeWords.set(word.id, word);
1190
+ this.serveWakeWord(word);
1191
+ this.log.info(`Custom wake word uploaded: ${word.phrase} (${word.id}, ${word.size} bytes)`);
1192
+ }
1193
+ const devices = this.esphome?.refreshWakeWords() ?? 0;
1194
+ return { id: word.id, phrase: word.phrase, languages: word.languages, size: word.size, devices };
1195
+ }
1196
+ /**
1197
+ * Remove an uploaded wake word: it is no longer offered or served. A device that already downloaded
1198
+ * it keeps its copy — the ESPHome API cannot delete a model from a device.
1199
+ */
1200
+ async deleteCustomWakeWord(id) {
1201
+ await this.loadCustomWakeWords();
1202
+ if (!wakeWords_1.WAKE_WORD_ID_RE.test(id) || !this.customWakeWords.has(id)) {
1203
+ throw new Error(`unknown wake word ${id}`);
1204
+ }
1205
+ for (const ext of ['json', 'tflite']) {
1206
+ await this.delFileAsync(this.namespace, `${wakeWords_1.WAKE_WORD_DIR}/${id}.${ext}`).catch(() => undefined);
1207
+ this.media?.unserve(`${wakeWords_1.WAKE_WORD_URL_PREFIX}${id}.${ext}`);
1208
+ }
1209
+ this.customWakeWords.delete(id);
1210
+ this.esphome?.refreshWakeWords();
1211
+ this.log.info(`Custom wake word deleted: ${id}`);
1212
+ }
921
1213
  /**
922
1214
  * Mirror everything an ESPHome satellite exposes besides the voice pipeline into
923
1215
  * `satellites.<id>.controls.*` — microphone gain and volume, noise suppression, wake-word and
@@ -1281,7 +1573,8 @@ class Assistant extends adapter_core_1.Adapter {
1281
1573
  await Promise.all(allowed.map(d => this.voice.announce(d, pcm, sampleRate)));
1282
1574
  delivered += allowed.length;
1283
1575
  if (listen) {
1284
- allowed.forEach(d => this.voice.listen(d));
1576
+ const playMs = (pcm.length / 2 / sampleRate) * 1000;
1577
+ allowed.forEach(d => this.voice.listen(d, playMs));
1285
1578
  }
1286
1579
  }
1287
1580
  catch (e) {
@@ -1526,7 +1819,36 @@ class Assistant extends adapter_core_1.Adapter {
1526
1819
  });
1527
1820
  }
1528
1821
  /** Invalidate the cached device/room/function listings when a room/function enum changes. */
1529
- onObjectChange(id, _obj) {
1822
+ /** Vendor and key of the cloud LLM: speech of the same vendor (OpenAI, OpenRouter) reuses the key. */
1823
+ async speechLlmKey() {
1824
+ const provider = this.config.provider || 'openai';
1825
+ const llm = this.llm ?? (provider === 'none' || provider === 'pi-agent' ? null : await (0, systemAi_1.resolveLlm)(this, this.config));
1826
+ return llm && !('error' in llm) ? { provider: llm.provider, apiKey: llm.apiKey } : null;
1827
+ }
1828
+ /**
1829
+ * Restart when the system-wide AI settings this instance takes change — also when they are still empty, so
1830
+ * setting them up in admin starts the instance. The credential is watched too: a new key needs no new id.
1831
+ */
1832
+ async watchSystemAi() {
1833
+ const native = await (0, systemAi_1.readSystemAi)(this);
1834
+ const credentialId = (0, systemAi_1.systemAiSelection)(native)?.credentialId || '';
1835
+ this.systemAiWatch = { fingerprint: (0, systemAi_1.systemAiFingerprint)(native), credentialId };
1836
+ await this.subscribeForeignObjectsAsync(systemAi_1.SYSTEM_AI_ID);
1837
+ if (credentialId) {
1838
+ await this.subscribeForeignObjectsAsync(credentialId);
1839
+ }
1840
+ }
1841
+ onObjectChange(id, obj) {
1842
+ if (this.systemAiWatch && (id === systemAi_1.SYSTEM_AI_ID || id === this.systemAiWatch.credentialId)) {
1843
+ // A credential has no fingerprint: a change of it is a new key.
1844
+ if (id === systemAi_1.SYSTEM_AI_ID && (0, systemAi_1.systemAiFingerprint)(obj?.native) === this.systemAiWatch.fingerprint) {
1845
+ return;
1846
+ }
1847
+ this.log.info(`The system-wide AI settings changed (${id}) — restarting.`);
1848
+ this.systemAiWatch = null;
1849
+ this.restart();
1850
+ return;
1851
+ }
1530
1852
  if (this.listCache && (id.startsWith('enum.rooms.') || id.startsWith('enum.functions.'))) {
1531
1853
  this.listCache.clear();
1532
1854
  this.log.debug(`list cache cleared (object changed: ${id})`);
@@ -1630,6 +1952,15 @@ class Assistant extends adapter_core_1.Adapter {
1630
1952
  await this.setStateAsync('dnd', { val: this.globalDnd, ack: true });
1631
1953
  return;
1632
1954
  }
1955
+ // Playback volume of a UDP satellite — not acked here: the satellite reports the level it took.
1956
+ const satVolume = id.match(/\.satellites\.([^.]+)\.volume$/);
1957
+ if (satVolume) {
1958
+ const device = this.satDeviceById.get(satVolume[1]) || satVolume[1];
1959
+ if (!this.voice?.volume(device, { level: Number(state.val) })) {
1960
+ this.log.warn(`Cannot set the volume of '${device}': it is not a registered UDP satellite.`);
1961
+ }
1962
+ return;
1963
+ }
1633
1964
  const satDnd = id.match(/\.satellites\.([^.]+)\.dnd$/);
1634
1965
  if (satDnd) {
1635
1966
  this.dndById.set(satDnd[1], !!state.val);
@@ -1775,14 +2106,23 @@ class Assistant extends adapter_core_1.Adapter {
1775
2106
  }
1776
2107
  return;
1777
2108
  }
1778
- // STT model dropdown (Vosk): suggest model names for the selected language.
1779
- if (obj.command === 'getSttModels') {
2109
+ // STT/TTS model dropdowns (Vosk, OpenRouter): suggest model names for the selected provider.
2110
+ if (obj.command === 'getSttModels' || obj.command === 'getTtsModels') {
1780
2111
  const msg = (obj.message || {});
1781
2112
  const cfg = this.config;
1782
- const provider = msg.sttProvider || cfg.sttProvider || 'openai';
1783
2113
  const language = msg.language || cfg.voiceLanguage || this.language || '';
2114
+ let models = [];
2115
+ try {
2116
+ models =
2117
+ obj.command === 'getSttModels'
2118
+ ? await (0, engines_1.listSttModels)(msg.sttProvider || cfg.sttProvider || 'openai', language)
2119
+ : await (0, engines_1.listTtsModels)(msg.ttsProvider || cfg.ttsProvider || 'openai');
2120
+ }
2121
+ catch (e) {
2122
+ this.log.warn(`${obj.command} failed: ${e.message}`);
2123
+ }
1784
2124
  if (obj.callback) {
1785
- this.sendTo(obj.from, obj.command, (0, engines_1.listSttModels)(provider, language), obj.callback);
2125
+ this.sendTo(obj.from, obj.command, models, obj.callback);
1786
2126
  }
1787
2127
  return;
1788
2128
  }
@@ -2014,6 +2354,41 @@ class Assistant extends adapter_core_1.Adapter {
2014
2354
  }
2015
2355
  return;
2016
2356
  }
2357
+ // Settings dialog / scripts: own wake words for the ESPHome satellites (microWakeWord models).
2358
+ // uploadWakeWord: { model: base64 .tflite, fileName?, config?: JSON text, phrase?, languages? }
2359
+ // → { ok, id, phrase, languages, size, devices } | { error }
2360
+ // deleteWakeWord: { id } → { ok } | { error }; listWakeWords → { wakeWords: [{id, phrase, …}] }
2361
+ if (obj.command === 'uploadWakeWord' || obj.command === 'deleteWakeWord' || obj.command === 'listWakeWords') {
2362
+ const m = (obj.message || {});
2363
+ let payload;
2364
+ try {
2365
+ if (obj.command === 'uploadWakeWord') {
2366
+ payload = { ok: true, ...(await this.addCustomWakeWord(m)) };
2367
+ }
2368
+ else if (obj.command === 'deleteWakeWord') {
2369
+ await this.deleteCustomWakeWord(typeof m.id === 'string' ? m.id : '');
2370
+ payload = { ok: true };
2371
+ }
2372
+ else {
2373
+ await this.loadCustomWakeWords();
2374
+ payload = {
2375
+ wakeWords: [...this.customWakeWords.values()].map(w => ({
2376
+ id: w.id,
2377
+ phrase: w.phrase,
2378
+ languages: w.languages,
2379
+ size: w.size,
2380
+ })),
2381
+ };
2382
+ }
2383
+ }
2384
+ catch (e) {
2385
+ payload = { error: e.message };
2386
+ }
2387
+ if (obj.callback) {
2388
+ this.sendTo(obj.from, obj.command, payload, obj.callback);
2389
+ }
2390
+ return;
2391
+ }
2017
2392
  // Scripts: everything an ESPHome satellite exposes besides the voice pipeline.
2018
2393
  // message = { device? }; without a device, every connected one is listed.
2019
2394
  if (obj.command === 'getControls') {
@@ -2122,10 +2497,20 @@ class Assistant extends adapter_core_1.Adapter {
2122
2497
  }
2123
2498
  return;
2124
2499
  }
2125
- // Chat: synthesize an answer to speech (WAV) via the configured TTS engine.
2500
+ // Chat and the voice test of the settings: synthesize to speech (WAV) via the configured TTS engine,
2501
+ // or via the unsaved settings of the Voice tab (`config`).
2126
2502
  if (obj.command === 'tts') {
2127
- const { text, language } = (obj.message || {});
2128
- const result = await this.synthesizeToWav(String(text ?? ''), language);
2503
+ const { text, language, config } = (obj.message || {});
2504
+ const cfg = config ? this.withVoiceForm(config) : this.config;
2505
+ const result = await this.synthesizeToWav(String(text ?? ''), language, cfg);
2506
+ if (obj.callback) {
2507
+ this.sendTo(obj.from, obj.command, result, obj.callback);
2508
+ }
2509
+ return;
2510
+ }
2511
+ // Voice test of the settings: recognize a recording with the (unsaved) speech-to-text settings.
2512
+ if (obj.command === 'stt') {
2513
+ const result = await this.transcribeForTest((obj.message || {}));
2129
2514
  if (obj.callback) {
2130
2515
  this.sendTo(obj.from, obj.command, result, obj.callback);
2131
2516
  }
@@ -2220,6 +2605,18 @@ class Assistant extends adapter_core_1.Adapter {
2220
2605
  }
2221
2606
  return;
2222
2607
  }
2608
+ // Admin chat debug view: one request with every step of its processing (message = { id }).
2609
+ if (obj.command === 'getJournalTrace') {
2610
+ const id = String(obj.message?.id || '');
2611
+ const entry = this.journal?.get(id);
2612
+ const result = entry
2613
+ ? { entry, steps: await this.journal.trace(id) }
2614
+ : { error: this.journal?.enabled ? 'unknown request' : 'journal disabled' };
2615
+ if (obj.callback) {
2616
+ this.sendTo(obj.from, obj.command, result, obj.callback);
2617
+ }
2618
+ return;
2619
+ }
2223
2620
  if (obj.command === 'clearJournal') {
2224
2621
  this.journal?.clear();
2225
2622
  if (obj.callback) {
@@ -2271,28 +2668,126 @@ class Assistant extends adapter_core_1.Adapter {
2271
2668
  */
2272
2669
  answer(question, source = '', origin = {}) {
2273
2670
  return this.requestOrigin.run({ source, room: origin.room || '' }, () => this.journal
2274
- ? this.journal.run(question, source, origin, () => this.answerTracked(question, source))
2671
+ ? this.journal.run(question, source, origin, () => {
2672
+ this.traceSpeechInput(question, origin);
2673
+ return this.answerTracked(question, source);
2674
+ })
2275
2675
  : this.answerTracked(question, source));
2276
2676
  }
2677
+ /** Speech recognition happened before the request: put it at the head of its trace. */
2678
+ traceSpeechInput(question, origin) {
2679
+ if (origin.sttMs === undefined) {
2680
+ return;
2681
+ }
2682
+ const cfg = this.config;
2683
+ const provider = cfg.sttProvider || 'openai';
2684
+ const audio = origin.audioMs !== undefined ? `, ${(origin.audioMs / 1000).toFixed(1)} s of audio` : '';
2685
+ this.journal?.step({
2686
+ stage: 'stt',
2687
+ title: `Speech recognition: ${provider}${audio}`,
2688
+ ms: origin.sttMs,
2689
+ data: {
2690
+ provider,
2691
+ fallback: cfg.sttFallback || undefined,
2692
+ model: (provider === 'openrouter' ? cfg.openrouterSttModel : cfg.sttModel || cfg.voskModel || '').trim() ||
2693
+ undefined,
2694
+ language: cfg.voiceLanguage || this.language || '',
2695
+ audioMs: origin.audioMs,
2696
+ wakeWord: origin.wakeWord,
2697
+ vad: origin.vad,
2698
+ transcript: question,
2699
+ },
2700
+ });
2701
+ }
2277
2702
  async answerTracked(question, source) {
2278
2703
  const useCtx = this.config.useConversationContext !== false;
2279
2704
  const history = useCtx ? this.context.get(source) : [];
2705
+ if (history.length) {
2706
+ this.journal?.step({
2707
+ stage: 'context',
2708
+ title: `Conversation context: ${history.length} earlier turn(s) of '${source}'`,
2709
+ data: history,
2710
+ });
2711
+ }
2280
2712
  const raw = await this.produceAnswer(question, history, source);
2281
2713
  // The LLM signals "I'm waiting for the user's reply" by appending the [[LISTEN]] control marker
2282
- // (see followUpHint). Strip it here so it's neither stored, shown nor spoken, and expose it as the
2283
- // follow-up flag. Single-user voice → a shared field is race-free enough (read right after await).
2284
- this.lastFollowUp = /\[\[LISTEN\]\]/i.test(raw);
2285
- if (this.lastFollowUp) {
2714
+ // (see followUpHint). Strip it here so it's neither stored, shown nor spoken, and keep it per source:
2715
+ // two satellites answered at the same time must not hand each other their mic-on.
2716
+ const followUp = /\[\[LISTEN\]\]/i.test(raw);
2717
+ this.followUps.set(source, followUp);
2718
+ if (followUp) {
2286
2719
  this.journal?.note({ followUp: true });
2287
2720
  }
2288
- const result = raw.replace(/\s*\[\[LISTEN\]\]\s*/gi, ' ').trim();
2289
- if (useCtx && result) {
2290
- this.context.add(source, question, result);
2721
+ let result = raw.replace(/\s*\[\[(?:LISTEN|DONE)\]\]\s*/gi, ' ').trim();
2722
+ // The LLM answers a switch command that worked with [[DONE]] alone (see doneHint): a beep on the satellite
2723
+ // instead of a sentence. Where nothing can beep (chat, Telegram), a short word. With more text, that is said.
2724
+ let toned = false;
2725
+ if (!result && /\[\[DONE\]\]/i.test(raw)) {
2726
+ toned = await this.playConfirmationTone(source);
2727
+ if (!toned) {
2728
+ result = this.doneText();
2729
+ }
2730
+ }
2731
+ this.journal?.step({
2732
+ stage: 'answer',
2733
+ title: toned
2734
+ ? 'Confirmed with a tone instead of words'
2735
+ : result
2736
+ ? `Answer: ${result.length} characters${followUp ? ', waits for a reply' : ''}`
2737
+ : 'No spoken answer',
2738
+ data: { answer: result, raw: raw !== result ? raw : undefined, followUp, tone: toned || undefined },
2739
+ });
2740
+ if (useCtx && (result || toned)) {
2741
+ this.context.add(source, question, result || this.doneText());
2291
2742
  }
2292
2743
  return result;
2293
2744
  }
2294
- /** Whether the LAST {@link answer} asked the user something (LLM-signalled via [[LISTEN]]). */
2295
- lastFollowUp = false;
2745
+ /**
2746
+ * System-prompt instruction for `confirmWithTone`: a switch command that worked is answered with [[DONE]]
2747
+ * alone, which becomes a beep on the satellite. Empty when the option (or voice) is off.
2748
+ */
2749
+ doneHint() {
2750
+ if (!this.config.confirmWithTone || !this.config.voiceEnabled) {
2751
+ return '';
2752
+ }
2753
+ const lang = String(this.config.voiceLanguage || this.language || 'en');
2754
+ if (lang === 'ru') {
2755
+ return 'Если запрос был только командой включить, выключить или установить устройства на значение и всё получилось, ответь только управляющим маркером [[DONE]] (вместо фразы прозвучит короткий сигнал). На вопросы, при ошибках или если нужно что-то сообщить, отвечай как обычно.';
2756
+ }
2757
+ if (lang === 'de') {
2758
+ return 'Wenn die Anfrage nur ein Befehl war, Geräte ein- oder auszuschalten oder auf einen Wert zu stellen, und alles geklappt hat, antworte ausschließlich mit dem Steuer-Marker [[DONE]] (statt eines Satzes ertönt ein kurzer Bestätigungston). Bei Fragen, bei Fehlern oder wenn du etwas mitteilen musst, antworte normal.';
2759
+ }
2760
+ return 'If the request was only a command to switch devices on or off or to set them to a value, and everything worked, answer with the control marker [[DONE]] alone (a short beep plays instead of a sentence). For questions, errors or when you have something to tell, answer normally.';
2761
+ }
2762
+ /** What [[DONE]] says where no beep can be played */
2763
+ doneText() {
2764
+ const lang = String(this.config.voiceLanguage || this.language || 'en');
2765
+ return lang === 'ru' ? 'Готово.' : lang === 'de' ? 'Erledigt.' : 'Done.';
2766
+ }
2767
+ /** Per source: whether its last {@link answer} asked the user something (LLM-signalled via [[LISTEN]]). */
2768
+ followUps = new Map();
2769
+ /**
2770
+ * Should the satellite behind `source` re-open its mic after this reply? Primarily the LLM's own
2771
+ * [[LISTEN]] signal, with a question mark in the reply as the fallback (the NLU and the local tiers
2772
+ * cannot emit the marker, but they do ask "Meinst du Büro oder Wohnzimmer?").
2773
+ */
2774
+ wantsFollowUp(source, answer) {
2775
+ if (!answer) {
2776
+ return null;
2777
+ }
2778
+ if (this.followUps.get(source)) {
2779
+ return 'llm';
2780
+ }
2781
+ return this.expectsFollowUp(answer) ? 'question' : null;
2782
+ }
2783
+ /** {@link wantsFollowUp} for a voice transport, logged, as the `listen` flag of the reply. */
2784
+ followUpFor(source, answer) {
2785
+ const why = this.wantsFollowUp(source, answer);
2786
+ if (why) {
2787
+ this.log.info(`→ mic-on to satellite '${source}' (${why === 'llm' ? 'LLM signalled a follow-up' : 'reply is a question'}).`);
2788
+ }
2789
+ return !!why;
2790
+ }
2296
2791
  /**
2297
2792
  * System-prompt instruction letting the LLM explicitly signal a follow-up: append [[LISTEN]] when it
2298
2793
  * asks the user something and waits for an answer. More reliable than a "?" heuristic and lets a
@@ -2369,6 +2864,7 @@ class Assistant extends adapter_core_1.Adapter {
2369
2864
  const n = this.stopRinging();
2370
2865
  this.log.info(`Ring stopped by voice (${n} session(s)).`);
2371
2866
  this.journal?.note({ tier: 'stop' });
2867
+ this.journal?.step({ stage: 'stop', title: `Stop word: silenced ${n} ringing timer/alarm(s)` });
2372
2868
  const lang = String(this.config.voiceLanguage || this.language || 'en');
2373
2869
  return lang === 'ru' ? 'Хорошо.' : lang === 'de' ? 'Ok.' : 'Okay.';
2374
2870
  }
@@ -2380,6 +2876,11 @@ class Assistant extends adapter_core_1.Adapter {
2380
2876
  if (this.pending.deliver(source, question)) {
2381
2877
  this.log.info(`Answer to "${asked}" from source '${source || 'any'}': ${question}`);
2382
2878
  this.journal?.note({ tier: 'askReply', detail: asked || undefined });
2879
+ this.journal?.step({
2880
+ stage: 'askReply',
2881
+ title: `Reply to the open question "${asked}"`,
2882
+ data: { question: asked, reply: question },
2883
+ });
2383
2884
  return '';
2384
2885
  }
2385
2886
  }
@@ -2390,9 +2891,17 @@ class Assistant extends adapter_core_1.Adapter {
2390
2891
  if (routine) {
2391
2892
  this.log.info(`Routine '${routine.name}' triggered (source='${source}').`);
2392
2893
  this.journal?.note({ tier: 'routine', detail: routine.name });
2894
+ this.journal?.step({ stage: 'routine', title: `Routine "${routine.name}" matched`, data: routine });
2393
2895
  await this.runRoutine(routine, source);
2394
2896
  return routine.reply;
2395
2897
  }
2898
+ if (this.routines.length) {
2899
+ this.journal?.step({
2900
+ stage: 'routine',
2901
+ title: `No routine matched (${this.routines.length} configured)`,
2902
+ data: { phrases: this.routines.map(r => ({ name: r.name, phrases: r.phrases })) },
2903
+ });
2904
+ }
2396
2905
  // Tier 0: rule-based NLU (device commands) — fastest, offline, free.
2397
2906
  if (this.config.useLocalNlu) {
2398
2907
  // "Spiele den Kanal X ab" first: the device NLU would find the TV in "… auf dem Fernseher" and
@@ -2401,9 +2910,19 @@ class Assistant extends adapter_core_1.Adapter {
2401
2910
  if (yt) {
2402
2911
  this.log.info(`YouTube request understood locally: ${yt.kind} "${yt.query}" (source='${source}').`);
2403
2912
  this.journal?.note({ tier: 'nlu', detail: `youtube ${yt.kind}` });
2913
+ const start = Date.now();
2404
2914
  const res = await this.playYouTube(yt.query, yt.kind, yt.player);
2915
+ this.journal?.step({
2916
+ stage: 'youtube',
2917
+ title: `YouTube ${yt.kind} "${yt.query}"${yt.player ? ` on "${yt.player}"` : ''}`,
2918
+ ms: Date.now() - start,
2919
+ data: { parsed: yt, result: res },
2920
+ });
2405
2921
  return res.text;
2406
2922
  }
2923
+ if (this.youtubePlayers.length) {
2924
+ this.journal?.step({ stage: 'youtube', title: 'Not a YouTube command' });
2925
+ }
2407
2926
  try {
2408
2927
  const handled = await this.tryLocalNlu(question, source);
2409
2928
  if (handled !== null) {
@@ -2413,61 +2932,205 @@ class Assistant extends adapter_core_1.Adapter {
2413
2932
  }
2414
2933
  catch (e) {
2415
2934
  this.log.debug(`NLU skipped: ${e.message}`);
2935
+ this.journal?.step({
2936
+ stage: 'nlu',
2937
+ level: 'warn',
2938
+ title: `NLU gave up, the request goes on: ${e.message}`,
2939
+ data: (0, journal_1.errorInfo)(e),
2940
+ });
2416
2941
  }
2417
2942
  }
2943
+ else {
2944
+ this.journal?.step({ stage: 'nlu', title: 'Rule-based NLU is switched off (useLocalNlu)' });
2945
+ }
2418
2946
  // Current weather from the configured adapter — prepended to the user turn of BOTH LLM tiers so
2419
2947
  // "how's the weather?" is answered from real data (the local model has no tools and would otherwise
2420
2948
  // invent a forecast; the cloud model saves a get_weather round-trip). Cached, see buildWeatherContext.
2421
2949
  const weather = await this.buildWeatherContext();
2422
2950
  // Tier 1a: local LLM — answers general questions offline; emits HANDOFF for anything needing tools.
2951
+ // With a cloud LLM it only goes first when asked to (`localLlmFirst`): on a Raspberry Pi a small model
2952
+ // needs 10–25 s, often only to hand off, and the cloud then starts from zero. By default it is the
2953
+ // fallback for when the cloud fails (see below).
2423
2954
  // Skipped inside an ongoing conversation (history present): the tool-free local model gets no history,
2424
2955
  // so it would answer a follow-up like "yes" context-free and swallow it — let the cloud LLM (which has
2425
2956
  // the thread and the tools) handle follow-ups instead. Without a cloud LLM there is nobody to hand
2426
2957
  // follow-ups to, so the local model takes them too.
2427
- if (this.config.useLocalLlm && this.localLlm && (!history.length || !this.agent)) {
2428
- try {
2429
- // The local model has no tools: without these lines it would invent the weather and guess
2430
- // who is at home.
2431
- const facts = [weather, this.buildPresenceContext()].filter(Boolean).join('\n\n');
2432
- const ans = await this.localLlm.ask(facts ? `${facts}\n\n${question}` : question);
2433
- if (ans && !(0, localLlm_1.isHandoff)(ans)) {
2434
- this.log.info(`Answered by local LLM (source='${source}').`);
2435
- this.journal?.note({ tier: 'localLlm', detail: this.localLlmName() });
2436
- return ans;
2437
- }
2438
- this.log.debug('Local LLM handed off to the cloud LLM.');
2439
- }
2440
- catch (e) {
2441
- this.log.debug(`Local LLM error: ${e.message}`);
2958
+ const useLocal = !!(this.config.useLocalLlm && this.localLlm);
2959
+ const cloud = !!(this.agent || this.piAgent);
2960
+ const localFirst = useLocal && (!cloud || !!this.config.localLlmFirst);
2961
+ if (localFirst && (!history.length || !cloud)) {
2962
+ const ans = await this.askLocalLlm(question, weather, source);
2963
+ if (ans) {
2964
+ return ans;
2442
2965
  }
2443
2966
  }
2444
- else if (this.config.useLocalLlm && this.localLlm) {
2967
+ else if (localFirst) {
2445
2968
  this.log.debug('Local LLM skipped (follow-up in an ongoing conversation → cloud LLM).');
2969
+ this.journal?.step({
2970
+ stage: 'localLlm',
2971
+ title: 'Local LLM skipped: follow-up in an ongoing conversation goes to the cloud LLM',
2972
+ });
2973
+ }
2974
+ else if (useLocal) {
2975
+ this.journal?.step({
2976
+ stage: 'localLlm',
2977
+ title: 'Local LLM not asked first: it is the fallback for when the cloud LLM fails (localLlmFirst)',
2978
+ });
2446
2979
  }
2447
2980
  // Tier 2: cloud LLM with full tool access. Inject a compact device list into the system prompt so
2448
2981
  // the model can act without a first `list_devices` round-trip (kept cached via prompt caching).
2982
+ if (this.piAgent) {
2983
+ return this.askPiAgent(this.piAgent, question, weather, source, useLocal && !localFirst);
2984
+ }
2449
2985
  if (!this.agent) {
2450
2986
  // Provider "none": no tier further down. Say so instead of failing — over voice an error is
2451
2987
  // read out as a technical message.
2452
2988
  this.log.info(`No local tier could answer and no LLM provider is configured (source='${source}').`);
2453
2989
  this.journal?.note({ tier: 'none' });
2990
+ this.journal?.step({
2991
+ stage: 'none',
2992
+ level: 'warn',
2993
+ title: 'Nothing local understood the request and no LLM provider is configured',
2994
+ });
2454
2995
  return this.localOnlyFallback();
2455
2996
  }
2456
2997
  const ctx = await this.buildDeviceContext();
2457
2998
  const mem = this.buildMemoryContext();
2458
2999
  // Only ask the model to emit the follow-up marker when satellites can actually re-open the mic.
2459
3000
  const followUp = this.config.voiceEnabled ? this.followUpHint() : '';
2460
- const parts = [this.config.systemPrompt || '', followUp, mem, ctx].filter(Boolean);
3001
+ const done = this.doneHint();
3002
+ const parts = [this.config.systemPrompt || '', followUp, done, mem, ctx].filter(Boolean);
2461
3003
  const sys = parts.length ? parts.join('\n\n') : undefined;
2462
3004
  this.log.info(`Cloud LLM [${this.agent.model || 'auto'}] (source='${source}', context=${history.length} turn(s)${mem ? ', memory' : ''}${weather ? ', weather' : ''}${this.presence?.configured ? ', presence' : ''}) — running tool loop…`);
2463
3005
  // Prepend the current date/time (and the weather line, if a source is configured) to the user turn —
2464
3006
  // cache-safe, see buildTimeContext. The stored history keeps the untouched question.
2465
- const questionForLlm = [this.buildTimeContext(), weather, question].filter(Boolean).join('\n\n');
3007
+ const time = this.buildTimeContext();
3008
+ const questionForLlm = [time, weather, question].filter(Boolean).join('\n\n');
2466
3009
  this.journal?.note({
2467
3010
  tier: 'cloud',
2468
- detail: `${this.config.provider || 'openai'} · ${this.agent.model || 'auto'}`,
3011
+ detail: `${this.llm?.provider || this.config.provider || 'openai'} · ${this.agent.model || 'auto'}`,
3012
+ });
3013
+ // What went into the prompt, piece by piece. The device list is only counted: the model's opening
3014
+ // step carries the whole system prompt, and with it the list.
3015
+ const deviceLines = ctx ? ctx.split('\n').length - 1 : 0;
3016
+ this.journal?.step({
3017
+ stage: 'context',
3018
+ title: `Prompt: ${deviceLines} devices${mem ? ', memory' : ''}${weather ? ', weather' : ''}${followUp ? ', follow-up marker' : ''}`,
3019
+ data: {
3020
+ systemPrompt: this.config.systemPrompt || '',
3021
+ followUpHint: followUp || undefined,
3022
+ doneHint: done || undefined,
3023
+ memory: mem || undefined,
3024
+ devices: deviceLines,
3025
+ time,
3026
+ weather: weather || undefined,
3027
+ },
2469
3028
  });
2470
- return this.agent.ask(questionForLlm, sys, history);
3029
+ try {
3030
+ return await this.agent.ask(questionForLlm, sys, history);
3031
+ }
3032
+ catch (e) {
3033
+ // Offline, quota, outage: the local model is the fallback — unless it was already asked above.
3034
+ if (useLocal && !localFirst) {
3035
+ this.log.warn(`Cloud LLM failed (${e.message}) — asking the local LLM instead.`);
3036
+ this.journal?.step({
3037
+ stage: 'localLlm',
3038
+ level: 'warn',
3039
+ title: `Cloud LLM failed (${e.message}) — asking the local LLM instead`,
3040
+ });
3041
+ const ans = await this.askLocalLlm(question, weather, source);
3042
+ if (ans) {
3043
+ return ans;
3044
+ }
3045
+ }
3046
+ throw e;
3047
+ }
3048
+ }
3049
+ /**
3050
+ * Cloud tier through the adapter pi-agent: one pi-agent conversation per source. The style, the
3051
+ * follow-up marker, the memory and the device list are the instructions of that conversation (pi-agent
3052
+ * re-configures it only when they change); time, weather and presence change every turn and go in front
3053
+ * of the question. The history stays in pi-agent, so nothing of {@link ConversationStore} is sent.
3054
+ */
3055
+ async askPiAgent(harness, question, weather, source, localFallback) {
3056
+ const followUp = this.config.voiceEnabled ? this.followUpHint() : '';
3057
+ // The device list maps names to state ids, so pi-agent can act without searching first. It is the
3058
+ // first to go when the instructions get too long — pi-agent can still search.
3059
+ const { text: instructions, dropped } = (0, piAgent_1.composeInstructions)([this.config.systemPrompt || '', followUp, this.doneHint()], [this.buildMemoryContext(), await this.buildDeviceContext()]);
3060
+ if (dropped) {
3061
+ this.log.debug(`pi-agent: ${dropped} context block(s) left out, the instructions would be too long.`);
3062
+ }
3063
+ const text = [this.buildTimeContext(), weather, this.buildPresenceContext(), question]
3064
+ .filter(Boolean)
3065
+ .join('\n\n');
3066
+ this.log.info(`Cloud tier ${harness.instance} (source='${source}') — asking pi-agent…`);
3067
+ this.journal?.note({ tier: 'cloud', detail: harness.instance });
3068
+ try {
3069
+ return await harness.ask({ text, conversation: source, instructions });
3070
+ }
3071
+ catch (e) {
3072
+ if (localFallback) {
3073
+ this.log.warn(`pi-agent failed (${e.message}) — asking the local LLM instead.`);
3074
+ const ans = await this.askLocalLlm(question, weather, source);
3075
+ if (ans) {
3076
+ return ans;
3077
+ }
3078
+ }
3079
+ throw e;
3080
+ }
3081
+ }
3082
+ /**
3083
+ * Ask the local model, bounded by `localLlmTimeoutSec`. Returns its answer, or null when it handed off,
3084
+ * timed out or failed — the caller then goes on to the next tier.
3085
+ */
3086
+ async askLocalLlm(question, weather, source) {
3087
+ if (!this.localLlm) {
3088
+ return null;
3089
+ }
3090
+ const timeoutMs = Math.max(1, Number(this.config.localLlmTimeoutSec) || LOCAL_LLM_TIMEOUT_SEC) * 1000;
3091
+ const name = this.localLlmName();
3092
+ // The local model has no tools: without these lines it would invent the weather and guess who is
3093
+ // at home.
3094
+ const facts = [weather, this.buildPresenceContext()].filter(Boolean).join('\n\n');
3095
+ const prompt = facts ? `${facts}\n\n${question}` : question;
3096
+ const busy = this.localLlm.running;
3097
+ const data = { model: name, timeoutMs, system: this.localLlm.systemPrompt(), prompt };
3098
+ const start = Date.now();
3099
+ try {
3100
+ const ans = await this.localLlm.ask(prompt, timeoutMs);
3101
+ const ms = Date.now() - start;
3102
+ if (ans && !(0, localLlm_1.isHandoff)(ans)) {
3103
+ this.log.info(`Answered by local LLM (source='${source}').`);
3104
+ this.journal?.note({ tier: 'localLlm', detail: name });
3105
+ this.journal?.step({
3106
+ stage: 'localLlm',
3107
+ title: `Local LLM ${name}: answered`,
3108
+ ms,
3109
+ data: { ...data, answer: ans },
3110
+ });
3111
+ return ans;
3112
+ }
3113
+ this.log.debug('Local LLM handed off.');
3114
+ this.journal?.step({
3115
+ stage: 'localLlm',
3116
+ title: busy
3117
+ ? `Local LLM ${name}: busy with another question → handed off`
3118
+ : `Local LLM ${name}: handed off`,
3119
+ ms,
3120
+ data: { ...data, answer: ans },
3121
+ });
3122
+ }
3123
+ catch (e) {
3124
+ this.log.info(`Local LLM gave no answer: ${e.message}`);
3125
+ this.journal?.step({
3126
+ stage: 'localLlm',
3127
+ level: 'warn',
3128
+ title: `Local LLM ${name}: ${e.message}`,
3129
+ ms: Date.now() - start,
3130
+ data: { ...data, error: (0, journal_1.errorInfo)(e) },
3131
+ });
3132
+ }
3133
+ return null;
2471
3134
  }
2472
3135
  /** Reply when nothing local understood the request and there is no cloud LLM (provider "none"). */
2473
3136
  localOnlyFallback() {
@@ -2506,10 +3169,11 @@ class Assistant extends adapter_core_1.Adapter {
2506
3169
  const room = d.room ? `${d.room}, ` : '';
2507
3170
  lines.push(`- ${d.name} (${room}${d.type}): ${ids.join(', ')}`);
2508
3171
  }
3172
+ const lang = String(this.config.voiceLanguage || this.language || 'en');
3173
+ lines.push(...this.satelliteVolumeLines(lang));
2509
3174
  if (!lines.length) {
2510
3175
  return '';
2511
3176
  }
2512
- const lang = String(this.config.voiceLanguage || this.language || 'en');
2513
3177
  const header = lang === 'ru'
2514
3178
  ? 'Известные устройства — «имя (комната, тип): stateId». Используй эти stateId напрямую с set_state/get_states; вызывай list_devices только если нужного устройства здесь нет.'
2515
3179
  : lang === 'de'
@@ -2517,6 +3181,41 @@ class Assistant extends adapter_core_1.Adapter {
2517
3181
  : 'Known devices — "name (room, type): stateId". Use these stateIds directly with set_state/get_states; only call list_devices if a needed device is missing here.';
2518
3182
  return `${header}\n${lines.join('\n')}`;
2519
3183
  }
3184
+ /**
3185
+ * The playback volume of every voice satellite as a line of the device list. Without it, "Lautstärke auf 50
3186
+ * Prozent" that the rules miss reaches the LLM (or pi-agent) with only the media devices of the room in view, and
3187
+ * the model turns those up instead of the speaker that heard the request. The scales differ per transport.
3188
+ */
3189
+ satelliteVolumeLines(lang) {
3190
+ const label = (satId) => lang === 'ru'
3191
+ ? `Громкость голосового спутника ${satId}`
3192
+ : lang === 'de'
3193
+ ? `Lautstärke des Sprachsatelliten ${satId}`
3194
+ : `Volume of the voice satellite ${satId}`;
3195
+ const lines = [];
3196
+ const seen = new Set();
3197
+ for (const [satId, instance] of this.nativeSatFrom) {
3198
+ seen.add(satId);
3199
+ lines.push(`- ${label(satId)} (${satId}, level.volume 0-100): ${instance}.volume`);
3200
+ }
3201
+ for (const [satId, device] of this.satDeviceById) {
3202
+ if (seen.has(satId)) {
3203
+ continue;
3204
+ }
3205
+ const player = this.esphome?.entities(device).find(e => e.kind === 'mediaPlayer');
3206
+ if (player) {
3207
+ seen.add(satId);
3208
+ const id = `${this.namespace}.satellites.${satId}.controls.${player.objectId}.volume`;
3209
+ lines.push(`- ${label(satId)} (${satId}, level.volume 0-1): ${id}`);
3210
+ }
3211
+ }
3212
+ for (const satId of this.satVolumeEnsured) {
3213
+ if (!seen.has(satId)) {
3214
+ lines.push(`- ${label(satId)} (${satId}, level.volume 0-100): ${this.namespace}.satellites.${satId}.volume`);
3215
+ }
3216
+ }
3217
+ return lines;
3218
+ }
2520
3219
  /**
2521
3220
  * List a port we listen on in the host's used-resources registry (js-controller ≥ 8), so the admin shows it
2522
3221
  * as occupied when someone configures another instance. `common.declareUsedResources` is set because the
@@ -2578,35 +3277,27 @@ class Assistant extends adapter_core_1.Adapter {
2578
3277
  else {
2579
3278
  // Switched off: do not keep what was recorded while it was on.
2580
3279
  await fs.promises.rm(file, { force: true }).catch(() => { });
3280
+ await fs.promises.rm((0, journal_1.traceDirFor)(file), { recursive: true, force: true }).catch(() => { });
2581
3281
  }
2582
3282
  }
2583
- /** Wrap the MCP connection so every state the pipeline reads or writes lands in the journal entry. */
3283
+ /**
3284
+ * Wrap the MCP connection so every state the pipeline reads or writes lands in the journal entry — and a
3285
+ * call the NLU makes directly, with its result, in the request's trace.
3286
+ */
2584
3287
  journalMcp(mcp) {
2585
3288
  return {
2586
3289
  listTools: () => mcp.listTools(),
2587
- callTool: (name, args) => {
2588
- this.journal?.stateAccess(name, args);
2589
- return mcp.callTool(name, args);
2590
- },
3290
+ callTool: (name, args) => this.journal
3291
+ ? this.journal.stateCall(name, args, () => mcp.callTool(name, args))
3292
+ : mcp.callTool(name, args),
2591
3293
  close: () => mcp.close(),
2592
3294
  };
2593
3295
  }
2594
- /** Wrap an LLM tool so its calls (arguments, result, duration) land in the journal entry. */
3296
+ /** Wrap an LLM tool so its calls (arguments, result, duration) land in the journal entry and its trace. */
2595
3297
  journalTool(tool) {
2596
3298
  return {
2597
3299
  ...tool,
2598
- run: async (args) => {
2599
- const start = Date.now();
2600
- try {
2601
- const result = await tool.run(args);
2602
- this.journal?.tool(tool.name, args, Date.now() - start, result);
2603
- return result;
2604
- }
2605
- catch (e) {
2606
- this.journal?.tool(tool.name, args, Date.now() - start, undefined, e.message);
2607
- throw e;
2608
- }
2609
- },
3300
+ run: args => (this.journal ? this.journal.toolCall(tool.name, args, () => tool.run(args)) : tool.run(args)),
2610
3301
  };
2611
3302
  }
2612
3303
  /** Short name of the local model for the journal ("qwen2.5-1.5b-instruct-q4_k_m"). */
@@ -2704,7 +3395,7 @@ class Assistant extends adapter_core_1.Adapter {
2704
3395
  dataDir,
2705
3396
  modelUrl: (this.config.localLlmModelUrl || '').trim() || localLlm_1.DEFAULT_LOCAL_MODEL_URL,
2706
3397
  systemPrompt: this.config.systemPrompt || '',
2707
- maxTokens: this.config.maxTokens || 512,
3398
+ maxTokens: Math.min(this.config.maxTokens || LOCAL_LLM_MAX_TOKENS, LOCAL_LLM_MAX_TOKENS),
2708
3399
  log: this.log,
2709
3400
  });
2710
3401
  this.lastLlmPct = -1;
@@ -2737,20 +3428,56 @@ class Assistant extends adapter_core_1.Adapter {
2737
3428
  }
2738
3429
  // Timer intents match device-independently (parse() checks them first), so build the NLU even
2739
3430
  // when no devices are known and always try timers before bailing out.
3431
+ const start = Date.now();
2740
3432
  const { rooms, devices } = await this.getNluDevices();
2741
- const intents = new nlu_1.Nlu(rooms, devices, this.getNluAliases()).parseAll(question);
2742
- if (!intents.length || !intents.every(i => this.canExecuteNlu(i, devices.length))) {
3433
+ const aliases = this.getNluAliases();
3434
+ const intents = new nlu_1.Nlu(rooms, devices, aliases, {
3435
+ weather: !!(this.config.weatherInstance || '').trim(),
3436
+ }).parseAll(question);
3437
+ const blockers = intents.map(i => this.nluBlocker(i, devices.length));
3438
+ this.journal?.step({
3439
+ stage: 'nlu',
3440
+ title: !intents.length
3441
+ ? `NLU: nothing recognised (${devices.length} devices, ${rooms.length} rooms known)`
3442
+ : blockers.some(Boolean)
3443
+ ? `NLU: ${intents.map(i => i.action).join(' + ')} — cannot run here: ${blockers.filter(Boolean).join('; ')}`
3444
+ : `NLU: ${intents.map(describeIntent).join(' + ')}`,
3445
+ level: intents.length && blockers.some(Boolean) ? 'warn' : undefined,
3446
+ ms: Date.now() - start,
3447
+ data: {
3448
+ devices: devices.length,
3449
+ rooms,
3450
+ aliases: aliases.length ? aliases : undefined,
3451
+ intents: intents.map((intent, i) => ({ ...intent, blockedBecause: blockers[i] || undefined })),
3452
+ },
3453
+ });
3454
+ if (!intents.length || blockers.some(Boolean)) {
2743
3455
  return null; // nothing matched, or one part cannot run here → let the LLM answer the whole thing
2744
3456
  }
2745
3457
  this.journal?.note({ tier: 'nlu', detail: intents.map(i => i.action).join(' + ') });
2746
3458
  const answers = [];
2747
3459
  let failed = false;
2748
3460
  for (const intent of intents) {
3461
+ const started = Date.now();
2749
3462
  try {
2750
- answers.push(await this.executeNluIntent(intent, source));
3463
+ const answer = await this.executeNluIntent(intent, source);
3464
+ answers.push(answer);
3465
+ this.journal?.step({
3466
+ stage: 'nlu',
3467
+ title: `Executed ${describeIntent(intent)}`,
3468
+ ms: Date.now() - started,
3469
+ data: { answer },
3470
+ });
2751
3471
  }
2752
3472
  catch (e) {
2753
3473
  failed = true;
3474
+ this.journal?.step({
3475
+ stage: 'nlu',
3476
+ level: 'error',
3477
+ title: `${describeIntent(intent)} failed: ${e.message}${answers.length ? '' : ' → the LLM takes over'}`,
3478
+ ms: Date.now() - started,
3479
+ data: (0, journal_1.errorInfo)(e),
3480
+ });
2754
3481
  // Nothing executed yet → hand the whole utterance to the LLM. Afterwards a state has already
2755
3482
  // been written and the LLM would repeat it, so report the failed part instead of bailing out.
2756
3483
  if (!answers.length) {
@@ -2770,38 +3497,51 @@ class Assistant extends adapter_core_1.Adapter {
2770
3497
  if (this.config.confirmWithTone &&
2771
3498
  answer &&
2772
3499
  !failed &&
2773
- intents.every(i => CONTROL_ACTIONS.has(i.action)) &&
3500
+ // a command for later says when it will run — a beep would hide that
3501
+ intents.every(i => CONTROL_ACTIONS.has(i.action) && !isDeferred(i)) &&
2774
3502
  (await this.playConfirmationTone(source))) {
2775
3503
  this.log.debug(`NLU confirmed with a tone instead of "${answer}".`);
3504
+ this.journal?.step({ stage: 'nlu', title: 'Confirmed with a tone instead of words', data: { answer } });
2776
3505
  return '';
2777
3506
  }
2778
3507
  return answer;
2779
3508
  }
2780
3509
  /**
2781
- * Can this intent run locally at all? Checked for **every** part of a combined command before anything
2782
- * is executed: a missing manager, no known devices or disabled writes send the whole utterance to the
2783
- * LLM (which can explain why) instead of silently doing nothing — or only half of it.
3510
+ * Why this intent cannot run locally ('' = it can). Checked for **every** part of a combined command
3511
+ * before anything is executed: a missing manager, no known devices or disabled writes send the whole
3512
+ * utterance to the LLM (which can explain why) instead of silently doing nothing — or only half of it.
2784
3513
  */
2785
- canExecuteNlu(intent, deviceCount) {
3514
+ nluBlocker(intent, deviceCount) {
2786
3515
  switch (intent.action) {
2787
3516
  case 'timerSet':
2788
3517
  case 'timerQuery':
2789
3518
  case 'timerCancel':
2790
- return !!this.timers;
3519
+ return this.timers ? '' : 'timers are not running';
2791
3520
  case 'alarmSet':
2792
3521
  case 'alarmQuery':
2793
3522
  case 'alarmCancel':
2794
- return !!this.alarms;
3523
+ return this.alarms ? '' : 'alarms are not running';
2795
3524
  case 'timeQuery':
2796
3525
  case 'dateQuery':
2797
- return true;
3526
+ return '';
3527
+ case 'weatherQuery':
3528
+ return (this.config.weatherInstance || '').trim() ? '' : 'no weather instance configured';
3529
+ case 'volumeSet':
3530
+ return this.voice || this.esphome || this.nativeSatFrom.size ? '' : 'no satellites';
2798
3531
  default:
2799
3532
  // Device intents; writes additionally require the coarse "allow control" toggle.
2800
- return (deviceCount > 0 &&
2801
- (!['on', 'off', 'level', 'color'].includes(intent.action) || !!this.config.allowWriteStates));
3533
+ if (!deviceCount) {
3534
+ return 'no devices known';
3535
+ }
3536
+ if (isDeferred(intent) && !(this.config.calendarInstance || '').trim()) {
3537
+ return 'a command for later needs a calendar (fullcalendar) instance in the settings';
3538
+ }
3539
+ return ['on', 'off', 'level', 'color'].includes(intent.action) && !this.config.allowWriteStates
3540
+ ? 'writing states is not allowed (allowWriteStates)'
3541
+ : '';
2802
3542
  }
2803
3543
  }
2804
- /** Execute one intent that {@link canExecuteNlu} accepted, and return its spoken answer. */
3544
+ /** Execute one intent that {@link nluBlocker} accepted, and return its spoken answer. */
2805
3545
  async executeNluIntent(intent, source) {
2806
3546
  switch (intent.action) {
2807
3547
  case 'timerSet':
@@ -2815,9 +3555,314 @@ class Assistant extends adapter_core_1.Adapter {
2815
3555
  case 'timeQuery':
2816
3556
  case 'dateQuery':
2817
3557
  return this.executeTimeIntent(intent);
3558
+ case 'weatherQuery':
3559
+ return this.executeWeatherIntent(intent);
3560
+ case 'volumeSet':
3561
+ return this.executeVolumeIntent(intent, source);
2818
3562
  default:
2819
- return this.executeIntent(intent);
3563
+ return isDeferred(intent) ? this.executeDeferredIntent(intent) : this.executeIntent(intent);
3564
+ }
3565
+ }
3566
+ /**
3567
+ * Send a message to the calendar instance (fullcalendar). Its answer `{error}` becomes an exception.
3568
+ *
3569
+ * @param command `addEvent`, `listEvents`, `deleteEvent`, …
3570
+ * @param message the parameters
3571
+ */
3572
+ async calendarCall(command, message) {
3573
+ const instance = (this.config.calendarInstance || '').trim().replace(/^system\.adapter\./, '');
3574
+ if (!instance) {
3575
+ throw new Error('No calendar instance configured in the assistant settings');
3576
+ }
3577
+ const alive = await this.getForeignStateAsync(`system.adapter.${instance}.alive`);
3578
+ if (!alive?.val) {
3579
+ throw new Error(`The calendar ${instance} is not running`);
3580
+ }
3581
+ const res = (await this.sendToAsync(instance, command, message, { timeout: 10000 }));
3582
+ if (!res) {
3583
+ throw new Error(`The calendar ${instance} did not answer`);
3584
+ }
3585
+ if (res.error) {
3586
+ throw new Error(res.error);
3587
+ }
3588
+ return res;
3589
+ }
3590
+ /**
3591
+ * Whether the per-device ACL forbids writing this state: an explicit entry, else the default of the device
3592
+ * type (locks are read-only). States outside the device list are not restricted (like `set_state`).
3593
+ */
3594
+ async writeForbidden(stateId) {
3595
+ const { devices } = await this.getNluDevices();
3596
+ const device = devices.find(d => Object.values(d.controls).includes(stateId));
3597
+ if (!device) {
3598
+ return false;
3599
+ }
3600
+ const entry = this.config.deviceAcl?.[(0, tools_1.deviceKey)(Object.values(device.controls))];
3601
+ return entry ? entry.write === false : device.type === 'lock';
3602
+ }
3603
+ /** "22:30", "morgen um 22:30", "30.10. um 7:00" — when a planned command runs, for the spoken answer */
3604
+ describeWhen(time, lang) {
3605
+ const pad = (n) => String(n).padStart(2, '0');
3606
+ const clock = `${time.getHours()}:${pad(time.getMinutes())}`;
3607
+ const now = new Date();
3608
+ const days = Math.round((new Date(time.getFullYear(), time.getMonth(), time.getDate()).getTime() -
3609
+ new Date(now.getFullYear(), now.getMonth(), now.getDate()).getTime()) /
3610
+ 86_400_000);
3611
+ const date = `${pad(time.getDate())}.${pad(time.getMonth() + 1)}.`;
3612
+ if (lang === 'ru') {
3613
+ return days === 0 ? `в ${clock}` : days === 1 ? `завтра в ${clock}` : `${date} в ${clock}`;
3614
+ }
3615
+ if (lang === 'de') {
3616
+ return days === 0 ? `um ${clock}` : days === 1 ? `morgen um ${clock}` : `am ${date} um ${clock}`;
3617
+ }
3618
+ return days === 0 ? `at ${clock}` : days === 1 ? `tomorrow at ${clock}` : `on ${date} at ${clock}`;
3619
+ }
3620
+ /**
3621
+ * A device command for later ("Licht in 2 Stunden aus", "выключи свет через 2 часа"): one event in the calendar
3622
+ * (fullcalendar) per write — the calendar shows it, survives restarts and can be cancelled there.
3623
+ */
3624
+ async executeDeferredIntent(intent) {
3625
+ const device = intent.device;
3626
+ if (!device || !intent.stateId) {
3627
+ throw new Error('deferred intent without a device');
3628
+ }
3629
+ const lang = String(this.config.voiceLanguage || this.language || 'en');
3630
+ const pick = (rus, ger, eng) => lang === 'ru' ? rus : lang === 'de' ? ger : eng;
3631
+ const room = intent.room || device.room;
3632
+ const where = room ? ` (${room})` : '';
3633
+ const dev = device.name;
3634
+ if (await this.writeForbidden(intent.stateId)) {
3635
+ return pick(`${dev}${where} только для чтения.`, `${dev}${where} ist schreibgeschützt.`, `${dev}${where} is read-only.`);
3636
+ }
3637
+ let time;
3638
+ if (intent.delaySec !== undefined) {
3639
+ time = new Date(Date.now() + intent.delaySec * 1000);
3640
+ }
3641
+ else {
3642
+ const { hour, minute, day } = intent.at;
3643
+ const now = new Date();
3644
+ time = new Date(now.getFullYear(), now.getMonth(), now.getDate() + (day ?? 0), hour, minute, 0, 0);
3645
+ if (time.getTime() <= now.getTime()) {
3646
+ if (day !== undefined) {
3647
+ return pick(`${this.describeWhen(time, lang)} уже прошло.`, `${this.describeWhen(time, lang).replace(/^um /, '')} ist schon vorbei.`, `${this.describeWhen(time, lang).replace(/^at /, '')} has already passed.`);
3648
+ }
3649
+ time.setDate(time.getDate() + 1); // "um 7 Uhr" after 7 o'clock: tomorrow
3650
+ }
3651
+ }
3652
+ const name = `${dev}${where}`;
3653
+ await this.calendarCall('addEvent', { oid: intent.stateId, value: intent.value, start: time.getTime(), name });
3654
+ // a dimmer with its own switch: the switch goes into the calendar, too
3655
+ if (intent.also) {
3656
+ await this.calendarCall('addEvent', {
3657
+ oid: intent.also.stateId,
3658
+ value: intent.also.value,
3659
+ start: time.getTime(),
3660
+ name,
3661
+ });
3662
+ }
3663
+ this.log.info(`Planned in the calendar: ${intent.stateId} = ${JSON.stringify(intent.value)} at ${time.toString()}`);
3664
+ const when = this.describeWhen(time, lang);
3665
+ switch (intent.action) {
3666
+ case 'on':
3667
+ return pick(`Хорошо, ${when} включу: ${dev}${where}.`, `Okay, ${dev}${where} wird ${when} eingeschaltet.`, `Okay, ${dev}${where} will be turned on ${when}.`);
3668
+ case 'off':
3669
+ return pick(`Хорошо, ${when} выключу: ${dev}${where}.`, `Okay, ${dev}${where} wird ${when} ausgeschaltet.`, `Okay, ${dev}${where} will be turned off ${when}.`);
3670
+ case 'level':
3671
+ return pick(`Хорошо, ${when} поставлю ${dev}${where} на ${String(intent.value)} %.`, `Okay, ${dev}${where} wird ${when} auf ${String(intent.value)} % gestellt.`, `Okay, ${dev}${where} will be set to ${String(intent.value)} % ${when}.`);
3672
+ default:
3673
+ return pick(`Хорошо, запланировано ${when}: ${dev}${where}.`, `Okay, für ${dev}${where} ${when} eingeplant.`, `Okay, planned for ${dev}${where} ${when}.`);
3674
+ }
3675
+ }
3676
+ /**
3677
+ * LLM tools for the calendar (fullcalendar): plan a state change for later, list what is planned, cancel it.
3678
+ * Planning respects the same write rules as `set_state` (the coarse switch and the per-device ACL).
3679
+ */
3680
+ buildCalendarTools() {
3681
+ const fail = (e) => JSON.stringify({ ok: false, error: e.message });
3682
+ const tools = [
3683
+ {
3684
+ name: 'list_scheduled',
3685
+ description: 'List the planned events of the calendar (state changes at a time, also recurring ones), in local time of the house. Optional "days" ahead (default 1 = the next 24 hours) or a "from"/"to" range as local time "YYYY-MM-DDTHH:mm".',
3686
+ parameters: {
3687
+ type: 'object',
3688
+ properties: {
3689
+ days: { type: 'number', description: 'How many days ahead, default 1' },
3690
+ from: { type: 'string', description: 'Start, local time YYYY-MM-DDTHH:mm' },
3691
+ to: { type: 'string', description: 'End, local time YYYY-MM-DDTHH:mm' },
3692
+ },
3693
+ additionalProperties: false,
3694
+ },
3695
+ run: async (args) => {
3696
+ try {
3697
+ const res = await this.calendarCall('listEvents', {
3698
+ days: typeof args.days === 'number' ? args.days : 1,
3699
+ from: args.from,
3700
+ to: args.to,
3701
+ max: 50,
3702
+ });
3703
+ return JSON.stringify({ ok: true, events: res.events });
3704
+ }
3705
+ catch (e) {
3706
+ return fail(e);
3707
+ }
3708
+ },
3709
+ },
3710
+ ];
3711
+ if (!this.config.allowWriteStates) {
3712
+ return tools;
2820
3713
  }
3714
+ tools.push({
3715
+ name: 'schedule_state',
3716
+ description: 'Plan a state change for later in the calendar: write "value" to state "id" after "in_minutes" or at "at" (local time of the house, "YYYY-MM-DDTHH:mm"). Use it for "in 2 hours", "at 10 pm", "tomorrow morning" — never set_state for something meant for later. With "duration_minutes" the state gets "end_value" after that time (for a boolean the opposite by default). Tell the user the planned time.',
3717
+ parameters: {
3718
+ type: 'object',
3719
+ properties: {
3720
+ id: { type: 'string', description: 'State ID' },
3721
+ value: { type: ['boolean', 'number', 'string'], description: 'Value to write' },
3722
+ in_minutes: { type: 'number', description: 'Delay in minutes from now' },
3723
+ at: { type: 'string', description: 'Local time YYYY-MM-DDTHH:mm' },
3724
+ duration_minutes: { type: 'number', description: 'Optional: write end_value this much later' },
3725
+ end_value: { type: ['boolean', 'number', 'string'], description: 'Value after the duration' },
3726
+ name: {
3727
+ type: 'string',
3728
+ description: 'Short title for the calendar, e.g. "Licht Wohnzimmer aus"',
3729
+ },
3730
+ },
3731
+ required: ['id', 'value'],
3732
+ additionalProperties: false,
3733
+ },
3734
+ run: async (args) => {
3735
+ try {
3736
+ const id = typeof args.id === 'string' ? args.id : '';
3737
+ if (await this.writeForbidden(id)) {
3738
+ return JSON.stringify({
3739
+ ok: false,
3740
+ error: `Writing "${id}" is not allowed: the device is read-only in the assistant settings.`,
3741
+ });
3742
+ }
3743
+ if (args.in_minutes === undefined && !args.at) {
3744
+ return JSON.stringify({ ok: false, error: 'Give "in_minutes" or "at"' });
3745
+ }
3746
+ const res = await this.calendarCall('addEvent', {
3747
+ oid: id,
3748
+ value: args.value,
3749
+ in: typeof args.in_minutes === 'number' ? Math.round(args.in_minutes * 60) : undefined,
3750
+ start: args.in_minutes === undefined ? args.at : undefined,
3751
+ duration: typeof args.duration_minutes === 'number'
3752
+ ? Math.round(args.duration_minutes * 60)
3753
+ : undefined,
3754
+ endValue: args.end_value,
3755
+ name: typeof args.name === 'string' ? args.name : undefined,
3756
+ });
3757
+ return JSON.stringify({ ok: true, eventId: res.id, start: res.start });
3758
+ }
3759
+ catch (e) {
3760
+ return fail(e);
3761
+ }
3762
+ },
3763
+ }, {
3764
+ name: 'cancel_scheduled',
3765
+ description: 'Delete a planned event from the calendar, by its "id" from list_scheduled or schedule_state.',
3766
+ parameters: {
3767
+ type: 'object',
3768
+ properties: { id: { type: 'string', description: 'Event ID' } },
3769
+ required: ['id'],
3770
+ additionalProperties: false,
3771
+ },
3772
+ run: async (args) => {
3773
+ try {
3774
+ await this.calendarCall('deleteEvent', { id: typeof args.id === 'string' ? args.id : '' });
3775
+ return JSON.stringify({ ok: true });
3776
+ }
3777
+ catch (e) {
3778
+ return fail(e);
3779
+ }
3780
+ },
3781
+ });
3782
+ return tools;
3783
+ }
3784
+ /**
3785
+ * Speak the weather straight from the configured adapter. Throws when the report has nothing for the
3786
+ * asked day (or the adapter has no mapper), so the question goes on to the LLM and its tool instead.
3787
+ */
3788
+ async executeWeatherIntent(intent) {
3789
+ const res = await this.readWeather(this.config.weatherInstance || '');
3790
+ if (!res.report) {
3791
+ throw new Error(res.error || 'no weather report');
3792
+ }
3793
+ const lang = String(this.config.voiceLanguage || this.language || 'en');
3794
+ const text = (0, weather_1.buildWeatherAnswer)(res.report, lang, intent.when);
3795
+ if (!text) {
3796
+ throw new Error(`the weather report has no data for '${intent.when || 'now'}'`);
3797
+ }
3798
+ return text;
3799
+ }
3800
+ /**
3801
+ * Change the playback volume of a satellite: the one in the named room, else the one that heard the
3802
+ * command. Each transport has its own knob — the native satellite's `volume` state, the media player
3803
+ * of an ESPHome device, the `volume` message of a UDP satellite.
3804
+ */
3805
+ async executeVolumeIntent(intent, source) {
3806
+ const lang = String(this.config.voiceLanguage || this.language || 'en');
3807
+ const satId = intent.room ? this.resolveSatelliteId(intent.room) : this.announceTargetForSource(source);
3808
+ const absolute = typeof intent.value === 'number' ? intent.value : null;
3809
+ const delta = intent.delta || 0;
3810
+ const next = (current) => Math.min(100, Math.max(0, Math.round(absolute ?? current + delta)));
3811
+ // 1 процент, 2 процента, 5 процентов (11–14 take the last form).
3812
+ const ruPercent = (n) => n % 100 > 10 && n % 100 < 20
3813
+ ? 'процентов'
3814
+ : n % 10 === 1
3815
+ ? 'процент'
3816
+ : n % 10 >= 2 && n % 10 <= 4
3817
+ ? 'процента'
3818
+ : 'процентов';
3819
+ const confirm = (level) => lang === 'ru'
3820
+ ? `Громкость ${level} ${ruPercent(level)}.`
3821
+ : lang === 'de'
3822
+ ? `Lautstärke ${level} Prozent.`
3823
+ : `Volume ${level} percent.`;
3824
+ if (satId) {
3825
+ const device = this.satDeviceById.get(satId) || satId;
3826
+ const instance = this.nativeSatFrom.get(satId);
3827
+ if (instance) {
3828
+ const stateId = `${instance}.volume`;
3829
+ const current = Number((await this.getForeignStateAsync(stateId))?.val ?? 50);
3830
+ const level = next(Number.isFinite(current) ? current : 50);
3831
+ await this.setForeignStateAsync(stateId, level);
3832
+ return confirm(level);
3833
+ }
3834
+ const player = this.esphome?.entities(device).find(e => e.kind === 'mediaPlayer');
3835
+ if (player) {
3836
+ const current = (player.value?.volume ?? 0.5) * 100;
3837
+ const level = next(current);
3838
+ if (this.esphome.setEntity(device, player.objectId, level / 100)) {
3839
+ return confirm(level);
3840
+ }
3841
+ }
3842
+ // A UDP satellite works out a relative step itself (it knows its level) and reports the result.
3843
+ if (this.voice?.has(device)) {
3844
+ this.voice.volume(device, absolute !== null ? { level: absolute } : { delta });
3845
+ if (absolute !== null) {
3846
+ return confirm(next(0));
3847
+ }
3848
+ return lang === 'ru'
3849
+ ? delta > 0
3850
+ ? 'Сделала громче.'
3851
+ : 'Сделала тише.'
3852
+ : lang === 'de'
3853
+ ? delta > 0
3854
+ ? 'Etwas lauter.'
3855
+ : 'Etwas leiser.'
3856
+ : delta > 0
3857
+ ? 'A bit louder.'
3858
+ : 'A bit quieter.';
3859
+ }
3860
+ }
3861
+ return lang === 'ru'
3862
+ ? 'Я не знаю, громкость какой колонки изменить.'
3863
+ : lang === 'de'
3864
+ ? 'Ich weiß nicht, bei welchem Lautsprecher ich die Lautstärke ändern soll.'
3865
+ : "I don't know which speaker's volume to change.";
2821
3866
  }
2822
3867
  /** Localized "this part did not work" sentence for a failed part of a combined command. */
2823
3868
  nluFailureText(intent) {
@@ -2868,14 +3913,36 @@ class Assistant extends adapter_core_1.Adapter {
2868
3913
  return [];
2869
3914
  }
2870
3915
  }
2871
- /** Device model for the NLU: friendly name, room, type and controls (controlType → state id). */
2872
- async getNluDevices() {
3916
+ /**
3917
+ * Device model for the NLU: friendly name, room, type and controls (controlType → state id). Cached
3918
+ * for {@link NLU_DEVICES_TTL_MS}; requests arriving while it is being built wait for the same build.
3919
+ */
3920
+ getNluDevices() {
2873
3921
  if (!this.mcp) {
2874
- return { rooms: [], devices: [] };
3922
+ return Promise.resolve({ rooms: [], devices: [] });
2875
3923
  }
2876
3924
  // Match names in the assistant's interaction language (voice language, else system) so a
2877
3925
  // Russian command matches Russian smartNames.
2878
3926
  const nluLang = (this.config.voiceLanguage || this.language || 'en');
3927
+ const now = Date.now();
3928
+ const hit = this.nluDevices;
3929
+ if (hit && hit.key === nluLang && now - hit.at < NLU_DEVICES_TTL_MS) {
3930
+ return hit.result;
3931
+ }
3932
+ const entry = { key: nluLang, at: now, result: this.buildNluDevices(nluLang) };
3933
+ this.nluDevices = entry;
3934
+ // An empty model is usually a failed build (the manager not ready yet) — don't keep it for a minute.
3935
+ void entry.result.then(r => {
3936
+ if (!r.devices.length && this.nluDevices === entry) {
3937
+ this.nluDevices = null;
3938
+ }
3939
+ });
3940
+ return entry.result;
3941
+ }
3942
+ async buildNluDevices(nluLang) {
3943
+ if (!this.mcp) {
3944
+ return { rooms: [], devices: [] };
3945
+ }
2879
3946
  try {
2880
3947
  const res = await this.mcp.callTool('list_devices', { language: nluLang });
2881
3948
  const parsed = JSON.parse(res.text);
@@ -3507,16 +4574,30 @@ class Assistant extends adapter_core_1.Adapter {
3507
4574
  if (!this.config.verifyWrites || !stateId) {
3508
4575
  return true;
3509
4576
  }
3510
- const deadline = Date.now() + ACK_TIMEOUT_MS;
4577
+ const start = Date.now();
4578
+ const deadline = start + ACK_TIMEOUT_MS;
3511
4579
  let last;
3512
4580
  while (Date.now() < deadline) {
3513
4581
  await this.delay(ACK_POLL_MS);
3514
4582
  last = await this.getForeignStateAsync(stateId).catch(() => null);
3515
4583
  if (last?.ack && valuesMatch(last.val, expected)) {
3516
4584
  this.log.debug(`write to ${stateId} confirmed by the device.`);
4585
+ this.journal?.step({
4586
+ stage: 'state',
4587
+ title: `${stateId} confirmed by the device`,
4588
+ ms: Date.now() - start,
4589
+ data: { expected, state: last },
4590
+ });
3517
4591
  return true;
3518
4592
  }
3519
4593
  }
4594
+ this.journal?.step({
4595
+ stage: 'state',
4596
+ level: 'warn',
4597
+ title: `${stateId}: no confirmation within ${ACK_TIMEOUT_MS} ms`,
4598
+ ms: Date.now() - start,
4599
+ data: { expected, state: last },
4600
+ });
3520
4601
  this.log.warn(`No confirmation for ${stateId} within ${ACK_TIMEOUT_MS} ms ` +
3521
4602
  `(last: ${JSON.stringify(last?.val)}, ack=${String(last?.ack)}). ` +
3522
4603
  'If this device never acknowledges, switch "Verify device feedback" off.');
@@ -4729,12 +5810,26 @@ class Assistant extends adapter_core_1.Adapter {
4729
5810
  */
4730
5811
  async runRoutine(routine, source) {
4731
5812
  const origin = this.announceTargetForSource(source);
4732
- for (const action of routine.actions) {
5813
+ for (const [i, action] of routine.actions.entries()) {
5814
+ const start = Date.now();
4733
5815
  try {
4734
5816
  await this.runTriggerAction({ id: `routine:${routine.name}`, when: [] }, action, origin);
5817
+ this.journal?.step({
5818
+ stage: 'routine',
5819
+ title: `Routine action ${i + 1}/${routine.actions.length}`,
5820
+ ms: Date.now() - start,
5821
+ data: action,
5822
+ });
4735
5823
  }
4736
5824
  catch (e) {
4737
5825
  this.log.warn(`Routine '${routine.name}': an action failed — ${e.message}`);
5826
+ this.journal?.step({
5827
+ stage: 'routine',
5828
+ level: 'error',
5829
+ title: `Routine action ${i + 1}/${routine.actions.length} failed: ${e.message}`,
5830
+ ms: Date.now() - start,
5831
+ data: { action, error: (0, journal_1.errorInfo)(e) },
5832
+ });
4738
5833
  }
4739
5834
  }
4740
5835
  }
@@ -5274,13 +6369,52 @@ class Assistant extends adapter_core_1.Adapter {
5274
6369
  return { ok: false, error: e.message };
5275
6370
  }
5276
6371
  }
5277
- /** Build the configured TTS engine (throws when no voice key is available). */
5278
- async buildTtsEngine() {
5279
- const cfg = this.config;
5280
- const mainApiKey = (await (0, credentials_1.resolveApiKey)(this, cfg)) || '';
5281
- const creds = await (0, credentials_1.resolveVoiceCredentials)(this, cfg, mainApiKey);
6372
+ /**
6373
+ * The saved configuration with the speech settings of an unsaved form on top. Only the speech fields count,
6374
+ * and a key only when one was typed: protected fields never reach the browser, so they are empty there.
6375
+ * Without fallbacks: a test must show whether the chosen engine works, not whether the reserve does.
6376
+ */
6377
+ withVoiceForm(form) {
6378
+ const cfg = { ...this.config, sttFallback: '', ttsFallback: '' };
6379
+ for (const key of VOICE_FORM_FIELDS) {
6380
+ const value = form[key];
6381
+ if (typeof value !== 'string' || (VOICE_FORM_SECRETS.includes(key) && !value.trim())) {
6382
+ continue;
6383
+ }
6384
+ cfg[key] = value;
6385
+ }
6386
+ return cfg;
6387
+ }
6388
+ /** Build the TTS engine of a configuration (throws when no voice key is available). */
6389
+ async buildTtsEngine(cfg = this.config) {
6390
+ const creds = await (0, credentials_1.resolveVoiceCredentials)(this, cfg, await this.speechLlmKey());
5282
6391
  return (0, engines_1.createTtsEngine)(cfg.ttsProvider || 'openai', this.voiceContext(cfg, creds));
5283
6392
  }
6393
+ /**
6394
+ * Speech recognition for the voice test of the settings: a recording of the browser, an uploaded file or the
6395
+ * test sentence the TTS just spoke. WAV is read directly, anything else goes through ffmpeg.
6396
+ */
6397
+ async transcribeForTest(msg) {
6398
+ try {
6399
+ if (!msg.audio) {
6400
+ return { error: 'no audio' };
6401
+ }
6402
+ const cfg = msg.config ? this.withVoiceForm(msg.config) : this.config;
6403
+ const data = Buffer.from(msg.audio, 'base64');
6404
+ const { pcm, sampleRate } = (msg.format || 'wav') === 'wav' && data.toString('latin1', 0, 4) === 'RIFF'
6405
+ ? (0, openrouter_1.pcmFromResponse)(data, '')
6406
+ : await this.decodeAudioBufferToPcm(data);
6407
+ const creds = await (0, credentials_1.resolveVoiceCredentials)(this, cfg, await this.speechLlmKey());
6408
+ const stt = (0, engines_1.createSttEngine)(cfg.sttProvider || 'openai', this.voiceContext(cfg, creds));
6409
+ const language = msg.language || cfg.voiceLanguage || this.language || '';
6410
+ const start = Date.now();
6411
+ const text = await stt.transcribe(pcm, sampleRate, language, await this.buildSttHints());
6412
+ return { text, ms: Date.now() - start, audioMs: Math.round((pcm.length / 2 / sampleRate) * 1000) };
6413
+ }
6414
+ catch (e) {
6415
+ return { error: e.message };
6416
+ }
6417
+ }
5284
6418
  /** True if backend TTS can run (a voice key is configured) — drives the Chat play button. */
5285
6419
  async isTtsAvailable() {
5286
6420
  try {
@@ -5292,15 +6426,23 @@ class Assistant extends adapter_core_1.Adapter {
5292
6426
  }
5293
6427
  }
5294
6428
  /** Synthesize `text` to a base64 WAV via the configured TTS engine (used by the Chat play button). */
5295
- async synthesizeToWav(text, language) {
6429
+ async synthesizeToWav(text, language, cfg = this.config) {
5296
6430
  if (!text.trim()) {
5297
6431
  return { error: 'empty text' };
5298
6432
  }
5299
6433
  try {
5300
- const tts = await this.buildTtsEngine();
5301
- const lang = language || this.config.voiceLanguage || this.language || '';
6434
+ const tts = await this.buildTtsEngine(cfg);
6435
+ const lang = language || cfg.voiceLanguage || this.language || '';
6436
+ const start = Date.now();
5302
6437
  const { pcm, sampleRate } = await tts.synthesize(text, lang);
5303
- return { audio: this.pcmToWav(pcm, sampleRate).toString('base64'), mime: 'audio/wav' };
6438
+ return {
6439
+ audio: this.pcmToWav(pcm, sampleRate).toString('base64'),
6440
+ mime: 'audio/wav',
6441
+ sampleRate,
6442
+ // a wrong rate shows up as an implausible length of the spoken text
6443
+ audioMs: Math.round((pcm.length / 2 / sampleRate) * 1000),
6444
+ ms: Date.now() - start,
6445
+ };
5304
6446
  }
5305
6447
  catch (e) {
5306
6448
  return { error: e.message };
@@ -5332,7 +6474,8 @@ class Assistant extends adapter_core_1.Adapter {
5332
6474
  async getModels(msg) {
5333
6475
  const cfg = this.config;
5334
6476
  const provider = msg.provider || cfg.provider || 'openai';
5335
- if (provider === 'none') {
6477
+ // `system`: the model is chosen in the system settings of admin
6478
+ if (provider === 'none' || provider === 'pi-agent' || provider === 'system') {
5336
6479
  return [];
5337
6480
  }
5338
6481
  const apiKey = (0, llm_1.effectiveApiKey)(provider, await (0, credentials_1.resolveApiKey)(this, cfg, {
@@ -5367,6 +6510,19 @@ class Assistant extends adapter_core_1.Adapter {
5367
6510
  if (provider === 'none') {
5368
6511
  return { error: 'No LLM provider selected.' };
5369
6512
  }
6513
+ if (provider === 'pi-agent') {
6514
+ return { error: 'pi-agent has its own connection test in its instance settings.' };
6515
+ }
6516
+ if (provider === 'system') {
6517
+ const llm = await (0, systemAi_1.resolveSystemLlm)(this, await (0, systemAi_1.readSystemAi)(this), true);
6518
+ if ('error' in llm) {
6519
+ return { error: llm.error };
6520
+ }
6521
+ const res = await new llm_1.LlmAgent({ ...llm, maxTokens: 16, tools: [], log: this.log }).testConnection();
6522
+ return res.ok
6523
+ ? { result: `Connection OK (${llm.provider} · ${llm.model || 'auto'})` }
6524
+ : { error: res.error || 'unknown error' };
6525
+ }
5370
6526
  const apiKey = (0, llm_1.effectiveApiKey)(provider, await (0, credentials_1.resolveApiKey)(this, cfg, {
5371
6527
  credentialType: msg.credentialType,
5372
6528
  apiKey: msg.apiKey,