iobroker.assistant 1.4.3 → 1.4.5

package/build/lib/voice/esphome.js

@@ -35,6 +35,7 @@ var __importStar = (this && this.__importStar) || (function () {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.EsphomeSatellites = exports.TIMER_EVENT = exports.DEFAULT_SPEECH_LEVEL = void 0;
 exports.speechLevelOf = speechLevelOf;
+exports.esphomeAddress = esphomeAddress;
 /**
  * ESPHome voice satellites — the fourth transport, next to the UDP server (`voiceServer.ts`) and the
  * Wyoming endpoint (`wyoming.ts`).
@@ -63,7 +64,7 @@ const esphomeProto_1 = require("./esphomeProto");
 const esphomeEntities_1 = require("./esphomeEntities");
 const vad_1 = require("./vad");
 const speechAudio_1 = require("./speechAudio");
-const stt_1 = require("./stt");
+const replyFormat_1 = require("./replyFormat");
 const captures_1 = require("./captures");
 /** `VoiceAssistantEvent` (api.proto) — the subset we emit. */
 const EVENT = {
@@ -136,8 +137,34 @@ const DEFAULT_SILENCE_MS = 900;
  */
 const LONG_REQUEST_MS = 2500;
 const LONG_PAUSE_FACTOR = 1.5;
+/**
+ * Sound needed before a loud moment counts as speech. On vitanova (2026-11-03) a single 100 ms sound at RMS 197
+ * after a false wake opened the gate (threshold 150), and Whisper made "и все просят." of the amplified silence
+ * around it. The shortest real answer, "да", lasts about 250 ms.
+ */
+const MIN_SPEECH_MS = 200;
+/**
+ * Close the utterance when the device stops sending audio without ending the stream. Seen on a ThirdReality
+ * speaker: the audio stopped after 2.7 s, and the satellite showed "listening" for the 30 s of
+ * {@link PIPELINE_TIMEOUT_MS}. A device streams every few tens of ms while the mic is open.
+ */
+const STREAM_STALL_MS = 2000;
+/** The same before the first audio arrives: a device needs a moment to open the microphone. */
+const STREAM_START_MS = 5000;
+/** How often a running capture checks for a stalled stream. */
+const STALL_CHECK_MS = 250;
 /** Silence threshold as a share of the speech threshold — keeps the default pair 250/150. */
 const SILENCE_SHARE = 0.6;
+/**
+ * The firmware answers an ERROR event with a stop request of its own (`signal_stop_`). One arriving this soon after
+ * our error belongs to it and needs no reply — the run is already over.
+ */
+const ERROR_ECHO_MS = 2000;
+/**
+ * What ends a run that produced no request. Home Assistant sends the same code when the speech recognition finds no
+ * text, and the Voice PE firmware deliberately plays no error sound for it.
+ */
+const NO_TEXT_ERROR = { code: 'stt-no-text-recognized', message: 'No text recognized' };
 /**
  * A dropped utterance whose loudest frame reached this share of the speech threshold was probably a voice
  * that was too quiet, not an empty room — worth telling the user where to lower the threshold.
@@ -148,6 +175,12 @@ function speechLevelOf(device) {
     const level = Math.round(Number(device.speechLevel));
     return Number.isFinite(level) && level > 0 ? level : 0;
 }
+/** How a configured device is identified: `ip`, or `ip:port` when it does not use the default port. */
+function esphomeAddress(device) {
+    const ip = (device.ip || '').trim().toLowerCase();
+    const port = Number(device.port) || DEFAULT_PORT;
+    return port === DEFAULT_PORT ? ip : `${ip}:${port}`;
+}
 /** `VoiceAssistantTimerEvent` in api.proto. */
 exports.TIMER_EVENT = { started: 0, updated: 1, cancelled: 2, finished: 3 };
 /**
@@ -277,13 +310,29 @@ class EsphomeConnection {
     password;
     /** Configured speech threshold, 0 = {@link DEFAULT_SPEECH_LEVEL}. */
     speechLevel;
+    /** `ip`, or `ip:port` for another port than the default — how the adapter recognises the table row. */
+    address;
+    /** The device's friendly name, for entities without a name of their own. */
+    friendlyName = '';
+    /**
+     * Number of the current pipeline run. A run whose number is no longer current was ended — the device asked to
+     * stop, started over, or went away — and whatever STT, answer or TTS is still in flight for it stays silent.
+     */
+    run = 0;
+    /** From RUN_START until we sent RUN_END: the device waits for that RUN_END. */
+    runOpen = false;
+    /** When we last sent an ERROR event (see {@link ERROR_ECHO_MS}). */
+    errorSentAt = 0;
+    /** The reply format last reported in the log, to log it again only when it changes. */
+    loggedFormat = '';
     constructor(device, opts) {
         this.opts = opts;
         this.host = device.ip.trim();
-        this.port = device.port || DEFAULT_PORT;
+        this.port = Number(device.port) || DEFAULT_PORT;
         this.room = (device.room || '').trim();
         this.password = device.password || '';
         this.speechLevel = speechLevelOf(device);
+        this.address = esphomeAddress(device);
         this.deviceName = this.host;
     }
     // ── lifecycle ───────────────────────────────────────────────────────────────────────────────────
@@ -339,6 +388,9 @@ class EsphomeConnection {
     onDisconnected() {
         this.clearTimers();
         this.abortCapture();
+        // A reply still being worked out must not land on the next connection, in the middle of another run.
+        this.run++;
+        this.runOpen = false;
         if (this.online) {
             this.opts.log.info(`ESPHome satellite offline: ${this.deviceName}`);
         }
@@ -372,11 +424,11 @@ class EsphomeConnection {
     setOffline() {
         if (this.online) {
             this.online = false;
-            this.opts.onStatus?.(this.deviceName, this.room, 'offline');
+            this.opts.onStatus?.(this.deviceName, this.room, 'offline', this.address);
         }
     }
     status(state) {
-        this.opts.onStatus?.(this.deviceName, this.room, state);
+        this.opts.onStatus?.(this.deviceName, this.room, state, this.address);
     }
     // ── protocol plumbing ───────────────────────────────────────────────────────────────────────────
     send(message) {
@@ -463,6 +515,7 @@ class EsphomeConnection {
             case 'DeviceInfoResponse': {
                 const info = message;
                 this.deviceName = info.getName() || this.host;
+                this.friendlyName = info.getFriendlyName?.() || info.getName() || '';
                 this.online = true;
                 this.opts.log.info(`ESPHome satellite online: ${this.deviceName} (${info.getModel()}) @ ${this.host}${this.room ? `, room ${this.room}` : ''}`);
                 this.status('idle');
@@ -540,7 +593,7 @@ class EsphomeConnection {
             }
             default:
                 if ((0, esphomeEntities_1.isListEntityMessage)(name)) {
-                    this.entities.addFromAnnouncement(name, message);
+                    this.entities.addFromAnnouncement(name, message, this.friendlyName);
                 }
                 else if ((0, esphomeEntities_1.isEntityStateMessage)(name)) {
                     const entity = this.entities.applyState(name, message);
@@ -584,13 +637,19 @@ class EsphomeConnection {
     // ── voice pipeline ──────────────────────────────────────────────────────────────────────────────
     onPipelineRequest(request) {
         if (!request.getStart()) {
-            this.abortCapture();
+            this.onStopRequest();
             return;
         }
         if (this.capture) {
             this.opts.log.debug(`ESPHome ${this.deviceName}: pipeline already running — ignoring wake`);
             return;
         }
+        if (this.runOpen) {
+            // The device started over while the last request was still being answered: that answer is not
+            // wanted any more, and its events would land in the middle of the new run.
+            this.opts.log.debug(`ESPHome ${this.deviceName}: new wake while answering — dropping the old reply`);
+        }
+        this.run++;
         const phrase = request.getWakeWordPhrase();
         this.opts.log.debug(`ESPHome ${this.deviceName}: wake "${phrase || '(follow-up)'}"`);
         const response = new esphomeProto_1.pb.VoiceAssistantResponse();
@@ -599,9 +658,11 @@ class EsphomeConnection {
         this.send(response);
         const timer = setTimeout(() => {
             this.opts.log.warn(`ESPHome ${this.deviceName}: no end of speech within ${PIPELINE_TIMEOUT_MS} ms`);
-            void this.finishCapture();
+            void this.finishCapture('pipeline-timeout');
         }, PIPELINE_TIMEOUT_MS);
         timer.unref();
+        const stallCheck = setInterval(() => this.checkStall(), STALL_CHECK_MS);
+        stallCheck.unref();
         const silenceMs = this.opts.silenceMs || DEFAULT_SILENCE_MS;
         this.capture = {
             chunks: [],
@@ -617,10 +678,14 @@ class EsphomeConnection {
                 startTimeoutMs: phrase ? NO_SPEECH_MS : NO_SPEECH_FOLLOW_UP_MS,
                 minStartLevel: this.speechLevel || exports.DEFAULT_SPEECH_LEVEL,
                 minEndLevel: Math.round((this.speechLevel || exports.DEFAULT_SPEECH_LEVEL) * SILENCE_SHARE),
+                minSpeechMs: MIN_SPEECH_MS,
             }),
             conversationId: request.getConversationId(),
             wakeWord: phrase || '',
             timer,
+            stallCheck,
+            startedAt: Date.now(),
+            lastAudioAt: 0,
             closed: false,
         };
         this.status('listening');
@@ -631,6 +696,29 @@ class EsphomeConnection {
         this.event(EVENT.STT_START);
         this.event(EVENT.STT_VAD_START);
     }
+    /**
+     * The device asks to end the conversation (`VoiceAssistantRequest{start:false}`): a button, or the wake word
+     * said again while a request is running — that is how ESPHome's voice_assistant stops. While it waits for an
+     * answer, the firmware leaves that state only on our RUN_END; without one it stays "in the conversation", and
+     * every further wake word just sends another stop (field report 2026-11-03).
+     */
+    onStopRequest() {
+        const running = !!this.capture || this.runOpen;
+        this.abortCapture();
+        if (running) {
+            this.run++; // whatever is still being answered stays silent
+            this.opts.log.debug(`ESPHome ${this.deviceName}: the device ended the conversation`);
+            this.event(EVENT.RUN_END);
+            this.status('idle');
+            return;
+        }
+        if (Date.now() - this.errorSentAt < ERROR_ECHO_MS) {
+            return; // the device's own reaction to our ERROR event
+        }
+        // Nothing is open on our side, but the device still thinks so (it missed a RUN_END): end it there too.
+        this.opts.log.debug(`ESPHome ${this.deviceName}: stop outside a conversation — ending it on the device`);
+        this.event(EVENT.RUN_END);
+    }
     onAudio(audio) {
         const capture = this.capture;
         if (!capture || capture.closed) {
@@ -638,6 +726,7 @@ class EsphomeConnection {
         }
         const chunk = Buffer.from(audio.getData_asU8());
         if (chunk.length) {
+            capture.lastAudioAt = Date.now();
             capture.chunks.push(chunk);
             const end = capture.detector.push(chunk);
             if (end) {
@@ -646,8 +735,23 @@ class EsphomeConnection {
             }
         }
         if (audio.getEnd()) {
-            void this.finishCapture();
+            void this.finishCapture('stream');
+        }
+    }
+    /** End a capture whose device stopped sending audio without ending the stream. */
+    checkStall() {
+        const capture = this.capture;
+        if (!capture || capture.closed) {
+            return;
+        }
+        const now = Date.now();
+        const quietMs = capture.lastAudioAt ? now - capture.lastAudioAt : now - capture.startedAt;
+        if (quietMs < (capture.lastAudioAt ? STREAM_STALL_MS : STREAM_START_MS)) {
+            return;
         }
+        const audioMs = Math.round(capture.chunks.reduce((sum, c) => sum + c.length, 0) / 2 / (MIC_SAMPLE_RATE / 1000));
+        this.opts.log.info(`ESPHome ${this.deviceName}: the device ${capture.lastAudioAt ? `stopped sending audio after ${(audioMs / 1000).toFixed(2)} s` : 'sent no audio'} without ending the stream — closing the utterance after ${(quietMs / 1000).toFixed(1)} s of nothing`);
+        void this.finishCapture('stall');
     }
     /** Hand a recording that never became a request to {@link EsphomeSatellitesOptions.onDiscarded}. */
     discard(capture, pcm, reason, vad) {
@@ -679,11 +783,21 @@ class EsphomeConnection {
         capture.closed = true;
         this.capture = null;
         clearTimeout(capture.timer);
+        clearInterval(capture.stallCheck);
+        const run = this.run;
+        /** False once the run was ended elsewhere — then nothing more may be sent for it. */
+        const current = () => {
+            if (run === this.run) {
+                return true;
+            }
+            this.opts.log.debug(`ESPHome ${this.deviceName}: conversation ended meanwhile — dropping the reply`);
+            return false;
+        };
         // Send this first: the device streams until it sees it.
         this.event(EVENT.STT_VAD_END);
         const pcm = Buffer.concat(capture.chunks);
         const seconds = pcm.length / 2 / MIC_SAMPLE_RATE;
-        const vad = { ...capture.detector.report(), end: end ?? 'stream' };
+        const vad = { ...capture.detector.report(), end };
         const startedAt = vad.speechStartMs !== undefined ? `, speech from ${(vad.speechStartMs / 1000).toFixed(2)} s` : '';
         const endedBy = vad.end === 'speech-end' ? `ended after ${vad.silenceMs} ms of silence` : `ended by ${vad.end}`;
         this.opts.log.debug(`ESPHome ${this.deviceName}: captured ${seconds.toFixed(2)} s ` +
@@ -692,7 +806,12 @@ class EsphomeConnection {
         if (!pcm.length || !capture.detector.sawSpeech) {
             const loudest = Math.round(capture.detector.analysedPeak);
             const gate = Math.round(capture.detector.startLevel);
-            if (pcm.length && loudest >= gate * QUIET_VOICE_SHARE) {
+            if (capture.detector.longestSoundMs > 0) {
+                // Something crossed the gate, but too briefly for a word: a click, a knock, a cough.
+                this.opts.log.debug(`ESPHome ${this.deviceName}: only a short sound (${capture.detector.longestSoundMs} ms, ` +
+                    `RMS ${loudest}) — too short for speech (${MIN_SPEECH_MS} ms), dropping the utterance`);
+            }
+            else if (pcm.length && loudest >= gate * QUIET_VOICE_SHARE) {
                 // Not an empty room: something came close to the gate. Typically a microphone that is quieter
                 // than the default threshold expects (a phone) — say where to change it.
                 this.opts.log.info(`ESPHome ${this.deviceName}: dropped the utterance — the loudest sound (RMS ${loudest}) stayed ` +
@@ -703,13 +822,15 @@ class EsphomeConnection {
                 this.opts.log.debug(`ESPHome ${this.deviceName}: nothing but silence${end === 'no-speech' ? ' (nobody started speaking)' : ''} — dropping the utterance`);
             }
             this.discard(capture, pcm, end === 'no-speech' ? 'no-speech' : 'silence', vad);
-            this.event(EVENT.RUN_END);
-            this.status('idle');
+            this.endWithoutRequest();
             return;
         }
         this.status('processing');
         try {
             const hints = this.opts.getHints ? await this.opts.getHints() : undefined;
+            if (!current()) {
+                return;
+            }
             const speech = (0, speechAudio_1.prepareForStt)(pcm, MIC_SAMPLE_RATE, { startMs: vad.speechStartMs, endLevel: vad.endLevel });
             vad.stt = { fromMs: speech.fromMs, toMs: speech.toMs, gain: speech.gain };
             this.opts.log.debug(`ESPHome ${this.deviceName}: speech recognition gets ${(speech.fromMs / 1000).toFixed(2)}–` +
@@ -717,16 +838,19 @@ class EsphomeConnection {
             const sttStart = Date.now();
             const text = (await this.opts.stt.transcribe(speech.pcm, MIC_SAMPLE_RATE, this.opts.language, hints)).trim();
             const sttMs = Date.now() - sttStart;
-            this.event(EVENT.STT_END, { text });
+            if (!current()) {
+                return;
+            }
             if (!text) {
                 // Speech was heard, so the user waits for an answer: say why none comes. Typically a word in another
                 // language than the STT listens in (a German "Ja" to a Russian STT comes back empty).
                 this.opts.log.info(`Voice Q (${this.deviceName}): (empty) — speech recognition returned no text for ${seconds.toFixed(1)} s of audio`);
                 this.discard(capture, pcm, 'empty-stt', vad);
-                this.event(EVENT.RUN_END);
-                this.status('idle');
+                // No STT_END without text — like Home Assistant, which reports this case as an error only.
+                this.endWithoutRequest();
                 return;
             }
+            this.event(EVENT.STT_END, { text });
             this.event(EVENT.INTENT_START);
             const answer = (await this.opts.answer(text, {
                 device: this.deviceName,
@@ -737,6 +861,9 @@ class EsphomeConnection {
                 // Numbers only, no audio: shows where a request was cut off without keeping recordings.
                 vad: { ...vad, levels: (0, captures_1.levelProfile)(pcm, MIC_SAMPLE_RATE) },
             })).trim();
+            if (!current()) {
+                return;
+            }
             const followUp = !!answer && this.opts.expectsFollowUp?.(answer, this.deviceName) === true;
             this.event(EVENT.INTENT_END, followUp ? { continue_conversation: '1' } : undefined);
             if (!answer) {
@@ -746,11 +873,17 @@ class EsphomeConnection {
             }
             const ttsStart = Date.now();
             const { pcm: ttsPcm, sampleRate } = await this.opts.tts.synthesize(answer, this.opts.language);
+            if (!current()) {
+                return;
+            }
             this.opts.onSpoken?.(this.deviceName, {
                 ms: Date.now() - ttsStart,
                 audioMs: (ttsPcm.length / 2 / sampleRate) * 1000,
             });
-            const url = this.publish(ttsPcm, sampleRate);
+            const url = await this.publish(ttsPcm, sampleRate);
+            if (!current()) {
+                return;
+            }
             this.status('speaking');
             this.event(EVENT.TTS_START, { text: answer });
             this.event(EVENT.TTS_END, { url });
@@ -760,12 +893,26 @@ class EsphomeConnection {
         catch (e) {
             const reason = e.message;
             this.opts.log.error(`ESPHome ${this.deviceName}: voice pipeline failed — ${reason}`);
+            if (run !== this.run) {
+                return; // the conversation is over already — nothing to tell the device
+            }
             // Without this the device stays ducked and its LED keeps spinning.
             this.event(EVENT.ERROR, { code: 'pipeline-failed', message: reason });
             this.event(EVENT.RUN_END);
             this.status('idle');
         }
     }
+    /**
+     * End a run that produced no request. A RUN_END right after STT_VAD_END gets lost: the firmware is still stopping
+     * its microphone then, ignores RUN_END in that state, and afterwards waits for an answer that never comes — the
+     * satellite stays in the conversation (field report 2026-11-03). An ERROR event puts it back to idle in any state,
+     * so it goes first, with the code Home Assistant uses for this case.
+     */
+    endWithoutRequest() {
+        this.event(EVENT.ERROR, NO_TEXT_ERROR);
+        this.event(EVENT.RUN_END);
+        this.status('idle');
+    }
     /**
      * Select which wake words listen on this device. Unknown ids are dropped and the list is trimmed
      * to what the firmware allows, because the device answers a rejected list with silence rather than
@@ -835,18 +982,19 @@ class EsphomeConnection {
         this.capture = null;
         capture.closed = true;
         clearTimeout(capture.timer);
+        clearInterval(capture.stallCheck);
     }
     // ── announcements ───────────────────────────────────────────────────────────────────────────────
     /**
      * Speak a ready PCM buffer on this device (timers, alarms, `tts.text`, jingles). With
      * `startConversation` the microphone opens once the clip has played, to catch the answer to a question.
      */
-    announce(pcm, sampleRate, startConversation = false) {
+    async announce(pcm, sampleRate, startConversation = false) {
         if (!this.online || !pcm.length) {
-            return Promise.resolve();
+            return;
         }
         const request = new esphomeProto_1.pb.VoiceAssistantAnnounceRequest();
-        request.setMediaId(this.publish(pcm, sampleRate));
+        request.setMediaId(await this.publish(pcm, sampleRate));
         request.setText('');
         request.setPreannounceMediaId('');
         // Needs feature flag 32 (START_CONVERSATION); on a device without it the clip still plays and
@@ -855,16 +1003,45 @@ class EsphomeConnection {
         this.status('speaking');
         this.send(request);
         this.armPlaybackTimeout();
-        return Promise.resolve();
+    }
+    /** The format replies should have on this device, from what its media player announced. */
+    replyFormat() {
+        const formats = this.entities
+            .list()
+            .filter(e => e.kind === 'mediaPlayer')
+            .flatMap(e => e.formats ?? []);
+        return (0, replyFormat_1.chooseReplyFormat)(formats, !!this.opts.encodeAudio);
     }
     /**
-     * Publish PCM as a WAV on the media server and build the URL the device should fetch.
+     * Publish PCM on the media server, in the format the device plays, and build the URL it should fetch.
      *
      * The host matters: on a multi-homed ioBroker machine the device can usually only reach one of the
      * addresses, and the local end of *this* socket is by definition one it can talk to.
      */
-    publish(pcm, sampleRate) {
-        return this.mediaUrl(this.opts.media.publish((0, stt_1.pcmToWav)(pcm, sampleRate), 'audio/wav'));
+    async publish(pcm, sampleRate) {
+        let format = this.replyFormat();
+        let encoded;
+        try {
+            encoded = await (0, replyFormat_1.encodeReply)(pcm, sampleRate, format, this.opts.encodeAudio);
+        }
+        catch (e) {
+            // MP3/Opus need ffmpeg. Without it a native format is the only chance, whether the device lists it or not.
+            const fallback = { ...format, codec: 'flac', source: `${format.source}, FLAC as fallback` };
+            this.opts.log.warn(`ESPHome ${this.deviceName}: cannot write the reply as ${format.codec.toUpperCase()} ` +
+                `(${e.message}) — sending FLAC, which the device may not play. Install ffmpeg.`);
+            format = fallback;
+            encoded = await (0, replyFormat_1.encodeReply)(pcm, sampleRate, format);
+        }
+        const key = `${format.codec}/${format.sampleRate}/${format.channels}`;
+        if (key !== this.loggedFormat) {
+            this.loggedFormat = key;
+            const what = !format.source
+                ? 'WAV (the device lists no formats)'
+                : `${format.codec.toUpperCase()}${format.sampleRate ? ` ${format.sampleRate} Hz` : ''}` +
+                    `${format.channels === 2 ? ' stereo' : ''} (device: ${format.source})`;
+            this.opts.log.info(`ESPHome ${this.deviceName}: spoken replies as ${what}`);
+        }
+        return this.mediaUrl(this.opts.media.publish(encoded.body, encoded.contentType));
     }
     /** Absolute URL of a path on the media server, with a host this device can reach. */
     mediaUrl(path) {
@@ -886,6 +1063,15 @@ class EsphomeConnection {
         this.playbackTimer.unref();
     }
     event(type, data) {
+        if (type === EVENT.RUN_START) {
+            this.runOpen = true;
+        }
+        else if (type === EVENT.RUN_END) {
+            this.runOpen = false;
+        }
+        else if (type === EVENT.ERROR) {
+            this.errorSentAt = Date.now();
+        }
         const message = new esphomeProto_1.pb.VoiceAssistantEventResponse();
         message.setEventType(type);
         const items = [];