npm - iobroker.assistant - Versions diffs - 1.4.2 → 1.4.4 - Mend

iobroker.assistant 1.4.2 → 1.4.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (9) hide show

package/README.md +7 -0
package/build/lib/voice/esphome.js +49 -4
package/build/lib/voice/esphome.js.map +1 -1
package/build/lib/voice/vad.js +42 -5
package/build/lib/voice/vad.js.map +1 -1
package/build/main.js +36 -2
package/build/main.js.map +1 -1
package/io-package.json +27 -27
package/package.json +1 -1

package/build/lib/voice/vad.js CHANGED Viewed

@@ -61,6 +61,8 @@ class SpeechDetector {
     silenceMs;
     longSilenceMs;
     longAfterMs;
+    minSpeechMs;
+    speechGapMs;
     maxMs;
     startTimeoutMs;
     skipMs;
@@ -79,6 +81,14 @@ class SpeechDetector {
     elapsedMs = 0;
     /** Start of the frame that crossed {@link startLevel}, in ms from the start of the recording; -1 = not yet. */
     speechStartMs = -1;
+    /** A crossing of {@link startLevel} that has not yet lasted {@link minSpeechMs}: its start, -1 = none. */
+    candidateMs = -1;
+    /** Quiet since the last loud frame of the candidate. */
+    candidateQuietMs = 0;
+    /** Sound above {@link endLevel} since the candidate (and then the speech) started, in ms. */
+    voicedMs = 0;
+    /** The longest sound that crossed {@link startLevel}, in ms — also one too short to count as speech. */
+    longestSoundMs = 0;
     /** Audio analysed so far, in milliseconds (exposed for logging). */
     totalMs = 0;
     /** Loudest frame seen so far (exposed for logging — the usual clue when a mic is mis-configured). */
@@ -88,7 +98,7 @@ class SpeechDetector {
      * it tells whether a dropped utterance was silence or a voice that was simply too quiet for the gate.
      */
     analysedPeak = 0;
-    constructor({ sampleRate = 16000, frameMs = 30, startLevel = 700, endLevel = 400, silenceMs = 900, longSilenceMs = 0, longAfterMs = 0, maxMs = 12000, startTimeoutMs = 0, skipMs = 0, adaptive = false, minStartLevel = 250, minEndLevel = 150, startFactor = 3.5, endFactor = 2, warmupMs = 150, } = {}) {
+    constructor({ sampleRate = 16000, frameMs = 30, startLevel = 700, endLevel = 400, silenceMs = 900, longSilenceMs = 0, longAfterMs = 0, minSpeechMs = 0, speechGapMs = 300, maxMs = 12000, startTimeoutMs = 0, skipMs = 0, adaptive = false, minStartLevel = 250, minEndLevel = 150, startFactor = 3.5, endFactor = 2, warmupMs = 150, } = {}) {
         this.frameBytes = Math.round((sampleRate * frameMs) / 1000) * 2;
         this.frameMs = frameMs;
         this.fixedStart = startLevel;
@@ -96,6 +106,8 @@ class SpeechDetector {
         this.silenceMs = silenceMs;
         this.longSilenceMs = longSilenceMs;
         this.longAfterMs = longAfterMs;
+        this.minSpeechMs = minSpeechMs;
+        this.speechGapMs = speechGapMs;
         this.maxMs = maxMs;
         this.startTimeoutMs = startTimeoutMs;
         this.skipMs = skipMs;
@@ -145,6 +157,7 @@ class SpeechDetector {
             startLevel: Math.round(this.startLevel),
             endLevel: Math.round(this.endLevel),
             speechStartMs: this.speechSeen ? this.speechStartMs : undefined,
+            voicedMs: this.speechSeen ? this.voicedMs : undefined,
             silenceMs: this.requiredSilenceMs,
         };
     }
@@ -184,7 +197,7 @@ class SpeechDetector {
                         ? 0
                         : level < this.floor
                             ? level
-                            : this.speechSeen
+                            : this.speechSeen || this.candidateMs >= 0
                                 ? this.floor
                                 : this.floor + (level - this.floor) * NOISE_RISE;
                 if (this.totalMs < this.warmupMs) {
@@ -192,10 +205,33 @@ class SpeechDetector {
                 }
             }
             if (!this.speechSeen) {
-                if (level >= this.startLevel) {
+                // A loud frame opens a candidate; it becomes speech once it has lasted `minSpeechMs`, and is
+                // forgotten when it falls quiet for `speechGapMs` before that — a click is not a request.
+                if (this.candidateMs < 0) {
+                    if (level >= this.startLevel) {
+                        this.candidateMs = this.elapsedMs - this.frameMs;
+                        this.candidateQuietMs = 0;
+                        this.voicedMs = this.frameMs;
+                        this.longestSoundMs = Math.max(this.longestSoundMs, this.voicedMs);
+                    }
+                }
+                else if (level >= this.endLevel) {
+                    this.voicedMs += this.frameMs;
+                    this.candidateQuietMs = 0;
+                    this.longestSoundMs = Math.max(this.longestSoundMs, this.voicedMs);
+                }
+                else {
+                    this.candidateQuietMs += this.frameMs;
+                    if (this.candidateQuietMs >= this.speechGapMs) {
+                        this.candidateMs = -1;
+                        this.voicedMs = 0;
+                    }
+                }
+                if (this.candidateMs >= 0 && this.voicedMs >= this.minSpeechMs) {
                     this.speechSeen = true;
-                    this.speechStartMs = this.elapsedMs - this.frameMs;
-                    this.silentMs = 0;
+                    this.speechStartMs = this.candidateMs;
+                    // A gap inside the candidate already counts towards the end of the utterance.
+                    this.silentMs = this.candidateQuietMs;
                 }
             }
             else if (level < this.endLevel) {
@@ -206,6 +242,7 @@ class SpeechDetector {
             }
             else {
                 this.silentMs = 0;
+                this.voicedMs += this.frameMs;
             }
             if (!this.speechSeen && this.startTimeoutMs > 0 && this.elapsedMs >= this.startTimeoutMs) {
                 verdict ||= 'no-speech';

package/build/lib/voice/vad.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"vad.js","sourceRoot":"","sources":["../../../src/lib/voice/vad.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;;;AASH,kBAWC;AAZD,kFAAkF;AAClF,SAAgB,GAAG,CAAC,GAAU;IAC1B,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC3C,IAAI,CAAC,OAAO,EAAE,CAAC;QACX,OAAO,CAAC,CAAC;IACb,CAAC;IACD,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,GAAG,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACrC,GAAG,IAAI,KAAK,GAAG,KAAK,CAAC;IACzB,CAAC;IACD,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,CAAC;AACpC,CAAC;AAwED;;;;GAIG;AACH,MAAM,UAAU,GAAG,IAAI,CAAC;AAExB,MAAa,cAAc;IACN,UAAU,CAAS;IACnB,OAAO,CAAS;IAChB,UAAU,CAAS;IACnB,QAAQ,CAAS;IACjB,SAAS,CAAS;IAClB,aAAa,CAAS;IACtB,WAAW,CAAS;IACpB,KAAK,CAAS;IACd,cAAc,CAAS;IACvB,MAAM,CAAS;IACf,QAAQ,CAAU;IAClB,aAAa,CAAS;IACtB,WAAW,CAAS;IACpB,WAAW,CAAS;IACpB,SAAS,CAAS;IAClB,QAAQ,CAAS;IAE1B,OAAO,GAAU,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACjC,UAAU,GAAG,KAAK,CAAC;IACnB,QAAQ,GAAG,CAAC,CAAC;IACrB,gFAAgF;IACxE,KAAK,GAAkB,IAAI,CAAC;IACpC,0EAA0E;IAClE,SAAS,GAAG,CAAC,CAAC;IACtB,+GAA+G;IAC/G,aAAa,GAAG,CAAC,CAAC,CAAC;IACnB,oEAAoE;IACpE,OAAO,GAAG,CAAC,CAAC;IACZ,qGAAqG;IACrG,IAAI,GAAG,CAAC,CAAC;IACT;;;OAGG;IACH,YAAY,GAAG,CAAC,CAAC;IAEjB,YAAY,EACR,UAAU,GAAG,KAAK,EAClB,OAAO,GAAG,EAAE,EACZ,UAAU,GAAG,GAAG,EAChB,QAAQ,GAAG,GAAG,EACd,SAAS,GAAG,GAAG,EACf,aAAa,GAAG,CAAC,EACjB,WAAW,GAAG,CAAC,EACf,KAAK,GAAG,KAAK,EACb,cAAc,GAAG,CAAC,EAClB,MAAM,GAAG,CAAC,EACV,QAAQ,GAAG,KAAK,EAChB,aAAa,GAAG,GAAG,EACnB,WAAW,GAAG,GAAG,EACjB,WAAW,GAAG,GAAG,EACjB,SAAS,GAAG,CAAC,EACb,QAAQ,GAAG,GAAG,MACS,EAAE;QACzB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,UAAU,GAAG,OAAO,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;QAChE,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACrC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC7B,CAAC;IAED,kGAAkG;IAClG,IAAI,SAAS;QACT,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;IAED,0FAA0F;IAC1F,IAAI,UAAU;QACV,OAAO,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;IAC3B,CAAC;IAED,0FAA0F;IAC1F,IAAI,UAAU;QACV,OAAO,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC;IAChH,CAAC;IAED,mGAAmG;IACnG,IAAI,QAAQ;QACR,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACjB,OAAO,IAAI,CAAC,QAAQ,CAAC;QACzB,CAAC;QACD,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;IACrG,CAAC;IAED;;;OAGG;IACH,IAAI,iBAAiB;QACjB,IAAI,CAAC,IAAI,CAAC,UAAU,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YAChF,OAAO,IAAI,CAAC,SAAS,CAAC;QAC1B,CAAC;QACD,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,aAAa,CAAC;QACrE,OAAO,QAAQ,IAAI,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC;IAC9E,CAAC;IAED,mDAAmD;IACnD,MAAM;QACF,OAAO;YACH,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;YAC3B,YAAY,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC;YAC3C,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC;YACvC,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC;YACvC,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC;YACnC,aAAa,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS;YAC/D,SAAS,EAAE,IAAI,CAAC,iBAAiB;SACpC,CAAC;IACN,CAAC;IAED,mGAAmG;IACnG,IAAI,CAAC,KAAY;QACb,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QAClF,IAAI,OAAO,GAAqB,IAAI,CAAC;QAErC,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YAC5C,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;YACxD,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACtD,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC;YAE/B,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC;YACzB,IAAI,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;gBACpB,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC;YACtB,CAAC;YAED,oFAAoF;YACpF,sFAAsF;YACtF,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBAChC,SAAS;YACb,CAAC;YACD,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,CAAC;YAC7B,IAAI,KAAK,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;gBAC5B,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;YAC9B,CAAC;YAED,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAChB,0EAA0E;gBAC1E,mFAAmF;gBACnF,oFAAoF;gBACpF,sFAAsF;gBACtF,qDAAqD;gBACrD,oEAAoE;gBACpE,mFAAmF;gBACnF,qFAAqF;gBACrF,yFAAyF;gBACzF,IAAI,CAAC,KAAK;oBACN,IAAI,CAAC,KAAK,KAAK,IAAI;wBACf,CAAC,CAAC,CAAC;wBACH,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK;4BAClB,CAAC,CAAC,KAAK;4BACP,CAAC,CAAC,IAAI,CAAC,UAAU;gCACf,CAAC,CAAC,IAAI,CAAC,KAAK;gCACZ,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,UAAU,CAAC;gBAC7D,IAAI,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;oBAC/B,SAAS,CAAC,2DAA2D;gBACzE,CAAC;YACL,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;gBACnB,IAAI,KAAK,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;oBAC3B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;oBACvB,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC;oBACnD,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;gBACtB,CAAC;YACL,CAAC;iBAAM,IAAI,KAAK,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAC/B,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,OAAO,CAAC;gBAC9B,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;oBAC1C,OAAO,KAAK,YAAY,CAAC;gBAC7B,CAAC;YACL,CAAC;iBAAM,CAAC;gBACJ,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtB,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,cAAc,GAAG,CAAC,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;gBACvF,OAAO,KAAK,WAAW,CAAC;YAC5B,CAAC;YACD,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;gBAC7B,OAAO,KAAK,SAAS,CAAC;YAC1B,CAAC;QACL,CAAC;QACD,OAAO,OAAO,CAAC;IACnB,CAAC;CACJ;AAjMD,wCAiMC","sourcesContent":["/*\n End-of-speech detection for satellites that stream microphone audio until they are told to stop.\n \n The Hannah/UDP protocol has the satellite decide when an utterance ended (`audio_end`), and Wyoming\n * clients send `audio-stop` themselves. ESPHome voice satellites do neither: they keep streaming until\n * the server sends `STT_VAD_END`/`STT_END`, so the decision has to happen here.\n \n This is a plain energy gate, not a neural VAD — good enough to close an utterance, and it costs\n * nothing per frame. Speech must first rise above `startLevel`; after that, `silenceMs` worth of frames\n * below `endLevel` ends it. `maxMs` is the hard stop for rooms that are simply loud.\n \n Fixed thresholds only work when the microphone level is known, and on a real satellite it is not.\n * Measured on a ThirdReality Voice & Music Assistant: the device plays a wake-acknowledgement chirp\n * into its own microphone that peaks at full scale (RMS ~16000, clipped), while the speech that follows\n * arrives at RMS 358–469 — below the 700 default for `startLevel` and right on the 400 for `endLevel`.\n * That breaks the gate twice over: `sawSpeech` is satisfied by the chirp rather than by any voice, so\n * the \"nothing but speech-free audio\" guard never fires, and half the real speech frames count as\n * silence, so the utterance can be cut off mid-sentence.\n \n Hence two opt-in knobs, both enabled by the ESPHome transport:\n \n - `skipMs` drops the leading chirp from the analysis only. What reaches STT is cut later, from where\n * speech was detected minus a margin (`speechAudio.ts`) — a fast speaker may already be talking.\n * - `adaptive` derives the thresholds from a tracked noise floor instead of fixed levels. The floor\n * follows the quietest frame seen (instant down, very slow up), so it settles on room noise even when\n * someone starts talking immediately — which a mean or median over a calibration window would not.\n \n And one for longer requests: `longSilenceMs` after `longAfterMs` of speech. In a quiet room the silence\n * threshold sits at its minimum, and a voice a few metres away dips below it between syllables — on a\n * ThirdReality speaker a whole \"Мессенджер\" (0.7 s) stayed under 150. A pause for thought on top of that\n * ended a long request mid-sentence, while short commands should keep closing quickly.\n /\n\n/\n A buffer that may be a view onto any `ArrayBufferLike` — what `Buffer.subarray()` and\n * `Buffer.concat()` hand back. Plain `Buffer` (= `Buffer<ArrayBuffer>`) assigns to it, not the reverse.\n /\nexport type Bytes = Buffer<ArrayBufferLike>;\n\n/* Root-mean-square amplitude (0…32767) of a mono 16-bit signed LE PCM buffer. /\nexport function rms(pcm: Bytes): number {\n const samples = Math.floor(pcm.length / 2);\n if (!samples) {\n return 0;\n }\n let sum = 0;\n for (let i = 0; i < samples; i++) {\n const value = pcm.readInt16LE(i 2);\n sum += value * value;\n }\n return Math.sqrt(sum / samples);\n}\n\nexport interface SpeechDetectorOptions {\n /** Sample rate of the incoming PCM (ESPHome satellites always capture at 16 kHz). /\n sampleRate?: number;\n /* Analysis window; 30 ms is the usual compromise between latency and stability. /\n frameMs?: number;\n /* RMS above which we consider speech to have started. Ignored when `adaptive` is on. /\n startLevel?: number;\n /* RMS below which a frame counts as silence. Ignored when `adaptive` is on. /\n endLevel?: number;\n /* Silence after speech that ends the utterance. /\n silenceMs?: number;\n /\n Silence that ends the utterance once the speaker has talked for {@link longAfterMs} (0 = always\n * {@link silenceMs}). Somebody dictating a longer request pauses to think, and a short command should\n * still close fast.\n /\n longSilenceMs?: number;\n /* Speech (from its start to the pause) after which {@link longSilenceMs} applies; 0 = never. /\n longAfterMs?: number;\n /* Hard cap on one utterance, in case the room never falls quiet. /\n maxMs?: number;\n /\n Give up when no speech has started this long after the stream began (the skipped lead-in included);\n * 0 = only {@link maxMs}. A false wake would otherwise keep the microphone open for the whole `maxMs`.\n /\n startTimeoutMs?: number;\n /\n Leading audio to leave out of the analysis — for devices that chirp into their own microphone\n * when the wake word fires. The audio itself is untouched; this only stops the chirp from being\n * mistaken for speech and from poisoning the noise floor.\n /\n skipMs?: number;\n /* Derive the thresholds from the tracked noise floor instead of using fixed levels. /\n adaptive?: boolean;\n /* Adaptive mode: `startLevel` is at least this, however quiet the room is. /\n minStartLevel?: number;\n /* Adaptive mode: `endLevel` is at least this. /\n minEndLevel?: number;\n /* Adaptive mode: speech threshold as a multiple of the noise floor. /\n startFactor?: number;\n /* Adaptive mode: silence threshold as a multiple of the noise floor. /\n endFactor?: number;\n /* Adaptive mode: settle the noise floor this long before speech may be declared. /\n warmupMs?: number;\n}\n\n/* Why an utterance ended: the speaker stopped, we hit the hard cap, or nobody started speaking at all. /\nexport type SpeechEnd = 'speech-end' \| 'timeout' \| 'no-speech';\n\n/* What the detector made of one recording — for the request trace and the diagnosis recordings. /\nexport interface VadReport {\n /* Loudest frame, the wake chirp included. /\n peak: number;\n /* Loudest frame after the skipped lead-in. /\n analysedPeak: number;\n noiseFloor: number;\n startLevel: number;\n endLevel: number;\n /* Where speech began, in ms from the start of the recording; absent when it never did. /\n speechStartMs?: number;\n /* Why the recording ended; 'stream' = the device stopped streaming by itself. /\n end?: SpeechEnd \| 'stream';\n /* Silence that was needed to end it at that point (longer once the speaker talked for a while). /\n silenceMs?: number;\n /* RMS per 100 ms of the whole recording. /\n levels?: number[];\n /* What went to speech recognition: the recording cut to the speech, and the gain applied to it. /\n stt?: { fromMs: number; toMs: number; gain: number };\n}\n\n/\n How fast the tracked noise floor may follow a rising ambient level, per analysed frame (30 ms).\n * Fast enough that a genuinely noisy room is matched within about a second, slow enough that a voice\n * cannot pull the floor up to its own level before it has been recognised as speech.\n /\nconst NOISE_RISE = 0.02;\n\nexport class SpeechDetector {\n private readonly frameBytes: number;\n private readonly frameMs: number;\n private readonly fixedStart: number;\n private readonly fixedEnd: number;\n private readonly silenceMs: number;\n private readonly longSilenceMs: number;\n private readonly longAfterMs: number;\n private readonly maxMs: number;\n private readonly startTimeoutMs: number;\n private readonly skipMs: number;\n private readonly adaptive: boolean;\n private readonly minStartLevel: number;\n private readonly minEndLevel: number;\n private readonly startFactor: number;\n private readonly endFactor: number;\n private readonly warmupMs: number;\n\n private pending: Bytes = Buffer.alloc(0);\n private speechSeen = false;\n private silentMs = 0;\n /* Tracked room level in adaptive mode; null until the first analysed frame. /\n private floor: number \| null = null;\n /* Audio seen so far including the skipped lead-in, in milliseconds. /\n private elapsedMs = 0;\n /* Start of the frame that crossed {@link startLevel}, in ms from the start of the recording; -1 = not yet. /\n speechStartMs = -1;\n /* Audio analysed so far, in milliseconds (exposed for logging). /\n totalMs = 0;\n /* Loudest frame seen so far (exposed for logging — the usual clue when a mic is mis-configured). /\n peak = 0;\n /\n Loudest frame after the skipped lead-in, i.e. without the wake chirp. Compared with {@link startLevel}\n * it tells whether a dropped utterance was silence or a voice that was simply too quiet for the gate.\n /\n analysedPeak = 0;\n\n constructor({\n sampleRate = 16000,\n frameMs = 30,\n startLevel = 700,\n endLevel = 400,\n silenceMs = 900,\n longSilenceMs = 0,\n longAfterMs = 0,\n maxMs = 12000,\n startTimeoutMs = 0,\n skipMs = 0,\n adaptive = false,\n minStartLevel = 250,\n minEndLevel = 150,\n startFactor = 3.5,\n endFactor = 2,\n warmupMs = 150,\n }: SpeechDetectorOptions = {}) {\n this.frameBytes = Math.round((sampleRate frameMs) / 1000) * 2;\n this.frameMs = frameMs;\n this.fixedStart = startLevel;\n this.fixedEnd = endLevel;\n this.silenceMs = silenceMs;\n this.longSilenceMs = longSilenceMs;\n this.longAfterMs = longAfterMs;\n this.maxMs = maxMs;\n this.startTimeoutMs = startTimeoutMs;\n this.skipMs = skipMs;\n this.adaptive = adaptive;\n this.minStartLevel = minStartLevel;\n this.minEndLevel = minEndLevel;\n this.startFactor = startFactor;\n this.endFactor = endFactor;\n this.warmupMs = warmupMs;\n }\n\n /** True once any frame crossed {@link startLevel} — an utterance that never did is room noise. /\n get sawSpeech(): boolean {\n return this.speechSeen;\n }\n\n /* Tracked room level (adaptive mode), for logging. 0 before the first analysed frame. /\n get noiseFloor(): number {\n return this.floor ?? 0;\n }\n\n /* Current speech threshold — fixed, or derived from the noise floor in adaptive mode. /\n get startLevel(): number {\n return this.adaptive ? Math.max(this.minStartLevel, (this.floor ?? 0) this.startFactor) : this.fixedStart;\n }\n\n /** Current silence threshold. Never above {@link startLevel}, so the gate keeps its hysteresis. /\n get endLevel(): number {\n if (!this.adaptive) {\n return this.fixedEnd;\n }\n return Math.min(this.startLevel, Math.max(this.minEndLevel, (this.floor ?? 0) this.endFactor));\n }\n\n /*\n Silence that ends the utterance right now: {@link silenceMs}, or {@link longSilenceMs} once the speaker\n * has talked for {@link longAfterMs} before the current pause.\n /\n get requiredSilenceMs(): number {\n if (!this.speechSeen \|\| !this.longAfterMs \|\| this.longSilenceMs <= this.silenceMs) {\n return this.silenceMs;\n }\n const spokenMs = this.elapsedMs - this.silentMs - this.speechStartMs;\n return spokenMs >= this.longAfterMs ? this.longSilenceMs : this.silenceMs;\n }\n\n /* The detector's view of the recording so far. /\n report(): VadReport {\n return {\n peak: Math.round(this.peak),\n analysedPeak: Math.round(this.analysedPeak),\n noiseFloor: Math.round(this.noiseFloor),\n startLevel: Math.round(this.startLevel),\n endLevel: Math.round(this.endLevel),\n speechStartMs: this.speechSeen ? this.speechStartMs : undefined,\n silenceMs: this.requiredSilenceMs,\n };\n }\n\n /* Feed one audio chunk (any size). Returns why the utterance ended, or null to keep listening. /\n push(chunk: Bytes): SpeechEnd \| null {\n this.pending = this.pending.length ? Buffer.concat([this.pending, chunk]) : chunk;\n let verdict: SpeechEnd \| null = null;\n\n while (this.pending.length >= this.frameBytes) {\n const frame = this.pending.subarray(0, this.frameBytes);\n this.pending = this.pending.subarray(this.frameBytes);\n this.elapsedMs += this.frameMs;\n\n const level = rms(frame);\n if (level > this.peak) {\n this.peak = level;\n }\n\n // The device's own wake chirp: loud, clipped, and not speech. Leaving it out of the\n // analysis is what makes `sawSpeech` mean anything, and keeps the noise floor honest.\n if (this.elapsedMs <= this.skipMs) {\n continue;\n }\n this.totalMs += this.frameMs;\n if (level > this.analysedPeak) {\n this.analysedPeak = level;\n }\n\n if (this.adaptive) {\n // Three rules, each there to defeat a specific way of getting this wrong:\n // - start at zero, i.e. maximally sensitive. Seeding from the first frame instead\n // lets someone who talks straight after the wake word install their own voice as\n // the room's noise floor, which shuts the gate for good — the same trap an average\n // or median over a calibration window falls into.\n // - fall instantly, so a room that went quiet is believed at once.\n // - rise slowly, and not at all once speech has been recognised: otherwise a long\n // utterance drags the floor, and with it `endLevel`, up to its own level and cuts\n // itself off mid-sentence. Falling stays on, so the pauses between words still count.\n this.floor =\n this.floor === null\n ? 0\n : level < this.floor\n ? level\n : this.speechSeen\n ? this.floor\n : this.floor + (level - this.floor) NOISE_RISE;\n if (this.totalMs < this.warmupMs) {\n continue; // let the floor settle before anything may count as speech\n }\n }\n\n if (!this.speechSeen) {\n if (level >= this.startLevel) {\n this.speechSeen = true;\n this.speechStartMs = this.elapsedMs - this.frameMs;\n this.silentMs = 0;\n }\n } else if (level < this.endLevel) {\n this.silentMs += this.frameMs;\n if (this.silentMs >= this.requiredSilenceMs) {\n verdict \|\|= 'speech-end';\n }\n } else {\n this.silentMs = 0;\n }\n\n if (!this.speechSeen && this.startTimeoutMs > 0 && this.elapsedMs >= this.startTimeoutMs) {\n verdict \|\|= 'no-speech';\n }\n if (this.totalMs >= this.maxMs) {\n verdict \|\|= 'timeout';\n }\n }\n return verdict;\n }\n}\n"]}
1	+ {"version":3,"file":"vad.js","sourceRoot":"","sources":["../../../src/lib/voice/vad.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;;;AASH,kBAWC;AAZD,kFAAkF;AAClF,SAAgB,GAAG,CAAC,GAAU;IAC1B,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC3C,IAAI,CAAC,OAAO,EAAE,CAAC;QACX,OAAO,CAAC,CAAC;IACb,CAAC;IACD,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,GAAG,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACrC,GAAG,IAAI,KAAK,GAAG,KAAK,CAAC;IACzB,CAAC;IACD,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,CAAC;AACpC,CAAC;AAuFD;;;;GAIG;AACH,MAAM,UAAU,GAAG,IAAI,CAAC;AAExB,MAAa,cAAc;IACN,UAAU,CAAS;IACnB,OAAO,CAAS;IAChB,UAAU,CAAS;IACnB,QAAQ,CAAS;IACjB,SAAS,CAAS;IAClB,aAAa,CAAS;IACtB,WAAW,CAAS;IACpB,WAAW,CAAS;IACpB,WAAW,CAAS;IACpB,KAAK,CAAS;IACd,cAAc,CAAS;IACvB,MAAM,CAAS;IACf,QAAQ,CAAU;IAClB,aAAa,CAAS;IACtB,WAAW,CAAS;IACpB,WAAW,CAAS;IACpB,SAAS,CAAS;IAClB,QAAQ,CAAS;IAE1B,OAAO,GAAU,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACjC,UAAU,GAAG,KAAK,CAAC;IACnB,QAAQ,GAAG,CAAC,CAAC;IACrB,gFAAgF;IACxE,KAAK,GAAkB,IAAI,CAAC;IACpC,0EAA0E;IAClE,SAAS,GAAG,CAAC,CAAC;IACtB,+GAA+G;IAC/G,aAAa,GAAG,CAAC,CAAC,CAAC;IACnB,0GAA0G;IAClG,WAAW,GAAG,CAAC,CAAC,CAAC;IACzB,wDAAwD;IAChD,gBAAgB,GAAG,CAAC,CAAC;IAC7B,6FAA6F;IAC7F,QAAQ,GAAG,CAAC,CAAC;IACb,wGAAwG;IACxG,cAAc,GAAG,CAAC,CAAC;IACnB,oEAAoE;IACpE,OAAO,GAAG,CAAC,CAAC;IACZ,qGAAqG;IACrG,IAAI,GAAG,CAAC,CAAC;IACT;;;OAGG;IACH,YAAY,GAAG,CAAC,CAAC;IAEjB,YAAY,EACR,UAAU,GAAG,KAAK,EAClB,OAAO,GAAG,EAAE,EACZ,UAAU,GAAG,GAAG,EAChB,QAAQ,GAAG,GAAG,EACd,SAAS,GAAG,GAAG,EACf,aAAa,GAAG,CAAC,EACjB,WAAW,GAAG,CAAC,EACf,WAAW,GAAG,CAAC,EACf,WAAW,GAAG,GAAG,EACjB,KAAK,GAAG,KAAK,EACb,cAAc,GAAG,CAAC,EAClB,MAAM,GAAG,CAAC,EACV,QAAQ,GAAG,KAAK,EAChB,aAAa,GAAG,GAAG,EACnB,WAAW,GAAG,GAAG,EACjB,WAAW,GAAG,GAAG,EACjB,SAAS,GAAG,CAAC,EACb,QAAQ,GAAG,GAAG,MACS,EAAE;QACzB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,UAAU,GAAG,OAAO,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;QAChE,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACrC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC7B,CAAC;IAED,kGAAkG;IAClG,IAAI,SAAS;QACT,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;IAED,0FAA0F;IAC1F,IAAI,UAAU;QACV,OAAO,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;IAC3B,CAAC;IAED,0FAA0F;IAC1F,IAAI,UAAU;QACV,OAAO,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC;IAChH,CAAC;IAED,mGAAmG;IACnG,IAAI,QAAQ;QACR,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACjB,OAAO,IAAI,CAAC,QAAQ,CAAC;QACzB,CAAC;QACD,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;IACrG,CAAC;IAED;;;OAGG;IACH,IAAI,iBAAiB;QACjB,IAAI,CAAC,IAAI,CAAC,UAAU,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YAChF,OAAO,IAAI,CAAC,SAAS,CAAC;QAC1B,CAAC;QACD,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,aAAa,CAAC;QACrE,OAAO,QAAQ,IAAI,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC;IAC9E,CAAC;IAED,mDAAmD;IACnD,MAAM;QACF,OAAO;YACH,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;YAC3B,YAAY,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC;YAC3C,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC;YACvC,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC;YACvC,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC;YACnC,aAAa,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS;YAC/D,QAAQ,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS;YACrD,SAAS,EAAE,IAAI,CAAC,iBAAiB;SACpC,CAAC;IACN,CAAC;IAED,mGAAmG;IACnG,IAAI,CAAC,KAAY;QACb,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QAClF,IAAI,OAAO,GAAqB,IAAI,CAAC;QAErC,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YAC5C,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;YACxD,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACtD,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC;YAE/B,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC;YACzB,IAAI,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;gBACpB,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC;YACtB,CAAC;YAED,oFAAoF;YACpF,sFAAsF;YACtF,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBAChC,SAAS;YACb,CAAC;YACD,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,CAAC;YAC7B,IAAI,KAAK,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;gBAC5B,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;YAC9B,CAAC;YAED,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAChB,0EAA0E;gBAC1E,mFAAmF;gBACnF,oFAAoF;gBACpF,sFAAsF;gBACtF,qDAAqD;gBACrD,oEAAoE;gBACpE,mFAAmF;gBACnF,qFAAqF;gBACrF,yFAAyF;gBACzF,IAAI,CAAC,KAAK;oBACN,IAAI,CAAC,KAAK,KAAK,IAAI;wBACf,CAAC,CAAC,CAAC;wBACH,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK;4BAClB,CAAC,CAAC,KAAK;4BACP,CAAC,CAAC,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,WAAW,IAAI,CAAC;gCACxC,CAAC,CAAC,IAAI,CAAC,KAAK;gCACZ,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,UAAU,CAAC;gBAC7D,IAAI,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;oBAC/B,SAAS,CAAC,2DAA2D;gBACzE,CAAC;YACL,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;gBACnB,6FAA6F;gBAC7F,0FAA0F;gBAC1F,IAAI,IAAI,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;oBACvB,IAAI,KAAK,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;wBAC3B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC;wBACjD,IAAI,CAAC,gBAAgB,GAAG,CAAC,CAAC;wBAC1B,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC;wBAC7B,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;oBACvE,CAAC;gBACL,CAAC;qBAAM,IAAI,KAAK,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;oBAChC,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,OAAO,CAAC;oBAC9B,IAAI,CAAC,gBAAgB,GAAG,CAAC,CAAC;oBAC1B,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACvE,CAAC;qBAAM,CAAC;oBACJ,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC,OAAO,CAAC;oBACtC,IAAI,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;wBAC5C,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC;wBACtB,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;oBACtB,CAAC;gBACL,CAAC;gBACD,IAAI,IAAI,CAAC,WAAW,IAAI,CAAC,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;oBAC7D,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;oBACvB,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,WAAW,CAAC;oBACtC,8EAA8E;oBAC9E,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC;gBAC1C,CAAC;YACL,CAAC;iBAAM,IAAI,KAAK,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAC/B,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,OAAO,CAAC;gBAC9B,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;oBAC1C,OAAO,KAAK,YAAY,CAAC;gBAC7B,CAAC;YACL,CAAC;iBAAM,CAAC;gBACJ,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;gBAClB,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,OAAO,CAAC;YAClC,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,cAAc,GAAG,CAAC,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;gBACvF,OAAO,KAAK,WAAW,CAAC;YAC5B,CAAC;YACD,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;gBAC7B,OAAO,KAAK,SAAS,CAAC;YAC1B,CAAC;QACL,CAAC;QACD,OAAO,OAAO,CAAC;IACnB,CAAC;CACJ;AAtOD,wCAsOC","sourcesContent":["/*\n End-of-speech detection for satellites that stream microphone audio until they are told to stop.\n \n The Hannah/UDP protocol has the satellite decide when an utterance ended (`audio_end`), and Wyoming\n * clients send `audio-stop` themselves. ESPHome voice satellites do neither: they keep streaming until\n * the server sends `STT_VAD_END`/`STT_END`, so the decision has to happen here.\n \n This is a plain energy gate, not a neural VAD — good enough to close an utterance, and it costs\n * nothing per frame. Speech must first rise above `startLevel`; after that, `silenceMs` worth of frames\n * below `endLevel` ends it. `maxMs` is the hard stop for rooms that are simply loud.\n \n Fixed thresholds only work when the microphone level is known, and on a real satellite it is not.\n * Measured on a ThirdReality Voice & Music Assistant: the device plays a wake-acknowledgement chirp\n * into its own microphone that peaks at full scale (RMS ~16000, clipped), while the speech that follows\n * arrives at RMS 358–469 — below the 700 default for `startLevel` and right on the 400 for `endLevel`.\n * That breaks the gate twice over: `sawSpeech` is satisfied by the chirp rather than by any voice, so\n * the \"nothing but speech-free audio\" guard never fires, and half the real speech frames count as\n * silence, so the utterance can be cut off mid-sentence.\n \n Hence two opt-in knobs, both enabled by the ESPHome transport:\n \n - `skipMs` drops the leading chirp from the analysis only. What reaches STT is cut later, from where\n * speech was detected minus a margin (`speechAudio.ts`) — a fast speaker may already be talking.\n * - `adaptive` derives the thresholds from a tracked noise floor instead of fixed levels. The floor\n * follows the quietest frame seen (instant down, very slow up), so it settles on room noise even when\n * someone starts talking immediately — which a mean or median over a calibration window would not.\n \n And one for longer requests: `longSilenceMs` after `longAfterMs` of speech. In a quiet room the silence\n * threshold sits at its minimum, and a voice a few metres away dips below it between syllables — on a\n * ThirdReality speaker a whole \"Мессенджер\" (0.7 s) stayed under 150. A pause for thought on top of that\n * ended a long request mid-sentence, while short commands should keep closing quickly.\n /\n\n/\n A buffer that may be a view onto any `ArrayBufferLike` — what `Buffer.subarray()` and\n * `Buffer.concat()` hand back. Plain `Buffer` (= `Buffer<ArrayBuffer>`) assigns to it, not the reverse.\n /\nexport type Bytes = Buffer<ArrayBufferLike>;\n\n/* Root-mean-square amplitude (0…32767) of a mono 16-bit signed LE PCM buffer. /\nexport function rms(pcm: Bytes): number {\n const samples = Math.floor(pcm.length / 2);\n if (!samples) {\n return 0;\n }\n let sum = 0;\n for (let i = 0; i < samples; i++) {\n const value = pcm.readInt16LE(i 2);\n sum += value * value;\n }\n return Math.sqrt(sum / samples);\n}\n\nexport interface SpeechDetectorOptions {\n /** Sample rate of the incoming PCM (ESPHome satellites always capture at 16 kHz). /\n sampleRate?: number;\n /* Analysis window; 30 ms is the usual compromise between latency and stability. /\n frameMs?: number;\n /* RMS above which we consider speech to have started. Ignored when `adaptive` is on. /\n startLevel?: number;\n /* RMS below which a frame counts as silence. Ignored when `adaptive` is on. /\n endLevel?: number;\n /* Silence after speech that ends the utterance. /\n silenceMs?: number;\n /\n Silence that ends the utterance once the speaker has talked for {@link longAfterMs} (0 = always\n * {@link silenceMs}). Somebody dictating a longer request pauses to think, and a short command should\n * still close fast.\n /\n longSilenceMs?: number;\n /* Speech (from its start to the pause) after which {@link longSilenceMs} applies; 0 = never. /\n longAfterMs?: number;\n /\n Sound above the silence threshold needed before a crossing of the speech threshold counts as speech; 0 = the\n * first loud frame does. A click or a knock is over in 100 ms, the shortest \"да\" takes about 250.\n /\n minSpeechMs?: number;\n /* A candidate start that falls quiet this long before reaching {@link minSpeechMs} is forgotten. /\n speechGapMs?: number;\n /* Hard cap on one utterance, in case the room never falls quiet. /\n maxMs?: number;\n /\n Give up when no speech has started this long after the stream began (the skipped lead-in included);\n * 0 = only {@link maxMs}. A false wake would otherwise keep the microphone open for the whole `maxMs`.\n /\n startTimeoutMs?: number;\n /\n Leading audio to leave out of the analysis — for devices that chirp into their own microphone\n * when the wake word fires. The audio itself is untouched; this only stops the chirp from being\n * mistaken for speech and from poisoning the noise floor.\n /\n skipMs?: number;\n /* Derive the thresholds from the tracked noise floor instead of using fixed levels. /\n adaptive?: boolean;\n /* Adaptive mode: `startLevel` is at least this, however quiet the room is. /\n minStartLevel?: number;\n /* Adaptive mode: `endLevel` is at least this. /\n minEndLevel?: number;\n /* Adaptive mode: speech threshold as a multiple of the noise floor. /\n startFactor?: number;\n /* Adaptive mode: silence threshold as a multiple of the noise floor. /\n endFactor?: number;\n /* Adaptive mode: settle the noise floor this long before speech may be declared. /\n warmupMs?: number;\n}\n\n/* Why an utterance ended: the speaker stopped, we hit the hard cap, or nobody started speaking at all. /\nexport type SpeechEnd = 'speech-end' \| 'timeout' \| 'no-speech';\n\n/\n Why a recording ended, from the transport's view: what the detector said, the device ending the stream\n * ('stream'), the device going quiet without ending it ('stall'), or the transport's last-resort timer.\n /\nexport type CaptureEnd = SpeechEnd \| 'stream' \| 'stall' \| 'pipeline-timeout';\n\n/* What the detector made of one recording — for the request trace and the diagnosis recordings. /\nexport interface VadReport {\n /* Loudest frame, the wake chirp included. /\n peak: number;\n /* Loudest frame after the skipped lead-in. /\n analysedPeak: number;\n noiseFloor: number;\n startLevel: number;\n endLevel: number;\n /* Where speech began, in ms from the start of the recording; absent when it never did. /\n speechStartMs?: number;\n /* Why the recording ended. /\n end?: CaptureEnd;\n /* Sound above the silence threshold since speech started, in ms. /\n voicedMs?: number;\n /* Silence that was needed to end it at that point (longer once the speaker talked for a while). /\n silenceMs?: number;\n /* RMS per 100 ms of the whole recording. /\n levels?: number[];\n /* What went to speech recognition: the recording cut to the speech, and the gain applied to it. /\n stt?: { fromMs: number; toMs: number; gain: number };\n}\n\n/\n How fast the tracked noise floor may follow a rising ambient level, per analysed frame (30 ms).\n * Fast enough that a genuinely noisy room is matched within about a second, slow enough that a voice\n * cannot pull the floor up to its own level before it has been recognised as speech.\n /\nconst NOISE_RISE = 0.02;\n\nexport class SpeechDetector {\n private readonly frameBytes: number;\n private readonly frameMs: number;\n private readonly fixedStart: number;\n private readonly fixedEnd: number;\n private readonly silenceMs: number;\n private readonly longSilenceMs: number;\n private readonly longAfterMs: number;\n private readonly minSpeechMs: number;\n private readonly speechGapMs: number;\n private readonly maxMs: number;\n private readonly startTimeoutMs: number;\n private readonly skipMs: number;\n private readonly adaptive: boolean;\n private readonly minStartLevel: number;\n private readonly minEndLevel: number;\n private readonly startFactor: number;\n private readonly endFactor: number;\n private readonly warmupMs: number;\n\n private pending: Bytes = Buffer.alloc(0);\n private speechSeen = false;\n private silentMs = 0;\n /* Tracked room level in adaptive mode; null until the first analysed frame. /\n private floor: number \| null = null;\n /* Audio seen so far including the skipped lead-in, in milliseconds. /\n private elapsedMs = 0;\n /* Start of the frame that crossed {@link startLevel}, in ms from the start of the recording; -1 = not yet. /\n speechStartMs = -1;\n /* A crossing of {@link startLevel} that has not yet lasted {@link minSpeechMs}: its start, -1 = none. /\n private candidateMs = -1;\n /* Quiet since the last loud frame of the candidate. /\n private candidateQuietMs = 0;\n /* Sound above {@link endLevel} since the candidate (and then the speech) started, in ms. /\n voicedMs = 0;\n /* The longest sound that crossed {@link startLevel}, in ms — also one too short to count as speech. /\n longestSoundMs = 0;\n /* Audio analysed so far, in milliseconds (exposed for logging). /\n totalMs = 0;\n /* Loudest frame seen so far (exposed for logging — the usual clue when a mic is mis-configured). /\n peak = 0;\n /\n Loudest frame after the skipped lead-in, i.e. without the wake chirp. Compared with {@link startLevel}\n * it tells whether a dropped utterance was silence or a voice that was simply too quiet for the gate.\n /\n analysedPeak = 0;\n\n constructor({\n sampleRate = 16000,\n frameMs = 30,\n startLevel = 700,\n endLevel = 400,\n silenceMs = 900,\n longSilenceMs = 0,\n longAfterMs = 0,\n minSpeechMs = 0,\n speechGapMs = 300,\n maxMs = 12000,\n startTimeoutMs = 0,\n skipMs = 0,\n adaptive = false,\n minStartLevel = 250,\n minEndLevel = 150,\n startFactor = 3.5,\n endFactor = 2,\n warmupMs = 150,\n }: SpeechDetectorOptions = {}) {\n this.frameBytes = Math.round((sampleRate frameMs) / 1000) * 2;\n this.frameMs = frameMs;\n this.fixedStart = startLevel;\n this.fixedEnd = endLevel;\n this.silenceMs = silenceMs;\n this.longSilenceMs = longSilenceMs;\n this.longAfterMs = longAfterMs;\n this.minSpeechMs = minSpeechMs;\n this.speechGapMs = speechGapMs;\n this.maxMs = maxMs;\n this.startTimeoutMs = startTimeoutMs;\n this.skipMs = skipMs;\n this.adaptive = adaptive;\n this.minStartLevel = minStartLevel;\n this.minEndLevel = minEndLevel;\n this.startFactor = startFactor;\n this.endFactor = endFactor;\n this.warmupMs = warmupMs;\n }\n\n /** True once any frame crossed {@link startLevel} — an utterance that never did is room noise. /\n get sawSpeech(): boolean {\n return this.speechSeen;\n }\n\n /* Tracked room level (adaptive mode), for logging. 0 before the first analysed frame. /\n get noiseFloor(): number {\n return this.floor ?? 0;\n }\n\n /* Current speech threshold — fixed, or derived from the noise floor in adaptive mode. /\n get startLevel(): number {\n return this.adaptive ? Math.max(this.minStartLevel, (this.floor ?? 0) this.startFactor) : this.fixedStart;\n }\n\n /** Current silence threshold. Never above {@link startLevel}, so the gate keeps its hysteresis. /\n get endLevel(): number {\n if (!this.adaptive) {\n return this.fixedEnd;\n }\n return Math.min(this.startLevel, Math.max(this.minEndLevel, (this.floor ?? 0) this.endFactor));\n }\n\n /*\n Silence that ends the utterance right now: {@link silenceMs}, or {@link longSilenceMs} once the speaker\n * has talked for {@link longAfterMs} before the current pause.\n /\n get requiredSilenceMs(): number {\n if (!this.speechSeen \|\| !this.longAfterMs \|\| this.longSilenceMs <= this.silenceMs) {\n return this.silenceMs;\n }\n const spokenMs = this.elapsedMs - this.silentMs - this.speechStartMs;\n return spokenMs >= this.longAfterMs ? this.longSilenceMs : this.silenceMs;\n }\n\n /* The detector's view of the recording so far. /\n report(): VadReport {\n return {\n peak: Math.round(this.peak),\n analysedPeak: Math.round(this.analysedPeak),\n noiseFloor: Math.round(this.noiseFloor),\n startLevel: Math.round(this.startLevel),\n endLevel: Math.round(this.endLevel),\n speechStartMs: this.speechSeen ? this.speechStartMs : undefined,\n voicedMs: this.speechSeen ? this.voicedMs : undefined,\n silenceMs: this.requiredSilenceMs,\n };\n }\n\n /* Feed one audio chunk (any size). Returns why the utterance ended, or null to keep listening. /\n push(chunk: Bytes): SpeechEnd \| null {\n this.pending = this.pending.length ? Buffer.concat([this.pending, chunk]) : chunk;\n let verdict: SpeechEnd \| null = null;\n\n while (this.pending.length >= this.frameBytes) {\n const frame = this.pending.subarray(0, this.frameBytes);\n this.pending = this.pending.subarray(this.frameBytes);\n this.elapsedMs += this.frameMs;\n\n const level = rms(frame);\n if (level > this.peak) {\n this.peak = level;\n }\n\n // The device's own wake chirp: loud, clipped, and not speech. Leaving it out of the\n // analysis is what makes `sawSpeech` mean anything, and keeps the noise floor honest.\n if (this.elapsedMs <= this.skipMs) {\n continue;\n }\n this.totalMs += this.frameMs;\n if (level > this.analysedPeak) {\n this.analysedPeak = level;\n }\n\n if (this.adaptive) {\n // Three rules, each there to defeat a specific way of getting this wrong:\n // - start at zero, i.e. maximally sensitive. Seeding from the first frame instead\n // lets someone who talks straight after the wake word install their own voice as\n // the room's noise floor, which shuts the gate for good — the same trap an average\n // or median over a calibration window falls into.\n // - fall instantly, so a room that went quiet is believed at once.\n // - rise slowly, and not at all once speech has been recognised: otherwise a long\n // utterance drags the floor, and with it `endLevel`, up to its own level and cuts\n // itself off mid-sentence. Falling stays on, so the pauses between words still count.\n this.floor =\n this.floor === null\n ? 0\n : level < this.floor\n ? level\n : this.speechSeen \|\| this.candidateMs >= 0\n ? this.floor\n : this.floor + (level - this.floor) NOISE_RISE;\n if (this.totalMs < this.warmupMs) {\n continue; // let the floor settle before anything may count as speech\n }\n }\n\n if (!this.speechSeen) {\n // A loud frame opens a candidate; it becomes speech once it has lasted `minSpeechMs`, and is\n // forgotten when it falls quiet for `speechGapMs` before that — a click is not a request.\n if (this.candidateMs < 0) {\n if (level >= this.startLevel) {\n this.candidateMs = this.elapsedMs - this.frameMs;\n this.candidateQuietMs = 0;\n this.voicedMs = this.frameMs;\n this.longestSoundMs = Math.max(this.longestSoundMs, this.voicedMs);\n }\n } else if (level >= this.endLevel) {\n this.voicedMs += this.frameMs;\n this.candidateQuietMs = 0;\n this.longestSoundMs = Math.max(this.longestSoundMs, this.voicedMs);\n } else {\n this.candidateQuietMs += this.frameMs;\n if (this.candidateQuietMs >= this.speechGapMs) {\n this.candidateMs = -1;\n this.voicedMs = 0;\n }\n }\n if (this.candidateMs >= 0 && this.voicedMs >= this.minSpeechMs) {\n this.speechSeen = true;\n this.speechStartMs = this.candidateMs;\n // A gap inside the candidate already counts towards the end of the utterance.\n this.silentMs = this.candidateQuietMs;\n }\n } else if (level < this.endLevel) {\n this.silentMs += this.frameMs;\n if (this.silentMs >= this.requiredSilenceMs) {\n verdict \|\|= 'speech-end';\n }\n } else {\n this.silentMs = 0;\n this.voicedMs += this.frameMs;\n }\n\n if (!this.speechSeen && this.startTimeoutMs > 0 && this.elapsedMs >= this.startTimeoutMs) {\n verdict \|\|= 'no-speech';\n }\n if (this.totalMs >= this.maxMs) {\n verdict \|\|= 'timeout';\n }\n }\n return verdict;\n }\n}\n"]}

package/build/main.js CHANGED Viewed

@@ -132,6 +132,8 @@ const VOICE_FORM_FIELDS = [
 ];
 /** Of those, the protected ones: the browser never gets them, so an empty value there means "keep the saved one" */
 const VOICE_FORM_SECRETS = ['voiceApiKey', 'openrouterSpeechKey', 'azureSpeechKey', 'awsSecretAccessKey'];
+/** An empty `ask` from the same caller is reported at most this often. */
+const EMPTY_ASK_REPORT_MS = 60_000;
 /** How long to wait for a device to confirm a write, and how often to look. */
 const ACK_TIMEOUT_MS = 2000;
 const ACK_POLL_MS = 150;
@@ -817,6 +819,25 @@ class Assistant extends adapter_core_1.Adapter {
             voice: (voices[provider] || '').trim() || undefined,
         });
     }
+    /** Per caller of `ask`: when an empty question was last reported, and how many came since. */
+    emptyAsks = new Map();
+    /**
+     * An empty `ask` is rejected, and reported once a minute per caller. Field report (2026-11-03): a caller asked
+     * with empty text every time an answer arrived — about once a second, each one a pi-agent request about "only
+     * the time" (the time context was the whole question) and paid tokens. A script that asks on every change of
+     * `text.response` loops exactly like that.
+     */
+    reportEmptyAsk(from) {
+        const now = Date.now();
+        const entry = this.emptyAsks.get(from) ?? { reportedAt: 0, count: 0 };
+        entry.count++;
+        if (now - entry.reportedAt >= EMPTY_ASK_REPORT_MS) {
+            this.log.warn(`Ignoring an empty question from ${from} (sendTo "ask")${entry.count > 1 ? ` — ${entry.count} since the last report` : ''}. A script that asks whenever text.response changes keeps doing this in a loop.`);
+            entry.reportedAt = now;
+            entry.count = 0;
+        }
+        this.emptyAsks.set(from, entry);
+    }
     /** Record the origin of the current text.request/response ('' = state write, 'chat' = message, else satellite). */
     setQuerySource(source) {
         this.setStateAsync('text.querySource', { val: source, ack: true }).catch(() => { });
@@ -2696,14 +2717,23 @@ class Assistant extends adapter_core_1.Adapter {
         const text = typeof message === 'string' ? message : message?.text;
         // Origin for text.querySource: caller-provided (e.g. 'telegram:Max') or 'chat' for the admin chat.
         const source = (typeof message === 'string' ? '' : message?.source?.trim()) || 'chat';
+        // Who sent it (javascript.0, telegram.0, admin.0 for the chat tab) — the way to find a script gone wrong.
+        const caller = String(obj.from || '').replace(/^system\.adapter\./, '') || 'unknown';
+        const question = String(text ?? '').trim();
+        if (!question) {
+            this.reportEmptyAsk(caller);
+            if (obj.callback) {
+                this.sendTo(obj.from, obj.command, { error: 'empty question' }, obj.callback);
+            }
+            return;
+        }
         if (!this.ready) {
             if (obj.callback) {
                 this.sendTo(obj.from, obj.command, { error: 'agent not ready' }, obj.callback);
             }
             return;
         }
-        const question = String(text ?? '');
-        this.log.info(`Q (${obj.command}): ${question}`);
+        this.log.info(`Q (${obj.command} from ${caller}): ${question}`);
         this.setStateAsync('text.request', { val: question, ack: true }).catch(() => { });
         this.setQuerySource(source);
         try {
@@ -2732,6 +2762,10 @@ class Assistant extends adapter_core_1.Adapter {
      * later turns have the full thread.
      */
     answer(question, source = '', origin = {}) {
+        if (!question.trim()) {
+            // Nothing to answer — and a model asked with nothing but the time context still costs a request.
+            return Promise.resolve('');
+        }
         const voice = VOICE_TRANSPORTS.has(origin.transport || '');
         return this.requestOrigin.run({ source, room: origin.room || '', voice }, () => this.journal
             ? this.journal.run(question, source, origin, () => {