dsh-live-voice 0.2.1 → 0.2.3

package/src/core/coordinator.ts

@@ -46,6 +46,7 @@ export class VoiceCoordinator {
       conversation: false,
       listening: false,
       recognizing: false,
+      pendingTranscriptions: 0,
       muted: normalizeSettings(settings).microphoneEnabled === false,
       speaking: false,
       paused: false,
@@ -60,6 +61,7 @@ export class VoiceCoordinator {
     };
     this.autoSendTimer = null;
     this.autoSendDraft = null;
+    this.holdToTalkRelease = false;
     this.interruptionTimer = null;
     this.interruptionTranscriptConfirmed = false;
     this.interruptionPausedSpeech = false;
@@ -186,6 +188,27 @@ export class VoiceCoordinator {
   startDictation() {
     return this.startListening(false);
   }
+  startHoldToTalk() {
+    this.holdToTalkRelease = false;
+    return this.startListening(false);
+  }
+  async releaseHoldToTalk() {
+    if (this.disposed || (!this.snapshot.listening && !this.snapshot.starting)) return;
+    this.holdToTalkRelease = true;
+    if (this.snapshot.starting) return;
+    await this.recognition.finish?.();
+    if (this.snapshot.pendingTranscriptions === 0) this._finishHoldToTalk();
+  }
+  _finishHoldToTalk() {
+    if (!this.holdToTalkRelease) return;
+    this.holdToTalkRelease = false;
+    const draft = this.composer.getDraft();
+    if (!draft.trim()) {
+      void this.stopListening();
+      return;
+    }
+    this.scheduleAutoSend(draft, { force: true, stopAfter: true });
+  }
   startConversation() {
     return this.startListening(true);
   }
@@ -269,6 +292,14 @@ export class VoiceCoordinator {
           onResult: (result) => {
             if (valid()) this.onResult(result);
           },
+          onProcessingChange: ({ pending = 0 } = {}) => {
+            if (!valid()) return;
+            const count = Number.isSafeInteger(pending) && pending >= 0 ? pending : 0;
+            this.patch({ pendingTranscriptions: count });
+            if (count > 0) this.cancelAutoSend();
+            else if (this.holdToTalkRelease) this._finishHoldToTalk();
+            else this.maybeScheduleAutoSend();
+          },
           onActivity: (active) => {
             if (!valid()) return;
             if (active) {
@@ -294,6 +325,7 @@ export class VoiceCoordinator {
           return;
         }
         this.patch({ listening: true, starting: false });
+        if (this.holdToTalkRelease) void this.releaseHoldToTalk();
         this._drain();
       } catch (error) {
         try {
@@ -378,8 +410,10 @@ export class VoiceCoordinator {
     }
     // Pending structured questions own recognition while waiting for a hands-free
     // answer. Keep their interim and final text out of the normal chat composer.
-    if (typeof this.composer.handleQuestionResult === 'function' &&
-        this.composer.handleQuestionResult({ final, interim })) {
+    if (
+      typeof this.composer.handleQuestionResult === 'function' &&
+      this.composer.handleQuestionResult({ final, interim })
+    ) {
       this.cancelAutoSend();
       this.transcript.reset();
       this.patch({ recognizing: !!interim });
@@ -441,7 +475,7 @@ export class VoiceCoordinator {
       }
       const next = this.transcript.update(this.composer.getDraft(), final, true);
       this.composer.setDraft(next);
-      if (this.snapshot.settings.sendingMode !== 'manual') this.scheduleAutoSend(next);
+      this.maybeScheduleAutoSend(next);
     }
     if (interim) {
       this.cancelAutoSend();
@@ -453,7 +487,16 @@ export class VoiceCoordinator {
     this.patch({ recognizing: !!interim });
     this._drain();
   }
-  scheduleAutoSend(draft) {
+  maybeScheduleAutoSend(draft = this.composer.getDraft()) {
+    if (
+      this.snapshot.settings.sendingMode === 'manual' ||
+      this.snapshot.recognizing ||
+      this.snapshot.pendingTranscriptions > 0
+    )
+      return;
+    this.scheduleAutoSend(draft);
+  }
+  scheduleAutoSend(draft, { force = false, stopAfter = false } = {}) {
     this.cancelAutoSend();
     if (!draft.trim() || typeof this.composer.submit !== 'function') return;
     const delay = this.snapshot.settings.autoSendDelaySeconds * 1000;
@@ -466,16 +509,19 @@ export class VoiceCoordinator {
       this.patch({ autoSendAt: null });
       if (
         !this.disposed &&
-        this.snapshot.settings.sendingMode !== 'manual' &&
+        (force || this.snapshot.settings.sendingMode !== 'manual') &&
         expected === this.composer.getDraft()
       ) {
         try {
-          this.composer.submit(this.snapshot.settings.sendingMode === 'steer' ? 'steer' : 'queue');
+          this.composer.submit(
+            this.snapshot.settings.sendingMode === 'steer' && !force ? 'steer' : 'queue',
+          );
           // Web Speech keeps a cumulative native result list for the lifetime of
           // one recognition instance. Start a fresh instance at the turn boundary
           // so the sent utterance cannot prefix the next one.
           this.transcript.reset();
           this.recognition.reset?.();
+          if (stopAfter) void this.stopListening();
         } catch (error) {
           this.patch({ error: message(error) });
         }
@@ -492,10 +538,11 @@ export class VoiceCoordinator {
     if (this.autoSendDraft !== null && draft !== this.autoSendDraft) this.cancelAutoSend();
   }
   stopListening() {
+    this.holdToTalkRelease = false;
     this.cancelAutoSend();
     ++this.epoch;
     this.inputController?.abort();
-    this.patch({ listening: false, recognizing: false, starting: false });
+    this.patch({ listening: false, recognizing: false, pendingTranscriptions: 0, starting: false });
     return this._input(async () => {
       try {
         await this._releaseInput();

package/src/core/settings.ts

@@ -36,7 +36,9 @@ export const defaultSettings = Object.freeze({
   recognitionProcessLocally: true,
   recognitionAutoInstall: true,
   voiceDetectionPreset: 'natural',
+  recognitionMaxUtteranceSeconds: 60,
   microphoneEnabled: true,
+  holdToTalkEnabled: true,
   announceAssistantMessages: true,
   interruptSpeechOnUserMessage: false,
   sendingMode: 'manual',
@@ -94,10 +96,20 @@ export function normalizeSettings(value) {
     voiceDetectionPreset: Object.hasOwn(voiceDetectionPresets, source.voiceDetectionPreset)
       ? source.voiceDetectionPreset
       : defaultSettings.voiceDetectionPreset,
+    recognitionMaxUtteranceSeconds:
+      Number.isInteger(source.recognitionMaxUtteranceSeconds) &&
+      source.recognitionMaxUtteranceSeconds >= 10 &&
+      source.recognitionMaxUtteranceSeconds <= 300
+        ? source.recognitionMaxUtteranceSeconds
+        : defaultSettings.recognitionMaxUtteranceSeconds,
     microphoneEnabled:
       typeof source.microphoneEnabled === 'boolean'
         ? source.microphoneEnabled
         : defaultSettings.microphoneEnabled,
+    holdToTalkEnabled:
+      typeof source.holdToTalkEnabled === 'boolean'
+        ? source.holdToTalkEnabled
+        : defaultSettings.holdToTalkEnabled,
     announceAssistantMessages:
       typeof source.announceAssistantMessages === 'boolean'
         ? source.announceAssistantMessages

package/src/engines/recognition/browser.ts

@@ -141,6 +141,9 @@ export class BrowserRecognitionEngine {
       timer: null,
       recognition: null,
       speaking: false,
+      finishing: false,
+      finishPromise: null,
+      resolveFinish: null,
     };
     this.session = session;
     const cancelled = new Promise((resolve, reject) => {
@@ -244,6 +247,12 @@ export class BrowserRecognitionEngine {
       session.recognition = null;
       this._activity(session, false);
       if (this.session !== session) return;
+      if (session.finishing) {
+        this.session = null;
+        session.signal?.removeEventListener('abort', session.cancel);
+        session.resolveFinish?.();
+        return;
+      }
       if (session.restarts >= this.maxRestarts) {
         this._fail(
           session,
@@ -273,6 +282,34 @@ export class BrowserRecognitionEngine {
     recognition.start();
   }
 
+  finish() {
+    const session = this.session;
+    if (!session) return Promise.resolve();
+    if (session.finishPromise) return session.finishPromise;
+    session.finishing = true;
+    session.finishPromise = new Promise((resolve) => {
+      session.resolveFinish = resolve;
+    });
+    if (session.timer !== null) {
+      (this.globals.clearTimeout ?? globalThis.clearTimeout)(session.timer);
+      session.timer = null;
+      this.session = null;
+      session.resolveFinish();
+      return session.finishPromise;
+    }
+    try {
+      session.recognition?.stop();
+      if (!session.recognition) {
+        this.session = null;
+        session.resolveFinish();
+      }
+    } catch {
+      this.session = null;
+      session.resolveFinish();
+    }
+    return session.finishPromise;
+  }
+
   reset() {
     const session = this.session;
     const recognition = session?.recognition;
@@ -310,5 +347,6 @@ export class BrowserRecognitionEngine {
       /* Browser already ended capture. */
     }
     this._activity(session, false);
+    session.resolveFinish?.();
   }
 }

package/src/engines/recognition/whisper-http.ts

@@ -39,12 +39,18 @@ export function encodeMonoPcm16Wav(samples, inputRate) {
   return buffer;
 }
 export class WhisperHttpRecognitionEngine {
-  constructor({ globals = globalThis, meter, voiceDetectionPreset = 'natural' } = {}) {
+  constructor({
+    globals = globalThis,
+    meter,
+    voiceDetectionPreset = 'natural',
+    maxUtteranceSeconds = 60,
+  } = {}) {
     this.g = globals;
     this.meter = meter;
     this.session = null;
     this.lang = 'pt-BR';
     this.voiceDetectionPreset = voiceDetectionPreset;
+    this.maxUtteranceSeconds = maxUtteranceSeconds;
     this.route = ROUTE;
   }
   get segmentation() {
@@ -73,7 +79,14 @@ export class WhisperHttpRecognitionEngine {
       };
     }
   }
-  async start({ lang = this.lang, signal, onResult, onActivity, onError } = {}) {
+  async start({
+    lang = this.lang,
+    signal,
+    onResult,
+    onActivity,
+    onError,
+    onProcessingChange,
+  } = {}) {
     await this.stop();
     if (signal?.aborted) throw abortError();
     const context = this.meter?.context,
@@ -95,16 +108,25 @@ export class WhisperHttpRecognitionEngine {
       samples: 0,
       voiced: false,
       silence: 0,
-      inflight: new Set(),
+      transcriptionQueue: [],
+      activeRequest: null,
+      draining: false,
       onResult,
       onActivity,
       onError,
+      onProcessingChange,
       lang,
       signal,
     };
     this.session = session;
     const valid = () => this.session === session && !signal?.aborted;
-    const submit = async () => {
+    const notifyProcessing = () =>
+      session.onProcessingChange?.({
+        queued: session.transcriptionQueue.length,
+        active: !!session.activeRequest,
+        pending: session.transcriptionQueue.length + (session.activeRequest ? 1 : 0),
+      });
+    const enqueue = () => {
       if (!session.voiced || session.samples < context.sampleRate * 0.25) {
         session.chunks = [];
         session.samples = 0;
@@ -122,29 +144,46 @@ export class WhisperHttpRecognitionEngine {
       session.samples = 0;
       session.voiced = false;
       session.silence = 0;
-      const request = new AbortController();
-      session.inflight.add(request);
+      session.transcriptionQueue.push(samples);
+      notifyProcessing();
+      void drain();
+    };
+    const drain = async () => {
+      if (session.draining || !valid()) return;
+      session.draining = true;
       try {
-        const response = await this.g.fetch(this.route + '/transcribe', {
-          method: 'POST',
-          credentials: 'same-origin',
-          headers: {
-            'content-type': 'audio/wav',
-            'x-dlv-client-id': session.operation,
-            'x-dlv-operation-id': id(),
-            'x-dlv-language': lang,
-          },
-          body: encodeMonoPcm16Wav(samples, context.sampleRate),
-          signal: request.signal,
-        });
-        const json = await response.json();
-        if (!response.ok || !json?.ok)
-          throw new Error(json?.error?.message || 'HTTP transcription failed.');
-        if (valid() && json.value.text) onResult?.({ final: json.value.text, interim: '' });
-      } catch (error) {
-        if (error.name !== 'AbortError' && valid()) onError?.(error);
+        while (valid() && session.transcriptionQueue.length) {
+          const samples = session.transcriptionQueue.shift();
+          const request = new AbortController();
+          session.activeRequest = request;
+          notifyProcessing();
+          try {
+            const response = await this.g.fetch(this.route + '/transcribe', {
+              method: 'POST',
+              credentials: 'same-origin',
+              headers: {
+                'content-type': 'audio/wav',
+                'x-dlv-client-id': session.operation,
+                'x-dlv-operation-id': id(),
+                'x-dlv-language': lang,
+              },
+              body: encodeMonoPcm16Wav(samples, context.sampleRate),
+              signal: request.signal,
+            });
+            const json = await response.json();
+            if (!response.ok || !json?.ok)
+              throw new Error(json?.error?.message || 'HTTP transcription failed.');
+            if (valid() && json.value.text) onResult?.({ final: json.value.text, interim: '' });
+          } catch (error) {
+            if (error.name !== 'AbortError' && valid()) onError?.(error);
+          } finally {
+            if (session.activeRequest === request) session.activeRequest = null;
+            notifyProcessing();
+          }
+        }
       } finally {
-        session.inflight.delete(request);
+        session.draining = false;
+        notifyProcessing();
       }
     };
     processor.onaudioprocess = (event) => {
@@ -164,14 +203,24 @@ export class WhisperHttpRecognitionEngine {
       if (
         (session.voiced &&
           session.silence > context.sampleRate * (this.segmentation.silenceMs / 1000)) ||
-        session.samples > context.sampleRate * 20
+        session.samples > context.sampleRate * this.maxUtteranceSeconds
       )
-        void submit();
+        enqueue();
     };
     source.connect(processor);
+    session.finish = enqueue;
     session.abort = () => this.stop();
     signal?.addEventListener('abort', session.abort, { once: true });
   }
+  finish() {
+    const session = this.session;
+    if (!session) return;
+    session.processor.onaudioprocess = null;
+    try {
+      this.meter?.source?.disconnect(session.processor);
+    } catch {}
+    session.finish?.();
+  }
   async stop() {
     const session = this.session;
     if (!session) return;
@@ -185,7 +234,9 @@ export class WhisperHttpRecognitionEngine {
       session.processor.disconnect();
       session.gain?.disconnect();
     } catch {}
-    for (const request of session.inflight) request.abort();
-    session.inflight.clear();
+    session.transcriptionQueue = [];
+    session.activeRequest?.abort();
+    session.activeRequest = null;
+    session.onProcessingChange?.({ queued: 0, active: false, pending: 0 });
   }
 }