dsh-live-voice 0.0.1-developing → 0.0.2

package/src/engines/recognition/whisper-http.ts

@@ -0,0 +1,191 @@
+// @ts-nocheck
+import { voiceDetectionPresets } from '../../core/settings.ts';
+
+const ROUTE = '/api/dsh-live-voice/whisper';
+const id = () => globalThis.crypto.randomUUID();
+const abortError = () =>
+  Object.assign(new Error('Whisper recognition was cancelled.'), { name: 'AbortError' });
+export function encodeMonoPcm16Wav(samples, inputRate) {
+  const ratio = inputRate / 16000,
+    length = Math.floor(samples.length / ratio),
+    out = new Int16Array(length);
+  for (let i = 0; i < length; i++) {
+    const start = Math.floor(i * ratio),
+      end = Math.max(start + 1, Math.floor((i + 1) * ratio));
+    let sum = 0;
+    for (let j = start; j < end && j < samples.length; j++) sum += samples[j];
+    const value = Math.max(-1, Math.min(1, sum / (end - start)));
+    out[i] = value < 0 ? value * 32768 : value * 32767;
+  }
+  const buffer = new ArrayBuffer(44 + out.byteLength),
+    view = new DataView(buffer),
+    text = (at, s) => {
+      for (let i = 0; i < s.length; i++) view.setUint8(at + i, s.charCodeAt(i));
+    };
+  text(0, 'RIFF');
+  view.setUint32(4, 36 + out.byteLength, true);
+  text(8, 'WAVE');
+  text(12, 'fmt ');
+  view.setUint32(16, 16, true);
+  view.setUint16(20, 1, true);
+  view.setUint16(22, 1, true);
+  view.setUint32(24, 16000, true);
+  view.setUint32(28, 32000, true);
+  view.setUint16(32, 2, true);
+  view.setUint16(34, 16, true);
+  text(36, 'data');
+  view.setUint32(40, out.byteLength, true);
+  new Int16Array(buffer, 44).set(out);
+  return buffer;
+}
+export class WhisperHttpRecognitionEngine {
+  constructor({ globals = globalThis, meter, voiceDetectionPreset = 'natural' } = {}) {
+    this.g = globals;
+    this.meter = meter;
+    this.session = null;
+    this.lang = 'pt-BR';
+    this.voiceDetectionPreset = voiceDetectionPreset;
+    this.route = ROUTE;
+  }
+  get segmentation() {
+    return voiceDetectionPresets[this.voiceDetectionPreset] || voiceDetectionPresets.natural;
+  }
+  async capability() {
+    try {
+      const response = await this.g.fetch(this.route + '/capabilities', {
+          credentials: 'same-origin',
+        }),
+        json = await response.json();
+      return json?.ok
+        ? json.value
+        : {
+            supported: false,
+            local: true,
+            streaming: false,
+            reason: json?.error?.message || 'Whisper HTTP host is unavailable.',
+          };
+    } catch (error) {
+      return {
+        supported: false,
+        local: true,
+        streaming: false,
+        reason: 'Whisper HTTP host connection failed: ' + error.message,
+      };
+    }
+  }
+  async start({ lang = this.lang, signal, onResult, onActivity, onError } = {}) {
+    await this.stop();
+    if (signal?.aborted) throw abortError();
+    const context = this.meter?.context,
+      source = this.meter?.source;
+    if (!context || !source || typeof context.createScriptProcessor !== 'function')
+      throw new Error('This browser cannot capture PCM audio for Whisper HTTP.');
+    const processor = context.createScriptProcessor(4096, 1, 1),
+      gain = context.createGain?.();
+    if (gain) {
+      gain.gain.value = 0;
+      processor.connect(gain);
+      gain.connect(context.destination);
+    } else processor.connect(context.destination);
+    const session = {
+      operation: id(),
+      processor,
+      gain,
+      chunks: [],
+      samples: 0,
+      voiced: false,
+      silence: 0,
+      inflight: new Set(),
+      onResult,
+      onActivity,
+      onError,
+      lang,
+      signal,
+    };
+    this.session = session;
+    const valid = () => this.session === session && !signal?.aborted;
+    const submit = async () => {
+      if (!session.voiced || session.samples < context.sampleRate * 0.25) {
+        session.chunks = [];
+        session.samples = 0;
+        session.voiced = false;
+        session.silence = 0;
+        return;
+      }
+      const samples = new Float32Array(session.samples);
+      let at = 0;
+      for (const chunk of session.chunks) {
+        samples.set(chunk, at);
+        at += chunk.length;
+      }
+      session.chunks = [];
+      session.samples = 0;
+      session.voiced = false;
+      session.silence = 0;
+      const request = new AbortController();
+      session.inflight.add(request);
+      try {
+        const response = await this.g.fetch(this.route + '/transcribe', {
+          method: 'POST',
+          credentials: 'same-origin',
+          headers: {
+            'content-type': 'audio/wav',
+            'x-dlv-client-id': session.operation,
+            'x-dlv-operation-id': id(),
+            'x-dlv-language': lang,
+          },
+          body: encodeMonoPcm16Wav(samples, context.sampleRate),
+          signal: request.signal,
+        });
+        const json = await response.json();
+        if (!response.ok || !json?.ok)
+          throw new Error(json?.error?.message || 'HTTP transcription failed.');
+        if (valid() && json.value.text) onResult?.({ final: json.value.text, interim: '' });
+      } catch (error) {
+        if (error.name !== 'AbortError' && valid()) onError?.(error);
+      } finally {
+        session.inflight.delete(request);
+      }
+    };
+    processor.onaudioprocess = (event) => {
+      if (!valid()) return;
+      const data = new Float32Array(event.inputBuffer.getChannelData(0)),
+        rms = Math.sqrt(data.reduce((sum, x) => sum + x * x, 0) / data.length);
+      if (rms > 0.012) {
+        session.voiced = true;
+        session.silence = 0;
+        onActivity?.(true);
+      } else if (session.voiced) {
+        session.silence += data.length;
+        onActivity?.(false);
+      }
+      session.chunks.push(data);
+      session.samples += data.length;
+      if (
+        (session.voiced &&
+          session.silence > context.sampleRate * (this.segmentation.silenceMs / 1000)) ||
+        session.samples > context.sampleRate * 20
+      )
+        void submit();
+    };
+    source.connect(processor);
+    session.abort = () => this.stop();
+    signal?.addEventListener('abort', session.abort, { once: true });
+  }
+  async stop() {
+    const session = this.session;
+    if (!session) return;
+    this.session = null;
+    session.signal?.removeEventListener('abort', session.abort);
+    session.processor.onaudioprocess = null;
+    try {
+      this.meter?.source?.disconnect(session.processor);
+    } catch {}
+    try {
+      session.processor.disconnect();
+      session.gain?.disconnect();
+    } catch {}
+    for (const request of session.inflight) request.abort();
+    session.inflight.clear();
+  }
+}

package/src/engines/speaking/browser.ts

@@ -0,0 +1,136 @@
+// @ts-nocheck
+const abortError = () =>
+  Object.assign(new Error('Speech playback was cancelled.'), { name: 'AbortError' });
+
+/** Local-only Web Speech synthesis. The browser synthesis queue is shared: use one owner.
+ * Inject { globals: { speechSynthesis, SpeechSynthesisUtterance } } for tests.
+ * capability() is synchronous; voices() returns native local voices (refresh after voiceschanged).
+ * speak() replaces prior speech and resolves only on completion. Cancellation rejects AbortError.
+ * voice accepts a listed voice object, voiceURI, or name; no remote/default fallback is used.
+ */
+export class BrowserSpeakingEngine {
+  constructor({ globals = globalThis, lang = 'pt-BR' } = {}) {
+    this.globals = globals;
+    this.lang = lang;
+    this.current = null;
+  }
+
+  voices() {
+    try {
+      return Array.from(this.globals.speechSynthesis?.getVoices?.() ?? []).filter(
+        (v) => v.localService === true,
+      );
+    } catch {
+      return [];
+    }
+  }
+
+  capability() {
+    const supported =
+      typeof this.globals.SpeechSynthesisUtterance === 'function' &&
+      typeof this.globals.speechSynthesis?.speak === 'function' &&
+      typeof this.globals.speechSynthesis?.cancel === 'function' &&
+      this.voices().length > 0;
+    return {
+      supported,
+      local: true,
+      voices: this.voices().map((voice) => ({
+        name: voice.name,
+        voiceURI: voice.voiceURI,
+        lang: voice.lang,
+      })),
+      pause: typeof this.globals.speechSynthesis?.pause === 'function',
+      resume: typeof this.globals.speechSynthesis?.resume === 'function',
+      ...(supported
+        ? {}
+        : {
+            reason:
+              'Local browser speech synthesis is unavailable. Enable an installed local system voice, refresh the voice list, or choose another local speaking engine. Remote voices are not allowed.',
+          }),
+    };
+  }
+
+  async speak(text, { voice, rate = 1, signal } = {}) {
+    // No await before replacement: concurrent calls cannot enqueue obsolete utterances.
+    this.stop();
+    if (signal?.aborted) throw abortError();
+    if (typeof text !== 'string') throw new TypeError('Speech text must be a string.');
+    if (!Number.isFinite(rate) || rate < 0.1 || rate > 10)
+      throw new RangeError('Speech rate must be between 0.1 and 10.');
+    const capability = this.capability();
+    if (!capability.supported) throw new Error(capability.reason);
+    const voices = this.voices();
+    const chosen =
+      voice == null
+        ? (voices.find((v) => v.lang?.toLowerCase() === this.lang.toLowerCase()) ?? voices[0])
+        : voices.find((v) =>
+            typeof voice === 'string' ? v.voiceURI === voice || v.name === voice : v === voice,
+          );
+    if (!chosen)
+      throw new Error(
+        'The selected voice is not an available local browser voice. Choose a voice from voices().',
+      );
+    if (!text.trim()) return;
+    const utterance = new this.globals.SpeechSynthesisUtterance(text);
+    utterance.voice = chosen;
+    utterance.lang = chosen.lang || this.lang;
+    utterance.rate = rate;
+    return new Promise((resolve, reject) => {
+      const finish = (error) => {
+        if (this.current !== job) return;
+        this.current = null;
+        utterance.onend = null;
+        utterance.onerror = null;
+        signal?.removeEventListener('abort', cancel);
+        error ? reject(error) : resolve();
+      };
+      const cancel = () => {
+        if (this.current === job) this.stop();
+      };
+      const job = { finish, utterance };
+      this.current = job;
+      utterance.onend = () => finish();
+      utterance.onerror = (event) =>
+        finish(
+          Object.assign(
+            new Error('Browser speech synthesis failed: ' + (event.error || 'unknown error')),
+            { code: event.error },
+          ),
+        );
+      signal?.addEventListener('abort', cancel, { once: true });
+      if (signal?.aborted) {
+        cancel();
+        return;
+      }
+      try {
+        // cancel() can leave the shared synthesizer paused in some browsers.
+        this.globals.speechSynthesis.resume?.();
+        this.globals.speechSynthesis.speak(utterance);
+      } catch (error) {
+        finish(error);
+      }
+    });
+  }
+
+  async stop() {
+    if (!this.current) return;
+    this.current.finish(abortError());
+    // Detach first: cancel may synchronously emit end/error.
+    try {
+      this.globals.speechSynthesis.cancel();
+    } catch {
+      /* Already released locally. */
+    }
+  }
+
+  pause() {
+    if (!this.current || typeof this.globals.speechSynthesis.pause !== 'function') return false;
+    this.globals.speechSynthesis.pause();
+    return true;
+  }
+  resume() {
+    if (!this.current || typeof this.globals.speechSynthesis.resume !== 'function') return false;
+    this.globals.speechSynthesis.resume();
+    return true;
+  }
+}

package/src/engines/speaking/qwen-http.ts

@@ -0,0 +1,119 @@
+// @ts-nocheck
+const BASE = '/api/dsh-live-voice/qwen';
+const cancelled = () => Object.assign(new Error('Speech was cancelled.'), { name: 'AbortError' });
+
+/** Fetches host-local Qwen WAV audio through authenticated DSH and plays it in
+ * the user's browser. The resident server owns model lifetime. */
+export class QwenHttpSpeakingEngine {
+  constructor({ globals = globalThis, lang = 'pt-BR' } = {}) {
+    this.g = globals;
+    this.lang = lang;
+    this.current = null;
+  }
+  async capability() {
+    try {
+      const response = await this.g.fetch(BASE + '/capabilities?kind=tts', {
+          credentials: 'same-origin',
+        }),
+        json = await response.json();
+      if (!response.ok || !json?.ok)
+        throw new Error(json?.error?.message || 'Qwen capability check failed.');
+      return { ...json.value, pause: true, resume: true };
+    } catch (error) {
+      return {
+        supported: false,
+        local: true,
+        location: 'host',
+        pause: true,
+        resume: true,
+        reason: 'Qwen speech server check failed: ' + (error?.message || error),
+      };
+    }
+  }
+  async speak(text, { rate = 1, signal, lang = this.lang, voice = 'aiden' } = {}) {
+    if (typeof text !== 'string') throw new TypeError('Speech text must be a string.');
+    if (!Number.isFinite(rate) || rate < 0.1 || rate > 3)
+      throw new RangeError('Speech rate must be between 0.1 and 3.');
+    if (signal?.aborted) throw cancelled();
+    await this.stop();
+    if (!text.trim()) return;
+    const operation = { abort: new AbortController(), audio: null, url: null };
+    this.current = operation;
+    const cancel = () => operation.abort.abort();
+    signal?.addEventListener('abort', cancel, { once: true });
+    if (signal?.aborted) cancel();
+    try {
+      const response = await this.g.fetch(BASE + '/speech', {
+        method: 'POST',
+        credentials: 'same-origin',
+        headers: { 'content-type': 'application/json' },
+        body: JSON.stringify({ text, lang, voice }),
+        signal: operation.abort.signal,
+      });
+      if (!response.ok) {
+        let body;
+        try {
+          body = await response.json();
+        } catch {}
+        throw new Error(body?.error?.message || `Qwen synthesis failed (${response.status}).`);
+      }
+      if (operation.abort.signal.aborted) throw cancelled();
+      const blob = await response.blob();
+      operation.url = this.g.URL.createObjectURL(blob);
+      const audio = (operation.audio = new this.g.Audio(operation.url));
+      audio.playbackRate = rate;
+      await new Promise((resolve, reject) => {
+        const done = () => {
+            cleanup();
+            resolve();
+          },
+          failed = () => {
+            cleanup();
+            reject(new Error('The browser could not play Qwen speech audio.'));
+          },
+          aborted = () => {
+            cleanup();
+            audio.pause();
+            reject(cancelled());
+          },
+          cleanup = () => {
+            audio.removeEventListener('ended', done);
+            audio.removeEventListener('error', failed);
+            operation.abort.signal.removeEventListener('abort', aborted);
+          };
+        audio.addEventListener('ended', done, { once: true });
+        audio.addEventListener('error', failed, { once: true });
+        operation.abort.signal.addEventListener('abort', aborted, { once: true });
+        Promise.resolve(audio.play()).catch(failed);
+      });
+    } catch (error) {
+      if (operation.abort.signal.aborted && error?.name !== 'AbortError') throw cancelled();
+      throw error;
+    } finally {
+      signal?.removeEventListener('abort', cancel);
+      if (operation.url) this.g.URL.revokeObjectURL(operation.url);
+      if (this.current === operation) this.current = null;
+    }
+  }
+  async stop() {
+    const operation = this.current;
+    if (!operation) return;
+    this.current = null;
+    operation.abort.abort();
+    operation.audio?.pause();
+    if (operation.url) {
+      this.g.URL.revokeObjectURL(operation.url);
+      operation.url = null;
+    }
+  }
+  pause() {
+    if (!this.current?.audio || this.current.audio.paused) return false;
+    this.current.audio.pause();
+    return true;
+  }
+  async resume() {
+    if (!this.current?.audio || !this.current.audio.paused) return false;
+    await this.current.audio.play();
+    return true;
+  }
+}

package/src/engines/speaking/say-client.ts

@@ -0,0 +1,96 @@
+// @ts-nocheck
+const CHANNEL = '/api';
+const cancelled = () => Object.assign(new Error('Speech was cancelled.'), { name: 'AbortError' });
+
+/** Browser transport for host-local macOS say; it never claims browser-local audio.
+ * rpc is ctx.connection.rpc. The long-running speak request is intentionally
+ * abortable, so a closed browser transport cancels the host process as well.
+ */
+export class SayClientEngine {
+  constructor({ rpc }) {
+    this.rpc = rpc;
+    this.clientId = globalThis.crypto.randomUUID();
+    this.current = null;
+  }
+  async request(endpoint, payload = {}, signal) {
+    const result = await this.rpc.call(CHANNEL, `dsh-live-voice/${endpoint}`, payload, signal);
+    if (!result?.ok) {
+      if (signal?.aborted || result?.error?.code === 'cancelled') throw cancelled();
+      throw Object.assign(new Error(result?.error?.message || 'Host speech is unavailable.'), {
+        code: result?.error?.code,
+      });
+    }
+    return result.value;
+  }
+  async capability() {
+    try {
+      return { ...(await this.request('capabilities')), local: true, location: 'host' };
+    } catch (error) {
+      return {
+        supported: false,
+        local: true,
+        location: 'host',
+        reason: `macOS say connection failed: ${error.message}`,
+      };
+    }
+  }
+  async speak(text, { voice, rate = 1, signal } = {}) {
+    if (typeof text !== 'string') throw new TypeError('Speech text must be a string.');
+    if (!Number.isFinite(rate) || rate < 0.1 || rate > 10)
+      throw new RangeError('Speech rate must be between 0.1 and 10.');
+    if (signal?.aborted) throw cancelled();
+    const previous = this.current;
+    previous?.abort.abort();
+    const operation = { operationId: globalThis.crypto.randomUUID(), abort: new AbortController() };
+    this.current = operation;
+    const cancel = () => operation.abort.abort();
+    signal?.addEventListener('abort', cancel, { once: true });
+    if (signal?.aborted) cancel();
+    try {
+      if (!text.trim()) return;
+      await this.request(
+        'speak',
+        {
+          clientId: this.clientId,
+          operationId: operation.operationId,
+          text,
+          ...(voice === undefined ? {} : { voice }),
+          rate: Math.round(rate * 175),
+        },
+        operation.abort.signal,
+      );
+      if (operation.abort.signal.aborted) throw cancelled();
+    } finally {
+      signal?.removeEventListener('abort', cancel);
+      if (this.current === operation) this.current = null;
+    }
+  }
+  async stop() {
+    const operation = this.current;
+    if (!operation) return;
+    this.current = null;
+    // Send ownership-scoped stop as well as cancelling the open request. Either
+    // arrival order is safe; an obsolete operation cannot stop its successor.
+    const stopping = this.request('stop', {
+      clientId: this.clientId,
+      operationId: operation.operationId,
+    });
+    operation.abort.abort();
+    await stopping;
+  }
+  async control(endpoint) {
+    const operation = this.current;
+    if (!operation) return false;
+    const value = await this.request(endpoint, {
+      clientId: this.clientId,
+      operationId: operation.operationId,
+    });
+    return value.applied;
+  }
+  pause() {
+    return this.control('pause');
+  }
+  resume() {
+    return this.control('resume');
+  }
+}