@olares/lares-core 0.15.27 → 0.15.29

package/voice/unified-conversation.js

@@ -3,10 +3,14 @@ import { createDeviceInput } from "./device-input.js";
 import { createPcmStreamPlayer } from "./pcm-player.js";
 import { createVoiceProtocolClient } from "./protocol-client.js";
 import { createStreamingCapture } from "./stream-client.js";
+import { isTransientVoiceError, transientVoiceError } from "./errors.js";
 import { mintRpcId } from "../larepass/rpc.js";
 
 const RATE = 24000;
 const PROGRESS_MS = 250;
+// A commit is final on the Host. If its transcript never comes back the
+// composer must stop waiting instead of sitting on 转写中 for minutes.
+const COMMIT_TIMEOUT_MS = 60_000;
 
 export function encodeBase64(bytes) {
   let text = "";
@@ -26,12 +30,12 @@ export function browserRealtimeSocket(ticket) {
  * microphone streams PCM into the socket, the Host transcribes, submits and speaks, and the reply
  * audio plays from the socket instead of a separate synthesis stream.
  */
-export function createUnifiedConversation({ request, socket: openSocket = browserRealtimeSocket, onPhase, mode = () => "queue", device = "device", metric = () => {}, readAloud = true, preset, presentationSession = false, endpointSilenceMs, idleTimeoutMs, recoveryTimeoutMs, onText = () => {}, onTranscript = null, preparation, media, player: createPlayer = createPcmStreamPlayer, input: inputPort }) {
+export function createUnifiedConversation({ request, socket: openSocket = browserRealtimeSocket, onPhase, mode = () => "queue", device = "device", metric = () => {}, readAloud = true, preset, presentationSession = false, endpointSilenceMs, idleTimeoutMs, recoveryTimeoutMs, commitTimeoutMs = COMMIT_TIMEOUT_MS, onText = () => {}, onTranscript = null, preparation, media, player: createPlayer = createPcmStreamPlayer, input: inputPort, transcribe = async () => { throw new Error("voice_unsupported"); } }) {
   const voiceId = mintRpcId();
   const headset = device === "headset";
   const headers = { "x-lares-voice-id": voiceId, "x-lares-voice-device": headset ? "headset" : "device" };
   let protocol = null, socketOpen = null, disposed = false, timer = null, lastSessionId = null, continuous = false;
-  let turn = null, reply = null, outputSerial = Promise.resolve(), confirmedReadAloud = readAloud;
+  let turn = null, reply = null, outputSerial = Promise.resolve(), confirmedReadAloud = readAloud, hostHeard = false;
   const call = async (path, body, signal) => {
     const res = await request(`/api/lares/voice${path}`, { method: body === undefined ? "GET" : "POST", headers, ...(body === undefined ? {} : { body }), signal });
     if (!res?.ok) throw new Error(res?.body?.error?.code || "voice_failed");
@@ -47,11 +51,18 @@ export function createUnifiedConversation({ request, socket: openSocket = browse
   } });
   const playback = {
     begin(event) {
-      reply = { id: event.response?.id, itemId: null, text: "", audio: false, playedMs: 0, chain: Promise.resolve() };
+      reply = { id: event.response?.id, itemId: null, text: "", audio: false, playedMs: 0, chain: Promise.resolve(), muted: false };
     },
     write(delta) {
       const current = reply;
       if (!current) return;
+      // Read-aloud was turned off locally but the Host still has audio on
+      // (a lost output update). Converge by cancelling that reply instead of
+      // playing something the user asked not to hear.
+      if (readAloud !== true) {
+        if (!current.muted) { current.muted = true; protocol?.stopOutput(); controller.observePlayback(false); }
+        return;
+      }
       if (!current.audio) { current.audio = true; current.chain = current.chain.then(() => player.start({ sampleRate: RATE, channels: 1, encoding: "pcm_s16le" })); controller.observePlayback(true); }
       current.chain = current.chain.then(() => { if (reply === current) player.write(decodeBase64(delta)); });
       current.chain.catch(() => {});
@@ -59,13 +70,27 @@ export function createUnifiedConversation({ request, socket: openSocket = browse
     async done() {
       const current = reply;
       if (!current) return;
-      if (current.audio) {
-        try { await current.chain; await player.finish(); } catch {}
-        if (reply !== current) return;
-        protocol?.playbackCompleted(current.itemId);
+      let failure = null;
+      try {
+        if (current.audio) {
+          try { await current.chain; await player.finish(); } catch (cause) { failure = cause; }
+          if (reply !== current) return;
+          if (failure) {
+            // Audio that never reached the speaker is a real, visible failure;
+            // release the Host gate too instead of leaving the call muted.
+            protocol?.stopOutput();
+            controller.notePlaybackError(failure.code || failure.message || "voice_playback_failed");
+            return;
+          }
+          protocol?.playbackCompleted(current.itemId);
+        }
+      } finally {
+        // Only this reply may clear playback: a newer reply already owns it.
+        if (reply === current) {
+          reply = null;
+          controller.observePlayback(false);
+        }
       }
-      if (reply === current) reply = null;
-      controller.observePlayback(false);
     },
     async stop() {
       const current = reply;
@@ -86,21 +111,51 @@ export function createUnifiedConversation({ request, socket: openSocket = browse
   const transcription = ({ onPartial }) => {
     if (!protocol) throw new Error("voice_disconnected");
     const current = take();
+    hostHeard = false;
     current.onPartial = onPartial;
+    // The promise exists for the whole take. An error while audio is still
+    // streaming has to reject it; waiting until finish() to attach reject left
+    // the composer on 转写中.
+    let settle, timer = null, committed = false;
+    const result = new Promise((resolve, reject) => {
+      settle = (error, text) => {
+        if (current.done) return;
+        current.done = true;
+        if (timer) { clearTimeout(timer); timer = null; }
+        if (turn === current) turn = null;
+        if (error) reject(error); else resolve(text ?? "");
+      };
+      current.resolve = (text) => settle(null, text);
+      current.reject = (error) => settle(error);
+    });
+    void result.catch(() => {});
     return {
-      write(bytes) { if (turn === current && protocol) protocol.append(encodeBase64(bytes)); },
+      write(bytes) { if (!current.done && protocol) protocol.append(encodeBase64(bytes)); },
+      get committed() { return committed; },
+      // The Host submits a conversation turn on commit, so its recording must
+      // not be uploaded again. Dictation only returns text and stays replayable.
+      get replayable() { return Boolean(onTranscript); },
       finish() {
-        return new Promise((resolve, reject) => {
-          current.resolve = resolve; current.reject = reject;
-          if (!onTranscript) protocol.setBusyTurn(mode());
-          protocol.commit();
-        });
+        if (current.done) return result;
+        if (!onTranscript) protocol.setBusyTurn(mode());
+        protocol.commit();
+        committed = true;
+        if (!timer) {
+          timer = setTimeout(() => {
+            timer = null;
+            if (current.done) return;
+            // The commit is final on the Host; this client just stops waiting so
+            // the composer returns to listening. A late transcript is ignored.
+            current.reject?.(transientVoiceError("voice_asr_timeout"));
+          }, commitTimeoutMs);
+          timer.unref?.();
+        }
+        return result;
       },
-      cancel() { if (turn === current) { turn = null; protocol?.discard(); } },
+      cancel() { if (!current.done) { protocol?.discard(); settle(null, ""); } },
     };
   };
-  const deviceInput = () => createDeviceInput({ media, preparation, onPhase, endpointSilenceMs,
-    transcribe: async () => { throw new Error("voice_unsupported"); },
+  const deviceInput = () => createDeviceInput({ media, preparation, onPhase, endpointSilenceMs, transcribe,
     streamInput: (stream, options) => createStreamingCapture(stream, { ...options, sampleRate: RATE, port: async () => transcription(options) }),
   });
   // Headset: the Host owns the microphone; the socket only opens and closes it.
@@ -109,16 +164,30 @@ export function createUnifiedConversation({ request, socket: openSocket = browse
     capture({ signal, onReady, onSpeech, onTranscribing }) {
       const current = take();
       current.onSpeech = onSpeech; current.onTranscribing = onTranscribing;
+      let timer = null;
+      const clear = () => { if (timer) { clearTimeout(timer); timer = null; } };
       return new Promise((resolve, reject) => {
-        current.resolve = (text) => { protocol?.deviceInput(false); resolve(text); };
-        current.reject = (error) => { protocol?.deviceInput(false); reject(error); };
-        signal.addEventListener("abort", () => { if (turn === current) { turn = null; protocol?.discard(); protocol?.deviceInput(false); resolve(""); } }, { once: true });
+        current.armCommit = () => {
+          if (timer || turn !== current) return;
+          // The Host owns the ASR request. Stop waiting if its transcript never
+          // comes back; the take settles without closing the call.
+          timer = setTimeout(() => {
+            timer = null;
+            if (turn !== current) return;
+            turn = null;
+            current.reject?.(transientVoiceError("voice_asr_timeout"));
+          }, commitTimeoutMs);
+          timer.unref?.();
+        };
+        current.resolve = (text) => { clear(); protocol?.deviceInput(false); resolve(text); };
+        current.reject = (error) => { clear(); protocol?.deviceInput(false); reject(error); };
+        signal.addEventListener("abort", () => { if (turn === current) { turn = null; clear(); protocol?.discard(); protocol?.deviceInput(false); resolve(""); } }, { once: true });
         if (!onTranscript) protocol.setBusyTurn(mode());
         protocol.deviceInput(true);
         onReady?.();
       });
     },
-    stop() { protocol?.commit(); },
+    stop() { const current = turn; current?.armCommit?.(); protocol?.commit(); },
     cancel() { const current = turn; turn = null; protocol?.discard(); protocol?.deviceInput(false); current?.resolve?.(""); },
   });
   const input = inputPort ?? (headset ? hostInput() : deviceInput());
@@ -128,7 +197,13 @@ export function createUnifiedConversation({ request, socket: openSocket = browse
       case "input_audio_buffer.speech_started": turn?.onSpeech?.(); break;
       case "input_audio_buffer.committed": turn?.onTranscribing?.(); break;
       case "conversation.item.input_audio_transcription.delta": if (turn?.onPartial) { turn.partial = `${turn.partial || ""}${event.delta}`; turn.onPartial(turn.partial); } break;
-      case "conversation.item.input_audio_transcription.completed": { const current = turn; turn = null; current?.resolve?.(event.transcript || ""); break; }
+      case "conversation.item.input_audio_transcription.completed": {
+        // The Host already submitted a non-empty transcript. An empty one did not,
+        // so a later blob fallback has to send the words itself.
+        hostHeard = Boolean(String(event.transcript || "").trim());
+        const current = turn; turn = null; current?.resolve?.(event.transcript || "");
+        break;
+      }
       case "response.created": playback.begin(event); break;
       case "response.output_item.added": if (reply) reply.itemId = event.item?.id || null; break;
       case "response.output_audio_transcript.delta":
@@ -138,7 +213,14 @@ export function createUnifiedConversation({ request, socket: openSocket = browse
       case "lares.device.playback": controller.observePlayback(event.playing === true); break;
       case "error": {
         const code = event.error?.code || "voice_failed";
-        if (turn && ["unsupported_audio_format", "audio_limit_exceeded", "turn_busy", "waiting_for_playback", "upstream_failed"].includes(code)) { const current = turn; turn = null; current.reject?.(new Error(code)); }
+        // Any error with a take open belongs to that take. Settling it here is
+        // what keeps the composer off 转写中; transient contention drops only
+        // the take, not the call.
+        if (turn) {
+          const current = turn;
+          turn = null;
+          current.reject?.(isTransientVoiceError(code) ? transientVoiceError(code) : Object.assign(new Error(code), { code }));
+        }
         if (code === "session_expired") void controller.disconnect("voice_idle_timeout");
         break;
       }
@@ -171,7 +253,14 @@ export function createUnifiedConversation({ request, socket: openSocket = browse
       lazy: true,
       create: async () => null,
       // The Host submitted this turn when its audio was committed; only the lifecycle remains to observe.
-      async submit() { await poll(); },
+      async submit({ text } = {}) {
+        // Audio that never reached the socket still has a recording. The Host did not
+        // see that take, so hand the transcript over as a typed turn instead of dropping it.
+        const typed = Boolean(text) && !hostHeard;
+        hostHeard = false;
+        if (typed) protocol?.say(text);
+        await poll();
+      },
       cancel: () => call("/session/cancel", {}),
       running: async () => (await call("/session")).running === true,
     },
@@ -179,17 +268,21 @@ export function createUnifiedConversation({ request, socket: openSocket = browse
   const poll = async () => {
     if (disposed || onTranscript) return;
     const state = await call("/session");
-    if (disposed || controller.getSnapshot().connection !== "open") return;
+    // A paused call still has a chat session. Stop watching only after it is closed,
+    // otherwise a dropped socket leaves the composer on "working" after the task ends.
+    if (disposed || controller.getSnapshot().connection === "closed") return;
     lastSessionId = state.sessionId;
     if (lastSessionId) controller.bindSession(lastSessionId);
     if (state.error && state.error !== "voice_cancelled") throw new Error(state.error);
     controller.observeAgent(Boolean(state.running) || Boolean(reply), Boolean(reply?.audio));
   };
   const watch = async () => {
-    if (disposed || controller.getSnapshot().connection !== "open") return;
+    if (disposed || controller.getSnapshot().connection === "closed") return;
     try { await poll(); }
-    catch (error) { await controller.disconnect(error.code || error.message || "voice_disconnected"); return; }
-    if (!disposed) timer = setTimeout(watch, 500);
+    catch (error) {
+      if (controller.getSnapshot().connection === "open") await controller.disconnect(error.code || error.message || "voice_disconnected");
+    }
+    if (!disposed && controller.getSnapshot().connection !== "closed") timer = setTimeout(watch, 500);
   };
   const stop = async () => {
     disposed = true; clearTimeout(timer);

package/voice/vad.js

@@ -0,0 +1,169 @@
+import { ENDPOINT, createEnergyEndpoint, createSpeechEndpoint, frameRms } from "./endpoint.js";
+
+/** Silero v5 frames are 512 samples at 16 kHz. */
+const FRAME_MS = 32;
+const ASSET_ROOT = "/api/lares/voice/vad/";
+
+let runtime = null;
+
+function loadScript(src) {
+  return new Promise((resolve, reject) => {
+    const existing = document.querySelector(`script[data-lares-vad="${src}"]`);
+    if (existing?.dataset.loaded === "true") { resolve(); return; }
+    const script = existing ?? document.createElement("script");
+    script.dataset.laresVad = src;
+    script.addEventListener("load", () => { script.dataset.loaded = "true"; resolve(); }, { once: true });
+    // Remove a failed tag so the next attempt creates a fresh one instead of
+    // attaching listeners to a script that will never fire again.
+    script.addEventListener("error", () => { script.remove(); reject(new Error(`failed to load ${src}`)); }, { once: true });
+    if (!existing) { script.src = src; document.head.append(script); }
+  });
+}
+
+async function loadRuntime() {
+  if (typeof document === "undefined") throw new Error("voice_unsupported");
+  await loadScript(`${ASSET_ROOT}ort.js`);
+  await loadScript(`${ASSET_ROOT}vad.js`);
+  if (!globalThis.vad?.MicVAD) throw new Error("voice_vad_unavailable");
+  return globalThis.vad;
+}
+
+/**
+ * One Silero session for the page. `MicVAD.new` builds the ONNX session, and
+ * `destroy` releases it, so a recording only pauses that session and the next
+ * one starts it again on a new stream.
+ */
+export function createVadSession(factory) {
+  let loading;
+  let detector;
+  let stream = null;
+  let ended = () => {};
+  let endpoint = createSpeechEndpoint();
+  let active = false;
+  let spoke = false;
+  let options = {};
+  let tail = Promise.resolve();
+
+  function load() {
+    loading ??= Promise.resolve()
+      .then(() => factory({
+        getStream: async () => stream,
+        resumeStream: async () => stream,
+        onFrame: (prob, frame) => {
+          if (!active) return;
+          const rms = frameRms(frame);
+          options.onLevel?.(rms);
+          if (!spoke && prob >= ENDPOINT.speechProb && rms >= ENDPOINT.floor) { spoke = true; options.onSpeech?.(); }
+          if (endpoint.push(prob, rms, FRAME_MS)) ended();
+        },
+      }))
+      .then((created) => {
+        detector = created;
+        return created;
+      })
+      .catch((err) => {
+        loading = undefined;
+        throw err;
+      });
+    return loading;
+  }
+
+  return {
+    preload() {
+      return load();
+    },
+    open(nextStream, onSpeechEnd, nextOptions = {}) {
+      const run = tail.then(async () => {
+        const created = await load();
+        stream = nextStream;
+        ended = onSpeechEnd;
+        spoke = false;
+        // Options belong to this recording, not to whichever open() came last.
+        options = nextOptions;
+        endpoint = createSpeechEndpoint({ silenceMs: options.silenceMs });
+        await created.start();
+        active = true;
+      });
+      tail = run.then(() => {}, () => {});
+      return run.then(() => ({
+        stop() {
+          if (!active) return;
+          active = false;
+          stream = null;
+          tail = tail.then(() => detector.pause()).then(() => {}, () => {});
+        },
+      }));
+    },
+  };
+}
+
+let session = null;
+let ready = false;
+let preloading = null;
+let retryAt = 0;
+
+function sileroSession() {
+  session ??= createVadSession(async (hooks) => {
+    const vad = await loadRuntime();
+    return vad.MicVAD.new({
+      model: "v5",
+      startOnLoad: false,
+      baseAssetPath: ASSET_ROOT,
+      onnxWASMBasePath: ASSET_ROOT,
+      getStream: hooks.getStream,
+      pauseStream: async () => {},
+      resumeStream: hooks.resumeStream,
+      // Olares Desktop's CEF rejects dynamically loaded AudioWorklet modules.
+      // ScriptProcessor feeds the same 16 kHz frames into the same Silero model.
+      processorType: "ScriptProcessor",
+      onFrameProcessed: (probs, frame) => hooks.onFrame(probs.isSpeech, frame),
+      onSpeechEnd: () => {},
+      onVADMisfire: () => {},
+      onSpeechStart: () => {},
+      onSpeechRealStart: () => {},
+    });
+  });
+  return session;
+}
+
+/**
+ * Warm the model once. Returns whether Silero is usable here; a webview without
+ * the runtime keeps the energy endpoint instead of failing the take.
+ */
+export function preloadVoiceVad() {
+  if (typeof document === "undefined") return Promise.resolve(false);
+  if (ready) return Promise.resolve(true);
+  // An unavailable runtime is retried eventually, not on every single take.
+  if (Date.now() < retryAt) return Promise.resolve(false);
+  preloading ??= sileroSession().preload().then(() => { ready = true; return true; }, () => {
+    retryAt = Date.now() + 60_000;
+    preloading = null;
+    return false;
+  });
+  return preloading;
+}
+
+export function voiceVadReady() { return ready; }
+
+/**
+ * Endpoint backed by Silero when the model is already loaded. `null` tells the
+ * caller to use the energy endpoint for this take, so the first recording is
+ * never delayed by model download.
+ */
+export function createVoiceVadEndpoint(stream, onSpeechEnd, onSpeech = () => {}, onLevel = () => {}, options = {}) {
+  if (!ready) { void preloadVoiceVad(); return null; }
+  let handle = null, fallback = null, stopped = false;
+  sileroSession().open(stream, onSpeechEnd, { ...options, onSpeech, onLevel }).then((value) => {
+    if (stopped) value?.stop(); else handle = value;
+  }, () => {
+    // The model failed after preload; keep this take usable with energy endpointing.
+    if (!stopped) fallback = createEnergyEndpoint(stream, onSpeechEnd, onSpeech, onLevel, options);
+  }).catch(() => {});
+  return {
+    stop() {
+      stopped = true;
+      handle?.stop();
+      fallback?.stop();
+    },
+  };
+}