@olares/lares-core 0.15.28 → 0.15.30

package/i18n/voice.js

@@ -132,6 +132,13 @@ export const ZH = {
   "error.voice_too_short": "录音太短",
   "error.voice_no_input": "没有检测到声音",
   "error.voice_failed": "转写失败，请重试",
+  "error.voice_asr_timeout": "转写等待超时，请重试",
+  "error.voice_audio_backpressure": "播放缓冲已满，请重试",
+  "error.voice_playback_failed": "播放失败，请重试",
+  "error.voice_stream_state": "语音连接状态异常，请重试",
+  "error.voice_interruption_timeout": "打断等待超时，已停止朗读",
+  "error.turn_busy": "上一轮还在处理，请再说一次",
+  "error.waiting_for_playback": "上一段回复还在播放，请稍后再说",
   "error.voice_unsupported": "当前浏览器不支持录音",
   "error.voice_bluetooth_not_ready": "蓝牙麦克风还没连上，请重试",
   "error.voice_bluetooth_unavailable": "这台机器的蓝牙还没就绪。请确认机器有蓝牙并已打开，然后在 Olares 市场重启 Lares，重启时会重新安装蓝牙组件",
@@ -436,6 +443,13 @@ export const EN = {
   "error.voice_too_short": "Recording too short",
   "error.voice_no_input": "No voice detected",
   "error.voice_failed": "Transcription failed, please retry",
+  "error.voice_asr_timeout": "Timed out waiting for the transcript, please retry",
+  "error.voice_audio_backpressure": "Playback buffer is full, please retry",
+  "error.voice_playback_failed": "Playback failed, please retry",
+  "error.voice_stream_state": "The voice connection lost its place, please retry",
+  "error.voice_interruption_timeout": "Interruption timed out; playback stopped",
+  "error.turn_busy": "The previous turn is still running, please speak again",
+  "error.waiting_for_playback": "The previous answer is still playing, please wait",
   "error.voice_unsupported": "This browser does not support recording",
   "error.voice_bluetooth_not_ready": "Bluetooth microphone is not ready, please retry",
   "error.voice_bluetooth_unavailable":

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@olares/lares-core",
-  "version": "0.15.28",
+  "version": "0.15.30",
   "description": "Lares 共享业务核心：Router、workspace、文件/媒体、Olares 身份（无 UI、无平台壳）",
   "type": "module",
   "repository": {
@@ -224,7 +224,9 @@
     "./router/default-selection": {
       "types": "./router/default-selection.d.ts",
       "default": "./router/default-selection.js"
-    }
+    },
+    "./voice/vad": "./voice/vad.js",
+    "./voice/errors": "./voice/errors.js"
   },
   "dependencies": {
     "emoji-regex": "11.0.0",

package/voice/browser-capture.js

@@ -162,18 +162,25 @@ export function createVoiceCapture({
     abort = new AbortController();
     try {
       let text = "";
+      let committed = false, replayable = true;
       if (streaming) {
         const port = streaming;
         streaming = null;
+        committed = port.committed === true;
+        replayable = port.replayable !== false;
         try {
           text = String(await port.finish() ?? "").trim();
         } catch (err) {
           if (dead || abort.signal.aborted || cancel) return;
           text = "";
         }
+        // finish() is the commit; the Host transcribed this audio and returned
+        // nothing. Uploading the blob again would submit the same take twice.
+        committed ||= port.committed === true;
+        replayable = port.replayable !== false;
       }
       // The socket can finish with nothing while the recording still has the words.
-      if (!text) text = String(await transcribe(blob, abort.signal) ?? "").trim();
+      if (!text && (!committed || replayable)) text = String(await transcribe(blob, abort.signal) ?? "").trim();
       if (dead || cancel) return;
       if (!text) {
         trace("no-input", {});

package/voice/controller.js

@@ -1,4 +1,5 @@
 import { createInterruptionGuard } from "./interruption.js";
+import { isTransientVoiceError, voiceErrorCode } from "./errors.js";
 import { mintRpcId } from "../larepass/rpc.js";
 
 /** Device, agent and playback ports are injected; UI frameworks own no turn state. */
@@ -6,6 +7,7 @@ export function createVoiceController({ input, agent, output, voiceId = mintRpcI
   let epoch = 0;
   let manualTurns = false;
   let take = null;
+  let playbackError = null;
   let idleTimer = null;
   let recoveryTimer = null;
   const clearTimers = () => { unschedule(idleTimer); unschedule(recoveryTimer); };
@@ -64,9 +66,13 @@ export function createVoiceController({ input, agent, output, voiceId = mintRpcI
     )) return;
     const run = { epoch, abort: new AbortController(), id: mintRpcId(), started: now() };
     take = run;
-    change({ capture: "recording", inputReady: false, error: null });
+    // A playback failure is not cleared by merely listening again; the next
+    // reply that actually renders clears it.
+    change({ capture: "recording", inputReady: false, error: playbackError });
     try {
-      const text = String(await input.capture({ signal: run.abort.signal, continuous: state.continuous && !manualTurns, onReady: () => { if (live(run)) change({ inputReady: true }); }, onLevel: (level) => { if (live(run)) change({ inputLevel: Math.max(0, Math.min(1, level)) }); }, onSpeech: () => {
+      // A call outlives each take. `manual` only means the finger ends the
+      // recording; it must not turn that take into a one-shot dictation.
+      const text = String(await input.capture({ signal: run.abort.signal, continuous: state.continuous, manual: manualTurns, onReady: () => { if (live(run)) change({ inputReady: true }); }, onLevel: (level) => { if (live(run)) change({ inputLevel: Math.max(0, Math.min(1, level)) }); }, onSpeech: () => {
         if (live(run)) { touch(); change({ userSpeaking: true }); if (interruptible() && state.playback !== "idle") interruption.suspect(); }
       }, onTranscribing: () => { if (live(run)) { interruption.speechEnded(); change({ capture: "transcribing", userSpeaking: false }); } } }) ?? "").trim();
       if (!live(run)) return;
@@ -87,8 +93,18 @@ export function createVoiceController({ input, agent, output, voiceId = mintRpcI
       // Acceptance is not completion; observeAgent receives the durable turn lifecycle.
     } catch (error) {
       if (live(run)) {
-        interruption.fail(error.code || error.message || "voice_failed");
-        await controller.disconnect(error.code || error.message || "voice_failed");
+        const code = voiceErrorCode(error) || "voice_failed";
+        if (isTransientVoiceError(code)) {
+          // The take lost a race with the previous turn or with a reply that
+          // still owns the shared audio link. Drop only this take: the call
+          // stays open and the composer returns to listening for the next one.
+          interruption.cancel();
+          report("take-dropped", run.started, { turnId: run.id, code });
+          change({ capture: "idle", userSpeaking: false, inputLevel: null });
+          return;
+        }
+        interruption.fail(code);
+        await controller.disconnect(code);
       }
     } finally {
       if (take === run) {
@@ -104,11 +120,19 @@ export function createVoiceController({ input, agent, output, voiceId = mintRpcI
     getSnapshot: () => state,
     noteActivity: touch,
     setOutputMuted(outputMuted) { change({ outputMuted }); },
+    /** Audio that never reached the speaker is visible, not silently swallowed. */
+    notePlaybackError(code) {
+      const value = voiceErrorCode(code) || "voice_playback_failed";
+      playbackError = value;
+      report("playback-error", now(), { code: value });
+      change({ error: value });
+    },
     bindSession(sessionId) { if (sessionId) change({ sessionId }); },
     subscribe(listener) { listeners.add(listener); return () => listeners.delete(listener); },
     async start({ sessionId = null, continuous = false, running = false, manual = false, shouldStart = () => true } = {}) {
       output.arm?.();
       manualTurns = manual;
+      playbackError = null;
       const startedEpoch = ++epoch;
       creating = null;
       await cancelInput();
@@ -137,7 +161,14 @@ export function createVoiceController({ input, agent, output, voiceId = mintRpcI
     observePlayback(playing) {
       if (!playing) interruption.outputEnded();
       if (playing !== (state.playback === "playing")) touch();
+      // The next reply that actually renders clears the previous audio failure.
+      if (playing && playbackError) { playbackError = null; change({ error: null }); }
       change({ playback: playing ? "playing" : "idle" });
+      // Output and microphone share one link. As soon as audio is rendered, a
+      // take that is still recording would only send the speaker back to ASR,
+      // and the Host rejects those frames. Stop it here instead of waiting for
+      // the next agent poll.
+      if (playing && take && !interruptible()) void cancelInput();
       if (!manualTurns && state.continuous && (!playing || interruptible())) void listen();
     },
     async pause() {

package/voice/device-input.js

@@ -1,44 +1,16 @@
 import { createVoiceCapture } from "./browser-capture.js";
-import { createSpeechEndpoint, frameRms } from "./endpoint.js";
+import { createEnergyEndpoint, createInputMeter } from "./endpoint.js";
+import { createVoiceVadEndpoint, preloadVoiceVad } from "./vad.js";
 
-export function createEnergyEndpoint(stream, end, onSpeech = () => {}, onLevel = () => {}, options = {}) {
-  const AudioCtx = globalThis.AudioContext || globalThis.webkitAudioContext;
-  if (!AudioCtx) throw new Error("voice_unsupported");
-  const context = new AudioCtx();
-  const source = context.createMediaStreamSource(stream);
-  const processor = context.createScriptProcessor(2048, 1, 1);
-  const gain = context.createGain();
-  gain.gain.value = 0;
-  const endpoint = createSpeechEndpoint(options);
-  let heard = false, consecutiveMs = 0;
-  const silenceTimer = setTimeout(() => { if (!heard) end(false); }, 10_000);
-  processor.onaudioprocess = (event) => {
-    const samples = event.inputBuffer.getChannelData(0);
-    const rms = frameRms(samples);
-    onLevel(rms);
-    consecutiveMs = rms >= 0.004 ? consecutiveMs + samples.length * 1000 / context.sampleRate : 0;
-    if (consecutiveMs >= (options.interruptionMs || 0) && rms >= 0.004 && !heard) { heard = true; onSpeech(); }
-    if (endpoint.push(1, rms, samples.length * 1000 / context.sampleRate)) end(true);
-  };
-  source.connect(processor); processor.connect(gain); gain.connect(context.destination);
-  void context.resume();
-  return { stop() { clearTimeout(silenceTimer); processor.onaudioprocess = null; source.disconnect(); processor.disconnect(); gain.disconnect(); void context.close(); } };
-}
+export { createEnergyEndpoint };
 
-function createInputMeter(stream, onLevel) {
-  const AudioCtx = globalThis.AudioContext || globalThis.webkitAudioContext;
-  if (!AudioCtx) return { stop() {} };
-  const context = new AudioCtx();
-  const source = context.createMediaStreamSource(stream);
-  const analyser = context.createAnalyser(); analyser.fftSize = 512;
-  source.connect(analyser);
-  const samples = new Float32Array(analyser.fftSize);
-  const timer = setInterval(() => { analyser.getFloatTimeDomainData(samples); onLevel(frameRms(samples)); }, 80);
-  void context.resume();
-  return { stop() { clearInterval(timer); source.disconnect(); analyser.disconnect(); void context.close(); } };
+/** Silero once it is loaded; the energy endpoint otherwise or on any VAD failure. */
+export function defaultEndpoint(stream, end, onSpeech, onLevel, options) {
+  return createVoiceVadEndpoint(stream, end, onSpeech, onLevel, options)
+    ?? createEnergyEndpoint(stream, end, onSpeech, onLevel, options);
 }
 
-export function createDeviceInput({ transcribe, onPhase = () => {}, endpoint = createEnergyEndpoint, endpointSilenceMs = 500, media, preparation, streamInput, echoTested = false } = {}) {
+export function createDeviceInput({ transcribe, onPhase = () => {}, endpoint = defaultEndpoint, endpointSilenceMs = 500, media, preparation, streamInput, echoTested = false } = {}) {
   let recorder = null;
   let settle = null;
   let timer = null;
@@ -49,8 +21,13 @@ export function createDeviceInput({ transcribe, onPhase = () => {}, endpoint = c
   };
   return {
     capabilities: { duplex: true, bargeIn: echoTested === true, echoTested: echoTested === true, listenWhileWorking: true, audio: "encoded" },
-    capture({ signal, onTranscribing, onReady, onSpeech, onLevel = () => {}, continuous = false } = {}) {
+    capture({ signal, onTranscribing, onReady, onSpeech, onLevel = () => {}, continuous = false, manual = false } = {}) {
       cancel();
+      // Hands-free calls end themselves. A manual hold is still one take of the same call.
+      const automatic = continuous && !manual;
+      // Only automatic turns need a speech model. The first take may use energy
+      // endpointing; the model is warmed for the next one instead of delaying it.
+      if (automatic) void preloadVoiceVad();
       return new Promise((resolve, reject) => {
         let ready = false;
         settle = resolve;
@@ -58,11 +35,8 @@ export function createDeviceInput({ transcribe, onPhase = () => {}, endpoint = c
         const prepared = preparation; preparation = null;
         recorder = createVoiceCapture({ media, streamInput, preparation: prepared, transcribe, submit: done, onStream: (stream) => {
           for (const track of stream.getTracks()) track.addEventListener?.("ended", () => { settle = null; cancel(); reject(new Error("voice_device_lost")); }, { once: true });
-          if (!continuous) return createInputMeter(stream, onLevel);
-          return endpoint(stream, (heard = true) => { if (heard) recorder?.stop(); else {
-            if (continuous) { cancel(); done(""); }
-            else { settle = null; cancel(); reject(new Error("voice_no_input")); }
-          } }, onSpeech, onLevel, { silenceMs: endpointSilenceMs, interruptionMs: echoTested ? 500 : 0 });
+          if (!automatic) return createInputMeter(stream, onLevel);
+          return endpoint(stream, (heard = true) => { if (heard) recorder?.stop(); else { cancel(); done(""); } }, onSpeech, onLevel, { silenceMs: endpointSilenceMs, interruptionMs: echoTested ? 500 : 0 });
         }, onPhase(phase, extra) {
           onPhase(phase, extra);
           if (phase === "recording" && !ready) {
@@ -71,7 +45,7 @@ export function createDeviceInput({ transcribe, onPhase = () => {}, endpoint = c
           }
           if (phase === "transcribing") onTranscribing?.();
           if (phase === "error") {
-            if (continuous && (extra.recognizedEmpty || !echoTested && ["voice_no_input", "voice_too_short"].includes(extra.error))) { done(""); return; }
+            if (continuous && (extra.recognizedEmpty || ["voice_no_input", "voice_too_short"].includes(extra.error))) { done(""); return; }
             clearTimeout(timer); signal?.removeEventListener("abort", cancel); settle = null; reject(new Error(extra.error)); }
         } });
         signal?.addEventListener("abort", cancel, { once: true });

package/voice/endpoint.js

@@ -2,8 +2,8 @@
  * Decide when a take is over from Silero's per-frame speech probability and the
  * frame's loudness. A conference mic hears the whole room, so a frame only
  * counts as the speaker when it is human speech and within `relative` of the
- * loudest speech already heard in this take. Farther voices then do not hold
- * the recording open.
+ * loudest speech heard recently in this take. Farther voices then do not hold
+ * the recording open, and a single loud cough does not raise the bar forever.
  */
 export const ENDPOINT = {
   speechProb: 0.5,
@@ -14,6 +14,9 @@ export const ENDPOINT = {
   relative: 0.3,
   floor: 0.004,
   smoothFrames: 5,
+  // The reference is a leaky maximum: it halves roughly every five seconds of
+  // speech-level audio, so a door slam cannot close the take for the rest of it.
+  referenceDecay: 0.995,
 };
 
 export function frameRms(frame) {
@@ -39,6 +42,7 @@ export function createSpeechEndpoint(options = {}) {
       if (recent.length > o.smoothFrames) recent.shift();
       const level = recent.reduce((sum, value) => sum + value, 0) / recent.length;
       const human = prob >= o.speechProb;
+      reference *= o.referenceDecay;
       if (human && level > reference) reference = level;
       const near = human && level >= Math.max(o.floor, reference * o.relative);
       if (near) {
@@ -55,3 +59,55 @@ export function createSpeechEndpoint(options = {}) {
     },
   };
 }
+
+/** Widest room-noise floor a quiet speaker is still expected to clear. */
+const MAX_NOISE_FLOOR = 0.02;
+
+/**
+ * Endpointing without a speech model. The room's floor is estimated from the
+ * frames before the speaker starts and only needs to be cleared by that much,
+ * so a loud room does not keep every take open and a quiet speaker still ends
+ * it. Used by webviews that cannot load Silero, and as its fallback.
+ */
+export function createEnergyEndpoint(stream, end, onSpeech = () => {}, onLevel = () => {}, options = {}) {
+  const AudioCtx = globalThis.AudioContext || globalThis.webkitAudioContext;
+  if (!AudioCtx) throw new Error("voice_unsupported");
+  const context = new AudioCtx();
+  const source = context.createMediaStreamSource(stream);
+  const processor = context.createScriptProcessor(2048, 1, 1);
+  const gain = context.createGain();
+  gain.gain.value = 0;
+  const endpoint = createSpeechEndpoint(options);
+  let heard = false, consecutiveMs = 0, noise = null, spoke = false;
+  const silenceTimer = setTimeout(() => { if (!heard) end(false); }, 10_000);
+  processor.onaudioprocess = (event) => {
+    const samples = event.inputBuffer.getChannelData(0);
+    const rms = frameRms(samples);
+    const frameMs = samples.length * 1000 / context.sampleRate;
+    onLevel(rms);
+    if (!spoke) noise = noise == null ? rms : noise * 0.9 + rms * 0.1;
+    const floor = Math.max(ENDPOINT.floor, Math.min(MAX_NOISE_FLOOR, (noise ?? ENDPOINT.floor / 2.5) * 2.5));
+    const human = rms >= floor;
+    if (human) spoke = true;
+    consecutiveMs = human ? consecutiveMs + frameMs : 0;
+    if (consecutiveMs >= (options.interruptionMs || 0) && human && !heard) { heard = true; onSpeech(); }
+    if (endpoint.push(human ? 1 : 0, rms, frameMs)) end(true);
+  };
+  source.connect(processor); processor.connect(gain); gain.connect(context.destination);
+  void context.resume();
+  return { stop() { clearTimeout(silenceTimer); processor.onaudioprocess = null; source.disconnect(); processor.disconnect(); gain.disconnect(); void context.close(); } };
+}
+
+/** Visible microphone level for a take that ends by the user's finger. */
+export function createInputMeter(stream, onLevel) {
+  const AudioCtx = globalThis.AudioContext || globalThis.webkitAudioContext;
+  if (!AudioCtx) return { stop() {} };
+  const context = new AudioCtx();
+  const source = context.createMediaStreamSource(stream);
+  const analyser = context.createAnalyser(); analyser.fftSize = 512;
+  source.connect(analyser);
+  const samples = new Float32Array(analyser.fftSize);
+  const timer = setInterval(() => { analyser.getFloatTimeDomainData(samples); onLevel(frameRms(samples)); }, 80);
+  void context.resume();
+  return { stop() { clearInterval(timer); source.disconnect(); analyser.disconnect(); void context.close(); } };
+}

package/voice/errors.js

@@ -0,0 +1,40 @@
+/**
+ * A take can lose a race with the turn that is still being submitted or with a
+ * reply that is still playing. That is contention on one shared audio link, not
+ * a broken voice session: the take is dropped and the call keeps listening.
+ *
+ * Everything else stays a real failure so it is never hidden from the user.
+ */
+export const TRANSIENT_VOICE_ERRORS = Object.freeze([
+  // The previous turn is still being transcribed/submitted (or the Host queued it).
+  "turn_busy",
+  // Half duplex: the reply owns the link, or its audio is still in flight.
+  "waiting_for_playback",
+  // One take exceeded the bounded transport buffers.
+  "voice_audio_backpressure",
+  "voice_stream_state",
+  // The take hit the transport's own size/limit for a single utterance.
+  "audio_limit_exceeded",
+  // The Host never returned a transcript for a committed take.
+  "voice_asr_timeout",
+]);
+
+const TRANSIENT = new Set(TRANSIENT_VOICE_ERRORS);
+
+export function voiceErrorCode(value) {
+  if (typeof value === "string") return value;
+  return value?.code || value?.message || "";
+}
+
+/** @returns {boolean} true when only this take failed, not the connection. */
+export function isTransientVoiceError(value) {
+  return TRANSIENT.has(voiceErrorCode(value));
+}
+
+/** Marks a rejection so the controller drops one take instead of the call. */
+export function transientVoiceError(code) {
+  const error = new Error(code);
+  error.code = code;
+  error.transient = true;
+  return error;
+}

package/voice/host-service.js

@@ -68,7 +68,7 @@ export function createVoiceHostService(api, { env = process.env, metric = () =>
       const restoredRequests = [...new Set(history.records.map((row) => row.taskId).filter(Boolean))].map((requestId) => ({ requestId }));
       scope.session = createVoiceAgentSession(api, { sessionId: boundSessionId,
         preset: scope.authorizedBinding?.agentPreset || env.LARES_VOICE_AGENT_PRESET || undefined, workspaceId: scope.authorizedBinding?.workspaceId || env.LARES_VOICE_WORKSPACE_ID || undefined, restoredRequests,
-        onChange: () => { if (scope.active) service.state(scope); },
+        onChange: () => service.advance(scope),
         metric: (fields) => metric({ voiceId: scope.id, ...fields }),
       });
       await scope.session.open();
@@ -146,9 +146,14 @@ export function createVoiceHostService(api, { env = process.env, metric = () =>
       scope.voiceRequests.set(body.requestId, operation);
       return operation;
     },
-    state(scope) {
+    /**
+     * Turns a durable task's completion into one pending spoken utterance.
+     * Called from session events, never from a read: a GET that loses its
+     * response must not consume the result it was about to return.
+     */
+    advance(scope) {
+      if (!scope.active || !scope.voiceState) return;
       const state = scope.session?.snapshot() || {};
-      if (!scope.voiceState) return { ...state, contextId: scope.contextId, active: scope.active === true };
       // Completion is keyed by durable request/turn, never by the session's aggregate idle flag.
       for (const task of state.tasks || []) {
         if (!["completed", "failed", "cancelled"].includes(task.status) || scope.deliveredTasks.has(task.requestId)) continue;
@@ -157,15 +162,26 @@ export function createVoiceHostService(api, { env = process.env, metric = () =>
         scope.voiceResults ??= [];
         scope.voiceResults.push(task);
       }
-      if (scope.voiceConfig.preset === "direct") return { ...state, contextId: scope.contextId, active: scope.active === true };
+      if (scope.voiceConfig.preset === "direct" || scope.voiceState.running) return;
       // Pending business work does not block the next speech turn.
-      if (scope.active && !scope.voiceState.running && scope.voiceResults?.length && !scope.voiceInputActive) {
-        const task = scope.voiceResults.shift();
-        scope.dialogue.observeTask(task);
-        const text = task.status === "completed" ? task.text : `Task ${task.status}: ${task.error || ""}`;
-        scope.voiceState.utterance = { key: `task:${task.requestId}`, turn: ++scope.voiceState.version, step: 0, text, done: true, silent: false };
-        append(scope, "assistant", text, `result:${task.requestId}`, task.requestId, { taskTurn: task.turn });
-      }
+      if (!scope.voiceResults?.length || scope.voiceInputActive) return;
+      const task = scope.voiceResults.shift();
+      scope.dialogue.observeTask(task);
+      const text = task.status === "completed" ? task.text : `Task ${task.status}: ${task.error || ""}`;
+      scope.voiceState.utterance = { key: `task:${task.requestId}`, turn: ++scope.voiceState.version, step: 0, text, done: true, silent: false };
+      append(scope, "assistant", text, `result:${task.requestId}`, task.requestId, { taskTurn: task.turn });
+    },
+    /**
+     * A queued task result moves into the sticky `utterance` field before the
+     * snapshot is returned. The move is idempotent and the utterance survives
+     * the read, so a lost response cannot consume the result; HTTP polls and
+     * the socket observe the same snapshot.
+     */
+    state(scope) {
+      service.advance(scope);
+      const state = scope.session?.snapshot() || {};
+      if (!scope.voiceState) return { ...state, contextId: scope.contextId, active: scope.active === true };
+      if (scope.voiceConfig.preset === "direct") return { ...state, contextId: scope.contextId, active: scope.active === true };
       return { ...state, ...scope.voiceState, contextId: scope.contextId, active: scope.active === true, backgroundRunning: state.running, summaryError: scope.summaryError || null, preset: scope.voiceConfig.preset };
     },
     async remove(sessionId) {