@olares/lares-core 0.15.10 → 0.15.11

package/voice/service.js

@@ -19,8 +19,15 @@ import {
 import { createCaptureSession } from "./capture.js";
 import { createDuplexCoordinator } from "./duplex.js";
 import { normalizeSttPcm } from "./pcm.js";
-import { speechSegments } from "./script.js";
-import { pcmToWav, silencePcm, synthesize, synthesizeStream } from "../router/tts.js";
+import { gapAfter, speechSegments } from "./script.js";
+import {
+  listTtsVoices,
+  pcmToWav,
+  pickTtsVoice,
+  silencePcm,
+  synthesize,
+  synthesizeStream,
+} from "../router/tts.js";
 import {
   STT_MAX_AUDIO_BYTES,
   VoiceError,
@@ -155,50 +162,167 @@ function cancelledSpeech() {
 
 const readouts = new WeakMap();
 
+/** model id → pinned voice. A failed lookup is not cached, so the next readout asks again. */
+const pinnedVoices = new Map();
+
+async function pinnedVoice(model, listVoices = listTtsVoices) {
+  if (pinnedVoices.has(model)) return pinnedVoices.get(model);
+  let voice = null;
+  try {
+    voice = pickTtsVoice(await listVoices(model));
+  } catch {
+    return null;
+  }
+  if (voice) pinnedVoices.set(model, voice);
+  return voice;
+}
+
+function deferred() {
+  let resolve;
+  let reject;
+  const promise = new Promise((ok, fail) => {
+    resolve = ok;
+    reject = fail;
+  });
+  promise.catch(() => {});
+  return { promise, resolve, reject };
+}
+
+function partIndex(value) {
+  return Number.isInteger(value) && value >= 0 ? value : null;
+}
+
+/** How long a readout of a streaming answer waits for its next sentence before it lets the headset go. */
+const STREAM_IDLE_MS = 30_000;
+
+function joinable(current, id, key, part) {
+  if (current?.id !== id) return null;
+  if (key && current.signal.aborted) throw cancelledSpeech();
+  if (key || part > 0) return current.signal.aborted ? null : current;
+  return current.active() ? current : null;
+}
+
 /**
- * Every open page asks to read the same answer when it concludes. A request
- * for the readout already under way joins it instead of restarting the
- * headset and cancelling the page that asked first.
+ * Every open page asks to read the same answer. A request for the readout
+ * already under way joins it instead of restarting the headset and cancelling
+ * the page that asked first.
+ *
+ * `options.key` names an answer that is still being written: each request
+ * carries the text so far, and `options.done` once it is complete. The readout
+ * appends the sentences it has not seen yet, so the headset reads on without a
+ * restart. A keyed request without `options.part` only hands over the text.
+ * Once interrupted, a key stays cancelled.
+ *
+ * `options.part` asks for one sentence instead of the whole answer, so a page
+ * can play it while the rest is still being written or synthesized. A later
+ * sentence joins its readout even after synthesis has finished.
+ *
+ * Without a key the text is the whole answer and names its readout.
+ *
+ * @returns {Promise<{ bluetooth: string, audio: Buffer } | { sentences: number }>}
  */
 export async function speakFromConfig(text, options = {}) {
   const config = options.config ?? readConfig();
   if (!config.speak) throw new VoiceError("voice_speak_disabled", 409, "speech playback is off");
-  const segments = speechSegments(text);
-  if (!segments.length) throw new VoiceError("voice_config_invalid", 400, "nothing to speak");
-  const model = options.model ?? await pickTtsModelId(await listModels(), config.ttsModel);
-  if (!model) throw new VoiceError("voice_tts_unavailable", 503, "no speech model available");
+  const key = typeof options.key === "string" && options.key ? options.key : null;
+  const done = key ? options.done === true : true;
+  const segments = speechSegments(text, { done });
+  const part = partIndex(options.part);
   const coordinator = options.coordinator ?? duplex;
-  const current = readouts.get(coordinator);
-  if (current?.text === text && current.active()) return current.result;
+  const id = key ?? text;
+  let readout = joinable(readouts.get(coordinator), id, key, part);
+  if (!readout) {
+    if (!segments.length) {
+      if (key && part === null) return { sentences: 0 };
+      throw new VoiceError("voice_config_invalid", 400, "nothing to speak");
+    }
+    const model = options.model ?? await pickTtsModelId(await listModels(), config.ttsModel);
+    if (!model) throw new VoiceError("voice_tts_unavailable", 503, "no speech model available");
+    readout = joinable(readouts.get(coordinator), id, key, part) ?? startReadout(coordinator, id, model, options);
+  }
+  readout.extend(segments, done);
+  if (part === null) return key ? { sentences: readout.segments.length } : readout.result;
+  if (part >= readout.segments.length) {
+    throw new VoiceError("voice_config_invalid", 400, `answer has ${readout.segments.length} sentence(s)`);
+  }
+  return readout.parts[part].promise;
+}
+
+function startReadout(coordinator, id, model, options) {
   const lease = coordinator.beginSpeech(options.signal);
-  const result = readAloud(segments, model, lease, options);
-  readouts.set(coordinator, { text, active: lease.active, result });
-  return result;
+  let wake = () => {};
+  const readout = {
+    id,
+    active: lease.active,
+    signal: lease.signal,
+    segments: [],
+    parts: [],
+    final: false,
+    extend(segments, done) {
+      if (readout.final) return;
+      for (const sentence of segments.slice(readout.segments.length)) {
+        readout.segments.push(sentence);
+        readout.parts.push(deferred());
+      }
+      if (done) readout.final = true;
+      wake();
+    },
+    /** true once more text arrives or the readout is cancelled, false after `idleMs` of silence. */
+    more(idleMs) {
+      return new Promise((resolve) => {
+        const settle = (value) => {
+          clearTimeout(timer);
+          lease.signal.removeEventListener("abort", woken);
+          wake = () => {};
+          resolve(value);
+        };
+        const woken = () => settle(true);
+        const timer = setTimeout(() => settle(false), idleMs);
+        wake = woken;
+        lease.signal.addEventListener("abort", woken, { once: true });
+      });
+    },
+  };
+  readout.result = pinnedVoice(model, options.listVoices)
+    .then((voice) => readAloud(readout, model, voice, lease, options));
+  readout.result.catch(() => {});
+  readouts.set(coordinator, readout);
+  return readout;
 }
 
-async function readAloud(segments, model, lease, options) {
+async function readAloud(readout, model, voice, lease, options) {
   const openPlayback = options.openPlayback ?? openBluetoothPlayback;
   const synthesize = options.synthesize ?? synthesizeStream;
   const playFile = options.playFile ?? playBluetoothAudio;
   const toWav = options.pcmToWav ?? pcmToWav;
   const gap = options.silencePcm ?? silencePcm;
+  const idleMs = options.idleMs ?? STREAM_IDLE_MS;
   let opening;
   let player;
   const abortPlayer = () => player?.abort?.();
   lease.signal.addEventListener("abort", abortPlayer);
   const pcm = [];
+  let sentence = [];
   let streamed = false;
   let fileAudio;
   const release = () => {
     lease.signal.removeEventListener("abort", abortPlayer);
     lease.finish();
   };
+  const dropParts = (err) => {
+    for (const part of readout.parts) part.reject(err);
+  };
+  const headsetState = async () => {
+    const opened = opening ? await opening : null;
+    return opened && !opened.skipped ? "played" : "skipped";
+  };
   // The speaker drains in real time; synthesis must not wait on it, or the
   // browser copy arrives only after the headset has finished.
   let feeding = Promise.resolve();
   const writeChunk = (chunk) => {
     if (!chunk?.length) return;
     pcm.push(chunk);
+    sentence.push(chunk);
     if (!lease.active()) return;
     opening ??= openPlayback().catch(() => ({ skipped: true }));
     feeding = feeding.then(async () => {
@@ -213,29 +337,41 @@ async function readAloud(segments, model, lease, options) {
     });
   };
   try {
-    for (const segment of segments) {
-      if (!lease.active()) break;
-      const outcome = await synthesize({ text: segment.text, model, signal: lease.signal }, writeChunk);
+    for (let index = 0; lease.active();) {
+      if (index >= readout.segments.length) {
+        if (readout.final) break;
+        if (!(await readout.more(idleMs))) readout.final = true;
+        continue;
+      }
+      sentence = [];
+      if (index > 0) writeChunk(gap(gapAfter(readout.segments[index - 1])));
+      const outcome = await synthesize({ text: readout.segments[index], model, voice, signal: lease.signal }, writeChunk);
       if (!outcome.streamed) {
         fileAudio = outcome.audio;
+        readout.parts[0].resolve({ audio: fileAudio, bluetooth: "skipped" });
+        readout.final = true;
         break;
       }
       streamed = true;
-      if (segment.gapMs > 0 && lease.active()) writeChunk(gap(segment.gapMs));
+      readout.parts[index].resolve({ audio: toWav(Buffer.concat(sentence)), bluetooth: await headsetState() });
+      index += 1;
     }
   } catch (err) {
     void opening?.then((opened) => opened.abort?.());
     release();
-    if (lease.signal.aborted) throw cancelledSpeech();
-    throw err;
+    const failure = lease.signal.aborted ? cancelledSpeech() : err;
+    dropParts(failure);
+    throw failure;
   }
-  const audio = streamed ? toWav(Buffer.concat(pcm)) : fileAudio;
   const headset = opening ? await opening : null;
   if (!lease.active()) {
     headset?.abort?.();
     release();
+    dropParts(cancelledSpeech());
     throw cancelledSpeech();
   }
+  if (!streamed) dropParts(new VoiceError("voice_config_invalid", 400, "this engine returns the answer whole"));
+  const audio = streamed ? toWav(Buffer.concat(pcm)) : fileAudio;
   if (streamed && headset && !headset.skipped) {
     void feeding
       .then(() => (headset.skipped || !lease.active() ? headset.abort?.() : headset.close()))

package/voice/speech.js

@@ -1,9 +1,9 @@
 /**
- * The utterance a turn may speak: the text of the assistant message that
- * concludes it. The agent loop stops stepping once a settled message carries
- * no tool call, so that message is the answer even while turn-stopping work
- * still holds the turn open. Reasoning and text said alongside a tool call
- * are intermediate and stay silent.
+ * The utterance a turn may speak: the text of the step being written, read
+ * while it streams. Only a step that ends without a tool call answers the
+ * turn, and that is not known until the step settles, so a step turns silent
+ * the moment a tool call shows up in it, and an interrupted step is silent
+ * too. Reasoning is never read.
  */
 
 function turnOf(event) {
@@ -13,6 +13,12 @@ function turnOf(event) {
   return Number.isFinite(number) ? number : null;
 }
 
+function stepOf(event) {
+  const data = event?.data ?? {};
+  const number = Number(data.step ?? data.message?.step);
+  return Number.isFinite(number) ? number : 0;
+}
+
 function messageContent(event) {
   const data = event?.data ?? {};
   const content = data.message?.content ?? data.content;
@@ -28,66 +34,238 @@ export function assistantText(event) {
     .trim();
 }
 
-/** The answer text when this event concludes its turn, otherwise "". */
-export function concludingReply(event) {
-  if (event?.type !== "assistant/message" || event.data?.interrupted) return "";
-  if (messageContent(event).some((block) => block?.type === "tool-call")) return "";
-  return assistantText(event);
+function isChunkEvent(event) {
+  return event?.type === "assistant/chunk" || event?.type === "assistant/live-chunk";
+}
+
+function opensToolCall(chunk) {
+  return chunk?.type === "tool-call-delta"
+    || (chunk?.type === "block-start" && chunk.blockType === "tool-call")
+    || (chunk?.type === "block-end" && chunk.block?.type === "tool-call");
+}
+
+function stepUtterance(turn, step) {
+  return { key: `${turn}:${step}`, turn, step, blocks: {}, text: "", done: false, silent: false };
+}
+
+function withBlock(state, index, text) {
+  const blocks = { ...state.blocks, [index]: text };
+  const joined = Object.keys(blocks)
+    .map(Number)
+    .sort((a, b) => a - b)
+    .map((at) => blocks[at].trim())
+    .filter(Boolean)
+    .join("\n");
+  return { ...state, blocks, text: joined };
 }
 
 /**
- * Which concluded answers a live view reads aloud: only those it watched
- * arrive while their turn was running. Opening a session, or history that is
- * already finished, stays silent.
+ * @typedef {{ key: string, turn: number, step: number, text: string, done: boolean, silent: boolean }} Utterance
+ * @param {Utterance | null} state
+ * @returns {Utterance | null}
+ */
+export function reduceUtterance(state, event) {
+  const type = event?.type;
+  if (type === "turn/start") return null;
+  if (type === "turn/end") return state && !state.done ? { ...state, done: true } : state;
+  const turn = turnOf(event);
+  if (turn === null) return state;
+  const step = stepOf(event);
+  const same = state?.turn === turn && state.step === step;
+  if (isChunkEvent(event)) {
+    if (same && state.done) return state;
+    const chunk = event.data?.chunk;
+    const current = same ? state : stepUtterance(turn, step);
+    if (opensToolCall(chunk)) return current.silent ? current : { ...current, silent: true };
+    const index = Number.isInteger(chunk?.index) ? chunk.index : 0;
+    if (chunk?.type === "text-delta" && typeof chunk.text === "string") {
+      return withBlock(current, index, `${current.blocks[index] ?? ""}${chunk.text}`);
+    }
+    if (chunk?.type === "block-end" && chunk.block?.type === "text" && typeof chunk.block.text === "string") {
+      return withBlock(current, index, chunk.block.text);
+    }
+    return state;
+  }
+  if (type === "assistant/message") {
+    const content = messageContent(event);
+    return {
+      ...stepUtterance(turn, step),
+      text: assistantText(event),
+      done: true,
+      silent: Boolean(event.data?.interrupted) || content.some((block) => block?.type === "tool-call"),
+    };
+  }
+  return state;
+}
+
+export function matchSpeechEvent(event) {
+  const type = event?.type;
+  if (type !== "turn/start" && type !== "turn/end" && type !== "assistant/message" && !isChunkEvent(event)) return null;
+  const turn = turnOf(event);
+  if (turn === null) return null;
+  return { id: String(turn), role: type === "turn/start" ? "start" : "update" };
+}
+
+/**
+ * Which utterances a live view reads aloud: only those it watched being
+ * written while their turn was running. Opening a session mid-answer, or
+ * history that is already finished, stays silent.
  */
 export function createReplySpeaker() {
   let armed = false;
-  let spokenSeq = -1;
+  let skipKey = null;
   return {
-    /** @returns {string} text to speak now, or "" */
-    observe({ sessionChanged, wasRunning, running, reply }) {
+    /** @returns {Utterance | null} the utterance to follow now, or null */
+    observe({ sessionChanged, wasRunning, running, utterance }) {
       if (sessionChanged) {
         armed = Boolean(running);
-        spokenSeq = reply?.seq ?? -1;
-        return "";
+        skipKey = utterance?.key ?? null;
+        return null;
       }
       if (!wasRunning && running) {
         armed = true;
-        // A turn start resets the reply, so one already here was loaded with
-        // history, not watched.
-        if (reply) spokenSeq = Math.max(spokenSeq, reply.seq);
-      }
-      let text = "";
-      if (armed && reply && reply.seq > spokenSeq) {
-        spokenSeq = reply.seq;
-        text = reply.text;
+        skipKey = utterance?.key ?? null;
       }
+      const follow = armed && utterance && utterance.key !== skipKey ? utterance : null;
       if (!running) armed = false;
-      return text;
+      return follow;
     },
   };
 }
 
-export function emptySpeechState() {
-  return { text: "", seq: -1 };
+/**
+ * Which state of an utterance the reader gets. Streaming reads while the step
+ * is written; otherwise the step is read once it has settled.
+ * @param {Utterance | null} utterance
+ */
+export function speakable(utterance, streaming) {
+  if (!utterance) return null;
+  return streaming || utterance.done || utterance.silent ? utterance : null;
 }
 
+/** Text past this may have closed a sentence; the host decides whether it did. */
+const BOUNDARY = /[.!?;。！？；…\n]/;
+
 /**
- * A steer that lands after the answer runs another step in the same turn, so
- * a turn may conclude more than once; `seq` tells the answers apart.
- * @param {{ text: string, seq: number }} state
+ * Reads one utterance at a time, sentence by sentence, while it is still being
+ * written. The host owns sentence boundaries (sentencex does not run in a
+ * page): the text is handed over whenever it may have finished a sentence, and
+ * the host answers how many sentences are complete. That keeps a headset
+ * reading on without gaps; the page fetches a sentence's audio just before it
+ * needs it and plays it through `play`.
+ *
+ * `halt` stops the page's own audio. `silence` also stops the host, for a step
+ * that turned out to call a tool while the headset was reading it.
+ *
+ * @param {{
+ *   send: (request: { key: string, text: string, done: boolean, part?: number }) => Promise<any>,
+ *   play: (sentence: { audio: unknown, bluetooth?: string }) => Promise<unknown>,
+ *   halt: () => void,
+ *   silence: () => void,
+ *   fail?: (err: unknown) => void,
+ * }} io
  */
-export function reduceSpeechEvent(state, event) {
-  const current = state ?? emptySpeechState();
-  if (event?.type === "turn/start") return emptySpeechState();
-  const text = concludingReply(event);
-  return text ? { text, seq: Number(event.seq) } : current;
-}
+export function createSentencePlayer({ send, play, halt, silence, fail = () => {} }) {
+  let gen = 0;
+  let current = null;
+  // An utterance is read once: finishing it, failing, or being stopped all end it for good.
+  let lastKey = null;
+  let silencedKey = null;
 
-export function matchSpeechEvent(event) {
-  const type = event?.type;
-  if (type !== "turn/start" && type !== "assistant/message") return null;
-  const turn = turnOf(event);
-  if (turn === null) return null;
-  return { id: String(turn), role: type === "turn/start" ? "start" : "update" };
+  const failed = (run, err) => {
+    if (run.gen !== gen) return;
+    gen += 1;
+    run.wake?.();
+    current = null;
+    if (err?.code !== "voice_cancelled") fail(err);
+  };
+
+  const handOver = (run) => {
+    if (run.sending || run.gen !== gen) return;
+    const due = (run.done && !run.sentDone) || BOUNDARY.test(run.text.slice(run.sentLength));
+    if (!due) return;
+    const { text, done } = run;
+    run.sending = send({ key: run.key, text, done })
+      .then((result) => {
+        run.sending = null;
+        run.sent = Math.max(run.sent, Number(result?.sentences) || 0);
+        run.sentLength = text.length;
+        run.sentDone = done;
+        const wake = run.wake;
+        run.wake = null;
+        wake?.();
+        handOver(run);
+      })
+      .catch((err) => failed(run, err));
+  };
+
+  const fetchPart = (run, part) => {
+    const pending = send({ key: run.key, text: run.text, done: run.done, part });
+    pending.catch(() => {});
+    return pending;
+  };
+
+  const read = async (run) => {
+    let next = null;
+    for (let part = 0; run.gen === gen;) {
+      if (part >= run.sent) {
+        if (run.sentDone) break;
+        await new Promise((resolve) => { run.wake = resolve; });
+        continue;
+      }
+      const sentence = await (next ?? fetchPart(run, part));
+      next = part + 1 < run.sent ? fetchPart(run, part + 1) : null;
+      if (run.gen !== gen) return;
+      await play(sentence);
+      part += 1;
+    }
+    if (run.gen === gen) current = null;
+  };
+
+  const stop = () => {
+    gen += 1;
+    current?.wake?.();
+    if (current) halt();
+    current = null;
+  };
+
+  return {
+    /** Feed the latest state of the utterance; null leaves the reading as it is. */
+    follow(utterance) {
+      if (!utterance) return;
+      if (utterance.silent) {
+        if (utterance.key !== lastKey || silencedKey === lastKey) return;
+        silencedKey = lastKey;
+        stop();
+        silence();
+        return;
+      }
+      if (!utterance.text) return;
+      if (current?.key !== utterance.key) {
+        if (utterance.key === lastKey) return;
+        if (current) stop();
+        gen += 1;
+        lastKey = utterance.key;
+        current = {
+          gen,
+          key: utterance.key,
+          text: utterance.text,
+          done: utterance.done,
+          sent: 0,
+          sentLength: 0,
+          sentDone: false,
+          sending: null,
+          wake: null,
+        };
+        const run = current;
+        handOver(run);
+        read(run).catch((err) => failed(run, err));
+        return;
+      }
+      current.text = utterance.text;
+      current.done = utterance.done;
+      handOver(current);
+    },
+    stop,
+  };
 }