@olares/lares-core 0.15.11 → 0.15.13

package/voice/service.js

@@ -1,3 +1,6 @@
+import { writeReport } from "../log/report.js";
+import { createPcmResampler, pcmSamples } from "./realtime-audio.js";
+import { realtimeModelIds, validatedRealtimeModels } from "./realtime.js";
 import { readConfig, validateConfigPatch, writeConfig } from "./config.js";
 import {
   chartModelId,
@@ -18,6 +21,7 @@ import {
 } from "./session.js";
 import { createCaptureSession } from "./capture.js";
 import { createDuplexCoordinator } from "./duplex.js";
+import { createDeviceLease } from "./lease.js";
 import { normalizeSttPcm } from "./pcm.js";
 import { gapAfter, speechSegments } from "./script.js";
 import {
@@ -43,6 +47,57 @@ import {
 export { STT_MAX_AUDIO_BYTES };
 
 const duplex = createDuplexCoordinator(stopBluetoothPlayback);
+const deviceLease = createDeviceLease();
+const scopes = new Map();
+
+export function voiceSessionPrompt(sessionId) {
+  return [...scopes.values()].some((scope) => scope.session?.snapshot().sessionId === sessionId && scope.active && Date.now() - scope.seen < 30_000);
+}
+
+export function voiceScope(id, owner = "", device = "headset") {
+  if (!/^[a-zA-Z0-9_-]{8,128}$/.test(id)) throw new VoiceError("voice_session_invalid", 400, "invalid voice connection");
+  const key = `${owner}:${id}`;
+  for (const [other, scope] of scopes) {
+    if (Date.now() - scope.seen > 120_000 && !deviceLease.owns(scope)) {
+      scopes.delete(other);
+      void scope.duplex.interrupt().catch(() => {});
+      scope.capture?.stop(true);
+      void scope.session?.close();
+    }
+  }
+  let scope = scopes.get(key);
+  if (!scope) {
+    if (scopes.size >= 128) throw new VoiceError("voice_busy", 429, "too many voice connections");
+    scope = { id, owner, device, seen: Date.now(), capture: null, duplex: null };
+    scope.duplex = createDuplexCoordinator(async () => {
+      if (deviceLease.owns(scope)) {
+        await stopBluetoothPlayback();
+        deviceLease.release(scope);
+      }
+    });
+    scopes.set(key, scope);
+  }
+  if (scope.device !== device) {
+    if (deviceLease.owns(scope)) throw new VoiceError("voice_device_busy", 409, "stop the current audio device first");
+    scope.device = device;
+  }
+  scope.seen = Date.now();
+  return scope;
+}
+
+function scopedPlayback(scope) {
+  if (scope.device === "device") return Promise.resolve({ skipped: true });
+  deviceLease.acquire(scope);
+  return openBluetoothPlayback({ waitForDrain: true }).then((player) => {
+    if (player.skipped) deviceLease.release(scope);
+    return {
+      ...player,
+      async write(chunk) { deviceLease.touch(scope); return player.write(chunk); },
+      async close() { try { return await player.close?.(); } finally { deviceLease.release(scope); } },
+      abort() { try { player.abort?.(); } finally { deviceLease.release(scope); } },
+    };
+  }, (error) => { deviceLease.release(scope); throw error; });
+}
 
 export async function voiceStatusPayload(options) {
   const config = readConfig();
@@ -79,6 +134,9 @@ export async function voiceModelsPayload(options) {
     models: ids,
     stt,
     tts,
+    chat: models.filter((model) => model.mode === "chat").map((model) => model.id),
+    realtime: realtimeModelIds(models),
+    realtimeReady: validatedRealtimeModels(models),
     selected: pickSttModelId(models, config.model),
     ttsSelected: pickTtsModelId(models, config.ttsModel),
     charts,
@@ -87,8 +145,10 @@ export async function voiceModelsPayload(options) {
 
 export async function saveVoiceConfig(body) {
   const patch = validateConfigPatch(body);
-  if (patch.model || patch.ttsModel) {
+  if (patch.model || patch.ttsModel || patch.realtimeModel || patch.backend === "realtime") {
     const models = await listModels();
+    if (patch.realtimeModel && !realtimeModelIds(models).includes(patch.realtimeModel)) throw new VoiceError("voice_model_unavailable", 400, "Realtime model is not available");
+    if (patch.backend === "realtime" && !validatedRealtimeModels(models).includes(patch.realtimeModel || readConfig().realtimeModel)) throw new VoiceError("voice_realtime_unavailable", 409, "Realtime model has not been validated");
     if (patch.model && !sttModelIds(models).includes(patch.model)) {
       throw new VoiceError("voice_model_unavailable", 400, "selected voice model is not available");
     }
@@ -228,7 +288,7 @@ export async function speakFromConfig(text, options = {}) {
   const done = key ? options.done === true : true;
   const segments = speechSegments(text, { done });
   const part = partIndex(options.part);
-  const coordinator = options.coordinator ?? duplex;
+  const coordinator = options.coordinator ?? options.scope?.duplex ?? duplex;
   const id = key ?? text;
   let readout = joinable(readouts.get(coordinator), id, key, part);
   if (!readout) {
@@ -245,6 +305,8 @@ export async function speakFromConfig(text, options = {}) {
   if (part >= readout.segments.length) {
     throw new VoiceError("voice_config_invalid", 400, `answer has ${readout.segments.length} sentence(s)`);
   }
+  readout.requestedPart = Math.max(readout.requestedPart, part);
+  readout.extend([], false);
   return readout.parts[part].promise;
 }
 
@@ -253,13 +315,16 @@ function startReadout(coordinator, id, model, options) {
   let wake = () => {};
   const readout = {
     id,
+    createdAt: Date.now(),
+    firstAudio: false,
     active: lease.active,
     signal: lease.signal,
     segments: [],
     parts: [],
     final: false,
+    requestedPart: 0,
     extend(segments, done) {
-      if (readout.final) return;
+      if (readout.final) { wake(); return; }
       for (const sentence of segments.slice(readout.segments.length)) {
         readout.segments.push(sentence);
         readout.parts.push(deferred());
@@ -283,7 +348,7 @@ function startReadout(coordinator, id, model, options) {
       });
     },
   };
-  readout.result = pinnedVoice(model, options.listVoices)
+  readout.result = Promise.resolve(options.config?.voice || pinnedVoice(model, options.listVoices))
     .then((voice) => readAloud(readout, model, voice, lease, options));
   readout.result.catch(() => {});
   readouts.set(coordinator, readout);
@@ -291,9 +356,13 @@ function startReadout(coordinator, id, model, options) {
 }
 
 async function readAloud(readout, model, voice, lease, options) {
-  const openPlayback = options.openPlayback ?? openBluetoothPlayback;
+  const openPlayback = options.openPlayback ?? (options.scope ? () => scopedPlayback(options.scope) : openBluetoothPlayback);
   const synthesize = options.synthesize ?? synthesizeStream;
-  const playFile = options.playFile ?? playBluetoothAudio;
+  const playFile = options.playFile ?? (options.scope ? async (audio) => {
+    if (options.scope.device === "device") return { played: false };
+    deviceLease.acquire(options.scope);
+    try { return await playBluetoothAudio(audio); } finally { deviceLease.release(options.scope); }
+  } : playBluetoothAudio);
   const toWav = options.pcmToWav ?? pcmToWav;
   const gap = options.silencePcm ?? silencePcm;
   const idleMs = options.idleMs ?? STREAM_IDLE_MS;
@@ -321,10 +390,15 @@ async function readAloud(readout, model, voice, lease, options) {
   let feeding = Promise.resolve();
   const writeChunk = (chunk) => {
     if (!chunk?.length) return;
+    if (!readout.firstAudio) {
+      readout.firstAudio = true;
+      if (options.scope) writeReport({ scope: "voice", event: "metric", voiceId: options.scope.id, stage: "synthesis_first_audio", durationMs: Date.now() - readout.createdAt, queueDepth: readout.segments.length });
+    }
     pcm.push(chunk);
     sentence.push(chunk);
     if (!lease.active()) return;
-    opening ??= openPlayback().catch(() => ({ skipped: true }));
+    opening ??= openPlayback();
+    opening.catch(() => {});
     feeding = feeding.then(async () => {
       player ??= await opening;
       if (player.skipped || !lease.active()) return;
@@ -335,6 +409,7 @@ async function readAloud(readout, model, voice, lease, options) {
         player.abort?.();
       }
     });
+    if (options.scope?.device === "headset") return feeding;
   };
   try {
     for (let index = 0; lease.active();) {
@@ -343,8 +418,12 @@ async function readAloud(readout, model, voice, lease, options) {
         if (!(await readout.more(idleMs))) readout.final = true;
         continue;
       }
+      if (options.key && index > readout.requestedPart + 1) {
+        if (!(await readout.more(idleMs))) throw new VoiceError("voice_playback_timeout", 408, "playback consumer stopped");
+        continue;
+      }
       sentence = [];
-      if (index > 0) writeChunk(gap(gapAfter(readout.segments[index - 1])));
+      if (index > 0) await writeChunk(gap(gapAfter(readout.segments[index - 1])));
       const outcome = await synthesize({ text: readout.segments[index], model, voice, signal: lease.signal }, writeChunk);
       if (!outcome.streamed) {
         fileAudio = outcome.audio;
@@ -392,8 +471,10 @@ async function readAloud(readout, model, voice, lease, options) {
   return { bluetooth, audio };
 }
 
-export function interruptSpeech() {
-  return duplex.interrupt();
+export function playbackStatus(scope) { return { playing: (scope?.duplex ?? duplex).isSpeaking() }; }
+
+export function interruptSpeech(scope) {
+  return (scope?.duplex ?? duplex).interrupt();
 }
 
 export function languageForTranscribe(requestedLanguage, configLanguage) {
@@ -401,47 +482,45 @@ export function languageForTranscribe(requestedLanguage, configLanguage) {
   return validateConfigPatch({ language: requestedLanguage }).language || undefined;
 }
 
-export async function transcribeFromHttp({ languageParam, contentType, bytes }) {
+export async function transcribeFromHttp({ languageParam, contentType, bytes, signal }) {
   const config = readConfig();
   const text = await transcribeAudio({
     bytes,
     contentType,
     language: languageForTranscribe(languageParam, config.language),
     preferred: config.model,
+    signal,
   });
   return { text };
 }
 
-const hostCapture = createCaptureSession({
-  open: beginHostCapture,
-  read: (from) => readHostCapture(from),
-  close: endHostCapture,
-  reset: resetHostCapture,
-  transcribe(wav, signal) {
-    const config = readConfig();
-    return transcribeAudio({
-      bytes: wav,
-      contentType: "audio/wav",
-      language: languageForTranscribe(null, config.language),
-      preferred: config.model,
-      signal,
-    });
-  },
-});
-
-export async function captureStart() {
-  await interruptSpeech();
-  return hostCapture.start();
+function createHostCapture(scope) {
+  return createCaptureSession({
+    async open() {
+      deviceLease.acquire(scope);
+      try { const opened = await beginHostCapture(); if (!opened.ready) deviceLease.release(scope); return opened; } catch (error) { deviceLease.release(scope); throw error; }
+    },
+    read(from) { if (!deviceLease.owns(scope)) throw new VoiceError("voice_device_busy", 409, "audio lease expired"); deviceLease.touch(scope); return readHostCapture(from); },
+    async close() { if (!deviceLease.owns(scope)) return; try { await endHostCapture(); } finally { deviceLease.release(scope); } },
+    reset: resetHostCapture,
+    transcribe(wav, signal) {
+      const config = readConfig();
+      return transcribeAudio({ bytes: wav, contentType: "audio/wav", language: languageForTranscribe(null, config.language), preferred: config.model, signal });
+    },
+  });
 }
-
-export function captureStop(cancel) {
-  hostCapture.stop(cancel);
-  return hostCapture.status();
+const defaultScope = { duplex };
+function captureFor(scope = defaultScope) { return scope.capture ??= createHostCapture(scope); }
+export async function captureStart(scope = defaultScope) {
+  deviceLease.acquire(scope);
+  await interruptSpeech(scope);
+  return captureFor(scope).start();
 }
-
-export function captureStatus() {
-  return hostCapture.status();
+export function captureStop(cancel, scope = defaultScope) {
+  captureFor(scope).stop(cancel);
+  return captureFor(scope).status();
 }
+export function captureStatus(scope = defaultScope) { return captureFor(scope).status(); }
 
 export async function transcribeAudio({ bytes, contentType, language, preferred, signal }) {
   const model = await resolveSttModel(preferred);
@@ -450,6 +529,7 @@ export async function transcribeAudio({ bytes, contentType, language, preferred,
   }
   if (bytes.length === 0) throw new VoiceError("voice_audio_unreadable", 400, "audio content is empty");
   const pcm = await normalizeSttPcm(bytes, contentType);
+  signal?.throwIfAborted();
   return transcribe({
     bytes: pcm,
     filename: "voice.wav",
@@ -460,3 +540,70 @@ export async function transcribeAudio({ bytes, contentType, language, preferred,
     signal,
   });
 }
+
+
+/** Host input and output share one physical lease; half-duplex is intentional. */
+export async function openRealtimeHostAudio(scope, send) {
+  if (scope.device !== "headset") return null;
+  deviceLease.acquire(scope);
+  let closed = false;
+  let capturing = false;
+  let inputPaused = true;
+  let offset = 0;
+  let player = null;
+  let timer = null;
+  let generation = 0;
+  const convert = createPcmResampler(16000);
+  const start = async () => {
+    if (closed || capturing || inputPaused) return;
+    const opened = await beginHostCapture();
+    if (!opened.ready) throw new VoiceError("voice_bluetooth_not_ready", 503, "headset is not ready");
+    capturing = true; offset = opened.offset;
+  };
+  const read = async () => {
+    if (closed) return;
+    if (!deviceLease.owns(scope)) throw new VoiceError("voice_device_busy", 409, "audio lease expired");
+    deviceLease.touch(scope);
+    if (capturing) {
+      const bytes = await readHostCapture(offset, { maxBytes: 32_000 });
+      offset += bytes.length;
+      if (capturing && bytes.length) send({ type: "input_audio_buffer.append", audio: Buffer.from(convert(pcmSamples(bytes))).toString("base64") });
+    }
+    if (!closed) timer = setTimeout(() => { void read().catch(() => scope.realtimeSocket?.close()); }, 100);
+  };
+  try { await start(); void read(); }
+  catch (error) { deviceLease.release(scope); throw error; }
+  return {
+    async pauseInput() { inputPaused = true; if (capturing) { capturing = false; await endHostCapture(); } },
+    async resumeInput() { inputPaused = false; if (!player) await start(); },
+    async write(bytes) {
+      if (closed) return;
+      if (capturing) { capturing = false; await endHostCapture(); }
+      if (!player) {
+        player = await openBluetoothPlayback({ waitForDrain: true });
+        if (player.skipped) throw new VoiceError("voice_device_unavailable", 503, "headset speaker is unavailable");
+      }
+      deviceLease.touch(scope);
+      await player.write(bytes);
+    },
+    async finish() {
+      const epoch = generation;
+      const current = player; player = null;
+      await current?.close();
+      if (!closed && epoch === generation) await start();
+    },
+    async interrupt() {
+      generation++;
+      player?.abort(); player = null;
+      await stopBluetoothPlayback();
+      if (!closed) await start();
+    },
+    async stop() {
+      if (closed) return;
+      closed = true; generation++; clearTimeout(timer);
+      player?.abort(); player = null;
+      try { if (deviceLease.owns(scope)) { await endHostCapture(); await stopBluetoothPlayback(); } }
+      finally { deviceLease.release(scope); }
+    },
+  };
+}

package/voice/session.js

@@ -65,6 +65,16 @@ export async function openBluetoothPlayback(options = {}) {
         });
         stream.end();
       });
+      if (options.waitForDrain) {
+        const until = Date.now() + 120_000;
+        while (true) {
+          const state = await askAgent("playback-status", {}, options);
+          if (state.error) throw new VoiceError("voice_host_upgrade_required", 409, "update the Lares host audio helper");
+          if (!state.playing) break;
+          if (Date.now() >= until) throw new VoiceError("voice_playback_timeout", 408, "host playback did not finish");
+          await new Promise((resolve) => setTimeout(resolve, 100));
+        }
+      }
     },
     abort() {
       stream.destroy();
@@ -102,7 +112,7 @@ export async function readHostCapture(from = 0, options = {}) {
   try {
     handle = await open(`${voiceDir(options)}/${CAPTURE_FILE}`, "r");
     const { size } = await handle.stat();
-    const length = size - start;
+    const length = Math.min(size - start, options.maxBytes ?? 64 * 1024);
     if (length <= 0) return Buffer.alloc(0);
     const buf = Buffer.alloc(length);
     const { bytesRead } = await handle.read(buf, 0, length, start);

package/voice/speech.js

@@ -165,7 +165,7 @@ const BOUNDARY = /[.!?;。！？；…\n]/;
  *   fail?: (err: unknown) => void,
  * }} io
  */
-export function createSentencePlayer({ send, play, halt, silence, fail = () => {} }) {
+export function createSentencePlayer({ send, play, halt, silence, fail = () => {}, onState = () => {} }) {
   let gen = 0;
   let current = null;
   // An utterance is read once: finishing it, failing, or being stopped all end it for good.
@@ -177,6 +177,7 @@ export function createSentencePlayer({ send, play, halt, silence, fail = () => {
     gen += 1;
     run.wake?.();
     current = null;
+    onState(false);
     if (err?.code !== "voice_cancelled") fail(err);
   };
 
@@ -219,7 +220,7 @@ export function createSentencePlayer({ send, play, halt, silence, fail = () => {
       await play(sentence);
       part += 1;
     }
-    if (run.gen === gen) current = null;
+    if (run.gen === gen) { current = null; onState(false); }
   };
 
   const stop = () => {
@@ -227,6 +228,7 @@ export function createSentencePlayer({ send, play, halt, silence, fail = () => {
     current?.wake?.();
     if (current) halt();
     current = null;
+    onState(false);
   };
 
   return {
@@ -258,6 +260,7 @@ export function createSentencePlayer({ send, play, halt, silence, fail = () => {
           wake: null,
         };
         const run = current;
+        onState(true);
         handOver(run);
         read(run).catch((err) => failed(run, err));
         return;
@@ -266,6 +269,9 @@ export function createSentencePlayer({ send, play, halt, silence, fail = () => {
       current.done = utterance.done;
       handOver(current);
     },
-    stop,
+    stop() {
+      stop();
+      silence();
+    },
   };
 }