@olares/lares-core 0.15.7 → 0.15.10

package/voice/service.js

@@ -1,17 +1,42 @@
 import { readConfig, validateConfigPatch, writeConfig } from "./config.js";
+import {
+  chartModelId,
+  installVoiceChart,
+  resumeVoiceChart,
+  stopVoiceChart,
+  voiceChartsStatus,
+  voiceChartStatus,
+} from "./install.js";
+import {
+  beginHostCapture,
+  endHostCapture,
+  openBluetoothPlayback,
+  playBluetoothAudio,
+  readHostCapture,
+  resetHostCapture,
+  stopBluetoothPlayback,
+} from "./session.js";
+import { createCaptureSession } from "./capture.js";
+import { createDuplexCoordinator } from "./duplex.js";
+import { normalizeSttPcm } from "./pcm.js";
+import { speechSegments } from "./script.js";
+import { pcmToWav, silencePcm, synthesize, synthesizeStream } from "../router/tts.js";
 import {
   STT_MAX_AUDIO_BYTES,
   VoiceError,
-  audioFilenameForContentType,
   listModels,
   pickSttModelId,
+  pickTtsModelId,
   resolveSttModel,
   sttModelIds,
   transcribe,
+  ttsModelIds,
 } from "../router/stt.js";
 
 export { STT_MAX_AUDIO_BYTES };
 
+const duplex = createDuplexCoordinator(stopBluetoothPlayback);
+
 export async function voiceStatusPayload(options) {
   const config = readConfig();
   let modelAvailable = false;
@@ -33,23 +58,208 @@ export async function voiceStatusPayload(options) {
 
 export async function voiceModelsPayload(options) {
   const models = await listModels(options);
+  const config = readConfig();
   const ids = models.map((model) => model.id);
   const stt = sttModelIds(models);
-  const selected = pickSttModelId(models, readConfig().model);
-  return { models: ids, stt, selected };
+  const tts = ttsModelIds(models);
+  let charts = null;
+  try {
+    charts = await voiceChartsStatus();
+  } catch {
+    charts = null;
+  }
+  return {
+    models: ids,
+    stt,
+    tts,
+    selected: pickSttModelId(models, config.model),
+    ttsSelected: pickTtsModelId(models, config.ttsModel),
+    charts,
+  };
 }
 
 export async function saveVoiceConfig(body) {
   const patch = validateConfigPatch(body);
-  if (patch.model) {
-    const available = sttModelIds(await listModels());
-    if (!available.includes(patch.model)) {
+  if (patch.model || patch.ttsModel) {
+    const models = await listModels();
+    if (patch.model && !sttModelIds(models).includes(patch.model)) {
       throw new VoiceError("voice_model_unavailable", 400, "selected voice model is not available");
     }
+    if (patch.ttsModel && !ttsModelIds(models).includes(patch.ttsModel)) {
+      throw new VoiceError("voice_model_unavailable", 400, "selected speech model is not available");
+    }
   }
   return writeConfig(patch);
 }
 
+export function installVoice(kind) {
+  return installVoiceChart(kind);
+}
+
+export function stopVoice(kind) {
+  return stopVoiceChart(kind);
+}
+
+export function resumeVoice(kind) {
+  return resumeVoiceChart(kind);
+}
+
+const PROBE_TIMEOUT_MS = 45_000;
+
+function probeWav() {
+  return pcmToWav(Buffer.alloc(6400), 16000);
+}
+
+/**
+ * Ask the chart's model through Router. An engine that answers — even to
+ * reject silence — is connected. A stopped chart is not.
+ * @param {"stt" | "tts"} kind
+ */
+export async function probeVoice(kind, options = {}) {
+  const status = await (options.status ?? voiceChartStatus)(kind);
+  if (status.phase !== "running") {
+    throw new VoiceError("voice_probe_failed", 409, "voice app is not running");
+  }
+  const models = await (options.listModels ?? listModels)({ refresh: true });
+  const ids = kind === "tts" ? ttsModelIds(models) : sttModelIds(models);
+  const model = chartModelId(kind, ids);
+  if (!model) throw new VoiceError("voice_probe_failed", 503, "voice model is not registered");
+  if (kind === "tts") {
+    const speak = options.synthesize ?? synthesize;
+    const audio = await speak({ text: "你好", model, timeoutMs: PROBE_TIMEOUT_MS });
+    if (!audio?.length) throw new VoiceError("voice_probe_failed", 502, "speech returned no audio");
+    return { ok: true, app: status.app, model };
+  }
+  const hear = options.transcribe ?? transcribe;
+  try {
+    await hear({
+      bytes: probeWav(),
+      filename: "probe.wav",
+      contentType: "audio/wav",
+      model,
+      once: true,
+      timeoutMs: PROBE_TIMEOUT_MS,
+    });
+  } catch (err) {
+    if (!(err instanceof VoiceError) || err.code !== "voice_audio_unreadable") {
+      const statusCode = err && typeof err === "object" && "status" in err ? Number(err.status) : 502;
+      throw new VoiceError("voice_probe_failed", statusCode || 502, err instanceof Error ? err.message : "probe failed");
+    }
+  }
+  return { ok: true, app: status.app, model };
+}
+
+function cancelledSpeech() {
+  return new VoiceError("voice_cancelled", 499, "speech was superseded");
+}
+
+const readouts = new WeakMap();
+
+/**
+ * Every open page asks to read the same answer when it concludes. A request
+ * for the readout already under way joins it instead of restarting the
+ * headset and cancelling the page that asked first.
+ */
+export async function speakFromConfig(text, options = {}) {
+  const config = options.config ?? readConfig();
+  if (!config.speak) throw new VoiceError("voice_speak_disabled", 409, "speech playback is off");
+  const segments = speechSegments(text);
+  if (!segments.length) throw new VoiceError("voice_config_invalid", 400, "nothing to speak");
+  const model = options.model ?? await pickTtsModelId(await listModels(), config.ttsModel);
+  if (!model) throw new VoiceError("voice_tts_unavailable", 503, "no speech model available");
+  const coordinator = options.coordinator ?? duplex;
+  const current = readouts.get(coordinator);
+  if (current?.text === text && current.active()) return current.result;
+  const lease = coordinator.beginSpeech(options.signal);
+  const result = readAloud(segments, model, lease, options);
+  readouts.set(coordinator, { text, active: lease.active, result });
+  return result;
+}
+
+async function readAloud(segments, model, lease, options) {
+  const openPlayback = options.openPlayback ?? openBluetoothPlayback;
+  const synthesize = options.synthesize ?? synthesizeStream;
+  const playFile = options.playFile ?? playBluetoothAudio;
+  const toWav = options.pcmToWav ?? pcmToWav;
+  const gap = options.silencePcm ?? silencePcm;
+  let opening;
+  let player;
+  const abortPlayer = () => player?.abort?.();
+  lease.signal.addEventListener("abort", abortPlayer);
+  const pcm = [];
+  let streamed = false;
+  let fileAudio;
+  const release = () => {
+    lease.signal.removeEventListener("abort", abortPlayer);
+    lease.finish();
+  };
+  // The speaker drains in real time; synthesis must not wait on it, or the
+  // browser copy arrives only after the headset has finished.
+  let feeding = Promise.resolve();
+  const writeChunk = (chunk) => {
+    if (!chunk?.length) return;
+    pcm.push(chunk);
+    if (!lease.active()) return;
+    opening ??= openPlayback().catch(() => ({ skipped: true }));
+    feeding = feeding.then(async () => {
+      player ??= await opening;
+      if (player.skipped || !lease.active()) return;
+      try {
+        await player.write(chunk);
+      } catch {
+        player.skipped = true;
+        player.abort?.();
+      }
+    });
+  };
+  try {
+    for (const segment of segments) {
+      if (!lease.active()) break;
+      const outcome = await synthesize({ text: segment.text, model, signal: lease.signal }, writeChunk);
+      if (!outcome.streamed) {
+        fileAudio = outcome.audio;
+        break;
+      }
+      streamed = true;
+      if (segment.gapMs > 0 && lease.active()) writeChunk(gap(segment.gapMs));
+    }
+  } catch (err) {
+    void opening?.then((opened) => opened.abort?.());
+    release();
+    if (lease.signal.aborted) throw cancelledSpeech();
+    throw err;
+  }
+  const audio = streamed ? toWav(Buffer.concat(pcm)) : fileAudio;
+  const headset = opening ? await opening : null;
+  if (!lease.active()) {
+    headset?.abort?.();
+    release();
+    throw cancelledSpeech();
+  }
+  if (streamed && headset && !headset.skipped) {
+    void feeding
+      .then(() => (headset.skipped || !lease.active() ? headset.abort?.() : headset.close()))
+      .catch(() => headset.abort?.())
+      .finally(release);
+    return { bluetooth: "played", audio };
+  }
+  headset?.abort?.();
+  let bluetooth = "skipped";
+  if (!streamed && fileAudio) {
+    try {
+      if ((await playFile(fileAudio)).played) bluetooth = "played";
+    } catch {
+      // Browser playback remains available.
+    }
+  }
+  release();
+  return { bluetooth, audio };
+}
+
+export function interruptSpeech() {
+  return duplex.interrupt();
+}
+
 export function languageForTranscribe(requestedLanguage, configLanguage) {
   if (requestedLanguage === null) return configLanguage || undefined;
   return validateConfigPatch({ language: requestedLanguage }).language || undefined;
@@ -66,18 +276,51 @@ export async function transcribeFromHttp({ languageParam, contentType, bytes })
   return { text };
 }
 
-export async function transcribeAudio({ bytes, contentType, language, preferred }) {
+const hostCapture = createCaptureSession({
+  open: beginHostCapture,
+  read: (from) => readHostCapture(from),
+  close: endHostCapture,
+  reset: resetHostCapture,
+  transcribe(wav, signal) {
+    const config = readConfig();
+    return transcribeAudio({
+      bytes: wav,
+      contentType: "audio/wav",
+      language: languageForTranscribe(null, config.language),
+      preferred: config.model,
+      signal,
+    });
+  },
+});
+
+export async function captureStart() {
+  await interruptSpeech();
+  return hostCapture.start();
+}
+
+export function captureStop(cancel) {
+  hostCapture.stop(cancel);
+  return hostCapture.status();
+}
+
+export function captureStatus() {
+  return hostCapture.status();
+}
+
+export async function transcribeAudio({ bytes, contentType, language, preferred, signal }) {
   const model = await resolveSttModel(preferred);
   if (!model) {
     throw new VoiceError("voice_model_unavailable", 503, "no voice model available in the Router catalog");
   }
   if (bytes.length === 0) throw new VoiceError("voice_audio_unreadable", 400, "audio content is empty");
+  const pcm = await normalizeSttPcm(bytes, contentType);
   return transcribe({
-    bytes,
-    filename: audioFilenameForContentType(contentType),
-    contentType,
+    bytes: pcm,
+    filename: "voice.wav",
+    contentType: "audio/wav",
     model,
     language,
     preferred,
+    signal,
   });
 }

package/voice/session.js

@@ -0,0 +1,128 @@
+import { randomUUID } from "node:crypto";
+import { once } from "node:events";
+import { createWriteStream } from "node:fs";
+import { mkdir, open, writeFile } from "node:fs/promises";
+import { VoiceError } from "../router/stt.js";
+import { askAgent, liveAgent, voiceDir } from "./agent.js";
+
+const CAPTURE_FILE = "capture.s16";
+
+/**
+ * Play speech on the host that owns the headset.
+ * No agent means this machine has no speaker to drive.
+ */
+export async function playBluetoothAudio(bytes, options = {}) {
+  if (!await liveAgent(options)) return { played: false, skipped: true };
+  const dir = voiceDir(options);
+  const id = options.id ?? randomUUID();
+  const file = `speech-${id}.wav`;
+  await mkdir(dir, { recursive: true });
+  await writeFile(`${dir}/${file}`, bytes);
+  const result = await askAgent("play", { file }, { ...options, id, timeoutMs: options.timeoutMs ?? 20_000 });
+  if (result.skipped) return { played: false, skipped: true };
+  if (result.playing) return { played: true, sink: result.sink };
+  throw new VoiceError("voice_failed", 502, result.error || "speaker did not play");
+}
+
+/**
+ * Open the host speaker so PCM can be written while later sentences are still synthesizing.
+ * No agent means there is no headset speaker.
+ */
+export async function openBluetoothPlayback(options = {}) {
+  if (!await liveAgent(options)) return { skipped: true };
+  const dir = voiceDir(options);
+  const id = options.id ?? randomUUID();
+  const file = `speech-${id}.pcm`;
+  const result = await askAgent("play-stream", { file, rate: 24000, channels: 1 }, {
+    ...options,
+    id,
+    timeoutMs: options.timeoutMs ?? 20_000,
+  });
+  if (result.skipped) return { skipped: true };
+  if (!result.playing) {
+    throw new VoiceError("voice_failed", 502, result.error || "speaker did not play");
+  }
+  if (options.writer) return { skipped: false, sink: result.sink, ...options.writer };
+  const stream = createWriteStream(`${dir}/${file}`);
+  const ready = new Promise((resolve, reject) => {
+    stream.once("open", resolve);
+    stream.once("error", reject);
+  });
+  return {
+    skipped: false,
+    sink: result.sink,
+    async write(chunk) {
+      await ready;
+      if (!stream.write(chunk)) await once(stream, "drain");
+    },
+    async close() {
+      await ready;
+      await new Promise((resolve, reject) => {
+        stream.once("finish", resolve);
+        stream.once("error", reject);
+        stream.once("close", () => {
+          if (!stream.writableFinished) reject(new Error("speaker stream closed"));
+        });
+        stream.end();
+      });
+    },
+    abort() {
+      stream.destroy();
+    },
+  };
+}
+
+/** Stop a readout without changing the headset microphone profile. */
+export async function stopBluetoothPlayback(options = {}) {
+  if (!await liveAgent(options)) return { stopped: false, skipped: true };
+  const result = await askAgent("playback-stop", {}, options);
+  return { stopped: result.stopped === true };
+}
+
+/** Start the headset recording on the machine. No agent means there is no headset mic. */
+export async function beginHostCapture(options = {}) {
+  if (!await liveAgent(options)) return { skipped: true };
+  const result = await askAgent("capture-start", {}, options);
+  if (result.skipped) return { skipped: true };
+  if (!result.ready) {
+    throw new VoiceError("voice_bluetooth_not_ready", 503, result.error || "bluetooth headset is not capturing");
+  }
+  return {
+    ready: true,
+    file: result.file || CAPTURE_FILE,
+    offset: Number(result.offset) || 0,
+    settle: result.settle === true,
+  };
+}
+
+/** Bytes written after `from`. The call link stays open, so the file keeps growing. */
+export async function readHostCapture(from = 0, options = {}) {
+  const start = Number.isFinite(from) ? Math.max(0, Math.trunc(from)) : 0;
+  let handle;
+  try {
+    handle = await open(`${voiceDir(options)}/${CAPTURE_FILE}`, "r");
+    const { size } = await handle.stat();
+    const length = size - start;
+    if (length <= 0) return Buffer.alloc(0);
+    const buf = Buffer.alloc(length);
+    const { bytesRead } = await handle.read(buf, 0, length, start);
+    return bytesRead === buf.length ? buf : buf.subarray(0, bytesRead);
+  } catch {
+    return Buffer.alloc(0);
+  } finally {
+    await handle?.close();
+  }
+}
+
+export async function endHostCapture(options = {}) {
+  if (!await liveAgent(options)) return { ready: true, skipped: true };
+  await askAgent("capture-stop", {}, options);
+  return { ready: true };
+}
+
+/** Recreate a capture transport that produced no microphone samples. */
+export async function resetHostCapture(options = {}) {
+  if (!await liveAgent(options)) return { ready: true, skipped: true };
+  await askAgent("capture-reset", {}, options);
+  return { ready: true };
+}

package/voice/speech.js

@@ -0,0 +1,93 @@
+/**
+ * The utterance a turn may speak: the text of the assistant message that
+ * concludes it. The agent loop stops stepping once a settled message carries
+ * no tool call, so that message is the answer even while turn-stopping work
+ * still holds the turn open. Reasoning and text said alongside a tool call
+ * are intermediate and stay silent.
+ */
+
+function turnOf(event) {
+  const data = event?.data ?? {};
+  const turn = data.turn ?? data.message?.turn;
+  const number = Number(turn);
+  return Number.isFinite(number) ? number : null;
+}
+
+function messageContent(event) {
+  const data = event?.data ?? {};
+  const content = data.message?.content ?? data.content;
+  return Array.isArray(content) ? content : [];
+}
+
+export function assistantText(event) {
+  return messageContent(event)
+    .filter((block) => block?.type === "text" && typeof block.text === "string")
+    .map((block) => block.text.trim())
+    .filter(Boolean)
+    .join("\n")
+    .trim();
+}
+
+/** The answer text when this event concludes its turn, otherwise "". */
+export function concludingReply(event) {
+  if (event?.type !== "assistant/message" || event.data?.interrupted) return "";
+  if (messageContent(event).some((block) => block?.type === "tool-call")) return "";
+  return assistantText(event);
+}
+
+/**
+ * Which concluded answers a live view reads aloud: only those it watched
+ * arrive while their turn was running. Opening a session, or history that is
+ * already finished, stays silent.
+ */
+export function createReplySpeaker() {
+  let armed = false;
+  let spokenSeq = -1;
+  return {
+    /** @returns {string} text to speak now, or "" */
+    observe({ sessionChanged, wasRunning, running, reply }) {
+      if (sessionChanged) {
+        armed = Boolean(running);
+        spokenSeq = reply?.seq ?? -1;
+        return "";
+      }
+      if (!wasRunning && running) {
+        armed = true;
+        // A turn start resets the reply, so one already here was loaded with
+        // history, not watched.
+        if (reply) spokenSeq = Math.max(spokenSeq, reply.seq);
+      }
+      let text = "";
+      if (armed && reply && reply.seq > spokenSeq) {
+        spokenSeq = reply.seq;
+        text = reply.text;
+      }
+      if (!running) armed = false;
+      return text;
+    },
+  };
+}
+
+export function emptySpeechState() {
+  return { text: "", seq: -1 };
+}
+
+/**
+ * A steer that lands after the answer runs another step in the same turn, so
+ * a turn may conclude more than once; `seq` tells the answers apart.
+ * @param {{ text: string, seq: number }} state
+ */
+export function reduceSpeechEvent(state, event) {
+  const current = state ?? emptySpeechState();
+  if (event?.type === "turn/start") return emptySpeechState();
+  const text = concludingReply(event);
+  return text ? { text, seq: Number(event.seq) } : current;
+}
+
+export function matchSpeechEvent(event) {
+  const type = event?.type;
+  if (type !== "turn/start" && type !== "assistant/message") return null;
+  const turn = turnOf(event);
+  if (turn === null) return null;
+  return { id: String(turn), role: type === "turn/start" ? "start" : "update" };
+}

package/files/preview-cache.js

@@ -1,100 +0,0 @@
-import { createHash } from "node:crypto";
-import { mkdir, readFile, stat, writeFile } from "node:fs/promises";
-import { join } from "node:path";
-import { runOlaresDownload } from "../drive/download.js";
-import { sanitizeFilename } from "./filename.js";
-
-const inflight = new Map();
-
-export function previewCacheRoot(env = process.env) {
-  const root = env.LARES_DATA_DIR?.trim();
-  return join(root || "/data/lares", "preview-cache");
-}
-
-function cacheDir(cacheRoot, source) {
-  const id = createHash("sha256").update(source).digest("hex").slice(0, 24);
-  return join(cacheRoot, id);
-}
-
-async function readMeta(path) {
-  try {
-    const parsed = JSON.parse(await readFile(path, "utf8"));
-    if (parsed && typeof parsed === "object") return parsed;
-  } catch {
-    // Missing or corrupt meta is a cache miss.
-  }
-  return null;
-}
-
-async function localSize(path) {
-  try {
-    const info = await stat(path);
-    return info.isFile() ? info.size : null;
-  } catch {
-    return null;
-  }
-}
-
-async function attachLocal(file, absolutePath) {
-  const info = await stat(absolutePath);
-  return {
-    ...file,
-    absolutePath,
-    size: info.size,
-    modifiedAt: info.mtimeMs,
-    device: info.dev,
-    inode: info.ino,
-  };
-}
-
-async function materializeOnce(file, options) {
-  const download = options.download ?? runOlaresDownload;
-  const cacheRoot = options.cacheRoot ?? previewCacheRoot(options.env);
-  const dir = cacheDir(cacheRoot, file.path);
-  const dest = join(dir, sanitizeFilename(file.name));
-  const metaPath = join(dir, "meta.json");
-  const expected = { source: file.path, size: file.size, modifiedAt: file.modifiedAt };
-
-  const meta = await readMeta(metaPath);
-  const size = await localSize(dest);
-  const metaMatches = meta?.source === expected.source
-    && meta?.size === expected.size
-    && meta?.modifiedAt === expected.modifiedAt;
-
-  if (metaMatches && size === expected.size) return attachLocal(file, dest);
-
-  await mkdir(dir, { recursive: true });
-  await writeFile(metaPath, JSON.stringify(expected));
-  const resume = metaMatches && size !== null && size < expected.size;
-  await download(file.path, dest, {
-    signal: options.signal,
-    overwrite: !resume,
-    resume,
-    ...(options.spawnFn ? { spawnFn: options.spawnFn } : {}),
-  });
-  const next = await localSize(dest);
-  if (next !== expected.size) {
-    const error = new Error(
-      `downloaded size ${next} does not match files backend size ${expected.size}`,
-    );
-    error.code = "files_unavailable";
-    throw error;
-  }
-  return attachLocal(file, dest);
-}
-
-/**
- * Copy one files-backend file into `$LARES_DATA_DIR/preview-cache` so range
- * streaming can use a local handle. Concurrent requests for the same source
- * share one download; a matching size/mtime reuses the cache.
- */
-export function materializeFilesFile(file, options = {}) {
-  const key = file.path;
-  const pending = inflight.get(key);
-  if (pending) return pending;
-  const done = materializeOnce(file, options).finally(() => {
-    if (inflight.get(key) === done) inflight.delete(key);
-  });
-  inflight.set(key, done);
-  return done;
-}