@olares/lares-core 0.15.10 → 0.15.12

package/voice/agent-session.js

@@ -0,0 +1,133 @@
+import { mintRpcId } from "../larepass/rpc.js";
+import { createSessionFollow } from "../larepass/mux.js";
+import { reduceUtterance } from "./speech.js";
+
+export const SPOKEN_PROMPT = "This is a voice conversation. Keep user-facing answers concise and natural for speech. Do not read reasoning or tool payloads aloud. Preserve the task's factual results and use the user's language.";
+
+/** dsh opens a turn before claiming input; rpcId associates input with that open turn. */
+export function createVoiceAgentSession(api, { sessionId = null, preset, workspaceId, restoredRequests = [], metric = () => {} } = {}) {
+  let creating;
+  let following;
+  let closed = false;
+  let utterance = null;
+  let error = null;
+  let version = 0;
+  let cursor = -1;
+  let openTurn = null;
+  let latestRequest = null;
+  let followTask = null;
+  let ready = null;
+  let readyTimer;
+  let observedRunning = false;
+  const requests = new Map();
+  for (const restored of restoredRequests.slice(-256)) {
+    requests.set(restored.requestId, { requestId: restored.requestId, turn: null, status: "unknown", utterance: null, error: null, at: Date.now(), firstTextAt: null });
+  }
+  const decoder = createSessionFollow();
+  const unfinished = (task) => ["accepted", "running", "waiting", "unknown"].includes(task.status);
+  const snapshot = () => ({ sessionId, running: observedRunning || [...requests.values()].some(unfinished), utterance, error, version,
+    requestId: latestRequest,
+    tasks: [...requests.values()].map(({ requestId, turn, status, utterance: reply, error: taskError }) => ({ requestId, turn, status, text: reply?.silent ? "" : reply?.text || "", error: taskError })),
+  });
+  const ensure = async () => {
+    if (sessionId) return sessionId;
+    creating ??= Promise.resolve(api.create({ agentPreset: preset, workspaceId })).then((value) => {
+      if (!value?.sessionId) throw new Error("voice_session_failed");
+      sessionId = value.sessionId;
+      return sessionId;
+    });
+    return creating;
+  };
+  const publish = (event, replay = false) => {
+    if (!event || typeof event !== "object") return;
+    if (typeof event.seq === "number") {
+      if (event.seq <= cursor) return;
+      cursor = event.seq;
+    }
+    if (event.type === "turn/start") { openTurn = event.data.turn; observedRunning = true; }
+    if (event.type === "user/message") {
+      const id = event.data?.requestId ?? event.data?.source?.rpcId;
+      const task = requests.get(id);
+      if (task && openTurn !== null) { task.turn = openTurn; task.status = "running"; }
+    }
+    const turn = event.data?.turn ?? openTurn;
+    const owned = [...requests.values()].filter((task) => task.turn != null && task.turn === turn && unfinished(task));
+    for (const task of owned) {
+      task.utterance = reduceUtterance(task.utterance, event);
+      if (event.type === "approval/asked") task.status = "waiting";
+      if (event.type === "approval/decided") task.status = "running";
+      if (event.type === "turn/end") {
+        const reason = event.data?.reason?.kind;
+        task.status = reason === "error" ? "failed" : reason === "cancelled" ? "cancelled" : "completed";
+        task.error = reason === "error" ? "voice_agent_failed" : reason === "cancelled" ? "voice_cancelled" : null;
+      }
+      if (!replay && task.utterance?.text && !task.firstTextAt) {
+        task.firstTextAt = Date.now();
+        metric({ stage: "agent_first_text", requestId: task.requestId, durationMs: task.firstTextAt - task.at });
+      }
+      if (!replay && task.requestId === latestRequest) { utterance = task.utterance; error = task.error; }
+    }
+    if (event.type === "turn/end" && openTurn === event.data?.turn) { openTurn = null; observedRunning = false; }
+    version++;
+  };
+  const follow = () => {
+    if (following && !following.signal.aborted) return ready;
+    const abort = new AbortController();
+    following = abort;
+    let resolveReady;
+    let rejectReady;
+    ready = new Promise((resolve, reject) => { resolveReady = resolve; rejectReady = reject; });
+    readyTimer = setTimeout(() => { rejectReady(new Error("voice_agent_timeout")); abort.abort(); }, 15_000);
+    ready.catch(() => {});
+    abort.signal.addEventListener("abort", () => rejectReady(new Error("voice_session_closed")), { once: true });
+    followTask = (async () => {
+      try {
+        for await (const frame of api.follow({ address: { kind: "session", sessionId }, assistantStream: true }, abort.signal)) {
+          const decoded = decoder.push(frame);
+          if (frame.type === "snapshot") {
+            for (const event of decoded?.events ?? []) publish(event, true);
+            clearTimeout(readyTimer); resolveReady();
+          } else for (const event of decoded?.events ?? []) publish(event);
+        }
+        if (!abort.signal.aborted) throw new Error("voice_agent_disconnected");
+      } catch (cause) {
+        clearTimeout(readyTimer); rejectReady(cause);
+        if (!abort.signal.aborted) { error = "voice_agent_disconnected"; version++; abort.abort(); }
+      }
+    })();
+    followTask.catch(() => {});
+    return ready;
+  };
+  return {
+    snapshot,
+    async open() { if (sessionId) await follow(); },
+    async resume() {
+      following?.abort(); await followTask;
+      closed = false; error = null; utterance = null; cursor = -1; openTurn = null; observedRunning = false; decoder.reset();
+      if (sessionId) await follow();
+    },
+    async submit(text, requestId = mintRpcId(), mode = "queue") {
+      if (closed) throw new Error("voice_session_closed");
+      if (!String(text).trim()) return snapshot();
+      if (requests.has(requestId)) return requests.get(requestId).operation || snapshot();
+      if (requests.size >= 256) throw new Error("voice_session_limit");
+      const task = { requestId, turn: null, status: "accepted", utterance: null, error: null, at: Date.now(), firstTextAt: null };
+      requests.set(requestId, task);
+      latestRequest = requestId; utterance = null; error = null; version++;
+      task.operation = (async () => {
+        try {
+          await ensure();
+          await follow();
+          await api.prompt({ sessionId, requestId, mode: mode === "steer" ? "steer" : "queue", content: [{ type: "text", text }] }, following.signal);
+          return snapshot();
+        } catch (cause) {
+          task.status = "unknown"; task.error = "submission_outcome_unknown"; version++;
+          throw cause;
+        }
+      })();
+      return task.operation;
+    },
+    async cancel() { if (sessionId) await api.cancel({ sessionId }); },
+    async close() { closed = true; clearTimeout(readyTimer); following?.abort(); await followTask; },
+  };
+}

package/voice/agent.js

@@ -29,7 +29,15 @@ export async function liveAgent(options = {}) {
   return agentIsLive(info, now()) ? info : null;
 }
 
-export async function askAgent(op, fields, options) {
+const requests = new Map();
+export function askAgent(op, fields, options = {}) {
+  const key = voiceDir(options);
+  const result = (requests.get(key) ?? Promise.resolve()).catch(() => {}).then(() => performAgentRequest(op, fields, options));
+  requests.set(key, result);
+  void result.finally(() => { if (requests.get(key) === result) requests.delete(key); }).catch(() => {});
+  return result;
+}
+async function performAgentRequest(op, fields, options) {
   const dir = voiceDir(options);
   const now = options.now ?? Date.now;
   const sleep = options.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));

package/voice/browser-capture.js

@@ -0,0 +1,274 @@
+import {
+  formatElapsed,
+  isRecordingTooShort,
+  pickRecordingMime,
+} from "./recorder.js";
+
+export { formatElapsed };
+
+/** Phone mic. Bare booleans are exact and iOS rejects the headset profile. */
+const DEVICE_AUDIO = {
+  channelCount: { ideal: 1 },
+  echoCancellation: { ideal: true },
+  noiseSuppression: { ideal: true },
+  autoGainControl: { ideal: true },
+};
+
+const CHUNK_MS = 250;
+
+function micFailure(err) {
+  const name = err?.name || "";
+  if (name === "NotAllowedError" || name === "PermissionDeniedError") return "voice_permission_denied";
+  if (name === "SecurityError") return "voice_unsupported";
+  return "voice_failed";
+}
+
+async function openDeviceMic(mediaDevices) {
+  try {
+    return await mediaDevices.getUserMedia({ audio: DEVICE_AUDIO });
+  } catch (err) {
+    if (micFailure(err) !== "voice_failed") throw err;
+    return mediaDevices.getUserMedia({ audio: true });
+  }
+}
+
+export function createVoiceCapture({
+  transcribe,
+  submit,
+  onStream,
+  onPhase,
+  onTrace,
+  now = Date.now,
+  media,
+}) {
+  const mediaDevices = media?.mediaDevices ?? globalThis.navigator?.mediaDevices;
+  const Recorder = media?.MediaRecorder ?? globalThis.MediaRecorder;
+  let recorder = null;
+  let detector = null;
+  let stream = null;
+  let chunks = [];
+  let started = 0;
+  let timer = 0;
+  let errorTimer = 0;
+  let finishTimer = 0;
+  let cancel = false;
+  let released = false;
+  let finished = false;
+  let dead = false;
+  let abort = null;
+  let phase = "idle";
+
+  const emit = (next, extra = {}) => {
+    phase = next;
+    onPhase?.(next, extra);
+  };
+
+  const trace = (event, fields) => {
+    try {
+      onTrace?.(event, fields);
+    } catch {
+      /* a trace must not change the take */
+    }
+  };
+
+  const stopTimer = () => {
+    if (timer) {
+      clearInterval(timer);
+      timer = 0;
+    }
+  };
+
+  const release = () => {
+    detector?.stop?.();
+    detector = null;
+    stream?.getTracks().forEach((track) => track.stop());
+    stream = null;
+    recorder = null;
+  };
+
+  const fail = (code) => {
+    if (dead) return;
+    if (errorTimer) clearTimeout(errorTimer);
+    emit("error", { error: code, elapsed: 0 });
+    errorTimer = setTimeout(() => {
+      errorTimer = 0;
+      if (!dead && phase === "error") emit("idle", { error: "", elapsed: 0 });
+    }, 3200);
+  };
+
+  const finish = async () => {
+    if (finished || dead) return;
+    finished = true;
+    stopTimer();
+    const took = now() - started;
+    const type = recorder?.mimeType || chunks[0]?.type || "audio/webm";
+    const blobChunks = chunks;
+    chunks = [];
+    release();
+    if (cancel) {
+      emit("idle", { error: "", elapsed: 0 });
+      return;
+    }
+    const blob = new Blob(blobChunks, { type });
+    trace("blob", { bytes: blob.size, type, ms: took, chunks: blobChunks.length });
+    if (isRecordingTooShort(took, blob.size)) {
+      trace("too-short", { bytes: blob.size, ms: took });
+      fail("voice_too_short");
+      return;
+    }
+    emit("transcribing", { elapsed: took, error: "" });
+    abort = new AbortController();
+    try {
+      const text = String(await transcribe(blob, abort.signal) ?? "").trim();
+      if (dead || cancel) return;
+      if (!text) {
+        trace("no-input", {});
+        fail("voice_no_input");
+        return;
+      }
+      trace("transcript", { chars: text.length });
+      emit("idle", { error: "", elapsed: 0 });
+      await submit?.(text);
+    } catch (err) {
+      if (dead || abort.signal.aborted || cancel) return;
+      const code = err instanceof Error ? err.message : "voice_failed";
+      trace("transcribe-error", { code });
+      fail(code);
+    } finally {
+      abort = null;
+    }
+  };
+
+  const scheduleFinish = () => {
+    if (finishTimer) clearTimeout(finishTimer);
+    finishTimer = setTimeout(() => {
+      finishTimer = 0;
+      void finish();
+    }, 30);
+  };
+
+  const endRecorder = () => {
+    if (!recorder || recorder.state === "inactive") {
+      scheduleFinish();
+      return;
+    }
+    try {
+      recorder.requestData?.();
+    } catch {
+      /* the final dataavailable still arrives from stop() */
+    }
+    try {
+      recorder.stop();
+    } catch {
+      scheduleFinish();
+    }
+  };
+
+  return {
+    get phase() {
+      return phase;
+    },
+    async start() {
+      if (dead || phase === "preparing" || phase === "recording" || phase === "transcribing") return;
+      if (typeof Recorder === "undefined" || !mediaDevices?.getUserMedia) {
+        fail("voice_unsupported");
+        return;
+      }
+      cancel = false;
+      released = false;
+      finished = false;
+      trace("open", {});
+      emit("preparing", { elapsed: 0, error: "" });
+      let next;
+      try {
+        next = await openDeviceMic(mediaDevices);
+      } catch (err) {
+        const code = micFailure(err);
+        trace("mic-fail", { code, name: err?.name || "" });
+        if (!dead) fail(code);
+        return;
+      }
+      if (dead || cancel) {
+        next.getTracks().forEach((track) => track.stop());
+        if (!dead && phase === "preparing") emit("idle", { error: "", elapsed: 0 });
+        return;
+      }
+      stream = next;
+      chunks = [];
+      const mime = pickRecordingMime(Recorder);
+      try {
+        recorder = new Recorder(stream, mime ? { mimeType: mime } : undefined);
+      } catch {
+        release();
+        fail("voice_unsupported");
+        return;
+      }
+      recorder.ondataavailable = (event) => {
+        if (event.data?.size) chunks.push(event.data);
+        if (recorder?.state === "inactive") scheduleFinish();
+      };
+      recorder.onstop = () => scheduleFinish();
+      try { detector = await onStream?.(stream); } catch { release(); fail("voice_unsupported"); return; }
+      started = now();
+      try {
+        recorder.start(CHUNK_MS);
+      } catch {
+        trace("record-fail", { mime });
+        release();
+        fail("voice_unsupported");
+        return;
+      }
+      trace("recording", { mime: recorder.mimeType || mime || "" });
+      if (released || cancel) {
+        endRecorder();
+        return;
+      }
+      emit("recording", { elapsed: 0, error: "" });
+      timer = setInterval(() => emit("recording", { elapsed: now() - started, error: "" }), 200);
+    },
+    /**
+     * Finger-up finishes the take. The mic opens asynchronously, so the finger
+     * often lifts while still preparing — that is not a cancel.
+     */
+    stop(aborting = false) {
+      if (phase === "transcribing") {
+        if (aborting) {
+          cancel = true;
+          abort?.abort();
+        }
+        return;
+      }
+      if (phase !== "preparing" && phase !== "recording") {
+        trace("stop-ignored", { aborting: Boolean(aborting), phase });
+        return;
+      }
+      trace("stop", {
+        aborting: Boolean(aborting),
+        phase,
+        recorder: recorder?.state || "none",
+      });
+      if (aborting) {
+        cancel = true;
+        released = false;
+        if (recorder && recorder.state !== "inactive") recorder.stop();
+        return;
+      }
+      released = true;
+      stopTimer();
+      if (recorder && recorder.state !== "inactive") {
+        emit("transcribing", { elapsed: now() - started, error: "" });
+        endRecorder();
+      }
+    },
+    dispose() {
+      dead = true;
+      cancel = true;
+      if (errorTimer) clearTimeout(errorTimer);
+      if (finishTimer) clearTimeout(finishTimer);
+      stopTimer();
+      abort?.abort();
+      if (recorder && recorder.state !== "inactive") recorder.stop();
+      release();
+    },
+  };
+}

package/voice/browser-playback.js

@@ -0,0 +1,30 @@
+export function createBrowserPlayer() {
+  let element = null;
+  let finish = null;
+  const stop = () => {
+    element?.pause();
+    finish?.();
+  };
+  return {
+    play(blob) {
+      stop();
+      return new Promise((resolve) => {
+        const url = URL.createObjectURL(blob);
+        const audio = new Audio(url);
+        const done = () => {
+          if (finish !== done) return;
+          finish = null;
+          element = null;
+          URL.revokeObjectURL(url);
+          resolve();
+        };
+        element = audio;
+        finish = done;
+        audio.addEventListener("ended", done, { once: true });
+        audio.addEventListener("error", done, { once: true });
+        audio.play().catch(done);
+      });
+    },
+    stop,
+  };
+}

package/voice/capture.js

@@ -7,6 +7,8 @@ const FRAME_MS = 32;
 const NO_INPUT_MS = 10_000;
 const POLL_MS = 40;
 const CLIENT_LOST_MS = 5000;
+const MAX_CAPTURE_MS = 60_000;
+const MAX_CAPTURE_BYTES = 16_000 * 2 * MAX_CAPTURE_MS / 1000;
 const HOST_RATE = 16_000;
 const PLAYBACK_SETTLE_MS = 320;
 const PLAYBACK_LEADING_MS = 2500;
@@ -125,7 +127,13 @@ export function createCaptureSession(io) {
       controller: new AbortController(),
     };
     job = run;
-    const opened = await io.open();
+    let opened;
+    try { opened = await io.open(); }
+    catch (error) {
+      run.phase = "error";
+      run.error = error?.code || "voice_failed";
+      throw error;
+    }
     if (run.cancel) {
       await io.close?.();
       run.phase = "idle";
@@ -161,7 +169,9 @@ export function createCaptureSession(io) {
       try {
         while (run === job && run.phase !== "done" && run.phase !== "error" && run.phase !== "idle") {
           if (run.phase === "recording" && now() - run.seen > CLIENT_LOST_MS) run.cancel = true;
-          const fresh = await io.read(offset);
+          const available = await io.read(offset);
+          const fresh = available.subarray(0, Math.max(0, MAX_CAPTURE_BYTES - pcm.length));
+          if (fresh.length < available.length || now() - started >= MAX_CAPTURE_MS) run.stop = true;
           offset += fresh.length;
           if (fresh.length) {
             pcm = Buffer.concat([pcm, fresh]);

package/voice/client.js

@@ -1,8 +1,14 @@
+import { mintRpcId } from "../larepass/rpc.js";
 import { createSnapshotStore } from "../tools/async.js";
 
 export const API = "/api/lares/voice";
 
 const settings = createSnapshotStore();
+export const voiceClientId = mintRpcId();
+export function voiceHeaders(device) {
+  const selected = device ?? settings.peek()?.config?.device;
+  return { "x-lares-voice-id": voiceClientId, "x-lares-voice-device": selected === "device" ? "device" : "headset" };
+}
 
 export function rememberedVoiceSettings() {
   return settings.peek();
@@ -21,6 +27,9 @@ export async function loadVoiceSettings(options = {}) {
       status,
       sttModels: Array.isArray(models.stt) ? models.stt : [],
       ttsModels: Array.isArray(models.tts) ? models.tts : [],
+      chatModels: Array.isArray(models.chat) ? models.chat : [],
+      realtimeModels: Array.isArray(models.realtime) ? models.realtime : [],
+      realtimeReady: Array.isArray(models.realtimeReady) ? models.realtimeReady : [],
       charts: {
         stt: chartView(models.charts?.stt),
         tts: chartView(models.charts?.tts),
@@ -32,6 +41,7 @@ export async function loadVoiceSettings(options = {}) {
 function chartView(row) {
   const phase = row?.phase;
   return {
+    app: typeof row?.app === "string" ? row.app : "",
     phase: phase === "running" || phase === "stopped" || phase === "busy" || phase === "failed" ? phase : "absent",
     state: typeof row?.state === "string" ? row.state : "",
   };
@@ -62,11 +72,12 @@ export function wavBlob(base64) {
   return new Blob([bytes], { type: "audio/wav" });
 }
 
-export async function postSpeak(text) {
+/** Hands over a streaming answer's text; with `part`, also returns that sentence's audio. */
+export async function postSpeak({ key, text, done, part }) {
   const res = await fetch(`${API}/speak`, {
     method: "POST",
-    headers: { "content-type": "application/json" },
-    body: JSON.stringify({ text }),
+    headers: { "content-type": "application/json", ...voiceHeaders() },
+    body: JSON.stringify(Number.isInteger(part) ? { key, text, done, part } : { key, text, done }),
   });
   let body = null;
   try {
@@ -74,7 +85,11 @@ export async function postSpeak(text) {
   } catch {
     body = null;
   }
-  if (!res.ok || typeof body?.audio !== "string") throw new Error(transcribeErrorCode(body));
+  if (res.ok && body && !Number.isInteger(part)) return { sentences: Number(body.sentences) || 0 };
+  if (!res.ok || typeof body?.audio !== "string") {
+    const code = transcribeErrorCode(body);
+    throw Object.assign(new Error(code), { code });
+  }
   return {
     bluetooth: body.bluetooth === "played" ? "played" : "skipped",
     blob: wavBlob(body.audio),
@@ -82,7 +97,7 @@ export async function postSpeak(text) {
 }
 
 export async function postPlaybackStop(signal) {
-  const res = await fetch(`${API}/playback/stop`, { method: "POST", signal });
+  const res = await fetch(`${API}/playback/stop`, { method: "POST", signal, headers: voiceHeaders() });
   if (!res.ok) throw new Error(String(res.status));
   return res.json();
 }
@@ -97,6 +112,53 @@ export async function saveVoiceSettings(patch) {
   return loadVoiceSettings({ force: true });
 }
 
+/** Reading mode and the GPU slicing it needs; see core/voice/gpu-mode.js. */
+export async function loadGpuMode() {
+  const res = await fetch(`${API}/gpu-mode`);
+  if (!res.ok) throw new Error(String(res.status));
+  return res.json();
+}
+
+/** A failed GPU request keeps the Host's code and params for gpuProblemText. */
+async function gpuResponse(res) {
+  const payload = await res.json().catch(() => null);
+  if (res.ok) return payload;
+  const error = payload?.error;
+  throw Object.assign(new Error(error?.message || String(res.status)), {
+    code: typeof error?.code === "string" ? error.code : "voice_gpu_failed",
+    params: error?.params && typeof error.params === "object" ? error.params : {},
+  });
+}
+
+export async function switchGpuMode(streaming) {
+  const res = await fetch(`${API}/gpu-mode`, {
+    method: "POST",
+    headers: { "content-type": "application/json" },
+    body: JSON.stringify({ streaming }),
+  });
+  return gpuResponse(res);
+}
+
+/** @param {{ app: string, enabled: boolean, memory: number }[]} apps memory in bytes */
+export async function saveSlicePlan(apps) {
+  const res = await fetch(`${API}/gpu-mode/slices`, {
+    method: "POST",
+    headers: { "content-type": "application/json" },
+    body: JSON.stringify({ apps }),
+  });
+  return gpuResponse(res);
+}
+
+/** @param {string} app @param {"resume" | "stop"} op */
+export async function postGpuApp(app, op) {
+  const res = await fetch(`${API}/gpu-mode/app`, {
+    method: "POST",
+    headers: { "content-type": "application/json" },
+    body: JSON.stringify({ app, op }),
+  });
+  return gpuResponse(res);
+}
+
 function transcribeErrorCode(payload) {
   const code = payload && typeof payload === "object" ? payload.error?.code : undefined;
   return typeof code === "string" ? code : "voice_failed";
@@ -140,19 +202,19 @@ async function readVoiceJson(res) {
 
 /** Host recording. `skipped` means the headset microphone is not available. */
 export async function postCaptureStart(signal) {
-  return readVoiceJson(await fetch(`${API}/capture`, { method: "POST", signal }));
+  return readVoiceJson(await fetch(`${API}/capture`, { method: "POST", signal, headers: voiceHeaders() }));
 }
 
 export async function getCapture(signal) {
-  return readVoiceJson(await fetch(`${API}/capture`, { signal }));
+  return readVoiceJson(await fetch(`${API}/capture`, { signal, headers: voiceHeaders() }));
 }
 
 export async function postCaptureStop(signal) {
-  return readVoiceJson(await fetch(`${API}/capture/stop`, { method: "POST", signal }));
+  return readVoiceJson(await fetch(`${API}/capture/stop`, { method: "POST", signal, headers: voiceHeaders() }));
 }
 
 export async function postCaptureCancel(signal) {
-  return readVoiceJson(await fetch(`${API}/capture/cancel`, { method: "POST", signal }));
+  return readVoiceJson(await fetch(`${API}/capture/cancel`, { method: "POST", signal, headers: voiceHeaders() }));
 }
 
 export async function postBluetoothReady(signal) {