@olares/lares-mobile 0.15.7 → 0.15.9

package/src/chat/voice.js

@@ -1,21 +1,57 @@
-import { TranscriptQueue } from "@olares/lares-core/voice/client";
 import {
   formatElapsed,
   isRecordingTooShort,
   pickRecordingMime,
-  RECORDING_AUDIO,
 } from "@olares/lares-core/voice/recorder";
 
 export { formatElapsed };
 
-export function createVoiceCapture({ transcribe, getDraft, setDraft, onPhase }) {
-  const queue = new TranscriptQueue();
+/** Phone mic. Bare booleans are exact and iOS rejects the headset profile. */
+const DEVICE_AUDIO = {
+  channelCount: { ideal: 1 },
+  echoCancellation: { ideal: true },
+  noiseSuppression: { ideal: true },
+  autoGainControl: { ideal: true },
+};
+
+const CHUNK_MS = 250;
+
+function micFailure(err) {
+  const name = err?.name || "";
+  if (name === "NotAllowedError" || name === "PermissionDeniedError") return "voice_permission_denied";
+  if (name === "SecurityError") return "voice_unsupported";
+  return "voice_failed";
+}
+
+async function openDeviceMic(mediaDevices) {
+  try {
+    return await mediaDevices.getUserMedia({ audio: DEVICE_AUDIO });
+  } catch (err) {
+    if (micFailure(err) !== "voice_failed") throw err;
+    return mediaDevices.getUserMedia({ audio: true });
+  }
+}
+
+export function createVoiceCapture({
+  transcribe,
+  submit,
+  onPhase,
+  onTrace,
+  now = Date.now,
+  media,
+}) {
+  const mediaDevices = media?.mediaDevices ?? globalThis.navigator?.mediaDevices;
+  const Recorder = media?.MediaRecorder ?? globalThis.MediaRecorder;
   let recorder = null;
   let stream = null;
   let chunks = [];
   let started = 0;
   let timer = 0;
+  let errorTimer = 0;
+  let finishTimer = 0;
   let cancel = false;
+  let released = false;
+  let finished = false;
   let dead = false;
   let abort = null;
   let phase = "idle";
@@ -25,6 +61,14 @@ export function createVoiceCapture({ transcribe, getDraft, setDraft, onPhase })
     onPhase?.(next, extra);
   };
 
+  const trace = (event, fields) => {
+    try {
+      onTrace?.(event, fields);
+    } catch {
+      /* a trace must not change the take */
+    }
+  };
+
   const stopTimer = () => {
     if (timer) {
       clearInterval(timer);
@@ -40,96 +84,182 @@ export function createVoiceCapture({ transcribe, getDraft, setDraft, onPhase })
 
   const fail = (code) => {
     if (dead) return;
+    if (errorTimer) clearTimeout(errorTimer);
     emit("error", { error: code, elapsed: 0 });
-    setTimeout(() => {
+    errorTimer = setTimeout(() => {
+      errorTimer = 0;
       if (!dead && phase === "error") emit("idle", { error: "", elapsed: 0 });
     }, 3200);
   };
 
   const finish = async () => {
+    if (finished || dead) return;
+    finished = true;
     stopTimer();
-    const took = Date.now() - started;
-    const type = recorder?.mimeType || "audio/webm";
+    const took = now() - started;
+    const type = recorder?.mimeType || chunks[0]?.type || "audio/webm";
     const blobChunks = chunks;
     chunks = [];
     release();
-    if (dead) return;
     if (cancel) {
-      emit("idle", { elapsed: 0 });
+      emit("idle", { error: "", elapsed: 0 });
       return;
     }
     const blob = new Blob(blobChunks, { type });
+    trace("blob", { bytes: blob.size, type, ms: took, chunks: blobChunks.length });
     if (isRecordingTooShort(took, blob.size)) {
+      trace("too-short", { bytes: blob.size, ms: took });
       fail("voice_too_short");
       return;
     }
-    emit("transcribing", { elapsed: took });
+    emit("transcribing", { elapsed: took, error: "" });
     abort = new AbortController();
     try {
-      const text = await transcribe(blob, abort.signal);
-      if (dead) return;
-      emit("idle", { elapsed: 0 });
-      queue.apply(text, true, getDraft(), setDraft);
+      const text = String(await transcribe(blob, abort.signal) ?? "").trim();
+      if (dead || cancel) return;
+      if (!text) {
+        trace("no-input", {});
+        fail("voice_no_input");
+        return;
+      }
+      trace("transcript", { chars: text.length, head: text.slice(0, 40) });
+      emit("idle", { error: "", elapsed: 0 });
+      submit?.(text);
     } catch (err) {
-      if (dead || abort.signal.aborted) return;
-      fail(err instanceof Error ? err.message : "voice_failed");
+      if (dead || abort.signal.aborted || cancel) return;
+      const code = err instanceof Error ? err.message : "voice_failed";
+      trace("transcribe-error", { code });
+      fail(code);
     } finally {
       abort = null;
     }
   };
 
+  const scheduleFinish = () => {
+    if (finishTimer) clearTimeout(finishTimer);
+    finishTimer = setTimeout(() => {
+      finishTimer = 0;
+      void finish();
+    }, 30);
+  };
+
+  const endRecorder = () => {
+    if (!recorder || recorder.state === "inactive") {
+      scheduleFinish();
+      return;
+    }
+    try {
+      recorder.requestData?.();
+    } catch {
+      /* the final dataavailable still arrives from stop() */
+    }
+    try {
+      recorder.stop();
+    } catch {
+      scheduleFinish();
+    }
+  };
+
   return {
     get phase() {
       return phase;
     },
     async start() {
-      if (dead || phase === "recording" || phase === "transcribing") return;
-      if (typeof MediaRecorder === "undefined" || !navigator.mediaDevices?.getUserMedia) {
+      if (dead || phase === "preparing" || phase === "recording" || phase === "transcribing") return;
+      if (typeof Recorder === "undefined" || !mediaDevices?.getUserMedia) {
         fail("voice_unsupported");
         return;
       }
       cancel = false;
+      released = false;
+      finished = false;
+      trace("open", {});
+      emit("preparing", { elapsed: 0, error: "" });
       let next;
       try {
-        next = await navigator.mediaDevices.getUserMedia({ audio: RECORDING_AUDIO });
-      } catch {
-        fail("voice_permission_denied");
+        next = await openDeviceMic(mediaDevices);
+      } catch (err) {
+        const code = micFailure(err);
+        trace("mic-fail", { code, name: err?.name || "" });
+        if (!dead) fail(code);
         return;
       }
       if (dead || cancel) {
         next.getTracks().forEach((track) => track.stop());
+        if (!dead && phase === "preparing") emit("idle", { error: "", elapsed: 0 });
         return;
       }
       stream = next;
       chunks = [];
-      const mime = pickRecordingMime();
-      recorder = new MediaRecorder(stream, mime ? { mimeType: mime } : undefined);
+      const mime = pickRecordingMime(Recorder);
+      try {
+        recorder = new Recorder(stream, mime ? { mimeType: mime } : undefined);
+      } catch {
+        release();
+        fail("voice_unsupported");
+        return;
+      }
       recorder.ondataavailable = (event) => {
         if (event.data?.size) chunks.push(event.data);
+        if (recorder?.state === "inactive") scheduleFinish();
       };
-      recorder.onstop = () => {
-        void finish();
-      };
-      started = Date.now();
+      recorder.onstop = () => scheduleFinish();
+      started = now();
+      try {
+        recorder.start(CHUNK_MS);
+      } catch {
+        trace("record-fail", { mime });
+        release();
+        fail("voice_unsupported");
+        return;
+      }
+      trace("recording", { mime: recorder.mimeType || mime || "" });
+      if (released || cancel) {
+        endRecorder();
+        return;
+      }
       emit("recording", { elapsed: 0, error: "" });
-      recorder.start();
-      timer = setInterval(() => emit("recording", { elapsed: Date.now() - started }), 200);
+      timer = setInterval(() => emit("recording", { elapsed: now() - started, error: "" }), 200);
     },
+    /**
+     * Finger-up finishes the take. The mic opens asynchronously, so the finger
+     * often lifts while still preparing — that is not a cancel.
+     */
     stop(aborting = false) {
-      cancel = Boolean(aborting);
-      if (recorder && recorder.state !== "inactive") recorder.stop();
-      else {
-        stopTimer();
-        release();
-        emit("idle", { elapsed: 0 });
+      if (phase === "transcribing") {
+        if (aborting) {
+          cancel = true;
+          abort?.abort();
+        }
+        return;
+      }
+      if (phase !== "preparing" && phase !== "recording") {
+        trace("stop-ignored", { aborting: Boolean(aborting), phase });
+        return;
+      }
+      trace("stop", {
+        aborting: Boolean(aborting),
+        phase,
+        recorder: recorder?.state || "none",
+      });
+      if (aborting) {
+        cancel = true;
+        released = false;
+        if (recorder && recorder.state !== "inactive") recorder.stop();
+        return;
+      }
+      released = true;
+      stopTimer();
+      if (recorder && recorder.state !== "inactive") {
+        emit("transcribing", { elapsed: now() - started, error: "" });
+        endRecorder();
       }
-    },
-    toggle() {
-      if (phase === "recording") this.stop(false);
-      else if (phase === "idle" || phase === "error") void this.start();
     },
     dispose() {
       dead = true;
+      cancel = true;
+      if (errorTimer) clearTimeout(errorTimer);
+      if (finishTimer) clearTimeout(finishTimer);
       stopTimer();
       abort?.abort();
       if (recorder && recorder.state !== "inactive") recorder.stop();

package/src/desktop/DesktopPreview.vue

@@ -2,9 +2,16 @@
   <section class="desktop-preview" :aria-label="t('preview')">
     <header class="desktop-preview__bar">
       <span class="desktop-preview__path" :title="path">{{ path }}</span>
+      <a
+        v-if="filesHref"
+        class="desktop-preview__action"
+        :href="filesHref"
+        target="_blank"
+        rel="noopener noreferrer"
+      >{{ t("openInFiles") }}</a>
       <a
         v-if="downloadHref"
-        class="desktop-preview__download"
+        class="desktop-preview__action"
         :href="downloadHref"
         download
       >{{ t("download") }}</a>
@@ -40,6 +47,7 @@ export default {
     error: { type: String, default: "" },
     mediaSrc: { type: String, default: "" },
     downloadHref: { type: String, default: "" },
+    filesHref: { type: String, default: "" },
     hrefFor: { type: Function, default: null },
     t: { type: Function, required: true },
   },
@@ -53,6 +61,6 @@ export default {
 .desktop-preview { position:absolute; z-index:8; display:flex; min-width:0; min-height:0; flex-direction:column; inset:0; padding:12px 20px 20px; background:var(--q-background-1); }
 .desktop-preview__bar { display:flex; min-width:0; flex:none; align-items:center; gap:12px; padding-bottom:10px; }
 .desktop-preview__path { min-width:0; flex:1; overflow:hidden; padding:0 4px; color:var(--q-ink-3); font-size:12px; line-height:18px; text-overflow:ellipsis; white-space:nowrap; }
-.desktop-preview__download { flex:none; color:var(--q-blue-default); font-size:12px; line-height:18px; text-decoration:none; }
+.desktop-preview__action { flex:none; color:var(--q-blue-default); font-size:12px; line-height:18px; text-decoration:none; }
 .desktop-preview__content { display:flex; min-height:0; flex:1; flex-direction:column; overflow:hidden; border:1px solid var(--q-separator); border-radius:14px; background:var(--q-background-2); }
 </style>

package/src/desktop/DesktopShell.vue

@@ -44,6 +44,7 @@
           ref="transcript"
           :items="state.viewItems"
           :running="state.running"
+          :running-since="state.runningSince"
           :loading="state.historyLoading || state.starting"
           :previews="state.previews"
           :media-url="mediaUrl"
@@ -95,6 +96,7 @@
             :error="state.preview.error"
             :media-src="state.previewMediaSrc"
             :download-href="state.previewDownloadHref"
+            :files-href="state.previewFilesHref"
             :href-for="previewHref"
             :t="t"
             @retry="$emit('open-file', state.preview.path)"

package/src/host.js

@@ -2,12 +2,18 @@ import { hostTarget, hostUrl, MODELS_PATH, probeHost } from "@olares/lares-core/
 import { callRpc, consumeMux, ensureSession, loadTranscript, MUX_PATH, sendPrompt } from "@olares/lares-core/larepass/chat";
 import { commandsUrl, normalizeCommands } from "@olares/lares-core/larepass/commands";
 import { normalizeReferenceCandidates, referencesUrl } from "@olares/lares-core/larepass/references";
-import { muxWsUrl } from "@olares/lares-core/larepass/mux";
+import { muxWsUrl, remoteStreamOpen, workspaceBaseline } from "@olares/lares-core/larepass/mux";
 import { RESPOND_PATH } from "@olares/lares-core/larepass/rpc";
 import { createHostSettings } from "@olares/lares-core/larepass/settings";
-import { downloadFileUrl, previewMetaUrl, rawFileUrl } from "@olares/lares-core/files/preview-workspace";
+import {
+  downloadFileUrl,
+  isPreviewPayload,
+  previewMetaUrl,
+  rawFileUrl,
+} from "@olares/lares-core/files/preview-workspace";
 import { FILES_UPLOAD_PATH, uploadFile } from "@olares/lares-core/files/upload-client";
 import { API as VOICE_API, postTranscribe } from "@olares/lares-core/voice/client";
+import { postReport, REPORT_PATH } from "@olares/lares-core/log/report";
 
 function pageOrigin() {
   try {
@@ -35,6 +41,72 @@ function socketProtocols(ports) {
   return value ? String(value) : undefined;
 }
 
+function listWorkspaces(ports) {
+  const Socket = globalThis.WebSocket;
+  if (typeof Socket !== "function") {
+    return Promise.resolve({
+      ok: false,
+      status: "error",
+      error: { code: "no-websocket", message: "WebSocket is not available" },
+    });
+  }
+  const streamId = typeof globalThis.crypto?.randomUUID === "function"
+    ? globalThis.crypto.randomUUID()
+    : `ws-${Date.now()}`;
+  return openMuxSocket(hostUrl({
+    baseUrl: hostTarget(ports).baseUrl,
+    proxyPrefix: hostTarget(ports).proxyPrefix,
+    path: MUX_PATH,
+  }), undefined, socketProtocols(ports)).then((opened) => {
+    if (!opened.ok || !opened.body) {
+      return {
+        ok: false,
+        status: opened.status || "error",
+        error: opened.error || { code: "workspace", message: "workspace follow failed" },
+      };
+    }
+    const socket = opened.body;
+    return new Promise((resolve) => {
+      let settled = false;
+      const finish = (result) => {
+        if (settled) return;
+        settled = true;
+        clearTimeout(timer);
+        try {
+          socket.send(JSON.stringify({ type: "cancel", streamId }));
+        } catch {
+          // already closing
+        }
+        try {
+          socket.close();
+        } catch {
+          // already closing
+        }
+        resolve(result);
+      };
+      const timer = setTimeout(() => {
+        finish({
+          ok: false,
+          status: "error",
+          error: { code: "workspace", message: "workspace follow timed out" },
+        });
+      }, 8000);
+      socket.addEventListener("message", (event) => {
+        let message;
+        try {
+          message = JSON.parse(String(event.data ?? ""));
+        } catch {
+          return;
+        }
+        const baseline = workspaceBaseline(message);
+        if (!baseline) return;
+        finish({ ok: true, status: "ok", value: baseline });
+      });
+      socket.send(JSON.stringify(remoteStreamOpen(streamId, "workspace/follow")));
+    });
+  });
+}
+
 function openMuxSocket(httpUrl, signal, protocols) {
   const Socket = globalThis.WebSocket;
   if (typeof Socket !== "function") {
@@ -82,19 +154,24 @@ function openMuxSocket(httpUrl, signal, protocols) {
   });
 }
 
+/** Audio and file bodies go out as bytes; only plain objects are JSON. */
+function isBinaryBody(body) {
+  return (typeof Blob !== "undefined" && body instanceof Blob)
+    || body instanceof ArrayBuffer
+    || ArrayBuffer.isView(body)
+    || (typeof FormData !== "undefined" && body instanceof FormData);
+}
+
 export async function defaultRequest(url, init = {}) {
+  const raw = init.body === undefined || typeof init.body === "string" || isBinaryBody(init.body);
   const res = await fetch(url, {
     method: init.method ?? "GET",
-    body: init.body === undefined
-      ? undefined
-      : typeof init.body === "string"
-        ? init.body
-        : JSON.stringify(init.body),
+    body: raw ? init.body : JSON.stringify(init.body),
     credentials: "include",
     redirect: "manual",
     signal: init.signal,
     headers: {
-      ...(init.body === undefined ? {} : { "content-type": "application/json" }),
+      ...(init.body === undefined || isBinaryBody(init.body) ? {} : { "content-type": "application/json" }),
       ...init.headers,
     },
   });
@@ -118,14 +195,20 @@ export function createHostClient(ports = {}) {
   const { baseUrl, proxyPrefix } = hostTarget(ports);
   const send = ports.request ?? defaultRequest;
   const urlFor = (path) => hostUrl({ baseUrl, proxyPrefix, path });
-  const request = (path, init) => send(urlFor(path), init);
-  const rpc = (method, payload) => callRpc(request, method, payload);
+  const direct = (url, init) => {
+    if (ports.missing) {
+      return Promise.resolve({ ok: false, status: 0, body: { error: { code: "missing" } } });
+    }
+    return send(url, init);
+  };
+  const request = (path, init) => direct(urlFor(path), init);
+  const rpc = (method, payload) => (
+    method === "workspace.list" ? listWorkspaces(ports) : callRpc(request, method, payload)
+  );
   return {
     urlFor,
     probe() {
-      if (!proxyPrefix && !baseUrl) {
-        return Promise.resolve({ status: "missing" });
-      }
+      if (ports.missing) return Promise.resolve({ status: "missing" });
       return probeHost((path) => request(path), MODELS_PATH);
     },
     rpc,
@@ -168,32 +251,59 @@ export function createHostClient(ports = {}) {
         const code = body?.error?.code || "file_preview_failed";
         throw new Error(code);
       }
+      if (!isPreviewPayload(body)) throw new Error("file_preview_failed");
       return body;
     },
-    mediaUrl(sessionId, path, modifiedAt) {
-      return urlFor(rawFileUrl(sessionId, path, modifiedAt));
+    mediaUrl(sessionId, path, modifiedAt, size) {
+      return urlFor(rawFileUrl(sessionId, path, modifiedAt, size));
     },
     downloadUrl(sessionId, path) {
       return urlFor(downloadFileUrl(sessionId, path));
     },
-    // Both carry a binary body, so neither can go through `request` as an RPC —
-    // but both must still ride the host's auth. `ports.request` is passed
-    // through rather than called here so a host that supplies none keeps the
-    // cookie `fetch` these two used before.
+    // Binary bodies stay off the JSON path. `direct` still uses the host's
+    // auth port, or cookie fetch when none was given, and does not call out
+    // when LarePass has no entrance.
     upload(sessionId, file, options) {
       return uploadFile(file, sessionId, {
         ...options,
         url: urlFor(FILES_UPLOAD_PATH),
-        request: ports.request,
+        request: direct,
       });
     },
     transcribe(blob, language, signal) {
       return postTranscribe(blob, language, signal, {
         baseUrl: urlFor(VOICE_API),
-        request: ports.request,
+        request: direct,
+      });
+    },
+    reportLog(scope, event, fields) {
+      return postReport({ scope, event, ...(fields ?? {}) }, {
+        request,
+        url: urlFor(REPORT_PATH),
       });
     },
+    async speak(text) {
+      const res = await request(`${VOICE_API}/speak`, { method: "POST", body: { text } });
+      const body = res?.body;
+      if (res?.ok && body && typeof body.audio === "string") {
+        return {
+          bluetooth: body.bluetooth === "played" ? "played" : "skipped",
+          audio: body.audio,
+        };
+      }
+      const error = body && typeof body === "object" ? body.error : null;
+      const code = typeof error?.code === "string" && error.code ? error.code : "voice_speak_failed";
+      const message = typeof error?.message === "string" && error.message ? error.message : code;
+      throw Object.assign(new Error(message), { code });
+    },
+    async interruptSpeech() {
+      const res = await request(`${VOICE_API}/playback/stop`, { method: "POST" });
+      return Boolean(res?.ok);
+    },
     async openMux(signal) {
+      if (ports.missing) {
+        return { ok: false, status: "missing", error: { code: "missing", message: "no entrance" } };
+      }
       try {
         return await openMuxSocket(urlFor(MUX_PATH), signal, socketProtocols(ports));
       } catch (err) {

package/src/index.d.ts

@@ -30,6 +30,8 @@ export type HostPorts = {
    * carry `X-Authorization`. Pass a function to have it read per connect.
    */
   socketProtocol?: string | string[] | (() => string | string[] | undefined);
+  /** LarePass sets this when myApps has no Lares app. The entrance page leaves it unset. */
+  missing?: boolean;
 };
 
 export const PC_TEST_PROXY: string;
@@ -50,6 +52,7 @@ export function laresPortsFromAccount(input?: {
   baseUrl?: string;
   env?: HostEnv;
   proxyPrefix?: string;
+  missing?: boolean;
 };
 
 export function defaultRequest(url: string, init?: HostRequestInit): Promise<HostResponse>;
@@ -72,6 +75,7 @@ export const LaresApp: DefineComponent<{
   env?: HostEnv;
   request?: (url: string, init?: HostRequestInit) => Promise<HostResponse>;
   socketProtocol?: HostPorts["socketProtocol"];
+  missing?: boolean;
 }>;
 
 export const LaresAgentSettings: DefineComponent<{
@@ -81,4 +85,5 @@ export const LaresAgentSettings: DefineComponent<{
   proxyPrefix?: string;
   env?: HostEnv;
   request?: (url: string, init?: HostRequestInit) => Promise<HostResponse>;
+  missing?: boolean;
 }>;