@ai-matrx/desktop-protocol 0.10.0 → 0.12.0

package/CHANGELOG.md

@@ -1,5 +1,24 @@
 # Changelog
 
+## 0.12.0
+
+- `TOOL_CATALOG`: `desktop_transcribe` is built — one op, `models.transcribe`, reached through
+  aidream's generic `POST /api/local-proxy/{device}/op/{cap.op}`. matrx-local's `speech_locales`
+  needs no op of its own (Whisper detects the language).
+
+Consumer action: none.
+
+## 0.11.0
+
+- `createDesktopTranscriber(client, options)` (`./client`): speech to text on the person's own
+  computer from any connected surface — the app's screen, the web app or the phone through the
+  relay. It writes the audio into the device's temp folder (`fs.write`, at most 4 MiB:
+  `DESKTOP_TRANSCRIBE_MAX_BYTES`), calls `models.transcribe`, removes the file, and reports that
+  call's `models.progress` (install steps, then each segment). Its return is structurally a
+  `@ai-matrx/browser-audio` `TranscriptionTransport`, so any voice input can transcribe locally.
+
+Consumer action: none.
+
 ## 0.10.0
 
 Protocol 1.2 adds the `apps` capability (SPEC §6): the desktop apps a person uses (Claude, Codex,

package/README.md

@@ -63,6 +63,16 @@ term.write("ls\n");
 - Background problems (malformed messages, sequence gaps, heartbeat death, reconnect alarm) go to
   `onDiagnostic` — loud by default (`console.warn`).
 
+## Local transcription
+
+```ts
+import { createDesktopTranscriber } from "@ai-matrx/desktop-protocol/client";
+
+const transcriber = createDesktopTranscriber(client, { model: "base", onProgress: (p) => show(p) });
+const { text, segments } = await transcriber.transcribe(recordingBlob); // on the person's computer
+// Or as a voice input's transport: new BrowserAudioKernel({ transport: transcriber })
+```
+
 ## React
 
 ```tsx

package/dist/client.cjs

@@ -20,10 +20,12 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 // src/client.ts
 var client_exports = {};
 __export(client_exports, {
+  DESKTOP_TRANSCRIBE_MAX_BYTES: () => DESKTOP_TRANSCRIBE_MAX_BYTES,
   DesktopProtocolError: () => DesktopProtocolError,
   RECONNECT_ALARM_ATTEMPTS: () => import_realtime.RECONNECT_ALARM_ATTEMPTS,
   bindDesktopWake: () => bindDesktopWake,
   createDesktopClient: () => createDesktopClient,
+  createDesktopTranscriber: () => createDesktopTranscriber,
   isDesktopProtocolError: () => isDesktopProtocolError
 });
 module.exports = __toCommonJS(client_exports);
@@ -1669,4 +1671,59 @@ function bindDesktopWake(client, options = {}) {
     targets.window.removeEventListener("focus", onWake);
   };
 }
+
+// src/client/transcribe.ts
+var DESKTOP_TRANSCRIBE_MAX_BYTES = 4 * 1024 * 1024;
+var EXT = {
+  "audio/webm": "webm",
+  "audio/ogg": "ogg",
+  "audio/mp4": "m4a",
+  "audio/mpeg": "mp3",
+  "audio/wav": "wav",
+  "audio/x-wav": "wav",
+  "video/webm": "webm",
+  "video/mp4": "mp4"
+};
+function base64(bytes) {
+  let binary = "";
+  for (let i = 0; i < bytes.length; i += 32768) binary += String.fromCharCode(...bytes.subarray(i, i + 32768));
+  return btoa(binary);
+}
+function createDesktopTranscriber(client, options = {}) {
+  let temp = null;
+  const tempDir = () => temp ??= client.request("sysinfo.get", {}).then((s) => s.paths.temp.replace(/\/$/, ""));
+  return {
+    async transcribe(file, signal) {
+      if (file.size > DESKTOP_TRANSCRIBE_MAX_BYTES) {
+        throw new DesktopProtocolError("PAYLOAD_TOO_LARGE", "Recording is larger than 4 MB", { retryable: false, data: { limit_bytes: DESKTOP_TRANSCRIBE_MAX_BYTES } });
+      }
+      const type = file.type.split(";")[0]?.trim() ?? "";
+      const name = `matrx-transcribe-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}.${EXT[type] ?? "webm"}`;
+      const path = `${await tempDir()}/${name}`;
+      const bytes = new Uint8Array(await file.arrayBuffer());
+      await client.request("fs.write", { path, content: base64(bytes), encoding: "base64", mode: 384 }, signal ? { signal } : {});
+      let requestId = "";
+      const off = options.onProgress ? client.on("models.progress", (payload) => {
+        if (payload.request_id === requestId) options.onProgress?.(payload.progress);
+      }) : () => void 0;
+      try {
+        const result = await client.request(
+          "models.transcribe",
+          { path, ...options.model ? { model: options.model } : {}, ...options.language ? { language: options.language } : {} },
+          { timeoutMs: options.timeoutMs ?? 15 * 6e4, onRequestId: (id) => requestId = id, ...signal ? { signal } : {} }
+        );
+        return {
+          text: result.text,
+          language: result.language,
+          duration: result.duration_s,
+          segments: result.segments.map((s) => ({ start: s.start_s, end: s.end_s, text: s.text })),
+          result
+        };
+      } finally {
+        off();
+        void client.request("fs.delete", { path, to_trash: false }).catch(() => void 0);
+      }
+    }
+  };
+}
 //# sourceMappingURL=client.cjs.map