@ai-matrx/desktop-protocol 0.10.0 → 0.12.0

package/dist/client.d.cts

@@ -1,8 +1,8 @@
 import { BackoffOptions } from '@ai-matrx/realtime';
 export { RECONNECT_ALARM_ATTEMPTS } from '@ai-matrx/realtime';
 import { z } from 'zod';
-import { W as Welcome, D as DesktopProtocolError, C as CloseOutcome, O as OpName, a as OpParams, b as OpResult, c as OPS, S as StreamMeta, E as EndReason, d as EVENTS, e as ClientType } from './registry-B3NqARgh.cjs';
-export { i as isDesktopProtocolError } from './registry-B3NqARgh.cjs';
+import { W as Welcome, D as DesktopProtocolError, C as CloseOutcome, O as OpName, a as OpParams, b as OpResult, c as OPS, S as StreamMeta, E as EndReason, d as EVENTS, e as ClientType, M as ModelsTranscribeResult, T as TranscriptionModelId, f as MlProgress } from './models-CxAHZMAM.cjs';
+export { i as isDesktopProtocolError } from './models-CxAHZMAM.cjs';
 import { FrameHeader } from './frame.cjs';
 
 /** The subset of the WHATWG WebSocket the client uses — browser, React Native, `ws` and Workers all fit. */
@@ -213,4 +213,44 @@ declare function bindDesktopWake(client: Pick<DesktopClient, "wake">, options?:
     targets?: WakeTargets;
 }): () => void;
 
-export { type ClientTimers, type DesktopClient, type DesktopClientDiagnostic, type DesktopClientOptions, type DesktopClientState, type DesktopClientStatus, DesktopProtocolError, type DesktopStream, type EventHandler, type EventName, type EventPayload, OpName, OpParams, OpResult, type RequestOptions, type StreamHandlers, type StreamOpName, type StreamOptions, type WakeOptions, type WakeTargets, type WebSocketFactory, type WebSocketLike, bindDesktopWake, createDesktopClient };
+/** fs.write carries at most 4 MiB of bytes (base64 in one message). */
+declare const DESKTOP_TRANSCRIBE_MAX_BYTES: number;
+interface DesktopTranscriberOptions {
+    /** Whisper size; omitted = the device's default. */
+    model?: TranscriptionModelId;
+    /** ISO 639-1; omitted = detect. */
+    language?: string;
+    /** Per call (first use installs the runtime and a model): default 15 minutes. */
+    timeoutMs?: number;
+    /** Progress for each call (install steps, then each segment as it is recognised). */
+    onProgress?: (progress: MlProgress) => void;
+}
+/** What a transcription returns: the shape `@ai-matrx/browser-audio`'s TranscriptionTransport expects. */
+interface DesktopTranscript {
+    text: string;
+    language: string;
+    duration: number;
+    segments: Array<{
+        start: number;
+        end: number;
+        text: string;
+    }>;
+    result: ModelsTranscribeResult;
+}
+interface DesktopTranscriber {
+    /** Transcribe recorded audio on the device: written to its temp folder, transcribed, removed. */
+    transcribe(file: Blob, signal?: AbortSignal): Promise<DesktopTranscript>;
+}
+/**
+ * Speech to text on the person's own computer, from any surface connected to it (the app's
+ * screen, the web app or the phone through the relay). Structurally a
+ * `@ai-matrx/browser-audio` TranscriptionTransport, so every voice input on the platform can
+ * transcribe locally: `new BrowserAudioKernel({ transport: createDesktopTranscriber(client) })`.
+ *
+ * Audio travels as one `fs.write` into the device's temp folder (at most 4 MiB; longer
+ * recordings are chunked by the kernel), then `models.transcribe`, then `fs.delete` — every step
+ * an ordinary op, so this works through every transport with no special path.
+ */
+declare function createDesktopTranscriber(client: DesktopClient, options?: DesktopTranscriberOptions): DesktopTranscriber;
+
+export { type ClientTimers, DESKTOP_TRANSCRIBE_MAX_BYTES, type DesktopClient, type DesktopClientDiagnostic, type DesktopClientOptions, type DesktopClientState, type DesktopClientStatus, DesktopProtocolError, type DesktopStream, type DesktopTranscriber, type DesktopTranscriberOptions, type DesktopTranscript, type EventHandler, type EventName, type EventPayload, OpName, OpParams, OpResult, type RequestOptions, type StreamHandlers, type StreamOpName, type StreamOptions, type WakeOptions, type WakeTargets, type WebSocketFactory, type WebSocketLike, bindDesktopWake, createDesktopClient, createDesktopTranscriber };

package/dist/client.d.ts

@@ -1,8 +1,8 @@
 import { BackoffOptions } from '@ai-matrx/realtime';
 export { RECONNECT_ALARM_ATTEMPTS } from '@ai-matrx/realtime';
 import { z } from 'zod';
-import { W as Welcome, D as DesktopProtocolError, C as CloseOutcome, O as OpName, a as OpParams, b as OpResult, c as OPS, S as StreamMeta, E as EndReason, d as EVENTS, e as ClientType } from './registry-B3NqARgh.js';
-export { i as isDesktopProtocolError } from './registry-B3NqARgh.js';
+import { W as Welcome, D as DesktopProtocolError, C as CloseOutcome, O as OpName, a as OpParams, b as OpResult, c as OPS, S as StreamMeta, E as EndReason, d as EVENTS, e as ClientType, M as ModelsTranscribeResult, T as TranscriptionModelId, f as MlProgress } from './models-CxAHZMAM.js';
+export { i as isDesktopProtocolError } from './models-CxAHZMAM.js';
 import { FrameHeader } from './frame.js';
 
 /** The subset of the WHATWG WebSocket the client uses — browser, React Native, `ws` and Workers all fit. */
@@ -213,4 +213,44 @@ declare function bindDesktopWake(client: Pick<DesktopClient, "wake">, options?:
     targets?: WakeTargets;
 }): () => void;
 
-export { type ClientTimers, type DesktopClient, type DesktopClientDiagnostic, type DesktopClientOptions, type DesktopClientState, type DesktopClientStatus, DesktopProtocolError, type DesktopStream, type EventHandler, type EventName, type EventPayload, OpName, OpParams, OpResult, type RequestOptions, type StreamHandlers, type StreamOpName, type StreamOptions, type WakeOptions, type WakeTargets, type WebSocketFactory, type WebSocketLike, bindDesktopWake, createDesktopClient };
+/** fs.write carries at most 4 MiB of bytes (base64 in one message). */
+declare const DESKTOP_TRANSCRIBE_MAX_BYTES: number;
+interface DesktopTranscriberOptions {
+    /** Whisper size; omitted = the device's default. */
+    model?: TranscriptionModelId;
+    /** ISO 639-1; omitted = detect. */
+    language?: string;
+    /** Per call (first use installs the runtime and a model): default 15 minutes. */
+    timeoutMs?: number;
+    /** Progress for each call (install steps, then each segment as it is recognised). */
+    onProgress?: (progress: MlProgress) => void;
+}
+/** What a transcription returns: the shape `@ai-matrx/browser-audio`'s TranscriptionTransport expects. */
+interface DesktopTranscript {
+    text: string;
+    language: string;
+    duration: number;
+    segments: Array<{
+        start: number;
+        end: number;
+        text: string;
+    }>;
+    result: ModelsTranscribeResult;
+}
+interface DesktopTranscriber {
+    /** Transcribe recorded audio on the device: written to its temp folder, transcribed, removed. */
+    transcribe(file: Blob, signal?: AbortSignal): Promise<DesktopTranscript>;
+}
+/**
+ * Speech to text on the person's own computer, from any surface connected to it (the app's
+ * screen, the web app or the phone through the relay). Structurally a
+ * `@ai-matrx/browser-audio` TranscriptionTransport, so every voice input on the platform can
+ * transcribe locally: `new BrowserAudioKernel({ transport: createDesktopTranscriber(client) })`.
+ *
+ * Audio travels as one `fs.write` into the device's temp folder (at most 4 MiB; longer
+ * recordings are chunked by the kernel), then `models.transcribe`, then `fs.delete` — every step
+ * an ordinary op, so this works through every transport with no special path.
+ */
+declare function createDesktopTranscriber(client: DesktopClient, options?: DesktopTranscriberOptions): DesktopTranscriber;
+
+export { type ClientTimers, DESKTOP_TRANSCRIBE_MAX_BYTES, type DesktopClient, type DesktopClientDiagnostic, type DesktopClientOptions, type DesktopClientState, type DesktopClientStatus, DesktopProtocolError, type DesktopStream, type DesktopTranscriber, type DesktopTranscriberOptions, type DesktopTranscript, type EventHandler, type EventName, type EventPayload, OpName, OpParams, OpResult, type RequestOptions, type StreamHandlers, type StreamOpName, type StreamOptions, type WakeOptions, type WakeTargets, type WebSocketFactory, type WebSocketLike, bindDesktopWake, createDesktopClient, createDesktopTranscriber };

package/dist/client.js

@@ -1639,11 +1639,68 @@ function bindDesktopWake(client, options = {}) {
     targets.window.removeEventListener("focus", onWake);
   };
 }
+
+// src/client/transcribe.ts
+var DESKTOP_TRANSCRIBE_MAX_BYTES = 4 * 1024 * 1024;
+var EXT = {
+  "audio/webm": "webm",
+  "audio/ogg": "ogg",
+  "audio/mp4": "m4a",
+  "audio/mpeg": "mp3",
+  "audio/wav": "wav",
+  "audio/x-wav": "wav",
+  "video/webm": "webm",
+  "video/mp4": "mp4"
+};
+function base64(bytes) {
+  let binary = "";
+  for (let i = 0; i < bytes.length; i += 32768) binary += String.fromCharCode(...bytes.subarray(i, i + 32768));
+  return btoa(binary);
+}
+function createDesktopTranscriber(client, options = {}) {
+  let temp = null;
+  const tempDir = () => temp ??= client.request("sysinfo.get", {}).then((s) => s.paths.temp.replace(/\/$/, ""));
+  return {
+    async transcribe(file, signal) {
+      if (file.size > DESKTOP_TRANSCRIBE_MAX_BYTES) {
+        throw new DesktopProtocolError("PAYLOAD_TOO_LARGE", "Recording is larger than 4 MB", { retryable: false, data: { limit_bytes: DESKTOP_TRANSCRIBE_MAX_BYTES } });
+      }
+      const type = file.type.split(";")[0]?.trim() ?? "";
+      const name = `matrx-transcribe-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}.${EXT[type] ?? "webm"}`;
+      const path = `${await tempDir()}/${name}`;
+      const bytes = new Uint8Array(await file.arrayBuffer());
+      await client.request("fs.write", { path, content: base64(bytes), encoding: "base64", mode: 384 }, signal ? { signal } : {});
+      let requestId = "";
+      const off = options.onProgress ? client.on("models.progress", (payload) => {
+        if (payload.request_id === requestId) options.onProgress?.(payload.progress);
+      }) : () => void 0;
+      try {
+        const result = await client.request(
+          "models.transcribe",
+          { path, ...options.model ? { model: options.model } : {}, ...options.language ? { language: options.language } : {} },
+          { timeoutMs: options.timeoutMs ?? 15 * 6e4, onRequestId: (id) => requestId = id, ...signal ? { signal } : {} }
+        );
+        return {
+          text: result.text,
+          language: result.language,
+          duration: result.duration_s,
+          segments: result.segments.map((s) => ({ start: s.start_s, end: s.end_s, text: s.text })),
+          result
+        };
+      } finally {
+        off();
+        void client.request("fs.delete", { path, to_trash: false }).catch(() => void 0);
+      }
+    }
+  };
+}
 export {
+  DESKTOP_TRANSCRIBE_MAX_BYTES,
   DesktopProtocolError,
   RECONNECT_ALARM_ATTEMPTS,
   bindDesktopWake,
   createDesktopClient,
+  createDesktopTranscriber,
   isDesktopProtocolError
 };
 //# sourceMappingURL=client.js.map