@pyai/sdk 0.2.2 → 0.2.3

package/dist/index.d.ts

@@ -420,6 +420,17 @@ export interface OmniServerFrame {
     event: string;
     [k: string]: unknown;
 }
+/** Canonical, sanitized transcript delivered by the native Omni demux. */
+export interface OmniTranscriptFrame extends OmniServerFrame {
+    event: "transcript";
+    role: "user" | "assistant";
+    text: string;
+    final: boolean;
+    mode: "delta" | "replace";
+    sequence?: number;
+}
+/** Normalize the live UTF-8 0x02 body and the documented direct-object legacy shape. */
+export declare function normalizeOmniTranscriptBody(bytes: Uint8Array): OmniTranscriptFrame | null;
 /** A binary agent-audio chunk delivered to {@link OmniConnectOptions.onAudio}. */
 export type OmniAudioChunk = ArrayBuffer | ArrayBufferView | Blob;
 export interface OmniToolDef {
@@ -539,6 +550,8 @@ export declare class OmniConnection {
     private closed;
     /** Serializes async Blob reads in {@link sendAudio} so frames stay ordered. */
     private blobTail;
+    /** Serializes inbound Blob decoding so browser frames stay ordered. */
+    private inboundTail;
     constructor(url: string, subprotocol: string, opts: OmniConnectOptions);
     private handleMessage;
     private dispatchFrame;

package/dist/index.js

@@ -272,6 +272,71 @@ export const OmniEvent = {
     /** Server fault frame. */
     Error: "error",
 };
+const OMNI_TRANSCRIPT_MAX_BYTES = 16_384;
+const OMNI_TRANSCRIPT_MAX_CHARS = 4_000;
+function omniTranscriptRole(value) {
+    if (value === "user" || value === "caller" || value === "human")
+        return "user";
+    if (value === "assistant" || value === "agent")
+        return "assistant";
+    return null;
+}
+function omniTranscriptText(value) {
+    return typeof value === "string"
+        && value.length > 0
+        && value.length <= OMNI_TRANSCRIPT_MAX_CHARS
+        && !/[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/.test(value)
+        ? value
+        : null;
+}
+/** Normalize the live UTF-8 0x02 body and the documented direct-object legacy shape. */
+export function normalizeOmniTranscriptBody(bytes) {
+    if (bytes.byteLength === 0 || bytes.byteLength > OMNI_TRANSCRIPT_MAX_BYTES)
+        return null;
+    let decoded;
+    try {
+        decoded = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
+    }
+    catch {
+        return null;
+    }
+    if (!decoded.trimStart().startsWith("{")) {
+        const text = omniTranscriptText(decoded);
+        return text
+            ? { event: "transcript", role: "user", text, final: false, mode: "delta" }
+            : null;
+    }
+    let value;
+    try {
+        value = JSON.parse(decoded);
+    }
+    catch {
+        return null;
+    }
+    if (!value || typeof value !== "object" || Array.isArray(value))
+        return null;
+    const payload = value;
+    const role = omniTranscriptRole(payload.role ?? payload.speaker);
+    const mode = typeof payload.delta === "string" ? "delta" : "replace";
+    const text = omniTranscriptText(mode === "delta" ? payload.delta : typeof payload.text === "string" ? payload.text : payload.transcript);
+    if (!role || !text)
+        return null;
+    if (payload.final !== undefined && typeof payload.final !== "boolean")
+        return null;
+    if (payload.sequence !== undefined
+        && (typeof payload.sequence !== "number"
+            || !Number.isSafeInteger(payload.sequence)
+            || payload.sequence < 0))
+        return null;
+    return {
+        event: "transcript",
+        role,
+        text,
+        final: payload.final === true,
+        mode,
+        ...(payload.sequence === undefined ? {} : { sequence: payload.sequence }),
+    };
+}
 /** Extract a byte view from a binary WS frame (Buffer / ArrayBuffer / typed
  * array). Returns null for a Blob or unknown (can't be read synchronously). */
 function omniToBytes(data) {
@@ -324,6 +389,8 @@ export class OmniConnection {
     closed = false;
     /** Serializes async Blob reads in {@link sendAudio} so frames stay ordered. */
     blobTail = Promise.resolve();
+    /** Serializes inbound Blob decoding so browser frames stay ordered. */
+    inboundTail = Promise.resolve();
     constructor(url, subprotocol, opts) {
         this.opts = opts;
         const WS = opts.webSocket ?? globalThis.WebSocket;
@@ -342,22 +409,34 @@ export class OmniConnection {
             }
             opts.onOpen?.();
         };
-        this.ws.onmessage = (ev) => this.handleMessage(ev.data);
+        this.ws.onmessage = (ev) => {
+            const reportDecodeError = (error) => {
+                opts.onError?.(error instanceof Error ? error : new Error("Could not decode Omni frame"));
+            };
+            if (typeof Blob !== "undefined" && ev.data instanceof Blob) {
+                this.inboundTail = this.inboundTail.then(() => this.handleMessage(ev.data)).catch(reportDecodeError);
+            }
+            else {
+                void this.handleMessage(ev.data).catch(reportDecodeError);
+            }
+        };
         this.ws.onerror = (ev) => opts.onError?.(ev instanceof Error ? ev : new Error("WebSocket error"));
         this.ws.onclose = (ev) => {
             this.closed = true;
             opts.onClose?.(ev.code, ev.reason);
         };
     }
-    handleMessage(data) {
+    async handleMessage(data) {
         // Server → client binary frames are TYPE-TAGGED by their first byte:
-        // 0x01 = agent audio (PCM16) · 0x02 = transcript JSON · 0x03 = control JSON.
+        // 0x01 = agent audio (PCM16) · 0x02 = transcript UTF-8 · 0x03 = control JSON.
         // (Treating every binary frame as audio, the old behavior, plays the
         // 0x03/0x02 frames as a glitch and drops every event/transcript.)
         if (typeof data !== "string") {
-            const bytes = omniToBytes(data);
+            const bytes = typeof Blob !== "undefined" && data instanceof Blob
+                ? new Uint8Array(await data.arrayBuffer())
+                : omniToBytes(data);
             if (!bytes) {
-                this.opts.onAudio?.(data); // Blob/unknown, best-effort
+                this.opts.onError?.(new Error("Unsupported Omni binary frame"));
                 return;
             }
             const tag = bytes[0];
@@ -365,18 +444,26 @@ export class OmniConnection {
                 this.opts.onAudio?.(bytes.slice(1)); // copy → aligned PCM16
                 return;
             }
-            if (tag === 0x02 || tag === 0x03) {
+            if (tag === 0x02) {
+                const transcript = normalizeOmniTranscriptBody(bytes.subarray(1));
+                if (transcript)
+                    this.dispatchFrame(transcript);
+                else
+                    this.opts.onError?.(new Error("Unparseable Omni transcript frame"));
+                return;
+            }
+            if (tag === 0x03) {
                 try {
                     const parsed = JSON.parse(new TextDecoder().decode(bytes.subarray(1)));
-                    // The 0x02 tag is authoritative for transcript even if the JSON omits `event`.
-                    this.dispatchFrame(tag === 0x02 ? { ...parsed, event: "transcript" } : parsed);
+                    this.dispatchFrame(parsed);
                 }
                 catch {
                     this.opts.onError?.(new Error("Unparseable Omni binary frame"));
                 }
                 return;
             }
-            this.opts.onAudio?.(data); // untagged, forward-compat as audio
+            const tagName = tag === undefined ? "empty" : `0x${tag.toString(16).padStart(2, "0")}`;
+            this.opts.onError?.(new Error(`Ignored unknown Omni binary frame tag ${tagName}`));
             return;
         }
         // Text frame (e.g. a server-side broker relay).

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pyai/sdk",
-  "version": "0.2.2",
+  "version": "0.2.3",
   "description": "Official TypeScript/JavaScript SDK for PyAI, speech-to-text (Hear), text-to-speech (Speak), realtime voice agents (Omni), and call compliance (Trace).",
   "type": "module",
   "main": "dist/index.js",

package/src/index.ts

@@ -586,6 +586,78 @@ export interface OmniServerFrame {
   [k: string]: unknown;
 }
 
+/** Canonical, sanitized transcript delivered by the native Omni demux. */
+export interface OmniTranscriptFrame extends OmniServerFrame {
+  event: "transcript";
+  role: "user" | "assistant";
+  text: string;
+  final: boolean;
+  mode: "delta" | "replace";
+  sequence?: number;
+}
+
+const OMNI_TRANSCRIPT_MAX_BYTES = 16_384;
+const OMNI_TRANSCRIPT_MAX_CHARS = 4_000;
+
+function omniTranscriptRole(value: unknown): "user" | "assistant" | null {
+  if (value === "user" || value === "caller" || value === "human") return "user";
+  if (value === "assistant" || value === "agent") return "assistant";
+  return null;
+}
+
+function omniTranscriptText(value: unknown): string | null {
+  return typeof value === "string"
+    && value.length > 0
+    && value.length <= OMNI_TRANSCRIPT_MAX_CHARS
+    && !/[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/.test(value)
+    ? value
+    : null;
+}
+
+/** Normalize the live UTF-8 0x02 body and the documented direct-object legacy shape. */
+export function normalizeOmniTranscriptBody(bytes: Uint8Array): OmniTranscriptFrame | null {
+  if (bytes.byteLength === 0 || bytes.byteLength > OMNI_TRANSCRIPT_MAX_BYTES) return null;
+  let decoded: string;
+  try {
+    decoded = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
+  } catch {
+    return null;
+  }
+  if (!decoded.trimStart().startsWith("{")) {
+    const text = omniTranscriptText(decoded);
+    return text
+      ? { event: "transcript", role: "user", text, final: false, mode: "delta" }
+      : null;
+  }
+  let value: unknown;
+  try {
+    value = JSON.parse(decoded);
+  } catch {
+    return null;
+  }
+  if (!value || typeof value !== "object" || Array.isArray(value)) return null;
+  const payload = value as Record<string, unknown>;
+  const role = omniTranscriptRole(payload.role ?? payload.speaker);
+  const mode = typeof payload.delta === "string" ? "delta" : "replace";
+  const text = omniTranscriptText(
+    mode === "delta" ? payload.delta : typeof payload.text === "string" ? payload.text : payload.transcript,
+  );
+  if (!role || !text) return null;
+  if (payload.final !== undefined && typeof payload.final !== "boolean") return null;
+  if (payload.sequence !== undefined
+    && (typeof payload.sequence !== "number"
+      || !Number.isSafeInteger(payload.sequence)
+      || payload.sequence < 0)) return null;
+  return {
+    event: "transcript",
+    role,
+    text,
+    final: payload.final === true,
+    mode,
+    ...(payload.sequence === undefined ? {} : { sequence: payload.sequence as number }),
+  };
+}
+
 /** A binary agent-audio chunk delivered to {@link OmniConnectOptions.onAudio}. */
 export type OmniAudioChunk = ArrayBuffer | ArrayBufferView | Blob;
 
@@ -741,6 +813,8 @@ export class OmniConnection {
   private closed = false;
   /** Serializes async Blob reads in {@link sendAudio} so frames stay ordered. */
   private blobTail: Promise<void> = Promise.resolve();
+  /** Serializes inbound Blob decoding so browser frames stay ordered. */
+  private inboundTail: Promise<void> = Promise.resolve();
 
   constructor(url: string, subprotocol: string, opts: OmniConnectOptions) {
     this.opts = opts;
@@ -761,7 +835,16 @@ export class OmniConnection {
       }
       opts.onOpen?.();
     };
-    this.ws.onmessage = (ev) => this.handleMessage(ev.data);
+    this.ws.onmessage = (ev) => {
+      const reportDecodeError = (error: unknown) => {
+        opts.onError?.(error instanceof Error ? error : new Error("Could not decode Omni frame"));
+      };
+      if (typeof Blob !== "undefined" && ev.data instanceof Blob) {
+        this.inboundTail = this.inboundTail.then(() => this.handleMessage(ev.data)).catch(reportDecodeError);
+      } else {
+        void this.handleMessage(ev.data).catch(reportDecodeError);
+      }
+    };
     this.ws.onerror = (ev) => opts.onError?.(ev instanceof Error ? ev : new Error("WebSocket error"));
     this.ws.onclose = (ev) => {
       this.closed = true;
@@ -769,15 +852,17 @@ export class OmniConnection {
     };
   }
 
-  private handleMessage(data: unknown): void {
+  private async handleMessage(data: unknown): Promise<void> {
     // Server → client binary frames are TYPE-TAGGED by their first byte:
-    // 0x01 = agent audio (PCM16) · 0x02 = transcript JSON · 0x03 = control JSON.
+    // 0x01 = agent audio (PCM16) · 0x02 = transcript UTF-8 · 0x03 = control JSON.
     // (Treating every binary frame as audio, the old behavior, plays the
     // 0x03/0x02 frames as a glitch and drops every event/transcript.)
     if (typeof data !== "string") {
-      const bytes = omniToBytes(data);
+      const bytes = typeof Blob !== "undefined" && data instanceof Blob
+        ? new Uint8Array(await data.arrayBuffer())
+        : omniToBytes(data);
       if (!bytes) {
-        this.opts.onAudio?.(data as OmniAudioChunk); // Blob/unknown, best-effort
+        this.opts.onError?.(new Error("Unsupported Omni binary frame"));
         return;
       }
       const tag = bytes[0];
@@ -785,17 +870,23 @@ export class OmniConnection {
         this.opts.onAudio?.(bytes.slice(1) as OmniAudioChunk); // copy → aligned PCM16
         return;
       }
-      if (tag === 0x02 || tag === 0x03) {
+      if (tag === 0x02) {
+        const transcript = normalizeOmniTranscriptBody(bytes.subarray(1));
+        if (transcript) this.dispatchFrame(transcript);
+        else this.opts.onError?.(new Error("Unparseable Omni transcript frame"));
+        return;
+      }
+      if (tag === 0x03) {
         try {
           const parsed = JSON.parse(new TextDecoder().decode(bytes.subarray(1))) as OmniServerFrame;
-          // The 0x02 tag is authoritative for transcript even if the JSON omits `event`.
-          this.dispatchFrame(tag === 0x02 ? { ...parsed, event: "transcript" } : parsed);
+          this.dispatchFrame(parsed);
         } catch {
           this.opts.onError?.(new Error("Unparseable Omni binary frame"));
         }
         return;
       }
-      this.opts.onAudio?.(data as OmniAudioChunk); // untagged, forward-compat as audio
+      const tagName = tag === undefined ? "empty" : `0x${tag.toString(16).padStart(2, "0")}`;
+      this.opts.onError?.(new Error(`Ignored unknown Omni binary frame tag ${tagName}`));
       return;
     }
     // Text frame (e.g. a server-side broker relay).