@xmov/avatar 2.0.0-alpha.38

package/src/agent/audio-debug.ts

@@ -0,0 +1,36 @@
+const DEFAULT_AUDIO_CHUNK_PREVIEW_BYTES = 64;
+
+type AudioChunkDebugMetadata = Record<string, unknown>;
+
+function toHex(byte: number) {
+  return byte.toString(16).padStart(2, "0");
+}
+
+export async function logAudioChunkContent(
+  label: string,
+  chunk: Blob | ArrayBuffer | ArrayBufferView,
+  metadata: AudioChunkDebugMetadata = {},
+  previewBytes = DEFAULT_AUDIO_CHUNK_PREVIEW_BYTES,
+) {
+  try {
+    const buffer = chunk instanceof Blob
+      ? await chunk.arrayBuffer()
+      : ArrayBuffer.isView(chunk)
+        ? chunk.buffer.slice(chunk.byteOffset, chunk.byteOffset + chunk.byteLength)
+        : chunk;
+    const bytes = new Uint8Array(buffer);
+    const preview = Array.from(bytes.slice(0, previewBytes));
+    const type = chunk instanceof Blob ? chunk.type : metadata.type;
+
+    console.log(label, {
+      ...metadata,
+      size: bytes.byteLength,
+      type,
+      previewLength: preview.length,
+      bytes: preview,
+      hex: preview.map(toHex).join(" "),
+    });
+  } catch (error) {
+    console.warn(label, "failed to read audio chunk", error);
+  }
+}

package/src/agent/audio-uplink.ts

@@ -0,0 +1,392 @@
+const AU_HEADER_BYTES = 8;
+const AU_VERSION = 1;
+const AUDIO_BLOCK_MS = 100;
+const MAX_FAREND_BUCKET_SKEW = 1;
+const MAX_PENDING_FAREND_BUCKETS = 3;
+const NEAREND_GRACE_BUCKETS = 1;
+const WEBM_EBML_MAGIC = new Uint8Array([0x1a, 0x45, 0xdf, 0xa3]);
+
+export const AU_STREAM_NEAREND = 1 as const;
+export const AU_STREAM_FAREND = 2 as const;
+
+export type AuStreamId = typeof AU_STREAM_NEAREND | typeof AU_STREAM_FAREND;
+export type AuPayload = Blob | ArrayBuffer | ArrayBufferView;
+
+export interface AgentAudioBlock {
+  data: AuPayload;
+  /** Epoch timestamp for the beginning of this 100 ms media block. */
+  timestamp: number;
+  /** Whether this block starts a new WebM container stream. */
+  isFirstChunk?: boolean;
+}
+
+interface PendingAudioBlock extends AgentAudioBlock {
+  streamId: AuStreamId;
+  tsMs: number;
+  frameKey: number;
+  dataParts?: AuPayload[];
+}
+
+interface AgentAudioUplinkOptions {
+  sendBinary(frame: ArrayBuffer): void;
+  onError?(error: Error): void;
+}
+
+function logAudioUplink(event: string, details: Record<string, unknown>) {
+  const logger = (globalThis as typeof globalThis & {
+    avatarSDKLogger?: { log?: (...args: unknown[]) => void };
+  }).avatarSDKLogger;
+  logger?.log?.("[Agent][AudioUplink]", event, details);
+}
+
+export function packAuAudioFrame(
+  streamId: AuStreamId,
+  tsMs: number,
+  payload: ArrayBuffer | ArrayBufferView,
+): ArrayBuffer {
+  if (streamId !== AU_STREAM_NEAREND && streamId !== AU_STREAM_FAREND) {
+    throw new Error(`AU stream_id 无效: ${streamId}`);
+  }
+  if (!Number.isFinite(tsMs) || tsMs < 0 || tsMs > 0xffffffff) {
+    throw new Error(`AU ts_ms 无效: ${tsMs}`);
+  }
+
+  const body = payload instanceof ArrayBuffer
+    ? new Uint8Array(payload)
+    : new Uint8Array(payload.buffer, payload.byteOffset, payload.byteLength);
+  const frame = new ArrayBuffer(AU_HEADER_BYTES + body.byteLength);
+  const view = new DataView(frame);
+  view.setUint8(0, 0x41);
+  view.setUint8(1, 0x55);
+  view.setUint8(2, AU_VERSION);
+  view.setUint8(3, streamId);
+  view.setUint32(4, Math.floor(tsMs), false);
+  new Uint8Array(frame, AU_HEADER_BYTES).set(body);
+  return frame;
+}
+
+async function toPayloadBytes(payload: AuPayload | AuPayload[]): Promise<Uint8Array> {
+  if (Array.isArray(payload)) {
+    const chunks = await Promise.all(payload.map(toPayloadBytes));
+    const merged = new Uint8Array(chunks.reduce((size, chunk) => size + chunk.byteLength, 0));
+    let offset = 0;
+    for (const chunk of chunks) {
+      merged.set(chunk, offset);
+      offset += chunk.byteLength;
+    }
+    return merged;
+  }
+  if (payload instanceof Blob) {
+    return new Uint8Array(await payload.arrayBuffer());
+  }
+  if (payload instanceof ArrayBuffer) {
+    return new Uint8Array(payload);
+  }
+  return new Uint8Array(payload.buffer, payload.byteOffset, payload.byteLength);
+}
+
+/**
+ * Owns the connection-level AU timeline and serializes nearend/farend writes.
+ * Nearend/farend are matched by 100 ms bucket. When independent capture clocks
+ * straddle a bucket boundary, an aged nearend may consume farend from
+ * the immediately preceding bucket. This keeps farend-before-nearend wire
+ * order without uploading farend-only buckets or dropping WebM continuations.
+ */
+export class AgentAudioUplink {
+  private readonly options: AgentAudioUplinkOptions;
+  private connectionStartedAt: number | null = null;
+  private connectionGeneration = 0;
+  private sendChain: Promise<void> = Promise.resolve();
+  private failed = false;
+  private readonly pendingFarend = new Map<number, PendingAudioBlock[]>();
+  private readonly pendingNearend = new Map<number, PendingAudioBlock[]>();
+  private readonly closedNearendBuckets = new Set<number>();
+  private readonly lastTimestampByStream = new Map<AuStreamId, number>();
+  private readonly lastSentTimestampByStream = new Map<AuStreamId, number>();
+  private readonly pendingWebmPrefixByStream = new Map<AuStreamId, Uint8Array>();
+  private farendStreamState: "unknown" | "ready" | "awaiting-header" = "unknown";
+
+  constructor(options: AgentAudioUplinkOptions) {
+    this.options = options;
+  }
+
+  reset(connectionStartedAt: number | null = null) {
+    this.connectionGeneration += 1;
+    this.connectionStartedAt = connectionStartedAt;
+    this.pendingFarend.clear();
+    this.pendingNearend.clear();
+    this.closedNearendBuckets.clear();
+    this.lastTimestampByStream.clear();
+    this.lastSentTimestampByStream.clear();
+    this.pendingWebmPrefixByStream.clear();
+    this.farendStreamState = "unknown";
+    this.failed = false;
+    this.sendChain = Promise.resolve();
+  }
+
+  startInput() {
+    this.failed = false;
+    this.pendingFarend.clear();
+    this.pendingNearend.clear();
+    this.closedNearendBuckets.clear();
+    this.pendingWebmPrefixByStream.clear();
+    this.farendStreamState = "unknown";
+  }
+
+  clearFarend() {
+    this.pendingFarend.clear();
+    this.pendingWebmPrefixByStream.delete(AU_STREAM_FAREND);
+    this.farendStreamState = "unknown";
+  }
+
+  offerFarend(block: AgentAudioBlock) {
+    if (block.isFirstChunk === false && this.farendStreamState !== "ready") {
+      logAudioUplink("farend.drop", {
+        reason: "awaiting_webm_header",
+        timestamp: block.timestamp,
+        isFirstChunk: false,
+      });
+      return;
+    }
+
+    const normalized = this.normalizeBlock(AU_STREAM_FAREND, block);
+    if (!normalized) {
+      if (block.isFirstChunk) {
+        this.abandonFarendStream();
+      }
+      return;
+    }
+
+    if (block.isFirstChunk) {
+      this.pendingFarend.clear();
+      this.pendingWebmPrefixByStream.delete(AU_STREAM_FAREND);
+      this.farendStreamState = "ready";
+    }
+
+    const bucket = this.pendingFarend.get(normalized.frameKey) ?? [];
+    bucket.push(normalized);
+    this.pendingFarend.set(normalized.frameKey, bucket);
+    logAudioUplink("farend.offer", {
+      tsMs: normalized.tsMs,
+      frameKey: normalized.frameKey,
+      isFirstChunk: Boolean(normalized.isFirstChunk),
+      pendingNearend: [...this.pendingNearend.keys()],
+    });
+    this.flushOlderNearend(normalized.frameKey);
+    this.prune(normalized.frameKey);
+  }
+
+  offerNearend(block: AgentAudioBlock) {
+    const nearend = this.normalizeBlock(AU_STREAM_NEAREND, block);
+    if (!nearend || this.closedNearendBuckets.has(nearend.frameKey)) {
+      return;
+    }
+
+    const bucket = this.pendingNearend.get(nearend.frameKey) ?? [];
+    bucket.push(nearend);
+    this.pendingNearend.set(nearend.frameKey, bucket);
+    this.flushExpiredNearend(nearend.frameKey);
+    this.prune(nearend.frameKey);
+  }
+
+  drain() {
+    for (const frameKey of [...this.pendingNearend.keys()].sort((a, b) => a - b)) {
+      this.flushBucket(frameKey, true);
+    }
+    return this.sendChain;
+  }
+
+  private normalizeBlock(streamId: AuStreamId, block: AgentAudioBlock): PendingAudioBlock | null {
+    const connectionStartedAt = this.connectionStartedAt;
+    if (connectionStartedAt === null || !Number.isFinite(block.timestamp)) {
+      return null;
+    }
+
+    const relativeTimestamp = Math.max(0, Math.floor(block.timestamp - connectionStartedAt));
+    const frameKey = Math.floor(relativeTimestamp / AUDIO_BLOCK_MS);
+    const lastTimestamp = this.lastTimestampByStream.get(streamId);
+    if (lastTimestamp !== undefined && relativeTimestamp <= lastTimestamp) {
+      return null;
+    }
+    this.lastTimestampByStream.set(streamId, relativeTimestamp);
+    return { ...block, streamId, tsMs: relativeTimestamp, frameKey };
+  }
+
+  private enqueue(blocks: PendingAudioBlock[]) {
+    if (this.failed) {
+      return;
+    }
+    const generation = this.connectionGeneration;
+    this.sendChain = this.sendChain
+      .then(async () => {
+        if (this.failed) {
+          return;
+        }
+        for (const block of blocks) {
+          const bytes = await toPayloadBytes(block.dataParts ?? block.data);
+          if (
+            this.failed
+            || generation !== this.connectionGeneration
+            || this.connectionStartedAt === null
+            || bytes.byteLength === 0
+          ) {
+            continue;
+          }
+          if (block.streamId === AU_STREAM_FAREND && block.isFirstChunk) {
+            this.pendingWebmPrefixByStream.delete(AU_STREAM_FAREND);
+          }
+          const payload = this.coalesceSplitWebmMagic(block.streamId, bytes);
+          if (!payload) {
+            continue;
+          }
+          const lastSentTimestamp = this.lastSentTimestampByStream.get(block.streamId);
+          if (lastSentTimestamp !== undefined && block.tsMs <= lastSentTimestamp) {
+            continue;
+          }
+          this.options.sendBinary(packAuAudioFrame(block.streamId, block.tsMs, payload));
+          this.lastSentTimestampByStream.set(block.streamId, block.tsMs);
+          logAudioUplink("frame.send", {
+            streamId: block.streamId,
+            tsMs: block.tsMs,
+            bytes: payload.byteLength,
+          });
+        }
+      })
+      .catch((error) => {
+        if (generation !== this.connectionGeneration) {
+          return;
+        }
+        this.failed = true;
+        this.pendingFarend.clear();
+        this.pendingNearend.clear();
+        const normalized = error instanceof Error ? error : new Error("AU 音频上行失败");
+        this.options.onError?.(normalized);
+      });
+  }
+
+  /**
+   * Chromium may emit the first WebM byte (0x1a) as its own MediaRecorder
+   * Blob and put the remaining EBML magic in the next Blob. A one-byte AU
+   * payload is not a valid timed audio block, so keep an incomplete WebM
+   * magic prefix and prepend it to the next block from the same stream.
+   */
+  private coalesceSplitWebmMagic(streamId: AuStreamId, payload: Uint8Array) {
+    const pendingPrefix = this.pendingWebmPrefixByStream.get(streamId);
+    if (pendingPrefix) {
+      const merged = new Uint8Array(pendingPrefix.byteLength + payload.byteLength);
+      merged.set(pendingPrefix);
+      merged.set(payload, pendingPrefix.byteLength);
+      this.pendingWebmPrefixByStream.delete(streamId);
+      payload = merged;
+    }
+
+    if (
+      payload.byteLength < WEBM_EBML_MAGIC.byteLength
+      && payload.every((byte, index) => byte === WEBM_EBML_MAGIC[index])
+    ) {
+      this.pendingWebmPrefixByStream.set(streamId, payload.slice());
+      return null;
+    }
+
+    return payload;
+  }
+
+  private flushOlderNearend(currentFrameKey: number) {
+    for (const frameKey of [...this.pendingNearend.keys()].sort((a, b) => a - b)) {
+      if (frameKey >= currentFrameKey) {
+        break;
+      }
+      this.flushBucket(frameKey, true);
+    }
+  }
+
+  private flushExpiredNearend(currentFrameKey: number) {
+    this.flushOlderNearend(currentFrameKey - NEAREND_GRACE_BUCKETS);
+  }
+
+  private flushBucket(frameKey: number, allowPreviousFarend = false) {
+    const nearend = this.takeNearend(frameKey);
+    if (!nearend) {
+      return;
+    }
+    const previousFarendKey = frameKey - MAX_FAREND_BUCKET_SKEW;
+    const hasMatchingFarend = this.pendingFarend.has(frameKey)
+      || (allowPreviousFarend && this.pendingFarend.has(previousFarendKey));
+    const farend = hasMatchingFarend
+      ? this.takeFarendThrough(frameKey)
+      : null;
+    this.closedNearendBuckets.add(frameKey);
+    logAudioUplink("bucket.flush", {
+      nearendFrameKey: frameKey,
+      pairedFarend: Boolean(farend),
+      pendingFarend: [...this.pendingFarend.keys()],
+    });
+    this.enqueue(farend ? [farend, nearend] : [nearend]);
+  }
+
+  private takeNearend(frameKey: number): PendingAudioBlock | null {
+    const blocks = this.pendingNearend.get(frameKey);
+    this.pendingNearend.delete(frameKey);
+    if (!blocks?.length) {
+      return null;
+    }
+
+    const first = blocks[0];
+    return {
+      ...first,
+      dataParts: blocks.flatMap((block) => block.dataParts ?? [block.data]),
+    };
+  }
+
+  private takeFarendThrough(frameKey: number): PendingAudioBlock | null {
+    const blocks: PendingAudioBlock[] = [];
+    for (const key of [...this.pendingFarend.keys()].sort((a, b) => a - b)) {
+      if (key > frameKey) {
+        break;
+      }
+      blocks.push(...this.pendingFarend.get(key)!);
+      this.pendingFarend.delete(key);
+    }
+    if (blocks.length === 0) {
+      return null;
+    }
+
+    const latest = blocks[blocks.length - 1];
+    return {
+      ...latest,
+      dataParts: blocks.map((block) => block.data),
+      isFirstChunk: blocks.some((block) => block.isFirstChunk),
+    };
+  }
+
+  private abandonFarendStream() {
+    logAudioUplink("farend.drop", {
+      reason: "stale_bucket_abandons_webm_stream",
+      pendingFarend: [...this.pendingFarend.keys()],
+    });
+    this.pendingFarend.clear();
+    this.pendingWebmPrefixByStream.delete(AU_STREAM_FAREND);
+    this.farendStreamState = "awaiting-header";
+  }
+
+  private prune(currentFrameKey: number) {
+    const oldestAllowed = currentFrameKey - MAX_PENDING_FAREND_BUCKETS;
+    if ([...this.pendingFarend.keys()].some((frameKey) => frameKey < oldestAllowed)) {
+      // Dropping an arbitrary WebM fragment would corrupt every continuation
+      // after it. Abandon this container and wait for the next explicit header.
+      this.abandonFarendStream();
+    }
+    for (const frameKey of this.pendingNearend.keys()) {
+      if (frameKey < oldestAllowed) {
+        this.flushBucket(frameKey, true);
+      }
+    }
+    for (const frameKey of this.closedNearendBuckets) {
+      if (frameKey < oldestAllowed) {
+        this.closedNearendBuckets.delete(frameKey);
+      }
+    }
+  }
+}
+
+export const AU_AUDIO_BLOCK_MS = AUDIO_BLOCK_MS;

package/src/agent/e2e-client.ts

@@ -0,0 +1,215 @@
+import type { AgentClientMessage, AgentServerEvent, AgentSocketState } from "./types";
+
+const MAX_BUFFERED_AMOUNT = 2 * 1024 * 1024;
+
+export interface AgentE2EClientOptions {
+  wsUrl: string;
+  token: string;
+  audioInputEnabled?: boolean;
+  WebSocketCtor?: typeof WebSocket;
+  onStateChange?: (state: AgentSocketState) => void;
+  onEvent?: (event: AgentServerEvent) => void;
+  onError?: (error: Error) => void;
+  onClose?: (event: CloseEvent) => void;
+}
+
+export class AgentE2EClient {
+  private ws: WebSocket | null = null;
+  private connectPromise: Promise<void> | null = null;
+  private rejectConnect: ((error: Error) => void) | null = null;
+  private audioInputEnabled: boolean;
+  private readonly WebSocketCtor: typeof WebSocket;
+  private readonly options: AgentE2EClientOptions;
+
+  constructor(options: AgentE2EClientOptions) {
+    this.options = options;
+    this.audioInputEnabled = options.audioInputEnabled ?? true;
+    const WebSocketCtor = options.WebSocketCtor || globalThis.WebSocket;
+    if (!WebSocketCtor) {
+      throw new Error("当前环境不支持 WebSocket");
+    }
+    this.WebSocketCtor = WebSocketCtor;
+  }
+
+  get isOpen() {
+    return this.ws?.readyState === this.WebSocketCtor.OPEN;
+  }
+
+  connect(): Promise<void> {
+    if (this.isOpen) {
+      return Promise.resolve();
+    }
+    if (this.connectPromise) {
+      return this.connectPromise;
+    }
+
+    this.options.onStateChange?.("connecting");
+
+    let connection: Promise<void>;
+    connection = new Promise<void>((resolve, reject) => {
+      this.rejectConnect = reject;
+      let settled = false;
+      const ws = new this.WebSocketCtor(this.buildWsUrl(this.options.wsUrl, this.options.token));
+      this.ws = ws;
+
+      ws.onopen = () => {
+        if (this.ws !== ws) {
+          return;
+        }
+        settled = true;
+        if (this.connectPromise === connection) {
+          this.connectPromise = null;
+        }
+        this.rejectConnect = null;
+        this.options.onStateChange?.("open");
+        resolve();
+      };
+
+      ws.onmessage = (message) => {
+        if (this.ws !== ws) {
+          return;
+        }
+        const event = this.parseMessage(message.data);
+        if (event) {
+          this.options.onEvent?.(event);
+        }
+      };
+
+      ws.onerror = () => {
+        if (this.ws !== ws) {
+          return;
+        }
+        const error = new Error("E2E WebSocket 连接异常");
+        if (!settled) {
+          settled = true;
+          if (this.connectPromise === connection) {
+            this.connectPromise = null;
+          }
+          this.rejectConnect = null;
+          reject(error);
+        }
+        this.options.onStateChange?.("error");
+        this.options.onError?.(error);
+      };
+
+      ws.onclose = (event) => {
+        if (this.ws !== ws) {
+          return;
+        }
+        this.ws = null;
+        if (!settled) {
+          settled = true;
+          if (this.connectPromise === connection) {
+            this.connectPromise = null;
+          }
+          this.rejectConnect = null;
+          reject(new Error(`E2E WebSocket 已关闭: ${event.code} ${event.reason || ""}`.trim()));
+        }
+        this.options.onStateChange?.("closed");
+        this.options.onClose?.(event);
+      };
+    });
+    this.connectPromise = connection;
+    return this.connectPromise;
+  }
+
+  disconnect(code = 1000, reason = "client_close") {
+    if (!this.ws) {
+      return;
+    }
+
+    const ws = this.ws;
+    this.ws = null;
+    this.connectPromise = null;
+    const rejectConnect = this.rejectConnect;
+    this.rejectConnect = null;
+    rejectConnect?.(new Error("E2E WebSocket 连接已取消"));
+
+    if (ws.readyState === this.WebSocketCtor.OPEN || ws.readyState === this.WebSocketCtor.CONNECTING) {
+      ws.close(code, reason);
+    }
+  }
+
+  send(message: AgentClientMessage) {
+    this.assertOpen();
+    this.ws?.send(JSON.stringify(message));
+  }
+
+  trySend(message: AgentClientMessage) {
+    this.assertOpen();
+    if (this.ws && this.ws.bufferedAmount > MAX_BUFFERED_AMOUNT) {
+      return false;
+    }
+    this.ws?.send(JSON.stringify(message));
+    return true;
+  }
+
+  sendBinary(frame: ArrayBuffer | ArrayBufferView | Blob) {
+    this.assertOpen();
+    if (this.ws && this.ws.bufferedAmount > MAX_BUFFERED_AMOUNT) {
+      const error = new Error("E2E WebSocket 发送缓冲区超限");
+      error.name = "WebSocketBackpressureError";
+      throw error;
+    }
+    this.ws?.send(frame as any);
+  }
+
+  ping() {
+    this.send({ type: "ping" });
+  }
+
+  setAudioInputEnabled(enabled: boolean) {
+    this.audioInputEnabled = enabled;
+  }
+
+  private assertOpen() {
+    if (!this.isOpen) {
+      throw new Error("E2E WebSocket 未连接");
+    }
+  }
+
+  private parseMessage(data: unknown): AgentServerEvent | null {
+    if (typeof data !== "string") {
+      return null;
+    }
+
+    try {
+      const parsed = JSON.parse(data);
+      if (parsed && typeof parsed.type === "string") {
+        return parsed as AgentServerEvent;
+      }
+      return null;
+    } catch (error) {
+      this.options.onError?.(new Error("E2E WebSocket 消息不是合法 JSON"));
+      return null;
+    }
+  }
+
+  private buildWsUrl(wsUrl: string, token: string) {
+    try {
+      const url = new URL(wsUrl);
+      if (token && !url.searchParams.has("token") && !url.searchParams.has("e2e_token")) {
+        url.searchParams.set("token", token);
+      }
+      this.updateAudioInputQuery(url.searchParams);
+      return url.toString();
+    } catch {
+      const [baseUrl, query = ""] = wsUrl.split("?", 2);
+      const searchParams = new URLSearchParams(query);
+      if (token && !searchParams.has("token") && !searchParams.has("e2e_token")) {
+        searchParams.set("token", token);
+      }
+      this.updateAudioInputQuery(searchParams);
+      const nextQuery = searchParams.toString();
+      return nextQuery ? `${baseUrl}?${nextQuery}` : baseUrl;
+    }
+  }
+
+  private updateAudioInputQuery(searchParams: URLSearchParams) {
+    if (!this.audioInputEnabled) {
+      searchParams.set("audio_input_off", "1");
+      return;
+    }
+    searchParams.delete("audio_input_off");
+  }
+}