@xmov/avatar 2.0.0-alpha.38

package/src/utils/audio-session.ts

@@ -0,0 +1,144 @@
+export type WebAudioSessionType = "playback" | "play-and-record";
+
+type WebAudioSession = {
+  type: string;
+};
+
+type NavigatorWithAudioSession = Navigator & {
+  audioSession?: WebAudioSession;
+};
+
+type WebAudioPlaybackSessionRestoredListener = () => void;
+
+const captureStates = new WeakMap<
+  WebAudioSession,
+  { count: number; previousType: string }
+>();
+const playbackSessionRestoredListeners = new WeakMap<
+  WebAudioSession,
+  Set<WebAudioPlaybackSessionRestoredListener>
+>();
+
+/**
+ * Keep Web Audio audible through the iOS Ring/Silent switch. Do not override
+ * a play-and-record session that may belong to an active microphone track.
+ */
+export function configureWebAudioPlaybackSession(
+  target: Navigator = navigator,
+): boolean {
+  const audioSession = getAudioSession(target);
+  if (!audioSession) {
+    return false;
+  }
+
+  const captureState = captureStates.get(audioSession);
+  if (captureState?.count || audioSession.type === "play-and-record") {
+    return setAudioSessionType(audioSession, "play-and-record");
+  }
+  return setAudioSessionType(audioSession, "playback");
+}
+
+export function beginWebAudioCaptureSession(
+  target: Navigator = navigator,
+): boolean {
+  const audioSession = getAudioSession(target);
+  if (!audioSession) {
+    return false;
+  }
+
+  const existingState = captureStates.get(audioSession);
+  const previousType = existingState?.previousType ?? audioSession.type;
+  if (!setAudioSessionType(audioSession, "play-and-record")) {
+    return false;
+  }
+
+  captureStates.set(audioSession, {
+    count: (existingState?.count ?? 0) + 1,
+    previousType,
+  });
+  return true;
+}
+
+export function subscribeWebAudioPlaybackSessionRestored(
+  listener: WebAudioPlaybackSessionRestoredListener,
+  target: Navigator = navigator,
+): () => void {
+  const audioSession = getAudioSession(target);
+  if (!audioSession) {
+    return () => {};
+  }
+
+  let listeners = playbackSessionRestoredListeners.get(audioSession);
+  if (!listeners) {
+    listeners = new Set();
+    playbackSessionRestoredListeners.set(audioSession, listeners);
+  }
+  listeners.add(listener);
+
+  return () => {
+    listeners?.delete(listener);
+    if (listeners?.size === 0) {
+      playbackSessionRestoredListeners.delete(audioSession);
+    }
+  };
+}
+
+export function endWebAudioCaptureSession(
+  target: Navigator = navigator,
+): boolean {
+  const audioSession = getAudioSession(target);
+  if (!audioSession) {
+    return false;
+  }
+
+  const captureState = captureStates.get(audioSession);
+  if (!captureState) {
+    return configureWebAudioPlaybackSession(target);
+  }
+
+  captureState.count -= 1;
+  if (captureState.count > 0) {
+    return setAudioSessionType(audioSession, "play-and-record");
+  }
+
+  captureStates.delete(audioSession);
+  const nextType = captureState.previousType === "play-and-record"
+    ? "play-and-record"
+    : "playback";
+  const updated = setAudioSessionType(audioSession, nextType);
+  if (updated && nextType === "playback") {
+    notifyPlaybackSessionRestored(audioSession);
+  }
+  return updated;
+}
+
+function notifyPlaybackSessionRestored(audioSession: WebAudioSession): void {
+  const listeners = playbackSessionRestoredListeners.get(audioSession);
+  if (!listeners) {
+    return;
+  }
+
+  for (const listener of [...listeners]) {
+    try {
+      listener();
+    } catch {
+      // Playback recovery must never break microphone cleanup.
+    }
+  }
+}
+
+function getAudioSession(target: Navigator): WebAudioSession | undefined {
+  return (target as NavigatorWithAudioSession).audioSession;
+}
+
+function setAudioSessionType(
+  audioSession: WebAudioSession,
+  type: WebAudioSessionType,
+): boolean {
+  try {
+    audioSession.type = type;
+    return audioSession.type === type;
+  } catch {
+    return false;
+  }
+}

package/src/utils/audio.ts

@@ -0,0 +1,44 @@
+// PCM转WAV的辅助函数
+export function pcmToWav(pcmData: Uint8Array) {
+  const wavHeader = new ArrayBuffer(44);
+  const view = new DataView(wavHeader);
+
+  /* RIFF identifier */
+  writeString(view, 0, "RIFF");
+  /* RIFF chunk length */
+  view.setUint32(4, 36 + pcmData.length, true);
+  /* RIFF type */
+  writeString(view, 8, "WAVE");
+  /* format chunk identifier */
+  writeString(view, 12, "fmt ");
+  /* format chunk length */
+  view.setUint32(16, 16, true);
+  /* sample format (raw) */
+  view.setUint16(20, 1, true);
+  /* channel count */
+  view.setUint16(22, 1, true); // mono
+  /* sample rate */
+  view.setUint32(24, 24000, true);
+  /* byte rate (sample rate * block align) */
+  view.setUint32(28, 24000 * 2, true);
+  /* block align (channel count * bytes per sample) */
+  view.setUint16(32, 2, true);
+  /* bits per sample */
+  view.setUint16(34, 16, true);
+  /* data chunk identifier */
+  writeString(view, 36, "data");
+  /* data chunk length */
+  view.setUint32(40, pcmData.length, true);
+
+  const wavData = new Uint8Array(wavHeader.byteLength + pcmData.length);
+  wavData.set(new Uint8Array(wavHeader), 0);
+  wavData.set(new Uint8Array(pcmData), wavHeader.byteLength);
+
+  return wavData.buffer;
+}
+
+function writeString(view: DataView, offset: number, string: string) {
+  for (let i = 0; i < string.length; i++) {
+    view.setUint8(offset + i, string.charCodeAt(i));
+  }
+}

package/src/utils/blueimp-md5.d.ts

@@ -0,0 +1,3 @@
+declare module 'blueimp-md5' {
+    export default function md5(str: string): string;
+} 

package/src/utils/capability-checker.ts

@@ -0,0 +1,379 @@
+import type { ICapabilityCheckItem, ICapabilityCheckReport } from '../types/capability';
+
+// ==================== 同步检查（constructor 使用） ====================
+
+/**
+ * 同步检查安全上下文（HTTPS / localhost）
+ */
+function checkSecureContextSync(): ICapabilityCheckItem {
+  const supported = window.isSecureContext;
+  return {
+    name: 'secureContext',
+    supported,
+    severity: 'fatal',
+    reason: supported ? undefined : '当前环境不是安全上下文，请使用 HTTPS 或 localhost',
+  };
+}
+
+/**
+ * 同步检查 WebGL2 支持（创建上下文）
+ */
+function checkWebGL2Sync(): ICapabilityCheckItem {
+  try {
+    const canvas = document.createElement('canvas');
+    const gl = canvas.getContext('webgl2', { failIfMajorPerformanceCaveat: true });
+    if (!gl) {
+      return {
+        name: 'webgl2',
+        supported: false,
+        severity: 'fatal',
+        reason: '浏览器不支持 WebGL2',
+      };
+    }
+    // 获取基本信息
+    const debugInfo = gl.getExtension('WEBGL_debug_renderer_info');
+    const detail = {
+      version: gl.getParameter(gl.VERSION),
+      vendor: gl.getParameter(gl.VENDOR),
+      renderer: gl.getParameter(gl.RENDERER),
+      gpuVendor: debugInfo ? gl.getParameter(debugInfo.UNMASKED_VENDOR_WEBGL) : undefined,
+      gpuRenderer: debugInfo ? gl.getParameter(debugInfo.UNMASKED_RENDERER_WEBGL) : undefined,
+    };
+    return {
+      name: 'webgl2',
+      supported: true,
+      severity: 'fatal',
+      detail,
+    };
+  } catch (e) {
+    return {
+      name: 'webgl2',
+      supported: false,
+      severity: 'fatal',
+      reason: 'WebGL2 初始化失败',
+      detail: e,
+    };
+  }
+}
+
+/**
+ * 同步检查音频系统（PCM 或 MSE 任一可用即可）
+ * 
+ * SDK 有两种音频播放器：
+ * 1. PCM 播放器：AudioWorklet + AudioContext（raw_audio = true）
+ * 2. MSE 播放器：MediaSource + HTMLAudioElement（raw_audio = false）
+ * 
+ * 两者二选一，任一可用则 SDK 音频能力可用
+ */
+function checkAudioSystemSync(): ICapabilityCheckItem {
+  // PCM 路径：需要 AudioContext + AudioWorkletNode
+  const hasAudioContext = typeof (window as any).AudioContext !== 'undefined'
+    || typeof (window as any).webkitAudioContext !== 'undefined';
+  const hasAudioWorklet = typeof (window as any).AudioWorkletNode !== 'undefined';
+  const pcmSupported = hasAudioContext && hasAudioWorklet;
+
+  // MSE 路径：需要 MediaSource + SourceBuffer
+  const hasMediaSource = typeof (window as any).MediaSource !== 'undefined';
+  const mseSupported = hasMediaSource;
+
+  const supported = pcmSupported || mseSupported;
+
+  const detail = {
+    pcm: {
+      supported: pcmSupported,
+      hasAudioContext,
+      hasAudioWorklet,
+    },
+    mse: {
+      supported: mseSupported,
+      hasMediaSource,
+    },
+  };
+
+  if (!supported) {
+    return {
+      name: 'audio',
+      supported: false,
+      severity: 'fatal',
+      reason: '音频播放能力不可用：PCM（AudioWorklet）和 MSE（MediaSource）均不支持',
+      detail,
+    };
+  }
+
+  return {
+    name: 'audio',
+    supported: true,
+    severity: 'fatal',
+    detail,
+  };
+}
+
+/**
+ * 同步检查 WebCodecs API 是否存在（不探测具体编解码器）
+ */
+function checkWebCodecsSync(): ICapabilityCheckItem {
+  const hasVideoDecoder = typeof (window as any).VideoDecoder !== 'undefined';
+  const supported = hasVideoDecoder;
+
+  return {
+    name: 'webcodecs',
+    supported,
+    severity: 'fatal',
+    reason: supported ? undefined : '浏览器不支持 WebCodecs VideoDecoder API',
+    detail: { hasVideoDecoder },
+  };
+}
+
+/**
+ * 同步检查 Web Worker 构造函数是否存在
+ */
+function checkWorkerSync(): ICapabilityCheckItem {
+  const supported = typeof Worker !== 'undefined';
+  return {
+    name: 'worker',
+    supported,
+    severity: 'fatal',
+    reason: supported ? undefined : '浏览器不支持 Web Worker',
+  };
+}
+
+/**
+ * 同步检查所有浏览器能力（用于 constructor 快速校验）
+ * 
+ * 检查项：
+ * - secureContext：安全上下文（HTTPS / localhost）
+ * - webgl2：WebGL2 渲染能力
+ * - audio：音频播放能力（PCM 或 MSE 任一可用即可）
+ * - webcodecs：WebCodecs API 存在性
+ * - worker：Web Worker 支持
+ * 
+ * 音频检查逻辑：SDK 有两种音频播放器（PCM 和 MSE），只需其中一种可用即可
+ * - PCM 播放器依赖：AudioContext + AudioWorkletNode
+ * - MSE 播放器依赖：MediaSource
+ * - 两者都不可用时，音频能力检查失败
+ */
+export function checkCapabilitySync(audioOnlyMode = false): ICapabilityCheckReport {
+  const items: ICapabilityCheckItem[] = [
+    checkSecureContextSync(),
+  ];
+
+  if (audioOnlyMode) {
+    // audioOnlyMode: only check secureContext, audio, and worker
+    items.push(checkAudioSystemSync());
+    items.push(checkWorkerSync());
+  } else {
+    // Full mode: check all capabilities
+    items.push(
+      checkWebGL2Sync(),
+      checkAudioSystemSync(),
+      checkWebCodecsSync(),
+      checkWorkerSync(),
+    );
+  }
+
+  const fatalCount = items.filter(i => i.severity === 'fatal' && !i.supported).length;
+  const warnCount = items.filter(i => i.severity === 'warn' && !i.supported).length;
+
+  return {
+    supported: fatalCount === 0,
+    fatalCount,
+    warnCount,
+    items,
+  };
+}
+
+// ==================== 异步检查（init 详细探测使用） ====================
+
+/**
+ * 异步检查 WebCodecs 详细编解码器支持
+ */
+async function checkWebCodecsDetailed(): Promise<ICapabilityCheckItem> {
+  try {
+    const hasVideoDecoder = typeof (window as any).VideoDecoder !== 'undefined';
+    const supported = hasVideoDecoder;
+
+    if (!supported) {
+      return {
+        name: 'webcodecs',
+        supported: false,
+        severity: 'fatal',
+        reason: '浏览器不支持 WebCodecs VideoDecoder API',
+        detail: { hasVideoDecoder },
+      };
+    }
+
+    // 探测具体编解码器支持
+    let codecSupport: Record<string, boolean> = {};
+    const testCodecs = [
+      'video/mp4; codecs="avc1.42E01E"',  // H.264 Baseline
+      'video/mp4; codecs="avc1.4D001F"',  // H.264 Main
+      'video/mp4; codecs="avc1.64001F"',  // H.264 High
+    ];
+    for (const codec of testCodecs) {
+      try {
+        const result = await (window as any).VideoDecoder.isConfigSupported({
+          codec,
+        });
+        codecSupport[codec] = result.supported;
+      } catch {
+        codecSupport[codec] = false;
+      }
+    }
+
+    return {
+      name: 'webcodecs',
+      supported: true,
+      severity: 'fatal',
+      detail: { hasVideoDecoder, codecSupport },
+    };
+  } catch (e) {
+    return {
+      name: 'webcodecs',
+      supported: false,
+      severity: 'fatal',
+      reason: 'WebCodecs 探测失败',
+      detail: e,
+    };
+  }
+}
+
+/**
+ * 异步检查 Worker 实际执行能力
+ */
+const WORKER_CHECK_TIMEOUT_MS = 3000;
+const WORKER_CHECK_MAX_ATTEMPTS = 2;
+
+type WorkerProbeResult =
+  | { status: 'supported' }
+  | { status: 'timeout' }
+  | { status: 'failed'; reason: string; detail?: unknown };
+
+function probeWorkerExecution(): Promise<WorkerProbeResult> {
+  return new Promise((resolve) => {
+    let worker: Worker | null = null;
+    let url: string | null = null;
+    let settled = false;
+    let timeout: ReturnType<typeof setTimeout> | null = null;
+
+    const finish = (result: WorkerProbeResult) => {
+      if (settled) return;
+      settled = true;
+      if (timeout !== null) clearTimeout(timeout);
+      worker?.terminate();
+      if (url) URL.revokeObjectURL(url);
+      resolve(result);
+    };
+
+    try {
+      const blob = new Blob(['self.postMessage("ok");'], { type: 'application/javascript' });
+      url = URL.createObjectURL(blob);
+      worker = new Worker(url);
+      worker.onmessage = (event) => {
+        finish(event.data === 'ok'
+          ? { status: 'supported' }
+          : { status: 'failed', reason: 'Web Worker 执行异常' });
+      };
+      worker.onerror = (event) => {
+        finish({ status: 'failed', reason: 'Web Worker 创建失败', detail: event });
+      };
+      timeout = setTimeout(() => {
+        finish({ status: 'timeout' });
+      }, WORKER_CHECK_TIMEOUT_MS);
+    } catch (error) {
+      finish({ status: 'failed', reason: 'Web Worker 异常', detail: error });
+    }
+  });
+}
+
+async function checkWorkerDetailed(): Promise<ICapabilityCheckItem> {
+  if (typeof Worker === 'undefined') {
+    return {
+      name: 'worker',
+      supported: false,
+      severity: 'fatal',
+      reason: '浏览器不支持 Web Worker',
+    };
+  }
+
+  for (let attempt = 1; attempt <= WORKER_CHECK_MAX_ATTEMPTS; attempt += 1) {
+    const result = await probeWorkerExecution();
+    if (result.status === 'supported') {
+      return {
+        name: 'worker',
+        supported: true,
+        severity: 'fatal',
+        detail: { attempts: attempt },
+      };
+    }
+    if (result.status === 'failed') {
+      return {
+        name: 'worker',
+        supported: false,
+        severity: 'fatal',
+        reason: result.reason,
+        detail: result.detail,
+      };
+    }
+  }
+
+  // iOS/WebKit can suspend or delay a Blob Worker while the page is being
+  // inspected, backgrounded, or recovering from a reload. The Worker API is
+  // present and no explicit error was raised, so a timing-only probe must not
+  // reject the entire SDK initialization. Real worker creation still reports
+  // its own fatal error later if the capability is genuinely unavailable.
+  return {
+    name: 'worker',
+    supported: false,
+    severity: 'warn',
+    reason: `Web Worker 创建超时（已重试 ${WORKER_CHECK_MAX_ATTEMPTS} 次）`,
+    detail: { attempts: WORKER_CHECK_MAX_ATTEMPTS },
+  };
+}
+
+/**
+ * 异步检查所有浏览器能力（详细版，用于 init 阶段详细探测）
+ * 
+ * 包含更详细的编解码器探测和 Worker 执行测试
+ */
+export async function checkCapability(audioOnlyMode = false): Promise<ICapabilityCheckReport> {
+  // 先执行同步检查
+  const syncReport = checkCapabilitySync(audioOnlyMode);
+
+  if (audioOnlyMode) {
+    // audioOnlyMode: only detailed check worker
+    const detailedItems = await Promise.all([
+      Promise.resolve(syncReport.items.find(i => i.name === 'secureContext')!),
+      Promise.resolve(syncReport.items.find(i => i.name === 'audio')!),
+      checkWorkerDetailed(),
+    ]);
+
+    const fatalCount = detailedItems.filter(i => i.severity === 'fatal' && !i.supported).length;
+    const warnCount = detailedItems.filter(i => i.severity === 'warn' && !i.supported).length;
+
+    return {
+      supported: fatalCount === 0,
+      fatalCount,
+      warnCount,
+      items: detailedItems,
+    };
+  }
+
+  // 替换 webcodecs 和 worker 为详细版本
+  const detailedItems = await Promise.all([
+    Promise.resolve(syncReport.items.find(i => i.name === 'secureContext')!),
+    Promise.resolve(syncReport.items.find(i => i.name === 'webgl2')!),
+    Promise.resolve(syncReport.items.find(i => i.name === 'audio')!),
+    checkWebCodecsDetailed(),
+    checkWorkerDetailed(),
+  ]);
+
+  const fatalCount = detailedItems.filter(i => i.severity === 'fatal' && !i.supported).length;
+  const warnCount = detailedItems.filter(i => i.severity === 'warn' && !i.supported).length;
+
+  return {
+    supported: fatalCount === 0,
+    fatalCount,
+    warnCount,
+    items: detailedItems,
+  };
+}

package/src/utils/encodeToken.ts

@@ -0,0 +1,100 @@
+import md5 from 'blueimp-md5';
+
+function encodeWithMd5(s: string): string {
+  return md5(s);
+}
+
+function escapeNonAsciiForPythonJson(value: string): string {
+  return value.replace(/[^\u0000-\u007E]/g, character => {
+    return `\\u${character.charCodeAt(0).toString(16).padStart(4, '0')}`;
+  });
+}
+
+function compareUnicodeCodePoints(left: string, right: string): number {
+  const leftPoints = Array.from(left, character => character.codePointAt(0)!);
+  const rightPoints = Array.from(right, character => character.codePointAt(0)!);
+  const length = Math.min(leftPoints.length, rightPoints.length);
+
+  for (let index = 0; index < length; index += 1) {
+    if (leftPoints[index] !== rightPoints[index]) {
+      return leftPoints[index] - rightPoints[index];
+    }
+  }
+
+  return leftPoints.length - rightPoints.length;
+}
+
+export interface Headers {
+  [key: string]: string;
+}
+
+function parseUrlWithoutBase(url: string): { isAbsolute: boolean; pathWithQuery: string } {
+  try {
+    // 尝试直接解析为绝对 URL（不含基准）
+    const urlObj = new URL(url);
+    // 若是绝对 URL，返回标记+完整路径+查询
+    return {
+      isAbsolute: true,
+      pathWithQuery: urlObj.pathname + urlObj.search
+    };
+  } catch {
+    // 解析失败，视为相对路径（path+query）
+    // 处理开头可能的斜杠，确保统一格式（如 "path" → "/path"）
+    let path = url.trim();
+    if (!path.startsWith('/')) {
+      path = '/' + path;
+    }
+    return {
+      isAbsolute: false,
+      pathWithQuery: path
+    };
+  }
+}
+
+export function headersNeedSign(
+  ak: string,
+  sk: string,
+  method: string,
+  url: string,
+  data: Record<string, any>
+): { headers: Headers; data: Record<string, any> } {
+  const headers: Headers = {};
+  const t = Math.floor(Date.now() / 1000);
+  const urlPathQuery = parseUrlWithoutBase(url).pathWithQuery;
+
+  // 递归排序对象的所有 key（包括嵌套对象）
+  function sortObjectKeys(obj: any): any {
+    if (obj === null || obj === undefined) {
+      return obj;
+    }
+    if (Array.isArray(obj)) {
+      return obj.map(item => sortObjectKeys(item));
+    }
+    if (typeof obj !== 'object') {
+      return obj;
+    }
+    const sortedKeys = Object.keys(obj).sort(compareUnicodeCodePoints);
+    const sortedObj: Record<string, any> = {};
+    for (const key of sortedKeys) {
+      sortedObj[key] = sortObjectKeys(obj[key]);
+    }
+    return sortedObj;
+  }
+
+  // 对象按 key 排序并去除空格
+  const sortedData = sortObjectKeys(data);
+
+  // 对齐 Python json.dumps(..., sort_keys=True) 的默认 ensure_ascii=True 行为。
+  const dataStr = escapeNonAsciiForPythonJson(JSON.stringify(sortedData)).replace(/ /g, '');
+
+  const oriSign = `${urlPathQuery.toLowerCase()}${method.toLowerCase()}${dataStr}${sk}${t}`;
+  const sign = encodeWithMd5(oriSign);
+
+  headers['X-APP-ID'] = ak;
+  headers['X-TOKEN'] = sign;
+  headers['X-TIMESTAMP'] = t.toString();
+  return {
+    headers,
+    data: sortedData,
+  };
+}

package/src/utils/face.ts

@@ -0,0 +1,29 @@
+// translate numpy.float16 tobytes() to Float32Array
+const halfToFloat = (h: number) => {
+  const s = (h & 0x8000) >> 15;
+  let e = (h & 0x7C00) >> 10;
+  let f = h & 0x03FF;
+
+  if (e === 0) {
+    // subnormal
+    return ((-1) ** s) * (f / 0x400) * 2 ** (-14);
+  } else if (e === 0x1F) {
+    // ±Inf or NaN
+    return f ? NaN : ((-1) ** s) * Infinity;
+  } else {
+    // normal
+    return ((-1) ** s) * (1 + f / 0x400) * 2 ** (e - 15);
+  }
+}
+
+export const translateBufferToFloatArray = (buf: any) => {
+  const count = buf.byteLength / 2;
+  const floats = new Float32Array(count);
+  const dv = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
+
+  for (let i = 0; i < count; i++) {
+    const h = dv.getUint16(i * 2, true);  // true = little‑endian
+    floats[i] = halfToFloat(h);
+  }
+  return Array.from(floats)
+}