crowdplaysdk 0.6.3 → 0.7.0

package/src/agentAudio.ts

@@ -0,0 +1,259 @@
+/**
+ * The agent audio tap, JavaScript side (engine 0.6.0+): the voice AI's call
+ * audio as it plays on this phone, in fixed frames of 16-bit mono PCM, for
+ * lip sync, visemes and meters. This is CALL audio (Path A), already
+ * compressed and decoded by the call; it never touches the recording.
+ *
+ * Pure TypeScript with no react-native import: option rules, the base64
+ * decoder, and the controller that orders start/stop and filters frames.
+ * index.ts wires the controller to the native module.
+ */
+
+/** Options for CrowdPlay.startAgentAudio() and useAgentAudio(). */
+export interface AgentAudioOptions {
+  /** Samples per frame, an integer of at least 1. Default 1024 (64 ms at
+   * 16 kHz). frameSize / sampleRate must be 10 ms to 1 s. */
+  frameSize?: number;
+  /** Output rate in Hz, an integer in 8000...48000. Default 16000. The
+   * call audio is resampled to it and downmixed to mono. */
+  sampleRate?: number;
+}
+
+/** Payload of the 'agentAudio' event: one frame of the voice AI's call
+ * audio, pulled for playout. */
+export interface AgentAudioFrame {
+  /** Base64 of Int16 little-endian mono PCM, exactly `frameSize` samples.
+   * decodeAgentAudio(frame.pcm) gives the samples. */
+  pcm: string;
+  sampleRate: number;
+  frameSize: number;
+  /** 0, 1, 2... per start, contiguous across delivered frames. */
+  seq: number;
+  /** True when this frame does not directly continue the previous frame
+   * this listener received: its first frame, a track switch or reconnect,
+   * a format change, or lost audio. Reset any smoothing state. */
+  discontinuity: boolean;
+  /** Which start this frame belongs to (frames of a stopped tap are
+   * dropped before they reach the app). */
+  tap: number;
+  /** When the frame's first sample reaches the listener, in Unix ms on this
+   * phone's clock (compare with Date.now()): its playout pull plus
+   * `outputLatencyMs`. Show the frame's mouth shape at this moment. It is
+   * usually already past when the frame arrives on the speaker (a frame is
+   * sent once its last sample is pulled, 64 ms at the defaults) and 100 to
+   * 300 ms ahead on Bluetooth. */
+  heardAtMs: number;
+  /** The audio route's delay from playout to the listener, in ms: one IO
+   * buffer plus what iOS reports for the route (a few ms on the speaker
+   * and wired headphones, 100 to 300 ms on Bluetooth). Re-read at most
+   * every 0.5 s, so a route change shows within half a second. */
+  outputLatencyMs: number;
+}
+
+export const AGENT_AUDIO_DEFAULT_FRAME_SIZE = 1024;
+export const AGENT_AUDIO_DEFAULT_SAMPLE_RATE = 16000;
+
+/** The engine's rules (AgentAudioTap.validate), in the engine's order, so
+ * a bad option fails here with the same wording instead of in native code.
+ * Doubles divide identically in JS and Swift, so the 10 ms and 1 s edges
+ * agree exactly. */
+export function normalizeAgentAudioOptions(
+  options?: AgentAudioOptions | null
+): { frameSize: number; sampleRate: number } {
+  if (options !== undefined && options !== null && typeof options !== 'object') {
+    throw new TypeError('crowdplaysdk: startAgentAudio() options must be an object');
+  }
+  const frameSize = options?.frameSize ?? AGENT_AUDIO_DEFAULT_FRAME_SIZE;
+  const sampleRate = options?.sampleRate ?? AGENT_AUDIO_DEFAULT_SAMPLE_RATE;
+  if (typeof sampleRate !== 'number' || !Number.isInteger(sampleRate)) {
+    throw new TypeError(`crowdplaysdk: agent audio tap: sampleRate must be an integer (got ${String(sampleRate)})`);
+  }
+  if (typeof frameSize !== 'number' || !Number.isInteger(frameSize)) {
+    throw new TypeError(`crowdplaysdk: agent audio tap: frameSize must be an integer (got ${String(frameSize)})`);
+  }
+  if (sampleRate < 8000 || sampleRate > 48000) {
+    throw new RangeError(`crowdplaysdk: agent audio tap: sampleRate ${sampleRate} is outside 8000...48000`);
+  }
+  if (frameSize < 1) {
+    throw new RangeError(`crowdplaysdk: agent audio tap: frameSize ${frameSize} must be at least 1`);
+  }
+  const seconds = frameSize / sampleRate;
+  if (seconds < 0.010 || seconds > 1.0) {
+    throw new RangeError(
+      `crowdplaysdk: agent audio tap: frameSize ${frameSize} at ${sampleRate} Hz is ` +
+        `${(seconds * 1000).toFixed(1)} ms; a frame must last 10 to 1000 ms`
+    );
+  }
+  return { frameSize, sampleRate };
+}
+
+const ALPHABET = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/';
+const INVALID = 255;
+const LOOKUP: Uint8Array = (() => {
+  const table = new Uint8Array(128).fill(INVALID);
+  for (let i = 0; i < ALPHABET.length; i++) table[ALPHABET.charCodeAt(i)] = i;
+  return table;
+})();
+
+function notBase64(): TypeError {
+  return new TypeError(
+    'crowdplaysdk: decodeAgentAudio() expects the base64 `pcm` string of an agentAudio event'
+  );
+}
+
+/**
+ * Decodes an agentAudio frame's `pcm` (base64 of Int16 little-endian mono)
+ * into samples, -32768...32767. Standard base64, padded or not; no atob
+ * needed. A trailing odd byte (never sent by the SDK) is ignored. Throws a
+ * TypeError on anything that is not base64.
+ */
+export function decodeAgentAudio(pcm: string): Int16Array {
+  if (typeof pcm !== 'string') throw notBase64();
+  let end = pcm.length;
+  while (end > 0 && pcm.charCodeAt(end - 1) === 61 /* = */) end--;
+  const padding = pcm.length - end;
+  if (padding > 2 || end % 4 === 1 || (padding > 0 && pcm.length % 4 !== 0)) throw notBase64();
+
+  const digit = (index: number): number => {
+    const code = pcm.charCodeAt(index);
+    const value = code < 128 ? (LOOKUP[code] ?? INVALID) : INVALID;
+    if (value === INVALID) throw notBase64();
+    return value;
+  };
+
+  const bytes = new Uint8Array(Math.floor((end * 3) / 4));
+  let out = 0;
+  let i = 0;
+  for (; i + 4 <= end; i += 4) {
+    const n = (digit(i) << 18) | (digit(i + 1) << 12) | (digit(i + 2) << 6) | digit(i + 3);
+    bytes[out++] = (n >> 16) & 0xff;
+    bytes[out++] = (n >> 8) & 0xff;
+    bytes[out++] = n & 0xff;
+  }
+  const rest = end - i;
+  if (rest === 2) {
+    const n = (digit(i) << 18) | (digit(i + 1) << 12);
+    bytes[out++] = (n >> 16) & 0xff;
+  } else if (rest === 3) {
+    const n = (digit(i) << 18) | (digit(i + 1) << 12) | (digit(i + 2) << 6);
+    bytes[out++] = (n >> 16) & 0xff;
+    bytes[out++] = (n >> 8) & 0xff;
+  }
+
+  const samples = new Int16Array(out >> 1);
+  for (let k = 0; k < samples.length; k++) {
+    const lo = bytes[2 * k] ?? 0;
+    const hi = bytes[2 * k + 1] ?? 0;
+    // Little-endian whatever the platform's byte order; storing into an
+    // Int16Array wraps 32768...65535 to the negative samples.
+    samples[k] = (hi << 8) | lo;
+  }
+  return samples;
+}
+
+/** The native methods the controller needs; optional because a bridge
+ * wired by an older crowdplaysdk does not have them. */
+export interface AgentAudioNative {
+  startAgentAudio?(options: { frameSize: number; sampleRate: number }): Promise<{ tap: number } | null>;
+  stopAgentAudio?(): Promise<void>;
+}
+
+export const STALE_BRIDGE_MESSAGE =
+  'crowdplaysdk: the agent audio tap is missing from the native bridge: this app\'s iOS ' +
+  'project still has the bridge files of an older crowdplaysdk. Re-run prebuild or wire.rb ' +
+  '(Expo: npx expo prebuild, or a build that runs it; bare React Native: ruby ' +
+  'node_modules/crowdplaysdk/ios/wire.rb <YourProjectName> from ios/), then Product > ' +
+  'Clean Build Folder before building.';
+
+interface TapNative {
+  startAgentAudio: NonNullable<AgentAudioNative['startAgentAudio']>;
+  stopAgentAudio: NonNullable<AgentAudioNative['stopAgentAudio']>;
+}
+
+function withTap(module: AgentAudioNative): TapNative {
+  if (typeof module.startAgentAudio !== 'function' || typeof module.stopAgentAudio !== 'function') {
+    throw new Error(STALE_BRIDGE_MESSAGE);
+  }
+  return module as TapNative;
+}
+
+export interface AgentAudioController {
+  start(options?: AgentAudioOptions | null): Promise<void>;
+  stop(): Promise<void>;
+  /** Wraps an app listener: drops frames of any tap but the active one and
+   * marks the first frame it passes, and any frame that does not follow
+   * the previous one it passed, as a discontinuity. */
+  filter(listener: (frame: AgentAudioFrame) => void): (frame: AgentAudioFrame) => void;
+  /** The tap whose frames are delivered, or null (tests and diagnostics). */
+  readonly activeTap: number | null;
+}
+
+/**
+ * Orders every start and stop through one promise chain (separate native
+ * calls are not guaranteed to run in call order), and tracks which tap's
+ * frames may reach the app. A stop takes effect for the app the moment it
+ * is CALLED: frames still in flight from the bridge are dropped, so none
+ * arrive after stopAgentAudio() is called, let alone after it resolves.
+ */
+export function createAgentAudio(getNative: () => AgentAudioNative): AgentAudioController {
+  let chain: Promise<void> = Promise.resolve();
+  let stops = 0;
+  let activeTap: number | null = null;
+
+  const enqueue = (operation: () => Promise<void>): Promise<void> => {
+    const run = chain.then(operation);
+    chain = run.catch(() => undefined);
+    return run;
+  };
+
+  return {
+    get activeTap() {
+      return activeTap;
+    },
+
+    start(options) {
+      let parsed: { frameSize: number; sampleRate: number };
+      let module: TapNative;
+      try {
+        parsed = normalizeAgentAudioOptions(options);
+        module = withTap(getNative());
+      } catch (error) {
+        return Promise.reject(error);
+      }
+      const stopsAtCall = stops;
+      return enqueue(async () => {
+        const result = await module.startAgentAudio(parsed);
+        const tap = result?.tap;
+        if (typeof tap !== 'number') throw new Error(STALE_BRIDGE_MESSAGE);
+        // A stop called after this start wins, whatever order they finish in.
+        if (stopsAtCall === stops) activeTap = tap;
+      });
+    },
+
+    stop() {
+      let module: TapNative;
+      try {
+        module = withTap(getNative());
+      } catch (error) {
+        return Promise.reject(error);
+      }
+      stops += 1;
+      activeTap = null;
+      return enqueue(async () => {
+        await module.stopAgentAudio();
+      });
+    },
+
+    filter(listener) {
+      let lastTap: number | undefined;
+      let lastSeq: number | undefined;
+      return (frame) => {
+        if (activeTap === null || frame.tap !== activeTap) return;
+        const continues = frame.tap === lastTap && lastSeq !== undefined && frame.seq === lastSeq + 1;
+        lastTap = frame.tap;
+        lastSeq = frame.seq;
+        listener(continues || frame.discontinuity ? frame : { ...frame, discontinuity: true });
+      };
+    },
+  };
+}

package/src/index.ts

@@ -4,9 +4,11 @@
  * The recording engine is 100% native (CrowdPlaySDK): studio-grade audio
  * (48 kHz / 24-bit, unprocessed) + 1080p30 video captured locally during a
  * live call, clock-synced across participants, uploaded to CrowdPlay with
- * retries and crash recovery. **No media ever crosses the JS bridge**,
- * only control calls and state events, so recording quality is identical
- * to a fully native app.
+ * retries and crash recovery. **No recording media ever crosses the JS
+ * bridge**, only control calls and state events, so recording quality is
+ * identical to a fully native app. The one audio stream that can cross it
+ * is the agent audio tap (startAgentAudio / useAgentAudio): the voice AI's
+ * call-path playout audio, and only while the app asks for it.
  *
  *     import CrowdPlay, { CrowdPlayConsentScreen, CrowdPlayVideoView } from 'crowdplaysdk';
  *
@@ -21,7 +23,10 @@
  * path that records without it.
  */
 
+import { useEffect, useRef } from 'react';
 import { NativeEventEmitter, NativeModules, Platform } from 'react-native';
+import { createAgentAudio, STALE_BRIDGE_MESSAGE } from './agentAudio';
+import type { AgentAudioFrame, AgentAudioNative, AgentAudioOptions } from './agentAudio';
 import type { VoiceActivityEvent, VoiceCaption } from './VoiceView';
 
 // ---------------------------------------------------------------------------
@@ -200,6 +205,11 @@ export interface CrowdPlayEvents {
   /** The latest live transcript line from the voice agent; see
    * useLatestCaption(). */
   caption: VoiceCaption;
+  /** The voice AI's call audio as it plays, one fixed frame at a time
+   * (engine 0.6.0+). Flows ONLY between startAgentAudio() and
+   * stopAgentAudio() or leave(); prefer useAgentAudio(). Decode `pcm` with
+   * decodeAgentAudio(). */
+  agentAudio: AgentAudioFrame;
 }
 
 export type CrowdPlayEventName = keyof CrowdPlayEvents;
@@ -226,7 +236,11 @@ interface NativeCrowdPlay {
   doctor(): Promise<DoctorCheck[]>;
   /** Start/stop the 'voiceActivity' fast stream (useVoiceActivity owns this). */
   startVoiceActivityUpdates(): void;
+  startVoiceActivityUpdatesWithInterval?(intervalMs: number): void;
   stopVoiceActivityUpdates(): void;
+  /** The agent audio tap; absent from bridge files wired by an older package. */
+  startAgentAudio?: AgentAudioNative['startAgentAudio'];
+  stopAgentAudio?: AgentAudioNative['stopAgentAudio'];
   addListener(eventName: string): void;
   removeListeners(count: number): void;
 }
@@ -256,6 +270,9 @@ function events(): NativeEventEmitter {
 
 let configured = false;
 
+// One controller for the app: the native tap is one stream per engine.
+const agentAudio = createAgentAudio(() => native());
+
 // ---------------------------------------------------------------------------
 // Public API
 // ---------------------------------------------------------------------------
@@ -412,18 +429,90 @@ const CrowdPlay = {
     }
   },
 
+  /**
+   * Starts the agent audio tap (engine 0.6.0+): the voice AI's call audio,
+   * resampled to `sampleRate` (default 16000) and downmixed to mono, as
+   * 'agentAudio' events of exactly `frameSize` samples (default 1024).
+   * Call it after join() or when the call screen mounts; it attaches when
+   * the AI's audio arrives and follows reconnects. It stays on until
+   * stopAgentAudio() or leave() (a failed join() leaves it on, so a retry
+   * gets frames); calling it again restarts it (new options, seq from 0).
+   * Rejects on invalid options, leaving a running tap as it was. Reads
+   * call audio only, never the recording. Prefer useAgentAudio().
+   */
+  startAgentAudio(options?: AgentAudioOptions): Promise<void> {
+    return agentAudio.start(options);
+  },
+
+  /** Stops the agent audio tap. From the moment this is called, no further
+   * 'agentAudio' frame reaches the app. A no-op when it is off. */
+  stopAgentAudio(): Promise<void> {
+    return agentAudio.stop();
+  },
+
   /** Subscribe to state events. Returns a subscription; call .remove(). */
   addListener<E extends CrowdPlayEventName>(
     event: E,
     listener: (payload: CrowdPlayEvents[E]) => void
   ): { remove(): void } {
-    const subscription = events().addListener(`crowdplay:${event}`, listener as (payload: unknown) => void);
+    let handler = listener as (payload: unknown) => void;
+    if (event === 'agentAudio') {
+      // An older bridge does not list the event, and listening to it there
+      // raises a red box in debug builds: name the fix instead.
+      const module = NativeModules.CrowdPlayReactNative as NativeCrowdPlay | undefined;
+      if (module && typeof module.startAgentAudio !== 'function') {
+        throw new Error(STALE_BRIDGE_MESSAGE);
+      }
+      handler = agentAudio.filter(listener as (frame: AgentAudioFrame) => void) as (payload: unknown) => void;
+    }
+    const subscription = events().addListener(`crowdplay:${event}`, handler);
     return { remove: () => subscription.remove() };
   },
 };
 
+/**
+ * Runs the agent audio tap while the calling component is mounted: starts
+ * on mount, stops on unmount, restarts when the options change. `onFrame`
+ * may change on every render (the latest one is called). Mount it in ONE
+ * place at a time (the call screen): the tap is one stream per app, and
+ * leave() stops it, so a screen that stays mounted across calls must
+ * remount (or call startAgentAudio()) after the next join. A failed join()
+ * does not stop it: a Retry on the same screen keeps getting frames. Does
+ * nothing where the native module is absent (Expo Go, previews).
+ */
+export function useAgentAudio(
+  onFrame: (frame: AgentAudioFrame) => void,
+  options?: AgentAudioOptions
+): void {
+  const latest = useRef(onFrame);
+  useEffect(() => {
+    latest.current = onFrame;
+  }, [onFrame]);
+  const frameSize = options?.frameSize;
+  const sampleRate = options?.sampleRate;
+  useEffect(() => {
+    if (!NativeModules.CrowdPlayReactNative) return;
+    const report = (error: unknown) =>
+      console.error(error instanceof Error ? error.message : String(error));
+    let subscription: { remove(): void };
+    try {
+      subscription = CrowdPlay.addListener('agentAudio', (frame) => latest.current(frame));
+    } catch (error) {
+      report(error);
+      return;
+    }
+    CrowdPlay.startAgentAudio({ frameSize, sampleRate }).catch(report);
+    return () => {
+      subscription.remove();
+      CrowdPlay.stopAgentAudio().catch(report);
+    };
+  }, [frameSize, sampleRate]);
+}
+
 export default CrowdPlay;
 export { CrowdPlayConsentScreen } from './ConsentScreen';
 export { CrowdPlayVideoView } from './VideoView';
 export { CrowdPlayVoiceView, VoiceReactive, useVoiceActivity, useLatestCaption } from './VoiceView';
-export type { CrowdPlayVoiceViewProps, VoiceActivity, VoiceActivityEvent, VoicePhase, VoiceCaption } from './VoiceView';
+export type { CrowdPlayVoiceViewProps, VoiceActivity, VoiceActivityEvent, VoiceActivityOptions, VoicePhase, VoiceCaption } from './VoiceView';
+export { decodeAgentAudio } from './agentAudio';
+export type { AgentAudioFrame, AgentAudioOptions } from './agentAudio';