crowdplaysdk 0.6.2 → 0.7.0

package/lib/agentAudio.js

@@ -0,0 +1,189 @@
+"use strict";
+/**
+ * The agent audio tap, JavaScript side (engine 0.6.0+): the voice AI's call
+ * audio as it plays on this phone, in fixed frames of 16-bit mono PCM, for
+ * lip sync, visemes and meters. This is CALL audio (Path A), already
+ * compressed and decoded by the call; it never touches the recording.
+ *
+ * Pure TypeScript with no react-native import: option rules, the base64
+ * decoder, and the controller that orders start/stop and filters frames.
+ * index.ts wires the controller to the native module.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.STALE_BRIDGE_MESSAGE = exports.AGENT_AUDIO_DEFAULT_SAMPLE_RATE = exports.AGENT_AUDIO_DEFAULT_FRAME_SIZE = void 0;
+exports.normalizeAgentAudioOptions = normalizeAgentAudioOptions;
+exports.decodeAgentAudio = decodeAgentAudio;
+exports.createAgentAudio = createAgentAudio;
+exports.AGENT_AUDIO_DEFAULT_FRAME_SIZE = 1024;
+exports.AGENT_AUDIO_DEFAULT_SAMPLE_RATE = 16000;
+/** The engine's rules (AgentAudioTap.validate), in the engine's order, so
+ * a bad option fails here with the same wording instead of in native code.
+ * Doubles divide identically in JS and Swift, so the 10 ms and 1 s edges
+ * agree exactly. */
+function normalizeAgentAudioOptions(options) {
+    if (options !== undefined && options !== null && typeof options !== 'object') {
+        throw new TypeError('crowdplaysdk: startAgentAudio() options must be an object');
+    }
+    const frameSize = options?.frameSize ?? exports.AGENT_AUDIO_DEFAULT_FRAME_SIZE;
+    const sampleRate = options?.sampleRate ?? exports.AGENT_AUDIO_DEFAULT_SAMPLE_RATE;
+    if (typeof sampleRate !== 'number' || !Number.isInteger(sampleRate)) {
+        throw new TypeError(`crowdplaysdk: agent audio tap: sampleRate must be an integer (got ${String(sampleRate)})`);
+    }
+    if (typeof frameSize !== 'number' || !Number.isInteger(frameSize)) {
+        throw new TypeError(`crowdplaysdk: agent audio tap: frameSize must be an integer (got ${String(frameSize)})`);
+    }
+    if (sampleRate < 8000 || sampleRate > 48000) {
+        throw new RangeError(`crowdplaysdk: agent audio tap: sampleRate ${sampleRate} is outside 8000...48000`);
+    }
+    if (frameSize < 1) {
+        throw new RangeError(`crowdplaysdk: agent audio tap: frameSize ${frameSize} must be at least 1`);
+    }
+    const seconds = frameSize / sampleRate;
+    if (seconds < 0.010 || seconds > 1.0) {
+        throw new RangeError(`crowdplaysdk: agent audio tap: frameSize ${frameSize} at ${sampleRate} Hz is ` +
+            `${(seconds * 1000).toFixed(1)} ms; a frame must last 10 to 1000 ms`);
+    }
+    return { frameSize, sampleRate };
+}
+const ALPHABET = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/';
+const INVALID = 255;
+const LOOKUP = (() => {
+    const table = new Uint8Array(128).fill(INVALID);
+    for (let i = 0; i < ALPHABET.length; i++)
+        table[ALPHABET.charCodeAt(i)] = i;
+    return table;
+})();
+function notBase64() {
+    return new TypeError('crowdplaysdk: decodeAgentAudio() expects the base64 `pcm` string of an agentAudio event');
+}
+/**
+ * Decodes an agentAudio frame's `pcm` (base64 of Int16 little-endian mono)
+ * into samples, -32768...32767. Standard base64, padded or not; no atob
+ * needed. A trailing odd byte (never sent by the SDK) is ignored. Throws a
+ * TypeError on anything that is not base64.
+ */
+function decodeAgentAudio(pcm) {
+    if (typeof pcm !== 'string')
+        throw notBase64();
+    let end = pcm.length;
+    while (end > 0 && pcm.charCodeAt(end - 1) === 61 /* = */)
+        end--;
+    const padding = pcm.length - end;
+    if (padding > 2 || end % 4 === 1 || (padding > 0 && pcm.length % 4 !== 0))
+        throw notBase64();
+    const digit = (index) => {
+        const code = pcm.charCodeAt(index);
+        const value = code < 128 ? (LOOKUP[code] ?? INVALID) : INVALID;
+        if (value === INVALID)
+            throw notBase64();
+        return value;
+    };
+    const bytes = new Uint8Array(Math.floor((end * 3) / 4));
+    let out = 0;
+    let i = 0;
+    for (; i + 4 <= end; i += 4) {
+        const n = (digit(i) << 18) | (digit(i + 1) << 12) | (digit(i + 2) << 6) | digit(i + 3);
+        bytes[out++] = (n >> 16) & 0xff;
+        bytes[out++] = (n >> 8) & 0xff;
+        bytes[out++] = n & 0xff;
+    }
+    const rest = end - i;
+    if (rest === 2) {
+        const n = (digit(i) << 18) | (digit(i + 1) << 12);
+        bytes[out++] = (n >> 16) & 0xff;
+    }
+    else if (rest === 3) {
+        const n = (digit(i) << 18) | (digit(i + 1) << 12) | (digit(i + 2) << 6);
+        bytes[out++] = (n >> 16) & 0xff;
+        bytes[out++] = (n >> 8) & 0xff;
+    }
+    const samples = new Int16Array(out >> 1);
+    for (let k = 0; k < samples.length; k++) {
+        const lo = bytes[2 * k] ?? 0;
+        const hi = bytes[2 * k + 1] ?? 0;
+        // Little-endian whatever the platform's byte order; storing into an
+        // Int16Array wraps 32768...65535 to the negative samples.
+        samples[k] = (hi << 8) | lo;
+    }
+    return samples;
+}
+exports.STALE_BRIDGE_MESSAGE = 'crowdplaysdk: the agent audio tap is missing from the native bridge: this app\'s iOS ' +
+    'project still has the bridge files of an older crowdplaysdk. Re-run prebuild or wire.rb ' +
+    '(Expo: npx expo prebuild, or a build that runs it; bare React Native: ruby ' +
+    'node_modules/crowdplaysdk/ios/wire.rb <YourProjectName> from ios/), then Product > ' +
+    'Clean Build Folder before building.';
+function withTap(module) {
+    if (typeof module.startAgentAudio !== 'function' || typeof module.stopAgentAudio !== 'function') {
+        throw new Error(exports.STALE_BRIDGE_MESSAGE);
+    }
+    return module;
+}
+/**
+ * Orders every start and stop through one promise chain (separate native
+ * calls are not guaranteed to run in call order), and tracks which tap's
+ * frames may reach the app. A stop takes effect for the app the moment it
+ * is CALLED: frames still in flight from the bridge are dropped, so none
+ * arrive after stopAgentAudio() is called, let alone after it resolves.
+ */
+function createAgentAudio(getNative) {
+    let chain = Promise.resolve();
+    let stops = 0;
+    let activeTap = null;
+    const enqueue = (operation) => {
+        const run = chain.then(operation);
+        chain = run.catch(() => undefined);
+        return run;
+    };
+    return {
+        get activeTap() {
+            return activeTap;
+        },
+        start(options) {
+            let parsed;
+            let module;
+            try {
+                parsed = normalizeAgentAudioOptions(options);
+                module = withTap(getNative());
+            }
+            catch (error) {
+                return Promise.reject(error);
+            }
+            const stopsAtCall = stops;
+            return enqueue(async () => {
+                const result = await module.startAgentAudio(parsed);
+                const tap = result?.tap;
+                if (typeof tap !== 'number')
+                    throw new Error(exports.STALE_BRIDGE_MESSAGE);
+                // A stop called after this start wins, whatever order they finish in.
+                if (stopsAtCall === stops)
+                    activeTap = tap;
+            });
+        },
+        stop() {
+            let module;
+            try {
+                module = withTap(getNative());
+            }
+            catch (error) {
+                return Promise.reject(error);
+            }
+            stops += 1;
+            activeTap = null;
+            return enqueue(async () => {
+                await module.stopAgentAudio();
+            });
+        },
+        filter(listener) {
+            let lastTap;
+            let lastSeq;
+            return (frame) => {
+                if (activeTap === null || frame.tap !== activeTap)
+                    return;
+                const continues = frame.tap === lastTap && lastSeq !== undefined && frame.seq === lastSeq + 1;
+                lastTap = frame.tap;
+                lastSeq = frame.seq;
+                listener(continues || frame.discontinuity ? frame : { ...frame, discontinuity: true });
+            };
+        },
+    };
+}

package/lib/index.d.ts

@@ -4,9 +4,11 @@
  * The recording engine is 100% native (CrowdPlaySDK): studio-grade audio
  * (48 kHz / 24-bit, unprocessed) + 1080p30 video captured locally during a
  * live call, clock-synced across participants, uploaded to CrowdPlay with
- * retries and crash recovery. **No media ever crosses the JS bridge**,
- * only control calls and state events, so recording quality is identical
- * to a fully native app.
+ * retries and crash recovery. **No recording media ever crosses the JS
+ * bridge**, only control calls and state events, so recording quality is
+ * identical to a fully native app. The one audio stream that can cross it
+ * is the agent audio tap (startAgentAudio / useAgentAudio): the voice AI's
+ * call-path playout audio, and only while the app asks for it.
  *
  *     import CrowdPlay, { CrowdPlayConsentScreen, CrowdPlayVideoView } from 'crowdplaysdk';
  *
@@ -20,6 +22,7 @@
  * native layer requires the consent record structurally; there is no code
  * path that records without it.
  */
+import type { AgentAudioFrame, AgentAudioOptions } from './agentAudio';
 import type { VoiceActivityEvent, VoiceCaption } from './VoiceView';
 /** Live-call (Path A) quality. The RECORDING is always 1080p30 + 24-bit WAV
  * regardless; raising these trades upload + thermal headroom (which the
@@ -207,6 +210,11 @@ export interface CrowdPlayEvents {
     /** The latest live transcript line from the voice agent; see
      * useLatestCaption(). */
     caption: VoiceCaption;
+    /** The voice AI's call audio as it plays, one fixed frame at a time
+     * (engine 0.6.0+). Flows ONLY between startAgentAudio() and
+     * stopAgentAudio() or leave(); prefer useAgentAudio(). Decode `pcm` with
+     * decodeAgentAudio(). */
+    agentAudio: AgentAudioFrame;
 }
 export type CrowdPlayEventName = keyof CrowdPlayEvents;
 declare const CrowdPlay: {
@@ -269,13 +277,41 @@ declare const CrowdPlay: {
      * Sam was eliminated."). Keep it to a sentence or two; long pushes are
      * truncated. GPT-Live provider only; other providers ignore it. */
     updateAgentContext(text: string): void;
+    /**
+     * Starts the agent audio tap (engine 0.6.0+): the voice AI's call audio,
+     * resampled to `sampleRate` (default 16000) and downmixed to mono, as
+     * 'agentAudio' events of exactly `frameSize` samples (default 1024).
+     * Call it after join() or when the call screen mounts; it attaches when
+     * the AI's audio arrives and follows reconnects. It stays on until
+     * stopAgentAudio() or leave() (a failed join() leaves it on, so a retry
+     * gets frames); calling it again restarts it (new options, seq from 0).
+     * Rejects on invalid options, leaving a running tap as it was. Reads
+     * call audio only, never the recording. Prefer useAgentAudio().
+     */
+    startAgentAudio(options?: AgentAudioOptions): Promise<void>;
+    /** Stops the agent audio tap. From the moment this is called, no further
+     * 'agentAudio' frame reaches the app. A no-op when it is off. */
+    stopAgentAudio(): Promise<void>;
     /** Subscribe to state events. Returns a subscription; call .remove(). */
     addListener<E extends CrowdPlayEventName>(event: E, listener: (payload: CrowdPlayEvents[E]) => void): {
         remove(): void;
     };
 };
+/**
+ * Runs the agent audio tap while the calling component is mounted: starts
+ * on mount, stops on unmount, restarts when the options change. `onFrame`
+ * may change on every render (the latest one is called). Mount it in ONE
+ * place at a time (the call screen): the tap is one stream per app, and
+ * leave() stops it, so a screen that stays mounted across calls must
+ * remount (or call startAgentAudio()) after the next join. A failed join()
+ * does not stop it: a Retry on the same screen keeps getting frames. Does
+ * nothing where the native module is absent (Expo Go, previews).
+ */
+export declare function useAgentAudio(onFrame: (frame: AgentAudioFrame) => void, options?: AgentAudioOptions): void;
 export default CrowdPlay;
 export { CrowdPlayConsentScreen } from './ConsentScreen';
 export { CrowdPlayVideoView } from './VideoView';
 export { CrowdPlayVoiceView, VoiceReactive, useVoiceActivity, useLatestCaption } from './VoiceView';
-export type { CrowdPlayVoiceViewProps, VoiceActivity, VoiceActivityEvent, VoicePhase, VoiceCaption } from './VoiceView';
+export type { CrowdPlayVoiceViewProps, VoiceActivity, VoiceActivityEvent, VoiceActivityOptions, VoicePhase, VoiceCaption } from './VoiceView';
+export { decodeAgentAudio } from './agentAudio';
+export type { AgentAudioFrame, AgentAudioOptions } from './agentAudio';

package/lib/index.js

@@ -5,9 +5,11 @@
  * The recording engine is 100% native (CrowdPlaySDK): studio-grade audio
  * (48 kHz / 24-bit, unprocessed) + 1080p30 video captured locally during a
  * live call, clock-synced across participants, uploaded to CrowdPlay with
- * retries and crash recovery. **No media ever crosses the JS bridge**,
- * only control calls and state events, so recording quality is identical
- * to a fully native app.
+ * retries and crash recovery. **No recording media ever crosses the JS
+ * bridge**, only control calls and state events, so recording quality is
+ * identical to a fully native app. The one audio stream that can cross it
+ * is the agent audio tap (startAgentAudio / useAgentAudio): the voice AI's
+ * call-path playout audio, and only while the app asks for it.
  *
  *     import CrowdPlay, { CrowdPlayConsentScreen, CrowdPlayVideoView } from 'crowdplaysdk';
  *
@@ -22,8 +24,11 @@
  * path that records without it.
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.useLatestCaption = exports.useVoiceActivity = exports.VoiceReactive = exports.CrowdPlayVoiceView = exports.CrowdPlayVideoView = exports.CrowdPlayConsentScreen = exports.CROWDPLAY_TERMS_URL = void 0;
+exports.decodeAgentAudio = exports.useLatestCaption = exports.useVoiceActivity = exports.VoiceReactive = exports.CrowdPlayVoiceView = exports.CrowdPlayVideoView = exports.CrowdPlayConsentScreen = exports.CROWDPLAY_TERMS_URL = void 0;
+exports.useAgentAudio = useAgentAudio;
+const react_1 = require("react");
 const react_native_1 = require("react-native");
+const agentAudio_1 = require("./agentAudio");
 /** The CrowdPlay Terms of Use and Privacy Notice every participant accepts,
  * as a PDF. CrowdPlayConsentScreen links here; a custom consent UI must too. */
 exports.CROWDPLAY_TERMS_URL = 'https://www.crowdplay.ai/legal/consumer-terms';
@@ -47,6 +52,8 @@ function events() {
     return emitter;
 }
 let configured = false;
+// One controller for the app: the native tap is one stream per engine.
+const agentAudio = (0, agentAudio_1.createAgentAudio)(() => native());
 // ---------------------------------------------------------------------------
 // Public API
 // ---------------------------------------------------------------------------
@@ -184,12 +191,77 @@ const CrowdPlay = {
             native().updateAgentContext(text.trim());
         }
     },
+    /**
+     * Starts the agent audio tap (engine 0.6.0+): the voice AI's call audio,
+     * resampled to `sampleRate` (default 16000) and downmixed to mono, as
+     * 'agentAudio' events of exactly `frameSize` samples (default 1024).
+     * Call it after join() or when the call screen mounts; it attaches when
+     * the AI's audio arrives and follows reconnects. It stays on until
+     * stopAgentAudio() or leave() (a failed join() leaves it on, so a retry
+     * gets frames); calling it again restarts it (new options, seq from 0).
+     * Rejects on invalid options, leaving a running tap as it was. Reads
+     * call audio only, never the recording. Prefer useAgentAudio().
+     */
+    startAgentAudio(options) {
+        return agentAudio.start(options);
+    },
+    /** Stops the agent audio tap. From the moment this is called, no further
+     * 'agentAudio' frame reaches the app. A no-op when it is off. */
+    stopAgentAudio() {
+        return agentAudio.stop();
+    },
     /** Subscribe to state events. Returns a subscription; call .remove(). */
     addListener(event, listener) {
-        const subscription = events().addListener(`crowdplay:${event}`, listener);
+        let handler = listener;
+        if (event === 'agentAudio') {
+            // An older bridge does not list the event, and listening to it there
+            // raises a red box in debug builds: name the fix instead.
+            const module = react_native_1.NativeModules.CrowdPlayReactNative;
+            if (module && typeof module.startAgentAudio !== 'function') {
+                throw new Error(agentAudio_1.STALE_BRIDGE_MESSAGE);
+            }
+            handler = agentAudio.filter(listener);
+        }
+        const subscription = events().addListener(`crowdplay:${event}`, handler);
         return { remove: () => subscription.remove() };
     },
 };
+/**
+ * Runs the agent audio tap while the calling component is mounted: starts
+ * on mount, stops on unmount, restarts when the options change. `onFrame`
+ * may change on every render (the latest one is called). Mount it in ONE
+ * place at a time (the call screen): the tap is one stream per app, and
+ * leave() stops it, so a screen that stays mounted across calls must
+ * remount (or call startAgentAudio()) after the next join. A failed join()
+ * does not stop it: a Retry on the same screen keeps getting frames. Does
+ * nothing where the native module is absent (Expo Go, previews).
+ */
+function useAgentAudio(onFrame, options) {
+    const latest = (0, react_1.useRef)(onFrame);
+    (0, react_1.useEffect)(() => {
+        latest.current = onFrame;
+    }, [onFrame]);
+    const frameSize = options?.frameSize;
+    const sampleRate = options?.sampleRate;
+    (0, react_1.useEffect)(() => {
+        if (!react_native_1.NativeModules.CrowdPlayReactNative)
+            return;
+        const report = (error) => console.error(error instanceof Error ? error.message : String(error));
+        let subscription;
+        try {
+            subscription = CrowdPlay.addListener('agentAudio', (frame) => latest.current(frame));
+        }
+        catch (error) {
+            report(error);
+            return;
+        }
+        CrowdPlay.startAgentAudio({ frameSize, sampleRate }).catch(report);
+        return () => {
+            subscription.remove();
+            CrowdPlay.stopAgentAudio().catch(report);
+        };
+    }, [frameSize, sampleRate]);
+}
 exports.default = CrowdPlay;
 var ConsentScreen_1 = require("./ConsentScreen");
 Object.defineProperty(exports, "CrowdPlayConsentScreen", { enumerable: true, get: function () { return ConsentScreen_1.CrowdPlayConsentScreen; } });
@@ -200,3 +272,5 @@ Object.defineProperty(exports, "CrowdPlayVoiceView", { enumerable: true, get: fu
 Object.defineProperty(exports, "VoiceReactive", { enumerable: true, get: function () { return VoiceView_1.VoiceReactive; } });
 Object.defineProperty(exports, "useVoiceActivity", { enumerable: true, get: function () { return VoiceView_1.useVoiceActivity; } });
 Object.defineProperty(exports, "useLatestCaption", { enumerable: true, get: function () { return VoiceView_1.useLatestCaption; } });
+var agentAudio_2 = require("./agentAudio");
+Object.defineProperty(exports, "decodeAgentAudio", { enumerable: true, get: function () { return agentAudio_2.decodeAgentAudio; } });

package/llms.txt

@@ -6,8 +6,10 @@
 > participant on a shared clock, and uploads everything to CrowdPlay
 > automatically. The engine is compiled native code (the same CrowdPlaySDK core
 > the native iOS SDK ships); JavaScript sends control calls and receives
-> state events. NO media ever crosses the JS bridge, so recording quality
-> is identical to a fully native app.
+> state events. NO recording media ever crosses the JS bridge, so recording
+> quality is identical to a fully native app. The only audio that can
+> cross it is the opt-in agent audio tap: the voice AI's call audio for lip
+> sync, and only while the app asks for it.
 
 This file is the complete integration contract for an AI coding agent.
 Follow it exactly. When this file and your instincts disagree, this file
@@ -81,6 +83,14 @@ EXPO (managed, prebuild or EAS Build; app builders such as X1):
     Xcode project. Do not add those Info.plist keys or any native code
     yourself. Expo SDK 50 or newer.
 3.  Rebuild: npx expo prebuild, a development build, or eas build.
+    UPDATING the package later: npx expo prebuild again so the plugin
+    re-pins the engine to the version the package ships with (0.6.3+). An
+    ios/ folder committed to git is not re-wired by eas build on its own:
+    run prebuild and commit the result. Then clean before a LOCAL build
+    (Xcode: Product > Clean Build Folder); an incremental build can reuse
+    the previous engine's cached interface and fail with "has no member
+    '$storageWarning'". That error after an update means "clean", never
+    "edit the bridge". EAS Build starts clean.
 4.  Expo Go and any preview that never ran the plugin cannot load native
     modules: there the consent text, the call and recording fail with a
     clear error and the video view shows a placeholder. That is EXPECTED;
@@ -92,7 +102,8 @@ BARE REACT NATIVE (no Expo):
 2.  cd ios && pod install        (installs RN's own pods; CrowdPlay has none)
 3.  gem install xcodeproj --user-install   (one-time; wire.rb needs it
     and says so itself if missing)
-    ruby node_modules/crowdplaysdk/ios/wire.rb <YourProjectName>
+    ruby ../node_modules/crowdplaysdk/ios/wire.rb <YourProjectName>
+    (from ios/, where step 2 left you; wire.rb must run there)
     Idempotent. It: adds the CrowdPlaySDK binary Swift package to the Xcode
     project, copies the three native bridge files into the app target,
     creates/points the Swift bridging header at the React headers, raises
@@ -100,7 +111,10 @@ BARE REACT NATIVE (no Expo):
     sets SWIFT_ENABLE_EXPLICIT_MODULES = YES on the app target (React Native
     0.8x sets it to NO project-wide on Xcode 26, and with implicit modules
     the compiled engine's interface cannot be imported). Rerun it any time;
-    it never duplicates. Never undo that setting to fix an unrelated error.
+    it never duplicates, and after updating the package it re-pins the
+    engine to the version the package ships with (0.6.3+); then Product >
+    Clean Build Folder before building. Never undo that setting to fix an
+    unrelated error.
 4.  Info.plist: NSMicrophoneUsageDescription, NSCameraUsageDescription,
     UIBackgroundModes = [audio]. A missing usage string makes iOS kill
     the app at first mic/camera access (uncatchable).
@@ -170,13 +184,23 @@ Default export `CrowdPlay`:
   setAudioOutput(output: 'automatic' | 'speaker'): void
       // 'automatic' = the connected device (AirPods, wired, ...);
       // pair with the 'audioRoute' event for the output menu
+  startAgentAudio(opts?: { frameSize?: number; sampleRate?: number }): Promise<void>
+  stopAgentAudio(): Promise<void>
+      // 0.7.0+ (engine 0.6.0): the voice AI's call audio as 'agentAudio'
+      // frames, for lip sync; see "Lip sync" below. Prefer useAgentAudio().
   addListener<E extends CrowdPlayEventName>(event: E,
       handler: (payload: CrowdPlayEvents[E]) => void): { remove(): void }
 
 Named exports: CrowdPlayConsentScreen (props: onConsent(grant)), CrowdPlayVideoView
 (props: participant: string | 'local', plus standard View style),
 CrowdPlayVoiceView (props: agentColor?, listeningColor?, idleColor?, size?),
-useVoiceActivity() -> { phase: 'connecting'|'idle'|'listening'|'speaking', energy: 0…1 }.
+useVoiceActivity(opts?: { intervalMs?: number })
+    -> { phase: 'connecting'|'idle'|'listening'|'speaking', energy: 0…1 }
+    // intervalMs: update rate, default 150, floor 33; mounted hooks share
+    // one stream at the smallest interval asked for,
+useAgentAudio(onFrame: (frame: AgentAudioFrame) => void,
+              opts?: { frameSize?: number; sampleRate?: number }): void,
+decodeAgentAudio(pcm: string): Int16Array   // frame.pcm -> samples
 
 The voice AI (apps with the Voice AI switch enabled on the dashboard):
 
@@ -272,6 +296,57 @@ count it as a generic user. Detect it in the participants event by name
   For speech bubbles, subtitles, language-learning corrections. Optional
   UI; the full timestamped transcript is always delivered with the
   session regardless.
+- Lip sync (0.7.0+, engine 0.6.0): the agent audio tap hands the app the
+  AI's voice as raw PCM while it plays, for mouth shapes, visemes or
+  custom meters (useVoiceActivity's energy is a few updates a second; this
+  is every sample). The recipe: start it after join() resolves or on the
+  call screen's mount, feed decodeAgentAudio(e.pcm) to the app's viseme
+  classifier, stop it on leave.
+
+      import { useAgentAudio, decodeAgentAudio } from 'crowdplaysdk';
+      function Avatar() {
+        useAgentAudio((frame) => {
+          const samples = decodeAgentAudio(frame.pcm); // Int16Array
+          visemes.feed(samples, { reset: frame.discontinuity,    // app code
+                                  showAt: frame.heardAtMs });
+        });                       // defaults: frameSize 1024, sampleRate 16000
+        return <Character mouth={visemes.current} />;
+      }
+
+  Imperative form: await CrowdPlay.startAgentAudio({ frameSize, sampleRate })
+  after join(), CrowdPlay.addListener('agentAudio', handler), and
+  CrowdPlay.stopAgentAudio() before or with leave().
+  RULES: frames are exactly frameSize samples, 16-bit mono, at sampleRate;
+  sampleRate an integer 8000 to 48000, frameSize an integer >= 1, a frame
+  10 ms to 1 s (otherwise start rejects and a running tap is left as it
+  was). seq counts frames from 0 per start; discontinuity is true on the
+  first frame a listener gets and after any gap (reconnect, dropped
+  audio): reset smoothing there. Starting again restarts it with the new
+  options. leave() turns it off: start it again after the next join (the
+  hook does that when its screen remounts). A failed join() leaves it on,
+  so a Retry on the same screen keeps getting frames. Mount useAgentAudio
+  in ONE component at a time; the tap is one stream for the whole app.
+  Nothing is delivered while no AI is in the room. It listens only to the
+  AI participant (identity "agent-" plus 8 lowercase hex, name "agent"),
+  never to a person whose name merely looks like it; the identity is not
+  reserved, so a participant that joins under exactly that identity is
+  taken for the AI (a person named exactly "agent" always is from the web
+  SDK, and now and then from a phone).
+  TIMING: frames are taken as the audio is pulled for playout. Each frame
+  carries heardAtMs, when its first sample reaches the listener (Unix ms
+  on the phone's clock, compare with Date.now()) = the pull plus
+  outputLatencyMs, the route's delay (a few ms on the speaker and wired
+  headphones, 100 to 300 ms on Bluetooth/AirPods). Show each frame's mouth
+  shape at heardAtMs, never on arrival: on AirPods frames arrive well
+  before they are heard, on the speaker just after (a frame is sent once
+  its last sample is pulled; smaller frames shorten that). On Bluetooth
+  the delay is the one iOS reports for the headphones, so the timing
+  there is only as exact as that figure.
+  It reads the CALL audio only: it never touches the recording, and
+  nothing crosses the bridge unless the app starts it. Never capture the
+  AI's audio any other way (rule 2). After updating to a package with it,
+  re-wire (prebuild or wire.rb) and Clean Build Folder; an old bridge
+  rejects with a message saying exactly that.
 - Testing: the AI appears only on a real iPhone (simulators have no
   microphone). Join a room, wait ~3 s, it speaks first.
 
@@ -330,6 +405,11 @@ Events (CrowdPlay.addListener):
                  // stopped. The call continues. The SDK shows nothing
                  // itself; listening is optional. Kept out of 'warning' so
                  // apps that render every warning do not show it.
+  'agentAudio'   { pcm: string; sampleRate; frameSize; seq; discontinuity; tap;
+                   heardAtMs; outputLatencyMs }
+                 // 0.7.0+: only between startAgentAudio() and
+                 // stopAgentAudio() or leave(). pcm = base64 Int16 LE mono,
+                 // decodeAgentAudio(pcm) -> Int16Array. Prefer useAgentAudio().
 
 ## Checking sessions (no AWS, no SDK, plain HTTPS)
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crowdplaysdk",
-  "version": "0.6.2",
+  "version": "0.7.0",
   "description": "CrowdPlay lossless conversation capture for React Native (iOS). Studio-grade per-participant recording during live video calls, delivered to CrowdPlay automatically.",
   "main": "lib/index.js",
   "types": "lib/index.d.ts",
@@ -17,6 +17,7 @@
   "scripts": {
     "build": "tsc",
     "typecheck": "tsc --noEmit",
+    "test": "tsc -p test/tsconfig.json && node --test --experimental-test-module-mocks \"test/*.test.mjs\"",
     "prepack": "npm run build"
   },
   "homepage": "https://github.com/symbiateam/crowdplaysdk",

package/src/VoiceView.tsx

@@ -33,11 +33,60 @@ export interface VoiceActivityEvent {
   connected: boolean;
 }
 
-/** Live phase + energy from the call engine. Subscribing starts the native
- * fast stream; the last unmount stops it. */
-export function useVoiceActivity(): VoiceActivity {
+export interface VoiceActivityOptions {
+  /** How often the native stream sends 'voiceActivity', in ms (floor 33).
+   * Default 150. With several hooks mounted, the stream runs at the
+   * smallest interval any of them asks for. */
+  intervalMs?: number;
+}
+
+/** Every mounted useVoiceActivity() shares ONE native stream: each hook
+ * registers its requested interval (undefined = the default) here, the
+ * stream runs at the smallest, and only the last unmount stops it. */
+const voiceRequests = new Map<object, number | undefined>();
+/** What the native stream was last told: 'stopped', 'default' (the
+ * zero-argument start, 150 ms, exactly as before intervals existed), or
+ * an interval in ms. */
+let voiceStream: 'stopped' | 'default' | number = 'stopped';
+
+function voiceIntervalOf(intervalMs: unknown): number | undefined {
+  if (typeof intervalMs !== 'number' || !Number.isFinite(intervalMs) || intervalMs <= 0) return undefined;
+  return Math.max(33, Math.round(intervalMs));
+}
+
+function applyVoiceRequests(module: {
+  startVoiceActivityUpdates?: () => void;
+  startVoiceActivityUpdatesWithInterval?: (intervalMs: number) => void;
+  stopVoiceActivityUpdates?: () => void;
+}): void {
+  if (voiceRequests.size === 0) {
+    if (voiceStream !== 'stopped') module.stopVoiceActivityUpdates?.();
+    voiceStream = 'stopped';
+    return;
+  }
+  let smallest: number | undefined;
+  for (const interval of voiceRequests.values()) {
+    if (interval !== undefined && (smallest === undefined || interval < smallest)) smallest = interval;
+  }
+  const want = smallest ?? 'default';
+  if (want === voiceStream) return;
+  if (want !== 'default' && typeof module.startVoiceActivityUpdatesWithInterval === 'function') {
+    module.startVoiceActivityUpdatesWithInterval(want);
+  } else {
+    // The default, or a bridge wired by an older package (fixed 150 ms).
+    module.startVoiceActivityUpdates?.();
+  }
+  voiceStream = want;
+}
+
+/** Live phase + energy from the call engine. The first mounted hook starts
+ * the native fast stream; the last unmount stops it. */
+export function useVoiceActivity(options?: VoiceActivityOptions): VoiceActivity {
   const [activity, setActivity] = useState<VoiceActivity>({ phase: 'connecting', energy: 0 });
   const energyRef = useRef(0);
+  const token = useRef<object>({}).current;
+  const intervalMs = voiceIntervalOf(options?.intervalMs);
+  const intervalRef = useRef(intervalMs);
 
   useEffect(() => {
     const module = NativeModules.CrowdPlayReactNative;
@@ -65,12 +114,24 @@ export function useVoiceActivity(): VoiceActivity {
       energyRef.current += (target - energyRef.current) * rate;
       setActivity({ phase, energy: energyRef.current });
     });
-    module.startVoiceActivityUpdates?.();
+    voiceRequests.set(token, intervalRef.current);
+    applyVoiceRequests(module);
     return () => {
       sub.remove();
-      module.stopVoiceActivityUpdates?.();
+      voiceRequests.delete(token);
+      applyVoiceRequests(module);
     };
-  }, []);
+  }, [token]);
+
+  // A changed interval updates this hook's request in place, so the shared
+  // stream never stops and restarts for it.
+  useEffect(() => {
+    intervalRef.current = intervalMs;
+    const module = NativeModules.CrowdPlayReactNative;
+    if (!module || !voiceRequests.has(token)) return;
+    voiceRequests.set(token, intervalMs);
+    applyVoiceRequests(module);
+  }, [token, intervalMs]);
 
   return activity;
 }