crowdplaysdk 0.6.2 → 0.7.0

package/README.md

@@ -3,8 +3,10 @@
 CrowdPlaySDK adds live calls and high-quality conversation capture to React
 Native iOS apps. It is the same compiled engine as the native CrowdPlaySDK
 Swift package, driven from JavaScript with control calls and state events.
-No media ever crosses the JS bridge, so recording quality is identical to a
-fully native app.
+No recording media ever crosses the JS bridge, so recording quality is
+identical to a fully native app. The one audio stream that can cross it is
+opt-in and is not the recording: the agent audio tap (below) hands the
+voice AI's call audio to JavaScript for lip sync, only while you ask for it.
 
 ## What the SDK handles
 
@@ -97,7 +99,7 @@ Bare React Native (no Expo):
 npm install crowdplaysdk
 cd ios && pod install
 gem install xcodeproj --user-install              # one time
-ruby node_modules/crowdplaysdk/ios/wire.rb <YourProjectName>
+ruby ../node_modules/crowdplaysdk/ios/wire.rb <YourProjectName>   # from ios/
 ```
 
 `wire.rb` is idempotent. It adds the compiled CrowdPlaySDK Swift package,
@@ -107,6 +109,18 @@ keeps explicit Swift modules on for the app target (React Native 0.8x turns
 them off project-wide on Xcode 26, which breaks importing the compiled
 engine). There is no CocoaPod of our own, so your Podfile stays untouched.
 
+**Updating.** `npm install crowdplaysdk@latest`, then wire again so the
+engine moves with the bridge: Expo, `npx expo prebuild` (or a build that
+runs prebuild; an `ios/` folder committed to git is not re-wired by
+`eas build`, so run prebuild yourself and commit the result); bare React
+Native, re-run `wire.rb`. Since 0.6.3 that also re-pins the engine to the
+version this package ships with. 0.6.2 and older only added the engine
+the first time, so upgrading in place kept the old engine. Before the
+next local build, clean (Xcode: Product > Clean Build Folder): an
+incremental build can reuse Xcode's cached copy of the previous engine's
+interface and fail on new symbols such as `storageWarning`. EAS Build
+starts clean every time.
+
 **2. Declare permissions in Info.plist** (bare React Native only; the Expo
 plugin does this for you).
 
@@ -168,6 +182,7 @@ CrowdPlay.addListener('warning', …)       // micSilent / clipping / crossTalk:
 CrowdPlay.addListener('storageWarning', …) // optional: message set = storage low (audio only or stopped)
 CrowdPlay.addListener('recording', …)     // error means NOT capturing; offer retryRecording()
 CrowdPlay.addListener('audioRoute', …)    // current output device, for the output menu
+CrowdPlay.addListener('agentAudio', …)    // opt-in, voice AI: the AI's voice as PCM (see Voice AI)
 CrowdPlay.setAudioOutput('speaker')       // or 'automatic' (the connected device)
 CrowdPlay.snapshot()                      // poll at 1 Hz for the REC timer and input level
 ```
@@ -198,8 +213,65 @@ the AI speaks:
 ```
 
 Restyle it with props, wrap your own artwork in `<VoiceReactive>`, or drive
-any visual from the `useVoiceActivity()` hook. Show live subtitles with
-`useLatestCaption()`, and identify the AI participant by name `agent`.
+any visual from the `useVoiceActivity()` hook (`useVoiceActivity({ intervalMs: 50 })`
+updates faster than the default 150 ms; the floor is 33 ms). Show live
+subtitles with `useLatestCaption()`, and identify the AI participant by name
+`agent`.
+
+### Lip sync: the agent audio tap
+
+For mouth shapes, visemes or your own meters, the SDK can hand you the AI's
+voice as it plays, in fixed frames of 16-bit mono PCM (crowdplaysdk 0.7.0+,
+engine 0.6.0):
+
+```tsx
+import { useAgentAudio, decodeAgentAudio } from 'crowdplaysdk';
+
+function Avatar() {
+  useAgentAudio((frame) => {
+    const samples = decodeAgentAudio(frame.pcm); // Int16Array, frame.frameSize samples
+    // your classifier; show the shape when the frame is heard, not on arrival
+    visemes.feed(samples, { reset: frame.discontinuity, showAt: frame.heardAtMs });
+  }, { frameSize: 1024, sampleRate: 16000 }); // the defaults: 64 ms frames
+  return <Character mouth={visemes.current} />;
+}
+```
+
+The recipe: start it after `join()` resolves or when the call screen
+mounts (the hook does it on mount), feed `decodeAgentAudio(e.pcm)` to your
+viseme classifier, and stop it on leave (`leave()` stops it; unmounting the
+hook's screen stops it too). Without the hook, call
+`CrowdPlay.startAgentAudio(options)` and `CrowdPlay.stopAgentAudio()` and
+listen with `CrowdPlay.addListener('agentAudio', …)`.
+
+- Every frame is exactly `frameSize` samples at `sampleRate` (integers;
+  `sampleRate` 8000 to 48000, a frame 10 ms to 1 s; anything else
+  rejects). `seq` counts frames from 0 for each start; `discontinuity` is
+  true on the first frame and after any gap (a reconnect, dropped audio):
+  reset your smoothing there.
+- It attaches when the AI's audio arrives and follows it across
+  reconnects; before that nothing is delivered. Starting again restarts it
+  with the new options; `leave()` turns it off, so start it again after the
+  next join. A failed `join()` leaves it on, so a Retry on the same screen
+  keeps getting frames. Mount `useAgentAudio` in one place at a time: the
+  tap is one stream for the whole app.
+- It listens only to the AI participant (identity `agent-` plus 8
+  lowercase hex, name `agent`), never to a person whose name merely looks
+  like it. That identity is not reserved, so someone who joins under that
+  exact identity is taken for the AI (a person named exactly `agent` always
+  is from the web SDK, and now and then from a phone).
+- Timing: frames are taken as the audio is pulled for playout, and each
+  one says when the user hears it. `heardAtMs` (Unix ms on the phone's
+  clock, compare with `Date.now()`) is the pull plus `outputLatencyMs`, the
+  route's delay: a few ms on the speaker or wired headphones, 100 to 300 ms
+  on Bluetooth. Show each frame's mouth shape at `heardAtMs`, not when the
+  frame arrives. On AirPods a frame arrives well before it is heard; on the
+  speaker it arrives just after it starts playing, because a frame is sent
+  once its last sample is pulled (smaller frames shorten that). On
+  Bluetooth the delay is the one iOS reports for the headphones, so the
+  timing there is only as exact as that figure.
+- It reads the call audio only. It never touches the recording, and nothing
+  crosses the bridge unless your app starts it.
 
 You can also override the voice, personality and context for a single call,
 which is how you give the AI memory of a particular user:

package/ios/CrowdPlayRNModule.swift

@@ -1,3 +1,4 @@
+import AVFoundation
 import Combine
 import Foundation
 import CrowdPlaySDK
@@ -26,6 +27,11 @@ final class CrowdPlayRNCore {
 
     func bind(_ module: CrowdPlayRNModule) {
         emitter = module
+        // A JS reload makes a new module, but this core and its engine live
+        // on: a tap the previous JS started must not feed the new one.
+        if agentAudioTap != nil, agentAudioOwner !== module {
+            stopAgentAudio()
+        }
         guard cancellables.isEmpty else { return } // bind once; emitter may rebind
 
         engine.$phase.sink { [weak self] phase in
@@ -155,13 +161,17 @@ final class CrowdPlayRNCore {
     // MARK: - Voice-agent activity (M8 voice UI)
 
     private var voiceTimer: Timer?
-
-    /// Fast (0.15 s) voice-activity stream for the animated voice UI , 
-    /// runs only while some JS component is subscribed, so the bridge
-    /// stays quiet for apps that never render it.
-    func startVoiceActivity() {
-        guard voiceTimer == nil else { return }
-        voiceTimer = Timer.scheduledTimer(withTimeInterval: 0.15, repeats: true) { _ in
+    private var voiceInterval: TimeInterval = 0
+
+    /// Fast (0.15 s by default) voice-activity stream for the animated voice
+    /// UI; it runs only while some JS component is subscribed, so the bridge
+    /// stays quiet for apps that never render it. A running stream is left
+    /// alone at the same interval and re-created at a new one.
+    func startVoiceActivity(interval: TimeInterval = 0.15) {
+        if voiceTimer != nil, voiceInterval == interval { return }
+        voiceTimer?.invalidate()
+        voiceInterval = interval
+        voiceTimer = Timer.scheduledTimer(withTimeInterval: interval, repeats: true) { _ in
             Task { @MainActor in CrowdPlayRNCore.shared.voiceTick() }
         }
     }
@@ -184,6 +194,62 @@ final class CrowdPlayRNCore {
         ])
     }
 
+    // MARK: - Agent audio tap (engine 0.6.0)
+
+    /// The running tap's id (sent with every frame so JS can drop frames of
+    /// a stopped tap) and the module whose JS started it. Ids only grow, and
+    /// this core outlives JS reloads, so an id is never reused.
+    private var agentAudioTap: Int?
+    private weak var agentAudioOwner: CrowdPlayRNModule?
+    private var agentAudioTapCount = 0
+    private var agentAudioOptions: (frameSize: Int, sampleRate: Int)?
+
+    func startAgentAudio(frameSize: Int, sampleRate: Int, owner: CrowdPlayRNModule) throws -> Int {
+        let tap = agentAudioTapCount + 1
+        try startEngineTap(frameSize: frameSize, sampleRate: sampleRate, tap: tap)
+        agentAudioTapCount = tap
+        agentAudioTap = tap
+        agentAudioOptions = (frameSize, sampleRate)
+        agentAudioOwner = owner
+        return tap
+    }
+
+    private func startEngineTap(frameSize: Int, sampleRate: Int, tap: Int) throws {
+        // Frames are sent from the engine's drain queue, never the main
+        // actor: one hop per frame would add jitter to lip sync.
+        try engine.startAgentAudioTap(frameSize: frameSize, sampleRate: sampleRate) { frame in
+            CrowdPlayRNAgentAudioEmitter.shared.send(frame, tap: tap)
+        }
+    }
+
+    func stopAgentAudio() {
+        guard agentAudioTap != nil else { return }
+        agentAudioTap = nil
+        agentAudioOwner = nil
+        agentAudioOptions = nil
+        engine.stopAgentAudioTap()
+    }
+
+    /// The engine stops the tap when a join fails, but JS still has it on:
+    /// a call screen with useAgentAudio and a Retry button stays mounted.
+    /// Start it again under the same id, so the retry's frames reach the
+    /// same listener.
+    func agentAudioAfterFailedJoin() {
+        guard let tap = agentAudioTap, let options = agentAudioOptions,
+              !engine.isAgentAudioTapActive else { return }
+        try? startEngineTap(frameSize: options.frameSize, sampleRate: options.sampleRate, tap: tap)
+    }
+
+    /// leave() turns the tap off for JS too. Forgotten BEFORE the engine's
+    /// leave() runs (which stops the engine's tap first thing), so a join
+    /// that fails while leave() is still disconnecting cannot bring it
+    /// back; a start JS makes during leave() stays on.
+    func agentAudioBeforeLeave() {
+        agentAudioTap = nil
+        agentAudioOwner = nil
+        agentAudioOptions = nil
+    }
+
     private func updateWarning(_ kind: String, active: Bool, message: String) {
         let was = activeWarnings.contains(kind)
         guard was != active else { return }
@@ -219,6 +285,87 @@ final class CrowdPlayRNCore {
     }
 }
 
+/// The drain queue's view of the JS emitter, for agent audio frames. The
+/// engine (and so the tap's handler) outlives the module across JS reloads,
+/// so the module is held weakly; it is set while the module has listeners
+/// (startObserving/stopObserving), so nothing is sent with none.
+final class CrowdPlayRNAgentAudioEmitter: @unchecked Sendable {
+    static let shared = CrowdPlayRNAgentAudioEmitter()
+
+    private let lock = NSLock()
+    private weak var module: CrowdPlayRNModule?
+
+    func attach(_ module: CrowdPlayRNModule) {
+        lock.lock()
+        self.module = module
+        lock.unlock()
+    }
+
+    /// Only the module that is attached detaches: an old module's late
+    /// stopObserving (JS reload) must not silence the new one.
+    func detach(_ module: CrowdPlayRNModule) {
+        lock.lock()
+        if self.module === module { self.module = nil }
+        lock.unlock()
+    }
+
+    func send(_ frame: CallEngine.AgentAudioFrame, tap: Int) {
+        lock.lock()
+        let target = module
+        lock.unlock()
+        guard let target else { return }
+        let hostNow = mach_absolute_time()
+        let wallNowMs = Date().timeIntervalSince1970 * 1000
+        let latencyMs = outputLatencyMs(at: hostNow)
+        target.sendEvent(withName: "crowdplay:agentAudio", body: [
+            "pcm": frame.pcm.base64EncodedString(),
+            "sampleRate": frame.sampleRate,
+            "frameSize": frame.frameSize,
+            "seq": NSNumber(value: frame.seq),
+            "discontinuity": frame.discontinuity,
+            "tap": tap,
+            "heardAtMs": Self.heardAtMs(hostTime: frame.hostTime, hostNow: hostNow,
+                                        wallNowMs: wallNowMs, latencyMs: latencyMs),
+            "outputLatencyMs": latencyMs,
+        ] as [String: Any])
+    }
+
+    // MARK: When a frame is heard (lip sync)
+
+    /// Mach ticks per millisecond (24,000 on Apple silicon).
+    static let ticksPerMs: Double = {
+        var timebase = mach_timebase_info_data_t()
+        mach_timebase_info(&timebase)
+        return 1e6 * Double(timebase.denom) / Double(timebase.numer)
+    }()
+
+    /// The frame's first sample on the wall clock JS reads (Date.now()):
+    /// its playout pull (`hostTime`) plus the output latency. Mach time and
+    /// the wall clock are read together at send, so the conversion carries
+    /// no clock offset.
+    static func heardAtMs(hostTime: UInt64, hostNow: UInt64, wallNowMs: Double, latencyMs: Double) -> Double {
+        wallNowMs + (Double(hostTime) - Double(hostNow)) / ticksPerMs + latencyMs
+    }
+
+    /// Read only from send(), which the engine calls on its one drain
+    /// queue, so it needs no lock.
+    private var latencyMs = 0.0
+    private var latencyReadAt: UInt64 = 0
+
+    /// From a playout pull to the listener: one IO buffer plus the route's
+    /// reported output latency (a few ms on the speaker, 100 to 300 ms on
+    /// Bluetooth). Read from the audio session at most every 0.5 s, so a
+    /// route change shows within half a second.
+    private func outputLatencyMs(at hostNow: UInt64) -> Double {
+        if latencyReadAt == 0 || Double(hostNow &- latencyReadAt) > 500 * Self.ticksPerMs {
+            let session = AVAudioSession.sharedInstance()
+            latencyMs = (session.outputLatency + session.ioBufferDuration) * 1000
+            latencyReadAt = hostNow
+        }
+        return latencyMs
+    }
+}
+
 /// ObjC-callable hook for background upload completion. From an ObjC
 /// AppDelegate (via the app's generated -Swift.h header):
 ///
@@ -240,22 +387,66 @@ public final class CrowdPlayRNModule: RCTEventEmitter {
     public override static func requiresMainQueueSetup() -> Bool { true }
 
     public override func supportedEvents() -> [String]! {
-        ["crowdplay:phase", "crowdplay:recording", "crowdplay:uploads", "crowdplay:participants", "crowdplay:warning", "crowdplay:storageWarning", "crowdplay:audioRoute", "crowdplay:voiceActivity", "crowdplay:caption", "crowdplay:agentToolCall"]
+        ["crowdplay:phase", "crowdplay:recording", "crowdplay:uploads", "crowdplay:participants", "crowdplay:warning", "crowdplay:storageWarning", "crowdplay:audioRoute", "crowdplay:voiceActivity", "crowdplay:caption", "crowdplay:agentToolCall", "crowdplay:agentAudio"]
     }
 
     @objc public func startVoiceActivityUpdates() {
         Task { @MainActor in CrowdPlayRNCore.shared.startVoiceActivity() }
     }
 
+    /// The same stream at an app-chosen interval (useVoiceActivity's
+    /// intervalMs), floored at 33 ms. The zero-argument method above stays
+    /// for bridges and JS that predate it.
+    @objc public func startVoiceActivityUpdatesWithInterval(_ intervalMs: NSNumber) {
+        let ms = intervalMs.doubleValue
+        let seconds = (ms.isFinite ? max(ms, 33) : 150) / 1000
+        Task { @MainActor in CrowdPlayRNCore.shared.startVoiceActivity(interval: seconds) }
+    }
+
     @objc public func stopVoiceActivityUpdates() {
         Task { @MainActor in CrowdPlayRNCore.shared.stopVoiceActivity() }
     }
 
+    /// Starts the agent audio tap; resolves `{ tap }`, the id every frame of
+    /// this start carries. The JS layer validated the options already; the
+    /// engine validates again and a refusal leaves a running tap as it was.
+    @objc public func startAgentAudio(_ options: NSDictionary,
+                                      resolver: @escaping RCTPromiseResolveBlock,
+                                      rejecter: @escaping RCTPromiseRejectBlock) {
+        let frameSize = options["frameSize"] == nil ? 1024 : options["frameSize"] as? Int
+        let sampleRate = options["sampleRate"] == nil ? 16000 : options["sampleRate"] as? Int
+        guard let frameSize, let sampleRate else {
+            rejecter("agent_audio_invalid", "agent audio tap: frameSize and sampleRate must be integers", nil)
+            return
+        }
+        Task { @MainActor in
+            do {
+                let tap = try CrowdPlayRNCore.shared.startAgentAudio(
+                    frameSize: frameSize, sampleRate: sampleRate, owner: self)
+                resolver(["tap": tap])
+            } catch {
+                rejecter("agent_audio_invalid", error.localizedDescription, error)
+            }
+        }
+    }
+
+    @objc public func stopAgentAudio(_ resolve: @escaping RCTPromiseResolveBlock,
+                                     rejecter _: @escaping RCTPromiseRejectBlock) {
+        Task { @MainActor in
+            CrowdPlayRNCore.shared.stopAgentAudio()
+            resolve(nil)
+        }
+    }
+
     public override func startObserving() {
+        CrowdPlayRNAgentAudioEmitter.shared.attach(self)
         Task { @MainActor in CrowdPlayRNCore.shared.bind(self) }
     }
 
+    /// Never stops the agent audio tap: the listener count can touch 0 for
+    /// a moment during a remount. The tap's frames just stop being sent.
     public override func stopObserving() {
+        CrowdPlayRNAgentAudioEmitter.shared.detach(self)
         Task { @MainActor in CrowdPlayRNCore.shared.unbind() }
     }
 
@@ -355,6 +546,7 @@ public final class CrowdPlayRNModule: RCTEventEmitter {
                                    consent: consent,
                                    agent: agentOptions)
             if case let .failed(message) = core.engine.phase {
+                core.agentAudioAfterFailedJoin()
                 rejecter("join_failed", message, nil)
             } else {
                 resolver(nil)
@@ -365,6 +557,7 @@ public final class CrowdPlayRNModule: RCTEventEmitter {
     @objc public func leave(_ resolve: @escaping RCTPromiseResolveBlock,
                             rejecter _: @escaping RCTPromiseRejectBlock) {
         Task { @MainActor in
+            CrowdPlayRNCore.shared.agentAudioBeforeLeave()
             await CrowdPlayRNCore.shared.engine.leave()
             resolve(nil)
         }

package/ios/CrowdPlayReactNative.m

@@ -32,7 +32,13 @@ RCT_EXTERN_METHOD(retryRecording)
 RCT_EXTERN_METHOD(retryUploads)
 RCT_EXTERN_METHOD(warmUp)
 RCT_EXTERN_METHOD(startVoiceActivityUpdates)
+RCT_EXTERN_METHOD(startVoiceActivityUpdatesWithInterval : (nonnull NSNumber *)intervalMs)
 RCT_EXTERN_METHOD(stopVoiceActivityUpdates)
+RCT_EXTERN_METHOD(startAgentAudio : (NSDictionary *)options
+                  resolver : (RCTPromiseResolveBlock)resolve
+                  rejecter : (RCTPromiseRejectBlock)reject)
+RCT_EXTERN_METHOD(stopAgentAudio : (RCTPromiseResolveBlock)resolve
+                  rejecter : (RCTPromiseRejectBlock)reject)
 RCT_EXTERN_METHOD(snapshot : (RCTPromiseResolveBlock)resolve
                   rejecter : (RCTPromiseRejectBlock)reject)
 RCT_EXTERN_METHOD(doctor : (RCTPromiseResolveBlock)resolve

package/ios/wire.rb

@@ -6,7 +6,8 @@
 #   ruby node_modules/crowdplaysdk/ios/wire.rb <YourProjectName>
 #
 # What it does (idempotent; safe to re-run):
-#   1. adds the CrowdPlaySDK Swift package (compiled engine) to the project
+#   1. adds the CrowdPlaySDK Swift package (compiled engine) to the project,
+#      or re-pins an existing one to SDK_VERSION (an upgrade)
 #   2. copies the three bridge files into your app group + target
 #   3. creates/points the Swift bridging header at the React headers, raises
 #      the app target to iOS 17 and keeps explicit Swift modules ON for it
@@ -31,7 +32,8 @@ SDK_URL = 'https://github.com/symbiateam/crowdplaysdk'
 # this still at 0.4.0, which broke every install (D-121). publish_rn.sh now
 # compiles the bridge against exactly this release before publishing.
 # 0.5.4: the storage guard and engine.storageWarning (D-148).
-SDK_VERSION = '0.5.4'
+# 0.6.0: the agent audio tap (startAgentAudioTap), which the bridge calls.
+SDK_VERSION = '0.6.0'
 BRIDGE_FILES = ['CrowdPlayRNModule.swift', 'CrowdPlayRNVideoView.swift', 'CrowdPlayReactNative.m'].freeze
 
 project_name = ARGV[0] or abort 'usage: ruby wire.rb <YourProjectName>'
@@ -44,17 +46,30 @@ target = project.targets.find { |t| t.name == project_name } ||
          project.targets.find { |t| t.product_type == 'com.apple.product-type.application' }
 abort 'no application target found' unless target
 
-# 1. CrowdPlaySDK Swift package (skip when already present).
-unless project.root_object.package_references.any? { |r| r.respond_to?(:repositoryURL) && r.repositoryURL == SDK_URL }
+# 1. CrowdPlaySDK Swift package, pinned to exactly SDK_VERSION. An existing
+#    reference is re-pinned, never skipped: step 2 always replaces the bridge
+#    files, and a new bridge against the old engine does not compile (0.6.2
+#    upgraded in place kept engine 0.5.3 and failed on storageWarning, D-151).
+requirement = { 'kind' => 'exactVersion', 'version' => SDK_VERSION }
+existing = project.root_object.package_references.select { |r| r.respond_to?(:repositoryURL) && r.repositoryURL == SDK_URL }
+if existing.empty?
   pkg = project.new(Xcodeproj::Project::Object::XCRemoteSwiftPackageReference)
   pkg.repositoryURL = SDK_URL
-  pkg.requirement = { 'kind' => 'exactVersion', 'version' => SDK_VERSION }
+  pkg.requirement = requirement
   project.root_object.package_references << pkg
   dep = project.new(Xcodeproj::Project::Object::XCSwiftPackageProductDependency)
   dep.package = pkg
   dep.product_name = 'CrowdPlaySDK'
   target.package_product_dependencies << dep
   puts "+ CrowdPlaySDK package #{SDK_VERSION}"
+else
+  existing.each do |pkg|
+    next if pkg.requirement == requirement
+
+    was = (pkg.requirement || {})['version'] || (pkg.requirement || {})['minimumVersion'] || 'unpinned'
+    pkg.requirement = requirement
+    puts "+ CrowdPlaySDK package #{was} -> #{SDK_VERSION}"
+  end
 end
 
 # 2. Bridge files into the app group + target.

package/lib/VoiceView.d.ts

@@ -27,9 +27,15 @@ export interface VoiceActivityEvent {
     localSpeaking: boolean;
     connected: boolean;
 }
-/** Live phase + energy from the call engine. Subscribing starts the native
- * fast stream; the last unmount stops it. */
-export declare function useVoiceActivity(): VoiceActivity;
+export interface VoiceActivityOptions {
+    /** How often the native stream sends 'voiceActivity', in ms (floor 33).
+     * Default 150. With several hooks mounted, the stream runs at the
+     * smallest interval any of them asks for. */
+    intervalMs?: number;
+}
+/** Live phase + energy from the call engine. The first mounted hook starts
+ * the native fast stream; the last unmount stops it. */
+export declare function useVoiceActivity(options?: VoiceActivityOptions): VoiceActivity;
 export interface VoiceCaption {
     role: 'agent' | 'user';
     text: string;

package/lib/VoiceView.js

@@ -51,11 +51,51 @@ exports.VoiceReactive = VoiceReactive;
 exports.CrowdPlayVoiceView = CrowdPlayVoiceView;
 const react_1 = __importStar(require("react"));
 const react_native_1 = require("react-native");
-/** Live phase + energy from the call engine. Subscribing starts the native
- * fast stream; the last unmount stops it. */
-function useVoiceActivity() {
+/** Every mounted useVoiceActivity() shares ONE native stream: each hook
+ * registers its requested interval (undefined = the default) here, the
+ * stream runs at the smallest, and only the last unmount stops it. */
+const voiceRequests = new Map();
+/** What the native stream was last told: 'stopped', 'default' (the
+ * zero-argument start, 150 ms, exactly as before intervals existed), or
+ * an interval in ms. */
+let voiceStream = 'stopped';
+function voiceIntervalOf(intervalMs) {
+    if (typeof intervalMs !== 'number' || !Number.isFinite(intervalMs) || intervalMs <= 0)
+        return undefined;
+    return Math.max(33, Math.round(intervalMs));
+}
+function applyVoiceRequests(module) {
+    if (voiceRequests.size === 0) {
+        if (voiceStream !== 'stopped')
+            module.stopVoiceActivityUpdates?.();
+        voiceStream = 'stopped';
+        return;
+    }
+    let smallest;
+    for (const interval of voiceRequests.values()) {
+        if (interval !== undefined && (smallest === undefined || interval < smallest))
+            smallest = interval;
+    }
+    const want = smallest ?? 'default';
+    if (want === voiceStream)
+        return;
+    if (want !== 'default' && typeof module.startVoiceActivityUpdatesWithInterval === 'function') {
+        module.startVoiceActivityUpdatesWithInterval(want);
+    }
+    else {
+        // The default, or a bridge wired by an older package (fixed 150 ms).
+        module.startVoiceActivityUpdates?.();
+    }
+    voiceStream = want;
+}
+/** Live phase + energy from the call engine. The first mounted hook starts
+ * the native fast stream; the last unmount stops it. */
+function useVoiceActivity(options) {
     const [activity, setActivity] = (0, react_1.useState)({ phase: 'connecting', energy: 0 });
     const energyRef = (0, react_1.useRef)(0);
+    const token = (0, react_1.useRef)({}).current;
+    const intervalMs = voiceIntervalOf(options?.intervalMs);
+    const intervalRef = (0, react_1.useRef)(intervalMs);
     (0, react_1.useEffect)(() => {
         const module = react_native_1.NativeModules.CrowdPlayReactNative;
         if (!module)
@@ -86,12 +126,24 @@ function useVoiceActivity() {
             energyRef.current += (target - energyRef.current) * rate;
             setActivity({ phase, energy: energyRef.current });
         });
-        module.startVoiceActivityUpdates?.();
+        voiceRequests.set(token, intervalRef.current);
+        applyVoiceRequests(module);
         return () => {
             sub.remove();
-            module.stopVoiceActivityUpdates?.();
+            voiceRequests.delete(token);
+            applyVoiceRequests(module);
         };
-    }, []);
+    }, [token]);
+    // A changed interval updates this hook's request in place, so the shared
+    // stream never stops and restarts for it.
+    (0, react_1.useEffect)(() => {
+        intervalRef.current = intervalMs;
+        const module = react_native_1.NativeModules.CrowdPlayReactNative;
+        if (!module || !voiceRequests.has(token))
+            return;
+        voiceRequests.set(token, intervalMs);
+        applyVoiceRequests(module);
+    }, [token, intervalMs]);
     return activity;
 }
 /** The latest live transcript line (speech bubbles, subtitles). Captions

package/lib/agentAudio.d.ts

@@ -0,0 +1,96 @@
+/**
+ * The agent audio tap, JavaScript side (engine 0.6.0+): the voice AI's call
+ * audio as it plays on this phone, in fixed frames of 16-bit mono PCM, for
+ * lip sync, visemes and meters. This is CALL audio (Path A), already
+ * compressed and decoded by the call; it never touches the recording.
+ *
+ * Pure TypeScript with no react-native import: option rules, the base64
+ * decoder, and the controller that orders start/stop and filters frames.
+ * index.ts wires the controller to the native module.
+ */
+/** Options for CrowdPlay.startAgentAudio() and useAgentAudio(). */
+export interface AgentAudioOptions {
+    /** Samples per frame, an integer of at least 1. Default 1024 (64 ms at
+     * 16 kHz). frameSize / sampleRate must be 10 ms to 1 s. */
+    frameSize?: number;
+    /** Output rate in Hz, an integer in 8000...48000. Default 16000. The
+     * call audio is resampled to it and downmixed to mono. */
+    sampleRate?: number;
+}
+/** Payload of the 'agentAudio' event: one frame of the voice AI's call
+ * audio, pulled for playout. */
+export interface AgentAudioFrame {
+    /** Base64 of Int16 little-endian mono PCM, exactly `frameSize` samples.
+     * decodeAgentAudio(frame.pcm) gives the samples. */
+    pcm: string;
+    sampleRate: number;
+    frameSize: number;
+    /** 0, 1, 2... per start, contiguous across delivered frames. */
+    seq: number;
+    /** True when this frame does not directly continue the previous frame
+     * this listener received: its first frame, a track switch or reconnect,
+     * a format change, or lost audio. Reset any smoothing state. */
+    discontinuity: boolean;
+    /** Which start this frame belongs to (frames of a stopped tap are
+     * dropped before they reach the app). */
+    tap: number;
+    /** When the frame's first sample reaches the listener, in Unix ms on this
+     * phone's clock (compare with Date.now()): its playout pull plus
+     * `outputLatencyMs`. Show the frame's mouth shape at this moment. It is
+     * usually already past when the frame arrives on the speaker (a frame is
+     * sent once its last sample is pulled, 64 ms at the defaults) and 100 to
+     * 300 ms ahead on Bluetooth. */
+    heardAtMs: number;
+    /** The audio route's delay from playout to the listener, in ms: one IO
+     * buffer plus what iOS reports for the route (a few ms on the speaker
+     * and wired headphones, 100 to 300 ms on Bluetooth). Re-read at most
+     * every 0.5 s, so a route change shows within half a second. */
+    outputLatencyMs: number;
+}
+export declare const AGENT_AUDIO_DEFAULT_FRAME_SIZE = 1024;
+export declare const AGENT_AUDIO_DEFAULT_SAMPLE_RATE = 16000;
+/** The engine's rules (AgentAudioTap.validate), in the engine's order, so
+ * a bad option fails here with the same wording instead of in native code.
+ * Doubles divide identically in JS and Swift, so the 10 ms and 1 s edges
+ * agree exactly. */
+export declare function normalizeAgentAudioOptions(options?: AgentAudioOptions | null): {
+    frameSize: number;
+    sampleRate: number;
+};
+/**
+ * Decodes an agentAudio frame's `pcm` (base64 of Int16 little-endian mono)
+ * into samples, -32768...32767. Standard base64, padded or not; no atob
+ * needed. A trailing odd byte (never sent by the SDK) is ignored. Throws a
+ * TypeError on anything that is not base64.
+ */
+export declare function decodeAgentAudio(pcm: string): Int16Array;
+/** The native methods the controller needs; optional because a bridge
+ * wired by an older crowdplaysdk does not have them. */
+export interface AgentAudioNative {
+    startAgentAudio?(options: {
+        frameSize: number;
+        sampleRate: number;
+    }): Promise<{
+        tap: number;
+    } | null>;
+    stopAgentAudio?(): Promise<void>;
+}
+export declare const STALE_BRIDGE_MESSAGE: string;
+export interface AgentAudioController {
+    start(options?: AgentAudioOptions | null): Promise<void>;
+    stop(): Promise<void>;
+    /** Wraps an app listener: drops frames of any tap but the active one and
+     * marks the first frame it passes, and any frame that does not follow
+     * the previous one it passed, as a discontinuity. */
+    filter(listener: (frame: AgentAudioFrame) => void): (frame: AgentAudioFrame) => void;
+    /** The tap whose frames are delivered, or null (tests and diagnostics). */
+    readonly activeTap: number | null;
+}
+/**
+ * Orders every start and stop through one promise chain (separate native
+ * calls are not guaranteed to run in call order), and tracks which tap's
+ * frames may reach the app. A stop takes effect for the app the moment it
+ * is CALLED: frames still in flight from the bridge are dropped, so none
+ * arrive after stopAgentAudio() is called, let alone after it resolves.
+ */
+export declare function createAgentAudio(getNative: () => AgentAudioNative): AgentAudioController;