crowdplaysdk 0.4.1 → 0.5.0

package/src/index.ts

@@ -1,11 +1,11 @@
 /**
- * crowdplaysdk — CrowdPlay's lossless capture SDK for React Native (iOS).
+ * crowdplaysdk: CrowdPlay's lossless capture SDK for React Native (iOS).
  *
  * The recording engine is 100% native (CrowdPlaySDK): studio-grade audio
  * (48 kHz / 24-bit, unprocessed) + 1080p30 video captured locally during a
  * live call, clock-synced across participants, uploaded to CrowdPlay with
- * retries and crash recovery. **No media ever crosses the JS bridge** —
- * only control calls and state events — so recording quality is identical
+ * retries and crash recovery. **No media ever crosses the JS bridge**,
+ * only control calls and state events, so recording quality is identical
  * to a fully native app.
  *
  *     import CrowdPlay, { CrowdPlayConsentScreen, CrowdPlayVideoView } from 'crowdplaysdk';
@@ -17,7 +17,7 @@
  *     await CrowdPlay.leave(); // recording stops; uploads continue automatically
  *
  * Consent is enforced twice: join() throws here without a grant, and the
- * native layer requires the consent record structurally — there is no code
+ * native layer requires the consent record structurally; there is no code
  * path that records without it.
  */
 
@@ -45,7 +45,7 @@ export interface CrowdPlayConfig {
   serverUrl: string;
   /** From the CrowdPlay dashboard. Also your data-access credential. */
   appKey: string;
-  /** AUDIO-ONLY app: the camera is never touched — no camera permission,
+  /** AUDIO-ONLY app: the camera is never touched: no camera permission,
    * no video captured or uploaded, voice-only calls. The audio path is
    * identical to an A/V app. Requires engine 0.3.0+. Default false. */
   audioOnly?: boolean;
@@ -54,14 +54,22 @@ export interface CrowdPlayConfig {
   videoUploadsOnCellular?: boolean;
   /** Upload segments while the call records, so the post-call wait is
    * roughly the final segment + manifests. Mid-call transfers run one at
-   * a time and pause automatically when call quality drops — the call
+   * a time and pause automatically when call quality drops; the call
    * always wins the uplink. Default true. */
   uploadDuringCall?: boolean;
+  /** What happens to OTHER apps' audio (Apple Music, Spotify, podcasts)
+   * while a call is live. 'interrupt' (default) pauses them at join, the
+   * classic call behaviour. 'mix' keeps them playing alongside the call
+   * and the recording, for shared-listening and DJ apps; 'mixDucked' also
+   * lowers them for the whole call. Music stays on the device and never
+   * rides the call; listeners must wear headphones or it lands in the
+   * recording. Requires engine 0.4.0+. */
+  otherAudio?: 'interrupt' | 'mix' | 'mixDucked';
   callQuality?: CallQuality;
 }
 
-/** Proof the participant agreed. Produce it with CrowdPlayConsentScreen, or —
- * if you render your own consent UI — call CrowdPlay.consentText() to show the
+/** Proof the participant agreed. Produce it with CrowdPlayConsentScreen, or,
+ * if you render your own consent UI, call CrowdPlay.consentText() to show the
  * REQUIRED wording and construct the grant at the moment of agreement. */
 export interface ConsentGrant {
   /** Date.now() at the moment the participant agreed. */
@@ -76,6 +84,18 @@ export interface ConsentGrant {
  * apply to the whole room. The app can never change the provider, the
  * model, or the API key from here.
  */
+/** A tool the app offers the voice AI for one session (D-103). Requires
+ * the app's dashboard config to use the GPT-Live provider with an AI brain
+ * (delegation) enabled; ignored otherwise. */
+export interface AgentTool {
+  /** Letters, digits, underscore, dash; max 64 chars. */
+  name: string;
+  /** What the tool does and when to use it, for the model. */
+  description: string;
+  /** JSON-schema object describing the arguments. Omit for none. */
+  parameters?: Record<string, unknown>;
+}
+
 export interface AgentOptions {
   /** A voice of the app's configured provider. Gemini: Charon, Aoede,
    * Fenrir, Kore, Puck. OpenAI: marin, cedar, alloy, ash, ballad, coral,
@@ -89,6 +109,9 @@ export interface AgentOptions {
    * keep a few sentences per user in your own storage and pass them
    * here. Max 4000 chars. */
   context?: string;
+  /** Tools the AI may call this session (D-103). Handle calls with
+   * CrowdPlay.onAgentToolCall(). Max 16. */
+  tools?: AgentTool[];
 }
 
 export type Phase = 'idle' | 'connecting' | 'connected' | 'failed';
@@ -100,10 +123,10 @@ export interface RecordingSnapshot {
   droppedSamples: number;
   /** Live input level. Speech sits around −25…−45 dBFS. */
   inputLevelDbfs: number;
-  /** True after 10 s of silence while unmuted — a dead/disconnected mic.
+  /** True after 10 s of silence while unmuted: a dead/disconnected mic.
    * Surface this loudly; it is fixable mid-call. */
   inputSilent: boolean;
-  /** Samples at digital full scale — a too-hot mic, irreversibly clipped. */
+  /** Samples at digital full scale: a too-hot mic, irreversibly clipped. */
   fullScaleSamples: number;
 }
 
@@ -141,7 +164,7 @@ export interface CrowdPlayEvents {
   /** Connection lifecycle. `error` is set when phase === 'failed'. */
   phase: { phase: Phase; error?: string };
   /** Recording state. `error` non-null means the session is NOT being
-   * captured — show it prominently and offer retryRecording(). */
+   * captured; show it prominently and offer retryRecording(). */
   recording: { isRecording: boolean; error?: string };
   /** Upload progress for every session with outstanding or recent work. */
   uploads: { sessions: UploadProgress[]; onWifi: boolean };
@@ -156,6 +179,9 @@ export interface CrowdPlayEvents {
    * "iPhone speaker"), connected candidates, and whether headphones are
    * attached. Drive the call screen's output menu from this (D-079). */
   audioRoute: { currentOutputName: string; detectedOutputs: string[]; headphonesConnected: boolean };
+  /** The voice AI asked the app to run a tool (D-103). Prefer
+   * CrowdPlay.onAgentToolCall(), which answers for you. */
+  agentToolCall: { callId: string; name: string; argumentsJSON: string };
 }
 
 export type CrowdPlayEventName = keyof CrowdPlayEvents;
@@ -169,6 +195,8 @@ interface NativeCrowdPlay {
   consentText(): Promise<string>;
   join(displayName: string, roomCode: string, consentGrantedAtMs: number,
        agent: Record<string, string> | null): Promise<void>;
+  respondToAgentToolCall(callId: string, output: string): void;
+  updateAgentContext(text: string): void;
   leave(): Promise<void>;
   setMicMuted(muted: boolean): Promise<void>;
   setCameraEnabled(enabled: boolean): Promise<void>;
@@ -189,8 +217,8 @@ function native(): NativeCrowdPlay {
       "crowdplaysdk: native module not found. iOS setup: add the CrowdPlaySDK " +
         "Swift package (https://github.com/symbiateam/crowdplaysdk) to your Xcode " +
         "project and add the two bridge files from node_modules/crowdplaysdk/ios " +
-        "to your app target — see the package README. " +
-        (Platform.OS !== 'ios' ? `(platform '${Platform.OS}' is not supported yet — iOS only.)` : '')
+        "to your app target; see the package README. " +
+        (Platform.OS !== 'ios' ? `(platform '${Platform.OS}' is not supported yet; iOS only.)` : '')
     );
   }
   return module;
@@ -240,7 +268,7 @@ const CrowdPlay = {
    * Join a room. Recording starts automatically on join and stops on
    * leave(). Rejects if the connection fails.
    *
-   * `consent` is REQUIRED — produce it with <CrowdPlayConsentScreen> or from
+   * `consent` is REQUIRED; produce it with <CrowdPlayConsentScreen> or from
    * your own UI at the moment of agreement. There is no way to record
    * without it, here or natively.
    */
@@ -271,6 +299,11 @@ const CrowdPlay = {
       put('voice', agent.voice, 60);
       put('systemPrompt', agent.systemPrompt, 8192);
       put('context', agent.context, 4000);
+      // Tools cross the bridge as one JSON string; the native layer and the
+      // server both re-validate (D-103).
+      if (Array.isArray(agent.tools) && agent.tools.length) {
+        put('toolsJSON', JSON.stringify(agent.tools.slice(0, 16)), 8192);
+      }
     }
     return native().join(displayName.trim(), roomCode, consent.grantedAtMs, agentFields);
   },
@@ -322,6 +355,41 @@ const CrowdPlay = {
     return native().doctor();
   },
 
+  /** Handle the voice AI's tool calls (D-103). The handler gets the tool
+   * name and its arguments (parsed JSON object) and returns the result as
+   * an object or JSON string; return null to decline on this device
+   * (another participant's device may answer; first answer wins). Ordinary
+   * app code, no AI here. The AI waits about 12 seconds per call. Returns
+   * a subscription; call .remove(). */
+  onAgentToolCall(
+    handler: (name: string, args: Record<string, unknown>) =>
+      Promise<Record<string, unknown> | string | null> | Record<string, unknown> | string | null
+  ): { remove(): void } {
+    const subscription = events().addListener(
+      'crowdplay:agentToolCall',
+      (payload: { callId: string; name: string; argumentsJSON: string }) => {
+        void (async () => {
+          let args: Record<string, unknown> = {};
+          try { args = JSON.parse(payload.argumentsJSON); } catch { /* {} */ }
+          const result = await handler(payload.name, args);
+          if (result === null || result === undefined) return;
+          const output = typeof result === 'string' ? result : JSON.stringify(result);
+          native().respondToAgentToolCall(payload.callId, output);
+        })();
+      }
+    );
+    return { remove: () => subscription.remove() };
+  },
+
+  /** Tell the AI what just happened in the app, mid-session ("Night ended.
+   * Sam was eliminated."). Keep it to a sentence or two; long pushes are
+   * truncated. GPT-Live provider only; other providers ignore it. */
+  updateAgentContext(text: string): void {
+    if (typeof text === 'string' && text.trim()) {
+      native().updateAgentContext(text.trim());
+    }
+  },
+
   /** Subscribe to state events. Returns a subscription; call .remove(). */
   addListener<E extends CrowdPlayEventName>(
     event: E,