crowdplaysdk 0.4.2 → 0.5.0

package/ios/CrowdPlayRNModule.swift

@@ -52,6 +52,17 @@ final class CrowdPlayRNCore {
                 ])
             }.store(in: &cancellables)
 
+        // Agent tool calls (D-103): the JS layer answers via
+        // respondToAgentToolCall, so the native auto-handler stays unset.
+        engine.$latestAgentToolCall
+            .compactMap { $0 }
+            .sink { [weak self] call in
+                self?.send("crowdplay:agentToolCall", [
+                    "callId": call.callId, "name": call.name,
+                    "argumentsJSON": call.argumentsJSON,
+                ])
+            }.store(in: &cancellables)
+
         engine.$uploads
             .combineLatest(engine.$uploadsOnWiFi)
             .sink { [weak self] uploads, onWifi in
@@ -220,7 +231,7 @@ public final class CrowdPlayRNModule: RCTEventEmitter {
     public override static func requiresMainQueueSetup() -> Bool { true }
 
     public override func supportedEvents() -> [String]! {
-        ["crowdplay:phase", "crowdplay:recording", "crowdplay:uploads", "crowdplay:participants", "crowdplay:warning", "crowdplay:audioRoute", "crowdplay:voiceActivity", "crowdplay:caption"]
+        ["crowdplay:phase", "crowdplay:recording", "crowdplay:uploads", "crowdplay:participants", "crowdplay:warning", "crowdplay:audioRoute", "crowdplay:voiceActivity", "crowdplay:caption", "crowdplay:agentToolCall"]
     }
 
     @objc public func startVoiceActivityUpdates() {
@@ -294,8 +305,30 @@ public final class CrowdPlayRNModule: RCTEventEmitter {
             let voice = agent["voice"] as? String
             let prompt = agent["systemPrompt"] as? String
             let context = agent["context"] as? String
-            if voice != nil || prompt != nil || context != nil {
-                agentOptions = CrowdPlayAgentOptions(voice: voice, systemPrompt: prompt, context: context)
+            // Tools cross the bridge as one JSON string (D-103); the server
+            // re-validates every entry.
+            var tools: [CrowdPlayAgentTool]?
+            if let toolsJSON = agent["toolsJSON"] as? String,
+               let data = toolsJSON.data(using: .utf8),
+               let entries = try? JSONSerialization.jsonObject(with: data) as? [[String: Any]] {
+                let parsed: [CrowdPlayAgentTool] = entries.compactMap { entry in
+                    guard let name = entry["name"] as? String,
+                          let description = entry["description"] as? String else { return nil }
+                    var parametersJSON = "{}"
+                    if let parameters = entry["parameters"],
+                       JSONSerialization.isValidJSONObject(parameters),
+                       let pData = try? JSONSerialization.data(withJSONObject: parameters),
+                       let pString = String(data: pData, encoding: .utf8) {
+                        parametersJSON = pString
+                    }
+                    return CrowdPlayAgentTool(name: name, description: description,
+                                              parametersJSON: parametersJSON)
+                }
+                if !parsed.isEmpty { tools = parsed }
+            }
+            if voice != nil || prompt != nil || context != nil || tools != nil {
+                agentOptions = CrowdPlayAgentOptions(voice: voice, systemPrompt: prompt,
+                                                     context: context, tools: tools)
             }
         }
         Task { @MainActor in
@@ -360,6 +393,19 @@ public final class CrowdPlayRNModule: RCTEventEmitter {
         }
     }
 
+    @objc public func respondToAgentToolCall(_ callId: NSString, output: NSString) {
+        Task { @MainActor in
+            await CrowdPlayRNCore.shared.engine.respondToAgentToolCall(
+                callId: callId as String, output: output as String)
+        }
+    }
+
+    @objc public func updateAgentContext(_ text: NSString) {
+        Task { @MainActor in
+            await CrowdPlayRNCore.shared.engine.updateAgentContext(text as String)
+        }
+    }
+
     @objc public func retryRecording() {
         Task { @MainActor in
             let core = CrowdPlayRNCore.shared

package/ios/CrowdPlayReactNative.m

@@ -25,6 +25,9 @@ RCT_EXTERN_METHOD(setCameraEnabled : (BOOL)enabled
                   resolver : (RCTPromiseResolveBlock)resolve
                   rejecter : (RCTPromiseRejectBlock)reject)
 RCT_EXTERN_METHOD(setAudioOutput : (NSString *)output)
+RCT_EXTERN_METHOD(respondToAgentToolCall : (NSString *)callId
+                  output : (NSString *)output)
+RCT_EXTERN_METHOD(updateAgentContext : (NSString *)text)
 RCT_EXTERN_METHOD(retryRecording)
 RCT_EXTERN_METHOD(retryUploads)
 RCT_EXTERN_METHOD(warmUp)

package/lib/index.d.ts

@@ -74,6 +74,17 @@ export interface ConsentGrant {
  * apply to the whole room. The app can never change the provider, the
  * model, or the API key from here.
  */
+/** A tool the app offers the voice AI for one session (D-103). Requires
+ * the app's dashboard config to use the GPT-Live provider with an AI brain
+ * (delegation) enabled; ignored otherwise. */
+export interface AgentTool {
+    /** Letters, digits, underscore, dash; max 64 chars. */
+    name: string;
+    /** What the tool does and when to use it, for the model. */
+    description: string;
+    /** JSON-schema object describing the arguments. Omit for none. */
+    parameters?: Record<string, unknown>;
+}
 export interface AgentOptions {
     /** A voice of the app's configured provider. Gemini: Charon, Aoede,
      * Fenrir, Kore, Puck. OpenAI: marin, cedar, alloy, ash, ballad, coral,
@@ -87,6 +98,9 @@ export interface AgentOptions {
      * keep a few sentences per user in your own storage and pass them
      * here. Max 4000 chars. */
     context?: string;
+    /** Tools the AI may call this session (D-103). Handle calls with
+     * CrowdPlay.onAgentToolCall(). Max 16. */
+    tools?: AgentTool[];
 }
 export type Phase = 'idle' | 'connecting' | 'connected' | 'failed';
 export interface RecordingSnapshot {
@@ -167,6 +181,13 @@ export interface CrowdPlayEvents {
         detectedOutputs: string[];
         headphonesConnected: boolean;
     };
+    /** The voice AI asked the app to run a tool (D-103). Prefer
+     * CrowdPlay.onAgentToolCall(), which answers for you. */
+    agentToolCall: {
+        callId: string;
+        name: string;
+        argumentsJSON: string;
+    };
 }
 export type CrowdPlayEventName = keyof CrowdPlayEvents;
 declare const CrowdPlay: {
@@ -216,6 +237,19 @@ declare const CrowdPlay: {
     /** Integration self-check: configuration, permissions, disk, backend
      * reachability, app-key auth. Every failing check names its fix. */
     doctor(): Promise<DoctorCheck[]>;
+    /** Handle the voice AI's tool calls (D-103). The handler gets the tool
+     * name and its arguments (parsed JSON object) and returns the result as
+     * an object or JSON string; return null to decline on this device
+     * (another participant's device may answer; first answer wins). Ordinary
+     * app code, no AI here. The AI waits about 12 seconds per call. Returns
+     * a subscription; call .remove(). */
+    onAgentToolCall(handler: (name: string, args: Record<string, unknown>) => Promise<Record<string, unknown> | string | null> | Record<string, unknown> | string | null): {
+        remove(): void;
+    };
+    /** Tell the AI what just happened in the app, mid-session ("Night ended.
+     * Sam was eliminated."). Keep it to a sentence or two; long pushes are
+     * truncated. GPT-Live provider only; other providers ignore it. */
+    updateAgentContext(text: string): void;
     /** Subscribe to state events. Returns a subscription; call .remove(). */
     addListener<E extends CrowdPlayEventName>(event: E, listener: (payload: CrowdPlayEvents[E]) => void): {
         remove(): void;

package/lib/index.js

@@ -102,6 +102,11 @@ const CrowdPlay = {
             put('voice', agent.voice, 60);
             put('systemPrompt', agent.systemPrompt, 8192);
             put('context', agent.context, 4000);
+            // Tools cross the bridge as one JSON string; the native layer and the
+            // server both re-validate (D-103).
+            if (Array.isArray(agent.tools) && agent.tools.length) {
+                put('toolsJSON', JSON.stringify(agent.tools.slice(0, 16)), 8192);
+            }
         }
         return native().join(displayName.trim(), roomCode, consent.grantedAtMs, agentFields);
     },
@@ -144,6 +149,37 @@ const CrowdPlay = {
     doctor() {
         return native().doctor();
     },
+    /** Handle the voice AI's tool calls (D-103). The handler gets the tool
+     * name and its arguments (parsed JSON object) and returns the result as
+     * an object or JSON string; return null to decline on this device
+     * (another participant's device may answer; first answer wins). Ordinary
+     * app code, no AI here. The AI waits about 12 seconds per call. Returns
+     * a subscription; call .remove(). */
+    onAgentToolCall(handler) {
+        const subscription = events().addListener('crowdplay:agentToolCall', (payload) => {
+            void (async () => {
+                let args = {};
+                try {
+                    args = JSON.parse(payload.argumentsJSON);
+                }
+                catch { /* {} */ }
+                const result = await handler(payload.name, args);
+                if (result === null || result === undefined)
+                    return;
+                const output = typeof result === 'string' ? result : JSON.stringify(result);
+                native().respondToAgentToolCall(payload.callId, output);
+            })();
+        });
+        return { remove: () => subscription.remove() };
+    },
+    /** Tell the AI what just happened in the app, mid-session ("Night ended.
+     * Sam was eliminated."). Keep it to a sentence or two; long pushes are
+     * truncated. GPT-Live provider only; other providers ignore it. */
+    updateAgentContext(text) {
+        if (typeof text === 'string' && text.trim()) {
+            native().updateAgentContext(text.trim());
+        }
+    },
     /** Subscribe to state events. Returns a subscription; call .remove(). */
     addListener(event, listener) {
         const subscription = events().addListener(`crowdplay:${event}`, listener);

package/llms.txt

@@ -169,13 +169,38 @@ the voice and the persona and give a short context text for one session:
     agent: { voice: 'Aoede', context: "The user's name is Sam. Last time you practised past tense." } });
 voice must be one of the configured provider's voices (Gemini: Charon,
 Aoede, Fenrir, Kore, Puck; OpenAI: marin, cedar, alloy, ash, ballad,
-coral, echo, sage, shimmer, verse; Grok: Ara, Rex, Sal, Eve, Leo), unknown
+coral, echo, sage, shimmer, verse; Grok: Ara, Rex, Sal, Eve, Leo;
+OpenAI GPT-Live: quartz, ripple, vesper, willow, stone, gleam, meridian,
+bossa, tempo, beacon, delta, cinder), unknown
 falls back to the dashboard voice. Caps: systemPrompt 8192 chars, context
 4000. The options of the join that CREATES the AI apply to the whole room;
 fixed for the session (leave and join again to change); provider, model
 and key cannot be changed from the app. MEMORY is built with `context`:
 keep a few sentences per user in the app's own storage and pass them at
 every join.
+Agent tools + live context (0.5.0+, D-103; needs the app's dashboard
+config on the OpenAI GPT-Live provider with an AI brain enabled —
+ignored otherwise, never an error). Tools let the AI SEE app state and
+ACT in the app: pass them at join, handle calls with ordinary JS.
+  await CrowdPlay.join({ displayName, roomCode, consent: grant,
+    agent: { tools: [
+      { name: 'get_game_state',
+        description: 'The current players, roles you know, and votes.' },
+      { name: 'cast_vote', description: 'Vote to eliminate one player.',
+        parameters: { type: 'object',
+          properties: { player: { type: 'string' } }, required: ['player'] } },
+    ] } });
+  const sub = CrowdPlay.onAgentToolCall(async (name, args) => {
+    if (name === 'get_game_state') return game.state();
+    if (name === 'cast_vote') return game.castVote(args.player);
+    return null; // decline on this device; first device to answer wins
+  });
+Return an object (or JSON string), or null to decline. The AI waits
+about 12 seconds per call, then carries on gracefully. Max 16 tools;
+names letters/digits/underscore/dash up to 64 chars. To push events to
+the AI as they happen (it has no other way to know):
+CrowdPlay.updateAgentContext('Night ended. Sam was eliminated.') — a
+sentence or two, long pushes are truncated.
 Conversations are recorded and transcribed by the platform automatically.
 RECOMMENDED: voice-AI apps should be AUDIO-ONLY (audioOnly: true; no
 camera permission, no video anywhere); a camera adds nothing to talking

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crowdplaysdk",
-  "version": "0.4.2",
+  "version": "0.5.0",
   "description": "CrowdPlay lossless conversation capture for React Native (iOS). Studio-grade per-participant recording during live video calls, delivered to CrowdPlay automatically.",
   "main": "lib/index.js",
   "types": "lib/index.d.ts",

package/src/index.ts

@@ -84,6 +84,18 @@ export interface ConsentGrant {
  * apply to the whole room. The app can never change the provider, the
  * model, or the API key from here.
  */
+/** A tool the app offers the voice AI for one session (D-103). Requires
+ * the app's dashboard config to use the GPT-Live provider with an AI brain
+ * (delegation) enabled; ignored otherwise. */
+export interface AgentTool {
+  /** Letters, digits, underscore, dash; max 64 chars. */
+  name: string;
+  /** What the tool does and when to use it, for the model. */
+  description: string;
+  /** JSON-schema object describing the arguments. Omit for none. */
+  parameters?: Record<string, unknown>;
+}
+
 export interface AgentOptions {
   /** A voice of the app's configured provider. Gemini: Charon, Aoede,
    * Fenrir, Kore, Puck. OpenAI: marin, cedar, alloy, ash, ballad, coral,
@@ -97,6 +109,9 @@ export interface AgentOptions {
    * keep a few sentences per user in your own storage and pass them
    * here. Max 4000 chars. */
   context?: string;
+  /** Tools the AI may call this session (D-103). Handle calls with
+   * CrowdPlay.onAgentToolCall(). Max 16. */
+  tools?: AgentTool[];
 }
 
 export type Phase = 'idle' | 'connecting' | 'connected' | 'failed';
@@ -164,6 +179,9 @@ export interface CrowdPlayEvents {
    * "iPhone speaker"), connected candidates, and whether headphones are
    * attached. Drive the call screen's output menu from this (D-079). */
   audioRoute: { currentOutputName: string; detectedOutputs: string[]; headphonesConnected: boolean };
+  /** The voice AI asked the app to run a tool (D-103). Prefer
+   * CrowdPlay.onAgentToolCall(), which answers for you. */
+  agentToolCall: { callId: string; name: string; argumentsJSON: string };
 }
 
 export type CrowdPlayEventName = keyof CrowdPlayEvents;
@@ -177,6 +195,8 @@ interface NativeCrowdPlay {
   consentText(): Promise<string>;
   join(displayName: string, roomCode: string, consentGrantedAtMs: number,
        agent: Record<string, string> | null): Promise<void>;
+  respondToAgentToolCall(callId: string, output: string): void;
+  updateAgentContext(text: string): void;
   leave(): Promise<void>;
   setMicMuted(muted: boolean): Promise<void>;
   setCameraEnabled(enabled: boolean): Promise<void>;
@@ -279,6 +299,11 @@ const CrowdPlay = {
       put('voice', agent.voice, 60);
       put('systemPrompt', agent.systemPrompt, 8192);
       put('context', agent.context, 4000);
+      // Tools cross the bridge as one JSON string; the native layer and the
+      // server both re-validate (D-103).
+      if (Array.isArray(agent.tools) && agent.tools.length) {
+        put('toolsJSON', JSON.stringify(agent.tools.slice(0, 16)), 8192);
+      }
     }
     return native().join(displayName.trim(), roomCode, consent.grantedAtMs, agentFields);
   },
@@ -330,6 +355,41 @@ const CrowdPlay = {
     return native().doctor();
   },
 
+  /** Handle the voice AI's tool calls (D-103). The handler gets the tool
+   * name and its arguments (parsed JSON object) and returns the result as
+   * an object or JSON string; return null to decline on this device
+   * (another participant's device may answer; first answer wins). Ordinary
+   * app code, no AI here. The AI waits about 12 seconds per call. Returns
+   * a subscription; call .remove(). */
+  onAgentToolCall(
+    handler: (name: string, args: Record<string, unknown>) =>
+      Promise<Record<string, unknown> | string | null> | Record<string, unknown> | string | null
+  ): { remove(): void } {
+    const subscription = events().addListener(
+      'crowdplay:agentToolCall',
+      (payload: { callId: string; name: string; argumentsJSON: string }) => {
+        void (async () => {
+          let args: Record<string, unknown> = {};
+          try { args = JSON.parse(payload.argumentsJSON); } catch { /* {} */ }
+          const result = await handler(payload.name, args);
+          if (result === null || result === undefined) return;
+          const output = typeof result === 'string' ? result : JSON.stringify(result);
+          native().respondToAgentToolCall(payload.callId, output);
+        })();
+      }
+    );
+    return { remove: () => subscription.remove() };
+  },
+
+  /** Tell the AI what just happened in the app, mid-session ("Night ended.
+   * Sam was eliminated."). Keep it to a sentence or two; long pushes are
+   * truncated. GPT-Live provider only; other providers ignore it. */
+  updateAgentContext(text: string): void {
+    if (typeof text === 'string' && text.trim()) {
+      native().updateAgentContext(text.trim());
+    }
+  },
+
   /** Subscribe to state events. Returns a subscription; call .remove(). */
   addListener<E extends CrowdPlayEventName>(
     event: E,