crowdplaysdk 0.3.3 → 0.3.4

package/ios/CrowdPlayRNModule.swift

@@ -42,6 +42,16 @@ final class CrowdPlayRNCore {
                 self?.send("crowdplay:recording", body)
             }.store(in: &cancellables)
 
+        // Live captions (M8 voice UI): each line the AI or user speaks, as
+        // broadcast by the agent on the data channel.
+        engine.$latestCaption
+            .compactMap { $0 }
+            .sink { [weak self] caption in
+                self?.send("crowdplay:caption", [
+                    "role": caption.role, "text": caption.text,
+                ])
+            }.store(in: &cancellables)
+
         engine.$uploads
             .combineLatest(engine.$uploadsOnWiFi)
             .sink { [weak self] uploads, onWifi in
@@ -210,7 +220,7 @@ public final class CrowdPlayRNModule: RCTEventEmitter {
     public override static func requiresMainQueueSetup() -> Bool { true }
 
     public override func supportedEvents() -> [String]! {
-        ["crowdplay:phase", "crowdplay:recording", "crowdplay:uploads", "crowdplay:participants", "crowdplay:warning", "crowdplay:audioRoute", "crowdplay:voiceActivity"]
+        ["crowdplay:phase", "crowdplay:recording", "crowdplay:uploads", "crowdplay:participants", "crowdplay:warning", "crowdplay:audioRoute", "crowdplay:voiceActivity", "crowdplay:caption"]
     }
 
     @objc public func startVoiceActivityUpdates() {

package/ios/wire.rb

@@ -25,7 +25,7 @@ SDK_URL = 'https://github.com/symbiateam/crowdplaysdk'
 # The ENGINE release this bridge was tested against. The npm package
 # version may be AHEAD of this (bridge/docs fixes ship without a new
 # engine binary); that is deliberate, not drift.
-SDK_VERSION = '0.3.3'
+SDK_VERSION = '0.3.4'
 BRIDGE_FILES = ['CrowdPlayRNModule.swift', 'CrowdPlayRNVideoView.swift', 'CrowdPlayReactNative.m'].freeze
 
 project_name = ARGV[0] or abort 'usage: ruby wire.rb <YourProjectName>'

package/lib/VoiceView.d.ts

@@ -20,7 +20,24 @@ export interface VoiceActivity {
 /** Live phase + energy from the call engine. Subscribing starts the native
  * fast stream; the last unmount stops it. */
 export declare function useVoiceActivity(): VoiceActivity;
+export interface VoiceCaption {
+    role: 'agent' | 'user';
+    text: string;
+}
+/** The latest live transcript line (speech bubbles, subtitles). Captions
+ * are broadcast by the voice agent as it speaks; null until the first one. */
+export declare function useLatestCaption(): VoiceCaption | null;
+/** Make ANY child view breathe, pulse, and glow with the conversation —
+ * the one-line path from the app's own art (mascot, logo, character) to a
+ * living AI presence. */
+export declare function VoiceReactive({ children, glowColor, }: {
+    children: React.ReactNode;
+    glowColor?: string;
+}): React.JSX.Element;
 export interface CrowdPlayVoiceViewProps {
+    /** Built-in visual shape: the breathing orb (default) or equalizer bars.
+     * Passing children switches to the halo (ring + glow around YOUR art). */
+    variant?: 'orb' | 'bars';
     /** Orb color while the AI is speaking. */
     agentColor?: string;
     /** Ring color while the user is speaking. */
@@ -29,5 +46,8 @@ export interface CrowdPlayVoiceViewProps {
     idleColor?: string;
     /** Diameter in points. */
     size?: number;
+    /** App-provided center content (mascot portrait, logo): rendered inside
+     * an animated halo instead of the orb. */
+    children?: React.ReactNode;
 }
-export declare function CrowdPlayVoiceView({ agentColor, listeningColor, idleColor, size, }: CrowdPlayVoiceViewProps): React.JSX.Element;
+export declare function CrowdPlayVoiceView({ variant, agentColor, listeningColor, idleColor, size, children, }: CrowdPlayVoiceViewProps): React.JSX.Element;

package/lib/VoiceView.js

@@ -46,6 +46,8 @@ var __importStar = (this && this.__importStar) || (function () {
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.useVoiceActivity = useVoiceActivity;
+exports.useLatestCaption = useLatestCaption;
+exports.VoiceReactive = VoiceReactive;
 exports.CrowdPlayVoiceView = CrowdPlayVoiceView;
 const react_1 = __importStar(require("react"));
 const react_native_1 = require("react-native");
@@ -92,7 +94,45 @@ function useVoiceActivity() {
     }, []);
     return activity;
 }
-function CrowdPlayVoiceView({ agentColor = '#598CFF', listeningColor = '#4DD9B3', idleColor = '#BFBFBF', size = 180, }) {
+/** The latest live transcript line (speech bubbles, subtitles). Captions
+ * are broadcast by the voice agent as it speaks; null until the first one. */
+function useLatestCaption() {
+    const [caption, setCaption] = (0, react_1.useState)(null);
+    (0, react_1.useEffect)(() => {
+        const module = react_native_1.NativeModules.CrowdPlayReactNative;
+        if (!module)
+            return;
+        const emitter = new react_native_1.NativeEventEmitter(module);
+        const sub = emitter.addListener('crowdplay:caption', (e) => setCaption(e));
+        return () => sub.remove();
+    }, []);
+    return caption;
+}
+/** Make ANY child view breathe, pulse, and glow with the conversation —
+ * the one-line path from the app's own art (mascot, logo, character) to a
+ * living AI presence. */
+function VoiceReactive({ children, glowColor = '#598CFF', }) {
+    const { energy } = useVoiceActivity();
+    const scale = (0, react_1.useRef)(new react_native_1.Animated.Value(1)).current;
+    const glow = (0, react_1.useRef)(new react_native_1.Animated.Value(0)).current;
+    (0, react_1.useEffect)(() => {
+        react_native_1.Animated.spring(scale, {
+            toValue: 1 + 0.08 * energy, useNativeDriver: true, speed: 20, bounciness: 12,
+        }).start();
+        react_native_1.Animated.timing(glow, {
+            toValue: energy, duration: 120, useNativeDriver: true,
+        }).start();
+    }, [energy, scale, glow]);
+    return (<react_native_1.View style={styles.container}>
+      <react_native_1.Animated.View pointerEvents="none" style={[react_native_1.StyleSheet.absoluteFillObject, styles.reactiveGlow, {
+                backgroundColor: glowColor,
+                opacity: react_native_1.Animated.multiply(glow, 0.35),
+                transform: [{ scale: 1.25 }],
+            }]}/>
+      <react_native_1.Animated.View style={{ transform: [{ scale }] }}>{children}</react_native_1.Animated.View>
+    </react_native_1.View>);
+}
+function CrowdPlayVoiceView({ variant = 'orb', agentColor = '#598CFF', listeningColor = '#4DD9B3', idleColor = '#BFBFBF', size = 180, children, }) {
     const { phase, energy } = useVoiceActivity();
     const breathe = (0, react_1.useRef)(new react_native_1.Animated.Value(0)).current;
     const scale = (0, react_1.useRef)(new react_native_1.Animated.Value(0.62)).current;
@@ -137,20 +177,55 @@ function CrowdPlayVoiceView({ agentColor = '#598CFF', listeningColor = '#4DD9B3'
                 opacity: 0.18 * dim + 0.25 * energy,
                 transform: [{ scale: react_native_1.Animated.multiply(halo, breatheScale) }],
             }]}/>
-      {phase === 'listening' && (<react_native_1.View style={[styles.circle, {
+      {(phase === 'listening' || children != null) && (<react_native_1.View style={[styles.circle, {
                     width: size * 0.8, height: size * 0.8, borderRadius: size * 0.4,
-                    borderWidth: Math.max(2, size * 0.015), borderColor: color,
+                    borderWidth: Math.max(2, size * (children != null ? 0.02 : 0.015)),
+                    borderColor: color,
                     backgroundColor: 'transparent',
                 }]}/>)}
-      <react_native_1.Animated.View style={[styles.circle, {
-                width: size, height: size, borderRadius: size / 2,
-                backgroundColor: color,
-                opacity: dim,
+      {children != null ? (
+        // Halo mode: the app's own art at the center, ring + glow around it.
+        <react_native_1.Animated.View style={{
+                width: size * 0.72, height: size * 0.72,
+                borderRadius: size * 0.36, overflow: 'hidden',
                 transform: [{ scale: react_native_1.Animated.multiply(scale, breatheScale) }],
-            }]}/>
+            }}>
+          {children}
+        </react_native_1.Animated.View>) : variant === 'bars' ? (<BarsVisual color={color} dim={dim} energy={energy} size={size}/>) : (<react_native_1.Animated.View style={[styles.circle, {
+                    width: size, height: size, borderRadius: size / 2,
+                    backgroundColor: color,
+                    opacity: dim,
+                    transform: [{ scale: react_native_1.Animated.multiply(scale, breatheScale) }],
+                }]}/>)}
+    </react_native_1.View>);
+}
+/** Equalizer-style bars: heights mix energy with per-bar sines so they
+ * dance during speech and rest at a low idle. */
+function BarsVisual({ color, dim, energy, size }) {
+    const [tick, setTick] = (0, react_1.useState)(0);
+    (0, react_1.useEffect)(() => {
+        const id = setInterval(() => setTick((t) => t + 1), 90);
+        return () => clearInterval(id);
+    }, []);
+    const t = tick * 0.09;
+    return (<react_native_1.View style={styles.barsRow}>
+      {[0, 1, 2, 3, 4].map((index) => {
+            const phase = index * 1.3;
+            const wave = 0.5 + 0.5 * Math.sin(t * (5 + index * 1.7) + phase);
+            const height = 0.15 + 0.08 * Math.sin(t * 1.4 + phase) + energy * wave * 0.8;
+            return (<react_native_1.View key={index} style={{
+                    width: size * 0.1,
+                    height: size * Math.min(1, height),
+                    borderRadius: size * 0.04,
+                    backgroundColor: color,
+                    opacity: dim,
+                }}/>);
+        })}
     </react_native_1.View>);
 }
 const styles = react_native_1.StyleSheet.create({
     container: { alignItems: 'center', justifyContent: 'center' },
     circle: { position: 'absolute' },
+    barsRow: { flexDirection: 'row', alignItems: 'center', columnGap: 8 },
+    reactiveGlow: { borderRadius: 9999 },
 });

package/lib/index.d.ts

@@ -193,5 +193,5 @@ declare const CrowdPlay: {
 export default CrowdPlay;
 export { CrowdPlayConsentScreen } from './ConsentScreen';
 export { CrowdPlayVideoView } from './VideoView';
-export { CrowdPlayVoiceView, useVoiceActivity } from './VoiceView';
-export type { CrowdPlayVoiceViewProps, VoiceActivity, VoicePhase } from './VoiceView';
+export { CrowdPlayVoiceView, VoiceReactive, useVoiceActivity, useLatestCaption } from './VoiceView';
+export type { CrowdPlayVoiceViewProps, VoiceActivity, VoicePhase, VoiceCaption } from './VoiceView';

package/lib/index.js

@@ -22,7 +22,7 @@
  * path that records without it.
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.useVoiceActivity = exports.CrowdPlayVoiceView = exports.CrowdPlayVideoView = exports.CrowdPlayConsentScreen = void 0;
+exports.useLatestCaption = exports.useVoiceActivity = exports.VoiceReactive = exports.CrowdPlayVoiceView = exports.CrowdPlayVideoView = exports.CrowdPlayConsentScreen = void 0;
 const react_native_1 = require("react-native");
 function native() {
     const module = react_native_1.NativeModules.CrowdPlayReactNative;
@@ -145,4 +145,6 @@ var VideoView_1 = require("./VideoView");
 Object.defineProperty(exports, "CrowdPlayVideoView", { enumerable: true, get: function () { return VideoView_1.CrowdPlayVideoView; } });
 var VoiceView_1 = require("./VoiceView");
 Object.defineProperty(exports, "CrowdPlayVoiceView", { enumerable: true, get: function () { return VoiceView_1.CrowdPlayVoiceView; } });
+Object.defineProperty(exports, "VoiceReactive", { enumerable: true, get: function () { return VoiceView_1.VoiceReactive; } });
 Object.defineProperty(exports, "useVoiceActivity", { enumerable: true, get: function () { return VoiceView_1.useVoiceActivity; } });
+Object.defineProperty(exports, "useLatestCaption", { enumerable: true, get: function () { return VoiceView_1.useLatestCaption; } });

package/llms.txt

@@ -151,13 +151,64 @@ Named exports: CrowdPlayConsentScreen (props: onConsent(grant)), CrowdPlayVideoV
 CrowdPlayVoiceView (props: agentColor?, listeningColor?, idleColor?, size?),
 useVoiceActivity() -> { phase: 'connecting'|'idle'|'listening'|'speaking', energy: 0…1 }.
 
-Voice-AI apps: when the app includes the CrowdPlay voice AI, render the AI
-participant with <CrowdPlayVoiceView /> — the standard animated orb that
-breathes when idle, rings while the user talks, and pulses while the AI
-talks. The AI publishes NO video: never show it as a camera tile or an
-empty box. Detect it in the participants event by identity prefix
-"agent-" or name "agent". For a fully custom visual, drive your own
-component from useVoiceActivity().
+The voice AI (apps with the Voice AI switch enabled on the dashboard):
+
+How it works — all server-side, nothing to wire up. When any user joins a
+call, CrowdPlay's servers put a live AI participant into the room within
+~3 seconds. It joins with name "agent" (identity "agent-<8hex>"), greets
+the user, and holds a real spoken conversation (speech-to-speech: it
+hears the user directly, answers in natural voice, can be interrupted).
+The app writes ZERO AI code — no model API calls, no AI keys, no chat
+backend, no STT/TTS. Persona (system prompt), voice, and model are set by
+the BUILDER on the CrowdPlay dashboard and apply from the next call.
+Conversations are recorded and transcribed by the platform automatically.
+RECOMMENDED: voice-AI apps should be AUDIO-ONLY (audioOnly: true — no
+camera permission, no video anywhere); a camera adds nothing to talking
+with an AI. Keep video only when the app ALSO has calls between people.
+
+The AI conversation screen — the approved standard layout: the animated
+visual front and center, large (it IS the screen); NO self camera
+preview and NO video tiles on this screen, ever; around it only the AI's
+name/persona, a small REC indicator, a leave button, and the small
+audio-output menu. One calm screen, one living visual, nothing competing
+with it.
+
+What the app builds — the AI's on-screen presence. The AI publishes
+AUDIO ONLY: never a camera tile, an empty box, or a static image; never
+count it as a generic user. Detect it in the participants event by name
+"agent" (or identity prefix "agent-").
+- The visual is a MENU, zero-effort to fully custom (0.3.4+ for 2-5):
+  1. Orb (default): <CrowdPlayVoiceView /> — breathes idle, rings with
+     the user's voice, pulses with the AI's. Props: agentColor,
+     listeningColor, idleColor, size.
+  2. Bars: <CrowdPlayVoiceView variant='bars' /> — equalizer look, good
+     under other content (scoreboards, question cards, quizzes).
+  3. Halo around the app's own art: pass a mascot portrait / logo /
+     character as children of <CrowdPlayVoiceView> — animated ring +
+     glow around whatever you provide.
+  4. Make ANY view alive: wrap it in <VoiceReactive> — breathing, pulse
+     and speech glow on the app's own art in one line.
+  5. Fully custom (best for characters): drive ANY visual from the
+     useVoiceActivity() hook — { phase: 'connecting'|'idle'|'listening'|
+     'speaking', energy: 0…1 smoothed loudness }. Example, an illustrated
+     tutor that reacts to the conversation:
+
+      function TutorAvatar() {
+        const { phase, energy } = useVoiceActivity();
+        return <Character                    // the app's own art
+          mouthOpen={phase === 'speaking' ? energy : 0}
+          pose={phase === 'listening' ? 'leanIn' : 'idleBlink'} />;
+      }
+
+  Any art pipeline works (Animated, Lottie, Rive, sprite frames) — bind
+  mouth/pose/glow to phase + energy.
+- Live captions (0.3.4+): each line the AI and the user say is streamed
+  as it is spoken — the useLatestCaption() hook returns { role, text }.
+  For speech bubbles, subtitles, language-learning corrections. Optional
+  UI; the full timestamped transcript is always delivered with the
+  session regardless.
+- Testing: the AI appears only on a real iPhone (simulators have no
+  microphone). Join a room, wait ~3 s, it speaks first.
 
 Types:
   CrowdPlayConfig { serverUrl; appKey; audioOnly?=false;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crowdplaysdk",
-  "version": "0.3.3",
+  "version": "0.3.4",
   "description": "CrowdPlay lossless conversation capture for React Native (iOS). Studio-grade per-participant recording during live video calls, delivered to CrowdPlay automatically.",
   "main": "lib/index.js",
   "types": "lib/index.d.ts",

package/src/VoiceView.tsx

@@ -73,7 +73,67 @@ export function useVoiceActivity(): VoiceActivity {
   return activity;
 }
 
+export interface VoiceCaption {
+  role: 'agent' | 'user';
+  text: string;
+}
+
+/** The latest live transcript line (speech bubbles, subtitles). Captions
+ * are broadcast by the voice agent as it speaks; null until the first one. */
+export function useLatestCaption(): VoiceCaption | null {
+  const [caption, setCaption] = useState<VoiceCaption | null>(null);
+  useEffect(() => {
+    const module = NativeModules.CrowdPlayReactNative;
+    if (!module) return;
+    const emitter = new NativeEventEmitter(module);
+    const sub = emitter.addListener('crowdplay:caption', (e: VoiceCaption) => setCaption(e));
+    return () => sub.remove();
+  }, []);
+  return caption;
+}
+
+/** Make ANY child view breathe, pulse, and glow with the conversation —
+ * the one-line path from the app's own art (mascot, logo, character) to a
+ * living AI presence. */
+export function VoiceReactive({
+  children,
+  glowColor = '#598CFF',
+}: {
+  children: React.ReactNode;
+  glowColor?: string;
+}): React.JSX.Element {
+  const { energy } = useVoiceActivity();
+  const scale = useRef(new Animated.Value(1)).current;
+  const glow = useRef(new Animated.Value(0)).current;
+
+  useEffect(() => {
+    Animated.spring(scale, {
+      toValue: 1 + 0.08 * energy, useNativeDriver: true, speed: 20, bounciness: 12,
+    }).start();
+    Animated.timing(glow, {
+      toValue: energy, duration: 120, useNativeDriver: true,
+    }).start();
+  }, [energy, scale, glow]);
+
+  return (
+    <View style={styles.container}>
+      <Animated.View
+        pointerEvents="none"
+        style={[StyleSheet.absoluteFillObject, styles.reactiveGlow, {
+          backgroundColor: glowColor,
+          opacity: Animated.multiply(glow, 0.35),
+          transform: [{ scale: 1.25 }],
+        }]}
+      />
+      <Animated.View style={{ transform: [{ scale }] }}>{children}</Animated.View>
+    </View>
+  );
+}
+
 export interface CrowdPlayVoiceViewProps {
+  /** Built-in visual shape: the breathing orb (default) or equalizer bars.
+   * Passing children switches to the halo (ring + glow around YOUR art). */
+  variant?: 'orb' | 'bars';
   /** Orb color while the AI is speaking. */
   agentColor?: string;
   /** Ring color while the user is speaking. */
@@ -82,13 +142,18 @@ export interface CrowdPlayVoiceViewProps {
   idleColor?: string;
   /** Diameter in points. */
   size?: number;
+  /** App-provided center content (mascot portrait, logo): rendered inside
+   * an animated halo instead of the orb. */
+  children?: React.ReactNode;
 }
 
 export function CrowdPlayVoiceView({
+  variant = 'orb',
   agentColor = '#598CFF',
   listeningColor = '#4DD9B3',
   idleColor = '#BFBFBF',
   size = 180,
+  children,
 }: CrowdPlayVoiceViewProps): React.JSX.Element {
   const { phase, energy } = useVoiceActivity();
   const breathe = useRef(new Animated.Value(0)).current;
@@ -148,23 +213,73 @@ export function CrowdPlayVoiceView({
           transform: [{ scale: Animated.multiply(halo, breatheScale) }],
         }]}
       />
-      {phase === 'listening' && (
+      {(phase === 'listening' || children != null) && (
         <View
           style={[styles.circle, {
             width: size * 0.8, height: size * 0.8, borderRadius: size * 0.4,
-            borderWidth: Math.max(2, size * 0.015), borderColor: color,
+            borderWidth: Math.max(2, size * (children != null ? 0.02 : 0.015)),
+            borderColor: color,
             backgroundColor: 'transparent',
           }]}
         />
       )}
-      <Animated.View
-        style={[styles.circle, {
-          width: size, height: size, borderRadius: size / 2,
-          backgroundColor: color,
-          opacity: dim,
-          transform: [{ scale: Animated.multiply(scale, breatheScale) }],
-        }]}
-      />
+      {children != null ? (
+        // Halo mode: the app's own art at the center, ring + glow around it.
+        <Animated.View
+          style={{
+            width: size * 0.72, height: size * 0.72,
+            borderRadius: size * 0.36, overflow: 'hidden',
+            transform: [{ scale: Animated.multiply(scale, breatheScale) }],
+          }}
+        >
+          {children}
+        </Animated.View>
+      ) : variant === 'bars' ? (
+        <BarsVisual color={color} dim={dim} energy={energy} size={size} />
+      ) : (
+        <Animated.View
+          style={[styles.circle, {
+            width: size, height: size, borderRadius: size / 2,
+            backgroundColor: color,
+            opacity: dim,
+            transform: [{ scale: Animated.multiply(scale, breatheScale) }],
+          }]}
+        />
+      )}
+    </View>
+  );
+}
+
+/** Equalizer-style bars: heights mix energy with per-bar sines so they
+ * dance during speech and rest at a low idle. */
+function BarsVisual({ color, dim, energy, size }: {
+  color: string; dim: number; energy: number; size: number;
+}): React.JSX.Element {
+  const [tick, setTick] = useState(0);
+  useEffect(() => {
+    const id = setInterval(() => setTick((t) => t + 1), 90);
+    return () => clearInterval(id);
+  }, []);
+  const t = tick * 0.09;
+  return (
+    <View style={styles.barsRow}>
+      {[0, 1, 2, 3, 4].map((index) => {
+        const phase = index * 1.3;
+        const wave = 0.5 + 0.5 * Math.sin(t * (5 + index * 1.7) + phase);
+        const height = 0.15 + 0.08 * Math.sin(t * 1.4 + phase) + energy * wave * 0.8;
+        return (
+          <View
+            key={index}
+            style={{
+              width: size * 0.1,
+              height: size * Math.min(1, height),
+              borderRadius: size * 0.04,
+              backgroundColor: color,
+              opacity: dim,
+            }}
+          />
+        );
+      })}
     </View>
   );
 }
@@ -172,4 +287,6 @@ export function CrowdPlayVoiceView({
 const styles = StyleSheet.create({
   container: { alignItems: 'center', justifyContent: 'center' },
   circle: { position: 'absolute' },
+  barsRow: { flexDirection: 'row', alignItems: 'center', columnGap: 8 },
+  reactiveGlow: { borderRadius: 9999 },
 });

package/src/index.ts

@@ -297,5 +297,5 @@ const CrowdPlay = {
 export default CrowdPlay;
 export { CrowdPlayConsentScreen } from './ConsentScreen';
 export { CrowdPlayVideoView } from './VideoView';
-export { CrowdPlayVoiceView, useVoiceActivity } from './VoiceView';
-export type { CrowdPlayVoiceViewProps, VoiceActivity, VoicePhase } from './VoiceView';
+export { CrowdPlayVoiceView, VoiceReactive, useVoiceActivity, useLatestCaption } from './VoiceView';
+export type { CrowdPlayVoiceViewProps, VoiceActivity, VoicePhase, VoiceCaption } from './VoiceView';