crowdplaysdk 0.4.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Examples/App.tsx +2 -2
- package/README.md +201 -119
- package/ios/CrowdPlayRNModule.swift +61 -9
- package/ios/CrowdPlayRNVideoView.swift +1 -1
- package/ios/CrowdPlayReactNative.m +4 -1
- package/ios/wire.rb +4 -4
- package/lib/ConsentScreen.js +1 -1
- package/lib/VideoView.d.ts +1 -1
- package/lib/VideoView.js +1 -1
- package/lib/VoiceView.d.ts +4 -4
- package/lib/VoiceView.js +5 -5
- package/lib/index.d.ts +54 -12
- package/lib/index.js +43 -7
- package/llms.txt +64 -33
- package/package.json +1 -1
- package/src/ConsentScreen.tsx +1 -1
- package/src/VideoView.tsx +1 -1
- package/src/VoiceView.tsx +5 -5
- package/src/index.ts +82 -14
package/Examples/App.tsx
CHANGED
|
@@ -91,7 +91,7 @@ export default function App(): React.JSX.Element {
|
|
|
91
91
|
|
|
92
92
|
return (
|
|
93
93
|
<SafeAreaView style={styles.screen}>
|
|
94
|
-
{/* The session is NOT being recorded when this is non-null
|
|
94
|
+
{/* The session is NOT being recorded when this is non-null; always show it. */}
|
|
95
95
|
{recordingError ? (
|
|
96
96
|
<View style={styles.banner}>
|
|
97
97
|
<Text style={styles.bannerText}>NOT RECORDING: {recordingError}</Text>
|
|
@@ -121,7 +121,7 @@ export default function App(): React.JSX.Element {
|
|
|
121
121
|
/>
|
|
122
122
|
</View>
|
|
123
123
|
|
|
124
|
-
{/* Audio output menu (D-079): standard in every build
|
|
124
|
+
{/* Audio output menu (D-079): standard in every build; shows the
|
|
125
125
|
device by name and lets the user flip to Speaker and back. */}
|
|
126
126
|
<View style={styles.controls}>
|
|
127
127
|
<Text>{route?.currentOutputName ?? 'audio output'}</Text>
|
package/README.md
CHANGED
|
@@ -1,150 +1,232 @@
|
|
|
1
1
|
# crowdplaysdk
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
3
|
+
CrowdPlaySDK adds live calls and high-quality conversation capture to React
|
|
4
|
+
Native iOS apps. It is the same compiled engine as the native CrowdPlaySDK
|
|
5
|
+
Swift package, driven from JavaScript with control calls and state events.
|
|
6
|
+
No media ever crosses the JS bridge, so recording quality is identical to a
|
|
7
|
+
fully native app.
|
|
8
8
|
|
|
9
|
-
|
|
10
|
-
native core (CrowdPlaySDK) our reference app runs; React Native drives it with
|
|
11
|
-
control calls and receives state events. Recording quality is identical to
|
|
12
|
-
a fully native app.
|
|
9
|
+
## What the SDK handles
|
|
13
10
|
|
|
14
|
-
|
|
11
|
+
* Joining and leaving calls
|
|
12
|
+
* Microphone and camera access
|
|
13
|
+
* Participant audio and video
|
|
14
|
+
* Mute, camera, and audio-output controls
|
|
15
|
+
* User consent before capture begins
|
|
16
|
+
* Per-participant data capture and synchronization
|
|
17
|
+
* Uploads, retries, and recovery
|
|
18
|
+
* Session lifecycle and common interruptions
|
|
15
19
|
|
|
20
|
+
Once a user consents and joins a call, capture and delivery happen
|
|
21
|
+
automatically.
|
|
22
|
+
|
|
23
|
+
## What you build
|
|
24
|
+
|
|
25
|
+
You build the actual product and user experience:
|
|
26
|
+
|
|
27
|
+
* Your app's screens, navigation, and design
|
|
28
|
+
* What users do in your app
|
|
29
|
+
* When and why a call happens
|
|
30
|
+
* The UI around the call
|
|
31
|
+
* Your own accounts, profiles, content, or other product features
|
|
32
|
+
|
|
33
|
+
For example, if you're building a tutoring app, **you build the tutoring
|
|
34
|
+
experience**. CrowdPlaySDK provides the call and capture infrastructure
|
|
35
|
+
underneath it.
|
|
36
|
+
|
|
37
|
+
Your integration is essentially:
|
|
38
|
+
|
|
39
|
+
**Your app → consent → join call → CrowdPlay handles the rest**
|
|
40
|
+
|
|
41
|
+
## Three kinds of calls
|
|
42
|
+
|
|
43
|
+
You pick one when you create your app on the dashboard, and you can change
|
|
44
|
+
it later:
|
|
45
|
+
|
|
46
|
+
* **Video + audio.** Face-to-face calls between people. Both are recorded.
|
|
47
|
+
* **Audio-only.** Voice calls between people, no camera anywhere in the app.
|
|
48
|
+
* **Voice AI.** Your users talk with a live AI voice instead of another
|
|
49
|
+
person. Audio only, no camera.
|
|
50
|
+
|
|
51
|
+
## Getting started
|
|
52
|
+
|
|
53
|
+
Sign in at **https://dashboard.crowdplay.ai**, create your app, and press
|
|
54
|
+
**Copy agent prompt**. The prompt is pre-filled with your key and tailored
|
|
55
|
+
to the tool you build with, so paste it into your coding agent and it will
|
|
56
|
+
build the app. Your dashboard then tracks setup and shows your recordings.
|
|
57
|
+
|
|
58
|
+
**Building in Swift?** Use the CrowdPlaySDK Swift package instead:
|
|
59
|
+
https://github.com/symbiateam/crowdplaysdk
|
|
60
|
+
|
|
61
|
+
## Setup (iOS)
|
|
62
|
+
|
|
63
|
+
Requirements: iOS 17+, React Native 0.71 or newer, a real device to test
|
|
64
|
+
(the Simulator has no camera or mic), and an **app key** from CrowdPlay.
|
|
65
|
+
iOS only for now; calls to the module on other platforms throw with a
|
|
66
|
+
clear message.
|
|
67
|
+
|
|
68
|
+
**1. Install the package and wire the engine.**
|
|
69
|
+
|
|
70
|
+
```bash
|
|
71
|
+
npm install crowdplaysdk
|
|
72
|
+
cd ios && pod install
|
|
73
|
+
gem install xcodeproj --user-install # one time
|
|
74
|
+
ruby node_modules/crowdplaysdk/ios/wire.rb <YourProjectName>
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
`wire.rb` is idempotent. It adds the compiled CrowdPlaySDK Swift package,
|
|
78
|
+
copies the three bridge files into your app target, and points the Swift
|
|
79
|
+
bridging header at the React headers. There is no CocoaPod of our own, so
|
|
80
|
+
your Podfile stays untouched.
|
|
81
|
+
|
|
82
|
+
**2. Declare permissions in Info.plist.**
|
|
83
|
+
|
|
84
|
+
```
|
|
85
|
+
NSMicrophoneUsageDescription = "Records your microphone during sessions."
|
|
86
|
+
NSCameraUsageDescription = "Records your camera during sessions."
|
|
87
|
+
UIBackgroundModes = [audio]
|
|
16
88
|
```
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
a small REC indicator during the call, join/leave, and a small
|
|
29
|
-
audio-output menu.
|
|
30
|
-
- Keep the SDK integration minimal: consent → join/leave → call screen.
|
|
31
|
-
The call screen MUST include a small audio-output menu showing the
|
|
32
|
-
current output device by name (from the 'audioRoute' event) with a
|
|
33
|
-
switch between the device and Speaker (CrowdPlay.setAudioOutput). Do
|
|
34
|
-
NOT build upload screens, diagnostics screens, or SDK settings.
|
|
35
|
-
Uploads happen automatically in the background.
|
|
36
|
-
|
|
37
|
-
- npm install crowdplaysdk
|
|
38
|
-
- In ios/: pod install, then run
|
|
39
|
-
gem install xcodeproj --user-install # one-time
|
|
40
|
-
ruby node_modules/crowdplaysdk/ios/wire.rb <MyProjectName>
|
|
41
|
-
(adds the compiled CrowdPlay engine + bridge to the Xcode project)
|
|
42
|
-
- Info.plist: NSMicrophoneUsageDescription, NSCameraUsageDescription,
|
|
43
|
-
UIBackgroundModes = [audio]
|
|
44
|
-
- My CrowdPlay app key: <APP KEY from the CrowdPlay dashboard>
|
|
45
|
-
- Follow node_modules/crowdplaysdk/llms.txt exactly: configure at
|
|
46
|
-
startup, the CrowdPlayConsentScreen before joining (join() throws
|
|
47
|
-
without a ConsentGrant), CrowdPlayVideoView tiles from the participants
|
|
48
|
-
event, and surface the warning events on the call screen.
|
|
49
|
-
- Never add, pin or import LiveKit/WebRTC yourself, and never write your
|
|
50
|
-
own AVAudioSession/AVCaptureSession code — the engine owns the mic and
|
|
51
|
-
camera.
|
|
52
|
-
- When done, run CrowdPlay.doctor() as a development check (not an app
|
|
53
|
-
screen) and fix anything actionable.
|
|
54
|
-
- Your FINAL message must walk me through testing on my real iPhone,
|
|
55
|
-
step by step (the Simulator has no camera or mic, so nothing is proven
|
|
56
|
-
until then): plugging it in, signing/Developer Mode, running from
|
|
57
|
-
Xcode, allowing permissions, doing a ~1-minute test call, and checking
|
|
58
|
-
my session at https://dashboard.crowdplay.ai. Do not just say the
|
|
59
|
-
implementation is complete.
|
|
89
|
+
|
|
90
|
+
**3. Let uploads finish while the app is suspended** (recommended). In your
|
|
91
|
+
AppDelegate:
|
|
92
|
+
|
|
93
|
+
```objc
|
|
94
|
+
// AppDelegate.mm, with #import "<YourProjectName>-Swift.h" at the top
|
|
95
|
+
- (void)application:(UIApplication *)application
|
|
96
|
+
handleEventsForBackgroundURLSession:(NSString *)identifier
|
|
97
|
+
completionHandler:(void (^)(void))completionHandler {
|
|
98
|
+
[CrowdPlayRNBackground handleWithCompletionHandler:completionHandler];
|
|
99
|
+
}
|
|
60
100
|
```
|
|
61
101
|
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
1. `npm install crowdplaysdk`
|
|
65
|
-
2. `cd ios && pod install`
|
|
66
|
-
(one-time: `gem install xcodeproj --user-install` — the wiring script needs it)
|
|
67
|
-
3. `ruby node_modules/crowdplaysdk/ios/wire.rb <YourProjectName>` —
|
|
68
|
-
idempotent; adds the CrowdPlaySDK Swift package (the compiled engine), copies
|
|
69
|
-
the three bridge files into your app target, and points the Swift
|
|
70
|
-
bridging header at the React headers. (No CocoaPod of our own: the
|
|
71
|
-
engine ships as a binary Swift package, which keeps your Podfile
|
|
72
|
-
untouched.)
|
|
73
|
-
4. Info.plist: `NSMicrophoneUsageDescription`, `NSCameraUsageDescription`,
|
|
74
|
-
`UIBackgroundModes = [audio]`.
|
|
75
|
-
5. Recommended, for uploads that finish while your app is suspended — in
|
|
76
|
-
your AppDelegate:
|
|
77
|
-
|
|
78
|
-
```objc
|
|
79
|
-
// AppDelegate.mm — add #import "<YourProjectName>-Swift.h" at the top
|
|
80
|
-
- (void)application:(UIApplication *)application
|
|
81
|
-
handleEventsForBackgroundURLSession:(NSString *)identifier
|
|
82
|
-
completionHandler:(void (^)(void))completionHandler {
|
|
83
|
-
[CrowdPlayRNBackground handleWithCompletionHandler:completionHandler];
|
|
84
|
-
}
|
|
85
|
-
```
|
|
86
|
-
|
|
87
|
-
(Swift AppDelegates call `CrowdPlaySDK.handleBackgroundURLSessionEvents(completionHandler:)`.)
|
|
102
|
+
Swift AppDelegates call `CrowdPlaySDK.handleBackgroundURLSessionEvents(completionHandler:)`.
|
|
88
103
|
|
|
89
104
|
## Usage
|
|
90
105
|
|
|
91
106
|
```tsx
|
|
92
107
|
import CrowdPlay, { CrowdPlayConsentScreen, CrowdPlayVideoView } from 'crowdplaysdk';
|
|
93
108
|
|
|
94
|
-
CrowdPlay.configure({
|
|
95
|
-
|
|
96
|
-
|
|
109
|
+
CrowdPlay.configure({
|
|
110
|
+
serverUrl: 'https://dashboard.crowdplay.ai',
|
|
111
|
+
appKey: 'liva_pk_…',
|
|
112
|
+
// audioOnly: true, // audio-only or Voice AI app: no camera at all
|
|
113
|
+
// otherAudio: 'mix', // keep Apple Music / Spotify playing (0.4.0+)
|
|
114
|
+
});
|
|
97
115
|
|
|
98
|
-
// 1. Consent (
|
|
116
|
+
// 1. Consent (required; join() throws without it):
|
|
99
117
|
<CrowdPlayConsentScreen onConsent={(grant) => setConsent(grant)} />
|
|
100
118
|
|
|
101
|
-
// 2. Join
|
|
119
|
+
// 2. Join. Recording starts automatically:
|
|
102
120
|
await CrowdPlay.join({ displayName, roomCode, consent });
|
|
103
121
|
|
|
104
122
|
// 3. Render the call:
|
|
105
|
-
<CrowdPlayVideoView participant="local" style={…} />
|
|
123
|
+
<CrowdPlayVideoView participant="local" style={…} /> // self view
|
|
106
124
|
{participants.map(p => <CrowdPlayVideoView participant={p.identity} … />)}
|
|
107
125
|
|
|
108
126
|
// 4. Controls:
|
|
109
|
-
await CrowdPlay.setMicMuted(true);
|
|
127
|
+
await CrowdPlay.setMicMuted(true); // mutes the call and records silence
|
|
110
128
|
await CrowdPlay.setCameraEnabled(false); // black frames, timeline continuous
|
|
111
129
|
await CrowdPlay.leave(); // stops recording; uploads continue
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
State arrives as events:
|
|
112
133
|
|
|
113
|
-
|
|
114
|
-
CrowdPlay.addListener('phase', …) // idle/connecting/connected/failed
|
|
134
|
+
```tsx
|
|
135
|
+
CrowdPlay.addListener('phase', …) // idle / connecting / connected / failed
|
|
115
136
|
CrowdPlay.addListener('participants', …) // who to render tiles for
|
|
116
137
|
CrowdPlay.addListener('uploads', …) // per-session progress
|
|
117
|
-
CrowdPlay.addListener('warning', …) // micSilent / clipping / crossTalk
|
|
118
|
-
CrowdPlay.addListener('recording', …) // error
|
|
119
|
-
CrowdPlay.addListener('audioRoute', …) // current output device
|
|
138
|
+
CrowdPlay.addListener('warning', …) // micSilent / clipping / crossTalk: show these
|
|
139
|
+
CrowdPlay.addListener('recording', …) // error means NOT capturing; offer retryRecording()
|
|
140
|
+
CrowdPlay.addListener('audioRoute', …) // current output device, for the output menu
|
|
120
141
|
CrowdPlay.setAudioOutput('speaker') // or 'automatic' (the connected device)
|
|
121
|
-
CrowdPlay.snapshot() // poll 1 Hz for REC timer
|
|
122
|
-
|
|
142
|
+
CrowdPlay.snapshot() // poll at 1 Hz for the REC timer and input level
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
That's the whole integration. Capture, sync, upload, retry and crash
|
|
146
|
+
recovery are invisible. The complete example lives in this package's
|
|
147
|
+
`Examples` folder: one file, consent → join → tiles → warnings → uploads.
|
|
148
|
+
|
|
149
|
+
## Voice AI
|
|
150
|
+
|
|
151
|
+
If you picked **Voice AI**, a live AI voice joins the call as an ordinary
|
|
152
|
+
participant, greets your user, and holds a real spoken conversation. Users
|
|
153
|
+
can interrupt it and it responds with natural timing. Both sides are
|
|
154
|
+
recorded and transcribed like any other call.
|
|
155
|
+
|
|
156
|
+
**Your app writes no AI code.** No model API calls, no API keys, no speech
|
|
157
|
+
recognition, no text-to-speech. You choose the AI's voice and personality
|
|
158
|
+
on the dashboard and can change them any time; everything else runs on
|
|
159
|
+
CrowdPlay's servers.
|
|
160
|
+
|
|
161
|
+
The only thing your app draws is what the AI looks like on screen. It has
|
|
162
|
+
no camera, so never give it a video tile. The SDK ships an animated orb
|
|
163
|
+
that breathes when idle, ripples with your user's voice, and pulses when
|
|
164
|
+
the AI speaks:
|
|
165
|
+
|
|
166
|
+
```tsx
|
|
167
|
+
<CrowdPlayVoiceView />
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
Restyle it with props, wrap your own artwork in `<VoiceReactive>`, or drive
|
|
171
|
+
any visual from the `useVoiceActivity()` hook. Show live subtitles with
|
|
172
|
+
`useLatestCaption()`, and identify the AI participant by name `agent`.
|
|
173
|
+
|
|
174
|
+
You can also override the voice, personality and context for a single call,
|
|
175
|
+
which is how you give the AI memory of a particular user:
|
|
176
|
+
|
|
177
|
+
```tsx
|
|
178
|
+
await CrowdPlay.join({
|
|
179
|
+
displayName, roomCode, consent,
|
|
180
|
+
agent: { voice: 'Aoede', context: 'Their name is Sam. Last time you discussed Kyoto.' },
|
|
181
|
+
});
|
|
182
|
+
```
|
|
183
|
+
|
|
184
|
+
See `llms.txt` in this package for the full menu of visuals and options.
|
|
185
|
+
|
|
186
|
+
## Verifying your integration
|
|
187
|
+
|
|
188
|
+
```tsx
|
|
189
|
+
const checks = await CrowdPlay.doctor();
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
Doctor verifies configuration, Info.plist strings, permissions, free disk,
|
|
193
|
+
backend reachability and your app key. Every failing check names its own
|
|
194
|
+
fix. Then record a one-minute session on a real device and confirm the
|
|
195
|
+
upload finishes.
|
|
196
|
+
|
|
197
|
+
## Accessing your recordings
|
|
198
|
+
|
|
199
|
+
Your app key is also your data-access credential: plain HTTPS, no AWS
|
|
200
|
+
account, no SDK required.
|
|
201
|
+
|
|
202
|
+
```bash
|
|
203
|
+
BASE=https://dashboard.crowdplay.ai
|
|
204
|
+
|
|
205
|
+
curl -H "x-liva-key: $APP_KEY" $BASE/sessions
|
|
206
|
+
curl -H "x-liva-key: $APP_KEY" $BASE/sessions/<sessionId>/files
|
|
123
207
|
```
|
|
124
208
|
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
quality with the app launched from the home screen — Metro/debugger
|
|
150
|
-
overhead can reproduce the same symptoms.
|
|
209
|
+
`deliverable/` holds what you actually use: one clean audio file per
|
|
210
|
+
person, aligned so they play together with no offset, plus a mixed version
|
|
211
|
+
and a quality report. You can only reach sessions recorded with your key.
|
|
212
|
+
|
|
213
|
+
## Things to know
|
|
214
|
+
|
|
215
|
+
- **Don't force-quit the app** after a session: iOS pauses background
|
|
216
|
+
transfers until next launch. They resume automatically and nothing is
|
|
217
|
+
lost.
|
|
218
|
+
- Sessions are roughly 2 GB per participant-hour, dominated by video. The
|
|
219
|
+
SDK refuses to join below 8 GB free disk.
|
|
220
|
+
- Participants should wear **wired or closed-back headphones**. A
|
|
221
|
+
loudspeaker pipes the far end into the local mic and contaminates the
|
|
222
|
+
track; the SDK surfaces this live through the `warning` event.
|
|
223
|
+
- **Slow join or glitchy first seconds is bad radio, not a bug.** Congested
|
|
224
|
+
2.4 GHz WiFi and Bluetooth headphones share an antenna. Capture
|
|
225
|
+
self-heals and the rest of the session is fine. Metro and the debugger
|
|
226
|
+
cause the same symptoms, so launch from the home screen when judging
|
|
227
|
+
quality.
|
|
228
|
+
- The npm version can run ahead of the engine version `wire.rb` pins;
|
|
229
|
+
bridge and doc fixes ship without a new engine binary.
|
|
230
|
+
|
|
231
|
+
`llms.txt` carries the full reference, including every event and warning
|
|
232
|
+
and the voice-AI options. It is written for coding agents.
|
|
@@ -52,6 +52,17 @@ final class CrowdPlayRNCore {
|
|
|
52
52
|
])
|
|
53
53
|
}.store(in: &cancellables)
|
|
54
54
|
|
|
55
|
+
// Agent tool calls (D-103): the JS layer answers via
|
|
56
|
+
// respondToAgentToolCall, so the native auto-handler stays unset.
|
|
57
|
+
engine.$latestAgentToolCall
|
|
58
|
+
.compactMap { $0 }
|
|
59
|
+
.sink { [weak self] call in
|
|
60
|
+
self?.send("crowdplay:agentToolCall", [
|
|
61
|
+
"callId": call.callId, "name": call.name,
|
|
62
|
+
"argumentsJSON": call.argumentsJSON,
|
|
63
|
+
])
|
|
64
|
+
}.store(in: &cancellables)
|
|
65
|
+
|
|
55
66
|
engine.$uploads
|
|
56
67
|
.combineLatest(engine.$uploadsOnWiFi)
|
|
57
68
|
.sink { [weak self] uploads, onWifi in
|
|
@@ -113,19 +124,19 @@ final class CrowdPlayRNCore {
|
|
|
113
124
|
let snapshot = engine.recordingSnapshot()
|
|
114
125
|
updateWarning("audioStalled",
|
|
115
126
|
active: snapshot?.audioStalled == true,
|
|
116
|
-
message: "Audio capture stalled
|
|
127
|
+
message: "Audio capture stalled; the engine is being restarted. "
|
|
117
128
|
+ "If this persists, leave and rejoin.")
|
|
118
129
|
updateWarning("micSilent",
|
|
119
130
|
active: snapshot?.inputSilent == true && !engine.isMicMuted,
|
|
120
|
-
message: "No sound is reaching the microphone
|
|
131
|
+
message: "No sound is reaching the microphone; check the headset "
|
|
121
132
|
+ "plug or switch to the phone microphone.")
|
|
122
133
|
updateWarning("clipping",
|
|
123
134
|
active: (snapshot?.fullScaleSamples ?? 0) > 100,
|
|
124
|
-
message: "The microphone is clipping
|
|
135
|
+
message: "The microphone is clipping; audio is distorting. Move "
|
|
125
136
|
+ "the mic away or lower the input.")
|
|
126
137
|
updateWarning("crossTalk",
|
|
127
138
|
active: engine.crossTalkRisk,
|
|
128
|
-
message: "Playing through the loudspeaker
|
|
139
|
+
message: "Playing through the loudspeaker; the other person's "
|
|
129
140
|
+ "voice is being recorded into this microphone. Use headphones.")
|
|
130
141
|
updateWarning("micPolicy",
|
|
131
142
|
active: engine.micPolicyViolation != nil,
|
|
@@ -136,7 +147,7 @@ final class CrowdPlayRNCore {
|
|
|
136
147
|
|
|
137
148
|
private var voiceTimer: Timer?
|
|
138
149
|
|
|
139
|
-
/// Fast (0.15 s) voice-activity stream for the animated voice UI
|
|
150
|
+
/// Fast (0.15 s) voice-activity stream for the animated voice UI ,
|
|
140
151
|
/// runs only while some JS component is subscribed, so the bridge
|
|
141
152
|
/// stays quiet for apps that never render it.
|
|
142
153
|
func startVoiceActivity() {
|
|
@@ -205,7 +216,7 @@ final class CrowdPlayRNCore {
|
|
|
205
216
|
/// [CrowdPlayRNBackground handleWithCompletionHandler:completionHandler];
|
|
206
217
|
///
|
|
207
218
|
/// iOS eventually stops delivering background upload events to apps that
|
|
208
|
-
/// skip this
|
|
219
|
+
/// skip this; uploads still finish, just less promptly.
|
|
209
220
|
@objc(CrowdPlayRNBackground)
|
|
210
221
|
public final class CrowdPlayRNBackground: NSObject {
|
|
211
222
|
@objc public static func handle(completionHandler: @escaping () -> Void) {
|
|
@@ -220,7 +231,7 @@ public final class CrowdPlayRNModule: RCTEventEmitter {
|
|
|
220
231
|
public override static func requiresMainQueueSetup() -> Bool { true }
|
|
221
232
|
|
|
222
233
|
public override func supportedEvents() -> [String]! {
|
|
223
|
-
["crowdplay:phase", "crowdplay:recording", "crowdplay:uploads", "crowdplay:participants", "crowdplay:warning", "crowdplay:audioRoute", "crowdplay:voiceActivity", "crowdplay:caption"]
|
|
234
|
+
["crowdplay:phase", "crowdplay:recording", "crowdplay:uploads", "crowdplay:participants", "crowdplay:warning", "crowdplay:audioRoute", "crowdplay:voiceActivity", "crowdplay:caption", "crowdplay:agentToolCall"]
|
|
224
235
|
}
|
|
225
236
|
|
|
226
237
|
@objc public func startVoiceActivityUpdates() {
|
|
@@ -258,6 +269,12 @@ public final class CrowdPlayRNModule: RCTEventEmitter {
|
|
|
258
269
|
if let duringCall = config["uploadDuringCall"] as? Bool {
|
|
259
270
|
configuration.uploadDuringCall = duringCall
|
|
260
271
|
}
|
|
272
|
+
// D-093: other apps' audio during the call. Unknown strings keep
|
|
273
|
+
// the default (.interrupt) rather than failing configure.
|
|
274
|
+
if let policy = config["otherAudio"] as? String,
|
|
275
|
+
let parsed = CrowdPlayConfiguration.OtherAudioPolicy(rawValue: policy) {
|
|
276
|
+
configuration.otherAudio = parsed
|
|
277
|
+
}
|
|
261
278
|
if let quality = config["callQuality"] as? [String: Any] {
|
|
262
279
|
var callQuality = CallQuality()
|
|
263
280
|
if let v = quality["videoWidth"] as? Int { callQuality.videoWidth = v }
|
|
@@ -288,8 +305,30 @@ public final class CrowdPlayRNModule: RCTEventEmitter {
|
|
|
288
305
|
let voice = agent["voice"] as? String
|
|
289
306
|
let prompt = agent["systemPrompt"] as? String
|
|
290
307
|
let context = agent["context"] as? String
|
|
291
|
-
|
|
292
|
-
|
|
308
|
+
// Tools cross the bridge as one JSON string (D-103); the server
|
|
309
|
+
// re-validates every entry.
|
|
310
|
+
var tools: [CrowdPlayAgentTool]?
|
|
311
|
+
if let toolsJSON = agent["toolsJSON"] as? String,
|
|
312
|
+
let data = toolsJSON.data(using: .utf8),
|
|
313
|
+
let entries = try? JSONSerialization.jsonObject(with: data) as? [[String: Any]] {
|
|
314
|
+
let parsed: [CrowdPlayAgentTool] = entries.compactMap { entry in
|
|
315
|
+
guard let name = entry["name"] as? String,
|
|
316
|
+
let description = entry["description"] as? String else { return nil }
|
|
317
|
+
var parametersJSON = "{}"
|
|
318
|
+
if let parameters = entry["parameters"],
|
|
319
|
+
JSONSerialization.isValidJSONObject(parameters),
|
|
320
|
+
let pData = try? JSONSerialization.data(withJSONObject: parameters),
|
|
321
|
+
let pString = String(data: pData, encoding: .utf8) {
|
|
322
|
+
parametersJSON = pString
|
|
323
|
+
}
|
|
324
|
+
return CrowdPlayAgentTool(name: name, description: description,
|
|
325
|
+
parametersJSON: parametersJSON)
|
|
326
|
+
}
|
|
327
|
+
if !parsed.isEmpty { tools = parsed }
|
|
328
|
+
}
|
|
329
|
+
if voice != nil || prompt != nil || context != nil || tools != nil {
|
|
330
|
+
agentOptions = CrowdPlayAgentOptions(voice: voice, systemPrompt: prompt,
|
|
331
|
+
context: context, tools: tools)
|
|
293
332
|
}
|
|
294
333
|
}
|
|
295
334
|
Task { @MainActor in
|
|
@@ -354,6 +393,19 @@ public final class CrowdPlayRNModule: RCTEventEmitter {
|
|
|
354
393
|
}
|
|
355
394
|
}
|
|
356
395
|
|
|
396
|
+
@objc public func respondToAgentToolCall(_ callId: NSString, output: NSString) {
|
|
397
|
+
Task { @MainActor in
|
|
398
|
+
await CrowdPlayRNCore.shared.engine.respondToAgentToolCall(
|
|
399
|
+
callId: callId as String, output: output as String)
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
@objc public func updateAgentContext(_ text: NSString) {
|
|
404
|
+
Task { @MainActor in
|
|
405
|
+
await CrowdPlayRNCore.shared.engine.updateAgentContext(text as String)
|
|
406
|
+
}
|
|
407
|
+
}
|
|
408
|
+
|
|
357
409
|
@objc public func retryRecording() {
|
|
358
410
|
Task { @MainActor in
|
|
359
411
|
let core = CrowdPlayRNCore.shared
|
|
@@ -5,7 +5,7 @@ import React
|
|
|
5
5
|
import UIKit
|
|
6
6
|
|
|
7
7
|
/// Native video tile hosted in the RN layout. Resolves the requested
|
|
8
|
-
/// participant's track from the shared engine on a short poll
|
|
8
|
+
/// participant's track from the shared engine on a short poll; idempotent
|
|
9
9
|
/// and reconnect-proof, and the video itself never touches JavaScript.
|
|
10
10
|
final class CrowdPlayRNVideoView: UIView {
|
|
11
11
|
private let videoView = VideoView()
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// Objective-C registration for the CrowdPlay RN bridge. Add this file AND the
|
|
2
2
|
// two Swift files to your app target (they compile alongside your app, with
|
|
3
|
-
// CrowdPlaySDK coming from the Swift package
|
|
3
|
+
// CrowdPlaySDK coming from the Swift package; see the package README).
|
|
4
4
|
#import <React/RCTBridgeModule.h>
|
|
5
5
|
#import <React/RCTEventEmitter.h>
|
|
6
6
|
#import <React/RCTViewManager.h>
|
|
@@ -25,6 +25,9 @@ RCT_EXTERN_METHOD(setCameraEnabled : (BOOL)enabled
|
|
|
25
25
|
resolver : (RCTPromiseResolveBlock)resolve
|
|
26
26
|
rejecter : (RCTPromiseRejectBlock)reject)
|
|
27
27
|
RCT_EXTERN_METHOD(setAudioOutput : (NSString *)output)
|
|
28
|
+
RCT_EXTERN_METHOD(respondToAgentToolCall : (NSString *)callId
|
|
29
|
+
output : (NSString *)output)
|
|
30
|
+
RCT_EXTERN_METHOD(updateAgentContext : (NSString *)text)
|
|
28
31
|
RCT_EXTERN_METHOD(retryRecording)
|
|
29
32
|
RCT_EXTERN_METHOD(retryUploads)
|
|
30
33
|
RCT_EXTERN_METHOD(warmUp)
|
package/ios/wire.rb
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
#
|
|
4
4
|
# ruby node_modules/crowdplaysdk/ios/wire.rb <YourProjectName>
|
|
5
5
|
#
|
|
6
|
-
# What it does (idempotent
|
|
6
|
+
# What it does (idempotent; safe to re-run):
|
|
7
7
|
# 1. adds the CrowdPlaySDK Swift package (compiled engine) to the project
|
|
8
8
|
# 2. copies the three bridge files into your app group + target
|
|
9
9
|
# 3. creates/points the Swift bridging header at the React headers
|
|
@@ -30,7 +30,7 @@ BRIDGE_FILES = ['CrowdPlayRNModule.swift', 'CrowdPlayRNVideoView.swift', 'CrowdP
|
|
|
30
30
|
|
|
31
31
|
project_name = ARGV[0] or abort 'usage: ruby wire.rb <YourProjectName>'
|
|
32
32
|
project_path = "#{project_name}.xcodeproj"
|
|
33
|
-
abort "#{project_path} not found
|
|
33
|
+
abort "#{project_path} not found; run from your app's ios/ directory" unless File.exist?(project_path)
|
|
34
34
|
|
|
35
35
|
package_dir = File.expand_path(File.dirname(__FILE__))
|
|
36
36
|
project = Xcodeproj::Project.open(project_path)
|
|
@@ -68,7 +68,7 @@ end
|
|
|
68
68
|
target.build_configurations.each do |config|
|
|
69
69
|
config.build_settings['SWIFT_VERSION'] ||= '5.0'
|
|
70
70
|
# CrowdPlaySDK requires iOS 17; RN templates default far lower. Only the APP
|
|
71
|
-
# target is raised
|
|
71
|
+
# target is raised; pods keep their own targets, which is fine.
|
|
72
72
|
current = config.build_settings['IPHONEOS_DEPLOYMENT_TARGET']
|
|
73
73
|
if current.nil? || current.to_f < 17.0
|
|
74
74
|
config.build_settings['IPHONEOS_DEPLOYMENT_TARGET'] = '17.0'
|
|
@@ -86,7 +86,7 @@ target.build_configurations.each do |config|
|
|
|
86
86
|
end
|
|
87
87
|
# The setting must point at wherever the file ACTUALLY went. When the
|
|
88
88
|
# project has no group named after itself (newer RN templates), the
|
|
89
|
-
# fallback above places files at the project root
|
|
89
|
+
# fallback above places files at the project root; hardcoding
|
|
90
90
|
# "<Project>/<header>" there produced a missing-input build failure
|
|
91
91
|
# (field-reported on the RN 0.87 template). Derive it instead:
|
|
92
92
|
# SRCROOT is the .xcodeproj's directory.
|
package/lib/ConsentScreen.js
CHANGED
|
@@ -65,7 +65,7 @@ function CrowdPlayConsentScreen({ onConsent, style }) {
|
|
|
65
65
|
})
|
|
66
66
|
.catch(() => {
|
|
67
67
|
if (!cancelled)
|
|
68
|
-
setText('Consent text unavailable
|
|
68
|
+
setText('Consent text unavailable. Check that CrowdPlay.configure() ran and the native module is installed.');
|
|
69
69
|
});
|
|
70
70
|
return () => {
|
|
71
71
|
cancelled = true;
|
package/lib/VideoView.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Native video tile. Renders the LOCAL camera preview or a REMOTE
|
|
3
3
|
* participant's live video (identified by the `identity` from the
|
|
4
|
-
* `participants` event). The video never touches JavaScript
|
|
4
|
+
* `participants` event). The video never touches JavaScript; this is a
|
|
5
5
|
* native LiveKit view hosted in your RN layout.
|
|
6
6
|
*
|
|
7
7
|
* <CrowdPlayVideoView participant="local" style={{ width: 120, height: 160 }} />
|
package/lib/VideoView.js
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
/**
|
|
3
3
|
* Native video tile. Renders the LOCAL camera preview or a REMOTE
|
|
4
4
|
* participant's live video (identified by the `identity` from the
|
|
5
|
-
* `participants` event). The video never touches JavaScript
|
|
5
|
+
* `participants` event). The video never touches JavaScript; this is a
|
|
6
6
|
* native LiveKit view hosted in your RN layout.
|
|
7
7
|
*
|
|
8
8
|
* <CrowdPlayVideoView participant="local" style={{ width: 120, height: 160 }} />
|
package/lib/VoiceView.d.ts
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* The standard voice-AI visual (founders, 2026-08-22): a breathing orb in
|
|
3
|
-
* the style of realtime assistants
|
|
3
|
+
* the style of realtime assistants: calm while idle, a cool ring while the
|
|
4
4
|
* user talks, an energetic pulse while the AI speaks. The React Native
|
|
5
5
|
* counterpart of the native SDK's CrowdPlayVoiceView, driven by the same
|
|
6
6
|
* engine signal over a fast bridge event that runs only while mounted.
|
|
7
7
|
*
|
|
8
8
|
* Two levels of customization:
|
|
9
|
-
* 1. Props on <CrowdPlayVoiceView
|
|
10
|
-
* 2. useVoiceActivity()
|
|
9
|
+
* 1. Props on <CrowdPlayVoiceView>: colors and size of the built-in orb.
|
|
10
|
+
* 2. useVoiceActivity(): the phase + smoothed energy underneath, for apps
|
|
11
11
|
* that want to draw something entirely their own.
|
|
12
12
|
*/
|
|
13
13
|
import React from 'react';
|
|
@@ -27,7 +27,7 @@ export interface VoiceCaption {
|
|
|
27
27
|
/** The latest live transcript line (speech bubbles, subtitles). Captions
|
|
28
28
|
* are broadcast by the voice agent as it speaks; null until the first one. */
|
|
29
29
|
export declare function useLatestCaption(): VoiceCaption | null;
|
|
30
|
-
/** Make ANY child view breathe, pulse, and glow with the conversation
|
|
30
|
+
/** Make ANY child view breathe, pulse, and glow with the conversation;
|
|
31
31
|
* the one-line path from the app's own art (mascot, logo, character) to a
|
|
32
32
|
* living AI presence. */
|
|
33
33
|
export declare function VoiceReactive({ children, glowColor, }: {
|