crowdplaysdk 0.4.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Examples/App.tsx +2 -2
- package/README.md +201 -119
- package/ios/CrowdPlayRNModule.swift +61 -9
- package/ios/CrowdPlayRNVideoView.swift +1 -1
- package/ios/CrowdPlayReactNative.m +4 -1
- package/ios/wire.rb +4 -4
- package/lib/ConsentScreen.js +1 -1
- package/lib/VideoView.d.ts +1 -1
- package/lib/VideoView.js +1 -1
- package/lib/VoiceView.d.ts +4 -4
- package/lib/VoiceView.js +5 -5
- package/lib/index.d.ts +54 -12
- package/lib/index.js +43 -7
- package/llms.txt +64 -33
- package/package.json +1 -1
- package/src/ConsentScreen.tsx +1 -1
- package/src/VideoView.tsx +1 -1
- package/src/VoiceView.tsx +5 -5
- package/src/index.ts +82 -14
package/lib/VoiceView.js
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
/**
|
|
3
3
|
* The standard voice-AI visual (founders, 2026-08-22): a breathing orb in
|
|
4
|
-
* the style of realtime assistants
|
|
4
|
+
* the style of realtime assistants: calm while idle, a cool ring while the
|
|
5
5
|
* user talks, an energetic pulse while the AI speaks. The React Native
|
|
6
6
|
* counterpart of the native SDK's CrowdPlayVoiceView, driven by the same
|
|
7
7
|
* engine signal over a fast bridge event that runs only while mounted.
|
|
8
8
|
*
|
|
9
9
|
* Two levels of customization:
|
|
10
|
-
* 1. Props on <CrowdPlayVoiceView
|
|
11
|
-
* 2. useVoiceActivity()
|
|
10
|
+
* 1. Props on <CrowdPlayVoiceView>: colors and size of the built-in orb.
|
|
11
|
+
* 2. useVoiceActivity(): the phase + smoothed energy underneath, for apps
|
|
12
12
|
* that want to draw something entirely their own.
|
|
13
13
|
*/
|
|
14
14
|
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
@@ -81,7 +81,7 @@ function useVoiceActivity() {
|
|
|
81
81
|
phase = 'idle';
|
|
82
82
|
target = 0;
|
|
83
83
|
}
|
|
84
|
-
// Fast attack, slow release
|
|
84
|
+
// Fast attack, slow release: speech onsets snap, tails breathe out.
|
|
85
85
|
const rate = target > energyRef.current ? 0.55 : 0.12;
|
|
86
86
|
energyRef.current += (target - energyRef.current) * rate;
|
|
87
87
|
setActivity({ phase, energy: energyRef.current });
|
|
@@ -108,7 +108,7 @@ function useLatestCaption() {
|
|
|
108
108
|
}, []);
|
|
109
109
|
return caption;
|
|
110
110
|
}
|
|
111
|
-
/** Make ANY child view breathe, pulse, and glow with the conversation
|
|
111
|
+
/** Make ANY child view breathe, pulse, and glow with the conversation;
|
|
112
112
|
* the one-line path from the app's own art (mascot, logo, character) to a
|
|
113
113
|
* living AI presence. */
|
|
114
114
|
function VoiceReactive({ children, glowColor = '#598CFF', }) {
|
package/lib/index.d.ts
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* crowdplaysdk
|
|
2
|
+
* crowdplaysdk: CrowdPlay's lossless capture SDK for React Native (iOS).
|
|
3
3
|
*
|
|
4
4
|
* The recording engine is 100% native (CrowdPlaySDK): studio-grade audio
|
|
5
5
|
* (48 kHz / 24-bit, unprocessed) + 1080p30 video captured locally during a
|
|
6
6
|
* live call, clock-synced across participants, uploaded to CrowdPlay with
|
|
7
|
-
* retries and crash recovery. **No media ever crosses the JS bridge
|
|
8
|
-
* only control calls and state events
|
|
7
|
+
* retries and crash recovery. **No media ever crosses the JS bridge**,
|
|
8
|
+
* only control calls and state events, so recording quality is identical
|
|
9
9
|
* to a fully native app.
|
|
10
10
|
*
|
|
11
11
|
* import CrowdPlay, { CrowdPlayConsentScreen, CrowdPlayVideoView } from 'crowdplaysdk';
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
* await CrowdPlay.leave(); // recording stops; uploads continue automatically
|
|
18
18
|
*
|
|
19
19
|
* Consent is enforced twice: join() throws here without a grant, and the
|
|
20
|
-
* native layer requires the consent record structurally
|
|
20
|
+
* native layer requires the consent record structurally; there is no code
|
|
21
21
|
* path that records without it.
|
|
22
22
|
*/
|
|
23
23
|
/** Live-call (Path A) quality. The RECORDING is always 1080p30 + 24-bit WAV
|
|
@@ -37,7 +37,7 @@ export interface CrowdPlayConfig {
|
|
|
37
37
|
serverUrl: string;
|
|
38
38
|
/** From the CrowdPlay dashboard. Also your data-access credential. */
|
|
39
39
|
appKey: string;
|
|
40
|
-
/** AUDIO-ONLY app: the camera is never touched
|
|
40
|
+
/** AUDIO-ONLY app: the camera is never touched: no camera permission,
|
|
41
41
|
* no video captured or uploaded, voice-only calls. The audio path is
|
|
42
42
|
* identical to an A/V app. Requires engine 0.3.0+. Default false. */
|
|
43
43
|
audioOnly?: boolean;
|
|
@@ -46,13 +46,21 @@ export interface CrowdPlayConfig {
|
|
|
46
46
|
videoUploadsOnCellular?: boolean;
|
|
47
47
|
/** Upload segments while the call records, so the post-call wait is
|
|
48
48
|
* roughly the final segment + manifests. Mid-call transfers run one at
|
|
49
|
-
* a time and pause automatically when call quality drops
|
|
49
|
+
* a time and pause automatically when call quality drops; the call
|
|
50
50
|
* always wins the uplink. Default true. */
|
|
51
51
|
uploadDuringCall?: boolean;
|
|
52
|
+
/** What happens to OTHER apps' audio (Apple Music, Spotify, podcasts)
|
|
53
|
+
* while a call is live. 'interrupt' (default) pauses them at join, the
|
|
54
|
+
* classic call behaviour. 'mix' keeps them playing alongside the call
|
|
55
|
+
* and the recording, for shared-listening and DJ apps; 'mixDucked' also
|
|
56
|
+
* lowers them for the whole call. Music stays on the device and never
|
|
57
|
+
* rides the call; listeners must wear headphones or it lands in the
|
|
58
|
+
* recording. Requires engine 0.4.0+. */
|
|
59
|
+
otherAudio?: 'interrupt' | 'mix' | 'mixDucked';
|
|
52
60
|
callQuality?: CallQuality;
|
|
53
61
|
}
|
|
54
|
-
/** Proof the participant agreed. Produce it with CrowdPlayConsentScreen, or
|
|
55
|
-
* if you render your own consent UI
|
|
62
|
+
/** Proof the participant agreed. Produce it with CrowdPlayConsentScreen, or,
|
|
63
|
+
* if you render your own consent UI, call CrowdPlay.consentText() to show the
|
|
56
64
|
* REQUIRED wording and construct the grant at the moment of agreement. */
|
|
57
65
|
export interface ConsentGrant {
|
|
58
66
|
/** Date.now() at the moment the participant agreed. */
|
|
@@ -66,6 +74,17 @@ export interface ConsentGrant {
|
|
|
66
74
|
* apply to the whole room. The app can never change the provider, the
|
|
67
75
|
* model, or the API key from here.
|
|
68
76
|
*/
|
|
77
|
+
/** A tool the app offers the voice AI for one session (D-103). Requires
|
|
78
|
+
* the app's dashboard config to use the GPT-Live provider with an AI brain
|
|
79
|
+
* (delegation) enabled; ignored otherwise. */
|
|
80
|
+
export interface AgentTool {
|
|
81
|
+
/** Letters, digits, underscore, dash; max 64 chars. */
|
|
82
|
+
name: string;
|
|
83
|
+
/** What the tool does and when to use it, for the model. */
|
|
84
|
+
description: string;
|
|
85
|
+
/** JSON-schema object describing the arguments. Omit for none. */
|
|
86
|
+
parameters?: Record<string, unknown>;
|
|
87
|
+
}
|
|
69
88
|
export interface AgentOptions {
|
|
70
89
|
/** A voice of the app's configured provider. Gemini: Charon, Aoede,
|
|
71
90
|
* Fenrir, Kore, Puck. OpenAI: marin, cedar, alloy, ash, ballad, coral,
|
|
@@ -79,6 +98,9 @@ export interface AgentOptions {
|
|
|
79
98
|
* keep a few sentences per user in your own storage and pass them
|
|
80
99
|
* here. Max 4000 chars. */
|
|
81
100
|
context?: string;
|
|
101
|
+
/** Tools the AI may call this session (D-103). Handle calls with
|
|
102
|
+
* CrowdPlay.onAgentToolCall(). Max 16. */
|
|
103
|
+
tools?: AgentTool[];
|
|
82
104
|
}
|
|
83
105
|
export type Phase = 'idle' | 'connecting' | 'connected' | 'failed';
|
|
84
106
|
export interface RecordingSnapshot {
|
|
@@ -88,10 +110,10 @@ export interface RecordingSnapshot {
|
|
|
88
110
|
droppedSamples: number;
|
|
89
111
|
/** Live input level. Speech sits around −25…−45 dBFS. */
|
|
90
112
|
inputLevelDbfs: number;
|
|
91
|
-
/** True after 10 s of silence while unmuted
|
|
113
|
+
/** True after 10 s of silence while unmuted: a dead/disconnected mic.
|
|
92
114
|
* Surface this loudly; it is fixable mid-call. */
|
|
93
115
|
inputSilent: boolean;
|
|
94
|
-
/** Samples at digital full scale
|
|
116
|
+
/** Samples at digital full scale: a too-hot mic, irreversibly clipped. */
|
|
95
117
|
fullScaleSamples: number;
|
|
96
118
|
}
|
|
97
119
|
export interface UploadKindProgress {
|
|
@@ -127,7 +149,7 @@ export interface CrowdPlayEvents {
|
|
|
127
149
|
error?: string;
|
|
128
150
|
};
|
|
129
151
|
/** Recording state. `error` non-null means the session is NOT being
|
|
130
|
-
* captured
|
|
152
|
+
* captured; show it prominently and offer retryRecording(). */
|
|
131
153
|
recording: {
|
|
132
154
|
isRecording: boolean;
|
|
133
155
|
error?: string;
|
|
@@ -159,6 +181,13 @@ export interface CrowdPlayEvents {
|
|
|
159
181
|
detectedOutputs: string[];
|
|
160
182
|
headphonesConnected: boolean;
|
|
161
183
|
};
|
|
184
|
+
/** The voice AI asked the app to run a tool (D-103). Prefer
|
|
185
|
+
* CrowdPlay.onAgentToolCall(), which answers for you. */
|
|
186
|
+
agentToolCall: {
|
|
187
|
+
callId: string;
|
|
188
|
+
name: string;
|
|
189
|
+
argumentsJSON: string;
|
|
190
|
+
};
|
|
162
191
|
}
|
|
163
192
|
export type CrowdPlayEventName = keyof CrowdPlayEvents;
|
|
164
193
|
declare const CrowdPlay: {
|
|
@@ -175,7 +204,7 @@ declare const CrowdPlay: {
|
|
|
175
204
|
* Join a room. Recording starts automatically on join and stops on
|
|
176
205
|
* leave(). Rejects if the connection fails.
|
|
177
206
|
*
|
|
178
|
-
* `consent` is REQUIRED
|
|
207
|
+
* `consent` is REQUIRED; produce it with <CrowdPlayConsentScreen> or from
|
|
179
208
|
* your own UI at the moment of agreement. There is no way to record
|
|
180
209
|
* without it, here or natively.
|
|
181
210
|
*/
|
|
@@ -208,6 +237,19 @@ declare const CrowdPlay: {
|
|
|
208
237
|
/** Integration self-check: configuration, permissions, disk, backend
|
|
209
238
|
* reachability, app-key auth. Every failing check names its fix. */
|
|
210
239
|
doctor(): Promise<DoctorCheck[]>;
|
|
240
|
+
/** Handle the voice AI's tool calls (D-103). The handler gets the tool
|
|
241
|
+
* name and its arguments (parsed JSON object) and returns the result as
|
|
242
|
+
* an object or JSON string; return null to decline on this device
|
|
243
|
+
* (another participant's device may answer; first answer wins). Ordinary
|
|
244
|
+
* app code, no AI here. The AI waits about 12 seconds per call. Returns
|
|
245
|
+
* a subscription; call .remove(). */
|
|
246
|
+
onAgentToolCall(handler: (name: string, args: Record<string, unknown>) => Promise<Record<string, unknown> | string | null> | Record<string, unknown> | string | null): {
|
|
247
|
+
remove(): void;
|
|
248
|
+
};
|
|
249
|
+
/** Tell the AI what just happened in the app, mid-session ("Night ended.
|
|
250
|
+
* Sam was eliminated."). Keep it to a sentence or two; long pushes are
|
|
251
|
+
* truncated. GPT-Live provider only; other providers ignore it. */
|
|
252
|
+
updateAgentContext(text: string): void;
|
|
211
253
|
/** Subscribe to state events. Returns a subscription; call .remove(). */
|
|
212
254
|
addListener<E extends CrowdPlayEventName>(event: E, listener: (payload: CrowdPlayEvents[E]) => void): {
|
|
213
255
|
remove(): void;
|
package/lib/index.js
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
/**
|
|
3
|
-
* crowdplaysdk
|
|
3
|
+
* crowdplaysdk: CrowdPlay's lossless capture SDK for React Native (iOS).
|
|
4
4
|
*
|
|
5
5
|
* The recording engine is 100% native (CrowdPlaySDK): studio-grade audio
|
|
6
6
|
* (48 kHz / 24-bit, unprocessed) + 1080p30 video captured locally during a
|
|
7
7
|
* live call, clock-synced across participants, uploaded to CrowdPlay with
|
|
8
|
-
* retries and crash recovery. **No media ever crosses the JS bridge
|
|
9
|
-
* only control calls and state events
|
|
8
|
+
* retries and crash recovery. **No media ever crosses the JS bridge**,
|
|
9
|
+
* only control calls and state events, so recording quality is identical
|
|
10
10
|
* to a fully native app.
|
|
11
11
|
*
|
|
12
12
|
* import CrowdPlay, { CrowdPlayConsentScreen, CrowdPlayVideoView } from 'crowdplaysdk';
|
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
* await CrowdPlay.leave(); // recording stops; uploads continue automatically
|
|
19
19
|
*
|
|
20
20
|
* Consent is enforced twice: join() throws here without a grant, and the
|
|
21
|
-
* native layer requires the consent record structurally
|
|
21
|
+
* native layer requires the consent record structurally; there is no code
|
|
22
22
|
* path that records without it.
|
|
23
23
|
*/
|
|
24
24
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
@@ -30,8 +30,8 @@ function native() {
|
|
|
30
30
|
throw new Error("crowdplaysdk: native module not found. iOS setup: add the CrowdPlaySDK " +
|
|
31
31
|
"Swift package (https://github.com/symbiateam/crowdplaysdk) to your Xcode " +
|
|
32
32
|
"project and add the two bridge files from node_modules/crowdplaysdk/ios " +
|
|
33
|
-
"to your app target
|
|
34
|
-
(react_native_1.Platform.OS !== 'ios' ? `(platform '${react_native_1.Platform.OS}' is not supported yet
|
|
33
|
+
"to your app target; see the package README. " +
|
|
34
|
+
(react_native_1.Platform.OS !== 'ios' ? `(platform '${react_native_1.Platform.OS}' is not supported yet; iOS only.)` : ''));
|
|
35
35
|
}
|
|
36
36
|
return module;
|
|
37
37
|
}
|
|
@@ -73,7 +73,7 @@ const CrowdPlay = {
|
|
|
73
73
|
* Join a room. Recording starts automatically on join and stops on
|
|
74
74
|
* leave(). Rejects if the connection fails.
|
|
75
75
|
*
|
|
76
|
-
* `consent` is REQUIRED
|
|
76
|
+
* `consent` is REQUIRED; produce it with <CrowdPlayConsentScreen> or from
|
|
77
77
|
* your own UI at the moment of agreement. There is no way to record
|
|
78
78
|
* without it, here or natively.
|
|
79
79
|
*/
|
|
@@ -102,6 +102,11 @@ const CrowdPlay = {
|
|
|
102
102
|
put('voice', agent.voice, 60);
|
|
103
103
|
put('systemPrompt', agent.systemPrompt, 8192);
|
|
104
104
|
put('context', agent.context, 4000);
|
|
105
|
+
// Tools cross the bridge as one JSON string; the native layer and the
|
|
106
|
+
// server both re-validate (D-103).
|
|
107
|
+
if (Array.isArray(agent.tools) && agent.tools.length) {
|
|
108
|
+
put('toolsJSON', JSON.stringify(agent.tools.slice(0, 16)), 8192);
|
|
109
|
+
}
|
|
105
110
|
}
|
|
106
111
|
return native().join(displayName.trim(), roomCode, consent.grantedAtMs, agentFields);
|
|
107
112
|
},
|
|
@@ -144,6 +149,37 @@ const CrowdPlay = {
|
|
|
144
149
|
doctor() {
|
|
145
150
|
return native().doctor();
|
|
146
151
|
},
|
|
152
|
+
/** Handle the voice AI's tool calls (D-103). The handler gets the tool
|
|
153
|
+
* name and its arguments (parsed JSON object) and returns the result as
|
|
154
|
+
* an object or JSON string; return null to decline on this device
|
|
155
|
+
* (another participant's device may answer; first answer wins). Ordinary
|
|
156
|
+
* app code, no AI here. The AI waits about 12 seconds per call. Returns
|
|
157
|
+
* a subscription; call .remove(). */
|
|
158
|
+
onAgentToolCall(handler) {
|
|
159
|
+
const subscription = events().addListener('crowdplay:agentToolCall', (payload) => {
|
|
160
|
+
void (async () => {
|
|
161
|
+
let args = {};
|
|
162
|
+
try {
|
|
163
|
+
args = JSON.parse(payload.argumentsJSON);
|
|
164
|
+
}
|
|
165
|
+
catch { /* {} */ }
|
|
166
|
+
const result = await handler(payload.name, args);
|
|
167
|
+
if (result === null || result === undefined)
|
|
168
|
+
return;
|
|
169
|
+
const output = typeof result === 'string' ? result : JSON.stringify(result);
|
|
170
|
+
native().respondToAgentToolCall(payload.callId, output);
|
|
171
|
+
})();
|
|
172
|
+
});
|
|
173
|
+
return { remove: () => subscription.remove() };
|
|
174
|
+
},
|
|
175
|
+
/** Tell the AI what just happened in the app, mid-session ("Night ended.
|
|
176
|
+
* Sam was eliminated."). Keep it to a sentence or two; long pushes are
|
|
177
|
+
* truncated. GPT-Live provider only; other providers ignore it. */
|
|
178
|
+
updateAgentContext(text) {
|
|
179
|
+
if (typeof text === 'string' && text.trim()) {
|
|
180
|
+
native().updateAgentContext(text.trim());
|
|
181
|
+
}
|
|
182
|
+
},
|
|
147
183
|
/** Subscribe to state events. Returns a subscription; call .remove(). */
|
|
148
184
|
addListener(event, listener) {
|
|
149
185
|
const subscription = events().addListener(`crowdplay:${event}`, listener);
|
package/llms.txt
CHANGED
|
@@ -29,7 +29,7 @@ Consequences:
|
|
|
29
29
|
- Recording starts automatically on join() and stops on leave(). Do not
|
|
30
30
|
build a separate record button.
|
|
31
31
|
|
|
32
|
-
## The HOST APP is the product
|
|
32
|
+
## The HOST APP is the product, not recording
|
|
33
33
|
|
|
34
34
|
The builder came to you with an app idea (a tutoring app, a language-
|
|
35
35
|
exchange app, a remote interview tool…). Build THAT app. The SDK records
|
|
@@ -37,11 +37,11 @@ its video calls invisibly; recording is infrastructure, not identity:
|
|
|
37
37
|
- The app's NAME, description, and screens are about the builder's idea.
|
|
38
38
|
Never name or describe the app around recording.
|
|
39
39
|
- Recording surfaces in the UI in exactly four places: the consent
|
|
40
|
-
screen before joining (required
|
|
40
|
+
screen before joining (required, never minimized away), a small REC
|
|
41
41
|
indicator during the call, join/leave, and a small audio-output menu
|
|
42
42
|
(REQUIRED standard control: show the current output device by name from
|
|
43
43
|
the 'audioRoute' event, with a switch between the device and Speaker
|
|
44
|
-
via CrowdPlay.setAudioOutput
|
|
44
|
+
via CrowdPlay.setAudioOutput; Bluetooth users expect it).
|
|
45
45
|
- Do NOT build upload-progress screens, diagnostics screens, SDK settings
|
|
46
46
|
pages, or recording libraries/playback. Uploads run automatically in
|
|
47
47
|
the background; doctor() is a development-time check, not a screen.
|
|
@@ -56,7 +56,7 @@ its video calls invisibly; recording is infrastructure, not identity:
|
|
|
56
56
|
add audio-session-touching RN packages like react-native-incall-manager)
|
|
57
57
|
in an app using this SDK. The engine owns the microphone and camera.
|
|
58
58
|
3. NEVER skip or fake consent. CrowdPlay.join() THROWS without a ConsentGrant,
|
|
59
|
-
and the user must actually SEE the consent wording first
|
|
59
|
+
and the user must actually SEE the consent wording first; use
|
|
60
60
|
<CrowdPlayConsentScreen/>, or show CrowdPlay.consentText() verbatim in your own
|
|
61
61
|
UI and construct { grantedAtMs: Date.now() } at the moment of
|
|
62
62
|
agreement. Fabricating a grant creates false legal evidence.
|
|
@@ -79,7 +79,7 @@ its video calls invisibly; recording is infrastructure, not identity:
|
|
|
79
79
|
4. Info.plist: NSMicrophoneUsageDescription, NSCameraUsageDescription,
|
|
80
80
|
UIBackgroundModes = [audio]. A missing usage string makes iOS kill
|
|
81
81
|
the app at first mic/camera access (uncatchable).
|
|
82
|
-
5. AppDelegate (recommended
|
|
82
|
+
5. AppDelegate (recommended; lets uploads finish while suspended):
|
|
83
83
|
// AppDelegate.mm, with #import "<YourProjectName>-Swift.h"
|
|
84
84
|
- (void)application:(UIApplication *)application
|
|
85
85
|
handleEventsForBackgroundURLSession:(NSString *)identifier
|
|
@@ -112,18 +112,18 @@ its video calls invisibly; recording is infrastructure, not identity:
|
|
|
112
112
|
already went out during the call). Show at most one quiet
|
|
113
113
|
"finishing up…" line while an 'uploads' session is incomplete; do
|
|
114
114
|
NOT build a dedicated upload screen.
|
|
115
|
-
10. Verify: const checks = await CrowdPlay.doctor();
|
|
115
|
+
10. Verify: const checks = await CrowdPlay.doctor(); every failing check's
|
|
116
116
|
detail names its own fix. NOTE: on a fresh install the "permissions"
|
|
117
|
-
check reports "not requested yet
|
|
118
|
-
That is not an integration bug
|
|
117
|
+
check reports "not requested yet (NORMAL)"; iOS prompts at first join.
|
|
118
|
+
That is not an integration bug; only a DENIED state needs action.
|
|
119
119
|
11. Hand off (REQUIRED final message): the Simulator has no camera or
|
|
120
120
|
mic, so nothing is proven until the app runs on a real iPhone. Do
|
|
121
|
-
not end with "the implementation is complete"
|
|
121
|
+
not end with "the implementation is complete"; your FINAL message
|
|
122
122
|
must walk the builder through: plugging in the iPhone (Trust +
|
|
123
123
|
Developer Mode), picking a signing Team in Xcode and pressing Run on
|
|
124
124
|
the device (via the .xcworkspace), allowing mic/camera permissions,
|
|
125
125
|
a ~1-minute test call, and checking the session at
|
|
126
|
-
https://dashboard.crowdplay.ai
|
|
126
|
+
https://dashboard.crowdplay.ai; that is the end-to-end proof.
|
|
127
127
|
|
|
128
128
|
## API reference (exact TypeScript surface)
|
|
129
129
|
|
|
@@ -155,12 +155,12 @@ useVoiceActivity() -> { phase: 'connecting'|'idle'|'listening'|'speaking', energ
|
|
|
155
155
|
|
|
156
156
|
The voice AI (apps with the Voice AI switch enabled on the dashboard):
|
|
157
157
|
|
|
158
|
-
How it works
|
|
158
|
+
How it works (all server-side, nothing to wire up). When any user joins a
|
|
159
159
|
call, CrowdPlay's servers put a live AI participant into the room within
|
|
160
160
|
~3 seconds. It joins with name "agent" (identity "agent-<8hex>"), greets
|
|
161
161
|
the user, and holds a real spoken conversation (speech-to-speech: it
|
|
162
162
|
hears the user directly, answers in natural voice, can be interrupted).
|
|
163
|
-
The app writes ZERO AI code
|
|
163
|
+
The app writes ZERO AI code: no model API calls, no AI keys, no chat
|
|
164
164
|
backend, no STT/TTS. Provider, model and API key are set by the BUILDER on
|
|
165
165
|
the CrowdPlay dashboard, and so are the DEFAULT persona and voice.
|
|
166
166
|
Per-join overrides (0.4.0+): the app MAY pass `agent` to join() to change
|
|
@@ -169,42 +169,67 @@ the voice and the persona and give a short context text for one session:
|
|
|
169
169
|
agent: { voice: 'Aoede', context: "The user's name is Sam. Last time you practised past tense." } });
|
|
170
170
|
voice must be one of the configured provider's voices (Gemini: Charon,
|
|
171
171
|
Aoede, Fenrir, Kore, Puck; OpenAI: marin, cedar, alloy, ash, ballad,
|
|
172
|
-
coral, echo, sage, shimmer, verse; Grok: Ara, Rex, Sal, Eve, Leo
|
|
172
|
+
coral, echo, sage, shimmer, verse; Grok: Ara, Rex, Sal, Eve, Leo;
|
|
173
|
+
OpenAI GPT-Live: quartz, ripple, vesper, willow, stone, gleam, meridian,
|
|
174
|
+
bossa, tempo, beacon, delta, cinder), unknown
|
|
173
175
|
falls back to the dashboard voice. Caps: systemPrompt 8192 chars, context
|
|
174
176
|
4000. The options of the join that CREATES the AI apply to the whole room;
|
|
175
177
|
fixed for the session (leave and join again to change); provider, model
|
|
176
178
|
and key cannot be changed from the app. MEMORY is built with `context`:
|
|
177
179
|
keep a few sentences per user in the app's own storage and pass them at
|
|
178
180
|
every join.
|
|
181
|
+
Agent tools + live context (0.5.0+, D-103; needs the app's dashboard
|
|
182
|
+
config on the OpenAI GPT-Live provider with an AI brain enabled —
|
|
183
|
+
ignored otherwise, never an error). Tools let the AI SEE app state and
|
|
184
|
+
ACT in the app: pass them at join, handle calls with ordinary JS.
|
|
185
|
+
await CrowdPlay.join({ displayName, roomCode, consent: grant,
|
|
186
|
+
agent: { tools: [
|
|
187
|
+
{ name: 'get_game_state',
|
|
188
|
+
description: 'The current players, roles you know, and votes.' },
|
|
189
|
+
{ name: 'cast_vote', description: 'Vote to eliminate one player.',
|
|
190
|
+
parameters: { type: 'object',
|
|
191
|
+
properties: { player: { type: 'string' } }, required: ['player'] } },
|
|
192
|
+
] } });
|
|
193
|
+
const sub = CrowdPlay.onAgentToolCall(async (name, args) => {
|
|
194
|
+
if (name === 'get_game_state') return game.state();
|
|
195
|
+
if (name === 'cast_vote') return game.castVote(args.player);
|
|
196
|
+
return null; // decline on this device; first device to answer wins
|
|
197
|
+
});
|
|
198
|
+
Return an object (or JSON string), or null to decline. The AI waits
|
|
199
|
+
about 12 seconds per call, then carries on gracefully. Max 16 tools;
|
|
200
|
+
names letters/digits/underscore/dash up to 64 chars. To push events to
|
|
201
|
+
the AI as they happen (it has no other way to know):
|
|
202
|
+
CrowdPlay.updateAgentContext('Night ended. Sam was eliminated.') — a
|
|
203
|
+
sentence or two, long pushes are truncated.
|
|
179
204
|
Conversations are recorded and transcribed by the platform automatically.
|
|
180
|
-
RECOMMENDED: voice-AI apps should be AUDIO-ONLY (audioOnly: true
|
|
205
|
+
RECOMMENDED: voice-AI apps should be AUDIO-ONLY (audioOnly: true; no
|
|
181
206
|
camera permission, no video anywhere); a camera adds nothing to talking
|
|
182
207
|
with an AI. Keep video only when the app ALSO has calls between people.
|
|
183
208
|
|
|
184
|
-
The AI conversation screen
|
|
209
|
+
The AI conversation screen, the approved standard layout: the animated
|
|
185
210
|
visual front and center, large (it IS the screen); NO self camera
|
|
186
211
|
preview and NO video tiles on this screen, ever; around it only the AI's
|
|
187
212
|
name/persona, a small REC indicator, a leave button, and the small
|
|
188
213
|
audio-output menu. One calm screen, one living visual, nothing competing
|
|
189
214
|
with it.
|
|
190
215
|
|
|
191
|
-
What the app builds
|
|
216
|
+
What the app builds: the AI's on-screen presence. The AI publishes
|
|
192
217
|
AUDIO ONLY: never a camera tile, an empty box, or a static image; never
|
|
193
218
|
count it as a generic user. Detect it in the participants event by name
|
|
194
219
|
"agent" (or identity prefix "agent-").
|
|
195
220
|
- The visual is a MENU, zero-effort to fully custom (0.3.4+ for 2-5):
|
|
196
|
-
1. Orb (default): <CrowdPlayVoiceView
|
|
221
|
+
1. Orb (default): <CrowdPlayVoiceView />, breathes idle, rings with
|
|
197
222
|
the user's voice, pulses with the AI's. Props: agentColor,
|
|
198
223
|
listeningColor, idleColor, size.
|
|
199
|
-
2. Bars: <CrowdPlayVoiceView variant='bars'
|
|
224
|
+
2. Bars: <CrowdPlayVoiceView variant='bars' />, equalizer look, good
|
|
200
225
|
under other content (scoreboards, question cards, quizzes).
|
|
201
226
|
3. Halo around the app's own art: pass a mascot portrait / logo /
|
|
202
|
-
character as children of <CrowdPlayVoiceView
|
|
227
|
+
character as children of <CrowdPlayVoiceView>, animated ring +
|
|
203
228
|
glow around whatever you provide.
|
|
204
|
-
4. Make ANY view alive: wrap it in <VoiceReactive
|
|
229
|
+
4. Make ANY view alive: wrap it in <VoiceReactive>, breathing, pulse
|
|
205
230
|
and speech glow on the app's own art in one line.
|
|
206
231
|
5. Fully custom (best for characters): drive ANY visual from the
|
|
207
|
-
useVoiceActivity() hook
|
|
232
|
+
useVoiceActivity() hook, returning { phase: 'connecting'|'idle'|'listening'|
|
|
208
233
|
'speaking', energy: 0…1 smoothed loudness }. Example, an illustrated
|
|
209
234
|
tutor that reacts to the conversation:
|
|
210
235
|
|
|
@@ -215,10 +240,10 @@ count it as a generic user. Detect it in the participants event by name
|
|
|
215
240
|
pose={phase === 'listening' ? 'leanIn' : 'idleBlink'} />;
|
|
216
241
|
}
|
|
217
242
|
|
|
218
|
-
Any art pipeline works (Animated, Lottie, Rive, sprite frames)
|
|
243
|
+
Any art pipeline works (Animated, Lottie, Rive, sprite frames): bind
|
|
219
244
|
mouth/pose/glow to phase + energy.
|
|
220
245
|
- Live captions (0.3.4+): each line the AI and the user say is streamed
|
|
221
|
-
as it is spoken
|
|
246
|
+
as it is spoken; the useLatestCaption() hook returns { role, text }.
|
|
222
247
|
For speech bubbles, subtitles, language-learning corrections. Optional
|
|
223
248
|
UI; the full timestamped transcript is always delivered with the
|
|
224
249
|
session regardless.
|
|
@@ -228,8 +253,14 @@ count it as a generic user. Detect it in the participants event by name
|
|
|
228
253
|
Types:
|
|
229
254
|
CrowdPlayConfig { serverUrl; appKey; audioOnly?=false;
|
|
230
255
|
videoUploadsOnCellular?=false;
|
|
231
|
-
uploadDuringCall?=true;
|
|
232
|
-
|
|
256
|
+
uploadDuringCall?=true;
|
|
257
|
+
otherAudio?='interrupt'|'mix'|'mixDucked'; callQuality? }
|
|
258
|
+
// otherAudio (0.4.0+): 'interrupt' pauses other apps'
|
|
259
|
+
// audio at join (classic call); 'mix' keeps Apple Music /
|
|
260
|
+
// Spotify playing through the call and the recording, for
|
|
261
|
+
// shared-listening apps; 'mixDucked' also lowers it. Music
|
|
262
|
+
// never rides the call; listeners must wear headphones.
|
|
263
|
+
// audioOnly (0.3.0+): the camera is never touched; no
|
|
233
264
|
// camera permission, no video captured or uploaded,
|
|
234
265
|
// voice-only calls. Skip NSCameraUsageDescription, skip
|
|
235
266
|
// CrowdPlayVideoView and every camera control.
|
|
@@ -242,8 +273,8 @@ Types:
|
|
|
242
273
|
ConsentGrant { grantedAtMs: number }
|
|
243
274
|
RecordingSnapshot { seconds; segmentsClosed; droppedSamples;
|
|
244
275
|
inputLevelDbfs; // speech ≈ −25…−45 dBFS
|
|
245
|
-
inputSilent; // dead mic
|
|
246
|
-
fullScaleSamples } // clipping
|
|
276
|
+
inputSilent; // dead mic: surface loudly
|
|
277
|
+
fullScaleSamples } // clipping: mic too hot
|
|
247
278
|
UploadProgress { sessionId; fraction; isComplete; failed;
|
|
248
279
|
audio: UploadKindProgress; video: UploadKindProgress }
|
|
249
280
|
DoctorCheck { id; passed; detail }
|
|
@@ -269,7 +300,7 @@ Events (CrowdPlay.addListener):
|
|
|
269
300
|
// this mic (wear headphones); micPolicy = unexpected
|
|
270
301
|
// input device.
|
|
271
302
|
|
|
272
|
-
## Accessing recordings (no AWS, no SDK
|
|
303
|
+
## Accessing recordings (no AWS, no SDK, plain HTTPS)
|
|
273
304
|
|
|
274
305
|
The app key doubles as the data credential:
|
|
275
306
|
GET https://dashboard.crowdplay.ai/sessions (x-liva-key header)
|
|
@@ -289,21 +320,21 @@ Humans use https://dashboard.crowdplay.ai directly (sign in, download).
|
|
|
289
320
|
pauses uploads until next launch (nothing lost).
|
|
290
321
|
- Participants should wear wired or closed-back headphones (crossTalk
|
|
291
322
|
warns live when they don't).
|
|
292
|
-
- SIMULATOR detection (engine 0.3.0+): on any Simulator
|
|
293
|
-
previews
|
|
323
|
+
- SIMULATOR detection (engine 0.3.0+): on any Simulator, including CLOUD
|
|
324
|
+
previews, doctor()'s "platform" check fails and join() rejects with a
|
|
294
325
|
clear message instead of a cryptic audio-engine error. Expected, not a
|
|
295
326
|
bug: no microphone exists there. A doctor() run also flips the "health
|
|
296
327
|
check" step on the builder's dashboard checklist automatically.
|
|
297
328
|
- Slow join or glitchy first seconds of a recording = poor radio
|
|
298
329
|
environment (congested 2.4 GHz WiFi + Bluetooth share the antenna) or
|
|
299
|
-
Metro/debugger overhead
|
|
330
|
+
Metro/debugger overhead; NOT an integration bug. Capture self-heals.
|
|
300
331
|
Advise: better network, wired headphones, evaluate from a home-screen
|
|
301
332
|
launch (release build).
|
|
302
333
|
- Build error mentioning LiveKit/WebRTC duplicate symbols: the app added
|
|
303
|
-
its own LiveKit dependency
|
|
334
|
+
its own LiveKit dependency; remove it (rule 1).
|
|
304
335
|
- "Swift bridging" or "<Project>-Swift.h not found" errors: rerun
|
|
305
336
|
ios/wire.rb, then a clean build (the header is generated at build time).
|
|
306
|
-
- 401/"unauthorized": wrong app key
|
|
337
|
+
- 401/"unauthorized": wrong app key; sign in at
|
|
307
338
|
https://dashboard.crowdplay.ai, reveal the key, and copy it again.
|
|
308
339
|
- Uploads crawling on an otherwise-fine network: check for a VPN, iCloud
|
|
309
340
|
Private Relay, or Low Data Mode on the phone (a VPN can cut throughput
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "crowdplaysdk",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.5.0",
|
|
4
4
|
"description": "CrowdPlay lossless conversation capture for React Native (iOS). Studio-grade per-participant recording during live video calls, delivered to CrowdPlay automatically.",
|
|
5
5
|
"main": "lib/index.js",
|
|
6
6
|
"types": "lib/index.d.ts",
|
package/src/ConsentScreen.tsx
CHANGED
|
@@ -33,7 +33,7 @@ export function CrowdPlayConsentScreen({ onConsent, style }: CrowdPlayConsentScr
|
|
|
33
33
|
if (!cancelled) setText(wording);
|
|
34
34
|
})
|
|
35
35
|
.catch(() => {
|
|
36
|
-
if (!cancelled) setText('Consent text unavailable
|
|
36
|
+
if (!cancelled) setText('Consent text unavailable. Check that CrowdPlay.configure() ran and the native module is installed.');
|
|
37
37
|
});
|
|
38
38
|
return () => {
|
|
39
39
|
cancelled = true;
|
package/src/VideoView.tsx
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Native video tile. Renders the LOCAL camera preview or a REMOTE
|
|
3
3
|
* participant's live video (identified by the `identity` from the
|
|
4
|
-
* `participants` event). The video never touches JavaScript
|
|
4
|
+
* `participants` event). The video never touches JavaScript; this is a
|
|
5
5
|
* native LiveKit view hosted in your RN layout.
|
|
6
6
|
*
|
|
7
7
|
* <CrowdPlayVideoView participant="local" style={{ width: 120, height: 160 }} />
|
package/src/VoiceView.tsx
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* The standard voice-AI visual (founders, 2026-08-22): a breathing orb in
|
|
3
|
-
* the style of realtime assistants
|
|
3
|
+
* the style of realtime assistants: calm while idle, a cool ring while the
|
|
4
4
|
* user talks, an energetic pulse while the AI speaks. The React Native
|
|
5
5
|
* counterpart of the native SDK's CrowdPlayVoiceView, driven by the same
|
|
6
6
|
* engine signal over a fast bridge event that runs only while mounted.
|
|
7
7
|
*
|
|
8
8
|
* Two levels of customization:
|
|
9
|
-
* 1. Props on <CrowdPlayVoiceView
|
|
10
|
-
* 2. useVoiceActivity()
|
|
9
|
+
* 1. Props on <CrowdPlayVoiceView>: colors and size of the built-in orb.
|
|
10
|
+
* 2. useVoiceActivity(): the phase + smoothed energy underneath, for apps
|
|
11
11
|
* that want to draw something entirely their own.
|
|
12
12
|
*/
|
|
13
13
|
|
|
@@ -58,7 +58,7 @@ export function useVoiceActivity(): VoiceActivity {
|
|
|
58
58
|
phase = 'idle';
|
|
59
59
|
target = 0;
|
|
60
60
|
}
|
|
61
|
-
// Fast attack, slow release
|
|
61
|
+
// Fast attack, slow release: speech onsets snap, tails breathe out.
|
|
62
62
|
const rate = target > energyRef.current ? 0.55 : 0.12;
|
|
63
63
|
energyRef.current += (target - energyRef.current) * rate;
|
|
64
64
|
setActivity({ phase, energy: energyRef.current });
|
|
@@ -92,7 +92,7 @@ export function useLatestCaption(): VoiceCaption | null {
|
|
|
92
92
|
return caption;
|
|
93
93
|
}
|
|
94
94
|
|
|
95
|
-
/** Make ANY child view breathe, pulse, and glow with the conversation
|
|
95
|
+
/** Make ANY child view breathe, pulse, and glow with the conversation;
|
|
96
96
|
* the one-line path from the app's own art (mascot, logo, character) to a
|
|
97
97
|
* living AI presence. */
|
|
98
98
|
export function VoiceReactive({
|