@akhilpulse/samadhaan-session-replay 0.4.3 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -5
- package/SamadhaanSessionReplay.podspec +17 -0
- package/android/src/main/java/com/akhilpulse/sessionreplay/SessionReplayPcmPlayerModule.kt +119 -0
- package/android/src/main/java/com/akhilpulse/sessionreplay/SessionReplayPixelCopyPackage.kt +4 -1
- package/ios/SessionReplayPcmPlayer.h +11 -0
- package/ios/SessionReplayPcmPlayer.m +161 -0
- package/package.json +3 -1
- package/src/SessionReplayProvider.tsx +23 -4
- package/src/index.ts +2 -0
- package/src/voice.ts +69 -3
package/README.md
CHANGED
|
@@ -122,15 +122,39 @@ if a module is missing that capability is simply skipped (no crash):
|
|
|
122
122
|
| Capability | Package | If missing |
|
|
123
123
|
|---------------------------|------------------------------------|------------|
|
|
124
124
|
| Shake detection | `react-native-shake` | Use `triggerAssist()` manually |
|
|
125
|
-
| Microphone (voice input) | `react-native-live-audio-stream` |
|
|
126
|
-
| Agent voice playback |
|
|
125
|
+
| Microphone (voice input) | `react-native-live-audio-stream` | Voice bar shows "Mic unavailable" — assistant can still speak + draw arrows |
|
|
126
|
+
| Agent voice playback | **built-in on Android (v0.5.0+) and iOS (v0.6.0+)**; custom `PcmAudioPlayer` optional | In Expo Go / stale builds without the native module, agent audio is dropped (arrows + transcript still work) |
|
|
127
127
|
|
|
128
128
|
```bash
|
|
129
129
|
npm install react-native-shake react-native-live-audio-stream
|
|
130
130
|
cd ios && pod install
|
|
131
131
|
```
|
|
132
132
|
|
|
133
|
-
|
|
133
|
+
#### Microphone permission (required for voice input)
|
|
134
|
+
|
|
135
|
+
- **Android** — declare in your app's `AndroidManifest.xml`:
|
|
136
|
+
```xml
|
|
137
|
+
<uses-permission android:name="android.permission.RECORD_AUDIO" />
|
|
138
|
+
```
|
|
139
|
+
Starting in **0.5.0 the SDK requests the runtime permission automatically**
|
|
140
|
+
the first time the assistant opens. If the user denies it, the voice bar
|
|
141
|
+
shows "Mic blocked — allow microphone access in Settings" instead of
|
|
142
|
+
pretending to listen.
|
|
143
|
+
- **iOS** — add `NSMicrophoneUsageDescription` to `Info.plist` or the OS will
|
|
144
|
+
kill the app when recording starts.
|
|
145
|
+
|
|
146
|
+
#### Agent voice playback
|
|
147
|
+
|
|
148
|
+
The agent streams **int16 mono PCM at 16 kHz**. The SDK ships built-in native
|
|
149
|
+
players used automatically when you don't pass `assistantAudioPlayer`:
|
|
150
|
+
|
|
151
|
+
- **Android (0.5.0+)** — AudioTrack streaming player. Rebuild the app after
|
|
152
|
+
upgrading (`cd android && ./gradlew clean && cd .. && npx react-native run-android`).
|
|
153
|
+
- **iOS (0.6.0+)** — AVAudioEngine player (autolinked via the bundled
|
|
154
|
+
`SamadhaanSessionReplay` podspec). After upgrading run
|
|
155
|
+
`cd ios && pod install`, then rebuild the app.
|
|
156
|
+
|
|
157
|
+
To override with your own player:
|
|
134
158
|
|
|
135
159
|
```tsx
|
|
136
160
|
import type { PcmAudioPlayer } from "@akhilpulse/samadhaan-session-replay";
|
|
@@ -145,8 +169,9 @@ const myPlayer: PcmAudioPlayer = {
|
|
|
145
169
|
<SessionReplayProvider config={{ appId: "app_xxx", assistantAudioPlayer: myPlayer }}>
|
|
146
170
|
```
|
|
147
171
|
|
|
148
|
-
After installing native deps
|
|
149
|
-
|
|
172
|
+
After installing native deps, **rebuild the app** — Android:
|
|
173
|
+
`cd android && ./gradlew clean && cd .. && npx react-native run-android`;
|
|
174
|
+
iOS: `cd ios && pod install && cd .. && npx react-native run-ios`.
|
|
150
175
|
|
|
151
176
|
## What it does
|
|
152
177
|
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
require "json"
|
|
2
|
+
|
|
3
|
+
package = JSON.parse(File.read(File.join(__dir__, "package.json")))
|
|
4
|
+
|
|
5
|
+
Pod::Spec.new do |s|
|
|
6
|
+
s.name = "SamadhaanSessionReplay"
|
|
7
|
+
s.version = package["version"]
|
|
8
|
+
s.summary = package["description"]
|
|
9
|
+
s.homepage = package["homepage"]
|
|
10
|
+
s.license = { :type => package["license"] || "UNLICENSED" }
|
|
11
|
+
s.authors = { "Akhil / Pulse Energy" => "support@pulseenergy.io" }
|
|
12
|
+
s.platforms = { :ios => "12.0" }
|
|
13
|
+
s.source = { :git => "https://samadhaan.pulseenergy.io", :tag => s.version.to_s }
|
|
14
|
+
s.source_files = "ios/**/*.{h,m,mm}"
|
|
15
|
+
s.frameworks = "AVFoundation"
|
|
16
|
+
s.dependency "React-Core"
|
|
17
|
+
end
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
package com.akhilpulse.sessionreplay
|
|
2
|
+
|
|
3
|
+
import android.media.AudioAttributes
|
|
4
|
+
import android.media.AudioFormat
|
|
5
|
+
import android.media.AudioTrack
|
|
6
|
+
import android.os.Handler
|
|
7
|
+
import android.os.HandlerThread
|
|
8
|
+
import android.util.Base64
|
|
9
|
+
import com.facebook.react.bridge.ReactApplicationContext
|
|
10
|
+
import com.facebook.react.bridge.ReactContextBaseJavaModule
|
|
11
|
+
import com.facebook.react.bridge.ReactMethod
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Built-in streaming PCM player for the Shake-to-Assist agent voice
|
|
15
|
+
* (v0.5.0+). Plays int16 mono PCM chunks (base64 over the bridge) through
|
|
16
|
+
* a single reused AudioTrack in streaming mode, so host apps get agent
|
|
17
|
+
* audio out of the box without wiring their own PcmAudioPlayer.
|
|
18
|
+
*
|
|
19
|
+
* All AudioTrack work happens on a dedicated HandlerThread — never on the
|
|
20
|
+
* React native-modules thread — because AudioTrack.write() in blocking
|
|
21
|
+
* mode can stall for the duration of the buffered audio.
|
|
22
|
+
*/
|
|
23
|
+
class SessionReplayPcmPlayerModule(reactContext: ReactApplicationContext) :
|
|
24
|
+
ReactContextBaseJavaModule(reactContext) {
|
|
25
|
+
|
|
26
|
+
override fun getName() = "SessionReplayPcmPlayer"
|
|
27
|
+
|
|
28
|
+
private var track: AudioTrack? = null
|
|
29
|
+
private var thread: HandlerThread? = null
|
|
30
|
+
private var handler: Handler? = null
|
|
31
|
+
|
|
32
|
+
private fun ensureHandler(): Handler {
|
|
33
|
+
val alive = thread
|
|
34
|
+
val h = handler
|
|
35
|
+
if (alive != null && alive.isAlive && h != null) return h
|
|
36
|
+
val nt = HandlerThread("SessionReplayPcmPlayer")
|
|
37
|
+
nt.start()
|
|
38
|
+
thread = nt
|
|
39
|
+
val nh = Handler(nt.looper)
|
|
40
|
+
handler = nh
|
|
41
|
+
return nh
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
private fun releaseTrack() {
|
|
45
|
+
try { track?.stop() } catch (_: Throwable) {}
|
|
46
|
+
try { track?.release() } catch (_: Throwable) {}
|
|
47
|
+
track = null
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
@ReactMethod
|
|
51
|
+
fun start(sampleRateIn: Int) {
|
|
52
|
+
ensureHandler().post {
|
|
53
|
+
try {
|
|
54
|
+
releaseTrack()
|
|
55
|
+
val sampleRate = if (sampleRateIn > 0) sampleRateIn else 16000
|
|
56
|
+
val minBuf = AudioTrack.getMinBufferSize(
|
|
57
|
+
sampleRate, AudioFormat.CHANNEL_OUT_MONO, AudioFormat.ENCODING_PCM_16BIT)
|
|
58
|
+
// ~0.5 s of 16-bit mono headroom, never below the platform minimum.
|
|
59
|
+
val bufBytes = maxOf(minBuf, sampleRate)
|
|
60
|
+
val t = AudioTrack.Builder()
|
|
61
|
+
.setAudioAttributes(
|
|
62
|
+
AudioAttributes.Builder()
|
|
63
|
+
.setUsage(AudioAttributes.USAGE_MEDIA)
|
|
64
|
+
.setContentType(AudioAttributes.CONTENT_TYPE_SPEECH)
|
|
65
|
+
.build())
|
|
66
|
+
.setAudioFormat(
|
|
67
|
+
AudioFormat.Builder()
|
|
68
|
+
.setEncoding(AudioFormat.ENCODING_PCM_16BIT)
|
|
69
|
+
.setSampleRate(sampleRate)
|
|
70
|
+
.setChannelMask(AudioFormat.CHANNEL_OUT_MONO)
|
|
71
|
+
.build())
|
|
72
|
+
.setBufferSizeInBytes(bufBytes)
|
|
73
|
+
.setTransferMode(AudioTrack.MODE_STREAM)
|
|
74
|
+
.build()
|
|
75
|
+
t.play()
|
|
76
|
+
track = t
|
|
77
|
+
} catch (_: Throwable) {
|
|
78
|
+
track = null
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
@ReactMethod
|
|
84
|
+
fun play(base64Pcm: String) {
|
|
85
|
+
ensureHandler().post {
|
|
86
|
+
val t = track ?: return@post
|
|
87
|
+
try {
|
|
88
|
+
val bytes = Base64.decode(base64Pcm, Base64.DEFAULT)
|
|
89
|
+
if (bytes.isNotEmpty()) t.write(bytes, 0, bytes.size)
|
|
90
|
+
} catch (_: Throwable) {}
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/** Barge-in: drop any queued agent audio immediately. */
|
|
95
|
+
@ReactMethod
|
|
96
|
+
fun clear() {
|
|
97
|
+
ensureHandler().post {
|
|
98
|
+
val t = track ?: return@post
|
|
99
|
+
try {
|
|
100
|
+
t.pause()
|
|
101
|
+
t.flush()
|
|
102
|
+
t.play()
|
|
103
|
+
} catch (_: Throwable) {}
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
@ReactMethod
|
|
108
|
+
fun stop() {
|
|
109
|
+
ensureHandler().post { releaseTrack() }
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
override fun invalidate() {
|
|
113
|
+
super.invalidate()
|
|
114
|
+
try { handler?.post { releaseTrack() } } catch (_: Throwable) {}
|
|
115
|
+
try { thread?.quitSafely() } catch (_: Throwable) {}
|
|
116
|
+
thread = null
|
|
117
|
+
handler = null
|
|
118
|
+
}
|
|
119
|
+
}
|
|
@@ -7,7 +7,10 @@ import com.facebook.react.uimanager.ViewManager
|
|
|
7
7
|
|
|
8
8
|
class SessionReplayPixelCopyPackage : ReactPackage {
|
|
9
9
|
override fun createNativeModules(reactContext: ReactApplicationContext): List<NativeModule> =
|
|
10
|
-
listOf(
|
|
10
|
+
listOf(
|
|
11
|
+
SessionReplayPixelCopyModule(reactContext),
|
|
12
|
+
SessionReplayPcmPlayerModule(reactContext),
|
|
13
|
+
)
|
|
11
14
|
|
|
12
15
|
override fun createViewManagers(reactContext: ReactApplicationContext): List<ViewManager<*, *>> =
|
|
13
16
|
emptyList()
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
#import <React/RCTBridgeModule.h>
|
|
2
|
+
#import <React/RCTInvalidating.h>
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Streaming PCM16 player for the Shake-to-Assist agent voice (iOS).
|
|
6
|
+
* Mirrors the Android SessionReplayPcmPlayerModule: start(sampleRate),
|
|
7
|
+
* play(base64 int16 mono chunks), clear() for barge-in, stop().
|
|
8
|
+
* Backed by AVAudioEngine + AVAudioPlayerNode.
|
|
9
|
+
*/
|
|
10
|
+
@interface SessionReplayPcmPlayer : NSObject <RCTBridgeModule, RCTInvalidating>
|
|
11
|
+
@end
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
#import "SessionReplayPcmPlayer.h"
|
|
2
|
+
#import <React/RCTLog.h>
|
|
3
|
+
#import <AVFoundation/AVFoundation.h>
|
|
4
|
+
|
|
5
|
+
@implementation SessionReplayPcmPlayer {
|
|
6
|
+
AVAudioEngine *_engine;
|
|
7
|
+
AVAudioPlayerNode *_player;
|
|
8
|
+
AVAudioFormat *_format;
|
|
9
|
+
double _sampleRate;
|
|
10
|
+
BOOL _running;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
RCT_EXPORT_MODULE(SessionReplayPcmPlayer);
|
|
14
|
+
|
|
15
|
+
// All calls run on a dedicated serial queue so bridge threads are never blocked
|
|
16
|
+
// and engine state is never touched concurrently.
|
|
17
|
+
- (dispatch_queue_t)methodQueue
|
|
18
|
+
{
|
|
19
|
+
static dispatch_queue_t queue;
|
|
20
|
+
static dispatch_once_t onceToken;
|
|
21
|
+
dispatch_once(&onceToken, ^{
|
|
22
|
+
queue = dispatch_queue_create("com.akhilpulse.sessionreplay.pcmplayer", DISPATCH_QUEUE_SERIAL);
|
|
23
|
+
});
|
|
24
|
+
return queue;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
+ (BOOL)requiresMainQueueSetup
|
|
28
|
+
{
|
|
29
|
+
return NO;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
- (void)setUpEngineWithSampleRate:(double)sampleRate
|
|
33
|
+
{
|
|
34
|
+
[self tearDownEngine];
|
|
35
|
+
|
|
36
|
+
// Voice assistant runs duplex (mic capture via react-native-live-audio-stream
|
|
37
|
+
// + agent playback here, simultaneously), so the session must stay
|
|
38
|
+
// record-capable. Only touch the category if it isn't already PlayAndRecord —
|
|
39
|
+
// and NEVER downgrade to a playback-only category, which would silently kill
|
|
40
|
+
// the mic while the UI still says "Listening".
|
|
41
|
+
AVAudioSession *session = [AVAudioSession sharedInstance];
|
|
42
|
+
if (![session.category isEqualToString:AVAudioSessionCategoryPlayAndRecord]) {
|
|
43
|
+
NSError *err = nil;
|
|
44
|
+
[session setCategory:AVAudioSessionCategoryPlayAndRecord
|
|
45
|
+
withOptions:(AVAudioSessionCategoryOptionDefaultToSpeaker |
|
|
46
|
+
AVAudioSessionCategoryOptionAllowBluetooth |
|
|
47
|
+
AVAudioSessionCategoryOptionMixWithOthers)
|
|
48
|
+
error:&err];
|
|
49
|
+
if (err) {
|
|
50
|
+
// Another library owns the session config — log and continue with
|
|
51
|
+
// whatever is active rather than fighting over it.
|
|
52
|
+
RCTLogWarn(@"[session-replay] could not set AVAudioSession category: %@", err);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
NSError *activeErr = nil;
|
|
56
|
+
[session setActive:YES error:&activeErr];
|
|
57
|
+
if (activeErr) {
|
|
58
|
+
RCTLogWarn(@"[session-replay] could not activate AVAudioSession: %@", activeErr);
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
_sampleRate = sampleRate > 0 ? sampleRate : 16000.0;
|
|
62
|
+
_engine = [[AVAudioEngine alloc] init];
|
|
63
|
+
_player = [[AVAudioPlayerNode alloc] init];
|
|
64
|
+
// Float32 non-interleaved mono — AVAudioPlayerNode's native scheduling format.
|
|
65
|
+
_format = [[AVAudioFormat alloc] initWithCommonFormat:AVAudioPCMFormatFloat32
|
|
66
|
+
sampleRate:_sampleRate
|
|
67
|
+
channels:1
|
|
68
|
+
interleaved:NO];
|
|
69
|
+
[_engine attachNode:_player];
|
|
70
|
+
[_engine connect:_player to:_engine.mainMixerNode format:_format];
|
|
71
|
+
|
|
72
|
+
NSError *startErr = nil;
|
|
73
|
+
if (![_engine startAndReturnError:&startErr]) {
|
|
74
|
+
RCTLogWarn(@"[session-replay] PCM player engine failed to start: %@", startErr);
|
|
75
|
+
[self tearDownEngine];
|
|
76
|
+
return;
|
|
77
|
+
}
|
|
78
|
+
[_player play];
|
|
79
|
+
_running = YES;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
- (void)tearDownEngine
|
|
83
|
+
{
|
|
84
|
+
if (_player) {
|
|
85
|
+
@try { [_player stop]; } @catch (__unused NSException *e) {}
|
|
86
|
+
}
|
|
87
|
+
if (_engine) {
|
|
88
|
+
@try { [_engine stop]; } @catch (__unused NSException *e) {}
|
|
89
|
+
}
|
|
90
|
+
_player = nil;
|
|
91
|
+
_engine = nil;
|
|
92
|
+
_format = nil;
|
|
93
|
+
_running = NO;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
RCT_EXPORT_METHOD(start:(double)sampleRate)
|
|
97
|
+
{
|
|
98
|
+
[self setUpEngineWithSampleRate:sampleRate];
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
RCT_EXPORT_METHOD(play:(NSString *)base64Pcm)
|
|
102
|
+
{
|
|
103
|
+
if (!base64Pcm.length) return;
|
|
104
|
+
if (!_running) {
|
|
105
|
+
// Host called play() before start() (or engine failed) — try a lazy start
|
|
106
|
+
// at the default assistant rate rather than dropping audio silently.
|
|
107
|
+
[self setUpEngineWithSampleRate:(_sampleRate > 0 ? _sampleRate : 16000.0)];
|
|
108
|
+
if (!_running) return;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
NSData *pcm = [[NSData alloc] initWithBase64EncodedString:base64Pcm
|
|
112
|
+
options:NSDataBase64DecodingIgnoreUnknownCharacters];
|
|
113
|
+
if (!pcm || pcm.length < 2) return;
|
|
114
|
+
|
|
115
|
+
AVAudioFrameCount frames = (AVAudioFrameCount)(pcm.length / 2);
|
|
116
|
+
AVAudioPCMBuffer *buffer = [[AVAudioPCMBuffer alloc] initWithPCMFormat:_format
|
|
117
|
+
frameCapacity:frames];
|
|
118
|
+
if (!buffer) return;
|
|
119
|
+
buffer.frameLength = frames;
|
|
120
|
+
|
|
121
|
+
const int16_t *src = (const int16_t *)pcm.bytes;
|
|
122
|
+
float *dst = buffer.floatChannelData[0];
|
|
123
|
+
for (AVAudioFrameCount i = 0; i < frames; i++) {
|
|
124
|
+
dst[i] = (float)src[i] / 32768.0f;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
// Engine can die on route changes (headphones unplugged) — restart once.
|
|
128
|
+
if (!_engine.isRunning) {
|
|
129
|
+
NSError *err = nil;
|
|
130
|
+
if (![_engine startAndReturnError:&err]) return;
|
|
131
|
+
[_player play];
|
|
132
|
+
}
|
|
133
|
+
[_player scheduleBuffer:buffer completionHandler:nil];
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
RCT_EXPORT_METHOD(clear)
|
|
137
|
+
{
|
|
138
|
+
// Barge-in: drop everything queued, keep the node ready for new chunks.
|
|
139
|
+
if (!_running || !_player) return;
|
|
140
|
+
@try {
|
|
141
|
+
[_player stop]; // discards all scheduled buffers
|
|
142
|
+
if (_engine.isRunning) [_player play];
|
|
143
|
+
} @catch (__unused NSException *e) {}
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
RCT_EXPORT_METHOD(stop)
|
|
147
|
+
{
|
|
148
|
+
[self tearDownEngine];
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
- (void)invalidate
|
|
152
|
+
{
|
|
153
|
+
[self tearDownEngine];
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
- (void)dealloc
|
|
157
|
+
{
|
|
158
|
+
[self tearDownEngine];
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
@end
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@akhilpulse/samadhaan-session-replay",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.6.0",
|
|
4
4
|
"description": "React Native SDK for Samadhaan Guided Support — session recording, live remote screen-share, take-control, guided-tour overlays, and Shake-to-Assist AI voice guide.",
|
|
5
5
|
"main": "src/index.ts",
|
|
6
6
|
"module": "src/index.ts",
|
|
@@ -10,6 +10,8 @@
|
|
|
10
10
|
"files": [
|
|
11
11
|
"src",
|
|
12
12
|
"android",
|
|
13
|
+
"ios",
|
|
14
|
+
"SamadhaanSessionReplay.podspec",
|
|
13
15
|
"react-native.config.js",
|
|
14
16
|
"README.md"
|
|
15
17
|
],
|
|
@@ -38,6 +38,8 @@ import { captureRef, captureScreen } from "react-native-view-shot";
|
|
|
38
38
|
import {
|
|
39
39
|
subscribeShake,
|
|
40
40
|
UltravoxVoiceSession,
|
|
41
|
+
createDefaultPcmPlayer,
|
|
42
|
+
type MicStatus,
|
|
41
43
|
type PcmAudioPlayer,
|
|
42
44
|
type VoiceState,
|
|
43
45
|
} from "./voice";
|
|
@@ -54,7 +56,7 @@ let pixelCopyDisabled = false; // turned true after repeated runtime failures to
|
|
|
54
56
|
let pixelCopyFailStreak = 0; // consecutive non-busy failures; resets on success
|
|
55
57
|
|
|
56
58
|
const STORAGE_KEY = "@akhilpulse/samadhaan-session-replay/sessionId";
|
|
57
|
-
const SDK_VERSION = "0.
|
|
59
|
+
const SDK_VERSION = "0.6.0";
|
|
58
60
|
|
|
59
61
|
// ---- Module-level session singleton --------------------------------------
|
|
60
62
|
// Multiple SessionReplayProvider mounts (intentional or via React StrictMode's
|
|
@@ -158,6 +160,7 @@ export function SessionReplayProvider({ config, children }: { config: Config; ch
|
|
|
158
160
|
|
|
159
161
|
// ---- Shake-to-Assist (AI voice guide) state ----
|
|
160
162
|
const [assistantPhase, setAssistantPhase] = useState<AssistantPhase>("idle");
|
|
163
|
+
const [assistMicStatus, setAssistMicStatus] = useState<MicStatus | null>(null);
|
|
161
164
|
const [assistantArrow, setAssistantArrow] = useState<
|
|
162
165
|
{ nx: number; ny: number; label?: string; instruction?: string } | null
|
|
163
166
|
>(null);
|
|
@@ -567,6 +570,7 @@ export function SessionReplayProvider({ config, children }: { config: Config; ch
|
|
|
567
570
|
try { voice?.stop(); } catch {}
|
|
568
571
|
setAssistantArrow(null);
|
|
569
572
|
setAssistantPhase("idle");
|
|
573
|
+
setAssistMicStatus(null);
|
|
570
574
|
if (asid && sessionId && ingestToken) {
|
|
571
575
|
fetch(`${serverUrl}/api/guided-support/assistant/end`, {
|
|
572
576
|
method: "POST",
|
|
@@ -610,15 +614,20 @@ export function SessionReplayProvider({ config, children }: { config: Config; ch
|
|
|
610
614
|
return;
|
|
611
615
|
}
|
|
612
616
|
assistantSessionIdRef.current = j.assistantSessionId;
|
|
617
|
+
setAssistMicStatus(null);
|
|
613
618
|
const voice = new UltravoxVoiceSession({
|
|
614
619
|
joinUrl: j.joinUrl,
|
|
615
620
|
sampleRate: 16000,
|
|
616
|
-
|
|
621
|
+
// Prefer the host-supplied player; otherwise fall back to the SDK's
|
|
622
|
+
// built-in Android AudioTrack player (v0.5.0+) so the agent's voice
|
|
623
|
+
// is audible out of the box.
|
|
624
|
+
audioPlayer: config.assistantAudioPlayer ?? createDefaultPcmPlayer() ?? undefined,
|
|
617
625
|
onState: (s) => {
|
|
618
626
|
// Natural socket close ("ended") tears the whole assistant down.
|
|
619
627
|
if (s === "ended") { endAssist(); return; }
|
|
620
628
|
setAssistantPhase(s);
|
|
621
629
|
},
|
|
630
|
+
onMicStatus: (status) => setAssistMicStatus(status),
|
|
622
631
|
onError: () => setAssistantPhase("error"),
|
|
623
632
|
});
|
|
624
633
|
voiceRef.current = voice;
|
|
@@ -903,7 +912,9 @@ export function SessionReplayProvider({ config, children }: { config: Config; ch
|
|
|
903
912
|
<View style={styles.assistDock} pointerEvents="box-none">
|
|
904
913
|
<View style={styles.assistDockCard}>
|
|
905
914
|
<VoiceWave phase={assistantPhase} />
|
|
906
|
-
<Text style={styles.assistStatusText}>
|
|
915
|
+
<Text style={styles.assistStatusText}>
|
|
916
|
+
{assistantStatusLabel(assistantPhase, assistMicStatus)}
|
|
917
|
+
</Text>
|
|
907
918
|
<Pressable
|
|
908
919
|
onPress={endAssist}
|
|
909
920
|
style={({ pressed }) => [styles.assistClose, pressed && styles.btnPressed]}
|
|
@@ -920,7 +931,15 @@ export function SessionReplayProvider({ config, children }: { config: Config; ch
|
|
|
920
931
|
}
|
|
921
932
|
|
|
922
933
|
// ---- Shake-to-Assist UI bits ----
|
|
923
|
-
function assistantStatusLabel(phase: AssistantPhase): string {
|
|
934
|
+
function assistantStatusLabel(phase: AssistantPhase, micStatus?: MicStatus | null): string {
|
|
935
|
+
// Be honest when we can't actually hear the user — a fake "Listening…"
|
|
936
|
+
// with a dead mic is the worst possible UX (bar stuck forever).
|
|
937
|
+
if (phase === "listening" && micStatus === "permission_denied") {
|
|
938
|
+
return "Mic blocked — allow microphone access in Settings";
|
|
939
|
+
}
|
|
940
|
+
if (phase === "listening" && micStatus === "unavailable") {
|
|
941
|
+
return "Mic unavailable — voice input is off";
|
|
942
|
+
}
|
|
924
943
|
switch (phase) {
|
|
925
944
|
case "connecting": return "Connecting…";
|
|
926
945
|
case "listening": return "Listening… ask me anything";
|
package/src/index.ts
CHANGED
package/src/voice.ts
CHANGED
|
@@ -21,6 +21,8 @@
|
|
|
21
21
|
* matching server/ultravox.ts (serverWebSocket inputSampleRate/outputSampleRate).
|
|
22
22
|
*/
|
|
23
23
|
|
|
24
|
+
import { NativeModules, PermissionsAndroid, Platform } from "react-native";
|
|
25
|
+
|
|
24
26
|
// Loads an optional native module without crashing Metro when it's absent.
|
|
25
27
|
function loadOptional<T = any>(name: string): T | null {
|
|
26
28
|
try {
|
|
@@ -31,6 +33,34 @@ function loadOptional<T = any>(name: string): T | null {
|
|
|
31
33
|
}
|
|
32
34
|
}
|
|
33
35
|
|
|
36
|
+
// ---- Mic permission --------------------------------------------------------
|
|
37
|
+
/**
|
|
38
|
+
* Android requires a runtime RECORD_AUDIO grant — react-native-live-audio-stream
|
|
39
|
+
* does NOT request it itself; without it the recorder silently captures nothing
|
|
40
|
+
* and the assistant can never hear the user. iOS shows its own prompt when the
|
|
41
|
+
* recorder starts (host app must have NSMicrophoneUsageDescription in Info.plist).
|
|
42
|
+
*/
|
|
43
|
+
async function ensureMicPermission(): Promise<boolean> {
|
|
44
|
+
if (Platform.OS !== "android") return true;
|
|
45
|
+
try {
|
|
46
|
+
const perm = PermissionsAndroid.PERMISSIONS.RECORD_AUDIO;
|
|
47
|
+
const already = await PermissionsAndroid.check(perm);
|
|
48
|
+
if (already) return true;
|
|
49
|
+
const res = await PermissionsAndroid.request(perm, {
|
|
50
|
+
title: "Microphone access",
|
|
51
|
+
message: "Needed so the voice assistant can hear you.",
|
|
52
|
+
buttonPositive: "Allow",
|
|
53
|
+
buttonNegative: "Not now",
|
|
54
|
+
});
|
|
55
|
+
return res === PermissionsAndroid.RESULTS.GRANTED;
|
|
56
|
+
} catch {
|
|
57
|
+
return false;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** Why voice input isn't working (surfaced so the UI can tell the user). */
|
|
62
|
+
export type MicStatus = "ok" | "permission_denied" | "unavailable";
|
|
63
|
+
|
|
34
64
|
// ---- Shake detection -----------------------------------------------------
|
|
35
65
|
/**
|
|
36
66
|
* Subscribe to phone-shake events. Returns an unsubscribe function.
|
|
@@ -106,6 +136,8 @@ export type StartVoiceOpts = {
|
|
|
106
136
|
/** Live transcript text (both user + agent), with role + final flag. */
|
|
107
137
|
onTranscript?: (t: { role: string; text: string; final: boolean }) => void;
|
|
108
138
|
onError?: (e: Error) => void;
|
|
139
|
+
/** Reports whether the microphone actually started (module + permission). */
|
|
140
|
+
onMicStatus?: (status: MicStatus) => void;
|
|
109
141
|
};
|
|
110
142
|
|
|
111
143
|
/**
|
|
@@ -140,7 +172,7 @@ export class UltravoxVoiceSession {
|
|
|
140
172
|
ws.onopen = () => {
|
|
141
173
|
if (this.stopped) return;
|
|
142
174
|
this.opts.audioPlayer?.start?.(this.sampleRate);
|
|
143
|
-
this.startMic();
|
|
175
|
+
void this.startMic(); // async: requests mic permission, reports onMicStatus
|
|
144
176
|
this.opts.onState?.("listening");
|
|
145
177
|
};
|
|
146
178
|
ws.onmessage = (ev: any) => {
|
|
@@ -184,9 +216,21 @@ export class UltravoxVoiceSession {
|
|
|
184
216
|
}
|
|
185
217
|
}
|
|
186
218
|
|
|
187
|
-
private startMic() {
|
|
219
|
+
private async startMic() {
|
|
188
220
|
const audioMod: any = loadOptional("react-native-live-audio-stream");
|
|
189
|
-
if (!audioMod)
|
|
221
|
+
if (!audioMod) {
|
|
222
|
+
// No mic module — assistant can still speak/guide, but tell the UI.
|
|
223
|
+
if (__DEV__) console.warn("[session-replay] react-native-live-audio-stream not installed — voice input disabled");
|
|
224
|
+
this.opts.onMicStatus?.("unavailable");
|
|
225
|
+
return;
|
|
226
|
+
}
|
|
227
|
+
const granted = await ensureMicPermission();
|
|
228
|
+
if (this.stopped) return;
|
|
229
|
+
if (!granted) {
|
|
230
|
+
if (__DEV__) console.warn("[session-replay] RECORD_AUDIO permission denied — voice input disabled");
|
|
231
|
+
this.opts.onMicStatus?.("permission_denied");
|
|
232
|
+
return;
|
|
233
|
+
}
|
|
190
234
|
const LiveAudioStream = audioMod.default || audioMod;
|
|
191
235
|
try {
|
|
192
236
|
LiveAudioStream.init({
|
|
@@ -209,9 +253,12 @@ export class UltravoxVoiceSession {
|
|
|
209
253
|
LiveAudioStream.start();
|
|
210
254
|
this.audioStream = LiveAudioStream;
|
|
211
255
|
this.micActive = true;
|
|
256
|
+
this.opts.onMicStatus?.("ok");
|
|
212
257
|
} catch (e) {
|
|
213
258
|
// Mic failed — keep the session alive for agent audio + arrows.
|
|
259
|
+
if (__DEV__) console.warn("[session-replay] mic capture failed to start:", e);
|
|
214
260
|
this.audioStream = null;
|
|
261
|
+
this.opts.onMicStatus?.("unavailable");
|
|
215
262
|
}
|
|
216
263
|
}
|
|
217
264
|
|
|
@@ -241,4 +288,23 @@ export class UltravoxVoiceSession {
|
|
|
241
288
|
}
|
|
242
289
|
}
|
|
243
290
|
|
|
291
|
+
// ---- Built-in agent-voice player (Android + iOS) ----------------------------
|
|
292
|
+
/**
|
|
293
|
+
* Returns a PcmAudioPlayer backed by the SDK's bundled native module
|
|
294
|
+
* (SessionReplayPcmPlayer): AudioTrack on Android (v0.5.0+), AVAudioEngine on
|
|
295
|
+
* iOS (v0.6.0+). Returns null when the native module isn't available (Expo Go,
|
|
296
|
+
* or host app not rebuilt / `pod install` not run after upgrading).
|
|
297
|
+
* Used automatically when the integrator doesn't pass `assistantAudioPlayer`.
|
|
298
|
+
*/
|
|
299
|
+
export function createDefaultPcmPlayer(): PcmAudioPlayer | null {
|
|
300
|
+
const native: any = (NativeModules as any).SessionReplayPcmPlayer;
|
|
301
|
+
if (!native || typeof native.play !== "function") return null;
|
|
302
|
+
return {
|
|
303
|
+
start: (sampleRate: number) => { try { native.start(sampleRate); } catch {} },
|
|
304
|
+
play: (pcm: ArrayBuffer) => { try { native.play(arrayBufferToBase64(pcm)); } catch {} },
|
|
305
|
+
clear: () => { try { native.clear(); } catch {} },
|
|
306
|
+
stop: () => { try { native.stop(); } catch {} },
|
|
307
|
+
};
|
|
308
|
+
}
|
|
309
|
+
|
|
244
310
|
export { arrayBufferToBase64 };
|