@bytexbyte/nxtlinq-ai-agent-react-native-development 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/context/NxtlinqAgentContext.d.ts +13 -0
- package/dist/context/NxtlinqAgentContext.d.ts.map +1 -0
- package/dist/context/NxtlinqAgentContext.js +42 -0
- package/dist/createNxtlinqAgent.d.ts +9 -0
- package/dist/createNxtlinqAgent.d.ts.map +1 -0
- package/dist/createNxtlinqAgent.js +16 -0
- package/dist/hooks/useNxtlinqAgent.d.ts +20 -0
- package/dist/hooks/useNxtlinqAgent.d.ts.map +1 -0
- package/dist/hooks/useNxtlinqAgent.js +25 -0
- package/dist/hooks/useNxtlinqVoice.d.ts +23 -0
- package/dist/hooks/useNxtlinqVoice.d.ts.map +1 -0
- package/dist/hooks/useNxtlinqVoice.js +79 -0
- package/dist/index.d.ts +14 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +12 -0
- package/dist/ports/createAsyncStoragePort.d.ts +9 -0
- package/dist/ports/createAsyncStoragePort.d.ts.map +1 -0
- package/dist/ports/createAsyncStoragePort.js +7 -0
- package/dist/ports/createRNHttpPort.d.ts +4 -0
- package/dist/ports/createRNHttpPort.d.ts.map +1 -0
- package/dist/ports/createRNHttpPort.js +9 -0
- package/dist/ports/createRNPlatformPorts.d.ts +14 -0
- package/dist/ports/createRNPlatformPorts.d.ts.map +1 -0
- package/dist/ports/createRNPlatformPorts.js +21 -0
- package/dist/ports/createRNWebRTCPort.d.ts +81 -0
- package/dist/ports/createRNWebRTCPort.d.ts.map +1 -0
- package/dist/ports/createRNWebRTCPort.js +235 -0
- package/dist/utils/ensureRecordAudioPermission.d.ts +7 -0
- package/dist/utils/ensureRecordAudioPermission.d.ts.map +1 -0
- package/dist/utils/ensureRecordAudioPermission.js +24 -0
- package/dist/utils/fetchUriAsDataUri.d.ts +11 -0
- package/dist/utils/fetchUriAsDataUri.d.ts.map +1 -0
- package/dist/utils/fetchUriAsDataUri.js +143 -0
- package/dist/utils/iosAudioSessionReady.d.ts +27 -0
- package/dist/utils/iosAudioSessionReady.d.ts.map +1 -0
- package/dist/utils/iosAudioSessionReady.js +93 -0
- package/dist/utils/iosWebRTCAudioSession.d.ts +5 -0
- package/dist/utils/iosWebRTCAudioSession.d.ts.map +1 -0
- package/dist/utils/iosWebRTCAudioSession.js +35 -0
- package/package.json +56 -0
- package/src/context/NxtlinqAgentContext.tsx +87 -0
- package/src/createNxtlinqAgent.ts +32 -0
- package/src/hooks/useNxtlinqAgent.ts +76 -0
- package/src/hooks/useNxtlinqVoice.ts +148 -0
- package/src/index.ts +74 -0
- package/src/ports/createAsyncStoragePort.ts +16 -0
- package/src/ports/createRNHttpPort.ts +11 -0
- package/src/ports/createRNPlatformPorts.ts +37 -0
- package/src/ports/createRNWebRTCPort.ts +327 -0
- package/src/utils/ensureRecordAudioPermission.ts +28 -0
- package/src/utils/fetchUriAsDataUri.ts +166 -0
- package/src/utils/iosAudioSessionReady.ts +106 -0
- package/src/utils/iosWebRTCAudioSession.ts +39 -0
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
import { VoiceNotSupportedError } from '@bytexbyte/nxtlinq-ai-agent-core-development';
|
|
2
|
+
import { ensureRecordAudioPermission } from '../utils/ensureRecordAudioPermission';
|
|
3
|
+
import { ensureIOSAudioSessionForVoiceCapture, getIOSVoiceCaptureGeneration, markIOSVoiceCaptureCompleted, } from '../utils/iosAudioSessionReady';
|
|
4
|
+
import { activateIOSWebRTCAudioSession, deactivateIOSWebRTCAudioSession, } from '../utils/iosWebRTCAudioSession';
|
|
5
|
+
const ICE_GATHERING_TIMEOUT_MS = 5000;
|
|
6
|
+
function bindDataChannelEvent(channel, type, handler) {
|
|
7
|
+
if (!handler)
|
|
8
|
+
return;
|
|
9
|
+
if (channel.addEventListener) {
|
|
10
|
+
if (type === 'message') {
|
|
11
|
+
channel.addEventListener('message', (event) => {
|
|
12
|
+
handler({
|
|
13
|
+
data: String(event?.data ?? ''),
|
|
14
|
+
});
|
|
15
|
+
});
|
|
16
|
+
}
|
|
17
|
+
else {
|
|
18
|
+
channel.addEventListener(type, () => {
|
|
19
|
+
handler();
|
|
20
|
+
});
|
|
21
|
+
}
|
|
22
|
+
return;
|
|
23
|
+
}
|
|
24
|
+
if (type === 'open') {
|
|
25
|
+
channel.onopen = handler;
|
|
26
|
+
}
|
|
27
|
+
else if (type === 'message') {
|
|
28
|
+
channel.onmessage = handler;
|
|
29
|
+
}
|
|
30
|
+
else {
|
|
31
|
+
channel.onclose = handler;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
function wrapDataChannel(channel) {
|
|
35
|
+
let onopenHandler = null;
|
|
36
|
+
let onmessageHandler = null;
|
|
37
|
+
let oncloseHandler = null;
|
|
38
|
+
return {
|
|
39
|
+
get readyState() {
|
|
40
|
+
return channel.readyState;
|
|
41
|
+
},
|
|
42
|
+
get onopen() {
|
|
43
|
+
return onopenHandler;
|
|
44
|
+
},
|
|
45
|
+
set onopen(handler) {
|
|
46
|
+
onopenHandler = handler;
|
|
47
|
+
if (handler) {
|
|
48
|
+
bindDataChannelEvent(channel, 'open', handler);
|
|
49
|
+
if (channel.readyState === 'open') {
|
|
50
|
+
queueMicrotask(() => handler());
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
},
|
|
54
|
+
get onmessage() {
|
|
55
|
+
return onmessageHandler;
|
|
56
|
+
},
|
|
57
|
+
set onmessage(handler) {
|
|
58
|
+
onmessageHandler = handler;
|
|
59
|
+
if (handler)
|
|
60
|
+
bindDataChannelEvent(channel, 'message', handler);
|
|
61
|
+
},
|
|
62
|
+
get onclose() {
|
|
63
|
+
return oncloseHandler;
|
|
64
|
+
},
|
|
65
|
+
set onclose(handler) {
|
|
66
|
+
oncloseHandler = handler;
|
|
67
|
+
if (handler)
|
|
68
|
+
bindDataChannelEvent(channel, 'close', handler);
|
|
69
|
+
},
|
|
70
|
+
send: (data) => channel.send(data),
|
|
71
|
+
close: () => channel.close(),
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
function wrapPeerConnection(pc) {
|
|
75
|
+
let connectionHandler = null;
|
|
76
|
+
let iceConnectionHandler = null;
|
|
77
|
+
let connectionBound = null;
|
|
78
|
+
let iceConnectionBound = null;
|
|
79
|
+
const wrapped = {
|
|
80
|
+
__nativePc: pc,
|
|
81
|
+
get connectionState() {
|
|
82
|
+
return pc.connectionState;
|
|
83
|
+
},
|
|
84
|
+
get iceConnectionState() {
|
|
85
|
+
return pc.iceConnectionState;
|
|
86
|
+
},
|
|
87
|
+
onconnectionstatechange: null,
|
|
88
|
+
oniceconnectionstatechange: null,
|
|
89
|
+
ontrack: null,
|
|
90
|
+
ondatachannel: null,
|
|
91
|
+
createDataChannel: (label) => wrapDataChannel(pc.createDataChannel(label)),
|
|
92
|
+
addTrack: (track, stream) => pc.addTrack(track, stream),
|
|
93
|
+
createOffer: () => pc.createOffer(),
|
|
94
|
+
setLocalDescription: (desc) => pc.setLocalDescription(desc),
|
|
95
|
+
setRemoteDescription: (desc) => pc.setRemoteDescription(desc),
|
|
96
|
+
getLocalDescription: () => {
|
|
97
|
+
const local = pc.localDescription;
|
|
98
|
+
return local ? { sdp: local.sdp, type: local.type } : null;
|
|
99
|
+
},
|
|
100
|
+
close: () => pc.close(),
|
|
101
|
+
};
|
|
102
|
+
Object.defineProperty(wrapped, 'onconnectionstatechange', {
|
|
103
|
+
get: () => connectionHandler,
|
|
104
|
+
set(handler) {
|
|
105
|
+
connectionHandler = handler;
|
|
106
|
+
if (connectionBound) {
|
|
107
|
+
pc.removeEventListener('connectionstatechange', connectionBound);
|
|
108
|
+
connectionBound = null;
|
|
109
|
+
}
|
|
110
|
+
if (handler) {
|
|
111
|
+
connectionBound = () => handler();
|
|
112
|
+
pc.addEventListener('connectionstatechange', connectionBound);
|
|
113
|
+
}
|
|
114
|
+
},
|
|
115
|
+
configurable: true,
|
|
116
|
+
});
|
|
117
|
+
Object.defineProperty(wrapped, 'oniceconnectionstatechange', {
|
|
118
|
+
get: () => iceConnectionHandler,
|
|
119
|
+
set(handler) {
|
|
120
|
+
iceConnectionHandler = handler;
|
|
121
|
+
if (iceConnectionBound) {
|
|
122
|
+
pc.removeEventListener('iceconnectionstatechange', iceConnectionBound);
|
|
123
|
+
iceConnectionBound = null;
|
|
124
|
+
}
|
|
125
|
+
if (handler) {
|
|
126
|
+
iceConnectionBound = () => handler();
|
|
127
|
+
pc.addEventListener('iceconnectionstatechange', iceConnectionBound);
|
|
128
|
+
}
|
|
129
|
+
},
|
|
130
|
+
configurable: true,
|
|
131
|
+
});
|
|
132
|
+
let trackHandler = null;
|
|
133
|
+
const trackListener = (event) => {
|
|
134
|
+
trackHandler?.({
|
|
135
|
+
streams: (event.streams ?? []),
|
|
136
|
+
track: event.track ?? { kind: 'audio' },
|
|
137
|
+
});
|
|
138
|
+
};
|
|
139
|
+
pc.addEventListener('track', trackListener);
|
|
140
|
+
Object.defineProperty(wrapped, 'ontrack', {
|
|
141
|
+
get: () => trackHandler,
|
|
142
|
+
set(handler) {
|
|
143
|
+
trackHandler = handler;
|
|
144
|
+
},
|
|
145
|
+
configurable: true,
|
|
146
|
+
});
|
|
147
|
+
let datachannelHandler = null;
|
|
148
|
+
const datachannelListener = (event) => {
|
|
149
|
+
if (!event.channel)
|
|
150
|
+
return;
|
|
151
|
+
datachannelHandler?.({ channel: wrapDataChannel(event.channel) });
|
|
152
|
+
};
|
|
153
|
+
pc.addEventListener('datachannel', datachannelListener);
|
|
154
|
+
Object.defineProperty(wrapped, 'ondatachannel', {
|
|
155
|
+
get: () => datachannelHandler,
|
|
156
|
+
set(handler) {
|
|
157
|
+
datachannelHandler = handler;
|
|
158
|
+
},
|
|
159
|
+
configurable: true,
|
|
160
|
+
});
|
|
161
|
+
return wrapped;
|
|
162
|
+
}
|
|
163
|
+
/**
|
|
164
|
+
* React Native WebRTC adapter — pass exports from `react-native-webrtc`.
|
|
165
|
+
*
|
|
166
|
+
* @example
|
|
167
|
+
* import { RTCPeerConnection, mediaDevices } from 'react-native-webrtc';
|
|
168
|
+
* const webrtc = createRNWebRTCPort({ RTCPeerConnection, mediaDevices });
|
|
169
|
+
*/
|
|
170
|
+
export function createRNWebRTCPort(module) {
|
|
171
|
+
if (!module?.RTCPeerConnection || !module?.mediaDevices?.getUserMedia) {
|
|
172
|
+
throw new VoiceNotSupportedError('react-native-webrtc is required — install and link the native module.');
|
|
173
|
+
}
|
|
174
|
+
return {
|
|
175
|
+
isSupported: () => Boolean(module.RTCPeerConnection && module.mediaDevices?.getUserMedia),
|
|
176
|
+
getUserMedia: async (constraints) => {
|
|
177
|
+
if (!constraints.audio) {
|
|
178
|
+
return (await module.mediaDevices.getUserMedia(constraints));
|
|
179
|
+
}
|
|
180
|
+
const generation = getIOSVoiceCaptureGeneration();
|
|
181
|
+
const maxAttempts = 3;
|
|
182
|
+
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
|
183
|
+
await ensureIOSAudioSessionForVoiceCapture();
|
|
184
|
+
await ensureRecordAudioPermission();
|
|
185
|
+
activateIOSWebRTCAudioSession();
|
|
186
|
+
const stream = (await module.mediaDevices.getUserMedia(constraints));
|
|
187
|
+
const tracks = stream.getAudioTracks();
|
|
188
|
+
const hasLiveTrack = tracks.some((track) => track.readyState === 'live');
|
|
189
|
+
if (hasLiveTrack || tracks.length === 0) {
|
|
190
|
+
markIOSVoiceCaptureCompleted();
|
|
191
|
+
return stream;
|
|
192
|
+
}
|
|
193
|
+
console.warn(`[nxtlinq] iOS mic track not live (gen=${generation}, attempt=${attempt}/${maxAttempts}), retrying…`);
|
|
194
|
+
for (const track of tracks) {
|
|
195
|
+
try {
|
|
196
|
+
track.stop();
|
|
197
|
+
}
|
|
198
|
+
catch {
|
|
199
|
+
/* noop */
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
deactivateIOSWebRTCAudioSession();
|
|
203
|
+
await new Promise((resolve) => {
|
|
204
|
+
setTimeout(resolve, 350 * attempt);
|
|
205
|
+
});
|
|
206
|
+
}
|
|
207
|
+
throw new Error('Failed to acquire live iOS microphone track after retries');
|
|
208
|
+
},
|
|
209
|
+
createPeerConnection: (config) => wrapPeerConnection(new module.RTCPeerConnection(config)),
|
|
210
|
+
waitForIceGathering: async (pc, timeoutMs = ICE_GATHERING_TIMEOUT_MS) => {
|
|
211
|
+
const native = pc.__nativePc;
|
|
212
|
+
if (native.iceGatheringState === 'complete')
|
|
213
|
+
return;
|
|
214
|
+
await new Promise((resolve) => {
|
|
215
|
+
let done = false;
|
|
216
|
+
const finish = () => {
|
|
217
|
+
if (done)
|
|
218
|
+
return;
|
|
219
|
+
done = true;
|
|
220
|
+
native.removeEventListener('icegatheringstatechange', check);
|
|
221
|
+
clearTimeout(timer);
|
|
222
|
+
resolve();
|
|
223
|
+
};
|
|
224
|
+
const check = () => {
|
|
225
|
+
if (native.iceGatheringState === 'complete')
|
|
226
|
+
finish();
|
|
227
|
+
};
|
|
228
|
+
native.addEventListener('icegatheringstatechange', check);
|
|
229
|
+
if (native.iceGatheringState === 'complete')
|
|
230
|
+
finish();
|
|
231
|
+
const timer = setTimeout(finish, timeoutMs);
|
|
232
|
+
});
|
|
233
|
+
},
|
|
234
|
+
};
|
|
235
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
export declare class MicrophonePermissionDeniedError extends Error {
|
|
2
|
+
readonly name = "MicrophonePermissionDeniedError";
|
|
3
|
+
constructor(message?: string);
|
|
4
|
+
}
|
|
5
|
+
/** Android requires runtime RECORD_AUDIO before `getUserMedia({ audio: true })`. */
|
|
6
|
+
export declare function ensureRecordAudioPermission(): Promise<void>;
|
|
7
|
+
//# sourceMappingURL=ensureRecordAudioPermission.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ensureRecordAudioPermission.d.ts","sourceRoot":"","sources":["../../src/utils/ensureRecordAudioPermission.ts"],"names":[],"mappings":"AAEA,qBAAa,+BAAgC,SAAQ,KAAK;IACxD,QAAQ,CAAC,IAAI,qCAAqC;gBAEtC,OAAO,SAAiC;CAGrD;AAED,oFAAoF;AACpF,wBAAsB,2BAA2B,IAAI,OAAO,CAAC,IAAI,CAAC,CAgBjE"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { Platform, PermissionsAndroid } from 'react-native';
|
|
2
|
+
export class MicrophonePermissionDeniedError extends Error {
|
|
3
|
+
constructor(message = 'Microphone permission denied') {
|
|
4
|
+
super(message);
|
|
5
|
+
this.name = 'MicrophonePermissionDeniedError';
|
|
6
|
+
}
|
|
7
|
+
}
|
|
8
|
+
/** Android requires runtime RECORD_AUDIO before `getUserMedia({ audio: true })`. */
|
|
9
|
+
export async function ensureRecordAudioPermission() {
|
|
10
|
+
if (Platform.OS !== 'android')
|
|
11
|
+
return;
|
|
12
|
+
const permission = PermissionsAndroid.PERMISSIONS.RECORD_AUDIO;
|
|
13
|
+
if (await PermissionsAndroid.check(permission))
|
|
14
|
+
return;
|
|
15
|
+
const result = await PermissionsAndroid.request(permission, {
|
|
16
|
+
title: 'Microphone',
|
|
17
|
+
message: 'Allow microphone access for voice chat.',
|
|
18
|
+
buttonPositive: 'OK',
|
|
19
|
+
buttonNegative: 'Cancel',
|
|
20
|
+
});
|
|
21
|
+
if (result !== PermissionsAndroid.RESULTS.GRANTED) {
|
|
22
|
+
throw new MicrophonePermissionDeniedError();
|
|
23
|
+
}
|
|
24
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { Attachment } from '@bytexbyte/nxtlinq-ai-agent-core-development';
|
|
2
|
+
/**
|
|
3
|
+
* Prefer HTTPS URL as-is (avoids huge base64 over SCTP). Local `file://` URIs are
|
|
4
|
+
* still read and encoded as data URIs.
|
|
5
|
+
*/
|
|
6
|
+
export declare function uriToVoiceImageAttachment(uri: string, name?: string): Promise<Attachment>;
|
|
7
|
+
/**
|
|
8
|
+
* Build an image {@link Attachment} from a remote URL or local file URI (RN / browser).
|
|
9
|
+
*/
|
|
10
|
+
export declare function fetchUriAsDataUriAttachment(uri: string, name?: string): Promise<Attachment>;
|
|
11
|
+
//# sourceMappingURL=fetchUriAsDataUri.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fetchUriAsDataUri.d.ts","sourceRoot":"","sources":["../../src/utils/fetchUriAsDataUri.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,8CAA8C,CAAC;AAyG/E;;;GAGG;AACH,wBAAsB,yBAAyB,CAC7C,GAAG,EAAE,MAAM,EACX,IAAI,SAAc,GACjB,OAAO,CAAC,UAAU,CAAC,CAsBrB;AAED;;GAEG;AACH,wBAAsB,2BAA2B,CAC/C,GAAG,EAAE,MAAM,EACX,IAAI,SAAc,GACjB,OAAO,CAAC,UAAU,CAAC,CAuBrB"}
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
function guessMime(uri) {
|
|
2
|
+
const lower = uri.split('?')[0]?.toLowerCase() ?? '';
|
|
3
|
+
if (lower.endsWith('.png'))
|
|
4
|
+
return 'image/png';
|
|
5
|
+
if (lower.endsWith('.webp'))
|
|
6
|
+
return 'image/webp';
|
|
7
|
+
if (lower.endsWith('.gif'))
|
|
8
|
+
return 'image/gif';
|
|
9
|
+
return 'image/jpeg';
|
|
10
|
+
}
|
|
11
|
+
function normalizeMime(header, fallback) {
|
|
12
|
+
const raw = header?.split(';')[0]?.trim().toLowerCase();
|
|
13
|
+
if (!raw || !raw.startsWith('image/'))
|
|
14
|
+
return fallback;
|
|
15
|
+
return raw === 'image/jpg' ? 'image/jpeg' : raw;
|
|
16
|
+
}
|
|
17
|
+
function bytesToBase64(bytes) {
|
|
18
|
+
if (typeof globalThis.Buffer !== 'undefined') {
|
|
19
|
+
return globalThis.Buffer.from(bytes).toString('base64');
|
|
20
|
+
}
|
|
21
|
+
let binary = '';
|
|
22
|
+
const chunk = 0x8000;
|
|
23
|
+
for (let i = 0; i < bytes.length; i += chunk) {
|
|
24
|
+
binary += String.fromCharCode(...bytes.subarray(i, i + chunk));
|
|
25
|
+
}
|
|
26
|
+
if (typeof globalThis.btoa !== 'function') {
|
|
27
|
+
throw new Error('No base64 encoder available in this runtime');
|
|
28
|
+
}
|
|
29
|
+
return globalThis.btoa(binary);
|
|
30
|
+
}
|
|
31
|
+
/** Read binary from fetch Response (RN Blob often lacks arrayBuffer()). */
|
|
32
|
+
async function readResponseBytes(res, fallbackMime) {
|
|
33
|
+
const headerMime = normalizeMime(res.headers.get('content-type'), fallbackMime);
|
|
34
|
+
if (typeof res.arrayBuffer === 'function') {
|
|
35
|
+
const buffer = await res.arrayBuffer();
|
|
36
|
+
return { bytes: new Uint8Array(buffer), mimeType: headerMime };
|
|
37
|
+
}
|
|
38
|
+
const blob = await res.blob();
|
|
39
|
+
if (typeof blob.arrayBuffer === 'function') {
|
|
40
|
+
const buffer = await blob.arrayBuffer();
|
|
41
|
+
return {
|
|
42
|
+
bytes: new Uint8Array(buffer),
|
|
43
|
+
mimeType: normalizeMime(blob.type, headerMime),
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
throw new Error('fetch response cannot be read as binary in this runtime');
|
|
47
|
+
}
|
|
48
|
+
/** XHR fallback for RN runtimes where fetch().arrayBuffer() is unavailable. */
|
|
49
|
+
function xhrGetBytes(uri) {
|
|
50
|
+
return new Promise((resolve, reject) => {
|
|
51
|
+
const xhr = new XMLHttpRequest();
|
|
52
|
+
xhr.onload = () => {
|
|
53
|
+
if (xhr.status < 200 || xhr.status >= 300) {
|
|
54
|
+
reject(new Error(`Failed to fetch image: ${xhr.status}`));
|
|
55
|
+
return;
|
|
56
|
+
}
|
|
57
|
+
const body = xhr.response;
|
|
58
|
+
if (!(body instanceof ArrayBuffer)) {
|
|
59
|
+
reject(new Error('XHR did not return ArrayBuffer'));
|
|
60
|
+
return;
|
|
61
|
+
}
|
|
62
|
+
resolve({
|
|
63
|
+
bytes: new Uint8Array(body),
|
|
64
|
+
mimeType: normalizeMime(xhr.getResponseHeader('Content-Type'), guessMime(uri)),
|
|
65
|
+
});
|
|
66
|
+
};
|
|
67
|
+
xhr.onerror = () => reject(new Error('Failed to fetch image'));
|
|
68
|
+
xhr.responseType = 'arraybuffer';
|
|
69
|
+
xhr.open('GET', uri);
|
|
70
|
+
xhr.send();
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
async function fetchUriBytes(uri) {
|
|
74
|
+
const fallbackMime = guessMime(uri);
|
|
75
|
+
try {
|
|
76
|
+
const res = await fetch(uri);
|
|
77
|
+
if (!res.ok) {
|
|
78
|
+
throw new Error(`Failed to fetch image: ${res.status}`);
|
|
79
|
+
}
|
|
80
|
+
return await readResponseBytes(res, fallbackMime);
|
|
81
|
+
}
|
|
82
|
+
catch (err) {
|
|
83
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
84
|
+
if (message.includes('arrayBuffer') ||
|
|
85
|
+
message.includes('binary in this runtime')) {
|
|
86
|
+
return xhrGetBytes(uri);
|
|
87
|
+
}
|
|
88
|
+
throw err;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* Prefer HTTPS URL as-is (avoids huge base64 over SCTP). Local `file://` URIs are
|
|
93
|
+
* still read and encoded as data URIs.
|
|
94
|
+
*/
|
|
95
|
+
export async function uriToVoiceImageAttachment(uri, name = 'image.jpg') {
|
|
96
|
+
const trimmed = uri.trim();
|
|
97
|
+
if (!trimmed) {
|
|
98
|
+
throw new Error('uriToVoiceImageAttachment: uri is empty');
|
|
99
|
+
}
|
|
100
|
+
if (trimmed.startsWith('data:')) {
|
|
101
|
+
return {
|
|
102
|
+
type: 'image',
|
|
103
|
+
url: trimmed,
|
|
104
|
+
name,
|
|
105
|
+
mimeType: trimmed.slice(5, trimmed.indexOf(';')) || guessMime(name),
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
if (/^https?:\/\//i.test(trimmed)) {
|
|
109
|
+
return {
|
|
110
|
+
type: 'image',
|
|
111
|
+
url: trimmed,
|
|
112
|
+
name,
|
|
113
|
+
mimeType: guessMime(trimmed),
|
|
114
|
+
};
|
|
115
|
+
}
|
|
116
|
+
return fetchUriAsDataUriAttachment(trimmed, name);
|
|
117
|
+
}
|
|
118
|
+
/**
|
|
119
|
+
* Build an image {@link Attachment} from a remote URL or local file URI (RN / browser).
|
|
120
|
+
*/
|
|
121
|
+
export async function fetchUriAsDataUriAttachment(uri, name = 'image.jpg') {
|
|
122
|
+
const trimmed = uri.trim();
|
|
123
|
+
if (!trimmed) {
|
|
124
|
+
throw new Error('fetchUriAsDataUriAttachment: uri is empty');
|
|
125
|
+
}
|
|
126
|
+
if (trimmed.startsWith('data:')) {
|
|
127
|
+
return {
|
|
128
|
+
type: 'image',
|
|
129
|
+
url: trimmed,
|
|
130
|
+
name,
|
|
131
|
+
mimeType: trimmed.slice(5, trimmed.indexOf(';')) || guessMime(name),
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
const { bytes, mimeType } = await fetchUriBytes(trimmed);
|
|
135
|
+
const base64 = bytesToBase64(bytes);
|
|
136
|
+
return {
|
|
137
|
+
type: 'image',
|
|
138
|
+
url: `data:${mimeType};base64,${base64}`,
|
|
139
|
+
name,
|
|
140
|
+
mimeType,
|
|
141
|
+
size: bytes.length,
|
|
142
|
+
};
|
|
143
|
+
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
/** Called by {@link AudioSessionWaker} when silent.mp3 starts (react-native-video). */
|
|
2
|
+
export declare function markIOSAudioSessionReady(): void;
|
|
3
|
+
/**
|
|
4
|
+
* Register a handler that replays silent audio (mounted {@link AudioSessionWaker}).
|
|
5
|
+
* Must NOT unmount the video view — react-native-video deactivates AVAudioSession on unregister.
|
|
6
|
+
*/
|
|
7
|
+
export declare function setIOSAudioSessionPrewarmHandler(handler: (() => void) | null): void;
|
|
8
|
+
/** Call after `getUserMedia({ audio: true })` succeeds so reconnect path is used. */
|
|
9
|
+
export declare function markIOSVoiceCaptureCompleted(): void;
|
|
10
|
+
/** Nudge silent playback after WebRTC releases the mic (text mode / session stop). */
|
|
11
|
+
export declare function notifyIOSVoiceSessionEnded(): void;
|
|
12
|
+
/**
|
|
13
|
+
* On iOS, WebRTC mic + speaker need AVAudioSession active (via react-native-video).
|
|
14
|
+
* Waits until {@link markIOSAudioSessionReady} or timeout.
|
|
15
|
+
*/
|
|
16
|
+
export declare function waitForIOSAudioSessionReady(timeoutMs?: number): Promise<void>;
|
|
17
|
+
/**
|
|
18
|
+
* Before voice capture on iOS.
|
|
19
|
+
* - Cold start: wait for waker, then sync WebRTC audio session.
|
|
20
|
+
* - Reconnect: deactivate → nudge waker → activate WebRTC (required after ~4+ cycles on iOS).
|
|
21
|
+
*/
|
|
22
|
+
export declare function ensureIOSAudioSessionForVoiceCapture(timeoutMs?: number): Promise<void>;
|
|
23
|
+
/** Allow WebRTC / AVAudioSession to release the mic before the next getUserMedia. */
|
|
24
|
+
export declare function waitForIOSVoiceCaptureRelease(): Promise<void>;
|
|
25
|
+
/** Current capture generation — useful for logging flaky reconnect attempts. */
|
|
26
|
+
export declare function getIOSVoiceCaptureGeneration(): number;
|
|
27
|
+
//# sourceMappingURL=iosAudioSessionReady.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"iosAudioSessionReady.d.ts","sourceRoot":"","sources":["../../src/utils/iosAudioSessionReady.ts"],"names":[],"mappings":"AAeA,uFAAuF;AACvF,wBAAgB,wBAAwB,IAAI,IAAI,CAK/C;AAED;;;GAGG;AACH,wBAAgB,gCAAgC,CAAC,OAAO,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,IAAI,GAAG,IAAI,CAEnF;AAED,qFAAqF;AACrF,wBAAgB,4BAA4B,IAAI,IAAI,CAGnD;AAED,sFAAsF;AACtF,wBAAgB,0BAA0B,IAAI,IAAI,CAGjD;AAED;;;GAGG;AACH,wBAAsB,2BAA2B,CAAC,SAAS,SAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAiBjF;AAYD;;;;GAIG;AACH,wBAAsB,oCAAoC,CACxD,SAAS,SAAO,GACf,OAAO,CAAC,IAAI,CAAC,CASf;AAED,qFAAqF;AACrF,wBAAsB,6BAA6B,IAAI,OAAO,CAAC,IAAI,CAAC,CAKnE;AAED,gFAAgF;AAChF,wBAAgB,4BAA4B,IAAI,MAAM,CAErD"}
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import { Platform } from 'react-native';
|
|
2
|
+
import { activateIOSWebRTCAudioSession, deactivateIOSWebRTCAudioSession, } from './iosWebRTCAudioSession';
|
|
3
|
+
let sessionReady = Platform.OS !== 'ios';
|
|
4
|
+
const pending = [];
|
|
5
|
+
let prewarmHandler = null;
|
|
6
|
+
let hasCompletedVoiceCapture = false;
|
|
7
|
+
let voiceCaptureGeneration = 0;
|
|
8
|
+
const RECONNECT_SETTLE_MS = 550;
|
|
9
|
+
const POST_STOP_SETTLE_MS = 550;
|
|
10
|
+
/** Called by {@link AudioSessionWaker} when silent.mp3 starts (react-native-video). */
|
|
11
|
+
export function markIOSAudioSessionReady() {
|
|
12
|
+
if (sessionReady)
|
|
13
|
+
return;
|
|
14
|
+
sessionReady = true;
|
|
15
|
+
for (const resolve of pending)
|
|
16
|
+
resolve();
|
|
17
|
+
pending.length = 0;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Register a handler that replays silent audio (mounted {@link AudioSessionWaker}).
|
|
21
|
+
* Must NOT unmount the video view — react-native-video deactivates AVAudioSession on unregister.
|
|
22
|
+
*/
|
|
23
|
+
export function setIOSAudioSessionPrewarmHandler(handler) {
|
|
24
|
+
prewarmHandler = handler;
|
|
25
|
+
}
|
|
26
|
+
/** Call after `getUserMedia({ audio: true })` succeeds so reconnect path is used. */
|
|
27
|
+
export function markIOSVoiceCaptureCompleted() {
|
|
28
|
+
hasCompletedVoiceCapture = true;
|
|
29
|
+
voiceCaptureGeneration += 1;
|
|
30
|
+
}
|
|
31
|
+
/** Nudge silent playback after WebRTC releases the mic (text mode / session stop). */
|
|
32
|
+
export function notifyIOSVoiceSessionEnded() {
|
|
33
|
+
if (Platform.OS !== 'ios')
|
|
34
|
+
return;
|
|
35
|
+
prewarmHandler?.();
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* On iOS, WebRTC mic + speaker need AVAudioSession active (via react-native-video).
|
|
39
|
+
* Waits until {@link markIOSAudioSessionReady} or timeout.
|
|
40
|
+
*/
|
|
41
|
+
export async function waitForIOSAudioSessionReady(timeoutMs = 5000) {
|
|
42
|
+
if (sessionReady)
|
|
43
|
+
return;
|
|
44
|
+
await Promise.race([
|
|
45
|
+
new Promise((resolve) => {
|
|
46
|
+
pending.push(resolve);
|
|
47
|
+
}),
|
|
48
|
+
new Promise((resolve) => {
|
|
49
|
+
setTimeout(() => {
|
|
50
|
+
if (!sessionReady) {
|
|
51
|
+
console.warn('[nxtlinq] iOS AVAudioSession was not activated in time; pass iosSilentAudioSource and react-native-video');
|
|
52
|
+
}
|
|
53
|
+
resolve();
|
|
54
|
+
}, timeoutMs);
|
|
55
|
+
}),
|
|
56
|
+
]);
|
|
57
|
+
}
|
|
58
|
+
function sleep(ms) {
|
|
59
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
60
|
+
}
|
|
61
|
+
async function nudgeAndSettle() {
|
|
62
|
+
notifyIOSVoiceSessionEnded();
|
|
63
|
+
await sleep(RECONNECT_SETTLE_MS);
|
|
64
|
+
activateIOSWebRTCAudioSession();
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Before voice capture on iOS.
|
|
68
|
+
* - Cold start: wait for waker, then sync WebRTC audio session.
|
|
69
|
+
* - Reconnect: deactivate → nudge waker → activate WebRTC (required after ~4+ cycles on iOS).
|
|
70
|
+
*/
|
|
71
|
+
export async function ensureIOSAudioSessionForVoiceCapture(timeoutMs = 5000) {
|
|
72
|
+
if (Platform.OS !== 'ios')
|
|
73
|
+
return;
|
|
74
|
+
if (!hasCompletedVoiceCapture) {
|
|
75
|
+
await waitForIOSAudioSessionReady(timeoutMs);
|
|
76
|
+
activateIOSWebRTCAudioSession();
|
|
77
|
+
return;
|
|
78
|
+
}
|
|
79
|
+
deactivateIOSWebRTCAudioSession();
|
|
80
|
+
await nudgeAndSettle();
|
|
81
|
+
}
|
|
82
|
+
/** Allow WebRTC / AVAudioSession to release the mic before the next getUserMedia. */
|
|
83
|
+
export async function waitForIOSVoiceCaptureRelease() {
|
|
84
|
+
if (Platform.OS !== 'ios')
|
|
85
|
+
return;
|
|
86
|
+
deactivateIOSWebRTCAudioSession();
|
|
87
|
+
notifyIOSVoiceSessionEnded();
|
|
88
|
+
await sleep(POST_STOP_SETTLE_MS);
|
|
89
|
+
}
|
|
90
|
+
/** Current capture generation — useful for logging flaky reconnect attempts. */
|
|
91
|
+
export function getIOSVoiceCaptureGeneration() {
|
|
92
|
+
return voiceCaptureGeneration;
|
|
93
|
+
}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
/** Tell WebRTC the AVAudioSession is active (after react-native-video waker). */
|
|
2
|
+
export declare function activateIOSWebRTCAudioSession(): void;
|
|
3
|
+
/** Release WebRTC audio session lock after voice stop (before next getUserMedia). */
|
|
4
|
+
export declare function deactivateIOSWebRTCAudioSession(): void;
|
|
5
|
+
//# sourceMappingURL=iosWebRTCAudioSession.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"iosWebRTCAudioSession.d.ts","sourceRoot":"","sources":["../../src/utils/iosWebRTCAudioSession.ts"],"names":[],"mappings":"AAsBA,iFAAiF;AACjF,wBAAgB,6BAA6B,IAAI,IAAI,CAMpD;AAED,qFAAqF;AACrF,wBAAgB,+BAA+B,IAAI,IAAI,CAMtD"}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { Platform } from 'react-native';
|
|
2
|
+
let rtcAudioSession;
|
|
3
|
+
function getRTCAudioSession() {
|
|
4
|
+
if (Platform.OS !== 'ios')
|
|
5
|
+
return null;
|
|
6
|
+
if (rtcAudioSession !== undefined)
|
|
7
|
+
return rtcAudioSession;
|
|
8
|
+
try {
|
|
9
|
+
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
|
10
|
+
const mod = require('react-native-webrtc');
|
|
11
|
+
rtcAudioSession = mod.RTCAudioSession ?? null;
|
|
12
|
+
}
|
|
13
|
+
catch {
|
|
14
|
+
rtcAudioSession = null;
|
|
15
|
+
}
|
|
16
|
+
return rtcAudioSession;
|
|
17
|
+
}
|
|
18
|
+
/** Tell WebRTC the AVAudioSession is active (after react-native-video waker). */
|
|
19
|
+
export function activateIOSWebRTCAudioSession() {
|
|
20
|
+
try {
|
|
21
|
+
getRTCAudioSession()?.audioSessionDidActivate();
|
|
22
|
+
}
|
|
23
|
+
catch (err) {
|
|
24
|
+
console.warn('[nxtlinq] RTCAudioSession.audioSessionDidActivate failed:', err);
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
/** Release WebRTC audio session lock after voice stop (before next getUserMedia). */
|
|
28
|
+
export function deactivateIOSWebRTCAudioSession() {
|
|
29
|
+
try {
|
|
30
|
+
getRTCAudioSession()?.audioSessionDidDeactivate();
|
|
31
|
+
}
|
|
32
|
+
catch (err) {
|
|
33
|
+
console.warn('[nxtlinq] RTCAudioSession.audioSessionDidDeactivate failed:', err);
|
|
34
|
+
}
|
|
35
|
+
}
|
package/package.json
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@bytexbyte/nxtlinq-ai-agent-react-native-development",
|
|
3
|
+
"version": "0.2.0",
|
|
4
|
+
"description": "React Native SDK for nxtlinq AI Agent — custom UI hooks and platform ports",
|
|
5
|
+
"main": "dist/index.js",
|
|
6
|
+
"react-native": "src/index.ts",
|
|
7
|
+
"types": "dist/index.d.ts",
|
|
8
|
+
"files": [
|
|
9
|
+
"dist",
|
|
10
|
+
"src"
|
|
11
|
+
],
|
|
12
|
+
"scripts": {
|
|
13
|
+
"build": "tsc",
|
|
14
|
+
"clean": "rm -rf dist",
|
|
15
|
+
"prepublishOnly": "yarn build"
|
|
16
|
+
},
|
|
17
|
+
"keywords": [
|
|
18
|
+
"nxtlinq",
|
|
19
|
+
"ai-agent",
|
|
20
|
+
"react-native",
|
|
21
|
+
"sdk"
|
|
22
|
+
],
|
|
23
|
+
"author": "ByteXByte",
|
|
24
|
+
"license": "MIT",
|
|
25
|
+
"repository": {
|
|
26
|
+
"type": "git",
|
|
27
|
+
"url": "https://dev.azure.com/nxtlinqLLC/nxtlinq/_git/nxtlinq-AI-Agent-SDK",
|
|
28
|
+
"directory": "packages/ai-agent-react-native"
|
|
29
|
+
},
|
|
30
|
+
"publishConfig": {
|
|
31
|
+
"access": "public",
|
|
32
|
+
"registry": "https://registry.npmjs.org/"
|
|
33
|
+
},
|
|
34
|
+
"sideEffects": false,
|
|
35
|
+
"peerDependencies": {
|
|
36
|
+
"react": ">=18.0.0",
|
|
37
|
+
"react-native": ">=0.72.0"
|
|
38
|
+
},
|
|
39
|
+
"peerDependenciesMeta": {
|
|
40
|
+
"@react-native-async-storage/async-storage": {
|
|
41
|
+
"optional": true
|
|
42
|
+
},
|
|
43
|
+
"react-native-webrtc": {
|
|
44
|
+
"optional": true
|
|
45
|
+
}
|
|
46
|
+
},
|
|
47
|
+
"dependencies": {
|
|
48
|
+
"@bytexbyte/nxtlinq-ai-agent-core-development": "^0.2.0"
|
|
49
|
+
},
|
|
50
|
+
"devDependencies": {
|
|
51
|
+
"@bytexbyte/nxtlinq-ai-agent-core-development": "workspace:^",
|
|
52
|
+
"@types/react": "^18.2.64",
|
|
53
|
+
"react": "^18.2.0",
|
|
54
|
+
"typescript": "^5.4.2"
|
|
55
|
+
}
|
|
56
|
+
}
|