@oryksa/react-native 1.0.0 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/voice.js ADDED
@@ -0,0 +1,338 @@
1
+ /*!
2
+ * @oryksa/react-native - the ORYKSA voice conversation (the same behaviour as the ORYKSA app and
3
+ * every other ORYKSA channel): the microphone stays open even while she speaks; only a human voice
4
+ * cuts her off and she stops on the word (the text stays in the chat); the last second before the cut
5
+ * is kept; 700 ms of silence closes a sentence, 15 s is the longest; a whisper gets a whispered,
6
+ * shorter answer; her voice is the ElevenLabs voice chosen in ORYKSA, and if it fails she stays
7
+ * silent and the text is shown (never a robot voice).
8
+ *
9
+ * Audio goes through a small adapter (see createDefaultAudio). License: MIT
10
+ */
11
+ "use strict";
12
+
13
+ const { Vad } = require("./vad");
14
+
15
+ const SR = 16000;
16
+ const PRE_ROLL_BYTES = SR * 2; // one second of 16 kHz PCM16
17
+
18
+ const B64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
19
+
20
+ /** base64 -> Uint8Array (no atob needed). */
21
+ function fromBase64(str) {
22
+ const clean = String(str).replace(/[^A-Za-z0-9+/]/g, "");
23
+ const out = new Uint8Array(Math.floor((clean.length * 3) / 4));
24
+ let o = 0;
25
+ for (let i = 0; i + 1 < clean.length; i += 4) {
26
+ const a = B64.indexOf(clean[i]), b = B64.indexOf(clean[i + 1]);
27
+ const c = i + 2 < clean.length ? B64.indexOf(clean[i + 2]) : -1;
28
+ const d = i + 3 < clean.length ? B64.indexOf(clean[i + 3]) : -1;
29
+ out[o++] = (a << 2) | (b >> 4);
30
+ if (c >= 0 && o < out.length) out[o++] = ((b & 15) << 4) | (c >> 2);
31
+ if (d >= 0 && o < out.length) out[o++] = ((c & 3) << 6) | d;
32
+ }
33
+ return out.subarray(0, o);
34
+ }
35
+
36
+ /** Uint8Array -> base64. */
37
+ function toBase64(bytes) {
38
+ let s = "";
39
+ for (let i = 0; i < bytes.length; i += 3) {
40
+ const a = bytes[i], b = bytes[i + 1], c = bytes[i + 2];
41
+ s += B64[a >> 2] + B64[((a & 3) << 4) | ((b || 0) >> 4)];
42
+ s += i + 1 < bytes.length ? B64[((b & 15) << 2) | ((c || 0) >> 6)] : "=";
43
+ s += i + 2 < bytes.length ? B64[c & 63] : "=";
44
+ }
45
+ return s;
46
+ }
47
+
48
+ /** 16 kHz mono PCM16 WAV (what the server transcribes). */
49
+ function wav(pcm, sampleRate) {
50
+ const sr = sampleRate || SR;
51
+ const out = new Uint8Array(44 + pcm.length);
52
+ const dv = new DataView(out.buffer);
53
+ const tag = (o, t) => { for (let i = 0; i < 4; i++) out[o + i] = t.charCodeAt(i); };
54
+ tag(0, "RIFF"); dv.setUint32(4, 36 + pcm.length, true); tag(8, "WAVE");
55
+ tag(12, "fmt "); dv.setUint32(16, 16, true); dv.setUint16(20, 1, true); dv.setUint16(22, 1, true);
56
+ dv.setUint32(24, sr, true); dv.setUint32(28, sr * 2, true); dv.setUint16(32, 2, true); dv.setUint16(34, 16, true);
57
+ tag(36, "data"); dv.setUint32(40, pcm.length, true);
58
+ out.set(pcm, 44);
59
+ return out;
60
+ }
61
+
62
+ function concat(parts, total) {
63
+ const out = new Uint8Array(total);
64
+ let o = 0;
65
+ for (const p of parts) { out.set(p, o); o += p.length; }
66
+ return out;
67
+ }
68
+
69
+ /**
70
+ * The default audio adapter for bare React Native apps. Install the three libraries:
71
+ * npm install react-native-live-audio-stream react-native-sound react-native-fs
72
+ * and add the microphone permission (Android RECORD_AUDIO, iOS NSMicrophoneUsageDescription).
73
+ * Expo or another audio stack: pass your own adapter with the same four functions.
74
+ *
75
+ * Adapter contract:
76
+ * startMic(onChunk) -> Promise<boolean> onChunk(Uint8Array PCM16 LE mono 16 kHz)
77
+ * stopMic() -> Promise<void>
78
+ * play(mp3Uint8) -> Promise<void> resolves when it finished or was stopped
79
+ * stopPlayback() -> Promise<void>
80
+ */
81
+ function createDefaultAudio() {
82
+ let LiveAudioStream, Sound, RNFS, RN;
83
+ try {
84
+ LiveAudioStream = require("react-native-live-audio-stream").default || require("react-native-live-audio-stream");
85
+ Sound = require("react-native-sound");
86
+ RNFS = require("react-native-fs");
87
+ RN = require("react-native");
88
+ } catch (_) {
89
+ return null; // libraries not installed: the chat hides the microphone
90
+ }
91
+ let sub = null;
92
+ let current = null;
93
+ let ended = null;
94
+ return {
95
+ async startMic(onChunk) {
96
+ try {
97
+ if (RN.Platform.OS === "android") {
98
+ const P = RN.PermissionsAndroid;
99
+ const r = await P.request(P.PERMISSIONS.RECORD_AUDIO);
100
+ if (r !== P.RESULTS.GRANTED) return false;
101
+ }
102
+ LiveAudioStream.init({ sampleRate: SR, channels: 1, bitsPerSample: 16, audioSource: 7, bufferSize: 4096, wavFile: "" });
103
+ sub = LiveAudioStream.on("data", (b64) => onChunk(fromBase64(b64)));
104
+ LiveAudioStream.start();
105
+ return true;
106
+ } catch (_) {
107
+ return false;
108
+ }
109
+ },
110
+ async stopMic() {
111
+ try { LiveAudioStream.stop(); } catch (_) { /* ignore */ }
112
+ if (sub && sub.remove) sub.remove();
113
+ sub = null;
114
+ },
115
+ async play(mp3) {
116
+ const file = RNFS.CachesDirectoryPath + "/oryksa_voice_" + Date.now() + ".mp3";
117
+ await RNFS.writeFile(file, toBase64(mp3), "base64");
118
+ Sound.setCategory("PlayAndRecord", true);
119
+ await new Promise((ok) => {
120
+ ended = ok;
121
+ const snd = new Sound(file, "", (err) => {
122
+ if (err) { ok(); return; }
123
+ current = snd;
124
+ snd.play(() => { snd.release(); current = null; ok(); });
125
+ });
126
+ });
127
+ RNFS.unlink(file).catch(() => {});
128
+ },
129
+ async stopPlayback() {
130
+ if (current) { try { current.stop(); current.release(); } catch (_) { /* ignore */ } current = null; }
131
+ if (ended) { ended(); ended = null; }
132
+ },
133
+ };
134
+ }
135
+
136
+ /**
137
+ * The voice conversation. Phases: starting, listening, hearing, thinking, speaking, muted, micError,
138
+ * noisy, notUnderstood (onPhase is called on every change).
139
+ */
140
+ class OryksaVoiceController {
141
+ /**
142
+ * @param {{client: object, audio?: object, appContext?: () => object, onUserText?: (t: string) => void,
143
+ * onReply?: (t: string) => void, onPhase?: (phase: string, ctl: OryksaVoiceController) => void}} opts
144
+ */
145
+ constructor(opts) {
146
+ const o = opts || {};
147
+ this.client = o.client;
148
+ this.audio = o.audio || createDefaultAudio();
149
+ this.appContext = o.appContext || null;
150
+ this.onUserText = o.onUserText || null;
151
+ this.onReply = o.onReply || null;
152
+ this.onPhase = o.onPhase || null;
153
+ this.vad = new Vad(SR);
154
+ this.phase = "starting";
155
+ this.lastReply = "";
156
+ this.lastHeard = "";
157
+ this._pcm = [];
158
+ this._pcmBytes = 0;
159
+ this._before = [];
160
+ this._beforeBytes = 0;
161
+ this._running = false;
162
+ this._closed = false;
163
+ this._speaking = false;
164
+ this._busy = false;
165
+ this._cut = false;
166
+ this._whispered = false;
167
+ this._noise = 0;
168
+ this._turn = 0;
169
+ this._heard = false;
170
+ }
171
+
172
+ _set(p) {
173
+ if (this._closed || this.phase === p) return;
174
+ this.phase = p;
175
+ if (this.onPhase) this.onPhase(p, this);
176
+ }
177
+
178
+ /** Opens the microphone and starts listening. */
179
+ async start() {
180
+ if (this._closed || this._running || this.phase === "muted") return;
181
+ if (!this.audio) { this._set("micError"); return; }
182
+ const ok = await this.audio.startMic((c) => this._onAudio(c));
183
+ if (!ok) { this._set("micError"); return; }
184
+ this._running = true;
185
+ this.vad.reset();
186
+ this._pcm = []; this._pcmBytes = 0;
187
+ this._heard = false;
188
+ this._set("listening");
189
+ setTimeout(() => { if (!this._closed && !this._heard && this.phase !== "muted") this._set("micError"); }, 4000);
190
+ }
191
+
192
+ _onAudio(chunk) {
193
+ if (this._closed || this.phase === "muted" || !chunk || chunk.length < 2) return;
194
+ if (!this._heard && Vad.rmsOf(chunk) > 0.0005) this._heard = true;
195
+
196
+ // While she SPEAKS the engine only decides whether she was cut off.
197
+ if (this._speaking) {
198
+ this._before.push(chunk);
199
+ this._beforeBytes += chunk.length;
200
+ while (this._beforeBytes > PRE_ROLL_BYTES && this._before.length > 1) this._beforeBytes -= this._before.shift().length;
201
+ if (this.vad.feed(chunk, true).isBargeIn) this._bargeIn();
202
+ return;
203
+ }
204
+
205
+ const step = this.vad.feed(chunk);
206
+ if (step.isBargeIn) { this._bargeIn(); return; }
207
+ if (step.isCapturing || this.vad.capturing) {
208
+ this._pcm.push(chunk);
209
+ this._pcmBytes += chunk.length;
210
+ if (!this._busy) this._set("hearing");
211
+ }
212
+ if (!step.isDone) return;
213
+
214
+ const raw = concat(this._pcm, this._pcmBytes);
215
+ this._pcm = []; this._pcmBytes = 0;
216
+ const whisper = this.vad.foiSussurro;
217
+ const stats = this._stats(false, Math.floor(raw.length / 32));
218
+ this.vad.newTurn();
219
+ if (step.enough && raw.length > SR / 2) {
220
+ this._noise = 0;
221
+ this._handle(wav(raw), whisper, stats);
222
+ } else if (step.isNoise) {
223
+ if (++this._noise >= 2) this._set("noisy");
224
+ } else if (!this._busy) {
225
+ this._set("listening");
226
+ }
227
+ }
228
+
229
+ /** The customer cut her off: stop the voice on the word, keep what he says. */
230
+ _bargeIn() {
231
+ this._turn++;
232
+ this._speaking = false;
233
+ this._busy = false;
234
+ if (this.audio) this.audio.stopPlayback();
235
+ this.vad.newTurn();
236
+ this.vad.capturing = true;
237
+ this._pcm = this._before.slice();
238
+ this._pcmBytes = this._beforeBytes;
239
+ this._before = []; this._beforeBytes = 0;
240
+ this._cut = true;
241
+ this._set("hearing");
242
+ }
243
+
244
+ /** Sends what was said right away (tap on the picture). */
245
+ sendNow() {
246
+ if (this._busy || this._speaking || !this.vad.capturing) return;
247
+ const raw = concat(this._pcm, this._pcmBytes);
248
+ const whisper = this.vad.foiSussurro;
249
+ const stats = this._stats(false, Math.floor(raw.length / 32));
250
+ this.vad.newTurn();
251
+ this._pcm = []; this._pcmBytes = 0;
252
+ if (raw.length > SR / 2) this._handle(wav(raw), whisper, stats);
253
+ }
254
+
255
+ /** Audio numbers of this turn, in the ORYKSA ecosystem format (quality report). */
256
+ _stats(empty, durMs) {
257
+ const r3 = (v) => Math.round(v * 1000) / 1000;
258
+ return { channel: "sdk_rn", peak: r3(this.vad.picoDaFrase), pitch: r3(this.vad.tomDaFrase), noise: r3(this.vad.base),
259
+ floor: Vad.bargeFloor, cut: false, whisper: this.vad.foiSussurro, self_cut: this._cut, empty, dur_ms: durMs };
260
+ }
261
+
262
+ async _handle(wavBytes, whisper, stats) {
263
+ const my = ++this._turn;
264
+ this._whispered = whisper;
265
+ this._busy = true;
266
+ this._set("thinking");
267
+ const text = await this.client.transcribe(wavBytes);
268
+ if (this._closed || my !== this._turn) return;
269
+ if (!text) {
270
+ await this.client.voiceStats(Object.assign({}, stats, { empty: true }));
271
+ this._busy = false;
272
+ this._set(text === null ? "notUnderstood" : "listening");
273
+ return;
274
+ }
275
+ this.lastHeard = text;
276
+ if (this.onUserText) this.onUserText(text);
277
+ let r = null;
278
+ try {
279
+ r = await this.client.sendAndWaitReply(text, { appContext: this.appContext ? this.appContext() : undefined, voice: true, whisper, voiceStats: stats });
280
+ } catch (_) { r = null; }
281
+ if (this._closed || my !== this._turn) return;
282
+ const reply = String((r && r.reply) || "").trim();
283
+ if (!reply) { this._busy = false; this._set("notUnderstood"); return; }
284
+ this.lastReply = reply;
285
+ if (this.onReply) this.onReply(reply);
286
+ const speech = String((r && r.speech) || "").trim();
287
+ await this._speak(speech || reply, my);
288
+ }
289
+
290
+ /** Speaks text with the AI's voice (for example a greeting). If the voice fails she stays silent. */
291
+ say(text) { return this._speak(text, ++this._turn); }
292
+
293
+ async _speak(text, my) {
294
+ this._before = []; this._beforeBytes = 0;
295
+ const done = () => {
296
+ this._cut = false;
297
+ this._speaking = false;
298
+ this._busy = false;
299
+ this.vad.newTurn();
300
+ if (!this._closed && my === this._turn) this._set("listening");
301
+ };
302
+ const mp3 = await this.client.tts(text, { whisper: this._whispered });
303
+ if (this._closed || my !== this._turn || !mp3 || !this.audio) { done(); return; } // never a robot voice
304
+ this._speaking = true;
305
+ this._set("speaking");
306
+ try { await this.audio.play(mp3); } catch (_) { /* silence + text */ }
307
+ if (my === this._turn) done();
308
+ }
309
+
310
+ /** Mute: she stops talking and listening. Nothing is lost: what she said stays in the chat. */
311
+ async toggleMute() {
312
+ if (this.phase === "muted") {
313
+ this.phase = "starting";
314
+ if (this.onPhase) this.onPhase("starting", this);
315
+ await this.start();
316
+ return;
317
+ }
318
+ this._turn++;
319
+ this._speaking = false;
320
+ this._busy = false;
321
+ if (this.audio) { await this.audio.stopPlayback(); await this.audio.stopMic(); }
322
+ this._running = false;
323
+ this.vad.newTurn();
324
+ this.phase = "muted";
325
+ if (this.onPhase) this.onPhase("muted", this);
326
+ }
327
+
328
+ /** Closes the microphone and the player. */
329
+ async close() {
330
+ if (this._closed) return;
331
+ this._closed = true;
332
+ this._turn++;
333
+ if (this.audio) { await this.audio.stopPlayback(); await this.audio.stopMic(); }
334
+ this._running = false;
335
+ }
336
+ }
337
+
338
+ module.exports = { OryksaVoiceController, createDefaultAudio, wav, fromBase64, toBase64 };