@oryksa/react-native 1.0.0 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/README.md +123 -98
- package/package.json +49 -9
- package/src/chat.js +225 -163
- package/src/client.js +133 -14
- package/src/index.d.ts +151 -53
- package/src/index.js +28 -18
- package/src/profanity.js +60 -0
- package/src/vad.js +264 -0
- package/src/voice-screen.js +141 -0
- package/src/voice.js +338 -0
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
/*!
|
|
2
|
+
* @oryksa/react-native - the voice screen, the same as the ORYKSA app: the photo of the AI with a
|
|
3
|
+
* halo, "I'm listening" / her answer, Mute and Close. License: MIT
|
|
4
|
+
*/
|
|
5
|
+
"use strict";
|
|
6
|
+
|
|
7
|
+
const React = require("react");
|
|
8
|
+
const RN = require("react-native");
|
|
9
|
+
const { OryksaVoiceController } = require("./voice");
|
|
10
|
+
|
|
11
|
+
const h = React.createElement;
|
|
12
|
+
const { View, Text, Image, Pressable, Modal, SafeAreaView, StyleSheet, Animated, Easing } = RN;
|
|
13
|
+
|
|
14
|
+
const VX = {
|
|
15
|
+
en: { listening: "I'm listening", listeningSub: "Speak to me. You can cut me off any time.", hearing: "Go on, I am listening.",
|
|
16
|
+
thinking: "One moment...", muted: "Paused", mutedSub: "Tap the mic to talk again.",
|
|
17
|
+
micError: "Microphone unavailable", micErrorSub: "Allow microphone access, or close other apps using it.",
|
|
18
|
+
noisy: "Too much background noise", noisySub: "I cannot tell your voice from the noise. Move somewhere quieter, or type to me.",
|
|
19
|
+
notUnderstood: "I could not understand", notUnderstoodSub: "Say it again, please.",
|
|
20
|
+
tapToSend: "Tap the picture to send what you said.", mute: "Mute", unmute: "Unmute", close: "Close" },
|
|
21
|
+
pt: { listening: "Estou a ouvir", listeningSub: "Fala comigo. Podes interromper-me quando quiseres.", hearing: "Continua, estou a ouvir.",
|
|
22
|
+
thinking: "Um momento...", muted: "Em pausa", mutedSub: "Toca no microfone para voltar a falar.",
|
|
23
|
+
micError: "Microfone indisponível", micErrorSub: "Permite o acesso ao microfone, ou fecha outras apps que o estejam a usar.",
|
|
24
|
+
noisy: "Demasiado barulho", noisySub: "Não consigo distinguir a tua voz do barulho. Vai para um sítio mais calmo, ou escreve-me.",
|
|
25
|
+
notUnderstood: "Não percebi", notUnderstoodSub: "Diz outra vez, por favor.",
|
|
26
|
+
tapToSend: "Toca na imagem para enviar o que disseste.", mute: "Silenciar", unmute: "Ativar som", close: "Fechar" },
|
|
27
|
+
br: { listening: "Estou ouvindo", listeningSub: "Fale comigo. Você pode me interromper quando quiser.", hearing: "Continue, estou ouvindo.",
|
|
28
|
+
thinking: "Um momento...", muted: "Em pausa", mutedSub: "Toque no microfone para voltar a falar.",
|
|
29
|
+
micError: "Microfone indisponível", micErrorSub: "Permita o acesso ao microfone, ou feche outros apps que estejam usando.",
|
|
30
|
+
noisy: "Barulho demais", noisySub: "Não consigo separar sua voz do barulho. Vá para um lugar mais calmo, ou digite para mim.",
|
|
31
|
+
notUnderstood: "Não entendi", notUnderstoodSub: "Fale de novo, por favor.",
|
|
32
|
+
tapToSend: "Toque na imagem para enviar o que você disse.", mute: "Silenciar", unmute: "Ativar som", close: "Fechar" },
|
|
33
|
+
es: { listening: "Te escucho", listeningSub: "Háblame. Puedes interrumpirme cuando quieras.", hearing: "Sigue, te escucho.",
|
|
34
|
+
thinking: "Un momento...", muted: "En pausa", mutedSub: "Toca el micrófono para volver a hablar.",
|
|
35
|
+
micError: "Micrófono no disponible", micErrorSub: "Permite el acceso al micrófono, o cierra otras apps que lo estén usando.",
|
|
36
|
+
noisy: "Demasiado ruido", noisySub: "No distingo tu voz del ruido. Ve a un sitio más tranquilo, o escríbeme.",
|
|
37
|
+
notUnderstood: "No te entendí", notUnderstoodSub: "Dilo otra vez, por favor.",
|
|
38
|
+
tapToSend: "Toca la imagen para enviar lo que dijiste.", mute: "Silenciar", unmute: "Activar sonido", close: "Cerrar" },
|
|
39
|
+
};
|
|
40
|
+
|
|
41
|
+
const FALLBACK_AVATAR = "https://oryksa.com/assets/img/avatar_official_oryksa.png";
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Full-screen voice conversation. Props: client, agent ({ name, avatar, business }), visible, onClose,
|
|
45
|
+
* lang, theme, appContext (() => ({ screen, title, items })), onUserText, onReply, audio (adapter).
|
|
46
|
+
*/
|
|
47
|
+
function OryksaVoiceScreen(props) {
|
|
48
|
+
const { client, agent, visible, onClose, lang = "en", theme, appContext, onUserText, onReply, audio } = props;
|
|
49
|
+
const th = Object.assign({ accent: "#5B57E0", ink: "#161B3D", soft: "#EEEBFB", background: "#FFFFFF", muted: "#6B7280" }, theme || {});
|
|
50
|
+
const t = VX[lang] || VX.en;
|
|
51
|
+
const [phase, setPhase] = React.useState("starting");
|
|
52
|
+
const ctlRef = React.useRef(null);
|
|
53
|
+
const pulse = React.useRef(new Animated.Value(1)).current;
|
|
54
|
+
|
|
55
|
+
React.useEffect(() => {
|
|
56
|
+
if (!visible) return undefined;
|
|
57
|
+
const ctl = new OryksaVoiceController({ client, audio, appContext, onUserText, onReply, onPhase: (p) => setPhase(p) });
|
|
58
|
+
ctlRef.current = ctl;
|
|
59
|
+
ctl.start();
|
|
60
|
+
const loop = Animated.loop(Animated.sequence([
|
|
61
|
+
Animated.timing(pulse, { toValue: 1.07, duration: 1000, easing: Easing.inOut(Easing.ease), useNativeDriver: true }),
|
|
62
|
+
Animated.timing(pulse, { toValue: 1, duration: 1000, easing: Easing.inOut(Easing.ease), useNativeDriver: true }),
|
|
63
|
+
]));
|
|
64
|
+
loop.start();
|
|
65
|
+
return () => { loop.stop(); ctl.close(); ctlRef.current = null; };
|
|
66
|
+
}, [visible, client]);
|
|
67
|
+
|
|
68
|
+
const ctl = ctlRef.current;
|
|
69
|
+
const pick = {
|
|
70
|
+
starting: [t.listening, t.listeningSub], listening: [t.listening, t.listeningSub], hearing: [t.listening, t.hearing],
|
|
71
|
+
// never the words the person said on screen: only "listening" in the interface language; her answer shows as text
|
|
72
|
+
thinking: [t.listening, t.thinking], speaking: [(agent && agent.name) || "ORYKSA", (ctl && ctl.lastReply) || ""],
|
|
73
|
+
muted: [t.muted, t.mutedSub], micError: [t.micError, t.micErrorSub], noisy: [t.noisy, t.noisySub],
|
|
74
|
+
notUnderstood: [t.notUnderstood, t.notUnderstoodSub],
|
|
75
|
+
}[phase] || [t.listening, t.listeningSub];
|
|
76
|
+
const active = phase === "hearing" || phase === "speaking";
|
|
77
|
+
const photo = (agent && agent.avatar) || FALLBACK_AVATAR;
|
|
78
|
+
const close = () => { if (ctlRef.current) ctlRef.current.close(); if (onClose) onClose(); };
|
|
79
|
+
const ring = (size, alpha) => h(Animated.View, {
|
|
80
|
+
key: size, style: [st.ring, { width: size, height: size, borderRadius: size / 2, backgroundColor: th.accent, opacity: alpha,
|
|
81
|
+
transform: [{ scale: active ? pulse : 1 }] }] });
|
|
82
|
+
|
|
83
|
+
return h(Modal, { visible: !!visible, animationType: "slide", onRequestClose: close },
|
|
84
|
+
h(SafeAreaView, { style: [st.screen, { backgroundColor: th.background }] },
|
|
85
|
+
h(View, { style: st.hd },
|
|
86
|
+
h(Image, { source: { uri: photo }, style: st.hdImg }),
|
|
87
|
+
h(View, { style: { flex: 1 } },
|
|
88
|
+
h(Text, { style: [st.hdName, { color: th.ink }] }, (agent && agent.name) || "ORYKSA"),
|
|
89
|
+
agent && agent.business ? h(Text, { style: { fontSize: 11, color: th.muted }, numberOfLines: 1 }, agent.business) : null),
|
|
90
|
+
h(Pressable, { onPress: close, accessibilityRole: "button", accessibilityLabel: t.close, hitSlop: 10 },
|
|
91
|
+
h(Text, { style: { fontSize: 26, color: th.muted, paddingHorizontal: 6 } }, "×"))),
|
|
92
|
+
h(View, { style: st.center },
|
|
93
|
+
h(Pressable, { onPress: () => ctlRef.current && ctlRef.current.sendNow(), style: st.halo },
|
|
94
|
+
ring(236, 0.06), ring(184, 0.09), ring(136, 0.13),
|
|
95
|
+
h(Image, { source: { uri: photo }, style: st.photo })),
|
|
96
|
+
h(Text, { style: [st.title, { color: th.ink }] }, pick[0]),
|
|
97
|
+
h(Text, { style: [st.sub, { color: th.muted }] }, pick[1]),
|
|
98
|
+
phase === "hearing" ? h(Text, { style: st.tap }, t.tapToSend) : null,
|
|
99
|
+
h(View, { style: st.row },
|
|
100
|
+
btn(h(MicIcon, { color: th.muted, crossed: phase === "muted" }), phase === "muted" ? t.unmute : t.mute, false, () => ctlRef.current && ctlRef.current.toggleMute(), th),
|
|
101
|
+
btn("×", t.close, true, close, th))),
|
|
102
|
+
h(Text, { style: st.pw }, "POWERED BY ", h(Text, { style: { color: th.accent, fontWeight: "800" } }, "ORYKSA"))));
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/** A microphone drawn with views (no emoji or icon font needed); crossed = muted. */
|
|
106
|
+
function MicIcon(props) {
|
|
107
|
+
const c = props.color || "#6B7280";
|
|
108
|
+
const k = (props.size || 22) / 22;
|
|
109
|
+
return h(View, { style: { width: 22 * k, height: 22 * k, alignItems: "center" } },
|
|
110
|
+
h(View, { style: { width: 8 * k, height: 12 * k, borderRadius: 4 * k, borderWidth: 2 * k, borderColor: c } }),
|
|
111
|
+
h(View, { style: { width: 14 * k, height: 7 * k, marginTop: -4 * k, borderBottomLeftRadius: 7 * k, borderBottomRightRadius: 7 * k, borderWidth: 2 * k, borderTopWidth: 0, borderColor: c } }),
|
|
112
|
+
h(View, { style: { width: 2 * k, height: 3 * k, backgroundColor: c } }),
|
|
113
|
+
props.crossed ? h(View, { style: { position: "absolute", top: 10 * k, left: -1 * k, width: 24 * k, height: 2 * k, backgroundColor: c, transform: [{ rotate: "45deg" }] } }) : null);
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
function btn(icon, label, cancel, onPress, th) {
|
|
117
|
+
const red = "#E05A52";
|
|
118
|
+
return h(Pressable, { onPress, accessibilityRole: "button", accessibilityLabel: label, style: { alignItems: "center" } },
|
|
119
|
+
h(View, { style: [st.cbtn, { backgroundColor: cancel ? "#FDF0EF" : th.background, borderColor: cancel ? "#F3C0BE" : "#ECECF6" }] },
|
|
120
|
+
typeof icon === "string" ? h(Text, { style: { fontSize: 22, color: cancel ? red : th.muted } }, icon) : icon),
|
|
121
|
+
h(Text, { style: { marginTop: 8, fontSize: 12, color: cancel ? red : th.muted } }, label));
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
const st = StyleSheet.create({
|
|
125
|
+
screen: { flex: 1 },
|
|
126
|
+
hd: { flexDirection: "row", alignItems: "center", gap: 10, paddingHorizontal: 16, paddingTop: 12 },
|
|
127
|
+
hdImg: { width: 38, height: 38, borderRadius: 19 },
|
|
128
|
+
hdName: { fontSize: 14.5, fontWeight: "700" },
|
|
129
|
+
center: { flex: 1, alignItems: "center", justifyContent: "center", paddingHorizontal: 30 },
|
|
130
|
+
halo: { width: 250, height: 250, alignItems: "center", justifyContent: "center" },
|
|
131
|
+
ring: { position: "absolute" },
|
|
132
|
+
photo: { width: 124, height: 124, borderRadius: 62 },
|
|
133
|
+
title: { marginTop: 14, fontSize: 22, fontWeight: "700", textAlign: "center" },
|
|
134
|
+
sub: { marginTop: 8, fontSize: 13.5, lineHeight: 20, textAlign: "center" },
|
|
135
|
+
tap: { marginTop: 10, fontSize: 11.5, color: "#AAB0CC" },
|
|
136
|
+
row: { flexDirection: "row", gap: 46, marginTop: 20 },
|
|
137
|
+
cbtn: { width: 58, height: 58, borderRadius: 29, borderWidth: 1.5, alignItems: "center", justifyContent: "center" },
|
|
138
|
+
pw: { textAlign: "center", fontSize: 10.5, letterSpacing: 1.3, color: "#9CA3AF", paddingBottom: 22 },
|
|
139
|
+
});
|
|
140
|
+
|
|
141
|
+
module.exports = { OryksaVoiceScreen, MicIcon, VOICE_TEXTS: VX };
|
package/src/voice.js
ADDED
|
@@ -0,0 +1,338 @@
|
|
|
1
|
+
/*!
|
|
2
|
+
* @oryksa/react-native - the ORYKSA voice conversation (the same behaviour as the ORYKSA app and
|
|
3
|
+
* every other ORYKSA channel): the microphone stays open even while she speaks; only a human voice
|
|
4
|
+
* cuts her off and she stops on the word (the text stays in the chat); the last second before the cut
|
|
5
|
+
* is kept; 700 ms of silence closes a sentence, 15 s is the longest; a whisper gets a whispered,
|
|
6
|
+
* shorter answer; her voice is the ElevenLabs voice chosen in ORYKSA, and if it fails she stays
|
|
7
|
+
* silent and the text is shown (never a robot voice).
|
|
8
|
+
*
|
|
9
|
+
* Audio goes through a small adapter (see createDefaultAudio). License: MIT
|
|
10
|
+
*/
|
|
11
|
+
"use strict";
|
|
12
|
+
|
|
13
|
+
const { Vad } = require("./vad");
|
|
14
|
+
|
|
15
|
+
const SR = 16000;
|
|
16
|
+
const PRE_ROLL_BYTES = SR * 2; // one second of 16 kHz PCM16
|
|
17
|
+
|
|
18
|
+
const B64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
|
|
19
|
+
|
|
20
|
+
/** base64 -> Uint8Array (no atob needed). */
|
|
21
|
+
function fromBase64(str) {
|
|
22
|
+
const clean = String(str).replace(/[^A-Za-z0-9+/]/g, "");
|
|
23
|
+
const out = new Uint8Array(Math.floor((clean.length * 3) / 4));
|
|
24
|
+
let o = 0;
|
|
25
|
+
for (let i = 0; i + 1 < clean.length; i += 4) {
|
|
26
|
+
const a = B64.indexOf(clean[i]), b = B64.indexOf(clean[i + 1]);
|
|
27
|
+
const c = i + 2 < clean.length ? B64.indexOf(clean[i + 2]) : -1;
|
|
28
|
+
const d = i + 3 < clean.length ? B64.indexOf(clean[i + 3]) : -1;
|
|
29
|
+
out[o++] = (a << 2) | (b >> 4);
|
|
30
|
+
if (c >= 0 && o < out.length) out[o++] = ((b & 15) << 4) | (c >> 2);
|
|
31
|
+
if (d >= 0 && o < out.length) out[o++] = ((c & 3) << 6) | d;
|
|
32
|
+
}
|
|
33
|
+
return out.subarray(0, o);
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/** Uint8Array -> base64. */
|
|
37
|
+
function toBase64(bytes) {
|
|
38
|
+
let s = "";
|
|
39
|
+
for (let i = 0; i < bytes.length; i += 3) {
|
|
40
|
+
const a = bytes[i], b = bytes[i + 1], c = bytes[i + 2];
|
|
41
|
+
s += B64[a >> 2] + B64[((a & 3) << 4) | ((b || 0) >> 4)];
|
|
42
|
+
s += i + 1 < bytes.length ? B64[((b & 15) << 2) | ((c || 0) >> 6)] : "=";
|
|
43
|
+
s += i + 2 < bytes.length ? B64[c & 63] : "=";
|
|
44
|
+
}
|
|
45
|
+
return s;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** 16 kHz mono PCM16 WAV (what the server transcribes). */
|
|
49
|
+
function wav(pcm, sampleRate) {
|
|
50
|
+
const sr = sampleRate || SR;
|
|
51
|
+
const out = new Uint8Array(44 + pcm.length);
|
|
52
|
+
const dv = new DataView(out.buffer);
|
|
53
|
+
const tag = (o, t) => { for (let i = 0; i < 4; i++) out[o + i] = t.charCodeAt(i); };
|
|
54
|
+
tag(0, "RIFF"); dv.setUint32(4, 36 + pcm.length, true); tag(8, "WAVE");
|
|
55
|
+
tag(12, "fmt "); dv.setUint32(16, 16, true); dv.setUint16(20, 1, true); dv.setUint16(22, 1, true);
|
|
56
|
+
dv.setUint32(24, sr, true); dv.setUint32(28, sr * 2, true); dv.setUint16(32, 2, true); dv.setUint16(34, 16, true);
|
|
57
|
+
tag(36, "data"); dv.setUint32(40, pcm.length, true);
|
|
58
|
+
out.set(pcm, 44);
|
|
59
|
+
return out;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function concat(parts, total) {
|
|
63
|
+
const out = new Uint8Array(total);
|
|
64
|
+
let o = 0;
|
|
65
|
+
for (const p of parts) { out.set(p, o); o += p.length; }
|
|
66
|
+
return out;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* The default audio adapter for bare React Native apps. Install the three libraries:
|
|
71
|
+
* npm install react-native-live-audio-stream react-native-sound react-native-fs
|
|
72
|
+
* and add the microphone permission (Android RECORD_AUDIO, iOS NSMicrophoneUsageDescription).
|
|
73
|
+
* Expo or another audio stack: pass your own adapter with the same four functions.
|
|
74
|
+
*
|
|
75
|
+
* Adapter contract:
|
|
76
|
+
* startMic(onChunk) -> Promise<boolean> onChunk(Uint8Array PCM16 LE mono 16 kHz)
|
|
77
|
+
* stopMic() -> Promise<void>
|
|
78
|
+
* play(mp3Uint8) -> Promise<void> resolves when it finished or was stopped
|
|
79
|
+
* stopPlayback() -> Promise<void>
|
|
80
|
+
*/
|
|
81
|
+
function createDefaultAudio() {
|
|
82
|
+
let LiveAudioStream, Sound, RNFS, RN;
|
|
83
|
+
try {
|
|
84
|
+
LiveAudioStream = require("react-native-live-audio-stream").default || require("react-native-live-audio-stream");
|
|
85
|
+
Sound = require("react-native-sound");
|
|
86
|
+
RNFS = require("react-native-fs");
|
|
87
|
+
RN = require("react-native");
|
|
88
|
+
} catch (_) {
|
|
89
|
+
return null; // libraries not installed: the chat hides the microphone
|
|
90
|
+
}
|
|
91
|
+
let sub = null;
|
|
92
|
+
let current = null;
|
|
93
|
+
let ended = null;
|
|
94
|
+
return {
|
|
95
|
+
async startMic(onChunk) {
|
|
96
|
+
try {
|
|
97
|
+
if (RN.Platform.OS === "android") {
|
|
98
|
+
const P = RN.PermissionsAndroid;
|
|
99
|
+
const r = await P.request(P.PERMISSIONS.RECORD_AUDIO);
|
|
100
|
+
if (r !== P.RESULTS.GRANTED) return false;
|
|
101
|
+
}
|
|
102
|
+
LiveAudioStream.init({ sampleRate: SR, channels: 1, bitsPerSample: 16, audioSource: 7, bufferSize: 4096, wavFile: "" });
|
|
103
|
+
sub = LiveAudioStream.on("data", (b64) => onChunk(fromBase64(b64)));
|
|
104
|
+
LiveAudioStream.start();
|
|
105
|
+
return true;
|
|
106
|
+
} catch (_) {
|
|
107
|
+
return false;
|
|
108
|
+
}
|
|
109
|
+
},
|
|
110
|
+
async stopMic() {
|
|
111
|
+
try { LiveAudioStream.stop(); } catch (_) { /* ignore */ }
|
|
112
|
+
if (sub && sub.remove) sub.remove();
|
|
113
|
+
sub = null;
|
|
114
|
+
},
|
|
115
|
+
async play(mp3) {
|
|
116
|
+
const file = RNFS.CachesDirectoryPath + "/oryksa_voice_" + Date.now() + ".mp3";
|
|
117
|
+
await RNFS.writeFile(file, toBase64(mp3), "base64");
|
|
118
|
+
Sound.setCategory("PlayAndRecord", true);
|
|
119
|
+
await new Promise((ok) => {
|
|
120
|
+
ended = ok;
|
|
121
|
+
const snd = new Sound(file, "", (err) => {
|
|
122
|
+
if (err) { ok(); return; }
|
|
123
|
+
current = snd;
|
|
124
|
+
snd.play(() => { snd.release(); current = null; ok(); });
|
|
125
|
+
});
|
|
126
|
+
});
|
|
127
|
+
RNFS.unlink(file).catch(() => {});
|
|
128
|
+
},
|
|
129
|
+
async stopPlayback() {
|
|
130
|
+
if (current) { try { current.stop(); current.release(); } catch (_) { /* ignore */ } current = null; }
|
|
131
|
+
if (ended) { ended(); ended = null; }
|
|
132
|
+
},
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* The voice conversation. Phases: starting, listening, hearing, thinking, speaking, muted, micError,
|
|
138
|
+
* noisy, notUnderstood (onPhase is called on every change).
|
|
139
|
+
*/
|
|
140
|
+
class OryksaVoiceController {
|
|
141
|
+
/**
|
|
142
|
+
* @param {{client: object, audio?: object, appContext?: () => object, onUserText?: (t: string) => void,
|
|
143
|
+
* onReply?: (t: string) => void, onPhase?: (phase: string, ctl: OryksaVoiceController) => void}} opts
|
|
144
|
+
*/
|
|
145
|
+
constructor(opts) {
|
|
146
|
+
const o = opts || {};
|
|
147
|
+
this.client = o.client;
|
|
148
|
+
this.audio = o.audio || createDefaultAudio();
|
|
149
|
+
this.appContext = o.appContext || null;
|
|
150
|
+
this.onUserText = o.onUserText || null;
|
|
151
|
+
this.onReply = o.onReply || null;
|
|
152
|
+
this.onPhase = o.onPhase || null;
|
|
153
|
+
this.vad = new Vad(SR);
|
|
154
|
+
this.phase = "starting";
|
|
155
|
+
this.lastReply = "";
|
|
156
|
+
this.lastHeard = "";
|
|
157
|
+
this._pcm = [];
|
|
158
|
+
this._pcmBytes = 0;
|
|
159
|
+
this._before = [];
|
|
160
|
+
this._beforeBytes = 0;
|
|
161
|
+
this._running = false;
|
|
162
|
+
this._closed = false;
|
|
163
|
+
this._speaking = false;
|
|
164
|
+
this._busy = false;
|
|
165
|
+
this._cut = false;
|
|
166
|
+
this._whispered = false;
|
|
167
|
+
this._noise = 0;
|
|
168
|
+
this._turn = 0;
|
|
169
|
+
this._heard = false;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
_set(p) {
|
|
173
|
+
if (this._closed || this.phase === p) return;
|
|
174
|
+
this.phase = p;
|
|
175
|
+
if (this.onPhase) this.onPhase(p, this);
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/** Opens the microphone and starts listening. */
|
|
179
|
+
async start() {
|
|
180
|
+
if (this._closed || this._running || this.phase === "muted") return;
|
|
181
|
+
if (!this.audio) { this._set("micError"); return; }
|
|
182
|
+
const ok = await this.audio.startMic((c) => this._onAudio(c));
|
|
183
|
+
if (!ok) { this._set("micError"); return; }
|
|
184
|
+
this._running = true;
|
|
185
|
+
this.vad.reset();
|
|
186
|
+
this._pcm = []; this._pcmBytes = 0;
|
|
187
|
+
this._heard = false;
|
|
188
|
+
this._set("listening");
|
|
189
|
+
setTimeout(() => { if (!this._closed && !this._heard && this.phase !== "muted") this._set("micError"); }, 4000);
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
_onAudio(chunk) {
|
|
193
|
+
if (this._closed || this.phase === "muted" || !chunk || chunk.length < 2) return;
|
|
194
|
+
if (!this._heard && Vad.rmsOf(chunk) > 0.0005) this._heard = true;
|
|
195
|
+
|
|
196
|
+
// While she SPEAKS the engine only decides whether she was cut off.
|
|
197
|
+
if (this._speaking) {
|
|
198
|
+
this._before.push(chunk);
|
|
199
|
+
this._beforeBytes += chunk.length;
|
|
200
|
+
while (this._beforeBytes > PRE_ROLL_BYTES && this._before.length > 1) this._beforeBytes -= this._before.shift().length;
|
|
201
|
+
if (this.vad.feed(chunk, true).isBargeIn) this._bargeIn();
|
|
202
|
+
return;
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
const step = this.vad.feed(chunk);
|
|
206
|
+
if (step.isBargeIn) { this._bargeIn(); return; }
|
|
207
|
+
if (step.isCapturing || this.vad.capturing) {
|
|
208
|
+
this._pcm.push(chunk);
|
|
209
|
+
this._pcmBytes += chunk.length;
|
|
210
|
+
if (!this._busy) this._set("hearing");
|
|
211
|
+
}
|
|
212
|
+
if (!step.isDone) return;
|
|
213
|
+
|
|
214
|
+
const raw = concat(this._pcm, this._pcmBytes);
|
|
215
|
+
this._pcm = []; this._pcmBytes = 0;
|
|
216
|
+
const whisper = this.vad.foiSussurro;
|
|
217
|
+
const stats = this._stats(false, Math.floor(raw.length / 32));
|
|
218
|
+
this.vad.newTurn();
|
|
219
|
+
if (step.enough && raw.length > SR / 2) {
|
|
220
|
+
this._noise = 0;
|
|
221
|
+
this._handle(wav(raw), whisper, stats);
|
|
222
|
+
} else if (step.isNoise) {
|
|
223
|
+
if (++this._noise >= 2) this._set("noisy");
|
|
224
|
+
} else if (!this._busy) {
|
|
225
|
+
this._set("listening");
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
/** The customer cut her off: stop the voice on the word, keep what he says. */
|
|
230
|
+
_bargeIn() {
|
|
231
|
+
this._turn++;
|
|
232
|
+
this._speaking = false;
|
|
233
|
+
this._busy = false;
|
|
234
|
+
if (this.audio) this.audio.stopPlayback();
|
|
235
|
+
this.vad.newTurn();
|
|
236
|
+
this.vad.capturing = true;
|
|
237
|
+
this._pcm = this._before.slice();
|
|
238
|
+
this._pcmBytes = this._beforeBytes;
|
|
239
|
+
this._before = []; this._beforeBytes = 0;
|
|
240
|
+
this._cut = true;
|
|
241
|
+
this._set("hearing");
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
/** Sends what was said right away (tap on the picture). */
|
|
245
|
+
sendNow() {
|
|
246
|
+
if (this._busy || this._speaking || !this.vad.capturing) return;
|
|
247
|
+
const raw = concat(this._pcm, this._pcmBytes);
|
|
248
|
+
const whisper = this.vad.foiSussurro;
|
|
249
|
+
const stats = this._stats(false, Math.floor(raw.length / 32));
|
|
250
|
+
this.vad.newTurn();
|
|
251
|
+
this._pcm = []; this._pcmBytes = 0;
|
|
252
|
+
if (raw.length > SR / 2) this._handle(wav(raw), whisper, stats);
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
/** Audio numbers of this turn, in the ORYKSA ecosystem format (quality report). */
|
|
256
|
+
_stats(empty, durMs) {
|
|
257
|
+
const r3 = (v) => Math.round(v * 1000) / 1000;
|
|
258
|
+
return { channel: "sdk_rn", peak: r3(this.vad.picoDaFrase), pitch: r3(this.vad.tomDaFrase), noise: r3(this.vad.base),
|
|
259
|
+
floor: Vad.bargeFloor, cut: false, whisper: this.vad.foiSussurro, self_cut: this._cut, empty, dur_ms: durMs };
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
async _handle(wavBytes, whisper, stats) {
|
|
263
|
+
const my = ++this._turn;
|
|
264
|
+
this._whispered = whisper;
|
|
265
|
+
this._busy = true;
|
|
266
|
+
this._set("thinking");
|
|
267
|
+
const text = await this.client.transcribe(wavBytes);
|
|
268
|
+
if (this._closed || my !== this._turn) return;
|
|
269
|
+
if (!text) {
|
|
270
|
+
await this.client.voiceStats(Object.assign({}, stats, { empty: true }));
|
|
271
|
+
this._busy = false;
|
|
272
|
+
this._set(text === null ? "notUnderstood" : "listening");
|
|
273
|
+
return;
|
|
274
|
+
}
|
|
275
|
+
this.lastHeard = text;
|
|
276
|
+
if (this.onUserText) this.onUserText(text);
|
|
277
|
+
let r = null;
|
|
278
|
+
try {
|
|
279
|
+
r = await this.client.sendAndWaitReply(text, { appContext: this.appContext ? this.appContext() : undefined, voice: true, whisper, voiceStats: stats });
|
|
280
|
+
} catch (_) { r = null; }
|
|
281
|
+
if (this._closed || my !== this._turn) return;
|
|
282
|
+
const reply = String((r && r.reply) || "").trim();
|
|
283
|
+
if (!reply) { this._busy = false; this._set("notUnderstood"); return; }
|
|
284
|
+
this.lastReply = reply;
|
|
285
|
+
if (this.onReply) this.onReply(reply);
|
|
286
|
+
const speech = String((r && r.speech) || "").trim();
|
|
287
|
+
await this._speak(speech || reply, my);
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
/** Speaks text with the AI's voice (for example a greeting). If the voice fails she stays silent. */
|
|
291
|
+
say(text) { return this._speak(text, ++this._turn); }
|
|
292
|
+
|
|
293
|
+
async _speak(text, my) {
|
|
294
|
+
this._before = []; this._beforeBytes = 0;
|
|
295
|
+
const done = () => {
|
|
296
|
+
this._cut = false;
|
|
297
|
+
this._speaking = false;
|
|
298
|
+
this._busy = false;
|
|
299
|
+
this.vad.newTurn();
|
|
300
|
+
if (!this._closed && my === this._turn) this._set("listening");
|
|
301
|
+
};
|
|
302
|
+
const mp3 = await this.client.tts(text, { whisper: this._whispered });
|
|
303
|
+
if (this._closed || my !== this._turn || !mp3 || !this.audio) { done(); return; } // never a robot voice
|
|
304
|
+
this._speaking = true;
|
|
305
|
+
this._set("speaking");
|
|
306
|
+
try { await this.audio.play(mp3); } catch (_) { /* silence + text */ }
|
|
307
|
+
if (my === this._turn) done();
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
/** Mute: she stops talking and listening. Nothing is lost: what she said stays in the chat. */
|
|
311
|
+
async toggleMute() {
|
|
312
|
+
if (this.phase === "muted") {
|
|
313
|
+
this.phase = "starting";
|
|
314
|
+
if (this.onPhase) this.onPhase("starting", this);
|
|
315
|
+
await this.start();
|
|
316
|
+
return;
|
|
317
|
+
}
|
|
318
|
+
this._turn++;
|
|
319
|
+
this._speaking = false;
|
|
320
|
+
this._busy = false;
|
|
321
|
+
if (this.audio) { await this.audio.stopPlayback(); await this.audio.stopMic(); }
|
|
322
|
+
this._running = false;
|
|
323
|
+
this.vad.newTurn();
|
|
324
|
+
this.phase = "muted";
|
|
325
|
+
if (this.onPhase) this.onPhase("muted", this);
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
/** Closes the microphone and the player. */
|
|
329
|
+
async close() {
|
|
330
|
+
if (this._closed) return;
|
|
331
|
+
this._closed = true;
|
|
332
|
+
this._turn++;
|
|
333
|
+
if (this.audio) { await this.audio.stopPlayback(); await this.audio.stopMic(); }
|
|
334
|
+
this._running = false;
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
module.exports = { OryksaVoiceController, createDefaultAudio, wav, fromBase64, toBase64 };
|