@memori.ai/memori-react 7.16.2 → 7.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -0
- package/dist/components/Avatar/AvatarView/AvatarComponent/lights/Lights.d.ts +27 -0
- package/dist/components/Avatar/AvatarView/AvatarComponent/lights/Lights.js +52 -0
- package/dist/components/Avatar/AvatarView/AvatarComponent/lights/Lights.js.map +1 -0
- package/dist/components/Avatar/AvatarView/AvatarComponent/positionControls/positionControls.css +19 -7
- package/dist/components/Avatar/AvatarView/AvatarComponent/positionControls/positionControls.js +7 -7
- package/dist/components/Avatar/AvatarView/AvatarComponent/positionControls/positionControls.js.map +1 -1
- package/dist/components/Avatar/AvatarView/index.js +2 -3
- package/dist/components/Avatar/AvatarView/index.js.map +1 -1
- package/dist/components/ChatTextArea/ChatTextArea.css +55 -60
- package/dist/components/MemoriWidget/MemoriWidget.js +215 -138
- package/dist/components/MemoriWidget/MemoriWidget.js.map +1 -1
- package/dist/components/SettingsDrawer/SettingsDrawer.css +5 -0
- package/dist/components/SettingsDrawer/SettingsDrawer.d.ts +2 -1
- package/dist/components/SettingsDrawer/SettingsDrawer.js +6 -3
- package/dist/components/SettingsDrawer/SettingsDrawer.js.map +1 -1
- package/dist/components/UploadButton/UploadButton.d.ts +5 -0
- package/dist/components/UploadButton/UploadButton.js +49 -48
- package/dist/components/UploadButton/UploadButton.js.map +1 -1
- package/dist/components/ui/Slider.css +59 -44
- package/dist/context/visemeContext.d.ts +1 -1
- package/dist/context/visemeContext.js +2 -2
- package/dist/context/visemeContext.js.map +1 -1
- package/dist/locales/de.json +1 -0
- package/dist/locales/en.json +1 -0
- package/dist/locales/es.json +1 -0
- package/dist/locales/fr.json +1 -0
- package/dist/locales/it.json +1 -0
- package/esm/components/Avatar/AvatarView/AvatarComponent/lights/Lights.d.ts +27 -0
- package/esm/components/Avatar/AvatarView/AvatarComponent/lights/Lights.js +48 -0
- package/esm/components/Avatar/AvatarView/AvatarComponent/lights/Lights.js.map +1 -0
- package/esm/components/Avatar/AvatarView/AvatarComponent/positionControls/positionControls.css +19 -7
- package/esm/components/Avatar/AvatarView/AvatarComponent/positionControls/positionControls.js +7 -7
- package/esm/components/Avatar/AvatarView/AvatarComponent/positionControls/positionControls.js.map +1 -1
- package/esm/components/Avatar/AvatarView/index.js +3 -4
- package/esm/components/Avatar/AvatarView/index.js.map +1 -1
- package/esm/components/ChatTextArea/ChatTextArea.css +55 -60
- package/esm/components/MemoriWidget/MemoriWidget.js +216 -139
- package/esm/components/MemoriWidget/MemoriWidget.js.map +1 -1
- package/esm/components/SettingsDrawer/SettingsDrawer.css +5 -0
- package/esm/components/SettingsDrawer/SettingsDrawer.d.ts +2 -1
- package/esm/components/SettingsDrawer/SettingsDrawer.js +6 -3
- package/esm/components/SettingsDrawer/SettingsDrawer.js.map +1 -1
- package/esm/components/UploadButton/UploadButton.d.ts +5 -0
- package/esm/components/UploadButton/UploadButton.js +50 -49
- package/esm/components/UploadButton/UploadButton.js.map +1 -1
- package/esm/components/ui/Slider.css +59 -44
- package/esm/context/visemeContext.d.ts +1 -1
- package/esm/context/visemeContext.js +2 -2
- package/esm/context/visemeContext.js.map +1 -1
- package/esm/locales/de.json +1 -0
- package/esm/locales/en.json +1 -0
- package/esm/locales/es.json +1 -0
- package/esm/locales/fr.json +1 -0
- package/esm/locales/it.json +1 -0
- package/package.json +1 -2
- package/src/components/Avatar/AvatarView/AvatarComponent/lights/Lights.tsx +145 -0
- package/src/components/Avatar/AvatarView/AvatarComponent/positionControls/positionControls.css +19 -7
- package/src/components/Avatar/AvatarView/AvatarComponent/positionControls/positionControls.tsx +6 -14
- package/src/components/Avatar/AvatarView/index.tsx +5 -14
- package/src/components/ChatTextArea/ChatTextArea.css +55 -60
- package/src/components/MemoriWidget/MemoriWidget.tsx +337 -187
- package/src/components/SettingsDrawer/SettingsDrawer.css +5 -0
- package/src/components/SettingsDrawer/SettingsDrawer.tsx +29 -11
- package/src/components/UploadButton/UploadButton.tsx +139 -118
- package/src/components/UploadButton/__snapshots__/UploadButton.test.tsx.snap +3 -52
- package/src/components/ui/Slider.css +59 -44
- package/src/context/visemeContext.tsx +2 -2
- package/src/locales/de.json +1 -0
- package/src/locales/en.json +1 -0
- package/src/locales/es.json +1 -0
- package/src/locales/fr.json +1 -0
- package/src/locales/it.json +1 -0
|
@@ -2,7 +2,7 @@ import { jsx as _jsx, jsxs as _jsxs } from "react/jsx-runtime";
|
|
|
2
2
|
import { useState, useEffect, useCallback, useRef, } from 'react';
|
|
3
3
|
import { useTranslation } from 'react-i18next';
|
|
4
4
|
import memoriApiClient from '@memori.ai/memori-api-client';
|
|
5
|
-
import { AudioContext } from 'standardized-audio-context';
|
|
5
|
+
import { AudioContext, } from 'standardized-audio-context';
|
|
6
6
|
import * as speechSdk from 'microsoft-cognitiveservices-speech-sdk';
|
|
7
7
|
import cx from 'classnames';
|
|
8
8
|
import { DateTime } from 'luxon';
|
|
@@ -231,7 +231,6 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
|
|
|
231
231
|
const [hideEmissions, setHideEmissions] = useState(false);
|
|
232
232
|
const { startProcessing, setAudioContext, addViseme, stopProcessing, resetVisemeQueue, } = useViseme();
|
|
233
233
|
useEffect(() => {
|
|
234
|
-
setIsPlayingAudio(!!speechSynthesizer);
|
|
235
234
|
memoriSpeaking = !!speechSynthesizer;
|
|
236
235
|
}, [speechSynthesizer]);
|
|
237
236
|
useEffect(() => {
|
|
@@ -248,12 +247,11 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
|
|
|
248
247
|
else {
|
|
249
248
|
defaultControlsPosition = 'bottom';
|
|
250
249
|
}
|
|
251
|
-
|
|
252
|
-
getLocalConfig('muteSpeaker', !defaultEnableAudio || !defaultSpeakerActive || autoStart)
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
setContinuousSpeech(microphoneMode === 'CONTINUOUS');
|
|
250
|
+
const muteSpeaker = autoStart ||
|
|
251
|
+
getLocalConfig('muteSpeaker', !defaultEnableAudio || !defaultSpeakerActive || autoStart);
|
|
252
|
+
setMuteSpeaker(muteSpeaker);
|
|
253
|
+
speakerMuted = muteSpeaker;
|
|
254
|
+
setContinuousSpeech(muteSpeaker ? false : microphoneMode === 'CONTINUOUS');
|
|
257
255
|
setContinuousSpeechTimeout(getLocalConfig('continuousSpeechTimeout', 2));
|
|
258
256
|
setControlsPosition(getLocalConfig('controlsPosition', defaultControlsPosition));
|
|
259
257
|
setAvatarType(getLocalConfig('avatarType', 'avatar3d'));
|
|
@@ -1184,32 +1182,44 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
|
|
|
1184
1182
|
document.dispatchEvent(e);
|
|
1185
1183
|
};
|
|
1186
1184
|
const speak = (text) => {
|
|
1185
|
+
console.debug('speak called with text:', text);
|
|
1187
1186
|
if (!AZURE_COGNITIVE_SERVICES_TTS_KEY || preview) {
|
|
1187
|
+
console.debug('No TTS key or preview mode, emitting end speak event');
|
|
1188
1188
|
emitEndSpeakEvent();
|
|
1189
1189
|
return;
|
|
1190
1190
|
}
|
|
1191
|
+
console.debug('Stopping listening before speaking');
|
|
1191
1192
|
stopListening();
|
|
1192
|
-
if (preview)
|
|
1193
|
+
if (preview) {
|
|
1194
|
+
console.debug('Preview mode, returning early');
|
|
1193
1195
|
return;
|
|
1196
|
+
}
|
|
1194
1197
|
if (speakerMuted) {
|
|
1198
|
+
console.debug('Speaker muted, skipping speech synthesis');
|
|
1195
1199
|
memoriSpeaking = false;
|
|
1196
1200
|
setMemoriTyping(false);
|
|
1197
1201
|
emitEndSpeakEvent();
|
|
1198
1202
|
if (continuousSpeech) {
|
|
1203
|
+
console.debug('Setting listening timeout for continuous speech');
|
|
1199
1204
|
setListeningTimeout();
|
|
1200
1205
|
}
|
|
1201
1206
|
return;
|
|
1202
1207
|
}
|
|
1203
|
-
if (audioDestination)
|
|
1208
|
+
if (audioDestination) {
|
|
1209
|
+
console.debug('Pausing existing audio destination');
|
|
1204
1210
|
audioDestination.pause();
|
|
1211
|
+
}
|
|
1205
1212
|
let isSafari = window.navigator.userAgent.includes('Safari') &&
|
|
1206
1213
|
!window.navigator.userAgent.includes('Chrome');
|
|
1207
1214
|
let isIOS = /iPad|iPhone|iPod/.test(navigator.userAgent);
|
|
1215
|
+
console.debug('Browser detection - Safari:', isSafari, 'iOS:', isIOS);
|
|
1208
1216
|
if (audioContext.state === 'interrupted') {
|
|
1217
|
+
console.debug('Audio context interrupted, attempting resume');
|
|
1209
1218
|
audioContext.resume().then(() => speak(text));
|
|
1210
1219
|
return;
|
|
1211
1220
|
}
|
|
1212
1221
|
if (audioContext.state === 'closed') {
|
|
1222
|
+
console.debug('Audio context closed, creating new context');
|
|
1213
1223
|
audioContext = new AudioContext();
|
|
1214
1224
|
let buffer = audioContext.createBuffer(1, 10000, 22050);
|
|
1215
1225
|
let source = audioContext.createBufferSource();
|
|
@@ -1217,6 +1227,7 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
|
|
|
1217
1227
|
source.connect(audioContext.destination);
|
|
1218
1228
|
}
|
|
1219
1229
|
else if (audioContext.state === 'suspended') {
|
|
1230
|
+
console.debug('Audio context suspended, stopping audio and creating new context');
|
|
1220
1231
|
stopAudio();
|
|
1221
1232
|
audioContext = new AudioContext();
|
|
1222
1233
|
let buffer = audioContext.createBuffer(1, 10000, 22050);
|
|
@@ -1225,77 +1236,91 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
|
|
|
1225
1236
|
source.connect(audioContext.destination);
|
|
1226
1237
|
}
|
|
1227
1238
|
if (!speechSynthesizer) {
|
|
1228
|
-
|
|
1229
|
-
audioDestination = new speechSdk.SpeakerAudioDestination();
|
|
1230
|
-
}
|
|
1231
|
-
let audioConfig = speechSdk.AudioConfig.fromSpeakerOutput(audioDestination);
|
|
1232
|
-
speechSynthesizer = new speechSdk.SpeechSynthesizer(speechConfig, audioConfig);
|
|
1239
|
+
initializeTTS();
|
|
1233
1240
|
}
|
|
1234
1241
|
const source = audioContext.createBufferSource();
|
|
1235
1242
|
source.addEventListener('ended', () => {
|
|
1243
|
+
console.debug('Audio source ended');
|
|
1236
1244
|
setIsPlayingAudio(false);
|
|
1237
1245
|
memoriSpeaking = false;
|
|
1238
1246
|
});
|
|
1239
1247
|
audioDestination.onAudioEnd = () => {
|
|
1248
|
+
console.debug('Audio destination ended');
|
|
1240
1249
|
setIsPlayingAudio(false);
|
|
1241
1250
|
memoriSpeaking = false;
|
|
1242
1251
|
source.disconnect();
|
|
1243
1252
|
emitEndSpeakEvent();
|
|
1244
1253
|
onEndSpeakStartListen();
|
|
1245
1254
|
};
|
|
1255
|
+
console.debug('Resetting viseme queue');
|
|
1246
1256
|
resetVisemeQueue();
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1257
|
+
if (speechSynthesizer) {
|
|
1258
|
+
speechSynthesizer.visemeReceived = function (_, e) {
|
|
1259
|
+
console.debug('Viseme received:', e.visemeId, 'at offset:', e.audioOffset);
|
|
1260
|
+
addViseme(e.visemeId, e.audioOffset);
|
|
1261
|
+
};
|
|
1262
|
+
}
|
|
1250
1263
|
const textToSpeak = escapeHTML(stripMarkdown(stripEmojis(stripHTML(stripOutputTags(text)))));
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1264
|
+
console.debug('Processed text to speak:', textToSpeak);
|
|
1265
|
+
setTimeout(() => {
|
|
1266
|
+
if (speechSynthesizer) {
|
|
1267
|
+
console.debug('Starting speech synthesis');
|
|
1268
|
+
speechSynthesizer.speakSsmlAsync(`<speak version="1.0" xmlns="http://www.w3.org/2001/10/synthesis" xmlns:mstts="https://www.w3.org/2001/mstts" xmlns:emo="http://www.w3.org/2009/10/emotionml" xml:lang="${getCultureCodeByLanguage(userLang)}"><voice name="${getTTSVoice(userLang)}"><s>${replaceTextWithPhonemes(textToSpeak, userLang.toLowerCase())}</s></voice></speak>`, result => {
|
|
1269
|
+
if (result) {
|
|
1270
|
+
console.debug('Speech synthesis successful');
|
|
1271
|
+
setIsPlayingAudio(true);
|
|
1272
|
+
memoriSpeaking = true;
|
|
1273
|
+
startProcessing(audioContext);
|
|
1274
|
+
try {
|
|
1275
|
+
console.debug('Decoding audio data');
|
|
1276
|
+
audioContext.decodeAudioData(result.audioData, function (buffer) {
|
|
1277
|
+
console.debug('Audio data decoded successfully');
|
|
1278
|
+
source.buffer = buffer;
|
|
1279
|
+
source.connect(audioContext.destination);
|
|
1280
|
+
if (history.length < 1 || (isSafari && isIOS)) {
|
|
1281
|
+
console.debug('Starting audio playback');
|
|
1282
|
+
source.start(0);
|
|
1283
|
+
}
|
|
1284
|
+
});
|
|
1285
|
+
audioContext.onstatechange = () => {
|
|
1286
|
+
console.debug('Audio context state changed to:', audioContext.state);
|
|
1287
|
+
if (audioContext.state === 'suspended' ||
|
|
1288
|
+
audioContext.state === 'closed') {
|
|
1289
|
+
source.disconnect();
|
|
1290
|
+
setIsPlayingAudio(false);
|
|
1291
|
+
stopProcessing();
|
|
1292
|
+
resetVisemeQueue();
|
|
1293
|
+
memoriSpeaking = false;
|
|
1294
|
+
}
|
|
1295
|
+
else if (audioContext.state === 'interrupted') {
|
|
1296
|
+
audioContext.resume();
|
|
1297
|
+
}
|
|
1298
|
+
};
|
|
1274
1299
|
audioContext.resume();
|
|
1300
|
+
if (speechSynthesizer) {
|
|
1301
|
+
console.debug('Closing speech synthesizer');
|
|
1302
|
+
speechSynthesizer.close();
|
|
1303
|
+
speechSynthesizer = null;
|
|
1304
|
+
}
|
|
1305
|
+
}
|
|
1306
|
+
catch (error) {
|
|
1307
|
+
console.error('Error processing audio data:', error);
|
|
1308
|
+
handleFallback(text);
|
|
1275
1309
|
}
|
|
1276
|
-
};
|
|
1277
|
-
audioContext.resume();
|
|
1278
|
-
if (speechSynthesizer) {
|
|
1279
|
-
speechSynthesizer.close();
|
|
1280
|
-
speechSynthesizer = null;
|
|
1281
1310
|
}
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1311
|
+
else {
|
|
1312
|
+
console.debug('No result from speech synthesis, using fallback');
|
|
1313
|
+
handleFallback(text);
|
|
1314
|
+
}
|
|
1315
|
+
}, error => {
|
|
1316
|
+
console.error('Speak error:', error);
|
|
1285
1317
|
handleFallback(text);
|
|
1286
|
-
}
|
|
1287
|
-
}
|
|
1288
|
-
else {
|
|
1289
|
-
handleFallback(text);
|
|
1318
|
+
});
|
|
1290
1319
|
}
|
|
1291
|
-
},
|
|
1292
|
-
console.error('Speak error:', error);
|
|
1293
|
-
handleFallback(text);
|
|
1294
|
-
});
|
|
1320
|
+
}, 100);
|
|
1295
1321
|
setMemoriTyping(false);
|
|
1296
1322
|
};
|
|
1297
1323
|
const handleFallback = (text) => {
|
|
1298
|
-
console.log('Falling back to browser speech synthesis');
|
|
1299
1324
|
window.speechSynthesis.speak(new SpeechSynthesisUtterance(text));
|
|
1300
1325
|
cleanup();
|
|
1301
1326
|
};
|
|
@@ -1304,22 +1329,35 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
|
|
|
1304
1329
|
stopProcessing();
|
|
1305
1330
|
resetVisemeQueue();
|
|
1306
1331
|
memoriSpeaking = false;
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1332
|
+
try {
|
|
1333
|
+
if (speechSynthesizer) {
|
|
1334
|
+
const currentSynthesizer = speechSynthesizer;
|
|
1335
|
+
speechSynthesizer = null;
|
|
1336
|
+
console.debug('Closing speech synthesizer');
|
|
1337
|
+
currentSynthesizer.close();
|
|
1338
|
+
}
|
|
1339
|
+
}
|
|
1340
|
+
catch (error) {
|
|
1341
|
+
console.debug('Error during synthesizer cleanup:', error);
|
|
1310
1342
|
speechSynthesizer = null;
|
|
1311
1343
|
}
|
|
1312
1344
|
emitEndSpeakEvent();
|
|
1313
1345
|
};
|
|
1314
|
-
const stopAudio = () => {
|
|
1346
|
+
const stopAudio = async () => {
|
|
1315
1347
|
setIsPlayingAudio(false);
|
|
1316
1348
|
memoriSpeaking = false;
|
|
1317
1349
|
try {
|
|
1318
1350
|
if (speechSynthesizer) {
|
|
1319
|
-
speechSynthesizer
|
|
1351
|
+
const currentSynthesizer = speechSynthesizer;
|
|
1320
1352
|
speechSynthesizer = null;
|
|
1353
|
+
try {
|
|
1354
|
+
currentSynthesizer.close();
|
|
1355
|
+
}
|
|
1356
|
+
catch (e) {
|
|
1357
|
+
console.debug('Error closing speech synthesizer:', e);
|
|
1358
|
+
}
|
|
1321
1359
|
}
|
|
1322
|
-
if (audioContext.state !== 'closed') {
|
|
1360
|
+
if ((audioContext === null || audioContext === void 0 ? void 0 : audioContext.state) !== 'closed') {
|
|
1323
1361
|
audioContext.close();
|
|
1324
1362
|
}
|
|
1325
1363
|
if (audioDestination) {
|
|
@@ -1346,21 +1384,14 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
|
|
|
1346
1384
|
}
|
|
1347
1385
|
}, [currentDialogState === null || currentDialogState === void 0 ? void 0 : currentDialogState.emission]);
|
|
1348
1386
|
const [transcript, setTranscript] = useState('');
|
|
1349
|
-
const resetTranscript = () => setTranscript('');
|
|
1350
1387
|
const [transcriptTimeout, setTranscriptTimeout] = useState(null);
|
|
1388
|
+
const [isSpeaking, setIsSpeaking] = useState(false);
|
|
1389
|
+
const resetTranscript = () => {
|
|
1390
|
+
setTranscript('');
|
|
1391
|
+
};
|
|
1351
1392
|
const setListeningTimeout = () => {
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
const message = stripDuplicates(transcript);
|
|
1355
|
-
if (message.length > 0 && listening) {
|
|
1356
|
-
sendMessage(message);
|
|
1357
|
-
resetTranscript();
|
|
1358
|
-
setUserMessage('');
|
|
1359
|
-
}
|
|
1360
|
-
else if (listening) {
|
|
1361
|
-
resetInteractionTimeout();
|
|
1362
|
-
}
|
|
1363
|
-
}, continuousSpeechTimeout * 1000);
|
|
1393
|
+
clearListeningTimeout();
|
|
1394
|
+
const timeout = setTimeout(handleTranscriptProcessing, continuousSpeechTimeout * 1000 + 300);
|
|
1364
1395
|
setTranscriptTimeout(timeout);
|
|
1365
1396
|
};
|
|
1366
1397
|
const clearListeningTimeout = () => {
|
|
@@ -1371,74 +1402,105 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
|
|
|
1371
1402
|
};
|
|
1372
1403
|
const resetListeningTimeout = () => {
|
|
1373
1404
|
clearListeningTimeout();
|
|
1374
|
-
if (continuousSpeech)
|
|
1405
|
+
if (continuousSpeech) {
|
|
1375
1406
|
setListeningTimeout();
|
|
1407
|
+
}
|
|
1376
1408
|
};
|
|
1377
1409
|
useEffect(() => {
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1410
|
+
if (!isSpeaking) {
|
|
1411
|
+
resetListeningTimeout();
|
|
1412
|
+
resetInteractionTimeout();
|
|
1413
|
+
}
|
|
1414
|
+
}, [transcript, isSpeaking]);
|
|
1415
|
+
useEffect(() => {
|
|
1416
|
+
return () => {
|
|
1417
|
+
clearListeningTimeout();
|
|
1418
|
+
};
|
|
1419
|
+
}, []);
|
|
1381
1420
|
const startListening = async () => {
|
|
1382
|
-
if (!AZURE_COGNITIVE_SERVICES_TTS_KEY)
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1421
|
+
if (!AZURE_COGNITIVE_SERVICES_TTS_KEY) {
|
|
1422
|
+
throw new Error('No TTS key available');
|
|
1423
|
+
}
|
|
1424
|
+
if (!sessionId) {
|
|
1425
|
+
throw new Error('No session ID available');
|
|
1426
|
+
}
|
|
1427
|
+
cleanup();
|
|
1386
1428
|
resetTranscript();
|
|
1387
|
-
if (hasTouchscreen())
|
|
1388
|
-
setEnableFocusChatInput(false);
|
|
1389
1429
|
try {
|
|
1390
|
-
navigator.mediaDevices
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
|
|
1403
|
-
recognizer = new speechSdk.SpeechRecognizer(speechConfig, audioConfig);
|
|
1404
|
-
setListening(true);
|
|
1405
|
-
recognizer.recognized = (_s, e) => {
|
|
1406
|
-
if (!e.result.text)
|
|
1407
|
-
return;
|
|
1408
|
-
if (e.result.reason === speechSdk.ResultReason.RecognizedSpeech) {
|
|
1409
|
-
let transcript = e.result.text;
|
|
1410
|
-
setTranscript(transcript || '');
|
|
1411
|
-
if ((transcript === null || transcript === void 0 ? void 0 : transcript.length) > 0) {
|
|
1412
|
-
const transcriptMessage = stripDuplicates(transcript);
|
|
1413
|
-
if (transcriptMessage.length > 0)
|
|
1414
|
-
setUserMessage(msg => `${msg} ${transcriptMessage}`);
|
|
1415
|
-
}
|
|
1416
|
-
}
|
|
1417
|
-
else if (e.result.reason === speechSdk.ResultReason.NoMatch) {
|
|
1418
|
-
console.debug('NOMATCH: Speech could not be recognized.');
|
|
1419
|
-
}
|
|
1420
|
-
};
|
|
1421
|
-
recognizer.canceled = (_s, e) => {
|
|
1422
|
-
if (e.reason === speechSdk.CancellationReason.Error) {
|
|
1423
|
-
console.debug(`"CANCELED: ErrorCode=${e.errorCode}`);
|
|
1424
|
-
console.debug(`"CANCELED: ErrorDetails=${e.errorDetails}`);
|
|
1425
|
-
console.debug('CANCELED: Did you set the speech resource key and region values?');
|
|
1426
|
-
}
|
|
1430
|
+
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
|
1431
|
+
setHasUserActivatedListening(true);
|
|
1432
|
+
speechConfig = setupSpeechConfig(AZURE_COGNITIVE_SERVICES_TTS_KEY);
|
|
1433
|
+
const audioConfig = speechSdk.AudioConfig.fromDefaultMicrophoneInput();
|
|
1434
|
+
recognizer = new speechSdk.SpeechRecognizer(speechConfig, audioConfig);
|
|
1435
|
+
setupRecognizerHandlers(recognizer);
|
|
1436
|
+
setListening(true);
|
|
1437
|
+
recognizer.startContinuousRecognitionAsync();
|
|
1438
|
+
recognizer.canceled = (_s, e) => {
|
|
1439
|
+
if (e.reason === speechSdk.CancellationReason.Error) {
|
|
1440
|
+
console.debug(`"CANCELED: ErrorCode=${e.errorCode}`);
|
|
1441
|
+
console.debug(`"CANCELED: ErrorDetails=${e.errorDetails}`);
|
|
1442
|
+
console.debug('CANCELED: Did you set the speech resource key and region values?');
|
|
1427
1443
|
stopListening();
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1444
|
+
cleanup();
|
|
1445
|
+
}
|
|
1446
|
+
stopListening();
|
|
1447
|
+
};
|
|
1448
|
+
recognizer.sessionStopped = (_s, _e) => {
|
|
1449
|
+
stopListening();
|
|
1432
1450
|
resetTranscript();
|
|
1433
|
-
|
|
1434
|
-
})
|
|
1435
|
-
.catch(console.debug);
|
|
1451
|
+
};
|
|
1436
1452
|
}
|
|
1437
1453
|
catch (error) {
|
|
1438
|
-
console.
|
|
1454
|
+
console.error('Error in startListening:', error);
|
|
1455
|
+
stopListening();
|
|
1456
|
+
throw error;
|
|
1457
|
+
}
|
|
1458
|
+
};
|
|
1459
|
+
const setupSpeechConfig = (AZURE_COGNITIVE_SERVICES_TTS_KEY) => {
|
|
1460
|
+
speechConfig = speechSdk.SpeechConfig.fromSubscription(AZURE_COGNITIVE_SERVICES_TTS_KEY, 'westeurope');
|
|
1461
|
+
speechConfig.speechRecognitionLanguage = getCultureCodeByLanguage(userLang);
|
|
1462
|
+
speechConfig.speechSynthesisLanguage = getCultureCodeByLanguage(userLang);
|
|
1463
|
+
speechConfig.speechSynthesisVoiceName = getTTSVoice(userLang);
|
|
1464
|
+
return speechConfig;
|
|
1465
|
+
};
|
|
1466
|
+
const setupRecognizerHandlers = (recognizer) => {
|
|
1467
|
+
if (recognizer) {
|
|
1468
|
+
recognizer.recognized = (_, event) => {
|
|
1469
|
+
handleRecognizedSpeech(event.result.text);
|
|
1470
|
+
};
|
|
1471
|
+
recognizer.properties.setProperty('SpeechServiceResponse_JsonResult', 'true');
|
|
1472
|
+
recognizer.properties.setProperty('SpeechServiceConnection_NoiseSuppression', 'true');
|
|
1473
|
+
recognizer.properties.setProperty('SpeechServiceConnection_SNRThresholdDb', '10.0');
|
|
1474
|
+
}
|
|
1475
|
+
};
|
|
1476
|
+
const handleRecognizedSpeech = (text) => {
|
|
1477
|
+
console.debug('Handling recognized speech:', text);
|
|
1478
|
+
if (!text || text.trim().length === 0) {
|
|
1479
|
+
console.debug('No valid text received from speech recognition');
|
|
1480
|
+
return;
|
|
1481
|
+
}
|
|
1482
|
+
setTranscript(text);
|
|
1483
|
+
setIsSpeaking(false);
|
|
1484
|
+
const message = stripDuplicates(text);
|
|
1485
|
+
console.debug('Stripped message:', message);
|
|
1486
|
+
if (message.length > 0) {
|
|
1487
|
+
setUserMessage(message);
|
|
1488
|
+
}
|
|
1489
|
+
};
|
|
1490
|
+
const handleTranscriptProcessing = () => {
|
|
1491
|
+
const message = stripDuplicates(transcript);
|
|
1492
|
+
if (message.length > 0 && listening) {
|
|
1493
|
+
sendMessage(message);
|
|
1494
|
+
resetTranscript();
|
|
1495
|
+
setUserMessage('');
|
|
1496
|
+
clearListening();
|
|
1497
|
+
}
|
|
1498
|
+
else if (listening) {
|
|
1499
|
+
resetInteractionTimeout();
|
|
1439
1500
|
}
|
|
1440
1501
|
};
|
|
1441
1502
|
const stopListening = () => {
|
|
1503
|
+
console.debug('Stopping speech recognition');
|
|
1442
1504
|
if (recognizer) {
|
|
1443
1505
|
recognizer.stopContinuousRecognitionAsync();
|
|
1444
1506
|
recognizer.close();
|
|
@@ -1447,9 +1509,9 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
|
|
|
1447
1509
|
setListening(false);
|
|
1448
1510
|
};
|
|
1449
1511
|
const clearListening = () => {
|
|
1450
|
-
setHasUserActivatedListening(false);
|
|
1451
1512
|
stopListening();
|
|
1452
1513
|
clearListeningTimeout();
|
|
1514
|
+
setIsSpeaking(false);
|
|
1453
1515
|
};
|
|
1454
1516
|
const resetListening = () => {
|
|
1455
1517
|
if (listening) {
|
|
@@ -1499,15 +1561,16 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
|
|
|
1499
1561
|
}
|
|
1500
1562
|
}, [continuousSpeech, hasUserActivatedListening]);
|
|
1501
1563
|
useEffect(() => {
|
|
1502
|
-
if (
|
|
1503
|
-
!isPlayingAudio &&
|
|
1564
|
+
if (!isPlayingAudio &&
|
|
1504
1565
|
continuousSpeech &&
|
|
1505
|
-
(hasUserActivatedListening || !requestedListening)
|
|
1566
|
+
(hasUserActivatedListening || !requestedListening) &&
|
|
1567
|
+
sessionId) {
|
|
1506
1568
|
startListening();
|
|
1569
|
+
}
|
|
1507
1570
|
else if (isPlayingAudio && listening) {
|
|
1508
1571
|
stopListening();
|
|
1509
1572
|
}
|
|
1510
|
-
}, [isPlayingAudio]);
|
|
1573
|
+
}, [isPlayingAudio, hasUserActivatedListening]);
|
|
1511
1574
|
useEffect(() => {
|
|
1512
1575
|
resetListening();
|
|
1513
1576
|
}, [language]);
|
|
@@ -1668,7 +1731,9 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
|
|
|
1668
1731
|
});
|
|
1669
1732
|
}
|
|
1670
1733
|
let storageBirthDate = getLocalConfig('birthDate', undefined);
|
|
1671
|
-
let birth = birthDate || storageBirthDate ||
|
|
1734
|
+
let birth = birthDate || storageBirthDate || (user === null || user === void 0 ? void 0 : user.birthDate);
|
|
1735
|
+
if (!birth && autoStart && initialSessionID)
|
|
1736
|
+
birth = '1970-01-01T10:24:03.845Z';
|
|
1672
1737
|
if (!sessionID && !!minAge && !birth) {
|
|
1673
1738
|
setShowAgeVerification(true);
|
|
1674
1739
|
setClickedStart(false);
|
|
@@ -1851,6 +1916,7 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
|
|
|
1851
1916
|
}
|
|
1852
1917
|
if ((!!(translatedMessages === null || translatedMessages === void 0 ? void 0 : translatedMessages.length) && translatedMessages.length > 1) ||
|
|
1853
1918
|
!initialQuestion) {
|
|
1919
|
+
console.log('[CLICK_START] Using existing chat history');
|
|
1854
1920
|
translateDialogState(currentState, userLang, undefined, !!(translatedMessages === null || translatedMessages === void 0 ? void 0 : translatedMessages.length))
|
|
1855
1921
|
.then(ts => {
|
|
1856
1922
|
let text = ts.translatedEmission || ts.emission;
|
|
@@ -1863,8 +1929,10 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
|
|
|
1863
1929
|
});
|
|
1864
1930
|
}
|
|
1865
1931
|
else {
|
|
1932
|
+
console.log('[CLICK_START] Using existing chat history with message from initial question');
|
|
1866
1933
|
translatedMessages = [];
|
|
1867
1934
|
setHistory([]);
|
|
1935
|
+
setMemoriTyping(true);
|
|
1868
1936
|
const response = await postTextEnteredEvent({
|
|
1869
1937
|
sessionId: sessionID,
|
|
1870
1938
|
text: initialQuestion,
|
|
@@ -1877,6 +1945,7 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
|
|
|
1877
1945
|
}
|
|
1878
1946
|
})
|
|
1879
1947
|
.finally(() => {
|
|
1948
|
+
setMemoriTyping(false);
|
|
1880
1949
|
setHasUserActivatedSpeak(true);
|
|
1881
1950
|
});
|
|
1882
1951
|
}
|
|
@@ -1903,6 +1972,9 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
|
|
|
1903
1972
|
}, [memoriPwd, memori, memoriTokens, birthDate, sessionId, userLang, position]);
|
|
1904
1973
|
useEffect(() => {
|
|
1905
1974
|
if (!clickedStart && autoStart) {
|
|
1975
|
+
if (AZURE_COGNITIVE_SERVICES_TTS_KEY && !speechSynthesizer) {
|
|
1976
|
+
initializeTTS();
|
|
1977
|
+
}
|
|
1906
1978
|
onClickStart();
|
|
1907
1979
|
}
|
|
1908
1980
|
}, [clickedStart, autoStart]);
|
|
@@ -2015,6 +2087,11 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
|
|
|
2015
2087
|
setSpeakerMuted: mute => {
|
|
2016
2088
|
speakerMuted = !!mute;
|
|
2017
2089
|
setMuteSpeaker(mute);
|
|
2090
|
+
let microphoneMode = getLocalConfig('microphoneMode', 'HOLD_TO_TALK');
|
|
2091
|
+
if (microphoneMode === 'CONTINUOUS' && mute) {
|
|
2092
|
+
setContinuousSpeech(false);
|
|
2093
|
+
setLocalConfig('microphoneMode', 'HOLD_TO_TALK');
|
|
2094
|
+
}
|
|
2018
2095
|
setLocalConfig('muteSpeaker', !!mute);
|
|
2019
2096
|
if (mute) {
|
|
2020
2097
|
stopAudio();
|
|
@@ -2225,7 +2302,7 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
|
|
|
2225
2302
|
setShowAgeVerification(false);
|
|
2226
2303
|
setClickedStart(false);
|
|
2227
2304
|
}
|
|
2228
|
-
} })), showSettingsDrawer && (_jsx(SettingsDrawer, { layout: selectedLayout, open: !!showSettingsDrawer, onClose: () => setShowSettingsDrawer(false), microphoneMode: continuousSpeech ? 'CONTINUOUS' : 'HOLD_TO_TALK', continuousSpeechTimeout: continuousSpeechTimeout, setMicrophoneMode: mode => setContinuousSpeech(mode === 'CONTINUOUS'), setContinuousSpeechTimeout: setContinuousSpeechTimeout, controlsPosition: controlsPosition, setControlsPosition: setControlsPosition, hideEmissions: hideEmissions, setHideEmissions: setHideEmissions, avatarType: avatarType, setAvatarType: setAvatarType, enablePositionControls: enablePositionControls, setEnablePositionControls: setEnablePositionControls, isAvatar3d: !!(integrationConfig === null || integrationConfig === void 0 ? void 0 : integrationConfig.avatarURL), additionalSettings: additionalSettings })), showPositionDrawer && (_jsx(PositionDrawer, { memori: memori, open: !!showPositionDrawer, venue: position, setVenue: setPosition, onClose: position => {
|
|
2305
|
+
} })), showSettingsDrawer && (_jsx(SettingsDrawer, { layout: selectedLayout, open: !!showSettingsDrawer, onClose: () => setShowSettingsDrawer(false), microphoneMode: continuousSpeech ? 'CONTINUOUS' : 'HOLD_TO_TALK', continuousSpeechTimeout: continuousSpeechTimeout, setMicrophoneMode: mode => setContinuousSpeech(mode === 'CONTINUOUS'), setContinuousSpeechTimeout: setContinuousSpeechTimeout, controlsPosition: controlsPosition, setControlsPosition: setControlsPosition, hideEmissions: hideEmissions, setHideEmissions: setHideEmissions, avatarType: avatarType, setAvatarType: setAvatarType, enablePositionControls: enablePositionControls, setEnablePositionControls: setEnablePositionControls, isAvatar3d: !!(integrationConfig === null || integrationConfig === void 0 ? void 0 : integrationConfig.avatarURL), additionalSettings: additionalSettings, speakerMuted: speakerMuted })), showPositionDrawer && (_jsx(PositionDrawer, { memori: memori, open: !!showPositionDrawer, venue: position, setVenue: setPosition, onClose: position => {
|
|
2229
2306
|
if (position)
|
|
2230
2307
|
applyPosition(position);
|
|
2231
2308
|
setShowPositionDrawer(false);
|