@memori.ai/memori-react 7.16.2 → 7.17.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +40 -0
- package/dist/components/Avatar/AvatarView/AvatarComponent/lights/Lights.d.ts +27 -0
- package/dist/components/Avatar/AvatarView/AvatarComponent/lights/Lights.js +52 -0
- package/dist/components/Avatar/AvatarView/AvatarComponent/lights/Lights.js.map +1 -0
- package/dist/components/Avatar/AvatarView/AvatarComponent/positionControls/positionControls.css +19 -7
- package/dist/components/Avatar/AvatarView/AvatarComponent/positionControls/positionControls.js +7 -7
- package/dist/components/Avatar/AvatarView/AvatarComponent/positionControls/positionControls.js.map +1 -1
- package/dist/components/Avatar/AvatarView/index.js +2 -3
- package/dist/components/Avatar/AvatarView/index.js.map +1 -1
- package/dist/components/ChatTextArea/ChatTextArea.css +55 -60
- package/dist/components/MemoriWidget/MemoriWidget.js +215 -138
- package/dist/components/MemoriWidget/MemoriWidget.js.map +1 -1
- package/dist/components/SettingsDrawer/SettingsDrawer.css +5 -0
- package/dist/components/SettingsDrawer/SettingsDrawer.d.ts +2 -1
- package/dist/components/SettingsDrawer/SettingsDrawer.js +6 -3
- package/dist/components/SettingsDrawer/SettingsDrawer.js.map +1 -1
- package/dist/components/UploadButton/UploadButton.d.ts +5 -0
- package/dist/components/UploadButton/UploadButton.js +49 -48
- package/dist/components/UploadButton/UploadButton.js.map +1 -1
- package/dist/components/ui/Slider.css +59 -44
- package/dist/context/visemeContext.d.ts +1 -1
- package/dist/context/visemeContext.js +2 -2
- package/dist/context/visemeContext.js.map +1 -1
- package/dist/helpers/constants.js +2 -2
- package/dist/locales/de.json +1 -0
- package/dist/locales/en.json +1 -0
- package/dist/locales/es.json +1 -0
- package/dist/locales/fr.json +1 -0
- package/dist/locales/it.json +1 -0
- package/esm/components/Avatar/AvatarView/AvatarComponent/lights/Lights.d.ts +27 -0
- package/esm/components/Avatar/AvatarView/AvatarComponent/lights/Lights.js +48 -0
- package/esm/components/Avatar/AvatarView/AvatarComponent/lights/Lights.js.map +1 -0
- package/esm/components/Avatar/AvatarView/AvatarComponent/positionControls/positionControls.css +19 -7
- package/esm/components/Avatar/AvatarView/AvatarComponent/positionControls/positionControls.js +7 -7
- package/esm/components/Avatar/AvatarView/AvatarComponent/positionControls/positionControls.js.map +1 -1
- package/esm/components/Avatar/AvatarView/index.js +3 -4
- package/esm/components/Avatar/AvatarView/index.js.map +1 -1
- package/esm/components/ChatTextArea/ChatTextArea.css +55 -60
- package/esm/components/MemoriWidget/MemoriWidget.js +216 -139
- package/esm/components/MemoriWidget/MemoriWidget.js.map +1 -1
- package/esm/components/SettingsDrawer/SettingsDrawer.css +5 -0
- package/esm/components/SettingsDrawer/SettingsDrawer.d.ts +2 -1
- package/esm/components/SettingsDrawer/SettingsDrawer.js +6 -3
- package/esm/components/SettingsDrawer/SettingsDrawer.js.map +1 -1
- package/esm/components/UploadButton/UploadButton.d.ts +5 -0
- package/esm/components/UploadButton/UploadButton.js +50 -49
- package/esm/components/UploadButton/UploadButton.js.map +1 -1
- package/esm/components/ui/Slider.css +59 -44
- package/esm/context/visemeContext.d.ts +1 -1
- package/esm/context/visemeContext.js +2 -2
- package/esm/context/visemeContext.js.map +1 -1
- package/esm/helpers/constants.js +2 -2
- package/esm/locales/de.json +1 -0
- package/esm/locales/en.json +1 -0
- package/esm/locales/es.json +1 -0
- package/esm/locales/fr.json +1 -0
- package/esm/locales/it.json +1 -0
- package/package.json +1 -2
- package/src/components/Avatar/AvatarView/AvatarComponent/lights/Lights.tsx +145 -0
- package/src/components/Avatar/AvatarView/AvatarComponent/positionControls/positionControls.css +19 -7
- package/src/components/Avatar/AvatarView/AvatarComponent/positionControls/positionControls.tsx +6 -14
- package/src/components/Avatar/AvatarView/index.tsx +5 -14
- package/src/components/ChatTextArea/ChatTextArea.css +55 -60
- package/src/components/MemoriWidget/MemoriWidget.tsx +337 -187
- package/src/components/SettingsDrawer/SettingsDrawer.css +5 -0
- package/src/components/SettingsDrawer/SettingsDrawer.tsx +29 -11
- package/src/components/StartPanel/__snapshots__/StartPanel.test.tsx.snap +2 -2
- package/src/components/UploadButton/UploadButton.tsx +139 -118
- package/src/components/UploadButton/__snapshots__/UploadButton.test.tsx.snap +3 -52
- package/src/components/layouts/__snapshots__/Chat.test.tsx.snap +2 -2
- package/src/components/layouts/__snapshots__/FullPage.test.tsx.snap +4 -4
- package/src/components/layouts/__snapshots__/Totem.test.tsx.snap +2 -2
- package/src/components/layouts/__snapshots__/ZoomedFullBody.test.tsx.snap +2 -2
- package/src/components/ui/Slider.css +59 -44
- package/src/context/visemeContext.tsx +2 -2
- package/src/helpers/constants.ts +2 -2
- package/src/locales/de.json +1 -0
- package/src/locales/en.json +1 -0
- package/src/locales/es.json +1 -0
- package/src/locales/fr.json +1 -0
- package/src/locales/it.json +1 -0
|
@@ -234,7 +234,6 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
|
|
|
234
234
|
const [hideEmissions, setHideEmissions] = (0, react_1.useState)(false);
|
|
235
235
|
const { startProcessing, setAudioContext, addViseme, stopProcessing, resetVisemeQueue, } = (0, visemeContext_1.useViseme)();
|
|
236
236
|
(0, react_1.useEffect)(() => {
|
|
237
|
-
setIsPlayingAudio(!!speechSynthesizer);
|
|
238
237
|
memoriSpeaking = !!speechSynthesizer;
|
|
239
238
|
}, [speechSynthesizer]);
|
|
240
239
|
(0, react_1.useEffect)(() => {
|
|
@@ -251,12 +250,11 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
|
|
|
251
250
|
else {
|
|
252
251
|
defaultControlsPosition = 'bottom';
|
|
253
252
|
}
|
|
254
|
-
|
|
255
|
-
(0, configuration_1.getLocalConfig)('muteSpeaker', !defaultEnableAudio || !defaultSpeakerActive || autoStart)
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
setContinuousSpeech(microphoneMode === 'CONTINUOUS');
|
|
253
|
+
const muteSpeaker = autoStart ||
|
|
254
|
+
(0, configuration_1.getLocalConfig)('muteSpeaker', !defaultEnableAudio || !defaultSpeakerActive || autoStart);
|
|
255
|
+
setMuteSpeaker(muteSpeaker);
|
|
256
|
+
speakerMuted = muteSpeaker;
|
|
257
|
+
setContinuousSpeech(muteSpeaker ? false : microphoneMode === 'CONTINUOUS');
|
|
260
258
|
setContinuousSpeechTimeout((0, configuration_1.getLocalConfig)('continuousSpeechTimeout', 2));
|
|
261
259
|
setControlsPosition((0, configuration_1.getLocalConfig)('controlsPosition', defaultControlsPosition));
|
|
262
260
|
setAvatarType((0, configuration_1.getLocalConfig)('avatarType', 'avatar3d'));
|
|
@@ -1187,32 +1185,44 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
|
|
|
1187
1185
|
document.dispatchEvent(e);
|
|
1188
1186
|
};
|
|
1189
1187
|
const speak = (text) => {
|
|
1188
|
+
console.debug('speak called with text:', text);
|
|
1190
1189
|
if (!AZURE_COGNITIVE_SERVICES_TTS_KEY || preview) {
|
|
1190
|
+
console.debug('No TTS key or preview mode, emitting end speak event');
|
|
1191
1191
|
emitEndSpeakEvent();
|
|
1192
1192
|
return;
|
|
1193
1193
|
}
|
|
1194
|
+
console.debug('Stopping listening before speaking');
|
|
1194
1195
|
stopListening();
|
|
1195
|
-
if (preview)
|
|
1196
|
+
if (preview) {
|
|
1197
|
+
console.debug('Preview mode, returning early');
|
|
1196
1198
|
return;
|
|
1199
|
+
}
|
|
1197
1200
|
if (speakerMuted) {
|
|
1201
|
+
console.debug('Speaker muted, skipping speech synthesis');
|
|
1198
1202
|
memoriSpeaking = false;
|
|
1199
1203
|
setMemoriTyping(false);
|
|
1200
1204
|
emitEndSpeakEvent();
|
|
1201
1205
|
if (continuousSpeech) {
|
|
1206
|
+
console.debug('Setting listening timeout for continuous speech');
|
|
1202
1207
|
setListeningTimeout();
|
|
1203
1208
|
}
|
|
1204
1209
|
return;
|
|
1205
1210
|
}
|
|
1206
|
-
if (audioDestination)
|
|
1211
|
+
if (audioDestination) {
|
|
1212
|
+
console.debug('Pausing existing audio destination');
|
|
1207
1213
|
audioDestination.pause();
|
|
1214
|
+
}
|
|
1208
1215
|
let isSafari = window.navigator.userAgent.includes('Safari') &&
|
|
1209
1216
|
!window.navigator.userAgent.includes('Chrome');
|
|
1210
1217
|
let isIOS = /iPad|iPhone|iPod/.test(navigator.userAgent);
|
|
1218
|
+
console.debug('Browser detection - Safari:', isSafari, 'iOS:', isIOS);
|
|
1211
1219
|
if (audioContext.state === 'interrupted') {
|
|
1220
|
+
console.debug('Audio context interrupted, attempting resume');
|
|
1212
1221
|
audioContext.resume().then(() => speak(text));
|
|
1213
1222
|
return;
|
|
1214
1223
|
}
|
|
1215
1224
|
if (audioContext.state === 'closed') {
|
|
1225
|
+
console.debug('Audio context closed, creating new context');
|
|
1216
1226
|
audioContext = new standardized_audio_context_1.AudioContext();
|
|
1217
1227
|
let buffer = audioContext.createBuffer(1, 10000, 22050);
|
|
1218
1228
|
let source = audioContext.createBufferSource();
|
|
@@ -1220,6 +1230,7 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
|
|
|
1220
1230
|
source.connect(audioContext.destination);
|
|
1221
1231
|
}
|
|
1222
1232
|
else if (audioContext.state === 'suspended') {
|
|
1233
|
+
console.debug('Audio context suspended, stopping audio and creating new context');
|
|
1223
1234
|
stopAudio();
|
|
1224
1235
|
audioContext = new standardized_audio_context_1.AudioContext();
|
|
1225
1236
|
let buffer = audioContext.createBuffer(1, 10000, 22050);
|
|
@@ -1228,77 +1239,91 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
|
|
|
1228
1239
|
source.connect(audioContext.destination);
|
|
1229
1240
|
}
|
|
1230
1241
|
if (!speechSynthesizer) {
|
|
1231
|
-
|
|
1232
|
-
audioDestination = new speechSdk.SpeakerAudioDestination();
|
|
1233
|
-
}
|
|
1234
|
-
let audioConfig = speechSdk.AudioConfig.fromSpeakerOutput(audioDestination);
|
|
1235
|
-
speechSynthesizer = new speechSdk.SpeechSynthesizer(speechConfig, audioConfig);
|
|
1242
|
+
initializeTTS();
|
|
1236
1243
|
}
|
|
1237
1244
|
const source = audioContext.createBufferSource();
|
|
1238
1245
|
source.addEventListener('ended', () => {
|
|
1246
|
+
console.debug('Audio source ended');
|
|
1239
1247
|
setIsPlayingAudio(false);
|
|
1240
1248
|
memoriSpeaking = false;
|
|
1241
1249
|
});
|
|
1242
1250
|
audioDestination.onAudioEnd = () => {
|
|
1251
|
+
console.debug('Audio destination ended');
|
|
1243
1252
|
setIsPlayingAudio(false);
|
|
1244
1253
|
memoriSpeaking = false;
|
|
1245
1254
|
source.disconnect();
|
|
1246
1255
|
emitEndSpeakEvent();
|
|
1247
1256
|
onEndSpeakStartListen();
|
|
1248
1257
|
};
|
|
1258
|
+
console.debug('Resetting viseme queue');
|
|
1249
1259
|
resetVisemeQueue();
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1260
|
+
if (speechSynthesizer) {
|
|
1261
|
+
speechSynthesizer.visemeReceived = function (_, e) {
|
|
1262
|
+
console.debug('Viseme received:', e.visemeId, 'at offset:', e.audioOffset);
|
|
1263
|
+
addViseme(e.visemeId, e.audioOffset);
|
|
1264
|
+
};
|
|
1265
|
+
}
|
|
1253
1266
|
const textToSpeak = (0, utils_1.escapeHTML)((0, utils_1.stripMarkdown)((0, utils_1.stripEmojis)((0, utils_1.stripHTML)((0, utils_1.stripOutputTags)(text)))));
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1267
|
+
console.debug('Processed text to speak:', textToSpeak);
|
|
1268
|
+
setTimeout(() => {
|
|
1269
|
+
if (speechSynthesizer) {
|
|
1270
|
+
console.debug('Starting speech synthesis');
|
|
1271
|
+
speechSynthesizer.speakSsmlAsync(`<speak version="1.0" xmlns="http://www.w3.org/2001/10/synthesis" xmlns:mstts="https://www.w3.org/2001/mstts" xmlns:emo="http://www.w3.org/2009/10/emotionml" xml:lang="${getCultureCodeByLanguage(userLang)}"><voice name="${getTTSVoice(userLang)}"><s>${replaceTextWithPhonemes(textToSpeak, userLang.toLowerCase())}</s></voice></speak>`, result => {
|
|
1272
|
+
if (result) {
|
|
1273
|
+
console.debug('Speech synthesis successful');
|
|
1274
|
+
setIsPlayingAudio(true);
|
|
1275
|
+
memoriSpeaking = true;
|
|
1276
|
+
startProcessing(audioContext);
|
|
1277
|
+
try {
|
|
1278
|
+
console.debug('Decoding audio data');
|
|
1279
|
+
audioContext.decodeAudioData(result.audioData, function (buffer) {
|
|
1280
|
+
console.debug('Audio data decoded successfully');
|
|
1281
|
+
source.buffer = buffer;
|
|
1282
|
+
source.connect(audioContext.destination);
|
|
1283
|
+
if (history.length < 1 || (isSafari && isIOS)) {
|
|
1284
|
+
console.debug('Starting audio playback');
|
|
1285
|
+
source.start(0);
|
|
1286
|
+
}
|
|
1287
|
+
});
|
|
1288
|
+
audioContext.onstatechange = () => {
|
|
1289
|
+
console.debug('Audio context state changed to:', audioContext.state);
|
|
1290
|
+
if (audioContext.state === 'suspended' ||
|
|
1291
|
+
audioContext.state === 'closed') {
|
|
1292
|
+
source.disconnect();
|
|
1293
|
+
setIsPlayingAudio(false);
|
|
1294
|
+
stopProcessing();
|
|
1295
|
+
resetVisemeQueue();
|
|
1296
|
+
memoriSpeaking = false;
|
|
1297
|
+
}
|
|
1298
|
+
else if (audioContext.state === 'interrupted') {
|
|
1299
|
+
audioContext.resume();
|
|
1300
|
+
}
|
|
1301
|
+
};
|
|
1277
1302
|
audioContext.resume();
|
|
1303
|
+
if (speechSynthesizer) {
|
|
1304
|
+
console.debug('Closing speech synthesizer');
|
|
1305
|
+
speechSynthesizer.close();
|
|
1306
|
+
speechSynthesizer = null;
|
|
1307
|
+
}
|
|
1308
|
+
}
|
|
1309
|
+
catch (error) {
|
|
1310
|
+
console.error('Error processing audio data:', error);
|
|
1311
|
+
handleFallback(text);
|
|
1278
1312
|
}
|
|
1279
|
-
};
|
|
1280
|
-
audioContext.resume();
|
|
1281
|
-
if (speechSynthesizer) {
|
|
1282
|
-
speechSynthesizer.close();
|
|
1283
|
-
speechSynthesizer = null;
|
|
1284
1313
|
}
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1314
|
+
else {
|
|
1315
|
+
console.debug('No result from speech synthesis, using fallback');
|
|
1316
|
+
handleFallback(text);
|
|
1317
|
+
}
|
|
1318
|
+
}, error => {
|
|
1319
|
+
console.error('Speak error:', error);
|
|
1288
1320
|
handleFallback(text);
|
|
1289
|
-
}
|
|
1290
|
-
}
|
|
1291
|
-
else {
|
|
1292
|
-
handleFallback(text);
|
|
1321
|
+
});
|
|
1293
1322
|
}
|
|
1294
|
-
},
|
|
1295
|
-
console.error('Speak error:', error);
|
|
1296
|
-
handleFallback(text);
|
|
1297
|
-
});
|
|
1323
|
+
}, 100);
|
|
1298
1324
|
setMemoriTyping(false);
|
|
1299
1325
|
};
|
|
1300
1326
|
const handleFallback = (text) => {
|
|
1301
|
-
console.log('Falling back to browser speech synthesis');
|
|
1302
1327
|
window.speechSynthesis.speak(new SpeechSynthesisUtterance(text));
|
|
1303
1328
|
cleanup();
|
|
1304
1329
|
};
|
|
@@ -1307,22 +1332,35 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
|
|
|
1307
1332
|
stopProcessing();
|
|
1308
1333
|
resetVisemeQueue();
|
|
1309
1334
|
memoriSpeaking = false;
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1335
|
+
try {
|
|
1336
|
+
if (speechSynthesizer) {
|
|
1337
|
+
const currentSynthesizer = speechSynthesizer;
|
|
1338
|
+
speechSynthesizer = null;
|
|
1339
|
+
console.debug('Closing speech synthesizer');
|
|
1340
|
+
currentSynthesizer.close();
|
|
1341
|
+
}
|
|
1342
|
+
}
|
|
1343
|
+
catch (error) {
|
|
1344
|
+
console.debug('Error during synthesizer cleanup:', error);
|
|
1313
1345
|
speechSynthesizer = null;
|
|
1314
1346
|
}
|
|
1315
1347
|
emitEndSpeakEvent();
|
|
1316
1348
|
};
|
|
1317
|
-
const stopAudio = () => {
|
|
1349
|
+
const stopAudio = async () => {
|
|
1318
1350
|
setIsPlayingAudio(false);
|
|
1319
1351
|
memoriSpeaking = false;
|
|
1320
1352
|
try {
|
|
1321
1353
|
if (speechSynthesizer) {
|
|
1322
|
-
speechSynthesizer
|
|
1354
|
+
const currentSynthesizer = speechSynthesizer;
|
|
1323
1355
|
speechSynthesizer = null;
|
|
1356
|
+
try {
|
|
1357
|
+
currentSynthesizer.close();
|
|
1358
|
+
}
|
|
1359
|
+
catch (e) {
|
|
1360
|
+
console.debug('Error closing speech synthesizer:', e);
|
|
1361
|
+
}
|
|
1324
1362
|
}
|
|
1325
|
-
if (audioContext.state !== 'closed') {
|
|
1363
|
+
if ((audioContext === null || audioContext === void 0 ? void 0 : audioContext.state) !== 'closed') {
|
|
1326
1364
|
audioContext.close();
|
|
1327
1365
|
}
|
|
1328
1366
|
if (audioDestination) {
|
|
@@ -1349,21 +1387,14 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
|
|
|
1349
1387
|
}
|
|
1350
1388
|
}, [currentDialogState === null || currentDialogState === void 0 ? void 0 : currentDialogState.emission]);
|
|
1351
1389
|
const [transcript, setTranscript] = (0, react_1.useState)('');
|
|
1352
|
-
const resetTranscript = () => setTranscript('');
|
|
1353
1390
|
const [transcriptTimeout, setTranscriptTimeout] = (0, react_1.useState)(null);
|
|
1391
|
+
const [isSpeaking, setIsSpeaking] = (0, react_1.useState)(false);
|
|
1392
|
+
const resetTranscript = () => {
|
|
1393
|
+
setTranscript('');
|
|
1394
|
+
};
|
|
1354
1395
|
const setListeningTimeout = () => {
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
const message = (0, utils_1.stripDuplicates)(transcript);
|
|
1358
|
-
if (message.length > 0 && listening) {
|
|
1359
|
-
sendMessage(message);
|
|
1360
|
-
resetTranscript();
|
|
1361
|
-
setUserMessage('');
|
|
1362
|
-
}
|
|
1363
|
-
else if (listening) {
|
|
1364
|
-
resetInteractionTimeout();
|
|
1365
|
-
}
|
|
1366
|
-
}, continuousSpeechTimeout * 1000);
|
|
1396
|
+
clearListeningTimeout();
|
|
1397
|
+
const timeout = setTimeout(handleTranscriptProcessing, continuousSpeechTimeout * 1000 + 300);
|
|
1367
1398
|
setTranscriptTimeout(timeout);
|
|
1368
1399
|
};
|
|
1369
1400
|
const clearListeningTimeout = () => {
|
|
@@ -1374,74 +1405,105 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
|
|
|
1374
1405
|
};
|
|
1375
1406
|
const resetListeningTimeout = () => {
|
|
1376
1407
|
clearListeningTimeout();
|
|
1377
|
-
if (continuousSpeech)
|
|
1408
|
+
if (continuousSpeech) {
|
|
1378
1409
|
setListeningTimeout();
|
|
1410
|
+
}
|
|
1379
1411
|
};
|
|
1380
1412
|
(0, react_1.useEffect)(() => {
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1413
|
+
if (!isSpeaking) {
|
|
1414
|
+
resetListeningTimeout();
|
|
1415
|
+
resetInteractionTimeout();
|
|
1416
|
+
}
|
|
1417
|
+
}, [transcript, isSpeaking]);
|
|
1418
|
+
(0, react_1.useEffect)(() => {
|
|
1419
|
+
return () => {
|
|
1420
|
+
clearListeningTimeout();
|
|
1421
|
+
};
|
|
1422
|
+
}, []);
|
|
1384
1423
|
const startListening = async () => {
|
|
1385
|
-
if (!AZURE_COGNITIVE_SERVICES_TTS_KEY)
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1424
|
+
if (!AZURE_COGNITIVE_SERVICES_TTS_KEY) {
|
|
1425
|
+
throw new Error('No TTS key available');
|
|
1426
|
+
}
|
|
1427
|
+
if (!sessionId) {
|
|
1428
|
+
throw new Error('No session ID available');
|
|
1429
|
+
}
|
|
1430
|
+
cleanup();
|
|
1389
1431
|
resetTranscript();
|
|
1390
|
-
if ((0, utils_1.hasTouchscreen)())
|
|
1391
|
-
setEnableFocusChatInput(false);
|
|
1392
1432
|
try {
|
|
1393
|
-
navigator.mediaDevices
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
recognizer = new speechSdk.SpeechRecognizer(speechConfig, audioConfig);
|
|
1407
|
-
setListening(true);
|
|
1408
|
-
recognizer.recognized = (_s, e) => {
|
|
1409
|
-
if (!e.result.text)
|
|
1410
|
-
return;
|
|
1411
|
-
if (e.result.reason === speechSdk.ResultReason.RecognizedSpeech) {
|
|
1412
|
-
let transcript = e.result.text;
|
|
1413
|
-
setTranscript(transcript || '');
|
|
1414
|
-
if ((transcript === null || transcript === void 0 ? void 0 : transcript.length) > 0) {
|
|
1415
|
-
const transcriptMessage = (0, utils_1.stripDuplicates)(transcript);
|
|
1416
|
-
if (transcriptMessage.length > 0)
|
|
1417
|
-
setUserMessage(msg => `${msg} ${transcriptMessage}`);
|
|
1418
|
-
}
|
|
1419
|
-
}
|
|
1420
|
-
else if (e.result.reason === speechSdk.ResultReason.NoMatch) {
|
|
1421
|
-
console.debug('NOMATCH: Speech could not be recognized.');
|
|
1422
|
-
}
|
|
1423
|
-
};
|
|
1424
|
-
recognizer.canceled = (_s, e) => {
|
|
1425
|
-
if (e.reason === speechSdk.CancellationReason.Error) {
|
|
1426
|
-
console.debug(`"CANCELED: ErrorCode=${e.errorCode}`);
|
|
1427
|
-
console.debug(`"CANCELED: ErrorDetails=${e.errorDetails}`);
|
|
1428
|
-
console.debug('CANCELED: Did you set the speech resource key and region values?');
|
|
1429
|
-
}
|
|
1433
|
+
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
|
1434
|
+
setHasUserActivatedListening(true);
|
|
1435
|
+
speechConfig = setupSpeechConfig(AZURE_COGNITIVE_SERVICES_TTS_KEY);
|
|
1436
|
+
const audioConfig = speechSdk.AudioConfig.fromDefaultMicrophoneInput();
|
|
1437
|
+
recognizer = new speechSdk.SpeechRecognizer(speechConfig, audioConfig);
|
|
1438
|
+
setupRecognizerHandlers(recognizer);
|
|
1439
|
+
setListening(true);
|
|
1440
|
+
recognizer.startContinuousRecognitionAsync();
|
|
1441
|
+
recognizer.canceled = (_s, e) => {
|
|
1442
|
+
if (e.reason === speechSdk.CancellationReason.Error) {
|
|
1443
|
+
console.debug(`"CANCELED: ErrorCode=${e.errorCode}`);
|
|
1444
|
+
console.debug(`"CANCELED: ErrorDetails=${e.errorDetails}`);
|
|
1445
|
+
console.debug('CANCELED: Did you set the speech resource key and region values?');
|
|
1430
1446
|
stopListening();
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1447
|
+
cleanup();
|
|
1448
|
+
}
|
|
1449
|
+
stopListening();
|
|
1450
|
+
};
|
|
1451
|
+
recognizer.sessionStopped = (_s, _e) => {
|
|
1452
|
+
stopListening();
|
|
1435
1453
|
resetTranscript();
|
|
1436
|
-
|
|
1437
|
-
})
|
|
1438
|
-
.catch(console.debug);
|
|
1454
|
+
};
|
|
1439
1455
|
}
|
|
1440
1456
|
catch (error) {
|
|
1441
|
-
console.
|
|
1457
|
+
console.error('Error in startListening:', error);
|
|
1458
|
+
stopListening();
|
|
1459
|
+
throw error;
|
|
1460
|
+
}
|
|
1461
|
+
};
|
|
1462
|
+
const setupSpeechConfig = (AZURE_COGNITIVE_SERVICES_TTS_KEY) => {
|
|
1463
|
+
speechConfig = speechSdk.SpeechConfig.fromSubscription(AZURE_COGNITIVE_SERVICES_TTS_KEY, 'westeurope');
|
|
1464
|
+
speechConfig.speechRecognitionLanguage = getCultureCodeByLanguage(userLang);
|
|
1465
|
+
speechConfig.speechSynthesisLanguage = getCultureCodeByLanguage(userLang);
|
|
1466
|
+
speechConfig.speechSynthesisVoiceName = getTTSVoice(userLang);
|
|
1467
|
+
return speechConfig;
|
|
1468
|
+
};
|
|
1469
|
+
const setupRecognizerHandlers = (recognizer) => {
|
|
1470
|
+
if (recognizer) {
|
|
1471
|
+
recognizer.recognized = (_, event) => {
|
|
1472
|
+
handleRecognizedSpeech(event.result.text);
|
|
1473
|
+
};
|
|
1474
|
+
recognizer.properties.setProperty('SpeechServiceResponse_JsonResult', 'true');
|
|
1475
|
+
recognizer.properties.setProperty('SpeechServiceConnection_NoiseSuppression', 'true');
|
|
1476
|
+
recognizer.properties.setProperty('SpeechServiceConnection_SNRThresholdDb', '10.0');
|
|
1477
|
+
}
|
|
1478
|
+
};
|
|
1479
|
+
const handleRecognizedSpeech = (text) => {
|
|
1480
|
+
console.debug('Handling recognized speech:', text);
|
|
1481
|
+
if (!text || text.trim().length === 0) {
|
|
1482
|
+
console.debug('No valid text received from speech recognition');
|
|
1483
|
+
return;
|
|
1484
|
+
}
|
|
1485
|
+
setTranscript(text);
|
|
1486
|
+
setIsSpeaking(false);
|
|
1487
|
+
const message = (0, utils_1.stripDuplicates)(text);
|
|
1488
|
+
console.debug('Stripped message:', message);
|
|
1489
|
+
if (message.length > 0) {
|
|
1490
|
+
setUserMessage(message);
|
|
1491
|
+
}
|
|
1492
|
+
};
|
|
1493
|
+
const handleTranscriptProcessing = () => {
|
|
1494
|
+
const message = (0, utils_1.stripDuplicates)(transcript);
|
|
1495
|
+
if (message.length > 0 && listening) {
|
|
1496
|
+
sendMessage(message);
|
|
1497
|
+
resetTranscript();
|
|
1498
|
+
setUserMessage('');
|
|
1499
|
+
clearListening();
|
|
1500
|
+
}
|
|
1501
|
+
else if (listening) {
|
|
1502
|
+
resetInteractionTimeout();
|
|
1442
1503
|
}
|
|
1443
1504
|
};
|
|
1444
1505
|
const stopListening = () => {
|
|
1506
|
+
console.debug('Stopping speech recognition');
|
|
1445
1507
|
if (recognizer) {
|
|
1446
1508
|
recognizer.stopContinuousRecognitionAsync();
|
|
1447
1509
|
recognizer.close();
|
|
@@ -1450,9 +1512,9 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
|
|
|
1450
1512
|
setListening(false);
|
|
1451
1513
|
};
|
|
1452
1514
|
const clearListening = () => {
|
|
1453
|
-
setHasUserActivatedListening(false);
|
|
1454
1515
|
stopListening();
|
|
1455
1516
|
clearListeningTimeout();
|
|
1517
|
+
setIsSpeaking(false);
|
|
1456
1518
|
};
|
|
1457
1519
|
const resetListening = () => {
|
|
1458
1520
|
if (listening) {
|
|
@@ -1502,15 +1564,16 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
|
|
|
1502
1564
|
}
|
|
1503
1565
|
}, [continuousSpeech, hasUserActivatedListening]);
|
|
1504
1566
|
(0, react_1.useEffect)(() => {
|
|
1505
|
-
if (
|
|
1506
|
-
!isPlayingAudio &&
|
|
1567
|
+
if (!isPlayingAudio &&
|
|
1507
1568
|
continuousSpeech &&
|
|
1508
|
-
(hasUserActivatedListening || !requestedListening)
|
|
1569
|
+
(hasUserActivatedListening || !requestedListening) &&
|
|
1570
|
+
sessionId) {
|
|
1509
1571
|
startListening();
|
|
1572
|
+
}
|
|
1510
1573
|
else if (isPlayingAudio && listening) {
|
|
1511
1574
|
stopListening();
|
|
1512
1575
|
}
|
|
1513
|
-
}, [isPlayingAudio]);
|
|
1576
|
+
}, [isPlayingAudio, hasUserActivatedListening]);
|
|
1514
1577
|
(0, react_1.useEffect)(() => {
|
|
1515
1578
|
resetListening();
|
|
1516
1579
|
}, [language]);
|
|
@@ -1671,7 +1734,9 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
|
|
|
1671
1734
|
});
|
|
1672
1735
|
}
|
|
1673
1736
|
let storageBirthDate = (0, configuration_1.getLocalConfig)('birthDate', undefined);
|
|
1674
|
-
let birth = birthDate || storageBirthDate ||
|
|
1737
|
+
let birth = birthDate || storageBirthDate || (user === null || user === void 0 ? void 0 : user.birthDate);
|
|
1738
|
+
if (!birth && autoStart && initialSessionID)
|
|
1739
|
+
birth = '1970-01-01T10:24:03.845Z';
|
|
1675
1740
|
if (!sessionID && !!minAge && !birth) {
|
|
1676
1741
|
setShowAgeVerification(true);
|
|
1677
1742
|
setClickedStart(false);
|
|
@@ -1854,6 +1919,7 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
|
|
|
1854
1919
|
}
|
|
1855
1920
|
if ((!!(translatedMessages === null || translatedMessages === void 0 ? void 0 : translatedMessages.length) && translatedMessages.length > 1) ||
|
|
1856
1921
|
!initialQuestion) {
|
|
1922
|
+
console.log('[CLICK_START] Using existing chat history');
|
|
1857
1923
|
translateDialogState(currentState, userLang, undefined, !!(translatedMessages === null || translatedMessages === void 0 ? void 0 : translatedMessages.length))
|
|
1858
1924
|
.then(ts => {
|
|
1859
1925
|
let text = ts.translatedEmission || ts.emission;
|
|
@@ -1866,8 +1932,10 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
|
|
|
1866
1932
|
});
|
|
1867
1933
|
}
|
|
1868
1934
|
else {
|
|
1935
|
+
console.log('[CLICK_START] Using existing chat history with message from initial question');
|
|
1869
1936
|
translatedMessages = [];
|
|
1870
1937
|
setHistory([]);
|
|
1938
|
+
setMemoriTyping(true);
|
|
1871
1939
|
const response = await postTextEnteredEvent({
|
|
1872
1940
|
sessionId: sessionID,
|
|
1873
1941
|
text: initialQuestion,
|
|
@@ -1880,6 +1948,7 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
|
|
|
1880
1948
|
}
|
|
1881
1949
|
})
|
|
1882
1950
|
.finally(() => {
|
|
1951
|
+
setMemoriTyping(false);
|
|
1883
1952
|
setHasUserActivatedSpeak(true);
|
|
1884
1953
|
});
|
|
1885
1954
|
}
|
|
@@ -1906,6 +1975,9 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
|
|
|
1906
1975
|
}, [memoriPwd, memori, memoriTokens, birthDate, sessionId, userLang, position]);
|
|
1907
1976
|
(0, react_1.useEffect)(() => {
|
|
1908
1977
|
if (!clickedStart && autoStart) {
|
|
1978
|
+
if (AZURE_COGNITIVE_SERVICES_TTS_KEY && !speechSynthesizer) {
|
|
1979
|
+
initializeTTS();
|
|
1980
|
+
}
|
|
1909
1981
|
onClickStart();
|
|
1910
1982
|
}
|
|
1911
1983
|
}, [clickedStart, autoStart]);
|
|
@@ -2018,6 +2090,11 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
|
|
|
2018
2090
|
setSpeakerMuted: mute => {
|
|
2019
2091
|
speakerMuted = !!mute;
|
|
2020
2092
|
setMuteSpeaker(mute);
|
|
2093
|
+
let microphoneMode = (0, configuration_1.getLocalConfig)('microphoneMode', 'HOLD_TO_TALK');
|
|
2094
|
+
if (microphoneMode === 'CONTINUOUS' && mute) {
|
|
2095
|
+
setContinuousSpeech(false);
|
|
2096
|
+
(0, configuration_1.setLocalConfig)('microphoneMode', 'HOLD_TO_TALK');
|
|
2097
|
+
}
|
|
2021
2098
|
(0, configuration_1.setLocalConfig)('muteSpeaker', !!mute);
|
|
2022
2099
|
if (mute) {
|
|
2023
2100
|
stopAudio();
|
|
@@ -2228,7 +2305,7 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
|
|
|
2228
2305
|
setShowAgeVerification(false);
|
|
2229
2306
|
setClickedStart(false);
|
|
2230
2307
|
}
|
|
2231
|
-
} })), showSettingsDrawer && ((0, jsx_runtime_1.jsx)(SettingsDrawer_1.default, { layout: selectedLayout, open: !!showSettingsDrawer, onClose: () => setShowSettingsDrawer(false), microphoneMode: continuousSpeech ? 'CONTINUOUS' : 'HOLD_TO_TALK', continuousSpeechTimeout: continuousSpeechTimeout, setMicrophoneMode: mode => setContinuousSpeech(mode === 'CONTINUOUS'), setContinuousSpeechTimeout: setContinuousSpeechTimeout, controlsPosition: controlsPosition, setControlsPosition: setControlsPosition, hideEmissions: hideEmissions, setHideEmissions: setHideEmissions, avatarType: avatarType, setAvatarType: setAvatarType, enablePositionControls: enablePositionControls, setEnablePositionControls: setEnablePositionControls, isAvatar3d: !!(integrationConfig === null || integrationConfig === void 0 ? void 0 : integrationConfig.avatarURL), additionalSettings: additionalSettings })), showPositionDrawer && ((0, jsx_runtime_1.jsx)(PositionDrawer_1.default, { memori: memori, open: !!showPositionDrawer, venue: position, setVenue: setPosition, onClose: position => {
|
|
2308
|
+
} })), showSettingsDrawer && ((0, jsx_runtime_1.jsx)(SettingsDrawer_1.default, { layout: selectedLayout, open: !!showSettingsDrawer, onClose: () => setShowSettingsDrawer(false), microphoneMode: continuousSpeech ? 'CONTINUOUS' : 'HOLD_TO_TALK', continuousSpeechTimeout: continuousSpeechTimeout, setMicrophoneMode: mode => setContinuousSpeech(mode === 'CONTINUOUS'), setContinuousSpeechTimeout: setContinuousSpeechTimeout, controlsPosition: controlsPosition, setControlsPosition: setControlsPosition, hideEmissions: hideEmissions, setHideEmissions: setHideEmissions, avatarType: avatarType, setAvatarType: setAvatarType, enablePositionControls: enablePositionControls, setEnablePositionControls: setEnablePositionControls, isAvatar3d: !!(integrationConfig === null || integrationConfig === void 0 ? void 0 : integrationConfig.avatarURL), additionalSettings: additionalSettings, speakerMuted: speakerMuted })), showPositionDrawer && ((0, jsx_runtime_1.jsx)(PositionDrawer_1.default, { memori: memori, open: !!showPositionDrawer, venue: position, setVenue: setPosition, onClose: position => {
|
|
2232
2309
|
if (position)
|
|
2233
2310
|
applyPosition(position);
|
|
2234
2311
|
setShowPositionDrawer(false);
|