@memori.ai/memori-react 7.16.2 → 7.17.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. package/CHANGELOG.md +40 -0
  2. package/dist/components/Avatar/AvatarView/AvatarComponent/lights/Lights.d.ts +27 -0
  3. package/dist/components/Avatar/AvatarView/AvatarComponent/lights/Lights.js +52 -0
  4. package/dist/components/Avatar/AvatarView/AvatarComponent/lights/Lights.js.map +1 -0
  5. package/dist/components/Avatar/AvatarView/AvatarComponent/positionControls/positionControls.css +19 -7
  6. package/dist/components/Avatar/AvatarView/AvatarComponent/positionControls/positionControls.js +7 -7
  7. package/dist/components/Avatar/AvatarView/AvatarComponent/positionControls/positionControls.js.map +1 -1
  8. package/dist/components/Avatar/AvatarView/index.js +2 -3
  9. package/dist/components/Avatar/AvatarView/index.js.map +1 -1
  10. package/dist/components/ChatTextArea/ChatTextArea.css +55 -60
  11. package/dist/components/MemoriWidget/MemoriWidget.js +215 -138
  12. package/dist/components/MemoriWidget/MemoriWidget.js.map +1 -1
  13. package/dist/components/SettingsDrawer/SettingsDrawer.css +5 -0
  14. package/dist/components/SettingsDrawer/SettingsDrawer.d.ts +2 -1
  15. package/dist/components/SettingsDrawer/SettingsDrawer.js +6 -3
  16. package/dist/components/SettingsDrawer/SettingsDrawer.js.map +1 -1
  17. package/dist/components/UploadButton/UploadButton.d.ts +5 -0
  18. package/dist/components/UploadButton/UploadButton.js +49 -48
  19. package/dist/components/UploadButton/UploadButton.js.map +1 -1
  20. package/dist/components/ui/Slider.css +59 -44
  21. package/dist/context/visemeContext.d.ts +1 -1
  22. package/dist/context/visemeContext.js +2 -2
  23. package/dist/context/visemeContext.js.map +1 -1
  24. package/dist/helpers/constants.js +2 -2
  25. package/dist/locales/de.json +1 -0
  26. package/dist/locales/en.json +1 -0
  27. package/dist/locales/es.json +1 -0
  28. package/dist/locales/fr.json +1 -0
  29. package/dist/locales/it.json +1 -0
  30. package/esm/components/Avatar/AvatarView/AvatarComponent/lights/Lights.d.ts +27 -0
  31. package/esm/components/Avatar/AvatarView/AvatarComponent/lights/Lights.js +48 -0
  32. package/esm/components/Avatar/AvatarView/AvatarComponent/lights/Lights.js.map +1 -0
  33. package/esm/components/Avatar/AvatarView/AvatarComponent/positionControls/positionControls.css +19 -7
  34. package/esm/components/Avatar/AvatarView/AvatarComponent/positionControls/positionControls.js +7 -7
  35. package/esm/components/Avatar/AvatarView/AvatarComponent/positionControls/positionControls.js.map +1 -1
  36. package/esm/components/Avatar/AvatarView/index.js +3 -4
  37. package/esm/components/Avatar/AvatarView/index.js.map +1 -1
  38. package/esm/components/ChatTextArea/ChatTextArea.css +55 -60
  39. package/esm/components/MemoriWidget/MemoriWidget.js +216 -139
  40. package/esm/components/MemoriWidget/MemoriWidget.js.map +1 -1
  41. package/esm/components/SettingsDrawer/SettingsDrawer.css +5 -0
  42. package/esm/components/SettingsDrawer/SettingsDrawer.d.ts +2 -1
  43. package/esm/components/SettingsDrawer/SettingsDrawer.js +6 -3
  44. package/esm/components/SettingsDrawer/SettingsDrawer.js.map +1 -1
  45. package/esm/components/UploadButton/UploadButton.d.ts +5 -0
  46. package/esm/components/UploadButton/UploadButton.js +50 -49
  47. package/esm/components/UploadButton/UploadButton.js.map +1 -1
  48. package/esm/components/ui/Slider.css +59 -44
  49. package/esm/context/visemeContext.d.ts +1 -1
  50. package/esm/context/visemeContext.js +2 -2
  51. package/esm/context/visemeContext.js.map +1 -1
  52. package/esm/helpers/constants.js +2 -2
  53. package/esm/locales/de.json +1 -0
  54. package/esm/locales/en.json +1 -0
  55. package/esm/locales/es.json +1 -0
  56. package/esm/locales/fr.json +1 -0
  57. package/esm/locales/it.json +1 -0
  58. package/package.json +1 -2
  59. package/src/components/Avatar/AvatarView/AvatarComponent/lights/Lights.tsx +145 -0
  60. package/src/components/Avatar/AvatarView/AvatarComponent/positionControls/positionControls.css +19 -7
  61. package/src/components/Avatar/AvatarView/AvatarComponent/positionControls/positionControls.tsx +6 -14
  62. package/src/components/Avatar/AvatarView/index.tsx +5 -14
  63. package/src/components/ChatTextArea/ChatTextArea.css +55 -60
  64. package/src/components/MemoriWidget/MemoriWidget.tsx +337 -187
  65. package/src/components/SettingsDrawer/SettingsDrawer.css +5 -0
  66. package/src/components/SettingsDrawer/SettingsDrawer.tsx +29 -11
  67. package/src/components/StartPanel/__snapshots__/StartPanel.test.tsx.snap +2 -2
  68. package/src/components/UploadButton/UploadButton.tsx +139 -118
  69. package/src/components/UploadButton/__snapshots__/UploadButton.test.tsx.snap +3 -52
  70. package/src/components/layouts/__snapshots__/Chat.test.tsx.snap +2 -2
  71. package/src/components/layouts/__snapshots__/FullPage.test.tsx.snap +4 -4
  72. package/src/components/layouts/__snapshots__/Totem.test.tsx.snap +2 -2
  73. package/src/components/layouts/__snapshots__/ZoomedFullBody.test.tsx.snap +2 -2
  74. package/src/components/ui/Slider.css +59 -44
  75. package/src/context/visemeContext.tsx +2 -2
  76. package/src/helpers/constants.ts +2 -2
  77. package/src/locales/de.json +1 -0
  78. package/src/locales/en.json +1 -0
  79. package/src/locales/es.json +1 -0
  80. package/src/locales/fr.json +1 -0
  81. package/src/locales/it.json +1 -0
@@ -234,7 +234,6 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
234
234
  const [hideEmissions, setHideEmissions] = (0, react_1.useState)(false);
235
235
  const { startProcessing, setAudioContext, addViseme, stopProcessing, resetVisemeQueue, } = (0, visemeContext_1.useViseme)();
236
236
  (0, react_1.useEffect)(() => {
237
- setIsPlayingAudio(!!speechSynthesizer);
238
237
  memoriSpeaking = !!speechSynthesizer;
239
238
  }, [speechSynthesizer]);
240
239
  (0, react_1.useEffect)(() => {
@@ -251,12 +250,11 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
251
250
  else {
252
251
  defaultControlsPosition = 'bottom';
253
252
  }
254
- setMuteSpeaker(autoStart ||
255
- (0, configuration_1.getLocalConfig)('muteSpeaker', !defaultEnableAudio || !defaultSpeakerActive || autoStart));
256
- speakerMuted =
257
- autoStart ||
258
- (0, configuration_1.getLocalConfig)('muteSpeaker', !defaultEnableAudio || !defaultSpeakerActive || autoStart);
259
- setContinuousSpeech(microphoneMode === 'CONTINUOUS');
253
+ const muteSpeaker = autoStart ||
254
+ (0, configuration_1.getLocalConfig)('muteSpeaker', !defaultEnableAudio || !defaultSpeakerActive || autoStart);
255
+ setMuteSpeaker(muteSpeaker);
256
+ speakerMuted = muteSpeaker;
257
+ setContinuousSpeech(muteSpeaker ? false : microphoneMode === 'CONTINUOUS');
260
258
  setContinuousSpeechTimeout((0, configuration_1.getLocalConfig)('continuousSpeechTimeout', 2));
261
259
  setControlsPosition((0, configuration_1.getLocalConfig)('controlsPosition', defaultControlsPosition));
262
260
  setAvatarType((0, configuration_1.getLocalConfig)('avatarType', 'avatar3d'));
@@ -1187,32 +1185,44 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
1187
1185
  document.dispatchEvent(e);
1188
1186
  };
1189
1187
  const speak = (text) => {
1188
+ console.debug('speak called with text:', text);
1190
1189
  if (!AZURE_COGNITIVE_SERVICES_TTS_KEY || preview) {
1190
+ console.debug('No TTS key or preview mode, emitting end speak event');
1191
1191
  emitEndSpeakEvent();
1192
1192
  return;
1193
1193
  }
1194
+ console.debug('Stopping listening before speaking');
1194
1195
  stopListening();
1195
- if (preview)
1196
+ if (preview) {
1197
+ console.debug('Preview mode, returning early');
1196
1198
  return;
1199
+ }
1197
1200
  if (speakerMuted) {
1201
+ console.debug('Speaker muted, skipping speech synthesis');
1198
1202
  memoriSpeaking = false;
1199
1203
  setMemoriTyping(false);
1200
1204
  emitEndSpeakEvent();
1201
1205
  if (continuousSpeech) {
1206
+ console.debug('Setting listening timeout for continuous speech');
1202
1207
  setListeningTimeout();
1203
1208
  }
1204
1209
  return;
1205
1210
  }
1206
- if (audioDestination)
1211
+ if (audioDestination) {
1212
+ console.debug('Pausing existing audio destination');
1207
1213
  audioDestination.pause();
1214
+ }
1208
1215
  let isSafari = window.navigator.userAgent.includes('Safari') &&
1209
1216
  !window.navigator.userAgent.includes('Chrome');
1210
1217
  let isIOS = /iPad|iPhone|iPod/.test(navigator.userAgent);
1218
+ console.debug('Browser detection - Safari:', isSafari, 'iOS:', isIOS);
1211
1219
  if (audioContext.state === 'interrupted') {
1220
+ console.debug('Audio context interrupted, attempting resume');
1212
1221
  audioContext.resume().then(() => speak(text));
1213
1222
  return;
1214
1223
  }
1215
1224
  if (audioContext.state === 'closed') {
1225
+ console.debug('Audio context closed, creating new context');
1216
1226
  audioContext = new standardized_audio_context_1.AudioContext();
1217
1227
  let buffer = audioContext.createBuffer(1, 10000, 22050);
1218
1228
  let source = audioContext.createBufferSource();
@@ -1220,6 +1230,7 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
1220
1230
  source.connect(audioContext.destination);
1221
1231
  }
1222
1232
  else if (audioContext.state === 'suspended') {
1233
+ console.debug('Audio context suspended, stopping audio and creating new context');
1223
1234
  stopAudio();
1224
1235
  audioContext = new standardized_audio_context_1.AudioContext();
1225
1236
  let buffer = audioContext.createBuffer(1, 10000, 22050);
@@ -1228,77 +1239,91 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
1228
1239
  source.connect(audioContext.destination);
1229
1240
  }
1230
1241
  if (!speechSynthesizer) {
1231
- if (!isIOS) {
1232
- audioDestination = new speechSdk.SpeakerAudioDestination();
1233
- }
1234
- let audioConfig = speechSdk.AudioConfig.fromSpeakerOutput(audioDestination);
1235
- speechSynthesizer = new speechSdk.SpeechSynthesizer(speechConfig, audioConfig);
1242
+ initializeTTS();
1236
1243
  }
1237
1244
  const source = audioContext.createBufferSource();
1238
1245
  source.addEventListener('ended', () => {
1246
+ console.debug('Audio source ended');
1239
1247
  setIsPlayingAudio(false);
1240
1248
  memoriSpeaking = false;
1241
1249
  });
1242
1250
  audioDestination.onAudioEnd = () => {
1251
+ console.debug('Audio destination ended');
1243
1252
  setIsPlayingAudio(false);
1244
1253
  memoriSpeaking = false;
1245
1254
  source.disconnect();
1246
1255
  emitEndSpeakEvent();
1247
1256
  onEndSpeakStartListen();
1248
1257
  };
1258
+ console.debug('Resetting viseme queue');
1249
1259
  resetVisemeQueue();
1250
- speechSynthesizer.visemeReceived = function (_, e) {
1251
- addViseme(e.visemeId, e.audioOffset);
1252
- };
1260
+ if (speechSynthesizer) {
1261
+ speechSynthesizer.visemeReceived = function (_, e) {
1262
+ console.debug('Viseme received:', e.visemeId, 'at offset:', e.audioOffset);
1263
+ addViseme(e.visemeId, e.audioOffset);
1264
+ };
1265
+ }
1253
1266
  const textToSpeak = (0, utils_1.escapeHTML)((0, utils_1.stripMarkdown)((0, utils_1.stripEmojis)((0, utils_1.stripHTML)((0, utils_1.stripOutputTags)(text)))));
1254
- speechSynthesizer.speakSsmlAsync(`<speak version="1.0" xmlns="http://www.w3.org/2001/10/synthesis" xmlns:mstts="https://www.w3.org/2001/mstts" xmlns:emo="http://www.w3.org/2009/10/emotionml" xml:lang="${getCultureCodeByLanguage(userLang)}"><voice name="${getTTSVoice(userLang)}"><s>${replaceTextWithPhonemes(textToSpeak, userLang.toLowerCase())}</s></voice></speak>`, result => {
1255
- if (result) {
1256
- setIsPlayingAudio(true);
1257
- memoriSpeaking = true;
1258
- startProcessing(audioContext);
1259
- try {
1260
- audioContext.decodeAudioData(result.audioData, function (buffer) {
1261
- source.buffer = buffer;
1262
- source.connect(audioContext.destination);
1263
- if (history.length < 1 || (isSafari && isIOS)) {
1264
- source.start(0);
1265
- }
1266
- });
1267
- audioContext.onstatechange = () => {
1268
- if (audioContext.state === 'suspended' ||
1269
- audioContext.state === 'closed') {
1270
- source.disconnect();
1271
- setIsPlayingAudio(false);
1272
- stopProcessing();
1273
- resetVisemeQueue();
1274
- memoriSpeaking = false;
1275
- }
1276
- else if (audioContext.state === 'interrupted') {
1267
+ console.debug('Processed text to speak:', textToSpeak);
1268
+ setTimeout(() => {
1269
+ if (speechSynthesizer) {
1270
+ console.debug('Starting speech synthesis');
1271
+ speechSynthesizer.speakSsmlAsync(`<speak version="1.0" xmlns="http://www.w3.org/2001/10/synthesis" xmlns:mstts="https://www.w3.org/2001/mstts" xmlns:emo="http://www.w3.org/2009/10/emotionml" xml:lang="${getCultureCodeByLanguage(userLang)}"><voice name="${getTTSVoice(userLang)}"><s>${replaceTextWithPhonemes(textToSpeak, userLang.toLowerCase())}</s></voice></speak>`, result => {
1272
+ if (result) {
1273
+ console.debug('Speech synthesis successful');
1274
+ setIsPlayingAudio(true);
1275
+ memoriSpeaking = true;
1276
+ startProcessing(audioContext);
1277
+ try {
1278
+ console.debug('Decoding audio data');
1279
+ audioContext.decodeAudioData(result.audioData, function (buffer) {
1280
+ console.debug('Audio data decoded successfully');
1281
+ source.buffer = buffer;
1282
+ source.connect(audioContext.destination);
1283
+ if (history.length < 1 || (isSafari && isIOS)) {
1284
+ console.debug('Starting audio playback');
1285
+ source.start(0);
1286
+ }
1287
+ });
1288
+ audioContext.onstatechange = () => {
1289
+ console.debug('Audio context state changed to:', audioContext.state);
1290
+ if (audioContext.state === 'suspended' ||
1291
+ audioContext.state === 'closed') {
1292
+ source.disconnect();
1293
+ setIsPlayingAudio(false);
1294
+ stopProcessing();
1295
+ resetVisemeQueue();
1296
+ memoriSpeaking = false;
1297
+ }
1298
+ else if (audioContext.state === 'interrupted') {
1299
+ audioContext.resume();
1300
+ }
1301
+ };
1277
1302
  audioContext.resume();
1303
+ if (speechSynthesizer) {
1304
+ console.debug('Closing speech synthesizer');
1305
+ speechSynthesizer.close();
1306
+ speechSynthesizer = null;
1307
+ }
1308
+ }
1309
+ catch (error) {
1310
+ console.error('Error processing audio data:', error);
1311
+ handleFallback(text);
1278
1312
  }
1279
- };
1280
- audioContext.resume();
1281
- if (speechSynthesizer) {
1282
- speechSynthesizer.close();
1283
- speechSynthesizer = null;
1284
1313
  }
1285
- }
1286
- catch (error) {
1287
- console.error('Error processing audio data:', error);
1314
+ else {
1315
+ console.debug('No result from speech synthesis, using fallback');
1316
+ handleFallback(text);
1317
+ }
1318
+ }, error => {
1319
+ console.error('Speak error:', error);
1288
1320
  handleFallback(text);
1289
- }
1290
- }
1291
- else {
1292
- handleFallback(text);
1321
+ });
1293
1322
  }
1294
- }, error => {
1295
- console.error('Speak error:', error);
1296
- handleFallback(text);
1297
- });
1323
+ }, 100);
1298
1324
  setMemoriTyping(false);
1299
1325
  };
1300
1326
  const handleFallback = (text) => {
1301
- console.log('Falling back to browser speech synthesis');
1302
1327
  window.speechSynthesis.speak(new SpeechSynthesisUtterance(text));
1303
1328
  cleanup();
1304
1329
  };
@@ -1307,22 +1332,35 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
1307
1332
  stopProcessing();
1308
1333
  resetVisemeQueue();
1309
1334
  memoriSpeaking = false;
1310
- if (speechSynthesizer) {
1311
- console.log('Closing speech synthesizer');
1312
- speechSynthesizer.close();
1335
+ try {
1336
+ if (speechSynthesizer) {
1337
+ const currentSynthesizer = speechSynthesizer;
1338
+ speechSynthesizer = null;
1339
+ console.debug('Closing speech synthesizer');
1340
+ currentSynthesizer.close();
1341
+ }
1342
+ }
1343
+ catch (error) {
1344
+ console.debug('Error during synthesizer cleanup:', error);
1313
1345
  speechSynthesizer = null;
1314
1346
  }
1315
1347
  emitEndSpeakEvent();
1316
1348
  };
1317
- const stopAudio = () => {
1349
+ const stopAudio = async () => {
1318
1350
  setIsPlayingAudio(false);
1319
1351
  memoriSpeaking = false;
1320
1352
  try {
1321
1353
  if (speechSynthesizer) {
1322
- speechSynthesizer.close();
1354
+ const currentSynthesizer = speechSynthesizer;
1323
1355
  speechSynthesizer = null;
1356
+ try {
1357
+ currentSynthesizer.close();
1358
+ }
1359
+ catch (e) {
1360
+ console.debug('Error closing speech synthesizer:', e);
1361
+ }
1324
1362
  }
1325
- if (audioContext.state !== 'closed') {
1363
+ if ((audioContext === null || audioContext === void 0 ? void 0 : audioContext.state) !== 'closed') {
1326
1364
  audioContext.close();
1327
1365
  }
1328
1366
  if (audioDestination) {
@@ -1349,21 +1387,14 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
1349
1387
  }
1350
1388
  }, [currentDialogState === null || currentDialogState === void 0 ? void 0 : currentDialogState.emission]);
1351
1389
  const [transcript, setTranscript] = (0, react_1.useState)('');
1352
- const resetTranscript = () => setTranscript('');
1353
1390
  const [transcriptTimeout, setTranscriptTimeout] = (0, react_1.useState)(null);
1391
+ const [isSpeaking, setIsSpeaking] = (0, react_1.useState)(false);
1392
+ const resetTranscript = () => {
1393
+ setTranscript('');
1394
+ };
1354
1395
  const setListeningTimeout = () => {
1355
- let timeout = setTimeout(async () => {
1356
- clearListening();
1357
- const message = (0, utils_1.stripDuplicates)(transcript);
1358
- if (message.length > 0 && listening) {
1359
- sendMessage(message);
1360
- resetTranscript();
1361
- setUserMessage('');
1362
- }
1363
- else if (listening) {
1364
- resetInteractionTimeout();
1365
- }
1366
- }, continuousSpeechTimeout * 1000);
1396
+ clearListeningTimeout();
1397
+ const timeout = setTimeout(handleTranscriptProcessing, continuousSpeechTimeout * 1000 + 300);
1367
1398
  setTranscriptTimeout(timeout);
1368
1399
  };
1369
1400
  const clearListeningTimeout = () => {
@@ -1374,74 +1405,105 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
1374
1405
  };
1375
1406
  const resetListeningTimeout = () => {
1376
1407
  clearListeningTimeout();
1377
- if (continuousSpeech)
1408
+ if (continuousSpeech) {
1378
1409
  setListeningTimeout();
1410
+ }
1379
1411
  };
1380
1412
  (0, react_1.useEffect)(() => {
1381
- resetListeningTimeout();
1382
- resetInteractionTimeout();
1383
- }, [transcript]);
1413
+ if (!isSpeaking) {
1414
+ resetListeningTimeout();
1415
+ resetInteractionTimeout();
1416
+ }
1417
+ }, [transcript, isSpeaking]);
1418
+ (0, react_1.useEffect)(() => {
1419
+ return () => {
1420
+ clearListeningTimeout();
1421
+ };
1422
+ }, []);
1384
1423
  const startListening = async () => {
1385
- if (!AZURE_COGNITIVE_SERVICES_TTS_KEY)
1386
- return;
1387
- clearListening();
1388
- setTranscript('');
1424
+ if (!AZURE_COGNITIVE_SERVICES_TTS_KEY) {
1425
+ throw new Error('No TTS key available');
1426
+ }
1427
+ if (!sessionId) {
1428
+ throw new Error('No session ID available');
1429
+ }
1430
+ cleanup();
1389
1431
  resetTranscript();
1390
- if ((0, utils_1.hasTouchscreen)())
1391
- setEnableFocusChatInput(false);
1392
1432
  try {
1393
- navigator.mediaDevices
1394
- .getUserMedia({ audio: true })
1395
- .then(function (_stream) {
1396
- setHasUserActivatedListening(true);
1397
- if (!speechConfig) {
1398
- speechConfig = speechSdk.SpeechConfig.fromSubscription(AZURE_COGNITIVE_SERVICES_TTS_KEY, 'westeurope');
1399
- speechConfig.speechRecognitionLanguage =
1400
- getCultureCodeByLanguage(userLang);
1401
- speechConfig.speechSynthesisLanguage =
1402
- getCultureCodeByLanguage(userLang);
1403
- speechConfig.speechSynthesisVoiceName = getTTSVoice(userLang);
1404
- }
1405
- const audioConfig = speechSdk.AudioConfig.fromDefaultMicrophoneInput();
1406
- recognizer = new speechSdk.SpeechRecognizer(speechConfig, audioConfig);
1407
- setListening(true);
1408
- recognizer.recognized = (_s, e) => {
1409
- if (!e.result.text)
1410
- return;
1411
- if (e.result.reason === speechSdk.ResultReason.RecognizedSpeech) {
1412
- let transcript = e.result.text;
1413
- setTranscript(transcript || '');
1414
- if ((transcript === null || transcript === void 0 ? void 0 : transcript.length) > 0) {
1415
- const transcriptMessage = (0, utils_1.stripDuplicates)(transcript);
1416
- if (transcriptMessage.length > 0)
1417
- setUserMessage(msg => `${msg} ${transcriptMessage}`);
1418
- }
1419
- }
1420
- else if (e.result.reason === speechSdk.ResultReason.NoMatch) {
1421
- console.debug('NOMATCH: Speech could not be recognized.');
1422
- }
1423
- };
1424
- recognizer.canceled = (_s, e) => {
1425
- if (e.reason === speechSdk.CancellationReason.Error) {
1426
- console.debug(`"CANCELED: ErrorCode=${e.errorCode}`);
1427
- console.debug(`"CANCELED: ErrorDetails=${e.errorDetails}`);
1428
- console.debug('CANCELED: Did you set the speech resource key and region values?');
1429
- }
1433
+ const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
1434
+ setHasUserActivatedListening(true);
1435
+ speechConfig = setupSpeechConfig(AZURE_COGNITIVE_SERVICES_TTS_KEY);
1436
+ const audioConfig = speechSdk.AudioConfig.fromDefaultMicrophoneInput();
1437
+ recognizer = new speechSdk.SpeechRecognizer(speechConfig, audioConfig);
1438
+ setupRecognizerHandlers(recognizer);
1439
+ setListening(true);
1440
+ recognizer.startContinuousRecognitionAsync();
1441
+ recognizer.canceled = (_s, e) => {
1442
+ if (e.reason === speechSdk.CancellationReason.Error) {
1443
+ console.debug(`"CANCELED: ErrorCode=${e.errorCode}`);
1444
+ console.debug(`"CANCELED: ErrorDetails=${e.errorDetails}`);
1445
+ console.debug('CANCELED: Did you set the speech resource key and region values?');
1430
1446
  stopListening();
1431
- };
1432
- recognizer.sessionStopped = (_s, _e) => {
1433
- stopListening();
1434
- };
1447
+ cleanup();
1448
+ }
1449
+ stopListening();
1450
+ };
1451
+ recognizer.sessionStopped = (_s, _e) => {
1452
+ stopListening();
1435
1453
  resetTranscript();
1436
- recognizer.startContinuousRecognitionAsync();
1437
- })
1438
- .catch(console.debug);
1454
+ };
1439
1455
  }
1440
1456
  catch (error) {
1441
- console.debug(error);
1457
+ console.error('Error in startListening:', error);
1458
+ stopListening();
1459
+ throw error;
1460
+ }
1461
+ };
1462
+ const setupSpeechConfig = (AZURE_COGNITIVE_SERVICES_TTS_KEY) => {
1463
+ speechConfig = speechSdk.SpeechConfig.fromSubscription(AZURE_COGNITIVE_SERVICES_TTS_KEY, 'westeurope');
1464
+ speechConfig.speechRecognitionLanguage = getCultureCodeByLanguage(userLang);
1465
+ speechConfig.speechSynthesisLanguage = getCultureCodeByLanguage(userLang);
1466
+ speechConfig.speechSynthesisVoiceName = getTTSVoice(userLang);
1467
+ return speechConfig;
1468
+ };
1469
+ const setupRecognizerHandlers = (recognizer) => {
1470
+ if (recognizer) {
1471
+ recognizer.recognized = (_, event) => {
1472
+ handleRecognizedSpeech(event.result.text);
1473
+ };
1474
+ recognizer.properties.setProperty('SpeechServiceResponse_JsonResult', 'true');
1475
+ recognizer.properties.setProperty('SpeechServiceConnection_NoiseSuppression', 'true');
1476
+ recognizer.properties.setProperty('SpeechServiceConnection_SNRThresholdDb', '10.0');
1477
+ }
1478
+ };
1479
+ const handleRecognizedSpeech = (text) => {
1480
+ console.debug('Handling recognized speech:', text);
1481
+ if (!text || text.trim().length === 0) {
1482
+ console.debug('No valid text received from speech recognition');
1483
+ return;
1484
+ }
1485
+ setTranscript(text);
1486
+ setIsSpeaking(false);
1487
+ const message = (0, utils_1.stripDuplicates)(text);
1488
+ console.debug('Stripped message:', message);
1489
+ if (message.length > 0) {
1490
+ setUserMessage(message);
1491
+ }
1492
+ };
1493
+ const handleTranscriptProcessing = () => {
1494
+ const message = (0, utils_1.stripDuplicates)(transcript);
1495
+ if (message.length > 0 && listening) {
1496
+ sendMessage(message);
1497
+ resetTranscript();
1498
+ setUserMessage('');
1499
+ clearListening();
1500
+ }
1501
+ else if (listening) {
1502
+ resetInteractionTimeout();
1442
1503
  }
1443
1504
  };
1444
1505
  const stopListening = () => {
1506
+ console.debug('Stopping speech recognition');
1445
1507
  if (recognizer) {
1446
1508
  recognizer.stopContinuousRecognitionAsync();
1447
1509
  recognizer.close();
@@ -1450,9 +1512,9 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
1450
1512
  setListening(false);
1451
1513
  };
1452
1514
  const clearListening = () => {
1453
- setHasUserActivatedListening(false);
1454
1515
  stopListening();
1455
1516
  clearListeningTimeout();
1517
+ setIsSpeaking(false);
1456
1518
  };
1457
1519
  const resetListening = () => {
1458
1520
  if (listening) {
@@ -1502,15 +1564,16 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
1502
1564
  }
1503
1565
  }, [continuousSpeech, hasUserActivatedListening]);
1504
1566
  (0, react_1.useEffect)(() => {
1505
- if (history.length > 1 &&
1506
- !isPlayingAudio &&
1567
+ if (!isPlayingAudio &&
1507
1568
  continuousSpeech &&
1508
- (hasUserActivatedListening || !requestedListening))
1569
+ (hasUserActivatedListening || !requestedListening) &&
1570
+ sessionId) {
1509
1571
  startListening();
1572
+ }
1510
1573
  else if (isPlayingAudio && listening) {
1511
1574
  stopListening();
1512
1575
  }
1513
- }, [isPlayingAudio]);
1576
+ }, [isPlayingAudio, hasUserActivatedListening]);
1514
1577
  (0, react_1.useEffect)(() => {
1515
1578
  resetListening();
1516
1579
  }, [language]);
@@ -1671,7 +1734,9 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
1671
1734
  });
1672
1735
  }
1673
1736
  let storageBirthDate = (0, configuration_1.getLocalConfig)('birthDate', undefined);
1674
- let birth = birthDate || storageBirthDate || undefined;
1737
+ let birth = birthDate || storageBirthDate || (user === null || user === void 0 ? void 0 : user.birthDate);
1738
+ if (!birth && autoStart && initialSessionID)
1739
+ birth = '1970-01-01T10:24:03.845Z';
1675
1740
  if (!sessionID && !!minAge && !birth) {
1676
1741
  setShowAgeVerification(true);
1677
1742
  setClickedStart(false);
@@ -1854,6 +1919,7 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
1854
1919
  }
1855
1920
  if ((!!(translatedMessages === null || translatedMessages === void 0 ? void 0 : translatedMessages.length) && translatedMessages.length > 1) ||
1856
1921
  !initialQuestion) {
1922
+ console.log('[CLICK_START] Using existing chat history');
1857
1923
  translateDialogState(currentState, userLang, undefined, !!(translatedMessages === null || translatedMessages === void 0 ? void 0 : translatedMessages.length))
1858
1924
  .then(ts => {
1859
1925
  let text = ts.translatedEmission || ts.emission;
@@ -1866,8 +1932,10 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
1866
1932
  });
1867
1933
  }
1868
1934
  else {
1935
+ console.log('[CLICK_START] Using existing chat history with message from initial question');
1869
1936
  translatedMessages = [];
1870
1937
  setHistory([]);
1938
+ setMemoriTyping(true);
1871
1939
  const response = await postTextEnteredEvent({
1872
1940
  sessionId: sessionID,
1873
1941
  text: initialQuestion,
@@ -1880,6 +1948,7 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
1880
1948
  }
1881
1949
  })
1882
1950
  .finally(() => {
1951
+ setMemoriTyping(false);
1883
1952
  setHasUserActivatedSpeak(true);
1884
1953
  });
1885
1954
  }
@@ -1906,6 +1975,9 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
1906
1975
  }, [memoriPwd, memori, memoriTokens, birthDate, sessionId, userLang, position]);
1907
1976
  (0, react_1.useEffect)(() => {
1908
1977
  if (!clickedStart && autoStart) {
1978
+ if (AZURE_COGNITIVE_SERVICES_TTS_KEY && !speechSynthesizer) {
1979
+ initializeTTS();
1980
+ }
1909
1981
  onClickStart();
1910
1982
  }
1911
1983
  }, [clickedStart, autoStart]);
@@ -2018,6 +2090,11 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
2018
2090
  setSpeakerMuted: mute => {
2019
2091
  speakerMuted = !!mute;
2020
2092
  setMuteSpeaker(mute);
2093
+ let microphoneMode = (0, configuration_1.getLocalConfig)('microphoneMode', 'HOLD_TO_TALK');
2094
+ if (microphoneMode === 'CONTINUOUS' && mute) {
2095
+ setContinuousSpeech(false);
2096
+ (0, configuration_1.setLocalConfig)('microphoneMode', 'HOLD_TO_TALK');
2097
+ }
2021
2098
  (0, configuration_1.setLocalConfig)('muteSpeaker', !!mute);
2022
2099
  if (mute) {
2023
2100
  stopAudio();
@@ -2228,7 +2305,7 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
2228
2305
  setShowAgeVerification(false);
2229
2306
  setClickedStart(false);
2230
2307
  }
2231
- } })), showSettingsDrawer && ((0, jsx_runtime_1.jsx)(SettingsDrawer_1.default, { layout: selectedLayout, open: !!showSettingsDrawer, onClose: () => setShowSettingsDrawer(false), microphoneMode: continuousSpeech ? 'CONTINUOUS' : 'HOLD_TO_TALK', continuousSpeechTimeout: continuousSpeechTimeout, setMicrophoneMode: mode => setContinuousSpeech(mode === 'CONTINUOUS'), setContinuousSpeechTimeout: setContinuousSpeechTimeout, controlsPosition: controlsPosition, setControlsPosition: setControlsPosition, hideEmissions: hideEmissions, setHideEmissions: setHideEmissions, avatarType: avatarType, setAvatarType: setAvatarType, enablePositionControls: enablePositionControls, setEnablePositionControls: setEnablePositionControls, isAvatar3d: !!(integrationConfig === null || integrationConfig === void 0 ? void 0 : integrationConfig.avatarURL), additionalSettings: additionalSettings })), showPositionDrawer && ((0, jsx_runtime_1.jsx)(PositionDrawer_1.default, { memori: memori, open: !!showPositionDrawer, venue: position, setVenue: setPosition, onClose: position => {
2308
+ } })), showSettingsDrawer && ((0, jsx_runtime_1.jsx)(SettingsDrawer_1.default, { layout: selectedLayout, open: !!showSettingsDrawer, onClose: () => setShowSettingsDrawer(false), microphoneMode: continuousSpeech ? 'CONTINUOUS' : 'HOLD_TO_TALK', continuousSpeechTimeout: continuousSpeechTimeout, setMicrophoneMode: mode => setContinuousSpeech(mode === 'CONTINUOUS'), setContinuousSpeechTimeout: setContinuousSpeechTimeout, controlsPosition: controlsPosition, setControlsPosition: setControlsPosition, hideEmissions: hideEmissions, setHideEmissions: setHideEmissions, avatarType: avatarType, setAvatarType: setAvatarType, enablePositionControls: enablePositionControls, setEnablePositionControls: setEnablePositionControls, isAvatar3d: !!(integrationConfig === null || integrationConfig === void 0 ? void 0 : integrationConfig.avatarURL), additionalSettings: additionalSettings, speakerMuted: speakerMuted })), showPositionDrawer && ((0, jsx_runtime_1.jsx)(PositionDrawer_1.default, { memori: memori, open: !!showPositionDrawer, venue: position, setVenue: setPosition, onClose: position => {
2232
2309
  if (position)
2233
2310
  applyPosition(position);
2234
2311
  setShowPositionDrawer(false);