@memori.ai/memori-react 2.7.2 → 2.8.1

package/src/components/MediaWidget/MediaWidget.tsx

@@ -5,7 +5,7 @@ import {
 import React, { useEffect, useState } from 'react';
 import Button from '../ui/Button';
 import LinkItemWidget from './LinkItemWidget';
-import MediaItemWidget from './MediaItemWidget';
+import MediaItemWidget, { Props as MediaItemProps } from './MediaItemWidget';
 import { Transition } from '@headlessui/react';
 import cx from 'classnames';
 import { useTranslation } from 'react-i18next';
@@ -19,6 +19,7 @@ export interface Props {
   baseUrl?: string;
   apiUrl?: string;
   translateTo?: string;
+  customMediaRenderer?: MediaItemProps['customMediaRenderer'];
 }
 
 const MediaWidget: React.FC<Props> = ({
@@ -30,6 +31,7 @@ const MediaWidget: React.FC<Props> = ({
   baseUrl,
   apiUrl,
   translateTo,
+  customMediaRenderer,
 }: Props) => {
   const { t } = useTranslation();
   const [showHints, setShowHints] = useState(true);
@@ -48,6 +50,7 @@ const MediaWidget: React.FC<Props> = ({
           translateTo={translateTo}
           baseURL={baseUrl}
           apiURL={apiUrl}
+          customMediaRenderer={customMediaRenderer}
         />
       )}
       {links?.length > 0 && <LinkItemWidget items={links} baseUrl={baseUrl} />}

package/src/components/MediaWidget/__snapshots__/MediaItemWidget.test.tsx.snap

@@ -151,6 +151,18 @@ exports[`renders MediaItemWidget unchanged with css snippet to show 1`] = `
 </div>
 `;
 
+exports[`renders MediaItemWidget unchanged with custom media renderer 1`] = `
+<div>
+  <div
+    class="memori-media-items"
+  >
+    <div
+      class="memori-media-items--grid"
+    />
+  </div>
+</div>
+`;
+
 exports[`renders MediaItemWidget unchanged with img 1`] = `
 <div>
   <div

package/src/components/MemoriWidget/MemoriWidget.stories.tsx

@@ -127,3 +127,24 @@ WithAzureSpeechKey.args = {
   tenant,
   AZURE_COGNITIVE_SERVICES_TTS_KEY: 'provide your key here',
 };
+
+export const WithCustomMediaRenderer = Template.bind({});
+WithCustomMediaRenderer.args = {
+  memori,
+  tenant,
+  customMediaRenderer: (mimeType: string) => (
+    <div
+      style={{
+        width: '100%',
+        height: '100%',
+        backgroundColor: 'black',
+        color: 'white',
+        display: 'flex',
+        justifyContent: 'center',
+        alignItems: 'center',
+      }}
+    >
+      {mimeType}
+    </div>
+  ),
+};

package/src/components/MemoriWidget/MemoriWidget.tsx

@@ -148,6 +148,7 @@ let audioContext: IAudioContext;
 
 let memoriPassword: string | undefined;
 let speakerMuted: boolean = false;
+let speakLanguage: string | undefined;
 
 export interface LayoutProps {
   Header?: typeof Header;
@@ -203,6 +204,7 @@ export interface Props {
   AZURE_COGNITIVE_SERVICES_TTS_KEY?: string;
   onStateChange?: (state?: DialogState) => void;
   additionalInfo?: OpenSession['additionalInfo'] & { [key: string]: string };
+  customMediaRenderer?: ChatProps['customMediaRenderer'];
 }
 
 const MemoriWidget = ({
@@ -235,6 +237,7 @@ const MemoriWidget = ({
   AZURE_COGNITIVE_SERVICES_TTS_KEY,
   onStateChange,
   additionalInfo,
+  customMediaRenderer,
 }: Props) => {
   const { t, i18n } = useTranslation();
 
@@ -271,7 +274,7 @@ const MemoriWidget = ({
     multilingual !== undefined
       ? multilingual
       : !!integrationConfig?.multilanguage;
-  const [userLang, setUserLang] = useState(
+  const [userLang, _setUserLang] = useState(
     memoriLang ??
       integrationConfig?.lang ??
       memori?.culture?.split('-')?.[0] ??
@@ -280,6 +283,10 @@ const MemoriWidget = ({
       i18n.language ??
       'IT'
   );
+  const setUserLang: typeof _setUserLang = lang => {
+    _setUserLang(lang);
+    speakLanguage = lang;
+  };
 
   const [loading, setLoading] = useState(false);
   const [memoriTyping, setMemoriTyping] = useState(false);
@@ -1141,6 +1148,13 @@ const MemoriWidget = ({
               : 'ru-RU-SvetlanaNeural'
           }`;
           break;
+        case 'PL':
+          voice = `${
+            memori.voiceType === 'MALE'
+              ? 'pl-PL-MarekNeural'
+              : 'pl-PL-AgnieszkaNeural'
+          }`;
+          break;
         default:
           voice = `${
             memori.voiceType === 'MALE'
@@ -1187,6 +1201,9 @@ const MemoriWidget = ({
       case 'RU':
         voice = 'ru-RU';
         break;
+      case 'PL':
+        voice = 'pl-PL';
+        break;
       default:
         voice = 'it-IT';
         break;
@@ -1266,177 +1283,182 @@ const MemoriWidget = ({
     // .replace(/qfe/gi, `<sub alias="Quota Filo Erba">QFE</sub>`)
   };
 
-  const speak = (text: string): void => {
-    if (!AZURE_COGNITIVE_SERVICES_TTS_KEY) return;
-    stopListening();
-    // stopAudio();
+  const speak = useCallback(
+    (text: string): void => {
+      if (!AZURE_COGNITIVE_SERVICES_TTS_KEY) return;
+      stopListening();
+      // stopAudio();
 
-    if (preview) return;
+      if (preview) return;
 
-    if (muteSpeaker || speakerMuted) {
-      // trigger start continuous listening if set, see MemoriChat
-      if (continuousSpeech) {
-        setListeningTimeout();
+      if (muteSpeaker || speakerMuted) {
+        // trigger start continuous listening if set, see MemoriChat
+        if (continuousSpeech) {
+          setListeningTimeout();
+        }
+        return;
       }
-      return;
-    }
 
-    if (audioDestination) audioDestination.pause();
-
-    // if (audioContext?.state === 'running') {
-    //   //   audioContext.suspend();
-    //   // }
-    //   // if (audioContext) {
-    //   audioContext.close().then(() => {
-    //     audioContext = new AudioContext();
-    //     let buffer = audioContext.createBuffer(1, 10000, 22050);
-    //     let source = audioContext.createBufferSource();
-    //     source.buffer = buffer;
-    //     source.connect(audioContext.destination);
-    //   });
-    // }
-
-    let isSafari =
-      window.navigator.userAgent.includes('Safari') &&
-      !window.navigator.userAgent.includes('Chrome');
-    let isIOS = /iPad|iPhone|iPod/.test(navigator.userAgent);
-    if ((audioContext.state as string) === 'interrupted') {
-      audioContext.resume().then(() => speak(text));
-      return;
-    }
-    if (isIOS && isSafari) {
-      audioContext.suspend();
-
-      if (isPlayingAudio) {
-        try {
-          if (speechSynthesizer) {
-            speechSynthesizer.close();
-            speechSynthesizer = null;
-          }
-          if (audioDestination) {
-            audioDestination.pause();
-            audioDestination.close();
-          }
-          if (audioContext) {
-            // audioContext.close().then(() => {
-            //   audioContext = new AudioContext();
-            //   let buffer = audioContext.createBuffer(1, 10000, 22050);
-            //   let source = audioContext.createBufferSource();
-            //   source.buffer = buffer;
-            //   source.connect(audioContext.destination);
-            // });
-            audioContext.destination.disconnect();
+      if (audioDestination) audioDestination.pause();
+
+      // if (audioContext?.state === 'running') {
+      //   //   audioContext.suspend();
+      //   // }
+      //   // if (audioContext) {
+      //   audioContext.close().then(() => {
+      //     audioContext = new AudioContext();
+      //     let buffer = audioContext.createBuffer(1, 10000, 22050);
+      //     let source = audioContext.createBufferSource();
+      //     source.buffer = buffer;
+      //     source.connect(audioContext.destination);
+      //   });
+      // }
+
+      let isSafari =
+        window.navigator.userAgent.includes('Safari') &&
+        !window.navigator.userAgent.includes('Chrome');
+      let isIOS = /iPad|iPhone|iPod/.test(navigator.userAgent);
+      if ((audioContext.state as string) === 'interrupted') {
+        audioContext.resume().then(() => speak(text));
+        return;
+      }
+      if (isIOS && isSafari) {
+        audioContext.suspend();
+
+        if (isPlayingAudio) {
+          try {
+            if (speechSynthesizer) {
+              speechSynthesizer.close();
+              speechSynthesizer = null;
+            }
+            if (audioDestination) {
+              audioDestination.pause();
+              audioDestination.close();
+            }
+            if (audioContext) {
+              // audioContext.close().then(() => {
+              //   audioContext = new AudioContext();
+              //   let buffer = audioContext.createBuffer(1, 10000, 22050);
+              //   let source = audioContext.createBufferSource();
+              //   source.buffer = buffer;
+              //   source.connect(audioContext.destination);
+              // });
+              audioContext.destination.disconnect();
+            }
+          } catch (e) {
+            console.error('stopAudio error: ', e);
           }
-        } catch (e) {
-          console.error('stopAudio error: ', e);
         }
       }
-    }
-    if (audioContext.state === 'closed') {
-      audioContext = new AudioContext();
-      let buffer = audioContext.createBuffer(1, 10000, 22050);
-      let source = audioContext.createBufferSource();
-      source.buffer = buffer;
-      source.connect(audioContext.destination);
-    } else if (audioContext.state === 'suspended') {
-      stopAudio();
+      if (audioContext.state === 'closed') {
+        audioContext = new AudioContext();
+        let buffer = audioContext.createBuffer(1, 10000, 22050);
+        let source = audioContext.createBufferSource();
+        source.buffer = buffer;
+        source.connect(audioContext.destination);
+      } else if (audioContext.state === 'suspended') {
+        stopAudio();
 
-      audioContext = new AudioContext();
-      let buffer = audioContext.createBuffer(1, 10000, 22050);
-      let source = audioContext.createBufferSource();
-      source.buffer = buffer;
-      source.connect(audioContext.destination);
-    }
+        audioContext = new AudioContext();
+        let buffer = audioContext.createBuffer(1, 10000, 22050);
+        let source = audioContext.createBufferSource();
+        source.buffer = buffer;
+        source.connect(audioContext.destination);
+      }
 
-    if (!speechSynthesizer) {
-      audioDestination = new speechSdk.SpeakerAudioDestination();
-      let audioConfig =
-        speechSdk.AudioConfig.fromSpeakerOutput(audioDestination);
-      speechSynthesizer = new speechSdk.SpeechSynthesizer(
-        speechConfig,
-        audioConfig
-      );
-    }
+      if (!speechSynthesizer) {
+        audioDestination = new speechSdk.SpeakerAudioDestination();
+        let audioConfig =
+          speechSdk.AudioConfig.fromSpeakerOutput(audioDestination);
+        speechSynthesizer = new speechSdk.SpeechSynthesizer(
+          speechConfig,
+          audioConfig
+        );
+      }
 
-    const source = audioContext.createBufferSource();
-    source.addEventListener('ended', () => {
-      setIsPlayingAudio(false);
-    });
-    audioDestination.onAudioEnd = () => {
-      setIsPlayingAudio(false);
-      source.disconnect();
+      const source = audioContext.createBufferSource();
+      source.addEventListener('ended', () => {
+        setIsPlayingAudio(false);
+      });
+      audioDestination.onAudioEnd = () => {
+        setIsPlayingAudio(false);
+        source.disconnect();
 
-      // trigger start continuous listening if set
-      // document.dispatchEvent(new Event('endSpeakStartListen'));
-      onEndSpeakStartListen();
-    };
+        // trigger start continuous listening if set
+        // document.dispatchEvent(new Event('endSpeakStartListen'));
+        onEndSpeakStartListen();
+      };
 
-    speechSynthesizer.speakSsmlAsync(
-      `<speak version="1.0" xmlns="http://www.w3.org/2001/10/synthesis" xmlns:mstts="https://www.w3.org/2001/mstts" xmlns:emo="http://www.w3.org/2009/10/emotionml" xml:lang="${getCultureCodeByLanguage(
-        userLang
-      )}"><voice name="${getTTSVoice(userLang)}"><s>${replaceTextWithPhonemes(
-        escapeHTML(text),
-        userLang.toLowerCase()
-      )}</s></voice></speak>`,
-      result => {
-        if (result) {
-          setIsPlayingAudio(true);
+      const lang = speakLanguage || userLang;
 
-          try {
-            // if (audioContext.destination.context.state === 'running') {
-            //   audioContext.destination.disconnect();
-            // }
-            audioContext.decodeAudioData(result.audioData, function (buffer) {
-              source.buffer = buffer;
-              source.connect(audioContext.destination);
-
-              if (history.length < 1 || (isSafari && isIOS)) {
-                source.start(0);
-              }
-            });
+      speechSynthesizer.speakSsmlAsync(
+        `<speak version="1.0" xmlns="http://www.w3.org/2001/10/synthesis" xmlns:mstts="https://www.w3.org/2001/mstts" xmlns:emo="http://www.w3.org/2009/10/emotionml" xml:lang="${getCultureCodeByLanguage(
+          lang
+        )}"><voice name="${getTTSVoice(lang)}"><s>${replaceTextWithPhonemes(
+          escapeHTML(text),
+          lang.toLowerCase()
+        )}</s></voice></speak>`,
+        result => {
+          if (result) {
+            setIsPlayingAudio(true);
 
-            audioContext.onstatechange = () => {
-              if (
-                audioContext.state === 'suspended' ||
-                audioContext.state === 'closed'
-              ) {
-                source.disconnect();
-                setIsPlayingAudio(false);
-              } else if ((audioContext.state as string) === 'interrupted') {
-                audioContext.resume();
-              }
-            };
+            try {
+              // if (audioContext.destination.context.state === 'running') {
+              //   audioContext.destination.disconnect();
+              // }
+              audioContext.decodeAudioData(result.audioData, function (buffer) {
+                source.buffer = buffer;
+                source.connect(audioContext.destination);
+
+                if (history.length < 1 || (isSafari && isIOS)) {
+                  source.start(0);
+                }
+              });
 
-            audioContext.resume();
+              audioContext.onstatechange = () => {
+                if (
+                  audioContext.state === 'suspended' ||
+                  audioContext.state === 'closed'
+                ) {
+                  source.disconnect();
+                  setIsPlayingAudio(false);
+                } else if ((audioContext.state as string) === 'interrupted') {
+                  audioContext.resume();
+                }
+              };
 
-            if (speechSynthesizer) {
-              speechSynthesizer.close();
-              speechSynthesizer = null;
-            }
-          } catch (e) {
-            console.error('speak error: ', e);
-            window.speechSynthesis.speak(new SpeechSynthesisUtterance(text));
-            setIsPlayingAudio(false);
+              audioContext.resume();
 
-            if (speechSynthesizer) {
-              speechSynthesizer.close();
-              speechSynthesizer = null;
+              if (speechSynthesizer) {
+                speechSynthesizer.close();
+                speechSynthesizer = null;
+              }
+            } catch (e) {
+              console.error('speak error: ', e);
+              window.speechSynthesis.speak(new SpeechSynthesisUtterance(text));
+              setIsPlayingAudio(false);
+
+              if (speechSynthesizer) {
+                speechSynthesizer.close();
+                speechSynthesizer = null;
+              }
             }
+          } else {
+            audioContext.resume();
+            setIsPlayingAudio(false);
           }
-        } else {
-          audioContext.resume();
+        },
+        error => {
+          console.error('speak:', error);
+          window.speechSynthesis.speak(new SpeechSynthesisUtterance(text));
           setIsPlayingAudio(false);
         }
-      },
-      error => {
-        console.error('speak:', error);
-        window.speechSynthesis.speak(new SpeechSynthesisUtterance(text));
-        setIsPlayingAudio(false);
-      }
-    );
+      );
 
-    setMemoriTyping(false);
-  };
+      setMemoriTyping(false);
+    },
+    [userLang, speakerMuted]
+  );
   const stopAudio = () => {
     setIsPlayingAudio(false);
     try {
@@ -2312,6 +2334,7 @@ const MemoriWidget = ({
     resetTranscript,
     listening,
     isPlayingAudio,
+    customMediaRenderer,
   };
 
   const integrationBackground =

package/src/index.tsx

@@ -40,6 +40,7 @@ export interface Props {
   AZURE_COGNITIVE_SERVICES_TTS_KEY?: string;
   onStateChange?: (state?: DialogState) => void;
   additionalInfo?: WidgetProps['additionalInfo'];
+  customMediaRenderer?: WidgetProps['customMediaRenderer'];
 }
 
 const getPreferredLanguages = () => {
@@ -87,6 +88,7 @@ const Memori: React.FC<Props> = ({
   AZURE_COGNITIVE_SERVICES_TTS_KEY,
   onStateChange,
   additionalInfo,
+  customMediaRenderer,
 }) => {
   const [memori, setMemori] = useState<IMemori>();
   const [speechKey, setSpeechKey] = useState<string | undefined>(
@@ -213,6 +215,7 @@ const Memori: React.FC<Props> = ({
       }
       onStateChange={onStateChange}
       additionalInfo={additionalInfo}
+      customMediaRenderer={customMediaRenderer}
       {...(tag && pin ? { personification: { tag, pin } } : {})}
     />
   ) : (