@memori.ai/memori-react 1.0.1 → 1.1.0

package/src/components/AvatarView/components/fullbodyAvatar.tsx

@@ -0,0 +1,88 @@
+import React, { memo, useEffect, useMemo } from 'react';
+import { AnimationMixer, Object3D, Vector3 } from 'three';
+import { useAnimations, useGLTF } from '@react-three/drei';
+import useEyeBlink from '../utils/useEyeBlink';
+import useHeadMovement from '../utils/useHeadMovement';
+import useMouthSpeaking from '../utils/useMouthSpeaking';
+import { dispose, useGraph } from '@react-three/fiber';
+import { correctMaterials, hideHands, isSkinnedMesh } from '../utils/utils';
+
+export interface AvatarProps {
+  url: string;
+  eyeBlink?: boolean;
+  headMovement?: boolean;
+  speaking?: boolean;
+  onLoaded?: () => void;
+  animation?:
+    | 'Idle'
+    | 'Idle 1'
+    | 'Idle 2'
+    | 'Idle 3'
+    | 'Loading'
+    | 'Sad'
+    | 'Talk 1'
+    | 'Talk 2'
+    | 'Talk 3';
+}
+
+const position = new Vector3(0, -1, 0);
+
+export default function Avatar({
+  url,
+  eyeBlink,
+  headMovement,
+  speaking,
+  onLoaded,
+  animation,
+}: AvatarProps) {
+  const { scene } = useGLTF(url);
+  const { animations } = useGLTF(
+    'https://assets.memori.ai/api/v2/asset/b11eef07-c45b-4959-9d43-be8105365c1f.glb'
+  );
+  const { nodes, materials } = useGraph(scene);
+
+  const anim = useAnimations(animations, scene);
+
+  useEyeBlink(eyeBlink, nodes);
+  useHeadMovement(headMovement, nodes);
+  useMouthSpeaking(!!speaking, nodes);
+
+  useEffect(() => {
+    correctMaterials(materials);
+
+    if (onLoaded) onLoaded();
+
+    return () => {
+      Object.values(materials).forEach(dispose);
+
+      Object.values(nodes).filter(isSkinnedMesh).forEach(dispose);
+    };
+  }, [materials, nodes, url, onLoaded]);
+
+  // useEffect(() => {
+  // console.log('animation', animation, animation && animation in anim.actions);
+  try {
+    if (animation && animation in anim.actions) {
+      Object.keys(anim.actions).forEach(name => {
+        let action = anim.actions[name];
+        if (!action) return;
+        else if (name === animation) action.play();
+        else action.stop();
+      });
+    } else {
+      Object.values(anim.actions).forEach(action => {
+        if (!action) return;
+        action?.reset().stop();
+      });
+    }
+  } catch (e) {
+    console.log(e);
+  }
+  // }, [animation]);
+
+  return (
+    <group position={position}>
+      <primitive object={scene} />
+    </group>
+  );
+}

package/src/components/AvatarView/index.tsx

@@ -1,6 +1,7 @@
 import { CSSProperties } from 'react';
 import React, { Suspense } from 'react';
 import Avatar from './components/avatar';
+import FullbodyAvatar, { AvatarProps } from './components/fullbodyAvatar';
 import Loader from './components/loader';
 import { Canvas } from '@react-three/fiber';
 import { OrbitControls, SpotLight, Environment } from '@react-three/drei';
@@ -15,15 +16,23 @@ export interface Props {
   speaking?: boolean;
   style?: CSSProperties;
   fallback?: React.ReactNode;
+  halfBody?: boolean;
+  animation?: AvatarProps['animation'];
 }
 
-const defaultStyle = {
+const defaultStylesHalfBody = {
   width: '250px',
   height: '250px',
   backgroundColor: 'white',
   borderRadius: '100%',
 };
 
+const defaultStylesFullBody = {
+  width: '500px',
+  height: '500px',
+  backgroundColor: 'white',
+};
+
 export default function AvatarView({
   url,
   fallbackImg,
@@ -33,11 +42,23 @@ export default function AvatarView({
   headMovement,
   speaking,
   fallback,
+  halfBody = true,
+  animation,
 }: Props) {
+  const defaultStyles = halfBody
+    ? defaultStylesHalfBody
+    : defaultStylesFullBody;
   return (
     <Canvas
-      style={style || defaultStyle}
-      camera={{ fov: 40, position: [0, 0, 0.6] }}
+      style={style || defaultStyles}
+      camera={
+        halfBody
+          ? {
+              fov: 40,
+              position: [0, 0, 0.6],
+            }
+          : { fov: 40, position: [0, 0, 3] }
+      }
     >
       <Suspense fallback={fallback || <Loader fallbackImg={fallbackImg} />}>
         {isAndroid() ? (
@@ -51,13 +72,25 @@ export default function AvatarView({
         ) : (
           <Environment preset="sunset" />
         )}
-        {rotateAvatar && <OrbitControls enablePan={false} enableZoom={false} />}
-        <Avatar
-          url={url}
-          eyeBlink={eyeBlink}
-          headMovement={headMovement}
-          speaking={speaking}
-        />
+        {rotateAvatar && halfBody && (
+          <OrbitControls enablePan={false} enableZoom={false} />
+        )}
+        {halfBody ? (
+          <Avatar
+            url={url}
+            eyeBlink={eyeBlink}
+            headMovement={headMovement}
+            speaking={speaking}
+          />
+        ) : (
+          <FullbodyAvatar
+            url={url}
+            eyeBlink={eyeBlink}
+            headMovement={headMovement}
+            speaking={speaking}
+            animation={animation}
+          />
+        )}
       </Suspense>
     </Canvas>
   );

package/src/components/Chat/Chat.test.tsx

@@ -20,7 +20,7 @@ const dialogState = {
 
 const DateTimeFormat = Intl.DateTimeFormat;
 beforeEach(() => {
-  jest.spyOn(global.Intl, 'DateTimeFormat').mockImplementation(
+  jest.spyOn(Intl, 'DateTimeFormat').mockImplementation(
     (locale, options) =>
       new DateTimeFormat(locale, {
         ...options,

package/src/components/ChatInputs/ChatInputs.tsx

@@ -46,7 +46,6 @@ const ChatInputs: React.FC<Props> = ({
   onTextareaPressEnter,
   showMicrophone = false,
   listening = false,
-  isPlayingAudio = false,
   stopAudio,
   startListening,
   stopListening,
@@ -104,7 +103,6 @@ const ChatInputs: React.FC<Props> = ({
               ? t('write_and_speak.micButtonPopoverListening') || 'Listening'
               : t('write_and_speak.micButtonPopover') || 'Start listening'
           }
-          disabled={isPlayingAudio}
           onClick={() => {
             if (listening) {
               stopListening();

package/src/components/Header/Header.tsx

@@ -26,6 +26,7 @@ export interface Props {
   hasUserActivatedSpeak?: boolean;
   showShare?: boolean;
   showSettings?: boolean;
+  showSpeaker?: boolean;
 }
 
 const Header: React.FC<Props> = ({
@@ -40,6 +41,7 @@ const Header: React.FC<Props> = ({
   hasUserActivatedSpeak = false,
   showShare = true,
   showSettings = true,
+  showSpeaker = true,
 }) => {
   const { t } = useTranslation();
 
@@ -58,14 +60,16 @@ const Header: React.FC<Props> = ({
           />
         </div>
       )}
-      <Button
-        primary
-        shape="circle"
-        className="memori-header--button"
-        icon={speakerMuted ? <SoundDeactivated /> : <Sound />}
-        onClick={() => setSpeakerMuted(!speakerMuted)}
-        title={t('widget.sound') || 'Sound'}
-      />
+      {showSpeaker && (
+        <Button
+          primary
+          shape="circle"
+          className="memori-header--button"
+          icon={speakerMuted ? <SoundDeactivated /> : <Sound />}
+          onClick={() => setSpeakerMuted(!speakerMuted)}
+          title={t('widget.sound') || 'Sound'}
+        />
+      )}
       <ExportHistoryButton
         history={history}
         memori={memori}

package/src/components/MediaWidget/MediaItemWidget.tsx

@@ -97,7 +97,13 @@ export const RenderMediaItem = ({
       return (
         <a
           className="memori-media-item--link"
-          href={item.url}
+          href={getResourceUrl({
+            resourceURI: item.url,
+            sessionID,
+            tenantID,
+            baseURL,
+            apiURL,
+          })}
           target="_blank"
           rel="noopener noreferrer"
           title={item.title}
@@ -115,7 +121,13 @@ export const RenderMediaItem = ({
       return (
         <a
           className="memori-media-item--link"
-          href={item.url}
+          href={getResourceUrl({
+            resourceURI: item.url,
+            sessionID,
+            tenantID,
+            baseURL,
+            apiURL,
+          })}
           target="_blank"
           rel="noopener noreferrer"
           title={item.title}
@@ -132,7 +144,13 @@ export const RenderMediaItem = ({
       return (
         <a
           className="memori-media-item--link"
-          href={item.url}
+          href={getResourceUrl({
+            resourceURI: item.url,
+            sessionID,
+            tenantID,
+            baseURL,
+            apiURL,
+          })}
           target="_blank"
           rel="noopener noreferrer"
           title={item.title}
@@ -151,7 +169,13 @@ export const RenderMediaItem = ({
       return (
         <a
           className="memori-media-item--link"
-          href={item.url}
+          href={getResourceUrl({
+            resourceURI: item.url,
+            sessionID,
+            tenantID,
+            baseURL,
+            apiURL,
+          })}
           target="_blank"
           rel="noopener noreferrer"
           title={item.title}
@@ -162,7 +186,13 @@ export const RenderMediaItem = ({
               <video
                 style={{ width: '100%', height: '100%' }}
                 controls
-                src={item.url}
+                src={getResourceUrl({
+                  resourceURI: item.url,
+                  sessionID,
+                  tenantID,
+                  baseURL,
+                  apiURL,
+                })}
                 title={item.title}
               />
             }
@@ -177,7 +207,13 @@ export const RenderMediaItem = ({
       return (
         <a
           className="memori-media-item--link"
-          href={item.url}
+          href={getResourceUrl({
+            resourceURI: item.url,
+            sessionID,
+            tenantID,
+            baseURL,
+            apiURL,
+          })}
           target="_blank"
           rel="noopener noreferrer"
           title={item.title}
@@ -188,7 +224,13 @@ export const RenderMediaItem = ({
               <audio
                 style={{ width: '100%', height: '100%' }}
                 controls
-                src={item.url}
+                src={getResourceUrl({
+                  resourceURI: item.url,
+                  sessionID,
+                  tenantID,
+                  baseURL,
+                  apiURL,
+                })}
               />
             }
             title={item.title}
@@ -199,7 +241,13 @@ export const RenderMediaItem = ({
     case 'model/gltf-binary':
       return (
         <ModelViewer
-          src={item.url!}
+          src={getResourceUrl({
+            resourceURI: item.url,
+            sessionID,
+            tenantID,
+            baseURL,
+            apiURL,
+          })}
           alt=""
           poster="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAMIAAADDCAYAAADQvc6UAAABRWlDQ1BJQ0MgUHJvZmlsZQAAKJFjYGASSSwoyGFhYGDIzSspCnJ3UoiIjFJgf8LAwSDCIMogwMCcmFxc4BgQ4ANUwgCjUcG3awyMIPqyLsis7PPOq3QdDFcvjV3jOD1boQVTPQrgSkktTgbSf4A4LbmgqISBgTEFyFYuLykAsTuAbJEioKOA7DkgdjqEvQHEToKwj4DVhAQ5A9k3gGyB5IxEoBmML4BsnSQk8XQkNtReEOBxcfXxUQg1Mjc0dyHgXNJBSWpFCYh2zi+oLMpMzyhRcASGUqqCZ16yno6CkYGRAQMDKMwhqj/fAIcloxgHQqxAjIHBEugw5sUIsSQpBobtQPdLciLEVJYzMPBHMDBsayhILEqEO4DxG0txmrERhM29nYGBddr//5/DGRjYNRkY/l7////39v///y4Dmn+LgeHANwDrkl1AuO+pmgAAADhlWElmTU0AKgAAAAgAAYdpAAQAAAABAAAAGgAAAAAAAqACAAQAAAABAAAAwqADAAQAAAABAAAAwwAAAAD9b/HnAAAHlklEQVR4Ae3dP3PTWBSGcbGzM6GCKqlIBRV0dHRJFarQ0eUT8LH4BnRU0NHR0UEFVdIlFRV7TzRksomPY8uykTk/zewQfKw/9znv4yvJynLv4uLiV2dBoDiBf4qP3/ARuCRABEFAoBEgghggQAQZQKAnYEaQBAQaASKIAQJEkAEEegJmBElAoBEgghggQAQZQKAnYEaQBAQaASKIAQJEkAEEegJmBElAoBEgghggQAQZQKAnYEaQBAQaASKIAQJEkAEEegJmBElAoBEgghggQAQZQKAnYEaQBAQaASKIAQJEkAEEegJmBElAoBEgghggQAQZQKAnYEaQBAQaASKIAQJEkAEEegJmBElAoBEgghggQAQZQKAnYEaQBAQaASKIAQJEkAEEegJmBElAoBEgghggQAQZQKAnYEaQBAQaASKIAQJEkAEEegJmBElAoBEgghggQAQZQKAnYEaQBAQaASKIAQJEkAEEegJmBElAoBEgghggQAQZQKAnYEaQBAQaASKIAQJEkAEEegJmBElAoBEgghggQAQZQKAnYEaQBAQaASKIAQJEkAEEegJmBElAoBEgghggQAQZQKAnYEaQBAQaASKIAQJEkAEEegJmBElAoBEgghggQAQZQKAnYEaQBAQaASKIAQJEkAEEegJmBElAoBEgghgg0Aj8i0JO4OzsrPv69Wv+hi2qPHr0qNvf39+iI97soRIh4f3z58/u7du3SXX7Xt7Z2enevHmzfQe+oSN2apSAPj09TSrb+XKI/f379+08+A0cNRE2ANkupk+ACNPvkSPcAAEibACyXUyfABGm3yNHuAECRNgAZLuYPgEirKlHu7u7XdyytGwHAd8jjNyng4OD7vnz51dbPT8/7z58+NB9+/bt6jU/TI+AGWHEnrx48eJ/EsSmHzx40L18+fLyzxF3ZVMjEyDCiEDjMYZZS5wiPXnyZFbJaxMhQIQRGzHvWR7XCyOCXsOmiDAi1HmPMMQjDpbpEiDCiL358eNHurW/5SnWdIBbXiDCiA38/Pnzrce2YyZ4//59F3ePLNMl4PbpiL2J0L979+7yDtHDhw8vtzzvdGnEXdvUigSIsCLAWavHp/+qM0BcXMd/q25n1vF57TYBp0a3mUzilePj4+7k5KSLb6gt6ydAhPUzXnoPR0dHl79WGTNCfBnn1uvSCJdegQhLI1vvCk+fPu2ePXt2tZOYEV6/fn31dz+shwAR1sP1cqvLntbEN9MxA9xcYjsxS1jWR4AIa2Ibzx0tc44fYX/16lV6NDFLXH+YL32jwiACRBiEbf5KcXoTIsQSpzXx4N28Ja4BQoK7rgXiydbHjx/P25TaQAJEGAguWy0+2Q8PD6/Ki4R8EVl+bzBOnZY95fq9rj9zAkTI2SxdidBHqG9+skdw43borCXO/ZcJdraPWdv22uIEiLA4q7nvvCug8WTqzQveOH26fodo7g6uFe/a17W3+nFBAkRYENRdb1vkkz1CH9cPsVy/jrhr27PqMYvENYNlHAIesRiBYwRy0V+8iXP8+/fvX11Mr7L7ECueb/r48eMqm7FuI2BGWDEG8cm+7G3NEOfmdcTQw4h9/55lhm7DekRYKQPZF2ArbXTAyu4kDYB2YxUzwg0gi/41ztHnfQG26HbGel/crVrm7tNY+/1btkOEAZ2M05r4FB7r9GbAIdxaZYrHdOsgJ/wCEQY0J74TmOKnbxxT9n3FgGGWWsVdowHtjt9Nnvf7yQM2aZU/TIAIAxrw6dOnAWtZZcoEnBpNuTuObWMEiLAx1HY0ZQJEmHJ3HNvGCBBhY6jtaMoEiJB0Z29vL6ls58vxPcO8/zfrdo5qvKO+d3Fx8Wu8zf1dW4p/cPzLly/dtv9Ts/EbcvGAHhHyfBIhZ6NSiIBTo0LNNtScABFyNiqFCBChULMNNSdAhJyNSiECRCjUbEPNCRAhZ6NSiAARCjXbUHMCRMjZqBQiQIRCzTbUnAARcjYqhQgQoVCzDTUnQIScjUohAkQo1GxDzQkQIWejUogAEQo121BzAkTI2agUIkCEQs021JwAEXI2KoUIEKFQsw01J0CEnI1KIQJEKNRsQ80JECFno1KIABEKNdtQcwJEyNmoFCJAhELNNtScABFyNiqFCBChULMNNSdAhJyNSiECRCjUbEPNCRAhZ6NSiAARCjXbUHMCRMjZqBQiQIRCzTbUnAARcjYqhQgQoVCzDTUnQIScjUohAkQo1GxDzQkQIWejUogAEQo121BzAkTI2agUIkCEQs021JwAEXI2KoUIEKFQsw01J0CEnI1KIQJEKNRsQ80JECFno1KIABEKNdtQcwJEyNmoFCJAhELNNtScABFyNiqFCBChULMNNSdAhJyNSiECRCjUbEPNCRAhZ6NSiAARCjXbUHMCRMjZqBQiQIRCzTbUnAARcjYqhQgQoVCzDTUnQIScjUohAkQo1GxDzQkQIWejUogAEQo121BzAkTI2agUIkCEQs021JwAEXI2KoUIEKFQsw01J0CEnI1KIQJEKNRsQ80JECFno1KIABEKNdtQcwJEyNmoFCJAhELNNtScABFyNiqFCBChULMNNSdAhJyNSiEC/wGgKKC4YMA4TAAAAABJRU5ErkJggg=="
         />
@@ -209,7 +257,13 @@ export const RenderMediaItem = ({
       return (
         <a
           className="memori-media-item--link"
-          href={item.url}
+          href={getResourceUrl({
+            resourceURI: item.url,
+            sessionID,
+            tenantID,
+            baseURL,
+            apiURL,
+          })}
           target="_blank"
           rel="noopener noreferrer"
           title={item.title}

package/src/components/MemoriWidget/MemoriWidget.stories.tsx

@@ -57,7 +57,23 @@ WithPublicPageIntegration.args = {
       ...JSON.parse(integration.customData ?? '{}'),
       avatar: 'readyplayerme',
       avatarURL:
-        'https://assets.memori.ai/api/v2/asset/b791f77c-1a94-4272-829e-eca82fcc62b7.glb' +
+        'https://assets.memori.ai/api/v2/asset/b791f77c-1a94-4272-829e-eca82fcc62b7.glb#' +
+        new Date(Date.now()).getTime(),
+    }),
+  },
+};
+
+export const WithPublicPageIntegrationAndFullbodyAvatar = Template.bind({});
+WithPublicPageIntegrationAndFullbodyAvatar.args = {
+  memori,
+  tenant,
+  integration: {
+    ...integration,
+    customData: JSON.stringify({
+      ...JSON.parse(integration.customData ?? '{}'),
+      avatar: 'readyplayerme-full',
+      avatarURL:
+        'https://models.readyplayer.me/63b55751f17e295642bf07a2.glb#' +
         new Date(Date.now()).getTime(),
     }),
   },

package/src/components/MemoriWidget/MemoriWidget.tsx

@@ -517,6 +517,9 @@ const MemoriWidget = ({
         translatedState = {
           ...state,
           emission: t.text,
+          hints:
+            state.hints ??
+            (state.state === 'G1' ? currentDialogState?.hints : []),
         };
       }
 
@@ -823,7 +826,12 @@ const MemoriWidget = ({
             generatedByAI: !!currentState.completion,
           });
           speak(emission);
-          setCurrentDialogState(currentState);
+          setCurrentDialogState({
+            ...currentState,
+            hints:
+              currentState.hints ??
+              (currentState.state === 'G1' ? currentDialogState?.hints : []),
+          });
         }
       }
     }
@@ -1140,6 +1148,13 @@ const MemoriWidget = ({
           console.error('stopAudio error: ', e);
         }
       }
+    }
+    if (audioContext.state === 'closed') {
+      audioContext = new AudioContext();
+      let buffer = audioContext.createBuffer(1, 10000, 22050);
+      let source = audioContext.createBufferSource();
+      source.buffer = buffer;
+      source.connect(audioContext.destination);
     } else if (audioContext.state === 'suspended') {
       stopAudio();
 
@@ -1235,10 +1250,10 @@ const MemoriWidget = ({
       }
     );
 
-    setIsPlayingAudio(false);
     setMemoriTyping(false);
   };
   const stopAudio = () => {
+    setIsPlayingAudio(false);
     try {
       if (speechSynthesizer) {
         speechSynthesizer.close();
@@ -1977,6 +1992,7 @@ const MemoriWidget = ({
       position={position}
       setShowPositionDrawer={setShowPositionDrawer}
       setShowSettingsDrawer={setShowSettingsDrawer}
+      showSpeaker={!!AZURE_COGNITIVE_SERVICES_TTS_KEY}
       speakerMuted={muteSpeaker}
       setSpeakerMuted={mute => {
         setMuteSpeaker(mute);
@@ -2006,6 +2022,7 @@ const MemoriWidget = ({
       setAvatar3dVisible={setAvatar3dVisible}
       hasUserActivatedSpeak={hasUserActivatedSpeak}
       isPlayingAudio={isPlayingAudio}
+      loading={memoriTyping}
       baseUrl={baseUrl}
       apiUrl={apiUrl}
     />