@memori.ai/memori-react 7.8.0-rc.0 → 7.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/dist/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.d.ts +1 -1
- package/dist/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.js +113 -103
- package/dist/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.js.map +1 -1
- package/dist/components/MemoriWidget/MemoriWidget.js +15 -24
- package/dist/components/MemoriWidget/MemoriWidget.js.map +1 -1
- package/dist/components/layouts/ZoomedFullBody.js +7 -0
- package/dist/components/layouts/ZoomedFullBody.js.map +1 -1
- package/dist/components/layouts/zoomed-full-body.css +1 -1
- package/dist/context/visemeContext.js +2 -2
- package/dist/context/visemeContext.js.map +1 -1
- package/esm/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.d.ts +1 -1
- package/esm/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.js +115 -105
- package/esm/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.js.map +1 -1
- package/esm/components/MemoriWidget/MemoriWidget.js +15 -24
- package/esm/components/MemoriWidget/MemoriWidget.js.map +1 -1
- package/esm/components/layouts/ZoomedFullBody.js +7 -0
- package/esm/components/layouts/ZoomedFullBody.js.map +1 -1
- package/esm/components/layouts/zoomed-full-body.css +1 -1
- package/esm/context/visemeContext.js +2 -2
- package/esm/context/visemeContext.js.map +1 -1
- package/package.json +1 -1
- package/src/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.tsx +86 -92
- package/src/components/MemoriWidget/MemoriWidget.tsx +26 -33
- package/src/components/layouts/ZoomedFullBody.tsx +10 -1
- package/src/components/layouts/zoomed-full-body.css +1 -1
- package/src/context/visemeContext.tsx +2 -2
- package/src/index.stories.tsx +41 -14
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
import { jsx as _jsx } from "react/jsx-runtime";
|
|
2
|
-
import { useEffect, useRef,
|
|
2
|
+
import { useEffect, useRef, useMemo, useCallback } from 'react';
|
|
3
3
|
import { Vector3, Euler, AnimationMixer, SkinnedMesh, MathUtils, LoopOnce, } from 'three';
|
|
4
4
|
import { useAnimations, useGLTF } from '@react-three/drei';
|
|
5
|
-
import {
|
|
6
|
-
import { correctMaterials, isSkinnedMesh } from '../../../../../helpers/utils';
|
|
5
|
+
import { useFrame } from '@react-three/fiber';
|
|
7
6
|
const AVATAR_POSITION = new Vector3(0, -1, 0);
|
|
8
7
|
const AVATAR_ROTATION = new Euler(0.175, 0, 0);
|
|
9
8
|
const AVATAR_POSITION_ZOOMED = new Vector3(0, -1.45, 0);
|
|
@@ -16,45 +15,44 @@ const BLINK_CONFIG = {
|
|
|
16
15
|
maxInterval: 5000,
|
|
17
16
|
blinkDuration: 150,
|
|
18
17
|
};
|
|
19
|
-
const
|
|
20
|
-
|
|
18
|
+
const EMOTION_SMOOTHING = 0.3;
|
|
19
|
+
const VISME_SMOOTHING = 0.5;
|
|
20
|
+
export default function FullbodyAvatar({ url, sex, currentBaseAction, timeScale, isZoomed, eyeBlink, updateCurrentViseme, setMorphTargetDictionary, setMorphTargetInfluences, emotionMorphTargets, }) {
|
|
21
21
|
const { scene } = useGLTF(url);
|
|
22
22
|
const { animations } = useGLTF(ANIMATION_URLS[sex]);
|
|
23
|
-
const { nodes, materials } = useGraph(scene);
|
|
24
23
|
const { actions } = useAnimations(animations, scene);
|
|
25
|
-
const
|
|
24
|
+
const mixerRef = useRef();
|
|
26
25
|
const headMeshRef = useRef();
|
|
27
26
|
const currentActionRef = useRef(null);
|
|
28
|
-
const
|
|
29
|
-
const
|
|
30
|
-
const
|
|
31
|
-
const
|
|
32
|
-
const
|
|
27
|
+
const isTransitioningToIdleRef = useRef(false);
|
|
28
|
+
const lastBlinkTimeRef = useRef(0);
|
|
29
|
+
const nextBlinkTimeRef = useRef(0);
|
|
30
|
+
const isBlinkingRef = useRef(false);
|
|
31
|
+
const blinkStartTimeRef = useRef(0);
|
|
33
32
|
const currentEmotionRef = useRef({});
|
|
34
33
|
const previousEmotionKeysRef = useRef(new Set());
|
|
35
|
-
|
|
36
|
-
|
|
34
|
+
const headMesh = useMemo(() => {
|
|
35
|
+
let foundMesh;
|
|
37
36
|
scene.traverse((object) => {
|
|
38
|
-
if (object instanceof SkinnedMesh
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
if (object.morphTargetDictionary && object.morphTargetInfluences) {
|
|
42
|
-
setMorphTargetDictionary(object.morphTargetDictionary);
|
|
43
|
-
const initialInfluences = Object.keys(object.morphTargetDictionary).reduce((acc, key) => ({ ...acc, [key]: 0 }), {});
|
|
44
|
-
setMorphTargetInfluences(initialInfluences);
|
|
45
|
-
}
|
|
46
|
-
}
|
|
37
|
+
if (object instanceof SkinnedMesh &&
|
|
38
|
+
(object.name === 'GBNL__Head' || object.name === 'Wolf3D_Avatar')) {
|
|
39
|
+
foundMesh = object;
|
|
47
40
|
}
|
|
48
41
|
});
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
Object.values(materials).forEach(material => material.dispose());
|
|
52
|
-
Object.values(nodes)
|
|
53
|
-
.filter(isSkinnedMesh)
|
|
54
|
-
.forEach(mesh => mesh.geometry.dispose());
|
|
55
|
-
};
|
|
56
|
-
}, [materials, nodes, url, onLoaded, scene]);
|
|
42
|
+
return foundMesh;
|
|
43
|
+
}, [scene]);
|
|
57
44
|
useEffect(() => {
|
|
45
|
+
if (headMesh) {
|
|
46
|
+
headMeshRef.current = headMesh;
|
|
47
|
+
if (headMesh.morphTargetDictionary && headMesh.morphTargetInfluences) {
|
|
48
|
+
setMorphTargetDictionary(headMesh.morphTargetDictionary);
|
|
49
|
+
const initialInfluences = Object.keys(headMesh.morphTargetDictionary).reduce((acc, key) => ({ ...acc, [key]: 0 }), {});
|
|
50
|
+
setMorphTargetInfluences(initialInfluences);
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
mixerRef.current = new AnimationMixer(scene);
|
|
54
|
+
}, [headMesh, scene, setMorphTargetDictionary, setMorphTargetInfluences]);
|
|
55
|
+
const handleAnimationChange = useCallback(() => {
|
|
58
56
|
if (!actions || !currentBaseAction.action)
|
|
59
57
|
return;
|
|
60
58
|
const newAction = actions[currentBaseAction.action];
|
|
@@ -67,7 +65,6 @@ export default function FullbodyAvatar({ url, sex, onLoaded, currentBaseAction,
|
|
|
67
65
|
if (currentActionRef.current) {
|
|
68
66
|
currentActionRef.current.fadeOut(fadeOutDuration);
|
|
69
67
|
}
|
|
70
|
-
console.log(newAction);
|
|
71
68
|
newAction.reset().fadeIn(fadeInDuration).play();
|
|
72
69
|
currentActionRef.current = newAction;
|
|
73
70
|
newAction.timeScale = timeScale;
|
|
@@ -78,91 +75,104 @@ export default function FullbodyAvatar({ url, sex, onLoaded, currentBaseAction,
|
|
|
78
75
|
currentBaseAction.action.startsWith('Tristezza')) {
|
|
79
76
|
newAction.setLoop(LoopOnce, 1);
|
|
80
77
|
newAction.clampWhenFinished = true;
|
|
81
|
-
|
|
78
|
+
isTransitioningToIdleRef.current = true;
|
|
82
79
|
}
|
|
83
80
|
}, [actions, currentBaseAction, timeScale]);
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
81
|
+
useEffect(() => {
|
|
82
|
+
handleAnimationChange();
|
|
83
|
+
}, [handleAnimationChange]);
|
|
84
|
+
const updateFrame = useCallback((currentTime) => {
|
|
85
|
+
var _a;
|
|
86
|
+
if (!headMeshRef.current ||
|
|
87
|
+
!headMeshRef.current.morphTargetDictionary ||
|
|
88
|
+
!headMeshRef.current.morphTargetInfluences)
|
|
89
|
+
return;
|
|
90
|
+
let blinkValue = 0;
|
|
91
|
+
if (eyeBlink) {
|
|
92
|
+
if (currentTime >= nextBlinkTimeRef.current && !isBlinkingRef.current) {
|
|
93
|
+
isBlinkingRef.current = true;
|
|
94
|
+
blinkStartTimeRef.current = currentTime;
|
|
95
|
+
lastBlinkTimeRef.current = currentTime;
|
|
96
|
+
nextBlinkTimeRef.current =
|
|
97
|
+
currentTime +
|
|
98
|
+
Math.random() *
|
|
99
|
+
(BLINK_CONFIG.maxInterval - BLINK_CONFIG.minInterval) +
|
|
100
|
+
BLINK_CONFIG.minInterval;
|
|
101
|
+
}
|
|
102
|
+
if (isBlinkingRef.current) {
|
|
103
|
+
const blinkProgress = (currentTime - blinkStartTimeRef.current) /
|
|
104
|
+
BLINK_CONFIG.blinkDuration;
|
|
105
|
+
if (blinkProgress <= 0.5) {
|
|
106
|
+
blinkValue = blinkProgress * 2;
|
|
100
107
|
}
|
|
101
|
-
if (
|
|
102
|
-
|
|
103
|
-
if (blinkProgress <= 0.5) {
|
|
104
|
-
blinkValue = blinkProgress * 2;
|
|
105
|
-
}
|
|
106
|
-
else if (blinkProgress <= 1) {
|
|
107
|
-
blinkValue = 2 - blinkProgress * 2;
|
|
108
|
-
}
|
|
109
|
-
else {
|
|
110
|
-
isBlinking.current = false;
|
|
111
|
-
blinkValue = 0;
|
|
112
|
-
}
|
|
108
|
+
else if (blinkProgress <= 1) {
|
|
109
|
+
blinkValue = 2 - blinkProgress * 2;
|
|
113
110
|
}
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
previousEmotionKeysRef.current.forEach(key => {
|
|
118
|
-
if (!currentEmotionKeys.has(key)) {
|
|
119
|
-
const index = headMeshRef.current.morphTargetDictionary[key];
|
|
120
|
-
if (typeof index === 'number') {
|
|
121
|
-
currentEmotionRef.current[key] = 0;
|
|
122
|
-
if (headMeshRef.current && headMeshRef.current.morphTargetInfluences) {
|
|
123
|
-
headMeshRef.current.morphTargetInfluences[index] = 0;
|
|
124
|
-
}
|
|
125
|
-
}
|
|
111
|
+
else {
|
|
112
|
+
isBlinkingRef.current = false;
|
|
113
|
+
blinkValue = 0;
|
|
126
114
|
}
|
|
127
|
-
}
|
|
128
|
-
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
const currentViseme = updateCurrentViseme(currentTime / 1000);
|
|
118
|
+
const currentEmotionKeys = new Set(Object.keys(emotionMorphTargets));
|
|
119
|
+
previousEmotionKeysRef.current.forEach(key => {
|
|
120
|
+
if (!currentEmotionKeys.has(key)) {
|
|
121
|
+
const index = headMeshRef.current.morphTargetDictionary[key];
|
|
129
122
|
if (typeof index === 'number') {
|
|
130
|
-
|
|
131
|
-
if (Object.prototype.hasOwnProperty.call(emotionMorphTargets, key)) {
|
|
132
|
-
const targetEmotionValue = emotionMorphTargets[key];
|
|
133
|
-
const currentEmotionValue = currentEmotionRef.current[key] || 0;
|
|
134
|
-
const newEmotionValue = MathUtils.lerp(currentEmotionValue, targetEmotionValue * 2, EMOTION_TRANSITION_SPEED);
|
|
135
|
-
currentEmotionRef.current[key] = newEmotionValue;
|
|
136
|
-
targetValue += newEmotionValue;
|
|
137
|
-
}
|
|
138
|
-
if (currentViseme && key === currentViseme.name) {
|
|
139
|
-
targetValue += currentViseme.weight * 1.3;
|
|
140
|
-
}
|
|
141
|
-
if (key === 'eyesClosed' && eyeBlink) {
|
|
142
|
-
targetValue += blinkValue;
|
|
143
|
-
}
|
|
144
|
-
targetValue = MathUtils.clamp(targetValue, 0, 1);
|
|
123
|
+
currentEmotionRef.current[key] = 0;
|
|
145
124
|
if (headMeshRef.current && headMeshRef.current.morphTargetInfluences) {
|
|
146
|
-
headMeshRef.current.morphTargetInfluences[index] =
|
|
125
|
+
headMeshRef.current.morphTargetInfluences[index] = 0;
|
|
147
126
|
}
|
|
148
127
|
}
|
|
149
|
-
}
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
const
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
128
|
+
}
|
|
129
|
+
});
|
|
130
|
+
Object.entries(headMeshRef.current.morphTargetDictionary).forEach(([key, index]) => {
|
|
131
|
+
if (typeof index === 'number') {
|
|
132
|
+
let targetValue = 0;
|
|
133
|
+
if (currentEmotionKeys.has(key)) {
|
|
134
|
+
const targetEmotionValue = emotionMorphTargets[key];
|
|
135
|
+
const currentEmotionValue = currentEmotionRef.current[key] || 0;
|
|
136
|
+
const newEmotionValue = MathUtils.lerp(currentEmotionValue, targetEmotionValue * 2.5, EMOTION_SMOOTHING);
|
|
137
|
+
currentEmotionRef.current[key] = newEmotionValue;
|
|
138
|
+
targetValue += newEmotionValue;
|
|
139
|
+
}
|
|
140
|
+
if (currentViseme && key === currentViseme.name) {
|
|
141
|
+
targetValue += currentViseme.weight * 1;
|
|
142
|
+
}
|
|
143
|
+
if (key === 'eyesClosed' && eyeBlink) {
|
|
144
|
+
targetValue += blinkValue;
|
|
145
|
+
}
|
|
146
|
+
targetValue = MathUtils.clamp(targetValue, 0, 1);
|
|
147
|
+
if (headMeshRef.current &&
|
|
148
|
+
headMeshRef.current.morphTargetInfluences) {
|
|
149
|
+
headMeshRef.current.morphTargetInfluences[index] = MathUtils.lerp(headMeshRef.current.morphTargetInfluences[index], targetValue, VISME_SMOOTHING);
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
});
|
|
153
|
+
previousEmotionKeysRef.current = currentEmotionKeys;
|
|
154
|
+
if (isTransitioningToIdleRef.current && currentActionRef.current) {
|
|
155
|
+
if (currentActionRef.current.time >=
|
|
156
|
+
currentActionRef.current.getClip().duration) {
|
|
157
|
+
const idleNumber = Math.floor(Math.random() * 5) + 1;
|
|
158
|
+
const idleAction = actions[`Idle${idleNumber === 3 ? 4 : idleNumber}`];
|
|
159
|
+
if (idleAction) {
|
|
160
|
+
currentActionRef.current.fadeOut(0.5);
|
|
161
|
+
idleAction.reset().fadeIn(0.5).play();
|
|
162
|
+
currentActionRef.current = idleAction;
|
|
163
|
+
isTransitioningToIdleRef.current = false;
|
|
162
164
|
}
|
|
163
165
|
}
|
|
164
|
-
mixer.current.update(0.01);
|
|
165
166
|
}
|
|
167
|
+
(_a = mixerRef.current) === null || _a === void 0 ? void 0 : _a.update(0.01);
|
|
168
|
+
}, [
|
|
169
|
+
actions,
|
|
170
|
+
emotionMorphTargets,
|
|
171
|
+
eyeBlink,
|
|
172
|
+
updateCurrentViseme,
|
|
173
|
+
]);
|
|
174
|
+
useFrame(state => {
|
|
175
|
+
updateFrame(state.clock.getElapsedTime() * 1000);
|
|
166
176
|
});
|
|
167
177
|
return (_jsx("group", { position: isZoomed ? AVATAR_POSITION_ZOOMED : AVATAR_POSITION, rotation: AVATAR_ROTATION, children: _jsx("primitive", { object: scene }) }));
|
|
168
178
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fullbodyAvatar.js","sourceRoot":"","sources":["../../../../../../src/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.tsx"],"names":[],"mappings":";AAAA,OAAc,EAAE,SAAS,EAAE,MAAM,EAAE,
|
|
1
|
+
{"version":3,"file":"fullbodyAvatar.js","sourceRoot":"","sources":["../../../../../../src/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.tsx"],"names":[],"mappings":";AAAA,OAAc,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,OAAO,CAAC;AACvE,OAAO,EACL,OAAO,EACP,KAAK,EACL,cAAc,EACd,WAAW,EAEX,SAAS,EAET,QAAQ,GACT,MAAM,OAAO,CAAC;AACf,OAAO,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAC3D,OAAO,EAAY,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AA+BxD,MAAM,eAAe,GAAG,IAAI,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AAC9C,MAAM,eAAe,GAAG,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;AAC/C,MAAM,sBAAsB,GAAG,IAAI,OAAO,CAAC,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;AAExD,MAAM,cAAc,GAAG;IACrB,IAAI,EAAE,gFAAgF;IACtF,MAAM,EACJ,gFAAgF;CACnF,CAAC;AAEF,MAAM,YAAY,GAAG;IACnB,WAAW,EAAE,IAAI;IACjB,WAAW,EAAE,IAAI;IACjB,aAAa,EAAE,GAAG;CACnB,CAAC;AAEF,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAC9B,MAAM,eAAe,GAAG,GAAG,CAAC;AAG5B,MAAM,CAAC,OAAO,UAAU,cAAc,CAAC,EACrC,GAAG,EACH,GAAG,EACH,iBAAiB,EACjB,SAAS,EACT,QAAQ,EACR,QAAQ,EACR,mBAAmB,EACnB,wBAAwB,EACxB,wBAAwB,EACxB,mBAAmB,GACC;IACpB,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IAC/B,MAAM,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC;IACpD,MAAM,EAAE,OAAO,EAAE,GAAG,aAAa,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;IAErD,MAAM,QAAQ,GAAG,MAAM,EAAkB,CAAC;IAC1C,MAAM,WAAW,GAAG,MAAM,EAAe,CAAC;IAC1C,MAAM,gBAAgB,GAAG,MAAM,CAAyB,IAAI,CAAC,CAAC;IAC9D,MAAM,wBAAwB,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAE/C,MAAM,gBAAgB,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;IACnC,MAAM,gBAAgB,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;IACnC,MAAM,aAAa,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IACpC,MAAM,iBAAiB,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;IAEpC,MAAM,iBAAiB,GAAG,MAAM,CAAyB,EAAE,CAAC,CAAC;IAC7D,MAAM,sBAAsB,GAAG,MAAM,CAAc,IAAI,GAAG,EAAE,CAAC,CAAC;IAG9D,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,EAAE;QAC5B,IAAI,SAAkC,CAAC;QACvC,KAAK,CAAC,QAAQ,CAAC,CAAC,MAAgB,EAAE,EAAE;YAClC,IACE,MAAM,YAAY,WAAW;gBAC7B,CAAC,MAAM,CAAC,IAAI,KAAK,YAAY,IAAI,MAAM,CAAC,IAAI,KAAK,eAAe,CAAC,EACjE;gBACA,SAAS,GAAG,MAAM,CAAC;aACpB;QACH,CAAC,CAAC,CAAC;QACH,OAAO,SAAS,CAAC;IACnB,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC;IAEZ,SAAS,CAAC,GAAG,EAAE;QACb,IAAI,QAAQ,EAAE;YACZ,WAAW,CAAC,OAAO,GAAG,QAAQ,CAAC;YAC/B,IAAI,QAAQ,CAAC,qBAAqB,IAAI,QAAQ,CAAC,qBAAqB,EAAE;gBACpE,wBAAwB,CAAC,QAAQ,CAAC,qBAAqB,CAAC,CAAC;gBACzD,MAAM,iBAAiB,GAAG,MAAM,CAAC,IAAI,CACnC,QAAQ,CAAC,qBAAqB,CAC/B,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,GAAG,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;gBACnD,wBAAwB,CAAC,iBAAiB,CAAC,CAAC;aAC7C;SACF;QACD,QAAQ,CAAC,OAAO,GAAG,IAAI,cAAc,CAAC,KAAK,CAAC,CAAC;IAC/C,CAAC,EAAE,CAAC,QAAQ,EAAE,KAAK,EAAE,wBAAwB,EAAE,wBAAwB,CAAC,CAAC,CAAC;IAG1E,MAAM,qBAAqB,GAAG,WAAW,CAAC,GAAG,EAAE;QAC7C,IAAI,CAAC,OAAO,IAAI,CAAC,iBAAiB,CAAC,MAAM;YAAE,OAAO;QAElD,MAAM,SAAS,GAAG,OAAO,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC;QACpD,IAAI,CAAC,SAAS,EAAE;YACd,OAAO,CAAC,IAAI,CACV,cAAc,iBAAiB,CAAC,MAAM,yBAAyB,CAChE,CAAC;YACF,OAAO;SACR;QAED,MAAM,eAAe,GAAG,GAAG,CAAC;QAC5B,MAAM,cAAc,GAAG,GAAG,CAAC;QAE3B,IAAI,gBAAgB,CAAC,OAAO,EAAE;YAC5B,gBAAgB,CAAC,OAAO,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC;SACnD;QAED,SAAS,CAAC,KAAK,EAAE,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,IAAI,EAAE,CAAC;QAChD,gBAAgB,CAAC,OAAO,GAAG,SAAS,CAAC;QACrC,SAAS,CAAC,SAAS,GAAG,SAAS,CAAC;QAEhC,IACE,iBAAiB,CAAC,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC;YAC5C,iBAAiB,CAAC,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC;YAC7C,iBAAiB,CAAC,MAAM,CAAC,UAAU,CAAC,UAAU,CAAC;YAC/C,iBAAiB,CAAC,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC;YAC7C,iBAAiB,CAAC,MAAM,CAAC,UAAU,CAAC,WAAW,CAAC,EAChD;YACA,SAAS,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;YAC/B,SAAS,CAAC,iBAAiB,GAAG,IAAI,CAAC;YACnC,wBAAwB,CAAC,OAAO,GAAG,IAAI,CAAC;SACzC;IACH,CAAC,EAAE,CAAC,OAAO,EAAE,iBAAiB,EAAE,SAAS,CAAC,CAAC,CAAC;IAE5C,SAAS,CAAC,GAAG,EAAE;QACb,qBAAqB,EAAE,CAAC;IAC1B,CAAC,EAAE,CAAC,qBAAqB,CAAC,CAAC,CAAC;IAG5B,MAAM,WAAW,GAAG,WAAW,CAC7B,CAAC,WAAmB,EAAE,EAAE;;QACtB,IACE,CAAC,WAAW,CAAC,OAAO;YACpB,CAAC,WAAW,CAAC,OAAO,CAAC,qBAAqB;YAC1C,CAAC,WAAW,CAAC,OAAO,CAAC,qBAAqB;YAE1C,OAAO;QAET,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,QAAQ,EAAE;YACZ,IAAI,WAAW,IAAI,gBAAgB,CAAC,OAAO,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE;gBACrE,aAAa,CAAC,OAAO,GAAG,IAAI,CAAC;gBAC7B,iBAAiB,CAAC,OAAO,GAAG,WAAW,CAAC;gBACxC,gBAAgB,CAAC,OAAO,GAAG,WAAW,CAAC;gBACvC,gBAAgB,CAAC,OAAO;oBACtB,WAAW;wBACX,IAAI,CAAC,MAAM,EAAE;4BACX,CAAC,YAAY,CAAC,WAAW,GAAG,YAAY,CAAC,WAAW,CAAC;wBACvD,YAAY,CAAC,WAAW,CAAC;aAC5B;YAED,IAAI,aAAa,CAAC,OAAO,EAAE;gBACzB,MAAM,aAAa,GACjB,CAAC,WAAW,GAAG,iBAAiB,CAAC,OAAO,CAAC;oBACzC,YAAY,CAAC,aAAa,CAAC;gBAC7B,IAAI,aAAa,IAAI,GAAG,EAAE;oBACxB,UAAU,GAAG,aAAa,GAAG,CAAC,CAAC;iBAChC;qBAAM,IAAI,aAAa,IAAI,CAAC,EAAE;oBAC7B,UAAU,GAAG,CAAC,GAAG,aAAa,GAAG,CAAC,CAAC;iBACpC;qBAAM;oBACL,aAAa,CAAC,OAAO,GAAG,KAAK,CAAC;oBAC9B,UAAU,GAAG,CAAC,CAAC;iBAChB;aACF;SACF;QAED,MAAM,aAAa,GAAG,mBAAmB,CAAC,WAAW,GAAG,IAAI,CAAC,CAAC;QAC9D,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC,CAAC;QAErE,sBAAsB,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE;YAC3C,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;gBAChC,MAAM,KAAK,GAAG,WAAW,CAAC,OAAQ,CAAC,qBAAsB,CAAC,GAAG,CAAC,CAAC;gBAC/D,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;oBAC7B,iBAAiB,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;oBACnC,IAAI,WAAW,CAAC,OAAO,IAAI,WAAW,CAAC,OAAO,CAAC,qBAAqB,EAAE;wBACpE,WAAW,CAAC,OAAO,CAAC,qBAAqB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;qBACtD;iBACF;aACF;QACH,CAAC,CAAC,CAAC;QAEH,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,OAAO,CAAC,qBAAqB,CAAC,CAAC,OAAO,CAC/D,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;YACf,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;gBAC7B,IAAI,WAAW,GAAG,CAAC,CAAC;gBAEpB,IAAI,kBAAkB,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;oBAC/B,MAAM,kBAAkB,GAAG,mBAAmB,CAAC,GAAG,CAAC,CAAC;oBACpD,MAAM,mBAAmB,GAAG,iBAAiB,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;oBAChE,MAAM,eAAe,GAAG,SAAS,CAAC,IAAI,CACpC,mBAAmB,EACnB,kBAAkB,GAAG,GAAG,EACxB,iBAAiB,CAClB,CAAC;oBACF,iBAAiB,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,eAAe,CAAC;oBACjD,WAAW,IAAI,eAAe,CAAC;iBAChC;gBAED,IAAI,aAAa,IAAI,GAAG,KAAK,aAAa,CAAC,IAAI,EAAE;oBAC/C,WAAW,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC;iBACzC;gBAED,IAAI,GAAG,KAAK,YAAY,IAAI,QAAQ,EAAE;oBACpC,WAAW,IAAI,UAAU,CAAC;iBAC3B;gBAED,WAAW,GAAG,SAAS,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;gBACjD,IACE,WAAW,CAAC,OAAO;oBACnB,WAAW,CAAC,OAAO,CAAC,qBAAqB,EACzC;oBACA,WAAW,CAAC,OAAO,CAAC,qBAAqB,CAAC,KAAK,CAAC,GAAG,SAAS,CAAC,IAAI,CAC/D,WAAW,CAAC,OAAO,CAAC,qBAAqB,CAAC,KAAK,CAAC,EAChD,WAAW,EACX,eAAe,CAChB,CAAC;iBACH;aACF;QACH,CAAC,CACF,CAAC;QAEF,sBAAsB,CAAC,OAAO,GAAG,kBAAkB,CAAC;QAEpD,IAAI,wBAAwB,CAAC,OAAO,IAAI,gBAAgB,CAAC,OAAO,EAAE;YAChE,IACE,gBAAgB,CAAC,OAAO,CAAC,IAAI;gBAC7B,gBAAgB,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,QAAQ,EAC3C;gBACA,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;gBACrD,MAAM,UAAU,GACd,OAAO,CAAC,OAAO,UAAU,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC;gBAEtD,IAAI,UAAU,EAAE;oBACd,gBAAgB,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;oBACtC,UAAU,CAAC,KAAK,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;oBACtC,gBAAgB,CAAC,OAAO,GAAG,UAAU,CAAC;oBACtC,wBAAwB,CAAC,OAAO,GAAG,KAAK,CAAC;iBAC1C;aACF;SACF;QAED,MAAA,QAAQ,CAAC,OAAO,0CAAE,MAAM,CAAC,IAAI,CAAC,CAAC;IACjC,CAAC,EACD;QACE,OAAO;QACP,mBAAmB;QACnB,QAAQ;QACR,mBAAmB;KACpB,CACF,CAAC;IAEF,QAAQ,CAAC,KAAK,CAAC,EAAE;QACf,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,cAAc,EAAE,GAAG,IAAI,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,OAAO,CACL,gBACE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,eAAe,EAC7D,QAAQ,EAAE,eAAe,YAEzB,oBAAW,MAAM,EAAE,KAAK,GAAI,GACtB,CACT,CAAC;AACJ,CAAC"}
|
|
@@ -216,7 +216,7 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
|
|
|
216
216
|
const [isPlayingAudio, setIsPlayingAudio] = useState(false);
|
|
217
217
|
const [controlsPosition, setControlsPosition] = useState('center');
|
|
218
218
|
const [hideEmissions, setHideEmissions] = useState(false);
|
|
219
|
-
const {
|
|
219
|
+
const { startProcessing, addViseme, stopProcessing, resetVisemeQueue } = useViseme();
|
|
220
220
|
useEffect(() => {
|
|
221
221
|
setIsPlayingAudio(!!speechSynthesizer);
|
|
222
222
|
memoriSpeaking = !!speechSynthesizer;
|
|
@@ -1312,49 +1312,41 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
|
|
|
1312
1312
|
if (result) {
|
|
1313
1313
|
setIsPlayingAudio(true);
|
|
1314
1314
|
memoriSpeaking = true;
|
|
1315
|
+
startProcessing();
|
|
1315
1316
|
try {
|
|
1316
1317
|
audioContext.decodeAudioData(result.audioData, function (buffer) {
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
currentSource.start();
|
|
1323
|
-
currentSource.onended = () => {
|
|
1324
|
-
console.log('ended');
|
|
1325
|
-
setIsPlayingAudio(false);
|
|
1326
|
-
memoriSpeaking = false;
|
|
1327
|
-
stopProcessing();
|
|
1328
|
-
resetVisemeQueue();
|
|
1329
|
-
emitEndSpeakEvent();
|
|
1330
|
-
};
|
|
1318
|
+
source.buffer = buffer;
|
|
1319
|
+
source.connect(audioContext.destination);
|
|
1320
|
+
if (history.length < 1 || (isSafari && isIOS)) {
|
|
1321
|
+
source.start(0);
|
|
1322
|
+
}
|
|
1331
1323
|
});
|
|
1332
1324
|
audioContext.onstatechange = () => {
|
|
1333
1325
|
if (audioContext.state === 'suspended' ||
|
|
1334
1326
|
audioContext.state === 'closed') {
|
|
1335
|
-
console.log('suspended');
|
|
1336
1327
|
source.disconnect();
|
|
1337
1328
|
setIsPlayingAudio(false);
|
|
1338
1329
|
stopProcessing();
|
|
1339
1330
|
resetVisemeQueue();
|
|
1340
1331
|
memoriSpeaking = false;
|
|
1341
|
-
emitEndSpeakEvent();
|
|
1342
1332
|
}
|
|
1343
1333
|
else if (audioContext.state === 'interrupted') {
|
|
1344
|
-
console.log('interrupted');
|
|
1345
|
-
stopProcessing();
|
|
1346
|
-
resetVisemeQueue();
|
|
1347
1334
|
audioContext.resume();
|
|
1348
1335
|
}
|
|
1349
1336
|
};
|
|
1337
|
+
audioContext.resume();
|
|
1338
|
+
if (speechSynthesizer) {
|
|
1339
|
+
speechSynthesizer.close();
|
|
1340
|
+
speechSynthesizer = null;
|
|
1341
|
+
}
|
|
1350
1342
|
}
|
|
1351
1343
|
catch (e) {
|
|
1352
1344
|
console.warn('speak error: ', e);
|
|
1353
1345
|
window.speechSynthesis.speak(new SpeechSynthesisUtterance(text));
|
|
1354
|
-
setIsPlayingAudio(false);
|
|
1355
|
-
memoriSpeaking = false;
|
|
1356
1346
|
stopProcessing();
|
|
1357
1347
|
resetVisemeQueue();
|
|
1348
|
+
setIsPlayingAudio(false);
|
|
1349
|
+
memoriSpeaking = false;
|
|
1358
1350
|
if (speechSynthesizer) {
|
|
1359
1351
|
speechSynthesizer.close();
|
|
1360
1352
|
speechSynthesizer = null;
|
|
@@ -1365,7 +1357,6 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
|
|
|
1365
1357
|
else {
|
|
1366
1358
|
audioContext.resume();
|
|
1367
1359
|
stopProcessing();
|
|
1368
|
-
resetVisemeQueue();
|
|
1369
1360
|
setIsPlayingAudio(false);
|
|
1370
1361
|
memoriSpeaking = false;
|
|
1371
1362
|
emitEndSpeakEvent();
|
|
@@ -1374,9 +1365,9 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
|
|
|
1374
1365
|
console.error('speak:', error);
|
|
1375
1366
|
window.speechSynthesis.speak(new SpeechSynthesisUtterance(text));
|
|
1376
1367
|
setIsPlayingAudio(false);
|
|
1377
|
-
memoriSpeaking = false;
|
|
1378
1368
|
stopProcessing();
|
|
1379
1369
|
resetVisemeQueue();
|
|
1370
|
+
memoriSpeaking = false;
|
|
1380
1371
|
emitEndSpeakEvent();
|
|
1381
1372
|
});
|
|
1382
1373
|
setMemoriTyping(false);
|