@memori.ai/memori-react 7.8.0-rc.0 → 7.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/dist/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.d.ts +1 -1
- package/dist/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.js +113 -103
- package/dist/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.js.map +1 -1
- package/dist/components/MemoriWidget/MemoriWidget.js +15 -24
- package/dist/components/MemoriWidget/MemoriWidget.js.map +1 -1
- package/dist/components/layouts/ZoomedFullBody.js +7 -0
- package/dist/components/layouts/ZoomedFullBody.js.map +1 -1
- package/dist/components/layouts/zoomed-full-body.css +1 -1
- package/dist/context/visemeContext.js +2 -2
- package/dist/context/visemeContext.js.map +1 -1
- package/esm/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.d.ts +1 -1
- package/esm/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.js +115 -105
- package/esm/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.js.map +1 -1
- package/esm/components/MemoriWidget/MemoriWidget.js +15 -24
- package/esm/components/MemoriWidget/MemoriWidget.js.map +1 -1
- package/esm/components/layouts/ZoomedFullBody.js +7 -0
- package/esm/components/layouts/ZoomedFullBody.js.map +1 -1
- package/esm/components/layouts/zoomed-full-body.css +1 -1
- package/esm/context/visemeContext.js +2 -2
- package/esm/context/visemeContext.js.map +1 -1
- package/package.json +1 -1
- package/src/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.tsx +86 -92
- package/src/components/MemoriWidget/MemoriWidget.tsx +26 -33
- package/src/components/layouts/ZoomedFullBody.tsx +10 -1
- package/src/components/layouts/zoomed-full-body.css +1 -1
- package/src/context/visemeContext.tsx +2 -2
- package/src/index.stories.tsx +41 -14
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,22 @@
|
|
|
1
1
|
|
|
2
2
|
|
|
3
|
+
## [7.8.0](https://github.com/memori-ai/memori-react/compare/v7.8.0-rc.1...v7.8.0) (2024-10-18)
|
|
4
|
+
|
|
5
|
+
## [7.8.0-rc.1](https://github.com/memori-ai/memori-react/compare/v7.8.0-rc.0...v7.8.0-rc.1) (2024-10-18)
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
### Features
|
|
9
|
+
|
|
10
|
+
* fine tuned lip sync params ([25f6d53](https://github.com/memori-ai/memori-react/commit/25f6d5328855564f8ae186e31d1f070d472dd0dc))
|
|
11
|
+
* improved fullbody avatar state management ([ddac623](https://github.com/memori-ai/memori-react/commit/ddac623afcd4ca58cd887b35d950cf23d3caf8f2))
|
|
12
|
+
* removed scroll beahviour for ZOOMED FULL BODY ([2c9e1ae](https://github.com/memori-ai/memori-react/commit/2c9e1ae9eb468d68fe9b0892f00b03c8fbb41ed4))
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
### Bug Fixes
|
|
16
|
+
|
|
17
|
+
* adjusted ZOOMED FULL BODY layout style ([cefab1e](https://github.com/memori-ai/memori-react/commit/cefab1e4759a4a01c50cd21c3c828ae6a1cbe3c1))
|
|
18
|
+
* fixed starting twice audio ([d108ea8](https://github.com/memori-ai/memori-react/commit/d108ea89265564b81c6a181347621676780b36af))
|
|
19
|
+
|
|
3
20
|
## [7.8.0-rc.0](https://github.com/memori-ai/memori-react/compare/v7.7.1...v7.8.0-rc.0) (2024-10-17)
|
|
4
21
|
|
|
5
22
|
|
|
@@ -22,5 +22,5 @@ interface FullbodyAvatarProps {
|
|
|
22
22
|
setMorphTargetInfluences: (morphTargetInfluences: Record<string, number>) => void;
|
|
23
23
|
emotionMorphTargets: Record<string, number>;
|
|
24
24
|
}
|
|
25
|
-
export default function FullbodyAvatar({ url, sex,
|
|
25
|
+
export default function FullbodyAvatar({ url, sex, currentBaseAction, timeScale, isZoomed, eyeBlink, updateCurrentViseme, setMorphTargetDictionary, setMorphTargetInfluences, emotionMorphTargets, }: FullbodyAvatarProps): JSX.Element;
|
|
26
26
|
export {};
|
|
@@ -5,7 +5,6 @@ const react_1 = require("react");
|
|
|
5
5
|
const three_1 = require("three");
|
|
6
6
|
const drei_1 = require("@react-three/drei");
|
|
7
7
|
const fiber_1 = require("@react-three/fiber");
|
|
8
|
-
const utils_1 = require("../../../../../helpers/utils");
|
|
9
8
|
const AVATAR_POSITION = new three_1.Vector3(0, -1, 0);
|
|
10
9
|
const AVATAR_ROTATION = new three_1.Euler(0.175, 0, 0);
|
|
11
10
|
const AVATAR_POSITION_ZOOMED = new three_1.Vector3(0, -1.45, 0);
|
|
@@ -18,45 +17,44 @@ const BLINK_CONFIG = {
|
|
|
18
17
|
maxInterval: 5000,
|
|
19
18
|
blinkDuration: 150,
|
|
20
19
|
};
|
|
21
|
-
const
|
|
22
|
-
|
|
20
|
+
const EMOTION_SMOOTHING = 0.3;
|
|
21
|
+
const VISME_SMOOTHING = 0.5;
|
|
22
|
+
function FullbodyAvatar({ url, sex, currentBaseAction, timeScale, isZoomed, eyeBlink, updateCurrentViseme, setMorphTargetDictionary, setMorphTargetInfluences, emotionMorphTargets, }) {
|
|
23
23
|
const { scene } = (0, drei_1.useGLTF)(url);
|
|
24
24
|
const { animations } = (0, drei_1.useGLTF)(ANIMATION_URLS[sex]);
|
|
25
|
-
const { nodes, materials } = (0, fiber_1.useGraph)(scene);
|
|
26
25
|
const { actions } = (0, drei_1.useAnimations)(animations, scene);
|
|
27
|
-
const
|
|
26
|
+
const mixerRef = (0, react_1.useRef)();
|
|
28
27
|
const headMeshRef = (0, react_1.useRef)();
|
|
29
28
|
const currentActionRef = (0, react_1.useRef)(null);
|
|
30
|
-
const
|
|
31
|
-
const
|
|
32
|
-
const
|
|
33
|
-
const
|
|
34
|
-
const
|
|
29
|
+
const isTransitioningToIdleRef = (0, react_1.useRef)(false);
|
|
30
|
+
const lastBlinkTimeRef = (0, react_1.useRef)(0);
|
|
31
|
+
const nextBlinkTimeRef = (0, react_1.useRef)(0);
|
|
32
|
+
const isBlinkingRef = (0, react_1.useRef)(false);
|
|
33
|
+
const blinkStartTimeRef = (0, react_1.useRef)(0);
|
|
35
34
|
const currentEmotionRef = (0, react_1.useRef)({});
|
|
36
35
|
const previousEmotionKeysRef = (0, react_1.useRef)(new Set());
|
|
37
|
-
(0, react_1.
|
|
38
|
-
|
|
36
|
+
const headMesh = (0, react_1.useMemo)(() => {
|
|
37
|
+
let foundMesh;
|
|
39
38
|
scene.traverse((object) => {
|
|
40
|
-
if (object instanceof three_1.SkinnedMesh
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
if (object.morphTargetDictionary && object.morphTargetInfluences) {
|
|
44
|
-
setMorphTargetDictionary(object.morphTargetDictionary);
|
|
45
|
-
const initialInfluences = Object.keys(object.morphTargetDictionary).reduce((acc, key) => ({ ...acc, [key]: 0 }), {});
|
|
46
|
-
setMorphTargetInfluences(initialInfluences);
|
|
47
|
-
}
|
|
48
|
-
}
|
|
39
|
+
if (object instanceof three_1.SkinnedMesh &&
|
|
40
|
+
(object.name === 'GBNL__Head' || object.name === 'Wolf3D_Avatar')) {
|
|
41
|
+
foundMesh = object;
|
|
49
42
|
}
|
|
50
43
|
});
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
Object.values(materials).forEach(material => material.dispose());
|
|
54
|
-
Object.values(nodes)
|
|
55
|
-
.filter(utils_1.isSkinnedMesh)
|
|
56
|
-
.forEach(mesh => mesh.geometry.dispose());
|
|
57
|
-
};
|
|
58
|
-
}, [materials, nodes, url, onLoaded, scene]);
|
|
44
|
+
return foundMesh;
|
|
45
|
+
}, [scene]);
|
|
59
46
|
(0, react_1.useEffect)(() => {
|
|
47
|
+
if (headMesh) {
|
|
48
|
+
headMeshRef.current = headMesh;
|
|
49
|
+
if (headMesh.morphTargetDictionary && headMesh.morphTargetInfluences) {
|
|
50
|
+
setMorphTargetDictionary(headMesh.morphTargetDictionary);
|
|
51
|
+
const initialInfluences = Object.keys(headMesh.morphTargetDictionary).reduce((acc, key) => ({ ...acc, [key]: 0 }), {});
|
|
52
|
+
setMorphTargetInfluences(initialInfluences);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
mixerRef.current = new three_1.AnimationMixer(scene);
|
|
56
|
+
}, [headMesh, scene, setMorphTargetDictionary, setMorphTargetInfluences]);
|
|
57
|
+
const handleAnimationChange = (0, react_1.useCallback)(() => {
|
|
60
58
|
if (!actions || !currentBaseAction.action)
|
|
61
59
|
return;
|
|
62
60
|
const newAction = actions[currentBaseAction.action];
|
|
@@ -69,7 +67,6 @@ function FullbodyAvatar({ url, sex, onLoaded, currentBaseAction, timeScale, isZo
|
|
|
69
67
|
if (currentActionRef.current) {
|
|
70
68
|
currentActionRef.current.fadeOut(fadeOutDuration);
|
|
71
69
|
}
|
|
72
|
-
console.log(newAction);
|
|
73
70
|
newAction.reset().fadeIn(fadeInDuration).play();
|
|
74
71
|
currentActionRef.current = newAction;
|
|
75
72
|
newAction.timeScale = timeScale;
|
|
@@ -80,91 +77,104 @@ function FullbodyAvatar({ url, sex, onLoaded, currentBaseAction, timeScale, isZo
|
|
|
80
77
|
currentBaseAction.action.startsWith('Tristezza')) {
|
|
81
78
|
newAction.setLoop(three_1.LoopOnce, 1);
|
|
82
79
|
newAction.clampWhenFinished = true;
|
|
83
|
-
|
|
80
|
+
isTransitioningToIdleRef.current = true;
|
|
84
81
|
}
|
|
85
82
|
}, [actions, currentBaseAction, timeScale]);
|
|
86
|
-
(0,
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
83
|
+
(0, react_1.useEffect)(() => {
|
|
84
|
+
handleAnimationChange();
|
|
85
|
+
}, [handleAnimationChange]);
|
|
86
|
+
const updateFrame = (0, react_1.useCallback)((currentTime) => {
|
|
87
|
+
var _a;
|
|
88
|
+
if (!headMeshRef.current ||
|
|
89
|
+
!headMeshRef.current.morphTargetDictionary ||
|
|
90
|
+
!headMeshRef.current.morphTargetInfluences)
|
|
91
|
+
return;
|
|
92
|
+
let blinkValue = 0;
|
|
93
|
+
if (eyeBlink) {
|
|
94
|
+
if (currentTime >= nextBlinkTimeRef.current && !isBlinkingRef.current) {
|
|
95
|
+
isBlinkingRef.current = true;
|
|
96
|
+
blinkStartTimeRef.current = currentTime;
|
|
97
|
+
lastBlinkTimeRef.current = currentTime;
|
|
98
|
+
nextBlinkTimeRef.current =
|
|
99
|
+
currentTime +
|
|
100
|
+
Math.random() *
|
|
101
|
+
(BLINK_CONFIG.maxInterval - BLINK_CONFIG.minInterval) +
|
|
102
|
+
BLINK_CONFIG.minInterval;
|
|
103
|
+
}
|
|
104
|
+
if (isBlinkingRef.current) {
|
|
105
|
+
const blinkProgress = (currentTime - blinkStartTimeRef.current) /
|
|
106
|
+
BLINK_CONFIG.blinkDuration;
|
|
107
|
+
if (blinkProgress <= 0.5) {
|
|
108
|
+
blinkValue = blinkProgress * 2;
|
|
102
109
|
}
|
|
103
|
-
if (
|
|
104
|
-
|
|
105
|
-
if (blinkProgress <= 0.5) {
|
|
106
|
-
blinkValue = blinkProgress * 2;
|
|
107
|
-
}
|
|
108
|
-
else if (blinkProgress <= 1) {
|
|
109
|
-
blinkValue = 2 - blinkProgress * 2;
|
|
110
|
-
}
|
|
111
|
-
else {
|
|
112
|
-
isBlinking.current = false;
|
|
113
|
-
blinkValue = 0;
|
|
114
|
-
}
|
|
110
|
+
else if (blinkProgress <= 1) {
|
|
111
|
+
blinkValue = 2 - blinkProgress * 2;
|
|
115
112
|
}
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
previousEmotionKeysRef.current.forEach(key => {
|
|
120
|
-
if (!currentEmotionKeys.has(key)) {
|
|
121
|
-
const index = headMeshRef.current.morphTargetDictionary[key];
|
|
122
|
-
if (typeof index === 'number') {
|
|
123
|
-
currentEmotionRef.current[key] = 0;
|
|
124
|
-
if (headMeshRef.current && headMeshRef.current.morphTargetInfluences) {
|
|
125
|
-
headMeshRef.current.morphTargetInfluences[index] = 0;
|
|
126
|
-
}
|
|
127
|
-
}
|
|
113
|
+
else {
|
|
114
|
+
isBlinkingRef.current = false;
|
|
115
|
+
blinkValue = 0;
|
|
128
116
|
}
|
|
129
|
-
}
|
|
130
|
-
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
const currentViseme = updateCurrentViseme(currentTime / 1000);
|
|
120
|
+
const currentEmotionKeys = new Set(Object.keys(emotionMorphTargets));
|
|
121
|
+
previousEmotionKeysRef.current.forEach(key => {
|
|
122
|
+
if (!currentEmotionKeys.has(key)) {
|
|
123
|
+
const index = headMeshRef.current.morphTargetDictionary[key];
|
|
131
124
|
if (typeof index === 'number') {
|
|
132
|
-
|
|
133
|
-
if (Object.prototype.hasOwnProperty.call(emotionMorphTargets, key)) {
|
|
134
|
-
const targetEmotionValue = emotionMorphTargets[key];
|
|
135
|
-
const currentEmotionValue = currentEmotionRef.current[key] || 0;
|
|
136
|
-
const newEmotionValue = three_1.MathUtils.lerp(currentEmotionValue, targetEmotionValue * 2, EMOTION_TRANSITION_SPEED);
|
|
137
|
-
currentEmotionRef.current[key] = newEmotionValue;
|
|
138
|
-
targetValue += newEmotionValue;
|
|
139
|
-
}
|
|
140
|
-
if (currentViseme && key === currentViseme.name) {
|
|
141
|
-
targetValue += currentViseme.weight * 1.3;
|
|
142
|
-
}
|
|
143
|
-
if (key === 'eyesClosed' && eyeBlink) {
|
|
144
|
-
targetValue += blinkValue;
|
|
145
|
-
}
|
|
146
|
-
targetValue = three_1.MathUtils.clamp(targetValue, 0, 1);
|
|
125
|
+
currentEmotionRef.current[key] = 0;
|
|
147
126
|
if (headMeshRef.current && headMeshRef.current.morphTargetInfluences) {
|
|
148
|
-
headMeshRef.current.morphTargetInfluences[index] =
|
|
127
|
+
headMeshRef.current.morphTargetInfluences[index] = 0;
|
|
149
128
|
}
|
|
150
129
|
}
|
|
151
|
-
}
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
const
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
130
|
+
}
|
|
131
|
+
});
|
|
132
|
+
Object.entries(headMeshRef.current.morphTargetDictionary).forEach(([key, index]) => {
|
|
133
|
+
if (typeof index === 'number') {
|
|
134
|
+
let targetValue = 0;
|
|
135
|
+
if (currentEmotionKeys.has(key)) {
|
|
136
|
+
const targetEmotionValue = emotionMorphTargets[key];
|
|
137
|
+
const currentEmotionValue = currentEmotionRef.current[key] || 0;
|
|
138
|
+
const newEmotionValue = three_1.MathUtils.lerp(currentEmotionValue, targetEmotionValue * 2.5, EMOTION_SMOOTHING);
|
|
139
|
+
currentEmotionRef.current[key] = newEmotionValue;
|
|
140
|
+
targetValue += newEmotionValue;
|
|
141
|
+
}
|
|
142
|
+
if (currentViseme && key === currentViseme.name) {
|
|
143
|
+
targetValue += currentViseme.weight * 1;
|
|
144
|
+
}
|
|
145
|
+
if (key === 'eyesClosed' && eyeBlink) {
|
|
146
|
+
targetValue += blinkValue;
|
|
147
|
+
}
|
|
148
|
+
targetValue = three_1.MathUtils.clamp(targetValue, 0, 1);
|
|
149
|
+
if (headMeshRef.current &&
|
|
150
|
+
headMeshRef.current.morphTargetInfluences) {
|
|
151
|
+
headMeshRef.current.morphTargetInfluences[index] = three_1.MathUtils.lerp(headMeshRef.current.morphTargetInfluences[index], targetValue, VISME_SMOOTHING);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
});
|
|
155
|
+
previousEmotionKeysRef.current = currentEmotionKeys;
|
|
156
|
+
if (isTransitioningToIdleRef.current && currentActionRef.current) {
|
|
157
|
+
if (currentActionRef.current.time >=
|
|
158
|
+
currentActionRef.current.getClip().duration) {
|
|
159
|
+
const idleNumber = Math.floor(Math.random() * 5) + 1;
|
|
160
|
+
const idleAction = actions[`Idle${idleNumber === 3 ? 4 : idleNumber}`];
|
|
161
|
+
if (idleAction) {
|
|
162
|
+
currentActionRef.current.fadeOut(0.5);
|
|
163
|
+
idleAction.reset().fadeIn(0.5).play();
|
|
164
|
+
currentActionRef.current = idleAction;
|
|
165
|
+
isTransitioningToIdleRef.current = false;
|
|
164
166
|
}
|
|
165
167
|
}
|
|
166
|
-
mixer.current.update(0.01);
|
|
167
168
|
}
|
|
169
|
+
(_a = mixerRef.current) === null || _a === void 0 ? void 0 : _a.update(0.01);
|
|
170
|
+
}, [
|
|
171
|
+
actions,
|
|
172
|
+
emotionMorphTargets,
|
|
173
|
+
eyeBlink,
|
|
174
|
+
updateCurrentViseme,
|
|
175
|
+
]);
|
|
176
|
+
(0, fiber_1.useFrame)(state => {
|
|
177
|
+
updateFrame(state.clock.getElapsedTime() * 1000);
|
|
168
178
|
});
|
|
169
179
|
return ((0, jsx_runtime_1.jsx)("group", { position: isZoomed ? AVATAR_POSITION_ZOOMED : AVATAR_POSITION, rotation: AVATAR_ROTATION, children: (0, jsx_runtime_1.jsx)("primitive", { object: scene }) }));
|
|
170
180
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fullbodyAvatar.js","sourceRoot":"","sources":["../../../../../../src/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.tsx"],"names":[],"mappings":";;;AAAA,
|
|
1
|
+
{"version":3,"file":"fullbodyAvatar.js","sourceRoot":"","sources":["../../../../../../src/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.tsx"],"names":[],"mappings":";;;AAAA,iCAAuE;AACvE,iCASe;AACf,4CAA2D;AAC3D,8CAAwD;AA+BxD,MAAM,eAAe,GAAG,IAAI,eAAO,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AAC9C,MAAM,eAAe,GAAG,IAAI,aAAK,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;AAC/C,MAAM,sBAAsB,GAAG,IAAI,eAAO,CAAC,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;AAExD,MAAM,cAAc,GAAG;IACrB,IAAI,EAAE,gFAAgF;IACtF,MAAM,EACJ,gFAAgF;CACnF,CAAC;AAEF,MAAM,YAAY,GAAG;IACnB,WAAW,EAAE,IAAI;IACjB,WAAW,EAAE,IAAI;IACjB,aAAa,EAAE,GAAG;CACnB,CAAC;AAEF,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAC9B,MAAM,eAAe,GAAG,GAAG,CAAC;AAG5B,SAAwB,cAAc,CAAC,EACrC,GAAG,EACH,GAAG,EACH,iBAAiB,EACjB,SAAS,EACT,QAAQ,EACR,QAAQ,EACR,mBAAmB,EACnB,wBAAwB,EACxB,wBAAwB,EACxB,mBAAmB,GACC;IACpB,MAAM,EAAE,KAAK,EAAE,GAAG,IAAA,cAAO,EAAC,GAAG,CAAC,CAAC;IAC/B,MAAM,EAAE,UAAU,EAAE,GAAG,IAAA,cAAO,EAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC;IACpD,MAAM,EAAE,OAAO,EAAE,GAAG,IAAA,oBAAa,EAAC,UAAU,EAAE,KAAK,CAAC,CAAC;IAErD,MAAM,QAAQ,GAAG,IAAA,cAAM,GAAkB,CAAC;IAC1C,MAAM,WAAW,GAAG,IAAA,cAAM,GAAe,CAAC;IAC1C,MAAM,gBAAgB,GAAG,IAAA,cAAM,EAAyB,IAAI,CAAC,CAAC;IAC9D,MAAM,wBAAwB,GAAG,IAAA,cAAM,EAAC,KAAK,CAAC,CAAC;IAE/C,MAAM,gBAAgB,GAAG,IAAA,cAAM,EAAC,CAAC,CAAC,CAAC;IACnC,MAAM,gBAAgB,GAAG,IAAA,cAAM,EAAC,CAAC,CAAC,CAAC;IACnC,MAAM,aAAa,GAAG,IAAA,cAAM,EAAC,KAAK,CAAC,CAAC;IACpC,MAAM,iBAAiB,GAAG,IAAA,cAAM,EAAC,CAAC,CAAC,CAAC;IAEpC,MAAM,iBAAiB,GAAG,IAAA,cAAM,EAAyB,EAAE,CAAC,CAAC;IAC7D,MAAM,sBAAsB,GAAG,IAAA,cAAM,EAAc,IAAI,GAAG,EAAE,CAAC,CAAC;IAG9D,MAAM,QAAQ,GAAG,IAAA,eAAO,EAAC,GAAG,EAAE;QAC5B,IAAI,SAAkC,CAAC;QACvC,KAAK,CAAC,QAAQ,CAAC,CAAC,MAAgB,EAAE,EAAE;YAClC,IACE,MAAM,YAAY,mBAAW;gBAC7B,CAAC,MAAM,CAAC,IAAI,KAAK,YAAY,IAAI,MAAM,CAAC,IAAI,KAAK,eAAe,CAAC,EACjE;gBACA,SAAS,GAAG,MAAM,CAAC;aACpB;QACH,CAAC,CAAC,CAAC;QACH,OAAO,SAAS,CAAC;IACnB,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC;IAEZ,IAAA,iBAAS,EAAC,GAAG,EAAE;QACb,IAAI,QAAQ,EAAE;YACZ,WAAW,CAAC,OAAO,GAAG,QAAQ,CAAC;YAC/B,IAAI,QAAQ,CAAC,qBAAqB,IAAI,QAAQ,CAAC,qBAAqB,EAAE;gBACpE,wBAAwB,CAAC,QAAQ,CAAC,qBAAqB,CAAC,CAAC;gBACzD,MAAM,iBAAiB,GAAG,MAAM,CAAC,IAAI,CACnC,QAAQ,CAAC,qBAAqB,CAC/B,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,GAAG,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;gBACnD,wBAAwB,CAAC,iBAAiB,CAAC,CAAC;aAC7C;SACF;QACD,QAAQ,CAAC,OAAO,GAAG,IAAI,sBAAc,CAAC,KAAK,CAAC,CAAC;IAC/C,CAAC,EAAE,CAAC,QAAQ,EAAE,KAAK,EAAE,wBAAwB,EAAE,wBAAwB,CAAC,CAAC,CAAC;IAG1E,MAAM,qBAAqB,GAAG,IAAA,mBAAW,EAAC,GAAG,EAAE;QAC7C,IAAI,CAAC,OAAO,IAAI,CAAC,iBAAiB,CAAC,MAAM;YAAE,OAAO;QAElD,MAAM,SAAS,GAAG,OAAO,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC;QACpD,IAAI,CAAC,SAAS,EAAE;YACd,OAAO,CAAC,IAAI,CACV,cAAc,iBAAiB,CAAC,MAAM,yBAAyB,CAChE,CAAC;YACF,OAAO;SACR;QAED,MAAM,eAAe,GAAG,GAAG,CAAC;QAC5B,MAAM,cAAc,GAAG,GAAG,CAAC;QAE3B,IAAI,gBAAgB,CAAC,OAAO,EAAE;YAC5B,gBAAgB,CAAC,OAAO,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC;SACnD;QAED,SAAS,CAAC,KAAK,EAAE,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,IAAI,EAAE,CAAC;QAChD,gBAAgB,CAAC,OAAO,GAAG,SAAS,CAAC;QACrC,SAAS,CAAC,SAAS,GAAG,SAAS,CAAC;QAEhC,IACE,iBAAiB,CAAC,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC;YAC5C,iBAAiB,CAAC,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC;YAC7C,iBAAiB,CAAC,MAAM,CAAC,UAAU,CAAC,UAAU,CAAC;YAC/C,iBAAiB,CAAC,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC;YAC7C,iBAAiB,CAAC,MAAM,CAAC,UAAU,CAAC,WAAW,CAAC,EAChD;YACA,SAAS,CAAC,OAAO,CAAC,gBAAQ,EAAE,CAAC,CAAC,CAAC;YAC/B,SAAS,CAAC,iBAAiB,GAAG,IAAI,CAAC;YACnC,wBAAwB,CAAC,OAAO,GAAG,IAAI,CAAC;SACzC;IACH,CAAC,EAAE,CAAC,OAAO,EAAE,iBAAiB,EAAE,SAAS,CAAC,CAAC,CAAC;IAE5C,IAAA,iBAAS,EAAC,GAAG,EAAE;QACb,qBAAqB,EAAE,CAAC;IAC1B,CAAC,EAAE,CAAC,qBAAqB,CAAC,CAAC,CAAC;IAG5B,MAAM,WAAW,GAAG,IAAA,mBAAW,EAC7B,CAAC,WAAmB,EAAE,EAAE;;QACtB,IACE,CAAC,WAAW,CAAC,OAAO;YACpB,CAAC,WAAW,CAAC,OAAO,CAAC,qBAAqB;YAC1C,CAAC,WAAW,CAAC,OAAO,CAAC,qBAAqB;YAE1C,OAAO;QAET,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,QAAQ,EAAE;YACZ,IAAI,WAAW,IAAI,gBAAgB,CAAC,OAAO,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE;gBACrE,aAAa,CAAC,OAAO,GAAG,IAAI,CAAC;gBAC7B,iBAAiB,CAAC,OAAO,GAAG,WAAW,CAAC;gBACxC,gBAAgB,CAAC,OAAO,GAAG,WAAW,CAAC;gBACvC,gBAAgB,CAAC,OAAO;oBACtB,WAAW;wBACX,IAAI,CAAC,MAAM,EAAE;4BACX,CAAC,YAAY,CAAC,WAAW,GAAG,YAAY,CAAC,WAAW,CAAC;wBACvD,YAAY,CAAC,WAAW,CAAC;aAC5B;YAED,IAAI,aAAa,CAAC,OAAO,EAAE;gBACzB,MAAM,aAAa,GACjB,CAAC,WAAW,GAAG,iBAAiB,CAAC,OAAO,CAAC;oBACzC,YAAY,CAAC,aAAa,CAAC;gBAC7B,IAAI,aAAa,IAAI,GAAG,EAAE;oBACxB,UAAU,GAAG,aAAa,GAAG,CAAC,CAAC;iBAChC;qBAAM,IAAI,aAAa,IAAI,CAAC,EAAE;oBAC7B,UAAU,GAAG,CAAC,GAAG,aAAa,GAAG,CAAC,CAAC;iBACpC;qBAAM;oBACL,aAAa,CAAC,OAAO,GAAG,KAAK,CAAC;oBAC9B,UAAU,GAAG,CAAC,CAAC;iBAChB;aACF;SACF;QAED,MAAM,aAAa,GAAG,mBAAmB,CAAC,WAAW,GAAG,IAAI,CAAC,CAAC;QAC9D,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC,CAAC;QAErE,sBAAsB,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE;YAC3C,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;gBAChC,MAAM,KAAK,GAAG,WAAW,CAAC,OAAQ,CAAC,qBAAsB,CAAC,GAAG,CAAC,CAAC;gBAC/D,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;oBAC7B,iBAAiB,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;oBACnC,IAAI,WAAW,CAAC,OAAO,IAAI,WAAW,CAAC,OAAO,CAAC,qBAAqB,EAAE;wBACpE,WAAW,CAAC,OAAO,CAAC,qBAAqB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;qBACtD;iBACF;aACF;QACH,CAAC,CAAC,CAAC;QAEH,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,OAAO,CAAC,qBAAqB,CAAC,CAAC,OAAO,CAC/D,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;YACf,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;gBAC7B,IAAI,WAAW,GAAG,CAAC,CAAC;gBAEpB,IAAI,kBAAkB,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;oBAC/B,MAAM,kBAAkB,GAAG,mBAAmB,CAAC,GAAG,CAAC,CAAC;oBACpD,MAAM,mBAAmB,GAAG,iBAAiB,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;oBAChE,MAAM,eAAe,GAAG,iBAAS,CAAC,IAAI,CACpC,mBAAmB,EACnB,kBAAkB,GAAG,GAAG,EACxB,iBAAiB,CAClB,CAAC;oBACF,iBAAiB,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,eAAe,CAAC;oBACjD,WAAW,IAAI,eAAe,CAAC;iBAChC;gBAED,IAAI,aAAa,IAAI,GAAG,KAAK,aAAa,CAAC,IAAI,EAAE;oBAC/C,WAAW,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC;iBACzC;gBAED,IAAI,GAAG,KAAK,YAAY,IAAI,QAAQ,EAAE;oBACpC,WAAW,IAAI,UAAU,CAAC;iBAC3B;gBAED,WAAW,GAAG,iBAAS,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;gBACjD,IACE,WAAW,CAAC,OAAO;oBACnB,WAAW,CAAC,OAAO,CAAC,qBAAqB,EACzC;oBACA,WAAW,CAAC,OAAO,CAAC,qBAAqB,CAAC,KAAK,CAAC,GAAG,iBAAS,CAAC,IAAI,CAC/D,WAAW,CAAC,OAAO,CAAC,qBAAqB,CAAC,KAAK,CAAC,EAChD,WAAW,EACX,eAAe,CAChB,CAAC;iBACH;aACF;QACH,CAAC,CACF,CAAC;QAEF,sBAAsB,CAAC,OAAO,GAAG,kBAAkB,CAAC;QAEpD,IAAI,wBAAwB,CAAC,OAAO,IAAI,gBAAgB,CAAC,OAAO,EAAE;YAChE,IACE,gBAAgB,CAAC,OAAO,CAAC,IAAI;gBAC7B,gBAAgB,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,QAAQ,EAC3C;gBACA,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;gBACrD,MAAM,UAAU,GACd,OAAO,CAAC,OAAO,UAAU,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC;gBAEtD,IAAI,UAAU,EAAE;oBACd,gBAAgB,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;oBACtC,UAAU,CAAC,KAAK,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;oBACtC,gBAAgB,CAAC,OAAO,GAAG,UAAU,CAAC;oBACtC,wBAAwB,CAAC,OAAO,GAAG,KAAK,CAAC;iBAC1C;aACF;SACF;QAED,MAAA,QAAQ,CAAC,OAAO,0CAAE,MAAM,CAAC,IAAI,CAAC,CAAC;IACjC,CAAC,EACD;QACE,OAAO;QACP,mBAAmB;QACnB,QAAQ;QACR,mBAAmB;KACpB,CACF,CAAC;IAEF,IAAA,gBAAQ,EAAC,KAAK,CAAC,EAAE;QACf,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,cAAc,EAAE,GAAG,IAAI,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,OAAO,CACL,kCACE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,eAAe,EAC7D,QAAQ,EAAE,eAAe,YAEzB,sCAAW,MAAM,EAAE,KAAK,GAAI,GACtB,CACT,CAAC;AACJ,CAAC;AAxOD,iCAwOC"}
|
|
@@ -219,7 +219,7 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
|
|
|
219
219
|
const [isPlayingAudio, setIsPlayingAudio] = (0, react_1.useState)(false);
|
|
220
220
|
const [controlsPosition, setControlsPosition] = (0, react_1.useState)('center');
|
|
221
221
|
const [hideEmissions, setHideEmissions] = (0, react_1.useState)(false);
|
|
222
|
-
const {
|
|
222
|
+
const { startProcessing, addViseme, stopProcessing, resetVisemeQueue } = (0, visemeContext_1.useViseme)();
|
|
223
223
|
(0, react_1.useEffect)(() => {
|
|
224
224
|
setIsPlayingAudio(!!speechSynthesizer);
|
|
225
225
|
memoriSpeaking = !!speechSynthesizer;
|
|
@@ -1315,49 +1315,41 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
|
|
|
1315
1315
|
if (result) {
|
|
1316
1316
|
setIsPlayingAudio(true);
|
|
1317
1317
|
memoriSpeaking = true;
|
|
1318
|
+
startProcessing();
|
|
1318
1319
|
try {
|
|
1319
1320
|
audioContext.decodeAudioData(result.audioData, function (buffer) {
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
currentSource.start();
|
|
1326
|
-
currentSource.onended = () => {
|
|
1327
|
-
console.log('ended');
|
|
1328
|
-
setIsPlayingAudio(false);
|
|
1329
|
-
memoriSpeaking = false;
|
|
1330
|
-
stopProcessing();
|
|
1331
|
-
resetVisemeQueue();
|
|
1332
|
-
emitEndSpeakEvent();
|
|
1333
|
-
};
|
|
1321
|
+
source.buffer = buffer;
|
|
1322
|
+
source.connect(audioContext.destination);
|
|
1323
|
+
if (history.length < 1 || (isSafari && isIOS)) {
|
|
1324
|
+
source.start(0);
|
|
1325
|
+
}
|
|
1334
1326
|
});
|
|
1335
1327
|
audioContext.onstatechange = () => {
|
|
1336
1328
|
if (audioContext.state === 'suspended' ||
|
|
1337
1329
|
audioContext.state === 'closed') {
|
|
1338
|
-
console.log('suspended');
|
|
1339
1330
|
source.disconnect();
|
|
1340
1331
|
setIsPlayingAudio(false);
|
|
1341
1332
|
stopProcessing();
|
|
1342
1333
|
resetVisemeQueue();
|
|
1343
1334
|
memoriSpeaking = false;
|
|
1344
|
-
emitEndSpeakEvent();
|
|
1345
1335
|
}
|
|
1346
1336
|
else if (audioContext.state === 'interrupted') {
|
|
1347
|
-
console.log('interrupted');
|
|
1348
|
-
stopProcessing();
|
|
1349
|
-
resetVisemeQueue();
|
|
1350
1337
|
audioContext.resume();
|
|
1351
1338
|
}
|
|
1352
1339
|
};
|
|
1340
|
+
audioContext.resume();
|
|
1341
|
+
if (speechSynthesizer) {
|
|
1342
|
+
speechSynthesizer.close();
|
|
1343
|
+
speechSynthesizer = null;
|
|
1344
|
+
}
|
|
1353
1345
|
}
|
|
1354
1346
|
catch (e) {
|
|
1355
1347
|
console.warn('speak error: ', e);
|
|
1356
1348
|
window.speechSynthesis.speak(new SpeechSynthesisUtterance(text));
|
|
1357
|
-
setIsPlayingAudio(false);
|
|
1358
|
-
memoriSpeaking = false;
|
|
1359
1349
|
stopProcessing();
|
|
1360
1350
|
resetVisemeQueue();
|
|
1351
|
+
setIsPlayingAudio(false);
|
|
1352
|
+
memoriSpeaking = false;
|
|
1361
1353
|
if (speechSynthesizer) {
|
|
1362
1354
|
speechSynthesizer.close();
|
|
1363
1355
|
speechSynthesizer = null;
|
|
@@ -1368,7 +1360,6 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
|
|
|
1368
1360
|
else {
|
|
1369
1361
|
audioContext.resume();
|
|
1370
1362
|
stopProcessing();
|
|
1371
|
-
resetVisemeQueue();
|
|
1372
1363
|
setIsPlayingAudio(false);
|
|
1373
1364
|
memoriSpeaking = false;
|
|
1374
1365
|
emitEndSpeakEvent();
|
|
@@ -1377,9 +1368,9 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
|
|
|
1377
1368
|
console.error('speak:', error);
|
|
1378
1369
|
window.speechSynthesis.speak(new SpeechSynthesisUtterance(text));
|
|
1379
1370
|
setIsPlayingAudio(false);
|
|
1380
|
-
memoriSpeaking = false;
|
|
1381
1371
|
stopProcessing();
|
|
1382
1372
|
resetVisemeQueue();
|
|
1373
|
+
memoriSpeaking = false;
|
|
1383
1374
|
emitEndSpeakEvent();
|
|
1384
1375
|
});
|
|
1385
1376
|
setMemoriTyping(false);
|