@convai/web-sdk 0.3.1-beta.3 → 0.3.2-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +535 -1077
- package/dist/core/AudioManager.d.ts.map +1 -0
- package/dist/core/AudioManager.js +262 -0
- package/dist/core/AudioManager.js.map +1 -0
- package/dist/core/BlendshapeQueue.d.ts +112 -0
- package/dist/core/BlendshapeQueue.d.ts.map +1 -0
- package/dist/core/BlendshapeQueue.js +172 -0
- package/dist/core/BlendshapeQueue.js.map +1 -0
- package/dist/{types/core → core}/ConvaiClient.d.ts +5 -15
- package/dist/core/ConvaiClient.d.ts.map +1 -0
- package/dist/core/ConvaiClient.js +602 -0
- package/dist/core/ConvaiClient.js.map +1 -0
- package/dist/core/EventEmitter.d.ts.map +1 -0
- package/dist/core/EventEmitter.js +68 -0
- package/dist/core/EventEmitter.js.map +1 -0
- package/dist/{types/core → core}/MessageHandler.d.ts +7 -0
- package/dist/core/MessageHandler.d.ts.map +1 -0
- package/dist/core/MessageHandler.js +332 -0
- package/dist/core/MessageHandler.js.map +1 -0
- package/dist/core/ScreenShareManager.d.ts.map +1 -0
- package/dist/core/ScreenShareManager.js +207 -0
- package/dist/core/ScreenShareManager.js.map +1 -0
- package/dist/core/VideoManager.d.ts.map +1 -0
- package/dist/core/VideoManager.js +205 -0
- package/dist/core/VideoManager.js.map +1 -0
- package/dist/{types/core → core}/index.d.ts +1 -0
- package/dist/core/index.d.ts.map +1 -0
- package/dist/core/index.js +14 -1970
- package/dist/core/index.js.map +1 -0
- package/dist/{types/core → core}/types.d.ts +9 -23
- package/dist/core/types.d.ts.map +1 -0
- package/dist/core/types.js +2 -0
- package/dist/core/types.js.map +1 -0
- package/dist/dev.d.ts +12 -0
- package/dist/dev.d.ts.map +1 -0
- package/dist/dev.js +12 -0
- package/dist/dev.js.map +1 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +6 -0
- package/dist/index.js.map +1 -0
- package/dist/lipsync-helpers/arkitOrder61.d.ts.map +1 -0
- package/dist/lipsync-helpers/arkitOrder61.js +287 -0
- package/dist/lipsync-helpers/arkitOrder61.js.map +1 -0
- package/dist/lipsync-helpers/createBlendshapeQueue.d.ts +86 -0
- package/dist/lipsync-helpers/createBlendshapeQueue.d.ts.map +1 -0
- package/dist/lipsync-helpers/createBlendshapeQueue.js +126 -0
- package/dist/lipsync-helpers/createBlendshapeQueue.js.map +1 -0
- package/dist/lipsync-helpers/declarativeMapping.d.ts +93 -0
- package/dist/lipsync-helpers/declarativeMapping.d.ts.map +1 -0
- package/dist/lipsync-helpers/declarativeMapping.js +228 -0
- package/dist/lipsync-helpers/declarativeMapping.js.map +1 -0
- package/dist/lipsync-helpers/index.d.ts +15 -0
- package/dist/lipsync-helpers/index.d.ts.map +1 -0
- package/dist/lipsync-helpers/index.js +31 -1164
- package/dist/lipsync-helpers/index.js.map +1 -0
- package/dist/lipsync-helpers/mappingTypes.d.ts +64 -0
- package/dist/lipsync-helpers/mappingTypes.d.ts.map +1 -0
- package/dist/lipsync-helpers/mappingTypes.js +28 -0
- package/dist/lipsync-helpers/mappingTypes.js.map +1 -0
- package/dist/lipsync-helpers/metahumanOrder251.d.ts +115 -0
- package/dist/lipsync-helpers/metahumanOrder251.d.ts.map +1 -0
- package/dist/lipsync-helpers/metahumanOrder251.js +432 -0
- package/dist/lipsync-helpers/metahumanOrder251.js.map +1 -0
- package/dist/lipsync-helpers/presetMappers.d.ts +24 -0
- package/dist/lipsync-helpers/presetMappers.d.ts.map +1 -0
- package/dist/lipsync-helpers/presetMappers.js +38 -0
- package/dist/lipsync-helpers/presetMappers.js.map +1 -0
- package/dist/react/components/ConvaiWidget.d.ts +68 -0
- package/dist/react/components/ConvaiWidget.d.ts.map +1 -0
- package/dist/react/components/ConvaiWidget.js +505 -0
- package/dist/react/components/ConvaiWidget.js.map +1 -0
- package/dist/react/components/index.d.ts +2 -0
- package/dist/react/components/index.d.ts.map +1 -0
- package/dist/react/components/index.js +3 -0
- package/dist/react/components/index.js.map +1 -0
- package/dist/react/components/rtc-widget/components/AudioSettingsPanel.d.ts +10 -0
- package/dist/react/components/rtc-widget/components/AudioSettingsPanel.d.ts.map +1 -0
- package/dist/react/components/rtc-widget/components/AudioSettingsPanel.js +316 -0
- package/dist/react/components/rtc-widget/components/AudioSettingsPanel.js.map +1 -0
- package/dist/react/components/rtc-widget/components/AudioVisualizer.d.ts +36 -0
- package/dist/react/components/rtc-widget/components/AudioVisualizer.d.ts.map +1 -0
- package/dist/react/components/rtc-widget/components/AudioVisualizer.js +259 -0
- package/dist/react/components/rtc-widget/components/AudioVisualizer.js.map +1 -0
- package/dist/react/components/rtc-widget/components/ConviMessage.d.ts +10 -0
- package/dist/react/components/rtc-widget/components/ConviMessage.d.ts.map +1 -0
- package/dist/react/components/rtc-widget/components/ConviMessage.js +14 -0
- package/dist/react/components/rtc-widget/components/ConviMessage.js.map +1 -0
- package/dist/react/components/rtc-widget/components/FloatingVideo.d.ts +9 -0
- package/dist/react/components/rtc-widget/components/FloatingVideo.d.ts.map +1 -0
- package/dist/react/components/rtc-widget/components/FloatingVideo.js +122 -0
- package/dist/react/components/rtc-widget/components/FloatingVideo.js.map +1 -0
- package/dist/react/components/rtc-widget/components/MarkdownRenderer.d.ts +7 -0
- package/dist/react/components/rtc-widget/components/MarkdownRenderer.d.ts.map +1 -0
- package/dist/react/components/rtc-widget/components/MarkdownRenderer.js +68 -0
- package/dist/react/components/rtc-widget/components/MarkdownRenderer.js.map +1 -0
- package/dist/react/components/rtc-widget/components/MessageBubble.d.ts +10 -0
- package/dist/react/components/rtc-widget/components/MessageBubble.d.ts.map +1 -0
- package/dist/react/components/rtc-widget/components/MessageBubble.js +23 -0
- package/dist/react/components/rtc-widget/components/MessageBubble.js.map +1 -0
- package/dist/react/components/rtc-widget/components/MessageList.d.ts +11 -0
- package/dist/react/components/rtc-widget/components/MessageList.d.ts.map +1 -0
- package/dist/react/components/rtc-widget/components/MessageList.js +89 -0
- package/dist/react/components/rtc-widget/components/MessageList.js.map +1 -0
- package/dist/react/components/rtc-widget/components/UserMessage.d.ts +9 -0
- package/dist/react/components/rtc-widget/components/UserMessage.d.ts.map +1 -0
- package/dist/react/components/rtc-widget/components/UserMessage.js +15 -0
- package/dist/react/components/rtc-widget/components/UserMessage.js.map +1 -0
- package/dist/react/components/rtc-widget/components/conviComponents/ConviButton.d.ts +6 -0
- package/dist/react/components/rtc-widget/components/conviComponents/ConviButton.d.ts.map +1 -0
- package/dist/react/components/rtc-widget/components/conviComponents/ConviButton.js +15 -0
- package/dist/react/components/rtc-widget/components/conviComponents/ConviButton.js.map +1 -0
- package/dist/react/components/rtc-widget/components/conviComponents/ConviFooter.d.ts +25 -0
- package/dist/react/components/rtc-widget/components/conviComponents/ConviFooter.d.ts.map +1 -0
- package/dist/react/components/rtc-widget/components/conviComponents/ConviFooter.js +172 -0
- package/dist/react/components/rtc-widget/components/conviComponents/ConviFooter.js.map +1 -0
- package/dist/react/components/rtc-widget/components/conviComponents/ConviHeader.d.ts +17 -0
- package/dist/react/components/rtc-widget/components/conviComponents/ConviHeader.d.ts.map +1 -0
- package/dist/react/components/rtc-widget/components/conviComponents/ConviHeader.js +66 -0
- package/dist/react/components/rtc-widget/components/conviComponents/ConviHeader.js.map +1 -0
- package/dist/react/components/rtc-widget/components/conviComponents/SettingsTray.d.ts +12 -0
- package/dist/react/components/rtc-widget/components/conviComponents/SettingsTray.d.ts.map +1 -0
- package/dist/react/components/rtc-widget/components/conviComponents/SettingsTray.js +68 -0
- package/dist/react/components/rtc-widget/components/conviComponents/SettingsTray.js.map +1 -0
- package/dist/react/components/rtc-widget/components/conviComponents/VoiceModeOverlay.d.ts +12 -0
- package/dist/react/components/rtc-widget/components/conviComponents/VoiceModeOverlay.d.ts.map +1 -0
- package/dist/react/components/rtc-widget/components/conviComponents/VoiceModeOverlay.js +255 -0
- package/dist/react/components/rtc-widget/components/conviComponents/VoiceModeOverlay.js.map +1 -0
- package/dist/react/components/rtc-widget/components/conviComponents/index.d.ts +6 -0
- package/dist/react/components/rtc-widget/components/conviComponents/index.d.ts.map +1 -0
- package/dist/react/components/rtc-widget/components/conviComponents/index.js +6 -0
- package/dist/react/components/rtc-widget/components/conviComponents/index.js.map +1 -0
- package/dist/react/components/rtc-widget/components/index.d.ts +9 -0
- package/dist/react/components/rtc-widget/components/index.d.ts.map +1 -0
- package/dist/react/components/rtc-widget/components/index.js +15 -0
- package/dist/react/components/rtc-widget/components/index.js.map +1 -0
- package/dist/react/components/rtc-widget/index.d.ts +6 -0
- package/dist/react/components/rtc-widget/index.d.ts.map +1 -0
- package/dist/react/components/rtc-widget/index.js +9 -0
- package/dist/react/components/rtc-widget/index.js.map +1 -0
- package/dist/react/components/rtc-widget/styles/framerConfig.d.ts +116 -0
- package/dist/react/components/rtc-widget/styles/framerConfig.d.ts.map +1 -0
- package/dist/react/components/rtc-widget/styles/framerConfig.js +73 -0
- package/dist/react/components/rtc-widget/styles/framerConfig.js.map +1 -0
- package/dist/react/components/rtc-widget/styles/icons.d.ts +28 -0
- package/dist/react/components/rtc-widget/styles/icons.d.ts.map +1 -0
- package/dist/react/components/rtc-widget/styles/icons.js +257 -0
- package/dist/react/components/rtc-widget/styles/icons.js.map +1 -0
- package/dist/react/components/rtc-widget/styles/index.d.ts +6 -0
- package/dist/react/components/rtc-widget/styles/index.d.ts.map +1 -0
- package/dist/react/components/rtc-widget/styles/index.js +9 -0
- package/dist/react/components/rtc-widget/styles/index.js.map +1 -0
- package/dist/react/components/rtc-widget/styles/styledComponents.d.ts +90 -0
- package/dist/react/components/rtc-widget/styles/styledComponents.d.ts.map +1 -0
- package/dist/react/components/rtc-widget/styles/styledComponents.js +663 -0
- package/dist/react/components/rtc-widget/styles/styledComponents.js.map +1 -0
- package/dist/react/components/rtc-widget/styles/theme.d.ts +188 -0
- package/dist/react/components/rtc-widget/styles/theme.d.ts.map +1 -0
- package/dist/react/components/rtc-widget/styles/theme.js +290 -0
- package/dist/react/components/rtc-widget/styles/theme.js.map +1 -0
- package/dist/react/components/rtc-widget/types/index.d.ts +60 -0
- package/dist/react/components/rtc-widget/types/index.d.ts.map +1 -0
- package/dist/react/components/rtc-widget/types/index.js +2 -0
- package/dist/react/components/rtc-widget/types/index.js.map +1 -0
- package/dist/react/hooks/index.d.ts +4 -0
- package/dist/react/hooks/index.d.ts.map +1 -0
- package/dist/react/hooks/index.js +6 -0
- package/dist/react/hooks/index.js.map +1 -0
- package/dist/react/hooks/useCharacterInfo.d.ts +17 -0
- package/dist/react/hooks/useCharacterInfo.d.ts.map +1 -0
- package/dist/react/hooks/useCharacterInfo.js +60 -0
- package/dist/react/hooks/useCharacterInfo.js.map +1 -0
- package/dist/react/hooks/useConvaiClient.d.ts +35 -0
- package/dist/react/hooks/useConvaiClient.d.ts.map +1 -0
- package/dist/react/hooks/useConvaiClient.js +179 -0
- package/dist/react/hooks/useConvaiClient.js.map +1 -0
- package/dist/react/hooks/useLocalCameraTrack.d.ts +22 -0
- package/dist/react/hooks/useLocalCameraTrack.d.ts.map +1 -0
- package/dist/react/hooks/useLocalCameraTrack.js +34 -0
- package/dist/react/hooks/useLocalCameraTrack.js.map +1 -0
- package/dist/react/index.d.ts +7 -0
- package/dist/react/index.d.ts.map +1 -0
- package/dist/react/index.js +13 -0
- package/dist/react/index.js.map +1 -0
- package/dist/types/index.d.ts.map +1 -0
- package/dist/types/index.js +2 -0
- package/dist/types/index.js.map +1 -0
- package/dist/utils/LatencyMonitor.d.ts.map +1 -0
- package/dist/utils/LatencyMonitor.js +136 -0
- package/dist/utils/LatencyMonitor.js.map +1 -0
- package/dist/utils/logger.d.ts.map +1 -0
- package/dist/utils/logger.js +96 -0
- package/dist/utils/logger.js.map +1 -0
- package/dist/utils/speakerManagement.d.ts.map +1 -0
- package/dist/utils/speakerManagement.js +64 -0
- package/dist/utils/speakerManagement.js.map +1 -0
- package/dist/{types/vanilla → vanilla}/AudioRenderer.d.ts +5 -0
- package/dist/vanilla/AudioRenderer.d.ts.map +1 -0
- package/dist/vanilla/AudioRenderer.js +135 -0
- package/dist/vanilla/AudioRenderer.js.map +1 -0
- package/dist/{types/vanilla → vanilla}/ConvaiWidget.d.ts +13 -6
- package/dist/vanilla/ConvaiWidget.d.ts.map +1 -0
- package/dist/vanilla/ConvaiWidget.js +1823 -0
- package/dist/vanilla/ConvaiWidget.js.map +1 -0
- package/dist/vanilla/icons.d.ts.map +1 -0
- package/dist/vanilla/icons.js +222 -0
- package/dist/vanilla/icons.js.map +1 -0
- package/dist/{types/vanilla → vanilla}/index.d.ts +1 -3
- package/dist/vanilla/index.d.ts.map +1 -0
- package/dist/vanilla/index.js +20 -5509
- package/dist/vanilla/index.js.map +1 -0
- package/dist/vanilla/styles.d.ts.map +1 -0
- package/dist/vanilla/styles.js +287 -0
- package/dist/vanilla/styles.js.map +1 -0
- package/dist/vanilla/types.d.ts +63 -0
- package/dist/vanilla/types.d.ts.map +1 -0
- package/dist/vanilla/types.js +2 -0
- package/dist/vanilla/types.js.map +1 -0
- package/package.json +33 -38
- package/CHANGELOG.md +0 -165
- package/dist/core/index.cjs +0 -1977
- package/dist/lipsync-helpers/index.cjs +0 -1195
- package/dist/types/core/AudioManager.d.ts.map +0 -1
- package/dist/types/core/ConvaiClient.d.ts.map +0 -1
- package/dist/types/core/EventEmitter.d.ts.map +0 -1
- package/dist/types/core/MessageHandler.d.ts.map +0 -1
- package/dist/types/core/ScreenShareManager.d.ts.map +0 -1
- package/dist/types/core/VideoManager.d.ts.map +0 -1
- package/dist/types/core/index.d.ts.map +0 -1
- package/dist/types/core/types.d.ts.map +0 -1
- package/dist/types/lipsync-helpers/arkitBlendshapeHelpers.d.ts +0 -80
- package/dist/types/lipsync-helpers/arkitBlendshapeHelpers.d.ts.map +0 -1
- package/dist/types/lipsync-helpers/arkitOrder61.d.ts.map +0 -1
- package/dist/types/lipsync-helpers/arkitPhonemeReference.d.ts +0 -155
- package/dist/types/lipsync-helpers/arkitPhonemeReference.d.ts.map +0 -1
- package/dist/types/lipsync-helpers/index.d.ts +0 -9
- package/dist/types/lipsync-helpers/index.d.ts.map +0 -1
- package/dist/types/lipsync-helpers/neurosyncBlendshapeMapper.d.ts +0 -30
- package/dist/types/lipsync-helpers/neurosyncBlendshapeMapper.d.ts.map +0 -1
- package/dist/types/types/index.d.ts.map +0 -1
- package/dist/types/utils/LatencyMonitor.d.ts.map +0 -1
- package/dist/types/utils/logger.d.ts.map +0 -1
- package/dist/types/utils/speakerManagement.d.ts.map +0 -1
- package/dist/types/vanilla/AudioRenderer.d.ts.map +0 -1
- package/dist/types/vanilla/ConvaiWidget.d.ts.map +0 -1
- package/dist/types/vanilla/icons.d.ts.map +0 -1
- package/dist/types/vanilla/index.d.ts.map +0 -1
- package/dist/types/vanilla/styles.d.ts.map +0 -1
- package/dist/types/vanilla/types.d.ts +0 -106
- package/dist/types/vanilla/types.d.ts.map +0 -1
- package/dist/umd/convai.umd.js +0 -1
- package/dist/vanilla/index.cjs +0 -5559
- /package/dist/{types/core → core}/AudioManager.d.ts +0 -0
- /package/dist/{types/core → core}/EventEmitter.d.ts +0 -0
- /package/dist/{types/core → core}/ScreenShareManager.d.ts +0 -0
- /package/dist/{types/core → core}/VideoManager.d.ts +0 -0
- /package/dist/{types/lipsync-helpers → lipsync-helpers}/arkitOrder61.d.ts +0 -0
- /package/dist/types/{types/index.d.ts → index.d.ts} +0 -0
- /package/dist/{types/utils → utils}/LatencyMonitor.d.ts +0 -0
- /package/dist/{types/utils → utils}/logger.d.ts +0 -0
- /package/dist/{types/utils → utils}/speakerManagement.d.ts +0 -0
- /package/dist/{types/vanilla → vanilla}/icons.d.ts +0 -0
- /package/dist/{types/vanilla → vanilla}/styles.d.ts +0 -0
|
@@ -1,1195 +0,0 @@
|
|
|
1
|
-
'use strict';
|
|
2
|
-
|
|
3
|
-
/**
|
|
4
|
-
* ARKit 61-Element Custom Order Mapper
|
|
5
|
-
* Maps the custom 61-element ARKit blendshape array to standard ARKit indices
|
|
6
|
-
*
|
|
7
|
-
* This ordering includes:
|
|
8
|
-
* - 52 standard ARKit facial blendshapes (in custom order)
|
|
9
|
-
* - 9 additional values for head and eye rotations
|
|
10
|
-
*/
|
|
11
|
-
/**
|
|
12
|
-
* Custom ARKit 61-element ordering
|
|
13
|
-
* This is the order in which blendshapes are received from the server
|
|
14
|
-
*/
|
|
15
|
-
const ARKIT_ORDER_61 = [
|
|
16
|
-
"EyeBlinkLeft", // 0
|
|
17
|
-
"EyeLookDownLeft", // 1
|
|
18
|
-
"EyeLookInLeft", // 2
|
|
19
|
-
"EyeLookOutLeft", // 3
|
|
20
|
-
"EyeLookUpLeft", // 4
|
|
21
|
-
"EyeSquintLeft", // 5
|
|
22
|
-
"EyeWideLeft", // 6
|
|
23
|
-
"EyeBlinkRight", // 7
|
|
24
|
-
"EyeLookDownRight", // 8
|
|
25
|
-
"EyeLookInRight", // 9
|
|
26
|
-
"EyeLookOutRight", // 10
|
|
27
|
-
"EyeLookUpRight", // 11
|
|
28
|
-
"EyeSquintRight", // 12
|
|
29
|
-
"EyeWideRight", // 13
|
|
30
|
-
"JawForward", // 14
|
|
31
|
-
"JawRight", // 15
|
|
32
|
-
"JawLeft", // 16
|
|
33
|
-
"JawOpen", // 17 🔑 KEY for mouth opening
|
|
34
|
-
"MouthClose", // 18 🔑 KEY for mouth closing
|
|
35
|
-
"MouthFunnel", // 19 🔑 KEY for "O" sounds
|
|
36
|
-
"MouthPucker", // 20 🔑 KEY for "U" sounds
|
|
37
|
-
"MouthRight", // 21
|
|
38
|
-
"MouthLeft", // 22
|
|
39
|
-
"MouthSmileLeft", // 23 🔑 KEY for "ee" sounds
|
|
40
|
-
"MouthSmileRight", // 24 🔑 KEY for "ee" sounds
|
|
41
|
-
"MouthFrownLeft", // 25
|
|
42
|
-
"MouthFrownRight", // 26
|
|
43
|
-
"MouthDimpleLeft", // 27
|
|
44
|
-
"MouthDimpleRight", // 28
|
|
45
|
-
"MouthStretchLeft", // 29 🔑 KEY for wide mouth
|
|
46
|
-
"MouthStretchRight", // 30 🔑 KEY for wide mouth
|
|
47
|
-
"MouthRollLower", // 31
|
|
48
|
-
"MouthRollUpper", // 32
|
|
49
|
-
"MouthShrugLower", // 33
|
|
50
|
-
"MouthShrugUpper", // 34
|
|
51
|
-
"MouthPressLeft", // 35
|
|
52
|
-
"MouthPressRight", // 36
|
|
53
|
-
"MouthLowerDownLeft", // 37
|
|
54
|
-
"MouthLowerDownRight", // 38
|
|
55
|
-
"MouthUpperUpLeft", // 39
|
|
56
|
-
"MouthUpperUpRight", // 40
|
|
57
|
-
"BrowDownLeft", // 41
|
|
58
|
-
"BrowDownRight", // 42
|
|
59
|
-
"BrowInnerUp", // 43
|
|
60
|
-
"BrowOuterUpLeft", // 44
|
|
61
|
-
"BrowOuterUpRight", // 45
|
|
62
|
-
"CheekPuff", // 46
|
|
63
|
-
"CheekSquintLeft", // 47
|
|
64
|
-
"CheekSquintRight", // 48
|
|
65
|
-
"NoseSneerLeft", // 49
|
|
66
|
-
"NoseSneerRight", // 50
|
|
67
|
-
"TongueOut", // 51 🔑 KEY for "th" sounds
|
|
68
|
-
"HeadYaw", // 52 - Head rotation left/right
|
|
69
|
-
"HeadPitch", // 53 - Head rotation up/down
|
|
70
|
-
"HeadRoll", // 54 - Head rotation tilt
|
|
71
|
-
"LeftEyeYaw", // 55 - Left eye rotation left/right
|
|
72
|
-
"LeftEyePitch", // 56 - Left eye rotation up/down
|
|
73
|
-
"LeftEyeRoll", // 57 - Left eye rotation roll
|
|
74
|
-
"RightEyeYaw", // 58 - Right eye rotation left/right
|
|
75
|
-
"RightEyePitch", // 59 - Right eye rotation up/down
|
|
76
|
-
"RightEyeRoll", // 60 - Right eye rotation roll
|
|
77
|
-
];
|
|
78
|
-
/**
|
|
79
|
-
* Standard ARKit blendshape order (52 elements)
|
|
80
|
-
* This is Apple's official ordering
|
|
81
|
-
*/
|
|
82
|
-
const STANDARD_ARKIT_ORDER = [
|
|
83
|
-
"BrowDownLeft", // 0
|
|
84
|
-
"BrowDownRight", // 1
|
|
85
|
-
"BrowInnerUp", // 2
|
|
86
|
-
"BrowOuterUpLeft", // 3
|
|
87
|
-
"BrowOuterUpRight", // 4
|
|
88
|
-
"CheekPuff", // 5
|
|
89
|
-
"CheekSquintLeft", // 6
|
|
90
|
-
"CheekSquintRight", // 7
|
|
91
|
-
"EyeBlinkLeft", // 8
|
|
92
|
-
"EyeBlinkRight", // 9
|
|
93
|
-
"EyeLookDownLeft", // 10
|
|
94
|
-
"EyeLookDownRight", // 11
|
|
95
|
-
"EyeLookInLeft", // 12
|
|
96
|
-
"EyeLookInRight", // 13
|
|
97
|
-
"EyeLookOutLeft", // 14
|
|
98
|
-
"EyeLookOutRight", // 15
|
|
99
|
-
"EyeLookUpLeft", // 16
|
|
100
|
-
"EyeLookUpRight", // 17
|
|
101
|
-
"EyeSquintLeft", // 18
|
|
102
|
-
"EyeSquintRight", // 19
|
|
103
|
-
"EyeWideLeft", // 20
|
|
104
|
-
"EyeWideRight", // 21
|
|
105
|
-
"JawForward", // 22
|
|
106
|
-
"JawLeft", // 23
|
|
107
|
-
"JawOpen", // 24
|
|
108
|
-
"JawRight", // 25
|
|
109
|
-
"MouthClose", // 26
|
|
110
|
-
"MouthDimpleLeft", // 27
|
|
111
|
-
"MouthDimpleRight", // 28
|
|
112
|
-
"MouthFrownLeft", // 29
|
|
113
|
-
"MouthFrownRight", // 30
|
|
114
|
-
"MouthFunnel", // 31
|
|
115
|
-
"MouthLeft", // 32
|
|
116
|
-
"MouthLowerDownLeft", // 33
|
|
117
|
-
"MouthLowerDownRight", // 34
|
|
118
|
-
"MouthPressLeft", // 35
|
|
119
|
-
"MouthPressRight", // 36
|
|
120
|
-
"MouthPucker", // 37
|
|
121
|
-
"MouthRight", // 38
|
|
122
|
-
"MouthRollLower", // 39
|
|
123
|
-
"MouthRollUpper", // 40
|
|
124
|
-
"MouthShrugLower", // 41
|
|
125
|
-
"MouthShrugUpper", // 42
|
|
126
|
-
"MouthSmileLeft", // 43
|
|
127
|
-
"MouthSmileRight", // 44
|
|
128
|
-
"MouthStretchLeft", // 45
|
|
129
|
-
"MouthStretchRight", // 46
|
|
130
|
-
"MouthUpperUpLeft", // 47
|
|
131
|
-
"MouthUpperUpRight", // 48
|
|
132
|
-
"NoseSneerLeft", // 49
|
|
133
|
-
"NoseSneerRight", // 50
|
|
134
|
-
"TongueOut", // 51
|
|
135
|
-
];
|
|
136
|
-
/**
|
|
137
|
-
* Mapping from custom 61-order indices to standard ARKit indices
|
|
138
|
-
* Maps ARKIT_ORDER_61[i] -> STANDARD_ARKIT_ORDER[j]
|
|
139
|
-
*/
|
|
140
|
-
const ORDER_61_TO_STANDARD_MAP = new Map();
|
|
141
|
-
// Build the mapping at module initialization
|
|
142
|
-
for (let i = 0; i < ARKIT_ORDER_61.length; i++) {
|
|
143
|
-
const name = ARKIT_ORDER_61[i];
|
|
144
|
-
// Only map the first 52 (standard ARKit blendshapes)
|
|
145
|
-
if (i < 52) {
|
|
146
|
-
const standardIndex = STANDARD_ARKIT_ORDER.indexOf(name);
|
|
147
|
-
if (standardIndex !== -1) {
|
|
148
|
-
ORDER_61_TO_STANDARD_MAP.set(i, standardIndex);
|
|
149
|
-
}
|
|
150
|
-
}
|
|
151
|
-
}
|
|
152
|
-
/**
|
|
153
|
-
* Convert 61-element custom order array to standard 52-element ARKit array
|
|
154
|
-
* @param values61 - 61-element array in custom order
|
|
155
|
-
* @returns 52-element array in standard ARKit order
|
|
156
|
-
*/
|
|
157
|
-
function convertOrder61ToStandard(values61) {
|
|
158
|
-
const standardArray = new Array(52).fill(0);
|
|
159
|
-
for (let i = 0; i < Math.min(52, values61.length); i++) {
|
|
160
|
-
const standardIndex = ORDER_61_TO_STANDARD_MAP.get(i);
|
|
161
|
-
if (standardIndex !== undefined) {
|
|
162
|
-
standardArray[standardIndex] = values61[i];
|
|
163
|
-
}
|
|
164
|
-
}
|
|
165
|
-
return standardArray;
|
|
166
|
-
}
|
|
167
|
-
/**
|
|
168
|
-
* Map 61-element array to named object (all 61 values)
|
|
169
|
-
* @param values61 - 61-element array in custom order
|
|
170
|
-
* @returns Object with blendshape/rotation names as keys
|
|
171
|
-
*/
|
|
172
|
-
function mapOrder61ToNames(values61) {
|
|
173
|
-
const mapped = {};
|
|
174
|
-
for (let i = 0; i < Math.min(values61.length, ARKIT_ORDER_61.length); i++) {
|
|
175
|
-
// Convert to camelCase (lowercase first letter)
|
|
176
|
-
const name = ARKIT_ORDER_61[i];
|
|
177
|
-
const camelCaseName = name.charAt(0).toLowerCase() + name.slice(1);
|
|
178
|
-
mapped[camelCaseName] = values61[i];
|
|
179
|
-
}
|
|
180
|
-
return mapped;
|
|
181
|
-
}
|
|
182
|
-
/**
|
|
183
|
-
* Get specific blendshape value by name from 61-element array
|
|
184
|
-
* @param values61 - 61-element array in custom order
|
|
185
|
-
* @param name - Name of the blendshape (case-insensitive)
|
|
186
|
-
* @returns The value, or 0 if not found
|
|
187
|
-
*/
|
|
188
|
-
function getBlendshapeValueFromOrder61(values61, name) {
|
|
189
|
-
const upperName = name.charAt(0).toUpperCase() + name.slice(1);
|
|
190
|
-
const index = ARKIT_ORDER_61.indexOf(upperName);
|
|
191
|
-
if (index !== -1 && index < values61.length) {
|
|
192
|
-
return values61[index];
|
|
193
|
-
}
|
|
194
|
-
return 0;
|
|
195
|
-
}
|
|
196
|
-
/**
|
|
197
|
-
* Extract head rotation values from 61-element array
|
|
198
|
-
* @param values61 - 61-element array in custom order
|
|
199
|
-
* @returns Object with head rotation values (yaw, pitch, roll)
|
|
200
|
-
*/
|
|
201
|
-
function getHeadRotation(values61) {
|
|
202
|
-
return {
|
|
203
|
-
yaw: values61[52] || 0,
|
|
204
|
-
pitch: values61[53] || 0,
|
|
205
|
-
roll: values61[54] || 0,
|
|
206
|
-
};
|
|
207
|
-
}
|
|
208
|
-
/**
|
|
209
|
-
* Extract eye rotation values from 61-element array
|
|
210
|
-
* @param values61 - 61-element array in custom order
|
|
211
|
-
* @returns Object with left and right eye rotation values
|
|
212
|
-
*/
|
|
213
|
-
function getEyeRotations(values61) {
|
|
214
|
-
return {
|
|
215
|
-
left: {
|
|
216
|
-
yaw: values61[55] || 0,
|
|
217
|
-
pitch: values61[56] || 0,
|
|
218
|
-
roll: values61[57] || 0,
|
|
219
|
-
},
|
|
220
|
-
right: {
|
|
221
|
-
yaw: values61[58] || 0,
|
|
222
|
-
pitch: values61[59] || 0,
|
|
223
|
-
roll: values61[60] || 0,
|
|
224
|
-
},
|
|
225
|
-
};
|
|
226
|
-
}
|
|
227
|
-
/**
|
|
228
|
-
* Get key lip-sync blendshapes from 61-element array
|
|
229
|
-
* These are the most important values for facial animation
|
|
230
|
-
*/
|
|
231
|
-
function getLipSyncValuesFromOrder61(values61) {
|
|
232
|
-
return {
|
|
233
|
-
jawOpen: values61[17] || 0,
|
|
234
|
-
mouthClose: values61[18] || 0,
|
|
235
|
-
mouthFunnel: values61[19] || 0,
|
|
236
|
-
mouthPucker: values61[20] || 0,
|
|
237
|
-
mouthSmileLeft: values61[23] || 0,
|
|
238
|
-
mouthSmileRight: values61[24] || 0,
|
|
239
|
-
mouthStretchLeft: values61[29] || 0,
|
|
240
|
-
mouthStretchRight: values61[30] || 0,
|
|
241
|
-
mouthRollLower: values61[31] || 0,
|
|
242
|
-
mouthRollUpper: values61[32] || 0,
|
|
243
|
-
tongueOut: values61[51] || 0,
|
|
244
|
-
};
|
|
245
|
-
}
|
|
246
|
-
/**
|
|
247
|
-
* Get active blendshapes from 61-element array (above threshold)
|
|
248
|
-
* @param values61 - 61-element array in custom order
|
|
249
|
-
* @param threshold - Minimum value to consider "active" (default: 0.01)
|
|
250
|
-
* @returns Sorted array of active blendshapes
|
|
251
|
-
*/
|
|
252
|
-
function getActiveBlendshapesFromOrder61(values61, threshold = 0.01) {
|
|
253
|
-
const active = [];
|
|
254
|
-
for (let i = 0; i < Math.min(values61.length, ARKIT_ORDER_61.length); i++) {
|
|
255
|
-
if (Math.abs(values61[i]) > threshold) {
|
|
256
|
-
const name = ARKIT_ORDER_61[i];
|
|
257
|
-
const camelCaseName = name.charAt(0).toLowerCase() + name.slice(1);
|
|
258
|
-
active.push({
|
|
259
|
-
name: camelCaseName,
|
|
260
|
-
value: values61[i],
|
|
261
|
-
index: i,
|
|
262
|
-
});
|
|
263
|
-
}
|
|
264
|
-
}
|
|
265
|
-
// Sort by absolute value (descending)
|
|
266
|
-
return active.sort((a, b) => Math.abs(b.value) - Math.abs(a.value));
|
|
267
|
-
}
|
|
268
|
-
/**
|
|
269
|
-
* Format 61-element array for logging
|
|
270
|
-
* @param values61 - 61-element array in custom order
|
|
271
|
-
* @param threshold - Only show values above this (default: 0.01)
|
|
272
|
-
* @param maxEntries - Maximum entries to show (default: 20)
|
|
273
|
-
*/
|
|
274
|
-
function formatOrder61ForLogging(values61, threshold = 0.01, maxEntries = 20) {
|
|
275
|
-
const active = getActiveBlendshapesFromOrder61(values61, threshold)
|
|
276
|
-
.slice(0, maxEntries)
|
|
277
|
-
.reduce((acc, item) => {
|
|
278
|
-
acc[item.name] = Number(item.value.toFixed(4));
|
|
279
|
-
return acc;
|
|
280
|
-
}, {});
|
|
281
|
-
return {
|
|
282
|
-
active,
|
|
283
|
-
total: values61.length,
|
|
284
|
-
activeCount: Object.keys(active).length,
|
|
285
|
-
headRotation: getHeadRotation(values61),
|
|
286
|
-
eyeRotations: getEyeRotations(values61),
|
|
287
|
-
};
|
|
288
|
-
}
|
|
289
|
-
|
|
290
|
-
/**
|
|
291
|
-
* ARKit Blendshape Helpers
|
|
292
|
-
* Utilities for working with ARKit blendshape arrays (52-60 elements)
|
|
293
|
-
*
|
|
294
|
-
* ARKit Standard: 52 blendshapes (indices 0-51)
|
|
295
|
-
* ARKit Extended: Up to 60 blendshapes with additional facial features
|
|
296
|
-
*/
|
|
297
|
-
/**
|
|
298
|
-
* ARKit Standard Blendshape Names (indices 0-51)
|
|
299
|
-
*/
|
|
300
|
-
const ARKIT_BLENDSHAPE_NAMES = [
|
|
301
|
-
"browDownLeft", // 0
|
|
302
|
-
"browDownRight", // 1
|
|
303
|
-
"browInnerUp", // 2
|
|
304
|
-
"browOuterUpLeft", // 3
|
|
305
|
-
"browOuterUpRight", // 4
|
|
306
|
-
"cheekPuff", // 5
|
|
307
|
-
"cheekSquintLeft", // 6
|
|
308
|
-
"cheekSquintRight", // 7
|
|
309
|
-
"eyeBlinkLeft", // 8
|
|
310
|
-
"eyeBlinkRight", // 9
|
|
311
|
-
"eyeLookDownLeft", // 10
|
|
312
|
-
"eyeLookDownRight", // 11
|
|
313
|
-
"eyeLookInLeft", // 12
|
|
314
|
-
"eyeLookInRight", // 13
|
|
315
|
-
"eyeLookOutLeft", // 14
|
|
316
|
-
"eyeLookOutRight", // 15
|
|
317
|
-
"eyeLookUpLeft", // 16
|
|
318
|
-
"eyeLookUpRight", // 17
|
|
319
|
-
"eyeSquintLeft", // 18
|
|
320
|
-
"eyeSquintRight", // 19
|
|
321
|
-
"eyeWideLeft", // 20
|
|
322
|
-
"eyeWideRight", // 21
|
|
323
|
-
"jawForward", // 22
|
|
324
|
-
"jawLeft", // 23
|
|
325
|
-
"jawOpen", // 24 🔑 KEY for mouth opening
|
|
326
|
-
"jawRight", // 25
|
|
327
|
-
"mouthClose", // 26 🔑 KEY for mouth closing
|
|
328
|
-
"mouthDimpleLeft", // 27
|
|
329
|
-
"mouthDimpleRight", // 28
|
|
330
|
-
"mouthFrownLeft", // 29
|
|
331
|
-
"mouthFrownRight", // 30
|
|
332
|
-
"mouthFunnel", // 31 🔑 KEY for "O" sounds
|
|
333
|
-
"mouthLeft", // 32
|
|
334
|
-
"mouthLowerDownLeft", // 33
|
|
335
|
-
"mouthLowerDownRight", // 34
|
|
336
|
-
"mouthPressLeft", // 35
|
|
337
|
-
"mouthPressRight", // 36
|
|
338
|
-
"mouthPucker", // 37 🔑 KEY for "U" sounds
|
|
339
|
-
"mouthRight", // 38
|
|
340
|
-
"mouthRollLower", // 39
|
|
341
|
-
"mouthRollUpper", // 40
|
|
342
|
-
"mouthShrugLower", // 41
|
|
343
|
-
"mouthShrugUpper", // 42
|
|
344
|
-
"mouthSmileLeft", // 43 🔑 KEY for "ee" sounds
|
|
345
|
-
"mouthSmileRight", // 44 🔑 KEY for "ee" sounds
|
|
346
|
-
"mouthStretchLeft", // 45 🔑 KEY for wide mouth
|
|
347
|
-
"mouthStretchRight", // 46 🔑 KEY for wide mouth
|
|
348
|
-
"mouthUpperUpLeft", // 47
|
|
349
|
-
"mouthUpperUpRight", // 48
|
|
350
|
-
"noseSneerLeft", // 49
|
|
351
|
-
"noseSneerRight", // 50
|
|
352
|
-
"tongueOut", // 51 🔑 KEY for "th" sounds
|
|
353
|
-
];
|
|
354
|
-
/**
|
|
355
|
-
* Extended blendshapes (indices 52-59) - if provided
|
|
356
|
-
* These are optional extensions beyond standard ARKit
|
|
357
|
-
*/
|
|
358
|
-
const ARKIT_EXTENDED_NAMES = [
|
|
359
|
-
"eyeDilation", // 52 - Pupil dilation
|
|
360
|
-
"eyeConstriction", // 53 - Pupil constriction
|
|
361
|
-
"mouthShrug", // 54 - Combined mouth shrug
|
|
362
|
-
"lipSuckUpper", // 55 - Upper lip suck
|
|
363
|
-
"lipSuckLower", // 56 - Lower lip suck
|
|
364
|
-
"lipFunnel", // 57 - Lip funnel position
|
|
365
|
-
"lipPress", // 58 - Lip press together
|
|
366
|
-
"reserved", // 59 - Reserved for future use
|
|
367
|
-
];
|
|
368
|
-
/**
|
|
369
|
-
* Map ARKit blendshape array to named object (ARKit standard only: 0-51)
|
|
370
|
-
* @param values - Array of blendshape values (length 52-60)
|
|
371
|
-
* @returns Object with blendshape names as keys
|
|
372
|
-
*/
|
|
373
|
-
function mapARKitBlendshapesToNames(values) {
|
|
374
|
-
const mapped = {};
|
|
375
|
-
// Map standard ARKit blendshapes (0-51)
|
|
376
|
-
for (let i = 0; i < Math.min(values.length, ARKIT_BLENDSHAPE_NAMES.length); i++) {
|
|
377
|
-
mapped[ARKIT_BLENDSHAPE_NAMES[i]] = values[i];
|
|
378
|
-
}
|
|
379
|
-
// Map extended blendshapes if available (52+)
|
|
380
|
-
if (values.length > 52) {
|
|
381
|
-
const extendedStart = 52;
|
|
382
|
-
for (let i = 0; i < Math.min(values.length - extendedStart, ARKIT_EXTENDED_NAMES.length); i++) {
|
|
383
|
-
mapped[ARKIT_EXTENDED_NAMES[i]] = values[extendedStart + i];
|
|
384
|
-
}
|
|
385
|
-
}
|
|
386
|
-
return mapped;
|
|
387
|
-
}
|
|
388
|
-
/**
|
|
389
|
-
* Get only active blendshapes (above threshold)
|
|
390
|
-
* @param values - Array of blendshape values
|
|
391
|
-
* @param threshold - Minimum value to consider "active" (default: 0.01)
|
|
392
|
-
* @returns Sorted array of active blendshapes
|
|
393
|
-
*/
|
|
394
|
-
function getActiveARKitBlendshapes(values, threshold = 0.01) {
|
|
395
|
-
const active = [];
|
|
396
|
-
// Check standard ARKit blendshapes
|
|
397
|
-
for (let i = 0; i < Math.min(values.length, ARKIT_BLENDSHAPE_NAMES.length); i++) {
|
|
398
|
-
if (Math.abs(values[i]) > threshold) {
|
|
399
|
-
active.push({
|
|
400
|
-
name: ARKIT_BLENDSHAPE_NAMES[i],
|
|
401
|
-
value: values[i],
|
|
402
|
-
index: i,
|
|
403
|
-
});
|
|
404
|
-
}
|
|
405
|
-
}
|
|
406
|
-
// Check extended blendshapes
|
|
407
|
-
if (values.length > 52) {
|
|
408
|
-
for (let i = 52; i < Math.min(values.length, 52 + ARKIT_EXTENDED_NAMES.length); i++) {
|
|
409
|
-
if (Math.abs(values[i]) > threshold) {
|
|
410
|
-
active.push({
|
|
411
|
-
name: ARKIT_EXTENDED_NAMES[i - 52],
|
|
412
|
-
value: values[i],
|
|
413
|
-
index: i,
|
|
414
|
-
});
|
|
415
|
-
}
|
|
416
|
-
}
|
|
417
|
-
}
|
|
418
|
-
// Sort by absolute value (descending)
|
|
419
|
-
return active.sort((a, b) => Math.abs(b.value) - Math.abs(a.value));
|
|
420
|
-
}
|
|
421
|
-
/**
|
|
422
|
-
* Get specific blendshape value by name
|
|
423
|
-
* @param values - Array of blendshape values
|
|
424
|
-
* @param name - Name of the blendshape
|
|
425
|
-
* @returns The value, or 0 if not found
|
|
426
|
-
*/
|
|
427
|
-
function getARKitBlendshapeValue(values, name) {
|
|
428
|
-
const index = ARKIT_BLENDSHAPE_NAMES.indexOf(name);
|
|
429
|
-
if (index !== -1 && index < values.length) {
|
|
430
|
-
return values[index];
|
|
431
|
-
}
|
|
432
|
-
const extendedIndex = ARKIT_EXTENDED_NAMES.indexOf(name);
|
|
433
|
-
if (extendedIndex !== -1) {
|
|
434
|
-
const actualIndex = 52 + extendedIndex;
|
|
435
|
-
if (actualIndex < values.length) {
|
|
436
|
-
return values[actualIndex];
|
|
437
|
-
}
|
|
438
|
-
}
|
|
439
|
-
return 0;
|
|
440
|
-
}
|
|
441
|
-
/**
|
|
442
|
-
* Format ARKit blendshapes for logging
|
|
443
|
-
* @param values - Array of blendshape values
|
|
444
|
-
* @param threshold - Only show values above this (default: 0.01)
|
|
445
|
-
* @param maxEntries - Maximum entries to show (default: 20)
|
|
446
|
-
*/
|
|
447
|
-
function formatARKitBlendshapesForLogging(values, threshold = 0.01, maxEntries = 20) {
|
|
448
|
-
const active = getActiveARKitBlendshapes(values, threshold)
|
|
449
|
-
.slice(0, maxEntries)
|
|
450
|
-
.reduce((acc, item) => {
|
|
451
|
-
acc[item.name] = Number(item.value.toFixed(4));
|
|
452
|
-
return acc;
|
|
453
|
-
}, {});
|
|
454
|
-
return {
|
|
455
|
-
active,
|
|
456
|
-
total: values.length,
|
|
457
|
-
activeCount: Object.keys(active).length,
|
|
458
|
-
};
|
|
459
|
-
}
|
|
460
|
-
/**
|
|
461
|
-
* Get the most important facial features for speech
|
|
462
|
-
* These are the key blendshapes for lip-sync
|
|
463
|
-
*/
|
|
464
|
-
const KEY_LIPSYNC_BLENDSHAPES = {
|
|
465
|
-
jaw: [24], // jawOpen
|
|
466
|
-
lips: [26, 31, 37, 39, 40], // mouthClose, mouthFunnel, mouthPucker, mouthRollLower, mouthRollUpper
|
|
467
|
-
smile: [43, 44, 45, 46], // mouthSmile*, mouthStretch*
|
|
468
|
-
tongue: [51], // tongueOut
|
|
469
|
-
};
|
|
470
|
-
/**
|
|
471
|
-
* Extract only lip-sync relevant blendshapes
|
|
472
|
-
* @param values - Full blendshape array
|
|
473
|
-
* @returns Object with categorized lip-sync blendshapes
|
|
474
|
-
*/
|
|
475
|
-
function getLipSyncBlendshapes(values) {
|
|
476
|
-
return {
|
|
477
|
-
jaw: values[24] || 0,
|
|
478
|
-
mouthClose: values[26] || 0,
|
|
479
|
-
mouthFunnel: values[31] || 0,
|
|
480
|
-
mouthPucker: values[37] || 0,
|
|
481
|
-
mouthRollLower: values[39] || 0,
|
|
482
|
-
mouthRollUpper: values[40] || 0,
|
|
483
|
-
mouthSmileLeft: values[43] || 0,
|
|
484
|
-
mouthSmileRight: values[44] || 0,
|
|
485
|
-
mouthStretchLeft: values[45] || 0,
|
|
486
|
-
mouthStretchRight: values[46] || 0,
|
|
487
|
-
tongueOut: values[51] || 0,
|
|
488
|
-
};
|
|
489
|
-
}
|
|
490
|
-
|
|
491
|
-
/**
|
|
492
|
-
* ARKit Blendshape Reference Values for Phonemes/Visemes
|
|
493
|
-
*
|
|
494
|
-
* This file provides reference values for ARKit's 52 standard facial blendshapes
|
|
495
|
-
* when forming different phonemes (speech sounds). These values are based on
|
|
496
|
-
* industry standards from systems like OVR LipSync, Oculus Audio, and ARKit face tracking.
|
|
497
|
-
*
|
|
498
|
-
* All blendshape values range from 0.0 (no activation) to 1.0 (full activation)
|
|
499
|
-
*
|
|
500
|
-
* ARKit Blendshapes: Indices 0-51
|
|
501
|
-
* Meta Human Extended: Indices 52-250
|
|
502
|
-
*/
|
|
503
|
-
/**
|
|
504
|
-
* Reference values for common phonemes
|
|
505
|
-
* Based on International Phonetic Alphabet (IPA) and Arpabet standards
|
|
506
|
-
*/
|
|
507
|
-
const ARKIT_PHONEME_REFERENCE_VALUES = {
|
|
508
|
-
// ============================================
|
|
509
|
-
// SILENCE / NEUTRAL
|
|
510
|
-
// ============================================
|
|
511
|
-
sil: {
|
|
512
|
-
jawOpen: 0.0,
|
|
513
|
-
mouthClose: 1.0,
|
|
514
|
-
description: "Silence - mouth closed, neutral expression"
|
|
515
|
-
},
|
|
516
|
-
// ============================================
|
|
517
|
-
// VOWELS
|
|
518
|
-
// ============================================
|
|
519
|
-
aa: {
|
|
520
|
-
jawOpen: 0.7,
|
|
521
|
-
mouthFunnel: 0.0,
|
|
522
|
-
mouthPucker: 0.0,
|
|
523
|
-
tongueOut: 0.1,
|
|
524
|
-
description: "Open back vowel - 'ah' (father, hot)"
|
|
525
|
-
},
|
|
526
|
-
ae: {
|
|
527
|
-
jawOpen: 0.5,
|
|
528
|
-
mouthStretchLeft: 0.4,
|
|
529
|
-
mouthStretchRight: 0.4,
|
|
530
|
-
description: "Near-open front vowel - 'a' (cat, bat)"
|
|
531
|
-
},
|
|
532
|
-
E: {
|
|
533
|
-
jawOpen: 0.4,
|
|
534
|
-
mouthStretchLeft: 0.3,
|
|
535
|
-
mouthStretchRight: 0.3,
|
|
536
|
-
mouthSmileLeft: 0.2,
|
|
537
|
-
mouthSmileRight: 0.2,
|
|
538
|
-
description: "Open-mid front vowel - 'eh' (bed, get)"
|
|
539
|
-
},
|
|
540
|
-
I: {
|
|
541
|
-
jawOpen: 0.1,
|
|
542
|
-
mouthStretchLeft: 0.6,
|
|
543
|
-
mouthStretchRight: 0.6,
|
|
544
|
-
mouthSmileLeft: 0.5,
|
|
545
|
-
mouthSmileRight: 0.5,
|
|
546
|
-
mouthClose: 0.3,
|
|
547
|
-
description: "Close front vowel - 'ee' (see, feet)"
|
|
548
|
-
},
|
|
549
|
-
ih: {
|
|
550
|
-
jawOpen: 0.2,
|
|
551
|
-
mouthStretchLeft: 0.4,
|
|
552
|
-
mouthStretchRight: 0.4,
|
|
553
|
-
mouthSmileLeft: 0.3,
|
|
554
|
-
mouthSmileRight: 0.3,
|
|
555
|
-
description: "Near-close front vowel - 'i' (sit, bit)"
|
|
556
|
-
},
|
|
557
|
-
O: {
|
|
558
|
-
jawOpen: 0.5,
|
|
559
|
-
mouthFunnel: 0.6,
|
|
560
|
-
mouthPucker: 0.3,
|
|
561
|
-
description: "Close-mid back rounded vowel - 'oh' (go, boat)"
|
|
562
|
-
},
|
|
563
|
-
aw: {
|
|
564
|
-
jawOpen: 0.6,
|
|
565
|
-
mouthFunnel: 0.4,
|
|
566
|
-
mouthPucker: 0.2,
|
|
567
|
-
description: "Open-mid back rounded vowel - 'aw' (thought, caught)"
|
|
568
|
-
},
|
|
569
|
-
U: {
|
|
570
|
-
jawOpen: 0.2,
|
|
571
|
-
mouthFunnel: 0.8,
|
|
572
|
-
mouthPucker: 0.9,
|
|
573
|
-
description: "Close back rounded vowel - 'oo' (food, boot)"
|
|
574
|
-
},
|
|
575
|
-
uh: {
|
|
576
|
-
jawOpen: 0.25,
|
|
577
|
-
mouthFunnel: 0.5,
|
|
578
|
-
mouthPucker: 0.6,
|
|
579
|
-
description: "Near-close back rounded vowel - 'u' (book, foot)"
|
|
580
|
-
},
|
|
581
|
-
er: {
|
|
582
|
-
jawOpen: 0.35,
|
|
583
|
-
mouthFunnel: 0.3,
|
|
584
|
-
mouthPucker: 0.1,
|
|
585
|
-
description: "R-colored mid-central vowel - 'er' (her, bird)"
|
|
586
|
-
},
|
|
587
|
-
ah: {
|
|
588
|
-
jawOpen: 0.4,
|
|
589
|
-
mouthFunnel: 0.1,
|
|
590
|
-
description: "Open-mid back vowel - 'uh' (but, cup)"
|
|
591
|
-
},
|
|
592
|
-
// ============================================
|
|
593
|
-
// DIPHTHONGS
|
|
594
|
-
// ============================================
|
|
595
|
-
ow: {
|
|
596
|
-
jawOpen: 0.6,
|
|
597
|
-
mouthFunnel: 0.3,
|
|
598
|
-
mouthPucker: 0.2,
|
|
599
|
-
description: "Diphthong - 'ow' (cow, house)"
|
|
600
|
-
},
|
|
601
|
-
ay: {
|
|
602
|
-
jawOpen: 0.5,
|
|
603
|
-
mouthStretchLeft: 0.3,
|
|
604
|
-
mouthStretchRight: 0.3,
|
|
605
|
-
description: "Diphthong - 'ai' (eye, my)"
|
|
606
|
-
},
|
|
607
|
-
oy: {
|
|
608
|
-
jawOpen: 0.5,
|
|
609
|
-
mouthFunnel: 0.4,
|
|
610
|
-
mouthPucker: 0.3,
|
|
611
|
-
description: "Diphthong - 'oy' (boy, coin)"
|
|
612
|
-
},
|
|
613
|
-
// ============================================
|
|
614
|
-
// CONSONANTS - BILABIALS (both lips)
|
|
615
|
-
// ============================================
|
|
616
|
-
PP: {
|
|
617
|
-
jawOpen: 0.0,
|
|
618
|
-
mouthClose: 1.0,
|
|
619
|
-
mouthPressLeft: 0.5,
|
|
620
|
-
mouthPressRight: 0.5,
|
|
621
|
-
description: "Bilabial stops/nasal - 'p', 'b', 'm' (pat, bat, mat)"
|
|
622
|
-
},
|
|
623
|
-
// ============================================
|
|
624
|
-
// CONSONANTS - LABIODENTAL (teeth on lip)
|
|
625
|
-
// ============================================
|
|
626
|
-
FF: {
|
|
627
|
-
jawOpen: 0.15,
|
|
628
|
-
mouthRollLower: 0.6,
|
|
629
|
-
mouthUpperUpLeft: 0.3,
|
|
630
|
-
mouthUpperUpRight: 0.3,
|
|
631
|
-
description: "Labiodental fricatives - 'f', 'v' (fat, vat)"
|
|
632
|
-
},
|
|
633
|
-
// ============================================
|
|
634
|
-
// CONSONANTS - DENTAL/INTERDENTAL
|
|
635
|
-
// ============================================
|
|
636
|
-
TH: {
|
|
637
|
-
jawOpen: 0.2,
|
|
638
|
-
tongueOut: 0.4,
|
|
639
|
-
mouthUpperUpLeft: 0.2,
|
|
640
|
-
mouthUpperUpRight: 0.2,
|
|
641
|
-
mouthRollLower: 0.2,
|
|
642
|
-
description: "Dental fricatives - 'th' (think, this)"
|
|
643
|
-
},
|
|
644
|
-
// ============================================
|
|
645
|
-
// CONSONANTS - ALVEOLAR (tongue to ridge)
|
|
646
|
-
// ============================================
|
|
647
|
-
DD: {
|
|
648
|
-
jawOpen: 0.3,
|
|
649
|
-
mouthClose: 0.2,
|
|
650
|
-
description: "Alveolar stops - 't', 'd' (tap, dap)"
|
|
651
|
-
},
|
|
652
|
-
SS: {
|
|
653
|
-
jawOpen: 0.15,
|
|
654
|
-
mouthStretchLeft: 0.4,
|
|
655
|
-
mouthStretchRight: 0.4,
|
|
656
|
-
mouthClose: 0.4,
|
|
657
|
-
description: "Alveolar sibilants - 's', 'z' (sap, zap)"
|
|
658
|
-
},
|
|
659
|
-
nn: {
|
|
660
|
-
jawOpen: 0.2,
|
|
661
|
-
mouthClose: 0.3,
|
|
662
|
-
description: "Alveolar nasal - 'n' (nap)"
|
|
663
|
-
},
|
|
664
|
-
l: {
|
|
665
|
-
jawOpen: 0.25,
|
|
666
|
-
tongueOut: 0.2,
|
|
667
|
-
description: "Alveolar lateral approximant - 'l' (lap)"
|
|
668
|
-
},
|
|
669
|
-
// ============================================
|
|
670
|
-
// CONSONANTS - POSTALVEOLAR (behind ridge)
|
|
671
|
-
// ============================================
|
|
672
|
-
CH: {
|
|
673
|
-
jawOpen: 0.2,
|
|
674
|
-
mouthFunnel: 0.3,
|
|
675
|
-
mouthPucker: 0.2,
|
|
676
|
-
description: "Postalveolar affricates - 'ch', 'j' (church, judge)"
|
|
677
|
-
},
|
|
678
|
-
sh: {
|
|
679
|
-
jawOpen: 0.2,
|
|
680
|
-
mouthFunnel: 0.4,
|
|
681
|
-
mouthPucker: 0.3,
|
|
682
|
-
description: "Postalveolar fricatives - 'sh', 'zh' (shoe, measure)"
|
|
683
|
-
},
|
|
684
|
-
// ============================================
|
|
685
|
-
// CONSONANTS - RETROFLEX
|
|
686
|
-
// ============================================
|
|
687
|
-
RR: {
|
|
688
|
-
jawOpen: 0.3,
|
|
689
|
-
mouthFunnel: 0.2,
|
|
690
|
-
mouthPucker: 0.15,
|
|
691
|
-
tongueOut: 0.1,
|
|
692
|
-
description: "Alveolar approximant - 'r' (red)"
|
|
693
|
-
},
|
|
694
|
-
// ============================================
|
|
695
|
-
// CONSONANTS - PALATAL
|
|
696
|
-
// ============================================
|
|
697
|
-
y: {
|
|
698
|
-
jawOpen: 0.2,
|
|
699
|
-
mouthStretchLeft: 0.35,
|
|
700
|
-
mouthStretchRight: 0.35,
|
|
701
|
-
mouthSmileLeft: 0.2,
|
|
702
|
-
mouthSmileRight: 0.2,
|
|
703
|
-
description: "Palatal approximant - 'y' (yes)"
|
|
704
|
-
},
|
|
705
|
-
// ============================================
|
|
706
|
-
// CONSONANTS - VELAR (back of tongue)
|
|
707
|
-
// ============================================
|
|
708
|
-
kk: {
|
|
709
|
-
jawOpen: 0.35,
|
|
710
|
-
description: "Velar stops/nasal - 'k', 'g', 'ng' (kite, go, sing)"
|
|
711
|
-
},
|
|
712
|
-
// ============================================
|
|
713
|
-
// CONSONANTS - GLOTTAL
|
|
714
|
-
// ============================================
|
|
715
|
-
hh: {
|
|
716
|
-
jawOpen: 0.3,
|
|
717
|
-
description: "Glottal fricative - 'h' (hat)"
|
|
718
|
-
},
|
|
719
|
-
// ============================================
|
|
720
|
-
// CONSONANTS - LABIO-VELAR
|
|
721
|
-
// ============================================
|
|
722
|
-
w: {
|
|
723
|
-
jawOpen: 0.15,
|
|
724
|
-
mouthPucker: 0.8,
|
|
725
|
-
mouthFunnel: 0.6,
|
|
726
|
-
description: "Labio-velar approximant - 'w' (wet)"
|
|
727
|
-
},
|
|
728
|
-
};
|
|
729
|
-
/**
|
|
730
|
-
* ARKit Blendshape Indices (0-51)
|
|
731
|
-
* These are Apple's standard 52 facial blendshapes
|
|
732
|
-
*/
|
|
733
|
-
const ARKIT_BLENDSHAPE_INDICES = {
|
|
734
|
-
browDownLeft: 0,
|
|
735
|
-
browDownRight: 1,
|
|
736
|
-
browInnerUp: 2,
|
|
737
|
-
browOuterUpLeft: 3,
|
|
738
|
-
browOuterUpRight: 4,
|
|
739
|
-
cheekPuff: 5,
|
|
740
|
-
cheekSquintLeft: 6,
|
|
741
|
-
cheekSquintRight: 7,
|
|
742
|
-
eyeBlinkLeft: 8,
|
|
743
|
-
eyeBlinkRight: 9,
|
|
744
|
-
eyeLookDownLeft: 10,
|
|
745
|
-
eyeLookDownRight: 11,
|
|
746
|
-
eyeLookInLeft: 12,
|
|
747
|
-
eyeLookInRight: 13,
|
|
748
|
-
eyeLookOutLeft: 14,
|
|
749
|
-
eyeLookOutRight: 15,
|
|
750
|
-
eyeLookUpLeft: 16,
|
|
751
|
-
eyeLookUpRight: 17,
|
|
752
|
-
eyeSquintLeft: 18,
|
|
753
|
-
eyeSquintRight: 19,
|
|
754
|
-
eyeWideLeft: 20,
|
|
755
|
-
eyeWideRight: 21,
|
|
756
|
-
jawForward: 22,
|
|
757
|
-
jawLeft: 23,
|
|
758
|
-
jawOpen: 24, // 🔑 KEY for mouth opening
|
|
759
|
-
jawRight: 25,
|
|
760
|
-
mouthClose: 26, // 🔑 KEY for mouth closing
|
|
761
|
-
mouthDimpleLeft: 27,
|
|
762
|
-
mouthDimpleRight: 28,
|
|
763
|
-
mouthFrownLeft: 29,
|
|
764
|
-
mouthFrownRight: 30,
|
|
765
|
-
mouthFunnel: 31, // 🔑 KEY for "O" and "oo" sounds
|
|
766
|
-
mouthLeft: 32,
|
|
767
|
-
mouthLowerDownLeft: 33,
|
|
768
|
-
mouthLowerDownRight: 34,
|
|
769
|
-
mouthPressLeft: 35,
|
|
770
|
-
mouthPressRight: 36,
|
|
771
|
-
mouthPucker: 37, // 🔑 KEY for "U" and "w" sounds
|
|
772
|
-
mouthRight: 38,
|
|
773
|
-
mouthRollLower: 39,
|
|
774
|
-
mouthRollUpper: 40,
|
|
775
|
-
mouthShrugLower: 41,
|
|
776
|
-
mouthShrugUpper: 42,
|
|
777
|
-
mouthSmileLeft: 43, // 🔑 KEY for "ee" sounds
|
|
778
|
-
mouthSmileRight: 44, // 🔑 KEY for "ee" sounds
|
|
779
|
-
mouthStretchLeft: 45, // 🔑 KEY for wide mouth shapes
|
|
780
|
-
mouthStretchRight: 46, // 🔑 KEY for wide mouth shapes
|
|
781
|
-
mouthUpperUpLeft: 47,
|
|
782
|
-
mouthUpperUpRight: 48,
|
|
783
|
-
noseSneerLeft: 49,
|
|
784
|
-
noseSneerRight: 50,
|
|
785
|
-
tongueOut: 51 // 🔑 KEY for "th" and "l" sounds
|
|
786
|
-
};
|
|
787
|
-
/**
|
|
788
|
-
* Generate a complete 251-length blendshape array for a specific phoneme
|
|
789
|
-
* @param phoneme - The phoneme key from ARKIT_PHONEME_REFERENCE_VALUES
|
|
790
|
-
* @returns Array of 251 blendshape values
|
|
791
|
-
*/
|
|
792
|
-
function generatePhonemeBlendshapes(phoneme) {
|
|
793
|
-
const blendshapes = new Array(251).fill(0);
|
|
794
|
-
const values = ARKIT_PHONEME_REFERENCE_VALUES[phoneme];
|
|
795
|
-
// Map phoneme values to ARKit indices (0-51)
|
|
796
|
-
for (const [key, value] of Object.entries(values)) {
|
|
797
|
-
if (key !== 'description' && key in ARKIT_BLENDSHAPE_INDICES) {
|
|
798
|
-
const index = ARKIT_BLENDSHAPE_INDICES[key];
|
|
799
|
-
blendshapes[index] = value;
|
|
800
|
-
}
|
|
801
|
-
}
|
|
802
|
-
return blendshapes;
|
|
803
|
-
}
|
|
804
|
-
/**
|
|
805
|
-
* Pre-generated sample blendshapes for testing and visualization
|
|
806
|
-
* Use these to test your visualization system
|
|
807
|
-
*/
|
|
808
|
-
const SAMPLE_PHONEME_TESTS = {
|
|
809
|
-
'Silence': generatePhonemeBlendshapes('sil'),
|
|
810
|
-
'A (ah - father)': generatePhonemeBlendshapes('aa'),
|
|
811
|
-
'E (eh - bed)': generatePhonemeBlendshapes('E'),
|
|
812
|
-
'I (ee - see)': generatePhonemeBlendshapes('I'),
|
|
813
|
-
'O (oh - go)': generatePhonemeBlendshapes('O'),
|
|
814
|
-
'U (oo - food)': generatePhonemeBlendshapes('U'),
|
|
815
|
-
'P/B/M': generatePhonemeBlendshapes('PP'),
|
|
816
|
-
'F/V': generatePhonemeBlendshapes('FF'),
|
|
817
|
-
'TH (think)': generatePhonemeBlendshapes('TH'),
|
|
818
|
-
'S/Z': generatePhonemeBlendshapes('SS'),
|
|
819
|
-
'SH (shoe)': generatePhonemeBlendshapes('sh'),
|
|
820
|
-
'CH (church)': generatePhonemeBlendshapes('CH'),
|
|
821
|
-
'W (wet)': generatePhonemeBlendshapes('w'),
|
|
822
|
-
'R (red)': generatePhonemeBlendshapes('RR'),
|
|
823
|
-
'L (lap)': generatePhonemeBlendshapes('l'),
|
|
824
|
-
'K/G': generatePhonemeBlendshapes('kk'),
|
|
825
|
-
};
|
|
826
|
-
/**
|
|
827
|
-
* Get a formatted string showing active blendshapes for a phoneme
|
|
828
|
-
* Useful for debugging and documentation
|
|
829
|
-
*/
|
|
830
|
-
function getPhonemeDescription(phoneme) {
|
|
831
|
-
const data = ARKIT_PHONEME_REFERENCE_VALUES[phoneme];
|
|
832
|
-
const lines = [data.description, ''];
|
|
833
|
-
for (const [key, value] of Object.entries(data)) {
|
|
834
|
-
if (key !== 'description') {
|
|
835
|
-
lines.push(` ${key}: ${value}`);
|
|
836
|
-
}
|
|
837
|
-
}
|
|
838
|
-
return lines.join('\n');
|
|
839
|
-
}
|
|
840
|
-
/**
|
|
841
|
-
* Quick reference guide for key blendshapes
|
|
842
|
-
*/
|
|
843
|
-
const BLENDSHAPE_QUICK_GUIDE = {
|
|
844
|
-
'Open mouth': ['jawOpen'],
|
|
845
|
-
'Close mouth': ['mouthClose'],
|
|
846
|
-
'Round lips (O, oo)': ['mouthFunnel', 'mouthPucker'],
|
|
847
|
-
'Smile (ee)': ['mouthSmileLeft', 'mouthSmileRight', 'mouthStretchLeft', 'mouthStretchRight'],
|
|
848
|
-
'Lips pressed (p, b, m)': ['mouthPressLeft', 'mouthPressRight', 'mouthClose'],
|
|
849
|
-
'Teeth on lip (f, v)': ['mouthRollLower', 'mouthUpperUpLeft', 'mouthUpperUpRight'],
|
|
850
|
-
'Tongue visible (th, l)': ['tongueOut'],
|
|
851
|
-
};
|
|
852
|
-
|
|
853
|
-
/**
|
|
854
|
-
* Neurosync Meta Human Blendshape Mapper
|
|
855
|
-
* Maps 251 blendshape indices to their corresponding names
|
|
856
|
-
* Based on Neurosync Meta Human specification
|
|
857
|
-
*/
|
|
858
|
-
const NEUROSYNC_BLENDSHAPE_NAMES = [
|
|
859
|
-
// 0-51: ARKit Standard Blendshapes
|
|
860
|
-
"browDownLeft", // 0
|
|
861
|
-
"browDownRight", // 1
|
|
862
|
-
"browInnerUp", // 2
|
|
863
|
-
"browOuterUpLeft", // 3
|
|
864
|
-
"browOuterUpRight", // 4
|
|
865
|
-
"cheekPuff", // 5
|
|
866
|
-
"cheekSquintLeft", // 6
|
|
867
|
-
"cheekSquintRight", // 7
|
|
868
|
-
"eyeBlinkLeft", // 8
|
|
869
|
-
"eyeBlinkRight", // 9
|
|
870
|
-
"eyeLookDownLeft", // 10
|
|
871
|
-
"eyeLookDownRight", // 11
|
|
872
|
-
"eyeLookInLeft", // 12
|
|
873
|
-
"eyeLookInRight", // 13
|
|
874
|
-
"eyeLookOutLeft", // 14
|
|
875
|
-
"eyeLookOutRight", // 15
|
|
876
|
-
"eyeLookUpLeft", // 16
|
|
877
|
-
"eyeLookUpRight", // 17
|
|
878
|
-
"eyeSquintLeft", // 18
|
|
879
|
-
"eyeSquintRight", // 19
|
|
880
|
-
"eyeWideLeft", // 20
|
|
881
|
-
"eyeWideRight", // 21
|
|
882
|
-
"jawForward", // 22
|
|
883
|
-
"jawLeft", // 23
|
|
884
|
-
"jawOpen", // 24
|
|
885
|
-
"jawRight", // 25
|
|
886
|
-
"mouthClose", // 26
|
|
887
|
-
"mouthDimpleLeft", // 27
|
|
888
|
-
"mouthDimpleRight", // 28
|
|
889
|
-
"mouthFrownLeft", // 29
|
|
890
|
-
"mouthFrownRight", // 30
|
|
891
|
-
"mouthFunnel", // 31
|
|
892
|
-
"mouthLeft", // 32
|
|
893
|
-
"mouthLowerDownLeft", // 33
|
|
894
|
-
"mouthLowerDownRight", // 34
|
|
895
|
-
"mouthPressLeft", // 35
|
|
896
|
-
"mouthPressRight", // 36
|
|
897
|
-
"mouthPucker", // 37
|
|
898
|
-
"mouthRight", // 38
|
|
899
|
-
"mouthRollLower", // 39
|
|
900
|
-
"mouthRollUpper", // 40
|
|
901
|
-
"mouthShrugLower", // 41
|
|
902
|
-
"mouthShrugUpper", // 42
|
|
903
|
-
"mouthSmileLeft", // 43
|
|
904
|
-
"mouthSmileRight", // 44
|
|
905
|
-
"mouthStretchLeft", // 45
|
|
906
|
-
"mouthStretchRight", // 46
|
|
907
|
-
"mouthUpperUpLeft", // 47
|
|
908
|
-
"mouthUpperUpRight", // 48
|
|
909
|
-
"noseSneerLeft", // 49
|
|
910
|
-
"noseSneerRight", // 50
|
|
911
|
-
"tongueOut", // 51
|
|
912
|
-
// 52-250: Meta Human Extended Blendshapes
|
|
913
|
-
"head_forward", // 52
|
|
914
|
-
"head_upDown", // 53
|
|
915
|
-
"head_tilt", // 54
|
|
916
|
-
"head_turn", // 55
|
|
917
|
-
"neck_stretch", // 56
|
|
918
|
-
"neck_tilt", // 57
|
|
919
|
-
"eyebrow_raiseInner_L", // 58
|
|
920
|
-
"eyebrow_raiseInner_R", // 59
|
|
921
|
-
"eyebrow_raiseOuter_L", // 60
|
|
922
|
-
"eyebrow_raiseOuter_R", // 61
|
|
923
|
-
"eyebrow_down_L", // 62
|
|
924
|
-
"eyebrow_down_R", // 63
|
|
925
|
-
"eye_blink_L", // 64
|
|
926
|
-
"eye_blink_R", // 65
|
|
927
|
-
"eye_wide_L", // 66
|
|
928
|
-
"eye_wide_R", // 67
|
|
929
|
-
"eye_squint_L", // 68
|
|
930
|
-
"eye_squint_R", // 69
|
|
931
|
-
"eye_look_up_L", // 70
|
|
932
|
-
"eye_look_up_R", // 71
|
|
933
|
-
"eye_look_down_L", // 72
|
|
934
|
-
"eye_look_down_R", // 73
|
|
935
|
-
"eye_look_in_L", // 74
|
|
936
|
-
"eye_look_in_R", // 75
|
|
937
|
-
"eye_look_out_L", // 76
|
|
938
|
-
"eye_look_out_R", // 77
|
|
939
|
-
"pupil_dilate_L", // 78
|
|
940
|
-
"pupil_dilate_R", // 79
|
|
941
|
-
"cheek_puff_L", // 80
|
|
942
|
-
"cheek_puff_R", // 81
|
|
943
|
-
"cheek_squint_L", // 82
|
|
944
|
-
"cheek_squint_R", // 83
|
|
945
|
-
"cheek_raiser_L", // 84
|
|
946
|
-
"cheek_raiser_R", // 85
|
|
947
|
-
"nose_wrinkle_L", // 86
|
|
948
|
-
"nose_wrinkle_R", // 87
|
|
949
|
-
"nose_nostril_raise_L", // 88
|
|
950
|
-
"nose_nostril_raise_R", // 89
|
|
951
|
-
"nose_nostril_compress_L", // 90
|
|
952
|
-
"nose_nostril_compress_R", // 91
|
|
953
|
-
"nose_nostril_dilate_L", // 92
|
|
954
|
-
"nose_nostril_dilate_R", // 93
|
|
955
|
-
"lip_upper_raise_L", // 94
|
|
956
|
-
"lip_upper_raise_R", // 95
|
|
957
|
-
"lip_upper_raiseInner_L", // 96
|
|
958
|
-
"lip_upper_raiseInner_R", // 97
|
|
959
|
-
"lip_upper_close", // 98
|
|
960
|
-
"lip_lower_close", // 99
|
|
961
|
-
"lip_lower_down_L", // 100
|
|
962
|
-
"lip_lower_down_R", // 101
|
|
963
|
-
"lip_lower_downInner_L", // 102
|
|
964
|
-
"lip_lower_downInner_R", // 103
|
|
965
|
-
"lip_corner_puller_L", // 104
|
|
966
|
-
"lip_corner_puller_R", // 105
|
|
967
|
-
"lip_corner_depressor_L", // 106
|
|
968
|
-
"lip_corner_depressor_R", // 107
|
|
969
|
-
"lip_stretcher_L", // 108
|
|
970
|
-
"lip_stretcher_R", // 109
|
|
971
|
-
"lip_funneler_LT", // 110
|
|
972
|
-
"lip_funneler_RT", // 111
|
|
973
|
-
"lip_funneler_LB", // 112
|
|
974
|
-
"lip_funneler_RB", // 113
|
|
975
|
-
"lip_pucker_L", // 114
|
|
976
|
-
"lip_pucker_R", // 115
|
|
977
|
-
"lip_press_L", // 116
|
|
978
|
-
"lip_press_R", // 117
|
|
979
|
-
"lip_tightener_L", // 118
|
|
980
|
-
"lip_tightener_R", // 119
|
|
981
|
-
"lip_suck_LT", // 120
|
|
982
|
-
"lip_suck_RT", // 121
|
|
983
|
-
"lip_suck_LB", // 122
|
|
984
|
-
"lip_suck_RB", // 123
|
|
985
|
-
"mouth_left", // 124
|
|
986
|
-
"mouth_right", // 125
|
|
987
|
-
"mouth_smile_L", // 126
|
|
988
|
-
"mouth_smile_R", // 127
|
|
989
|
-
"mouth_frown_L", // 128
|
|
990
|
-
"mouth_frown_R", // 129
|
|
991
|
-
"mouth_dimpler_L", // 130
|
|
992
|
-
"mouth_dimpler_R", // 131
|
|
993
|
-
"mouth_stretch_L", // 132
|
|
994
|
-
"mouth_stretch_R", // 133
|
|
995
|
-
"mouth_shrug_upper", // 134
|
|
996
|
-
"mouth_shrug_lower", // 135
|
|
997
|
-
"mouth_roll_upper_L", // 136
|
|
998
|
-
"mouth_roll_upper_R", // 137
|
|
999
|
-
"mouth_roll_lower_L", // 138
|
|
1000
|
-
"mouth_roll_lower_R", // 139
|
|
1001
|
-
"jaw_open", // 140
|
|
1002
|
-
"jaw_forward", // 141
|
|
1003
|
-
"jaw_left", // 142
|
|
1004
|
-
"jaw_right", // 143
|
|
1005
|
-
"jaw_clench", // 144
|
|
1006
|
-
"jaw_thrust", // 145
|
|
1007
|
-
"tongue_out", // 146
|
|
1008
|
-
"tongue_up", // 147
|
|
1009
|
-
"tongue_down", // 148
|
|
1010
|
-
"tongue_wide", // 149
|
|
1011
|
-
"tongue_narrow", // 150
|
|
1012
|
-
"tongue_curl_up", // 151
|
|
1013
|
-
"tongue_roll", // 152
|
|
1014
|
-
"tongue_bulge_L", // 153
|
|
1015
|
-
"tongue_bulge_R", // 154
|
|
1016
|
-
"tongue_press", // 155
|
|
1017
|
-
"midLipLowerDown_L", // 156
|
|
1018
|
-
"midLipLowerDown_R", // 157
|
|
1019
|
-
"midLipUpperUp_L", // 158
|
|
1020
|
-
"midLipUpperUp_R", // 159
|
|
1021
|
-
"midNostrilSharpener_L", // 160
|
|
1022
|
-
"midNostrilSharpener_R", // 161
|
|
1023
|
-
"viseme_sil", // 162
|
|
1024
|
-
"viseme_PP", // 163
|
|
1025
|
-
"viseme_FF", // 164
|
|
1026
|
-
"viseme_TH", // 165
|
|
1027
|
-
"viseme_DD", // 166
|
|
1028
|
-
"viseme_kk", // 167
|
|
1029
|
-
"viseme_CH", // 168
|
|
1030
|
-
"viseme_SS", // 169
|
|
1031
|
-
"viseme_nn", // 170
|
|
1032
|
-
"viseme_RR", // 171
|
|
1033
|
-
"viseme_aa", // 172
|
|
1034
|
-
"viseme_E", // 173
|
|
1035
|
-
"viseme_I", // 174
|
|
1036
|
-
"viseme_O", // 175
|
|
1037
|
-
"viseme_U", // 176
|
|
1038
|
-
"lid_tightener_L", // 177
|
|
1039
|
-
"lid_tightener_R", // 178
|
|
1040
|
-
"eyelid_upper_raise_L", // 179
|
|
1041
|
-
"eyelid_upper_raise_R", // 180
|
|
1042
|
-
"eyelid_lower_down_L", // 181
|
|
1043
|
-
"eyelid_lower_down_R", // 182
|
|
1044
|
-
"eye_parallel_look_down", // 183
|
|
1045
|
-
"eye_parallel_look_up", // 184
|
|
1046
|
-
"innerBrow_raiser_L", // 185
|
|
1047
|
-
"innerBrow_raiser_R", // 186
|
|
1048
|
-
"outerBrow_raiser_L", // 187
|
|
1049
|
-
"outerBrow_raiser_R", // 188
|
|
1050
|
-
"lipCorner_down_L", // 189
|
|
1051
|
-
"lipCorner_down_R", // 190
|
|
1052
|
-
"lipCorner_wide_L", // 191
|
|
1053
|
-
"lipCorner_wide_R", // 192
|
|
1054
|
-
"lipLower_down_L", // 193
|
|
1055
|
-
"lipLower_down_R", // 194
|
|
1056
|
-
"lipUpper_up_L", // 195
|
|
1057
|
-
"lipUpper_up_R", // 196
|
|
1058
|
-
"wrinkler", // 197
|
|
1059
|
-
"chin_raiser", // 198
|
|
1060
|
-
"chin_raiser_T", // 199
|
|
1061
|
-
"chin_raiser_B", // 200
|
|
1062
|
-
"lips_toward", // 201
|
|
1063
|
-
"lips_blow", // 202
|
|
1064
|
-
"puff_suck_lower", // 203
|
|
1065
|
-
"puff_suck_upper", // 204
|
|
1066
|
-
"puff_suck_upper_L", // 205
|
|
1067
|
-
"puff_suck_upper_R", // 206
|
|
1068
|
-
"puff_suck_lower_L", // 207
|
|
1069
|
-
"puff_suck_lower_R", // 208
|
|
1070
|
-
"phone_aa", // 209
|
|
1071
|
-
"phone_ow", // 210
|
|
1072
|
-
"phone_ee", // 211
|
|
1073
|
-
"phone_er", // 212
|
|
1074
|
-
"phone_uw", // 213
|
|
1075
|
-
"phone_l", // 214
|
|
1076
|
-
"phone_w", // 215
|
|
1077
|
-
"phone_m", // 216
|
|
1078
|
-
"phone_f", // 217
|
|
1079
|
-
"phone_th", // 218
|
|
1080
|
-
"phone_s", // 219
|
|
1081
|
-
"phone_sh", // 220
|
|
1082
|
-
"phone_t", // 221
|
|
1083
|
-
"phone_k", // 222
|
|
1084
|
-
"phone_b", // 223
|
|
1085
|
-
"phone_d", // 224
|
|
1086
|
-
"phone_g", // 225
|
|
1087
|
-
"phone_n", // 226
|
|
1088
|
-
"phone_r", // 227
|
|
1089
|
-
"phone_y", // 228
|
|
1090
|
-
"expression_neutral", // 229
|
|
1091
|
-
"expression_happy", // 230
|
|
1092
|
-
"expression_sad", // 231
|
|
1093
|
-
"expression_angry", // 232
|
|
1094
|
-
"expression_surprised", // 233
|
|
1095
|
-
"expression_disgusted", // 234
|
|
1096
|
-
"expression_afraid", // 235
|
|
1097
|
-
"expression_contempt", // 236
|
|
1098
|
-
"occlusion_upper", // 237
|
|
1099
|
-
"occlusion_lower", // 238
|
|
1100
|
-
"teeth_upper", // 239
|
|
1101
|
-
"teeth_lower", // 240
|
|
1102
|
-
"teeth_fwdBack_upper", // 241
|
|
1103
|
-
"teeth_fwdBack_lower", // 242
|
|
1104
|
-
"teeth_width_upper", // 243
|
|
1105
|
-
"teeth_width_lower", // 244
|
|
1106
|
-
"gums_upper", // 245
|
|
1107
|
-
"gums_lower", // 246
|
|
1108
|
-
"tongue_tip", // 247
|
|
1109
|
-
"tongue_thickness", // 248
|
|
1110
|
-
"soft_palate", // 249
|
|
1111
|
-
"emotion_intensity", // 250
|
|
1112
|
-
];
|
|
1113
|
-
/**
|
|
1114
|
-
* Maps blendshape values to their names
|
|
1115
|
-
*/
|
|
1116
|
-
function mapBlendshapesToNames(values) {
|
|
1117
|
-
const mapped = {};
|
|
1118
|
-
values.forEach((value, index) => {
|
|
1119
|
-
if (index < NEUROSYNC_BLENDSHAPE_NAMES.length) {
|
|
1120
|
-
const name = NEUROSYNC_BLENDSHAPE_NAMES[index];
|
|
1121
|
-
mapped[name] = value;
|
|
1122
|
-
}
|
|
1123
|
-
else {
|
|
1124
|
-
// Handle any indices beyond our known names
|
|
1125
|
-
mapped[`unknown_${index}`] = value;
|
|
1126
|
-
}
|
|
1127
|
-
});
|
|
1128
|
-
return mapped;
|
|
1129
|
-
}
|
|
1130
|
-
/**
|
|
1131
|
-
* Formats blendshapes for logging with filtering
|
|
1132
|
-
* @param values - Array of 251 blendshape values
|
|
1133
|
-
* @param threshold - Only log values above this threshold (default: 0.01)
|
|
1134
|
-
* @param maxEntries - Maximum number of entries to log (default: 20)
|
|
1135
|
-
*/
|
|
1136
|
-
function formatBlendshapesForLogging(values, threshold = 0.01, maxEntries = 20) {
|
|
1137
|
-
const mapped = mapBlendshapesToNames(values);
|
|
1138
|
-
// Filter values above threshold and sort by value (descending)
|
|
1139
|
-
const activeBlendshapes = Object.entries(mapped)
|
|
1140
|
-
.filter(([_, value]) => Math.abs(value) > threshold)
|
|
1141
|
-
.sort(([, a], [, b]) => Math.abs(b) - Math.abs(a))
|
|
1142
|
-
.slice(0, maxEntries)
|
|
1143
|
-
.reduce((acc, [name, value]) => {
|
|
1144
|
-
acc[name] = Number(value.toFixed(4));
|
|
1145
|
-
return acc;
|
|
1146
|
-
}, {});
|
|
1147
|
-
return {
|
|
1148
|
-
active: activeBlendshapes,
|
|
1149
|
-
total: values.length,
|
|
1150
|
-
activeCount: Object.keys(activeBlendshapes).length,
|
|
1151
|
-
};
|
|
1152
|
-
}
|
|
1153
|
-
/**
|
|
1154
|
-
* Gets the most significant blendshapes
|
|
1155
|
-
*/
|
|
1156
|
-
function getTopBlendshapes(values, count = 10) {
|
|
1157
|
-
return values
|
|
1158
|
-
.map((value, index) => ({
|
|
1159
|
-
name: NEUROSYNC_BLENDSHAPE_NAMES[index] || `unknown_${index}`,
|
|
1160
|
-
value,
|
|
1161
|
-
index,
|
|
1162
|
-
}))
|
|
1163
|
-
.filter((item) => Math.abs(item.value) > 0.001)
|
|
1164
|
-
.sort((a, b) => Math.abs(b.value) - Math.abs(a.value))
|
|
1165
|
-
.slice(0, count);
|
|
1166
|
-
}
|
|
1167
|
-
|
|
1168
|
-
exports.ARKIT_BLENDSHAPE_INDICES = ARKIT_BLENDSHAPE_INDICES;
|
|
1169
|
-
exports.ARKIT_BLENDSHAPE_NAMES = ARKIT_BLENDSHAPE_NAMES;
|
|
1170
|
-
exports.ARKIT_EXTENDED_NAMES = ARKIT_EXTENDED_NAMES;
|
|
1171
|
-
exports.ARKIT_ORDER_61 = ARKIT_ORDER_61;
|
|
1172
|
-
exports.ARKIT_PHONEME_REFERENCE_VALUES = ARKIT_PHONEME_REFERENCE_VALUES;
|
|
1173
|
-
exports.BLENDSHAPE_QUICK_GUIDE = BLENDSHAPE_QUICK_GUIDE;
|
|
1174
|
-
exports.KEY_LIPSYNC_BLENDSHAPES = KEY_LIPSYNC_BLENDSHAPES;
|
|
1175
|
-
exports.NEUROSYNC_BLENDSHAPE_NAMES = NEUROSYNC_BLENDSHAPE_NAMES;
|
|
1176
|
-
exports.SAMPLE_PHONEME_TESTS = SAMPLE_PHONEME_TESTS;
|
|
1177
|
-
exports.STANDARD_ARKIT_ORDER = STANDARD_ARKIT_ORDER;
|
|
1178
|
-
exports.convertOrder61ToStandard = convertOrder61ToStandard;
|
|
1179
|
-
exports.formatARKitBlendshapesForLogging = formatARKitBlendshapesForLogging;
|
|
1180
|
-
exports.formatBlendshapesForLogging = formatBlendshapesForLogging;
|
|
1181
|
-
exports.formatOrder61ForLogging = formatOrder61ForLogging;
|
|
1182
|
-
exports.generatePhonemeBlendshapes = generatePhonemeBlendshapes;
|
|
1183
|
-
exports.getARKitBlendshapeValue = getARKitBlendshapeValue;
|
|
1184
|
-
exports.getActiveARKitBlendshapes = getActiveARKitBlendshapes;
|
|
1185
|
-
exports.getActiveBlendshapesFromOrder61 = getActiveBlendshapesFromOrder61;
|
|
1186
|
-
exports.getBlendshapeValueFromOrder61 = getBlendshapeValueFromOrder61;
|
|
1187
|
-
exports.getEyeRotations = getEyeRotations;
|
|
1188
|
-
exports.getHeadRotation = getHeadRotation;
|
|
1189
|
-
exports.getLipSyncBlendshapes = getLipSyncBlendshapes;
|
|
1190
|
-
exports.getLipSyncValuesFromOrder61 = getLipSyncValuesFromOrder61;
|
|
1191
|
-
exports.getPhonemeDescription = getPhonemeDescription;
|
|
1192
|
-
exports.getTopBlendshapes = getTopBlendshapes;
|
|
1193
|
-
exports.mapARKitBlendshapesToNames = mapARKitBlendshapesToNames;
|
|
1194
|
-
exports.mapBlendshapesToNames = mapBlendshapesToNames;
|
|
1195
|
-
exports.mapOrder61ToNames = mapOrder61ToNames;
|