@convai/web-sdk 0.3.1-beta.3 → 0.3.2-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (263) hide show
  1. package/README.md +535 -1077
  2. package/dist/core/AudioManager.d.ts.map +1 -0
  3. package/dist/core/AudioManager.js +262 -0
  4. package/dist/core/AudioManager.js.map +1 -0
  5. package/dist/core/BlendshapeQueue.d.ts +112 -0
  6. package/dist/core/BlendshapeQueue.d.ts.map +1 -0
  7. package/dist/core/BlendshapeQueue.js +172 -0
  8. package/dist/core/BlendshapeQueue.js.map +1 -0
  9. package/dist/{types/core → core}/ConvaiClient.d.ts +5 -15
  10. package/dist/core/ConvaiClient.d.ts.map +1 -0
  11. package/dist/core/ConvaiClient.js +602 -0
  12. package/dist/core/ConvaiClient.js.map +1 -0
  13. package/dist/core/EventEmitter.d.ts.map +1 -0
  14. package/dist/core/EventEmitter.js +68 -0
  15. package/dist/core/EventEmitter.js.map +1 -0
  16. package/dist/{types/core → core}/MessageHandler.d.ts +7 -0
  17. package/dist/core/MessageHandler.d.ts.map +1 -0
  18. package/dist/core/MessageHandler.js +332 -0
  19. package/dist/core/MessageHandler.js.map +1 -0
  20. package/dist/core/ScreenShareManager.d.ts.map +1 -0
  21. package/dist/core/ScreenShareManager.js +207 -0
  22. package/dist/core/ScreenShareManager.js.map +1 -0
  23. package/dist/core/VideoManager.d.ts.map +1 -0
  24. package/dist/core/VideoManager.js +205 -0
  25. package/dist/core/VideoManager.js.map +1 -0
  26. package/dist/{types/core → core}/index.d.ts +1 -0
  27. package/dist/core/index.d.ts.map +1 -0
  28. package/dist/core/index.js +14 -1970
  29. package/dist/core/index.js.map +1 -0
  30. package/dist/{types/core → core}/types.d.ts +9 -23
  31. package/dist/core/types.d.ts.map +1 -0
  32. package/dist/core/types.js +2 -0
  33. package/dist/core/types.js.map +1 -0
  34. package/dist/dev.d.ts +12 -0
  35. package/dist/dev.d.ts.map +1 -0
  36. package/dist/dev.js +12 -0
  37. package/dist/dev.js.map +1 -0
  38. package/dist/index.d.ts +4 -0
  39. package/dist/index.d.ts.map +1 -0
  40. package/dist/index.js +6 -0
  41. package/dist/index.js.map +1 -0
  42. package/dist/lipsync-helpers/arkitOrder61.d.ts.map +1 -0
  43. package/dist/lipsync-helpers/arkitOrder61.js +287 -0
  44. package/dist/lipsync-helpers/arkitOrder61.js.map +1 -0
  45. package/dist/lipsync-helpers/createBlendshapeQueue.d.ts +86 -0
  46. package/dist/lipsync-helpers/createBlendshapeQueue.d.ts.map +1 -0
  47. package/dist/lipsync-helpers/createBlendshapeQueue.js +126 -0
  48. package/dist/lipsync-helpers/createBlendshapeQueue.js.map +1 -0
  49. package/dist/lipsync-helpers/declarativeMapping.d.ts +93 -0
  50. package/dist/lipsync-helpers/declarativeMapping.d.ts.map +1 -0
  51. package/dist/lipsync-helpers/declarativeMapping.js +228 -0
  52. package/dist/lipsync-helpers/declarativeMapping.js.map +1 -0
  53. package/dist/lipsync-helpers/index.d.ts +15 -0
  54. package/dist/lipsync-helpers/index.d.ts.map +1 -0
  55. package/dist/lipsync-helpers/index.js +31 -1164
  56. package/dist/lipsync-helpers/index.js.map +1 -0
  57. package/dist/lipsync-helpers/mappingTypes.d.ts +64 -0
  58. package/dist/lipsync-helpers/mappingTypes.d.ts.map +1 -0
  59. package/dist/lipsync-helpers/mappingTypes.js +28 -0
  60. package/dist/lipsync-helpers/mappingTypes.js.map +1 -0
  61. package/dist/lipsync-helpers/metahumanOrder251.d.ts +115 -0
  62. package/dist/lipsync-helpers/metahumanOrder251.d.ts.map +1 -0
  63. package/dist/lipsync-helpers/metahumanOrder251.js +432 -0
  64. package/dist/lipsync-helpers/metahumanOrder251.js.map +1 -0
  65. package/dist/lipsync-helpers/presetMappers.d.ts +24 -0
  66. package/dist/lipsync-helpers/presetMappers.d.ts.map +1 -0
  67. package/dist/lipsync-helpers/presetMappers.js +38 -0
  68. package/dist/lipsync-helpers/presetMappers.js.map +1 -0
  69. package/dist/react/components/ConvaiWidget.d.ts +68 -0
  70. package/dist/react/components/ConvaiWidget.d.ts.map +1 -0
  71. package/dist/react/components/ConvaiWidget.js +505 -0
  72. package/dist/react/components/ConvaiWidget.js.map +1 -0
  73. package/dist/react/components/index.d.ts +2 -0
  74. package/dist/react/components/index.d.ts.map +1 -0
  75. package/dist/react/components/index.js +3 -0
  76. package/dist/react/components/index.js.map +1 -0
  77. package/dist/react/components/rtc-widget/components/AudioSettingsPanel.d.ts +10 -0
  78. package/dist/react/components/rtc-widget/components/AudioSettingsPanel.d.ts.map +1 -0
  79. package/dist/react/components/rtc-widget/components/AudioSettingsPanel.js +316 -0
  80. package/dist/react/components/rtc-widget/components/AudioSettingsPanel.js.map +1 -0
  81. package/dist/react/components/rtc-widget/components/AudioVisualizer.d.ts +36 -0
  82. package/dist/react/components/rtc-widget/components/AudioVisualizer.d.ts.map +1 -0
  83. package/dist/react/components/rtc-widget/components/AudioVisualizer.js +259 -0
  84. package/dist/react/components/rtc-widget/components/AudioVisualizer.js.map +1 -0
  85. package/dist/react/components/rtc-widget/components/ConviMessage.d.ts +10 -0
  86. package/dist/react/components/rtc-widget/components/ConviMessage.d.ts.map +1 -0
  87. package/dist/react/components/rtc-widget/components/ConviMessage.js +14 -0
  88. package/dist/react/components/rtc-widget/components/ConviMessage.js.map +1 -0
  89. package/dist/react/components/rtc-widget/components/FloatingVideo.d.ts +9 -0
  90. package/dist/react/components/rtc-widget/components/FloatingVideo.d.ts.map +1 -0
  91. package/dist/react/components/rtc-widget/components/FloatingVideo.js +122 -0
  92. package/dist/react/components/rtc-widget/components/FloatingVideo.js.map +1 -0
  93. package/dist/react/components/rtc-widget/components/MarkdownRenderer.d.ts +7 -0
  94. package/dist/react/components/rtc-widget/components/MarkdownRenderer.d.ts.map +1 -0
  95. package/dist/react/components/rtc-widget/components/MarkdownRenderer.js +68 -0
  96. package/dist/react/components/rtc-widget/components/MarkdownRenderer.js.map +1 -0
  97. package/dist/react/components/rtc-widget/components/MessageBubble.d.ts +10 -0
  98. package/dist/react/components/rtc-widget/components/MessageBubble.d.ts.map +1 -0
  99. package/dist/react/components/rtc-widget/components/MessageBubble.js +23 -0
  100. package/dist/react/components/rtc-widget/components/MessageBubble.js.map +1 -0
  101. package/dist/react/components/rtc-widget/components/MessageList.d.ts +11 -0
  102. package/dist/react/components/rtc-widget/components/MessageList.d.ts.map +1 -0
  103. package/dist/react/components/rtc-widget/components/MessageList.js +89 -0
  104. package/dist/react/components/rtc-widget/components/MessageList.js.map +1 -0
  105. package/dist/react/components/rtc-widget/components/UserMessage.d.ts +9 -0
  106. package/dist/react/components/rtc-widget/components/UserMessage.d.ts.map +1 -0
  107. package/dist/react/components/rtc-widget/components/UserMessage.js +15 -0
  108. package/dist/react/components/rtc-widget/components/UserMessage.js.map +1 -0
  109. package/dist/react/components/rtc-widget/components/conviComponents/ConviButton.d.ts +6 -0
  110. package/dist/react/components/rtc-widget/components/conviComponents/ConviButton.d.ts.map +1 -0
  111. package/dist/react/components/rtc-widget/components/conviComponents/ConviButton.js +15 -0
  112. package/dist/react/components/rtc-widget/components/conviComponents/ConviButton.js.map +1 -0
  113. package/dist/react/components/rtc-widget/components/conviComponents/ConviFooter.d.ts +25 -0
  114. package/dist/react/components/rtc-widget/components/conviComponents/ConviFooter.d.ts.map +1 -0
  115. package/dist/react/components/rtc-widget/components/conviComponents/ConviFooter.js +172 -0
  116. package/dist/react/components/rtc-widget/components/conviComponents/ConviFooter.js.map +1 -0
  117. package/dist/react/components/rtc-widget/components/conviComponents/ConviHeader.d.ts +17 -0
  118. package/dist/react/components/rtc-widget/components/conviComponents/ConviHeader.d.ts.map +1 -0
  119. package/dist/react/components/rtc-widget/components/conviComponents/ConviHeader.js +66 -0
  120. package/dist/react/components/rtc-widget/components/conviComponents/ConviHeader.js.map +1 -0
  121. package/dist/react/components/rtc-widget/components/conviComponents/SettingsTray.d.ts +12 -0
  122. package/dist/react/components/rtc-widget/components/conviComponents/SettingsTray.d.ts.map +1 -0
  123. package/dist/react/components/rtc-widget/components/conviComponents/SettingsTray.js +68 -0
  124. package/dist/react/components/rtc-widget/components/conviComponents/SettingsTray.js.map +1 -0
  125. package/dist/react/components/rtc-widget/components/conviComponents/VoiceModeOverlay.d.ts +12 -0
  126. package/dist/react/components/rtc-widget/components/conviComponents/VoiceModeOverlay.d.ts.map +1 -0
  127. package/dist/react/components/rtc-widget/components/conviComponents/VoiceModeOverlay.js +255 -0
  128. package/dist/react/components/rtc-widget/components/conviComponents/VoiceModeOverlay.js.map +1 -0
  129. package/dist/react/components/rtc-widget/components/conviComponents/index.d.ts +6 -0
  130. package/dist/react/components/rtc-widget/components/conviComponents/index.d.ts.map +1 -0
  131. package/dist/react/components/rtc-widget/components/conviComponents/index.js +6 -0
  132. package/dist/react/components/rtc-widget/components/conviComponents/index.js.map +1 -0
  133. package/dist/react/components/rtc-widget/components/index.d.ts +9 -0
  134. package/dist/react/components/rtc-widget/components/index.d.ts.map +1 -0
  135. package/dist/react/components/rtc-widget/components/index.js +15 -0
  136. package/dist/react/components/rtc-widget/components/index.js.map +1 -0
  137. package/dist/react/components/rtc-widget/index.d.ts +6 -0
  138. package/dist/react/components/rtc-widget/index.d.ts.map +1 -0
  139. package/dist/react/components/rtc-widget/index.js +9 -0
  140. package/dist/react/components/rtc-widget/index.js.map +1 -0
  141. package/dist/react/components/rtc-widget/styles/framerConfig.d.ts +116 -0
  142. package/dist/react/components/rtc-widget/styles/framerConfig.d.ts.map +1 -0
  143. package/dist/react/components/rtc-widget/styles/framerConfig.js +73 -0
  144. package/dist/react/components/rtc-widget/styles/framerConfig.js.map +1 -0
  145. package/dist/react/components/rtc-widget/styles/icons.d.ts +28 -0
  146. package/dist/react/components/rtc-widget/styles/icons.d.ts.map +1 -0
  147. package/dist/react/components/rtc-widget/styles/icons.js +257 -0
  148. package/dist/react/components/rtc-widget/styles/icons.js.map +1 -0
  149. package/dist/react/components/rtc-widget/styles/index.d.ts +6 -0
  150. package/dist/react/components/rtc-widget/styles/index.d.ts.map +1 -0
  151. package/dist/react/components/rtc-widget/styles/index.js +9 -0
  152. package/dist/react/components/rtc-widget/styles/index.js.map +1 -0
  153. package/dist/react/components/rtc-widget/styles/styledComponents.d.ts +90 -0
  154. package/dist/react/components/rtc-widget/styles/styledComponents.d.ts.map +1 -0
  155. package/dist/react/components/rtc-widget/styles/styledComponents.js +663 -0
  156. package/dist/react/components/rtc-widget/styles/styledComponents.js.map +1 -0
  157. package/dist/react/components/rtc-widget/styles/theme.d.ts +188 -0
  158. package/dist/react/components/rtc-widget/styles/theme.d.ts.map +1 -0
  159. package/dist/react/components/rtc-widget/styles/theme.js +290 -0
  160. package/dist/react/components/rtc-widget/styles/theme.js.map +1 -0
  161. package/dist/react/components/rtc-widget/types/index.d.ts +60 -0
  162. package/dist/react/components/rtc-widget/types/index.d.ts.map +1 -0
  163. package/dist/react/components/rtc-widget/types/index.js +2 -0
  164. package/dist/react/components/rtc-widget/types/index.js.map +1 -0
  165. package/dist/react/hooks/index.d.ts +4 -0
  166. package/dist/react/hooks/index.d.ts.map +1 -0
  167. package/dist/react/hooks/index.js +6 -0
  168. package/dist/react/hooks/index.js.map +1 -0
  169. package/dist/react/hooks/useCharacterInfo.d.ts +17 -0
  170. package/dist/react/hooks/useCharacterInfo.d.ts.map +1 -0
  171. package/dist/react/hooks/useCharacterInfo.js +60 -0
  172. package/dist/react/hooks/useCharacterInfo.js.map +1 -0
  173. package/dist/react/hooks/useConvaiClient.d.ts +35 -0
  174. package/dist/react/hooks/useConvaiClient.d.ts.map +1 -0
  175. package/dist/react/hooks/useConvaiClient.js +179 -0
  176. package/dist/react/hooks/useConvaiClient.js.map +1 -0
  177. package/dist/react/hooks/useLocalCameraTrack.d.ts +22 -0
  178. package/dist/react/hooks/useLocalCameraTrack.d.ts.map +1 -0
  179. package/dist/react/hooks/useLocalCameraTrack.js +34 -0
  180. package/dist/react/hooks/useLocalCameraTrack.js.map +1 -0
  181. package/dist/react/index.d.ts +7 -0
  182. package/dist/react/index.d.ts.map +1 -0
  183. package/dist/react/index.js +13 -0
  184. package/dist/react/index.js.map +1 -0
  185. package/dist/types/index.d.ts.map +1 -0
  186. package/dist/types/index.js +2 -0
  187. package/dist/types/index.js.map +1 -0
  188. package/dist/utils/LatencyMonitor.d.ts.map +1 -0
  189. package/dist/utils/LatencyMonitor.js +136 -0
  190. package/dist/utils/LatencyMonitor.js.map +1 -0
  191. package/dist/utils/logger.d.ts.map +1 -0
  192. package/dist/utils/logger.js +96 -0
  193. package/dist/utils/logger.js.map +1 -0
  194. package/dist/utils/speakerManagement.d.ts.map +1 -0
  195. package/dist/utils/speakerManagement.js +64 -0
  196. package/dist/utils/speakerManagement.js.map +1 -0
  197. package/dist/{types/vanilla → vanilla}/AudioRenderer.d.ts +5 -0
  198. package/dist/vanilla/AudioRenderer.d.ts.map +1 -0
  199. package/dist/vanilla/AudioRenderer.js +135 -0
  200. package/dist/vanilla/AudioRenderer.js.map +1 -0
  201. package/dist/{types/vanilla → vanilla}/ConvaiWidget.d.ts +13 -6
  202. package/dist/vanilla/ConvaiWidget.d.ts.map +1 -0
  203. package/dist/vanilla/ConvaiWidget.js +1823 -0
  204. package/dist/vanilla/ConvaiWidget.js.map +1 -0
  205. package/dist/vanilla/icons.d.ts.map +1 -0
  206. package/dist/vanilla/icons.js +222 -0
  207. package/dist/vanilla/icons.js.map +1 -0
  208. package/dist/{types/vanilla → vanilla}/index.d.ts +1 -3
  209. package/dist/vanilla/index.d.ts.map +1 -0
  210. package/dist/vanilla/index.js +20 -5509
  211. package/dist/vanilla/index.js.map +1 -0
  212. package/dist/vanilla/styles.d.ts.map +1 -0
  213. package/dist/vanilla/styles.js +287 -0
  214. package/dist/vanilla/styles.js.map +1 -0
  215. package/dist/vanilla/types.d.ts +63 -0
  216. package/dist/vanilla/types.d.ts.map +1 -0
  217. package/dist/vanilla/types.js +2 -0
  218. package/dist/vanilla/types.js.map +1 -0
  219. package/package.json +33 -38
  220. package/CHANGELOG.md +0 -165
  221. package/dist/core/index.cjs +0 -1977
  222. package/dist/lipsync-helpers/index.cjs +0 -1195
  223. package/dist/types/core/AudioManager.d.ts.map +0 -1
  224. package/dist/types/core/ConvaiClient.d.ts.map +0 -1
  225. package/dist/types/core/EventEmitter.d.ts.map +0 -1
  226. package/dist/types/core/MessageHandler.d.ts.map +0 -1
  227. package/dist/types/core/ScreenShareManager.d.ts.map +0 -1
  228. package/dist/types/core/VideoManager.d.ts.map +0 -1
  229. package/dist/types/core/index.d.ts.map +0 -1
  230. package/dist/types/core/types.d.ts.map +0 -1
  231. package/dist/types/lipsync-helpers/arkitBlendshapeHelpers.d.ts +0 -80
  232. package/dist/types/lipsync-helpers/arkitBlendshapeHelpers.d.ts.map +0 -1
  233. package/dist/types/lipsync-helpers/arkitOrder61.d.ts.map +0 -1
  234. package/dist/types/lipsync-helpers/arkitPhonemeReference.d.ts +0 -155
  235. package/dist/types/lipsync-helpers/arkitPhonemeReference.d.ts.map +0 -1
  236. package/dist/types/lipsync-helpers/index.d.ts +0 -9
  237. package/dist/types/lipsync-helpers/index.d.ts.map +0 -1
  238. package/dist/types/lipsync-helpers/neurosyncBlendshapeMapper.d.ts +0 -30
  239. package/dist/types/lipsync-helpers/neurosyncBlendshapeMapper.d.ts.map +0 -1
  240. package/dist/types/types/index.d.ts.map +0 -1
  241. package/dist/types/utils/LatencyMonitor.d.ts.map +0 -1
  242. package/dist/types/utils/logger.d.ts.map +0 -1
  243. package/dist/types/utils/speakerManagement.d.ts.map +0 -1
  244. package/dist/types/vanilla/AudioRenderer.d.ts.map +0 -1
  245. package/dist/types/vanilla/ConvaiWidget.d.ts.map +0 -1
  246. package/dist/types/vanilla/icons.d.ts.map +0 -1
  247. package/dist/types/vanilla/index.d.ts.map +0 -1
  248. package/dist/types/vanilla/styles.d.ts.map +0 -1
  249. package/dist/types/vanilla/types.d.ts +0 -106
  250. package/dist/types/vanilla/types.d.ts.map +0 -1
  251. package/dist/umd/convai.umd.js +0 -1
  252. package/dist/vanilla/index.cjs +0 -5559
  253. /package/dist/{types/core → core}/AudioManager.d.ts +0 -0
  254. /package/dist/{types/core → core}/EventEmitter.d.ts +0 -0
  255. /package/dist/{types/core → core}/ScreenShareManager.d.ts +0 -0
  256. /package/dist/{types/core → core}/VideoManager.d.ts +0 -0
  257. /package/dist/{types/lipsync-helpers → lipsync-helpers}/arkitOrder61.d.ts +0 -0
  258. /package/dist/types/{types/index.d.ts → index.d.ts} +0 -0
  259. /package/dist/{types/utils → utils}/LatencyMonitor.d.ts +0 -0
  260. /package/dist/{types/utils → utils}/logger.d.ts +0 -0
  261. /package/dist/{types/utils → utils}/speakerManagement.d.ts +0 -0
  262. /package/dist/{types/vanilla → vanilla}/icons.d.ts +0 -0
  263. /package/dist/{types/vanilla → vanilla}/styles.d.ts +0 -0
@@ -1,1195 +0,0 @@
1
- 'use strict';
2
-
3
- /**
4
- * ARKit 61-Element Custom Order Mapper
5
- * Maps the custom 61-element ARKit blendshape array to standard ARKit indices
6
- *
7
- * This ordering includes:
8
- * - 52 standard ARKit facial blendshapes (in custom order)
9
- * - 9 additional values for head and eye rotations
10
- */
11
- /**
12
- * Custom ARKit 61-element ordering
13
- * This is the order in which blendshapes are received from the server
14
- */
15
- const ARKIT_ORDER_61 = [
16
- "EyeBlinkLeft", // 0
17
- "EyeLookDownLeft", // 1
18
- "EyeLookInLeft", // 2
19
- "EyeLookOutLeft", // 3
20
- "EyeLookUpLeft", // 4
21
- "EyeSquintLeft", // 5
22
- "EyeWideLeft", // 6
23
- "EyeBlinkRight", // 7
24
- "EyeLookDownRight", // 8
25
- "EyeLookInRight", // 9
26
- "EyeLookOutRight", // 10
27
- "EyeLookUpRight", // 11
28
- "EyeSquintRight", // 12
29
- "EyeWideRight", // 13
30
- "JawForward", // 14
31
- "JawRight", // 15
32
- "JawLeft", // 16
33
- "JawOpen", // 17 🔑 KEY for mouth opening
34
- "MouthClose", // 18 🔑 KEY for mouth closing
35
- "MouthFunnel", // 19 🔑 KEY for "O" sounds
36
- "MouthPucker", // 20 🔑 KEY for "U" sounds
37
- "MouthRight", // 21
38
- "MouthLeft", // 22
39
- "MouthSmileLeft", // 23 🔑 KEY for "ee" sounds
40
- "MouthSmileRight", // 24 🔑 KEY for "ee" sounds
41
- "MouthFrownLeft", // 25
42
- "MouthFrownRight", // 26
43
- "MouthDimpleLeft", // 27
44
- "MouthDimpleRight", // 28
45
- "MouthStretchLeft", // 29 🔑 KEY for wide mouth
46
- "MouthStretchRight", // 30 🔑 KEY for wide mouth
47
- "MouthRollLower", // 31
48
- "MouthRollUpper", // 32
49
- "MouthShrugLower", // 33
50
- "MouthShrugUpper", // 34
51
- "MouthPressLeft", // 35
52
- "MouthPressRight", // 36
53
- "MouthLowerDownLeft", // 37
54
- "MouthLowerDownRight", // 38
55
- "MouthUpperUpLeft", // 39
56
- "MouthUpperUpRight", // 40
57
- "BrowDownLeft", // 41
58
- "BrowDownRight", // 42
59
- "BrowInnerUp", // 43
60
- "BrowOuterUpLeft", // 44
61
- "BrowOuterUpRight", // 45
62
- "CheekPuff", // 46
63
- "CheekSquintLeft", // 47
64
- "CheekSquintRight", // 48
65
- "NoseSneerLeft", // 49
66
- "NoseSneerRight", // 50
67
- "TongueOut", // 51 🔑 KEY for "th" sounds
68
- "HeadYaw", // 52 - Head rotation left/right
69
- "HeadPitch", // 53 - Head rotation up/down
70
- "HeadRoll", // 54 - Head rotation tilt
71
- "LeftEyeYaw", // 55 - Left eye rotation left/right
72
- "LeftEyePitch", // 56 - Left eye rotation up/down
73
- "LeftEyeRoll", // 57 - Left eye rotation roll
74
- "RightEyeYaw", // 58 - Right eye rotation left/right
75
- "RightEyePitch", // 59 - Right eye rotation up/down
76
- "RightEyeRoll", // 60 - Right eye rotation roll
77
- ];
78
- /**
79
- * Standard ARKit blendshape order (52 elements)
80
- * This is Apple's official ordering
81
- */
82
- const STANDARD_ARKIT_ORDER = [
83
- "BrowDownLeft", // 0
84
- "BrowDownRight", // 1
85
- "BrowInnerUp", // 2
86
- "BrowOuterUpLeft", // 3
87
- "BrowOuterUpRight", // 4
88
- "CheekPuff", // 5
89
- "CheekSquintLeft", // 6
90
- "CheekSquintRight", // 7
91
- "EyeBlinkLeft", // 8
92
- "EyeBlinkRight", // 9
93
- "EyeLookDownLeft", // 10
94
- "EyeLookDownRight", // 11
95
- "EyeLookInLeft", // 12
96
- "EyeLookInRight", // 13
97
- "EyeLookOutLeft", // 14
98
- "EyeLookOutRight", // 15
99
- "EyeLookUpLeft", // 16
100
- "EyeLookUpRight", // 17
101
- "EyeSquintLeft", // 18
102
- "EyeSquintRight", // 19
103
- "EyeWideLeft", // 20
104
- "EyeWideRight", // 21
105
- "JawForward", // 22
106
- "JawLeft", // 23
107
- "JawOpen", // 24
108
- "JawRight", // 25
109
- "MouthClose", // 26
110
- "MouthDimpleLeft", // 27
111
- "MouthDimpleRight", // 28
112
- "MouthFrownLeft", // 29
113
- "MouthFrownRight", // 30
114
- "MouthFunnel", // 31
115
- "MouthLeft", // 32
116
- "MouthLowerDownLeft", // 33
117
- "MouthLowerDownRight", // 34
118
- "MouthPressLeft", // 35
119
- "MouthPressRight", // 36
120
- "MouthPucker", // 37
121
- "MouthRight", // 38
122
- "MouthRollLower", // 39
123
- "MouthRollUpper", // 40
124
- "MouthShrugLower", // 41
125
- "MouthShrugUpper", // 42
126
- "MouthSmileLeft", // 43
127
- "MouthSmileRight", // 44
128
- "MouthStretchLeft", // 45
129
- "MouthStretchRight", // 46
130
- "MouthUpperUpLeft", // 47
131
- "MouthUpperUpRight", // 48
132
- "NoseSneerLeft", // 49
133
- "NoseSneerRight", // 50
134
- "TongueOut", // 51
135
- ];
136
- /**
137
- * Mapping from custom 61-order indices to standard ARKit indices
138
- * Maps ARKIT_ORDER_61[i] -> STANDARD_ARKIT_ORDER[j]
139
- */
140
- const ORDER_61_TO_STANDARD_MAP = new Map();
141
- // Build the mapping at module initialization
142
- for (let i = 0; i < ARKIT_ORDER_61.length; i++) {
143
- const name = ARKIT_ORDER_61[i];
144
- // Only map the first 52 (standard ARKit blendshapes)
145
- if (i < 52) {
146
- const standardIndex = STANDARD_ARKIT_ORDER.indexOf(name);
147
- if (standardIndex !== -1) {
148
- ORDER_61_TO_STANDARD_MAP.set(i, standardIndex);
149
- }
150
- }
151
- }
152
- /**
153
- * Convert 61-element custom order array to standard 52-element ARKit array
154
- * @param values61 - 61-element array in custom order
155
- * @returns 52-element array in standard ARKit order
156
- */
157
- function convertOrder61ToStandard(values61) {
158
- const standardArray = new Array(52).fill(0);
159
- for (let i = 0; i < Math.min(52, values61.length); i++) {
160
- const standardIndex = ORDER_61_TO_STANDARD_MAP.get(i);
161
- if (standardIndex !== undefined) {
162
- standardArray[standardIndex] = values61[i];
163
- }
164
- }
165
- return standardArray;
166
- }
167
- /**
168
- * Map 61-element array to named object (all 61 values)
169
- * @param values61 - 61-element array in custom order
170
- * @returns Object with blendshape/rotation names as keys
171
- */
172
- function mapOrder61ToNames(values61) {
173
- const mapped = {};
174
- for (let i = 0; i < Math.min(values61.length, ARKIT_ORDER_61.length); i++) {
175
- // Convert to camelCase (lowercase first letter)
176
- const name = ARKIT_ORDER_61[i];
177
- const camelCaseName = name.charAt(0).toLowerCase() + name.slice(1);
178
- mapped[camelCaseName] = values61[i];
179
- }
180
- return mapped;
181
- }
182
- /**
183
- * Get specific blendshape value by name from 61-element array
184
- * @param values61 - 61-element array in custom order
185
- * @param name - Name of the blendshape (case-insensitive)
186
- * @returns The value, or 0 if not found
187
- */
188
- function getBlendshapeValueFromOrder61(values61, name) {
189
- const upperName = name.charAt(0).toUpperCase() + name.slice(1);
190
- const index = ARKIT_ORDER_61.indexOf(upperName);
191
- if (index !== -1 && index < values61.length) {
192
- return values61[index];
193
- }
194
- return 0;
195
- }
196
- /**
197
- * Extract head rotation values from 61-element array
198
- * @param values61 - 61-element array in custom order
199
- * @returns Object with head rotation values (yaw, pitch, roll)
200
- */
201
- function getHeadRotation(values61) {
202
- return {
203
- yaw: values61[52] || 0,
204
- pitch: values61[53] || 0,
205
- roll: values61[54] || 0,
206
- };
207
- }
208
- /**
209
- * Extract eye rotation values from 61-element array
210
- * @param values61 - 61-element array in custom order
211
- * @returns Object with left and right eye rotation values
212
- */
213
- function getEyeRotations(values61) {
214
- return {
215
- left: {
216
- yaw: values61[55] || 0,
217
- pitch: values61[56] || 0,
218
- roll: values61[57] || 0,
219
- },
220
- right: {
221
- yaw: values61[58] || 0,
222
- pitch: values61[59] || 0,
223
- roll: values61[60] || 0,
224
- },
225
- };
226
- }
227
- /**
228
- * Get key lip-sync blendshapes from 61-element array
229
- * These are the most important values for facial animation
230
- */
231
- function getLipSyncValuesFromOrder61(values61) {
232
- return {
233
- jawOpen: values61[17] || 0,
234
- mouthClose: values61[18] || 0,
235
- mouthFunnel: values61[19] || 0,
236
- mouthPucker: values61[20] || 0,
237
- mouthSmileLeft: values61[23] || 0,
238
- mouthSmileRight: values61[24] || 0,
239
- mouthStretchLeft: values61[29] || 0,
240
- mouthStretchRight: values61[30] || 0,
241
- mouthRollLower: values61[31] || 0,
242
- mouthRollUpper: values61[32] || 0,
243
- tongueOut: values61[51] || 0,
244
- };
245
- }
246
- /**
247
- * Get active blendshapes from 61-element array (above threshold)
248
- * @param values61 - 61-element array in custom order
249
- * @param threshold - Minimum value to consider "active" (default: 0.01)
250
- * @returns Sorted array of active blendshapes
251
- */
252
- function getActiveBlendshapesFromOrder61(values61, threshold = 0.01) {
253
- const active = [];
254
- for (let i = 0; i < Math.min(values61.length, ARKIT_ORDER_61.length); i++) {
255
- if (Math.abs(values61[i]) > threshold) {
256
- const name = ARKIT_ORDER_61[i];
257
- const camelCaseName = name.charAt(0).toLowerCase() + name.slice(1);
258
- active.push({
259
- name: camelCaseName,
260
- value: values61[i],
261
- index: i,
262
- });
263
- }
264
- }
265
- // Sort by absolute value (descending)
266
- return active.sort((a, b) => Math.abs(b.value) - Math.abs(a.value));
267
- }
268
- /**
269
- * Format 61-element array for logging
270
- * @param values61 - 61-element array in custom order
271
- * @param threshold - Only show values above this (default: 0.01)
272
- * @param maxEntries - Maximum entries to show (default: 20)
273
- */
274
- function formatOrder61ForLogging(values61, threshold = 0.01, maxEntries = 20) {
275
- const active = getActiveBlendshapesFromOrder61(values61, threshold)
276
- .slice(0, maxEntries)
277
- .reduce((acc, item) => {
278
- acc[item.name] = Number(item.value.toFixed(4));
279
- return acc;
280
- }, {});
281
- return {
282
- active,
283
- total: values61.length,
284
- activeCount: Object.keys(active).length,
285
- headRotation: getHeadRotation(values61),
286
- eyeRotations: getEyeRotations(values61),
287
- };
288
- }
289
-
290
- /**
291
- * ARKit Blendshape Helpers
292
- * Utilities for working with ARKit blendshape arrays (52-60 elements)
293
- *
294
- * ARKit Standard: 52 blendshapes (indices 0-51)
295
- * ARKit Extended: Up to 60 blendshapes with additional facial features
296
- */
297
- /**
298
- * ARKit Standard Blendshape Names (indices 0-51)
299
- */
300
- const ARKIT_BLENDSHAPE_NAMES = [
301
- "browDownLeft", // 0
302
- "browDownRight", // 1
303
- "browInnerUp", // 2
304
- "browOuterUpLeft", // 3
305
- "browOuterUpRight", // 4
306
- "cheekPuff", // 5
307
- "cheekSquintLeft", // 6
308
- "cheekSquintRight", // 7
309
- "eyeBlinkLeft", // 8
310
- "eyeBlinkRight", // 9
311
- "eyeLookDownLeft", // 10
312
- "eyeLookDownRight", // 11
313
- "eyeLookInLeft", // 12
314
- "eyeLookInRight", // 13
315
- "eyeLookOutLeft", // 14
316
- "eyeLookOutRight", // 15
317
- "eyeLookUpLeft", // 16
318
- "eyeLookUpRight", // 17
319
- "eyeSquintLeft", // 18
320
- "eyeSquintRight", // 19
321
- "eyeWideLeft", // 20
322
- "eyeWideRight", // 21
323
- "jawForward", // 22
324
- "jawLeft", // 23
325
- "jawOpen", // 24 🔑 KEY for mouth opening
326
- "jawRight", // 25
327
- "mouthClose", // 26 🔑 KEY for mouth closing
328
- "mouthDimpleLeft", // 27
329
- "mouthDimpleRight", // 28
330
- "mouthFrownLeft", // 29
331
- "mouthFrownRight", // 30
332
- "mouthFunnel", // 31 🔑 KEY for "O" sounds
333
- "mouthLeft", // 32
334
- "mouthLowerDownLeft", // 33
335
- "mouthLowerDownRight", // 34
336
- "mouthPressLeft", // 35
337
- "mouthPressRight", // 36
338
- "mouthPucker", // 37 🔑 KEY for "U" sounds
339
- "mouthRight", // 38
340
- "mouthRollLower", // 39
341
- "mouthRollUpper", // 40
342
- "mouthShrugLower", // 41
343
- "mouthShrugUpper", // 42
344
- "mouthSmileLeft", // 43 🔑 KEY for "ee" sounds
345
- "mouthSmileRight", // 44 🔑 KEY for "ee" sounds
346
- "mouthStretchLeft", // 45 🔑 KEY for wide mouth
347
- "mouthStretchRight", // 46 🔑 KEY for wide mouth
348
- "mouthUpperUpLeft", // 47
349
- "mouthUpperUpRight", // 48
350
- "noseSneerLeft", // 49
351
- "noseSneerRight", // 50
352
- "tongueOut", // 51 🔑 KEY for "th" sounds
353
- ];
354
- /**
355
- * Extended blendshapes (indices 52-59) - if provided
356
- * These are optional extensions beyond standard ARKit
357
- */
358
- const ARKIT_EXTENDED_NAMES = [
359
- "eyeDilation", // 52 - Pupil dilation
360
- "eyeConstriction", // 53 - Pupil constriction
361
- "mouthShrug", // 54 - Combined mouth shrug
362
- "lipSuckUpper", // 55 - Upper lip suck
363
- "lipSuckLower", // 56 - Lower lip suck
364
- "lipFunnel", // 57 - Lip funnel position
365
- "lipPress", // 58 - Lip press together
366
- "reserved", // 59 - Reserved for future use
367
- ];
368
- /**
369
- * Map ARKit blendshape array to named object (ARKit standard only: 0-51)
370
- * @param values - Array of blendshape values (length 52-60)
371
- * @returns Object with blendshape names as keys
372
- */
373
- function mapARKitBlendshapesToNames(values) {
374
- const mapped = {};
375
- // Map standard ARKit blendshapes (0-51)
376
- for (let i = 0; i < Math.min(values.length, ARKIT_BLENDSHAPE_NAMES.length); i++) {
377
- mapped[ARKIT_BLENDSHAPE_NAMES[i]] = values[i];
378
- }
379
- // Map extended blendshapes if available (52+)
380
- if (values.length > 52) {
381
- const extendedStart = 52;
382
- for (let i = 0; i < Math.min(values.length - extendedStart, ARKIT_EXTENDED_NAMES.length); i++) {
383
- mapped[ARKIT_EXTENDED_NAMES[i]] = values[extendedStart + i];
384
- }
385
- }
386
- return mapped;
387
- }
388
- /**
389
- * Get only active blendshapes (above threshold)
390
- * @param values - Array of blendshape values
391
- * @param threshold - Minimum value to consider "active" (default: 0.01)
392
- * @returns Sorted array of active blendshapes
393
- */
394
- function getActiveARKitBlendshapes(values, threshold = 0.01) {
395
- const active = [];
396
- // Check standard ARKit blendshapes
397
- for (let i = 0; i < Math.min(values.length, ARKIT_BLENDSHAPE_NAMES.length); i++) {
398
- if (Math.abs(values[i]) > threshold) {
399
- active.push({
400
- name: ARKIT_BLENDSHAPE_NAMES[i],
401
- value: values[i],
402
- index: i,
403
- });
404
- }
405
- }
406
- // Check extended blendshapes
407
- if (values.length > 52) {
408
- for (let i = 52; i < Math.min(values.length, 52 + ARKIT_EXTENDED_NAMES.length); i++) {
409
- if (Math.abs(values[i]) > threshold) {
410
- active.push({
411
- name: ARKIT_EXTENDED_NAMES[i - 52],
412
- value: values[i],
413
- index: i,
414
- });
415
- }
416
- }
417
- }
418
- // Sort by absolute value (descending)
419
- return active.sort((a, b) => Math.abs(b.value) - Math.abs(a.value));
420
- }
421
- /**
422
- * Get specific blendshape value by name
423
- * @param values - Array of blendshape values
424
- * @param name - Name of the blendshape
425
- * @returns The value, or 0 if not found
426
- */
427
- function getARKitBlendshapeValue(values, name) {
428
- const index = ARKIT_BLENDSHAPE_NAMES.indexOf(name);
429
- if (index !== -1 && index < values.length) {
430
- return values[index];
431
- }
432
- const extendedIndex = ARKIT_EXTENDED_NAMES.indexOf(name);
433
- if (extendedIndex !== -1) {
434
- const actualIndex = 52 + extendedIndex;
435
- if (actualIndex < values.length) {
436
- return values[actualIndex];
437
- }
438
- }
439
- return 0;
440
- }
441
- /**
442
- * Format ARKit blendshapes for logging
443
- * @param values - Array of blendshape values
444
- * @param threshold - Only show values above this (default: 0.01)
445
- * @param maxEntries - Maximum entries to show (default: 20)
446
- */
447
- function formatARKitBlendshapesForLogging(values, threshold = 0.01, maxEntries = 20) {
448
- const active = getActiveARKitBlendshapes(values, threshold)
449
- .slice(0, maxEntries)
450
- .reduce((acc, item) => {
451
- acc[item.name] = Number(item.value.toFixed(4));
452
- return acc;
453
- }, {});
454
- return {
455
- active,
456
- total: values.length,
457
- activeCount: Object.keys(active).length,
458
- };
459
- }
460
- /**
461
- * Get the most important facial features for speech
462
- * These are the key blendshapes for lip-sync
463
- */
464
- const KEY_LIPSYNC_BLENDSHAPES = {
465
- jaw: [24], // jawOpen
466
- lips: [26, 31, 37, 39, 40], // mouthClose, mouthFunnel, mouthPucker, mouthRollLower, mouthRollUpper
467
- smile: [43, 44, 45, 46], // mouthSmile*, mouthStretch*
468
- tongue: [51], // tongueOut
469
- };
470
- /**
471
- * Extract only lip-sync relevant blendshapes
472
- * @param values - Full blendshape array
473
- * @returns Object with categorized lip-sync blendshapes
474
- */
475
- function getLipSyncBlendshapes(values) {
476
- return {
477
- jaw: values[24] || 0,
478
- mouthClose: values[26] || 0,
479
- mouthFunnel: values[31] || 0,
480
- mouthPucker: values[37] || 0,
481
- mouthRollLower: values[39] || 0,
482
- mouthRollUpper: values[40] || 0,
483
- mouthSmileLeft: values[43] || 0,
484
- mouthSmileRight: values[44] || 0,
485
- mouthStretchLeft: values[45] || 0,
486
- mouthStretchRight: values[46] || 0,
487
- tongueOut: values[51] || 0,
488
- };
489
- }
490
-
491
- /**
492
- * ARKit Blendshape Reference Values for Phonemes/Visemes
493
- *
494
- * This file provides reference values for ARKit's 52 standard facial blendshapes
495
- * when forming different phonemes (speech sounds). These values are based on
496
- * industry standards from systems like OVR LipSync, Oculus Audio, and ARKit face tracking.
497
- *
498
- * All blendshape values range from 0.0 (no activation) to 1.0 (full activation)
499
- *
500
- * ARKit Blendshapes: Indices 0-51
501
- * Meta Human Extended: Indices 52-250
502
- */
503
- /**
504
- * Reference values for common phonemes
505
- * Based on International Phonetic Alphabet (IPA) and Arpabet standards
506
- */
507
- const ARKIT_PHONEME_REFERENCE_VALUES = {
508
- // ============================================
509
- // SILENCE / NEUTRAL
510
- // ============================================
511
- sil: {
512
- jawOpen: 0.0,
513
- mouthClose: 1.0,
514
- description: "Silence - mouth closed, neutral expression"
515
- },
516
- // ============================================
517
- // VOWELS
518
- // ============================================
519
- aa: {
520
- jawOpen: 0.7,
521
- mouthFunnel: 0.0,
522
- mouthPucker: 0.0,
523
- tongueOut: 0.1,
524
- description: "Open back vowel - 'ah' (father, hot)"
525
- },
526
- ae: {
527
- jawOpen: 0.5,
528
- mouthStretchLeft: 0.4,
529
- mouthStretchRight: 0.4,
530
- description: "Near-open front vowel - 'a' (cat, bat)"
531
- },
532
- E: {
533
- jawOpen: 0.4,
534
- mouthStretchLeft: 0.3,
535
- mouthStretchRight: 0.3,
536
- mouthSmileLeft: 0.2,
537
- mouthSmileRight: 0.2,
538
- description: "Open-mid front vowel - 'eh' (bed, get)"
539
- },
540
- I: {
541
- jawOpen: 0.1,
542
- mouthStretchLeft: 0.6,
543
- mouthStretchRight: 0.6,
544
- mouthSmileLeft: 0.5,
545
- mouthSmileRight: 0.5,
546
- mouthClose: 0.3,
547
- description: "Close front vowel - 'ee' (see, feet)"
548
- },
549
- ih: {
550
- jawOpen: 0.2,
551
- mouthStretchLeft: 0.4,
552
- mouthStretchRight: 0.4,
553
- mouthSmileLeft: 0.3,
554
- mouthSmileRight: 0.3,
555
- description: "Near-close front vowel - 'i' (sit, bit)"
556
- },
557
- O: {
558
- jawOpen: 0.5,
559
- mouthFunnel: 0.6,
560
- mouthPucker: 0.3,
561
- description: "Close-mid back rounded vowel - 'oh' (go, boat)"
562
- },
563
- aw: {
564
- jawOpen: 0.6,
565
- mouthFunnel: 0.4,
566
- mouthPucker: 0.2,
567
- description: "Open-mid back rounded vowel - 'aw' (thought, caught)"
568
- },
569
- U: {
570
- jawOpen: 0.2,
571
- mouthFunnel: 0.8,
572
- mouthPucker: 0.9,
573
- description: "Close back rounded vowel - 'oo' (food, boot)"
574
- },
575
- uh: {
576
- jawOpen: 0.25,
577
- mouthFunnel: 0.5,
578
- mouthPucker: 0.6,
579
- description: "Near-close back rounded vowel - 'u' (book, foot)"
580
- },
581
- er: {
582
- jawOpen: 0.35,
583
- mouthFunnel: 0.3,
584
- mouthPucker: 0.1,
585
- description: "R-colored mid-central vowel - 'er' (her, bird)"
586
- },
587
- ah: {
588
- jawOpen: 0.4,
589
- mouthFunnel: 0.1,
590
- description: "Open-mid back vowel - 'uh' (but, cup)"
591
- },
592
- // ============================================
593
- // DIPHTHONGS
594
- // ============================================
595
- ow: {
596
- jawOpen: 0.6,
597
- mouthFunnel: 0.3,
598
- mouthPucker: 0.2,
599
- description: "Diphthong - 'ow' (cow, house)"
600
- },
601
- ay: {
602
- jawOpen: 0.5,
603
- mouthStretchLeft: 0.3,
604
- mouthStretchRight: 0.3,
605
- description: "Diphthong - 'ai' (eye, my)"
606
- },
607
- oy: {
608
- jawOpen: 0.5,
609
- mouthFunnel: 0.4,
610
- mouthPucker: 0.3,
611
- description: "Diphthong - 'oy' (boy, coin)"
612
- },
613
- // ============================================
614
- // CONSONANTS - BILABIALS (both lips)
615
- // ============================================
616
- PP: {
617
- jawOpen: 0.0,
618
- mouthClose: 1.0,
619
- mouthPressLeft: 0.5,
620
- mouthPressRight: 0.5,
621
- description: "Bilabial stops/nasal - 'p', 'b', 'm' (pat, bat, mat)"
622
- },
623
- // ============================================
624
- // CONSONANTS - LABIODENTAL (teeth on lip)
625
- // ============================================
626
- FF: {
627
- jawOpen: 0.15,
628
- mouthRollLower: 0.6,
629
- mouthUpperUpLeft: 0.3,
630
- mouthUpperUpRight: 0.3,
631
- description: "Labiodental fricatives - 'f', 'v' (fat, vat)"
632
- },
633
- // ============================================
634
- // CONSONANTS - DENTAL/INTERDENTAL
635
- // ============================================
636
- TH: {
637
- jawOpen: 0.2,
638
- tongueOut: 0.4,
639
- mouthUpperUpLeft: 0.2,
640
- mouthUpperUpRight: 0.2,
641
- mouthRollLower: 0.2,
642
- description: "Dental fricatives - 'th' (think, this)"
643
- },
644
- // ============================================
645
- // CONSONANTS - ALVEOLAR (tongue to ridge)
646
- // ============================================
647
- DD: {
648
- jawOpen: 0.3,
649
- mouthClose: 0.2,
650
- description: "Alveolar stops - 't', 'd' (tap, dap)"
651
- },
652
- SS: {
653
- jawOpen: 0.15,
654
- mouthStretchLeft: 0.4,
655
- mouthStretchRight: 0.4,
656
- mouthClose: 0.4,
657
- description: "Alveolar sibilants - 's', 'z' (sap, zap)"
658
- },
659
- nn: {
660
- jawOpen: 0.2,
661
- mouthClose: 0.3,
662
- description: "Alveolar nasal - 'n' (nap)"
663
- },
664
- l: {
665
- jawOpen: 0.25,
666
- tongueOut: 0.2,
667
- description: "Alveolar lateral approximant - 'l' (lap)"
668
- },
669
- // ============================================
670
- // CONSONANTS - POSTALVEOLAR (behind ridge)
671
- // ============================================
672
- CH: {
673
- jawOpen: 0.2,
674
- mouthFunnel: 0.3,
675
- mouthPucker: 0.2,
676
- description: "Postalveolar affricates - 'ch', 'j' (church, judge)"
677
- },
678
- sh: {
679
- jawOpen: 0.2,
680
- mouthFunnel: 0.4,
681
- mouthPucker: 0.3,
682
- description: "Postalveolar fricatives - 'sh', 'zh' (shoe, measure)"
683
- },
684
- // ============================================
685
- // CONSONANTS - RETROFLEX
686
- // ============================================
687
- RR: {
688
- jawOpen: 0.3,
689
- mouthFunnel: 0.2,
690
- mouthPucker: 0.15,
691
- tongueOut: 0.1,
692
- description: "Alveolar approximant - 'r' (red)"
693
- },
694
- // ============================================
695
- // CONSONANTS - PALATAL
696
- // ============================================
697
- y: {
698
- jawOpen: 0.2,
699
- mouthStretchLeft: 0.35,
700
- mouthStretchRight: 0.35,
701
- mouthSmileLeft: 0.2,
702
- mouthSmileRight: 0.2,
703
- description: "Palatal approximant - 'y' (yes)"
704
- },
705
- // ============================================
706
- // CONSONANTS - VELAR (back of tongue)
707
- // ============================================
708
- kk: {
709
- jawOpen: 0.35,
710
- description: "Velar stops/nasal - 'k', 'g', 'ng' (kite, go, sing)"
711
- },
712
- // ============================================
713
- // CONSONANTS - GLOTTAL
714
- // ============================================
715
- hh: {
716
- jawOpen: 0.3,
717
- description: "Glottal fricative - 'h' (hat)"
718
- },
719
- // ============================================
720
- // CONSONANTS - LABIO-VELAR
721
- // ============================================
722
- w: {
723
- jawOpen: 0.15,
724
- mouthPucker: 0.8,
725
- mouthFunnel: 0.6,
726
- description: "Labio-velar approximant - 'w' (wet)"
727
- },
728
- };
729
- /**
730
- * ARKit Blendshape Indices (0-51)
731
- * These are Apple's standard 52 facial blendshapes
732
- */
733
- const ARKIT_BLENDSHAPE_INDICES = {
734
- browDownLeft: 0,
735
- browDownRight: 1,
736
- browInnerUp: 2,
737
- browOuterUpLeft: 3,
738
- browOuterUpRight: 4,
739
- cheekPuff: 5,
740
- cheekSquintLeft: 6,
741
- cheekSquintRight: 7,
742
- eyeBlinkLeft: 8,
743
- eyeBlinkRight: 9,
744
- eyeLookDownLeft: 10,
745
- eyeLookDownRight: 11,
746
- eyeLookInLeft: 12,
747
- eyeLookInRight: 13,
748
- eyeLookOutLeft: 14,
749
- eyeLookOutRight: 15,
750
- eyeLookUpLeft: 16,
751
- eyeLookUpRight: 17,
752
- eyeSquintLeft: 18,
753
- eyeSquintRight: 19,
754
- eyeWideLeft: 20,
755
- eyeWideRight: 21,
756
- jawForward: 22,
757
- jawLeft: 23,
758
- jawOpen: 24, // 🔑 KEY for mouth opening
759
- jawRight: 25,
760
- mouthClose: 26, // 🔑 KEY for mouth closing
761
- mouthDimpleLeft: 27,
762
- mouthDimpleRight: 28,
763
- mouthFrownLeft: 29,
764
- mouthFrownRight: 30,
765
- mouthFunnel: 31, // 🔑 KEY for "O" and "oo" sounds
766
- mouthLeft: 32,
767
- mouthLowerDownLeft: 33,
768
- mouthLowerDownRight: 34,
769
- mouthPressLeft: 35,
770
- mouthPressRight: 36,
771
- mouthPucker: 37, // 🔑 KEY for "U" and "w" sounds
772
- mouthRight: 38,
773
- mouthRollLower: 39,
774
- mouthRollUpper: 40,
775
- mouthShrugLower: 41,
776
- mouthShrugUpper: 42,
777
- mouthSmileLeft: 43, // 🔑 KEY for "ee" sounds
778
- mouthSmileRight: 44, // 🔑 KEY for "ee" sounds
779
- mouthStretchLeft: 45, // 🔑 KEY for wide mouth shapes
780
- mouthStretchRight: 46, // 🔑 KEY for wide mouth shapes
781
- mouthUpperUpLeft: 47,
782
- mouthUpperUpRight: 48,
783
- noseSneerLeft: 49,
784
- noseSneerRight: 50,
785
- tongueOut: 51 // 🔑 KEY for "th" and "l" sounds
786
- };
787
- /**
788
- * Generate a complete 251-length blendshape array for a specific phoneme
789
- * @param phoneme - The phoneme key from ARKIT_PHONEME_REFERENCE_VALUES
790
- * @returns Array of 251 blendshape values
791
- */
792
- function generatePhonemeBlendshapes(phoneme) {
793
- const blendshapes = new Array(251).fill(0);
794
- const values = ARKIT_PHONEME_REFERENCE_VALUES[phoneme];
795
- // Map phoneme values to ARKit indices (0-51)
796
- for (const [key, value] of Object.entries(values)) {
797
- if (key !== 'description' && key in ARKIT_BLENDSHAPE_INDICES) {
798
- const index = ARKIT_BLENDSHAPE_INDICES[key];
799
- blendshapes[index] = value;
800
- }
801
- }
802
- return blendshapes;
803
- }
804
- /**
805
- * Pre-generated sample blendshapes for testing and visualization
806
- * Use these to test your visualization system
807
- */
808
- const SAMPLE_PHONEME_TESTS = {
809
- 'Silence': generatePhonemeBlendshapes('sil'),
810
- 'A (ah - father)': generatePhonemeBlendshapes('aa'),
811
- 'E (eh - bed)': generatePhonemeBlendshapes('E'),
812
- 'I (ee - see)': generatePhonemeBlendshapes('I'),
813
- 'O (oh - go)': generatePhonemeBlendshapes('O'),
814
- 'U (oo - food)': generatePhonemeBlendshapes('U'),
815
- 'P/B/M': generatePhonemeBlendshapes('PP'),
816
- 'F/V': generatePhonemeBlendshapes('FF'),
817
- 'TH (think)': generatePhonemeBlendshapes('TH'),
818
- 'S/Z': generatePhonemeBlendshapes('SS'),
819
- 'SH (shoe)': generatePhonemeBlendshapes('sh'),
820
- 'CH (church)': generatePhonemeBlendshapes('CH'),
821
- 'W (wet)': generatePhonemeBlendshapes('w'),
822
- 'R (red)': generatePhonemeBlendshapes('RR'),
823
- 'L (lap)': generatePhonemeBlendshapes('l'),
824
- 'K/G': generatePhonemeBlendshapes('kk'),
825
- };
826
- /**
827
- * Get a formatted string showing active blendshapes for a phoneme
828
- * Useful for debugging and documentation
829
- */
830
- function getPhonemeDescription(phoneme) {
831
- const data = ARKIT_PHONEME_REFERENCE_VALUES[phoneme];
832
- const lines = [data.description, ''];
833
- for (const [key, value] of Object.entries(data)) {
834
- if (key !== 'description') {
835
- lines.push(` ${key}: ${value}`);
836
- }
837
- }
838
- return lines.join('\n');
839
- }
840
- /**
841
- * Quick reference guide for key blendshapes
842
- */
843
- const BLENDSHAPE_QUICK_GUIDE = {
844
- 'Open mouth': ['jawOpen'],
845
- 'Close mouth': ['mouthClose'],
846
- 'Round lips (O, oo)': ['mouthFunnel', 'mouthPucker'],
847
- 'Smile (ee)': ['mouthSmileLeft', 'mouthSmileRight', 'mouthStretchLeft', 'mouthStretchRight'],
848
- 'Lips pressed (p, b, m)': ['mouthPressLeft', 'mouthPressRight', 'mouthClose'],
849
- 'Teeth on lip (f, v)': ['mouthRollLower', 'mouthUpperUpLeft', 'mouthUpperUpRight'],
850
- 'Tongue visible (th, l)': ['tongueOut'],
851
- };
852
-
853
- /**
854
- * Neurosync Meta Human Blendshape Mapper
855
- * Maps 251 blendshape indices to their corresponding names
856
- * Based on Neurosync Meta Human specification
857
- */
858
- const NEUROSYNC_BLENDSHAPE_NAMES = [
859
- // 0-51: ARKit Standard Blendshapes
860
- "browDownLeft", // 0
861
- "browDownRight", // 1
862
- "browInnerUp", // 2
863
- "browOuterUpLeft", // 3
864
- "browOuterUpRight", // 4
865
- "cheekPuff", // 5
866
- "cheekSquintLeft", // 6
867
- "cheekSquintRight", // 7
868
- "eyeBlinkLeft", // 8
869
- "eyeBlinkRight", // 9
870
- "eyeLookDownLeft", // 10
871
- "eyeLookDownRight", // 11
872
- "eyeLookInLeft", // 12
873
- "eyeLookInRight", // 13
874
- "eyeLookOutLeft", // 14
875
- "eyeLookOutRight", // 15
876
- "eyeLookUpLeft", // 16
877
- "eyeLookUpRight", // 17
878
- "eyeSquintLeft", // 18
879
- "eyeSquintRight", // 19
880
- "eyeWideLeft", // 20
881
- "eyeWideRight", // 21
882
- "jawForward", // 22
883
- "jawLeft", // 23
884
- "jawOpen", // 24
885
- "jawRight", // 25
886
- "mouthClose", // 26
887
- "mouthDimpleLeft", // 27
888
- "mouthDimpleRight", // 28
889
- "mouthFrownLeft", // 29
890
- "mouthFrownRight", // 30
891
- "mouthFunnel", // 31
892
- "mouthLeft", // 32
893
- "mouthLowerDownLeft", // 33
894
- "mouthLowerDownRight", // 34
895
- "mouthPressLeft", // 35
896
- "mouthPressRight", // 36
897
- "mouthPucker", // 37
898
- "mouthRight", // 38
899
- "mouthRollLower", // 39
900
- "mouthRollUpper", // 40
901
- "mouthShrugLower", // 41
902
- "mouthShrugUpper", // 42
903
- "mouthSmileLeft", // 43
904
- "mouthSmileRight", // 44
905
- "mouthStretchLeft", // 45
906
- "mouthStretchRight", // 46
907
- "mouthUpperUpLeft", // 47
908
- "mouthUpperUpRight", // 48
909
- "noseSneerLeft", // 49
910
- "noseSneerRight", // 50
911
- "tongueOut", // 51
912
- // 52-250: Meta Human Extended Blendshapes
913
- "head_forward", // 52
914
- "head_upDown", // 53
915
- "head_tilt", // 54
916
- "head_turn", // 55
917
- "neck_stretch", // 56
918
- "neck_tilt", // 57
919
- "eyebrow_raiseInner_L", // 58
920
- "eyebrow_raiseInner_R", // 59
921
- "eyebrow_raiseOuter_L", // 60
922
- "eyebrow_raiseOuter_R", // 61
923
- "eyebrow_down_L", // 62
924
- "eyebrow_down_R", // 63
925
- "eye_blink_L", // 64
926
- "eye_blink_R", // 65
927
- "eye_wide_L", // 66
928
- "eye_wide_R", // 67
929
- "eye_squint_L", // 68
930
- "eye_squint_R", // 69
931
- "eye_look_up_L", // 70
932
- "eye_look_up_R", // 71
933
- "eye_look_down_L", // 72
934
- "eye_look_down_R", // 73
935
- "eye_look_in_L", // 74
936
- "eye_look_in_R", // 75
937
- "eye_look_out_L", // 76
938
- "eye_look_out_R", // 77
939
- "pupil_dilate_L", // 78
940
- "pupil_dilate_R", // 79
941
- "cheek_puff_L", // 80
942
- "cheek_puff_R", // 81
943
- "cheek_squint_L", // 82
944
- "cheek_squint_R", // 83
945
- "cheek_raiser_L", // 84
946
- "cheek_raiser_R", // 85
947
- "nose_wrinkle_L", // 86
948
- "nose_wrinkle_R", // 87
949
- "nose_nostril_raise_L", // 88
950
- "nose_nostril_raise_R", // 89
951
- "nose_nostril_compress_L", // 90
952
- "nose_nostril_compress_R", // 91
953
- "nose_nostril_dilate_L", // 92
954
- "nose_nostril_dilate_R", // 93
955
- "lip_upper_raise_L", // 94
956
- "lip_upper_raise_R", // 95
957
- "lip_upper_raiseInner_L", // 96
958
- "lip_upper_raiseInner_R", // 97
959
- "lip_upper_close", // 98
960
- "lip_lower_close", // 99
961
- "lip_lower_down_L", // 100
962
- "lip_lower_down_R", // 101
963
- "lip_lower_downInner_L", // 102
964
- "lip_lower_downInner_R", // 103
965
- "lip_corner_puller_L", // 104
966
- "lip_corner_puller_R", // 105
967
- "lip_corner_depressor_L", // 106
968
- "lip_corner_depressor_R", // 107
969
- "lip_stretcher_L", // 108
970
- "lip_stretcher_R", // 109
971
- "lip_funneler_LT", // 110
972
- "lip_funneler_RT", // 111
973
- "lip_funneler_LB", // 112
974
- "lip_funneler_RB", // 113
975
- "lip_pucker_L", // 114
976
- "lip_pucker_R", // 115
977
- "lip_press_L", // 116
978
- "lip_press_R", // 117
979
- "lip_tightener_L", // 118
980
- "lip_tightener_R", // 119
981
- "lip_suck_LT", // 120
982
- "lip_suck_RT", // 121
983
- "lip_suck_LB", // 122
984
- "lip_suck_RB", // 123
985
- "mouth_left", // 124
986
- "mouth_right", // 125
987
- "mouth_smile_L", // 126
988
- "mouth_smile_R", // 127
989
- "mouth_frown_L", // 128
990
- "mouth_frown_R", // 129
991
- "mouth_dimpler_L", // 130
992
- "mouth_dimpler_R", // 131
993
- "mouth_stretch_L", // 132
994
- "mouth_stretch_R", // 133
995
- "mouth_shrug_upper", // 134
996
- "mouth_shrug_lower", // 135
997
- "mouth_roll_upper_L", // 136
998
- "mouth_roll_upper_R", // 137
999
- "mouth_roll_lower_L", // 138
1000
- "mouth_roll_lower_R", // 139
1001
- "jaw_open", // 140
1002
- "jaw_forward", // 141
1003
- "jaw_left", // 142
1004
- "jaw_right", // 143
1005
- "jaw_clench", // 144
1006
- "jaw_thrust", // 145
1007
- "tongue_out", // 146
1008
- "tongue_up", // 147
1009
- "tongue_down", // 148
1010
- "tongue_wide", // 149
1011
- "tongue_narrow", // 150
1012
- "tongue_curl_up", // 151
1013
- "tongue_roll", // 152
1014
- "tongue_bulge_L", // 153
1015
- "tongue_bulge_R", // 154
1016
- "tongue_press", // 155
1017
- "midLipLowerDown_L", // 156
1018
- "midLipLowerDown_R", // 157
1019
- "midLipUpperUp_L", // 158
1020
- "midLipUpperUp_R", // 159
1021
- "midNostrilSharpener_L", // 160
1022
- "midNostrilSharpener_R", // 161
1023
- "viseme_sil", // 162
1024
- "viseme_PP", // 163
1025
- "viseme_FF", // 164
1026
- "viseme_TH", // 165
1027
- "viseme_DD", // 166
1028
- "viseme_kk", // 167
1029
- "viseme_CH", // 168
1030
- "viseme_SS", // 169
1031
- "viseme_nn", // 170
1032
- "viseme_RR", // 171
1033
- "viseme_aa", // 172
1034
- "viseme_E", // 173
1035
- "viseme_I", // 174
1036
- "viseme_O", // 175
1037
- "viseme_U", // 176
1038
- "lid_tightener_L", // 177
1039
- "lid_tightener_R", // 178
1040
- "eyelid_upper_raise_L", // 179
1041
- "eyelid_upper_raise_R", // 180
1042
- "eyelid_lower_down_L", // 181
1043
- "eyelid_lower_down_R", // 182
1044
- "eye_parallel_look_down", // 183
1045
- "eye_parallel_look_up", // 184
1046
- "innerBrow_raiser_L", // 185
1047
- "innerBrow_raiser_R", // 186
1048
- "outerBrow_raiser_L", // 187
1049
- "outerBrow_raiser_R", // 188
1050
- "lipCorner_down_L", // 189
1051
- "lipCorner_down_R", // 190
1052
- "lipCorner_wide_L", // 191
1053
- "lipCorner_wide_R", // 192
1054
- "lipLower_down_L", // 193
1055
- "lipLower_down_R", // 194
1056
- "lipUpper_up_L", // 195
1057
- "lipUpper_up_R", // 196
1058
- "wrinkler", // 197
1059
- "chin_raiser", // 198
1060
- "chin_raiser_T", // 199
1061
- "chin_raiser_B", // 200
1062
- "lips_toward", // 201
1063
- "lips_blow", // 202
1064
- "puff_suck_lower", // 203
1065
- "puff_suck_upper", // 204
1066
- "puff_suck_upper_L", // 205
1067
- "puff_suck_upper_R", // 206
1068
- "puff_suck_lower_L", // 207
1069
- "puff_suck_lower_R", // 208
1070
- "phone_aa", // 209
1071
- "phone_ow", // 210
1072
- "phone_ee", // 211
1073
- "phone_er", // 212
1074
- "phone_uw", // 213
1075
- "phone_l", // 214
1076
- "phone_w", // 215
1077
- "phone_m", // 216
1078
- "phone_f", // 217
1079
- "phone_th", // 218
1080
- "phone_s", // 219
1081
- "phone_sh", // 220
1082
- "phone_t", // 221
1083
- "phone_k", // 222
1084
- "phone_b", // 223
1085
- "phone_d", // 224
1086
- "phone_g", // 225
1087
- "phone_n", // 226
1088
- "phone_r", // 227
1089
- "phone_y", // 228
1090
- "expression_neutral", // 229
1091
- "expression_happy", // 230
1092
- "expression_sad", // 231
1093
- "expression_angry", // 232
1094
- "expression_surprised", // 233
1095
- "expression_disgusted", // 234
1096
- "expression_afraid", // 235
1097
- "expression_contempt", // 236
1098
- "occlusion_upper", // 237
1099
- "occlusion_lower", // 238
1100
- "teeth_upper", // 239
1101
- "teeth_lower", // 240
1102
- "teeth_fwdBack_upper", // 241
1103
- "teeth_fwdBack_lower", // 242
1104
- "teeth_width_upper", // 243
1105
- "teeth_width_lower", // 244
1106
- "gums_upper", // 245
1107
- "gums_lower", // 246
1108
- "tongue_tip", // 247
1109
- "tongue_thickness", // 248
1110
- "soft_palate", // 249
1111
- "emotion_intensity", // 250
1112
- ];
1113
- /**
1114
- * Maps blendshape values to their names
1115
- */
1116
- function mapBlendshapesToNames(values) {
1117
- const mapped = {};
1118
- values.forEach((value, index) => {
1119
- if (index < NEUROSYNC_BLENDSHAPE_NAMES.length) {
1120
- const name = NEUROSYNC_BLENDSHAPE_NAMES[index];
1121
- mapped[name] = value;
1122
- }
1123
- else {
1124
- // Handle any indices beyond our known names
1125
- mapped[`unknown_${index}`] = value;
1126
- }
1127
- });
1128
- return mapped;
1129
- }
1130
- /**
1131
- * Formats blendshapes for logging with filtering
1132
- * @param values - Array of 251 blendshape values
1133
- * @param threshold - Only log values above this threshold (default: 0.01)
1134
- * @param maxEntries - Maximum number of entries to log (default: 20)
1135
- */
1136
- function formatBlendshapesForLogging(values, threshold = 0.01, maxEntries = 20) {
1137
- const mapped = mapBlendshapesToNames(values);
1138
- // Filter values above threshold and sort by value (descending)
1139
- const activeBlendshapes = Object.entries(mapped)
1140
- .filter(([_, value]) => Math.abs(value) > threshold)
1141
- .sort(([, a], [, b]) => Math.abs(b) - Math.abs(a))
1142
- .slice(0, maxEntries)
1143
- .reduce((acc, [name, value]) => {
1144
- acc[name] = Number(value.toFixed(4));
1145
- return acc;
1146
- }, {});
1147
- return {
1148
- active: activeBlendshapes,
1149
- total: values.length,
1150
- activeCount: Object.keys(activeBlendshapes).length,
1151
- };
1152
- }
1153
- /**
1154
- * Gets the most significant blendshapes
1155
- */
1156
- function getTopBlendshapes(values, count = 10) {
1157
- return values
1158
- .map((value, index) => ({
1159
- name: NEUROSYNC_BLENDSHAPE_NAMES[index] || `unknown_${index}`,
1160
- value,
1161
- index,
1162
- }))
1163
- .filter((item) => Math.abs(item.value) > 0.001)
1164
- .sort((a, b) => Math.abs(b.value) - Math.abs(a.value))
1165
- .slice(0, count);
1166
- }
1167
-
1168
- exports.ARKIT_BLENDSHAPE_INDICES = ARKIT_BLENDSHAPE_INDICES;
1169
- exports.ARKIT_BLENDSHAPE_NAMES = ARKIT_BLENDSHAPE_NAMES;
1170
- exports.ARKIT_EXTENDED_NAMES = ARKIT_EXTENDED_NAMES;
1171
- exports.ARKIT_ORDER_61 = ARKIT_ORDER_61;
1172
- exports.ARKIT_PHONEME_REFERENCE_VALUES = ARKIT_PHONEME_REFERENCE_VALUES;
1173
- exports.BLENDSHAPE_QUICK_GUIDE = BLENDSHAPE_QUICK_GUIDE;
1174
- exports.KEY_LIPSYNC_BLENDSHAPES = KEY_LIPSYNC_BLENDSHAPES;
1175
- exports.NEUROSYNC_BLENDSHAPE_NAMES = NEUROSYNC_BLENDSHAPE_NAMES;
1176
- exports.SAMPLE_PHONEME_TESTS = SAMPLE_PHONEME_TESTS;
1177
- exports.STANDARD_ARKIT_ORDER = STANDARD_ARKIT_ORDER;
1178
- exports.convertOrder61ToStandard = convertOrder61ToStandard;
1179
- exports.formatARKitBlendshapesForLogging = formatARKitBlendshapesForLogging;
1180
- exports.formatBlendshapesForLogging = formatBlendshapesForLogging;
1181
- exports.formatOrder61ForLogging = formatOrder61ForLogging;
1182
- exports.generatePhonemeBlendshapes = generatePhonemeBlendshapes;
1183
- exports.getARKitBlendshapeValue = getARKitBlendshapeValue;
1184
- exports.getActiveARKitBlendshapes = getActiveARKitBlendshapes;
1185
- exports.getActiveBlendshapesFromOrder61 = getActiveBlendshapesFromOrder61;
1186
- exports.getBlendshapeValueFromOrder61 = getBlendshapeValueFromOrder61;
1187
- exports.getEyeRotations = getEyeRotations;
1188
- exports.getHeadRotation = getHeadRotation;
1189
- exports.getLipSyncBlendshapes = getLipSyncBlendshapes;
1190
- exports.getLipSyncValuesFromOrder61 = getLipSyncValuesFromOrder61;
1191
- exports.getPhonemeDescription = getPhonemeDescription;
1192
- exports.getTopBlendshapes = getTopBlendshapes;
1193
- exports.mapARKitBlendshapesToNames = mapARKitBlendshapesToNames;
1194
- exports.mapBlendshapesToNames = mapBlendshapesToNames;
1195
- exports.mapOrder61ToNames = mapOrder61ToNames;