@convai/web-sdk 0.3.2-beta.0 → 1.0.0-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/dist/core/BlendshapeQueue.d.ts +75 -42
  2. package/dist/core/BlendshapeQueue.d.ts.map +1 -1
  3. package/dist/core/BlendshapeQueue.js +177 -103
  4. package/dist/core/BlendshapeQueue.js.map +1 -1
  5. package/dist/core/ConvaiClient.d.ts +0 -14
  6. package/dist/core/ConvaiClient.d.ts.map +1 -1
  7. package/dist/core/ConvaiClient.js +13 -24
  8. package/dist/core/ConvaiClient.js.map +1 -1
  9. package/dist/core/MessageHandler.d.ts.map +1 -1
  10. package/dist/core/MessageHandler.js +6 -3
  11. package/dist/core/MessageHandler.js.map +1 -1
  12. package/dist/core/index.d.ts +0 -1
  13. package/dist/core/index.d.ts.map +1 -1
  14. package/dist/core/index.js.map +1 -1
  15. package/dist/core/types.d.ts +29 -11
  16. package/dist/core/types.d.ts.map +1 -1
  17. package/dist/lipsync-helpers/createBlendshapeQueue.d.ts +86 -0
  18. package/dist/lipsync-helpers/createBlendshapeQueue.d.ts.map +1 -0
  19. package/dist/lipsync-helpers/createBlendshapeQueue.js +126 -0
  20. package/dist/lipsync-helpers/createBlendshapeQueue.js.map +1 -0
  21. package/dist/lipsync-helpers/declarativeMapping.d.ts +93 -0
  22. package/dist/lipsync-helpers/declarativeMapping.d.ts.map +1 -0
  23. package/dist/lipsync-helpers/declarativeMapping.js +228 -0
  24. package/dist/lipsync-helpers/declarativeMapping.js.map +1 -0
  25. package/dist/lipsync-helpers/index.d.ts +9 -4
  26. package/dist/lipsync-helpers/index.d.ts.map +1 -1
  27. package/dist/lipsync-helpers/index.js +24 -12
  28. package/dist/lipsync-helpers/index.js.map +1 -1
  29. package/dist/lipsync-helpers/mappingTypes.d.ts +64 -0
  30. package/dist/lipsync-helpers/mappingTypes.d.ts.map +1 -0
  31. package/dist/lipsync-helpers/mappingTypes.js +28 -0
  32. package/dist/lipsync-helpers/mappingTypes.js.map +1 -0
  33. package/dist/lipsync-helpers/presetMappers.d.ts +24 -0
  34. package/dist/lipsync-helpers/presetMappers.d.ts.map +1 -0
  35. package/dist/lipsync-helpers/presetMappers.js +38 -0
  36. package/dist/lipsync-helpers/presetMappers.js.map +1 -0
  37. package/dist/react/hooks/useConvaiClient.d.ts.map +1 -1
  38. package/dist/react/hooks/useConvaiClient.js +0 -4
  39. package/dist/react/hooks/useConvaiClient.js.map +1 -1
  40. package/dist/types/index.d.ts +13 -0
  41. package/dist/types/index.d.ts.map +1 -1
  42. package/dist/vanilla/ConvaiWidget.d.ts +13 -6
  43. package/dist/vanilla/ConvaiWidget.d.ts.map +1 -1
  44. package/dist/vanilla/ConvaiWidget.js +108 -70
  45. package/dist/vanilla/ConvaiWidget.js.map +1 -1
  46. package/dist/vanilla/types.d.ts +29 -2
  47. package/dist/vanilla/types.d.ts.map +1 -1
  48. package/package.json +1 -1
  49. package/dist/lipsync-helpers/arkitBlendshapeHelpers.d.ts +0 -80
  50. package/dist/lipsync-helpers/arkitBlendshapeHelpers.d.ts.map +0 -1
  51. package/dist/lipsync-helpers/arkitBlendshapeHelpers.js +0 -201
  52. package/dist/lipsync-helpers/arkitBlendshapeHelpers.js.map +0 -1
  53. package/dist/lipsync-helpers/arkitPhonemeReference.d.ts +0 -155
  54. package/dist/lipsync-helpers/arkitPhonemeReference.d.ts.map +0 -1
  55. package/dist/lipsync-helpers/arkitPhonemeReference.js +0 -362
  56. package/dist/lipsync-helpers/arkitPhonemeReference.js.map +0 -1
  57. package/dist/lipsync-helpers/neurosyncBlendshapeMapper.d.ts +0 -30
  58. package/dist/lipsync-helpers/neurosyncBlendshapeMapper.d.ts.map +0 -1
  59. package/dist/lipsync-helpers/neurosyncBlendshapeMapper.js +0 -315
  60. package/dist/lipsync-helpers/neurosyncBlendshapeMapper.js.map +0 -1
@@ -1,155 +0,0 @@
1
- /**
2
- * ARKit Blendshape Reference Values for Phonemes/Visemes
3
- *
4
- * This file provides reference values for ARKit's 52 standard facial blendshapes
5
- * when forming different phonemes (speech sounds). These values are based on
6
- * industry standards from systems like OVR LipSync, Oculus Audio, and ARKit face tracking.
7
- *
8
- * All blendshape values range from 0.0 (no activation) to 1.0 (full activation)
9
- *
10
- * ARKit Blendshapes: Indices 0-51
11
- * Meta Human Extended: Indices 52-250
12
- */
13
- export interface PhonemeBlendshapes {
14
- jawOpen?: number;
15
- jawForward?: number;
16
- jawLeft?: number;
17
- jawRight?: number;
18
- mouthClose?: number;
19
- mouthFunnel?: number;
20
- mouthPucker?: number;
21
- mouthLeft?: number;
22
- mouthRight?: number;
23
- mouthSmileLeft?: number;
24
- mouthSmileRight?: number;
25
- mouthStretchLeft?: number;
26
- mouthStretchRight?: number;
27
- mouthFrownLeft?: number;
28
- mouthFrownRight?: number;
29
- mouthDimpleLeft?: number;
30
- mouthDimpleRight?: number;
31
- mouthRollLower?: number;
32
- mouthRollUpper?: number;
33
- mouthShrugLower?: number;
34
- mouthShrugUpper?: number;
35
- mouthPressLeft?: number;
36
- mouthPressRight?: number;
37
- mouthLowerDownLeft?: number;
38
- mouthLowerDownRight?: number;
39
- mouthUpperUpLeft?: number;
40
- mouthUpperUpRight?: number;
41
- tongueOut?: number;
42
- noseSneerLeft?: number;
43
- noseSneerRight?: number;
44
- cheekPuff?: number;
45
- description: string;
46
- }
47
- /**
48
- * Reference values for common phonemes
49
- * Based on International Phonetic Alphabet (IPA) and Arpabet standards
50
- */
51
- export declare const ARKIT_PHONEME_REFERENCE_VALUES: Record<string, PhonemeBlendshapes>;
52
- /**
53
- * ARKit Blendshape Indices (0-51)
54
- * These are Apple's standard 52 facial blendshapes
55
- */
56
- export declare const ARKIT_BLENDSHAPE_INDICES: {
57
- browDownLeft: number;
58
- browDownRight: number;
59
- browInnerUp: number;
60
- browOuterUpLeft: number;
61
- browOuterUpRight: number;
62
- cheekPuff: number;
63
- cheekSquintLeft: number;
64
- cheekSquintRight: number;
65
- eyeBlinkLeft: number;
66
- eyeBlinkRight: number;
67
- eyeLookDownLeft: number;
68
- eyeLookDownRight: number;
69
- eyeLookInLeft: number;
70
- eyeLookInRight: number;
71
- eyeLookOutLeft: number;
72
- eyeLookOutRight: number;
73
- eyeLookUpLeft: number;
74
- eyeLookUpRight: number;
75
- eyeSquintLeft: number;
76
- eyeSquintRight: number;
77
- eyeWideLeft: number;
78
- eyeWideRight: number;
79
- jawForward: number;
80
- jawLeft: number;
81
- jawOpen: number;
82
- jawRight: number;
83
- mouthClose: number;
84
- mouthDimpleLeft: number;
85
- mouthDimpleRight: number;
86
- mouthFrownLeft: number;
87
- mouthFrownRight: number;
88
- mouthFunnel: number;
89
- mouthLeft: number;
90
- mouthLowerDownLeft: number;
91
- mouthLowerDownRight: number;
92
- mouthPressLeft: number;
93
- mouthPressRight: number;
94
- mouthPucker: number;
95
- mouthRight: number;
96
- mouthRollLower: number;
97
- mouthRollUpper: number;
98
- mouthShrugLower: number;
99
- mouthShrugUpper: number;
100
- mouthSmileLeft: number;
101
- mouthSmileRight: number;
102
- mouthStretchLeft: number;
103
- mouthStretchRight: number;
104
- mouthUpperUpLeft: number;
105
- mouthUpperUpRight: number;
106
- noseSneerLeft: number;
107
- noseSneerRight: number;
108
- tongueOut: number;
109
- };
110
- /**
111
- * Generate a complete 251-length blendshape array for a specific phoneme
112
- * @param phoneme - The phoneme key from ARKIT_PHONEME_REFERENCE_VALUES
113
- * @returns Array of 251 blendshape values
114
- */
115
- export declare function generatePhonemeBlendshapes(phoneme: keyof typeof ARKIT_PHONEME_REFERENCE_VALUES): number[];
116
- /**
117
- * Pre-generated sample blendshapes for testing and visualization
118
- * Use these to test your visualization system
119
- */
120
- export declare const SAMPLE_PHONEME_TESTS: {
121
- Silence: number[];
122
- 'A (ah - father)': number[];
123
- 'E (eh - bed)': number[];
124
- 'I (ee - see)': number[];
125
- 'O (oh - go)': number[];
126
- 'U (oo - food)': number[];
127
- 'P/B/M': number[];
128
- 'F/V': number[];
129
- 'TH (think)': number[];
130
- 'S/Z': number[];
131
- 'SH (shoe)': number[];
132
- 'CH (church)': number[];
133
- 'W (wet)': number[];
134
- 'R (red)': number[];
135
- 'L (lap)': number[];
136
- 'K/G': number[];
137
- };
138
- /**
139
- * Get a formatted string showing active blendshapes for a phoneme
140
- * Useful for debugging and documentation
141
- */
142
- export declare function getPhonemeDescription(phoneme: keyof typeof ARKIT_PHONEME_REFERENCE_VALUES): string;
143
- /**
144
- * Quick reference guide for key blendshapes
145
- */
146
- export declare const BLENDSHAPE_QUICK_GUIDE: {
147
- 'Open mouth': string[];
148
- 'Close mouth': string[];
149
- 'Round lips (O, oo)': string[];
150
- 'Smile (ee)': string[];
151
- 'Lips pressed (p, b, m)': string[];
152
- 'Teeth on lip (f, v)': string[];
153
- 'Tongue visible (th, l)': string[];
154
- };
155
- //# sourceMappingURL=arkitPhonemeReference.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"arkitPhonemeReference.d.ts","sourceRoot":"","sources":["../../src/lipsync-helpers/arkitPhonemeReference.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,MAAM,WAAW,kBAAkB;IAEjC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAGlB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IAGpB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAG1B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAG3B,SAAS,CAAC,EAAE,MAAM,CAAC;IAGnB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAC;IAGxB,SAAS,CAAC,EAAE,MAAM,CAAC;IAGnB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;;GAGG;AACH,eAAO,MAAM,8BAA8B,EAAE,MAAM,CAAC,MAAM,EAAE,kBAAkB,CAqQ7E,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAqDpC,CAAC;AAEF;;;;GAIG;AACH,wBAAgB,0BAA0B,CAAC,OAAO,EAAE,MAAM,OAAO,8BAA8B,GAAG,MAAM,EAAE,CAazG;AAED;;;GAGG;AACH,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;CAiBhC,CAAC;AAEF;;;GAGG;AACH,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,MAAM,OAAO,8BAA8B,GAAG,MAAM,CAWlG;AAED;;GAEG;AACH,eAAO,MAAM,sBAAsB;;;;;;;;CAQlC,CAAC"}
@@ -1,362 +0,0 @@
1
- /**
2
- * ARKit Blendshape Reference Values for Phonemes/Visemes
3
- *
4
- * This file provides reference values for ARKit's 52 standard facial blendshapes
5
- * when forming different phonemes (speech sounds). These values are based on
6
- * industry standards from systems like OVR LipSync, Oculus Audio, and ARKit face tracking.
7
- *
8
- * All blendshape values range from 0.0 (no activation) to 1.0 (full activation)
9
- *
10
- * ARKit Blendshapes: Indices 0-51
11
- * Meta Human Extended: Indices 52-250
12
- */
13
- /**
14
- * Reference values for common phonemes
15
- * Based on International Phonetic Alphabet (IPA) and Arpabet standards
16
- */
17
- export const ARKIT_PHONEME_REFERENCE_VALUES = {
18
- // ============================================
19
- // SILENCE / NEUTRAL
20
- // ============================================
21
- sil: {
22
- jawOpen: 0.0,
23
- mouthClose: 1.0,
24
- description: "Silence - mouth closed, neutral expression"
25
- },
26
- // ============================================
27
- // VOWELS
28
- // ============================================
29
- aa: {
30
- jawOpen: 0.7,
31
- mouthFunnel: 0.0,
32
- mouthPucker: 0.0,
33
- tongueOut: 0.1,
34
- description: "Open back vowel - 'ah' (father, hot)"
35
- },
36
- ae: {
37
- jawOpen: 0.5,
38
- mouthStretchLeft: 0.4,
39
- mouthStretchRight: 0.4,
40
- description: "Near-open front vowel - 'a' (cat, bat)"
41
- },
42
- E: {
43
- jawOpen: 0.4,
44
- mouthStretchLeft: 0.3,
45
- mouthStretchRight: 0.3,
46
- mouthSmileLeft: 0.2,
47
- mouthSmileRight: 0.2,
48
- description: "Open-mid front vowel - 'eh' (bed, get)"
49
- },
50
- I: {
51
- jawOpen: 0.1,
52
- mouthStretchLeft: 0.6,
53
- mouthStretchRight: 0.6,
54
- mouthSmileLeft: 0.5,
55
- mouthSmileRight: 0.5,
56
- mouthClose: 0.3,
57
- description: "Close front vowel - 'ee' (see, feet)"
58
- },
59
- ih: {
60
- jawOpen: 0.2,
61
- mouthStretchLeft: 0.4,
62
- mouthStretchRight: 0.4,
63
- mouthSmileLeft: 0.3,
64
- mouthSmileRight: 0.3,
65
- description: "Near-close front vowel - 'i' (sit, bit)"
66
- },
67
- O: {
68
- jawOpen: 0.5,
69
- mouthFunnel: 0.6,
70
- mouthPucker: 0.3,
71
- description: "Close-mid back rounded vowel - 'oh' (go, boat)"
72
- },
73
- aw: {
74
- jawOpen: 0.6,
75
- mouthFunnel: 0.4,
76
- mouthPucker: 0.2,
77
- description: "Open-mid back rounded vowel - 'aw' (thought, caught)"
78
- },
79
- U: {
80
- jawOpen: 0.2,
81
- mouthFunnel: 0.8,
82
- mouthPucker: 0.9,
83
- description: "Close back rounded vowel - 'oo' (food, boot)"
84
- },
85
- uh: {
86
- jawOpen: 0.25,
87
- mouthFunnel: 0.5,
88
- mouthPucker: 0.6,
89
- description: "Near-close back rounded vowel - 'u' (book, foot)"
90
- },
91
- er: {
92
- jawOpen: 0.35,
93
- mouthFunnel: 0.3,
94
- mouthPucker: 0.1,
95
- description: "R-colored mid-central vowel - 'er' (her, bird)"
96
- },
97
- ah: {
98
- jawOpen: 0.4,
99
- mouthFunnel: 0.1,
100
- description: "Open-mid back vowel - 'uh' (but, cup)"
101
- },
102
- // ============================================
103
- // DIPHTHONGS
104
- // ============================================
105
- ow: {
106
- jawOpen: 0.6,
107
- mouthFunnel: 0.3,
108
- mouthPucker: 0.2,
109
- description: "Diphthong - 'ow' (cow, house)"
110
- },
111
- ay: {
112
- jawOpen: 0.5,
113
- mouthStretchLeft: 0.3,
114
- mouthStretchRight: 0.3,
115
- description: "Diphthong - 'ai' (eye, my)"
116
- },
117
- oy: {
118
- jawOpen: 0.5,
119
- mouthFunnel: 0.4,
120
- mouthPucker: 0.3,
121
- description: "Diphthong - 'oy' (boy, coin)"
122
- },
123
- // ============================================
124
- // CONSONANTS - BILABIALS (both lips)
125
- // ============================================
126
- PP: {
127
- jawOpen: 0.0,
128
- mouthClose: 1.0,
129
- mouthPressLeft: 0.5,
130
- mouthPressRight: 0.5,
131
- description: "Bilabial stops/nasal - 'p', 'b', 'm' (pat, bat, mat)"
132
- },
133
- // ============================================
134
- // CONSONANTS - LABIODENTAL (teeth on lip)
135
- // ============================================
136
- FF: {
137
- jawOpen: 0.15,
138
- mouthRollLower: 0.6,
139
- mouthUpperUpLeft: 0.3,
140
- mouthUpperUpRight: 0.3,
141
- description: "Labiodental fricatives - 'f', 'v' (fat, vat)"
142
- },
143
- // ============================================
144
- // CONSONANTS - DENTAL/INTERDENTAL
145
- // ============================================
146
- TH: {
147
- jawOpen: 0.2,
148
- tongueOut: 0.4,
149
- mouthUpperUpLeft: 0.2,
150
- mouthUpperUpRight: 0.2,
151
- mouthRollLower: 0.2,
152
- description: "Dental fricatives - 'th' (think, this)"
153
- },
154
- // ============================================
155
- // CONSONANTS - ALVEOLAR (tongue to ridge)
156
- // ============================================
157
- DD: {
158
- jawOpen: 0.3,
159
- mouthClose: 0.2,
160
- description: "Alveolar stops - 't', 'd' (tap, dap)"
161
- },
162
- SS: {
163
- jawOpen: 0.15,
164
- mouthStretchLeft: 0.4,
165
- mouthStretchRight: 0.4,
166
- mouthClose: 0.4,
167
- description: "Alveolar sibilants - 's', 'z' (sap, zap)"
168
- },
169
- nn: {
170
- jawOpen: 0.2,
171
- mouthClose: 0.3,
172
- description: "Alveolar nasal - 'n' (nap)"
173
- },
174
- l: {
175
- jawOpen: 0.25,
176
- tongueOut: 0.2,
177
- description: "Alveolar lateral approximant - 'l' (lap)"
178
- },
179
- // ============================================
180
- // CONSONANTS - POSTALVEOLAR (behind ridge)
181
- // ============================================
182
- CH: {
183
- jawOpen: 0.2,
184
- mouthFunnel: 0.3,
185
- mouthPucker: 0.2,
186
- description: "Postalveolar affricates - 'ch', 'j' (church, judge)"
187
- },
188
- sh: {
189
- jawOpen: 0.2,
190
- mouthFunnel: 0.4,
191
- mouthPucker: 0.3,
192
- description: "Postalveolar fricatives - 'sh', 'zh' (shoe, measure)"
193
- },
194
- // ============================================
195
- // CONSONANTS - RETROFLEX
196
- // ============================================
197
- RR: {
198
- jawOpen: 0.3,
199
- mouthFunnel: 0.2,
200
- mouthPucker: 0.15,
201
- tongueOut: 0.1,
202
- description: "Alveolar approximant - 'r' (red)"
203
- },
204
- // ============================================
205
- // CONSONANTS - PALATAL
206
- // ============================================
207
- y: {
208
- jawOpen: 0.2,
209
- mouthStretchLeft: 0.35,
210
- mouthStretchRight: 0.35,
211
- mouthSmileLeft: 0.2,
212
- mouthSmileRight: 0.2,
213
- description: "Palatal approximant - 'y' (yes)"
214
- },
215
- // ============================================
216
- // CONSONANTS - VELAR (back of tongue)
217
- // ============================================
218
- kk: {
219
- jawOpen: 0.35,
220
- description: "Velar stops/nasal - 'k', 'g', 'ng' (kite, go, sing)"
221
- },
222
- // ============================================
223
- // CONSONANTS - GLOTTAL
224
- // ============================================
225
- hh: {
226
- jawOpen: 0.3,
227
- description: "Glottal fricative - 'h' (hat)"
228
- },
229
- // ============================================
230
- // CONSONANTS - LABIO-VELAR
231
- // ============================================
232
- w: {
233
- jawOpen: 0.15,
234
- mouthPucker: 0.8,
235
- mouthFunnel: 0.6,
236
- description: "Labio-velar approximant - 'w' (wet)"
237
- },
238
- };
239
- /**
240
- * ARKit Blendshape Indices (0-51)
241
- * These are Apple's standard 52 facial blendshapes
242
- */
243
- export const ARKIT_BLENDSHAPE_INDICES = {
244
- browDownLeft: 0,
245
- browDownRight: 1,
246
- browInnerUp: 2,
247
- browOuterUpLeft: 3,
248
- browOuterUpRight: 4,
249
- cheekPuff: 5,
250
- cheekSquintLeft: 6,
251
- cheekSquintRight: 7,
252
- eyeBlinkLeft: 8,
253
- eyeBlinkRight: 9,
254
- eyeLookDownLeft: 10,
255
- eyeLookDownRight: 11,
256
- eyeLookInLeft: 12,
257
- eyeLookInRight: 13,
258
- eyeLookOutLeft: 14,
259
- eyeLookOutRight: 15,
260
- eyeLookUpLeft: 16,
261
- eyeLookUpRight: 17,
262
- eyeSquintLeft: 18,
263
- eyeSquintRight: 19,
264
- eyeWideLeft: 20,
265
- eyeWideRight: 21,
266
- jawForward: 22,
267
- jawLeft: 23,
268
- jawOpen: 24, // 🔑 KEY for mouth opening
269
- jawRight: 25,
270
- mouthClose: 26, // 🔑 KEY for mouth closing
271
- mouthDimpleLeft: 27,
272
- mouthDimpleRight: 28,
273
- mouthFrownLeft: 29,
274
- mouthFrownRight: 30,
275
- mouthFunnel: 31, // 🔑 KEY for "O" and "oo" sounds
276
- mouthLeft: 32,
277
- mouthLowerDownLeft: 33,
278
- mouthLowerDownRight: 34,
279
- mouthPressLeft: 35,
280
- mouthPressRight: 36,
281
- mouthPucker: 37, // 🔑 KEY for "U" and "w" sounds
282
- mouthRight: 38,
283
- mouthRollLower: 39,
284
- mouthRollUpper: 40,
285
- mouthShrugLower: 41,
286
- mouthShrugUpper: 42,
287
- mouthSmileLeft: 43, // 🔑 KEY for "ee" sounds
288
- mouthSmileRight: 44, // 🔑 KEY for "ee" sounds
289
- mouthStretchLeft: 45, // 🔑 KEY for wide mouth shapes
290
- mouthStretchRight: 46, // 🔑 KEY for wide mouth shapes
291
- mouthUpperUpLeft: 47,
292
- mouthUpperUpRight: 48,
293
- noseSneerLeft: 49,
294
- noseSneerRight: 50,
295
- tongueOut: 51 // 🔑 KEY for "th" and "l" sounds
296
- };
297
- /**
298
- * Generate a complete 251-length blendshape array for a specific phoneme
299
- * @param phoneme - The phoneme key from ARKIT_PHONEME_REFERENCE_VALUES
300
- * @returns Array of 251 blendshape values
301
- */
302
- export function generatePhonemeBlendshapes(phoneme) {
303
- const blendshapes = new Array(251).fill(0);
304
- const values = ARKIT_PHONEME_REFERENCE_VALUES[phoneme];
305
- // Map phoneme values to ARKit indices (0-51)
306
- for (const [key, value] of Object.entries(values)) {
307
- if (key !== 'description' && key in ARKIT_BLENDSHAPE_INDICES) {
308
- const index = ARKIT_BLENDSHAPE_INDICES[key];
309
- blendshapes[index] = value;
310
- }
311
- }
312
- return blendshapes;
313
- }
314
- /**
315
- * Pre-generated sample blendshapes for testing and visualization
316
- * Use these to test your visualization system
317
- */
318
- export const SAMPLE_PHONEME_TESTS = {
319
- 'Silence': generatePhonemeBlendshapes('sil'),
320
- 'A (ah - father)': generatePhonemeBlendshapes('aa'),
321
- 'E (eh - bed)': generatePhonemeBlendshapes('E'),
322
- 'I (ee - see)': generatePhonemeBlendshapes('I'),
323
- 'O (oh - go)': generatePhonemeBlendshapes('O'),
324
- 'U (oo - food)': generatePhonemeBlendshapes('U'),
325
- 'P/B/M': generatePhonemeBlendshapes('PP'),
326
- 'F/V': generatePhonemeBlendshapes('FF'),
327
- 'TH (think)': generatePhonemeBlendshapes('TH'),
328
- 'S/Z': generatePhonemeBlendshapes('SS'),
329
- 'SH (shoe)': generatePhonemeBlendshapes('sh'),
330
- 'CH (church)': generatePhonemeBlendshapes('CH'),
331
- 'W (wet)': generatePhonemeBlendshapes('w'),
332
- 'R (red)': generatePhonemeBlendshapes('RR'),
333
- 'L (lap)': generatePhonemeBlendshapes('l'),
334
- 'K/G': generatePhonemeBlendshapes('kk'),
335
- };
336
- /**
337
- * Get a formatted string showing active blendshapes for a phoneme
338
- * Useful for debugging and documentation
339
- */
340
- export function getPhonemeDescription(phoneme) {
341
- const data = ARKIT_PHONEME_REFERENCE_VALUES[phoneme];
342
- const lines = [data.description, ''];
343
- for (const [key, value] of Object.entries(data)) {
344
- if (key !== 'description') {
345
- lines.push(` ${key}: ${value}`);
346
- }
347
- }
348
- return lines.join('\n');
349
- }
350
- /**
351
- * Quick reference guide for key blendshapes
352
- */
353
- export const BLENDSHAPE_QUICK_GUIDE = {
354
- 'Open mouth': ['jawOpen'],
355
- 'Close mouth': ['mouthClose'],
356
- 'Round lips (O, oo)': ['mouthFunnel', 'mouthPucker'],
357
- 'Smile (ee)': ['mouthSmileLeft', 'mouthSmileRight', 'mouthStretchLeft', 'mouthStretchRight'],
358
- 'Lips pressed (p, b, m)': ['mouthPressLeft', 'mouthPressRight', 'mouthClose'],
359
- 'Teeth on lip (f, v)': ['mouthRollLower', 'mouthUpperUpLeft', 'mouthUpperUpRight'],
360
- 'Tongue visible (th, l)': ['tongueOut'],
361
- };
362
- //# sourceMappingURL=arkitPhonemeReference.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"arkitPhonemeReference.js","sourceRoot":"","sources":["../../src/lipsync-helpers/arkitPhonemeReference.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAoDH;;;GAGG;AACH,MAAM,CAAC,MAAM,8BAA8B,GAAuC;IAChF,+CAA+C;IAC/C,oBAAoB;IACpB,+CAA+C;IAC/C,GAAG,EAAE;QACH,OAAO,EAAE,GAAG;QACZ,UAAU,EAAE,GAAG;QACf,WAAW,EAAE,4CAA4C;KAC1D;IAED,+CAA+C;IAC/C,SAAS;IACT,+CAA+C;IAE/C,EAAE,EAAE;QACF,OAAO,EAAE,GAAG;QACZ,WAAW,EAAE,GAAG;QAChB,WAAW,EAAE,GAAG;QAChB,SAAS,EAAE,GAAG;QACd,WAAW,EAAE,sCAAsC;KACpD;IAED,EAAE,EAAE;QACF,OAAO,EAAE,GAAG;QACZ,gBAAgB,EAAE,GAAG;QACrB,iBAAiB,EAAE,GAAG;QACtB,WAAW,EAAE,wCAAwC;KACtD;IAED,CAAC,EAAE;QACD,OAAO,EAAE,GAAG;QACZ,gBAAgB,EAAE,GAAG;QACrB,iBAAiB,EAAE,GAAG;QACtB,cAAc,EAAE,GAAG;QACnB,eAAe,EAAE,GAAG;QACpB,WAAW,EAAE,wCAAwC;KACtD;IAED,CAAC,EAAE;QACD,OAAO,EAAE,GAAG;QACZ,gBAAgB,EAAE,GAAG;QACrB,iBAAiB,EAAE,GAAG;QACtB,cAAc,EAAE,GAAG;QACnB,eAAe,EAAE,GAAG;QACpB,UAAU,EAAE,GAAG;QACf,WAAW,EAAE,sCAAsC;KACpD;IAED,EAAE,EAAE;QACF,OAAO,EAAE,GAAG;QACZ,gBAAgB,EAAE,GAAG;QACrB,iBAAiB,EAAE,GAAG;QACtB,cAAc,EAAE,GAAG;QACnB,eAAe,EAAE,GAAG;QACpB,WAAW,EAAE,yCAAyC;KACvD;IAED,CAAC,EAAE;QACD,OAAO,EAAE,GAAG;QACZ,WAAW,EAAE,GAAG;QAChB,WAAW,EAAE,GAAG;QAChB,WAAW,EAAE,gDAAgD;KAC9D;IAED,EAAE,EAAE;QACF,OAAO,EAAE,GAAG;QACZ,WAAW,EAAE,GAAG;QAChB,WAAW,EAAE,GAAG;QAChB,WAAW,EAAE,sDAAsD;KACpE;IAED,CAAC,EAAE;QACD,OAAO,EAAE,GAAG;QACZ,WAAW,EAAE,GAAG;QAChB,WAAW,EAAE,GAAG;QAChB,WAAW,EAAE,8CAA8C;KAC5D;IAED,EAAE,EAAE;QACF,OAAO,EAAE,IAAI;QACb,WAAW,EAAE,GAAG;QAChB,WAAW,EAAE,GAAG;QAChB,WAAW,EAAE,kDAAkD;KAChE;IAED,EAAE,EAAE;QACF,OAAO,EAAE,IAAI;QACb,WAAW,EAAE,GAAG;QAChB,WAAW,EAAE,GAAG;QAChB,WAAW,EAAE,gDAAgD;KAC9D;IAED,EAAE,EAAE;QACF,OAAO,EAAE,GAAG;QACZ,WAAW,EAAE,GAAG;QAChB,WAAW,EAAE,uCAAuC;KACrD;IAED,+CAA+C;IAC/C,aAAa;IACb,+CAA+C;IAE/C,EAAE,EAAE;QACF,OAAO,EAAE,GAAG;QACZ,WAAW,EAAE,GAAG;QAChB,WAAW,EAAE,GAAG;QAChB,WAAW,EAAE,+BAA+B;KAC7C;IAED,EAAE,EAAE;QACF,OAAO,EAAE,GAAG;QACZ,gBAAgB,EAAE,GAAG;QACrB,iBAAiB,EAAE,GAAG;QACtB,WAAW,EAAE,4BAA4B;KAC1C;IAED,EAAE,EAAE;QACF,OAAO,EAAE,GAAG;QACZ,WAAW,EAAE,GAAG;QAChB,WAAW,EAAE,GAAG;QAChB,WAAW,EAAE,8BAA8B;KAC5C;IAED,+CAA+C;IAC/C,qCAAqC;IACrC,+CAA+C;IAE/C,EAAE,EAAE;QACF,OAAO,EAAE,GAAG;QACZ,UAAU,EAAE,GAAG;QACf,cAAc,EAAE,GAAG;QACnB,eAAe,EAAE,GAAG;QACpB,WAAW,EAAE,sDAAsD;KACpE;IAED,+CAA+C;IAC/C,0CAA0C;IAC1C,+CAA+C;IAE/C,EAAE,EAAE;QACF,OAAO,EAAE,IAAI;QACb,cAAc,EAAE,GAAG;QACnB,gBAAgB,EAAE,GAAG;QACrB,iBAAiB,EAAE,GAAG;QACtB,WAAW,EAAE,8CAA8C;KAC5D;IAED,+CAA+C;IAC/C,kCAAkC;IAClC,+CAA+C;IAE/C,EAAE,EAAE;QACF,OAAO,EAAE,GAAG;QACZ,SAAS,EAAE,GAAG;QACd,gBAAgB,EAAE,GAAG;QACrB,iBAAiB,EAAE,GAAG;QACtB,cAAc,EAAE,GAAG;QACnB,WAAW,EAAE,wCAAwC;KACtD;IAED,+CAA+C;IAC/C,0CAA0C;IAC1C,+CAA+C;IAE/C,EAAE,EAAE;QACF,OAAO,EAAE,GAAG;QACZ,UAAU,EAAE,GAAG;QACf,WAAW,EAAE,sCAAsC;KACpD;IAED,EAAE,EAAE;QACF,OAAO,EAAE,IAAI;QACb,gBAAgB,EAAE,GAAG;QACrB,iBAAiB,EAAE,GAAG;QACtB,UAAU,EAAE,GAAG;QACf,WAAW,EAAE,0CAA0C;KACxD;IAED,EAAE,EAAE;QACF,OAAO,EAAE,GAAG;QACZ,UAAU,EAAE,GAAG;QACf,WAAW,EAAE,4BAA4B;KAC1C;IAED,CAAC,EAAE;QACD,OAAO,EAAE,IAAI;QACb,SAAS,EAAE,GAAG;QACd,WAAW,EAAE,0CAA0C;KACxD;IAED,+CAA+C;IAC/C,2CAA2C;IAC3C,+CAA+C;IAE/C,EAAE,EAAE;QACF,OAAO,EAAE,GAAG;QACZ,WAAW,EAAE,GAAG;QAChB,WAAW,EAAE,GAAG;QAChB,WAAW,EAAE,qDAAqD;KACnE;IAED,EAAE,EAAE;QACF,OAAO,EAAE,GAAG;QACZ,WAAW,EAAE,GAAG;QAChB,WAAW,EAAE,GAAG;QAChB,WAAW,EAAE,sDAAsD;KACpE;IAED,+CAA+C;IAC/C,yBAAyB;IACzB,+CAA+C;IAE/C,EAAE,EAAE;QACF,OAAO,EAAE,GAAG;QACZ,WAAW,EAAE,GAAG;QAChB,WAAW,EAAE,IAAI;QACjB,SAAS,EAAE,GAAG;QACd,WAAW,EAAE,kCAAkC;KAChD;IAED,+CAA+C;IAC/C,uBAAuB;IACvB,+CAA+C;IAE/C,CAAC,EAAE;QACD,OAAO,EAAE,GAAG;QACZ,gBAAgB,EAAE,IAAI;QACtB,iBAAiB,EAAE,IAAI;QACvB,cAAc,EAAE,GAAG;QACnB,eAAe,EAAE,GAAG;QACpB,WAAW,EAAE,iCAAiC;KAC/C;IAED,+CAA+C;IAC/C,sCAAsC;IACtC,+CAA+C;IAE/C,EAAE,EAAE;QACF,OAAO,EAAE,IAAI;QACb,WAAW,EAAE,qDAAqD;KACnE;IAED,+CAA+C;IAC/C,uBAAuB;IACvB,+CAA+C;IAE/C,EAAE,EAAE;QACF,OAAO,EAAE,GAAG;QACZ,WAAW,EAAE,+BAA+B;KAC7C;IAED,+CAA+C;IAC/C,2BAA2B;IAC3B,+CAA+C;IAE/C,CAAC,EAAE;QACD,OAAO,EAAE,IAAI;QACb,WAAW,EAAE,GAAG;QAChB,WAAW,EAAE,GAAG;QAChB,WAAW,EAAE,qCAAqC;KACnD;CACF,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG;IACtC,YAAY,EAAE,CAAC;IACf,aAAa,EAAE,CAAC;IAChB,WAAW,EAAE,CAAC;IACd,eAAe,EAAE,CAAC;IAClB,gBAAgB,EAAE,CAAC;IACnB,SAAS,EAAE,CAAC;IACZ,eAAe,EAAE,CAAC;IAClB,gBAAgB,EAAE,CAAC;IACnB,YAAY,EAAE,CAAC;IACf,aAAa,EAAE,CAAC;IAChB,eAAe,EAAE,EAAE;IACnB,gBAAgB,EAAE,EAAE;IACpB,aAAa,EAAE,EAAE;IACjB,cAAc,EAAE,EAAE;IAClB,cAAc,EAAE,EAAE;IAClB,eAAe,EAAE,EAAE;IACnB,aAAa,EAAE,EAAE;IACjB,cAAc,EAAE,EAAE;IAClB,aAAa,EAAE,EAAE;IACjB,cAAc,EAAE,EAAE;IAClB,WAAW,EAAE,EAAE;IACf,YAAY,EAAE,EAAE;IAChB,UAAU,EAAE,EAAE;IACd,OAAO,EAAE,EAAE;IACX,OAAO,EAAE,EAAE,EAAW,2BAA2B;IACjD,QAAQ,EAAE,EAAE;IACZ,UAAU,EAAE,EAAE,EAAQ,2BAA2B;IACjD,eAAe,EAAE,EAAE;IACnB,gBAAgB,EAAE,EAAE;IACpB,cAAc,EAAE,EAAE;IAClB,eAAe,EAAE,EAAE;IACnB,WAAW,EAAE,EAAE,EAAO,iCAAiC;IACvD,SAAS,EAAE,EAAE;IACb,kBAAkB,EAAE,EAAE;IACtB,mBAAmB,EAAE,EAAE;IACvB,cAAc,EAAE,EAAE;IAClB,eAAe,EAAE,EAAE;IACnB,WAAW,EAAE,EAAE,EAAO,gCAAgC;IACtD,UAAU,EAAE,EAAE;IACd,cAAc,EAAE,EAAE;IAClB,cAAc,EAAE,EAAE;IAClB,eAAe,EAAE,EAAE;IACnB,eAAe,EAAE,EAAE;IACnB,cAAc,EAAE,EAAE,EAAI,yBAAyB;IAC/C,eAAe,EAAE,EAAE,EAAG,yBAAyB;IAC/C,gBAAgB,EAAE,EAAE,EAAE,+BAA+B;IACrD,iBAAiB,EAAE,EAAE,EAAC,+BAA+B;IACrD,gBAAgB,EAAE,EAAE;IACpB,iBAAiB,EAAE,EAAE;IACrB,aAAa,EAAE,EAAE;IACjB,cAAc,EAAE,EAAE;IAClB,SAAS,EAAE,EAAE,CAAS,iCAAiC;CACxD,CAAC;AAEF;;;;GAIG;AACH,MAAM,UAAU,0BAA0B,CAAC,OAAoD;IAC7F,MAAM,WAAW,GAAG,IAAI,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC3C,MAAM,MAAM,GAAG,8BAA8B,CAAC,OAAO,CAAC,CAAC;IAEvD,6CAA6C;IAC7C,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAClD,IAAI,GAAG,KAAK,aAAa,IAAI,GAAG,IAAI,wBAAwB,EAAE,CAAC;YAC7D,MAAM,KAAK,GAAG,wBAAwB,CAAC,GAA4C,CAAC,CAAC;YACrF,WAAW,CAAC,KAAK,CAAC,GAAG,KAAe,CAAC;QACvC,CAAC;IACH,CAAC;IAED,OAAO,WAAW,CAAC;AACrB,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG;IAClC,SAAS,EAAE,0BAA0B,CAAC,KAAK,CAAC;IAC5C,iBAAiB,EAAE,0BAA0B,CAAC,IAAI,CAAC;IACnD,cAAc,EAAE,0BAA0B,CAAC,GAAG,CAAC;IAC/C,cAAc,EAAE,0BAA0B,CAAC,GAAG,CAAC;IAC/C,aAAa,EAAE,0BAA0B,CAAC,GAAG,CAAC;IAC9C,eAAe,EAAE,0BAA0B,CAAC,GAAG,CAAC;IAChD,OAAO,EAAE,0BAA0B,CAAC,IAAI,CAAC;IACzC,KAAK,EAAE,0BAA0B,CAAC,IAAI,CAAC;IACvC,YAAY,EAAE,0BAA0B,CAAC,IAAI,CAAC;IAC9C,KAAK,EAAE,0BAA0B,CAAC,IAAI,CAAC;IACvC,WAAW,EAAE,0BAA0B,CAAC,IAAI,CAAC;IAC7C,aAAa,EAAE,0BAA0B,CAAC,IAAI,CAAC;IAC/C,SAAS,EAAE,0BAA0B,CAAC,GAAG,CAAC;IAC1C,SAAS,EAAE,0BAA0B,CAAC,IAAI,CAAC;IAC3C,SAAS,EAAE,0BAA0B,CAAC,GAAG,CAAC;IAC1C,KAAK,EAAE,0BAA0B,CAAC,IAAI,CAAC;CACxC,CAAC;AAEF;;;GAGG;AACH,MAAM,UAAU,qBAAqB,CAAC,OAAoD;IACxF,MAAM,IAAI,GAAG,8BAA8B,CAAC,OAAO,CAAC,CAAC;IACrD,MAAM,KAAK,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;IAErC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QAChD,IAAI,GAAG,KAAK,aAAa,EAAE,CAAC;YAC1B,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG,KAAK,KAAK,EAAE,CAAC,CAAC;QACnC,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG;IACpC,YAAY,EAAE,CAAC,SAAS,CAAC;IACzB,aAAa,EAAE,CAAC,YAAY,CAAC;IAC7B,oBAAoB,EAAE,CAAC,aAAa,EAAE,aAAa,CAAC;IACpD,YAAY,EAAE,CAAC,gBAAgB,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,mBAAmB,CAAC;IAC5F,wBAAwB,EAAE,CAAC,gBAAgB,EAAE,iBAAiB,EAAE,YAAY,CAAC;IAC7E,qBAAqB,EAAE,CAAC,gBAAgB,EAAE,kBAAkB,EAAE,mBAAmB,CAAC;IAClF,wBAAwB,EAAE,CAAC,WAAW,CAAC;CACxC,CAAC"}
@@ -1,30 +0,0 @@
1
- /**
2
- * Neurosync Meta Human Blendshape Mapper
3
- * Maps 251 blendshape indices to their corresponding names
4
- * Based on Neurosync Meta Human specification
5
- */
6
- export declare const NEUROSYNC_BLENDSHAPE_NAMES: string[];
7
- /**
8
- * Maps blendshape values to their names
9
- */
10
- export declare function mapBlendshapesToNames(values: number[]): Record<string, number>;
11
- /**
12
- * Formats blendshapes for logging with filtering
13
- * @param values - Array of 251 blendshape values
14
- * @param threshold - Only log values above this threshold (default: 0.01)
15
- * @param maxEntries - Maximum number of entries to log (default: 20)
16
- */
17
- export declare function formatBlendshapesForLogging(values: number[], threshold?: number, maxEntries?: number): {
18
- active: Record<string, number>;
19
- total: number;
20
- activeCount: number;
21
- };
22
- /**
23
- * Gets the most significant blendshapes
24
- */
25
- export declare function getTopBlendshapes(values: number[], count?: number): Array<{
26
- name: string;
27
- value: number;
28
- index: number;
29
- }>;
30
- //# sourceMappingURL=neurosyncBlendshapeMapper.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"neurosyncBlendshapeMapper.d.ts","sourceRoot":"","sources":["../../src/lipsync-helpers/neurosyncBlendshapeMapper.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,eAAO,MAAM,0BAA0B,EAAE,MAAM,EA+P9C,CAAC;AAEF;;GAEG;AACH,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAc9E;AAED;;;;;GAKG;AACH,wBAAgB,2BAA2B,CACzC,MAAM,EAAE,MAAM,EAAE,EAChB,SAAS,GAAE,MAAa,EACxB,UAAU,GAAE,MAAW,GACtB;IAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,MAAM,CAAA;CAAE,CAkBxE;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAC/B,MAAM,EAAE,MAAM,EAAE,EAChB,KAAK,GAAE,MAAW,GACjB,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAUvD"}