@npcify/web 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +68 -4
- package/dist/index.d.ts +258 -8
- package/dist/index.js +2085 -824
- package/examples/index.html +1 -0
- package/examples/main.js +40 -12
- package/examples/three-game.js +13 -3
- package/package.json +3 -3
package/dist/index.js
CHANGED
|
@@ -5,693 +5,186 @@ var STAGING_PROTOCOL_VERSION = 1;
|
|
|
5
5
|
var SERVER_RENDER_PROTOCOL_VERSION = 3;
|
|
6
6
|
var MAXIMUM_PLAYBACK_DURATION_SECONDS = 10 * 60;
|
|
7
7
|
|
|
8
|
-
// ../ts-shared/src/tts
|
|
9
|
-
var
|
|
10
|
-
|
|
11
|
-
id: "qbfQuoRJv1T3ei3qV4bc",
|
|
12
|
-
name: "Kazuha",
|
|
13
|
-
gender: "female",
|
|
14
|
-
description: "Conversation: Warm & Friendly",
|
|
15
|
-
lang: ["Japanese"]
|
|
16
|
-
},
|
|
17
|
-
{
|
|
18
|
-
id: "NO5A3b3sSzDyJQF7MiNS",
|
|
19
|
-
name: "Shohei",
|
|
20
|
-
gender: "male",
|
|
21
|
-
description: "Conversation: Serious, Clear and Measured",
|
|
22
|
-
lang: ["Japanese"]
|
|
23
|
-
},
|
|
24
|
-
{
|
|
25
|
-
id: "3321Alera3fXjEWjjbAX",
|
|
26
|
-
name: "Akane",
|
|
27
|
-
gender: "female",
|
|
28
|
-
description: "Calm: Soft, Warm and Soothing",
|
|
29
|
-
lang: ["Japanese"]
|
|
30
|
-
},
|
|
31
|
-
{
|
|
32
|
-
id: "4sirbXwrtRlmPV80MJkQ",
|
|
33
|
-
name: "Sora",
|
|
34
|
-
gender: "male",
|
|
35
|
-
description: "Calm: Clear and Airy",
|
|
36
|
-
lang: ["Japanese"]
|
|
37
|
-
},
|
|
38
|
-
{
|
|
39
|
-
id: "HxuFAkkGVeQs1sDIMF5g",
|
|
40
|
-
name: "Claire",
|
|
41
|
-
gender: "female",
|
|
42
|
-
description: "Lively: Cute & Cheerful JP",
|
|
43
|
-
lang: ["Japanese"]
|
|
44
|
-
},
|
|
45
|
-
{
|
|
46
|
-
id: "tGhb4uYSV8sWBI31DYU8",
|
|
47
|
-
name: "Gary",
|
|
48
|
-
gender: "male",
|
|
49
|
-
description: "Lively: Joyful, Clear and Husky",
|
|
50
|
-
lang: ["Japanese"]
|
|
51
|
-
},
|
|
52
|
-
{
|
|
53
|
-
id: "8PfKHL4nZToWC3pbz9U9",
|
|
54
|
-
name: "Rose",
|
|
55
|
-
gender: "female",
|
|
56
|
-
description: "Mature narration: Inviting, Clear and Steady",
|
|
57
|
-
lang: ["Japanese"]
|
|
58
|
-
},
|
|
59
|
-
{
|
|
60
|
-
id: "Mv8AjrYZCBkdsmDHNwcB",
|
|
61
|
-
name: "Ishibashi",
|
|
62
|
-
gender: "male",
|
|
63
|
-
description: "Mature narration: Inviting, Natural and Smoky",
|
|
64
|
-
lang: ["Japanese"]
|
|
65
|
-
},
|
|
66
|
-
{
|
|
67
|
-
id: "6awt6FKyZGV0HyQEwisX",
|
|
68
|
-
name: "Suzu",
|
|
69
|
-
gender: "female",
|
|
70
|
-
description: "Character: Cute, Clear & Calm",
|
|
71
|
-
lang: ["Japanese"]
|
|
72
|
-
},
|
|
73
|
-
{
|
|
74
|
-
id: "ZcX76PwFSkrkyI78RBTK",
|
|
75
|
-
name: "Nayuta",
|
|
76
|
-
gender: "male",
|
|
77
|
-
description: "Character: Dark Anime Hero",
|
|
78
|
-
lang: ["Japanese"]
|
|
79
|
-
},
|
|
80
|
-
{
|
|
81
|
-
id: "9lHjugDhwqoxA5MhX0az",
|
|
82
|
-
name: "Anna Su",
|
|
83
|
-
gender: "female",
|
|
84
|
-
description: "Conversation: Casual, Friendly and Bright",
|
|
85
|
-
lang: ["Chinese"]
|
|
86
|
-
},
|
|
87
|
-
{
|
|
88
|
-
id: "UwT0JPexcCbH107hq7i5",
|
|
89
|
-
name: "James",
|
|
90
|
-
gender: "male",
|
|
91
|
-
description: "Conversation: Young, Friendly and Positive",
|
|
92
|
-
lang: ["Chinese"]
|
|
93
|
-
},
|
|
94
|
-
{
|
|
95
|
-
id: "APSIkVZudNbPAwyPoeVO",
|
|
96
|
-
name: "Sage",
|
|
97
|
-
gender: "female",
|
|
98
|
-
description: "Calm: Soothing & Gentle",
|
|
99
|
-
lang: ["Chinese"]
|
|
100
|
-
},
|
|
101
|
-
{
|
|
102
|
-
id: "Z5oklYidFFuqcVQNGzYK",
|
|
103
|
-
name: "Aman",
|
|
104
|
-
gender: "male",
|
|
105
|
-
description: "Calm: ASMR, Soft and Whisper",
|
|
106
|
-
lang: ["Chinese"]
|
|
107
|
-
},
|
|
108
|
-
{
|
|
109
|
-
id: "ByhETIclHirOlWnWKhHc",
|
|
110
|
-
name: "Shan Shan",
|
|
111
|
-
gender: "female",
|
|
112
|
-
description: "Lively: Young Energetic Female",
|
|
113
|
-
lang: ["Chinese"]
|
|
114
|
-
},
|
|
115
|
-
{
|
|
116
|
-
id: "5mZxJZhSmJTjL7GoYfYI",
|
|
117
|
-
name: "Karo Yang",
|
|
118
|
-
gender: "male",
|
|
119
|
-
description: "Lively: Clear, Lively and Energetic",
|
|
120
|
-
lang: ["Chinese"]
|
|
121
|
-
},
|
|
122
|
-
{
|
|
123
|
-
id: "xDISamJf8LV5rG5A2te1",
|
|
124
|
-
name: "Aki | Deep & Suspenseful",
|
|
125
|
-
gender: "female",
|
|
126
|
-
description: "Mature narration: Deep & Suspenseful",
|
|
127
|
-
lang: ["Chinese"]
|
|
128
|
-
},
|
|
129
|
-
{
|
|
130
|
-
id: "WuLq5z7nEcrhppO0ZQJw",
|
|
131
|
-
name: "Martin Li",
|
|
132
|
-
gender: "male",
|
|
133
|
-
description: "Mature narration: Raspy, Serious and Deep",
|
|
134
|
-
lang: ["Chinese"]
|
|
135
|
-
},
|
|
136
|
-
{
|
|
137
|
-
id: "BT0SH7Hb8NRPpT5xbCgg",
|
|
138
|
-
name: "Lulu",
|
|
139
|
-
gender: "female",
|
|
140
|
-
description: "Character: Cute Little Witch",
|
|
141
|
-
lang: ["Chinese"]
|
|
142
|
-
},
|
|
143
|
-
{
|
|
144
|
-
id: "YaHyfWIwVxVEIhSTnKOi",
|
|
145
|
-
name: "Xiao Bai",
|
|
146
|
-
gender: "male",
|
|
147
|
-
description: "Character: Nasal, Slow and Annoying",
|
|
148
|
-
lang: ["Chinese"]
|
|
149
|
-
},
|
|
150
|
-
{
|
|
151
|
-
id: "8jHHF8rMqMlg8if2mOUe",
|
|
152
|
-
name: "Han",
|
|
153
|
-
gender: "female",
|
|
154
|
-
description: "Conversation: Natural, Casual and Spontaneous",
|
|
155
|
-
lang: ["Korean"]
|
|
156
|
-
},
|
|
157
|
-
{
|
|
158
|
-
id: "pb3lVZVjdFWbkhPKlelB",
|
|
159
|
-
name: "Harry Kim",
|
|
160
|
-
gender: "male",
|
|
161
|
-
description: "Conversation: Soft, Natural and Steady",
|
|
162
|
-
lang: ["Korean"]
|
|
163
|
-
},
|
|
164
|
-
{
|
|
165
|
-
id: "7oLyBHyhxAjrctX6ZQlw",
|
|
166
|
-
name: "Bomisori",
|
|
167
|
-
gender: "female",
|
|
168
|
-
description: "Calm: Warm and Clear",
|
|
169
|
-
lang: ["Korean"]
|
|
170
|
-
},
|
|
171
|
-
{
|
|
172
|
-
id: "BbsagRO6ohd8MKPS2Ob0",
|
|
173
|
-
name: "Jin Geon Song",
|
|
174
|
-
gender: "male",
|
|
175
|
-
description: "Calm: Neutral, Calm and Steady",
|
|
176
|
-
lang: ["Korean"]
|
|
177
|
-
},
|
|
178
|
-
{
|
|
179
|
-
id: "bQlkYuipD5BHEhntA5iz",
|
|
180
|
-
name: "JY",
|
|
181
|
-
gender: "female",
|
|
182
|
-
description: "Lively: Trendy K-Culture Vlog Girl",
|
|
183
|
-
lang: ["Korean"]
|
|
184
|
-
},
|
|
185
|
-
{
|
|
186
|
-
id: "1W00IGEmNmwmsDeYy7ag",
|
|
187
|
-
name: "Krys",
|
|
188
|
-
gender: "male",
|
|
189
|
-
description: "Lively: Cheerful, Clear and Measured",
|
|
190
|
-
lang: ["Korean"]
|
|
191
|
-
},
|
|
192
|
-
{
|
|
193
|
-
id: "ksaI0TCD9BstzEzlxj4q",
|
|
194
|
-
name: "Seulki",
|
|
195
|
-
gender: "female",
|
|
196
|
-
description: "Mature narration: Inviting, Calm and Measured",
|
|
197
|
-
lang: ["Korean"]
|
|
198
|
-
},
|
|
199
|
-
{
|
|
200
|
-
id: "FQ3MuLxZh0jHcZmA5vW1",
|
|
201
|
-
name: "Dohyeon",
|
|
202
|
-
gender: "male",
|
|
203
|
-
description: "Mature narration: Whisper, Measured and Neutral",
|
|
204
|
-
lang: ["Korean"]
|
|
205
|
-
},
|
|
206
|
-
{
|
|
207
|
-
id: "DGDjWy2CQ8Ishu0meQte",
|
|
208
|
-
name: "Chaemong",
|
|
209
|
-
gender: "female",
|
|
210
|
-
description: "Character: Young and Sassy",
|
|
211
|
-
lang: ["Korean"]
|
|
212
|
-
},
|
|
213
|
-
{
|
|
214
|
-
id: "ZZ4xhVcc83kZBfNIlIIz",
|
|
215
|
-
name: "Manbo",
|
|
216
|
-
gender: "male",
|
|
217
|
-
description: "Character: Encouraging, Clear and Measured",
|
|
218
|
-
lang: ["Korean"]
|
|
219
|
-
}
|
|
220
|
-
];
|
|
8
|
+
// ../ts-shared/src/tts.ts
|
|
9
|
+
var TTS_PROVIDERS = ["11labs", "gemini"];
|
|
10
|
+
var DEFAULT_TTS_PROVIDER = TTS_PROVIDERS[0];
|
|
221
11
|
|
|
222
|
-
// ../ts-shared/src/
|
|
223
|
-
var
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
name: "
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
},
|
|
238
|
-
{
|
|
239
|
-
id: "ftDdhfYtmfGP0tFlBYA1",
|
|
240
|
-
name: "Alisha",
|
|
241
|
-
gender: "female",
|
|
242
|
-
description: "Calm: Soft and Engaging",
|
|
243
|
-
lang: ["English"]
|
|
244
|
-
},
|
|
245
|
-
{
|
|
246
|
-
id: "GUDYcgRAONiI1nXDcNQQ",
|
|
247
|
-
name: "Milo",
|
|
248
|
-
gender: "male",
|
|
249
|
-
description: "Calm: Soothing and Meditative",
|
|
250
|
-
lang: ["English"]
|
|
251
|
-
},
|
|
252
|
-
{
|
|
253
|
-
id: "r1KmysJdVYZjJCm4mL3b",
|
|
254
|
-
name: "Jessica",
|
|
255
|
-
gender: "female",
|
|
256
|
-
description: "Lively: Playful, Bright, Warm",
|
|
257
|
-
lang: ["English"]
|
|
258
|
-
},
|
|
259
|
-
{
|
|
260
|
-
id: "o2zd9K5QOO7ppTb04Lx0",
|
|
261
|
-
name: "Gabriel Ripley",
|
|
262
|
-
gender: "male",
|
|
263
|
-
description: "Lively: Facetious, Quick-Spoken",
|
|
264
|
-
lang: ["English"]
|
|
265
|
-
},
|
|
266
|
-
{
|
|
267
|
-
id: "RILOU7YmBhvwJGDGjNmP",
|
|
268
|
-
name: "Jane",
|
|
269
|
-
gender: "female",
|
|
270
|
-
description: "Mature narration: Professional Audiobook Reader",
|
|
271
|
-
lang: ["English"]
|
|
272
|
-
},
|
|
273
|
-
{
|
|
274
|
-
id: "wAGzRVkxKEs8La0lmdrE",
|
|
275
|
-
name: "Sully",
|
|
276
|
-
gender: "male",
|
|
277
|
-
description: "Mature narration: Mature, Deep and Intriguing",
|
|
278
|
-
lang: ["English"]
|
|
279
|
-
},
|
|
280
|
-
{
|
|
281
|
-
id: "flHkNRp1BlvT73UL6gyz",
|
|
282
|
-
name: "Jessica Anne Bogart",
|
|
283
|
-
gender: "female",
|
|
284
|
-
description: "Character: Eloquent Villain",
|
|
285
|
-
lang: ["English"]
|
|
286
|
-
},
|
|
287
|
-
{
|
|
288
|
-
id: "IRHApOXLvnW57QJPQH2P",
|
|
289
|
-
name: "Adam - American, Dark and Tough",
|
|
290
|
-
gender: "male",
|
|
291
|
-
description: "Character: American, Dark and Tough",
|
|
292
|
-
lang: ["English"]
|
|
293
|
-
},
|
|
294
|
-
{
|
|
295
|
-
id: "lvQdCgwZfBuOzxyV5pxu",
|
|
296
|
-
name: "Audia",
|
|
297
|
-
gender: "female",
|
|
298
|
-
description: "Conversation: Warm, Clear and Welcoming",
|
|
299
|
-
lang: ["French"]
|
|
300
|
-
},
|
|
301
|
-
{
|
|
302
|
-
id: "kENkNtk0xyzG09WW40xE",
|
|
303
|
-
name: "Marcel - Warm and Conversational",
|
|
304
|
-
gender: "male",
|
|
305
|
-
description: "Conversation: Warm and Conversational",
|
|
306
|
-
lang: ["French"]
|
|
307
|
-
},
|
|
308
|
-
{
|
|
309
|
-
id: "kwhMCf63M8O3rCfnQ3oQ",
|
|
310
|
-
name: "Caroline",
|
|
311
|
-
gender: "female",
|
|
312
|
-
description: "Calm: Warm, Sweet and Soft",
|
|
313
|
-
lang: ["French"]
|
|
314
|
-
},
|
|
315
|
-
{
|
|
316
|
-
id: "YV28ox2c5Cuh5rim0LrW",
|
|
317
|
-
name: "Marcel - Warm, Soft and Intimate",
|
|
318
|
-
gender: "male",
|
|
319
|
-
description: "Calm: Warm, Soft and Intimate",
|
|
320
|
-
lang: ["French"]
|
|
321
|
-
},
|
|
322
|
-
{
|
|
323
|
-
id: "KSyQzmsYhFbuOhqj1Xxv",
|
|
324
|
-
name: "Lea",
|
|
325
|
-
gender: "female",
|
|
326
|
-
description: "Lively: Energetic and Modern",
|
|
327
|
-
lang: ["French"]
|
|
328
|
-
},
|
|
329
|
-
{
|
|
330
|
-
id: "4TfTGcPwoefWe878B0rm",
|
|
331
|
-
name: "Troy",
|
|
332
|
-
gender: "male",
|
|
333
|
-
description: "Lively: Cartoon Hero",
|
|
334
|
-
lang: ["French"]
|
|
335
|
-
},
|
|
336
|
-
{
|
|
337
|
-
id: "WeAAwKYcS06VmXw086yZ",
|
|
338
|
-
name: "Victoria",
|
|
339
|
-
gender: "female",
|
|
340
|
-
description: "Mature narration: Warm and calm",
|
|
341
|
-
lang: ["French"]
|
|
342
|
-
},
|
|
343
|
-
{
|
|
344
|
-
id: "TTtB1x9U8PF0Vgf20IAP",
|
|
345
|
-
name: "Adrien",
|
|
346
|
-
gender: "male",
|
|
347
|
-
description: "Mature narration: Deep, Comforting and Warm",
|
|
348
|
-
lang: ["French"]
|
|
349
|
-
},
|
|
350
|
-
{
|
|
351
|
-
id: "ZYOBieLaunTiQrTrvNQq",
|
|
352
|
-
name: "Morgane",
|
|
353
|
-
gender: "female",
|
|
354
|
-
description: "Character: Gentle and Expressive",
|
|
355
|
-
lang: ["French"]
|
|
356
|
-
},
|
|
357
|
-
{
|
|
358
|
-
id: "RBhYSNMNu6b2CGZ9Fn1M",
|
|
359
|
-
name: "Felix Tabarnak",
|
|
360
|
-
gender: "male",
|
|
361
|
-
description: "Character: Confident and Witty",
|
|
362
|
-
lang: ["French"]
|
|
363
|
-
},
|
|
364
|
-
{
|
|
365
|
-
id: "MuTiG4dbrEGYEy3XP4iP",
|
|
366
|
-
name: "Rossana",
|
|
367
|
-
gender: "female",
|
|
368
|
-
description: "Conversation: Warm Italian Conversational",
|
|
369
|
-
lang: ["Italian"]
|
|
370
|
-
},
|
|
371
|
-
{
|
|
372
|
-
id: "5wE9Co8DESyhUw0GgAHG",
|
|
373
|
-
name: "Alessandro Zanierato",
|
|
374
|
-
gender: "male",
|
|
375
|
-
description: "Conversation: Casual",
|
|
376
|
-
lang: ["Italian"]
|
|
377
|
-
},
|
|
378
|
-
{
|
|
379
|
-
id: "PS6wM7QnnPTUDE3t4bbl",
|
|
380
|
-
name: "Serena",
|
|
381
|
-
gender: "female",
|
|
382
|
-
description: "Calm: Soft, Soothing and Gentle",
|
|
383
|
-
lang: ["Italian"]
|
|
384
|
-
},
|
|
385
|
-
{
|
|
386
|
-
id: "TnICJ0opDnjladqWtQ6k",
|
|
387
|
-
name: "Livio",
|
|
388
|
-
gender: "male",
|
|
389
|
-
description: "Calm: Meditative and Balanced",
|
|
390
|
-
lang: ["Italian"]
|
|
391
|
-
},
|
|
392
|
-
{
|
|
393
|
-
id: "ZRKmc75tGxpIMNTEiwe0",
|
|
394
|
-
name: "Mimmi",
|
|
395
|
-
gender: "female",
|
|
396
|
-
description: "Lively: Playful and Cartoonish",
|
|
397
|
-
lang: ["Italian"]
|
|
398
|
-
},
|
|
399
|
-
{
|
|
400
|
-
id: "nH7uLS5UdEnvKEOAXtlQ",
|
|
401
|
-
name: "Vittorio",
|
|
402
|
-
gender: "male",
|
|
403
|
-
description: "Lively: Believable, Friendly and Rich",
|
|
404
|
-
lang: ["Italian"]
|
|
405
|
-
},
|
|
406
|
-
{
|
|
407
|
-
id: "fNmw8sukfGuvWVOp33Ge",
|
|
408
|
-
name: "Tosca",
|
|
409
|
-
gender: "female",
|
|
410
|
-
description: "Mature narration: Mature, Whispery and Seductive",
|
|
411
|
-
lang: ["Italian"]
|
|
412
|
-
},
|
|
413
|
-
{
|
|
414
|
-
id: "YGp1lBJLaHhfIFT0yeDE",
|
|
415
|
-
name: "RaffaeleR",
|
|
416
|
-
gender: "male",
|
|
417
|
-
description: "Mature narration: Deep, Rich and Steady",
|
|
418
|
-
lang: ["Italian"]
|
|
419
|
-
},
|
|
420
|
-
{
|
|
421
|
-
id: "G0q9AYE8QsarSbMtaIEu",
|
|
422
|
-
name: "Elettra",
|
|
423
|
-
gender: "female",
|
|
424
|
-
description: "Character: Mature actress",
|
|
425
|
-
lang: ["Italian"]
|
|
426
|
-
},
|
|
427
|
-
{
|
|
428
|
-
id: "TdZgLZ1h5vbFJrgh8QqK",
|
|
429
|
-
name: "Vito Pepe",
|
|
430
|
-
gender: "male",
|
|
431
|
-
description: "Character: Profound and Confident",
|
|
432
|
-
lang: ["Italian"]
|
|
433
|
-
},
|
|
434
|
-
{
|
|
435
|
-
id: "CaJslL1xziwefCeTNzHv",
|
|
436
|
-
name: "Cristina Campos",
|
|
437
|
-
gender: "female",
|
|
438
|
-
description: "Conversation: Friendly and Soft",
|
|
439
|
-
lang: ["Spanish"]
|
|
440
|
-
},
|
|
441
|
-
{
|
|
442
|
-
id: "YExhVa4bZONzeingloMX",
|
|
443
|
-
name: "Juan Carlos",
|
|
444
|
-
gender: "male",
|
|
445
|
-
description: "Conversation: Warm and Conversational",
|
|
446
|
-
lang: ["Spanish"]
|
|
447
|
-
},
|
|
448
|
-
{
|
|
449
|
-
id: "2Lb1en5ujrODDIqmp7F3",
|
|
450
|
-
name: "Jhenny",
|
|
451
|
-
gender: "female",
|
|
452
|
-
description: "Calm: Soft, Calm, Sweet",
|
|
453
|
-
lang: ["Spanish"]
|
|
454
|
-
},
|
|
455
|
-
{
|
|
456
|
-
id: "iDEmt5MnqUotdwCIVplo",
|
|
457
|
-
name: "Enrique Mondrag\xF3n",
|
|
458
|
-
gender: "male",
|
|
459
|
-
description: "Calm: Elegant and Dynamic",
|
|
460
|
-
lang: ["Spanish"]
|
|
461
|
-
},
|
|
462
|
-
{
|
|
463
|
-
id: "JYyJjNPfmNJdaby8LdZs",
|
|
464
|
-
name: "Marlene",
|
|
465
|
-
gender: "female",
|
|
466
|
-
description: "Lively: Melodic, Pleasant and Bright",
|
|
467
|
-
lang: ["Spanish"]
|
|
468
|
-
},
|
|
469
|
-
{
|
|
470
|
-
id: "9F4C8ztpNUmXkdDDbz3J",
|
|
471
|
-
name: "Dan",
|
|
472
|
-
gender: "male",
|
|
473
|
-
description: "Lively: Upbeat, Dynamic and Friendly",
|
|
474
|
-
lang: ["Spanish"]
|
|
475
|
-
},
|
|
476
|
-
{
|
|
477
|
-
id: "2rigMbVWLdqtBSCahJFX",
|
|
478
|
-
name: "Tatiana Martin",
|
|
479
|
-
gender: "female",
|
|
480
|
-
description: "Mature narration: Wise-speaking, calm",
|
|
481
|
-
lang: ["Spanish"]
|
|
482
|
-
},
|
|
483
|
-
{
|
|
484
|
-
id: "k8cFOyAg7B9qwBlDDNTC",
|
|
485
|
-
name: "Miguel",
|
|
486
|
-
gender: "male",
|
|
487
|
-
description: "Mature narration: Deep, Rich and Cinematic",
|
|
488
|
-
lang: ["Spanish"]
|
|
489
|
-
},
|
|
490
|
-
{
|
|
491
|
-
id: "U9tZtg3uJtVgXPkvosWR",
|
|
492
|
-
name: "Kate",
|
|
493
|
-
gender: "female",
|
|
494
|
-
description: "Character: Playful Latina Storyteller",
|
|
495
|
-
lang: ["Spanish"]
|
|
496
|
-
},
|
|
497
|
-
{
|
|
498
|
-
id: "HMCmDsbKeaSZp5LMOYKR",
|
|
499
|
-
name: "Johnny",
|
|
500
|
-
gender: "male",
|
|
501
|
-
description: "Character: Mocking, Sarcastic and Rich",
|
|
502
|
-
lang: ["Spanish"]
|
|
12
|
+
// ../ts-shared/src/ai-settings.ts
|
|
13
|
+
var GPT_IMAGE_MODELS = [
|
|
14
|
+
"gpt-image-2.5-flare",
|
|
15
|
+
"gpt-image-2.5-sunburst"
|
|
16
|
+
];
|
|
17
|
+
var GEMINI_SPEECH_MODEL_VENDORS = {
|
|
18
|
+
"gemini-3.5-transcribe-live": "gemini",
|
|
19
|
+
"gemini-3.5-transcribe": "gemini"
|
|
20
|
+
};
|
|
21
|
+
var AI_ROLES = {
|
|
22
|
+
character_draft: {
|
|
23
|
+
name: "Character draft",
|
|
24
|
+
source: "llm",
|
|
25
|
+
models: [],
|
|
26
|
+
options: ["temperature"]
|
|
503
27
|
},
|
|
504
|
-
{
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
lang: ["Russian"]
|
|
28
|
+
motion_planner: {
|
|
29
|
+
name: "Motion planner",
|
|
30
|
+
source: "llm",
|
|
31
|
+
models: [],
|
|
32
|
+
options: []
|
|
510
33
|
},
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
name: "
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
34
|
+
// Not charged.
|
|
35
|
+
prompt_check: {
|
|
36
|
+
name: "Prompt check",
|
|
37
|
+
source: "openai",
|
|
38
|
+
models: ["gpt-6-luna"],
|
|
39
|
+
options: []
|
|
517
40
|
},
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
name: "
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
41
|
+
// Priced by OPENAI_IMAGE_PRICES of the server (flare and sunburst alike).
|
|
42
|
+
design_sheet_image: {
|
|
43
|
+
name: "Design sheet image",
|
|
44
|
+
source: "openai",
|
|
45
|
+
models: GPT_IMAGE_MODELS,
|
|
46
|
+
options: []
|
|
524
47
|
},
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
name: "
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
48
|
+
// Not charged per call.
|
|
49
|
+
tts: {
|
|
50
|
+
name: "ElevenLabs text to speech",
|
|
51
|
+
source: "elevenlabs",
|
|
52
|
+
models: ["eleven_v3_conversational", "eleven_v3"],
|
|
53
|
+
options: []
|
|
531
54
|
},
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
name: "
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
55
|
+
// Not charged per call.
|
|
56
|
+
gemini_tts: {
|
|
57
|
+
name: "Gemini text to speech",
|
|
58
|
+
source: "gemini",
|
|
59
|
+
models: ["gemini-3.8-flash-tts", "gemini-3.8-flash-lite-tts"],
|
|
60
|
+
options: []
|
|
538
61
|
},
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
62
|
+
// Priced by SPEECH_RECOGNITION_PRICES of the server (docs/voice-input.md).
|
|
63
|
+
// Audio that arrives while the player speaks. The model selects the vendor.
|
|
64
|
+
speech_input_live: {
|
|
65
|
+
name: "Speech input (live)",
|
|
66
|
+
source: "openai",
|
|
67
|
+
models: [
|
|
68
|
+
"gpt-transcribe",
|
|
69
|
+
"gpt-live-transcribe",
|
|
70
|
+
"gemini-3.5-transcribe-live",
|
|
71
|
+
"gemini-3.5-transcribe"
|
|
72
|
+
],
|
|
73
|
+
modelVendors: GEMINI_SPEECH_MODEL_VENDORS,
|
|
74
|
+
options: []
|
|
545
75
|
},
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
name: "
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
76
|
+
// One complete recording (an upload of an engine SDK).
|
|
77
|
+
speech_input_file: {
|
|
78
|
+
name: "Speech input (file)",
|
|
79
|
+
source: "openai",
|
|
80
|
+
models: ["gpt-transcribe", "gemini-3.5-transcribe"],
|
|
81
|
+
modelVendors: GEMINI_SPEECH_MODEL_VENDORS,
|
|
82
|
+
options: []
|
|
552
83
|
},
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
name: "
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
84
|
+
// Priced by OPENAI_PRICES.text of the generator.
|
|
85
|
+
generator_agent: {
|
|
86
|
+
name: "Generator agent",
|
|
87
|
+
source: "openai",
|
|
88
|
+
models: ["gpt-6.1-sol"],
|
|
89
|
+
options: ["reasoningEffort"]
|
|
559
90
|
},
|
|
560
|
-
{
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
lang: ["Russian"]
|
|
91
|
+
generator_reading: {
|
|
92
|
+
name: "Generator face reading",
|
|
93
|
+
source: "openai",
|
|
94
|
+
models: ["gpt-6.1-sol"],
|
|
95
|
+
options: ["reasoningEffort"]
|
|
566
96
|
},
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
name: "
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
97
|
+
// Priced by OPENAI_PRICES of the generator (flare and sunburst alike).
|
|
98
|
+
generator_image: {
|
|
99
|
+
name: "Generator images",
|
|
100
|
+
source: "openai",
|
|
101
|
+
models: GPT_IMAGE_MODELS,
|
|
102
|
+
options: []
|
|
573
103
|
}
|
|
574
|
-
|
|
104
|
+
};
|
|
105
|
+
var AI_ROLE_NAMES = Object.keys(AI_ROLES);
|
|
575
106
|
|
|
576
|
-
// ../ts-shared/src/
|
|
577
|
-
var
|
|
578
|
-
var
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
{
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
}
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
{
|
|
658
|
-
id: "93nuHbke4dTER9x2pDwE",
|
|
659
|
-
name: "Adam",
|
|
660
|
-
gender: "male",
|
|
661
|
-
description: "Warm and Friendly",
|
|
662
|
-
lang: ["Chinese", "French", "Korean", "Spanish", "Russian"]
|
|
663
|
-
},
|
|
664
|
-
{
|
|
665
|
-
id: "6MoEUz34rbRrmmyxgRm4",
|
|
666
|
-
name: "Manav",
|
|
667
|
-
gender: "male",
|
|
668
|
-
description: "Husky and Conversational",
|
|
669
|
-
lang: ["Spanish", "Japanese", "Chinese", "Italian"]
|
|
670
|
-
},
|
|
671
|
-
{
|
|
672
|
-
id: "gOupLcAkjEnguROwi4oS",
|
|
673
|
-
name: "Darian",
|
|
674
|
-
gender: "male",
|
|
675
|
-
description: "Warm, Grounded Baritone",
|
|
676
|
-
lang: ["English"]
|
|
677
|
-
},
|
|
678
|
-
{
|
|
679
|
-
id: "ktkP7Nsj67dw2zcplQYt",
|
|
680
|
-
name: "Lawrence",
|
|
681
|
-
gender: "male",
|
|
682
|
-
description: "Bright, Informative",
|
|
683
|
-
lang: ["English"]
|
|
684
|
-
},
|
|
685
|
-
{
|
|
686
|
-
id: "10NkTYmU7tSz3Kkl3Lex",
|
|
687
|
-
name: "Kaelen",
|
|
688
|
-
gender: "male",
|
|
689
|
-
description: "Rugged, Gravelly Bass-Baritone",
|
|
690
|
-
lang: ["English"]
|
|
691
|
-
},
|
|
692
|
-
...asianTtsVoices,
|
|
693
|
-
...europeanTtsVoices
|
|
694
|
-
];
|
|
107
|
+
// ../ts-shared/src/speech-input.ts
|
|
108
|
+
var INPUT_MODES = ["options", "text", "voice"];
|
|
109
|
+
var InputModeError = class extends Error {
|
|
110
|
+
constructor(message) {
|
|
111
|
+
super(message);
|
|
112
|
+
this.name = "NpcifyInputModeError";
|
|
113
|
+
}
|
|
114
|
+
};
|
|
115
|
+
var permittedInputModes = function(permissions) {
|
|
116
|
+
if (permissions.baked || !permissions.allowCustomReply) {
|
|
117
|
+
return ["options"];
|
|
118
|
+
}
|
|
119
|
+
return ["options", "text", "voice"];
|
|
120
|
+
};
|
|
121
|
+
var isInputMode = function(value) {
|
|
122
|
+
return INPUT_MODES.includes(value);
|
|
123
|
+
};
|
|
124
|
+
var parseInputModes = function(value) {
|
|
125
|
+
if (value === void 0 || value === null) {
|
|
126
|
+
return void 0;
|
|
127
|
+
}
|
|
128
|
+
if (!Array.isArray(value)) {
|
|
129
|
+
throw new InputModeError(
|
|
130
|
+
'inputModes must be an array of "options", "text" and "voice".'
|
|
131
|
+
);
|
|
132
|
+
}
|
|
133
|
+
if (value.length === 0) {
|
|
134
|
+
throw new InputModeError("inputModes must have at least one mode.");
|
|
135
|
+
}
|
|
136
|
+
const modes = [];
|
|
137
|
+
for (const mode of value) {
|
|
138
|
+
if (!isInputMode(mode)) {
|
|
139
|
+
throw new InputModeError(
|
|
140
|
+
`inputModes has an unknown mode ${JSON.stringify(mode)}. Use "options", "text" or "voice".`
|
|
141
|
+
);
|
|
142
|
+
}
|
|
143
|
+
if (modes.includes(mode)) {
|
|
144
|
+
throw new InputModeError(`inputModes has "${mode}" two times.`);
|
|
145
|
+
}
|
|
146
|
+
modes.push(mode);
|
|
147
|
+
}
|
|
148
|
+
return modes;
|
|
149
|
+
};
|
|
150
|
+
var resolveInputModes = function(configured, permissions) {
|
|
151
|
+
const permitted = permittedInputModes(permissions);
|
|
152
|
+
if (!configured) {
|
|
153
|
+
return permitted;
|
|
154
|
+
}
|
|
155
|
+
if (permissions.baked && !(configured.length === 1 && configured[0] === "options")) {
|
|
156
|
+
throw new InputModeError(
|
|
157
|
+
'A baked (offline) conversation accepts only reply options. Set inputModes to ["options"] or remove it.'
|
|
158
|
+
);
|
|
159
|
+
}
|
|
160
|
+
const refused = configured.filter((mode) => !permitted.includes(mode));
|
|
161
|
+
if (refused.length > 0) {
|
|
162
|
+
throw new InputModeError(
|
|
163
|
+
`This character accepts only its reply options (Allow custom replies is off). Remove ${refused.map((mode) => `"${mode}"`).join(" and ")} from inputModes.`
|
|
164
|
+
);
|
|
165
|
+
}
|
|
166
|
+
return [...configured];
|
|
167
|
+
};
|
|
168
|
+
var SPEECH_AUDIO_FORMAT = Object.freeze({
|
|
169
|
+
encoding: "pcm_s16le",
|
|
170
|
+
sampleRate: 16e3,
|
|
171
|
+
channels: 1
|
|
172
|
+
});
|
|
173
|
+
var SPEECH_BYTES_PER_SECOND = 32e3;
|
|
174
|
+
var SPEECH_MAX_DURATION_MS = 3e4;
|
|
175
|
+
var SPEECH_MAX_AUDIO_BYTES = SPEECH_MAX_DURATION_MS / 1e3 * SPEECH_BYTES_PER_SECOND;
|
|
176
|
+
var SPEECH_MAX_UPLOAD_BYTES = SPEECH_MAX_AUDIO_BYTES + 4096;
|
|
177
|
+
var SPEECH_FRAME_MS = 100;
|
|
178
|
+
var SPEECH_MIN_HOLD_MS = 300;
|
|
179
|
+
var SPEECH_EVENTS = Object.freeze({
|
|
180
|
+
start: "speech:start",
|
|
181
|
+
audio: "speech:audio",
|
|
182
|
+
stop: "speech:stop",
|
|
183
|
+
cancel: "speech:cancel",
|
|
184
|
+
partial: "speech:partial",
|
|
185
|
+
turn: "speech:turn",
|
|
186
|
+
turnFailed: "speech:turn_failed"
|
|
187
|
+
});
|
|
695
188
|
|
|
696
189
|
// ../ts-shared/src/product-usage.ts
|
|
697
190
|
var productUsageEvents = [
|
|
@@ -701,7 +194,11 @@ var productUsageEvents = [
|
|
|
701
194
|
"checkout_submitted",
|
|
702
195
|
"checkout_failed",
|
|
703
196
|
"checkout_closed",
|
|
704
|
-
"paywall_shown"
|
|
197
|
+
"paywall_shown",
|
|
198
|
+
"playback_blocked",
|
|
199
|
+
"playback_start_pressed",
|
|
200
|
+
"reply_options_shown",
|
|
201
|
+
"page_left"
|
|
705
202
|
];
|
|
706
203
|
|
|
707
204
|
// ../ts-shared/src/analytics.ts
|
|
@@ -3806,8 +3303,14 @@ var LiveConversation = class {
|
|
|
3806
3303
|
this.dependencies = dependencies;
|
|
3807
3304
|
}
|
|
3808
3305
|
sending = false;
|
|
3306
|
+
/** A typed turn is in flight. */
|
|
3307
|
+
get inFlight() {
|
|
3308
|
+
return this.sending;
|
|
3309
|
+
}
|
|
3809
3310
|
pending = null;
|
|
3311
|
+
revision = 0;
|
|
3810
3312
|
reset() {
|
|
3313
|
+
this.revision += 1;
|
|
3811
3314
|
this.pending = null;
|
|
3812
3315
|
}
|
|
3813
3316
|
received(payload) {
|
|
@@ -3820,6 +3323,7 @@ var LiveConversation = class {
|
|
|
3820
3323
|
throw new Error("A reply is already being generated. Please wait.");
|
|
3821
3324
|
}
|
|
3822
3325
|
this.sending = true;
|
|
3326
|
+
const revision = this.revision;
|
|
3823
3327
|
const { telemetry, socket } = this.dependencies;
|
|
3824
3328
|
try {
|
|
3825
3329
|
telemetry.begin(conversationId);
|
|
@@ -3827,10 +3331,10 @@ var LiveConversation = class {
|
|
|
3827
3331
|
const silent = options?.silent === true;
|
|
3828
3332
|
const parentStagingId = options?.parentStagingId;
|
|
3829
3333
|
const pending = this.pending;
|
|
3830
|
-
const request = pending?.conversationId === conversationId && pending.message === message && pending.silent === silent && pending.parentStagingId === parentStagingId ? pending : {
|
|
3334
|
+
const request = !options?.requestId && pending?.conversationId === conversationId && pending.message === message && pending.silent === silent && pending.parentStagingId === parentStagingId ? pending : {
|
|
3831
3335
|
conversationId,
|
|
3832
3336
|
message,
|
|
3833
|
-
requestId: globalThis.crypto.randomUUID(),
|
|
3337
|
+
requestId: options?.requestId ?? globalThis.crypto.randomUUID(),
|
|
3834
3338
|
silent,
|
|
3835
3339
|
parentStagingId
|
|
3836
3340
|
};
|
|
@@ -3864,6 +3368,9 @@ var LiveConversation = class {
|
|
|
3864
3368
|
if (this.pending?.requestId === request.requestId) {
|
|
3865
3369
|
this.pending = null;
|
|
3866
3370
|
}
|
|
3371
|
+
if (revision !== this.revision) {
|
|
3372
|
+
return payload;
|
|
3373
|
+
}
|
|
3867
3374
|
await this.dependencies.handleResult(
|
|
3868
3375
|
payload,
|
|
3869
3376
|
payload.taskId !== request.requestId,
|
|
@@ -4994,7 +4501,7 @@ var disposeSceneContext = function(ctx) {
|
|
|
4994
4501
|
|
|
4995
4502
|
// ../vrm-renderer/src/preview-controller.ts
|
|
4996
4503
|
import { VRMUtils as VRMUtils4 } from "@pixiv/three-vrm";
|
|
4997
|
-
import { Box3, GridHelper, Vector3 as
|
|
4504
|
+
import { Box3, GridHelper, Vector3 as Vector34 } from "three";
|
|
4998
4505
|
|
|
4999
4506
|
// ../vrm-renderer/src/loaders.ts
|
|
5000
4507
|
import { VRMLoaderPlugin as VRMLoaderPlugin2, VRMUtils as VRMUtils2 } from "@pixiv/three-vrm";
|
|
@@ -5016,7 +4523,33 @@ import { AnimationMixer, LoopOnce, LoopRepeat } from "three";
|
|
|
5016
4523
|
import {
|
|
5017
4524
|
createVRMAnimationClip as createPixivVRMAnimationClip
|
|
5018
4525
|
} from "@pixiv/three-vrm-animation";
|
|
4526
|
+
import {
|
|
4527
|
+
AnimationClip,
|
|
4528
|
+
Euler,
|
|
4529
|
+
Quaternion as Quaternion2,
|
|
4530
|
+
QuaternionKeyframeTrack,
|
|
4531
|
+
QuaternionLinearInterpolant,
|
|
4532
|
+
Vector3 as Vector33,
|
|
4533
|
+
VectorKeyframeTrack
|
|
4534
|
+
} from "three";
|
|
5019
4535
|
var NPCIFY_VRM_NORMALIZATION_ROOT_NAME = "NPCifyNormalizationRoot";
|
|
4536
|
+
var TORSO_BONES = [
|
|
4537
|
+
"hips",
|
|
4538
|
+
"spine",
|
|
4539
|
+
"chest",
|
|
4540
|
+
"upperChest"
|
|
4541
|
+
];
|
|
4542
|
+
var HEAD_BONES = ["neck", "head"];
|
|
4543
|
+
var AXIAL_BONES = [...TORSO_BONES, ...HEAD_BONES];
|
|
4544
|
+
var PARENT_OF_HEAD_BONES = [
|
|
4545
|
+
...TORSO_BONES,
|
|
4546
|
+
"neck"
|
|
4547
|
+
];
|
|
4548
|
+
var IDENTITY_QUATERNION = new Quaternion2();
|
|
4549
|
+
var UP_AXIS = new Vector33(0, 1, 0);
|
|
4550
|
+
var MAX_HEAD_PITCH_CORRECTION_RAD = 35 * Math.PI / 180;
|
|
4551
|
+
var NECK_CORRECTION_SHARE = 0.35;
|
|
4552
|
+
var EPSILON = 1e-5;
|
|
5020
4553
|
var canonicalAnimationTargets = /* @__PURE__ */ new WeakMap();
|
|
5021
4554
|
var isNpcifyNormalizedVrm = function(vrm) {
|
|
5022
4555
|
return vrm.scene.children.some(
|
|
@@ -5042,39 +4575,331 @@ var canonicalAnimationTarget = function(vrm) {
|
|
|
5042
4575
|
canonicalAnimationTargets.set(vrm, target);
|
|
5043
4576
|
return target;
|
|
5044
4577
|
};
|
|
5045
|
-
var
|
|
5046
|
-
|
|
4578
|
+
var resolveAxialBoneNodeNames = function(vrm) {
|
|
4579
|
+
const result = /* @__PURE__ */ new Map();
|
|
4580
|
+
const seenNodeNames = /* @__PURE__ */ new Set();
|
|
4581
|
+
for (const bone of AXIAL_BONES) {
|
|
4582
|
+
const nodeName = vrm.humanoid?.getNormalizedBoneNode?.(bone)?.name;
|
|
4583
|
+
if (nodeName && !seenNodeNames.has(nodeName)) {
|
|
4584
|
+
seenNodeNames.add(nodeName);
|
|
4585
|
+
result.set(bone, nodeName);
|
|
4586
|
+
}
|
|
4587
|
+
}
|
|
4588
|
+
return result;
|
|
5047
4589
|
};
|
|
5048
|
-
|
|
5049
|
-
|
|
5050
|
-
|
|
5051
|
-
|
|
5052
|
-
|
|
5053
|
-
|
|
5054
|
-
|
|
5055
|
-
(name) => vrm.humanoid.getRawBoneNode(name)
|
|
5056
|
-
)
|
|
4590
|
+
var findQuaternionTrack = function(clip, nodeName) {
|
|
4591
|
+
if (!nodeName) {
|
|
4592
|
+
return void 0;
|
|
4593
|
+
}
|
|
4594
|
+
const trackName = `${nodeName}.quaternion`;
|
|
4595
|
+
return clip.tracks.find(
|
|
4596
|
+
(track) => track.name === trackName && track.times.length > 0 && track.values.length >= track.times.length * 4
|
|
5057
4597
|
);
|
|
5058
|
-
const mixer = new AnimationMixer(vrm.scene);
|
|
5059
|
-
return { vrm, mixer, activeAction: null, animationFrameId: null };
|
|
5060
4598
|
};
|
|
5061
|
-
var
|
|
5062
|
-
|
|
5063
|
-
|
|
5064
|
-
if (session.vrm.scene.parent) {
|
|
5065
|
-
session.vrm.scene.parent.remove(session.vrm.scene);
|
|
5066
|
-
}
|
|
5067
|
-
setCameraCollisionBackend(ctx, null);
|
|
5068
|
-
session.mixer.uncacheRoot(session.vrm.scene);
|
|
5069
|
-
if (options.dispose) {
|
|
5070
|
-
disposeVrm(session.vrm);
|
|
4599
|
+
var sampleBoneQuaternion = function(interpolant, time, target) {
|
|
4600
|
+
if (!interpolant) {
|
|
4601
|
+
return target.identity();
|
|
5071
4602
|
}
|
|
4603
|
+
const values = interpolant.evaluate(time);
|
|
4604
|
+
return target.set(values[0] ?? 0, values[1] ?? 0, values[2] ?? 0, values[3] ?? 1).normalize();
|
|
5072
4605
|
};
|
|
5073
|
-
var
|
|
5074
|
-
|
|
4606
|
+
var evaluateChainOrientation = function(bones, interpolants, time, rootPreRotation, target, scratch) {
|
|
4607
|
+
target.copy(rootPreRotation);
|
|
4608
|
+
for (const bone of bones) {
|
|
4609
|
+
const interpolant = interpolants.get(bone);
|
|
4610
|
+
if (interpolant) {
|
|
4611
|
+
sampleBoneQuaternion(interpolant, time, scratch);
|
|
4612
|
+
target.multiply(scratch);
|
|
4613
|
+
}
|
|
4614
|
+
}
|
|
4615
|
+
return target;
|
|
5075
4616
|
};
|
|
5076
|
-
var
|
|
5077
|
-
|
|
4617
|
+
var rotateQuaternionTrackInParentSpace = function(track, parentBones, interpolants, rootPreRotation, worldCorrection) {
|
|
4618
|
+
const values = new Float32Array(track.values.length);
|
|
4619
|
+
const parentQuat = new Quaternion2();
|
|
4620
|
+
const parentInvQuat = new Quaternion2();
|
|
4621
|
+
const localCorrection = new Quaternion2();
|
|
4622
|
+
const keyframeQuat = new Quaternion2();
|
|
4623
|
+
const prevOrigQuat = new Quaternion2();
|
|
4624
|
+
const prevNewQuat = new Quaternion2();
|
|
4625
|
+
const scratchQuat = new Quaternion2();
|
|
4626
|
+
const count = track.times.length;
|
|
4627
|
+
for (let i = 0; i < count; i += 1) {
|
|
4628
|
+
const time = track.times[i] ?? 0;
|
|
4629
|
+
const offset = i * 4;
|
|
4630
|
+
keyframeQuat.set(
|
|
4631
|
+
track.values[offset] ?? 0,
|
|
4632
|
+
track.values[offset + 1] ?? 0,
|
|
4633
|
+
track.values[offset + 2] ?? 0,
|
|
4634
|
+
track.values[offset + 3] ?? 1
|
|
4635
|
+
).normalize();
|
|
4636
|
+
const origDot = i > 0 ? prevOrigQuat.dot(keyframeQuat) : 1;
|
|
4637
|
+
prevOrigQuat.copy(keyframeQuat);
|
|
4638
|
+
if (parentBones.length === 0) {
|
|
4639
|
+
keyframeQuat.premultiply(worldCorrection).normalize();
|
|
4640
|
+
} else {
|
|
4641
|
+
evaluateChainOrientation(
|
|
4642
|
+
parentBones,
|
|
4643
|
+
interpolants,
|
|
4644
|
+
time,
|
|
4645
|
+
rootPreRotation,
|
|
4646
|
+
parentQuat,
|
|
4647
|
+
scratchQuat
|
|
4648
|
+
);
|
|
4649
|
+
parentInvQuat.copy(parentQuat).invert();
|
|
4650
|
+
localCorrection.copy(parentInvQuat).multiply(worldCorrection).multiply(parentQuat);
|
|
4651
|
+
keyframeQuat.premultiply(localCorrection).normalize();
|
|
4652
|
+
}
|
|
4653
|
+
if (i > 0 && origDot >= 0 && prevNewQuat.dot(keyframeQuat) < 0) {
|
|
4654
|
+
keyframeQuat.set(
|
|
4655
|
+
-keyframeQuat.x,
|
|
4656
|
+
-keyframeQuat.y,
|
|
4657
|
+
-keyframeQuat.z,
|
|
4658
|
+
-keyframeQuat.w
|
|
4659
|
+
);
|
|
4660
|
+
}
|
|
4661
|
+
prevNewQuat.copy(keyframeQuat);
|
|
4662
|
+
values[offset] = keyframeQuat.x;
|
|
4663
|
+
values[offset + 1] = keyframeQuat.y;
|
|
4664
|
+
values[offset + 2] = keyframeQuat.z;
|
|
4665
|
+
values[offset + 3] = keyframeQuat.w;
|
|
4666
|
+
}
|
|
4667
|
+
return new QuaternionKeyframeTrack(
|
|
4668
|
+
track.name,
|
|
4669
|
+
Array.from(track.times),
|
|
4670
|
+
Array.from(values)
|
|
4671
|
+
);
|
|
4672
|
+
};
|
|
4673
|
+
var rotateHipsPositionTrack = function(track, yawRadians) {
|
|
4674
|
+
const count = Math.floor(track.values.length / 3);
|
|
4675
|
+
if (count === 0) {
|
|
4676
|
+
return new VectorKeyframeTrack(
|
|
4677
|
+
track.name,
|
|
4678
|
+
Array.from(track.times),
|
|
4679
|
+
Array.from(track.values)
|
|
4680
|
+
);
|
|
4681
|
+
}
|
|
4682
|
+
let meanX = 0;
|
|
4683
|
+
let meanZ = 0;
|
|
4684
|
+
for (let i = 0; i < count; i += 1) {
|
|
4685
|
+
meanX += track.values[i * 3] ?? 0;
|
|
4686
|
+
meanZ += track.values[i * 3 + 2] ?? 0;
|
|
4687
|
+
}
|
|
4688
|
+
meanX /= count;
|
|
4689
|
+
meanZ /= count;
|
|
4690
|
+
const cosYaw = Math.cos(yawRadians);
|
|
4691
|
+
const sinYaw = Math.sin(yawRadians);
|
|
4692
|
+
const values = new Float32Array(track.values.length);
|
|
4693
|
+
for (let i = 0; i < count; i += 1) {
|
|
4694
|
+
const offset = i * 3;
|
|
4695
|
+
const dx = (track.values[offset] ?? 0) - meanX;
|
|
4696
|
+
const y = track.values[offset + 1] ?? 0;
|
|
4697
|
+
const dz = (track.values[offset + 2] ?? 0) - meanZ;
|
|
4698
|
+
values[offset] = meanX + dx * cosYaw + dz * sinYaw;
|
|
4699
|
+
values[offset + 1] = y;
|
|
4700
|
+
values[offset + 2] = meanZ - dx * sinYaw + dz * cosYaw;
|
|
4701
|
+
}
|
|
4702
|
+
return new VectorKeyframeTrack(
|
|
4703
|
+
track.name,
|
|
4704
|
+
Array.from(track.times),
|
|
4705
|
+
Array.from(values)
|
|
4706
|
+
);
|
|
4707
|
+
};
|
|
4708
|
+
var normalizeIdleAnimationClip = function(clip, vrm) {
|
|
4709
|
+
const boneNodeNames = resolveAxialBoneNodeNames(vrm);
|
|
4710
|
+
if (boneNodeNames.size === 0) {
|
|
4711
|
+
return clip;
|
|
4712
|
+
}
|
|
4713
|
+
const interpolants = /* @__PURE__ */ new Map();
|
|
4714
|
+
for (const bone of AXIAL_BONES) {
|
|
4715
|
+
const track = findQuaternionTrack(clip, boneNodeNames.get(bone));
|
|
4716
|
+
if (track) {
|
|
4717
|
+
interpolants.set(
|
|
4718
|
+
bone,
|
|
4719
|
+
new QuaternionLinearInterpolant(track.times, track.values, 4)
|
|
4720
|
+
);
|
|
4721
|
+
}
|
|
4722
|
+
}
|
|
4723
|
+
if (interpolants.size === 0) {
|
|
4724
|
+
return clip;
|
|
4725
|
+
}
|
|
4726
|
+
const duration = clip.duration > 0 ? clip.duration : 0;
|
|
4727
|
+
const sampleCount = duration > 0 ? Math.max(16, Math.min(120, Math.ceil(duration * 30))) : 1;
|
|
4728
|
+
const sampleTimes = [];
|
|
4729
|
+
for (let i = 0; i < sampleCount; i += 1) {
|
|
4730
|
+
sampleTimes.push(duration > 0 ? (i + 0.5) / sampleCount * duration : 0);
|
|
4731
|
+
}
|
|
4732
|
+
const chainQuat = new Quaternion2();
|
|
4733
|
+
const scratchQuat = new Quaternion2();
|
|
4734
|
+
const forwardVec = new Vector33();
|
|
4735
|
+
const rootTorsoBone = TORSO_BONES.find((bone) => interpolants.has(bone));
|
|
4736
|
+
const rootPreRotation = new Quaternion2();
|
|
4737
|
+
let torsoYawCorrection = 0;
|
|
4738
|
+
if (rootTorsoBone) {
|
|
4739
|
+
let sumX = 0;
|
|
4740
|
+
let sumZ = 0;
|
|
4741
|
+
for (const time of sampleTimes) {
|
|
4742
|
+
evaluateChainOrientation(
|
|
4743
|
+
TORSO_BONES,
|
|
4744
|
+
interpolants,
|
|
4745
|
+
time,
|
|
4746
|
+
IDENTITY_QUATERNION,
|
|
4747
|
+
chainQuat,
|
|
4748
|
+
scratchQuat
|
|
4749
|
+
);
|
|
4750
|
+
forwardVec.set(0, 0, 1).applyQuaternion(chainQuat);
|
|
4751
|
+
const horizLen = Math.hypot(forwardVec.x, forwardVec.z);
|
|
4752
|
+
if (horizLen > EPSILON) {
|
|
4753
|
+
sumX += forwardVec.x / horizLen;
|
|
4754
|
+
sumZ += forwardVec.z / horizLen;
|
|
4755
|
+
}
|
|
4756
|
+
}
|
|
4757
|
+
if (Math.hypot(sumX, sumZ) > EPSILON) {
|
|
4758
|
+
const meanTorsoYaw = Math.atan2(sumX, sumZ);
|
|
4759
|
+
if (Math.abs(meanTorsoYaw) > EPSILON) {
|
|
4760
|
+
torsoYawCorrection = -meanTorsoYaw;
|
|
4761
|
+
rootPreRotation.setFromAxisAngle(UP_AXIS, torsoYawCorrection);
|
|
4762
|
+
}
|
|
4763
|
+
}
|
|
4764
|
+
}
|
|
4765
|
+
const hasNeckTrack = interpolants.has("neck");
|
|
4766
|
+
const hasHeadTrack = interpolants.has("head");
|
|
4767
|
+
let neckWorldCorrection = null;
|
|
4768
|
+
let headWorldCorrection = null;
|
|
4769
|
+
if (hasNeckTrack || hasHeadTrack) {
|
|
4770
|
+
const meanHeadForward = new Vector33();
|
|
4771
|
+
for (const time of sampleTimes) {
|
|
4772
|
+
evaluateChainOrientation(
|
|
4773
|
+
AXIAL_BONES,
|
|
4774
|
+
interpolants,
|
|
4775
|
+
time,
|
|
4776
|
+
rootPreRotation,
|
|
4777
|
+
chainQuat,
|
|
4778
|
+
scratchQuat
|
|
4779
|
+
);
|
|
4780
|
+
forwardVec.set(0, 0, 1).applyQuaternion(chainQuat);
|
|
4781
|
+
meanHeadForward.add(forwardVec);
|
|
4782
|
+
}
|
|
4783
|
+
if (meanHeadForward.lengthSq() > EPSILON * EPSILON) {
|
|
4784
|
+
meanHeadForward.normalize();
|
|
4785
|
+
const meanHeadYaw = Math.atan2(meanHeadForward.x, meanHeadForward.z);
|
|
4786
|
+
const rawHeadPitch = Math.atan2(
|
|
4787
|
+
-meanHeadForward.y,
|
|
4788
|
+
Math.hypot(meanHeadForward.x, meanHeadForward.z)
|
|
4789
|
+
);
|
|
4790
|
+
const clampedHeadPitch = Math.max(
|
|
4791
|
+
-MAX_HEAD_PITCH_CORRECTION_RAD,
|
|
4792
|
+
Math.min(MAX_HEAD_PITCH_CORRECTION_RAD, rawHeadPitch)
|
|
4793
|
+
);
|
|
4794
|
+
if (Math.abs(meanHeadYaw) > EPSILON || Math.abs(clampedHeadPitch) > EPSILON) {
|
|
4795
|
+
const totalWorldCorrection = new Quaternion2().setFromEuler(
|
|
4796
|
+
new Euler(-clampedHeadPitch, -meanHeadYaw, 0, "XYZ")
|
|
4797
|
+
);
|
|
4798
|
+
if (hasNeckTrack && hasHeadTrack) {
|
|
4799
|
+
neckWorldCorrection = new Quaternion2().setFromEuler(
|
|
4800
|
+
new Euler(
|
|
4801
|
+
-clampedHeadPitch * NECK_CORRECTION_SHARE,
|
|
4802
|
+
-meanHeadYaw * NECK_CORRECTION_SHARE,
|
|
4803
|
+
0,
|
|
4804
|
+
"XYZ"
|
|
4805
|
+
)
|
|
4806
|
+
);
|
|
4807
|
+
headWorldCorrection = neckWorldCorrection.clone().invert().multiply(totalWorldCorrection);
|
|
4808
|
+
} else if (hasNeckTrack) {
|
|
4809
|
+
neckWorldCorrection = totalWorldCorrection;
|
|
4810
|
+
} else {
|
|
4811
|
+
headWorldCorrection = totalWorldCorrection;
|
|
4812
|
+
}
|
|
4813
|
+
}
|
|
4814
|
+
}
|
|
4815
|
+
}
|
|
4816
|
+
const hasTorsoCorrection = Math.abs(torsoYawCorrection) > EPSILON;
|
|
4817
|
+
if (!hasTorsoCorrection && !neckWorldCorrection && !headWorldCorrection) {
|
|
4818
|
+
return clip;
|
|
4819
|
+
}
|
|
4820
|
+
const rootTorsoNodeName = rootTorsoBone ? boneNodeNames.get(rootTorsoBone) : void 0;
|
|
4821
|
+
const rootTorsoQuatTrackName = rootTorsoNodeName ? `${rootTorsoNodeName}.quaternion` : null;
|
|
4822
|
+
const hipsNodeName = boneNodeNames.get("hips");
|
|
4823
|
+
const hipsPosTrackName = hipsNodeName && rootTorsoBone === "hips" ? `${hipsNodeName}.position` : null;
|
|
4824
|
+
const neckNodeName = boneNodeNames.get("neck");
|
|
4825
|
+
const neckQuatTrackName = neckNodeName ? `${neckNodeName}.quaternion` : null;
|
|
4826
|
+
const headNodeName = boneNodeNames.get("head");
|
|
4827
|
+
const headQuatTrackName = headNodeName ? `${headNodeName}.quaternion` : null;
|
|
4828
|
+
const nextTracks = clip.tracks.map((track) => {
|
|
4829
|
+
if (hasTorsoCorrection && track.name === rootTorsoQuatTrackName) {
|
|
4830
|
+
return rotateQuaternionTrackInParentSpace(
|
|
4831
|
+
track,
|
|
4832
|
+
[],
|
|
4833
|
+
interpolants,
|
|
4834
|
+
IDENTITY_QUATERNION,
|
|
4835
|
+
rootPreRotation
|
|
4836
|
+
);
|
|
4837
|
+
}
|
|
4838
|
+
if (hasTorsoCorrection && track.name === hipsPosTrackName) {
|
|
4839
|
+
return rotateHipsPositionTrack(track, torsoYawCorrection);
|
|
4840
|
+
}
|
|
4841
|
+
if (neckWorldCorrection && track.name === neckQuatTrackName) {
|
|
4842
|
+
return rotateQuaternionTrackInParentSpace(
|
|
4843
|
+
track,
|
|
4844
|
+
TORSO_BONES,
|
|
4845
|
+
interpolants,
|
|
4846
|
+
rootPreRotation,
|
|
4847
|
+
neckWorldCorrection
|
|
4848
|
+
);
|
|
4849
|
+
}
|
|
4850
|
+
if (headWorldCorrection && track.name === headQuatTrackName) {
|
|
4851
|
+
return rotateQuaternionTrackInParentSpace(
|
|
4852
|
+
track,
|
|
4853
|
+
PARENT_OF_HEAD_BONES,
|
|
4854
|
+
interpolants,
|
|
4855
|
+
rootPreRotation,
|
|
4856
|
+
headWorldCorrection
|
|
4857
|
+
);
|
|
4858
|
+
}
|
|
4859
|
+
return track;
|
|
4860
|
+
});
|
|
4861
|
+
return new AnimationClip(clip.name, clip.duration, nextTracks, clip.blendMode);
|
|
4862
|
+
};
|
|
4863
|
+
var createVRMAnimationClip = function(animation, vrm, options) {
|
|
4864
|
+
const clip = createPixivVRMAnimationClip(
|
|
4865
|
+
animation,
|
|
4866
|
+
canonicalAnimationTarget(vrm)
|
|
4867
|
+
);
|
|
4868
|
+
if (options?.normalizeIdleOrientation) {
|
|
4869
|
+
return normalizeIdleAnimationClip(clip, vrm);
|
|
4870
|
+
}
|
|
4871
|
+
return clip;
|
|
4872
|
+
};
|
|
4873
|
+
|
|
4874
|
+
// ../vrm-renderer/src/session.ts
|
|
4875
|
+
var attachVrm = function(ctx, vrm) {
|
|
4876
|
+
ctx.scene.add(vrm.scene);
|
|
4877
|
+
setCameraCollisionBackend(
|
|
4878
|
+
ctx,
|
|
4879
|
+
new HumanoidCapsuleCollisionBackend(
|
|
4880
|
+
(name) => vrm.humanoid.getRawBoneNode(name)
|
|
4881
|
+
)
|
|
4882
|
+
);
|
|
4883
|
+
const mixer = new AnimationMixer(vrm.scene);
|
|
4884
|
+
return { vrm, mixer, activeAction: null, animationFrameId: null };
|
|
4885
|
+
};
|
|
4886
|
+
var detachVrm = function(ctx, session, options = {}) {
|
|
4887
|
+
stopRenderLoop(session);
|
|
4888
|
+
session.mixer.stopAllAction();
|
|
4889
|
+
if (session.vrm.scene.parent) {
|
|
4890
|
+
session.vrm.scene.parent.remove(session.vrm.scene);
|
|
4891
|
+
}
|
|
4892
|
+
setCameraCollisionBackend(ctx, null);
|
|
4893
|
+
session.mixer.uncacheRoot(session.vrm.scene);
|
|
4894
|
+
if (options.dispose) {
|
|
4895
|
+
disposeVrm(session.vrm);
|
|
4896
|
+
}
|
|
4897
|
+
};
|
|
4898
|
+
var disposeVrm = function(vrm) {
|
|
4899
|
+
disposeObject3D(vrm.scene);
|
|
4900
|
+
};
|
|
4901
|
+
var disposeObject3D = function(object) {
|
|
4902
|
+
VRMUtils3.deepDispose(object);
|
|
5078
4903
|
};
|
|
5079
4904
|
var stopRenderLoop = function(session) {
|
|
5080
4905
|
if (session.animationFrameId === null) {
|
|
@@ -5380,16 +5205,20 @@ var RemoteAnimationController = class {
|
|
|
5380
5205
|
}
|
|
5381
5206
|
};
|
|
5382
5207
|
var AnimationController = class {
|
|
5383
|
-
constructor(vrm, idleAnimationId) {
|
|
5208
|
+
constructor(vrm, idleAnimationId, options) {
|
|
5384
5209
|
this.vrm = vrm;
|
|
5385
5210
|
this.mixer = new AnimationMixer2(this.vrm.scene);
|
|
5386
5211
|
this.idleAnimationId = idleAnimationId ?? null;
|
|
5212
|
+
this.normalizeIdleOrientation = options?.normalizeIdleOrientation ?? false;
|
|
5387
5213
|
this.mixer.addEventListener("finished", this.handleActionFinished);
|
|
5388
5214
|
}
|
|
5389
5215
|
prepareAnimations;
|
|
5390
5216
|
mixer;
|
|
5391
5217
|
clips = /* @__PURE__ */ new Map();
|
|
5218
|
+
rawClips = /* @__PURE__ */ new Map();
|
|
5219
|
+
normalizedIdleClips = /* @__PURE__ */ new Map();
|
|
5392
5220
|
configs = /* @__PURE__ */ new Map();
|
|
5221
|
+
normalizeIdleOrientation;
|
|
5393
5222
|
previewMixer = null;
|
|
5394
5223
|
restorePreview = null;
|
|
5395
5224
|
activeAction = null;
|
|
@@ -5415,8 +5244,10 @@ var AnimationController = class {
|
|
|
5415
5244
|
return this.clips.has(id);
|
|
5416
5245
|
}
|
|
5417
5246
|
addAnimation(animationId, animation, config) {
|
|
5418
|
-
const
|
|
5419
|
-
this.
|
|
5247
|
+
const rawClip = createVRMAnimationClip(animation, this.vrm);
|
|
5248
|
+
this.rawClips.set(animationId, rawClip);
|
|
5249
|
+
this.normalizedIdleClips.delete(animationId);
|
|
5250
|
+
this.clips.set(animationId, this.resolveClipForId(animationId, rawClip));
|
|
5420
5251
|
this.configs.set(animationId, {
|
|
5421
5252
|
loop: config?.loop ?? defaultAnimationConfig.loop,
|
|
5422
5253
|
weight: config?.weight ?? defaultAnimationConfig.weight
|
|
@@ -5446,7 +5277,20 @@ var AnimationController = class {
|
|
|
5446
5277
|
return restore;
|
|
5447
5278
|
}
|
|
5448
5279
|
setIdleAnimation(animationId) {
|
|
5280
|
+
const previousIdleId = this.idleAnimationId;
|
|
5449
5281
|
this.idleAnimationId = animationId;
|
|
5282
|
+
if (previousIdleId && previousIdleId !== animationId) {
|
|
5283
|
+
const previousRaw = this.rawClips.get(previousIdleId);
|
|
5284
|
+
if (previousRaw) {
|
|
5285
|
+
this.clips.set(previousIdleId, previousRaw);
|
|
5286
|
+
}
|
|
5287
|
+
}
|
|
5288
|
+
if (animationId) {
|
|
5289
|
+
const rawClip = this.rawClips.get(animationId);
|
|
5290
|
+
if (rawClip) {
|
|
5291
|
+
this.clips.set(animationId, this.resolveClipForId(animationId, rawClip));
|
|
5292
|
+
}
|
|
5293
|
+
}
|
|
5450
5294
|
this.playIdle();
|
|
5451
5295
|
}
|
|
5452
5296
|
clearAnimations() {
|
|
@@ -5454,6 +5298,8 @@ var AnimationController = class {
|
|
|
5454
5298
|
this.mixer.stopAllAction();
|
|
5455
5299
|
this.mixer.uncacheRoot(this.vrm.scene);
|
|
5456
5300
|
this.clips.clear();
|
|
5301
|
+
this.rawClips.clear();
|
|
5302
|
+
this.normalizedIdleClips.clear();
|
|
5457
5303
|
this.configs.clear();
|
|
5458
5304
|
this.activeAction = null;
|
|
5459
5305
|
this.activeAnimationId = null;
|
|
@@ -5461,6 +5307,18 @@ var AnimationController = class {
|
|
|
5461
5307
|
this.idleAnimationId = null;
|
|
5462
5308
|
this.vrm.humanoid.resetNormalizedPose();
|
|
5463
5309
|
}
|
|
5310
|
+
resolveClipForId(animationId, rawClip) {
|
|
5311
|
+
if (!this.normalizeIdleOrientation || animationId !== this.idleAnimationId) {
|
|
5312
|
+
return rawClip;
|
|
5313
|
+
}
|
|
5314
|
+
const cached = this.normalizedIdleClips.get(animationId);
|
|
5315
|
+
if (cached) {
|
|
5316
|
+
return cached;
|
|
5317
|
+
}
|
|
5318
|
+
const normalized = normalizeIdleAnimationClip(rawClip, this.vrm);
|
|
5319
|
+
this.normalizedIdleClips.set(animationId, normalized);
|
|
5320
|
+
return normalized;
|
|
5321
|
+
}
|
|
5464
5322
|
playAnimation(animationId, fadeDuration = 0.25, loopOverride) {
|
|
5465
5323
|
this.restorePreview?.();
|
|
5466
5324
|
const clip = this.clips.get(animationId);
|
|
@@ -6229,6 +6087,7 @@ var StreamPlaybackController = class {
|
|
|
6229
6087
|
force: options?.force === true
|
|
6230
6088
|
});
|
|
6231
6089
|
}
|
|
6090
|
+
/** Returns the playback result for a timeline event. */
|
|
6232
6091
|
handleEvent(event) {
|
|
6233
6092
|
if (this.streamEventCallback) {
|
|
6234
6093
|
this.safeInvoke("onStreamEvent", () => this.streamEventCallback?.(event));
|
|
@@ -6272,14 +6131,13 @@ var StreamPlaybackController = class {
|
|
|
6272
6131
|
if (event.type === "staging" || event.type === "staging_prefix" || event.type === "staging_live") {
|
|
6273
6132
|
this.recordStreamEvent(event, "timeline");
|
|
6274
6133
|
this.dependencies.telemetry.recordFirstStaging();
|
|
6275
|
-
|
|
6134
|
+
return this.playTimeline({
|
|
6276
6135
|
taskId: event.taskId,
|
|
6277
6136
|
stagingId: event.stagingId,
|
|
6278
6137
|
timeline: event.timeline,
|
|
6279
6138
|
complete: event.complete,
|
|
6280
6139
|
revision: event.revision
|
|
6281
6140
|
});
|
|
6282
|
-
return;
|
|
6283
6141
|
}
|
|
6284
6142
|
if (event.type === "options") {
|
|
6285
6143
|
this.recordStreamEvent(event, "options");
|
|
@@ -6638,8 +6496,18 @@ var decodeBase64 = function(value) {
|
|
|
6638
6496
|
}
|
|
6639
6497
|
return bytes;
|
|
6640
6498
|
};
|
|
6641
|
-
var
|
|
6642
|
-
return
|
|
6499
|
+
var managedMediaSource = function() {
|
|
6500
|
+
return globalThis.ManagedMediaSource;
|
|
6501
|
+
};
|
|
6502
|
+
var needsManagedMediaSource = function() {
|
|
6503
|
+
return typeof MediaSource === "undefined" && Boolean(managedMediaSource());
|
|
6504
|
+
};
|
|
6505
|
+
var streamingMediaSource = function(mimeType) {
|
|
6506
|
+
if (typeof URL?.createObjectURL !== "function") {
|
|
6507
|
+
return null;
|
|
6508
|
+
}
|
|
6509
|
+
const source = typeof MediaSource === "undefined" ? managedMediaSource() : MediaSource;
|
|
6510
|
+
return source?.isTypeSupported(mimeType) ? source : null;
|
|
6643
6511
|
};
|
|
6644
6512
|
var StreamingAudioBuffer = class {
|
|
6645
6513
|
constructor(mimeType) {
|
|
@@ -6682,8 +6550,9 @@ var StreamingAudioBuffer = class {
|
|
|
6682
6550
|
if (this.objectUrl) {
|
|
6683
6551
|
return this.objectUrl;
|
|
6684
6552
|
}
|
|
6685
|
-
|
|
6686
|
-
|
|
6553
|
+
const Source2 = streamingMediaSource(this.mimeType);
|
|
6554
|
+
if (Source2) {
|
|
6555
|
+
this.mediaSource = new Source2();
|
|
6687
6556
|
this.objectUrl = URL.createObjectURL(this.mediaSource);
|
|
6688
6557
|
this.mediaSource.addEventListener("sourceopen", this.handleSourceOpen, {
|
|
6689
6558
|
once: true
|
|
@@ -6775,7 +6644,7 @@ var StreamingAudioBuffer = class {
|
|
|
6775
6644
|
}
|
|
6776
6645
|
};
|
|
6777
6646
|
finishFallbackIfReady() {
|
|
6778
|
-
if (!this.completed || this.objectUrl ||
|
|
6647
|
+
if (!this.completed || this.objectUrl || streamingMediaSource(this.mimeType) || this.failedError || this.disposed) {
|
|
6779
6648
|
return;
|
|
6780
6649
|
}
|
|
6781
6650
|
const blobParts = this.chunks.map((chunk) => Uint8Array.from(chunk).buffer);
|
|
@@ -7102,7 +6971,8 @@ var AudioPlayer = class {
|
|
|
7102
6971
|
this.visemePlayer.cleanup();
|
|
7103
6972
|
if (this.audioElement) {
|
|
7104
6973
|
this.audioElement.srcObject = null;
|
|
7105
|
-
this.audioElement.src
|
|
6974
|
+
this.audioElement.removeAttribute("src");
|
|
6975
|
+
this.audioElement.load();
|
|
7106
6976
|
}
|
|
7107
6977
|
this.streams.forEach((stream) => stream.dispose());
|
|
7108
6978
|
this.streams.clear();
|
|
@@ -7119,8 +6989,10 @@ var AudioPlayer = class {
|
|
|
7119
6989
|
this.streams.delete(streamId);
|
|
7120
6990
|
}
|
|
7121
6991
|
buildSilentWav = function() {
|
|
7122
|
-
const
|
|
7123
|
-
const
|
|
6992
|
+
const sampleRate = 44100;
|
|
6993
|
+
const bytesPerSample = 2;
|
|
6994
|
+
const dataSize = Math.round(sampleRate * 0.1) * bytesPerSample;
|
|
6995
|
+
const bytes = new Uint8Array(44 + dataSize);
|
|
7124
6996
|
const view = new DataView(bytes.buffer);
|
|
7125
6997
|
const writeText = function(offset, value) {
|
|
7126
6998
|
Array.from(value).forEach((character, index) => {
|
|
@@ -7128,19 +7000,18 @@ var AudioPlayer = class {
|
|
|
7128
7000
|
});
|
|
7129
7001
|
};
|
|
7130
7002
|
writeText(0, "RIFF");
|
|
7131
|
-
view.setUint32(4, 36 +
|
|
7003
|
+
view.setUint32(4, 36 + dataSize, true);
|
|
7132
7004
|
writeText(8, "WAVE");
|
|
7133
7005
|
writeText(12, "fmt ");
|
|
7134
7006
|
view.setUint32(16, 16, true);
|
|
7135
7007
|
view.setUint16(20, 1, true);
|
|
7136
7008
|
view.setUint16(22, 1, true);
|
|
7137
|
-
view.setUint32(24,
|
|
7138
|
-
view.setUint32(28,
|
|
7139
|
-
view.setUint16(32,
|
|
7140
|
-
view.setUint16(34, 8, true);
|
|
7009
|
+
view.setUint32(24, sampleRate, true);
|
|
7010
|
+
view.setUint32(28, sampleRate * bytesPerSample, true);
|
|
7011
|
+
view.setUint16(32, bytesPerSample, true);
|
|
7012
|
+
view.setUint16(34, bytesPerSample * 8, true);
|
|
7141
7013
|
writeText(36, "data");
|
|
7142
|
-
view.setUint32(40,
|
|
7143
|
-
bytes.fill(128, 44);
|
|
7014
|
+
view.setUint32(40, dataSize, true);
|
|
7144
7015
|
return new Blob([bytes], { type: "audio/wav" });
|
|
7145
7016
|
};
|
|
7146
7017
|
async resolveSourceUrl(source) {
|
|
@@ -7176,6 +7047,9 @@ var AudioPlayer = class {
|
|
|
7176
7047
|
element.volume = this.volumeValue;
|
|
7177
7048
|
element.muted = this.mutedValue;
|
|
7178
7049
|
element.playbackRate = this.playbackRateValue;
|
|
7050
|
+
if (needsManagedMediaSource()) {
|
|
7051
|
+
element.disableRemotePlayback = true;
|
|
7052
|
+
}
|
|
7179
7053
|
if (this.crossOriginValue !== null) {
|
|
7180
7054
|
element.crossOrigin = this.crossOriginValue;
|
|
7181
7055
|
}
|
|
@@ -7259,6 +7133,14 @@ var CharacterExpressions = class {
|
|
|
7259
7133
|
clearEmotion(mouthMode) {
|
|
7260
7134
|
this.mixer.clearEmotion(mouthMode);
|
|
7261
7135
|
}
|
|
7136
|
+
/** Set a temporary emotion overlay. Timeline emotions and visemes keep their state. */
|
|
7137
|
+
setOverlay(weights) {
|
|
7138
|
+
this.mixer.setOverlay(weights);
|
|
7139
|
+
}
|
|
7140
|
+
/** Ease the temporary overlay back to the current timeline or idle face. */
|
|
7141
|
+
clearOverlay() {
|
|
7142
|
+
this.mixer.clearOverlay();
|
|
7143
|
+
}
|
|
7262
7144
|
/** Reset all expression layer targets to zero. */
|
|
7263
7145
|
reset() {
|
|
7264
7146
|
this.mixer.reset();
|
|
@@ -7433,7 +7315,7 @@ var FacialPerformanceInspector = class {
|
|
|
7433
7315
|
};
|
|
7434
7316
|
|
|
7435
7317
|
// ../../clients/web-client/src/character/idle-face.ts
|
|
7436
|
-
import { Vector3 as
|
|
7318
|
+
import { Vector3 as Vector35 } from "three";
|
|
7437
7319
|
var IdleFace = class {
|
|
7438
7320
|
constructor(vrm, random) {
|
|
7439
7321
|
this.vrm = vrm;
|
|
@@ -7441,7 +7323,7 @@ var IdleFace = class {
|
|
|
7441
7323
|
}
|
|
7442
7324
|
time = 0;
|
|
7443
7325
|
timeline;
|
|
7444
|
-
targetPosition = new
|
|
7326
|
+
targetPosition = new Vector35();
|
|
7445
7327
|
/** Advances by `delta` seconds, turns the eyes, and returns the blink weight. */
|
|
7446
7328
|
update(delta, frame) {
|
|
7447
7329
|
if (Number.isFinite(delta) && delta > 0) {
|
|
@@ -7629,6 +7511,7 @@ var PerformanceWeightMixer = class {
|
|
|
7629
7511
|
};
|
|
7630
7512
|
|
|
7631
7513
|
// ../../clients/web-client/src/character/facial-performance-mixer.ts
|
|
7514
|
+
var VisemeResponseRate = 30;
|
|
7632
7515
|
var DEFAULT_PLAY_DURATION_MS = 1500;
|
|
7633
7516
|
var MOUTH_CHANNELS = [
|
|
7634
7517
|
VRMExpressionPresetName.Aa,
|
|
@@ -7656,7 +7539,8 @@ var FacialPerformanceLayer = {
|
|
|
7656
7539
|
Emotion: "emotion",
|
|
7657
7540
|
Viseme: "viseme",
|
|
7658
7541
|
Blink: "blink",
|
|
7659
|
-
Gaze: "gaze"
|
|
7542
|
+
Gaze: "gaze",
|
|
7543
|
+
Overlay: "overlay"
|
|
7660
7544
|
};
|
|
7661
7545
|
var getFacialPerformanceLayerId = function(channel) {
|
|
7662
7546
|
const normalized = channel.toLowerCase();
|
|
@@ -7807,6 +7691,22 @@ var FacialPerformanceMixer = class {
|
|
|
7807
7691
|
}
|
|
7808
7692
|
this.releaseVisemeMouthMode();
|
|
7809
7693
|
}
|
|
7694
|
+
/** Blend temporary emotions without changing timeline targets or mouth ownership. */
|
|
7695
|
+
setOverlay(weights) {
|
|
7696
|
+
this.clearOverlay();
|
|
7697
|
+
Object.entries(weights).forEach(([channel, weight]) => {
|
|
7698
|
+
if (this.availableChannelSet.has(channel) && getFacialPerformanceLayerId(channel) === FacialPerformanceLayer.Emotion && Number.isFinite(weight)) {
|
|
7699
|
+
this.weightMixer.setTarget(
|
|
7700
|
+
FacialPerformanceLayer.Overlay,
|
|
7701
|
+
channel,
|
|
7702
|
+
weight
|
|
7703
|
+
);
|
|
7704
|
+
}
|
|
7705
|
+
});
|
|
7706
|
+
}
|
|
7707
|
+
clearOverlay() {
|
|
7708
|
+
this.weightMixer.clearLayerTargets(FacialPerformanceLayer.Overlay);
|
|
7709
|
+
}
|
|
7810
7710
|
releaseVisemeMouthMode() {
|
|
7811
7711
|
if (this.activeMouthMode !== "viseme") {
|
|
7812
7712
|
return;
|
|
@@ -7924,7 +7824,9 @@ var FacialPerformanceMixer = class {
|
|
|
7924
7824
|
this.weightMixer.registerLayer({
|
|
7925
7825
|
id: FacialPerformanceLayer.Viseme,
|
|
7926
7826
|
priority: 200,
|
|
7927
|
-
blendMode: "override"
|
|
7827
|
+
blendMode: "override",
|
|
7828
|
+
// A 33 ms time constant: an 80 ms lip closure reaches 90% closed.
|
|
7829
|
+
responseRate: VisemeResponseRate
|
|
7928
7830
|
});
|
|
7929
7831
|
this.weightMixer.registerLayer({
|
|
7930
7832
|
id: FacialPerformanceLayer.Blink,
|
|
@@ -7936,6 +7838,11 @@ var FacialPerformanceMixer = class {
|
|
|
7936
7838
|
priority: 400,
|
|
7937
7839
|
blendMode: "override"
|
|
7938
7840
|
});
|
|
7841
|
+
this.weightMixer.registerLayer({
|
|
7842
|
+
id: FacialPerformanceLayer.Overlay,
|
|
7843
|
+
priority: 500,
|
|
7844
|
+
blendMode: "maximum"
|
|
7845
|
+
});
|
|
7939
7846
|
}
|
|
7940
7847
|
updateIdleFace(delta, values) {
|
|
7941
7848
|
const manager = this.vrm?.expressionManager;
|
|
@@ -8060,10 +7967,11 @@ var FacialPerformanceMixer = class {
|
|
|
8060
7967
|
|
|
8061
7968
|
// ../../clients/web-client/src/character/mesh.ts
|
|
8062
7969
|
import {
|
|
8063
|
-
Euler,
|
|
7970
|
+
Euler as Euler2,
|
|
8064
7971
|
Mesh,
|
|
7972
|
+
Object3D,
|
|
8065
7973
|
SkinnedMesh,
|
|
8066
|
-
Vector3 as
|
|
7974
|
+
Vector3 as Vector36
|
|
8067
7975
|
} from "three";
|
|
8068
7976
|
var isSkinnedMesh = function(object) {
|
|
8069
7977
|
return object instanceof SkinnedMesh || object.isSkinnedMesh === true;
|
|
@@ -8072,19 +7980,30 @@ var isMesh = function(object) {
|
|
|
8072
7980
|
return isSkinnedMesh(object) || object instanceof Mesh || object.isMesh === true;
|
|
8073
7981
|
};
|
|
8074
7982
|
var applyVector3 = function(target, value) {
|
|
8075
|
-
if (value instanceof
|
|
7983
|
+
if (value instanceof Vector36) {
|
|
8076
7984
|
target.copy(value);
|
|
8077
7985
|
return;
|
|
8078
7986
|
}
|
|
8079
7987
|
target.set(value[0], value[1], value[2]);
|
|
8080
7988
|
};
|
|
8081
7989
|
var applyEuler = function(target, value) {
|
|
8082
|
-
if (value instanceof
|
|
7990
|
+
if (value instanceof Euler2) {
|
|
8083
7991
|
target.copy(value);
|
|
8084
7992
|
return;
|
|
8085
7993
|
}
|
|
8086
7994
|
target.set(value[0], value[1], value[2]);
|
|
8087
7995
|
};
|
|
7996
|
+
var isObject3D = function(value) {
|
|
7997
|
+
return value instanceof Object3D || typeof value === "object" && value !== null && value.isObject3D === true;
|
|
7998
|
+
};
|
|
7999
|
+
var finiteCoordinate = function(value, axis) {
|
|
8000
|
+
if (typeof value !== "number" || !Number.isFinite(value)) {
|
|
8001
|
+
throw new RangeError(
|
|
8002
|
+
`lookAt needs a finite ${axis} coordinate, not ${String(value)}`
|
|
8003
|
+
);
|
|
8004
|
+
}
|
|
8005
|
+
return value;
|
|
8006
|
+
};
|
|
8088
8007
|
var CharacterSceneRig = class {
|
|
8089
8008
|
currentVrm = null;
|
|
8090
8009
|
cachedMeshes = [];
|
|
@@ -8093,10 +8012,14 @@ var CharacterSceneRig = class {
|
|
|
8093
8012
|
cachedSkeletons = [];
|
|
8094
8013
|
shadowOptions = null;
|
|
8095
8014
|
transformOptions = null;
|
|
8015
|
+
lookAtAnchor = null;
|
|
8016
|
+
/** The target that setVrmLookAtTarget set on the current model. */
|
|
8017
|
+
vrmLookAtTarget = null;
|
|
8096
8018
|
get vrm() {
|
|
8097
8019
|
return this.currentVrm;
|
|
8098
8020
|
}
|
|
8099
8021
|
setVrm(vrm) {
|
|
8022
|
+
this.setVrmLookAtTarget(null);
|
|
8100
8023
|
this.currentVrm = vrm;
|
|
8101
8024
|
this.refreshMeshes();
|
|
8102
8025
|
if (this.shadowOptions) {
|
|
@@ -8106,6 +8029,64 @@ var CharacterSceneRig = class {
|
|
|
8106
8029
|
this.applyTransform(this.transformOptions);
|
|
8107
8030
|
}
|
|
8108
8031
|
}
|
|
8032
|
+
/**
|
|
8033
|
+
* The `Object3D` of a `lookAt()` argument: an `Object3D` as it is, a point
|
|
8034
|
+
* or three coordinates as a world position that this rig keeps, or null.
|
|
8035
|
+
* Coordinates that are not finite numbers throw a `RangeError` and change
|
|
8036
|
+
* nothing.
|
|
8037
|
+
*/
|
|
8038
|
+
resolveLookAtTarget(targetOrX, y, z) {
|
|
8039
|
+
if (targetOrX === null) {
|
|
8040
|
+
return null;
|
|
8041
|
+
}
|
|
8042
|
+
if (typeof targetOrX === "number") {
|
|
8043
|
+
return this.updateLookAtAnchor(targetOrX, y, z);
|
|
8044
|
+
}
|
|
8045
|
+
if (isObject3D(targetOrX)) {
|
|
8046
|
+
return targetOrX;
|
|
8047
|
+
}
|
|
8048
|
+
if (Array.isArray(targetOrX)) {
|
|
8049
|
+
return this.updateLookAtAnchor(targetOrX[0], targetOrX[1], targetOrX[2]);
|
|
8050
|
+
}
|
|
8051
|
+
if (typeof targetOrX === "object") {
|
|
8052
|
+
const point = targetOrX;
|
|
8053
|
+
return this.updateLookAtAnchor(point.x, point.y, point.z);
|
|
8054
|
+
}
|
|
8055
|
+
throw new TypeError(
|
|
8056
|
+
"lookAt needs an Object3D, a point, [x, y, z] or three coordinates"
|
|
8057
|
+
);
|
|
8058
|
+
}
|
|
8059
|
+
/**
|
|
8060
|
+
* Sets three-vrm's own look-at target (`vrm.lookAt.target`) of the current
|
|
8061
|
+
* model. Null removes the target that this method set; a target that other
|
|
8062
|
+
* code set stays. The IdleFace of a template face does not move the eyes
|
|
8063
|
+
* while `vrm.lookAt.target` is set.
|
|
8064
|
+
*/
|
|
8065
|
+
setVrmLookAtTarget(target) {
|
|
8066
|
+
const lookAt = this.currentVrm?.lookAt;
|
|
8067
|
+
if (lookAt) {
|
|
8068
|
+
if (target) {
|
|
8069
|
+
lookAt.target = target;
|
|
8070
|
+
} else if (this.vrmLookAtTarget && lookAt.target === this.vrmLookAtTarget) {
|
|
8071
|
+
lookAt.target = void 0;
|
|
8072
|
+
}
|
|
8073
|
+
}
|
|
8074
|
+
this.vrmLookAtTarget = lookAt ? target : null;
|
|
8075
|
+
}
|
|
8076
|
+
updateLookAtAnchor(x, y, z) {
|
|
8077
|
+
const position = [
|
|
8078
|
+
finiteCoordinate(x, "x"),
|
|
8079
|
+
finiteCoordinate(y, "y"),
|
|
8080
|
+
finiteCoordinate(z, "z")
|
|
8081
|
+
];
|
|
8082
|
+
if (!this.lookAtAnchor) {
|
|
8083
|
+
this.lookAtAnchor = new Object3D();
|
|
8084
|
+
this.lookAtAnchor.name = "npcify-look-at-target";
|
|
8085
|
+
}
|
|
8086
|
+
this.lookAtAnchor.position.set(...position);
|
|
8087
|
+
this.lookAtAnchor.updateMatrixWorld(true);
|
|
8088
|
+
return this.lookAtAnchor;
|
|
8089
|
+
}
|
|
8109
8090
|
/** Root Three.js group containing the VRM hierarchy (`vrm.scene`). */
|
|
8110
8091
|
get scene() {
|
|
8111
8092
|
return this.currentVrm?.scene ?? null;
|
|
@@ -8296,6 +8277,11 @@ var IncrementalStaging = class {
|
|
|
8296
8277
|
const dialogue = timeline.dialogue.map((next, index) => {
|
|
8297
8278
|
const previous = this.timeline.dialogue[index];
|
|
8298
8279
|
if (previous?.type === "speech" && previous.pending && next.type === "speech") {
|
|
8280
|
+
for (const key of Object.keys(previous)) {
|
|
8281
|
+
if (!(key in next)) {
|
|
8282
|
+
Reflect.deleteProperty(previous, key);
|
|
8283
|
+
}
|
|
8284
|
+
}
|
|
8299
8285
|
return Object.assign(previous, next, { pending: next.pending });
|
|
8300
8286
|
}
|
|
8301
8287
|
return next;
|
|
@@ -8358,6 +8344,30 @@ var IncrementalStaging = class {
|
|
|
8358
8344
|
}
|
|
8359
8345
|
};
|
|
8360
8346
|
|
|
8347
|
+
// ../../clients/web-client/src/character/playback-clock.ts
|
|
8348
|
+
var PlaybackClock = class {
|
|
8349
|
+
pausedAt = null;
|
|
8350
|
+
pausedSeconds = 0;
|
|
8351
|
+
wallTime() {
|
|
8352
|
+
return performance.now() / 1e3;
|
|
8353
|
+
}
|
|
8354
|
+
get paused() {
|
|
8355
|
+
return this.pausedAt !== null;
|
|
8356
|
+
}
|
|
8357
|
+
now() {
|
|
8358
|
+
return (this.pausedAt ?? this.wallTime()) - this.pausedSeconds;
|
|
8359
|
+
}
|
|
8360
|
+
pause() {
|
|
8361
|
+
this.pausedAt ??= this.wallTime();
|
|
8362
|
+
}
|
|
8363
|
+
resume() {
|
|
8364
|
+
if (this.pausedAt !== null) {
|
|
8365
|
+
this.pausedSeconds += this.wallTime() - this.pausedAt;
|
|
8366
|
+
this.pausedAt = null;
|
|
8367
|
+
}
|
|
8368
|
+
}
|
|
8369
|
+
};
|
|
8370
|
+
|
|
8361
8371
|
// ../../clients/web-client/src/character/staging-character-cues.ts
|
|
8362
8372
|
var findCharacterSequence = function(characters, sequence, fromIndex) {
|
|
8363
8373
|
if (sequence.length === 0) {
|
|
@@ -8459,12 +8469,6 @@ var buildStagingCharacterCues = function(text, wordTimings, duration, characterT
|
|
|
8459
8469
|
};
|
|
8460
8470
|
|
|
8461
8471
|
// ../../clients/web-client/src/character/staging-player.ts
|
|
8462
|
-
var nowSeconds = function() {
|
|
8463
|
-
if (typeof performance !== "undefined") {
|
|
8464
|
-
return performance.now() / 1e3;
|
|
8465
|
-
}
|
|
8466
|
-
return Date.now() / 1e3;
|
|
8467
|
-
};
|
|
8468
8472
|
var abortError2 = function() {
|
|
8469
8473
|
const error = new Error("Staging playback was cancelled");
|
|
8470
8474
|
error.name = "AbortError";
|
|
@@ -8496,6 +8500,8 @@ var StagingPlayer = class {
|
|
|
8496
8500
|
this.facialMixer = facialMixer;
|
|
8497
8501
|
this.animationController = animationController;
|
|
8498
8502
|
}
|
|
8503
|
+
clock = new PlaybackClock();
|
|
8504
|
+
pauseRevision = 0;
|
|
8499
8505
|
playbackController = null;
|
|
8500
8506
|
generation = 0;
|
|
8501
8507
|
incremental = null;
|
|
@@ -8505,6 +8511,40 @@ var StagingPlayer = class {
|
|
|
8505
8511
|
activeExpression = null;
|
|
8506
8512
|
activeExpressionMouthMode = null;
|
|
8507
8513
|
activeAnimation = null;
|
|
8514
|
+
get paused() {
|
|
8515
|
+
return this.clock.paused;
|
|
8516
|
+
}
|
|
8517
|
+
pause() {
|
|
8518
|
+
this.pauseRevision += 1;
|
|
8519
|
+
this.clock.pause();
|
|
8520
|
+
this.audioPlayer.element?.pause();
|
|
8521
|
+
}
|
|
8522
|
+
async resume() {
|
|
8523
|
+
if (!this.paused) {
|
|
8524
|
+
return;
|
|
8525
|
+
}
|
|
8526
|
+
const generation = this.generation;
|
|
8527
|
+
const pauseRevision = this.pauseRevision;
|
|
8528
|
+
if (this.pendingSpeech?.started && this.pendingSpeech.audio.type !== "silent") {
|
|
8529
|
+
await this.audioPlayer.element?.play();
|
|
8530
|
+
}
|
|
8531
|
+
if (generation !== this.generation) {
|
|
8532
|
+
return;
|
|
8533
|
+
}
|
|
8534
|
+
if (pauseRevision !== this.pauseRevision) {
|
|
8535
|
+
this.audioPlayer.element?.pause();
|
|
8536
|
+
return;
|
|
8537
|
+
}
|
|
8538
|
+
this.clock.resume();
|
|
8539
|
+
}
|
|
8540
|
+
async waitUntilResumed(signal) {
|
|
8541
|
+
while (this.paused) {
|
|
8542
|
+
await waitFrame(signal);
|
|
8543
|
+
}
|
|
8544
|
+
if (signal.aborted) {
|
|
8545
|
+
throw abortError2();
|
|
8546
|
+
}
|
|
8547
|
+
}
|
|
8508
8548
|
async play(timeline, options) {
|
|
8509
8549
|
this.cancel(false);
|
|
8510
8550
|
const controller = new AbortController();
|
|
@@ -8543,6 +8583,7 @@ var StagingPlayer = class {
|
|
|
8543
8583
|
if (controller.signal.aborted) {
|
|
8544
8584
|
throw abortError2();
|
|
8545
8585
|
}
|
|
8586
|
+
await this.waitUntilResumed(controller.signal);
|
|
8546
8587
|
if (block.startTime > cursor) {
|
|
8547
8588
|
beginTimeline();
|
|
8548
8589
|
await this.playTimedRange(
|
|
@@ -8626,12 +8667,12 @@ var StagingPlayer = class {
|
|
|
8626
8667
|
}
|
|
8627
8668
|
}
|
|
8628
8669
|
startCursor(block, mediaClock) {
|
|
8629
|
-
const startedAt =
|
|
8670
|
+
const startedAt = this.clock.now();
|
|
8630
8671
|
this.cursor = () => ({
|
|
8631
8672
|
blockId: block.id,
|
|
8632
8673
|
offsetSeconds: Math.min(
|
|
8633
8674
|
block.endTime - block.startTime,
|
|
8634
|
-
Math.max(0, mediaClock ? mediaClock() :
|
|
8675
|
+
Math.max(0, mediaClock ? mediaClock() : this.clock.now() - startedAt)
|
|
8635
8676
|
)
|
|
8636
8677
|
});
|
|
8637
8678
|
this.onCursor?.(this.cursor());
|
|
@@ -8655,6 +8696,7 @@ var StagingPlayer = class {
|
|
|
8655
8696
|
this.generation += 1;
|
|
8656
8697
|
this.playbackController?.abort();
|
|
8657
8698
|
this.playbackController = null;
|
|
8699
|
+
this.clock.resume();
|
|
8658
8700
|
this.incremental = null;
|
|
8659
8701
|
this.clearCursor();
|
|
8660
8702
|
this.onCursor = void 0;
|
|
@@ -8766,6 +8808,9 @@ var StagingPlayer = class {
|
|
|
8766
8808
|
return;
|
|
8767
8809
|
}
|
|
8768
8810
|
started = true;
|
|
8811
|
+
if (this.paused) {
|
|
8812
|
+
this.audioPlayer.element?.pause();
|
|
8813
|
+
}
|
|
8769
8814
|
if (this.pendingSpeech?.id === speech.id) {
|
|
8770
8815
|
this.pendingSpeech.started = true;
|
|
8771
8816
|
}
|
|
@@ -8786,6 +8831,7 @@ var StagingPlayer = class {
|
|
|
8786
8831
|
let ticking = true;
|
|
8787
8832
|
const tick = async () => {
|
|
8788
8833
|
while (ticking && !signal.aborted) {
|
|
8834
|
+
await this.waitUntilResumed(signal);
|
|
8789
8835
|
if (started) {
|
|
8790
8836
|
observePhonemes();
|
|
8791
8837
|
const localTime = Math.min(
|
|
@@ -8824,14 +8870,15 @@ var StagingPlayer = class {
|
|
|
8824
8870
|
}
|
|
8825
8871
|
}
|
|
8826
8872
|
async playSilentSpeech(speech, timeline, signal, revealCharacters) {
|
|
8827
|
-
const startedAt =
|
|
8873
|
+
const startedAt = this.clock.now();
|
|
8828
8874
|
const duration = Math.max(speech.endTime - speech.startTime, 0);
|
|
8829
8875
|
let nextVisemeIndex = 0;
|
|
8830
8876
|
while (true) {
|
|
8831
8877
|
if (signal.aborted) {
|
|
8832
8878
|
throw abortError2();
|
|
8833
8879
|
}
|
|
8834
|
-
|
|
8880
|
+
await this.waitUntilResumed(signal);
|
|
8881
|
+
const elapsed = Math.min(this.clock.now() - startedAt, duration);
|
|
8835
8882
|
this.applyTracks(speech.startTime + elapsed, true, timeline);
|
|
8836
8883
|
revealCharacters(elapsed);
|
|
8837
8884
|
let viseme = speech.visemes[nextVisemeIndex];
|
|
@@ -8851,13 +8898,14 @@ var StagingPlayer = class {
|
|
|
8851
8898
|
revealCharacters(Number.POSITIVE_INFINITY);
|
|
8852
8899
|
}
|
|
8853
8900
|
async playTimedRange(startTime, endTime, speechActive, timeline, signal) {
|
|
8854
|
-
const startedAt =
|
|
8901
|
+
const startedAt = this.clock.now();
|
|
8855
8902
|
const duration = Math.max(endTime - startTime, 0);
|
|
8856
8903
|
while (true) {
|
|
8857
8904
|
if (signal.aborted) {
|
|
8858
8905
|
throw abortError2();
|
|
8859
8906
|
}
|
|
8860
|
-
|
|
8907
|
+
await this.waitUntilResumed(signal);
|
|
8908
|
+
const elapsed = Math.min(this.clock.now() - startedAt, duration);
|
|
8861
8909
|
this.applyTracks(startTime + elapsed, speechActive, timeline);
|
|
8862
8910
|
if (elapsed >= duration) {
|
|
8863
8911
|
return;
|
|
@@ -8886,8 +8934,9 @@ var StagingPlayer = class {
|
|
|
8886
8934
|
if (animation?.animation === this.activeAnimation?.animation && animation?.startTime === this.activeAnimation?.startTime && animation?.loop === this.activeAnimation?.loop) {
|
|
8887
8935
|
return;
|
|
8888
8936
|
}
|
|
8937
|
+
const continuesIdle = !this.activeAnimation && animation?.startTime === 0 && animation?.animation === this.animationController.getIdleAnimationId?.();
|
|
8889
8938
|
this.activeAnimation = animation ? { ...animation } : null;
|
|
8890
|
-
if (animation) {
|
|
8939
|
+
if (animation && !continuesIdle) {
|
|
8891
8940
|
this.animationController.playAnimation(
|
|
8892
8941
|
animation.animation,
|
|
8893
8942
|
0.25,
|
|
@@ -9338,6 +9387,8 @@ var CharacterRuntime = class {
|
|
|
9338
9387
|
streamPlayback;
|
|
9339
9388
|
defaultParent;
|
|
9340
9389
|
currentGazeTarget = null;
|
|
9390
|
+
/** The current gaze target came from `lookAt()`. */
|
|
9391
|
+
lookAtActive = false;
|
|
9341
9392
|
thinkingState = false;
|
|
9342
9393
|
errorCallback;
|
|
9343
9394
|
constructor(options) {
|
|
@@ -9354,13 +9405,10 @@ var CharacterRuntime = class {
|
|
|
9354
9405
|
animationId,
|
|
9355
9406
|
assetProfile: this.assetProfile
|
|
9356
9407
|
}));
|
|
9357
|
-
this.lazyAnimations = new LazyAnimations(
|
|
9358
|
-
|
|
9359
|
-
(id, animation
|
|
9360
|
-
|
|
9361
|
-
this.animationController?.addAnimation?.(id, animation, { loop: false });
|
|
9362
|
-
}
|
|
9363
|
-
);
|
|
9408
|
+
this.lazyAnimations = new LazyAnimations(loadAnimation, (id, animation) => {
|
|
9409
|
+
this.animations.set(id, animation);
|
|
9410
|
+
this.animationController?.addAnimation?.(id, animation, { loop: false });
|
|
9411
|
+
});
|
|
9364
9412
|
this.performanceRecorder = new PerformanceRecorder({
|
|
9365
9413
|
characterId: this.characterId
|
|
9366
9414
|
});
|
|
@@ -9427,6 +9475,7 @@ var CharacterRuntime = class {
|
|
|
9427
9475
|
set vrm(value) {
|
|
9428
9476
|
this.sceneRig.setVrm(value);
|
|
9429
9477
|
this.facialMixer.setVrm(value);
|
|
9478
|
+
this.applyGaze();
|
|
9430
9479
|
if (value && this.defaultParent && !value.scene?.parent) {
|
|
9431
9480
|
this.sceneRig.attachTo(this.defaultParent);
|
|
9432
9481
|
}
|
|
@@ -9579,6 +9628,20 @@ var CharacterRuntime = class {
|
|
|
9579
9628
|
});
|
|
9580
9629
|
return unlocked;
|
|
9581
9630
|
}
|
|
9631
|
+
/** Freeze the current NPCP timeline, audio, face, and animation. */
|
|
9632
|
+
pausePlayback() {
|
|
9633
|
+
this.stagingPlayer?.pause();
|
|
9634
|
+
}
|
|
9635
|
+
/** Continue the same timeline. Call from a gesture if the browser requires it. */
|
|
9636
|
+
async resumePlayback() {
|
|
9637
|
+
await this.stagingPlayer?.resume();
|
|
9638
|
+
}
|
|
9639
|
+
/** Cancel current output and pending audio/animation loads. */
|
|
9640
|
+
interruptPlayback() {
|
|
9641
|
+
if (this.animationController) {
|
|
9642
|
+
this.streamPlayback.cleanup();
|
|
9643
|
+
}
|
|
9644
|
+
}
|
|
9582
9645
|
getExpressions() {
|
|
9583
9646
|
return this.facialMixer.getChannelNames();
|
|
9584
9647
|
}
|
|
@@ -9603,10 +9666,38 @@ var CharacterRuntime = class {
|
|
|
9603
9666
|
clearEmotion(mouthMode) {
|
|
9604
9667
|
this.facialMixer.clearEmotion(mouthMode);
|
|
9605
9668
|
}
|
|
9606
|
-
/**
|
|
9669
|
+
/**
|
|
9670
|
+
* The object the eyes of a template face follow (the camera); null looks
|
|
9671
|
+
* ahead. It replaces a `lookAt()` target. Other faces do not follow it.
|
|
9672
|
+
*/
|
|
9607
9673
|
setGazeTarget(target) {
|
|
9608
9674
|
this.currentGazeTarget = target;
|
|
9609
|
-
this.
|
|
9675
|
+
this.lookAtActive = false;
|
|
9676
|
+
this.applyGaze();
|
|
9677
|
+
}
|
|
9678
|
+
lookAt(targetOrX, y, z) {
|
|
9679
|
+
const target = this.sceneRig.resolveLookAtTarget(targetOrX, y, z);
|
|
9680
|
+
this.currentGazeTarget = target;
|
|
9681
|
+
this.lookAtActive = target !== null;
|
|
9682
|
+
this.applyGaze();
|
|
9683
|
+
}
|
|
9684
|
+
/** Reset the character's gaze target so the eyes look straight ahead. */
|
|
9685
|
+
clearLookAt() {
|
|
9686
|
+
this.setGazeTarget(null);
|
|
9687
|
+
}
|
|
9688
|
+
/** Briefly look at an object; release restores the prior target and gaze mode. */
|
|
9689
|
+
lookAtTemporarily(target) {
|
|
9690
|
+
const previous = this.currentGazeTarget;
|
|
9691
|
+
const wasActive = this.lookAtActive;
|
|
9692
|
+
this.lookAt(target);
|
|
9693
|
+
return () => {
|
|
9694
|
+
if (this.currentGazeTarget !== target || !this.lookAtActive) {
|
|
9695
|
+
return;
|
|
9696
|
+
}
|
|
9697
|
+
this.currentGazeTarget = previous;
|
|
9698
|
+
this.lookAtActive = wasActive;
|
|
9699
|
+
this.applyGaze();
|
|
9700
|
+
};
|
|
9610
9701
|
}
|
|
9611
9702
|
startPerformanceRecording(options) {
|
|
9612
9703
|
this.performanceRecorder.start(options);
|
|
@@ -9656,6 +9747,9 @@ var CharacterRuntime = class {
|
|
|
9656
9747
|
}
|
|
9657
9748
|
/** Advance skeletal animations, lip-sync visemes, facial expressions, and VRM physics by `delta` seconds. */
|
|
9658
9749
|
update(delta) {
|
|
9750
|
+
if (this.stagingPlayer?.paused) {
|
|
9751
|
+
return;
|
|
9752
|
+
}
|
|
9659
9753
|
this.animationController?.update(delta);
|
|
9660
9754
|
this.visemePlayer.update(delta);
|
|
9661
9755
|
this.facialMixer.update(delta);
|
|
@@ -9688,15 +9782,24 @@ var CharacterRuntime = class {
|
|
|
9688
9782
|
}
|
|
9689
9783
|
return this.stagingPlayer;
|
|
9690
9784
|
};
|
|
9785
|
+
/**
|
|
9786
|
+
* A template face follows the gaze target with its IdleFace. Only a
|
|
9787
|
+
* `lookAt()` target moves the eyes of another face, through three-vrm's
|
|
9788
|
+
* `vrm.lookAt.target`; on a template face that target would stop the
|
|
9789
|
+
* IdleFace (its gaze cone, idle eye moves and look cues).
|
|
9790
|
+
*/
|
|
9791
|
+
applyGaze() {
|
|
9792
|
+
this.facialMixer.setGazeTarget(this.currentGazeTarget);
|
|
9793
|
+
this.sceneRig.setVrmLookAtTarget(
|
|
9794
|
+
this.lookAtActive && !this.facialMixer.hasIdleFace ? this.currentGazeTarget : null
|
|
9795
|
+
);
|
|
9796
|
+
}
|
|
9691
9797
|
resetRuntimeAssets() {
|
|
9692
9798
|
this.lazyAnimations.reset();
|
|
9693
|
-
|
|
9694
|
-
|
|
9695
|
-
|
|
9696
|
-
|
|
9697
|
-
this.audioPlayer?.reset();
|
|
9698
|
-
this.facialMixer.reset();
|
|
9699
|
-
}
|
|
9799
|
+
this.stagingPlayer?.cleanup();
|
|
9800
|
+
this.stagingPlayer = null;
|
|
9801
|
+
this.audioPlayer?.reset();
|
|
9802
|
+
this.facialMixer.reset();
|
|
9700
9803
|
if (this.animationController instanceof AnimationController) {
|
|
9701
9804
|
this.animationController.clearAnimations();
|
|
9702
9805
|
}
|
|
@@ -9706,20 +9809,10 @@ var CharacterRuntime = class {
|
|
|
9706
9809
|
this.facialMixer.setVrm(null);
|
|
9707
9810
|
}
|
|
9708
9811
|
disposeRuntimeAssets() {
|
|
9709
|
-
this.
|
|
9710
|
-
|
|
9711
|
-
this.stagingPlayer.cleanup();
|
|
9712
|
-
this.stagingPlayer = null;
|
|
9713
|
-
}
|
|
9812
|
+
this.sceneRig.disposeVrm();
|
|
9813
|
+
this.resetRuntimeAssets();
|
|
9714
9814
|
this.audioPlayer?.cleanup();
|
|
9715
9815
|
this.audioPlayer = null;
|
|
9716
|
-
if (this.animationController instanceof AnimationController) {
|
|
9717
|
-
this.animationController.clearAnimations();
|
|
9718
|
-
}
|
|
9719
|
-
this.animationController = void 0;
|
|
9720
|
-
this.animations.clear();
|
|
9721
|
-
this.facialMixer.setVrm(null);
|
|
9722
|
-
this.sceneRig.disposeVrm();
|
|
9723
9816
|
}
|
|
9724
9817
|
handleCallbackError = (context2, error) => {
|
|
9725
9818
|
const normalized = error instanceof Error ? error : new Error(String(error ?? "Error"));
|
|
@@ -9746,6 +9839,18 @@ var ConversationSocket = class {
|
|
|
9746
9839
|
this.socket?.disconnect();
|
|
9747
9840
|
this.socket = null;
|
|
9748
9841
|
}
|
|
9842
|
+
/** An event with an acknowledgement. Needs `ensureConnected` first. */
|
|
9843
|
+
async request(event, payload, timeoutMs = 1e4) {
|
|
9844
|
+
const socket = this.socket;
|
|
9845
|
+
if (!socket?.connected) {
|
|
9846
|
+
throw new Error("The conversation connection is not open.");
|
|
9847
|
+
}
|
|
9848
|
+
return await socket.timeout(timeoutMs).emitWithAck(event, payload);
|
|
9849
|
+
}
|
|
9850
|
+
/** An event without an acknowledgement, in order with the other events. */
|
|
9851
|
+
send(event, payload) {
|
|
9852
|
+
this.socket?.emit(event, payload);
|
|
9853
|
+
}
|
|
9749
9854
|
async ensureConnected(conversationId) {
|
|
9750
9855
|
if (typeof window === "undefined") {
|
|
9751
9856
|
return;
|
|
@@ -9800,6 +9905,41 @@ var ConversationSocket = class {
|
|
|
9800
9905
|
}
|
|
9801
9906
|
this.options.handleStreamEvent(event);
|
|
9802
9907
|
});
|
|
9908
|
+
this.socket.on(SPEECH_EVENTS.partial, (payload) => {
|
|
9909
|
+
const event = payload;
|
|
9910
|
+
if (typeof event?.sessionId === "string" && typeof event.text === "string") {
|
|
9911
|
+
this.options.handleSpeechEvent?.({ type: "partial", event });
|
|
9912
|
+
}
|
|
9913
|
+
});
|
|
9914
|
+
this.socket.on(SPEECH_EVENTS.turn, (payload) => {
|
|
9915
|
+
let candidate;
|
|
9916
|
+
try {
|
|
9917
|
+
candidate = decodePerformance(
|
|
9918
|
+
payload,
|
|
9919
|
+
"turn"
|
|
9920
|
+
);
|
|
9921
|
+
} catch (error) {
|
|
9922
|
+
this.options.handleCallbackError("speech.turn.decode", error);
|
|
9923
|
+
return;
|
|
9924
|
+
}
|
|
9925
|
+
if (candidate?.conversationId !== this.options.getConversationId() || typeof candidate.requestId !== "string" || !candidate.turn || typeof candidate.turn !== "object") {
|
|
9926
|
+
return;
|
|
9927
|
+
}
|
|
9928
|
+
this.options.handleSpeechEvent?.({
|
|
9929
|
+
type: "turn",
|
|
9930
|
+
requestId: candidate.requestId,
|
|
9931
|
+
turn: candidate.turn
|
|
9932
|
+
});
|
|
9933
|
+
});
|
|
9934
|
+
this.socket.on(SPEECH_EVENTS.turnFailed, (payload) => {
|
|
9935
|
+
const event = payload;
|
|
9936
|
+
if (typeof event?.requestId === "string" && event.conversationId === this.options.getConversationId()) {
|
|
9937
|
+
this.options.handleSpeechEvent?.({ type: "turnFailed", event });
|
|
9938
|
+
}
|
|
9939
|
+
});
|
|
9940
|
+
this.socket.on("disconnect", () => {
|
|
9941
|
+
this.options.handleSpeechEvent?.({ type: "disconnect" });
|
|
9942
|
+
});
|
|
9803
9943
|
this.socket.on("conversations:turn", (payload) => {
|
|
9804
9944
|
let candidate;
|
|
9805
9945
|
try {
|
|
@@ -9852,56 +9992,1056 @@ var ConversationSocket = class {
|
|
|
9852
9992
|
}
|
|
9853
9993
|
};
|
|
9854
9994
|
|
|
9855
|
-
// ../../clients/web-client/src/
|
|
9856
|
-
var
|
|
9857
|
-
|
|
9858
|
-
|
|
9859
|
-
|
|
9860
|
-
|
|
9861
|
-
|
|
9862
|
-
|
|
9863
|
-
|
|
9864
|
-
|
|
9865
|
-
|
|
9866
|
-
|
|
9867
|
-
|
|
9868
|
-
|
|
9869
|
-
|
|
9870
|
-
|
|
9871
|
-
|
|
9872
|
-
|
|
9873
|
-
|
|
9874
|
-
|
|
9875
|
-
|
|
9876
|
-
|
|
9877
|
-
|
|
9878
|
-
|
|
9879
|
-
|
|
9880
|
-
|
|
9881
|
-
assetProfile: resolvedProfile
|
|
9882
|
-
})
|
|
9883
|
-
});
|
|
9884
|
-
this.authoring = options.authoring ?? false;
|
|
9885
|
-
this.bakedAsset = options.bakedConversation;
|
|
9886
|
-
this.initialAssetInfo = options.initialAssetInfo ?? null;
|
|
9887
|
-
this.conversationSocket = new ConversationSocket({
|
|
9888
|
-
baseUrl: this.baseUrl,
|
|
9889
|
-
apiToken: this.apiToken,
|
|
9890
|
-
getConversationId: () => this.conversationId,
|
|
9891
|
-
handleStreamEvent: (event) => {
|
|
9892
|
-
if (event.type !== "text_delta") {
|
|
9893
|
-
this.thinkingState = false;
|
|
9995
|
+
// ../../clients/web-client/src/voice/push-to-talk.ts
|
|
9996
|
+
var INTERACTIVE = 'input, textarea, select, button, a[href], summary, [contenteditable]:not([contenteditable="false"]), [role="button"], [role="textbox"], [role="checkbox"], [role="switch"], [role="menuitem"], [role="tab"], [role="slider"], [role="option"]';
|
|
9997
|
+
var elementOf = function(target) {
|
|
9998
|
+
return target && typeof target.closest === "function" ? target : null;
|
|
9999
|
+
};
|
|
10000
|
+
var bindPushToTalk = function(target, options = {}) {
|
|
10001
|
+
const element = options.element ?? null;
|
|
10002
|
+
const key = options.key === void 0 ? "Space" : options.key;
|
|
10003
|
+
const minHoldMs = options.minHoldMs ?? SPEECH_MIN_HOLD_MS;
|
|
10004
|
+
let active = null;
|
|
10005
|
+
let pointerId = null;
|
|
10006
|
+
let pressedAt = 0;
|
|
10007
|
+
const begin = function(source) {
|
|
10008
|
+
if (active || target.listeningState !== "idle") {
|
|
10009
|
+
return false;
|
|
10010
|
+
}
|
|
10011
|
+
if (options.enabled && !options.enabled()) {
|
|
10012
|
+
return false;
|
|
10013
|
+
}
|
|
10014
|
+
active = source;
|
|
10015
|
+
pressedAt = performance.now();
|
|
10016
|
+
const listening = typeof options.listening === "function" ? options.listening() : options.listening ?? {};
|
|
10017
|
+
target.startListening({ ...listening, endOnSilenceMs: null }).then(
|
|
10018
|
+
(result) => {
|
|
10019
|
+
if (result) {
|
|
10020
|
+
options.onResult?.(result);
|
|
9894
10021
|
}
|
|
9895
|
-
this.streamPlayback.handleEvent(event);
|
|
9896
|
-
},
|
|
9897
|
-
handleTurn: async (turn) => {
|
|
9898
|
-
await this.handleStagingResult(turn, { source: "turn" });
|
|
9899
10022
|
},
|
|
9900
|
-
|
|
9901
|
-
|
|
9902
|
-
|
|
9903
|
-
|
|
9904
|
-
|
|
10023
|
+
(error) => {
|
|
10024
|
+
active = null;
|
|
10025
|
+
options.onError?.(error);
|
|
10026
|
+
}
|
|
10027
|
+
);
|
|
10028
|
+
return true;
|
|
10029
|
+
};
|
|
10030
|
+
const end = function() {
|
|
10031
|
+
if (!active) {
|
|
10032
|
+
return;
|
|
10033
|
+
}
|
|
10034
|
+
active = null;
|
|
10035
|
+
pointerId = null;
|
|
10036
|
+
if (performance.now() - pressedAt < minHoldMs) {
|
|
10037
|
+
target.cancelListening();
|
|
10038
|
+
options.onShortPress?.();
|
|
10039
|
+
return;
|
|
10040
|
+
}
|
|
10041
|
+
const starting = target.listeningState === "starting";
|
|
10042
|
+
target.stopListening().then(
|
|
10043
|
+
(result) => {
|
|
10044
|
+
if (starting && result === null) {
|
|
10045
|
+
options.onShortPress?.();
|
|
10046
|
+
}
|
|
10047
|
+
},
|
|
10048
|
+
() => void 0
|
|
10049
|
+
);
|
|
10050
|
+
};
|
|
10051
|
+
const cancel = function() {
|
|
10052
|
+
if (!active) {
|
|
10053
|
+
return;
|
|
10054
|
+
}
|
|
10055
|
+
active = null;
|
|
10056
|
+
pointerId = null;
|
|
10057
|
+
target.cancelListening();
|
|
10058
|
+
};
|
|
10059
|
+
const isOwnElement = function(node) {
|
|
10060
|
+
return Boolean(element && node && element.contains(node));
|
|
10061
|
+
};
|
|
10062
|
+
const onKeyDown = function(event) {
|
|
10063
|
+
if (event.key === "Escape" || event.code === "Escape") {
|
|
10064
|
+
cancel();
|
|
10065
|
+
return;
|
|
10066
|
+
}
|
|
10067
|
+
const node = elementOf(event.target);
|
|
10068
|
+
const ownElement = isOwnElement(node);
|
|
10069
|
+
const pressesElement = ownElement && (event.code === "Space" || event.code === "Enter");
|
|
10070
|
+
if (!pressesElement && (!key || event.code !== key)) {
|
|
10071
|
+
return;
|
|
10072
|
+
}
|
|
10073
|
+
if (event.repeat) {
|
|
10074
|
+
if (active === "key") {
|
|
10075
|
+
event.preventDefault();
|
|
10076
|
+
}
|
|
10077
|
+
return;
|
|
10078
|
+
}
|
|
10079
|
+
if (event.ctrlKey || event.metaKey || event.altKey || event.isComposing) {
|
|
10080
|
+
return;
|
|
10081
|
+
}
|
|
10082
|
+
if (!ownElement && node?.closest(INTERACTIVE)) {
|
|
10083
|
+
return;
|
|
10084
|
+
}
|
|
10085
|
+
if (begin("key")) {
|
|
10086
|
+
event.preventDefault();
|
|
10087
|
+
}
|
|
10088
|
+
};
|
|
10089
|
+
const onKeyUp = function(event) {
|
|
10090
|
+
if (active !== "key") {
|
|
10091
|
+
return;
|
|
10092
|
+
}
|
|
10093
|
+
const ownElement = isOwnElement(elementOf(event.target));
|
|
10094
|
+
if (event.code === key || ownElement && (event.code === "Space" || event.code === "Enter")) {
|
|
10095
|
+
event.preventDefault();
|
|
10096
|
+
end();
|
|
10097
|
+
}
|
|
10098
|
+
};
|
|
10099
|
+
const onVisibility = function() {
|
|
10100
|
+
if (document.visibilityState === "hidden") {
|
|
10101
|
+
cancel();
|
|
10102
|
+
}
|
|
10103
|
+
};
|
|
10104
|
+
const onPointerDown = function(event) {
|
|
10105
|
+
if (event.pointerType === "mouse" && event.button !== 0) {
|
|
10106
|
+
return;
|
|
10107
|
+
}
|
|
10108
|
+
event.preventDefault();
|
|
10109
|
+
if (!begin("pointer")) {
|
|
10110
|
+
return;
|
|
10111
|
+
}
|
|
10112
|
+
pointerId = event.pointerId;
|
|
10113
|
+
try {
|
|
10114
|
+
element?.setPointerCapture(event.pointerId);
|
|
10115
|
+
} catch {
|
|
10116
|
+
}
|
|
10117
|
+
};
|
|
10118
|
+
const onPointerUp = function(event) {
|
|
10119
|
+
if (active === "pointer" && event.pointerId === pointerId) {
|
|
10120
|
+
end();
|
|
10121
|
+
}
|
|
10122
|
+
};
|
|
10123
|
+
const onPointerCancel = function(event) {
|
|
10124
|
+
if (active === "pointer" && event.pointerId === pointerId) {
|
|
10125
|
+
cancel();
|
|
10126
|
+
}
|
|
10127
|
+
};
|
|
10128
|
+
const prevent = function(event) {
|
|
10129
|
+
event.preventDefault();
|
|
10130
|
+
};
|
|
10131
|
+
const restoreStyle = [];
|
|
10132
|
+
if (element) {
|
|
10133
|
+
for (const [property, value] of [
|
|
10134
|
+
["touch-action", "none"],
|
|
10135
|
+
["user-select", "none"],
|
|
10136
|
+
["-webkit-user-select", "none"],
|
|
10137
|
+
["-webkit-touch-callout", "none"]
|
|
10138
|
+
]) {
|
|
10139
|
+
const previous = element.style.getPropertyValue(property);
|
|
10140
|
+
element.style.setProperty(property, value);
|
|
10141
|
+
restoreStyle.push(() => {
|
|
10142
|
+
if (previous) {
|
|
10143
|
+
element.style.setProperty(property, previous);
|
|
10144
|
+
} else {
|
|
10145
|
+
element.style.removeProperty(property);
|
|
10146
|
+
}
|
|
10147
|
+
});
|
|
10148
|
+
}
|
|
10149
|
+
element.addEventListener("pointerdown", onPointerDown);
|
|
10150
|
+
window.addEventListener("pointerup", onPointerUp, true);
|
|
10151
|
+
window.addEventListener("pointercancel", onPointerCancel, true);
|
|
10152
|
+
element.addEventListener("contextmenu", prevent);
|
|
10153
|
+
element.addEventListener("selectstart", prevent);
|
|
10154
|
+
element.addEventListener("touchstart", prevent, { passive: false });
|
|
10155
|
+
}
|
|
10156
|
+
window.addEventListener("keydown", onKeyDown);
|
|
10157
|
+
window.addEventListener("keyup", onKeyUp);
|
|
10158
|
+
window.addEventListener("blur", cancel);
|
|
10159
|
+
document.addEventListener("visibilitychange", onVisibility);
|
|
10160
|
+
return () => {
|
|
10161
|
+
cancel();
|
|
10162
|
+
if (element) {
|
|
10163
|
+
element.removeEventListener("pointerdown", onPointerDown);
|
|
10164
|
+
window.removeEventListener("pointerup", onPointerUp, true);
|
|
10165
|
+
window.removeEventListener("pointercancel", onPointerCancel, true);
|
|
10166
|
+
element.removeEventListener("contextmenu", prevent);
|
|
10167
|
+
element.removeEventListener("selectstart", prevent);
|
|
10168
|
+
element.removeEventListener("touchstart", prevent);
|
|
10169
|
+
restoreStyle.forEach((restore) => restore());
|
|
10170
|
+
}
|
|
10171
|
+
window.removeEventListener("keydown", onKeyDown);
|
|
10172
|
+
window.removeEventListener("keyup", onKeyUp);
|
|
10173
|
+
window.removeEventListener("blur", cancel);
|
|
10174
|
+
document.removeEventListener("visibilitychange", onVisibility);
|
|
10175
|
+
};
|
|
10176
|
+
};
|
|
10177
|
+
|
|
10178
|
+
// ../../clients/web-client/src/voice/microphone-capture.ts
|
|
10179
|
+
var FRAME_SAMPLES = SPEECH_AUDIO_FORMAT.sampleRate * SPEECH_FRAME_MS / 1e3;
|
|
10180
|
+
var MicrophoneError = class extends Error {
|
|
10181
|
+
constructor(code, message) {
|
|
10182
|
+
super(message);
|
|
10183
|
+
this.code = code;
|
|
10184
|
+
this.name = "NpcifyMicrophoneError";
|
|
10185
|
+
}
|
|
10186
|
+
};
|
|
10187
|
+
var Pcm16Framer = class {
|
|
10188
|
+
constructor(inputRate, emit) {
|
|
10189
|
+
this.emit = emit;
|
|
10190
|
+
this.ratio = inputRate / SPEECH_AUDIO_FORMAT.sampleRate;
|
|
10191
|
+
}
|
|
10192
|
+
ratio;
|
|
10193
|
+
frame = new Int16Array(FRAME_SAMPLES);
|
|
10194
|
+
length = 0;
|
|
10195
|
+
sum = 0;
|
|
10196
|
+
count = 0;
|
|
10197
|
+
phase = 0;
|
|
10198
|
+
push(samples) {
|
|
10199
|
+
for (let index = 0; index < samples.length; index += 1) {
|
|
10200
|
+
this.sum += samples[index] ?? 0;
|
|
10201
|
+
this.count += 1;
|
|
10202
|
+
this.phase += 1;
|
|
10203
|
+
while (this.phase >= this.ratio) {
|
|
10204
|
+
this.phase -= this.ratio;
|
|
10205
|
+
this.write(this.sum / Math.max(1, this.count));
|
|
10206
|
+
if (this.ratio >= 1) {
|
|
10207
|
+
this.sum = 0;
|
|
10208
|
+
this.count = 0;
|
|
10209
|
+
}
|
|
10210
|
+
}
|
|
10211
|
+
if (this.ratio < 1) {
|
|
10212
|
+
this.sum = 0;
|
|
10213
|
+
this.count = 0;
|
|
10214
|
+
}
|
|
10215
|
+
}
|
|
10216
|
+
}
|
|
10217
|
+
flush() {
|
|
10218
|
+
if (this.count > 0 && this.ratio >= 1) {
|
|
10219
|
+
this.write(this.sum / this.count);
|
|
10220
|
+
this.sum = 0;
|
|
10221
|
+
this.count = 0;
|
|
10222
|
+
}
|
|
10223
|
+
if (this.length > 0) {
|
|
10224
|
+
this.emit(this.frame.slice(0, this.length));
|
|
10225
|
+
this.length = 0;
|
|
10226
|
+
}
|
|
10227
|
+
}
|
|
10228
|
+
write(value) {
|
|
10229
|
+
const clamped = Math.max(-1, Math.min(1, value));
|
|
10230
|
+
this.frame[this.length] = Math.round(
|
|
10231
|
+
clamped < 0 ? clamped * 32768 : clamped * 32767
|
|
10232
|
+
);
|
|
10233
|
+
this.length += 1;
|
|
10234
|
+
if (this.length === FRAME_SAMPLES) {
|
|
10235
|
+
this.emit(this.frame);
|
|
10236
|
+
this.frame = new Int16Array(FRAME_SAMPLES);
|
|
10237
|
+
this.length = 0;
|
|
10238
|
+
}
|
|
10239
|
+
}
|
|
10240
|
+
};
|
|
10241
|
+
var levelOf = function(pcm) {
|
|
10242
|
+
if (pcm.length === 0) {
|
|
10243
|
+
return 0;
|
|
10244
|
+
}
|
|
10245
|
+
let sum = 0;
|
|
10246
|
+
for (const sample of pcm) {
|
|
10247
|
+
sum += (sample / 32768) ** 2;
|
|
10248
|
+
}
|
|
10249
|
+
return Math.sqrt(sum / pcm.length);
|
|
10250
|
+
};
|
|
10251
|
+
var WORKLET_SOURCE = `
|
|
10252
|
+
const FRAME = ${FRAME_SAMPLES}
|
|
10253
|
+
class NpcifyCaptureProcessor extends AudioWorkletProcessor {
|
|
10254
|
+
constructor() {
|
|
10255
|
+
super()
|
|
10256
|
+
this.ratio = sampleRate / ${SPEECH_AUDIO_FORMAT.sampleRate}
|
|
10257
|
+
this.frame = new Int16Array(FRAME)
|
|
10258
|
+
this.length = 0
|
|
10259
|
+
this.sum = 0
|
|
10260
|
+
this.count = 0
|
|
10261
|
+
this.phase = 0
|
|
10262
|
+
this.port.onmessage = (event) => {
|
|
10263
|
+
if (event.data === 'flush') {
|
|
10264
|
+
if (this.count > 0 && this.ratio >= 1) {
|
|
10265
|
+
this.write(this.sum / this.count)
|
|
10266
|
+
this.sum = 0
|
|
10267
|
+
this.count = 0
|
|
10268
|
+
}
|
|
10269
|
+
if (this.length > 0) {
|
|
10270
|
+
this.post(this.frame.slice(0, this.length))
|
|
10271
|
+
this.length = 0
|
|
10272
|
+
}
|
|
10273
|
+
this.port.postMessage({ type: 'flushed' })
|
|
10274
|
+
}
|
|
10275
|
+
}
|
|
10276
|
+
}
|
|
10277
|
+
write(value) {
|
|
10278
|
+
const clamped = Math.max(-1, Math.min(1, value))
|
|
10279
|
+
this.frame[this.length++] = Math.round(clamped < 0 ? clamped * 32768 : clamped * 32767)
|
|
10280
|
+
if (this.length === FRAME) {
|
|
10281
|
+
this.post(this.frame)
|
|
10282
|
+
this.frame = new Int16Array(FRAME)
|
|
10283
|
+
this.length = 0
|
|
10284
|
+
}
|
|
10285
|
+
}
|
|
10286
|
+
post(pcm) {
|
|
10287
|
+
this.port.postMessage({ type: 'frame', pcm }, [pcm.buffer])
|
|
10288
|
+
}
|
|
10289
|
+
process(inputs) {
|
|
10290
|
+
const channels = inputs[0]
|
|
10291
|
+
if (!channels || channels.length === 0) return true
|
|
10292
|
+
const size = channels[0].length
|
|
10293
|
+
for (let index = 0; index < size; index += 1) {
|
|
10294
|
+
let value = 0
|
|
10295
|
+
for (const channel of channels) value += channel[index]
|
|
10296
|
+
this.sum += value / channels.length
|
|
10297
|
+
this.count += 1
|
|
10298
|
+
this.phase += 1
|
|
10299
|
+
while (this.phase >= this.ratio) {
|
|
10300
|
+
this.phase -= this.ratio
|
|
10301
|
+
this.write(this.sum / Math.max(1, this.count))
|
|
10302
|
+
if (this.ratio >= 1) {
|
|
10303
|
+
this.sum = 0
|
|
10304
|
+
this.count = 0
|
|
10305
|
+
}
|
|
10306
|
+
}
|
|
10307
|
+
if (this.ratio < 1) {
|
|
10308
|
+
this.sum = 0
|
|
10309
|
+
this.count = 0
|
|
10310
|
+
}
|
|
10311
|
+
}
|
|
10312
|
+
return true
|
|
10313
|
+
}
|
|
10314
|
+
}
|
|
10315
|
+
registerProcessor('npcify-capture', NpcifyCaptureProcessor)
|
|
10316
|
+
`;
|
|
10317
|
+
var MicrophoneCapture = class {
|
|
10318
|
+
context = null;
|
|
10319
|
+
workletReady = null;
|
|
10320
|
+
graph = null;
|
|
10321
|
+
generation = 0;
|
|
10322
|
+
async start(onFrame) {
|
|
10323
|
+
this.cancel();
|
|
10324
|
+
const generation = ++this.generation;
|
|
10325
|
+
const AudioContextClass = globalThis.AudioContext ?? globalThis.webkitAudioContext;
|
|
10326
|
+
if (!navigator.mediaDevices?.getUserMedia || !AudioContextClass) {
|
|
10327
|
+
throw new MicrophoneError(
|
|
10328
|
+
"MICROPHONE_UNAVAILABLE",
|
|
10329
|
+
"This browser cannot record audio."
|
|
10330
|
+
);
|
|
10331
|
+
}
|
|
10332
|
+
this.context ??= new AudioContextClass({ latencyHint: "interactive" });
|
|
10333
|
+
const context2 = this.context;
|
|
10334
|
+
const resumed = context2.resume().catch(() => void 0);
|
|
10335
|
+
let stream;
|
|
10336
|
+
try {
|
|
10337
|
+
stream = await navigator.mediaDevices.getUserMedia({
|
|
10338
|
+
audio: {
|
|
10339
|
+
channelCount: 1,
|
|
10340
|
+
echoCancellation: true,
|
|
10341
|
+
noiseSuppression: true,
|
|
10342
|
+
autoGainControl: true
|
|
10343
|
+
}
|
|
10344
|
+
});
|
|
10345
|
+
} catch (error) {
|
|
10346
|
+
const name = error?.name;
|
|
10347
|
+
throw name === "NotAllowedError" || name === "SecurityError" ? new MicrophoneError(
|
|
10348
|
+
"MICROPHONE_PERMISSION_DENIED",
|
|
10349
|
+
"Microphone access was denied."
|
|
10350
|
+
) : new MicrophoneError(
|
|
10351
|
+
"MICROPHONE_UNAVAILABLE",
|
|
10352
|
+
"No microphone is available."
|
|
10353
|
+
);
|
|
10354
|
+
}
|
|
10355
|
+
await resumed;
|
|
10356
|
+
if (generation !== this.generation) {
|
|
10357
|
+
stream.getTracks().forEach((track) => track.stop());
|
|
10358
|
+
return;
|
|
10359
|
+
}
|
|
10360
|
+
const deliver = (pcm) => onFrame({ pcm, level: levelOf(pcm) });
|
|
10361
|
+
const source = context2.createMediaStreamSource(stream);
|
|
10362
|
+
const sink = context2.createGain();
|
|
10363
|
+
sink.gain.value = 0;
|
|
10364
|
+
sink.connect(context2.destination);
|
|
10365
|
+
const graph = await this.workletGraph(context2, stream, source, sink, deliver) ?? this.scriptGraph(context2, stream, source, sink, deliver);
|
|
10366
|
+
if (generation !== this.generation) {
|
|
10367
|
+
this.close(graph, false);
|
|
10368
|
+
return;
|
|
10369
|
+
}
|
|
10370
|
+
this.graph = graph;
|
|
10371
|
+
}
|
|
10372
|
+
async stop() {
|
|
10373
|
+
const graph = this.graph;
|
|
10374
|
+
this.graph = null;
|
|
10375
|
+
this.generation += 1;
|
|
10376
|
+
if (!graph) {
|
|
10377
|
+
return;
|
|
10378
|
+
}
|
|
10379
|
+
try {
|
|
10380
|
+
await graph.flush();
|
|
10381
|
+
} finally {
|
|
10382
|
+
this.close(graph);
|
|
10383
|
+
}
|
|
10384
|
+
}
|
|
10385
|
+
cancel() {
|
|
10386
|
+
const graph = this.graph;
|
|
10387
|
+
this.graph = null;
|
|
10388
|
+
this.generation += 1;
|
|
10389
|
+
if (graph) {
|
|
10390
|
+
this.close(graph);
|
|
10391
|
+
}
|
|
10392
|
+
}
|
|
10393
|
+
close(graph, suspend = true) {
|
|
10394
|
+
graph.stream.getTracks().forEach((track) => track.stop());
|
|
10395
|
+
graph.source.disconnect();
|
|
10396
|
+
graph.node.disconnect();
|
|
10397
|
+
graph.sink.disconnect();
|
|
10398
|
+
if (graph.node instanceof AudioWorkletNode) {
|
|
10399
|
+
graph.node.port.onmessage = null;
|
|
10400
|
+
} else {
|
|
10401
|
+
graph.node.onaudioprocess = null;
|
|
10402
|
+
}
|
|
10403
|
+
if (suspend) {
|
|
10404
|
+
void this.context?.suspend().catch(() => void 0);
|
|
10405
|
+
}
|
|
10406
|
+
}
|
|
10407
|
+
async workletGraph(context2, stream, source, sink, deliver) {
|
|
10408
|
+
if (!context2.audioWorklet || typeof AudioWorkletNode === "undefined") {
|
|
10409
|
+
return null;
|
|
10410
|
+
}
|
|
10411
|
+
this.workletReady ??= (async () => {
|
|
10412
|
+
const url = URL.createObjectURL(
|
|
10413
|
+
new Blob([WORKLET_SOURCE], { type: "application/javascript" })
|
|
10414
|
+
);
|
|
10415
|
+
try {
|
|
10416
|
+
await context2.audioWorklet.addModule(url);
|
|
10417
|
+
return true;
|
|
10418
|
+
} catch {
|
|
10419
|
+
return false;
|
|
10420
|
+
} finally {
|
|
10421
|
+
URL.revokeObjectURL(url);
|
|
10422
|
+
}
|
|
10423
|
+
})();
|
|
10424
|
+
if (!await this.workletReady) {
|
|
10425
|
+
return null;
|
|
10426
|
+
}
|
|
10427
|
+
const node = new AudioWorkletNode(context2, "npcify-capture", {
|
|
10428
|
+
numberOfInputs: 1,
|
|
10429
|
+
numberOfOutputs: 1,
|
|
10430
|
+
channelCount: 1,
|
|
10431
|
+
channelCountMode: "explicit"
|
|
10432
|
+
});
|
|
10433
|
+
let flushed = null;
|
|
10434
|
+
node.port.onmessage = (event) => {
|
|
10435
|
+
if (event.data.type === "frame" && event.data.pcm) {
|
|
10436
|
+
deliver(event.data.pcm);
|
|
10437
|
+
} else if (event.data.type === "flushed") {
|
|
10438
|
+
flushed?.();
|
|
10439
|
+
}
|
|
10440
|
+
};
|
|
10441
|
+
source.connect(node);
|
|
10442
|
+
node.connect(sink);
|
|
10443
|
+
return {
|
|
10444
|
+
stream,
|
|
10445
|
+
source,
|
|
10446
|
+
node,
|
|
10447
|
+
sink,
|
|
10448
|
+
flush: () => new Promise((resolve) => {
|
|
10449
|
+
const timer = setTimeout(resolve, 250);
|
|
10450
|
+
flushed = () => {
|
|
10451
|
+
clearTimeout(timer);
|
|
10452
|
+
resolve();
|
|
10453
|
+
};
|
|
10454
|
+
node.port.postMessage("flush");
|
|
10455
|
+
})
|
|
10456
|
+
};
|
|
10457
|
+
}
|
|
10458
|
+
/** The fallback without AudioWorklet: a ScriptProcessorNode on the main thread. */
|
|
10459
|
+
scriptGraph(context2, stream, source, sink, deliver) {
|
|
10460
|
+
const framer = new Pcm16Framer(context2.sampleRate, deliver);
|
|
10461
|
+
const node = context2.createScriptProcessor(2048, 1, 1);
|
|
10462
|
+
node.onaudioprocess = (event) => {
|
|
10463
|
+
framer.push(event.inputBuffer.getChannelData(0));
|
|
10464
|
+
};
|
|
10465
|
+
source.connect(node);
|
|
10466
|
+
node.connect(sink);
|
|
10467
|
+
return {
|
|
10468
|
+
stream,
|
|
10469
|
+
source,
|
|
10470
|
+
node,
|
|
10471
|
+
sink,
|
|
10472
|
+
flush: () => {
|
|
10473
|
+
framer.flush();
|
|
10474
|
+
return Promise.resolve();
|
|
10475
|
+
}
|
|
10476
|
+
};
|
|
10477
|
+
}
|
|
10478
|
+
};
|
|
10479
|
+
|
|
10480
|
+
// ../../clients/web-client/src/voice/silence-detector.ts
|
|
10481
|
+
var MIN_SPEECH_LEVEL = 0.015;
|
|
10482
|
+
var SPEECH_TO_NOISE = 3;
|
|
10483
|
+
var SPEECH_START_MS = 200;
|
|
10484
|
+
var SilenceDetector = class {
|
|
10485
|
+
constructor(silenceMs) {
|
|
10486
|
+
this.silenceMs = silenceMs;
|
|
10487
|
+
}
|
|
10488
|
+
floor = null;
|
|
10489
|
+
loudMs = 0;
|
|
10490
|
+
quietMs = 0;
|
|
10491
|
+
speaking = false;
|
|
10492
|
+
/** True when the speech has ended. */
|
|
10493
|
+
push(level, frameMs = SPEECH_FRAME_MS) {
|
|
10494
|
+
const threshold = Math.max(
|
|
10495
|
+
MIN_SPEECH_LEVEL,
|
|
10496
|
+
(this.floor ?? MIN_SPEECH_LEVEL / SPEECH_TO_NOISE) * SPEECH_TO_NOISE
|
|
10497
|
+
);
|
|
10498
|
+
if (level >= threshold) {
|
|
10499
|
+
this.loudMs += frameMs;
|
|
10500
|
+
this.quietMs = 0;
|
|
10501
|
+
if (this.loudMs >= SPEECH_START_MS) {
|
|
10502
|
+
this.speaking = true;
|
|
10503
|
+
}
|
|
10504
|
+
return false;
|
|
10505
|
+
}
|
|
10506
|
+
this.floor = this.floor === null ? level : this.floor * 0.9 + level * 0.1;
|
|
10507
|
+
this.loudMs = 0;
|
|
10508
|
+
if (!this.speaking) {
|
|
10509
|
+
return false;
|
|
10510
|
+
}
|
|
10511
|
+
this.quietMs += frameMs;
|
|
10512
|
+
return this.quietMs >= this.silenceMs;
|
|
10513
|
+
}
|
|
10514
|
+
get heardSpeech() {
|
|
10515
|
+
return this.speaking;
|
|
10516
|
+
}
|
|
10517
|
+
};
|
|
10518
|
+
|
|
10519
|
+
// ../../clients/web-client/src/voice/voice-input.ts
|
|
10520
|
+
var SpeechInputError = class extends Error {
|
|
10521
|
+
constructor(code, message) {
|
|
10522
|
+
super(message ?? code);
|
|
10523
|
+
this.code = code;
|
|
10524
|
+
this.name = "NpcifySpeechInputError";
|
|
10525
|
+
}
|
|
10526
|
+
};
|
|
10527
|
+
var deferred = function() {
|
|
10528
|
+
let resolve;
|
|
10529
|
+
let reject;
|
|
10530
|
+
const promise = new Promise((done, fail) => {
|
|
10531
|
+
resolve = done;
|
|
10532
|
+
reject = fail;
|
|
10533
|
+
});
|
|
10534
|
+
promise.catch(() => void 0);
|
|
10535
|
+
return { promise, resolve, reject };
|
|
10536
|
+
};
|
|
10537
|
+
var CODE = /^[A-Z][A-Z0-9_]{2,63}$/u;
|
|
10538
|
+
var speechError = function(error, fallback) {
|
|
10539
|
+
if (error instanceof SpeechInputError) {
|
|
10540
|
+
return error;
|
|
10541
|
+
}
|
|
10542
|
+
if (error instanceof Error && "code" in error && typeof error.code === "string") {
|
|
10543
|
+
return new SpeechInputError(
|
|
10544
|
+
error.code,
|
|
10545
|
+
error.message
|
|
10546
|
+
);
|
|
10547
|
+
}
|
|
10548
|
+
return new SpeechInputError(
|
|
10549
|
+
fallback,
|
|
10550
|
+
error instanceof Error ? error.message : void 0
|
|
10551
|
+
);
|
|
10552
|
+
};
|
|
10553
|
+
var VoiceInput = class {
|
|
10554
|
+
constructor(deps) {
|
|
10555
|
+
this.deps = deps;
|
|
10556
|
+
}
|
|
10557
|
+
state = "idle";
|
|
10558
|
+
session = null;
|
|
10559
|
+
capture = null;
|
|
10560
|
+
/**
|
|
10561
|
+
* The session that the capture records for. A start that was cancelled
|
|
10562
|
+
* never touches the capture when a later session owns it.
|
|
10563
|
+
*/
|
|
10564
|
+
captureOwner = null;
|
|
10565
|
+
turns = /* @__PURE__ */ new Map();
|
|
10566
|
+
get listeningState() {
|
|
10567
|
+
return this.state;
|
|
10568
|
+
}
|
|
10569
|
+
/** A turn that the server started has not delivered its final payload. */
|
|
10570
|
+
get turnPending() {
|
|
10571
|
+
return this.turns.size > 0;
|
|
10572
|
+
}
|
|
10573
|
+
/**
|
|
10574
|
+
* Starts listening. Resolves with the result when the listening ends (by
|
|
10575
|
+
* `stop`, the silence rule or the time limit), or with null when it is
|
|
10576
|
+
* cancelled.
|
|
10577
|
+
*/
|
|
10578
|
+
async start(options = {}) {
|
|
10579
|
+
if (this.session) {
|
|
10580
|
+
throw new SpeechInputError(
|
|
10581
|
+
"SPEECH_ALREADY_LISTENING",
|
|
10582
|
+
"Listening has already started."
|
|
10583
|
+
);
|
|
10584
|
+
}
|
|
10585
|
+
this.deps.assertVoiceAllowed();
|
|
10586
|
+
if (this.turnPending || this.deps.turnInFlight()) {
|
|
10587
|
+
throw new SpeechInputError(
|
|
10588
|
+
"TURN_IN_PROGRESS",
|
|
10589
|
+
"A reply is already being generated. Please wait."
|
|
10590
|
+
);
|
|
10591
|
+
}
|
|
10592
|
+
const conversationId = this.deps.conversationId();
|
|
10593
|
+
if (!conversationId) {
|
|
10594
|
+
throw new SpeechInputError(
|
|
10595
|
+
"CONVERSATION_REQUIRED",
|
|
10596
|
+
"Create a conversation before listening."
|
|
10597
|
+
);
|
|
10598
|
+
}
|
|
10599
|
+
const maxDurationMs = Math.min(
|
|
10600
|
+
SPEECH_MAX_DURATION_MS,
|
|
10601
|
+
Math.max(1e3, options.maxDurationMs ?? SPEECH_MAX_DURATION_MS)
|
|
10602
|
+
);
|
|
10603
|
+
const session = {
|
|
10604
|
+
requestId: globalThis.crypto.randomUUID(),
|
|
10605
|
+
options,
|
|
10606
|
+
sessionId: null,
|
|
10607
|
+
sequence: 0,
|
|
10608
|
+
bytes: 0,
|
|
10609
|
+
pending: [],
|
|
10610
|
+
timer: null,
|
|
10611
|
+
silence: options.endOnSilenceMs && options.endOnSilenceMs > 0 ? new SilenceDetector(options.endOnSilenceMs) : null,
|
|
10612
|
+
stopRequested: false,
|
|
10613
|
+
stopping: null,
|
|
10614
|
+
captureClosing: null,
|
|
10615
|
+
captureClosed: false,
|
|
10616
|
+
disconnected: deferred(),
|
|
10617
|
+
done: deferred()
|
|
10618
|
+
};
|
|
10619
|
+
this.session = session;
|
|
10620
|
+
this.setState("starting");
|
|
10621
|
+
const socket = this.deps.socket();
|
|
10622
|
+
this.capture ??= (this.deps.createCapture ?? (() => new MicrophoneCapture()))();
|
|
10623
|
+
const capture = this.capture;
|
|
10624
|
+
this.captureOwner = session;
|
|
10625
|
+
const [acknowledged, captured] = await Promise.allSettled([
|
|
10626
|
+
socket.ensureConnected(conversationId).then(
|
|
10627
|
+
() => socket.request(SPEECH_EVENTS.start, {
|
|
10628
|
+
conversationId,
|
|
10629
|
+
requestId: session.requestId,
|
|
10630
|
+
parentStagingId: options.parentStagingId,
|
|
10631
|
+
language: options.language ?? null,
|
|
10632
|
+
send: options.send !== false,
|
|
10633
|
+
silent: options.silent === true,
|
|
10634
|
+
renderSessionId: this.deps.renderSessionId(),
|
|
10635
|
+
format: SPEECH_AUDIO_FORMAT
|
|
10636
|
+
})
|
|
10637
|
+
),
|
|
10638
|
+
capture.start((frame) => this.frame(session, frame))
|
|
10639
|
+
]);
|
|
10640
|
+
const ack = acknowledged.status === "fulfilled" ? acknowledged.value : null;
|
|
10641
|
+
const serverSession = ack?.ok ? ack.data.sessionId : null;
|
|
10642
|
+
if (session !== this.session) {
|
|
10643
|
+
this.dropCapture(session);
|
|
10644
|
+
if (serverSession) {
|
|
10645
|
+
socket.send(SPEECH_EVENTS.cancel, { sessionId: serverSession });
|
|
10646
|
+
}
|
|
10647
|
+
return session.done.promise;
|
|
10648
|
+
}
|
|
10649
|
+
let failure = null;
|
|
10650
|
+
if (captured.status === "rejected") {
|
|
10651
|
+
failure = speechError(captured.reason, "MICROPHONE_UNAVAILABLE");
|
|
10652
|
+
} else if (acknowledged.status === "rejected") {
|
|
10653
|
+
failure = speechError(acknowledged.reason, "SPEECH_UNAVAILABLE");
|
|
10654
|
+
} else if (!ack?.ok) {
|
|
10655
|
+
failure = new SpeechInputError(
|
|
10656
|
+
ack && !ack.ok ? ack.error : "SPEECH_UNAVAILABLE"
|
|
10657
|
+
);
|
|
10658
|
+
}
|
|
10659
|
+
if (failure) {
|
|
10660
|
+
this.dropCapture(session);
|
|
10661
|
+
if (serverSession) {
|
|
10662
|
+
socket.send(SPEECH_EVENTS.cancel, { sessionId: serverSession });
|
|
10663
|
+
}
|
|
10664
|
+
this.end(session);
|
|
10665
|
+
session.done.reject(failure);
|
|
10666
|
+
return session.done.promise;
|
|
10667
|
+
}
|
|
10668
|
+
session.sessionId = serverSession;
|
|
10669
|
+
for (const audio of session.pending.splice(0)) {
|
|
10670
|
+
this.sendFrame(session, audio);
|
|
10671
|
+
}
|
|
10672
|
+
if (session.stopRequested) {
|
|
10673
|
+
void this.stop().catch(() => void 0);
|
|
10674
|
+
return session.done.promise;
|
|
10675
|
+
}
|
|
10676
|
+
this.setState("listening");
|
|
10677
|
+
session.timer = setTimeout(() => {
|
|
10678
|
+
void this.stop().catch(() => void 0);
|
|
10679
|
+
}, maxDurationMs);
|
|
10680
|
+
return session.done.promise;
|
|
10681
|
+
}
|
|
10682
|
+
/**
|
|
10683
|
+
* Ends the recording and resolves with the result as soon as the text is
|
|
10684
|
+
* recognised. A stop before the microphone gave any audio (a permission
|
|
10685
|
+
* prompt, a slow device) cancels and resolves null.
|
|
10686
|
+
*/
|
|
10687
|
+
async stop() {
|
|
10688
|
+
const session = this.session;
|
|
10689
|
+
if (!session) {
|
|
10690
|
+
return null;
|
|
10691
|
+
}
|
|
10692
|
+
if (session.stopping) {
|
|
10693
|
+
return session.stopping;
|
|
10694
|
+
}
|
|
10695
|
+
if (!session.sessionId) {
|
|
10696
|
+
if (session.bytes === 0) {
|
|
10697
|
+
this.cancel();
|
|
10698
|
+
return null;
|
|
10699
|
+
}
|
|
10700
|
+
session.stopRequested = true;
|
|
10701
|
+
void this.closeCapture(session);
|
|
10702
|
+
return session.done.promise;
|
|
10703
|
+
}
|
|
10704
|
+
const stopping = deferred();
|
|
10705
|
+
session.stopping = stopping.promise;
|
|
10706
|
+
this.finish(session).then(stopping.resolve, stopping.reject);
|
|
10707
|
+
return session.stopping;
|
|
10708
|
+
}
|
|
10709
|
+
/** Discards the recording. Nothing is recognised or charged. */
|
|
10710
|
+
cancel() {
|
|
10711
|
+
const session = this.session;
|
|
10712
|
+
if (!session || session.stopping) {
|
|
10713
|
+
return;
|
|
10714
|
+
}
|
|
10715
|
+
this.dropCapture(session);
|
|
10716
|
+
if (session.sessionId) {
|
|
10717
|
+
this.deps.socket().send(SPEECH_EVENTS.cancel, { sessionId: session.sessionId });
|
|
10718
|
+
}
|
|
10719
|
+
this.end(session);
|
|
10720
|
+
session.done.resolve(null);
|
|
10721
|
+
}
|
|
10722
|
+
/** The conversation ended: no listening, and no turn waits any more. */
|
|
10723
|
+
reset() {
|
|
10724
|
+
this.cancel();
|
|
10725
|
+
for (const [requestId, turn] of this.turns) {
|
|
10726
|
+
clearTimeout(turn.timer);
|
|
10727
|
+
turn.done.reject(
|
|
10728
|
+
new SpeechInputError("CONVERSATION_ENDED", "The conversation ended.")
|
|
10729
|
+
);
|
|
10730
|
+
this.turns.delete(requestId);
|
|
10731
|
+
}
|
|
10732
|
+
}
|
|
10733
|
+
handleSocketEvent(event) {
|
|
10734
|
+
if (event.type === "partial") {
|
|
10735
|
+
if (this.session?.sessionId === event.event.sessionId) {
|
|
10736
|
+
this.deps.onTranscript({ text: event.event.text, final: false });
|
|
10737
|
+
}
|
|
10738
|
+
return;
|
|
10739
|
+
}
|
|
10740
|
+
if (event.type === "turn") {
|
|
10741
|
+
const turn = this.turns.get(event.requestId);
|
|
10742
|
+
if (!turn) {
|
|
10743
|
+
return;
|
|
10744
|
+
}
|
|
10745
|
+
this.turns.delete(event.requestId);
|
|
10746
|
+
clearTimeout(turn.timer);
|
|
10747
|
+
void this.deps.deliverTurn(event.turn, event.requestId, turn.options.silent === true).then(
|
|
10748
|
+
() => turn.done.resolve(event.turn),
|
|
10749
|
+
(error) => turn.done.reject(error)
|
|
10750
|
+
);
|
|
10751
|
+
return;
|
|
10752
|
+
}
|
|
10753
|
+
if (event.type === "turnFailed") {
|
|
10754
|
+
const turn = this.turns.get(event.event.requestId);
|
|
10755
|
+
if (!turn) {
|
|
10756
|
+
return;
|
|
10757
|
+
}
|
|
10758
|
+
this.turns.delete(event.event.requestId);
|
|
10759
|
+
clearTimeout(turn.timer);
|
|
10760
|
+
const reason = event.event.error;
|
|
10761
|
+
turn.done.reject(
|
|
10762
|
+
new SpeechInputError(CODE.test(reason) ? reason : "TURN_FAILED", reason)
|
|
10763
|
+
);
|
|
10764
|
+
return;
|
|
10765
|
+
}
|
|
10766
|
+
const session = this.session;
|
|
10767
|
+
if (session) {
|
|
10768
|
+
const failure = new SpeechInputError(
|
|
10769
|
+
"SPEECH_DISCONNECTED",
|
|
10770
|
+
session.stopping ? "The connection closed while the speech was recognised." : "The connection closed while listening."
|
|
10771
|
+
);
|
|
10772
|
+
session.disconnected.reject(failure);
|
|
10773
|
+
if (!session.stopping) {
|
|
10774
|
+
this.dropCapture(session);
|
|
10775
|
+
this.end(session);
|
|
10776
|
+
session.done.reject(failure);
|
|
10777
|
+
}
|
|
10778
|
+
}
|
|
10779
|
+
for (const requestId of [...this.turns.keys()]) {
|
|
10780
|
+
this.recover(requestId);
|
|
10781
|
+
}
|
|
10782
|
+
}
|
|
10783
|
+
frame(session, frame) {
|
|
10784
|
+
if (session !== this.session || session.captureClosed) {
|
|
10785
|
+
return;
|
|
10786
|
+
}
|
|
10787
|
+
const audio = new Uint8Array(
|
|
10788
|
+
frame.pcm.buffer,
|
|
10789
|
+
frame.pcm.byteOffset,
|
|
10790
|
+
frame.pcm.byteLength
|
|
10791
|
+
).slice();
|
|
10792
|
+
if (session.sessionId) {
|
|
10793
|
+
this.sendFrame(session, audio);
|
|
10794
|
+
} else if (session.bytes + audio.length <= SPEECH_MAX_AUDIO_BYTES) {
|
|
10795
|
+
session.bytes += audio.length;
|
|
10796
|
+
session.pending.push(audio);
|
|
10797
|
+
}
|
|
10798
|
+
if (!session.stopping && session.silence?.push(frame.level)) {
|
|
10799
|
+
void this.stop().catch(() => void 0);
|
|
10800
|
+
}
|
|
10801
|
+
}
|
|
10802
|
+
sendFrame(session, audio) {
|
|
10803
|
+
if (!session.sessionId) {
|
|
10804
|
+
return;
|
|
10805
|
+
}
|
|
10806
|
+
this.deps.socket().send(SPEECH_EVENTS.audio, {
|
|
10807
|
+
sessionId: session.sessionId,
|
|
10808
|
+
sequence: session.sequence,
|
|
10809
|
+
audio
|
|
10810
|
+
});
|
|
10811
|
+
session.sequence += 1;
|
|
10812
|
+
}
|
|
10813
|
+
async finish(session) {
|
|
10814
|
+
if (session.timer) {
|
|
10815
|
+
clearTimeout(session.timer);
|
|
10816
|
+
}
|
|
10817
|
+
this.setState("recognizing");
|
|
10818
|
+
try {
|
|
10819
|
+
await this.closeCapture(session);
|
|
10820
|
+
const ack = await Promise.race([
|
|
10821
|
+
this.deps.socket().request(
|
|
10822
|
+
SPEECH_EVENTS.stop,
|
|
10823
|
+
{ sessionId: session.sessionId },
|
|
10824
|
+
3e4
|
|
10825
|
+
),
|
|
10826
|
+
session.disconnected.promise
|
|
10827
|
+
]);
|
|
10828
|
+
if (!ack.ok) {
|
|
10829
|
+
throw new SpeechInputError(ack.error);
|
|
10830
|
+
}
|
|
10831
|
+
const result = this.result(session, ack.data);
|
|
10832
|
+
this.end(session);
|
|
10833
|
+
this.deps.onTranscript({ text: result.text, final: true });
|
|
10834
|
+
session.done.resolve(result);
|
|
10835
|
+
return result;
|
|
10836
|
+
} catch (error) {
|
|
10837
|
+
const failure = speechError(error, "SPEECH_RECOGNITION_FAILED");
|
|
10838
|
+
this.end(session);
|
|
10839
|
+
session.done.reject(failure);
|
|
10840
|
+
throw failure;
|
|
10841
|
+
}
|
|
10842
|
+
}
|
|
10843
|
+
result(session, transcript) {
|
|
10844
|
+
const text = transcript.text;
|
|
10845
|
+
const turn = transcript.sent ? this.track(session, text) : null;
|
|
10846
|
+
let sending = null;
|
|
10847
|
+
return {
|
|
10848
|
+
text,
|
|
10849
|
+
language: transcript.language,
|
|
10850
|
+
durationMs: transcript.durationMs,
|
|
10851
|
+
sent: transcript.sent,
|
|
10852
|
+
turn,
|
|
10853
|
+
send: () => {
|
|
10854
|
+
if (turn) {
|
|
10855
|
+
return turn;
|
|
10856
|
+
}
|
|
10857
|
+
if (!text) {
|
|
10858
|
+
return Promise.reject(
|
|
10859
|
+
new SpeechInputError("SPEECH_EMPTY", "No speech was recognised.")
|
|
10860
|
+
);
|
|
10861
|
+
}
|
|
10862
|
+
this.deps.assertVoiceAllowed();
|
|
10863
|
+
sending ??= this.deps.sendText(text, {
|
|
10864
|
+
requestId: session.requestId,
|
|
10865
|
+
parentStagingId: session.options.parentStagingId,
|
|
10866
|
+
silent: session.options.silent === true
|
|
10867
|
+
});
|
|
10868
|
+
return sending;
|
|
10869
|
+
}
|
|
10870
|
+
};
|
|
10871
|
+
}
|
|
10872
|
+
track(session, text) {
|
|
10873
|
+
const done = deferred();
|
|
10874
|
+
const timer = setTimeout(
|
|
10875
|
+
() => this.recover(session.requestId),
|
|
10876
|
+
this.deps.turnTimeoutMs ?? 12e4
|
|
10877
|
+
);
|
|
10878
|
+
this.turns.set(session.requestId, {
|
|
10879
|
+
text,
|
|
10880
|
+
options: session.options,
|
|
10881
|
+
timer,
|
|
10882
|
+
done
|
|
10883
|
+
});
|
|
10884
|
+
this.deps.beginTurn();
|
|
10885
|
+
return done.promise;
|
|
10886
|
+
}
|
|
10887
|
+
/** The idempotent HTTP request returns the turn that the server started. */
|
|
10888
|
+
recover(requestId) {
|
|
10889
|
+
const turn = this.turns.get(requestId);
|
|
10890
|
+
if (!turn) {
|
|
10891
|
+
return;
|
|
10892
|
+
}
|
|
10893
|
+
this.turns.delete(requestId);
|
|
10894
|
+
clearTimeout(turn.timer);
|
|
10895
|
+
this.deps.sendText(turn.text, {
|
|
10896
|
+
requestId,
|
|
10897
|
+
parentStagingId: turn.options.parentStagingId,
|
|
10898
|
+
silent: turn.options.silent === true
|
|
10899
|
+
}).then(turn.done.resolve, turn.done.reject);
|
|
10900
|
+
}
|
|
10901
|
+
/** Stops the capture of `session`; its last partial frame arrives first. */
|
|
10902
|
+
closeCapture(session) {
|
|
10903
|
+
session.captureClosing ??= (async () => {
|
|
10904
|
+
if (this.captureOwner === session) {
|
|
10905
|
+
this.captureOwner = null;
|
|
10906
|
+
await this.capture?.stop();
|
|
10907
|
+
}
|
|
10908
|
+
session.captureClosed = true;
|
|
10909
|
+
})();
|
|
10910
|
+
return session.captureClosing;
|
|
10911
|
+
}
|
|
10912
|
+
/** Ends the capture of `session` without its last frame, unless a later session owns it. */
|
|
10913
|
+
dropCapture(session) {
|
|
10914
|
+
if (this.captureOwner === session) {
|
|
10915
|
+
this.captureOwner = null;
|
|
10916
|
+
this.capture?.cancel();
|
|
10917
|
+
}
|
|
10918
|
+
}
|
|
10919
|
+
end(session) {
|
|
10920
|
+
if (session.timer) {
|
|
10921
|
+
clearTimeout(session.timer);
|
|
10922
|
+
session.timer = null;
|
|
10923
|
+
}
|
|
10924
|
+
if (this.session === session) {
|
|
10925
|
+
this.session = null;
|
|
10926
|
+
this.setState("idle");
|
|
10927
|
+
}
|
|
10928
|
+
}
|
|
10929
|
+
setState(state) {
|
|
10930
|
+
if (this.state !== state) {
|
|
10931
|
+
this.state = state;
|
|
10932
|
+
this.deps.onState(state);
|
|
10933
|
+
}
|
|
10934
|
+
}
|
|
10935
|
+
};
|
|
10936
|
+
|
|
10937
|
+
// ../../clients/web-client/src/npcify.ts
|
|
10938
|
+
var Character = class extends CharacterRuntime {
|
|
10939
|
+
conversationSocket;
|
|
10940
|
+
renderSessionId = null;
|
|
10941
|
+
initialAssetInfo;
|
|
10942
|
+
stagingResultCallback = null;
|
|
10943
|
+
thinkingCallback = null;
|
|
10944
|
+
notifiedStagingIds = /* @__PURE__ */ new Map();
|
|
10945
|
+
liveConversation;
|
|
10946
|
+
baked = null;
|
|
10947
|
+
authoring;
|
|
10948
|
+
normalizeIdleOrientation;
|
|
10949
|
+
bakedAsset;
|
|
10950
|
+
animationRefreshId = 0;
|
|
10951
|
+
voice;
|
|
10952
|
+
configuredInputModes;
|
|
10953
|
+
resolvedInputModes = null;
|
|
10954
|
+
latestOptions = [];
|
|
10955
|
+
listeningStateCallback = null;
|
|
10956
|
+
transcriptCallback = null;
|
|
10957
|
+
conversationId = null;
|
|
10958
|
+
constructor(options) {
|
|
10959
|
+
const resolvedToken = (options.apiToken ?? options.apiKey)?.trim() ?? "";
|
|
10960
|
+
const resolvedProfile = options.assetProfile ?? "original";
|
|
10961
|
+
super({
|
|
10962
|
+
...options,
|
|
10963
|
+
apiToken: resolvedToken,
|
|
10964
|
+
assetProfile: resolvedProfile,
|
|
10965
|
+
loadAnimation: (animationId) => loadVrma({
|
|
10966
|
+
baseUrl: options.baseUrl,
|
|
10967
|
+
characterId: options.characterId,
|
|
10968
|
+
apiToken: resolvedToken,
|
|
10969
|
+
animationId,
|
|
10970
|
+
assetProfile: resolvedProfile
|
|
10971
|
+
})
|
|
10972
|
+
});
|
|
10973
|
+
this.authoring = options.authoring ?? false;
|
|
10974
|
+
this.normalizeIdleOrientation = options.normalizeIdleOrientation ?? false;
|
|
10975
|
+
this.bakedAsset = options.bakedConversation;
|
|
10976
|
+
this.initialAssetInfo = options.initialAssetInfo ?? null;
|
|
10977
|
+
this.configuredInputModes = parseInputModes(options.inputModes);
|
|
10978
|
+
if (this.configuredInputModes && this.bakedAsset && !this.authoring) {
|
|
10979
|
+
resolveInputModes(this.configuredInputModes, {
|
|
10980
|
+
allowCustomReply: false,
|
|
10981
|
+
baked: true
|
|
10982
|
+
});
|
|
10983
|
+
}
|
|
10984
|
+
this.conversationSocket = new ConversationSocket({
|
|
10985
|
+
baseUrl: this.baseUrl,
|
|
10986
|
+
apiToken: this.apiToken,
|
|
10987
|
+
getConversationId: () => this.conversationId,
|
|
10988
|
+
handleStreamEvent: (event) => {
|
|
10989
|
+
if (event.type !== "text_delta") {
|
|
10990
|
+
this.thinkingState = false;
|
|
10991
|
+
}
|
|
10992
|
+
this.streamPlayback.handleEvent(event);
|
|
10993
|
+
},
|
|
10994
|
+
handleTurn: async (turn) => {
|
|
10995
|
+
await this.handleStagingResult(turn, { source: "turn" });
|
|
10996
|
+
},
|
|
10997
|
+
handleCallbackError: this.handleCallbackError,
|
|
10998
|
+
handleSpeechEvent: (event) => this.voice.handleSocketEvent(event)
|
|
10999
|
+
});
|
|
11000
|
+
this.voice = new VoiceInput({
|
|
11001
|
+
socket: () => this.conversationSocket,
|
|
11002
|
+
conversationId: () => this.conversationId,
|
|
11003
|
+
renderSessionId: () => this.renderSessionId,
|
|
11004
|
+
assertVoiceAllowed: () => this.assertInputMode("voice"),
|
|
11005
|
+
turnInFlight: () => this.liveConversation.inFlight,
|
|
11006
|
+
beginTurn: () => {
|
|
11007
|
+
if (this.conversationId) {
|
|
11008
|
+
this.telemetry.begin(this.conversationId);
|
|
11009
|
+
}
|
|
11010
|
+
this.invokeThinkingCallback({ taskId: null, stagingId: null });
|
|
11011
|
+
},
|
|
11012
|
+
deliverTurn: (payload, requestId, silent) => this.handleStagingResult(payload, {
|
|
11013
|
+
source: "api",
|
|
11014
|
+
forcePlayback: payload.taskId !== requestId,
|
|
11015
|
+
silent
|
|
11016
|
+
}),
|
|
11017
|
+
sendText: (text, turnOptions) => {
|
|
11018
|
+
if (!this.conversationId) {
|
|
11019
|
+
return Promise.reject(new Error("Conversation ID is required."));
|
|
11020
|
+
}
|
|
11021
|
+
return this.liveConversation.send(this.conversationId, text, turnOptions);
|
|
11022
|
+
},
|
|
11023
|
+
onState: (state) => {
|
|
11024
|
+
if (this.listeningStateCallback) {
|
|
11025
|
+
invokeSafeCallback(
|
|
11026
|
+
"onListeningState",
|
|
11027
|
+
this.handleCallbackError,
|
|
11028
|
+
() => this.listeningStateCallback?.(state)
|
|
11029
|
+
);
|
|
11030
|
+
}
|
|
11031
|
+
},
|
|
11032
|
+
onTranscript: (event) => {
|
|
11033
|
+
if (this.transcriptCallback) {
|
|
11034
|
+
invokeSafeCallback(
|
|
11035
|
+
"onTranscript",
|
|
11036
|
+
this.handleCallbackError,
|
|
11037
|
+
() => this.transcriptCallback?.(event)
|
|
11038
|
+
);
|
|
11039
|
+
}
|
|
11040
|
+
}
|
|
11041
|
+
});
|
|
11042
|
+
this.liveConversation = new LiveConversation({
|
|
11043
|
+
baseUrl: this.baseUrl,
|
|
11044
|
+
apiToken: this.apiToken,
|
|
9905
11045
|
telemetry: this.telemetry,
|
|
9906
11046
|
socket: () => this.conversationSocket,
|
|
9907
11047
|
renderSession: () => this.renderSessionId,
|
|
@@ -9924,7 +11064,12 @@ var Character = class extends CharacterRuntime {
|
|
|
9924
11064
|
this.resetAssets();
|
|
9925
11065
|
const progressReporter = createAssetProgressReporter(callback);
|
|
9926
11066
|
const info = await this.getAssetInfo();
|
|
9927
|
-
|
|
11067
|
+
const baked = !this.authoring && Boolean(this.bakedAsset || info.bakedConversationEnabled);
|
|
11068
|
+
this.resolvedInputModes = resolveInputModes(this.configuredInputModes, {
|
|
11069
|
+
allowCustomReply: info.allowCustomReply !== false,
|
|
11070
|
+
baked
|
|
11071
|
+
});
|
|
11072
|
+
this.baked = baked ? await loadBakedConversation({
|
|
9928
11073
|
baseUrl: this.baseUrl,
|
|
9929
11074
|
characterId: this.characterId,
|
|
9930
11075
|
apiToken: this.apiToken,
|
|
@@ -9949,7 +11094,8 @@ var Character = class extends CharacterRuntime {
|
|
|
9949
11094
|
}
|
|
9950
11095
|
const animationController = new AnimationController(
|
|
9951
11096
|
this.vrm,
|
|
9952
|
-
info.idleAnimationId ?? null
|
|
11097
|
+
info.idleAnimationId ?? null,
|
|
11098
|
+
{ normalizeIdleOrientation: this.normalizeIdleOrientation }
|
|
9953
11099
|
);
|
|
9954
11100
|
animationController.prepareAnimations = (ids) => this.lazyAnimations.ensureMany(ids);
|
|
9955
11101
|
this.animationController = animationController;
|
|
@@ -9992,6 +11138,10 @@ var Character = class extends CharacterRuntime {
|
|
|
9992
11138
|
if (info.bakedConversationEnabled || this.bakedAsset) {
|
|
9993
11139
|
throw new Error("Baked conversation requires local rendering.");
|
|
9994
11140
|
}
|
|
11141
|
+
this.resolvedInputModes = resolveInputModes(this.configuredInputModes, {
|
|
11142
|
+
allowCustomReply: info.allowCustomReply !== false,
|
|
11143
|
+
baked: false
|
|
11144
|
+
});
|
|
9995
11145
|
this.animationController = new RemoteAnimationController();
|
|
9996
11146
|
}
|
|
9997
11147
|
async createConversation(options) {
|
|
@@ -10034,6 +11184,7 @@ var Character = class extends CharacterRuntime {
|
|
|
10034
11184
|
return targetConversationId;
|
|
10035
11185
|
}
|
|
10036
11186
|
async disconnect() {
|
|
11187
|
+
this.voice.reset();
|
|
10037
11188
|
this.thinkingState = false;
|
|
10038
11189
|
if (this.animationController) {
|
|
10039
11190
|
this.streamPlayback.cleanup();
|
|
@@ -10069,6 +11220,10 @@ var Character = class extends CharacterRuntime {
|
|
|
10069
11220
|
maybeMessageOrOptions,
|
|
10070
11221
|
maybeOptions
|
|
10071
11222
|
);
|
|
11223
|
+
this.assertReplyMode(message);
|
|
11224
|
+
if (this.voice.turnPending) {
|
|
11225
|
+
throw new Error("A reply is already being generated. Please wait.");
|
|
11226
|
+
}
|
|
10072
11227
|
if (this.baked) {
|
|
10073
11228
|
const result = this.baked.reply(message, options?.parentStagingId);
|
|
10074
11229
|
if (result.ended) {
|
|
@@ -10108,6 +11263,55 @@ var Character = class extends CharacterRuntime {
|
|
|
10108
11263
|
get isBakedConversation() {
|
|
10109
11264
|
return this.baked !== null;
|
|
10110
11265
|
}
|
|
11266
|
+
/**
|
|
11267
|
+
* The input modes in use: the configured modes, or every mode that the
|
|
11268
|
+
* character permits. Before `loadAssets`, the configuration or all modes.
|
|
11269
|
+
*/
|
|
11270
|
+
get inputModes() {
|
|
11271
|
+
return this.resolvedInputModes ?? this.configuredInputModes ?? permittedInputModes({ allowCustomReply: true, baked: false });
|
|
11272
|
+
}
|
|
11273
|
+
get listeningState() {
|
|
11274
|
+
return this.voice.listeningState;
|
|
11275
|
+
}
|
|
11276
|
+
/**
|
|
11277
|
+
* Opens the microphone and streams the speech to the server. Resolves with
|
|
11278
|
+
* the result when listening ends (`stopListening`, `endOnSilenceMs` or
|
|
11279
|
+
* `maxDurationMs`), or with null when it is cancelled. The first call asks
|
|
11280
|
+
* for the microphone permission.
|
|
11281
|
+
*/
|
|
11282
|
+
startListening(options) {
|
|
11283
|
+
return this.voice.start(options);
|
|
11284
|
+
}
|
|
11285
|
+
/**
|
|
11286
|
+
* Ends the recording; resolves as soon as the server recognised the text.
|
|
11287
|
+
* Before the microphone gave any audio, it cancels and resolves null.
|
|
11288
|
+
*/
|
|
11289
|
+
stopListening() {
|
|
11290
|
+
return this.voice.stop();
|
|
11291
|
+
}
|
|
11292
|
+
/** Discards the recording. Nothing is recognised or charged. */
|
|
11293
|
+
cancelListening() {
|
|
11294
|
+
this.voice.cancel();
|
|
11295
|
+
}
|
|
11296
|
+
/** Press and hold an element or a key (default Space) to talk. Returns an unbind function. */
|
|
11297
|
+
bindPushToTalk(options) {
|
|
11298
|
+
return bindPushToTalk(this, options);
|
|
11299
|
+
}
|
|
11300
|
+
onListeningState(callback) {
|
|
11301
|
+
return bindCallbackSlot(
|
|
11302
|
+
() => this.listeningStateCallback,
|
|
11303
|
+
(next) => this.listeningStateCallback = next,
|
|
11304
|
+
callback
|
|
11305
|
+
);
|
|
11306
|
+
}
|
|
11307
|
+
/** Partial text while the player speaks (`final: false`), then the final text. */
|
|
11308
|
+
onTranscript(callback) {
|
|
11309
|
+
return bindCallbackSlot(
|
|
11310
|
+
() => this.transcriptCallback,
|
|
11311
|
+
(next) => this.transcriptCallback = next,
|
|
11312
|
+
callback
|
|
11313
|
+
);
|
|
11314
|
+
}
|
|
10111
11315
|
async getConversationWorldline(conversationId = this.conversationId) {
|
|
10112
11316
|
if (!conversationId) {
|
|
10113
11317
|
throw new Error("Conversation ID is required.");
|
|
@@ -10131,6 +11335,13 @@ var Character = class extends CharacterRuntime {
|
|
|
10131
11335
|
timeline: options?.silent ? toSilentTimeline(normalized.timeline) : normalized.timeline
|
|
10132
11336
|
});
|
|
10133
11337
|
}
|
|
11338
|
+
/**
|
|
11339
|
+
* Plays one live staging event from another transport, such as an actor
|
|
11340
|
+
* stage. A timeline event returns its playback result.
|
|
11341
|
+
*/
|
|
11342
|
+
playStreamEvent(event) {
|
|
11343
|
+
return this.streamPlayback.handleEvent(event);
|
|
11344
|
+
}
|
|
10134
11345
|
setRenderSessionId(renderSessionId) {
|
|
10135
11346
|
this.renderSessionId = renderSessionId?.trim() || null;
|
|
10136
11347
|
}
|
|
@@ -10164,6 +11375,7 @@ var Character = class extends CharacterRuntime {
|
|
|
10164
11375
|
async handleStagingResult(payload, options) {
|
|
10165
11376
|
this.thinkingState = false;
|
|
10166
11377
|
const normalized = normalizeStagingResultPayload(payload);
|
|
11378
|
+
this.latestOptions = normalized.options;
|
|
10167
11379
|
if (options.source === "turn") {
|
|
10168
11380
|
this.liveConversation.received(normalized);
|
|
10169
11381
|
}
|
|
@@ -10190,6 +11402,44 @@ var Character = class extends CharacterRuntime {
|
|
|
10190
11402
|
void this.streamPlayback.playPayload(normalized);
|
|
10191
11403
|
}
|
|
10192
11404
|
}
|
|
11405
|
+
/** Throws when the input modes do not include `mode`. */
|
|
11406
|
+
assertInputMode(mode) {
|
|
11407
|
+
if (mode === "voice" && this.baked) {
|
|
11408
|
+
throw new InputModeError(
|
|
11409
|
+
"Voice input is not available in a baked (offline) conversation."
|
|
11410
|
+
);
|
|
11411
|
+
}
|
|
11412
|
+
const modes = this.inputModes;
|
|
11413
|
+
if (!modes.includes(mode)) {
|
|
11414
|
+
throw new InputModeError(
|
|
11415
|
+
`The "${mode}" input mode is off: inputModes is ${JSON.stringify(modes)}.`
|
|
11416
|
+
);
|
|
11417
|
+
}
|
|
11418
|
+
}
|
|
11419
|
+
/**
|
|
11420
|
+
* A typed reply needs the `text` mode; one of the current reply options
|
|
11421
|
+
* needs `options` or `text`. Without a configuration the server's rules
|
|
11422
|
+
* apply alone, as before.
|
|
11423
|
+
*/
|
|
11424
|
+
assertReplyMode(message) {
|
|
11425
|
+
if (!this.configuredInputModes) {
|
|
11426
|
+
return;
|
|
11427
|
+
}
|
|
11428
|
+
const modes = this.inputModes;
|
|
11429
|
+
if (modes.includes("text")) {
|
|
11430
|
+
return;
|
|
11431
|
+
}
|
|
11432
|
+
const reply = message.trim();
|
|
11433
|
+
const isOption = this.latestOptions.some(
|
|
11434
|
+
(option) => option.label.trim() === reply
|
|
11435
|
+
);
|
|
11436
|
+
if (isOption && modes.includes("options")) {
|
|
11437
|
+
return;
|
|
11438
|
+
}
|
|
11439
|
+
throw new InputModeError(
|
|
11440
|
+
isOption ? `Reply options are off: inputModes is ${JSON.stringify(modes)}.` : `Typed replies are off: inputModes is ${JSON.stringify(modes)}. Send one of the reply options or use startListening().`
|
|
11441
|
+
);
|
|
11442
|
+
}
|
|
10193
11443
|
invokeThinkingCallback(payload) {
|
|
10194
11444
|
this.thinkingState = true;
|
|
10195
11445
|
if (this.thinkingCallback) {
|
|
@@ -10202,6 +11452,12 @@ var Character = class extends CharacterRuntime {
|
|
|
10202
11452
|
}
|
|
10203
11453
|
};
|
|
10204
11454
|
var Npcify = class extends Character {
|
|
11455
|
+
constructor(options) {
|
|
11456
|
+
super({
|
|
11457
|
+
...options,
|
|
11458
|
+
normalizeIdleOrientation: options.normalizeIdleOrientation ?? true
|
|
11459
|
+
});
|
|
11460
|
+
}
|
|
10205
11461
|
};
|
|
10206
11462
|
|
|
10207
11463
|
// ../../clients/web-client/src/character/character.ts
|
|
@@ -10782,13 +12038,18 @@ export {
|
|
|
10782
12038
|
CharacterSceneRig,
|
|
10783
12039
|
FacialPerformanceLayer,
|
|
10784
12040
|
FacialPerformanceMixer,
|
|
12041
|
+
INPUT_MODES,
|
|
12042
|
+
InputModeError,
|
|
12043
|
+
MicrophoneError,
|
|
10785
12044
|
Npcify,
|
|
10786
12045
|
Character as NpcifyCharacter,
|
|
10787
12046
|
PERFORMANCE_FILE_EXTENSION,
|
|
10788
12047
|
PERFORMANCE_MEDIA_TYPE,
|
|
10789
12048
|
PerformanceRecordingSchemaVersion,
|
|
10790
12049
|
RemoteAnimationController,
|
|
12050
|
+
SpeechInputError,
|
|
10791
12051
|
analyzePerformanceRecording,
|
|
12052
|
+
bindPushToTalk,
|
|
10792
12053
|
createCharacter,
|
|
10793
12054
|
decodePerformance,
|
|
10794
12055
|
encodePerformance,
|