@npcify/web 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -5,693 +5,186 @@ var STAGING_PROTOCOL_VERSION = 1;
5
5
  var SERVER_RENDER_PROTOCOL_VERSION = 3;
6
6
  var MAXIMUM_PLAYBACK_DURATION_SECONDS = 10 * 60;
7
7
 
8
- // ../ts-shared/src/tts-voices-asian.ts
9
- var asianTtsVoices = [
10
- {
11
- id: "qbfQuoRJv1T3ei3qV4bc",
12
- name: "Kazuha",
13
- gender: "female",
14
- description: "Conversation: Warm & Friendly",
15
- lang: ["Japanese"]
16
- },
17
- {
18
- id: "NO5A3b3sSzDyJQF7MiNS",
19
- name: "Shohei",
20
- gender: "male",
21
- description: "Conversation: Serious, Clear and Measured",
22
- lang: ["Japanese"]
23
- },
24
- {
25
- id: "3321Alera3fXjEWjjbAX",
26
- name: "Akane",
27
- gender: "female",
28
- description: "Calm: Soft, Warm and Soothing",
29
- lang: ["Japanese"]
30
- },
31
- {
32
- id: "4sirbXwrtRlmPV80MJkQ",
33
- name: "Sora",
34
- gender: "male",
35
- description: "Calm: Clear and Airy",
36
- lang: ["Japanese"]
37
- },
38
- {
39
- id: "HxuFAkkGVeQs1sDIMF5g",
40
- name: "Claire",
41
- gender: "female",
42
- description: "Lively: Cute & Cheerful JP",
43
- lang: ["Japanese"]
44
- },
45
- {
46
- id: "tGhb4uYSV8sWBI31DYU8",
47
- name: "Gary",
48
- gender: "male",
49
- description: "Lively: Joyful, Clear and Husky",
50
- lang: ["Japanese"]
51
- },
52
- {
53
- id: "8PfKHL4nZToWC3pbz9U9",
54
- name: "Rose",
55
- gender: "female",
56
- description: "Mature narration: Inviting, Clear and Steady",
57
- lang: ["Japanese"]
58
- },
59
- {
60
- id: "Mv8AjrYZCBkdsmDHNwcB",
61
- name: "Ishibashi",
62
- gender: "male",
63
- description: "Mature narration: Inviting, Natural and Smoky",
64
- lang: ["Japanese"]
65
- },
66
- {
67
- id: "6awt6FKyZGV0HyQEwisX",
68
- name: "Suzu",
69
- gender: "female",
70
- description: "Character: Cute, Clear & Calm",
71
- lang: ["Japanese"]
72
- },
73
- {
74
- id: "ZcX76PwFSkrkyI78RBTK",
75
- name: "Nayuta",
76
- gender: "male",
77
- description: "Character: Dark Anime Hero",
78
- lang: ["Japanese"]
79
- },
80
- {
81
- id: "9lHjugDhwqoxA5MhX0az",
82
- name: "Anna Su",
83
- gender: "female",
84
- description: "Conversation: Casual, Friendly and Bright",
85
- lang: ["Chinese"]
86
- },
87
- {
88
- id: "UwT0JPexcCbH107hq7i5",
89
- name: "James",
90
- gender: "male",
91
- description: "Conversation: Young, Friendly and Positive",
92
- lang: ["Chinese"]
93
- },
94
- {
95
- id: "APSIkVZudNbPAwyPoeVO",
96
- name: "Sage",
97
- gender: "female",
98
- description: "Calm: Soothing & Gentle",
99
- lang: ["Chinese"]
100
- },
101
- {
102
- id: "Z5oklYidFFuqcVQNGzYK",
103
- name: "Aman",
104
- gender: "male",
105
- description: "Calm: ASMR, Soft and Whisper",
106
- lang: ["Chinese"]
107
- },
108
- {
109
- id: "ByhETIclHirOlWnWKhHc",
110
- name: "Shan Shan",
111
- gender: "female",
112
- description: "Lively: Young Energetic Female",
113
- lang: ["Chinese"]
114
- },
115
- {
116
- id: "5mZxJZhSmJTjL7GoYfYI",
117
- name: "Karo Yang",
118
- gender: "male",
119
- description: "Lively: Clear, Lively and Energetic",
120
- lang: ["Chinese"]
121
- },
122
- {
123
- id: "xDISamJf8LV5rG5A2te1",
124
- name: "Aki | Deep & Suspenseful",
125
- gender: "female",
126
- description: "Mature narration: Deep & Suspenseful",
127
- lang: ["Chinese"]
128
- },
129
- {
130
- id: "WuLq5z7nEcrhppO0ZQJw",
131
- name: "Martin Li",
132
- gender: "male",
133
- description: "Mature narration: Raspy, Serious and Deep",
134
- lang: ["Chinese"]
135
- },
136
- {
137
- id: "BT0SH7Hb8NRPpT5xbCgg",
138
- name: "Lulu",
139
- gender: "female",
140
- description: "Character: Cute Little Witch",
141
- lang: ["Chinese"]
142
- },
143
- {
144
- id: "YaHyfWIwVxVEIhSTnKOi",
145
- name: "Xiao Bai",
146
- gender: "male",
147
- description: "Character: Nasal, Slow and Annoying",
148
- lang: ["Chinese"]
149
- },
150
- {
151
- id: "8jHHF8rMqMlg8if2mOUe",
152
- name: "Han",
153
- gender: "female",
154
- description: "Conversation: Natural, Casual and Spontaneous",
155
- lang: ["Korean"]
156
- },
157
- {
158
- id: "pb3lVZVjdFWbkhPKlelB",
159
- name: "Harry Kim",
160
- gender: "male",
161
- description: "Conversation: Soft, Natural and Steady",
162
- lang: ["Korean"]
163
- },
164
- {
165
- id: "7oLyBHyhxAjrctX6ZQlw",
166
- name: "Bomisori",
167
- gender: "female",
168
- description: "Calm: Warm and Clear",
169
- lang: ["Korean"]
170
- },
171
- {
172
- id: "BbsagRO6ohd8MKPS2Ob0",
173
- name: "Jin Geon Song",
174
- gender: "male",
175
- description: "Calm: Neutral, Calm and Steady",
176
- lang: ["Korean"]
177
- },
178
- {
179
- id: "bQlkYuipD5BHEhntA5iz",
180
- name: "JY",
181
- gender: "female",
182
- description: "Lively: Trendy K-Culture Vlog Girl",
183
- lang: ["Korean"]
184
- },
185
- {
186
- id: "1W00IGEmNmwmsDeYy7ag",
187
- name: "Krys",
188
- gender: "male",
189
- description: "Lively: Cheerful, Clear and Measured",
190
- lang: ["Korean"]
191
- },
192
- {
193
- id: "ksaI0TCD9BstzEzlxj4q",
194
- name: "Seulki",
195
- gender: "female",
196
- description: "Mature narration: Inviting, Calm and Measured",
197
- lang: ["Korean"]
198
- },
199
- {
200
- id: "FQ3MuLxZh0jHcZmA5vW1",
201
- name: "Dohyeon",
202
- gender: "male",
203
- description: "Mature narration: Whisper, Measured and Neutral",
204
- lang: ["Korean"]
205
- },
206
- {
207
- id: "DGDjWy2CQ8Ishu0meQte",
208
- name: "Chaemong",
209
- gender: "female",
210
- description: "Character: Young and Sassy",
211
- lang: ["Korean"]
212
- },
213
- {
214
- id: "ZZ4xhVcc83kZBfNIlIIz",
215
- name: "Manbo",
216
- gender: "male",
217
- description: "Character: Encouraging, Clear and Measured",
218
- lang: ["Korean"]
219
- }
220
- ];
8
+ // ../ts-shared/src/tts.ts
9
+ var TTS_PROVIDERS = ["11labs", "gemini"];
10
+ var DEFAULT_TTS_PROVIDER = TTS_PROVIDERS[0];
221
11
 
222
- // ../ts-shared/src/tts-voices-european.ts
223
- var europeanTtsVoices = [
224
- {
225
- id: "kdmDKE6EkgrWrrykO9Qt",
226
- name: "Alexandra",
227
- gender: "female",
228
- description: "Conversation: Natural and Relatable",
229
- lang: ["English"]
230
- },
231
- {
232
- id: "UgBBYS2sOqTuMpoF3BR0",
233
- name: "Mark",
234
- gender: "male",
235
- description: "Conversation: Natural and Casual",
236
- lang: ["English"]
237
- },
238
- {
239
- id: "ftDdhfYtmfGP0tFlBYA1",
240
- name: "Alisha",
241
- gender: "female",
242
- description: "Calm: Soft and Engaging",
243
- lang: ["English"]
244
- },
245
- {
246
- id: "GUDYcgRAONiI1nXDcNQQ",
247
- name: "Milo",
248
- gender: "male",
249
- description: "Calm: Soothing and Meditative",
250
- lang: ["English"]
251
- },
252
- {
253
- id: "r1KmysJdVYZjJCm4mL3b",
254
- name: "Jessica",
255
- gender: "female",
256
- description: "Lively: Playful, Bright, Warm",
257
- lang: ["English"]
258
- },
259
- {
260
- id: "o2zd9K5QOO7ppTb04Lx0",
261
- name: "Gabriel Ripley",
262
- gender: "male",
263
- description: "Lively: Facetious, Quick-Spoken",
264
- lang: ["English"]
265
- },
266
- {
267
- id: "RILOU7YmBhvwJGDGjNmP",
268
- name: "Jane",
269
- gender: "female",
270
- description: "Mature narration: Professional Audiobook Reader",
271
- lang: ["English"]
272
- },
273
- {
274
- id: "wAGzRVkxKEs8La0lmdrE",
275
- name: "Sully",
276
- gender: "male",
277
- description: "Mature narration: Mature, Deep and Intriguing",
278
- lang: ["English"]
279
- },
280
- {
281
- id: "flHkNRp1BlvT73UL6gyz",
282
- name: "Jessica Anne Bogart",
283
- gender: "female",
284
- description: "Character: Eloquent Villain",
285
- lang: ["English"]
286
- },
287
- {
288
- id: "IRHApOXLvnW57QJPQH2P",
289
- name: "Adam - American, Dark and Tough",
290
- gender: "male",
291
- description: "Character: American, Dark and Tough",
292
- lang: ["English"]
293
- },
294
- {
295
- id: "lvQdCgwZfBuOzxyV5pxu",
296
- name: "Audia",
297
- gender: "female",
298
- description: "Conversation: Warm, Clear and Welcoming",
299
- lang: ["French"]
300
- },
301
- {
302
- id: "kENkNtk0xyzG09WW40xE",
303
- name: "Marcel - Warm and Conversational",
304
- gender: "male",
305
- description: "Conversation: Warm and Conversational",
306
- lang: ["French"]
307
- },
308
- {
309
- id: "kwhMCf63M8O3rCfnQ3oQ",
310
- name: "Caroline",
311
- gender: "female",
312
- description: "Calm: Warm, Sweet and Soft",
313
- lang: ["French"]
314
- },
315
- {
316
- id: "YV28ox2c5Cuh5rim0LrW",
317
- name: "Marcel - Warm, Soft and Intimate",
318
- gender: "male",
319
- description: "Calm: Warm, Soft and Intimate",
320
- lang: ["French"]
321
- },
322
- {
323
- id: "KSyQzmsYhFbuOhqj1Xxv",
324
- name: "Lea",
325
- gender: "female",
326
- description: "Lively: Energetic and Modern",
327
- lang: ["French"]
328
- },
329
- {
330
- id: "4TfTGcPwoefWe878B0rm",
331
- name: "Troy",
332
- gender: "male",
333
- description: "Lively: Cartoon Hero",
334
- lang: ["French"]
335
- },
336
- {
337
- id: "WeAAwKYcS06VmXw086yZ",
338
- name: "Victoria",
339
- gender: "female",
340
- description: "Mature narration: Warm and calm",
341
- lang: ["French"]
342
- },
343
- {
344
- id: "TTtB1x9U8PF0Vgf20IAP",
345
- name: "Adrien",
346
- gender: "male",
347
- description: "Mature narration: Deep, Comforting and Warm",
348
- lang: ["French"]
349
- },
350
- {
351
- id: "ZYOBieLaunTiQrTrvNQq",
352
- name: "Morgane",
353
- gender: "female",
354
- description: "Character: Gentle and Expressive",
355
- lang: ["French"]
356
- },
357
- {
358
- id: "RBhYSNMNu6b2CGZ9Fn1M",
359
- name: "Felix Tabarnak",
360
- gender: "male",
361
- description: "Character: Confident and Witty",
362
- lang: ["French"]
363
- },
364
- {
365
- id: "MuTiG4dbrEGYEy3XP4iP",
366
- name: "Rossana",
367
- gender: "female",
368
- description: "Conversation: Warm Italian Conversational",
369
- lang: ["Italian"]
370
- },
371
- {
372
- id: "5wE9Co8DESyhUw0GgAHG",
373
- name: "Alessandro Zanierato",
374
- gender: "male",
375
- description: "Conversation: Casual",
376
- lang: ["Italian"]
377
- },
378
- {
379
- id: "PS6wM7QnnPTUDE3t4bbl",
380
- name: "Serena",
381
- gender: "female",
382
- description: "Calm: Soft, Soothing and Gentle",
383
- lang: ["Italian"]
384
- },
385
- {
386
- id: "TnICJ0opDnjladqWtQ6k",
387
- name: "Livio",
388
- gender: "male",
389
- description: "Calm: Meditative and Balanced",
390
- lang: ["Italian"]
391
- },
392
- {
393
- id: "ZRKmc75tGxpIMNTEiwe0",
394
- name: "Mimmi",
395
- gender: "female",
396
- description: "Lively: Playful and Cartoonish",
397
- lang: ["Italian"]
398
- },
399
- {
400
- id: "nH7uLS5UdEnvKEOAXtlQ",
401
- name: "Vittorio",
402
- gender: "male",
403
- description: "Lively: Believable, Friendly and Rich",
404
- lang: ["Italian"]
405
- },
406
- {
407
- id: "fNmw8sukfGuvWVOp33Ge",
408
- name: "Tosca",
409
- gender: "female",
410
- description: "Mature narration: Mature, Whispery and Seductive",
411
- lang: ["Italian"]
412
- },
413
- {
414
- id: "YGp1lBJLaHhfIFT0yeDE",
415
- name: "RaffaeleR",
416
- gender: "male",
417
- description: "Mature narration: Deep, Rich and Steady",
418
- lang: ["Italian"]
419
- },
420
- {
421
- id: "G0q9AYE8QsarSbMtaIEu",
422
- name: "Elettra",
423
- gender: "female",
424
- description: "Character: Mature actress",
425
- lang: ["Italian"]
426
- },
427
- {
428
- id: "TdZgLZ1h5vbFJrgh8QqK",
429
- name: "Vito Pepe",
430
- gender: "male",
431
- description: "Character: Profound and Confident",
432
- lang: ["Italian"]
433
- },
434
- {
435
- id: "CaJslL1xziwefCeTNzHv",
436
- name: "Cristina Campos",
437
- gender: "female",
438
- description: "Conversation: Friendly and Soft",
439
- lang: ["Spanish"]
440
- },
441
- {
442
- id: "YExhVa4bZONzeingloMX",
443
- name: "Juan Carlos",
444
- gender: "male",
445
- description: "Conversation: Warm and Conversational",
446
- lang: ["Spanish"]
447
- },
448
- {
449
- id: "2Lb1en5ujrODDIqmp7F3",
450
- name: "Jhenny",
451
- gender: "female",
452
- description: "Calm: Soft, Calm, Sweet",
453
- lang: ["Spanish"]
454
- },
455
- {
456
- id: "iDEmt5MnqUotdwCIVplo",
457
- name: "Enrique Mondrag\xF3n",
458
- gender: "male",
459
- description: "Calm: Elegant and Dynamic",
460
- lang: ["Spanish"]
461
- },
462
- {
463
- id: "JYyJjNPfmNJdaby8LdZs",
464
- name: "Marlene",
465
- gender: "female",
466
- description: "Lively: Melodic, Pleasant and Bright",
467
- lang: ["Spanish"]
468
- },
469
- {
470
- id: "9F4C8ztpNUmXkdDDbz3J",
471
- name: "Dan",
472
- gender: "male",
473
- description: "Lively: Upbeat, Dynamic and Friendly",
474
- lang: ["Spanish"]
475
- },
476
- {
477
- id: "2rigMbVWLdqtBSCahJFX",
478
- name: "Tatiana Martin",
479
- gender: "female",
480
- description: "Mature narration: Wise-speaking, calm",
481
- lang: ["Spanish"]
482
- },
483
- {
484
- id: "k8cFOyAg7B9qwBlDDNTC",
485
- name: "Miguel",
486
- gender: "male",
487
- description: "Mature narration: Deep, Rich and Cinematic",
488
- lang: ["Spanish"]
489
- },
490
- {
491
- id: "U9tZtg3uJtVgXPkvosWR",
492
- name: "Kate",
493
- gender: "female",
494
- description: "Character: Playful Latina Storyteller",
495
- lang: ["Spanish"]
496
- },
497
- {
498
- id: "HMCmDsbKeaSZp5LMOYKR",
499
- name: "Johnny",
500
- gender: "male",
501
- description: "Character: Mocking, Sarcastic and Rich",
502
- lang: ["Spanish"]
12
+ // ../ts-shared/src/ai-settings.ts
13
+ var GPT_IMAGE_MODELS = [
14
+ "gpt-image-2.5-flare",
15
+ "gpt-image-2.5-sunburst"
16
+ ];
17
+ var GEMINI_SPEECH_MODEL_VENDORS = {
18
+ "gemini-3.5-transcribe-live": "gemini",
19
+ "gemini-3.5-transcribe": "gemini"
20
+ };
21
+ var AI_ROLES = {
22
+ character_draft: {
23
+ name: "Character draft",
24
+ source: "llm",
25
+ models: [],
26
+ options: ["temperature"]
503
27
  },
504
- {
505
- id: "dHAwRJVaEPhU907QLTPW",
506
- name: "Natalia",
507
- gender: "female",
508
- description: "Conversation: Gentle and Soothing",
509
- lang: ["Russian"]
28
+ motion_planner: {
29
+ name: "Motion planner",
30
+ source: "llm",
31
+ models: [],
32
+ options: []
510
33
  },
511
- {
512
- id: "qJBO8ZmKp4te7NTtYgzz",
513
- name: "Egor T",
514
- gender: "male",
515
- description: "Conversation: Natural, Soothing & Relatable",
516
- lang: ["Russian"]
34
+ // Not charged.
35
+ prompt_check: {
36
+ name: "Prompt check",
37
+ source: "openai",
38
+ models: ["gpt-6-luna"],
39
+ options: []
517
40
  },
518
- {
519
- id: "ymDCYd8puC7gYjxIamPt",
520
- name: "Marina",
521
- gender: "female",
522
- description: "Calm: Soft, Clear and Warm",
523
- lang: ["Russian"]
41
+ // Priced by OPENAI_IMAGE_PRICES of the server (flare and sunburst alike).
42
+ design_sheet_image: {
43
+ name: "Design sheet image",
44
+ source: "openai",
45
+ models: GPT_IMAGE_MODELS,
46
+ options: []
524
47
  },
525
- {
526
- id: "e5A3tSm7sjA7Sbiz2Qac",
527
- name: "Maksim",
528
- gender: "male",
529
- description: "Calm: Deep and Steady",
530
- lang: ["Russian"]
48
+ // Not charged per call.
49
+ tts: {
50
+ name: "ElevenLabs text to speech",
51
+ source: "elevenlabs",
52
+ models: ["eleven_v3_conversational", "eleven_v3"],
53
+ options: []
531
54
  },
532
- {
533
- id: "AB9XsbSA4eLG12t2myjN",
534
- name: "Larisa Actrisa",
535
- gender: "female",
536
- description: "Lively: Confident and Clear",
537
- lang: ["Russian"]
55
+ // Not charged per call.
56
+ gemini_tts: {
57
+ name: "Gemini text to speech",
58
+ source: "gemini",
59
+ models: ["gemini-3.8-flash-tts", "gemini-3.8-flash-lite-tts"],
60
+ options: []
538
61
  },
539
- {
540
- id: "m0OQuJtWCw1V23P0pQmG",
541
- name: "Eugene",
542
- gender: "male",
543
- description: "Lively: Bright, Clear & Engaging",
544
- lang: ["Russian"]
62
+ // Priced by SPEECH_RECOGNITION_PRICES of the server (docs/voice-input.md).
63
+ // Audio that arrives while the player speaks. The model selects the vendor.
64
+ speech_input_live: {
65
+ name: "Speech input (live)",
66
+ source: "openai",
67
+ models: [
68
+ "gpt-transcribe",
69
+ "gpt-live-transcribe",
70
+ "gemini-3.5-transcribe-live",
71
+ "gemini-3.5-transcribe"
72
+ ],
73
+ modelVendors: GEMINI_SPEECH_MODEL_VENDORS,
74
+ options: []
545
75
  },
546
- {
547
- id: "d60rsXo2p0OwikDR5bS7",
548
- name: "Olga Orlova",
549
- gender: "female",
550
- description: "Mature narration: Calm, Clear and Serious",
551
- lang: ["Russian"]
76
+ // One complete recording (an upload of an engine SDK).
77
+ speech_input_file: {
78
+ name: "Speech input (file)",
79
+ source: "openai",
80
+ models: ["gpt-transcribe", "gemini-3.5-transcribe"],
81
+ modelVendors: GEMINI_SPEECH_MODEL_VENDORS,
82
+ options: []
552
83
  },
553
- {
554
- id: "dWNEQ4rqRY6thcusuDyq",
555
- name: "Fuad",
556
- gender: "male",
557
- description: "Mature narration: Clear, Deep and Rich",
558
- lang: ["Russian"]
84
+ // Priced by OPENAI_PRICES.text of the generator.
85
+ generator_agent: {
86
+ name: "Generator agent",
87
+ source: "openai",
88
+ models: ["gpt-6.1-sol"],
89
+ options: ["reasoningEffort"]
559
90
  },
560
- {
561
- id: "TPIitICAZ8CqlGZ81AKm",
562
- name: "Elen Kuragina",
563
- gender: "female",
564
- description: "Character: Golden & Dangerous",
565
- lang: ["Russian"]
91
+ generator_reading: {
92
+ name: "Generator face reading",
93
+ source: "openai",
94
+ models: ["gpt-6.1-sol"],
95
+ options: ["reasoningEffort"]
566
96
  },
567
- {
568
- id: "6A9D8WSMm4rFsg2DWFeE",
569
- name: "Egor Gadzhiyev",
570
- gender: "male",
571
- description: "Character: Clear and Commanding",
572
- lang: ["Russian"]
97
+ // Priced by OPENAI_PRICES of the generator (flare and sunburst alike).
98
+ generator_image: {
99
+ name: "Generator images",
100
+ source: "openai",
101
+ models: GPT_IMAGE_MODELS,
102
+ options: []
573
103
  }
574
- ];
104
+ };
105
+ var AI_ROLE_NAMES = Object.keys(AI_ROLES);
575
106
 
576
- // ../ts-shared/src/tts.ts
577
- var TTS_PROVIDERS = ["11labs"];
578
- var DEFAULT_TTS_PROVIDER = TTS_PROVIDERS[0];
579
- var ttsVoices = [
580
- {
581
- id: "ngvNHfiCrXLPAHcTrZK1",
582
- name: "Aki",
583
- gender: "female",
584
- description: "Friendly, Clear and Natural",
585
- lang: ["Japanese"]
586
- },
587
- {
588
- id: "8kgj5469z1URcH4MB2G4",
589
- name: "Sakuya",
590
- gender: "female",
591
- description: "Cheerful and Clear",
592
- lang: ["Japanese"]
593
- },
594
- {
595
- id: "KgETZ36CCLD1Cob4xpkv",
596
- name: "Romaco",
597
- description: "Cheerful, Clear and Full",
598
- gender: "female",
599
- lang: ["Japanese"]
600
- },
601
- {
602
- id: "lhTvHflPVOqgSWyuWQry",
603
- name: "Hina",
604
- description: "Cute and Friendly",
605
- gender: "female",
606
- lang: ["Japanese"]
607
- },
608
- {
609
- id: "cGbEKHsmg38m62yxIWFk",
610
- name: "Aira",
611
- gender: "male",
612
- description: "Neutral, Clear and Steady",
613
- lang: ["Japanese"]
614
- },
615
- {
616
- id: "A4AyGcPAjb1pHgflyZZp",
617
- name: "Ritsuto",
618
- gender: "male",
619
- description: "Anime Prince Voice",
620
- lang: ["Japanese"]
621
- },
622
- {
623
- id: "hkfHEbBvdQFNX4uWHqRF",
624
- name: "Stacy",
625
- gender: "female",
626
- description: "Young, Sweet and Cute",
627
- lang: ["Chinese"]
628
- },
629
- {
630
- id: "JZLpE3AGwpKYZI2X65hN",
631
- name: "Mingyao Ye",
632
- gender: "female",
633
- description: "Sad and Broken Hearted",
634
- lang: ["Chinese"]
635
- },
636
- {
637
- id: "bhJUNIXWQQ94l8eI2VUf",
638
- name: "Amy",
639
- gender: "female",
640
- description: "Young, Natural and Friendly",
641
- lang: ["Chinese", "English", "French", "Italian"]
642
- },
643
- {
644
- id: "54Cze5LrTSyLgbO6Fhlc",
645
- name: "Caty",
646
- gender: "female",
647
- description: "Droll, Wry and Dry",
648
- lang: ["Chinese", "English", "Korean", "French"]
649
- },
650
- {
651
- id: "kbrsaic1zriFXx1pgRYN",
652
- name: "Evan",
653
- gender: "male",
654
- description: "Warm, Soothing and Soft",
655
- lang: ["Chinese"]
656
- },
657
- {
658
- id: "93nuHbke4dTER9x2pDwE",
659
- name: "Adam",
660
- gender: "male",
661
- description: "Warm and Friendly",
662
- lang: ["Chinese", "French", "Korean", "Spanish", "Russian"]
663
- },
664
- {
665
- id: "6MoEUz34rbRrmmyxgRm4",
666
- name: "Manav",
667
- gender: "male",
668
- description: "Husky and Conversational",
669
- lang: ["Spanish", "Japanese", "Chinese", "Italian"]
670
- },
671
- {
672
- id: "gOupLcAkjEnguROwi4oS",
673
- name: "Darian",
674
- gender: "male",
675
- description: "Warm, Grounded Baritone",
676
- lang: ["English"]
677
- },
678
- {
679
- id: "ktkP7Nsj67dw2zcplQYt",
680
- name: "Lawrence",
681
- gender: "male",
682
- description: "Bright, Informative",
683
- lang: ["English"]
684
- },
685
- {
686
- id: "10NkTYmU7tSz3Kkl3Lex",
687
- name: "Kaelen",
688
- gender: "male",
689
- description: "Rugged, Gravelly Bass-Baritone",
690
- lang: ["English"]
691
- },
692
- ...asianTtsVoices,
693
- ...europeanTtsVoices
694
- ];
107
+ // ../ts-shared/src/speech-input.ts
108
+ var INPUT_MODES = ["options", "text", "voice"];
109
+ var InputModeError = class extends Error {
110
+ constructor(message) {
111
+ super(message);
112
+ this.name = "NpcifyInputModeError";
113
+ }
114
+ };
115
+ var permittedInputModes = function(permissions) {
116
+ if (permissions.baked || !permissions.allowCustomReply) {
117
+ return ["options"];
118
+ }
119
+ return ["options", "text", "voice"];
120
+ };
121
+ var isInputMode = function(value) {
122
+ return INPUT_MODES.includes(value);
123
+ };
124
+ var parseInputModes = function(value) {
125
+ if (value === void 0 || value === null) {
126
+ return void 0;
127
+ }
128
+ if (!Array.isArray(value)) {
129
+ throw new InputModeError(
130
+ 'inputModes must be an array of "options", "text" and "voice".'
131
+ );
132
+ }
133
+ if (value.length === 0) {
134
+ throw new InputModeError("inputModes must have at least one mode.");
135
+ }
136
+ const modes = [];
137
+ for (const mode of value) {
138
+ if (!isInputMode(mode)) {
139
+ throw new InputModeError(
140
+ `inputModes has an unknown mode ${JSON.stringify(mode)}. Use "options", "text" or "voice".`
141
+ );
142
+ }
143
+ if (modes.includes(mode)) {
144
+ throw new InputModeError(`inputModes has "${mode}" two times.`);
145
+ }
146
+ modes.push(mode);
147
+ }
148
+ return modes;
149
+ };
150
+ var resolveInputModes = function(configured, permissions) {
151
+ const permitted = permittedInputModes(permissions);
152
+ if (!configured) {
153
+ return permitted;
154
+ }
155
+ if (permissions.baked && !(configured.length === 1 && configured[0] === "options")) {
156
+ throw new InputModeError(
157
+ 'A baked (offline) conversation accepts only reply options. Set inputModes to ["options"] or remove it.'
158
+ );
159
+ }
160
+ const refused = configured.filter((mode) => !permitted.includes(mode));
161
+ if (refused.length > 0) {
162
+ throw new InputModeError(
163
+ `This character accepts only its reply options (Allow custom replies is off). Remove ${refused.map((mode) => `"${mode}"`).join(" and ")} from inputModes.`
164
+ );
165
+ }
166
+ return [...configured];
167
+ };
168
+ var SPEECH_AUDIO_FORMAT = Object.freeze({
169
+ encoding: "pcm_s16le",
170
+ sampleRate: 16e3,
171
+ channels: 1
172
+ });
173
+ var SPEECH_BYTES_PER_SECOND = 32e3;
174
+ var SPEECH_MAX_DURATION_MS = 3e4;
175
+ var SPEECH_MAX_AUDIO_BYTES = SPEECH_MAX_DURATION_MS / 1e3 * SPEECH_BYTES_PER_SECOND;
176
+ var SPEECH_MAX_UPLOAD_BYTES = SPEECH_MAX_AUDIO_BYTES + 4096;
177
+ var SPEECH_FRAME_MS = 100;
178
+ var SPEECH_MIN_HOLD_MS = 300;
179
+ var SPEECH_EVENTS = Object.freeze({
180
+ start: "speech:start",
181
+ audio: "speech:audio",
182
+ stop: "speech:stop",
183
+ cancel: "speech:cancel",
184
+ partial: "speech:partial",
185
+ turn: "speech:turn",
186
+ turnFailed: "speech:turn_failed"
187
+ });
695
188
 
696
189
  // ../ts-shared/src/product-usage.ts
697
190
  var productUsageEvents = [
@@ -701,7 +194,11 @@ var productUsageEvents = [
701
194
  "checkout_submitted",
702
195
  "checkout_failed",
703
196
  "checkout_closed",
704
- "paywall_shown"
197
+ "paywall_shown",
198
+ "playback_blocked",
199
+ "playback_start_pressed",
200
+ "reply_options_shown",
201
+ "page_left"
705
202
  ];
706
203
 
707
204
  // ../ts-shared/src/analytics.ts
@@ -3806,8 +3303,14 @@ var LiveConversation = class {
3806
3303
  this.dependencies = dependencies;
3807
3304
  }
3808
3305
  sending = false;
3306
+ /** A typed turn is in flight. */
3307
+ get inFlight() {
3308
+ return this.sending;
3309
+ }
3809
3310
  pending = null;
3311
+ revision = 0;
3810
3312
  reset() {
3313
+ this.revision += 1;
3811
3314
  this.pending = null;
3812
3315
  }
3813
3316
  received(payload) {
@@ -3820,6 +3323,7 @@ var LiveConversation = class {
3820
3323
  throw new Error("A reply is already being generated. Please wait.");
3821
3324
  }
3822
3325
  this.sending = true;
3326
+ const revision = this.revision;
3823
3327
  const { telemetry, socket } = this.dependencies;
3824
3328
  try {
3825
3329
  telemetry.begin(conversationId);
@@ -3827,10 +3331,10 @@ var LiveConversation = class {
3827
3331
  const silent = options?.silent === true;
3828
3332
  const parentStagingId = options?.parentStagingId;
3829
3333
  const pending = this.pending;
3830
- const request = pending?.conversationId === conversationId && pending.message === message && pending.silent === silent && pending.parentStagingId === parentStagingId ? pending : {
3334
+ const request = !options?.requestId && pending?.conversationId === conversationId && pending.message === message && pending.silent === silent && pending.parentStagingId === parentStagingId ? pending : {
3831
3335
  conversationId,
3832
3336
  message,
3833
- requestId: globalThis.crypto.randomUUID(),
3337
+ requestId: options?.requestId ?? globalThis.crypto.randomUUID(),
3834
3338
  silent,
3835
3339
  parentStagingId
3836
3340
  };
@@ -3864,6 +3368,9 @@ var LiveConversation = class {
3864
3368
  if (this.pending?.requestId === request.requestId) {
3865
3369
  this.pending = null;
3866
3370
  }
3371
+ if (revision !== this.revision) {
3372
+ return payload;
3373
+ }
3867
3374
  await this.dependencies.handleResult(
3868
3375
  payload,
3869
3376
  payload.taskId !== request.requestId,
@@ -4994,7 +4501,7 @@ var disposeSceneContext = function(ctx) {
4994
4501
 
4995
4502
  // ../vrm-renderer/src/preview-controller.ts
4996
4503
  import { VRMUtils as VRMUtils4 } from "@pixiv/three-vrm";
4997
- import { Box3, GridHelper, Vector3 as Vector33 } from "three";
4504
+ import { Box3, GridHelper, Vector3 as Vector34 } from "three";
4998
4505
 
4999
4506
  // ../vrm-renderer/src/loaders.ts
5000
4507
  import { VRMLoaderPlugin as VRMLoaderPlugin2, VRMUtils as VRMUtils2 } from "@pixiv/three-vrm";
@@ -5016,7 +4523,33 @@ import { AnimationMixer, LoopOnce, LoopRepeat } from "three";
5016
4523
  import {
5017
4524
  createVRMAnimationClip as createPixivVRMAnimationClip
5018
4525
  } from "@pixiv/three-vrm-animation";
4526
+ import {
4527
+ AnimationClip,
4528
+ Euler,
4529
+ Quaternion as Quaternion2,
4530
+ QuaternionKeyframeTrack,
4531
+ QuaternionLinearInterpolant,
4532
+ Vector3 as Vector33,
4533
+ VectorKeyframeTrack
4534
+ } from "three";
5019
4535
  var NPCIFY_VRM_NORMALIZATION_ROOT_NAME = "NPCifyNormalizationRoot";
4536
+ var TORSO_BONES = [
4537
+ "hips",
4538
+ "spine",
4539
+ "chest",
4540
+ "upperChest"
4541
+ ];
4542
+ var HEAD_BONES = ["neck", "head"];
4543
+ var AXIAL_BONES = [...TORSO_BONES, ...HEAD_BONES];
4544
+ var PARENT_OF_HEAD_BONES = [
4545
+ ...TORSO_BONES,
4546
+ "neck"
4547
+ ];
4548
+ var IDENTITY_QUATERNION = new Quaternion2();
4549
+ var UP_AXIS = new Vector33(0, 1, 0);
4550
+ var MAX_HEAD_PITCH_CORRECTION_RAD = 35 * Math.PI / 180;
4551
+ var NECK_CORRECTION_SHARE = 0.35;
4552
+ var EPSILON = 1e-5;
5020
4553
  var canonicalAnimationTargets = /* @__PURE__ */ new WeakMap();
5021
4554
  var isNpcifyNormalizedVrm = function(vrm) {
5022
4555
  return vrm.scene.children.some(
@@ -5042,39 +4575,331 @@ var canonicalAnimationTarget = function(vrm) {
5042
4575
  canonicalAnimationTargets.set(vrm, target);
5043
4576
  return target;
5044
4577
  };
5045
- var createVRMAnimationClip = function(animation, vrm) {
5046
- return createPixivVRMAnimationClip(animation, canonicalAnimationTarget(vrm));
4578
+ var resolveAxialBoneNodeNames = function(vrm) {
4579
+ const result = /* @__PURE__ */ new Map();
4580
+ const seenNodeNames = /* @__PURE__ */ new Set();
4581
+ for (const bone of AXIAL_BONES) {
4582
+ const nodeName = vrm.humanoid?.getNormalizedBoneNode?.(bone)?.name;
4583
+ if (nodeName && !seenNodeNames.has(nodeName)) {
4584
+ seenNodeNames.add(nodeName);
4585
+ result.set(bone, nodeName);
4586
+ }
4587
+ }
4588
+ return result;
5047
4589
  };
5048
-
5049
- // ../vrm-renderer/src/session.ts
5050
- var attachVrm = function(ctx, vrm) {
5051
- ctx.scene.add(vrm.scene);
5052
- setCameraCollisionBackend(
5053
- ctx,
5054
- new HumanoidCapsuleCollisionBackend(
5055
- (name) => vrm.humanoid.getRawBoneNode(name)
5056
- )
4590
+ var findQuaternionTrack = function(clip, nodeName) {
4591
+ if (!nodeName) {
4592
+ return void 0;
4593
+ }
4594
+ const trackName = `${nodeName}.quaternion`;
4595
+ return clip.tracks.find(
4596
+ (track) => track.name === trackName && track.times.length > 0 && track.values.length >= track.times.length * 4
5057
4597
  );
5058
- const mixer = new AnimationMixer(vrm.scene);
5059
- return { vrm, mixer, activeAction: null, animationFrameId: null };
5060
4598
  };
5061
- var detachVrm = function(ctx, session, options = {}) {
5062
- stopRenderLoop(session);
5063
- session.mixer.stopAllAction();
5064
- if (session.vrm.scene.parent) {
5065
- session.vrm.scene.parent.remove(session.vrm.scene);
5066
- }
5067
- setCameraCollisionBackend(ctx, null);
5068
- session.mixer.uncacheRoot(session.vrm.scene);
5069
- if (options.dispose) {
5070
- disposeVrm(session.vrm);
4599
+ var sampleBoneQuaternion = function(interpolant, time, target) {
4600
+ if (!interpolant) {
4601
+ return target.identity();
5071
4602
  }
4603
+ const values = interpolant.evaluate(time);
4604
+ return target.set(values[0] ?? 0, values[1] ?? 0, values[2] ?? 0, values[3] ?? 1).normalize();
5072
4605
  };
5073
- var disposeVrm = function(vrm) {
5074
- disposeObject3D(vrm.scene);
4606
+ var evaluateChainOrientation = function(bones, interpolants, time, rootPreRotation, target, scratch) {
4607
+ target.copy(rootPreRotation);
4608
+ for (const bone of bones) {
4609
+ const interpolant = interpolants.get(bone);
4610
+ if (interpolant) {
4611
+ sampleBoneQuaternion(interpolant, time, scratch);
4612
+ target.multiply(scratch);
4613
+ }
4614
+ }
4615
+ return target;
5075
4616
  };
5076
- var disposeObject3D = function(object) {
5077
- VRMUtils3.deepDispose(object);
4617
+ var rotateQuaternionTrackInParentSpace = function(track, parentBones, interpolants, rootPreRotation, worldCorrection) {
4618
+ const values = new Float32Array(track.values.length);
4619
+ const parentQuat = new Quaternion2();
4620
+ const parentInvQuat = new Quaternion2();
4621
+ const localCorrection = new Quaternion2();
4622
+ const keyframeQuat = new Quaternion2();
4623
+ const prevOrigQuat = new Quaternion2();
4624
+ const prevNewQuat = new Quaternion2();
4625
+ const scratchQuat = new Quaternion2();
4626
+ const count = track.times.length;
4627
+ for (let i = 0; i < count; i += 1) {
4628
+ const time = track.times[i] ?? 0;
4629
+ const offset = i * 4;
4630
+ keyframeQuat.set(
4631
+ track.values[offset] ?? 0,
4632
+ track.values[offset + 1] ?? 0,
4633
+ track.values[offset + 2] ?? 0,
4634
+ track.values[offset + 3] ?? 1
4635
+ ).normalize();
4636
+ const origDot = i > 0 ? prevOrigQuat.dot(keyframeQuat) : 1;
4637
+ prevOrigQuat.copy(keyframeQuat);
4638
+ if (parentBones.length === 0) {
4639
+ keyframeQuat.premultiply(worldCorrection).normalize();
4640
+ } else {
4641
+ evaluateChainOrientation(
4642
+ parentBones,
4643
+ interpolants,
4644
+ time,
4645
+ rootPreRotation,
4646
+ parentQuat,
4647
+ scratchQuat
4648
+ );
4649
+ parentInvQuat.copy(parentQuat).invert();
4650
+ localCorrection.copy(parentInvQuat).multiply(worldCorrection).multiply(parentQuat);
4651
+ keyframeQuat.premultiply(localCorrection).normalize();
4652
+ }
4653
+ if (i > 0 && origDot >= 0 && prevNewQuat.dot(keyframeQuat) < 0) {
4654
+ keyframeQuat.set(
4655
+ -keyframeQuat.x,
4656
+ -keyframeQuat.y,
4657
+ -keyframeQuat.z,
4658
+ -keyframeQuat.w
4659
+ );
4660
+ }
4661
+ prevNewQuat.copy(keyframeQuat);
4662
+ values[offset] = keyframeQuat.x;
4663
+ values[offset + 1] = keyframeQuat.y;
4664
+ values[offset + 2] = keyframeQuat.z;
4665
+ values[offset + 3] = keyframeQuat.w;
4666
+ }
4667
+ return new QuaternionKeyframeTrack(
4668
+ track.name,
4669
+ Array.from(track.times),
4670
+ Array.from(values)
4671
+ );
4672
+ };
4673
+ var rotateHipsPositionTrack = function(track, yawRadians) {
4674
+ const count = Math.floor(track.values.length / 3);
4675
+ if (count === 0) {
4676
+ return new VectorKeyframeTrack(
4677
+ track.name,
4678
+ Array.from(track.times),
4679
+ Array.from(track.values)
4680
+ );
4681
+ }
4682
+ let meanX = 0;
4683
+ let meanZ = 0;
4684
+ for (let i = 0; i < count; i += 1) {
4685
+ meanX += track.values[i * 3] ?? 0;
4686
+ meanZ += track.values[i * 3 + 2] ?? 0;
4687
+ }
4688
+ meanX /= count;
4689
+ meanZ /= count;
4690
+ const cosYaw = Math.cos(yawRadians);
4691
+ const sinYaw = Math.sin(yawRadians);
4692
+ const values = new Float32Array(track.values.length);
4693
+ for (let i = 0; i < count; i += 1) {
4694
+ const offset = i * 3;
4695
+ const dx = (track.values[offset] ?? 0) - meanX;
4696
+ const y = track.values[offset + 1] ?? 0;
4697
+ const dz = (track.values[offset + 2] ?? 0) - meanZ;
4698
+ values[offset] = meanX + dx * cosYaw + dz * sinYaw;
4699
+ values[offset + 1] = y;
4700
+ values[offset + 2] = meanZ - dx * sinYaw + dz * cosYaw;
4701
+ }
4702
+ return new VectorKeyframeTrack(
4703
+ track.name,
4704
+ Array.from(track.times),
4705
+ Array.from(values)
4706
+ );
4707
+ };
4708
+ var normalizeIdleAnimationClip = function(clip, vrm) {
4709
+ const boneNodeNames = resolveAxialBoneNodeNames(vrm);
4710
+ if (boneNodeNames.size === 0) {
4711
+ return clip;
4712
+ }
4713
+ const interpolants = /* @__PURE__ */ new Map();
4714
+ for (const bone of AXIAL_BONES) {
4715
+ const track = findQuaternionTrack(clip, boneNodeNames.get(bone));
4716
+ if (track) {
4717
+ interpolants.set(
4718
+ bone,
4719
+ new QuaternionLinearInterpolant(track.times, track.values, 4)
4720
+ );
4721
+ }
4722
+ }
4723
+ if (interpolants.size === 0) {
4724
+ return clip;
4725
+ }
4726
+ const duration = clip.duration > 0 ? clip.duration : 0;
4727
+ const sampleCount = duration > 0 ? Math.max(16, Math.min(120, Math.ceil(duration * 30))) : 1;
4728
+ const sampleTimes = [];
4729
+ for (let i = 0; i < sampleCount; i += 1) {
4730
+ sampleTimes.push(duration > 0 ? (i + 0.5) / sampleCount * duration : 0);
4731
+ }
4732
+ const chainQuat = new Quaternion2();
4733
+ const scratchQuat = new Quaternion2();
4734
+ const forwardVec = new Vector33();
4735
+ const rootTorsoBone = TORSO_BONES.find((bone) => interpolants.has(bone));
4736
+ const rootPreRotation = new Quaternion2();
4737
+ let torsoYawCorrection = 0;
4738
+ if (rootTorsoBone) {
4739
+ let sumX = 0;
4740
+ let sumZ = 0;
4741
+ for (const time of sampleTimes) {
4742
+ evaluateChainOrientation(
4743
+ TORSO_BONES,
4744
+ interpolants,
4745
+ time,
4746
+ IDENTITY_QUATERNION,
4747
+ chainQuat,
4748
+ scratchQuat
4749
+ );
4750
+ forwardVec.set(0, 0, 1).applyQuaternion(chainQuat);
4751
+ const horizLen = Math.hypot(forwardVec.x, forwardVec.z);
4752
+ if (horizLen > EPSILON) {
4753
+ sumX += forwardVec.x / horizLen;
4754
+ sumZ += forwardVec.z / horizLen;
4755
+ }
4756
+ }
4757
+ if (Math.hypot(sumX, sumZ) > EPSILON) {
4758
+ const meanTorsoYaw = Math.atan2(sumX, sumZ);
4759
+ if (Math.abs(meanTorsoYaw) > EPSILON) {
4760
+ torsoYawCorrection = -meanTorsoYaw;
4761
+ rootPreRotation.setFromAxisAngle(UP_AXIS, torsoYawCorrection);
4762
+ }
4763
+ }
4764
+ }
4765
+ const hasNeckTrack = interpolants.has("neck");
4766
+ const hasHeadTrack = interpolants.has("head");
4767
+ let neckWorldCorrection = null;
4768
+ let headWorldCorrection = null;
4769
+ if (hasNeckTrack || hasHeadTrack) {
4770
+ const meanHeadForward = new Vector33();
4771
+ for (const time of sampleTimes) {
4772
+ evaluateChainOrientation(
4773
+ AXIAL_BONES,
4774
+ interpolants,
4775
+ time,
4776
+ rootPreRotation,
4777
+ chainQuat,
4778
+ scratchQuat
4779
+ );
4780
+ forwardVec.set(0, 0, 1).applyQuaternion(chainQuat);
4781
+ meanHeadForward.add(forwardVec);
4782
+ }
4783
+ if (meanHeadForward.lengthSq() > EPSILON * EPSILON) {
4784
+ meanHeadForward.normalize();
4785
+ const meanHeadYaw = Math.atan2(meanHeadForward.x, meanHeadForward.z);
4786
+ const rawHeadPitch = Math.atan2(
4787
+ -meanHeadForward.y,
4788
+ Math.hypot(meanHeadForward.x, meanHeadForward.z)
4789
+ );
4790
+ const clampedHeadPitch = Math.max(
4791
+ -MAX_HEAD_PITCH_CORRECTION_RAD,
4792
+ Math.min(MAX_HEAD_PITCH_CORRECTION_RAD, rawHeadPitch)
4793
+ );
4794
+ if (Math.abs(meanHeadYaw) > EPSILON || Math.abs(clampedHeadPitch) > EPSILON) {
4795
+ const totalWorldCorrection = new Quaternion2().setFromEuler(
4796
+ new Euler(-clampedHeadPitch, -meanHeadYaw, 0, "XYZ")
4797
+ );
4798
+ if (hasNeckTrack && hasHeadTrack) {
4799
+ neckWorldCorrection = new Quaternion2().setFromEuler(
4800
+ new Euler(
4801
+ -clampedHeadPitch * NECK_CORRECTION_SHARE,
4802
+ -meanHeadYaw * NECK_CORRECTION_SHARE,
4803
+ 0,
4804
+ "XYZ"
4805
+ )
4806
+ );
4807
+ headWorldCorrection = neckWorldCorrection.clone().invert().multiply(totalWorldCorrection);
4808
+ } else if (hasNeckTrack) {
4809
+ neckWorldCorrection = totalWorldCorrection;
4810
+ } else {
4811
+ headWorldCorrection = totalWorldCorrection;
4812
+ }
4813
+ }
4814
+ }
4815
+ }
4816
+ const hasTorsoCorrection = Math.abs(torsoYawCorrection) > EPSILON;
4817
+ if (!hasTorsoCorrection && !neckWorldCorrection && !headWorldCorrection) {
4818
+ return clip;
4819
+ }
4820
+ const rootTorsoNodeName = rootTorsoBone ? boneNodeNames.get(rootTorsoBone) : void 0;
4821
+ const rootTorsoQuatTrackName = rootTorsoNodeName ? `${rootTorsoNodeName}.quaternion` : null;
4822
+ const hipsNodeName = boneNodeNames.get("hips");
4823
+ const hipsPosTrackName = hipsNodeName && rootTorsoBone === "hips" ? `${hipsNodeName}.position` : null;
4824
+ const neckNodeName = boneNodeNames.get("neck");
4825
+ const neckQuatTrackName = neckNodeName ? `${neckNodeName}.quaternion` : null;
4826
+ const headNodeName = boneNodeNames.get("head");
4827
+ const headQuatTrackName = headNodeName ? `${headNodeName}.quaternion` : null;
4828
+ const nextTracks = clip.tracks.map((track) => {
4829
+ if (hasTorsoCorrection && track.name === rootTorsoQuatTrackName) {
4830
+ return rotateQuaternionTrackInParentSpace(
4831
+ track,
4832
+ [],
4833
+ interpolants,
4834
+ IDENTITY_QUATERNION,
4835
+ rootPreRotation
4836
+ );
4837
+ }
4838
+ if (hasTorsoCorrection && track.name === hipsPosTrackName) {
4839
+ return rotateHipsPositionTrack(track, torsoYawCorrection);
4840
+ }
4841
+ if (neckWorldCorrection && track.name === neckQuatTrackName) {
4842
+ return rotateQuaternionTrackInParentSpace(
4843
+ track,
4844
+ TORSO_BONES,
4845
+ interpolants,
4846
+ rootPreRotation,
4847
+ neckWorldCorrection
4848
+ );
4849
+ }
4850
+ if (headWorldCorrection && track.name === headQuatTrackName) {
4851
+ return rotateQuaternionTrackInParentSpace(
4852
+ track,
4853
+ PARENT_OF_HEAD_BONES,
4854
+ interpolants,
4855
+ rootPreRotation,
4856
+ headWorldCorrection
4857
+ );
4858
+ }
4859
+ return track;
4860
+ });
4861
+ return new AnimationClip(clip.name, clip.duration, nextTracks, clip.blendMode);
4862
+ };
4863
+ var createVRMAnimationClip = function(animation, vrm, options) {
4864
+ const clip = createPixivVRMAnimationClip(
4865
+ animation,
4866
+ canonicalAnimationTarget(vrm)
4867
+ );
4868
+ if (options?.normalizeIdleOrientation) {
4869
+ return normalizeIdleAnimationClip(clip, vrm);
4870
+ }
4871
+ return clip;
4872
+ };
4873
+
4874
+ // ../vrm-renderer/src/session.ts
4875
+ var attachVrm = function(ctx, vrm) {
4876
+ ctx.scene.add(vrm.scene);
4877
+ setCameraCollisionBackend(
4878
+ ctx,
4879
+ new HumanoidCapsuleCollisionBackend(
4880
+ (name) => vrm.humanoid.getRawBoneNode(name)
4881
+ )
4882
+ );
4883
+ const mixer = new AnimationMixer(vrm.scene);
4884
+ return { vrm, mixer, activeAction: null, animationFrameId: null };
4885
+ };
4886
+ var detachVrm = function(ctx, session, options = {}) {
4887
+ stopRenderLoop(session);
4888
+ session.mixer.stopAllAction();
4889
+ if (session.vrm.scene.parent) {
4890
+ session.vrm.scene.parent.remove(session.vrm.scene);
4891
+ }
4892
+ setCameraCollisionBackend(ctx, null);
4893
+ session.mixer.uncacheRoot(session.vrm.scene);
4894
+ if (options.dispose) {
4895
+ disposeVrm(session.vrm);
4896
+ }
4897
+ };
4898
+ var disposeVrm = function(vrm) {
4899
+ disposeObject3D(vrm.scene);
4900
+ };
4901
+ var disposeObject3D = function(object) {
4902
+ VRMUtils3.deepDispose(object);
5078
4903
  };
5079
4904
  var stopRenderLoop = function(session) {
5080
4905
  if (session.animationFrameId === null) {
@@ -5380,16 +5205,20 @@ var RemoteAnimationController = class {
5380
5205
  }
5381
5206
  };
5382
5207
  var AnimationController = class {
5383
- constructor(vrm, idleAnimationId) {
5208
+ constructor(vrm, idleAnimationId, options) {
5384
5209
  this.vrm = vrm;
5385
5210
  this.mixer = new AnimationMixer2(this.vrm.scene);
5386
5211
  this.idleAnimationId = idleAnimationId ?? null;
5212
+ this.normalizeIdleOrientation = options?.normalizeIdleOrientation ?? false;
5387
5213
  this.mixer.addEventListener("finished", this.handleActionFinished);
5388
5214
  }
5389
5215
  prepareAnimations;
5390
5216
  mixer;
5391
5217
  clips = /* @__PURE__ */ new Map();
5218
+ rawClips = /* @__PURE__ */ new Map();
5219
+ normalizedIdleClips = /* @__PURE__ */ new Map();
5392
5220
  configs = /* @__PURE__ */ new Map();
5221
+ normalizeIdleOrientation;
5393
5222
  previewMixer = null;
5394
5223
  restorePreview = null;
5395
5224
  activeAction = null;
@@ -5415,8 +5244,10 @@ var AnimationController = class {
5415
5244
  return this.clips.has(id);
5416
5245
  }
5417
5246
  addAnimation(animationId, animation, config) {
5418
- const clip = createVRMAnimationClip(animation, this.vrm);
5419
- this.clips.set(animationId, clip);
5247
+ const rawClip = createVRMAnimationClip(animation, this.vrm);
5248
+ this.rawClips.set(animationId, rawClip);
5249
+ this.normalizedIdleClips.delete(animationId);
5250
+ this.clips.set(animationId, this.resolveClipForId(animationId, rawClip));
5420
5251
  this.configs.set(animationId, {
5421
5252
  loop: config?.loop ?? defaultAnimationConfig.loop,
5422
5253
  weight: config?.weight ?? defaultAnimationConfig.weight
@@ -5446,7 +5277,20 @@ var AnimationController = class {
5446
5277
  return restore;
5447
5278
  }
5448
5279
  setIdleAnimation(animationId) {
5280
+ const previousIdleId = this.idleAnimationId;
5449
5281
  this.idleAnimationId = animationId;
5282
+ if (previousIdleId && previousIdleId !== animationId) {
5283
+ const previousRaw = this.rawClips.get(previousIdleId);
5284
+ if (previousRaw) {
5285
+ this.clips.set(previousIdleId, previousRaw);
5286
+ }
5287
+ }
5288
+ if (animationId) {
5289
+ const rawClip = this.rawClips.get(animationId);
5290
+ if (rawClip) {
5291
+ this.clips.set(animationId, this.resolveClipForId(animationId, rawClip));
5292
+ }
5293
+ }
5450
5294
  this.playIdle();
5451
5295
  }
5452
5296
  clearAnimations() {
@@ -5454,6 +5298,8 @@ var AnimationController = class {
5454
5298
  this.mixer.stopAllAction();
5455
5299
  this.mixer.uncacheRoot(this.vrm.scene);
5456
5300
  this.clips.clear();
5301
+ this.rawClips.clear();
5302
+ this.normalizedIdleClips.clear();
5457
5303
  this.configs.clear();
5458
5304
  this.activeAction = null;
5459
5305
  this.activeAnimationId = null;
@@ -5461,6 +5307,18 @@ var AnimationController = class {
5461
5307
  this.idleAnimationId = null;
5462
5308
  this.vrm.humanoid.resetNormalizedPose();
5463
5309
  }
5310
+ resolveClipForId(animationId, rawClip) {
5311
+ if (!this.normalizeIdleOrientation || animationId !== this.idleAnimationId) {
5312
+ return rawClip;
5313
+ }
5314
+ const cached = this.normalizedIdleClips.get(animationId);
5315
+ if (cached) {
5316
+ return cached;
5317
+ }
5318
+ const normalized = normalizeIdleAnimationClip(rawClip, this.vrm);
5319
+ this.normalizedIdleClips.set(animationId, normalized);
5320
+ return normalized;
5321
+ }
5464
5322
  playAnimation(animationId, fadeDuration = 0.25, loopOverride) {
5465
5323
  this.restorePreview?.();
5466
5324
  const clip = this.clips.get(animationId);
@@ -6229,6 +6087,7 @@ var StreamPlaybackController = class {
6229
6087
  force: options?.force === true
6230
6088
  });
6231
6089
  }
6090
+ /** Returns the playback result for a timeline event. */
6232
6091
  handleEvent(event) {
6233
6092
  if (this.streamEventCallback) {
6234
6093
  this.safeInvoke("onStreamEvent", () => this.streamEventCallback?.(event));
@@ -6272,14 +6131,13 @@ var StreamPlaybackController = class {
6272
6131
  if (event.type === "staging" || event.type === "staging_prefix" || event.type === "staging_live") {
6273
6132
  this.recordStreamEvent(event, "timeline");
6274
6133
  this.dependencies.telemetry.recordFirstStaging();
6275
- void this.playTimeline({
6134
+ return this.playTimeline({
6276
6135
  taskId: event.taskId,
6277
6136
  stagingId: event.stagingId,
6278
6137
  timeline: event.timeline,
6279
6138
  complete: event.complete,
6280
6139
  revision: event.revision
6281
6140
  });
6282
- return;
6283
6141
  }
6284
6142
  if (event.type === "options") {
6285
6143
  this.recordStreamEvent(event, "options");
@@ -6638,8 +6496,18 @@ var decodeBase64 = function(value) {
6638
6496
  }
6639
6497
  return bytes;
6640
6498
  };
6641
- var canUseMediaSource = function(mimeType) {
6642
- return typeof MediaSource !== "undefined" && typeof URL?.createObjectURL === "function" && MediaSource.isTypeSupported(mimeType);
6499
+ var managedMediaSource = function() {
6500
+ return globalThis.ManagedMediaSource;
6501
+ };
6502
+ var needsManagedMediaSource = function() {
6503
+ return typeof MediaSource === "undefined" && Boolean(managedMediaSource());
6504
+ };
6505
+ var streamingMediaSource = function(mimeType) {
6506
+ if (typeof URL?.createObjectURL !== "function") {
6507
+ return null;
6508
+ }
6509
+ const source = typeof MediaSource === "undefined" ? managedMediaSource() : MediaSource;
6510
+ return source?.isTypeSupported(mimeType) ? source : null;
6643
6511
  };
6644
6512
  var StreamingAudioBuffer = class {
6645
6513
  constructor(mimeType) {
@@ -6682,8 +6550,9 @@ var StreamingAudioBuffer = class {
6682
6550
  if (this.objectUrl) {
6683
6551
  return this.objectUrl;
6684
6552
  }
6685
- if (canUseMediaSource(this.mimeType)) {
6686
- this.mediaSource = new MediaSource();
6553
+ const Source2 = streamingMediaSource(this.mimeType);
6554
+ if (Source2) {
6555
+ this.mediaSource = new Source2();
6687
6556
  this.objectUrl = URL.createObjectURL(this.mediaSource);
6688
6557
  this.mediaSource.addEventListener("sourceopen", this.handleSourceOpen, {
6689
6558
  once: true
@@ -6775,7 +6644,7 @@ var StreamingAudioBuffer = class {
6775
6644
  }
6776
6645
  };
6777
6646
  finishFallbackIfReady() {
6778
- if (!this.completed || this.objectUrl || canUseMediaSource(this.mimeType) || this.failedError || this.disposed) {
6647
+ if (!this.completed || this.objectUrl || streamingMediaSource(this.mimeType) || this.failedError || this.disposed) {
6779
6648
  return;
6780
6649
  }
6781
6650
  const blobParts = this.chunks.map((chunk) => Uint8Array.from(chunk).buffer);
@@ -7102,7 +6971,8 @@ var AudioPlayer = class {
7102
6971
  this.visemePlayer.cleanup();
7103
6972
  if (this.audioElement) {
7104
6973
  this.audioElement.srcObject = null;
7105
- this.audioElement.src = "";
6974
+ this.audioElement.removeAttribute("src");
6975
+ this.audioElement.load();
7106
6976
  }
7107
6977
  this.streams.forEach((stream) => stream.dispose());
7108
6978
  this.streams.clear();
@@ -7119,8 +6989,10 @@ var AudioPlayer = class {
7119
6989
  this.streams.delete(streamId);
7120
6990
  }
7121
6991
  buildSilentWav = function() {
7122
- const sampleCount = 8;
7123
- const bytes = new Uint8Array(44 + sampleCount);
6992
+ const sampleRate = 44100;
6993
+ const bytesPerSample = 2;
6994
+ const dataSize = Math.round(sampleRate * 0.1) * bytesPerSample;
6995
+ const bytes = new Uint8Array(44 + dataSize);
7124
6996
  const view = new DataView(bytes.buffer);
7125
6997
  const writeText = function(offset, value) {
7126
6998
  Array.from(value).forEach((character, index) => {
@@ -7128,19 +7000,18 @@ var AudioPlayer = class {
7128
7000
  });
7129
7001
  };
7130
7002
  writeText(0, "RIFF");
7131
- view.setUint32(4, 36 + sampleCount, true);
7003
+ view.setUint32(4, 36 + dataSize, true);
7132
7004
  writeText(8, "WAVE");
7133
7005
  writeText(12, "fmt ");
7134
7006
  view.setUint32(16, 16, true);
7135
7007
  view.setUint16(20, 1, true);
7136
7008
  view.setUint16(22, 1, true);
7137
- view.setUint32(24, 8e3, true);
7138
- view.setUint32(28, 8e3, true);
7139
- view.setUint16(32, 1, true);
7140
- view.setUint16(34, 8, true);
7009
+ view.setUint32(24, sampleRate, true);
7010
+ view.setUint32(28, sampleRate * bytesPerSample, true);
7011
+ view.setUint16(32, bytesPerSample, true);
7012
+ view.setUint16(34, bytesPerSample * 8, true);
7141
7013
  writeText(36, "data");
7142
- view.setUint32(40, sampleCount, true);
7143
- bytes.fill(128, 44);
7014
+ view.setUint32(40, dataSize, true);
7144
7015
  return new Blob([bytes], { type: "audio/wav" });
7145
7016
  };
7146
7017
  async resolveSourceUrl(source) {
@@ -7176,6 +7047,9 @@ var AudioPlayer = class {
7176
7047
  element.volume = this.volumeValue;
7177
7048
  element.muted = this.mutedValue;
7178
7049
  element.playbackRate = this.playbackRateValue;
7050
+ if (needsManagedMediaSource()) {
7051
+ element.disableRemotePlayback = true;
7052
+ }
7179
7053
  if (this.crossOriginValue !== null) {
7180
7054
  element.crossOrigin = this.crossOriginValue;
7181
7055
  }
@@ -7259,6 +7133,14 @@ var CharacterExpressions = class {
7259
7133
  clearEmotion(mouthMode) {
7260
7134
  this.mixer.clearEmotion(mouthMode);
7261
7135
  }
7136
+ /** Set a temporary emotion overlay. Timeline emotions and visemes keep their state. */
7137
+ setOverlay(weights) {
7138
+ this.mixer.setOverlay(weights);
7139
+ }
7140
+ /** Ease the temporary overlay back to the current timeline or idle face. */
7141
+ clearOverlay() {
7142
+ this.mixer.clearOverlay();
7143
+ }
7262
7144
  /** Reset all expression layer targets to zero. */
7263
7145
  reset() {
7264
7146
  this.mixer.reset();
@@ -7433,7 +7315,7 @@ var FacialPerformanceInspector = class {
7433
7315
  };
7434
7316
 
7435
7317
  // ../../clients/web-client/src/character/idle-face.ts
7436
- import { Vector3 as Vector34 } from "three";
7318
+ import { Vector3 as Vector35 } from "three";
7437
7319
  var IdleFace = class {
7438
7320
  constructor(vrm, random) {
7439
7321
  this.vrm = vrm;
@@ -7441,7 +7323,7 @@ var IdleFace = class {
7441
7323
  }
7442
7324
  time = 0;
7443
7325
  timeline;
7444
- targetPosition = new Vector34();
7326
+ targetPosition = new Vector35();
7445
7327
  /** Advances by `delta` seconds, turns the eyes, and returns the blink weight. */
7446
7328
  update(delta, frame) {
7447
7329
  if (Number.isFinite(delta) && delta > 0) {
@@ -7629,6 +7511,7 @@ var PerformanceWeightMixer = class {
7629
7511
  };
7630
7512
 
7631
7513
  // ../../clients/web-client/src/character/facial-performance-mixer.ts
7514
+ var VisemeResponseRate = 30;
7632
7515
  var DEFAULT_PLAY_DURATION_MS = 1500;
7633
7516
  var MOUTH_CHANNELS = [
7634
7517
  VRMExpressionPresetName.Aa,
@@ -7656,7 +7539,8 @@ var FacialPerformanceLayer = {
7656
7539
  Emotion: "emotion",
7657
7540
  Viseme: "viseme",
7658
7541
  Blink: "blink",
7659
- Gaze: "gaze"
7542
+ Gaze: "gaze",
7543
+ Overlay: "overlay"
7660
7544
  };
7661
7545
  var getFacialPerformanceLayerId = function(channel) {
7662
7546
  const normalized = channel.toLowerCase();
@@ -7807,6 +7691,22 @@ var FacialPerformanceMixer = class {
7807
7691
  }
7808
7692
  this.releaseVisemeMouthMode();
7809
7693
  }
7694
+ /** Blend temporary emotions without changing timeline targets or mouth ownership. */
7695
+ setOverlay(weights) {
7696
+ this.clearOverlay();
7697
+ Object.entries(weights).forEach(([channel, weight]) => {
7698
+ if (this.availableChannelSet.has(channel) && getFacialPerformanceLayerId(channel) === FacialPerformanceLayer.Emotion && Number.isFinite(weight)) {
7699
+ this.weightMixer.setTarget(
7700
+ FacialPerformanceLayer.Overlay,
7701
+ channel,
7702
+ weight
7703
+ );
7704
+ }
7705
+ });
7706
+ }
7707
+ clearOverlay() {
7708
+ this.weightMixer.clearLayerTargets(FacialPerformanceLayer.Overlay);
7709
+ }
7810
7710
  releaseVisemeMouthMode() {
7811
7711
  if (this.activeMouthMode !== "viseme") {
7812
7712
  return;
@@ -7924,7 +7824,9 @@ var FacialPerformanceMixer = class {
7924
7824
  this.weightMixer.registerLayer({
7925
7825
  id: FacialPerformanceLayer.Viseme,
7926
7826
  priority: 200,
7927
- blendMode: "override"
7827
+ blendMode: "override",
7828
+ // A 33 ms time constant: an 80 ms lip closure reaches 90% closed.
7829
+ responseRate: VisemeResponseRate
7928
7830
  });
7929
7831
  this.weightMixer.registerLayer({
7930
7832
  id: FacialPerformanceLayer.Blink,
@@ -7936,6 +7838,11 @@ var FacialPerformanceMixer = class {
7936
7838
  priority: 400,
7937
7839
  blendMode: "override"
7938
7840
  });
7841
+ this.weightMixer.registerLayer({
7842
+ id: FacialPerformanceLayer.Overlay,
7843
+ priority: 500,
7844
+ blendMode: "maximum"
7845
+ });
7939
7846
  }
7940
7847
  updateIdleFace(delta, values) {
7941
7848
  const manager = this.vrm?.expressionManager;
@@ -8060,10 +7967,11 @@ var FacialPerformanceMixer = class {
8060
7967
 
8061
7968
  // ../../clients/web-client/src/character/mesh.ts
8062
7969
  import {
8063
- Euler,
7970
+ Euler as Euler2,
8064
7971
  Mesh,
7972
+ Object3D,
8065
7973
  SkinnedMesh,
8066
- Vector3 as Vector35
7974
+ Vector3 as Vector36
8067
7975
  } from "three";
8068
7976
  var isSkinnedMesh = function(object) {
8069
7977
  return object instanceof SkinnedMesh || object.isSkinnedMesh === true;
@@ -8072,19 +7980,30 @@ var isMesh = function(object) {
8072
7980
  return isSkinnedMesh(object) || object instanceof Mesh || object.isMesh === true;
8073
7981
  };
8074
7982
  var applyVector3 = function(target, value) {
8075
- if (value instanceof Vector35) {
7983
+ if (value instanceof Vector36) {
8076
7984
  target.copy(value);
8077
7985
  return;
8078
7986
  }
8079
7987
  target.set(value[0], value[1], value[2]);
8080
7988
  };
8081
7989
  var applyEuler = function(target, value) {
8082
- if (value instanceof Euler) {
7990
+ if (value instanceof Euler2) {
8083
7991
  target.copy(value);
8084
7992
  return;
8085
7993
  }
8086
7994
  target.set(value[0], value[1], value[2]);
8087
7995
  };
7996
+ var isObject3D = function(value) {
7997
+ return value instanceof Object3D || typeof value === "object" && value !== null && value.isObject3D === true;
7998
+ };
7999
+ var finiteCoordinate = function(value, axis) {
8000
+ if (typeof value !== "number" || !Number.isFinite(value)) {
8001
+ throw new RangeError(
8002
+ `lookAt needs a finite ${axis} coordinate, not ${String(value)}`
8003
+ );
8004
+ }
8005
+ return value;
8006
+ };
8088
8007
  var CharacterSceneRig = class {
8089
8008
  currentVrm = null;
8090
8009
  cachedMeshes = [];
@@ -8093,10 +8012,14 @@ var CharacterSceneRig = class {
8093
8012
  cachedSkeletons = [];
8094
8013
  shadowOptions = null;
8095
8014
  transformOptions = null;
8015
+ lookAtAnchor = null;
8016
+ /** The target that setVrmLookAtTarget set on the current model. */
8017
+ vrmLookAtTarget = null;
8096
8018
  get vrm() {
8097
8019
  return this.currentVrm;
8098
8020
  }
8099
8021
  setVrm(vrm) {
8022
+ this.setVrmLookAtTarget(null);
8100
8023
  this.currentVrm = vrm;
8101
8024
  this.refreshMeshes();
8102
8025
  if (this.shadowOptions) {
@@ -8106,6 +8029,64 @@ var CharacterSceneRig = class {
8106
8029
  this.applyTransform(this.transformOptions);
8107
8030
  }
8108
8031
  }
8032
+ /**
8033
+ * The `Object3D` of a `lookAt()` argument: an `Object3D` as it is, a point
8034
+ * or three coordinates as a world position that this rig keeps, or null.
8035
+ * Coordinates that are not finite numbers throw a `RangeError` and change
8036
+ * nothing.
8037
+ */
8038
+ resolveLookAtTarget(targetOrX, y, z) {
8039
+ if (targetOrX === null) {
8040
+ return null;
8041
+ }
8042
+ if (typeof targetOrX === "number") {
8043
+ return this.updateLookAtAnchor(targetOrX, y, z);
8044
+ }
8045
+ if (isObject3D(targetOrX)) {
8046
+ return targetOrX;
8047
+ }
8048
+ if (Array.isArray(targetOrX)) {
8049
+ return this.updateLookAtAnchor(targetOrX[0], targetOrX[1], targetOrX[2]);
8050
+ }
8051
+ if (typeof targetOrX === "object") {
8052
+ const point = targetOrX;
8053
+ return this.updateLookAtAnchor(point.x, point.y, point.z);
8054
+ }
8055
+ throw new TypeError(
8056
+ "lookAt needs an Object3D, a point, [x, y, z] or three coordinates"
8057
+ );
8058
+ }
8059
+ /**
8060
+ * Sets three-vrm's own look-at target (`vrm.lookAt.target`) of the current
8061
+ * model. Null removes the target that this method set; a target that other
8062
+ * code set stays. The IdleFace of a template face does not move the eyes
8063
+ * while `vrm.lookAt.target` is set.
8064
+ */
8065
+ setVrmLookAtTarget(target) {
8066
+ const lookAt = this.currentVrm?.lookAt;
8067
+ if (lookAt) {
8068
+ if (target) {
8069
+ lookAt.target = target;
8070
+ } else if (this.vrmLookAtTarget && lookAt.target === this.vrmLookAtTarget) {
8071
+ lookAt.target = void 0;
8072
+ }
8073
+ }
8074
+ this.vrmLookAtTarget = lookAt ? target : null;
8075
+ }
8076
+ updateLookAtAnchor(x, y, z) {
8077
+ const position = [
8078
+ finiteCoordinate(x, "x"),
8079
+ finiteCoordinate(y, "y"),
8080
+ finiteCoordinate(z, "z")
8081
+ ];
8082
+ if (!this.lookAtAnchor) {
8083
+ this.lookAtAnchor = new Object3D();
8084
+ this.lookAtAnchor.name = "npcify-look-at-target";
8085
+ }
8086
+ this.lookAtAnchor.position.set(...position);
8087
+ this.lookAtAnchor.updateMatrixWorld(true);
8088
+ return this.lookAtAnchor;
8089
+ }
8109
8090
  /** Root Three.js group containing the VRM hierarchy (`vrm.scene`). */
8110
8091
  get scene() {
8111
8092
  return this.currentVrm?.scene ?? null;
@@ -8296,6 +8277,11 @@ var IncrementalStaging = class {
8296
8277
  const dialogue = timeline.dialogue.map((next, index) => {
8297
8278
  const previous = this.timeline.dialogue[index];
8298
8279
  if (previous?.type === "speech" && previous.pending && next.type === "speech") {
8280
+ for (const key of Object.keys(previous)) {
8281
+ if (!(key in next)) {
8282
+ Reflect.deleteProperty(previous, key);
8283
+ }
8284
+ }
8299
8285
  return Object.assign(previous, next, { pending: next.pending });
8300
8286
  }
8301
8287
  return next;
@@ -8358,6 +8344,30 @@ var IncrementalStaging = class {
8358
8344
  }
8359
8345
  };
8360
8346
 
8347
+ // ../../clients/web-client/src/character/playback-clock.ts
8348
+ var PlaybackClock = class {
8349
+ pausedAt = null;
8350
+ pausedSeconds = 0;
8351
+ wallTime() {
8352
+ return performance.now() / 1e3;
8353
+ }
8354
+ get paused() {
8355
+ return this.pausedAt !== null;
8356
+ }
8357
+ now() {
8358
+ return (this.pausedAt ?? this.wallTime()) - this.pausedSeconds;
8359
+ }
8360
+ pause() {
8361
+ this.pausedAt ??= this.wallTime();
8362
+ }
8363
+ resume() {
8364
+ if (this.pausedAt !== null) {
8365
+ this.pausedSeconds += this.wallTime() - this.pausedAt;
8366
+ this.pausedAt = null;
8367
+ }
8368
+ }
8369
+ };
8370
+
8361
8371
  // ../../clients/web-client/src/character/staging-character-cues.ts
8362
8372
  var findCharacterSequence = function(characters, sequence, fromIndex) {
8363
8373
  if (sequence.length === 0) {
@@ -8459,12 +8469,6 @@ var buildStagingCharacterCues = function(text, wordTimings, duration, characterT
8459
8469
  };
8460
8470
 
8461
8471
  // ../../clients/web-client/src/character/staging-player.ts
8462
- var nowSeconds = function() {
8463
- if (typeof performance !== "undefined") {
8464
- return performance.now() / 1e3;
8465
- }
8466
- return Date.now() / 1e3;
8467
- };
8468
8472
  var abortError2 = function() {
8469
8473
  const error = new Error("Staging playback was cancelled");
8470
8474
  error.name = "AbortError";
@@ -8496,6 +8500,8 @@ var StagingPlayer = class {
8496
8500
  this.facialMixer = facialMixer;
8497
8501
  this.animationController = animationController;
8498
8502
  }
8503
+ clock = new PlaybackClock();
8504
+ pauseRevision = 0;
8499
8505
  playbackController = null;
8500
8506
  generation = 0;
8501
8507
  incremental = null;
@@ -8505,6 +8511,40 @@ var StagingPlayer = class {
8505
8511
  activeExpression = null;
8506
8512
  activeExpressionMouthMode = null;
8507
8513
  activeAnimation = null;
8514
+ get paused() {
8515
+ return this.clock.paused;
8516
+ }
8517
+ pause() {
8518
+ this.pauseRevision += 1;
8519
+ this.clock.pause();
8520
+ this.audioPlayer.element?.pause();
8521
+ }
8522
+ async resume() {
8523
+ if (!this.paused) {
8524
+ return;
8525
+ }
8526
+ const generation = this.generation;
8527
+ const pauseRevision = this.pauseRevision;
8528
+ if (this.pendingSpeech?.started && this.pendingSpeech.audio.type !== "silent") {
8529
+ await this.audioPlayer.element?.play();
8530
+ }
8531
+ if (generation !== this.generation) {
8532
+ return;
8533
+ }
8534
+ if (pauseRevision !== this.pauseRevision) {
8535
+ this.audioPlayer.element?.pause();
8536
+ return;
8537
+ }
8538
+ this.clock.resume();
8539
+ }
8540
+ async waitUntilResumed(signal) {
8541
+ while (this.paused) {
8542
+ await waitFrame(signal);
8543
+ }
8544
+ if (signal.aborted) {
8545
+ throw abortError2();
8546
+ }
8547
+ }
8508
8548
  async play(timeline, options) {
8509
8549
  this.cancel(false);
8510
8550
  const controller = new AbortController();
@@ -8543,6 +8583,7 @@ var StagingPlayer = class {
8543
8583
  if (controller.signal.aborted) {
8544
8584
  throw abortError2();
8545
8585
  }
8586
+ await this.waitUntilResumed(controller.signal);
8546
8587
  if (block.startTime > cursor) {
8547
8588
  beginTimeline();
8548
8589
  await this.playTimedRange(
@@ -8626,12 +8667,12 @@ var StagingPlayer = class {
8626
8667
  }
8627
8668
  }
8628
8669
  startCursor(block, mediaClock) {
8629
- const startedAt = nowSeconds();
8670
+ const startedAt = this.clock.now();
8630
8671
  this.cursor = () => ({
8631
8672
  blockId: block.id,
8632
8673
  offsetSeconds: Math.min(
8633
8674
  block.endTime - block.startTime,
8634
- Math.max(0, mediaClock ? mediaClock() : nowSeconds() - startedAt)
8675
+ Math.max(0, mediaClock ? mediaClock() : this.clock.now() - startedAt)
8635
8676
  )
8636
8677
  });
8637
8678
  this.onCursor?.(this.cursor());
@@ -8655,6 +8696,7 @@ var StagingPlayer = class {
8655
8696
  this.generation += 1;
8656
8697
  this.playbackController?.abort();
8657
8698
  this.playbackController = null;
8699
+ this.clock.resume();
8658
8700
  this.incremental = null;
8659
8701
  this.clearCursor();
8660
8702
  this.onCursor = void 0;
@@ -8766,6 +8808,9 @@ var StagingPlayer = class {
8766
8808
  return;
8767
8809
  }
8768
8810
  started = true;
8811
+ if (this.paused) {
8812
+ this.audioPlayer.element?.pause();
8813
+ }
8769
8814
  if (this.pendingSpeech?.id === speech.id) {
8770
8815
  this.pendingSpeech.started = true;
8771
8816
  }
@@ -8786,6 +8831,7 @@ var StagingPlayer = class {
8786
8831
  let ticking = true;
8787
8832
  const tick = async () => {
8788
8833
  while (ticking && !signal.aborted) {
8834
+ await this.waitUntilResumed(signal);
8789
8835
  if (started) {
8790
8836
  observePhonemes();
8791
8837
  const localTime = Math.min(
@@ -8824,14 +8870,15 @@ var StagingPlayer = class {
8824
8870
  }
8825
8871
  }
8826
8872
  async playSilentSpeech(speech, timeline, signal, revealCharacters) {
8827
- const startedAt = nowSeconds();
8873
+ const startedAt = this.clock.now();
8828
8874
  const duration = Math.max(speech.endTime - speech.startTime, 0);
8829
8875
  let nextVisemeIndex = 0;
8830
8876
  while (true) {
8831
8877
  if (signal.aborted) {
8832
8878
  throw abortError2();
8833
8879
  }
8834
- const elapsed = Math.min(nowSeconds() - startedAt, duration);
8880
+ await this.waitUntilResumed(signal);
8881
+ const elapsed = Math.min(this.clock.now() - startedAt, duration);
8835
8882
  this.applyTracks(speech.startTime + elapsed, true, timeline);
8836
8883
  revealCharacters(elapsed);
8837
8884
  let viseme = speech.visemes[nextVisemeIndex];
@@ -8851,13 +8898,14 @@ var StagingPlayer = class {
8851
8898
  revealCharacters(Number.POSITIVE_INFINITY);
8852
8899
  }
8853
8900
  async playTimedRange(startTime, endTime, speechActive, timeline, signal) {
8854
- const startedAt = nowSeconds();
8901
+ const startedAt = this.clock.now();
8855
8902
  const duration = Math.max(endTime - startTime, 0);
8856
8903
  while (true) {
8857
8904
  if (signal.aborted) {
8858
8905
  throw abortError2();
8859
8906
  }
8860
- const elapsed = Math.min(nowSeconds() - startedAt, duration);
8907
+ await this.waitUntilResumed(signal);
8908
+ const elapsed = Math.min(this.clock.now() - startedAt, duration);
8861
8909
  this.applyTracks(startTime + elapsed, speechActive, timeline);
8862
8910
  if (elapsed >= duration) {
8863
8911
  return;
@@ -8886,8 +8934,9 @@ var StagingPlayer = class {
8886
8934
  if (animation?.animation === this.activeAnimation?.animation && animation?.startTime === this.activeAnimation?.startTime && animation?.loop === this.activeAnimation?.loop) {
8887
8935
  return;
8888
8936
  }
8937
+ const continuesIdle = !this.activeAnimation && animation?.startTime === 0 && animation?.animation === this.animationController.getIdleAnimationId?.();
8889
8938
  this.activeAnimation = animation ? { ...animation } : null;
8890
- if (animation) {
8939
+ if (animation && !continuesIdle) {
8891
8940
  this.animationController.playAnimation(
8892
8941
  animation.animation,
8893
8942
  0.25,
@@ -9338,6 +9387,8 @@ var CharacterRuntime = class {
9338
9387
  streamPlayback;
9339
9388
  defaultParent;
9340
9389
  currentGazeTarget = null;
9390
+ /** The current gaze target came from `lookAt()`. */
9391
+ lookAtActive = false;
9341
9392
  thinkingState = false;
9342
9393
  errorCallback;
9343
9394
  constructor(options) {
@@ -9354,13 +9405,10 @@ var CharacterRuntime = class {
9354
9405
  animationId,
9355
9406
  assetProfile: this.assetProfile
9356
9407
  }));
9357
- this.lazyAnimations = new LazyAnimations(
9358
- loadAnimation,
9359
- (id, animation) => {
9360
- this.animations.set(id, animation);
9361
- this.animationController?.addAnimation?.(id, animation, { loop: false });
9362
- }
9363
- );
9408
+ this.lazyAnimations = new LazyAnimations(loadAnimation, (id, animation) => {
9409
+ this.animations.set(id, animation);
9410
+ this.animationController?.addAnimation?.(id, animation, { loop: false });
9411
+ });
9364
9412
  this.performanceRecorder = new PerformanceRecorder({
9365
9413
  characterId: this.characterId
9366
9414
  });
@@ -9427,6 +9475,7 @@ var CharacterRuntime = class {
9427
9475
  set vrm(value) {
9428
9476
  this.sceneRig.setVrm(value);
9429
9477
  this.facialMixer.setVrm(value);
9478
+ this.applyGaze();
9430
9479
  if (value && this.defaultParent && !value.scene?.parent) {
9431
9480
  this.sceneRig.attachTo(this.defaultParent);
9432
9481
  }
@@ -9579,6 +9628,20 @@ var CharacterRuntime = class {
9579
9628
  });
9580
9629
  return unlocked;
9581
9630
  }
9631
+ /** Freeze the current NPCP timeline, audio, face, and animation. */
9632
+ pausePlayback() {
9633
+ this.stagingPlayer?.pause();
9634
+ }
9635
+ /** Continue the same timeline. Call from a gesture if the browser requires it. */
9636
+ async resumePlayback() {
9637
+ await this.stagingPlayer?.resume();
9638
+ }
9639
+ /** Cancel current output and pending audio/animation loads. */
9640
+ interruptPlayback() {
9641
+ if (this.animationController) {
9642
+ this.streamPlayback.cleanup();
9643
+ }
9644
+ }
9582
9645
  getExpressions() {
9583
9646
  return this.facialMixer.getChannelNames();
9584
9647
  }
@@ -9603,10 +9666,38 @@ var CharacterRuntime = class {
9603
9666
  clearEmotion(mouthMode) {
9604
9667
  this.facialMixer.clearEmotion(mouthMode);
9605
9668
  }
9606
- /** The object the eyes of a template face follow (the camera); null looks ahead. */
9669
+ /**
9670
+ * The object the eyes of a template face follow (the camera); null looks
9671
+ * ahead. It replaces a `lookAt()` target. Other faces do not follow it.
9672
+ */
9607
9673
  setGazeTarget(target) {
9608
9674
  this.currentGazeTarget = target;
9609
- this.facialMixer.setGazeTarget(target);
9675
+ this.lookAtActive = false;
9676
+ this.applyGaze();
9677
+ }
9678
+ lookAt(targetOrX, y, z) {
9679
+ const target = this.sceneRig.resolveLookAtTarget(targetOrX, y, z);
9680
+ this.currentGazeTarget = target;
9681
+ this.lookAtActive = target !== null;
9682
+ this.applyGaze();
9683
+ }
9684
+ /** Reset the character's gaze target so the eyes look straight ahead. */
9685
+ clearLookAt() {
9686
+ this.setGazeTarget(null);
9687
+ }
9688
+ /** Briefly look at an object; release restores the prior target and gaze mode. */
9689
+ lookAtTemporarily(target) {
9690
+ const previous = this.currentGazeTarget;
9691
+ const wasActive = this.lookAtActive;
9692
+ this.lookAt(target);
9693
+ return () => {
9694
+ if (this.currentGazeTarget !== target || !this.lookAtActive) {
9695
+ return;
9696
+ }
9697
+ this.currentGazeTarget = previous;
9698
+ this.lookAtActive = wasActive;
9699
+ this.applyGaze();
9700
+ };
9610
9701
  }
9611
9702
  startPerformanceRecording(options) {
9612
9703
  this.performanceRecorder.start(options);
@@ -9656,6 +9747,9 @@ var CharacterRuntime = class {
9656
9747
  }
9657
9748
  /** Advance skeletal animations, lip-sync visemes, facial expressions, and VRM physics by `delta` seconds. */
9658
9749
  update(delta) {
9750
+ if (this.stagingPlayer?.paused) {
9751
+ return;
9752
+ }
9659
9753
  this.animationController?.update(delta);
9660
9754
  this.visemePlayer.update(delta);
9661
9755
  this.facialMixer.update(delta);
@@ -9688,15 +9782,24 @@ var CharacterRuntime = class {
9688
9782
  }
9689
9783
  return this.stagingPlayer;
9690
9784
  };
9785
+ /**
9786
+ * A template face follows the gaze target with its IdleFace. Only a
9787
+ * `lookAt()` target moves the eyes of another face, through three-vrm's
9788
+ * `vrm.lookAt.target`; on a template face that target would stop the
9789
+ * IdleFace (its gaze cone, idle eye moves and look cues).
9790
+ */
9791
+ applyGaze() {
9792
+ this.facialMixer.setGazeTarget(this.currentGazeTarget);
9793
+ this.sceneRig.setVrmLookAtTarget(
9794
+ this.lookAtActive && !this.facialMixer.hasIdleFace ? this.currentGazeTarget : null
9795
+ );
9796
+ }
9691
9797
  resetRuntimeAssets() {
9692
9798
  this.lazyAnimations.reset();
9693
- if (this.stagingPlayer) {
9694
- this.stagingPlayer.cleanup();
9695
- this.stagingPlayer = null;
9696
- } else {
9697
- this.audioPlayer?.reset();
9698
- this.facialMixer.reset();
9699
- }
9799
+ this.stagingPlayer?.cleanup();
9800
+ this.stagingPlayer = null;
9801
+ this.audioPlayer?.reset();
9802
+ this.facialMixer.reset();
9700
9803
  if (this.animationController instanceof AnimationController) {
9701
9804
  this.animationController.clearAnimations();
9702
9805
  }
@@ -9706,20 +9809,10 @@ var CharacterRuntime = class {
9706
9809
  this.facialMixer.setVrm(null);
9707
9810
  }
9708
9811
  disposeRuntimeAssets() {
9709
- this.lazyAnimations.reset();
9710
- if (this.stagingPlayer) {
9711
- this.stagingPlayer.cleanup();
9712
- this.stagingPlayer = null;
9713
- }
9812
+ this.sceneRig.disposeVrm();
9813
+ this.resetRuntimeAssets();
9714
9814
  this.audioPlayer?.cleanup();
9715
9815
  this.audioPlayer = null;
9716
- if (this.animationController instanceof AnimationController) {
9717
- this.animationController.clearAnimations();
9718
- }
9719
- this.animationController = void 0;
9720
- this.animations.clear();
9721
- this.facialMixer.setVrm(null);
9722
- this.sceneRig.disposeVrm();
9723
9816
  }
9724
9817
  handleCallbackError = (context2, error) => {
9725
9818
  const normalized = error instanceof Error ? error : new Error(String(error ?? "Error"));
@@ -9746,6 +9839,18 @@ var ConversationSocket = class {
9746
9839
  this.socket?.disconnect();
9747
9840
  this.socket = null;
9748
9841
  }
9842
+ /** An event with an acknowledgement. Needs `ensureConnected` first. */
9843
+ async request(event, payload, timeoutMs = 1e4) {
9844
+ const socket = this.socket;
9845
+ if (!socket?.connected) {
9846
+ throw new Error("The conversation connection is not open.");
9847
+ }
9848
+ return await socket.timeout(timeoutMs).emitWithAck(event, payload);
9849
+ }
9850
+ /** An event without an acknowledgement, in order with the other events. */
9851
+ send(event, payload) {
9852
+ this.socket?.emit(event, payload);
9853
+ }
9749
9854
  async ensureConnected(conversationId) {
9750
9855
  if (typeof window === "undefined") {
9751
9856
  return;
@@ -9800,6 +9905,41 @@ var ConversationSocket = class {
9800
9905
  }
9801
9906
  this.options.handleStreamEvent(event);
9802
9907
  });
9908
+ this.socket.on(SPEECH_EVENTS.partial, (payload) => {
9909
+ const event = payload;
9910
+ if (typeof event?.sessionId === "string" && typeof event.text === "string") {
9911
+ this.options.handleSpeechEvent?.({ type: "partial", event });
9912
+ }
9913
+ });
9914
+ this.socket.on(SPEECH_EVENTS.turn, (payload) => {
9915
+ let candidate;
9916
+ try {
9917
+ candidate = decodePerformance(
9918
+ payload,
9919
+ "turn"
9920
+ );
9921
+ } catch (error) {
9922
+ this.options.handleCallbackError("speech.turn.decode", error);
9923
+ return;
9924
+ }
9925
+ if (candidate?.conversationId !== this.options.getConversationId() || typeof candidate.requestId !== "string" || !candidate.turn || typeof candidate.turn !== "object") {
9926
+ return;
9927
+ }
9928
+ this.options.handleSpeechEvent?.({
9929
+ type: "turn",
9930
+ requestId: candidate.requestId,
9931
+ turn: candidate.turn
9932
+ });
9933
+ });
9934
+ this.socket.on(SPEECH_EVENTS.turnFailed, (payload) => {
9935
+ const event = payload;
9936
+ if (typeof event?.requestId === "string" && event.conversationId === this.options.getConversationId()) {
9937
+ this.options.handleSpeechEvent?.({ type: "turnFailed", event });
9938
+ }
9939
+ });
9940
+ this.socket.on("disconnect", () => {
9941
+ this.options.handleSpeechEvent?.({ type: "disconnect" });
9942
+ });
9803
9943
  this.socket.on("conversations:turn", (payload) => {
9804
9944
  let candidate;
9805
9945
  try {
@@ -9852,56 +9992,1056 @@ var ConversationSocket = class {
9852
9992
  }
9853
9993
  };
9854
9994
 
9855
- // ../../clients/web-client/src/npcify.ts
9856
- var Character = class extends CharacterRuntime {
9857
- conversationSocket;
9858
- renderSessionId = null;
9859
- initialAssetInfo;
9860
- stagingResultCallback = null;
9861
- thinkingCallback = null;
9862
- notifiedStagingIds = /* @__PURE__ */ new Map();
9863
- liveConversation;
9864
- baked = null;
9865
- authoring;
9866
- bakedAsset;
9867
- animationRefreshId = 0;
9868
- conversationId = null;
9869
- constructor(options) {
9870
- const resolvedToken = (options.apiToken ?? options.apiKey)?.trim() ?? "";
9871
- const resolvedProfile = options.assetProfile ?? "original";
9872
- super({
9873
- ...options,
9874
- apiToken: resolvedToken,
9875
- assetProfile: resolvedProfile,
9876
- loadAnimation: (animationId) => loadVrma({
9877
- baseUrl: options.baseUrl,
9878
- characterId: options.characterId,
9879
- apiToken: resolvedToken,
9880
- animationId,
9881
- assetProfile: resolvedProfile
9882
- })
9883
- });
9884
- this.authoring = options.authoring ?? false;
9885
- this.bakedAsset = options.bakedConversation;
9886
- this.initialAssetInfo = options.initialAssetInfo ?? null;
9887
- this.conversationSocket = new ConversationSocket({
9888
- baseUrl: this.baseUrl,
9889
- apiToken: this.apiToken,
9890
- getConversationId: () => this.conversationId,
9891
- handleStreamEvent: (event) => {
9892
- if (event.type !== "text_delta") {
9893
- this.thinkingState = false;
9995
+ // ../../clients/web-client/src/voice/push-to-talk.ts
9996
+ var INTERACTIVE = 'input, textarea, select, button, a[href], summary, [contenteditable]:not([contenteditable="false"]), [role="button"], [role="textbox"], [role="checkbox"], [role="switch"], [role="menuitem"], [role="tab"], [role="slider"], [role="option"]';
9997
+ var elementOf = function(target) {
9998
+ return target && typeof target.closest === "function" ? target : null;
9999
+ };
10000
+ var bindPushToTalk = function(target, options = {}) {
10001
+ const element = options.element ?? null;
10002
+ const key = options.key === void 0 ? "Space" : options.key;
10003
+ const minHoldMs = options.minHoldMs ?? SPEECH_MIN_HOLD_MS;
10004
+ let active = null;
10005
+ let pointerId = null;
10006
+ let pressedAt = 0;
10007
+ const begin = function(source) {
10008
+ if (active || target.listeningState !== "idle") {
10009
+ return false;
10010
+ }
10011
+ if (options.enabled && !options.enabled()) {
10012
+ return false;
10013
+ }
10014
+ active = source;
10015
+ pressedAt = performance.now();
10016
+ const listening = typeof options.listening === "function" ? options.listening() : options.listening ?? {};
10017
+ target.startListening({ ...listening, endOnSilenceMs: null }).then(
10018
+ (result) => {
10019
+ if (result) {
10020
+ options.onResult?.(result);
9894
10021
  }
9895
- this.streamPlayback.handleEvent(event);
9896
- },
9897
- handleTurn: async (turn) => {
9898
- await this.handleStagingResult(turn, { source: "turn" });
9899
10022
  },
9900
- handleCallbackError: this.handleCallbackError
9901
- });
9902
- this.liveConversation = new LiveConversation({
9903
- baseUrl: this.baseUrl,
9904
- apiToken: this.apiToken,
10023
+ (error) => {
10024
+ active = null;
10025
+ options.onError?.(error);
10026
+ }
10027
+ );
10028
+ return true;
10029
+ };
10030
+ const end = function() {
10031
+ if (!active) {
10032
+ return;
10033
+ }
10034
+ active = null;
10035
+ pointerId = null;
10036
+ if (performance.now() - pressedAt < minHoldMs) {
10037
+ target.cancelListening();
10038
+ options.onShortPress?.();
10039
+ return;
10040
+ }
10041
+ const starting = target.listeningState === "starting";
10042
+ target.stopListening().then(
10043
+ (result) => {
10044
+ if (starting && result === null) {
10045
+ options.onShortPress?.();
10046
+ }
10047
+ },
10048
+ () => void 0
10049
+ );
10050
+ };
10051
+ const cancel = function() {
10052
+ if (!active) {
10053
+ return;
10054
+ }
10055
+ active = null;
10056
+ pointerId = null;
10057
+ target.cancelListening();
10058
+ };
10059
+ const isOwnElement = function(node) {
10060
+ return Boolean(element && node && element.contains(node));
10061
+ };
10062
+ const onKeyDown = function(event) {
10063
+ if (event.key === "Escape" || event.code === "Escape") {
10064
+ cancel();
10065
+ return;
10066
+ }
10067
+ const node = elementOf(event.target);
10068
+ const ownElement = isOwnElement(node);
10069
+ const pressesElement = ownElement && (event.code === "Space" || event.code === "Enter");
10070
+ if (!pressesElement && (!key || event.code !== key)) {
10071
+ return;
10072
+ }
10073
+ if (event.repeat) {
10074
+ if (active === "key") {
10075
+ event.preventDefault();
10076
+ }
10077
+ return;
10078
+ }
10079
+ if (event.ctrlKey || event.metaKey || event.altKey || event.isComposing) {
10080
+ return;
10081
+ }
10082
+ if (!ownElement && node?.closest(INTERACTIVE)) {
10083
+ return;
10084
+ }
10085
+ if (begin("key")) {
10086
+ event.preventDefault();
10087
+ }
10088
+ };
10089
+ const onKeyUp = function(event) {
10090
+ if (active !== "key") {
10091
+ return;
10092
+ }
10093
+ const ownElement = isOwnElement(elementOf(event.target));
10094
+ if (event.code === key || ownElement && (event.code === "Space" || event.code === "Enter")) {
10095
+ event.preventDefault();
10096
+ end();
10097
+ }
10098
+ };
10099
+ const onVisibility = function() {
10100
+ if (document.visibilityState === "hidden") {
10101
+ cancel();
10102
+ }
10103
+ };
10104
+ const onPointerDown = function(event) {
10105
+ if (event.pointerType === "mouse" && event.button !== 0) {
10106
+ return;
10107
+ }
10108
+ event.preventDefault();
10109
+ if (!begin("pointer")) {
10110
+ return;
10111
+ }
10112
+ pointerId = event.pointerId;
10113
+ try {
10114
+ element?.setPointerCapture(event.pointerId);
10115
+ } catch {
10116
+ }
10117
+ };
10118
+ const onPointerUp = function(event) {
10119
+ if (active === "pointer" && event.pointerId === pointerId) {
10120
+ end();
10121
+ }
10122
+ };
10123
+ const onPointerCancel = function(event) {
10124
+ if (active === "pointer" && event.pointerId === pointerId) {
10125
+ cancel();
10126
+ }
10127
+ };
10128
+ const prevent = function(event) {
10129
+ event.preventDefault();
10130
+ };
10131
+ const restoreStyle = [];
10132
+ if (element) {
10133
+ for (const [property, value] of [
10134
+ ["touch-action", "none"],
10135
+ ["user-select", "none"],
10136
+ ["-webkit-user-select", "none"],
10137
+ ["-webkit-touch-callout", "none"]
10138
+ ]) {
10139
+ const previous = element.style.getPropertyValue(property);
10140
+ element.style.setProperty(property, value);
10141
+ restoreStyle.push(() => {
10142
+ if (previous) {
10143
+ element.style.setProperty(property, previous);
10144
+ } else {
10145
+ element.style.removeProperty(property);
10146
+ }
10147
+ });
10148
+ }
10149
+ element.addEventListener("pointerdown", onPointerDown);
10150
+ window.addEventListener("pointerup", onPointerUp, true);
10151
+ window.addEventListener("pointercancel", onPointerCancel, true);
10152
+ element.addEventListener("contextmenu", prevent);
10153
+ element.addEventListener("selectstart", prevent);
10154
+ element.addEventListener("touchstart", prevent, { passive: false });
10155
+ }
10156
+ window.addEventListener("keydown", onKeyDown);
10157
+ window.addEventListener("keyup", onKeyUp);
10158
+ window.addEventListener("blur", cancel);
10159
+ document.addEventListener("visibilitychange", onVisibility);
10160
+ return () => {
10161
+ cancel();
10162
+ if (element) {
10163
+ element.removeEventListener("pointerdown", onPointerDown);
10164
+ window.removeEventListener("pointerup", onPointerUp, true);
10165
+ window.removeEventListener("pointercancel", onPointerCancel, true);
10166
+ element.removeEventListener("contextmenu", prevent);
10167
+ element.removeEventListener("selectstart", prevent);
10168
+ element.removeEventListener("touchstart", prevent);
10169
+ restoreStyle.forEach((restore) => restore());
10170
+ }
10171
+ window.removeEventListener("keydown", onKeyDown);
10172
+ window.removeEventListener("keyup", onKeyUp);
10173
+ window.removeEventListener("blur", cancel);
10174
+ document.removeEventListener("visibilitychange", onVisibility);
10175
+ };
10176
+ };
10177
+
10178
+ // ../../clients/web-client/src/voice/microphone-capture.ts
10179
+ var FRAME_SAMPLES = SPEECH_AUDIO_FORMAT.sampleRate * SPEECH_FRAME_MS / 1e3;
10180
+ var MicrophoneError = class extends Error {
10181
+ constructor(code, message) {
10182
+ super(message);
10183
+ this.code = code;
10184
+ this.name = "NpcifyMicrophoneError";
10185
+ }
10186
+ };
10187
+ var Pcm16Framer = class {
10188
+ constructor(inputRate, emit) {
10189
+ this.emit = emit;
10190
+ this.ratio = inputRate / SPEECH_AUDIO_FORMAT.sampleRate;
10191
+ }
10192
+ ratio;
10193
+ frame = new Int16Array(FRAME_SAMPLES);
10194
+ length = 0;
10195
+ sum = 0;
10196
+ count = 0;
10197
+ phase = 0;
10198
+ push(samples) {
10199
+ for (let index = 0; index < samples.length; index += 1) {
10200
+ this.sum += samples[index] ?? 0;
10201
+ this.count += 1;
10202
+ this.phase += 1;
10203
+ while (this.phase >= this.ratio) {
10204
+ this.phase -= this.ratio;
10205
+ this.write(this.sum / Math.max(1, this.count));
10206
+ if (this.ratio >= 1) {
10207
+ this.sum = 0;
10208
+ this.count = 0;
10209
+ }
10210
+ }
10211
+ if (this.ratio < 1) {
10212
+ this.sum = 0;
10213
+ this.count = 0;
10214
+ }
10215
+ }
10216
+ }
10217
+ flush() {
10218
+ if (this.count > 0 && this.ratio >= 1) {
10219
+ this.write(this.sum / this.count);
10220
+ this.sum = 0;
10221
+ this.count = 0;
10222
+ }
10223
+ if (this.length > 0) {
10224
+ this.emit(this.frame.slice(0, this.length));
10225
+ this.length = 0;
10226
+ }
10227
+ }
10228
+ write(value) {
10229
+ const clamped = Math.max(-1, Math.min(1, value));
10230
+ this.frame[this.length] = Math.round(
10231
+ clamped < 0 ? clamped * 32768 : clamped * 32767
10232
+ );
10233
+ this.length += 1;
10234
+ if (this.length === FRAME_SAMPLES) {
10235
+ this.emit(this.frame);
10236
+ this.frame = new Int16Array(FRAME_SAMPLES);
10237
+ this.length = 0;
10238
+ }
10239
+ }
10240
+ };
10241
+ var levelOf = function(pcm) {
10242
+ if (pcm.length === 0) {
10243
+ return 0;
10244
+ }
10245
+ let sum = 0;
10246
+ for (const sample of pcm) {
10247
+ sum += (sample / 32768) ** 2;
10248
+ }
10249
+ return Math.sqrt(sum / pcm.length);
10250
+ };
10251
+ var WORKLET_SOURCE = `
10252
+ const FRAME = ${FRAME_SAMPLES}
10253
+ class NpcifyCaptureProcessor extends AudioWorkletProcessor {
10254
+ constructor() {
10255
+ super()
10256
+ this.ratio = sampleRate / ${SPEECH_AUDIO_FORMAT.sampleRate}
10257
+ this.frame = new Int16Array(FRAME)
10258
+ this.length = 0
10259
+ this.sum = 0
10260
+ this.count = 0
10261
+ this.phase = 0
10262
+ this.port.onmessage = (event) => {
10263
+ if (event.data === 'flush') {
10264
+ if (this.count > 0 && this.ratio >= 1) {
10265
+ this.write(this.sum / this.count)
10266
+ this.sum = 0
10267
+ this.count = 0
10268
+ }
10269
+ if (this.length > 0) {
10270
+ this.post(this.frame.slice(0, this.length))
10271
+ this.length = 0
10272
+ }
10273
+ this.port.postMessage({ type: 'flushed' })
10274
+ }
10275
+ }
10276
+ }
10277
+ write(value) {
10278
+ const clamped = Math.max(-1, Math.min(1, value))
10279
+ this.frame[this.length++] = Math.round(clamped < 0 ? clamped * 32768 : clamped * 32767)
10280
+ if (this.length === FRAME) {
10281
+ this.post(this.frame)
10282
+ this.frame = new Int16Array(FRAME)
10283
+ this.length = 0
10284
+ }
10285
+ }
10286
+ post(pcm) {
10287
+ this.port.postMessage({ type: 'frame', pcm }, [pcm.buffer])
10288
+ }
10289
+ process(inputs) {
10290
+ const channels = inputs[0]
10291
+ if (!channels || channels.length === 0) return true
10292
+ const size = channels[0].length
10293
+ for (let index = 0; index < size; index += 1) {
10294
+ let value = 0
10295
+ for (const channel of channels) value += channel[index]
10296
+ this.sum += value / channels.length
10297
+ this.count += 1
10298
+ this.phase += 1
10299
+ while (this.phase >= this.ratio) {
10300
+ this.phase -= this.ratio
10301
+ this.write(this.sum / Math.max(1, this.count))
10302
+ if (this.ratio >= 1) {
10303
+ this.sum = 0
10304
+ this.count = 0
10305
+ }
10306
+ }
10307
+ if (this.ratio < 1) {
10308
+ this.sum = 0
10309
+ this.count = 0
10310
+ }
10311
+ }
10312
+ return true
10313
+ }
10314
+ }
10315
+ registerProcessor('npcify-capture', NpcifyCaptureProcessor)
10316
+ `;
10317
+ var MicrophoneCapture = class {
10318
+ context = null;
10319
+ workletReady = null;
10320
+ graph = null;
10321
+ generation = 0;
10322
+ async start(onFrame) {
10323
+ this.cancel();
10324
+ const generation = ++this.generation;
10325
+ const AudioContextClass = globalThis.AudioContext ?? globalThis.webkitAudioContext;
10326
+ if (!navigator.mediaDevices?.getUserMedia || !AudioContextClass) {
10327
+ throw new MicrophoneError(
10328
+ "MICROPHONE_UNAVAILABLE",
10329
+ "This browser cannot record audio."
10330
+ );
10331
+ }
10332
+ this.context ??= new AudioContextClass({ latencyHint: "interactive" });
10333
+ const context2 = this.context;
10334
+ const resumed = context2.resume().catch(() => void 0);
10335
+ let stream;
10336
+ try {
10337
+ stream = await navigator.mediaDevices.getUserMedia({
10338
+ audio: {
10339
+ channelCount: 1,
10340
+ echoCancellation: true,
10341
+ noiseSuppression: true,
10342
+ autoGainControl: true
10343
+ }
10344
+ });
10345
+ } catch (error) {
10346
+ const name = error?.name;
10347
+ throw name === "NotAllowedError" || name === "SecurityError" ? new MicrophoneError(
10348
+ "MICROPHONE_PERMISSION_DENIED",
10349
+ "Microphone access was denied."
10350
+ ) : new MicrophoneError(
10351
+ "MICROPHONE_UNAVAILABLE",
10352
+ "No microphone is available."
10353
+ );
10354
+ }
10355
+ await resumed;
10356
+ if (generation !== this.generation) {
10357
+ stream.getTracks().forEach((track) => track.stop());
10358
+ return;
10359
+ }
10360
+ const deliver = (pcm) => onFrame({ pcm, level: levelOf(pcm) });
10361
+ const source = context2.createMediaStreamSource(stream);
10362
+ const sink = context2.createGain();
10363
+ sink.gain.value = 0;
10364
+ sink.connect(context2.destination);
10365
+ const graph = await this.workletGraph(context2, stream, source, sink, deliver) ?? this.scriptGraph(context2, stream, source, sink, deliver);
10366
+ if (generation !== this.generation) {
10367
+ this.close(graph, false);
10368
+ return;
10369
+ }
10370
+ this.graph = graph;
10371
+ }
10372
+ async stop() {
10373
+ const graph = this.graph;
10374
+ this.graph = null;
10375
+ this.generation += 1;
10376
+ if (!graph) {
10377
+ return;
10378
+ }
10379
+ try {
10380
+ await graph.flush();
10381
+ } finally {
10382
+ this.close(graph);
10383
+ }
10384
+ }
10385
+ cancel() {
10386
+ const graph = this.graph;
10387
+ this.graph = null;
10388
+ this.generation += 1;
10389
+ if (graph) {
10390
+ this.close(graph);
10391
+ }
10392
+ }
10393
+ close(graph, suspend = true) {
10394
+ graph.stream.getTracks().forEach((track) => track.stop());
10395
+ graph.source.disconnect();
10396
+ graph.node.disconnect();
10397
+ graph.sink.disconnect();
10398
+ if (graph.node instanceof AudioWorkletNode) {
10399
+ graph.node.port.onmessage = null;
10400
+ } else {
10401
+ graph.node.onaudioprocess = null;
10402
+ }
10403
+ if (suspend) {
10404
+ void this.context?.suspend().catch(() => void 0);
10405
+ }
10406
+ }
10407
+ async workletGraph(context2, stream, source, sink, deliver) {
10408
+ if (!context2.audioWorklet || typeof AudioWorkletNode === "undefined") {
10409
+ return null;
10410
+ }
10411
+ this.workletReady ??= (async () => {
10412
+ const url = URL.createObjectURL(
10413
+ new Blob([WORKLET_SOURCE], { type: "application/javascript" })
10414
+ );
10415
+ try {
10416
+ await context2.audioWorklet.addModule(url);
10417
+ return true;
10418
+ } catch {
10419
+ return false;
10420
+ } finally {
10421
+ URL.revokeObjectURL(url);
10422
+ }
10423
+ })();
10424
+ if (!await this.workletReady) {
10425
+ return null;
10426
+ }
10427
+ const node = new AudioWorkletNode(context2, "npcify-capture", {
10428
+ numberOfInputs: 1,
10429
+ numberOfOutputs: 1,
10430
+ channelCount: 1,
10431
+ channelCountMode: "explicit"
10432
+ });
10433
+ let flushed = null;
10434
+ node.port.onmessage = (event) => {
10435
+ if (event.data.type === "frame" && event.data.pcm) {
10436
+ deliver(event.data.pcm);
10437
+ } else if (event.data.type === "flushed") {
10438
+ flushed?.();
10439
+ }
10440
+ };
10441
+ source.connect(node);
10442
+ node.connect(sink);
10443
+ return {
10444
+ stream,
10445
+ source,
10446
+ node,
10447
+ sink,
10448
+ flush: () => new Promise((resolve) => {
10449
+ const timer = setTimeout(resolve, 250);
10450
+ flushed = () => {
10451
+ clearTimeout(timer);
10452
+ resolve();
10453
+ };
10454
+ node.port.postMessage("flush");
10455
+ })
10456
+ };
10457
+ }
10458
+ /** The fallback without AudioWorklet: a ScriptProcessorNode on the main thread. */
10459
+ scriptGraph(context2, stream, source, sink, deliver) {
10460
+ const framer = new Pcm16Framer(context2.sampleRate, deliver);
10461
+ const node = context2.createScriptProcessor(2048, 1, 1);
10462
+ node.onaudioprocess = (event) => {
10463
+ framer.push(event.inputBuffer.getChannelData(0));
10464
+ };
10465
+ source.connect(node);
10466
+ node.connect(sink);
10467
+ return {
10468
+ stream,
10469
+ source,
10470
+ node,
10471
+ sink,
10472
+ flush: () => {
10473
+ framer.flush();
10474
+ return Promise.resolve();
10475
+ }
10476
+ };
10477
+ }
10478
+ };
10479
+
10480
+ // ../../clients/web-client/src/voice/silence-detector.ts
10481
+ var MIN_SPEECH_LEVEL = 0.015;
10482
+ var SPEECH_TO_NOISE = 3;
10483
+ var SPEECH_START_MS = 200;
10484
+ var SilenceDetector = class {
10485
+ constructor(silenceMs) {
10486
+ this.silenceMs = silenceMs;
10487
+ }
10488
+ floor = null;
10489
+ loudMs = 0;
10490
+ quietMs = 0;
10491
+ speaking = false;
10492
+ /** True when the speech has ended. */
10493
+ push(level, frameMs = SPEECH_FRAME_MS) {
10494
+ const threshold = Math.max(
10495
+ MIN_SPEECH_LEVEL,
10496
+ (this.floor ?? MIN_SPEECH_LEVEL / SPEECH_TO_NOISE) * SPEECH_TO_NOISE
10497
+ );
10498
+ if (level >= threshold) {
10499
+ this.loudMs += frameMs;
10500
+ this.quietMs = 0;
10501
+ if (this.loudMs >= SPEECH_START_MS) {
10502
+ this.speaking = true;
10503
+ }
10504
+ return false;
10505
+ }
10506
+ this.floor = this.floor === null ? level : this.floor * 0.9 + level * 0.1;
10507
+ this.loudMs = 0;
10508
+ if (!this.speaking) {
10509
+ return false;
10510
+ }
10511
+ this.quietMs += frameMs;
10512
+ return this.quietMs >= this.silenceMs;
10513
+ }
10514
+ get heardSpeech() {
10515
+ return this.speaking;
10516
+ }
10517
+ };
10518
+
10519
+ // ../../clients/web-client/src/voice/voice-input.ts
10520
+ var SpeechInputError = class extends Error {
10521
+ constructor(code, message) {
10522
+ super(message ?? code);
10523
+ this.code = code;
10524
+ this.name = "NpcifySpeechInputError";
10525
+ }
10526
+ };
10527
+ var deferred = function() {
10528
+ let resolve;
10529
+ let reject;
10530
+ const promise = new Promise((done, fail) => {
10531
+ resolve = done;
10532
+ reject = fail;
10533
+ });
10534
+ promise.catch(() => void 0);
10535
+ return { promise, resolve, reject };
10536
+ };
10537
+ var CODE = /^[A-Z][A-Z0-9_]{2,63}$/u;
10538
+ var speechError = function(error, fallback) {
10539
+ if (error instanceof SpeechInputError) {
10540
+ return error;
10541
+ }
10542
+ if (error instanceof Error && "code" in error && typeof error.code === "string") {
10543
+ return new SpeechInputError(
10544
+ error.code,
10545
+ error.message
10546
+ );
10547
+ }
10548
+ return new SpeechInputError(
10549
+ fallback,
10550
+ error instanceof Error ? error.message : void 0
10551
+ );
10552
+ };
10553
+ var VoiceInput = class {
10554
+ constructor(deps) {
10555
+ this.deps = deps;
10556
+ }
10557
+ state = "idle";
10558
+ session = null;
10559
+ capture = null;
10560
+ /**
10561
+ * The session that the capture records for. A start that was cancelled
10562
+ * never touches the capture when a later session owns it.
10563
+ */
10564
+ captureOwner = null;
10565
+ turns = /* @__PURE__ */ new Map();
10566
+ get listeningState() {
10567
+ return this.state;
10568
+ }
10569
+ /** A turn that the server started has not delivered its final payload. */
10570
+ get turnPending() {
10571
+ return this.turns.size > 0;
10572
+ }
10573
+ /**
10574
+ * Starts listening. Resolves with the result when the listening ends (by
10575
+ * `stop`, the silence rule or the time limit), or with null when it is
10576
+ * cancelled.
10577
+ */
10578
+ async start(options = {}) {
10579
+ if (this.session) {
10580
+ throw new SpeechInputError(
10581
+ "SPEECH_ALREADY_LISTENING",
10582
+ "Listening has already started."
10583
+ );
10584
+ }
10585
+ this.deps.assertVoiceAllowed();
10586
+ if (this.turnPending || this.deps.turnInFlight()) {
10587
+ throw new SpeechInputError(
10588
+ "TURN_IN_PROGRESS",
10589
+ "A reply is already being generated. Please wait."
10590
+ );
10591
+ }
10592
+ const conversationId = this.deps.conversationId();
10593
+ if (!conversationId) {
10594
+ throw new SpeechInputError(
10595
+ "CONVERSATION_REQUIRED",
10596
+ "Create a conversation before listening."
10597
+ );
10598
+ }
10599
+ const maxDurationMs = Math.min(
10600
+ SPEECH_MAX_DURATION_MS,
10601
+ Math.max(1e3, options.maxDurationMs ?? SPEECH_MAX_DURATION_MS)
10602
+ );
10603
+ const session = {
10604
+ requestId: globalThis.crypto.randomUUID(),
10605
+ options,
10606
+ sessionId: null,
10607
+ sequence: 0,
10608
+ bytes: 0,
10609
+ pending: [],
10610
+ timer: null,
10611
+ silence: options.endOnSilenceMs && options.endOnSilenceMs > 0 ? new SilenceDetector(options.endOnSilenceMs) : null,
10612
+ stopRequested: false,
10613
+ stopping: null,
10614
+ captureClosing: null,
10615
+ captureClosed: false,
10616
+ disconnected: deferred(),
10617
+ done: deferred()
10618
+ };
10619
+ this.session = session;
10620
+ this.setState("starting");
10621
+ const socket = this.deps.socket();
10622
+ this.capture ??= (this.deps.createCapture ?? (() => new MicrophoneCapture()))();
10623
+ const capture = this.capture;
10624
+ this.captureOwner = session;
10625
+ const [acknowledged, captured] = await Promise.allSettled([
10626
+ socket.ensureConnected(conversationId).then(
10627
+ () => socket.request(SPEECH_EVENTS.start, {
10628
+ conversationId,
10629
+ requestId: session.requestId,
10630
+ parentStagingId: options.parentStagingId,
10631
+ language: options.language ?? null,
10632
+ send: options.send !== false,
10633
+ silent: options.silent === true,
10634
+ renderSessionId: this.deps.renderSessionId(),
10635
+ format: SPEECH_AUDIO_FORMAT
10636
+ })
10637
+ ),
10638
+ capture.start((frame) => this.frame(session, frame))
10639
+ ]);
10640
+ const ack = acknowledged.status === "fulfilled" ? acknowledged.value : null;
10641
+ const serverSession = ack?.ok ? ack.data.sessionId : null;
10642
+ if (session !== this.session) {
10643
+ this.dropCapture(session);
10644
+ if (serverSession) {
10645
+ socket.send(SPEECH_EVENTS.cancel, { sessionId: serverSession });
10646
+ }
10647
+ return session.done.promise;
10648
+ }
10649
+ let failure = null;
10650
+ if (captured.status === "rejected") {
10651
+ failure = speechError(captured.reason, "MICROPHONE_UNAVAILABLE");
10652
+ } else if (acknowledged.status === "rejected") {
10653
+ failure = speechError(acknowledged.reason, "SPEECH_UNAVAILABLE");
10654
+ } else if (!ack?.ok) {
10655
+ failure = new SpeechInputError(
10656
+ ack && !ack.ok ? ack.error : "SPEECH_UNAVAILABLE"
10657
+ );
10658
+ }
10659
+ if (failure) {
10660
+ this.dropCapture(session);
10661
+ if (serverSession) {
10662
+ socket.send(SPEECH_EVENTS.cancel, { sessionId: serverSession });
10663
+ }
10664
+ this.end(session);
10665
+ session.done.reject(failure);
10666
+ return session.done.promise;
10667
+ }
10668
+ session.sessionId = serverSession;
10669
+ for (const audio of session.pending.splice(0)) {
10670
+ this.sendFrame(session, audio);
10671
+ }
10672
+ if (session.stopRequested) {
10673
+ void this.stop().catch(() => void 0);
10674
+ return session.done.promise;
10675
+ }
10676
+ this.setState("listening");
10677
+ session.timer = setTimeout(() => {
10678
+ void this.stop().catch(() => void 0);
10679
+ }, maxDurationMs);
10680
+ return session.done.promise;
10681
+ }
10682
+ /**
10683
+ * Ends the recording and resolves with the result as soon as the text is
10684
+ * recognised. A stop before the microphone gave any audio (a permission
10685
+ * prompt, a slow device) cancels and resolves null.
10686
+ */
10687
+ async stop() {
10688
+ const session = this.session;
10689
+ if (!session) {
10690
+ return null;
10691
+ }
10692
+ if (session.stopping) {
10693
+ return session.stopping;
10694
+ }
10695
+ if (!session.sessionId) {
10696
+ if (session.bytes === 0) {
10697
+ this.cancel();
10698
+ return null;
10699
+ }
10700
+ session.stopRequested = true;
10701
+ void this.closeCapture(session);
10702
+ return session.done.promise;
10703
+ }
10704
+ const stopping = deferred();
10705
+ session.stopping = stopping.promise;
10706
+ this.finish(session).then(stopping.resolve, stopping.reject);
10707
+ return session.stopping;
10708
+ }
10709
+ /** Discards the recording. Nothing is recognised or charged. */
10710
+ cancel() {
10711
+ const session = this.session;
10712
+ if (!session || session.stopping) {
10713
+ return;
10714
+ }
10715
+ this.dropCapture(session);
10716
+ if (session.sessionId) {
10717
+ this.deps.socket().send(SPEECH_EVENTS.cancel, { sessionId: session.sessionId });
10718
+ }
10719
+ this.end(session);
10720
+ session.done.resolve(null);
10721
+ }
10722
+ /** The conversation ended: no listening, and no turn waits any more. */
10723
+ reset() {
10724
+ this.cancel();
10725
+ for (const [requestId, turn] of this.turns) {
10726
+ clearTimeout(turn.timer);
10727
+ turn.done.reject(
10728
+ new SpeechInputError("CONVERSATION_ENDED", "The conversation ended.")
10729
+ );
10730
+ this.turns.delete(requestId);
10731
+ }
10732
+ }
10733
+ handleSocketEvent(event) {
10734
+ if (event.type === "partial") {
10735
+ if (this.session?.sessionId === event.event.sessionId) {
10736
+ this.deps.onTranscript({ text: event.event.text, final: false });
10737
+ }
10738
+ return;
10739
+ }
10740
+ if (event.type === "turn") {
10741
+ const turn = this.turns.get(event.requestId);
10742
+ if (!turn) {
10743
+ return;
10744
+ }
10745
+ this.turns.delete(event.requestId);
10746
+ clearTimeout(turn.timer);
10747
+ void this.deps.deliverTurn(event.turn, event.requestId, turn.options.silent === true).then(
10748
+ () => turn.done.resolve(event.turn),
10749
+ (error) => turn.done.reject(error)
10750
+ );
10751
+ return;
10752
+ }
10753
+ if (event.type === "turnFailed") {
10754
+ const turn = this.turns.get(event.event.requestId);
10755
+ if (!turn) {
10756
+ return;
10757
+ }
10758
+ this.turns.delete(event.event.requestId);
10759
+ clearTimeout(turn.timer);
10760
+ const reason = event.event.error;
10761
+ turn.done.reject(
10762
+ new SpeechInputError(CODE.test(reason) ? reason : "TURN_FAILED", reason)
10763
+ );
10764
+ return;
10765
+ }
10766
+ const session = this.session;
10767
+ if (session) {
10768
+ const failure = new SpeechInputError(
10769
+ "SPEECH_DISCONNECTED",
10770
+ session.stopping ? "The connection closed while the speech was recognised." : "The connection closed while listening."
10771
+ );
10772
+ session.disconnected.reject(failure);
10773
+ if (!session.stopping) {
10774
+ this.dropCapture(session);
10775
+ this.end(session);
10776
+ session.done.reject(failure);
10777
+ }
10778
+ }
10779
+ for (const requestId of [...this.turns.keys()]) {
10780
+ this.recover(requestId);
10781
+ }
10782
+ }
10783
+ frame(session, frame) {
10784
+ if (session !== this.session || session.captureClosed) {
10785
+ return;
10786
+ }
10787
+ const audio = new Uint8Array(
10788
+ frame.pcm.buffer,
10789
+ frame.pcm.byteOffset,
10790
+ frame.pcm.byteLength
10791
+ ).slice();
10792
+ if (session.sessionId) {
10793
+ this.sendFrame(session, audio);
10794
+ } else if (session.bytes + audio.length <= SPEECH_MAX_AUDIO_BYTES) {
10795
+ session.bytes += audio.length;
10796
+ session.pending.push(audio);
10797
+ }
10798
+ if (!session.stopping && session.silence?.push(frame.level)) {
10799
+ void this.stop().catch(() => void 0);
10800
+ }
10801
+ }
10802
+ sendFrame(session, audio) {
10803
+ if (!session.sessionId) {
10804
+ return;
10805
+ }
10806
+ this.deps.socket().send(SPEECH_EVENTS.audio, {
10807
+ sessionId: session.sessionId,
10808
+ sequence: session.sequence,
10809
+ audio
10810
+ });
10811
+ session.sequence += 1;
10812
+ }
10813
+ async finish(session) {
10814
+ if (session.timer) {
10815
+ clearTimeout(session.timer);
10816
+ }
10817
+ this.setState("recognizing");
10818
+ try {
10819
+ await this.closeCapture(session);
10820
+ const ack = await Promise.race([
10821
+ this.deps.socket().request(
10822
+ SPEECH_EVENTS.stop,
10823
+ { sessionId: session.sessionId },
10824
+ 3e4
10825
+ ),
10826
+ session.disconnected.promise
10827
+ ]);
10828
+ if (!ack.ok) {
10829
+ throw new SpeechInputError(ack.error);
10830
+ }
10831
+ const result = this.result(session, ack.data);
10832
+ this.end(session);
10833
+ this.deps.onTranscript({ text: result.text, final: true });
10834
+ session.done.resolve(result);
10835
+ return result;
10836
+ } catch (error) {
10837
+ const failure = speechError(error, "SPEECH_RECOGNITION_FAILED");
10838
+ this.end(session);
10839
+ session.done.reject(failure);
10840
+ throw failure;
10841
+ }
10842
+ }
10843
+ result(session, transcript) {
10844
+ const text = transcript.text;
10845
+ const turn = transcript.sent ? this.track(session, text) : null;
10846
+ let sending = null;
10847
+ return {
10848
+ text,
10849
+ language: transcript.language,
10850
+ durationMs: transcript.durationMs,
10851
+ sent: transcript.sent,
10852
+ turn,
10853
+ send: () => {
10854
+ if (turn) {
10855
+ return turn;
10856
+ }
10857
+ if (!text) {
10858
+ return Promise.reject(
10859
+ new SpeechInputError("SPEECH_EMPTY", "No speech was recognised.")
10860
+ );
10861
+ }
10862
+ this.deps.assertVoiceAllowed();
10863
+ sending ??= this.deps.sendText(text, {
10864
+ requestId: session.requestId,
10865
+ parentStagingId: session.options.parentStagingId,
10866
+ silent: session.options.silent === true
10867
+ });
10868
+ return sending;
10869
+ }
10870
+ };
10871
+ }
10872
+ track(session, text) {
10873
+ const done = deferred();
10874
+ const timer = setTimeout(
10875
+ () => this.recover(session.requestId),
10876
+ this.deps.turnTimeoutMs ?? 12e4
10877
+ );
10878
+ this.turns.set(session.requestId, {
10879
+ text,
10880
+ options: session.options,
10881
+ timer,
10882
+ done
10883
+ });
10884
+ this.deps.beginTurn();
10885
+ return done.promise;
10886
+ }
10887
+ /** The idempotent HTTP request returns the turn that the server started. */
10888
+ recover(requestId) {
10889
+ const turn = this.turns.get(requestId);
10890
+ if (!turn) {
10891
+ return;
10892
+ }
10893
+ this.turns.delete(requestId);
10894
+ clearTimeout(turn.timer);
10895
+ this.deps.sendText(turn.text, {
10896
+ requestId,
10897
+ parentStagingId: turn.options.parentStagingId,
10898
+ silent: turn.options.silent === true
10899
+ }).then(turn.done.resolve, turn.done.reject);
10900
+ }
10901
+ /** Stops the capture of `session`; its last partial frame arrives first. */
10902
+ closeCapture(session) {
10903
+ session.captureClosing ??= (async () => {
10904
+ if (this.captureOwner === session) {
10905
+ this.captureOwner = null;
10906
+ await this.capture?.stop();
10907
+ }
10908
+ session.captureClosed = true;
10909
+ })();
10910
+ return session.captureClosing;
10911
+ }
10912
+ /** Ends the capture of `session` without its last frame, unless a later session owns it. */
10913
+ dropCapture(session) {
10914
+ if (this.captureOwner === session) {
10915
+ this.captureOwner = null;
10916
+ this.capture?.cancel();
10917
+ }
10918
+ }
10919
+ end(session) {
10920
+ if (session.timer) {
10921
+ clearTimeout(session.timer);
10922
+ session.timer = null;
10923
+ }
10924
+ if (this.session === session) {
10925
+ this.session = null;
10926
+ this.setState("idle");
10927
+ }
10928
+ }
10929
+ setState(state) {
10930
+ if (this.state !== state) {
10931
+ this.state = state;
10932
+ this.deps.onState(state);
10933
+ }
10934
+ }
10935
+ };
10936
+
10937
+ // ../../clients/web-client/src/npcify.ts
10938
+ var Character = class extends CharacterRuntime {
10939
+ conversationSocket;
10940
+ renderSessionId = null;
10941
+ initialAssetInfo;
10942
+ stagingResultCallback = null;
10943
+ thinkingCallback = null;
10944
+ notifiedStagingIds = /* @__PURE__ */ new Map();
10945
+ liveConversation;
10946
+ baked = null;
10947
+ authoring;
10948
+ normalizeIdleOrientation;
10949
+ bakedAsset;
10950
+ animationRefreshId = 0;
10951
+ voice;
10952
+ configuredInputModes;
10953
+ resolvedInputModes = null;
10954
+ latestOptions = [];
10955
+ listeningStateCallback = null;
10956
+ transcriptCallback = null;
10957
+ conversationId = null;
10958
+ constructor(options) {
10959
+ const resolvedToken = (options.apiToken ?? options.apiKey)?.trim() ?? "";
10960
+ const resolvedProfile = options.assetProfile ?? "original";
10961
+ super({
10962
+ ...options,
10963
+ apiToken: resolvedToken,
10964
+ assetProfile: resolvedProfile,
10965
+ loadAnimation: (animationId) => loadVrma({
10966
+ baseUrl: options.baseUrl,
10967
+ characterId: options.characterId,
10968
+ apiToken: resolvedToken,
10969
+ animationId,
10970
+ assetProfile: resolvedProfile
10971
+ })
10972
+ });
10973
+ this.authoring = options.authoring ?? false;
10974
+ this.normalizeIdleOrientation = options.normalizeIdleOrientation ?? false;
10975
+ this.bakedAsset = options.bakedConversation;
10976
+ this.initialAssetInfo = options.initialAssetInfo ?? null;
10977
+ this.configuredInputModes = parseInputModes(options.inputModes);
10978
+ if (this.configuredInputModes && this.bakedAsset && !this.authoring) {
10979
+ resolveInputModes(this.configuredInputModes, {
10980
+ allowCustomReply: false,
10981
+ baked: true
10982
+ });
10983
+ }
10984
+ this.conversationSocket = new ConversationSocket({
10985
+ baseUrl: this.baseUrl,
10986
+ apiToken: this.apiToken,
10987
+ getConversationId: () => this.conversationId,
10988
+ handleStreamEvent: (event) => {
10989
+ if (event.type !== "text_delta") {
10990
+ this.thinkingState = false;
10991
+ }
10992
+ this.streamPlayback.handleEvent(event);
10993
+ },
10994
+ handleTurn: async (turn) => {
10995
+ await this.handleStagingResult(turn, { source: "turn" });
10996
+ },
10997
+ handleCallbackError: this.handleCallbackError,
10998
+ handleSpeechEvent: (event) => this.voice.handleSocketEvent(event)
10999
+ });
11000
+ this.voice = new VoiceInput({
11001
+ socket: () => this.conversationSocket,
11002
+ conversationId: () => this.conversationId,
11003
+ renderSessionId: () => this.renderSessionId,
11004
+ assertVoiceAllowed: () => this.assertInputMode("voice"),
11005
+ turnInFlight: () => this.liveConversation.inFlight,
11006
+ beginTurn: () => {
11007
+ if (this.conversationId) {
11008
+ this.telemetry.begin(this.conversationId);
11009
+ }
11010
+ this.invokeThinkingCallback({ taskId: null, stagingId: null });
11011
+ },
11012
+ deliverTurn: (payload, requestId, silent) => this.handleStagingResult(payload, {
11013
+ source: "api",
11014
+ forcePlayback: payload.taskId !== requestId,
11015
+ silent
11016
+ }),
11017
+ sendText: (text, turnOptions) => {
11018
+ if (!this.conversationId) {
11019
+ return Promise.reject(new Error("Conversation ID is required."));
11020
+ }
11021
+ return this.liveConversation.send(this.conversationId, text, turnOptions);
11022
+ },
11023
+ onState: (state) => {
11024
+ if (this.listeningStateCallback) {
11025
+ invokeSafeCallback(
11026
+ "onListeningState",
11027
+ this.handleCallbackError,
11028
+ () => this.listeningStateCallback?.(state)
11029
+ );
11030
+ }
11031
+ },
11032
+ onTranscript: (event) => {
11033
+ if (this.transcriptCallback) {
11034
+ invokeSafeCallback(
11035
+ "onTranscript",
11036
+ this.handleCallbackError,
11037
+ () => this.transcriptCallback?.(event)
11038
+ );
11039
+ }
11040
+ }
11041
+ });
11042
+ this.liveConversation = new LiveConversation({
11043
+ baseUrl: this.baseUrl,
11044
+ apiToken: this.apiToken,
9905
11045
  telemetry: this.telemetry,
9906
11046
  socket: () => this.conversationSocket,
9907
11047
  renderSession: () => this.renderSessionId,
@@ -9924,7 +11064,12 @@ var Character = class extends CharacterRuntime {
9924
11064
  this.resetAssets();
9925
11065
  const progressReporter = createAssetProgressReporter(callback);
9926
11066
  const info = await this.getAssetInfo();
9927
- this.baked = !this.authoring && (this.bakedAsset || info.bakedConversationEnabled) ? await loadBakedConversation({
11067
+ const baked = !this.authoring && Boolean(this.bakedAsset || info.bakedConversationEnabled);
11068
+ this.resolvedInputModes = resolveInputModes(this.configuredInputModes, {
11069
+ allowCustomReply: info.allowCustomReply !== false,
11070
+ baked
11071
+ });
11072
+ this.baked = baked ? await loadBakedConversation({
9928
11073
  baseUrl: this.baseUrl,
9929
11074
  characterId: this.characterId,
9930
11075
  apiToken: this.apiToken,
@@ -9949,7 +11094,8 @@ var Character = class extends CharacterRuntime {
9949
11094
  }
9950
11095
  const animationController = new AnimationController(
9951
11096
  this.vrm,
9952
- info.idleAnimationId ?? null
11097
+ info.idleAnimationId ?? null,
11098
+ { normalizeIdleOrientation: this.normalizeIdleOrientation }
9953
11099
  );
9954
11100
  animationController.prepareAnimations = (ids) => this.lazyAnimations.ensureMany(ids);
9955
11101
  this.animationController = animationController;
@@ -9992,6 +11138,10 @@ var Character = class extends CharacterRuntime {
9992
11138
  if (info.bakedConversationEnabled || this.bakedAsset) {
9993
11139
  throw new Error("Baked conversation requires local rendering.");
9994
11140
  }
11141
+ this.resolvedInputModes = resolveInputModes(this.configuredInputModes, {
11142
+ allowCustomReply: info.allowCustomReply !== false,
11143
+ baked: false
11144
+ });
9995
11145
  this.animationController = new RemoteAnimationController();
9996
11146
  }
9997
11147
  async createConversation(options) {
@@ -10034,6 +11184,7 @@ var Character = class extends CharacterRuntime {
10034
11184
  return targetConversationId;
10035
11185
  }
10036
11186
  async disconnect() {
11187
+ this.voice.reset();
10037
11188
  this.thinkingState = false;
10038
11189
  if (this.animationController) {
10039
11190
  this.streamPlayback.cleanup();
@@ -10069,6 +11220,10 @@ var Character = class extends CharacterRuntime {
10069
11220
  maybeMessageOrOptions,
10070
11221
  maybeOptions
10071
11222
  );
11223
+ this.assertReplyMode(message);
11224
+ if (this.voice.turnPending) {
11225
+ throw new Error("A reply is already being generated. Please wait.");
11226
+ }
10072
11227
  if (this.baked) {
10073
11228
  const result = this.baked.reply(message, options?.parentStagingId);
10074
11229
  if (result.ended) {
@@ -10108,6 +11263,55 @@ var Character = class extends CharacterRuntime {
10108
11263
  get isBakedConversation() {
10109
11264
  return this.baked !== null;
10110
11265
  }
11266
+ /**
11267
+ * The input modes in use: the configured modes, or every mode that the
11268
+ * character permits. Before `loadAssets`, the configuration or all modes.
11269
+ */
11270
+ get inputModes() {
11271
+ return this.resolvedInputModes ?? this.configuredInputModes ?? permittedInputModes({ allowCustomReply: true, baked: false });
11272
+ }
11273
+ get listeningState() {
11274
+ return this.voice.listeningState;
11275
+ }
11276
+ /**
11277
+ * Opens the microphone and streams the speech to the server. Resolves with
11278
+ * the result when listening ends (`stopListening`, `endOnSilenceMs` or
11279
+ * `maxDurationMs`), or with null when it is cancelled. The first call asks
11280
+ * for the microphone permission.
11281
+ */
11282
+ startListening(options) {
11283
+ return this.voice.start(options);
11284
+ }
11285
+ /**
11286
+ * Ends the recording; resolves as soon as the server recognised the text.
11287
+ * Before the microphone gave any audio, it cancels and resolves null.
11288
+ */
11289
+ stopListening() {
11290
+ return this.voice.stop();
11291
+ }
11292
+ /** Discards the recording. Nothing is recognised or charged. */
11293
+ cancelListening() {
11294
+ this.voice.cancel();
11295
+ }
11296
+ /** Press and hold an element or a key (default Space) to talk. Returns an unbind function. */
11297
+ bindPushToTalk(options) {
11298
+ return bindPushToTalk(this, options);
11299
+ }
11300
+ onListeningState(callback) {
11301
+ return bindCallbackSlot(
11302
+ () => this.listeningStateCallback,
11303
+ (next) => this.listeningStateCallback = next,
11304
+ callback
11305
+ );
11306
+ }
11307
+ /** Partial text while the player speaks (`final: false`), then the final text. */
11308
+ onTranscript(callback) {
11309
+ return bindCallbackSlot(
11310
+ () => this.transcriptCallback,
11311
+ (next) => this.transcriptCallback = next,
11312
+ callback
11313
+ );
11314
+ }
10111
11315
  async getConversationWorldline(conversationId = this.conversationId) {
10112
11316
  if (!conversationId) {
10113
11317
  throw new Error("Conversation ID is required.");
@@ -10131,6 +11335,13 @@ var Character = class extends CharacterRuntime {
10131
11335
  timeline: options?.silent ? toSilentTimeline(normalized.timeline) : normalized.timeline
10132
11336
  });
10133
11337
  }
11338
+ /**
11339
+ * Plays one live staging event from another transport, such as an actor
11340
+ * stage. A timeline event returns its playback result.
11341
+ */
11342
+ playStreamEvent(event) {
11343
+ return this.streamPlayback.handleEvent(event);
11344
+ }
10134
11345
  setRenderSessionId(renderSessionId) {
10135
11346
  this.renderSessionId = renderSessionId?.trim() || null;
10136
11347
  }
@@ -10164,6 +11375,7 @@ var Character = class extends CharacterRuntime {
10164
11375
  async handleStagingResult(payload, options) {
10165
11376
  this.thinkingState = false;
10166
11377
  const normalized = normalizeStagingResultPayload(payload);
11378
+ this.latestOptions = normalized.options;
10167
11379
  if (options.source === "turn") {
10168
11380
  this.liveConversation.received(normalized);
10169
11381
  }
@@ -10190,6 +11402,44 @@ var Character = class extends CharacterRuntime {
10190
11402
  void this.streamPlayback.playPayload(normalized);
10191
11403
  }
10192
11404
  }
11405
+ /** Throws when the input modes do not include `mode`. */
11406
+ assertInputMode(mode) {
11407
+ if (mode === "voice" && this.baked) {
11408
+ throw new InputModeError(
11409
+ "Voice input is not available in a baked (offline) conversation."
11410
+ );
11411
+ }
11412
+ const modes = this.inputModes;
11413
+ if (!modes.includes(mode)) {
11414
+ throw new InputModeError(
11415
+ `The "${mode}" input mode is off: inputModes is ${JSON.stringify(modes)}.`
11416
+ );
11417
+ }
11418
+ }
11419
+ /**
11420
+ * A typed reply needs the `text` mode; one of the current reply options
11421
+ * needs `options` or `text`. Without a configuration the server's rules
11422
+ * apply alone, as before.
11423
+ */
11424
+ assertReplyMode(message) {
11425
+ if (!this.configuredInputModes) {
11426
+ return;
11427
+ }
11428
+ const modes = this.inputModes;
11429
+ if (modes.includes("text")) {
11430
+ return;
11431
+ }
11432
+ const reply = message.trim();
11433
+ const isOption = this.latestOptions.some(
11434
+ (option) => option.label.trim() === reply
11435
+ );
11436
+ if (isOption && modes.includes("options")) {
11437
+ return;
11438
+ }
11439
+ throw new InputModeError(
11440
+ isOption ? `Reply options are off: inputModes is ${JSON.stringify(modes)}.` : `Typed replies are off: inputModes is ${JSON.stringify(modes)}. Send one of the reply options or use startListening().`
11441
+ );
11442
+ }
10193
11443
  invokeThinkingCallback(payload) {
10194
11444
  this.thinkingState = true;
10195
11445
  if (this.thinkingCallback) {
@@ -10202,6 +11452,12 @@ var Character = class extends CharacterRuntime {
10202
11452
  }
10203
11453
  };
10204
11454
  var Npcify = class extends Character {
11455
+ constructor(options) {
11456
+ super({
11457
+ ...options,
11458
+ normalizeIdleOrientation: options.normalizeIdleOrientation ?? true
11459
+ });
11460
+ }
10205
11461
  };
10206
11462
 
10207
11463
  // ../../clients/web-client/src/character/character.ts
@@ -10782,13 +12038,18 @@ export {
10782
12038
  CharacterSceneRig,
10783
12039
  FacialPerformanceLayer,
10784
12040
  FacialPerformanceMixer,
12041
+ INPUT_MODES,
12042
+ InputModeError,
12043
+ MicrophoneError,
10785
12044
  Npcify,
10786
12045
  Character as NpcifyCharacter,
10787
12046
  PERFORMANCE_FILE_EXTENSION,
10788
12047
  PERFORMANCE_MEDIA_TYPE,
10789
12048
  PerformanceRecordingSchemaVersion,
10790
12049
  RemoteAnimationController,
12050
+ SpeechInputError,
10791
12051
  analyzePerformanceRecording,
12052
+ bindPushToTalk,
10792
12053
  createCharacter,
10793
12054
  decodePerformance,
10794
12055
  encodePerformance,