@npcify/web 0.4.1 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -34,6 +34,33 @@ var isValidMotionFramesEvent = function(event) {
34
34
  // ../ts-shared/src/llm.ts
35
35
  var STAGING_PROTOCOL_VERSION = 1;
36
36
 
37
+ // ../ts-shared/src/variants.ts
38
+ var DEFAULT_VARIANT_KEY = "default";
39
+ var NUDE_VARIANT_KEY = "nude";
40
+ var VARIANT_KEY_PATTERN = /^[a-z][a-z0-9_-]{0,39}$/u;
41
+ var isValidVariantKey = function(value) {
42
+ return typeof value === "string" && VARIANT_KEY_PATTERN.test(value);
43
+ };
44
+ var STAGING_TOOL_RESULT_STATUSES = [
45
+ "applied",
46
+ "noop",
47
+ "rejected"
48
+ ];
49
+ var isStagingToolCall = function(value) {
50
+ if (!value || typeof value !== "object" || Array.isArray(value)) {
51
+ return false;
52
+ }
53
+ const candidate = value;
54
+ if (typeof candidate.id !== "string" || !candidate.id.trim() || candidate.name !== "switch_variant" || typeof candidate.timestamp !== "number" || !Number.isFinite(candidate.timestamp) || candidate.timestamp < 0 || !candidate.arguments || typeof candidate.arguments !== "object" || Array.isArray(candidate.arguments) || !candidate.result || typeof candidate.result !== "object" || Array.isArray(candidate.result)) {
55
+ return false;
56
+ }
57
+ const args = candidate.arguments;
58
+ const result = candidate.result;
59
+ return isValidVariantKey(args.variant) && typeof result.ok === "boolean" && STAGING_TOOL_RESULT_STATUSES.includes(
60
+ String(result.status)
61
+ ) && isValidVariantKey(result.previousVariant) && isValidVariantKey(result.activeVariant) && (result.error === void 0 || typeof result.error === "string");
62
+ };
63
+
37
64
  // ../ts-shared/src/server-rendering.ts
38
65
  var SERVER_RENDER_PROTOCOL_VERSION = 3;
39
66
  var MAXIMUM_PLAYBACK_DURATION_SECONDS = 10 * 60;
@@ -42,7 +69,465 @@ var MAXIMUM_PLAYBACK_DURATION_SECONDS = 10 * 60;
42
69
  var TTS_PROVIDERS = ["11labs", "gemini"];
43
70
  var DEFAULT_TTS_PROVIDER = TTS_PROVIDERS[0];
44
71
 
72
+ // ../ts-shared/src/llm-providers/catalog.ts
73
+ var record = function(value) {
74
+ return value && typeof value === "object" && !Array.isArray(value) ? value : {};
75
+ };
76
+ var rate = function(value) {
77
+ if (typeof value !== "number" && typeof value !== "string") {
78
+ return null;
79
+ }
80
+ if (value === "") {
81
+ return null;
82
+ }
83
+ const result = Number(value);
84
+ return Number.isFinite(result) && result >= 0 ? result : null;
85
+ };
86
+ var catalogRows = function(payload) {
87
+ if (!Array.isArray(payload)) {
88
+ throw new Error("Provider returned an invalid model catalog");
89
+ }
90
+ return payload.map(record);
91
+ };
92
+ var parseCompatibleModels = function(payload) {
93
+ return catalogRows(record(payload).data).flatMap((row) => {
94
+ const pricing = record(row.pricing);
95
+ const input = rate(pricing.prompt);
96
+ const output = rate(pricing.completion);
97
+ const cached = rate(pricing.input_cache_read);
98
+ const write = rate(pricing.input_cache_write);
99
+ const modalities = record(row.architecture).output_modalities;
100
+ const parameters = row.supported_parameters;
101
+ if (typeof row.id !== "string" || !row.id || row.deprecated || input === null || output === null || Array.isArray(modalities) && (modalities.length !== 1 || modalities[0] !== "text") || Array.isArray(pricing.overrides) && pricing.overrides.length > 0 || Array.isArray(parameters) && !parameters.includes("structured_outputs") && !parameters.includes("response_format") || (rate(pricing.request) ?? 0) > 0) {
102
+ return [];
103
+ }
104
+ return [
105
+ {
106
+ id: row.id,
107
+ name: typeof row.name === "string" ? row.name : row.id,
108
+ inputUsd: input * 1e6,
109
+ outputUsd: output * 1e6,
110
+ cachedInputUsd: (cached ?? input) * 1e6,
111
+ ...write !== null ? { cacheWriteUsd: write * 1e6 } : {},
112
+ contextLength: rate(row.max_tokens ?? row.context_length)
113
+ }
114
+ ];
115
+ });
116
+ };
117
+
118
+ // ../ts-shared/src/llm-providers/usage.ts
119
+ var validUsd = function(value) {
120
+ return typeof value === "number" && Number.isFinite(value) && value >= 0;
121
+ };
122
+ var validateUsageCounts = function(usage) {
123
+ const counts = [
124
+ usage.prompt_tokens,
125
+ usage.completion_tokens,
126
+ usage.total_tokens,
127
+ // Providers can explicitly return null for unavailable cache counters.
128
+ usage.prompt_tokens_details?.cached_tokens ?? 0,
129
+ usage.prompt_tokens_details?.cache_write_tokens ?? 0
130
+ ];
131
+ if (counts.some(
132
+ (value) => value !== void 0 && (!Number.isSafeInteger(value) || value < 0)
133
+ )) {
134
+ throw new Error("Provider returned invalid token usage");
135
+ }
136
+ const cached = usage.prompt_tokens_details?.cached_tokens ?? 0;
137
+ const written = usage.prompt_tokens_details?.cache_write_tokens ?? 0;
138
+ if (usage.prompt_tokens !== void 0 && cached + written > usage.prompt_tokens) {
139
+ throw new Error("Provider returned invalid cache usage");
140
+ }
141
+ };
142
+ var quoteTokenUsage = function(price, usage) {
143
+ validateUsageCounts(usage);
144
+ const input = usage.prompt_tokens;
145
+ const output = usage.completion_tokens;
146
+ if (input === void 0 || output === void 0) {
147
+ throw new Error("Provider did not report billable token usage");
148
+ }
149
+ const cached = usage.prompt_tokens_details?.cached_tokens ?? 0;
150
+ const written = usage.prompt_tokens_details?.cache_write_tokens ?? 0;
151
+ const writeRate = price.cacheWriteUsd ?? price.inputUsd;
152
+ if (![price.inputUsd, price.outputUsd, price.cachedInputUsd, writeRate].every(
153
+ validUsd
154
+ )) {
155
+ throw new Error("Provider returned invalid token prices");
156
+ }
157
+ return {
158
+ usd: ((input - cached - written) * price.inputUsd + cached * price.cachedInputUsd + written * writeRate + output * price.outputUsd) / 1e6,
159
+ source: "token-rates"
160
+ };
161
+ };
162
+
163
+ // ../ts-shared/src/llm-providers/baseten.ts
164
+ var baseten = {
165
+ name: "Baseten",
166
+ baseUrl: "https://inference.baseten.co/v1",
167
+ modelsUrl: "https://inference.baseten.co/v1/models",
168
+ billingMode: "token-rates",
169
+ parseModels: parseCompatibleModels,
170
+ requestOptions: () => ({}),
171
+ quoteUsage: quoteTokenUsage
172
+ };
173
+
174
+ // ../ts-shared/src/llm-providers/deepinfra.ts
175
+ var deepinfra = {
176
+ name: "DeepInfra",
177
+ keyCheckUrl: "https://api.deepinfra.com/v1/me",
178
+ baseUrl: "https://api.deepinfra.com/v1/openai",
179
+ modelsUrl: "https://api.deepinfra.com/models/list",
180
+ billingMode: "token-rates",
181
+ parseModels: (payload) => catalogRows(payload).flatMap((row) => {
182
+ const tags = Array.isArray(row.tags) ? row.tags : [];
183
+ const pricing = record(row.pricing);
184
+ const input = rate(pricing.cents_per_input_token);
185
+ const output = rate(pricing.cents_per_output_token);
186
+ if (typeof row.model_name !== "string" || !row.model_name || row.deprecated || row.type !== "text-generation" || !tags.includes("openai") || !tags.includes("structured-output") || pricing.type !== "tokens" || input === null || output === null) {
187
+ return [];
188
+ }
189
+ return [
190
+ {
191
+ id: row.model_name,
192
+ name: typeof row.name === "string" ? row.name : row.model_name,
193
+ inputUsd: input / 100 * 1e6,
194
+ outputUsd: output / 100 * 1e6,
195
+ cachedInputUsd: input / 100 * (rate(pricing.rate_per_input_token_cached) ?? 1) * 1e6,
196
+ contextLength: rate(row.max_tokens ?? row.context_length),
197
+ canDisableReasoning: tags.includes("can-disable-reasoning")
198
+ }
199
+ ];
200
+ }),
201
+ requestOptions: ({ model, conversationId, legacyPriority }) => ({
202
+ ...conversationId ? { prompt_cache_key: conversationId } : {},
203
+ ...!model || model.canDisableReasoning ? { reasoning_effort: "none" } : {},
204
+ ...legacyPriority ? { service_tier: "priority" } : {}
205
+ }),
206
+ quoteUsage: quoteTokenUsage
207
+ };
208
+
209
+ // ../ts-shared/src/llm-providers/gemini.ts
210
+ var GEMINI_MODELS = {
211
+ "gemini-3.8-flash": {
212
+ id: "gemini-3.8-flash",
213
+ name: "Gemini 3.8 Flash",
214
+ inputUsd: 0.75,
215
+ outputUsd: 3.75,
216
+ cachedInputUsd: 0.075,
217
+ contextLength: 1048576
218
+ }
219
+ };
220
+ var gemini = {
221
+ name: "Google Gemini",
222
+ baseUrl: "https://generativelanguage.googleapis.com/v1beta/openai",
223
+ modelsUrl: "https://generativelanguage.googleapis.com/v1beta/openai/models",
224
+ billingMode: "token-rates",
225
+ parseModels: (payload) => {
226
+ const seen = /* @__PURE__ */ new Set();
227
+ return catalogRows(record(payload).data).flatMap((row) => {
228
+ const rawId = typeof row.id === "string" ? row.id.trim() : "";
229
+ const id = rawId.replace(/^models\//, "");
230
+ const model = Object.hasOwn(GEMINI_MODELS, id) ? GEMINI_MODELS[id] : void 0;
231
+ if (!model || row.deprecated || seen.has(id)) {
232
+ return [];
233
+ }
234
+ seen.add(id);
235
+ return [{ ...model }];
236
+ });
237
+ },
238
+ requestOptions: () => ({ reasoning_effort: "low" }),
239
+ quoteUsage: (price, usage) => {
240
+ validateUsageCounts(usage);
241
+ if (usage.prompt_tokens !== void 0 && usage.completion_tokens !== void 0 && usage.total_tokens !== void 0 && usage.total_tokens < usage.prompt_tokens + usage.completion_tokens) {
242
+ throw new Error("Provider returned invalid token usage");
243
+ }
244
+ const outputTokens = usage.total_tokens !== void 0 && usage.prompt_tokens !== void 0 ? usage.total_tokens - usage.prompt_tokens : usage.completion_tokens;
245
+ return quoteTokenUsage(price, {
246
+ ...usage,
247
+ completion_tokens: outputTokens
248
+ });
249
+ }
250
+ };
251
+
252
+ // ../ts-shared/src/llm-providers/openrouter.ts
253
+ var openrouter = {
254
+ name: "OpenRouter",
255
+ keyCheckUrl: "https://openrouter.ai/api/v1/key",
256
+ baseUrl: "https://openrouter.ai/api/v1",
257
+ modelsUrl: "https://openrouter.ai/api/v1/models",
258
+ billingMode: "reported-cost",
259
+ parseModels: parseCompatibleModels,
260
+ requestOptions: ({ conversationId }) => conversationId ? { session_id: conversationId } : {},
261
+ quoteUsage: (_price, usage) => {
262
+ validateUsageCounts(usage);
263
+ if (!validUsd(usage.cost)) {
264
+ throw new Error("OpenRouter did not report a valid account cost");
265
+ }
266
+ return { usd: usage.cost, source: "provider-reported" };
267
+ },
268
+ needsUsageRecovery: (usage) => usage.cost === void 0 || usage.cost === null,
269
+ recoverUsage: async (usage, apiKey) => {
270
+ const id = usage.billing?.generationId;
271
+ if (!id) {
272
+ throw new Error("OpenRouter generation ID is missing");
273
+ }
274
+ const url = new URL("https://openrouter.ai/api/v1/generation");
275
+ url.searchParams.set("id", id);
276
+ const response = await fetch(url, {
277
+ headers: { Authorization: `Bearer ${apiKey}` },
278
+ signal: AbortSignal.timeout(15e3),
279
+ redirect: "error"
280
+ });
281
+ if (!response.ok) {
282
+ throw new Error(`OpenRouter usage lookup failed (${response.status})`);
283
+ }
284
+ const data = record(record(await response.json()).data);
285
+ const finished = typeof data.finish_reason === "string" && data.finish_reason.length > 0 || data.cancelled === true;
286
+ if (data.id !== id || !validUsd(data.total_cost) || !finished) {
287
+ throw new Error("OpenRouter generation cost is unavailable");
288
+ }
289
+ return {
290
+ ...usage,
291
+ cost: data.total_cost,
292
+ prompt_tokens_details: {
293
+ ...usage.prompt_tokens_details,
294
+ ...Number.isSafeInteger(data.native_tokens_cached) && Number(data.native_tokens_cached) >= 0 ? { cached_tokens: Number(data.native_tokens_cached) } : {}
295
+ },
296
+ ...Number.isSafeInteger(data.native_tokens_prompt) && Number(data.native_tokens_prompt) >= 0 ? { prompt_tokens: Number(data.native_tokens_prompt) } : {},
297
+ ...Number.isSafeInteger(data.native_tokens_completion) && Number(data.native_tokens_completion) >= 0 ? { completion_tokens: Number(data.native_tokens_completion) } : {}
298
+ };
299
+ }
300
+ };
301
+
302
+ // ../ts-shared/src/llm-providers.ts
303
+ var LLM_PROVIDERS = {
304
+ deepinfra,
305
+ baseten,
306
+ openrouter,
307
+ gemini
308
+ };
309
+
310
+ // ../ts-shared/src/ai-providers.ts
311
+ var staticModel = function(id, name, capabilities, priceSummary) {
312
+ return Object.freeze({
313
+ id,
314
+ name,
315
+ capabilities: [...capabilities],
316
+ ...priceSummary ? { priceSummary } : {}
317
+ });
318
+ };
319
+ var AI_PROVIDERS = Object.freeze({
320
+ deepinfra: {
321
+ name: "DeepInfra",
322
+ capabilities: ["llm"],
323
+ defaultBaseUrl: LLM_PROVIDERS.deepinfra.baseUrl,
324
+ supportsCreatorChat: true,
325
+ staticModels: []
326
+ },
327
+ openrouter: {
328
+ name: "OpenRouter",
329
+ capabilities: ["llm"],
330
+ defaultBaseUrl: LLM_PROVIDERS.openrouter.baseUrl,
331
+ supportsCreatorChat: true,
332
+ staticModels: []
333
+ },
334
+ baseten: {
335
+ name: "Baseten",
336
+ capabilities: ["llm"],
337
+ defaultBaseUrl: LLM_PROVIDERS.baseten.baseUrl,
338
+ supportsCreatorChat: true,
339
+ staticModels: []
340
+ },
341
+ gemini: {
342
+ name: "Google Gemini",
343
+ capabilities: ["llm", "tts", "stt", "vision"],
344
+ defaultBaseUrl: "https://generativelanguage.googleapis.com/v1beta",
345
+ supportsCreatorChat: true,
346
+ staticModels: [
347
+ staticModel("gemini-3.8-flash-tts", "Gemini 3.8 Flash TTS", ["tts"]),
348
+ staticModel(
349
+ "gemini-3.8-flash-lite-tts",
350
+ "Gemini 3.8 Flash Lite TTS",
351
+ ["tts"]
352
+ ),
353
+ staticModel(
354
+ "gemini-3.5-transcribe-live",
355
+ "Gemini 3.5 Transcribe Live",
356
+ ["stt"],
357
+ "$3.50 input / $21 output per 1M tokens"
358
+ ),
359
+ staticModel(
360
+ "gemini-3.5-transcribe",
361
+ "Gemini 3.5 Transcribe",
362
+ ["stt"],
363
+ "$2 input / $12 output per 1M tokens"
364
+ ),
365
+ staticModel(
366
+ "gemini-3.5-flash-lite",
367
+ "Gemini 3.5 Flash Lite",
368
+ ["vision"],
369
+ "$0.30 input / $2.50 output per 1M tokens"
370
+ )
371
+ ]
372
+ },
373
+ openai: {
374
+ name: "OpenAI",
375
+ capabilities: ["llm", "stt", "vision", "image"],
376
+ defaultBaseUrl: "https://api.openai.com/v1",
377
+ supportsCreatorChat: false,
378
+ forbidOpenRouterBaseUrl: true,
379
+ staticModels: [
380
+ staticModel(
381
+ "gpt-6.1-sol",
382
+ "GPT-6.1 Sol",
383
+ ["llm"],
384
+ "$2 input / $10 output per 1M tokens"
385
+ ),
386
+ staticModel(
387
+ "gpt-6-luna",
388
+ "GPT-6 Luna",
389
+ ["llm", "vision"],
390
+ "$0.10 input / $0.50 output per 1M tokens"
391
+ ),
392
+ staticModel(
393
+ "gpt-image-2.5-flare",
394
+ "GPT Image 2.5 Flare",
395
+ ["image"],
396
+ "$5 text / $8 image input / $30 output per 1M tokens"
397
+ ),
398
+ staticModel(
399
+ "gpt-image-2.5-sunburst",
400
+ "GPT Image 2.5 Sunburst",
401
+ ["image"],
402
+ "$5 text / $8 image input / $30 output per 1M tokens"
403
+ ),
404
+ staticModel(
405
+ "gpt-transcribe",
406
+ "GPT Transcribe",
407
+ ["stt"],
408
+ "$0.0045/min"
409
+ ),
410
+ staticModel(
411
+ "gpt-live-transcribe",
412
+ "GPT Live Transcribe",
413
+ ["stt"],
414
+ "$0.017/min"
415
+ )
416
+ ]
417
+ },
418
+ anthropic: {
419
+ name: "Anthropic",
420
+ capabilities: ["llm"],
421
+ defaultBaseUrl: "https://api.anthropic.com",
422
+ supportsCreatorChat: false,
423
+ staticModels: [
424
+ staticModel(
425
+ "claude-opus-5-5",
426
+ "Claude Opus 5.5",
427
+ ["llm"],
428
+ "$4 input / $20 output per 1M tokens"
429
+ ),
430
+ staticModel(
431
+ "claude-sonnet-5-5",
432
+ "Claude Sonnet 5.5",
433
+ ["llm"],
434
+ "$2 input / $10 output per 1M tokens"
435
+ ),
436
+ staticModel(
437
+ "claude-haiku-5-5",
438
+ "Claude Haiku 5.5",
439
+ ["llm"],
440
+ "$0.10 input / $0.50 output per 1M tokens"
441
+ )
442
+ ]
443
+ },
444
+ elevenlabs: {
445
+ name: "ElevenLabs",
446
+ capabilities: ["tts", "stt"],
447
+ defaultBaseUrl: "https://api.elevenlabs.io/v1",
448
+ supportsCreatorChat: false,
449
+ staticModels: [
450
+ staticModel(
451
+ "eleven_v3_conversational",
452
+ "Eleven v3 Conversational",
453
+ ["tts"]
454
+ ),
455
+ staticModel("eleven_v3", "Eleven v3", ["tts"]),
456
+ staticModel(
457
+ "scribe_v2_realtime",
458
+ "Scribe v2 Realtime",
459
+ ["stt"],
460
+ "$0.39/hr"
461
+ ),
462
+ staticModel("scribe_v2", "Scribe v2", ["stt"], "$0.22/hr")
463
+ ]
464
+ },
465
+ tripo: {
466
+ name: "Tripo",
467
+ capabilities: ["3d"],
468
+ defaultBaseUrl: "https://openapi.tripo3d.ai/v3",
469
+ supportsCreatorChat: false,
470
+ staticModels: [
471
+ staticModel(
472
+ "P2-20260801",
473
+ "Tripo P2",
474
+ ["3d"],
475
+ "$0.01/credit (P2 4K)"
476
+ ),
477
+ staticModel(
478
+ "v1.0-20240301",
479
+ "Tripo Rig v1.0",
480
+ ["3d"],
481
+ "$0.01/credit (25 credits/rig)"
482
+ )
483
+ ]
484
+ },
485
+ meshy: {
486
+ name: "Meshy",
487
+ capabilities: ["3d"],
488
+ defaultBaseUrl: "https://api.meshy.ai",
489
+ supportsCreatorChat: false,
490
+ staticModels: [
491
+ staticModel(
492
+ "meshy-7.1",
493
+ "Meshy 7.1",
494
+ ["3d"],
495
+ "$0.02/credit (20-30 credits/model)"
496
+ ),
497
+ staticModel(
498
+ "meshy-6",
499
+ "Meshy 6",
500
+ ["3d"],
501
+ "$0.02/credit (20-30 credits/model)"
502
+ )
503
+ ]
504
+ },
505
+ unirig: {
506
+ name: "UniRig",
507
+ capabilities: ["3d"],
508
+ defaultBaseUrl: "https://modal.run",
509
+ supportsCreatorChat: false,
510
+ staticModels: [
511
+ staticModel(
512
+ "unirig-v1",
513
+ "UniRig (Modal GPU)",
514
+ ["3d"],
515
+ "Modal A10 GPU billed per second"
516
+ )
517
+ ]
518
+ }
519
+ });
520
+
45
521
  // ../ts-shared/src/ai-settings.ts
522
+ var AI_VENDOR_NAMES = {
523
+ openai: AI_PROVIDERS.openai.name,
524
+ tripo: AI_PROVIDERS.tripo.name,
525
+ meshy: AI_PROVIDERS.meshy.name,
526
+ unirig: AI_PROVIDERS.unirig.name,
527
+ elevenlabs: AI_PROVIDERS.elevenlabs.name,
528
+ gemini: AI_PROVIDERS.gemini.name,
529
+ anthropic: AI_PROVIDERS.anthropic.name
530
+ };
46
531
  var GPT_IMAGE_MODELS = [
47
532
  "gpt-image-2.5-flare",
48
533
  "gpt-image-2.5-sunburst"
@@ -50,62 +535,124 @@ var GPT_IMAGE_MODELS = [
50
535
  var AI_ROLES = {
51
536
  character_draft: {
52
537
  name: "Character draft",
53
- source: "llm",
54
- models: [],
538
+ capability: "llm",
539
+ chatLlm: true,
55
540
  options: ["temperature"]
56
541
  },
57
542
  motion_planner: {
58
543
  name: "Motion planner",
59
- source: "llm",
60
- models: [],
544
+ capability: "llm",
545
+ chatLlm: true,
61
546
  options: []
62
547
  },
63
- // Not charged.
64
548
  prompt_check: {
65
549
  name: "Prompt check",
66
- source: "openai",
67
- models: ["gpt-6-luna"],
550
+ capability: "llm",
551
+ providers: ["openai"],
552
+ allowedModels: ["gpt-6-luna"],
68
553
  options: []
69
554
  },
70
- // Priced by OPENAI_IMAGE_PRICES of the server (flare and sunburst alike).
71
555
  design_sheet_image: {
72
556
  name: "Design sheet image",
73
- source: "openai",
74
- models: GPT_IMAGE_MODELS,
557
+ capability: "image",
558
+ providers: ["openai"],
559
+ allowedModels: GPT_IMAGE_MODELS,
75
560
  options: []
76
561
  },
77
- // Not charged per call.
78
562
  tts: {
79
563
  name: "ElevenLabs text to speech",
80
- source: "elevenlabs",
81
- models: ["eleven_v3_conversational", "eleven_v3"],
564
+ capability: "tts",
565
+ providers: ["elevenlabs"],
566
+ allowedModels: ["eleven_v3_conversational", "eleven_v3"],
82
567
  options: []
83
568
  },
84
- // Not charged per call.
85
569
  gemini_tts: {
86
570
  name: "Gemini text to speech",
87
- source: "gemini",
88
- models: ["gemini-3.8-flash-tts", "gemini-3.8-flash-lite-tts"],
571
+ capability: "tts",
572
+ providers: ["gemini"],
573
+ allowedModels: ["gemini-3.8-flash-tts", "gemini-3.8-flash-lite-tts"],
574
+ options: []
575
+ },
576
+ stt_openai: {
577
+ name: "OpenAI speech recognition",
578
+ capability: "stt",
579
+ providers: ["openai"],
580
+ allowedModels: ["gpt-transcribe", "gpt-live-transcribe"],
581
+ options: []
582
+ },
583
+ stt_gemini: {
584
+ name: "Gemini speech recognition",
585
+ capability: "stt",
586
+ providers: ["gemini"],
587
+ allowedModels: ["gemini-3.5-transcribe-live"],
588
+ options: []
589
+ },
590
+ stt_elevenlabs: {
591
+ name: "ElevenLabs speech recognition",
592
+ capability: "stt",
593
+ providers: ["elevenlabs"],
594
+ allowedModels: ["scribe_v2_realtime"],
595
+ options: []
596
+ },
597
+ vision_gemini: {
598
+ name: "Gemini sight",
599
+ capability: "vision",
600
+ providers: ["gemini"],
601
+ allowedModels: ["gemini-3.5-flash-lite"],
602
+ options: []
603
+ },
604
+ vision_openai: {
605
+ name: "OpenAI sight",
606
+ capability: "vision",
607
+ providers: ["openai"],
608
+ allowedModels: ["gpt-6-luna"],
89
609
  options: []
90
610
  },
91
- // Priced by OPENAI_PRICES.text of the generator.
92
611
  generator_agent: {
93
612
  name: "Generator agent",
94
- source: "openai",
95
- models: ["gpt-6.1-sol"],
613
+ capability: "llm",
614
+ providers: ["openai", "anthropic"],
615
+ allowedModels: [
616
+ "gpt-6.1-sol",
617
+ "claude-opus-5-5",
618
+ "claude-sonnet-5-5",
619
+ "claude-haiku-5-5"
620
+ ],
96
621
  options: ["reasoningEffort"]
97
622
  },
98
623
  generator_reading: {
99
624
  name: "Generator face reading",
100
- source: "openai",
101
- models: ["gpt-6.1-sol"],
625
+ capability: "llm",
626
+ providers: ["openai"],
627
+ allowedModels: ["gpt-6.1-sol"],
102
628
  options: ["reasoningEffort"]
103
629
  },
104
- // Priced by OPENAI_PRICES of the generator (flare and sunburst alike).
105
630
  generator_image: {
106
631
  name: "Generator images",
107
- source: "openai",
108
- models: GPT_IMAGE_MODELS,
632
+ capability: "image",
633
+ providers: ["openai"],
634
+ allowedModels: GPT_IMAGE_MODELS,
635
+ options: []
636
+ },
637
+ generator_3d: {
638
+ name: "Generator 3D head",
639
+ capability: "3d",
640
+ providers: ["tripo"],
641
+ allowedModels: ["P2-20260801"],
642
+ options: []
643
+ },
644
+ generator_3d_body: {
645
+ name: "Generator 3D body",
646
+ capability: "3d",
647
+ providers: ["tripo", "meshy"],
648
+ allowedModels: ["P2-20260801", "meshy-7.1", "meshy-6"],
649
+ options: []
650
+ },
651
+ generator_rig: {
652
+ name: "Generator 3D rig",
653
+ capability: "3d",
654
+ providers: ["tripo", "unirig"],
655
+ allowedModels: ["v1.0-20240301", "P2-20260801", "unirig-v1"],
109
656
  options: []
110
657
  }
111
658
  };
@@ -242,6 +789,43 @@ var SPEECH_INPUT_PROVIDER_PROFILES = Object.freeze({
242
789
  }
243
790
  });
244
791
 
792
+ // ../ts-shared/src/sight.ts
793
+ var SIGHT_EVENTS = Object.freeze({
794
+ /** Client to server: one JPEG frame, acknowledged with a `SightFrameAck`. */
795
+ frame: "conversations:sight",
796
+ /** Server to the conversation's room: one `SightObservationEvent`. */
797
+ observation: "sight_observation"
798
+ });
799
+ var SIGHT_LIMITS = Object.freeze({
800
+ /** The largest JPEG frame, in bytes (200 KiB). */
801
+ maxFrameBytes: 200 * 1024,
802
+ /** The longest edge of a frame or a crop, in pixels. */
803
+ maxEdgePx: 1280,
804
+ /** The shortest time between two frames of one conversation. */
805
+ minFrameIntervalMs: 1e3,
806
+ maxCrops: 4,
807
+ /** The largest JPEG crop, in bytes (100 KiB). */
808
+ maxCropBytes: 100 * 1024,
809
+ maxCropLabelLength: 40
810
+ });
811
+ var SIGHT_CAPTURE_SIZE = Object.freeze({
812
+ width: 640,
813
+ height: 360,
814
+ jpegQuality: 0.7
815
+ });
816
+ var SIGHT_PROVIDER_PROFILES = Object.freeze({
817
+ gemini: {
818
+ model: "gemini-3.5-flash-lite",
819
+ frameToObservationMs: 1280,
820
+ usdPerFrame: 29e-5
821
+ },
822
+ openai: {
823
+ model: "gpt-6-luna",
824
+ frameToObservationMs: 2140,
825
+ usdPerFrame: 74e-6
826
+ }
827
+ });
828
+
245
829
  // ../ts-shared/src/product-usage.ts
246
830
  var productUsageEvents = [
247
831
  "page_viewed",
@@ -254,7 +838,8 @@ var productUsageEvents = [
254
838
  "playback_blocked",
255
839
  "playback_start_pressed",
256
840
  "reply_options_shown",
257
- "page_left"
841
+ "page_left",
842
+ "creation_mode_chosen"
258
843
  ];
259
844
 
260
845
  // ../ts-shared/src/analytics.ts
@@ -2636,7 +3221,7 @@ var PERFORMANCE_KINDS = {
2636
3221
  cachedOpening: 8
2637
3222
  };
2638
3223
  var PERFORMANCE_MAJOR_VERSION = 1;
2639
- var PERFORMANCE_MINOR_VERSION = 1;
3224
+ var PERFORMANCE_MINOR_VERSION = 2;
2640
3225
  var HEADER_SIZE = 24;
2641
3226
  var MAGIC = 1346588750;
2642
3227
  var crcTable = Uint32Array.from({ length: 256 }, (_, index) => {
@@ -2788,6 +3373,68 @@ var decodePerformance = function(input, expectedKind) {
2788
3373
  return wireValue(value);
2789
3374
  };
2790
3375
 
3376
+ // ../ts-shared/src/player-memory.ts
3377
+ var MEMORY_IDENTIFIER_MAX_LENGTH = 128;
3378
+ var MEMORY_IMPORT_MAX_BYTES = 512 * 1024;
3379
+ var normalizeMemoryIdentifier = function(value) {
3380
+ if (typeof value !== "string") {
3381
+ return null;
3382
+ }
3383
+ const cleaned = value.normalize("NFKC").replace(/[\u0000-\u001f\u007f]/gu, "").trim();
3384
+ if (!cleaned) {
3385
+ return null;
3386
+ }
3387
+ return cleaned.slice(0, MEMORY_IDENTIFIER_MAX_LENGTH).trim() || null;
3388
+ };
3389
+
3390
+ // ../ts-shared/src/message-time.ts
3391
+ var MESSAGE_TIME_ZONE_MAX_LENGTH = 64;
3392
+ var TIME_ZONE_PATTERN = /^[A-Za-z0-9_+-]+(?:\/[A-Za-z0-9_+-]+)*$/u;
3393
+ var OFFSET_PATTERN = /^[+-](?:[01]\d|2[0-3]):[0-5]\d$/u;
3394
+ var normalizeMessageTimeZone = function(value) {
3395
+ if (typeof value !== "string") {
3396
+ return null;
3397
+ }
3398
+ const trimmed = value.trim();
3399
+ if (!trimmed || trimmed.length > MESSAGE_TIME_ZONE_MAX_LENGTH) {
3400
+ return null;
3401
+ }
3402
+ if (trimmed === "Z" || trimmed === "z" || trimmed === "+00:00" || trimmed === "-00:00") {
3403
+ return "UTC";
3404
+ }
3405
+ if (!OFFSET_PATTERN.test(trimmed) && !TIME_ZONE_PATTERN.test(trimmed)) {
3406
+ return null;
3407
+ }
3408
+ try {
3409
+ new Intl.DateTimeFormat("en-US", { timeZone: trimmed }).format(/* @__PURE__ */ new Date(0));
3410
+ return trimmed;
3411
+ } catch {
3412
+ return null;
3413
+ }
3414
+ };
3415
+ var formatUtcOffsetMinutes = function(offsetMinutesEastOfUtc) {
3416
+ if (!Number.isFinite(offsetMinutesEastOfUtc) || offsetMinutesEastOfUtc === 0) {
3417
+ return "UTC";
3418
+ }
3419
+ const total = Math.trunc(offsetMinutesEastOfUtc);
3420
+ const sign = total >= 0 ? "+" : "-";
3421
+ const abs = Math.abs(total);
3422
+ const hours = String(Math.floor(abs / 60)).padStart(2, "0");
3423
+ const minutes = String(abs % 60).padStart(2, "0");
3424
+ return `${sign}${hours}:${minutes}`;
3425
+ };
3426
+ var detectClientTimeZone = function(now = /* @__PURE__ */ new Date()) {
3427
+ try {
3428
+ const resolved = Intl.DateTimeFormat().resolvedOptions().timeZone;
3429
+ const normalized = normalizeMessageTimeZone(resolved);
3430
+ if (normalized) {
3431
+ return normalized;
3432
+ }
3433
+ } catch {
3434
+ }
3435
+ return formatUtcOffsetMinutes(-now.getTimezoneOffset());
3436
+ };
3437
+
2791
3438
  // ../../clients/web-client/src/api/performance.ts
2792
3439
  var readPerformanceBytes = async function(response) {
2793
3440
  const maximum = PERFORMANCE_MAX_BYTES + 65535;
@@ -3031,7 +3678,9 @@ var hasTimelineShape = function(value) {
3031
3678
  }
3032
3679
  previousDialogueEnd = Number(block.endTime);
3033
3680
  return true;
3034
- }) && isSameTime(previousDialogueEnd, value.duration) && hasOrderedItems(value.expressions, value.duration, hasExpressionShape) && hasOrderedItems(value.animations, value.duration, hasAnimationShape);
3681
+ }) && isSameTime(previousDialogueEnd, value.duration) && hasOrderedItems(value.expressions, value.duration, hasExpressionShape) && hasOrderedItems(value.animations, value.duration, hasAnimationShape) && (value.initialVariant === void 0 || isValidVariantKey(value.initialVariant)) && (value.activeVariant === void 0 || isValidVariantKey(value.activeVariant)) && (value.toolCalls === void 0 || Array.isArray(value.toolCalls) && value.toolCalls.every(
3682
+ (call) => isStagingToolCall(call) && isWithinDuration(call.timestamp, value.duration)
3683
+ ));
3035
3684
  };
3036
3685
  var normalizeStagingResultPayload = function(value) {
3037
3686
  if (!isRecord(value)) {
@@ -3141,14 +3790,20 @@ var BakedConversation = class {
3141
3790
  conversationId,
3142
3791
  openingStagingId: this.asset.rootStagingId,
3143
3792
  activeStagingId: this.activeId,
3144
- stagings: [...this.nodes.values()].map((node) => ({
3145
- id: node.id,
3146
- parentId: node.parentId,
3147
- triggerText: node.triggerText,
3148
- text: node.payload.text,
3149
- options: structuredClone(node.payload.options),
3150
- createdAt: "1970-01-01T00:00:00.000Z"
3151
- }))
3793
+ stagings: [...this.nodes.values()].map((node) => {
3794
+ const { initialVariant, activeVariant, toolCalls } = node.payload.timeline;
3795
+ return {
3796
+ id: node.id,
3797
+ parentId: node.parentId,
3798
+ triggerText: node.triggerText,
3799
+ text: node.payload.text,
3800
+ options: structuredClone(node.payload.options),
3801
+ createdAt: "1970-01-01T00:00:00.000Z",
3802
+ ...initialVariant !== void 0 ? { initialVariant } : {},
3803
+ ...activeVariant !== void 0 ? { activeVariant } : {},
3804
+ ...toolCalls ? { toolCalls: structuredClone(toolCalls) } : {}
3805
+ };
3806
+ })
3152
3807
  };
3153
3808
  }
3154
3809
  child(parentId, label) {
@@ -3188,7 +3843,9 @@ var createConversation = async function({
3188
3843
  avatarId,
3189
3844
  apiToken,
3190
3845
  baseUrl,
3191
- renderSessionId
3846
+ renderSessionId,
3847
+ hostedWebChat,
3848
+ memoryIdentifier
3192
3849
  }) {
3193
3850
  const url = new URL("/conversations", baseUrl);
3194
3851
  const res = await fetch(url.toString(), {
@@ -3202,7 +3859,12 @@ var createConversation = async function({
3202
3859
  credentials: apiToken ? "omit" : "include",
3203
3860
  method: "POST",
3204
3861
  // This SDK plays live motion frames (ADR 0011).
3205
- body: JSON.stringify({ avatarId, capabilities: [MOTION_FRAMES_CAPABILITY] })
3862
+ body: JSON.stringify({
3863
+ avatarId,
3864
+ capabilities: [MOTION_FRAMES_CAPABILITY],
3865
+ ...hostedWebChat ? { hostedWebChat: true } : {},
3866
+ ...memoryIdentifier !== void 0 ? { memoryIdentifier } : {}
3867
+ })
3206
3868
  });
3207
3869
  if (!res.ok) {
3208
3870
  throw Object.assign(new Error(await res.text()), { status: res.status });
@@ -3225,7 +3887,8 @@ var createStaging = async function({
3225
3887
  requestId,
3226
3888
  silent,
3227
3889
  parentStagingId,
3228
- renderSessionId
3890
+ renderSessionId,
3891
+ memoryIdentifier
3229
3892
  }) {
3230
3893
  const url = new URL(`/conversations/${conversationId}/stagings`, baseUrl);
3231
3894
  const requestHeaders = withRenderSession(
@@ -3256,7 +3919,9 @@ var createStaging = async function({
3256
3919
  silent: silent === true,
3257
3920
  incrementalPlayback: true,
3258
3921
  parentStagingId,
3259
- capabilities: [MOTION_FRAMES_CAPABILITY]
3922
+ capabilities: [MOTION_FRAMES_CAPABILITY],
3923
+ timeZone: detectClientTimeZone(),
3924
+ ...memoryIdentifier !== void 0 ? { memoryIdentifier } : {}
3260
3925
  })
3261
3926
  });
3262
3927
  } catch (error) {
@@ -3440,7 +4105,8 @@ var LiveConversation = class {
3440
4105
  silent,
3441
4106
  parentStagingId,
3442
4107
  headers: telemetry.traceHeaders,
3443
- renderSessionId: this.dependencies.renderSession()
4108
+ renderSessionId: this.dependencies.renderSession(),
4109
+ memoryIdentifier: this.dependencies.memoryIdentifier?.()
3444
4110
  });
3445
4111
  } catch (error) {
3446
4112
  if (request.completed) {
@@ -3702,25 +4368,61 @@ var loadGltfWithProgress = async function({
3702
4368
  });
3703
4369
  }
3704
4370
  };
3705
- var loadVrm = async ({
3706
- baseUrl,
3707
- characterId,
3708
- apiToken,
3709
- onProgress,
3710
- assetProfile
3711
- }) => {
3712
- const url = new URL(`/avatars/${characterId}/download-url`, baseUrl);
3713
- if (assetProfile && assetProfile !== "original") {
3714
- url.searchParams.set("assetProfile", assetProfile);
4371
+ var fetchVrmDownloadUrl = async function(baseUrl, characterId, options) {
4372
+ const url = new URL(
4373
+ `/avatars/${encodeURIComponent(characterId)}/download-url`,
4374
+ baseUrl
4375
+ );
4376
+ if (options?.assetProfile && options.assetProfile !== "original") {
4377
+ url.searchParams.set("assetProfile", options.assetProfile);
3715
4378
  }
3716
4379
  const res = await fetch(url.toString(), {
3717
- headers: buildAuthHeaders(apiToken, { allowEmpty: true }),
3718
- credentials: apiToken ? "omit" : "include"
4380
+ headers: buildAuthHeaders(options?.apiToken, { allowEmpty: true }),
4381
+ credentials: options?.apiToken ? "omit" : "include",
4382
+ ...options?.signal ? { signal: options.signal } : {}
4383
+ });
4384
+ if (!res.ok) {
4385
+ throw new Error(await res.text());
4386
+ }
4387
+ const { url: vrmUrl } = await res.json();
4388
+ return vrmUrl;
4389
+ };
4390
+ var fetchVariantVrmDownloadUrl = async function(baseUrl, characterId, variantKey, options) {
4391
+ if (variantKey === DEFAULT_VARIANT_KEY) {
4392
+ return await fetchVrmDownloadUrl(baseUrl, characterId, options);
4393
+ }
4394
+ const url = new URL(
4395
+ `/avatars/${encodeURIComponent(characterId)}/variants/${encodeURIComponent(variantKey)}/download-url`,
4396
+ baseUrl
4397
+ );
4398
+ if (options?.assetProfile && options.assetProfile !== "original") {
4399
+ url.searchParams.set("assetProfile", options.assetProfile);
4400
+ }
4401
+ const res = await fetch(url.toString(), {
4402
+ headers: buildAuthHeaders(options?.apiToken, { allowEmpty: true }),
4403
+ credentials: options?.apiToken ? "omit" : "include",
4404
+ ...options?.signal ? { signal: options.signal } : {}
3719
4405
  });
3720
4406
  if (!res.ok) {
3721
4407
  throw new Error(await res.text());
3722
4408
  }
3723
4409
  const { url: vrmUrl } = await res.json();
4410
+ return vrmUrl;
4411
+ };
4412
+ var loadVrm = async function({
4413
+ baseUrl,
4414
+ characterId,
4415
+ variantKey,
4416
+ apiToken,
4417
+ onProgress,
4418
+ assetProfile
4419
+ }) {
4420
+ const vrmUrl = await fetchVariantVrmDownloadUrl(
4421
+ baseUrl,
4422
+ characterId,
4423
+ variantKey ?? DEFAULT_VARIANT_KEY,
4424
+ { apiToken, assetProfile }
4425
+ );
3724
4426
  const gltf = await loadGltfWithProgress({
3725
4427
  url: vrmUrl,
3726
4428
  onProgress,
@@ -3732,6 +4434,23 @@ var loadVrm = async ({
3732
4434
  VRMUtils.combineSkeletons(gltf.scene);
3733
4435
  return gltf.userData.vrm;
3734
4436
  };
4437
+ var loadVariantVrm = async function({
4438
+ baseUrl,
4439
+ characterId,
4440
+ variantKey,
4441
+ apiToken,
4442
+ onProgress,
4443
+ assetProfile
4444
+ }) {
4445
+ return await loadVrm({
4446
+ baseUrl,
4447
+ characterId,
4448
+ variantKey,
4449
+ apiToken,
4450
+ onProgress,
4451
+ assetProfile
4452
+ });
4453
+ };
3735
4454
  var loadVrma = async ({
3736
4455
  baseUrl,
3737
4456
  characterId,
@@ -5314,6 +6033,70 @@ var AnimationController = class {
5314
6033
  activeAnimationId = null;
5315
6034
  idleAnimationId;
5316
6035
  pendingReturnToIdleAction = null;
6036
+ setVrm(nextVrm, rawAnimations) {
6037
+ if (nextVrm === this.vrm) {
6038
+ return;
6039
+ }
6040
+ const currentAnimationId = this.activeAnimationId;
6041
+ const currentAction = this.activeAction;
6042
+ const currentTime = currentAction ? currentAction.time : 0;
6043
+ const currentLoop = currentAction ? currentAction.loop === LoopRepeat2 : void 0;
6044
+ const hadPendingReturnToIdle = this.pendingReturnToIdleAction !== null && this.pendingReturnToIdleAction === currentAction;
6045
+ const savedConfigs = new Map(this.configs);
6046
+ this.restorePreview?.();
6047
+ this.mixer.removeEventListener("finished", this.handleActionFinished);
6048
+ this.mixer.stopAllAction();
6049
+ this.mixer.uncacheRoot(this.vrm.scene);
6050
+ this.vrm.humanoid?.resetNormalizedPose?.();
6051
+ this.vrm = nextVrm;
6052
+ this.mixer = new AnimationMixer2(this.vrm.scene);
6053
+ this.mixer.addEventListener("finished", this.handleActionFinished);
6054
+ this.clips.clear();
6055
+ this.rawClips.clear();
6056
+ this.normalizedIdleClips.clear();
6057
+ this.configs.clear();
6058
+ this.activeAction = null;
6059
+ this.activeAnimationId = null;
6060
+ this.pendingReturnToIdleAction = null;
6061
+ for (const [animationId, animation] of rawAnimations) {
6062
+ const config = savedConfigs.get(animationId) ?? {
6063
+ loop: animationId === this.idleAnimationId,
6064
+ weight: defaultAnimationConfig.weight
6065
+ };
6066
+ this.addAnimation(animationId, animation, config);
6067
+ }
6068
+ const neededIds = [];
6069
+ if (currentAnimationId && !this.clips.has(currentAnimationId)) {
6070
+ neededIds.push(currentAnimationId);
6071
+ }
6072
+ if (this.idleAnimationId && !this.clips.has(this.idleAnimationId) && !neededIds.includes(this.idleAnimationId)) {
6073
+ neededIds.push(this.idleAnimationId);
6074
+ }
6075
+ if (neededIds.length > 0 && this.prepareAnimations) {
6076
+ void this.prepareAnimations(neededIds).catch(() => void 0);
6077
+ }
6078
+ if (currentAnimationId && this.clips.has(currentAnimationId)) {
6079
+ this.playAnimation(currentAnimationId, 0, currentLoop);
6080
+ const resumedAction = this.getActiveAction();
6081
+ if (resumedAction) {
6082
+ const clipDuration = resumedAction.getClip().duration;
6083
+ if (clipDuration > 0 && Number.isFinite(currentTime)) {
6084
+ resumedAction.time = resumedAction.loop === LoopRepeat2 ? (currentTime % clipDuration + clipDuration) % clipDuration : Math.min(Math.max(0, currentTime), clipDuration);
6085
+ }
6086
+ if (hadPendingReturnToIdle && !currentLoop && this.idleAnimationId) {
6087
+ this.pendingReturnToIdleAction = resumedAction;
6088
+ }
6089
+ }
6090
+ this.mixer.update(0);
6091
+ return;
6092
+ }
6093
+ if (this.idleAnimationId && this.clips.has(this.idleAnimationId)) {
6094
+ this.playIdle(0);
6095
+ this.mixer.update(0);
6096
+ return;
6097
+ }
6098
+ this.vrm.humanoid?.resetNormalizedPose?.();
6099
+ }
5317
6100
  getAnimationMixer() {
5318
6101
  return this.mixer;
5319
6102
  }
@@ -5989,6 +6772,35 @@ var PerformanceRecorder = class {
5989
6772
  };
5990
6773
 
5991
6774
  // ../../clients/web-client/src/playback-utils.ts
6775
+ var deliverStagingResult = async function(payload, options) {
6776
+ const normalized = normalizeStagingResultPayload(payload);
6777
+ options.setOptions(normalized.options);
6778
+ if (options.source === "turn") {
6779
+ options.onTurnReceived(normalized);
6780
+ }
6781
+ const force = options.forcePlayback === true || options.source === "replay";
6782
+ const shouldNotify = force || shouldNotifyStagingResult(options.notifiedStagingIds, normalized.stagingId);
6783
+ if (shouldNotify && options.onStagingResult) {
6784
+ try {
6785
+ await options.onStagingResult(normalized);
6786
+ } catch (error) {
6787
+ options.onCallbackError("onStagingResult", error);
6788
+ }
6789
+ }
6790
+ const shouldPlay = !options.hasPlayed(normalized.stagingId) || force;
6791
+ options.recordEvent({
6792
+ type: "staging.result",
6793
+ source: options.source,
6794
+ stagingId: normalized.stagingId,
6795
+ taskId: normalized.taskId ?? "",
6796
+ playback: shouldPlay ? "requested" : "already-streamed"
6797
+ });
6798
+ if (shouldPlay) {
6799
+ options.playRequested(normalized, { force, silent: options.silent });
6800
+ } else {
6801
+ options.playStreamed(normalized);
6802
+ }
6803
+ };
5992
6804
  var toSilentTimeline = function(timeline) {
5993
6805
  return {
5994
6806
  ...timeline,
@@ -6115,6 +6927,16 @@ var StreamPlaybackController = class {
6115
6927
  get activeStagingId() {
6116
6928
  return this.activePlayback?.stagingId ?? null;
6117
6929
  }
6930
+ /**
6931
+ * A staging plays or waits to play (its audio or the rest of its stream).
6932
+ * False after it completed or when autoplay blocked it; the last staging
6933
+ * stays the active staging after that.
6934
+ */
6935
+ get playing() {
6936
+ const active = this.activePlayback;
6937
+ const state = active ? this.states.get(active.stagingId) : void 0;
6938
+ return state !== void 0 && !state.completed && !state.blocked;
6939
+ }
6118
6940
  onStreamEvent(callback) {
6119
6941
  return bindCallbackSlot(
6120
6942
  () => this.streamEventCallback,
@@ -6411,7 +7233,11 @@ var StreamPlaybackController = class {
6411
7233
  () => this.characterRevealCallback?.(event, characterIndex, character)
6412
7234
  );
6413
7235
  }
6414
- }
7236
+ },
7237
+ onToolCall: (toolCall) => {
7238
+ this.dependencies.onToolCall?.(toolCall);
7239
+ },
7240
+ onVariantCue: (variantKey, toolCall) => this.dependencies.onVariantCue?.(variantKey, toolCall)
6415
7241
  };
6416
7242
  try {
6417
7243
  await player.play(params.timeline, playbackOptions);
@@ -7176,22 +8002,201 @@ var AudioPlayer = class {
7176
8002
  }
7177
8003
  };
7178
8004
 
7179
- // ../../clients/web-client/src/character/expressions.ts
7180
- var CharacterExpressions = class {
7181
- constructor(mixer) {
7182
- this.mixer = mixer;
8005
+ // ../../clients/web-client/src/character/character-variants.ts
8006
+ var disposeOrphanedVrm = function(vrm) {
8007
+ if (vrm.scene?.parent) {
8008
+ vrm.scene.parent.remove(vrm.scene);
7183
8009
  }
7184
- /** All available VRM expression channel names on the loaded character. */
7185
- get all() {
7186
- return this.mixer.getChannelNames();
8010
+ if (vrm.scene && typeof vrm.scene.traverse === "function") {
8011
+ disposeVrm(vrm);
7187
8012
  }
7188
- /** Available emotion expression channel names (`happy`, `angry`, `sad`, etc.). */
7189
- get emotions() {
7190
- return this.mixer.getEmotionChannelNames();
8013
+ };
8014
+ var CharacterVariantController = class {
8015
+ constructor(options) {
8016
+ this.options = options;
7191
8017
  }
7192
- /** Available mouth/viseme expression channel names (`aa`, `ee`, `ih`, `oh`, `ou`). */
7193
- get visemes() {
7194
- return this.mixer.getVisemeChannelNames();
8018
+ variantVrms = /* @__PURE__ */ new Map();
8019
+ variantsList = [];
8020
+ activeVariantKey = DEFAULT_VARIANT_KEY;
8021
+ remoteMode = false;
8022
+ loadEpoch = 0;
8023
+ switchRevision = 0;
8024
+ pendingLoads = /* @__PURE__ */ new Map();
8025
+ listeners = {
8026
+ variantChange: /* @__PURE__ */ new Set(),
8027
+ toolCall: /* @__PURE__ */ new Set(),
8028
+ vrmChange: /* @__PURE__ */ new Set()
8029
+ };
8030
+ get variants() {
8031
+ return this.variantsList.map((variant) => ({ ...variant }));
8032
+ }
8033
+ get activeVariant() {
8034
+ return this.activeVariantKey;
8035
+ }
8036
+ get isRemote() {
8037
+ return this.remoteMode;
8038
+ }
8039
+ on(event, listener) {
8040
+ const bucket = this.listeners[event];
8041
+ bucket.add(listener);
8042
+ return () => {
8043
+ bucket.delete(listener);
8044
+ };
8045
+ }
8046
+ off(event, listener) {
8047
+ this.listeners[event].delete(listener);
8048
+ }
8049
+ emit(event, payload) {
8050
+ for (const listener of [...this.listeners[event]]) {
8051
+ invokeSafeCallback(
8052
+ `on(${event})`,
8053
+ this.options.handleCallbackError,
8054
+ () => listener(payload)
8055
+ );
8056
+ }
8057
+ }
8058
+ configureVariants(variants, options) {
8059
+ this.variantsList = variants ? variants.map((entry) => ({ ...entry })) : [];
8060
+ if (options?.remote !== void 0) {
8061
+ this.remoteMode = options.remote;
8062
+ }
8063
+ }
8064
+ registerDefaultVrm(vrm) {
8065
+ this.variantVrms.set(DEFAULT_VARIANT_KEY, vrm);
8066
+ this.activeVariantKey = DEFAULT_VARIANT_KEY;
8067
+ this.remoteMode = false;
8068
+ }
8069
+ syncDirectVrm(vrm, previousVrm) {
8070
+ if (vrm && !this.variantVrms.has(this.activeVariantKey)) {
8071
+ this.variantVrms.set(this.activeVariantKey, vrm);
8072
+ }
8073
+ if (vrm && vrm !== previousVrm) {
8074
+ this.emit("vrmChange", vrm);
8075
+ }
8076
+ }
8077
+ async preloadVariants(eager) {
8078
+ const keys = this.variantsList.map((variant) => variant.key).filter((key) => key !== DEFAULT_VARIANT_KEY);
8079
+ if (keys.length === 0) {
8080
+ return;
8081
+ }
8082
+ if (eager) {
8083
+ await Promise.all(keys.map((key) => this.ensureVariant(key)));
8084
+ return;
8085
+ }
8086
+ for (const key of keys) {
8087
+ void this.ensureVariant(key).catch(() => {
8088
+ });
8089
+ }
8090
+ }
8091
+ async ensureVariant(variantKey) {
8092
+ const cached = this.variantVrms.get(variantKey);
8093
+ if (cached) {
8094
+ return cached;
8095
+ }
8096
+ const existing = this.pendingLoads.get(variantKey);
8097
+ if (existing) {
8098
+ return await existing;
8099
+ }
8100
+ const epoch = this.loadEpoch;
8101
+ const request = this.options.loadVariant(variantKey).then((loadedVrm) => {
8102
+ this.pendingLoads.delete(variantKey);
8103
+ if (epoch !== this.loadEpoch) {
8104
+ disposeOrphanedVrm(loadedVrm);
8105
+ throw new Error("Character assets were reset while loading variant.");
8106
+ }
8107
+ this.variantVrms.set(variantKey, loadedVrm);
8108
+ return loadedVrm;
8109
+ }).catch((error) => {
8110
+ this.pendingLoads.delete(variantKey);
8111
+ throw error;
8112
+ });
8113
+ this.pendingLoads.set(variantKey, request);
8114
+ return await request;
8115
+ }
8116
+ async switchVariant(variantKey, toolCall = null) {
8117
+ if (!isValidVariantKey(variantKey)) {
8118
+ throw new Error(`Invalid variant key: ${variantKey}`);
8119
+ }
8120
+ const revision = ++this.switchRevision;
8121
+ if (this.remoteMode) {
8122
+ if (variantKey !== DEFAULT_VARIANT_KEY && this.variantsList.length > 0 && !this.variantsList.some((entry) => entry.key === variantKey)) {
8123
+ throw new Error(`Unknown variant: ${variantKey}`);
8124
+ }
8125
+ if (this.activeVariantKey === variantKey) {
8126
+ return;
8127
+ }
8128
+ const previousVariant = this.activeVariantKey;
8129
+ this.activeVariantKey = variantKey;
8130
+ this.emit("variantChange", {
8131
+ previousVariant,
8132
+ activeVariant: variantKey,
8133
+ toolCall
8134
+ });
8135
+ return;
8136
+ }
8137
+ const cached = this.variantVrms.get(variantKey);
8138
+ if (cached) {
8139
+ this.applyActiveVariant(variantKey, cached, toolCall);
8140
+ return;
8141
+ }
8142
+ if (variantKey === DEFAULT_VARIANT_KEY && this.activeVariantKey === DEFAULT_VARIANT_KEY) {
8143
+ return;
8144
+ }
8145
+ const loaded = await this.ensureVariant(variantKey);
8146
+ if (revision !== this.switchRevision) {
8147
+ return;
8148
+ }
8149
+ this.applyActiveVariant(variantKey, loaded, toolCall);
8150
+ }
8151
+ reset() {
8152
+ this.loadEpoch += 1;
8153
+ this.switchRevision += 1;
8154
+ this.pendingLoads.clear();
8155
+ const cached = [...this.variantVrms.values()];
8156
+ this.variantVrms.clear();
8157
+ this.activeVariantKey = DEFAULT_VARIANT_KEY;
8158
+ this.remoteMode = false;
8159
+ return cached;
8160
+ }
8161
+ applyActiveVariant(variantKey, nextVrm, toolCall) {
8162
+ const previousVariant = this.activeVariantKey;
8163
+ const previousVrm = this.options.getCurrentVrm();
8164
+ if (previousVariant === variantKey && previousVrm === nextVrm) {
8165
+ return;
8166
+ }
8167
+ if (previousVrm !== nextVrm) {
8168
+ this.options.applyVrmSwap(nextVrm);
8169
+ }
8170
+ this.activeVariantKey = variantKey;
8171
+ if (previousVrm !== nextVrm) {
8172
+ this.emit("vrmChange", nextVrm);
8173
+ }
8174
+ if (previousVariant !== variantKey) {
8175
+ this.emit("variantChange", {
8176
+ previousVariant,
8177
+ activeVariant: variantKey,
8178
+ toolCall
8179
+ });
8180
+ }
8181
+ }
8182
+ };
8183
+
8184
+ // ../../clients/web-client/src/character/expressions.ts
8185
+ var CharacterExpressions = class {
8186
+ constructor(mixer) {
8187
+ this.mixer = mixer;
8188
+ }
8189
+ /** All available VRM expression channel names on the loaded character. */
8190
+ get all() {
8191
+ return this.mixer.getChannelNames();
8192
+ }
8193
+ /** Available emotion expression channel names (`happy`, `angry`, `sad`, etc.). */
8194
+ get emotions() {
8195
+ return this.mixer.getEmotionChannelNames();
8196
+ }
8197
+ /** Available mouth/viseme expression channel names (`aa`, `ee`, `ih`, `oh`, `ou`). */
8198
+ get visemes() {
8199
+ return this.mixer.getVisemeChannelNames();
7195
8200
  }
7196
8201
  /** Current applied weights for all expression channels in `[0, 1]`. */
7197
8202
  get weights() {
@@ -7443,6 +8448,76 @@ var IdleFace = class {
7443
8448
  }
7444
8449
  };
7445
8450
 
8451
+ // ../../clients/web-client/src/character/listening-face.ts
8452
+ var LISTENING_FACE_LAYER = "listening";
8453
+ var CHANNELS = ["relaxed", "happy"];
8454
+ var REST = 0.01;
8455
+ var ListeningFace = class {
8456
+ constructor(weights) {
8457
+ this.weights = weights;
8458
+ }
8459
+ channel = null;
8460
+ weight = 0;
8461
+ saved = /* @__PURE__ */ new Map();
8462
+ /** The preset in use, or null when the model has none of them. */
8463
+ get channelName() {
8464
+ return this.channel;
8465
+ }
8466
+ /** Call after the layers were registered for a new model. */
8467
+ setChannels(available, manager) {
8468
+ this.restoreOverrides(manager);
8469
+ this.channel = CHANNELS.find((name) => available.has(name)) ?? null;
8470
+ }
8471
+ set(weight, options) {
8472
+ this.weight = Math.max(0, Math.min(1, weight));
8473
+ if (options?.immediate && this.weight === 0) {
8474
+ this.weights.clearLayerTargets(LISTENING_FACE_LAYER, { immediate: true });
8475
+ }
8476
+ }
8477
+ /** Before the weight mixer updates: an active emotion takes the face. */
8478
+ beforeUpdate(emotionActive) {
8479
+ if (!this.channel) {
8480
+ return;
8481
+ }
8482
+ const target = emotionActive ? 0 : this.weight;
8483
+ if (target === 0 && this.weights.getLayerValue(LISTENING_FACE_LAYER, this.channel) < REST) {
8484
+ this.weights.clearLayerTargets(LISTENING_FACE_LAYER, { immediate: true });
8485
+ return;
8486
+ }
8487
+ this.weights.setTarget(LISTENING_FACE_LAYER, this.channel, target);
8488
+ }
8489
+ /** After the values went to the expression manager, before `vrm.update()`. */
8490
+ afterUpdate(manager, otherLayerValue) {
8491
+ const channel = this.channel;
8492
+ const owned = channel !== null && this.weights.getLayerValue(LISTENING_FACE_LAYER, channel) > 0 && otherLayerValue(channel) === 0;
8493
+ if (!owned) {
8494
+ this.restoreOverrides(manager);
8495
+ return;
8496
+ }
8497
+ const expression = manager.getExpression(channel);
8498
+ if (!expression || this.saved.size > 0) {
8499
+ return;
8500
+ }
8501
+ ;
8502
+ ["overrideBlink", "overrideLookAt"].forEach((key) => {
8503
+ if (expression[key] === "block") {
8504
+ this.saved.set(key, expression[key]);
8505
+ expression[key] = "blend";
8506
+ }
8507
+ });
8508
+ }
8509
+ /** Gives the preset its own overrides back. */
8510
+ restoreOverrides(manager) {
8511
+ const expression = this.channel !== null ? manager?.getExpression(this.channel) : null;
8512
+ if (expression) {
8513
+ this.saved.forEach((value, key) => {
8514
+ expression[key] = value;
8515
+ });
8516
+ }
8517
+ this.saved.clear();
8518
+ }
8519
+ };
8520
+
7446
8521
  // ../../clients/web-client/src/character/performance-weight-mixer.ts
7447
8522
  var PRECISION = 1e3;
7448
8523
  var DEFAULT_RESPONSE_RATE = 12;
@@ -7521,6 +8596,18 @@ var PerformanceWeightMixer = class {
7521
8596
  }
7522
8597
  layer.channels.set(channel, { value: 0, target });
7523
8598
  }
8599
+ restoreChannelState(layerId, channel, value, target) {
8600
+ const layer = this.getLayer(layerId);
8601
+ const state = layer.channels.get(channel);
8602
+ const nextValue = clamp012(value);
8603
+ const nextTarget = clamp012(target);
8604
+ if (state) {
8605
+ state.value = nextValue;
8606
+ state.target = nextTarget;
8607
+ return;
8608
+ }
8609
+ layer.channels.set(channel, { value: nextValue, target: nextTarget });
8610
+ }
7524
8611
  setExclusiveTargets(layerId, channels, selected, weight) {
7525
8612
  channels.forEach((channel) => {
7526
8613
  this.setTarget(layerId, channel, channel === selected ? weight : 0);
@@ -7679,17 +8766,34 @@ var FacialPerformanceMixer = class {
7679
8766
  restoreMouthAfterVisemeRelease = false;
7680
8767
  idleFace = null;
7681
8768
  gazeTarget = null;
7682
- setVrm(vrm) {
7683
- this.restoreExpressionMouthOwnership();
8769
+ listeningFace = new ListeningFace(this.weightMixer);
8770
+ setVrm(vrm, options) {
8771
+ const preserveState = options?.preserveState === true && vrm !== null;
8772
+ const savedMouthMode = preserveState ? this.activeMouthMode : "expression";
8773
+ const savedRestoreAfterRelease = preserveState ? this.restoreMouthAfterVisemeRelease : false;
8774
+ const savedSnapshot = preserveState ? this.weightMixer.getSnapshot() : null;
8775
+ this.vrm?.expressionManager?.resetValues?.();
8776
+ this.listeningFace.restoreOverrides(this.vrm?.expressionManager);
8777
+ if (preserveState) {
8778
+ this.restoreMouthOverrides();
8779
+ } else {
8780
+ this.restoreExpressionMouthOwnership();
8781
+ this.clearCueTimeouts();
8782
+ }
7684
8783
  this.vrm = vrm;
7685
8784
  this.idleFace = vrm && detectTemplateFace(vrm) ? new IdleFace(vrm, this.options.idleRandom) : null;
7686
8785
  this.inspector.setVrm(vrm);
7687
- this.clearCueTimeouts();
7688
8786
  this.availableChannels = [];
7689
8787
  this.availableChannelSet.clear();
7690
8788
  this.channelsByLayer.clear();
7691
8789
  this.configureLayers();
7692
8790
  if (!vrm?.expressionManager) {
8791
+ this.listeningFace.setChannels(this.availableChannelSet, null);
8792
+ if (preserveState) {
8793
+ this.activeMouthMode = "expression";
8794
+ this.restoreMouthAfterVisemeRelease = false;
8795
+ this.clearCueTimeouts();
8796
+ }
7693
8797
  return;
7694
8798
  }
7695
8799
  const manager = vrm.expressionManager;
@@ -7705,7 +8809,43 @@ var FacialPerformanceMixer = class {
7705
8809
  this.weightMixer.setTarget(layerId, name, 0);
7706
8810
  });
7707
8811
  this.availableChannels = names;
8812
+ this.listeningFace.setChannels(this.availableChannelSet, manager);
7708
8813
  manager.resetValues();
8814
+ if (!preserveState || !savedSnapshot) {
8815
+ return;
8816
+ }
8817
+ this.cueTimeouts.forEach((timeoutId, channel) => {
8818
+ if (!this.availableChannelSet.has(channel)) {
8819
+ clearTimeout(timeoutId);
8820
+ this.cueTimeouts.delete(channel);
8821
+ }
8822
+ });
8823
+ Object.entries(savedSnapshot).forEach(([layerId, layerSnapshot]) => {
8824
+ Object.entries(layerSnapshot.channels).forEach(
8825
+ ([channel, channelState]) => {
8826
+ if (this.availableChannelSet.has(channel)) {
8827
+ this.weightMixer.restoreChannelState(
8828
+ layerId,
8829
+ channel,
8830
+ channelState.value,
8831
+ channelState.target
8832
+ );
8833
+ }
8834
+ }
8835
+ );
8836
+ });
8837
+ this.activeMouthMode = savedMouthMode;
8838
+ this.restoreMouthAfterVisemeRelease = savedRestoreAfterRelease;
8839
+ if (savedMouthMode === "viseme") {
8840
+ this.getLayerChannels(FacialPerformanceLayer.Emotion).forEach((name) => {
8841
+ const expression = manager.getExpression(name);
8842
+ if (expression) {
8843
+ this.originalMouthOverrides.set(name, expression.overrideMouth);
8844
+ expression.overrideMouth = "none";
8845
+ }
8846
+ });
8847
+ }
8848
+ this.update(0);
7709
8849
  }
7710
8850
  getChannelNames() {
7711
8851
  return this.availableChannels.slice();
@@ -7807,6 +8947,19 @@ var FacialPerformanceMixer = class {
7807
8947
  clearOverlay() {
7808
8948
  this.weightMixer.clearLayerTargets(FacialPerformanceLayer.Overlay);
7809
8949
  }
8950
+ /**
8951
+ * The attentive face of the listening reflexes (`relaxed`, else `happy`)
8952
+ * at `weight`. It eases in and out on its own layer, gives way to any
8953
+ * emotion and leaves the mouth layers alone. `immediate` with weight 0
8954
+ * removes it in this frame.
8955
+ */
8956
+ setListeningFace(weight, options) {
8957
+ this.listeningFace.set(weight, options);
8958
+ }
8959
+ /** The preset of the attentive face, or null when the model has none. */
8960
+ get listeningFaceChannel() {
8961
+ return this.listeningFace.channelName;
8962
+ }
7810
8963
  releaseVisemeMouthMode() {
7811
8964
  if (this.activeMouthMode !== "viseme") {
7812
8965
  return;
@@ -7886,16 +9039,24 @@ var FacialPerformanceMixer = class {
7886
9039
  }
7887
9040
  reset() {
7888
9041
  this.weightMixer.resetTargets();
9042
+ this.listeningFace.set(0);
7889
9043
  }
7890
9044
  update(delta) {
7891
9045
  if (!this.vrm?.expressionManager) {
7892
9046
  return;
7893
9047
  }
7894
9048
  const manager = this.vrm.expressionManager;
9049
+ this.listeningFace.beforeUpdate(
9050
+ this.weightMixer.isLayerActive(FacialPerformanceLayer.Emotion)
9051
+ );
7895
9052
  const values = this.weightMixer.update(delta);
7896
9053
  values.forEach((value, name) => {
7897
9054
  manager.setValue(name, value);
7898
9055
  });
9056
+ this.listeningFace.afterUpdate(
9057
+ manager,
9058
+ (channel) => this.weightMixer.getLayerValue(FacialPerformanceLayer.Emotion, channel) + this.weightMixer.getLayerValue(FacialPerformanceLayer.Overlay, channel)
9059
+ );
7899
9060
  this.updateIdleFace(delta, values);
7900
9061
  if (this.restoreMouthAfterVisemeRelease && this.areVisemesReleased()) {
7901
9062
  this.restoreExpressionMouthOwnership();
@@ -7938,6 +9099,12 @@ var FacialPerformanceMixer = class {
7938
9099
  priority: 400,
7939
9100
  blendMode: "override"
7940
9101
  });
9102
+ this.weightMixer.registerLayer({
9103
+ id: LISTENING_FACE_LAYER,
9104
+ priority: 450,
9105
+ blendMode: "maximum",
9106
+ responseRate: 8
9107
+ });
7941
9108
  this.weightMixer.registerLayer({
7942
9109
  id: FacialPerformanceLayer.Overlay,
7943
9110
  priority: 500,
@@ -8118,8 +9285,28 @@ var CharacterSceneRig = class {
8118
9285
  get vrm() {
8119
9286
  return this.currentVrm;
8120
9287
  }
8121
- setVrm(vrm) {
9288
+ setVrm(vrm, fallbackParent) {
9289
+ const previousVrm = this.currentVrm;
9290
+ const previousScene = previousVrm?.scene ?? null;
9291
+ const previousParent = previousScene?.parent ?? null;
8122
9292
  this.setVrmLookAtTarget(null);
9293
+ if (previousScene && previousParent && previousVrm !== vrm) {
9294
+ previousParent.remove(previousScene);
9295
+ }
9296
+ if (previousScene && vrm?.scene && previousScene !== vrm.scene) {
9297
+ if (previousScene.position && vrm.scene.position?.copy) {
9298
+ vrm.scene.position.copy(previousScene.position);
9299
+ }
9300
+ if (previousScene.rotation && vrm.scene.rotation?.copy) {
9301
+ vrm.scene.rotation.copy(previousScene.rotation);
9302
+ }
9303
+ if (previousScene.scale && vrm.scene.scale?.copy) {
9304
+ vrm.scene.scale.copy(previousScene.scale);
9305
+ }
9306
+ if (typeof previousScene.visible === "boolean") {
9307
+ vrm.scene.visible = previousScene.visible;
9308
+ }
9309
+ }
8123
9310
  this.currentVrm = vrm;
8124
9311
  this.refreshMeshes();
8125
9312
  if (this.shadowOptions) {
@@ -8128,6 +9315,10 @@ var CharacterSceneRig = class {
8128
9315
  if (this.transformOptions) {
8129
9316
  this.applyTransform(this.transformOptions);
8130
9317
  }
9318
+ const targetParent = previousParent ?? fallbackParent ?? null;
9319
+ if (vrm?.scene && targetParent && vrm.scene.parent !== targetParent) {
9320
+ targetParent.add(vrm.scene);
9321
+ }
8131
9322
  }
8132
9323
  /**
8133
9324
  * The `Object3D` of a `lookAt()` argument: an `Object3D` as it is, a point
@@ -8169,6 +9360,7 @@ var CharacterSceneRig = class {
8169
9360
  lookAt.target = target;
8170
9361
  } else if (this.vrmLookAtTarget && lookAt.target === this.vrmLookAtTarget) {
8171
9362
  lookAt.target = void 0;
9363
+ lookAt.reset();
8172
9364
  }
8173
9365
  }
8174
9366
  this.vrmLookAtTarget = lookAt ? target : null;
@@ -8310,6 +9502,22 @@ var CharacterSceneRig = class {
8310
9502
  disposeVrm(vrm);
8311
9503
  }
8312
9504
  }
9505
+ disposeAllVrms(vrms) {
9506
+ const unique = new Set(vrms);
9507
+ if (this.currentVrm) {
9508
+ unique.add(this.currentVrm);
9509
+ }
9510
+ this.detach();
9511
+ this.setVrm(null);
9512
+ for (const vrm of unique) {
9513
+ if (vrm.scene?.parent) {
9514
+ vrm.scene.parent.remove(vrm.scene);
9515
+ }
9516
+ if (vrm.scene && typeof vrm.scene.traverse === "function") {
9517
+ disposeVrm(vrm);
9518
+ }
9519
+ }
9520
+ }
8313
9521
  applyShadows(options) {
8314
9522
  this.cachedMeshes.forEach((mesh) => {
8315
9523
  mesh.castShadow = options.cast;
@@ -8337,107 +9545,373 @@ var CharacterSceneRig = class {
8337
9545
  }
8338
9546
  };
8339
9547
 
8340
- // ../../clients/web-client/src/character/live-motion.ts
8341
- import { Quaternion as Quaternion3, Vector3 as Vector37 } from "three";
8342
- var PARENTS = {
8343
- spine: "hips",
8344
- chest: "spine",
8345
- upperChest: "chest",
8346
- neck: "upperChest",
8347
- head: "neck",
8348
- leftShoulder: "upperChest",
8349
- leftUpperArm: "leftShoulder",
8350
- leftLowerArm: "leftUpperArm",
8351
- leftHand: "leftLowerArm",
8352
- rightShoulder: "upperChest",
8353
- rightUpperArm: "rightShoulder",
8354
- rightLowerArm: "rightUpperArm",
8355
- rightHand: "rightLowerArm",
8356
- leftUpperLeg: "hips",
8357
- leftLowerLeg: "leftUpperLeg",
8358
- leftFoot: "leftLowerLeg",
8359
- leftToes: "leftFoot",
8360
- rightUpperLeg: "hips",
8361
- rightLowerLeg: "rightUpperLeg",
8362
- rightFoot: "rightLowerLeg",
8363
- rightToes: "rightFoot"
8364
- };
8365
- var MAX_TRACKS = 4;
8366
- var approach = function(value, target, delta, seconds) {
8367
- if (seconds <= 0) {
8368
- return target;
9548
+ // ../../clients/web-client/src/character/character-scene-access.ts
9549
+ var CharacterSceneAccess = class {
9550
+ sceneRig = new CharacterSceneRig();
9551
+ defaultParent = null;
9552
+ /** True once the character VRM model is loaded. */
9553
+ get ready() {
9554
+ return this.sceneRig.vrm !== null;
8369
9555
  }
8370
- const step = delta / seconds;
8371
- return value < target ? Math.min(target, value + step) : Math.max(target, value - step);
8372
- };
8373
- var LiveMotionPlayer = class {
8374
- vrm = null;
8375
- tracks = /* @__PURE__ */ new Map();
8376
- rootMode;
8377
- blendIn;
8378
- blendOut;
8379
- weight = 0;
8380
- /** The last applied pose, to blend out of it after the frames end. */
8381
- lastPose = null;
8382
- scratch = {
8383
- from: new Quaternion3(),
8384
- to: new Quaternion3(),
8385
- carry: /* @__PURE__ */ new Map(),
8386
- position: new Vector37()
8387
- };
8388
- constructor(options = {}) {
8389
- this.rootMode = options.rootMode ?? "anchored";
8390
- this.blendIn = options.blendInSeconds ?? 0.3;
8391
- this.blendOut = options.blendOutSeconds ?? 0.5;
9556
+ /** Root Three.js `Group` (`vrm.scene`) to add to a `THREE.Scene`. */
9557
+ get scene() {
9558
+ return this.sceneRig.scene;
8392
9559
  }
8393
- setVrm(vrm) {
8394
- this.vrm = vrm;
8395
- this.weight = 0;
8396
- this.lastPose = null;
9560
+ /** Alias for `scene` for Three.js `Object3D` workflows. */
9561
+ get object3d() {
9562
+ return this.sceneRig.object3d;
8397
9563
  }
8398
- /** Adds the frames of a `motion_frames` event; invalid events are dropped. */
8399
- push(received) {
8400
- const frames = received.frames;
8401
- const event = frames instanceof ArrayBuffer ? { ...received, frames: new Uint8Array(frames) } : received;
8402
- if (!isValidMotionFramesEvent(event)) {
8403
- return false;
8404
- }
8405
- const values = readMotionFrames(
8406
- event.frames,
8407
- event.frameCount,
8408
- event.bones.length
8409
- );
8410
- const track = this.track(event.stagingId, {
8411
- fps: event.fps,
8412
- bones: event.bones,
8413
- restHipsHeight: event.restHipsHeight
8414
- });
8415
- for (let frame = 0; frame < event.frameCount; frame += 1) {
8416
- const index = event.startFrame + frame;
8417
- const existing = track.frames.get(index);
8418
- if (existing && existing.revision > event.revision) {
8419
- continue;
8420
- }
8421
- track.frames.set(index, {
8422
- revision: event.revision,
8423
- values: values.subarray(frame * track.width, (frame + 1) * track.width)
8424
- });
8425
- }
8426
- return true;
9564
+ /** Alias for `scene`. */
9565
+ get root() {
9566
+ return this.sceneRig.root;
8427
9567
  }
8428
- /** Loads the stored frames of a staging (a replay). */
8429
- async load(stagingId, track, fetchBytes = async (url) => new Uint8Array(await (await fetch(url)).arrayBuffer())) {
8430
- if (this.tracks.has(stagingId)) {
8431
- return;
8432
- }
8433
- const data = track.frames.type === "bytes" ? track.frames.data : await fetchBytes(track.frames.url);
8434
- const values = readMotionFrames(data, track.frameCount, track.bones.length);
8435
- const stored = this.track(stagingId, track);
8436
- for (let frame = 0; frame < track.frameCount; frame += 1) {
8437
- stored.frames.set(frame, {
8438
- revision: Number.MAX_SAFE_INTEGER,
8439
- values: values.subarray(
8440
- frame * stored.width,
9568
+ /** Primary character mesh (`SkinnedMesh` preferred, otherwise first `Mesh`). */
9569
+ get mesh() {
9570
+ return this.sceneRig.mesh;
9571
+ }
9572
+ /** All Three.js `Mesh` instances in the character hierarchy. */
9573
+ get meshes() {
9574
+ return this.sceneRig.meshes;
9575
+ }
9576
+ /** All Three.js `SkinnedMesh` instances in the character hierarchy. */
9577
+ get skinnedMeshes() {
9578
+ return this.sceneRig.skinnedMeshes;
9579
+ }
9580
+ /** Deduplicated Three.js `Material` instances across all character meshes. */
9581
+ get materials() {
9582
+ return this.sceneRig.materials;
9583
+ }
9584
+ /** Deduplicated Three.js `Skeleton` instances across all character skinned meshes. */
9585
+ get skeletons() {
9586
+ return this.sceneRig.skeletons;
9587
+ }
9588
+ /** Three-VRM humanoid rig when loaded. */
9589
+ get humanoid() {
9590
+ return this.sceneRig.humanoid;
9591
+ }
9592
+ /** Look up a raw humanoid bone `Object3D` by VRM bone name (for example `'head'` or `'rightHand'`). */
9593
+ getBone(name) {
9594
+ return this.sceneRig.getBone(name);
9595
+ }
9596
+ /** Look up a normalized humanoid bone `Object3D` by VRM bone name. */
9597
+ getNormalizedBone(name) {
9598
+ return this.sceneRig.getNormalizedBone(name);
9599
+ }
9600
+ /** Attach the character root `Group` to a Three.js `Scene` or parent `Object3D`. */
9601
+ attachTo(parent) {
9602
+ this.defaultParent = parent;
9603
+ return this.sceneRig.attachTo(parent);
9604
+ }
9605
+ /** Detach the character root `Group` from its current parent `Object3D`. */
9606
+ detach() {
9607
+ this.defaultParent = null;
9608
+ this.sceneRig.detach();
9609
+ }
9610
+ /** Configure `castShadow` and `receiveShadow` on all character meshes. */
9611
+ setShadows(options) {
9612
+ this.sceneRig.setShadows(options);
9613
+ }
9614
+ /** Configure `position`, `rotation`, and `scale` on the character root `Group`. */
9615
+ setTransform(options) {
9616
+ this.sceneRig.setTransform(options);
9617
+ }
9618
+ /** Re-index cached meshes, materials, and skeletons after custom scene graph edits. */
9619
+ refreshMeshes() {
9620
+ this.sceneRig.refreshMeshes();
9621
+ }
9622
+ };
9623
+
9624
+ // ../../clients/web-client/src/character/listening-reflex.ts
9625
+ import { Object3D as Object3D2, Vector3 as Vector38 } from "three";
9626
+
9627
+ // ../../clients/web-client/src/character/head-nod.ts
9628
+ import { Quaternion as Quaternion3, Vector3 as Vector37 } from "three";
9629
+ var SHARES = [
9630
+ ["neck", 0.4],
9631
+ ["head", 0.6]
9632
+ ];
9633
+ var DAMPING = 0.6;
9634
+ var FREQUENCY_TIMES_LENGTH = 1.4;
9635
+ var PULSE_PART = 0.3;
9636
+ var MAX_STEP_SECONDS = 1 / 240;
9637
+ var REST_DEGREES = 0.01;
9638
+ var X_AXIS = new Vector37(1, 0, 0);
9639
+ var HeadNod = class {
9640
+ pitch = 0;
9641
+ speed = 0;
9642
+ elapsed = 0;
9643
+ pulseSeconds = 0;
9644
+ amplitude = 0;
9645
+ omega = 0;
9646
+ applied = [];
9647
+ scratch = new Quaternion3();
9648
+ /** The current pitch in degrees; positive turns the face down. */
9649
+ get pitchDegrees() {
9650
+ return this.pitch;
9651
+ }
9652
+ get active() {
9653
+ return this.elapsed < this.pulseSeconds || Math.abs(this.pitch) > REST_DEGREES || Math.abs(this.speed) > REST_DEGREES;
9654
+ }
9655
+ start(nod) {
9656
+ const seconds = Math.max(nod.durationMs, 1) / 1e3;
9657
+ this.amplitude = nod.amplitudeDegrees;
9658
+ this.omega = 2 * Math.PI * FREQUENCY_TIMES_LENGTH / seconds;
9659
+ this.pulseSeconds = seconds * PULSE_PART;
9660
+ this.elapsed = 0;
9661
+ }
9662
+ /** Stops the nod at once; the next `apply` removes its pitch. */
9663
+ cancel() {
9664
+ this.pitch = 0;
9665
+ this.speed = 0;
9666
+ this.elapsed = 0;
9667
+ this.pulseSeconds = 0;
9668
+ }
9669
+ update(delta) {
9670
+ if (!(delta > 0) || !this.active) {
9671
+ if (!this.active) {
9672
+ this.pitch = 0;
9673
+ this.speed = 0;
9674
+ }
9675
+ return this.pitch;
9676
+ }
9677
+ let left = Math.min(delta, 0.25);
9678
+ while (left > 0) {
9679
+ const step = Math.min(left, MAX_STEP_SECONDS);
9680
+ const target = this.elapsed < this.pulseSeconds ? this.amplitude : 0;
9681
+ const omega = this.omega;
9682
+ const acceleration = omega * omega * (target - this.pitch) - 2 * DAMPING * omega * this.speed;
9683
+ this.speed += acceleration * step;
9684
+ this.pitch += this.speed * step;
9685
+ this.elapsed += step;
9686
+ left -= step;
9687
+ }
9688
+ if (!this.active) {
9689
+ this.pitch = 0;
9690
+ this.speed = 0;
9691
+ }
9692
+ return this.pitch;
9693
+ }
9694
+ /** Writes the current pitch onto the normalized neck and head of `vrm`. */
9695
+ apply(vrm) {
9696
+ this.applied.forEach(({ node, base, result }) => {
9697
+ if (node.quaternion.equals(result)) {
9698
+ node.quaternion.copy(base);
9699
+ }
9700
+ });
9701
+ this.applied = [];
9702
+ const humanoid = vrm?.humanoid;
9703
+ if (!humanoid || this.pitch === 0) {
9704
+ return;
9705
+ }
9706
+ const flip = vrm.meta.metaVersion === "0" && !isNpcifyNormalizedVrm(vrm) ? -1 : 1;
9707
+ const radians = this.pitch * Math.PI / 180 * flip;
9708
+ let carried = 0;
9709
+ SHARES.forEach(([bone, share]) => {
9710
+ const node = humanoid.getNormalizedBoneNode(bone);
9711
+ if (!node) {
9712
+ carried += share;
9713
+ return;
9714
+ }
9715
+ const angle = radians * (share + carried);
9716
+ carried = 0;
9717
+ const base = node.quaternion.clone();
9718
+ node.quaternion.multiply(this.scratch.setFromAxisAngle(X_AXIS, angle));
9719
+ this.applied.push({ node, base, result: node.quaternion.clone() });
9720
+ });
9721
+ }
9722
+ };
9723
+
9724
+ // ../../clients/web-client/src/character/listening-reflex.ts
9725
+ var LISTENING_FACE_WEIGHT = 0.2;
9726
+ var FRONT_DISTANCE = 1.5;
9727
+ var ListeningReflex = class {
9728
+ constructor(host, activity) {
9729
+ this.host = host;
9730
+ this.activity = activity;
9731
+ activity.onNod((nod) => {
9732
+ this.pendingNod = nod;
9733
+ });
9734
+ }
9735
+ nod = new HeadNod();
9736
+ pendingNod = null;
9737
+ releaseGaze = null;
9738
+ front = new Object3D2();
9739
+ origin = new Vector38();
9740
+ forward = new Vector38();
9741
+ yielded = false;
9742
+ /** Call after the animation mixer and live motion, before the facial mixer and `vrm.update()`. */
9743
+ update(delta, frame) {
9744
+ const vrm = this.host.vrm();
9745
+ if (frame.stagingActive) {
9746
+ if (!this.yielded) {
9747
+ this.yielded = true;
9748
+ this.host.facialMixer.setListeningFace(0, { immediate: true });
9749
+ this.nod.cancel();
9750
+ this.endGaze();
9751
+ }
9752
+ this.pendingNod = null;
9753
+ this.nod.apply(vrm);
9754
+ return;
9755
+ }
9756
+ this.yielded = false;
9757
+ const engaged = this.activity.state !== "idle" && vrm !== null;
9758
+ this.host.facialMixer.setListeningFace(engaged ? LISTENING_FACE_WEIGHT : 0);
9759
+ if (engaged) {
9760
+ this.beginGaze(vrm);
9761
+ } else {
9762
+ this.endGaze();
9763
+ }
9764
+ const nod = this.pendingNod;
9765
+ this.pendingNod = null;
9766
+ if (frame.liveMotionWeight > 0) {
9767
+ this.nod.cancel();
9768
+ } else if (nod && this.activity.state === "speaking" && vrm) {
9769
+ this.nod.start(nod);
9770
+ }
9771
+ this.nod.update(delta);
9772
+ this.nod.apply(vrm);
9773
+ }
9774
+ /** A new model or the end of the conversation: no gaze, face or nod remains. */
9775
+ reset() {
9776
+ this.pendingNod = null;
9777
+ this.nod.cancel();
9778
+ this.nod.apply(null);
9779
+ this.endGaze();
9780
+ this.host.facialMixer.setListeningFace(0, { immediate: true });
9781
+ }
9782
+ beginGaze(vrm) {
9783
+ if (!this.releaseGaze) {
9784
+ if (this.host.lookAtActive()) {
9785
+ return;
9786
+ }
9787
+ const target = this.host.gazeTarget() ?? this.front;
9788
+ this.releaseGaze = this.host.lookAtTemporarily(target);
9789
+ }
9790
+ this.placeFront(vrm);
9791
+ }
9792
+ endGaze() {
9793
+ const release = this.releaseGaze;
9794
+ this.releaseGaze = null;
9795
+ release?.();
9796
+ }
9797
+ /** The point in front of the head, along the character's forward axis. */
9798
+ placeFront(vrm) {
9799
+ const head = vrm.humanoid?.getNormalizedBoneNode("head");
9800
+ if (!head) {
9801
+ return;
9802
+ }
9803
+ const flip = vrm.meta.metaVersion === "0" && !isNpcifyNormalizedVrm(vrm);
9804
+ const root = vrm.scene;
9805
+ const origin = root.getWorldPosition(this.origin);
9806
+ const forward = root.localToWorld(
9807
+ this.forward.set(0, 0, flip ? -FRONT_DISTANCE : FRONT_DISTANCE)
9808
+ ).sub(origin);
9809
+ head.getWorldPosition(this.front.position).add(forward);
9810
+ this.front.updateMatrixWorld(true);
9811
+ }
9812
+ };
9813
+
9814
+ // ../../clients/web-client/src/character/live-motion.ts
9815
+ import { Quaternion as Quaternion4, Vector3 as Vector39 } from "three";
9816
+ var PARENTS = {
9817
+ spine: "hips",
9818
+ chest: "spine",
9819
+ upperChest: "chest",
9820
+ neck: "upperChest",
9821
+ head: "neck",
9822
+ leftShoulder: "upperChest",
9823
+ leftUpperArm: "leftShoulder",
9824
+ leftLowerArm: "leftUpperArm",
9825
+ leftHand: "leftLowerArm",
9826
+ rightShoulder: "upperChest",
9827
+ rightUpperArm: "rightShoulder",
9828
+ rightLowerArm: "rightUpperArm",
9829
+ rightHand: "rightLowerArm",
9830
+ leftUpperLeg: "hips",
9831
+ leftLowerLeg: "leftUpperLeg",
9832
+ leftFoot: "leftLowerLeg",
9833
+ leftToes: "leftFoot",
9834
+ rightUpperLeg: "hips",
9835
+ rightLowerLeg: "rightUpperLeg",
9836
+ rightFoot: "rightLowerLeg",
9837
+ rightToes: "rightFoot"
9838
+ };
9839
+ var MAX_TRACKS = 4;
9840
+ var approach = function(value, target, delta, seconds) {
9841
+ if (seconds <= 0) {
9842
+ return target;
9843
+ }
9844
+ const step = delta / seconds;
9845
+ return value < target ? Math.min(target, value + step) : Math.max(target, value - step);
9846
+ };
9847
+ var LiveMotionPlayer = class {
9848
+ vrm = null;
9849
+ tracks = /* @__PURE__ */ new Map();
9850
+ rootMode;
9851
+ blendIn;
9852
+ blendOut;
9853
+ weight = 0;
9854
+ /** The last applied pose, to blend out of it after the frames end. */
9855
+ lastPose = null;
9856
+ scratch = {
9857
+ from: new Quaternion4(),
9858
+ to: new Quaternion4(),
9859
+ carry: /* @__PURE__ */ new Map(),
9860
+ position: new Vector39()
9861
+ };
9862
+ constructor(options = {}) {
9863
+ this.rootMode = options.rootMode ?? "anchored";
9864
+ this.blendIn = options.blendInSeconds ?? 0.3;
9865
+ this.blendOut = options.blendOutSeconds ?? 0.5;
9866
+ }
9867
+ setVrm(vrm) {
9868
+ this.vrm = vrm;
9869
+ this.weight = 0;
9870
+ this.lastPose = null;
9871
+ }
9872
+ /** Adds the frames of a `motion_frames` event; invalid events are dropped. */
9873
+ push(received) {
9874
+ const frames = received.frames;
9875
+ const event = frames instanceof ArrayBuffer ? { ...received, frames: new Uint8Array(frames) } : received;
9876
+ if (!isValidMotionFramesEvent(event)) {
9877
+ return false;
9878
+ }
9879
+ const values = readMotionFrames(
9880
+ event.frames,
9881
+ event.frameCount,
9882
+ event.bones.length
9883
+ );
9884
+ const track = this.track(event.stagingId, {
9885
+ fps: event.fps,
9886
+ bones: event.bones,
9887
+ restHipsHeight: event.restHipsHeight
9888
+ });
9889
+ for (let frame = 0; frame < event.frameCount; frame += 1) {
9890
+ const index = event.startFrame + frame;
9891
+ const existing = track.frames.get(index);
9892
+ if (existing && existing.revision > event.revision) {
9893
+ continue;
9894
+ }
9895
+ track.frames.set(index, {
9896
+ revision: event.revision,
9897
+ values: values.subarray(frame * track.width, (frame + 1) * track.width)
9898
+ });
9899
+ }
9900
+ return true;
9901
+ }
9902
+ /** Loads the stored frames of a staging (a replay). */
9903
+ async load(stagingId, track, fetchBytes = async (url) => new Uint8Array(await (await fetch(url)).arrayBuffer())) {
9904
+ if (this.tracks.has(stagingId)) {
9905
+ return;
9906
+ }
9907
+ const data = track.frames.type === "bytes" ? track.frames.data : await fetchBytes(track.frames.url);
9908
+ const values = readMotionFrames(data, track.frameCount, track.bones.length);
9909
+ const stored = this.track(stagingId, track);
9910
+ for (let frame = 0; frame < track.frameCount; frame += 1) {
9911
+ stored.frames.set(frame, {
9912
+ revision: Number.MAX_SAFE_INTEGER,
9913
+ values: values.subarray(
9914
+ frame * stored.width,
8441
9915
  (frame + 1) * stored.width
8442
9916
  )
8443
9917
  });
@@ -8545,7 +10019,7 @@ var LiveMotionPlayer = class {
8545
10019
  carry.clear();
8546
10020
  track.bones.forEach((bone, slot) => {
8547
10021
  const offset = 3 + slot * 4;
8548
- const rotation = new Quaternion3(
10022
+ const rotation = new Quaternion4(
8549
10023
  values[offset] ?? 0,
8550
10024
  values[offset + 1] ?? 0,
8551
10025
  values[offset + 2] ?? 0,
@@ -8592,6 +10066,166 @@ var LiveMotionPlayer = class {
8592
10066
  }
8593
10067
  };
8594
10068
 
10069
+ // ../../clients/web-client/src/voice/speech-nod-trigger.ts
10070
+ var MIN_SPEECH_LEVEL = 0.015;
10071
+ var SPEECH_TO_NOISE = 3;
10072
+ var NOD_PAUSE_MS = 250;
10073
+ var NOD_MIN_SPEECH_MS = 1200;
10074
+ var NOD_MIN_INTERVAL_MS = 1800;
10075
+ var NOD_INTERVAL_JITTER_MS = 1400;
10076
+ var NOD_SKIP_CHANCE = 0.2;
10077
+ var NOD_MIN_DEGREES = 4;
10078
+ var NOD_MAX_DEGREES = 6;
10079
+ var NOD_MIN_DURATION_MS = 320;
10080
+ var NOD_MAX_DURATION_MS = 400;
10081
+ var SpeechNodTrigger = class {
10082
+ constructor(random = Math.random) {
10083
+ this.random = random;
10084
+ this.intervalMs = this.drawInterval();
10085
+ }
10086
+ floor = null;
10087
+ speechMs = 0;
10088
+ quietMs = 0;
10089
+ sinceNodMs = Number.POSITIVE_INFINITY;
10090
+ intervalMs;
10091
+ /** True while the last frame was loud enough to be speech. */
10092
+ get voiced() {
10093
+ return this.speechMs > 0 && this.quietMs === 0;
10094
+ }
10095
+ /** Reads one frame; returns a nod when one is due now. */
10096
+ push(level, frameMs = SPEECH_FRAME_MS) {
10097
+ if (!Number.isFinite(level) || !(frameMs > 0)) {
10098
+ return null;
10099
+ }
10100
+ this.sinceNodMs += frameMs;
10101
+ const threshold = Math.max(
10102
+ MIN_SPEECH_LEVEL,
10103
+ (this.floor ?? MIN_SPEECH_LEVEL / SPEECH_TO_NOISE) * SPEECH_TO_NOISE
10104
+ );
10105
+ if (level >= threshold) {
10106
+ this.speechMs += this.quietMs + frameMs;
10107
+ this.quietMs = 0;
10108
+ return null;
10109
+ }
10110
+ this.floor = this.floor === null ? level : this.floor * 0.9 + level * 0.1;
10111
+ if (this.speechMs === 0) {
10112
+ return null;
10113
+ }
10114
+ this.quietMs += frameMs;
10115
+ if (this.quietMs < NOD_PAUSE_MS) {
10116
+ return null;
10117
+ }
10118
+ const run = this.speechMs;
10119
+ this.speechMs = 0;
10120
+ this.quietMs = 0;
10121
+ if (run < NOD_MIN_SPEECH_MS || this.sinceNodMs < this.intervalMs) {
10122
+ return null;
10123
+ }
10124
+ if (this.random() < NOD_SKIP_CHANCE) {
10125
+ return null;
10126
+ }
10127
+ this.sinceNodMs = 0;
10128
+ this.intervalMs = this.drawInterval();
10129
+ return {
10130
+ amplitudeDegrees: NOD_MIN_DEGREES + (NOD_MAX_DEGREES - NOD_MIN_DEGREES) * this.random(),
10131
+ durationMs: NOD_MIN_DURATION_MS + (NOD_MAX_DURATION_MS - NOD_MIN_DURATION_MS) * this.random()
10132
+ };
10133
+ }
10134
+ /** A new listening session: the speech run starts again; the noise floor stays. */
10135
+ reset() {
10136
+ this.speechMs = 0;
10137
+ this.quietMs = 0;
10138
+ this.sinceNodMs = Number.POSITIVE_INFINITY;
10139
+ }
10140
+ drawInterval() {
10141
+ return NOD_MIN_INTERVAL_MS + NOD_INTERVAL_JITTER_MS * this.random();
10142
+ }
10143
+ };
10144
+
10145
+ // ../../clients/web-client/src/character/player-activity.ts
10146
+ var TYPING_IDLE_MS = 1500;
10147
+ var PlayerActivityTracker = class {
10148
+ listening = false;
10149
+ typing = false;
10150
+ typingTimer = null;
10151
+ current = "idle";
10152
+ trigger;
10153
+ changeListeners = /* @__PURE__ */ new Set();
10154
+ nodListeners = /* @__PURE__ */ new Set();
10155
+ constructor(options = {}) {
10156
+ this.trigger = new SpeechNodTrigger(options.random);
10157
+ }
10158
+ get state() {
10159
+ return this.current;
10160
+ }
10161
+ /** The listening session is (or is no longer) in state `listening`. */
10162
+ setListening(active) {
10163
+ if (active === this.listening) {
10164
+ return;
10165
+ }
10166
+ this.listening = active;
10167
+ this.trigger.reset();
10168
+ this.resolve();
10169
+ }
10170
+ /**
10171
+ * Input in a text field: `true` at each input event, `false` when the
10172
+ * player sends or leaves the field. Without input, typing ends after 1.5 s.
10173
+ */
10174
+ notifyTyping(active) {
10175
+ this.clearTypingTimer();
10176
+ this.typing = active;
10177
+ if (active) {
10178
+ this.typingTimer = setTimeout(() => {
10179
+ this.typingTimer = null;
10180
+ this.typing = false;
10181
+ this.resolve();
10182
+ }, TYPING_IDLE_MS);
10183
+ }
10184
+ this.resolve();
10185
+ }
10186
+ /** One microphone frame of the listening session; returns a nod when one is due. */
10187
+ pushVoiceLevel(level, frameMs) {
10188
+ if (!this.listening) {
10189
+ return null;
10190
+ }
10191
+ const nod = this.trigger.push(level, frameMs);
10192
+ if (nod) {
10193
+ this.nodListeners.forEach((listener) => listener(nod));
10194
+ }
10195
+ return nod;
10196
+ }
10197
+ onChange(listener) {
10198
+ this.changeListeners.add(listener);
10199
+ return () => this.changeListeners.delete(listener);
10200
+ }
10201
+ onNod(listener) {
10202
+ this.nodListeners.add(listener);
10203
+ return () => this.nodListeners.delete(listener);
10204
+ }
10205
+ /** Ends all activity, for example when the conversation ends. */
10206
+ reset() {
10207
+ this.clearTypingTimer();
10208
+ this.typing = false;
10209
+ this.listening = false;
10210
+ this.trigger.reset();
10211
+ this.resolve();
10212
+ }
10213
+ resolve() {
10214
+ const next = this.listening ? "speaking" : this.typing ? "typing" : "idle";
10215
+ if (next === this.current) {
10216
+ return;
10217
+ }
10218
+ this.current = next;
10219
+ this.changeListeners.forEach((listener) => listener(next));
10220
+ }
10221
+ clearTypingTimer() {
10222
+ if (this.typingTimer !== null) {
10223
+ clearTimeout(this.typingTimer);
10224
+ this.typingTimer = null;
10225
+ }
10226
+ }
10227
+ };
10228
+
8595
10229
  // ../../clients/web-client/src/character/incremental-staging.ts
8596
10230
  var dialogueKey = function(block) {
8597
10231
  return JSON.stringify(block, (key, value) => {
@@ -8867,6 +10501,9 @@ var StagingPlayer = class {
8867
10501
  activeExpressionMouthMode = null;
8868
10502
  activeAnimation = null;
8869
10503
  currentTrackTime = null;
10504
+ playbackElapsed = 0;
10505
+ appliedToolCallIds = /* @__PURE__ */ new Set();
10506
+ activeOptions;
8870
10507
  get paused() {
8871
10508
  return this.clock.paused;
8872
10509
  }
@@ -8920,6 +10557,9 @@ var StagingPlayer = class {
8920
10557
  this.incremental = incremental;
8921
10558
  timeline = incremental.timeline;
8922
10559
  this.onCursor = options?.onCursor;
10560
+ this.activeOptions = options;
10561
+ this.playbackElapsed = 0;
10562
+ this.appliedToolCallIds.clear();
8923
10563
  let timelineStarted = false;
8924
10564
  const beginTimeline = () => {
8925
10565
  if (timelineStarted) {
@@ -8929,8 +10569,16 @@ var StagingPlayer = class {
8929
10569
  options?.onTimelineStart?.(timeline);
8930
10570
  };
8931
10571
  try {
8932
- let cursor = 0;
8933
- for (let index = 0; ; index += 1) {
10572
+ await options?.onVariantCue?.(
10573
+ timeline.initialVariant ?? DEFAULT_VARIANT_KEY,
10574
+ null
10575
+ );
10576
+ if (controller.signal.aborted) {
10577
+ throw abortError2();
10578
+ }
10579
+ this.applyToolCalls(0, timeline);
10580
+ let cursor = 0;
10581
+ for (let index = 0; ; index += 1) {
8934
10582
  const block = await incremental.next(index, controller.signal, () => {
8935
10583
  this.clearPerformance();
8936
10584
  this.clearCursor();
@@ -8978,6 +10626,7 @@ var StagingPlayer = class {
8978
10626
  }
8979
10627
  await incremental.finishBlock(index, controller.signal);
8980
10628
  cursor = block.endTime;
10629
+ this.playbackElapsed = Math.max(this.playbackElapsed, cursor);
8981
10630
  }
8982
10631
  if (timeline.duration > cursor) {
8983
10632
  await this.animationController.prepareAnimations?.(
@@ -8999,10 +10648,15 @@ var StagingPlayer = class {
8999
10648
  beginTimeline();
9000
10649
  }
9001
10650
  this.applyTracks(timeline.duration, false, timeline);
10651
+ if (timeline.activeVariant) {
10652
+ await options?.onVariantCue?.(timeline.activeVariant, null);
10653
+ }
9002
10654
  } finally {
9003
10655
  if (this.generation === generation) {
9004
10656
  this.playbackController = null;
9005
10657
  this.incremental = null;
10658
+ this.activeOptions = void 0;
10659
+ this.appliedToolCallIds.clear();
9006
10660
  this.clearCursor();
9007
10661
  this.onCursor = void 0;
9008
10662
  this.clearPerformance();
@@ -9011,6 +10665,10 @@ var StagingPlayer = class {
9011
10665
  }
9012
10666
  updateTimeline(timeline, complete = true) {
9013
10667
  this.incremental?.update(timeline, complete);
10668
+ if (this.incremental) {
10669
+ const elapsed = this.currentTrackTime ?? this.playbackElapsed;
10670
+ this.applyToolCalls(elapsed, this.incremental.timeline);
10671
+ }
9014
10672
  const current = this.incremental?.timeline.dialogue.find(
9015
10673
  (block) => block.id === this.pendingSpeech?.id
9016
10674
  );
@@ -9061,6 +10719,9 @@ var StagingPlayer = class {
9061
10719
  this.playbackController = null;
9062
10720
  this.clock.resume();
9063
10721
  this.incremental = null;
10722
+ this.activeOptions = void 0;
10723
+ this.playbackElapsed = 0;
10724
+ this.appliedToolCallIds.clear();
9064
10725
  this.clearCursor();
9065
10726
  this.onCursor = void 0;
9066
10727
  this.audioPlayer.stop();
@@ -9276,8 +10937,29 @@ var StagingPlayer = class {
9276
10937
  await waitFrame(signal);
9277
10938
  }
9278
10939
  }
10940
+ applyToolCalls(elapsed, timeline) {
10941
+ const calls = timeline.toolCalls;
10942
+ if (!calls || calls.length === 0) {
10943
+ return;
10944
+ }
10945
+ const ordered = calls.length > 1 ? [...calls].sort((left, right) => left.timestamp - right.timestamp) : calls;
10946
+ for (const call of ordered) {
10947
+ if (this.appliedToolCallIds.has(call.id) || elapsed < call.timestamp) {
10948
+ continue;
10949
+ }
10950
+ this.appliedToolCallIds.add(call.id);
10951
+ this.activeOptions?.onToolCall?.(call);
10952
+ if (call.name === "switch_variant" && call.result.ok) {
10953
+ void Promise.resolve(
10954
+ this.activeOptions?.onVariantCue?.(call.result.activeVariant, call)
10955
+ ).catch(() => void 0);
10956
+ }
10957
+ }
10958
+ }
9279
10959
  applyTracks(time, speechActive, timeline) {
9280
10960
  this.currentTrackTime = time;
10961
+ this.playbackElapsed = Math.max(this.playbackElapsed, time);
10962
+ this.applyToolCalls(time, timeline);
9281
10963
  const expression = activeClipAt(timeline.expressions, time);
9282
10964
  const mouthMode = speechActive ? "viseme" : "expression";
9283
10965
  if (expression !== this.activeExpression || expression && mouthMode !== this.activeExpressionMouthMode) {
@@ -9731,7 +11413,7 @@ var VisemePlayer = class {
9731
11413
  };
9732
11414
 
9733
11415
  // ../../clients/web-client/src/character/character-runtime.ts
9734
- var CharacterRuntime = class {
11416
+ var CharacterRuntime = class extends CharacterSceneAccess {
9735
11417
  baseUrl;
9736
11418
  characterId;
9737
11419
  apiToken;
@@ -9741,8 +11423,8 @@ var CharacterRuntime = class {
9741
11423
  animation;
9742
11424
  animations = /* @__PURE__ */ new Map();
9743
11425
  animationController;
9744
- sceneRig = new CharacterSceneRig();
9745
11426
  lazyAnimations;
11427
+ variantController;
9746
11428
  audioPlayer = null;
9747
11429
  stagingPlayer = null;
9748
11430
  facialMixer;
@@ -9751,13 +11433,15 @@ var CharacterRuntime = class {
9751
11433
  telemetry;
9752
11434
  streamPlayback;
9753
11435
  liveMotion;
9754
- defaultParent;
11436
+ playerActivityTracker = new PlayerActivityTracker();
11437
+ listeningReflex;
9755
11438
  currentGazeTarget = null;
9756
11439
  /** The current gaze target came from `lookAt()`. */
9757
11440
  lookAtActive = false;
9758
11441
  thinkingState = false;
9759
11442
  errorCallback;
9760
11443
  constructor(options) {
11444
+ super();
9761
11445
  this.baseUrl = options.baseUrl;
9762
11446
  this.characterId = options.characterId;
9763
11447
  this.apiToken = options.apiToken;
@@ -9771,6 +11455,13 @@ var CharacterRuntime = class {
9771
11455
  animationId,
9772
11456
  assetProfile: this.assetProfile
9773
11457
  }));
11458
+ const loadVariant = options.loadVariant ?? ((variantKey) => loadVariantVrm({
11459
+ baseUrl: this.baseUrl,
11460
+ characterId: this.characterId,
11461
+ variantKey,
11462
+ apiToken: this.apiToken,
11463
+ assetProfile: this.assetProfile
11464
+ }));
9774
11465
  this.lazyAnimations = new LazyAnimations(loadAnimation, (id, animation) => {
9775
11466
  this.animations.set(id, animation);
9776
11467
  this.animationController?.addAnimation?.(id, animation, { loop: false });
@@ -9800,6 +11491,36 @@ var CharacterRuntime = class {
9800
11491
  apiToken: this.apiToken
9801
11492
  });
9802
11493
  this.liveMotion = new LiveMotionPlayer(options.liveMotion);
11494
+ this.listeningReflex = new ListeningReflex(
11495
+ {
11496
+ vrm: () => this.sceneRig.vrm,
11497
+ gazeTarget: () => this.currentGazeTarget,
11498
+ lookAtActive: () => this.lookAtActive,
11499
+ lookAtTemporarily: (target) => this.lookAtTemporarily(target),
11500
+ facialMixer: this.facialMixer
11501
+ },
11502
+ this.playerActivityTracker
11503
+ );
11504
+ this.variantController = new CharacterVariantController({
11505
+ loadVariant,
11506
+ getCurrentVrm: () => this.sceneRig.vrm,
11507
+ handleCallbackError: this.handleCallbackError,
11508
+ applyVrmSwap: (nextVrm) => {
11509
+ this.sceneRig.setVrm(nextVrm, this.defaultParent);
11510
+ this.facialMixer.setVrm(nextVrm, { preserveState: true });
11511
+ this.animationController?.setVrm?.(nextVrm, this.animations);
11512
+ this.liveMotion.setVrm(nextVrm);
11513
+ this.applyGaze();
11514
+ if (this.performanceRecorder.isRecording) {
11515
+ this.facialMixer.capturePerformanceModel();
11516
+ }
11517
+ }
11518
+ });
11519
+ if (options.initialAssetInfo?.variants) {
11520
+ this.variantController.configureVariants(
11521
+ options.initialAssetInfo.variants
11522
+ );
11523
+ }
9803
11524
  this.streamPlayback = new StreamPlaybackController({
9804
11525
  getStagingPlayer: this.getStagingPlayer,
9805
11526
  telemetry: this.telemetry,
@@ -9813,6 +11534,19 @@ var CharacterRuntime = class {
9813
11534
  void this.liveMotion.load(stagingId, track).catch((error) => {
9814
11535
  this.handleCallbackError("liveMotion", error);
9815
11536
  });
11537
+ },
11538
+ onToolCall: (toolCall) => {
11539
+ this.variantController.emit("toolCall", toolCall);
11540
+ },
11541
+ onVariantCue: async (variantKey, toolCall) => {
11542
+ try {
11543
+ await this.variantController.switchVariant(variantKey, toolCall);
11544
+ } catch (error) {
11545
+ this.reportError(
11546
+ error instanceof Error ? error : new Error(String(error)),
11547
+ "switchVariant"
11548
+ );
11549
+ }
9816
11550
  }
9817
11551
  });
9818
11552
  if (options.gazeTarget !== void 0) {
@@ -9848,53 +11582,28 @@ var CharacterRuntime = class {
9848
11582
  return this.sceneRig.vrm;
9849
11583
  }
9850
11584
  set vrm(value) {
9851
- this.sceneRig.setVrm(value);
11585
+ const previous = this.sceneRig.vrm;
11586
+ this.sceneRig.setVrm(value, this.defaultParent);
9852
11587
  this.facialMixer.setVrm(value);
9853
11588
  this.liveMotion.setVrm(value);
9854
11589
  this.applyGaze();
9855
- if (value && this.defaultParent && !value.scene?.parent) {
9856
- this.sceneRig.attachTo(this.defaultParent);
9857
- }
11590
+ this.variantController.syncDirectVrm(value, previous);
9858
11591
  }
9859
- /** True once the character VRM model is loaded. */
9860
- get ready() {
9861
- return this.sceneRig.vrm !== null;
9862
- }
9863
- /** Root Three.js `Group` (`vrm.scene`) to add to a `THREE.Scene`. */
9864
- get scene() {
9865
- return this.sceneRig.scene;
9866
- }
9867
- /** Alias for `scene` for Three.js `Object3D` workflows. */
9868
- get object3d() {
9869
- return this.sceneRig.object3d;
9870
- }
9871
- /** Alias for `scene`. */
9872
- get root() {
9873
- return this.sceneRig.root;
9874
- }
9875
- /** Primary character mesh (`SkinnedMesh` preferred, otherwise first `Mesh`). */
9876
- get mesh() {
9877
- return this.sceneRig.mesh;
9878
- }
9879
- /** All Three.js `Mesh` instances in the character hierarchy. */
9880
- get meshes() {
9881
- return this.sceneRig.meshes;
11592
+ /** Allowed appearance variants configured for this character (`default` first). */
11593
+ get variants() {
11594
+ return this.variantController.variants;
9882
11595
  }
9883
- /** All Three.js `SkinnedMesh` instances in the character hierarchy. */
9884
- get skinnedMeshes() {
9885
- return this.sceneRig.skinnedMeshes;
11596
+ /** Currently active appearance variant key (defaults to `'default'`). */
11597
+ get activeVariant() {
11598
+ return this.variantController.activeVariant;
9886
11599
  }
9887
- /** Deduplicated Three.js `Material` instances across all character meshes. */
9888
- get materials() {
9889
- return this.sceneRig.materials;
11600
+ /** Cached Three-VRM instances keyed by variant key. */
11601
+ get variantVrms() {
11602
+ return this.variantController.variantVrms;
9890
11603
  }
9891
- /** Deduplicated Three.js `Skeleton` instances across all character skinned meshes. */
9892
- get skeletons() {
9893
- return this.sceneRig.skeletons;
9894
- }
9895
- /** Three-VRM humanoid rig when loaded. */
9896
- get humanoid() {
9897
- return this.sceneRig.humanoid;
11604
+ /** Switch the character's active appearance variant, loading on-demand if needed. */
11605
+ async switchVariant(variantKey) {
11606
+ await this.variantController.switchVariant(variantKey, null);
9898
11607
  }
9899
11608
  /** Underlying Three.js `AnimationMixer` driving skeletal animations. */
9900
11609
  get mixer() {
@@ -9947,36 +11656,6 @@ var CharacterRuntime = class {
9947
11656
  cursor: this.cursor
9948
11657
  };
9949
11658
  }
9950
- /** Look up a raw humanoid bone `Object3D` by VRM bone name (for example `'head'` or `'rightHand'`). */
9951
- getBone(name) {
9952
- return this.sceneRig.getBone(name);
9953
- }
9954
- /** Look up a normalized humanoid bone `Object3D` by VRM bone name. */
9955
- getNormalizedBone(name) {
9956
- return this.sceneRig.getNormalizedBone(name);
9957
- }
9958
- /** Attach the character root `Group` to a Three.js `Scene` or parent `Object3D`. */
9959
- attachTo(parent) {
9960
- this.defaultParent = parent;
9961
- return this.sceneRig.attachTo(parent);
9962
- }
9963
- /** Detach the character root `Group` from its current parent `Object3D`. */
9964
- detach() {
9965
- this.defaultParent = null;
9966
- this.sceneRig.detach();
9967
- }
9968
- /** Configure `castShadow` and `receiveShadow` on all character meshes. */
9969
- setShadows(options) {
9970
- this.sceneRig.setShadows(options);
9971
- }
9972
- /** Configure `position`, `rotation`, and `scale` on the character root `Group`. */
9973
- setTransform(options) {
9974
- this.sceneRig.setTransform(options);
9975
- }
9976
- /** Re-index cached meshes, materials, and skeletons after custom scene graph edits. */
9977
- refreshMeshes() {
9978
- this.sceneRig.refreshMeshes();
9979
- }
9980
11659
  async ensureAnimation(animationId) {
9981
11660
  await this.lazyAnimations.ensure(animationId);
9982
11661
  }
@@ -10075,6 +11754,28 @@ var CharacterRuntime = class {
10075
11754
  this.applyGaze();
10076
11755
  };
10077
11756
  }
11757
+ /**
11758
+ * Tells the character that the player types (`true` at each input event
11759
+ * of the text field) or stopped (`false`, for example after sending). The
11760
+ * character looks at the player and shows a subtle attentive face; typing
11761
+ * ends by itself after 1.5 s without input. Local only: no request.
11762
+ */
11763
+ notifyUserTyping(active) {
11764
+ this.playerActivityTracker.notifyTyping(active);
11765
+ }
11766
+ /** `'speaking'` while listening, `'typing'` after typed input, else `'idle'`. */
11767
+ get playerActivity() {
11768
+ return this.playerActivityTracker.state;
11769
+ }
11770
+ onPlayerActivity(callback) {
11771
+ return this.playerActivityTracker.onChange((state) => {
11772
+ try {
11773
+ callback(state);
11774
+ } catch (error) {
11775
+ this.handleCallbackError("onPlayerActivity", error);
11776
+ }
11777
+ });
11778
+ }
10078
11779
  startPerformanceRecording(options) {
10079
11780
  this.performanceRecorder.start(options);
10080
11781
  this.facialMixer.capturePerformanceModel();
@@ -10121,6 +11822,21 @@ var CharacterRuntime = class {
10121
11822
  onTimelineStart(callback) {
10122
11823
  return this.streamPlayback.onTimelineStart(callback);
10123
11824
  }
11825
+ on(event, listener) {
11826
+ return this.variantController.on(event, listener);
11827
+ }
11828
+ off(event, listener) {
11829
+ this.variantController.off(event, listener);
11830
+ }
11831
+ onVrmChange(callback) {
11832
+ return this.variantController.on("vrmChange", callback);
11833
+ }
11834
+ onVariantChange(callback) {
11835
+ return this.variantController.on("variantChange", callback);
11836
+ }
11837
+ onToolCall(callback) {
11838
+ return this.variantController.on("toolCall", callback);
11839
+ }
10124
11840
  /** Advance skeletal animations, lip-sync visemes, facial expressions, and VRM physics by `delta` seconds. */
10125
11841
  update(delta) {
10126
11842
  if (this.stagingPlayer?.paused) {
@@ -10132,6 +11848,10 @@ var CharacterRuntime = class {
10132
11848
  this.streamPlayback.activeStagingId,
10133
11849
  this.stagingPlayer?.trackTime ?? null
10134
11850
  );
11851
+ this.listeningReflex.update(delta, {
11852
+ stagingActive: this.streamPlayback.playing || this.speaking || (this.stagingPlayer?.trackTime ?? null) !== null,
11853
+ liveMotionWeight: this.liveMotion.liveWeight
11854
+ });
10135
11855
  this.visemePlayer.update(delta);
10136
11856
  this.facialMixer.update(delta);
10137
11857
  this.vrm?.update(delta);
@@ -10144,73 +11864,595 @@ var CharacterRuntime = class {
10144
11864
  });
10145
11865
  }
10146
11866
  }
10147
- getAudioPlayer = () => {
10148
- if (!this.audioPlayer) {
10149
- this.audioPlayer = new AudioPlayer(this.visemePlayer);
11867
+ getAudioPlayer = () => {
11868
+ if (!this.audioPlayer) {
11869
+ this.audioPlayer = new AudioPlayer(this.visemePlayer);
11870
+ }
11871
+ return this.audioPlayer;
11872
+ };
11873
+ getStagingPlayer = () => {
11874
+ if (!this.animationController) {
11875
+ throw new Error("Animation controller is not initialized.");
11876
+ }
11877
+ if (!this.stagingPlayer) {
11878
+ this.stagingPlayer = new StagingPlayer(
11879
+ this.getAudioPlayer(),
11880
+ this.facialMixer,
11881
+ this.animationController
11882
+ );
11883
+ }
11884
+ return this.stagingPlayer;
11885
+ };
11886
+ /**
11887
+ * A template face follows the gaze target with its IdleFace. Only a
11888
+ * `lookAt()` target moves the eyes of another face, through three-vrm's
11889
+ * `vrm.lookAt.target`; on a template face that target would stop the
11890
+ * IdleFace (its gaze cone, idle eye moves and look cues).
11891
+ */
11892
+ applyGaze() {
11893
+ this.facialMixer.setGazeTarget(this.currentGazeTarget);
11894
+ this.sceneRig.setVrmLookAtTarget(
11895
+ this.lookAtActive && !this.facialMixer.hasIdleFace ? this.currentGazeTarget : null
11896
+ );
11897
+ }
11898
+ resetRuntimeAssets(options) {
11899
+ const cachedVrms = this.variantController.reset();
11900
+ this.listeningReflex.reset();
11901
+ this.playerActivityTracker.reset();
11902
+ this.lazyAnimations.reset();
11903
+ this.stagingPlayer?.cleanup();
11904
+ this.stagingPlayer = null;
11905
+ this.audioPlayer?.reset();
11906
+ this.facialMixer.reset();
11907
+ if (this.animationController instanceof AnimationController) {
11908
+ this.animationController.clearAnimations();
11909
+ }
11910
+ this.animationController = void 0;
11911
+ this.animations.clear();
11912
+ if (options?.disposeVrms !== false) {
11913
+ this.sceneRig.disposeAllVrms(cachedVrms);
11914
+ } else {
11915
+ this.sceneRig.setVrm(null);
11916
+ }
11917
+ this.facialMixer.setVrm(null);
11918
+ this.liveMotion.setVrm(null);
11919
+ }
11920
+ disposeRuntimeAssets() {
11921
+ this.resetRuntimeAssets({ disposeVrms: true });
11922
+ this.audioPlayer?.cleanup();
11923
+ this.audioPlayer = null;
11924
+ }
11925
+ handleCallbackError = (context2, error) => {
11926
+ const normalized = error instanceof Error ? error : new Error(String(error ?? "Error"));
11927
+ this.reportError(normalized, `${context2} callback`);
11928
+ };
11929
+ reportError = (error, context2) => {
11930
+ if (this.errorCallback) {
11931
+ this.errorCallback(error);
11932
+ return;
11933
+ }
11934
+ console.error(context2 ? `${context2}:` : "Npcify:", error);
11935
+ };
11936
+ };
11937
+
11938
+ // ../../clients/web-client/src/conversation-socket.ts
11939
+ import { io } from "socket.io-client";
11940
+
11941
+ // ../../clients/web-client/src/sight/sight-error.ts
11942
+ var SightError = class extends Error {
11943
+ constructor(code, message) {
11944
+ super(message ?? code);
11945
+ this.code = code;
11946
+ this.name = "NpcifySightError";
11947
+ }
11948
+ };
11949
+ var SIGHT_STOP_CODES = /* @__PURE__ */ new Set([
11950
+ "SIGHT_DISABLED",
11951
+ "INSUFFICIENT_POINTS",
11952
+ "CONVERSATION_ACCESS_DENIED"
11953
+ ]);
11954
+ var SIGHT_OFF_MESSAGE = "Sight is off for this character. Its owner turns it on in the Sight tab.";
11955
+ var STOP_MESSAGES = {
11956
+ SIGHT_DISABLED: SIGHT_OFF_MESSAGE,
11957
+ INSUFFICIENT_POINTS: "The owner of the character has no points to pay for sight.",
11958
+ CONVERSATION_ACCESS_DENIED: "This connection has no access to the conversation."
11959
+ };
11960
+ var sightStopError = function(code) {
11961
+ return new SightError(code, STOP_MESSAGES[code] ?? code);
11962
+ };
11963
+ var toSightError = function(error, fallback) {
11964
+ if (error instanceof SightError) {
11965
+ return error;
11966
+ }
11967
+ const code = error?.code;
11968
+ return new SightError(
11969
+ typeof code === "string" && code ? code : fallback,
11970
+ error instanceof Error ? error.message : void 0
11971
+ );
11972
+ };
11973
+
11974
+ // ../../clients/web-client/src/sight/frame-source.ts
11975
+ var LUMA_GRID = Object.freeze({ width: 32, height: 18 });
11976
+ var SMALLER_JPEG_QUALITY = 0.5;
11977
+ var HAVE_CURRENT_DATA = 2;
11978
+ var fitCaptureSize = function(width, height, limit = SIGHT_CAPTURE_SIZE) {
11979
+ const scale = Math.min(1, limit.width / width, limit.height / height);
11980
+ return {
11981
+ width: Math.max(1, Math.round(width * scale)),
11982
+ height: Math.max(1, Math.round(height * scale))
11983
+ };
11984
+ };
11985
+ var lumaOf = function(rgba) {
11986
+ const luma = new Uint8Array(Math.floor(rgba.length / 4));
11987
+ for (let index = 0; index < luma.length; index += 1) {
11988
+ const offset = index * 4;
11989
+ luma[index] = Math.round(
11990
+ 0.299 * (rgba[offset] ?? 0) + 0.587 * (rgba[offset + 1] ?? 0) + 0.114 * (rgba[offset + 2] ?? 0)
11991
+ );
11992
+ }
11993
+ return luma;
11994
+ };
11995
+ var meanLumaChange = function(previous, next) {
11996
+ const length = Math.min(previous.length, next.length);
11997
+ if (length === 0) {
11998
+ return 0;
11999
+ }
12000
+ let sum = 0;
12001
+ for (let index = 0; index < length; index += 1) {
12002
+ sum += Math.abs((next[index] ?? 0) - (previous[index] ?? 0));
12003
+ }
12004
+ return sum / length;
12005
+ };
12006
+ var encodeJpeg = function(canvas, quality) {
12007
+ return new Promise((resolve) => {
12008
+ canvas.toBlob(resolve, "image/jpeg", quality);
12009
+ });
12010
+ };
12011
+ var VideoFrameSource = class {
12012
+ constructor(stream, owned) {
12013
+ this.stream = stream;
12014
+ this.owned = owned;
12015
+ const settings = stream.getVideoTracks()[0]?.getSettings?.();
12016
+ this.kind = settings?.displaySurface ? "screen" : "camera";
12017
+ const video = document.createElement("video");
12018
+ video.muted = true;
12019
+ video.playsInline = true;
12020
+ video.setAttribute("playsinline", "");
12021
+ video.srcObject = stream;
12022
+ void video.play().catch(() => void 0);
12023
+ this.video = video;
12024
+ this.frameCanvas = document.createElement("canvas");
12025
+ const grid = document.createElement("canvas");
12026
+ grid.width = LUMA_GRID.width;
12027
+ grid.height = LUMA_GRID.height;
12028
+ this.gridContext = grid.getContext("2d", { willReadFrequently: true });
12029
+ if (this.gridContext) {
12030
+ this.gridContext.imageSmoothingQuality = "high";
12031
+ }
12032
+ }
12033
+ kind;
12034
+ video;
12035
+ frameCanvas;
12036
+ frameContext = null;
12037
+ gridContext;
12038
+ stopped = false;
12039
+ get live() {
12040
+ return !this.stopped && this.stream.getVideoTracks().some((track) => track.readyState === "live");
12041
+ }
12042
+ async capture() {
12043
+ const picture = this.pictureSize();
12044
+ if (!picture) {
12045
+ return null;
12046
+ }
12047
+ const size = fitCaptureSize(picture.width, picture.height);
12048
+ const canvas = this.frameCanvas;
12049
+ if (canvas.width !== size.width || canvas.height !== size.height) {
12050
+ canvas.width = size.width;
12051
+ canvas.height = size.height;
12052
+ }
12053
+ this.frameContext ??= canvas.getContext("2d");
12054
+ const context2 = this.frameContext;
12055
+ if (!context2) {
12056
+ return null;
12057
+ }
12058
+ context2.imageSmoothingQuality = "high";
12059
+ context2.drawImage(this.video, 0, 0, size.width, size.height);
12060
+ let blob = await encodeJpeg(canvas, SIGHT_CAPTURE_SIZE.jpegQuality);
12061
+ if (blob && blob.size > SIGHT_LIMITS.maxFrameBytes) {
12062
+ blob = await encodeJpeg(canvas, SMALLER_JPEG_QUALITY);
12063
+ }
12064
+ if (!blob || blob.size > SIGHT_LIMITS.maxFrameBytes) {
12065
+ return null;
12066
+ }
12067
+ return { jpeg: await blob.arrayBuffer(), ...size };
12068
+ }
12069
+ luminance() {
12070
+ const context2 = this.gridContext;
12071
+ if (!context2 || !this.pictureSize()) {
12072
+ return null;
12073
+ }
12074
+ context2.drawImage(this.video, 0, 0, LUMA_GRID.width, LUMA_GRID.height);
12075
+ return lumaOf(
12076
+ context2.getImageData(0, 0, LUMA_GRID.width, LUMA_GRID.height).data
12077
+ );
12078
+ }
12079
+ stop() {
12080
+ if (this.stopped) {
12081
+ return;
12082
+ }
12083
+ this.stopped = true;
12084
+ this.video.pause();
12085
+ this.video.srcObject = null;
12086
+ if (this.owned) {
12087
+ this.stream.getTracks().forEach((track) => track.stop());
12088
+ }
12089
+ }
12090
+ pictureSize() {
12091
+ const { video } = this;
12092
+ if (this.stopped || video.readyState < HAVE_CURRENT_DATA || !video.videoWidth || !video.videoHeight) {
12093
+ return null;
12094
+ }
12095
+ return { width: video.videoWidth, height: video.videoHeight };
12096
+ }
12097
+ };
12098
+ var openFrameSource = async function(source) {
12099
+ if (source !== "camera") {
12100
+ if (typeof source?.getVideoTracks !== "function") {
12101
+ throw new SightError(
12102
+ "CAMERA_UNAVAILABLE",
12103
+ 'The sight source must be "camera" or a MediaStream.'
12104
+ );
12105
+ }
12106
+ if (source.getVideoTracks().length === 0) {
12107
+ throw new SightError(
12108
+ "CAMERA_UNAVAILABLE",
12109
+ "The stream has no video track."
12110
+ );
12111
+ }
12112
+ return new VideoFrameSource(source, false);
12113
+ }
12114
+ const devices = typeof navigator === "undefined" ? void 0 : navigator.mediaDevices;
12115
+ if (!devices?.getUserMedia) {
12116
+ throw new SightError(
12117
+ "CAMERA_UNAVAILABLE",
12118
+ "This browser cannot open the camera."
12119
+ );
12120
+ }
12121
+ let stream;
12122
+ try {
12123
+ stream = await devices.getUserMedia({
12124
+ video: {
12125
+ width: { ideal: SIGHT_CAPTURE_SIZE.width },
12126
+ height: { ideal: SIGHT_CAPTURE_SIZE.height }
12127
+ },
12128
+ audio: false
12129
+ });
12130
+ } catch (error) {
12131
+ const name = error?.name;
12132
+ throw name === "NotAllowedError" || name === "SecurityError" ? new SightError("CAMERA_PERMISSION_DENIED", "Camera access was denied.") : new SightError("CAMERA_UNAVAILABLE", "No camera is available.");
12133
+ }
12134
+ return new VideoFrameSource(stream, true);
12135
+ };
12136
+
12137
+ // ../../clients/web-client/src/sight/sight-input.ts
12138
+ var SIGHT_CHECK_MS = 250;
12139
+ var SIGHT_INTERVAL_MS = 5e3;
12140
+ var SIGHT_MOTION_THRESHOLD = 8;
12141
+ var ACK_TIMEOUT_MS = 2e4;
12142
+ var systemClock = {
12143
+ now: () => Date.now(),
12144
+ setInterval: (callback, ms) => setInterval(callback, ms),
12145
+ clearInterval: (handle) => clearInterval(handle)
12146
+ };
12147
+ var settle = async function(run) {
12148
+ return run();
12149
+ };
12150
+ var documentHidden = function() {
12151
+ return typeof document !== "undefined" && document.hidden === true;
12152
+ };
12153
+ var intervalOf = function(value) {
12154
+ return typeof value === "number" && value > 0 ? Math.max(SIGHT_LIMITS.minFrameIntervalMs, value) : SIGHT_INTERVAL_MS;
12155
+ };
12156
+ var thresholdOf = function(motion) {
12157
+ if (motion === false) {
12158
+ return null;
12159
+ }
12160
+ const threshold = motion?.threshold;
12161
+ return typeof threshold === "number" && Number.isFinite(threshold) ? Math.max(0, threshold) : SIGHT_MOTION_THRESHOLD;
12162
+ };
12163
+ var isSightObservationEvent = function(value) {
12164
+ const event = value;
12165
+ const observation = event?.observation;
12166
+ return typeof event?.conversationId === "string" && typeof event.observationId === "string" && typeof event.summary === "string" && typeof observation === "object" && observation !== null && typeof observation.people === "number" && Array.isArray(observation.persons) && Array.isArray(observation.text);
12167
+ };
12168
+ var SightInput = class {
12169
+ constructor(deps) {
12170
+ this.deps = deps;
12171
+ this.clock = deps.clock ?? systemClock;
12172
+ this.hidden = deps.hidden ?? documentHidden;
12173
+ }
12174
+ clock;
12175
+ hidden;
12176
+ session = null;
12177
+ /** Why sight cannot start; null when it can. */
12178
+ unavailable = "Sight needs the character info: load the character first.";
12179
+ current = "unavailable";
12180
+ stateCallback = null;
12181
+ observationCallback = null;
12182
+ get state() {
12183
+ return this.current;
12184
+ }
12185
+ /**
12186
+ * The character info was read. Sight is available only when the owner
12187
+ * turned it on, and never in a baked conversation. Sight that becomes
12188
+ * unavailable stops.
12189
+ */
12190
+ configure(settings) {
12191
+ this.unavailable = settings.baked ? "Sight is not available in a baked (offline) conversation." : settings.enabled ? null : SIGHT_OFF_MESSAGE;
12192
+ if (this.unavailable) {
12193
+ this.stop();
12194
+ } else {
12195
+ this.publish();
12196
+ }
12197
+ }
12198
+ /**
12199
+ * Opens the source and joins the conversation. Resolves when frames can
12200
+ * go; the first frame goes when the video has a picture.
12201
+ */
12202
+ async start(options = {}) {
12203
+ if (this.unavailable) {
12204
+ throw new SightError("SIGHT_DISABLED", this.unavailable);
12205
+ }
12206
+ if (this.session) {
12207
+ throw new SightError(
12208
+ "SIGHT_ALREADY_STARTED",
12209
+ "Sight has already started."
12210
+ );
12211
+ }
12212
+ const conversationId = this.deps.conversationId();
12213
+ if (!conversationId) {
12214
+ throw new SightError(
12215
+ "CONVERSATION_REQUIRED",
12216
+ "Create a conversation before sight starts."
12217
+ );
12218
+ }
12219
+ const session = {
12220
+ intervalMs: intervalOf(options.intervalMs),
12221
+ threshold: thresholdOf(options.motion),
12222
+ onError: options.onError,
12223
+ source: null,
12224
+ timer: null,
12225
+ grid: null,
12226
+ lastSentAt: Number.NEGATIVE_INFINITY,
12227
+ motion: false,
12228
+ inFlight: false,
12229
+ rejoin: false
12230
+ };
12231
+ this.session = session;
12232
+ this.publish();
12233
+ const open = this.deps.openSource ?? openFrameSource;
12234
+ const [joined, opened] = await Promise.allSettled([
12235
+ settle(() => this.deps.socket().ensureConnected(conversationId)),
12236
+ settle(() => open(options.source ?? "camera"))
12237
+ ]);
12238
+ const source = opened.status === "fulfilled" ? opened.value : null;
12239
+ if (session !== this.session) {
12240
+ source?.stop();
12241
+ return;
12242
+ }
12243
+ const failure = opened.status === "rejected" ? toSightError(opened.reason, "CAMERA_UNAVAILABLE") : joined.status === "rejected" ? toSightError(joined.reason, "CONVERSATION_DISCONNECTED") : null;
12244
+ if (failure || !source) {
12245
+ source?.stop();
12246
+ this.session = null;
12247
+ this.publish();
12248
+ throw failure ?? new SightError("CAMERA_UNAVAILABLE");
12249
+ }
12250
+ session.source = source;
12251
+ session.timer = this.clock.setInterval(
12252
+ () => this.check(session),
12253
+ SIGHT_CHECK_MS
12254
+ );
12255
+ this.publish();
12256
+ this.check(session);
12257
+ }
12258
+ /** Stops the frames and releases the source. Does nothing when sight is off. */
12259
+ stop() {
12260
+ if (this.session) {
12261
+ this.end(this.session);
12262
+ }
12263
+ this.publish();
12264
+ }
12265
+ /** An observation of the current conversation (the socket checked the ID). */
12266
+ handleObservation(event) {
12267
+ if (this.observationCallback) {
12268
+ invokeSafeCallback(
12269
+ "onSight",
12270
+ this.deps.handleCallbackError,
12271
+ () => this.observationCallback?.(event)
12272
+ );
12273
+ }
12274
+ }
12275
+ onObservation(callback) {
12276
+ return bindCallbackSlot(
12277
+ () => this.observationCallback,
12278
+ (next) => this.observationCallback = next,
12279
+ callback
12280
+ );
12281
+ }
12282
+ onState(callback) {
12283
+ return bindCallbackSlot(
12284
+ () => this.stateCallback,
12285
+ (next) => this.stateCallback = next,
12286
+ callback
12287
+ );
12288
+ }
12289
+ /** One check: the end of the source, a hidden page, motion, then a due frame. */
12290
+ check(session) {
12291
+ const source = session.source;
12292
+ if (session !== this.session || !source) {
12293
+ return;
12294
+ }
12295
+ if (!source.live) {
12296
+ this.fail(
12297
+ session,
12298
+ new SightError("SIGHT_SOURCE_ENDED", "The video of sight ended.")
12299
+ );
12300
+ return;
12301
+ }
12302
+ if (this.hidden()) {
12303
+ session.grid = null;
12304
+ return;
12305
+ }
12306
+ if (session.threshold !== null) {
12307
+ const grid = source.luminance();
12308
+ if (grid && session.grid && meanLumaChange(session.grid, grid) > session.threshold) {
12309
+ session.motion = true;
12310
+ }
12311
+ session.grid = grid;
12312
+ }
12313
+ this.sendWhenDue(session);
12314
+ }
12315
+ /**
12316
+ * A frame is due at the interval, or after motion when one second passed
12317
+ * since the last frame. While a frame waits for its acknowledgement, a due
12318
+ * frame waits too: the acknowledgement sends it.
12319
+ */
12320
+ sendWhenDue(session) {
12321
+ if (session !== this.session || session.inFlight || this.hidden()) {
12322
+ return;
10150
12323
  }
10151
- return this.audioPlayer;
10152
- };
10153
- getStagingPlayer = () => {
10154
- if (!this.animationController) {
10155
- throw new Error("Animation controller is not initialized.");
12324
+ const elapsed = this.clock.now() - session.lastSentAt;
12325
+ if (elapsed >= session.intervalMs || session.motion && elapsed >= SIGHT_LIMITS.minFrameIntervalMs) {
12326
+ void this.send(session);
10156
12327
  }
10157
- if (!this.stagingPlayer) {
10158
- this.stagingPlayer = new StagingPlayer(
10159
- this.getAudioPlayer(),
10160
- this.facialMixer,
10161
- this.animationController
12328
+ }
12329
+ async send(session) {
12330
+ const source = session.source;
12331
+ if (!source) {
12332
+ return;
12333
+ }
12334
+ session.inFlight = true;
12335
+ let sent = false;
12336
+ try {
12337
+ const capturedAt = this.clock.now();
12338
+ let frame;
12339
+ try {
12340
+ frame = await source.capture();
12341
+ } catch (error) {
12342
+ this.fail(
12343
+ session,
12344
+ new SightError(
12345
+ "CAMERA_UNAVAILABLE",
12346
+ error instanceof Error ? error.message : "The video gave no picture."
12347
+ )
12348
+ );
12349
+ return;
12350
+ }
12351
+ if (!frame || session !== this.session) {
12352
+ return;
12353
+ }
12354
+ const conversationId = this.deps.conversationId();
12355
+ if (!conversationId) {
12356
+ this.stop();
12357
+ return;
12358
+ }
12359
+ session.lastSentAt = this.clock.now();
12360
+ session.motion = false;
12361
+ sent = true;
12362
+ const socket = this.deps.socket();
12363
+ if (session.rejoin) {
12364
+ await socket.ensureConnected(conversationId);
12365
+ session.rejoin = false;
12366
+ }
12367
+ const request = {
12368
+ conversationId,
12369
+ capturedAt,
12370
+ width: frame.width,
12371
+ height: frame.height,
12372
+ source: source.kind,
12373
+ jpeg: frame.jpeg
12374
+ };
12375
+ const ack = await socket.request(
12376
+ SIGHT_EVENTS.frame,
12377
+ request,
12378
+ ACK_TIMEOUT_MS
10162
12379
  );
12380
+ const code = ack && !ack.ok ? ack.error : null;
12381
+ if (code && SIGHT_STOP_CODES.has(code)) {
12382
+ this.fail(session, sightStopError(code));
12383
+ }
12384
+ } catch (error) {
12385
+ this.failed(session, error);
12386
+ } finally {
12387
+ session.inFlight = false;
12388
+ if (sent) {
12389
+ this.sendWhenDue(session);
12390
+ }
10163
12391
  }
10164
- return this.stagingPlayer;
10165
- };
12392
+ }
10166
12393
  /**
10167
- * A template face follows the gaze target with its IdleFace. Only a
10168
- * `lookAt()` target moves the eyes of another face, through three-vrm's
10169
- * `vrm.lookAt.target`; on a template face that target would stop the
10170
- * IdleFace (its gaze cone, idle eye moves and look cues).
12394
+ * A frame that did not reach the server or got no acknowledgement. A
12395
+ * refused conversation stops sight. A timeout skips the frame; a lost
12396
+ * connection also joins the conversation again before the next frame.
10171
12397
  */
10172
- applyGaze() {
10173
- this.facialMixer.setGazeTarget(this.currentGazeTarget);
10174
- this.sceneRig.setVrmLookAtTarget(
10175
- this.lookAtActive && !this.facialMixer.hasIdleFace ? this.currentGazeTarget : null
10176
- );
12398
+ failed(session, error) {
12399
+ const code = error?.code;
12400
+ if (typeof code === "string" && SIGHT_STOP_CODES.has(code)) {
12401
+ this.fail(session, sightStopError(code));
12402
+ return;
12403
+ }
12404
+ if (code !== "CONVERSATION_TIMEOUT") {
12405
+ session.rejoin = true;
12406
+ }
10177
12407
  }
10178
- resetRuntimeAssets() {
10179
- this.lazyAnimations.reset();
10180
- this.stagingPlayer?.cleanup();
10181
- this.stagingPlayer = null;
10182
- this.audioPlayer?.reset();
10183
- this.facialMixer.reset();
10184
- if (this.animationController instanceof AnimationController) {
10185
- this.animationController.clearAnimations();
12408
+ /** Stops sight because of `error`, and reports it. */
12409
+ fail(session, error) {
12410
+ if (session !== this.session) {
12411
+ return;
12412
+ }
12413
+ this.end(session);
12414
+ if (error.code === "SIGHT_DISABLED") {
12415
+ this.unavailable = SIGHT_OFF_MESSAGE;
12416
+ }
12417
+ this.publish(error);
12418
+ const onError = session.onError;
12419
+ if (onError) {
12420
+ invokeSafeCallback(
12421
+ "startSight onError",
12422
+ this.deps.handleCallbackError,
12423
+ () => onError(error)
12424
+ );
10186
12425
  }
10187
- this.animationController = void 0;
10188
- this.animations.clear();
10189
- this.sceneRig.setVrm(null);
10190
- this.facialMixer.setVrm(null);
10191
- this.liveMotion.setVrm(null);
10192
12426
  }
10193
- disposeRuntimeAssets() {
10194
- this.sceneRig.disposeVrm();
10195
- this.resetRuntimeAssets();
10196
- this.audioPlayer?.cleanup();
10197
- this.audioPlayer = null;
12427
+ end(session) {
12428
+ if (this.session !== session) {
12429
+ return;
12430
+ }
12431
+ this.session = null;
12432
+ if (session.timer !== null) {
12433
+ this.clock.clearInterval(session.timer);
12434
+ session.timer = null;
12435
+ }
12436
+ session.source?.stop();
10198
12437
  }
10199
- handleCallbackError = (context2, error) => {
10200
- const normalized = error instanceof Error ? error : new Error(String(error ?? "Error"));
10201
- this.reportError(normalized, `${context2} callback`);
10202
- };
10203
- reportError = (error, context2) => {
10204
- if (this.errorCallback) {
10205
- this.errorCallback(error);
12438
+ publish(error) {
12439
+ const session = this.session;
12440
+ const next = session ? session.source ? "on" : "starting" : this.unavailable ? "unavailable" : "off";
12441
+ if (next === this.current && !error) {
10206
12442
  return;
10207
12443
  }
10208
- console.error(context2 ? `${context2}:` : "Npcify:", error);
10209
- };
12444
+ this.current = next;
12445
+ if (this.stateCallback) {
12446
+ invokeSafeCallback(
12447
+ "onSightState",
12448
+ this.deps.handleCallbackError,
12449
+ () => this.stateCallback?.(next, error)
12450
+ );
12451
+ }
12452
+ }
10210
12453
  };
10211
12454
 
10212
12455
  // ../../clients/web-client/src/conversation-socket.ts
10213
- import { io } from "socket.io-client";
10214
12456
  var ConversationConnectionError = class extends Error {
10215
12457
  constructor(code, message, reason = null) {
10216
12458
  super(message);
@@ -10563,6 +12805,11 @@ var ConversationSocket = class {
10563
12805
  this.options.handleSpeechEvent?.({ type: "turnFailed", event });
10564
12806
  }
10565
12807
  });
12808
+ socket.on(SIGHT_EVENTS.observation, (payload) => {
12809
+ if (isSightObservationEvent(payload) && payload.conversationId === this.options.getConversationId()) {
12810
+ this.options.handleSightEvent?.(payload);
12811
+ }
12812
+ });
10566
12813
  socket.on("conversations:turn", (payload) => {
10567
12814
  let candidate;
10568
12815
  try {
@@ -10591,6 +12838,96 @@ var ConversationSocket = class {
10591
12838
  }
10592
12839
  };
10593
12840
 
12841
+ // ../../clients/web-client/src/input-mode-rules.ts
12842
+ var InputModeRules = class {
12843
+ configured;
12844
+ resolved = null;
12845
+ /** The character's voice input switch, after the character loads. */
12846
+ speechInput = null;
12847
+ latestOptions = [];
12848
+ /**
12849
+ * Throws an `InputModeError` at once for a malformed configuration, and
12850
+ * for a baked asset with a mode other than `options`.
12851
+ */
12852
+ constructor(inputModes, bakedAsset) {
12853
+ this.configured = parseInputModes(inputModes);
12854
+ if (this.configured && bakedAsset) {
12855
+ resolveInputModes(this.configured, {
12856
+ allowCustomReply: false,
12857
+ baked: true,
12858
+ speechInput: false
12859
+ });
12860
+ }
12861
+ }
12862
+ /**
12863
+ * The character loaded. Throws an `InputModeError` when the configuration
12864
+ * has a mode that the character does not permit.
12865
+ */
12866
+ load(permissions) {
12867
+ this.speechInput = permissions.speechInput;
12868
+ this.resolved = resolveInputModes(this.configured, permissions);
12869
+ }
12870
+ /**
12871
+ * The input modes in use: the configured modes, or every mode that the
12872
+ * character permits. Before the character loads, the configuration or all
12873
+ * modes.
12874
+ */
12875
+ get modes() {
12876
+ return this.resolved ?? this.configured ?? permittedInputModes({
12877
+ allowCustomReply: true,
12878
+ baked: false,
12879
+ speechInput: true
12880
+ });
12881
+ }
12882
+ /** The reply options of the latest result, for `assertReply`. */
12883
+ setOptions(options) {
12884
+ this.latestOptions = options;
12885
+ }
12886
+ /** Throws when the input modes do not include `mode`. */
12887
+ assertMode(mode, baked) {
12888
+ if (mode === "voice" && baked) {
12889
+ throw new InputModeError(
12890
+ "Voice input is not available in a baked (offline) conversation."
12891
+ );
12892
+ }
12893
+ if (mode === "voice" && this.speechInput === false) {
12894
+ throw new InputModeError(
12895
+ "Voice input is off for this character. Its owner turns it on in the STT tab."
12896
+ );
12897
+ }
12898
+ const modes = this.modes;
12899
+ if (!modes.includes(mode)) {
12900
+ throw new InputModeError(
12901
+ `The "${mode}" input mode is off: inputModes is ${JSON.stringify(modes)}.`
12902
+ );
12903
+ }
12904
+ }
12905
+ /**
12906
+ * A typed reply needs the `text` mode; one of the current reply options
12907
+ * needs `options` or `text`. Without a configuration the server's rules
12908
+ * apply alone, as before.
12909
+ */
12910
+ assertReply(message) {
12911
+ if (!this.configured) {
12912
+ return;
12913
+ }
12914
+ const modes = this.modes;
12915
+ if (modes.includes("text")) {
12916
+ return;
12917
+ }
12918
+ const reply = message.trim();
12919
+ const isOption = this.latestOptions.some(
12920
+ (option) => option.label.trim() === reply
12921
+ );
12922
+ if (isOption && modes.includes("options")) {
12923
+ return;
12924
+ }
12925
+ throw new InputModeError(
12926
+ isOption ? `Reply options are off: inputModes is ${JSON.stringify(modes)}.` : `Typed replies are off: inputModes is ${JSON.stringify(modes)}. Send one of the reply options or use startListening().`
12927
+ );
12928
+ }
12929
+ };
12930
+
10594
12931
  // ../../clients/web-client/src/voice/push-to-talk.ts
10595
12932
  var INTERACTIVE = 'input, textarea, select, button, a[href], summary, [contenteditable]:not([contenteditable="false"]), [role="button"], [role="textbox"], [role="checkbox"], [role="switch"], [role="menuitem"], [role="tab"], [role="slider"], [role="option"]';
10596
12933
  var elementOf = function(target) {
@@ -11077,8 +13414,8 @@ var MicrophoneCapture = class {
11077
13414
  };
11078
13415
 
11079
13416
  // ../../clients/web-client/src/voice/silence-detector.ts
11080
- var MIN_SPEECH_LEVEL = 0.015;
11081
- var SPEECH_TO_NOISE = 3;
13417
+ var MIN_SPEECH_LEVEL2 = 0.015;
13418
+ var SPEECH_TO_NOISE2 = 3;
11082
13419
  var SPEECH_START_MS = 200;
11083
13420
  var SilenceDetector = class {
11084
13421
  constructor(silenceMs) {
@@ -11091,8 +13428,8 @@ var SilenceDetector = class {
11091
13428
  /** True when the speech has ended. */
11092
13429
  push(level, frameMs = SPEECH_FRAME_MS) {
11093
13430
  const threshold = Math.max(
11094
- MIN_SPEECH_LEVEL,
11095
- (this.floor ?? MIN_SPEECH_LEVEL / SPEECH_TO_NOISE) * SPEECH_TO_NOISE
13431
+ MIN_SPEECH_LEVEL2,
13432
+ (this.floor ?? MIN_SPEECH_LEVEL2 / SPEECH_TO_NOISE2) * SPEECH_TO_NOISE2
11096
13433
  );
11097
13434
  if (level >= threshold) {
11098
13435
  this.loudMs += frameMs;
@@ -11231,6 +13568,7 @@ var VoiceInput = class {
11231
13568
  send: options.send !== false,
11232
13569
  silent: options.silent === true,
11233
13570
  renderSessionId: this.deps.renderSessionId(),
13571
+ timeZone: detectClientTimeZone(),
11234
13572
  format: SPEECH_AUDIO_FORMAT
11235
13573
  })
11236
13574
  ),
@@ -11383,6 +13721,7 @@ var VoiceInput = class {
11383
13721
  if (session !== this.session || session.captureClosed) {
11384
13722
  return;
11385
13723
  }
13724
+ this.deps.activity?.pushVoiceLevel(frame.level);
11386
13725
  const audio = new Uint8Array(
11387
13726
  frame.pcm.buffer,
11388
13727
  frame.pcm.byteOffset,
@@ -11528,6 +13867,7 @@ var VoiceInput = class {
11528
13867
  setState(state) {
11529
13868
  if (this.state !== state) {
11530
13869
  this.state = state;
13870
+ this.deps.activity?.setListening(state === "listening");
11531
13871
  this.deps.onState(state);
11532
13872
  }
11533
13873
  }
@@ -11546,15 +13886,14 @@ var Character = class extends CharacterRuntime {
11546
13886
  /** The conversation whose live motion was warmed up. */
11547
13887
  liveMotionPrepared = null;
11548
13888
  authoring;
13889
+ hostedWebChat;
13890
+ memoryIdentifier;
11549
13891
  normalizeIdleOrientation;
11550
13892
  bakedAsset;
11551
13893
  animationRefreshId = 0;
11552
13894
  voice;
11553
- configuredInputModes;
11554
- resolvedInputModes = null;
11555
- /** The character's voice input switch, after the character loads. */
11556
- speechInput = null;
11557
- latestOptions = [];
13895
+ sight;
13896
+ inputs;
11558
13897
  listeningStateCallback = null;
11559
13898
  transcriptCallback = null;
11560
13899
  connectionClosedCallback = null;
@@ -11575,17 +13914,15 @@ var Character = class extends CharacterRuntime {
11575
13914
  })
11576
13915
  });
11577
13916
  this.authoring = options.authoring ?? false;
13917
+ this.hostedWebChat = options.hostedWebChat === true && !resolvedToken;
13918
+ this.memoryIdentifier = normalizeMemoryIdentifier(options.memoryIdentifier);
11578
13919
  this.normalizeIdleOrientation = options.normalizeIdleOrientation ?? false;
11579
13920
  this.bakedAsset = options.bakedConversation;
11580
13921
  this.initialAssetInfo = options.initialAssetInfo ?? null;
11581
- this.configuredInputModes = parseInputModes(options.inputModes);
11582
- if (this.configuredInputModes && this.bakedAsset && !this.authoring) {
11583
- resolveInputModes(this.configuredInputModes, {
11584
- allowCustomReply: false,
11585
- baked: true,
11586
- speechInput: false
11587
- });
11588
- }
13922
+ this.inputs = new InputModeRules(
13923
+ options.inputModes,
13924
+ Boolean(this.bakedAsset) && !this.authoring
13925
+ );
11589
13926
  this.conversationSocket = new ConversationSocket({
11590
13927
  baseUrl: this.baseUrl,
11591
13928
  apiToken: this.apiToken,
@@ -11601,17 +13938,23 @@ var Character = class extends CharacterRuntime {
11601
13938
  },
11602
13939
  handleCallbackError: this.handleCallbackError,
11603
13940
  handleSpeechEvent: (event) => this.voice.handleSocketEvent(event),
13941
+ handleSightEvent: (event) => this.sight.handleObservation(event),
11604
13942
  handleConnectionClosed: (event) => invokeSafeCallback(
11605
13943
  "onConnectionClosed",
11606
13944
  this.handleCallbackError,
11607
13945
  () => this.connectionClosedCallback?.(event)
11608
13946
  )
11609
13947
  });
13948
+ this.sight = new SightInput({
13949
+ socket: () => this.conversationSocket,
13950
+ conversationId: () => this.conversationId,
13951
+ handleCallbackError: this.handleCallbackError
13952
+ });
11610
13953
  this.voice = new VoiceInput({
11611
13954
  socket: () => this.conversationSocket,
11612
13955
  conversationId: () => this.conversationId,
11613
13956
  renderSessionId: () => this.renderSessionId,
11614
- assertVoiceAllowed: () => this.assertInputMode("voice"),
13957
+ assertVoiceAllowed: () => this.inputs.assertMode("voice", this.baked !== null),
11615
13958
  turnInFlight: () => this.liveConversation.inFlight,
11616
13959
  beginTurn: () => {
11617
13960
  if (this.conversationId) {
@@ -11639,6 +13982,7 @@ var Character = class extends CharacterRuntime {
11639
13982
  );
11640
13983
  }
11641
13984
  },
13985
+ activity: this.playerActivityTracker,
11642
13986
  onTranscript: (event) => {
11643
13987
  if (this.transcriptCallback) {
11644
13988
  invokeSafeCallback(
@@ -11655,6 +13999,7 @@ var Character = class extends CharacterRuntime {
11655
13999
  telemetry: this.telemetry,
11656
14000
  socket: () => this.conversationSocket,
11657
14001
  renderSession: () => this.renderSessionId,
14002
+ memoryIdentifier: () => this.apiToken ? this.memoryIdentifier : void 0,
11658
14003
  thinking: () => this.invokeThinkingCallback({ taskId: null, stagingId: null }),
11659
14004
  handleResult: (payload, forcePlayback, silent) => this.handleStagingResult(payload, {
11660
14005
  source: "api",
@@ -11675,12 +14020,13 @@ var Character = class extends CharacterRuntime {
11675
14020
  const progressReporter = createAssetProgressReporter(callback);
11676
14021
  const info = await this.getAssetInfo();
11677
14022
  const baked = !this.authoring && Boolean(this.bakedAsset || info.bakedConversationEnabled);
11678
- this.speechInput = info.speechInputEnabled === true;
11679
- this.resolvedInputModes = resolveInputModes(this.configuredInputModes, {
14023
+ this.variantController.configureVariants(info.variants, { remote: false });
14024
+ this.inputs.load({
11680
14025
  allowCustomReply: info.allowCustomReply !== false,
11681
14026
  baked,
11682
- speechInput: this.speechInput
14027
+ speechInput: info.speechInputEnabled === true
11683
14028
  });
14029
+ this.sight.configure({ enabled: info.sight?.enabled === true, baked });
11684
14030
  this.baked = baked ? await loadBakedConversation({
11685
14031
  baseUrl: this.baseUrl,
11686
14032
  characterId: this.characterId,
@@ -11698,8 +14044,10 @@ var Character = class extends CharacterRuntime {
11698
14044
  onProgress: progressReporter.updateResource,
11699
14045
  assetProfile: this.assetProfile
11700
14046
  }),
11701
- this.baked ? this.lazyAnimations.ensureMany(animationIds) : info.idleAnimationId ? this.lazyAnimations.ensure(info.idleAnimationId) : Promise.resolve()
14047
+ this.baked ? this.lazyAnimations.ensureMany(animationIds) : info.idleAnimationId ? this.lazyAnimations.ensure(info.idleAnimationId) : Promise.resolve(),
14048
+ this.baked ? this.variantController.preloadVariants(true) : Promise.resolve()
11702
14049
  ]);
14050
+ this.variantController.registerDefaultVrm(vrm);
11703
14051
  this.vrm = vrm;
11704
14052
  if (this.performanceRecorder.isRecording) {
11705
14053
  this.facialMixer.capturePerformanceModel();
@@ -11717,6 +14065,9 @@ var Character = class extends CharacterRuntime {
11717
14065
  });
11718
14066
  }
11719
14067
  animationController.playIdle(0);
14068
+ if (!this.baked) {
14069
+ void this.variantController.preloadVariants(false);
14070
+ }
11720
14071
  }
11721
14072
  async reloadAnimations(info) {
11722
14073
  const requestId = ++this.animationRefreshId;
@@ -11750,11 +14101,15 @@ var Character = class extends CharacterRuntime {
11750
14101
  if (info.bakedConversationEnabled || this.bakedAsset) {
11751
14102
  throw new Error("Baked conversation requires local rendering.");
11752
14103
  }
11753
- this.speechInput = info.speechInputEnabled === true;
11754
- this.resolvedInputModes = resolveInputModes(this.configuredInputModes, {
14104
+ this.variantController.configureVariants(info.variants, { remote: true });
14105
+ this.inputs.load({
11755
14106
  allowCustomReply: info.allowCustomReply !== false,
11756
14107
  baked: false,
11757
- speechInput: this.speechInput
14108
+ speechInput: info.speechInputEnabled === true
14109
+ });
14110
+ this.sight.configure({
14111
+ enabled: info.sight?.enabled === true,
14112
+ baked: false
11758
14113
  });
11759
14114
  this.animationController = new RemoteAnimationController();
11760
14115
  }
@@ -11788,7 +14143,9 @@ var Character = class extends CharacterRuntime {
11788
14143
  baseUrl: this.baseUrl,
11789
14144
  apiToken: this.apiToken,
11790
14145
  avatarId: this.characterId,
11791
- renderSessionId: this.renderSessionId
14146
+ renderSessionId: this.renderSessionId,
14147
+ hostedWebChat: this.hostedWebChat,
14148
+ memoryIdentifier: this.apiToken ? this.memoryIdentifier : void 0
11792
14149
  });
11793
14150
  if (!response.conversation?.id) {
11794
14151
  throw new Error("Failed to create conversation.");
@@ -11818,6 +14175,7 @@ var Character = class extends CharacterRuntime {
11818
14175
  }
11819
14176
  async disconnect() {
11820
14177
  this.voice.reset();
14178
+ this.sight.stop();
11821
14179
  this.thinkingState = false;
11822
14180
  if (this.animationController) {
11823
14181
  this.streamPlayback.cleanup();
@@ -11839,7 +14197,7 @@ var Character = class extends CharacterRuntime {
11839
14197
  await this.endConversation();
11840
14198
  if (options?.disposeVrm === false) {
11841
14199
  this.detach();
11842
- this.resetRuntimeAssets();
14200
+ this.resetRuntimeAssets({ disposeVrms: false });
11843
14201
  this.audioPlayer?.cleanup();
11844
14202
  this.audioPlayer = null;
11845
14203
  return;
@@ -11853,7 +14211,7 @@ var Character = class extends CharacterRuntime {
11853
14211
  maybeMessageOrOptions,
11854
14212
  maybeOptions
11855
14213
  );
11856
- this.assertReplyMode(message);
14214
+ this.inputs.assertReply(message);
11857
14215
  if (this.voice.turnPending) {
11858
14216
  throw new Error("A reply is already being generated. Please wait.");
11859
14217
  }
@@ -11901,11 +14259,7 @@ var Character = class extends CharacterRuntime {
11901
14259
  * character permits. Before `loadAssets`, the configuration or all modes.
11902
14260
  */
11903
14261
  get inputModes() {
11904
- return this.resolvedInputModes ?? this.configuredInputModes ?? permittedInputModes({
11905
- allowCustomReply: true,
11906
- baked: false,
11907
- speechInput: true
11908
- });
14262
+ return this.inputs.modes;
11909
14263
  }
11910
14264
  get listeningState() {
11911
14265
  return this.voice.listeningState;
@@ -11949,6 +14303,35 @@ var Character = class extends CharacterRuntime {
11949
14303
  callback
11950
14304
  );
11951
14305
  }
14306
+ /**
14307
+ * `'unavailable'` until the character info turns sight on (its owner's
14308
+ * Sight tab), and in a baked conversation; then `'off'`, `'starting'` or
14309
+ * `'on'`.
14310
+ */
14311
+ get sightState() {
14312
+ return this.sight.state;
14313
+ }
14314
+ /**
14315
+ * Lets the character see the player's camera (docs/sight.md). Needs a
14316
+ * conversation; the first call asks for the camera permission. Rejects
14317
+ * with a `SightError`: `SIGHT_DISABLED` when sight is unavailable,
14318
+ * `CONVERSATION_REQUIRED`, `CAMERA_PERMISSION_DENIED`, ...
14319
+ */
14320
+ startSight(options) {
14321
+ return this.sight.start(options);
14322
+ }
14323
+ /** Stops the frames. Closes the camera that `startSight` opened. */
14324
+ stopSight() {
14325
+ this.sight.stop();
14326
+ }
14327
+ /** What the character saw: each observation of the current conversation. */
14328
+ onSight(callback) {
14329
+ return this.sight.onObservation(callback);
14330
+ }
14331
+ /** The sight state. `error` is the `SightError` that stopped sight. */
14332
+ onSightState(callback) {
14333
+ return this.sight.onState(callback);
14334
+ }
11952
14335
  /**
11953
14336
  * The server closed the conversation connection: a refused or failed join,
11954
14337
  * or an API key that was rotated, edited or deleted. The next `connect`,
@@ -11995,6 +14378,15 @@ var Character = class extends CharacterRuntime {
11995
14378
  setRenderSessionId(renderSessionId) {
11996
14379
  this.renderSessionId = renderSessionId?.trim() || null;
11997
14380
  }
14381
+ setHostedWebChat(enabled) {
14382
+ this.hostedWebChat = enabled === true && !this.apiToken;
14383
+ }
14384
+ setMemoryIdentifier(memoryIdentifier) {
14385
+ this.memoryIdentifier = normalizeMemoryIdentifier(memoryIdentifier);
14386
+ }
14387
+ getMemoryIdentifier() {
14388
+ return this.memoryIdentifier;
14389
+ }
11998
14390
  onStagingResult(callback) {
11999
14391
  return bindCallbackSlot(
12000
14392
  () => this.stagingResultCallback,
@@ -12024,76 +14416,22 @@ var Character = class extends CharacterRuntime {
12024
14416
  };
12025
14417
  async handleStagingResult(payload, options) {
12026
14418
  this.thinkingState = false;
12027
- const normalized = normalizeStagingResultPayload(payload);
12028
- this.latestOptions = normalized.options;
12029
- if (options.source === "turn") {
12030
- this.liveConversation.received(normalized);
12031
- }
12032
- const force = options.forcePlayback === true || options.source === "replay";
12033
- const shouldNotify = force || shouldNotifyStagingResult(this.notifiedStagingIds, normalized.stagingId);
12034
- if (shouldNotify && this.stagingResultCallback) {
12035
- try {
12036
- await this.stagingResultCallback(normalized);
12037
- } catch (error) {
12038
- this.handleCallbackError("onStagingResult", error);
14419
+ await deliverStagingResult(payload, {
14420
+ ...options,
14421
+ notifiedStagingIds: this.notifiedStagingIds,
14422
+ hasPlayed: (stagingId) => this.streamPlayback.hasPlayed(stagingId),
14423
+ setOptions: (next) => this.inputs.setOptions(next),
14424
+ onTurnReceived: (normalized) => this.liveConversation.received(normalized),
14425
+ onStagingResult: this.stagingResultCallback,
14426
+ onCallbackError: this.handleCallbackError,
14427
+ recordEvent: (event) => this.performanceRecorder.recordEvent(event),
14428
+ playRequested: (normalized, playOptions) => {
14429
+ void this.playStaging(normalized, playOptions);
14430
+ },
14431
+ playStreamed: (normalized) => {
14432
+ void this.streamPlayback.playPayload(normalized);
12039
14433
  }
12040
- }
12041
- const shouldPlay = !this.streamPlayback.hasPlayed(normalized.stagingId) || force;
12042
- this.performanceRecorder.recordEvent({
12043
- type: "staging.result",
12044
- source: options.source,
12045
- stagingId: normalized.stagingId,
12046
- taskId: normalized.taskId ?? "",
12047
- playback: shouldPlay ? "requested" : "already-streamed"
12048
14434
  });
12049
- if (shouldPlay) {
12050
- void this.playStaging(normalized, { force, silent: options.silent });
12051
- } else {
12052
- void this.streamPlayback.playPayload(normalized);
12053
- }
12054
- }
12055
- /** Throws when the input modes do not include `mode`. */
12056
- assertInputMode(mode) {
12057
- if (mode === "voice" && this.baked) {
12058
- throw new InputModeError(
12059
- "Voice input is not available in a baked (offline) conversation."
12060
- );
12061
- }
12062
- if (mode === "voice" && this.speechInput === false) {
12063
- throw new InputModeError(
12064
- "Voice input is off for this character. Its owner turns it on in the STT tab."
12065
- );
12066
- }
12067
- const modes = this.inputModes;
12068
- if (!modes.includes(mode)) {
12069
- throw new InputModeError(
12070
- `The "${mode}" input mode is off: inputModes is ${JSON.stringify(modes)}.`
12071
- );
12072
- }
12073
- }
12074
- /**
12075
- * A typed reply needs the `text` mode; one of the current reply options
12076
- * needs `options` or `text`. Without a configuration the server's rules
12077
- * apply alone, as before.
12078
- */
12079
- assertReplyMode(message) {
12080
- if (!this.configuredInputModes) {
12081
- return;
12082
- }
12083
- const modes = this.inputModes;
12084
- if (modes.includes("text")) {
12085
- return;
12086
- }
12087
- const reply = message.trim();
12088
- const isOption = this.latestOptions.some(
12089
- (option) => option.label.trim() === reply
12090
- );
12091
- if (isOption && modes.includes("options")) {
12092
- return;
12093
- }
12094
- throw new InputModeError(
12095
- isOption ? `Reply options are off: inputModes is ${JSON.stringify(modes)}.` : `Typed replies are off: inputModes is ${JSON.stringify(modes)}. Send one of the reply options or use startListening().`
12096
- );
12097
14435
  }
12098
14436
  invokeThinkingCallback(payload) {
12099
14437
  this.thinkingState = true;
@@ -12604,6 +14942,7 @@ var mountNpcify = async function(options) {
12604
14942
  camera: null,
12605
14943
  renderer: null,
12606
14944
  controls: null,
14945
+ setMemoryIdentifier: (memoryIdentifier) => npcify.setMemoryIdentifier(memoryIdentifier),
12607
14946
  dispose: async () => {
12608
14947
  remote.dispose();
12609
14948
  await npcify.dispose();
@@ -12617,6 +14956,7 @@ var mountNpcify = async function(options) {
12617
14956
  let context2;
12618
14957
  let session;
12619
14958
  let observer;
14959
+ let unsubscribeVrm;
12620
14960
  let frame = 0;
12621
14961
  let disposed = false;
12622
14962
  const dispose = async function() {
@@ -12626,10 +14966,11 @@ var mountNpcify = async function(options) {
12626
14966
  disposed = true;
12627
14967
  cancelAnimationFrame(frame);
12628
14968
  observer?.disconnect();
12629
- await npcify.dispose({ disposeVrm: false });
14969
+ unsubscribeVrm?.();
12630
14970
  if (context2 && session) {
12631
- detachVrm(context2, session, { dispose: true });
14971
+ detachVrm(context2, session, { dispose: false });
12632
14972
  }
14973
+ await npcify.dispose();
12633
14974
  if (context2) {
12634
14975
  disposeSceneContext(context2);
12635
14976
  }
@@ -12641,6 +14982,16 @@ var mountNpcify = async function(options) {
12641
14982
  }
12642
14983
  context2 = createSceneContext({ canvas: options.canvas });
12643
14984
  session = attachVrm(context2, npcify.vrm);
14985
+ unsubscribeVrm = npcify.on("vrmChange", (nextVrm) => {
14986
+ if (context2) {
14987
+ setCameraCollisionBackend(
14988
+ context2,
14989
+ new HumanoidCapsuleCollisionBackend(
14990
+ (name) => nextVrm.humanoid.getRawBoneNode(name)
14991
+ )
14992
+ );
14993
+ }
14994
+ });
12644
14995
  applyCameraPreset(context2, "full_body");
12645
14996
  npcify.setGazeTarget(context2.camera);
12646
14997
  const resize = () => {
@@ -12674,6 +15025,7 @@ var mountNpcify = async function(options) {
12674
15025
  camera: context2.camera,
12675
15026
  renderer: context2.renderer,
12676
15027
  controls: context2.controls,
15028
+ setMemoryIdentifier: (memoryIdentifier) => npcify.setMemoryIdentifier(memoryIdentifier),
12677
15029
  dispose
12678
15030
  };
12679
15031
  } catch (error) {
@@ -12692,25 +15044,32 @@ export {
12692
15044
  CharacterRuntime,
12693
15045
  CharacterSceneRig,
12694
15046
  ConversationConnectionError,
15047
+ DEFAULT_VARIANT_KEY,
12695
15048
  FacialPerformanceLayer,
12696
15049
  FacialPerformanceMixer,
12697
15050
  INPUT_MODES,
12698
15051
  InputModeError,
12699
15052
  LiveMotionPlayer,
12700
15053
  MicrophoneError,
15054
+ NUDE_VARIANT_KEY,
12701
15055
  Npcify,
12702
15056
  Character as NpcifyCharacter,
12703
15057
  PERFORMANCE_FILE_EXTENSION,
12704
15058
  PERFORMANCE_MEDIA_TYPE,
12705
15059
  PerformanceRecordingSchemaVersion,
12706
15060
  RemoteAnimationController,
15061
+ SightError,
12707
15062
  SpeechInputError,
12708
15063
  analyzePerformanceRecording,
12709
15064
  bindPushToTalk,
12710
15065
  createCharacter,
12711
15066
  decodePerformance,
12712
15067
  encodePerformance,
15068
+ fetchVariantVrmDownloadUrl,
15069
+ fetchVrmDownloadUrl,
12713
15070
  loadBakedConversation,
12714
15071
  loadCharacter,
15072
+ loadVariantVrm,
15073
+ loadVrm,
12715
15074
  mountNpcify
12716
15075
  };