@npcify/web 0.4.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +75 -0
- package/dist/index.d.ts +463 -57
- package/dist/index.js +2757 -398
- package/examples/index.html +1 -0
- package/examples/main.js +22 -0
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -34,6 +34,33 @@ var isValidMotionFramesEvent = function(event) {
|
|
|
34
34
|
// ../ts-shared/src/llm.ts
|
|
35
35
|
var STAGING_PROTOCOL_VERSION = 1;
|
|
36
36
|
|
|
37
|
+
// ../ts-shared/src/variants.ts
|
|
38
|
+
var DEFAULT_VARIANT_KEY = "default";
|
|
39
|
+
var NUDE_VARIANT_KEY = "nude";
|
|
40
|
+
var VARIANT_KEY_PATTERN = /^[a-z][a-z0-9_-]{0,39}$/u;
|
|
41
|
+
var isValidVariantKey = function(value) {
|
|
42
|
+
return typeof value === "string" && VARIANT_KEY_PATTERN.test(value);
|
|
43
|
+
};
|
|
44
|
+
var STAGING_TOOL_RESULT_STATUSES = [
|
|
45
|
+
"applied",
|
|
46
|
+
"noop",
|
|
47
|
+
"rejected"
|
|
48
|
+
];
|
|
49
|
+
var isStagingToolCall = function(value) {
|
|
50
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
51
|
+
return false;
|
|
52
|
+
}
|
|
53
|
+
const candidate = value;
|
|
54
|
+
if (typeof candidate.id !== "string" || !candidate.id.trim() || candidate.name !== "switch_variant" || typeof candidate.timestamp !== "number" || !Number.isFinite(candidate.timestamp) || candidate.timestamp < 0 || !candidate.arguments || typeof candidate.arguments !== "object" || Array.isArray(candidate.arguments) || !candidate.result || typeof candidate.result !== "object" || Array.isArray(candidate.result)) {
|
|
55
|
+
return false;
|
|
56
|
+
}
|
|
57
|
+
const args = candidate.arguments;
|
|
58
|
+
const result = candidate.result;
|
|
59
|
+
return isValidVariantKey(args.variant) && typeof result.ok === "boolean" && STAGING_TOOL_RESULT_STATUSES.includes(
|
|
60
|
+
String(result.status)
|
|
61
|
+
) && isValidVariantKey(result.previousVariant) && isValidVariantKey(result.activeVariant) && (result.error === void 0 || typeof result.error === "string");
|
|
62
|
+
};
|
|
63
|
+
|
|
37
64
|
// ../ts-shared/src/server-rendering.ts
|
|
38
65
|
var SERVER_RENDER_PROTOCOL_VERSION = 3;
|
|
39
66
|
var MAXIMUM_PLAYBACK_DURATION_SECONDS = 10 * 60;
|
|
@@ -42,7 +69,465 @@ var MAXIMUM_PLAYBACK_DURATION_SECONDS = 10 * 60;
|
|
|
42
69
|
var TTS_PROVIDERS = ["11labs", "gemini"];
|
|
43
70
|
var DEFAULT_TTS_PROVIDER = TTS_PROVIDERS[0];
|
|
44
71
|
|
|
72
|
+
// ../ts-shared/src/llm-providers/catalog.ts
|
|
73
|
+
var record = function(value) {
|
|
74
|
+
return value && typeof value === "object" && !Array.isArray(value) ? value : {};
|
|
75
|
+
};
|
|
76
|
+
var rate = function(value) {
|
|
77
|
+
if (typeof value !== "number" && typeof value !== "string") {
|
|
78
|
+
return null;
|
|
79
|
+
}
|
|
80
|
+
if (value === "") {
|
|
81
|
+
return null;
|
|
82
|
+
}
|
|
83
|
+
const result = Number(value);
|
|
84
|
+
return Number.isFinite(result) && result >= 0 ? result : null;
|
|
85
|
+
};
|
|
86
|
+
var catalogRows = function(payload) {
|
|
87
|
+
if (!Array.isArray(payload)) {
|
|
88
|
+
throw new Error("Provider returned an invalid model catalog");
|
|
89
|
+
}
|
|
90
|
+
return payload.map(record);
|
|
91
|
+
};
|
|
92
|
+
var parseCompatibleModels = function(payload) {
|
|
93
|
+
return catalogRows(record(payload).data).flatMap((row) => {
|
|
94
|
+
const pricing = record(row.pricing);
|
|
95
|
+
const input = rate(pricing.prompt);
|
|
96
|
+
const output = rate(pricing.completion);
|
|
97
|
+
const cached = rate(pricing.input_cache_read);
|
|
98
|
+
const write = rate(pricing.input_cache_write);
|
|
99
|
+
const modalities = record(row.architecture).output_modalities;
|
|
100
|
+
const parameters = row.supported_parameters;
|
|
101
|
+
if (typeof row.id !== "string" || !row.id || row.deprecated || input === null || output === null || Array.isArray(modalities) && (modalities.length !== 1 || modalities[0] !== "text") || Array.isArray(pricing.overrides) && pricing.overrides.length > 0 || Array.isArray(parameters) && !parameters.includes("structured_outputs") && !parameters.includes("response_format") || (rate(pricing.request) ?? 0) > 0) {
|
|
102
|
+
return [];
|
|
103
|
+
}
|
|
104
|
+
return [
|
|
105
|
+
{
|
|
106
|
+
id: row.id,
|
|
107
|
+
name: typeof row.name === "string" ? row.name : row.id,
|
|
108
|
+
inputUsd: input * 1e6,
|
|
109
|
+
outputUsd: output * 1e6,
|
|
110
|
+
cachedInputUsd: (cached ?? input) * 1e6,
|
|
111
|
+
...write !== null ? { cacheWriteUsd: write * 1e6 } : {},
|
|
112
|
+
contextLength: rate(row.max_tokens ?? row.context_length)
|
|
113
|
+
}
|
|
114
|
+
];
|
|
115
|
+
});
|
|
116
|
+
};
|
|
117
|
+
|
|
118
|
+
// ../ts-shared/src/llm-providers/usage.ts
|
|
119
|
+
var validUsd = function(value) {
|
|
120
|
+
return typeof value === "number" && Number.isFinite(value) && value >= 0;
|
|
121
|
+
};
|
|
122
|
+
var validateUsageCounts = function(usage) {
|
|
123
|
+
const counts = [
|
|
124
|
+
usage.prompt_tokens,
|
|
125
|
+
usage.completion_tokens,
|
|
126
|
+
usage.total_tokens,
|
|
127
|
+
// Providers can explicitly return null for unavailable cache counters.
|
|
128
|
+
usage.prompt_tokens_details?.cached_tokens ?? 0,
|
|
129
|
+
usage.prompt_tokens_details?.cache_write_tokens ?? 0
|
|
130
|
+
];
|
|
131
|
+
if (counts.some(
|
|
132
|
+
(value) => value !== void 0 && (!Number.isSafeInteger(value) || value < 0)
|
|
133
|
+
)) {
|
|
134
|
+
throw new Error("Provider returned invalid token usage");
|
|
135
|
+
}
|
|
136
|
+
const cached = usage.prompt_tokens_details?.cached_tokens ?? 0;
|
|
137
|
+
const written = usage.prompt_tokens_details?.cache_write_tokens ?? 0;
|
|
138
|
+
if (usage.prompt_tokens !== void 0 && cached + written > usage.prompt_tokens) {
|
|
139
|
+
throw new Error("Provider returned invalid cache usage");
|
|
140
|
+
}
|
|
141
|
+
};
|
|
142
|
+
var quoteTokenUsage = function(price, usage) {
|
|
143
|
+
validateUsageCounts(usage);
|
|
144
|
+
const input = usage.prompt_tokens;
|
|
145
|
+
const output = usage.completion_tokens;
|
|
146
|
+
if (input === void 0 || output === void 0) {
|
|
147
|
+
throw new Error("Provider did not report billable token usage");
|
|
148
|
+
}
|
|
149
|
+
const cached = usage.prompt_tokens_details?.cached_tokens ?? 0;
|
|
150
|
+
const written = usage.prompt_tokens_details?.cache_write_tokens ?? 0;
|
|
151
|
+
const writeRate = price.cacheWriteUsd ?? price.inputUsd;
|
|
152
|
+
if (![price.inputUsd, price.outputUsd, price.cachedInputUsd, writeRate].every(
|
|
153
|
+
validUsd
|
|
154
|
+
)) {
|
|
155
|
+
throw new Error("Provider returned invalid token prices");
|
|
156
|
+
}
|
|
157
|
+
return {
|
|
158
|
+
usd: ((input - cached - written) * price.inputUsd + cached * price.cachedInputUsd + written * writeRate + output * price.outputUsd) / 1e6,
|
|
159
|
+
source: "token-rates"
|
|
160
|
+
};
|
|
161
|
+
};
|
|
162
|
+
|
|
163
|
+
// ../ts-shared/src/llm-providers/baseten.ts
|
|
164
|
+
var baseten = {
|
|
165
|
+
name: "Baseten",
|
|
166
|
+
baseUrl: "https://inference.baseten.co/v1",
|
|
167
|
+
modelsUrl: "https://inference.baseten.co/v1/models",
|
|
168
|
+
billingMode: "token-rates",
|
|
169
|
+
parseModels: parseCompatibleModels,
|
|
170
|
+
requestOptions: () => ({}),
|
|
171
|
+
quoteUsage: quoteTokenUsage
|
|
172
|
+
};
|
|
173
|
+
|
|
174
|
+
// ../ts-shared/src/llm-providers/deepinfra.ts
|
|
175
|
+
var deepinfra = {
|
|
176
|
+
name: "DeepInfra",
|
|
177
|
+
keyCheckUrl: "https://api.deepinfra.com/v1/me",
|
|
178
|
+
baseUrl: "https://api.deepinfra.com/v1/openai",
|
|
179
|
+
modelsUrl: "https://api.deepinfra.com/models/list",
|
|
180
|
+
billingMode: "token-rates",
|
|
181
|
+
parseModels: (payload) => catalogRows(payload).flatMap((row) => {
|
|
182
|
+
const tags = Array.isArray(row.tags) ? row.tags : [];
|
|
183
|
+
const pricing = record(row.pricing);
|
|
184
|
+
const input = rate(pricing.cents_per_input_token);
|
|
185
|
+
const output = rate(pricing.cents_per_output_token);
|
|
186
|
+
if (typeof row.model_name !== "string" || !row.model_name || row.deprecated || row.type !== "text-generation" || !tags.includes("openai") || !tags.includes("structured-output") || pricing.type !== "tokens" || input === null || output === null) {
|
|
187
|
+
return [];
|
|
188
|
+
}
|
|
189
|
+
return [
|
|
190
|
+
{
|
|
191
|
+
id: row.model_name,
|
|
192
|
+
name: typeof row.name === "string" ? row.name : row.model_name,
|
|
193
|
+
inputUsd: input / 100 * 1e6,
|
|
194
|
+
outputUsd: output / 100 * 1e6,
|
|
195
|
+
cachedInputUsd: input / 100 * (rate(pricing.rate_per_input_token_cached) ?? 1) * 1e6,
|
|
196
|
+
contextLength: rate(row.max_tokens ?? row.context_length),
|
|
197
|
+
canDisableReasoning: tags.includes("can-disable-reasoning")
|
|
198
|
+
}
|
|
199
|
+
];
|
|
200
|
+
}),
|
|
201
|
+
requestOptions: ({ model, conversationId, legacyPriority }) => ({
|
|
202
|
+
...conversationId ? { prompt_cache_key: conversationId } : {},
|
|
203
|
+
...!model || model.canDisableReasoning ? { reasoning_effort: "none" } : {},
|
|
204
|
+
...legacyPriority ? { service_tier: "priority" } : {}
|
|
205
|
+
}),
|
|
206
|
+
quoteUsage: quoteTokenUsage
|
|
207
|
+
};
|
|
208
|
+
|
|
209
|
+
// ../ts-shared/src/llm-providers/gemini.ts
|
|
210
|
+
var GEMINI_MODELS = {
|
|
211
|
+
"gemini-3.8-flash": {
|
|
212
|
+
id: "gemini-3.8-flash",
|
|
213
|
+
name: "Gemini 3.8 Flash",
|
|
214
|
+
inputUsd: 0.75,
|
|
215
|
+
outputUsd: 3.75,
|
|
216
|
+
cachedInputUsd: 0.075,
|
|
217
|
+
contextLength: 1048576
|
|
218
|
+
}
|
|
219
|
+
};
|
|
220
|
+
var gemini = {
|
|
221
|
+
name: "Google Gemini",
|
|
222
|
+
baseUrl: "https://generativelanguage.googleapis.com/v1beta/openai",
|
|
223
|
+
modelsUrl: "https://generativelanguage.googleapis.com/v1beta/openai/models",
|
|
224
|
+
billingMode: "token-rates",
|
|
225
|
+
parseModels: (payload) => {
|
|
226
|
+
const seen = /* @__PURE__ */ new Set();
|
|
227
|
+
return catalogRows(record(payload).data).flatMap((row) => {
|
|
228
|
+
const rawId = typeof row.id === "string" ? row.id.trim() : "";
|
|
229
|
+
const id = rawId.replace(/^models\//, "");
|
|
230
|
+
const model = Object.hasOwn(GEMINI_MODELS, id) ? GEMINI_MODELS[id] : void 0;
|
|
231
|
+
if (!model || row.deprecated || seen.has(id)) {
|
|
232
|
+
return [];
|
|
233
|
+
}
|
|
234
|
+
seen.add(id);
|
|
235
|
+
return [{ ...model }];
|
|
236
|
+
});
|
|
237
|
+
},
|
|
238
|
+
requestOptions: () => ({ reasoning_effort: "low" }),
|
|
239
|
+
quoteUsage: (price, usage) => {
|
|
240
|
+
validateUsageCounts(usage);
|
|
241
|
+
if (usage.prompt_tokens !== void 0 && usage.completion_tokens !== void 0 && usage.total_tokens !== void 0 && usage.total_tokens < usage.prompt_tokens + usage.completion_tokens) {
|
|
242
|
+
throw new Error("Provider returned invalid token usage");
|
|
243
|
+
}
|
|
244
|
+
const outputTokens = usage.total_tokens !== void 0 && usage.prompt_tokens !== void 0 ? usage.total_tokens - usage.prompt_tokens : usage.completion_tokens;
|
|
245
|
+
return quoteTokenUsage(price, {
|
|
246
|
+
...usage,
|
|
247
|
+
completion_tokens: outputTokens
|
|
248
|
+
});
|
|
249
|
+
}
|
|
250
|
+
};
|
|
251
|
+
|
|
252
|
+
// ../ts-shared/src/llm-providers/openrouter.ts
|
|
253
|
+
var openrouter = {
|
|
254
|
+
name: "OpenRouter",
|
|
255
|
+
keyCheckUrl: "https://openrouter.ai/api/v1/key",
|
|
256
|
+
baseUrl: "https://openrouter.ai/api/v1",
|
|
257
|
+
modelsUrl: "https://openrouter.ai/api/v1/models",
|
|
258
|
+
billingMode: "reported-cost",
|
|
259
|
+
parseModels: parseCompatibleModels,
|
|
260
|
+
requestOptions: ({ conversationId }) => conversationId ? { session_id: conversationId } : {},
|
|
261
|
+
quoteUsage: (_price, usage) => {
|
|
262
|
+
validateUsageCounts(usage);
|
|
263
|
+
if (!validUsd(usage.cost)) {
|
|
264
|
+
throw new Error("OpenRouter did not report a valid account cost");
|
|
265
|
+
}
|
|
266
|
+
return { usd: usage.cost, source: "provider-reported" };
|
|
267
|
+
},
|
|
268
|
+
needsUsageRecovery: (usage) => usage.cost === void 0 || usage.cost === null,
|
|
269
|
+
recoverUsage: async (usage, apiKey) => {
|
|
270
|
+
const id = usage.billing?.generationId;
|
|
271
|
+
if (!id) {
|
|
272
|
+
throw new Error("OpenRouter generation ID is missing");
|
|
273
|
+
}
|
|
274
|
+
const url = new URL("https://openrouter.ai/api/v1/generation");
|
|
275
|
+
url.searchParams.set("id", id);
|
|
276
|
+
const response = await fetch(url, {
|
|
277
|
+
headers: { Authorization: `Bearer ${apiKey}` },
|
|
278
|
+
signal: AbortSignal.timeout(15e3),
|
|
279
|
+
redirect: "error"
|
|
280
|
+
});
|
|
281
|
+
if (!response.ok) {
|
|
282
|
+
throw new Error(`OpenRouter usage lookup failed (${response.status})`);
|
|
283
|
+
}
|
|
284
|
+
const data = record(record(await response.json()).data);
|
|
285
|
+
const finished = typeof data.finish_reason === "string" && data.finish_reason.length > 0 || data.cancelled === true;
|
|
286
|
+
if (data.id !== id || !validUsd(data.total_cost) || !finished) {
|
|
287
|
+
throw new Error("OpenRouter generation cost is unavailable");
|
|
288
|
+
}
|
|
289
|
+
return {
|
|
290
|
+
...usage,
|
|
291
|
+
cost: data.total_cost,
|
|
292
|
+
prompt_tokens_details: {
|
|
293
|
+
...usage.prompt_tokens_details,
|
|
294
|
+
...Number.isSafeInteger(data.native_tokens_cached) && Number(data.native_tokens_cached) >= 0 ? { cached_tokens: Number(data.native_tokens_cached) } : {}
|
|
295
|
+
},
|
|
296
|
+
...Number.isSafeInteger(data.native_tokens_prompt) && Number(data.native_tokens_prompt) >= 0 ? { prompt_tokens: Number(data.native_tokens_prompt) } : {},
|
|
297
|
+
...Number.isSafeInteger(data.native_tokens_completion) && Number(data.native_tokens_completion) >= 0 ? { completion_tokens: Number(data.native_tokens_completion) } : {}
|
|
298
|
+
};
|
|
299
|
+
}
|
|
300
|
+
};
|
|
301
|
+
|
|
302
|
+
// ../ts-shared/src/llm-providers.ts
|
|
303
|
+
var LLM_PROVIDERS = {
|
|
304
|
+
deepinfra,
|
|
305
|
+
baseten,
|
|
306
|
+
openrouter,
|
|
307
|
+
gemini
|
|
308
|
+
};
|
|
309
|
+
|
|
310
|
+
// ../ts-shared/src/ai-providers.ts
|
|
311
|
+
var staticModel = function(id, name, capabilities, priceSummary) {
|
|
312
|
+
return Object.freeze({
|
|
313
|
+
id,
|
|
314
|
+
name,
|
|
315
|
+
capabilities: [...capabilities],
|
|
316
|
+
...priceSummary ? { priceSummary } : {}
|
|
317
|
+
});
|
|
318
|
+
};
|
|
319
|
+
var AI_PROVIDERS = Object.freeze({
|
|
320
|
+
deepinfra: {
|
|
321
|
+
name: "DeepInfra",
|
|
322
|
+
capabilities: ["llm"],
|
|
323
|
+
defaultBaseUrl: LLM_PROVIDERS.deepinfra.baseUrl,
|
|
324
|
+
supportsCreatorChat: true,
|
|
325
|
+
staticModels: []
|
|
326
|
+
},
|
|
327
|
+
openrouter: {
|
|
328
|
+
name: "OpenRouter",
|
|
329
|
+
capabilities: ["llm"],
|
|
330
|
+
defaultBaseUrl: LLM_PROVIDERS.openrouter.baseUrl,
|
|
331
|
+
supportsCreatorChat: true,
|
|
332
|
+
staticModels: []
|
|
333
|
+
},
|
|
334
|
+
baseten: {
|
|
335
|
+
name: "Baseten",
|
|
336
|
+
capabilities: ["llm"],
|
|
337
|
+
defaultBaseUrl: LLM_PROVIDERS.baseten.baseUrl,
|
|
338
|
+
supportsCreatorChat: true,
|
|
339
|
+
staticModels: []
|
|
340
|
+
},
|
|
341
|
+
gemini: {
|
|
342
|
+
name: "Google Gemini",
|
|
343
|
+
capabilities: ["llm", "tts", "stt", "vision"],
|
|
344
|
+
defaultBaseUrl: "https://generativelanguage.googleapis.com/v1beta",
|
|
345
|
+
supportsCreatorChat: true,
|
|
346
|
+
staticModels: [
|
|
347
|
+
staticModel("gemini-3.8-flash-tts", "Gemini 3.8 Flash TTS", ["tts"]),
|
|
348
|
+
staticModel(
|
|
349
|
+
"gemini-3.8-flash-lite-tts",
|
|
350
|
+
"Gemini 3.8 Flash Lite TTS",
|
|
351
|
+
["tts"]
|
|
352
|
+
),
|
|
353
|
+
staticModel(
|
|
354
|
+
"gemini-3.5-transcribe-live",
|
|
355
|
+
"Gemini 3.5 Transcribe Live",
|
|
356
|
+
["stt"],
|
|
357
|
+
"$3.50 input / $21 output per 1M tokens"
|
|
358
|
+
),
|
|
359
|
+
staticModel(
|
|
360
|
+
"gemini-3.5-transcribe",
|
|
361
|
+
"Gemini 3.5 Transcribe",
|
|
362
|
+
["stt"],
|
|
363
|
+
"$2 input / $12 output per 1M tokens"
|
|
364
|
+
),
|
|
365
|
+
staticModel(
|
|
366
|
+
"gemini-3.5-flash-lite",
|
|
367
|
+
"Gemini 3.5 Flash Lite",
|
|
368
|
+
["vision"],
|
|
369
|
+
"$0.30 input / $2.50 output per 1M tokens"
|
|
370
|
+
)
|
|
371
|
+
]
|
|
372
|
+
},
|
|
373
|
+
openai: {
|
|
374
|
+
name: "OpenAI",
|
|
375
|
+
capabilities: ["llm", "stt", "vision", "image"],
|
|
376
|
+
defaultBaseUrl: "https://api.openai.com/v1",
|
|
377
|
+
supportsCreatorChat: false,
|
|
378
|
+
forbidOpenRouterBaseUrl: true,
|
|
379
|
+
staticModels: [
|
|
380
|
+
staticModel(
|
|
381
|
+
"gpt-6.1-sol",
|
|
382
|
+
"GPT-6.1 Sol",
|
|
383
|
+
["llm"],
|
|
384
|
+
"$2 input / $10 output per 1M tokens"
|
|
385
|
+
),
|
|
386
|
+
staticModel(
|
|
387
|
+
"gpt-6-luna",
|
|
388
|
+
"GPT-6 Luna",
|
|
389
|
+
["llm", "vision"],
|
|
390
|
+
"$0.10 input / $0.50 output per 1M tokens"
|
|
391
|
+
),
|
|
392
|
+
staticModel(
|
|
393
|
+
"gpt-image-2.5-flare",
|
|
394
|
+
"GPT Image 2.5 Flare",
|
|
395
|
+
["image"],
|
|
396
|
+
"$5 text / $8 image input / $30 output per 1M tokens"
|
|
397
|
+
),
|
|
398
|
+
staticModel(
|
|
399
|
+
"gpt-image-2.5-sunburst",
|
|
400
|
+
"GPT Image 2.5 Sunburst",
|
|
401
|
+
["image"],
|
|
402
|
+
"$5 text / $8 image input / $30 output per 1M tokens"
|
|
403
|
+
),
|
|
404
|
+
staticModel(
|
|
405
|
+
"gpt-transcribe",
|
|
406
|
+
"GPT Transcribe",
|
|
407
|
+
["stt"],
|
|
408
|
+
"$0.0045/min"
|
|
409
|
+
),
|
|
410
|
+
staticModel(
|
|
411
|
+
"gpt-live-transcribe",
|
|
412
|
+
"GPT Live Transcribe",
|
|
413
|
+
["stt"],
|
|
414
|
+
"$0.017/min"
|
|
415
|
+
)
|
|
416
|
+
]
|
|
417
|
+
},
|
|
418
|
+
anthropic: {
|
|
419
|
+
name: "Anthropic",
|
|
420
|
+
capabilities: ["llm"],
|
|
421
|
+
defaultBaseUrl: "https://api.anthropic.com",
|
|
422
|
+
supportsCreatorChat: false,
|
|
423
|
+
staticModels: [
|
|
424
|
+
staticModel(
|
|
425
|
+
"claude-opus-5-5",
|
|
426
|
+
"Claude Opus 5.5",
|
|
427
|
+
["llm"],
|
|
428
|
+
"$4 input / $20 output per 1M tokens"
|
|
429
|
+
),
|
|
430
|
+
staticModel(
|
|
431
|
+
"claude-sonnet-5-5",
|
|
432
|
+
"Claude Sonnet 5.5",
|
|
433
|
+
["llm"],
|
|
434
|
+
"$2 input / $10 output per 1M tokens"
|
|
435
|
+
),
|
|
436
|
+
staticModel(
|
|
437
|
+
"claude-haiku-5-5",
|
|
438
|
+
"Claude Haiku 5.5",
|
|
439
|
+
["llm"],
|
|
440
|
+
"$0.10 input / $0.50 output per 1M tokens"
|
|
441
|
+
)
|
|
442
|
+
]
|
|
443
|
+
},
|
|
444
|
+
elevenlabs: {
|
|
445
|
+
name: "ElevenLabs",
|
|
446
|
+
capabilities: ["tts", "stt"],
|
|
447
|
+
defaultBaseUrl: "https://api.elevenlabs.io/v1",
|
|
448
|
+
supportsCreatorChat: false,
|
|
449
|
+
staticModels: [
|
|
450
|
+
staticModel(
|
|
451
|
+
"eleven_v3_conversational",
|
|
452
|
+
"Eleven v3 Conversational",
|
|
453
|
+
["tts"]
|
|
454
|
+
),
|
|
455
|
+
staticModel("eleven_v3", "Eleven v3", ["tts"]),
|
|
456
|
+
staticModel(
|
|
457
|
+
"scribe_v2_realtime",
|
|
458
|
+
"Scribe v2 Realtime",
|
|
459
|
+
["stt"],
|
|
460
|
+
"$0.39/hr"
|
|
461
|
+
),
|
|
462
|
+
staticModel("scribe_v2", "Scribe v2", ["stt"], "$0.22/hr")
|
|
463
|
+
]
|
|
464
|
+
},
|
|
465
|
+
tripo: {
|
|
466
|
+
name: "Tripo",
|
|
467
|
+
capabilities: ["3d"],
|
|
468
|
+
defaultBaseUrl: "https://openapi.tripo3d.ai/v3",
|
|
469
|
+
supportsCreatorChat: false,
|
|
470
|
+
staticModels: [
|
|
471
|
+
staticModel(
|
|
472
|
+
"P2-20260801",
|
|
473
|
+
"Tripo P2",
|
|
474
|
+
["3d"],
|
|
475
|
+
"$0.01/credit (P2 4K)"
|
|
476
|
+
),
|
|
477
|
+
staticModel(
|
|
478
|
+
"v1.0-20240301",
|
|
479
|
+
"Tripo Rig v1.0",
|
|
480
|
+
["3d"],
|
|
481
|
+
"$0.01/credit (25 credits/rig)"
|
|
482
|
+
)
|
|
483
|
+
]
|
|
484
|
+
},
|
|
485
|
+
meshy: {
|
|
486
|
+
name: "Meshy",
|
|
487
|
+
capabilities: ["3d"],
|
|
488
|
+
defaultBaseUrl: "https://api.meshy.ai",
|
|
489
|
+
supportsCreatorChat: false,
|
|
490
|
+
staticModels: [
|
|
491
|
+
staticModel(
|
|
492
|
+
"meshy-7.1",
|
|
493
|
+
"Meshy 7.1",
|
|
494
|
+
["3d"],
|
|
495
|
+
"$0.02/credit (20-30 credits/model)"
|
|
496
|
+
),
|
|
497
|
+
staticModel(
|
|
498
|
+
"meshy-6",
|
|
499
|
+
"Meshy 6",
|
|
500
|
+
["3d"],
|
|
501
|
+
"$0.02/credit (20-30 credits/model)"
|
|
502
|
+
)
|
|
503
|
+
]
|
|
504
|
+
},
|
|
505
|
+
unirig: {
|
|
506
|
+
name: "UniRig",
|
|
507
|
+
capabilities: ["3d"],
|
|
508
|
+
defaultBaseUrl: "https://modal.run",
|
|
509
|
+
supportsCreatorChat: false,
|
|
510
|
+
staticModels: [
|
|
511
|
+
staticModel(
|
|
512
|
+
"unirig-v1",
|
|
513
|
+
"UniRig (Modal GPU)",
|
|
514
|
+
["3d"],
|
|
515
|
+
"Modal A10 GPU billed per second"
|
|
516
|
+
)
|
|
517
|
+
]
|
|
518
|
+
}
|
|
519
|
+
});
|
|
520
|
+
|
|
45
521
|
// ../ts-shared/src/ai-settings.ts
|
|
522
|
+
var AI_VENDOR_NAMES = {
|
|
523
|
+
openai: AI_PROVIDERS.openai.name,
|
|
524
|
+
tripo: AI_PROVIDERS.tripo.name,
|
|
525
|
+
meshy: AI_PROVIDERS.meshy.name,
|
|
526
|
+
unirig: AI_PROVIDERS.unirig.name,
|
|
527
|
+
elevenlabs: AI_PROVIDERS.elevenlabs.name,
|
|
528
|
+
gemini: AI_PROVIDERS.gemini.name,
|
|
529
|
+
anthropic: AI_PROVIDERS.anthropic.name
|
|
530
|
+
};
|
|
46
531
|
var GPT_IMAGE_MODELS = [
|
|
47
532
|
"gpt-image-2.5-flare",
|
|
48
533
|
"gpt-image-2.5-sunburst"
|
|
@@ -50,62 +535,124 @@ var GPT_IMAGE_MODELS = [
|
|
|
50
535
|
var AI_ROLES = {
|
|
51
536
|
character_draft: {
|
|
52
537
|
name: "Character draft",
|
|
53
|
-
|
|
54
|
-
|
|
538
|
+
capability: "llm",
|
|
539
|
+
chatLlm: true,
|
|
55
540
|
options: ["temperature"]
|
|
56
541
|
},
|
|
57
542
|
motion_planner: {
|
|
58
543
|
name: "Motion planner",
|
|
59
|
-
|
|
60
|
-
|
|
544
|
+
capability: "llm",
|
|
545
|
+
chatLlm: true,
|
|
61
546
|
options: []
|
|
62
547
|
},
|
|
63
|
-
// Not charged.
|
|
64
548
|
prompt_check: {
|
|
65
549
|
name: "Prompt check",
|
|
66
|
-
|
|
67
|
-
|
|
550
|
+
capability: "llm",
|
|
551
|
+
providers: ["openai"],
|
|
552
|
+
allowedModels: ["gpt-6-luna"],
|
|
68
553
|
options: []
|
|
69
554
|
},
|
|
70
|
-
// Priced by OPENAI_IMAGE_PRICES of the server (flare and sunburst alike).
|
|
71
555
|
design_sheet_image: {
|
|
72
556
|
name: "Design sheet image",
|
|
73
|
-
|
|
74
|
-
|
|
557
|
+
capability: "image",
|
|
558
|
+
providers: ["openai"],
|
|
559
|
+
allowedModels: GPT_IMAGE_MODELS,
|
|
75
560
|
options: []
|
|
76
561
|
},
|
|
77
|
-
// Not charged per call.
|
|
78
562
|
tts: {
|
|
79
563
|
name: "ElevenLabs text to speech",
|
|
80
|
-
|
|
81
|
-
|
|
564
|
+
capability: "tts",
|
|
565
|
+
providers: ["elevenlabs"],
|
|
566
|
+
allowedModels: ["eleven_v3_conversational", "eleven_v3"],
|
|
82
567
|
options: []
|
|
83
568
|
},
|
|
84
|
-
// Not charged per call.
|
|
85
569
|
gemini_tts: {
|
|
86
570
|
name: "Gemini text to speech",
|
|
87
|
-
|
|
88
|
-
|
|
571
|
+
capability: "tts",
|
|
572
|
+
providers: ["gemini"],
|
|
573
|
+
allowedModels: ["gemini-3.8-flash-tts", "gemini-3.8-flash-lite-tts"],
|
|
574
|
+
options: []
|
|
575
|
+
},
|
|
576
|
+
stt_openai: {
|
|
577
|
+
name: "OpenAI speech recognition",
|
|
578
|
+
capability: "stt",
|
|
579
|
+
providers: ["openai"],
|
|
580
|
+
allowedModels: ["gpt-transcribe", "gpt-live-transcribe"],
|
|
581
|
+
options: []
|
|
582
|
+
},
|
|
583
|
+
stt_gemini: {
|
|
584
|
+
name: "Gemini speech recognition",
|
|
585
|
+
capability: "stt",
|
|
586
|
+
providers: ["gemini"],
|
|
587
|
+
allowedModels: ["gemini-3.5-transcribe-live"],
|
|
588
|
+
options: []
|
|
589
|
+
},
|
|
590
|
+
stt_elevenlabs: {
|
|
591
|
+
name: "ElevenLabs speech recognition",
|
|
592
|
+
capability: "stt",
|
|
593
|
+
providers: ["elevenlabs"],
|
|
594
|
+
allowedModels: ["scribe_v2_realtime"],
|
|
595
|
+
options: []
|
|
596
|
+
},
|
|
597
|
+
vision_gemini: {
|
|
598
|
+
name: "Gemini sight",
|
|
599
|
+
capability: "vision",
|
|
600
|
+
providers: ["gemini"],
|
|
601
|
+
allowedModels: ["gemini-3.5-flash-lite"],
|
|
602
|
+
options: []
|
|
603
|
+
},
|
|
604
|
+
vision_openai: {
|
|
605
|
+
name: "OpenAI sight",
|
|
606
|
+
capability: "vision",
|
|
607
|
+
providers: ["openai"],
|
|
608
|
+
allowedModels: ["gpt-6-luna"],
|
|
89
609
|
options: []
|
|
90
610
|
},
|
|
91
|
-
// Priced by OPENAI_PRICES.text of the generator.
|
|
92
611
|
generator_agent: {
|
|
93
612
|
name: "Generator agent",
|
|
94
|
-
|
|
95
|
-
|
|
613
|
+
capability: "llm",
|
|
614
|
+
providers: ["openai", "anthropic"],
|
|
615
|
+
allowedModels: [
|
|
616
|
+
"gpt-6.1-sol",
|
|
617
|
+
"claude-opus-5-5",
|
|
618
|
+
"claude-sonnet-5-5",
|
|
619
|
+
"claude-haiku-5-5"
|
|
620
|
+
],
|
|
96
621
|
options: ["reasoningEffort"]
|
|
97
622
|
},
|
|
98
623
|
generator_reading: {
|
|
99
624
|
name: "Generator face reading",
|
|
100
|
-
|
|
101
|
-
|
|
625
|
+
capability: "llm",
|
|
626
|
+
providers: ["openai"],
|
|
627
|
+
allowedModels: ["gpt-6.1-sol"],
|
|
102
628
|
options: ["reasoningEffort"]
|
|
103
629
|
},
|
|
104
|
-
// Priced by OPENAI_PRICES of the generator (flare and sunburst alike).
|
|
105
630
|
generator_image: {
|
|
106
631
|
name: "Generator images",
|
|
107
|
-
|
|
108
|
-
|
|
632
|
+
capability: "image",
|
|
633
|
+
providers: ["openai"],
|
|
634
|
+
allowedModels: GPT_IMAGE_MODELS,
|
|
635
|
+
options: []
|
|
636
|
+
},
|
|
637
|
+
generator_3d: {
|
|
638
|
+
name: "Generator 3D head",
|
|
639
|
+
capability: "3d",
|
|
640
|
+
providers: ["tripo"],
|
|
641
|
+
allowedModels: ["P2-20260801"],
|
|
642
|
+
options: []
|
|
643
|
+
},
|
|
644
|
+
generator_3d_body: {
|
|
645
|
+
name: "Generator 3D body",
|
|
646
|
+
capability: "3d",
|
|
647
|
+
providers: ["tripo", "meshy"],
|
|
648
|
+
allowedModels: ["P2-20260801", "meshy-7.1", "meshy-6"],
|
|
649
|
+
options: []
|
|
650
|
+
},
|
|
651
|
+
generator_rig: {
|
|
652
|
+
name: "Generator 3D rig",
|
|
653
|
+
capability: "3d",
|
|
654
|
+
providers: ["tripo", "unirig"],
|
|
655
|
+
allowedModels: ["v1.0-20240301", "P2-20260801", "unirig-v1"],
|
|
109
656
|
options: []
|
|
110
657
|
}
|
|
111
658
|
};
|
|
@@ -242,6 +789,43 @@ var SPEECH_INPUT_PROVIDER_PROFILES = Object.freeze({
|
|
|
242
789
|
}
|
|
243
790
|
});
|
|
244
791
|
|
|
792
|
+
// ../ts-shared/src/sight.ts
|
|
793
|
+
var SIGHT_EVENTS = Object.freeze({
|
|
794
|
+
/** Client to server: one JPEG frame, acknowledged with a `SightFrameAck`. */
|
|
795
|
+
frame: "conversations:sight",
|
|
796
|
+
/** Server to the conversation's room: one `SightObservationEvent`. */
|
|
797
|
+
observation: "sight_observation"
|
|
798
|
+
});
|
|
799
|
+
var SIGHT_LIMITS = Object.freeze({
|
|
800
|
+
/** The largest JPEG frame, in bytes (200 KiB). */
|
|
801
|
+
maxFrameBytes: 200 * 1024,
|
|
802
|
+
/** The longest edge of a frame or a crop, in pixels. */
|
|
803
|
+
maxEdgePx: 1280,
|
|
804
|
+
/** The shortest time between two frames of one conversation. */
|
|
805
|
+
minFrameIntervalMs: 1e3,
|
|
806
|
+
maxCrops: 4,
|
|
807
|
+
/** The largest JPEG crop, in bytes (100 KiB). */
|
|
808
|
+
maxCropBytes: 100 * 1024,
|
|
809
|
+
maxCropLabelLength: 40
|
|
810
|
+
});
|
|
811
|
+
var SIGHT_CAPTURE_SIZE = Object.freeze({
|
|
812
|
+
width: 640,
|
|
813
|
+
height: 360,
|
|
814
|
+
jpegQuality: 0.7
|
|
815
|
+
});
|
|
816
|
+
var SIGHT_PROVIDER_PROFILES = Object.freeze({
|
|
817
|
+
gemini: {
|
|
818
|
+
model: "gemini-3.5-flash-lite",
|
|
819
|
+
frameToObservationMs: 1280,
|
|
820
|
+
usdPerFrame: 29e-5
|
|
821
|
+
},
|
|
822
|
+
openai: {
|
|
823
|
+
model: "gpt-6-luna",
|
|
824
|
+
frameToObservationMs: 2140,
|
|
825
|
+
usdPerFrame: 74e-6
|
|
826
|
+
}
|
|
827
|
+
});
|
|
828
|
+
|
|
245
829
|
// ../ts-shared/src/product-usage.ts
|
|
246
830
|
var productUsageEvents = [
|
|
247
831
|
"page_viewed",
|
|
@@ -254,7 +838,8 @@ var productUsageEvents = [
|
|
|
254
838
|
"playback_blocked",
|
|
255
839
|
"playback_start_pressed",
|
|
256
840
|
"reply_options_shown",
|
|
257
|
-
"page_left"
|
|
841
|
+
"page_left",
|
|
842
|
+
"creation_mode_chosen"
|
|
258
843
|
];
|
|
259
844
|
|
|
260
845
|
// ../ts-shared/src/analytics.ts
|
|
@@ -2636,7 +3221,7 @@ var PERFORMANCE_KINDS = {
|
|
|
2636
3221
|
cachedOpening: 8
|
|
2637
3222
|
};
|
|
2638
3223
|
var PERFORMANCE_MAJOR_VERSION = 1;
|
|
2639
|
-
var PERFORMANCE_MINOR_VERSION =
|
|
3224
|
+
var PERFORMANCE_MINOR_VERSION = 2;
|
|
2640
3225
|
var HEADER_SIZE = 24;
|
|
2641
3226
|
var MAGIC = 1346588750;
|
|
2642
3227
|
var crcTable = Uint32Array.from({ length: 256 }, (_, index) => {
|
|
@@ -2788,6 +3373,68 @@ var decodePerformance = function(input, expectedKind) {
|
|
|
2788
3373
|
return wireValue(value);
|
|
2789
3374
|
};
|
|
2790
3375
|
|
|
3376
|
+
// ../ts-shared/src/player-memory.ts
|
|
3377
|
+
var MEMORY_IDENTIFIER_MAX_LENGTH = 128;
|
|
3378
|
+
var MEMORY_IMPORT_MAX_BYTES = 512 * 1024;
|
|
3379
|
+
var normalizeMemoryIdentifier = function(value) {
|
|
3380
|
+
if (typeof value !== "string") {
|
|
3381
|
+
return null;
|
|
3382
|
+
}
|
|
3383
|
+
const cleaned = value.normalize("NFKC").replace(/[\u0000-\u001f\u007f]/gu, "").trim();
|
|
3384
|
+
if (!cleaned) {
|
|
3385
|
+
return null;
|
|
3386
|
+
}
|
|
3387
|
+
return cleaned.slice(0, MEMORY_IDENTIFIER_MAX_LENGTH).trim() || null;
|
|
3388
|
+
};
|
|
3389
|
+
|
|
3390
|
+
// ../ts-shared/src/message-time.ts
|
|
3391
|
+
var MESSAGE_TIME_ZONE_MAX_LENGTH = 64;
|
|
3392
|
+
var TIME_ZONE_PATTERN = /^[A-Za-z0-9_+-]+(?:\/[A-Za-z0-9_+-]+)*$/u;
|
|
3393
|
+
var OFFSET_PATTERN = /^[+-](?:[01]\d|2[0-3]):[0-5]\d$/u;
|
|
3394
|
+
var normalizeMessageTimeZone = function(value) {
|
|
3395
|
+
if (typeof value !== "string") {
|
|
3396
|
+
return null;
|
|
3397
|
+
}
|
|
3398
|
+
const trimmed = value.trim();
|
|
3399
|
+
if (!trimmed || trimmed.length > MESSAGE_TIME_ZONE_MAX_LENGTH) {
|
|
3400
|
+
return null;
|
|
3401
|
+
}
|
|
3402
|
+
if (trimmed === "Z" || trimmed === "z" || trimmed === "+00:00" || trimmed === "-00:00") {
|
|
3403
|
+
return "UTC";
|
|
3404
|
+
}
|
|
3405
|
+
if (!OFFSET_PATTERN.test(trimmed) && !TIME_ZONE_PATTERN.test(trimmed)) {
|
|
3406
|
+
return null;
|
|
3407
|
+
}
|
|
3408
|
+
try {
|
|
3409
|
+
new Intl.DateTimeFormat("en-US", { timeZone: trimmed }).format(/* @__PURE__ */ new Date(0));
|
|
3410
|
+
return trimmed;
|
|
3411
|
+
} catch {
|
|
3412
|
+
return null;
|
|
3413
|
+
}
|
|
3414
|
+
};
|
|
3415
|
+
var formatUtcOffsetMinutes = function(offsetMinutesEastOfUtc) {
|
|
3416
|
+
if (!Number.isFinite(offsetMinutesEastOfUtc) || offsetMinutesEastOfUtc === 0) {
|
|
3417
|
+
return "UTC";
|
|
3418
|
+
}
|
|
3419
|
+
const total = Math.trunc(offsetMinutesEastOfUtc);
|
|
3420
|
+
const sign = total >= 0 ? "+" : "-";
|
|
3421
|
+
const abs = Math.abs(total);
|
|
3422
|
+
const hours = String(Math.floor(abs / 60)).padStart(2, "0");
|
|
3423
|
+
const minutes = String(abs % 60).padStart(2, "0");
|
|
3424
|
+
return `${sign}${hours}:${minutes}`;
|
|
3425
|
+
};
|
|
3426
|
+
var detectClientTimeZone = function(now = /* @__PURE__ */ new Date()) {
|
|
3427
|
+
try {
|
|
3428
|
+
const resolved = Intl.DateTimeFormat().resolvedOptions().timeZone;
|
|
3429
|
+
const normalized = normalizeMessageTimeZone(resolved);
|
|
3430
|
+
if (normalized) {
|
|
3431
|
+
return normalized;
|
|
3432
|
+
}
|
|
3433
|
+
} catch {
|
|
3434
|
+
}
|
|
3435
|
+
return formatUtcOffsetMinutes(-now.getTimezoneOffset());
|
|
3436
|
+
};
|
|
3437
|
+
|
|
2791
3438
|
// ../../clients/web-client/src/api/performance.ts
|
|
2792
3439
|
var readPerformanceBytes = async function(response) {
|
|
2793
3440
|
const maximum = PERFORMANCE_MAX_BYTES + 65535;
|
|
@@ -3031,7 +3678,9 @@ var hasTimelineShape = function(value) {
|
|
|
3031
3678
|
}
|
|
3032
3679
|
previousDialogueEnd = Number(block.endTime);
|
|
3033
3680
|
return true;
|
|
3034
|
-
}) && isSameTime(previousDialogueEnd, value.duration) && hasOrderedItems(value.expressions, value.duration, hasExpressionShape) && hasOrderedItems(value.animations, value.duration, hasAnimationShape)
|
|
3681
|
+
}) && isSameTime(previousDialogueEnd, value.duration) && hasOrderedItems(value.expressions, value.duration, hasExpressionShape) && hasOrderedItems(value.animations, value.duration, hasAnimationShape) && (value.initialVariant === void 0 || isValidVariantKey(value.initialVariant)) && (value.activeVariant === void 0 || isValidVariantKey(value.activeVariant)) && (value.toolCalls === void 0 || Array.isArray(value.toolCalls) && value.toolCalls.every(
|
|
3682
|
+
(call) => isStagingToolCall(call) && isWithinDuration(call.timestamp, value.duration)
|
|
3683
|
+
));
|
|
3035
3684
|
};
|
|
3036
3685
|
var normalizeStagingResultPayload = function(value) {
|
|
3037
3686
|
if (!isRecord(value)) {
|
|
@@ -3141,14 +3790,20 @@ var BakedConversation = class {
|
|
|
3141
3790
|
conversationId,
|
|
3142
3791
|
openingStagingId: this.asset.rootStagingId,
|
|
3143
3792
|
activeStagingId: this.activeId,
|
|
3144
|
-
stagings: [...this.nodes.values()].map((node) =>
|
|
3145
|
-
|
|
3146
|
-
|
|
3147
|
-
|
|
3148
|
-
|
|
3149
|
-
|
|
3150
|
-
|
|
3151
|
-
|
|
3793
|
+
stagings: [...this.nodes.values()].map((node) => {
|
|
3794
|
+
const { initialVariant, activeVariant, toolCalls } = node.payload.timeline;
|
|
3795
|
+
return {
|
|
3796
|
+
id: node.id,
|
|
3797
|
+
parentId: node.parentId,
|
|
3798
|
+
triggerText: node.triggerText,
|
|
3799
|
+
text: node.payload.text,
|
|
3800
|
+
options: structuredClone(node.payload.options),
|
|
3801
|
+
createdAt: "1970-01-01T00:00:00.000Z",
|
|
3802
|
+
...initialVariant !== void 0 ? { initialVariant } : {},
|
|
3803
|
+
...activeVariant !== void 0 ? { activeVariant } : {},
|
|
3804
|
+
...toolCalls ? { toolCalls: structuredClone(toolCalls) } : {}
|
|
3805
|
+
};
|
|
3806
|
+
})
|
|
3152
3807
|
};
|
|
3153
3808
|
}
|
|
3154
3809
|
child(parentId, label) {
|
|
@@ -3188,7 +3843,9 @@ var createConversation = async function({
|
|
|
3188
3843
|
avatarId,
|
|
3189
3844
|
apiToken,
|
|
3190
3845
|
baseUrl,
|
|
3191
|
-
renderSessionId
|
|
3846
|
+
renderSessionId,
|
|
3847
|
+
hostedWebChat,
|
|
3848
|
+
memoryIdentifier
|
|
3192
3849
|
}) {
|
|
3193
3850
|
const url = new URL("/conversations", baseUrl);
|
|
3194
3851
|
const res = await fetch(url.toString(), {
|
|
@@ -3202,7 +3859,12 @@ var createConversation = async function({
|
|
|
3202
3859
|
credentials: apiToken ? "omit" : "include",
|
|
3203
3860
|
method: "POST",
|
|
3204
3861
|
// This SDK plays live motion frames (ADR 0011).
|
|
3205
|
-
body: JSON.stringify({
|
|
3862
|
+
body: JSON.stringify({
|
|
3863
|
+
avatarId,
|
|
3864
|
+
capabilities: [MOTION_FRAMES_CAPABILITY],
|
|
3865
|
+
...hostedWebChat ? { hostedWebChat: true } : {},
|
|
3866
|
+
...memoryIdentifier !== void 0 ? { memoryIdentifier } : {}
|
|
3867
|
+
})
|
|
3206
3868
|
});
|
|
3207
3869
|
if (!res.ok) {
|
|
3208
3870
|
throw Object.assign(new Error(await res.text()), { status: res.status });
|
|
@@ -3225,7 +3887,8 @@ var createStaging = async function({
|
|
|
3225
3887
|
requestId,
|
|
3226
3888
|
silent,
|
|
3227
3889
|
parentStagingId,
|
|
3228
|
-
renderSessionId
|
|
3890
|
+
renderSessionId,
|
|
3891
|
+
memoryIdentifier
|
|
3229
3892
|
}) {
|
|
3230
3893
|
const url = new URL(`/conversations/${conversationId}/stagings`, baseUrl);
|
|
3231
3894
|
const requestHeaders = withRenderSession(
|
|
@@ -3256,7 +3919,9 @@ var createStaging = async function({
|
|
|
3256
3919
|
silent: silent === true,
|
|
3257
3920
|
incrementalPlayback: true,
|
|
3258
3921
|
parentStagingId,
|
|
3259
|
-
capabilities: [MOTION_FRAMES_CAPABILITY]
|
|
3922
|
+
capabilities: [MOTION_FRAMES_CAPABILITY],
|
|
3923
|
+
timeZone: detectClientTimeZone(),
|
|
3924
|
+
...memoryIdentifier !== void 0 ? { memoryIdentifier } : {}
|
|
3260
3925
|
})
|
|
3261
3926
|
});
|
|
3262
3927
|
} catch (error) {
|
|
@@ -3440,7 +4105,8 @@ var LiveConversation = class {
|
|
|
3440
4105
|
silent,
|
|
3441
4106
|
parentStagingId,
|
|
3442
4107
|
headers: telemetry.traceHeaders,
|
|
3443
|
-
renderSessionId: this.dependencies.renderSession()
|
|
4108
|
+
renderSessionId: this.dependencies.renderSession(),
|
|
4109
|
+
memoryIdentifier: this.dependencies.memoryIdentifier?.()
|
|
3444
4110
|
});
|
|
3445
4111
|
} catch (error) {
|
|
3446
4112
|
if (request.completed) {
|
|
@@ -3702,25 +4368,61 @@ var loadGltfWithProgress = async function({
|
|
|
3702
4368
|
});
|
|
3703
4369
|
}
|
|
3704
4370
|
};
|
|
3705
|
-
var
|
|
3706
|
-
|
|
3707
|
-
|
|
3708
|
-
|
|
3709
|
-
|
|
3710
|
-
assetProfile
|
|
3711
|
-
|
|
3712
|
-
const url = new URL(`/avatars/${characterId}/download-url`, baseUrl);
|
|
3713
|
-
if (assetProfile && assetProfile !== "original") {
|
|
3714
|
-
url.searchParams.set("assetProfile", assetProfile);
|
|
4371
|
+
var fetchVrmDownloadUrl = async function(baseUrl, characterId, options) {
|
|
4372
|
+
const url = new URL(
|
|
4373
|
+
`/avatars/${encodeURIComponent(characterId)}/download-url`,
|
|
4374
|
+
baseUrl
|
|
4375
|
+
);
|
|
4376
|
+
if (options?.assetProfile && options.assetProfile !== "original") {
|
|
4377
|
+
url.searchParams.set("assetProfile", options.assetProfile);
|
|
3715
4378
|
}
|
|
3716
4379
|
const res = await fetch(url.toString(), {
|
|
3717
|
-
headers: buildAuthHeaders(apiToken, { allowEmpty: true }),
|
|
3718
|
-
credentials: apiToken ? "omit" : "include"
|
|
4380
|
+
headers: buildAuthHeaders(options?.apiToken, { allowEmpty: true }),
|
|
4381
|
+
credentials: options?.apiToken ? "omit" : "include",
|
|
4382
|
+
...options?.signal ? { signal: options.signal } : {}
|
|
4383
|
+
});
|
|
4384
|
+
if (!res.ok) {
|
|
4385
|
+
throw new Error(await res.text());
|
|
4386
|
+
}
|
|
4387
|
+
const { url: vrmUrl } = await res.json();
|
|
4388
|
+
return vrmUrl;
|
|
4389
|
+
};
|
|
4390
|
+
var fetchVariantVrmDownloadUrl = async function(baseUrl, characterId, variantKey, options) {
|
|
4391
|
+
if (variantKey === DEFAULT_VARIANT_KEY) {
|
|
4392
|
+
return await fetchVrmDownloadUrl(baseUrl, characterId, options);
|
|
4393
|
+
}
|
|
4394
|
+
const url = new URL(
|
|
4395
|
+
`/avatars/${encodeURIComponent(characterId)}/variants/${encodeURIComponent(variantKey)}/download-url`,
|
|
4396
|
+
baseUrl
|
|
4397
|
+
);
|
|
4398
|
+
if (options?.assetProfile && options.assetProfile !== "original") {
|
|
4399
|
+
url.searchParams.set("assetProfile", options.assetProfile);
|
|
4400
|
+
}
|
|
4401
|
+
const res = await fetch(url.toString(), {
|
|
4402
|
+
headers: buildAuthHeaders(options?.apiToken, { allowEmpty: true }),
|
|
4403
|
+
credentials: options?.apiToken ? "omit" : "include",
|
|
4404
|
+
...options?.signal ? { signal: options.signal } : {}
|
|
3719
4405
|
});
|
|
3720
4406
|
if (!res.ok) {
|
|
3721
4407
|
throw new Error(await res.text());
|
|
3722
4408
|
}
|
|
3723
4409
|
const { url: vrmUrl } = await res.json();
|
|
4410
|
+
return vrmUrl;
|
|
4411
|
+
};
|
|
4412
|
+
var loadVrm = async function({
|
|
4413
|
+
baseUrl,
|
|
4414
|
+
characterId,
|
|
4415
|
+
variantKey,
|
|
4416
|
+
apiToken,
|
|
4417
|
+
onProgress,
|
|
4418
|
+
assetProfile
|
|
4419
|
+
}) {
|
|
4420
|
+
const vrmUrl = await fetchVariantVrmDownloadUrl(
|
|
4421
|
+
baseUrl,
|
|
4422
|
+
characterId,
|
|
4423
|
+
variantKey ?? DEFAULT_VARIANT_KEY,
|
|
4424
|
+
{ apiToken, assetProfile }
|
|
4425
|
+
);
|
|
3724
4426
|
const gltf = await loadGltfWithProgress({
|
|
3725
4427
|
url: vrmUrl,
|
|
3726
4428
|
onProgress,
|
|
@@ -3732,6 +4434,23 @@ var loadVrm = async ({
|
|
|
3732
4434
|
VRMUtils.combineSkeletons(gltf.scene);
|
|
3733
4435
|
return gltf.userData.vrm;
|
|
3734
4436
|
};
|
|
4437
|
+
var loadVariantVrm = async function({
|
|
4438
|
+
baseUrl,
|
|
4439
|
+
characterId,
|
|
4440
|
+
variantKey,
|
|
4441
|
+
apiToken,
|
|
4442
|
+
onProgress,
|
|
4443
|
+
assetProfile
|
|
4444
|
+
}) {
|
|
4445
|
+
return await loadVrm({
|
|
4446
|
+
baseUrl,
|
|
4447
|
+
characterId,
|
|
4448
|
+
variantKey,
|
|
4449
|
+
apiToken,
|
|
4450
|
+
onProgress,
|
|
4451
|
+
assetProfile
|
|
4452
|
+
});
|
|
4453
|
+
};
|
|
3735
4454
|
var loadVrma = async ({
|
|
3736
4455
|
baseUrl,
|
|
3737
4456
|
characterId,
|
|
@@ -5314,6 +6033,70 @@ var AnimationController = class {
|
|
|
5314
6033
|
activeAnimationId = null;
|
|
5315
6034
|
idleAnimationId;
|
|
5316
6035
|
pendingReturnToIdleAction = null;
|
|
6036
|
+
setVrm(nextVrm, rawAnimations) {
|
|
6037
|
+
if (nextVrm === this.vrm) {
|
|
6038
|
+
return;
|
|
6039
|
+
}
|
|
6040
|
+
const currentAnimationId = this.activeAnimationId;
|
|
6041
|
+
const currentAction = this.activeAction;
|
|
6042
|
+
const currentTime = currentAction ? currentAction.time : 0;
|
|
6043
|
+
const currentLoop = currentAction ? currentAction.loop === LoopRepeat2 : void 0;
|
|
6044
|
+
const hadPendingReturnToIdle = this.pendingReturnToIdleAction !== null && this.pendingReturnToIdleAction === currentAction;
|
|
6045
|
+
const savedConfigs = new Map(this.configs);
|
|
6046
|
+
this.restorePreview?.();
|
|
6047
|
+
this.mixer.removeEventListener("finished", this.handleActionFinished);
|
|
6048
|
+
this.mixer.stopAllAction();
|
|
6049
|
+
this.mixer.uncacheRoot(this.vrm.scene);
|
|
6050
|
+
this.vrm.humanoid?.resetNormalizedPose?.();
|
|
6051
|
+
this.vrm = nextVrm;
|
|
6052
|
+
this.mixer = new AnimationMixer2(this.vrm.scene);
|
|
6053
|
+
this.mixer.addEventListener("finished", this.handleActionFinished);
|
|
6054
|
+
this.clips.clear();
|
|
6055
|
+
this.rawClips.clear();
|
|
6056
|
+
this.normalizedIdleClips.clear();
|
|
6057
|
+
this.configs.clear();
|
|
6058
|
+
this.activeAction = null;
|
|
6059
|
+
this.activeAnimationId = null;
|
|
6060
|
+
this.pendingReturnToIdleAction = null;
|
|
6061
|
+
for (const [animationId, animation] of rawAnimations) {
|
|
6062
|
+
const config = savedConfigs.get(animationId) ?? {
|
|
6063
|
+
loop: animationId === this.idleAnimationId,
|
|
6064
|
+
weight: defaultAnimationConfig.weight
|
|
6065
|
+
};
|
|
6066
|
+
this.addAnimation(animationId, animation, config);
|
|
6067
|
+
}
|
|
6068
|
+
const neededIds = [];
|
|
6069
|
+
if (currentAnimationId && !this.clips.has(currentAnimationId)) {
|
|
6070
|
+
neededIds.push(currentAnimationId);
|
|
6071
|
+
}
|
|
6072
|
+
if (this.idleAnimationId && !this.clips.has(this.idleAnimationId) && !neededIds.includes(this.idleAnimationId)) {
|
|
6073
|
+
neededIds.push(this.idleAnimationId);
|
|
6074
|
+
}
|
|
6075
|
+
if (neededIds.length > 0 && this.prepareAnimations) {
|
|
6076
|
+
void this.prepareAnimations(neededIds).catch(() => void 0);
|
|
6077
|
+
}
|
|
6078
|
+
if (currentAnimationId && this.clips.has(currentAnimationId)) {
|
|
6079
|
+
this.playAnimation(currentAnimationId, 0, currentLoop);
|
|
6080
|
+
const resumedAction = this.getActiveAction();
|
|
6081
|
+
if (resumedAction) {
|
|
6082
|
+
const clipDuration = resumedAction.getClip().duration;
|
|
6083
|
+
if (clipDuration > 0 && Number.isFinite(currentTime)) {
|
|
6084
|
+
resumedAction.time = resumedAction.loop === LoopRepeat2 ? (currentTime % clipDuration + clipDuration) % clipDuration : Math.min(Math.max(0, currentTime), clipDuration);
|
|
6085
|
+
}
|
|
6086
|
+
if (hadPendingReturnToIdle && !currentLoop && this.idleAnimationId) {
|
|
6087
|
+
this.pendingReturnToIdleAction = resumedAction;
|
|
6088
|
+
}
|
|
6089
|
+
}
|
|
6090
|
+
this.mixer.update(0);
|
|
6091
|
+
return;
|
|
6092
|
+
}
|
|
6093
|
+
if (this.idleAnimationId && this.clips.has(this.idleAnimationId)) {
|
|
6094
|
+
this.playIdle(0);
|
|
6095
|
+
this.mixer.update(0);
|
|
6096
|
+
return;
|
|
6097
|
+
}
|
|
6098
|
+
this.vrm.humanoid?.resetNormalizedPose?.();
|
|
6099
|
+
}
|
|
5317
6100
|
getAnimationMixer() {
|
|
5318
6101
|
return this.mixer;
|
|
5319
6102
|
}
|
|
@@ -5989,6 +6772,35 @@ var PerformanceRecorder = class {
|
|
|
5989
6772
|
};
|
|
5990
6773
|
|
|
5991
6774
|
// ../../clients/web-client/src/playback-utils.ts
|
|
6775
|
+
var deliverStagingResult = async function(payload, options) {
|
|
6776
|
+
const normalized = normalizeStagingResultPayload(payload);
|
|
6777
|
+
options.setOptions(normalized.options);
|
|
6778
|
+
if (options.source === "turn") {
|
|
6779
|
+
options.onTurnReceived(normalized);
|
|
6780
|
+
}
|
|
6781
|
+
const force = options.forcePlayback === true || options.source === "replay";
|
|
6782
|
+
const shouldNotify = force || shouldNotifyStagingResult(options.notifiedStagingIds, normalized.stagingId);
|
|
6783
|
+
if (shouldNotify && options.onStagingResult) {
|
|
6784
|
+
try {
|
|
6785
|
+
await options.onStagingResult(normalized);
|
|
6786
|
+
} catch (error) {
|
|
6787
|
+
options.onCallbackError("onStagingResult", error);
|
|
6788
|
+
}
|
|
6789
|
+
}
|
|
6790
|
+
const shouldPlay = !options.hasPlayed(normalized.stagingId) || force;
|
|
6791
|
+
options.recordEvent({
|
|
6792
|
+
type: "staging.result",
|
|
6793
|
+
source: options.source,
|
|
6794
|
+
stagingId: normalized.stagingId,
|
|
6795
|
+
taskId: normalized.taskId ?? "",
|
|
6796
|
+
playback: shouldPlay ? "requested" : "already-streamed"
|
|
6797
|
+
});
|
|
6798
|
+
if (shouldPlay) {
|
|
6799
|
+
options.playRequested(normalized, { force, silent: options.silent });
|
|
6800
|
+
} else {
|
|
6801
|
+
options.playStreamed(normalized);
|
|
6802
|
+
}
|
|
6803
|
+
};
|
|
5992
6804
|
var toSilentTimeline = function(timeline) {
|
|
5993
6805
|
return {
|
|
5994
6806
|
...timeline,
|
|
@@ -6115,6 +6927,16 @@ var StreamPlaybackController = class {
|
|
|
6115
6927
|
get activeStagingId() {
|
|
6116
6928
|
return this.activePlayback?.stagingId ?? null;
|
|
6117
6929
|
}
|
|
6930
|
+
/**
|
|
6931
|
+
* A staging plays or waits to play (its audio or the rest of its stream).
|
|
6932
|
+
* False after it completed or when autoplay blocked it; the last staging
|
|
6933
|
+
* stays the active staging after that.
|
|
6934
|
+
*/
|
|
6935
|
+
get playing() {
|
|
6936
|
+
const active = this.activePlayback;
|
|
6937
|
+
const state = active ? this.states.get(active.stagingId) : void 0;
|
|
6938
|
+
return state !== void 0 && !state.completed && !state.blocked;
|
|
6939
|
+
}
|
|
6118
6940
|
onStreamEvent(callback) {
|
|
6119
6941
|
return bindCallbackSlot(
|
|
6120
6942
|
() => this.streamEventCallback,
|
|
@@ -6411,7 +7233,11 @@ var StreamPlaybackController = class {
|
|
|
6411
7233
|
() => this.characterRevealCallback?.(event, characterIndex, character)
|
|
6412
7234
|
);
|
|
6413
7235
|
}
|
|
6414
|
-
}
|
|
7236
|
+
},
|
|
7237
|
+
onToolCall: (toolCall) => {
|
|
7238
|
+
this.dependencies.onToolCall?.(toolCall);
|
|
7239
|
+
},
|
|
7240
|
+
onVariantCue: (variantKey, toolCall) => this.dependencies.onVariantCue?.(variantKey, toolCall)
|
|
6415
7241
|
};
|
|
6416
7242
|
try {
|
|
6417
7243
|
await player.play(params.timeline, playbackOptions);
|
|
@@ -7176,22 +8002,201 @@ var AudioPlayer = class {
|
|
|
7176
8002
|
}
|
|
7177
8003
|
};
|
|
7178
8004
|
|
|
7179
|
-
// ../../clients/web-client/src/character/
|
|
7180
|
-
var
|
|
7181
|
-
|
|
7182
|
-
|
|
8005
|
+
// ../../clients/web-client/src/character/character-variants.ts
|
|
8006
|
+
var disposeOrphanedVrm = function(vrm) {
|
|
8007
|
+
if (vrm.scene?.parent) {
|
|
8008
|
+
vrm.scene.parent.remove(vrm.scene);
|
|
7183
8009
|
}
|
|
7184
|
-
|
|
7185
|
-
|
|
7186
|
-
return this.mixer.getChannelNames();
|
|
8010
|
+
if (vrm.scene && typeof vrm.scene.traverse === "function") {
|
|
8011
|
+
disposeVrm(vrm);
|
|
7187
8012
|
}
|
|
7188
|
-
|
|
7189
|
-
|
|
7190
|
-
|
|
8013
|
+
};
|
|
8014
|
+
var CharacterVariantController = class {
|
|
8015
|
+
constructor(options) {
|
|
8016
|
+
this.options = options;
|
|
7191
8017
|
}
|
|
7192
|
-
|
|
7193
|
-
|
|
7194
|
-
|
|
8018
|
+
variantVrms = /* @__PURE__ */ new Map();
|
|
8019
|
+
variantsList = [];
|
|
8020
|
+
activeVariantKey = DEFAULT_VARIANT_KEY;
|
|
8021
|
+
remoteMode = false;
|
|
8022
|
+
loadEpoch = 0;
|
|
8023
|
+
switchRevision = 0;
|
|
8024
|
+
pendingLoads = /* @__PURE__ */ new Map();
|
|
8025
|
+
listeners = {
|
|
8026
|
+
variantChange: /* @__PURE__ */ new Set(),
|
|
8027
|
+
toolCall: /* @__PURE__ */ new Set(),
|
|
8028
|
+
vrmChange: /* @__PURE__ */ new Set()
|
|
8029
|
+
};
|
|
8030
|
+
get variants() {
|
|
8031
|
+
return this.variantsList.map((variant) => ({ ...variant }));
|
|
8032
|
+
}
|
|
8033
|
+
get activeVariant() {
|
|
8034
|
+
return this.activeVariantKey;
|
|
8035
|
+
}
|
|
8036
|
+
get isRemote() {
|
|
8037
|
+
return this.remoteMode;
|
|
8038
|
+
}
|
|
8039
|
+
on(event, listener) {
|
|
8040
|
+
const bucket = this.listeners[event];
|
|
8041
|
+
bucket.add(listener);
|
|
8042
|
+
return () => {
|
|
8043
|
+
bucket.delete(listener);
|
|
8044
|
+
};
|
|
8045
|
+
}
|
|
8046
|
+
off(event, listener) {
|
|
8047
|
+
this.listeners[event].delete(listener);
|
|
8048
|
+
}
|
|
8049
|
+
emit(event, payload) {
|
|
8050
|
+
for (const listener of [...this.listeners[event]]) {
|
|
8051
|
+
invokeSafeCallback(
|
|
8052
|
+
`on(${event})`,
|
|
8053
|
+
this.options.handleCallbackError,
|
|
8054
|
+
() => listener(payload)
|
|
8055
|
+
);
|
|
8056
|
+
}
|
|
8057
|
+
}
|
|
8058
|
+
configureVariants(variants, options) {
|
|
8059
|
+
this.variantsList = variants ? variants.map((entry) => ({ ...entry })) : [];
|
|
8060
|
+
if (options?.remote !== void 0) {
|
|
8061
|
+
this.remoteMode = options.remote;
|
|
8062
|
+
}
|
|
8063
|
+
}
|
|
8064
|
+
registerDefaultVrm(vrm) {
|
|
8065
|
+
this.variantVrms.set(DEFAULT_VARIANT_KEY, vrm);
|
|
8066
|
+
this.activeVariantKey = DEFAULT_VARIANT_KEY;
|
|
8067
|
+
this.remoteMode = false;
|
|
8068
|
+
}
|
|
8069
|
+
syncDirectVrm(vrm, previousVrm) {
|
|
8070
|
+
if (vrm && !this.variantVrms.has(this.activeVariantKey)) {
|
|
8071
|
+
this.variantVrms.set(this.activeVariantKey, vrm);
|
|
8072
|
+
}
|
|
8073
|
+
if (vrm && vrm !== previousVrm) {
|
|
8074
|
+
this.emit("vrmChange", vrm);
|
|
8075
|
+
}
|
|
8076
|
+
}
|
|
8077
|
+
async preloadVariants(eager) {
|
|
8078
|
+
const keys = this.variantsList.map((variant) => variant.key).filter((key) => key !== DEFAULT_VARIANT_KEY);
|
|
8079
|
+
if (keys.length === 0) {
|
|
8080
|
+
return;
|
|
8081
|
+
}
|
|
8082
|
+
if (eager) {
|
|
8083
|
+
await Promise.all(keys.map((key) => this.ensureVariant(key)));
|
|
8084
|
+
return;
|
|
8085
|
+
}
|
|
8086
|
+
for (const key of keys) {
|
|
8087
|
+
void this.ensureVariant(key).catch(() => {
|
|
8088
|
+
});
|
|
8089
|
+
}
|
|
8090
|
+
}
|
|
8091
|
+
async ensureVariant(variantKey) {
|
|
8092
|
+
const cached = this.variantVrms.get(variantKey);
|
|
8093
|
+
if (cached) {
|
|
8094
|
+
return cached;
|
|
8095
|
+
}
|
|
8096
|
+
const existing = this.pendingLoads.get(variantKey);
|
|
8097
|
+
if (existing) {
|
|
8098
|
+
return await existing;
|
|
8099
|
+
}
|
|
8100
|
+
const epoch = this.loadEpoch;
|
|
8101
|
+
const request = this.options.loadVariant(variantKey).then((loadedVrm) => {
|
|
8102
|
+
this.pendingLoads.delete(variantKey);
|
|
8103
|
+
if (epoch !== this.loadEpoch) {
|
|
8104
|
+
disposeOrphanedVrm(loadedVrm);
|
|
8105
|
+
throw new Error("Character assets were reset while loading variant.");
|
|
8106
|
+
}
|
|
8107
|
+
this.variantVrms.set(variantKey, loadedVrm);
|
|
8108
|
+
return loadedVrm;
|
|
8109
|
+
}).catch((error) => {
|
|
8110
|
+
this.pendingLoads.delete(variantKey);
|
|
8111
|
+
throw error;
|
|
8112
|
+
});
|
|
8113
|
+
this.pendingLoads.set(variantKey, request);
|
|
8114
|
+
return await request;
|
|
8115
|
+
}
|
|
8116
|
+
async switchVariant(variantKey, toolCall = null) {
|
|
8117
|
+
if (!isValidVariantKey(variantKey)) {
|
|
8118
|
+
throw new Error(`Invalid variant key: ${variantKey}`);
|
|
8119
|
+
}
|
|
8120
|
+
const revision = ++this.switchRevision;
|
|
8121
|
+
if (this.remoteMode) {
|
|
8122
|
+
if (variantKey !== DEFAULT_VARIANT_KEY && this.variantsList.length > 0 && !this.variantsList.some((entry) => entry.key === variantKey)) {
|
|
8123
|
+
throw new Error(`Unknown variant: ${variantKey}`);
|
|
8124
|
+
}
|
|
8125
|
+
if (this.activeVariantKey === variantKey) {
|
|
8126
|
+
return;
|
|
8127
|
+
}
|
|
8128
|
+
const previousVariant = this.activeVariantKey;
|
|
8129
|
+
this.activeVariantKey = variantKey;
|
|
8130
|
+
this.emit("variantChange", {
|
|
8131
|
+
previousVariant,
|
|
8132
|
+
activeVariant: variantKey,
|
|
8133
|
+
toolCall
|
|
8134
|
+
});
|
|
8135
|
+
return;
|
|
8136
|
+
}
|
|
8137
|
+
const cached = this.variantVrms.get(variantKey);
|
|
8138
|
+
if (cached) {
|
|
8139
|
+
this.applyActiveVariant(variantKey, cached, toolCall);
|
|
8140
|
+
return;
|
|
8141
|
+
}
|
|
8142
|
+
if (variantKey === DEFAULT_VARIANT_KEY && this.activeVariantKey === DEFAULT_VARIANT_KEY) {
|
|
8143
|
+
return;
|
|
8144
|
+
}
|
|
8145
|
+
const loaded = await this.ensureVariant(variantKey);
|
|
8146
|
+
if (revision !== this.switchRevision) {
|
|
8147
|
+
return;
|
|
8148
|
+
}
|
|
8149
|
+
this.applyActiveVariant(variantKey, loaded, toolCall);
|
|
8150
|
+
}
|
|
8151
|
+
reset() {
|
|
8152
|
+
this.loadEpoch += 1;
|
|
8153
|
+
this.switchRevision += 1;
|
|
8154
|
+
this.pendingLoads.clear();
|
|
8155
|
+
const cached = [...this.variantVrms.values()];
|
|
8156
|
+
this.variantVrms.clear();
|
|
8157
|
+
this.activeVariantKey = DEFAULT_VARIANT_KEY;
|
|
8158
|
+
this.remoteMode = false;
|
|
8159
|
+
return cached;
|
|
8160
|
+
}
|
|
8161
|
+
applyActiveVariant(variantKey, nextVrm, toolCall) {
|
|
8162
|
+
const previousVariant = this.activeVariantKey;
|
|
8163
|
+
const previousVrm = this.options.getCurrentVrm();
|
|
8164
|
+
if (previousVariant === variantKey && previousVrm === nextVrm) {
|
|
8165
|
+
return;
|
|
8166
|
+
}
|
|
8167
|
+
if (previousVrm !== nextVrm) {
|
|
8168
|
+
this.options.applyVrmSwap(nextVrm);
|
|
8169
|
+
}
|
|
8170
|
+
this.activeVariantKey = variantKey;
|
|
8171
|
+
if (previousVrm !== nextVrm) {
|
|
8172
|
+
this.emit("vrmChange", nextVrm);
|
|
8173
|
+
}
|
|
8174
|
+
if (previousVariant !== variantKey) {
|
|
8175
|
+
this.emit("variantChange", {
|
|
8176
|
+
previousVariant,
|
|
8177
|
+
activeVariant: variantKey,
|
|
8178
|
+
toolCall
|
|
8179
|
+
});
|
|
8180
|
+
}
|
|
8181
|
+
}
|
|
8182
|
+
};
|
|
8183
|
+
|
|
8184
|
+
// ../../clients/web-client/src/character/expressions.ts
|
|
8185
|
+
var CharacterExpressions = class {
|
|
8186
|
+
constructor(mixer) {
|
|
8187
|
+
this.mixer = mixer;
|
|
8188
|
+
}
|
|
8189
|
+
/** All available VRM expression channel names on the loaded character. */
|
|
8190
|
+
get all() {
|
|
8191
|
+
return this.mixer.getChannelNames();
|
|
8192
|
+
}
|
|
8193
|
+
/** Available emotion expression channel names (`happy`, `angry`, `sad`, etc.). */
|
|
8194
|
+
get emotions() {
|
|
8195
|
+
return this.mixer.getEmotionChannelNames();
|
|
8196
|
+
}
|
|
8197
|
+
/** Available mouth/viseme expression channel names (`aa`, `ee`, `ih`, `oh`, `ou`). */
|
|
8198
|
+
get visemes() {
|
|
8199
|
+
return this.mixer.getVisemeChannelNames();
|
|
7195
8200
|
}
|
|
7196
8201
|
/** Current applied weights for all expression channels in `[0, 1]`. */
|
|
7197
8202
|
get weights() {
|
|
@@ -7443,6 +8448,76 @@ var IdleFace = class {
|
|
|
7443
8448
|
}
|
|
7444
8449
|
};
|
|
7445
8450
|
|
|
8451
|
+
// ../../clients/web-client/src/character/listening-face.ts
|
|
8452
|
+
var LISTENING_FACE_LAYER = "listening";
|
|
8453
|
+
var CHANNELS = ["relaxed", "happy"];
|
|
8454
|
+
var REST = 0.01;
|
|
8455
|
+
var ListeningFace = class {
|
|
8456
|
+
constructor(weights) {
|
|
8457
|
+
this.weights = weights;
|
|
8458
|
+
}
|
|
8459
|
+
channel = null;
|
|
8460
|
+
weight = 0;
|
|
8461
|
+
saved = /* @__PURE__ */ new Map();
|
|
8462
|
+
/** The preset in use, or null when the model has none of them. */
|
|
8463
|
+
get channelName() {
|
|
8464
|
+
return this.channel;
|
|
8465
|
+
}
|
|
8466
|
+
/** Call after the layers were registered for a new model. */
|
|
8467
|
+
setChannels(available, manager) {
|
|
8468
|
+
this.restoreOverrides(manager);
|
|
8469
|
+
this.channel = CHANNELS.find((name) => available.has(name)) ?? null;
|
|
8470
|
+
}
|
|
8471
|
+
set(weight, options) {
|
|
8472
|
+
this.weight = Math.max(0, Math.min(1, weight));
|
|
8473
|
+
if (options?.immediate && this.weight === 0) {
|
|
8474
|
+
this.weights.clearLayerTargets(LISTENING_FACE_LAYER, { immediate: true });
|
|
8475
|
+
}
|
|
8476
|
+
}
|
|
8477
|
+
/** Before the weight mixer updates: an active emotion takes the face. */
|
|
8478
|
+
beforeUpdate(emotionActive) {
|
|
8479
|
+
if (!this.channel) {
|
|
8480
|
+
return;
|
|
8481
|
+
}
|
|
8482
|
+
const target = emotionActive ? 0 : this.weight;
|
|
8483
|
+
if (target === 0 && this.weights.getLayerValue(LISTENING_FACE_LAYER, this.channel) < REST) {
|
|
8484
|
+
this.weights.clearLayerTargets(LISTENING_FACE_LAYER, { immediate: true });
|
|
8485
|
+
return;
|
|
8486
|
+
}
|
|
8487
|
+
this.weights.setTarget(LISTENING_FACE_LAYER, this.channel, target);
|
|
8488
|
+
}
|
|
8489
|
+
/** After the values went to the expression manager, before `vrm.update()`. */
|
|
8490
|
+
afterUpdate(manager, otherLayerValue) {
|
|
8491
|
+
const channel = this.channel;
|
|
8492
|
+
const owned = channel !== null && this.weights.getLayerValue(LISTENING_FACE_LAYER, channel) > 0 && otherLayerValue(channel) === 0;
|
|
8493
|
+
if (!owned) {
|
|
8494
|
+
this.restoreOverrides(manager);
|
|
8495
|
+
return;
|
|
8496
|
+
}
|
|
8497
|
+
const expression = manager.getExpression(channel);
|
|
8498
|
+
if (!expression || this.saved.size > 0) {
|
|
8499
|
+
return;
|
|
8500
|
+
}
|
|
8501
|
+
;
|
|
8502
|
+
["overrideBlink", "overrideLookAt"].forEach((key) => {
|
|
8503
|
+
if (expression[key] === "block") {
|
|
8504
|
+
this.saved.set(key, expression[key]);
|
|
8505
|
+
expression[key] = "blend";
|
|
8506
|
+
}
|
|
8507
|
+
});
|
|
8508
|
+
}
|
|
8509
|
+
/** Gives the preset its own overrides back. */
|
|
8510
|
+
restoreOverrides(manager) {
|
|
8511
|
+
const expression = this.channel !== null ? manager?.getExpression(this.channel) : null;
|
|
8512
|
+
if (expression) {
|
|
8513
|
+
this.saved.forEach((value, key) => {
|
|
8514
|
+
expression[key] = value;
|
|
8515
|
+
});
|
|
8516
|
+
}
|
|
8517
|
+
this.saved.clear();
|
|
8518
|
+
}
|
|
8519
|
+
};
|
|
8520
|
+
|
|
7446
8521
|
// ../../clients/web-client/src/character/performance-weight-mixer.ts
|
|
7447
8522
|
var PRECISION = 1e3;
|
|
7448
8523
|
var DEFAULT_RESPONSE_RATE = 12;
|
|
@@ -7521,6 +8596,18 @@ var PerformanceWeightMixer = class {
|
|
|
7521
8596
|
}
|
|
7522
8597
|
layer.channels.set(channel, { value: 0, target });
|
|
7523
8598
|
}
|
|
8599
|
+
restoreChannelState(layerId, channel, value, target) {
|
|
8600
|
+
const layer = this.getLayer(layerId);
|
|
8601
|
+
const state = layer.channels.get(channel);
|
|
8602
|
+
const nextValue = clamp012(value);
|
|
8603
|
+
const nextTarget = clamp012(target);
|
|
8604
|
+
if (state) {
|
|
8605
|
+
state.value = nextValue;
|
|
8606
|
+
state.target = nextTarget;
|
|
8607
|
+
return;
|
|
8608
|
+
}
|
|
8609
|
+
layer.channels.set(channel, { value: nextValue, target: nextTarget });
|
|
8610
|
+
}
|
|
7524
8611
|
setExclusiveTargets(layerId, channels, selected, weight) {
|
|
7525
8612
|
channels.forEach((channel) => {
|
|
7526
8613
|
this.setTarget(layerId, channel, channel === selected ? weight : 0);
|
|
@@ -7679,17 +8766,34 @@ var FacialPerformanceMixer = class {
|
|
|
7679
8766
|
restoreMouthAfterVisemeRelease = false;
|
|
7680
8767
|
idleFace = null;
|
|
7681
8768
|
gazeTarget = null;
|
|
7682
|
-
|
|
7683
|
-
|
|
8769
|
+
listeningFace = new ListeningFace(this.weightMixer);
|
|
8770
|
+
setVrm(vrm, options) {
|
|
8771
|
+
const preserveState = options?.preserveState === true && vrm !== null;
|
|
8772
|
+
const savedMouthMode = preserveState ? this.activeMouthMode : "expression";
|
|
8773
|
+
const savedRestoreAfterRelease = preserveState ? this.restoreMouthAfterVisemeRelease : false;
|
|
8774
|
+
const savedSnapshot = preserveState ? this.weightMixer.getSnapshot() : null;
|
|
8775
|
+
this.vrm?.expressionManager?.resetValues?.();
|
|
8776
|
+
this.listeningFace.restoreOverrides(this.vrm?.expressionManager);
|
|
8777
|
+
if (preserveState) {
|
|
8778
|
+
this.restoreMouthOverrides();
|
|
8779
|
+
} else {
|
|
8780
|
+
this.restoreExpressionMouthOwnership();
|
|
8781
|
+
this.clearCueTimeouts();
|
|
8782
|
+
}
|
|
7684
8783
|
this.vrm = vrm;
|
|
7685
8784
|
this.idleFace = vrm && detectTemplateFace(vrm) ? new IdleFace(vrm, this.options.idleRandom) : null;
|
|
7686
8785
|
this.inspector.setVrm(vrm);
|
|
7687
|
-
this.clearCueTimeouts();
|
|
7688
8786
|
this.availableChannels = [];
|
|
7689
8787
|
this.availableChannelSet.clear();
|
|
7690
8788
|
this.channelsByLayer.clear();
|
|
7691
8789
|
this.configureLayers();
|
|
7692
8790
|
if (!vrm?.expressionManager) {
|
|
8791
|
+
this.listeningFace.setChannels(this.availableChannelSet, null);
|
|
8792
|
+
if (preserveState) {
|
|
8793
|
+
this.activeMouthMode = "expression";
|
|
8794
|
+
this.restoreMouthAfterVisemeRelease = false;
|
|
8795
|
+
this.clearCueTimeouts();
|
|
8796
|
+
}
|
|
7693
8797
|
return;
|
|
7694
8798
|
}
|
|
7695
8799
|
const manager = vrm.expressionManager;
|
|
@@ -7705,7 +8809,43 @@ var FacialPerformanceMixer = class {
|
|
|
7705
8809
|
this.weightMixer.setTarget(layerId, name, 0);
|
|
7706
8810
|
});
|
|
7707
8811
|
this.availableChannels = names;
|
|
8812
|
+
this.listeningFace.setChannels(this.availableChannelSet, manager);
|
|
7708
8813
|
manager.resetValues();
|
|
8814
|
+
if (!preserveState || !savedSnapshot) {
|
|
8815
|
+
return;
|
|
8816
|
+
}
|
|
8817
|
+
this.cueTimeouts.forEach((timeoutId, channel) => {
|
|
8818
|
+
if (!this.availableChannelSet.has(channel)) {
|
|
8819
|
+
clearTimeout(timeoutId);
|
|
8820
|
+
this.cueTimeouts.delete(channel);
|
|
8821
|
+
}
|
|
8822
|
+
});
|
|
8823
|
+
Object.entries(savedSnapshot).forEach(([layerId, layerSnapshot]) => {
|
|
8824
|
+
Object.entries(layerSnapshot.channels).forEach(
|
|
8825
|
+
([channel, channelState]) => {
|
|
8826
|
+
if (this.availableChannelSet.has(channel)) {
|
|
8827
|
+
this.weightMixer.restoreChannelState(
|
|
8828
|
+
layerId,
|
|
8829
|
+
channel,
|
|
8830
|
+
channelState.value,
|
|
8831
|
+
channelState.target
|
|
8832
|
+
);
|
|
8833
|
+
}
|
|
8834
|
+
}
|
|
8835
|
+
);
|
|
8836
|
+
});
|
|
8837
|
+
this.activeMouthMode = savedMouthMode;
|
|
8838
|
+
this.restoreMouthAfterVisemeRelease = savedRestoreAfterRelease;
|
|
8839
|
+
if (savedMouthMode === "viseme") {
|
|
8840
|
+
this.getLayerChannels(FacialPerformanceLayer.Emotion).forEach((name) => {
|
|
8841
|
+
const expression = manager.getExpression(name);
|
|
8842
|
+
if (expression) {
|
|
8843
|
+
this.originalMouthOverrides.set(name, expression.overrideMouth);
|
|
8844
|
+
expression.overrideMouth = "none";
|
|
8845
|
+
}
|
|
8846
|
+
});
|
|
8847
|
+
}
|
|
8848
|
+
this.update(0);
|
|
7709
8849
|
}
|
|
7710
8850
|
getChannelNames() {
|
|
7711
8851
|
return this.availableChannels.slice();
|
|
@@ -7807,6 +8947,19 @@ var FacialPerformanceMixer = class {
|
|
|
7807
8947
|
clearOverlay() {
|
|
7808
8948
|
this.weightMixer.clearLayerTargets(FacialPerformanceLayer.Overlay);
|
|
7809
8949
|
}
|
|
8950
|
+
/**
|
|
8951
|
+
* The attentive face of the listening reflexes (`relaxed`, else `happy`)
|
|
8952
|
+
* at `weight`. It eases in and out on its own layer, gives way to any
|
|
8953
|
+
* emotion and leaves the mouth layers alone. `immediate` with weight 0
|
|
8954
|
+
* removes it in this frame.
|
|
8955
|
+
*/
|
|
8956
|
+
setListeningFace(weight, options) {
|
|
8957
|
+
this.listeningFace.set(weight, options);
|
|
8958
|
+
}
|
|
8959
|
+
/** The preset of the attentive face, or null when the model has none. */
|
|
8960
|
+
get listeningFaceChannel() {
|
|
8961
|
+
return this.listeningFace.channelName;
|
|
8962
|
+
}
|
|
7810
8963
|
releaseVisemeMouthMode() {
|
|
7811
8964
|
if (this.activeMouthMode !== "viseme") {
|
|
7812
8965
|
return;
|
|
@@ -7886,16 +9039,24 @@ var FacialPerformanceMixer = class {
|
|
|
7886
9039
|
}
|
|
7887
9040
|
reset() {
|
|
7888
9041
|
this.weightMixer.resetTargets();
|
|
9042
|
+
this.listeningFace.set(0);
|
|
7889
9043
|
}
|
|
7890
9044
|
update(delta) {
|
|
7891
9045
|
if (!this.vrm?.expressionManager) {
|
|
7892
9046
|
return;
|
|
7893
9047
|
}
|
|
7894
9048
|
const manager = this.vrm.expressionManager;
|
|
9049
|
+
this.listeningFace.beforeUpdate(
|
|
9050
|
+
this.weightMixer.isLayerActive(FacialPerformanceLayer.Emotion)
|
|
9051
|
+
);
|
|
7895
9052
|
const values = this.weightMixer.update(delta);
|
|
7896
9053
|
values.forEach((value, name) => {
|
|
7897
9054
|
manager.setValue(name, value);
|
|
7898
9055
|
});
|
|
9056
|
+
this.listeningFace.afterUpdate(
|
|
9057
|
+
manager,
|
|
9058
|
+
(channel) => this.weightMixer.getLayerValue(FacialPerformanceLayer.Emotion, channel) + this.weightMixer.getLayerValue(FacialPerformanceLayer.Overlay, channel)
|
|
9059
|
+
);
|
|
7899
9060
|
this.updateIdleFace(delta, values);
|
|
7900
9061
|
if (this.restoreMouthAfterVisemeRelease && this.areVisemesReleased()) {
|
|
7901
9062
|
this.restoreExpressionMouthOwnership();
|
|
@@ -7938,6 +9099,12 @@ var FacialPerformanceMixer = class {
|
|
|
7938
9099
|
priority: 400,
|
|
7939
9100
|
blendMode: "override"
|
|
7940
9101
|
});
|
|
9102
|
+
this.weightMixer.registerLayer({
|
|
9103
|
+
id: LISTENING_FACE_LAYER,
|
|
9104
|
+
priority: 450,
|
|
9105
|
+
blendMode: "maximum",
|
|
9106
|
+
responseRate: 8
|
|
9107
|
+
});
|
|
7941
9108
|
this.weightMixer.registerLayer({
|
|
7942
9109
|
id: FacialPerformanceLayer.Overlay,
|
|
7943
9110
|
priority: 500,
|
|
@@ -8118,8 +9285,28 @@ var CharacterSceneRig = class {
|
|
|
8118
9285
|
get vrm() {
|
|
8119
9286
|
return this.currentVrm;
|
|
8120
9287
|
}
|
|
8121
|
-
setVrm(vrm) {
|
|
9288
|
+
setVrm(vrm, fallbackParent) {
|
|
9289
|
+
const previousVrm = this.currentVrm;
|
|
9290
|
+
const previousScene = previousVrm?.scene ?? null;
|
|
9291
|
+
const previousParent = previousScene?.parent ?? null;
|
|
8122
9292
|
this.setVrmLookAtTarget(null);
|
|
9293
|
+
if (previousScene && previousParent && previousVrm !== vrm) {
|
|
9294
|
+
previousParent.remove(previousScene);
|
|
9295
|
+
}
|
|
9296
|
+
if (previousScene && vrm?.scene && previousScene !== vrm.scene) {
|
|
9297
|
+
if (previousScene.position && vrm.scene.position?.copy) {
|
|
9298
|
+
vrm.scene.position.copy(previousScene.position);
|
|
9299
|
+
}
|
|
9300
|
+
if (previousScene.rotation && vrm.scene.rotation?.copy) {
|
|
9301
|
+
vrm.scene.rotation.copy(previousScene.rotation);
|
|
9302
|
+
}
|
|
9303
|
+
if (previousScene.scale && vrm.scene.scale?.copy) {
|
|
9304
|
+
vrm.scene.scale.copy(previousScene.scale);
|
|
9305
|
+
}
|
|
9306
|
+
if (typeof previousScene.visible === "boolean") {
|
|
9307
|
+
vrm.scene.visible = previousScene.visible;
|
|
9308
|
+
}
|
|
9309
|
+
}
|
|
8123
9310
|
this.currentVrm = vrm;
|
|
8124
9311
|
this.refreshMeshes();
|
|
8125
9312
|
if (this.shadowOptions) {
|
|
@@ -8128,6 +9315,10 @@ var CharacterSceneRig = class {
|
|
|
8128
9315
|
if (this.transformOptions) {
|
|
8129
9316
|
this.applyTransform(this.transformOptions);
|
|
8130
9317
|
}
|
|
9318
|
+
const targetParent = previousParent ?? fallbackParent ?? null;
|
|
9319
|
+
if (vrm?.scene && targetParent && vrm.scene.parent !== targetParent) {
|
|
9320
|
+
targetParent.add(vrm.scene);
|
|
9321
|
+
}
|
|
8131
9322
|
}
|
|
8132
9323
|
/**
|
|
8133
9324
|
* The `Object3D` of a `lookAt()` argument: an `Object3D` as it is, a point
|
|
@@ -8169,6 +9360,7 @@ var CharacterSceneRig = class {
|
|
|
8169
9360
|
lookAt.target = target;
|
|
8170
9361
|
} else if (this.vrmLookAtTarget && lookAt.target === this.vrmLookAtTarget) {
|
|
8171
9362
|
lookAt.target = void 0;
|
|
9363
|
+
lookAt.reset();
|
|
8172
9364
|
}
|
|
8173
9365
|
}
|
|
8174
9366
|
this.vrmLookAtTarget = lookAt ? target : null;
|
|
@@ -8310,6 +9502,22 @@ var CharacterSceneRig = class {
|
|
|
8310
9502
|
disposeVrm(vrm);
|
|
8311
9503
|
}
|
|
8312
9504
|
}
|
|
9505
|
+
disposeAllVrms(vrms) {
|
|
9506
|
+
const unique = new Set(vrms);
|
|
9507
|
+
if (this.currentVrm) {
|
|
9508
|
+
unique.add(this.currentVrm);
|
|
9509
|
+
}
|
|
9510
|
+
this.detach();
|
|
9511
|
+
this.setVrm(null);
|
|
9512
|
+
for (const vrm of unique) {
|
|
9513
|
+
if (vrm.scene?.parent) {
|
|
9514
|
+
vrm.scene.parent.remove(vrm.scene);
|
|
9515
|
+
}
|
|
9516
|
+
if (vrm.scene && typeof vrm.scene.traverse === "function") {
|
|
9517
|
+
disposeVrm(vrm);
|
|
9518
|
+
}
|
|
9519
|
+
}
|
|
9520
|
+
}
|
|
8313
9521
|
applyShadows(options) {
|
|
8314
9522
|
this.cachedMeshes.forEach((mesh) => {
|
|
8315
9523
|
mesh.castShadow = options.cast;
|
|
@@ -8337,107 +9545,373 @@ var CharacterSceneRig = class {
|
|
|
8337
9545
|
}
|
|
8338
9546
|
};
|
|
8339
9547
|
|
|
8340
|
-
// ../../clients/web-client/src/character/
|
|
8341
|
-
|
|
8342
|
-
|
|
8343
|
-
|
|
8344
|
-
|
|
8345
|
-
|
|
8346
|
-
|
|
8347
|
-
head: "neck",
|
|
8348
|
-
leftShoulder: "upperChest",
|
|
8349
|
-
leftUpperArm: "leftShoulder",
|
|
8350
|
-
leftLowerArm: "leftUpperArm",
|
|
8351
|
-
leftHand: "leftLowerArm",
|
|
8352
|
-
rightShoulder: "upperChest",
|
|
8353
|
-
rightUpperArm: "rightShoulder",
|
|
8354
|
-
rightLowerArm: "rightUpperArm",
|
|
8355
|
-
rightHand: "rightLowerArm",
|
|
8356
|
-
leftUpperLeg: "hips",
|
|
8357
|
-
leftLowerLeg: "leftUpperLeg",
|
|
8358
|
-
leftFoot: "leftLowerLeg",
|
|
8359
|
-
leftToes: "leftFoot",
|
|
8360
|
-
rightUpperLeg: "hips",
|
|
8361
|
-
rightLowerLeg: "rightUpperLeg",
|
|
8362
|
-
rightFoot: "rightLowerLeg",
|
|
8363
|
-
rightToes: "rightFoot"
|
|
8364
|
-
};
|
|
8365
|
-
var MAX_TRACKS = 4;
|
|
8366
|
-
var approach = function(value, target, delta, seconds) {
|
|
8367
|
-
if (seconds <= 0) {
|
|
8368
|
-
return target;
|
|
9548
|
+
// ../../clients/web-client/src/character/character-scene-access.ts
|
|
9549
|
+
var CharacterSceneAccess = class {
|
|
9550
|
+
sceneRig = new CharacterSceneRig();
|
|
9551
|
+
defaultParent = null;
|
|
9552
|
+
/** True once the character VRM model is loaded. */
|
|
9553
|
+
get ready() {
|
|
9554
|
+
return this.sceneRig.vrm !== null;
|
|
8369
9555
|
}
|
|
8370
|
-
|
|
8371
|
-
|
|
8372
|
-
|
|
8373
|
-
var LiveMotionPlayer = class {
|
|
8374
|
-
vrm = null;
|
|
8375
|
-
tracks = /* @__PURE__ */ new Map();
|
|
8376
|
-
rootMode;
|
|
8377
|
-
blendIn;
|
|
8378
|
-
blendOut;
|
|
8379
|
-
weight = 0;
|
|
8380
|
-
/** The last applied pose, to blend out of it after the frames end. */
|
|
8381
|
-
lastPose = null;
|
|
8382
|
-
scratch = {
|
|
8383
|
-
from: new Quaternion3(),
|
|
8384
|
-
to: new Quaternion3(),
|
|
8385
|
-
carry: /* @__PURE__ */ new Map(),
|
|
8386
|
-
position: new Vector37()
|
|
8387
|
-
};
|
|
8388
|
-
constructor(options = {}) {
|
|
8389
|
-
this.rootMode = options.rootMode ?? "anchored";
|
|
8390
|
-
this.blendIn = options.blendInSeconds ?? 0.3;
|
|
8391
|
-
this.blendOut = options.blendOutSeconds ?? 0.5;
|
|
9556
|
+
/** Root Three.js `Group` (`vrm.scene`) to add to a `THREE.Scene`. */
|
|
9557
|
+
get scene() {
|
|
9558
|
+
return this.sceneRig.scene;
|
|
8392
9559
|
}
|
|
8393
|
-
|
|
8394
|
-
|
|
8395
|
-
this.
|
|
8396
|
-
this.lastPose = null;
|
|
9560
|
+
/** Alias for `scene` for Three.js `Object3D` workflows. */
|
|
9561
|
+
get object3d() {
|
|
9562
|
+
return this.sceneRig.object3d;
|
|
8397
9563
|
}
|
|
8398
|
-
/**
|
|
8399
|
-
|
|
8400
|
-
|
|
8401
|
-
const event = frames instanceof ArrayBuffer ? { ...received, frames: new Uint8Array(frames) } : received;
|
|
8402
|
-
if (!isValidMotionFramesEvent(event)) {
|
|
8403
|
-
return false;
|
|
8404
|
-
}
|
|
8405
|
-
const values = readMotionFrames(
|
|
8406
|
-
event.frames,
|
|
8407
|
-
event.frameCount,
|
|
8408
|
-
event.bones.length
|
|
8409
|
-
);
|
|
8410
|
-
const track = this.track(event.stagingId, {
|
|
8411
|
-
fps: event.fps,
|
|
8412
|
-
bones: event.bones,
|
|
8413
|
-
restHipsHeight: event.restHipsHeight
|
|
8414
|
-
});
|
|
8415
|
-
for (let frame = 0; frame < event.frameCount; frame += 1) {
|
|
8416
|
-
const index = event.startFrame + frame;
|
|
8417
|
-
const existing = track.frames.get(index);
|
|
8418
|
-
if (existing && existing.revision > event.revision) {
|
|
8419
|
-
continue;
|
|
8420
|
-
}
|
|
8421
|
-
track.frames.set(index, {
|
|
8422
|
-
revision: event.revision,
|
|
8423
|
-
values: values.subarray(frame * track.width, (frame + 1) * track.width)
|
|
8424
|
-
});
|
|
8425
|
-
}
|
|
8426
|
-
return true;
|
|
9564
|
+
/** Alias for `scene`. */
|
|
9565
|
+
get root() {
|
|
9566
|
+
return this.sceneRig.root;
|
|
8427
9567
|
}
|
|
8428
|
-
/**
|
|
8429
|
-
|
|
8430
|
-
|
|
8431
|
-
|
|
8432
|
-
|
|
8433
|
-
|
|
8434
|
-
|
|
8435
|
-
|
|
8436
|
-
|
|
8437
|
-
|
|
8438
|
-
|
|
8439
|
-
|
|
8440
|
-
|
|
9568
|
+
/** Primary character mesh (`SkinnedMesh` preferred, otherwise first `Mesh`). */
|
|
9569
|
+
get mesh() {
|
|
9570
|
+
return this.sceneRig.mesh;
|
|
9571
|
+
}
|
|
9572
|
+
/** All Three.js `Mesh` instances in the character hierarchy. */
|
|
9573
|
+
get meshes() {
|
|
9574
|
+
return this.sceneRig.meshes;
|
|
9575
|
+
}
|
|
9576
|
+
/** All Three.js `SkinnedMesh` instances in the character hierarchy. */
|
|
9577
|
+
get skinnedMeshes() {
|
|
9578
|
+
return this.sceneRig.skinnedMeshes;
|
|
9579
|
+
}
|
|
9580
|
+
/** Deduplicated Three.js `Material` instances across all character meshes. */
|
|
9581
|
+
get materials() {
|
|
9582
|
+
return this.sceneRig.materials;
|
|
9583
|
+
}
|
|
9584
|
+
/** Deduplicated Three.js `Skeleton` instances across all character skinned meshes. */
|
|
9585
|
+
get skeletons() {
|
|
9586
|
+
return this.sceneRig.skeletons;
|
|
9587
|
+
}
|
|
9588
|
+
/** Three-VRM humanoid rig when loaded. */
|
|
9589
|
+
get humanoid() {
|
|
9590
|
+
return this.sceneRig.humanoid;
|
|
9591
|
+
}
|
|
9592
|
+
/** Look up a raw humanoid bone `Object3D` by VRM bone name (for example `'head'` or `'rightHand'`). */
|
|
9593
|
+
getBone(name) {
|
|
9594
|
+
return this.sceneRig.getBone(name);
|
|
9595
|
+
}
|
|
9596
|
+
/** Look up a normalized humanoid bone `Object3D` by VRM bone name. */
|
|
9597
|
+
getNormalizedBone(name) {
|
|
9598
|
+
return this.sceneRig.getNormalizedBone(name);
|
|
9599
|
+
}
|
|
9600
|
+
/** Attach the character root `Group` to a Three.js `Scene` or parent `Object3D`. */
|
|
9601
|
+
attachTo(parent) {
|
|
9602
|
+
this.defaultParent = parent;
|
|
9603
|
+
return this.sceneRig.attachTo(parent);
|
|
9604
|
+
}
|
|
9605
|
+
/** Detach the character root `Group` from its current parent `Object3D`. */
|
|
9606
|
+
detach() {
|
|
9607
|
+
this.defaultParent = null;
|
|
9608
|
+
this.sceneRig.detach();
|
|
9609
|
+
}
|
|
9610
|
+
/** Configure `castShadow` and `receiveShadow` on all character meshes. */
|
|
9611
|
+
setShadows(options) {
|
|
9612
|
+
this.sceneRig.setShadows(options);
|
|
9613
|
+
}
|
|
9614
|
+
/** Configure `position`, `rotation`, and `scale` on the character root `Group`. */
|
|
9615
|
+
setTransform(options) {
|
|
9616
|
+
this.sceneRig.setTransform(options);
|
|
9617
|
+
}
|
|
9618
|
+
/** Re-index cached meshes, materials, and skeletons after custom scene graph edits. */
|
|
9619
|
+
refreshMeshes() {
|
|
9620
|
+
this.sceneRig.refreshMeshes();
|
|
9621
|
+
}
|
|
9622
|
+
};
|
|
9623
|
+
|
|
9624
|
+
// ../../clients/web-client/src/character/listening-reflex.ts
|
|
9625
|
+
import { Object3D as Object3D2, Vector3 as Vector38 } from "three";
|
|
9626
|
+
|
|
9627
|
+
// ../../clients/web-client/src/character/head-nod.ts
|
|
9628
|
+
import { Quaternion as Quaternion3, Vector3 as Vector37 } from "three";
|
|
9629
|
+
var SHARES = [
|
|
9630
|
+
["neck", 0.4],
|
|
9631
|
+
["head", 0.6]
|
|
9632
|
+
];
|
|
9633
|
+
var DAMPING = 0.6;
|
|
9634
|
+
var FREQUENCY_TIMES_LENGTH = 1.4;
|
|
9635
|
+
var PULSE_PART = 0.3;
|
|
9636
|
+
var MAX_STEP_SECONDS = 1 / 240;
|
|
9637
|
+
var REST_DEGREES = 0.01;
|
|
9638
|
+
var X_AXIS = new Vector37(1, 0, 0);
|
|
9639
|
+
var HeadNod = class {
|
|
9640
|
+
pitch = 0;
|
|
9641
|
+
speed = 0;
|
|
9642
|
+
elapsed = 0;
|
|
9643
|
+
pulseSeconds = 0;
|
|
9644
|
+
amplitude = 0;
|
|
9645
|
+
omega = 0;
|
|
9646
|
+
applied = [];
|
|
9647
|
+
scratch = new Quaternion3();
|
|
9648
|
+
/** The current pitch in degrees; positive turns the face down. */
|
|
9649
|
+
get pitchDegrees() {
|
|
9650
|
+
return this.pitch;
|
|
9651
|
+
}
|
|
9652
|
+
get active() {
|
|
9653
|
+
return this.elapsed < this.pulseSeconds || Math.abs(this.pitch) > REST_DEGREES || Math.abs(this.speed) > REST_DEGREES;
|
|
9654
|
+
}
|
|
9655
|
+
start(nod) {
|
|
9656
|
+
const seconds = Math.max(nod.durationMs, 1) / 1e3;
|
|
9657
|
+
this.amplitude = nod.amplitudeDegrees;
|
|
9658
|
+
this.omega = 2 * Math.PI * FREQUENCY_TIMES_LENGTH / seconds;
|
|
9659
|
+
this.pulseSeconds = seconds * PULSE_PART;
|
|
9660
|
+
this.elapsed = 0;
|
|
9661
|
+
}
|
|
9662
|
+
/** Stops the nod at once; the next `apply` removes its pitch. */
|
|
9663
|
+
cancel() {
|
|
9664
|
+
this.pitch = 0;
|
|
9665
|
+
this.speed = 0;
|
|
9666
|
+
this.elapsed = 0;
|
|
9667
|
+
this.pulseSeconds = 0;
|
|
9668
|
+
}
|
|
9669
|
+
update(delta) {
|
|
9670
|
+
if (!(delta > 0) || !this.active) {
|
|
9671
|
+
if (!this.active) {
|
|
9672
|
+
this.pitch = 0;
|
|
9673
|
+
this.speed = 0;
|
|
9674
|
+
}
|
|
9675
|
+
return this.pitch;
|
|
9676
|
+
}
|
|
9677
|
+
let left = Math.min(delta, 0.25);
|
|
9678
|
+
while (left > 0) {
|
|
9679
|
+
const step = Math.min(left, MAX_STEP_SECONDS);
|
|
9680
|
+
const target = this.elapsed < this.pulseSeconds ? this.amplitude : 0;
|
|
9681
|
+
const omega = this.omega;
|
|
9682
|
+
const acceleration = omega * omega * (target - this.pitch) - 2 * DAMPING * omega * this.speed;
|
|
9683
|
+
this.speed += acceleration * step;
|
|
9684
|
+
this.pitch += this.speed * step;
|
|
9685
|
+
this.elapsed += step;
|
|
9686
|
+
left -= step;
|
|
9687
|
+
}
|
|
9688
|
+
if (!this.active) {
|
|
9689
|
+
this.pitch = 0;
|
|
9690
|
+
this.speed = 0;
|
|
9691
|
+
}
|
|
9692
|
+
return this.pitch;
|
|
9693
|
+
}
|
|
9694
|
+
/** Writes the current pitch onto the normalized neck and head of `vrm`. */
|
|
9695
|
+
apply(vrm) {
|
|
9696
|
+
this.applied.forEach(({ node, base, result }) => {
|
|
9697
|
+
if (node.quaternion.equals(result)) {
|
|
9698
|
+
node.quaternion.copy(base);
|
|
9699
|
+
}
|
|
9700
|
+
});
|
|
9701
|
+
this.applied = [];
|
|
9702
|
+
const humanoid = vrm?.humanoid;
|
|
9703
|
+
if (!humanoid || this.pitch === 0) {
|
|
9704
|
+
return;
|
|
9705
|
+
}
|
|
9706
|
+
const flip = vrm.meta.metaVersion === "0" && !isNpcifyNormalizedVrm(vrm) ? -1 : 1;
|
|
9707
|
+
const radians = this.pitch * Math.PI / 180 * flip;
|
|
9708
|
+
let carried = 0;
|
|
9709
|
+
SHARES.forEach(([bone, share]) => {
|
|
9710
|
+
const node = humanoid.getNormalizedBoneNode(bone);
|
|
9711
|
+
if (!node) {
|
|
9712
|
+
carried += share;
|
|
9713
|
+
return;
|
|
9714
|
+
}
|
|
9715
|
+
const angle = radians * (share + carried);
|
|
9716
|
+
carried = 0;
|
|
9717
|
+
const base = node.quaternion.clone();
|
|
9718
|
+
node.quaternion.multiply(this.scratch.setFromAxisAngle(X_AXIS, angle));
|
|
9719
|
+
this.applied.push({ node, base, result: node.quaternion.clone() });
|
|
9720
|
+
});
|
|
9721
|
+
}
|
|
9722
|
+
};
|
|
9723
|
+
|
|
9724
|
+
// ../../clients/web-client/src/character/listening-reflex.ts
|
|
9725
|
+
var LISTENING_FACE_WEIGHT = 0.2;
|
|
9726
|
+
var FRONT_DISTANCE = 1.5;
|
|
9727
|
+
var ListeningReflex = class {
|
|
9728
|
+
constructor(host, activity) {
|
|
9729
|
+
this.host = host;
|
|
9730
|
+
this.activity = activity;
|
|
9731
|
+
activity.onNod((nod) => {
|
|
9732
|
+
this.pendingNod = nod;
|
|
9733
|
+
});
|
|
9734
|
+
}
|
|
9735
|
+
nod = new HeadNod();
|
|
9736
|
+
pendingNod = null;
|
|
9737
|
+
releaseGaze = null;
|
|
9738
|
+
front = new Object3D2();
|
|
9739
|
+
origin = new Vector38();
|
|
9740
|
+
forward = new Vector38();
|
|
9741
|
+
yielded = false;
|
|
9742
|
+
/** Call after the animation mixer and live motion, before the facial mixer and `vrm.update()`. */
|
|
9743
|
+
update(delta, frame) {
|
|
9744
|
+
const vrm = this.host.vrm();
|
|
9745
|
+
if (frame.stagingActive) {
|
|
9746
|
+
if (!this.yielded) {
|
|
9747
|
+
this.yielded = true;
|
|
9748
|
+
this.host.facialMixer.setListeningFace(0, { immediate: true });
|
|
9749
|
+
this.nod.cancel();
|
|
9750
|
+
this.endGaze();
|
|
9751
|
+
}
|
|
9752
|
+
this.pendingNod = null;
|
|
9753
|
+
this.nod.apply(vrm);
|
|
9754
|
+
return;
|
|
9755
|
+
}
|
|
9756
|
+
this.yielded = false;
|
|
9757
|
+
const engaged = this.activity.state !== "idle" && vrm !== null;
|
|
9758
|
+
this.host.facialMixer.setListeningFace(engaged ? LISTENING_FACE_WEIGHT : 0);
|
|
9759
|
+
if (engaged) {
|
|
9760
|
+
this.beginGaze(vrm);
|
|
9761
|
+
} else {
|
|
9762
|
+
this.endGaze();
|
|
9763
|
+
}
|
|
9764
|
+
const nod = this.pendingNod;
|
|
9765
|
+
this.pendingNod = null;
|
|
9766
|
+
if (frame.liveMotionWeight > 0) {
|
|
9767
|
+
this.nod.cancel();
|
|
9768
|
+
} else if (nod && this.activity.state === "speaking" && vrm) {
|
|
9769
|
+
this.nod.start(nod);
|
|
9770
|
+
}
|
|
9771
|
+
this.nod.update(delta);
|
|
9772
|
+
this.nod.apply(vrm);
|
|
9773
|
+
}
|
|
9774
|
+
/** A new model or the end of the conversation: no gaze, face or nod remains. */
|
|
9775
|
+
reset() {
|
|
9776
|
+
this.pendingNod = null;
|
|
9777
|
+
this.nod.cancel();
|
|
9778
|
+
this.nod.apply(null);
|
|
9779
|
+
this.endGaze();
|
|
9780
|
+
this.host.facialMixer.setListeningFace(0, { immediate: true });
|
|
9781
|
+
}
|
|
9782
|
+
beginGaze(vrm) {
|
|
9783
|
+
if (!this.releaseGaze) {
|
|
9784
|
+
if (this.host.lookAtActive()) {
|
|
9785
|
+
return;
|
|
9786
|
+
}
|
|
9787
|
+
const target = this.host.gazeTarget() ?? this.front;
|
|
9788
|
+
this.releaseGaze = this.host.lookAtTemporarily(target);
|
|
9789
|
+
}
|
|
9790
|
+
this.placeFront(vrm);
|
|
9791
|
+
}
|
|
9792
|
+
endGaze() {
|
|
9793
|
+
const release = this.releaseGaze;
|
|
9794
|
+
this.releaseGaze = null;
|
|
9795
|
+
release?.();
|
|
9796
|
+
}
|
|
9797
|
+
/** The point in front of the head, along the character's forward axis. */
|
|
9798
|
+
placeFront(vrm) {
|
|
9799
|
+
const head = vrm.humanoid?.getNormalizedBoneNode("head");
|
|
9800
|
+
if (!head) {
|
|
9801
|
+
return;
|
|
9802
|
+
}
|
|
9803
|
+
const flip = vrm.meta.metaVersion === "0" && !isNpcifyNormalizedVrm(vrm);
|
|
9804
|
+
const root = vrm.scene;
|
|
9805
|
+
const origin = root.getWorldPosition(this.origin);
|
|
9806
|
+
const forward = root.localToWorld(
|
|
9807
|
+
this.forward.set(0, 0, flip ? -FRONT_DISTANCE : FRONT_DISTANCE)
|
|
9808
|
+
).sub(origin);
|
|
9809
|
+
head.getWorldPosition(this.front.position).add(forward);
|
|
9810
|
+
this.front.updateMatrixWorld(true);
|
|
9811
|
+
}
|
|
9812
|
+
};
|
|
9813
|
+
|
|
9814
|
+
// ../../clients/web-client/src/character/live-motion.ts
|
|
9815
|
+
import { Quaternion as Quaternion4, Vector3 as Vector39 } from "three";
|
|
9816
|
+
var PARENTS = {
|
|
9817
|
+
spine: "hips",
|
|
9818
|
+
chest: "spine",
|
|
9819
|
+
upperChest: "chest",
|
|
9820
|
+
neck: "upperChest",
|
|
9821
|
+
head: "neck",
|
|
9822
|
+
leftShoulder: "upperChest",
|
|
9823
|
+
leftUpperArm: "leftShoulder",
|
|
9824
|
+
leftLowerArm: "leftUpperArm",
|
|
9825
|
+
leftHand: "leftLowerArm",
|
|
9826
|
+
rightShoulder: "upperChest",
|
|
9827
|
+
rightUpperArm: "rightShoulder",
|
|
9828
|
+
rightLowerArm: "rightUpperArm",
|
|
9829
|
+
rightHand: "rightLowerArm",
|
|
9830
|
+
leftUpperLeg: "hips",
|
|
9831
|
+
leftLowerLeg: "leftUpperLeg",
|
|
9832
|
+
leftFoot: "leftLowerLeg",
|
|
9833
|
+
leftToes: "leftFoot",
|
|
9834
|
+
rightUpperLeg: "hips",
|
|
9835
|
+
rightLowerLeg: "rightUpperLeg",
|
|
9836
|
+
rightFoot: "rightLowerLeg",
|
|
9837
|
+
rightToes: "rightFoot"
|
|
9838
|
+
};
|
|
9839
|
+
var MAX_TRACKS = 4;
|
|
9840
|
+
var approach = function(value, target, delta, seconds) {
|
|
9841
|
+
if (seconds <= 0) {
|
|
9842
|
+
return target;
|
|
9843
|
+
}
|
|
9844
|
+
const step = delta / seconds;
|
|
9845
|
+
return value < target ? Math.min(target, value + step) : Math.max(target, value - step);
|
|
9846
|
+
};
|
|
9847
|
+
var LiveMotionPlayer = class {
|
|
9848
|
+
vrm = null;
|
|
9849
|
+
tracks = /* @__PURE__ */ new Map();
|
|
9850
|
+
rootMode;
|
|
9851
|
+
blendIn;
|
|
9852
|
+
blendOut;
|
|
9853
|
+
weight = 0;
|
|
9854
|
+
/** The last applied pose, to blend out of it after the frames end. */
|
|
9855
|
+
lastPose = null;
|
|
9856
|
+
scratch = {
|
|
9857
|
+
from: new Quaternion4(),
|
|
9858
|
+
to: new Quaternion4(),
|
|
9859
|
+
carry: /* @__PURE__ */ new Map(),
|
|
9860
|
+
position: new Vector39()
|
|
9861
|
+
};
|
|
9862
|
+
constructor(options = {}) {
|
|
9863
|
+
this.rootMode = options.rootMode ?? "anchored";
|
|
9864
|
+
this.blendIn = options.blendInSeconds ?? 0.3;
|
|
9865
|
+
this.blendOut = options.blendOutSeconds ?? 0.5;
|
|
9866
|
+
}
|
|
9867
|
+
setVrm(vrm) {
|
|
9868
|
+
this.vrm = vrm;
|
|
9869
|
+
this.weight = 0;
|
|
9870
|
+
this.lastPose = null;
|
|
9871
|
+
}
|
|
9872
|
+
/** Adds the frames of a `motion_frames` event; invalid events are dropped. */
|
|
9873
|
+
push(received) {
|
|
9874
|
+
const frames = received.frames;
|
|
9875
|
+
const event = frames instanceof ArrayBuffer ? { ...received, frames: new Uint8Array(frames) } : received;
|
|
9876
|
+
if (!isValidMotionFramesEvent(event)) {
|
|
9877
|
+
return false;
|
|
9878
|
+
}
|
|
9879
|
+
const values = readMotionFrames(
|
|
9880
|
+
event.frames,
|
|
9881
|
+
event.frameCount,
|
|
9882
|
+
event.bones.length
|
|
9883
|
+
);
|
|
9884
|
+
const track = this.track(event.stagingId, {
|
|
9885
|
+
fps: event.fps,
|
|
9886
|
+
bones: event.bones,
|
|
9887
|
+
restHipsHeight: event.restHipsHeight
|
|
9888
|
+
});
|
|
9889
|
+
for (let frame = 0; frame < event.frameCount; frame += 1) {
|
|
9890
|
+
const index = event.startFrame + frame;
|
|
9891
|
+
const existing = track.frames.get(index);
|
|
9892
|
+
if (existing && existing.revision > event.revision) {
|
|
9893
|
+
continue;
|
|
9894
|
+
}
|
|
9895
|
+
track.frames.set(index, {
|
|
9896
|
+
revision: event.revision,
|
|
9897
|
+
values: values.subarray(frame * track.width, (frame + 1) * track.width)
|
|
9898
|
+
});
|
|
9899
|
+
}
|
|
9900
|
+
return true;
|
|
9901
|
+
}
|
|
9902
|
+
/** Loads the stored frames of a staging (a replay). */
|
|
9903
|
+
async load(stagingId, track, fetchBytes = async (url) => new Uint8Array(await (await fetch(url)).arrayBuffer())) {
|
|
9904
|
+
if (this.tracks.has(stagingId)) {
|
|
9905
|
+
return;
|
|
9906
|
+
}
|
|
9907
|
+
const data = track.frames.type === "bytes" ? track.frames.data : await fetchBytes(track.frames.url);
|
|
9908
|
+
const values = readMotionFrames(data, track.frameCount, track.bones.length);
|
|
9909
|
+
const stored = this.track(stagingId, track);
|
|
9910
|
+
for (let frame = 0; frame < track.frameCount; frame += 1) {
|
|
9911
|
+
stored.frames.set(frame, {
|
|
9912
|
+
revision: Number.MAX_SAFE_INTEGER,
|
|
9913
|
+
values: values.subarray(
|
|
9914
|
+
frame * stored.width,
|
|
8441
9915
|
(frame + 1) * stored.width
|
|
8442
9916
|
)
|
|
8443
9917
|
});
|
|
@@ -8545,7 +10019,7 @@ var LiveMotionPlayer = class {
|
|
|
8545
10019
|
carry.clear();
|
|
8546
10020
|
track.bones.forEach((bone, slot) => {
|
|
8547
10021
|
const offset = 3 + slot * 4;
|
|
8548
|
-
const rotation = new
|
|
10022
|
+
const rotation = new Quaternion4(
|
|
8549
10023
|
values[offset] ?? 0,
|
|
8550
10024
|
values[offset + 1] ?? 0,
|
|
8551
10025
|
values[offset + 2] ?? 0,
|
|
@@ -8592,6 +10066,166 @@ var LiveMotionPlayer = class {
|
|
|
8592
10066
|
}
|
|
8593
10067
|
};
|
|
8594
10068
|
|
|
10069
|
+
// ../../clients/web-client/src/voice/speech-nod-trigger.ts
|
|
10070
|
+
var MIN_SPEECH_LEVEL = 0.015;
|
|
10071
|
+
var SPEECH_TO_NOISE = 3;
|
|
10072
|
+
var NOD_PAUSE_MS = 250;
|
|
10073
|
+
var NOD_MIN_SPEECH_MS = 1200;
|
|
10074
|
+
var NOD_MIN_INTERVAL_MS = 1800;
|
|
10075
|
+
var NOD_INTERVAL_JITTER_MS = 1400;
|
|
10076
|
+
var NOD_SKIP_CHANCE = 0.2;
|
|
10077
|
+
var NOD_MIN_DEGREES = 4;
|
|
10078
|
+
var NOD_MAX_DEGREES = 6;
|
|
10079
|
+
var NOD_MIN_DURATION_MS = 320;
|
|
10080
|
+
var NOD_MAX_DURATION_MS = 400;
|
|
10081
|
+
var SpeechNodTrigger = class {
|
|
10082
|
+
constructor(random = Math.random) {
|
|
10083
|
+
this.random = random;
|
|
10084
|
+
this.intervalMs = this.drawInterval();
|
|
10085
|
+
}
|
|
10086
|
+
floor = null;
|
|
10087
|
+
speechMs = 0;
|
|
10088
|
+
quietMs = 0;
|
|
10089
|
+
sinceNodMs = Number.POSITIVE_INFINITY;
|
|
10090
|
+
intervalMs;
|
|
10091
|
+
/** True while the last frame was loud enough to be speech. */
|
|
10092
|
+
get voiced() {
|
|
10093
|
+
return this.speechMs > 0 && this.quietMs === 0;
|
|
10094
|
+
}
|
|
10095
|
+
/** Reads one frame; returns a nod when one is due now. */
|
|
10096
|
+
push(level, frameMs = SPEECH_FRAME_MS) {
|
|
10097
|
+
if (!Number.isFinite(level) || !(frameMs > 0)) {
|
|
10098
|
+
return null;
|
|
10099
|
+
}
|
|
10100
|
+
this.sinceNodMs += frameMs;
|
|
10101
|
+
const threshold = Math.max(
|
|
10102
|
+
MIN_SPEECH_LEVEL,
|
|
10103
|
+
(this.floor ?? MIN_SPEECH_LEVEL / SPEECH_TO_NOISE) * SPEECH_TO_NOISE
|
|
10104
|
+
);
|
|
10105
|
+
if (level >= threshold) {
|
|
10106
|
+
this.speechMs += this.quietMs + frameMs;
|
|
10107
|
+
this.quietMs = 0;
|
|
10108
|
+
return null;
|
|
10109
|
+
}
|
|
10110
|
+
this.floor = this.floor === null ? level : this.floor * 0.9 + level * 0.1;
|
|
10111
|
+
if (this.speechMs === 0) {
|
|
10112
|
+
return null;
|
|
10113
|
+
}
|
|
10114
|
+
this.quietMs += frameMs;
|
|
10115
|
+
if (this.quietMs < NOD_PAUSE_MS) {
|
|
10116
|
+
return null;
|
|
10117
|
+
}
|
|
10118
|
+
const run = this.speechMs;
|
|
10119
|
+
this.speechMs = 0;
|
|
10120
|
+
this.quietMs = 0;
|
|
10121
|
+
if (run < NOD_MIN_SPEECH_MS || this.sinceNodMs < this.intervalMs) {
|
|
10122
|
+
return null;
|
|
10123
|
+
}
|
|
10124
|
+
if (this.random() < NOD_SKIP_CHANCE) {
|
|
10125
|
+
return null;
|
|
10126
|
+
}
|
|
10127
|
+
this.sinceNodMs = 0;
|
|
10128
|
+
this.intervalMs = this.drawInterval();
|
|
10129
|
+
return {
|
|
10130
|
+
amplitudeDegrees: NOD_MIN_DEGREES + (NOD_MAX_DEGREES - NOD_MIN_DEGREES) * this.random(),
|
|
10131
|
+
durationMs: NOD_MIN_DURATION_MS + (NOD_MAX_DURATION_MS - NOD_MIN_DURATION_MS) * this.random()
|
|
10132
|
+
};
|
|
10133
|
+
}
|
|
10134
|
+
/** A new listening session: the speech run starts again; the noise floor stays. */
|
|
10135
|
+
reset() {
|
|
10136
|
+
this.speechMs = 0;
|
|
10137
|
+
this.quietMs = 0;
|
|
10138
|
+
this.sinceNodMs = Number.POSITIVE_INFINITY;
|
|
10139
|
+
}
|
|
10140
|
+
drawInterval() {
|
|
10141
|
+
return NOD_MIN_INTERVAL_MS + NOD_INTERVAL_JITTER_MS * this.random();
|
|
10142
|
+
}
|
|
10143
|
+
};
|
|
10144
|
+
|
|
10145
|
+
// ../../clients/web-client/src/character/player-activity.ts
|
|
10146
|
+
var TYPING_IDLE_MS = 1500;
|
|
10147
|
+
var PlayerActivityTracker = class {
|
|
10148
|
+
listening = false;
|
|
10149
|
+
typing = false;
|
|
10150
|
+
typingTimer = null;
|
|
10151
|
+
current = "idle";
|
|
10152
|
+
trigger;
|
|
10153
|
+
changeListeners = /* @__PURE__ */ new Set();
|
|
10154
|
+
nodListeners = /* @__PURE__ */ new Set();
|
|
10155
|
+
constructor(options = {}) {
|
|
10156
|
+
this.trigger = new SpeechNodTrigger(options.random);
|
|
10157
|
+
}
|
|
10158
|
+
get state() {
|
|
10159
|
+
return this.current;
|
|
10160
|
+
}
|
|
10161
|
+
/** The listening session is (or is no longer) in state `listening`. */
|
|
10162
|
+
setListening(active) {
|
|
10163
|
+
if (active === this.listening) {
|
|
10164
|
+
return;
|
|
10165
|
+
}
|
|
10166
|
+
this.listening = active;
|
|
10167
|
+
this.trigger.reset();
|
|
10168
|
+
this.resolve();
|
|
10169
|
+
}
|
|
10170
|
+
/**
|
|
10171
|
+
* Input in a text field: `true` at each input event, `false` when the
|
|
10172
|
+
* player sends or leaves the field. Without input, typing ends after 1.5 s.
|
|
10173
|
+
*/
|
|
10174
|
+
notifyTyping(active) {
|
|
10175
|
+
this.clearTypingTimer();
|
|
10176
|
+
this.typing = active;
|
|
10177
|
+
if (active) {
|
|
10178
|
+
this.typingTimer = setTimeout(() => {
|
|
10179
|
+
this.typingTimer = null;
|
|
10180
|
+
this.typing = false;
|
|
10181
|
+
this.resolve();
|
|
10182
|
+
}, TYPING_IDLE_MS);
|
|
10183
|
+
}
|
|
10184
|
+
this.resolve();
|
|
10185
|
+
}
|
|
10186
|
+
/** One microphone frame of the listening session; returns a nod when one is due. */
|
|
10187
|
+
pushVoiceLevel(level, frameMs) {
|
|
10188
|
+
if (!this.listening) {
|
|
10189
|
+
return null;
|
|
10190
|
+
}
|
|
10191
|
+
const nod = this.trigger.push(level, frameMs);
|
|
10192
|
+
if (nod) {
|
|
10193
|
+
this.nodListeners.forEach((listener) => listener(nod));
|
|
10194
|
+
}
|
|
10195
|
+
return nod;
|
|
10196
|
+
}
|
|
10197
|
+
onChange(listener) {
|
|
10198
|
+
this.changeListeners.add(listener);
|
|
10199
|
+
return () => this.changeListeners.delete(listener);
|
|
10200
|
+
}
|
|
10201
|
+
onNod(listener) {
|
|
10202
|
+
this.nodListeners.add(listener);
|
|
10203
|
+
return () => this.nodListeners.delete(listener);
|
|
10204
|
+
}
|
|
10205
|
+
/** Ends all activity, for example when the conversation ends. */
|
|
10206
|
+
reset() {
|
|
10207
|
+
this.clearTypingTimer();
|
|
10208
|
+
this.typing = false;
|
|
10209
|
+
this.listening = false;
|
|
10210
|
+
this.trigger.reset();
|
|
10211
|
+
this.resolve();
|
|
10212
|
+
}
|
|
10213
|
+
resolve() {
|
|
10214
|
+
const next = this.listening ? "speaking" : this.typing ? "typing" : "idle";
|
|
10215
|
+
if (next === this.current) {
|
|
10216
|
+
return;
|
|
10217
|
+
}
|
|
10218
|
+
this.current = next;
|
|
10219
|
+
this.changeListeners.forEach((listener) => listener(next));
|
|
10220
|
+
}
|
|
10221
|
+
clearTypingTimer() {
|
|
10222
|
+
if (this.typingTimer !== null) {
|
|
10223
|
+
clearTimeout(this.typingTimer);
|
|
10224
|
+
this.typingTimer = null;
|
|
10225
|
+
}
|
|
10226
|
+
}
|
|
10227
|
+
};
|
|
10228
|
+
|
|
8595
10229
|
// ../../clients/web-client/src/character/incremental-staging.ts
|
|
8596
10230
|
var dialogueKey = function(block) {
|
|
8597
10231
|
return JSON.stringify(block, (key, value) => {
|
|
@@ -8867,6 +10501,9 @@ var StagingPlayer = class {
|
|
|
8867
10501
|
activeExpressionMouthMode = null;
|
|
8868
10502
|
activeAnimation = null;
|
|
8869
10503
|
currentTrackTime = null;
|
|
10504
|
+
playbackElapsed = 0;
|
|
10505
|
+
appliedToolCallIds = /* @__PURE__ */ new Set();
|
|
10506
|
+
activeOptions;
|
|
8870
10507
|
get paused() {
|
|
8871
10508
|
return this.clock.paused;
|
|
8872
10509
|
}
|
|
@@ -8920,6 +10557,9 @@ var StagingPlayer = class {
|
|
|
8920
10557
|
this.incremental = incremental;
|
|
8921
10558
|
timeline = incremental.timeline;
|
|
8922
10559
|
this.onCursor = options?.onCursor;
|
|
10560
|
+
this.activeOptions = options;
|
|
10561
|
+
this.playbackElapsed = 0;
|
|
10562
|
+
this.appliedToolCallIds.clear();
|
|
8923
10563
|
let timelineStarted = false;
|
|
8924
10564
|
const beginTimeline = () => {
|
|
8925
10565
|
if (timelineStarted) {
|
|
@@ -8929,8 +10569,16 @@ var StagingPlayer = class {
|
|
|
8929
10569
|
options?.onTimelineStart?.(timeline);
|
|
8930
10570
|
};
|
|
8931
10571
|
try {
|
|
8932
|
-
|
|
8933
|
-
|
|
10572
|
+
await options?.onVariantCue?.(
|
|
10573
|
+
timeline.initialVariant ?? DEFAULT_VARIANT_KEY,
|
|
10574
|
+
null
|
|
10575
|
+
);
|
|
10576
|
+
if (controller.signal.aborted) {
|
|
10577
|
+
throw abortError2();
|
|
10578
|
+
}
|
|
10579
|
+
this.applyToolCalls(0, timeline);
|
|
10580
|
+
let cursor = 0;
|
|
10581
|
+
for (let index = 0; ; index += 1) {
|
|
8934
10582
|
const block = await incremental.next(index, controller.signal, () => {
|
|
8935
10583
|
this.clearPerformance();
|
|
8936
10584
|
this.clearCursor();
|
|
@@ -8978,6 +10626,7 @@ var StagingPlayer = class {
|
|
|
8978
10626
|
}
|
|
8979
10627
|
await incremental.finishBlock(index, controller.signal);
|
|
8980
10628
|
cursor = block.endTime;
|
|
10629
|
+
this.playbackElapsed = Math.max(this.playbackElapsed, cursor);
|
|
8981
10630
|
}
|
|
8982
10631
|
if (timeline.duration > cursor) {
|
|
8983
10632
|
await this.animationController.prepareAnimations?.(
|
|
@@ -8999,10 +10648,15 @@ var StagingPlayer = class {
|
|
|
8999
10648
|
beginTimeline();
|
|
9000
10649
|
}
|
|
9001
10650
|
this.applyTracks(timeline.duration, false, timeline);
|
|
10651
|
+
if (timeline.activeVariant) {
|
|
10652
|
+
await options?.onVariantCue?.(timeline.activeVariant, null);
|
|
10653
|
+
}
|
|
9002
10654
|
} finally {
|
|
9003
10655
|
if (this.generation === generation) {
|
|
9004
10656
|
this.playbackController = null;
|
|
9005
10657
|
this.incremental = null;
|
|
10658
|
+
this.activeOptions = void 0;
|
|
10659
|
+
this.appliedToolCallIds.clear();
|
|
9006
10660
|
this.clearCursor();
|
|
9007
10661
|
this.onCursor = void 0;
|
|
9008
10662
|
this.clearPerformance();
|
|
@@ -9011,6 +10665,10 @@ var StagingPlayer = class {
|
|
|
9011
10665
|
}
|
|
9012
10666
|
updateTimeline(timeline, complete = true) {
|
|
9013
10667
|
this.incremental?.update(timeline, complete);
|
|
10668
|
+
if (this.incremental) {
|
|
10669
|
+
const elapsed = this.currentTrackTime ?? this.playbackElapsed;
|
|
10670
|
+
this.applyToolCalls(elapsed, this.incremental.timeline);
|
|
10671
|
+
}
|
|
9014
10672
|
const current = this.incremental?.timeline.dialogue.find(
|
|
9015
10673
|
(block) => block.id === this.pendingSpeech?.id
|
|
9016
10674
|
);
|
|
@@ -9061,6 +10719,9 @@ var StagingPlayer = class {
|
|
|
9061
10719
|
this.playbackController = null;
|
|
9062
10720
|
this.clock.resume();
|
|
9063
10721
|
this.incremental = null;
|
|
10722
|
+
this.activeOptions = void 0;
|
|
10723
|
+
this.playbackElapsed = 0;
|
|
10724
|
+
this.appliedToolCallIds.clear();
|
|
9064
10725
|
this.clearCursor();
|
|
9065
10726
|
this.onCursor = void 0;
|
|
9066
10727
|
this.audioPlayer.stop();
|
|
@@ -9276,8 +10937,29 @@ var StagingPlayer = class {
|
|
|
9276
10937
|
await waitFrame(signal);
|
|
9277
10938
|
}
|
|
9278
10939
|
}
|
|
10940
|
+
applyToolCalls(elapsed, timeline) {
|
|
10941
|
+
const calls = timeline.toolCalls;
|
|
10942
|
+
if (!calls || calls.length === 0) {
|
|
10943
|
+
return;
|
|
10944
|
+
}
|
|
10945
|
+
const ordered = calls.length > 1 ? [...calls].sort((left, right) => left.timestamp - right.timestamp) : calls;
|
|
10946
|
+
for (const call of ordered) {
|
|
10947
|
+
if (this.appliedToolCallIds.has(call.id) || elapsed < call.timestamp) {
|
|
10948
|
+
continue;
|
|
10949
|
+
}
|
|
10950
|
+
this.appliedToolCallIds.add(call.id);
|
|
10951
|
+
this.activeOptions?.onToolCall?.(call);
|
|
10952
|
+
if (call.name === "switch_variant" && call.result.ok) {
|
|
10953
|
+
void Promise.resolve(
|
|
10954
|
+
this.activeOptions?.onVariantCue?.(call.result.activeVariant, call)
|
|
10955
|
+
).catch(() => void 0);
|
|
10956
|
+
}
|
|
10957
|
+
}
|
|
10958
|
+
}
|
|
9279
10959
|
applyTracks(time, speechActive, timeline) {
|
|
9280
10960
|
this.currentTrackTime = time;
|
|
10961
|
+
this.playbackElapsed = Math.max(this.playbackElapsed, time);
|
|
10962
|
+
this.applyToolCalls(time, timeline);
|
|
9281
10963
|
const expression = activeClipAt(timeline.expressions, time);
|
|
9282
10964
|
const mouthMode = speechActive ? "viseme" : "expression";
|
|
9283
10965
|
if (expression !== this.activeExpression || expression && mouthMode !== this.activeExpressionMouthMode) {
|
|
@@ -9731,7 +11413,7 @@ var VisemePlayer = class {
|
|
|
9731
11413
|
};
|
|
9732
11414
|
|
|
9733
11415
|
// ../../clients/web-client/src/character/character-runtime.ts
|
|
9734
|
-
var CharacterRuntime = class {
|
|
11416
|
+
var CharacterRuntime = class extends CharacterSceneAccess {
|
|
9735
11417
|
baseUrl;
|
|
9736
11418
|
characterId;
|
|
9737
11419
|
apiToken;
|
|
@@ -9741,8 +11423,8 @@ var CharacterRuntime = class {
|
|
|
9741
11423
|
animation;
|
|
9742
11424
|
animations = /* @__PURE__ */ new Map();
|
|
9743
11425
|
animationController;
|
|
9744
|
-
sceneRig = new CharacterSceneRig();
|
|
9745
11426
|
lazyAnimations;
|
|
11427
|
+
variantController;
|
|
9746
11428
|
audioPlayer = null;
|
|
9747
11429
|
stagingPlayer = null;
|
|
9748
11430
|
facialMixer;
|
|
@@ -9751,13 +11433,15 @@ var CharacterRuntime = class {
|
|
|
9751
11433
|
telemetry;
|
|
9752
11434
|
streamPlayback;
|
|
9753
11435
|
liveMotion;
|
|
9754
|
-
|
|
11436
|
+
playerActivityTracker = new PlayerActivityTracker();
|
|
11437
|
+
listeningReflex;
|
|
9755
11438
|
currentGazeTarget = null;
|
|
9756
11439
|
/** The current gaze target came from `lookAt()`. */
|
|
9757
11440
|
lookAtActive = false;
|
|
9758
11441
|
thinkingState = false;
|
|
9759
11442
|
errorCallback;
|
|
9760
11443
|
constructor(options) {
|
|
11444
|
+
super();
|
|
9761
11445
|
this.baseUrl = options.baseUrl;
|
|
9762
11446
|
this.characterId = options.characterId;
|
|
9763
11447
|
this.apiToken = options.apiToken;
|
|
@@ -9771,6 +11455,13 @@ var CharacterRuntime = class {
|
|
|
9771
11455
|
animationId,
|
|
9772
11456
|
assetProfile: this.assetProfile
|
|
9773
11457
|
}));
|
|
11458
|
+
const loadVariant = options.loadVariant ?? ((variantKey) => loadVariantVrm({
|
|
11459
|
+
baseUrl: this.baseUrl,
|
|
11460
|
+
characterId: this.characterId,
|
|
11461
|
+
variantKey,
|
|
11462
|
+
apiToken: this.apiToken,
|
|
11463
|
+
assetProfile: this.assetProfile
|
|
11464
|
+
}));
|
|
9774
11465
|
this.lazyAnimations = new LazyAnimations(loadAnimation, (id, animation) => {
|
|
9775
11466
|
this.animations.set(id, animation);
|
|
9776
11467
|
this.animationController?.addAnimation?.(id, animation, { loop: false });
|
|
@@ -9800,6 +11491,36 @@ var CharacterRuntime = class {
|
|
|
9800
11491
|
apiToken: this.apiToken
|
|
9801
11492
|
});
|
|
9802
11493
|
this.liveMotion = new LiveMotionPlayer(options.liveMotion);
|
|
11494
|
+
this.listeningReflex = new ListeningReflex(
|
|
11495
|
+
{
|
|
11496
|
+
vrm: () => this.sceneRig.vrm,
|
|
11497
|
+
gazeTarget: () => this.currentGazeTarget,
|
|
11498
|
+
lookAtActive: () => this.lookAtActive,
|
|
11499
|
+
lookAtTemporarily: (target) => this.lookAtTemporarily(target),
|
|
11500
|
+
facialMixer: this.facialMixer
|
|
11501
|
+
},
|
|
11502
|
+
this.playerActivityTracker
|
|
11503
|
+
);
|
|
11504
|
+
this.variantController = new CharacterVariantController({
|
|
11505
|
+
loadVariant,
|
|
11506
|
+
getCurrentVrm: () => this.sceneRig.vrm,
|
|
11507
|
+
handleCallbackError: this.handleCallbackError,
|
|
11508
|
+
applyVrmSwap: (nextVrm) => {
|
|
11509
|
+
this.sceneRig.setVrm(nextVrm, this.defaultParent);
|
|
11510
|
+
this.facialMixer.setVrm(nextVrm, { preserveState: true });
|
|
11511
|
+
this.animationController?.setVrm?.(nextVrm, this.animations);
|
|
11512
|
+
this.liveMotion.setVrm(nextVrm);
|
|
11513
|
+
this.applyGaze();
|
|
11514
|
+
if (this.performanceRecorder.isRecording) {
|
|
11515
|
+
this.facialMixer.capturePerformanceModel();
|
|
11516
|
+
}
|
|
11517
|
+
}
|
|
11518
|
+
});
|
|
11519
|
+
if (options.initialAssetInfo?.variants) {
|
|
11520
|
+
this.variantController.configureVariants(
|
|
11521
|
+
options.initialAssetInfo.variants
|
|
11522
|
+
);
|
|
11523
|
+
}
|
|
9803
11524
|
this.streamPlayback = new StreamPlaybackController({
|
|
9804
11525
|
getStagingPlayer: this.getStagingPlayer,
|
|
9805
11526
|
telemetry: this.telemetry,
|
|
@@ -9813,6 +11534,19 @@ var CharacterRuntime = class {
|
|
|
9813
11534
|
void this.liveMotion.load(stagingId, track).catch((error) => {
|
|
9814
11535
|
this.handleCallbackError("liveMotion", error);
|
|
9815
11536
|
});
|
|
11537
|
+
},
|
|
11538
|
+
onToolCall: (toolCall) => {
|
|
11539
|
+
this.variantController.emit("toolCall", toolCall);
|
|
11540
|
+
},
|
|
11541
|
+
onVariantCue: async (variantKey, toolCall) => {
|
|
11542
|
+
try {
|
|
11543
|
+
await this.variantController.switchVariant(variantKey, toolCall);
|
|
11544
|
+
} catch (error) {
|
|
11545
|
+
this.reportError(
|
|
11546
|
+
error instanceof Error ? error : new Error(String(error)),
|
|
11547
|
+
"switchVariant"
|
|
11548
|
+
);
|
|
11549
|
+
}
|
|
9816
11550
|
}
|
|
9817
11551
|
});
|
|
9818
11552
|
if (options.gazeTarget !== void 0) {
|
|
@@ -9848,53 +11582,28 @@ var CharacterRuntime = class {
|
|
|
9848
11582
|
return this.sceneRig.vrm;
|
|
9849
11583
|
}
|
|
9850
11584
|
set vrm(value) {
|
|
9851
|
-
this.sceneRig.
|
|
11585
|
+
const previous = this.sceneRig.vrm;
|
|
11586
|
+
this.sceneRig.setVrm(value, this.defaultParent);
|
|
9852
11587
|
this.facialMixer.setVrm(value);
|
|
9853
11588
|
this.liveMotion.setVrm(value);
|
|
9854
11589
|
this.applyGaze();
|
|
9855
|
-
|
|
9856
|
-
this.sceneRig.attachTo(this.defaultParent);
|
|
9857
|
-
}
|
|
11590
|
+
this.variantController.syncDirectVrm(value, previous);
|
|
9858
11591
|
}
|
|
9859
|
-
/**
|
|
9860
|
-
get
|
|
9861
|
-
return this.
|
|
9862
|
-
}
|
|
9863
|
-
/** Root Three.js `Group` (`vrm.scene`) to add to a `THREE.Scene`. */
|
|
9864
|
-
get scene() {
|
|
9865
|
-
return this.sceneRig.scene;
|
|
9866
|
-
}
|
|
9867
|
-
/** Alias for `scene` for Three.js `Object3D` workflows. */
|
|
9868
|
-
get object3d() {
|
|
9869
|
-
return this.sceneRig.object3d;
|
|
9870
|
-
}
|
|
9871
|
-
/** Alias for `scene`. */
|
|
9872
|
-
get root() {
|
|
9873
|
-
return this.sceneRig.root;
|
|
9874
|
-
}
|
|
9875
|
-
/** Primary character mesh (`SkinnedMesh` preferred, otherwise first `Mesh`). */
|
|
9876
|
-
get mesh() {
|
|
9877
|
-
return this.sceneRig.mesh;
|
|
9878
|
-
}
|
|
9879
|
-
/** All Three.js `Mesh` instances in the character hierarchy. */
|
|
9880
|
-
get meshes() {
|
|
9881
|
-
return this.sceneRig.meshes;
|
|
11592
|
+
/** Allowed appearance variants configured for this character (`default` first). */
|
|
11593
|
+
get variants() {
|
|
11594
|
+
return this.variantController.variants;
|
|
9882
11595
|
}
|
|
9883
|
-
/**
|
|
9884
|
-
get
|
|
9885
|
-
return this.
|
|
11596
|
+
/** Currently active appearance variant key (defaults to `'default'`). */
|
|
11597
|
+
get activeVariant() {
|
|
11598
|
+
return this.variantController.activeVariant;
|
|
9886
11599
|
}
|
|
9887
|
-
/**
|
|
9888
|
-
get
|
|
9889
|
-
return this.
|
|
11600
|
+
/** Cached Three-VRM instances keyed by variant key. */
|
|
11601
|
+
get variantVrms() {
|
|
11602
|
+
return this.variantController.variantVrms;
|
|
9890
11603
|
}
|
|
9891
|
-
/**
|
|
9892
|
-
|
|
9893
|
-
|
|
9894
|
-
}
|
|
9895
|
-
/** Three-VRM humanoid rig when loaded. */
|
|
9896
|
-
get humanoid() {
|
|
9897
|
-
return this.sceneRig.humanoid;
|
|
11604
|
+
/** Switch the character's active appearance variant, loading on-demand if needed. */
|
|
11605
|
+
async switchVariant(variantKey) {
|
|
11606
|
+
await this.variantController.switchVariant(variantKey, null);
|
|
9898
11607
|
}
|
|
9899
11608
|
/** Underlying Three.js `AnimationMixer` driving skeletal animations. */
|
|
9900
11609
|
get mixer() {
|
|
@@ -9947,36 +11656,6 @@ var CharacterRuntime = class {
|
|
|
9947
11656
|
cursor: this.cursor
|
|
9948
11657
|
};
|
|
9949
11658
|
}
|
|
9950
|
-
/** Look up a raw humanoid bone `Object3D` by VRM bone name (for example `'head'` or `'rightHand'`). */
|
|
9951
|
-
getBone(name) {
|
|
9952
|
-
return this.sceneRig.getBone(name);
|
|
9953
|
-
}
|
|
9954
|
-
/** Look up a normalized humanoid bone `Object3D` by VRM bone name. */
|
|
9955
|
-
getNormalizedBone(name) {
|
|
9956
|
-
return this.sceneRig.getNormalizedBone(name);
|
|
9957
|
-
}
|
|
9958
|
-
/** Attach the character root `Group` to a Three.js `Scene` or parent `Object3D`. */
|
|
9959
|
-
attachTo(parent) {
|
|
9960
|
-
this.defaultParent = parent;
|
|
9961
|
-
return this.sceneRig.attachTo(parent);
|
|
9962
|
-
}
|
|
9963
|
-
/** Detach the character root `Group` from its current parent `Object3D`. */
|
|
9964
|
-
detach() {
|
|
9965
|
-
this.defaultParent = null;
|
|
9966
|
-
this.sceneRig.detach();
|
|
9967
|
-
}
|
|
9968
|
-
/** Configure `castShadow` and `receiveShadow` on all character meshes. */
|
|
9969
|
-
setShadows(options) {
|
|
9970
|
-
this.sceneRig.setShadows(options);
|
|
9971
|
-
}
|
|
9972
|
-
/** Configure `position`, `rotation`, and `scale` on the character root `Group`. */
|
|
9973
|
-
setTransform(options) {
|
|
9974
|
-
this.sceneRig.setTransform(options);
|
|
9975
|
-
}
|
|
9976
|
-
/** Re-index cached meshes, materials, and skeletons after custom scene graph edits. */
|
|
9977
|
-
refreshMeshes() {
|
|
9978
|
-
this.sceneRig.refreshMeshes();
|
|
9979
|
-
}
|
|
9980
11659
|
async ensureAnimation(animationId) {
|
|
9981
11660
|
await this.lazyAnimations.ensure(animationId);
|
|
9982
11661
|
}
|
|
@@ -10075,6 +11754,28 @@ var CharacterRuntime = class {
|
|
|
10075
11754
|
this.applyGaze();
|
|
10076
11755
|
};
|
|
10077
11756
|
}
|
|
11757
|
+
/**
|
|
11758
|
+
* Tells the character that the player types (`true` at each input event
|
|
11759
|
+
* of the text field) or stopped (`false`, for example after sending). The
|
|
11760
|
+
* character looks at the player and shows a subtle attentive face; typing
|
|
11761
|
+
* ends by itself after 1.5 s without input. Local only: no request.
|
|
11762
|
+
*/
|
|
11763
|
+
notifyUserTyping(active) {
|
|
11764
|
+
this.playerActivityTracker.notifyTyping(active);
|
|
11765
|
+
}
|
|
11766
|
+
/** `'speaking'` while listening, `'typing'` after typed input, else `'idle'`. */
|
|
11767
|
+
get playerActivity() {
|
|
11768
|
+
return this.playerActivityTracker.state;
|
|
11769
|
+
}
|
|
11770
|
+
onPlayerActivity(callback) {
|
|
11771
|
+
return this.playerActivityTracker.onChange((state) => {
|
|
11772
|
+
try {
|
|
11773
|
+
callback(state);
|
|
11774
|
+
} catch (error) {
|
|
11775
|
+
this.handleCallbackError("onPlayerActivity", error);
|
|
11776
|
+
}
|
|
11777
|
+
});
|
|
11778
|
+
}
|
|
10078
11779
|
startPerformanceRecording(options) {
|
|
10079
11780
|
this.performanceRecorder.start(options);
|
|
10080
11781
|
this.facialMixer.capturePerformanceModel();
|
|
@@ -10121,6 +11822,21 @@ var CharacterRuntime = class {
|
|
|
10121
11822
|
onTimelineStart(callback) {
|
|
10122
11823
|
return this.streamPlayback.onTimelineStart(callback);
|
|
10123
11824
|
}
|
|
11825
|
+
on(event, listener) {
|
|
11826
|
+
return this.variantController.on(event, listener);
|
|
11827
|
+
}
|
|
11828
|
+
off(event, listener) {
|
|
11829
|
+
this.variantController.off(event, listener);
|
|
11830
|
+
}
|
|
11831
|
+
onVrmChange(callback) {
|
|
11832
|
+
return this.variantController.on("vrmChange", callback);
|
|
11833
|
+
}
|
|
11834
|
+
onVariantChange(callback) {
|
|
11835
|
+
return this.variantController.on("variantChange", callback);
|
|
11836
|
+
}
|
|
11837
|
+
onToolCall(callback) {
|
|
11838
|
+
return this.variantController.on("toolCall", callback);
|
|
11839
|
+
}
|
|
10124
11840
|
/** Advance skeletal animations, lip-sync visemes, facial expressions, and VRM physics by `delta` seconds. */
|
|
10125
11841
|
update(delta) {
|
|
10126
11842
|
if (this.stagingPlayer?.paused) {
|
|
@@ -10132,6 +11848,10 @@ var CharacterRuntime = class {
|
|
|
10132
11848
|
this.streamPlayback.activeStagingId,
|
|
10133
11849
|
this.stagingPlayer?.trackTime ?? null
|
|
10134
11850
|
);
|
|
11851
|
+
this.listeningReflex.update(delta, {
|
|
11852
|
+
stagingActive: this.streamPlayback.playing || this.speaking || (this.stagingPlayer?.trackTime ?? null) !== null,
|
|
11853
|
+
liveMotionWeight: this.liveMotion.liveWeight
|
|
11854
|
+
});
|
|
10135
11855
|
this.visemePlayer.update(delta);
|
|
10136
11856
|
this.facialMixer.update(delta);
|
|
10137
11857
|
this.vrm?.update(delta);
|
|
@@ -10144,73 +11864,595 @@ var CharacterRuntime = class {
|
|
|
10144
11864
|
});
|
|
10145
11865
|
}
|
|
10146
11866
|
}
|
|
10147
|
-
getAudioPlayer = () => {
|
|
10148
|
-
if (!this.audioPlayer) {
|
|
10149
|
-
this.audioPlayer = new AudioPlayer(this.visemePlayer);
|
|
11867
|
+
getAudioPlayer = () => {
|
|
11868
|
+
if (!this.audioPlayer) {
|
|
11869
|
+
this.audioPlayer = new AudioPlayer(this.visemePlayer);
|
|
11870
|
+
}
|
|
11871
|
+
return this.audioPlayer;
|
|
11872
|
+
};
|
|
11873
|
+
getStagingPlayer = () => {
|
|
11874
|
+
if (!this.animationController) {
|
|
11875
|
+
throw new Error("Animation controller is not initialized.");
|
|
11876
|
+
}
|
|
11877
|
+
if (!this.stagingPlayer) {
|
|
11878
|
+
this.stagingPlayer = new StagingPlayer(
|
|
11879
|
+
this.getAudioPlayer(),
|
|
11880
|
+
this.facialMixer,
|
|
11881
|
+
this.animationController
|
|
11882
|
+
);
|
|
11883
|
+
}
|
|
11884
|
+
return this.stagingPlayer;
|
|
11885
|
+
};
|
|
11886
|
+
/**
|
|
11887
|
+
* A template face follows the gaze target with its IdleFace. Only a
|
|
11888
|
+
* `lookAt()` target moves the eyes of another face, through three-vrm's
|
|
11889
|
+
* `vrm.lookAt.target`; on a template face that target would stop the
|
|
11890
|
+
* IdleFace (its gaze cone, idle eye moves and look cues).
|
|
11891
|
+
*/
|
|
11892
|
+
applyGaze() {
|
|
11893
|
+
this.facialMixer.setGazeTarget(this.currentGazeTarget);
|
|
11894
|
+
this.sceneRig.setVrmLookAtTarget(
|
|
11895
|
+
this.lookAtActive && !this.facialMixer.hasIdleFace ? this.currentGazeTarget : null
|
|
11896
|
+
);
|
|
11897
|
+
}
|
|
11898
|
+
resetRuntimeAssets(options) {
|
|
11899
|
+
const cachedVrms = this.variantController.reset();
|
|
11900
|
+
this.listeningReflex.reset();
|
|
11901
|
+
this.playerActivityTracker.reset();
|
|
11902
|
+
this.lazyAnimations.reset();
|
|
11903
|
+
this.stagingPlayer?.cleanup();
|
|
11904
|
+
this.stagingPlayer = null;
|
|
11905
|
+
this.audioPlayer?.reset();
|
|
11906
|
+
this.facialMixer.reset();
|
|
11907
|
+
if (this.animationController instanceof AnimationController) {
|
|
11908
|
+
this.animationController.clearAnimations();
|
|
11909
|
+
}
|
|
11910
|
+
this.animationController = void 0;
|
|
11911
|
+
this.animations.clear();
|
|
11912
|
+
if (options?.disposeVrms !== false) {
|
|
11913
|
+
this.sceneRig.disposeAllVrms(cachedVrms);
|
|
11914
|
+
} else {
|
|
11915
|
+
this.sceneRig.setVrm(null);
|
|
11916
|
+
}
|
|
11917
|
+
this.facialMixer.setVrm(null);
|
|
11918
|
+
this.liveMotion.setVrm(null);
|
|
11919
|
+
}
|
|
11920
|
+
disposeRuntimeAssets() {
|
|
11921
|
+
this.resetRuntimeAssets({ disposeVrms: true });
|
|
11922
|
+
this.audioPlayer?.cleanup();
|
|
11923
|
+
this.audioPlayer = null;
|
|
11924
|
+
}
|
|
11925
|
+
handleCallbackError = (context2, error) => {
|
|
11926
|
+
const normalized = error instanceof Error ? error : new Error(String(error ?? "Error"));
|
|
11927
|
+
this.reportError(normalized, `${context2} callback`);
|
|
11928
|
+
};
|
|
11929
|
+
reportError = (error, context2) => {
|
|
11930
|
+
if (this.errorCallback) {
|
|
11931
|
+
this.errorCallback(error);
|
|
11932
|
+
return;
|
|
11933
|
+
}
|
|
11934
|
+
console.error(context2 ? `${context2}:` : "Npcify:", error);
|
|
11935
|
+
};
|
|
11936
|
+
};
|
|
11937
|
+
|
|
11938
|
+
// ../../clients/web-client/src/conversation-socket.ts
|
|
11939
|
+
import { io } from "socket.io-client";
|
|
11940
|
+
|
|
11941
|
+
// ../../clients/web-client/src/sight/sight-error.ts
|
|
11942
|
+
var SightError = class extends Error {
|
|
11943
|
+
constructor(code, message) {
|
|
11944
|
+
super(message ?? code);
|
|
11945
|
+
this.code = code;
|
|
11946
|
+
this.name = "NpcifySightError";
|
|
11947
|
+
}
|
|
11948
|
+
};
|
|
11949
|
+
var SIGHT_STOP_CODES = /* @__PURE__ */ new Set([
|
|
11950
|
+
"SIGHT_DISABLED",
|
|
11951
|
+
"INSUFFICIENT_POINTS",
|
|
11952
|
+
"CONVERSATION_ACCESS_DENIED"
|
|
11953
|
+
]);
|
|
11954
|
+
var SIGHT_OFF_MESSAGE = "Sight is off for this character. Its owner turns it on in the Sight tab.";
|
|
11955
|
+
var STOP_MESSAGES = {
|
|
11956
|
+
SIGHT_DISABLED: SIGHT_OFF_MESSAGE,
|
|
11957
|
+
INSUFFICIENT_POINTS: "The owner of the character has no points to pay for sight.",
|
|
11958
|
+
CONVERSATION_ACCESS_DENIED: "This connection has no access to the conversation."
|
|
11959
|
+
};
|
|
11960
|
+
var sightStopError = function(code) {
|
|
11961
|
+
return new SightError(code, STOP_MESSAGES[code] ?? code);
|
|
11962
|
+
};
|
|
11963
|
+
var toSightError = function(error, fallback) {
|
|
11964
|
+
if (error instanceof SightError) {
|
|
11965
|
+
return error;
|
|
11966
|
+
}
|
|
11967
|
+
const code = error?.code;
|
|
11968
|
+
return new SightError(
|
|
11969
|
+
typeof code === "string" && code ? code : fallback,
|
|
11970
|
+
error instanceof Error ? error.message : void 0
|
|
11971
|
+
);
|
|
11972
|
+
};
|
|
11973
|
+
|
|
11974
|
+
// ../../clients/web-client/src/sight/frame-source.ts
|
|
11975
|
+
var LUMA_GRID = Object.freeze({ width: 32, height: 18 });
|
|
11976
|
+
var SMALLER_JPEG_QUALITY = 0.5;
|
|
11977
|
+
var HAVE_CURRENT_DATA = 2;
|
|
11978
|
+
var fitCaptureSize = function(width, height, limit = SIGHT_CAPTURE_SIZE) {
|
|
11979
|
+
const scale = Math.min(1, limit.width / width, limit.height / height);
|
|
11980
|
+
return {
|
|
11981
|
+
width: Math.max(1, Math.round(width * scale)),
|
|
11982
|
+
height: Math.max(1, Math.round(height * scale))
|
|
11983
|
+
};
|
|
11984
|
+
};
|
|
11985
|
+
var lumaOf = function(rgba) {
|
|
11986
|
+
const luma = new Uint8Array(Math.floor(rgba.length / 4));
|
|
11987
|
+
for (let index = 0; index < luma.length; index += 1) {
|
|
11988
|
+
const offset = index * 4;
|
|
11989
|
+
luma[index] = Math.round(
|
|
11990
|
+
0.299 * (rgba[offset] ?? 0) + 0.587 * (rgba[offset + 1] ?? 0) + 0.114 * (rgba[offset + 2] ?? 0)
|
|
11991
|
+
);
|
|
11992
|
+
}
|
|
11993
|
+
return luma;
|
|
11994
|
+
};
|
|
11995
|
+
var meanLumaChange = function(previous, next) {
|
|
11996
|
+
const length = Math.min(previous.length, next.length);
|
|
11997
|
+
if (length === 0) {
|
|
11998
|
+
return 0;
|
|
11999
|
+
}
|
|
12000
|
+
let sum = 0;
|
|
12001
|
+
for (let index = 0; index < length; index += 1) {
|
|
12002
|
+
sum += Math.abs((next[index] ?? 0) - (previous[index] ?? 0));
|
|
12003
|
+
}
|
|
12004
|
+
return sum / length;
|
|
12005
|
+
};
|
|
12006
|
+
var encodeJpeg = function(canvas, quality) {
|
|
12007
|
+
return new Promise((resolve) => {
|
|
12008
|
+
canvas.toBlob(resolve, "image/jpeg", quality);
|
|
12009
|
+
});
|
|
12010
|
+
};
|
|
12011
|
+
var VideoFrameSource = class {
|
|
12012
|
+
constructor(stream, owned) {
|
|
12013
|
+
this.stream = stream;
|
|
12014
|
+
this.owned = owned;
|
|
12015
|
+
const settings = stream.getVideoTracks()[0]?.getSettings?.();
|
|
12016
|
+
this.kind = settings?.displaySurface ? "screen" : "camera";
|
|
12017
|
+
const video = document.createElement("video");
|
|
12018
|
+
video.muted = true;
|
|
12019
|
+
video.playsInline = true;
|
|
12020
|
+
video.setAttribute("playsinline", "");
|
|
12021
|
+
video.srcObject = stream;
|
|
12022
|
+
void video.play().catch(() => void 0);
|
|
12023
|
+
this.video = video;
|
|
12024
|
+
this.frameCanvas = document.createElement("canvas");
|
|
12025
|
+
const grid = document.createElement("canvas");
|
|
12026
|
+
grid.width = LUMA_GRID.width;
|
|
12027
|
+
grid.height = LUMA_GRID.height;
|
|
12028
|
+
this.gridContext = grid.getContext("2d", { willReadFrequently: true });
|
|
12029
|
+
if (this.gridContext) {
|
|
12030
|
+
this.gridContext.imageSmoothingQuality = "high";
|
|
12031
|
+
}
|
|
12032
|
+
}
|
|
12033
|
+
kind;
|
|
12034
|
+
video;
|
|
12035
|
+
frameCanvas;
|
|
12036
|
+
frameContext = null;
|
|
12037
|
+
gridContext;
|
|
12038
|
+
stopped = false;
|
|
12039
|
+
get live() {
|
|
12040
|
+
return !this.stopped && this.stream.getVideoTracks().some((track) => track.readyState === "live");
|
|
12041
|
+
}
|
|
12042
|
+
async capture() {
|
|
12043
|
+
const picture = this.pictureSize();
|
|
12044
|
+
if (!picture) {
|
|
12045
|
+
return null;
|
|
12046
|
+
}
|
|
12047
|
+
const size = fitCaptureSize(picture.width, picture.height);
|
|
12048
|
+
const canvas = this.frameCanvas;
|
|
12049
|
+
if (canvas.width !== size.width || canvas.height !== size.height) {
|
|
12050
|
+
canvas.width = size.width;
|
|
12051
|
+
canvas.height = size.height;
|
|
12052
|
+
}
|
|
12053
|
+
this.frameContext ??= canvas.getContext("2d");
|
|
12054
|
+
const context2 = this.frameContext;
|
|
12055
|
+
if (!context2) {
|
|
12056
|
+
return null;
|
|
12057
|
+
}
|
|
12058
|
+
context2.imageSmoothingQuality = "high";
|
|
12059
|
+
context2.drawImage(this.video, 0, 0, size.width, size.height);
|
|
12060
|
+
let blob = await encodeJpeg(canvas, SIGHT_CAPTURE_SIZE.jpegQuality);
|
|
12061
|
+
if (blob && blob.size > SIGHT_LIMITS.maxFrameBytes) {
|
|
12062
|
+
blob = await encodeJpeg(canvas, SMALLER_JPEG_QUALITY);
|
|
12063
|
+
}
|
|
12064
|
+
if (!blob || blob.size > SIGHT_LIMITS.maxFrameBytes) {
|
|
12065
|
+
return null;
|
|
12066
|
+
}
|
|
12067
|
+
return { jpeg: await blob.arrayBuffer(), ...size };
|
|
12068
|
+
}
|
|
12069
|
+
luminance() {
|
|
12070
|
+
const context2 = this.gridContext;
|
|
12071
|
+
if (!context2 || !this.pictureSize()) {
|
|
12072
|
+
return null;
|
|
12073
|
+
}
|
|
12074
|
+
context2.drawImage(this.video, 0, 0, LUMA_GRID.width, LUMA_GRID.height);
|
|
12075
|
+
return lumaOf(
|
|
12076
|
+
context2.getImageData(0, 0, LUMA_GRID.width, LUMA_GRID.height).data
|
|
12077
|
+
);
|
|
12078
|
+
}
|
|
12079
|
+
stop() {
|
|
12080
|
+
if (this.stopped) {
|
|
12081
|
+
return;
|
|
12082
|
+
}
|
|
12083
|
+
this.stopped = true;
|
|
12084
|
+
this.video.pause();
|
|
12085
|
+
this.video.srcObject = null;
|
|
12086
|
+
if (this.owned) {
|
|
12087
|
+
this.stream.getTracks().forEach((track) => track.stop());
|
|
12088
|
+
}
|
|
12089
|
+
}
|
|
12090
|
+
pictureSize() {
|
|
12091
|
+
const { video } = this;
|
|
12092
|
+
if (this.stopped || video.readyState < HAVE_CURRENT_DATA || !video.videoWidth || !video.videoHeight) {
|
|
12093
|
+
return null;
|
|
12094
|
+
}
|
|
12095
|
+
return { width: video.videoWidth, height: video.videoHeight };
|
|
12096
|
+
}
|
|
12097
|
+
};
|
|
12098
|
+
var openFrameSource = async function(source) {
|
|
12099
|
+
if (source !== "camera") {
|
|
12100
|
+
if (typeof source?.getVideoTracks !== "function") {
|
|
12101
|
+
throw new SightError(
|
|
12102
|
+
"CAMERA_UNAVAILABLE",
|
|
12103
|
+
'The sight source must be "camera" or a MediaStream.'
|
|
12104
|
+
);
|
|
12105
|
+
}
|
|
12106
|
+
if (source.getVideoTracks().length === 0) {
|
|
12107
|
+
throw new SightError(
|
|
12108
|
+
"CAMERA_UNAVAILABLE",
|
|
12109
|
+
"The stream has no video track."
|
|
12110
|
+
);
|
|
12111
|
+
}
|
|
12112
|
+
return new VideoFrameSource(source, false);
|
|
12113
|
+
}
|
|
12114
|
+
const devices = typeof navigator === "undefined" ? void 0 : navigator.mediaDevices;
|
|
12115
|
+
if (!devices?.getUserMedia) {
|
|
12116
|
+
throw new SightError(
|
|
12117
|
+
"CAMERA_UNAVAILABLE",
|
|
12118
|
+
"This browser cannot open the camera."
|
|
12119
|
+
);
|
|
12120
|
+
}
|
|
12121
|
+
let stream;
|
|
12122
|
+
try {
|
|
12123
|
+
stream = await devices.getUserMedia({
|
|
12124
|
+
video: {
|
|
12125
|
+
width: { ideal: SIGHT_CAPTURE_SIZE.width },
|
|
12126
|
+
height: { ideal: SIGHT_CAPTURE_SIZE.height }
|
|
12127
|
+
},
|
|
12128
|
+
audio: false
|
|
12129
|
+
});
|
|
12130
|
+
} catch (error) {
|
|
12131
|
+
const name = error?.name;
|
|
12132
|
+
throw name === "NotAllowedError" || name === "SecurityError" ? new SightError("CAMERA_PERMISSION_DENIED", "Camera access was denied.") : new SightError("CAMERA_UNAVAILABLE", "No camera is available.");
|
|
12133
|
+
}
|
|
12134
|
+
return new VideoFrameSource(stream, true);
|
|
12135
|
+
};
|
|
12136
|
+
|
|
12137
|
+
// ../../clients/web-client/src/sight/sight-input.ts
|
|
12138
|
+
var SIGHT_CHECK_MS = 250;
|
|
12139
|
+
var SIGHT_INTERVAL_MS = 5e3;
|
|
12140
|
+
var SIGHT_MOTION_THRESHOLD = 8;
|
|
12141
|
+
var ACK_TIMEOUT_MS = 2e4;
|
|
12142
|
+
var systemClock = {
|
|
12143
|
+
now: () => Date.now(),
|
|
12144
|
+
setInterval: (callback, ms) => setInterval(callback, ms),
|
|
12145
|
+
clearInterval: (handle) => clearInterval(handle)
|
|
12146
|
+
};
|
|
12147
|
+
var settle = async function(run) {
|
|
12148
|
+
return run();
|
|
12149
|
+
};
|
|
12150
|
+
var documentHidden = function() {
|
|
12151
|
+
return typeof document !== "undefined" && document.hidden === true;
|
|
12152
|
+
};
|
|
12153
|
+
var intervalOf = function(value) {
|
|
12154
|
+
return typeof value === "number" && value > 0 ? Math.max(SIGHT_LIMITS.minFrameIntervalMs, value) : SIGHT_INTERVAL_MS;
|
|
12155
|
+
};
|
|
12156
|
+
var thresholdOf = function(motion) {
|
|
12157
|
+
if (motion === false) {
|
|
12158
|
+
return null;
|
|
12159
|
+
}
|
|
12160
|
+
const threshold = motion?.threshold;
|
|
12161
|
+
return typeof threshold === "number" && Number.isFinite(threshold) ? Math.max(0, threshold) : SIGHT_MOTION_THRESHOLD;
|
|
12162
|
+
};
|
|
12163
|
+
var isSightObservationEvent = function(value) {
|
|
12164
|
+
const event = value;
|
|
12165
|
+
const observation = event?.observation;
|
|
12166
|
+
return typeof event?.conversationId === "string" && typeof event.observationId === "string" && typeof event.summary === "string" && typeof observation === "object" && observation !== null && typeof observation.people === "number" && Array.isArray(observation.persons) && Array.isArray(observation.text);
|
|
12167
|
+
};
|
|
12168
|
+
var SightInput = class {
|
|
12169
|
+
constructor(deps) {
|
|
12170
|
+
this.deps = deps;
|
|
12171
|
+
this.clock = deps.clock ?? systemClock;
|
|
12172
|
+
this.hidden = deps.hidden ?? documentHidden;
|
|
12173
|
+
}
|
|
12174
|
+
clock;
|
|
12175
|
+
hidden;
|
|
12176
|
+
session = null;
|
|
12177
|
+
/** Why sight cannot start; null when it can. */
|
|
12178
|
+
unavailable = "Sight needs the character info: load the character first.";
|
|
12179
|
+
current = "unavailable";
|
|
12180
|
+
stateCallback = null;
|
|
12181
|
+
observationCallback = null;
|
|
12182
|
+
get state() {
|
|
12183
|
+
return this.current;
|
|
12184
|
+
}
|
|
12185
|
+
/**
|
|
12186
|
+
* The character info was read. Sight is available only when the owner
|
|
12187
|
+
* turned it on, and never in a baked conversation. Sight that becomes
|
|
12188
|
+
* unavailable stops.
|
|
12189
|
+
*/
|
|
12190
|
+
configure(settings) {
|
|
12191
|
+
this.unavailable = settings.baked ? "Sight is not available in a baked (offline) conversation." : settings.enabled ? null : SIGHT_OFF_MESSAGE;
|
|
12192
|
+
if (this.unavailable) {
|
|
12193
|
+
this.stop();
|
|
12194
|
+
} else {
|
|
12195
|
+
this.publish();
|
|
12196
|
+
}
|
|
12197
|
+
}
|
|
12198
|
+
/**
|
|
12199
|
+
* Opens the source and joins the conversation. Resolves when frames can
|
|
12200
|
+
* go; the first frame goes when the video has a picture.
|
|
12201
|
+
*/
|
|
12202
|
+
async start(options = {}) {
|
|
12203
|
+
if (this.unavailable) {
|
|
12204
|
+
throw new SightError("SIGHT_DISABLED", this.unavailable);
|
|
12205
|
+
}
|
|
12206
|
+
if (this.session) {
|
|
12207
|
+
throw new SightError(
|
|
12208
|
+
"SIGHT_ALREADY_STARTED",
|
|
12209
|
+
"Sight has already started."
|
|
12210
|
+
);
|
|
12211
|
+
}
|
|
12212
|
+
const conversationId = this.deps.conversationId();
|
|
12213
|
+
if (!conversationId) {
|
|
12214
|
+
throw new SightError(
|
|
12215
|
+
"CONVERSATION_REQUIRED",
|
|
12216
|
+
"Create a conversation before sight starts."
|
|
12217
|
+
);
|
|
12218
|
+
}
|
|
12219
|
+
const session = {
|
|
12220
|
+
intervalMs: intervalOf(options.intervalMs),
|
|
12221
|
+
threshold: thresholdOf(options.motion),
|
|
12222
|
+
onError: options.onError,
|
|
12223
|
+
source: null,
|
|
12224
|
+
timer: null,
|
|
12225
|
+
grid: null,
|
|
12226
|
+
lastSentAt: Number.NEGATIVE_INFINITY,
|
|
12227
|
+
motion: false,
|
|
12228
|
+
inFlight: false,
|
|
12229
|
+
rejoin: false
|
|
12230
|
+
};
|
|
12231
|
+
this.session = session;
|
|
12232
|
+
this.publish();
|
|
12233
|
+
const open = this.deps.openSource ?? openFrameSource;
|
|
12234
|
+
const [joined, opened] = await Promise.allSettled([
|
|
12235
|
+
settle(() => this.deps.socket().ensureConnected(conversationId)),
|
|
12236
|
+
settle(() => open(options.source ?? "camera"))
|
|
12237
|
+
]);
|
|
12238
|
+
const source = opened.status === "fulfilled" ? opened.value : null;
|
|
12239
|
+
if (session !== this.session) {
|
|
12240
|
+
source?.stop();
|
|
12241
|
+
return;
|
|
12242
|
+
}
|
|
12243
|
+
const failure = opened.status === "rejected" ? toSightError(opened.reason, "CAMERA_UNAVAILABLE") : joined.status === "rejected" ? toSightError(joined.reason, "CONVERSATION_DISCONNECTED") : null;
|
|
12244
|
+
if (failure || !source) {
|
|
12245
|
+
source?.stop();
|
|
12246
|
+
this.session = null;
|
|
12247
|
+
this.publish();
|
|
12248
|
+
throw failure ?? new SightError("CAMERA_UNAVAILABLE");
|
|
12249
|
+
}
|
|
12250
|
+
session.source = source;
|
|
12251
|
+
session.timer = this.clock.setInterval(
|
|
12252
|
+
() => this.check(session),
|
|
12253
|
+
SIGHT_CHECK_MS
|
|
12254
|
+
);
|
|
12255
|
+
this.publish();
|
|
12256
|
+
this.check(session);
|
|
12257
|
+
}
|
|
12258
|
+
/** Stops the frames and releases the source. Does nothing when sight is off. */
|
|
12259
|
+
stop() {
|
|
12260
|
+
if (this.session) {
|
|
12261
|
+
this.end(this.session);
|
|
12262
|
+
}
|
|
12263
|
+
this.publish();
|
|
12264
|
+
}
|
|
12265
|
+
/** An observation of the current conversation (the socket checked the ID). */
|
|
12266
|
+
handleObservation(event) {
|
|
12267
|
+
if (this.observationCallback) {
|
|
12268
|
+
invokeSafeCallback(
|
|
12269
|
+
"onSight",
|
|
12270
|
+
this.deps.handleCallbackError,
|
|
12271
|
+
() => this.observationCallback?.(event)
|
|
12272
|
+
);
|
|
12273
|
+
}
|
|
12274
|
+
}
|
|
12275
|
+
onObservation(callback) {
|
|
12276
|
+
return bindCallbackSlot(
|
|
12277
|
+
() => this.observationCallback,
|
|
12278
|
+
(next) => this.observationCallback = next,
|
|
12279
|
+
callback
|
|
12280
|
+
);
|
|
12281
|
+
}
|
|
12282
|
+
onState(callback) {
|
|
12283
|
+
return bindCallbackSlot(
|
|
12284
|
+
() => this.stateCallback,
|
|
12285
|
+
(next) => this.stateCallback = next,
|
|
12286
|
+
callback
|
|
12287
|
+
);
|
|
12288
|
+
}
|
|
12289
|
+
/** One check: the end of the source, a hidden page, motion, then a due frame. */
|
|
12290
|
+
check(session) {
|
|
12291
|
+
const source = session.source;
|
|
12292
|
+
if (session !== this.session || !source) {
|
|
12293
|
+
return;
|
|
12294
|
+
}
|
|
12295
|
+
if (!source.live) {
|
|
12296
|
+
this.fail(
|
|
12297
|
+
session,
|
|
12298
|
+
new SightError("SIGHT_SOURCE_ENDED", "The video of sight ended.")
|
|
12299
|
+
);
|
|
12300
|
+
return;
|
|
12301
|
+
}
|
|
12302
|
+
if (this.hidden()) {
|
|
12303
|
+
session.grid = null;
|
|
12304
|
+
return;
|
|
12305
|
+
}
|
|
12306
|
+
if (session.threshold !== null) {
|
|
12307
|
+
const grid = source.luminance();
|
|
12308
|
+
if (grid && session.grid && meanLumaChange(session.grid, grid) > session.threshold) {
|
|
12309
|
+
session.motion = true;
|
|
12310
|
+
}
|
|
12311
|
+
session.grid = grid;
|
|
12312
|
+
}
|
|
12313
|
+
this.sendWhenDue(session);
|
|
12314
|
+
}
|
|
12315
|
+
/**
|
|
12316
|
+
* A frame is due at the interval, or after motion when one second passed
|
|
12317
|
+
* since the last frame. While a frame waits for its acknowledgement, a due
|
|
12318
|
+
* frame waits too: the acknowledgement sends it.
|
|
12319
|
+
*/
|
|
12320
|
+
sendWhenDue(session) {
|
|
12321
|
+
if (session !== this.session || session.inFlight || this.hidden()) {
|
|
12322
|
+
return;
|
|
10150
12323
|
}
|
|
10151
|
-
|
|
10152
|
-
|
|
10153
|
-
|
|
10154
|
-
if (!this.animationController) {
|
|
10155
|
-
throw new Error("Animation controller is not initialized.");
|
|
12324
|
+
const elapsed = this.clock.now() - session.lastSentAt;
|
|
12325
|
+
if (elapsed >= session.intervalMs || session.motion && elapsed >= SIGHT_LIMITS.minFrameIntervalMs) {
|
|
12326
|
+
void this.send(session);
|
|
10156
12327
|
}
|
|
10157
|
-
|
|
10158
|
-
|
|
10159
|
-
|
|
10160
|
-
|
|
10161
|
-
|
|
12328
|
+
}
|
|
12329
|
+
async send(session) {
|
|
12330
|
+
const source = session.source;
|
|
12331
|
+
if (!source) {
|
|
12332
|
+
return;
|
|
12333
|
+
}
|
|
12334
|
+
session.inFlight = true;
|
|
12335
|
+
let sent = false;
|
|
12336
|
+
try {
|
|
12337
|
+
const capturedAt = this.clock.now();
|
|
12338
|
+
let frame;
|
|
12339
|
+
try {
|
|
12340
|
+
frame = await source.capture();
|
|
12341
|
+
} catch (error) {
|
|
12342
|
+
this.fail(
|
|
12343
|
+
session,
|
|
12344
|
+
new SightError(
|
|
12345
|
+
"CAMERA_UNAVAILABLE",
|
|
12346
|
+
error instanceof Error ? error.message : "The video gave no picture."
|
|
12347
|
+
)
|
|
12348
|
+
);
|
|
12349
|
+
return;
|
|
12350
|
+
}
|
|
12351
|
+
if (!frame || session !== this.session) {
|
|
12352
|
+
return;
|
|
12353
|
+
}
|
|
12354
|
+
const conversationId = this.deps.conversationId();
|
|
12355
|
+
if (!conversationId) {
|
|
12356
|
+
this.stop();
|
|
12357
|
+
return;
|
|
12358
|
+
}
|
|
12359
|
+
session.lastSentAt = this.clock.now();
|
|
12360
|
+
session.motion = false;
|
|
12361
|
+
sent = true;
|
|
12362
|
+
const socket = this.deps.socket();
|
|
12363
|
+
if (session.rejoin) {
|
|
12364
|
+
await socket.ensureConnected(conversationId);
|
|
12365
|
+
session.rejoin = false;
|
|
12366
|
+
}
|
|
12367
|
+
const request = {
|
|
12368
|
+
conversationId,
|
|
12369
|
+
capturedAt,
|
|
12370
|
+
width: frame.width,
|
|
12371
|
+
height: frame.height,
|
|
12372
|
+
source: source.kind,
|
|
12373
|
+
jpeg: frame.jpeg
|
|
12374
|
+
};
|
|
12375
|
+
const ack = await socket.request(
|
|
12376
|
+
SIGHT_EVENTS.frame,
|
|
12377
|
+
request,
|
|
12378
|
+
ACK_TIMEOUT_MS
|
|
10162
12379
|
);
|
|
12380
|
+
const code = ack && !ack.ok ? ack.error : null;
|
|
12381
|
+
if (code && SIGHT_STOP_CODES.has(code)) {
|
|
12382
|
+
this.fail(session, sightStopError(code));
|
|
12383
|
+
}
|
|
12384
|
+
} catch (error) {
|
|
12385
|
+
this.failed(session, error);
|
|
12386
|
+
} finally {
|
|
12387
|
+
session.inFlight = false;
|
|
12388
|
+
if (sent) {
|
|
12389
|
+
this.sendWhenDue(session);
|
|
12390
|
+
}
|
|
10163
12391
|
}
|
|
10164
|
-
|
|
10165
|
-
};
|
|
12392
|
+
}
|
|
10166
12393
|
/**
|
|
10167
|
-
* A
|
|
10168
|
-
*
|
|
10169
|
-
*
|
|
10170
|
-
* IdleFace (its gaze cone, idle eye moves and look cues).
|
|
12394
|
+
* A frame that did not reach the server or got no acknowledgement. A
|
|
12395
|
+
* refused conversation stops sight. A timeout skips the frame; a lost
|
|
12396
|
+
* connection also joins the conversation again before the next frame.
|
|
10171
12397
|
*/
|
|
10172
|
-
|
|
10173
|
-
|
|
10174
|
-
|
|
10175
|
-
this.
|
|
10176
|
-
|
|
12398
|
+
failed(session, error) {
|
|
12399
|
+
const code = error?.code;
|
|
12400
|
+
if (typeof code === "string" && SIGHT_STOP_CODES.has(code)) {
|
|
12401
|
+
this.fail(session, sightStopError(code));
|
|
12402
|
+
return;
|
|
12403
|
+
}
|
|
12404
|
+
if (code !== "CONVERSATION_TIMEOUT") {
|
|
12405
|
+
session.rejoin = true;
|
|
12406
|
+
}
|
|
10177
12407
|
}
|
|
10178
|
-
|
|
10179
|
-
|
|
10180
|
-
this.
|
|
10181
|
-
|
|
10182
|
-
|
|
10183
|
-
this.
|
|
10184
|
-
if (
|
|
10185
|
-
this.
|
|
12408
|
+
/** Stops sight because of `error`, and reports it. */
|
|
12409
|
+
fail(session, error) {
|
|
12410
|
+
if (session !== this.session) {
|
|
12411
|
+
return;
|
|
12412
|
+
}
|
|
12413
|
+
this.end(session);
|
|
12414
|
+
if (error.code === "SIGHT_DISABLED") {
|
|
12415
|
+
this.unavailable = SIGHT_OFF_MESSAGE;
|
|
12416
|
+
}
|
|
12417
|
+
this.publish(error);
|
|
12418
|
+
const onError = session.onError;
|
|
12419
|
+
if (onError) {
|
|
12420
|
+
invokeSafeCallback(
|
|
12421
|
+
"startSight onError",
|
|
12422
|
+
this.deps.handleCallbackError,
|
|
12423
|
+
() => onError(error)
|
|
12424
|
+
);
|
|
10186
12425
|
}
|
|
10187
|
-
this.animationController = void 0;
|
|
10188
|
-
this.animations.clear();
|
|
10189
|
-
this.sceneRig.setVrm(null);
|
|
10190
|
-
this.facialMixer.setVrm(null);
|
|
10191
|
-
this.liveMotion.setVrm(null);
|
|
10192
12426
|
}
|
|
10193
|
-
|
|
10194
|
-
this.
|
|
10195
|
-
|
|
10196
|
-
|
|
10197
|
-
this.
|
|
12427
|
+
end(session) {
|
|
12428
|
+
if (this.session !== session) {
|
|
12429
|
+
return;
|
|
12430
|
+
}
|
|
12431
|
+
this.session = null;
|
|
12432
|
+
if (session.timer !== null) {
|
|
12433
|
+
this.clock.clearInterval(session.timer);
|
|
12434
|
+
session.timer = null;
|
|
12435
|
+
}
|
|
12436
|
+
session.source?.stop();
|
|
10198
12437
|
}
|
|
10199
|
-
|
|
10200
|
-
const
|
|
10201
|
-
this.
|
|
10202
|
-
|
|
10203
|
-
reportError = (error, context2) => {
|
|
10204
|
-
if (this.errorCallback) {
|
|
10205
|
-
this.errorCallback(error);
|
|
12438
|
+
publish(error) {
|
|
12439
|
+
const session = this.session;
|
|
12440
|
+
const next = session ? session.source ? "on" : "starting" : this.unavailable ? "unavailable" : "off";
|
|
12441
|
+
if (next === this.current && !error) {
|
|
10206
12442
|
return;
|
|
10207
12443
|
}
|
|
10208
|
-
|
|
10209
|
-
|
|
12444
|
+
this.current = next;
|
|
12445
|
+
if (this.stateCallback) {
|
|
12446
|
+
invokeSafeCallback(
|
|
12447
|
+
"onSightState",
|
|
12448
|
+
this.deps.handleCallbackError,
|
|
12449
|
+
() => this.stateCallback?.(next, error)
|
|
12450
|
+
);
|
|
12451
|
+
}
|
|
12452
|
+
}
|
|
10210
12453
|
};
|
|
10211
12454
|
|
|
10212
12455
|
// ../../clients/web-client/src/conversation-socket.ts
|
|
10213
|
-
import { io } from "socket.io-client";
|
|
10214
12456
|
var ConversationConnectionError = class extends Error {
|
|
10215
12457
|
constructor(code, message, reason = null) {
|
|
10216
12458
|
super(message);
|
|
@@ -10563,6 +12805,11 @@ var ConversationSocket = class {
|
|
|
10563
12805
|
this.options.handleSpeechEvent?.({ type: "turnFailed", event });
|
|
10564
12806
|
}
|
|
10565
12807
|
});
|
|
12808
|
+
socket.on(SIGHT_EVENTS.observation, (payload) => {
|
|
12809
|
+
if (isSightObservationEvent(payload) && payload.conversationId === this.options.getConversationId()) {
|
|
12810
|
+
this.options.handleSightEvent?.(payload);
|
|
12811
|
+
}
|
|
12812
|
+
});
|
|
10566
12813
|
socket.on("conversations:turn", (payload) => {
|
|
10567
12814
|
let candidate;
|
|
10568
12815
|
try {
|
|
@@ -10591,6 +12838,96 @@ var ConversationSocket = class {
|
|
|
10591
12838
|
}
|
|
10592
12839
|
};
|
|
10593
12840
|
|
|
12841
|
+
// ../../clients/web-client/src/input-mode-rules.ts
|
|
12842
|
+
var InputModeRules = class {
|
|
12843
|
+
configured;
|
|
12844
|
+
resolved = null;
|
|
12845
|
+
/** The character's voice input switch, after the character loads. */
|
|
12846
|
+
speechInput = null;
|
|
12847
|
+
latestOptions = [];
|
|
12848
|
+
/**
|
|
12849
|
+
* Throws an `InputModeError` at once for a malformed configuration, and
|
|
12850
|
+
* for a baked asset with a mode other than `options`.
|
|
12851
|
+
*/
|
|
12852
|
+
constructor(inputModes, bakedAsset) {
|
|
12853
|
+
this.configured = parseInputModes(inputModes);
|
|
12854
|
+
if (this.configured && bakedAsset) {
|
|
12855
|
+
resolveInputModes(this.configured, {
|
|
12856
|
+
allowCustomReply: false,
|
|
12857
|
+
baked: true,
|
|
12858
|
+
speechInput: false
|
|
12859
|
+
});
|
|
12860
|
+
}
|
|
12861
|
+
}
|
|
12862
|
+
/**
|
|
12863
|
+
* The character loaded. Throws an `InputModeError` when the configuration
|
|
12864
|
+
* has a mode that the character does not permit.
|
|
12865
|
+
*/
|
|
12866
|
+
load(permissions) {
|
|
12867
|
+
this.speechInput = permissions.speechInput;
|
|
12868
|
+
this.resolved = resolveInputModes(this.configured, permissions);
|
|
12869
|
+
}
|
|
12870
|
+
/**
|
|
12871
|
+
* The input modes in use: the configured modes, or every mode that the
|
|
12872
|
+
* character permits. Before the character loads, the configuration or all
|
|
12873
|
+
* modes.
|
|
12874
|
+
*/
|
|
12875
|
+
get modes() {
|
|
12876
|
+
return this.resolved ?? this.configured ?? permittedInputModes({
|
|
12877
|
+
allowCustomReply: true,
|
|
12878
|
+
baked: false,
|
|
12879
|
+
speechInput: true
|
|
12880
|
+
});
|
|
12881
|
+
}
|
|
12882
|
+
/** The reply options of the latest result, for `assertReply`. */
|
|
12883
|
+
setOptions(options) {
|
|
12884
|
+
this.latestOptions = options;
|
|
12885
|
+
}
|
|
12886
|
+
/** Throws when the input modes do not include `mode`. */
|
|
12887
|
+
assertMode(mode, baked) {
|
|
12888
|
+
if (mode === "voice" && baked) {
|
|
12889
|
+
throw new InputModeError(
|
|
12890
|
+
"Voice input is not available in a baked (offline) conversation."
|
|
12891
|
+
);
|
|
12892
|
+
}
|
|
12893
|
+
if (mode === "voice" && this.speechInput === false) {
|
|
12894
|
+
throw new InputModeError(
|
|
12895
|
+
"Voice input is off for this character. Its owner turns it on in the STT tab."
|
|
12896
|
+
);
|
|
12897
|
+
}
|
|
12898
|
+
const modes = this.modes;
|
|
12899
|
+
if (!modes.includes(mode)) {
|
|
12900
|
+
throw new InputModeError(
|
|
12901
|
+
`The "${mode}" input mode is off: inputModes is ${JSON.stringify(modes)}.`
|
|
12902
|
+
);
|
|
12903
|
+
}
|
|
12904
|
+
}
|
|
12905
|
+
/**
|
|
12906
|
+
* A typed reply needs the `text` mode; one of the current reply options
|
|
12907
|
+
* needs `options` or `text`. Without a configuration the server's rules
|
|
12908
|
+
* apply alone, as before.
|
|
12909
|
+
*/
|
|
12910
|
+
assertReply(message) {
|
|
12911
|
+
if (!this.configured) {
|
|
12912
|
+
return;
|
|
12913
|
+
}
|
|
12914
|
+
const modes = this.modes;
|
|
12915
|
+
if (modes.includes("text")) {
|
|
12916
|
+
return;
|
|
12917
|
+
}
|
|
12918
|
+
const reply = message.trim();
|
|
12919
|
+
const isOption = this.latestOptions.some(
|
|
12920
|
+
(option) => option.label.trim() === reply
|
|
12921
|
+
);
|
|
12922
|
+
if (isOption && modes.includes("options")) {
|
|
12923
|
+
return;
|
|
12924
|
+
}
|
|
12925
|
+
throw new InputModeError(
|
|
12926
|
+
isOption ? `Reply options are off: inputModes is ${JSON.stringify(modes)}.` : `Typed replies are off: inputModes is ${JSON.stringify(modes)}. Send one of the reply options or use startListening().`
|
|
12927
|
+
);
|
|
12928
|
+
}
|
|
12929
|
+
};
|
|
12930
|
+
|
|
10594
12931
|
// ../../clients/web-client/src/voice/push-to-talk.ts
|
|
10595
12932
|
var INTERACTIVE = 'input, textarea, select, button, a[href], summary, [contenteditable]:not([contenteditable="false"]), [role="button"], [role="textbox"], [role="checkbox"], [role="switch"], [role="menuitem"], [role="tab"], [role="slider"], [role="option"]';
|
|
10596
12933
|
var elementOf = function(target) {
|
|
@@ -11077,8 +13414,8 @@ var MicrophoneCapture = class {
|
|
|
11077
13414
|
};
|
|
11078
13415
|
|
|
11079
13416
|
// ../../clients/web-client/src/voice/silence-detector.ts
|
|
11080
|
-
var
|
|
11081
|
-
var
|
|
13417
|
+
var MIN_SPEECH_LEVEL2 = 0.015;
|
|
13418
|
+
var SPEECH_TO_NOISE2 = 3;
|
|
11082
13419
|
var SPEECH_START_MS = 200;
|
|
11083
13420
|
var SilenceDetector = class {
|
|
11084
13421
|
constructor(silenceMs) {
|
|
@@ -11091,8 +13428,8 @@ var SilenceDetector = class {
|
|
|
11091
13428
|
/** True when the speech has ended. */
|
|
11092
13429
|
push(level, frameMs = SPEECH_FRAME_MS) {
|
|
11093
13430
|
const threshold = Math.max(
|
|
11094
|
-
|
|
11095
|
-
(this.floor ??
|
|
13431
|
+
MIN_SPEECH_LEVEL2,
|
|
13432
|
+
(this.floor ?? MIN_SPEECH_LEVEL2 / SPEECH_TO_NOISE2) * SPEECH_TO_NOISE2
|
|
11096
13433
|
);
|
|
11097
13434
|
if (level >= threshold) {
|
|
11098
13435
|
this.loudMs += frameMs;
|
|
@@ -11231,6 +13568,7 @@ var VoiceInput = class {
|
|
|
11231
13568
|
send: options.send !== false,
|
|
11232
13569
|
silent: options.silent === true,
|
|
11233
13570
|
renderSessionId: this.deps.renderSessionId(),
|
|
13571
|
+
timeZone: detectClientTimeZone(),
|
|
11234
13572
|
format: SPEECH_AUDIO_FORMAT
|
|
11235
13573
|
})
|
|
11236
13574
|
),
|
|
@@ -11383,6 +13721,7 @@ var VoiceInput = class {
|
|
|
11383
13721
|
if (session !== this.session || session.captureClosed) {
|
|
11384
13722
|
return;
|
|
11385
13723
|
}
|
|
13724
|
+
this.deps.activity?.pushVoiceLevel(frame.level);
|
|
11386
13725
|
const audio = new Uint8Array(
|
|
11387
13726
|
frame.pcm.buffer,
|
|
11388
13727
|
frame.pcm.byteOffset,
|
|
@@ -11528,6 +13867,7 @@ var VoiceInput = class {
|
|
|
11528
13867
|
setState(state) {
|
|
11529
13868
|
if (this.state !== state) {
|
|
11530
13869
|
this.state = state;
|
|
13870
|
+
this.deps.activity?.setListening(state === "listening");
|
|
11531
13871
|
this.deps.onState(state);
|
|
11532
13872
|
}
|
|
11533
13873
|
}
|
|
@@ -11546,15 +13886,14 @@ var Character = class extends CharacterRuntime {
|
|
|
11546
13886
|
/** The conversation whose live motion was warmed up. */
|
|
11547
13887
|
liveMotionPrepared = null;
|
|
11548
13888
|
authoring;
|
|
13889
|
+
hostedWebChat;
|
|
13890
|
+
memoryIdentifier;
|
|
11549
13891
|
normalizeIdleOrientation;
|
|
11550
13892
|
bakedAsset;
|
|
11551
13893
|
animationRefreshId = 0;
|
|
11552
13894
|
voice;
|
|
11553
|
-
|
|
11554
|
-
|
|
11555
|
-
/** The character's voice input switch, after the character loads. */
|
|
11556
|
-
speechInput = null;
|
|
11557
|
-
latestOptions = [];
|
|
13895
|
+
sight;
|
|
13896
|
+
inputs;
|
|
11558
13897
|
listeningStateCallback = null;
|
|
11559
13898
|
transcriptCallback = null;
|
|
11560
13899
|
connectionClosedCallback = null;
|
|
@@ -11575,17 +13914,15 @@ var Character = class extends CharacterRuntime {
|
|
|
11575
13914
|
})
|
|
11576
13915
|
});
|
|
11577
13916
|
this.authoring = options.authoring ?? false;
|
|
13917
|
+
this.hostedWebChat = options.hostedWebChat === true && !resolvedToken;
|
|
13918
|
+
this.memoryIdentifier = normalizeMemoryIdentifier(options.memoryIdentifier);
|
|
11578
13919
|
this.normalizeIdleOrientation = options.normalizeIdleOrientation ?? false;
|
|
11579
13920
|
this.bakedAsset = options.bakedConversation;
|
|
11580
13921
|
this.initialAssetInfo = options.initialAssetInfo ?? null;
|
|
11581
|
-
this.
|
|
11582
|
-
|
|
11583
|
-
|
|
11584
|
-
|
|
11585
|
-
baked: true,
|
|
11586
|
-
speechInput: false
|
|
11587
|
-
});
|
|
11588
|
-
}
|
|
13922
|
+
this.inputs = new InputModeRules(
|
|
13923
|
+
options.inputModes,
|
|
13924
|
+
Boolean(this.bakedAsset) && !this.authoring
|
|
13925
|
+
);
|
|
11589
13926
|
this.conversationSocket = new ConversationSocket({
|
|
11590
13927
|
baseUrl: this.baseUrl,
|
|
11591
13928
|
apiToken: this.apiToken,
|
|
@@ -11601,17 +13938,23 @@ var Character = class extends CharacterRuntime {
|
|
|
11601
13938
|
},
|
|
11602
13939
|
handleCallbackError: this.handleCallbackError,
|
|
11603
13940
|
handleSpeechEvent: (event) => this.voice.handleSocketEvent(event),
|
|
13941
|
+
handleSightEvent: (event) => this.sight.handleObservation(event),
|
|
11604
13942
|
handleConnectionClosed: (event) => invokeSafeCallback(
|
|
11605
13943
|
"onConnectionClosed",
|
|
11606
13944
|
this.handleCallbackError,
|
|
11607
13945
|
() => this.connectionClosedCallback?.(event)
|
|
11608
13946
|
)
|
|
11609
13947
|
});
|
|
13948
|
+
this.sight = new SightInput({
|
|
13949
|
+
socket: () => this.conversationSocket,
|
|
13950
|
+
conversationId: () => this.conversationId,
|
|
13951
|
+
handleCallbackError: this.handleCallbackError
|
|
13952
|
+
});
|
|
11610
13953
|
this.voice = new VoiceInput({
|
|
11611
13954
|
socket: () => this.conversationSocket,
|
|
11612
13955
|
conversationId: () => this.conversationId,
|
|
11613
13956
|
renderSessionId: () => this.renderSessionId,
|
|
11614
|
-
assertVoiceAllowed: () => this.
|
|
13957
|
+
assertVoiceAllowed: () => this.inputs.assertMode("voice", this.baked !== null),
|
|
11615
13958
|
turnInFlight: () => this.liveConversation.inFlight,
|
|
11616
13959
|
beginTurn: () => {
|
|
11617
13960
|
if (this.conversationId) {
|
|
@@ -11639,6 +13982,7 @@ var Character = class extends CharacterRuntime {
|
|
|
11639
13982
|
);
|
|
11640
13983
|
}
|
|
11641
13984
|
},
|
|
13985
|
+
activity: this.playerActivityTracker,
|
|
11642
13986
|
onTranscript: (event) => {
|
|
11643
13987
|
if (this.transcriptCallback) {
|
|
11644
13988
|
invokeSafeCallback(
|
|
@@ -11655,6 +13999,7 @@ var Character = class extends CharacterRuntime {
|
|
|
11655
13999
|
telemetry: this.telemetry,
|
|
11656
14000
|
socket: () => this.conversationSocket,
|
|
11657
14001
|
renderSession: () => this.renderSessionId,
|
|
14002
|
+
memoryIdentifier: () => this.apiToken ? this.memoryIdentifier : void 0,
|
|
11658
14003
|
thinking: () => this.invokeThinkingCallback({ taskId: null, stagingId: null }),
|
|
11659
14004
|
handleResult: (payload, forcePlayback, silent) => this.handleStagingResult(payload, {
|
|
11660
14005
|
source: "api",
|
|
@@ -11675,12 +14020,13 @@ var Character = class extends CharacterRuntime {
|
|
|
11675
14020
|
const progressReporter = createAssetProgressReporter(callback);
|
|
11676
14021
|
const info = await this.getAssetInfo();
|
|
11677
14022
|
const baked = !this.authoring && Boolean(this.bakedAsset || info.bakedConversationEnabled);
|
|
11678
|
-
this.
|
|
11679
|
-
this.
|
|
14023
|
+
this.variantController.configureVariants(info.variants, { remote: false });
|
|
14024
|
+
this.inputs.load({
|
|
11680
14025
|
allowCustomReply: info.allowCustomReply !== false,
|
|
11681
14026
|
baked,
|
|
11682
|
-
speechInput:
|
|
14027
|
+
speechInput: info.speechInputEnabled === true
|
|
11683
14028
|
});
|
|
14029
|
+
this.sight.configure({ enabled: info.sight?.enabled === true, baked });
|
|
11684
14030
|
this.baked = baked ? await loadBakedConversation({
|
|
11685
14031
|
baseUrl: this.baseUrl,
|
|
11686
14032
|
characterId: this.characterId,
|
|
@@ -11698,8 +14044,10 @@ var Character = class extends CharacterRuntime {
|
|
|
11698
14044
|
onProgress: progressReporter.updateResource,
|
|
11699
14045
|
assetProfile: this.assetProfile
|
|
11700
14046
|
}),
|
|
11701
|
-
this.baked ? this.lazyAnimations.ensureMany(animationIds) : info.idleAnimationId ? this.lazyAnimations.ensure(info.idleAnimationId) : Promise.resolve()
|
|
14047
|
+
this.baked ? this.lazyAnimations.ensureMany(animationIds) : info.idleAnimationId ? this.lazyAnimations.ensure(info.idleAnimationId) : Promise.resolve(),
|
|
14048
|
+
this.baked ? this.variantController.preloadVariants(true) : Promise.resolve()
|
|
11702
14049
|
]);
|
|
14050
|
+
this.variantController.registerDefaultVrm(vrm);
|
|
11703
14051
|
this.vrm = vrm;
|
|
11704
14052
|
if (this.performanceRecorder.isRecording) {
|
|
11705
14053
|
this.facialMixer.capturePerformanceModel();
|
|
@@ -11717,6 +14065,9 @@ var Character = class extends CharacterRuntime {
|
|
|
11717
14065
|
});
|
|
11718
14066
|
}
|
|
11719
14067
|
animationController.playIdle(0);
|
|
14068
|
+
if (!this.baked) {
|
|
14069
|
+
void this.variantController.preloadVariants(false);
|
|
14070
|
+
}
|
|
11720
14071
|
}
|
|
11721
14072
|
async reloadAnimations(info) {
|
|
11722
14073
|
const requestId = ++this.animationRefreshId;
|
|
@@ -11750,11 +14101,15 @@ var Character = class extends CharacterRuntime {
|
|
|
11750
14101
|
if (info.bakedConversationEnabled || this.bakedAsset) {
|
|
11751
14102
|
throw new Error("Baked conversation requires local rendering.");
|
|
11752
14103
|
}
|
|
11753
|
-
this.
|
|
11754
|
-
this.
|
|
14104
|
+
this.variantController.configureVariants(info.variants, { remote: true });
|
|
14105
|
+
this.inputs.load({
|
|
11755
14106
|
allowCustomReply: info.allowCustomReply !== false,
|
|
11756
14107
|
baked: false,
|
|
11757
|
-
speechInput:
|
|
14108
|
+
speechInput: info.speechInputEnabled === true
|
|
14109
|
+
});
|
|
14110
|
+
this.sight.configure({
|
|
14111
|
+
enabled: info.sight?.enabled === true,
|
|
14112
|
+
baked: false
|
|
11758
14113
|
});
|
|
11759
14114
|
this.animationController = new RemoteAnimationController();
|
|
11760
14115
|
}
|
|
@@ -11788,7 +14143,9 @@ var Character = class extends CharacterRuntime {
|
|
|
11788
14143
|
baseUrl: this.baseUrl,
|
|
11789
14144
|
apiToken: this.apiToken,
|
|
11790
14145
|
avatarId: this.characterId,
|
|
11791
|
-
renderSessionId: this.renderSessionId
|
|
14146
|
+
renderSessionId: this.renderSessionId,
|
|
14147
|
+
hostedWebChat: this.hostedWebChat,
|
|
14148
|
+
memoryIdentifier: this.apiToken ? this.memoryIdentifier : void 0
|
|
11792
14149
|
});
|
|
11793
14150
|
if (!response.conversation?.id) {
|
|
11794
14151
|
throw new Error("Failed to create conversation.");
|
|
@@ -11818,6 +14175,7 @@ var Character = class extends CharacterRuntime {
|
|
|
11818
14175
|
}
|
|
11819
14176
|
async disconnect() {
|
|
11820
14177
|
this.voice.reset();
|
|
14178
|
+
this.sight.stop();
|
|
11821
14179
|
this.thinkingState = false;
|
|
11822
14180
|
if (this.animationController) {
|
|
11823
14181
|
this.streamPlayback.cleanup();
|
|
@@ -11839,7 +14197,7 @@ var Character = class extends CharacterRuntime {
|
|
|
11839
14197
|
await this.endConversation();
|
|
11840
14198
|
if (options?.disposeVrm === false) {
|
|
11841
14199
|
this.detach();
|
|
11842
|
-
this.resetRuntimeAssets();
|
|
14200
|
+
this.resetRuntimeAssets({ disposeVrms: false });
|
|
11843
14201
|
this.audioPlayer?.cleanup();
|
|
11844
14202
|
this.audioPlayer = null;
|
|
11845
14203
|
return;
|
|
@@ -11853,7 +14211,7 @@ var Character = class extends CharacterRuntime {
|
|
|
11853
14211
|
maybeMessageOrOptions,
|
|
11854
14212
|
maybeOptions
|
|
11855
14213
|
);
|
|
11856
|
-
this.
|
|
14214
|
+
this.inputs.assertReply(message);
|
|
11857
14215
|
if (this.voice.turnPending) {
|
|
11858
14216
|
throw new Error("A reply is already being generated. Please wait.");
|
|
11859
14217
|
}
|
|
@@ -11901,11 +14259,7 @@ var Character = class extends CharacterRuntime {
|
|
|
11901
14259
|
* character permits. Before `loadAssets`, the configuration or all modes.
|
|
11902
14260
|
*/
|
|
11903
14261
|
get inputModes() {
|
|
11904
|
-
return this.
|
|
11905
|
-
allowCustomReply: true,
|
|
11906
|
-
baked: false,
|
|
11907
|
-
speechInput: true
|
|
11908
|
-
});
|
|
14262
|
+
return this.inputs.modes;
|
|
11909
14263
|
}
|
|
11910
14264
|
get listeningState() {
|
|
11911
14265
|
return this.voice.listeningState;
|
|
@@ -11949,6 +14303,35 @@ var Character = class extends CharacterRuntime {
|
|
|
11949
14303
|
callback
|
|
11950
14304
|
);
|
|
11951
14305
|
}
|
|
14306
|
+
/**
|
|
14307
|
+
* `'unavailable'` until the character info turns sight on (its owner's
|
|
14308
|
+
* Sight tab), and in a baked conversation; then `'off'`, `'starting'` or
|
|
14309
|
+
* `'on'`.
|
|
14310
|
+
*/
|
|
14311
|
+
get sightState() {
|
|
14312
|
+
return this.sight.state;
|
|
14313
|
+
}
|
|
14314
|
+
/**
|
|
14315
|
+
* Lets the character see the player's camera (docs/sight.md). Needs a
|
|
14316
|
+
* conversation; the first call asks for the camera permission. Rejects
|
|
14317
|
+
* with a `SightError`: `SIGHT_DISABLED` when sight is unavailable,
|
|
14318
|
+
* `CONVERSATION_REQUIRED`, `CAMERA_PERMISSION_DENIED`, ...
|
|
14319
|
+
*/
|
|
14320
|
+
startSight(options) {
|
|
14321
|
+
return this.sight.start(options);
|
|
14322
|
+
}
|
|
14323
|
+
/** Stops the frames. Closes the camera that `startSight` opened. */
|
|
14324
|
+
stopSight() {
|
|
14325
|
+
this.sight.stop();
|
|
14326
|
+
}
|
|
14327
|
+
/** What the character saw: each observation of the current conversation. */
|
|
14328
|
+
onSight(callback) {
|
|
14329
|
+
return this.sight.onObservation(callback);
|
|
14330
|
+
}
|
|
14331
|
+
/** The sight state. `error` is the `SightError` that stopped sight. */
|
|
14332
|
+
onSightState(callback) {
|
|
14333
|
+
return this.sight.onState(callback);
|
|
14334
|
+
}
|
|
11952
14335
|
/**
|
|
11953
14336
|
* The server closed the conversation connection: a refused or failed join,
|
|
11954
14337
|
* or an API key that was rotated, edited or deleted. The next `connect`,
|
|
@@ -11995,6 +14378,15 @@ var Character = class extends CharacterRuntime {
|
|
|
11995
14378
|
setRenderSessionId(renderSessionId) {
|
|
11996
14379
|
this.renderSessionId = renderSessionId?.trim() || null;
|
|
11997
14380
|
}
|
|
14381
|
+
setHostedWebChat(enabled) {
|
|
14382
|
+
this.hostedWebChat = enabled === true && !this.apiToken;
|
|
14383
|
+
}
|
|
14384
|
+
setMemoryIdentifier(memoryIdentifier) {
|
|
14385
|
+
this.memoryIdentifier = normalizeMemoryIdentifier(memoryIdentifier);
|
|
14386
|
+
}
|
|
14387
|
+
getMemoryIdentifier() {
|
|
14388
|
+
return this.memoryIdentifier;
|
|
14389
|
+
}
|
|
11998
14390
|
onStagingResult(callback) {
|
|
11999
14391
|
return bindCallbackSlot(
|
|
12000
14392
|
() => this.stagingResultCallback,
|
|
@@ -12024,76 +14416,22 @@ var Character = class extends CharacterRuntime {
|
|
|
12024
14416
|
};
|
|
12025
14417
|
async handleStagingResult(payload, options) {
|
|
12026
14418
|
this.thinkingState = false;
|
|
12027
|
-
|
|
12028
|
-
|
|
12029
|
-
|
|
12030
|
-
this.
|
|
12031
|
-
|
|
12032
|
-
|
|
12033
|
-
|
|
12034
|
-
|
|
12035
|
-
|
|
12036
|
-
|
|
12037
|
-
|
|
12038
|
-
|
|
14419
|
+
await deliverStagingResult(payload, {
|
|
14420
|
+
...options,
|
|
14421
|
+
notifiedStagingIds: this.notifiedStagingIds,
|
|
14422
|
+
hasPlayed: (stagingId) => this.streamPlayback.hasPlayed(stagingId),
|
|
14423
|
+
setOptions: (next) => this.inputs.setOptions(next),
|
|
14424
|
+
onTurnReceived: (normalized) => this.liveConversation.received(normalized),
|
|
14425
|
+
onStagingResult: this.stagingResultCallback,
|
|
14426
|
+
onCallbackError: this.handleCallbackError,
|
|
14427
|
+
recordEvent: (event) => this.performanceRecorder.recordEvent(event),
|
|
14428
|
+
playRequested: (normalized, playOptions) => {
|
|
14429
|
+
void this.playStaging(normalized, playOptions);
|
|
14430
|
+
},
|
|
14431
|
+
playStreamed: (normalized) => {
|
|
14432
|
+
void this.streamPlayback.playPayload(normalized);
|
|
12039
14433
|
}
|
|
12040
|
-
}
|
|
12041
|
-
const shouldPlay = !this.streamPlayback.hasPlayed(normalized.stagingId) || force;
|
|
12042
|
-
this.performanceRecorder.recordEvent({
|
|
12043
|
-
type: "staging.result",
|
|
12044
|
-
source: options.source,
|
|
12045
|
-
stagingId: normalized.stagingId,
|
|
12046
|
-
taskId: normalized.taskId ?? "",
|
|
12047
|
-
playback: shouldPlay ? "requested" : "already-streamed"
|
|
12048
14434
|
});
|
|
12049
|
-
if (shouldPlay) {
|
|
12050
|
-
void this.playStaging(normalized, { force, silent: options.silent });
|
|
12051
|
-
} else {
|
|
12052
|
-
void this.streamPlayback.playPayload(normalized);
|
|
12053
|
-
}
|
|
12054
|
-
}
|
|
12055
|
-
/** Throws when the input modes do not include `mode`. */
|
|
12056
|
-
assertInputMode(mode) {
|
|
12057
|
-
if (mode === "voice" && this.baked) {
|
|
12058
|
-
throw new InputModeError(
|
|
12059
|
-
"Voice input is not available in a baked (offline) conversation."
|
|
12060
|
-
);
|
|
12061
|
-
}
|
|
12062
|
-
if (mode === "voice" && this.speechInput === false) {
|
|
12063
|
-
throw new InputModeError(
|
|
12064
|
-
"Voice input is off for this character. Its owner turns it on in the STT tab."
|
|
12065
|
-
);
|
|
12066
|
-
}
|
|
12067
|
-
const modes = this.inputModes;
|
|
12068
|
-
if (!modes.includes(mode)) {
|
|
12069
|
-
throw new InputModeError(
|
|
12070
|
-
`The "${mode}" input mode is off: inputModes is ${JSON.stringify(modes)}.`
|
|
12071
|
-
);
|
|
12072
|
-
}
|
|
12073
|
-
}
|
|
12074
|
-
/**
|
|
12075
|
-
* A typed reply needs the `text` mode; one of the current reply options
|
|
12076
|
-
* needs `options` or `text`. Without a configuration the server's rules
|
|
12077
|
-
* apply alone, as before.
|
|
12078
|
-
*/
|
|
12079
|
-
assertReplyMode(message) {
|
|
12080
|
-
if (!this.configuredInputModes) {
|
|
12081
|
-
return;
|
|
12082
|
-
}
|
|
12083
|
-
const modes = this.inputModes;
|
|
12084
|
-
if (modes.includes("text")) {
|
|
12085
|
-
return;
|
|
12086
|
-
}
|
|
12087
|
-
const reply = message.trim();
|
|
12088
|
-
const isOption = this.latestOptions.some(
|
|
12089
|
-
(option) => option.label.trim() === reply
|
|
12090
|
-
);
|
|
12091
|
-
if (isOption && modes.includes("options")) {
|
|
12092
|
-
return;
|
|
12093
|
-
}
|
|
12094
|
-
throw new InputModeError(
|
|
12095
|
-
isOption ? `Reply options are off: inputModes is ${JSON.stringify(modes)}.` : `Typed replies are off: inputModes is ${JSON.stringify(modes)}. Send one of the reply options or use startListening().`
|
|
12096
|
-
);
|
|
12097
14435
|
}
|
|
12098
14436
|
invokeThinkingCallback(payload) {
|
|
12099
14437
|
this.thinkingState = true;
|
|
@@ -12604,6 +14942,7 @@ var mountNpcify = async function(options) {
|
|
|
12604
14942
|
camera: null,
|
|
12605
14943
|
renderer: null,
|
|
12606
14944
|
controls: null,
|
|
14945
|
+
setMemoryIdentifier: (memoryIdentifier) => npcify.setMemoryIdentifier(memoryIdentifier),
|
|
12607
14946
|
dispose: async () => {
|
|
12608
14947
|
remote.dispose();
|
|
12609
14948
|
await npcify.dispose();
|
|
@@ -12617,6 +14956,7 @@ var mountNpcify = async function(options) {
|
|
|
12617
14956
|
let context2;
|
|
12618
14957
|
let session;
|
|
12619
14958
|
let observer;
|
|
14959
|
+
let unsubscribeVrm;
|
|
12620
14960
|
let frame = 0;
|
|
12621
14961
|
let disposed = false;
|
|
12622
14962
|
const dispose = async function() {
|
|
@@ -12626,10 +14966,11 @@ var mountNpcify = async function(options) {
|
|
|
12626
14966
|
disposed = true;
|
|
12627
14967
|
cancelAnimationFrame(frame);
|
|
12628
14968
|
observer?.disconnect();
|
|
12629
|
-
|
|
14969
|
+
unsubscribeVrm?.();
|
|
12630
14970
|
if (context2 && session) {
|
|
12631
|
-
detachVrm(context2, session, { dispose:
|
|
14971
|
+
detachVrm(context2, session, { dispose: false });
|
|
12632
14972
|
}
|
|
14973
|
+
await npcify.dispose();
|
|
12633
14974
|
if (context2) {
|
|
12634
14975
|
disposeSceneContext(context2);
|
|
12635
14976
|
}
|
|
@@ -12641,6 +14982,16 @@ var mountNpcify = async function(options) {
|
|
|
12641
14982
|
}
|
|
12642
14983
|
context2 = createSceneContext({ canvas: options.canvas });
|
|
12643
14984
|
session = attachVrm(context2, npcify.vrm);
|
|
14985
|
+
unsubscribeVrm = npcify.on("vrmChange", (nextVrm) => {
|
|
14986
|
+
if (context2) {
|
|
14987
|
+
setCameraCollisionBackend(
|
|
14988
|
+
context2,
|
|
14989
|
+
new HumanoidCapsuleCollisionBackend(
|
|
14990
|
+
(name) => nextVrm.humanoid.getRawBoneNode(name)
|
|
14991
|
+
)
|
|
14992
|
+
);
|
|
14993
|
+
}
|
|
14994
|
+
});
|
|
12644
14995
|
applyCameraPreset(context2, "full_body");
|
|
12645
14996
|
npcify.setGazeTarget(context2.camera);
|
|
12646
14997
|
const resize = () => {
|
|
@@ -12674,6 +15025,7 @@ var mountNpcify = async function(options) {
|
|
|
12674
15025
|
camera: context2.camera,
|
|
12675
15026
|
renderer: context2.renderer,
|
|
12676
15027
|
controls: context2.controls,
|
|
15028
|
+
setMemoryIdentifier: (memoryIdentifier) => npcify.setMemoryIdentifier(memoryIdentifier),
|
|
12677
15029
|
dispose
|
|
12678
15030
|
};
|
|
12679
15031
|
} catch (error) {
|
|
@@ -12692,25 +15044,32 @@ export {
|
|
|
12692
15044
|
CharacterRuntime,
|
|
12693
15045
|
CharacterSceneRig,
|
|
12694
15046
|
ConversationConnectionError,
|
|
15047
|
+
DEFAULT_VARIANT_KEY,
|
|
12695
15048
|
FacialPerformanceLayer,
|
|
12696
15049
|
FacialPerformanceMixer,
|
|
12697
15050
|
INPUT_MODES,
|
|
12698
15051
|
InputModeError,
|
|
12699
15052
|
LiveMotionPlayer,
|
|
12700
15053
|
MicrophoneError,
|
|
15054
|
+
NUDE_VARIANT_KEY,
|
|
12701
15055
|
Npcify,
|
|
12702
15056
|
Character as NpcifyCharacter,
|
|
12703
15057
|
PERFORMANCE_FILE_EXTENSION,
|
|
12704
15058
|
PERFORMANCE_MEDIA_TYPE,
|
|
12705
15059
|
PerformanceRecordingSchemaVersion,
|
|
12706
15060
|
RemoteAnimationController,
|
|
15061
|
+
SightError,
|
|
12707
15062
|
SpeechInputError,
|
|
12708
15063
|
analyzePerformanceRecording,
|
|
12709
15064
|
bindPushToTalk,
|
|
12710
15065
|
createCharacter,
|
|
12711
15066
|
decodePerformance,
|
|
12712
15067
|
encodePerformance,
|
|
15068
|
+
fetchVariantVrmDownloadUrl,
|
|
15069
|
+
fetchVrmDownloadUrl,
|
|
12713
15070
|
loadBakedConversation,
|
|
12714
15071
|
loadCharacter,
|
|
15072
|
+
loadVariantVrm,
|
|
15073
|
+
loadVrm,
|
|
12715
15074
|
mountNpcify
|
|
12716
15075
|
};
|