@npcify/web 0.5.0 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -0
- package/dist/index.d.ts +346 -44
- package/dist/index.js +3010 -279
- package/package.json +3 -3
package/dist/index.js
CHANGED
|
@@ -34,6 +34,33 @@ var isValidMotionFramesEvent = function(event) {
|
|
|
34
34
|
// ../ts-shared/src/llm.ts
|
|
35
35
|
var STAGING_PROTOCOL_VERSION = 1;
|
|
36
36
|
|
|
37
|
+
// ../ts-shared/src/variants.ts
|
|
38
|
+
var DEFAULT_VARIANT_KEY = "default";
|
|
39
|
+
var NUDE_VARIANT_KEY = "nude";
|
|
40
|
+
var VARIANT_KEY_PATTERN = /^[a-z][a-z0-9_-]{0,39}$/u;
|
|
41
|
+
var isValidVariantKey = function(value) {
|
|
42
|
+
return typeof value === "string" && VARIANT_KEY_PATTERN.test(value);
|
|
43
|
+
};
|
|
44
|
+
var STAGING_TOOL_RESULT_STATUSES = [
|
|
45
|
+
"applied",
|
|
46
|
+
"noop",
|
|
47
|
+
"rejected"
|
|
48
|
+
];
|
|
49
|
+
var isStagingToolCall = function(value) {
|
|
50
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
51
|
+
return false;
|
|
52
|
+
}
|
|
53
|
+
const candidate = value;
|
|
54
|
+
if (typeof candidate.id !== "string" || !candidate.id.trim() || candidate.name !== "switch_variant" || typeof candidate.timestamp !== "number" || !Number.isFinite(candidate.timestamp) || candidate.timestamp < 0 || !candidate.arguments || typeof candidate.arguments !== "object" || Array.isArray(candidate.arguments) || !candidate.result || typeof candidate.result !== "object" || Array.isArray(candidate.result)) {
|
|
55
|
+
return false;
|
|
56
|
+
}
|
|
57
|
+
const args = candidate.arguments;
|
|
58
|
+
const result = candidate.result;
|
|
59
|
+
return isValidVariantKey(args.variant) && typeof result.ok === "boolean" && STAGING_TOOL_RESULT_STATUSES.includes(
|
|
60
|
+
String(result.status)
|
|
61
|
+
) && isValidVariantKey(result.previousVariant) && isValidVariantKey(result.activeVariant) && (result.error === void 0 || typeof result.error === "string");
|
|
62
|
+
};
|
|
63
|
+
|
|
37
64
|
// ../ts-shared/src/server-rendering.ts
|
|
38
65
|
var SERVER_RENDER_PROTOCOL_VERSION = 3;
|
|
39
66
|
var MAXIMUM_PLAYBACK_DURATION_SECONDS = 10 * 60;
|
|
@@ -42,7 +69,465 @@ var MAXIMUM_PLAYBACK_DURATION_SECONDS = 10 * 60;
|
|
|
42
69
|
var TTS_PROVIDERS = ["11labs", "gemini"];
|
|
43
70
|
var DEFAULT_TTS_PROVIDER = TTS_PROVIDERS[0];
|
|
44
71
|
|
|
72
|
+
// ../ts-shared/src/llm-providers/catalog.ts
|
|
73
|
+
var record = function(value) {
|
|
74
|
+
return value && typeof value === "object" && !Array.isArray(value) ? value : {};
|
|
75
|
+
};
|
|
76
|
+
var rate = function(value) {
|
|
77
|
+
if (typeof value !== "number" && typeof value !== "string") {
|
|
78
|
+
return null;
|
|
79
|
+
}
|
|
80
|
+
if (value === "") {
|
|
81
|
+
return null;
|
|
82
|
+
}
|
|
83
|
+
const result = Number(value);
|
|
84
|
+
return Number.isFinite(result) && result >= 0 ? result : null;
|
|
85
|
+
};
|
|
86
|
+
var catalogRows = function(payload) {
|
|
87
|
+
if (!Array.isArray(payload)) {
|
|
88
|
+
throw new Error("Provider returned an invalid model catalog");
|
|
89
|
+
}
|
|
90
|
+
return payload.map(record);
|
|
91
|
+
};
|
|
92
|
+
var parseCompatibleModels = function(payload) {
|
|
93
|
+
return catalogRows(record(payload).data).flatMap((row) => {
|
|
94
|
+
const pricing = record(row.pricing);
|
|
95
|
+
const input = rate(pricing.prompt);
|
|
96
|
+
const output = rate(pricing.completion);
|
|
97
|
+
const cached = rate(pricing.input_cache_read);
|
|
98
|
+
const write = rate(pricing.input_cache_write);
|
|
99
|
+
const modalities = record(row.architecture).output_modalities;
|
|
100
|
+
const parameters = row.supported_parameters;
|
|
101
|
+
if (typeof row.id !== "string" || !row.id || row.deprecated || input === null || output === null || Array.isArray(modalities) && (modalities.length !== 1 || modalities[0] !== "text") || Array.isArray(pricing.overrides) && pricing.overrides.length > 0 || Array.isArray(parameters) && !parameters.includes("structured_outputs") && !parameters.includes("response_format") || (rate(pricing.request) ?? 0) > 0) {
|
|
102
|
+
return [];
|
|
103
|
+
}
|
|
104
|
+
return [
|
|
105
|
+
{
|
|
106
|
+
id: row.id,
|
|
107
|
+
name: typeof row.name === "string" ? row.name : row.id,
|
|
108
|
+
inputUsd: input * 1e6,
|
|
109
|
+
outputUsd: output * 1e6,
|
|
110
|
+
cachedInputUsd: (cached ?? input) * 1e6,
|
|
111
|
+
...write !== null ? { cacheWriteUsd: write * 1e6 } : {},
|
|
112
|
+
contextLength: rate(row.max_tokens ?? row.context_length)
|
|
113
|
+
}
|
|
114
|
+
];
|
|
115
|
+
});
|
|
116
|
+
};
|
|
117
|
+
|
|
118
|
+
// ../ts-shared/src/llm-providers/usage.ts
|
|
119
|
+
var validUsd = function(value) {
|
|
120
|
+
return typeof value === "number" && Number.isFinite(value) && value >= 0;
|
|
121
|
+
};
|
|
122
|
+
var validateUsageCounts = function(usage) {
|
|
123
|
+
const counts = [
|
|
124
|
+
usage.prompt_tokens,
|
|
125
|
+
usage.completion_tokens,
|
|
126
|
+
usage.total_tokens,
|
|
127
|
+
// Providers can explicitly return null for unavailable cache counters.
|
|
128
|
+
usage.prompt_tokens_details?.cached_tokens ?? 0,
|
|
129
|
+
usage.prompt_tokens_details?.cache_write_tokens ?? 0
|
|
130
|
+
];
|
|
131
|
+
if (counts.some(
|
|
132
|
+
(value) => value !== void 0 && (!Number.isSafeInteger(value) || value < 0)
|
|
133
|
+
)) {
|
|
134
|
+
throw new Error("Provider returned invalid token usage");
|
|
135
|
+
}
|
|
136
|
+
const cached = usage.prompt_tokens_details?.cached_tokens ?? 0;
|
|
137
|
+
const written = usage.prompt_tokens_details?.cache_write_tokens ?? 0;
|
|
138
|
+
if (usage.prompt_tokens !== void 0 && cached + written > usage.prompt_tokens) {
|
|
139
|
+
throw new Error("Provider returned invalid cache usage");
|
|
140
|
+
}
|
|
141
|
+
};
|
|
142
|
+
var quoteTokenUsage = function(price, usage) {
|
|
143
|
+
validateUsageCounts(usage);
|
|
144
|
+
const input = usage.prompt_tokens;
|
|
145
|
+
const output = usage.completion_tokens;
|
|
146
|
+
if (input === void 0 || output === void 0) {
|
|
147
|
+
throw new Error("Provider did not report billable token usage");
|
|
148
|
+
}
|
|
149
|
+
const cached = usage.prompt_tokens_details?.cached_tokens ?? 0;
|
|
150
|
+
const written = usage.prompt_tokens_details?.cache_write_tokens ?? 0;
|
|
151
|
+
const writeRate = price.cacheWriteUsd ?? price.inputUsd;
|
|
152
|
+
if (![price.inputUsd, price.outputUsd, price.cachedInputUsd, writeRate].every(
|
|
153
|
+
validUsd
|
|
154
|
+
)) {
|
|
155
|
+
throw new Error("Provider returned invalid token prices");
|
|
156
|
+
}
|
|
157
|
+
return {
|
|
158
|
+
usd: ((input - cached - written) * price.inputUsd + cached * price.cachedInputUsd + written * writeRate + output * price.outputUsd) / 1e6,
|
|
159
|
+
source: "token-rates"
|
|
160
|
+
};
|
|
161
|
+
};
|
|
162
|
+
|
|
163
|
+
// ../ts-shared/src/llm-providers/baseten.ts
|
|
164
|
+
var baseten = {
|
|
165
|
+
name: "Baseten",
|
|
166
|
+
baseUrl: "https://inference.baseten.co/v1",
|
|
167
|
+
modelsUrl: "https://inference.baseten.co/v1/models",
|
|
168
|
+
billingMode: "token-rates",
|
|
169
|
+
parseModels: parseCompatibleModels,
|
|
170
|
+
requestOptions: () => ({}),
|
|
171
|
+
quoteUsage: quoteTokenUsage
|
|
172
|
+
};
|
|
173
|
+
|
|
174
|
+
// ../ts-shared/src/llm-providers/deepinfra.ts
|
|
175
|
+
var deepinfra = {
|
|
176
|
+
name: "DeepInfra",
|
|
177
|
+
keyCheckUrl: "https://api.deepinfra.com/v1/me",
|
|
178
|
+
baseUrl: "https://api.deepinfra.com/v1/openai",
|
|
179
|
+
modelsUrl: "https://api.deepinfra.com/models/list",
|
|
180
|
+
billingMode: "token-rates",
|
|
181
|
+
parseModels: (payload) => catalogRows(payload).flatMap((row) => {
|
|
182
|
+
const tags = Array.isArray(row.tags) ? row.tags : [];
|
|
183
|
+
const pricing = record(row.pricing);
|
|
184
|
+
const input = rate(pricing.cents_per_input_token);
|
|
185
|
+
const output = rate(pricing.cents_per_output_token);
|
|
186
|
+
if (typeof row.model_name !== "string" || !row.model_name || row.deprecated || row.type !== "text-generation" || !tags.includes("openai") || !tags.includes("structured-output") || pricing.type !== "tokens" || input === null || output === null) {
|
|
187
|
+
return [];
|
|
188
|
+
}
|
|
189
|
+
return [
|
|
190
|
+
{
|
|
191
|
+
id: row.model_name,
|
|
192
|
+
name: typeof row.name === "string" ? row.name : row.model_name,
|
|
193
|
+
inputUsd: input / 100 * 1e6,
|
|
194
|
+
outputUsd: output / 100 * 1e6,
|
|
195
|
+
cachedInputUsd: input / 100 * (rate(pricing.rate_per_input_token_cached) ?? 1) * 1e6,
|
|
196
|
+
contextLength: rate(row.max_tokens ?? row.context_length),
|
|
197
|
+
canDisableReasoning: tags.includes("can-disable-reasoning")
|
|
198
|
+
}
|
|
199
|
+
];
|
|
200
|
+
}),
|
|
201
|
+
requestOptions: ({ model, conversationId, legacyPriority }) => ({
|
|
202
|
+
...conversationId ? { prompt_cache_key: conversationId } : {},
|
|
203
|
+
...!model || model.canDisableReasoning ? { reasoning_effort: "none" } : {},
|
|
204
|
+
...legacyPriority ? { service_tier: "priority" } : {}
|
|
205
|
+
}),
|
|
206
|
+
quoteUsage: quoteTokenUsage
|
|
207
|
+
};
|
|
208
|
+
|
|
209
|
+
// ../ts-shared/src/llm-providers/gemini.ts
|
|
210
|
+
var GEMINI_MODELS = {
|
|
211
|
+
"gemini-3.8-flash": {
|
|
212
|
+
id: "gemini-3.8-flash",
|
|
213
|
+
name: "Gemini 3.8 Flash",
|
|
214
|
+
inputUsd: 0.75,
|
|
215
|
+
outputUsd: 3.75,
|
|
216
|
+
cachedInputUsd: 0.075,
|
|
217
|
+
contextLength: 1048576
|
|
218
|
+
}
|
|
219
|
+
};
|
|
220
|
+
var gemini = {
|
|
221
|
+
name: "Google Gemini",
|
|
222
|
+
baseUrl: "https://generativelanguage.googleapis.com/v1beta/openai",
|
|
223
|
+
modelsUrl: "https://generativelanguage.googleapis.com/v1beta/openai/models",
|
|
224
|
+
billingMode: "token-rates",
|
|
225
|
+
parseModels: (payload) => {
|
|
226
|
+
const seen = /* @__PURE__ */ new Set();
|
|
227
|
+
return catalogRows(record(payload).data).flatMap((row) => {
|
|
228
|
+
const rawId = typeof row.id === "string" ? row.id.trim() : "";
|
|
229
|
+
const id = rawId.replace(/^models\//, "");
|
|
230
|
+
const model = Object.hasOwn(GEMINI_MODELS, id) ? GEMINI_MODELS[id] : void 0;
|
|
231
|
+
if (!model || row.deprecated || seen.has(id)) {
|
|
232
|
+
return [];
|
|
233
|
+
}
|
|
234
|
+
seen.add(id);
|
|
235
|
+
return [{ ...model }];
|
|
236
|
+
});
|
|
237
|
+
},
|
|
238
|
+
requestOptions: () => ({ reasoning_effort: "low" }),
|
|
239
|
+
quoteUsage: (price, usage) => {
|
|
240
|
+
validateUsageCounts(usage);
|
|
241
|
+
if (usage.prompt_tokens !== void 0 && usage.completion_tokens !== void 0 && usage.total_tokens !== void 0 && usage.total_tokens < usage.prompt_tokens + usage.completion_tokens) {
|
|
242
|
+
throw new Error("Provider returned invalid token usage");
|
|
243
|
+
}
|
|
244
|
+
const outputTokens = usage.total_tokens !== void 0 && usage.prompt_tokens !== void 0 ? usage.total_tokens - usage.prompt_tokens : usage.completion_tokens;
|
|
245
|
+
return quoteTokenUsage(price, {
|
|
246
|
+
...usage,
|
|
247
|
+
completion_tokens: outputTokens
|
|
248
|
+
});
|
|
249
|
+
}
|
|
250
|
+
};
|
|
251
|
+
|
|
252
|
+
// ../ts-shared/src/llm-providers/openrouter.ts
|
|
253
|
+
var openrouter = {
|
|
254
|
+
name: "OpenRouter",
|
|
255
|
+
keyCheckUrl: "https://openrouter.ai/api/v1/key",
|
|
256
|
+
baseUrl: "https://openrouter.ai/api/v1",
|
|
257
|
+
modelsUrl: "https://openrouter.ai/api/v1/models",
|
|
258
|
+
billingMode: "reported-cost",
|
|
259
|
+
parseModels: parseCompatibleModels,
|
|
260
|
+
requestOptions: ({ conversationId }) => conversationId ? { session_id: conversationId } : {},
|
|
261
|
+
quoteUsage: (_price, usage) => {
|
|
262
|
+
validateUsageCounts(usage);
|
|
263
|
+
if (!validUsd(usage.cost)) {
|
|
264
|
+
throw new Error("OpenRouter did not report a valid account cost");
|
|
265
|
+
}
|
|
266
|
+
return { usd: usage.cost, source: "provider-reported" };
|
|
267
|
+
},
|
|
268
|
+
needsUsageRecovery: (usage) => usage.cost === void 0 || usage.cost === null,
|
|
269
|
+
recoverUsage: async (usage, apiKey) => {
|
|
270
|
+
const id = usage.billing?.generationId;
|
|
271
|
+
if (!id) {
|
|
272
|
+
throw new Error("OpenRouter generation ID is missing");
|
|
273
|
+
}
|
|
274
|
+
const url = new URL("https://openrouter.ai/api/v1/generation");
|
|
275
|
+
url.searchParams.set("id", id);
|
|
276
|
+
const response = await fetch(url, {
|
|
277
|
+
headers: { Authorization: `Bearer ${apiKey}` },
|
|
278
|
+
signal: AbortSignal.timeout(15e3),
|
|
279
|
+
redirect: "error"
|
|
280
|
+
});
|
|
281
|
+
if (!response.ok) {
|
|
282
|
+
throw new Error(`OpenRouter usage lookup failed (${response.status})`);
|
|
283
|
+
}
|
|
284
|
+
const data = record(record(await response.json()).data);
|
|
285
|
+
const finished = typeof data.finish_reason === "string" && data.finish_reason.length > 0 || data.cancelled === true;
|
|
286
|
+
if (data.id !== id || !validUsd(data.total_cost) || !finished) {
|
|
287
|
+
throw new Error("OpenRouter generation cost is unavailable");
|
|
288
|
+
}
|
|
289
|
+
return {
|
|
290
|
+
...usage,
|
|
291
|
+
cost: data.total_cost,
|
|
292
|
+
prompt_tokens_details: {
|
|
293
|
+
...usage.prompt_tokens_details,
|
|
294
|
+
...Number.isSafeInteger(data.native_tokens_cached) && Number(data.native_tokens_cached) >= 0 ? { cached_tokens: Number(data.native_tokens_cached) } : {}
|
|
295
|
+
},
|
|
296
|
+
...Number.isSafeInteger(data.native_tokens_prompt) && Number(data.native_tokens_prompt) >= 0 ? { prompt_tokens: Number(data.native_tokens_prompt) } : {},
|
|
297
|
+
...Number.isSafeInteger(data.native_tokens_completion) && Number(data.native_tokens_completion) >= 0 ? { completion_tokens: Number(data.native_tokens_completion) } : {}
|
|
298
|
+
};
|
|
299
|
+
}
|
|
300
|
+
};
|
|
301
|
+
|
|
302
|
+
// ../ts-shared/src/llm-providers.ts
|
|
303
|
+
var LLM_PROVIDERS = {
|
|
304
|
+
deepinfra,
|
|
305
|
+
baseten,
|
|
306
|
+
openrouter,
|
|
307
|
+
gemini
|
|
308
|
+
};
|
|
309
|
+
|
|
310
|
+
// ../ts-shared/src/ai-providers.ts
|
|
311
|
+
var staticModel = function(id, name, capabilities, priceSummary) {
|
|
312
|
+
return Object.freeze({
|
|
313
|
+
id,
|
|
314
|
+
name,
|
|
315
|
+
capabilities: [...capabilities],
|
|
316
|
+
...priceSummary ? { priceSummary } : {}
|
|
317
|
+
});
|
|
318
|
+
};
|
|
319
|
+
var AI_PROVIDERS = Object.freeze({
|
|
320
|
+
deepinfra: {
|
|
321
|
+
name: "DeepInfra",
|
|
322
|
+
capabilities: ["llm"],
|
|
323
|
+
defaultBaseUrl: LLM_PROVIDERS.deepinfra.baseUrl,
|
|
324
|
+
supportsCreatorChat: true,
|
|
325
|
+
staticModels: []
|
|
326
|
+
},
|
|
327
|
+
openrouter: {
|
|
328
|
+
name: "OpenRouter",
|
|
329
|
+
capabilities: ["llm"],
|
|
330
|
+
defaultBaseUrl: LLM_PROVIDERS.openrouter.baseUrl,
|
|
331
|
+
supportsCreatorChat: true,
|
|
332
|
+
staticModels: []
|
|
333
|
+
},
|
|
334
|
+
baseten: {
|
|
335
|
+
name: "Baseten",
|
|
336
|
+
capabilities: ["llm"],
|
|
337
|
+
defaultBaseUrl: LLM_PROVIDERS.baseten.baseUrl,
|
|
338
|
+
supportsCreatorChat: true,
|
|
339
|
+
staticModels: []
|
|
340
|
+
},
|
|
341
|
+
gemini: {
|
|
342
|
+
name: "Google Gemini",
|
|
343
|
+
capabilities: ["llm", "tts", "stt", "vision"],
|
|
344
|
+
defaultBaseUrl: "https://generativelanguage.googleapis.com/v1beta",
|
|
345
|
+
supportsCreatorChat: true,
|
|
346
|
+
staticModels: [
|
|
347
|
+
staticModel("gemini-3.8-flash-tts", "Gemini 3.8 Flash TTS", ["tts"]),
|
|
348
|
+
staticModel(
|
|
349
|
+
"gemini-3.8-flash-lite-tts",
|
|
350
|
+
"Gemini 3.8 Flash Lite TTS",
|
|
351
|
+
["tts"]
|
|
352
|
+
),
|
|
353
|
+
staticModel(
|
|
354
|
+
"gemini-3.5-transcribe-live",
|
|
355
|
+
"Gemini 3.5 Transcribe Live",
|
|
356
|
+
["stt"],
|
|
357
|
+
"$3.50 input / $21 output per 1M tokens"
|
|
358
|
+
),
|
|
359
|
+
staticModel(
|
|
360
|
+
"gemini-3.5-transcribe",
|
|
361
|
+
"Gemini 3.5 Transcribe",
|
|
362
|
+
["stt"],
|
|
363
|
+
"$2 input / $12 output per 1M tokens"
|
|
364
|
+
),
|
|
365
|
+
staticModel(
|
|
366
|
+
"gemini-3.5-flash-lite",
|
|
367
|
+
"Gemini 3.5 Flash Lite",
|
|
368
|
+
["vision"],
|
|
369
|
+
"$0.30 input / $2.50 output per 1M tokens"
|
|
370
|
+
)
|
|
371
|
+
]
|
|
372
|
+
},
|
|
373
|
+
openai: {
|
|
374
|
+
name: "OpenAI",
|
|
375
|
+
capabilities: ["llm", "stt", "vision", "image"],
|
|
376
|
+
defaultBaseUrl: "https://api.openai.com/v1",
|
|
377
|
+
supportsCreatorChat: false,
|
|
378
|
+
forbidOpenRouterBaseUrl: true,
|
|
379
|
+
staticModels: [
|
|
380
|
+
staticModel(
|
|
381
|
+
"gpt-6.1-sol",
|
|
382
|
+
"GPT-6.1 Sol",
|
|
383
|
+
["llm"],
|
|
384
|
+
"$2 input / $10 output per 1M tokens"
|
|
385
|
+
),
|
|
386
|
+
staticModel(
|
|
387
|
+
"gpt-6-luna",
|
|
388
|
+
"GPT-6 Luna",
|
|
389
|
+
["llm", "vision"],
|
|
390
|
+
"$0.10 input / $0.50 output per 1M tokens"
|
|
391
|
+
),
|
|
392
|
+
staticModel(
|
|
393
|
+
"gpt-image-2.5-flare",
|
|
394
|
+
"GPT Image 2.5 Flare",
|
|
395
|
+
["image"],
|
|
396
|
+
"$5 text / $8 image input / $30 output per 1M tokens"
|
|
397
|
+
),
|
|
398
|
+
staticModel(
|
|
399
|
+
"gpt-image-2.5-sunburst",
|
|
400
|
+
"GPT Image 2.5 Sunburst",
|
|
401
|
+
["image"],
|
|
402
|
+
"$5 text / $8 image input / $30 output per 1M tokens"
|
|
403
|
+
),
|
|
404
|
+
staticModel(
|
|
405
|
+
"gpt-transcribe",
|
|
406
|
+
"GPT Transcribe",
|
|
407
|
+
["stt"],
|
|
408
|
+
"$0.0045/min"
|
|
409
|
+
),
|
|
410
|
+
staticModel(
|
|
411
|
+
"gpt-live-transcribe",
|
|
412
|
+
"GPT Live Transcribe",
|
|
413
|
+
["stt"],
|
|
414
|
+
"$0.017/min"
|
|
415
|
+
)
|
|
416
|
+
]
|
|
417
|
+
},
|
|
418
|
+
anthropic: {
|
|
419
|
+
name: "Anthropic",
|
|
420
|
+
capabilities: ["llm"],
|
|
421
|
+
defaultBaseUrl: "https://api.anthropic.com",
|
|
422
|
+
supportsCreatorChat: false,
|
|
423
|
+
staticModels: [
|
|
424
|
+
staticModel(
|
|
425
|
+
"claude-opus-5-5",
|
|
426
|
+
"Claude Opus 5.5",
|
|
427
|
+
["llm"],
|
|
428
|
+
"$4 input / $20 output per 1M tokens"
|
|
429
|
+
),
|
|
430
|
+
staticModel(
|
|
431
|
+
"claude-sonnet-5-5",
|
|
432
|
+
"Claude Sonnet 5.5",
|
|
433
|
+
["llm"],
|
|
434
|
+
"$2 input / $10 output per 1M tokens"
|
|
435
|
+
),
|
|
436
|
+
staticModel(
|
|
437
|
+
"claude-haiku-5-5",
|
|
438
|
+
"Claude Haiku 5.5",
|
|
439
|
+
["llm"],
|
|
440
|
+
"$0.10 input / $0.50 output per 1M tokens"
|
|
441
|
+
)
|
|
442
|
+
]
|
|
443
|
+
},
|
|
444
|
+
elevenlabs: {
|
|
445
|
+
name: "ElevenLabs",
|
|
446
|
+
capabilities: ["tts", "stt"],
|
|
447
|
+
defaultBaseUrl: "https://api.elevenlabs.io/v1",
|
|
448
|
+
supportsCreatorChat: false,
|
|
449
|
+
staticModels: [
|
|
450
|
+
staticModel(
|
|
451
|
+
"eleven_v3_conversational",
|
|
452
|
+
"Eleven v3 Conversational",
|
|
453
|
+
["tts"]
|
|
454
|
+
),
|
|
455
|
+
staticModel("eleven_v3", "Eleven v3", ["tts"]),
|
|
456
|
+
staticModel(
|
|
457
|
+
"scribe_v2_realtime",
|
|
458
|
+
"Scribe v2 Realtime",
|
|
459
|
+
["stt"],
|
|
460
|
+
"$0.39/hr"
|
|
461
|
+
),
|
|
462
|
+
staticModel("scribe_v2", "Scribe v2", ["stt"], "$0.22/hr")
|
|
463
|
+
]
|
|
464
|
+
},
|
|
465
|
+
tripo: {
|
|
466
|
+
name: "Tripo",
|
|
467
|
+
capabilities: ["3d"],
|
|
468
|
+
defaultBaseUrl: "https://openapi.tripo3d.ai/v3",
|
|
469
|
+
supportsCreatorChat: false,
|
|
470
|
+
staticModels: [
|
|
471
|
+
staticModel(
|
|
472
|
+
"P2-20260801",
|
|
473
|
+
"Tripo P2",
|
|
474
|
+
["3d"],
|
|
475
|
+
"$0.01/credit (P2 4K)"
|
|
476
|
+
),
|
|
477
|
+
staticModel(
|
|
478
|
+
"v1.0-20240301",
|
|
479
|
+
"Tripo Rig v1.0",
|
|
480
|
+
["3d"],
|
|
481
|
+
"$0.01/credit (25 credits/rig)"
|
|
482
|
+
)
|
|
483
|
+
]
|
|
484
|
+
},
|
|
485
|
+
meshy: {
|
|
486
|
+
name: "Meshy",
|
|
487
|
+
capabilities: ["3d"],
|
|
488
|
+
defaultBaseUrl: "https://api.meshy.ai",
|
|
489
|
+
supportsCreatorChat: false,
|
|
490
|
+
staticModels: [
|
|
491
|
+
staticModel(
|
|
492
|
+
"meshy-7.1",
|
|
493
|
+
"Meshy 7.1",
|
|
494
|
+
["3d"],
|
|
495
|
+
"$0.02/credit (20-30 credits/model)"
|
|
496
|
+
),
|
|
497
|
+
staticModel(
|
|
498
|
+
"meshy-6",
|
|
499
|
+
"Meshy 6",
|
|
500
|
+
["3d"],
|
|
501
|
+
"$0.02/credit (20-30 credits/model)"
|
|
502
|
+
)
|
|
503
|
+
]
|
|
504
|
+
},
|
|
505
|
+
unirig: {
|
|
506
|
+
name: "UniRig",
|
|
507
|
+
capabilities: ["3d"],
|
|
508
|
+
defaultBaseUrl: "https://modal.run",
|
|
509
|
+
supportsCreatorChat: false,
|
|
510
|
+
staticModels: [
|
|
511
|
+
staticModel(
|
|
512
|
+
"unirig-v1",
|
|
513
|
+
"UniRig (Modal GPU)",
|
|
514
|
+
["3d"],
|
|
515
|
+
"Modal A10 GPU billed per second"
|
|
516
|
+
)
|
|
517
|
+
]
|
|
518
|
+
}
|
|
519
|
+
});
|
|
520
|
+
|
|
45
521
|
// ../ts-shared/src/ai-settings.ts
|
|
522
|
+
var AI_VENDOR_NAMES = {
|
|
523
|
+
openai: AI_PROVIDERS.openai.name,
|
|
524
|
+
tripo: AI_PROVIDERS.tripo.name,
|
|
525
|
+
meshy: AI_PROVIDERS.meshy.name,
|
|
526
|
+
unirig: AI_PROVIDERS.unirig.name,
|
|
527
|
+
elevenlabs: AI_PROVIDERS.elevenlabs.name,
|
|
528
|
+
gemini: AI_PROVIDERS.gemini.name,
|
|
529
|
+
anthropic: AI_PROVIDERS.anthropic.name
|
|
530
|
+
};
|
|
46
531
|
var GPT_IMAGE_MODELS = [
|
|
47
532
|
"gpt-image-2.5-flare",
|
|
48
533
|
"gpt-image-2.5-sunburst"
|
|
@@ -50,76 +535,124 @@ var GPT_IMAGE_MODELS = [
|
|
|
50
535
|
var AI_ROLES = {
|
|
51
536
|
character_draft: {
|
|
52
537
|
name: "Character draft",
|
|
53
|
-
|
|
54
|
-
|
|
538
|
+
capability: "llm",
|
|
539
|
+
chatLlm: true,
|
|
55
540
|
options: ["temperature"]
|
|
56
541
|
},
|
|
57
542
|
motion_planner: {
|
|
58
543
|
name: "Motion planner",
|
|
59
|
-
|
|
60
|
-
|
|
544
|
+
capability: "llm",
|
|
545
|
+
chatLlm: true,
|
|
61
546
|
options: []
|
|
62
547
|
},
|
|
63
|
-
// Not charged.
|
|
64
548
|
prompt_check: {
|
|
65
549
|
name: "Prompt check",
|
|
66
|
-
|
|
67
|
-
|
|
550
|
+
capability: "llm",
|
|
551
|
+
providers: ["openai"],
|
|
552
|
+
allowedModels: ["gpt-6-luna"],
|
|
68
553
|
options: []
|
|
69
554
|
},
|
|
70
|
-
// Priced by OPENAI_IMAGE_PRICES of the server (flare and sunburst alike).
|
|
71
555
|
design_sheet_image: {
|
|
72
556
|
name: "Design sheet image",
|
|
73
|
-
|
|
74
|
-
|
|
557
|
+
capability: "image",
|
|
558
|
+
providers: ["openai"],
|
|
559
|
+
allowedModels: GPT_IMAGE_MODELS,
|
|
75
560
|
options: []
|
|
76
561
|
},
|
|
77
|
-
// Not charged per call.
|
|
78
562
|
tts: {
|
|
79
563
|
name: "ElevenLabs text to speech",
|
|
80
|
-
|
|
81
|
-
|
|
564
|
+
capability: "tts",
|
|
565
|
+
providers: ["elevenlabs"],
|
|
566
|
+
allowedModels: ["eleven_v3_conversational", "eleven_v3"],
|
|
82
567
|
options: []
|
|
83
568
|
},
|
|
84
|
-
// Not charged per call.
|
|
85
569
|
gemini_tts: {
|
|
86
570
|
name: "Gemini text to speech",
|
|
87
|
-
|
|
88
|
-
|
|
571
|
+
capability: "tts",
|
|
572
|
+
providers: ["gemini"],
|
|
573
|
+
allowedModels: ["gemini-3.8-flash-tts", "gemini-3.8-flash-lite-tts"],
|
|
574
|
+
options: []
|
|
575
|
+
},
|
|
576
|
+
stt_openai: {
|
|
577
|
+
name: "OpenAI speech recognition",
|
|
578
|
+
capability: "stt",
|
|
579
|
+
providers: ["openai"],
|
|
580
|
+
allowedModels: ["gpt-transcribe", "gpt-live-transcribe"],
|
|
581
|
+
options: []
|
|
582
|
+
},
|
|
583
|
+
stt_gemini: {
|
|
584
|
+
name: "Gemini speech recognition",
|
|
585
|
+
capability: "stt",
|
|
586
|
+
providers: ["gemini"],
|
|
587
|
+
allowedModels: ["gemini-3.5-transcribe-live"],
|
|
588
|
+
options: []
|
|
589
|
+
},
|
|
590
|
+
stt_elevenlabs: {
|
|
591
|
+
name: "ElevenLabs speech recognition",
|
|
592
|
+
capability: "stt",
|
|
593
|
+
providers: ["elevenlabs"],
|
|
594
|
+
allowedModels: ["scribe_v2_realtime"],
|
|
89
595
|
options: []
|
|
90
596
|
},
|
|
91
|
-
// Sight (docs/sight.md): one role for each provider that an owner can
|
|
92
|
-
// choose. Priced by VISION_PRICES of the server.
|
|
93
597
|
vision_gemini: {
|
|
94
598
|
name: "Gemini sight",
|
|
95
|
-
|
|
96
|
-
|
|
599
|
+
capability: "vision",
|
|
600
|
+
providers: ["gemini"],
|
|
601
|
+
allowedModels: ["gemini-3.5-flash-lite"],
|
|
97
602
|
options: []
|
|
98
603
|
},
|
|
99
604
|
vision_openai: {
|
|
100
605
|
name: "OpenAI sight",
|
|
101
|
-
|
|
102
|
-
|
|
606
|
+
capability: "vision",
|
|
607
|
+
providers: ["openai"],
|
|
608
|
+
allowedModels: ["gpt-6-luna"],
|
|
103
609
|
options: []
|
|
104
610
|
},
|
|
105
|
-
// Priced by OPENAI_PRICES.text of the generator.
|
|
106
611
|
generator_agent: {
|
|
107
612
|
name: "Generator agent",
|
|
108
|
-
|
|
109
|
-
|
|
613
|
+
capability: "llm",
|
|
614
|
+
providers: ["openai", "anthropic"],
|
|
615
|
+
allowedModels: [
|
|
616
|
+
"gpt-6.1-sol",
|
|
617
|
+
"claude-opus-5-5",
|
|
618
|
+
"claude-sonnet-5-5",
|
|
619
|
+
"claude-haiku-5-5"
|
|
620
|
+
],
|
|
110
621
|
options: ["reasoningEffort"]
|
|
111
622
|
},
|
|
112
623
|
generator_reading: {
|
|
113
624
|
name: "Generator face reading",
|
|
114
|
-
|
|
115
|
-
|
|
625
|
+
capability: "llm",
|
|
626
|
+
providers: ["openai"],
|
|
627
|
+
allowedModels: ["gpt-6.1-sol"],
|
|
116
628
|
options: ["reasoningEffort"]
|
|
117
629
|
},
|
|
118
|
-
// Priced by OPENAI_PRICES of the generator (flare and sunburst alike).
|
|
119
630
|
generator_image: {
|
|
120
631
|
name: "Generator images",
|
|
121
|
-
|
|
122
|
-
|
|
632
|
+
capability: "image",
|
|
633
|
+
providers: ["openai"],
|
|
634
|
+
allowedModels: GPT_IMAGE_MODELS,
|
|
635
|
+
options: []
|
|
636
|
+
},
|
|
637
|
+
generator_3d: {
|
|
638
|
+
name: "Generator 3D head",
|
|
639
|
+
capability: "3d",
|
|
640
|
+
providers: ["tripo"],
|
|
641
|
+
allowedModels: ["P2-20260801"],
|
|
642
|
+
options: []
|
|
643
|
+
},
|
|
644
|
+
generator_3d_body: {
|
|
645
|
+
name: "Generator 3D body",
|
|
646
|
+
capability: "3d",
|
|
647
|
+
providers: ["tripo", "meshy"],
|
|
648
|
+
allowedModels: ["P2-20260801", "meshy-7.1", "meshy-6"],
|
|
649
|
+
options: []
|
|
650
|
+
},
|
|
651
|
+
generator_rig: {
|
|
652
|
+
name: "Generator 3D rig",
|
|
653
|
+
capability: "3d",
|
|
654
|
+
providers: ["tripo", "unirig"],
|
|
655
|
+
allowedModels: ["v1.0-20240301", "P2-20260801", "unirig-v1"],
|
|
123
656
|
options: []
|
|
124
657
|
}
|
|
125
658
|
};
|
|
@@ -305,7 +838,8 @@ var productUsageEvents = [
|
|
|
305
838
|
"playback_blocked",
|
|
306
839
|
"playback_start_pressed",
|
|
307
840
|
"reply_options_shown",
|
|
308
|
-
"page_left"
|
|
841
|
+
"page_left",
|
|
842
|
+
"creation_mode_chosen"
|
|
309
843
|
];
|
|
310
844
|
|
|
311
845
|
// ../ts-shared/src/analytics.ts
|
|
@@ -2687,7 +3221,7 @@ var PERFORMANCE_KINDS = {
|
|
|
2687
3221
|
cachedOpening: 8
|
|
2688
3222
|
};
|
|
2689
3223
|
var PERFORMANCE_MAJOR_VERSION = 1;
|
|
2690
|
-
var PERFORMANCE_MINOR_VERSION =
|
|
3224
|
+
var PERFORMANCE_MINOR_VERSION = 2;
|
|
2691
3225
|
var HEADER_SIZE = 24;
|
|
2692
3226
|
var MAGIC = 1346588750;
|
|
2693
3227
|
var crcTable = Uint32Array.from({ length: 256 }, (_, index) => {
|
|
@@ -2839,6 +3373,68 @@ var decodePerformance = function(input, expectedKind) {
|
|
|
2839
3373
|
return wireValue(value);
|
|
2840
3374
|
};
|
|
2841
3375
|
|
|
3376
|
+
// ../ts-shared/src/player-memory.ts
|
|
3377
|
+
var MEMORY_IDENTIFIER_MAX_LENGTH = 128;
|
|
3378
|
+
var MEMORY_IMPORT_MAX_BYTES = 512 * 1024;
|
|
3379
|
+
var normalizeMemoryIdentifier = function(value) {
|
|
3380
|
+
if (typeof value !== "string") {
|
|
3381
|
+
return null;
|
|
3382
|
+
}
|
|
3383
|
+
const cleaned = value.normalize("NFKC").replace(/[\u0000-\u001f\u007f]/gu, "").trim();
|
|
3384
|
+
if (!cleaned) {
|
|
3385
|
+
return null;
|
|
3386
|
+
}
|
|
3387
|
+
return cleaned.slice(0, MEMORY_IDENTIFIER_MAX_LENGTH).trim() || null;
|
|
3388
|
+
};
|
|
3389
|
+
|
|
3390
|
+
// ../ts-shared/src/message-time.ts
|
|
3391
|
+
var MESSAGE_TIME_ZONE_MAX_LENGTH = 64;
|
|
3392
|
+
var TIME_ZONE_PATTERN = /^[A-Za-z0-9_+-]+(?:\/[A-Za-z0-9_+-]+)*$/u;
|
|
3393
|
+
var OFFSET_PATTERN = /^[+-](?:[01]\d|2[0-3]):[0-5]\d$/u;
|
|
3394
|
+
var normalizeMessageTimeZone = function(value) {
|
|
3395
|
+
if (typeof value !== "string") {
|
|
3396
|
+
return null;
|
|
3397
|
+
}
|
|
3398
|
+
const trimmed = value.trim();
|
|
3399
|
+
if (!trimmed || trimmed.length > MESSAGE_TIME_ZONE_MAX_LENGTH) {
|
|
3400
|
+
return null;
|
|
3401
|
+
}
|
|
3402
|
+
if (trimmed === "Z" || trimmed === "z" || trimmed === "+00:00" || trimmed === "-00:00") {
|
|
3403
|
+
return "UTC";
|
|
3404
|
+
}
|
|
3405
|
+
if (!OFFSET_PATTERN.test(trimmed) && !TIME_ZONE_PATTERN.test(trimmed)) {
|
|
3406
|
+
return null;
|
|
3407
|
+
}
|
|
3408
|
+
try {
|
|
3409
|
+
new Intl.DateTimeFormat("en-US", { timeZone: trimmed }).format(/* @__PURE__ */ new Date(0));
|
|
3410
|
+
return trimmed;
|
|
3411
|
+
} catch {
|
|
3412
|
+
return null;
|
|
3413
|
+
}
|
|
3414
|
+
};
|
|
3415
|
+
var formatUtcOffsetMinutes = function(offsetMinutesEastOfUtc) {
|
|
3416
|
+
if (!Number.isFinite(offsetMinutesEastOfUtc) || offsetMinutesEastOfUtc === 0) {
|
|
3417
|
+
return "UTC";
|
|
3418
|
+
}
|
|
3419
|
+
const total = Math.trunc(offsetMinutesEastOfUtc);
|
|
3420
|
+
const sign = total >= 0 ? "+" : "-";
|
|
3421
|
+
const abs = Math.abs(total);
|
|
3422
|
+
const hours = String(Math.floor(abs / 60)).padStart(2, "0");
|
|
3423
|
+
const minutes = String(abs % 60).padStart(2, "0");
|
|
3424
|
+
return `${sign}${hours}:${minutes}`;
|
|
3425
|
+
};
|
|
3426
|
+
var detectClientTimeZone = function(now = /* @__PURE__ */ new Date()) {
|
|
3427
|
+
try {
|
|
3428
|
+
const resolved = Intl.DateTimeFormat().resolvedOptions().timeZone;
|
|
3429
|
+
const normalized = normalizeMessageTimeZone(resolved);
|
|
3430
|
+
if (normalized) {
|
|
3431
|
+
return normalized;
|
|
3432
|
+
}
|
|
3433
|
+
} catch {
|
|
3434
|
+
}
|
|
3435
|
+
return formatUtcOffsetMinutes(-now.getTimezoneOffset());
|
|
3436
|
+
};
|
|
3437
|
+
|
|
2842
3438
|
// ../../clients/web-client/src/api/performance.ts
|
|
2843
3439
|
var readPerformanceBytes = async function(response) {
|
|
2844
3440
|
const maximum = PERFORMANCE_MAX_BYTES + 65535;
|
|
@@ -3082,7 +3678,9 @@ var hasTimelineShape = function(value) {
|
|
|
3082
3678
|
}
|
|
3083
3679
|
previousDialogueEnd = Number(block.endTime);
|
|
3084
3680
|
return true;
|
|
3085
|
-
}) && isSameTime(previousDialogueEnd, value.duration) && hasOrderedItems(value.expressions, value.duration, hasExpressionShape) && hasOrderedItems(value.animations, value.duration, hasAnimationShape)
|
|
3681
|
+
}) && isSameTime(previousDialogueEnd, value.duration) && hasOrderedItems(value.expressions, value.duration, hasExpressionShape) && hasOrderedItems(value.animations, value.duration, hasAnimationShape) && (value.initialVariant === void 0 || isValidVariantKey(value.initialVariant)) && (value.activeVariant === void 0 || isValidVariantKey(value.activeVariant)) && (value.toolCalls === void 0 || Array.isArray(value.toolCalls) && value.toolCalls.every(
|
|
3682
|
+
(call) => isStagingToolCall(call) && isWithinDuration(call.timestamp, value.duration)
|
|
3683
|
+
));
|
|
3086
3684
|
};
|
|
3087
3685
|
var normalizeStagingResultPayload = function(value) {
|
|
3088
3686
|
if (!isRecord(value)) {
|
|
@@ -3192,14 +3790,20 @@ var BakedConversation = class {
|
|
|
3192
3790
|
conversationId,
|
|
3193
3791
|
openingStagingId: this.asset.rootStagingId,
|
|
3194
3792
|
activeStagingId: this.activeId,
|
|
3195
|
-
stagings: [...this.nodes.values()].map((node) =>
|
|
3196
|
-
|
|
3197
|
-
|
|
3198
|
-
|
|
3199
|
-
|
|
3200
|
-
|
|
3201
|
-
|
|
3202
|
-
|
|
3793
|
+
stagings: [...this.nodes.values()].map((node) => {
|
|
3794
|
+
const { initialVariant, activeVariant, toolCalls } = node.payload.timeline;
|
|
3795
|
+
return {
|
|
3796
|
+
id: node.id,
|
|
3797
|
+
parentId: node.parentId,
|
|
3798
|
+
triggerText: node.triggerText,
|
|
3799
|
+
text: node.payload.text,
|
|
3800
|
+
options: structuredClone(node.payload.options),
|
|
3801
|
+
createdAt: "1970-01-01T00:00:00.000Z",
|
|
3802
|
+
...initialVariant !== void 0 ? { initialVariant } : {},
|
|
3803
|
+
...activeVariant !== void 0 ? { activeVariant } : {},
|
|
3804
|
+
...toolCalls ? { toolCalls: structuredClone(toolCalls) } : {}
|
|
3805
|
+
};
|
|
3806
|
+
})
|
|
3203
3807
|
};
|
|
3204
3808
|
}
|
|
3205
3809
|
child(parentId, label) {
|
|
@@ -3239,7 +3843,9 @@ var createConversation = async function({
|
|
|
3239
3843
|
avatarId,
|
|
3240
3844
|
apiToken,
|
|
3241
3845
|
baseUrl,
|
|
3242
|
-
renderSessionId
|
|
3846
|
+
renderSessionId,
|
|
3847
|
+
hostedWebChat,
|
|
3848
|
+
memoryIdentifier
|
|
3243
3849
|
}) {
|
|
3244
3850
|
const url = new URL("/conversations", baseUrl);
|
|
3245
3851
|
const res = await fetch(url.toString(), {
|
|
@@ -3253,7 +3859,12 @@ var createConversation = async function({
|
|
|
3253
3859
|
credentials: apiToken ? "omit" : "include",
|
|
3254
3860
|
method: "POST",
|
|
3255
3861
|
// This SDK plays live motion frames (ADR 0011).
|
|
3256
|
-
body: JSON.stringify({
|
|
3862
|
+
body: JSON.stringify({
|
|
3863
|
+
avatarId,
|
|
3864
|
+
capabilities: [MOTION_FRAMES_CAPABILITY],
|
|
3865
|
+
...hostedWebChat ? { hostedWebChat: true } : {},
|
|
3866
|
+
...memoryIdentifier !== void 0 ? { memoryIdentifier } : {}
|
|
3867
|
+
})
|
|
3257
3868
|
});
|
|
3258
3869
|
if (!res.ok) {
|
|
3259
3870
|
throw Object.assign(new Error(await res.text()), { status: res.status });
|
|
@@ -3276,7 +3887,8 @@ var createStaging = async function({
|
|
|
3276
3887
|
requestId,
|
|
3277
3888
|
silent,
|
|
3278
3889
|
parentStagingId,
|
|
3279
|
-
renderSessionId
|
|
3890
|
+
renderSessionId,
|
|
3891
|
+
memoryIdentifier
|
|
3280
3892
|
}) {
|
|
3281
3893
|
const url = new URL(`/conversations/${conversationId}/stagings`, baseUrl);
|
|
3282
3894
|
const requestHeaders = withRenderSession(
|
|
@@ -3307,7 +3919,9 @@ var createStaging = async function({
|
|
|
3307
3919
|
silent: silent === true,
|
|
3308
3920
|
incrementalPlayback: true,
|
|
3309
3921
|
parentStagingId,
|
|
3310
|
-
capabilities: [MOTION_FRAMES_CAPABILITY]
|
|
3922
|
+
capabilities: [MOTION_FRAMES_CAPABILITY],
|
|
3923
|
+
timeZone: detectClientTimeZone(),
|
|
3924
|
+
...memoryIdentifier !== void 0 ? { memoryIdentifier } : {}
|
|
3311
3925
|
})
|
|
3312
3926
|
});
|
|
3313
3927
|
} catch (error) {
|
|
@@ -3491,7 +4105,8 @@ var LiveConversation = class {
|
|
|
3491
4105
|
silent,
|
|
3492
4106
|
parentStagingId,
|
|
3493
4107
|
headers: telemetry.traceHeaders,
|
|
3494
|
-
renderSessionId: this.dependencies.renderSession()
|
|
4108
|
+
renderSessionId: this.dependencies.renderSession(),
|
|
4109
|
+
memoryIdentifier: this.dependencies.memoryIdentifier?.()
|
|
3495
4110
|
});
|
|
3496
4111
|
} catch (error) {
|
|
3497
4112
|
if (request.completed) {
|
|
@@ -3753,25 +4368,61 @@ var loadGltfWithProgress = async function({
|
|
|
3753
4368
|
});
|
|
3754
4369
|
}
|
|
3755
4370
|
};
|
|
3756
|
-
var
|
|
3757
|
-
|
|
3758
|
-
|
|
3759
|
-
|
|
3760
|
-
|
|
3761
|
-
assetProfile
|
|
3762
|
-
|
|
3763
|
-
const url = new URL(`/avatars/${characterId}/download-url`, baseUrl);
|
|
3764
|
-
if (assetProfile && assetProfile !== "original") {
|
|
3765
|
-
url.searchParams.set("assetProfile", assetProfile);
|
|
4371
|
+
var fetchVrmDownloadUrl = async function(baseUrl, characterId, options) {
|
|
4372
|
+
const url = new URL(
|
|
4373
|
+
`/avatars/${encodeURIComponent(characterId)}/download-url`,
|
|
4374
|
+
baseUrl
|
|
4375
|
+
);
|
|
4376
|
+
if (options?.assetProfile && options.assetProfile !== "original") {
|
|
4377
|
+
url.searchParams.set("assetProfile", options.assetProfile);
|
|
3766
4378
|
}
|
|
3767
4379
|
const res = await fetch(url.toString(), {
|
|
3768
|
-
headers: buildAuthHeaders(apiToken, { allowEmpty: true }),
|
|
3769
|
-
credentials: apiToken ? "omit" : "include"
|
|
4380
|
+
headers: buildAuthHeaders(options?.apiToken, { allowEmpty: true }),
|
|
4381
|
+
credentials: options?.apiToken ? "omit" : "include",
|
|
4382
|
+
...options?.signal ? { signal: options.signal } : {}
|
|
4383
|
+
});
|
|
4384
|
+
if (!res.ok) {
|
|
4385
|
+
throw new Error(await res.text());
|
|
4386
|
+
}
|
|
4387
|
+
const { url: vrmUrl } = await res.json();
|
|
4388
|
+
return vrmUrl;
|
|
4389
|
+
};
|
|
4390
|
+
var fetchVariantVrmDownloadUrl = async function(baseUrl, characterId, variantKey, options) {
|
|
4391
|
+
if (variantKey === DEFAULT_VARIANT_KEY) {
|
|
4392
|
+
return await fetchVrmDownloadUrl(baseUrl, characterId, options);
|
|
4393
|
+
}
|
|
4394
|
+
const url = new URL(
|
|
4395
|
+
`/avatars/${encodeURIComponent(characterId)}/variants/${encodeURIComponent(variantKey)}/download-url`,
|
|
4396
|
+
baseUrl
|
|
4397
|
+
);
|
|
4398
|
+
if (options?.assetProfile && options.assetProfile !== "original") {
|
|
4399
|
+
url.searchParams.set("assetProfile", options.assetProfile);
|
|
4400
|
+
}
|
|
4401
|
+
const res = await fetch(url.toString(), {
|
|
4402
|
+
headers: buildAuthHeaders(options?.apiToken, { allowEmpty: true }),
|
|
4403
|
+
credentials: options?.apiToken ? "omit" : "include",
|
|
4404
|
+
...options?.signal ? { signal: options.signal } : {}
|
|
3770
4405
|
});
|
|
3771
4406
|
if (!res.ok) {
|
|
3772
4407
|
throw new Error(await res.text());
|
|
3773
4408
|
}
|
|
3774
4409
|
const { url: vrmUrl } = await res.json();
|
|
4410
|
+
return vrmUrl;
|
|
4411
|
+
};
|
|
4412
|
+
var loadVrm = async function({
|
|
4413
|
+
baseUrl,
|
|
4414
|
+
characterId,
|
|
4415
|
+
variantKey,
|
|
4416
|
+
apiToken,
|
|
4417
|
+
onProgress,
|
|
4418
|
+
assetProfile
|
|
4419
|
+
}) {
|
|
4420
|
+
const vrmUrl = await fetchVariantVrmDownloadUrl(
|
|
4421
|
+
baseUrl,
|
|
4422
|
+
characterId,
|
|
4423
|
+
variantKey ?? DEFAULT_VARIANT_KEY,
|
|
4424
|
+
{ apiToken, assetProfile }
|
|
4425
|
+
);
|
|
3775
4426
|
const gltf = await loadGltfWithProgress({
|
|
3776
4427
|
url: vrmUrl,
|
|
3777
4428
|
onProgress,
|
|
@@ -3783,6 +4434,23 @@ var loadVrm = async ({
|
|
|
3783
4434
|
VRMUtils.combineSkeletons(gltf.scene);
|
|
3784
4435
|
return gltf.userData.vrm;
|
|
3785
4436
|
};
|
|
4437
|
+
var loadVariantVrm = async function({
|
|
4438
|
+
baseUrl,
|
|
4439
|
+
characterId,
|
|
4440
|
+
variantKey,
|
|
4441
|
+
apiToken,
|
|
4442
|
+
onProgress,
|
|
4443
|
+
assetProfile
|
|
4444
|
+
}) {
|
|
4445
|
+
return await loadVrm({
|
|
4446
|
+
baseUrl,
|
|
4447
|
+
characterId,
|
|
4448
|
+
variantKey,
|
|
4449
|
+
apiToken,
|
|
4450
|
+
onProgress,
|
|
4451
|
+
assetProfile
|
|
4452
|
+
});
|
|
4453
|
+
};
|
|
3786
4454
|
var loadVrma = async ({
|
|
3787
4455
|
baseUrl,
|
|
3788
4456
|
characterId,
|
|
@@ -4641,7 +5309,7 @@ var disposeSceneContext = function(ctx) {
|
|
|
4641
5309
|
|
|
4642
5310
|
// ../vrm-renderer/src/preview-controller.ts
|
|
4643
5311
|
import { VRMUtils as VRMUtils4 } from "@pixiv/three-vrm";
|
|
4644
|
-
import { Box3, GridHelper, Vector3 as
|
|
5312
|
+
import { Box3, GridHelper, Vector3 as Vector38 } from "three";
|
|
4645
5313
|
|
|
4646
5314
|
// ../vrm-renderer/src/loaders.ts
|
|
4647
5315
|
import { VRMLoaderPlugin as VRMLoaderPlugin2, VRMUtils as VRMUtils2 } from "@pixiv/three-vrm";
|
|
@@ -4663,71 +5331,1086 @@ import { AnimationMixer, LoopOnce, LoopRepeat } from "three";
|
|
|
4663
5331
|
import {
|
|
4664
5332
|
createVRMAnimationClip as createPixivVRMAnimationClip
|
|
4665
5333
|
} from "@pixiv/three-vrm-animation";
|
|
5334
|
+
|
|
5335
|
+
// ../vrm-renderer/src/face-reach.ts
|
|
4666
5336
|
import {
|
|
4667
5337
|
AnimationClip,
|
|
4668
|
-
|
|
4669
|
-
Quaternion as Quaternion2,
|
|
5338
|
+
Quaternion as Quaternion4,
|
|
4670
5339
|
QuaternionKeyframeTrack,
|
|
5340
|
+
Vector3 as Vector35
|
|
5341
|
+
} from "three";
|
|
5342
|
+
|
|
5343
|
+
// ../vrm-renderer/src/hand-body.ts
|
|
5344
|
+
import {
|
|
5345
|
+
LinearInterpolant,
|
|
5346
|
+
Quaternion as Quaternion3,
|
|
4671
5347
|
QuaternionLinearInterpolant,
|
|
4672
|
-
|
|
4673
|
-
|
|
5348
|
+
SkinnedMesh,
|
|
5349
|
+
Vector3 as Vector34
|
|
4674
5350
|
} from "three";
|
|
4675
|
-
|
|
4676
|
-
|
|
4677
|
-
|
|
4678
|
-
|
|
4679
|
-
|
|
4680
|
-
|
|
4681
|
-
|
|
4682
|
-
|
|
4683
|
-
|
|
4684
|
-
var
|
|
4685
|
-
|
|
4686
|
-
|
|
4687
|
-
|
|
4688
|
-
|
|
4689
|
-
var
|
|
4690
|
-
|
|
4691
|
-
|
|
4692
|
-
|
|
4693
|
-
|
|
4694
|
-
|
|
4695
|
-
|
|
4696
|
-
|
|
5351
|
+
|
|
5352
|
+
// ../vrm-renderer/src/hand-clearance-core.ts
|
|
5353
|
+
import { Quaternion as Quaternion2, Vector3 as Vector33 } from "three";
|
|
5354
|
+
var MAP_HEIGHT_STEP = 0.02;
|
|
5355
|
+
var MAP_ANGLE_BINS = 24;
|
|
5356
|
+
var perpendicular = function(axis) {
|
|
5357
|
+
const helper = Math.abs(axis.z) < 0.9 ? new Vector33(0, 0, 1) : new Vector33(1, 0, 0);
|
|
5358
|
+
return helper.sub(axis.clone().multiplyScalar(helper.dot(axis))).normalize();
|
|
5359
|
+
};
|
|
5360
|
+
var angleBin = function(angle) {
|
|
5361
|
+
const turn = angle / (2 * Math.PI);
|
|
5362
|
+
const value = (turn - Math.floor(turn)) * MAP_ANGLE_BINS;
|
|
5363
|
+
return value;
|
|
5364
|
+
};
|
|
5365
|
+
var buildRadialMap = function(bone, origin, axis, points) {
|
|
5366
|
+
if (points.length < 16) {
|
|
5367
|
+
return null;
|
|
5368
|
+
}
|
|
5369
|
+
const unit = axis.clone().normalize();
|
|
5370
|
+
const reference = perpendicular(unit);
|
|
5371
|
+
const binormal = new Vector33().crossVectors(unit, reference);
|
|
5372
|
+
let min = Infinity;
|
|
5373
|
+
let max2 = -Infinity;
|
|
5374
|
+
const offset = new Vector33();
|
|
5375
|
+
for (const point of points) {
|
|
5376
|
+
const height = offset.subVectors(point, origin).dot(unit);
|
|
5377
|
+
min = Math.min(min, height);
|
|
5378
|
+
max2 = Math.max(max2, height);
|
|
5379
|
+
}
|
|
5380
|
+
const start = Math.floor(min / MAP_HEIGHT_STEP) * MAP_HEIGHT_STEP;
|
|
5381
|
+
const heights = Math.max(1, Math.ceil((max2 - start) / MAP_HEIGHT_STEP) + 1);
|
|
5382
|
+
const radius = new Float32Array(heights * MAP_ANGLE_BINS);
|
|
5383
|
+
const top = new Float32Array(MAP_ANGLE_BINS).fill(-Infinity);
|
|
5384
|
+
const bottom = new Float32Array(MAP_ANGLE_BINS).fill(Infinity);
|
|
5385
|
+
for (const point of points) {
|
|
5386
|
+
offset.subVectors(point, origin);
|
|
5387
|
+
const height = offset.dot(unit);
|
|
5388
|
+
const x = offset.dot(reference);
|
|
5389
|
+
const y = offset.dot(binormal);
|
|
5390
|
+
const row = Math.min(
|
|
5391
|
+
heights - 1,
|
|
5392
|
+
Math.max(0, Math.round((height - start) / MAP_HEIGHT_STEP))
|
|
5393
|
+
);
|
|
5394
|
+
const column = Math.floor(angleBin(Math.atan2(y, x))) % MAP_ANGLE_BINS;
|
|
5395
|
+
const index = row * MAP_ANGLE_BINS + column;
|
|
5396
|
+
radius[index] = Math.max(radius[index], Math.hypot(x, y));
|
|
5397
|
+
top[column] = Math.max(top[column], height);
|
|
5398
|
+
bottom[column] = Math.min(bottom[column], height);
|
|
5399
|
+
}
|
|
5400
|
+
for (let column = 0; column < MAP_ANGLE_BINS; column += 1) {
|
|
5401
|
+
if (!Number.isFinite(top[column])) {
|
|
5402
|
+
top[column] = min;
|
|
5403
|
+
bottom[column] = min;
|
|
5404
|
+
}
|
|
5405
|
+
}
|
|
5406
|
+
const filled = radius.slice();
|
|
5407
|
+
for (let row = 0; row < heights; row += 1) {
|
|
5408
|
+
for (let column = 0; column < MAP_ANGLE_BINS; column += 1) {
|
|
5409
|
+
const index = row * MAP_ANGLE_BINS + column;
|
|
5410
|
+
if (radius[index] > 0) {
|
|
5411
|
+
continue;
|
|
5412
|
+
}
|
|
5413
|
+
const left = radius[row * MAP_ANGLE_BINS + (column + MAP_ANGLE_BINS - 1) % MAP_ANGLE_BINS];
|
|
5414
|
+
const right = radius[row * MAP_ANGLE_BINS + (column + 1) % MAP_ANGLE_BINS];
|
|
5415
|
+
filled[index] = Math.max(left, right);
|
|
5416
|
+
}
|
|
5417
|
+
}
|
|
5418
|
+
return {
|
|
5419
|
+
bone,
|
|
5420
|
+
origin: origin.clone(),
|
|
5421
|
+
axis: unit,
|
|
5422
|
+
reference,
|
|
5423
|
+
start,
|
|
5424
|
+
heights,
|
|
5425
|
+
radius: filled,
|
|
5426
|
+
top,
|
|
5427
|
+
bottom,
|
|
5428
|
+
maxRadius: filled.reduce((a, b) => Math.max(a, b), 0)
|
|
5429
|
+
};
|
|
5430
|
+
};
|
|
5431
|
+
var scratchOffset = new Vector33();
|
|
5432
|
+
var NO_DIRECTION = new Vector33();
|
|
5433
|
+
var scratchBinormal = new Vector33();
|
|
5434
|
+
var queryRadialMap = function(map, point) {
|
|
5435
|
+
const offset = scratchOffset.subVectors(point, map.origin);
|
|
5436
|
+
const height = offset.dot(map.axis);
|
|
5437
|
+
const binormal = scratchBinormal.crossVectors(map.axis, map.reference);
|
|
5438
|
+
const x = offset.dot(map.reference);
|
|
5439
|
+
const y = offset.dot(binormal);
|
|
5440
|
+
const distance = Math.hypot(x, y);
|
|
5441
|
+
const row = (height - map.start) / MAP_HEIGHT_STEP;
|
|
5442
|
+
if (row < -0.5 || row > map.heights - 0.5) {
|
|
5443
|
+
return { depth: -Infinity, direction: NO_DIRECTION };
|
|
5444
|
+
}
|
|
5445
|
+
if (distance >= map.maxRadius) {
|
|
5446
|
+
return { depth: map.maxRadius - distance, direction: NO_DIRECTION };
|
|
5447
|
+
}
|
|
5448
|
+
const angle = angleBin(Math.atan2(y, x)) - 0.5;
|
|
5449
|
+
const r0 = Math.max(0, Math.min(map.heights - 1, Math.floor(row)));
|
|
5450
|
+
const r1 = Math.min(map.heights - 1, r0 + 1);
|
|
5451
|
+
const c0 = (Math.floor(angle) % MAP_ANGLE_BINS + MAP_ANGLE_BINS) % MAP_ANGLE_BINS;
|
|
5452
|
+
const c1 = (c0 + 1) % MAP_ANGLE_BINS;
|
|
5453
|
+
const fc = angle - Math.floor(angle);
|
|
5454
|
+
const at = (r, c) => map.radius[r * MAP_ANGLE_BINS + c];
|
|
5455
|
+
const surface = Math.max(at(r0, c0), at(r0, c1), at(r1, c0), at(r1, c1));
|
|
5456
|
+
const radial = surface - distance;
|
|
5457
|
+
if (radial <= 0) {
|
|
5458
|
+
return { depth: radial, direction: NO_DIRECTION };
|
|
5459
|
+
}
|
|
5460
|
+
const up = (1 - fc) * map.top[c0] + fc * map.top[c1] - height;
|
|
5461
|
+
const down = height - ((1 - fc) * map.bottom[c0] + fc * map.bottom[c1]);
|
|
5462
|
+
if (up < radial && up <= down) {
|
|
5463
|
+
return { depth: Math.max(0, up), direction: map.axis.clone() };
|
|
5464
|
+
}
|
|
5465
|
+
if (down < radial) {
|
|
5466
|
+
return { depth: Math.max(0, down), direction: map.axis.clone().negate() };
|
|
5467
|
+
}
|
|
5468
|
+
const direction = distance > 1e-6 ? map.reference.clone().multiplyScalar(x / distance).add(binormal.clone().multiplyScalar(y / distance)) : map.reference.clone();
|
|
5469
|
+
return { depth: radial, direction };
|
|
5470
|
+
};
|
|
5471
|
+
var marchPoint = new Vector33();
|
|
5472
|
+
var escapeAlong = function(map, point, direction, maxDistance = 0.3, step = 5e-3) {
|
|
5473
|
+
for (let distance = step; distance <= maxDistance; distance += step) {
|
|
5474
|
+
marchPoint.copy(point).addScaledVector(direction, distance);
|
|
5475
|
+
if (queryRadialMap(map, marchPoint).depth <= 0) {
|
|
5476
|
+
return distance;
|
|
5477
|
+
}
|
|
5478
|
+
}
|
|
5479
|
+
return Infinity;
|
|
5480
|
+
};
|
|
5481
|
+
var rotationBetween = function(a, b) {
|
|
5482
|
+
return new Quaternion2().setFromUnitVectors(
|
|
5483
|
+
a.clone().normalize(),
|
|
5484
|
+
b.clone().normalize()
|
|
4697
5485
|
);
|
|
4698
5486
|
};
|
|
4699
|
-
var
|
|
4700
|
-
|
|
4701
|
-
|
|
5487
|
+
var quaternionToRotationVector = function(q) {
|
|
5488
|
+
const w = q.w < 0 ? -q.w : q.w;
|
|
5489
|
+
const sign = q.w < 0 ? -1 : 1;
|
|
5490
|
+
const sinHalf = Math.hypot(q.x, q.y, q.z);
|
|
5491
|
+
if (sinHalf < 1e-9) {
|
|
5492
|
+
return new Vector33(2 * q.x * sign, 2 * q.y * sign, 2 * q.z * sign);
|
|
4702
5493
|
}
|
|
4703
|
-
const
|
|
4704
|
-
|
|
4705
|
-
|
|
5494
|
+
const angle = 2 * Math.atan2(sinHalf, w);
|
|
5495
|
+
return new Vector33(q.x, q.y, q.z).multiplyScalar(sign * angle / sinHalf);
|
|
5496
|
+
};
|
|
5497
|
+
var rotationVectorToQuaternion = function(v) {
|
|
5498
|
+
const angle = v.length();
|
|
5499
|
+
if (angle < 1e-9) {
|
|
5500
|
+
return new Quaternion2(v.x / 2, v.y / 2, v.z / 2, 1).normalize();
|
|
4706
5501
|
}
|
|
4707
|
-
|
|
4708
|
-
|
|
4709
|
-
|
|
4710
|
-
|
|
5502
|
+
return new Quaternion2().setFromAxisAngle(v.clone().divideScalar(angle), angle);
|
|
5503
|
+
};
|
|
5504
|
+
var widenAndSmooth = function(corrections, hold, sigma) {
|
|
5505
|
+
const count = corrections.length;
|
|
5506
|
+
const held = corrections.map((_, frame) => {
|
|
5507
|
+
let best = corrections[frame];
|
|
5508
|
+
for (let k = Math.max(0, frame - hold); k <= Math.min(count - 1, frame + hold); k += 1) {
|
|
5509
|
+
if (corrections[k].lengthSq() > best.lengthSq()) {
|
|
5510
|
+
best = corrections[k];
|
|
4711
5511
|
}
|
|
4712
|
-
return Reflect.get(source, property, receiver);
|
|
4713
5512
|
}
|
|
5513
|
+
return best;
|
|
5514
|
+
});
|
|
5515
|
+
const reach = Math.ceil(3 * sigma);
|
|
5516
|
+
return held.map((_, frame) => {
|
|
5517
|
+
const sum = new Vector33();
|
|
5518
|
+
let weight = 0;
|
|
5519
|
+
for (let k = -reach; k <= reach; k += 1) {
|
|
5520
|
+
const index = frame + k;
|
|
5521
|
+
if (index < 0 || index >= count) {
|
|
5522
|
+
continue;
|
|
5523
|
+
}
|
|
5524
|
+
const w = Math.exp(-0.5 * (k / sigma) ** 2);
|
|
5525
|
+
sum.addScaledVector(held[index], w);
|
|
5526
|
+
weight += w;
|
|
5527
|
+
}
|
|
5528
|
+
return sum.divideScalar(weight);
|
|
4714
5529
|
});
|
|
4715
|
-
canonicalAnimationTargets.set(vrm, target);
|
|
4716
|
-
return target;
|
|
4717
5530
|
};
|
|
4718
|
-
var
|
|
4719
|
-
const
|
|
4720
|
-
const
|
|
4721
|
-
|
|
4722
|
-
|
|
4723
|
-
|
|
4724
|
-
|
|
4725
|
-
|
|
5531
|
+
var distanceToSegment = function(point, a, b) {
|
|
5532
|
+
const ab = b.clone().sub(a);
|
|
5533
|
+
const t = Math.max(
|
|
5534
|
+
0,
|
|
5535
|
+
Math.min(1, point.clone().sub(a).dot(ab) / Math.max(ab.lengthSq(), 1e-12))
|
|
5536
|
+
);
|
|
5537
|
+
return point.distanceTo(a.clone().addScaledVector(ab, t));
|
|
5538
|
+
};
|
|
5539
|
+
var HAND_SKIN_DISTANCE = 0.035;
|
|
5540
|
+
var handSkeletonSegments = function(wrist, elbow, fingers) {
|
|
5541
|
+
const segments = [];
|
|
5542
|
+
for (const chain of fingers) {
|
|
5543
|
+
let previous = wrist;
|
|
5544
|
+
for (const joint of chain) {
|
|
5545
|
+
segments.push([previous, joint]);
|
|
5546
|
+
previous = joint;
|
|
5547
|
+
}
|
|
5548
|
+
if (chain.length >= 2) {
|
|
5549
|
+
const last = chain[chain.length - 1];
|
|
5550
|
+
const before = chain[chain.length - 2];
|
|
5551
|
+
segments.push([last, last.clone().add(last.clone().sub(before))]);
|
|
4726
5552
|
}
|
|
4727
5553
|
}
|
|
4728
|
-
|
|
5554
|
+
if (!segments.length) {
|
|
5555
|
+
const direction = wrist.clone().sub(elbow).normalize();
|
|
5556
|
+
segments.push([wrist, wrist.clone().addScaledVector(direction, 0.15)]);
|
|
5557
|
+
}
|
|
5558
|
+
return segments;
|
|
4729
5559
|
};
|
|
4730
|
-
var
|
|
5560
|
+
var isHandSkin = function(point, segments) {
|
|
5561
|
+
return segments.some(
|
|
5562
|
+
([a, b]) => distanceToSegment(point, a, b) <= HAND_SKIN_DISTANCE
|
|
5563
|
+
);
|
|
5564
|
+
};
|
|
5565
|
+
|
|
5566
|
+
// ../vrm-renderer/src/hand-body.ts
|
|
5567
|
+
var SIDES = ["left", "right"];
|
|
5568
|
+
var TORSO_PARTS = [
|
|
5569
|
+
"hips",
|
|
5570
|
+
"spine",
|
|
5571
|
+
"chest",
|
|
5572
|
+
"upperChest",
|
|
5573
|
+
"neck"
|
|
5574
|
+
];
|
|
5575
|
+
var LIMB_PARTS = [
|
|
5576
|
+
["leftUpperLeg", "leftLowerLeg"],
|
|
5577
|
+
["rightUpperLeg", "rightLowerLeg"],
|
|
5578
|
+
["leftLowerLeg", "leftFoot"],
|
|
5579
|
+
["rightLowerLeg", "rightFoot"],
|
|
5580
|
+
["leftShoulder", "leftUpperArm"],
|
|
5581
|
+
["rightShoulder", "rightUpperArm"]
|
|
5582
|
+
];
|
|
5583
|
+
var FINGERS = ["Thumb", "Index", "Middle", "Ring", "Little"];
|
|
5584
|
+
var isHandBone = function(name, side) {
|
|
5585
|
+
if (!name.startsWith(side)) {
|
|
5586
|
+
return false;
|
|
5587
|
+
}
|
|
5588
|
+
const rest = name.slice(side.length);
|
|
5589
|
+
return rest === "Hand" || FINGERS.some((finger) => rest.startsWith(finger));
|
|
5590
|
+
};
|
|
5591
|
+
var MAX_HAND_POINTS = 48;
|
|
5592
|
+
var MAX_PART_POINTS = 6e3;
|
|
5593
|
+
var UP = new Vector34(0, 1, 0);
|
|
5594
|
+
var bodyModels = /* @__PURE__ */ new WeakMap();
|
|
5595
|
+
var humanBoneNames = function(vrm) {
|
|
5596
|
+
return Object.keys(vrm.humanoid.humanBones).filter(
|
|
5597
|
+
(name) => Boolean(vrm.humanoid.getNormalizedBoneNode(name))
|
|
5598
|
+
);
|
|
5599
|
+
};
|
|
5600
|
+
var withRestPose = function(vrm, run) {
|
|
5601
|
+
const names = humanBoneNames(vrm);
|
|
5602
|
+
const saved = names.map((name) => {
|
|
5603
|
+
const node = vrm.humanoid.getNormalizedBoneNode(name);
|
|
5604
|
+
return [node, node.position.clone(), node.quaternion.clone()];
|
|
5605
|
+
});
|
|
5606
|
+
vrm.humanoid.resetNormalizedPose();
|
|
5607
|
+
vrm.humanoid.update();
|
|
5608
|
+
vrm.scene.updateMatrixWorld(true);
|
|
5609
|
+
try {
|
|
5610
|
+
return run();
|
|
5611
|
+
} finally {
|
|
5612
|
+
for (const [node, position, quaternion] of saved) {
|
|
5613
|
+
node.position.copy(position);
|
|
5614
|
+
node.quaternion.copy(quaternion);
|
|
5615
|
+
}
|
|
5616
|
+
vrm.humanoid.update();
|
|
5617
|
+
vrm.scene.updateMatrixWorld(true);
|
|
5618
|
+
}
|
|
5619
|
+
};
|
|
5620
|
+
var restMouth = function(vrm, toRig) {
|
|
5621
|
+
const expression = vrm.expressionManager?.getExpression("aa");
|
|
5622
|
+
const sum = new Vector34();
|
|
5623
|
+
const vertex = new Vector34();
|
|
5624
|
+
let count = 0;
|
|
5625
|
+
for (const bind of expression?._binds ?? []) {
|
|
5626
|
+
if (!bind.primitives || bind.index === void 0) {
|
|
5627
|
+
continue;
|
|
5628
|
+
}
|
|
5629
|
+
for (const mesh of bind.primitives) {
|
|
5630
|
+
const morph = mesh.geometry.morphAttributes.position?.[bind.index];
|
|
5631
|
+
if (!morph || !mesh.isSkinnedMesh) {
|
|
5632
|
+
continue;
|
|
5633
|
+
}
|
|
5634
|
+
for (let i = 0; i < morph.count; i += 1) {
|
|
5635
|
+
if (Math.hypot(morph.getX(i), morph.getY(i), morph.getZ(i)) > 2e-3) {
|
|
5636
|
+
mesh.getVertexPosition(i, vertex);
|
|
5637
|
+
sum.add(vertex.applyMatrix4(mesh.matrixWorld).applyMatrix4(toRig));
|
|
5638
|
+
count += 1;
|
|
5639
|
+
}
|
|
5640
|
+
}
|
|
5641
|
+
}
|
|
5642
|
+
}
|
|
5643
|
+
return count ? sum.divideScalar(count) : null;
|
|
5644
|
+
};
|
|
5645
|
+
var buildBodyModel = function(vrm) {
|
|
5646
|
+
const humanoid = vrm.humanoid;
|
|
5647
|
+
const names = humanBoneNames(vrm);
|
|
5648
|
+
for (const required of [
|
|
5649
|
+
"hips",
|
|
5650
|
+
"leftUpperArm",
|
|
5651
|
+
"rightUpperArm",
|
|
5652
|
+
"leftHand",
|
|
5653
|
+
"rightHand"
|
|
5654
|
+
]) {
|
|
5655
|
+
if (!names.includes(required)) {
|
|
5656
|
+
return null;
|
|
5657
|
+
}
|
|
5658
|
+
}
|
|
5659
|
+
return withRestPose(vrm, () => {
|
|
5660
|
+
const toRig = humanoid.normalizedHumanBonesRoot.matrixWorld.clone().invert();
|
|
5661
|
+
const rest = /* @__PURE__ */ new Map();
|
|
5662
|
+
const nameOfNormalized = /* @__PURE__ */ new Map();
|
|
5663
|
+
const nameOfRaw = /* @__PURE__ */ new Map();
|
|
5664
|
+
for (const name of names) {
|
|
5665
|
+
const node = humanoid.getNormalizedBoneNode(name);
|
|
5666
|
+
nameOfNormalized.set(node, name);
|
|
5667
|
+
rest.set(name, node.getWorldPosition(new Vector34()).applyMatrix4(toRig));
|
|
5668
|
+
const raw = humanoid.getRawBoneNode(name);
|
|
5669
|
+
if (raw) {
|
|
5670
|
+
nameOfRaw.set(raw, name);
|
|
5671
|
+
}
|
|
5672
|
+
}
|
|
5673
|
+
const parent = /* @__PURE__ */ new Map();
|
|
5674
|
+
const depth = /* @__PURE__ */ new Map();
|
|
5675
|
+
for (const name of names) {
|
|
5676
|
+
let node = humanoid.getNormalizedBoneNode(name).parent;
|
|
5677
|
+
while (node && !nameOfNormalized.has(node)) {
|
|
5678
|
+
node = node.parent;
|
|
5679
|
+
}
|
|
5680
|
+
parent.set(name, node ? nameOfNormalized.get(node) : null);
|
|
5681
|
+
}
|
|
5682
|
+
const depthOf = (name) => {
|
|
5683
|
+
const known = depth.get(name);
|
|
5684
|
+
if (known !== void 0) {
|
|
5685
|
+
return known;
|
|
5686
|
+
}
|
|
5687
|
+
const up = parent.get(name);
|
|
5688
|
+
const value = up ? depthOf(up) + 1 : 0;
|
|
5689
|
+
depth.set(name, value);
|
|
5690
|
+
return value;
|
|
5691
|
+
};
|
|
5692
|
+
const order = [...names].sort((a, b) => depthOf(a) - depthOf(b));
|
|
5693
|
+
const attachCache = /* @__PURE__ */ new Map();
|
|
5694
|
+
const attachOf = (bone) => {
|
|
5695
|
+
if (attachCache.has(bone)) {
|
|
5696
|
+
return attachCache.get(bone);
|
|
5697
|
+
}
|
|
5698
|
+
let node = bone;
|
|
5699
|
+
while (node && !nameOfRaw.has(node)) {
|
|
5700
|
+
node = node.parent;
|
|
5701
|
+
}
|
|
5702
|
+
const result = node ? nameOfRaw.get(node) : null;
|
|
5703
|
+
attachCache.set(bone, result);
|
|
5704
|
+
return result;
|
|
5705
|
+
};
|
|
5706
|
+
const fingerChains = (side) => FINGERS.map(
|
|
5707
|
+
(finger) => ["Metacarpal", "Proximal", "Intermediate", "Distal"].map(
|
|
5708
|
+
(segment) => rest.get(`${side}${finger}${segment}`)
|
|
5709
|
+
).filter((joint) => Boolean(joint))
|
|
5710
|
+
).filter((chain) => chain.length > 0);
|
|
5711
|
+
const handSegments = {
|
|
5712
|
+
left: handSkeletonSegments(
|
|
5713
|
+
rest.get("leftHand"),
|
|
5714
|
+
rest.get("leftLowerArm") ?? rest.get("leftHand"),
|
|
5715
|
+
fingerChains("left")
|
|
5716
|
+
),
|
|
5717
|
+
right: handSkeletonSegments(
|
|
5718
|
+
rest.get("rightHand"),
|
|
5719
|
+
rest.get("rightLowerArm") ?? rest.get("rightHand"),
|
|
5720
|
+
fingerChains("right")
|
|
5721
|
+
)
|
|
5722
|
+
};
|
|
5723
|
+
const partPoints = /* @__PURE__ */ new Map();
|
|
5724
|
+
const handPoints = { left: [], right: [] };
|
|
5725
|
+
let total = 0;
|
|
5726
|
+
vrm.scene.traverse((object) => {
|
|
5727
|
+
if (object instanceof SkinnedMesh && object.visible) {
|
|
5728
|
+
total += object.geometry.getAttribute("position")?.count ?? 0;
|
|
5729
|
+
}
|
|
5730
|
+
});
|
|
5731
|
+
const stride = Math.max(1, Math.floor(total / (MAX_PART_POINTS * 8)));
|
|
5732
|
+
const vertex = new Vector34();
|
|
5733
|
+
let counter = 0;
|
|
5734
|
+
vrm.scene.traverse((object) => {
|
|
5735
|
+
if (!(object instanceof SkinnedMesh) || !object.visible) {
|
|
5736
|
+
return;
|
|
5737
|
+
}
|
|
5738
|
+
const geometry = object.geometry;
|
|
5739
|
+
const position = geometry.getAttribute("position");
|
|
5740
|
+
const skinIndex = geometry.getAttribute("skinIndex");
|
|
5741
|
+
const skinWeight = geometry.getAttribute("skinWeight");
|
|
5742
|
+
if (!position || !skinIndex || !skinWeight) {
|
|
5743
|
+
return;
|
|
5744
|
+
}
|
|
5745
|
+
for (let i = 0; i < position.count; i += 1, counter += 1) {
|
|
5746
|
+
if (counter % stride !== 0) {
|
|
5747
|
+
continue;
|
|
5748
|
+
}
|
|
5749
|
+
let best = 0;
|
|
5750
|
+
let bestWeight = -1;
|
|
5751
|
+
for (let k = 0; k < 4; k += 1) {
|
|
5752
|
+
const weight = skinWeight.getComponent(i, k);
|
|
5753
|
+
if (weight > bestWeight) {
|
|
5754
|
+
bestWeight = weight;
|
|
5755
|
+
best = skinIndex.getComponent(i, k);
|
|
5756
|
+
}
|
|
5757
|
+
}
|
|
5758
|
+
const bone = object.skeleton.bones[best];
|
|
5759
|
+
const attach = bone ? attachOf(bone) : null;
|
|
5760
|
+
if (!attach) {
|
|
5761
|
+
continue;
|
|
5762
|
+
}
|
|
5763
|
+
object.getVertexPosition(i, vertex);
|
|
5764
|
+
const point = vertex.applyMatrix4(object.matrixWorld).applyMatrix4(toRig).clone();
|
|
5765
|
+
const side = SIDES.find((s) => isHandBone(attach, s));
|
|
5766
|
+
if (side && (nameOfRaw.get(bone) !== attach || !isHandSkin(point, handSegments[side]))) {
|
|
5767
|
+
continue;
|
|
5768
|
+
}
|
|
5769
|
+
if (side) {
|
|
5770
|
+
handPoints[side].push({ bone: attach, rest: point });
|
|
5771
|
+
continue;
|
|
5772
|
+
}
|
|
5773
|
+
const list = partPoints.get(attach) ?? [];
|
|
5774
|
+
list.push(point);
|
|
5775
|
+
partPoints.set(attach, list);
|
|
5776
|
+
}
|
|
5777
|
+
});
|
|
5778
|
+
const maps = [];
|
|
5779
|
+
for (const bone of TORSO_PARTS) {
|
|
5780
|
+
const origin = rest.get(bone);
|
|
5781
|
+
const points = partPoints.get(bone);
|
|
5782
|
+
const map = origin && points ? buildRadialMap(bone, origin, UP, points) : null;
|
|
5783
|
+
if (map) maps.push(map);
|
|
5784
|
+
}
|
|
5785
|
+
for (const [bone, end] of LIMB_PARTS) {
|
|
5786
|
+
const origin = rest.get(bone);
|
|
5787
|
+
const target = rest.get(end);
|
|
5788
|
+
const points = partPoints.get(bone);
|
|
5789
|
+
const map = origin && target && points ? buildRadialMap(bone, origin, target.clone().sub(origin), points) : null;
|
|
5790
|
+
if (map) maps.push(map);
|
|
5791
|
+
}
|
|
5792
|
+
const hands = { left: [], right: [] };
|
|
5793
|
+
for (const side of SIDES) {
|
|
5794
|
+
const list = handPoints[side];
|
|
5795
|
+
const step = Math.max(1, Math.floor(list.length / MAX_HAND_POINTS));
|
|
5796
|
+
for (let i = 0; i < list.length; i += step) {
|
|
5797
|
+
hands[side].push(list[i]);
|
|
5798
|
+
}
|
|
5799
|
+
const wrist = `${side}Hand`;
|
|
5800
|
+
hands[side].push({ bone: wrist, rest: rest.get(wrist).clone() });
|
|
5801
|
+
}
|
|
5802
|
+
if (!maps.length) {
|
|
5803
|
+
return null;
|
|
5804
|
+
}
|
|
5805
|
+
const forward = new Vector34(0, 0, 1);
|
|
5806
|
+
const foot = rest.get("leftFoot");
|
|
5807
|
+
const toes = rest.get("leftToes");
|
|
5808
|
+
if (foot && toes) {
|
|
5809
|
+
const step = toes.clone().sub(foot).setY(0);
|
|
5810
|
+
if (step.lengthSq() > 1e-6) {
|
|
5811
|
+
forward.copy(step.normalize());
|
|
5812
|
+
}
|
|
5813
|
+
}
|
|
5814
|
+
const handLength = { left: 0, right: 0 };
|
|
5815
|
+
for (const side of SIDES) {
|
|
5816
|
+
const wrist = rest.get(`${side}Hand`);
|
|
5817
|
+
for (const point of hands[side]) {
|
|
5818
|
+
handLength[side] = Math.max(
|
|
5819
|
+
handLength[side],
|
|
5820
|
+
point.rest.distanceTo(wrist)
|
|
5821
|
+
);
|
|
5822
|
+
}
|
|
5823
|
+
}
|
|
5824
|
+
const mouth = restMouth(vrm, toRig);
|
|
5825
|
+
return { rest, parent, order, maps, hands, forward, mouth, handLength };
|
|
5826
|
+
});
|
|
5827
|
+
};
|
|
5828
|
+
var bodyModelOf = function(vrm) {
|
|
5829
|
+
if (!bodyModels.has(vrm)) {
|
|
5830
|
+
const humanoid = vrm.humanoid;
|
|
5831
|
+
let model = null;
|
|
5832
|
+
if (humanoid?.humanBones && typeof humanoid.getRawBoneNode === "function" && typeof humanoid.resetNormalizedPose === "function") {
|
|
5833
|
+
try {
|
|
5834
|
+
model = buildBodyModel(vrm);
|
|
5835
|
+
} catch {
|
|
5836
|
+
model = null;
|
|
5837
|
+
}
|
|
5838
|
+
}
|
|
5839
|
+
bodyModels.set(vrm, model);
|
|
5840
|
+
}
|
|
5841
|
+
return bodyModels.get(vrm) ?? null;
|
|
5842
|
+
};
|
|
5843
|
+
var solvePose = function(model, local, hips, only, into) {
|
|
5844
|
+
const pose = into ?? { rotation: /* @__PURE__ */ new Map(), position: /* @__PURE__ */ new Map() };
|
|
5845
|
+
for (const name of model.order) {
|
|
5846
|
+
if (only && !only.has(name)) {
|
|
5847
|
+
continue;
|
|
5848
|
+
}
|
|
5849
|
+
const up = model.parent.get(name) ?? null;
|
|
5850
|
+
const q = local.get(name);
|
|
5851
|
+
if (!up) {
|
|
5852
|
+
pose.rotation.set(name, (q ?? new Quaternion3()).clone());
|
|
5853
|
+
pose.position.set(name, (hips ?? model.rest.get(name)).clone());
|
|
5854
|
+
continue;
|
|
5855
|
+
}
|
|
5856
|
+
const parentRotation = pose.rotation.get(up);
|
|
5857
|
+
const parentPosition = pose.position.get(up);
|
|
5858
|
+
pose.rotation.set(
|
|
5859
|
+
name,
|
|
5860
|
+
q ? parentRotation.clone().multiply(q) : parentRotation.clone()
|
|
5861
|
+
);
|
|
5862
|
+
pose.position.set(
|
|
5863
|
+
name,
|
|
5864
|
+
model.rest.get(name).clone().sub(model.rest.get(up)).applyQuaternion(parentRotation).add(parentPosition)
|
|
5865
|
+
);
|
|
5866
|
+
}
|
|
5867
|
+
return pose;
|
|
5868
|
+
};
|
|
5869
|
+
var pointInPose = function(model, pose, point) {
|
|
5870
|
+
return point.rest.clone().sub(model.rest.get(point.bone)).applyQuaternion(pose.rotation.get(point.bone)).add(pose.position.get(point.bone));
|
|
5871
|
+
};
|
|
5872
|
+
var trackOf = function(clip, name) {
|
|
5873
|
+
return clip.tracks.find(
|
|
5874
|
+
(track) => track.name === name && track.times.length > 0
|
|
5875
|
+
);
|
|
5876
|
+
};
|
|
5877
|
+
var sampleClipFrames = function(model, vrm, clip, fps) {
|
|
5878
|
+
if (clip.duration <= 0) {
|
|
5879
|
+
return null;
|
|
5880
|
+
}
|
|
5881
|
+
const count = Math.max(2, Math.round(clip.duration * fps) + 1);
|
|
5882
|
+
const times = Array.from(
|
|
5883
|
+
{ length: count },
|
|
5884
|
+
(_, i) => Math.min(clip.duration, i / fps)
|
|
5885
|
+
);
|
|
5886
|
+
const nodeName = (name) => vrm.humanoid.getNormalizedBoneNode(name)?.name;
|
|
5887
|
+
const interpolants = /* @__PURE__ */ new Map();
|
|
5888
|
+
for (const name of model.order) {
|
|
5889
|
+
const node = nodeName(name);
|
|
5890
|
+
const track = node ? trackOf(clip, `${node}.quaternion`) : void 0;
|
|
5891
|
+
if (track) {
|
|
5892
|
+
interpolants.set(
|
|
5893
|
+
name,
|
|
5894
|
+
new QuaternionLinearInterpolant(track.times, track.values, 4)
|
|
5895
|
+
);
|
|
5896
|
+
}
|
|
5897
|
+
}
|
|
5898
|
+
if (!interpolants.has("leftUpperArm") && !interpolants.has("rightUpperArm")) {
|
|
5899
|
+
return null;
|
|
5900
|
+
}
|
|
5901
|
+
const hipsNode = nodeName("hips");
|
|
5902
|
+
const hipsTrack = hipsNode ? trackOf(clip, `${hipsNode}.position`) : void 0;
|
|
5903
|
+
const hipsInterpolant = hipsTrack ? new LinearInterpolant(hipsTrack.times, hipsTrack.values, 3) : void 0;
|
|
5904
|
+
const frames = times.map((time) => {
|
|
5905
|
+
const local = /* @__PURE__ */ new Map();
|
|
5906
|
+
for (const [name, interpolant] of interpolants) {
|
|
5907
|
+
const v = interpolant.evaluate(time);
|
|
5908
|
+
local.set(name, new Quaternion3(v[0], v[1], v[2], v[3]).normalize());
|
|
5909
|
+
}
|
|
5910
|
+
const h = hipsInterpolant?.evaluate(time);
|
|
5911
|
+
return { local, hips: h ? new Vector34(h[0], h[1], h[2]) : null };
|
|
5912
|
+
});
|
|
5913
|
+
return { times, frames, nodeName };
|
|
5914
|
+
};
|
|
5915
|
+
|
|
5916
|
+
// ../vrm-renderer/src/face-reach.ts
|
|
5917
|
+
var SOMA_REST = {
|
|
5918
|
+
hips: [0, 1, 0],
|
|
5919
|
+
spine: [-1e-4, 1.05, -5e-4],
|
|
5920
|
+
chest: [-1e-4, 1.1213, -8e-4],
|
|
5921
|
+
upperChest: [-1e-4, 1.1968, -9e-3],
|
|
5922
|
+
neck: [-2e-3, 1.4599, -0.0145],
|
|
5923
|
+
head: [-11e-4, 1.5994, 0.0241],
|
|
5924
|
+
leftShoulder: [0.0161, 1.4292, 0.0421],
|
|
5925
|
+
leftUpperArm: [0.1653, 1.4292, -0.0129],
|
|
5926
|
+
leftLowerArm: [0.4527, 1.4292, -0.0129],
|
|
5927
|
+
leftHand: [0.7236, 1.4292, -0.0129],
|
|
5928
|
+
rightShoulder: [-0.0139, 1.4286, 0.0431],
|
|
5929
|
+
rightUpperArm: [-0.1643, 1.4286, -0.0123],
|
|
5930
|
+
rightLowerArm: [-0.4517, 1.4286, -0.0123],
|
|
5931
|
+
rightHand: [-0.723, 1.4286, -0.0123]
|
|
5932
|
+
};
|
|
5933
|
+
var SOMA_PARENT = {
|
|
5934
|
+
spine: "hips",
|
|
5935
|
+
chest: "spine",
|
|
5936
|
+
upperChest: "chest",
|
|
5937
|
+
neck: "upperChest",
|
|
5938
|
+
head: "neck",
|
|
5939
|
+
leftShoulder: "upperChest",
|
|
5940
|
+
leftUpperArm: "leftShoulder",
|
|
5941
|
+
leftLowerArm: "leftUpperArm",
|
|
5942
|
+
leftHand: "leftLowerArm",
|
|
5943
|
+
rightShoulder: "upperChest",
|
|
5944
|
+
rightUpperArm: "rightShoulder",
|
|
5945
|
+
rightLowerArm: "rightUpperArm",
|
|
5946
|
+
rightHand: "rightLowerArm"
|
|
5947
|
+
};
|
|
5948
|
+
var SOMA_ORDER = Object.keys(SOMA_REST);
|
|
5949
|
+
var SOMA_MOUTH = new Vector35(2e-4, -0.0257, 0.1011);
|
|
5950
|
+
var SOMA_HAND_LENGTH = 0.19;
|
|
5951
|
+
var PALM_SHARE = 0.45;
|
|
5952
|
+
var NEAR = 0.05;
|
|
5953
|
+
var FAR = 0.09;
|
|
5954
|
+
var isKimodoAnimation = function(animation) {
|
|
5955
|
+
const hips = animation.restHipsPosition;
|
|
5956
|
+
return Math.abs(hips.x) < 1e-3 && Math.abs(hips.y - 1) < 1e-3 && Math.abs(hips.z) < 1e-3;
|
|
5957
|
+
};
|
|
5958
|
+
var smoothstep = function(edge0, edge1, x) {
|
|
5959
|
+
const t = Math.max(0, Math.min(1, (x - edge0) / (edge1 - edge0)));
|
|
5960
|
+
return t * t * (3 - 2 * t);
|
|
5961
|
+
};
|
|
5962
|
+
var somaPose = function(frame) {
|
|
5963
|
+
const pose = { rotation: /* @__PURE__ */ new Map(), position: /* @__PURE__ */ new Map() };
|
|
5964
|
+
for (const name of SOMA_ORDER) {
|
|
5965
|
+
const local = frame.local.get(name) ?? new Quaternion4();
|
|
5966
|
+
const up = SOMA_PARENT[name];
|
|
5967
|
+
const rest = new Vector35().fromArray(SOMA_REST[name]);
|
|
5968
|
+
if (!up) {
|
|
5969
|
+
pose.rotation.set(name, local.clone());
|
|
5970
|
+
pose.position.set(name, rest);
|
|
5971
|
+
continue;
|
|
5972
|
+
}
|
|
5973
|
+
const parentRotation = pose.rotation.get(up);
|
|
5974
|
+
pose.rotation.set(name, parentRotation.clone().multiply(local));
|
|
5975
|
+
pose.position.set(
|
|
5976
|
+
name,
|
|
5977
|
+
rest.sub(new Vector35().fromArray(SOMA_REST[up])).applyQuaternion(parentRotation).add(pose.position.get(up))
|
|
5978
|
+
);
|
|
5979
|
+
}
|
|
5980
|
+
return pose;
|
|
5981
|
+
};
|
|
5982
|
+
var palmOf = function(pose, rest, side, handLength) {
|
|
5983
|
+
const hand = `${side}Hand`;
|
|
5984
|
+
const lower = `${side}LowerArm`;
|
|
5985
|
+
const direction = rest(hand).clone().sub(rest(lower)).normalize();
|
|
5986
|
+
return direction.multiplyScalar(PALM_SHARE * handLength).applyQuaternion(pose.rotation.get(hand)).add(pose.position.get(hand));
|
|
5987
|
+
};
|
|
5988
|
+
var reachTarget = function(model, frame, pose, side) {
|
|
5989
|
+
if (!model.mouth) {
|
|
5990
|
+
return null;
|
|
5991
|
+
}
|
|
5992
|
+
const soma = somaPose(frame);
|
|
5993
|
+
const somaRest = (bone) => new Vector35().fromArray(SOMA_REST[bone]);
|
|
5994
|
+
const somaPalm = palmOf(soma, somaRest, side, SOMA_HAND_LENGTH);
|
|
5995
|
+
const somaHead = soma.rotation.get("head");
|
|
5996
|
+
const somaMouth = SOMA_MOUTH.clone().applyQuaternion(somaHead).add(soma.position.get("head"));
|
|
5997
|
+
const distance = somaPalm.distanceTo(somaMouth);
|
|
5998
|
+
const weight = 1 - smoothstep(NEAR, FAR, distance);
|
|
5999
|
+
if (weight <= 0) {
|
|
6000
|
+
return null;
|
|
6001
|
+
}
|
|
6002
|
+
const head = model.rest.get("head");
|
|
6003
|
+
const faceOffset = model.mouth.clone().sub(head);
|
|
6004
|
+
const scale = faceOffset.length() / SOMA_MOUTH.length();
|
|
6005
|
+
const relative = somaPalm.clone().sub(somaMouth).applyQuaternion(somaHead.clone().invert()).multiplyScalar(scale);
|
|
6006
|
+
const headRotation = pose.rotation.get("head");
|
|
6007
|
+
const mouth = faceOffset.applyQuaternion(headRotation).add(pose.position.get("head"));
|
|
6008
|
+
return {
|
|
6009
|
+
weight,
|
|
6010
|
+
target: relative.applyQuaternion(headRotation).add(mouth)
|
|
6011
|
+
};
|
|
6012
|
+
};
|
|
6013
|
+
var solveReach = function(model, pose, side, target) {
|
|
6014
|
+
const upper = `${side}UpperArm`;
|
|
6015
|
+
const lower = `${side}LowerArm`;
|
|
6016
|
+
const s = pose.position.get(upper);
|
|
6017
|
+
const e = pose.position.get(lower);
|
|
6018
|
+
const rest = (bone) => model.rest.get(bone);
|
|
6019
|
+
const palm = palmOf(pose, rest, side, model.handLength[side]);
|
|
6020
|
+
const forearm = palm.clone().sub(e);
|
|
6021
|
+
const a = e.distanceTo(s);
|
|
6022
|
+
const b = forearm.length();
|
|
6023
|
+
const wanted = Math.max(
|
|
6024
|
+
Math.abs(a - b) + 1e-4,
|
|
6025
|
+
Math.min(a + b - 1e-4, target.distanceTo(s))
|
|
6026
|
+
);
|
|
6027
|
+
const current = palm.distanceTo(s);
|
|
6028
|
+
const angleFor = (c) => Math.acos(Math.max(-1, Math.min(1, (a * a + b * b - c * c) / (2 * a * b))));
|
|
6029
|
+
const bend = angleFor(wanted) - angleFor(current);
|
|
6030
|
+
const upperDirection = e.clone().sub(s);
|
|
6031
|
+
let axis = new Vector35().crossVectors(upperDirection, forearm);
|
|
6032
|
+
if (axis.lengthSq() < 1e-10) {
|
|
6033
|
+
axis = new Vector35().crossVectors(upperDirection, model.forward);
|
|
6034
|
+
}
|
|
6035
|
+
axis.normalize();
|
|
6036
|
+
const elbow = new Quaternion4().setFromAxisAngle(axis, -bend);
|
|
6037
|
+
const bentPalm = forearm.clone().applyQuaternion(elbow).add(e);
|
|
6038
|
+
const shoulder = rotationBetween(bentPalm.sub(s), target.clone().sub(s));
|
|
6039
|
+
return {
|
|
6040
|
+
shoulder: quaternionToRotationVector(shoulder),
|
|
6041
|
+
elbow: quaternionToRotationVector(elbow)
|
|
6042
|
+
};
|
|
6043
|
+
};
|
|
6044
|
+
var rewrite = function(model, frames, bone, corrections, before) {
|
|
6045
|
+
return frames.map((source, index) => {
|
|
6046
|
+
const frame = before ? before(source, index) : source;
|
|
6047
|
+
const pose = solvePose(model, frame.local, frame.hips);
|
|
6048
|
+
const parent = model.parent.get(bone);
|
|
6049
|
+
const parentRotation = pose.rotation.get(parent);
|
|
6050
|
+
const world = parentRotation.clone().multiply(frame.local.get(bone) ?? new Quaternion4());
|
|
6051
|
+
return parentRotation.clone().invert().multiply(rotationVectorToQuaternion(corrections[index]).multiply(world)).normalize();
|
|
6052
|
+
});
|
|
6053
|
+
};
|
|
6054
|
+
var toTrack = function(name, times, values) {
|
|
6055
|
+
const flat = [];
|
|
6056
|
+
let previous = null;
|
|
6057
|
+
for (const q of values) {
|
|
6058
|
+
if (previous && previous.dot(q) < 0) {
|
|
6059
|
+
q.set(-q.x, -q.y, -q.z, -q.w);
|
|
6060
|
+
}
|
|
6061
|
+
previous = q;
|
|
6062
|
+
flat.push(q.x, q.y, q.z, q.w);
|
|
6063
|
+
}
|
|
6064
|
+
return new QuaternionKeyframeTrack(name, [...times], flat);
|
|
6065
|
+
};
|
|
6066
|
+
var applyFaceReach = function(clip, vrm, animation, fps = 30) {
|
|
6067
|
+
if (!isKimodoAnimation(animation)) {
|
|
6068
|
+
return clip;
|
|
6069
|
+
}
|
|
6070
|
+
const model = bodyModelOf(vrm);
|
|
6071
|
+
if (!model?.mouth || !model.rest.get("head")) {
|
|
6072
|
+
return clip;
|
|
6073
|
+
}
|
|
6074
|
+
const sampled = sampleClipFrames(model, vrm, clip, fps);
|
|
6075
|
+
if (!sampled) {
|
|
6076
|
+
return clip;
|
|
6077
|
+
}
|
|
6078
|
+
const { times, frames, nodeName } = sampled;
|
|
6079
|
+
const replaced = /* @__PURE__ */ new Map();
|
|
6080
|
+
for (const side of SIDES) {
|
|
6081
|
+
const upper = `${side}UpperArm`;
|
|
6082
|
+
const lower = `${side}LowerArm`;
|
|
6083
|
+
const upperNode = nodeName(upper);
|
|
6084
|
+
const lowerNode = nodeName(lower);
|
|
6085
|
+
if (!upperNode || !lowerNode || !model.parent.get(upper)) {
|
|
6086
|
+
continue;
|
|
6087
|
+
}
|
|
6088
|
+
const elbows = [];
|
|
6089
|
+
const shoulders = [];
|
|
6090
|
+
let any = false;
|
|
6091
|
+
for (const frame of frames) {
|
|
6092
|
+
const pose = solvePose(model, frame.local, frame.hips);
|
|
6093
|
+
const reach = reachTarget(model, frame, pose, side);
|
|
6094
|
+
if (!reach) {
|
|
6095
|
+
elbows.push(new Vector35());
|
|
6096
|
+
shoulders.push(new Vector35());
|
|
6097
|
+
continue;
|
|
6098
|
+
}
|
|
6099
|
+
any = true;
|
|
6100
|
+
const solved = solveReach(model, pose, side, reach.target);
|
|
6101
|
+
elbows.push(solved.elbow.multiplyScalar(reach.weight));
|
|
6102
|
+
shoulders.push(solved.shoulder.multiplyScalar(reach.weight));
|
|
6103
|
+
}
|
|
6104
|
+
if (!any) {
|
|
6105
|
+
continue;
|
|
6106
|
+
}
|
|
6107
|
+
const elbow = widenAndSmooth(elbows, 0, 2);
|
|
6108
|
+
const shoulder = widenAndSmooth(shoulders, 0, 2);
|
|
6109
|
+
const lowerValues = rewrite(model, frames, lower, elbow);
|
|
6110
|
+
const withElbow = (frame, index) => {
|
|
6111
|
+
const local = new Map(frame.local);
|
|
6112
|
+
local.set(lower, lowerValues[index].clone());
|
|
6113
|
+
return { local, hips: frame.hips };
|
|
6114
|
+
};
|
|
6115
|
+
const upperValues = rewrite(model, frames, upper, shoulder, withElbow);
|
|
6116
|
+
replaced.set(
|
|
6117
|
+
`${upperNode}.quaternion`,
|
|
6118
|
+
toTrack(`${upperNode}.quaternion`, times, upperValues)
|
|
6119
|
+
);
|
|
6120
|
+
replaced.set(
|
|
6121
|
+
`${lowerNode}.quaternion`,
|
|
6122
|
+
toTrack(`${lowerNode}.quaternion`, times, lowerValues)
|
|
6123
|
+
);
|
|
6124
|
+
}
|
|
6125
|
+
if (!replaced.size) {
|
|
6126
|
+
return clip;
|
|
6127
|
+
}
|
|
6128
|
+
const tracks = clip.tracks.filter((track) => !replaced.has(track.name));
|
|
6129
|
+
tracks.push(...replaced.values());
|
|
6130
|
+
return new AnimationClip(clip.name, clip.duration, tracks, clip.blendMode);
|
|
6131
|
+
};
|
|
6132
|
+
|
|
6133
|
+
// ../vrm-renderer/src/hand-clearance.ts
|
|
6134
|
+
import {
|
|
6135
|
+
AnimationClip as AnimationClip2,
|
|
6136
|
+
LinearInterpolant as LinearInterpolant2,
|
|
6137
|
+
Quaternion as Quaternion5,
|
|
6138
|
+
QuaternionKeyframeTrack as QuaternionKeyframeTrack2,
|
|
6139
|
+
QuaternionLinearInterpolant as QuaternionLinearInterpolant2,
|
|
6140
|
+
Vector3 as Vector36
|
|
6141
|
+
} from "three";
|
|
6142
|
+
var MAX_CORRECTION = 35 * Math.PI / 180;
|
|
6143
|
+
var MAX_ITERATIONS = 4;
|
|
6144
|
+
var deepestContact = function(model, pose, side) {
|
|
6145
|
+
let best = null;
|
|
6146
|
+
const points = model.hands[side].map(
|
|
6147
|
+
(handPoint) => pointInPose(model, pose, handPoint)
|
|
6148
|
+
);
|
|
6149
|
+
const inverse = new Quaternion5();
|
|
6150
|
+
const local = new Vector36();
|
|
6151
|
+
for (const map of model.maps) {
|
|
6152
|
+
const bone = map.bone;
|
|
6153
|
+
const rotation = pose.rotation.get(bone);
|
|
6154
|
+
const position = pose.position.get(bone);
|
|
6155
|
+
if (!rotation || !position) {
|
|
6156
|
+
continue;
|
|
6157
|
+
}
|
|
6158
|
+
inverse.copy(rotation).invert();
|
|
6159
|
+
const rest = model.rest.get(bone);
|
|
6160
|
+
const torso = TORSO_PARTS.includes(bone);
|
|
6161
|
+
for (const point of points) {
|
|
6162
|
+
local.copy(point).sub(position).applyQuaternion(inverse).add(rest);
|
|
6163
|
+
const hit = queryRadialMap(map, local);
|
|
6164
|
+
if (hit.depth <= 0) {
|
|
6165
|
+
continue;
|
|
6166
|
+
}
|
|
6167
|
+
if (torso) {
|
|
6168
|
+
const front = escapeAlong(map, local, model.forward);
|
|
6169
|
+
if (front <= 1.5 * hit.depth + 0.01) {
|
|
6170
|
+
hit.depth = front;
|
|
6171
|
+
hit.direction = model.forward.clone();
|
|
6172
|
+
}
|
|
6173
|
+
}
|
|
6174
|
+
if (!best || hit.depth > best.depth) {
|
|
6175
|
+
best = {
|
|
6176
|
+
bone: map.bone,
|
|
6177
|
+
point,
|
|
6178
|
+
depth: hit.depth,
|
|
6179
|
+
direction: hit.direction.clone().applyQuaternion(rotation)
|
|
6180
|
+
};
|
|
6181
|
+
}
|
|
6182
|
+
}
|
|
6183
|
+
}
|
|
6184
|
+
return best;
|
|
6185
|
+
};
|
|
6186
|
+
var ARM_CHAIN = function(model, side) {
|
|
6187
|
+
const result = /* @__PURE__ */ new Set();
|
|
6188
|
+
for (const name of model.order) {
|
|
6189
|
+
let node = name;
|
|
6190
|
+
while (node) {
|
|
6191
|
+
if (node === `${side}UpperArm`) {
|
|
6192
|
+
result.add(name);
|
|
6193
|
+
break;
|
|
6194
|
+
}
|
|
6195
|
+
node = model.parent.get(node) ?? null;
|
|
6196
|
+
}
|
|
6197
|
+
}
|
|
6198
|
+
return result;
|
|
6199
|
+
};
|
|
6200
|
+
var frameCorrections = function(model, frames, side, start, margin, debug) {
|
|
6201
|
+
const upper = `${side}UpperArm`;
|
|
6202
|
+
const chain = ARM_CHAIN(model, side);
|
|
6203
|
+
const upperParent = model.parent.get(upper);
|
|
6204
|
+
return frames.map((frame, index) => {
|
|
6205
|
+
const pose = solvePose(model, frame.local, frame.hips);
|
|
6206
|
+
const original = frame.local.get(upper) ?? new Quaternion5();
|
|
6207
|
+
const parentRotation = pose.rotation.get(upperParent);
|
|
6208
|
+
const parentInverse = parentRotation.clone().invert();
|
|
6209
|
+
const correction = rotationVectorToQuaternion(start[index]);
|
|
6210
|
+
const local = new Map(frame.local);
|
|
6211
|
+
const apply = () => {
|
|
6212
|
+
const world = parentRotation.clone().multiply(original);
|
|
6213
|
+
local.set(
|
|
6214
|
+
upper,
|
|
6215
|
+
parentInverse.clone().multiply(correction.clone().multiply(world))
|
|
6216
|
+
);
|
|
6217
|
+
solvePose(model, local, frame.hips, chain, pose);
|
|
6218
|
+
};
|
|
6219
|
+
apply();
|
|
6220
|
+
const added = new Quaternion5();
|
|
6221
|
+
for (let iteration = 0; iteration < MAX_ITERATIONS; iteration += 1) {
|
|
6222
|
+
const contact = deepestContact(model, pose, side);
|
|
6223
|
+
if (!contact) {
|
|
6224
|
+
break;
|
|
6225
|
+
}
|
|
6226
|
+
const shoulder = pose.position.get(upper);
|
|
6227
|
+
const from = contact.point.clone().sub(shoulder);
|
|
6228
|
+
const to = contact.point.clone().addScaledVector(contact.direction, contact.depth + margin).sub(shoulder);
|
|
6229
|
+
const step = rotationBetween(from, to);
|
|
6230
|
+
const next = step.clone().multiply(added);
|
|
6231
|
+
debug?.({
|
|
6232
|
+
side,
|
|
6233
|
+
frame: index,
|
|
6234
|
+
iteration,
|
|
6235
|
+
bone: contact.bone,
|
|
6236
|
+
depth: contact.depth,
|
|
6237
|
+
direction: contact.direction.toArray(),
|
|
6238
|
+
stepDegrees: quaternionToRotationVector(step).length() * 180 / Math.PI,
|
|
6239
|
+
totalDegrees: quaternionToRotationVector(next).length() * 180 / Math.PI
|
|
6240
|
+
});
|
|
6241
|
+
const combined = step.clone().multiply(correction);
|
|
6242
|
+
if (quaternionToRotationVector(combined).length() > MAX_CORRECTION) {
|
|
6243
|
+
break;
|
|
6244
|
+
}
|
|
6245
|
+
correction.premultiply(step);
|
|
6246
|
+
added.copy(next);
|
|
6247
|
+
apply();
|
|
6248
|
+
}
|
|
6249
|
+
return quaternionToRotationVector(added);
|
|
6250
|
+
});
|
|
6251
|
+
};
|
|
6252
|
+
var applyHandClearance = function(clip, vrm, options = {}) {
|
|
6253
|
+
const model = bodyModelOf(vrm);
|
|
6254
|
+
if (!model || clip.duration <= 0) {
|
|
6255
|
+
return clip;
|
|
6256
|
+
}
|
|
6257
|
+
const margin = options.margin ?? 0.015;
|
|
6258
|
+
const fps = options.fps ?? 30;
|
|
6259
|
+
const sampled = sampleClipFrames(model, vrm, clip, fps);
|
|
6260
|
+
if (!sampled) {
|
|
6261
|
+
return clip;
|
|
6262
|
+
}
|
|
6263
|
+
const { times, frames, nodeName } = sampled;
|
|
6264
|
+
const count = times.length;
|
|
6265
|
+
const replaced = /* @__PURE__ */ new Map();
|
|
6266
|
+
for (const side of SIDES) {
|
|
6267
|
+
const upper = `${side}UpperArm`;
|
|
6268
|
+
const upperNode = nodeName(upper);
|
|
6269
|
+
if (!upperNode || !model.parent.get(upper)) {
|
|
6270
|
+
continue;
|
|
6271
|
+
}
|
|
6272
|
+
let total = frames.map(() => new Vector36());
|
|
6273
|
+
let changed = false;
|
|
6274
|
+
for (let pass = 0; pass < 2; pass += 1) {
|
|
6275
|
+
const raw = frameCorrections(
|
|
6276
|
+
model,
|
|
6277
|
+
frames,
|
|
6278
|
+
side,
|
|
6279
|
+
total,
|
|
6280
|
+
margin,
|
|
6281
|
+
options.onDebug
|
|
6282
|
+
);
|
|
6283
|
+
if (raw.every((v) => v.lengthSq() < 1e-10)) {
|
|
6284
|
+
break;
|
|
6285
|
+
}
|
|
6286
|
+
changed = true;
|
|
6287
|
+
const smooth = widenAndSmooth(raw, 4, 3);
|
|
6288
|
+
total = total.map(
|
|
6289
|
+
(t, i) => quaternionToRotationVector(
|
|
6290
|
+
rotationVectorToQuaternion(smooth[i]).multiply(
|
|
6291
|
+
rotationVectorToQuaternion(t)
|
|
6292
|
+
)
|
|
6293
|
+
)
|
|
6294
|
+
);
|
|
6295
|
+
}
|
|
6296
|
+
if (options.onReport) {
|
|
6297
|
+
const left = frameCorrections(model, frames, side, total, margin);
|
|
6298
|
+
const degrees = total.map((v) => v.length() * 180 / Math.PI);
|
|
6299
|
+
options.onReport({
|
|
6300
|
+
side,
|
|
6301
|
+
maps: model.maps.map((m) => `${m.bone}:${m.heights}`),
|
|
6302
|
+
handPoints: model.hands[side].length,
|
|
6303
|
+
maxDegrees: Math.max(0, ...degrees),
|
|
6304
|
+
correctedFrames: degrees.filter((d) => d > 0.5).length,
|
|
6305
|
+
residualDegrees: Math.max(
|
|
6306
|
+
0,
|
|
6307
|
+
...left.map((v) => v.length() * 180 / Math.PI)
|
|
6308
|
+
)
|
|
6309
|
+
});
|
|
6310
|
+
}
|
|
6311
|
+
if (!changed) {
|
|
6312
|
+
continue;
|
|
6313
|
+
}
|
|
6314
|
+
const values = new Float32Array(count * 4);
|
|
6315
|
+
let previous = null;
|
|
6316
|
+
frames.forEach((frame, index) => {
|
|
6317
|
+
const pose = solvePose(model, frame.local, frame.hips);
|
|
6318
|
+
const parentRotation = pose.rotation.get(model.parent.get(upper));
|
|
6319
|
+
const original = frame.local.get(upper) ?? new Quaternion5();
|
|
6320
|
+
const world = parentRotation.clone().multiply(original);
|
|
6321
|
+
const corrected = parentRotation.clone().invert().multiply(rotationVectorToQuaternion(total[index]).multiply(world)).normalize();
|
|
6322
|
+
if (previous && previous.dot(corrected) < 0) {
|
|
6323
|
+
corrected.set(-corrected.x, -corrected.y, -corrected.z, -corrected.w);
|
|
6324
|
+
}
|
|
6325
|
+
previous = corrected;
|
|
6326
|
+
values.set(
|
|
6327
|
+
[corrected.x, corrected.y, corrected.z, corrected.w],
|
|
6328
|
+
index * 4
|
|
6329
|
+
);
|
|
6330
|
+
});
|
|
6331
|
+
replaced.set(
|
|
6332
|
+
`${upperNode}.quaternion`,
|
|
6333
|
+
new QuaternionKeyframeTrack2(
|
|
6334
|
+
`${upperNode}.quaternion`,
|
|
6335
|
+
times,
|
|
6336
|
+
Array.from(values)
|
|
6337
|
+
)
|
|
6338
|
+
);
|
|
6339
|
+
}
|
|
6340
|
+
if (!replaced.size) {
|
|
6341
|
+
return clip;
|
|
6342
|
+
}
|
|
6343
|
+
const tracks = clip.tracks.filter((track) => !replaced.has(track.name));
|
|
6344
|
+
tracks.push(...replaced.values());
|
|
6345
|
+
return new AnimationClip2(clip.name, clip.duration, tracks, clip.blendMode);
|
|
6346
|
+
};
|
|
6347
|
+
|
|
6348
|
+
// ../vrm-renderer/src/animation.ts
|
|
6349
|
+
import {
|
|
6350
|
+
AnimationClip as AnimationClip3,
|
|
6351
|
+
Euler,
|
|
6352
|
+
Quaternion as Quaternion6,
|
|
6353
|
+
QuaternionKeyframeTrack as QuaternionKeyframeTrack3,
|
|
6354
|
+
QuaternionLinearInterpolant as QuaternionLinearInterpolant3,
|
|
6355
|
+
Vector3 as Vector37,
|
|
6356
|
+
VectorKeyframeTrack
|
|
6357
|
+
} from "three";
|
|
6358
|
+
var NPCIFY_VRM_NORMALIZATION_ROOT_NAME = "NPCifyNormalizationRoot";
|
|
6359
|
+
var TORSO_BONES = [
|
|
6360
|
+
"hips",
|
|
6361
|
+
"spine",
|
|
6362
|
+
"chest",
|
|
6363
|
+
"upperChest"
|
|
6364
|
+
];
|
|
6365
|
+
var HEAD_BONES = ["neck", "head"];
|
|
6366
|
+
var AXIAL_BONES = [...TORSO_BONES, ...HEAD_BONES];
|
|
6367
|
+
var PARENT_OF_HEAD_BONES = [
|
|
6368
|
+
...TORSO_BONES,
|
|
6369
|
+
"neck"
|
|
6370
|
+
];
|
|
6371
|
+
var IDENTITY_QUATERNION = new Quaternion6();
|
|
6372
|
+
var UP_AXIS = new Vector37(0, 1, 0);
|
|
6373
|
+
var MAX_HEAD_PITCH_CORRECTION_RAD = 35 * Math.PI / 180;
|
|
6374
|
+
var NECK_CORRECTION_SHARE = 0.35;
|
|
6375
|
+
var EPSILON = 1e-5;
|
|
6376
|
+
var canonicalAnimationTargets = /* @__PURE__ */ new WeakMap();
|
|
6377
|
+
var isNpcifyNormalizedVrm = function(vrm) {
|
|
6378
|
+
return vrm.scene.children.some(
|
|
6379
|
+
(child) => child.name === NPCIFY_VRM_NORMALIZATION_ROOT_NAME
|
|
6380
|
+
);
|
|
6381
|
+
};
|
|
6382
|
+
var canonicalAnimationTarget = function(vrm) {
|
|
6383
|
+
if (vrm.meta.metaVersion !== "0" || !isNpcifyNormalizedVrm(vrm)) {
|
|
6384
|
+
return vrm;
|
|
6385
|
+
}
|
|
6386
|
+
const cached = canonicalAnimationTargets.get(vrm);
|
|
6387
|
+
if (cached) {
|
|
6388
|
+
return cached;
|
|
6389
|
+
}
|
|
6390
|
+
const target = new Proxy(vrm, {
|
|
6391
|
+
get(source, property, receiver) {
|
|
6392
|
+
if (property === "meta") {
|
|
6393
|
+
return { ...source.meta, metaVersion: "1" };
|
|
6394
|
+
}
|
|
6395
|
+
return Reflect.get(source, property, receiver);
|
|
6396
|
+
}
|
|
6397
|
+
});
|
|
6398
|
+
canonicalAnimationTargets.set(vrm, target);
|
|
6399
|
+
return target;
|
|
6400
|
+
};
|
|
6401
|
+
var resolveAxialBoneNodeNames = function(vrm) {
|
|
6402
|
+
const result = /* @__PURE__ */ new Map();
|
|
6403
|
+
const seenNodeNames = /* @__PURE__ */ new Set();
|
|
6404
|
+
for (const bone of AXIAL_BONES) {
|
|
6405
|
+
const nodeName = vrm.humanoid?.getNormalizedBoneNode?.(bone)?.name;
|
|
6406
|
+
if (nodeName && !seenNodeNames.has(nodeName)) {
|
|
6407
|
+
seenNodeNames.add(nodeName);
|
|
6408
|
+
result.set(bone, nodeName);
|
|
6409
|
+
}
|
|
6410
|
+
}
|
|
6411
|
+
return result;
|
|
6412
|
+
};
|
|
6413
|
+
var findQuaternionTrack = function(clip, nodeName) {
|
|
4731
6414
|
if (!nodeName) {
|
|
4732
6415
|
return void 0;
|
|
4733
6416
|
}
|
|
@@ -4756,13 +6439,13 @@ var evaluateChainOrientation = function(bones, interpolants, time, rootPreRotati
|
|
|
4756
6439
|
};
|
|
4757
6440
|
var rotateQuaternionTrackInParentSpace = function(track, parentBones, interpolants, rootPreRotation, worldCorrection) {
|
|
4758
6441
|
const values = new Float32Array(track.values.length);
|
|
4759
|
-
const parentQuat = new
|
|
4760
|
-
const parentInvQuat = new
|
|
4761
|
-
const localCorrection = new
|
|
4762
|
-
const keyframeQuat = new
|
|
4763
|
-
const prevOrigQuat = new
|
|
4764
|
-
const prevNewQuat = new
|
|
4765
|
-
const scratchQuat = new
|
|
6442
|
+
const parentQuat = new Quaternion6();
|
|
6443
|
+
const parentInvQuat = new Quaternion6();
|
|
6444
|
+
const localCorrection = new Quaternion6();
|
|
6445
|
+
const keyframeQuat = new Quaternion6();
|
|
6446
|
+
const prevOrigQuat = new Quaternion6();
|
|
6447
|
+
const prevNewQuat = new Quaternion6();
|
|
6448
|
+
const scratchQuat = new Quaternion6();
|
|
4766
6449
|
const count = track.times.length;
|
|
4767
6450
|
for (let i = 0; i < count; i += 1) {
|
|
4768
6451
|
const time = track.times[i] ?? 0;
|
|
@@ -4804,7 +6487,7 @@ var rotateQuaternionTrackInParentSpace = function(track, parentBones, interpolan
|
|
|
4804
6487
|
values[offset + 2] = keyframeQuat.z;
|
|
4805
6488
|
values[offset + 3] = keyframeQuat.w;
|
|
4806
6489
|
}
|
|
4807
|
-
return new
|
|
6490
|
+
return new QuaternionKeyframeTrack3(
|
|
4808
6491
|
track.name,
|
|
4809
6492
|
Array.from(track.times),
|
|
4810
6493
|
Array.from(values)
|
|
@@ -4856,7 +6539,7 @@ var normalizeIdleAnimationClip = function(clip, vrm) {
|
|
|
4856
6539
|
if (track) {
|
|
4857
6540
|
interpolants.set(
|
|
4858
6541
|
bone,
|
|
4859
|
-
new
|
|
6542
|
+
new QuaternionLinearInterpolant3(track.times, track.values, 4)
|
|
4860
6543
|
);
|
|
4861
6544
|
}
|
|
4862
6545
|
}
|
|
@@ -4869,11 +6552,11 @@ var normalizeIdleAnimationClip = function(clip, vrm) {
|
|
|
4869
6552
|
for (let i = 0; i < sampleCount; i += 1) {
|
|
4870
6553
|
sampleTimes.push(duration > 0 ? (i + 0.5) / sampleCount * duration : 0);
|
|
4871
6554
|
}
|
|
4872
|
-
const chainQuat = new
|
|
4873
|
-
const scratchQuat = new
|
|
4874
|
-
const forwardVec = new
|
|
6555
|
+
const chainQuat = new Quaternion6();
|
|
6556
|
+
const scratchQuat = new Quaternion6();
|
|
6557
|
+
const forwardVec = new Vector37();
|
|
4875
6558
|
const rootTorsoBone = TORSO_BONES.find((bone) => interpolants.has(bone));
|
|
4876
|
-
const rootPreRotation = new
|
|
6559
|
+
const rootPreRotation = new Quaternion6();
|
|
4877
6560
|
let torsoYawCorrection = 0;
|
|
4878
6561
|
if (rootTorsoBone) {
|
|
4879
6562
|
let sumX = 0;
|
|
@@ -4907,7 +6590,7 @@ var normalizeIdleAnimationClip = function(clip, vrm) {
|
|
|
4907
6590
|
let neckWorldCorrection = null;
|
|
4908
6591
|
let headWorldCorrection = null;
|
|
4909
6592
|
if (hasNeckTrack || hasHeadTrack) {
|
|
4910
|
-
const meanHeadForward = new
|
|
6593
|
+
const meanHeadForward = new Vector37();
|
|
4911
6594
|
for (const time of sampleTimes) {
|
|
4912
6595
|
evaluateChainOrientation(
|
|
4913
6596
|
AXIAL_BONES,
|
|
@@ -4932,11 +6615,11 @@ var normalizeIdleAnimationClip = function(clip, vrm) {
|
|
|
4932
6615
|
Math.min(MAX_HEAD_PITCH_CORRECTION_RAD, rawHeadPitch)
|
|
4933
6616
|
);
|
|
4934
6617
|
if (Math.abs(meanHeadYaw) > EPSILON || Math.abs(clampedHeadPitch) > EPSILON) {
|
|
4935
|
-
const totalWorldCorrection = new
|
|
6618
|
+
const totalWorldCorrection = new Quaternion6().setFromEuler(
|
|
4936
6619
|
new Euler(-clampedHeadPitch, -meanHeadYaw, 0, "XYZ")
|
|
4937
6620
|
);
|
|
4938
6621
|
if (hasNeckTrack && hasHeadTrack) {
|
|
4939
|
-
neckWorldCorrection = new
|
|
6622
|
+
neckWorldCorrection = new Quaternion6().setFromEuler(
|
|
4940
6623
|
new Euler(
|
|
4941
6624
|
-clampedHeadPitch * NECK_CORRECTION_SHARE,
|
|
4942
6625
|
-meanHeadYaw * NECK_CORRECTION_SHARE,
|
|
@@ -4998,17 +6681,22 @@ var normalizeIdleAnimationClip = function(clip, vrm) {
|
|
|
4998
6681
|
}
|
|
4999
6682
|
return track;
|
|
5000
6683
|
});
|
|
5001
|
-
return new
|
|
6684
|
+
return new AnimationClip3(clip.name, clip.duration, nextTracks, clip.blendMode);
|
|
5002
6685
|
};
|
|
5003
6686
|
var createVRMAnimationClip = function(animation, vrm, options) {
|
|
5004
6687
|
const clip = createPixivVRMAnimationClip(
|
|
5005
6688
|
animation,
|
|
5006
6689
|
canonicalAnimationTarget(vrm)
|
|
5007
6690
|
);
|
|
5008
|
-
|
|
5009
|
-
|
|
5010
|
-
|
|
5011
|
-
|
|
6691
|
+
const oriented = options?.normalizeIdleOrientation ? normalizeIdleAnimationClip(clip, vrm) : clip;
|
|
6692
|
+
if (options?.handClearance === false) {
|
|
6693
|
+
return oriented;
|
|
6694
|
+
}
|
|
6695
|
+
return applyHandClearance(
|
|
6696
|
+
applyFaceReach(oriented, vrm, animation),
|
|
6697
|
+
vrm,
|
|
6698
|
+
options?.handClearanceOptions
|
|
6699
|
+
);
|
|
5012
6700
|
};
|
|
5013
6701
|
|
|
5014
6702
|
// ../vrm-renderer/src/session.ts
|
|
@@ -5365,6 +7053,70 @@ var AnimationController = class {
|
|
|
5365
7053
|
activeAnimationId = null;
|
|
5366
7054
|
idleAnimationId;
|
|
5367
7055
|
pendingReturnToIdleAction = null;
|
|
7056
|
+
setVrm(nextVrm, rawAnimations) {
|
|
7057
|
+
if (nextVrm === this.vrm) {
|
|
7058
|
+
return;
|
|
7059
|
+
}
|
|
7060
|
+
const currentAnimationId = this.activeAnimationId;
|
|
7061
|
+
const currentAction = this.activeAction;
|
|
7062
|
+
const currentTime = currentAction ? currentAction.time : 0;
|
|
7063
|
+
const currentLoop = currentAction ? currentAction.loop === LoopRepeat2 : void 0;
|
|
7064
|
+
const hadPendingReturnToIdle = this.pendingReturnToIdleAction !== null && this.pendingReturnToIdleAction === currentAction;
|
|
7065
|
+
const savedConfigs = new Map(this.configs);
|
|
7066
|
+
this.restorePreview?.();
|
|
7067
|
+
this.mixer.removeEventListener("finished", this.handleActionFinished);
|
|
7068
|
+
this.mixer.stopAllAction();
|
|
7069
|
+
this.mixer.uncacheRoot(this.vrm.scene);
|
|
7070
|
+
this.vrm.humanoid?.resetNormalizedPose?.();
|
|
7071
|
+
this.vrm = nextVrm;
|
|
7072
|
+
this.mixer = new AnimationMixer2(this.vrm.scene);
|
|
7073
|
+
this.mixer.addEventListener("finished", this.handleActionFinished);
|
|
7074
|
+
this.clips.clear();
|
|
7075
|
+
this.rawClips.clear();
|
|
7076
|
+
this.normalizedIdleClips.clear();
|
|
7077
|
+
this.configs.clear();
|
|
7078
|
+
this.activeAction = null;
|
|
7079
|
+
this.activeAnimationId = null;
|
|
7080
|
+
this.pendingReturnToIdleAction = null;
|
|
7081
|
+
for (const [animationId, animation] of rawAnimations) {
|
|
7082
|
+
const config = savedConfigs.get(animationId) ?? {
|
|
7083
|
+
loop: animationId === this.idleAnimationId,
|
|
7084
|
+
weight: defaultAnimationConfig.weight
|
|
7085
|
+
};
|
|
7086
|
+
this.addAnimation(animationId, animation, config);
|
|
7087
|
+
}
|
|
7088
|
+
const neededIds = [];
|
|
7089
|
+
if (currentAnimationId && !this.clips.has(currentAnimationId)) {
|
|
7090
|
+
neededIds.push(currentAnimationId);
|
|
7091
|
+
}
|
|
7092
|
+
if (this.idleAnimationId && !this.clips.has(this.idleAnimationId) && !neededIds.includes(this.idleAnimationId)) {
|
|
7093
|
+
neededIds.push(this.idleAnimationId);
|
|
7094
|
+
}
|
|
7095
|
+
if (neededIds.length > 0 && this.prepareAnimations) {
|
|
7096
|
+
void this.prepareAnimations(neededIds).catch(() => void 0);
|
|
7097
|
+
}
|
|
7098
|
+
if (currentAnimationId && this.clips.has(currentAnimationId)) {
|
|
7099
|
+
this.playAnimation(currentAnimationId, 0, currentLoop);
|
|
7100
|
+
const resumedAction = this.getActiveAction();
|
|
7101
|
+
if (resumedAction) {
|
|
7102
|
+
const clipDuration = resumedAction.getClip().duration;
|
|
7103
|
+
if (clipDuration > 0 && Number.isFinite(currentTime)) {
|
|
7104
|
+
resumedAction.time = resumedAction.loop === LoopRepeat2 ? (currentTime % clipDuration + clipDuration) % clipDuration : Math.min(Math.max(0, currentTime), clipDuration);
|
|
7105
|
+
}
|
|
7106
|
+
if (hadPendingReturnToIdle && !currentLoop && this.idleAnimationId) {
|
|
7107
|
+
this.pendingReturnToIdleAction = resumedAction;
|
|
7108
|
+
}
|
|
7109
|
+
}
|
|
7110
|
+
this.mixer.update(0);
|
|
7111
|
+
return;
|
|
7112
|
+
}
|
|
7113
|
+
if (this.idleAnimationId && this.clips.has(this.idleAnimationId)) {
|
|
7114
|
+
this.playIdle(0);
|
|
7115
|
+
this.mixer.update(0);
|
|
7116
|
+
return;
|
|
7117
|
+
}
|
|
7118
|
+
this.vrm.humanoid?.resetNormalizedPose?.();
|
|
7119
|
+
}
|
|
5368
7120
|
getAnimationMixer() {
|
|
5369
7121
|
return this.mixer;
|
|
5370
7122
|
}
|
|
@@ -6040,6 +7792,35 @@ var PerformanceRecorder = class {
|
|
|
6040
7792
|
};
|
|
6041
7793
|
|
|
6042
7794
|
// ../../clients/web-client/src/playback-utils.ts
|
|
7795
|
+
var deliverStagingResult = async function(payload, options) {
|
|
7796
|
+
const normalized = normalizeStagingResultPayload(payload);
|
|
7797
|
+
options.setOptions(normalized.options);
|
|
7798
|
+
if (options.source === "turn") {
|
|
7799
|
+
options.onTurnReceived(normalized);
|
|
7800
|
+
}
|
|
7801
|
+
const force = options.forcePlayback === true || options.source === "replay";
|
|
7802
|
+
const shouldNotify = force || shouldNotifyStagingResult(options.notifiedStagingIds, normalized.stagingId);
|
|
7803
|
+
if (shouldNotify && options.onStagingResult) {
|
|
7804
|
+
try {
|
|
7805
|
+
await options.onStagingResult(normalized);
|
|
7806
|
+
} catch (error) {
|
|
7807
|
+
options.onCallbackError("onStagingResult", error);
|
|
7808
|
+
}
|
|
7809
|
+
}
|
|
7810
|
+
const shouldPlay = !options.hasPlayed(normalized.stagingId) || force;
|
|
7811
|
+
options.recordEvent({
|
|
7812
|
+
type: "staging.result",
|
|
7813
|
+
source: options.source,
|
|
7814
|
+
stagingId: normalized.stagingId,
|
|
7815
|
+
taskId: normalized.taskId ?? "",
|
|
7816
|
+
playback: shouldPlay ? "requested" : "already-streamed"
|
|
7817
|
+
});
|
|
7818
|
+
if (shouldPlay) {
|
|
7819
|
+
options.playRequested(normalized, { force, silent: options.silent });
|
|
7820
|
+
} else {
|
|
7821
|
+
options.playStreamed(normalized);
|
|
7822
|
+
}
|
|
7823
|
+
};
|
|
6043
7824
|
var toSilentTimeline = function(timeline) {
|
|
6044
7825
|
return {
|
|
6045
7826
|
...timeline,
|
|
@@ -6166,6 +7947,16 @@ var StreamPlaybackController = class {
|
|
|
6166
7947
|
get activeStagingId() {
|
|
6167
7948
|
return this.activePlayback?.stagingId ?? null;
|
|
6168
7949
|
}
|
|
7950
|
+
/**
|
|
7951
|
+
* A staging plays or waits to play (its audio or the rest of its stream).
|
|
7952
|
+
* False after it completed or when autoplay blocked it; the last staging
|
|
7953
|
+
* stays the active staging after that.
|
|
7954
|
+
*/
|
|
7955
|
+
get playing() {
|
|
7956
|
+
const active = this.activePlayback;
|
|
7957
|
+
const state = active ? this.states.get(active.stagingId) : void 0;
|
|
7958
|
+
return state !== void 0 && !state.completed && !state.blocked;
|
|
7959
|
+
}
|
|
6169
7960
|
onStreamEvent(callback) {
|
|
6170
7961
|
return bindCallbackSlot(
|
|
6171
7962
|
() => this.streamEventCallback,
|
|
@@ -6462,7 +8253,11 @@ var StreamPlaybackController = class {
|
|
|
6462
8253
|
() => this.characterRevealCallback?.(event, characterIndex, character)
|
|
6463
8254
|
);
|
|
6464
8255
|
}
|
|
6465
|
-
}
|
|
8256
|
+
},
|
|
8257
|
+
onToolCall: (toolCall) => {
|
|
8258
|
+
this.dependencies.onToolCall?.(toolCall);
|
|
8259
|
+
},
|
|
8260
|
+
onVariantCue: (variantKey, toolCall) => this.dependencies.onVariantCue?.(variantKey, toolCall)
|
|
6466
8261
|
};
|
|
6467
8262
|
try {
|
|
6468
8263
|
await player.play(params.timeline, playbackOptions);
|
|
@@ -7227,6 +9022,185 @@ var AudioPlayer = class {
|
|
|
7227
9022
|
}
|
|
7228
9023
|
};
|
|
7229
9024
|
|
|
9025
|
+
// ../../clients/web-client/src/character/character-variants.ts
|
|
9026
|
+
var disposeOrphanedVrm = function(vrm) {
|
|
9027
|
+
if (vrm.scene?.parent) {
|
|
9028
|
+
vrm.scene.parent.remove(vrm.scene);
|
|
9029
|
+
}
|
|
9030
|
+
if (vrm.scene && typeof vrm.scene.traverse === "function") {
|
|
9031
|
+
disposeVrm(vrm);
|
|
9032
|
+
}
|
|
9033
|
+
};
|
|
9034
|
+
var CharacterVariantController = class {
|
|
9035
|
+
constructor(options) {
|
|
9036
|
+
this.options = options;
|
|
9037
|
+
}
|
|
9038
|
+
variantVrms = /* @__PURE__ */ new Map();
|
|
9039
|
+
variantsList = [];
|
|
9040
|
+
activeVariantKey = DEFAULT_VARIANT_KEY;
|
|
9041
|
+
remoteMode = false;
|
|
9042
|
+
loadEpoch = 0;
|
|
9043
|
+
switchRevision = 0;
|
|
9044
|
+
pendingLoads = /* @__PURE__ */ new Map();
|
|
9045
|
+
listeners = {
|
|
9046
|
+
variantChange: /* @__PURE__ */ new Set(),
|
|
9047
|
+
toolCall: /* @__PURE__ */ new Set(),
|
|
9048
|
+
vrmChange: /* @__PURE__ */ new Set()
|
|
9049
|
+
};
|
|
9050
|
+
get variants() {
|
|
9051
|
+
return this.variantsList.map((variant) => ({ ...variant }));
|
|
9052
|
+
}
|
|
9053
|
+
get activeVariant() {
|
|
9054
|
+
return this.activeVariantKey;
|
|
9055
|
+
}
|
|
9056
|
+
get isRemote() {
|
|
9057
|
+
return this.remoteMode;
|
|
9058
|
+
}
|
|
9059
|
+
on(event, listener) {
|
|
9060
|
+
const bucket = this.listeners[event];
|
|
9061
|
+
bucket.add(listener);
|
|
9062
|
+
return () => {
|
|
9063
|
+
bucket.delete(listener);
|
|
9064
|
+
};
|
|
9065
|
+
}
|
|
9066
|
+
off(event, listener) {
|
|
9067
|
+
this.listeners[event].delete(listener);
|
|
9068
|
+
}
|
|
9069
|
+
emit(event, payload) {
|
|
9070
|
+
for (const listener of [...this.listeners[event]]) {
|
|
9071
|
+
invokeSafeCallback(
|
|
9072
|
+
`on(${event})`,
|
|
9073
|
+
this.options.handleCallbackError,
|
|
9074
|
+
() => listener(payload)
|
|
9075
|
+
);
|
|
9076
|
+
}
|
|
9077
|
+
}
|
|
9078
|
+
configureVariants(variants, options) {
|
|
9079
|
+
this.variantsList = variants ? variants.map((entry) => ({ ...entry })) : [];
|
|
9080
|
+
if (options?.remote !== void 0) {
|
|
9081
|
+
this.remoteMode = options.remote;
|
|
9082
|
+
}
|
|
9083
|
+
}
|
|
9084
|
+
registerDefaultVrm(vrm) {
|
|
9085
|
+
this.variantVrms.set(DEFAULT_VARIANT_KEY, vrm);
|
|
9086
|
+
this.activeVariantKey = DEFAULT_VARIANT_KEY;
|
|
9087
|
+
this.remoteMode = false;
|
|
9088
|
+
}
|
|
9089
|
+
syncDirectVrm(vrm, previousVrm) {
|
|
9090
|
+
if (vrm && !this.variantVrms.has(this.activeVariantKey)) {
|
|
9091
|
+
this.variantVrms.set(this.activeVariantKey, vrm);
|
|
9092
|
+
}
|
|
9093
|
+
if (vrm && vrm !== previousVrm) {
|
|
9094
|
+
this.emit("vrmChange", vrm);
|
|
9095
|
+
}
|
|
9096
|
+
}
|
|
9097
|
+
async preloadVariants(eager) {
|
|
9098
|
+
const keys = this.variantsList.map((variant) => variant.key).filter((key) => key !== DEFAULT_VARIANT_KEY);
|
|
9099
|
+
if (keys.length === 0) {
|
|
9100
|
+
return;
|
|
9101
|
+
}
|
|
9102
|
+
if (eager) {
|
|
9103
|
+
await Promise.all(keys.map((key) => this.ensureVariant(key)));
|
|
9104
|
+
return;
|
|
9105
|
+
}
|
|
9106
|
+
for (const key of keys) {
|
|
9107
|
+
void this.ensureVariant(key).catch(() => {
|
|
9108
|
+
});
|
|
9109
|
+
}
|
|
9110
|
+
}
|
|
9111
|
+
async ensureVariant(variantKey) {
|
|
9112
|
+
const cached = this.variantVrms.get(variantKey);
|
|
9113
|
+
if (cached) {
|
|
9114
|
+
return cached;
|
|
9115
|
+
}
|
|
9116
|
+
const existing = this.pendingLoads.get(variantKey);
|
|
9117
|
+
if (existing) {
|
|
9118
|
+
return await existing;
|
|
9119
|
+
}
|
|
9120
|
+
const epoch = this.loadEpoch;
|
|
9121
|
+
const request = this.options.loadVariant(variantKey).then((loadedVrm) => {
|
|
9122
|
+
this.pendingLoads.delete(variantKey);
|
|
9123
|
+
if (epoch !== this.loadEpoch) {
|
|
9124
|
+
disposeOrphanedVrm(loadedVrm);
|
|
9125
|
+
throw new Error("Character assets were reset while loading variant.");
|
|
9126
|
+
}
|
|
9127
|
+
this.variantVrms.set(variantKey, loadedVrm);
|
|
9128
|
+
return loadedVrm;
|
|
9129
|
+
}).catch((error) => {
|
|
9130
|
+
this.pendingLoads.delete(variantKey);
|
|
9131
|
+
throw error;
|
|
9132
|
+
});
|
|
9133
|
+
this.pendingLoads.set(variantKey, request);
|
|
9134
|
+
return await request;
|
|
9135
|
+
}
|
|
9136
|
+
async switchVariant(variantKey, toolCall = null) {
|
|
9137
|
+
if (!isValidVariantKey(variantKey)) {
|
|
9138
|
+
throw new Error(`Invalid variant key: ${variantKey}`);
|
|
9139
|
+
}
|
|
9140
|
+
const revision = ++this.switchRevision;
|
|
9141
|
+
if (this.remoteMode) {
|
|
9142
|
+
if (variantKey !== DEFAULT_VARIANT_KEY && this.variantsList.length > 0 && !this.variantsList.some((entry) => entry.key === variantKey)) {
|
|
9143
|
+
throw new Error(`Unknown variant: ${variantKey}`);
|
|
9144
|
+
}
|
|
9145
|
+
if (this.activeVariantKey === variantKey) {
|
|
9146
|
+
return;
|
|
9147
|
+
}
|
|
9148
|
+
const previousVariant = this.activeVariantKey;
|
|
9149
|
+
this.activeVariantKey = variantKey;
|
|
9150
|
+
this.emit("variantChange", {
|
|
9151
|
+
previousVariant,
|
|
9152
|
+
activeVariant: variantKey,
|
|
9153
|
+
toolCall
|
|
9154
|
+
});
|
|
9155
|
+
return;
|
|
9156
|
+
}
|
|
9157
|
+
const cached = this.variantVrms.get(variantKey);
|
|
9158
|
+
if (cached) {
|
|
9159
|
+
this.applyActiveVariant(variantKey, cached, toolCall);
|
|
9160
|
+
return;
|
|
9161
|
+
}
|
|
9162
|
+
if (variantKey === DEFAULT_VARIANT_KEY && this.activeVariantKey === DEFAULT_VARIANT_KEY) {
|
|
9163
|
+
return;
|
|
9164
|
+
}
|
|
9165
|
+
const loaded = await this.ensureVariant(variantKey);
|
|
9166
|
+
if (revision !== this.switchRevision) {
|
|
9167
|
+
return;
|
|
9168
|
+
}
|
|
9169
|
+
this.applyActiveVariant(variantKey, loaded, toolCall);
|
|
9170
|
+
}
|
|
9171
|
+
reset() {
|
|
9172
|
+
this.loadEpoch += 1;
|
|
9173
|
+
this.switchRevision += 1;
|
|
9174
|
+
this.pendingLoads.clear();
|
|
9175
|
+
const cached = [...this.variantVrms.values()];
|
|
9176
|
+
this.variantVrms.clear();
|
|
9177
|
+
this.activeVariantKey = DEFAULT_VARIANT_KEY;
|
|
9178
|
+
this.remoteMode = false;
|
|
9179
|
+
return cached;
|
|
9180
|
+
}
|
|
9181
|
+
applyActiveVariant(variantKey, nextVrm, toolCall) {
|
|
9182
|
+
const previousVariant = this.activeVariantKey;
|
|
9183
|
+
const previousVrm = this.options.getCurrentVrm();
|
|
9184
|
+
if (previousVariant === variantKey && previousVrm === nextVrm) {
|
|
9185
|
+
return;
|
|
9186
|
+
}
|
|
9187
|
+
if (previousVrm !== nextVrm) {
|
|
9188
|
+
this.options.applyVrmSwap(nextVrm);
|
|
9189
|
+
}
|
|
9190
|
+
this.activeVariantKey = variantKey;
|
|
9191
|
+
if (previousVrm !== nextVrm) {
|
|
9192
|
+
this.emit("vrmChange", nextVrm);
|
|
9193
|
+
}
|
|
9194
|
+
if (previousVariant !== variantKey) {
|
|
9195
|
+
this.emit("variantChange", {
|
|
9196
|
+
previousVariant,
|
|
9197
|
+
activeVariant: variantKey,
|
|
9198
|
+
toolCall
|
|
9199
|
+
});
|
|
9200
|
+
}
|
|
9201
|
+
}
|
|
9202
|
+
};
|
|
9203
|
+
|
|
7230
9204
|
// ../../clients/web-client/src/character/expressions.ts
|
|
7231
9205
|
var CharacterExpressions = class {
|
|
7232
9206
|
constructor(mixer) {
|
|
@@ -7466,7 +9440,7 @@ var FacialPerformanceInspector = class {
|
|
|
7466
9440
|
};
|
|
7467
9441
|
|
|
7468
9442
|
// ../../clients/web-client/src/character/idle-face.ts
|
|
7469
|
-
import { Vector3 as
|
|
9443
|
+
import { Vector3 as Vector39 } from "three";
|
|
7470
9444
|
var IdleFace = class {
|
|
7471
9445
|
constructor(vrm, random) {
|
|
7472
9446
|
this.vrm = vrm;
|
|
@@ -7474,7 +9448,7 @@ var IdleFace = class {
|
|
|
7474
9448
|
}
|
|
7475
9449
|
time = 0;
|
|
7476
9450
|
timeline;
|
|
7477
|
-
targetPosition = new
|
|
9451
|
+
targetPosition = new Vector39();
|
|
7478
9452
|
/** Advances by `delta` seconds, turns the eyes, and returns the blink weight. */
|
|
7479
9453
|
update(delta, frame) {
|
|
7480
9454
|
if (Number.isFinite(delta) && delta > 0) {
|
|
@@ -7494,6 +9468,76 @@ var IdleFace = class {
|
|
|
7494
9468
|
}
|
|
7495
9469
|
};
|
|
7496
9470
|
|
|
9471
|
+
// ../../clients/web-client/src/character/listening-face.ts
|
|
9472
|
+
var LISTENING_FACE_LAYER = "listening";
|
|
9473
|
+
var CHANNELS = ["relaxed", "happy"];
|
|
9474
|
+
var REST = 0.01;
|
|
9475
|
+
var ListeningFace = class {
|
|
9476
|
+
constructor(weights) {
|
|
9477
|
+
this.weights = weights;
|
|
9478
|
+
}
|
|
9479
|
+
channel = null;
|
|
9480
|
+
weight = 0;
|
|
9481
|
+
saved = /* @__PURE__ */ new Map();
|
|
9482
|
+
/** The preset in use, or null when the model has none of them. */
|
|
9483
|
+
get channelName() {
|
|
9484
|
+
return this.channel;
|
|
9485
|
+
}
|
|
9486
|
+
/** Call after the layers were registered for a new model. */
|
|
9487
|
+
setChannels(available, manager) {
|
|
9488
|
+
this.restoreOverrides(manager);
|
|
9489
|
+
this.channel = CHANNELS.find((name) => available.has(name)) ?? null;
|
|
9490
|
+
}
|
|
9491
|
+
set(weight, options) {
|
|
9492
|
+
this.weight = Math.max(0, Math.min(1, weight));
|
|
9493
|
+
if (options?.immediate && this.weight === 0) {
|
|
9494
|
+
this.weights.clearLayerTargets(LISTENING_FACE_LAYER, { immediate: true });
|
|
9495
|
+
}
|
|
9496
|
+
}
|
|
9497
|
+
/** Before the weight mixer updates: an active emotion takes the face. */
|
|
9498
|
+
beforeUpdate(emotionActive) {
|
|
9499
|
+
if (!this.channel) {
|
|
9500
|
+
return;
|
|
9501
|
+
}
|
|
9502
|
+
const target = emotionActive ? 0 : this.weight;
|
|
9503
|
+
if (target === 0 && this.weights.getLayerValue(LISTENING_FACE_LAYER, this.channel) < REST) {
|
|
9504
|
+
this.weights.clearLayerTargets(LISTENING_FACE_LAYER, { immediate: true });
|
|
9505
|
+
return;
|
|
9506
|
+
}
|
|
9507
|
+
this.weights.setTarget(LISTENING_FACE_LAYER, this.channel, target);
|
|
9508
|
+
}
|
|
9509
|
+
/** After the values went to the expression manager, before `vrm.update()`. */
|
|
9510
|
+
afterUpdate(manager, otherLayerValue) {
|
|
9511
|
+
const channel = this.channel;
|
|
9512
|
+
const owned = channel !== null && this.weights.getLayerValue(LISTENING_FACE_LAYER, channel) > 0 && otherLayerValue(channel) === 0;
|
|
9513
|
+
if (!owned) {
|
|
9514
|
+
this.restoreOverrides(manager);
|
|
9515
|
+
return;
|
|
9516
|
+
}
|
|
9517
|
+
const expression = manager.getExpression(channel);
|
|
9518
|
+
if (!expression || this.saved.size > 0) {
|
|
9519
|
+
return;
|
|
9520
|
+
}
|
|
9521
|
+
;
|
|
9522
|
+
["overrideBlink", "overrideLookAt"].forEach((key) => {
|
|
9523
|
+
if (expression[key] === "block") {
|
|
9524
|
+
this.saved.set(key, expression[key]);
|
|
9525
|
+
expression[key] = "blend";
|
|
9526
|
+
}
|
|
9527
|
+
});
|
|
9528
|
+
}
|
|
9529
|
+
/** Gives the preset its own overrides back. */
|
|
9530
|
+
restoreOverrides(manager) {
|
|
9531
|
+
const expression = this.channel !== null ? manager?.getExpression(this.channel) : null;
|
|
9532
|
+
if (expression) {
|
|
9533
|
+
this.saved.forEach((value, key) => {
|
|
9534
|
+
expression[key] = value;
|
|
9535
|
+
});
|
|
9536
|
+
}
|
|
9537
|
+
this.saved.clear();
|
|
9538
|
+
}
|
|
9539
|
+
};
|
|
9540
|
+
|
|
7497
9541
|
// ../../clients/web-client/src/character/performance-weight-mixer.ts
|
|
7498
9542
|
var PRECISION = 1e3;
|
|
7499
9543
|
var DEFAULT_RESPONSE_RATE = 12;
|
|
@@ -7572,6 +9616,18 @@ var PerformanceWeightMixer = class {
|
|
|
7572
9616
|
}
|
|
7573
9617
|
layer.channels.set(channel, { value: 0, target });
|
|
7574
9618
|
}
|
|
9619
|
+
restoreChannelState(layerId, channel, value, target) {
|
|
9620
|
+
const layer = this.getLayer(layerId);
|
|
9621
|
+
const state = layer.channels.get(channel);
|
|
9622
|
+
const nextValue = clamp012(value);
|
|
9623
|
+
const nextTarget = clamp012(target);
|
|
9624
|
+
if (state) {
|
|
9625
|
+
state.value = nextValue;
|
|
9626
|
+
state.target = nextTarget;
|
|
9627
|
+
return;
|
|
9628
|
+
}
|
|
9629
|
+
layer.channels.set(channel, { value: nextValue, target: nextTarget });
|
|
9630
|
+
}
|
|
7575
9631
|
setExclusiveTargets(layerId, channels, selected, weight) {
|
|
7576
9632
|
channels.forEach((channel) => {
|
|
7577
9633
|
this.setTarget(layerId, channel, channel === selected ? weight : 0);
|
|
@@ -7730,17 +9786,34 @@ var FacialPerformanceMixer = class {
|
|
|
7730
9786
|
restoreMouthAfterVisemeRelease = false;
|
|
7731
9787
|
idleFace = null;
|
|
7732
9788
|
gazeTarget = null;
|
|
7733
|
-
|
|
7734
|
-
|
|
9789
|
+
listeningFace = new ListeningFace(this.weightMixer);
|
|
9790
|
+
setVrm(vrm, options) {
|
|
9791
|
+
const preserveState = options?.preserveState === true && vrm !== null;
|
|
9792
|
+
const savedMouthMode = preserveState ? this.activeMouthMode : "expression";
|
|
9793
|
+
const savedRestoreAfterRelease = preserveState ? this.restoreMouthAfterVisemeRelease : false;
|
|
9794
|
+
const savedSnapshot = preserveState ? this.weightMixer.getSnapshot() : null;
|
|
9795
|
+
this.vrm?.expressionManager?.resetValues?.();
|
|
9796
|
+
this.listeningFace.restoreOverrides(this.vrm?.expressionManager);
|
|
9797
|
+
if (preserveState) {
|
|
9798
|
+
this.restoreMouthOverrides();
|
|
9799
|
+
} else {
|
|
9800
|
+
this.restoreExpressionMouthOwnership();
|
|
9801
|
+
this.clearCueTimeouts();
|
|
9802
|
+
}
|
|
7735
9803
|
this.vrm = vrm;
|
|
7736
9804
|
this.idleFace = vrm && detectTemplateFace(vrm) ? new IdleFace(vrm, this.options.idleRandom) : null;
|
|
7737
9805
|
this.inspector.setVrm(vrm);
|
|
7738
|
-
this.clearCueTimeouts();
|
|
7739
9806
|
this.availableChannels = [];
|
|
7740
9807
|
this.availableChannelSet.clear();
|
|
7741
9808
|
this.channelsByLayer.clear();
|
|
7742
9809
|
this.configureLayers();
|
|
7743
9810
|
if (!vrm?.expressionManager) {
|
|
9811
|
+
this.listeningFace.setChannels(this.availableChannelSet, null);
|
|
9812
|
+
if (preserveState) {
|
|
9813
|
+
this.activeMouthMode = "expression";
|
|
9814
|
+
this.restoreMouthAfterVisemeRelease = false;
|
|
9815
|
+
this.clearCueTimeouts();
|
|
9816
|
+
}
|
|
7744
9817
|
return;
|
|
7745
9818
|
}
|
|
7746
9819
|
const manager = vrm.expressionManager;
|
|
@@ -7756,7 +9829,43 @@ var FacialPerformanceMixer = class {
|
|
|
7756
9829
|
this.weightMixer.setTarget(layerId, name, 0);
|
|
7757
9830
|
});
|
|
7758
9831
|
this.availableChannels = names;
|
|
9832
|
+
this.listeningFace.setChannels(this.availableChannelSet, manager);
|
|
7759
9833
|
manager.resetValues();
|
|
9834
|
+
if (!preserveState || !savedSnapshot) {
|
|
9835
|
+
return;
|
|
9836
|
+
}
|
|
9837
|
+
this.cueTimeouts.forEach((timeoutId, channel) => {
|
|
9838
|
+
if (!this.availableChannelSet.has(channel)) {
|
|
9839
|
+
clearTimeout(timeoutId);
|
|
9840
|
+
this.cueTimeouts.delete(channel);
|
|
9841
|
+
}
|
|
9842
|
+
});
|
|
9843
|
+
Object.entries(savedSnapshot).forEach(([layerId, layerSnapshot]) => {
|
|
9844
|
+
Object.entries(layerSnapshot.channels).forEach(
|
|
9845
|
+
([channel, channelState]) => {
|
|
9846
|
+
if (this.availableChannelSet.has(channel)) {
|
|
9847
|
+
this.weightMixer.restoreChannelState(
|
|
9848
|
+
layerId,
|
|
9849
|
+
channel,
|
|
9850
|
+
channelState.value,
|
|
9851
|
+
channelState.target
|
|
9852
|
+
);
|
|
9853
|
+
}
|
|
9854
|
+
}
|
|
9855
|
+
);
|
|
9856
|
+
});
|
|
9857
|
+
this.activeMouthMode = savedMouthMode;
|
|
9858
|
+
this.restoreMouthAfterVisemeRelease = savedRestoreAfterRelease;
|
|
9859
|
+
if (savedMouthMode === "viseme") {
|
|
9860
|
+
this.getLayerChannels(FacialPerformanceLayer.Emotion).forEach((name) => {
|
|
9861
|
+
const expression = manager.getExpression(name);
|
|
9862
|
+
if (expression) {
|
|
9863
|
+
this.originalMouthOverrides.set(name, expression.overrideMouth);
|
|
9864
|
+
expression.overrideMouth = "none";
|
|
9865
|
+
}
|
|
9866
|
+
});
|
|
9867
|
+
}
|
|
9868
|
+
this.update(0);
|
|
7760
9869
|
}
|
|
7761
9870
|
getChannelNames() {
|
|
7762
9871
|
return this.availableChannels.slice();
|
|
@@ -7858,6 +9967,19 @@ var FacialPerformanceMixer = class {
|
|
|
7858
9967
|
clearOverlay() {
|
|
7859
9968
|
this.weightMixer.clearLayerTargets(FacialPerformanceLayer.Overlay);
|
|
7860
9969
|
}
|
|
9970
|
+
/**
|
|
9971
|
+
* The attentive face of the listening reflexes (`relaxed`, else `happy`)
|
|
9972
|
+
* at `weight`. It eases in and out on its own layer, gives way to any
|
|
9973
|
+
* emotion and leaves the mouth layers alone. `immediate` with weight 0
|
|
9974
|
+
* removes it in this frame.
|
|
9975
|
+
*/
|
|
9976
|
+
setListeningFace(weight, options) {
|
|
9977
|
+
this.listeningFace.set(weight, options);
|
|
9978
|
+
}
|
|
9979
|
+
/** The preset of the attentive face, or null when the model has none. */
|
|
9980
|
+
get listeningFaceChannel() {
|
|
9981
|
+
return this.listeningFace.channelName;
|
|
9982
|
+
}
|
|
7861
9983
|
releaseVisemeMouthMode() {
|
|
7862
9984
|
if (this.activeMouthMode !== "viseme") {
|
|
7863
9985
|
return;
|
|
@@ -7937,16 +10059,24 @@ var FacialPerformanceMixer = class {
|
|
|
7937
10059
|
}
|
|
7938
10060
|
reset() {
|
|
7939
10061
|
this.weightMixer.resetTargets();
|
|
10062
|
+
this.listeningFace.set(0);
|
|
7940
10063
|
}
|
|
7941
10064
|
update(delta) {
|
|
7942
10065
|
if (!this.vrm?.expressionManager) {
|
|
7943
10066
|
return;
|
|
7944
10067
|
}
|
|
7945
10068
|
const manager = this.vrm.expressionManager;
|
|
10069
|
+
this.listeningFace.beforeUpdate(
|
|
10070
|
+
this.weightMixer.isLayerActive(FacialPerformanceLayer.Emotion)
|
|
10071
|
+
);
|
|
7946
10072
|
const values = this.weightMixer.update(delta);
|
|
7947
10073
|
values.forEach((value, name) => {
|
|
7948
10074
|
manager.setValue(name, value);
|
|
7949
10075
|
});
|
|
10076
|
+
this.listeningFace.afterUpdate(
|
|
10077
|
+
manager,
|
|
10078
|
+
(channel) => this.weightMixer.getLayerValue(FacialPerformanceLayer.Emotion, channel) + this.weightMixer.getLayerValue(FacialPerformanceLayer.Overlay, channel)
|
|
10079
|
+
);
|
|
7950
10080
|
this.updateIdleFace(delta, values);
|
|
7951
10081
|
if (this.restoreMouthAfterVisemeRelease && this.areVisemesReleased()) {
|
|
7952
10082
|
this.restoreExpressionMouthOwnership();
|
|
@@ -7989,6 +10119,12 @@ var FacialPerformanceMixer = class {
|
|
|
7989
10119
|
priority: 400,
|
|
7990
10120
|
blendMode: "override"
|
|
7991
10121
|
});
|
|
10122
|
+
this.weightMixer.registerLayer({
|
|
10123
|
+
id: LISTENING_FACE_LAYER,
|
|
10124
|
+
priority: 450,
|
|
10125
|
+
blendMode: "maximum",
|
|
10126
|
+
responseRate: 8
|
|
10127
|
+
});
|
|
7992
10128
|
this.weightMixer.registerLayer({
|
|
7993
10129
|
id: FacialPerformanceLayer.Overlay,
|
|
7994
10130
|
priority: 500,
|
|
@@ -8121,17 +10257,17 @@ import {
|
|
|
8121
10257
|
Euler as Euler2,
|
|
8122
10258
|
Mesh,
|
|
8123
10259
|
Object3D,
|
|
8124
|
-
SkinnedMesh,
|
|
8125
|
-
Vector3 as
|
|
10260
|
+
SkinnedMesh as SkinnedMesh2,
|
|
10261
|
+
Vector3 as Vector310
|
|
8126
10262
|
} from "three";
|
|
8127
10263
|
var isSkinnedMesh = function(object) {
|
|
8128
|
-
return object instanceof
|
|
10264
|
+
return object instanceof SkinnedMesh2 || object.isSkinnedMesh === true;
|
|
8129
10265
|
};
|
|
8130
10266
|
var isMesh = function(object) {
|
|
8131
10267
|
return isSkinnedMesh(object) || object instanceof Mesh || object.isMesh === true;
|
|
8132
10268
|
};
|
|
8133
10269
|
var applyVector3 = function(target, value) {
|
|
8134
|
-
if (value instanceof
|
|
10270
|
+
if (value instanceof Vector310) {
|
|
8135
10271
|
target.copy(value);
|
|
8136
10272
|
return;
|
|
8137
10273
|
}
|
|
@@ -8169,8 +10305,28 @@ var CharacterSceneRig = class {
|
|
|
8169
10305
|
get vrm() {
|
|
8170
10306
|
return this.currentVrm;
|
|
8171
10307
|
}
|
|
8172
|
-
setVrm(vrm) {
|
|
10308
|
+
setVrm(vrm, fallbackParent) {
|
|
10309
|
+
const previousVrm = this.currentVrm;
|
|
10310
|
+
const previousScene = previousVrm?.scene ?? null;
|
|
10311
|
+
const previousParent = previousScene?.parent ?? null;
|
|
8173
10312
|
this.setVrmLookAtTarget(null);
|
|
10313
|
+
if (previousScene && previousParent && previousVrm !== vrm) {
|
|
10314
|
+
previousParent.remove(previousScene);
|
|
10315
|
+
}
|
|
10316
|
+
if (previousScene && vrm?.scene && previousScene !== vrm.scene) {
|
|
10317
|
+
if (previousScene.position && vrm.scene.position?.copy) {
|
|
10318
|
+
vrm.scene.position.copy(previousScene.position);
|
|
10319
|
+
}
|
|
10320
|
+
if (previousScene.rotation && vrm.scene.rotation?.copy) {
|
|
10321
|
+
vrm.scene.rotation.copy(previousScene.rotation);
|
|
10322
|
+
}
|
|
10323
|
+
if (previousScene.scale && vrm.scene.scale?.copy) {
|
|
10324
|
+
vrm.scene.scale.copy(previousScene.scale);
|
|
10325
|
+
}
|
|
10326
|
+
if (typeof previousScene.visible === "boolean") {
|
|
10327
|
+
vrm.scene.visible = previousScene.visible;
|
|
10328
|
+
}
|
|
10329
|
+
}
|
|
8174
10330
|
this.currentVrm = vrm;
|
|
8175
10331
|
this.refreshMeshes();
|
|
8176
10332
|
if (this.shadowOptions) {
|
|
@@ -8179,6 +10335,10 @@ var CharacterSceneRig = class {
|
|
|
8179
10335
|
if (this.transformOptions) {
|
|
8180
10336
|
this.applyTransform(this.transformOptions);
|
|
8181
10337
|
}
|
|
10338
|
+
const targetParent = previousParent ?? fallbackParent ?? null;
|
|
10339
|
+
if (vrm?.scene && targetParent && vrm.scene.parent !== targetParent) {
|
|
10340
|
+
targetParent.add(vrm.scene);
|
|
10341
|
+
}
|
|
8182
10342
|
}
|
|
8183
10343
|
/**
|
|
8184
10344
|
* The `Object3D` of a `lookAt()` argument: an `Object3D` as it is, a point
|
|
@@ -8220,6 +10380,7 @@ var CharacterSceneRig = class {
|
|
|
8220
10380
|
lookAt.target = target;
|
|
8221
10381
|
} else if (this.vrmLookAtTarget && lookAt.target === this.vrmLookAtTarget) {
|
|
8222
10382
|
lookAt.target = void 0;
|
|
10383
|
+
lookAt.reset();
|
|
8223
10384
|
}
|
|
8224
10385
|
}
|
|
8225
10386
|
this.vrmLookAtTarget = lookAt ? target : null;
|
|
@@ -8353,43 +10514,325 @@ var CharacterSceneRig = class {
|
|
|
8353
10514
|
this.cachedMaterials = Array.from(materials);
|
|
8354
10515
|
this.cachedSkeletons = Array.from(skeletons);
|
|
8355
10516
|
}
|
|
8356
|
-
disposeVrm() {
|
|
8357
|
-
const vrm = this.currentVrm;
|
|
8358
|
-
this.detach();
|
|
8359
|
-
this.setVrm(null);
|
|
8360
|
-
if (vrm?.scene && typeof vrm.scene.traverse === "function") {
|
|
8361
|
-
disposeVrm(vrm);
|
|
10517
|
+
disposeVrm() {
|
|
10518
|
+
const vrm = this.currentVrm;
|
|
10519
|
+
this.detach();
|
|
10520
|
+
this.setVrm(null);
|
|
10521
|
+
if (vrm?.scene && typeof vrm.scene.traverse === "function") {
|
|
10522
|
+
disposeVrm(vrm);
|
|
10523
|
+
}
|
|
10524
|
+
}
|
|
10525
|
+
disposeAllVrms(vrms) {
|
|
10526
|
+
const unique = new Set(vrms);
|
|
10527
|
+
if (this.currentVrm) {
|
|
10528
|
+
unique.add(this.currentVrm);
|
|
10529
|
+
}
|
|
10530
|
+
this.detach();
|
|
10531
|
+
this.setVrm(null);
|
|
10532
|
+
for (const vrm of unique) {
|
|
10533
|
+
if (vrm.scene?.parent) {
|
|
10534
|
+
vrm.scene.parent.remove(vrm.scene);
|
|
10535
|
+
}
|
|
10536
|
+
if (vrm.scene && typeof vrm.scene.traverse === "function") {
|
|
10537
|
+
disposeVrm(vrm);
|
|
10538
|
+
}
|
|
10539
|
+
}
|
|
10540
|
+
}
|
|
10541
|
+
applyShadows(options) {
|
|
10542
|
+
this.cachedMeshes.forEach((mesh) => {
|
|
10543
|
+
mesh.castShadow = options.cast;
|
|
10544
|
+
mesh.receiveShadow = options.receive;
|
|
10545
|
+
});
|
|
10546
|
+
}
|
|
10547
|
+
applyTransform(options) {
|
|
10548
|
+
const scene = this.scene;
|
|
10549
|
+
if (!scene) {
|
|
10550
|
+
return;
|
|
10551
|
+
}
|
|
10552
|
+
if (options.position !== void 0) {
|
|
10553
|
+
applyVector3(scene.position, options.position);
|
|
10554
|
+
}
|
|
10555
|
+
if (options.rotation !== void 0) {
|
|
10556
|
+
applyEuler(scene.rotation, options.rotation);
|
|
10557
|
+
}
|
|
10558
|
+
if (options.scale !== void 0) {
|
|
10559
|
+
if (typeof options.scale === "number") {
|
|
10560
|
+
scene.scale.setScalar(options.scale);
|
|
10561
|
+
} else {
|
|
10562
|
+
applyVector3(scene.scale, options.scale);
|
|
10563
|
+
}
|
|
10564
|
+
}
|
|
10565
|
+
}
|
|
10566
|
+
};
|
|
10567
|
+
|
|
10568
|
+
// ../../clients/web-client/src/character/character-scene-access.ts
|
|
10569
|
+
var CharacterSceneAccess = class {
|
|
10570
|
+
sceneRig = new CharacterSceneRig();
|
|
10571
|
+
defaultParent = null;
|
|
10572
|
+
/** True once the character VRM model is loaded. */
|
|
10573
|
+
get ready() {
|
|
10574
|
+
return this.sceneRig.vrm !== null;
|
|
10575
|
+
}
|
|
10576
|
+
/** Root Three.js `Group` (`vrm.scene`) to add to a `THREE.Scene`. */
|
|
10577
|
+
get scene() {
|
|
10578
|
+
return this.sceneRig.scene;
|
|
10579
|
+
}
|
|
10580
|
+
/** Alias for `scene` for Three.js `Object3D` workflows. */
|
|
10581
|
+
get object3d() {
|
|
10582
|
+
return this.sceneRig.object3d;
|
|
10583
|
+
}
|
|
10584
|
+
/** Alias for `scene`. */
|
|
10585
|
+
get root() {
|
|
10586
|
+
return this.sceneRig.root;
|
|
10587
|
+
}
|
|
10588
|
+
/** Primary character mesh (`SkinnedMesh` preferred, otherwise first `Mesh`). */
|
|
10589
|
+
get mesh() {
|
|
10590
|
+
return this.sceneRig.mesh;
|
|
10591
|
+
}
|
|
10592
|
+
/** All Three.js `Mesh` instances in the character hierarchy. */
|
|
10593
|
+
get meshes() {
|
|
10594
|
+
return this.sceneRig.meshes;
|
|
10595
|
+
}
|
|
10596
|
+
/** All Three.js `SkinnedMesh` instances in the character hierarchy. */
|
|
10597
|
+
get skinnedMeshes() {
|
|
10598
|
+
return this.sceneRig.skinnedMeshes;
|
|
10599
|
+
}
|
|
10600
|
+
/** Deduplicated Three.js `Material` instances across all character meshes. */
|
|
10601
|
+
get materials() {
|
|
10602
|
+
return this.sceneRig.materials;
|
|
10603
|
+
}
|
|
10604
|
+
/** Deduplicated Three.js `Skeleton` instances across all character skinned meshes. */
|
|
10605
|
+
get skeletons() {
|
|
10606
|
+
return this.sceneRig.skeletons;
|
|
10607
|
+
}
|
|
10608
|
+
/** Three-VRM humanoid rig when loaded. */
|
|
10609
|
+
get humanoid() {
|
|
10610
|
+
return this.sceneRig.humanoid;
|
|
10611
|
+
}
|
|
10612
|
+
/** Look up a raw humanoid bone `Object3D` by VRM bone name (for example `'head'` or `'rightHand'`). */
|
|
10613
|
+
getBone(name) {
|
|
10614
|
+
return this.sceneRig.getBone(name);
|
|
10615
|
+
}
|
|
10616
|
+
/** Look up a normalized humanoid bone `Object3D` by VRM bone name. */
|
|
10617
|
+
getNormalizedBone(name) {
|
|
10618
|
+
return this.sceneRig.getNormalizedBone(name);
|
|
10619
|
+
}
|
|
10620
|
+
/** Attach the character root `Group` to a Three.js `Scene` or parent `Object3D`. */
|
|
10621
|
+
attachTo(parent) {
|
|
10622
|
+
this.defaultParent = parent;
|
|
10623
|
+
return this.sceneRig.attachTo(parent);
|
|
10624
|
+
}
|
|
10625
|
+
/** Detach the character root `Group` from its current parent `Object3D`. */
|
|
10626
|
+
detach() {
|
|
10627
|
+
this.defaultParent = null;
|
|
10628
|
+
this.sceneRig.detach();
|
|
10629
|
+
}
|
|
10630
|
+
/** Configure `castShadow` and `receiveShadow` on all character meshes. */
|
|
10631
|
+
setShadows(options) {
|
|
10632
|
+
this.sceneRig.setShadows(options);
|
|
10633
|
+
}
|
|
10634
|
+
/** Configure `position`, `rotation`, and `scale` on the character root `Group`. */
|
|
10635
|
+
setTransform(options) {
|
|
10636
|
+
this.sceneRig.setTransform(options);
|
|
10637
|
+
}
|
|
10638
|
+
/** Re-index cached meshes, materials, and skeletons after custom scene graph edits. */
|
|
10639
|
+
refreshMeshes() {
|
|
10640
|
+
this.sceneRig.refreshMeshes();
|
|
10641
|
+
}
|
|
10642
|
+
};
|
|
10643
|
+
|
|
10644
|
+
// ../../clients/web-client/src/character/listening-reflex.ts
|
|
10645
|
+
import { Object3D as Object3D2, Vector3 as Vector312 } from "three";
|
|
10646
|
+
|
|
10647
|
+
// ../../clients/web-client/src/character/head-nod.ts
|
|
10648
|
+
import { Quaternion as Quaternion7, Vector3 as Vector311 } from "three";
|
|
10649
|
+
var SHARES = [
|
|
10650
|
+
["neck", 0.4],
|
|
10651
|
+
["head", 0.6]
|
|
10652
|
+
];
|
|
10653
|
+
var DAMPING = 0.6;
|
|
10654
|
+
var FREQUENCY_TIMES_LENGTH = 1.4;
|
|
10655
|
+
var PULSE_PART = 0.3;
|
|
10656
|
+
var MAX_STEP_SECONDS = 1 / 240;
|
|
10657
|
+
var REST_DEGREES = 0.01;
|
|
10658
|
+
var X_AXIS = new Vector311(1, 0, 0);
|
|
10659
|
+
var HeadNod = class {
|
|
10660
|
+
pitch = 0;
|
|
10661
|
+
speed = 0;
|
|
10662
|
+
elapsed = 0;
|
|
10663
|
+
pulseSeconds = 0;
|
|
10664
|
+
amplitude = 0;
|
|
10665
|
+
omega = 0;
|
|
10666
|
+
applied = [];
|
|
10667
|
+
scratch = new Quaternion7();
|
|
10668
|
+
/** The current pitch in degrees; positive turns the face down. */
|
|
10669
|
+
get pitchDegrees() {
|
|
10670
|
+
return this.pitch;
|
|
10671
|
+
}
|
|
10672
|
+
get active() {
|
|
10673
|
+
return this.elapsed < this.pulseSeconds || Math.abs(this.pitch) > REST_DEGREES || Math.abs(this.speed) > REST_DEGREES;
|
|
10674
|
+
}
|
|
10675
|
+
start(nod) {
|
|
10676
|
+
const seconds = Math.max(nod.durationMs, 1) / 1e3;
|
|
10677
|
+
this.amplitude = nod.amplitudeDegrees;
|
|
10678
|
+
this.omega = 2 * Math.PI * FREQUENCY_TIMES_LENGTH / seconds;
|
|
10679
|
+
this.pulseSeconds = seconds * PULSE_PART;
|
|
10680
|
+
this.elapsed = 0;
|
|
10681
|
+
}
|
|
10682
|
+
/** Stops the nod at once; the next `apply` removes its pitch. */
|
|
10683
|
+
cancel() {
|
|
10684
|
+
this.pitch = 0;
|
|
10685
|
+
this.speed = 0;
|
|
10686
|
+
this.elapsed = 0;
|
|
10687
|
+
this.pulseSeconds = 0;
|
|
10688
|
+
}
|
|
10689
|
+
update(delta) {
|
|
10690
|
+
if (!(delta > 0) || !this.active) {
|
|
10691
|
+
if (!this.active) {
|
|
10692
|
+
this.pitch = 0;
|
|
10693
|
+
this.speed = 0;
|
|
10694
|
+
}
|
|
10695
|
+
return this.pitch;
|
|
10696
|
+
}
|
|
10697
|
+
let left = Math.min(delta, 0.25);
|
|
10698
|
+
while (left > 0) {
|
|
10699
|
+
const step = Math.min(left, MAX_STEP_SECONDS);
|
|
10700
|
+
const target = this.elapsed < this.pulseSeconds ? this.amplitude : 0;
|
|
10701
|
+
const omega = this.omega;
|
|
10702
|
+
const acceleration = omega * omega * (target - this.pitch) - 2 * DAMPING * omega * this.speed;
|
|
10703
|
+
this.speed += acceleration * step;
|
|
10704
|
+
this.pitch += this.speed * step;
|
|
10705
|
+
this.elapsed += step;
|
|
10706
|
+
left -= step;
|
|
10707
|
+
}
|
|
10708
|
+
if (!this.active) {
|
|
10709
|
+
this.pitch = 0;
|
|
10710
|
+
this.speed = 0;
|
|
8362
10711
|
}
|
|
10712
|
+
return this.pitch;
|
|
8363
10713
|
}
|
|
8364
|
-
|
|
8365
|
-
|
|
8366
|
-
|
|
8367
|
-
|
|
10714
|
+
/** Writes the current pitch onto the normalized neck and head of `vrm`. */
|
|
10715
|
+
apply(vrm) {
|
|
10716
|
+
this.applied.forEach(({ node, base, result }) => {
|
|
10717
|
+
if (node.quaternion.equals(result)) {
|
|
10718
|
+
node.quaternion.copy(base);
|
|
10719
|
+
}
|
|
8368
10720
|
});
|
|
8369
|
-
|
|
8370
|
-
|
|
8371
|
-
|
|
8372
|
-
if (!scene) {
|
|
10721
|
+
this.applied = [];
|
|
10722
|
+
const humanoid = vrm?.humanoid;
|
|
10723
|
+
if (!humanoid || this.pitch === 0) {
|
|
8373
10724
|
return;
|
|
8374
10725
|
}
|
|
8375
|
-
|
|
8376
|
-
|
|
10726
|
+
const flip = vrm.meta.metaVersion === "0" && !isNpcifyNormalizedVrm(vrm) ? -1 : 1;
|
|
10727
|
+
const radians = this.pitch * Math.PI / 180 * flip;
|
|
10728
|
+
let carried = 0;
|
|
10729
|
+
SHARES.forEach(([bone, share]) => {
|
|
10730
|
+
const node = humanoid.getNormalizedBoneNode(bone);
|
|
10731
|
+
if (!node) {
|
|
10732
|
+
carried += share;
|
|
10733
|
+
return;
|
|
10734
|
+
}
|
|
10735
|
+
const angle = radians * (share + carried);
|
|
10736
|
+
carried = 0;
|
|
10737
|
+
const base = node.quaternion.clone();
|
|
10738
|
+
node.quaternion.multiply(this.scratch.setFromAxisAngle(X_AXIS, angle));
|
|
10739
|
+
this.applied.push({ node, base, result: node.quaternion.clone() });
|
|
10740
|
+
});
|
|
10741
|
+
}
|
|
10742
|
+
};
|
|
10743
|
+
|
|
10744
|
+
// ../../clients/web-client/src/character/listening-reflex.ts
|
|
10745
|
+
var LISTENING_FACE_WEIGHT = 0.2;
|
|
10746
|
+
var FRONT_DISTANCE = 1.5;
|
|
10747
|
+
var ListeningReflex = class {
|
|
10748
|
+
constructor(host, activity) {
|
|
10749
|
+
this.host = host;
|
|
10750
|
+
this.activity = activity;
|
|
10751
|
+
activity.onNod((nod) => {
|
|
10752
|
+
this.pendingNod = nod;
|
|
10753
|
+
});
|
|
10754
|
+
}
|
|
10755
|
+
nod = new HeadNod();
|
|
10756
|
+
pendingNod = null;
|
|
10757
|
+
releaseGaze = null;
|
|
10758
|
+
front = new Object3D2();
|
|
10759
|
+
origin = new Vector312();
|
|
10760
|
+
forward = new Vector312();
|
|
10761
|
+
yielded = false;
|
|
10762
|
+
/** Call after the animation mixer and live motion, before the facial mixer and `vrm.update()`. */
|
|
10763
|
+
update(delta, frame) {
|
|
10764
|
+
const vrm = this.host.vrm();
|
|
10765
|
+
if (frame.stagingActive) {
|
|
10766
|
+
if (!this.yielded) {
|
|
10767
|
+
this.yielded = true;
|
|
10768
|
+
this.host.facialMixer.setListeningFace(0, { immediate: true });
|
|
10769
|
+
this.nod.cancel();
|
|
10770
|
+
this.endGaze();
|
|
10771
|
+
}
|
|
10772
|
+
this.pendingNod = null;
|
|
10773
|
+
this.nod.apply(vrm);
|
|
10774
|
+
return;
|
|
10775
|
+
}
|
|
10776
|
+
this.yielded = false;
|
|
10777
|
+
const engaged = this.activity.state !== "idle" && vrm !== null;
|
|
10778
|
+
this.host.facialMixer.setListeningFace(engaged ? LISTENING_FACE_WEIGHT : 0);
|
|
10779
|
+
if (engaged) {
|
|
10780
|
+
this.beginGaze(vrm);
|
|
10781
|
+
} else {
|
|
10782
|
+
this.endGaze();
|
|
8377
10783
|
}
|
|
8378
|
-
|
|
8379
|
-
|
|
10784
|
+
const nod = this.pendingNod;
|
|
10785
|
+
this.pendingNod = null;
|
|
10786
|
+
if (frame.liveMotionWeight > 0) {
|
|
10787
|
+
this.nod.cancel();
|
|
10788
|
+
} else if (nod && this.activity.state === "speaking" && vrm) {
|
|
10789
|
+
this.nod.start(nod);
|
|
8380
10790
|
}
|
|
8381
|
-
|
|
8382
|
-
|
|
8383
|
-
|
|
8384
|
-
|
|
8385
|
-
|
|
10791
|
+
this.nod.update(delta);
|
|
10792
|
+
this.nod.apply(vrm);
|
|
10793
|
+
}
|
|
10794
|
+
/** A new model or the end of the conversation: no gaze, face or nod remains. */
|
|
10795
|
+
reset() {
|
|
10796
|
+
this.pendingNod = null;
|
|
10797
|
+
this.nod.cancel();
|
|
10798
|
+
this.nod.apply(null);
|
|
10799
|
+
this.endGaze();
|
|
10800
|
+
this.host.facialMixer.setListeningFace(0, { immediate: true });
|
|
10801
|
+
}
|
|
10802
|
+
beginGaze(vrm) {
|
|
10803
|
+
if (!this.releaseGaze) {
|
|
10804
|
+
if (this.host.lookAtActive()) {
|
|
10805
|
+
return;
|
|
8386
10806
|
}
|
|
10807
|
+
const target = this.host.gazeTarget() ?? this.front;
|
|
10808
|
+
this.releaseGaze = this.host.lookAtTemporarily(target);
|
|
10809
|
+
}
|
|
10810
|
+
this.placeFront(vrm);
|
|
10811
|
+
}
|
|
10812
|
+
endGaze() {
|
|
10813
|
+
const release = this.releaseGaze;
|
|
10814
|
+
this.releaseGaze = null;
|
|
10815
|
+
release?.();
|
|
10816
|
+
}
|
|
10817
|
+
/** The point in front of the head, along the character's forward axis. */
|
|
10818
|
+
placeFront(vrm) {
|
|
10819
|
+
const head = vrm.humanoid?.getNormalizedBoneNode("head");
|
|
10820
|
+
if (!head) {
|
|
10821
|
+
return;
|
|
8387
10822
|
}
|
|
10823
|
+
const flip = vrm.meta.metaVersion === "0" && !isNpcifyNormalizedVrm(vrm);
|
|
10824
|
+
const root = vrm.scene;
|
|
10825
|
+
const origin = root.getWorldPosition(this.origin);
|
|
10826
|
+
const forward = root.localToWorld(
|
|
10827
|
+
this.forward.set(0, 0, flip ? -FRONT_DISTANCE : FRONT_DISTANCE)
|
|
10828
|
+
).sub(origin);
|
|
10829
|
+
head.getWorldPosition(this.front.position).add(forward);
|
|
10830
|
+
this.front.updateMatrixWorld(true);
|
|
8388
10831
|
}
|
|
8389
10832
|
};
|
|
8390
10833
|
|
|
8391
10834
|
// ../../clients/web-client/src/character/live-motion.ts
|
|
8392
|
-
import { Quaternion as
|
|
10835
|
+
import { Quaternion as Quaternion8, Vector3 as Vector313 } from "three";
|
|
8393
10836
|
var PARENTS = {
|
|
8394
10837
|
spine: "hips",
|
|
8395
10838
|
chest: "spine",
|
|
@@ -8431,10 +10874,10 @@ var LiveMotionPlayer = class {
|
|
|
8431
10874
|
/** The last applied pose, to blend out of it after the frames end. */
|
|
8432
10875
|
lastPose = null;
|
|
8433
10876
|
scratch = {
|
|
8434
|
-
from: new
|
|
8435
|
-
to: new
|
|
10877
|
+
from: new Quaternion8(),
|
|
10878
|
+
to: new Quaternion8(),
|
|
8436
10879
|
carry: /* @__PURE__ */ new Map(),
|
|
8437
|
-
position: new
|
|
10880
|
+
position: new Vector313()
|
|
8438
10881
|
};
|
|
8439
10882
|
constructor(options = {}) {
|
|
8440
10883
|
this.rootMode = options.rootMode ?? "anchored";
|
|
@@ -8596,7 +11039,7 @@ var LiveMotionPlayer = class {
|
|
|
8596
11039
|
carry.clear();
|
|
8597
11040
|
track.bones.forEach((bone, slot) => {
|
|
8598
11041
|
const offset = 3 + slot * 4;
|
|
8599
|
-
const rotation = new
|
|
11042
|
+
const rotation = new Quaternion8(
|
|
8600
11043
|
values[offset] ?? 0,
|
|
8601
11044
|
values[offset + 1] ?? 0,
|
|
8602
11045
|
values[offset + 2] ?? 0,
|
|
@@ -8643,6 +11086,166 @@ var LiveMotionPlayer = class {
|
|
|
8643
11086
|
}
|
|
8644
11087
|
};
|
|
8645
11088
|
|
|
11089
|
+
// ../../clients/web-client/src/voice/speech-nod-trigger.ts
|
|
11090
|
+
var MIN_SPEECH_LEVEL = 0.015;
|
|
11091
|
+
var SPEECH_TO_NOISE = 3;
|
|
11092
|
+
var NOD_PAUSE_MS = 250;
|
|
11093
|
+
var NOD_MIN_SPEECH_MS = 1200;
|
|
11094
|
+
var NOD_MIN_INTERVAL_MS = 1800;
|
|
11095
|
+
var NOD_INTERVAL_JITTER_MS = 1400;
|
|
11096
|
+
var NOD_SKIP_CHANCE = 0.2;
|
|
11097
|
+
var NOD_MIN_DEGREES = 4;
|
|
11098
|
+
var NOD_MAX_DEGREES = 6;
|
|
11099
|
+
var NOD_MIN_DURATION_MS = 320;
|
|
11100
|
+
var NOD_MAX_DURATION_MS = 400;
|
|
11101
|
+
var SpeechNodTrigger = class {
|
|
11102
|
+
constructor(random = Math.random) {
|
|
11103
|
+
this.random = random;
|
|
11104
|
+
this.intervalMs = this.drawInterval();
|
|
11105
|
+
}
|
|
11106
|
+
floor = null;
|
|
11107
|
+
speechMs = 0;
|
|
11108
|
+
quietMs = 0;
|
|
11109
|
+
sinceNodMs = Number.POSITIVE_INFINITY;
|
|
11110
|
+
intervalMs;
|
|
11111
|
+
/** True while the last frame was loud enough to be speech. */
|
|
11112
|
+
get voiced() {
|
|
11113
|
+
return this.speechMs > 0 && this.quietMs === 0;
|
|
11114
|
+
}
|
|
11115
|
+
/** Reads one frame; returns a nod when one is due now. */
|
|
11116
|
+
push(level, frameMs = SPEECH_FRAME_MS) {
|
|
11117
|
+
if (!Number.isFinite(level) || !(frameMs > 0)) {
|
|
11118
|
+
return null;
|
|
11119
|
+
}
|
|
11120
|
+
this.sinceNodMs += frameMs;
|
|
11121
|
+
const threshold = Math.max(
|
|
11122
|
+
MIN_SPEECH_LEVEL,
|
|
11123
|
+
(this.floor ?? MIN_SPEECH_LEVEL / SPEECH_TO_NOISE) * SPEECH_TO_NOISE
|
|
11124
|
+
);
|
|
11125
|
+
if (level >= threshold) {
|
|
11126
|
+
this.speechMs += this.quietMs + frameMs;
|
|
11127
|
+
this.quietMs = 0;
|
|
11128
|
+
return null;
|
|
11129
|
+
}
|
|
11130
|
+
this.floor = this.floor === null ? level : this.floor * 0.9 + level * 0.1;
|
|
11131
|
+
if (this.speechMs === 0) {
|
|
11132
|
+
return null;
|
|
11133
|
+
}
|
|
11134
|
+
this.quietMs += frameMs;
|
|
11135
|
+
if (this.quietMs < NOD_PAUSE_MS) {
|
|
11136
|
+
return null;
|
|
11137
|
+
}
|
|
11138
|
+
const run = this.speechMs;
|
|
11139
|
+
this.speechMs = 0;
|
|
11140
|
+
this.quietMs = 0;
|
|
11141
|
+
if (run < NOD_MIN_SPEECH_MS || this.sinceNodMs < this.intervalMs) {
|
|
11142
|
+
return null;
|
|
11143
|
+
}
|
|
11144
|
+
if (this.random() < NOD_SKIP_CHANCE) {
|
|
11145
|
+
return null;
|
|
11146
|
+
}
|
|
11147
|
+
this.sinceNodMs = 0;
|
|
11148
|
+
this.intervalMs = this.drawInterval();
|
|
11149
|
+
return {
|
|
11150
|
+
amplitudeDegrees: NOD_MIN_DEGREES + (NOD_MAX_DEGREES - NOD_MIN_DEGREES) * this.random(),
|
|
11151
|
+
durationMs: NOD_MIN_DURATION_MS + (NOD_MAX_DURATION_MS - NOD_MIN_DURATION_MS) * this.random()
|
|
11152
|
+
};
|
|
11153
|
+
}
|
|
11154
|
+
/** A new listening session: the speech run starts again; the noise floor stays. */
|
|
11155
|
+
reset() {
|
|
11156
|
+
this.speechMs = 0;
|
|
11157
|
+
this.quietMs = 0;
|
|
11158
|
+
this.sinceNodMs = Number.POSITIVE_INFINITY;
|
|
11159
|
+
}
|
|
11160
|
+
drawInterval() {
|
|
11161
|
+
return NOD_MIN_INTERVAL_MS + NOD_INTERVAL_JITTER_MS * this.random();
|
|
11162
|
+
}
|
|
11163
|
+
};
|
|
11164
|
+
|
|
11165
|
+
// ../../clients/web-client/src/character/player-activity.ts
|
|
11166
|
+
var TYPING_IDLE_MS = 1500;
|
|
11167
|
+
var PlayerActivityTracker = class {
|
|
11168
|
+
listening = false;
|
|
11169
|
+
typing = false;
|
|
11170
|
+
typingTimer = null;
|
|
11171
|
+
current = "idle";
|
|
11172
|
+
trigger;
|
|
11173
|
+
changeListeners = /* @__PURE__ */ new Set();
|
|
11174
|
+
nodListeners = /* @__PURE__ */ new Set();
|
|
11175
|
+
constructor(options = {}) {
|
|
11176
|
+
this.trigger = new SpeechNodTrigger(options.random);
|
|
11177
|
+
}
|
|
11178
|
+
get state() {
|
|
11179
|
+
return this.current;
|
|
11180
|
+
}
|
|
11181
|
+
/** The listening session is (or is no longer) in state `listening`. */
|
|
11182
|
+
setListening(active) {
|
|
11183
|
+
if (active === this.listening) {
|
|
11184
|
+
return;
|
|
11185
|
+
}
|
|
11186
|
+
this.listening = active;
|
|
11187
|
+
this.trigger.reset();
|
|
11188
|
+
this.resolve();
|
|
11189
|
+
}
|
|
11190
|
+
/**
|
|
11191
|
+
* Input in a text field: `true` at each input event, `false` when the
|
|
11192
|
+
* player sends or leaves the field. Without input, typing ends after 1.5 s.
|
|
11193
|
+
*/
|
|
11194
|
+
notifyTyping(active) {
|
|
11195
|
+
this.clearTypingTimer();
|
|
11196
|
+
this.typing = active;
|
|
11197
|
+
if (active) {
|
|
11198
|
+
this.typingTimer = setTimeout(() => {
|
|
11199
|
+
this.typingTimer = null;
|
|
11200
|
+
this.typing = false;
|
|
11201
|
+
this.resolve();
|
|
11202
|
+
}, TYPING_IDLE_MS);
|
|
11203
|
+
}
|
|
11204
|
+
this.resolve();
|
|
11205
|
+
}
|
|
11206
|
+
/** One microphone frame of the listening session; returns a nod when one is due. */
|
|
11207
|
+
pushVoiceLevel(level, frameMs) {
|
|
11208
|
+
if (!this.listening) {
|
|
11209
|
+
return null;
|
|
11210
|
+
}
|
|
11211
|
+
const nod = this.trigger.push(level, frameMs);
|
|
11212
|
+
if (nod) {
|
|
11213
|
+
this.nodListeners.forEach((listener) => listener(nod));
|
|
11214
|
+
}
|
|
11215
|
+
return nod;
|
|
11216
|
+
}
|
|
11217
|
+
onChange(listener) {
|
|
11218
|
+
this.changeListeners.add(listener);
|
|
11219
|
+
return () => this.changeListeners.delete(listener);
|
|
11220
|
+
}
|
|
11221
|
+
onNod(listener) {
|
|
11222
|
+
this.nodListeners.add(listener);
|
|
11223
|
+
return () => this.nodListeners.delete(listener);
|
|
11224
|
+
}
|
|
11225
|
+
/** Ends all activity, for example when the conversation ends. */
|
|
11226
|
+
reset() {
|
|
11227
|
+
this.clearTypingTimer();
|
|
11228
|
+
this.typing = false;
|
|
11229
|
+
this.listening = false;
|
|
11230
|
+
this.trigger.reset();
|
|
11231
|
+
this.resolve();
|
|
11232
|
+
}
|
|
11233
|
+
resolve() {
|
|
11234
|
+
const next = this.listening ? "speaking" : this.typing ? "typing" : "idle";
|
|
11235
|
+
if (next === this.current) {
|
|
11236
|
+
return;
|
|
11237
|
+
}
|
|
11238
|
+
this.current = next;
|
|
11239
|
+
this.changeListeners.forEach((listener) => listener(next));
|
|
11240
|
+
}
|
|
11241
|
+
clearTypingTimer() {
|
|
11242
|
+
if (this.typingTimer !== null) {
|
|
11243
|
+
clearTimeout(this.typingTimer);
|
|
11244
|
+
this.typingTimer = null;
|
|
11245
|
+
}
|
|
11246
|
+
}
|
|
11247
|
+
};
|
|
11248
|
+
|
|
8646
11249
|
// ../../clients/web-client/src/character/incremental-staging.ts
|
|
8647
11250
|
var dialogueKey = function(block) {
|
|
8648
11251
|
return JSON.stringify(block, (key, value) => {
|
|
@@ -8918,6 +11521,9 @@ var StagingPlayer = class {
|
|
|
8918
11521
|
activeExpressionMouthMode = null;
|
|
8919
11522
|
activeAnimation = null;
|
|
8920
11523
|
currentTrackTime = null;
|
|
11524
|
+
playbackElapsed = 0;
|
|
11525
|
+
appliedToolCallIds = /* @__PURE__ */ new Set();
|
|
11526
|
+
activeOptions;
|
|
8921
11527
|
get paused() {
|
|
8922
11528
|
return this.clock.paused;
|
|
8923
11529
|
}
|
|
@@ -8971,6 +11577,9 @@ var StagingPlayer = class {
|
|
|
8971
11577
|
this.incremental = incremental;
|
|
8972
11578
|
timeline = incremental.timeline;
|
|
8973
11579
|
this.onCursor = options?.onCursor;
|
|
11580
|
+
this.activeOptions = options;
|
|
11581
|
+
this.playbackElapsed = 0;
|
|
11582
|
+
this.appliedToolCallIds.clear();
|
|
8974
11583
|
let timelineStarted = false;
|
|
8975
11584
|
const beginTimeline = () => {
|
|
8976
11585
|
if (timelineStarted) {
|
|
@@ -8980,6 +11589,14 @@ var StagingPlayer = class {
|
|
|
8980
11589
|
options?.onTimelineStart?.(timeline);
|
|
8981
11590
|
};
|
|
8982
11591
|
try {
|
|
11592
|
+
await options?.onVariantCue?.(
|
|
11593
|
+
timeline.initialVariant ?? DEFAULT_VARIANT_KEY,
|
|
11594
|
+
null
|
|
11595
|
+
);
|
|
11596
|
+
if (controller.signal.aborted) {
|
|
11597
|
+
throw abortError2();
|
|
11598
|
+
}
|
|
11599
|
+
this.applyToolCalls(0, timeline);
|
|
8983
11600
|
let cursor = 0;
|
|
8984
11601
|
for (let index = 0; ; index += 1) {
|
|
8985
11602
|
const block = await incremental.next(index, controller.signal, () => {
|
|
@@ -9029,6 +11646,7 @@ var StagingPlayer = class {
|
|
|
9029
11646
|
}
|
|
9030
11647
|
await incremental.finishBlock(index, controller.signal);
|
|
9031
11648
|
cursor = block.endTime;
|
|
11649
|
+
this.playbackElapsed = Math.max(this.playbackElapsed, cursor);
|
|
9032
11650
|
}
|
|
9033
11651
|
if (timeline.duration > cursor) {
|
|
9034
11652
|
await this.animationController.prepareAnimations?.(
|
|
@@ -9050,10 +11668,15 @@ var StagingPlayer = class {
|
|
|
9050
11668
|
beginTimeline();
|
|
9051
11669
|
}
|
|
9052
11670
|
this.applyTracks(timeline.duration, false, timeline);
|
|
11671
|
+
if (timeline.activeVariant) {
|
|
11672
|
+
await options?.onVariantCue?.(timeline.activeVariant, null);
|
|
11673
|
+
}
|
|
9053
11674
|
} finally {
|
|
9054
11675
|
if (this.generation === generation) {
|
|
9055
11676
|
this.playbackController = null;
|
|
9056
11677
|
this.incremental = null;
|
|
11678
|
+
this.activeOptions = void 0;
|
|
11679
|
+
this.appliedToolCallIds.clear();
|
|
9057
11680
|
this.clearCursor();
|
|
9058
11681
|
this.onCursor = void 0;
|
|
9059
11682
|
this.clearPerformance();
|
|
@@ -9062,6 +11685,10 @@ var StagingPlayer = class {
|
|
|
9062
11685
|
}
|
|
9063
11686
|
updateTimeline(timeline, complete = true) {
|
|
9064
11687
|
this.incremental?.update(timeline, complete);
|
|
11688
|
+
if (this.incremental) {
|
|
11689
|
+
const elapsed = this.currentTrackTime ?? this.playbackElapsed;
|
|
11690
|
+
this.applyToolCalls(elapsed, this.incremental.timeline);
|
|
11691
|
+
}
|
|
9065
11692
|
const current = this.incremental?.timeline.dialogue.find(
|
|
9066
11693
|
(block) => block.id === this.pendingSpeech?.id
|
|
9067
11694
|
);
|
|
@@ -9112,6 +11739,9 @@ var StagingPlayer = class {
|
|
|
9112
11739
|
this.playbackController = null;
|
|
9113
11740
|
this.clock.resume();
|
|
9114
11741
|
this.incremental = null;
|
|
11742
|
+
this.activeOptions = void 0;
|
|
11743
|
+
this.playbackElapsed = 0;
|
|
11744
|
+
this.appliedToolCallIds.clear();
|
|
9115
11745
|
this.clearCursor();
|
|
9116
11746
|
this.onCursor = void 0;
|
|
9117
11747
|
this.audioPlayer.stop();
|
|
@@ -9327,8 +11957,29 @@ var StagingPlayer = class {
|
|
|
9327
11957
|
await waitFrame(signal);
|
|
9328
11958
|
}
|
|
9329
11959
|
}
|
|
11960
|
+
applyToolCalls(elapsed, timeline) {
|
|
11961
|
+
const calls = timeline.toolCalls;
|
|
11962
|
+
if (!calls || calls.length === 0) {
|
|
11963
|
+
return;
|
|
11964
|
+
}
|
|
11965
|
+
const ordered = calls.length > 1 ? [...calls].sort((left, right) => left.timestamp - right.timestamp) : calls;
|
|
11966
|
+
for (const call of ordered) {
|
|
11967
|
+
if (this.appliedToolCallIds.has(call.id) || elapsed < call.timestamp) {
|
|
11968
|
+
continue;
|
|
11969
|
+
}
|
|
11970
|
+
this.appliedToolCallIds.add(call.id);
|
|
11971
|
+
this.activeOptions?.onToolCall?.(call);
|
|
11972
|
+
if (call.name === "switch_variant" && call.result.ok) {
|
|
11973
|
+
void Promise.resolve(
|
|
11974
|
+
this.activeOptions?.onVariantCue?.(call.result.activeVariant, call)
|
|
11975
|
+
).catch(() => void 0);
|
|
11976
|
+
}
|
|
11977
|
+
}
|
|
11978
|
+
}
|
|
9330
11979
|
applyTracks(time, speechActive, timeline) {
|
|
9331
11980
|
this.currentTrackTime = time;
|
|
11981
|
+
this.playbackElapsed = Math.max(this.playbackElapsed, time);
|
|
11982
|
+
this.applyToolCalls(time, timeline);
|
|
9332
11983
|
const expression = activeClipAt(timeline.expressions, time);
|
|
9333
11984
|
const mouthMode = speechActive ? "viseme" : "expression";
|
|
9334
11985
|
if (expression !== this.activeExpression || expression && mouthMode !== this.activeExpressionMouthMode) {
|
|
@@ -9782,7 +12433,7 @@ var VisemePlayer = class {
|
|
|
9782
12433
|
};
|
|
9783
12434
|
|
|
9784
12435
|
// ../../clients/web-client/src/character/character-runtime.ts
|
|
9785
|
-
var CharacterRuntime = class {
|
|
12436
|
+
var CharacterRuntime = class extends CharacterSceneAccess {
|
|
9786
12437
|
baseUrl;
|
|
9787
12438
|
characterId;
|
|
9788
12439
|
apiToken;
|
|
@@ -9792,8 +12443,8 @@ var CharacterRuntime = class {
|
|
|
9792
12443
|
animation;
|
|
9793
12444
|
animations = /* @__PURE__ */ new Map();
|
|
9794
12445
|
animationController;
|
|
9795
|
-
sceneRig = new CharacterSceneRig();
|
|
9796
12446
|
lazyAnimations;
|
|
12447
|
+
variantController;
|
|
9797
12448
|
audioPlayer = null;
|
|
9798
12449
|
stagingPlayer = null;
|
|
9799
12450
|
facialMixer;
|
|
@@ -9802,13 +12453,15 @@ var CharacterRuntime = class {
|
|
|
9802
12453
|
telemetry;
|
|
9803
12454
|
streamPlayback;
|
|
9804
12455
|
liveMotion;
|
|
9805
|
-
|
|
12456
|
+
playerActivityTracker = new PlayerActivityTracker();
|
|
12457
|
+
listeningReflex;
|
|
9806
12458
|
currentGazeTarget = null;
|
|
9807
12459
|
/** The current gaze target came from `lookAt()`. */
|
|
9808
12460
|
lookAtActive = false;
|
|
9809
12461
|
thinkingState = false;
|
|
9810
12462
|
errorCallback;
|
|
9811
12463
|
constructor(options) {
|
|
12464
|
+
super();
|
|
9812
12465
|
this.baseUrl = options.baseUrl;
|
|
9813
12466
|
this.characterId = options.characterId;
|
|
9814
12467
|
this.apiToken = options.apiToken;
|
|
@@ -9822,6 +12475,13 @@ var CharacterRuntime = class {
|
|
|
9822
12475
|
animationId,
|
|
9823
12476
|
assetProfile: this.assetProfile
|
|
9824
12477
|
}));
|
|
12478
|
+
const loadVariant = options.loadVariant ?? ((variantKey) => loadVariantVrm({
|
|
12479
|
+
baseUrl: this.baseUrl,
|
|
12480
|
+
characterId: this.characterId,
|
|
12481
|
+
variantKey,
|
|
12482
|
+
apiToken: this.apiToken,
|
|
12483
|
+
assetProfile: this.assetProfile
|
|
12484
|
+
}));
|
|
9825
12485
|
this.lazyAnimations = new LazyAnimations(loadAnimation, (id, animation) => {
|
|
9826
12486
|
this.animations.set(id, animation);
|
|
9827
12487
|
this.animationController?.addAnimation?.(id, animation, { loop: false });
|
|
@@ -9851,6 +12511,36 @@ var CharacterRuntime = class {
|
|
|
9851
12511
|
apiToken: this.apiToken
|
|
9852
12512
|
});
|
|
9853
12513
|
this.liveMotion = new LiveMotionPlayer(options.liveMotion);
|
|
12514
|
+
this.listeningReflex = new ListeningReflex(
|
|
12515
|
+
{
|
|
12516
|
+
vrm: () => this.sceneRig.vrm,
|
|
12517
|
+
gazeTarget: () => this.currentGazeTarget,
|
|
12518
|
+
lookAtActive: () => this.lookAtActive,
|
|
12519
|
+
lookAtTemporarily: (target) => this.lookAtTemporarily(target),
|
|
12520
|
+
facialMixer: this.facialMixer
|
|
12521
|
+
},
|
|
12522
|
+
this.playerActivityTracker
|
|
12523
|
+
);
|
|
12524
|
+
this.variantController = new CharacterVariantController({
|
|
12525
|
+
loadVariant,
|
|
12526
|
+
getCurrentVrm: () => this.sceneRig.vrm,
|
|
12527
|
+
handleCallbackError: this.handleCallbackError,
|
|
12528
|
+
applyVrmSwap: (nextVrm) => {
|
|
12529
|
+
this.sceneRig.setVrm(nextVrm, this.defaultParent);
|
|
12530
|
+
this.facialMixer.setVrm(nextVrm, { preserveState: true });
|
|
12531
|
+
this.animationController?.setVrm?.(nextVrm, this.animations);
|
|
12532
|
+
this.liveMotion.setVrm(nextVrm);
|
|
12533
|
+
this.applyGaze();
|
|
12534
|
+
if (this.performanceRecorder.isRecording) {
|
|
12535
|
+
this.facialMixer.capturePerformanceModel();
|
|
12536
|
+
}
|
|
12537
|
+
}
|
|
12538
|
+
});
|
|
12539
|
+
if (options.initialAssetInfo?.variants) {
|
|
12540
|
+
this.variantController.configureVariants(
|
|
12541
|
+
options.initialAssetInfo.variants
|
|
12542
|
+
);
|
|
12543
|
+
}
|
|
9854
12544
|
this.streamPlayback = new StreamPlaybackController({
|
|
9855
12545
|
getStagingPlayer: this.getStagingPlayer,
|
|
9856
12546
|
telemetry: this.telemetry,
|
|
@@ -9864,6 +12554,19 @@ var CharacterRuntime = class {
|
|
|
9864
12554
|
void this.liveMotion.load(stagingId, track).catch((error) => {
|
|
9865
12555
|
this.handleCallbackError("liveMotion", error);
|
|
9866
12556
|
});
|
|
12557
|
+
},
|
|
12558
|
+
onToolCall: (toolCall) => {
|
|
12559
|
+
this.variantController.emit("toolCall", toolCall);
|
|
12560
|
+
},
|
|
12561
|
+
onVariantCue: async (variantKey, toolCall) => {
|
|
12562
|
+
try {
|
|
12563
|
+
await this.variantController.switchVariant(variantKey, toolCall);
|
|
12564
|
+
} catch (error) {
|
|
12565
|
+
this.reportError(
|
|
12566
|
+
error instanceof Error ? error : new Error(String(error)),
|
|
12567
|
+
"switchVariant"
|
|
12568
|
+
);
|
|
12569
|
+
}
|
|
9867
12570
|
}
|
|
9868
12571
|
});
|
|
9869
12572
|
if (options.gazeTarget !== void 0) {
|
|
@@ -9899,53 +12602,28 @@ var CharacterRuntime = class {
|
|
|
9899
12602
|
return this.sceneRig.vrm;
|
|
9900
12603
|
}
|
|
9901
12604
|
set vrm(value) {
|
|
9902
|
-
this.sceneRig.
|
|
12605
|
+
const previous = this.sceneRig.vrm;
|
|
12606
|
+
this.sceneRig.setVrm(value, this.defaultParent);
|
|
9903
12607
|
this.facialMixer.setVrm(value);
|
|
9904
12608
|
this.liveMotion.setVrm(value);
|
|
9905
12609
|
this.applyGaze();
|
|
9906
|
-
|
|
9907
|
-
this.sceneRig.attachTo(this.defaultParent);
|
|
9908
|
-
}
|
|
9909
|
-
}
|
|
9910
|
-
/** True once the character VRM model is loaded. */
|
|
9911
|
-
get ready() {
|
|
9912
|
-
return this.sceneRig.vrm !== null;
|
|
9913
|
-
}
|
|
9914
|
-
/** Root Three.js `Group` (`vrm.scene`) to add to a `THREE.Scene`. */
|
|
9915
|
-
get scene() {
|
|
9916
|
-
return this.sceneRig.scene;
|
|
9917
|
-
}
|
|
9918
|
-
/** Alias for `scene` for Three.js `Object3D` workflows. */
|
|
9919
|
-
get object3d() {
|
|
9920
|
-
return this.sceneRig.object3d;
|
|
9921
|
-
}
|
|
9922
|
-
/** Alias for `scene`. */
|
|
9923
|
-
get root() {
|
|
9924
|
-
return this.sceneRig.root;
|
|
9925
|
-
}
|
|
9926
|
-
/** Primary character mesh (`SkinnedMesh` preferred, otherwise first `Mesh`). */
|
|
9927
|
-
get mesh() {
|
|
9928
|
-
return this.sceneRig.mesh;
|
|
9929
|
-
}
|
|
9930
|
-
/** All Three.js `Mesh` instances in the character hierarchy. */
|
|
9931
|
-
get meshes() {
|
|
9932
|
-
return this.sceneRig.meshes;
|
|
12610
|
+
this.variantController.syncDirectVrm(value, previous);
|
|
9933
12611
|
}
|
|
9934
|
-
/**
|
|
9935
|
-
get
|
|
9936
|
-
return this.
|
|
12612
|
+
/** Allowed appearance variants configured for this character (`default` first). */
|
|
12613
|
+
get variants() {
|
|
12614
|
+
return this.variantController.variants;
|
|
9937
12615
|
}
|
|
9938
|
-
/**
|
|
9939
|
-
get
|
|
9940
|
-
return this.
|
|
12616
|
+
/** Currently active appearance variant key (defaults to `'default'`). */
|
|
12617
|
+
get activeVariant() {
|
|
12618
|
+
return this.variantController.activeVariant;
|
|
9941
12619
|
}
|
|
9942
|
-
/**
|
|
9943
|
-
get
|
|
9944
|
-
return this.
|
|
12620
|
+
/** Cached Three-VRM instances keyed by variant key. */
|
|
12621
|
+
get variantVrms() {
|
|
12622
|
+
return this.variantController.variantVrms;
|
|
9945
12623
|
}
|
|
9946
|
-
/**
|
|
9947
|
-
|
|
9948
|
-
|
|
12624
|
+
/** Switch the character's active appearance variant, loading on-demand if needed. */
|
|
12625
|
+
async switchVariant(variantKey) {
|
|
12626
|
+
await this.variantController.switchVariant(variantKey, null);
|
|
9949
12627
|
}
|
|
9950
12628
|
/** Underlying Three.js `AnimationMixer` driving skeletal animations. */
|
|
9951
12629
|
get mixer() {
|
|
@@ -9998,36 +12676,6 @@ var CharacterRuntime = class {
|
|
|
9998
12676
|
cursor: this.cursor
|
|
9999
12677
|
};
|
|
10000
12678
|
}
|
|
10001
|
-
/** Look up a raw humanoid bone `Object3D` by VRM bone name (for example `'head'` or `'rightHand'`). */
|
|
10002
|
-
getBone(name) {
|
|
10003
|
-
return this.sceneRig.getBone(name);
|
|
10004
|
-
}
|
|
10005
|
-
/** Look up a normalized humanoid bone `Object3D` by VRM bone name. */
|
|
10006
|
-
getNormalizedBone(name) {
|
|
10007
|
-
return this.sceneRig.getNormalizedBone(name);
|
|
10008
|
-
}
|
|
10009
|
-
/** Attach the character root `Group` to a Three.js `Scene` or parent `Object3D`. */
|
|
10010
|
-
attachTo(parent) {
|
|
10011
|
-
this.defaultParent = parent;
|
|
10012
|
-
return this.sceneRig.attachTo(parent);
|
|
10013
|
-
}
|
|
10014
|
-
/** Detach the character root `Group` from its current parent `Object3D`. */
|
|
10015
|
-
detach() {
|
|
10016
|
-
this.defaultParent = null;
|
|
10017
|
-
this.sceneRig.detach();
|
|
10018
|
-
}
|
|
10019
|
-
/** Configure `castShadow` and `receiveShadow` on all character meshes. */
|
|
10020
|
-
setShadows(options) {
|
|
10021
|
-
this.sceneRig.setShadows(options);
|
|
10022
|
-
}
|
|
10023
|
-
/** Configure `position`, `rotation`, and `scale` on the character root `Group`. */
|
|
10024
|
-
setTransform(options) {
|
|
10025
|
-
this.sceneRig.setTransform(options);
|
|
10026
|
-
}
|
|
10027
|
-
/** Re-index cached meshes, materials, and skeletons after custom scene graph edits. */
|
|
10028
|
-
refreshMeshes() {
|
|
10029
|
-
this.sceneRig.refreshMeshes();
|
|
10030
|
-
}
|
|
10031
12679
|
async ensureAnimation(animationId) {
|
|
10032
12680
|
await this.lazyAnimations.ensure(animationId);
|
|
10033
12681
|
}
|
|
@@ -10126,6 +12774,28 @@ var CharacterRuntime = class {
|
|
|
10126
12774
|
this.applyGaze();
|
|
10127
12775
|
};
|
|
10128
12776
|
}
|
|
12777
|
+
/**
|
|
12778
|
+
* Tells the character that the player types (`true` at each input event
|
|
12779
|
+
* of the text field) or stopped (`false`, for example after sending). The
|
|
12780
|
+
* character looks at the player and shows a subtle attentive face; typing
|
|
12781
|
+
* ends by itself after 1.5 s without input. Local only: no request.
|
|
12782
|
+
*/
|
|
12783
|
+
notifyUserTyping(active) {
|
|
12784
|
+
this.playerActivityTracker.notifyTyping(active);
|
|
12785
|
+
}
|
|
12786
|
+
/** `'speaking'` while listening, `'typing'` after typed input, else `'idle'`. */
|
|
12787
|
+
get playerActivity() {
|
|
12788
|
+
return this.playerActivityTracker.state;
|
|
12789
|
+
}
|
|
12790
|
+
onPlayerActivity(callback) {
|
|
12791
|
+
return this.playerActivityTracker.onChange((state) => {
|
|
12792
|
+
try {
|
|
12793
|
+
callback(state);
|
|
12794
|
+
} catch (error) {
|
|
12795
|
+
this.handleCallbackError("onPlayerActivity", error);
|
|
12796
|
+
}
|
|
12797
|
+
});
|
|
12798
|
+
}
|
|
10129
12799
|
startPerformanceRecording(options) {
|
|
10130
12800
|
this.performanceRecorder.start(options);
|
|
10131
12801
|
this.facialMixer.capturePerformanceModel();
|
|
@@ -10172,6 +12842,21 @@ var CharacterRuntime = class {
|
|
|
10172
12842
|
onTimelineStart(callback) {
|
|
10173
12843
|
return this.streamPlayback.onTimelineStart(callback);
|
|
10174
12844
|
}
|
|
12845
|
+
on(event, listener) {
|
|
12846
|
+
return this.variantController.on(event, listener);
|
|
12847
|
+
}
|
|
12848
|
+
off(event, listener) {
|
|
12849
|
+
this.variantController.off(event, listener);
|
|
12850
|
+
}
|
|
12851
|
+
onVrmChange(callback) {
|
|
12852
|
+
return this.variantController.on("vrmChange", callback);
|
|
12853
|
+
}
|
|
12854
|
+
onVariantChange(callback) {
|
|
12855
|
+
return this.variantController.on("variantChange", callback);
|
|
12856
|
+
}
|
|
12857
|
+
onToolCall(callback) {
|
|
12858
|
+
return this.variantController.on("toolCall", callback);
|
|
12859
|
+
}
|
|
10175
12860
|
/** Advance skeletal animations, lip-sync visemes, facial expressions, and VRM physics by `delta` seconds. */
|
|
10176
12861
|
update(delta) {
|
|
10177
12862
|
if (this.stagingPlayer?.paused) {
|
|
@@ -10183,6 +12868,10 @@ var CharacterRuntime = class {
|
|
|
10183
12868
|
this.streamPlayback.activeStagingId,
|
|
10184
12869
|
this.stagingPlayer?.trackTime ?? null
|
|
10185
12870
|
);
|
|
12871
|
+
this.listeningReflex.update(delta, {
|
|
12872
|
+
stagingActive: this.streamPlayback.playing || this.speaking || (this.stagingPlayer?.trackTime ?? null) !== null,
|
|
12873
|
+
liveMotionWeight: this.liveMotion.liveWeight
|
|
12874
|
+
});
|
|
10186
12875
|
this.visemePlayer.update(delta);
|
|
10187
12876
|
this.facialMixer.update(delta);
|
|
10188
12877
|
this.vrm?.update(delta);
|
|
@@ -10226,7 +12915,10 @@ var CharacterRuntime = class {
|
|
|
10226
12915
|
this.lookAtActive && !this.facialMixer.hasIdleFace ? this.currentGazeTarget : null
|
|
10227
12916
|
);
|
|
10228
12917
|
}
|
|
10229
|
-
resetRuntimeAssets() {
|
|
12918
|
+
resetRuntimeAssets(options) {
|
|
12919
|
+
const cachedVrms = this.variantController.reset();
|
|
12920
|
+
this.listeningReflex.reset();
|
|
12921
|
+
this.playerActivityTracker.reset();
|
|
10230
12922
|
this.lazyAnimations.reset();
|
|
10231
12923
|
this.stagingPlayer?.cleanup();
|
|
10232
12924
|
this.stagingPlayer = null;
|
|
@@ -10237,13 +12929,16 @@ var CharacterRuntime = class {
|
|
|
10237
12929
|
}
|
|
10238
12930
|
this.animationController = void 0;
|
|
10239
12931
|
this.animations.clear();
|
|
10240
|
-
|
|
12932
|
+
if (options?.disposeVrms !== false) {
|
|
12933
|
+
this.sceneRig.disposeAllVrms(cachedVrms);
|
|
12934
|
+
} else {
|
|
12935
|
+
this.sceneRig.setVrm(null);
|
|
12936
|
+
}
|
|
10241
12937
|
this.facialMixer.setVrm(null);
|
|
10242
12938
|
this.liveMotion.setVrm(null);
|
|
10243
12939
|
}
|
|
10244
12940
|
disposeRuntimeAssets() {
|
|
10245
|
-
this.
|
|
10246
|
-
this.resetRuntimeAssets();
|
|
12941
|
+
this.resetRuntimeAssets({ disposeVrms: true });
|
|
10247
12942
|
this.audioPlayer?.cleanup();
|
|
10248
12943
|
this.audioPlayer = null;
|
|
10249
12944
|
}
|
|
@@ -11739,8 +14434,8 @@ var MicrophoneCapture = class {
|
|
|
11739
14434
|
};
|
|
11740
14435
|
|
|
11741
14436
|
// ../../clients/web-client/src/voice/silence-detector.ts
|
|
11742
|
-
var
|
|
11743
|
-
var
|
|
14437
|
+
var MIN_SPEECH_LEVEL2 = 0.015;
|
|
14438
|
+
var SPEECH_TO_NOISE2 = 3;
|
|
11744
14439
|
var SPEECH_START_MS = 200;
|
|
11745
14440
|
var SilenceDetector = class {
|
|
11746
14441
|
constructor(silenceMs) {
|
|
@@ -11753,8 +14448,8 @@ var SilenceDetector = class {
|
|
|
11753
14448
|
/** True when the speech has ended. */
|
|
11754
14449
|
push(level, frameMs = SPEECH_FRAME_MS) {
|
|
11755
14450
|
const threshold = Math.max(
|
|
11756
|
-
|
|
11757
|
-
(this.floor ??
|
|
14451
|
+
MIN_SPEECH_LEVEL2,
|
|
14452
|
+
(this.floor ?? MIN_SPEECH_LEVEL2 / SPEECH_TO_NOISE2) * SPEECH_TO_NOISE2
|
|
11758
14453
|
);
|
|
11759
14454
|
if (level >= threshold) {
|
|
11760
14455
|
this.loudMs += frameMs;
|
|
@@ -11893,6 +14588,7 @@ var VoiceInput = class {
|
|
|
11893
14588
|
send: options.send !== false,
|
|
11894
14589
|
silent: options.silent === true,
|
|
11895
14590
|
renderSessionId: this.deps.renderSessionId(),
|
|
14591
|
+
timeZone: detectClientTimeZone(),
|
|
11896
14592
|
format: SPEECH_AUDIO_FORMAT
|
|
11897
14593
|
})
|
|
11898
14594
|
),
|
|
@@ -12045,6 +14741,7 @@ var VoiceInput = class {
|
|
|
12045
14741
|
if (session !== this.session || session.captureClosed) {
|
|
12046
14742
|
return;
|
|
12047
14743
|
}
|
|
14744
|
+
this.deps.activity?.pushVoiceLevel(frame.level);
|
|
12048
14745
|
const audio = new Uint8Array(
|
|
12049
14746
|
frame.pcm.buffer,
|
|
12050
14747
|
frame.pcm.byteOffset,
|
|
@@ -12190,6 +14887,7 @@ var VoiceInput = class {
|
|
|
12190
14887
|
setState(state) {
|
|
12191
14888
|
if (this.state !== state) {
|
|
12192
14889
|
this.state = state;
|
|
14890
|
+
this.deps.activity?.setListening(state === "listening");
|
|
12193
14891
|
this.deps.onState(state);
|
|
12194
14892
|
}
|
|
12195
14893
|
}
|
|
@@ -12208,6 +14906,8 @@ var Character = class extends CharacterRuntime {
|
|
|
12208
14906
|
/** The conversation whose live motion was warmed up. */
|
|
12209
14907
|
liveMotionPrepared = null;
|
|
12210
14908
|
authoring;
|
|
14909
|
+
hostedWebChat;
|
|
14910
|
+
memoryIdentifier;
|
|
12211
14911
|
normalizeIdleOrientation;
|
|
12212
14912
|
bakedAsset;
|
|
12213
14913
|
animationRefreshId = 0;
|
|
@@ -12234,6 +14934,8 @@ var Character = class extends CharacterRuntime {
|
|
|
12234
14934
|
})
|
|
12235
14935
|
});
|
|
12236
14936
|
this.authoring = options.authoring ?? false;
|
|
14937
|
+
this.hostedWebChat = options.hostedWebChat === true && !resolvedToken;
|
|
14938
|
+
this.memoryIdentifier = normalizeMemoryIdentifier(options.memoryIdentifier);
|
|
12237
14939
|
this.normalizeIdleOrientation = options.normalizeIdleOrientation ?? false;
|
|
12238
14940
|
this.bakedAsset = options.bakedConversation;
|
|
12239
14941
|
this.initialAssetInfo = options.initialAssetInfo ?? null;
|
|
@@ -12300,6 +15002,7 @@ var Character = class extends CharacterRuntime {
|
|
|
12300
15002
|
);
|
|
12301
15003
|
}
|
|
12302
15004
|
},
|
|
15005
|
+
activity: this.playerActivityTracker,
|
|
12303
15006
|
onTranscript: (event) => {
|
|
12304
15007
|
if (this.transcriptCallback) {
|
|
12305
15008
|
invokeSafeCallback(
|
|
@@ -12316,6 +15019,7 @@ var Character = class extends CharacterRuntime {
|
|
|
12316
15019
|
telemetry: this.telemetry,
|
|
12317
15020
|
socket: () => this.conversationSocket,
|
|
12318
15021
|
renderSession: () => this.renderSessionId,
|
|
15022
|
+
memoryIdentifier: () => this.apiToken ? this.memoryIdentifier : void 0,
|
|
12319
15023
|
thinking: () => this.invokeThinkingCallback({ taskId: null, stagingId: null }),
|
|
12320
15024
|
handleResult: (payload, forcePlayback, silent) => this.handleStagingResult(payload, {
|
|
12321
15025
|
source: "api",
|
|
@@ -12336,6 +15040,7 @@ var Character = class extends CharacterRuntime {
|
|
|
12336
15040
|
const progressReporter = createAssetProgressReporter(callback);
|
|
12337
15041
|
const info = await this.getAssetInfo();
|
|
12338
15042
|
const baked = !this.authoring && Boolean(this.bakedAsset || info.bakedConversationEnabled);
|
|
15043
|
+
this.variantController.configureVariants(info.variants, { remote: false });
|
|
12339
15044
|
this.inputs.load({
|
|
12340
15045
|
allowCustomReply: info.allowCustomReply !== false,
|
|
12341
15046
|
baked,
|
|
@@ -12359,8 +15064,10 @@ var Character = class extends CharacterRuntime {
|
|
|
12359
15064
|
onProgress: progressReporter.updateResource,
|
|
12360
15065
|
assetProfile: this.assetProfile
|
|
12361
15066
|
}),
|
|
12362
|
-
this.baked ? this.lazyAnimations.ensureMany(animationIds) : info.idleAnimationId ? this.lazyAnimations.ensure(info.idleAnimationId) : Promise.resolve()
|
|
15067
|
+
this.baked ? this.lazyAnimations.ensureMany(animationIds) : info.idleAnimationId ? this.lazyAnimations.ensure(info.idleAnimationId) : Promise.resolve(),
|
|
15068
|
+
this.baked ? this.variantController.preloadVariants(true) : Promise.resolve()
|
|
12363
15069
|
]);
|
|
15070
|
+
this.variantController.registerDefaultVrm(vrm);
|
|
12364
15071
|
this.vrm = vrm;
|
|
12365
15072
|
if (this.performanceRecorder.isRecording) {
|
|
12366
15073
|
this.facialMixer.capturePerformanceModel();
|
|
@@ -12378,6 +15085,9 @@ var Character = class extends CharacterRuntime {
|
|
|
12378
15085
|
});
|
|
12379
15086
|
}
|
|
12380
15087
|
animationController.playIdle(0);
|
|
15088
|
+
if (!this.baked) {
|
|
15089
|
+
void this.variantController.preloadVariants(false);
|
|
15090
|
+
}
|
|
12381
15091
|
}
|
|
12382
15092
|
async reloadAnimations(info) {
|
|
12383
15093
|
const requestId = ++this.animationRefreshId;
|
|
@@ -12411,6 +15121,7 @@ var Character = class extends CharacterRuntime {
|
|
|
12411
15121
|
if (info.bakedConversationEnabled || this.bakedAsset) {
|
|
12412
15122
|
throw new Error("Baked conversation requires local rendering.");
|
|
12413
15123
|
}
|
|
15124
|
+
this.variantController.configureVariants(info.variants, { remote: true });
|
|
12414
15125
|
this.inputs.load({
|
|
12415
15126
|
allowCustomReply: info.allowCustomReply !== false,
|
|
12416
15127
|
baked: false,
|
|
@@ -12452,7 +15163,9 @@ var Character = class extends CharacterRuntime {
|
|
|
12452
15163
|
baseUrl: this.baseUrl,
|
|
12453
15164
|
apiToken: this.apiToken,
|
|
12454
15165
|
avatarId: this.characterId,
|
|
12455
|
-
renderSessionId: this.renderSessionId
|
|
15166
|
+
renderSessionId: this.renderSessionId,
|
|
15167
|
+
hostedWebChat: this.hostedWebChat,
|
|
15168
|
+
memoryIdentifier: this.apiToken ? this.memoryIdentifier : void 0
|
|
12456
15169
|
});
|
|
12457
15170
|
if (!response.conversation?.id) {
|
|
12458
15171
|
throw new Error("Failed to create conversation.");
|
|
@@ -12504,7 +15217,7 @@ var Character = class extends CharacterRuntime {
|
|
|
12504
15217
|
await this.endConversation();
|
|
12505
15218
|
if (options?.disposeVrm === false) {
|
|
12506
15219
|
this.detach();
|
|
12507
|
-
this.resetRuntimeAssets();
|
|
15220
|
+
this.resetRuntimeAssets({ disposeVrms: false });
|
|
12508
15221
|
this.audioPlayer?.cleanup();
|
|
12509
15222
|
this.audioPlayer = null;
|
|
12510
15223
|
return;
|
|
@@ -12685,6 +15398,15 @@ var Character = class extends CharacterRuntime {
|
|
|
12685
15398
|
setRenderSessionId(renderSessionId) {
|
|
12686
15399
|
this.renderSessionId = renderSessionId?.trim() || null;
|
|
12687
15400
|
}
|
|
15401
|
+
setHostedWebChat(enabled) {
|
|
15402
|
+
this.hostedWebChat = enabled === true && !this.apiToken;
|
|
15403
|
+
}
|
|
15404
|
+
setMemoryIdentifier(memoryIdentifier) {
|
|
15405
|
+
this.memoryIdentifier = normalizeMemoryIdentifier(memoryIdentifier);
|
|
15406
|
+
}
|
|
15407
|
+
getMemoryIdentifier() {
|
|
15408
|
+
return this.memoryIdentifier;
|
|
15409
|
+
}
|
|
12688
15410
|
onStagingResult(callback) {
|
|
12689
15411
|
return bindCallbackSlot(
|
|
12690
15412
|
() => this.stagingResultCallback,
|
|
@@ -12714,33 +15436,22 @@ var Character = class extends CharacterRuntime {
|
|
|
12714
15436
|
};
|
|
12715
15437
|
async handleStagingResult(payload, options) {
|
|
12716
15438
|
this.thinkingState = false;
|
|
12717
|
-
|
|
12718
|
-
|
|
12719
|
-
|
|
12720
|
-
this.
|
|
12721
|
-
|
|
12722
|
-
|
|
12723
|
-
|
|
12724
|
-
|
|
12725
|
-
|
|
12726
|
-
|
|
12727
|
-
|
|
12728
|
-
|
|
15439
|
+
await deliverStagingResult(payload, {
|
|
15440
|
+
...options,
|
|
15441
|
+
notifiedStagingIds: this.notifiedStagingIds,
|
|
15442
|
+
hasPlayed: (stagingId) => this.streamPlayback.hasPlayed(stagingId),
|
|
15443
|
+
setOptions: (next) => this.inputs.setOptions(next),
|
|
15444
|
+
onTurnReceived: (normalized) => this.liveConversation.received(normalized),
|
|
15445
|
+
onStagingResult: this.stagingResultCallback,
|
|
15446
|
+
onCallbackError: this.handleCallbackError,
|
|
15447
|
+
recordEvent: (event) => this.performanceRecorder.recordEvent(event),
|
|
15448
|
+
playRequested: (normalized, playOptions) => {
|
|
15449
|
+
void this.playStaging(normalized, playOptions);
|
|
15450
|
+
},
|
|
15451
|
+
playStreamed: (normalized) => {
|
|
15452
|
+
void this.streamPlayback.playPayload(normalized);
|
|
12729
15453
|
}
|
|
12730
|
-
}
|
|
12731
|
-
const shouldPlay = !this.streamPlayback.hasPlayed(normalized.stagingId) || force;
|
|
12732
|
-
this.performanceRecorder.recordEvent({
|
|
12733
|
-
type: "staging.result",
|
|
12734
|
-
source: options.source,
|
|
12735
|
-
stagingId: normalized.stagingId,
|
|
12736
|
-
taskId: normalized.taskId ?? "",
|
|
12737
|
-
playback: shouldPlay ? "requested" : "already-streamed"
|
|
12738
15454
|
});
|
|
12739
|
-
if (shouldPlay) {
|
|
12740
|
-
void this.playStaging(normalized, { force, silent: options.silent });
|
|
12741
|
-
} else {
|
|
12742
|
-
void this.streamPlayback.playPayload(normalized);
|
|
12743
|
-
}
|
|
12744
15455
|
}
|
|
12745
15456
|
invokeThinkingCallback(payload) {
|
|
12746
15457
|
this.thinkingState = true;
|
|
@@ -13251,6 +15962,7 @@ var mountNpcify = async function(options) {
|
|
|
13251
15962
|
camera: null,
|
|
13252
15963
|
renderer: null,
|
|
13253
15964
|
controls: null,
|
|
15965
|
+
setMemoryIdentifier: (memoryIdentifier) => npcify.setMemoryIdentifier(memoryIdentifier),
|
|
13254
15966
|
dispose: async () => {
|
|
13255
15967
|
remote.dispose();
|
|
13256
15968
|
await npcify.dispose();
|
|
@@ -13264,6 +15976,7 @@ var mountNpcify = async function(options) {
|
|
|
13264
15976
|
let context2;
|
|
13265
15977
|
let session;
|
|
13266
15978
|
let observer;
|
|
15979
|
+
let unsubscribeVrm;
|
|
13267
15980
|
let frame = 0;
|
|
13268
15981
|
let disposed = false;
|
|
13269
15982
|
const dispose = async function() {
|
|
@@ -13273,10 +15986,11 @@ var mountNpcify = async function(options) {
|
|
|
13273
15986
|
disposed = true;
|
|
13274
15987
|
cancelAnimationFrame(frame);
|
|
13275
15988
|
observer?.disconnect();
|
|
13276
|
-
|
|
15989
|
+
unsubscribeVrm?.();
|
|
13277
15990
|
if (context2 && session) {
|
|
13278
|
-
detachVrm(context2, session, { dispose:
|
|
15991
|
+
detachVrm(context2, session, { dispose: false });
|
|
13279
15992
|
}
|
|
15993
|
+
await npcify.dispose();
|
|
13280
15994
|
if (context2) {
|
|
13281
15995
|
disposeSceneContext(context2);
|
|
13282
15996
|
}
|
|
@@ -13288,6 +16002,16 @@ var mountNpcify = async function(options) {
|
|
|
13288
16002
|
}
|
|
13289
16003
|
context2 = createSceneContext({ canvas: options.canvas });
|
|
13290
16004
|
session = attachVrm(context2, npcify.vrm);
|
|
16005
|
+
unsubscribeVrm = npcify.on("vrmChange", (nextVrm) => {
|
|
16006
|
+
if (context2) {
|
|
16007
|
+
setCameraCollisionBackend(
|
|
16008
|
+
context2,
|
|
16009
|
+
new HumanoidCapsuleCollisionBackend(
|
|
16010
|
+
(name) => nextVrm.humanoid.getRawBoneNode(name)
|
|
16011
|
+
)
|
|
16012
|
+
);
|
|
16013
|
+
}
|
|
16014
|
+
});
|
|
13291
16015
|
applyCameraPreset(context2, "full_body");
|
|
13292
16016
|
npcify.setGazeTarget(context2.camera);
|
|
13293
16017
|
const resize = () => {
|
|
@@ -13321,6 +16045,7 @@ var mountNpcify = async function(options) {
|
|
|
13321
16045
|
camera: context2.camera,
|
|
13322
16046
|
renderer: context2.renderer,
|
|
13323
16047
|
controls: context2.controls,
|
|
16048
|
+
setMemoryIdentifier: (memoryIdentifier) => npcify.setMemoryIdentifier(memoryIdentifier),
|
|
13324
16049
|
dispose
|
|
13325
16050
|
};
|
|
13326
16051
|
} catch (error) {
|
|
@@ -13339,12 +16064,14 @@ export {
|
|
|
13339
16064
|
CharacterRuntime,
|
|
13340
16065
|
CharacterSceneRig,
|
|
13341
16066
|
ConversationConnectionError,
|
|
16067
|
+
DEFAULT_VARIANT_KEY,
|
|
13342
16068
|
FacialPerformanceLayer,
|
|
13343
16069
|
FacialPerformanceMixer,
|
|
13344
16070
|
INPUT_MODES,
|
|
13345
16071
|
InputModeError,
|
|
13346
16072
|
LiveMotionPlayer,
|
|
13347
16073
|
MicrophoneError,
|
|
16074
|
+
NUDE_VARIANT_KEY,
|
|
13348
16075
|
Npcify,
|
|
13349
16076
|
Character as NpcifyCharacter,
|
|
13350
16077
|
PERFORMANCE_FILE_EXTENSION,
|
|
@@ -13358,7 +16085,11 @@ export {
|
|
|
13358
16085
|
createCharacter,
|
|
13359
16086
|
decodePerformance,
|
|
13360
16087
|
encodePerformance,
|
|
16088
|
+
fetchVariantVrmDownloadUrl,
|
|
16089
|
+
fetchVrmDownloadUrl,
|
|
13361
16090
|
loadBakedConversation,
|
|
13362
16091
|
loadCharacter,
|
|
16092
|
+
loadVariantVrm,
|
|
16093
|
+
loadVrm,
|
|
13363
16094
|
mountNpcify
|
|
13364
16095
|
};
|