@npcify/web 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -0
- package/dist/index.d.ts +346 -44
- package/dist/index.js +1914 -203
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -34,6 +34,33 @@ var isValidMotionFramesEvent = function(event) {
|
|
|
34
34
|
// ../ts-shared/src/llm.ts
|
|
35
35
|
var STAGING_PROTOCOL_VERSION = 1;
|
|
36
36
|
|
|
37
|
+
// ../ts-shared/src/variants.ts
|
|
38
|
+
var DEFAULT_VARIANT_KEY = "default";
|
|
39
|
+
var NUDE_VARIANT_KEY = "nude";
|
|
40
|
+
var VARIANT_KEY_PATTERN = /^[a-z][a-z0-9_-]{0,39}$/u;
|
|
41
|
+
var isValidVariantKey = function(value) {
|
|
42
|
+
return typeof value === "string" && VARIANT_KEY_PATTERN.test(value);
|
|
43
|
+
};
|
|
44
|
+
var STAGING_TOOL_RESULT_STATUSES = [
|
|
45
|
+
"applied",
|
|
46
|
+
"noop",
|
|
47
|
+
"rejected"
|
|
48
|
+
];
|
|
49
|
+
var isStagingToolCall = function(value) {
|
|
50
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
51
|
+
return false;
|
|
52
|
+
}
|
|
53
|
+
const candidate = value;
|
|
54
|
+
if (typeof candidate.id !== "string" || !candidate.id.trim() || candidate.name !== "switch_variant" || typeof candidate.timestamp !== "number" || !Number.isFinite(candidate.timestamp) || candidate.timestamp < 0 || !candidate.arguments || typeof candidate.arguments !== "object" || Array.isArray(candidate.arguments) || !candidate.result || typeof candidate.result !== "object" || Array.isArray(candidate.result)) {
|
|
55
|
+
return false;
|
|
56
|
+
}
|
|
57
|
+
const args = candidate.arguments;
|
|
58
|
+
const result = candidate.result;
|
|
59
|
+
return isValidVariantKey(args.variant) && typeof result.ok === "boolean" && STAGING_TOOL_RESULT_STATUSES.includes(
|
|
60
|
+
String(result.status)
|
|
61
|
+
) && isValidVariantKey(result.previousVariant) && isValidVariantKey(result.activeVariant) && (result.error === void 0 || typeof result.error === "string");
|
|
62
|
+
};
|
|
63
|
+
|
|
37
64
|
// ../ts-shared/src/server-rendering.ts
|
|
38
65
|
var SERVER_RENDER_PROTOCOL_VERSION = 3;
|
|
39
66
|
var MAXIMUM_PLAYBACK_DURATION_SECONDS = 10 * 60;
|
|
@@ -42,7 +69,465 @@ var MAXIMUM_PLAYBACK_DURATION_SECONDS = 10 * 60;
|
|
|
42
69
|
var TTS_PROVIDERS = ["11labs", "gemini"];
|
|
43
70
|
var DEFAULT_TTS_PROVIDER = TTS_PROVIDERS[0];
|
|
44
71
|
|
|
72
|
+
// ../ts-shared/src/llm-providers/catalog.ts
|
|
73
|
+
var record = function(value) {
|
|
74
|
+
return value && typeof value === "object" && !Array.isArray(value) ? value : {};
|
|
75
|
+
};
|
|
76
|
+
var rate = function(value) {
|
|
77
|
+
if (typeof value !== "number" && typeof value !== "string") {
|
|
78
|
+
return null;
|
|
79
|
+
}
|
|
80
|
+
if (value === "") {
|
|
81
|
+
return null;
|
|
82
|
+
}
|
|
83
|
+
const result = Number(value);
|
|
84
|
+
return Number.isFinite(result) && result >= 0 ? result : null;
|
|
85
|
+
};
|
|
86
|
+
var catalogRows = function(payload) {
|
|
87
|
+
if (!Array.isArray(payload)) {
|
|
88
|
+
throw new Error("Provider returned an invalid model catalog");
|
|
89
|
+
}
|
|
90
|
+
return payload.map(record);
|
|
91
|
+
};
|
|
92
|
+
var parseCompatibleModels = function(payload) {
|
|
93
|
+
return catalogRows(record(payload).data).flatMap((row) => {
|
|
94
|
+
const pricing = record(row.pricing);
|
|
95
|
+
const input = rate(pricing.prompt);
|
|
96
|
+
const output = rate(pricing.completion);
|
|
97
|
+
const cached = rate(pricing.input_cache_read);
|
|
98
|
+
const write = rate(pricing.input_cache_write);
|
|
99
|
+
const modalities = record(row.architecture).output_modalities;
|
|
100
|
+
const parameters = row.supported_parameters;
|
|
101
|
+
if (typeof row.id !== "string" || !row.id || row.deprecated || input === null || output === null || Array.isArray(modalities) && (modalities.length !== 1 || modalities[0] !== "text") || Array.isArray(pricing.overrides) && pricing.overrides.length > 0 || Array.isArray(parameters) && !parameters.includes("structured_outputs") && !parameters.includes("response_format") || (rate(pricing.request) ?? 0) > 0) {
|
|
102
|
+
return [];
|
|
103
|
+
}
|
|
104
|
+
return [
|
|
105
|
+
{
|
|
106
|
+
id: row.id,
|
|
107
|
+
name: typeof row.name === "string" ? row.name : row.id,
|
|
108
|
+
inputUsd: input * 1e6,
|
|
109
|
+
outputUsd: output * 1e6,
|
|
110
|
+
cachedInputUsd: (cached ?? input) * 1e6,
|
|
111
|
+
...write !== null ? { cacheWriteUsd: write * 1e6 } : {},
|
|
112
|
+
contextLength: rate(row.max_tokens ?? row.context_length)
|
|
113
|
+
}
|
|
114
|
+
];
|
|
115
|
+
});
|
|
116
|
+
};
|
|
117
|
+
|
|
118
|
+
// ../ts-shared/src/llm-providers/usage.ts
|
|
119
|
+
var validUsd = function(value) {
|
|
120
|
+
return typeof value === "number" && Number.isFinite(value) && value >= 0;
|
|
121
|
+
};
|
|
122
|
+
var validateUsageCounts = function(usage) {
|
|
123
|
+
const counts = [
|
|
124
|
+
usage.prompt_tokens,
|
|
125
|
+
usage.completion_tokens,
|
|
126
|
+
usage.total_tokens,
|
|
127
|
+
// Providers can explicitly return null for unavailable cache counters.
|
|
128
|
+
usage.prompt_tokens_details?.cached_tokens ?? 0,
|
|
129
|
+
usage.prompt_tokens_details?.cache_write_tokens ?? 0
|
|
130
|
+
];
|
|
131
|
+
if (counts.some(
|
|
132
|
+
(value) => value !== void 0 && (!Number.isSafeInteger(value) || value < 0)
|
|
133
|
+
)) {
|
|
134
|
+
throw new Error("Provider returned invalid token usage");
|
|
135
|
+
}
|
|
136
|
+
const cached = usage.prompt_tokens_details?.cached_tokens ?? 0;
|
|
137
|
+
const written = usage.prompt_tokens_details?.cache_write_tokens ?? 0;
|
|
138
|
+
if (usage.prompt_tokens !== void 0 && cached + written > usage.prompt_tokens) {
|
|
139
|
+
throw new Error("Provider returned invalid cache usage");
|
|
140
|
+
}
|
|
141
|
+
};
|
|
142
|
+
var quoteTokenUsage = function(price, usage) {
|
|
143
|
+
validateUsageCounts(usage);
|
|
144
|
+
const input = usage.prompt_tokens;
|
|
145
|
+
const output = usage.completion_tokens;
|
|
146
|
+
if (input === void 0 || output === void 0) {
|
|
147
|
+
throw new Error("Provider did not report billable token usage");
|
|
148
|
+
}
|
|
149
|
+
const cached = usage.prompt_tokens_details?.cached_tokens ?? 0;
|
|
150
|
+
const written = usage.prompt_tokens_details?.cache_write_tokens ?? 0;
|
|
151
|
+
const writeRate = price.cacheWriteUsd ?? price.inputUsd;
|
|
152
|
+
if (![price.inputUsd, price.outputUsd, price.cachedInputUsd, writeRate].every(
|
|
153
|
+
validUsd
|
|
154
|
+
)) {
|
|
155
|
+
throw new Error("Provider returned invalid token prices");
|
|
156
|
+
}
|
|
157
|
+
return {
|
|
158
|
+
usd: ((input - cached - written) * price.inputUsd + cached * price.cachedInputUsd + written * writeRate + output * price.outputUsd) / 1e6,
|
|
159
|
+
source: "token-rates"
|
|
160
|
+
};
|
|
161
|
+
};
|
|
162
|
+
|
|
163
|
+
// ../ts-shared/src/llm-providers/baseten.ts
|
|
164
|
+
var baseten = {
|
|
165
|
+
name: "Baseten",
|
|
166
|
+
baseUrl: "https://inference.baseten.co/v1",
|
|
167
|
+
modelsUrl: "https://inference.baseten.co/v1/models",
|
|
168
|
+
billingMode: "token-rates",
|
|
169
|
+
parseModels: parseCompatibleModels,
|
|
170
|
+
requestOptions: () => ({}),
|
|
171
|
+
quoteUsage: quoteTokenUsage
|
|
172
|
+
};
|
|
173
|
+
|
|
174
|
+
// ../ts-shared/src/llm-providers/deepinfra.ts
|
|
175
|
+
var deepinfra = {
|
|
176
|
+
name: "DeepInfra",
|
|
177
|
+
keyCheckUrl: "https://api.deepinfra.com/v1/me",
|
|
178
|
+
baseUrl: "https://api.deepinfra.com/v1/openai",
|
|
179
|
+
modelsUrl: "https://api.deepinfra.com/models/list",
|
|
180
|
+
billingMode: "token-rates",
|
|
181
|
+
parseModels: (payload) => catalogRows(payload).flatMap((row) => {
|
|
182
|
+
const tags = Array.isArray(row.tags) ? row.tags : [];
|
|
183
|
+
const pricing = record(row.pricing);
|
|
184
|
+
const input = rate(pricing.cents_per_input_token);
|
|
185
|
+
const output = rate(pricing.cents_per_output_token);
|
|
186
|
+
if (typeof row.model_name !== "string" || !row.model_name || row.deprecated || row.type !== "text-generation" || !tags.includes("openai") || !tags.includes("structured-output") || pricing.type !== "tokens" || input === null || output === null) {
|
|
187
|
+
return [];
|
|
188
|
+
}
|
|
189
|
+
return [
|
|
190
|
+
{
|
|
191
|
+
id: row.model_name,
|
|
192
|
+
name: typeof row.name === "string" ? row.name : row.model_name,
|
|
193
|
+
inputUsd: input / 100 * 1e6,
|
|
194
|
+
outputUsd: output / 100 * 1e6,
|
|
195
|
+
cachedInputUsd: input / 100 * (rate(pricing.rate_per_input_token_cached) ?? 1) * 1e6,
|
|
196
|
+
contextLength: rate(row.max_tokens ?? row.context_length),
|
|
197
|
+
canDisableReasoning: tags.includes("can-disable-reasoning")
|
|
198
|
+
}
|
|
199
|
+
];
|
|
200
|
+
}),
|
|
201
|
+
requestOptions: ({ model, conversationId, legacyPriority }) => ({
|
|
202
|
+
...conversationId ? { prompt_cache_key: conversationId } : {},
|
|
203
|
+
...!model || model.canDisableReasoning ? { reasoning_effort: "none" } : {},
|
|
204
|
+
...legacyPriority ? { service_tier: "priority" } : {}
|
|
205
|
+
}),
|
|
206
|
+
quoteUsage: quoteTokenUsage
|
|
207
|
+
};
|
|
208
|
+
|
|
209
|
+
// ../ts-shared/src/llm-providers/gemini.ts
|
|
210
|
+
var GEMINI_MODELS = {
|
|
211
|
+
"gemini-3.8-flash": {
|
|
212
|
+
id: "gemini-3.8-flash",
|
|
213
|
+
name: "Gemini 3.8 Flash",
|
|
214
|
+
inputUsd: 0.75,
|
|
215
|
+
outputUsd: 3.75,
|
|
216
|
+
cachedInputUsd: 0.075,
|
|
217
|
+
contextLength: 1048576
|
|
218
|
+
}
|
|
219
|
+
};
|
|
220
|
+
var gemini = {
|
|
221
|
+
name: "Google Gemini",
|
|
222
|
+
baseUrl: "https://generativelanguage.googleapis.com/v1beta/openai",
|
|
223
|
+
modelsUrl: "https://generativelanguage.googleapis.com/v1beta/openai/models",
|
|
224
|
+
billingMode: "token-rates",
|
|
225
|
+
parseModels: (payload) => {
|
|
226
|
+
const seen = /* @__PURE__ */ new Set();
|
|
227
|
+
return catalogRows(record(payload).data).flatMap((row) => {
|
|
228
|
+
const rawId = typeof row.id === "string" ? row.id.trim() : "";
|
|
229
|
+
const id = rawId.replace(/^models\//, "");
|
|
230
|
+
const model = Object.hasOwn(GEMINI_MODELS, id) ? GEMINI_MODELS[id] : void 0;
|
|
231
|
+
if (!model || row.deprecated || seen.has(id)) {
|
|
232
|
+
return [];
|
|
233
|
+
}
|
|
234
|
+
seen.add(id);
|
|
235
|
+
return [{ ...model }];
|
|
236
|
+
});
|
|
237
|
+
},
|
|
238
|
+
requestOptions: () => ({ reasoning_effort: "low" }),
|
|
239
|
+
quoteUsage: (price, usage) => {
|
|
240
|
+
validateUsageCounts(usage);
|
|
241
|
+
if (usage.prompt_tokens !== void 0 && usage.completion_tokens !== void 0 && usage.total_tokens !== void 0 && usage.total_tokens < usage.prompt_tokens + usage.completion_tokens) {
|
|
242
|
+
throw new Error("Provider returned invalid token usage");
|
|
243
|
+
}
|
|
244
|
+
const outputTokens = usage.total_tokens !== void 0 && usage.prompt_tokens !== void 0 ? usage.total_tokens - usage.prompt_tokens : usage.completion_tokens;
|
|
245
|
+
return quoteTokenUsage(price, {
|
|
246
|
+
...usage,
|
|
247
|
+
completion_tokens: outputTokens
|
|
248
|
+
});
|
|
249
|
+
}
|
|
250
|
+
};
|
|
251
|
+
|
|
252
|
+
// ../ts-shared/src/llm-providers/openrouter.ts
|
|
253
|
+
var openrouter = {
|
|
254
|
+
name: "OpenRouter",
|
|
255
|
+
keyCheckUrl: "https://openrouter.ai/api/v1/key",
|
|
256
|
+
baseUrl: "https://openrouter.ai/api/v1",
|
|
257
|
+
modelsUrl: "https://openrouter.ai/api/v1/models",
|
|
258
|
+
billingMode: "reported-cost",
|
|
259
|
+
parseModels: parseCompatibleModels,
|
|
260
|
+
requestOptions: ({ conversationId }) => conversationId ? { session_id: conversationId } : {},
|
|
261
|
+
quoteUsage: (_price, usage) => {
|
|
262
|
+
validateUsageCounts(usage);
|
|
263
|
+
if (!validUsd(usage.cost)) {
|
|
264
|
+
throw new Error("OpenRouter did not report a valid account cost");
|
|
265
|
+
}
|
|
266
|
+
return { usd: usage.cost, source: "provider-reported" };
|
|
267
|
+
},
|
|
268
|
+
needsUsageRecovery: (usage) => usage.cost === void 0 || usage.cost === null,
|
|
269
|
+
recoverUsage: async (usage, apiKey) => {
|
|
270
|
+
const id = usage.billing?.generationId;
|
|
271
|
+
if (!id) {
|
|
272
|
+
throw new Error("OpenRouter generation ID is missing");
|
|
273
|
+
}
|
|
274
|
+
const url = new URL("https://openrouter.ai/api/v1/generation");
|
|
275
|
+
url.searchParams.set("id", id);
|
|
276
|
+
const response = await fetch(url, {
|
|
277
|
+
headers: { Authorization: `Bearer ${apiKey}` },
|
|
278
|
+
signal: AbortSignal.timeout(15e3),
|
|
279
|
+
redirect: "error"
|
|
280
|
+
});
|
|
281
|
+
if (!response.ok) {
|
|
282
|
+
throw new Error(`OpenRouter usage lookup failed (${response.status})`);
|
|
283
|
+
}
|
|
284
|
+
const data = record(record(await response.json()).data);
|
|
285
|
+
const finished = typeof data.finish_reason === "string" && data.finish_reason.length > 0 || data.cancelled === true;
|
|
286
|
+
if (data.id !== id || !validUsd(data.total_cost) || !finished) {
|
|
287
|
+
throw new Error("OpenRouter generation cost is unavailable");
|
|
288
|
+
}
|
|
289
|
+
return {
|
|
290
|
+
...usage,
|
|
291
|
+
cost: data.total_cost,
|
|
292
|
+
prompt_tokens_details: {
|
|
293
|
+
...usage.prompt_tokens_details,
|
|
294
|
+
...Number.isSafeInteger(data.native_tokens_cached) && Number(data.native_tokens_cached) >= 0 ? { cached_tokens: Number(data.native_tokens_cached) } : {}
|
|
295
|
+
},
|
|
296
|
+
...Number.isSafeInteger(data.native_tokens_prompt) && Number(data.native_tokens_prompt) >= 0 ? { prompt_tokens: Number(data.native_tokens_prompt) } : {},
|
|
297
|
+
...Number.isSafeInteger(data.native_tokens_completion) && Number(data.native_tokens_completion) >= 0 ? { completion_tokens: Number(data.native_tokens_completion) } : {}
|
|
298
|
+
};
|
|
299
|
+
}
|
|
300
|
+
};
|
|
301
|
+
|
|
302
|
+
// ../ts-shared/src/llm-providers.ts
|
|
303
|
+
var LLM_PROVIDERS = {
|
|
304
|
+
deepinfra,
|
|
305
|
+
baseten,
|
|
306
|
+
openrouter,
|
|
307
|
+
gemini
|
|
308
|
+
};
|
|
309
|
+
|
|
310
|
+
// ../ts-shared/src/ai-providers.ts
|
|
311
|
+
var staticModel = function(id, name, capabilities, priceSummary) {
|
|
312
|
+
return Object.freeze({
|
|
313
|
+
id,
|
|
314
|
+
name,
|
|
315
|
+
capabilities: [...capabilities],
|
|
316
|
+
...priceSummary ? { priceSummary } : {}
|
|
317
|
+
});
|
|
318
|
+
};
|
|
319
|
+
var AI_PROVIDERS = Object.freeze({
|
|
320
|
+
deepinfra: {
|
|
321
|
+
name: "DeepInfra",
|
|
322
|
+
capabilities: ["llm"],
|
|
323
|
+
defaultBaseUrl: LLM_PROVIDERS.deepinfra.baseUrl,
|
|
324
|
+
supportsCreatorChat: true,
|
|
325
|
+
staticModels: []
|
|
326
|
+
},
|
|
327
|
+
openrouter: {
|
|
328
|
+
name: "OpenRouter",
|
|
329
|
+
capabilities: ["llm"],
|
|
330
|
+
defaultBaseUrl: LLM_PROVIDERS.openrouter.baseUrl,
|
|
331
|
+
supportsCreatorChat: true,
|
|
332
|
+
staticModels: []
|
|
333
|
+
},
|
|
334
|
+
baseten: {
|
|
335
|
+
name: "Baseten",
|
|
336
|
+
capabilities: ["llm"],
|
|
337
|
+
defaultBaseUrl: LLM_PROVIDERS.baseten.baseUrl,
|
|
338
|
+
supportsCreatorChat: true,
|
|
339
|
+
staticModels: []
|
|
340
|
+
},
|
|
341
|
+
gemini: {
|
|
342
|
+
name: "Google Gemini",
|
|
343
|
+
capabilities: ["llm", "tts", "stt", "vision"],
|
|
344
|
+
defaultBaseUrl: "https://generativelanguage.googleapis.com/v1beta",
|
|
345
|
+
supportsCreatorChat: true,
|
|
346
|
+
staticModels: [
|
|
347
|
+
staticModel("gemini-3.8-flash-tts", "Gemini 3.8 Flash TTS", ["tts"]),
|
|
348
|
+
staticModel(
|
|
349
|
+
"gemini-3.8-flash-lite-tts",
|
|
350
|
+
"Gemini 3.8 Flash Lite TTS",
|
|
351
|
+
["tts"]
|
|
352
|
+
),
|
|
353
|
+
staticModel(
|
|
354
|
+
"gemini-3.5-transcribe-live",
|
|
355
|
+
"Gemini 3.5 Transcribe Live",
|
|
356
|
+
["stt"],
|
|
357
|
+
"$3.50 input / $21 output per 1M tokens"
|
|
358
|
+
),
|
|
359
|
+
staticModel(
|
|
360
|
+
"gemini-3.5-transcribe",
|
|
361
|
+
"Gemini 3.5 Transcribe",
|
|
362
|
+
["stt"],
|
|
363
|
+
"$2 input / $12 output per 1M tokens"
|
|
364
|
+
),
|
|
365
|
+
staticModel(
|
|
366
|
+
"gemini-3.5-flash-lite",
|
|
367
|
+
"Gemini 3.5 Flash Lite",
|
|
368
|
+
["vision"],
|
|
369
|
+
"$0.30 input / $2.50 output per 1M tokens"
|
|
370
|
+
)
|
|
371
|
+
]
|
|
372
|
+
},
|
|
373
|
+
openai: {
|
|
374
|
+
name: "OpenAI",
|
|
375
|
+
capabilities: ["llm", "stt", "vision", "image"],
|
|
376
|
+
defaultBaseUrl: "https://api.openai.com/v1",
|
|
377
|
+
supportsCreatorChat: false,
|
|
378
|
+
forbidOpenRouterBaseUrl: true,
|
|
379
|
+
staticModels: [
|
|
380
|
+
staticModel(
|
|
381
|
+
"gpt-6.1-sol",
|
|
382
|
+
"GPT-6.1 Sol",
|
|
383
|
+
["llm"],
|
|
384
|
+
"$2 input / $10 output per 1M tokens"
|
|
385
|
+
),
|
|
386
|
+
staticModel(
|
|
387
|
+
"gpt-6-luna",
|
|
388
|
+
"GPT-6 Luna",
|
|
389
|
+
["llm", "vision"],
|
|
390
|
+
"$0.10 input / $0.50 output per 1M tokens"
|
|
391
|
+
),
|
|
392
|
+
staticModel(
|
|
393
|
+
"gpt-image-2.5-flare",
|
|
394
|
+
"GPT Image 2.5 Flare",
|
|
395
|
+
["image"],
|
|
396
|
+
"$5 text / $8 image input / $30 output per 1M tokens"
|
|
397
|
+
),
|
|
398
|
+
staticModel(
|
|
399
|
+
"gpt-image-2.5-sunburst",
|
|
400
|
+
"GPT Image 2.5 Sunburst",
|
|
401
|
+
["image"],
|
|
402
|
+
"$5 text / $8 image input / $30 output per 1M tokens"
|
|
403
|
+
),
|
|
404
|
+
staticModel(
|
|
405
|
+
"gpt-transcribe",
|
|
406
|
+
"GPT Transcribe",
|
|
407
|
+
["stt"],
|
|
408
|
+
"$0.0045/min"
|
|
409
|
+
),
|
|
410
|
+
staticModel(
|
|
411
|
+
"gpt-live-transcribe",
|
|
412
|
+
"GPT Live Transcribe",
|
|
413
|
+
["stt"],
|
|
414
|
+
"$0.017/min"
|
|
415
|
+
)
|
|
416
|
+
]
|
|
417
|
+
},
|
|
418
|
+
anthropic: {
|
|
419
|
+
name: "Anthropic",
|
|
420
|
+
capabilities: ["llm"],
|
|
421
|
+
defaultBaseUrl: "https://api.anthropic.com",
|
|
422
|
+
supportsCreatorChat: false,
|
|
423
|
+
staticModels: [
|
|
424
|
+
staticModel(
|
|
425
|
+
"claude-opus-5-5",
|
|
426
|
+
"Claude Opus 5.5",
|
|
427
|
+
["llm"],
|
|
428
|
+
"$4 input / $20 output per 1M tokens"
|
|
429
|
+
),
|
|
430
|
+
staticModel(
|
|
431
|
+
"claude-sonnet-5-5",
|
|
432
|
+
"Claude Sonnet 5.5",
|
|
433
|
+
["llm"],
|
|
434
|
+
"$2 input / $10 output per 1M tokens"
|
|
435
|
+
),
|
|
436
|
+
staticModel(
|
|
437
|
+
"claude-haiku-5-5",
|
|
438
|
+
"Claude Haiku 5.5",
|
|
439
|
+
["llm"],
|
|
440
|
+
"$0.10 input / $0.50 output per 1M tokens"
|
|
441
|
+
)
|
|
442
|
+
]
|
|
443
|
+
},
|
|
444
|
+
elevenlabs: {
|
|
445
|
+
name: "ElevenLabs",
|
|
446
|
+
capabilities: ["tts", "stt"],
|
|
447
|
+
defaultBaseUrl: "https://api.elevenlabs.io/v1",
|
|
448
|
+
supportsCreatorChat: false,
|
|
449
|
+
staticModels: [
|
|
450
|
+
staticModel(
|
|
451
|
+
"eleven_v3_conversational",
|
|
452
|
+
"Eleven v3 Conversational",
|
|
453
|
+
["tts"]
|
|
454
|
+
),
|
|
455
|
+
staticModel("eleven_v3", "Eleven v3", ["tts"]),
|
|
456
|
+
staticModel(
|
|
457
|
+
"scribe_v2_realtime",
|
|
458
|
+
"Scribe v2 Realtime",
|
|
459
|
+
["stt"],
|
|
460
|
+
"$0.39/hr"
|
|
461
|
+
),
|
|
462
|
+
staticModel("scribe_v2", "Scribe v2", ["stt"], "$0.22/hr")
|
|
463
|
+
]
|
|
464
|
+
},
|
|
465
|
+
tripo: {
|
|
466
|
+
name: "Tripo",
|
|
467
|
+
capabilities: ["3d"],
|
|
468
|
+
defaultBaseUrl: "https://openapi.tripo3d.ai/v3",
|
|
469
|
+
supportsCreatorChat: false,
|
|
470
|
+
staticModels: [
|
|
471
|
+
staticModel(
|
|
472
|
+
"P2-20260801",
|
|
473
|
+
"Tripo P2",
|
|
474
|
+
["3d"],
|
|
475
|
+
"$0.01/credit (P2 4K)"
|
|
476
|
+
),
|
|
477
|
+
staticModel(
|
|
478
|
+
"v1.0-20240301",
|
|
479
|
+
"Tripo Rig v1.0",
|
|
480
|
+
["3d"],
|
|
481
|
+
"$0.01/credit (25 credits/rig)"
|
|
482
|
+
)
|
|
483
|
+
]
|
|
484
|
+
},
|
|
485
|
+
meshy: {
|
|
486
|
+
name: "Meshy",
|
|
487
|
+
capabilities: ["3d"],
|
|
488
|
+
defaultBaseUrl: "https://api.meshy.ai",
|
|
489
|
+
supportsCreatorChat: false,
|
|
490
|
+
staticModels: [
|
|
491
|
+
staticModel(
|
|
492
|
+
"meshy-7.1",
|
|
493
|
+
"Meshy 7.1",
|
|
494
|
+
["3d"],
|
|
495
|
+
"$0.02/credit (20-30 credits/model)"
|
|
496
|
+
),
|
|
497
|
+
staticModel(
|
|
498
|
+
"meshy-6",
|
|
499
|
+
"Meshy 6",
|
|
500
|
+
["3d"],
|
|
501
|
+
"$0.02/credit (20-30 credits/model)"
|
|
502
|
+
)
|
|
503
|
+
]
|
|
504
|
+
},
|
|
505
|
+
unirig: {
|
|
506
|
+
name: "UniRig",
|
|
507
|
+
capabilities: ["3d"],
|
|
508
|
+
defaultBaseUrl: "https://modal.run",
|
|
509
|
+
supportsCreatorChat: false,
|
|
510
|
+
staticModels: [
|
|
511
|
+
staticModel(
|
|
512
|
+
"unirig-v1",
|
|
513
|
+
"UniRig (Modal GPU)",
|
|
514
|
+
["3d"],
|
|
515
|
+
"Modal A10 GPU billed per second"
|
|
516
|
+
)
|
|
517
|
+
]
|
|
518
|
+
}
|
|
519
|
+
});
|
|
520
|
+
|
|
45
521
|
// ../ts-shared/src/ai-settings.ts
|
|
522
|
+
var AI_VENDOR_NAMES = {
|
|
523
|
+
openai: AI_PROVIDERS.openai.name,
|
|
524
|
+
tripo: AI_PROVIDERS.tripo.name,
|
|
525
|
+
meshy: AI_PROVIDERS.meshy.name,
|
|
526
|
+
unirig: AI_PROVIDERS.unirig.name,
|
|
527
|
+
elevenlabs: AI_PROVIDERS.elevenlabs.name,
|
|
528
|
+
gemini: AI_PROVIDERS.gemini.name,
|
|
529
|
+
anthropic: AI_PROVIDERS.anthropic.name
|
|
530
|
+
};
|
|
46
531
|
var GPT_IMAGE_MODELS = [
|
|
47
532
|
"gpt-image-2.5-flare",
|
|
48
533
|
"gpt-image-2.5-sunburst"
|
|
@@ -50,76 +535,124 @@ var GPT_IMAGE_MODELS = [
|
|
|
50
535
|
var AI_ROLES = {
|
|
51
536
|
character_draft: {
|
|
52
537
|
name: "Character draft",
|
|
53
|
-
|
|
54
|
-
|
|
538
|
+
capability: "llm",
|
|
539
|
+
chatLlm: true,
|
|
55
540
|
options: ["temperature"]
|
|
56
541
|
},
|
|
57
542
|
motion_planner: {
|
|
58
543
|
name: "Motion planner",
|
|
59
|
-
|
|
60
|
-
|
|
544
|
+
capability: "llm",
|
|
545
|
+
chatLlm: true,
|
|
61
546
|
options: []
|
|
62
547
|
},
|
|
63
|
-
// Not charged.
|
|
64
548
|
prompt_check: {
|
|
65
549
|
name: "Prompt check",
|
|
66
|
-
|
|
67
|
-
|
|
550
|
+
capability: "llm",
|
|
551
|
+
providers: ["openai"],
|
|
552
|
+
allowedModels: ["gpt-6-luna"],
|
|
68
553
|
options: []
|
|
69
554
|
},
|
|
70
|
-
// Priced by OPENAI_IMAGE_PRICES of the server (flare and sunburst alike).
|
|
71
555
|
design_sheet_image: {
|
|
72
556
|
name: "Design sheet image",
|
|
73
|
-
|
|
74
|
-
|
|
557
|
+
capability: "image",
|
|
558
|
+
providers: ["openai"],
|
|
559
|
+
allowedModels: GPT_IMAGE_MODELS,
|
|
75
560
|
options: []
|
|
76
561
|
},
|
|
77
|
-
// Not charged per call.
|
|
78
562
|
tts: {
|
|
79
563
|
name: "ElevenLabs text to speech",
|
|
80
|
-
|
|
81
|
-
|
|
564
|
+
capability: "tts",
|
|
565
|
+
providers: ["elevenlabs"],
|
|
566
|
+
allowedModels: ["eleven_v3_conversational", "eleven_v3"],
|
|
82
567
|
options: []
|
|
83
568
|
},
|
|
84
|
-
// Not charged per call.
|
|
85
569
|
gemini_tts: {
|
|
86
570
|
name: "Gemini text to speech",
|
|
87
|
-
|
|
88
|
-
|
|
571
|
+
capability: "tts",
|
|
572
|
+
providers: ["gemini"],
|
|
573
|
+
allowedModels: ["gemini-3.8-flash-tts", "gemini-3.8-flash-lite-tts"],
|
|
574
|
+
options: []
|
|
575
|
+
},
|
|
576
|
+
stt_openai: {
|
|
577
|
+
name: "OpenAI speech recognition",
|
|
578
|
+
capability: "stt",
|
|
579
|
+
providers: ["openai"],
|
|
580
|
+
allowedModels: ["gpt-transcribe", "gpt-live-transcribe"],
|
|
581
|
+
options: []
|
|
582
|
+
},
|
|
583
|
+
stt_gemini: {
|
|
584
|
+
name: "Gemini speech recognition",
|
|
585
|
+
capability: "stt",
|
|
586
|
+
providers: ["gemini"],
|
|
587
|
+
allowedModels: ["gemini-3.5-transcribe-live"],
|
|
588
|
+
options: []
|
|
589
|
+
},
|
|
590
|
+
stt_elevenlabs: {
|
|
591
|
+
name: "ElevenLabs speech recognition",
|
|
592
|
+
capability: "stt",
|
|
593
|
+
providers: ["elevenlabs"],
|
|
594
|
+
allowedModels: ["scribe_v2_realtime"],
|
|
89
595
|
options: []
|
|
90
596
|
},
|
|
91
|
-
// Sight (docs/sight.md): one role for each provider that an owner can
|
|
92
|
-
// choose. Priced by VISION_PRICES of the server.
|
|
93
597
|
vision_gemini: {
|
|
94
598
|
name: "Gemini sight",
|
|
95
|
-
|
|
96
|
-
|
|
599
|
+
capability: "vision",
|
|
600
|
+
providers: ["gemini"],
|
|
601
|
+
allowedModels: ["gemini-3.5-flash-lite"],
|
|
97
602
|
options: []
|
|
98
603
|
},
|
|
99
604
|
vision_openai: {
|
|
100
605
|
name: "OpenAI sight",
|
|
101
|
-
|
|
102
|
-
|
|
606
|
+
capability: "vision",
|
|
607
|
+
providers: ["openai"],
|
|
608
|
+
allowedModels: ["gpt-6-luna"],
|
|
103
609
|
options: []
|
|
104
610
|
},
|
|
105
|
-
// Priced by OPENAI_PRICES.text of the generator.
|
|
106
611
|
generator_agent: {
|
|
107
612
|
name: "Generator agent",
|
|
108
|
-
|
|
109
|
-
|
|
613
|
+
capability: "llm",
|
|
614
|
+
providers: ["openai", "anthropic"],
|
|
615
|
+
allowedModels: [
|
|
616
|
+
"gpt-6.1-sol",
|
|
617
|
+
"claude-opus-5-5",
|
|
618
|
+
"claude-sonnet-5-5",
|
|
619
|
+
"claude-haiku-5-5"
|
|
620
|
+
],
|
|
110
621
|
options: ["reasoningEffort"]
|
|
111
622
|
},
|
|
112
623
|
generator_reading: {
|
|
113
624
|
name: "Generator face reading",
|
|
114
|
-
|
|
115
|
-
|
|
625
|
+
capability: "llm",
|
|
626
|
+
providers: ["openai"],
|
|
627
|
+
allowedModels: ["gpt-6.1-sol"],
|
|
116
628
|
options: ["reasoningEffort"]
|
|
117
629
|
},
|
|
118
|
-
// Priced by OPENAI_PRICES of the generator (flare and sunburst alike).
|
|
119
630
|
generator_image: {
|
|
120
631
|
name: "Generator images",
|
|
121
|
-
|
|
122
|
-
|
|
632
|
+
capability: "image",
|
|
633
|
+
providers: ["openai"],
|
|
634
|
+
allowedModels: GPT_IMAGE_MODELS,
|
|
635
|
+
options: []
|
|
636
|
+
},
|
|
637
|
+
generator_3d: {
|
|
638
|
+
name: "Generator 3D head",
|
|
639
|
+
capability: "3d",
|
|
640
|
+
providers: ["tripo"],
|
|
641
|
+
allowedModels: ["P2-20260801"],
|
|
642
|
+
options: []
|
|
643
|
+
},
|
|
644
|
+
generator_3d_body: {
|
|
645
|
+
name: "Generator 3D body",
|
|
646
|
+
capability: "3d",
|
|
647
|
+
providers: ["tripo", "meshy"],
|
|
648
|
+
allowedModels: ["P2-20260801", "meshy-7.1", "meshy-6"],
|
|
649
|
+
options: []
|
|
650
|
+
},
|
|
651
|
+
generator_rig: {
|
|
652
|
+
name: "Generator 3D rig",
|
|
653
|
+
capability: "3d",
|
|
654
|
+
providers: ["tripo", "unirig"],
|
|
655
|
+
allowedModels: ["v1.0-20240301", "P2-20260801", "unirig-v1"],
|
|
123
656
|
options: []
|
|
124
657
|
}
|
|
125
658
|
};
|
|
@@ -305,7 +838,8 @@ var productUsageEvents = [
|
|
|
305
838
|
"playback_blocked",
|
|
306
839
|
"playback_start_pressed",
|
|
307
840
|
"reply_options_shown",
|
|
308
|
-
"page_left"
|
|
841
|
+
"page_left",
|
|
842
|
+
"creation_mode_chosen"
|
|
309
843
|
];
|
|
310
844
|
|
|
311
845
|
// ../ts-shared/src/analytics.ts
|
|
@@ -2687,7 +3221,7 @@ var PERFORMANCE_KINDS = {
|
|
|
2687
3221
|
cachedOpening: 8
|
|
2688
3222
|
};
|
|
2689
3223
|
var PERFORMANCE_MAJOR_VERSION = 1;
|
|
2690
|
-
var PERFORMANCE_MINOR_VERSION =
|
|
3224
|
+
var PERFORMANCE_MINOR_VERSION = 2;
|
|
2691
3225
|
var HEADER_SIZE = 24;
|
|
2692
3226
|
var MAGIC = 1346588750;
|
|
2693
3227
|
var crcTable = Uint32Array.from({ length: 256 }, (_, index) => {
|
|
@@ -2839,6 +3373,68 @@ var decodePerformance = function(input, expectedKind) {
|
|
|
2839
3373
|
return wireValue(value);
|
|
2840
3374
|
};
|
|
2841
3375
|
|
|
3376
|
+
// ../ts-shared/src/player-memory.ts
|
|
3377
|
+
var MEMORY_IDENTIFIER_MAX_LENGTH = 128;
|
|
3378
|
+
var MEMORY_IMPORT_MAX_BYTES = 512 * 1024;
|
|
3379
|
+
var normalizeMemoryIdentifier = function(value) {
|
|
3380
|
+
if (typeof value !== "string") {
|
|
3381
|
+
return null;
|
|
3382
|
+
}
|
|
3383
|
+
const cleaned = value.normalize("NFKC").replace(/[\u0000-\u001f\u007f]/gu, "").trim();
|
|
3384
|
+
if (!cleaned) {
|
|
3385
|
+
return null;
|
|
3386
|
+
}
|
|
3387
|
+
return cleaned.slice(0, MEMORY_IDENTIFIER_MAX_LENGTH).trim() || null;
|
|
3388
|
+
};
|
|
3389
|
+
|
|
3390
|
+
// ../ts-shared/src/message-time.ts
|
|
3391
|
+
var MESSAGE_TIME_ZONE_MAX_LENGTH = 64;
|
|
3392
|
+
var TIME_ZONE_PATTERN = /^[A-Za-z0-9_+-]+(?:\/[A-Za-z0-9_+-]+)*$/u;
|
|
3393
|
+
var OFFSET_PATTERN = /^[+-](?:[01]\d|2[0-3]):[0-5]\d$/u;
|
|
3394
|
+
var normalizeMessageTimeZone = function(value) {
|
|
3395
|
+
if (typeof value !== "string") {
|
|
3396
|
+
return null;
|
|
3397
|
+
}
|
|
3398
|
+
const trimmed = value.trim();
|
|
3399
|
+
if (!trimmed || trimmed.length > MESSAGE_TIME_ZONE_MAX_LENGTH) {
|
|
3400
|
+
return null;
|
|
3401
|
+
}
|
|
3402
|
+
if (trimmed === "Z" || trimmed === "z" || trimmed === "+00:00" || trimmed === "-00:00") {
|
|
3403
|
+
return "UTC";
|
|
3404
|
+
}
|
|
3405
|
+
if (!OFFSET_PATTERN.test(trimmed) && !TIME_ZONE_PATTERN.test(trimmed)) {
|
|
3406
|
+
return null;
|
|
3407
|
+
}
|
|
3408
|
+
try {
|
|
3409
|
+
new Intl.DateTimeFormat("en-US", { timeZone: trimmed }).format(/* @__PURE__ */ new Date(0));
|
|
3410
|
+
return trimmed;
|
|
3411
|
+
} catch {
|
|
3412
|
+
return null;
|
|
3413
|
+
}
|
|
3414
|
+
};
|
|
3415
|
+
var formatUtcOffsetMinutes = function(offsetMinutesEastOfUtc) {
|
|
3416
|
+
if (!Number.isFinite(offsetMinutesEastOfUtc) || offsetMinutesEastOfUtc === 0) {
|
|
3417
|
+
return "UTC";
|
|
3418
|
+
}
|
|
3419
|
+
const total = Math.trunc(offsetMinutesEastOfUtc);
|
|
3420
|
+
const sign = total >= 0 ? "+" : "-";
|
|
3421
|
+
const abs = Math.abs(total);
|
|
3422
|
+
const hours = String(Math.floor(abs / 60)).padStart(2, "0");
|
|
3423
|
+
const minutes = String(abs % 60).padStart(2, "0");
|
|
3424
|
+
return `${sign}${hours}:${minutes}`;
|
|
3425
|
+
};
|
|
3426
|
+
var detectClientTimeZone = function(now = /* @__PURE__ */ new Date()) {
|
|
3427
|
+
try {
|
|
3428
|
+
const resolved = Intl.DateTimeFormat().resolvedOptions().timeZone;
|
|
3429
|
+
const normalized = normalizeMessageTimeZone(resolved);
|
|
3430
|
+
if (normalized) {
|
|
3431
|
+
return normalized;
|
|
3432
|
+
}
|
|
3433
|
+
} catch {
|
|
3434
|
+
}
|
|
3435
|
+
return formatUtcOffsetMinutes(-now.getTimezoneOffset());
|
|
3436
|
+
};
|
|
3437
|
+
|
|
2842
3438
|
// ../../clients/web-client/src/api/performance.ts
|
|
2843
3439
|
var readPerformanceBytes = async function(response) {
|
|
2844
3440
|
const maximum = PERFORMANCE_MAX_BYTES + 65535;
|
|
@@ -3082,7 +3678,9 @@ var hasTimelineShape = function(value) {
|
|
|
3082
3678
|
}
|
|
3083
3679
|
previousDialogueEnd = Number(block.endTime);
|
|
3084
3680
|
return true;
|
|
3085
|
-
}) && isSameTime(previousDialogueEnd, value.duration) && hasOrderedItems(value.expressions, value.duration, hasExpressionShape) && hasOrderedItems(value.animations, value.duration, hasAnimationShape)
|
|
3681
|
+
}) && isSameTime(previousDialogueEnd, value.duration) && hasOrderedItems(value.expressions, value.duration, hasExpressionShape) && hasOrderedItems(value.animations, value.duration, hasAnimationShape) && (value.initialVariant === void 0 || isValidVariantKey(value.initialVariant)) && (value.activeVariant === void 0 || isValidVariantKey(value.activeVariant)) && (value.toolCalls === void 0 || Array.isArray(value.toolCalls) && value.toolCalls.every(
|
|
3682
|
+
(call) => isStagingToolCall(call) && isWithinDuration(call.timestamp, value.duration)
|
|
3683
|
+
));
|
|
3086
3684
|
};
|
|
3087
3685
|
var normalizeStagingResultPayload = function(value) {
|
|
3088
3686
|
if (!isRecord(value)) {
|
|
@@ -3192,14 +3790,20 @@ var BakedConversation = class {
|
|
|
3192
3790
|
conversationId,
|
|
3193
3791
|
openingStagingId: this.asset.rootStagingId,
|
|
3194
3792
|
activeStagingId: this.activeId,
|
|
3195
|
-
stagings: [...this.nodes.values()].map((node) =>
|
|
3196
|
-
|
|
3197
|
-
|
|
3198
|
-
|
|
3199
|
-
|
|
3200
|
-
|
|
3201
|
-
|
|
3202
|
-
|
|
3793
|
+
stagings: [...this.nodes.values()].map((node) => {
|
|
3794
|
+
const { initialVariant, activeVariant, toolCalls } = node.payload.timeline;
|
|
3795
|
+
return {
|
|
3796
|
+
id: node.id,
|
|
3797
|
+
parentId: node.parentId,
|
|
3798
|
+
triggerText: node.triggerText,
|
|
3799
|
+
text: node.payload.text,
|
|
3800
|
+
options: structuredClone(node.payload.options),
|
|
3801
|
+
createdAt: "1970-01-01T00:00:00.000Z",
|
|
3802
|
+
...initialVariant !== void 0 ? { initialVariant } : {},
|
|
3803
|
+
...activeVariant !== void 0 ? { activeVariant } : {},
|
|
3804
|
+
...toolCalls ? { toolCalls: structuredClone(toolCalls) } : {}
|
|
3805
|
+
};
|
|
3806
|
+
})
|
|
3203
3807
|
};
|
|
3204
3808
|
}
|
|
3205
3809
|
child(parentId, label) {
|
|
@@ -3239,7 +3843,9 @@ var createConversation = async function({
|
|
|
3239
3843
|
avatarId,
|
|
3240
3844
|
apiToken,
|
|
3241
3845
|
baseUrl,
|
|
3242
|
-
renderSessionId
|
|
3846
|
+
renderSessionId,
|
|
3847
|
+
hostedWebChat,
|
|
3848
|
+
memoryIdentifier
|
|
3243
3849
|
}) {
|
|
3244
3850
|
const url = new URL("/conversations", baseUrl);
|
|
3245
3851
|
const res = await fetch(url.toString(), {
|
|
@@ -3253,7 +3859,12 @@ var createConversation = async function({
|
|
|
3253
3859
|
credentials: apiToken ? "omit" : "include",
|
|
3254
3860
|
method: "POST",
|
|
3255
3861
|
// This SDK plays live motion frames (ADR 0011).
|
|
3256
|
-
body: JSON.stringify({
|
|
3862
|
+
body: JSON.stringify({
|
|
3863
|
+
avatarId,
|
|
3864
|
+
capabilities: [MOTION_FRAMES_CAPABILITY],
|
|
3865
|
+
...hostedWebChat ? { hostedWebChat: true } : {},
|
|
3866
|
+
...memoryIdentifier !== void 0 ? { memoryIdentifier } : {}
|
|
3867
|
+
})
|
|
3257
3868
|
});
|
|
3258
3869
|
if (!res.ok) {
|
|
3259
3870
|
throw Object.assign(new Error(await res.text()), { status: res.status });
|
|
@@ -3276,7 +3887,8 @@ var createStaging = async function({
|
|
|
3276
3887
|
requestId,
|
|
3277
3888
|
silent,
|
|
3278
3889
|
parentStagingId,
|
|
3279
|
-
renderSessionId
|
|
3890
|
+
renderSessionId,
|
|
3891
|
+
memoryIdentifier
|
|
3280
3892
|
}) {
|
|
3281
3893
|
const url = new URL(`/conversations/${conversationId}/stagings`, baseUrl);
|
|
3282
3894
|
const requestHeaders = withRenderSession(
|
|
@@ -3307,7 +3919,9 @@ var createStaging = async function({
|
|
|
3307
3919
|
silent: silent === true,
|
|
3308
3920
|
incrementalPlayback: true,
|
|
3309
3921
|
parentStagingId,
|
|
3310
|
-
capabilities: [MOTION_FRAMES_CAPABILITY]
|
|
3922
|
+
capabilities: [MOTION_FRAMES_CAPABILITY],
|
|
3923
|
+
timeZone: detectClientTimeZone(),
|
|
3924
|
+
...memoryIdentifier !== void 0 ? { memoryIdentifier } : {}
|
|
3311
3925
|
})
|
|
3312
3926
|
});
|
|
3313
3927
|
} catch (error) {
|
|
@@ -3491,7 +4105,8 @@ var LiveConversation = class {
|
|
|
3491
4105
|
silent,
|
|
3492
4106
|
parentStagingId,
|
|
3493
4107
|
headers: telemetry.traceHeaders,
|
|
3494
|
-
renderSessionId: this.dependencies.renderSession()
|
|
4108
|
+
renderSessionId: this.dependencies.renderSession(),
|
|
4109
|
+
memoryIdentifier: this.dependencies.memoryIdentifier?.()
|
|
3495
4110
|
});
|
|
3496
4111
|
} catch (error) {
|
|
3497
4112
|
if (request.completed) {
|
|
@@ -3753,25 +4368,61 @@ var loadGltfWithProgress = async function({
|
|
|
3753
4368
|
});
|
|
3754
4369
|
}
|
|
3755
4370
|
};
|
|
3756
|
-
var
|
|
3757
|
-
|
|
3758
|
-
|
|
3759
|
-
|
|
3760
|
-
|
|
3761
|
-
assetProfile
|
|
3762
|
-
|
|
3763
|
-
const url = new URL(`/avatars/${characterId}/download-url`, baseUrl);
|
|
3764
|
-
if (assetProfile && assetProfile !== "original") {
|
|
3765
|
-
url.searchParams.set("assetProfile", assetProfile);
|
|
4371
|
+
var fetchVrmDownloadUrl = async function(baseUrl, characterId, options) {
|
|
4372
|
+
const url = new URL(
|
|
4373
|
+
`/avatars/${encodeURIComponent(characterId)}/download-url`,
|
|
4374
|
+
baseUrl
|
|
4375
|
+
);
|
|
4376
|
+
if (options?.assetProfile && options.assetProfile !== "original") {
|
|
4377
|
+
url.searchParams.set("assetProfile", options.assetProfile);
|
|
3766
4378
|
}
|
|
3767
4379
|
const res = await fetch(url.toString(), {
|
|
3768
|
-
headers: buildAuthHeaders(apiToken, { allowEmpty: true }),
|
|
3769
|
-
credentials: apiToken ? "omit" : "include"
|
|
4380
|
+
headers: buildAuthHeaders(options?.apiToken, { allowEmpty: true }),
|
|
4381
|
+
credentials: options?.apiToken ? "omit" : "include",
|
|
4382
|
+
...options?.signal ? { signal: options.signal } : {}
|
|
4383
|
+
});
|
|
4384
|
+
if (!res.ok) {
|
|
4385
|
+
throw new Error(await res.text());
|
|
4386
|
+
}
|
|
4387
|
+
const { url: vrmUrl } = await res.json();
|
|
4388
|
+
return vrmUrl;
|
|
4389
|
+
};
|
|
4390
|
+
var fetchVariantVrmDownloadUrl = async function(baseUrl, characterId, variantKey, options) {
|
|
4391
|
+
if (variantKey === DEFAULT_VARIANT_KEY) {
|
|
4392
|
+
return await fetchVrmDownloadUrl(baseUrl, characterId, options);
|
|
4393
|
+
}
|
|
4394
|
+
const url = new URL(
|
|
4395
|
+
`/avatars/${encodeURIComponent(characterId)}/variants/${encodeURIComponent(variantKey)}/download-url`,
|
|
4396
|
+
baseUrl
|
|
4397
|
+
);
|
|
4398
|
+
if (options?.assetProfile && options.assetProfile !== "original") {
|
|
4399
|
+
url.searchParams.set("assetProfile", options.assetProfile);
|
|
4400
|
+
}
|
|
4401
|
+
const res = await fetch(url.toString(), {
|
|
4402
|
+
headers: buildAuthHeaders(options?.apiToken, { allowEmpty: true }),
|
|
4403
|
+
credentials: options?.apiToken ? "omit" : "include",
|
|
4404
|
+
...options?.signal ? { signal: options.signal } : {}
|
|
3770
4405
|
});
|
|
3771
4406
|
if (!res.ok) {
|
|
3772
4407
|
throw new Error(await res.text());
|
|
3773
4408
|
}
|
|
3774
4409
|
const { url: vrmUrl } = await res.json();
|
|
4410
|
+
return vrmUrl;
|
|
4411
|
+
};
|
|
4412
|
+
var loadVrm = async function({
|
|
4413
|
+
baseUrl,
|
|
4414
|
+
characterId,
|
|
4415
|
+
variantKey,
|
|
4416
|
+
apiToken,
|
|
4417
|
+
onProgress,
|
|
4418
|
+
assetProfile
|
|
4419
|
+
}) {
|
|
4420
|
+
const vrmUrl = await fetchVariantVrmDownloadUrl(
|
|
4421
|
+
baseUrl,
|
|
4422
|
+
characterId,
|
|
4423
|
+
variantKey ?? DEFAULT_VARIANT_KEY,
|
|
4424
|
+
{ apiToken, assetProfile }
|
|
4425
|
+
);
|
|
3775
4426
|
const gltf = await loadGltfWithProgress({
|
|
3776
4427
|
url: vrmUrl,
|
|
3777
4428
|
onProgress,
|
|
@@ -3783,6 +4434,23 @@ var loadVrm = async ({
|
|
|
3783
4434
|
VRMUtils.combineSkeletons(gltf.scene);
|
|
3784
4435
|
return gltf.userData.vrm;
|
|
3785
4436
|
};
|
|
4437
|
+
var loadVariantVrm = async function({
|
|
4438
|
+
baseUrl,
|
|
4439
|
+
characterId,
|
|
4440
|
+
variantKey,
|
|
4441
|
+
apiToken,
|
|
4442
|
+
onProgress,
|
|
4443
|
+
assetProfile
|
|
4444
|
+
}) {
|
|
4445
|
+
return await loadVrm({
|
|
4446
|
+
baseUrl,
|
|
4447
|
+
characterId,
|
|
4448
|
+
variantKey,
|
|
4449
|
+
apiToken,
|
|
4450
|
+
onProgress,
|
|
4451
|
+
assetProfile
|
|
4452
|
+
});
|
|
4453
|
+
};
|
|
3786
4454
|
var loadVrma = async ({
|
|
3787
4455
|
baseUrl,
|
|
3788
4456
|
characterId,
|
|
@@ -5365,6 +6033,70 @@ var AnimationController = class {
|
|
|
5365
6033
|
activeAnimationId = null;
|
|
5366
6034
|
idleAnimationId;
|
|
5367
6035
|
pendingReturnToIdleAction = null;
|
|
6036
|
+
setVrm(nextVrm, rawAnimations) {
|
|
6037
|
+
if (nextVrm === this.vrm) {
|
|
6038
|
+
return;
|
|
6039
|
+
}
|
|
6040
|
+
const currentAnimationId = this.activeAnimationId;
|
|
6041
|
+
const currentAction = this.activeAction;
|
|
6042
|
+
const currentTime = currentAction ? currentAction.time : 0;
|
|
6043
|
+
const currentLoop = currentAction ? currentAction.loop === LoopRepeat2 : void 0;
|
|
6044
|
+
const hadPendingReturnToIdle = this.pendingReturnToIdleAction !== null && this.pendingReturnToIdleAction === currentAction;
|
|
6045
|
+
const savedConfigs = new Map(this.configs);
|
|
6046
|
+
this.restorePreview?.();
|
|
6047
|
+
this.mixer.removeEventListener("finished", this.handleActionFinished);
|
|
6048
|
+
this.mixer.stopAllAction();
|
|
6049
|
+
this.mixer.uncacheRoot(this.vrm.scene);
|
|
6050
|
+
this.vrm.humanoid?.resetNormalizedPose?.();
|
|
6051
|
+
this.vrm = nextVrm;
|
|
6052
|
+
this.mixer = new AnimationMixer2(this.vrm.scene);
|
|
6053
|
+
this.mixer.addEventListener("finished", this.handleActionFinished);
|
|
6054
|
+
this.clips.clear();
|
|
6055
|
+
this.rawClips.clear();
|
|
6056
|
+
this.normalizedIdleClips.clear();
|
|
6057
|
+
this.configs.clear();
|
|
6058
|
+
this.activeAction = null;
|
|
6059
|
+
this.activeAnimationId = null;
|
|
6060
|
+
this.pendingReturnToIdleAction = null;
|
|
6061
|
+
for (const [animationId, animation] of rawAnimations) {
|
|
6062
|
+
const config = savedConfigs.get(animationId) ?? {
|
|
6063
|
+
loop: animationId === this.idleAnimationId,
|
|
6064
|
+
weight: defaultAnimationConfig.weight
|
|
6065
|
+
};
|
|
6066
|
+
this.addAnimation(animationId, animation, config);
|
|
6067
|
+
}
|
|
6068
|
+
const neededIds = [];
|
|
6069
|
+
if (currentAnimationId && !this.clips.has(currentAnimationId)) {
|
|
6070
|
+
neededIds.push(currentAnimationId);
|
|
6071
|
+
}
|
|
6072
|
+
if (this.idleAnimationId && !this.clips.has(this.idleAnimationId) && !neededIds.includes(this.idleAnimationId)) {
|
|
6073
|
+
neededIds.push(this.idleAnimationId);
|
|
6074
|
+
}
|
|
6075
|
+
if (neededIds.length > 0 && this.prepareAnimations) {
|
|
6076
|
+
void this.prepareAnimations(neededIds).catch(() => void 0);
|
|
6077
|
+
}
|
|
6078
|
+
if (currentAnimationId && this.clips.has(currentAnimationId)) {
|
|
6079
|
+
this.playAnimation(currentAnimationId, 0, currentLoop);
|
|
6080
|
+
const resumedAction = this.getActiveAction();
|
|
6081
|
+
if (resumedAction) {
|
|
6082
|
+
const clipDuration = resumedAction.getClip().duration;
|
|
6083
|
+
if (clipDuration > 0 && Number.isFinite(currentTime)) {
|
|
6084
|
+
resumedAction.time = resumedAction.loop === LoopRepeat2 ? (currentTime % clipDuration + clipDuration) % clipDuration : Math.min(Math.max(0, currentTime), clipDuration);
|
|
6085
|
+
}
|
|
6086
|
+
if (hadPendingReturnToIdle && !currentLoop && this.idleAnimationId) {
|
|
6087
|
+
this.pendingReturnToIdleAction = resumedAction;
|
|
6088
|
+
}
|
|
6089
|
+
}
|
|
6090
|
+
this.mixer.update(0);
|
|
6091
|
+
return;
|
|
6092
|
+
}
|
|
6093
|
+
if (this.idleAnimationId && this.clips.has(this.idleAnimationId)) {
|
|
6094
|
+
this.playIdle(0);
|
|
6095
|
+
this.mixer.update(0);
|
|
6096
|
+
return;
|
|
6097
|
+
}
|
|
6098
|
+
this.vrm.humanoid?.resetNormalizedPose?.();
|
|
6099
|
+
}
|
|
5368
6100
|
getAnimationMixer() {
|
|
5369
6101
|
return this.mixer;
|
|
5370
6102
|
}
|
|
@@ -6040,6 +6772,35 @@ var PerformanceRecorder = class {
|
|
|
6040
6772
|
};
|
|
6041
6773
|
|
|
6042
6774
|
// ../../clients/web-client/src/playback-utils.ts
|
|
6775
|
+
var deliverStagingResult = async function(payload, options) {
|
|
6776
|
+
const normalized = normalizeStagingResultPayload(payload);
|
|
6777
|
+
options.setOptions(normalized.options);
|
|
6778
|
+
if (options.source === "turn") {
|
|
6779
|
+
options.onTurnReceived(normalized);
|
|
6780
|
+
}
|
|
6781
|
+
const force = options.forcePlayback === true || options.source === "replay";
|
|
6782
|
+
const shouldNotify = force || shouldNotifyStagingResult(options.notifiedStagingIds, normalized.stagingId);
|
|
6783
|
+
if (shouldNotify && options.onStagingResult) {
|
|
6784
|
+
try {
|
|
6785
|
+
await options.onStagingResult(normalized);
|
|
6786
|
+
} catch (error) {
|
|
6787
|
+
options.onCallbackError("onStagingResult", error);
|
|
6788
|
+
}
|
|
6789
|
+
}
|
|
6790
|
+
const shouldPlay = !options.hasPlayed(normalized.stagingId) || force;
|
|
6791
|
+
options.recordEvent({
|
|
6792
|
+
type: "staging.result",
|
|
6793
|
+
source: options.source,
|
|
6794
|
+
stagingId: normalized.stagingId,
|
|
6795
|
+
taskId: normalized.taskId ?? "",
|
|
6796
|
+
playback: shouldPlay ? "requested" : "already-streamed"
|
|
6797
|
+
});
|
|
6798
|
+
if (shouldPlay) {
|
|
6799
|
+
options.playRequested(normalized, { force, silent: options.silent });
|
|
6800
|
+
} else {
|
|
6801
|
+
options.playStreamed(normalized);
|
|
6802
|
+
}
|
|
6803
|
+
};
|
|
6043
6804
|
var toSilentTimeline = function(timeline) {
|
|
6044
6805
|
return {
|
|
6045
6806
|
...timeline,
|
|
@@ -6166,6 +6927,16 @@ var StreamPlaybackController = class {
|
|
|
6166
6927
|
get activeStagingId() {
|
|
6167
6928
|
return this.activePlayback?.stagingId ?? null;
|
|
6168
6929
|
}
|
|
6930
|
+
/**
|
|
6931
|
+
* A staging plays or waits to play (its audio or the rest of its stream).
|
|
6932
|
+
* False after it completed or when autoplay blocked it; the last staging
|
|
6933
|
+
* stays the active staging after that.
|
|
6934
|
+
*/
|
|
6935
|
+
get playing() {
|
|
6936
|
+
const active = this.activePlayback;
|
|
6937
|
+
const state = active ? this.states.get(active.stagingId) : void 0;
|
|
6938
|
+
return state !== void 0 && !state.completed && !state.blocked;
|
|
6939
|
+
}
|
|
6169
6940
|
onStreamEvent(callback) {
|
|
6170
6941
|
return bindCallbackSlot(
|
|
6171
6942
|
() => this.streamEventCallback,
|
|
@@ -6462,7 +7233,11 @@ var StreamPlaybackController = class {
|
|
|
6462
7233
|
() => this.characterRevealCallback?.(event, characterIndex, character)
|
|
6463
7234
|
);
|
|
6464
7235
|
}
|
|
6465
|
-
}
|
|
7236
|
+
},
|
|
7237
|
+
onToolCall: (toolCall) => {
|
|
7238
|
+
this.dependencies.onToolCall?.(toolCall);
|
|
7239
|
+
},
|
|
7240
|
+
onVariantCue: (variantKey, toolCall) => this.dependencies.onVariantCue?.(variantKey, toolCall)
|
|
6466
7241
|
};
|
|
6467
7242
|
try {
|
|
6468
7243
|
await player.play(params.timeline, playbackOptions);
|
|
@@ -7227,37 +8002,216 @@ var AudioPlayer = class {
|
|
|
7227
8002
|
}
|
|
7228
8003
|
};
|
|
7229
8004
|
|
|
7230
|
-
// ../../clients/web-client/src/character/
|
|
7231
|
-
var
|
|
7232
|
-
|
|
7233
|
-
|
|
8005
|
+
// ../../clients/web-client/src/character/character-variants.ts
|
|
8006
|
+
var disposeOrphanedVrm = function(vrm) {
|
|
8007
|
+
if (vrm.scene?.parent) {
|
|
8008
|
+
vrm.scene.parent.remove(vrm.scene);
|
|
7234
8009
|
}
|
|
7235
|
-
|
|
7236
|
-
|
|
7237
|
-
return this.mixer.getChannelNames();
|
|
8010
|
+
if (vrm.scene && typeof vrm.scene.traverse === "function") {
|
|
8011
|
+
disposeVrm(vrm);
|
|
7238
8012
|
}
|
|
7239
|
-
|
|
7240
|
-
|
|
7241
|
-
|
|
8013
|
+
};
|
|
8014
|
+
var CharacterVariantController = class {
|
|
8015
|
+
constructor(options) {
|
|
8016
|
+
this.options = options;
|
|
7242
8017
|
}
|
|
7243
|
-
|
|
7244
|
-
|
|
7245
|
-
|
|
8018
|
+
variantVrms = /* @__PURE__ */ new Map();
|
|
8019
|
+
variantsList = [];
|
|
8020
|
+
activeVariantKey = DEFAULT_VARIANT_KEY;
|
|
8021
|
+
remoteMode = false;
|
|
8022
|
+
loadEpoch = 0;
|
|
8023
|
+
switchRevision = 0;
|
|
8024
|
+
pendingLoads = /* @__PURE__ */ new Map();
|
|
8025
|
+
listeners = {
|
|
8026
|
+
variantChange: /* @__PURE__ */ new Set(),
|
|
8027
|
+
toolCall: /* @__PURE__ */ new Set(),
|
|
8028
|
+
vrmChange: /* @__PURE__ */ new Set()
|
|
8029
|
+
};
|
|
8030
|
+
get variants() {
|
|
8031
|
+
return this.variantsList.map((variant) => ({ ...variant }));
|
|
7246
8032
|
}
|
|
7247
|
-
|
|
7248
|
-
|
|
7249
|
-
return this.mixer.getWeights();
|
|
8033
|
+
get activeVariant() {
|
|
8034
|
+
return this.activeVariantKey;
|
|
7250
8035
|
}
|
|
7251
|
-
|
|
7252
|
-
|
|
7253
|
-
return this.mixer.getMouthOpenness();
|
|
8036
|
+
get isRemote() {
|
|
8037
|
+
return this.remoteMode;
|
|
7254
8038
|
}
|
|
7255
|
-
|
|
7256
|
-
|
|
7257
|
-
|
|
8039
|
+
on(event, listener) {
|
|
8040
|
+
const bucket = this.listeners[event];
|
|
8041
|
+
bucket.add(listener);
|
|
8042
|
+
return () => {
|
|
8043
|
+
bucket.delete(listener);
|
|
8044
|
+
};
|
|
7258
8045
|
}
|
|
7259
|
-
|
|
7260
|
-
|
|
8046
|
+
off(event, listener) {
|
|
8047
|
+
this.listeners[event].delete(listener);
|
|
8048
|
+
}
|
|
8049
|
+
emit(event, payload) {
|
|
8050
|
+
for (const listener of [...this.listeners[event]]) {
|
|
8051
|
+
invokeSafeCallback(
|
|
8052
|
+
`on(${event})`,
|
|
8053
|
+
this.options.handleCallbackError,
|
|
8054
|
+
() => listener(payload)
|
|
8055
|
+
);
|
|
8056
|
+
}
|
|
8057
|
+
}
|
|
8058
|
+
configureVariants(variants, options) {
|
|
8059
|
+
this.variantsList = variants ? variants.map((entry) => ({ ...entry })) : [];
|
|
8060
|
+
if (options?.remote !== void 0) {
|
|
8061
|
+
this.remoteMode = options.remote;
|
|
8062
|
+
}
|
|
8063
|
+
}
|
|
8064
|
+
registerDefaultVrm(vrm) {
|
|
8065
|
+
this.variantVrms.set(DEFAULT_VARIANT_KEY, vrm);
|
|
8066
|
+
this.activeVariantKey = DEFAULT_VARIANT_KEY;
|
|
8067
|
+
this.remoteMode = false;
|
|
8068
|
+
}
|
|
8069
|
+
syncDirectVrm(vrm, previousVrm) {
|
|
8070
|
+
if (vrm && !this.variantVrms.has(this.activeVariantKey)) {
|
|
8071
|
+
this.variantVrms.set(this.activeVariantKey, vrm);
|
|
8072
|
+
}
|
|
8073
|
+
if (vrm && vrm !== previousVrm) {
|
|
8074
|
+
this.emit("vrmChange", vrm);
|
|
8075
|
+
}
|
|
8076
|
+
}
|
|
8077
|
+
async preloadVariants(eager) {
|
|
8078
|
+
const keys = this.variantsList.map((variant) => variant.key).filter((key) => key !== DEFAULT_VARIANT_KEY);
|
|
8079
|
+
if (keys.length === 0) {
|
|
8080
|
+
return;
|
|
8081
|
+
}
|
|
8082
|
+
if (eager) {
|
|
8083
|
+
await Promise.all(keys.map((key) => this.ensureVariant(key)));
|
|
8084
|
+
return;
|
|
8085
|
+
}
|
|
8086
|
+
for (const key of keys) {
|
|
8087
|
+
void this.ensureVariant(key).catch(() => {
|
|
8088
|
+
});
|
|
8089
|
+
}
|
|
8090
|
+
}
|
|
8091
|
+
async ensureVariant(variantKey) {
|
|
8092
|
+
const cached = this.variantVrms.get(variantKey);
|
|
8093
|
+
if (cached) {
|
|
8094
|
+
return cached;
|
|
8095
|
+
}
|
|
8096
|
+
const existing = this.pendingLoads.get(variantKey);
|
|
8097
|
+
if (existing) {
|
|
8098
|
+
return await existing;
|
|
8099
|
+
}
|
|
8100
|
+
const epoch = this.loadEpoch;
|
|
8101
|
+
const request = this.options.loadVariant(variantKey).then((loadedVrm) => {
|
|
8102
|
+
this.pendingLoads.delete(variantKey);
|
|
8103
|
+
if (epoch !== this.loadEpoch) {
|
|
8104
|
+
disposeOrphanedVrm(loadedVrm);
|
|
8105
|
+
throw new Error("Character assets were reset while loading variant.");
|
|
8106
|
+
}
|
|
8107
|
+
this.variantVrms.set(variantKey, loadedVrm);
|
|
8108
|
+
return loadedVrm;
|
|
8109
|
+
}).catch((error) => {
|
|
8110
|
+
this.pendingLoads.delete(variantKey);
|
|
8111
|
+
throw error;
|
|
8112
|
+
});
|
|
8113
|
+
this.pendingLoads.set(variantKey, request);
|
|
8114
|
+
return await request;
|
|
8115
|
+
}
|
|
8116
|
+
async switchVariant(variantKey, toolCall = null) {
|
|
8117
|
+
if (!isValidVariantKey(variantKey)) {
|
|
8118
|
+
throw new Error(`Invalid variant key: ${variantKey}`);
|
|
8119
|
+
}
|
|
8120
|
+
const revision = ++this.switchRevision;
|
|
8121
|
+
if (this.remoteMode) {
|
|
8122
|
+
if (variantKey !== DEFAULT_VARIANT_KEY && this.variantsList.length > 0 && !this.variantsList.some((entry) => entry.key === variantKey)) {
|
|
8123
|
+
throw new Error(`Unknown variant: ${variantKey}`);
|
|
8124
|
+
}
|
|
8125
|
+
if (this.activeVariantKey === variantKey) {
|
|
8126
|
+
return;
|
|
8127
|
+
}
|
|
8128
|
+
const previousVariant = this.activeVariantKey;
|
|
8129
|
+
this.activeVariantKey = variantKey;
|
|
8130
|
+
this.emit("variantChange", {
|
|
8131
|
+
previousVariant,
|
|
8132
|
+
activeVariant: variantKey,
|
|
8133
|
+
toolCall
|
|
8134
|
+
});
|
|
8135
|
+
return;
|
|
8136
|
+
}
|
|
8137
|
+
const cached = this.variantVrms.get(variantKey);
|
|
8138
|
+
if (cached) {
|
|
8139
|
+
this.applyActiveVariant(variantKey, cached, toolCall);
|
|
8140
|
+
return;
|
|
8141
|
+
}
|
|
8142
|
+
if (variantKey === DEFAULT_VARIANT_KEY && this.activeVariantKey === DEFAULT_VARIANT_KEY) {
|
|
8143
|
+
return;
|
|
8144
|
+
}
|
|
8145
|
+
const loaded = await this.ensureVariant(variantKey);
|
|
8146
|
+
if (revision !== this.switchRevision) {
|
|
8147
|
+
return;
|
|
8148
|
+
}
|
|
8149
|
+
this.applyActiveVariant(variantKey, loaded, toolCall);
|
|
8150
|
+
}
|
|
8151
|
+
reset() {
|
|
8152
|
+
this.loadEpoch += 1;
|
|
8153
|
+
this.switchRevision += 1;
|
|
8154
|
+
this.pendingLoads.clear();
|
|
8155
|
+
const cached = [...this.variantVrms.values()];
|
|
8156
|
+
this.variantVrms.clear();
|
|
8157
|
+
this.activeVariantKey = DEFAULT_VARIANT_KEY;
|
|
8158
|
+
this.remoteMode = false;
|
|
8159
|
+
return cached;
|
|
8160
|
+
}
|
|
8161
|
+
applyActiveVariant(variantKey, nextVrm, toolCall) {
|
|
8162
|
+
const previousVariant = this.activeVariantKey;
|
|
8163
|
+
const previousVrm = this.options.getCurrentVrm();
|
|
8164
|
+
if (previousVariant === variantKey && previousVrm === nextVrm) {
|
|
8165
|
+
return;
|
|
8166
|
+
}
|
|
8167
|
+
if (previousVrm !== nextVrm) {
|
|
8168
|
+
this.options.applyVrmSwap(nextVrm);
|
|
8169
|
+
}
|
|
8170
|
+
this.activeVariantKey = variantKey;
|
|
8171
|
+
if (previousVrm !== nextVrm) {
|
|
8172
|
+
this.emit("vrmChange", nextVrm);
|
|
8173
|
+
}
|
|
8174
|
+
if (previousVariant !== variantKey) {
|
|
8175
|
+
this.emit("variantChange", {
|
|
8176
|
+
previousVariant,
|
|
8177
|
+
activeVariant: variantKey,
|
|
8178
|
+
toolCall
|
|
8179
|
+
});
|
|
8180
|
+
}
|
|
8181
|
+
}
|
|
8182
|
+
};
|
|
8183
|
+
|
|
8184
|
+
// ../../clients/web-client/src/character/expressions.ts
|
|
8185
|
+
var CharacterExpressions = class {
|
|
8186
|
+
constructor(mixer) {
|
|
8187
|
+
this.mixer = mixer;
|
|
8188
|
+
}
|
|
8189
|
+
/** All available VRM expression channel names on the loaded character. */
|
|
8190
|
+
get all() {
|
|
8191
|
+
return this.mixer.getChannelNames();
|
|
8192
|
+
}
|
|
8193
|
+
/** Available emotion expression channel names (`happy`, `angry`, `sad`, etc.). */
|
|
8194
|
+
get emotions() {
|
|
8195
|
+
return this.mixer.getEmotionChannelNames();
|
|
8196
|
+
}
|
|
8197
|
+
/** Available mouth/viseme expression channel names (`aa`, `ee`, `ih`, `oh`, `ou`). */
|
|
8198
|
+
get visemes() {
|
|
8199
|
+
return this.mixer.getVisemeChannelNames();
|
|
8200
|
+
}
|
|
8201
|
+
/** Current applied weights for all expression channels in `[0, 1]`. */
|
|
8202
|
+
get weights() {
|
|
8203
|
+
return this.mixer.getWeights();
|
|
8204
|
+
}
|
|
8205
|
+
/** Real-time aggregate mouth openness in `[0, 1]`. */
|
|
8206
|
+
get mouthOpenness() {
|
|
8207
|
+
return this.mixer.getMouthOpenness();
|
|
8208
|
+
}
|
|
8209
|
+
/** Active mouth ownership mode (`'expression'` or `'viseme'`). */
|
|
8210
|
+
get mouthMode() {
|
|
8211
|
+
return this.mixer.mouthMode;
|
|
8212
|
+
}
|
|
8213
|
+
/** Whether automatic idle blinking and eye movement are active on this face. */
|
|
8214
|
+
get hasIdleFace() {
|
|
7261
8215
|
return this.mixer.hasIdleFace;
|
|
7262
8216
|
}
|
|
7263
8217
|
/** Full performance frame snapshot including layered weights and applied values. */
|
|
@@ -7494,6 +8448,76 @@ var IdleFace = class {
|
|
|
7494
8448
|
}
|
|
7495
8449
|
};
|
|
7496
8450
|
|
|
8451
|
+
// ../../clients/web-client/src/character/listening-face.ts
|
|
8452
|
+
var LISTENING_FACE_LAYER = "listening";
|
|
8453
|
+
var CHANNELS = ["relaxed", "happy"];
|
|
8454
|
+
var REST = 0.01;
|
|
8455
|
+
var ListeningFace = class {
|
|
8456
|
+
constructor(weights) {
|
|
8457
|
+
this.weights = weights;
|
|
8458
|
+
}
|
|
8459
|
+
channel = null;
|
|
8460
|
+
weight = 0;
|
|
8461
|
+
saved = /* @__PURE__ */ new Map();
|
|
8462
|
+
/** The preset in use, or null when the model has none of them. */
|
|
8463
|
+
get channelName() {
|
|
8464
|
+
return this.channel;
|
|
8465
|
+
}
|
|
8466
|
+
/** Call after the layers were registered for a new model. */
|
|
8467
|
+
setChannels(available, manager) {
|
|
8468
|
+
this.restoreOverrides(manager);
|
|
8469
|
+
this.channel = CHANNELS.find((name) => available.has(name)) ?? null;
|
|
8470
|
+
}
|
|
8471
|
+
set(weight, options) {
|
|
8472
|
+
this.weight = Math.max(0, Math.min(1, weight));
|
|
8473
|
+
if (options?.immediate && this.weight === 0) {
|
|
8474
|
+
this.weights.clearLayerTargets(LISTENING_FACE_LAYER, { immediate: true });
|
|
8475
|
+
}
|
|
8476
|
+
}
|
|
8477
|
+
/** Before the weight mixer updates: an active emotion takes the face. */
|
|
8478
|
+
beforeUpdate(emotionActive) {
|
|
8479
|
+
if (!this.channel) {
|
|
8480
|
+
return;
|
|
8481
|
+
}
|
|
8482
|
+
const target = emotionActive ? 0 : this.weight;
|
|
8483
|
+
if (target === 0 && this.weights.getLayerValue(LISTENING_FACE_LAYER, this.channel) < REST) {
|
|
8484
|
+
this.weights.clearLayerTargets(LISTENING_FACE_LAYER, { immediate: true });
|
|
8485
|
+
return;
|
|
8486
|
+
}
|
|
8487
|
+
this.weights.setTarget(LISTENING_FACE_LAYER, this.channel, target);
|
|
8488
|
+
}
|
|
8489
|
+
/** After the values went to the expression manager, before `vrm.update()`. */
|
|
8490
|
+
afterUpdate(manager, otherLayerValue) {
|
|
8491
|
+
const channel = this.channel;
|
|
8492
|
+
const owned = channel !== null && this.weights.getLayerValue(LISTENING_FACE_LAYER, channel) > 0 && otherLayerValue(channel) === 0;
|
|
8493
|
+
if (!owned) {
|
|
8494
|
+
this.restoreOverrides(manager);
|
|
8495
|
+
return;
|
|
8496
|
+
}
|
|
8497
|
+
const expression = manager.getExpression(channel);
|
|
8498
|
+
if (!expression || this.saved.size > 0) {
|
|
8499
|
+
return;
|
|
8500
|
+
}
|
|
8501
|
+
;
|
|
8502
|
+
["overrideBlink", "overrideLookAt"].forEach((key) => {
|
|
8503
|
+
if (expression[key] === "block") {
|
|
8504
|
+
this.saved.set(key, expression[key]);
|
|
8505
|
+
expression[key] = "blend";
|
|
8506
|
+
}
|
|
8507
|
+
});
|
|
8508
|
+
}
|
|
8509
|
+
/** Gives the preset its own overrides back. */
|
|
8510
|
+
restoreOverrides(manager) {
|
|
8511
|
+
const expression = this.channel !== null ? manager?.getExpression(this.channel) : null;
|
|
8512
|
+
if (expression) {
|
|
8513
|
+
this.saved.forEach((value, key) => {
|
|
8514
|
+
expression[key] = value;
|
|
8515
|
+
});
|
|
8516
|
+
}
|
|
8517
|
+
this.saved.clear();
|
|
8518
|
+
}
|
|
8519
|
+
};
|
|
8520
|
+
|
|
7497
8521
|
// ../../clients/web-client/src/character/performance-weight-mixer.ts
|
|
7498
8522
|
var PRECISION = 1e3;
|
|
7499
8523
|
var DEFAULT_RESPONSE_RATE = 12;
|
|
@@ -7572,6 +8596,18 @@ var PerformanceWeightMixer = class {
|
|
|
7572
8596
|
}
|
|
7573
8597
|
layer.channels.set(channel, { value: 0, target });
|
|
7574
8598
|
}
|
|
8599
|
+
restoreChannelState(layerId, channel, value, target) {
|
|
8600
|
+
const layer = this.getLayer(layerId);
|
|
8601
|
+
const state = layer.channels.get(channel);
|
|
8602
|
+
const nextValue = clamp012(value);
|
|
8603
|
+
const nextTarget = clamp012(target);
|
|
8604
|
+
if (state) {
|
|
8605
|
+
state.value = nextValue;
|
|
8606
|
+
state.target = nextTarget;
|
|
8607
|
+
return;
|
|
8608
|
+
}
|
|
8609
|
+
layer.channels.set(channel, { value: nextValue, target: nextTarget });
|
|
8610
|
+
}
|
|
7575
8611
|
setExclusiveTargets(layerId, channels, selected, weight) {
|
|
7576
8612
|
channels.forEach((channel) => {
|
|
7577
8613
|
this.setTarget(layerId, channel, channel === selected ? weight : 0);
|
|
@@ -7730,17 +8766,34 @@ var FacialPerformanceMixer = class {
|
|
|
7730
8766
|
restoreMouthAfterVisemeRelease = false;
|
|
7731
8767
|
idleFace = null;
|
|
7732
8768
|
gazeTarget = null;
|
|
7733
|
-
|
|
7734
|
-
|
|
8769
|
+
listeningFace = new ListeningFace(this.weightMixer);
|
|
8770
|
+
setVrm(vrm, options) {
|
|
8771
|
+
const preserveState = options?.preserveState === true && vrm !== null;
|
|
8772
|
+
const savedMouthMode = preserveState ? this.activeMouthMode : "expression";
|
|
8773
|
+
const savedRestoreAfterRelease = preserveState ? this.restoreMouthAfterVisemeRelease : false;
|
|
8774
|
+
const savedSnapshot = preserveState ? this.weightMixer.getSnapshot() : null;
|
|
8775
|
+
this.vrm?.expressionManager?.resetValues?.();
|
|
8776
|
+
this.listeningFace.restoreOverrides(this.vrm?.expressionManager);
|
|
8777
|
+
if (preserveState) {
|
|
8778
|
+
this.restoreMouthOverrides();
|
|
8779
|
+
} else {
|
|
8780
|
+
this.restoreExpressionMouthOwnership();
|
|
8781
|
+
this.clearCueTimeouts();
|
|
8782
|
+
}
|
|
7735
8783
|
this.vrm = vrm;
|
|
7736
8784
|
this.idleFace = vrm && detectTemplateFace(vrm) ? new IdleFace(vrm, this.options.idleRandom) : null;
|
|
7737
8785
|
this.inspector.setVrm(vrm);
|
|
7738
|
-
this.clearCueTimeouts();
|
|
7739
8786
|
this.availableChannels = [];
|
|
7740
8787
|
this.availableChannelSet.clear();
|
|
7741
8788
|
this.channelsByLayer.clear();
|
|
7742
8789
|
this.configureLayers();
|
|
7743
8790
|
if (!vrm?.expressionManager) {
|
|
8791
|
+
this.listeningFace.setChannels(this.availableChannelSet, null);
|
|
8792
|
+
if (preserveState) {
|
|
8793
|
+
this.activeMouthMode = "expression";
|
|
8794
|
+
this.restoreMouthAfterVisemeRelease = false;
|
|
8795
|
+
this.clearCueTimeouts();
|
|
8796
|
+
}
|
|
7744
8797
|
return;
|
|
7745
8798
|
}
|
|
7746
8799
|
const manager = vrm.expressionManager;
|
|
@@ -7756,7 +8809,43 @@ var FacialPerformanceMixer = class {
|
|
|
7756
8809
|
this.weightMixer.setTarget(layerId, name, 0);
|
|
7757
8810
|
});
|
|
7758
8811
|
this.availableChannels = names;
|
|
8812
|
+
this.listeningFace.setChannels(this.availableChannelSet, manager);
|
|
7759
8813
|
manager.resetValues();
|
|
8814
|
+
if (!preserveState || !savedSnapshot) {
|
|
8815
|
+
return;
|
|
8816
|
+
}
|
|
8817
|
+
this.cueTimeouts.forEach((timeoutId, channel) => {
|
|
8818
|
+
if (!this.availableChannelSet.has(channel)) {
|
|
8819
|
+
clearTimeout(timeoutId);
|
|
8820
|
+
this.cueTimeouts.delete(channel);
|
|
8821
|
+
}
|
|
8822
|
+
});
|
|
8823
|
+
Object.entries(savedSnapshot).forEach(([layerId, layerSnapshot]) => {
|
|
8824
|
+
Object.entries(layerSnapshot.channels).forEach(
|
|
8825
|
+
([channel, channelState]) => {
|
|
8826
|
+
if (this.availableChannelSet.has(channel)) {
|
|
8827
|
+
this.weightMixer.restoreChannelState(
|
|
8828
|
+
layerId,
|
|
8829
|
+
channel,
|
|
8830
|
+
channelState.value,
|
|
8831
|
+
channelState.target
|
|
8832
|
+
);
|
|
8833
|
+
}
|
|
8834
|
+
}
|
|
8835
|
+
);
|
|
8836
|
+
});
|
|
8837
|
+
this.activeMouthMode = savedMouthMode;
|
|
8838
|
+
this.restoreMouthAfterVisemeRelease = savedRestoreAfterRelease;
|
|
8839
|
+
if (savedMouthMode === "viseme") {
|
|
8840
|
+
this.getLayerChannels(FacialPerformanceLayer.Emotion).forEach((name) => {
|
|
8841
|
+
const expression = manager.getExpression(name);
|
|
8842
|
+
if (expression) {
|
|
8843
|
+
this.originalMouthOverrides.set(name, expression.overrideMouth);
|
|
8844
|
+
expression.overrideMouth = "none";
|
|
8845
|
+
}
|
|
8846
|
+
});
|
|
8847
|
+
}
|
|
8848
|
+
this.update(0);
|
|
7760
8849
|
}
|
|
7761
8850
|
getChannelNames() {
|
|
7762
8851
|
return this.availableChannels.slice();
|
|
@@ -7858,6 +8947,19 @@ var FacialPerformanceMixer = class {
|
|
|
7858
8947
|
clearOverlay() {
|
|
7859
8948
|
this.weightMixer.clearLayerTargets(FacialPerformanceLayer.Overlay);
|
|
7860
8949
|
}
|
|
8950
|
+
/**
|
|
8951
|
+
* The attentive face of the listening reflexes (`relaxed`, else `happy`)
|
|
8952
|
+
* at `weight`. It eases in and out on its own layer, gives way to any
|
|
8953
|
+
* emotion and leaves the mouth layers alone. `immediate` with weight 0
|
|
8954
|
+
* removes it in this frame.
|
|
8955
|
+
*/
|
|
8956
|
+
setListeningFace(weight, options) {
|
|
8957
|
+
this.listeningFace.set(weight, options);
|
|
8958
|
+
}
|
|
8959
|
+
/** The preset of the attentive face, or null when the model has none. */
|
|
8960
|
+
get listeningFaceChannel() {
|
|
8961
|
+
return this.listeningFace.channelName;
|
|
8962
|
+
}
|
|
7861
8963
|
releaseVisemeMouthMode() {
|
|
7862
8964
|
if (this.activeMouthMode !== "viseme") {
|
|
7863
8965
|
return;
|
|
@@ -7937,16 +9039,24 @@ var FacialPerformanceMixer = class {
|
|
|
7937
9039
|
}
|
|
7938
9040
|
reset() {
|
|
7939
9041
|
this.weightMixer.resetTargets();
|
|
9042
|
+
this.listeningFace.set(0);
|
|
7940
9043
|
}
|
|
7941
9044
|
update(delta) {
|
|
7942
9045
|
if (!this.vrm?.expressionManager) {
|
|
7943
9046
|
return;
|
|
7944
9047
|
}
|
|
7945
9048
|
const manager = this.vrm.expressionManager;
|
|
9049
|
+
this.listeningFace.beforeUpdate(
|
|
9050
|
+
this.weightMixer.isLayerActive(FacialPerformanceLayer.Emotion)
|
|
9051
|
+
);
|
|
7946
9052
|
const values = this.weightMixer.update(delta);
|
|
7947
9053
|
values.forEach((value, name) => {
|
|
7948
9054
|
manager.setValue(name, value);
|
|
7949
9055
|
});
|
|
9056
|
+
this.listeningFace.afterUpdate(
|
|
9057
|
+
manager,
|
|
9058
|
+
(channel) => this.weightMixer.getLayerValue(FacialPerformanceLayer.Emotion, channel) + this.weightMixer.getLayerValue(FacialPerformanceLayer.Overlay, channel)
|
|
9059
|
+
);
|
|
7950
9060
|
this.updateIdleFace(delta, values);
|
|
7951
9061
|
if (this.restoreMouthAfterVisemeRelease && this.areVisemesReleased()) {
|
|
7952
9062
|
this.restoreExpressionMouthOwnership();
|
|
@@ -7989,6 +9099,12 @@ var FacialPerformanceMixer = class {
|
|
|
7989
9099
|
priority: 400,
|
|
7990
9100
|
blendMode: "override"
|
|
7991
9101
|
});
|
|
9102
|
+
this.weightMixer.registerLayer({
|
|
9103
|
+
id: LISTENING_FACE_LAYER,
|
|
9104
|
+
priority: 450,
|
|
9105
|
+
blendMode: "maximum",
|
|
9106
|
+
responseRate: 8
|
|
9107
|
+
});
|
|
7992
9108
|
this.weightMixer.registerLayer({
|
|
7993
9109
|
id: FacialPerformanceLayer.Overlay,
|
|
7994
9110
|
priority: 500,
|
|
@@ -8169,8 +9285,28 @@ var CharacterSceneRig = class {
|
|
|
8169
9285
|
get vrm() {
|
|
8170
9286
|
return this.currentVrm;
|
|
8171
9287
|
}
|
|
8172
|
-
setVrm(vrm) {
|
|
9288
|
+
setVrm(vrm, fallbackParent) {
|
|
9289
|
+
const previousVrm = this.currentVrm;
|
|
9290
|
+
const previousScene = previousVrm?.scene ?? null;
|
|
9291
|
+
const previousParent = previousScene?.parent ?? null;
|
|
8173
9292
|
this.setVrmLookAtTarget(null);
|
|
9293
|
+
if (previousScene && previousParent && previousVrm !== vrm) {
|
|
9294
|
+
previousParent.remove(previousScene);
|
|
9295
|
+
}
|
|
9296
|
+
if (previousScene && vrm?.scene && previousScene !== vrm.scene) {
|
|
9297
|
+
if (previousScene.position && vrm.scene.position?.copy) {
|
|
9298
|
+
vrm.scene.position.copy(previousScene.position);
|
|
9299
|
+
}
|
|
9300
|
+
if (previousScene.rotation && vrm.scene.rotation?.copy) {
|
|
9301
|
+
vrm.scene.rotation.copy(previousScene.rotation);
|
|
9302
|
+
}
|
|
9303
|
+
if (previousScene.scale && vrm.scene.scale?.copy) {
|
|
9304
|
+
vrm.scene.scale.copy(previousScene.scale);
|
|
9305
|
+
}
|
|
9306
|
+
if (typeof previousScene.visible === "boolean") {
|
|
9307
|
+
vrm.scene.visible = previousScene.visible;
|
|
9308
|
+
}
|
|
9309
|
+
}
|
|
8174
9310
|
this.currentVrm = vrm;
|
|
8175
9311
|
this.refreshMeshes();
|
|
8176
9312
|
if (this.shadowOptions) {
|
|
@@ -8179,6 +9315,10 @@ var CharacterSceneRig = class {
|
|
|
8179
9315
|
if (this.transformOptions) {
|
|
8180
9316
|
this.applyTransform(this.transformOptions);
|
|
8181
9317
|
}
|
|
9318
|
+
const targetParent = previousParent ?? fallbackParent ?? null;
|
|
9319
|
+
if (vrm?.scene && targetParent && vrm.scene.parent !== targetParent) {
|
|
9320
|
+
targetParent.add(vrm.scene);
|
|
9321
|
+
}
|
|
8182
9322
|
}
|
|
8183
9323
|
/**
|
|
8184
9324
|
* The `Object3D` of a `lookAt()` argument: an `Object3D` as it is, a point
|
|
@@ -8220,6 +9360,7 @@ var CharacterSceneRig = class {
|
|
|
8220
9360
|
lookAt.target = target;
|
|
8221
9361
|
} else if (this.vrmLookAtTarget && lookAt.target === this.vrmLookAtTarget) {
|
|
8222
9362
|
lookAt.target = void 0;
|
|
9363
|
+
lookAt.reset();
|
|
8223
9364
|
}
|
|
8224
9365
|
}
|
|
8225
9366
|
this.vrmLookAtTarget = lookAt ? target : null;
|
|
@@ -8361,6 +9502,22 @@ var CharacterSceneRig = class {
|
|
|
8361
9502
|
disposeVrm(vrm);
|
|
8362
9503
|
}
|
|
8363
9504
|
}
|
|
9505
|
+
disposeAllVrms(vrms) {
|
|
9506
|
+
const unique = new Set(vrms);
|
|
9507
|
+
if (this.currentVrm) {
|
|
9508
|
+
unique.add(this.currentVrm);
|
|
9509
|
+
}
|
|
9510
|
+
this.detach();
|
|
9511
|
+
this.setVrm(null);
|
|
9512
|
+
for (const vrm of unique) {
|
|
9513
|
+
if (vrm.scene?.parent) {
|
|
9514
|
+
vrm.scene.parent.remove(vrm.scene);
|
|
9515
|
+
}
|
|
9516
|
+
if (vrm.scene && typeof vrm.scene.traverse === "function") {
|
|
9517
|
+
disposeVrm(vrm);
|
|
9518
|
+
}
|
|
9519
|
+
}
|
|
9520
|
+
}
|
|
8364
9521
|
applyShadows(options) {
|
|
8365
9522
|
this.cachedMeshes.forEach((mesh) => {
|
|
8366
9523
|
mesh.castShadow = options.cast;
|
|
@@ -8388,8 +9545,274 @@ var CharacterSceneRig = class {
|
|
|
8388
9545
|
}
|
|
8389
9546
|
};
|
|
8390
9547
|
|
|
8391
|
-
// ../../clients/web-client/src/character/
|
|
9548
|
+
// ../../clients/web-client/src/character/character-scene-access.ts
|
|
9549
|
+
var CharacterSceneAccess = class {
|
|
9550
|
+
sceneRig = new CharacterSceneRig();
|
|
9551
|
+
defaultParent = null;
|
|
9552
|
+
/** True once the character VRM model is loaded. */
|
|
9553
|
+
get ready() {
|
|
9554
|
+
return this.sceneRig.vrm !== null;
|
|
9555
|
+
}
|
|
9556
|
+
/** Root Three.js `Group` (`vrm.scene`) to add to a `THREE.Scene`. */
|
|
9557
|
+
get scene() {
|
|
9558
|
+
return this.sceneRig.scene;
|
|
9559
|
+
}
|
|
9560
|
+
/** Alias for `scene` for Three.js `Object3D` workflows. */
|
|
9561
|
+
get object3d() {
|
|
9562
|
+
return this.sceneRig.object3d;
|
|
9563
|
+
}
|
|
9564
|
+
/** Alias for `scene`. */
|
|
9565
|
+
get root() {
|
|
9566
|
+
return this.sceneRig.root;
|
|
9567
|
+
}
|
|
9568
|
+
/** Primary character mesh (`SkinnedMesh` preferred, otherwise first `Mesh`). */
|
|
9569
|
+
get mesh() {
|
|
9570
|
+
return this.sceneRig.mesh;
|
|
9571
|
+
}
|
|
9572
|
+
/** All Three.js `Mesh` instances in the character hierarchy. */
|
|
9573
|
+
get meshes() {
|
|
9574
|
+
return this.sceneRig.meshes;
|
|
9575
|
+
}
|
|
9576
|
+
/** All Three.js `SkinnedMesh` instances in the character hierarchy. */
|
|
9577
|
+
get skinnedMeshes() {
|
|
9578
|
+
return this.sceneRig.skinnedMeshes;
|
|
9579
|
+
}
|
|
9580
|
+
/** Deduplicated Three.js `Material` instances across all character meshes. */
|
|
9581
|
+
get materials() {
|
|
9582
|
+
return this.sceneRig.materials;
|
|
9583
|
+
}
|
|
9584
|
+
/** Deduplicated Three.js `Skeleton` instances across all character skinned meshes. */
|
|
9585
|
+
get skeletons() {
|
|
9586
|
+
return this.sceneRig.skeletons;
|
|
9587
|
+
}
|
|
9588
|
+
/** Three-VRM humanoid rig when loaded. */
|
|
9589
|
+
get humanoid() {
|
|
9590
|
+
return this.sceneRig.humanoid;
|
|
9591
|
+
}
|
|
9592
|
+
/** Look up a raw humanoid bone `Object3D` by VRM bone name (for example `'head'` or `'rightHand'`). */
|
|
9593
|
+
getBone(name) {
|
|
9594
|
+
return this.sceneRig.getBone(name);
|
|
9595
|
+
}
|
|
9596
|
+
/** Look up a normalized humanoid bone `Object3D` by VRM bone name. */
|
|
9597
|
+
getNormalizedBone(name) {
|
|
9598
|
+
return this.sceneRig.getNormalizedBone(name);
|
|
9599
|
+
}
|
|
9600
|
+
/** Attach the character root `Group` to a Three.js `Scene` or parent `Object3D`. */
|
|
9601
|
+
attachTo(parent) {
|
|
9602
|
+
this.defaultParent = parent;
|
|
9603
|
+
return this.sceneRig.attachTo(parent);
|
|
9604
|
+
}
|
|
9605
|
+
/** Detach the character root `Group` from its current parent `Object3D`. */
|
|
9606
|
+
detach() {
|
|
9607
|
+
this.defaultParent = null;
|
|
9608
|
+
this.sceneRig.detach();
|
|
9609
|
+
}
|
|
9610
|
+
/** Configure `castShadow` and `receiveShadow` on all character meshes. */
|
|
9611
|
+
setShadows(options) {
|
|
9612
|
+
this.sceneRig.setShadows(options);
|
|
9613
|
+
}
|
|
9614
|
+
/** Configure `position`, `rotation`, and `scale` on the character root `Group`. */
|
|
9615
|
+
setTransform(options) {
|
|
9616
|
+
this.sceneRig.setTransform(options);
|
|
9617
|
+
}
|
|
9618
|
+
/** Re-index cached meshes, materials, and skeletons after custom scene graph edits. */
|
|
9619
|
+
refreshMeshes() {
|
|
9620
|
+
this.sceneRig.refreshMeshes();
|
|
9621
|
+
}
|
|
9622
|
+
};
|
|
9623
|
+
|
|
9624
|
+
// ../../clients/web-client/src/character/listening-reflex.ts
|
|
9625
|
+
import { Object3D as Object3D2, Vector3 as Vector38 } from "three";
|
|
9626
|
+
|
|
9627
|
+
// ../../clients/web-client/src/character/head-nod.ts
|
|
8392
9628
|
import { Quaternion as Quaternion3, Vector3 as Vector37 } from "three";
|
|
9629
|
+
var SHARES = [
|
|
9630
|
+
["neck", 0.4],
|
|
9631
|
+
["head", 0.6]
|
|
9632
|
+
];
|
|
9633
|
+
var DAMPING = 0.6;
|
|
9634
|
+
var FREQUENCY_TIMES_LENGTH = 1.4;
|
|
9635
|
+
var PULSE_PART = 0.3;
|
|
9636
|
+
var MAX_STEP_SECONDS = 1 / 240;
|
|
9637
|
+
var REST_DEGREES = 0.01;
|
|
9638
|
+
var X_AXIS = new Vector37(1, 0, 0);
|
|
9639
|
+
var HeadNod = class {
|
|
9640
|
+
pitch = 0;
|
|
9641
|
+
speed = 0;
|
|
9642
|
+
elapsed = 0;
|
|
9643
|
+
pulseSeconds = 0;
|
|
9644
|
+
amplitude = 0;
|
|
9645
|
+
omega = 0;
|
|
9646
|
+
applied = [];
|
|
9647
|
+
scratch = new Quaternion3();
|
|
9648
|
+
/** The current pitch in degrees; positive turns the face down. */
|
|
9649
|
+
get pitchDegrees() {
|
|
9650
|
+
return this.pitch;
|
|
9651
|
+
}
|
|
9652
|
+
get active() {
|
|
9653
|
+
return this.elapsed < this.pulseSeconds || Math.abs(this.pitch) > REST_DEGREES || Math.abs(this.speed) > REST_DEGREES;
|
|
9654
|
+
}
|
|
9655
|
+
start(nod) {
|
|
9656
|
+
const seconds = Math.max(nod.durationMs, 1) / 1e3;
|
|
9657
|
+
this.amplitude = nod.amplitudeDegrees;
|
|
9658
|
+
this.omega = 2 * Math.PI * FREQUENCY_TIMES_LENGTH / seconds;
|
|
9659
|
+
this.pulseSeconds = seconds * PULSE_PART;
|
|
9660
|
+
this.elapsed = 0;
|
|
9661
|
+
}
|
|
9662
|
+
/** Stops the nod at once; the next `apply` removes its pitch. */
|
|
9663
|
+
cancel() {
|
|
9664
|
+
this.pitch = 0;
|
|
9665
|
+
this.speed = 0;
|
|
9666
|
+
this.elapsed = 0;
|
|
9667
|
+
this.pulseSeconds = 0;
|
|
9668
|
+
}
|
|
9669
|
+
update(delta) {
|
|
9670
|
+
if (!(delta > 0) || !this.active) {
|
|
9671
|
+
if (!this.active) {
|
|
9672
|
+
this.pitch = 0;
|
|
9673
|
+
this.speed = 0;
|
|
9674
|
+
}
|
|
9675
|
+
return this.pitch;
|
|
9676
|
+
}
|
|
9677
|
+
let left = Math.min(delta, 0.25);
|
|
9678
|
+
while (left > 0) {
|
|
9679
|
+
const step = Math.min(left, MAX_STEP_SECONDS);
|
|
9680
|
+
const target = this.elapsed < this.pulseSeconds ? this.amplitude : 0;
|
|
9681
|
+
const omega = this.omega;
|
|
9682
|
+
const acceleration = omega * omega * (target - this.pitch) - 2 * DAMPING * omega * this.speed;
|
|
9683
|
+
this.speed += acceleration * step;
|
|
9684
|
+
this.pitch += this.speed * step;
|
|
9685
|
+
this.elapsed += step;
|
|
9686
|
+
left -= step;
|
|
9687
|
+
}
|
|
9688
|
+
if (!this.active) {
|
|
9689
|
+
this.pitch = 0;
|
|
9690
|
+
this.speed = 0;
|
|
9691
|
+
}
|
|
9692
|
+
return this.pitch;
|
|
9693
|
+
}
|
|
9694
|
+
/** Writes the current pitch onto the normalized neck and head of `vrm`. */
|
|
9695
|
+
apply(vrm) {
|
|
9696
|
+
this.applied.forEach(({ node, base, result }) => {
|
|
9697
|
+
if (node.quaternion.equals(result)) {
|
|
9698
|
+
node.quaternion.copy(base);
|
|
9699
|
+
}
|
|
9700
|
+
});
|
|
9701
|
+
this.applied = [];
|
|
9702
|
+
const humanoid = vrm?.humanoid;
|
|
9703
|
+
if (!humanoid || this.pitch === 0) {
|
|
9704
|
+
return;
|
|
9705
|
+
}
|
|
9706
|
+
const flip = vrm.meta.metaVersion === "0" && !isNpcifyNormalizedVrm(vrm) ? -1 : 1;
|
|
9707
|
+
const radians = this.pitch * Math.PI / 180 * flip;
|
|
9708
|
+
let carried = 0;
|
|
9709
|
+
SHARES.forEach(([bone, share]) => {
|
|
9710
|
+
const node = humanoid.getNormalizedBoneNode(bone);
|
|
9711
|
+
if (!node) {
|
|
9712
|
+
carried += share;
|
|
9713
|
+
return;
|
|
9714
|
+
}
|
|
9715
|
+
const angle = radians * (share + carried);
|
|
9716
|
+
carried = 0;
|
|
9717
|
+
const base = node.quaternion.clone();
|
|
9718
|
+
node.quaternion.multiply(this.scratch.setFromAxisAngle(X_AXIS, angle));
|
|
9719
|
+
this.applied.push({ node, base, result: node.quaternion.clone() });
|
|
9720
|
+
});
|
|
9721
|
+
}
|
|
9722
|
+
};
|
|
9723
|
+
|
|
9724
|
+
// ../../clients/web-client/src/character/listening-reflex.ts
|
|
9725
|
+
var LISTENING_FACE_WEIGHT = 0.2;
|
|
9726
|
+
var FRONT_DISTANCE = 1.5;
|
|
9727
|
+
var ListeningReflex = class {
|
|
9728
|
+
constructor(host, activity) {
|
|
9729
|
+
this.host = host;
|
|
9730
|
+
this.activity = activity;
|
|
9731
|
+
activity.onNod((nod) => {
|
|
9732
|
+
this.pendingNod = nod;
|
|
9733
|
+
});
|
|
9734
|
+
}
|
|
9735
|
+
nod = new HeadNod();
|
|
9736
|
+
pendingNod = null;
|
|
9737
|
+
releaseGaze = null;
|
|
9738
|
+
front = new Object3D2();
|
|
9739
|
+
origin = new Vector38();
|
|
9740
|
+
forward = new Vector38();
|
|
9741
|
+
yielded = false;
|
|
9742
|
+
/** Call after the animation mixer and live motion, before the facial mixer and `vrm.update()`. */
|
|
9743
|
+
update(delta, frame) {
|
|
9744
|
+
const vrm = this.host.vrm();
|
|
9745
|
+
if (frame.stagingActive) {
|
|
9746
|
+
if (!this.yielded) {
|
|
9747
|
+
this.yielded = true;
|
|
9748
|
+
this.host.facialMixer.setListeningFace(0, { immediate: true });
|
|
9749
|
+
this.nod.cancel();
|
|
9750
|
+
this.endGaze();
|
|
9751
|
+
}
|
|
9752
|
+
this.pendingNod = null;
|
|
9753
|
+
this.nod.apply(vrm);
|
|
9754
|
+
return;
|
|
9755
|
+
}
|
|
9756
|
+
this.yielded = false;
|
|
9757
|
+
const engaged = this.activity.state !== "idle" && vrm !== null;
|
|
9758
|
+
this.host.facialMixer.setListeningFace(engaged ? LISTENING_FACE_WEIGHT : 0);
|
|
9759
|
+
if (engaged) {
|
|
9760
|
+
this.beginGaze(vrm);
|
|
9761
|
+
} else {
|
|
9762
|
+
this.endGaze();
|
|
9763
|
+
}
|
|
9764
|
+
const nod = this.pendingNod;
|
|
9765
|
+
this.pendingNod = null;
|
|
9766
|
+
if (frame.liveMotionWeight > 0) {
|
|
9767
|
+
this.nod.cancel();
|
|
9768
|
+
} else if (nod && this.activity.state === "speaking" && vrm) {
|
|
9769
|
+
this.nod.start(nod);
|
|
9770
|
+
}
|
|
9771
|
+
this.nod.update(delta);
|
|
9772
|
+
this.nod.apply(vrm);
|
|
9773
|
+
}
|
|
9774
|
+
/** A new model or the end of the conversation: no gaze, face or nod remains. */
|
|
9775
|
+
reset() {
|
|
9776
|
+
this.pendingNod = null;
|
|
9777
|
+
this.nod.cancel();
|
|
9778
|
+
this.nod.apply(null);
|
|
9779
|
+
this.endGaze();
|
|
9780
|
+
this.host.facialMixer.setListeningFace(0, { immediate: true });
|
|
9781
|
+
}
|
|
9782
|
+
beginGaze(vrm) {
|
|
9783
|
+
if (!this.releaseGaze) {
|
|
9784
|
+
if (this.host.lookAtActive()) {
|
|
9785
|
+
return;
|
|
9786
|
+
}
|
|
9787
|
+
const target = this.host.gazeTarget() ?? this.front;
|
|
9788
|
+
this.releaseGaze = this.host.lookAtTemporarily(target);
|
|
9789
|
+
}
|
|
9790
|
+
this.placeFront(vrm);
|
|
9791
|
+
}
|
|
9792
|
+
endGaze() {
|
|
9793
|
+
const release = this.releaseGaze;
|
|
9794
|
+
this.releaseGaze = null;
|
|
9795
|
+
release?.();
|
|
9796
|
+
}
|
|
9797
|
+
/** The point in front of the head, along the character's forward axis. */
|
|
9798
|
+
placeFront(vrm) {
|
|
9799
|
+
const head = vrm.humanoid?.getNormalizedBoneNode("head");
|
|
9800
|
+
if (!head) {
|
|
9801
|
+
return;
|
|
9802
|
+
}
|
|
9803
|
+
const flip = vrm.meta.metaVersion === "0" && !isNpcifyNormalizedVrm(vrm);
|
|
9804
|
+
const root = vrm.scene;
|
|
9805
|
+
const origin = root.getWorldPosition(this.origin);
|
|
9806
|
+
const forward = root.localToWorld(
|
|
9807
|
+
this.forward.set(0, 0, flip ? -FRONT_DISTANCE : FRONT_DISTANCE)
|
|
9808
|
+
).sub(origin);
|
|
9809
|
+
head.getWorldPosition(this.front.position).add(forward);
|
|
9810
|
+
this.front.updateMatrixWorld(true);
|
|
9811
|
+
}
|
|
9812
|
+
};
|
|
9813
|
+
|
|
9814
|
+
// ../../clients/web-client/src/character/live-motion.ts
|
|
9815
|
+
import { Quaternion as Quaternion4, Vector3 as Vector39 } from "three";
|
|
8393
9816
|
var PARENTS = {
|
|
8394
9817
|
spine: "hips",
|
|
8395
9818
|
chest: "spine",
|
|
@@ -8431,10 +9854,10 @@ var LiveMotionPlayer = class {
|
|
|
8431
9854
|
/** The last applied pose, to blend out of it after the frames end. */
|
|
8432
9855
|
lastPose = null;
|
|
8433
9856
|
scratch = {
|
|
8434
|
-
from: new
|
|
8435
|
-
to: new
|
|
9857
|
+
from: new Quaternion4(),
|
|
9858
|
+
to: new Quaternion4(),
|
|
8436
9859
|
carry: /* @__PURE__ */ new Map(),
|
|
8437
|
-
position: new
|
|
9860
|
+
position: new Vector39()
|
|
8438
9861
|
};
|
|
8439
9862
|
constructor(options = {}) {
|
|
8440
9863
|
this.rootMode = options.rootMode ?? "anchored";
|
|
@@ -8596,7 +10019,7 @@ var LiveMotionPlayer = class {
|
|
|
8596
10019
|
carry.clear();
|
|
8597
10020
|
track.bones.forEach((bone, slot) => {
|
|
8598
10021
|
const offset = 3 + slot * 4;
|
|
8599
|
-
const rotation = new
|
|
10022
|
+
const rotation = new Quaternion4(
|
|
8600
10023
|
values[offset] ?? 0,
|
|
8601
10024
|
values[offset + 1] ?? 0,
|
|
8602
10025
|
values[offset + 2] ?? 0,
|
|
@@ -8643,6 +10066,166 @@ var LiveMotionPlayer = class {
|
|
|
8643
10066
|
}
|
|
8644
10067
|
};
|
|
8645
10068
|
|
|
10069
|
+
// ../../clients/web-client/src/voice/speech-nod-trigger.ts
|
|
10070
|
+
var MIN_SPEECH_LEVEL = 0.015;
|
|
10071
|
+
var SPEECH_TO_NOISE = 3;
|
|
10072
|
+
var NOD_PAUSE_MS = 250;
|
|
10073
|
+
var NOD_MIN_SPEECH_MS = 1200;
|
|
10074
|
+
var NOD_MIN_INTERVAL_MS = 1800;
|
|
10075
|
+
var NOD_INTERVAL_JITTER_MS = 1400;
|
|
10076
|
+
var NOD_SKIP_CHANCE = 0.2;
|
|
10077
|
+
var NOD_MIN_DEGREES = 4;
|
|
10078
|
+
var NOD_MAX_DEGREES = 6;
|
|
10079
|
+
var NOD_MIN_DURATION_MS = 320;
|
|
10080
|
+
var NOD_MAX_DURATION_MS = 400;
|
|
10081
|
+
var SpeechNodTrigger = class {
|
|
10082
|
+
constructor(random = Math.random) {
|
|
10083
|
+
this.random = random;
|
|
10084
|
+
this.intervalMs = this.drawInterval();
|
|
10085
|
+
}
|
|
10086
|
+
floor = null;
|
|
10087
|
+
speechMs = 0;
|
|
10088
|
+
quietMs = 0;
|
|
10089
|
+
sinceNodMs = Number.POSITIVE_INFINITY;
|
|
10090
|
+
intervalMs;
|
|
10091
|
+
/** True while the last frame was loud enough to be speech. */
|
|
10092
|
+
get voiced() {
|
|
10093
|
+
return this.speechMs > 0 && this.quietMs === 0;
|
|
10094
|
+
}
|
|
10095
|
+
/** Reads one frame; returns a nod when one is due now. */
|
|
10096
|
+
push(level, frameMs = SPEECH_FRAME_MS) {
|
|
10097
|
+
if (!Number.isFinite(level) || !(frameMs > 0)) {
|
|
10098
|
+
return null;
|
|
10099
|
+
}
|
|
10100
|
+
this.sinceNodMs += frameMs;
|
|
10101
|
+
const threshold = Math.max(
|
|
10102
|
+
MIN_SPEECH_LEVEL,
|
|
10103
|
+
(this.floor ?? MIN_SPEECH_LEVEL / SPEECH_TO_NOISE) * SPEECH_TO_NOISE
|
|
10104
|
+
);
|
|
10105
|
+
if (level >= threshold) {
|
|
10106
|
+
this.speechMs += this.quietMs + frameMs;
|
|
10107
|
+
this.quietMs = 0;
|
|
10108
|
+
return null;
|
|
10109
|
+
}
|
|
10110
|
+
this.floor = this.floor === null ? level : this.floor * 0.9 + level * 0.1;
|
|
10111
|
+
if (this.speechMs === 0) {
|
|
10112
|
+
return null;
|
|
10113
|
+
}
|
|
10114
|
+
this.quietMs += frameMs;
|
|
10115
|
+
if (this.quietMs < NOD_PAUSE_MS) {
|
|
10116
|
+
return null;
|
|
10117
|
+
}
|
|
10118
|
+
const run = this.speechMs;
|
|
10119
|
+
this.speechMs = 0;
|
|
10120
|
+
this.quietMs = 0;
|
|
10121
|
+
if (run < NOD_MIN_SPEECH_MS || this.sinceNodMs < this.intervalMs) {
|
|
10122
|
+
return null;
|
|
10123
|
+
}
|
|
10124
|
+
if (this.random() < NOD_SKIP_CHANCE) {
|
|
10125
|
+
return null;
|
|
10126
|
+
}
|
|
10127
|
+
this.sinceNodMs = 0;
|
|
10128
|
+
this.intervalMs = this.drawInterval();
|
|
10129
|
+
return {
|
|
10130
|
+
amplitudeDegrees: NOD_MIN_DEGREES + (NOD_MAX_DEGREES - NOD_MIN_DEGREES) * this.random(),
|
|
10131
|
+
durationMs: NOD_MIN_DURATION_MS + (NOD_MAX_DURATION_MS - NOD_MIN_DURATION_MS) * this.random()
|
|
10132
|
+
};
|
|
10133
|
+
}
|
|
10134
|
+
/** A new listening session: the speech run starts again; the noise floor stays. */
|
|
10135
|
+
reset() {
|
|
10136
|
+
this.speechMs = 0;
|
|
10137
|
+
this.quietMs = 0;
|
|
10138
|
+
this.sinceNodMs = Number.POSITIVE_INFINITY;
|
|
10139
|
+
}
|
|
10140
|
+
drawInterval() {
|
|
10141
|
+
return NOD_MIN_INTERVAL_MS + NOD_INTERVAL_JITTER_MS * this.random();
|
|
10142
|
+
}
|
|
10143
|
+
};
|
|
10144
|
+
|
|
10145
|
+
// ../../clients/web-client/src/character/player-activity.ts
|
|
10146
|
+
var TYPING_IDLE_MS = 1500;
|
|
10147
|
+
var PlayerActivityTracker = class {
|
|
10148
|
+
listening = false;
|
|
10149
|
+
typing = false;
|
|
10150
|
+
typingTimer = null;
|
|
10151
|
+
current = "idle";
|
|
10152
|
+
trigger;
|
|
10153
|
+
changeListeners = /* @__PURE__ */ new Set();
|
|
10154
|
+
nodListeners = /* @__PURE__ */ new Set();
|
|
10155
|
+
constructor(options = {}) {
|
|
10156
|
+
this.trigger = new SpeechNodTrigger(options.random);
|
|
10157
|
+
}
|
|
10158
|
+
get state() {
|
|
10159
|
+
return this.current;
|
|
10160
|
+
}
|
|
10161
|
+
/** The listening session is (or is no longer) in state `listening`. */
|
|
10162
|
+
setListening(active) {
|
|
10163
|
+
if (active === this.listening) {
|
|
10164
|
+
return;
|
|
10165
|
+
}
|
|
10166
|
+
this.listening = active;
|
|
10167
|
+
this.trigger.reset();
|
|
10168
|
+
this.resolve();
|
|
10169
|
+
}
|
|
10170
|
+
/**
|
|
10171
|
+
* Input in a text field: `true` at each input event, `false` when the
|
|
10172
|
+
* player sends or leaves the field. Without input, typing ends after 1.5 s.
|
|
10173
|
+
*/
|
|
10174
|
+
notifyTyping(active) {
|
|
10175
|
+
this.clearTypingTimer();
|
|
10176
|
+
this.typing = active;
|
|
10177
|
+
if (active) {
|
|
10178
|
+
this.typingTimer = setTimeout(() => {
|
|
10179
|
+
this.typingTimer = null;
|
|
10180
|
+
this.typing = false;
|
|
10181
|
+
this.resolve();
|
|
10182
|
+
}, TYPING_IDLE_MS);
|
|
10183
|
+
}
|
|
10184
|
+
this.resolve();
|
|
10185
|
+
}
|
|
10186
|
+
/** One microphone frame of the listening session; returns a nod when one is due. */
|
|
10187
|
+
pushVoiceLevel(level, frameMs) {
|
|
10188
|
+
if (!this.listening) {
|
|
10189
|
+
return null;
|
|
10190
|
+
}
|
|
10191
|
+
const nod = this.trigger.push(level, frameMs);
|
|
10192
|
+
if (nod) {
|
|
10193
|
+
this.nodListeners.forEach((listener) => listener(nod));
|
|
10194
|
+
}
|
|
10195
|
+
return nod;
|
|
10196
|
+
}
|
|
10197
|
+
onChange(listener) {
|
|
10198
|
+
this.changeListeners.add(listener);
|
|
10199
|
+
return () => this.changeListeners.delete(listener);
|
|
10200
|
+
}
|
|
10201
|
+
onNod(listener) {
|
|
10202
|
+
this.nodListeners.add(listener);
|
|
10203
|
+
return () => this.nodListeners.delete(listener);
|
|
10204
|
+
}
|
|
10205
|
+
/** Ends all activity, for example when the conversation ends. */
|
|
10206
|
+
reset() {
|
|
10207
|
+
this.clearTypingTimer();
|
|
10208
|
+
this.typing = false;
|
|
10209
|
+
this.listening = false;
|
|
10210
|
+
this.trigger.reset();
|
|
10211
|
+
this.resolve();
|
|
10212
|
+
}
|
|
10213
|
+
resolve() {
|
|
10214
|
+
const next = this.listening ? "speaking" : this.typing ? "typing" : "idle";
|
|
10215
|
+
if (next === this.current) {
|
|
10216
|
+
return;
|
|
10217
|
+
}
|
|
10218
|
+
this.current = next;
|
|
10219
|
+
this.changeListeners.forEach((listener) => listener(next));
|
|
10220
|
+
}
|
|
10221
|
+
clearTypingTimer() {
|
|
10222
|
+
if (this.typingTimer !== null) {
|
|
10223
|
+
clearTimeout(this.typingTimer);
|
|
10224
|
+
this.typingTimer = null;
|
|
10225
|
+
}
|
|
10226
|
+
}
|
|
10227
|
+
};
|
|
10228
|
+
|
|
8646
10229
|
// ../../clients/web-client/src/character/incremental-staging.ts
|
|
8647
10230
|
var dialogueKey = function(block) {
|
|
8648
10231
|
return JSON.stringify(block, (key, value) => {
|
|
@@ -8918,6 +10501,9 @@ var StagingPlayer = class {
|
|
|
8918
10501
|
activeExpressionMouthMode = null;
|
|
8919
10502
|
activeAnimation = null;
|
|
8920
10503
|
currentTrackTime = null;
|
|
10504
|
+
playbackElapsed = 0;
|
|
10505
|
+
appliedToolCallIds = /* @__PURE__ */ new Set();
|
|
10506
|
+
activeOptions;
|
|
8921
10507
|
get paused() {
|
|
8922
10508
|
return this.clock.paused;
|
|
8923
10509
|
}
|
|
@@ -8971,6 +10557,9 @@ var StagingPlayer = class {
|
|
|
8971
10557
|
this.incremental = incremental;
|
|
8972
10558
|
timeline = incremental.timeline;
|
|
8973
10559
|
this.onCursor = options?.onCursor;
|
|
10560
|
+
this.activeOptions = options;
|
|
10561
|
+
this.playbackElapsed = 0;
|
|
10562
|
+
this.appliedToolCallIds.clear();
|
|
8974
10563
|
let timelineStarted = false;
|
|
8975
10564
|
const beginTimeline = () => {
|
|
8976
10565
|
if (timelineStarted) {
|
|
@@ -8980,6 +10569,14 @@ var StagingPlayer = class {
|
|
|
8980
10569
|
options?.onTimelineStart?.(timeline);
|
|
8981
10570
|
};
|
|
8982
10571
|
try {
|
|
10572
|
+
await options?.onVariantCue?.(
|
|
10573
|
+
timeline.initialVariant ?? DEFAULT_VARIANT_KEY,
|
|
10574
|
+
null
|
|
10575
|
+
);
|
|
10576
|
+
if (controller.signal.aborted) {
|
|
10577
|
+
throw abortError2();
|
|
10578
|
+
}
|
|
10579
|
+
this.applyToolCalls(0, timeline);
|
|
8983
10580
|
let cursor = 0;
|
|
8984
10581
|
for (let index = 0; ; index += 1) {
|
|
8985
10582
|
const block = await incremental.next(index, controller.signal, () => {
|
|
@@ -9029,6 +10626,7 @@ var StagingPlayer = class {
|
|
|
9029
10626
|
}
|
|
9030
10627
|
await incremental.finishBlock(index, controller.signal);
|
|
9031
10628
|
cursor = block.endTime;
|
|
10629
|
+
this.playbackElapsed = Math.max(this.playbackElapsed, cursor);
|
|
9032
10630
|
}
|
|
9033
10631
|
if (timeline.duration > cursor) {
|
|
9034
10632
|
await this.animationController.prepareAnimations?.(
|
|
@@ -9050,10 +10648,15 @@ var StagingPlayer = class {
|
|
|
9050
10648
|
beginTimeline();
|
|
9051
10649
|
}
|
|
9052
10650
|
this.applyTracks(timeline.duration, false, timeline);
|
|
10651
|
+
if (timeline.activeVariant) {
|
|
10652
|
+
await options?.onVariantCue?.(timeline.activeVariant, null);
|
|
10653
|
+
}
|
|
9053
10654
|
} finally {
|
|
9054
10655
|
if (this.generation === generation) {
|
|
9055
10656
|
this.playbackController = null;
|
|
9056
10657
|
this.incremental = null;
|
|
10658
|
+
this.activeOptions = void 0;
|
|
10659
|
+
this.appliedToolCallIds.clear();
|
|
9057
10660
|
this.clearCursor();
|
|
9058
10661
|
this.onCursor = void 0;
|
|
9059
10662
|
this.clearPerformance();
|
|
@@ -9062,6 +10665,10 @@ var StagingPlayer = class {
|
|
|
9062
10665
|
}
|
|
9063
10666
|
updateTimeline(timeline, complete = true) {
|
|
9064
10667
|
this.incremental?.update(timeline, complete);
|
|
10668
|
+
if (this.incremental) {
|
|
10669
|
+
const elapsed = this.currentTrackTime ?? this.playbackElapsed;
|
|
10670
|
+
this.applyToolCalls(elapsed, this.incremental.timeline);
|
|
10671
|
+
}
|
|
9065
10672
|
const current = this.incremental?.timeline.dialogue.find(
|
|
9066
10673
|
(block) => block.id === this.pendingSpeech?.id
|
|
9067
10674
|
);
|
|
@@ -9112,6 +10719,9 @@ var StagingPlayer = class {
|
|
|
9112
10719
|
this.playbackController = null;
|
|
9113
10720
|
this.clock.resume();
|
|
9114
10721
|
this.incremental = null;
|
|
10722
|
+
this.activeOptions = void 0;
|
|
10723
|
+
this.playbackElapsed = 0;
|
|
10724
|
+
this.appliedToolCallIds.clear();
|
|
9115
10725
|
this.clearCursor();
|
|
9116
10726
|
this.onCursor = void 0;
|
|
9117
10727
|
this.audioPlayer.stop();
|
|
@@ -9327,8 +10937,29 @@ var StagingPlayer = class {
|
|
|
9327
10937
|
await waitFrame(signal);
|
|
9328
10938
|
}
|
|
9329
10939
|
}
|
|
10940
|
+
applyToolCalls(elapsed, timeline) {
|
|
10941
|
+
const calls = timeline.toolCalls;
|
|
10942
|
+
if (!calls || calls.length === 0) {
|
|
10943
|
+
return;
|
|
10944
|
+
}
|
|
10945
|
+
const ordered = calls.length > 1 ? [...calls].sort((left, right) => left.timestamp - right.timestamp) : calls;
|
|
10946
|
+
for (const call of ordered) {
|
|
10947
|
+
if (this.appliedToolCallIds.has(call.id) || elapsed < call.timestamp) {
|
|
10948
|
+
continue;
|
|
10949
|
+
}
|
|
10950
|
+
this.appliedToolCallIds.add(call.id);
|
|
10951
|
+
this.activeOptions?.onToolCall?.(call);
|
|
10952
|
+
if (call.name === "switch_variant" && call.result.ok) {
|
|
10953
|
+
void Promise.resolve(
|
|
10954
|
+
this.activeOptions?.onVariantCue?.(call.result.activeVariant, call)
|
|
10955
|
+
).catch(() => void 0);
|
|
10956
|
+
}
|
|
10957
|
+
}
|
|
10958
|
+
}
|
|
9330
10959
|
applyTracks(time, speechActive, timeline) {
|
|
9331
10960
|
this.currentTrackTime = time;
|
|
10961
|
+
this.playbackElapsed = Math.max(this.playbackElapsed, time);
|
|
10962
|
+
this.applyToolCalls(time, timeline);
|
|
9332
10963
|
const expression = activeClipAt(timeline.expressions, time);
|
|
9333
10964
|
const mouthMode = speechActive ? "viseme" : "expression";
|
|
9334
10965
|
if (expression !== this.activeExpression || expression && mouthMode !== this.activeExpressionMouthMode) {
|
|
@@ -9782,7 +11413,7 @@ var VisemePlayer = class {
|
|
|
9782
11413
|
};
|
|
9783
11414
|
|
|
9784
11415
|
// ../../clients/web-client/src/character/character-runtime.ts
|
|
9785
|
-
var CharacterRuntime = class {
|
|
11416
|
+
var CharacterRuntime = class extends CharacterSceneAccess {
|
|
9786
11417
|
baseUrl;
|
|
9787
11418
|
characterId;
|
|
9788
11419
|
apiToken;
|
|
@@ -9792,8 +11423,8 @@ var CharacterRuntime = class {
|
|
|
9792
11423
|
animation;
|
|
9793
11424
|
animations = /* @__PURE__ */ new Map();
|
|
9794
11425
|
animationController;
|
|
9795
|
-
sceneRig = new CharacterSceneRig();
|
|
9796
11426
|
lazyAnimations;
|
|
11427
|
+
variantController;
|
|
9797
11428
|
audioPlayer = null;
|
|
9798
11429
|
stagingPlayer = null;
|
|
9799
11430
|
facialMixer;
|
|
@@ -9802,13 +11433,15 @@ var CharacterRuntime = class {
|
|
|
9802
11433
|
telemetry;
|
|
9803
11434
|
streamPlayback;
|
|
9804
11435
|
liveMotion;
|
|
9805
|
-
|
|
11436
|
+
playerActivityTracker = new PlayerActivityTracker();
|
|
11437
|
+
listeningReflex;
|
|
9806
11438
|
currentGazeTarget = null;
|
|
9807
11439
|
/** The current gaze target came from `lookAt()`. */
|
|
9808
11440
|
lookAtActive = false;
|
|
9809
11441
|
thinkingState = false;
|
|
9810
11442
|
errorCallback;
|
|
9811
11443
|
constructor(options) {
|
|
11444
|
+
super();
|
|
9812
11445
|
this.baseUrl = options.baseUrl;
|
|
9813
11446
|
this.characterId = options.characterId;
|
|
9814
11447
|
this.apiToken = options.apiToken;
|
|
@@ -9822,6 +11455,13 @@ var CharacterRuntime = class {
|
|
|
9822
11455
|
animationId,
|
|
9823
11456
|
assetProfile: this.assetProfile
|
|
9824
11457
|
}));
|
|
11458
|
+
const loadVariant = options.loadVariant ?? ((variantKey) => loadVariantVrm({
|
|
11459
|
+
baseUrl: this.baseUrl,
|
|
11460
|
+
characterId: this.characterId,
|
|
11461
|
+
variantKey,
|
|
11462
|
+
apiToken: this.apiToken,
|
|
11463
|
+
assetProfile: this.assetProfile
|
|
11464
|
+
}));
|
|
9825
11465
|
this.lazyAnimations = new LazyAnimations(loadAnimation, (id, animation) => {
|
|
9826
11466
|
this.animations.set(id, animation);
|
|
9827
11467
|
this.animationController?.addAnimation?.(id, animation, { loop: false });
|
|
@@ -9851,6 +11491,36 @@ var CharacterRuntime = class {
|
|
|
9851
11491
|
apiToken: this.apiToken
|
|
9852
11492
|
});
|
|
9853
11493
|
this.liveMotion = new LiveMotionPlayer(options.liveMotion);
|
|
11494
|
+
this.listeningReflex = new ListeningReflex(
|
|
11495
|
+
{
|
|
11496
|
+
vrm: () => this.sceneRig.vrm,
|
|
11497
|
+
gazeTarget: () => this.currentGazeTarget,
|
|
11498
|
+
lookAtActive: () => this.lookAtActive,
|
|
11499
|
+
lookAtTemporarily: (target) => this.lookAtTemporarily(target),
|
|
11500
|
+
facialMixer: this.facialMixer
|
|
11501
|
+
},
|
|
11502
|
+
this.playerActivityTracker
|
|
11503
|
+
);
|
|
11504
|
+
this.variantController = new CharacterVariantController({
|
|
11505
|
+
loadVariant,
|
|
11506
|
+
getCurrentVrm: () => this.sceneRig.vrm,
|
|
11507
|
+
handleCallbackError: this.handleCallbackError,
|
|
11508
|
+
applyVrmSwap: (nextVrm) => {
|
|
11509
|
+
this.sceneRig.setVrm(nextVrm, this.defaultParent);
|
|
11510
|
+
this.facialMixer.setVrm(nextVrm, { preserveState: true });
|
|
11511
|
+
this.animationController?.setVrm?.(nextVrm, this.animations);
|
|
11512
|
+
this.liveMotion.setVrm(nextVrm);
|
|
11513
|
+
this.applyGaze();
|
|
11514
|
+
if (this.performanceRecorder.isRecording) {
|
|
11515
|
+
this.facialMixer.capturePerformanceModel();
|
|
11516
|
+
}
|
|
11517
|
+
}
|
|
11518
|
+
});
|
|
11519
|
+
if (options.initialAssetInfo?.variants) {
|
|
11520
|
+
this.variantController.configureVariants(
|
|
11521
|
+
options.initialAssetInfo.variants
|
|
11522
|
+
);
|
|
11523
|
+
}
|
|
9854
11524
|
this.streamPlayback = new StreamPlaybackController({
|
|
9855
11525
|
getStagingPlayer: this.getStagingPlayer,
|
|
9856
11526
|
telemetry: this.telemetry,
|
|
@@ -9864,6 +11534,19 @@ var CharacterRuntime = class {
|
|
|
9864
11534
|
void this.liveMotion.load(stagingId, track).catch((error) => {
|
|
9865
11535
|
this.handleCallbackError("liveMotion", error);
|
|
9866
11536
|
});
|
|
11537
|
+
},
|
|
11538
|
+
onToolCall: (toolCall) => {
|
|
11539
|
+
this.variantController.emit("toolCall", toolCall);
|
|
11540
|
+
},
|
|
11541
|
+
onVariantCue: async (variantKey, toolCall) => {
|
|
11542
|
+
try {
|
|
11543
|
+
await this.variantController.switchVariant(variantKey, toolCall);
|
|
11544
|
+
} catch (error) {
|
|
11545
|
+
this.reportError(
|
|
11546
|
+
error instanceof Error ? error : new Error(String(error)),
|
|
11547
|
+
"switchVariant"
|
|
11548
|
+
);
|
|
11549
|
+
}
|
|
9867
11550
|
}
|
|
9868
11551
|
});
|
|
9869
11552
|
if (options.gazeTarget !== void 0) {
|
|
@@ -9899,53 +11582,28 @@ var CharacterRuntime = class {
|
|
|
9899
11582
|
return this.sceneRig.vrm;
|
|
9900
11583
|
}
|
|
9901
11584
|
set vrm(value) {
|
|
9902
|
-
this.sceneRig.
|
|
11585
|
+
const previous = this.sceneRig.vrm;
|
|
11586
|
+
this.sceneRig.setVrm(value, this.defaultParent);
|
|
9903
11587
|
this.facialMixer.setVrm(value);
|
|
9904
11588
|
this.liveMotion.setVrm(value);
|
|
9905
11589
|
this.applyGaze();
|
|
9906
|
-
|
|
9907
|
-
this.sceneRig.attachTo(this.defaultParent);
|
|
9908
|
-
}
|
|
9909
|
-
}
|
|
9910
|
-
/** True once the character VRM model is loaded. */
|
|
9911
|
-
get ready() {
|
|
9912
|
-
return this.sceneRig.vrm !== null;
|
|
9913
|
-
}
|
|
9914
|
-
/** Root Three.js `Group` (`vrm.scene`) to add to a `THREE.Scene`. */
|
|
9915
|
-
get scene() {
|
|
9916
|
-
return this.sceneRig.scene;
|
|
9917
|
-
}
|
|
9918
|
-
/** Alias for `scene` for Three.js `Object3D` workflows. */
|
|
9919
|
-
get object3d() {
|
|
9920
|
-
return this.sceneRig.object3d;
|
|
9921
|
-
}
|
|
9922
|
-
/** Alias for `scene`. */
|
|
9923
|
-
get root() {
|
|
9924
|
-
return this.sceneRig.root;
|
|
9925
|
-
}
|
|
9926
|
-
/** Primary character mesh (`SkinnedMesh` preferred, otherwise first `Mesh`). */
|
|
9927
|
-
get mesh() {
|
|
9928
|
-
return this.sceneRig.mesh;
|
|
9929
|
-
}
|
|
9930
|
-
/** All Three.js `Mesh` instances in the character hierarchy. */
|
|
9931
|
-
get meshes() {
|
|
9932
|
-
return this.sceneRig.meshes;
|
|
11590
|
+
this.variantController.syncDirectVrm(value, previous);
|
|
9933
11591
|
}
|
|
9934
|
-
/**
|
|
9935
|
-
get
|
|
9936
|
-
return this.
|
|
11592
|
+
/** Allowed appearance variants configured for this character (`default` first). */
|
|
11593
|
+
get variants() {
|
|
11594
|
+
return this.variantController.variants;
|
|
9937
11595
|
}
|
|
9938
|
-
/**
|
|
9939
|
-
get
|
|
9940
|
-
return this.
|
|
11596
|
+
/** Currently active appearance variant key (defaults to `'default'`). */
|
|
11597
|
+
get activeVariant() {
|
|
11598
|
+
return this.variantController.activeVariant;
|
|
9941
11599
|
}
|
|
9942
|
-
/**
|
|
9943
|
-
get
|
|
9944
|
-
return this.
|
|
11600
|
+
/** Cached Three-VRM instances keyed by variant key. */
|
|
11601
|
+
get variantVrms() {
|
|
11602
|
+
return this.variantController.variantVrms;
|
|
9945
11603
|
}
|
|
9946
|
-
/**
|
|
9947
|
-
|
|
9948
|
-
|
|
11604
|
+
/** Switch the character's active appearance variant, loading on-demand if needed. */
|
|
11605
|
+
async switchVariant(variantKey) {
|
|
11606
|
+
await this.variantController.switchVariant(variantKey, null);
|
|
9949
11607
|
}
|
|
9950
11608
|
/** Underlying Three.js `AnimationMixer` driving skeletal animations. */
|
|
9951
11609
|
get mixer() {
|
|
@@ -9998,36 +11656,6 @@ var CharacterRuntime = class {
|
|
|
9998
11656
|
cursor: this.cursor
|
|
9999
11657
|
};
|
|
10000
11658
|
}
|
|
10001
|
-
/** Look up a raw humanoid bone `Object3D` by VRM bone name (for example `'head'` or `'rightHand'`). */
|
|
10002
|
-
getBone(name) {
|
|
10003
|
-
return this.sceneRig.getBone(name);
|
|
10004
|
-
}
|
|
10005
|
-
/** Look up a normalized humanoid bone `Object3D` by VRM bone name. */
|
|
10006
|
-
getNormalizedBone(name) {
|
|
10007
|
-
return this.sceneRig.getNormalizedBone(name);
|
|
10008
|
-
}
|
|
10009
|
-
/** Attach the character root `Group` to a Three.js `Scene` or parent `Object3D`. */
|
|
10010
|
-
attachTo(parent) {
|
|
10011
|
-
this.defaultParent = parent;
|
|
10012
|
-
return this.sceneRig.attachTo(parent);
|
|
10013
|
-
}
|
|
10014
|
-
/** Detach the character root `Group` from its current parent `Object3D`. */
|
|
10015
|
-
detach() {
|
|
10016
|
-
this.defaultParent = null;
|
|
10017
|
-
this.sceneRig.detach();
|
|
10018
|
-
}
|
|
10019
|
-
/** Configure `castShadow` and `receiveShadow` on all character meshes. */
|
|
10020
|
-
setShadows(options) {
|
|
10021
|
-
this.sceneRig.setShadows(options);
|
|
10022
|
-
}
|
|
10023
|
-
/** Configure `position`, `rotation`, and `scale` on the character root `Group`. */
|
|
10024
|
-
setTransform(options) {
|
|
10025
|
-
this.sceneRig.setTransform(options);
|
|
10026
|
-
}
|
|
10027
|
-
/** Re-index cached meshes, materials, and skeletons after custom scene graph edits. */
|
|
10028
|
-
refreshMeshes() {
|
|
10029
|
-
this.sceneRig.refreshMeshes();
|
|
10030
|
-
}
|
|
10031
11659
|
async ensureAnimation(animationId) {
|
|
10032
11660
|
await this.lazyAnimations.ensure(animationId);
|
|
10033
11661
|
}
|
|
@@ -10126,6 +11754,28 @@ var CharacterRuntime = class {
|
|
|
10126
11754
|
this.applyGaze();
|
|
10127
11755
|
};
|
|
10128
11756
|
}
|
|
11757
|
+
/**
|
|
11758
|
+
* Tells the character that the player types (`true` at each input event
|
|
11759
|
+
* of the text field) or stopped (`false`, for example after sending). The
|
|
11760
|
+
* character looks at the player and shows a subtle attentive face; typing
|
|
11761
|
+
* ends by itself after 1.5 s without input. Local only: no request.
|
|
11762
|
+
*/
|
|
11763
|
+
notifyUserTyping(active) {
|
|
11764
|
+
this.playerActivityTracker.notifyTyping(active);
|
|
11765
|
+
}
|
|
11766
|
+
/** `'speaking'` while listening, `'typing'` after typed input, else `'idle'`. */
|
|
11767
|
+
get playerActivity() {
|
|
11768
|
+
return this.playerActivityTracker.state;
|
|
11769
|
+
}
|
|
11770
|
+
onPlayerActivity(callback) {
|
|
11771
|
+
return this.playerActivityTracker.onChange((state) => {
|
|
11772
|
+
try {
|
|
11773
|
+
callback(state);
|
|
11774
|
+
} catch (error) {
|
|
11775
|
+
this.handleCallbackError("onPlayerActivity", error);
|
|
11776
|
+
}
|
|
11777
|
+
});
|
|
11778
|
+
}
|
|
10129
11779
|
startPerformanceRecording(options) {
|
|
10130
11780
|
this.performanceRecorder.start(options);
|
|
10131
11781
|
this.facialMixer.capturePerformanceModel();
|
|
@@ -10172,6 +11822,21 @@ var CharacterRuntime = class {
|
|
|
10172
11822
|
onTimelineStart(callback) {
|
|
10173
11823
|
return this.streamPlayback.onTimelineStart(callback);
|
|
10174
11824
|
}
|
|
11825
|
+
on(event, listener) {
|
|
11826
|
+
return this.variantController.on(event, listener);
|
|
11827
|
+
}
|
|
11828
|
+
off(event, listener) {
|
|
11829
|
+
this.variantController.off(event, listener);
|
|
11830
|
+
}
|
|
11831
|
+
onVrmChange(callback) {
|
|
11832
|
+
return this.variantController.on("vrmChange", callback);
|
|
11833
|
+
}
|
|
11834
|
+
onVariantChange(callback) {
|
|
11835
|
+
return this.variantController.on("variantChange", callback);
|
|
11836
|
+
}
|
|
11837
|
+
onToolCall(callback) {
|
|
11838
|
+
return this.variantController.on("toolCall", callback);
|
|
11839
|
+
}
|
|
10175
11840
|
/** Advance skeletal animations, lip-sync visemes, facial expressions, and VRM physics by `delta` seconds. */
|
|
10176
11841
|
update(delta) {
|
|
10177
11842
|
if (this.stagingPlayer?.paused) {
|
|
@@ -10183,6 +11848,10 @@ var CharacterRuntime = class {
|
|
|
10183
11848
|
this.streamPlayback.activeStagingId,
|
|
10184
11849
|
this.stagingPlayer?.trackTime ?? null
|
|
10185
11850
|
);
|
|
11851
|
+
this.listeningReflex.update(delta, {
|
|
11852
|
+
stagingActive: this.streamPlayback.playing || this.speaking || (this.stagingPlayer?.trackTime ?? null) !== null,
|
|
11853
|
+
liveMotionWeight: this.liveMotion.liveWeight
|
|
11854
|
+
});
|
|
10186
11855
|
this.visemePlayer.update(delta);
|
|
10187
11856
|
this.facialMixer.update(delta);
|
|
10188
11857
|
this.vrm?.update(delta);
|
|
@@ -10226,7 +11895,10 @@ var CharacterRuntime = class {
|
|
|
10226
11895
|
this.lookAtActive && !this.facialMixer.hasIdleFace ? this.currentGazeTarget : null
|
|
10227
11896
|
);
|
|
10228
11897
|
}
|
|
10229
|
-
resetRuntimeAssets() {
|
|
11898
|
+
resetRuntimeAssets(options) {
|
|
11899
|
+
const cachedVrms = this.variantController.reset();
|
|
11900
|
+
this.listeningReflex.reset();
|
|
11901
|
+
this.playerActivityTracker.reset();
|
|
10230
11902
|
this.lazyAnimations.reset();
|
|
10231
11903
|
this.stagingPlayer?.cleanup();
|
|
10232
11904
|
this.stagingPlayer = null;
|
|
@@ -10237,13 +11909,16 @@ var CharacterRuntime = class {
|
|
|
10237
11909
|
}
|
|
10238
11910
|
this.animationController = void 0;
|
|
10239
11911
|
this.animations.clear();
|
|
10240
|
-
|
|
11912
|
+
if (options?.disposeVrms !== false) {
|
|
11913
|
+
this.sceneRig.disposeAllVrms(cachedVrms);
|
|
11914
|
+
} else {
|
|
11915
|
+
this.sceneRig.setVrm(null);
|
|
11916
|
+
}
|
|
10241
11917
|
this.facialMixer.setVrm(null);
|
|
10242
11918
|
this.liveMotion.setVrm(null);
|
|
10243
11919
|
}
|
|
10244
11920
|
disposeRuntimeAssets() {
|
|
10245
|
-
this.
|
|
10246
|
-
this.resetRuntimeAssets();
|
|
11921
|
+
this.resetRuntimeAssets({ disposeVrms: true });
|
|
10247
11922
|
this.audioPlayer?.cleanup();
|
|
10248
11923
|
this.audioPlayer = null;
|
|
10249
11924
|
}
|
|
@@ -11739,8 +13414,8 @@ var MicrophoneCapture = class {
|
|
|
11739
13414
|
};
|
|
11740
13415
|
|
|
11741
13416
|
// ../../clients/web-client/src/voice/silence-detector.ts
|
|
11742
|
-
var
|
|
11743
|
-
var
|
|
13417
|
+
var MIN_SPEECH_LEVEL2 = 0.015;
|
|
13418
|
+
var SPEECH_TO_NOISE2 = 3;
|
|
11744
13419
|
var SPEECH_START_MS = 200;
|
|
11745
13420
|
var SilenceDetector = class {
|
|
11746
13421
|
constructor(silenceMs) {
|
|
@@ -11753,8 +13428,8 @@ var SilenceDetector = class {
|
|
|
11753
13428
|
/** True when the speech has ended. */
|
|
11754
13429
|
push(level, frameMs = SPEECH_FRAME_MS) {
|
|
11755
13430
|
const threshold = Math.max(
|
|
11756
|
-
|
|
11757
|
-
(this.floor ??
|
|
13431
|
+
MIN_SPEECH_LEVEL2,
|
|
13432
|
+
(this.floor ?? MIN_SPEECH_LEVEL2 / SPEECH_TO_NOISE2) * SPEECH_TO_NOISE2
|
|
11758
13433
|
);
|
|
11759
13434
|
if (level >= threshold) {
|
|
11760
13435
|
this.loudMs += frameMs;
|
|
@@ -11893,6 +13568,7 @@ var VoiceInput = class {
|
|
|
11893
13568
|
send: options.send !== false,
|
|
11894
13569
|
silent: options.silent === true,
|
|
11895
13570
|
renderSessionId: this.deps.renderSessionId(),
|
|
13571
|
+
timeZone: detectClientTimeZone(),
|
|
11896
13572
|
format: SPEECH_AUDIO_FORMAT
|
|
11897
13573
|
})
|
|
11898
13574
|
),
|
|
@@ -12045,6 +13721,7 @@ var VoiceInput = class {
|
|
|
12045
13721
|
if (session !== this.session || session.captureClosed) {
|
|
12046
13722
|
return;
|
|
12047
13723
|
}
|
|
13724
|
+
this.deps.activity?.pushVoiceLevel(frame.level);
|
|
12048
13725
|
const audio = new Uint8Array(
|
|
12049
13726
|
frame.pcm.buffer,
|
|
12050
13727
|
frame.pcm.byteOffset,
|
|
@@ -12190,6 +13867,7 @@ var VoiceInput = class {
|
|
|
12190
13867
|
setState(state) {
|
|
12191
13868
|
if (this.state !== state) {
|
|
12192
13869
|
this.state = state;
|
|
13870
|
+
this.deps.activity?.setListening(state === "listening");
|
|
12193
13871
|
this.deps.onState(state);
|
|
12194
13872
|
}
|
|
12195
13873
|
}
|
|
@@ -12208,6 +13886,8 @@ var Character = class extends CharacterRuntime {
|
|
|
12208
13886
|
/** The conversation whose live motion was warmed up. */
|
|
12209
13887
|
liveMotionPrepared = null;
|
|
12210
13888
|
authoring;
|
|
13889
|
+
hostedWebChat;
|
|
13890
|
+
memoryIdentifier;
|
|
12211
13891
|
normalizeIdleOrientation;
|
|
12212
13892
|
bakedAsset;
|
|
12213
13893
|
animationRefreshId = 0;
|
|
@@ -12234,6 +13914,8 @@ var Character = class extends CharacterRuntime {
|
|
|
12234
13914
|
})
|
|
12235
13915
|
});
|
|
12236
13916
|
this.authoring = options.authoring ?? false;
|
|
13917
|
+
this.hostedWebChat = options.hostedWebChat === true && !resolvedToken;
|
|
13918
|
+
this.memoryIdentifier = normalizeMemoryIdentifier(options.memoryIdentifier);
|
|
12237
13919
|
this.normalizeIdleOrientation = options.normalizeIdleOrientation ?? false;
|
|
12238
13920
|
this.bakedAsset = options.bakedConversation;
|
|
12239
13921
|
this.initialAssetInfo = options.initialAssetInfo ?? null;
|
|
@@ -12300,6 +13982,7 @@ var Character = class extends CharacterRuntime {
|
|
|
12300
13982
|
);
|
|
12301
13983
|
}
|
|
12302
13984
|
},
|
|
13985
|
+
activity: this.playerActivityTracker,
|
|
12303
13986
|
onTranscript: (event) => {
|
|
12304
13987
|
if (this.transcriptCallback) {
|
|
12305
13988
|
invokeSafeCallback(
|
|
@@ -12316,6 +13999,7 @@ var Character = class extends CharacterRuntime {
|
|
|
12316
13999
|
telemetry: this.telemetry,
|
|
12317
14000
|
socket: () => this.conversationSocket,
|
|
12318
14001
|
renderSession: () => this.renderSessionId,
|
|
14002
|
+
memoryIdentifier: () => this.apiToken ? this.memoryIdentifier : void 0,
|
|
12319
14003
|
thinking: () => this.invokeThinkingCallback({ taskId: null, stagingId: null }),
|
|
12320
14004
|
handleResult: (payload, forcePlayback, silent) => this.handleStagingResult(payload, {
|
|
12321
14005
|
source: "api",
|
|
@@ -12336,6 +14020,7 @@ var Character = class extends CharacterRuntime {
|
|
|
12336
14020
|
const progressReporter = createAssetProgressReporter(callback);
|
|
12337
14021
|
const info = await this.getAssetInfo();
|
|
12338
14022
|
const baked = !this.authoring && Boolean(this.bakedAsset || info.bakedConversationEnabled);
|
|
14023
|
+
this.variantController.configureVariants(info.variants, { remote: false });
|
|
12339
14024
|
this.inputs.load({
|
|
12340
14025
|
allowCustomReply: info.allowCustomReply !== false,
|
|
12341
14026
|
baked,
|
|
@@ -12359,8 +14044,10 @@ var Character = class extends CharacterRuntime {
|
|
|
12359
14044
|
onProgress: progressReporter.updateResource,
|
|
12360
14045
|
assetProfile: this.assetProfile
|
|
12361
14046
|
}),
|
|
12362
|
-
this.baked ? this.lazyAnimations.ensureMany(animationIds) : info.idleAnimationId ? this.lazyAnimations.ensure(info.idleAnimationId) : Promise.resolve()
|
|
14047
|
+
this.baked ? this.lazyAnimations.ensureMany(animationIds) : info.idleAnimationId ? this.lazyAnimations.ensure(info.idleAnimationId) : Promise.resolve(),
|
|
14048
|
+
this.baked ? this.variantController.preloadVariants(true) : Promise.resolve()
|
|
12363
14049
|
]);
|
|
14050
|
+
this.variantController.registerDefaultVrm(vrm);
|
|
12364
14051
|
this.vrm = vrm;
|
|
12365
14052
|
if (this.performanceRecorder.isRecording) {
|
|
12366
14053
|
this.facialMixer.capturePerformanceModel();
|
|
@@ -12378,6 +14065,9 @@ var Character = class extends CharacterRuntime {
|
|
|
12378
14065
|
});
|
|
12379
14066
|
}
|
|
12380
14067
|
animationController.playIdle(0);
|
|
14068
|
+
if (!this.baked) {
|
|
14069
|
+
void this.variantController.preloadVariants(false);
|
|
14070
|
+
}
|
|
12381
14071
|
}
|
|
12382
14072
|
async reloadAnimations(info) {
|
|
12383
14073
|
const requestId = ++this.animationRefreshId;
|
|
@@ -12411,6 +14101,7 @@ var Character = class extends CharacterRuntime {
|
|
|
12411
14101
|
if (info.bakedConversationEnabled || this.bakedAsset) {
|
|
12412
14102
|
throw new Error("Baked conversation requires local rendering.");
|
|
12413
14103
|
}
|
|
14104
|
+
this.variantController.configureVariants(info.variants, { remote: true });
|
|
12414
14105
|
this.inputs.load({
|
|
12415
14106
|
allowCustomReply: info.allowCustomReply !== false,
|
|
12416
14107
|
baked: false,
|
|
@@ -12452,7 +14143,9 @@ var Character = class extends CharacterRuntime {
|
|
|
12452
14143
|
baseUrl: this.baseUrl,
|
|
12453
14144
|
apiToken: this.apiToken,
|
|
12454
14145
|
avatarId: this.characterId,
|
|
12455
|
-
renderSessionId: this.renderSessionId
|
|
14146
|
+
renderSessionId: this.renderSessionId,
|
|
14147
|
+
hostedWebChat: this.hostedWebChat,
|
|
14148
|
+
memoryIdentifier: this.apiToken ? this.memoryIdentifier : void 0
|
|
12456
14149
|
});
|
|
12457
14150
|
if (!response.conversation?.id) {
|
|
12458
14151
|
throw new Error("Failed to create conversation.");
|
|
@@ -12504,7 +14197,7 @@ var Character = class extends CharacterRuntime {
|
|
|
12504
14197
|
await this.endConversation();
|
|
12505
14198
|
if (options?.disposeVrm === false) {
|
|
12506
14199
|
this.detach();
|
|
12507
|
-
this.resetRuntimeAssets();
|
|
14200
|
+
this.resetRuntimeAssets({ disposeVrms: false });
|
|
12508
14201
|
this.audioPlayer?.cleanup();
|
|
12509
14202
|
this.audioPlayer = null;
|
|
12510
14203
|
return;
|
|
@@ -12685,6 +14378,15 @@ var Character = class extends CharacterRuntime {
|
|
|
12685
14378
|
setRenderSessionId(renderSessionId) {
|
|
12686
14379
|
this.renderSessionId = renderSessionId?.trim() || null;
|
|
12687
14380
|
}
|
|
14381
|
+
setHostedWebChat(enabled) {
|
|
14382
|
+
this.hostedWebChat = enabled === true && !this.apiToken;
|
|
14383
|
+
}
|
|
14384
|
+
setMemoryIdentifier(memoryIdentifier) {
|
|
14385
|
+
this.memoryIdentifier = normalizeMemoryIdentifier(memoryIdentifier);
|
|
14386
|
+
}
|
|
14387
|
+
getMemoryIdentifier() {
|
|
14388
|
+
return this.memoryIdentifier;
|
|
14389
|
+
}
|
|
12688
14390
|
onStagingResult(callback) {
|
|
12689
14391
|
return bindCallbackSlot(
|
|
12690
14392
|
() => this.stagingResultCallback,
|
|
@@ -12714,33 +14416,22 @@ var Character = class extends CharacterRuntime {
|
|
|
12714
14416
|
};
|
|
12715
14417
|
async handleStagingResult(payload, options) {
|
|
12716
14418
|
this.thinkingState = false;
|
|
12717
|
-
|
|
12718
|
-
|
|
12719
|
-
|
|
12720
|
-
this.
|
|
12721
|
-
|
|
12722
|
-
|
|
12723
|
-
|
|
12724
|
-
|
|
12725
|
-
|
|
12726
|
-
|
|
12727
|
-
|
|
12728
|
-
|
|
14419
|
+
await deliverStagingResult(payload, {
|
|
14420
|
+
...options,
|
|
14421
|
+
notifiedStagingIds: this.notifiedStagingIds,
|
|
14422
|
+
hasPlayed: (stagingId) => this.streamPlayback.hasPlayed(stagingId),
|
|
14423
|
+
setOptions: (next) => this.inputs.setOptions(next),
|
|
14424
|
+
onTurnReceived: (normalized) => this.liveConversation.received(normalized),
|
|
14425
|
+
onStagingResult: this.stagingResultCallback,
|
|
14426
|
+
onCallbackError: this.handleCallbackError,
|
|
14427
|
+
recordEvent: (event) => this.performanceRecorder.recordEvent(event),
|
|
14428
|
+
playRequested: (normalized, playOptions) => {
|
|
14429
|
+
void this.playStaging(normalized, playOptions);
|
|
14430
|
+
},
|
|
14431
|
+
playStreamed: (normalized) => {
|
|
14432
|
+
void this.streamPlayback.playPayload(normalized);
|
|
12729
14433
|
}
|
|
12730
|
-
}
|
|
12731
|
-
const shouldPlay = !this.streamPlayback.hasPlayed(normalized.stagingId) || force;
|
|
12732
|
-
this.performanceRecorder.recordEvent({
|
|
12733
|
-
type: "staging.result",
|
|
12734
|
-
source: options.source,
|
|
12735
|
-
stagingId: normalized.stagingId,
|
|
12736
|
-
taskId: normalized.taskId ?? "",
|
|
12737
|
-
playback: shouldPlay ? "requested" : "already-streamed"
|
|
12738
14434
|
});
|
|
12739
|
-
if (shouldPlay) {
|
|
12740
|
-
void this.playStaging(normalized, { force, silent: options.silent });
|
|
12741
|
-
} else {
|
|
12742
|
-
void this.streamPlayback.playPayload(normalized);
|
|
12743
|
-
}
|
|
12744
14435
|
}
|
|
12745
14436
|
invokeThinkingCallback(payload) {
|
|
12746
14437
|
this.thinkingState = true;
|
|
@@ -13251,6 +14942,7 @@ var mountNpcify = async function(options) {
|
|
|
13251
14942
|
camera: null,
|
|
13252
14943
|
renderer: null,
|
|
13253
14944
|
controls: null,
|
|
14945
|
+
setMemoryIdentifier: (memoryIdentifier) => npcify.setMemoryIdentifier(memoryIdentifier),
|
|
13254
14946
|
dispose: async () => {
|
|
13255
14947
|
remote.dispose();
|
|
13256
14948
|
await npcify.dispose();
|
|
@@ -13264,6 +14956,7 @@ var mountNpcify = async function(options) {
|
|
|
13264
14956
|
let context2;
|
|
13265
14957
|
let session;
|
|
13266
14958
|
let observer;
|
|
14959
|
+
let unsubscribeVrm;
|
|
13267
14960
|
let frame = 0;
|
|
13268
14961
|
let disposed = false;
|
|
13269
14962
|
const dispose = async function() {
|
|
@@ -13273,10 +14966,11 @@ var mountNpcify = async function(options) {
|
|
|
13273
14966
|
disposed = true;
|
|
13274
14967
|
cancelAnimationFrame(frame);
|
|
13275
14968
|
observer?.disconnect();
|
|
13276
|
-
|
|
14969
|
+
unsubscribeVrm?.();
|
|
13277
14970
|
if (context2 && session) {
|
|
13278
|
-
detachVrm(context2, session, { dispose:
|
|
14971
|
+
detachVrm(context2, session, { dispose: false });
|
|
13279
14972
|
}
|
|
14973
|
+
await npcify.dispose();
|
|
13280
14974
|
if (context2) {
|
|
13281
14975
|
disposeSceneContext(context2);
|
|
13282
14976
|
}
|
|
@@ -13288,6 +14982,16 @@ var mountNpcify = async function(options) {
|
|
|
13288
14982
|
}
|
|
13289
14983
|
context2 = createSceneContext({ canvas: options.canvas });
|
|
13290
14984
|
session = attachVrm(context2, npcify.vrm);
|
|
14985
|
+
unsubscribeVrm = npcify.on("vrmChange", (nextVrm) => {
|
|
14986
|
+
if (context2) {
|
|
14987
|
+
setCameraCollisionBackend(
|
|
14988
|
+
context2,
|
|
14989
|
+
new HumanoidCapsuleCollisionBackend(
|
|
14990
|
+
(name) => nextVrm.humanoid.getRawBoneNode(name)
|
|
14991
|
+
)
|
|
14992
|
+
);
|
|
14993
|
+
}
|
|
14994
|
+
});
|
|
13291
14995
|
applyCameraPreset(context2, "full_body");
|
|
13292
14996
|
npcify.setGazeTarget(context2.camera);
|
|
13293
14997
|
const resize = () => {
|
|
@@ -13321,6 +15025,7 @@ var mountNpcify = async function(options) {
|
|
|
13321
15025
|
camera: context2.camera,
|
|
13322
15026
|
renderer: context2.renderer,
|
|
13323
15027
|
controls: context2.controls,
|
|
15028
|
+
setMemoryIdentifier: (memoryIdentifier) => npcify.setMemoryIdentifier(memoryIdentifier),
|
|
13324
15029
|
dispose
|
|
13325
15030
|
};
|
|
13326
15031
|
} catch (error) {
|
|
@@ -13339,12 +15044,14 @@ export {
|
|
|
13339
15044
|
CharacterRuntime,
|
|
13340
15045
|
CharacterSceneRig,
|
|
13341
15046
|
ConversationConnectionError,
|
|
15047
|
+
DEFAULT_VARIANT_KEY,
|
|
13342
15048
|
FacialPerformanceLayer,
|
|
13343
15049
|
FacialPerformanceMixer,
|
|
13344
15050
|
INPUT_MODES,
|
|
13345
15051
|
InputModeError,
|
|
13346
15052
|
LiveMotionPlayer,
|
|
13347
15053
|
MicrophoneError,
|
|
15054
|
+
NUDE_VARIANT_KEY,
|
|
13348
15055
|
Npcify,
|
|
13349
15056
|
Character as NpcifyCharacter,
|
|
13350
15057
|
PERFORMANCE_FILE_EXTENSION,
|
|
@@ -13358,7 +15065,11 @@ export {
|
|
|
13358
15065
|
createCharacter,
|
|
13359
15066
|
decodePerformance,
|
|
13360
15067
|
encodePerformance,
|
|
15068
|
+
fetchVariantVrmDownloadUrl,
|
|
15069
|
+
fetchVrmDownloadUrl,
|
|
13361
15070
|
loadBakedConversation,
|
|
13362
15071
|
loadCharacter,
|
|
15072
|
+
loadVariantVrm,
|
|
15073
|
+
loadVrm,
|
|
13363
15074
|
mountNpcify
|
|
13364
15075
|
};
|