@npcify/web 0.3.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +25 -10
- package/dist/index.d.ts +194 -5
- package/dist/index.js +825 -168
- package/examples/main.js +15 -12
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1,3 +1,36 @@
|
|
|
1
|
+
// ../ts-shared/src/motion-frames.ts
|
|
2
|
+
var MOTION_FRAMES_CAPABILITY = "motion_frames";
|
|
3
|
+
var motionFrameWidth = function(boneCount) {
|
|
4
|
+
return 3 + 4 * boneCount;
|
|
5
|
+
};
|
|
6
|
+
var MAX_MOTION_FRAMES_PER_EVENT = 3600;
|
|
7
|
+
var MAX_MOTION_BONES = 64;
|
|
8
|
+
var isCount = function(value, max2) {
|
|
9
|
+
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 && value <= max2;
|
|
10
|
+
};
|
|
11
|
+
var readMotionFrames = function(data, frameCount, boneCount) {
|
|
12
|
+
const values = frameCount * motionFrameWidth(boneCount);
|
|
13
|
+
if (data.byteLength !== values * 4) {
|
|
14
|
+
throw new Error("Motion frames do not match their frame count.");
|
|
15
|
+
}
|
|
16
|
+
const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
|
|
17
|
+
const frames = new Float32Array(values);
|
|
18
|
+
for (let index = 0; index < values; index += 1) {
|
|
19
|
+
frames[index] = view.getFloat32(index * 4, true);
|
|
20
|
+
}
|
|
21
|
+
return frames;
|
|
22
|
+
};
|
|
23
|
+
var isValidMotionFramesEvent = function(event) {
|
|
24
|
+
if (!event || typeof event !== "object") {
|
|
25
|
+
return false;
|
|
26
|
+
}
|
|
27
|
+
const value = event;
|
|
28
|
+
const bones = value.bones;
|
|
29
|
+
const frameCount = value.frameCount;
|
|
30
|
+
const frames = value.frames;
|
|
31
|
+
return value.type === "motion_frames" && typeof value.taskId === "string" && typeof value.stagingId === "string" && isCount(value.revision, Number.MAX_SAFE_INTEGER) && typeof value.fps === "number" && value.fps > 0 && value.fps <= 240 && isCount(value.startFrame, 24 * 60 * 60 * 240) && isCount(frameCount, MAX_MOTION_FRAMES_PER_EVENT) && Array.isArray(bones) && bones.length > 0 && bones.length <= MAX_MOTION_BONES && bones.every((bone) => typeof bone === "string") && typeof value.restHipsHeight === "number" && value.restHipsHeight > 0 && Number.isFinite(value.restHipsHeight) && frames instanceof Uint8Array && frames.byteLength === frameCount * motionFrameWidth(bones.length) * 4;
|
|
32
|
+
};
|
|
33
|
+
|
|
1
34
|
// ../ts-shared/src/llm.ts
|
|
2
35
|
var STAGING_PROTOCOL_VERSION = 1;
|
|
3
36
|
|
|
@@ -14,10 +47,6 @@ var GPT_IMAGE_MODELS = [
|
|
|
14
47
|
"gpt-image-2.5-flare",
|
|
15
48
|
"gpt-image-2.5-sunburst"
|
|
16
49
|
];
|
|
17
|
-
var GEMINI_SPEECH_MODEL_VENDORS = {
|
|
18
|
-
"gemini-3.5-transcribe-live": "gemini",
|
|
19
|
-
"gemini-3.5-transcribe": "gemini"
|
|
20
|
-
};
|
|
21
50
|
var AI_ROLES = {
|
|
22
51
|
character_draft: {
|
|
23
52
|
name: "Character draft",
|
|
@@ -59,28 +88,6 @@ var AI_ROLES = {
|
|
|
59
88
|
models: ["gemini-3.8-flash-tts", "gemini-3.8-flash-lite-tts"],
|
|
60
89
|
options: []
|
|
61
90
|
},
|
|
62
|
-
// Priced by SPEECH_RECOGNITION_PRICES of the server (docs/voice-input.md).
|
|
63
|
-
// Audio that arrives while the player speaks. The model selects the vendor.
|
|
64
|
-
speech_input_live: {
|
|
65
|
-
name: "Speech input (live)",
|
|
66
|
-
source: "openai",
|
|
67
|
-
models: [
|
|
68
|
-
"gpt-transcribe",
|
|
69
|
-
"gpt-live-transcribe",
|
|
70
|
-
"gemini-3.5-transcribe-live",
|
|
71
|
-
"gemini-3.5-transcribe"
|
|
72
|
-
],
|
|
73
|
-
modelVendors: GEMINI_SPEECH_MODEL_VENDORS,
|
|
74
|
-
options: []
|
|
75
|
-
},
|
|
76
|
-
// One complete recording (an upload of an engine SDK).
|
|
77
|
-
speech_input_file: {
|
|
78
|
-
name: "Speech input (file)",
|
|
79
|
-
source: "openai",
|
|
80
|
-
models: ["gpt-transcribe", "gemini-3.5-transcribe"],
|
|
81
|
-
modelVendors: GEMINI_SPEECH_MODEL_VENDORS,
|
|
82
|
-
options: []
|
|
83
|
-
},
|
|
84
91
|
// Priced by OPENAI_PRICES.text of the generator.
|
|
85
92
|
generator_agent: {
|
|
86
93
|
name: "Generator agent",
|
|
@@ -116,7 +123,7 @@ var permittedInputModes = function(permissions) {
|
|
|
116
123
|
if (permissions.baked || !permissions.allowCustomReply) {
|
|
117
124
|
return ["options"];
|
|
118
125
|
}
|
|
119
|
-
return ["options", "text", "voice"];
|
|
126
|
+
return permissions.speechInput ? ["options", "text", "voice"] : ["options", "text"];
|
|
120
127
|
};
|
|
121
128
|
var isInputMode = function(value) {
|
|
122
129
|
return INPUT_MODES.includes(value);
|
|
@@ -159,8 +166,9 @@ var resolveInputModes = function(configured, permissions) {
|
|
|
159
166
|
}
|
|
160
167
|
const refused = configured.filter((mode) => !permitted.includes(mode));
|
|
161
168
|
if (refused.length > 0) {
|
|
169
|
+
const names = refused.map((mode) => `"${mode}"`).join(" and ");
|
|
162
170
|
throw new InputModeError(
|
|
163
|
-
`This character accepts only its reply options (Allow custom replies is off). Remove ${
|
|
171
|
+
permissions.allowCustomReply ? `Voice input is off for this character (its owner turns it on in the STT tab). Remove ${names} from inputModes.` : `This character accepts only its reply options (Allow custom replies is off). Remove ${names} from inputModes.`
|
|
164
172
|
);
|
|
165
173
|
}
|
|
166
174
|
return [...configured];
|
|
@@ -186,6 +194,54 @@ var SPEECH_EVENTS = Object.freeze({
|
|
|
186
194
|
turnFailed: "speech:turn_failed"
|
|
187
195
|
});
|
|
188
196
|
|
|
197
|
+
// ../ts-shared/src/speech-input-providers.ts
|
|
198
|
+
var SPEECH_INPUT_PROVIDER_PROFILES = Object.freeze({
|
|
199
|
+
// The file model also streams: 120 ms slower than gpt-live-transcribe,
|
|
200
|
+
// at a 3.8 times lower price and with fewer errors.
|
|
201
|
+
openai: {
|
|
202
|
+
live: {
|
|
203
|
+
model: "gpt-transcribe",
|
|
204
|
+
releaseToTextMs: 783,
|
|
205
|
+
usdPerMinute: 45e-4,
|
|
206
|
+
approximate: false
|
|
207
|
+
},
|
|
208
|
+
file: {
|
|
209
|
+
model: "gpt-transcribe",
|
|
210
|
+
releaseToTextMs: 867,
|
|
211
|
+
usdPerMinute: 45e-4,
|
|
212
|
+
approximate: false
|
|
213
|
+
}
|
|
214
|
+
},
|
|
215
|
+
gemini: {
|
|
216
|
+
live: {
|
|
217
|
+
model: "gemini-3.5-transcribe-live",
|
|
218
|
+
releaseToTextMs: 266,
|
|
219
|
+
usdPerMinute: 9e-3,
|
|
220
|
+
approximate: true
|
|
221
|
+
},
|
|
222
|
+
file: {
|
|
223
|
+
model: "gemini-3.5-transcribe",
|
|
224
|
+
releaseToTextMs: 1263,
|
|
225
|
+
usdPerMinute: 5e-3,
|
|
226
|
+
approximate: true
|
|
227
|
+
}
|
|
228
|
+
},
|
|
229
|
+
elevenlabs: {
|
|
230
|
+
live: {
|
|
231
|
+
model: "scribe_v2_realtime",
|
|
232
|
+
releaseToTextMs: 208,
|
|
233
|
+
usdPerMinute: 0.39 / 60,
|
|
234
|
+
approximate: false
|
|
235
|
+
},
|
|
236
|
+
file: {
|
|
237
|
+
model: "scribe_v2",
|
|
238
|
+
releaseToTextMs: 925,
|
|
239
|
+
usdPerMinute: 0.22 / 60,
|
|
240
|
+
approximate: false
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
});
|
|
244
|
+
|
|
189
245
|
// ../ts-shared/src/product-usage.ts
|
|
190
246
|
var productUsageEvents = [
|
|
191
247
|
"page_viewed",
|
|
@@ -2579,6 +2635,8 @@ var PERFORMANCE_KINDS = {
|
|
|
2579
2635
|
storedTimeline: 7,
|
|
2580
2636
|
cachedOpening: 8
|
|
2581
2637
|
};
|
|
2638
|
+
var PERFORMANCE_MAJOR_VERSION = 1;
|
|
2639
|
+
var PERFORMANCE_MINOR_VERSION = 1;
|
|
2582
2640
|
var HEADER_SIZE = 24;
|
|
2583
2641
|
var MAGIC = 1346588750;
|
|
2584
2642
|
var crcTable = Uint32Array.from({ length: 256 }, (_, index) => {
|
|
@@ -2650,8 +2708,8 @@ var encodePerformance = function(kind, value, options = {}) {
|
|
|
2650
2708
|
const bytes = new Uint8Array(HEADER_SIZE + body.length);
|
|
2651
2709
|
const header = new DataView(bytes.buffer);
|
|
2652
2710
|
header.setUint32(0, MAGIC, true);
|
|
2653
|
-
header.setUint8(4,
|
|
2654
|
-
header.setUint8(5,
|
|
2711
|
+
header.setUint8(4, PERFORMANCE_MAJOR_VERSION);
|
|
2712
|
+
header.setUint8(5, PERFORMANCE_MINOR_VERSION);
|
|
2655
2713
|
header.setUint8(6, PERFORMANCE_KINDS[kind]);
|
|
2656
2714
|
header.setUint8(7, options.compress ? 1 : 0);
|
|
2657
2715
|
header.setUint16(8, HEADER_SIZE, true);
|
|
@@ -2688,7 +2746,7 @@ var decodePerformance = function(input, expectedKind) {
|
|
|
2688
2746
|
throw new Error("Truncated performance header.");
|
|
2689
2747
|
}
|
|
2690
2748
|
const header = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
2691
|
-
if (header.getUint32(0, true) !== MAGIC || header.getUint8(4) !==
|
|
2749
|
+
if (header.getUint32(0, true) !== MAGIC || header.getUint8(4) !== PERFORMANCE_MAJOR_VERSION) {
|
|
2692
2750
|
throw new Error("Unsupported performance format or major version.");
|
|
2693
2751
|
}
|
|
2694
2752
|
if (header.getUint8(6) !== PERFORMANCE_KINDS[expectedKind]) {
|
|
@@ -3143,7 +3201,8 @@ var createConversation = async function({
|
|
|
3143
3201
|
),
|
|
3144
3202
|
credentials: apiToken ? "omit" : "include",
|
|
3145
3203
|
method: "POST",
|
|
3146
|
-
|
|
3204
|
+
// This SDK plays live motion frames (ADR 0011).
|
|
3205
|
+
body: JSON.stringify({ avatarId, capabilities: [MOTION_FRAMES_CAPABILITY] })
|
|
3147
3206
|
});
|
|
3148
3207
|
if (!res.ok) {
|
|
3149
3208
|
throw Object.assign(new Error(await res.text()), { status: res.status });
|
|
@@ -3196,7 +3255,8 @@ var createStaging = async function({
|
|
|
3196
3255
|
content: message,
|
|
3197
3256
|
silent: silent === true,
|
|
3198
3257
|
incrementalPlayback: true,
|
|
3199
|
-
parentStagingId
|
|
3258
|
+
parentStagingId,
|
|
3259
|
+
capabilities: [MOTION_FRAMES_CAPABILITY]
|
|
3200
3260
|
})
|
|
3201
3261
|
});
|
|
3202
3262
|
} catch (error) {
|
|
@@ -3296,6 +3356,35 @@ var getConversationWorldline = async function(options) {
|
|
|
3296
3356
|
}
|
|
3297
3357
|
return await response.json();
|
|
3298
3358
|
};
|
|
3359
|
+
var warmUpLiveMotion = async function({
|
|
3360
|
+
conversationId,
|
|
3361
|
+
apiToken,
|
|
3362
|
+
baseUrl,
|
|
3363
|
+
renderSessionId
|
|
3364
|
+
}) {
|
|
3365
|
+
const url = new URL(`/conversations/${conversationId}/motion/warm-up`, baseUrl);
|
|
3366
|
+
try {
|
|
3367
|
+
const res = await fetch(url.toString(), {
|
|
3368
|
+
headers: withRenderSession(
|
|
3369
|
+
buildAuthHeaders(apiToken, {
|
|
3370
|
+
json: true,
|
|
3371
|
+
allowEmpty: true
|
|
3372
|
+
}),
|
|
3373
|
+
renderSessionId
|
|
3374
|
+
),
|
|
3375
|
+
credentials: apiToken ? "omit" : "include",
|
|
3376
|
+
method: "POST",
|
|
3377
|
+
body: JSON.stringify({ capabilities: [MOTION_FRAMES_CAPABILITY] })
|
|
3378
|
+
});
|
|
3379
|
+
if (!res.ok) {
|
|
3380
|
+
return false;
|
|
3381
|
+
}
|
|
3382
|
+
const payload = await res.json();
|
|
3383
|
+
return payload.warming === true;
|
|
3384
|
+
} catch {
|
|
3385
|
+
return false;
|
|
3386
|
+
}
|
|
3387
|
+
};
|
|
3299
3388
|
|
|
3300
3389
|
// ../../clients/web-client/src/live-conversation.ts
|
|
3301
3390
|
var LiveConversation = class {
|
|
@@ -6022,6 +6111,10 @@ var StreamPlaybackController = class {
|
|
|
6022
6111
|
get cursor() {
|
|
6023
6112
|
return this.currentCursorValue;
|
|
6024
6113
|
}
|
|
6114
|
+
/** The staging that plays or waits to play, or null. */
|
|
6115
|
+
get activeStagingId() {
|
|
6116
|
+
return this.activePlayback?.stagingId ?? null;
|
|
6117
|
+
}
|
|
6025
6118
|
onStreamEvent(callback) {
|
|
6026
6119
|
return bindCallbackSlot(
|
|
6027
6120
|
() => this.streamEventCallback,
|
|
@@ -6115,6 +6208,10 @@ var StreamPlaybackController = class {
|
|
|
6115
6208
|
this.dependencies.telemetry.recordFirstTextDelta();
|
|
6116
6209
|
return;
|
|
6117
6210
|
}
|
|
6211
|
+
if (event.type === "motion_frames") {
|
|
6212
|
+
this.dependencies.onMotionFrames?.(event);
|
|
6213
|
+
return;
|
|
6214
|
+
}
|
|
6118
6215
|
const player = this.dependencies.getStagingPlayer();
|
|
6119
6216
|
if (event.type === "speech_chunk") {
|
|
6120
6217
|
player.appendSpeechChunk(event.streamId, event.mimeType, event.data);
|
|
@@ -6228,6 +6325,9 @@ var StreamPlaybackController = class {
|
|
|
6228
6325
|
this.activePlayback = { taskId: params.taskId, stagingId: params.stagingId };
|
|
6229
6326
|
this.playedStagingIds.add(params.stagingId);
|
|
6230
6327
|
this.recordPlaybackEvent(params, "requested");
|
|
6328
|
+
if (params.timeline.motion) {
|
|
6329
|
+
this.dependencies.onMotionTrack?.(params.stagingId, params.timeline.motion);
|
|
6330
|
+
}
|
|
6231
6331
|
const existing = this.states.get(params.stagingId);
|
|
6232
6332
|
const state = {
|
|
6233
6333
|
taskId: params.taskId,
|
|
@@ -8237,6 +8337,261 @@ var CharacterSceneRig = class {
|
|
|
8237
8337
|
}
|
|
8238
8338
|
};
|
|
8239
8339
|
|
|
8340
|
+
// ../../clients/web-client/src/character/live-motion.ts
|
|
8341
|
+
import { Quaternion as Quaternion3, Vector3 as Vector37 } from "three";
|
|
8342
|
+
var PARENTS = {
|
|
8343
|
+
spine: "hips",
|
|
8344
|
+
chest: "spine",
|
|
8345
|
+
upperChest: "chest",
|
|
8346
|
+
neck: "upperChest",
|
|
8347
|
+
head: "neck",
|
|
8348
|
+
leftShoulder: "upperChest",
|
|
8349
|
+
leftUpperArm: "leftShoulder",
|
|
8350
|
+
leftLowerArm: "leftUpperArm",
|
|
8351
|
+
leftHand: "leftLowerArm",
|
|
8352
|
+
rightShoulder: "upperChest",
|
|
8353
|
+
rightUpperArm: "rightShoulder",
|
|
8354
|
+
rightLowerArm: "rightUpperArm",
|
|
8355
|
+
rightHand: "rightLowerArm",
|
|
8356
|
+
leftUpperLeg: "hips",
|
|
8357
|
+
leftLowerLeg: "leftUpperLeg",
|
|
8358
|
+
leftFoot: "leftLowerLeg",
|
|
8359
|
+
leftToes: "leftFoot",
|
|
8360
|
+
rightUpperLeg: "hips",
|
|
8361
|
+
rightLowerLeg: "rightUpperLeg",
|
|
8362
|
+
rightFoot: "rightLowerLeg",
|
|
8363
|
+
rightToes: "rightFoot"
|
|
8364
|
+
};
|
|
8365
|
+
var MAX_TRACKS = 4;
|
|
8366
|
+
var approach = function(value, target, delta, seconds) {
|
|
8367
|
+
if (seconds <= 0) {
|
|
8368
|
+
return target;
|
|
8369
|
+
}
|
|
8370
|
+
const step = delta / seconds;
|
|
8371
|
+
return value < target ? Math.min(target, value + step) : Math.max(target, value - step);
|
|
8372
|
+
};
|
|
8373
|
+
var LiveMotionPlayer = class {
|
|
8374
|
+
vrm = null;
|
|
8375
|
+
tracks = /* @__PURE__ */ new Map();
|
|
8376
|
+
rootMode;
|
|
8377
|
+
blendIn;
|
|
8378
|
+
blendOut;
|
|
8379
|
+
weight = 0;
|
|
8380
|
+
/** The last applied pose, to blend out of it after the frames end. */
|
|
8381
|
+
lastPose = null;
|
|
8382
|
+
scratch = {
|
|
8383
|
+
from: new Quaternion3(),
|
|
8384
|
+
to: new Quaternion3(),
|
|
8385
|
+
carry: /* @__PURE__ */ new Map(),
|
|
8386
|
+
position: new Vector37()
|
|
8387
|
+
};
|
|
8388
|
+
constructor(options = {}) {
|
|
8389
|
+
this.rootMode = options.rootMode ?? "anchored";
|
|
8390
|
+
this.blendIn = options.blendInSeconds ?? 0.3;
|
|
8391
|
+
this.blendOut = options.blendOutSeconds ?? 0.5;
|
|
8392
|
+
}
|
|
8393
|
+
setVrm(vrm) {
|
|
8394
|
+
this.vrm = vrm;
|
|
8395
|
+
this.weight = 0;
|
|
8396
|
+
this.lastPose = null;
|
|
8397
|
+
}
|
|
8398
|
+
/** Adds the frames of a `motion_frames` event; invalid events are dropped. */
|
|
8399
|
+
push(received) {
|
|
8400
|
+
const frames = received.frames;
|
|
8401
|
+
const event = frames instanceof ArrayBuffer ? { ...received, frames: new Uint8Array(frames) } : received;
|
|
8402
|
+
if (!isValidMotionFramesEvent(event)) {
|
|
8403
|
+
return false;
|
|
8404
|
+
}
|
|
8405
|
+
const values = readMotionFrames(
|
|
8406
|
+
event.frames,
|
|
8407
|
+
event.frameCount,
|
|
8408
|
+
event.bones.length
|
|
8409
|
+
);
|
|
8410
|
+
const track = this.track(event.stagingId, {
|
|
8411
|
+
fps: event.fps,
|
|
8412
|
+
bones: event.bones,
|
|
8413
|
+
restHipsHeight: event.restHipsHeight
|
|
8414
|
+
});
|
|
8415
|
+
for (let frame = 0; frame < event.frameCount; frame += 1) {
|
|
8416
|
+
const index = event.startFrame + frame;
|
|
8417
|
+
const existing = track.frames.get(index);
|
|
8418
|
+
if (existing && existing.revision > event.revision) {
|
|
8419
|
+
continue;
|
|
8420
|
+
}
|
|
8421
|
+
track.frames.set(index, {
|
|
8422
|
+
revision: event.revision,
|
|
8423
|
+
values: values.subarray(frame * track.width, (frame + 1) * track.width)
|
|
8424
|
+
});
|
|
8425
|
+
}
|
|
8426
|
+
return true;
|
|
8427
|
+
}
|
|
8428
|
+
/** Loads the stored frames of a staging (a replay). */
|
|
8429
|
+
async load(stagingId, track, fetchBytes = async (url) => new Uint8Array(await (await fetch(url)).arrayBuffer())) {
|
|
8430
|
+
if (this.tracks.has(stagingId)) {
|
|
8431
|
+
return;
|
|
8432
|
+
}
|
|
8433
|
+
const data = track.frames.type === "bytes" ? track.frames.data : await fetchBytes(track.frames.url);
|
|
8434
|
+
const values = readMotionFrames(data, track.frameCount, track.bones.length);
|
|
8435
|
+
const stored = this.track(stagingId, track);
|
|
8436
|
+
for (let frame = 0; frame < track.frameCount; frame += 1) {
|
|
8437
|
+
stored.frames.set(frame, {
|
|
8438
|
+
revision: Number.MAX_SAFE_INTEGER,
|
|
8439
|
+
values: values.subarray(
|
|
8440
|
+
frame * stored.width,
|
|
8441
|
+
(frame + 1) * stored.width
|
|
8442
|
+
)
|
|
8443
|
+
});
|
|
8444
|
+
}
|
|
8445
|
+
}
|
|
8446
|
+
has(stagingId) {
|
|
8447
|
+
return (this.tracks.get(stagingId)?.frames.size ?? 0) > 0;
|
|
8448
|
+
}
|
|
8449
|
+
forget(stagingId) {
|
|
8450
|
+
this.tracks.delete(stagingId);
|
|
8451
|
+
}
|
|
8452
|
+
/** The current live weight, 0 (idle) to 1 (live frames). */
|
|
8453
|
+
get liveWeight() {
|
|
8454
|
+
return this.weight;
|
|
8455
|
+
}
|
|
8456
|
+
/**
|
|
8457
|
+
* Applies the live pose of `stagingId` at `time` seconds of its staging
|
|
8458
|
+
* clock; `time` null means that no staging plays now.
|
|
8459
|
+
*/
|
|
8460
|
+
update(delta, stagingId, time) {
|
|
8461
|
+
const vrm = this.vrm;
|
|
8462
|
+
if (!vrm?.humanoid) {
|
|
8463
|
+
return;
|
|
8464
|
+
}
|
|
8465
|
+
const track = stagingId ? this.tracks.get(stagingId) : void 0;
|
|
8466
|
+
let pose = null;
|
|
8467
|
+
if (track && time !== null && time >= 0) {
|
|
8468
|
+
pose = this.sample(track, time);
|
|
8469
|
+
}
|
|
8470
|
+
this.weight = approach(
|
|
8471
|
+
this.weight,
|
|
8472
|
+
pose ? 1 : 0,
|
|
8473
|
+
Math.max(0, delta),
|
|
8474
|
+
pose ? this.blendIn : this.blendOut
|
|
8475
|
+
);
|
|
8476
|
+
if (pose) {
|
|
8477
|
+
this.lastPose = pose;
|
|
8478
|
+
}
|
|
8479
|
+
const applied = pose ?? this.lastPose;
|
|
8480
|
+
if (!applied || this.weight <= 0) {
|
|
8481
|
+
if (this.weight <= 0) {
|
|
8482
|
+
this.lastPose = null;
|
|
8483
|
+
}
|
|
8484
|
+
return;
|
|
8485
|
+
}
|
|
8486
|
+
this.apply(vrm, applied.track, applied.values, this.weight);
|
|
8487
|
+
}
|
|
8488
|
+
track(stagingId, meta) {
|
|
8489
|
+
let track = this.tracks.get(stagingId);
|
|
8490
|
+
if (!track) {
|
|
8491
|
+
track = {
|
|
8492
|
+
fps: meta.fps,
|
|
8493
|
+
bones: [...meta.bones],
|
|
8494
|
+
restHipsHeight: meta.restHipsHeight,
|
|
8495
|
+
width: motionFrameWidth(meta.bones.length),
|
|
8496
|
+
frames: /* @__PURE__ */ new Map(),
|
|
8497
|
+
origin: null
|
|
8498
|
+
};
|
|
8499
|
+
this.tracks.set(stagingId, track);
|
|
8500
|
+
while (this.tracks.size > MAX_TRACKS) {
|
|
8501
|
+
const oldest = this.tracks.keys().next().value;
|
|
8502
|
+
if (oldest === void 0 || oldest === stagingId) {
|
|
8503
|
+
break;
|
|
8504
|
+
}
|
|
8505
|
+
this.tracks.delete(oldest);
|
|
8506
|
+
}
|
|
8507
|
+
}
|
|
8508
|
+
return track;
|
|
8509
|
+
}
|
|
8510
|
+
/** The interpolated frame at `time`, or null when its frames are missing. */
|
|
8511
|
+
sample(track, time) {
|
|
8512
|
+
const position = time * track.fps;
|
|
8513
|
+
const first = Math.floor(position);
|
|
8514
|
+
const a = track.frames.get(first);
|
|
8515
|
+
if (!a) {
|
|
8516
|
+
return null;
|
|
8517
|
+
}
|
|
8518
|
+
track.origin ??= { x: a.values[0] ?? 0, z: a.values[2] ?? 0 };
|
|
8519
|
+
const b = track.frames.get(first + 1);
|
|
8520
|
+
const amount = position - first;
|
|
8521
|
+
if (!b || amount <= 0) {
|
|
8522
|
+
return { track, values: a.values };
|
|
8523
|
+
}
|
|
8524
|
+
const values = new Float32Array(track.width);
|
|
8525
|
+
for (let index = 0; index < 3; index += 1) {
|
|
8526
|
+
values[index] = (a.values[index] ?? 0) * (1 - amount) + (b.values[index] ?? 0) * amount;
|
|
8527
|
+
}
|
|
8528
|
+
const from = this.scratch.from;
|
|
8529
|
+
const to = this.scratch.to;
|
|
8530
|
+
for (let offset = 3; offset < track.width; offset += 4) {
|
|
8531
|
+
from.fromArray(a.values, offset);
|
|
8532
|
+
to.fromArray(b.values, offset);
|
|
8533
|
+
from.slerp(to, amount);
|
|
8534
|
+
values[offset] = from.x;
|
|
8535
|
+
values[offset + 1] = from.y;
|
|
8536
|
+
values[offset + 2] = from.z;
|
|
8537
|
+
values[offset + 3] = from.w;
|
|
8538
|
+
}
|
|
8539
|
+
return { track, values };
|
|
8540
|
+
}
|
|
8541
|
+
apply(vrm, track, values, weight) {
|
|
8542
|
+
const humanoid = vrm.humanoid;
|
|
8543
|
+
const flip = vrm.meta.metaVersion === "0" && !isNpcifyNormalizedVrm(vrm);
|
|
8544
|
+
const carry = this.scratch.carry;
|
|
8545
|
+
carry.clear();
|
|
8546
|
+
track.bones.forEach((bone, slot) => {
|
|
8547
|
+
const offset = 3 + slot * 4;
|
|
8548
|
+
const rotation = new Quaternion3(
|
|
8549
|
+
values[offset] ?? 0,
|
|
8550
|
+
values[offset + 1] ?? 0,
|
|
8551
|
+
values[offset + 2] ?? 0,
|
|
8552
|
+
values[offset + 3] ?? 1
|
|
8553
|
+
);
|
|
8554
|
+
if (flip) {
|
|
8555
|
+
rotation.x = -rotation.x;
|
|
8556
|
+
rotation.z = -rotation.z;
|
|
8557
|
+
}
|
|
8558
|
+
const inherited = carry.get(bone);
|
|
8559
|
+
if (inherited) {
|
|
8560
|
+
rotation.premultiply(inherited);
|
|
8561
|
+
}
|
|
8562
|
+
const node = humanoid.getNormalizedBoneNode(bone);
|
|
8563
|
+
if (!node) {
|
|
8564
|
+
for (const [child, parent] of Object.entries(PARENTS)) {
|
|
8565
|
+
if (parent === bone) {
|
|
8566
|
+
carry.set(child, rotation.clone());
|
|
8567
|
+
}
|
|
8568
|
+
}
|
|
8569
|
+
return;
|
|
8570
|
+
}
|
|
8571
|
+
node.quaternion.slerp(rotation, weight);
|
|
8572
|
+
});
|
|
8573
|
+
const hips = humanoid.getNormalizedBoneNode("hips");
|
|
8574
|
+
const rest = humanoid.normalizedRestPose.hips?.position;
|
|
8575
|
+
if (!hips || !rest) {
|
|
8576
|
+
return;
|
|
8577
|
+
}
|
|
8578
|
+
const scale = (rest[1] ?? 1) / (track.restHipsHeight || 1);
|
|
8579
|
+
let x = values[0] ?? 0;
|
|
8580
|
+
const y = values[1] ?? 0;
|
|
8581
|
+
let z = values[2] ?? 0;
|
|
8582
|
+
if (this.rootMode === "anchored" && track.origin) {
|
|
8583
|
+
x -= track.origin.x;
|
|
8584
|
+
z -= track.origin.z;
|
|
8585
|
+
}
|
|
8586
|
+
const target = this.scratch.position.set(
|
|
8587
|
+
(flip ? -x : x) * scale + (this.rootMode === "anchored" ? rest[0] ?? 0 : 0),
|
|
8588
|
+
y * scale,
|
|
8589
|
+
(flip ? -z : z) * scale + (this.rootMode === "anchored" ? rest[2] ?? 0 : 0)
|
|
8590
|
+
);
|
|
8591
|
+
hips.position.lerp(target, weight);
|
|
8592
|
+
}
|
|
8593
|
+
};
|
|
8594
|
+
|
|
8240
8595
|
// ../../clients/web-client/src/character/incremental-staging.ts
|
|
8241
8596
|
var dialogueKey = function(block) {
|
|
8242
8597
|
return JSON.stringify(block, (key, value) => {
|
|
@@ -8511,9 +8866,17 @@ var StagingPlayer = class {
|
|
|
8511
8866
|
activeExpression = null;
|
|
8512
8867
|
activeExpressionMouthMode = null;
|
|
8513
8868
|
activeAnimation = null;
|
|
8869
|
+
currentTrackTime = null;
|
|
8514
8870
|
get paused() {
|
|
8515
8871
|
return this.clock.paused;
|
|
8516
8872
|
}
|
|
8873
|
+
/**
|
|
8874
|
+
* Seconds of the staging clock that the tracks show now, or null when no
|
|
8875
|
+
* staging plays (also while playback waits for the next dialogue block).
|
|
8876
|
+
*/
|
|
8877
|
+
get trackTime() {
|
|
8878
|
+
return this.currentTrackTime;
|
|
8879
|
+
}
|
|
8517
8880
|
pause() {
|
|
8518
8881
|
this.pauseRevision += 1;
|
|
8519
8882
|
this.clock.pause();
|
|
@@ -8914,6 +9277,7 @@ var StagingPlayer = class {
|
|
|
8914
9277
|
}
|
|
8915
9278
|
}
|
|
8916
9279
|
applyTracks(time, speechActive, timeline) {
|
|
9280
|
+
this.currentTrackTime = time;
|
|
8917
9281
|
const expression = activeClipAt(timeline.expressions, time);
|
|
8918
9282
|
const mouthMode = speechActive ? "viseme" : "expression";
|
|
8919
9283
|
if (expression !== this.activeExpression || expression && mouthMode !== this.activeExpressionMouthMode) {
|
|
@@ -8947,6 +9311,7 @@ var StagingPlayer = class {
|
|
|
8947
9311
|
}
|
|
8948
9312
|
}
|
|
8949
9313
|
clearPerformance() {
|
|
9314
|
+
this.currentTrackTime = null;
|
|
8950
9315
|
this.activeExpression = null;
|
|
8951
9316
|
this.activeExpressionMouthMode = null;
|
|
8952
9317
|
this.activeAnimation = null;
|
|
@@ -9385,6 +9750,7 @@ var CharacterRuntime = class {
|
|
|
9385
9750
|
performanceRecorder;
|
|
9386
9751
|
telemetry;
|
|
9387
9752
|
streamPlayback;
|
|
9753
|
+
liveMotion;
|
|
9388
9754
|
defaultParent;
|
|
9389
9755
|
currentGazeTarget = null;
|
|
9390
9756
|
/** The current gaze target came from `lookAt()`. */
|
|
@@ -9433,12 +9799,21 @@ var CharacterRuntime = class {
|
|
|
9433
9799
|
baseUrl: this.baseUrl,
|
|
9434
9800
|
apiToken: this.apiToken
|
|
9435
9801
|
});
|
|
9802
|
+
this.liveMotion = new LiveMotionPlayer(options.liveMotion);
|
|
9436
9803
|
this.streamPlayback = new StreamPlaybackController({
|
|
9437
9804
|
getStagingPlayer: this.getStagingPlayer,
|
|
9438
9805
|
telemetry: this.telemetry,
|
|
9439
9806
|
handleCallbackError: this.handleCallbackError,
|
|
9440
9807
|
reportError: this.reportError,
|
|
9441
|
-
recordEvent: (event) => this.performanceRecorder.recordEvent(event)
|
|
9808
|
+
recordEvent: (event) => this.performanceRecorder.recordEvent(event),
|
|
9809
|
+
onMotionFrames: (event) => {
|
|
9810
|
+
this.liveMotion.push(event);
|
|
9811
|
+
},
|
|
9812
|
+
onMotionTrack: (stagingId, track) => {
|
|
9813
|
+
void this.liveMotion.load(stagingId, track).catch((error) => {
|
|
9814
|
+
this.handleCallbackError("liveMotion", error);
|
|
9815
|
+
});
|
|
9816
|
+
}
|
|
9442
9817
|
});
|
|
9443
9818
|
if (options.gazeTarget !== void 0) {
|
|
9444
9819
|
this.setGazeTarget(options.gazeTarget);
|
|
@@ -9475,6 +9850,7 @@ var CharacterRuntime = class {
|
|
|
9475
9850
|
set vrm(value) {
|
|
9476
9851
|
this.sceneRig.setVrm(value);
|
|
9477
9852
|
this.facialMixer.setVrm(value);
|
|
9853
|
+
this.liveMotion.setVrm(value);
|
|
9478
9854
|
this.applyGaze();
|
|
9479
9855
|
if (value && this.defaultParent && !value.scene?.parent) {
|
|
9480
9856
|
this.sceneRig.attachTo(this.defaultParent);
|
|
@@ -9751,6 +10127,11 @@ var CharacterRuntime = class {
|
|
|
9751
10127
|
return;
|
|
9752
10128
|
}
|
|
9753
10129
|
this.animationController?.update(delta);
|
|
10130
|
+
this.liveMotion.update(
|
|
10131
|
+
delta,
|
|
10132
|
+
this.streamPlayback.activeStagingId,
|
|
10133
|
+
this.stagingPlayer?.trackTime ?? null
|
|
10134
|
+
);
|
|
9754
10135
|
this.visemePlayer.update(delta);
|
|
9755
10136
|
this.facialMixer.update(delta);
|
|
9756
10137
|
this.vrm?.update(delta);
|
|
@@ -9807,6 +10188,7 @@ var CharacterRuntime = class {
|
|
|
9807
10188
|
this.animations.clear();
|
|
9808
10189
|
this.sceneRig.setVrm(null);
|
|
9809
10190
|
this.facialMixer.setVrm(null);
|
|
10191
|
+
this.liveMotion.setVrm(null);
|
|
9810
10192
|
}
|
|
9811
10193
|
disposeRuntimeAssets() {
|
|
9812
10194
|
this.sceneRig.disposeVrm();
|
|
@@ -9829,28 +10211,101 @@ var CharacterRuntime = class {
|
|
|
9829
10211
|
|
|
9830
10212
|
// ../../clients/web-client/src/conversation-socket.ts
|
|
9831
10213
|
import { io } from "socket.io-client";
|
|
10214
|
+
var ConversationConnectionError = class extends Error {
|
|
10215
|
+
constructor(code, message, reason = null) {
|
|
10216
|
+
super(message);
|
|
10217
|
+
this.code = code;
|
|
10218
|
+
this.reason = reason;
|
|
10219
|
+
this.name = "NpcifyConversationConnectionError";
|
|
10220
|
+
}
|
|
10221
|
+
};
|
|
10222
|
+
var JOIN_EVENT = "conversations:join";
|
|
10223
|
+
var JOIN_TIMEOUT_MS = 1e4;
|
|
10224
|
+
var ACCESS_DENIED = "CONVERSATION_ACCESS_DENIED";
|
|
10225
|
+
var JOIN_FAILED = "CONVERSATION_JOIN_FAILED";
|
|
10226
|
+
var CLIENT_CLOSE = "io client disconnect";
|
|
10227
|
+
var SOCKET_IO_TIMEOUT = "operation has timed out";
|
|
10228
|
+
var joinMessage = function(code) {
|
|
10229
|
+
if (code === ACCESS_DENIED) {
|
|
10230
|
+
return "Conversation streaming access was denied.";
|
|
10231
|
+
}
|
|
10232
|
+
if (code === JOIN_FAILED) {
|
|
10233
|
+
return "The server could not join the conversation. Try again later.";
|
|
10234
|
+
}
|
|
10235
|
+
return `The server did not join the conversation (${code}). Try again later.`;
|
|
10236
|
+
};
|
|
10237
|
+
var closeMessage = function(reason, detail) {
|
|
10238
|
+
if (reason === "io server disconnect") {
|
|
10239
|
+
return "The server closed the conversation connection.";
|
|
10240
|
+
}
|
|
10241
|
+
if (reason === CLIENT_CLOSE) {
|
|
10242
|
+
return "The conversation connection closed.";
|
|
10243
|
+
}
|
|
10244
|
+
if (reason === "connect_error") {
|
|
10245
|
+
return `The server refused the conversation connection (${detail || "no reason"}).`;
|
|
10246
|
+
}
|
|
10247
|
+
return `The conversation connection was lost (${reason}).`;
|
|
10248
|
+
};
|
|
10249
|
+
var acknowledgementError = function(error, lost) {
|
|
10250
|
+
if (error.message === SOCKET_IO_TIMEOUT) {
|
|
10251
|
+
return new ConversationConnectionError(
|
|
10252
|
+
"CONVERSATION_TIMEOUT",
|
|
10253
|
+
"The server did not answer in time."
|
|
10254
|
+
);
|
|
10255
|
+
}
|
|
10256
|
+
return new ConversationConnectionError(
|
|
10257
|
+
"CONVERSATION_DISCONNECTED",
|
|
10258
|
+
closeMessage(lost ?? "transport close"),
|
|
10259
|
+
lost
|
|
10260
|
+
);
|
|
10261
|
+
};
|
|
10262
|
+
var refusedConversation = function(payload) {
|
|
10263
|
+
const exception = payload;
|
|
10264
|
+
const named = exception?.cause?.data?.conversationId;
|
|
10265
|
+
if (exception?.message !== ACCESS_DENIED || typeof named !== "string") {
|
|
10266
|
+
return null;
|
|
10267
|
+
}
|
|
10268
|
+
return named.trim() || null;
|
|
10269
|
+
};
|
|
9832
10270
|
var ConversationSocket = class {
|
|
9833
10271
|
options;
|
|
9834
|
-
|
|
10272
|
+
connection = null;
|
|
10273
|
+
/**
|
|
10274
|
+
* The conversations that the server refused. A join with the same
|
|
10275
|
+
* credentials fails the same way, so the SDK does not try them again until
|
|
10276
|
+
* `disconnect`.
|
|
10277
|
+
*/
|
|
10278
|
+
refused = /* @__PURE__ */ new Set();
|
|
9835
10279
|
constructor(options) {
|
|
9836
10280
|
this.options = options;
|
|
9837
10281
|
}
|
|
10282
|
+
/** Closes the connection and forgets the refused conversations. */
|
|
9838
10283
|
disconnect() {
|
|
9839
|
-
this.
|
|
9840
|
-
this.
|
|
10284
|
+
this.refused.clear();
|
|
10285
|
+
if (this.connection) {
|
|
10286
|
+
this.close(this.connection, CLIENT_CLOSE);
|
|
10287
|
+
}
|
|
9841
10288
|
}
|
|
9842
10289
|
/** An event with an acknowledgement. Needs `ensureConnected` first. */
|
|
9843
10290
|
async request(event, payload, timeoutMs = 1e4) {
|
|
9844
|
-
const
|
|
9845
|
-
if (!socket
|
|
9846
|
-
throw new
|
|
10291
|
+
const connection = this.connection;
|
|
10292
|
+
if (!connection?.socket.connected) {
|
|
10293
|
+
throw new ConversationConnectionError(
|
|
10294
|
+
"CONVERSATION_DISCONNECTED",
|
|
10295
|
+
"The conversation connection is not open."
|
|
10296
|
+
);
|
|
9847
10297
|
}
|
|
9848
|
-
return
|
|
10298
|
+
return this.acknowledged(connection, event, payload, timeoutMs, null);
|
|
9849
10299
|
}
|
|
9850
10300
|
/** An event without an acknowledgement, in order with the other events. */
|
|
9851
10301
|
send(event, payload) {
|
|
9852
|
-
this.socket
|
|
10302
|
+
this.connection?.socket.emit(event, payload);
|
|
9853
10303
|
}
|
|
10304
|
+
/**
|
|
10305
|
+
* Joins the room of the conversation. A connection that the server closed
|
|
10306
|
+
* is not used again: the join opens a new one. A refused conversation fails
|
|
10307
|
+
* at once, without a connection.
|
|
10308
|
+
*/
|
|
9854
10309
|
async ensureConnected(conversationId) {
|
|
9855
10310
|
if (typeof window === "undefined") {
|
|
9856
10311
|
return;
|
|
@@ -9859,135 +10314,279 @@ var ConversationSocket = class {
|
|
|
9859
10314
|
if (!normalized) {
|
|
9860
10315
|
return;
|
|
9861
10316
|
}
|
|
9862
|
-
if (
|
|
9863
|
-
|
|
9864
|
-
|
|
9865
|
-
|
|
9866
|
-
|
|
9867
|
-
|
|
9868
|
-
|
|
9869
|
-
|
|
9870
|
-
|
|
9871
|
-
|
|
9872
|
-
|
|
9873
|
-
|
|
9874
|
-
|
|
9875
|
-
|
|
9876
|
-
|
|
9877
|
-
|
|
9878
|
-
|
|
9879
|
-
|
|
9880
|
-
|
|
9881
|
-
|
|
9882
|
-
|
|
9883
|
-
|
|
9884
|
-
}
|
|
9885
|
-
|
|
9886
|
-
|
|
9887
|
-
|
|
9888
|
-
|
|
9889
|
-
|
|
9890
|
-
|
|
9891
|
-
|
|
9892
|
-
|
|
9893
|
-
|
|
9894
|
-
|
|
9895
|
-
|
|
9896
|
-
|
|
9897
|
-
|
|
9898
|
-
|
|
9899
|
-
|
|
9900
|
-
|
|
9901
|
-
|
|
9902
|
-
|
|
9903
|
-
|
|
9904
|
-
|
|
9905
|
-
|
|
9906
|
-
|
|
9907
|
-
|
|
9908
|
-
|
|
9909
|
-
|
|
9910
|
-
|
|
9911
|
-
|
|
9912
|
-
|
|
10317
|
+
if (this.refused.has(normalized)) {
|
|
10318
|
+
throw new ConversationConnectionError(
|
|
10319
|
+
ACCESS_DENIED,
|
|
10320
|
+
joinMessage(ACCESS_DENIED)
|
|
10321
|
+
);
|
|
10322
|
+
}
|
|
10323
|
+
const connection = this.connection ?? this.open();
|
|
10324
|
+
await this.connected(connection);
|
|
10325
|
+
await this.acknowledged(
|
|
10326
|
+
connection,
|
|
10327
|
+
JOIN_EVENT,
|
|
10328
|
+
{ conversationId: normalized },
|
|
10329
|
+
this.options.joinTimeoutMs ?? JOIN_TIMEOUT_MS,
|
|
10330
|
+
normalized
|
|
10331
|
+
);
|
|
10332
|
+
}
|
|
10333
|
+
open() {
|
|
10334
|
+
const socketBaseUrl = (() => {
|
|
10335
|
+
try {
|
|
10336
|
+
return new URL(this.options.baseUrl).origin;
|
|
10337
|
+
} catch {
|
|
10338
|
+
return window.location.origin;
|
|
10339
|
+
}
|
|
10340
|
+
})();
|
|
10341
|
+
const socket = io(socketBaseUrl, {
|
|
10342
|
+
transports: ["websocket"],
|
|
10343
|
+
withCredentials: true,
|
|
10344
|
+
forceNew: true,
|
|
10345
|
+
auth: {
|
|
10346
|
+
apiToken: this.options.apiToken
|
|
10347
|
+
}
|
|
10348
|
+
});
|
|
10349
|
+
const connection = {
|
|
10350
|
+
socket,
|
|
10351
|
+
pending: /* @__PURE__ */ new Set(),
|
|
10352
|
+
connectedBefore: false,
|
|
10353
|
+
lost: null,
|
|
10354
|
+
closed: null,
|
|
10355
|
+
failure: null
|
|
10356
|
+
};
|
|
10357
|
+
this.connection = connection;
|
|
10358
|
+
this.watch(connection);
|
|
10359
|
+
this.receive(socket);
|
|
10360
|
+
return connection;
|
|
10361
|
+
}
|
|
10362
|
+
/** The lifecycle of one connection (docs/streaming_reply.md#web-sdk). */
|
|
10363
|
+
watch(connection) {
|
|
10364
|
+
const { socket } = connection;
|
|
10365
|
+
socket.on("connect", () => {
|
|
10366
|
+
if (!connection.connectedBefore) {
|
|
10367
|
+
connection.connectedBefore = true;
|
|
10368
|
+
return;
|
|
10369
|
+
}
|
|
10370
|
+
const activeConversationId = this.options.getConversationId()?.trim();
|
|
10371
|
+
if (activeConversationId && !this.refused.has(activeConversationId)) {
|
|
10372
|
+
socket.emit(JOIN_EVENT, { conversationId: activeConversationId });
|
|
10373
|
+
}
|
|
10374
|
+
});
|
|
10375
|
+
socket.on("exception", (payload) => {
|
|
10376
|
+
const conversationId = refusedConversation(payload);
|
|
10377
|
+
if (conversationId) {
|
|
10378
|
+
connection.failure ??= { code: ACCESS_DENIED, conversationId };
|
|
10379
|
+
this.refused.add(conversationId);
|
|
10380
|
+
}
|
|
10381
|
+
});
|
|
10382
|
+
socket.on("disconnect", (reason) => {
|
|
10383
|
+
if (!socket.active) {
|
|
10384
|
+
this.close(connection, reason);
|
|
10385
|
+
return;
|
|
10386
|
+
}
|
|
10387
|
+
connection.lost = reason;
|
|
10388
|
+
this.options.handleSpeechEvent?.({ type: "disconnect" });
|
|
10389
|
+
});
|
|
10390
|
+
socket.on("connect_error", (error) => {
|
|
10391
|
+
if (!socket.active) {
|
|
10392
|
+
this.close(connection, "connect_error", error.message);
|
|
10393
|
+
}
|
|
10394
|
+
});
|
|
10395
|
+
}
|
|
10396
|
+
/**
|
|
10397
|
+
* Closes the connection for good: socket.io does not open it again. The
|
|
10398
|
+
* waits fail now, and the next join opens a new connection.
|
|
10399
|
+
*/
|
|
10400
|
+
close(connection, reason, detail) {
|
|
10401
|
+
if (connection.closed) {
|
|
10402
|
+
return;
|
|
10403
|
+
}
|
|
10404
|
+
connection.closed = { reason, message: closeMessage(reason, detail) };
|
|
10405
|
+
if (this.connection === connection) {
|
|
10406
|
+
this.connection = null;
|
|
10407
|
+
}
|
|
10408
|
+
const { socket } = connection;
|
|
10409
|
+
socket.removeAllListeners();
|
|
10410
|
+
if (connection.connectedBefore) {
|
|
10411
|
+
this.options.handleSpeechEvent?.({ type: "disconnect" });
|
|
10412
|
+
}
|
|
10413
|
+
const pending = [...connection.pending];
|
|
10414
|
+
connection.pending.clear();
|
|
10415
|
+
for (const entry of pending) {
|
|
10416
|
+
entry.fail(this.closeError(connection, entry.conversationId));
|
|
10417
|
+
}
|
|
10418
|
+
socket.disconnect();
|
|
10419
|
+
if (reason !== CLIENT_CLOSE) {
|
|
10420
|
+
this.options.handleConnectionClosed?.({
|
|
10421
|
+
reason,
|
|
10422
|
+
code: connection.failure?.code ?? null,
|
|
10423
|
+
conversationId: connection.failure?.conversationId ?? null
|
|
9913
10424
|
});
|
|
9914
|
-
|
|
9915
|
-
|
|
9916
|
-
|
|
9917
|
-
|
|
9918
|
-
|
|
9919
|
-
|
|
9920
|
-
|
|
9921
|
-
|
|
9922
|
-
|
|
9923
|
-
|
|
10425
|
+
}
|
|
10426
|
+
}
|
|
10427
|
+
closeError(connection, conversationId) {
|
|
10428
|
+
if (conversationId !== null && this.refused.has(conversationId)) {
|
|
10429
|
+
return new ConversationConnectionError(
|
|
10430
|
+
ACCESS_DENIED,
|
|
10431
|
+
joinMessage(ACCESS_DENIED)
|
|
10432
|
+
);
|
|
10433
|
+
}
|
|
10434
|
+
return new ConversationConnectionError(
|
|
10435
|
+
"CONVERSATION_DISCONNECTED",
|
|
10436
|
+
connection.closed?.message ?? closeMessage(CLIENT_CLOSE),
|
|
10437
|
+
connection.closed?.reason ?? null
|
|
10438
|
+
);
|
|
10439
|
+
}
|
|
10440
|
+
/**
|
|
10441
|
+
* Waits for the first connection. A connection error that socket.io
|
|
10442
|
+
* recovers from also ends the wait: the join then waits in socket.io's
|
|
10443
|
+
* buffer, for the time of its acknowledgement at most.
|
|
10444
|
+
*/
|
|
10445
|
+
connected(connection) {
|
|
10446
|
+
const { socket } = connection;
|
|
10447
|
+
if (connection.closed) {
|
|
10448
|
+
return Promise.reject(this.closeError(connection, null));
|
|
10449
|
+
}
|
|
10450
|
+
if (socket.connected || connection.connectedBefore) {
|
|
10451
|
+
return Promise.resolve();
|
|
10452
|
+
}
|
|
10453
|
+
return new Promise((resolve, reject) => {
|
|
10454
|
+
const entry = { conversationId: null, fail: reject };
|
|
10455
|
+
const done = () => {
|
|
10456
|
+
if (connection.pending.delete(entry)) {
|
|
10457
|
+
socket.off("connect", done);
|
|
10458
|
+
socket.off("connect_error", done);
|
|
10459
|
+
resolve();
|
|
9924
10460
|
}
|
|
9925
|
-
|
|
10461
|
+
};
|
|
10462
|
+
connection.pending.add(entry);
|
|
10463
|
+
socket.on("connect", done);
|
|
10464
|
+
socket.on("connect_error", done);
|
|
10465
|
+
});
|
|
10466
|
+
}
|
|
10467
|
+
/**
|
|
10468
|
+
* Emits an event and waits for its acknowledgement. A join (`joining`)
|
|
10469
|
+
* that the server does not accept fails with the code of its
|
|
10470
|
+
* acknowledgement. A close of the connection fails the wait at once; a
|
|
10471
|
+
* timeout fails it with `CONVERSATION_TIMEOUT`.
|
|
10472
|
+
*/
|
|
10473
|
+
acknowledged(connection, event, payload, timeoutMs, joining) {
|
|
10474
|
+
return new Promise((resolve, reject) => {
|
|
10475
|
+
if (connection.closed) {
|
|
10476
|
+
reject(this.closeError(connection, joining));
|
|
10477
|
+
return;
|
|
10478
|
+
}
|
|
10479
|
+
const entry = { conversationId: joining, fail: reject };
|
|
10480
|
+
connection.pending.add(entry);
|
|
10481
|
+
connection.socket.timeout(timeoutMs).emit(event, payload, (error, result) => {
|
|
10482
|
+
if (!connection.pending.delete(entry)) {
|
|
9926
10483
|
return;
|
|
9927
10484
|
}
|
|
9928
|
-
this.
|
|
9929
|
-
|
|
9930
|
-
|
|
9931
|
-
|
|
9932
|
-
|
|
9933
|
-
});
|
|
9934
|
-
this.socket.on(SPEECH_EVENTS.turnFailed, (payload) => {
|
|
9935
|
-
const event = payload;
|
|
9936
|
-
if (typeof event?.requestId === "string" && event.conversationId === this.options.getConversationId()) {
|
|
9937
|
-
this.options.handleSpeechEvent?.({ type: "turnFailed", event });
|
|
10485
|
+
const failure = error ? acknowledgementError(error, connection.lost) : joining === null ? null : this.joinFailure(connection, joining, result);
|
|
10486
|
+
if (failure) {
|
|
10487
|
+
reject(failure);
|
|
10488
|
+
} else {
|
|
10489
|
+
resolve(result);
|
|
9938
10490
|
}
|
|
9939
10491
|
});
|
|
9940
|
-
|
|
9941
|
-
|
|
9942
|
-
|
|
9943
|
-
|
|
9944
|
-
|
|
9945
|
-
|
|
9946
|
-
|
|
9947
|
-
|
|
9948
|
-
|
|
9949
|
-
|
|
9950
|
-
|
|
9951
|
-
|
|
9952
|
-
|
|
9953
|
-
|
|
9954
|
-
|
|
9955
|
-
|
|
9956
|
-
|
|
9957
|
-
|
|
9958
|
-
|
|
9959
|
-
|
|
9960
|
-
|
|
9961
|
-
|
|
9962
|
-
|
|
9963
|
-
|
|
9964
|
-
|
|
9965
|
-
|
|
9966
|
-
|
|
10492
|
+
});
|
|
10493
|
+
}
|
|
10494
|
+
/** The error of a join that the server did not accept; null after a join. */
|
|
10495
|
+
joinFailure(connection, conversationId, result) {
|
|
10496
|
+
const ack = result;
|
|
10497
|
+
if (ack?.ok) {
|
|
10498
|
+
return null;
|
|
10499
|
+
}
|
|
10500
|
+
const named = typeof ack?.error === "string" ? ack.error.trim() : "";
|
|
10501
|
+
const code = named || JOIN_FAILED;
|
|
10502
|
+
connection.failure ??= { code, conversationId };
|
|
10503
|
+
if (code === ACCESS_DENIED) {
|
|
10504
|
+
this.refused.add(conversationId);
|
|
10505
|
+
}
|
|
10506
|
+
this.close(connection, "io server disconnect");
|
|
10507
|
+
return new ConversationConnectionError(code, joinMessage(code));
|
|
10508
|
+
}
|
|
10509
|
+
/** The events that the server sends to the conversation's room. */
|
|
10510
|
+
receive(socket) {
|
|
10511
|
+
socket.on("conversations:stream", (payload) => {
|
|
10512
|
+
let candidate;
|
|
10513
|
+
try {
|
|
10514
|
+
candidate = decodePerformance(
|
|
10515
|
+
payload,
|
|
10516
|
+
"stream"
|
|
10517
|
+
);
|
|
10518
|
+
} catch (error) {
|
|
10519
|
+
this.options.handleCallbackError("stream.decode", error);
|
|
10520
|
+
return;
|
|
10521
|
+
}
|
|
10522
|
+
if (typeof candidate?.conversationId !== "string") {
|
|
10523
|
+
return;
|
|
10524
|
+
}
|
|
10525
|
+
if (candidate.conversationId !== this.options.getConversationId()) {
|
|
10526
|
+
return;
|
|
10527
|
+
}
|
|
10528
|
+
const event = candidate.event;
|
|
10529
|
+
if (!event || typeof event !== "object") {
|
|
10530
|
+
return;
|
|
10531
|
+
}
|
|
10532
|
+
this.options.handleStreamEvent(event);
|
|
10533
|
+
});
|
|
10534
|
+
socket.on(SPEECH_EVENTS.partial, (payload) => {
|
|
10535
|
+
const event = payload;
|
|
10536
|
+
if (typeof event?.sessionId === "string" && typeof event.text === "string") {
|
|
10537
|
+
this.options.handleSpeechEvent?.({ type: "partial", event });
|
|
10538
|
+
}
|
|
10539
|
+
});
|
|
10540
|
+
socket.on(SPEECH_EVENTS.turn, (payload) => {
|
|
10541
|
+
let candidate;
|
|
10542
|
+
try {
|
|
10543
|
+
candidate = decodePerformance(
|
|
10544
|
+
payload,
|
|
10545
|
+
"turn"
|
|
10546
|
+
);
|
|
10547
|
+
} catch (error) {
|
|
10548
|
+
this.options.handleCallbackError("speech.turn.decode", error);
|
|
10549
|
+
return;
|
|
10550
|
+
}
|
|
10551
|
+
if (candidate?.conversationId !== this.options.getConversationId() || typeof candidate.requestId !== "string" || !candidate.turn || typeof candidate.turn !== "object") {
|
|
10552
|
+
return;
|
|
10553
|
+
}
|
|
10554
|
+
this.options.handleSpeechEvent?.({
|
|
10555
|
+
type: "turn",
|
|
10556
|
+
requestId: candidate.requestId,
|
|
10557
|
+
turn: candidate.turn
|
|
9967
10558
|
});
|
|
9968
|
-
|
|
9969
|
-
|
|
9970
|
-
|
|
9971
|
-
|
|
9972
|
-
}
|
|
9973
|
-
|
|
9974
|
-
|
|
10559
|
+
});
|
|
10560
|
+
socket.on(SPEECH_EVENTS.turnFailed, (payload) => {
|
|
10561
|
+
const event = payload;
|
|
10562
|
+
if (typeof event?.requestId === "string" && event.conversationId === this.options.getConversationId()) {
|
|
10563
|
+
this.options.handleSpeechEvent?.({ type: "turnFailed", event });
|
|
10564
|
+
}
|
|
10565
|
+
});
|
|
10566
|
+
socket.on("conversations:turn", (payload) => {
|
|
10567
|
+
let candidate;
|
|
10568
|
+
try {
|
|
10569
|
+
candidate = decodePerformance(
|
|
10570
|
+
payload,
|
|
10571
|
+
"turn"
|
|
10572
|
+
);
|
|
10573
|
+
} catch (error) {
|
|
10574
|
+
this.options.handleCallbackError("turn.decode", error);
|
|
10575
|
+
return;
|
|
10576
|
+
}
|
|
10577
|
+
if (typeof candidate?.conversationId !== "string") {
|
|
10578
|
+
return;
|
|
10579
|
+
}
|
|
10580
|
+
if (candidate.conversationId !== this.options.getConversationId()) {
|
|
10581
|
+
return;
|
|
10582
|
+
}
|
|
10583
|
+
const turn = candidate.turn;
|
|
10584
|
+
if (!turn || typeof turn !== "object") {
|
|
10585
|
+
return;
|
|
10586
|
+
}
|
|
10587
|
+
void this.options.handleTurn(turn).catch((error) => {
|
|
10588
|
+
this.options.handleCallbackError("onTurn", error);
|
|
9975
10589
|
});
|
|
9976
|
-
}
|
|
9977
|
-
await new Promise((resolve, reject) => {
|
|
9978
|
-
this.socket.timeout(1e4).emit(
|
|
9979
|
-
"conversations:join",
|
|
9980
|
-
{ conversationId: normalized },
|
|
9981
|
-
(error, result) => {
|
|
9982
|
-
if (error || !result?.ok) {
|
|
9983
|
-
reject(
|
|
9984
|
-
error ?? new Error("Conversation streaming access was denied.")
|
|
9985
|
-
);
|
|
9986
|
-
} else {
|
|
9987
|
-
resolve();
|
|
9988
|
-
}
|
|
9989
|
-
}
|
|
9990
|
-
);
|
|
9991
10590
|
});
|
|
9992
10591
|
}
|
|
9993
10592
|
};
|
|
@@ -10944,6 +11543,8 @@ var Character = class extends CharacterRuntime {
|
|
|
10944
11543
|
notifiedStagingIds = /* @__PURE__ */ new Map();
|
|
10945
11544
|
liveConversation;
|
|
10946
11545
|
baked = null;
|
|
11546
|
+
/** The conversation whose live motion was warmed up. */
|
|
11547
|
+
liveMotionPrepared = null;
|
|
10947
11548
|
authoring;
|
|
10948
11549
|
normalizeIdleOrientation;
|
|
10949
11550
|
bakedAsset;
|
|
@@ -10951,9 +11552,12 @@ var Character = class extends CharacterRuntime {
|
|
|
10951
11552
|
voice;
|
|
10952
11553
|
configuredInputModes;
|
|
10953
11554
|
resolvedInputModes = null;
|
|
11555
|
+
/** The character's voice input switch, after the character loads. */
|
|
11556
|
+
speechInput = null;
|
|
10954
11557
|
latestOptions = [];
|
|
10955
11558
|
listeningStateCallback = null;
|
|
10956
11559
|
transcriptCallback = null;
|
|
11560
|
+
connectionClosedCallback = null;
|
|
10957
11561
|
conversationId = null;
|
|
10958
11562
|
constructor(options) {
|
|
10959
11563
|
const resolvedToken = (options.apiToken ?? options.apiKey)?.trim() ?? "";
|
|
@@ -10978,7 +11582,8 @@ var Character = class extends CharacterRuntime {
|
|
|
10978
11582
|
if (this.configuredInputModes && this.bakedAsset && !this.authoring) {
|
|
10979
11583
|
resolveInputModes(this.configuredInputModes, {
|
|
10980
11584
|
allowCustomReply: false,
|
|
10981
|
-
baked: true
|
|
11585
|
+
baked: true,
|
|
11586
|
+
speechInput: false
|
|
10982
11587
|
});
|
|
10983
11588
|
}
|
|
10984
11589
|
this.conversationSocket = new ConversationSocket({
|
|
@@ -10995,7 +11600,12 @@ var Character = class extends CharacterRuntime {
|
|
|
10995
11600
|
await this.handleStagingResult(turn, { source: "turn" });
|
|
10996
11601
|
},
|
|
10997
11602
|
handleCallbackError: this.handleCallbackError,
|
|
10998
|
-
handleSpeechEvent: (event) => this.voice.handleSocketEvent(event)
|
|
11603
|
+
handleSpeechEvent: (event) => this.voice.handleSocketEvent(event),
|
|
11604
|
+
handleConnectionClosed: (event) => invokeSafeCallback(
|
|
11605
|
+
"onConnectionClosed",
|
|
11606
|
+
this.handleCallbackError,
|
|
11607
|
+
() => this.connectionClosedCallback?.(event)
|
|
11608
|
+
)
|
|
10999
11609
|
});
|
|
11000
11610
|
this.voice = new VoiceInput({
|
|
11001
11611
|
socket: () => this.conversationSocket,
|
|
@@ -11065,9 +11675,11 @@ var Character = class extends CharacterRuntime {
|
|
|
11065
11675
|
const progressReporter = createAssetProgressReporter(callback);
|
|
11066
11676
|
const info = await this.getAssetInfo();
|
|
11067
11677
|
const baked = !this.authoring && Boolean(this.bakedAsset || info.bakedConversationEnabled);
|
|
11678
|
+
this.speechInput = info.speechInputEnabled === true;
|
|
11068
11679
|
this.resolvedInputModes = resolveInputModes(this.configuredInputModes, {
|
|
11069
11680
|
allowCustomReply: info.allowCustomReply !== false,
|
|
11070
|
-
baked
|
|
11681
|
+
baked,
|
|
11682
|
+
speechInput: this.speechInput
|
|
11071
11683
|
});
|
|
11072
11684
|
this.baked = baked ? await loadBakedConversation({
|
|
11073
11685
|
baseUrl: this.baseUrl,
|
|
@@ -11138,12 +11750,33 @@ var Character = class extends CharacterRuntime {
|
|
|
11138
11750
|
if (info.bakedConversationEnabled || this.bakedAsset) {
|
|
11139
11751
|
throw new Error("Baked conversation requires local rendering.");
|
|
11140
11752
|
}
|
|
11753
|
+
this.speechInput = info.speechInputEnabled === true;
|
|
11141
11754
|
this.resolvedInputModes = resolveInputModes(this.configuredInputModes, {
|
|
11142
11755
|
allowCustomReply: info.allowCustomReply !== false,
|
|
11143
|
-
baked: false
|
|
11756
|
+
baked: false,
|
|
11757
|
+
speechInput: this.speechInput
|
|
11144
11758
|
});
|
|
11145
11759
|
this.animationController = new RemoteAnimationController();
|
|
11146
11760
|
}
|
|
11761
|
+
/**
|
|
11762
|
+
* Warms up live motion for the current conversation, once, on the
|
|
11763
|
+
* player's intent: call it at the first focus or key in a reply field or
|
|
11764
|
+
* when a talk key goes down. It does nothing for a character without live
|
|
11765
|
+
* motion; the server limits warm-ups per player (ADR 0011).
|
|
11766
|
+
*/
|
|
11767
|
+
async prepareLiveMotion() {
|
|
11768
|
+
const conversationId = this.conversationId;
|
|
11769
|
+
if (this.baked || !conversationId || this.liveMotionPrepared === conversationId) {
|
|
11770
|
+
return false;
|
|
11771
|
+
}
|
|
11772
|
+
this.liveMotionPrepared = conversationId;
|
|
11773
|
+
return warmUpLiveMotion({
|
|
11774
|
+
conversationId,
|
|
11775
|
+
baseUrl: this.baseUrl,
|
|
11776
|
+
apiToken: this.apiToken,
|
|
11777
|
+
renderSessionId: this.renderSessionId
|
|
11778
|
+
});
|
|
11779
|
+
}
|
|
11147
11780
|
async createConversation(options) {
|
|
11148
11781
|
const response = this.baked ? {
|
|
11149
11782
|
conversation: {
|
|
@@ -11268,7 +11901,11 @@ var Character = class extends CharacterRuntime {
|
|
|
11268
11901
|
* character permits. Before `loadAssets`, the configuration or all modes.
|
|
11269
11902
|
*/
|
|
11270
11903
|
get inputModes() {
|
|
11271
|
-
return this.resolvedInputModes ?? this.configuredInputModes ?? permittedInputModes({
|
|
11904
|
+
return this.resolvedInputModes ?? this.configuredInputModes ?? permittedInputModes({
|
|
11905
|
+
allowCustomReply: true,
|
|
11906
|
+
baked: false,
|
|
11907
|
+
speechInput: true
|
|
11908
|
+
});
|
|
11272
11909
|
}
|
|
11273
11910
|
get listeningState() {
|
|
11274
11911
|
return this.voice.listeningState;
|
|
@@ -11312,6 +11949,19 @@ var Character = class extends CharacterRuntime {
|
|
|
11312
11949
|
callback
|
|
11313
11950
|
);
|
|
11314
11951
|
}
|
|
11952
|
+
/**
|
|
11953
|
+
* The server closed the conversation connection: a refused or failed join,
|
|
11954
|
+
* or an API key that was rotated, edited or deleted. The next `connect`,
|
|
11955
|
+
* `sendMessage` or `startListening` opens a new connection; a refused
|
|
11956
|
+
* conversation fails at once with `CONVERSATION_ACCESS_DENIED`.
|
|
11957
|
+
*/
|
|
11958
|
+
onConnectionClosed(callback) {
|
|
11959
|
+
return bindCallbackSlot(
|
|
11960
|
+
() => this.connectionClosedCallback,
|
|
11961
|
+
(next) => this.connectionClosedCallback = next,
|
|
11962
|
+
callback
|
|
11963
|
+
);
|
|
11964
|
+
}
|
|
11315
11965
|
async getConversationWorldline(conversationId = this.conversationId) {
|
|
11316
11966
|
if (!conversationId) {
|
|
11317
11967
|
throw new Error("Conversation ID is required.");
|
|
@@ -11409,6 +12059,11 @@ var Character = class extends CharacterRuntime {
|
|
|
11409
12059
|
"Voice input is not available in a baked (offline) conversation."
|
|
11410
12060
|
);
|
|
11411
12061
|
}
|
|
12062
|
+
if (mode === "voice" && this.speechInput === false) {
|
|
12063
|
+
throw new InputModeError(
|
|
12064
|
+
"Voice input is off for this character. Its owner turns it on in the STT tab."
|
|
12065
|
+
);
|
|
12066
|
+
}
|
|
11412
12067
|
const modes = this.inputModes;
|
|
11413
12068
|
if (!modes.includes(mode)) {
|
|
11414
12069
|
throw new InputModeError(
|
|
@@ -12036,10 +12691,12 @@ export {
|
|
|
12036
12691
|
CharacterExpressions,
|
|
12037
12692
|
CharacterRuntime,
|
|
12038
12693
|
CharacterSceneRig,
|
|
12694
|
+
ConversationConnectionError,
|
|
12039
12695
|
FacialPerformanceLayer,
|
|
12040
12696
|
FacialPerformanceMixer,
|
|
12041
12697
|
INPUT_MODES,
|
|
12042
12698
|
InputModeError,
|
|
12699
|
+
LiveMotionPlayer,
|
|
12043
12700
|
MicrophoneError,
|
|
12044
12701
|
Npcify,
|
|
12045
12702
|
Character as NpcifyCharacter,
|