@npcify/web 0.4.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -0
- package/dist/index.d.ts +183 -2
- package/dist/index.js +754 -136
- package/package.json +3 -3
package/dist/index.js
CHANGED
|
@@ -1,3 +1,36 @@
|
|
|
1
|
+
// ../ts-shared/src/motion-frames.ts
|
|
2
|
+
var MOTION_FRAMES_CAPABILITY = "motion_frames";
|
|
3
|
+
var motionFrameWidth = function(boneCount) {
|
|
4
|
+
return 3 + 4 * boneCount;
|
|
5
|
+
};
|
|
6
|
+
var MAX_MOTION_FRAMES_PER_EVENT = 3600;
|
|
7
|
+
var MAX_MOTION_BONES = 64;
|
|
8
|
+
var isCount = function(value, max2) {
|
|
9
|
+
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 && value <= max2;
|
|
10
|
+
};
|
|
11
|
+
var readMotionFrames = function(data, frameCount, boneCount) {
|
|
12
|
+
const values = frameCount * motionFrameWidth(boneCount);
|
|
13
|
+
if (data.byteLength !== values * 4) {
|
|
14
|
+
throw new Error("Motion frames do not match their frame count.");
|
|
15
|
+
}
|
|
16
|
+
const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
|
|
17
|
+
const frames = new Float32Array(values);
|
|
18
|
+
for (let index = 0; index < values; index += 1) {
|
|
19
|
+
frames[index] = view.getFloat32(index * 4, true);
|
|
20
|
+
}
|
|
21
|
+
return frames;
|
|
22
|
+
};
|
|
23
|
+
var isValidMotionFramesEvent = function(event) {
|
|
24
|
+
if (!event || typeof event !== "object") {
|
|
25
|
+
return false;
|
|
26
|
+
}
|
|
27
|
+
const value = event;
|
|
28
|
+
const bones = value.bones;
|
|
29
|
+
const frameCount = value.frameCount;
|
|
30
|
+
const frames = value.frames;
|
|
31
|
+
return value.type === "motion_frames" && typeof value.taskId === "string" && typeof value.stagingId === "string" && isCount(value.revision, Number.MAX_SAFE_INTEGER) && typeof value.fps === "number" && value.fps > 0 && value.fps <= 240 && isCount(value.startFrame, 24 * 60 * 60 * 240) && isCount(frameCount, MAX_MOTION_FRAMES_PER_EVENT) && Array.isArray(bones) && bones.length > 0 && bones.length <= MAX_MOTION_BONES && bones.every((bone) => typeof bone === "string") && typeof value.restHipsHeight === "number" && value.restHipsHeight > 0 && Number.isFinite(value.restHipsHeight) && frames instanceof Uint8Array && frames.byteLength === frameCount * motionFrameWidth(bones.length) * 4;
|
|
32
|
+
};
|
|
33
|
+
|
|
1
34
|
// ../ts-shared/src/llm.ts
|
|
2
35
|
var STAGING_PROTOCOL_VERSION = 1;
|
|
3
36
|
|
|
@@ -2602,6 +2635,8 @@ var PERFORMANCE_KINDS = {
|
|
|
2602
2635
|
storedTimeline: 7,
|
|
2603
2636
|
cachedOpening: 8
|
|
2604
2637
|
};
|
|
2638
|
+
var PERFORMANCE_MAJOR_VERSION = 1;
|
|
2639
|
+
var PERFORMANCE_MINOR_VERSION = 1;
|
|
2605
2640
|
var HEADER_SIZE = 24;
|
|
2606
2641
|
var MAGIC = 1346588750;
|
|
2607
2642
|
var crcTable = Uint32Array.from({ length: 256 }, (_, index) => {
|
|
@@ -2673,8 +2708,8 @@ var encodePerformance = function(kind, value, options = {}) {
|
|
|
2673
2708
|
const bytes = new Uint8Array(HEADER_SIZE + body.length);
|
|
2674
2709
|
const header = new DataView(bytes.buffer);
|
|
2675
2710
|
header.setUint32(0, MAGIC, true);
|
|
2676
|
-
header.setUint8(4,
|
|
2677
|
-
header.setUint8(5,
|
|
2711
|
+
header.setUint8(4, PERFORMANCE_MAJOR_VERSION);
|
|
2712
|
+
header.setUint8(5, PERFORMANCE_MINOR_VERSION);
|
|
2678
2713
|
header.setUint8(6, PERFORMANCE_KINDS[kind]);
|
|
2679
2714
|
header.setUint8(7, options.compress ? 1 : 0);
|
|
2680
2715
|
header.setUint16(8, HEADER_SIZE, true);
|
|
@@ -2711,7 +2746,7 @@ var decodePerformance = function(input, expectedKind) {
|
|
|
2711
2746
|
throw new Error("Truncated performance header.");
|
|
2712
2747
|
}
|
|
2713
2748
|
const header = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
2714
|
-
if (header.getUint32(0, true) !== MAGIC || header.getUint8(4) !==
|
|
2749
|
+
if (header.getUint32(0, true) !== MAGIC || header.getUint8(4) !== PERFORMANCE_MAJOR_VERSION) {
|
|
2715
2750
|
throw new Error("Unsupported performance format or major version.");
|
|
2716
2751
|
}
|
|
2717
2752
|
if (header.getUint8(6) !== PERFORMANCE_KINDS[expectedKind]) {
|
|
@@ -3166,7 +3201,8 @@ var createConversation = async function({
|
|
|
3166
3201
|
),
|
|
3167
3202
|
credentials: apiToken ? "omit" : "include",
|
|
3168
3203
|
method: "POST",
|
|
3169
|
-
|
|
3204
|
+
// This SDK plays live motion frames (ADR 0011).
|
|
3205
|
+
body: JSON.stringify({ avatarId, capabilities: [MOTION_FRAMES_CAPABILITY] })
|
|
3170
3206
|
});
|
|
3171
3207
|
if (!res.ok) {
|
|
3172
3208
|
throw Object.assign(new Error(await res.text()), { status: res.status });
|
|
@@ -3219,7 +3255,8 @@ var createStaging = async function({
|
|
|
3219
3255
|
content: message,
|
|
3220
3256
|
silent: silent === true,
|
|
3221
3257
|
incrementalPlayback: true,
|
|
3222
|
-
parentStagingId
|
|
3258
|
+
parentStagingId,
|
|
3259
|
+
capabilities: [MOTION_FRAMES_CAPABILITY]
|
|
3223
3260
|
})
|
|
3224
3261
|
});
|
|
3225
3262
|
} catch (error) {
|
|
@@ -3319,6 +3356,35 @@ var getConversationWorldline = async function(options) {
|
|
|
3319
3356
|
}
|
|
3320
3357
|
return await response.json();
|
|
3321
3358
|
};
|
|
3359
|
+
var warmUpLiveMotion = async function({
|
|
3360
|
+
conversationId,
|
|
3361
|
+
apiToken,
|
|
3362
|
+
baseUrl,
|
|
3363
|
+
renderSessionId
|
|
3364
|
+
}) {
|
|
3365
|
+
const url = new URL(`/conversations/${conversationId}/motion/warm-up`, baseUrl);
|
|
3366
|
+
try {
|
|
3367
|
+
const res = await fetch(url.toString(), {
|
|
3368
|
+
headers: withRenderSession(
|
|
3369
|
+
buildAuthHeaders(apiToken, {
|
|
3370
|
+
json: true,
|
|
3371
|
+
allowEmpty: true
|
|
3372
|
+
}),
|
|
3373
|
+
renderSessionId
|
|
3374
|
+
),
|
|
3375
|
+
credentials: apiToken ? "omit" : "include",
|
|
3376
|
+
method: "POST",
|
|
3377
|
+
body: JSON.stringify({ capabilities: [MOTION_FRAMES_CAPABILITY] })
|
|
3378
|
+
});
|
|
3379
|
+
if (!res.ok) {
|
|
3380
|
+
return false;
|
|
3381
|
+
}
|
|
3382
|
+
const payload = await res.json();
|
|
3383
|
+
return payload.warming === true;
|
|
3384
|
+
} catch {
|
|
3385
|
+
return false;
|
|
3386
|
+
}
|
|
3387
|
+
};
|
|
3322
3388
|
|
|
3323
3389
|
// ../../clients/web-client/src/live-conversation.ts
|
|
3324
3390
|
var LiveConversation = class {
|
|
@@ -6045,6 +6111,10 @@ var StreamPlaybackController = class {
|
|
|
6045
6111
|
get cursor() {
|
|
6046
6112
|
return this.currentCursorValue;
|
|
6047
6113
|
}
|
|
6114
|
+
/** The staging that plays or waits to play, or null. */
|
|
6115
|
+
get activeStagingId() {
|
|
6116
|
+
return this.activePlayback?.stagingId ?? null;
|
|
6117
|
+
}
|
|
6048
6118
|
onStreamEvent(callback) {
|
|
6049
6119
|
return bindCallbackSlot(
|
|
6050
6120
|
() => this.streamEventCallback,
|
|
@@ -6138,6 +6208,10 @@ var StreamPlaybackController = class {
|
|
|
6138
6208
|
this.dependencies.telemetry.recordFirstTextDelta();
|
|
6139
6209
|
return;
|
|
6140
6210
|
}
|
|
6211
|
+
if (event.type === "motion_frames") {
|
|
6212
|
+
this.dependencies.onMotionFrames?.(event);
|
|
6213
|
+
return;
|
|
6214
|
+
}
|
|
6141
6215
|
const player = this.dependencies.getStagingPlayer();
|
|
6142
6216
|
if (event.type === "speech_chunk") {
|
|
6143
6217
|
player.appendSpeechChunk(event.streamId, event.mimeType, event.data);
|
|
@@ -6251,6 +6325,9 @@ var StreamPlaybackController = class {
|
|
|
6251
6325
|
this.activePlayback = { taskId: params.taskId, stagingId: params.stagingId };
|
|
6252
6326
|
this.playedStagingIds.add(params.stagingId);
|
|
6253
6327
|
this.recordPlaybackEvent(params, "requested");
|
|
6328
|
+
if (params.timeline.motion) {
|
|
6329
|
+
this.dependencies.onMotionTrack?.(params.stagingId, params.timeline.motion);
|
|
6330
|
+
}
|
|
6254
6331
|
const existing = this.states.get(params.stagingId);
|
|
6255
6332
|
const state = {
|
|
6256
6333
|
taskId: params.taskId,
|
|
@@ -8260,6 +8337,261 @@ var CharacterSceneRig = class {
|
|
|
8260
8337
|
}
|
|
8261
8338
|
};
|
|
8262
8339
|
|
|
8340
|
+
// ../../clients/web-client/src/character/live-motion.ts
|
|
8341
|
+
import { Quaternion as Quaternion3, Vector3 as Vector37 } from "three";
|
|
8342
|
+
var PARENTS = {
|
|
8343
|
+
spine: "hips",
|
|
8344
|
+
chest: "spine",
|
|
8345
|
+
upperChest: "chest",
|
|
8346
|
+
neck: "upperChest",
|
|
8347
|
+
head: "neck",
|
|
8348
|
+
leftShoulder: "upperChest",
|
|
8349
|
+
leftUpperArm: "leftShoulder",
|
|
8350
|
+
leftLowerArm: "leftUpperArm",
|
|
8351
|
+
leftHand: "leftLowerArm",
|
|
8352
|
+
rightShoulder: "upperChest",
|
|
8353
|
+
rightUpperArm: "rightShoulder",
|
|
8354
|
+
rightLowerArm: "rightUpperArm",
|
|
8355
|
+
rightHand: "rightLowerArm",
|
|
8356
|
+
leftUpperLeg: "hips",
|
|
8357
|
+
leftLowerLeg: "leftUpperLeg",
|
|
8358
|
+
leftFoot: "leftLowerLeg",
|
|
8359
|
+
leftToes: "leftFoot",
|
|
8360
|
+
rightUpperLeg: "hips",
|
|
8361
|
+
rightLowerLeg: "rightUpperLeg",
|
|
8362
|
+
rightFoot: "rightLowerLeg",
|
|
8363
|
+
rightToes: "rightFoot"
|
|
8364
|
+
};
|
|
8365
|
+
var MAX_TRACKS = 4;
|
|
8366
|
+
var approach = function(value, target, delta, seconds) {
|
|
8367
|
+
if (seconds <= 0) {
|
|
8368
|
+
return target;
|
|
8369
|
+
}
|
|
8370
|
+
const step = delta / seconds;
|
|
8371
|
+
return value < target ? Math.min(target, value + step) : Math.max(target, value - step);
|
|
8372
|
+
};
|
|
8373
|
+
var LiveMotionPlayer = class {
|
|
8374
|
+
vrm = null;
|
|
8375
|
+
tracks = /* @__PURE__ */ new Map();
|
|
8376
|
+
rootMode;
|
|
8377
|
+
blendIn;
|
|
8378
|
+
blendOut;
|
|
8379
|
+
weight = 0;
|
|
8380
|
+
/** The last applied pose, to blend out of it after the frames end. */
|
|
8381
|
+
lastPose = null;
|
|
8382
|
+
scratch = {
|
|
8383
|
+
from: new Quaternion3(),
|
|
8384
|
+
to: new Quaternion3(),
|
|
8385
|
+
carry: /* @__PURE__ */ new Map(),
|
|
8386
|
+
position: new Vector37()
|
|
8387
|
+
};
|
|
8388
|
+
constructor(options = {}) {
|
|
8389
|
+
this.rootMode = options.rootMode ?? "anchored";
|
|
8390
|
+
this.blendIn = options.blendInSeconds ?? 0.3;
|
|
8391
|
+
this.blendOut = options.blendOutSeconds ?? 0.5;
|
|
8392
|
+
}
|
|
8393
|
+
setVrm(vrm) {
|
|
8394
|
+
this.vrm = vrm;
|
|
8395
|
+
this.weight = 0;
|
|
8396
|
+
this.lastPose = null;
|
|
8397
|
+
}
|
|
8398
|
+
/** Adds the frames of a `motion_frames` event; invalid events are dropped. */
|
|
8399
|
+
push(received) {
|
|
8400
|
+
const frames = received.frames;
|
|
8401
|
+
const event = frames instanceof ArrayBuffer ? { ...received, frames: new Uint8Array(frames) } : received;
|
|
8402
|
+
if (!isValidMotionFramesEvent(event)) {
|
|
8403
|
+
return false;
|
|
8404
|
+
}
|
|
8405
|
+
const values = readMotionFrames(
|
|
8406
|
+
event.frames,
|
|
8407
|
+
event.frameCount,
|
|
8408
|
+
event.bones.length
|
|
8409
|
+
);
|
|
8410
|
+
const track = this.track(event.stagingId, {
|
|
8411
|
+
fps: event.fps,
|
|
8412
|
+
bones: event.bones,
|
|
8413
|
+
restHipsHeight: event.restHipsHeight
|
|
8414
|
+
});
|
|
8415
|
+
for (let frame = 0; frame < event.frameCount; frame += 1) {
|
|
8416
|
+
const index = event.startFrame + frame;
|
|
8417
|
+
const existing = track.frames.get(index);
|
|
8418
|
+
if (existing && existing.revision > event.revision) {
|
|
8419
|
+
continue;
|
|
8420
|
+
}
|
|
8421
|
+
track.frames.set(index, {
|
|
8422
|
+
revision: event.revision,
|
|
8423
|
+
values: values.subarray(frame * track.width, (frame + 1) * track.width)
|
|
8424
|
+
});
|
|
8425
|
+
}
|
|
8426
|
+
return true;
|
|
8427
|
+
}
|
|
8428
|
+
/** Loads the stored frames of a staging (a replay). */
|
|
8429
|
+
async load(stagingId, track, fetchBytes = async (url) => new Uint8Array(await (await fetch(url)).arrayBuffer())) {
|
|
8430
|
+
if (this.tracks.has(stagingId)) {
|
|
8431
|
+
return;
|
|
8432
|
+
}
|
|
8433
|
+
const data = track.frames.type === "bytes" ? track.frames.data : await fetchBytes(track.frames.url);
|
|
8434
|
+
const values = readMotionFrames(data, track.frameCount, track.bones.length);
|
|
8435
|
+
const stored = this.track(stagingId, track);
|
|
8436
|
+
for (let frame = 0; frame < track.frameCount; frame += 1) {
|
|
8437
|
+
stored.frames.set(frame, {
|
|
8438
|
+
revision: Number.MAX_SAFE_INTEGER,
|
|
8439
|
+
values: values.subarray(
|
|
8440
|
+
frame * stored.width,
|
|
8441
|
+
(frame + 1) * stored.width
|
|
8442
|
+
)
|
|
8443
|
+
});
|
|
8444
|
+
}
|
|
8445
|
+
}
|
|
8446
|
+
has(stagingId) {
|
|
8447
|
+
return (this.tracks.get(stagingId)?.frames.size ?? 0) > 0;
|
|
8448
|
+
}
|
|
8449
|
+
forget(stagingId) {
|
|
8450
|
+
this.tracks.delete(stagingId);
|
|
8451
|
+
}
|
|
8452
|
+
/** The current live weight, 0 (idle) to 1 (live frames). */
|
|
8453
|
+
get liveWeight() {
|
|
8454
|
+
return this.weight;
|
|
8455
|
+
}
|
|
8456
|
+
/**
|
|
8457
|
+
* Applies the live pose of `stagingId` at `time` seconds of its staging
|
|
8458
|
+
* clock; `time` null means that no staging plays now.
|
|
8459
|
+
*/
|
|
8460
|
+
update(delta, stagingId, time) {
|
|
8461
|
+
const vrm = this.vrm;
|
|
8462
|
+
if (!vrm?.humanoid) {
|
|
8463
|
+
return;
|
|
8464
|
+
}
|
|
8465
|
+
const track = stagingId ? this.tracks.get(stagingId) : void 0;
|
|
8466
|
+
let pose = null;
|
|
8467
|
+
if (track && time !== null && time >= 0) {
|
|
8468
|
+
pose = this.sample(track, time);
|
|
8469
|
+
}
|
|
8470
|
+
this.weight = approach(
|
|
8471
|
+
this.weight,
|
|
8472
|
+
pose ? 1 : 0,
|
|
8473
|
+
Math.max(0, delta),
|
|
8474
|
+
pose ? this.blendIn : this.blendOut
|
|
8475
|
+
);
|
|
8476
|
+
if (pose) {
|
|
8477
|
+
this.lastPose = pose;
|
|
8478
|
+
}
|
|
8479
|
+
const applied = pose ?? this.lastPose;
|
|
8480
|
+
if (!applied || this.weight <= 0) {
|
|
8481
|
+
if (this.weight <= 0) {
|
|
8482
|
+
this.lastPose = null;
|
|
8483
|
+
}
|
|
8484
|
+
return;
|
|
8485
|
+
}
|
|
8486
|
+
this.apply(vrm, applied.track, applied.values, this.weight);
|
|
8487
|
+
}
|
|
8488
|
+
track(stagingId, meta) {
|
|
8489
|
+
let track = this.tracks.get(stagingId);
|
|
8490
|
+
if (!track) {
|
|
8491
|
+
track = {
|
|
8492
|
+
fps: meta.fps,
|
|
8493
|
+
bones: [...meta.bones],
|
|
8494
|
+
restHipsHeight: meta.restHipsHeight,
|
|
8495
|
+
width: motionFrameWidth(meta.bones.length),
|
|
8496
|
+
frames: /* @__PURE__ */ new Map(),
|
|
8497
|
+
origin: null
|
|
8498
|
+
};
|
|
8499
|
+
this.tracks.set(stagingId, track);
|
|
8500
|
+
while (this.tracks.size > MAX_TRACKS) {
|
|
8501
|
+
const oldest = this.tracks.keys().next().value;
|
|
8502
|
+
if (oldest === void 0 || oldest === stagingId) {
|
|
8503
|
+
break;
|
|
8504
|
+
}
|
|
8505
|
+
this.tracks.delete(oldest);
|
|
8506
|
+
}
|
|
8507
|
+
}
|
|
8508
|
+
return track;
|
|
8509
|
+
}
|
|
8510
|
+
/** The interpolated frame at `time`, or null when its frames are missing. */
|
|
8511
|
+
sample(track, time) {
|
|
8512
|
+
const position = time * track.fps;
|
|
8513
|
+
const first = Math.floor(position);
|
|
8514
|
+
const a = track.frames.get(first);
|
|
8515
|
+
if (!a) {
|
|
8516
|
+
return null;
|
|
8517
|
+
}
|
|
8518
|
+
track.origin ??= { x: a.values[0] ?? 0, z: a.values[2] ?? 0 };
|
|
8519
|
+
const b = track.frames.get(first + 1);
|
|
8520
|
+
const amount = position - first;
|
|
8521
|
+
if (!b || amount <= 0) {
|
|
8522
|
+
return { track, values: a.values };
|
|
8523
|
+
}
|
|
8524
|
+
const values = new Float32Array(track.width);
|
|
8525
|
+
for (let index = 0; index < 3; index += 1) {
|
|
8526
|
+
values[index] = (a.values[index] ?? 0) * (1 - amount) + (b.values[index] ?? 0) * amount;
|
|
8527
|
+
}
|
|
8528
|
+
const from = this.scratch.from;
|
|
8529
|
+
const to = this.scratch.to;
|
|
8530
|
+
for (let offset = 3; offset < track.width; offset += 4) {
|
|
8531
|
+
from.fromArray(a.values, offset);
|
|
8532
|
+
to.fromArray(b.values, offset);
|
|
8533
|
+
from.slerp(to, amount);
|
|
8534
|
+
values[offset] = from.x;
|
|
8535
|
+
values[offset + 1] = from.y;
|
|
8536
|
+
values[offset + 2] = from.z;
|
|
8537
|
+
values[offset + 3] = from.w;
|
|
8538
|
+
}
|
|
8539
|
+
return { track, values };
|
|
8540
|
+
}
|
|
8541
|
+
apply(vrm, track, values, weight) {
|
|
8542
|
+
const humanoid = vrm.humanoid;
|
|
8543
|
+
const flip = vrm.meta.metaVersion === "0" && !isNpcifyNormalizedVrm(vrm);
|
|
8544
|
+
const carry = this.scratch.carry;
|
|
8545
|
+
carry.clear();
|
|
8546
|
+
track.bones.forEach((bone, slot) => {
|
|
8547
|
+
const offset = 3 + slot * 4;
|
|
8548
|
+
const rotation = new Quaternion3(
|
|
8549
|
+
values[offset] ?? 0,
|
|
8550
|
+
values[offset + 1] ?? 0,
|
|
8551
|
+
values[offset + 2] ?? 0,
|
|
8552
|
+
values[offset + 3] ?? 1
|
|
8553
|
+
);
|
|
8554
|
+
if (flip) {
|
|
8555
|
+
rotation.x = -rotation.x;
|
|
8556
|
+
rotation.z = -rotation.z;
|
|
8557
|
+
}
|
|
8558
|
+
const inherited = carry.get(bone);
|
|
8559
|
+
if (inherited) {
|
|
8560
|
+
rotation.premultiply(inherited);
|
|
8561
|
+
}
|
|
8562
|
+
const node = humanoid.getNormalizedBoneNode(bone);
|
|
8563
|
+
if (!node) {
|
|
8564
|
+
for (const [child, parent] of Object.entries(PARENTS)) {
|
|
8565
|
+
if (parent === bone) {
|
|
8566
|
+
carry.set(child, rotation.clone());
|
|
8567
|
+
}
|
|
8568
|
+
}
|
|
8569
|
+
return;
|
|
8570
|
+
}
|
|
8571
|
+
node.quaternion.slerp(rotation, weight);
|
|
8572
|
+
});
|
|
8573
|
+
const hips = humanoid.getNormalizedBoneNode("hips");
|
|
8574
|
+
const rest = humanoid.normalizedRestPose.hips?.position;
|
|
8575
|
+
if (!hips || !rest) {
|
|
8576
|
+
return;
|
|
8577
|
+
}
|
|
8578
|
+
const scale = (rest[1] ?? 1) / (track.restHipsHeight || 1);
|
|
8579
|
+
let x = values[0] ?? 0;
|
|
8580
|
+
const y = values[1] ?? 0;
|
|
8581
|
+
let z = values[2] ?? 0;
|
|
8582
|
+
if (this.rootMode === "anchored" && track.origin) {
|
|
8583
|
+
x -= track.origin.x;
|
|
8584
|
+
z -= track.origin.z;
|
|
8585
|
+
}
|
|
8586
|
+
const target = this.scratch.position.set(
|
|
8587
|
+
(flip ? -x : x) * scale + (this.rootMode === "anchored" ? rest[0] ?? 0 : 0),
|
|
8588
|
+
y * scale,
|
|
8589
|
+
(flip ? -z : z) * scale + (this.rootMode === "anchored" ? rest[2] ?? 0 : 0)
|
|
8590
|
+
);
|
|
8591
|
+
hips.position.lerp(target, weight);
|
|
8592
|
+
}
|
|
8593
|
+
};
|
|
8594
|
+
|
|
8263
8595
|
// ../../clients/web-client/src/character/incremental-staging.ts
|
|
8264
8596
|
var dialogueKey = function(block) {
|
|
8265
8597
|
return JSON.stringify(block, (key, value) => {
|
|
@@ -8534,9 +8866,17 @@ var StagingPlayer = class {
|
|
|
8534
8866
|
activeExpression = null;
|
|
8535
8867
|
activeExpressionMouthMode = null;
|
|
8536
8868
|
activeAnimation = null;
|
|
8869
|
+
currentTrackTime = null;
|
|
8537
8870
|
get paused() {
|
|
8538
8871
|
return this.clock.paused;
|
|
8539
8872
|
}
|
|
8873
|
+
/**
|
|
8874
|
+
* Seconds of the staging clock that the tracks show now, or null when no
|
|
8875
|
+
* staging plays (also while playback waits for the next dialogue block).
|
|
8876
|
+
*/
|
|
8877
|
+
get trackTime() {
|
|
8878
|
+
return this.currentTrackTime;
|
|
8879
|
+
}
|
|
8540
8880
|
pause() {
|
|
8541
8881
|
this.pauseRevision += 1;
|
|
8542
8882
|
this.clock.pause();
|
|
@@ -8937,6 +9277,7 @@ var StagingPlayer = class {
|
|
|
8937
9277
|
}
|
|
8938
9278
|
}
|
|
8939
9279
|
applyTracks(time, speechActive, timeline) {
|
|
9280
|
+
this.currentTrackTime = time;
|
|
8940
9281
|
const expression = activeClipAt(timeline.expressions, time);
|
|
8941
9282
|
const mouthMode = speechActive ? "viseme" : "expression";
|
|
8942
9283
|
if (expression !== this.activeExpression || expression && mouthMode !== this.activeExpressionMouthMode) {
|
|
@@ -8970,6 +9311,7 @@ var StagingPlayer = class {
|
|
|
8970
9311
|
}
|
|
8971
9312
|
}
|
|
8972
9313
|
clearPerformance() {
|
|
9314
|
+
this.currentTrackTime = null;
|
|
8973
9315
|
this.activeExpression = null;
|
|
8974
9316
|
this.activeExpressionMouthMode = null;
|
|
8975
9317
|
this.activeAnimation = null;
|
|
@@ -9408,6 +9750,7 @@ var CharacterRuntime = class {
|
|
|
9408
9750
|
performanceRecorder;
|
|
9409
9751
|
telemetry;
|
|
9410
9752
|
streamPlayback;
|
|
9753
|
+
liveMotion;
|
|
9411
9754
|
defaultParent;
|
|
9412
9755
|
currentGazeTarget = null;
|
|
9413
9756
|
/** The current gaze target came from `lookAt()`. */
|
|
@@ -9456,12 +9799,21 @@ var CharacterRuntime = class {
|
|
|
9456
9799
|
baseUrl: this.baseUrl,
|
|
9457
9800
|
apiToken: this.apiToken
|
|
9458
9801
|
});
|
|
9802
|
+
this.liveMotion = new LiveMotionPlayer(options.liveMotion);
|
|
9459
9803
|
this.streamPlayback = new StreamPlaybackController({
|
|
9460
9804
|
getStagingPlayer: this.getStagingPlayer,
|
|
9461
9805
|
telemetry: this.telemetry,
|
|
9462
9806
|
handleCallbackError: this.handleCallbackError,
|
|
9463
9807
|
reportError: this.reportError,
|
|
9464
|
-
recordEvent: (event) => this.performanceRecorder.recordEvent(event)
|
|
9808
|
+
recordEvent: (event) => this.performanceRecorder.recordEvent(event),
|
|
9809
|
+
onMotionFrames: (event) => {
|
|
9810
|
+
this.liveMotion.push(event);
|
|
9811
|
+
},
|
|
9812
|
+
onMotionTrack: (stagingId, track) => {
|
|
9813
|
+
void this.liveMotion.load(stagingId, track).catch((error) => {
|
|
9814
|
+
this.handleCallbackError("liveMotion", error);
|
|
9815
|
+
});
|
|
9816
|
+
}
|
|
9465
9817
|
});
|
|
9466
9818
|
if (options.gazeTarget !== void 0) {
|
|
9467
9819
|
this.setGazeTarget(options.gazeTarget);
|
|
@@ -9498,6 +9850,7 @@ var CharacterRuntime = class {
|
|
|
9498
9850
|
set vrm(value) {
|
|
9499
9851
|
this.sceneRig.setVrm(value);
|
|
9500
9852
|
this.facialMixer.setVrm(value);
|
|
9853
|
+
this.liveMotion.setVrm(value);
|
|
9501
9854
|
this.applyGaze();
|
|
9502
9855
|
if (value && this.defaultParent && !value.scene?.parent) {
|
|
9503
9856
|
this.sceneRig.attachTo(this.defaultParent);
|
|
@@ -9774,6 +10127,11 @@ var CharacterRuntime = class {
|
|
|
9774
10127
|
return;
|
|
9775
10128
|
}
|
|
9776
10129
|
this.animationController?.update(delta);
|
|
10130
|
+
this.liveMotion.update(
|
|
10131
|
+
delta,
|
|
10132
|
+
this.streamPlayback.activeStagingId,
|
|
10133
|
+
this.stagingPlayer?.trackTime ?? null
|
|
10134
|
+
);
|
|
9777
10135
|
this.visemePlayer.update(delta);
|
|
9778
10136
|
this.facialMixer.update(delta);
|
|
9779
10137
|
this.vrm?.update(delta);
|
|
@@ -9830,6 +10188,7 @@ var CharacterRuntime = class {
|
|
|
9830
10188
|
this.animations.clear();
|
|
9831
10189
|
this.sceneRig.setVrm(null);
|
|
9832
10190
|
this.facialMixer.setVrm(null);
|
|
10191
|
+
this.liveMotion.setVrm(null);
|
|
9833
10192
|
}
|
|
9834
10193
|
disposeRuntimeAssets() {
|
|
9835
10194
|
this.sceneRig.disposeVrm();
|
|
@@ -9852,28 +10211,101 @@ var CharacterRuntime = class {
|
|
|
9852
10211
|
|
|
9853
10212
|
// ../../clients/web-client/src/conversation-socket.ts
|
|
9854
10213
|
import { io } from "socket.io-client";
|
|
10214
|
+
var ConversationConnectionError = class extends Error {
|
|
10215
|
+
constructor(code, message, reason = null) {
|
|
10216
|
+
super(message);
|
|
10217
|
+
this.code = code;
|
|
10218
|
+
this.reason = reason;
|
|
10219
|
+
this.name = "NpcifyConversationConnectionError";
|
|
10220
|
+
}
|
|
10221
|
+
};
|
|
10222
|
+
var JOIN_EVENT = "conversations:join";
|
|
10223
|
+
var JOIN_TIMEOUT_MS = 1e4;
|
|
10224
|
+
var ACCESS_DENIED = "CONVERSATION_ACCESS_DENIED";
|
|
10225
|
+
var JOIN_FAILED = "CONVERSATION_JOIN_FAILED";
|
|
10226
|
+
var CLIENT_CLOSE = "io client disconnect";
|
|
10227
|
+
var SOCKET_IO_TIMEOUT = "operation has timed out";
|
|
10228
|
+
var joinMessage = function(code) {
|
|
10229
|
+
if (code === ACCESS_DENIED) {
|
|
10230
|
+
return "Conversation streaming access was denied.";
|
|
10231
|
+
}
|
|
10232
|
+
if (code === JOIN_FAILED) {
|
|
10233
|
+
return "The server could not join the conversation. Try again later.";
|
|
10234
|
+
}
|
|
10235
|
+
return `The server did not join the conversation (${code}). Try again later.`;
|
|
10236
|
+
};
|
|
10237
|
+
var closeMessage = function(reason, detail) {
|
|
10238
|
+
if (reason === "io server disconnect") {
|
|
10239
|
+
return "The server closed the conversation connection.";
|
|
10240
|
+
}
|
|
10241
|
+
if (reason === CLIENT_CLOSE) {
|
|
10242
|
+
return "The conversation connection closed.";
|
|
10243
|
+
}
|
|
10244
|
+
if (reason === "connect_error") {
|
|
10245
|
+
return `The server refused the conversation connection (${detail || "no reason"}).`;
|
|
10246
|
+
}
|
|
10247
|
+
return `The conversation connection was lost (${reason}).`;
|
|
10248
|
+
};
|
|
10249
|
+
var acknowledgementError = function(error, lost) {
|
|
10250
|
+
if (error.message === SOCKET_IO_TIMEOUT) {
|
|
10251
|
+
return new ConversationConnectionError(
|
|
10252
|
+
"CONVERSATION_TIMEOUT",
|
|
10253
|
+
"The server did not answer in time."
|
|
10254
|
+
);
|
|
10255
|
+
}
|
|
10256
|
+
return new ConversationConnectionError(
|
|
10257
|
+
"CONVERSATION_DISCONNECTED",
|
|
10258
|
+
closeMessage(lost ?? "transport close"),
|
|
10259
|
+
lost
|
|
10260
|
+
);
|
|
10261
|
+
};
|
|
10262
|
+
var refusedConversation = function(payload) {
|
|
10263
|
+
const exception = payload;
|
|
10264
|
+
const named = exception?.cause?.data?.conversationId;
|
|
10265
|
+
if (exception?.message !== ACCESS_DENIED || typeof named !== "string") {
|
|
10266
|
+
return null;
|
|
10267
|
+
}
|
|
10268
|
+
return named.trim() || null;
|
|
10269
|
+
};
|
|
9855
10270
|
var ConversationSocket = class {
|
|
9856
10271
|
options;
|
|
9857
|
-
|
|
10272
|
+
connection = null;
|
|
10273
|
+
/**
|
|
10274
|
+
* The conversations that the server refused. A join with the same
|
|
10275
|
+
* credentials fails the same way, so the SDK does not try them again until
|
|
10276
|
+
* `disconnect`.
|
|
10277
|
+
*/
|
|
10278
|
+
refused = /* @__PURE__ */ new Set();
|
|
9858
10279
|
constructor(options) {
|
|
9859
10280
|
this.options = options;
|
|
9860
10281
|
}
|
|
10282
|
+
/** Closes the connection and forgets the refused conversations. */
|
|
9861
10283
|
disconnect() {
|
|
9862
|
-
this.
|
|
9863
|
-
this.
|
|
10284
|
+
this.refused.clear();
|
|
10285
|
+
if (this.connection) {
|
|
10286
|
+
this.close(this.connection, CLIENT_CLOSE);
|
|
10287
|
+
}
|
|
9864
10288
|
}
|
|
9865
10289
|
/** An event with an acknowledgement. Needs `ensureConnected` first. */
|
|
9866
10290
|
async request(event, payload, timeoutMs = 1e4) {
|
|
9867
|
-
const
|
|
9868
|
-
if (!socket
|
|
9869
|
-
throw new
|
|
10291
|
+
const connection = this.connection;
|
|
10292
|
+
if (!connection?.socket.connected) {
|
|
10293
|
+
throw new ConversationConnectionError(
|
|
10294
|
+
"CONVERSATION_DISCONNECTED",
|
|
10295
|
+
"The conversation connection is not open."
|
|
10296
|
+
);
|
|
9870
10297
|
}
|
|
9871
|
-
return
|
|
10298
|
+
return this.acknowledged(connection, event, payload, timeoutMs, null);
|
|
9872
10299
|
}
|
|
9873
10300
|
/** An event without an acknowledgement, in order with the other events. */
|
|
9874
10301
|
send(event, payload) {
|
|
9875
|
-
this.socket
|
|
10302
|
+
this.connection?.socket.emit(event, payload);
|
|
9876
10303
|
}
|
|
10304
|
+
/**
|
|
10305
|
+
* Joins the room of the conversation. A connection that the server closed
|
|
10306
|
+
* is not used again: the join opens a new one. A refused conversation fails
|
|
10307
|
+
* at once, without a connection.
|
|
10308
|
+
*/
|
|
9877
10309
|
async ensureConnected(conversationId) {
|
|
9878
10310
|
if (typeof window === "undefined") {
|
|
9879
10311
|
return;
|
|
@@ -9882,135 +10314,279 @@ var ConversationSocket = class {
|
|
|
9882
10314
|
if (!normalized) {
|
|
9883
10315
|
return;
|
|
9884
10316
|
}
|
|
9885
|
-
if (
|
|
9886
|
-
|
|
9887
|
-
|
|
9888
|
-
|
|
9889
|
-
|
|
9890
|
-
|
|
9891
|
-
|
|
9892
|
-
|
|
9893
|
-
|
|
9894
|
-
|
|
9895
|
-
|
|
9896
|
-
|
|
9897
|
-
|
|
9898
|
-
|
|
9899
|
-
|
|
9900
|
-
|
|
9901
|
-
|
|
9902
|
-
|
|
9903
|
-
|
|
9904
|
-
|
|
9905
|
-
|
|
9906
|
-
|
|
9907
|
-
}
|
|
9908
|
-
|
|
9909
|
-
|
|
9910
|
-
|
|
9911
|
-
|
|
9912
|
-
|
|
9913
|
-
|
|
9914
|
-
|
|
9915
|
-
|
|
9916
|
-
|
|
9917
|
-
|
|
9918
|
-
|
|
9919
|
-
|
|
9920
|
-
|
|
9921
|
-
|
|
9922
|
-
|
|
9923
|
-
|
|
9924
|
-
|
|
9925
|
-
|
|
9926
|
-
|
|
9927
|
-
|
|
9928
|
-
|
|
9929
|
-
|
|
9930
|
-
|
|
9931
|
-
|
|
9932
|
-
|
|
9933
|
-
|
|
9934
|
-
|
|
9935
|
-
|
|
10317
|
+
if (this.refused.has(normalized)) {
|
|
10318
|
+
throw new ConversationConnectionError(
|
|
10319
|
+
ACCESS_DENIED,
|
|
10320
|
+
joinMessage(ACCESS_DENIED)
|
|
10321
|
+
);
|
|
10322
|
+
}
|
|
10323
|
+
const connection = this.connection ?? this.open();
|
|
10324
|
+
await this.connected(connection);
|
|
10325
|
+
await this.acknowledged(
|
|
10326
|
+
connection,
|
|
10327
|
+
JOIN_EVENT,
|
|
10328
|
+
{ conversationId: normalized },
|
|
10329
|
+
this.options.joinTimeoutMs ?? JOIN_TIMEOUT_MS,
|
|
10330
|
+
normalized
|
|
10331
|
+
);
|
|
10332
|
+
}
|
|
10333
|
+
open() {
|
|
10334
|
+
const socketBaseUrl = (() => {
|
|
10335
|
+
try {
|
|
10336
|
+
return new URL(this.options.baseUrl).origin;
|
|
10337
|
+
} catch {
|
|
10338
|
+
return window.location.origin;
|
|
10339
|
+
}
|
|
10340
|
+
})();
|
|
10341
|
+
const socket = io(socketBaseUrl, {
|
|
10342
|
+
transports: ["websocket"],
|
|
10343
|
+
withCredentials: true,
|
|
10344
|
+
forceNew: true,
|
|
10345
|
+
auth: {
|
|
10346
|
+
apiToken: this.options.apiToken
|
|
10347
|
+
}
|
|
10348
|
+
});
|
|
10349
|
+
const connection = {
|
|
10350
|
+
socket,
|
|
10351
|
+
pending: /* @__PURE__ */ new Set(),
|
|
10352
|
+
connectedBefore: false,
|
|
10353
|
+
lost: null,
|
|
10354
|
+
closed: null,
|
|
10355
|
+
failure: null
|
|
10356
|
+
};
|
|
10357
|
+
this.connection = connection;
|
|
10358
|
+
this.watch(connection);
|
|
10359
|
+
this.receive(socket);
|
|
10360
|
+
return connection;
|
|
10361
|
+
}
|
|
10362
|
+
/** The lifecycle of one connection (docs/streaming_reply.md#web-sdk). */
|
|
10363
|
+
watch(connection) {
|
|
10364
|
+
const { socket } = connection;
|
|
10365
|
+
socket.on("connect", () => {
|
|
10366
|
+
if (!connection.connectedBefore) {
|
|
10367
|
+
connection.connectedBefore = true;
|
|
10368
|
+
return;
|
|
10369
|
+
}
|
|
10370
|
+
const activeConversationId = this.options.getConversationId()?.trim();
|
|
10371
|
+
if (activeConversationId && !this.refused.has(activeConversationId)) {
|
|
10372
|
+
socket.emit(JOIN_EVENT, { conversationId: activeConversationId });
|
|
10373
|
+
}
|
|
10374
|
+
});
|
|
10375
|
+
socket.on("exception", (payload) => {
|
|
10376
|
+
const conversationId = refusedConversation(payload);
|
|
10377
|
+
if (conversationId) {
|
|
10378
|
+
connection.failure ??= { code: ACCESS_DENIED, conversationId };
|
|
10379
|
+
this.refused.add(conversationId);
|
|
10380
|
+
}
|
|
10381
|
+
});
|
|
10382
|
+
socket.on("disconnect", (reason) => {
|
|
10383
|
+
if (!socket.active) {
|
|
10384
|
+
this.close(connection, reason);
|
|
10385
|
+
return;
|
|
10386
|
+
}
|
|
10387
|
+
connection.lost = reason;
|
|
10388
|
+
this.options.handleSpeechEvent?.({ type: "disconnect" });
|
|
10389
|
+
});
|
|
10390
|
+
socket.on("connect_error", (error) => {
|
|
10391
|
+
if (!socket.active) {
|
|
10392
|
+
this.close(connection, "connect_error", error.message);
|
|
10393
|
+
}
|
|
10394
|
+
});
|
|
10395
|
+
}
|
|
10396
|
+
/**
|
|
10397
|
+
* Closes the connection for good: socket.io does not open it again. The
|
|
10398
|
+
* waits fail now, and the next join opens a new connection.
|
|
10399
|
+
*/
|
|
10400
|
+
close(connection, reason, detail) {
|
|
10401
|
+
if (connection.closed) {
|
|
10402
|
+
return;
|
|
10403
|
+
}
|
|
10404
|
+
connection.closed = { reason, message: closeMessage(reason, detail) };
|
|
10405
|
+
if (this.connection === connection) {
|
|
10406
|
+
this.connection = null;
|
|
10407
|
+
}
|
|
10408
|
+
const { socket } = connection;
|
|
10409
|
+
socket.removeAllListeners();
|
|
10410
|
+
if (connection.connectedBefore) {
|
|
10411
|
+
this.options.handleSpeechEvent?.({ type: "disconnect" });
|
|
10412
|
+
}
|
|
10413
|
+
const pending = [...connection.pending];
|
|
10414
|
+
connection.pending.clear();
|
|
10415
|
+
for (const entry of pending) {
|
|
10416
|
+
entry.fail(this.closeError(connection, entry.conversationId));
|
|
10417
|
+
}
|
|
10418
|
+
socket.disconnect();
|
|
10419
|
+
if (reason !== CLIENT_CLOSE) {
|
|
10420
|
+
this.options.handleConnectionClosed?.({
|
|
10421
|
+
reason,
|
|
10422
|
+
code: connection.failure?.code ?? null,
|
|
10423
|
+
conversationId: connection.failure?.conversationId ?? null
|
|
9936
10424
|
});
|
|
9937
|
-
|
|
9938
|
-
|
|
9939
|
-
|
|
9940
|
-
|
|
9941
|
-
|
|
9942
|
-
|
|
9943
|
-
|
|
9944
|
-
|
|
9945
|
-
|
|
9946
|
-
|
|
10425
|
+
}
|
|
10426
|
+
}
|
|
10427
|
+
closeError(connection, conversationId) {
|
|
10428
|
+
if (conversationId !== null && this.refused.has(conversationId)) {
|
|
10429
|
+
return new ConversationConnectionError(
|
|
10430
|
+
ACCESS_DENIED,
|
|
10431
|
+
joinMessage(ACCESS_DENIED)
|
|
10432
|
+
);
|
|
10433
|
+
}
|
|
10434
|
+
return new ConversationConnectionError(
|
|
10435
|
+
"CONVERSATION_DISCONNECTED",
|
|
10436
|
+
connection.closed?.message ?? closeMessage(CLIENT_CLOSE),
|
|
10437
|
+
connection.closed?.reason ?? null
|
|
10438
|
+
);
|
|
10439
|
+
}
|
|
10440
|
+
/**
|
|
10441
|
+
* Waits for the first connection. A connection error that socket.io
|
|
10442
|
+
* recovers from also ends the wait: the join then waits in socket.io's
|
|
10443
|
+
* buffer, for the time of its acknowledgement at most.
|
|
10444
|
+
*/
|
|
10445
|
+
connected(connection) {
|
|
10446
|
+
const { socket } = connection;
|
|
10447
|
+
if (connection.closed) {
|
|
10448
|
+
return Promise.reject(this.closeError(connection, null));
|
|
10449
|
+
}
|
|
10450
|
+
if (socket.connected || connection.connectedBefore) {
|
|
10451
|
+
return Promise.resolve();
|
|
10452
|
+
}
|
|
10453
|
+
return new Promise((resolve, reject) => {
|
|
10454
|
+
const entry = { conversationId: null, fail: reject };
|
|
10455
|
+
const done = () => {
|
|
10456
|
+
if (connection.pending.delete(entry)) {
|
|
10457
|
+
socket.off("connect", done);
|
|
10458
|
+
socket.off("connect_error", done);
|
|
10459
|
+
resolve();
|
|
9947
10460
|
}
|
|
9948
|
-
|
|
10461
|
+
};
|
|
10462
|
+
connection.pending.add(entry);
|
|
10463
|
+
socket.on("connect", done);
|
|
10464
|
+
socket.on("connect_error", done);
|
|
10465
|
+
});
|
|
10466
|
+
}
|
|
10467
|
+
/**
|
|
10468
|
+
* Emits an event and waits for its acknowledgement. A join (`joining`)
|
|
10469
|
+
* that the server does not accept fails with the code of its
|
|
10470
|
+
* acknowledgement. A close of the connection fails the wait at once; a
|
|
10471
|
+
* timeout fails it with `CONVERSATION_TIMEOUT`.
|
|
10472
|
+
*/
|
|
10473
|
+
acknowledged(connection, event, payload, timeoutMs, joining) {
|
|
10474
|
+
return new Promise((resolve, reject) => {
|
|
10475
|
+
if (connection.closed) {
|
|
10476
|
+
reject(this.closeError(connection, joining));
|
|
10477
|
+
return;
|
|
10478
|
+
}
|
|
10479
|
+
const entry = { conversationId: joining, fail: reject };
|
|
10480
|
+
connection.pending.add(entry);
|
|
10481
|
+
connection.socket.timeout(timeoutMs).emit(event, payload, (error, result) => {
|
|
10482
|
+
if (!connection.pending.delete(entry)) {
|
|
9949
10483
|
return;
|
|
9950
10484
|
}
|
|
9951
|
-
this.
|
|
9952
|
-
|
|
9953
|
-
|
|
9954
|
-
|
|
9955
|
-
|
|
9956
|
-
});
|
|
9957
|
-
this.socket.on(SPEECH_EVENTS.turnFailed, (payload) => {
|
|
9958
|
-
const event = payload;
|
|
9959
|
-
if (typeof event?.requestId === "string" && event.conversationId === this.options.getConversationId()) {
|
|
9960
|
-
this.options.handleSpeechEvent?.({ type: "turnFailed", event });
|
|
10485
|
+
const failure = error ? acknowledgementError(error, connection.lost) : joining === null ? null : this.joinFailure(connection, joining, result);
|
|
10486
|
+
if (failure) {
|
|
10487
|
+
reject(failure);
|
|
10488
|
+
} else {
|
|
10489
|
+
resolve(result);
|
|
9961
10490
|
}
|
|
9962
10491
|
});
|
|
9963
|
-
|
|
9964
|
-
|
|
9965
|
-
|
|
9966
|
-
|
|
9967
|
-
|
|
9968
|
-
|
|
9969
|
-
|
|
9970
|
-
|
|
9971
|
-
|
|
9972
|
-
|
|
9973
|
-
|
|
9974
|
-
|
|
9975
|
-
|
|
9976
|
-
|
|
9977
|
-
|
|
9978
|
-
|
|
9979
|
-
|
|
9980
|
-
|
|
9981
|
-
|
|
9982
|
-
|
|
9983
|
-
|
|
9984
|
-
|
|
9985
|
-
|
|
9986
|
-
|
|
9987
|
-
|
|
9988
|
-
|
|
9989
|
-
|
|
10492
|
+
});
|
|
10493
|
+
}
|
|
10494
|
+
/** The error of a join that the server did not accept; null after a join. */
|
|
10495
|
+
joinFailure(connection, conversationId, result) {
|
|
10496
|
+
const ack = result;
|
|
10497
|
+
if (ack?.ok) {
|
|
10498
|
+
return null;
|
|
10499
|
+
}
|
|
10500
|
+
const named = typeof ack?.error === "string" ? ack.error.trim() : "";
|
|
10501
|
+
const code = named || JOIN_FAILED;
|
|
10502
|
+
connection.failure ??= { code, conversationId };
|
|
10503
|
+
if (code === ACCESS_DENIED) {
|
|
10504
|
+
this.refused.add(conversationId);
|
|
10505
|
+
}
|
|
10506
|
+
this.close(connection, "io server disconnect");
|
|
10507
|
+
return new ConversationConnectionError(code, joinMessage(code));
|
|
10508
|
+
}
|
|
10509
|
+
/** The events that the server sends to the conversation's room. */
|
|
10510
|
+
receive(socket) {
|
|
10511
|
+
socket.on("conversations:stream", (payload) => {
|
|
10512
|
+
let candidate;
|
|
10513
|
+
try {
|
|
10514
|
+
candidate = decodePerformance(
|
|
10515
|
+
payload,
|
|
10516
|
+
"stream"
|
|
10517
|
+
);
|
|
10518
|
+
} catch (error) {
|
|
10519
|
+
this.options.handleCallbackError("stream.decode", error);
|
|
10520
|
+
return;
|
|
10521
|
+
}
|
|
10522
|
+
if (typeof candidate?.conversationId !== "string") {
|
|
10523
|
+
return;
|
|
10524
|
+
}
|
|
10525
|
+
if (candidate.conversationId !== this.options.getConversationId()) {
|
|
10526
|
+
return;
|
|
10527
|
+
}
|
|
10528
|
+
const event = candidate.event;
|
|
10529
|
+
if (!event || typeof event !== "object") {
|
|
10530
|
+
return;
|
|
10531
|
+
}
|
|
10532
|
+
this.options.handleStreamEvent(event);
|
|
10533
|
+
});
|
|
10534
|
+
socket.on(SPEECH_EVENTS.partial, (payload) => {
|
|
10535
|
+
const event = payload;
|
|
10536
|
+
if (typeof event?.sessionId === "string" && typeof event.text === "string") {
|
|
10537
|
+
this.options.handleSpeechEvent?.({ type: "partial", event });
|
|
10538
|
+
}
|
|
10539
|
+
});
|
|
10540
|
+
socket.on(SPEECH_EVENTS.turn, (payload) => {
|
|
10541
|
+
let candidate;
|
|
10542
|
+
try {
|
|
10543
|
+
candidate = decodePerformance(
|
|
10544
|
+
payload,
|
|
10545
|
+
"turn"
|
|
10546
|
+
);
|
|
10547
|
+
} catch (error) {
|
|
10548
|
+
this.options.handleCallbackError("speech.turn.decode", error);
|
|
10549
|
+
return;
|
|
10550
|
+
}
|
|
10551
|
+
if (candidate?.conversationId !== this.options.getConversationId() || typeof candidate.requestId !== "string" || !candidate.turn || typeof candidate.turn !== "object") {
|
|
10552
|
+
return;
|
|
10553
|
+
}
|
|
10554
|
+
this.options.handleSpeechEvent?.({
|
|
10555
|
+
type: "turn",
|
|
10556
|
+
requestId: candidate.requestId,
|
|
10557
|
+
turn: candidate.turn
|
|
9990
10558
|
});
|
|
9991
|
-
|
|
9992
|
-
|
|
9993
|
-
|
|
9994
|
-
|
|
9995
|
-
}
|
|
9996
|
-
|
|
9997
|
-
|
|
10559
|
+
});
|
|
10560
|
+
socket.on(SPEECH_EVENTS.turnFailed, (payload) => {
|
|
10561
|
+
const event = payload;
|
|
10562
|
+
if (typeof event?.requestId === "string" && event.conversationId === this.options.getConversationId()) {
|
|
10563
|
+
this.options.handleSpeechEvent?.({ type: "turnFailed", event });
|
|
10564
|
+
}
|
|
10565
|
+
});
|
|
10566
|
+
socket.on("conversations:turn", (payload) => {
|
|
10567
|
+
let candidate;
|
|
10568
|
+
try {
|
|
10569
|
+
candidate = decodePerformance(
|
|
10570
|
+
payload,
|
|
10571
|
+
"turn"
|
|
10572
|
+
);
|
|
10573
|
+
} catch (error) {
|
|
10574
|
+
this.options.handleCallbackError("turn.decode", error);
|
|
10575
|
+
return;
|
|
10576
|
+
}
|
|
10577
|
+
if (typeof candidate?.conversationId !== "string") {
|
|
10578
|
+
return;
|
|
10579
|
+
}
|
|
10580
|
+
if (candidate.conversationId !== this.options.getConversationId()) {
|
|
10581
|
+
return;
|
|
10582
|
+
}
|
|
10583
|
+
const turn = candidate.turn;
|
|
10584
|
+
if (!turn || typeof turn !== "object") {
|
|
10585
|
+
return;
|
|
10586
|
+
}
|
|
10587
|
+
void this.options.handleTurn(turn).catch((error) => {
|
|
10588
|
+
this.options.handleCallbackError("onTurn", error);
|
|
9998
10589
|
});
|
|
9999
|
-
}
|
|
10000
|
-
await new Promise((resolve, reject) => {
|
|
10001
|
-
this.socket.timeout(1e4).emit(
|
|
10002
|
-
"conversations:join",
|
|
10003
|
-
{ conversationId: normalized },
|
|
10004
|
-
(error, result) => {
|
|
10005
|
-
if (error || !result?.ok) {
|
|
10006
|
-
reject(
|
|
10007
|
-
error ?? new Error("Conversation streaming access was denied.")
|
|
10008
|
-
);
|
|
10009
|
-
} else {
|
|
10010
|
-
resolve();
|
|
10011
|
-
}
|
|
10012
|
-
}
|
|
10013
|
-
);
|
|
10014
10590
|
});
|
|
10015
10591
|
}
|
|
10016
10592
|
};
|
|
@@ -10967,6 +11543,8 @@ var Character = class extends CharacterRuntime {
|
|
|
10967
11543
|
notifiedStagingIds = /* @__PURE__ */ new Map();
|
|
10968
11544
|
liveConversation;
|
|
10969
11545
|
baked = null;
|
|
11546
|
+
/** The conversation whose live motion was warmed up. */
|
|
11547
|
+
liveMotionPrepared = null;
|
|
10970
11548
|
authoring;
|
|
10971
11549
|
normalizeIdleOrientation;
|
|
10972
11550
|
bakedAsset;
|
|
@@ -10979,6 +11557,7 @@ var Character = class extends CharacterRuntime {
|
|
|
10979
11557
|
latestOptions = [];
|
|
10980
11558
|
listeningStateCallback = null;
|
|
10981
11559
|
transcriptCallback = null;
|
|
11560
|
+
connectionClosedCallback = null;
|
|
10982
11561
|
conversationId = null;
|
|
10983
11562
|
constructor(options) {
|
|
10984
11563
|
const resolvedToken = (options.apiToken ?? options.apiKey)?.trim() ?? "";
|
|
@@ -11021,7 +11600,12 @@ var Character = class extends CharacterRuntime {
|
|
|
11021
11600
|
await this.handleStagingResult(turn, { source: "turn" });
|
|
11022
11601
|
},
|
|
11023
11602
|
handleCallbackError: this.handleCallbackError,
|
|
11024
|
-
handleSpeechEvent: (event) => this.voice.handleSocketEvent(event)
|
|
11603
|
+
handleSpeechEvent: (event) => this.voice.handleSocketEvent(event),
|
|
11604
|
+
handleConnectionClosed: (event) => invokeSafeCallback(
|
|
11605
|
+
"onConnectionClosed",
|
|
11606
|
+
this.handleCallbackError,
|
|
11607
|
+
() => this.connectionClosedCallback?.(event)
|
|
11608
|
+
)
|
|
11025
11609
|
});
|
|
11026
11610
|
this.voice = new VoiceInput({
|
|
11027
11611
|
socket: () => this.conversationSocket,
|
|
@@ -11174,6 +11758,25 @@ var Character = class extends CharacterRuntime {
|
|
|
11174
11758
|
});
|
|
11175
11759
|
this.animationController = new RemoteAnimationController();
|
|
11176
11760
|
}
|
|
11761
|
+
/**
|
|
11762
|
+
* Warms up live motion for the current conversation, once, on the
|
|
11763
|
+
* player's intent: call it at the first focus or key in a reply field or
|
|
11764
|
+
* when a talk key goes down. It does nothing for a character without live
|
|
11765
|
+
* motion; the server limits warm-ups per player (ADR 0011).
|
|
11766
|
+
*/
|
|
11767
|
+
async prepareLiveMotion() {
|
|
11768
|
+
const conversationId = this.conversationId;
|
|
11769
|
+
if (this.baked || !conversationId || this.liveMotionPrepared === conversationId) {
|
|
11770
|
+
return false;
|
|
11771
|
+
}
|
|
11772
|
+
this.liveMotionPrepared = conversationId;
|
|
11773
|
+
return warmUpLiveMotion({
|
|
11774
|
+
conversationId,
|
|
11775
|
+
baseUrl: this.baseUrl,
|
|
11776
|
+
apiToken: this.apiToken,
|
|
11777
|
+
renderSessionId: this.renderSessionId
|
|
11778
|
+
});
|
|
11779
|
+
}
|
|
11177
11780
|
async createConversation(options) {
|
|
11178
11781
|
const response = this.baked ? {
|
|
11179
11782
|
conversation: {
|
|
@@ -11346,6 +11949,19 @@ var Character = class extends CharacterRuntime {
|
|
|
11346
11949
|
callback
|
|
11347
11950
|
);
|
|
11348
11951
|
}
|
|
11952
|
+
/**
|
|
11953
|
+
* The server closed the conversation connection: a refused or failed join,
|
|
11954
|
+
* or an API key that was rotated, edited or deleted. The next `connect`,
|
|
11955
|
+
* `sendMessage` or `startListening` opens a new connection; a refused
|
|
11956
|
+
* conversation fails at once with `CONVERSATION_ACCESS_DENIED`.
|
|
11957
|
+
*/
|
|
11958
|
+
onConnectionClosed(callback) {
|
|
11959
|
+
return bindCallbackSlot(
|
|
11960
|
+
() => this.connectionClosedCallback,
|
|
11961
|
+
(next) => this.connectionClosedCallback = next,
|
|
11962
|
+
callback
|
|
11963
|
+
);
|
|
11964
|
+
}
|
|
11349
11965
|
async getConversationWorldline(conversationId = this.conversationId) {
|
|
11350
11966
|
if (!conversationId) {
|
|
11351
11967
|
throw new Error("Conversation ID is required.");
|
|
@@ -12075,10 +12691,12 @@ export {
|
|
|
12075
12691
|
CharacterExpressions,
|
|
12076
12692
|
CharacterRuntime,
|
|
12077
12693
|
CharacterSceneRig,
|
|
12694
|
+
ConversationConnectionError,
|
|
12078
12695
|
FacialPerformanceLayer,
|
|
12079
12696
|
FacialPerformanceMixer,
|
|
12080
12697
|
INPUT_MODES,
|
|
12081
12698
|
InputModeError,
|
|
12699
|
+
LiveMotionPlayer,
|
|
12082
12700
|
MicrophoneError,
|
|
12083
12701
|
Npcify,
|
|
12084
12702
|
Character as NpcifyCharacter,
|