@npcify/web 0.3.0 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,3 +1,36 @@
1
+ // ../ts-shared/src/motion-frames.ts
2
+ var MOTION_FRAMES_CAPABILITY = "motion_frames";
3
+ var motionFrameWidth = function(boneCount) {
4
+ return 3 + 4 * boneCount;
5
+ };
6
+ var MAX_MOTION_FRAMES_PER_EVENT = 3600;
7
+ var MAX_MOTION_BONES = 64;
8
+ var isCount = function(value, max2) {
9
+ return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 && value <= max2;
10
+ };
11
+ var readMotionFrames = function(data, frameCount, boneCount) {
12
+ const values = frameCount * motionFrameWidth(boneCount);
13
+ if (data.byteLength !== values * 4) {
14
+ throw new Error("Motion frames do not match their frame count.");
15
+ }
16
+ const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
17
+ const frames = new Float32Array(values);
18
+ for (let index = 0; index < values; index += 1) {
19
+ frames[index] = view.getFloat32(index * 4, true);
20
+ }
21
+ return frames;
22
+ };
23
+ var isValidMotionFramesEvent = function(event) {
24
+ if (!event || typeof event !== "object") {
25
+ return false;
26
+ }
27
+ const value = event;
28
+ const bones = value.bones;
29
+ const frameCount = value.frameCount;
30
+ const frames = value.frames;
31
+ return value.type === "motion_frames" && typeof value.taskId === "string" && typeof value.stagingId === "string" && isCount(value.revision, Number.MAX_SAFE_INTEGER) && typeof value.fps === "number" && value.fps > 0 && value.fps <= 240 && isCount(value.startFrame, 24 * 60 * 60 * 240) && isCount(frameCount, MAX_MOTION_FRAMES_PER_EVENT) && Array.isArray(bones) && bones.length > 0 && bones.length <= MAX_MOTION_BONES && bones.every((bone) => typeof bone === "string") && typeof value.restHipsHeight === "number" && value.restHipsHeight > 0 && Number.isFinite(value.restHipsHeight) && frames instanceof Uint8Array && frames.byteLength === frameCount * motionFrameWidth(bones.length) * 4;
32
+ };
33
+
1
34
  // ../ts-shared/src/llm.ts
2
35
  var STAGING_PROTOCOL_VERSION = 1;
3
36
 
@@ -14,10 +47,6 @@ var GPT_IMAGE_MODELS = [
14
47
  "gpt-image-2.5-flare",
15
48
  "gpt-image-2.5-sunburst"
16
49
  ];
17
- var GEMINI_SPEECH_MODEL_VENDORS = {
18
- "gemini-3.5-transcribe-live": "gemini",
19
- "gemini-3.5-transcribe": "gemini"
20
- };
21
50
  var AI_ROLES = {
22
51
  character_draft: {
23
52
  name: "Character draft",
@@ -59,28 +88,6 @@ var AI_ROLES = {
59
88
  models: ["gemini-3.8-flash-tts", "gemini-3.8-flash-lite-tts"],
60
89
  options: []
61
90
  },
62
- // Priced by SPEECH_RECOGNITION_PRICES of the server (docs/voice-input.md).
63
- // Audio that arrives while the player speaks. The model selects the vendor.
64
- speech_input_live: {
65
- name: "Speech input (live)",
66
- source: "openai",
67
- models: [
68
- "gpt-transcribe",
69
- "gpt-live-transcribe",
70
- "gemini-3.5-transcribe-live",
71
- "gemini-3.5-transcribe"
72
- ],
73
- modelVendors: GEMINI_SPEECH_MODEL_VENDORS,
74
- options: []
75
- },
76
- // One complete recording (an upload of an engine SDK).
77
- speech_input_file: {
78
- name: "Speech input (file)",
79
- source: "openai",
80
- models: ["gpt-transcribe", "gemini-3.5-transcribe"],
81
- modelVendors: GEMINI_SPEECH_MODEL_VENDORS,
82
- options: []
83
- },
84
91
  // Priced by OPENAI_PRICES.text of the generator.
85
92
  generator_agent: {
86
93
  name: "Generator agent",
@@ -116,7 +123,7 @@ var permittedInputModes = function(permissions) {
116
123
  if (permissions.baked || !permissions.allowCustomReply) {
117
124
  return ["options"];
118
125
  }
119
- return ["options", "text", "voice"];
126
+ return permissions.speechInput ? ["options", "text", "voice"] : ["options", "text"];
120
127
  };
121
128
  var isInputMode = function(value) {
122
129
  return INPUT_MODES.includes(value);
@@ -159,8 +166,9 @@ var resolveInputModes = function(configured, permissions) {
159
166
  }
160
167
  const refused = configured.filter((mode) => !permitted.includes(mode));
161
168
  if (refused.length > 0) {
169
+ const names = refused.map((mode) => `"${mode}"`).join(" and ");
162
170
  throw new InputModeError(
163
- `This character accepts only its reply options (Allow custom replies is off). Remove ${refused.map((mode) => `"${mode}"`).join(" and ")} from inputModes.`
171
+ permissions.allowCustomReply ? `Voice input is off for this character (its owner turns it on in the STT tab). Remove ${names} from inputModes.` : `This character accepts only its reply options (Allow custom replies is off). Remove ${names} from inputModes.`
164
172
  );
165
173
  }
166
174
  return [...configured];
@@ -186,6 +194,54 @@ var SPEECH_EVENTS = Object.freeze({
186
194
  turnFailed: "speech:turn_failed"
187
195
  });
188
196
 
197
+ // ../ts-shared/src/speech-input-providers.ts
198
+ var SPEECH_INPUT_PROVIDER_PROFILES = Object.freeze({
199
+ // The file model also streams: 120 ms slower than gpt-live-transcribe,
200
+ // at a 3.8 times lower price and with fewer errors.
201
+ openai: {
202
+ live: {
203
+ model: "gpt-transcribe",
204
+ releaseToTextMs: 783,
205
+ usdPerMinute: 45e-4,
206
+ approximate: false
207
+ },
208
+ file: {
209
+ model: "gpt-transcribe",
210
+ releaseToTextMs: 867,
211
+ usdPerMinute: 45e-4,
212
+ approximate: false
213
+ }
214
+ },
215
+ gemini: {
216
+ live: {
217
+ model: "gemini-3.5-transcribe-live",
218
+ releaseToTextMs: 266,
219
+ usdPerMinute: 9e-3,
220
+ approximate: true
221
+ },
222
+ file: {
223
+ model: "gemini-3.5-transcribe",
224
+ releaseToTextMs: 1263,
225
+ usdPerMinute: 5e-3,
226
+ approximate: true
227
+ }
228
+ },
229
+ elevenlabs: {
230
+ live: {
231
+ model: "scribe_v2_realtime",
232
+ releaseToTextMs: 208,
233
+ usdPerMinute: 0.39 / 60,
234
+ approximate: false
235
+ },
236
+ file: {
237
+ model: "scribe_v2",
238
+ releaseToTextMs: 925,
239
+ usdPerMinute: 0.22 / 60,
240
+ approximate: false
241
+ }
242
+ }
243
+ });
244
+
189
245
  // ../ts-shared/src/product-usage.ts
190
246
  var productUsageEvents = [
191
247
  "page_viewed",
@@ -2579,6 +2635,8 @@ var PERFORMANCE_KINDS = {
2579
2635
  storedTimeline: 7,
2580
2636
  cachedOpening: 8
2581
2637
  };
2638
+ var PERFORMANCE_MAJOR_VERSION = 1;
2639
+ var PERFORMANCE_MINOR_VERSION = 1;
2582
2640
  var HEADER_SIZE = 24;
2583
2641
  var MAGIC = 1346588750;
2584
2642
  var crcTable = Uint32Array.from({ length: 256 }, (_, index) => {
@@ -2650,8 +2708,8 @@ var encodePerformance = function(kind, value, options = {}) {
2650
2708
  const bytes = new Uint8Array(HEADER_SIZE + body.length);
2651
2709
  const header = new DataView(bytes.buffer);
2652
2710
  header.setUint32(0, MAGIC, true);
2653
- header.setUint8(4, 1);
2654
- header.setUint8(5, 0);
2711
+ header.setUint8(4, PERFORMANCE_MAJOR_VERSION);
2712
+ header.setUint8(5, PERFORMANCE_MINOR_VERSION);
2655
2713
  header.setUint8(6, PERFORMANCE_KINDS[kind]);
2656
2714
  header.setUint8(7, options.compress ? 1 : 0);
2657
2715
  header.setUint16(8, HEADER_SIZE, true);
@@ -2688,7 +2746,7 @@ var decodePerformance = function(input, expectedKind) {
2688
2746
  throw new Error("Truncated performance header.");
2689
2747
  }
2690
2748
  const header = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
2691
- if (header.getUint32(0, true) !== MAGIC || header.getUint8(4) !== 1) {
2749
+ if (header.getUint32(0, true) !== MAGIC || header.getUint8(4) !== PERFORMANCE_MAJOR_VERSION) {
2692
2750
  throw new Error("Unsupported performance format or major version.");
2693
2751
  }
2694
2752
  if (header.getUint8(6) !== PERFORMANCE_KINDS[expectedKind]) {
@@ -3143,7 +3201,8 @@ var createConversation = async function({
3143
3201
  ),
3144
3202
  credentials: apiToken ? "omit" : "include",
3145
3203
  method: "POST",
3146
- body: JSON.stringify({ avatarId })
3204
+ // This SDK plays live motion frames (ADR 0011).
3205
+ body: JSON.stringify({ avatarId, capabilities: [MOTION_FRAMES_CAPABILITY] })
3147
3206
  });
3148
3207
  if (!res.ok) {
3149
3208
  throw Object.assign(new Error(await res.text()), { status: res.status });
@@ -3196,7 +3255,8 @@ var createStaging = async function({
3196
3255
  content: message,
3197
3256
  silent: silent === true,
3198
3257
  incrementalPlayback: true,
3199
- parentStagingId
3258
+ parentStagingId,
3259
+ capabilities: [MOTION_FRAMES_CAPABILITY]
3200
3260
  })
3201
3261
  });
3202
3262
  } catch (error) {
@@ -3296,6 +3356,35 @@ var getConversationWorldline = async function(options) {
3296
3356
  }
3297
3357
  return await response.json();
3298
3358
  };
3359
+ var warmUpLiveMotion = async function({
3360
+ conversationId,
3361
+ apiToken,
3362
+ baseUrl,
3363
+ renderSessionId
3364
+ }) {
3365
+ const url = new URL(`/conversations/${conversationId}/motion/warm-up`, baseUrl);
3366
+ try {
3367
+ const res = await fetch(url.toString(), {
3368
+ headers: withRenderSession(
3369
+ buildAuthHeaders(apiToken, {
3370
+ json: true,
3371
+ allowEmpty: true
3372
+ }),
3373
+ renderSessionId
3374
+ ),
3375
+ credentials: apiToken ? "omit" : "include",
3376
+ method: "POST",
3377
+ body: JSON.stringify({ capabilities: [MOTION_FRAMES_CAPABILITY] })
3378
+ });
3379
+ if (!res.ok) {
3380
+ return false;
3381
+ }
3382
+ const payload = await res.json();
3383
+ return payload.warming === true;
3384
+ } catch {
3385
+ return false;
3386
+ }
3387
+ };
3299
3388
 
3300
3389
  // ../../clients/web-client/src/live-conversation.ts
3301
3390
  var LiveConversation = class {
@@ -6022,6 +6111,10 @@ var StreamPlaybackController = class {
6022
6111
  get cursor() {
6023
6112
  return this.currentCursorValue;
6024
6113
  }
6114
+ /** The staging that plays or waits to play, or null. */
6115
+ get activeStagingId() {
6116
+ return this.activePlayback?.stagingId ?? null;
6117
+ }
6025
6118
  onStreamEvent(callback) {
6026
6119
  return bindCallbackSlot(
6027
6120
  () => this.streamEventCallback,
@@ -6115,6 +6208,10 @@ var StreamPlaybackController = class {
6115
6208
  this.dependencies.telemetry.recordFirstTextDelta();
6116
6209
  return;
6117
6210
  }
6211
+ if (event.type === "motion_frames") {
6212
+ this.dependencies.onMotionFrames?.(event);
6213
+ return;
6214
+ }
6118
6215
  const player = this.dependencies.getStagingPlayer();
6119
6216
  if (event.type === "speech_chunk") {
6120
6217
  player.appendSpeechChunk(event.streamId, event.mimeType, event.data);
@@ -6228,6 +6325,9 @@ var StreamPlaybackController = class {
6228
6325
  this.activePlayback = { taskId: params.taskId, stagingId: params.stagingId };
6229
6326
  this.playedStagingIds.add(params.stagingId);
6230
6327
  this.recordPlaybackEvent(params, "requested");
6328
+ if (params.timeline.motion) {
6329
+ this.dependencies.onMotionTrack?.(params.stagingId, params.timeline.motion);
6330
+ }
6231
6331
  const existing = this.states.get(params.stagingId);
6232
6332
  const state = {
6233
6333
  taskId: params.taskId,
@@ -8237,6 +8337,261 @@ var CharacterSceneRig = class {
8237
8337
  }
8238
8338
  };
8239
8339
 
8340
+ // ../../clients/web-client/src/character/live-motion.ts
8341
+ import { Quaternion as Quaternion3, Vector3 as Vector37 } from "three";
8342
+ var PARENTS = {
8343
+ spine: "hips",
8344
+ chest: "spine",
8345
+ upperChest: "chest",
8346
+ neck: "upperChest",
8347
+ head: "neck",
8348
+ leftShoulder: "upperChest",
8349
+ leftUpperArm: "leftShoulder",
8350
+ leftLowerArm: "leftUpperArm",
8351
+ leftHand: "leftLowerArm",
8352
+ rightShoulder: "upperChest",
8353
+ rightUpperArm: "rightShoulder",
8354
+ rightLowerArm: "rightUpperArm",
8355
+ rightHand: "rightLowerArm",
8356
+ leftUpperLeg: "hips",
8357
+ leftLowerLeg: "leftUpperLeg",
8358
+ leftFoot: "leftLowerLeg",
8359
+ leftToes: "leftFoot",
8360
+ rightUpperLeg: "hips",
8361
+ rightLowerLeg: "rightUpperLeg",
8362
+ rightFoot: "rightLowerLeg",
8363
+ rightToes: "rightFoot"
8364
+ };
8365
+ var MAX_TRACKS = 4;
8366
+ var approach = function(value, target, delta, seconds) {
8367
+ if (seconds <= 0) {
8368
+ return target;
8369
+ }
8370
+ const step = delta / seconds;
8371
+ return value < target ? Math.min(target, value + step) : Math.max(target, value - step);
8372
+ };
8373
+ var LiveMotionPlayer = class {
8374
+ vrm = null;
8375
+ tracks = /* @__PURE__ */ new Map();
8376
+ rootMode;
8377
+ blendIn;
8378
+ blendOut;
8379
+ weight = 0;
8380
+ /** The last applied pose, to blend out of it after the frames end. */
8381
+ lastPose = null;
8382
+ scratch = {
8383
+ from: new Quaternion3(),
8384
+ to: new Quaternion3(),
8385
+ carry: /* @__PURE__ */ new Map(),
8386
+ position: new Vector37()
8387
+ };
8388
+ constructor(options = {}) {
8389
+ this.rootMode = options.rootMode ?? "anchored";
8390
+ this.blendIn = options.blendInSeconds ?? 0.3;
8391
+ this.blendOut = options.blendOutSeconds ?? 0.5;
8392
+ }
8393
+ setVrm(vrm) {
8394
+ this.vrm = vrm;
8395
+ this.weight = 0;
8396
+ this.lastPose = null;
8397
+ }
8398
+ /** Adds the frames of a `motion_frames` event; invalid events are dropped. */
8399
+ push(received) {
8400
+ const frames = received.frames;
8401
+ const event = frames instanceof ArrayBuffer ? { ...received, frames: new Uint8Array(frames) } : received;
8402
+ if (!isValidMotionFramesEvent(event)) {
8403
+ return false;
8404
+ }
8405
+ const values = readMotionFrames(
8406
+ event.frames,
8407
+ event.frameCount,
8408
+ event.bones.length
8409
+ );
8410
+ const track = this.track(event.stagingId, {
8411
+ fps: event.fps,
8412
+ bones: event.bones,
8413
+ restHipsHeight: event.restHipsHeight
8414
+ });
8415
+ for (let frame = 0; frame < event.frameCount; frame += 1) {
8416
+ const index = event.startFrame + frame;
8417
+ const existing = track.frames.get(index);
8418
+ if (existing && existing.revision > event.revision) {
8419
+ continue;
8420
+ }
8421
+ track.frames.set(index, {
8422
+ revision: event.revision,
8423
+ values: values.subarray(frame * track.width, (frame + 1) * track.width)
8424
+ });
8425
+ }
8426
+ return true;
8427
+ }
8428
+ /** Loads the stored frames of a staging (a replay). */
8429
+ async load(stagingId, track, fetchBytes = async (url) => new Uint8Array(await (await fetch(url)).arrayBuffer())) {
8430
+ if (this.tracks.has(stagingId)) {
8431
+ return;
8432
+ }
8433
+ const data = track.frames.type === "bytes" ? track.frames.data : await fetchBytes(track.frames.url);
8434
+ const values = readMotionFrames(data, track.frameCount, track.bones.length);
8435
+ const stored = this.track(stagingId, track);
8436
+ for (let frame = 0; frame < track.frameCount; frame += 1) {
8437
+ stored.frames.set(frame, {
8438
+ revision: Number.MAX_SAFE_INTEGER,
8439
+ values: values.subarray(
8440
+ frame * stored.width,
8441
+ (frame + 1) * stored.width
8442
+ )
8443
+ });
8444
+ }
8445
+ }
8446
+ has(stagingId) {
8447
+ return (this.tracks.get(stagingId)?.frames.size ?? 0) > 0;
8448
+ }
8449
+ forget(stagingId) {
8450
+ this.tracks.delete(stagingId);
8451
+ }
8452
+ /** The current live weight, 0 (idle) to 1 (live frames). */
8453
+ get liveWeight() {
8454
+ return this.weight;
8455
+ }
8456
+ /**
8457
+ * Applies the live pose of `stagingId` at `time` seconds of its staging
8458
+ * clock; `time` null means that no staging plays now.
8459
+ */
8460
+ update(delta, stagingId, time) {
8461
+ const vrm = this.vrm;
8462
+ if (!vrm?.humanoid) {
8463
+ return;
8464
+ }
8465
+ const track = stagingId ? this.tracks.get(stagingId) : void 0;
8466
+ let pose = null;
8467
+ if (track && time !== null && time >= 0) {
8468
+ pose = this.sample(track, time);
8469
+ }
8470
+ this.weight = approach(
8471
+ this.weight,
8472
+ pose ? 1 : 0,
8473
+ Math.max(0, delta),
8474
+ pose ? this.blendIn : this.blendOut
8475
+ );
8476
+ if (pose) {
8477
+ this.lastPose = pose;
8478
+ }
8479
+ const applied = pose ?? this.lastPose;
8480
+ if (!applied || this.weight <= 0) {
8481
+ if (this.weight <= 0) {
8482
+ this.lastPose = null;
8483
+ }
8484
+ return;
8485
+ }
8486
+ this.apply(vrm, applied.track, applied.values, this.weight);
8487
+ }
8488
+ track(stagingId, meta) {
8489
+ let track = this.tracks.get(stagingId);
8490
+ if (!track) {
8491
+ track = {
8492
+ fps: meta.fps,
8493
+ bones: [...meta.bones],
8494
+ restHipsHeight: meta.restHipsHeight,
8495
+ width: motionFrameWidth(meta.bones.length),
8496
+ frames: /* @__PURE__ */ new Map(),
8497
+ origin: null
8498
+ };
8499
+ this.tracks.set(stagingId, track);
8500
+ while (this.tracks.size > MAX_TRACKS) {
8501
+ const oldest = this.tracks.keys().next().value;
8502
+ if (oldest === void 0 || oldest === stagingId) {
8503
+ break;
8504
+ }
8505
+ this.tracks.delete(oldest);
8506
+ }
8507
+ }
8508
+ return track;
8509
+ }
8510
+ /** The interpolated frame at `time`, or null when its frames are missing. */
8511
+ sample(track, time) {
8512
+ const position = time * track.fps;
8513
+ const first = Math.floor(position);
8514
+ const a = track.frames.get(first);
8515
+ if (!a) {
8516
+ return null;
8517
+ }
8518
+ track.origin ??= { x: a.values[0] ?? 0, z: a.values[2] ?? 0 };
8519
+ const b = track.frames.get(first + 1);
8520
+ const amount = position - first;
8521
+ if (!b || amount <= 0) {
8522
+ return { track, values: a.values };
8523
+ }
8524
+ const values = new Float32Array(track.width);
8525
+ for (let index = 0; index < 3; index += 1) {
8526
+ values[index] = (a.values[index] ?? 0) * (1 - amount) + (b.values[index] ?? 0) * amount;
8527
+ }
8528
+ const from = this.scratch.from;
8529
+ const to = this.scratch.to;
8530
+ for (let offset = 3; offset < track.width; offset += 4) {
8531
+ from.fromArray(a.values, offset);
8532
+ to.fromArray(b.values, offset);
8533
+ from.slerp(to, amount);
8534
+ values[offset] = from.x;
8535
+ values[offset + 1] = from.y;
8536
+ values[offset + 2] = from.z;
8537
+ values[offset + 3] = from.w;
8538
+ }
8539
+ return { track, values };
8540
+ }
8541
+ apply(vrm, track, values, weight) {
8542
+ const humanoid = vrm.humanoid;
8543
+ const flip = vrm.meta.metaVersion === "0" && !isNpcifyNormalizedVrm(vrm);
8544
+ const carry = this.scratch.carry;
8545
+ carry.clear();
8546
+ track.bones.forEach((bone, slot) => {
8547
+ const offset = 3 + slot * 4;
8548
+ const rotation = new Quaternion3(
8549
+ values[offset] ?? 0,
8550
+ values[offset + 1] ?? 0,
8551
+ values[offset + 2] ?? 0,
8552
+ values[offset + 3] ?? 1
8553
+ );
8554
+ if (flip) {
8555
+ rotation.x = -rotation.x;
8556
+ rotation.z = -rotation.z;
8557
+ }
8558
+ const inherited = carry.get(bone);
8559
+ if (inherited) {
8560
+ rotation.premultiply(inherited);
8561
+ }
8562
+ const node = humanoid.getNormalizedBoneNode(bone);
8563
+ if (!node) {
8564
+ for (const [child, parent] of Object.entries(PARENTS)) {
8565
+ if (parent === bone) {
8566
+ carry.set(child, rotation.clone());
8567
+ }
8568
+ }
8569
+ return;
8570
+ }
8571
+ node.quaternion.slerp(rotation, weight);
8572
+ });
8573
+ const hips = humanoid.getNormalizedBoneNode("hips");
8574
+ const rest = humanoid.normalizedRestPose.hips?.position;
8575
+ if (!hips || !rest) {
8576
+ return;
8577
+ }
8578
+ const scale = (rest[1] ?? 1) / (track.restHipsHeight || 1);
8579
+ let x = values[0] ?? 0;
8580
+ const y = values[1] ?? 0;
8581
+ let z = values[2] ?? 0;
8582
+ if (this.rootMode === "anchored" && track.origin) {
8583
+ x -= track.origin.x;
8584
+ z -= track.origin.z;
8585
+ }
8586
+ const target = this.scratch.position.set(
8587
+ (flip ? -x : x) * scale + (this.rootMode === "anchored" ? rest[0] ?? 0 : 0),
8588
+ y * scale,
8589
+ (flip ? -z : z) * scale + (this.rootMode === "anchored" ? rest[2] ?? 0 : 0)
8590
+ );
8591
+ hips.position.lerp(target, weight);
8592
+ }
8593
+ };
8594
+
8240
8595
  // ../../clients/web-client/src/character/incremental-staging.ts
8241
8596
  var dialogueKey = function(block) {
8242
8597
  return JSON.stringify(block, (key, value) => {
@@ -8511,9 +8866,17 @@ var StagingPlayer = class {
8511
8866
  activeExpression = null;
8512
8867
  activeExpressionMouthMode = null;
8513
8868
  activeAnimation = null;
8869
+ currentTrackTime = null;
8514
8870
  get paused() {
8515
8871
  return this.clock.paused;
8516
8872
  }
8873
+ /**
8874
+ * Seconds of the staging clock that the tracks show now, or null when no
8875
+ * staging plays (also while playback waits for the next dialogue block).
8876
+ */
8877
+ get trackTime() {
8878
+ return this.currentTrackTime;
8879
+ }
8517
8880
  pause() {
8518
8881
  this.pauseRevision += 1;
8519
8882
  this.clock.pause();
@@ -8914,6 +9277,7 @@ var StagingPlayer = class {
8914
9277
  }
8915
9278
  }
8916
9279
  applyTracks(time, speechActive, timeline) {
9280
+ this.currentTrackTime = time;
8917
9281
  const expression = activeClipAt(timeline.expressions, time);
8918
9282
  const mouthMode = speechActive ? "viseme" : "expression";
8919
9283
  if (expression !== this.activeExpression || expression && mouthMode !== this.activeExpressionMouthMode) {
@@ -8947,6 +9311,7 @@ var StagingPlayer = class {
8947
9311
  }
8948
9312
  }
8949
9313
  clearPerformance() {
9314
+ this.currentTrackTime = null;
8950
9315
  this.activeExpression = null;
8951
9316
  this.activeExpressionMouthMode = null;
8952
9317
  this.activeAnimation = null;
@@ -9385,6 +9750,7 @@ var CharacterRuntime = class {
9385
9750
  performanceRecorder;
9386
9751
  telemetry;
9387
9752
  streamPlayback;
9753
+ liveMotion;
9388
9754
  defaultParent;
9389
9755
  currentGazeTarget = null;
9390
9756
  /** The current gaze target came from `lookAt()`. */
@@ -9433,12 +9799,21 @@ var CharacterRuntime = class {
9433
9799
  baseUrl: this.baseUrl,
9434
9800
  apiToken: this.apiToken
9435
9801
  });
9802
+ this.liveMotion = new LiveMotionPlayer(options.liveMotion);
9436
9803
  this.streamPlayback = new StreamPlaybackController({
9437
9804
  getStagingPlayer: this.getStagingPlayer,
9438
9805
  telemetry: this.telemetry,
9439
9806
  handleCallbackError: this.handleCallbackError,
9440
9807
  reportError: this.reportError,
9441
- recordEvent: (event) => this.performanceRecorder.recordEvent(event)
9808
+ recordEvent: (event) => this.performanceRecorder.recordEvent(event),
9809
+ onMotionFrames: (event) => {
9810
+ this.liveMotion.push(event);
9811
+ },
9812
+ onMotionTrack: (stagingId, track) => {
9813
+ void this.liveMotion.load(stagingId, track).catch((error) => {
9814
+ this.handleCallbackError("liveMotion", error);
9815
+ });
9816
+ }
9442
9817
  });
9443
9818
  if (options.gazeTarget !== void 0) {
9444
9819
  this.setGazeTarget(options.gazeTarget);
@@ -9475,6 +9850,7 @@ var CharacterRuntime = class {
9475
9850
  set vrm(value) {
9476
9851
  this.sceneRig.setVrm(value);
9477
9852
  this.facialMixer.setVrm(value);
9853
+ this.liveMotion.setVrm(value);
9478
9854
  this.applyGaze();
9479
9855
  if (value && this.defaultParent && !value.scene?.parent) {
9480
9856
  this.sceneRig.attachTo(this.defaultParent);
@@ -9751,6 +10127,11 @@ var CharacterRuntime = class {
9751
10127
  return;
9752
10128
  }
9753
10129
  this.animationController?.update(delta);
10130
+ this.liveMotion.update(
10131
+ delta,
10132
+ this.streamPlayback.activeStagingId,
10133
+ this.stagingPlayer?.trackTime ?? null
10134
+ );
9754
10135
  this.visemePlayer.update(delta);
9755
10136
  this.facialMixer.update(delta);
9756
10137
  this.vrm?.update(delta);
@@ -9807,6 +10188,7 @@ var CharacterRuntime = class {
9807
10188
  this.animations.clear();
9808
10189
  this.sceneRig.setVrm(null);
9809
10190
  this.facialMixer.setVrm(null);
10191
+ this.liveMotion.setVrm(null);
9810
10192
  }
9811
10193
  disposeRuntimeAssets() {
9812
10194
  this.sceneRig.disposeVrm();
@@ -9829,28 +10211,101 @@ var CharacterRuntime = class {
9829
10211
 
9830
10212
  // ../../clients/web-client/src/conversation-socket.ts
9831
10213
  import { io } from "socket.io-client";
10214
+ var ConversationConnectionError = class extends Error {
10215
+ constructor(code, message, reason = null) {
10216
+ super(message);
10217
+ this.code = code;
10218
+ this.reason = reason;
10219
+ this.name = "NpcifyConversationConnectionError";
10220
+ }
10221
+ };
10222
+ var JOIN_EVENT = "conversations:join";
10223
+ var JOIN_TIMEOUT_MS = 1e4;
10224
+ var ACCESS_DENIED = "CONVERSATION_ACCESS_DENIED";
10225
+ var JOIN_FAILED = "CONVERSATION_JOIN_FAILED";
10226
+ var CLIENT_CLOSE = "io client disconnect";
10227
+ var SOCKET_IO_TIMEOUT = "operation has timed out";
10228
+ var joinMessage = function(code) {
10229
+ if (code === ACCESS_DENIED) {
10230
+ return "Conversation streaming access was denied.";
10231
+ }
10232
+ if (code === JOIN_FAILED) {
10233
+ return "The server could not join the conversation. Try again later.";
10234
+ }
10235
+ return `The server did not join the conversation (${code}). Try again later.`;
10236
+ };
10237
+ var closeMessage = function(reason, detail) {
10238
+ if (reason === "io server disconnect") {
10239
+ return "The server closed the conversation connection.";
10240
+ }
10241
+ if (reason === CLIENT_CLOSE) {
10242
+ return "The conversation connection closed.";
10243
+ }
10244
+ if (reason === "connect_error") {
10245
+ return `The server refused the conversation connection (${detail || "no reason"}).`;
10246
+ }
10247
+ return `The conversation connection was lost (${reason}).`;
10248
+ };
10249
+ var acknowledgementError = function(error, lost) {
10250
+ if (error.message === SOCKET_IO_TIMEOUT) {
10251
+ return new ConversationConnectionError(
10252
+ "CONVERSATION_TIMEOUT",
10253
+ "The server did not answer in time."
10254
+ );
10255
+ }
10256
+ return new ConversationConnectionError(
10257
+ "CONVERSATION_DISCONNECTED",
10258
+ closeMessage(lost ?? "transport close"),
10259
+ lost
10260
+ );
10261
+ };
10262
+ var refusedConversation = function(payload) {
10263
+ const exception = payload;
10264
+ const named = exception?.cause?.data?.conversationId;
10265
+ if (exception?.message !== ACCESS_DENIED || typeof named !== "string") {
10266
+ return null;
10267
+ }
10268
+ return named.trim() || null;
10269
+ };
9832
10270
  var ConversationSocket = class {
9833
10271
  options;
9834
- socket = null;
10272
+ connection = null;
10273
+ /**
10274
+ * The conversations that the server refused. A join with the same
10275
+ * credentials fails the same way, so the SDK does not try them again until
10276
+ * `disconnect`.
10277
+ */
10278
+ refused = /* @__PURE__ */ new Set();
9835
10279
  constructor(options) {
9836
10280
  this.options = options;
9837
10281
  }
10282
+ /** Closes the connection and forgets the refused conversations. */
9838
10283
  disconnect() {
9839
- this.socket?.disconnect();
9840
- this.socket = null;
10284
+ this.refused.clear();
10285
+ if (this.connection) {
10286
+ this.close(this.connection, CLIENT_CLOSE);
10287
+ }
9841
10288
  }
9842
10289
  /** An event with an acknowledgement. Needs `ensureConnected` first. */
9843
10290
  async request(event, payload, timeoutMs = 1e4) {
9844
- const socket = this.socket;
9845
- if (!socket?.connected) {
9846
- throw new Error("The conversation connection is not open.");
10291
+ const connection = this.connection;
10292
+ if (!connection?.socket.connected) {
10293
+ throw new ConversationConnectionError(
10294
+ "CONVERSATION_DISCONNECTED",
10295
+ "The conversation connection is not open."
10296
+ );
9847
10297
  }
9848
- return await socket.timeout(timeoutMs).emitWithAck(event, payload);
10298
+ return this.acknowledged(connection, event, payload, timeoutMs, null);
9849
10299
  }
9850
10300
  /** An event without an acknowledgement, in order with the other events. */
9851
10301
  send(event, payload) {
9852
- this.socket?.emit(event, payload);
10302
+ this.connection?.socket.emit(event, payload);
9853
10303
  }
10304
+ /**
10305
+ * Joins the room of the conversation. A connection that the server closed
10306
+ * is not used again: the join opens a new one. A refused conversation fails
10307
+ * at once, without a connection.
10308
+ */
9854
10309
  async ensureConnected(conversationId) {
9855
10310
  if (typeof window === "undefined") {
9856
10311
  return;
@@ -9859,135 +10314,279 @@ var ConversationSocket = class {
9859
10314
  if (!normalized) {
9860
10315
  return;
9861
10316
  }
9862
- if (!this.socket) {
9863
- const socketBaseUrl = (() => {
9864
- try {
9865
- return new URL(this.options.baseUrl).origin;
9866
- } catch {
9867
- return window.location.origin;
9868
- }
9869
- })();
9870
- this.socket = io(socketBaseUrl, {
9871
- transports: ["websocket"],
9872
- withCredentials: true,
9873
- auth: {
9874
- apiToken: this.options.apiToken
9875
- }
9876
- });
9877
- this.socket.on("connect", () => {
9878
- const activeConversationId = this.options.getConversationId()?.trim();
9879
- if (activeConversationId) {
9880
- this.socket?.emit("conversations:join", {
9881
- conversationId: activeConversationId
9882
- });
9883
- }
9884
- });
9885
- this.socket.on("conversations:stream", (payload) => {
9886
- let candidate;
9887
- try {
9888
- candidate = decodePerformance(
9889
- payload,
9890
- "stream"
9891
- );
9892
- } catch (error) {
9893
- this.options.handleCallbackError("stream.decode", error);
9894
- return;
9895
- }
9896
- if (typeof candidate?.conversationId !== "string") {
9897
- return;
9898
- }
9899
- if (candidate.conversationId !== this.options.getConversationId()) {
9900
- return;
9901
- }
9902
- const event = candidate.event;
9903
- if (!event || typeof event !== "object") {
9904
- return;
9905
- }
9906
- this.options.handleStreamEvent(event);
9907
- });
9908
- this.socket.on(SPEECH_EVENTS.partial, (payload) => {
9909
- const event = payload;
9910
- if (typeof event?.sessionId === "string" && typeof event.text === "string") {
9911
- this.options.handleSpeechEvent?.({ type: "partial", event });
9912
- }
10317
+ if (this.refused.has(normalized)) {
10318
+ throw new ConversationConnectionError(
10319
+ ACCESS_DENIED,
10320
+ joinMessage(ACCESS_DENIED)
10321
+ );
10322
+ }
10323
+ const connection = this.connection ?? this.open();
10324
+ await this.connected(connection);
10325
+ await this.acknowledged(
10326
+ connection,
10327
+ JOIN_EVENT,
10328
+ { conversationId: normalized },
10329
+ this.options.joinTimeoutMs ?? JOIN_TIMEOUT_MS,
10330
+ normalized
10331
+ );
10332
+ }
10333
+ open() {
10334
+ const socketBaseUrl = (() => {
10335
+ try {
10336
+ return new URL(this.options.baseUrl).origin;
10337
+ } catch {
10338
+ return window.location.origin;
10339
+ }
10340
+ })();
10341
+ const socket = io(socketBaseUrl, {
10342
+ transports: ["websocket"],
10343
+ withCredentials: true,
10344
+ forceNew: true,
10345
+ auth: {
10346
+ apiToken: this.options.apiToken
10347
+ }
10348
+ });
10349
+ const connection = {
10350
+ socket,
10351
+ pending: /* @__PURE__ */ new Set(),
10352
+ connectedBefore: false,
10353
+ lost: null,
10354
+ closed: null,
10355
+ failure: null
10356
+ };
10357
+ this.connection = connection;
10358
+ this.watch(connection);
10359
+ this.receive(socket);
10360
+ return connection;
10361
+ }
10362
+ /** The lifecycle of one connection (docs/streaming_reply.md#web-sdk). */
10363
+ watch(connection) {
10364
+ const { socket } = connection;
10365
+ socket.on("connect", () => {
10366
+ if (!connection.connectedBefore) {
10367
+ connection.connectedBefore = true;
10368
+ return;
10369
+ }
10370
+ const activeConversationId = this.options.getConversationId()?.trim();
10371
+ if (activeConversationId && !this.refused.has(activeConversationId)) {
10372
+ socket.emit(JOIN_EVENT, { conversationId: activeConversationId });
10373
+ }
10374
+ });
10375
+ socket.on("exception", (payload) => {
10376
+ const conversationId = refusedConversation(payload);
10377
+ if (conversationId) {
10378
+ connection.failure ??= { code: ACCESS_DENIED, conversationId };
10379
+ this.refused.add(conversationId);
10380
+ }
10381
+ });
10382
+ socket.on("disconnect", (reason) => {
10383
+ if (!socket.active) {
10384
+ this.close(connection, reason);
10385
+ return;
10386
+ }
10387
+ connection.lost = reason;
10388
+ this.options.handleSpeechEvent?.({ type: "disconnect" });
10389
+ });
10390
+ socket.on("connect_error", (error) => {
10391
+ if (!socket.active) {
10392
+ this.close(connection, "connect_error", error.message);
10393
+ }
10394
+ });
10395
+ }
10396
+ /**
10397
+ * Closes the connection for good: socket.io does not open it again. The
10398
+ * waits fail now, and the next join opens a new connection.
10399
+ */
10400
+ close(connection, reason, detail) {
10401
+ if (connection.closed) {
10402
+ return;
10403
+ }
10404
+ connection.closed = { reason, message: closeMessage(reason, detail) };
10405
+ if (this.connection === connection) {
10406
+ this.connection = null;
10407
+ }
10408
+ const { socket } = connection;
10409
+ socket.removeAllListeners();
10410
+ if (connection.connectedBefore) {
10411
+ this.options.handleSpeechEvent?.({ type: "disconnect" });
10412
+ }
10413
+ const pending = [...connection.pending];
10414
+ connection.pending.clear();
10415
+ for (const entry of pending) {
10416
+ entry.fail(this.closeError(connection, entry.conversationId));
10417
+ }
10418
+ socket.disconnect();
10419
+ if (reason !== CLIENT_CLOSE) {
10420
+ this.options.handleConnectionClosed?.({
10421
+ reason,
10422
+ code: connection.failure?.code ?? null,
10423
+ conversationId: connection.failure?.conversationId ?? null
9913
10424
  });
9914
- this.socket.on(SPEECH_EVENTS.turn, (payload) => {
9915
- let candidate;
9916
- try {
9917
- candidate = decodePerformance(
9918
- payload,
9919
- "turn"
9920
- );
9921
- } catch (error) {
9922
- this.options.handleCallbackError("speech.turn.decode", error);
9923
- return;
10425
+ }
10426
+ }
10427
+ closeError(connection, conversationId) {
10428
+ if (conversationId !== null && this.refused.has(conversationId)) {
10429
+ return new ConversationConnectionError(
10430
+ ACCESS_DENIED,
10431
+ joinMessage(ACCESS_DENIED)
10432
+ );
10433
+ }
10434
+ return new ConversationConnectionError(
10435
+ "CONVERSATION_DISCONNECTED",
10436
+ connection.closed?.message ?? closeMessage(CLIENT_CLOSE),
10437
+ connection.closed?.reason ?? null
10438
+ );
10439
+ }
10440
+ /**
10441
+ * Waits for the first connection. A connection error that socket.io
10442
+ * recovers from also ends the wait: the join then waits in socket.io's
10443
+ * buffer, for the time of its acknowledgement at most.
10444
+ */
10445
+ connected(connection) {
10446
+ const { socket } = connection;
10447
+ if (connection.closed) {
10448
+ return Promise.reject(this.closeError(connection, null));
10449
+ }
10450
+ if (socket.connected || connection.connectedBefore) {
10451
+ return Promise.resolve();
10452
+ }
10453
+ return new Promise((resolve, reject) => {
10454
+ const entry = { conversationId: null, fail: reject };
10455
+ const done = () => {
10456
+ if (connection.pending.delete(entry)) {
10457
+ socket.off("connect", done);
10458
+ socket.off("connect_error", done);
10459
+ resolve();
9924
10460
  }
9925
- if (candidate?.conversationId !== this.options.getConversationId() || typeof candidate.requestId !== "string" || !candidate.turn || typeof candidate.turn !== "object") {
10461
+ };
10462
+ connection.pending.add(entry);
10463
+ socket.on("connect", done);
10464
+ socket.on("connect_error", done);
10465
+ });
10466
+ }
10467
+ /**
10468
+ * Emits an event and waits for its acknowledgement. A join (`joining`)
10469
+ * that the server does not accept fails with the code of its
10470
+ * acknowledgement. A close of the connection fails the wait at once; a
10471
+ * timeout fails it with `CONVERSATION_TIMEOUT`.
10472
+ */
10473
+ acknowledged(connection, event, payload, timeoutMs, joining) {
10474
+ return new Promise((resolve, reject) => {
10475
+ if (connection.closed) {
10476
+ reject(this.closeError(connection, joining));
10477
+ return;
10478
+ }
10479
+ const entry = { conversationId: joining, fail: reject };
10480
+ connection.pending.add(entry);
10481
+ connection.socket.timeout(timeoutMs).emit(event, payload, (error, result) => {
10482
+ if (!connection.pending.delete(entry)) {
9926
10483
  return;
9927
10484
  }
9928
- this.options.handleSpeechEvent?.({
9929
- type: "turn",
9930
- requestId: candidate.requestId,
9931
- turn: candidate.turn
9932
- });
9933
- });
9934
- this.socket.on(SPEECH_EVENTS.turnFailed, (payload) => {
9935
- const event = payload;
9936
- if (typeof event?.requestId === "string" && event.conversationId === this.options.getConversationId()) {
9937
- this.options.handleSpeechEvent?.({ type: "turnFailed", event });
10485
+ const failure = error ? acknowledgementError(error, connection.lost) : joining === null ? null : this.joinFailure(connection, joining, result);
10486
+ if (failure) {
10487
+ reject(failure);
10488
+ } else {
10489
+ resolve(result);
9938
10490
  }
9939
10491
  });
9940
- this.socket.on("disconnect", () => {
9941
- this.options.handleSpeechEvent?.({ type: "disconnect" });
9942
- });
9943
- this.socket.on("conversations:turn", (payload) => {
9944
- let candidate;
9945
- try {
9946
- candidate = decodePerformance(
9947
- payload,
9948
- "turn"
9949
- );
9950
- } catch (error) {
9951
- this.options.handleCallbackError("turn.decode", error);
9952
- return;
9953
- }
9954
- if (typeof candidate?.conversationId !== "string") {
9955
- return;
9956
- }
9957
- if (candidate.conversationId !== this.options.getConversationId()) {
9958
- return;
9959
- }
9960
- const turn = candidate.turn;
9961
- if (!turn || typeof turn !== "object") {
9962
- return;
9963
- }
9964
- void this.options.handleTurn(turn).catch((error) => {
9965
- this.options.handleCallbackError("onTurn", error);
9966
- });
10492
+ });
10493
+ }
10494
+ /** The error of a join that the server did not accept; null after a join. */
10495
+ joinFailure(connection, conversationId, result) {
10496
+ const ack = result;
10497
+ if (ack?.ok) {
10498
+ return null;
10499
+ }
10500
+ const named = typeof ack?.error === "string" ? ack.error.trim() : "";
10501
+ const code = named || JOIN_FAILED;
10502
+ connection.failure ??= { code, conversationId };
10503
+ if (code === ACCESS_DENIED) {
10504
+ this.refused.add(conversationId);
10505
+ }
10506
+ this.close(connection, "io server disconnect");
10507
+ return new ConversationConnectionError(code, joinMessage(code));
10508
+ }
10509
+ /** The events that the server sends to the conversation's room. */
10510
+ receive(socket) {
10511
+ socket.on("conversations:stream", (payload) => {
10512
+ let candidate;
10513
+ try {
10514
+ candidate = decodePerformance(
10515
+ payload,
10516
+ "stream"
10517
+ );
10518
+ } catch (error) {
10519
+ this.options.handleCallbackError("stream.decode", error);
10520
+ return;
10521
+ }
10522
+ if (typeof candidate?.conversationId !== "string") {
10523
+ return;
10524
+ }
10525
+ if (candidate.conversationId !== this.options.getConversationId()) {
10526
+ return;
10527
+ }
10528
+ const event = candidate.event;
10529
+ if (!event || typeof event !== "object") {
10530
+ return;
10531
+ }
10532
+ this.options.handleStreamEvent(event);
10533
+ });
10534
+ socket.on(SPEECH_EVENTS.partial, (payload) => {
10535
+ const event = payload;
10536
+ if (typeof event?.sessionId === "string" && typeof event.text === "string") {
10537
+ this.options.handleSpeechEvent?.({ type: "partial", event });
10538
+ }
10539
+ });
10540
+ socket.on(SPEECH_EVENTS.turn, (payload) => {
10541
+ let candidate;
10542
+ try {
10543
+ candidate = decodePerformance(
10544
+ payload,
10545
+ "turn"
10546
+ );
10547
+ } catch (error) {
10548
+ this.options.handleCallbackError("speech.turn.decode", error);
10549
+ return;
10550
+ }
10551
+ if (candidate?.conversationId !== this.options.getConversationId() || typeof candidate.requestId !== "string" || !candidate.turn || typeof candidate.turn !== "object") {
10552
+ return;
10553
+ }
10554
+ this.options.handleSpeechEvent?.({
10555
+ type: "turn",
10556
+ requestId: candidate.requestId,
10557
+ turn: candidate.turn
9967
10558
  });
9968
- await new Promise((resolve) => {
9969
- if (this.socket?.connected) {
9970
- resolve();
9971
- return;
9972
- }
9973
- this.socket?.once("connect", () => resolve());
9974
- this.socket?.once("connect_error", () => resolve());
10559
+ });
10560
+ socket.on(SPEECH_EVENTS.turnFailed, (payload) => {
10561
+ const event = payload;
10562
+ if (typeof event?.requestId === "string" && event.conversationId === this.options.getConversationId()) {
10563
+ this.options.handleSpeechEvent?.({ type: "turnFailed", event });
10564
+ }
10565
+ });
10566
+ socket.on("conversations:turn", (payload) => {
10567
+ let candidate;
10568
+ try {
10569
+ candidate = decodePerformance(
10570
+ payload,
10571
+ "turn"
10572
+ );
10573
+ } catch (error) {
10574
+ this.options.handleCallbackError("turn.decode", error);
10575
+ return;
10576
+ }
10577
+ if (typeof candidate?.conversationId !== "string") {
10578
+ return;
10579
+ }
10580
+ if (candidate.conversationId !== this.options.getConversationId()) {
10581
+ return;
10582
+ }
10583
+ const turn = candidate.turn;
10584
+ if (!turn || typeof turn !== "object") {
10585
+ return;
10586
+ }
10587
+ void this.options.handleTurn(turn).catch((error) => {
10588
+ this.options.handleCallbackError("onTurn", error);
9975
10589
  });
9976
- }
9977
- await new Promise((resolve, reject) => {
9978
- this.socket.timeout(1e4).emit(
9979
- "conversations:join",
9980
- { conversationId: normalized },
9981
- (error, result) => {
9982
- if (error || !result?.ok) {
9983
- reject(
9984
- error ?? new Error("Conversation streaming access was denied.")
9985
- );
9986
- } else {
9987
- resolve();
9988
- }
9989
- }
9990
- );
9991
10590
  });
9992
10591
  }
9993
10592
  };
@@ -10944,6 +11543,8 @@ var Character = class extends CharacterRuntime {
10944
11543
  notifiedStagingIds = /* @__PURE__ */ new Map();
10945
11544
  liveConversation;
10946
11545
  baked = null;
11546
+ /** The conversation whose live motion was warmed up. */
11547
+ liveMotionPrepared = null;
10947
11548
  authoring;
10948
11549
  normalizeIdleOrientation;
10949
11550
  bakedAsset;
@@ -10951,9 +11552,12 @@ var Character = class extends CharacterRuntime {
10951
11552
  voice;
10952
11553
  configuredInputModes;
10953
11554
  resolvedInputModes = null;
11555
+ /** The character's voice input switch, after the character loads. */
11556
+ speechInput = null;
10954
11557
  latestOptions = [];
10955
11558
  listeningStateCallback = null;
10956
11559
  transcriptCallback = null;
11560
+ connectionClosedCallback = null;
10957
11561
  conversationId = null;
10958
11562
  constructor(options) {
10959
11563
  const resolvedToken = (options.apiToken ?? options.apiKey)?.trim() ?? "";
@@ -10978,7 +11582,8 @@ var Character = class extends CharacterRuntime {
10978
11582
  if (this.configuredInputModes && this.bakedAsset && !this.authoring) {
10979
11583
  resolveInputModes(this.configuredInputModes, {
10980
11584
  allowCustomReply: false,
10981
- baked: true
11585
+ baked: true,
11586
+ speechInput: false
10982
11587
  });
10983
11588
  }
10984
11589
  this.conversationSocket = new ConversationSocket({
@@ -10995,7 +11600,12 @@ var Character = class extends CharacterRuntime {
10995
11600
  await this.handleStagingResult(turn, { source: "turn" });
10996
11601
  },
10997
11602
  handleCallbackError: this.handleCallbackError,
10998
- handleSpeechEvent: (event) => this.voice.handleSocketEvent(event)
11603
+ handleSpeechEvent: (event) => this.voice.handleSocketEvent(event),
11604
+ handleConnectionClosed: (event) => invokeSafeCallback(
11605
+ "onConnectionClosed",
11606
+ this.handleCallbackError,
11607
+ () => this.connectionClosedCallback?.(event)
11608
+ )
10999
11609
  });
11000
11610
  this.voice = new VoiceInput({
11001
11611
  socket: () => this.conversationSocket,
@@ -11065,9 +11675,11 @@ var Character = class extends CharacterRuntime {
11065
11675
  const progressReporter = createAssetProgressReporter(callback);
11066
11676
  const info = await this.getAssetInfo();
11067
11677
  const baked = !this.authoring && Boolean(this.bakedAsset || info.bakedConversationEnabled);
11678
+ this.speechInput = info.speechInputEnabled === true;
11068
11679
  this.resolvedInputModes = resolveInputModes(this.configuredInputModes, {
11069
11680
  allowCustomReply: info.allowCustomReply !== false,
11070
- baked
11681
+ baked,
11682
+ speechInput: this.speechInput
11071
11683
  });
11072
11684
  this.baked = baked ? await loadBakedConversation({
11073
11685
  baseUrl: this.baseUrl,
@@ -11138,12 +11750,33 @@ var Character = class extends CharacterRuntime {
11138
11750
  if (info.bakedConversationEnabled || this.bakedAsset) {
11139
11751
  throw new Error("Baked conversation requires local rendering.");
11140
11752
  }
11753
+ this.speechInput = info.speechInputEnabled === true;
11141
11754
  this.resolvedInputModes = resolveInputModes(this.configuredInputModes, {
11142
11755
  allowCustomReply: info.allowCustomReply !== false,
11143
- baked: false
11756
+ baked: false,
11757
+ speechInput: this.speechInput
11144
11758
  });
11145
11759
  this.animationController = new RemoteAnimationController();
11146
11760
  }
11761
+ /**
11762
+ * Warms up live motion for the current conversation, once, on the
11763
+ * player's intent: call it at the first focus or key in a reply field or
11764
+ * when a talk key goes down. It does nothing for a character without live
11765
+ * motion; the server limits warm-ups per player (ADR 0011).
11766
+ */
11767
+ async prepareLiveMotion() {
11768
+ const conversationId = this.conversationId;
11769
+ if (this.baked || !conversationId || this.liveMotionPrepared === conversationId) {
11770
+ return false;
11771
+ }
11772
+ this.liveMotionPrepared = conversationId;
11773
+ return warmUpLiveMotion({
11774
+ conversationId,
11775
+ baseUrl: this.baseUrl,
11776
+ apiToken: this.apiToken,
11777
+ renderSessionId: this.renderSessionId
11778
+ });
11779
+ }
11147
11780
  async createConversation(options) {
11148
11781
  const response = this.baked ? {
11149
11782
  conversation: {
@@ -11268,7 +11901,11 @@ var Character = class extends CharacterRuntime {
11268
11901
  * character permits. Before `loadAssets`, the configuration or all modes.
11269
11902
  */
11270
11903
  get inputModes() {
11271
- return this.resolvedInputModes ?? this.configuredInputModes ?? permittedInputModes({ allowCustomReply: true, baked: false });
11904
+ return this.resolvedInputModes ?? this.configuredInputModes ?? permittedInputModes({
11905
+ allowCustomReply: true,
11906
+ baked: false,
11907
+ speechInput: true
11908
+ });
11272
11909
  }
11273
11910
  get listeningState() {
11274
11911
  return this.voice.listeningState;
@@ -11312,6 +11949,19 @@ var Character = class extends CharacterRuntime {
11312
11949
  callback
11313
11950
  );
11314
11951
  }
11952
+ /**
11953
+ * The server closed the conversation connection: a refused or failed join,
11954
+ * or an API key that was rotated, edited or deleted. The next `connect`,
11955
+ * `sendMessage` or `startListening` opens a new connection; a refused
11956
+ * conversation fails at once with `CONVERSATION_ACCESS_DENIED`.
11957
+ */
11958
+ onConnectionClosed(callback) {
11959
+ return bindCallbackSlot(
11960
+ () => this.connectionClosedCallback,
11961
+ (next) => this.connectionClosedCallback = next,
11962
+ callback
11963
+ );
11964
+ }
11315
11965
  async getConversationWorldline(conversationId = this.conversationId) {
11316
11966
  if (!conversationId) {
11317
11967
  throw new Error("Conversation ID is required.");
@@ -11409,6 +12059,11 @@ var Character = class extends CharacterRuntime {
11409
12059
  "Voice input is not available in a baked (offline) conversation."
11410
12060
  );
11411
12061
  }
12062
+ if (mode === "voice" && this.speechInput === false) {
12063
+ throw new InputModeError(
12064
+ "Voice input is off for this character. Its owner turns it on in the STT tab."
12065
+ );
12066
+ }
11412
12067
  const modes = this.inputModes;
11413
12068
  if (!modes.includes(mode)) {
11414
12069
  throw new InputModeError(
@@ -12036,10 +12691,12 @@ export {
12036
12691
  CharacterExpressions,
12037
12692
  CharacterRuntime,
12038
12693
  CharacterSceneRig,
12694
+ ConversationConnectionError,
12039
12695
  FacialPerformanceLayer,
12040
12696
  FacialPerformanceMixer,
12041
12697
  INPUT_MODES,
12042
12698
  InputModeError,
12699
+ LiveMotionPlayer,
12043
12700
  MicrophoneError,
12044
12701
  Npcify,
12045
12702
  Character as NpcifyCharacter,