@npcify/web 0.4.0 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,3 +1,36 @@
1
+ // ../ts-shared/src/motion-frames.ts
2
+ var MOTION_FRAMES_CAPABILITY = "motion_frames";
3
+ var motionFrameWidth = function(boneCount) {
4
+ return 3 + 4 * boneCount;
5
+ };
6
+ var MAX_MOTION_FRAMES_PER_EVENT = 3600;
7
+ var MAX_MOTION_BONES = 64;
8
+ var isCount = function(value, max2) {
9
+ return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 && value <= max2;
10
+ };
11
+ var readMotionFrames = function(data, frameCount, boneCount) {
12
+ const values = frameCount * motionFrameWidth(boneCount);
13
+ if (data.byteLength !== values * 4) {
14
+ throw new Error("Motion frames do not match their frame count.");
15
+ }
16
+ const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
17
+ const frames = new Float32Array(values);
18
+ for (let index = 0; index < values; index += 1) {
19
+ frames[index] = view.getFloat32(index * 4, true);
20
+ }
21
+ return frames;
22
+ };
23
+ var isValidMotionFramesEvent = function(event) {
24
+ if (!event || typeof event !== "object") {
25
+ return false;
26
+ }
27
+ const value = event;
28
+ const bones = value.bones;
29
+ const frameCount = value.frameCount;
30
+ const frames = value.frames;
31
+ return value.type === "motion_frames" && typeof value.taskId === "string" && typeof value.stagingId === "string" && isCount(value.revision, Number.MAX_SAFE_INTEGER) && typeof value.fps === "number" && value.fps > 0 && value.fps <= 240 && isCount(value.startFrame, 24 * 60 * 60 * 240) && isCount(frameCount, MAX_MOTION_FRAMES_PER_EVENT) && Array.isArray(bones) && bones.length > 0 && bones.length <= MAX_MOTION_BONES && bones.every((bone) => typeof bone === "string") && typeof value.restHipsHeight === "number" && value.restHipsHeight > 0 && Number.isFinite(value.restHipsHeight) && frames instanceof Uint8Array && frames.byteLength === frameCount * motionFrameWidth(bones.length) * 4;
32
+ };
33
+
1
34
  // ../ts-shared/src/llm.ts
2
35
  var STAGING_PROTOCOL_VERSION = 1;
3
36
 
@@ -2602,6 +2635,8 @@ var PERFORMANCE_KINDS = {
2602
2635
  storedTimeline: 7,
2603
2636
  cachedOpening: 8
2604
2637
  };
2638
+ var PERFORMANCE_MAJOR_VERSION = 1;
2639
+ var PERFORMANCE_MINOR_VERSION = 1;
2605
2640
  var HEADER_SIZE = 24;
2606
2641
  var MAGIC = 1346588750;
2607
2642
  var crcTable = Uint32Array.from({ length: 256 }, (_, index) => {
@@ -2673,8 +2708,8 @@ var encodePerformance = function(kind, value, options = {}) {
2673
2708
  const bytes = new Uint8Array(HEADER_SIZE + body.length);
2674
2709
  const header = new DataView(bytes.buffer);
2675
2710
  header.setUint32(0, MAGIC, true);
2676
- header.setUint8(4, 1);
2677
- header.setUint8(5, 0);
2711
+ header.setUint8(4, PERFORMANCE_MAJOR_VERSION);
2712
+ header.setUint8(5, PERFORMANCE_MINOR_VERSION);
2678
2713
  header.setUint8(6, PERFORMANCE_KINDS[kind]);
2679
2714
  header.setUint8(7, options.compress ? 1 : 0);
2680
2715
  header.setUint16(8, HEADER_SIZE, true);
@@ -2711,7 +2746,7 @@ var decodePerformance = function(input, expectedKind) {
2711
2746
  throw new Error("Truncated performance header.");
2712
2747
  }
2713
2748
  const header = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
2714
- if (header.getUint32(0, true) !== MAGIC || header.getUint8(4) !== 1) {
2749
+ if (header.getUint32(0, true) !== MAGIC || header.getUint8(4) !== PERFORMANCE_MAJOR_VERSION) {
2715
2750
  throw new Error("Unsupported performance format or major version.");
2716
2751
  }
2717
2752
  if (header.getUint8(6) !== PERFORMANCE_KINDS[expectedKind]) {
@@ -3166,7 +3201,8 @@ var createConversation = async function({
3166
3201
  ),
3167
3202
  credentials: apiToken ? "omit" : "include",
3168
3203
  method: "POST",
3169
- body: JSON.stringify({ avatarId })
3204
+ // This SDK plays live motion frames (ADR 0011).
3205
+ body: JSON.stringify({ avatarId, capabilities: [MOTION_FRAMES_CAPABILITY] })
3170
3206
  });
3171
3207
  if (!res.ok) {
3172
3208
  throw Object.assign(new Error(await res.text()), { status: res.status });
@@ -3219,7 +3255,8 @@ var createStaging = async function({
3219
3255
  content: message,
3220
3256
  silent: silent === true,
3221
3257
  incrementalPlayback: true,
3222
- parentStagingId
3258
+ parentStagingId,
3259
+ capabilities: [MOTION_FRAMES_CAPABILITY]
3223
3260
  })
3224
3261
  });
3225
3262
  } catch (error) {
@@ -3319,6 +3356,35 @@ var getConversationWorldline = async function(options) {
3319
3356
  }
3320
3357
  return await response.json();
3321
3358
  };
3359
+ var warmUpLiveMotion = async function({
3360
+ conversationId,
3361
+ apiToken,
3362
+ baseUrl,
3363
+ renderSessionId
3364
+ }) {
3365
+ const url = new URL(`/conversations/${conversationId}/motion/warm-up`, baseUrl);
3366
+ try {
3367
+ const res = await fetch(url.toString(), {
3368
+ headers: withRenderSession(
3369
+ buildAuthHeaders(apiToken, {
3370
+ json: true,
3371
+ allowEmpty: true
3372
+ }),
3373
+ renderSessionId
3374
+ ),
3375
+ credentials: apiToken ? "omit" : "include",
3376
+ method: "POST",
3377
+ body: JSON.stringify({ capabilities: [MOTION_FRAMES_CAPABILITY] })
3378
+ });
3379
+ if (!res.ok) {
3380
+ return false;
3381
+ }
3382
+ const payload = await res.json();
3383
+ return payload.warming === true;
3384
+ } catch {
3385
+ return false;
3386
+ }
3387
+ };
3322
3388
 
3323
3389
  // ../../clients/web-client/src/live-conversation.ts
3324
3390
  var LiveConversation = class {
@@ -6045,6 +6111,10 @@ var StreamPlaybackController = class {
6045
6111
  get cursor() {
6046
6112
  return this.currentCursorValue;
6047
6113
  }
6114
+ /** The staging that plays or waits to play, or null. */
6115
+ get activeStagingId() {
6116
+ return this.activePlayback?.stagingId ?? null;
6117
+ }
6048
6118
  onStreamEvent(callback) {
6049
6119
  return bindCallbackSlot(
6050
6120
  () => this.streamEventCallback,
@@ -6138,6 +6208,10 @@ var StreamPlaybackController = class {
6138
6208
  this.dependencies.telemetry.recordFirstTextDelta();
6139
6209
  return;
6140
6210
  }
6211
+ if (event.type === "motion_frames") {
6212
+ this.dependencies.onMotionFrames?.(event);
6213
+ return;
6214
+ }
6141
6215
  const player = this.dependencies.getStagingPlayer();
6142
6216
  if (event.type === "speech_chunk") {
6143
6217
  player.appendSpeechChunk(event.streamId, event.mimeType, event.data);
@@ -6251,6 +6325,9 @@ var StreamPlaybackController = class {
6251
6325
  this.activePlayback = { taskId: params.taskId, stagingId: params.stagingId };
6252
6326
  this.playedStagingIds.add(params.stagingId);
6253
6327
  this.recordPlaybackEvent(params, "requested");
6328
+ if (params.timeline.motion) {
6329
+ this.dependencies.onMotionTrack?.(params.stagingId, params.timeline.motion);
6330
+ }
6254
6331
  const existing = this.states.get(params.stagingId);
6255
6332
  const state = {
6256
6333
  taskId: params.taskId,
@@ -8260,6 +8337,261 @@ var CharacterSceneRig = class {
8260
8337
  }
8261
8338
  };
8262
8339
 
8340
+ // ../../clients/web-client/src/character/live-motion.ts
8341
+ import { Quaternion as Quaternion3, Vector3 as Vector37 } from "three";
8342
+ var PARENTS = {
8343
+ spine: "hips",
8344
+ chest: "spine",
8345
+ upperChest: "chest",
8346
+ neck: "upperChest",
8347
+ head: "neck",
8348
+ leftShoulder: "upperChest",
8349
+ leftUpperArm: "leftShoulder",
8350
+ leftLowerArm: "leftUpperArm",
8351
+ leftHand: "leftLowerArm",
8352
+ rightShoulder: "upperChest",
8353
+ rightUpperArm: "rightShoulder",
8354
+ rightLowerArm: "rightUpperArm",
8355
+ rightHand: "rightLowerArm",
8356
+ leftUpperLeg: "hips",
8357
+ leftLowerLeg: "leftUpperLeg",
8358
+ leftFoot: "leftLowerLeg",
8359
+ leftToes: "leftFoot",
8360
+ rightUpperLeg: "hips",
8361
+ rightLowerLeg: "rightUpperLeg",
8362
+ rightFoot: "rightLowerLeg",
8363
+ rightToes: "rightFoot"
8364
+ };
8365
+ var MAX_TRACKS = 4;
8366
+ var approach = function(value, target, delta, seconds) {
8367
+ if (seconds <= 0) {
8368
+ return target;
8369
+ }
8370
+ const step = delta / seconds;
8371
+ return value < target ? Math.min(target, value + step) : Math.max(target, value - step);
8372
+ };
8373
+ var LiveMotionPlayer = class {
8374
+ vrm = null;
8375
+ tracks = /* @__PURE__ */ new Map();
8376
+ rootMode;
8377
+ blendIn;
8378
+ blendOut;
8379
+ weight = 0;
8380
+ /** The last applied pose, to blend out of it after the frames end. */
8381
+ lastPose = null;
8382
+ scratch = {
8383
+ from: new Quaternion3(),
8384
+ to: new Quaternion3(),
8385
+ carry: /* @__PURE__ */ new Map(),
8386
+ position: new Vector37()
8387
+ };
8388
+ constructor(options = {}) {
8389
+ this.rootMode = options.rootMode ?? "anchored";
8390
+ this.blendIn = options.blendInSeconds ?? 0.3;
8391
+ this.blendOut = options.blendOutSeconds ?? 0.5;
8392
+ }
8393
+ setVrm(vrm) {
8394
+ this.vrm = vrm;
8395
+ this.weight = 0;
8396
+ this.lastPose = null;
8397
+ }
8398
+ /** Adds the frames of a `motion_frames` event; invalid events are dropped. */
8399
+ push(received) {
8400
+ const frames = received.frames;
8401
+ const event = frames instanceof ArrayBuffer ? { ...received, frames: new Uint8Array(frames) } : received;
8402
+ if (!isValidMotionFramesEvent(event)) {
8403
+ return false;
8404
+ }
8405
+ const values = readMotionFrames(
8406
+ event.frames,
8407
+ event.frameCount,
8408
+ event.bones.length
8409
+ );
8410
+ const track = this.track(event.stagingId, {
8411
+ fps: event.fps,
8412
+ bones: event.bones,
8413
+ restHipsHeight: event.restHipsHeight
8414
+ });
8415
+ for (let frame = 0; frame < event.frameCount; frame += 1) {
8416
+ const index = event.startFrame + frame;
8417
+ const existing = track.frames.get(index);
8418
+ if (existing && existing.revision > event.revision) {
8419
+ continue;
8420
+ }
8421
+ track.frames.set(index, {
8422
+ revision: event.revision,
8423
+ values: values.subarray(frame * track.width, (frame + 1) * track.width)
8424
+ });
8425
+ }
8426
+ return true;
8427
+ }
8428
+ /** Loads the stored frames of a staging (a replay). */
8429
+ async load(stagingId, track, fetchBytes = async (url) => new Uint8Array(await (await fetch(url)).arrayBuffer())) {
8430
+ if (this.tracks.has(stagingId)) {
8431
+ return;
8432
+ }
8433
+ const data = track.frames.type === "bytes" ? track.frames.data : await fetchBytes(track.frames.url);
8434
+ const values = readMotionFrames(data, track.frameCount, track.bones.length);
8435
+ const stored = this.track(stagingId, track);
8436
+ for (let frame = 0; frame < track.frameCount; frame += 1) {
8437
+ stored.frames.set(frame, {
8438
+ revision: Number.MAX_SAFE_INTEGER,
8439
+ values: values.subarray(
8440
+ frame * stored.width,
8441
+ (frame + 1) * stored.width
8442
+ )
8443
+ });
8444
+ }
8445
+ }
8446
+ has(stagingId) {
8447
+ return (this.tracks.get(stagingId)?.frames.size ?? 0) > 0;
8448
+ }
8449
+ forget(stagingId) {
8450
+ this.tracks.delete(stagingId);
8451
+ }
8452
+ /** The current live weight, 0 (idle) to 1 (live frames). */
8453
+ get liveWeight() {
8454
+ return this.weight;
8455
+ }
8456
+ /**
8457
+ * Applies the live pose of `stagingId` at `time` seconds of its staging
8458
+ * clock; `time` null means that no staging plays now.
8459
+ */
8460
+ update(delta, stagingId, time) {
8461
+ const vrm = this.vrm;
8462
+ if (!vrm?.humanoid) {
8463
+ return;
8464
+ }
8465
+ const track = stagingId ? this.tracks.get(stagingId) : void 0;
8466
+ let pose = null;
8467
+ if (track && time !== null && time >= 0) {
8468
+ pose = this.sample(track, time);
8469
+ }
8470
+ this.weight = approach(
8471
+ this.weight,
8472
+ pose ? 1 : 0,
8473
+ Math.max(0, delta),
8474
+ pose ? this.blendIn : this.blendOut
8475
+ );
8476
+ if (pose) {
8477
+ this.lastPose = pose;
8478
+ }
8479
+ const applied = pose ?? this.lastPose;
8480
+ if (!applied || this.weight <= 0) {
8481
+ if (this.weight <= 0) {
8482
+ this.lastPose = null;
8483
+ }
8484
+ return;
8485
+ }
8486
+ this.apply(vrm, applied.track, applied.values, this.weight);
8487
+ }
8488
+ track(stagingId, meta) {
8489
+ let track = this.tracks.get(stagingId);
8490
+ if (!track) {
8491
+ track = {
8492
+ fps: meta.fps,
8493
+ bones: [...meta.bones],
8494
+ restHipsHeight: meta.restHipsHeight,
8495
+ width: motionFrameWidth(meta.bones.length),
8496
+ frames: /* @__PURE__ */ new Map(),
8497
+ origin: null
8498
+ };
8499
+ this.tracks.set(stagingId, track);
8500
+ while (this.tracks.size > MAX_TRACKS) {
8501
+ const oldest = this.tracks.keys().next().value;
8502
+ if (oldest === void 0 || oldest === stagingId) {
8503
+ break;
8504
+ }
8505
+ this.tracks.delete(oldest);
8506
+ }
8507
+ }
8508
+ return track;
8509
+ }
8510
+ /** The interpolated frame at `time`, or null when its frames are missing. */
8511
+ sample(track, time) {
8512
+ const position = time * track.fps;
8513
+ const first = Math.floor(position);
8514
+ const a = track.frames.get(first);
8515
+ if (!a) {
8516
+ return null;
8517
+ }
8518
+ track.origin ??= { x: a.values[0] ?? 0, z: a.values[2] ?? 0 };
8519
+ const b = track.frames.get(first + 1);
8520
+ const amount = position - first;
8521
+ if (!b || amount <= 0) {
8522
+ return { track, values: a.values };
8523
+ }
8524
+ const values = new Float32Array(track.width);
8525
+ for (let index = 0; index < 3; index += 1) {
8526
+ values[index] = (a.values[index] ?? 0) * (1 - amount) + (b.values[index] ?? 0) * amount;
8527
+ }
8528
+ const from = this.scratch.from;
8529
+ const to = this.scratch.to;
8530
+ for (let offset = 3; offset < track.width; offset += 4) {
8531
+ from.fromArray(a.values, offset);
8532
+ to.fromArray(b.values, offset);
8533
+ from.slerp(to, amount);
8534
+ values[offset] = from.x;
8535
+ values[offset + 1] = from.y;
8536
+ values[offset + 2] = from.z;
8537
+ values[offset + 3] = from.w;
8538
+ }
8539
+ return { track, values };
8540
+ }
8541
+ apply(vrm, track, values, weight) {
8542
+ const humanoid = vrm.humanoid;
8543
+ const flip = vrm.meta.metaVersion === "0" && !isNpcifyNormalizedVrm(vrm);
8544
+ const carry = this.scratch.carry;
8545
+ carry.clear();
8546
+ track.bones.forEach((bone, slot) => {
8547
+ const offset = 3 + slot * 4;
8548
+ const rotation = new Quaternion3(
8549
+ values[offset] ?? 0,
8550
+ values[offset + 1] ?? 0,
8551
+ values[offset + 2] ?? 0,
8552
+ values[offset + 3] ?? 1
8553
+ );
8554
+ if (flip) {
8555
+ rotation.x = -rotation.x;
8556
+ rotation.z = -rotation.z;
8557
+ }
8558
+ const inherited = carry.get(bone);
8559
+ if (inherited) {
8560
+ rotation.premultiply(inherited);
8561
+ }
8562
+ const node = humanoid.getNormalizedBoneNode(bone);
8563
+ if (!node) {
8564
+ for (const [child, parent] of Object.entries(PARENTS)) {
8565
+ if (parent === bone) {
8566
+ carry.set(child, rotation.clone());
8567
+ }
8568
+ }
8569
+ return;
8570
+ }
8571
+ node.quaternion.slerp(rotation, weight);
8572
+ });
8573
+ const hips = humanoid.getNormalizedBoneNode("hips");
8574
+ const rest = humanoid.normalizedRestPose.hips?.position;
8575
+ if (!hips || !rest) {
8576
+ return;
8577
+ }
8578
+ const scale = (rest[1] ?? 1) / (track.restHipsHeight || 1);
8579
+ let x = values[0] ?? 0;
8580
+ const y = values[1] ?? 0;
8581
+ let z = values[2] ?? 0;
8582
+ if (this.rootMode === "anchored" && track.origin) {
8583
+ x -= track.origin.x;
8584
+ z -= track.origin.z;
8585
+ }
8586
+ const target = this.scratch.position.set(
8587
+ (flip ? -x : x) * scale + (this.rootMode === "anchored" ? rest[0] ?? 0 : 0),
8588
+ y * scale,
8589
+ (flip ? -z : z) * scale + (this.rootMode === "anchored" ? rest[2] ?? 0 : 0)
8590
+ );
8591
+ hips.position.lerp(target, weight);
8592
+ }
8593
+ };
8594
+
8263
8595
  // ../../clients/web-client/src/character/incremental-staging.ts
8264
8596
  var dialogueKey = function(block) {
8265
8597
  return JSON.stringify(block, (key, value) => {
@@ -8534,9 +8866,17 @@ var StagingPlayer = class {
8534
8866
  activeExpression = null;
8535
8867
  activeExpressionMouthMode = null;
8536
8868
  activeAnimation = null;
8869
+ currentTrackTime = null;
8537
8870
  get paused() {
8538
8871
  return this.clock.paused;
8539
8872
  }
8873
+ /**
8874
+ * Seconds of the staging clock that the tracks show now, or null when no
8875
+ * staging plays (also while playback waits for the next dialogue block).
8876
+ */
8877
+ get trackTime() {
8878
+ return this.currentTrackTime;
8879
+ }
8540
8880
  pause() {
8541
8881
  this.pauseRevision += 1;
8542
8882
  this.clock.pause();
@@ -8937,6 +9277,7 @@ var StagingPlayer = class {
8937
9277
  }
8938
9278
  }
8939
9279
  applyTracks(time, speechActive, timeline) {
9280
+ this.currentTrackTime = time;
8940
9281
  const expression = activeClipAt(timeline.expressions, time);
8941
9282
  const mouthMode = speechActive ? "viseme" : "expression";
8942
9283
  if (expression !== this.activeExpression || expression && mouthMode !== this.activeExpressionMouthMode) {
@@ -8970,6 +9311,7 @@ var StagingPlayer = class {
8970
9311
  }
8971
9312
  }
8972
9313
  clearPerformance() {
9314
+ this.currentTrackTime = null;
8973
9315
  this.activeExpression = null;
8974
9316
  this.activeExpressionMouthMode = null;
8975
9317
  this.activeAnimation = null;
@@ -9408,6 +9750,7 @@ var CharacterRuntime = class {
9408
9750
  performanceRecorder;
9409
9751
  telemetry;
9410
9752
  streamPlayback;
9753
+ liveMotion;
9411
9754
  defaultParent;
9412
9755
  currentGazeTarget = null;
9413
9756
  /** The current gaze target came from `lookAt()`. */
@@ -9456,12 +9799,21 @@ var CharacterRuntime = class {
9456
9799
  baseUrl: this.baseUrl,
9457
9800
  apiToken: this.apiToken
9458
9801
  });
9802
+ this.liveMotion = new LiveMotionPlayer(options.liveMotion);
9459
9803
  this.streamPlayback = new StreamPlaybackController({
9460
9804
  getStagingPlayer: this.getStagingPlayer,
9461
9805
  telemetry: this.telemetry,
9462
9806
  handleCallbackError: this.handleCallbackError,
9463
9807
  reportError: this.reportError,
9464
- recordEvent: (event) => this.performanceRecorder.recordEvent(event)
9808
+ recordEvent: (event) => this.performanceRecorder.recordEvent(event),
9809
+ onMotionFrames: (event) => {
9810
+ this.liveMotion.push(event);
9811
+ },
9812
+ onMotionTrack: (stagingId, track) => {
9813
+ void this.liveMotion.load(stagingId, track).catch((error) => {
9814
+ this.handleCallbackError("liveMotion", error);
9815
+ });
9816
+ }
9465
9817
  });
9466
9818
  if (options.gazeTarget !== void 0) {
9467
9819
  this.setGazeTarget(options.gazeTarget);
@@ -9498,6 +9850,7 @@ var CharacterRuntime = class {
9498
9850
  set vrm(value) {
9499
9851
  this.sceneRig.setVrm(value);
9500
9852
  this.facialMixer.setVrm(value);
9853
+ this.liveMotion.setVrm(value);
9501
9854
  this.applyGaze();
9502
9855
  if (value && this.defaultParent && !value.scene?.parent) {
9503
9856
  this.sceneRig.attachTo(this.defaultParent);
@@ -9774,6 +10127,11 @@ var CharacterRuntime = class {
9774
10127
  return;
9775
10128
  }
9776
10129
  this.animationController?.update(delta);
10130
+ this.liveMotion.update(
10131
+ delta,
10132
+ this.streamPlayback.activeStagingId,
10133
+ this.stagingPlayer?.trackTime ?? null
10134
+ );
9777
10135
  this.visemePlayer.update(delta);
9778
10136
  this.facialMixer.update(delta);
9779
10137
  this.vrm?.update(delta);
@@ -9830,6 +10188,7 @@ var CharacterRuntime = class {
9830
10188
  this.animations.clear();
9831
10189
  this.sceneRig.setVrm(null);
9832
10190
  this.facialMixer.setVrm(null);
10191
+ this.liveMotion.setVrm(null);
9833
10192
  }
9834
10193
  disposeRuntimeAssets() {
9835
10194
  this.sceneRig.disposeVrm();
@@ -9852,28 +10211,101 @@ var CharacterRuntime = class {
9852
10211
 
9853
10212
  // ../../clients/web-client/src/conversation-socket.ts
9854
10213
  import { io } from "socket.io-client";
10214
+ var ConversationConnectionError = class extends Error {
10215
+ constructor(code, message, reason = null) {
10216
+ super(message);
10217
+ this.code = code;
10218
+ this.reason = reason;
10219
+ this.name = "NpcifyConversationConnectionError";
10220
+ }
10221
+ };
10222
+ var JOIN_EVENT = "conversations:join";
10223
+ var JOIN_TIMEOUT_MS = 1e4;
10224
+ var ACCESS_DENIED = "CONVERSATION_ACCESS_DENIED";
10225
+ var JOIN_FAILED = "CONVERSATION_JOIN_FAILED";
10226
+ var CLIENT_CLOSE = "io client disconnect";
10227
+ var SOCKET_IO_TIMEOUT = "operation has timed out";
10228
+ var joinMessage = function(code) {
10229
+ if (code === ACCESS_DENIED) {
10230
+ return "Conversation streaming access was denied.";
10231
+ }
10232
+ if (code === JOIN_FAILED) {
10233
+ return "The server could not join the conversation. Try again later.";
10234
+ }
10235
+ return `The server did not join the conversation (${code}). Try again later.`;
10236
+ };
10237
+ var closeMessage = function(reason, detail) {
10238
+ if (reason === "io server disconnect") {
10239
+ return "The server closed the conversation connection.";
10240
+ }
10241
+ if (reason === CLIENT_CLOSE) {
10242
+ return "The conversation connection closed.";
10243
+ }
10244
+ if (reason === "connect_error") {
10245
+ return `The server refused the conversation connection (${detail || "no reason"}).`;
10246
+ }
10247
+ return `The conversation connection was lost (${reason}).`;
10248
+ };
10249
+ var acknowledgementError = function(error, lost) {
10250
+ if (error.message === SOCKET_IO_TIMEOUT) {
10251
+ return new ConversationConnectionError(
10252
+ "CONVERSATION_TIMEOUT",
10253
+ "The server did not answer in time."
10254
+ );
10255
+ }
10256
+ return new ConversationConnectionError(
10257
+ "CONVERSATION_DISCONNECTED",
10258
+ closeMessage(lost ?? "transport close"),
10259
+ lost
10260
+ );
10261
+ };
10262
+ var refusedConversation = function(payload) {
10263
+ const exception = payload;
10264
+ const named = exception?.cause?.data?.conversationId;
10265
+ if (exception?.message !== ACCESS_DENIED || typeof named !== "string") {
10266
+ return null;
10267
+ }
10268
+ return named.trim() || null;
10269
+ };
9855
10270
  var ConversationSocket = class {
9856
10271
  options;
9857
- socket = null;
10272
+ connection = null;
10273
+ /**
10274
+ * The conversations that the server refused. A join with the same
10275
+ * credentials fails the same way, so the SDK does not try them again until
10276
+ * `disconnect`.
10277
+ */
10278
+ refused = /* @__PURE__ */ new Set();
9858
10279
  constructor(options) {
9859
10280
  this.options = options;
9860
10281
  }
10282
+ /** Closes the connection and forgets the refused conversations. */
9861
10283
  disconnect() {
9862
- this.socket?.disconnect();
9863
- this.socket = null;
10284
+ this.refused.clear();
10285
+ if (this.connection) {
10286
+ this.close(this.connection, CLIENT_CLOSE);
10287
+ }
9864
10288
  }
9865
10289
  /** An event with an acknowledgement. Needs `ensureConnected` first. */
9866
10290
  async request(event, payload, timeoutMs = 1e4) {
9867
- const socket = this.socket;
9868
- if (!socket?.connected) {
9869
- throw new Error("The conversation connection is not open.");
10291
+ const connection = this.connection;
10292
+ if (!connection?.socket.connected) {
10293
+ throw new ConversationConnectionError(
10294
+ "CONVERSATION_DISCONNECTED",
10295
+ "The conversation connection is not open."
10296
+ );
9870
10297
  }
9871
- return await socket.timeout(timeoutMs).emitWithAck(event, payload);
10298
+ return this.acknowledged(connection, event, payload, timeoutMs, null);
9872
10299
  }
9873
10300
  /** An event without an acknowledgement, in order with the other events. */
9874
10301
  send(event, payload) {
9875
- this.socket?.emit(event, payload);
10302
+ this.connection?.socket.emit(event, payload);
9876
10303
  }
10304
+ /**
10305
+ * Joins the room of the conversation. A connection that the server closed
10306
+ * is not used again: the join opens a new one. A refused conversation fails
10307
+ * at once, without a connection.
10308
+ */
9877
10309
  async ensureConnected(conversationId) {
9878
10310
  if (typeof window === "undefined") {
9879
10311
  return;
@@ -9882,135 +10314,279 @@ var ConversationSocket = class {
9882
10314
  if (!normalized) {
9883
10315
  return;
9884
10316
  }
9885
- if (!this.socket) {
9886
- const socketBaseUrl = (() => {
9887
- try {
9888
- return new URL(this.options.baseUrl).origin;
9889
- } catch {
9890
- return window.location.origin;
9891
- }
9892
- })();
9893
- this.socket = io(socketBaseUrl, {
9894
- transports: ["websocket"],
9895
- withCredentials: true,
9896
- auth: {
9897
- apiToken: this.options.apiToken
9898
- }
9899
- });
9900
- this.socket.on("connect", () => {
9901
- const activeConversationId = this.options.getConversationId()?.trim();
9902
- if (activeConversationId) {
9903
- this.socket?.emit("conversations:join", {
9904
- conversationId: activeConversationId
9905
- });
9906
- }
9907
- });
9908
- this.socket.on("conversations:stream", (payload) => {
9909
- let candidate;
9910
- try {
9911
- candidate = decodePerformance(
9912
- payload,
9913
- "stream"
9914
- );
9915
- } catch (error) {
9916
- this.options.handleCallbackError("stream.decode", error);
9917
- return;
9918
- }
9919
- if (typeof candidate?.conversationId !== "string") {
9920
- return;
9921
- }
9922
- if (candidate.conversationId !== this.options.getConversationId()) {
9923
- return;
9924
- }
9925
- const event = candidate.event;
9926
- if (!event || typeof event !== "object") {
9927
- return;
9928
- }
9929
- this.options.handleStreamEvent(event);
9930
- });
9931
- this.socket.on(SPEECH_EVENTS.partial, (payload) => {
9932
- const event = payload;
9933
- if (typeof event?.sessionId === "string" && typeof event.text === "string") {
9934
- this.options.handleSpeechEvent?.({ type: "partial", event });
9935
- }
10317
+ if (this.refused.has(normalized)) {
10318
+ throw new ConversationConnectionError(
10319
+ ACCESS_DENIED,
10320
+ joinMessage(ACCESS_DENIED)
10321
+ );
10322
+ }
10323
+ const connection = this.connection ?? this.open();
10324
+ await this.connected(connection);
10325
+ await this.acknowledged(
10326
+ connection,
10327
+ JOIN_EVENT,
10328
+ { conversationId: normalized },
10329
+ this.options.joinTimeoutMs ?? JOIN_TIMEOUT_MS,
10330
+ normalized
10331
+ );
10332
+ }
10333
+ open() {
10334
+ const socketBaseUrl = (() => {
10335
+ try {
10336
+ return new URL(this.options.baseUrl).origin;
10337
+ } catch {
10338
+ return window.location.origin;
10339
+ }
10340
+ })();
10341
+ const socket = io(socketBaseUrl, {
10342
+ transports: ["websocket"],
10343
+ withCredentials: true,
10344
+ forceNew: true,
10345
+ auth: {
10346
+ apiToken: this.options.apiToken
10347
+ }
10348
+ });
10349
+ const connection = {
10350
+ socket,
10351
+ pending: /* @__PURE__ */ new Set(),
10352
+ connectedBefore: false,
10353
+ lost: null,
10354
+ closed: null,
10355
+ failure: null
10356
+ };
10357
+ this.connection = connection;
10358
+ this.watch(connection);
10359
+ this.receive(socket);
10360
+ return connection;
10361
+ }
10362
+ /** The lifecycle of one connection (docs/streaming_reply.md#web-sdk). */
10363
+ watch(connection) {
10364
+ const { socket } = connection;
10365
+ socket.on("connect", () => {
10366
+ if (!connection.connectedBefore) {
10367
+ connection.connectedBefore = true;
10368
+ return;
10369
+ }
10370
+ const activeConversationId = this.options.getConversationId()?.trim();
10371
+ if (activeConversationId && !this.refused.has(activeConversationId)) {
10372
+ socket.emit(JOIN_EVENT, { conversationId: activeConversationId });
10373
+ }
10374
+ });
10375
+ socket.on("exception", (payload) => {
10376
+ const conversationId = refusedConversation(payload);
10377
+ if (conversationId) {
10378
+ connection.failure ??= { code: ACCESS_DENIED, conversationId };
10379
+ this.refused.add(conversationId);
10380
+ }
10381
+ });
10382
+ socket.on("disconnect", (reason) => {
10383
+ if (!socket.active) {
10384
+ this.close(connection, reason);
10385
+ return;
10386
+ }
10387
+ connection.lost = reason;
10388
+ this.options.handleSpeechEvent?.({ type: "disconnect" });
10389
+ });
10390
+ socket.on("connect_error", (error) => {
10391
+ if (!socket.active) {
10392
+ this.close(connection, "connect_error", error.message);
10393
+ }
10394
+ });
10395
+ }
10396
+ /**
10397
+ * Closes the connection for good: socket.io does not open it again. The
10398
+ * waits fail now, and the next join opens a new connection.
10399
+ */
10400
+ close(connection, reason, detail) {
10401
+ if (connection.closed) {
10402
+ return;
10403
+ }
10404
+ connection.closed = { reason, message: closeMessage(reason, detail) };
10405
+ if (this.connection === connection) {
10406
+ this.connection = null;
10407
+ }
10408
+ const { socket } = connection;
10409
+ socket.removeAllListeners();
10410
+ if (connection.connectedBefore) {
10411
+ this.options.handleSpeechEvent?.({ type: "disconnect" });
10412
+ }
10413
+ const pending = [...connection.pending];
10414
+ connection.pending.clear();
10415
+ for (const entry of pending) {
10416
+ entry.fail(this.closeError(connection, entry.conversationId));
10417
+ }
10418
+ socket.disconnect();
10419
+ if (reason !== CLIENT_CLOSE) {
10420
+ this.options.handleConnectionClosed?.({
10421
+ reason,
10422
+ code: connection.failure?.code ?? null,
10423
+ conversationId: connection.failure?.conversationId ?? null
9936
10424
  });
9937
- this.socket.on(SPEECH_EVENTS.turn, (payload) => {
9938
- let candidate;
9939
- try {
9940
- candidate = decodePerformance(
9941
- payload,
9942
- "turn"
9943
- );
9944
- } catch (error) {
9945
- this.options.handleCallbackError("speech.turn.decode", error);
9946
- return;
10425
+ }
10426
+ }
10427
+ closeError(connection, conversationId) {
10428
+ if (conversationId !== null && this.refused.has(conversationId)) {
10429
+ return new ConversationConnectionError(
10430
+ ACCESS_DENIED,
10431
+ joinMessage(ACCESS_DENIED)
10432
+ );
10433
+ }
10434
+ return new ConversationConnectionError(
10435
+ "CONVERSATION_DISCONNECTED",
10436
+ connection.closed?.message ?? closeMessage(CLIENT_CLOSE),
10437
+ connection.closed?.reason ?? null
10438
+ );
10439
+ }
10440
+ /**
10441
+ * Waits for the first connection. A connection error that socket.io
10442
+ * recovers from also ends the wait: the join then waits in socket.io's
10443
+ * buffer, for the time of its acknowledgement at most.
10444
+ */
10445
+ connected(connection) {
10446
+ const { socket } = connection;
10447
+ if (connection.closed) {
10448
+ return Promise.reject(this.closeError(connection, null));
10449
+ }
10450
+ if (socket.connected || connection.connectedBefore) {
10451
+ return Promise.resolve();
10452
+ }
10453
+ return new Promise((resolve, reject) => {
10454
+ const entry = { conversationId: null, fail: reject };
10455
+ const done = () => {
10456
+ if (connection.pending.delete(entry)) {
10457
+ socket.off("connect", done);
10458
+ socket.off("connect_error", done);
10459
+ resolve();
9947
10460
  }
9948
- if (candidate?.conversationId !== this.options.getConversationId() || typeof candidate.requestId !== "string" || !candidate.turn || typeof candidate.turn !== "object") {
10461
+ };
10462
+ connection.pending.add(entry);
10463
+ socket.on("connect", done);
10464
+ socket.on("connect_error", done);
10465
+ });
10466
+ }
10467
+ /**
10468
+ * Emits an event and waits for its acknowledgement. A join (`joining`)
10469
+ * that the server does not accept fails with the code of its
10470
+ * acknowledgement. A close of the connection fails the wait at once; a
10471
+ * timeout fails it with `CONVERSATION_TIMEOUT`.
10472
+ */
10473
+ acknowledged(connection, event, payload, timeoutMs, joining) {
10474
+ return new Promise((resolve, reject) => {
10475
+ if (connection.closed) {
10476
+ reject(this.closeError(connection, joining));
10477
+ return;
10478
+ }
10479
+ const entry = { conversationId: joining, fail: reject };
10480
+ connection.pending.add(entry);
10481
+ connection.socket.timeout(timeoutMs).emit(event, payload, (error, result) => {
10482
+ if (!connection.pending.delete(entry)) {
9949
10483
  return;
9950
10484
  }
9951
- this.options.handleSpeechEvent?.({
9952
- type: "turn",
9953
- requestId: candidate.requestId,
9954
- turn: candidate.turn
9955
- });
9956
- });
9957
- this.socket.on(SPEECH_EVENTS.turnFailed, (payload) => {
9958
- const event = payload;
9959
- if (typeof event?.requestId === "string" && event.conversationId === this.options.getConversationId()) {
9960
- this.options.handleSpeechEvent?.({ type: "turnFailed", event });
10485
+ const failure = error ? acknowledgementError(error, connection.lost) : joining === null ? null : this.joinFailure(connection, joining, result);
10486
+ if (failure) {
10487
+ reject(failure);
10488
+ } else {
10489
+ resolve(result);
9961
10490
  }
9962
10491
  });
9963
- this.socket.on("disconnect", () => {
9964
- this.options.handleSpeechEvent?.({ type: "disconnect" });
9965
- });
9966
- this.socket.on("conversations:turn", (payload) => {
9967
- let candidate;
9968
- try {
9969
- candidate = decodePerformance(
9970
- payload,
9971
- "turn"
9972
- );
9973
- } catch (error) {
9974
- this.options.handleCallbackError("turn.decode", error);
9975
- return;
9976
- }
9977
- if (typeof candidate?.conversationId !== "string") {
9978
- return;
9979
- }
9980
- if (candidate.conversationId !== this.options.getConversationId()) {
9981
- return;
9982
- }
9983
- const turn = candidate.turn;
9984
- if (!turn || typeof turn !== "object") {
9985
- return;
9986
- }
9987
- void this.options.handleTurn(turn).catch((error) => {
9988
- this.options.handleCallbackError("onTurn", error);
9989
- });
10492
+ });
10493
+ }
10494
+ /** The error of a join that the server did not accept; null after a join. */
10495
+ joinFailure(connection, conversationId, result) {
10496
+ const ack = result;
10497
+ if (ack?.ok) {
10498
+ return null;
10499
+ }
10500
+ const named = typeof ack?.error === "string" ? ack.error.trim() : "";
10501
+ const code = named || JOIN_FAILED;
10502
+ connection.failure ??= { code, conversationId };
10503
+ if (code === ACCESS_DENIED) {
10504
+ this.refused.add(conversationId);
10505
+ }
10506
+ this.close(connection, "io server disconnect");
10507
+ return new ConversationConnectionError(code, joinMessage(code));
10508
+ }
10509
+ /** The events that the server sends to the conversation's room. */
10510
+ receive(socket) {
10511
+ socket.on("conversations:stream", (payload) => {
10512
+ let candidate;
10513
+ try {
10514
+ candidate = decodePerformance(
10515
+ payload,
10516
+ "stream"
10517
+ );
10518
+ } catch (error) {
10519
+ this.options.handleCallbackError("stream.decode", error);
10520
+ return;
10521
+ }
10522
+ if (typeof candidate?.conversationId !== "string") {
10523
+ return;
10524
+ }
10525
+ if (candidate.conversationId !== this.options.getConversationId()) {
10526
+ return;
10527
+ }
10528
+ const event = candidate.event;
10529
+ if (!event || typeof event !== "object") {
10530
+ return;
10531
+ }
10532
+ this.options.handleStreamEvent(event);
10533
+ });
10534
+ socket.on(SPEECH_EVENTS.partial, (payload) => {
10535
+ const event = payload;
10536
+ if (typeof event?.sessionId === "string" && typeof event.text === "string") {
10537
+ this.options.handleSpeechEvent?.({ type: "partial", event });
10538
+ }
10539
+ });
10540
+ socket.on(SPEECH_EVENTS.turn, (payload) => {
10541
+ let candidate;
10542
+ try {
10543
+ candidate = decodePerformance(
10544
+ payload,
10545
+ "turn"
10546
+ );
10547
+ } catch (error) {
10548
+ this.options.handleCallbackError("speech.turn.decode", error);
10549
+ return;
10550
+ }
10551
+ if (candidate?.conversationId !== this.options.getConversationId() || typeof candidate.requestId !== "string" || !candidate.turn || typeof candidate.turn !== "object") {
10552
+ return;
10553
+ }
10554
+ this.options.handleSpeechEvent?.({
10555
+ type: "turn",
10556
+ requestId: candidate.requestId,
10557
+ turn: candidate.turn
9990
10558
  });
9991
- await new Promise((resolve) => {
9992
- if (this.socket?.connected) {
9993
- resolve();
9994
- return;
9995
- }
9996
- this.socket?.once("connect", () => resolve());
9997
- this.socket?.once("connect_error", () => resolve());
10559
+ });
10560
+ socket.on(SPEECH_EVENTS.turnFailed, (payload) => {
10561
+ const event = payload;
10562
+ if (typeof event?.requestId === "string" && event.conversationId === this.options.getConversationId()) {
10563
+ this.options.handleSpeechEvent?.({ type: "turnFailed", event });
10564
+ }
10565
+ });
10566
+ socket.on("conversations:turn", (payload) => {
10567
+ let candidate;
10568
+ try {
10569
+ candidate = decodePerformance(
10570
+ payload,
10571
+ "turn"
10572
+ );
10573
+ } catch (error) {
10574
+ this.options.handleCallbackError("turn.decode", error);
10575
+ return;
10576
+ }
10577
+ if (typeof candidate?.conversationId !== "string") {
10578
+ return;
10579
+ }
10580
+ if (candidate.conversationId !== this.options.getConversationId()) {
10581
+ return;
10582
+ }
10583
+ const turn = candidate.turn;
10584
+ if (!turn || typeof turn !== "object") {
10585
+ return;
10586
+ }
10587
+ void this.options.handleTurn(turn).catch((error) => {
10588
+ this.options.handleCallbackError("onTurn", error);
9998
10589
  });
9999
- }
10000
- await new Promise((resolve, reject) => {
10001
- this.socket.timeout(1e4).emit(
10002
- "conversations:join",
10003
- { conversationId: normalized },
10004
- (error, result) => {
10005
- if (error || !result?.ok) {
10006
- reject(
10007
- error ?? new Error("Conversation streaming access was denied.")
10008
- );
10009
- } else {
10010
- resolve();
10011
- }
10012
- }
10013
- );
10014
10590
  });
10015
10591
  }
10016
10592
  };
@@ -10967,6 +11543,8 @@ var Character = class extends CharacterRuntime {
10967
11543
  notifiedStagingIds = /* @__PURE__ */ new Map();
10968
11544
  liveConversation;
10969
11545
  baked = null;
11546
+ /** The conversation whose live motion was warmed up. */
11547
+ liveMotionPrepared = null;
10970
11548
  authoring;
10971
11549
  normalizeIdleOrientation;
10972
11550
  bakedAsset;
@@ -10979,6 +11557,7 @@ var Character = class extends CharacterRuntime {
10979
11557
  latestOptions = [];
10980
11558
  listeningStateCallback = null;
10981
11559
  transcriptCallback = null;
11560
+ connectionClosedCallback = null;
10982
11561
  conversationId = null;
10983
11562
  constructor(options) {
10984
11563
  const resolvedToken = (options.apiToken ?? options.apiKey)?.trim() ?? "";
@@ -11021,7 +11600,12 @@ var Character = class extends CharacterRuntime {
11021
11600
  await this.handleStagingResult(turn, { source: "turn" });
11022
11601
  },
11023
11602
  handleCallbackError: this.handleCallbackError,
11024
- handleSpeechEvent: (event) => this.voice.handleSocketEvent(event)
11603
+ handleSpeechEvent: (event) => this.voice.handleSocketEvent(event),
11604
+ handleConnectionClosed: (event) => invokeSafeCallback(
11605
+ "onConnectionClosed",
11606
+ this.handleCallbackError,
11607
+ () => this.connectionClosedCallback?.(event)
11608
+ )
11025
11609
  });
11026
11610
  this.voice = new VoiceInput({
11027
11611
  socket: () => this.conversationSocket,
@@ -11174,6 +11758,25 @@ var Character = class extends CharacterRuntime {
11174
11758
  });
11175
11759
  this.animationController = new RemoteAnimationController();
11176
11760
  }
11761
+ /**
11762
+ * Warms up live motion for the current conversation, once, on the
11763
+ * player's intent: call it at the first focus or key in a reply field or
11764
+ * when a talk key goes down. It does nothing for a character without live
11765
+ * motion; the server limits warm-ups per player (ADR 0011).
11766
+ */
11767
+ async prepareLiveMotion() {
11768
+ const conversationId = this.conversationId;
11769
+ if (this.baked || !conversationId || this.liveMotionPrepared === conversationId) {
11770
+ return false;
11771
+ }
11772
+ this.liveMotionPrepared = conversationId;
11773
+ return warmUpLiveMotion({
11774
+ conversationId,
11775
+ baseUrl: this.baseUrl,
11776
+ apiToken: this.apiToken,
11777
+ renderSessionId: this.renderSessionId
11778
+ });
11779
+ }
11177
11780
  async createConversation(options) {
11178
11781
  const response = this.baked ? {
11179
11782
  conversation: {
@@ -11346,6 +11949,19 @@ var Character = class extends CharacterRuntime {
11346
11949
  callback
11347
11950
  );
11348
11951
  }
11952
+ /**
11953
+ * The server closed the conversation connection: a refused or failed join,
11954
+ * or an API key that was rotated, edited or deleted. The next `connect`,
11955
+ * `sendMessage` or `startListening` opens a new connection; a refused
11956
+ * conversation fails at once with `CONVERSATION_ACCESS_DENIED`.
11957
+ */
11958
+ onConnectionClosed(callback) {
11959
+ return bindCallbackSlot(
11960
+ () => this.connectionClosedCallback,
11961
+ (next) => this.connectionClosedCallback = next,
11962
+ callback
11963
+ );
11964
+ }
11349
11965
  async getConversationWorldline(conversationId = this.conversationId) {
11350
11966
  if (!conversationId) {
11351
11967
  throw new Error("Conversation ID is required.");
@@ -12075,10 +12691,12 @@ export {
12075
12691
  CharacterExpressions,
12076
12692
  CharacterRuntime,
12077
12693
  CharacterSceneRig,
12694
+ ConversationConnectionError,
12078
12695
  FacialPerformanceLayer,
12079
12696
  FacialPerformanceMixer,
12080
12697
  INPUT_MODES,
12081
12698
  InputModeError,
12699
+ LiveMotionPlayer,
12082
12700
  MicrophoneError,
12083
12701
  Npcify,
12084
12702
  Character as NpcifyCharacter,