@npcify/web 0.4.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,3 +1,36 @@
1
+ // ../ts-shared/src/motion-frames.ts
2
+ var MOTION_FRAMES_CAPABILITY = "motion_frames";
3
+ var motionFrameWidth = function(boneCount) {
4
+ return 3 + 4 * boneCount;
5
+ };
6
+ var MAX_MOTION_FRAMES_PER_EVENT = 3600;
7
+ var MAX_MOTION_BONES = 64;
8
+ var isCount = function(value, max2) {
9
+ return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 && value <= max2;
10
+ };
11
+ var readMotionFrames = function(data, frameCount, boneCount) {
12
+ const values = frameCount * motionFrameWidth(boneCount);
13
+ if (data.byteLength !== values * 4) {
14
+ throw new Error("Motion frames do not match their frame count.");
15
+ }
16
+ const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
17
+ const frames = new Float32Array(values);
18
+ for (let index = 0; index < values; index += 1) {
19
+ frames[index] = view.getFloat32(index * 4, true);
20
+ }
21
+ return frames;
22
+ };
23
+ var isValidMotionFramesEvent = function(event) {
24
+ if (!event || typeof event !== "object") {
25
+ return false;
26
+ }
27
+ const value = event;
28
+ const bones = value.bones;
29
+ const frameCount = value.frameCount;
30
+ const frames = value.frames;
31
+ return value.type === "motion_frames" && typeof value.taskId === "string" && typeof value.stagingId === "string" && isCount(value.revision, Number.MAX_SAFE_INTEGER) && typeof value.fps === "number" && value.fps > 0 && value.fps <= 240 && isCount(value.startFrame, 24 * 60 * 60 * 240) && isCount(frameCount, MAX_MOTION_FRAMES_PER_EVENT) && Array.isArray(bones) && bones.length > 0 && bones.length <= MAX_MOTION_BONES && bones.every((bone) => typeof bone === "string") && typeof value.restHipsHeight === "number" && value.restHipsHeight > 0 && Number.isFinite(value.restHipsHeight) && frames instanceof Uint8Array && frames.byteLength === frameCount * motionFrameWidth(bones.length) * 4;
32
+ };
33
+
1
34
  // ../ts-shared/src/llm.ts
2
35
  var STAGING_PROTOCOL_VERSION = 1;
3
36
 
@@ -55,6 +88,20 @@ var AI_ROLES = {
55
88
  models: ["gemini-3.8-flash-tts", "gemini-3.8-flash-lite-tts"],
56
89
  options: []
57
90
  },
91
+ // Sight (docs/sight.md): one role for each provider that an owner can
92
+ // choose. Priced by VISION_PRICES of the server.
93
+ vision_gemini: {
94
+ name: "Gemini sight",
95
+ source: "gemini",
96
+ models: ["gemini-3.5-flash-lite"],
97
+ options: []
98
+ },
99
+ vision_openai: {
100
+ name: "OpenAI sight",
101
+ source: "openai",
102
+ models: ["gpt-6-luna"],
103
+ options: []
104
+ },
58
105
  // Priced by OPENAI_PRICES.text of the generator.
59
106
  generator_agent: {
60
107
  name: "Generator agent",
@@ -209,6 +256,43 @@ var SPEECH_INPUT_PROVIDER_PROFILES = Object.freeze({
209
256
  }
210
257
  });
211
258
 
259
+ // ../ts-shared/src/sight.ts
260
+ var SIGHT_EVENTS = Object.freeze({
261
+ /** Client to server: one JPEG frame, acknowledged with a `SightFrameAck`. */
262
+ frame: "conversations:sight",
263
+ /** Server to the conversation's room: one `SightObservationEvent`. */
264
+ observation: "sight_observation"
265
+ });
266
+ var SIGHT_LIMITS = Object.freeze({
267
+ /** The largest JPEG frame, in bytes (200 KiB). */
268
+ maxFrameBytes: 200 * 1024,
269
+ /** The longest edge of a frame or a crop, in pixels. */
270
+ maxEdgePx: 1280,
271
+ /** The shortest time between two frames of one conversation. */
272
+ minFrameIntervalMs: 1e3,
273
+ maxCrops: 4,
274
+ /** The largest JPEG crop, in bytes (100 KiB). */
275
+ maxCropBytes: 100 * 1024,
276
+ maxCropLabelLength: 40
277
+ });
278
+ var SIGHT_CAPTURE_SIZE = Object.freeze({
279
+ width: 640,
280
+ height: 360,
281
+ jpegQuality: 0.7
282
+ });
283
+ var SIGHT_PROVIDER_PROFILES = Object.freeze({
284
+ gemini: {
285
+ model: "gemini-3.5-flash-lite",
286
+ frameToObservationMs: 1280,
287
+ usdPerFrame: 29e-5
288
+ },
289
+ openai: {
290
+ model: "gpt-6-luna",
291
+ frameToObservationMs: 2140,
292
+ usdPerFrame: 74e-6
293
+ }
294
+ });
295
+
212
296
  // ../ts-shared/src/product-usage.ts
213
297
  var productUsageEvents = [
214
298
  "page_viewed",
@@ -2602,6 +2686,8 @@ var PERFORMANCE_KINDS = {
2602
2686
  storedTimeline: 7,
2603
2687
  cachedOpening: 8
2604
2688
  };
2689
+ var PERFORMANCE_MAJOR_VERSION = 1;
2690
+ var PERFORMANCE_MINOR_VERSION = 1;
2605
2691
  var HEADER_SIZE = 24;
2606
2692
  var MAGIC = 1346588750;
2607
2693
  var crcTable = Uint32Array.from({ length: 256 }, (_, index) => {
@@ -2673,8 +2759,8 @@ var encodePerformance = function(kind, value, options = {}) {
2673
2759
  const bytes = new Uint8Array(HEADER_SIZE + body.length);
2674
2760
  const header = new DataView(bytes.buffer);
2675
2761
  header.setUint32(0, MAGIC, true);
2676
- header.setUint8(4, 1);
2677
- header.setUint8(5, 0);
2762
+ header.setUint8(4, PERFORMANCE_MAJOR_VERSION);
2763
+ header.setUint8(5, PERFORMANCE_MINOR_VERSION);
2678
2764
  header.setUint8(6, PERFORMANCE_KINDS[kind]);
2679
2765
  header.setUint8(7, options.compress ? 1 : 0);
2680
2766
  header.setUint16(8, HEADER_SIZE, true);
@@ -2711,7 +2797,7 @@ var decodePerformance = function(input, expectedKind) {
2711
2797
  throw new Error("Truncated performance header.");
2712
2798
  }
2713
2799
  const header = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
2714
- if (header.getUint32(0, true) !== MAGIC || header.getUint8(4) !== 1) {
2800
+ if (header.getUint32(0, true) !== MAGIC || header.getUint8(4) !== PERFORMANCE_MAJOR_VERSION) {
2715
2801
  throw new Error("Unsupported performance format or major version.");
2716
2802
  }
2717
2803
  if (header.getUint8(6) !== PERFORMANCE_KINDS[expectedKind]) {
@@ -3166,7 +3252,8 @@ var createConversation = async function({
3166
3252
  ),
3167
3253
  credentials: apiToken ? "omit" : "include",
3168
3254
  method: "POST",
3169
- body: JSON.stringify({ avatarId })
3255
+ // This SDK plays live motion frames (ADR 0011).
3256
+ body: JSON.stringify({ avatarId, capabilities: [MOTION_FRAMES_CAPABILITY] })
3170
3257
  });
3171
3258
  if (!res.ok) {
3172
3259
  throw Object.assign(new Error(await res.text()), { status: res.status });
@@ -3219,7 +3306,8 @@ var createStaging = async function({
3219
3306
  content: message,
3220
3307
  silent: silent === true,
3221
3308
  incrementalPlayback: true,
3222
- parentStagingId
3309
+ parentStagingId,
3310
+ capabilities: [MOTION_FRAMES_CAPABILITY]
3223
3311
  })
3224
3312
  });
3225
3313
  } catch (error) {
@@ -3319,6 +3407,35 @@ var getConversationWorldline = async function(options) {
3319
3407
  }
3320
3408
  return await response.json();
3321
3409
  };
3410
+ var warmUpLiveMotion = async function({
3411
+ conversationId,
3412
+ apiToken,
3413
+ baseUrl,
3414
+ renderSessionId
3415
+ }) {
3416
+ const url = new URL(`/conversations/${conversationId}/motion/warm-up`, baseUrl);
3417
+ try {
3418
+ const res = await fetch(url.toString(), {
3419
+ headers: withRenderSession(
3420
+ buildAuthHeaders(apiToken, {
3421
+ json: true,
3422
+ allowEmpty: true
3423
+ }),
3424
+ renderSessionId
3425
+ ),
3426
+ credentials: apiToken ? "omit" : "include",
3427
+ method: "POST",
3428
+ body: JSON.stringify({ capabilities: [MOTION_FRAMES_CAPABILITY] })
3429
+ });
3430
+ if (!res.ok) {
3431
+ return false;
3432
+ }
3433
+ const payload = await res.json();
3434
+ return payload.warming === true;
3435
+ } catch {
3436
+ return false;
3437
+ }
3438
+ };
3322
3439
 
3323
3440
  // ../../clients/web-client/src/live-conversation.ts
3324
3441
  var LiveConversation = class {
@@ -6045,6 +6162,10 @@ var StreamPlaybackController = class {
6045
6162
  get cursor() {
6046
6163
  return this.currentCursorValue;
6047
6164
  }
6165
+ /** The staging that plays or waits to play, or null. */
6166
+ get activeStagingId() {
6167
+ return this.activePlayback?.stagingId ?? null;
6168
+ }
6048
6169
  onStreamEvent(callback) {
6049
6170
  return bindCallbackSlot(
6050
6171
  () => this.streamEventCallback,
@@ -6138,6 +6259,10 @@ var StreamPlaybackController = class {
6138
6259
  this.dependencies.telemetry.recordFirstTextDelta();
6139
6260
  return;
6140
6261
  }
6262
+ if (event.type === "motion_frames") {
6263
+ this.dependencies.onMotionFrames?.(event);
6264
+ return;
6265
+ }
6141
6266
  const player = this.dependencies.getStagingPlayer();
6142
6267
  if (event.type === "speech_chunk") {
6143
6268
  player.appendSpeechChunk(event.streamId, event.mimeType, event.data);
@@ -6251,6 +6376,9 @@ var StreamPlaybackController = class {
6251
6376
  this.activePlayback = { taskId: params.taskId, stagingId: params.stagingId };
6252
6377
  this.playedStagingIds.add(params.stagingId);
6253
6378
  this.recordPlaybackEvent(params, "requested");
6379
+ if (params.timeline.motion) {
6380
+ this.dependencies.onMotionTrack?.(params.stagingId, params.timeline.motion);
6381
+ }
6254
6382
  const existing = this.states.get(params.stagingId);
6255
6383
  const state = {
6256
6384
  taskId: params.taskId,
@@ -8260,6 +8388,261 @@ var CharacterSceneRig = class {
8260
8388
  }
8261
8389
  };
8262
8390
 
8391
+ // ../../clients/web-client/src/character/live-motion.ts
8392
+ import { Quaternion as Quaternion3, Vector3 as Vector37 } from "three";
8393
+ var PARENTS = {
8394
+ spine: "hips",
8395
+ chest: "spine",
8396
+ upperChest: "chest",
8397
+ neck: "upperChest",
8398
+ head: "neck",
8399
+ leftShoulder: "upperChest",
8400
+ leftUpperArm: "leftShoulder",
8401
+ leftLowerArm: "leftUpperArm",
8402
+ leftHand: "leftLowerArm",
8403
+ rightShoulder: "upperChest",
8404
+ rightUpperArm: "rightShoulder",
8405
+ rightLowerArm: "rightUpperArm",
8406
+ rightHand: "rightLowerArm",
8407
+ leftUpperLeg: "hips",
8408
+ leftLowerLeg: "leftUpperLeg",
8409
+ leftFoot: "leftLowerLeg",
8410
+ leftToes: "leftFoot",
8411
+ rightUpperLeg: "hips",
8412
+ rightLowerLeg: "rightUpperLeg",
8413
+ rightFoot: "rightLowerLeg",
8414
+ rightToes: "rightFoot"
8415
+ };
8416
+ var MAX_TRACKS = 4;
8417
+ var approach = function(value, target, delta, seconds) {
8418
+ if (seconds <= 0) {
8419
+ return target;
8420
+ }
8421
+ const step = delta / seconds;
8422
+ return value < target ? Math.min(target, value + step) : Math.max(target, value - step);
8423
+ };
8424
+ var LiveMotionPlayer = class {
8425
+ vrm = null;
8426
+ tracks = /* @__PURE__ */ new Map();
8427
+ rootMode;
8428
+ blendIn;
8429
+ blendOut;
8430
+ weight = 0;
8431
+ /** The last applied pose, to blend out of it after the frames end. */
8432
+ lastPose = null;
8433
+ scratch = {
8434
+ from: new Quaternion3(),
8435
+ to: new Quaternion3(),
8436
+ carry: /* @__PURE__ */ new Map(),
8437
+ position: new Vector37()
8438
+ };
8439
+ constructor(options = {}) {
8440
+ this.rootMode = options.rootMode ?? "anchored";
8441
+ this.blendIn = options.blendInSeconds ?? 0.3;
8442
+ this.blendOut = options.blendOutSeconds ?? 0.5;
8443
+ }
8444
+ setVrm(vrm) {
8445
+ this.vrm = vrm;
8446
+ this.weight = 0;
8447
+ this.lastPose = null;
8448
+ }
8449
+ /** Adds the frames of a `motion_frames` event; invalid events are dropped. */
8450
+ push(received) {
8451
+ const frames = received.frames;
8452
+ const event = frames instanceof ArrayBuffer ? { ...received, frames: new Uint8Array(frames) } : received;
8453
+ if (!isValidMotionFramesEvent(event)) {
8454
+ return false;
8455
+ }
8456
+ const values = readMotionFrames(
8457
+ event.frames,
8458
+ event.frameCount,
8459
+ event.bones.length
8460
+ );
8461
+ const track = this.track(event.stagingId, {
8462
+ fps: event.fps,
8463
+ bones: event.bones,
8464
+ restHipsHeight: event.restHipsHeight
8465
+ });
8466
+ for (let frame = 0; frame < event.frameCount; frame += 1) {
8467
+ const index = event.startFrame + frame;
8468
+ const existing = track.frames.get(index);
8469
+ if (existing && existing.revision > event.revision) {
8470
+ continue;
8471
+ }
8472
+ track.frames.set(index, {
8473
+ revision: event.revision,
8474
+ values: values.subarray(frame * track.width, (frame + 1) * track.width)
8475
+ });
8476
+ }
8477
+ return true;
8478
+ }
8479
+ /** Loads the stored frames of a staging (a replay). */
8480
+ async load(stagingId, track, fetchBytes = async (url) => new Uint8Array(await (await fetch(url)).arrayBuffer())) {
8481
+ if (this.tracks.has(stagingId)) {
8482
+ return;
8483
+ }
8484
+ const data = track.frames.type === "bytes" ? track.frames.data : await fetchBytes(track.frames.url);
8485
+ const values = readMotionFrames(data, track.frameCount, track.bones.length);
8486
+ const stored = this.track(stagingId, track);
8487
+ for (let frame = 0; frame < track.frameCount; frame += 1) {
8488
+ stored.frames.set(frame, {
8489
+ revision: Number.MAX_SAFE_INTEGER,
8490
+ values: values.subarray(
8491
+ frame * stored.width,
8492
+ (frame + 1) * stored.width
8493
+ )
8494
+ });
8495
+ }
8496
+ }
8497
+ has(stagingId) {
8498
+ return (this.tracks.get(stagingId)?.frames.size ?? 0) > 0;
8499
+ }
8500
+ forget(stagingId) {
8501
+ this.tracks.delete(stagingId);
8502
+ }
8503
+ /** The current live weight, 0 (idle) to 1 (live frames). */
8504
+ get liveWeight() {
8505
+ return this.weight;
8506
+ }
8507
+ /**
8508
+ * Applies the live pose of `stagingId` at `time` seconds of its staging
8509
+ * clock; `time` null means that no staging plays now.
8510
+ */
8511
+ update(delta, stagingId, time) {
8512
+ const vrm = this.vrm;
8513
+ if (!vrm?.humanoid) {
8514
+ return;
8515
+ }
8516
+ const track = stagingId ? this.tracks.get(stagingId) : void 0;
8517
+ let pose = null;
8518
+ if (track && time !== null && time >= 0) {
8519
+ pose = this.sample(track, time);
8520
+ }
8521
+ this.weight = approach(
8522
+ this.weight,
8523
+ pose ? 1 : 0,
8524
+ Math.max(0, delta),
8525
+ pose ? this.blendIn : this.blendOut
8526
+ );
8527
+ if (pose) {
8528
+ this.lastPose = pose;
8529
+ }
8530
+ const applied = pose ?? this.lastPose;
8531
+ if (!applied || this.weight <= 0) {
8532
+ if (this.weight <= 0) {
8533
+ this.lastPose = null;
8534
+ }
8535
+ return;
8536
+ }
8537
+ this.apply(vrm, applied.track, applied.values, this.weight);
8538
+ }
8539
+ track(stagingId, meta) {
8540
+ let track = this.tracks.get(stagingId);
8541
+ if (!track) {
8542
+ track = {
8543
+ fps: meta.fps,
8544
+ bones: [...meta.bones],
8545
+ restHipsHeight: meta.restHipsHeight,
8546
+ width: motionFrameWidth(meta.bones.length),
8547
+ frames: /* @__PURE__ */ new Map(),
8548
+ origin: null
8549
+ };
8550
+ this.tracks.set(stagingId, track);
8551
+ while (this.tracks.size > MAX_TRACKS) {
8552
+ const oldest = this.tracks.keys().next().value;
8553
+ if (oldest === void 0 || oldest === stagingId) {
8554
+ break;
8555
+ }
8556
+ this.tracks.delete(oldest);
8557
+ }
8558
+ }
8559
+ return track;
8560
+ }
8561
+ /** The interpolated frame at `time`, or null when its frames are missing. */
8562
+ sample(track, time) {
8563
+ const position = time * track.fps;
8564
+ const first = Math.floor(position);
8565
+ const a = track.frames.get(first);
8566
+ if (!a) {
8567
+ return null;
8568
+ }
8569
+ track.origin ??= { x: a.values[0] ?? 0, z: a.values[2] ?? 0 };
8570
+ const b = track.frames.get(first + 1);
8571
+ const amount = position - first;
8572
+ if (!b || amount <= 0) {
8573
+ return { track, values: a.values };
8574
+ }
8575
+ const values = new Float32Array(track.width);
8576
+ for (let index = 0; index < 3; index += 1) {
8577
+ values[index] = (a.values[index] ?? 0) * (1 - amount) + (b.values[index] ?? 0) * amount;
8578
+ }
8579
+ const from = this.scratch.from;
8580
+ const to = this.scratch.to;
8581
+ for (let offset = 3; offset < track.width; offset += 4) {
8582
+ from.fromArray(a.values, offset);
8583
+ to.fromArray(b.values, offset);
8584
+ from.slerp(to, amount);
8585
+ values[offset] = from.x;
8586
+ values[offset + 1] = from.y;
8587
+ values[offset + 2] = from.z;
8588
+ values[offset + 3] = from.w;
8589
+ }
8590
+ return { track, values };
8591
+ }
8592
+ apply(vrm, track, values, weight) {
8593
+ const humanoid = vrm.humanoid;
8594
+ const flip = vrm.meta.metaVersion === "0" && !isNpcifyNormalizedVrm(vrm);
8595
+ const carry = this.scratch.carry;
8596
+ carry.clear();
8597
+ track.bones.forEach((bone, slot) => {
8598
+ const offset = 3 + slot * 4;
8599
+ const rotation = new Quaternion3(
8600
+ values[offset] ?? 0,
8601
+ values[offset + 1] ?? 0,
8602
+ values[offset + 2] ?? 0,
8603
+ values[offset + 3] ?? 1
8604
+ );
8605
+ if (flip) {
8606
+ rotation.x = -rotation.x;
8607
+ rotation.z = -rotation.z;
8608
+ }
8609
+ const inherited = carry.get(bone);
8610
+ if (inherited) {
8611
+ rotation.premultiply(inherited);
8612
+ }
8613
+ const node = humanoid.getNormalizedBoneNode(bone);
8614
+ if (!node) {
8615
+ for (const [child, parent] of Object.entries(PARENTS)) {
8616
+ if (parent === bone) {
8617
+ carry.set(child, rotation.clone());
8618
+ }
8619
+ }
8620
+ return;
8621
+ }
8622
+ node.quaternion.slerp(rotation, weight);
8623
+ });
8624
+ const hips = humanoid.getNormalizedBoneNode("hips");
8625
+ const rest = humanoid.normalizedRestPose.hips?.position;
8626
+ if (!hips || !rest) {
8627
+ return;
8628
+ }
8629
+ const scale = (rest[1] ?? 1) / (track.restHipsHeight || 1);
8630
+ let x = values[0] ?? 0;
8631
+ const y = values[1] ?? 0;
8632
+ let z = values[2] ?? 0;
8633
+ if (this.rootMode === "anchored" && track.origin) {
8634
+ x -= track.origin.x;
8635
+ z -= track.origin.z;
8636
+ }
8637
+ const target = this.scratch.position.set(
8638
+ (flip ? -x : x) * scale + (this.rootMode === "anchored" ? rest[0] ?? 0 : 0),
8639
+ y * scale,
8640
+ (flip ? -z : z) * scale + (this.rootMode === "anchored" ? rest[2] ?? 0 : 0)
8641
+ );
8642
+ hips.position.lerp(target, weight);
8643
+ }
8644
+ };
8645
+
8263
8646
  // ../../clients/web-client/src/character/incremental-staging.ts
8264
8647
  var dialogueKey = function(block) {
8265
8648
  return JSON.stringify(block, (key, value) => {
@@ -8534,9 +8917,17 @@ var StagingPlayer = class {
8534
8917
  activeExpression = null;
8535
8918
  activeExpressionMouthMode = null;
8536
8919
  activeAnimation = null;
8920
+ currentTrackTime = null;
8537
8921
  get paused() {
8538
8922
  return this.clock.paused;
8539
8923
  }
8924
+ /**
8925
+ * Seconds of the staging clock that the tracks show now, or null when no
8926
+ * staging plays (also while playback waits for the next dialogue block).
8927
+ */
8928
+ get trackTime() {
8929
+ return this.currentTrackTime;
8930
+ }
8540
8931
  pause() {
8541
8932
  this.pauseRevision += 1;
8542
8933
  this.clock.pause();
@@ -8937,6 +9328,7 @@ var StagingPlayer = class {
8937
9328
  }
8938
9329
  }
8939
9330
  applyTracks(time, speechActive, timeline) {
9331
+ this.currentTrackTime = time;
8940
9332
  const expression = activeClipAt(timeline.expressions, time);
8941
9333
  const mouthMode = speechActive ? "viseme" : "expression";
8942
9334
  if (expression !== this.activeExpression || expression && mouthMode !== this.activeExpressionMouthMode) {
@@ -8970,6 +9362,7 @@ var StagingPlayer = class {
8970
9362
  }
8971
9363
  }
8972
9364
  clearPerformance() {
9365
+ this.currentTrackTime = null;
8973
9366
  this.activeExpression = null;
8974
9367
  this.activeExpressionMouthMode = null;
8975
9368
  this.activeAnimation = null;
@@ -9408,6 +9801,7 @@ var CharacterRuntime = class {
9408
9801
  performanceRecorder;
9409
9802
  telemetry;
9410
9803
  streamPlayback;
9804
+ liveMotion;
9411
9805
  defaultParent;
9412
9806
  currentGazeTarget = null;
9413
9807
  /** The current gaze target came from `lookAt()`. */
@@ -9456,12 +9850,21 @@ var CharacterRuntime = class {
9456
9850
  baseUrl: this.baseUrl,
9457
9851
  apiToken: this.apiToken
9458
9852
  });
9853
+ this.liveMotion = new LiveMotionPlayer(options.liveMotion);
9459
9854
  this.streamPlayback = new StreamPlaybackController({
9460
9855
  getStagingPlayer: this.getStagingPlayer,
9461
9856
  telemetry: this.telemetry,
9462
9857
  handleCallbackError: this.handleCallbackError,
9463
9858
  reportError: this.reportError,
9464
- recordEvent: (event) => this.performanceRecorder.recordEvent(event)
9859
+ recordEvent: (event) => this.performanceRecorder.recordEvent(event),
9860
+ onMotionFrames: (event) => {
9861
+ this.liveMotion.push(event);
9862
+ },
9863
+ onMotionTrack: (stagingId, track) => {
9864
+ void this.liveMotion.load(stagingId, track).catch((error) => {
9865
+ this.handleCallbackError("liveMotion", error);
9866
+ });
9867
+ }
9465
9868
  });
9466
9869
  if (options.gazeTarget !== void 0) {
9467
9870
  this.setGazeTarget(options.gazeTarget);
@@ -9498,6 +9901,7 @@ var CharacterRuntime = class {
9498
9901
  set vrm(value) {
9499
9902
  this.sceneRig.setVrm(value);
9500
9903
  this.facialMixer.setVrm(value);
9904
+ this.liveMotion.setVrm(value);
9501
9905
  this.applyGaze();
9502
9906
  if (value && this.defaultParent && !value.scene?.parent) {
9503
9907
  this.sceneRig.attachTo(this.defaultParent);
@@ -9774,6 +10178,11 @@ var CharacterRuntime = class {
9774
10178
  return;
9775
10179
  }
9776
10180
  this.animationController?.update(delta);
10181
+ this.liveMotion.update(
10182
+ delta,
10183
+ this.streamPlayback.activeStagingId,
10184
+ this.stagingPlayer?.trackTime ?? null
10185
+ );
9777
10186
  this.visemePlayer.update(delta);
9778
10187
  this.facialMixer.update(delta);
9779
10188
  this.vrm?.update(delta);
@@ -9830,6 +10239,7 @@ var CharacterRuntime = class {
9830
10239
  this.animations.clear();
9831
10240
  this.sceneRig.setVrm(null);
9832
10241
  this.facialMixer.setVrm(null);
10242
+ this.liveMotion.setVrm(null);
9833
10243
  }
9834
10244
  disposeRuntimeAssets() {
9835
10245
  this.sceneRig.disposeVrm();
@@ -9852,184 +10262,1012 @@ var CharacterRuntime = class {
9852
10262
 
9853
10263
  // ../../clients/web-client/src/conversation-socket.ts
9854
10264
  import { io } from "socket.io-client";
9855
- var ConversationSocket = class {
9856
- options;
9857
- socket = null;
9858
- constructor(options) {
9859
- this.options = options;
10265
+
10266
+ // ../../clients/web-client/src/sight/sight-error.ts
10267
+ var SightError = class extends Error {
10268
+ constructor(code, message) {
10269
+ super(message ?? code);
10270
+ this.code = code;
10271
+ this.name = "NpcifySightError";
9860
10272
  }
9861
- disconnect() {
9862
- this.socket?.disconnect();
9863
- this.socket = null;
10273
+ };
10274
+ var SIGHT_STOP_CODES = /* @__PURE__ */ new Set([
10275
+ "SIGHT_DISABLED",
10276
+ "INSUFFICIENT_POINTS",
10277
+ "CONVERSATION_ACCESS_DENIED"
10278
+ ]);
10279
+ var SIGHT_OFF_MESSAGE = "Sight is off for this character. Its owner turns it on in the Sight tab.";
10280
+ var STOP_MESSAGES = {
10281
+ SIGHT_DISABLED: SIGHT_OFF_MESSAGE,
10282
+ INSUFFICIENT_POINTS: "The owner of the character has no points to pay for sight.",
10283
+ CONVERSATION_ACCESS_DENIED: "This connection has no access to the conversation."
10284
+ };
10285
+ var sightStopError = function(code) {
10286
+ return new SightError(code, STOP_MESSAGES[code] ?? code);
10287
+ };
10288
+ var toSightError = function(error, fallback) {
10289
+ if (error instanceof SightError) {
10290
+ return error;
9864
10291
  }
9865
- /** An event with an acknowledgement. Needs `ensureConnected` first. */
9866
- async request(event, payload, timeoutMs = 1e4) {
9867
- const socket = this.socket;
9868
- if (!socket?.connected) {
9869
- throw new Error("The conversation connection is not open.");
10292
+ const code = error?.code;
10293
+ return new SightError(
10294
+ typeof code === "string" && code ? code : fallback,
10295
+ error instanceof Error ? error.message : void 0
10296
+ );
10297
+ };
10298
+
10299
+ // ../../clients/web-client/src/sight/frame-source.ts
10300
+ var LUMA_GRID = Object.freeze({ width: 32, height: 18 });
10301
+ var SMALLER_JPEG_QUALITY = 0.5;
10302
+ var HAVE_CURRENT_DATA = 2;
10303
+ var fitCaptureSize = function(width, height, limit = SIGHT_CAPTURE_SIZE) {
10304
+ const scale = Math.min(1, limit.width / width, limit.height / height);
10305
+ return {
10306
+ width: Math.max(1, Math.round(width * scale)),
10307
+ height: Math.max(1, Math.round(height * scale))
10308
+ };
10309
+ };
10310
+ var lumaOf = function(rgba) {
10311
+ const luma = new Uint8Array(Math.floor(rgba.length / 4));
10312
+ for (let index = 0; index < luma.length; index += 1) {
10313
+ const offset = index * 4;
10314
+ luma[index] = Math.round(
10315
+ 0.299 * (rgba[offset] ?? 0) + 0.587 * (rgba[offset + 1] ?? 0) + 0.114 * (rgba[offset + 2] ?? 0)
10316
+ );
10317
+ }
10318
+ return luma;
10319
+ };
10320
+ var meanLumaChange = function(previous, next) {
10321
+ const length = Math.min(previous.length, next.length);
10322
+ if (length === 0) {
10323
+ return 0;
10324
+ }
10325
+ let sum = 0;
10326
+ for (let index = 0; index < length; index += 1) {
10327
+ sum += Math.abs((next[index] ?? 0) - (previous[index] ?? 0));
10328
+ }
10329
+ return sum / length;
10330
+ };
10331
+ var encodeJpeg = function(canvas, quality) {
10332
+ return new Promise((resolve) => {
10333
+ canvas.toBlob(resolve, "image/jpeg", quality);
10334
+ });
10335
+ };
10336
+ var VideoFrameSource = class {
10337
+ constructor(stream, owned) {
10338
+ this.stream = stream;
10339
+ this.owned = owned;
10340
+ const settings = stream.getVideoTracks()[0]?.getSettings?.();
10341
+ this.kind = settings?.displaySurface ? "screen" : "camera";
10342
+ const video = document.createElement("video");
10343
+ video.muted = true;
10344
+ video.playsInline = true;
10345
+ video.setAttribute("playsinline", "");
10346
+ video.srcObject = stream;
10347
+ void video.play().catch(() => void 0);
10348
+ this.video = video;
10349
+ this.frameCanvas = document.createElement("canvas");
10350
+ const grid = document.createElement("canvas");
10351
+ grid.width = LUMA_GRID.width;
10352
+ grid.height = LUMA_GRID.height;
10353
+ this.gridContext = grid.getContext("2d", { willReadFrequently: true });
10354
+ if (this.gridContext) {
10355
+ this.gridContext.imageSmoothingQuality = "high";
10356
+ }
10357
+ }
10358
+ kind;
10359
+ video;
10360
+ frameCanvas;
10361
+ frameContext = null;
10362
+ gridContext;
10363
+ stopped = false;
10364
+ get live() {
10365
+ return !this.stopped && this.stream.getVideoTracks().some((track) => track.readyState === "live");
10366
+ }
10367
+ async capture() {
10368
+ const picture = this.pictureSize();
10369
+ if (!picture) {
10370
+ return null;
9870
10371
  }
9871
- return await socket.timeout(timeoutMs).emitWithAck(event, payload);
10372
+ const size = fitCaptureSize(picture.width, picture.height);
10373
+ const canvas = this.frameCanvas;
10374
+ if (canvas.width !== size.width || canvas.height !== size.height) {
10375
+ canvas.width = size.width;
10376
+ canvas.height = size.height;
10377
+ }
10378
+ this.frameContext ??= canvas.getContext("2d");
10379
+ const context2 = this.frameContext;
10380
+ if (!context2) {
10381
+ return null;
10382
+ }
10383
+ context2.imageSmoothingQuality = "high";
10384
+ context2.drawImage(this.video, 0, 0, size.width, size.height);
10385
+ let blob = await encodeJpeg(canvas, SIGHT_CAPTURE_SIZE.jpegQuality);
10386
+ if (blob && blob.size > SIGHT_LIMITS.maxFrameBytes) {
10387
+ blob = await encodeJpeg(canvas, SMALLER_JPEG_QUALITY);
10388
+ }
10389
+ if (!blob || blob.size > SIGHT_LIMITS.maxFrameBytes) {
10390
+ return null;
10391
+ }
10392
+ return { jpeg: await blob.arrayBuffer(), ...size };
9872
10393
  }
9873
- /** An event without an acknowledgement, in order with the other events. */
9874
- send(event, payload) {
9875
- this.socket?.emit(event, payload);
10394
+ luminance() {
10395
+ const context2 = this.gridContext;
10396
+ if (!context2 || !this.pictureSize()) {
10397
+ return null;
10398
+ }
10399
+ context2.drawImage(this.video, 0, 0, LUMA_GRID.width, LUMA_GRID.height);
10400
+ return lumaOf(
10401
+ context2.getImageData(0, 0, LUMA_GRID.width, LUMA_GRID.height).data
10402
+ );
9876
10403
  }
9877
- async ensureConnected(conversationId) {
9878
- if (typeof window === "undefined") {
10404
+ stop() {
10405
+ if (this.stopped) {
9879
10406
  return;
9880
10407
  }
9881
- const normalized = conversationId.trim();
9882
- if (!normalized) {
9883
- return;
10408
+ this.stopped = true;
10409
+ this.video.pause();
10410
+ this.video.srcObject = null;
10411
+ if (this.owned) {
10412
+ this.stream.getTracks().forEach((track) => track.stop());
9884
10413
  }
9885
- if (!this.socket) {
9886
- const socketBaseUrl = (() => {
9887
- try {
9888
- return new URL(this.options.baseUrl).origin;
9889
- } catch {
9890
- return window.location.origin;
9891
- }
9892
- })();
9893
- this.socket = io(socketBaseUrl, {
9894
- transports: ["websocket"],
9895
- withCredentials: true,
9896
- auth: {
9897
- apiToken: this.options.apiToken
9898
- }
9899
- });
9900
- this.socket.on("connect", () => {
9901
- const activeConversationId = this.options.getConversationId()?.trim();
9902
- if (activeConversationId) {
9903
- this.socket?.emit("conversations:join", {
9904
- conversationId: activeConversationId
9905
- });
9906
- }
9907
- });
9908
- this.socket.on("conversations:stream", (payload) => {
9909
- let candidate;
9910
- try {
9911
- candidate = decodePerformance(
9912
- payload,
9913
- "stream"
9914
- );
9915
- } catch (error) {
9916
- this.options.handleCallbackError("stream.decode", error);
9917
- return;
9918
- }
9919
- if (typeof candidate?.conversationId !== "string") {
9920
- return;
9921
- }
9922
- if (candidate.conversationId !== this.options.getConversationId()) {
9923
- return;
9924
- }
9925
- const event = candidate.event;
9926
- if (!event || typeof event !== "object") {
9927
- return;
9928
- }
9929
- this.options.handleStreamEvent(event);
9930
- });
9931
- this.socket.on(SPEECH_EVENTS.partial, (payload) => {
9932
- const event = payload;
9933
- if (typeof event?.sessionId === "string" && typeof event.text === "string") {
9934
- this.options.handleSpeechEvent?.({ type: "partial", event });
9935
- }
9936
- });
9937
- this.socket.on(SPEECH_EVENTS.turn, (payload) => {
9938
- let candidate;
9939
- try {
9940
- candidate = decodePerformance(
9941
- payload,
9942
- "turn"
9943
- );
9944
- } catch (error) {
9945
- this.options.handleCallbackError("speech.turn.decode", error);
9946
- return;
9947
- }
9948
- if (candidate?.conversationId !== this.options.getConversationId() || typeof candidate.requestId !== "string" || !candidate.turn || typeof candidate.turn !== "object") {
9949
- return;
9950
- }
9951
- this.options.handleSpeechEvent?.({
9952
- type: "turn",
9953
- requestId: candidate.requestId,
9954
- turn: candidate.turn
9955
- });
9956
- });
9957
- this.socket.on(SPEECH_EVENTS.turnFailed, (payload) => {
9958
- const event = payload;
9959
- if (typeof event?.requestId === "string" && event.conversationId === this.options.getConversationId()) {
9960
- this.options.handleSpeechEvent?.({ type: "turnFailed", event });
9961
- }
9962
- });
9963
- this.socket.on("disconnect", () => {
9964
- this.options.handleSpeechEvent?.({ type: "disconnect" });
9965
- });
9966
- this.socket.on("conversations:turn", (payload) => {
9967
- let candidate;
9968
- try {
9969
- candidate = decodePerformance(
9970
- payload,
9971
- "turn"
9972
- );
9973
- } catch (error) {
9974
- this.options.handleCallbackError("turn.decode", error);
9975
- return;
9976
- }
9977
- if (typeof candidate?.conversationId !== "string") {
9978
- return;
9979
- }
9980
- if (candidate.conversationId !== this.options.getConversationId()) {
9981
- return;
9982
- }
9983
- const turn = candidate.turn;
9984
- if (!turn || typeof turn !== "object") {
9985
- return;
9986
- }
9987
- void this.options.handleTurn(turn).catch((error) => {
9988
- this.options.handleCallbackError("onTurn", error);
9989
- });
9990
- });
9991
- await new Promise((resolve) => {
9992
- if (this.socket?.connected) {
9993
- resolve();
9994
- return;
9995
- }
9996
- this.socket?.once("connect", () => resolve());
9997
- this.socket?.once("connect_error", () => resolve());
9998
- });
10414
+ }
10415
+ pictureSize() {
10416
+ const { video } = this;
10417
+ if (this.stopped || video.readyState < HAVE_CURRENT_DATA || !video.videoWidth || !video.videoHeight) {
10418
+ return null;
9999
10419
  }
10000
- await new Promise((resolve, reject) => {
10001
- this.socket.timeout(1e4).emit(
10002
- "conversations:join",
10003
- { conversationId: normalized },
10004
- (error, result) => {
10005
- if (error || !result?.ok) {
10006
- reject(
10007
- error ?? new Error("Conversation streaming access was denied.")
10008
- );
10009
- } else {
10010
- resolve();
10011
- }
10012
- }
10420
+ return { width: video.videoWidth, height: video.videoHeight };
10421
+ }
10422
+ };
10423
+ var openFrameSource = async function(source) {
10424
+ if (source !== "camera") {
10425
+ if (typeof source?.getVideoTracks !== "function") {
10426
+ throw new SightError(
10427
+ "CAMERA_UNAVAILABLE",
10428
+ 'The sight source must be "camera" or a MediaStream.'
10429
+ );
10430
+ }
10431
+ if (source.getVideoTracks().length === 0) {
10432
+ throw new SightError(
10433
+ "CAMERA_UNAVAILABLE",
10434
+ "The stream has no video track."
10013
10435
  );
10436
+ }
10437
+ return new VideoFrameSource(source, false);
10438
+ }
10439
+ const devices = typeof navigator === "undefined" ? void 0 : navigator.mediaDevices;
10440
+ if (!devices?.getUserMedia) {
10441
+ throw new SightError(
10442
+ "CAMERA_UNAVAILABLE",
10443
+ "This browser cannot open the camera."
10444
+ );
10445
+ }
10446
+ let stream;
10447
+ try {
10448
+ stream = await devices.getUserMedia({
10449
+ video: {
10450
+ width: { ideal: SIGHT_CAPTURE_SIZE.width },
10451
+ height: { ideal: SIGHT_CAPTURE_SIZE.height }
10452
+ },
10453
+ audio: false
10014
10454
  });
10455
+ } catch (error) {
10456
+ const name = error?.name;
10457
+ throw name === "NotAllowedError" || name === "SecurityError" ? new SightError("CAMERA_PERMISSION_DENIED", "Camera access was denied.") : new SightError("CAMERA_UNAVAILABLE", "No camera is available.");
10015
10458
  }
10459
+ return new VideoFrameSource(stream, true);
10016
10460
  };
10017
10461
 
10018
- // ../../clients/web-client/src/voice/push-to-talk.ts
10019
- var INTERACTIVE = 'input, textarea, select, button, a[href], summary, [contenteditable]:not([contenteditable="false"]), [role="button"], [role="textbox"], [role="checkbox"], [role="switch"], [role="menuitem"], [role="tab"], [role="slider"], [role="option"]';
10020
- var elementOf = function(target) {
10021
- return target && typeof target.closest === "function" ? target : null;
10462
+ // ../../clients/web-client/src/sight/sight-input.ts
10463
+ var SIGHT_CHECK_MS = 250;
10464
+ var SIGHT_INTERVAL_MS = 5e3;
10465
+ var SIGHT_MOTION_THRESHOLD = 8;
10466
+ var ACK_TIMEOUT_MS = 2e4;
10467
+ var systemClock = {
10468
+ now: () => Date.now(),
10469
+ setInterval: (callback, ms) => setInterval(callback, ms),
10470
+ clearInterval: (handle) => clearInterval(handle)
10471
+ };
10472
+ var settle = async function(run) {
10473
+ return run();
10474
+ };
10475
+ var documentHidden = function() {
10476
+ return typeof document !== "undefined" && document.hidden === true;
10477
+ };
10478
+ var intervalOf = function(value) {
10479
+ return typeof value === "number" && value > 0 ? Math.max(SIGHT_LIMITS.minFrameIntervalMs, value) : SIGHT_INTERVAL_MS;
10480
+ };
10481
+ var thresholdOf = function(motion) {
10482
+ if (motion === false) {
10483
+ return null;
10484
+ }
10485
+ const threshold = motion?.threshold;
10486
+ return typeof threshold === "number" && Number.isFinite(threshold) ? Math.max(0, threshold) : SIGHT_MOTION_THRESHOLD;
10022
10487
  };
10023
- var bindPushToTalk = function(target, options = {}) {
10024
- const element = options.element ?? null;
10025
- const key = options.key === void 0 ? "Space" : options.key;
10026
- const minHoldMs = options.minHoldMs ?? SPEECH_MIN_HOLD_MS;
10027
- let active = null;
10028
- let pointerId = null;
10029
- let pressedAt = 0;
10030
- const begin = function(source) {
10031
- if (active || target.listeningState !== "idle") {
10032
- return false;
10488
+ var isSightObservationEvent = function(value) {
10489
+ const event = value;
10490
+ const observation = event?.observation;
10491
+ return typeof event?.conversationId === "string" && typeof event.observationId === "string" && typeof event.summary === "string" && typeof observation === "object" && observation !== null && typeof observation.people === "number" && Array.isArray(observation.persons) && Array.isArray(observation.text);
10492
+ };
10493
+ var SightInput = class {
10494
+ constructor(deps) {
10495
+ this.deps = deps;
10496
+ this.clock = deps.clock ?? systemClock;
10497
+ this.hidden = deps.hidden ?? documentHidden;
10498
+ }
10499
+ clock;
10500
+ hidden;
10501
+ session = null;
10502
+ /** Why sight cannot start; null when it can. */
10503
+ unavailable = "Sight needs the character info: load the character first.";
10504
+ current = "unavailable";
10505
+ stateCallback = null;
10506
+ observationCallback = null;
10507
+ get state() {
10508
+ return this.current;
10509
+ }
10510
+ /**
10511
+ * The character info was read. Sight is available only when the owner
10512
+ * turned it on, and never in a baked conversation. Sight that becomes
10513
+ * unavailable stops.
10514
+ */
10515
+ configure(settings) {
10516
+ this.unavailable = settings.baked ? "Sight is not available in a baked (offline) conversation." : settings.enabled ? null : SIGHT_OFF_MESSAGE;
10517
+ if (this.unavailable) {
10518
+ this.stop();
10519
+ } else {
10520
+ this.publish();
10521
+ }
10522
+ }
10523
+ /**
10524
+ * Opens the source and joins the conversation. Resolves when frames can
10525
+ * go; the first frame goes when the video has a picture.
10526
+ */
10527
+ async start(options = {}) {
10528
+ if (this.unavailable) {
10529
+ throw new SightError("SIGHT_DISABLED", this.unavailable);
10530
+ }
10531
+ if (this.session) {
10532
+ throw new SightError(
10533
+ "SIGHT_ALREADY_STARTED",
10534
+ "Sight has already started."
10535
+ );
10536
+ }
10537
+ const conversationId = this.deps.conversationId();
10538
+ if (!conversationId) {
10539
+ throw new SightError(
10540
+ "CONVERSATION_REQUIRED",
10541
+ "Create a conversation before sight starts."
10542
+ );
10543
+ }
10544
+ const session = {
10545
+ intervalMs: intervalOf(options.intervalMs),
10546
+ threshold: thresholdOf(options.motion),
10547
+ onError: options.onError,
10548
+ source: null,
10549
+ timer: null,
10550
+ grid: null,
10551
+ lastSentAt: Number.NEGATIVE_INFINITY,
10552
+ motion: false,
10553
+ inFlight: false,
10554
+ rejoin: false
10555
+ };
10556
+ this.session = session;
10557
+ this.publish();
10558
+ const open = this.deps.openSource ?? openFrameSource;
10559
+ const [joined, opened] = await Promise.allSettled([
10560
+ settle(() => this.deps.socket().ensureConnected(conversationId)),
10561
+ settle(() => open(options.source ?? "camera"))
10562
+ ]);
10563
+ const source = opened.status === "fulfilled" ? opened.value : null;
10564
+ if (session !== this.session) {
10565
+ source?.stop();
10566
+ return;
10567
+ }
10568
+ const failure = opened.status === "rejected" ? toSightError(opened.reason, "CAMERA_UNAVAILABLE") : joined.status === "rejected" ? toSightError(joined.reason, "CONVERSATION_DISCONNECTED") : null;
10569
+ if (failure || !source) {
10570
+ source?.stop();
10571
+ this.session = null;
10572
+ this.publish();
10573
+ throw failure ?? new SightError("CAMERA_UNAVAILABLE");
10574
+ }
10575
+ session.source = source;
10576
+ session.timer = this.clock.setInterval(
10577
+ () => this.check(session),
10578
+ SIGHT_CHECK_MS
10579
+ );
10580
+ this.publish();
10581
+ this.check(session);
10582
+ }
10583
+ /** Stops the frames and releases the source. Does nothing when sight is off. */
10584
+ stop() {
10585
+ if (this.session) {
10586
+ this.end(this.session);
10587
+ }
10588
+ this.publish();
10589
+ }
10590
+ /** An observation of the current conversation (the socket checked the ID). */
10591
+ handleObservation(event) {
10592
+ if (this.observationCallback) {
10593
+ invokeSafeCallback(
10594
+ "onSight",
10595
+ this.deps.handleCallbackError,
10596
+ () => this.observationCallback?.(event)
10597
+ );
10598
+ }
10599
+ }
10600
+ onObservation(callback) {
10601
+ return bindCallbackSlot(
10602
+ () => this.observationCallback,
10603
+ (next) => this.observationCallback = next,
10604
+ callback
10605
+ );
10606
+ }
10607
+ onState(callback) {
10608
+ return bindCallbackSlot(
10609
+ () => this.stateCallback,
10610
+ (next) => this.stateCallback = next,
10611
+ callback
10612
+ );
10613
+ }
10614
+ /** One check: the end of the source, a hidden page, motion, then a due frame. */
10615
+ check(session) {
10616
+ const source = session.source;
10617
+ if (session !== this.session || !source) {
10618
+ return;
10619
+ }
10620
+ if (!source.live) {
10621
+ this.fail(
10622
+ session,
10623
+ new SightError("SIGHT_SOURCE_ENDED", "The video of sight ended.")
10624
+ );
10625
+ return;
10626
+ }
10627
+ if (this.hidden()) {
10628
+ session.grid = null;
10629
+ return;
10630
+ }
10631
+ if (session.threshold !== null) {
10632
+ const grid = source.luminance();
10633
+ if (grid && session.grid && meanLumaChange(session.grid, grid) > session.threshold) {
10634
+ session.motion = true;
10635
+ }
10636
+ session.grid = grid;
10637
+ }
10638
+ this.sendWhenDue(session);
10639
+ }
10640
+ /**
10641
+ * A frame is due at the interval, or after motion when one second passed
10642
+ * since the last frame. While a frame waits for its acknowledgement, a due
10643
+ * frame waits too: the acknowledgement sends it.
10644
+ */
10645
+ sendWhenDue(session) {
10646
+ if (session !== this.session || session.inFlight || this.hidden()) {
10647
+ return;
10648
+ }
10649
+ const elapsed = this.clock.now() - session.lastSentAt;
10650
+ if (elapsed >= session.intervalMs || session.motion && elapsed >= SIGHT_LIMITS.minFrameIntervalMs) {
10651
+ void this.send(session);
10652
+ }
10653
+ }
10654
+ async send(session) {
10655
+ const source = session.source;
10656
+ if (!source) {
10657
+ return;
10658
+ }
10659
+ session.inFlight = true;
10660
+ let sent = false;
10661
+ try {
10662
+ const capturedAt = this.clock.now();
10663
+ let frame;
10664
+ try {
10665
+ frame = await source.capture();
10666
+ } catch (error) {
10667
+ this.fail(
10668
+ session,
10669
+ new SightError(
10670
+ "CAMERA_UNAVAILABLE",
10671
+ error instanceof Error ? error.message : "The video gave no picture."
10672
+ )
10673
+ );
10674
+ return;
10675
+ }
10676
+ if (!frame || session !== this.session) {
10677
+ return;
10678
+ }
10679
+ const conversationId = this.deps.conversationId();
10680
+ if (!conversationId) {
10681
+ this.stop();
10682
+ return;
10683
+ }
10684
+ session.lastSentAt = this.clock.now();
10685
+ session.motion = false;
10686
+ sent = true;
10687
+ const socket = this.deps.socket();
10688
+ if (session.rejoin) {
10689
+ await socket.ensureConnected(conversationId);
10690
+ session.rejoin = false;
10691
+ }
10692
+ const request = {
10693
+ conversationId,
10694
+ capturedAt,
10695
+ width: frame.width,
10696
+ height: frame.height,
10697
+ source: source.kind,
10698
+ jpeg: frame.jpeg
10699
+ };
10700
+ const ack = await socket.request(
10701
+ SIGHT_EVENTS.frame,
10702
+ request,
10703
+ ACK_TIMEOUT_MS
10704
+ );
10705
+ const code = ack && !ack.ok ? ack.error : null;
10706
+ if (code && SIGHT_STOP_CODES.has(code)) {
10707
+ this.fail(session, sightStopError(code));
10708
+ }
10709
+ } catch (error) {
10710
+ this.failed(session, error);
10711
+ } finally {
10712
+ session.inFlight = false;
10713
+ if (sent) {
10714
+ this.sendWhenDue(session);
10715
+ }
10716
+ }
10717
+ }
10718
+ /**
10719
+ * A frame that did not reach the server or got no acknowledgement. A
10720
+ * refused conversation stops sight. A timeout skips the frame; a lost
10721
+ * connection also joins the conversation again before the next frame.
10722
+ */
10723
+ failed(session, error) {
10724
+ const code = error?.code;
10725
+ if (typeof code === "string" && SIGHT_STOP_CODES.has(code)) {
10726
+ this.fail(session, sightStopError(code));
10727
+ return;
10728
+ }
10729
+ if (code !== "CONVERSATION_TIMEOUT") {
10730
+ session.rejoin = true;
10731
+ }
10732
+ }
10733
+ /** Stops sight because of `error`, and reports it. */
10734
+ fail(session, error) {
10735
+ if (session !== this.session) {
10736
+ return;
10737
+ }
10738
+ this.end(session);
10739
+ if (error.code === "SIGHT_DISABLED") {
10740
+ this.unavailable = SIGHT_OFF_MESSAGE;
10741
+ }
10742
+ this.publish(error);
10743
+ const onError = session.onError;
10744
+ if (onError) {
10745
+ invokeSafeCallback(
10746
+ "startSight onError",
10747
+ this.deps.handleCallbackError,
10748
+ () => onError(error)
10749
+ );
10750
+ }
10751
+ }
10752
+ end(session) {
10753
+ if (this.session !== session) {
10754
+ return;
10755
+ }
10756
+ this.session = null;
10757
+ if (session.timer !== null) {
10758
+ this.clock.clearInterval(session.timer);
10759
+ session.timer = null;
10760
+ }
10761
+ session.source?.stop();
10762
+ }
10763
+ publish(error) {
10764
+ const session = this.session;
10765
+ const next = session ? session.source ? "on" : "starting" : this.unavailable ? "unavailable" : "off";
10766
+ if (next === this.current && !error) {
10767
+ return;
10768
+ }
10769
+ this.current = next;
10770
+ if (this.stateCallback) {
10771
+ invokeSafeCallback(
10772
+ "onSightState",
10773
+ this.deps.handleCallbackError,
10774
+ () => this.stateCallback?.(next, error)
10775
+ );
10776
+ }
10777
+ }
10778
+ };
10779
+
10780
+ // ../../clients/web-client/src/conversation-socket.ts
10781
+ var ConversationConnectionError = class extends Error {
10782
+ constructor(code, message, reason = null) {
10783
+ super(message);
10784
+ this.code = code;
10785
+ this.reason = reason;
10786
+ this.name = "NpcifyConversationConnectionError";
10787
+ }
10788
+ };
10789
+ var JOIN_EVENT = "conversations:join";
10790
+ var JOIN_TIMEOUT_MS = 1e4;
10791
+ var ACCESS_DENIED = "CONVERSATION_ACCESS_DENIED";
10792
+ var JOIN_FAILED = "CONVERSATION_JOIN_FAILED";
10793
+ var CLIENT_CLOSE = "io client disconnect";
10794
+ var SOCKET_IO_TIMEOUT = "operation has timed out";
10795
+ var joinMessage = function(code) {
10796
+ if (code === ACCESS_DENIED) {
10797
+ return "Conversation streaming access was denied.";
10798
+ }
10799
+ if (code === JOIN_FAILED) {
10800
+ return "The server could not join the conversation. Try again later.";
10801
+ }
10802
+ return `The server did not join the conversation (${code}). Try again later.`;
10803
+ };
10804
+ var closeMessage = function(reason, detail) {
10805
+ if (reason === "io server disconnect") {
10806
+ return "The server closed the conversation connection.";
10807
+ }
10808
+ if (reason === CLIENT_CLOSE) {
10809
+ return "The conversation connection closed.";
10810
+ }
10811
+ if (reason === "connect_error") {
10812
+ return `The server refused the conversation connection (${detail || "no reason"}).`;
10813
+ }
10814
+ return `The conversation connection was lost (${reason}).`;
10815
+ };
10816
+ var acknowledgementError = function(error, lost) {
10817
+ if (error.message === SOCKET_IO_TIMEOUT) {
10818
+ return new ConversationConnectionError(
10819
+ "CONVERSATION_TIMEOUT",
10820
+ "The server did not answer in time."
10821
+ );
10822
+ }
10823
+ return new ConversationConnectionError(
10824
+ "CONVERSATION_DISCONNECTED",
10825
+ closeMessage(lost ?? "transport close"),
10826
+ lost
10827
+ );
10828
+ };
10829
+ var refusedConversation = function(payload) {
10830
+ const exception = payload;
10831
+ const named = exception?.cause?.data?.conversationId;
10832
+ if (exception?.message !== ACCESS_DENIED || typeof named !== "string") {
10833
+ return null;
10834
+ }
10835
+ return named.trim() || null;
10836
+ };
10837
+ var ConversationSocket = class {
10838
+ options;
10839
+ connection = null;
10840
+ /**
10841
+ * The conversations that the server refused. A join with the same
10842
+ * credentials fails the same way, so the SDK does not try them again until
10843
+ * `disconnect`.
10844
+ */
10845
+ refused = /* @__PURE__ */ new Set();
10846
+ constructor(options) {
10847
+ this.options = options;
10848
+ }
10849
+ /** Closes the connection and forgets the refused conversations. */
10850
+ disconnect() {
10851
+ this.refused.clear();
10852
+ if (this.connection) {
10853
+ this.close(this.connection, CLIENT_CLOSE);
10854
+ }
10855
+ }
10856
+ /** An event with an acknowledgement. Needs `ensureConnected` first. */
10857
+ async request(event, payload, timeoutMs = 1e4) {
10858
+ const connection = this.connection;
10859
+ if (!connection?.socket.connected) {
10860
+ throw new ConversationConnectionError(
10861
+ "CONVERSATION_DISCONNECTED",
10862
+ "The conversation connection is not open."
10863
+ );
10864
+ }
10865
+ return this.acknowledged(connection, event, payload, timeoutMs, null);
10866
+ }
10867
+ /** An event without an acknowledgement, in order with the other events. */
10868
+ send(event, payload) {
10869
+ this.connection?.socket.emit(event, payload);
10870
+ }
10871
+ /**
10872
+ * Joins the room of the conversation. A connection that the server closed
10873
+ * is not used again: the join opens a new one. A refused conversation fails
10874
+ * at once, without a connection.
10875
+ */
10876
+ async ensureConnected(conversationId) {
10877
+ if (typeof window === "undefined") {
10878
+ return;
10879
+ }
10880
+ const normalized = conversationId.trim();
10881
+ if (!normalized) {
10882
+ return;
10883
+ }
10884
+ if (this.refused.has(normalized)) {
10885
+ throw new ConversationConnectionError(
10886
+ ACCESS_DENIED,
10887
+ joinMessage(ACCESS_DENIED)
10888
+ );
10889
+ }
10890
+ const connection = this.connection ?? this.open();
10891
+ await this.connected(connection);
10892
+ await this.acknowledged(
10893
+ connection,
10894
+ JOIN_EVENT,
10895
+ { conversationId: normalized },
10896
+ this.options.joinTimeoutMs ?? JOIN_TIMEOUT_MS,
10897
+ normalized
10898
+ );
10899
+ }
10900
+ open() {
10901
+ const socketBaseUrl = (() => {
10902
+ try {
10903
+ return new URL(this.options.baseUrl).origin;
10904
+ } catch {
10905
+ return window.location.origin;
10906
+ }
10907
+ })();
10908
+ const socket = io(socketBaseUrl, {
10909
+ transports: ["websocket"],
10910
+ withCredentials: true,
10911
+ forceNew: true,
10912
+ auth: {
10913
+ apiToken: this.options.apiToken
10914
+ }
10915
+ });
10916
+ const connection = {
10917
+ socket,
10918
+ pending: /* @__PURE__ */ new Set(),
10919
+ connectedBefore: false,
10920
+ lost: null,
10921
+ closed: null,
10922
+ failure: null
10923
+ };
10924
+ this.connection = connection;
10925
+ this.watch(connection);
10926
+ this.receive(socket);
10927
+ return connection;
10928
+ }
10929
+ /** The lifecycle of one connection (docs/streaming_reply.md#web-sdk). */
10930
+ watch(connection) {
10931
+ const { socket } = connection;
10932
+ socket.on("connect", () => {
10933
+ if (!connection.connectedBefore) {
10934
+ connection.connectedBefore = true;
10935
+ return;
10936
+ }
10937
+ const activeConversationId = this.options.getConversationId()?.trim();
10938
+ if (activeConversationId && !this.refused.has(activeConversationId)) {
10939
+ socket.emit(JOIN_EVENT, { conversationId: activeConversationId });
10940
+ }
10941
+ });
10942
+ socket.on("exception", (payload) => {
10943
+ const conversationId = refusedConversation(payload);
10944
+ if (conversationId) {
10945
+ connection.failure ??= { code: ACCESS_DENIED, conversationId };
10946
+ this.refused.add(conversationId);
10947
+ }
10948
+ });
10949
+ socket.on("disconnect", (reason) => {
10950
+ if (!socket.active) {
10951
+ this.close(connection, reason);
10952
+ return;
10953
+ }
10954
+ connection.lost = reason;
10955
+ this.options.handleSpeechEvent?.({ type: "disconnect" });
10956
+ });
10957
+ socket.on("connect_error", (error) => {
10958
+ if (!socket.active) {
10959
+ this.close(connection, "connect_error", error.message);
10960
+ }
10961
+ });
10962
+ }
10963
+ /**
10964
+ * Closes the connection for good: socket.io does not open it again. The
10965
+ * waits fail now, and the next join opens a new connection.
10966
+ */
10967
+ close(connection, reason, detail) {
10968
+ if (connection.closed) {
10969
+ return;
10970
+ }
10971
+ connection.closed = { reason, message: closeMessage(reason, detail) };
10972
+ if (this.connection === connection) {
10973
+ this.connection = null;
10974
+ }
10975
+ const { socket } = connection;
10976
+ socket.removeAllListeners();
10977
+ if (connection.connectedBefore) {
10978
+ this.options.handleSpeechEvent?.({ type: "disconnect" });
10979
+ }
10980
+ const pending = [...connection.pending];
10981
+ connection.pending.clear();
10982
+ for (const entry of pending) {
10983
+ entry.fail(this.closeError(connection, entry.conversationId));
10984
+ }
10985
+ socket.disconnect();
10986
+ if (reason !== CLIENT_CLOSE) {
10987
+ this.options.handleConnectionClosed?.({
10988
+ reason,
10989
+ code: connection.failure?.code ?? null,
10990
+ conversationId: connection.failure?.conversationId ?? null
10991
+ });
10992
+ }
10993
+ }
10994
+ closeError(connection, conversationId) {
10995
+ if (conversationId !== null && this.refused.has(conversationId)) {
10996
+ return new ConversationConnectionError(
10997
+ ACCESS_DENIED,
10998
+ joinMessage(ACCESS_DENIED)
10999
+ );
11000
+ }
11001
+ return new ConversationConnectionError(
11002
+ "CONVERSATION_DISCONNECTED",
11003
+ connection.closed?.message ?? closeMessage(CLIENT_CLOSE),
11004
+ connection.closed?.reason ?? null
11005
+ );
11006
+ }
11007
+ /**
11008
+ * Waits for the first connection. A connection error that socket.io
11009
+ * recovers from also ends the wait: the join then waits in socket.io's
11010
+ * buffer, for the time of its acknowledgement at most.
11011
+ */
11012
+ connected(connection) {
11013
+ const { socket } = connection;
11014
+ if (connection.closed) {
11015
+ return Promise.reject(this.closeError(connection, null));
11016
+ }
11017
+ if (socket.connected || connection.connectedBefore) {
11018
+ return Promise.resolve();
11019
+ }
11020
+ return new Promise((resolve, reject) => {
11021
+ const entry = { conversationId: null, fail: reject };
11022
+ const done = () => {
11023
+ if (connection.pending.delete(entry)) {
11024
+ socket.off("connect", done);
11025
+ socket.off("connect_error", done);
11026
+ resolve();
11027
+ }
11028
+ };
11029
+ connection.pending.add(entry);
11030
+ socket.on("connect", done);
11031
+ socket.on("connect_error", done);
11032
+ });
11033
+ }
11034
+ /**
11035
+ * Emits an event and waits for its acknowledgement. A join (`joining`)
11036
+ * that the server does not accept fails with the code of its
11037
+ * acknowledgement. A close of the connection fails the wait at once; a
11038
+ * timeout fails it with `CONVERSATION_TIMEOUT`.
11039
+ */
11040
+ acknowledged(connection, event, payload, timeoutMs, joining) {
11041
+ return new Promise((resolve, reject) => {
11042
+ if (connection.closed) {
11043
+ reject(this.closeError(connection, joining));
11044
+ return;
11045
+ }
11046
+ const entry = { conversationId: joining, fail: reject };
11047
+ connection.pending.add(entry);
11048
+ connection.socket.timeout(timeoutMs).emit(event, payload, (error, result) => {
11049
+ if (!connection.pending.delete(entry)) {
11050
+ return;
11051
+ }
11052
+ const failure = error ? acknowledgementError(error, connection.lost) : joining === null ? null : this.joinFailure(connection, joining, result);
11053
+ if (failure) {
11054
+ reject(failure);
11055
+ } else {
11056
+ resolve(result);
11057
+ }
11058
+ });
11059
+ });
11060
+ }
11061
+ /** The error of a join that the server did not accept; null after a join. */
11062
+ joinFailure(connection, conversationId, result) {
11063
+ const ack = result;
11064
+ if (ack?.ok) {
11065
+ return null;
11066
+ }
11067
+ const named = typeof ack?.error === "string" ? ack.error.trim() : "";
11068
+ const code = named || JOIN_FAILED;
11069
+ connection.failure ??= { code, conversationId };
11070
+ if (code === ACCESS_DENIED) {
11071
+ this.refused.add(conversationId);
11072
+ }
11073
+ this.close(connection, "io server disconnect");
11074
+ return new ConversationConnectionError(code, joinMessage(code));
11075
+ }
11076
+ /** The events that the server sends to the conversation's room. */
11077
+ receive(socket) {
11078
+ socket.on("conversations:stream", (payload) => {
11079
+ let candidate;
11080
+ try {
11081
+ candidate = decodePerformance(
11082
+ payload,
11083
+ "stream"
11084
+ );
11085
+ } catch (error) {
11086
+ this.options.handleCallbackError("stream.decode", error);
11087
+ return;
11088
+ }
11089
+ if (typeof candidate?.conversationId !== "string") {
11090
+ return;
11091
+ }
11092
+ if (candidate.conversationId !== this.options.getConversationId()) {
11093
+ return;
11094
+ }
11095
+ const event = candidate.event;
11096
+ if (!event || typeof event !== "object") {
11097
+ return;
11098
+ }
11099
+ this.options.handleStreamEvent(event);
11100
+ });
11101
+ socket.on(SPEECH_EVENTS.partial, (payload) => {
11102
+ const event = payload;
11103
+ if (typeof event?.sessionId === "string" && typeof event.text === "string") {
11104
+ this.options.handleSpeechEvent?.({ type: "partial", event });
11105
+ }
11106
+ });
11107
+ socket.on(SPEECH_EVENTS.turn, (payload) => {
11108
+ let candidate;
11109
+ try {
11110
+ candidate = decodePerformance(
11111
+ payload,
11112
+ "turn"
11113
+ );
11114
+ } catch (error) {
11115
+ this.options.handleCallbackError("speech.turn.decode", error);
11116
+ return;
11117
+ }
11118
+ if (candidate?.conversationId !== this.options.getConversationId() || typeof candidate.requestId !== "string" || !candidate.turn || typeof candidate.turn !== "object") {
11119
+ return;
11120
+ }
11121
+ this.options.handleSpeechEvent?.({
11122
+ type: "turn",
11123
+ requestId: candidate.requestId,
11124
+ turn: candidate.turn
11125
+ });
11126
+ });
11127
+ socket.on(SPEECH_EVENTS.turnFailed, (payload) => {
11128
+ const event = payload;
11129
+ if (typeof event?.requestId === "string" && event.conversationId === this.options.getConversationId()) {
11130
+ this.options.handleSpeechEvent?.({ type: "turnFailed", event });
11131
+ }
11132
+ });
11133
+ socket.on(SIGHT_EVENTS.observation, (payload) => {
11134
+ if (isSightObservationEvent(payload) && payload.conversationId === this.options.getConversationId()) {
11135
+ this.options.handleSightEvent?.(payload);
11136
+ }
11137
+ });
11138
+ socket.on("conversations:turn", (payload) => {
11139
+ let candidate;
11140
+ try {
11141
+ candidate = decodePerformance(
11142
+ payload,
11143
+ "turn"
11144
+ );
11145
+ } catch (error) {
11146
+ this.options.handleCallbackError("turn.decode", error);
11147
+ return;
11148
+ }
11149
+ if (typeof candidate?.conversationId !== "string") {
11150
+ return;
11151
+ }
11152
+ if (candidate.conversationId !== this.options.getConversationId()) {
11153
+ return;
11154
+ }
11155
+ const turn = candidate.turn;
11156
+ if (!turn || typeof turn !== "object") {
11157
+ return;
11158
+ }
11159
+ void this.options.handleTurn(turn).catch((error) => {
11160
+ this.options.handleCallbackError("onTurn", error);
11161
+ });
11162
+ });
11163
+ }
11164
+ };
11165
+
11166
+ // ../../clients/web-client/src/input-mode-rules.ts
11167
+ var InputModeRules = class {
11168
+ configured;
11169
+ resolved = null;
11170
+ /** The character's voice input switch, after the character loads. */
11171
+ speechInput = null;
11172
+ latestOptions = [];
11173
+ /**
11174
+ * Throws an `InputModeError` at once for a malformed configuration, and
11175
+ * for a baked asset with a mode other than `options`.
11176
+ */
11177
+ constructor(inputModes, bakedAsset) {
11178
+ this.configured = parseInputModes(inputModes);
11179
+ if (this.configured && bakedAsset) {
11180
+ resolveInputModes(this.configured, {
11181
+ allowCustomReply: false,
11182
+ baked: true,
11183
+ speechInput: false
11184
+ });
11185
+ }
11186
+ }
11187
+ /**
11188
+ * The character loaded. Throws an `InputModeError` when the configuration
11189
+ * has a mode that the character does not permit.
11190
+ */
11191
+ load(permissions) {
11192
+ this.speechInput = permissions.speechInput;
11193
+ this.resolved = resolveInputModes(this.configured, permissions);
11194
+ }
11195
+ /**
11196
+ * The input modes in use: the configured modes, or every mode that the
11197
+ * character permits. Before the character loads, the configuration or all
11198
+ * modes.
11199
+ */
11200
+ get modes() {
11201
+ return this.resolved ?? this.configured ?? permittedInputModes({
11202
+ allowCustomReply: true,
11203
+ baked: false,
11204
+ speechInput: true
11205
+ });
11206
+ }
11207
+ /** The reply options of the latest result, for `assertReply`. */
11208
+ setOptions(options) {
11209
+ this.latestOptions = options;
11210
+ }
11211
+ /** Throws when the input modes do not include `mode`. */
11212
+ assertMode(mode, baked) {
11213
+ if (mode === "voice" && baked) {
11214
+ throw new InputModeError(
11215
+ "Voice input is not available in a baked (offline) conversation."
11216
+ );
11217
+ }
11218
+ if (mode === "voice" && this.speechInput === false) {
11219
+ throw new InputModeError(
11220
+ "Voice input is off for this character. Its owner turns it on in the STT tab."
11221
+ );
11222
+ }
11223
+ const modes = this.modes;
11224
+ if (!modes.includes(mode)) {
11225
+ throw new InputModeError(
11226
+ `The "${mode}" input mode is off: inputModes is ${JSON.stringify(modes)}.`
11227
+ );
11228
+ }
11229
+ }
11230
+ /**
11231
+ * A typed reply needs the `text` mode; one of the current reply options
11232
+ * needs `options` or `text`. Without a configuration the server's rules
11233
+ * apply alone, as before.
11234
+ */
11235
+ assertReply(message) {
11236
+ if (!this.configured) {
11237
+ return;
11238
+ }
11239
+ const modes = this.modes;
11240
+ if (modes.includes("text")) {
11241
+ return;
11242
+ }
11243
+ const reply = message.trim();
11244
+ const isOption = this.latestOptions.some(
11245
+ (option) => option.label.trim() === reply
11246
+ );
11247
+ if (isOption && modes.includes("options")) {
11248
+ return;
11249
+ }
11250
+ throw new InputModeError(
11251
+ isOption ? `Reply options are off: inputModes is ${JSON.stringify(modes)}.` : `Typed replies are off: inputModes is ${JSON.stringify(modes)}. Send one of the reply options or use startListening().`
11252
+ );
11253
+ }
11254
+ };
11255
+
11256
+ // ../../clients/web-client/src/voice/push-to-talk.ts
11257
+ var INTERACTIVE = 'input, textarea, select, button, a[href], summary, [contenteditable]:not([contenteditable="false"]), [role="button"], [role="textbox"], [role="checkbox"], [role="switch"], [role="menuitem"], [role="tab"], [role="slider"], [role="option"]';
11258
+ var elementOf = function(target) {
11259
+ return target && typeof target.closest === "function" ? target : null;
11260
+ };
11261
+ var bindPushToTalk = function(target, options = {}) {
11262
+ const element = options.element ?? null;
11263
+ const key = options.key === void 0 ? "Space" : options.key;
11264
+ const minHoldMs = options.minHoldMs ?? SPEECH_MIN_HOLD_MS;
11265
+ let active = null;
11266
+ let pointerId = null;
11267
+ let pressedAt = 0;
11268
+ const begin = function(source) {
11269
+ if (active || target.listeningState !== "idle") {
11270
+ return false;
10033
11271
  }
10034
11272
  if (options.enabled && !options.enabled()) {
10035
11273
  return false;
@@ -10967,18 +12205,18 @@ var Character = class extends CharacterRuntime {
10967
12205
  notifiedStagingIds = /* @__PURE__ */ new Map();
10968
12206
  liveConversation;
10969
12207
  baked = null;
12208
+ /** The conversation whose live motion was warmed up. */
12209
+ liveMotionPrepared = null;
10970
12210
  authoring;
10971
12211
  normalizeIdleOrientation;
10972
12212
  bakedAsset;
10973
12213
  animationRefreshId = 0;
10974
12214
  voice;
10975
- configuredInputModes;
10976
- resolvedInputModes = null;
10977
- /** The character's voice input switch, after the character loads. */
10978
- speechInput = null;
10979
- latestOptions = [];
12215
+ sight;
12216
+ inputs;
10980
12217
  listeningStateCallback = null;
10981
12218
  transcriptCallback = null;
12219
+ connectionClosedCallback = null;
10982
12220
  conversationId = null;
10983
12221
  constructor(options) {
10984
12222
  const resolvedToken = (options.apiToken ?? options.apiKey)?.trim() ?? "";
@@ -10999,14 +12237,10 @@ var Character = class extends CharacterRuntime {
10999
12237
  this.normalizeIdleOrientation = options.normalizeIdleOrientation ?? false;
11000
12238
  this.bakedAsset = options.bakedConversation;
11001
12239
  this.initialAssetInfo = options.initialAssetInfo ?? null;
11002
- this.configuredInputModes = parseInputModes(options.inputModes);
11003
- if (this.configuredInputModes && this.bakedAsset && !this.authoring) {
11004
- resolveInputModes(this.configuredInputModes, {
11005
- allowCustomReply: false,
11006
- baked: true,
11007
- speechInput: false
11008
- });
11009
- }
12240
+ this.inputs = new InputModeRules(
12241
+ options.inputModes,
12242
+ Boolean(this.bakedAsset) && !this.authoring
12243
+ );
11010
12244
  this.conversationSocket = new ConversationSocket({
11011
12245
  baseUrl: this.baseUrl,
11012
12246
  apiToken: this.apiToken,
@@ -11021,13 +12255,24 @@ var Character = class extends CharacterRuntime {
11021
12255
  await this.handleStagingResult(turn, { source: "turn" });
11022
12256
  },
11023
12257
  handleCallbackError: this.handleCallbackError,
11024
- handleSpeechEvent: (event) => this.voice.handleSocketEvent(event)
12258
+ handleSpeechEvent: (event) => this.voice.handleSocketEvent(event),
12259
+ handleSightEvent: (event) => this.sight.handleObservation(event),
12260
+ handleConnectionClosed: (event) => invokeSafeCallback(
12261
+ "onConnectionClosed",
12262
+ this.handleCallbackError,
12263
+ () => this.connectionClosedCallback?.(event)
12264
+ )
12265
+ });
12266
+ this.sight = new SightInput({
12267
+ socket: () => this.conversationSocket,
12268
+ conversationId: () => this.conversationId,
12269
+ handleCallbackError: this.handleCallbackError
11025
12270
  });
11026
12271
  this.voice = new VoiceInput({
11027
12272
  socket: () => this.conversationSocket,
11028
12273
  conversationId: () => this.conversationId,
11029
12274
  renderSessionId: () => this.renderSessionId,
11030
- assertVoiceAllowed: () => this.assertInputMode("voice"),
12275
+ assertVoiceAllowed: () => this.inputs.assertMode("voice", this.baked !== null),
11031
12276
  turnInFlight: () => this.liveConversation.inFlight,
11032
12277
  beginTurn: () => {
11033
12278
  if (this.conversationId) {
@@ -11091,12 +12336,12 @@ var Character = class extends CharacterRuntime {
11091
12336
  const progressReporter = createAssetProgressReporter(callback);
11092
12337
  const info = await this.getAssetInfo();
11093
12338
  const baked = !this.authoring && Boolean(this.bakedAsset || info.bakedConversationEnabled);
11094
- this.speechInput = info.speechInputEnabled === true;
11095
- this.resolvedInputModes = resolveInputModes(this.configuredInputModes, {
12339
+ this.inputs.load({
11096
12340
  allowCustomReply: info.allowCustomReply !== false,
11097
12341
  baked,
11098
- speechInput: this.speechInput
12342
+ speechInput: info.speechInputEnabled === true
11099
12343
  });
12344
+ this.sight.configure({ enabled: info.sight?.enabled === true, baked });
11100
12345
  this.baked = baked ? await loadBakedConversation({
11101
12346
  baseUrl: this.baseUrl,
11102
12347
  characterId: this.characterId,
@@ -11166,14 +12411,36 @@ var Character = class extends CharacterRuntime {
11166
12411
  if (info.bakedConversationEnabled || this.bakedAsset) {
11167
12412
  throw new Error("Baked conversation requires local rendering.");
11168
12413
  }
11169
- this.speechInput = info.speechInputEnabled === true;
11170
- this.resolvedInputModes = resolveInputModes(this.configuredInputModes, {
12414
+ this.inputs.load({
11171
12415
  allowCustomReply: info.allowCustomReply !== false,
11172
12416
  baked: false,
11173
- speechInput: this.speechInput
12417
+ speechInput: info.speechInputEnabled === true
12418
+ });
12419
+ this.sight.configure({
12420
+ enabled: info.sight?.enabled === true,
12421
+ baked: false
11174
12422
  });
11175
12423
  this.animationController = new RemoteAnimationController();
11176
12424
  }
12425
+ /**
12426
+ * Warms up live motion for the current conversation, once, on the
12427
+ * player's intent: call it at the first focus or key in a reply field or
12428
+ * when a talk key goes down. It does nothing for a character without live
12429
+ * motion; the server limits warm-ups per player (ADR 0011).
12430
+ */
12431
+ async prepareLiveMotion() {
12432
+ const conversationId = this.conversationId;
12433
+ if (this.baked || !conversationId || this.liveMotionPrepared === conversationId) {
12434
+ return false;
12435
+ }
12436
+ this.liveMotionPrepared = conversationId;
12437
+ return warmUpLiveMotion({
12438
+ conversationId,
12439
+ baseUrl: this.baseUrl,
12440
+ apiToken: this.apiToken,
12441
+ renderSessionId: this.renderSessionId
12442
+ });
12443
+ }
11177
12444
  async createConversation(options) {
11178
12445
  const response = this.baked ? {
11179
12446
  conversation: {
@@ -11215,6 +12482,7 @@ var Character = class extends CharacterRuntime {
11215
12482
  }
11216
12483
  async disconnect() {
11217
12484
  this.voice.reset();
12485
+ this.sight.stop();
11218
12486
  this.thinkingState = false;
11219
12487
  if (this.animationController) {
11220
12488
  this.streamPlayback.cleanup();
@@ -11250,7 +12518,7 @@ var Character = class extends CharacterRuntime {
11250
12518
  maybeMessageOrOptions,
11251
12519
  maybeOptions
11252
12520
  );
11253
- this.assertReplyMode(message);
12521
+ this.inputs.assertReply(message);
11254
12522
  if (this.voice.turnPending) {
11255
12523
  throw new Error("A reply is already being generated. Please wait.");
11256
12524
  }
@@ -11298,11 +12566,7 @@ var Character = class extends CharacterRuntime {
11298
12566
  * character permits. Before `loadAssets`, the configuration or all modes.
11299
12567
  */
11300
12568
  get inputModes() {
11301
- return this.resolvedInputModes ?? this.configuredInputModes ?? permittedInputModes({
11302
- allowCustomReply: true,
11303
- baked: false,
11304
- speechInput: true
11305
- });
12569
+ return this.inputs.modes;
11306
12570
  }
11307
12571
  get listeningState() {
11308
12572
  return this.voice.listeningState;
@@ -11346,6 +12610,48 @@ var Character = class extends CharacterRuntime {
11346
12610
  callback
11347
12611
  );
11348
12612
  }
12613
+ /**
12614
+ * `'unavailable'` until the character info turns sight on (its owner's
12615
+ * Sight tab), and in a baked conversation; then `'off'`, `'starting'` or
12616
+ * `'on'`.
12617
+ */
12618
+ get sightState() {
12619
+ return this.sight.state;
12620
+ }
12621
+ /**
12622
+ * Lets the character see the player's camera (docs/sight.md). Needs a
12623
+ * conversation; the first call asks for the camera permission. Rejects
12624
+ * with a `SightError`: `SIGHT_DISABLED` when sight is unavailable,
12625
+ * `CONVERSATION_REQUIRED`, `CAMERA_PERMISSION_DENIED`, ...
12626
+ */
12627
+ startSight(options) {
12628
+ return this.sight.start(options);
12629
+ }
12630
+ /** Stops the frames. Closes the camera that `startSight` opened. */
12631
+ stopSight() {
12632
+ this.sight.stop();
12633
+ }
12634
+ /** What the character saw: each observation of the current conversation. */
12635
+ onSight(callback) {
12636
+ return this.sight.onObservation(callback);
12637
+ }
12638
+ /** The sight state. `error` is the `SightError` that stopped sight. */
12639
+ onSightState(callback) {
12640
+ return this.sight.onState(callback);
12641
+ }
12642
+ /**
12643
+ * The server closed the conversation connection: a refused or failed join,
12644
+ * or an API key that was rotated, edited or deleted. The next `connect`,
12645
+ * `sendMessage` or `startListening` opens a new connection; a refused
12646
+ * conversation fails at once with `CONVERSATION_ACCESS_DENIED`.
12647
+ */
12648
+ onConnectionClosed(callback) {
12649
+ return bindCallbackSlot(
12650
+ () => this.connectionClosedCallback,
12651
+ (next) => this.connectionClosedCallback = next,
12652
+ callback
12653
+ );
12654
+ }
11349
12655
  async getConversationWorldline(conversationId = this.conversationId) {
11350
12656
  if (!conversationId) {
11351
12657
  throw new Error("Conversation ID is required.");
@@ -11409,7 +12715,7 @@ var Character = class extends CharacterRuntime {
11409
12715
  async handleStagingResult(payload, options) {
11410
12716
  this.thinkingState = false;
11411
12717
  const normalized = normalizeStagingResultPayload(payload);
11412
- this.latestOptions = normalized.options;
12718
+ this.inputs.setOptions(normalized.options);
11413
12719
  if (options.source === "turn") {
11414
12720
  this.liveConversation.received(normalized);
11415
12721
  }
@@ -11436,49 +12742,6 @@ var Character = class extends CharacterRuntime {
11436
12742
  void this.streamPlayback.playPayload(normalized);
11437
12743
  }
11438
12744
  }
11439
- /** Throws when the input modes do not include `mode`. */
11440
- assertInputMode(mode) {
11441
- if (mode === "voice" && this.baked) {
11442
- throw new InputModeError(
11443
- "Voice input is not available in a baked (offline) conversation."
11444
- );
11445
- }
11446
- if (mode === "voice" && this.speechInput === false) {
11447
- throw new InputModeError(
11448
- "Voice input is off for this character. Its owner turns it on in the STT tab."
11449
- );
11450
- }
11451
- const modes = this.inputModes;
11452
- if (!modes.includes(mode)) {
11453
- throw new InputModeError(
11454
- `The "${mode}" input mode is off: inputModes is ${JSON.stringify(modes)}.`
11455
- );
11456
- }
11457
- }
11458
- /**
11459
- * A typed reply needs the `text` mode; one of the current reply options
11460
- * needs `options` or `text`. Without a configuration the server's rules
11461
- * apply alone, as before.
11462
- */
11463
- assertReplyMode(message) {
11464
- if (!this.configuredInputModes) {
11465
- return;
11466
- }
11467
- const modes = this.inputModes;
11468
- if (modes.includes("text")) {
11469
- return;
11470
- }
11471
- const reply = message.trim();
11472
- const isOption = this.latestOptions.some(
11473
- (option) => option.label.trim() === reply
11474
- );
11475
- if (isOption && modes.includes("options")) {
11476
- return;
11477
- }
11478
- throw new InputModeError(
11479
- isOption ? `Reply options are off: inputModes is ${JSON.stringify(modes)}.` : `Typed replies are off: inputModes is ${JSON.stringify(modes)}. Send one of the reply options or use startListening().`
11480
- );
11481
- }
11482
12745
  invokeThinkingCallback(payload) {
11483
12746
  this.thinkingState = true;
11484
12747
  if (this.thinkingCallback) {
@@ -12075,10 +13338,12 @@ export {
12075
13338
  CharacterExpressions,
12076
13339
  CharacterRuntime,
12077
13340
  CharacterSceneRig,
13341
+ ConversationConnectionError,
12078
13342
  FacialPerformanceLayer,
12079
13343
  FacialPerformanceMixer,
12080
13344
  INPUT_MODES,
12081
13345
  InputModeError,
13346
+ LiveMotionPlayer,
12082
13347
  MicrophoneError,
12083
13348
  Npcify,
12084
13349
  Character as NpcifyCharacter,
@@ -12086,6 +13351,7 @@ export {
12086
13351
  PERFORMANCE_MEDIA_TYPE,
12087
13352
  PerformanceRecordingSchemaVersion,
12088
13353
  RemoteAnimationController,
13354
+ SightError,
12089
13355
  SpeechInputError,
12090
13356
  analyzePerformanceRecording,
12091
13357
  bindPushToTalk,