@npcify/web 0.2.1 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -6,9 +6,186 @@ var SERVER_RENDER_PROTOCOL_VERSION = 3;
6
6
  var MAXIMUM_PLAYBACK_DURATION_SECONDS = 10 * 60;
7
7
 
8
8
  // ../ts-shared/src/tts.ts
9
- var TTS_PROVIDERS = ["11labs"];
9
+ var TTS_PROVIDERS = ["11labs", "gemini"];
10
10
  var DEFAULT_TTS_PROVIDER = TTS_PROVIDERS[0];
11
11
 
12
+ // ../ts-shared/src/ai-settings.ts
13
+ var GPT_IMAGE_MODELS = [
14
+ "gpt-image-2.5-flare",
15
+ "gpt-image-2.5-sunburst"
16
+ ];
17
+ var GEMINI_SPEECH_MODEL_VENDORS = {
18
+ "gemini-3.5-transcribe-live": "gemini",
19
+ "gemini-3.5-transcribe": "gemini"
20
+ };
21
+ var AI_ROLES = {
22
+ character_draft: {
23
+ name: "Character draft",
24
+ source: "llm",
25
+ models: [],
26
+ options: ["temperature"]
27
+ },
28
+ motion_planner: {
29
+ name: "Motion planner",
30
+ source: "llm",
31
+ models: [],
32
+ options: []
33
+ },
34
+ // Not charged.
35
+ prompt_check: {
36
+ name: "Prompt check",
37
+ source: "openai",
38
+ models: ["gpt-6-luna"],
39
+ options: []
40
+ },
41
+ // Priced by OPENAI_IMAGE_PRICES of the server (flare and sunburst alike).
42
+ design_sheet_image: {
43
+ name: "Design sheet image",
44
+ source: "openai",
45
+ models: GPT_IMAGE_MODELS,
46
+ options: []
47
+ },
48
+ // Not charged per call.
49
+ tts: {
50
+ name: "ElevenLabs text to speech",
51
+ source: "elevenlabs",
52
+ models: ["eleven_v3_conversational", "eleven_v3"],
53
+ options: []
54
+ },
55
+ // Not charged per call.
56
+ gemini_tts: {
57
+ name: "Gemini text to speech",
58
+ source: "gemini",
59
+ models: ["gemini-3.8-flash-tts", "gemini-3.8-flash-lite-tts"],
60
+ options: []
61
+ },
62
+ // Priced by SPEECH_RECOGNITION_PRICES of the server (docs/voice-input.md).
63
+ // Audio that arrives while the player speaks. The model selects the vendor.
64
+ speech_input_live: {
65
+ name: "Speech input (live)",
66
+ source: "openai",
67
+ models: [
68
+ "gpt-transcribe",
69
+ "gpt-live-transcribe",
70
+ "gemini-3.5-transcribe-live",
71
+ "gemini-3.5-transcribe"
72
+ ],
73
+ modelVendors: GEMINI_SPEECH_MODEL_VENDORS,
74
+ options: []
75
+ },
76
+ // One complete recording (an upload of an engine SDK).
77
+ speech_input_file: {
78
+ name: "Speech input (file)",
79
+ source: "openai",
80
+ models: ["gpt-transcribe", "gemini-3.5-transcribe"],
81
+ modelVendors: GEMINI_SPEECH_MODEL_VENDORS,
82
+ options: []
83
+ },
84
+ // Priced by OPENAI_PRICES.text of the generator.
85
+ generator_agent: {
86
+ name: "Generator agent",
87
+ source: "openai",
88
+ models: ["gpt-6.1-sol"],
89
+ options: ["reasoningEffort"]
90
+ },
91
+ generator_reading: {
92
+ name: "Generator face reading",
93
+ source: "openai",
94
+ models: ["gpt-6.1-sol"],
95
+ options: ["reasoningEffort"]
96
+ },
97
+ // Priced by OPENAI_PRICES of the generator (flare and sunburst alike).
98
+ generator_image: {
99
+ name: "Generator images",
100
+ source: "openai",
101
+ models: GPT_IMAGE_MODELS,
102
+ options: []
103
+ }
104
+ };
105
+ var AI_ROLE_NAMES = Object.keys(AI_ROLES);
106
+
107
+ // ../ts-shared/src/speech-input.ts
108
+ var INPUT_MODES = ["options", "text", "voice"];
109
+ var InputModeError = class extends Error {
110
+ constructor(message) {
111
+ super(message);
112
+ this.name = "NpcifyInputModeError";
113
+ }
114
+ };
115
+ var permittedInputModes = function(permissions) {
116
+ if (permissions.baked || !permissions.allowCustomReply) {
117
+ return ["options"];
118
+ }
119
+ return ["options", "text", "voice"];
120
+ };
121
+ var isInputMode = function(value) {
122
+ return INPUT_MODES.includes(value);
123
+ };
124
+ var parseInputModes = function(value) {
125
+ if (value === void 0 || value === null) {
126
+ return void 0;
127
+ }
128
+ if (!Array.isArray(value)) {
129
+ throw new InputModeError(
130
+ 'inputModes must be an array of "options", "text" and "voice".'
131
+ );
132
+ }
133
+ if (value.length === 0) {
134
+ throw new InputModeError("inputModes must have at least one mode.");
135
+ }
136
+ const modes = [];
137
+ for (const mode of value) {
138
+ if (!isInputMode(mode)) {
139
+ throw new InputModeError(
140
+ `inputModes has an unknown mode ${JSON.stringify(mode)}. Use "options", "text" or "voice".`
141
+ );
142
+ }
143
+ if (modes.includes(mode)) {
144
+ throw new InputModeError(`inputModes has "${mode}" two times.`);
145
+ }
146
+ modes.push(mode);
147
+ }
148
+ return modes;
149
+ };
150
+ var resolveInputModes = function(configured, permissions) {
151
+ const permitted = permittedInputModes(permissions);
152
+ if (!configured) {
153
+ return permitted;
154
+ }
155
+ if (permissions.baked && !(configured.length === 1 && configured[0] === "options")) {
156
+ throw new InputModeError(
157
+ 'A baked (offline) conversation accepts only reply options. Set inputModes to ["options"] or remove it.'
158
+ );
159
+ }
160
+ const refused = configured.filter((mode) => !permitted.includes(mode));
161
+ if (refused.length > 0) {
162
+ throw new InputModeError(
163
+ `This character accepts only its reply options (Allow custom replies is off). Remove ${refused.map((mode) => `"${mode}"`).join(" and ")} from inputModes.`
164
+ );
165
+ }
166
+ return [...configured];
167
+ };
168
+ var SPEECH_AUDIO_FORMAT = Object.freeze({
169
+ encoding: "pcm_s16le",
170
+ sampleRate: 16e3,
171
+ channels: 1
172
+ });
173
+ var SPEECH_BYTES_PER_SECOND = 32e3;
174
+ var SPEECH_MAX_DURATION_MS = 3e4;
175
+ var SPEECH_MAX_AUDIO_BYTES = SPEECH_MAX_DURATION_MS / 1e3 * SPEECH_BYTES_PER_SECOND;
176
+ var SPEECH_MAX_UPLOAD_BYTES = SPEECH_MAX_AUDIO_BYTES + 4096;
177
+ var SPEECH_FRAME_MS = 100;
178
+ var SPEECH_MIN_HOLD_MS = 300;
179
+ var SPEECH_EVENTS = Object.freeze({
180
+ start: "speech:start",
181
+ audio: "speech:audio",
182
+ stop: "speech:stop",
183
+ cancel: "speech:cancel",
184
+ partial: "speech:partial",
185
+ turn: "speech:turn",
186
+ turnFailed: "speech:turn_failed"
187
+ });
188
+
12
189
  // ../ts-shared/src/product-usage.ts
13
190
  var productUsageEvents = [
14
191
  "page_viewed",
@@ -17,7 +194,11 @@ var productUsageEvents = [
17
194
  "checkout_submitted",
18
195
  "checkout_failed",
19
196
  "checkout_closed",
20
- "paywall_shown"
197
+ "paywall_shown",
198
+ "playback_blocked",
199
+ "playback_start_pressed",
200
+ "reply_options_shown",
201
+ "page_left"
21
202
  ];
22
203
 
23
204
  // ../ts-shared/src/analytics.ts
@@ -3122,8 +3303,14 @@ var LiveConversation = class {
3122
3303
  this.dependencies = dependencies;
3123
3304
  }
3124
3305
  sending = false;
3306
+ /** A typed turn is in flight. */
3307
+ get inFlight() {
3308
+ return this.sending;
3309
+ }
3125
3310
  pending = null;
3311
+ revision = 0;
3126
3312
  reset() {
3313
+ this.revision += 1;
3127
3314
  this.pending = null;
3128
3315
  }
3129
3316
  received(payload) {
@@ -3136,6 +3323,7 @@ var LiveConversation = class {
3136
3323
  throw new Error("A reply is already being generated. Please wait.");
3137
3324
  }
3138
3325
  this.sending = true;
3326
+ const revision = this.revision;
3139
3327
  const { telemetry, socket } = this.dependencies;
3140
3328
  try {
3141
3329
  telemetry.begin(conversationId);
@@ -3143,10 +3331,10 @@ var LiveConversation = class {
3143
3331
  const silent = options?.silent === true;
3144
3332
  const parentStagingId = options?.parentStagingId;
3145
3333
  const pending = this.pending;
3146
- const request = pending?.conversationId === conversationId && pending.message === message && pending.silent === silent && pending.parentStagingId === parentStagingId ? pending : {
3334
+ const request = !options?.requestId && pending?.conversationId === conversationId && pending.message === message && pending.silent === silent && pending.parentStagingId === parentStagingId ? pending : {
3147
3335
  conversationId,
3148
3336
  message,
3149
- requestId: globalThis.crypto.randomUUID(),
3337
+ requestId: options?.requestId ?? globalThis.crypto.randomUUID(),
3150
3338
  silent,
3151
3339
  parentStagingId
3152
3340
  };
@@ -3180,6 +3368,9 @@ var LiveConversation = class {
3180
3368
  if (this.pending?.requestId === request.requestId) {
3181
3369
  this.pending = null;
3182
3370
  }
3371
+ if (revision !== this.revision) {
3372
+ return payload;
3373
+ }
3183
3374
  await this.dependencies.handleResult(
3184
3375
  payload,
3185
3376
  payload.taskId !== request.requestId,
@@ -4310,7 +4501,7 @@ var disposeSceneContext = function(ctx) {
4310
4501
 
4311
4502
  // ../vrm-renderer/src/preview-controller.ts
4312
4503
  import { VRMUtils as VRMUtils4 } from "@pixiv/three-vrm";
4313
- import { Box3, GridHelper, Vector3 as Vector33 } from "three";
4504
+ import { Box3, GridHelper, Vector3 as Vector34 } from "three";
4314
4505
 
4315
4506
  // ../vrm-renderer/src/loaders.ts
4316
4507
  import { VRMLoaderPlugin as VRMLoaderPlugin2, VRMUtils as VRMUtils2 } from "@pixiv/three-vrm";
@@ -4332,7 +4523,33 @@ import { AnimationMixer, LoopOnce, LoopRepeat } from "three";
4332
4523
  import {
4333
4524
  createVRMAnimationClip as createPixivVRMAnimationClip
4334
4525
  } from "@pixiv/three-vrm-animation";
4526
+ import {
4527
+ AnimationClip,
4528
+ Euler,
4529
+ Quaternion as Quaternion2,
4530
+ QuaternionKeyframeTrack,
4531
+ QuaternionLinearInterpolant,
4532
+ Vector3 as Vector33,
4533
+ VectorKeyframeTrack
4534
+ } from "three";
4335
4535
  var NPCIFY_VRM_NORMALIZATION_ROOT_NAME = "NPCifyNormalizationRoot";
4536
+ var TORSO_BONES = [
4537
+ "hips",
4538
+ "spine",
4539
+ "chest",
4540
+ "upperChest"
4541
+ ];
4542
+ var HEAD_BONES = ["neck", "head"];
4543
+ var AXIAL_BONES = [...TORSO_BONES, ...HEAD_BONES];
4544
+ var PARENT_OF_HEAD_BONES = [
4545
+ ...TORSO_BONES,
4546
+ "neck"
4547
+ ];
4548
+ var IDENTITY_QUATERNION = new Quaternion2();
4549
+ var UP_AXIS = new Vector33(0, 1, 0);
4550
+ var MAX_HEAD_PITCH_CORRECTION_RAD = 35 * Math.PI / 180;
4551
+ var NECK_CORRECTION_SHARE = 0.35;
4552
+ var EPSILON = 1e-5;
4336
4553
  var canonicalAnimationTargets = /* @__PURE__ */ new WeakMap();
4337
4554
  var isNpcifyNormalizedVrm = function(vrm) {
4338
4555
  return vrm.scene.children.some(
@@ -4358,8 +4575,300 @@ var canonicalAnimationTarget = function(vrm) {
4358
4575
  canonicalAnimationTargets.set(vrm, target);
4359
4576
  return target;
4360
4577
  };
4361
- var createVRMAnimationClip = function(animation, vrm) {
4362
- return createPixivVRMAnimationClip(animation, canonicalAnimationTarget(vrm));
4578
+ var resolveAxialBoneNodeNames = function(vrm) {
4579
+ const result = /* @__PURE__ */ new Map();
4580
+ const seenNodeNames = /* @__PURE__ */ new Set();
4581
+ for (const bone of AXIAL_BONES) {
4582
+ const nodeName = vrm.humanoid?.getNormalizedBoneNode?.(bone)?.name;
4583
+ if (nodeName && !seenNodeNames.has(nodeName)) {
4584
+ seenNodeNames.add(nodeName);
4585
+ result.set(bone, nodeName);
4586
+ }
4587
+ }
4588
+ return result;
4589
+ };
4590
+ var findQuaternionTrack = function(clip, nodeName) {
4591
+ if (!nodeName) {
4592
+ return void 0;
4593
+ }
4594
+ const trackName = `${nodeName}.quaternion`;
4595
+ return clip.tracks.find(
4596
+ (track) => track.name === trackName && track.times.length > 0 && track.values.length >= track.times.length * 4
4597
+ );
4598
+ };
4599
+ var sampleBoneQuaternion = function(interpolant, time, target) {
4600
+ if (!interpolant) {
4601
+ return target.identity();
4602
+ }
4603
+ const values = interpolant.evaluate(time);
4604
+ return target.set(values[0] ?? 0, values[1] ?? 0, values[2] ?? 0, values[3] ?? 1).normalize();
4605
+ };
4606
+ var evaluateChainOrientation = function(bones, interpolants, time, rootPreRotation, target, scratch) {
4607
+ target.copy(rootPreRotation);
4608
+ for (const bone of bones) {
4609
+ const interpolant = interpolants.get(bone);
4610
+ if (interpolant) {
4611
+ sampleBoneQuaternion(interpolant, time, scratch);
4612
+ target.multiply(scratch);
4613
+ }
4614
+ }
4615
+ return target;
4616
+ };
4617
+ var rotateQuaternionTrackInParentSpace = function(track, parentBones, interpolants, rootPreRotation, worldCorrection) {
4618
+ const values = new Float32Array(track.values.length);
4619
+ const parentQuat = new Quaternion2();
4620
+ const parentInvQuat = new Quaternion2();
4621
+ const localCorrection = new Quaternion2();
4622
+ const keyframeQuat = new Quaternion2();
4623
+ const prevOrigQuat = new Quaternion2();
4624
+ const prevNewQuat = new Quaternion2();
4625
+ const scratchQuat = new Quaternion2();
4626
+ const count = track.times.length;
4627
+ for (let i = 0; i < count; i += 1) {
4628
+ const time = track.times[i] ?? 0;
4629
+ const offset = i * 4;
4630
+ keyframeQuat.set(
4631
+ track.values[offset] ?? 0,
4632
+ track.values[offset + 1] ?? 0,
4633
+ track.values[offset + 2] ?? 0,
4634
+ track.values[offset + 3] ?? 1
4635
+ ).normalize();
4636
+ const origDot = i > 0 ? prevOrigQuat.dot(keyframeQuat) : 1;
4637
+ prevOrigQuat.copy(keyframeQuat);
4638
+ if (parentBones.length === 0) {
4639
+ keyframeQuat.premultiply(worldCorrection).normalize();
4640
+ } else {
4641
+ evaluateChainOrientation(
4642
+ parentBones,
4643
+ interpolants,
4644
+ time,
4645
+ rootPreRotation,
4646
+ parentQuat,
4647
+ scratchQuat
4648
+ );
4649
+ parentInvQuat.copy(parentQuat).invert();
4650
+ localCorrection.copy(parentInvQuat).multiply(worldCorrection).multiply(parentQuat);
4651
+ keyframeQuat.premultiply(localCorrection).normalize();
4652
+ }
4653
+ if (i > 0 && origDot >= 0 && prevNewQuat.dot(keyframeQuat) < 0) {
4654
+ keyframeQuat.set(
4655
+ -keyframeQuat.x,
4656
+ -keyframeQuat.y,
4657
+ -keyframeQuat.z,
4658
+ -keyframeQuat.w
4659
+ );
4660
+ }
4661
+ prevNewQuat.copy(keyframeQuat);
4662
+ values[offset] = keyframeQuat.x;
4663
+ values[offset + 1] = keyframeQuat.y;
4664
+ values[offset + 2] = keyframeQuat.z;
4665
+ values[offset + 3] = keyframeQuat.w;
4666
+ }
4667
+ return new QuaternionKeyframeTrack(
4668
+ track.name,
4669
+ Array.from(track.times),
4670
+ Array.from(values)
4671
+ );
4672
+ };
4673
+ var rotateHipsPositionTrack = function(track, yawRadians) {
4674
+ const count = Math.floor(track.values.length / 3);
4675
+ if (count === 0) {
4676
+ return new VectorKeyframeTrack(
4677
+ track.name,
4678
+ Array.from(track.times),
4679
+ Array.from(track.values)
4680
+ );
4681
+ }
4682
+ let meanX = 0;
4683
+ let meanZ = 0;
4684
+ for (let i = 0; i < count; i += 1) {
4685
+ meanX += track.values[i * 3] ?? 0;
4686
+ meanZ += track.values[i * 3 + 2] ?? 0;
4687
+ }
4688
+ meanX /= count;
4689
+ meanZ /= count;
4690
+ const cosYaw = Math.cos(yawRadians);
4691
+ const sinYaw = Math.sin(yawRadians);
4692
+ const values = new Float32Array(track.values.length);
4693
+ for (let i = 0; i < count; i += 1) {
4694
+ const offset = i * 3;
4695
+ const dx = (track.values[offset] ?? 0) - meanX;
4696
+ const y = track.values[offset + 1] ?? 0;
4697
+ const dz = (track.values[offset + 2] ?? 0) - meanZ;
4698
+ values[offset] = meanX + dx * cosYaw + dz * sinYaw;
4699
+ values[offset + 1] = y;
4700
+ values[offset + 2] = meanZ - dx * sinYaw + dz * cosYaw;
4701
+ }
4702
+ return new VectorKeyframeTrack(
4703
+ track.name,
4704
+ Array.from(track.times),
4705
+ Array.from(values)
4706
+ );
4707
+ };
4708
+ var normalizeIdleAnimationClip = function(clip, vrm) {
4709
+ const boneNodeNames = resolveAxialBoneNodeNames(vrm);
4710
+ if (boneNodeNames.size === 0) {
4711
+ return clip;
4712
+ }
4713
+ const interpolants = /* @__PURE__ */ new Map();
4714
+ for (const bone of AXIAL_BONES) {
4715
+ const track = findQuaternionTrack(clip, boneNodeNames.get(bone));
4716
+ if (track) {
4717
+ interpolants.set(
4718
+ bone,
4719
+ new QuaternionLinearInterpolant(track.times, track.values, 4)
4720
+ );
4721
+ }
4722
+ }
4723
+ if (interpolants.size === 0) {
4724
+ return clip;
4725
+ }
4726
+ const duration = clip.duration > 0 ? clip.duration : 0;
4727
+ const sampleCount = duration > 0 ? Math.max(16, Math.min(120, Math.ceil(duration * 30))) : 1;
4728
+ const sampleTimes = [];
4729
+ for (let i = 0; i < sampleCount; i += 1) {
4730
+ sampleTimes.push(duration > 0 ? (i + 0.5) / sampleCount * duration : 0);
4731
+ }
4732
+ const chainQuat = new Quaternion2();
4733
+ const scratchQuat = new Quaternion2();
4734
+ const forwardVec = new Vector33();
4735
+ const rootTorsoBone = TORSO_BONES.find((bone) => interpolants.has(bone));
4736
+ const rootPreRotation = new Quaternion2();
4737
+ let torsoYawCorrection = 0;
4738
+ if (rootTorsoBone) {
4739
+ let sumX = 0;
4740
+ let sumZ = 0;
4741
+ for (const time of sampleTimes) {
4742
+ evaluateChainOrientation(
4743
+ TORSO_BONES,
4744
+ interpolants,
4745
+ time,
4746
+ IDENTITY_QUATERNION,
4747
+ chainQuat,
4748
+ scratchQuat
4749
+ );
4750
+ forwardVec.set(0, 0, 1).applyQuaternion(chainQuat);
4751
+ const horizLen = Math.hypot(forwardVec.x, forwardVec.z);
4752
+ if (horizLen > EPSILON) {
4753
+ sumX += forwardVec.x / horizLen;
4754
+ sumZ += forwardVec.z / horizLen;
4755
+ }
4756
+ }
4757
+ if (Math.hypot(sumX, sumZ) > EPSILON) {
4758
+ const meanTorsoYaw = Math.atan2(sumX, sumZ);
4759
+ if (Math.abs(meanTorsoYaw) > EPSILON) {
4760
+ torsoYawCorrection = -meanTorsoYaw;
4761
+ rootPreRotation.setFromAxisAngle(UP_AXIS, torsoYawCorrection);
4762
+ }
4763
+ }
4764
+ }
4765
+ const hasNeckTrack = interpolants.has("neck");
4766
+ const hasHeadTrack = interpolants.has("head");
4767
+ let neckWorldCorrection = null;
4768
+ let headWorldCorrection = null;
4769
+ if (hasNeckTrack || hasHeadTrack) {
4770
+ const meanHeadForward = new Vector33();
4771
+ for (const time of sampleTimes) {
4772
+ evaluateChainOrientation(
4773
+ AXIAL_BONES,
4774
+ interpolants,
4775
+ time,
4776
+ rootPreRotation,
4777
+ chainQuat,
4778
+ scratchQuat
4779
+ );
4780
+ forwardVec.set(0, 0, 1).applyQuaternion(chainQuat);
4781
+ meanHeadForward.add(forwardVec);
4782
+ }
4783
+ if (meanHeadForward.lengthSq() > EPSILON * EPSILON) {
4784
+ meanHeadForward.normalize();
4785
+ const meanHeadYaw = Math.atan2(meanHeadForward.x, meanHeadForward.z);
4786
+ const rawHeadPitch = Math.atan2(
4787
+ -meanHeadForward.y,
4788
+ Math.hypot(meanHeadForward.x, meanHeadForward.z)
4789
+ );
4790
+ const clampedHeadPitch = Math.max(
4791
+ -MAX_HEAD_PITCH_CORRECTION_RAD,
4792
+ Math.min(MAX_HEAD_PITCH_CORRECTION_RAD, rawHeadPitch)
4793
+ );
4794
+ if (Math.abs(meanHeadYaw) > EPSILON || Math.abs(clampedHeadPitch) > EPSILON) {
4795
+ const totalWorldCorrection = new Quaternion2().setFromEuler(
4796
+ new Euler(-clampedHeadPitch, -meanHeadYaw, 0, "XYZ")
4797
+ );
4798
+ if (hasNeckTrack && hasHeadTrack) {
4799
+ neckWorldCorrection = new Quaternion2().setFromEuler(
4800
+ new Euler(
4801
+ -clampedHeadPitch * NECK_CORRECTION_SHARE,
4802
+ -meanHeadYaw * NECK_CORRECTION_SHARE,
4803
+ 0,
4804
+ "XYZ"
4805
+ )
4806
+ );
4807
+ headWorldCorrection = neckWorldCorrection.clone().invert().multiply(totalWorldCorrection);
4808
+ } else if (hasNeckTrack) {
4809
+ neckWorldCorrection = totalWorldCorrection;
4810
+ } else {
4811
+ headWorldCorrection = totalWorldCorrection;
4812
+ }
4813
+ }
4814
+ }
4815
+ }
4816
+ const hasTorsoCorrection = Math.abs(torsoYawCorrection) > EPSILON;
4817
+ if (!hasTorsoCorrection && !neckWorldCorrection && !headWorldCorrection) {
4818
+ return clip;
4819
+ }
4820
+ const rootTorsoNodeName = rootTorsoBone ? boneNodeNames.get(rootTorsoBone) : void 0;
4821
+ const rootTorsoQuatTrackName = rootTorsoNodeName ? `${rootTorsoNodeName}.quaternion` : null;
4822
+ const hipsNodeName = boneNodeNames.get("hips");
4823
+ const hipsPosTrackName = hipsNodeName && rootTorsoBone === "hips" ? `${hipsNodeName}.position` : null;
4824
+ const neckNodeName = boneNodeNames.get("neck");
4825
+ const neckQuatTrackName = neckNodeName ? `${neckNodeName}.quaternion` : null;
4826
+ const headNodeName = boneNodeNames.get("head");
4827
+ const headQuatTrackName = headNodeName ? `${headNodeName}.quaternion` : null;
4828
+ const nextTracks = clip.tracks.map((track) => {
4829
+ if (hasTorsoCorrection && track.name === rootTorsoQuatTrackName) {
4830
+ return rotateQuaternionTrackInParentSpace(
4831
+ track,
4832
+ [],
4833
+ interpolants,
4834
+ IDENTITY_QUATERNION,
4835
+ rootPreRotation
4836
+ );
4837
+ }
4838
+ if (hasTorsoCorrection && track.name === hipsPosTrackName) {
4839
+ return rotateHipsPositionTrack(track, torsoYawCorrection);
4840
+ }
4841
+ if (neckWorldCorrection && track.name === neckQuatTrackName) {
4842
+ return rotateQuaternionTrackInParentSpace(
4843
+ track,
4844
+ TORSO_BONES,
4845
+ interpolants,
4846
+ rootPreRotation,
4847
+ neckWorldCorrection
4848
+ );
4849
+ }
4850
+ if (headWorldCorrection && track.name === headQuatTrackName) {
4851
+ return rotateQuaternionTrackInParentSpace(
4852
+ track,
4853
+ PARENT_OF_HEAD_BONES,
4854
+ interpolants,
4855
+ rootPreRotation,
4856
+ headWorldCorrection
4857
+ );
4858
+ }
4859
+ return track;
4860
+ });
4861
+ return new AnimationClip(clip.name, clip.duration, nextTracks, clip.blendMode);
4862
+ };
4863
+ var createVRMAnimationClip = function(animation, vrm, options) {
4864
+ const clip = createPixivVRMAnimationClip(
4865
+ animation,
4866
+ canonicalAnimationTarget(vrm)
4867
+ );
4868
+ if (options?.normalizeIdleOrientation) {
4869
+ return normalizeIdleAnimationClip(clip, vrm);
4870
+ }
4871
+ return clip;
4363
4872
  };
4364
4873
 
4365
4874
  // ../vrm-renderer/src/session.ts
@@ -4696,16 +5205,20 @@ var RemoteAnimationController = class {
4696
5205
  }
4697
5206
  };
4698
5207
  var AnimationController = class {
4699
- constructor(vrm, idleAnimationId) {
5208
+ constructor(vrm, idleAnimationId, options) {
4700
5209
  this.vrm = vrm;
4701
5210
  this.mixer = new AnimationMixer2(this.vrm.scene);
4702
5211
  this.idleAnimationId = idleAnimationId ?? null;
5212
+ this.normalizeIdleOrientation = options?.normalizeIdleOrientation ?? false;
4703
5213
  this.mixer.addEventListener("finished", this.handleActionFinished);
4704
5214
  }
4705
5215
  prepareAnimations;
4706
5216
  mixer;
4707
5217
  clips = /* @__PURE__ */ new Map();
5218
+ rawClips = /* @__PURE__ */ new Map();
5219
+ normalizedIdleClips = /* @__PURE__ */ new Map();
4708
5220
  configs = /* @__PURE__ */ new Map();
5221
+ normalizeIdleOrientation;
4709
5222
  previewMixer = null;
4710
5223
  restorePreview = null;
4711
5224
  activeAction = null;
@@ -4731,8 +5244,10 @@ var AnimationController = class {
4731
5244
  return this.clips.has(id);
4732
5245
  }
4733
5246
  addAnimation(animationId, animation, config) {
4734
- const clip = createVRMAnimationClip(animation, this.vrm);
4735
- this.clips.set(animationId, clip);
5247
+ const rawClip = createVRMAnimationClip(animation, this.vrm);
5248
+ this.rawClips.set(animationId, rawClip);
5249
+ this.normalizedIdleClips.delete(animationId);
5250
+ this.clips.set(animationId, this.resolveClipForId(animationId, rawClip));
4736
5251
  this.configs.set(animationId, {
4737
5252
  loop: config?.loop ?? defaultAnimationConfig.loop,
4738
5253
  weight: config?.weight ?? defaultAnimationConfig.weight
@@ -4762,7 +5277,20 @@ var AnimationController = class {
4762
5277
  return restore;
4763
5278
  }
4764
5279
  setIdleAnimation(animationId) {
5280
+ const previousIdleId = this.idleAnimationId;
4765
5281
  this.idleAnimationId = animationId;
5282
+ if (previousIdleId && previousIdleId !== animationId) {
5283
+ const previousRaw = this.rawClips.get(previousIdleId);
5284
+ if (previousRaw) {
5285
+ this.clips.set(previousIdleId, previousRaw);
5286
+ }
5287
+ }
5288
+ if (animationId) {
5289
+ const rawClip = this.rawClips.get(animationId);
5290
+ if (rawClip) {
5291
+ this.clips.set(animationId, this.resolveClipForId(animationId, rawClip));
5292
+ }
5293
+ }
4766
5294
  this.playIdle();
4767
5295
  }
4768
5296
  clearAnimations() {
@@ -4770,6 +5298,8 @@ var AnimationController = class {
4770
5298
  this.mixer.stopAllAction();
4771
5299
  this.mixer.uncacheRoot(this.vrm.scene);
4772
5300
  this.clips.clear();
5301
+ this.rawClips.clear();
5302
+ this.normalizedIdleClips.clear();
4773
5303
  this.configs.clear();
4774
5304
  this.activeAction = null;
4775
5305
  this.activeAnimationId = null;
@@ -4777,6 +5307,18 @@ var AnimationController = class {
4777
5307
  this.idleAnimationId = null;
4778
5308
  this.vrm.humanoid.resetNormalizedPose();
4779
5309
  }
5310
+ resolveClipForId(animationId, rawClip) {
5311
+ if (!this.normalizeIdleOrientation || animationId !== this.idleAnimationId) {
5312
+ return rawClip;
5313
+ }
5314
+ const cached = this.normalizedIdleClips.get(animationId);
5315
+ if (cached) {
5316
+ return cached;
5317
+ }
5318
+ const normalized = normalizeIdleAnimationClip(rawClip, this.vrm);
5319
+ this.normalizedIdleClips.set(animationId, normalized);
5320
+ return normalized;
5321
+ }
4780
5322
  playAnimation(animationId, fadeDuration = 0.25, loopOverride) {
4781
5323
  this.restorePreview?.();
4782
5324
  const clip = this.clips.get(animationId);
@@ -5545,6 +6087,7 @@ var StreamPlaybackController = class {
5545
6087
  force: options?.force === true
5546
6088
  });
5547
6089
  }
6090
+ /** Returns the playback result for a timeline event. */
5548
6091
  handleEvent(event) {
5549
6092
  if (this.streamEventCallback) {
5550
6093
  this.safeInvoke("onStreamEvent", () => this.streamEventCallback?.(event));
@@ -5588,14 +6131,13 @@ var StreamPlaybackController = class {
5588
6131
  if (event.type === "staging" || event.type === "staging_prefix" || event.type === "staging_live") {
5589
6132
  this.recordStreamEvent(event, "timeline");
5590
6133
  this.dependencies.telemetry.recordFirstStaging();
5591
- void this.playTimeline({
6134
+ return this.playTimeline({
5592
6135
  taskId: event.taskId,
5593
6136
  stagingId: event.stagingId,
5594
6137
  timeline: event.timeline,
5595
6138
  complete: event.complete,
5596
6139
  revision: event.revision
5597
6140
  });
5598
- return;
5599
6141
  }
5600
6142
  if (event.type === "options") {
5601
6143
  this.recordStreamEvent(event, "options");
@@ -5954,8 +6496,18 @@ var decodeBase64 = function(value) {
5954
6496
  }
5955
6497
  return bytes;
5956
6498
  };
5957
- var canUseMediaSource = function(mimeType) {
5958
- return typeof MediaSource !== "undefined" && typeof URL?.createObjectURL === "function" && MediaSource.isTypeSupported(mimeType);
6499
+ var managedMediaSource = function() {
6500
+ return globalThis.ManagedMediaSource;
6501
+ };
6502
+ var needsManagedMediaSource = function() {
6503
+ return typeof MediaSource === "undefined" && Boolean(managedMediaSource());
6504
+ };
6505
+ var streamingMediaSource = function(mimeType) {
6506
+ if (typeof URL?.createObjectURL !== "function") {
6507
+ return null;
6508
+ }
6509
+ const source = typeof MediaSource === "undefined" ? managedMediaSource() : MediaSource;
6510
+ return source?.isTypeSupported(mimeType) ? source : null;
5959
6511
  };
5960
6512
  var StreamingAudioBuffer = class {
5961
6513
  constructor(mimeType) {
@@ -5998,8 +6550,9 @@ var StreamingAudioBuffer = class {
5998
6550
  if (this.objectUrl) {
5999
6551
  return this.objectUrl;
6000
6552
  }
6001
- if (canUseMediaSource(this.mimeType)) {
6002
- this.mediaSource = new MediaSource();
6553
+ const Source2 = streamingMediaSource(this.mimeType);
6554
+ if (Source2) {
6555
+ this.mediaSource = new Source2();
6003
6556
  this.objectUrl = URL.createObjectURL(this.mediaSource);
6004
6557
  this.mediaSource.addEventListener("sourceopen", this.handleSourceOpen, {
6005
6558
  once: true
@@ -6091,7 +6644,7 @@ var StreamingAudioBuffer = class {
6091
6644
  }
6092
6645
  };
6093
6646
  finishFallbackIfReady() {
6094
- if (!this.completed || this.objectUrl || canUseMediaSource(this.mimeType) || this.failedError || this.disposed) {
6647
+ if (!this.completed || this.objectUrl || streamingMediaSource(this.mimeType) || this.failedError || this.disposed) {
6095
6648
  return;
6096
6649
  }
6097
6650
  const blobParts = this.chunks.map((chunk) => Uint8Array.from(chunk).buffer);
@@ -6418,7 +6971,8 @@ var AudioPlayer = class {
6418
6971
  this.visemePlayer.cleanup();
6419
6972
  if (this.audioElement) {
6420
6973
  this.audioElement.srcObject = null;
6421
- this.audioElement.src = "";
6974
+ this.audioElement.removeAttribute("src");
6975
+ this.audioElement.load();
6422
6976
  }
6423
6977
  this.streams.forEach((stream) => stream.dispose());
6424
6978
  this.streams.clear();
@@ -6435,8 +6989,10 @@ var AudioPlayer = class {
6435
6989
  this.streams.delete(streamId);
6436
6990
  }
6437
6991
  buildSilentWav = function() {
6438
- const sampleCount = 8;
6439
- const bytes = new Uint8Array(44 + sampleCount);
6992
+ const sampleRate = 44100;
6993
+ const bytesPerSample = 2;
6994
+ const dataSize = Math.round(sampleRate * 0.1) * bytesPerSample;
6995
+ const bytes = new Uint8Array(44 + dataSize);
6440
6996
  const view = new DataView(bytes.buffer);
6441
6997
  const writeText = function(offset, value) {
6442
6998
  Array.from(value).forEach((character, index) => {
@@ -6444,19 +7000,18 @@ var AudioPlayer = class {
6444
7000
  });
6445
7001
  };
6446
7002
  writeText(0, "RIFF");
6447
- view.setUint32(4, 36 + sampleCount, true);
7003
+ view.setUint32(4, 36 + dataSize, true);
6448
7004
  writeText(8, "WAVE");
6449
7005
  writeText(12, "fmt ");
6450
7006
  view.setUint32(16, 16, true);
6451
7007
  view.setUint16(20, 1, true);
6452
7008
  view.setUint16(22, 1, true);
6453
- view.setUint32(24, 8e3, true);
6454
- view.setUint32(28, 8e3, true);
6455
- view.setUint16(32, 1, true);
6456
- view.setUint16(34, 8, true);
7009
+ view.setUint32(24, sampleRate, true);
7010
+ view.setUint32(28, sampleRate * bytesPerSample, true);
7011
+ view.setUint16(32, bytesPerSample, true);
7012
+ view.setUint16(34, bytesPerSample * 8, true);
6457
7013
  writeText(36, "data");
6458
- view.setUint32(40, sampleCount, true);
6459
- bytes.fill(128, 44);
7014
+ view.setUint32(40, dataSize, true);
6460
7015
  return new Blob([bytes], { type: "audio/wav" });
6461
7016
  };
6462
7017
  async resolveSourceUrl(source) {
@@ -6492,6 +7047,9 @@ var AudioPlayer = class {
6492
7047
  element.volume = this.volumeValue;
6493
7048
  element.muted = this.mutedValue;
6494
7049
  element.playbackRate = this.playbackRateValue;
7050
+ if (needsManagedMediaSource()) {
7051
+ element.disableRemotePlayback = true;
7052
+ }
6495
7053
  if (this.crossOriginValue !== null) {
6496
7054
  element.crossOrigin = this.crossOriginValue;
6497
7055
  }
@@ -6575,6 +7133,14 @@ var CharacterExpressions = class {
6575
7133
  clearEmotion(mouthMode) {
6576
7134
  this.mixer.clearEmotion(mouthMode);
6577
7135
  }
7136
+ /** Set a temporary emotion overlay. Timeline emotions and visemes keep their state. */
7137
+ setOverlay(weights) {
7138
+ this.mixer.setOverlay(weights);
7139
+ }
7140
+ /** Ease the temporary overlay back to the current timeline or idle face. */
7141
+ clearOverlay() {
7142
+ this.mixer.clearOverlay();
7143
+ }
6578
7144
  /** Reset all expression layer targets to zero. */
6579
7145
  reset() {
6580
7146
  this.mixer.reset();
@@ -6749,7 +7315,7 @@ var FacialPerformanceInspector = class {
6749
7315
  };
6750
7316
 
6751
7317
  // ../../clients/web-client/src/character/idle-face.ts
6752
- import { Vector3 as Vector34 } from "three";
7318
+ import { Vector3 as Vector35 } from "three";
6753
7319
  var IdleFace = class {
6754
7320
  constructor(vrm, random) {
6755
7321
  this.vrm = vrm;
@@ -6757,7 +7323,7 @@ var IdleFace = class {
6757
7323
  }
6758
7324
  time = 0;
6759
7325
  timeline;
6760
- targetPosition = new Vector34();
7326
+ targetPosition = new Vector35();
6761
7327
  /** Advances by `delta` seconds, turns the eyes, and returns the blink weight. */
6762
7328
  update(delta, frame) {
6763
7329
  if (Number.isFinite(delta) && delta > 0) {
@@ -6945,6 +7511,7 @@ var PerformanceWeightMixer = class {
6945
7511
  };
6946
7512
 
6947
7513
  // ../../clients/web-client/src/character/facial-performance-mixer.ts
7514
+ var VisemeResponseRate = 30;
6948
7515
  var DEFAULT_PLAY_DURATION_MS = 1500;
6949
7516
  var MOUTH_CHANNELS = [
6950
7517
  VRMExpressionPresetName.Aa,
@@ -6972,7 +7539,8 @@ var FacialPerformanceLayer = {
6972
7539
  Emotion: "emotion",
6973
7540
  Viseme: "viseme",
6974
7541
  Blink: "blink",
6975
- Gaze: "gaze"
7542
+ Gaze: "gaze",
7543
+ Overlay: "overlay"
6976
7544
  };
6977
7545
  var getFacialPerformanceLayerId = function(channel) {
6978
7546
  const normalized = channel.toLowerCase();
@@ -7123,6 +7691,22 @@ var FacialPerformanceMixer = class {
7123
7691
  }
7124
7692
  this.releaseVisemeMouthMode();
7125
7693
  }
7694
+ /** Blend temporary emotions without changing timeline targets or mouth ownership. */
7695
+ setOverlay(weights) {
7696
+ this.clearOverlay();
7697
+ Object.entries(weights).forEach(([channel, weight]) => {
7698
+ if (this.availableChannelSet.has(channel) && getFacialPerformanceLayerId(channel) === FacialPerformanceLayer.Emotion && Number.isFinite(weight)) {
7699
+ this.weightMixer.setTarget(
7700
+ FacialPerformanceLayer.Overlay,
7701
+ channel,
7702
+ weight
7703
+ );
7704
+ }
7705
+ });
7706
+ }
7707
+ clearOverlay() {
7708
+ this.weightMixer.clearLayerTargets(FacialPerformanceLayer.Overlay);
7709
+ }
7126
7710
  releaseVisemeMouthMode() {
7127
7711
  if (this.activeMouthMode !== "viseme") {
7128
7712
  return;
@@ -7240,7 +7824,9 @@ var FacialPerformanceMixer = class {
7240
7824
  this.weightMixer.registerLayer({
7241
7825
  id: FacialPerformanceLayer.Viseme,
7242
7826
  priority: 200,
7243
- blendMode: "override"
7827
+ blendMode: "override",
7828
+ // A 33 ms time constant: an 80 ms lip closure reaches 90% closed.
7829
+ responseRate: VisemeResponseRate
7244
7830
  });
7245
7831
  this.weightMixer.registerLayer({
7246
7832
  id: FacialPerformanceLayer.Blink,
@@ -7252,6 +7838,11 @@ var FacialPerformanceMixer = class {
7252
7838
  priority: 400,
7253
7839
  blendMode: "override"
7254
7840
  });
7841
+ this.weightMixer.registerLayer({
7842
+ id: FacialPerformanceLayer.Overlay,
7843
+ priority: 500,
7844
+ blendMode: "maximum"
7845
+ });
7255
7846
  }
7256
7847
  updateIdleFace(delta, values) {
7257
7848
  const manager = this.vrm?.expressionManager;
@@ -7376,11 +7967,11 @@ var FacialPerformanceMixer = class {
7376
7967
 
7377
7968
  // ../../clients/web-client/src/character/mesh.ts
7378
7969
  import {
7379
- Euler,
7970
+ Euler as Euler2,
7380
7971
  Mesh,
7381
7972
  Object3D,
7382
7973
  SkinnedMesh,
7383
- Vector3 as Vector35
7974
+ Vector3 as Vector36
7384
7975
  } from "three";
7385
7976
  var isSkinnedMesh = function(object) {
7386
7977
  return object instanceof SkinnedMesh || object.isSkinnedMesh === true;
@@ -7389,14 +7980,14 @@ var isMesh = function(object) {
7389
7980
  return isSkinnedMesh(object) || object instanceof Mesh || object.isMesh === true;
7390
7981
  };
7391
7982
  var applyVector3 = function(target, value) {
7392
- if (value instanceof Vector35) {
7983
+ if (value instanceof Vector36) {
7393
7984
  target.copy(value);
7394
7985
  return;
7395
7986
  }
7396
7987
  target.set(value[0], value[1], value[2]);
7397
7988
  };
7398
7989
  var applyEuler = function(target, value) {
7399
- if (value instanceof Euler) {
7990
+ if (value instanceof Euler2) {
7400
7991
  target.copy(value);
7401
7992
  return;
7402
7993
  }
@@ -7686,6 +8277,11 @@ var IncrementalStaging = class {
7686
8277
  const dialogue = timeline.dialogue.map((next, index) => {
7687
8278
  const previous = this.timeline.dialogue[index];
7688
8279
  if (previous?.type === "speech" && previous.pending && next.type === "speech") {
8280
+ for (const key of Object.keys(previous)) {
8281
+ if (!(key in next)) {
8282
+ Reflect.deleteProperty(previous, key);
8283
+ }
8284
+ }
7689
8285
  return Object.assign(previous, next, { pending: next.pending });
7690
8286
  }
7691
8287
  return next;
@@ -7748,6 +8344,30 @@ var IncrementalStaging = class {
7748
8344
  }
7749
8345
  };
7750
8346
 
8347
+ // ../../clients/web-client/src/character/playback-clock.ts
8348
+ var PlaybackClock = class {
8349
+ pausedAt = null;
8350
+ pausedSeconds = 0;
8351
+ wallTime() {
8352
+ return performance.now() / 1e3;
8353
+ }
8354
+ get paused() {
8355
+ return this.pausedAt !== null;
8356
+ }
8357
+ now() {
8358
+ return (this.pausedAt ?? this.wallTime()) - this.pausedSeconds;
8359
+ }
8360
+ pause() {
8361
+ this.pausedAt ??= this.wallTime();
8362
+ }
8363
+ resume() {
8364
+ if (this.pausedAt !== null) {
8365
+ this.pausedSeconds += this.wallTime() - this.pausedAt;
8366
+ this.pausedAt = null;
8367
+ }
8368
+ }
8369
+ };
8370
+
7751
8371
  // ../../clients/web-client/src/character/staging-character-cues.ts
7752
8372
  var findCharacterSequence = function(characters, sequence, fromIndex) {
7753
8373
  if (sequence.length === 0) {
@@ -7849,12 +8469,6 @@ var buildStagingCharacterCues = function(text, wordTimings, duration, characterT
7849
8469
  };
7850
8470
 
7851
8471
  // ../../clients/web-client/src/character/staging-player.ts
7852
- var nowSeconds = function() {
7853
- if (typeof performance !== "undefined") {
7854
- return performance.now() / 1e3;
7855
- }
7856
- return Date.now() / 1e3;
7857
- };
7858
8472
  var abortError2 = function() {
7859
8473
  const error = new Error("Staging playback was cancelled");
7860
8474
  error.name = "AbortError";
@@ -7886,6 +8500,8 @@ var StagingPlayer = class {
7886
8500
  this.facialMixer = facialMixer;
7887
8501
  this.animationController = animationController;
7888
8502
  }
8503
+ clock = new PlaybackClock();
8504
+ pauseRevision = 0;
7889
8505
  playbackController = null;
7890
8506
  generation = 0;
7891
8507
  incremental = null;
@@ -7895,6 +8511,40 @@ var StagingPlayer = class {
7895
8511
  activeExpression = null;
7896
8512
  activeExpressionMouthMode = null;
7897
8513
  activeAnimation = null;
8514
+ get paused() {
8515
+ return this.clock.paused;
8516
+ }
8517
+ pause() {
8518
+ this.pauseRevision += 1;
8519
+ this.clock.pause();
8520
+ this.audioPlayer.element?.pause();
8521
+ }
8522
+ async resume() {
8523
+ if (!this.paused) {
8524
+ return;
8525
+ }
8526
+ const generation = this.generation;
8527
+ const pauseRevision = this.pauseRevision;
8528
+ if (this.pendingSpeech?.started && this.pendingSpeech.audio.type !== "silent") {
8529
+ await this.audioPlayer.element?.play();
8530
+ }
8531
+ if (generation !== this.generation) {
8532
+ return;
8533
+ }
8534
+ if (pauseRevision !== this.pauseRevision) {
8535
+ this.audioPlayer.element?.pause();
8536
+ return;
8537
+ }
8538
+ this.clock.resume();
8539
+ }
8540
+ async waitUntilResumed(signal) {
8541
+ while (this.paused) {
8542
+ await waitFrame(signal);
8543
+ }
8544
+ if (signal.aborted) {
8545
+ throw abortError2();
8546
+ }
8547
+ }
7898
8548
  async play(timeline, options) {
7899
8549
  this.cancel(false);
7900
8550
  const controller = new AbortController();
@@ -7933,6 +8583,7 @@ var StagingPlayer = class {
7933
8583
  if (controller.signal.aborted) {
7934
8584
  throw abortError2();
7935
8585
  }
8586
+ await this.waitUntilResumed(controller.signal);
7936
8587
  if (block.startTime > cursor) {
7937
8588
  beginTimeline();
7938
8589
  await this.playTimedRange(
@@ -8016,12 +8667,12 @@ var StagingPlayer = class {
8016
8667
  }
8017
8668
  }
8018
8669
  startCursor(block, mediaClock) {
8019
- const startedAt = nowSeconds();
8670
+ const startedAt = this.clock.now();
8020
8671
  this.cursor = () => ({
8021
8672
  blockId: block.id,
8022
8673
  offsetSeconds: Math.min(
8023
8674
  block.endTime - block.startTime,
8024
- Math.max(0, mediaClock ? mediaClock() : nowSeconds() - startedAt)
8675
+ Math.max(0, mediaClock ? mediaClock() : this.clock.now() - startedAt)
8025
8676
  )
8026
8677
  });
8027
8678
  this.onCursor?.(this.cursor());
@@ -8045,6 +8696,7 @@ var StagingPlayer = class {
8045
8696
  this.generation += 1;
8046
8697
  this.playbackController?.abort();
8047
8698
  this.playbackController = null;
8699
+ this.clock.resume();
8048
8700
  this.incremental = null;
8049
8701
  this.clearCursor();
8050
8702
  this.onCursor = void 0;
@@ -8156,6 +8808,9 @@ var StagingPlayer = class {
8156
8808
  return;
8157
8809
  }
8158
8810
  started = true;
8811
+ if (this.paused) {
8812
+ this.audioPlayer.element?.pause();
8813
+ }
8159
8814
  if (this.pendingSpeech?.id === speech.id) {
8160
8815
  this.pendingSpeech.started = true;
8161
8816
  }
@@ -8176,6 +8831,7 @@ var StagingPlayer = class {
8176
8831
  let ticking = true;
8177
8832
  const tick = async () => {
8178
8833
  while (ticking && !signal.aborted) {
8834
+ await this.waitUntilResumed(signal);
8179
8835
  if (started) {
8180
8836
  observePhonemes();
8181
8837
  const localTime = Math.min(
@@ -8214,14 +8870,15 @@ var StagingPlayer = class {
8214
8870
  }
8215
8871
  }
8216
8872
  async playSilentSpeech(speech, timeline, signal, revealCharacters) {
8217
- const startedAt = nowSeconds();
8873
+ const startedAt = this.clock.now();
8218
8874
  const duration = Math.max(speech.endTime - speech.startTime, 0);
8219
8875
  let nextVisemeIndex = 0;
8220
8876
  while (true) {
8221
8877
  if (signal.aborted) {
8222
8878
  throw abortError2();
8223
8879
  }
8224
- const elapsed = Math.min(nowSeconds() - startedAt, duration);
8880
+ await this.waitUntilResumed(signal);
8881
+ const elapsed = Math.min(this.clock.now() - startedAt, duration);
8225
8882
  this.applyTracks(speech.startTime + elapsed, true, timeline);
8226
8883
  revealCharacters(elapsed);
8227
8884
  let viseme = speech.visemes[nextVisemeIndex];
@@ -8241,13 +8898,14 @@ var StagingPlayer = class {
8241
8898
  revealCharacters(Number.POSITIVE_INFINITY);
8242
8899
  }
8243
8900
  async playTimedRange(startTime, endTime, speechActive, timeline, signal) {
8244
- const startedAt = nowSeconds();
8901
+ const startedAt = this.clock.now();
8245
8902
  const duration = Math.max(endTime - startTime, 0);
8246
8903
  while (true) {
8247
8904
  if (signal.aborted) {
8248
8905
  throw abortError2();
8249
8906
  }
8250
- const elapsed = Math.min(nowSeconds() - startedAt, duration);
8907
+ await this.waitUntilResumed(signal);
8908
+ const elapsed = Math.min(this.clock.now() - startedAt, duration);
8251
8909
  this.applyTracks(startTime + elapsed, speechActive, timeline);
8252
8910
  if (elapsed >= duration) {
8253
8911
  return;
@@ -8747,13 +9405,10 @@ var CharacterRuntime = class {
8747
9405
  animationId,
8748
9406
  assetProfile: this.assetProfile
8749
9407
  }));
8750
- this.lazyAnimations = new LazyAnimations(
8751
- loadAnimation,
8752
- (id, animation) => {
8753
- this.animations.set(id, animation);
8754
- this.animationController?.addAnimation?.(id, animation, { loop: false });
8755
- }
8756
- );
9408
+ this.lazyAnimations = new LazyAnimations(loadAnimation, (id, animation) => {
9409
+ this.animations.set(id, animation);
9410
+ this.animationController?.addAnimation?.(id, animation, { loop: false });
9411
+ });
8757
9412
  this.performanceRecorder = new PerformanceRecorder({
8758
9413
  characterId: this.characterId
8759
9414
  });
@@ -8973,6 +9628,20 @@ var CharacterRuntime = class {
8973
9628
  });
8974
9629
  return unlocked;
8975
9630
  }
9631
+ /** Freeze the current NPCP timeline, audio, face, and animation. */
9632
+ pausePlayback() {
9633
+ this.stagingPlayer?.pause();
9634
+ }
9635
+ /** Continue the same timeline. Call from a gesture if the browser requires it. */
9636
+ async resumePlayback() {
9637
+ await this.stagingPlayer?.resume();
9638
+ }
9639
+ /** Cancel current output and pending audio/animation loads. */
9640
+ interruptPlayback() {
9641
+ if (this.animationController) {
9642
+ this.streamPlayback.cleanup();
9643
+ }
9644
+ }
8976
9645
  getExpressions() {
8977
9646
  return this.facialMixer.getChannelNames();
8978
9647
  }
@@ -9016,6 +9685,20 @@ var CharacterRuntime = class {
9016
9685
  clearLookAt() {
9017
9686
  this.setGazeTarget(null);
9018
9687
  }
9688
+ /** Briefly look at an object; release restores the prior target and gaze mode. */
9689
+ lookAtTemporarily(target) {
9690
+ const previous = this.currentGazeTarget;
9691
+ const wasActive = this.lookAtActive;
9692
+ this.lookAt(target);
9693
+ return () => {
9694
+ if (this.currentGazeTarget !== target || !this.lookAtActive) {
9695
+ return;
9696
+ }
9697
+ this.currentGazeTarget = previous;
9698
+ this.lookAtActive = wasActive;
9699
+ this.applyGaze();
9700
+ };
9701
+ }
9019
9702
  startPerformanceRecording(options) {
9020
9703
  this.performanceRecorder.start(options);
9021
9704
  this.facialMixer.capturePerformanceModel();
@@ -9064,6 +9747,9 @@ var CharacterRuntime = class {
9064
9747
  }
9065
9748
  /** Advance skeletal animations, lip-sync visemes, facial expressions, and VRM physics by `delta` seconds. */
9066
9749
  update(delta) {
9750
+ if (this.stagingPlayer?.paused) {
9751
+ return;
9752
+ }
9067
9753
  this.animationController?.update(delta);
9068
9754
  this.visemePlayer.update(delta);
9069
9755
  this.facialMixer.update(delta);
@@ -9153,6 +9839,18 @@ var ConversationSocket = class {
9153
9839
  this.socket?.disconnect();
9154
9840
  this.socket = null;
9155
9841
  }
9842
+ /** An event with an acknowledgement. Needs `ensureConnected` first. */
9843
+ async request(event, payload, timeoutMs = 1e4) {
9844
+ const socket = this.socket;
9845
+ if (!socket?.connected) {
9846
+ throw new Error("The conversation connection is not open.");
9847
+ }
9848
+ return await socket.timeout(timeoutMs).emitWithAck(event, payload);
9849
+ }
9850
+ /** An event without an acknowledgement, in order with the other events. */
9851
+ send(event, payload) {
9852
+ this.socket?.emit(event, payload);
9853
+ }
9156
9854
  async ensureConnected(conversationId) {
9157
9855
  if (typeof window === "undefined") {
9158
9856
  return;
@@ -9207,6 +9905,41 @@ var ConversationSocket = class {
9207
9905
  }
9208
9906
  this.options.handleStreamEvent(event);
9209
9907
  });
9908
+ this.socket.on(SPEECH_EVENTS.partial, (payload) => {
9909
+ const event = payload;
9910
+ if (typeof event?.sessionId === "string" && typeof event.text === "string") {
9911
+ this.options.handleSpeechEvent?.({ type: "partial", event });
9912
+ }
9913
+ });
9914
+ this.socket.on(SPEECH_EVENTS.turn, (payload) => {
9915
+ let candidate;
9916
+ try {
9917
+ candidate = decodePerformance(
9918
+ payload,
9919
+ "turn"
9920
+ );
9921
+ } catch (error) {
9922
+ this.options.handleCallbackError("speech.turn.decode", error);
9923
+ return;
9924
+ }
9925
+ if (candidate?.conversationId !== this.options.getConversationId() || typeof candidate.requestId !== "string" || !candidate.turn || typeof candidate.turn !== "object") {
9926
+ return;
9927
+ }
9928
+ this.options.handleSpeechEvent?.({
9929
+ type: "turn",
9930
+ requestId: candidate.requestId,
9931
+ turn: candidate.turn
9932
+ });
9933
+ });
9934
+ this.socket.on(SPEECH_EVENTS.turnFailed, (payload) => {
9935
+ const event = payload;
9936
+ if (typeof event?.requestId === "string" && event.conversationId === this.options.getConversationId()) {
9937
+ this.options.handleSpeechEvent?.({ type: "turnFailed", event });
9938
+ }
9939
+ });
9940
+ this.socket.on("disconnect", () => {
9941
+ this.options.handleSpeechEvent?.({ type: "disconnect" });
9942
+ });
9210
9943
  this.socket.on("conversations:turn", (payload) => {
9211
9944
  let candidate;
9212
9945
  try {
@@ -9259,59 +9992,1059 @@ var ConversationSocket = class {
9259
9992
  }
9260
9993
  };
9261
9994
 
9262
- // ../../clients/web-client/src/npcify.ts
9263
- var Character = class extends CharacterRuntime {
9264
- conversationSocket;
9265
- renderSessionId = null;
9266
- initialAssetInfo;
9267
- stagingResultCallback = null;
9268
- thinkingCallback = null;
9269
- notifiedStagingIds = /* @__PURE__ */ new Map();
9270
- liveConversation;
9271
- baked = null;
9272
- authoring;
9273
- bakedAsset;
9274
- animationRefreshId = 0;
9275
- conversationId = null;
9276
- constructor(options) {
9277
- const resolvedToken = (options.apiToken ?? options.apiKey)?.trim() ?? "";
9278
- const resolvedProfile = options.assetProfile ?? "original";
9279
- super({
9280
- ...options,
9281
- apiToken: resolvedToken,
9282
- assetProfile: resolvedProfile,
9283
- loadAnimation: (animationId) => loadVrma({
9284
- baseUrl: options.baseUrl,
9285
- characterId: options.characterId,
9286
- apiToken: resolvedToken,
9287
- animationId,
9288
- assetProfile: resolvedProfile
9289
- })
9290
- });
9291
- this.authoring = options.authoring ?? false;
9292
- this.bakedAsset = options.bakedConversation;
9293
- this.initialAssetInfo = options.initialAssetInfo ?? null;
9294
- this.conversationSocket = new ConversationSocket({
9295
- baseUrl: this.baseUrl,
9296
- apiToken: this.apiToken,
9297
- getConversationId: () => this.conversationId,
9298
- handleStreamEvent: (event) => {
9299
- if (event.type !== "text_delta") {
9300
- this.thinkingState = false;
9995
+ // ../../clients/web-client/src/voice/push-to-talk.ts
9996
+ var INTERACTIVE = 'input, textarea, select, button, a[href], summary, [contenteditable]:not([contenteditable="false"]), [role="button"], [role="textbox"], [role="checkbox"], [role="switch"], [role="menuitem"], [role="tab"], [role="slider"], [role="option"]';
9997
+ var elementOf = function(target) {
9998
+ return target && typeof target.closest === "function" ? target : null;
9999
+ };
10000
+ var bindPushToTalk = function(target, options = {}) {
10001
+ const element = options.element ?? null;
10002
+ const key = options.key === void 0 ? "Space" : options.key;
10003
+ const minHoldMs = options.minHoldMs ?? SPEECH_MIN_HOLD_MS;
10004
+ let active = null;
10005
+ let pointerId = null;
10006
+ let pressedAt = 0;
10007
+ const begin = function(source) {
10008
+ if (active || target.listeningState !== "idle") {
10009
+ return false;
10010
+ }
10011
+ if (options.enabled && !options.enabled()) {
10012
+ return false;
10013
+ }
10014
+ active = source;
10015
+ pressedAt = performance.now();
10016
+ const listening = typeof options.listening === "function" ? options.listening() : options.listening ?? {};
10017
+ target.startListening({ ...listening, endOnSilenceMs: null }).then(
10018
+ (result) => {
10019
+ if (result) {
10020
+ options.onResult?.(result);
9301
10021
  }
9302
- this.streamPlayback.handleEvent(event);
9303
10022
  },
9304
- handleTurn: async (turn) => {
9305
- await this.handleStagingResult(turn, { source: "turn" });
10023
+ (error) => {
10024
+ active = null;
10025
+ options.onError?.(error);
10026
+ }
10027
+ );
10028
+ return true;
10029
+ };
10030
+ const end = function() {
10031
+ if (!active) {
10032
+ return;
10033
+ }
10034
+ active = null;
10035
+ pointerId = null;
10036
+ if (performance.now() - pressedAt < minHoldMs) {
10037
+ target.cancelListening();
10038
+ options.onShortPress?.();
10039
+ return;
10040
+ }
10041
+ const starting = target.listeningState === "starting";
10042
+ target.stopListening().then(
10043
+ (result) => {
10044
+ if (starting && result === null) {
10045
+ options.onShortPress?.();
10046
+ }
9306
10047
  },
9307
- handleCallbackError: this.handleCallbackError
9308
- });
9309
- this.liveConversation = new LiveConversation({
9310
- baseUrl: this.baseUrl,
9311
- apiToken: this.apiToken,
9312
- telemetry: this.telemetry,
9313
- socket: () => this.conversationSocket,
9314
- renderSession: () => this.renderSessionId,
10048
+ () => void 0
10049
+ );
10050
+ };
10051
+ const cancel = function() {
10052
+ if (!active) {
10053
+ return;
10054
+ }
10055
+ active = null;
10056
+ pointerId = null;
10057
+ target.cancelListening();
10058
+ };
10059
+ const isOwnElement = function(node) {
10060
+ return Boolean(element && node && element.contains(node));
10061
+ };
10062
+ const onKeyDown = function(event) {
10063
+ if (event.key === "Escape" || event.code === "Escape") {
10064
+ cancel();
10065
+ return;
10066
+ }
10067
+ const node = elementOf(event.target);
10068
+ const ownElement = isOwnElement(node);
10069
+ const pressesElement = ownElement && (event.code === "Space" || event.code === "Enter");
10070
+ if (!pressesElement && (!key || event.code !== key)) {
10071
+ return;
10072
+ }
10073
+ if (event.repeat) {
10074
+ if (active === "key") {
10075
+ event.preventDefault();
10076
+ }
10077
+ return;
10078
+ }
10079
+ if (event.ctrlKey || event.metaKey || event.altKey || event.isComposing) {
10080
+ return;
10081
+ }
10082
+ if (!ownElement && node?.closest(INTERACTIVE)) {
10083
+ return;
10084
+ }
10085
+ if (begin("key")) {
10086
+ event.preventDefault();
10087
+ }
10088
+ };
10089
+ const onKeyUp = function(event) {
10090
+ if (active !== "key") {
10091
+ return;
10092
+ }
10093
+ const ownElement = isOwnElement(elementOf(event.target));
10094
+ if (event.code === key || ownElement && (event.code === "Space" || event.code === "Enter")) {
10095
+ event.preventDefault();
10096
+ end();
10097
+ }
10098
+ };
10099
+ const onVisibility = function() {
10100
+ if (document.visibilityState === "hidden") {
10101
+ cancel();
10102
+ }
10103
+ };
10104
+ const onPointerDown = function(event) {
10105
+ if (event.pointerType === "mouse" && event.button !== 0) {
10106
+ return;
10107
+ }
10108
+ event.preventDefault();
10109
+ if (!begin("pointer")) {
10110
+ return;
10111
+ }
10112
+ pointerId = event.pointerId;
10113
+ try {
10114
+ element?.setPointerCapture(event.pointerId);
10115
+ } catch {
10116
+ }
10117
+ };
10118
+ const onPointerUp = function(event) {
10119
+ if (active === "pointer" && event.pointerId === pointerId) {
10120
+ end();
10121
+ }
10122
+ };
10123
+ const onPointerCancel = function(event) {
10124
+ if (active === "pointer" && event.pointerId === pointerId) {
10125
+ cancel();
10126
+ }
10127
+ };
10128
+ const prevent = function(event) {
10129
+ event.preventDefault();
10130
+ };
10131
+ const restoreStyle = [];
10132
+ if (element) {
10133
+ for (const [property, value] of [
10134
+ ["touch-action", "none"],
10135
+ ["user-select", "none"],
10136
+ ["-webkit-user-select", "none"],
10137
+ ["-webkit-touch-callout", "none"]
10138
+ ]) {
10139
+ const previous = element.style.getPropertyValue(property);
10140
+ element.style.setProperty(property, value);
10141
+ restoreStyle.push(() => {
10142
+ if (previous) {
10143
+ element.style.setProperty(property, previous);
10144
+ } else {
10145
+ element.style.removeProperty(property);
10146
+ }
10147
+ });
10148
+ }
10149
+ element.addEventListener("pointerdown", onPointerDown);
10150
+ window.addEventListener("pointerup", onPointerUp, true);
10151
+ window.addEventListener("pointercancel", onPointerCancel, true);
10152
+ element.addEventListener("contextmenu", prevent);
10153
+ element.addEventListener("selectstart", prevent);
10154
+ element.addEventListener("touchstart", prevent, { passive: false });
10155
+ }
10156
+ window.addEventListener("keydown", onKeyDown);
10157
+ window.addEventListener("keyup", onKeyUp);
10158
+ window.addEventListener("blur", cancel);
10159
+ document.addEventListener("visibilitychange", onVisibility);
10160
+ return () => {
10161
+ cancel();
10162
+ if (element) {
10163
+ element.removeEventListener("pointerdown", onPointerDown);
10164
+ window.removeEventListener("pointerup", onPointerUp, true);
10165
+ window.removeEventListener("pointercancel", onPointerCancel, true);
10166
+ element.removeEventListener("contextmenu", prevent);
10167
+ element.removeEventListener("selectstart", prevent);
10168
+ element.removeEventListener("touchstart", prevent);
10169
+ restoreStyle.forEach((restore) => restore());
10170
+ }
10171
+ window.removeEventListener("keydown", onKeyDown);
10172
+ window.removeEventListener("keyup", onKeyUp);
10173
+ window.removeEventListener("blur", cancel);
10174
+ document.removeEventListener("visibilitychange", onVisibility);
10175
+ };
10176
+ };
10177
+
10178
+ // ../../clients/web-client/src/voice/microphone-capture.ts
10179
+ var FRAME_SAMPLES = SPEECH_AUDIO_FORMAT.sampleRate * SPEECH_FRAME_MS / 1e3;
10180
+ var MicrophoneError = class extends Error {
10181
+ constructor(code, message) {
10182
+ super(message);
10183
+ this.code = code;
10184
+ this.name = "NpcifyMicrophoneError";
10185
+ }
10186
+ };
10187
+ var Pcm16Framer = class {
10188
+ constructor(inputRate, emit) {
10189
+ this.emit = emit;
10190
+ this.ratio = inputRate / SPEECH_AUDIO_FORMAT.sampleRate;
10191
+ }
10192
+ ratio;
10193
+ frame = new Int16Array(FRAME_SAMPLES);
10194
+ length = 0;
10195
+ sum = 0;
10196
+ count = 0;
10197
+ phase = 0;
10198
+ push(samples) {
10199
+ for (let index = 0; index < samples.length; index += 1) {
10200
+ this.sum += samples[index] ?? 0;
10201
+ this.count += 1;
10202
+ this.phase += 1;
10203
+ while (this.phase >= this.ratio) {
10204
+ this.phase -= this.ratio;
10205
+ this.write(this.sum / Math.max(1, this.count));
10206
+ if (this.ratio >= 1) {
10207
+ this.sum = 0;
10208
+ this.count = 0;
10209
+ }
10210
+ }
10211
+ if (this.ratio < 1) {
10212
+ this.sum = 0;
10213
+ this.count = 0;
10214
+ }
10215
+ }
10216
+ }
10217
+ flush() {
10218
+ if (this.count > 0 && this.ratio >= 1) {
10219
+ this.write(this.sum / this.count);
10220
+ this.sum = 0;
10221
+ this.count = 0;
10222
+ }
10223
+ if (this.length > 0) {
10224
+ this.emit(this.frame.slice(0, this.length));
10225
+ this.length = 0;
10226
+ }
10227
+ }
10228
+ write(value) {
10229
+ const clamped = Math.max(-1, Math.min(1, value));
10230
+ this.frame[this.length] = Math.round(
10231
+ clamped < 0 ? clamped * 32768 : clamped * 32767
10232
+ );
10233
+ this.length += 1;
10234
+ if (this.length === FRAME_SAMPLES) {
10235
+ this.emit(this.frame);
10236
+ this.frame = new Int16Array(FRAME_SAMPLES);
10237
+ this.length = 0;
10238
+ }
10239
+ }
10240
+ };
10241
+ var levelOf = function(pcm) {
10242
+ if (pcm.length === 0) {
10243
+ return 0;
10244
+ }
10245
+ let sum = 0;
10246
+ for (const sample of pcm) {
10247
+ sum += (sample / 32768) ** 2;
10248
+ }
10249
+ return Math.sqrt(sum / pcm.length);
10250
+ };
10251
+ var WORKLET_SOURCE = `
10252
+ const FRAME = ${FRAME_SAMPLES}
10253
+ class NpcifyCaptureProcessor extends AudioWorkletProcessor {
10254
+ constructor() {
10255
+ super()
10256
+ this.ratio = sampleRate / ${SPEECH_AUDIO_FORMAT.sampleRate}
10257
+ this.frame = new Int16Array(FRAME)
10258
+ this.length = 0
10259
+ this.sum = 0
10260
+ this.count = 0
10261
+ this.phase = 0
10262
+ this.port.onmessage = (event) => {
10263
+ if (event.data === 'flush') {
10264
+ if (this.count > 0 && this.ratio >= 1) {
10265
+ this.write(this.sum / this.count)
10266
+ this.sum = 0
10267
+ this.count = 0
10268
+ }
10269
+ if (this.length > 0) {
10270
+ this.post(this.frame.slice(0, this.length))
10271
+ this.length = 0
10272
+ }
10273
+ this.port.postMessage({ type: 'flushed' })
10274
+ }
10275
+ }
10276
+ }
10277
+ write(value) {
10278
+ const clamped = Math.max(-1, Math.min(1, value))
10279
+ this.frame[this.length++] = Math.round(clamped < 0 ? clamped * 32768 : clamped * 32767)
10280
+ if (this.length === FRAME) {
10281
+ this.post(this.frame)
10282
+ this.frame = new Int16Array(FRAME)
10283
+ this.length = 0
10284
+ }
10285
+ }
10286
+ post(pcm) {
10287
+ this.port.postMessage({ type: 'frame', pcm }, [pcm.buffer])
10288
+ }
10289
+ process(inputs) {
10290
+ const channels = inputs[0]
10291
+ if (!channels || channels.length === 0) return true
10292
+ const size = channels[0].length
10293
+ for (let index = 0; index < size; index += 1) {
10294
+ let value = 0
10295
+ for (const channel of channels) value += channel[index]
10296
+ this.sum += value / channels.length
10297
+ this.count += 1
10298
+ this.phase += 1
10299
+ while (this.phase >= this.ratio) {
10300
+ this.phase -= this.ratio
10301
+ this.write(this.sum / Math.max(1, this.count))
10302
+ if (this.ratio >= 1) {
10303
+ this.sum = 0
10304
+ this.count = 0
10305
+ }
10306
+ }
10307
+ if (this.ratio < 1) {
10308
+ this.sum = 0
10309
+ this.count = 0
10310
+ }
10311
+ }
10312
+ return true
10313
+ }
10314
+ }
10315
+ registerProcessor('npcify-capture', NpcifyCaptureProcessor)
10316
+ `;
10317
+ var MicrophoneCapture = class {
10318
+ context = null;
10319
+ workletReady = null;
10320
+ graph = null;
10321
+ generation = 0;
10322
+ async start(onFrame) {
10323
+ this.cancel();
10324
+ const generation = ++this.generation;
10325
+ const AudioContextClass = globalThis.AudioContext ?? globalThis.webkitAudioContext;
10326
+ if (!navigator.mediaDevices?.getUserMedia || !AudioContextClass) {
10327
+ throw new MicrophoneError(
10328
+ "MICROPHONE_UNAVAILABLE",
10329
+ "This browser cannot record audio."
10330
+ );
10331
+ }
10332
+ this.context ??= new AudioContextClass({ latencyHint: "interactive" });
10333
+ const context2 = this.context;
10334
+ const resumed = context2.resume().catch(() => void 0);
10335
+ let stream;
10336
+ try {
10337
+ stream = await navigator.mediaDevices.getUserMedia({
10338
+ audio: {
10339
+ channelCount: 1,
10340
+ echoCancellation: true,
10341
+ noiseSuppression: true,
10342
+ autoGainControl: true
10343
+ }
10344
+ });
10345
+ } catch (error) {
10346
+ const name = error?.name;
10347
+ throw name === "NotAllowedError" || name === "SecurityError" ? new MicrophoneError(
10348
+ "MICROPHONE_PERMISSION_DENIED",
10349
+ "Microphone access was denied."
10350
+ ) : new MicrophoneError(
10351
+ "MICROPHONE_UNAVAILABLE",
10352
+ "No microphone is available."
10353
+ );
10354
+ }
10355
+ await resumed;
10356
+ if (generation !== this.generation) {
10357
+ stream.getTracks().forEach((track) => track.stop());
10358
+ return;
10359
+ }
10360
+ const deliver = (pcm) => onFrame({ pcm, level: levelOf(pcm) });
10361
+ const source = context2.createMediaStreamSource(stream);
10362
+ const sink = context2.createGain();
10363
+ sink.gain.value = 0;
10364
+ sink.connect(context2.destination);
10365
+ const graph = await this.workletGraph(context2, stream, source, sink, deliver) ?? this.scriptGraph(context2, stream, source, sink, deliver);
10366
+ if (generation !== this.generation) {
10367
+ this.close(graph, false);
10368
+ return;
10369
+ }
10370
+ this.graph = graph;
10371
+ }
10372
+ async stop() {
10373
+ const graph = this.graph;
10374
+ this.graph = null;
10375
+ this.generation += 1;
10376
+ if (!graph) {
10377
+ return;
10378
+ }
10379
+ try {
10380
+ await graph.flush();
10381
+ } finally {
10382
+ this.close(graph);
10383
+ }
10384
+ }
10385
+ cancel() {
10386
+ const graph = this.graph;
10387
+ this.graph = null;
10388
+ this.generation += 1;
10389
+ if (graph) {
10390
+ this.close(graph);
10391
+ }
10392
+ }
10393
+ close(graph, suspend = true) {
10394
+ graph.stream.getTracks().forEach((track) => track.stop());
10395
+ graph.source.disconnect();
10396
+ graph.node.disconnect();
10397
+ graph.sink.disconnect();
10398
+ if (graph.node instanceof AudioWorkletNode) {
10399
+ graph.node.port.onmessage = null;
10400
+ } else {
10401
+ graph.node.onaudioprocess = null;
10402
+ }
10403
+ if (suspend) {
10404
+ void this.context?.suspend().catch(() => void 0);
10405
+ }
10406
+ }
10407
+ async workletGraph(context2, stream, source, sink, deliver) {
10408
+ if (!context2.audioWorklet || typeof AudioWorkletNode === "undefined") {
10409
+ return null;
10410
+ }
10411
+ this.workletReady ??= (async () => {
10412
+ const url = URL.createObjectURL(
10413
+ new Blob([WORKLET_SOURCE], { type: "application/javascript" })
10414
+ );
10415
+ try {
10416
+ await context2.audioWorklet.addModule(url);
10417
+ return true;
10418
+ } catch {
10419
+ return false;
10420
+ } finally {
10421
+ URL.revokeObjectURL(url);
10422
+ }
10423
+ })();
10424
+ if (!await this.workletReady) {
10425
+ return null;
10426
+ }
10427
+ const node = new AudioWorkletNode(context2, "npcify-capture", {
10428
+ numberOfInputs: 1,
10429
+ numberOfOutputs: 1,
10430
+ channelCount: 1,
10431
+ channelCountMode: "explicit"
10432
+ });
10433
+ let flushed = null;
10434
+ node.port.onmessage = (event) => {
10435
+ if (event.data.type === "frame" && event.data.pcm) {
10436
+ deliver(event.data.pcm);
10437
+ } else if (event.data.type === "flushed") {
10438
+ flushed?.();
10439
+ }
10440
+ };
10441
+ source.connect(node);
10442
+ node.connect(sink);
10443
+ return {
10444
+ stream,
10445
+ source,
10446
+ node,
10447
+ sink,
10448
+ flush: () => new Promise((resolve) => {
10449
+ const timer = setTimeout(resolve, 250);
10450
+ flushed = () => {
10451
+ clearTimeout(timer);
10452
+ resolve();
10453
+ };
10454
+ node.port.postMessage("flush");
10455
+ })
10456
+ };
10457
+ }
10458
+ /** The fallback without AudioWorklet: a ScriptProcessorNode on the main thread. */
10459
+ scriptGraph(context2, stream, source, sink, deliver) {
10460
+ const framer = new Pcm16Framer(context2.sampleRate, deliver);
10461
+ const node = context2.createScriptProcessor(2048, 1, 1);
10462
+ node.onaudioprocess = (event) => {
10463
+ framer.push(event.inputBuffer.getChannelData(0));
10464
+ };
10465
+ source.connect(node);
10466
+ node.connect(sink);
10467
+ return {
10468
+ stream,
10469
+ source,
10470
+ node,
10471
+ sink,
10472
+ flush: () => {
10473
+ framer.flush();
10474
+ return Promise.resolve();
10475
+ }
10476
+ };
10477
+ }
10478
+ };
10479
+
10480
+ // ../../clients/web-client/src/voice/silence-detector.ts
10481
+ var MIN_SPEECH_LEVEL = 0.015;
10482
+ var SPEECH_TO_NOISE = 3;
10483
+ var SPEECH_START_MS = 200;
10484
+ var SilenceDetector = class {
10485
+ constructor(silenceMs) {
10486
+ this.silenceMs = silenceMs;
10487
+ }
10488
+ floor = null;
10489
+ loudMs = 0;
10490
+ quietMs = 0;
10491
+ speaking = false;
10492
+ /** True when the speech has ended. */
10493
+ push(level, frameMs = SPEECH_FRAME_MS) {
10494
+ const threshold = Math.max(
10495
+ MIN_SPEECH_LEVEL,
10496
+ (this.floor ?? MIN_SPEECH_LEVEL / SPEECH_TO_NOISE) * SPEECH_TO_NOISE
10497
+ );
10498
+ if (level >= threshold) {
10499
+ this.loudMs += frameMs;
10500
+ this.quietMs = 0;
10501
+ if (this.loudMs >= SPEECH_START_MS) {
10502
+ this.speaking = true;
10503
+ }
10504
+ return false;
10505
+ }
10506
+ this.floor = this.floor === null ? level : this.floor * 0.9 + level * 0.1;
10507
+ this.loudMs = 0;
10508
+ if (!this.speaking) {
10509
+ return false;
10510
+ }
10511
+ this.quietMs += frameMs;
10512
+ return this.quietMs >= this.silenceMs;
10513
+ }
10514
+ get heardSpeech() {
10515
+ return this.speaking;
10516
+ }
10517
+ };
10518
+
10519
+ // ../../clients/web-client/src/voice/voice-input.ts
10520
+ var SpeechInputError = class extends Error {
10521
+ constructor(code, message) {
10522
+ super(message ?? code);
10523
+ this.code = code;
10524
+ this.name = "NpcifySpeechInputError";
10525
+ }
10526
+ };
10527
+ var deferred = function() {
10528
+ let resolve;
10529
+ let reject;
10530
+ const promise = new Promise((done, fail) => {
10531
+ resolve = done;
10532
+ reject = fail;
10533
+ });
10534
+ promise.catch(() => void 0);
10535
+ return { promise, resolve, reject };
10536
+ };
10537
+ var CODE = /^[A-Z][A-Z0-9_]{2,63}$/u;
10538
+ var speechError = function(error, fallback) {
10539
+ if (error instanceof SpeechInputError) {
10540
+ return error;
10541
+ }
10542
+ if (error instanceof Error && "code" in error && typeof error.code === "string") {
10543
+ return new SpeechInputError(
10544
+ error.code,
10545
+ error.message
10546
+ );
10547
+ }
10548
+ return new SpeechInputError(
10549
+ fallback,
10550
+ error instanceof Error ? error.message : void 0
10551
+ );
10552
+ };
10553
+ var VoiceInput = class {
10554
+ constructor(deps) {
10555
+ this.deps = deps;
10556
+ }
10557
+ state = "idle";
10558
+ session = null;
10559
+ capture = null;
10560
+ /**
10561
+ * The session that the capture records for. A start that was cancelled
10562
+ * never touches the capture when a later session owns it.
10563
+ */
10564
+ captureOwner = null;
10565
+ turns = /* @__PURE__ */ new Map();
10566
+ get listeningState() {
10567
+ return this.state;
10568
+ }
10569
+ /** A turn that the server started has not delivered its final payload. */
10570
+ get turnPending() {
10571
+ return this.turns.size > 0;
10572
+ }
10573
+ /**
10574
+ * Starts listening. Resolves with the result when the listening ends (by
10575
+ * `stop`, the silence rule or the time limit), or with null when it is
10576
+ * cancelled.
10577
+ */
10578
+ async start(options = {}) {
10579
+ if (this.session) {
10580
+ throw new SpeechInputError(
10581
+ "SPEECH_ALREADY_LISTENING",
10582
+ "Listening has already started."
10583
+ );
10584
+ }
10585
+ this.deps.assertVoiceAllowed();
10586
+ if (this.turnPending || this.deps.turnInFlight()) {
10587
+ throw new SpeechInputError(
10588
+ "TURN_IN_PROGRESS",
10589
+ "A reply is already being generated. Please wait."
10590
+ );
10591
+ }
10592
+ const conversationId = this.deps.conversationId();
10593
+ if (!conversationId) {
10594
+ throw new SpeechInputError(
10595
+ "CONVERSATION_REQUIRED",
10596
+ "Create a conversation before listening."
10597
+ );
10598
+ }
10599
+ const maxDurationMs = Math.min(
10600
+ SPEECH_MAX_DURATION_MS,
10601
+ Math.max(1e3, options.maxDurationMs ?? SPEECH_MAX_DURATION_MS)
10602
+ );
10603
+ const session = {
10604
+ requestId: globalThis.crypto.randomUUID(),
10605
+ options,
10606
+ sessionId: null,
10607
+ sequence: 0,
10608
+ bytes: 0,
10609
+ pending: [],
10610
+ timer: null,
10611
+ silence: options.endOnSilenceMs && options.endOnSilenceMs > 0 ? new SilenceDetector(options.endOnSilenceMs) : null,
10612
+ stopRequested: false,
10613
+ stopping: null,
10614
+ captureClosing: null,
10615
+ captureClosed: false,
10616
+ disconnected: deferred(),
10617
+ done: deferred()
10618
+ };
10619
+ this.session = session;
10620
+ this.setState("starting");
10621
+ const socket = this.deps.socket();
10622
+ this.capture ??= (this.deps.createCapture ?? (() => new MicrophoneCapture()))();
10623
+ const capture = this.capture;
10624
+ this.captureOwner = session;
10625
+ const [acknowledged, captured] = await Promise.allSettled([
10626
+ socket.ensureConnected(conversationId).then(
10627
+ () => socket.request(SPEECH_EVENTS.start, {
10628
+ conversationId,
10629
+ requestId: session.requestId,
10630
+ parentStagingId: options.parentStagingId,
10631
+ language: options.language ?? null,
10632
+ send: options.send !== false,
10633
+ silent: options.silent === true,
10634
+ renderSessionId: this.deps.renderSessionId(),
10635
+ format: SPEECH_AUDIO_FORMAT
10636
+ })
10637
+ ),
10638
+ capture.start((frame) => this.frame(session, frame))
10639
+ ]);
10640
+ const ack = acknowledged.status === "fulfilled" ? acknowledged.value : null;
10641
+ const serverSession = ack?.ok ? ack.data.sessionId : null;
10642
+ if (session !== this.session) {
10643
+ this.dropCapture(session);
10644
+ if (serverSession) {
10645
+ socket.send(SPEECH_EVENTS.cancel, { sessionId: serverSession });
10646
+ }
10647
+ return session.done.promise;
10648
+ }
10649
+ let failure = null;
10650
+ if (captured.status === "rejected") {
10651
+ failure = speechError(captured.reason, "MICROPHONE_UNAVAILABLE");
10652
+ } else if (acknowledged.status === "rejected") {
10653
+ failure = speechError(acknowledged.reason, "SPEECH_UNAVAILABLE");
10654
+ } else if (!ack?.ok) {
10655
+ failure = new SpeechInputError(
10656
+ ack && !ack.ok ? ack.error : "SPEECH_UNAVAILABLE"
10657
+ );
10658
+ }
10659
+ if (failure) {
10660
+ this.dropCapture(session);
10661
+ if (serverSession) {
10662
+ socket.send(SPEECH_EVENTS.cancel, { sessionId: serverSession });
10663
+ }
10664
+ this.end(session);
10665
+ session.done.reject(failure);
10666
+ return session.done.promise;
10667
+ }
10668
+ session.sessionId = serverSession;
10669
+ for (const audio of session.pending.splice(0)) {
10670
+ this.sendFrame(session, audio);
10671
+ }
10672
+ if (session.stopRequested) {
10673
+ void this.stop().catch(() => void 0);
10674
+ return session.done.promise;
10675
+ }
10676
+ this.setState("listening");
10677
+ session.timer = setTimeout(() => {
10678
+ void this.stop().catch(() => void 0);
10679
+ }, maxDurationMs);
10680
+ return session.done.promise;
10681
+ }
10682
+ /**
10683
+ * Ends the recording and resolves with the result as soon as the text is
10684
+ * recognised. A stop before the microphone gave any audio (a permission
10685
+ * prompt, a slow device) cancels and resolves null.
10686
+ */
10687
+ async stop() {
10688
+ const session = this.session;
10689
+ if (!session) {
10690
+ return null;
10691
+ }
10692
+ if (session.stopping) {
10693
+ return session.stopping;
10694
+ }
10695
+ if (!session.sessionId) {
10696
+ if (session.bytes === 0) {
10697
+ this.cancel();
10698
+ return null;
10699
+ }
10700
+ session.stopRequested = true;
10701
+ void this.closeCapture(session);
10702
+ return session.done.promise;
10703
+ }
10704
+ const stopping = deferred();
10705
+ session.stopping = stopping.promise;
10706
+ this.finish(session).then(stopping.resolve, stopping.reject);
10707
+ return session.stopping;
10708
+ }
10709
+ /** Discards the recording. Nothing is recognised or charged. */
10710
+ cancel() {
10711
+ const session = this.session;
10712
+ if (!session || session.stopping) {
10713
+ return;
10714
+ }
10715
+ this.dropCapture(session);
10716
+ if (session.sessionId) {
10717
+ this.deps.socket().send(SPEECH_EVENTS.cancel, { sessionId: session.sessionId });
10718
+ }
10719
+ this.end(session);
10720
+ session.done.resolve(null);
10721
+ }
10722
+ /** The conversation ended: no listening, and no turn waits any more. */
10723
+ reset() {
10724
+ this.cancel();
10725
+ for (const [requestId, turn] of this.turns) {
10726
+ clearTimeout(turn.timer);
10727
+ turn.done.reject(
10728
+ new SpeechInputError("CONVERSATION_ENDED", "The conversation ended.")
10729
+ );
10730
+ this.turns.delete(requestId);
10731
+ }
10732
+ }
10733
+ handleSocketEvent(event) {
10734
+ if (event.type === "partial") {
10735
+ if (this.session?.sessionId === event.event.sessionId) {
10736
+ this.deps.onTranscript({ text: event.event.text, final: false });
10737
+ }
10738
+ return;
10739
+ }
10740
+ if (event.type === "turn") {
10741
+ const turn = this.turns.get(event.requestId);
10742
+ if (!turn) {
10743
+ return;
10744
+ }
10745
+ this.turns.delete(event.requestId);
10746
+ clearTimeout(turn.timer);
10747
+ void this.deps.deliverTurn(event.turn, event.requestId, turn.options.silent === true).then(
10748
+ () => turn.done.resolve(event.turn),
10749
+ (error) => turn.done.reject(error)
10750
+ );
10751
+ return;
10752
+ }
10753
+ if (event.type === "turnFailed") {
10754
+ const turn = this.turns.get(event.event.requestId);
10755
+ if (!turn) {
10756
+ return;
10757
+ }
10758
+ this.turns.delete(event.event.requestId);
10759
+ clearTimeout(turn.timer);
10760
+ const reason = event.event.error;
10761
+ turn.done.reject(
10762
+ new SpeechInputError(CODE.test(reason) ? reason : "TURN_FAILED", reason)
10763
+ );
10764
+ return;
10765
+ }
10766
+ const session = this.session;
10767
+ if (session) {
10768
+ const failure = new SpeechInputError(
10769
+ "SPEECH_DISCONNECTED",
10770
+ session.stopping ? "The connection closed while the speech was recognised." : "The connection closed while listening."
10771
+ );
10772
+ session.disconnected.reject(failure);
10773
+ if (!session.stopping) {
10774
+ this.dropCapture(session);
10775
+ this.end(session);
10776
+ session.done.reject(failure);
10777
+ }
10778
+ }
10779
+ for (const requestId of [...this.turns.keys()]) {
10780
+ this.recover(requestId);
10781
+ }
10782
+ }
10783
+ frame(session, frame) {
10784
+ if (session !== this.session || session.captureClosed) {
10785
+ return;
10786
+ }
10787
+ const audio = new Uint8Array(
10788
+ frame.pcm.buffer,
10789
+ frame.pcm.byteOffset,
10790
+ frame.pcm.byteLength
10791
+ ).slice();
10792
+ if (session.sessionId) {
10793
+ this.sendFrame(session, audio);
10794
+ } else if (session.bytes + audio.length <= SPEECH_MAX_AUDIO_BYTES) {
10795
+ session.bytes += audio.length;
10796
+ session.pending.push(audio);
10797
+ }
10798
+ if (!session.stopping && session.silence?.push(frame.level)) {
10799
+ void this.stop().catch(() => void 0);
10800
+ }
10801
+ }
10802
+ sendFrame(session, audio) {
10803
+ if (!session.sessionId) {
10804
+ return;
10805
+ }
10806
+ this.deps.socket().send(SPEECH_EVENTS.audio, {
10807
+ sessionId: session.sessionId,
10808
+ sequence: session.sequence,
10809
+ audio
10810
+ });
10811
+ session.sequence += 1;
10812
+ }
10813
+ async finish(session) {
10814
+ if (session.timer) {
10815
+ clearTimeout(session.timer);
10816
+ }
10817
+ this.setState("recognizing");
10818
+ try {
10819
+ await this.closeCapture(session);
10820
+ const ack = await Promise.race([
10821
+ this.deps.socket().request(
10822
+ SPEECH_EVENTS.stop,
10823
+ { sessionId: session.sessionId },
10824
+ 3e4
10825
+ ),
10826
+ session.disconnected.promise
10827
+ ]);
10828
+ if (!ack.ok) {
10829
+ throw new SpeechInputError(ack.error);
10830
+ }
10831
+ const result = this.result(session, ack.data);
10832
+ this.end(session);
10833
+ this.deps.onTranscript({ text: result.text, final: true });
10834
+ session.done.resolve(result);
10835
+ return result;
10836
+ } catch (error) {
10837
+ const failure = speechError(error, "SPEECH_RECOGNITION_FAILED");
10838
+ this.end(session);
10839
+ session.done.reject(failure);
10840
+ throw failure;
10841
+ }
10842
+ }
10843
+ result(session, transcript) {
10844
+ const text = transcript.text;
10845
+ const turn = transcript.sent ? this.track(session, text) : null;
10846
+ let sending = null;
10847
+ return {
10848
+ text,
10849
+ language: transcript.language,
10850
+ durationMs: transcript.durationMs,
10851
+ sent: transcript.sent,
10852
+ turn,
10853
+ send: () => {
10854
+ if (turn) {
10855
+ return turn;
10856
+ }
10857
+ if (!text) {
10858
+ return Promise.reject(
10859
+ new SpeechInputError("SPEECH_EMPTY", "No speech was recognised.")
10860
+ );
10861
+ }
10862
+ this.deps.assertVoiceAllowed();
10863
+ sending ??= this.deps.sendText(text, {
10864
+ requestId: session.requestId,
10865
+ parentStagingId: session.options.parentStagingId,
10866
+ silent: session.options.silent === true
10867
+ });
10868
+ return sending;
10869
+ }
10870
+ };
10871
+ }
10872
+ track(session, text) {
10873
+ const done = deferred();
10874
+ const timer = setTimeout(
10875
+ () => this.recover(session.requestId),
10876
+ this.deps.turnTimeoutMs ?? 12e4
10877
+ );
10878
+ this.turns.set(session.requestId, {
10879
+ text,
10880
+ options: session.options,
10881
+ timer,
10882
+ done
10883
+ });
10884
+ this.deps.beginTurn();
10885
+ return done.promise;
10886
+ }
10887
+ /** The idempotent HTTP request returns the turn that the server started. */
10888
+ recover(requestId) {
10889
+ const turn = this.turns.get(requestId);
10890
+ if (!turn) {
10891
+ return;
10892
+ }
10893
+ this.turns.delete(requestId);
10894
+ clearTimeout(turn.timer);
10895
+ this.deps.sendText(turn.text, {
10896
+ requestId,
10897
+ parentStagingId: turn.options.parentStagingId,
10898
+ silent: turn.options.silent === true
10899
+ }).then(turn.done.resolve, turn.done.reject);
10900
+ }
10901
+ /** Stops the capture of `session`; its last partial frame arrives first. */
10902
+ closeCapture(session) {
10903
+ session.captureClosing ??= (async () => {
10904
+ if (this.captureOwner === session) {
10905
+ this.captureOwner = null;
10906
+ await this.capture?.stop();
10907
+ }
10908
+ session.captureClosed = true;
10909
+ })();
10910
+ return session.captureClosing;
10911
+ }
10912
+ /** Ends the capture of `session` without its last frame, unless a later session owns it. */
10913
+ dropCapture(session) {
10914
+ if (this.captureOwner === session) {
10915
+ this.captureOwner = null;
10916
+ this.capture?.cancel();
10917
+ }
10918
+ }
10919
+ end(session) {
10920
+ if (session.timer) {
10921
+ clearTimeout(session.timer);
10922
+ session.timer = null;
10923
+ }
10924
+ if (this.session === session) {
10925
+ this.session = null;
10926
+ this.setState("idle");
10927
+ }
10928
+ }
10929
+ setState(state) {
10930
+ if (this.state !== state) {
10931
+ this.state = state;
10932
+ this.deps.onState(state);
10933
+ }
10934
+ }
10935
+ };
10936
+
10937
+ // ../../clients/web-client/src/npcify.ts
10938
+ var Character = class extends CharacterRuntime {
10939
+ conversationSocket;
10940
+ renderSessionId = null;
10941
+ initialAssetInfo;
10942
+ stagingResultCallback = null;
10943
+ thinkingCallback = null;
10944
+ notifiedStagingIds = /* @__PURE__ */ new Map();
10945
+ liveConversation;
10946
+ baked = null;
10947
+ authoring;
10948
+ normalizeIdleOrientation;
10949
+ bakedAsset;
10950
+ animationRefreshId = 0;
10951
+ voice;
10952
+ configuredInputModes;
10953
+ resolvedInputModes = null;
10954
+ latestOptions = [];
10955
+ listeningStateCallback = null;
10956
+ transcriptCallback = null;
10957
+ conversationId = null;
10958
+ constructor(options) {
10959
+ const resolvedToken = (options.apiToken ?? options.apiKey)?.trim() ?? "";
10960
+ const resolvedProfile = options.assetProfile ?? "original";
10961
+ super({
10962
+ ...options,
10963
+ apiToken: resolvedToken,
10964
+ assetProfile: resolvedProfile,
10965
+ loadAnimation: (animationId) => loadVrma({
10966
+ baseUrl: options.baseUrl,
10967
+ characterId: options.characterId,
10968
+ apiToken: resolvedToken,
10969
+ animationId,
10970
+ assetProfile: resolvedProfile
10971
+ })
10972
+ });
10973
+ this.authoring = options.authoring ?? false;
10974
+ this.normalizeIdleOrientation = options.normalizeIdleOrientation ?? false;
10975
+ this.bakedAsset = options.bakedConversation;
10976
+ this.initialAssetInfo = options.initialAssetInfo ?? null;
10977
+ this.configuredInputModes = parseInputModes(options.inputModes);
10978
+ if (this.configuredInputModes && this.bakedAsset && !this.authoring) {
10979
+ resolveInputModes(this.configuredInputModes, {
10980
+ allowCustomReply: false,
10981
+ baked: true
10982
+ });
10983
+ }
10984
+ this.conversationSocket = new ConversationSocket({
10985
+ baseUrl: this.baseUrl,
10986
+ apiToken: this.apiToken,
10987
+ getConversationId: () => this.conversationId,
10988
+ handleStreamEvent: (event) => {
10989
+ if (event.type !== "text_delta") {
10990
+ this.thinkingState = false;
10991
+ }
10992
+ this.streamPlayback.handleEvent(event);
10993
+ },
10994
+ handleTurn: async (turn) => {
10995
+ await this.handleStagingResult(turn, { source: "turn" });
10996
+ },
10997
+ handleCallbackError: this.handleCallbackError,
10998
+ handleSpeechEvent: (event) => this.voice.handleSocketEvent(event)
10999
+ });
11000
+ this.voice = new VoiceInput({
11001
+ socket: () => this.conversationSocket,
11002
+ conversationId: () => this.conversationId,
11003
+ renderSessionId: () => this.renderSessionId,
11004
+ assertVoiceAllowed: () => this.assertInputMode("voice"),
11005
+ turnInFlight: () => this.liveConversation.inFlight,
11006
+ beginTurn: () => {
11007
+ if (this.conversationId) {
11008
+ this.telemetry.begin(this.conversationId);
11009
+ }
11010
+ this.invokeThinkingCallback({ taskId: null, stagingId: null });
11011
+ },
11012
+ deliverTurn: (payload, requestId, silent) => this.handleStagingResult(payload, {
11013
+ source: "api",
11014
+ forcePlayback: payload.taskId !== requestId,
11015
+ silent
11016
+ }),
11017
+ sendText: (text, turnOptions) => {
11018
+ if (!this.conversationId) {
11019
+ return Promise.reject(new Error("Conversation ID is required."));
11020
+ }
11021
+ return this.liveConversation.send(this.conversationId, text, turnOptions);
11022
+ },
11023
+ onState: (state) => {
11024
+ if (this.listeningStateCallback) {
11025
+ invokeSafeCallback(
11026
+ "onListeningState",
11027
+ this.handleCallbackError,
11028
+ () => this.listeningStateCallback?.(state)
11029
+ );
11030
+ }
11031
+ },
11032
+ onTranscript: (event) => {
11033
+ if (this.transcriptCallback) {
11034
+ invokeSafeCallback(
11035
+ "onTranscript",
11036
+ this.handleCallbackError,
11037
+ () => this.transcriptCallback?.(event)
11038
+ );
11039
+ }
11040
+ }
11041
+ });
11042
+ this.liveConversation = new LiveConversation({
11043
+ baseUrl: this.baseUrl,
11044
+ apiToken: this.apiToken,
11045
+ telemetry: this.telemetry,
11046
+ socket: () => this.conversationSocket,
11047
+ renderSession: () => this.renderSessionId,
9315
11048
  thinking: () => this.invokeThinkingCallback({ taskId: null, stagingId: null }),
9316
11049
  handleResult: (payload, forcePlayback, silent) => this.handleStagingResult(payload, {
9317
11050
  source: "api",
@@ -9331,7 +11064,12 @@ var Character = class extends CharacterRuntime {
9331
11064
  this.resetAssets();
9332
11065
  const progressReporter = createAssetProgressReporter(callback);
9333
11066
  const info = await this.getAssetInfo();
9334
- this.baked = !this.authoring && (this.bakedAsset || info.bakedConversationEnabled) ? await loadBakedConversation({
11067
+ const baked = !this.authoring && Boolean(this.bakedAsset || info.bakedConversationEnabled);
11068
+ this.resolvedInputModes = resolveInputModes(this.configuredInputModes, {
11069
+ allowCustomReply: info.allowCustomReply !== false,
11070
+ baked
11071
+ });
11072
+ this.baked = baked ? await loadBakedConversation({
9335
11073
  baseUrl: this.baseUrl,
9336
11074
  characterId: this.characterId,
9337
11075
  apiToken: this.apiToken,
@@ -9356,7 +11094,8 @@ var Character = class extends CharacterRuntime {
9356
11094
  }
9357
11095
  const animationController = new AnimationController(
9358
11096
  this.vrm,
9359
- info.idleAnimationId ?? null
11097
+ info.idleAnimationId ?? null,
11098
+ { normalizeIdleOrientation: this.normalizeIdleOrientation }
9360
11099
  );
9361
11100
  animationController.prepareAnimations = (ids) => this.lazyAnimations.ensureMany(ids);
9362
11101
  this.animationController = animationController;
@@ -9399,6 +11138,10 @@ var Character = class extends CharacterRuntime {
9399
11138
  if (info.bakedConversationEnabled || this.bakedAsset) {
9400
11139
  throw new Error("Baked conversation requires local rendering.");
9401
11140
  }
11141
+ this.resolvedInputModes = resolveInputModes(this.configuredInputModes, {
11142
+ allowCustomReply: info.allowCustomReply !== false,
11143
+ baked: false
11144
+ });
9402
11145
  this.animationController = new RemoteAnimationController();
9403
11146
  }
9404
11147
  async createConversation(options) {
@@ -9441,6 +11184,7 @@ var Character = class extends CharacterRuntime {
9441
11184
  return targetConversationId;
9442
11185
  }
9443
11186
  async disconnect() {
11187
+ this.voice.reset();
9444
11188
  this.thinkingState = false;
9445
11189
  if (this.animationController) {
9446
11190
  this.streamPlayback.cleanup();
@@ -9476,6 +11220,10 @@ var Character = class extends CharacterRuntime {
9476
11220
  maybeMessageOrOptions,
9477
11221
  maybeOptions
9478
11222
  );
11223
+ this.assertReplyMode(message);
11224
+ if (this.voice.turnPending) {
11225
+ throw new Error("A reply is already being generated. Please wait.");
11226
+ }
9479
11227
  if (this.baked) {
9480
11228
  const result = this.baked.reply(message, options?.parentStagingId);
9481
11229
  if (result.ended) {
@@ -9515,6 +11263,55 @@ var Character = class extends CharacterRuntime {
9515
11263
  get isBakedConversation() {
9516
11264
  return this.baked !== null;
9517
11265
  }
11266
+ /**
11267
+ * The input modes in use: the configured modes, or every mode that the
11268
+ * character permits. Before `loadAssets`, the configuration or all modes.
11269
+ */
11270
+ get inputModes() {
11271
+ return this.resolvedInputModes ?? this.configuredInputModes ?? permittedInputModes({ allowCustomReply: true, baked: false });
11272
+ }
11273
+ get listeningState() {
11274
+ return this.voice.listeningState;
11275
+ }
11276
+ /**
11277
+ * Opens the microphone and streams the speech to the server. Resolves with
11278
+ * the result when listening ends (`stopListening`, `endOnSilenceMs` or
11279
+ * `maxDurationMs`), or with null when it is cancelled. The first call asks
11280
+ * for the microphone permission.
11281
+ */
11282
+ startListening(options) {
11283
+ return this.voice.start(options);
11284
+ }
11285
+ /**
11286
+ * Ends the recording; resolves as soon as the server recognised the text.
11287
+ * Before the microphone gave any audio, it cancels and resolves null.
11288
+ */
11289
+ stopListening() {
11290
+ return this.voice.stop();
11291
+ }
11292
+ /** Discards the recording. Nothing is recognised or charged. */
11293
+ cancelListening() {
11294
+ this.voice.cancel();
11295
+ }
11296
+ /** Press and hold an element or a key (default Space) to talk. Returns an unbind function. */
11297
+ bindPushToTalk(options) {
11298
+ return bindPushToTalk(this, options);
11299
+ }
11300
+ onListeningState(callback) {
11301
+ return bindCallbackSlot(
11302
+ () => this.listeningStateCallback,
11303
+ (next) => this.listeningStateCallback = next,
11304
+ callback
11305
+ );
11306
+ }
11307
+ /** Partial text while the player speaks (`final: false`), then the final text. */
11308
+ onTranscript(callback) {
11309
+ return bindCallbackSlot(
11310
+ () => this.transcriptCallback,
11311
+ (next) => this.transcriptCallback = next,
11312
+ callback
11313
+ );
11314
+ }
9518
11315
  async getConversationWorldline(conversationId = this.conversationId) {
9519
11316
  if (!conversationId) {
9520
11317
  throw new Error("Conversation ID is required.");
@@ -9538,6 +11335,13 @@ var Character = class extends CharacterRuntime {
9538
11335
  timeline: options?.silent ? toSilentTimeline(normalized.timeline) : normalized.timeline
9539
11336
  });
9540
11337
  }
11338
+ /**
11339
+ * Plays one live staging event from another transport, such as an actor
11340
+ * stage. A timeline event returns its playback result.
11341
+ */
11342
+ playStreamEvent(event) {
11343
+ return this.streamPlayback.handleEvent(event);
11344
+ }
9541
11345
  setRenderSessionId(renderSessionId) {
9542
11346
  this.renderSessionId = renderSessionId?.trim() || null;
9543
11347
  }
@@ -9571,6 +11375,7 @@ var Character = class extends CharacterRuntime {
9571
11375
  async handleStagingResult(payload, options) {
9572
11376
  this.thinkingState = false;
9573
11377
  const normalized = normalizeStagingResultPayload(payload);
11378
+ this.latestOptions = normalized.options;
9574
11379
  if (options.source === "turn") {
9575
11380
  this.liveConversation.received(normalized);
9576
11381
  }
@@ -9597,6 +11402,44 @@ var Character = class extends CharacterRuntime {
9597
11402
  void this.streamPlayback.playPayload(normalized);
9598
11403
  }
9599
11404
  }
11405
+ /** Throws when the input modes do not include `mode`. */
11406
+ assertInputMode(mode) {
11407
+ if (mode === "voice" && this.baked) {
11408
+ throw new InputModeError(
11409
+ "Voice input is not available in a baked (offline) conversation."
11410
+ );
11411
+ }
11412
+ const modes = this.inputModes;
11413
+ if (!modes.includes(mode)) {
11414
+ throw new InputModeError(
11415
+ `The "${mode}" input mode is off: inputModes is ${JSON.stringify(modes)}.`
11416
+ );
11417
+ }
11418
+ }
11419
+ /**
11420
+ * A typed reply needs the `text` mode; one of the current reply options
11421
+ * needs `options` or `text`. Without a configuration the server's rules
11422
+ * apply alone, as before.
11423
+ */
11424
+ assertReplyMode(message) {
11425
+ if (!this.configuredInputModes) {
11426
+ return;
11427
+ }
11428
+ const modes = this.inputModes;
11429
+ if (modes.includes("text")) {
11430
+ return;
11431
+ }
11432
+ const reply = message.trim();
11433
+ const isOption = this.latestOptions.some(
11434
+ (option) => option.label.trim() === reply
11435
+ );
11436
+ if (isOption && modes.includes("options")) {
11437
+ return;
11438
+ }
11439
+ throw new InputModeError(
11440
+ isOption ? `Reply options are off: inputModes is ${JSON.stringify(modes)}.` : `Typed replies are off: inputModes is ${JSON.stringify(modes)}. Send one of the reply options or use startListening().`
11441
+ );
11442
+ }
9600
11443
  invokeThinkingCallback(payload) {
9601
11444
  this.thinkingState = true;
9602
11445
  if (this.thinkingCallback) {
@@ -9609,6 +11452,12 @@ var Character = class extends CharacterRuntime {
9609
11452
  }
9610
11453
  };
9611
11454
  var Npcify = class extends Character {
11455
+ constructor(options) {
11456
+ super({
11457
+ ...options,
11458
+ normalizeIdleOrientation: options.normalizeIdleOrientation ?? true
11459
+ });
11460
+ }
9612
11461
  };
9613
11462
 
9614
11463
  // ../../clients/web-client/src/character/character.ts
@@ -10189,13 +12038,18 @@ export {
10189
12038
  CharacterSceneRig,
10190
12039
  FacialPerformanceLayer,
10191
12040
  FacialPerformanceMixer,
12041
+ INPUT_MODES,
12042
+ InputModeError,
12043
+ MicrophoneError,
10192
12044
  Npcify,
10193
12045
  Character as NpcifyCharacter,
10194
12046
  PERFORMANCE_FILE_EXTENSION,
10195
12047
  PERFORMANCE_MEDIA_TYPE,
10196
12048
  PerformanceRecordingSchemaVersion,
10197
12049
  RemoteAnimationController,
12050
+ SpeechInputError,
10198
12051
  analyzePerformanceRecording,
12052
+ bindPushToTalk,
10199
12053
  createCharacter,
10200
12054
  decodePerformance,
10201
12055
  encodePerformance,