@npcify/web 0.2.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -6,9 +6,209 @@ var SERVER_RENDER_PROTOCOL_VERSION = 3;
6
6
  var MAXIMUM_PLAYBACK_DURATION_SECONDS = 10 * 60;
7
7
 
8
8
  // ../ts-shared/src/tts.ts
9
- var TTS_PROVIDERS = ["11labs"];
9
+ var TTS_PROVIDERS = ["11labs", "gemini"];
10
10
  var DEFAULT_TTS_PROVIDER = TTS_PROVIDERS[0];
11
11
 
12
+ // ../ts-shared/src/ai-settings.ts
13
+ var GPT_IMAGE_MODELS = [
14
+ "gpt-image-2.5-flare",
15
+ "gpt-image-2.5-sunburst"
16
+ ];
17
+ var AI_ROLES = {
18
+ character_draft: {
19
+ name: "Character draft",
20
+ source: "llm",
21
+ models: [],
22
+ options: ["temperature"]
23
+ },
24
+ motion_planner: {
25
+ name: "Motion planner",
26
+ source: "llm",
27
+ models: [],
28
+ options: []
29
+ },
30
+ // Not charged.
31
+ prompt_check: {
32
+ name: "Prompt check",
33
+ source: "openai",
34
+ models: ["gpt-6-luna"],
35
+ options: []
36
+ },
37
+ // Priced by OPENAI_IMAGE_PRICES of the server (flare and sunburst alike).
38
+ design_sheet_image: {
39
+ name: "Design sheet image",
40
+ source: "openai",
41
+ models: GPT_IMAGE_MODELS,
42
+ options: []
43
+ },
44
+ // Not charged per call.
45
+ tts: {
46
+ name: "ElevenLabs text to speech",
47
+ source: "elevenlabs",
48
+ models: ["eleven_v3_conversational", "eleven_v3"],
49
+ options: []
50
+ },
51
+ // Not charged per call.
52
+ gemini_tts: {
53
+ name: "Gemini text to speech",
54
+ source: "gemini",
55
+ models: ["gemini-3.8-flash-tts", "gemini-3.8-flash-lite-tts"],
56
+ options: []
57
+ },
58
+ // Priced by OPENAI_PRICES.text of the generator.
59
+ generator_agent: {
60
+ name: "Generator agent",
61
+ source: "openai",
62
+ models: ["gpt-6.1-sol"],
63
+ options: ["reasoningEffort"]
64
+ },
65
+ generator_reading: {
66
+ name: "Generator face reading",
67
+ source: "openai",
68
+ models: ["gpt-6.1-sol"],
69
+ options: ["reasoningEffort"]
70
+ },
71
+ // Priced by OPENAI_PRICES of the generator (flare and sunburst alike).
72
+ generator_image: {
73
+ name: "Generator images",
74
+ source: "openai",
75
+ models: GPT_IMAGE_MODELS,
76
+ options: []
77
+ }
78
+ };
79
+ var AI_ROLE_NAMES = Object.keys(AI_ROLES);
80
+
81
+ // ../ts-shared/src/speech-input.ts
82
+ var INPUT_MODES = ["options", "text", "voice"];
83
+ var InputModeError = class extends Error {
84
+ constructor(message) {
85
+ super(message);
86
+ this.name = "NpcifyInputModeError";
87
+ }
88
+ };
89
+ var permittedInputModes = function(permissions) {
90
+ if (permissions.baked || !permissions.allowCustomReply) {
91
+ return ["options"];
92
+ }
93
+ return permissions.speechInput ? ["options", "text", "voice"] : ["options", "text"];
94
+ };
95
+ var isInputMode = function(value) {
96
+ return INPUT_MODES.includes(value);
97
+ };
98
+ var parseInputModes = function(value) {
99
+ if (value === void 0 || value === null) {
100
+ return void 0;
101
+ }
102
+ if (!Array.isArray(value)) {
103
+ throw new InputModeError(
104
+ 'inputModes must be an array of "options", "text" and "voice".'
105
+ );
106
+ }
107
+ if (value.length === 0) {
108
+ throw new InputModeError("inputModes must have at least one mode.");
109
+ }
110
+ const modes = [];
111
+ for (const mode of value) {
112
+ if (!isInputMode(mode)) {
113
+ throw new InputModeError(
114
+ `inputModes has an unknown mode ${JSON.stringify(mode)}. Use "options", "text" or "voice".`
115
+ );
116
+ }
117
+ if (modes.includes(mode)) {
118
+ throw new InputModeError(`inputModes has "${mode}" two times.`);
119
+ }
120
+ modes.push(mode);
121
+ }
122
+ return modes;
123
+ };
124
+ var resolveInputModes = function(configured, permissions) {
125
+ const permitted = permittedInputModes(permissions);
126
+ if (!configured) {
127
+ return permitted;
128
+ }
129
+ if (permissions.baked && !(configured.length === 1 && configured[0] === "options")) {
130
+ throw new InputModeError(
131
+ 'A baked (offline) conversation accepts only reply options. Set inputModes to ["options"] or remove it.'
132
+ );
133
+ }
134
+ const refused = configured.filter((mode) => !permitted.includes(mode));
135
+ if (refused.length > 0) {
136
+ const names = refused.map((mode) => `"${mode}"`).join(" and ");
137
+ throw new InputModeError(
138
+ permissions.allowCustomReply ? `Voice input is off for this character (its owner turns it on in the STT tab). Remove ${names} from inputModes.` : `This character accepts only its reply options (Allow custom replies is off). Remove ${names} from inputModes.`
139
+ );
140
+ }
141
+ return [...configured];
142
+ };
143
+ var SPEECH_AUDIO_FORMAT = Object.freeze({
144
+ encoding: "pcm_s16le",
145
+ sampleRate: 16e3,
146
+ channels: 1
147
+ });
148
+ var SPEECH_BYTES_PER_SECOND = 32e3;
149
+ var SPEECH_MAX_DURATION_MS = 3e4;
150
+ var SPEECH_MAX_AUDIO_BYTES = SPEECH_MAX_DURATION_MS / 1e3 * SPEECH_BYTES_PER_SECOND;
151
+ var SPEECH_MAX_UPLOAD_BYTES = SPEECH_MAX_AUDIO_BYTES + 4096;
152
+ var SPEECH_FRAME_MS = 100;
153
+ var SPEECH_MIN_HOLD_MS = 300;
154
+ var SPEECH_EVENTS = Object.freeze({
155
+ start: "speech:start",
156
+ audio: "speech:audio",
157
+ stop: "speech:stop",
158
+ cancel: "speech:cancel",
159
+ partial: "speech:partial",
160
+ turn: "speech:turn",
161
+ turnFailed: "speech:turn_failed"
162
+ });
163
+
164
+ // ../ts-shared/src/speech-input-providers.ts
165
+ var SPEECH_INPUT_PROVIDER_PROFILES = Object.freeze({
166
+ // The file model also streams: 120 ms slower than gpt-live-transcribe,
167
+ // at a 3.8 times lower price and with fewer errors.
168
+ openai: {
169
+ live: {
170
+ model: "gpt-transcribe",
171
+ releaseToTextMs: 783,
172
+ usdPerMinute: 45e-4,
173
+ approximate: false
174
+ },
175
+ file: {
176
+ model: "gpt-transcribe",
177
+ releaseToTextMs: 867,
178
+ usdPerMinute: 45e-4,
179
+ approximate: false
180
+ }
181
+ },
182
+ gemini: {
183
+ live: {
184
+ model: "gemini-3.5-transcribe-live",
185
+ releaseToTextMs: 266,
186
+ usdPerMinute: 9e-3,
187
+ approximate: true
188
+ },
189
+ file: {
190
+ model: "gemini-3.5-transcribe",
191
+ releaseToTextMs: 1263,
192
+ usdPerMinute: 5e-3,
193
+ approximate: true
194
+ }
195
+ },
196
+ elevenlabs: {
197
+ live: {
198
+ model: "scribe_v2_realtime",
199
+ releaseToTextMs: 208,
200
+ usdPerMinute: 0.39 / 60,
201
+ approximate: false
202
+ },
203
+ file: {
204
+ model: "scribe_v2",
205
+ releaseToTextMs: 925,
206
+ usdPerMinute: 0.22 / 60,
207
+ approximate: false
208
+ }
209
+ }
210
+ });
211
+
12
212
  // ../ts-shared/src/product-usage.ts
13
213
  var productUsageEvents = [
14
214
  "page_viewed",
@@ -17,7 +217,11 @@ var productUsageEvents = [
17
217
  "checkout_submitted",
18
218
  "checkout_failed",
19
219
  "checkout_closed",
20
- "paywall_shown"
220
+ "paywall_shown",
221
+ "playback_blocked",
222
+ "playback_start_pressed",
223
+ "reply_options_shown",
224
+ "page_left"
21
225
  ];
22
226
 
23
227
  // ../ts-shared/src/analytics.ts
@@ -3122,8 +3326,14 @@ var LiveConversation = class {
3122
3326
  this.dependencies = dependencies;
3123
3327
  }
3124
3328
  sending = false;
3329
+ /** A typed turn is in flight. */
3330
+ get inFlight() {
3331
+ return this.sending;
3332
+ }
3125
3333
  pending = null;
3334
+ revision = 0;
3126
3335
  reset() {
3336
+ this.revision += 1;
3127
3337
  this.pending = null;
3128
3338
  }
3129
3339
  received(payload) {
@@ -3136,6 +3346,7 @@ var LiveConversation = class {
3136
3346
  throw new Error("A reply is already being generated. Please wait.");
3137
3347
  }
3138
3348
  this.sending = true;
3349
+ const revision = this.revision;
3139
3350
  const { telemetry, socket } = this.dependencies;
3140
3351
  try {
3141
3352
  telemetry.begin(conversationId);
@@ -3143,10 +3354,10 @@ var LiveConversation = class {
3143
3354
  const silent = options?.silent === true;
3144
3355
  const parentStagingId = options?.parentStagingId;
3145
3356
  const pending = this.pending;
3146
- const request = pending?.conversationId === conversationId && pending.message === message && pending.silent === silent && pending.parentStagingId === parentStagingId ? pending : {
3357
+ const request = !options?.requestId && pending?.conversationId === conversationId && pending.message === message && pending.silent === silent && pending.parentStagingId === parentStagingId ? pending : {
3147
3358
  conversationId,
3148
3359
  message,
3149
- requestId: globalThis.crypto.randomUUID(),
3360
+ requestId: options?.requestId ?? globalThis.crypto.randomUUID(),
3150
3361
  silent,
3151
3362
  parentStagingId
3152
3363
  };
@@ -3180,6 +3391,9 @@ var LiveConversation = class {
3180
3391
  if (this.pending?.requestId === request.requestId) {
3181
3392
  this.pending = null;
3182
3393
  }
3394
+ if (revision !== this.revision) {
3395
+ return payload;
3396
+ }
3183
3397
  await this.dependencies.handleResult(
3184
3398
  payload,
3185
3399
  payload.taskId !== request.requestId,
@@ -4310,7 +4524,7 @@ var disposeSceneContext = function(ctx) {
4310
4524
 
4311
4525
  // ../vrm-renderer/src/preview-controller.ts
4312
4526
  import { VRMUtils as VRMUtils4 } from "@pixiv/three-vrm";
4313
- import { Box3, GridHelper, Vector3 as Vector33 } from "three";
4527
+ import { Box3, GridHelper, Vector3 as Vector34 } from "three";
4314
4528
 
4315
4529
  // ../vrm-renderer/src/loaders.ts
4316
4530
  import { VRMLoaderPlugin as VRMLoaderPlugin2, VRMUtils as VRMUtils2 } from "@pixiv/three-vrm";
@@ -4332,7 +4546,33 @@ import { AnimationMixer, LoopOnce, LoopRepeat } from "three";
4332
4546
  import {
4333
4547
  createVRMAnimationClip as createPixivVRMAnimationClip
4334
4548
  } from "@pixiv/three-vrm-animation";
4549
+ import {
4550
+ AnimationClip,
4551
+ Euler,
4552
+ Quaternion as Quaternion2,
4553
+ QuaternionKeyframeTrack,
4554
+ QuaternionLinearInterpolant,
4555
+ Vector3 as Vector33,
4556
+ VectorKeyframeTrack
4557
+ } from "three";
4335
4558
  var NPCIFY_VRM_NORMALIZATION_ROOT_NAME = "NPCifyNormalizationRoot";
4559
+ var TORSO_BONES = [
4560
+ "hips",
4561
+ "spine",
4562
+ "chest",
4563
+ "upperChest"
4564
+ ];
4565
+ var HEAD_BONES = ["neck", "head"];
4566
+ var AXIAL_BONES = [...TORSO_BONES, ...HEAD_BONES];
4567
+ var PARENT_OF_HEAD_BONES = [
4568
+ ...TORSO_BONES,
4569
+ "neck"
4570
+ ];
4571
+ var IDENTITY_QUATERNION = new Quaternion2();
4572
+ var UP_AXIS = new Vector33(0, 1, 0);
4573
+ var MAX_HEAD_PITCH_CORRECTION_RAD = 35 * Math.PI / 180;
4574
+ var NECK_CORRECTION_SHARE = 0.35;
4575
+ var EPSILON = 1e-5;
4336
4576
  var canonicalAnimationTargets = /* @__PURE__ */ new WeakMap();
4337
4577
  var isNpcifyNormalizedVrm = function(vrm) {
4338
4578
  return vrm.scene.children.some(
@@ -4358,8 +4598,300 @@ var canonicalAnimationTarget = function(vrm) {
4358
4598
  canonicalAnimationTargets.set(vrm, target);
4359
4599
  return target;
4360
4600
  };
4361
- var createVRMAnimationClip = function(animation, vrm) {
4362
- return createPixivVRMAnimationClip(animation, canonicalAnimationTarget(vrm));
4601
+ var resolveAxialBoneNodeNames = function(vrm) {
4602
+ const result = /* @__PURE__ */ new Map();
4603
+ const seenNodeNames = /* @__PURE__ */ new Set();
4604
+ for (const bone of AXIAL_BONES) {
4605
+ const nodeName = vrm.humanoid?.getNormalizedBoneNode?.(bone)?.name;
4606
+ if (nodeName && !seenNodeNames.has(nodeName)) {
4607
+ seenNodeNames.add(nodeName);
4608
+ result.set(bone, nodeName);
4609
+ }
4610
+ }
4611
+ return result;
4612
+ };
4613
+ var findQuaternionTrack = function(clip, nodeName) {
4614
+ if (!nodeName) {
4615
+ return void 0;
4616
+ }
4617
+ const trackName = `${nodeName}.quaternion`;
4618
+ return clip.tracks.find(
4619
+ (track) => track.name === trackName && track.times.length > 0 && track.values.length >= track.times.length * 4
4620
+ );
4621
+ };
4622
+ var sampleBoneQuaternion = function(interpolant, time, target) {
4623
+ if (!interpolant) {
4624
+ return target.identity();
4625
+ }
4626
+ const values = interpolant.evaluate(time);
4627
+ return target.set(values[0] ?? 0, values[1] ?? 0, values[2] ?? 0, values[3] ?? 1).normalize();
4628
+ };
4629
+ var evaluateChainOrientation = function(bones, interpolants, time, rootPreRotation, target, scratch) {
4630
+ target.copy(rootPreRotation);
4631
+ for (const bone of bones) {
4632
+ const interpolant = interpolants.get(bone);
4633
+ if (interpolant) {
4634
+ sampleBoneQuaternion(interpolant, time, scratch);
4635
+ target.multiply(scratch);
4636
+ }
4637
+ }
4638
+ return target;
4639
+ };
4640
+ var rotateQuaternionTrackInParentSpace = function(track, parentBones, interpolants, rootPreRotation, worldCorrection) {
4641
+ const values = new Float32Array(track.values.length);
4642
+ const parentQuat = new Quaternion2();
4643
+ const parentInvQuat = new Quaternion2();
4644
+ const localCorrection = new Quaternion2();
4645
+ const keyframeQuat = new Quaternion2();
4646
+ const prevOrigQuat = new Quaternion2();
4647
+ const prevNewQuat = new Quaternion2();
4648
+ const scratchQuat = new Quaternion2();
4649
+ const count = track.times.length;
4650
+ for (let i = 0; i < count; i += 1) {
4651
+ const time = track.times[i] ?? 0;
4652
+ const offset = i * 4;
4653
+ keyframeQuat.set(
4654
+ track.values[offset] ?? 0,
4655
+ track.values[offset + 1] ?? 0,
4656
+ track.values[offset + 2] ?? 0,
4657
+ track.values[offset + 3] ?? 1
4658
+ ).normalize();
4659
+ const origDot = i > 0 ? prevOrigQuat.dot(keyframeQuat) : 1;
4660
+ prevOrigQuat.copy(keyframeQuat);
4661
+ if (parentBones.length === 0) {
4662
+ keyframeQuat.premultiply(worldCorrection).normalize();
4663
+ } else {
4664
+ evaluateChainOrientation(
4665
+ parentBones,
4666
+ interpolants,
4667
+ time,
4668
+ rootPreRotation,
4669
+ parentQuat,
4670
+ scratchQuat
4671
+ );
4672
+ parentInvQuat.copy(parentQuat).invert();
4673
+ localCorrection.copy(parentInvQuat).multiply(worldCorrection).multiply(parentQuat);
4674
+ keyframeQuat.premultiply(localCorrection).normalize();
4675
+ }
4676
+ if (i > 0 && origDot >= 0 && prevNewQuat.dot(keyframeQuat) < 0) {
4677
+ keyframeQuat.set(
4678
+ -keyframeQuat.x,
4679
+ -keyframeQuat.y,
4680
+ -keyframeQuat.z,
4681
+ -keyframeQuat.w
4682
+ );
4683
+ }
4684
+ prevNewQuat.copy(keyframeQuat);
4685
+ values[offset] = keyframeQuat.x;
4686
+ values[offset + 1] = keyframeQuat.y;
4687
+ values[offset + 2] = keyframeQuat.z;
4688
+ values[offset + 3] = keyframeQuat.w;
4689
+ }
4690
+ return new QuaternionKeyframeTrack(
4691
+ track.name,
4692
+ Array.from(track.times),
4693
+ Array.from(values)
4694
+ );
4695
+ };
4696
+ var rotateHipsPositionTrack = function(track, yawRadians) {
4697
+ const count = Math.floor(track.values.length / 3);
4698
+ if (count === 0) {
4699
+ return new VectorKeyframeTrack(
4700
+ track.name,
4701
+ Array.from(track.times),
4702
+ Array.from(track.values)
4703
+ );
4704
+ }
4705
+ let meanX = 0;
4706
+ let meanZ = 0;
4707
+ for (let i = 0; i < count; i += 1) {
4708
+ meanX += track.values[i * 3] ?? 0;
4709
+ meanZ += track.values[i * 3 + 2] ?? 0;
4710
+ }
4711
+ meanX /= count;
4712
+ meanZ /= count;
4713
+ const cosYaw = Math.cos(yawRadians);
4714
+ const sinYaw = Math.sin(yawRadians);
4715
+ const values = new Float32Array(track.values.length);
4716
+ for (let i = 0; i < count; i += 1) {
4717
+ const offset = i * 3;
4718
+ const dx = (track.values[offset] ?? 0) - meanX;
4719
+ const y = track.values[offset + 1] ?? 0;
4720
+ const dz = (track.values[offset + 2] ?? 0) - meanZ;
4721
+ values[offset] = meanX + dx * cosYaw + dz * sinYaw;
4722
+ values[offset + 1] = y;
4723
+ values[offset + 2] = meanZ - dx * sinYaw + dz * cosYaw;
4724
+ }
4725
+ return new VectorKeyframeTrack(
4726
+ track.name,
4727
+ Array.from(track.times),
4728
+ Array.from(values)
4729
+ );
4730
+ };
4731
+ var normalizeIdleAnimationClip = function(clip, vrm) {
4732
+ const boneNodeNames = resolveAxialBoneNodeNames(vrm);
4733
+ if (boneNodeNames.size === 0) {
4734
+ return clip;
4735
+ }
4736
+ const interpolants = /* @__PURE__ */ new Map();
4737
+ for (const bone of AXIAL_BONES) {
4738
+ const track = findQuaternionTrack(clip, boneNodeNames.get(bone));
4739
+ if (track) {
4740
+ interpolants.set(
4741
+ bone,
4742
+ new QuaternionLinearInterpolant(track.times, track.values, 4)
4743
+ );
4744
+ }
4745
+ }
4746
+ if (interpolants.size === 0) {
4747
+ return clip;
4748
+ }
4749
+ const duration = clip.duration > 0 ? clip.duration : 0;
4750
+ const sampleCount = duration > 0 ? Math.max(16, Math.min(120, Math.ceil(duration * 30))) : 1;
4751
+ const sampleTimes = [];
4752
+ for (let i = 0; i < sampleCount; i += 1) {
4753
+ sampleTimes.push(duration > 0 ? (i + 0.5) / sampleCount * duration : 0);
4754
+ }
4755
+ const chainQuat = new Quaternion2();
4756
+ const scratchQuat = new Quaternion2();
4757
+ const forwardVec = new Vector33();
4758
+ const rootTorsoBone = TORSO_BONES.find((bone) => interpolants.has(bone));
4759
+ const rootPreRotation = new Quaternion2();
4760
+ let torsoYawCorrection = 0;
4761
+ if (rootTorsoBone) {
4762
+ let sumX = 0;
4763
+ let sumZ = 0;
4764
+ for (const time of sampleTimes) {
4765
+ evaluateChainOrientation(
4766
+ TORSO_BONES,
4767
+ interpolants,
4768
+ time,
4769
+ IDENTITY_QUATERNION,
4770
+ chainQuat,
4771
+ scratchQuat
4772
+ );
4773
+ forwardVec.set(0, 0, 1).applyQuaternion(chainQuat);
4774
+ const horizLen = Math.hypot(forwardVec.x, forwardVec.z);
4775
+ if (horizLen > EPSILON) {
4776
+ sumX += forwardVec.x / horizLen;
4777
+ sumZ += forwardVec.z / horizLen;
4778
+ }
4779
+ }
4780
+ if (Math.hypot(sumX, sumZ) > EPSILON) {
4781
+ const meanTorsoYaw = Math.atan2(sumX, sumZ);
4782
+ if (Math.abs(meanTorsoYaw) > EPSILON) {
4783
+ torsoYawCorrection = -meanTorsoYaw;
4784
+ rootPreRotation.setFromAxisAngle(UP_AXIS, torsoYawCorrection);
4785
+ }
4786
+ }
4787
+ }
4788
+ const hasNeckTrack = interpolants.has("neck");
4789
+ const hasHeadTrack = interpolants.has("head");
4790
+ let neckWorldCorrection = null;
4791
+ let headWorldCorrection = null;
4792
+ if (hasNeckTrack || hasHeadTrack) {
4793
+ const meanHeadForward = new Vector33();
4794
+ for (const time of sampleTimes) {
4795
+ evaluateChainOrientation(
4796
+ AXIAL_BONES,
4797
+ interpolants,
4798
+ time,
4799
+ rootPreRotation,
4800
+ chainQuat,
4801
+ scratchQuat
4802
+ );
4803
+ forwardVec.set(0, 0, 1).applyQuaternion(chainQuat);
4804
+ meanHeadForward.add(forwardVec);
4805
+ }
4806
+ if (meanHeadForward.lengthSq() > EPSILON * EPSILON) {
4807
+ meanHeadForward.normalize();
4808
+ const meanHeadYaw = Math.atan2(meanHeadForward.x, meanHeadForward.z);
4809
+ const rawHeadPitch = Math.atan2(
4810
+ -meanHeadForward.y,
4811
+ Math.hypot(meanHeadForward.x, meanHeadForward.z)
4812
+ );
4813
+ const clampedHeadPitch = Math.max(
4814
+ -MAX_HEAD_PITCH_CORRECTION_RAD,
4815
+ Math.min(MAX_HEAD_PITCH_CORRECTION_RAD, rawHeadPitch)
4816
+ );
4817
+ if (Math.abs(meanHeadYaw) > EPSILON || Math.abs(clampedHeadPitch) > EPSILON) {
4818
+ const totalWorldCorrection = new Quaternion2().setFromEuler(
4819
+ new Euler(-clampedHeadPitch, -meanHeadYaw, 0, "XYZ")
4820
+ );
4821
+ if (hasNeckTrack && hasHeadTrack) {
4822
+ neckWorldCorrection = new Quaternion2().setFromEuler(
4823
+ new Euler(
4824
+ -clampedHeadPitch * NECK_CORRECTION_SHARE,
4825
+ -meanHeadYaw * NECK_CORRECTION_SHARE,
4826
+ 0,
4827
+ "XYZ"
4828
+ )
4829
+ );
4830
+ headWorldCorrection = neckWorldCorrection.clone().invert().multiply(totalWorldCorrection);
4831
+ } else if (hasNeckTrack) {
4832
+ neckWorldCorrection = totalWorldCorrection;
4833
+ } else {
4834
+ headWorldCorrection = totalWorldCorrection;
4835
+ }
4836
+ }
4837
+ }
4838
+ }
4839
+ const hasTorsoCorrection = Math.abs(torsoYawCorrection) > EPSILON;
4840
+ if (!hasTorsoCorrection && !neckWorldCorrection && !headWorldCorrection) {
4841
+ return clip;
4842
+ }
4843
+ const rootTorsoNodeName = rootTorsoBone ? boneNodeNames.get(rootTorsoBone) : void 0;
4844
+ const rootTorsoQuatTrackName = rootTorsoNodeName ? `${rootTorsoNodeName}.quaternion` : null;
4845
+ const hipsNodeName = boneNodeNames.get("hips");
4846
+ const hipsPosTrackName = hipsNodeName && rootTorsoBone === "hips" ? `${hipsNodeName}.position` : null;
4847
+ const neckNodeName = boneNodeNames.get("neck");
4848
+ const neckQuatTrackName = neckNodeName ? `${neckNodeName}.quaternion` : null;
4849
+ const headNodeName = boneNodeNames.get("head");
4850
+ const headQuatTrackName = headNodeName ? `${headNodeName}.quaternion` : null;
4851
+ const nextTracks = clip.tracks.map((track) => {
4852
+ if (hasTorsoCorrection && track.name === rootTorsoQuatTrackName) {
4853
+ return rotateQuaternionTrackInParentSpace(
4854
+ track,
4855
+ [],
4856
+ interpolants,
4857
+ IDENTITY_QUATERNION,
4858
+ rootPreRotation
4859
+ );
4860
+ }
4861
+ if (hasTorsoCorrection && track.name === hipsPosTrackName) {
4862
+ return rotateHipsPositionTrack(track, torsoYawCorrection);
4863
+ }
4864
+ if (neckWorldCorrection && track.name === neckQuatTrackName) {
4865
+ return rotateQuaternionTrackInParentSpace(
4866
+ track,
4867
+ TORSO_BONES,
4868
+ interpolants,
4869
+ rootPreRotation,
4870
+ neckWorldCorrection
4871
+ );
4872
+ }
4873
+ if (headWorldCorrection && track.name === headQuatTrackName) {
4874
+ return rotateQuaternionTrackInParentSpace(
4875
+ track,
4876
+ PARENT_OF_HEAD_BONES,
4877
+ interpolants,
4878
+ rootPreRotation,
4879
+ headWorldCorrection
4880
+ );
4881
+ }
4882
+ return track;
4883
+ });
4884
+ return new AnimationClip(clip.name, clip.duration, nextTracks, clip.blendMode);
4885
+ };
4886
+ var createVRMAnimationClip = function(animation, vrm, options) {
4887
+ const clip = createPixivVRMAnimationClip(
4888
+ animation,
4889
+ canonicalAnimationTarget(vrm)
4890
+ );
4891
+ if (options?.normalizeIdleOrientation) {
4892
+ return normalizeIdleAnimationClip(clip, vrm);
4893
+ }
4894
+ return clip;
4363
4895
  };
4364
4896
 
4365
4897
  // ../vrm-renderer/src/session.ts
@@ -4696,16 +5228,20 @@ var RemoteAnimationController = class {
4696
5228
  }
4697
5229
  };
4698
5230
  var AnimationController = class {
4699
- constructor(vrm, idleAnimationId) {
5231
+ constructor(vrm, idleAnimationId, options) {
4700
5232
  this.vrm = vrm;
4701
5233
  this.mixer = new AnimationMixer2(this.vrm.scene);
4702
5234
  this.idleAnimationId = idleAnimationId ?? null;
5235
+ this.normalizeIdleOrientation = options?.normalizeIdleOrientation ?? false;
4703
5236
  this.mixer.addEventListener("finished", this.handleActionFinished);
4704
5237
  }
4705
5238
  prepareAnimations;
4706
5239
  mixer;
4707
5240
  clips = /* @__PURE__ */ new Map();
5241
+ rawClips = /* @__PURE__ */ new Map();
5242
+ normalizedIdleClips = /* @__PURE__ */ new Map();
4708
5243
  configs = /* @__PURE__ */ new Map();
5244
+ normalizeIdleOrientation;
4709
5245
  previewMixer = null;
4710
5246
  restorePreview = null;
4711
5247
  activeAction = null;
@@ -4731,8 +5267,10 @@ var AnimationController = class {
4731
5267
  return this.clips.has(id);
4732
5268
  }
4733
5269
  addAnimation(animationId, animation, config) {
4734
- const clip = createVRMAnimationClip(animation, this.vrm);
4735
- this.clips.set(animationId, clip);
5270
+ const rawClip = createVRMAnimationClip(animation, this.vrm);
5271
+ this.rawClips.set(animationId, rawClip);
5272
+ this.normalizedIdleClips.delete(animationId);
5273
+ this.clips.set(animationId, this.resolveClipForId(animationId, rawClip));
4736
5274
  this.configs.set(animationId, {
4737
5275
  loop: config?.loop ?? defaultAnimationConfig.loop,
4738
5276
  weight: config?.weight ?? defaultAnimationConfig.weight
@@ -4762,7 +5300,20 @@ var AnimationController = class {
4762
5300
  return restore;
4763
5301
  }
4764
5302
  setIdleAnimation(animationId) {
5303
+ const previousIdleId = this.idleAnimationId;
4765
5304
  this.idleAnimationId = animationId;
5305
+ if (previousIdleId && previousIdleId !== animationId) {
5306
+ const previousRaw = this.rawClips.get(previousIdleId);
5307
+ if (previousRaw) {
5308
+ this.clips.set(previousIdleId, previousRaw);
5309
+ }
5310
+ }
5311
+ if (animationId) {
5312
+ const rawClip = this.rawClips.get(animationId);
5313
+ if (rawClip) {
5314
+ this.clips.set(animationId, this.resolveClipForId(animationId, rawClip));
5315
+ }
5316
+ }
4766
5317
  this.playIdle();
4767
5318
  }
4768
5319
  clearAnimations() {
@@ -4770,6 +5321,8 @@ var AnimationController = class {
4770
5321
  this.mixer.stopAllAction();
4771
5322
  this.mixer.uncacheRoot(this.vrm.scene);
4772
5323
  this.clips.clear();
5324
+ this.rawClips.clear();
5325
+ this.normalizedIdleClips.clear();
4773
5326
  this.configs.clear();
4774
5327
  this.activeAction = null;
4775
5328
  this.activeAnimationId = null;
@@ -4777,6 +5330,18 @@ var AnimationController = class {
4777
5330
  this.idleAnimationId = null;
4778
5331
  this.vrm.humanoid.resetNormalizedPose();
4779
5332
  }
5333
+ resolveClipForId(animationId, rawClip) {
5334
+ if (!this.normalizeIdleOrientation || animationId !== this.idleAnimationId) {
5335
+ return rawClip;
5336
+ }
5337
+ const cached = this.normalizedIdleClips.get(animationId);
5338
+ if (cached) {
5339
+ return cached;
5340
+ }
5341
+ const normalized = normalizeIdleAnimationClip(rawClip, this.vrm);
5342
+ this.normalizedIdleClips.set(animationId, normalized);
5343
+ return normalized;
5344
+ }
4780
5345
  playAnimation(animationId, fadeDuration = 0.25, loopOverride) {
4781
5346
  this.restorePreview?.();
4782
5347
  const clip = this.clips.get(animationId);
@@ -5545,6 +6110,7 @@ var StreamPlaybackController = class {
5545
6110
  force: options?.force === true
5546
6111
  });
5547
6112
  }
6113
+ /** Returns the playback result for a timeline event. */
5548
6114
  handleEvent(event) {
5549
6115
  if (this.streamEventCallback) {
5550
6116
  this.safeInvoke("onStreamEvent", () => this.streamEventCallback?.(event));
@@ -5588,14 +6154,13 @@ var StreamPlaybackController = class {
5588
6154
  if (event.type === "staging" || event.type === "staging_prefix" || event.type === "staging_live") {
5589
6155
  this.recordStreamEvent(event, "timeline");
5590
6156
  this.dependencies.telemetry.recordFirstStaging();
5591
- void this.playTimeline({
6157
+ return this.playTimeline({
5592
6158
  taskId: event.taskId,
5593
6159
  stagingId: event.stagingId,
5594
6160
  timeline: event.timeline,
5595
6161
  complete: event.complete,
5596
6162
  revision: event.revision
5597
6163
  });
5598
- return;
5599
6164
  }
5600
6165
  if (event.type === "options") {
5601
6166
  this.recordStreamEvent(event, "options");
@@ -5954,8 +6519,18 @@ var decodeBase64 = function(value) {
5954
6519
  }
5955
6520
  return bytes;
5956
6521
  };
5957
- var canUseMediaSource = function(mimeType) {
5958
- return typeof MediaSource !== "undefined" && typeof URL?.createObjectURL === "function" && MediaSource.isTypeSupported(mimeType);
6522
+ var managedMediaSource = function() {
6523
+ return globalThis.ManagedMediaSource;
6524
+ };
6525
+ var needsManagedMediaSource = function() {
6526
+ return typeof MediaSource === "undefined" && Boolean(managedMediaSource());
6527
+ };
6528
+ var streamingMediaSource = function(mimeType) {
6529
+ if (typeof URL?.createObjectURL !== "function") {
6530
+ return null;
6531
+ }
6532
+ const source = typeof MediaSource === "undefined" ? managedMediaSource() : MediaSource;
6533
+ return source?.isTypeSupported(mimeType) ? source : null;
5959
6534
  };
5960
6535
  var StreamingAudioBuffer = class {
5961
6536
  constructor(mimeType) {
@@ -5998,8 +6573,9 @@ var StreamingAudioBuffer = class {
5998
6573
  if (this.objectUrl) {
5999
6574
  return this.objectUrl;
6000
6575
  }
6001
- if (canUseMediaSource(this.mimeType)) {
6002
- this.mediaSource = new MediaSource();
6576
+ const Source2 = streamingMediaSource(this.mimeType);
6577
+ if (Source2) {
6578
+ this.mediaSource = new Source2();
6003
6579
  this.objectUrl = URL.createObjectURL(this.mediaSource);
6004
6580
  this.mediaSource.addEventListener("sourceopen", this.handleSourceOpen, {
6005
6581
  once: true
@@ -6091,7 +6667,7 @@ var StreamingAudioBuffer = class {
6091
6667
  }
6092
6668
  };
6093
6669
  finishFallbackIfReady() {
6094
- if (!this.completed || this.objectUrl || canUseMediaSource(this.mimeType) || this.failedError || this.disposed) {
6670
+ if (!this.completed || this.objectUrl || streamingMediaSource(this.mimeType) || this.failedError || this.disposed) {
6095
6671
  return;
6096
6672
  }
6097
6673
  const blobParts = this.chunks.map((chunk) => Uint8Array.from(chunk).buffer);
@@ -6418,7 +6994,8 @@ var AudioPlayer = class {
6418
6994
  this.visemePlayer.cleanup();
6419
6995
  if (this.audioElement) {
6420
6996
  this.audioElement.srcObject = null;
6421
- this.audioElement.src = "";
6997
+ this.audioElement.removeAttribute("src");
6998
+ this.audioElement.load();
6422
6999
  }
6423
7000
  this.streams.forEach((stream) => stream.dispose());
6424
7001
  this.streams.clear();
@@ -6435,8 +7012,10 @@ var AudioPlayer = class {
6435
7012
  this.streams.delete(streamId);
6436
7013
  }
6437
7014
  buildSilentWav = function() {
6438
- const sampleCount = 8;
6439
- const bytes = new Uint8Array(44 + sampleCount);
7015
+ const sampleRate = 44100;
7016
+ const bytesPerSample = 2;
7017
+ const dataSize = Math.round(sampleRate * 0.1) * bytesPerSample;
7018
+ const bytes = new Uint8Array(44 + dataSize);
6440
7019
  const view = new DataView(bytes.buffer);
6441
7020
  const writeText = function(offset, value) {
6442
7021
  Array.from(value).forEach((character, index) => {
@@ -6444,19 +7023,18 @@ var AudioPlayer = class {
6444
7023
  });
6445
7024
  };
6446
7025
  writeText(0, "RIFF");
6447
- view.setUint32(4, 36 + sampleCount, true);
7026
+ view.setUint32(4, 36 + dataSize, true);
6448
7027
  writeText(8, "WAVE");
6449
7028
  writeText(12, "fmt ");
6450
7029
  view.setUint32(16, 16, true);
6451
7030
  view.setUint16(20, 1, true);
6452
7031
  view.setUint16(22, 1, true);
6453
- view.setUint32(24, 8e3, true);
6454
- view.setUint32(28, 8e3, true);
6455
- view.setUint16(32, 1, true);
6456
- view.setUint16(34, 8, true);
7032
+ view.setUint32(24, sampleRate, true);
7033
+ view.setUint32(28, sampleRate * bytesPerSample, true);
7034
+ view.setUint16(32, bytesPerSample, true);
7035
+ view.setUint16(34, bytesPerSample * 8, true);
6457
7036
  writeText(36, "data");
6458
- view.setUint32(40, sampleCount, true);
6459
- bytes.fill(128, 44);
7037
+ view.setUint32(40, dataSize, true);
6460
7038
  return new Blob([bytes], { type: "audio/wav" });
6461
7039
  };
6462
7040
  async resolveSourceUrl(source) {
@@ -6492,6 +7070,9 @@ var AudioPlayer = class {
6492
7070
  element.volume = this.volumeValue;
6493
7071
  element.muted = this.mutedValue;
6494
7072
  element.playbackRate = this.playbackRateValue;
7073
+ if (needsManagedMediaSource()) {
7074
+ element.disableRemotePlayback = true;
7075
+ }
6495
7076
  if (this.crossOriginValue !== null) {
6496
7077
  element.crossOrigin = this.crossOriginValue;
6497
7078
  }
@@ -6575,6 +7156,14 @@ var CharacterExpressions = class {
6575
7156
  clearEmotion(mouthMode) {
6576
7157
  this.mixer.clearEmotion(mouthMode);
6577
7158
  }
7159
+ /** Set a temporary emotion overlay. Timeline emotions and visemes keep their state. */
7160
+ setOverlay(weights) {
7161
+ this.mixer.setOverlay(weights);
7162
+ }
7163
+ /** Ease the temporary overlay back to the current timeline or idle face. */
7164
+ clearOverlay() {
7165
+ this.mixer.clearOverlay();
7166
+ }
6578
7167
  /** Reset all expression layer targets to zero. */
6579
7168
  reset() {
6580
7169
  this.mixer.reset();
@@ -6749,7 +7338,7 @@ var FacialPerformanceInspector = class {
6749
7338
  };
6750
7339
 
6751
7340
  // ../../clients/web-client/src/character/idle-face.ts
6752
- import { Vector3 as Vector34 } from "three";
7341
+ import { Vector3 as Vector35 } from "three";
6753
7342
  var IdleFace = class {
6754
7343
  constructor(vrm, random) {
6755
7344
  this.vrm = vrm;
@@ -6757,7 +7346,7 @@ var IdleFace = class {
6757
7346
  }
6758
7347
  time = 0;
6759
7348
  timeline;
6760
- targetPosition = new Vector34();
7349
+ targetPosition = new Vector35();
6761
7350
  /** Advances by `delta` seconds, turns the eyes, and returns the blink weight. */
6762
7351
  update(delta, frame) {
6763
7352
  if (Number.isFinite(delta) && delta > 0) {
@@ -6945,6 +7534,7 @@ var PerformanceWeightMixer = class {
6945
7534
  };
6946
7535
 
6947
7536
  // ../../clients/web-client/src/character/facial-performance-mixer.ts
7537
+ var VisemeResponseRate = 30;
6948
7538
  var DEFAULT_PLAY_DURATION_MS = 1500;
6949
7539
  var MOUTH_CHANNELS = [
6950
7540
  VRMExpressionPresetName.Aa,
@@ -6972,7 +7562,8 @@ var FacialPerformanceLayer = {
6972
7562
  Emotion: "emotion",
6973
7563
  Viseme: "viseme",
6974
7564
  Blink: "blink",
6975
- Gaze: "gaze"
7565
+ Gaze: "gaze",
7566
+ Overlay: "overlay"
6976
7567
  };
6977
7568
  var getFacialPerformanceLayerId = function(channel) {
6978
7569
  const normalized = channel.toLowerCase();
@@ -7123,6 +7714,22 @@ var FacialPerformanceMixer = class {
7123
7714
  }
7124
7715
  this.releaseVisemeMouthMode();
7125
7716
  }
7717
+ /** Blend temporary emotions without changing timeline targets or mouth ownership. */
7718
+ setOverlay(weights) {
7719
+ this.clearOverlay();
7720
+ Object.entries(weights).forEach(([channel, weight]) => {
7721
+ if (this.availableChannelSet.has(channel) && getFacialPerformanceLayerId(channel) === FacialPerformanceLayer.Emotion && Number.isFinite(weight)) {
7722
+ this.weightMixer.setTarget(
7723
+ FacialPerformanceLayer.Overlay,
7724
+ channel,
7725
+ weight
7726
+ );
7727
+ }
7728
+ });
7729
+ }
7730
+ clearOverlay() {
7731
+ this.weightMixer.clearLayerTargets(FacialPerformanceLayer.Overlay);
7732
+ }
7126
7733
  releaseVisemeMouthMode() {
7127
7734
  if (this.activeMouthMode !== "viseme") {
7128
7735
  return;
@@ -7240,7 +7847,9 @@ var FacialPerformanceMixer = class {
7240
7847
  this.weightMixer.registerLayer({
7241
7848
  id: FacialPerformanceLayer.Viseme,
7242
7849
  priority: 200,
7243
- blendMode: "override"
7850
+ blendMode: "override",
7851
+ // A 33 ms time constant: an 80 ms lip closure reaches 90% closed.
7852
+ responseRate: VisemeResponseRate
7244
7853
  });
7245
7854
  this.weightMixer.registerLayer({
7246
7855
  id: FacialPerformanceLayer.Blink,
@@ -7252,6 +7861,11 @@ var FacialPerformanceMixer = class {
7252
7861
  priority: 400,
7253
7862
  blendMode: "override"
7254
7863
  });
7864
+ this.weightMixer.registerLayer({
7865
+ id: FacialPerformanceLayer.Overlay,
7866
+ priority: 500,
7867
+ blendMode: "maximum"
7868
+ });
7255
7869
  }
7256
7870
  updateIdleFace(delta, values) {
7257
7871
  const manager = this.vrm?.expressionManager;
@@ -7376,11 +7990,11 @@ var FacialPerformanceMixer = class {
7376
7990
 
7377
7991
  // ../../clients/web-client/src/character/mesh.ts
7378
7992
  import {
7379
- Euler,
7993
+ Euler as Euler2,
7380
7994
  Mesh,
7381
7995
  Object3D,
7382
7996
  SkinnedMesh,
7383
- Vector3 as Vector35
7997
+ Vector3 as Vector36
7384
7998
  } from "three";
7385
7999
  var isSkinnedMesh = function(object) {
7386
8000
  return object instanceof SkinnedMesh || object.isSkinnedMesh === true;
@@ -7389,14 +8003,14 @@ var isMesh = function(object) {
7389
8003
  return isSkinnedMesh(object) || object instanceof Mesh || object.isMesh === true;
7390
8004
  };
7391
8005
  var applyVector3 = function(target, value) {
7392
- if (value instanceof Vector35) {
8006
+ if (value instanceof Vector36) {
7393
8007
  target.copy(value);
7394
8008
  return;
7395
8009
  }
7396
8010
  target.set(value[0], value[1], value[2]);
7397
8011
  };
7398
8012
  var applyEuler = function(target, value) {
7399
- if (value instanceof Euler) {
8013
+ if (value instanceof Euler2) {
7400
8014
  target.copy(value);
7401
8015
  return;
7402
8016
  }
@@ -7686,6 +8300,11 @@ var IncrementalStaging = class {
7686
8300
  const dialogue = timeline.dialogue.map((next, index) => {
7687
8301
  const previous = this.timeline.dialogue[index];
7688
8302
  if (previous?.type === "speech" && previous.pending && next.type === "speech") {
8303
+ for (const key of Object.keys(previous)) {
8304
+ if (!(key in next)) {
8305
+ Reflect.deleteProperty(previous, key);
8306
+ }
8307
+ }
7689
8308
  return Object.assign(previous, next, { pending: next.pending });
7690
8309
  }
7691
8310
  return next;
@@ -7748,6 +8367,30 @@ var IncrementalStaging = class {
7748
8367
  }
7749
8368
  };
7750
8369
 
8370
+ // ../../clients/web-client/src/character/playback-clock.ts
8371
+ var PlaybackClock = class {
8372
+ pausedAt = null;
8373
+ pausedSeconds = 0;
8374
+ wallTime() {
8375
+ return performance.now() / 1e3;
8376
+ }
8377
+ get paused() {
8378
+ return this.pausedAt !== null;
8379
+ }
8380
+ now() {
8381
+ return (this.pausedAt ?? this.wallTime()) - this.pausedSeconds;
8382
+ }
8383
+ pause() {
8384
+ this.pausedAt ??= this.wallTime();
8385
+ }
8386
+ resume() {
8387
+ if (this.pausedAt !== null) {
8388
+ this.pausedSeconds += this.wallTime() - this.pausedAt;
8389
+ this.pausedAt = null;
8390
+ }
8391
+ }
8392
+ };
8393
+
7751
8394
  // ../../clients/web-client/src/character/staging-character-cues.ts
7752
8395
  var findCharacterSequence = function(characters, sequence, fromIndex) {
7753
8396
  if (sequence.length === 0) {
@@ -7849,12 +8492,6 @@ var buildStagingCharacterCues = function(text, wordTimings, duration, characterT
7849
8492
  };
7850
8493
 
7851
8494
  // ../../clients/web-client/src/character/staging-player.ts
7852
- var nowSeconds = function() {
7853
- if (typeof performance !== "undefined") {
7854
- return performance.now() / 1e3;
7855
- }
7856
- return Date.now() / 1e3;
7857
- };
7858
8495
  var abortError2 = function() {
7859
8496
  const error = new Error("Staging playback was cancelled");
7860
8497
  error.name = "AbortError";
@@ -7886,6 +8523,8 @@ var StagingPlayer = class {
7886
8523
  this.facialMixer = facialMixer;
7887
8524
  this.animationController = animationController;
7888
8525
  }
8526
+ clock = new PlaybackClock();
8527
+ pauseRevision = 0;
7889
8528
  playbackController = null;
7890
8529
  generation = 0;
7891
8530
  incremental = null;
@@ -7895,6 +8534,40 @@ var StagingPlayer = class {
7895
8534
  activeExpression = null;
7896
8535
  activeExpressionMouthMode = null;
7897
8536
  activeAnimation = null;
8537
+ get paused() {
8538
+ return this.clock.paused;
8539
+ }
8540
+ pause() {
8541
+ this.pauseRevision += 1;
8542
+ this.clock.pause();
8543
+ this.audioPlayer.element?.pause();
8544
+ }
8545
+ async resume() {
8546
+ if (!this.paused) {
8547
+ return;
8548
+ }
8549
+ const generation = this.generation;
8550
+ const pauseRevision = this.pauseRevision;
8551
+ if (this.pendingSpeech?.started && this.pendingSpeech.audio.type !== "silent") {
8552
+ await this.audioPlayer.element?.play();
8553
+ }
8554
+ if (generation !== this.generation) {
8555
+ return;
8556
+ }
8557
+ if (pauseRevision !== this.pauseRevision) {
8558
+ this.audioPlayer.element?.pause();
8559
+ return;
8560
+ }
8561
+ this.clock.resume();
8562
+ }
8563
+ async waitUntilResumed(signal) {
8564
+ while (this.paused) {
8565
+ await waitFrame(signal);
8566
+ }
8567
+ if (signal.aborted) {
8568
+ throw abortError2();
8569
+ }
8570
+ }
7898
8571
  async play(timeline, options) {
7899
8572
  this.cancel(false);
7900
8573
  const controller = new AbortController();
@@ -7933,6 +8606,7 @@ var StagingPlayer = class {
7933
8606
  if (controller.signal.aborted) {
7934
8607
  throw abortError2();
7935
8608
  }
8609
+ await this.waitUntilResumed(controller.signal);
7936
8610
  if (block.startTime > cursor) {
7937
8611
  beginTimeline();
7938
8612
  await this.playTimedRange(
@@ -8016,12 +8690,12 @@ var StagingPlayer = class {
8016
8690
  }
8017
8691
  }
8018
8692
  startCursor(block, mediaClock) {
8019
- const startedAt = nowSeconds();
8693
+ const startedAt = this.clock.now();
8020
8694
  this.cursor = () => ({
8021
8695
  blockId: block.id,
8022
8696
  offsetSeconds: Math.min(
8023
8697
  block.endTime - block.startTime,
8024
- Math.max(0, mediaClock ? mediaClock() : nowSeconds() - startedAt)
8698
+ Math.max(0, mediaClock ? mediaClock() : this.clock.now() - startedAt)
8025
8699
  )
8026
8700
  });
8027
8701
  this.onCursor?.(this.cursor());
@@ -8045,6 +8719,7 @@ var StagingPlayer = class {
8045
8719
  this.generation += 1;
8046
8720
  this.playbackController?.abort();
8047
8721
  this.playbackController = null;
8722
+ this.clock.resume();
8048
8723
  this.incremental = null;
8049
8724
  this.clearCursor();
8050
8725
  this.onCursor = void 0;
@@ -8156,6 +8831,9 @@ var StagingPlayer = class {
8156
8831
  return;
8157
8832
  }
8158
8833
  started = true;
8834
+ if (this.paused) {
8835
+ this.audioPlayer.element?.pause();
8836
+ }
8159
8837
  if (this.pendingSpeech?.id === speech.id) {
8160
8838
  this.pendingSpeech.started = true;
8161
8839
  }
@@ -8176,6 +8854,7 @@ var StagingPlayer = class {
8176
8854
  let ticking = true;
8177
8855
  const tick = async () => {
8178
8856
  while (ticking && !signal.aborted) {
8857
+ await this.waitUntilResumed(signal);
8179
8858
  if (started) {
8180
8859
  observePhonemes();
8181
8860
  const localTime = Math.min(
@@ -8214,14 +8893,15 @@ var StagingPlayer = class {
8214
8893
  }
8215
8894
  }
8216
8895
  async playSilentSpeech(speech, timeline, signal, revealCharacters) {
8217
- const startedAt = nowSeconds();
8896
+ const startedAt = this.clock.now();
8218
8897
  const duration = Math.max(speech.endTime - speech.startTime, 0);
8219
8898
  let nextVisemeIndex = 0;
8220
8899
  while (true) {
8221
8900
  if (signal.aborted) {
8222
8901
  throw abortError2();
8223
8902
  }
8224
- const elapsed = Math.min(nowSeconds() - startedAt, duration);
8903
+ await this.waitUntilResumed(signal);
8904
+ const elapsed = Math.min(this.clock.now() - startedAt, duration);
8225
8905
  this.applyTracks(speech.startTime + elapsed, true, timeline);
8226
8906
  revealCharacters(elapsed);
8227
8907
  let viseme = speech.visemes[nextVisemeIndex];
@@ -8241,13 +8921,14 @@ var StagingPlayer = class {
8241
8921
  revealCharacters(Number.POSITIVE_INFINITY);
8242
8922
  }
8243
8923
  async playTimedRange(startTime, endTime, speechActive, timeline, signal) {
8244
- const startedAt = nowSeconds();
8924
+ const startedAt = this.clock.now();
8245
8925
  const duration = Math.max(endTime - startTime, 0);
8246
8926
  while (true) {
8247
8927
  if (signal.aborted) {
8248
8928
  throw abortError2();
8249
8929
  }
8250
- const elapsed = Math.min(nowSeconds() - startedAt, duration);
8930
+ await this.waitUntilResumed(signal);
8931
+ const elapsed = Math.min(this.clock.now() - startedAt, duration);
8251
8932
  this.applyTracks(startTime + elapsed, speechActive, timeline);
8252
8933
  if (elapsed >= duration) {
8253
8934
  return;
@@ -8747,13 +9428,10 @@ var CharacterRuntime = class {
8747
9428
  animationId,
8748
9429
  assetProfile: this.assetProfile
8749
9430
  }));
8750
- this.lazyAnimations = new LazyAnimations(
8751
- loadAnimation,
8752
- (id, animation) => {
8753
- this.animations.set(id, animation);
8754
- this.animationController?.addAnimation?.(id, animation, { loop: false });
8755
- }
8756
- );
9431
+ this.lazyAnimations = new LazyAnimations(loadAnimation, (id, animation) => {
9432
+ this.animations.set(id, animation);
9433
+ this.animationController?.addAnimation?.(id, animation, { loop: false });
9434
+ });
8757
9435
  this.performanceRecorder = new PerformanceRecorder({
8758
9436
  characterId: this.characterId
8759
9437
  });
@@ -8973,6 +9651,20 @@ var CharacterRuntime = class {
8973
9651
  });
8974
9652
  return unlocked;
8975
9653
  }
9654
+ /** Freeze the current NPCP timeline, audio, face, and animation. */
9655
+ pausePlayback() {
9656
+ this.stagingPlayer?.pause();
9657
+ }
9658
+ /** Continue the same timeline. Call from a gesture if the browser requires it. */
9659
+ async resumePlayback() {
9660
+ await this.stagingPlayer?.resume();
9661
+ }
9662
+ /** Cancel current output and pending audio/animation loads. */
9663
+ interruptPlayback() {
9664
+ if (this.animationController) {
9665
+ this.streamPlayback.cleanup();
9666
+ }
9667
+ }
8976
9668
  getExpressions() {
8977
9669
  return this.facialMixer.getChannelNames();
8978
9670
  }
@@ -9016,6 +9708,20 @@ var CharacterRuntime = class {
9016
9708
  clearLookAt() {
9017
9709
  this.setGazeTarget(null);
9018
9710
  }
9711
+ /** Briefly look at an object; release restores the prior target and gaze mode. */
9712
+ lookAtTemporarily(target) {
9713
+ const previous = this.currentGazeTarget;
9714
+ const wasActive = this.lookAtActive;
9715
+ this.lookAt(target);
9716
+ return () => {
9717
+ if (this.currentGazeTarget !== target || !this.lookAtActive) {
9718
+ return;
9719
+ }
9720
+ this.currentGazeTarget = previous;
9721
+ this.lookAtActive = wasActive;
9722
+ this.applyGaze();
9723
+ };
9724
+ }
9019
9725
  startPerformanceRecording(options) {
9020
9726
  this.performanceRecorder.start(options);
9021
9727
  this.facialMixer.capturePerformanceModel();
@@ -9064,6 +9770,9 @@ var CharacterRuntime = class {
9064
9770
  }
9065
9771
  /** Advance skeletal animations, lip-sync visemes, facial expressions, and VRM physics by `delta` seconds. */
9066
9772
  update(delta) {
9773
+ if (this.stagingPlayer?.paused) {
9774
+ return;
9775
+ }
9067
9776
  this.animationController?.update(delta);
9068
9777
  this.visemePlayer.update(delta);
9069
9778
  this.facialMixer.update(delta);
@@ -9153,6 +9862,18 @@ var ConversationSocket = class {
9153
9862
  this.socket?.disconnect();
9154
9863
  this.socket = null;
9155
9864
  }
9865
+ /** An event with an acknowledgement. Needs `ensureConnected` first. */
9866
+ async request(event, payload, timeoutMs = 1e4) {
9867
+ const socket = this.socket;
9868
+ if (!socket?.connected) {
9869
+ throw new Error("The conversation connection is not open.");
9870
+ }
9871
+ return await socket.timeout(timeoutMs).emitWithAck(event, payload);
9872
+ }
9873
+ /** An event without an acknowledgement, in order with the other events. */
9874
+ send(event, payload) {
9875
+ this.socket?.emit(event, payload);
9876
+ }
9156
9877
  async ensureConnected(conversationId) {
9157
9878
  if (typeof window === "undefined") {
9158
9879
  return;
@@ -9207,6 +9928,41 @@ var ConversationSocket = class {
9207
9928
  }
9208
9929
  this.options.handleStreamEvent(event);
9209
9930
  });
9931
+ this.socket.on(SPEECH_EVENTS.partial, (payload) => {
9932
+ const event = payload;
9933
+ if (typeof event?.sessionId === "string" && typeof event.text === "string") {
9934
+ this.options.handleSpeechEvent?.({ type: "partial", event });
9935
+ }
9936
+ });
9937
+ this.socket.on(SPEECH_EVENTS.turn, (payload) => {
9938
+ let candidate;
9939
+ try {
9940
+ candidate = decodePerformance(
9941
+ payload,
9942
+ "turn"
9943
+ );
9944
+ } catch (error) {
9945
+ this.options.handleCallbackError("speech.turn.decode", error);
9946
+ return;
9947
+ }
9948
+ if (candidate?.conversationId !== this.options.getConversationId() || typeof candidate.requestId !== "string" || !candidate.turn || typeof candidate.turn !== "object") {
9949
+ return;
9950
+ }
9951
+ this.options.handleSpeechEvent?.({
9952
+ type: "turn",
9953
+ requestId: candidate.requestId,
9954
+ turn: candidate.turn
9955
+ });
9956
+ });
9957
+ this.socket.on(SPEECH_EVENTS.turnFailed, (payload) => {
9958
+ const event = payload;
9959
+ if (typeof event?.requestId === "string" && event.conversationId === this.options.getConversationId()) {
9960
+ this.options.handleSpeechEvent?.({ type: "turnFailed", event });
9961
+ }
9962
+ });
9963
+ this.socket.on("disconnect", () => {
9964
+ this.options.handleSpeechEvent?.({ type: "disconnect" });
9965
+ });
9210
9966
  this.socket.on("conversations:turn", (payload) => {
9211
9967
  let candidate;
9212
9968
  try {
@@ -9259,58 +10015,1061 @@ var ConversationSocket = class {
9259
10015
  }
9260
10016
  };
9261
10017
 
9262
- // ../../clients/web-client/src/npcify.ts
9263
- var Character = class extends CharacterRuntime {
9264
- conversationSocket;
9265
- renderSessionId = null;
9266
- initialAssetInfo;
9267
- stagingResultCallback = null;
9268
- thinkingCallback = null;
9269
- notifiedStagingIds = /* @__PURE__ */ new Map();
9270
- liveConversation;
9271
- baked = null;
9272
- authoring;
9273
- bakedAsset;
9274
- animationRefreshId = 0;
9275
- conversationId = null;
9276
- constructor(options) {
9277
- const resolvedToken = (options.apiToken ?? options.apiKey)?.trim() ?? "";
9278
- const resolvedProfile = options.assetProfile ?? "original";
9279
- super({
9280
- ...options,
9281
- apiToken: resolvedToken,
9282
- assetProfile: resolvedProfile,
9283
- loadAnimation: (animationId) => loadVrma({
9284
- baseUrl: options.baseUrl,
9285
- characterId: options.characterId,
9286
- apiToken: resolvedToken,
9287
- animationId,
9288
- assetProfile: resolvedProfile
9289
- })
9290
- });
9291
- this.authoring = options.authoring ?? false;
9292
- this.bakedAsset = options.bakedConversation;
9293
- this.initialAssetInfo = options.initialAssetInfo ?? null;
9294
- this.conversationSocket = new ConversationSocket({
9295
- baseUrl: this.baseUrl,
9296
- apiToken: this.apiToken,
9297
- getConversationId: () => this.conversationId,
9298
- handleStreamEvent: (event) => {
9299
- if (event.type !== "text_delta") {
9300
- this.thinkingState = false;
10018
+ // ../../clients/web-client/src/voice/push-to-talk.ts
10019
+ var INTERACTIVE = 'input, textarea, select, button, a[href], summary, [contenteditable]:not([contenteditable="false"]), [role="button"], [role="textbox"], [role="checkbox"], [role="switch"], [role="menuitem"], [role="tab"], [role="slider"], [role="option"]';
10020
+ var elementOf = function(target) {
10021
+ return target && typeof target.closest === "function" ? target : null;
10022
+ };
10023
+ var bindPushToTalk = function(target, options = {}) {
10024
+ const element = options.element ?? null;
10025
+ const key = options.key === void 0 ? "Space" : options.key;
10026
+ const minHoldMs = options.minHoldMs ?? SPEECH_MIN_HOLD_MS;
10027
+ let active = null;
10028
+ let pointerId = null;
10029
+ let pressedAt = 0;
10030
+ const begin = function(source) {
10031
+ if (active || target.listeningState !== "idle") {
10032
+ return false;
10033
+ }
10034
+ if (options.enabled && !options.enabled()) {
10035
+ return false;
10036
+ }
10037
+ active = source;
10038
+ pressedAt = performance.now();
10039
+ const listening = typeof options.listening === "function" ? options.listening() : options.listening ?? {};
10040
+ target.startListening({ ...listening, endOnSilenceMs: null }).then(
10041
+ (result) => {
10042
+ if (result) {
10043
+ options.onResult?.(result);
9301
10044
  }
9302
- this.streamPlayback.handleEvent(event);
9303
- },
9304
- handleTurn: async (turn) => {
9305
- await this.handleStagingResult(turn, { source: "turn" });
9306
10045
  },
9307
- handleCallbackError: this.handleCallbackError
9308
- });
9309
- this.liveConversation = new LiveConversation({
9310
- baseUrl: this.baseUrl,
9311
- apiToken: this.apiToken,
9312
- telemetry: this.telemetry,
9313
- socket: () => this.conversationSocket,
10046
+ (error) => {
10047
+ active = null;
10048
+ options.onError?.(error);
10049
+ }
10050
+ );
10051
+ return true;
10052
+ };
10053
+ const end = function() {
10054
+ if (!active) {
10055
+ return;
10056
+ }
10057
+ active = null;
10058
+ pointerId = null;
10059
+ if (performance.now() - pressedAt < minHoldMs) {
10060
+ target.cancelListening();
10061
+ options.onShortPress?.();
10062
+ return;
10063
+ }
10064
+ const starting = target.listeningState === "starting";
10065
+ target.stopListening().then(
10066
+ (result) => {
10067
+ if (starting && result === null) {
10068
+ options.onShortPress?.();
10069
+ }
10070
+ },
10071
+ () => void 0
10072
+ );
10073
+ };
10074
+ const cancel = function() {
10075
+ if (!active) {
10076
+ return;
10077
+ }
10078
+ active = null;
10079
+ pointerId = null;
10080
+ target.cancelListening();
10081
+ };
10082
+ const isOwnElement = function(node) {
10083
+ return Boolean(element && node && element.contains(node));
10084
+ };
10085
+ const onKeyDown = function(event) {
10086
+ if (event.key === "Escape" || event.code === "Escape") {
10087
+ cancel();
10088
+ return;
10089
+ }
10090
+ const node = elementOf(event.target);
10091
+ const ownElement = isOwnElement(node);
10092
+ const pressesElement = ownElement && (event.code === "Space" || event.code === "Enter");
10093
+ if (!pressesElement && (!key || event.code !== key)) {
10094
+ return;
10095
+ }
10096
+ if (event.repeat) {
10097
+ if (active === "key") {
10098
+ event.preventDefault();
10099
+ }
10100
+ return;
10101
+ }
10102
+ if (event.ctrlKey || event.metaKey || event.altKey || event.isComposing) {
10103
+ return;
10104
+ }
10105
+ if (!ownElement && node?.closest(INTERACTIVE)) {
10106
+ return;
10107
+ }
10108
+ if (begin("key")) {
10109
+ event.preventDefault();
10110
+ }
10111
+ };
10112
+ const onKeyUp = function(event) {
10113
+ if (active !== "key") {
10114
+ return;
10115
+ }
10116
+ const ownElement = isOwnElement(elementOf(event.target));
10117
+ if (event.code === key || ownElement && (event.code === "Space" || event.code === "Enter")) {
10118
+ event.preventDefault();
10119
+ end();
10120
+ }
10121
+ };
10122
+ const onVisibility = function() {
10123
+ if (document.visibilityState === "hidden") {
10124
+ cancel();
10125
+ }
10126
+ };
10127
+ const onPointerDown = function(event) {
10128
+ if (event.pointerType === "mouse" && event.button !== 0) {
10129
+ return;
10130
+ }
10131
+ event.preventDefault();
10132
+ if (!begin("pointer")) {
10133
+ return;
10134
+ }
10135
+ pointerId = event.pointerId;
10136
+ try {
10137
+ element?.setPointerCapture(event.pointerId);
10138
+ } catch {
10139
+ }
10140
+ };
10141
+ const onPointerUp = function(event) {
10142
+ if (active === "pointer" && event.pointerId === pointerId) {
10143
+ end();
10144
+ }
10145
+ };
10146
+ const onPointerCancel = function(event) {
10147
+ if (active === "pointer" && event.pointerId === pointerId) {
10148
+ cancel();
10149
+ }
10150
+ };
10151
+ const prevent = function(event) {
10152
+ event.preventDefault();
10153
+ };
10154
+ const restoreStyle = [];
10155
+ if (element) {
10156
+ for (const [property, value] of [
10157
+ ["touch-action", "none"],
10158
+ ["user-select", "none"],
10159
+ ["-webkit-user-select", "none"],
10160
+ ["-webkit-touch-callout", "none"]
10161
+ ]) {
10162
+ const previous = element.style.getPropertyValue(property);
10163
+ element.style.setProperty(property, value);
10164
+ restoreStyle.push(() => {
10165
+ if (previous) {
10166
+ element.style.setProperty(property, previous);
10167
+ } else {
10168
+ element.style.removeProperty(property);
10169
+ }
10170
+ });
10171
+ }
10172
+ element.addEventListener("pointerdown", onPointerDown);
10173
+ window.addEventListener("pointerup", onPointerUp, true);
10174
+ window.addEventListener("pointercancel", onPointerCancel, true);
10175
+ element.addEventListener("contextmenu", prevent);
10176
+ element.addEventListener("selectstart", prevent);
10177
+ element.addEventListener("touchstart", prevent, { passive: false });
10178
+ }
10179
+ window.addEventListener("keydown", onKeyDown);
10180
+ window.addEventListener("keyup", onKeyUp);
10181
+ window.addEventListener("blur", cancel);
10182
+ document.addEventListener("visibilitychange", onVisibility);
10183
+ return () => {
10184
+ cancel();
10185
+ if (element) {
10186
+ element.removeEventListener("pointerdown", onPointerDown);
10187
+ window.removeEventListener("pointerup", onPointerUp, true);
10188
+ window.removeEventListener("pointercancel", onPointerCancel, true);
10189
+ element.removeEventListener("contextmenu", prevent);
10190
+ element.removeEventListener("selectstart", prevent);
10191
+ element.removeEventListener("touchstart", prevent);
10192
+ restoreStyle.forEach((restore) => restore());
10193
+ }
10194
+ window.removeEventListener("keydown", onKeyDown);
10195
+ window.removeEventListener("keyup", onKeyUp);
10196
+ window.removeEventListener("blur", cancel);
10197
+ document.removeEventListener("visibilitychange", onVisibility);
10198
+ };
10199
+ };
10200
+
10201
+ // ../../clients/web-client/src/voice/microphone-capture.ts
10202
+ var FRAME_SAMPLES = SPEECH_AUDIO_FORMAT.sampleRate * SPEECH_FRAME_MS / 1e3;
10203
+ var MicrophoneError = class extends Error {
10204
+ constructor(code, message) {
10205
+ super(message);
10206
+ this.code = code;
10207
+ this.name = "NpcifyMicrophoneError";
10208
+ }
10209
+ };
10210
+ var Pcm16Framer = class {
10211
+ constructor(inputRate, emit) {
10212
+ this.emit = emit;
10213
+ this.ratio = inputRate / SPEECH_AUDIO_FORMAT.sampleRate;
10214
+ }
10215
+ ratio;
10216
+ frame = new Int16Array(FRAME_SAMPLES);
10217
+ length = 0;
10218
+ sum = 0;
10219
+ count = 0;
10220
+ phase = 0;
10221
+ push(samples) {
10222
+ for (let index = 0; index < samples.length; index += 1) {
10223
+ this.sum += samples[index] ?? 0;
10224
+ this.count += 1;
10225
+ this.phase += 1;
10226
+ while (this.phase >= this.ratio) {
10227
+ this.phase -= this.ratio;
10228
+ this.write(this.sum / Math.max(1, this.count));
10229
+ if (this.ratio >= 1) {
10230
+ this.sum = 0;
10231
+ this.count = 0;
10232
+ }
10233
+ }
10234
+ if (this.ratio < 1) {
10235
+ this.sum = 0;
10236
+ this.count = 0;
10237
+ }
10238
+ }
10239
+ }
10240
+ flush() {
10241
+ if (this.count > 0 && this.ratio >= 1) {
10242
+ this.write(this.sum / this.count);
10243
+ this.sum = 0;
10244
+ this.count = 0;
10245
+ }
10246
+ if (this.length > 0) {
10247
+ this.emit(this.frame.slice(0, this.length));
10248
+ this.length = 0;
10249
+ }
10250
+ }
10251
+ write(value) {
10252
+ const clamped = Math.max(-1, Math.min(1, value));
10253
+ this.frame[this.length] = Math.round(
10254
+ clamped < 0 ? clamped * 32768 : clamped * 32767
10255
+ );
10256
+ this.length += 1;
10257
+ if (this.length === FRAME_SAMPLES) {
10258
+ this.emit(this.frame);
10259
+ this.frame = new Int16Array(FRAME_SAMPLES);
10260
+ this.length = 0;
10261
+ }
10262
+ }
10263
+ };
10264
+ var levelOf = function(pcm) {
10265
+ if (pcm.length === 0) {
10266
+ return 0;
10267
+ }
10268
+ let sum = 0;
10269
+ for (const sample of pcm) {
10270
+ sum += (sample / 32768) ** 2;
10271
+ }
10272
+ return Math.sqrt(sum / pcm.length);
10273
+ };
10274
+ var WORKLET_SOURCE = `
10275
+ const FRAME = ${FRAME_SAMPLES}
10276
+ class NpcifyCaptureProcessor extends AudioWorkletProcessor {
10277
+ constructor() {
10278
+ super()
10279
+ this.ratio = sampleRate / ${SPEECH_AUDIO_FORMAT.sampleRate}
10280
+ this.frame = new Int16Array(FRAME)
10281
+ this.length = 0
10282
+ this.sum = 0
10283
+ this.count = 0
10284
+ this.phase = 0
10285
+ this.port.onmessage = (event) => {
10286
+ if (event.data === 'flush') {
10287
+ if (this.count > 0 && this.ratio >= 1) {
10288
+ this.write(this.sum / this.count)
10289
+ this.sum = 0
10290
+ this.count = 0
10291
+ }
10292
+ if (this.length > 0) {
10293
+ this.post(this.frame.slice(0, this.length))
10294
+ this.length = 0
10295
+ }
10296
+ this.port.postMessage({ type: 'flushed' })
10297
+ }
10298
+ }
10299
+ }
10300
+ write(value) {
10301
+ const clamped = Math.max(-1, Math.min(1, value))
10302
+ this.frame[this.length++] = Math.round(clamped < 0 ? clamped * 32768 : clamped * 32767)
10303
+ if (this.length === FRAME) {
10304
+ this.post(this.frame)
10305
+ this.frame = new Int16Array(FRAME)
10306
+ this.length = 0
10307
+ }
10308
+ }
10309
+ post(pcm) {
10310
+ this.port.postMessage({ type: 'frame', pcm }, [pcm.buffer])
10311
+ }
10312
+ process(inputs) {
10313
+ const channels = inputs[0]
10314
+ if (!channels || channels.length === 0) return true
10315
+ const size = channels[0].length
10316
+ for (let index = 0; index < size; index += 1) {
10317
+ let value = 0
10318
+ for (const channel of channels) value += channel[index]
10319
+ this.sum += value / channels.length
10320
+ this.count += 1
10321
+ this.phase += 1
10322
+ while (this.phase >= this.ratio) {
10323
+ this.phase -= this.ratio
10324
+ this.write(this.sum / Math.max(1, this.count))
10325
+ if (this.ratio >= 1) {
10326
+ this.sum = 0
10327
+ this.count = 0
10328
+ }
10329
+ }
10330
+ if (this.ratio < 1) {
10331
+ this.sum = 0
10332
+ this.count = 0
10333
+ }
10334
+ }
10335
+ return true
10336
+ }
10337
+ }
10338
+ registerProcessor('npcify-capture', NpcifyCaptureProcessor)
10339
+ `;
10340
+ var MicrophoneCapture = class {
10341
+ context = null;
10342
+ workletReady = null;
10343
+ graph = null;
10344
+ generation = 0;
10345
+ async start(onFrame) {
10346
+ this.cancel();
10347
+ const generation = ++this.generation;
10348
+ const AudioContextClass = globalThis.AudioContext ?? globalThis.webkitAudioContext;
10349
+ if (!navigator.mediaDevices?.getUserMedia || !AudioContextClass) {
10350
+ throw new MicrophoneError(
10351
+ "MICROPHONE_UNAVAILABLE",
10352
+ "This browser cannot record audio."
10353
+ );
10354
+ }
10355
+ this.context ??= new AudioContextClass({ latencyHint: "interactive" });
10356
+ const context2 = this.context;
10357
+ const resumed = context2.resume().catch(() => void 0);
10358
+ let stream;
10359
+ try {
10360
+ stream = await navigator.mediaDevices.getUserMedia({
10361
+ audio: {
10362
+ channelCount: 1,
10363
+ echoCancellation: true,
10364
+ noiseSuppression: true,
10365
+ autoGainControl: true
10366
+ }
10367
+ });
10368
+ } catch (error) {
10369
+ const name = error?.name;
10370
+ throw name === "NotAllowedError" || name === "SecurityError" ? new MicrophoneError(
10371
+ "MICROPHONE_PERMISSION_DENIED",
10372
+ "Microphone access was denied."
10373
+ ) : new MicrophoneError(
10374
+ "MICROPHONE_UNAVAILABLE",
10375
+ "No microphone is available."
10376
+ );
10377
+ }
10378
+ await resumed;
10379
+ if (generation !== this.generation) {
10380
+ stream.getTracks().forEach((track) => track.stop());
10381
+ return;
10382
+ }
10383
+ const deliver = (pcm) => onFrame({ pcm, level: levelOf(pcm) });
10384
+ const source = context2.createMediaStreamSource(stream);
10385
+ const sink = context2.createGain();
10386
+ sink.gain.value = 0;
10387
+ sink.connect(context2.destination);
10388
+ const graph = await this.workletGraph(context2, stream, source, sink, deliver) ?? this.scriptGraph(context2, stream, source, sink, deliver);
10389
+ if (generation !== this.generation) {
10390
+ this.close(graph, false);
10391
+ return;
10392
+ }
10393
+ this.graph = graph;
10394
+ }
10395
+ async stop() {
10396
+ const graph = this.graph;
10397
+ this.graph = null;
10398
+ this.generation += 1;
10399
+ if (!graph) {
10400
+ return;
10401
+ }
10402
+ try {
10403
+ await graph.flush();
10404
+ } finally {
10405
+ this.close(graph);
10406
+ }
10407
+ }
10408
+ cancel() {
10409
+ const graph = this.graph;
10410
+ this.graph = null;
10411
+ this.generation += 1;
10412
+ if (graph) {
10413
+ this.close(graph);
10414
+ }
10415
+ }
10416
+ close(graph, suspend = true) {
10417
+ graph.stream.getTracks().forEach((track) => track.stop());
10418
+ graph.source.disconnect();
10419
+ graph.node.disconnect();
10420
+ graph.sink.disconnect();
10421
+ if (graph.node instanceof AudioWorkletNode) {
10422
+ graph.node.port.onmessage = null;
10423
+ } else {
10424
+ graph.node.onaudioprocess = null;
10425
+ }
10426
+ if (suspend) {
10427
+ void this.context?.suspend().catch(() => void 0);
10428
+ }
10429
+ }
10430
+ async workletGraph(context2, stream, source, sink, deliver) {
10431
+ if (!context2.audioWorklet || typeof AudioWorkletNode === "undefined") {
10432
+ return null;
10433
+ }
10434
+ this.workletReady ??= (async () => {
10435
+ const url = URL.createObjectURL(
10436
+ new Blob([WORKLET_SOURCE], { type: "application/javascript" })
10437
+ );
10438
+ try {
10439
+ await context2.audioWorklet.addModule(url);
10440
+ return true;
10441
+ } catch {
10442
+ return false;
10443
+ } finally {
10444
+ URL.revokeObjectURL(url);
10445
+ }
10446
+ })();
10447
+ if (!await this.workletReady) {
10448
+ return null;
10449
+ }
10450
+ const node = new AudioWorkletNode(context2, "npcify-capture", {
10451
+ numberOfInputs: 1,
10452
+ numberOfOutputs: 1,
10453
+ channelCount: 1,
10454
+ channelCountMode: "explicit"
10455
+ });
10456
+ let flushed = null;
10457
+ node.port.onmessage = (event) => {
10458
+ if (event.data.type === "frame" && event.data.pcm) {
10459
+ deliver(event.data.pcm);
10460
+ } else if (event.data.type === "flushed") {
10461
+ flushed?.();
10462
+ }
10463
+ };
10464
+ source.connect(node);
10465
+ node.connect(sink);
10466
+ return {
10467
+ stream,
10468
+ source,
10469
+ node,
10470
+ sink,
10471
+ flush: () => new Promise((resolve) => {
10472
+ const timer = setTimeout(resolve, 250);
10473
+ flushed = () => {
10474
+ clearTimeout(timer);
10475
+ resolve();
10476
+ };
10477
+ node.port.postMessage("flush");
10478
+ })
10479
+ };
10480
+ }
10481
+ /** The fallback without AudioWorklet: a ScriptProcessorNode on the main thread. */
10482
+ scriptGraph(context2, stream, source, sink, deliver) {
10483
+ const framer = new Pcm16Framer(context2.sampleRate, deliver);
10484
+ const node = context2.createScriptProcessor(2048, 1, 1);
10485
+ node.onaudioprocess = (event) => {
10486
+ framer.push(event.inputBuffer.getChannelData(0));
10487
+ };
10488
+ source.connect(node);
10489
+ node.connect(sink);
10490
+ return {
10491
+ stream,
10492
+ source,
10493
+ node,
10494
+ sink,
10495
+ flush: () => {
10496
+ framer.flush();
10497
+ return Promise.resolve();
10498
+ }
10499
+ };
10500
+ }
10501
+ };
10502
+
10503
+ // ../../clients/web-client/src/voice/silence-detector.ts
10504
+ var MIN_SPEECH_LEVEL = 0.015;
10505
+ var SPEECH_TO_NOISE = 3;
10506
+ var SPEECH_START_MS = 200;
10507
+ var SilenceDetector = class {
10508
+ constructor(silenceMs) {
10509
+ this.silenceMs = silenceMs;
10510
+ }
10511
+ floor = null;
10512
+ loudMs = 0;
10513
+ quietMs = 0;
10514
+ speaking = false;
10515
+ /** True when the speech has ended. */
10516
+ push(level, frameMs = SPEECH_FRAME_MS) {
10517
+ const threshold = Math.max(
10518
+ MIN_SPEECH_LEVEL,
10519
+ (this.floor ?? MIN_SPEECH_LEVEL / SPEECH_TO_NOISE) * SPEECH_TO_NOISE
10520
+ );
10521
+ if (level >= threshold) {
10522
+ this.loudMs += frameMs;
10523
+ this.quietMs = 0;
10524
+ if (this.loudMs >= SPEECH_START_MS) {
10525
+ this.speaking = true;
10526
+ }
10527
+ return false;
10528
+ }
10529
+ this.floor = this.floor === null ? level : this.floor * 0.9 + level * 0.1;
10530
+ this.loudMs = 0;
10531
+ if (!this.speaking) {
10532
+ return false;
10533
+ }
10534
+ this.quietMs += frameMs;
10535
+ return this.quietMs >= this.silenceMs;
10536
+ }
10537
+ get heardSpeech() {
10538
+ return this.speaking;
10539
+ }
10540
+ };
10541
+
10542
+ // ../../clients/web-client/src/voice/voice-input.ts
10543
+ var SpeechInputError = class extends Error {
10544
+ constructor(code, message) {
10545
+ super(message ?? code);
10546
+ this.code = code;
10547
+ this.name = "NpcifySpeechInputError";
10548
+ }
10549
+ };
10550
+ var deferred = function() {
10551
+ let resolve;
10552
+ let reject;
10553
+ const promise = new Promise((done, fail) => {
10554
+ resolve = done;
10555
+ reject = fail;
10556
+ });
10557
+ promise.catch(() => void 0);
10558
+ return { promise, resolve, reject };
10559
+ };
10560
+ var CODE = /^[A-Z][A-Z0-9_]{2,63}$/u;
10561
+ var speechError = function(error, fallback) {
10562
+ if (error instanceof SpeechInputError) {
10563
+ return error;
10564
+ }
10565
+ if (error instanceof Error && "code" in error && typeof error.code === "string") {
10566
+ return new SpeechInputError(
10567
+ error.code,
10568
+ error.message
10569
+ );
10570
+ }
10571
+ return new SpeechInputError(
10572
+ fallback,
10573
+ error instanceof Error ? error.message : void 0
10574
+ );
10575
+ };
10576
+ var VoiceInput = class {
10577
+ constructor(deps) {
10578
+ this.deps = deps;
10579
+ }
10580
+ state = "idle";
10581
+ session = null;
10582
+ capture = null;
10583
+ /**
10584
+ * The session that the capture records for. A start that was cancelled
10585
+ * never touches the capture when a later session owns it.
10586
+ */
10587
+ captureOwner = null;
10588
+ turns = /* @__PURE__ */ new Map();
10589
+ get listeningState() {
10590
+ return this.state;
10591
+ }
10592
+ /** A turn that the server started has not delivered its final payload. */
10593
+ get turnPending() {
10594
+ return this.turns.size > 0;
10595
+ }
10596
+ /**
10597
+ * Starts listening. Resolves with the result when the listening ends (by
10598
+ * `stop`, the silence rule or the time limit), or with null when it is
10599
+ * cancelled.
10600
+ */
10601
+ async start(options = {}) {
10602
+ if (this.session) {
10603
+ throw new SpeechInputError(
10604
+ "SPEECH_ALREADY_LISTENING",
10605
+ "Listening has already started."
10606
+ );
10607
+ }
10608
+ this.deps.assertVoiceAllowed();
10609
+ if (this.turnPending || this.deps.turnInFlight()) {
10610
+ throw new SpeechInputError(
10611
+ "TURN_IN_PROGRESS",
10612
+ "A reply is already being generated. Please wait."
10613
+ );
10614
+ }
10615
+ const conversationId = this.deps.conversationId();
10616
+ if (!conversationId) {
10617
+ throw new SpeechInputError(
10618
+ "CONVERSATION_REQUIRED",
10619
+ "Create a conversation before listening."
10620
+ );
10621
+ }
10622
+ const maxDurationMs = Math.min(
10623
+ SPEECH_MAX_DURATION_MS,
10624
+ Math.max(1e3, options.maxDurationMs ?? SPEECH_MAX_DURATION_MS)
10625
+ );
10626
+ const session = {
10627
+ requestId: globalThis.crypto.randomUUID(),
10628
+ options,
10629
+ sessionId: null,
10630
+ sequence: 0,
10631
+ bytes: 0,
10632
+ pending: [],
10633
+ timer: null,
10634
+ silence: options.endOnSilenceMs && options.endOnSilenceMs > 0 ? new SilenceDetector(options.endOnSilenceMs) : null,
10635
+ stopRequested: false,
10636
+ stopping: null,
10637
+ captureClosing: null,
10638
+ captureClosed: false,
10639
+ disconnected: deferred(),
10640
+ done: deferred()
10641
+ };
10642
+ this.session = session;
10643
+ this.setState("starting");
10644
+ const socket = this.deps.socket();
10645
+ this.capture ??= (this.deps.createCapture ?? (() => new MicrophoneCapture()))();
10646
+ const capture = this.capture;
10647
+ this.captureOwner = session;
10648
+ const [acknowledged, captured] = await Promise.allSettled([
10649
+ socket.ensureConnected(conversationId).then(
10650
+ () => socket.request(SPEECH_EVENTS.start, {
10651
+ conversationId,
10652
+ requestId: session.requestId,
10653
+ parentStagingId: options.parentStagingId,
10654
+ language: options.language ?? null,
10655
+ send: options.send !== false,
10656
+ silent: options.silent === true,
10657
+ renderSessionId: this.deps.renderSessionId(),
10658
+ format: SPEECH_AUDIO_FORMAT
10659
+ })
10660
+ ),
10661
+ capture.start((frame) => this.frame(session, frame))
10662
+ ]);
10663
+ const ack = acknowledged.status === "fulfilled" ? acknowledged.value : null;
10664
+ const serverSession = ack?.ok ? ack.data.sessionId : null;
10665
+ if (session !== this.session) {
10666
+ this.dropCapture(session);
10667
+ if (serverSession) {
10668
+ socket.send(SPEECH_EVENTS.cancel, { sessionId: serverSession });
10669
+ }
10670
+ return session.done.promise;
10671
+ }
10672
+ let failure = null;
10673
+ if (captured.status === "rejected") {
10674
+ failure = speechError(captured.reason, "MICROPHONE_UNAVAILABLE");
10675
+ } else if (acknowledged.status === "rejected") {
10676
+ failure = speechError(acknowledged.reason, "SPEECH_UNAVAILABLE");
10677
+ } else if (!ack?.ok) {
10678
+ failure = new SpeechInputError(
10679
+ ack && !ack.ok ? ack.error : "SPEECH_UNAVAILABLE"
10680
+ );
10681
+ }
10682
+ if (failure) {
10683
+ this.dropCapture(session);
10684
+ if (serverSession) {
10685
+ socket.send(SPEECH_EVENTS.cancel, { sessionId: serverSession });
10686
+ }
10687
+ this.end(session);
10688
+ session.done.reject(failure);
10689
+ return session.done.promise;
10690
+ }
10691
+ session.sessionId = serverSession;
10692
+ for (const audio of session.pending.splice(0)) {
10693
+ this.sendFrame(session, audio);
10694
+ }
10695
+ if (session.stopRequested) {
10696
+ void this.stop().catch(() => void 0);
10697
+ return session.done.promise;
10698
+ }
10699
+ this.setState("listening");
10700
+ session.timer = setTimeout(() => {
10701
+ void this.stop().catch(() => void 0);
10702
+ }, maxDurationMs);
10703
+ return session.done.promise;
10704
+ }
10705
+ /**
10706
+ * Ends the recording and resolves with the result as soon as the text is
10707
+ * recognised. A stop before the microphone gave any audio (a permission
10708
+ * prompt, a slow device) cancels and resolves null.
10709
+ */
10710
+ async stop() {
10711
+ const session = this.session;
10712
+ if (!session) {
10713
+ return null;
10714
+ }
10715
+ if (session.stopping) {
10716
+ return session.stopping;
10717
+ }
10718
+ if (!session.sessionId) {
10719
+ if (session.bytes === 0) {
10720
+ this.cancel();
10721
+ return null;
10722
+ }
10723
+ session.stopRequested = true;
10724
+ void this.closeCapture(session);
10725
+ return session.done.promise;
10726
+ }
10727
+ const stopping = deferred();
10728
+ session.stopping = stopping.promise;
10729
+ this.finish(session).then(stopping.resolve, stopping.reject);
10730
+ return session.stopping;
10731
+ }
10732
+ /** Discards the recording. Nothing is recognised or charged. */
10733
+ cancel() {
10734
+ const session = this.session;
10735
+ if (!session || session.stopping) {
10736
+ return;
10737
+ }
10738
+ this.dropCapture(session);
10739
+ if (session.sessionId) {
10740
+ this.deps.socket().send(SPEECH_EVENTS.cancel, { sessionId: session.sessionId });
10741
+ }
10742
+ this.end(session);
10743
+ session.done.resolve(null);
10744
+ }
10745
+ /** The conversation ended: no listening, and no turn waits any more. */
10746
+ reset() {
10747
+ this.cancel();
10748
+ for (const [requestId, turn] of this.turns) {
10749
+ clearTimeout(turn.timer);
10750
+ turn.done.reject(
10751
+ new SpeechInputError("CONVERSATION_ENDED", "The conversation ended.")
10752
+ );
10753
+ this.turns.delete(requestId);
10754
+ }
10755
+ }
10756
+ handleSocketEvent(event) {
10757
+ if (event.type === "partial") {
10758
+ if (this.session?.sessionId === event.event.sessionId) {
10759
+ this.deps.onTranscript({ text: event.event.text, final: false });
10760
+ }
10761
+ return;
10762
+ }
10763
+ if (event.type === "turn") {
10764
+ const turn = this.turns.get(event.requestId);
10765
+ if (!turn) {
10766
+ return;
10767
+ }
10768
+ this.turns.delete(event.requestId);
10769
+ clearTimeout(turn.timer);
10770
+ void this.deps.deliverTurn(event.turn, event.requestId, turn.options.silent === true).then(
10771
+ () => turn.done.resolve(event.turn),
10772
+ (error) => turn.done.reject(error)
10773
+ );
10774
+ return;
10775
+ }
10776
+ if (event.type === "turnFailed") {
10777
+ const turn = this.turns.get(event.event.requestId);
10778
+ if (!turn) {
10779
+ return;
10780
+ }
10781
+ this.turns.delete(event.event.requestId);
10782
+ clearTimeout(turn.timer);
10783
+ const reason = event.event.error;
10784
+ turn.done.reject(
10785
+ new SpeechInputError(CODE.test(reason) ? reason : "TURN_FAILED", reason)
10786
+ );
10787
+ return;
10788
+ }
10789
+ const session = this.session;
10790
+ if (session) {
10791
+ const failure = new SpeechInputError(
10792
+ "SPEECH_DISCONNECTED",
10793
+ session.stopping ? "The connection closed while the speech was recognised." : "The connection closed while listening."
10794
+ );
10795
+ session.disconnected.reject(failure);
10796
+ if (!session.stopping) {
10797
+ this.dropCapture(session);
10798
+ this.end(session);
10799
+ session.done.reject(failure);
10800
+ }
10801
+ }
10802
+ for (const requestId of [...this.turns.keys()]) {
10803
+ this.recover(requestId);
10804
+ }
10805
+ }
10806
+ frame(session, frame) {
10807
+ if (session !== this.session || session.captureClosed) {
10808
+ return;
10809
+ }
10810
+ const audio = new Uint8Array(
10811
+ frame.pcm.buffer,
10812
+ frame.pcm.byteOffset,
10813
+ frame.pcm.byteLength
10814
+ ).slice();
10815
+ if (session.sessionId) {
10816
+ this.sendFrame(session, audio);
10817
+ } else if (session.bytes + audio.length <= SPEECH_MAX_AUDIO_BYTES) {
10818
+ session.bytes += audio.length;
10819
+ session.pending.push(audio);
10820
+ }
10821
+ if (!session.stopping && session.silence?.push(frame.level)) {
10822
+ void this.stop().catch(() => void 0);
10823
+ }
10824
+ }
10825
+ sendFrame(session, audio) {
10826
+ if (!session.sessionId) {
10827
+ return;
10828
+ }
10829
+ this.deps.socket().send(SPEECH_EVENTS.audio, {
10830
+ sessionId: session.sessionId,
10831
+ sequence: session.sequence,
10832
+ audio
10833
+ });
10834
+ session.sequence += 1;
10835
+ }
10836
+ async finish(session) {
10837
+ if (session.timer) {
10838
+ clearTimeout(session.timer);
10839
+ }
10840
+ this.setState("recognizing");
10841
+ try {
10842
+ await this.closeCapture(session);
10843
+ const ack = await Promise.race([
10844
+ this.deps.socket().request(
10845
+ SPEECH_EVENTS.stop,
10846
+ { sessionId: session.sessionId },
10847
+ 3e4
10848
+ ),
10849
+ session.disconnected.promise
10850
+ ]);
10851
+ if (!ack.ok) {
10852
+ throw new SpeechInputError(ack.error);
10853
+ }
10854
+ const result = this.result(session, ack.data);
10855
+ this.end(session);
10856
+ this.deps.onTranscript({ text: result.text, final: true });
10857
+ session.done.resolve(result);
10858
+ return result;
10859
+ } catch (error) {
10860
+ const failure = speechError(error, "SPEECH_RECOGNITION_FAILED");
10861
+ this.end(session);
10862
+ session.done.reject(failure);
10863
+ throw failure;
10864
+ }
10865
+ }
10866
+ result(session, transcript) {
10867
+ const text = transcript.text;
10868
+ const turn = transcript.sent ? this.track(session, text) : null;
10869
+ let sending = null;
10870
+ return {
10871
+ text,
10872
+ language: transcript.language,
10873
+ durationMs: transcript.durationMs,
10874
+ sent: transcript.sent,
10875
+ turn,
10876
+ send: () => {
10877
+ if (turn) {
10878
+ return turn;
10879
+ }
10880
+ if (!text) {
10881
+ return Promise.reject(
10882
+ new SpeechInputError("SPEECH_EMPTY", "No speech was recognised.")
10883
+ );
10884
+ }
10885
+ this.deps.assertVoiceAllowed();
10886
+ sending ??= this.deps.sendText(text, {
10887
+ requestId: session.requestId,
10888
+ parentStagingId: session.options.parentStagingId,
10889
+ silent: session.options.silent === true
10890
+ });
10891
+ return sending;
10892
+ }
10893
+ };
10894
+ }
10895
+ track(session, text) {
10896
+ const done = deferred();
10897
+ const timer = setTimeout(
10898
+ () => this.recover(session.requestId),
10899
+ this.deps.turnTimeoutMs ?? 12e4
10900
+ );
10901
+ this.turns.set(session.requestId, {
10902
+ text,
10903
+ options: session.options,
10904
+ timer,
10905
+ done
10906
+ });
10907
+ this.deps.beginTurn();
10908
+ return done.promise;
10909
+ }
10910
+ /** The idempotent HTTP request returns the turn that the server started. */
10911
+ recover(requestId) {
10912
+ const turn = this.turns.get(requestId);
10913
+ if (!turn) {
10914
+ return;
10915
+ }
10916
+ this.turns.delete(requestId);
10917
+ clearTimeout(turn.timer);
10918
+ this.deps.sendText(turn.text, {
10919
+ requestId,
10920
+ parentStagingId: turn.options.parentStagingId,
10921
+ silent: turn.options.silent === true
10922
+ }).then(turn.done.resolve, turn.done.reject);
10923
+ }
10924
+ /** Stops the capture of `session`; its last partial frame arrives first. */
10925
+ closeCapture(session) {
10926
+ session.captureClosing ??= (async () => {
10927
+ if (this.captureOwner === session) {
10928
+ this.captureOwner = null;
10929
+ await this.capture?.stop();
10930
+ }
10931
+ session.captureClosed = true;
10932
+ })();
10933
+ return session.captureClosing;
10934
+ }
10935
+ /** Ends the capture of `session` without its last frame, unless a later session owns it. */
10936
+ dropCapture(session) {
10937
+ if (this.captureOwner === session) {
10938
+ this.captureOwner = null;
10939
+ this.capture?.cancel();
10940
+ }
10941
+ }
10942
+ end(session) {
10943
+ if (session.timer) {
10944
+ clearTimeout(session.timer);
10945
+ session.timer = null;
10946
+ }
10947
+ if (this.session === session) {
10948
+ this.session = null;
10949
+ this.setState("idle");
10950
+ }
10951
+ }
10952
+ setState(state) {
10953
+ if (this.state !== state) {
10954
+ this.state = state;
10955
+ this.deps.onState(state);
10956
+ }
10957
+ }
10958
+ };
10959
+
10960
+ // ../../clients/web-client/src/npcify.ts
10961
+ var Character = class extends CharacterRuntime {
10962
+ conversationSocket;
10963
+ renderSessionId = null;
10964
+ initialAssetInfo;
10965
+ stagingResultCallback = null;
10966
+ thinkingCallback = null;
10967
+ notifiedStagingIds = /* @__PURE__ */ new Map();
10968
+ liveConversation;
10969
+ baked = null;
10970
+ authoring;
10971
+ normalizeIdleOrientation;
10972
+ bakedAsset;
10973
+ animationRefreshId = 0;
10974
+ voice;
10975
+ configuredInputModes;
10976
+ resolvedInputModes = null;
10977
+ /** The character's voice input switch, after the character loads. */
10978
+ speechInput = null;
10979
+ latestOptions = [];
10980
+ listeningStateCallback = null;
10981
+ transcriptCallback = null;
10982
+ conversationId = null;
10983
+ constructor(options) {
10984
+ const resolvedToken = (options.apiToken ?? options.apiKey)?.trim() ?? "";
10985
+ const resolvedProfile = options.assetProfile ?? "original";
10986
+ super({
10987
+ ...options,
10988
+ apiToken: resolvedToken,
10989
+ assetProfile: resolvedProfile,
10990
+ loadAnimation: (animationId) => loadVrma({
10991
+ baseUrl: options.baseUrl,
10992
+ characterId: options.characterId,
10993
+ apiToken: resolvedToken,
10994
+ animationId,
10995
+ assetProfile: resolvedProfile
10996
+ })
10997
+ });
10998
+ this.authoring = options.authoring ?? false;
10999
+ this.normalizeIdleOrientation = options.normalizeIdleOrientation ?? false;
11000
+ this.bakedAsset = options.bakedConversation;
11001
+ this.initialAssetInfo = options.initialAssetInfo ?? null;
11002
+ this.configuredInputModes = parseInputModes(options.inputModes);
11003
+ if (this.configuredInputModes && this.bakedAsset && !this.authoring) {
11004
+ resolveInputModes(this.configuredInputModes, {
11005
+ allowCustomReply: false,
11006
+ baked: true,
11007
+ speechInput: false
11008
+ });
11009
+ }
11010
+ this.conversationSocket = new ConversationSocket({
11011
+ baseUrl: this.baseUrl,
11012
+ apiToken: this.apiToken,
11013
+ getConversationId: () => this.conversationId,
11014
+ handleStreamEvent: (event) => {
11015
+ if (event.type !== "text_delta") {
11016
+ this.thinkingState = false;
11017
+ }
11018
+ this.streamPlayback.handleEvent(event);
11019
+ },
11020
+ handleTurn: async (turn) => {
11021
+ await this.handleStagingResult(turn, { source: "turn" });
11022
+ },
11023
+ handleCallbackError: this.handleCallbackError,
11024
+ handleSpeechEvent: (event) => this.voice.handleSocketEvent(event)
11025
+ });
11026
+ this.voice = new VoiceInput({
11027
+ socket: () => this.conversationSocket,
11028
+ conversationId: () => this.conversationId,
11029
+ renderSessionId: () => this.renderSessionId,
11030
+ assertVoiceAllowed: () => this.assertInputMode("voice"),
11031
+ turnInFlight: () => this.liveConversation.inFlight,
11032
+ beginTurn: () => {
11033
+ if (this.conversationId) {
11034
+ this.telemetry.begin(this.conversationId);
11035
+ }
11036
+ this.invokeThinkingCallback({ taskId: null, stagingId: null });
11037
+ },
11038
+ deliverTurn: (payload, requestId, silent) => this.handleStagingResult(payload, {
11039
+ source: "api",
11040
+ forcePlayback: payload.taskId !== requestId,
11041
+ silent
11042
+ }),
11043
+ sendText: (text, turnOptions) => {
11044
+ if (!this.conversationId) {
11045
+ return Promise.reject(new Error("Conversation ID is required."));
11046
+ }
11047
+ return this.liveConversation.send(this.conversationId, text, turnOptions);
11048
+ },
11049
+ onState: (state) => {
11050
+ if (this.listeningStateCallback) {
11051
+ invokeSafeCallback(
11052
+ "onListeningState",
11053
+ this.handleCallbackError,
11054
+ () => this.listeningStateCallback?.(state)
11055
+ );
11056
+ }
11057
+ },
11058
+ onTranscript: (event) => {
11059
+ if (this.transcriptCallback) {
11060
+ invokeSafeCallback(
11061
+ "onTranscript",
11062
+ this.handleCallbackError,
11063
+ () => this.transcriptCallback?.(event)
11064
+ );
11065
+ }
11066
+ }
11067
+ });
11068
+ this.liveConversation = new LiveConversation({
11069
+ baseUrl: this.baseUrl,
11070
+ apiToken: this.apiToken,
11071
+ telemetry: this.telemetry,
11072
+ socket: () => this.conversationSocket,
9314
11073
  renderSession: () => this.renderSessionId,
9315
11074
  thinking: () => this.invokeThinkingCallback({ taskId: null, stagingId: null }),
9316
11075
  handleResult: (payload, forcePlayback, silent) => this.handleStagingResult(payload, {
@@ -9331,7 +11090,14 @@ var Character = class extends CharacterRuntime {
9331
11090
  this.resetAssets();
9332
11091
  const progressReporter = createAssetProgressReporter(callback);
9333
11092
  const info = await this.getAssetInfo();
9334
- this.baked = !this.authoring && (this.bakedAsset || info.bakedConversationEnabled) ? await loadBakedConversation({
11093
+ const baked = !this.authoring && Boolean(this.bakedAsset || info.bakedConversationEnabled);
11094
+ this.speechInput = info.speechInputEnabled === true;
11095
+ this.resolvedInputModes = resolveInputModes(this.configuredInputModes, {
11096
+ allowCustomReply: info.allowCustomReply !== false,
11097
+ baked,
11098
+ speechInput: this.speechInput
11099
+ });
11100
+ this.baked = baked ? await loadBakedConversation({
9335
11101
  baseUrl: this.baseUrl,
9336
11102
  characterId: this.characterId,
9337
11103
  apiToken: this.apiToken,
@@ -9356,7 +11122,8 @@ var Character = class extends CharacterRuntime {
9356
11122
  }
9357
11123
  const animationController = new AnimationController(
9358
11124
  this.vrm,
9359
- info.idleAnimationId ?? null
11125
+ info.idleAnimationId ?? null,
11126
+ { normalizeIdleOrientation: this.normalizeIdleOrientation }
9360
11127
  );
9361
11128
  animationController.prepareAnimations = (ids) => this.lazyAnimations.ensureMany(ids);
9362
11129
  this.animationController = animationController;
@@ -9399,6 +11166,12 @@ var Character = class extends CharacterRuntime {
9399
11166
  if (info.bakedConversationEnabled || this.bakedAsset) {
9400
11167
  throw new Error("Baked conversation requires local rendering.");
9401
11168
  }
11169
+ this.speechInput = info.speechInputEnabled === true;
11170
+ this.resolvedInputModes = resolveInputModes(this.configuredInputModes, {
11171
+ allowCustomReply: info.allowCustomReply !== false,
11172
+ baked: false,
11173
+ speechInput: this.speechInput
11174
+ });
9402
11175
  this.animationController = new RemoteAnimationController();
9403
11176
  }
9404
11177
  async createConversation(options) {
@@ -9441,6 +11214,7 @@ var Character = class extends CharacterRuntime {
9441
11214
  return targetConversationId;
9442
11215
  }
9443
11216
  async disconnect() {
11217
+ this.voice.reset();
9444
11218
  this.thinkingState = false;
9445
11219
  if (this.animationController) {
9446
11220
  this.streamPlayback.cleanup();
@@ -9476,6 +11250,10 @@ var Character = class extends CharacterRuntime {
9476
11250
  maybeMessageOrOptions,
9477
11251
  maybeOptions
9478
11252
  );
11253
+ this.assertReplyMode(message);
11254
+ if (this.voice.turnPending) {
11255
+ throw new Error("A reply is already being generated. Please wait.");
11256
+ }
9479
11257
  if (this.baked) {
9480
11258
  const result = this.baked.reply(message, options?.parentStagingId);
9481
11259
  if (result.ended) {
@@ -9515,6 +11293,59 @@ var Character = class extends CharacterRuntime {
9515
11293
  get isBakedConversation() {
9516
11294
  return this.baked !== null;
9517
11295
  }
11296
+ /**
11297
+ * The input modes in use: the configured modes, or every mode that the
11298
+ * character permits. Before `loadAssets`, the configuration or all modes.
11299
+ */
11300
+ get inputModes() {
11301
+ return this.resolvedInputModes ?? this.configuredInputModes ?? permittedInputModes({
11302
+ allowCustomReply: true,
11303
+ baked: false,
11304
+ speechInput: true
11305
+ });
11306
+ }
11307
+ get listeningState() {
11308
+ return this.voice.listeningState;
11309
+ }
11310
+ /**
11311
+ * Opens the microphone and streams the speech to the server. Resolves with
11312
+ * the result when listening ends (`stopListening`, `endOnSilenceMs` or
11313
+ * `maxDurationMs`), or with null when it is cancelled. The first call asks
11314
+ * for the microphone permission.
11315
+ */
11316
+ startListening(options) {
11317
+ return this.voice.start(options);
11318
+ }
11319
+ /**
11320
+ * Ends the recording; resolves as soon as the server recognised the text.
11321
+ * Before the microphone gave any audio, it cancels and resolves null.
11322
+ */
11323
+ stopListening() {
11324
+ return this.voice.stop();
11325
+ }
11326
+ /** Discards the recording. Nothing is recognised or charged. */
11327
+ cancelListening() {
11328
+ this.voice.cancel();
11329
+ }
11330
+ /** Press and hold an element or a key (default Space) to talk. Returns an unbind function. */
11331
+ bindPushToTalk(options) {
11332
+ return bindPushToTalk(this, options);
11333
+ }
11334
+ onListeningState(callback) {
11335
+ return bindCallbackSlot(
11336
+ () => this.listeningStateCallback,
11337
+ (next) => this.listeningStateCallback = next,
11338
+ callback
11339
+ );
11340
+ }
11341
+ /** Partial text while the player speaks (`final: false`), then the final text. */
11342
+ onTranscript(callback) {
11343
+ return bindCallbackSlot(
11344
+ () => this.transcriptCallback,
11345
+ (next) => this.transcriptCallback = next,
11346
+ callback
11347
+ );
11348
+ }
9518
11349
  async getConversationWorldline(conversationId = this.conversationId) {
9519
11350
  if (!conversationId) {
9520
11351
  throw new Error("Conversation ID is required.");
@@ -9538,6 +11369,13 @@ var Character = class extends CharacterRuntime {
9538
11369
  timeline: options?.silent ? toSilentTimeline(normalized.timeline) : normalized.timeline
9539
11370
  });
9540
11371
  }
11372
+ /**
11373
+ * Plays one live staging event from another transport, such as an actor
11374
+ * stage. A timeline event returns its playback result.
11375
+ */
11376
+ playStreamEvent(event) {
11377
+ return this.streamPlayback.handleEvent(event);
11378
+ }
9541
11379
  setRenderSessionId(renderSessionId) {
9542
11380
  this.renderSessionId = renderSessionId?.trim() || null;
9543
11381
  }
@@ -9571,6 +11409,7 @@ var Character = class extends CharacterRuntime {
9571
11409
  async handleStagingResult(payload, options) {
9572
11410
  this.thinkingState = false;
9573
11411
  const normalized = normalizeStagingResultPayload(payload);
11412
+ this.latestOptions = normalized.options;
9574
11413
  if (options.source === "turn") {
9575
11414
  this.liveConversation.received(normalized);
9576
11415
  }
@@ -9597,6 +11436,49 @@ var Character = class extends CharacterRuntime {
9597
11436
  void this.streamPlayback.playPayload(normalized);
9598
11437
  }
9599
11438
  }
11439
+ /** Throws when the input modes do not include `mode`. */
11440
+ assertInputMode(mode) {
11441
+ if (mode === "voice" && this.baked) {
11442
+ throw new InputModeError(
11443
+ "Voice input is not available in a baked (offline) conversation."
11444
+ );
11445
+ }
11446
+ if (mode === "voice" && this.speechInput === false) {
11447
+ throw new InputModeError(
11448
+ "Voice input is off for this character. Its owner turns it on in the STT tab."
11449
+ );
11450
+ }
11451
+ const modes = this.inputModes;
11452
+ if (!modes.includes(mode)) {
11453
+ throw new InputModeError(
11454
+ `The "${mode}" input mode is off: inputModes is ${JSON.stringify(modes)}.`
11455
+ );
11456
+ }
11457
+ }
11458
+ /**
11459
+ * A typed reply needs the `text` mode; one of the current reply options
11460
+ * needs `options` or `text`. Without a configuration the server's rules
11461
+ * apply alone, as before.
11462
+ */
11463
+ assertReplyMode(message) {
11464
+ if (!this.configuredInputModes) {
11465
+ return;
11466
+ }
11467
+ const modes = this.inputModes;
11468
+ if (modes.includes("text")) {
11469
+ return;
11470
+ }
11471
+ const reply = message.trim();
11472
+ const isOption = this.latestOptions.some(
11473
+ (option) => option.label.trim() === reply
11474
+ );
11475
+ if (isOption && modes.includes("options")) {
11476
+ return;
11477
+ }
11478
+ throw new InputModeError(
11479
+ isOption ? `Reply options are off: inputModes is ${JSON.stringify(modes)}.` : `Typed replies are off: inputModes is ${JSON.stringify(modes)}. Send one of the reply options or use startListening().`
11480
+ );
11481
+ }
9600
11482
  invokeThinkingCallback(payload) {
9601
11483
  this.thinkingState = true;
9602
11484
  if (this.thinkingCallback) {
@@ -9609,6 +11491,12 @@ var Character = class extends CharacterRuntime {
9609
11491
  }
9610
11492
  };
9611
11493
  var Npcify = class extends Character {
11494
+ constructor(options) {
11495
+ super({
11496
+ ...options,
11497
+ normalizeIdleOrientation: options.normalizeIdleOrientation ?? true
11498
+ });
11499
+ }
9612
11500
  };
9613
11501
 
9614
11502
  // ../../clients/web-client/src/character/character.ts
@@ -10189,13 +12077,18 @@ export {
10189
12077
  CharacterSceneRig,
10190
12078
  FacialPerformanceLayer,
10191
12079
  FacialPerformanceMixer,
12080
+ INPUT_MODES,
12081
+ InputModeError,
12082
+ MicrophoneError,
10192
12083
  Npcify,
10193
12084
  Character as NpcifyCharacter,
10194
12085
  PERFORMANCE_FILE_EXTENSION,
10195
12086
  PERFORMANCE_MEDIA_TYPE,
10196
12087
  PerformanceRecordingSchemaVersion,
10197
12088
  RemoteAnimationController,
12089
+ SpeechInputError,
10198
12090
  analyzePerformanceRecording,
12091
+ bindPushToTalk,
10199
12092
  createCharacter,
10200
12093
  decodePerformance,
10201
12094
  encodePerformance,