@npcify/web 0.1.3 → 0.1.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.ts CHANGED
@@ -317,6 +317,7 @@ interface AssetLoadProgress {
317
317
 
318
318
  interface AnimationControllerLike {
319
319
  prepareAnimations?: (ids: string[]) => Promise<void>;
320
+ previewAnimation?: (animation: VRMAnimation) => () => void;
320
321
  hasAnimation?: (id: string) => boolean;
321
322
  playAnimation: (animationId: string, fadeDuration?: number, loopOverride?: boolean) => void;
322
323
  playIdle: (fadeDuration?: number) => void;
@@ -560,6 +561,7 @@ declare class FacialPerformanceMixer {
560
561
  getChannelNames(): string[];
561
562
  getEmotionChannelNames(): string[];
562
563
  getVisemeChannelNames(): string[];
564
+ get usesDiscreteMouths(): boolean;
563
565
  playCue(channel: FacialChannelName, options?: FacialCueOptions): void;
564
566
  setEmotion(channel: FacialChannelName, weight: number, mouthMode: FacialMouthMode): void;
565
567
  clearEmotion(mouthMode?: FacialMouthMode): void;
@@ -633,6 +635,7 @@ declare class VisemePlayer {
633
635
  private phase;
634
636
  private running;
635
637
  private timelineRevision;
638
+ private readonly heldMouths;
636
639
  constructor(facialMixer: FacialPerformanceMixer, recorder?: PerformanceRecorder | undefined);
637
640
  attachAudioElement(element: HTMLAudioElement | null): void;
638
641
  setVisemes(visemes: ConversationTimelineEntry[]): void;
@@ -642,6 +645,8 @@ declare class VisemePlayer {
642
645
  getPerformanceSnapshot(): VisemePerformanceFrameSnapshot;
643
646
  update(_delta: number): void;
644
647
  private readonly handlePlaying;
648
+ private updateHeldMouth;
649
+ private cueDuration;
645
650
  private readonly handleEnded;
646
651
  private collectDueCueIndices;
647
652
  private shouldCloseActiveCue;
package/dist/index.js CHANGED
@@ -3835,204 +3835,61 @@ import {
3835
3835
  VRMExpressionTextureTransformBind
3836
3836
  } from "@pixiv/three-vrm";
3837
3837
 
3838
- // ../vrm-renderer/src/texture-expressions-shader.ts
3839
- var sampleMap = "texture2D( map, mapUv )";
3840
- var sampleShade = "texture2D( shadeMultiplyTexture, shadeMultiplyTextureUv )";
3841
- var mouthShader = `
3842
- uniform float npcifyMouthWeights[6];
3843
- uniform float npcifyMouthTexelWidth;
3844
- vec4 npcifySampleMouth(sampler2D atlas, vec2 uv) {
3845
- // Clamp inside the neutral column before adding each whole-cell offset.
3846
- // Interpolating these offsets would slide the face across adjacent cells.
3847
- uv.x = clamp(uv.x, 0.5 * npcifyMouthTexelWidth, 1.0 / 6.0 - 0.5 * npcifyMouthTexelWidth);
3848
- vec4 color = vec4(0.0);
3849
- ${Array.from(
3850
- { length: 6 },
3851
- (_, column) => `
3852
- if (npcifyMouthWeights[${column}] > 0.0) {
3853
- color += npcifyMouthWeights[${column}] * texture2D(atlas, uv + vec2(${column}.0 / 6.0, 0.0));
3854
- }`
3855
- ).join("\n")}
3856
- return color;
3857
- }
3858
- `;
3859
- var createMouthAtlasShader = function(material) {
3860
- if (!material.fragmentShader.includes(sampleMap) || !material.fragmentShader.includes(sampleShade)) {
3861
- return null;
3838
+ // ../vrm-renderer/src/held-mouth-timeline.ts
3839
+ var MinimumExposure = 0.1;
3840
+ var MaximumLinkedGap = 0.3;
3841
+ var UnlinkedExposure = 0.2;
3842
+ var Tolerance = 1e-6;
3843
+ var HeldMouthTimeline = class _HeldMouthTimeline {
3844
+ cues = [];
3845
+ next = 0;
3846
+ shown = null;
3847
+ shownAt = -Infinity;
3848
+ previousTime = -Infinity;
3849
+ static endTime(cue, next, speechEnd = Infinity) {
3850
+ const nextTime = next?.timestamp ?? speechEnd;
3851
+ return Math.min(
3852
+ speechEnd,
3853
+ nextTime - cue.timestamp <= MaximumLinkedGap + Tolerance ? nextTime : cue.timestamp + UnlinkedExposure
3854
+ );
3862
3855
  }
3863
- const fragment = material.fragmentShader;
3864
- const cacheKey = material.customProgramCacheKey;
3865
- const weights = { value: new Float32Array([1, 0, 0, 0, 0, 0]) };
3866
- const texel = { value: 1 / 4096 };
3867
- const uniforms = material.uniforms;
3868
- uniforms.npcifyMouthWeights = weights;
3869
- uniforms.npcifyMouthTexelWidth = texel;
3870
- material.fragmentShader = mouthShader + fragment.replace(sampleMap, "npcifySampleMouth( map, mapUv )").replace(
3871
- sampleShade,
3872
- "npcifySampleMouth( shadeMultiplyTexture, shadeMultiplyTextureUv )"
3873
- );
3874
- material.customProgramCacheKey = function() {
3875
- return `${cacheKey.call(material)},npcify-mouth-atlas-v1`;
3876
- };
3877
- material.needsUpdate = true;
3878
- return {
3879
- update(values) {
3880
- weights.value.set(values);
3881
- const image = material.map?.image;
3882
- if (image && typeof image === "object" && "width" in image && typeof image.width === "number" && image.width > 0) {
3883
- texel.value = 1 / image.width;
3884
- }
3885
- },
3886
- dispose() {
3887
- material.fragmentShader = fragment;
3888
- material.customProgramCacheKey = cacheKey;
3889
- delete uniforms.npcifyMouthWeights;
3890
- delete uniforms.npcifyMouthTexelWidth;
3891
- }
3892
- };
3893
- };
3894
-
3895
- // ../vrm-renderer/src/texture-expressions-pixels.ts
3896
- import { DataTexture, Source as Source2, SRGBColorSpace as SRGBColorSpace2 } from "three";
3897
- var rawImage = function(image) {
3898
- if (!image || typeof image !== "object" || !("data" in image) || !("width" in image) || !("height" in image)) {
3899
- return false;
3856
+ setCues(cues) {
3857
+ this.cues = [...cues].sort((a, b) => a.timestamp - b.timestamp);
3858
+ this.reset();
3900
3859
  }
3901
- return image.data instanceof Uint8Array && typeof image.width === "number" && typeof image.height === "number" && image.width >= 6 && image.height >= 7 && Number.isInteger(image.width) && Number.isInteger(image.height) && image.data.length === image.width * image.height * 4;
3902
- };
3903
- var linear = Float32Array.from({ length: 256 }, (_, value) => {
3904
- const s = value / 255;
3905
- return s <= 0.04045 ? s / 12.92 : ((s + 0.055) / 1.055) ** 2.4;
3906
- });
3907
- var srgb = Uint8Array.from({ length: 4097 }, (_, value) => {
3908
- const s = value / 4096;
3909
- return Math.round(
3910
- 255 * (s <= 31308e-7 ? 12.92 * s : 1.055 * s ** (1 / 2.4) - 0.055)
3911
- );
3912
- });
3913
- var PixelColumns = class {
3914
- constructor(original) {
3915
- this.original = original;
3916
- }
3917
- image = null;
3918
- columns = [];
3919
- previous = new Float32Array(6).fill(-1);
3920
- source = new Source2({ data: new Uint8Array(4), width: 1, height: 1 });
3921
- update(weights) {
3922
- const image = this.original.image;
3923
- if (!rawImage(image)) {
3924
- return false;
3860
+ reset() {
3861
+ this.next = 0;
3862
+ this.shown = null;
3863
+ this.shownAt = -Infinity;
3864
+ this.previousTime = -Infinity;
3865
+ }
3866
+ sample(time, speechEnd = Infinity) {
3867
+ if (!Number.isFinite(time) || time < 0) {
3868
+ this.reset();
3869
+ return null;
3925
3870
  }
3926
- const changed = image !== this.image;
3927
- if (changed) {
3928
- this.prepare(image);
3871
+ if (time < this.previousTime) {
3872
+ this.reset();
3929
3873
  }
3930
- if (!changed && weights.every(
3931
- (value, column) => Math.abs(value - this.previous[column]) < 1e-5
3932
- )) {
3933
- return false;
3874
+ this.previousTime = time;
3875
+ while (this.cues[this.next] && this.cues[this.next].timestamp <= time + Tolerance) {
3876
+ this.next += 1;
3934
3877
  }
3935
- this.previous.set(weights);
3936
- const data = this.source.data.data;
3937
- const active = Array.from(weights.entries()).filter(([, weight]) => weight > 0).map(([column, weight]) => ({ pixels: this.columns[column], weight }));
3938
- const isSrgb = this.original.colorSpace === SRGBColorSpace2;
3939
- for (let i = 0; i < data.length; i += 1) {
3940
- let value = 0;
3941
- for (let column = 0; column < active.length; column += 1) {
3942
- const source = active[column];
3943
- value += source.pixels[i] * source.weight;
3944
- }
3945
- data[i] = isSrgb && i % 4 !== 3 ? srgb[Math.max(0, Math.min(4096, Math.round(value * 4096)))] : Math.round(value * 255);
3878
+ const cue = this.cues[this.next - 1];
3879
+ const end = cue ? _HeldMouthTimeline.endTime(cue, this.cues[this.next], speechEnd) : 0;
3880
+ const weight = cue?.weight ?? 1;
3881
+ if (!cue || time + Tolerance >= end || !Number.isFinite(weight) || weight <= 0) {
3882
+ this.shown = null;
3883
+ return null;
3946
3884
  }
3947
- return true;
3948
- }
3949
- prepare(image) {
3950
- this.image = image;
3951
- const width = Math.max(1, Math.round(image.width / 6));
3952
- const length = width * image.height * 4;
3953
- this.source.data = {
3954
- data: new Uint8Array(length),
3955
- width,
3956
- height: image.height
3957
- };
3958
- const isSrgb = this.original.colorSpace === SRGBColorSpace2;
3959
- this.columns = Array.from({ length: 6 }, (_, column) => {
3960
- const pixels = new Float32Array(length);
3961
- for (let y = 0; y < image.height; y += 1) {
3962
- for (let x = 0; x < width; x += 1) {
3963
- const sample = ((x + 0.5) / width + column) * image.width / 6 - 0.5;
3964
- const left = Math.max(
3965
- 0,
3966
- Math.min(image.width - 1, Math.floor(sample))
3967
- );
3968
- const right = Math.min(image.width - 1, left + 1);
3969
- const mix = Math.max(0, sample - left);
3970
- for (let channel = 0; channel < 4; channel += 1) {
3971
- const a = image.data[(y * image.width + left) * 4 + channel];
3972
- const b = image.data[(y * image.width + right) * 4 + channel];
3973
- const convert = isSrgb && channel !== 3;
3974
- const first = convert ? linear[a] : a / 255;
3975
- const second = convert ? linear[b] : b / 255;
3976
- pixels[(y * width + x) * 4 + channel] = first + (second - first) * mix;
3977
- }
3978
- }
3979
- }
3980
- return pixels;
3981
- });
3982
- }
3983
- dispose() {
3984
- this.columns = [];
3985
- this.image = null;
3986
- this.source.data = null;
3987
- }
3988
- };
3989
- var createMouthAtlasPixels = function(material) {
3990
- const properties = ["map", "shadeMultiplyTexture"];
3991
- if (properties.some(
3992
- (name) => material[name] && !rawImage(material[name].image)
3993
- )) {
3994
- return null;
3995
- }
3996
- const columns = /* @__PURE__ */ new Map();
3997
- const textures = properties.flatMap((name) => {
3998
- const original = material[name];
3999
- if (!original) {
4000
- return [];
3885
+ if (!this.shown || cue.name !== this.shown.name && time - this.shownAt + Tolerance >= MinimumExposure) {
3886
+ this.shown = cue;
3887
+ this.shownAt = time;
3888
+ } else if (cue.name === this.shown.name) {
3889
+ this.shown = cue;
4001
3890
  }
4002
- let pixels = columns.get(original.colorSpace);
4003
- if (!pixels) {
4004
- pixels = new PixelColumns(original);
4005
- columns.set(original.colorSpace, pixels);
4006
- }
4007
- const texture = new DataTexture();
4008
- texture.copy(original);
4009
- texture.source = pixels.source;
4010
- texture.generateMipmaps = false;
4011
- texture.minFilter = original.magFilter;
4012
- material[name] = texture;
4013
- return [{ name, original, texture, pixels }];
4014
- });
4015
- return {
4016
- update(weights) {
4017
- const changed = new Set(
4018
- [...columns.values()].filter((pixels) => pixels.update(weights))
4019
- );
4020
- for (const { texture, pixels } of textures) {
4021
- texture.repeat.x = 6;
4022
- if (changed.has(pixels)) {
4023
- texture.needsUpdate = true;
4024
- }
4025
- }
4026
- },
4027
- dispose() {
4028
- for (const { name, original, texture } of textures) {
4029
- material[name] = original;
4030
- texture.dispose();
4031
- original.dispose();
4032
- }
4033
- columns.forEach((pixels) => pixels.dispose());
4034
- }
4035
- };
3891
+ return this.shown;
3892
+ }
4036
3893
  };
4037
3894
 
4038
3895
  // ../vrm-renderer/src/texture-expressions.ts
@@ -4097,72 +3954,56 @@ var findAtlasBindings = function(vrm) {
4097
3954
  }
4098
3955
  return bindings;
4099
3956
  };
4100
- var enableTextureExpressionBlending = function(vrm, options = {}) {
4101
- if (installed.has(vrm)) {
3957
+ var hasTextureMouthExpressions = function(manager) {
3958
+ return !!manager && installed.has(manager);
3959
+ };
3960
+ var enableTextureExpressionPlayback = function(vrm) {
3961
+ const manager = vrm.expressionManager;
3962
+ if (!manager || installed.has(manager)) {
4102
3963
  return;
4103
3964
  }
4104
3965
  const bindings = findAtlasBindings(vrm);
4105
3966
  if (!bindings.length) {
4106
3967
  return;
4107
3968
  }
4108
- const materials = [
4109
- ...new Set(bindings.map(({ bind }) => bind.material))
4110
- ];
4111
- const outputs = materials.map(
4112
- (material) => options.backend === "pixels" ? createMouthAtlasPixels(material) : createMouthAtlasShader(material)
3969
+ installed.add(manager);
3970
+ const materials = new Set(
3971
+ bindings.map(({ bind }) => bind.material)
4113
3972
  );
4114
- if (outputs.some((output) => !output)) {
4115
- outputs.forEach((output) => output?.dispose());
4116
- return;
4117
- }
4118
- installed.add(vrm);
4119
- for (const [index, material] of materials.entries()) {
4120
- const output = outputs[index];
3973
+ for (const material of materials) {
3974
+ const mouths = bindings.filter(({ bind }) => bind.material === material);
4121
3975
  const targets = new Float32Array(6);
4122
- const weights = new Float32Array([1, 0, 0, 0, 0, 0]);
4123
- for (const { expression, bind, column } of bindings) {
4124
- if (bind.material !== material) {
4125
- continue;
4126
- }
3976
+ let selected = 0;
3977
+ for (const { expression, bind, column } of mouths) {
4127
3978
  expression.isBinary = false;
4128
3979
  expression.deleteBind(bind);
4129
3980
  expression.addBind({
4130
3981
  clearAppliedWeight: () => {
4131
3982
  targets[column] = 0;
3983
+ bind.clearAppliedWeight();
4132
3984
  },
4133
3985
  applyWeight: (weight) => {
4134
- targets[column] = Number.isFinite(weight) ? Math.max(0, Math.min(1, weight)) : 0;
3986
+ targets[column] = Number.isFinite(weight) ? Math.max(0, weight) : 0;
4135
3987
  }
4136
3988
  });
4137
3989
  }
4138
3990
  const originalUpdate = material.update;
4139
3991
  material.update = function(delta) {
4140
- let total = 0;
3992
+ let winner = selected;
3993
+ let weight = targets[winner];
4141
3994
  for (let column = 1; column < 6; column += 1) {
4142
- total += targets[column];
4143
- }
4144
- targets[0] = Math.max(0, 1 - total);
4145
- const scale = Math.max(1, total);
4146
- let distance = 0;
4147
- for (let column = 0; column < 6; column += 1) {
4148
- targets[column] = targets[column] / scale;
4149
- distance = Math.max(
4150
- distance,
4151
- Math.abs(targets[column] - weights[column])
4152
- );
4153
- }
4154
- const step = Number.isFinite(delta) ? Math.max(0, delta) / 0.08 : 0;
4155
- const factor = distance > 0 ? Math.min(1, step / distance) : 1;
4156
- for (let column = 0; column < 6; column += 1) {
4157
- weights[column] = weights[column] + (targets[column] - weights[column]) * factor;
3995
+ if (targets[column] > weight) {
3996
+ winner = column;
3997
+ weight = targets[column];
3998
+ }
4158
3999
  }
4159
- output.update(weights);
4000
+ selected = weight > 0 ? winner : 0;
4001
+ mouths.find(({ column }) => column === selected)?.bind.applyWeight(1);
4160
4002
  originalUpdate.call(material, delta);
4161
4003
  };
4162
4004
  const dispose = function() {
4163
4005
  material.removeEventListener("dispose", dispose);
4164
4006
  material.update = originalUpdate;
4165
- output.dispose();
4166
4007
  };
4167
4008
  material.addEventListener("dispose", dispose);
4168
4009
  }
@@ -4492,7 +4333,7 @@ var loadVrm = async ({
4492
4333
  });
4493
4334
  VRMUtils4.removeUnnecessaryVertices(gltf.scene);
4494
4335
  VRMUtils4.combineSkeletons(gltf.scene);
4495
- enableTextureExpressionBlending(gltf.userData.vrm);
4336
+ enableTextureExpressionPlayback(gltf.userData.vrm);
4496
4337
  return gltf.userData.vrm;
4497
4338
  };
4498
4339
  var loadVrma = async ({
@@ -4624,6 +4465,8 @@ var AnimationController = class {
4624
4465
  mixer;
4625
4466
  clips = /* @__PURE__ */ new Map();
4626
4467
  configs = /* @__PURE__ */ new Map();
4468
+ previewMixer = null;
4469
+ restorePreview = null;
4627
4470
  activeAction = null;
4628
4471
  idleAnimationId;
4629
4472
  pendingReturnToIdleAction = null;
@@ -4638,11 +4481,35 @@ var AnimationController = class {
4638
4481
  weight: config?.weight ?? defaultAnimationConfig.weight
4639
4482
  });
4640
4483
  }
4484
+ previewAnimation(animation) {
4485
+ this.restorePreview?.();
4486
+ const clip = createVRMAnimationClip(animation, this.vrm);
4487
+ const previewMixer = new AnimationMixer2(this.vrm.scene);
4488
+ previewMixer.clipAction(clip).setLoop(LoopRepeat2, Infinity).reset().play();
4489
+ this.previewMixer = previewMixer;
4490
+ const restore = () => {
4491
+ if (this.restorePreview !== restore) {
4492
+ return;
4493
+ }
4494
+ this.restorePreview = null;
4495
+ previewMixer.stopAllAction();
4496
+ previewMixer.uncacheRoot(this.vrm.scene);
4497
+ this.previewMixer = null;
4498
+ if (!this.activeAction) {
4499
+ this.vrm.humanoid.resetNormalizedPose();
4500
+ }
4501
+ this.mixer.update(0);
4502
+ };
4503
+ this.restorePreview = restore;
4504
+ previewMixer.update(0);
4505
+ return restore;
4506
+ }
4641
4507
  setIdleAnimation(animationId) {
4642
4508
  this.idleAnimationId = animationId;
4643
4509
  this.playIdle();
4644
4510
  }
4645
4511
  clearAnimations() {
4512
+ this.restorePreview?.();
4646
4513
  this.mixer.stopAllAction();
4647
4514
  this.mixer.uncacheRoot(this.vrm.scene);
4648
4515
  this.clips.clear();
@@ -4653,6 +4520,7 @@ var AnimationController = class {
4653
4520
  this.vrm.humanoid.resetNormalizedPose();
4654
4521
  }
4655
4522
  playAnimation(animationId, fadeDuration = 0.25, loopOverride) {
4523
+ this.restorePreview?.();
4656
4524
  const clip = this.clips.get(animationId);
4657
4525
  if (!clip) {
4658
4526
  throw new Error(`Animation ${animationId} not found`);
@@ -4686,6 +4554,7 @@ var AnimationController = class {
4686
4554
  }
4687
4555
  }
4688
4556
  playIdle(fadeDuration = 0.25) {
4557
+ this.restorePreview?.();
4689
4558
  if (!this.idleAnimationId || !this.clips.has(this.idleAnimationId)) {
4690
4559
  this.mixer.stopAllAction();
4691
4560
  this.activeAction = null;
@@ -4699,6 +4568,7 @@ var AnimationController = class {
4699
4568
  this.playAnimation(this.idleAnimationId, fadeDuration, true);
4700
4569
  }
4701
4570
  stopAnimation(fadeDuration = 0.25) {
4571
+ this.restorePreview?.();
4702
4572
  if (!this.activeAction) {
4703
4573
  return;
4704
4574
  }
@@ -4707,7 +4577,8 @@ var AnimationController = class {
4707
4577
  this.pendingReturnToIdleAction = null;
4708
4578
  }
4709
4579
  update(delta) {
4710
- this.mixer.update(delta);
4580
+ ;
4581
+ (this.previewMixer ?? this.mixer).update(delta);
4711
4582
  }
4712
4583
  handleActionFinished = (event) => {
4713
4584
  if (!this.idleAnimationId || !this.pendingReturnToIdleAction || event.action !== this.pendingReturnToIdleAction || !this.clips.has(this.idleAnimationId)) {
@@ -5359,6 +5230,7 @@ var PerformanceWeightMixer = class {
5359
5230
  ...definition,
5360
5231
  id,
5361
5232
  responseRate: normalizeResponseRate(definition.responseRate),
5233
+ transition: definition.transition ?? "smooth",
5362
5234
  weight
5363
5235
  },
5364
5236
  order: this.nextLayerOrder,
@@ -5435,12 +5307,13 @@ var PerformanceWeightMixer = class {
5435
5307
  const resolved = /* @__PURE__ */ new Map();
5436
5308
  this.orderedLayers.forEach((layer) => {
5437
5309
  const responseRate = layer.definition.responseRate;
5310
+ const frameDelta = layer.definition.transition === "step" ? 0 : delta;
5438
5311
  layer.value = clamp01(
5439
- smoothValue(layer.value, layer.target, responseRate, delta)
5312
+ smoothValue(layer.value, layer.target, responseRate, frameDelta)
5440
5313
  );
5441
5314
  layer.channels.forEach((channel, name) => {
5442
5315
  channel.value = clamp01(
5443
- smoothValue(channel.value, channel.target, responseRate, delta)
5316
+ smoothValue(channel.value, channel.target, responseRate, frameDelta)
5444
5317
  );
5445
5318
  const current = resolved.get(name) ?? 0;
5446
5319
  const contribution = channel.value * layer.value;
@@ -5566,6 +5439,9 @@ var FacialPerformanceMixer = class {
5566
5439
  getVisemeChannelNames() {
5567
5440
  return this.getLayerChannels(FacialPerformanceLayer.Viseme);
5568
5441
  }
5442
+ get usesDiscreteMouths() {
5443
+ return hasTextureMouthExpressions(this.vrm?.expressionManager);
5444
+ }
5569
5445
  playCue(channel, options) {
5570
5446
  if (!this.availableChannelSet.has(channel)) {
5571
5447
  return;
@@ -5726,7 +5602,8 @@ var FacialPerformanceMixer = class {
5726
5602
  this.weightMixer.registerLayer({
5727
5603
  id: FacialPerformanceLayer.Viseme,
5728
5604
  priority: 200,
5729
- blendMode: "override"
5605
+ blendMode: "override",
5606
+ transition: this.usesDiscreteMouths ? "step" : "smooth"
5730
5607
  });
5731
5608
  this.weightMixer.registerLayer({
5732
5609
  id: FacialPerformanceLayer.Blink,
@@ -6475,6 +6352,7 @@ var VisemePlayer = class {
6475
6352
  phase = "idle";
6476
6353
  running = false;
6477
6354
  timelineRevision = 0;
6355
+ heldMouths = new HeldMouthTimeline();
6478
6356
  attachAudioElement(element) {
6479
6357
  if (this.audioElement === element) {
6480
6358
  return;
@@ -6509,6 +6387,7 @@ var VisemePlayer = class {
6509
6387
  this.rawVisemes = visemes.map((viseme) => ({ ...viseme }));
6510
6388
  this.normalizedTimeline = normalizeVisemeTimeline(this.rawVisemes);
6511
6389
  this.visemes = this.normalizedTimeline.entries;
6390
+ this.heldMouths.setCues(this.visemes);
6512
6391
  this.timelineRevision += 1;
6513
6392
  this.resetPlaybackState();
6514
6393
  this.recordTimeline();
@@ -6570,6 +6449,10 @@ var VisemePlayer = class {
6570
6449
  if (currentTime === null) {
6571
6450
  return;
6572
6451
  }
6452
+ if (this.facialMixer.usesDiscreteMouths) {
6453
+ this.updateHeldMouth(currentTime);
6454
+ return;
6455
+ }
6573
6456
  const dueCueIndices = this.collectDueCueIndices(currentTime);
6574
6457
  const latestDueCueIndex = dueCueIndices.at(-1);
6575
6458
  dueCueIndices.slice(0, -1).forEach((cueIndex) => {
@@ -6600,6 +6483,34 @@ var VisemePlayer = class {
6600
6483
  state: "playing"
6601
6484
  });
6602
6485
  };
6486
+ updateHeldMouth(currentTime) {
6487
+ const duration = this.audioElement?.duration;
6488
+ const speechEnd = Number.isFinite(duration) ? duration : void 0;
6489
+ const cue = this.heldMouths.sample(currentTime, speechEnd);
6490
+ if (!cue) {
6491
+ if (this.activeCueIndex !== null) {
6492
+ this.releaseToNeutral("scheduled");
6493
+ }
6494
+ return;
6495
+ }
6496
+ if (cue === this.visemes[this.activeCueIndex ?? -1]) {
6497
+ return;
6498
+ }
6499
+ const index = this.visemes.indexOf(cue);
6500
+ this.facialMixer.playCue(cue.name, { weight: 1, durationMs: 0 });
6501
+ this.activeCueIndex = index;
6502
+ this.nextVisemeIndex = index + 1;
6503
+ this.phase = "open";
6504
+ this.recordCueDecision(index, currentTime, "played");
6505
+ }
6506
+ cueDuration(cue, next) {
6507
+ if (this.facialMixer.usesDiscreteMouths) {
6508
+ const duration = this.audioElement?.duration;
6509
+ const speechEnd = Number.isFinite(duration) ? duration : void 0;
6510
+ return (HeldMouthTimeline.endTime(cue, next, speechEnd) - cue.timestamp) * 1e3;
6511
+ }
6512
+ return getVisemeDurationMs(cue, next);
6513
+ }
6603
6514
  handleEnded = () => {
6604
6515
  this.recorder?.recordEvent({
6605
6516
  type: "audio.state",
@@ -6728,7 +6639,7 @@ var VisemePlayer = class {
6728
6639
  if (!cue) {
6729
6640
  return;
6730
6641
  }
6731
- const openDurationMs = getVisemeDurationMs(cue, this.visemes[cueIndex + 1]);
6642
+ const openDurationMs = this.cueDuration(cue, this.visemes[cueIndex + 1]);
6732
6643
  const elapsedMs = Math.max(currentTime - cue.timestamp, 0) * 1e3;
6733
6644
  this.recorder?.recordEvent({
6734
6645
  type: "viseme.cue",
@@ -6773,6 +6684,7 @@ var VisemePlayer = class {
6773
6684
  this.audioElement.removeEventListener("ended", this.handleEnded);
6774
6685
  }
6775
6686
  resetPlaybackState() {
6687
+ this.heldMouths.reset();
6776
6688
  this.running = false;
6777
6689
  this.nextVisemeIndex = 0;
6778
6690
  this.activeCueIndex = null;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@npcify/web",
3
- "version": "0.1.3",
3
+ "version": "0.1.4",
4
4
  "description": "Framework-independent Npcify characters, live conversations and baked performances for the web",
5
5
  "type": "module",
6
6
  "license": "UNLICENSED",
@@ -35,8 +35,8 @@
35
35
  "tsup": "^8.5.0",
36
36
  "typescript": "^5.9.3",
37
37
  "@npcify/shared": "^0.1.0",
38
- "@npcify/vrm-renderer": "^0.1.0",
39
- "@npcify/web-client": "^0.0.0"
38
+ "@npcify/web-client": "^0.0.0",
39
+ "@npcify/vrm-renderer": "^0.1.0"
40
40
  },
41
41
  "repository": {
42
42
  "type": "git",