@npcify/web 0.1.3 → 0.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.ts CHANGED
@@ -317,6 +317,7 @@ interface AssetLoadProgress {
317
317
 
318
318
  interface AnimationControllerLike {
319
319
  prepareAnimations?: (ids: string[]) => Promise<void>;
320
+ previewAnimation?: (animation: VRMAnimation) => () => void;
320
321
  hasAnimation?: (id: string) => boolean;
321
322
  playAnimation: (animationId: string, fadeDuration?: number, loopOverride?: boolean) => void;
322
323
  playIdle: (fadeDuration?: number) => void;
@@ -560,6 +561,7 @@ declare class FacialPerformanceMixer {
560
561
  getChannelNames(): string[];
561
562
  getEmotionChannelNames(): string[];
562
563
  getVisemeChannelNames(): string[];
564
+ get usesDiscreteMouths(): boolean;
563
565
  playCue(channel: FacialChannelName, options?: FacialCueOptions): void;
564
566
  setEmotion(channel: FacialChannelName, weight: number, mouthMode: FacialMouthMode): void;
565
567
  clearEmotion(mouthMode?: FacialMouthMode): void;
@@ -633,6 +635,7 @@ declare class VisemePlayer {
633
635
  private phase;
634
636
  private running;
635
637
  private timelineRevision;
638
+ private readonly heldMouths;
636
639
  constructor(facialMixer: FacialPerformanceMixer, recorder?: PerformanceRecorder | undefined);
637
640
  attachAudioElement(element: HTMLAudioElement | null): void;
638
641
  setVisemes(visemes: ConversationTimelineEntry[]): void;
@@ -642,6 +645,8 @@ declare class VisemePlayer {
642
645
  getPerformanceSnapshot(): VisemePerformanceFrameSnapshot;
643
646
  update(_delta: number): void;
644
647
  private readonly handlePlaying;
648
+ private updateHeldMouth;
649
+ private cueDuration;
645
650
  private readonly handleEnded;
646
651
  private collectDueCueIndices;
647
652
  private shouldCloseActiveCue;
package/dist/index.js CHANGED
@@ -3477,10 +3477,15 @@ var AVATAR_COLOR_SETTINGS = {
3477
3477
  toneMapping: NoToneMapping,
3478
3478
  outputColorSpace: SRGBColorSpace
3479
3479
  };
3480
+ var AVATAR_LIGHTS = {
3481
+ ambientIntensity: 0.7 * Math.PI / 1.9,
3482
+ directionalIntensity: 1.2 * Math.PI / 1.9,
3483
+ directionalPosition: [1, 1.5, 1]
3484
+ };
3480
3485
  var addAvatarLights = function(scene, {
3481
- ambientIntensity = 0.7,
3482
- directionalIntensity = 1.2,
3483
- directionalPosition = [1, 1.5, 1]
3486
+ ambientIntensity = AVATAR_LIGHTS.ambientIntensity,
3487
+ directionalIntensity = AVATAR_LIGHTS.directionalIntensity,
3488
+ directionalPosition = [...AVATAR_LIGHTS.directionalPosition]
3484
3489
  } = {}) {
3485
3490
  scene.add(new AmbientLight(16777215, ambientIntensity));
3486
3491
  const key = new DirectionalLight(16777215, directionalIntensity);
@@ -3829,210 +3834,73 @@ import {
3829
3834
  Texture as Texture2
3830
3835
  } from "three";
3831
3836
 
3837
+ // ../vrm-renderer/src/generated-character.ts
3838
+ import {
3839
+ MToonMaterial as MToonMaterial2,
3840
+ VRMExpressionMaterialColorBind
3841
+ } from "@pixiv/three-vrm";
3842
+
3832
3843
  // ../vrm-renderer/src/texture-expressions.ts
3833
3844
  import {
3834
3845
  MToonMaterial,
3835
3846
  VRMExpressionTextureTransformBind
3836
3847
  } from "@pixiv/three-vrm";
3837
3848
 
3838
- // ../vrm-renderer/src/texture-expressions-shader.ts
3839
- var sampleMap = "texture2D( map, mapUv )";
3840
- var sampleShade = "texture2D( shadeMultiplyTexture, shadeMultiplyTextureUv )";
3841
- var mouthShader = `
3842
- uniform float npcifyMouthWeights[6];
3843
- uniform float npcifyMouthTexelWidth;
3844
- vec4 npcifySampleMouth(sampler2D atlas, vec2 uv) {
3845
- // Clamp inside the neutral column before adding each whole-cell offset.
3846
- // Interpolating these offsets would slide the face across adjacent cells.
3847
- uv.x = clamp(uv.x, 0.5 * npcifyMouthTexelWidth, 1.0 / 6.0 - 0.5 * npcifyMouthTexelWidth);
3848
- vec4 color = vec4(0.0);
3849
- ${Array.from(
3850
- { length: 6 },
3851
- (_, column) => `
3852
- if (npcifyMouthWeights[${column}] > 0.0) {
3853
- color += npcifyMouthWeights[${column}] * texture2D(atlas, uv + vec2(${column}.0 / 6.0, 0.0));
3854
- }`
3855
- ).join("\n")}
3856
- return color;
3857
- }
3858
- `;
3859
- var createMouthAtlasShader = function(material) {
3860
- if (!material.fragmentShader.includes(sampleMap) || !material.fragmentShader.includes(sampleShade)) {
3861
- return null;
3849
+ // ../vrm-renderer/src/held-mouth-timeline.ts
3850
+ var MinimumExposure = 0.1;
3851
+ var MaximumLinkedGap = 0.3;
3852
+ var UnlinkedExposure = 0.2;
3853
+ var Tolerance = 1e-6;
3854
+ var HeldMouthTimeline = class _HeldMouthTimeline {
3855
+ cues = [];
3856
+ next = 0;
3857
+ shown = null;
3858
+ shownAt = -Infinity;
3859
+ previousTime = -Infinity;
3860
+ static endTime(cue, next, speechEnd = Infinity) {
3861
+ const nextTime = next?.timestamp ?? speechEnd;
3862
+ return Math.min(
3863
+ speechEnd,
3864
+ nextTime - cue.timestamp <= MaximumLinkedGap + Tolerance ? nextTime : cue.timestamp + UnlinkedExposure
3865
+ );
3862
3866
  }
3863
- const fragment = material.fragmentShader;
3864
- const cacheKey = material.customProgramCacheKey;
3865
- const weights = { value: new Float32Array([1, 0, 0, 0, 0, 0]) };
3866
- const texel = { value: 1 / 4096 };
3867
- const uniforms = material.uniforms;
3868
- uniforms.npcifyMouthWeights = weights;
3869
- uniforms.npcifyMouthTexelWidth = texel;
3870
- material.fragmentShader = mouthShader + fragment.replace(sampleMap, "npcifySampleMouth( map, mapUv )").replace(
3871
- sampleShade,
3872
- "npcifySampleMouth( shadeMultiplyTexture, shadeMultiplyTextureUv )"
3873
- );
3874
- material.customProgramCacheKey = function() {
3875
- return `${cacheKey.call(material)},npcify-mouth-atlas-v1`;
3876
- };
3877
- material.needsUpdate = true;
3878
- return {
3879
- update(values) {
3880
- weights.value.set(values);
3881
- const image = material.map?.image;
3882
- if (image && typeof image === "object" && "width" in image && typeof image.width === "number" && image.width > 0) {
3883
- texel.value = 1 / image.width;
3884
- }
3885
- },
3886
- dispose() {
3887
- material.fragmentShader = fragment;
3888
- material.customProgramCacheKey = cacheKey;
3889
- delete uniforms.npcifyMouthWeights;
3890
- delete uniforms.npcifyMouthTexelWidth;
3891
- }
3892
- };
3893
- };
3894
-
3895
- // ../vrm-renderer/src/texture-expressions-pixels.ts
3896
- import { DataTexture, Source as Source2, SRGBColorSpace as SRGBColorSpace2 } from "three";
3897
- var rawImage = function(image) {
3898
- if (!image || typeof image !== "object" || !("data" in image) || !("width" in image) || !("height" in image)) {
3899
- return false;
3867
+ setCues(cues) {
3868
+ this.cues = [...cues].sort((a, b) => a.timestamp - b.timestamp);
3869
+ this.reset();
3900
3870
  }
3901
- return image.data instanceof Uint8Array && typeof image.width === "number" && typeof image.height === "number" && image.width >= 6 && image.height >= 7 && Number.isInteger(image.width) && Number.isInteger(image.height) && image.data.length === image.width * image.height * 4;
3902
- };
3903
- var linear = Float32Array.from({ length: 256 }, (_, value) => {
3904
- const s = value / 255;
3905
- return s <= 0.04045 ? s / 12.92 : ((s + 0.055) / 1.055) ** 2.4;
3906
- });
3907
- var srgb = Uint8Array.from({ length: 4097 }, (_, value) => {
3908
- const s = value / 4096;
3909
- return Math.round(
3910
- 255 * (s <= 31308e-7 ? 12.92 * s : 1.055 * s ** (1 / 2.4) - 0.055)
3911
- );
3912
- });
3913
- var PixelColumns = class {
3914
- constructor(original) {
3915
- this.original = original;
3916
- }
3917
- image = null;
3918
- columns = [];
3919
- previous = new Float32Array(6).fill(-1);
3920
- source = new Source2({ data: new Uint8Array(4), width: 1, height: 1 });
3921
- update(weights) {
3922
- const image = this.original.image;
3923
- if (!rawImage(image)) {
3924
- return false;
3925
- }
3926
- const changed = image !== this.image;
3927
- if (changed) {
3928
- this.prepare(image);
3871
+ reset() {
3872
+ this.next = 0;
3873
+ this.shown = null;
3874
+ this.shownAt = -Infinity;
3875
+ this.previousTime = -Infinity;
3876
+ }
3877
+ sample(time, speechEnd = Infinity) {
3878
+ if (!Number.isFinite(time) || time < 0) {
3879
+ this.reset();
3880
+ return null;
3929
3881
  }
3930
- if (!changed && weights.every(
3931
- (value, column) => Math.abs(value - this.previous[column]) < 1e-5
3932
- )) {
3933
- return false;
3882
+ if (time < this.previousTime) {
3883
+ this.reset();
3934
3884
  }
3935
- this.previous.set(weights);
3936
- const data = this.source.data.data;
3937
- const active = Array.from(weights.entries()).filter(([, weight]) => weight > 0).map(([column, weight]) => ({ pixels: this.columns[column], weight }));
3938
- const isSrgb = this.original.colorSpace === SRGBColorSpace2;
3939
- for (let i = 0; i < data.length; i += 1) {
3940
- let value = 0;
3941
- for (let column = 0; column < active.length; column += 1) {
3942
- const source = active[column];
3943
- value += source.pixels[i] * source.weight;
3944
- }
3945
- data[i] = isSrgb && i % 4 !== 3 ? srgb[Math.max(0, Math.min(4096, Math.round(value * 4096)))] : Math.round(value * 255);
3885
+ this.previousTime = time;
3886
+ while (this.cues[this.next] && this.cues[this.next].timestamp <= time + Tolerance) {
3887
+ this.next += 1;
3946
3888
  }
3947
- return true;
3948
- }
3949
- prepare(image) {
3950
- this.image = image;
3951
- const width = Math.max(1, Math.round(image.width / 6));
3952
- const length = width * image.height * 4;
3953
- this.source.data = {
3954
- data: new Uint8Array(length),
3955
- width,
3956
- height: image.height
3957
- };
3958
- const isSrgb = this.original.colorSpace === SRGBColorSpace2;
3959
- this.columns = Array.from({ length: 6 }, (_, column) => {
3960
- const pixels = new Float32Array(length);
3961
- for (let y = 0; y < image.height; y += 1) {
3962
- for (let x = 0; x < width; x += 1) {
3963
- const sample = ((x + 0.5) / width + column) * image.width / 6 - 0.5;
3964
- const left = Math.max(
3965
- 0,
3966
- Math.min(image.width - 1, Math.floor(sample))
3967
- );
3968
- const right = Math.min(image.width - 1, left + 1);
3969
- const mix = Math.max(0, sample - left);
3970
- for (let channel = 0; channel < 4; channel += 1) {
3971
- const a = image.data[(y * image.width + left) * 4 + channel];
3972
- const b = image.data[(y * image.width + right) * 4 + channel];
3973
- const convert = isSrgb && channel !== 3;
3974
- const first = convert ? linear[a] : a / 255;
3975
- const second = convert ? linear[b] : b / 255;
3976
- pixels[(y * width + x) * 4 + channel] = first + (second - first) * mix;
3977
- }
3978
- }
3979
- }
3980
- return pixels;
3981
- });
3982
- }
3983
- dispose() {
3984
- this.columns = [];
3985
- this.image = null;
3986
- this.source.data = null;
3987
- }
3988
- };
3989
- var createMouthAtlasPixels = function(material) {
3990
- const properties = ["map", "shadeMultiplyTexture"];
3991
- if (properties.some(
3992
- (name) => material[name] && !rawImage(material[name].image)
3993
- )) {
3994
- return null;
3995
- }
3996
- const columns = /* @__PURE__ */ new Map();
3997
- const textures = properties.flatMap((name) => {
3998
- const original = material[name];
3999
- if (!original) {
4000
- return [];
3889
+ const cue = this.cues[this.next - 1];
3890
+ const end = cue ? _HeldMouthTimeline.endTime(cue, this.cues[this.next], speechEnd) : 0;
3891
+ const weight = cue?.weight ?? 1;
3892
+ if (!cue || time + Tolerance >= end || !Number.isFinite(weight) || weight <= 0) {
3893
+ this.shown = null;
3894
+ return null;
4001
3895
  }
4002
- let pixels = columns.get(original.colorSpace);
4003
- if (!pixels) {
4004
- pixels = new PixelColumns(original);
4005
- columns.set(original.colorSpace, pixels);
4006
- }
4007
- const texture = new DataTexture();
4008
- texture.copy(original);
4009
- texture.source = pixels.source;
4010
- texture.generateMipmaps = false;
4011
- texture.minFilter = original.magFilter;
4012
- material[name] = texture;
4013
- return [{ name, original, texture, pixels }];
4014
- });
4015
- return {
4016
- update(weights) {
4017
- const changed = new Set(
4018
- [...columns.values()].filter((pixels) => pixels.update(weights))
4019
- );
4020
- for (const { texture, pixels } of textures) {
4021
- texture.repeat.x = 6;
4022
- if (changed.has(pixels)) {
4023
- texture.needsUpdate = true;
4024
- }
4025
- }
4026
- },
4027
- dispose() {
4028
- for (const { name, original, texture } of textures) {
4029
- material[name] = original;
4030
- texture.dispose();
4031
- original.dispose();
4032
- }
4033
- columns.forEach((pixels) => pixels.dispose());
3896
+ if (!this.shown || cue.name !== this.shown.name && time - this.shownAt + Tolerance >= MinimumExposure) {
3897
+ this.shown = cue;
3898
+ this.shownAt = time;
3899
+ } else if (cue.name === this.shown.name) {
3900
+ this.shown = cue;
4034
3901
  }
4035
- };
3902
+ return this.shown;
3903
+ }
4036
3904
  };
4037
3905
 
4038
3906
  // ../vrm-renderer/src/texture-expressions.ts
@@ -4097,77 +3965,126 @@ var findAtlasBindings = function(vrm) {
4097
3965
  }
4098
3966
  return bindings;
4099
3967
  };
4100
- var enableTextureExpressionBlending = function(vrm, options = {}) {
4101
- if (installed.has(vrm)) {
4102
- return;
4103
- }
4104
- const bindings = findAtlasBindings(vrm);
4105
- if (!bindings.length) {
3968
+ var hasTextureMouthExpressions = function(manager) {
3969
+ return !!manager && installed.has(manager);
3970
+ };
3971
+ var installTextureMouthPlayback = function(vrm, bindings) {
3972
+ const manager = vrm.expressionManager;
3973
+ if (!manager || installed.has(manager) || !bindings.length) {
4106
3974
  return;
4107
3975
  }
4108
- const materials = [
4109
- ...new Set(bindings.map(({ bind }) => bind.material))
4110
- ];
4111
- const outputs = materials.map(
4112
- (material) => options.backend === "pixels" ? createMouthAtlasPixels(material) : createMouthAtlasShader(material)
3976
+ installed.add(manager);
3977
+ const materials = new Set(
3978
+ bindings.map(({ bind }) => bind.material)
4113
3979
  );
4114
- if (outputs.some((output) => !output)) {
4115
- outputs.forEach((output) => output?.dispose());
4116
- return;
4117
- }
4118
- installed.add(vrm);
4119
- for (const [index, material] of materials.entries()) {
4120
- const output = outputs[index];
3980
+ for (const material of materials) {
3981
+ const mouths = bindings.filter(({ bind }) => bind.material === material);
4121
3982
  const targets = new Float32Array(6);
4122
- const weights = new Float32Array([1, 0, 0, 0, 0, 0]);
4123
- for (const { expression, bind, column } of bindings) {
4124
- if (bind.material !== material) {
4125
- continue;
4126
- }
3983
+ let selected = 0;
3984
+ for (const { expression, bind, column } of mouths) {
4127
3985
  expression.isBinary = false;
4128
3986
  expression.deleteBind(bind);
4129
3987
  expression.addBind({
4130
3988
  clearAppliedWeight: () => {
4131
3989
  targets[column] = 0;
3990
+ bind.clearAppliedWeight();
4132
3991
  },
4133
3992
  applyWeight: (weight) => {
4134
- targets[column] = Number.isFinite(weight) ? Math.max(0, Math.min(1, weight)) : 0;
3993
+ targets[column] = Number.isFinite(weight) ? Math.max(0, weight) : 0;
4135
3994
  }
4136
3995
  });
4137
3996
  }
4138
3997
  const originalUpdate = material.update;
4139
3998
  material.update = function(delta) {
4140
- let total = 0;
3999
+ let winner = selected;
4000
+ let weight = targets[winner];
4141
4001
  for (let column = 1; column < 6; column += 1) {
4142
- total += targets[column];
4143
- }
4144
- targets[0] = Math.max(0, 1 - total);
4145
- const scale = Math.max(1, total);
4146
- let distance = 0;
4147
- for (let column = 0; column < 6; column += 1) {
4148
- targets[column] = targets[column] / scale;
4149
- distance = Math.max(
4150
- distance,
4151
- Math.abs(targets[column] - weights[column])
4152
- );
4153
- }
4154
- const step = Number.isFinite(delta) ? Math.max(0, delta) / 0.08 : 0;
4155
- const factor = distance > 0 ? Math.min(1, step / distance) : 1;
4156
- for (let column = 0; column < 6; column += 1) {
4157
- weights[column] = weights[column] + (targets[column] - weights[column]) * factor;
4002
+ if (targets[column] > weight) {
4003
+ winner = column;
4004
+ weight = targets[column];
4005
+ }
4158
4006
  }
4159
- output.update(weights);
4007
+ selected = weight > 0 ? winner : 0;
4008
+ mouths.find(({ column }) => column === selected)?.bind.applyWeight(1);
4160
4009
  originalUpdate.call(material, delta);
4161
4010
  };
4162
4011
  const dispose = function() {
4163
4012
  material.removeEventListener("dispose", dispose);
4164
4013
  material.update = originalUpdate;
4165
- output.dispose();
4166
4014
  };
4167
4015
  material.addEventListener("dispose", dispose);
4168
4016
  }
4169
4017
  };
4170
4018
 
4019
+ // ../vrm-renderer/src/generated-character.ts
4020
+ var TOLERANCE = 1e-4;
4021
+ var prepared = /* @__PURE__ */ new WeakMap();
4022
+ var sameColour = function(a, b) {
4023
+ return Math.abs(a.r - b.r) < TOLERANCE && Math.abs(a.g - b.g) < TOLERANCE && Math.abs(a.b - b.b) < TOLERANCE;
4024
+ };
4025
+ var sceneMToonMaterials = function(scene) {
4026
+ const materials = /* @__PURE__ */ new Set();
4027
+ scene.traverse((object) => {
4028
+ const material = object.material;
4029
+ for (const entry of Array.isArray(material) ? material : [material]) {
4030
+ if (entry instanceof MToonMaterial2) {
4031
+ materials.add(entry);
4032
+ }
4033
+ }
4034
+ });
4035
+ return materials;
4036
+ };
4037
+ var sharesFaceRecipe = function(material, faces) {
4038
+ return !!material.map && material.shadeMultiplyTexture?.image === material.map.image && faces.some(
4039
+ (face) => sameColour(material.color, face.color) && sameColour(material.shadeColorFactor, face.shadeColorFactor)
4040
+ );
4041
+ };
4042
+ var detectGeneratedCharacter = function(vrm) {
4043
+ const known = prepared.get(vrm);
4044
+ if (known) {
4045
+ return known;
4046
+ }
4047
+ const mouths = findAtlasBindings(vrm);
4048
+ if (!mouths.length) {
4049
+ return null;
4050
+ }
4051
+ const faces = [
4052
+ ...new Set(mouths.map(({ bind }) => bind.material))
4053
+ ];
4054
+ const coloured = /* @__PURE__ */ new Set();
4055
+ for (const expression of vrm.expressionManager?.expressions ?? []) {
4056
+ for (const bind of expression.binds) {
4057
+ if (bind instanceof VRMExpressionMaterialColorBind) {
4058
+ coloured.add(bind.material);
4059
+ }
4060
+ }
4061
+ }
4062
+ const materials = [...sceneMToonMaterials(vrm.scene)].filter(
4063
+ (material) => !coloured.has(material) && (faces.includes(material) || sharesFaceRecipe(material, faces))
4064
+ );
4065
+ return { faces, mouths, materials };
4066
+ };
4067
+ var normaliseLitFactor = function(material) {
4068
+ const { r, g, b } = material.color;
4069
+ if (!(r > 0) || r >= 1 - TOLERANCE || Math.abs(r - g) >= TOLERANCE || Math.abs(r - b) >= TOLERANCE) {
4070
+ return;
4071
+ }
4072
+ material.color.multiplyScalar(1 / r);
4073
+ material.shadeColorFactor.multiplyScalar(1 / r);
4074
+ };
4075
+ var prepareGeneratedCharacter = function(vrm) {
4076
+ const character = detectGeneratedCharacter(vrm);
4077
+ if (!character || prepared.has(vrm)) {
4078
+ return character;
4079
+ }
4080
+ for (const material of character.materials) {
4081
+ normaliseLitFactor(material);
4082
+ }
4083
+ installTextureMouthPlayback(vrm, character.mouths);
4084
+ prepared.set(vrm, character);
4085
+ return character;
4086
+ };
4087
+
4171
4088
  // ../vrm-renderer/src/session.ts
4172
4089
  import { VRMUtils as VRMUtils2 } from "@pixiv/three-vrm";
4173
4090
  import { AnimationMixer, LoopOnce, LoopRepeat } from "three";
@@ -4492,7 +4409,7 @@ var loadVrm = async ({
4492
4409
  });
4493
4410
  VRMUtils4.removeUnnecessaryVertices(gltf.scene);
4494
4411
  VRMUtils4.combineSkeletons(gltf.scene);
4495
- enableTextureExpressionBlending(gltf.userData.vrm);
4412
+ prepareGeneratedCharacter(gltf.userData.vrm);
4496
4413
  return gltf.userData.vrm;
4497
4414
  };
4498
4415
  var loadVrma = async ({
@@ -4624,6 +4541,8 @@ var AnimationController = class {
4624
4541
  mixer;
4625
4542
  clips = /* @__PURE__ */ new Map();
4626
4543
  configs = /* @__PURE__ */ new Map();
4544
+ previewMixer = null;
4545
+ restorePreview = null;
4627
4546
  activeAction = null;
4628
4547
  idleAnimationId;
4629
4548
  pendingReturnToIdleAction = null;
@@ -4638,11 +4557,35 @@ var AnimationController = class {
4638
4557
  weight: config?.weight ?? defaultAnimationConfig.weight
4639
4558
  });
4640
4559
  }
4560
+ previewAnimation(animation) {
4561
+ this.restorePreview?.();
4562
+ const clip = createVRMAnimationClip(animation, this.vrm);
4563
+ const previewMixer = new AnimationMixer2(this.vrm.scene);
4564
+ previewMixer.clipAction(clip).setLoop(LoopRepeat2, Infinity).reset().play();
4565
+ this.previewMixer = previewMixer;
4566
+ const restore = () => {
4567
+ if (this.restorePreview !== restore) {
4568
+ return;
4569
+ }
4570
+ this.restorePreview = null;
4571
+ previewMixer.stopAllAction();
4572
+ previewMixer.uncacheRoot(this.vrm.scene);
4573
+ this.previewMixer = null;
4574
+ if (!this.activeAction) {
4575
+ this.vrm.humanoid.resetNormalizedPose();
4576
+ }
4577
+ this.mixer.update(0);
4578
+ };
4579
+ this.restorePreview = restore;
4580
+ previewMixer.update(0);
4581
+ return restore;
4582
+ }
4641
4583
  setIdleAnimation(animationId) {
4642
4584
  this.idleAnimationId = animationId;
4643
4585
  this.playIdle();
4644
4586
  }
4645
4587
  clearAnimations() {
4588
+ this.restorePreview?.();
4646
4589
  this.mixer.stopAllAction();
4647
4590
  this.mixer.uncacheRoot(this.vrm.scene);
4648
4591
  this.clips.clear();
@@ -4653,6 +4596,7 @@ var AnimationController = class {
4653
4596
  this.vrm.humanoid.resetNormalizedPose();
4654
4597
  }
4655
4598
  playAnimation(animationId, fadeDuration = 0.25, loopOverride) {
4599
+ this.restorePreview?.();
4656
4600
  const clip = this.clips.get(animationId);
4657
4601
  if (!clip) {
4658
4602
  throw new Error(`Animation ${animationId} not found`);
@@ -4686,6 +4630,7 @@ var AnimationController = class {
4686
4630
  }
4687
4631
  }
4688
4632
  playIdle(fadeDuration = 0.25) {
4633
+ this.restorePreview?.();
4689
4634
  if (!this.idleAnimationId || !this.clips.has(this.idleAnimationId)) {
4690
4635
  this.mixer.stopAllAction();
4691
4636
  this.activeAction = null;
@@ -4699,6 +4644,7 @@ var AnimationController = class {
4699
4644
  this.playAnimation(this.idleAnimationId, fadeDuration, true);
4700
4645
  }
4701
4646
  stopAnimation(fadeDuration = 0.25) {
4647
+ this.restorePreview?.();
4702
4648
  if (!this.activeAction) {
4703
4649
  return;
4704
4650
  }
@@ -4707,7 +4653,8 @@ var AnimationController = class {
4707
4653
  this.pendingReturnToIdleAction = null;
4708
4654
  }
4709
4655
  update(delta) {
4710
- this.mixer.update(delta);
4656
+ ;
4657
+ (this.previewMixer ?? this.mixer).update(delta);
4711
4658
  }
4712
4659
  handleActionFinished = (event) => {
4713
4660
  if (!this.idleAnimationId || !this.pendingReturnToIdleAction || event.action !== this.pendingReturnToIdleAction || !this.clips.has(this.idleAnimationId)) {
@@ -5359,6 +5306,7 @@ var PerformanceWeightMixer = class {
5359
5306
  ...definition,
5360
5307
  id,
5361
5308
  responseRate: normalizeResponseRate(definition.responseRate),
5309
+ transition: definition.transition ?? "smooth",
5362
5310
  weight
5363
5311
  },
5364
5312
  order: this.nextLayerOrder,
@@ -5435,12 +5383,13 @@ var PerformanceWeightMixer = class {
5435
5383
  const resolved = /* @__PURE__ */ new Map();
5436
5384
  this.orderedLayers.forEach((layer) => {
5437
5385
  const responseRate = layer.definition.responseRate;
5386
+ const frameDelta = layer.definition.transition === "step" ? 0 : delta;
5438
5387
  layer.value = clamp01(
5439
- smoothValue(layer.value, layer.target, responseRate, delta)
5388
+ smoothValue(layer.value, layer.target, responseRate, frameDelta)
5440
5389
  );
5441
5390
  layer.channels.forEach((channel, name) => {
5442
5391
  channel.value = clamp01(
5443
- smoothValue(channel.value, channel.target, responseRate, delta)
5392
+ smoothValue(channel.value, channel.target, responseRate, frameDelta)
5444
5393
  );
5445
5394
  const current = resolved.get(name) ?? 0;
5446
5395
  const contribution = channel.value * layer.value;
@@ -5566,6 +5515,9 @@ var FacialPerformanceMixer = class {
5566
5515
  getVisemeChannelNames() {
5567
5516
  return this.getLayerChannels(FacialPerformanceLayer.Viseme);
5568
5517
  }
5518
+ get usesDiscreteMouths() {
5519
+ return hasTextureMouthExpressions(this.vrm?.expressionManager);
5520
+ }
5569
5521
  playCue(channel, options) {
5570
5522
  if (!this.availableChannelSet.has(channel)) {
5571
5523
  return;
@@ -5726,7 +5678,8 @@ var FacialPerformanceMixer = class {
5726
5678
  this.weightMixer.registerLayer({
5727
5679
  id: FacialPerformanceLayer.Viseme,
5728
5680
  priority: 200,
5729
- blendMode: "override"
5681
+ blendMode: "override",
5682
+ transition: this.usesDiscreteMouths ? "step" : "smooth"
5730
5683
  });
5731
5684
  this.weightMixer.registerLayer({
5732
5685
  id: FacialPerformanceLayer.Blink,
@@ -6475,6 +6428,7 @@ var VisemePlayer = class {
6475
6428
  phase = "idle";
6476
6429
  running = false;
6477
6430
  timelineRevision = 0;
6431
+ heldMouths = new HeldMouthTimeline();
6478
6432
  attachAudioElement(element) {
6479
6433
  if (this.audioElement === element) {
6480
6434
  return;
@@ -6509,6 +6463,7 @@ var VisemePlayer = class {
6509
6463
  this.rawVisemes = visemes.map((viseme) => ({ ...viseme }));
6510
6464
  this.normalizedTimeline = normalizeVisemeTimeline(this.rawVisemes);
6511
6465
  this.visemes = this.normalizedTimeline.entries;
6466
+ this.heldMouths.setCues(this.visemes);
6512
6467
  this.timelineRevision += 1;
6513
6468
  this.resetPlaybackState();
6514
6469
  this.recordTimeline();
@@ -6570,6 +6525,10 @@ var VisemePlayer = class {
6570
6525
  if (currentTime === null) {
6571
6526
  return;
6572
6527
  }
6528
+ if (this.facialMixer.usesDiscreteMouths) {
6529
+ this.updateHeldMouth(currentTime);
6530
+ return;
6531
+ }
6573
6532
  const dueCueIndices = this.collectDueCueIndices(currentTime);
6574
6533
  const latestDueCueIndex = dueCueIndices.at(-1);
6575
6534
  dueCueIndices.slice(0, -1).forEach((cueIndex) => {
@@ -6600,6 +6559,34 @@ var VisemePlayer = class {
6600
6559
  state: "playing"
6601
6560
  });
6602
6561
  };
6562
+ updateHeldMouth(currentTime) {
6563
+ const duration = this.audioElement?.duration;
6564
+ const speechEnd = Number.isFinite(duration) ? duration : void 0;
6565
+ const cue = this.heldMouths.sample(currentTime, speechEnd);
6566
+ if (!cue) {
6567
+ if (this.activeCueIndex !== null) {
6568
+ this.releaseToNeutral("scheduled");
6569
+ }
6570
+ return;
6571
+ }
6572
+ if (cue === this.visemes[this.activeCueIndex ?? -1]) {
6573
+ return;
6574
+ }
6575
+ const index = this.visemes.indexOf(cue);
6576
+ this.facialMixer.playCue(cue.name, { weight: 1, durationMs: 0 });
6577
+ this.activeCueIndex = index;
6578
+ this.nextVisemeIndex = index + 1;
6579
+ this.phase = "open";
6580
+ this.recordCueDecision(index, currentTime, "played");
6581
+ }
6582
+ cueDuration(cue, next) {
6583
+ if (this.facialMixer.usesDiscreteMouths) {
6584
+ const duration = this.audioElement?.duration;
6585
+ const speechEnd = Number.isFinite(duration) ? duration : void 0;
6586
+ return (HeldMouthTimeline.endTime(cue, next, speechEnd) - cue.timestamp) * 1e3;
6587
+ }
6588
+ return getVisemeDurationMs(cue, next);
6589
+ }
6603
6590
  handleEnded = () => {
6604
6591
  this.recorder?.recordEvent({
6605
6592
  type: "audio.state",
@@ -6728,7 +6715,7 @@ var VisemePlayer = class {
6728
6715
  if (!cue) {
6729
6716
  return;
6730
6717
  }
6731
- const openDurationMs = getVisemeDurationMs(cue, this.visemes[cueIndex + 1]);
6718
+ const openDurationMs = this.cueDuration(cue, this.visemes[cueIndex + 1]);
6732
6719
  const elapsedMs = Math.max(currentTime - cue.timestamp, 0) * 1e3;
6733
6720
  this.recorder?.recordEvent({
6734
6721
  type: "viseme.cue",
@@ -6773,6 +6760,7 @@ var VisemePlayer = class {
6773
6760
  this.audioElement.removeEventListener("ended", this.handleEnded);
6774
6761
  }
6775
6762
  resetPlaybackState() {
6763
+ this.heldMouths.reset();
6776
6764
  this.running = false;
6777
6765
  this.nextVisemeIndex = 0;
6778
6766
  this.activeCueIndex = null;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@npcify/web",
3
- "version": "0.1.3",
3
+ "version": "0.1.5",
4
4
  "description": "Framework-independent Npcify characters, live conversations and baked performances for the web",
5
5
  "type": "module",
6
6
  "license": "UNLICENSED",