@npcify/web 0.1.3 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +5 -0
- package/dist/index.js +148 -236
- package/package.json +3 -3
package/dist/index.d.ts
CHANGED
|
@@ -317,6 +317,7 @@ interface AssetLoadProgress {
|
|
|
317
317
|
|
|
318
318
|
interface AnimationControllerLike {
|
|
319
319
|
prepareAnimations?: (ids: string[]) => Promise<void>;
|
|
320
|
+
previewAnimation?: (animation: VRMAnimation) => () => void;
|
|
320
321
|
hasAnimation?: (id: string) => boolean;
|
|
321
322
|
playAnimation: (animationId: string, fadeDuration?: number, loopOverride?: boolean) => void;
|
|
322
323
|
playIdle: (fadeDuration?: number) => void;
|
|
@@ -560,6 +561,7 @@ declare class FacialPerformanceMixer {
|
|
|
560
561
|
getChannelNames(): string[];
|
|
561
562
|
getEmotionChannelNames(): string[];
|
|
562
563
|
getVisemeChannelNames(): string[];
|
|
564
|
+
get usesDiscreteMouths(): boolean;
|
|
563
565
|
playCue(channel: FacialChannelName, options?: FacialCueOptions): void;
|
|
564
566
|
setEmotion(channel: FacialChannelName, weight: number, mouthMode: FacialMouthMode): void;
|
|
565
567
|
clearEmotion(mouthMode?: FacialMouthMode): void;
|
|
@@ -633,6 +635,7 @@ declare class VisemePlayer {
|
|
|
633
635
|
private phase;
|
|
634
636
|
private running;
|
|
635
637
|
private timelineRevision;
|
|
638
|
+
private readonly heldMouths;
|
|
636
639
|
constructor(facialMixer: FacialPerformanceMixer, recorder?: PerformanceRecorder | undefined);
|
|
637
640
|
attachAudioElement(element: HTMLAudioElement | null): void;
|
|
638
641
|
setVisemes(visemes: ConversationTimelineEntry[]): void;
|
|
@@ -642,6 +645,8 @@ declare class VisemePlayer {
|
|
|
642
645
|
getPerformanceSnapshot(): VisemePerformanceFrameSnapshot;
|
|
643
646
|
update(_delta: number): void;
|
|
644
647
|
private readonly handlePlaying;
|
|
648
|
+
private updateHeldMouth;
|
|
649
|
+
private cueDuration;
|
|
645
650
|
private readonly handleEnded;
|
|
646
651
|
private collectDueCueIndices;
|
|
647
652
|
private shouldCloseActiveCue;
|
package/dist/index.js
CHANGED
|
@@ -3835,204 +3835,61 @@ import {
|
|
|
3835
3835
|
VRMExpressionTextureTransformBind
|
|
3836
3836
|
} from "@pixiv/three-vrm";
|
|
3837
3837
|
|
|
3838
|
-
// ../vrm-renderer/src/
|
|
3839
|
-
var
|
|
3840
|
-
var
|
|
3841
|
-
var
|
|
3842
|
-
|
|
3843
|
-
|
|
3844
|
-
|
|
3845
|
-
|
|
3846
|
-
|
|
3847
|
-
|
|
3848
|
-
|
|
3849
|
-
|
|
3850
|
-
|
|
3851
|
-
|
|
3852
|
-
|
|
3853
|
-
|
|
3854
|
-
|
|
3855
|
-
).join("\n")}
|
|
3856
|
-
return color;
|
|
3857
|
-
}
|
|
3858
|
-
`;
|
|
3859
|
-
var createMouthAtlasShader = function(material) {
|
|
3860
|
-
if (!material.fragmentShader.includes(sampleMap) || !material.fragmentShader.includes(sampleShade)) {
|
|
3861
|
-
return null;
|
|
3838
|
+
// ../vrm-renderer/src/held-mouth-timeline.ts
|
|
3839
|
+
var MinimumExposure = 0.1;
|
|
3840
|
+
var MaximumLinkedGap = 0.3;
|
|
3841
|
+
var UnlinkedExposure = 0.2;
|
|
3842
|
+
var Tolerance = 1e-6;
|
|
3843
|
+
var HeldMouthTimeline = class _HeldMouthTimeline {
|
|
3844
|
+
cues = [];
|
|
3845
|
+
next = 0;
|
|
3846
|
+
shown = null;
|
|
3847
|
+
shownAt = -Infinity;
|
|
3848
|
+
previousTime = -Infinity;
|
|
3849
|
+
static endTime(cue, next, speechEnd = Infinity) {
|
|
3850
|
+
const nextTime = next?.timestamp ?? speechEnd;
|
|
3851
|
+
return Math.min(
|
|
3852
|
+
speechEnd,
|
|
3853
|
+
nextTime - cue.timestamp <= MaximumLinkedGap + Tolerance ? nextTime : cue.timestamp + UnlinkedExposure
|
|
3854
|
+
);
|
|
3862
3855
|
}
|
|
3863
|
-
|
|
3864
|
-
|
|
3865
|
-
|
|
3866
|
-
const texel = { value: 1 / 4096 };
|
|
3867
|
-
const uniforms = material.uniforms;
|
|
3868
|
-
uniforms.npcifyMouthWeights = weights;
|
|
3869
|
-
uniforms.npcifyMouthTexelWidth = texel;
|
|
3870
|
-
material.fragmentShader = mouthShader + fragment.replace(sampleMap, "npcifySampleMouth( map, mapUv )").replace(
|
|
3871
|
-
sampleShade,
|
|
3872
|
-
"npcifySampleMouth( shadeMultiplyTexture, shadeMultiplyTextureUv )"
|
|
3873
|
-
);
|
|
3874
|
-
material.customProgramCacheKey = function() {
|
|
3875
|
-
return `${cacheKey.call(material)},npcify-mouth-atlas-v1`;
|
|
3876
|
-
};
|
|
3877
|
-
material.needsUpdate = true;
|
|
3878
|
-
return {
|
|
3879
|
-
update(values) {
|
|
3880
|
-
weights.value.set(values);
|
|
3881
|
-
const image = material.map?.image;
|
|
3882
|
-
if (image && typeof image === "object" && "width" in image && typeof image.width === "number" && image.width > 0) {
|
|
3883
|
-
texel.value = 1 / image.width;
|
|
3884
|
-
}
|
|
3885
|
-
},
|
|
3886
|
-
dispose() {
|
|
3887
|
-
material.fragmentShader = fragment;
|
|
3888
|
-
material.customProgramCacheKey = cacheKey;
|
|
3889
|
-
delete uniforms.npcifyMouthWeights;
|
|
3890
|
-
delete uniforms.npcifyMouthTexelWidth;
|
|
3891
|
-
}
|
|
3892
|
-
};
|
|
3893
|
-
};
|
|
3894
|
-
|
|
3895
|
-
// ../vrm-renderer/src/texture-expressions-pixels.ts
|
|
3896
|
-
import { DataTexture, Source as Source2, SRGBColorSpace as SRGBColorSpace2 } from "three";
|
|
3897
|
-
var rawImage = function(image) {
|
|
3898
|
-
if (!image || typeof image !== "object" || !("data" in image) || !("width" in image) || !("height" in image)) {
|
|
3899
|
-
return false;
|
|
3856
|
+
setCues(cues) {
|
|
3857
|
+
this.cues = [...cues].sort((a, b) => a.timestamp - b.timestamp);
|
|
3858
|
+
this.reset();
|
|
3900
3859
|
}
|
|
3901
|
-
|
|
3902
|
-
|
|
3903
|
-
|
|
3904
|
-
|
|
3905
|
-
|
|
3906
|
-
}
|
|
3907
|
-
|
|
3908
|
-
|
|
3909
|
-
|
|
3910
|
-
|
|
3911
|
-
);
|
|
3912
|
-
});
|
|
3913
|
-
var PixelColumns = class {
|
|
3914
|
-
constructor(original) {
|
|
3915
|
-
this.original = original;
|
|
3916
|
-
}
|
|
3917
|
-
image = null;
|
|
3918
|
-
columns = [];
|
|
3919
|
-
previous = new Float32Array(6).fill(-1);
|
|
3920
|
-
source = new Source2({ data: new Uint8Array(4), width: 1, height: 1 });
|
|
3921
|
-
update(weights) {
|
|
3922
|
-
const image = this.original.image;
|
|
3923
|
-
if (!rawImage(image)) {
|
|
3924
|
-
return false;
|
|
3860
|
+
reset() {
|
|
3861
|
+
this.next = 0;
|
|
3862
|
+
this.shown = null;
|
|
3863
|
+
this.shownAt = -Infinity;
|
|
3864
|
+
this.previousTime = -Infinity;
|
|
3865
|
+
}
|
|
3866
|
+
sample(time, speechEnd = Infinity) {
|
|
3867
|
+
if (!Number.isFinite(time) || time < 0) {
|
|
3868
|
+
this.reset();
|
|
3869
|
+
return null;
|
|
3925
3870
|
}
|
|
3926
|
-
|
|
3927
|
-
|
|
3928
|
-
this.prepare(image);
|
|
3871
|
+
if (time < this.previousTime) {
|
|
3872
|
+
this.reset();
|
|
3929
3873
|
}
|
|
3930
|
-
|
|
3931
|
-
|
|
3932
|
-
|
|
3933
|
-
return false;
|
|
3874
|
+
this.previousTime = time;
|
|
3875
|
+
while (this.cues[this.next] && this.cues[this.next].timestamp <= time + Tolerance) {
|
|
3876
|
+
this.next += 1;
|
|
3934
3877
|
}
|
|
3935
|
-
this.
|
|
3936
|
-
const
|
|
3937
|
-
const
|
|
3938
|
-
|
|
3939
|
-
|
|
3940
|
-
|
|
3941
|
-
for (let column = 0; column < active.length; column += 1) {
|
|
3942
|
-
const source = active[column];
|
|
3943
|
-
value += source.pixels[i] * source.weight;
|
|
3944
|
-
}
|
|
3945
|
-
data[i] = isSrgb && i % 4 !== 3 ? srgb[Math.max(0, Math.min(4096, Math.round(value * 4096)))] : Math.round(value * 255);
|
|
3878
|
+
const cue = this.cues[this.next - 1];
|
|
3879
|
+
const end = cue ? _HeldMouthTimeline.endTime(cue, this.cues[this.next], speechEnd) : 0;
|
|
3880
|
+
const weight = cue?.weight ?? 1;
|
|
3881
|
+
if (!cue || time + Tolerance >= end || !Number.isFinite(weight) || weight <= 0) {
|
|
3882
|
+
this.shown = null;
|
|
3883
|
+
return null;
|
|
3946
3884
|
}
|
|
3947
|
-
|
|
3948
|
-
|
|
3949
|
-
|
|
3950
|
-
this.
|
|
3951
|
-
|
|
3952
|
-
const length = width * image.height * 4;
|
|
3953
|
-
this.source.data = {
|
|
3954
|
-
data: new Uint8Array(length),
|
|
3955
|
-
width,
|
|
3956
|
-
height: image.height
|
|
3957
|
-
};
|
|
3958
|
-
const isSrgb = this.original.colorSpace === SRGBColorSpace2;
|
|
3959
|
-
this.columns = Array.from({ length: 6 }, (_, column) => {
|
|
3960
|
-
const pixels = new Float32Array(length);
|
|
3961
|
-
for (let y = 0; y < image.height; y += 1) {
|
|
3962
|
-
for (let x = 0; x < width; x += 1) {
|
|
3963
|
-
const sample = ((x + 0.5) / width + column) * image.width / 6 - 0.5;
|
|
3964
|
-
const left = Math.max(
|
|
3965
|
-
0,
|
|
3966
|
-
Math.min(image.width - 1, Math.floor(sample))
|
|
3967
|
-
);
|
|
3968
|
-
const right = Math.min(image.width - 1, left + 1);
|
|
3969
|
-
const mix = Math.max(0, sample - left);
|
|
3970
|
-
for (let channel = 0; channel < 4; channel += 1) {
|
|
3971
|
-
const a = image.data[(y * image.width + left) * 4 + channel];
|
|
3972
|
-
const b = image.data[(y * image.width + right) * 4 + channel];
|
|
3973
|
-
const convert = isSrgb && channel !== 3;
|
|
3974
|
-
const first = convert ? linear[a] : a / 255;
|
|
3975
|
-
const second = convert ? linear[b] : b / 255;
|
|
3976
|
-
pixels[(y * width + x) * 4 + channel] = first + (second - first) * mix;
|
|
3977
|
-
}
|
|
3978
|
-
}
|
|
3979
|
-
}
|
|
3980
|
-
return pixels;
|
|
3981
|
-
});
|
|
3982
|
-
}
|
|
3983
|
-
dispose() {
|
|
3984
|
-
this.columns = [];
|
|
3985
|
-
this.image = null;
|
|
3986
|
-
this.source.data = null;
|
|
3987
|
-
}
|
|
3988
|
-
};
|
|
3989
|
-
var createMouthAtlasPixels = function(material) {
|
|
3990
|
-
const properties = ["map", "shadeMultiplyTexture"];
|
|
3991
|
-
if (properties.some(
|
|
3992
|
-
(name) => material[name] && !rawImage(material[name].image)
|
|
3993
|
-
)) {
|
|
3994
|
-
return null;
|
|
3995
|
-
}
|
|
3996
|
-
const columns = /* @__PURE__ */ new Map();
|
|
3997
|
-
const textures = properties.flatMap((name) => {
|
|
3998
|
-
const original = material[name];
|
|
3999
|
-
if (!original) {
|
|
4000
|
-
return [];
|
|
3885
|
+
if (!this.shown || cue.name !== this.shown.name && time - this.shownAt + Tolerance >= MinimumExposure) {
|
|
3886
|
+
this.shown = cue;
|
|
3887
|
+
this.shownAt = time;
|
|
3888
|
+
} else if (cue.name === this.shown.name) {
|
|
3889
|
+
this.shown = cue;
|
|
4001
3890
|
}
|
|
4002
|
-
|
|
4003
|
-
|
|
4004
|
-
pixels = new PixelColumns(original);
|
|
4005
|
-
columns.set(original.colorSpace, pixels);
|
|
4006
|
-
}
|
|
4007
|
-
const texture = new DataTexture();
|
|
4008
|
-
texture.copy(original);
|
|
4009
|
-
texture.source = pixels.source;
|
|
4010
|
-
texture.generateMipmaps = false;
|
|
4011
|
-
texture.minFilter = original.magFilter;
|
|
4012
|
-
material[name] = texture;
|
|
4013
|
-
return [{ name, original, texture, pixels }];
|
|
4014
|
-
});
|
|
4015
|
-
return {
|
|
4016
|
-
update(weights) {
|
|
4017
|
-
const changed = new Set(
|
|
4018
|
-
[...columns.values()].filter((pixels) => pixels.update(weights))
|
|
4019
|
-
);
|
|
4020
|
-
for (const { texture, pixels } of textures) {
|
|
4021
|
-
texture.repeat.x = 6;
|
|
4022
|
-
if (changed.has(pixels)) {
|
|
4023
|
-
texture.needsUpdate = true;
|
|
4024
|
-
}
|
|
4025
|
-
}
|
|
4026
|
-
},
|
|
4027
|
-
dispose() {
|
|
4028
|
-
for (const { name, original, texture } of textures) {
|
|
4029
|
-
material[name] = original;
|
|
4030
|
-
texture.dispose();
|
|
4031
|
-
original.dispose();
|
|
4032
|
-
}
|
|
4033
|
-
columns.forEach((pixels) => pixels.dispose());
|
|
4034
|
-
}
|
|
4035
|
-
};
|
|
3891
|
+
return this.shown;
|
|
3892
|
+
}
|
|
4036
3893
|
};
|
|
4037
3894
|
|
|
4038
3895
|
// ../vrm-renderer/src/texture-expressions.ts
|
|
@@ -4097,72 +3954,56 @@ var findAtlasBindings = function(vrm) {
|
|
|
4097
3954
|
}
|
|
4098
3955
|
return bindings;
|
|
4099
3956
|
};
|
|
4100
|
-
var
|
|
4101
|
-
|
|
3957
|
+
var hasTextureMouthExpressions = function(manager) {
|
|
3958
|
+
return !!manager && installed.has(manager);
|
|
3959
|
+
};
|
|
3960
|
+
var enableTextureExpressionPlayback = function(vrm) {
|
|
3961
|
+
const manager = vrm.expressionManager;
|
|
3962
|
+
if (!manager || installed.has(manager)) {
|
|
4102
3963
|
return;
|
|
4103
3964
|
}
|
|
4104
3965
|
const bindings = findAtlasBindings(vrm);
|
|
4105
3966
|
if (!bindings.length) {
|
|
4106
3967
|
return;
|
|
4107
3968
|
}
|
|
4108
|
-
|
|
4109
|
-
|
|
4110
|
-
|
|
4111
|
-
const outputs = materials.map(
|
|
4112
|
-
(material) => options.backend === "pixels" ? createMouthAtlasPixels(material) : createMouthAtlasShader(material)
|
|
3969
|
+
installed.add(manager);
|
|
3970
|
+
const materials = new Set(
|
|
3971
|
+
bindings.map(({ bind }) => bind.material)
|
|
4113
3972
|
);
|
|
4114
|
-
|
|
4115
|
-
|
|
4116
|
-
return;
|
|
4117
|
-
}
|
|
4118
|
-
installed.add(vrm);
|
|
4119
|
-
for (const [index, material] of materials.entries()) {
|
|
4120
|
-
const output = outputs[index];
|
|
3973
|
+
for (const material of materials) {
|
|
3974
|
+
const mouths = bindings.filter(({ bind }) => bind.material === material);
|
|
4121
3975
|
const targets = new Float32Array(6);
|
|
4122
|
-
|
|
4123
|
-
for (const { expression, bind, column } of
|
|
4124
|
-
if (bind.material !== material) {
|
|
4125
|
-
continue;
|
|
4126
|
-
}
|
|
3976
|
+
let selected = 0;
|
|
3977
|
+
for (const { expression, bind, column } of mouths) {
|
|
4127
3978
|
expression.isBinary = false;
|
|
4128
3979
|
expression.deleteBind(bind);
|
|
4129
3980
|
expression.addBind({
|
|
4130
3981
|
clearAppliedWeight: () => {
|
|
4131
3982
|
targets[column] = 0;
|
|
3983
|
+
bind.clearAppliedWeight();
|
|
4132
3984
|
},
|
|
4133
3985
|
applyWeight: (weight) => {
|
|
4134
|
-
targets[column] = Number.isFinite(weight) ? Math.max(0,
|
|
3986
|
+
targets[column] = Number.isFinite(weight) ? Math.max(0, weight) : 0;
|
|
4135
3987
|
}
|
|
4136
3988
|
});
|
|
4137
3989
|
}
|
|
4138
3990
|
const originalUpdate = material.update;
|
|
4139
3991
|
material.update = function(delta) {
|
|
4140
|
-
let
|
|
3992
|
+
let winner = selected;
|
|
3993
|
+
let weight = targets[winner];
|
|
4141
3994
|
for (let column = 1; column < 6; column += 1) {
|
|
4142
|
-
|
|
4143
|
-
|
|
4144
|
-
|
|
4145
|
-
|
|
4146
|
-
let distance = 0;
|
|
4147
|
-
for (let column = 0; column < 6; column += 1) {
|
|
4148
|
-
targets[column] = targets[column] / scale;
|
|
4149
|
-
distance = Math.max(
|
|
4150
|
-
distance,
|
|
4151
|
-
Math.abs(targets[column] - weights[column])
|
|
4152
|
-
);
|
|
4153
|
-
}
|
|
4154
|
-
const step = Number.isFinite(delta) ? Math.max(0, delta) / 0.08 : 0;
|
|
4155
|
-
const factor = distance > 0 ? Math.min(1, step / distance) : 1;
|
|
4156
|
-
for (let column = 0; column < 6; column += 1) {
|
|
4157
|
-
weights[column] = weights[column] + (targets[column] - weights[column]) * factor;
|
|
3995
|
+
if (targets[column] > weight) {
|
|
3996
|
+
winner = column;
|
|
3997
|
+
weight = targets[column];
|
|
3998
|
+
}
|
|
4158
3999
|
}
|
|
4159
|
-
|
|
4000
|
+
selected = weight > 0 ? winner : 0;
|
|
4001
|
+
mouths.find(({ column }) => column === selected)?.bind.applyWeight(1);
|
|
4160
4002
|
originalUpdate.call(material, delta);
|
|
4161
4003
|
};
|
|
4162
4004
|
const dispose = function() {
|
|
4163
4005
|
material.removeEventListener("dispose", dispose);
|
|
4164
4006
|
material.update = originalUpdate;
|
|
4165
|
-
output.dispose();
|
|
4166
4007
|
};
|
|
4167
4008
|
material.addEventListener("dispose", dispose);
|
|
4168
4009
|
}
|
|
@@ -4492,7 +4333,7 @@ var loadVrm = async ({
|
|
|
4492
4333
|
});
|
|
4493
4334
|
VRMUtils4.removeUnnecessaryVertices(gltf.scene);
|
|
4494
4335
|
VRMUtils4.combineSkeletons(gltf.scene);
|
|
4495
|
-
|
|
4336
|
+
enableTextureExpressionPlayback(gltf.userData.vrm);
|
|
4496
4337
|
return gltf.userData.vrm;
|
|
4497
4338
|
};
|
|
4498
4339
|
var loadVrma = async ({
|
|
@@ -4624,6 +4465,8 @@ var AnimationController = class {
|
|
|
4624
4465
|
mixer;
|
|
4625
4466
|
clips = /* @__PURE__ */ new Map();
|
|
4626
4467
|
configs = /* @__PURE__ */ new Map();
|
|
4468
|
+
previewMixer = null;
|
|
4469
|
+
restorePreview = null;
|
|
4627
4470
|
activeAction = null;
|
|
4628
4471
|
idleAnimationId;
|
|
4629
4472
|
pendingReturnToIdleAction = null;
|
|
@@ -4638,11 +4481,35 @@ var AnimationController = class {
|
|
|
4638
4481
|
weight: config?.weight ?? defaultAnimationConfig.weight
|
|
4639
4482
|
});
|
|
4640
4483
|
}
|
|
4484
|
+
previewAnimation(animation) {
|
|
4485
|
+
this.restorePreview?.();
|
|
4486
|
+
const clip = createVRMAnimationClip(animation, this.vrm);
|
|
4487
|
+
const previewMixer = new AnimationMixer2(this.vrm.scene);
|
|
4488
|
+
previewMixer.clipAction(clip).setLoop(LoopRepeat2, Infinity).reset().play();
|
|
4489
|
+
this.previewMixer = previewMixer;
|
|
4490
|
+
const restore = () => {
|
|
4491
|
+
if (this.restorePreview !== restore) {
|
|
4492
|
+
return;
|
|
4493
|
+
}
|
|
4494
|
+
this.restorePreview = null;
|
|
4495
|
+
previewMixer.stopAllAction();
|
|
4496
|
+
previewMixer.uncacheRoot(this.vrm.scene);
|
|
4497
|
+
this.previewMixer = null;
|
|
4498
|
+
if (!this.activeAction) {
|
|
4499
|
+
this.vrm.humanoid.resetNormalizedPose();
|
|
4500
|
+
}
|
|
4501
|
+
this.mixer.update(0);
|
|
4502
|
+
};
|
|
4503
|
+
this.restorePreview = restore;
|
|
4504
|
+
previewMixer.update(0);
|
|
4505
|
+
return restore;
|
|
4506
|
+
}
|
|
4641
4507
|
setIdleAnimation(animationId) {
|
|
4642
4508
|
this.idleAnimationId = animationId;
|
|
4643
4509
|
this.playIdle();
|
|
4644
4510
|
}
|
|
4645
4511
|
clearAnimations() {
|
|
4512
|
+
this.restorePreview?.();
|
|
4646
4513
|
this.mixer.stopAllAction();
|
|
4647
4514
|
this.mixer.uncacheRoot(this.vrm.scene);
|
|
4648
4515
|
this.clips.clear();
|
|
@@ -4653,6 +4520,7 @@ var AnimationController = class {
|
|
|
4653
4520
|
this.vrm.humanoid.resetNormalizedPose();
|
|
4654
4521
|
}
|
|
4655
4522
|
playAnimation(animationId, fadeDuration = 0.25, loopOverride) {
|
|
4523
|
+
this.restorePreview?.();
|
|
4656
4524
|
const clip = this.clips.get(animationId);
|
|
4657
4525
|
if (!clip) {
|
|
4658
4526
|
throw new Error(`Animation ${animationId} not found`);
|
|
@@ -4686,6 +4554,7 @@ var AnimationController = class {
|
|
|
4686
4554
|
}
|
|
4687
4555
|
}
|
|
4688
4556
|
playIdle(fadeDuration = 0.25) {
|
|
4557
|
+
this.restorePreview?.();
|
|
4689
4558
|
if (!this.idleAnimationId || !this.clips.has(this.idleAnimationId)) {
|
|
4690
4559
|
this.mixer.stopAllAction();
|
|
4691
4560
|
this.activeAction = null;
|
|
@@ -4699,6 +4568,7 @@ var AnimationController = class {
|
|
|
4699
4568
|
this.playAnimation(this.idleAnimationId, fadeDuration, true);
|
|
4700
4569
|
}
|
|
4701
4570
|
stopAnimation(fadeDuration = 0.25) {
|
|
4571
|
+
this.restorePreview?.();
|
|
4702
4572
|
if (!this.activeAction) {
|
|
4703
4573
|
return;
|
|
4704
4574
|
}
|
|
@@ -4707,7 +4577,8 @@ var AnimationController = class {
|
|
|
4707
4577
|
this.pendingReturnToIdleAction = null;
|
|
4708
4578
|
}
|
|
4709
4579
|
update(delta) {
|
|
4710
|
-
|
|
4580
|
+
;
|
|
4581
|
+
(this.previewMixer ?? this.mixer).update(delta);
|
|
4711
4582
|
}
|
|
4712
4583
|
handleActionFinished = (event) => {
|
|
4713
4584
|
if (!this.idleAnimationId || !this.pendingReturnToIdleAction || event.action !== this.pendingReturnToIdleAction || !this.clips.has(this.idleAnimationId)) {
|
|
@@ -5359,6 +5230,7 @@ var PerformanceWeightMixer = class {
|
|
|
5359
5230
|
...definition,
|
|
5360
5231
|
id,
|
|
5361
5232
|
responseRate: normalizeResponseRate(definition.responseRate),
|
|
5233
|
+
transition: definition.transition ?? "smooth",
|
|
5362
5234
|
weight
|
|
5363
5235
|
},
|
|
5364
5236
|
order: this.nextLayerOrder,
|
|
@@ -5435,12 +5307,13 @@ var PerformanceWeightMixer = class {
|
|
|
5435
5307
|
const resolved = /* @__PURE__ */ new Map();
|
|
5436
5308
|
this.orderedLayers.forEach((layer) => {
|
|
5437
5309
|
const responseRate = layer.definition.responseRate;
|
|
5310
|
+
const frameDelta = layer.definition.transition === "step" ? 0 : delta;
|
|
5438
5311
|
layer.value = clamp01(
|
|
5439
|
-
smoothValue(layer.value, layer.target, responseRate,
|
|
5312
|
+
smoothValue(layer.value, layer.target, responseRate, frameDelta)
|
|
5440
5313
|
);
|
|
5441
5314
|
layer.channels.forEach((channel, name) => {
|
|
5442
5315
|
channel.value = clamp01(
|
|
5443
|
-
smoothValue(channel.value, channel.target, responseRate,
|
|
5316
|
+
smoothValue(channel.value, channel.target, responseRate, frameDelta)
|
|
5444
5317
|
);
|
|
5445
5318
|
const current = resolved.get(name) ?? 0;
|
|
5446
5319
|
const contribution = channel.value * layer.value;
|
|
@@ -5566,6 +5439,9 @@ var FacialPerformanceMixer = class {
|
|
|
5566
5439
|
getVisemeChannelNames() {
|
|
5567
5440
|
return this.getLayerChannels(FacialPerformanceLayer.Viseme);
|
|
5568
5441
|
}
|
|
5442
|
+
get usesDiscreteMouths() {
|
|
5443
|
+
return hasTextureMouthExpressions(this.vrm?.expressionManager);
|
|
5444
|
+
}
|
|
5569
5445
|
playCue(channel, options) {
|
|
5570
5446
|
if (!this.availableChannelSet.has(channel)) {
|
|
5571
5447
|
return;
|
|
@@ -5726,7 +5602,8 @@ var FacialPerformanceMixer = class {
|
|
|
5726
5602
|
this.weightMixer.registerLayer({
|
|
5727
5603
|
id: FacialPerformanceLayer.Viseme,
|
|
5728
5604
|
priority: 200,
|
|
5729
|
-
blendMode: "override"
|
|
5605
|
+
blendMode: "override",
|
|
5606
|
+
transition: this.usesDiscreteMouths ? "step" : "smooth"
|
|
5730
5607
|
});
|
|
5731
5608
|
this.weightMixer.registerLayer({
|
|
5732
5609
|
id: FacialPerformanceLayer.Blink,
|
|
@@ -6475,6 +6352,7 @@ var VisemePlayer = class {
|
|
|
6475
6352
|
phase = "idle";
|
|
6476
6353
|
running = false;
|
|
6477
6354
|
timelineRevision = 0;
|
|
6355
|
+
heldMouths = new HeldMouthTimeline();
|
|
6478
6356
|
attachAudioElement(element) {
|
|
6479
6357
|
if (this.audioElement === element) {
|
|
6480
6358
|
return;
|
|
@@ -6509,6 +6387,7 @@ var VisemePlayer = class {
|
|
|
6509
6387
|
this.rawVisemes = visemes.map((viseme) => ({ ...viseme }));
|
|
6510
6388
|
this.normalizedTimeline = normalizeVisemeTimeline(this.rawVisemes);
|
|
6511
6389
|
this.visemes = this.normalizedTimeline.entries;
|
|
6390
|
+
this.heldMouths.setCues(this.visemes);
|
|
6512
6391
|
this.timelineRevision += 1;
|
|
6513
6392
|
this.resetPlaybackState();
|
|
6514
6393
|
this.recordTimeline();
|
|
@@ -6570,6 +6449,10 @@ var VisemePlayer = class {
|
|
|
6570
6449
|
if (currentTime === null) {
|
|
6571
6450
|
return;
|
|
6572
6451
|
}
|
|
6452
|
+
if (this.facialMixer.usesDiscreteMouths) {
|
|
6453
|
+
this.updateHeldMouth(currentTime);
|
|
6454
|
+
return;
|
|
6455
|
+
}
|
|
6573
6456
|
const dueCueIndices = this.collectDueCueIndices(currentTime);
|
|
6574
6457
|
const latestDueCueIndex = dueCueIndices.at(-1);
|
|
6575
6458
|
dueCueIndices.slice(0, -1).forEach((cueIndex) => {
|
|
@@ -6600,6 +6483,34 @@ var VisemePlayer = class {
|
|
|
6600
6483
|
state: "playing"
|
|
6601
6484
|
});
|
|
6602
6485
|
};
|
|
6486
|
+
updateHeldMouth(currentTime) {
|
|
6487
|
+
const duration = this.audioElement?.duration;
|
|
6488
|
+
const speechEnd = Number.isFinite(duration) ? duration : void 0;
|
|
6489
|
+
const cue = this.heldMouths.sample(currentTime, speechEnd);
|
|
6490
|
+
if (!cue) {
|
|
6491
|
+
if (this.activeCueIndex !== null) {
|
|
6492
|
+
this.releaseToNeutral("scheduled");
|
|
6493
|
+
}
|
|
6494
|
+
return;
|
|
6495
|
+
}
|
|
6496
|
+
if (cue === this.visemes[this.activeCueIndex ?? -1]) {
|
|
6497
|
+
return;
|
|
6498
|
+
}
|
|
6499
|
+
const index = this.visemes.indexOf(cue);
|
|
6500
|
+
this.facialMixer.playCue(cue.name, { weight: 1, durationMs: 0 });
|
|
6501
|
+
this.activeCueIndex = index;
|
|
6502
|
+
this.nextVisemeIndex = index + 1;
|
|
6503
|
+
this.phase = "open";
|
|
6504
|
+
this.recordCueDecision(index, currentTime, "played");
|
|
6505
|
+
}
|
|
6506
|
+
cueDuration(cue, next) {
|
|
6507
|
+
if (this.facialMixer.usesDiscreteMouths) {
|
|
6508
|
+
const duration = this.audioElement?.duration;
|
|
6509
|
+
const speechEnd = Number.isFinite(duration) ? duration : void 0;
|
|
6510
|
+
return (HeldMouthTimeline.endTime(cue, next, speechEnd) - cue.timestamp) * 1e3;
|
|
6511
|
+
}
|
|
6512
|
+
return getVisemeDurationMs(cue, next);
|
|
6513
|
+
}
|
|
6603
6514
|
handleEnded = () => {
|
|
6604
6515
|
this.recorder?.recordEvent({
|
|
6605
6516
|
type: "audio.state",
|
|
@@ -6728,7 +6639,7 @@ var VisemePlayer = class {
|
|
|
6728
6639
|
if (!cue) {
|
|
6729
6640
|
return;
|
|
6730
6641
|
}
|
|
6731
|
-
const openDurationMs =
|
|
6642
|
+
const openDurationMs = this.cueDuration(cue, this.visemes[cueIndex + 1]);
|
|
6732
6643
|
const elapsedMs = Math.max(currentTime - cue.timestamp, 0) * 1e3;
|
|
6733
6644
|
this.recorder?.recordEvent({
|
|
6734
6645
|
type: "viseme.cue",
|
|
@@ -6773,6 +6684,7 @@ var VisemePlayer = class {
|
|
|
6773
6684
|
this.audioElement.removeEventListener("ended", this.handleEnded);
|
|
6774
6685
|
}
|
|
6775
6686
|
resetPlaybackState() {
|
|
6687
|
+
this.heldMouths.reset();
|
|
6776
6688
|
this.running = false;
|
|
6777
6689
|
this.nextVisemeIndex = 0;
|
|
6778
6690
|
this.activeCueIndex = null;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@npcify/web",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.4",
|
|
4
4
|
"description": "Framework-independent Npcify characters, live conversations and baked performances for the web",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "UNLICENSED",
|
|
@@ -35,8 +35,8 @@
|
|
|
35
35
|
"tsup": "^8.5.0",
|
|
36
36
|
"typescript": "^5.9.3",
|
|
37
37
|
"@npcify/shared": "^0.1.0",
|
|
38
|
-
"@npcify/
|
|
39
|
-
"@npcify/
|
|
38
|
+
"@npcify/web-client": "^0.0.0",
|
|
39
|
+
"@npcify/vrm-renderer": "^0.1.0"
|
|
40
40
|
},
|
|
41
41
|
"repository": {
|
|
42
42
|
"type": "git",
|