@npcify/web 0.1.3 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +5 -0
- package/dist/index.js +231 -243
- package/package.json +1 -1
package/dist/index.d.ts
CHANGED
|
@@ -317,6 +317,7 @@ interface AssetLoadProgress {
|
|
|
317
317
|
|
|
318
318
|
interface AnimationControllerLike {
|
|
319
319
|
prepareAnimations?: (ids: string[]) => Promise<void>;
|
|
320
|
+
previewAnimation?: (animation: VRMAnimation) => () => void;
|
|
320
321
|
hasAnimation?: (id: string) => boolean;
|
|
321
322
|
playAnimation: (animationId: string, fadeDuration?: number, loopOverride?: boolean) => void;
|
|
322
323
|
playIdle: (fadeDuration?: number) => void;
|
|
@@ -560,6 +561,7 @@ declare class FacialPerformanceMixer {
|
|
|
560
561
|
getChannelNames(): string[];
|
|
561
562
|
getEmotionChannelNames(): string[];
|
|
562
563
|
getVisemeChannelNames(): string[];
|
|
564
|
+
get usesDiscreteMouths(): boolean;
|
|
563
565
|
playCue(channel: FacialChannelName, options?: FacialCueOptions): void;
|
|
564
566
|
setEmotion(channel: FacialChannelName, weight: number, mouthMode: FacialMouthMode): void;
|
|
565
567
|
clearEmotion(mouthMode?: FacialMouthMode): void;
|
|
@@ -633,6 +635,7 @@ declare class VisemePlayer {
|
|
|
633
635
|
private phase;
|
|
634
636
|
private running;
|
|
635
637
|
private timelineRevision;
|
|
638
|
+
private readonly heldMouths;
|
|
636
639
|
constructor(facialMixer: FacialPerformanceMixer, recorder?: PerformanceRecorder | undefined);
|
|
637
640
|
attachAudioElement(element: HTMLAudioElement | null): void;
|
|
638
641
|
setVisemes(visemes: ConversationTimelineEntry[]): void;
|
|
@@ -642,6 +645,8 @@ declare class VisemePlayer {
|
|
|
642
645
|
getPerformanceSnapshot(): VisemePerformanceFrameSnapshot;
|
|
643
646
|
update(_delta: number): void;
|
|
644
647
|
private readonly handlePlaying;
|
|
648
|
+
private updateHeldMouth;
|
|
649
|
+
private cueDuration;
|
|
645
650
|
private readonly handleEnded;
|
|
646
651
|
private collectDueCueIndices;
|
|
647
652
|
private shouldCloseActiveCue;
|
package/dist/index.js
CHANGED
|
@@ -3477,10 +3477,15 @@ var AVATAR_COLOR_SETTINGS = {
|
|
|
3477
3477
|
toneMapping: NoToneMapping,
|
|
3478
3478
|
outputColorSpace: SRGBColorSpace
|
|
3479
3479
|
};
|
|
3480
|
+
var AVATAR_LIGHTS = {
|
|
3481
|
+
ambientIntensity: 0.7 * Math.PI / 1.9,
|
|
3482
|
+
directionalIntensity: 1.2 * Math.PI / 1.9,
|
|
3483
|
+
directionalPosition: [1, 1.5, 1]
|
|
3484
|
+
};
|
|
3480
3485
|
var addAvatarLights = function(scene, {
|
|
3481
|
-
ambientIntensity =
|
|
3482
|
-
directionalIntensity =
|
|
3483
|
-
directionalPosition = [
|
|
3486
|
+
ambientIntensity = AVATAR_LIGHTS.ambientIntensity,
|
|
3487
|
+
directionalIntensity = AVATAR_LIGHTS.directionalIntensity,
|
|
3488
|
+
directionalPosition = [...AVATAR_LIGHTS.directionalPosition]
|
|
3484
3489
|
} = {}) {
|
|
3485
3490
|
scene.add(new AmbientLight(16777215, ambientIntensity));
|
|
3486
3491
|
const key = new DirectionalLight(16777215, directionalIntensity);
|
|
@@ -3829,210 +3834,73 @@ import {
|
|
|
3829
3834
|
Texture as Texture2
|
|
3830
3835
|
} from "three";
|
|
3831
3836
|
|
|
3837
|
+
// ../vrm-renderer/src/generated-character.ts
|
|
3838
|
+
import {
|
|
3839
|
+
MToonMaterial as MToonMaterial2,
|
|
3840
|
+
VRMExpressionMaterialColorBind
|
|
3841
|
+
} from "@pixiv/three-vrm";
|
|
3842
|
+
|
|
3832
3843
|
// ../vrm-renderer/src/texture-expressions.ts
|
|
3833
3844
|
import {
|
|
3834
3845
|
MToonMaterial,
|
|
3835
3846
|
VRMExpressionTextureTransformBind
|
|
3836
3847
|
} from "@pixiv/three-vrm";
|
|
3837
3848
|
|
|
3838
|
-
// ../vrm-renderer/src/
|
|
3839
|
-
var
|
|
3840
|
-
var
|
|
3841
|
-
var
|
|
3842
|
-
|
|
3843
|
-
|
|
3844
|
-
|
|
3845
|
-
|
|
3846
|
-
|
|
3847
|
-
|
|
3848
|
-
|
|
3849
|
-
|
|
3850
|
-
|
|
3851
|
-
|
|
3852
|
-
|
|
3853
|
-
|
|
3854
|
-
|
|
3855
|
-
).join("\n")}
|
|
3856
|
-
return color;
|
|
3857
|
-
}
|
|
3858
|
-
`;
|
|
3859
|
-
var createMouthAtlasShader = function(material) {
|
|
3860
|
-
if (!material.fragmentShader.includes(sampleMap) || !material.fragmentShader.includes(sampleShade)) {
|
|
3861
|
-
return null;
|
|
3849
|
+
// ../vrm-renderer/src/held-mouth-timeline.ts
|
|
3850
|
+
var MinimumExposure = 0.1;
|
|
3851
|
+
var MaximumLinkedGap = 0.3;
|
|
3852
|
+
var UnlinkedExposure = 0.2;
|
|
3853
|
+
var Tolerance = 1e-6;
|
|
3854
|
+
var HeldMouthTimeline = class _HeldMouthTimeline {
|
|
3855
|
+
cues = [];
|
|
3856
|
+
next = 0;
|
|
3857
|
+
shown = null;
|
|
3858
|
+
shownAt = -Infinity;
|
|
3859
|
+
previousTime = -Infinity;
|
|
3860
|
+
static endTime(cue, next, speechEnd = Infinity) {
|
|
3861
|
+
const nextTime = next?.timestamp ?? speechEnd;
|
|
3862
|
+
return Math.min(
|
|
3863
|
+
speechEnd,
|
|
3864
|
+
nextTime - cue.timestamp <= MaximumLinkedGap + Tolerance ? nextTime : cue.timestamp + UnlinkedExposure
|
|
3865
|
+
);
|
|
3862
3866
|
}
|
|
3863
|
-
|
|
3864
|
-
|
|
3865
|
-
|
|
3866
|
-
const texel = { value: 1 / 4096 };
|
|
3867
|
-
const uniforms = material.uniforms;
|
|
3868
|
-
uniforms.npcifyMouthWeights = weights;
|
|
3869
|
-
uniforms.npcifyMouthTexelWidth = texel;
|
|
3870
|
-
material.fragmentShader = mouthShader + fragment.replace(sampleMap, "npcifySampleMouth( map, mapUv )").replace(
|
|
3871
|
-
sampleShade,
|
|
3872
|
-
"npcifySampleMouth( shadeMultiplyTexture, shadeMultiplyTextureUv )"
|
|
3873
|
-
);
|
|
3874
|
-
material.customProgramCacheKey = function() {
|
|
3875
|
-
return `${cacheKey.call(material)},npcify-mouth-atlas-v1`;
|
|
3876
|
-
};
|
|
3877
|
-
material.needsUpdate = true;
|
|
3878
|
-
return {
|
|
3879
|
-
update(values) {
|
|
3880
|
-
weights.value.set(values);
|
|
3881
|
-
const image = material.map?.image;
|
|
3882
|
-
if (image && typeof image === "object" && "width" in image && typeof image.width === "number" && image.width > 0) {
|
|
3883
|
-
texel.value = 1 / image.width;
|
|
3884
|
-
}
|
|
3885
|
-
},
|
|
3886
|
-
dispose() {
|
|
3887
|
-
material.fragmentShader = fragment;
|
|
3888
|
-
material.customProgramCacheKey = cacheKey;
|
|
3889
|
-
delete uniforms.npcifyMouthWeights;
|
|
3890
|
-
delete uniforms.npcifyMouthTexelWidth;
|
|
3891
|
-
}
|
|
3892
|
-
};
|
|
3893
|
-
};
|
|
3894
|
-
|
|
3895
|
-
// ../vrm-renderer/src/texture-expressions-pixels.ts
|
|
3896
|
-
import { DataTexture, Source as Source2, SRGBColorSpace as SRGBColorSpace2 } from "three";
|
|
3897
|
-
var rawImage = function(image) {
|
|
3898
|
-
if (!image || typeof image !== "object" || !("data" in image) || !("width" in image) || !("height" in image)) {
|
|
3899
|
-
return false;
|
|
3867
|
+
setCues(cues) {
|
|
3868
|
+
this.cues = [...cues].sort((a, b) => a.timestamp - b.timestamp);
|
|
3869
|
+
this.reset();
|
|
3900
3870
|
}
|
|
3901
|
-
|
|
3902
|
-
|
|
3903
|
-
|
|
3904
|
-
|
|
3905
|
-
|
|
3906
|
-
}
|
|
3907
|
-
|
|
3908
|
-
|
|
3909
|
-
|
|
3910
|
-
|
|
3911
|
-
);
|
|
3912
|
-
});
|
|
3913
|
-
var PixelColumns = class {
|
|
3914
|
-
constructor(original) {
|
|
3915
|
-
this.original = original;
|
|
3916
|
-
}
|
|
3917
|
-
image = null;
|
|
3918
|
-
columns = [];
|
|
3919
|
-
previous = new Float32Array(6).fill(-1);
|
|
3920
|
-
source = new Source2({ data: new Uint8Array(4), width: 1, height: 1 });
|
|
3921
|
-
update(weights) {
|
|
3922
|
-
const image = this.original.image;
|
|
3923
|
-
if (!rawImage(image)) {
|
|
3924
|
-
return false;
|
|
3925
|
-
}
|
|
3926
|
-
const changed = image !== this.image;
|
|
3927
|
-
if (changed) {
|
|
3928
|
-
this.prepare(image);
|
|
3871
|
+
reset() {
|
|
3872
|
+
this.next = 0;
|
|
3873
|
+
this.shown = null;
|
|
3874
|
+
this.shownAt = -Infinity;
|
|
3875
|
+
this.previousTime = -Infinity;
|
|
3876
|
+
}
|
|
3877
|
+
sample(time, speechEnd = Infinity) {
|
|
3878
|
+
if (!Number.isFinite(time) || time < 0) {
|
|
3879
|
+
this.reset();
|
|
3880
|
+
return null;
|
|
3929
3881
|
}
|
|
3930
|
-
if (
|
|
3931
|
-
|
|
3932
|
-
)) {
|
|
3933
|
-
return false;
|
|
3882
|
+
if (time < this.previousTime) {
|
|
3883
|
+
this.reset();
|
|
3934
3884
|
}
|
|
3935
|
-
this.
|
|
3936
|
-
|
|
3937
|
-
|
|
3938
|
-
const isSrgb = this.original.colorSpace === SRGBColorSpace2;
|
|
3939
|
-
for (let i = 0; i < data.length; i += 1) {
|
|
3940
|
-
let value = 0;
|
|
3941
|
-
for (let column = 0; column < active.length; column += 1) {
|
|
3942
|
-
const source = active[column];
|
|
3943
|
-
value += source.pixels[i] * source.weight;
|
|
3944
|
-
}
|
|
3945
|
-
data[i] = isSrgb && i % 4 !== 3 ? srgb[Math.max(0, Math.min(4096, Math.round(value * 4096)))] : Math.round(value * 255);
|
|
3885
|
+
this.previousTime = time;
|
|
3886
|
+
while (this.cues[this.next] && this.cues[this.next].timestamp <= time + Tolerance) {
|
|
3887
|
+
this.next += 1;
|
|
3946
3888
|
}
|
|
3947
|
-
|
|
3948
|
-
|
|
3949
|
-
|
|
3950
|
-
|
|
3951
|
-
|
|
3952
|
-
|
|
3953
|
-
this.source.data = {
|
|
3954
|
-
data: new Uint8Array(length),
|
|
3955
|
-
width,
|
|
3956
|
-
height: image.height
|
|
3957
|
-
};
|
|
3958
|
-
const isSrgb = this.original.colorSpace === SRGBColorSpace2;
|
|
3959
|
-
this.columns = Array.from({ length: 6 }, (_, column) => {
|
|
3960
|
-
const pixels = new Float32Array(length);
|
|
3961
|
-
for (let y = 0; y < image.height; y += 1) {
|
|
3962
|
-
for (let x = 0; x < width; x += 1) {
|
|
3963
|
-
const sample = ((x + 0.5) / width + column) * image.width / 6 - 0.5;
|
|
3964
|
-
const left = Math.max(
|
|
3965
|
-
0,
|
|
3966
|
-
Math.min(image.width - 1, Math.floor(sample))
|
|
3967
|
-
);
|
|
3968
|
-
const right = Math.min(image.width - 1, left + 1);
|
|
3969
|
-
const mix = Math.max(0, sample - left);
|
|
3970
|
-
for (let channel = 0; channel < 4; channel += 1) {
|
|
3971
|
-
const a = image.data[(y * image.width + left) * 4 + channel];
|
|
3972
|
-
const b = image.data[(y * image.width + right) * 4 + channel];
|
|
3973
|
-
const convert = isSrgb && channel !== 3;
|
|
3974
|
-
const first = convert ? linear[a] : a / 255;
|
|
3975
|
-
const second = convert ? linear[b] : b / 255;
|
|
3976
|
-
pixels[(y * width + x) * 4 + channel] = first + (second - first) * mix;
|
|
3977
|
-
}
|
|
3978
|
-
}
|
|
3979
|
-
}
|
|
3980
|
-
return pixels;
|
|
3981
|
-
});
|
|
3982
|
-
}
|
|
3983
|
-
dispose() {
|
|
3984
|
-
this.columns = [];
|
|
3985
|
-
this.image = null;
|
|
3986
|
-
this.source.data = null;
|
|
3987
|
-
}
|
|
3988
|
-
};
|
|
3989
|
-
var createMouthAtlasPixels = function(material) {
|
|
3990
|
-
const properties = ["map", "shadeMultiplyTexture"];
|
|
3991
|
-
if (properties.some(
|
|
3992
|
-
(name) => material[name] && !rawImage(material[name].image)
|
|
3993
|
-
)) {
|
|
3994
|
-
return null;
|
|
3995
|
-
}
|
|
3996
|
-
const columns = /* @__PURE__ */ new Map();
|
|
3997
|
-
const textures = properties.flatMap((name) => {
|
|
3998
|
-
const original = material[name];
|
|
3999
|
-
if (!original) {
|
|
4000
|
-
return [];
|
|
3889
|
+
const cue = this.cues[this.next - 1];
|
|
3890
|
+
const end = cue ? _HeldMouthTimeline.endTime(cue, this.cues[this.next], speechEnd) : 0;
|
|
3891
|
+
const weight = cue?.weight ?? 1;
|
|
3892
|
+
if (!cue || time + Tolerance >= end || !Number.isFinite(weight) || weight <= 0) {
|
|
3893
|
+
this.shown = null;
|
|
3894
|
+
return null;
|
|
4001
3895
|
}
|
|
4002
|
-
|
|
4003
|
-
|
|
4004
|
-
|
|
4005
|
-
|
|
4006
|
-
|
|
4007
|
-
const texture = new DataTexture();
|
|
4008
|
-
texture.copy(original);
|
|
4009
|
-
texture.source = pixels.source;
|
|
4010
|
-
texture.generateMipmaps = false;
|
|
4011
|
-
texture.minFilter = original.magFilter;
|
|
4012
|
-
material[name] = texture;
|
|
4013
|
-
return [{ name, original, texture, pixels }];
|
|
4014
|
-
});
|
|
4015
|
-
return {
|
|
4016
|
-
update(weights) {
|
|
4017
|
-
const changed = new Set(
|
|
4018
|
-
[...columns.values()].filter((pixels) => pixels.update(weights))
|
|
4019
|
-
);
|
|
4020
|
-
for (const { texture, pixels } of textures) {
|
|
4021
|
-
texture.repeat.x = 6;
|
|
4022
|
-
if (changed.has(pixels)) {
|
|
4023
|
-
texture.needsUpdate = true;
|
|
4024
|
-
}
|
|
4025
|
-
}
|
|
4026
|
-
},
|
|
4027
|
-
dispose() {
|
|
4028
|
-
for (const { name, original, texture } of textures) {
|
|
4029
|
-
material[name] = original;
|
|
4030
|
-
texture.dispose();
|
|
4031
|
-
original.dispose();
|
|
4032
|
-
}
|
|
4033
|
-
columns.forEach((pixels) => pixels.dispose());
|
|
3896
|
+
if (!this.shown || cue.name !== this.shown.name && time - this.shownAt + Tolerance >= MinimumExposure) {
|
|
3897
|
+
this.shown = cue;
|
|
3898
|
+
this.shownAt = time;
|
|
3899
|
+
} else if (cue.name === this.shown.name) {
|
|
3900
|
+
this.shown = cue;
|
|
4034
3901
|
}
|
|
4035
|
-
|
|
3902
|
+
return this.shown;
|
|
3903
|
+
}
|
|
4036
3904
|
};
|
|
4037
3905
|
|
|
4038
3906
|
// ../vrm-renderer/src/texture-expressions.ts
|
|
@@ -4097,77 +3965,126 @@ var findAtlasBindings = function(vrm) {
|
|
|
4097
3965
|
}
|
|
4098
3966
|
return bindings;
|
|
4099
3967
|
};
|
|
4100
|
-
var
|
|
4101
|
-
|
|
4102
|
-
|
|
4103
|
-
|
|
4104
|
-
const
|
|
4105
|
-
if (!bindings.length) {
|
|
3968
|
+
var hasTextureMouthExpressions = function(manager) {
|
|
3969
|
+
return !!manager && installed.has(manager);
|
|
3970
|
+
};
|
|
3971
|
+
var installTextureMouthPlayback = function(vrm, bindings) {
|
|
3972
|
+
const manager = vrm.expressionManager;
|
|
3973
|
+
if (!manager || installed.has(manager) || !bindings.length) {
|
|
4106
3974
|
return;
|
|
4107
3975
|
}
|
|
4108
|
-
|
|
4109
|
-
|
|
4110
|
-
|
|
4111
|
-
const outputs = materials.map(
|
|
4112
|
-
(material) => options.backend === "pixels" ? createMouthAtlasPixels(material) : createMouthAtlasShader(material)
|
|
3976
|
+
installed.add(manager);
|
|
3977
|
+
const materials = new Set(
|
|
3978
|
+
bindings.map(({ bind }) => bind.material)
|
|
4113
3979
|
);
|
|
4114
|
-
|
|
4115
|
-
|
|
4116
|
-
return;
|
|
4117
|
-
}
|
|
4118
|
-
installed.add(vrm);
|
|
4119
|
-
for (const [index, material] of materials.entries()) {
|
|
4120
|
-
const output = outputs[index];
|
|
3980
|
+
for (const material of materials) {
|
|
3981
|
+
const mouths = bindings.filter(({ bind }) => bind.material === material);
|
|
4121
3982
|
const targets = new Float32Array(6);
|
|
4122
|
-
|
|
4123
|
-
for (const { expression, bind, column } of
|
|
4124
|
-
if (bind.material !== material) {
|
|
4125
|
-
continue;
|
|
4126
|
-
}
|
|
3983
|
+
let selected = 0;
|
|
3984
|
+
for (const { expression, bind, column } of mouths) {
|
|
4127
3985
|
expression.isBinary = false;
|
|
4128
3986
|
expression.deleteBind(bind);
|
|
4129
3987
|
expression.addBind({
|
|
4130
3988
|
clearAppliedWeight: () => {
|
|
4131
3989
|
targets[column] = 0;
|
|
3990
|
+
bind.clearAppliedWeight();
|
|
4132
3991
|
},
|
|
4133
3992
|
applyWeight: (weight) => {
|
|
4134
|
-
targets[column] = Number.isFinite(weight) ? Math.max(0,
|
|
3993
|
+
targets[column] = Number.isFinite(weight) ? Math.max(0, weight) : 0;
|
|
4135
3994
|
}
|
|
4136
3995
|
});
|
|
4137
3996
|
}
|
|
4138
3997
|
const originalUpdate = material.update;
|
|
4139
3998
|
material.update = function(delta) {
|
|
4140
|
-
let
|
|
3999
|
+
let winner = selected;
|
|
4000
|
+
let weight = targets[winner];
|
|
4141
4001
|
for (let column = 1; column < 6; column += 1) {
|
|
4142
|
-
|
|
4143
|
-
|
|
4144
|
-
|
|
4145
|
-
|
|
4146
|
-
let distance = 0;
|
|
4147
|
-
for (let column = 0; column < 6; column += 1) {
|
|
4148
|
-
targets[column] = targets[column] / scale;
|
|
4149
|
-
distance = Math.max(
|
|
4150
|
-
distance,
|
|
4151
|
-
Math.abs(targets[column] - weights[column])
|
|
4152
|
-
);
|
|
4153
|
-
}
|
|
4154
|
-
const step = Number.isFinite(delta) ? Math.max(0, delta) / 0.08 : 0;
|
|
4155
|
-
const factor = distance > 0 ? Math.min(1, step / distance) : 1;
|
|
4156
|
-
for (let column = 0; column < 6; column += 1) {
|
|
4157
|
-
weights[column] = weights[column] + (targets[column] - weights[column]) * factor;
|
|
4002
|
+
if (targets[column] > weight) {
|
|
4003
|
+
winner = column;
|
|
4004
|
+
weight = targets[column];
|
|
4005
|
+
}
|
|
4158
4006
|
}
|
|
4159
|
-
|
|
4007
|
+
selected = weight > 0 ? winner : 0;
|
|
4008
|
+
mouths.find(({ column }) => column === selected)?.bind.applyWeight(1);
|
|
4160
4009
|
originalUpdate.call(material, delta);
|
|
4161
4010
|
};
|
|
4162
4011
|
const dispose = function() {
|
|
4163
4012
|
material.removeEventListener("dispose", dispose);
|
|
4164
4013
|
material.update = originalUpdate;
|
|
4165
|
-
output.dispose();
|
|
4166
4014
|
};
|
|
4167
4015
|
material.addEventListener("dispose", dispose);
|
|
4168
4016
|
}
|
|
4169
4017
|
};
|
|
4170
4018
|
|
|
4019
|
+
// ../vrm-renderer/src/generated-character.ts
|
|
4020
|
+
var TOLERANCE = 1e-4;
|
|
4021
|
+
var prepared = /* @__PURE__ */ new WeakMap();
|
|
4022
|
+
var sameColour = function(a, b) {
|
|
4023
|
+
return Math.abs(a.r - b.r) < TOLERANCE && Math.abs(a.g - b.g) < TOLERANCE && Math.abs(a.b - b.b) < TOLERANCE;
|
|
4024
|
+
};
|
|
4025
|
+
var sceneMToonMaterials = function(scene) {
|
|
4026
|
+
const materials = /* @__PURE__ */ new Set();
|
|
4027
|
+
scene.traverse((object) => {
|
|
4028
|
+
const material = object.material;
|
|
4029
|
+
for (const entry of Array.isArray(material) ? material : [material]) {
|
|
4030
|
+
if (entry instanceof MToonMaterial2) {
|
|
4031
|
+
materials.add(entry);
|
|
4032
|
+
}
|
|
4033
|
+
}
|
|
4034
|
+
});
|
|
4035
|
+
return materials;
|
|
4036
|
+
};
|
|
4037
|
+
var sharesFaceRecipe = function(material, faces) {
|
|
4038
|
+
return !!material.map && material.shadeMultiplyTexture?.image === material.map.image && faces.some(
|
|
4039
|
+
(face) => sameColour(material.color, face.color) && sameColour(material.shadeColorFactor, face.shadeColorFactor)
|
|
4040
|
+
);
|
|
4041
|
+
};
|
|
4042
|
+
var detectGeneratedCharacter = function(vrm) {
|
|
4043
|
+
const known = prepared.get(vrm);
|
|
4044
|
+
if (known) {
|
|
4045
|
+
return known;
|
|
4046
|
+
}
|
|
4047
|
+
const mouths = findAtlasBindings(vrm);
|
|
4048
|
+
if (!mouths.length) {
|
|
4049
|
+
return null;
|
|
4050
|
+
}
|
|
4051
|
+
const faces = [
|
|
4052
|
+
...new Set(mouths.map(({ bind }) => bind.material))
|
|
4053
|
+
];
|
|
4054
|
+
const coloured = /* @__PURE__ */ new Set();
|
|
4055
|
+
for (const expression of vrm.expressionManager?.expressions ?? []) {
|
|
4056
|
+
for (const bind of expression.binds) {
|
|
4057
|
+
if (bind instanceof VRMExpressionMaterialColorBind) {
|
|
4058
|
+
coloured.add(bind.material);
|
|
4059
|
+
}
|
|
4060
|
+
}
|
|
4061
|
+
}
|
|
4062
|
+
const materials = [...sceneMToonMaterials(vrm.scene)].filter(
|
|
4063
|
+
(material) => !coloured.has(material) && (faces.includes(material) || sharesFaceRecipe(material, faces))
|
|
4064
|
+
);
|
|
4065
|
+
return { faces, mouths, materials };
|
|
4066
|
+
};
|
|
4067
|
+
var normaliseLitFactor = function(material) {
|
|
4068
|
+
const { r, g, b } = material.color;
|
|
4069
|
+
if (!(r > 0) || r >= 1 - TOLERANCE || Math.abs(r - g) >= TOLERANCE || Math.abs(r - b) >= TOLERANCE) {
|
|
4070
|
+
return;
|
|
4071
|
+
}
|
|
4072
|
+
material.color.multiplyScalar(1 / r);
|
|
4073
|
+
material.shadeColorFactor.multiplyScalar(1 / r);
|
|
4074
|
+
};
|
|
4075
|
+
var prepareGeneratedCharacter = function(vrm) {
|
|
4076
|
+
const character = detectGeneratedCharacter(vrm);
|
|
4077
|
+
if (!character || prepared.has(vrm)) {
|
|
4078
|
+
return character;
|
|
4079
|
+
}
|
|
4080
|
+
for (const material of character.materials) {
|
|
4081
|
+
normaliseLitFactor(material);
|
|
4082
|
+
}
|
|
4083
|
+
installTextureMouthPlayback(vrm, character.mouths);
|
|
4084
|
+
prepared.set(vrm, character);
|
|
4085
|
+
return character;
|
|
4086
|
+
};
|
|
4087
|
+
|
|
4171
4088
|
// ../vrm-renderer/src/session.ts
|
|
4172
4089
|
import { VRMUtils as VRMUtils2 } from "@pixiv/three-vrm";
|
|
4173
4090
|
import { AnimationMixer, LoopOnce, LoopRepeat } from "three";
|
|
@@ -4492,7 +4409,7 @@ var loadVrm = async ({
|
|
|
4492
4409
|
});
|
|
4493
4410
|
VRMUtils4.removeUnnecessaryVertices(gltf.scene);
|
|
4494
4411
|
VRMUtils4.combineSkeletons(gltf.scene);
|
|
4495
|
-
|
|
4412
|
+
prepareGeneratedCharacter(gltf.userData.vrm);
|
|
4496
4413
|
return gltf.userData.vrm;
|
|
4497
4414
|
};
|
|
4498
4415
|
var loadVrma = async ({
|
|
@@ -4624,6 +4541,8 @@ var AnimationController = class {
|
|
|
4624
4541
|
mixer;
|
|
4625
4542
|
clips = /* @__PURE__ */ new Map();
|
|
4626
4543
|
configs = /* @__PURE__ */ new Map();
|
|
4544
|
+
previewMixer = null;
|
|
4545
|
+
restorePreview = null;
|
|
4627
4546
|
activeAction = null;
|
|
4628
4547
|
idleAnimationId;
|
|
4629
4548
|
pendingReturnToIdleAction = null;
|
|
@@ -4638,11 +4557,35 @@ var AnimationController = class {
|
|
|
4638
4557
|
weight: config?.weight ?? defaultAnimationConfig.weight
|
|
4639
4558
|
});
|
|
4640
4559
|
}
|
|
4560
|
+
previewAnimation(animation) {
|
|
4561
|
+
this.restorePreview?.();
|
|
4562
|
+
const clip = createVRMAnimationClip(animation, this.vrm);
|
|
4563
|
+
const previewMixer = new AnimationMixer2(this.vrm.scene);
|
|
4564
|
+
previewMixer.clipAction(clip).setLoop(LoopRepeat2, Infinity).reset().play();
|
|
4565
|
+
this.previewMixer = previewMixer;
|
|
4566
|
+
const restore = () => {
|
|
4567
|
+
if (this.restorePreview !== restore) {
|
|
4568
|
+
return;
|
|
4569
|
+
}
|
|
4570
|
+
this.restorePreview = null;
|
|
4571
|
+
previewMixer.stopAllAction();
|
|
4572
|
+
previewMixer.uncacheRoot(this.vrm.scene);
|
|
4573
|
+
this.previewMixer = null;
|
|
4574
|
+
if (!this.activeAction) {
|
|
4575
|
+
this.vrm.humanoid.resetNormalizedPose();
|
|
4576
|
+
}
|
|
4577
|
+
this.mixer.update(0);
|
|
4578
|
+
};
|
|
4579
|
+
this.restorePreview = restore;
|
|
4580
|
+
previewMixer.update(0);
|
|
4581
|
+
return restore;
|
|
4582
|
+
}
|
|
4641
4583
|
setIdleAnimation(animationId) {
|
|
4642
4584
|
this.idleAnimationId = animationId;
|
|
4643
4585
|
this.playIdle();
|
|
4644
4586
|
}
|
|
4645
4587
|
clearAnimations() {
|
|
4588
|
+
this.restorePreview?.();
|
|
4646
4589
|
this.mixer.stopAllAction();
|
|
4647
4590
|
this.mixer.uncacheRoot(this.vrm.scene);
|
|
4648
4591
|
this.clips.clear();
|
|
@@ -4653,6 +4596,7 @@ var AnimationController = class {
|
|
|
4653
4596
|
this.vrm.humanoid.resetNormalizedPose();
|
|
4654
4597
|
}
|
|
4655
4598
|
playAnimation(animationId, fadeDuration = 0.25, loopOverride) {
|
|
4599
|
+
this.restorePreview?.();
|
|
4656
4600
|
const clip = this.clips.get(animationId);
|
|
4657
4601
|
if (!clip) {
|
|
4658
4602
|
throw new Error(`Animation ${animationId} not found`);
|
|
@@ -4686,6 +4630,7 @@ var AnimationController = class {
|
|
|
4686
4630
|
}
|
|
4687
4631
|
}
|
|
4688
4632
|
playIdle(fadeDuration = 0.25) {
|
|
4633
|
+
this.restorePreview?.();
|
|
4689
4634
|
if (!this.idleAnimationId || !this.clips.has(this.idleAnimationId)) {
|
|
4690
4635
|
this.mixer.stopAllAction();
|
|
4691
4636
|
this.activeAction = null;
|
|
@@ -4699,6 +4644,7 @@ var AnimationController = class {
|
|
|
4699
4644
|
this.playAnimation(this.idleAnimationId, fadeDuration, true);
|
|
4700
4645
|
}
|
|
4701
4646
|
stopAnimation(fadeDuration = 0.25) {
|
|
4647
|
+
this.restorePreview?.();
|
|
4702
4648
|
if (!this.activeAction) {
|
|
4703
4649
|
return;
|
|
4704
4650
|
}
|
|
@@ -4707,7 +4653,8 @@ var AnimationController = class {
|
|
|
4707
4653
|
this.pendingReturnToIdleAction = null;
|
|
4708
4654
|
}
|
|
4709
4655
|
update(delta) {
|
|
4710
|
-
|
|
4656
|
+
;
|
|
4657
|
+
(this.previewMixer ?? this.mixer).update(delta);
|
|
4711
4658
|
}
|
|
4712
4659
|
handleActionFinished = (event) => {
|
|
4713
4660
|
if (!this.idleAnimationId || !this.pendingReturnToIdleAction || event.action !== this.pendingReturnToIdleAction || !this.clips.has(this.idleAnimationId)) {
|
|
@@ -5359,6 +5306,7 @@ var PerformanceWeightMixer = class {
|
|
|
5359
5306
|
...definition,
|
|
5360
5307
|
id,
|
|
5361
5308
|
responseRate: normalizeResponseRate(definition.responseRate),
|
|
5309
|
+
transition: definition.transition ?? "smooth",
|
|
5362
5310
|
weight
|
|
5363
5311
|
},
|
|
5364
5312
|
order: this.nextLayerOrder,
|
|
@@ -5435,12 +5383,13 @@ var PerformanceWeightMixer = class {
|
|
|
5435
5383
|
const resolved = /* @__PURE__ */ new Map();
|
|
5436
5384
|
this.orderedLayers.forEach((layer) => {
|
|
5437
5385
|
const responseRate = layer.definition.responseRate;
|
|
5386
|
+
const frameDelta = layer.definition.transition === "step" ? 0 : delta;
|
|
5438
5387
|
layer.value = clamp01(
|
|
5439
|
-
smoothValue(layer.value, layer.target, responseRate,
|
|
5388
|
+
smoothValue(layer.value, layer.target, responseRate, frameDelta)
|
|
5440
5389
|
);
|
|
5441
5390
|
layer.channels.forEach((channel, name) => {
|
|
5442
5391
|
channel.value = clamp01(
|
|
5443
|
-
smoothValue(channel.value, channel.target, responseRate,
|
|
5392
|
+
smoothValue(channel.value, channel.target, responseRate, frameDelta)
|
|
5444
5393
|
);
|
|
5445
5394
|
const current = resolved.get(name) ?? 0;
|
|
5446
5395
|
const contribution = channel.value * layer.value;
|
|
@@ -5566,6 +5515,9 @@ var FacialPerformanceMixer = class {
|
|
|
5566
5515
|
getVisemeChannelNames() {
|
|
5567
5516
|
return this.getLayerChannels(FacialPerformanceLayer.Viseme);
|
|
5568
5517
|
}
|
|
5518
|
+
get usesDiscreteMouths() {
|
|
5519
|
+
return hasTextureMouthExpressions(this.vrm?.expressionManager);
|
|
5520
|
+
}
|
|
5569
5521
|
playCue(channel, options) {
|
|
5570
5522
|
if (!this.availableChannelSet.has(channel)) {
|
|
5571
5523
|
return;
|
|
@@ -5726,7 +5678,8 @@ var FacialPerformanceMixer = class {
|
|
|
5726
5678
|
this.weightMixer.registerLayer({
|
|
5727
5679
|
id: FacialPerformanceLayer.Viseme,
|
|
5728
5680
|
priority: 200,
|
|
5729
|
-
blendMode: "override"
|
|
5681
|
+
blendMode: "override",
|
|
5682
|
+
transition: this.usesDiscreteMouths ? "step" : "smooth"
|
|
5730
5683
|
});
|
|
5731
5684
|
this.weightMixer.registerLayer({
|
|
5732
5685
|
id: FacialPerformanceLayer.Blink,
|
|
@@ -6475,6 +6428,7 @@ var VisemePlayer = class {
|
|
|
6475
6428
|
phase = "idle";
|
|
6476
6429
|
running = false;
|
|
6477
6430
|
timelineRevision = 0;
|
|
6431
|
+
heldMouths = new HeldMouthTimeline();
|
|
6478
6432
|
attachAudioElement(element) {
|
|
6479
6433
|
if (this.audioElement === element) {
|
|
6480
6434
|
return;
|
|
@@ -6509,6 +6463,7 @@ var VisemePlayer = class {
|
|
|
6509
6463
|
this.rawVisemes = visemes.map((viseme) => ({ ...viseme }));
|
|
6510
6464
|
this.normalizedTimeline = normalizeVisemeTimeline(this.rawVisemes);
|
|
6511
6465
|
this.visemes = this.normalizedTimeline.entries;
|
|
6466
|
+
this.heldMouths.setCues(this.visemes);
|
|
6512
6467
|
this.timelineRevision += 1;
|
|
6513
6468
|
this.resetPlaybackState();
|
|
6514
6469
|
this.recordTimeline();
|
|
@@ -6570,6 +6525,10 @@ var VisemePlayer = class {
|
|
|
6570
6525
|
if (currentTime === null) {
|
|
6571
6526
|
return;
|
|
6572
6527
|
}
|
|
6528
|
+
if (this.facialMixer.usesDiscreteMouths) {
|
|
6529
|
+
this.updateHeldMouth(currentTime);
|
|
6530
|
+
return;
|
|
6531
|
+
}
|
|
6573
6532
|
const dueCueIndices = this.collectDueCueIndices(currentTime);
|
|
6574
6533
|
const latestDueCueIndex = dueCueIndices.at(-1);
|
|
6575
6534
|
dueCueIndices.slice(0, -1).forEach((cueIndex) => {
|
|
@@ -6600,6 +6559,34 @@ var VisemePlayer = class {
|
|
|
6600
6559
|
state: "playing"
|
|
6601
6560
|
});
|
|
6602
6561
|
};
|
|
6562
|
+
updateHeldMouth(currentTime) {
|
|
6563
|
+
const duration = this.audioElement?.duration;
|
|
6564
|
+
const speechEnd = Number.isFinite(duration) ? duration : void 0;
|
|
6565
|
+
const cue = this.heldMouths.sample(currentTime, speechEnd);
|
|
6566
|
+
if (!cue) {
|
|
6567
|
+
if (this.activeCueIndex !== null) {
|
|
6568
|
+
this.releaseToNeutral("scheduled");
|
|
6569
|
+
}
|
|
6570
|
+
return;
|
|
6571
|
+
}
|
|
6572
|
+
if (cue === this.visemes[this.activeCueIndex ?? -1]) {
|
|
6573
|
+
return;
|
|
6574
|
+
}
|
|
6575
|
+
const index = this.visemes.indexOf(cue);
|
|
6576
|
+
this.facialMixer.playCue(cue.name, { weight: 1, durationMs: 0 });
|
|
6577
|
+
this.activeCueIndex = index;
|
|
6578
|
+
this.nextVisemeIndex = index + 1;
|
|
6579
|
+
this.phase = "open";
|
|
6580
|
+
this.recordCueDecision(index, currentTime, "played");
|
|
6581
|
+
}
|
|
6582
|
+
cueDuration(cue, next) {
|
|
6583
|
+
if (this.facialMixer.usesDiscreteMouths) {
|
|
6584
|
+
const duration = this.audioElement?.duration;
|
|
6585
|
+
const speechEnd = Number.isFinite(duration) ? duration : void 0;
|
|
6586
|
+
return (HeldMouthTimeline.endTime(cue, next, speechEnd) - cue.timestamp) * 1e3;
|
|
6587
|
+
}
|
|
6588
|
+
return getVisemeDurationMs(cue, next);
|
|
6589
|
+
}
|
|
6603
6590
|
handleEnded = () => {
|
|
6604
6591
|
this.recorder?.recordEvent({
|
|
6605
6592
|
type: "audio.state",
|
|
@@ -6728,7 +6715,7 @@ var VisemePlayer = class {
|
|
|
6728
6715
|
if (!cue) {
|
|
6729
6716
|
return;
|
|
6730
6717
|
}
|
|
6731
|
-
const openDurationMs =
|
|
6718
|
+
const openDurationMs = this.cueDuration(cue, this.visemes[cueIndex + 1]);
|
|
6732
6719
|
const elapsedMs = Math.max(currentTime - cue.timestamp, 0) * 1e3;
|
|
6733
6720
|
this.recorder?.recordEvent({
|
|
6734
6721
|
type: "viseme.cue",
|
|
@@ -6773,6 +6760,7 @@ var VisemePlayer = class {
|
|
|
6773
6760
|
this.audioElement.removeEventListener("ended", this.handleEnded);
|
|
6774
6761
|
}
|
|
6775
6762
|
resetPlaybackState() {
|
|
6763
|
+
this.heldMouths.reset();
|
|
6776
6764
|
this.running = false;
|
|
6777
6765
|
this.nextVisemeIndex = 0;
|
|
6778
6766
|
this.activeCueIndex = null;
|