@omote/core 0.6.2 → 0.6.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -13
- package/dist/index.d.mts +330 -52
- package/dist/index.d.ts +330 -52
- package/dist/index.js +469 -75
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +469 -75
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -32,7 +32,9 @@ var index_exports = {};
|
|
|
32
32
|
__export(index_exports, {
|
|
33
33
|
A2EOrchestrator: () => A2EOrchestrator,
|
|
34
34
|
A2EProcessor: () => A2EProcessor,
|
|
35
|
+
ALL_AUS: () => ALL_AUS,
|
|
35
36
|
ARKIT_BLENDSHAPES: () => ARKIT_BLENDSHAPES,
|
|
37
|
+
AU_TO_ARKIT: () => AU_TO_ARKIT,
|
|
36
38
|
AnimationGraph: () => AnimationGraph,
|
|
37
39
|
AudioChunkCoalescer: () => AudioChunkCoalescer,
|
|
38
40
|
AudioEnergyAnalyzer: () => AudioEnergyAnalyzer,
|
|
@@ -43,13 +45,18 @@ __export(index_exports, {
|
|
|
43
45
|
ConsoleExporter: () => ConsoleExporter,
|
|
44
46
|
DEFAULT_ANIMATION_CONFIG: () => DEFAULT_ANIMATION_CONFIG,
|
|
45
47
|
DEFAULT_LOGGING_CONFIG: () => DEFAULT_LOGGING_CONFIG,
|
|
48
|
+
DEFAULT_MODEL_URLS: () => DEFAULT_MODEL_URLS,
|
|
46
49
|
EMOTION_NAMES: () => EMOTION_NAMES,
|
|
50
|
+
EMOTION_TO_AU: () => EMOTION_TO_AU,
|
|
47
51
|
EMOTION_VECTOR_SIZE: () => EMOTION_VECTOR_SIZE,
|
|
48
52
|
EmotionController: () => EmotionController,
|
|
49
53
|
EmotionPresets: () => EmotionPresets,
|
|
54
|
+
EmotionResolver: () => EmotionResolver,
|
|
50
55
|
EmphasisDetector: () => EmphasisDetector,
|
|
51
56
|
EventEmitter: () => EventEmitter,
|
|
57
|
+
FaceCompositor: () => FaceCompositor,
|
|
52
58
|
FullFacePipeline: () => FullFacePipeline,
|
|
59
|
+
HF_CDN_URLS: () => HF_CDN_URLS,
|
|
53
60
|
INFERENCE_LATENCY_BUCKETS: () => INFERENCE_LATENCY_BUCKETS,
|
|
54
61
|
InterruptionHandler: () => InterruptionHandler,
|
|
55
62
|
LAM_BLENDSHAPES: () => LAM_BLENDSHAPES,
|
|
@@ -84,6 +91,7 @@ __export(index_exports, {
|
|
|
84
91
|
calculateRMS: () => calculateRMS,
|
|
85
92
|
configureCacheLimit: () => configureCacheLimit,
|
|
86
93
|
configureLogging: () => configureLogging,
|
|
94
|
+
configureModelUrls: () => configureModelUrls,
|
|
87
95
|
configureTelemetry: () => configureTelemetry,
|
|
88
96
|
createA2E: () => createA2E,
|
|
89
97
|
createEmotionVector: () => createEmotionVector,
|
|
@@ -114,6 +122,7 @@ __export(index_exports, {
|
|
|
114
122
|
noopLogger: () => noopLogger,
|
|
115
123
|
preloadModels: () => preloadModels,
|
|
116
124
|
resetLoggingConfig: () => resetLoggingConfig,
|
|
125
|
+
resetModelUrls: () => resetModelUrls,
|
|
117
126
|
resolveBackend: () => resolveBackend,
|
|
118
127
|
setLogLevel: () => setLogLevel,
|
|
119
128
|
setLoggingEnabled: () => setLoggingEnabled,
|
|
@@ -2962,7 +2971,7 @@ var _Wav2Vec2Inference = class _Wav2Vec2Inference {
|
|
|
2962
2971
|
} else {
|
|
2963
2972
|
logger3.info("Fetching external model data", {
|
|
2964
2973
|
dataUrl,
|
|
2965
|
-
note: "This may be a large download
|
|
2974
|
+
note: "This may be a large download"
|
|
2966
2975
|
});
|
|
2967
2976
|
externalDataBuffer = await fetchWithCache(dataUrl);
|
|
2968
2977
|
}
|
|
@@ -2970,6 +2979,9 @@ var _Wav2Vec2Inference = class _Wav2Vec2Inference {
|
|
|
2970
2979
|
size: formatBytes(externalDataBuffer.byteLength)
|
|
2971
2980
|
});
|
|
2972
2981
|
} catch (err) {
|
|
2982
|
+
if (typeof this.config.externalDataUrl === "string") {
|
|
2983
|
+
throw new Error(`Failed to fetch external data: ${dataUrl} \u2014 ${err.message}`);
|
|
2984
|
+
}
|
|
2973
2985
|
logger3.debug("No external data file found (single-file model)", {
|
|
2974
2986
|
dataUrl,
|
|
2975
2987
|
error: err.message
|
|
@@ -3093,28 +3105,6 @@ var _Wav2Vec2Inference = class _Wav2Vec2Inference {
|
|
|
3093
3105
|
};
|
|
3094
3106
|
return this.queueInference(feeds);
|
|
3095
3107
|
}
|
|
3096
|
-
/**
|
|
3097
|
-
* Decode CTC logits to text using greedy decoding
|
|
3098
|
-
*/
|
|
3099
|
-
decodeCTC(logits) {
|
|
3100
|
-
const tokens = [];
|
|
3101
|
-
let prevToken = -1;
|
|
3102
|
-
for (const frame of logits) {
|
|
3103
|
-
let maxIdx = 0;
|
|
3104
|
-
let maxVal = frame[0];
|
|
3105
|
-
for (let i = 1; i < frame.length; i++) {
|
|
3106
|
-
if (frame[i] > maxVal) {
|
|
3107
|
-
maxVal = frame[i];
|
|
3108
|
-
maxIdx = i;
|
|
3109
|
-
}
|
|
3110
|
-
}
|
|
3111
|
-
if (maxIdx !== prevToken && maxIdx !== 0) {
|
|
3112
|
-
tokens.push(maxIdx);
|
|
3113
|
-
}
|
|
3114
|
-
prevToken = maxIdx;
|
|
3115
|
-
}
|
|
3116
|
-
return tokens.map((t) => CTC_VOCAB[t] === "|" ? " " : CTC_VOCAB[t]).join("");
|
|
3117
|
-
}
|
|
3118
3108
|
/**
|
|
3119
3109
|
* Queue inference to serialize ONNX session calls
|
|
3120
3110
|
*/
|
|
@@ -3142,37 +3132,25 @@ var _Wav2Vec2Inference = class _Wav2Vec2Inference {
|
|
|
3142
3132
|
})
|
|
3143
3133
|
]);
|
|
3144
3134
|
const inferenceTimeMs = performance.now() - startTime;
|
|
3145
|
-
const asrOutput = results["asr_logits"];
|
|
3146
3135
|
const blendshapeOutput = results["blendshapes"];
|
|
3147
|
-
if (!
|
|
3148
|
-
throw new Error("Missing
|
|
3136
|
+
if (!blendshapeOutput) {
|
|
3137
|
+
throw new Error("Missing blendshapes output from model");
|
|
3149
3138
|
}
|
|
3150
|
-
const asrData = asrOutput.data;
|
|
3151
3139
|
const blendshapeData = blendshapeOutput.data;
|
|
3152
|
-
const numASRFrames = asrOutput.dims[1];
|
|
3153
3140
|
const numA2EFrames = blendshapeOutput.dims[1];
|
|
3154
|
-
const asrVocabSize = asrOutput.dims[2];
|
|
3155
3141
|
const numBlendshapes = blendshapeOutput.dims[2];
|
|
3156
|
-
const asrLogits = [];
|
|
3157
3142
|
const blendshapes = [];
|
|
3158
|
-
for (let f = 0; f < numASRFrames; f++) {
|
|
3159
|
-
asrLogits.push(asrData.slice(f * asrVocabSize, (f + 1) * asrVocabSize));
|
|
3160
|
-
}
|
|
3161
3143
|
for (let f = 0; f < numA2EFrames; f++) {
|
|
3162
3144
|
const rawFrame = blendshapeData.slice(f * numBlendshapes, (f + 1) * numBlendshapes);
|
|
3163
3145
|
blendshapes.push(symmetrizeBlendshapes(rawFrame));
|
|
3164
3146
|
}
|
|
3165
|
-
const text = this.decodeCTC(asrLogits);
|
|
3166
3147
|
logger3.trace("Inference completed", {
|
|
3167
3148
|
inferenceTimeMs: Math.round(inferenceTimeMs * 100) / 100,
|
|
3168
|
-
numA2EFrames
|
|
3169
|
-
numASRFrames,
|
|
3170
|
-
textLength: text.length
|
|
3149
|
+
numA2EFrames
|
|
3171
3150
|
});
|
|
3172
3151
|
span?.setAttributes({
|
|
3173
3152
|
"inference.duration_ms": inferenceTimeMs,
|
|
3174
|
-
"inference.a2e_frames": numA2EFrames
|
|
3175
|
-
"inference.asr_frames": numASRFrames
|
|
3153
|
+
"inference.a2e_frames": numA2EFrames
|
|
3176
3154
|
});
|
|
3177
3155
|
span?.end();
|
|
3178
3156
|
telemetry?.recordHistogram("omote.inference.latency", inferenceTimeMs, {
|
|
@@ -3186,11 +3164,7 @@ var _Wav2Vec2Inference = class _Wav2Vec2Inference {
|
|
|
3186
3164
|
});
|
|
3187
3165
|
resolve({
|
|
3188
3166
|
blendshapes,
|
|
3189
|
-
asrLogits,
|
|
3190
|
-
text,
|
|
3191
3167
|
numFrames: numA2EFrames,
|
|
3192
|
-
numA2EFrames,
|
|
3193
|
-
numASRFrames,
|
|
3194
3168
|
inferenceTimeMs
|
|
3195
3169
|
});
|
|
3196
3170
|
} catch (err) {
|
|
@@ -5553,6 +5527,51 @@ var SenseVoiceWorker = class {
|
|
|
5553
5527
|
}
|
|
5554
5528
|
};
|
|
5555
5529
|
|
|
5530
|
+
// src/inference/defaultModelUrls.ts
|
|
5531
|
+
var HF = "https://huggingface.co";
|
|
5532
|
+
var HF_MODEL_URLS = {
|
|
5533
|
+
/** LAM A2E model — fp16 external data (385KB graph + 192MB weights, WebGPU) — 52 ARKit blendshapes */
|
|
5534
|
+
lam: `${HF}/omote-ai/lam-a2e/resolve/main/model_fp16.onnx`,
|
|
5535
|
+
/** wav2arkit_cpu A2E model graph (1.86MB, WASM) — Safari/iOS fallback */
|
|
5536
|
+
wav2arkitCpu: `${HF}/myned-ai/wav2arkit_cpu/resolve/main/wav2arkit_cpu.onnx`,
|
|
5537
|
+
/** SenseVoice ASR model (228MB int8, WASM) — speech recognition + emotion + language */
|
|
5538
|
+
senseVoice: `${HF}/omote-ai/sensevoice-asr/resolve/main/model.int8.onnx`,
|
|
5539
|
+
/** Silero VAD model (~2MB, WASM) — voice activity detection */
|
|
5540
|
+
sileroVad: `${HF}/deepghs/silero-vad-onnx/resolve/main/silero_vad.onnx`
|
|
5541
|
+
};
|
|
5542
|
+
var _overrides = {};
|
|
5543
|
+
var DEFAULT_MODEL_URLS = new Proxy(
|
|
5544
|
+
{},
|
|
5545
|
+
{
|
|
5546
|
+
get(_target, prop) {
|
|
5547
|
+
const key = prop;
|
|
5548
|
+
return _overrides[key] ?? HF_MODEL_URLS[key];
|
|
5549
|
+
},
|
|
5550
|
+
ownKeys() {
|
|
5551
|
+
return Object.keys(HF_MODEL_URLS);
|
|
5552
|
+
},
|
|
5553
|
+
getOwnPropertyDescriptor(_target, prop) {
|
|
5554
|
+
if (prop in HF_MODEL_URLS) {
|
|
5555
|
+
return { configurable: true, enumerable: true, value: this.get(_target, prop, _target) };
|
|
5556
|
+
}
|
|
5557
|
+
return void 0;
|
|
5558
|
+
}
|
|
5559
|
+
}
|
|
5560
|
+
);
|
|
5561
|
+
function configureModelUrls(urls) {
|
|
5562
|
+
for (const [key, url] of Object.entries(urls)) {
|
|
5563
|
+
if (key in HF_MODEL_URLS && typeof url === "string") {
|
|
5564
|
+
_overrides[key] = url;
|
|
5565
|
+
}
|
|
5566
|
+
}
|
|
5567
|
+
}
|
|
5568
|
+
function resetModelUrls() {
|
|
5569
|
+
for (const key of Object.keys(_overrides)) {
|
|
5570
|
+
delete _overrides[key];
|
|
5571
|
+
}
|
|
5572
|
+
}
|
|
5573
|
+
var HF_CDN_URLS = HF_MODEL_URLS;
|
|
5574
|
+
|
|
5556
5575
|
// src/inference/UnifiedInferenceWorker.ts
|
|
5557
5576
|
var logger8 = createLogger("UnifiedInferenceWorker");
|
|
5558
5577
|
var WASM_CDN_PATH3 = "https://cdn.jsdelivr.net/npm/onnxruntime-web@1.23.2/dist/";
|
|
@@ -6794,11 +6813,12 @@ var SileroVADUnifiedAdapter = class {
|
|
|
6794
6813
|
|
|
6795
6814
|
// src/inference/createSenseVoice.ts
|
|
6796
6815
|
var logger9 = createLogger("createSenseVoice");
|
|
6797
|
-
function createSenseVoice(config) {
|
|
6816
|
+
function createSenseVoice(config = {}) {
|
|
6817
|
+
const modelUrl = config.modelUrl ?? DEFAULT_MODEL_URLS.senseVoice;
|
|
6798
6818
|
if (config.unifiedWorker) {
|
|
6799
6819
|
logger9.info("Creating SenseVoiceUnifiedAdapter (shared unified worker)");
|
|
6800
6820
|
return new SenseVoiceUnifiedAdapter(config.unifiedWorker, {
|
|
6801
|
-
modelUrl
|
|
6821
|
+
modelUrl,
|
|
6802
6822
|
tokensUrl: config.tokensUrl,
|
|
6803
6823
|
language: config.language,
|
|
6804
6824
|
textNorm: config.textNorm
|
|
@@ -6811,7 +6831,7 @@ function createSenseVoice(config) {
|
|
|
6811
6831
|
}
|
|
6812
6832
|
logger9.info("Creating SenseVoiceWorker (off-main-thread)");
|
|
6813
6833
|
return new SenseVoiceWorker({
|
|
6814
|
-
modelUrl
|
|
6834
|
+
modelUrl,
|
|
6815
6835
|
tokensUrl: config.tokensUrl,
|
|
6816
6836
|
language: config.language,
|
|
6817
6837
|
textNorm: config.textNorm
|
|
@@ -6820,7 +6840,7 @@ function createSenseVoice(config) {
|
|
|
6820
6840
|
if (useWorker === false) {
|
|
6821
6841
|
logger9.info("Creating SenseVoiceInference (main thread)");
|
|
6822
6842
|
return new SenseVoiceInference({
|
|
6823
|
-
modelUrl
|
|
6843
|
+
modelUrl,
|
|
6824
6844
|
tokensUrl: config.tokensUrl,
|
|
6825
6845
|
language: config.language,
|
|
6826
6846
|
textNorm: config.textNorm
|
|
@@ -6829,7 +6849,7 @@ function createSenseVoice(config) {
|
|
|
6829
6849
|
if (SenseVoiceWorker.isSupported() && !isIOS()) {
|
|
6830
6850
|
logger9.info("Auto-detected: creating SenseVoiceWorker (off-main-thread)");
|
|
6831
6851
|
return new SenseVoiceWorker({
|
|
6832
|
-
modelUrl
|
|
6852
|
+
modelUrl,
|
|
6833
6853
|
tokensUrl: config.tokensUrl,
|
|
6834
6854
|
language: config.language,
|
|
6835
6855
|
textNorm: config.textNorm
|
|
@@ -6839,7 +6859,7 @@ function createSenseVoice(config) {
|
|
|
6839
6859
|
reason: isIOS() ? "iOS (shared ORT instance)" : "Worker unsupported"
|
|
6840
6860
|
});
|
|
6841
6861
|
return new SenseVoiceInference({
|
|
6842
|
-
modelUrl
|
|
6862
|
+
modelUrl,
|
|
6843
6863
|
tokensUrl: config.tokensUrl,
|
|
6844
6864
|
language: config.language,
|
|
6845
6865
|
textNorm: config.textNorm
|
|
@@ -7706,9 +7726,11 @@ var Wav2ArkitCpuWorker = class {
|
|
|
7706
7726
|
|
|
7707
7727
|
// src/inference/createA2E.ts
|
|
7708
7728
|
var logger12 = createLogger("createA2E");
|
|
7709
|
-
function createA2E(config) {
|
|
7729
|
+
function createA2E(config = {}) {
|
|
7710
7730
|
const mode = config.mode ?? "auto";
|
|
7711
7731
|
const fallbackOnError = config.fallbackOnError ?? true;
|
|
7732
|
+
const gpuModelUrl = config.gpuModelUrl ?? DEFAULT_MODEL_URLS.lam;
|
|
7733
|
+
const cpuModelUrl = config.cpuModelUrl ?? DEFAULT_MODEL_URLS.wav2arkitCpu;
|
|
7712
7734
|
let useCpu;
|
|
7713
7735
|
if (mode === "cpu") {
|
|
7714
7736
|
useCpu = true;
|
|
@@ -7727,23 +7749,24 @@ function createA2E(config) {
|
|
|
7727
7749
|
if (config.unifiedWorker) {
|
|
7728
7750
|
logger12.info("Creating Wav2ArkitCpuUnifiedAdapter (404MB, WASM, shared unified worker)");
|
|
7729
7751
|
return new Wav2ArkitCpuUnifiedAdapter(config.unifiedWorker, {
|
|
7730
|
-
modelUrl:
|
|
7752
|
+
modelUrl: cpuModelUrl
|
|
7731
7753
|
});
|
|
7732
7754
|
}
|
|
7733
7755
|
if (config.useWorker && Wav2ArkitCpuWorker.isSupported() && !isIOS()) {
|
|
7734
7756
|
logger12.info("Creating Wav2ArkitCpuWorker (404MB, WASM, off-main-thread)");
|
|
7735
7757
|
return new Wav2ArkitCpuWorker({
|
|
7736
|
-
modelUrl:
|
|
7758
|
+
modelUrl: cpuModelUrl
|
|
7737
7759
|
});
|
|
7738
7760
|
}
|
|
7739
7761
|
logger12.info("Creating Wav2ArkitCpuInference (404MB, WASM)");
|
|
7740
7762
|
return new Wav2ArkitCpuInference({
|
|
7741
|
-
modelUrl:
|
|
7763
|
+
modelUrl: cpuModelUrl
|
|
7742
7764
|
});
|
|
7743
7765
|
}
|
|
7766
|
+
const gpuExternalDataUrl = config.gpuExternalDataUrl !== void 0 ? config.gpuExternalDataUrl : void 0;
|
|
7744
7767
|
const gpuInstance = new Wav2Vec2Inference({
|
|
7745
|
-
modelUrl:
|
|
7746
|
-
externalDataUrl:
|
|
7768
|
+
modelUrl: gpuModelUrl,
|
|
7769
|
+
externalDataUrl: gpuExternalDataUrl,
|
|
7747
7770
|
backend: config.gpuBackend ?? "auto",
|
|
7748
7771
|
numIdentityClasses: config.numIdentityClasses
|
|
7749
7772
|
});
|
|
@@ -7759,6 +7782,7 @@ var A2EWithFallback = class {
|
|
|
7759
7782
|
this.hasFallenBack = false;
|
|
7760
7783
|
this.implementation = gpuInstance;
|
|
7761
7784
|
this.config = config;
|
|
7785
|
+
this.resolvedCpuModelUrl = config.cpuModelUrl ?? DEFAULT_MODEL_URLS.wav2arkitCpu;
|
|
7762
7786
|
}
|
|
7763
7787
|
get modelId() {
|
|
7764
7788
|
return this.implementation.modelId;
|
|
@@ -7787,17 +7811,17 @@ var A2EWithFallback = class {
|
|
|
7787
7811
|
}
|
|
7788
7812
|
if (this.config.unifiedWorker) {
|
|
7789
7813
|
this.implementation = new Wav2ArkitCpuUnifiedAdapter(this.config.unifiedWorker, {
|
|
7790
|
-
modelUrl: this.
|
|
7814
|
+
modelUrl: this.resolvedCpuModelUrl
|
|
7791
7815
|
});
|
|
7792
7816
|
logger12.info("Fallback to Wav2ArkitCpuUnifiedAdapter successful");
|
|
7793
7817
|
} else if (this.config.useWorker && Wav2ArkitCpuWorker.isSupported() && !isIOS()) {
|
|
7794
7818
|
this.implementation = new Wav2ArkitCpuWorker({
|
|
7795
|
-
modelUrl: this.
|
|
7819
|
+
modelUrl: this.resolvedCpuModelUrl
|
|
7796
7820
|
});
|
|
7797
7821
|
logger12.info("Fallback to Wav2ArkitCpuWorker successful");
|
|
7798
7822
|
} else {
|
|
7799
7823
|
this.implementation = new Wav2ArkitCpuInference({
|
|
7800
|
-
modelUrl: this.
|
|
7824
|
+
modelUrl: this.resolvedCpuModelUrl
|
|
7801
7825
|
});
|
|
7802
7826
|
logger12.info("Fallback to Wav2ArkitCpuInference successful");
|
|
7803
7827
|
}
|
|
@@ -8987,10 +9011,12 @@ function supportsVADWorker() {
|
|
|
8987
9011
|
}
|
|
8988
9012
|
return true;
|
|
8989
9013
|
}
|
|
8990
|
-
function createSileroVAD(config) {
|
|
9014
|
+
function createSileroVAD(config = {}) {
|
|
9015
|
+
const modelUrl = config.modelUrl ?? DEFAULT_MODEL_URLS.sileroVad;
|
|
9016
|
+
const resolvedConfig = { ...config, modelUrl };
|
|
8991
9017
|
if (config.unifiedWorker) {
|
|
8992
9018
|
logger15.info("Creating SileroVADUnifiedAdapter (shared unified worker)");
|
|
8993
|
-
return new SileroVADUnifiedAdapter(config.unifiedWorker,
|
|
9019
|
+
return new SileroVADUnifiedAdapter(config.unifiedWorker, resolvedConfig);
|
|
8994
9020
|
}
|
|
8995
9021
|
const fallbackOnError = config.fallbackOnError ?? true;
|
|
8996
9022
|
let useWorker;
|
|
@@ -9010,24 +9036,24 @@ function createSileroVAD(config) {
|
|
|
9010
9036
|
if (useWorker) {
|
|
9011
9037
|
logger15.info("Creating SileroVADWorker (off-main-thread)");
|
|
9012
9038
|
const worker = new SileroVADWorker({
|
|
9013
|
-
modelUrl
|
|
9039
|
+
modelUrl,
|
|
9014
9040
|
sampleRate: config.sampleRate,
|
|
9015
9041
|
threshold: config.threshold,
|
|
9016
9042
|
preSpeechBufferChunks: config.preSpeechBufferChunks
|
|
9017
9043
|
});
|
|
9018
9044
|
if (fallbackOnError) {
|
|
9019
|
-
return new VADWorkerWithFallback(worker,
|
|
9045
|
+
return new VADWorkerWithFallback(worker, resolvedConfig);
|
|
9020
9046
|
}
|
|
9021
9047
|
return worker;
|
|
9022
9048
|
}
|
|
9023
9049
|
logger15.info("Creating SileroVADInference (main thread)");
|
|
9024
|
-
return new SileroVADInference(
|
|
9050
|
+
return new SileroVADInference(resolvedConfig);
|
|
9025
9051
|
}
|
|
9026
9052
|
var VADWorkerWithFallback = class {
|
|
9027
|
-
constructor(worker,
|
|
9053
|
+
constructor(worker, resolvedConfig) {
|
|
9028
9054
|
this.hasFallenBack = false;
|
|
9029
9055
|
this.implementation = worker;
|
|
9030
|
-
this.
|
|
9056
|
+
this.resolvedConfig = resolvedConfig;
|
|
9031
9057
|
}
|
|
9032
9058
|
get backend() {
|
|
9033
9059
|
if (!this.isLoaded) return null;
|
|
@@ -9053,7 +9079,7 @@ var VADWorkerWithFallback = class {
|
|
|
9053
9079
|
await this.implementation.dispose();
|
|
9054
9080
|
} catch {
|
|
9055
9081
|
}
|
|
9056
|
-
this.implementation = new SileroVADInference(this.
|
|
9082
|
+
this.implementation = new SileroVADInference(this.resolvedConfig);
|
|
9057
9083
|
this.hasFallenBack = true;
|
|
9058
9084
|
logger15.info("Fallback to SileroVADInference successful");
|
|
9059
9085
|
return await this.implementation.load();
|
|
@@ -10101,17 +10127,29 @@ var AnimationGraph = class extends EventEmitter {
|
|
|
10101
10127
|
// src/animation/ProceduralLifeLayer.ts
|
|
10102
10128
|
var import_simplex_noise = require("simplex-noise");
|
|
10103
10129
|
var simplex2d = (0, import_simplex_noise.createNoise2D)();
|
|
10130
|
+
var LIFE_BS_INDEX = /* @__PURE__ */ new Map();
|
|
10131
|
+
for (let i = 0; i < LAM_BLENDSHAPES.length; i++) {
|
|
10132
|
+
LIFE_BS_INDEX.set(LAM_BLENDSHAPES[i], i);
|
|
10133
|
+
}
|
|
10104
10134
|
var PHASE_OPEN = 0;
|
|
10105
10135
|
var PHASE_CLOSING = 1;
|
|
10106
10136
|
var PHASE_CLOSED = 2;
|
|
10107
10137
|
var PHASE_OPENING = 3;
|
|
10108
|
-
var BLINK_CLOSE_DURATION = 0.
|
|
10138
|
+
var BLINK_CLOSE_DURATION = 0.092;
|
|
10109
10139
|
var BLINK_HOLD_DURATION = 0.04;
|
|
10110
|
-
var BLINK_OPEN_DURATION = 0.
|
|
10140
|
+
var BLINK_OPEN_DURATION = 0.242;
|
|
10111
10141
|
var BLINK_ASYMMETRY_DELAY = 8e-3;
|
|
10142
|
+
var BLINK_IBI_MU = Math.log(5.97);
|
|
10143
|
+
var BLINK_IBI_SIGMA = 0.89;
|
|
10112
10144
|
var GAZE_BREAK_DURATION = 0.12;
|
|
10113
10145
|
var GAZE_BREAK_HOLD_DURATION = 0.3;
|
|
10114
10146
|
var GAZE_BREAK_RETURN_DURATION = 0.15;
|
|
10147
|
+
var GAZE_STATE_PARAMS = {
|
|
10148
|
+
idle: { interval: [2, 5], amplitude: [0.15, 0.4] },
|
|
10149
|
+
listening: { interval: [4, 10], amplitude: [0.1, 0.25] },
|
|
10150
|
+
thinking: { interval: [1, 3], amplitude: [0.2, 0.5] },
|
|
10151
|
+
speaking: { interval: [2, 6], amplitude: [0.15, 0.35] }
|
|
10152
|
+
};
|
|
10115
10153
|
var EYE_NOISE_X_FREQ = 0.8;
|
|
10116
10154
|
var EYE_NOISE_Y_FREQ = 0.6;
|
|
10117
10155
|
var EYE_NOISE_X_PHASE = 73.1;
|
|
@@ -10139,6 +10177,12 @@ function smoothStep(t) {
|
|
|
10139
10177
|
function softClamp(v, max) {
|
|
10140
10178
|
return Math.tanh(v / max) * max;
|
|
10141
10179
|
}
|
|
10180
|
+
function sampleLogNormal(mu, sigma) {
|
|
10181
|
+
const u1 = Math.random();
|
|
10182
|
+
const u2 = Math.random();
|
|
10183
|
+
const z = Math.sqrt(-2 * Math.log(u1 || 1e-10)) * Math.cos(2 * Math.PI * u2);
|
|
10184
|
+
return Math.exp(mu + sigma * z);
|
|
10185
|
+
}
|
|
10142
10186
|
var ProceduralLifeLayer = class {
|
|
10143
10187
|
constructor(config) {
|
|
10144
10188
|
// Blink state
|
|
@@ -10151,7 +10195,7 @@ var ProceduralLifeLayer = class {
|
|
|
10151
10195
|
// Eye contact (smoothed)
|
|
10152
10196
|
this.smoothedEyeX = 0;
|
|
10153
10197
|
this.smoothedEyeY = 0;
|
|
10154
|
-
// Eye micro-motion
|
|
10198
|
+
// Eye micro-motion
|
|
10155
10199
|
this.eyeNoiseTime = 0;
|
|
10156
10200
|
// Gaze break state
|
|
10157
10201
|
this.gazeBreakTimer = 0;
|
|
@@ -10161,6 +10205,8 @@ var ProceduralLifeLayer = class {
|
|
|
10161
10205
|
this.gazeBreakTargetY = 0;
|
|
10162
10206
|
this.gazeBreakCurrentX = 0;
|
|
10163
10207
|
this.gazeBreakCurrentY = 0;
|
|
10208
|
+
// Conversational state for gaze
|
|
10209
|
+
this.currentState = null;
|
|
10164
10210
|
// Breathing / postural sway
|
|
10165
10211
|
this.microMotionTime = 0;
|
|
10166
10212
|
this.breathingPhase = 0;
|
|
@@ -10169,6 +10215,7 @@ var ProceduralLifeLayer = class {
|
|
|
10169
10215
|
this.previousEnergy = 0;
|
|
10170
10216
|
this.emphasisLevel = 0;
|
|
10171
10217
|
this.blinkIntervalRange = config?.blinkIntervalRange ?? [2.5, 6];
|
|
10218
|
+
this.useLogNormalBlinks = !config?.blinkIntervalRange;
|
|
10172
10219
|
this.gazeBreakIntervalRange = config?.gazeBreakIntervalRange ?? [3, 8];
|
|
10173
10220
|
this.gazeBreakAmplitudeRange = config?.gazeBreakAmplitudeRange ?? [0.15, 0.4];
|
|
10174
10221
|
this.eyeNoiseAmplitude = config?.eyeNoiseAmplitude ?? 0.06;
|
|
@@ -10178,7 +10225,7 @@ var ProceduralLifeLayer = class {
|
|
|
10178
10225
|
this.posturalSwayAmplitude = config?.posturalSwayAmplitude ?? 2e-3;
|
|
10179
10226
|
this.eyeMaxDeviation = config?.eyeMaxDeviation ?? 0.8;
|
|
10180
10227
|
this.eyeSmoothing = config?.eyeSmoothing ?? 15;
|
|
10181
|
-
this.blinkInterval =
|
|
10228
|
+
this.blinkInterval = this.nextBlinkInterval();
|
|
10182
10229
|
this.gazeBreakInterval = randomRange(...this.gazeBreakIntervalRange);
|
|
10183
10230
|
}
|
|
10184
10231
|
/**
|
|
@@ -10193,6 +10240,7 @@ var ProceduralLifeLayer = class {
|
|
|
10193
10240
|
const eyeTargetY = input?.eyeTargetY ?? 0;
|
|
10194
10241
|
const audioEnergy = input?.audioEnergy ?? 0;
|
|
10195
10242
|
const isSpeaking = input?.isSpeaking ?? false;
|
|
10243
|
+
this.currentState = input?.state ?? null;
|
|
10196
10244
|
const safeDelta = Math.min(delta, 0.1);
|
|
10197
10245
|
const blendshapes = {};
|
|
10198
10246
|
this.updateBlinks(delta);
|
|
@@ -10231,6 +10279,12 @@ var ProceduralLifeLayer = class {
|
|
|
10231
10279
|
const swayAmp = this.posturalSwayAmplitude;
|
|
10232
10280
|
const swayX = Math.sin(this.microMotionTime * 0.7) * swayAmp + Math.sin(this.microMotionTime * 1.3) * swayAmp * 0.5;
|
|
10233
10281
|
const swayY = Math.sin(this.microMotionTime * 0.5) * swayAmp * 0.75 + Math.sin(this.microMotionTime * 0.9) * swayAmp * 0.5;
|
|
10282
|
+
const breathVal = Math.sin(this.breathingPhase);
|
|
10283
|
+
if (breathVal > 0) {
|
|
10284
|
+
blendshapes["jawOpen"] = breathVal * 0.015;
|
|
10285
|
+
blendshapes["noseSneerLeft"] = breathVal * 8e-3;
|
|
10286
|
+
blendshapes["noseSneerRight"] = breathVal * 8e-3;
|
|
10287
|
+
}
|
|
10234
10288
|
return {
|
|
10235
10289
|
blendshapes,
|
|
10236
10290
|
headDelta: {
|
|
@@ -10239,12 +10293,35 @@ var ProceduralLifeLayer = class {
|
|
|
10239
10293
|
}
|
|
10240
10294
|
};
|
|
10241
10295
|
}
|
|
10296
|
+
/**
|
|
10297
|
+
* Write life layer output directly to a Float32Array[52] in LAM_BLENDSHAPES order.
|
|
10298
|
+
*
|
|
10299
|
+
* Includes micro-jitter (0.4% amplitude simplex noise on all channels) to
|
|
10300
|
+
* break uncanny stillness on undriven channels.
|
|
10301
|
+
*
|
|
10302
|
+
* @param delta - Time since last frame in seconds
|
|
10303
|
+
* @param input - Per-frame input
|
|
10304
|
+
* @param out - Pre-allocated Float32Array(52) to write into
|
|
10305
|
+
*/
|
|
10306
|
+
updateToArray(delta, input, out) {
|
|
10307
|
+
out.fill(0);
|
|
10308
|
+
const result = this.update(delta, input);
|
|
10309
|
+
for (const [name, value] of Object.entries(result.blendshapes)) {
|
|
10310
|
+
const idx = LIFE_BS_INDEX.get(name);
|
|
10311
|
+
if (idx !== void 0) {
|
|
10312
|
+
out[idx] = value;
|
|
10313
|
+
}
|
|
10314
|
+
}
|
|
10315
|
+
for (let i = 0; i < 52; i++) {
|
|
10316
|
+
out[i] += simplex2d(this.noiseTime * 0.3, i * 7.13) * 4e-3;
|
|
10317
|
+
}
|
|
10318
|
+
}
|
|
10242
10319
|
/**
|
|
10243
10320
|
* Reset all internal state to initial values.
|
|
10244
10321
|
*/
|
|
10245
10322
|
reset() {
|
|
10246
10323
|
this.blinkTimer = 0;
|
|
10247
|
-
this.blinkInterval =
|
|
10324
|
+
this.blinkInterval = this.nextBlinkInterval();
|
|
10248
10325
|
this.blinkPhase = PHASE_OPEN;
|
|
10249
10326
|
this.blinkProgress = 0;
|
|
10250
10327
|
this.asymmetryRight = 0.97;
|
|
@@ -10261,6 +10338,7 @@ var ProceduralLifeLayer = class {
|
|
|
10261
10338
|
this.gazeBreakTargetY = 0;
|
|
10262
10339
|
this.gazeBreakCurrentX = 0;
|
|
10263
10340
|
this.gazeBreakCurrentY = 0;
|
|
10341
|
+
this.currentState = null;
|
|
10264
10342
|
this.microMotionTime = 0;
|
|
10265
10343
|
this.breathingPhase = 0;
|
|
10266
10344
|
this.noiseTime = 0;
|
|
@@ -10268,6 +10346,21 @@ var ProceduralLifeLayer = class {
|
|
|
10268
10346
|
this.emphasisLevel = 0;
|
|
10269
10347
|
}
|
|
10270
10348
|
// =====================================================================
|
|
10349
|
+
// PRIVATE: Blink interval sampling
|
|
10350
|
+
// =====================================================================
|
|
10351
|
+
/**
|
|
10352
|
+
* Sample next blink interval.
|
|
10353
|
+
* Uses log-normal distribution (PMC3565584) when using default config,
|
|
10354
|
+
* or uniform random when custom blinkIntervalRange is provided.
|
|
10355
|
+
*/
|
|
10356
|
+
nextBlinkInterval() {
|
|
10357
|
+
if (this.useLogNormalBlinks) {
|
|
10358
|
+
const sample = sampleLogNormal(BLINK_IBI_MU, BLINK_IBI_SIGMA);
|
|
10359
|
+
return clamp(sample, 1.5, 12);
|
|
10360
|
+
}
|
|
10361
|
+
return randomRange(...this.blinkIntervalRange);
|
|
10362
|
+
}
|
|
10363
|
+
// =====================================================================
|
|
10271
10364
|
// PRIVATE: Blink system
|
|
10272
10365
|
// =====================================================================
|
|
10273
10366
|
updateBlinks(delta) {
|
|
@@ -10276,7 +10369,7 @@ var ProceduralLifeLayer = class {
|
|
|
10276
10369
|
this.blinkPhase = PHASE_CLOSING;
|
|
10277
10370
|
this.blinkProgress = 0;
|
|
10278
10371
|
this.blinkTimer = 0;
|
|
10279
|
-
this.blinkInterval =
|
|
10372
|
+
this.blinkInterval = this.nextBlinkInterval();
|
|
10280
10373
|
this.asymmetryRight = 0.95 + Math.random() * 0.08;
|
|
10281
10374
|
}
|
|
10282
10375
|
if (this.blinkPhase > PHASE_OPEN) {
|
|
@@ -10332,18 +10425,32 @@ var ProceduralLifeLayer = class {
|
|
|
10332
10425
|
return { x, y };
|
|
10333
10426
|
}
|
|
10334
10427
|
// =====================================================================
|
|
10335
|
-
// PRIVATE: Gaze breaks
|
|
10428
|
+
// PRIVATE: Gaze breaks (state-dependent)
|
|
10336
10429
|
// =====================================================================
|
|
10430
|
+
/**
|
|
10431
|
+
* Get active gaze parameters — uses state-dependent params when
|
|
10432
|
+
* conversational state is provided, otherwise falls back to config ranges.
|
|
10433
|
+
*/
|
|
10434
|
+
getActiveGazeParams() {
|
|
10435
|
+
if (this.currentState && GAZE_STATE_PARAMS[this.currentState]) {
|
|
10436
|
+
return GAZE_STATE_PARAMS[this.currentState];
|
|
10437
|
+
}
|
|
10438
|
+
return {
|
|
10439
|
+
interval: this.gazeBreakIntervalRange,
|
|
10440
|
+
amplitude: this.gazeBreakAmplitudeRange
|
|
10441
|
+
};
|
|
10442
|
+
}
|
|
10337
10443
|
updateGazeBreaks(delta) {
|
|
10338
10444
|
this.gazeBreakTimer += delta;
|
|
10339
10445
|
if (this.gazeBreakTimer >= this.gazeBreakInterval && this.gazeBreakPhase === PHASE_OPEN) {
|
|
10340
10446
|
this.gazeBreakPhase = PHASE_CLOSING;
|
|
10341
10447
|
this.gazeBreakProgress = 0;
|
|
10342
10448
|
this.gazeBreakTimer = 0;
|
|
10343
|
-
const
|
|
10449
|
+
const params = this.getActiveGazeParams();
|
|
10450
|
+
const amp = randomRange(...params.amplitude);
|
|
10344
10451
|
this.gazeBreakTargetX = (Math.random() - 0.5) * 2 * amp;
|
|
10345
10452
|
this.gazeBreakTargetY = (Math.random() - 0.5) * amp * 0.4;
|
|
10346
|
-
this.gazeBreakInterval = randomRange(...
|
|
10453
|
+
this.gazeBreakInterval = randomRange(...params.interval);
|
|
10347
10454
|
}
|
|
10348
10455
|
if (this.gazeBreakPhase > PHASE_OPEN) {
|
|
10349
10456
|
this.gazeBreakProgress += delta;
|
|
@@ -10408,6 +10515,293 @@ var ProceduralLifeLayer = class {
|
|
|
10408
10515
|
}
|
|
10409
10516
|
};
|
|
10410
10517
|
|
|
10518
|
+
// src/face/FACSMapping.ts
|
|
10519
|
+
var EMOTION_TO_AU = {
|
|
10520
|
+
joy: [
|
|
10521
|
+
{ au: "AU6", intensity: 0.7, region: "upper" },
|
|
10522
|
+
// cheek raise (Duchenne)
|
|
10523
|
+
{ au: "AU12", intensity: 0.8, region: "lower" }
|
|
10524
|
+
// lip corner pull (smile)
|
|
10525
|
+
],
|
|
10526
|
+
anger: [
|
|
10527
|
+
{ au: "AU4", intensity: 0.8, region: "upper" },
|
|
10528
|
+
// brow lower
|
|
10529
|
+
{ au: "AU5", intensity: 0.4, region: "upper" },
|
|
10530
|
+
// upper lid raise
|
|
10531
|
+
{ au: "AU7", intensity: 0.3, region: "upper" },
|
|
10532
|
+
// lid tighten
|
|
10533
|
+
{ au: "AU23", intensity: 0.6, region: "lower" }
|
|
10534
|
+
// lip tighten
|
|
10535
|
+
],
|
|
10536
|
+
sadness: [
|
|
10537
|
+
{ au: "AU1", intensity: 0.7, region: "upper" },
|
|
10538
|
+
// inner brow raise
|
|
10539
|
+
{ au: "AU4", intensity: 0.3, region: "upper" },
|
|
10540
|
+
// brow lower (furrow)
|
|
10541
|
+
{ au: "AU15", intensity: 0.5, region: "lower" }
|
|
10542
|
+
// lip corner depress
|
|
10543
|
+
],
|
|
10544
|
+
fear: [
|
|
10545
|
+
{ au: "AU1", intensity: 0.6, region: "upper" },
|
|
10546
|
+
// inner brow raise
|
|
10547
|
+
{ au: "AU2", intensity: 0.5, region: "upper" },
|
|
10548
|
+
// outer brow raise
|
|
10549
|
+
{ au: "AU4", intensity: 0.3, region: "upper" },
|
|
10550
|
+
// brow lower
|
|
10551
|
+
{ au: "AU5", intensity: 0.5, region: "upper" },
|
|
10552
|
+
// upper lid raise
|
|
10553
|
+
{ au: "AU20", intensity: 0.4, region: "lower" }
|
|
10554
|
+
// lip stretch
|
|
10555
|
+
],
|
|
10556
|
+
disgust: [
|
|
10557
|
+
{ au: "AU9", intensity: 0.7, region: "upper" },
|
|
10558
|
+
// nose wrinkle
|
|
10559
|
+
{ au: "AU10", intensity: 0.5, region: "lower" },
|
|
10560
|
+
// upper lip raise
|
|
10561
|
+
{ au: "AU15", intensity: 0.4, region: "lower" }
|
|
10562
|
+
// lip corner depress
|
|
10563
|
+
],
|
|
10564
|
+
amazement: [
|
|
10565
|
+
{ au: "AU1", intensity: 0.6, region: "upper" },
|
|
10566
|
+
// inner brow raise
|
|
10567
|
+
{ au: "AU2", intensity: 0.7, region: "upper" },
|
|
10568
|
+
// outer brow raise
|
|
10569
|
+
{ au: "AU5", intensity: 0.6, region: "upper" },
|
|
10570
|
+
// upper lid raise
|
|
10571
|
+
{ au: "AU26", intensity: 0.4, region: "lower" }
|
|
10572
|
+
// jaw drop
|
|
10573
|
+
],
|
|
10574
|
+
grief: [
|
|
10575
|
+
{ au: "AU1", intensity: 0.8, region: "upper" },
|
|
10576
|
+
// inner brow raise
|
|
10577
|
+
{ au: "AU4", intensity: 0.5, region: "upper" },
|
|
10578
|
+
// brow lower
|
|
10579
|
+
{ au: "AU6", intensity: 0.3, region: "upper" },
|
|
10580
|
+
// cheek raise (grief cry)
|
|
10581
|
+
{ au: "AU15", intensity: 0.6, region: "lower" }
|
|
10582
|
+
// lip corner depress
|
|
10583
|
+
],
|
|
10584
|
+
cheekiness: [
|
|
10585
|
+
{ au: "AU2", intensity: 0.4, region: "upper" },
|
|
10586
|
+
// outer brow raise
|
|
10587
|
+
{ au: "AU6", intensity: 0.4, region: "upper" },
|
|
10588
|
+
// cheek raise
|
|
10589
|
+
{ au: "AU12", intensity: 0.6, region: "lower" }
|
|
10590
|
+
// lip corner pull (smirk)
|
|
10591
|
+
],
|
|
10592
|
+
pain: [
|
|
10593
|
+
{ au: "AU4", intensity: 0.7, region: "upper" },
|
|
10594
|
+
// brow lower
|
|
10595
|
+
{ au: "AU6", intensity: 0.4, region: "upper" },
|
|
10596
|
+
// cheek raise (orbicularis)
|
|
10597
|
+
{ au: "AU7", intensity: 0.7, region: "upper" },
|
|
10598
|
+
// lid tighten (squint)
|
|
10599
|
+
{ au: "AU9", intensity: 0.5, region: "upper" }
|
|
10600
|
+
// nose wrinkle
|
|
10601
|
+
],
|
|
10602
|
+
outofbreath: [
|
|
10603
|
+
{ au: "AU1", intensity: 0.3, region: "upper" },
|
|
10604
|
+
// inner brow raise
|
|
10605
|
+
{ au: "AU25", intensity: 0.3, region: "lower" },
|
|
10606
|
+
// lips part
|
|
10607
|
+
{ au: "AU26", intensity: 0.5, region: "lower" }
|
|
10608
|
+
// jaw drop
|
|
10609
|
+
]
|
|
10610
|
+
};
|
|
10611
|
+
var AU_TO_ARKIT = {
|
|
10612
|
+
"AU1": [{ blendshape: "browInnerUp", weight: 1 }],
|
|
10613
|
+
"AU2": [{ blendshape: "browOuterUpLeft", weight: 1 }, { blendshape: "browOuterUpRight", weight: 1 }],
|
|
10614
|
+
"AU4": [{ blendshape: "browDownLeft", weight: 1 }, { blendshape: "browDownRight", weight: 1 }],
|
|
10615
|
+
"AU5": [{ blendshape: "eyeWideLeft", weight: 1 }, { blendshape: "eyeWideRight", weight: 1 }],
|
|
10616
|
+
"AU6": [{ blendshape: "cheekSquintLeft", weight: 1 }, { blendshape: "cheekSquintRight", weight: 1 }],
|
|
10617
|
+
"AU7": [{ blendshape: "eyeSquintLeft", weight: 1 }, { blendshape: "eyeSquintRight", weight: 1 }],
|
|
10618
|
+
"AU9": [{ blendshape: "noseSneerLeft", weight: 1 }, { blendshape: "noseSneerRight", weight: 1 }],
|
|
10619
|
+
"AU10": [{ blendshape: "mouthUpperUpLeft", weight: 1 }, { blendshape: "mouthUpperUpRight", weight: 1 }],
|
|
10620
|
+
"AU12": [{ blendshape: "mouthSmileLeft", weight: 1 }, { blendshape: "mouthSmileRight", weight: 1 }],
|
|
10621
|
+
"AU15": [{ blendshape: "mouthFrownLeft", weight: 1 }, { blendshape: "mouthFrownRight", weight: 1 }],
|
|
10622
|
+
"AU20": [{ blendshape: "mouthStretchLeft", weight: 1 }, { blendshape: "mouthStretchRight", weight: 1 }],
|
|
10623
|
+
"AU23": [{ blendshape: "mouthPressLeft", weight: 1 }, { blendshape: "mouthPressRight", weight: 1 }],
|
|
10624
|
+
"AU25": [{ blendshape: "jawOpen", weight: 0.3 }],
|
|
10625
|
+
"AU26": [{ blendshape: "jawOpen", weight: 1 }]
|
|
10626
|
+
};
|
|
10627
|
+
var ALL_AUS = [...new Set(
|
|
10628
|
+
Object.values(EMOTION_TO_AU).flatMap((activations) => activations.map((a) => a.au))
|
|
10629
|
+
)];
|
|
10630
|
+
|
|
10631
|
+
// src/face/EmotionResolver.ts
|
|
10632
|
+
var BS_INDEX = /* @__PURE__ */ new Map();
|
|
10633
|
+
for (let i = 0; i < LAM_BLENDSHAPES.length; i++) {
|
|
10634
|
+
BS_INDEX.set(LAM_BLENDSHAPES[i], i);
|
|
10635
|
+
}
|
|
10636
|
+
var EmotionResolver = class {
|
|
10637
|
+
constructor() {
|
|
10638
|
+
this.upperBuffer = new Float32Array(52);
|
|
10639
|
+
this.lowerBuffer = new Float32Array(52);
|
|
10640
|
+
}
|
|
10641
|
+
/**
|
|
10642
|
+
* Resolve emotion weights to upper/lower face blendshape contributions.
|
|
10643
|
+
*
|
|
10644
|
+
* @param weights - Emotion channel weights from EmotionController
|
|
10645
|
+
* @param intensity - Global intensity multiplier (0-2). Default: 1.0
|
|
10646
|
+
* @returns Upper and lower face blendshape arrays (52 channels each)
|
|
10647
|
+
*/
|
|
10648
|
+
resolve(weights, intensity = 1) {
|
|
10649
|
+
const upper = this.upperBuffer;
|
|
10650
|
+
const lower = this.lowerBuffer;
|
|
10651
|
+
upper.fill(0);
|
|
10652
|
+
lower.fill(0);
|
|
10653
|
+
for (const emotionName of EMOTION_NAMES) {
|
|
10654
|
+
const emotionWeight = weights[emotionName];
|
|
10655
|
+
if (!emotionWeight || emotionWeight < 0.01) continue;
|
|
10656
|
+
const auActivations = EMOTION_TO_AU[emotionName];
|
|
10657
|
+
if (!auActivations) continue;
|
|
10658
|
+
for (const activation of auActivations) {
|
|
10659
|
+
const arkitMappings = AU_TO_ARKIT[activation.au];
|
|
10660
|
+
if (!arkitMappings) continue;
|
|
10661
|
+
const target = activation.region === "upper" ? upper : lower;
|
|
10662
|
+
const scale = emotionWeight * activation.intensity * intensity;
|
|
10663
|
+
for (const mapping of arkitMappings) {
|
|
10664
|
+
const idx = BS_INDEX.get(mapping.blendshape);
|
|
10665
|
+
if (idx !== void 0) {
|
|
10666
|
+
target[idx] += mapping.weight * scale;
|
|
10667
|
+
}
|
|
10668
|
+
}
|
|
10669
|
+
}
|
|
10670
|
+
}
|
|
10671
|
+
for (let i = 0; i < 52; i++) {
|
|
10672
|
+
if (upper[i] > 1) upper[i] = 1;
|
|
10673
|
+
if (lower[i] > 1) lower[i] = 1;
|
|
10674
|
+
}
|
|
10675
|
+
return {
|
|
10676
|
+
upper: new Float32Array(upper),
|
|
10677
|
+
lower: new Float32Array(lower)
|
|
10678
|
+
};
|
|
10679
|
+
}
|
|
10680
|
+
};
|
|
10681
|
+
|
|
10682
|
+
// src/face/FaceCompositor.ts
|
|
10683
|
+
function smoothstep(t) {
|
|
10684
|
+
return t * t * (3 - 2 * t);
|
|
10685
|
+
}
|
|
10686
|
+
var BS_INDEX2 = /* @__PURE__ */ new Map();
|
|
10687
|
+
for (let i = 0; i < LAM_BLENDSHAPES.length; i++) {
|
|
10688
|
+
BS_INDEX2.set(LAM_BLENDSHAPES[i], i);
|
|
10689
|
+
}
|
|
10690
|
+
var IDX_MOUTH_CLOSE = BS_INDEX2.get("mouthClose");
|
|
10691
|
+
var IS_EYE_CHANNEL = new Array(52).fill(false);
|
|
10692
|
+
for (const name of LAM_BLENDSHAPES) {
|
|
10693
|
+
if (name.startsWith("eyeBlink") || name.startsWith("eyeLook")) {
|
|
10694
|
+
IS_EYE_CHANNEL[BS_INDEX2.get(name)] = true;
|
|
10695
|
+
}
|
|
10696
|
+
}
|
|
10697
|
+
var FaceCompositor = class {
|
|
10698
|
+
constructor(config) {
|
|
10699
|
+
this.emotionResolver = new EmotionResolver();
|
|
10700
|
+
// Pre-allocated buffers
|
|
10701
|
+
this.smoothedUpper = new Float32Array(52);
|
|
10702
|
+
this.smoothedLower = new Float32Array(52);
|
|
10703
|
+
this.lifeBuffer = new Float32Array(52);
|
|
10704
|
+
// Profile arrays (pre-expanded to 52 channels)
|
|
10705
|
+
this.multiplier = new Float32Array(52).fill(1);
|
|
10706
|
+
this.offset = new Float32Array(52);
|
|
10707
|
+
this.lifeLayer = config?.lifeLayer ?? new ProceduralLifeLayer();
|
|
10708
|
+
this.emotionSmoothing = config?.emotionSmoothing ?? 0.12;
|
|
10709
|
+
if (config?.profile) {
|
|
10710
|
+
this.applyProfileArrays(config.profile);
|
|
10711
|
+
}
|
|
10712
|
+
}
|
|
10713
|
+
/**
|
|
10714
|
+
* Compose a single output frame from the 5-stage signal chain.
|
|
10715
|
+
*
|
|
10716
|
+
* @param base - A2E raw output (Float32Array[52], LAM_BLENDSHAPES order)
|
|
10717
|
+
* @param input - Per-frame input (deltaTime, emotion, life layer params)
|
|
10718
|
+
* @returns Float32Array[52] with all values clamped to [0, 1]
|
|
10719
|
+
*/
|
|
10720
|
+
compose(base, input) {
|
|
10721
|
+
const out = new Float32Array(52);
|
|
10722
|
+
out.set(base);
|
|
10723
|
+
const emotion = input.emotion ?? this.stickyEmotion;
|
|
10724
|
+
if (emotion) {
|
|
10725
|
+
const resolved = this.emotionResolver.resolve(
|
|
10726
|
+
emotion,
|
|
10727
|
+
input.emotionIntensity ?? 1
|
|
10728
|
+
);
|
|
10729
|
+
const k = this.emotionSmoothing;
|
|
10730
|
+
for (let i = 0; i < 52; i++) {
|
|
10731
|
+
this.smoothedUpper[i] += (resolved.upper[i] - this.smoothedUpper[i]) * k;
|
|
10732
|
+
this.smoothedLower[i] += (resolved.lower[i] - this.smoothedLower[i]) * k;
|
|
10733
|
+
}
|
|
10734
|
+
const mc = base[IDX_MOUTH_CLOSE];
|
|
10735
|
+
const bilabialSuppress = mc <= 0.3 ? 1 : mc >= 0.7 ? 0.1 : 1 - 0.9 * smoothstep((mc - 0.3) * 2.5);
|
|
10736
|
+
for (let i = 0; i < 52; i++) {
|
|
10737
|
+
out[i] += this.smoothedUpper[i];
|
|
10738
|
+
}
|
|
10739
|
+
for (let i = 0; i < 52; i++) {
|
|
10740
|
+
out[i] *= 1 + this.smoothedLower[i] * bilabialSuppress;
|
|
10741
|
+
}
|
|
10742
|
+
}
|
|
10743
|
+
this.lifeLayer.updateToArray(input.deltaTime, input, this.lifeBuffer);
|
|
10744
|
+
for (let i = 0; i < 52; i++) {
|
|
10745
|
+
if (IS_EYE_CHANNEL[i]) {
|
|
10746
|
+
out[i] = this.lifeBuffer[i];
|
|
10747
|
+
} else {
|
|
10748
|
+
out[i] += this.lifeBuffer[i];
|
|
10749
|
+
}
|
|
10750
|
+
}
|
|
10751
|
+
for (let i = 0; i < 52; i++) {
|
|
10752
|
+
out[i] = out[i] * this.multiplier[i] + this.offset[i];
|
|
10753
|
+
}
|
|
10754
|
+
for (let i = 0; i < 52; i++) {
|
|
10755
|
+
if (out[i] < 0) out[i] = 0;
|
|
10756
|
+
else if (out[i] > 1) out[i] = 1;
|
|
10757
|
+
}
|
|
10758
|
+
return out;
|
|
10759
|
+
}
|
|
10760
|
+
/**
|
|
10761
|
+
* Set sticky emotion (used when input.emotion is not provided).
|
|
10762
|
+
*/
|
|
10763
|
+
setEmotion(weights) {
|
|
10764
|
+
this.stickyEmotion = weights;
|
|
10765
|
+
}
|
|
10766
|
+
/**
|
|
10767
|
+
* Update character profile at runtime.
|
|
10768
|
+
*/
|
|
10769
|
+
setProfile(profile) {
|
|
10770
|
+
this.multiplier.fill(1);
|
|
10771
|
+
this.offset.fill(0);
|
|
10772
|
+
this.applyProfileArrays(profile);
|
|
10773
|
+
}
|
|
10774
|
+
/**
|
|
10775
|
+
* Reset all smoothing state and life layer.
|
|
10776
|
+
*/
|
|
10777
|
+
reset() {
|
|
10778
|
+
this.smoothedUpper.fill(0);
|
|
10779
|
+
this.smoothedLower.fill(0);
|
|
10780
|
+
this.lifeBuffer.fill(0);
|
|
10781
|
+
this.stickyEmotion = void 0;
|
|
10782
|
+
this.lifeLayer.reset();
|
|
10783
|
+
}
|
|
10784
|
+
/** Expand partial profile maps into dense Float32Arrays */
|
|
10785
|
+
applyProfileArrays(profile) {
|
|
10786
|
+
if (profile.multiplier) {
|
|
10787
|
+
for (const [name, value] of Object.entries(profile.multiplier)) {
|
|
10788
|
+
const idx = BS_INDEX2.get(name);
|
|
10789
|
+
if (idx !== void 0 && value !== void 0) {
|
|
10790
|
+
this.multiplier[idx] = value;
|
|
10791
|
+
}
|
|
10792
|
+
}
|
|
10793
|
+
}
|
|
10794
|
+
if (profile.offset) {
|
|
10795
|
+
for (const [name, value] of Object.entries(profile.offset)) {
|
|
10796
|
+
const idx = BS_INDEX2.get(name);
|
|
10797
|
+
if (idx !== void 0 && value !== void 0) {
|
|
10798
|
+
this.offset[idx] = value;
|
|
10799
|
+
}
|
|
10800
|
+
}
|
|
10801
|
+
}
|
|
10802
|
+
}
|
|
10803
|
+
};
|
|
10804
|
+
|
|
10411
10805
|
// src/orchestration/MicLipSync.ts
|
|
10412
10806
|
var logger18 = createLogger("MicLipSync");
|
|
10413
10807
|
var MicLipSync = class extends EventEmitter {
|