@npcify/web 0.1.4 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +34 -1
- package/dist/index.js +250 -21
- package/package.json +3 -3
package/dist/index.d.ts
CHANGED
|
@@ -80,8 +80,16 @@ interface ConversationWorldline {
|
|
|
80
80
|
interface ConversationTimelineEntry {
|
|
81
81
|
name: string;
|
|
82
82
|
timestamp: number;
|
|
83
|
+
/** Optional measured/estimated release time, local to the speech audio. */
|
|
84
|
+
endTime?: number;
|
|
83
85
|
weight?: number;
|
|
84
86
|
}
|
|
87
|
+
interface PhonemeTiming {
|
|
88
|
+
phoneme: string;
|
|
89
|
+
startTime: number;
|
|
90
|
+
endTime: number;
|
|
91
|
+
confidence: number;
|
|
92
|
+
}
|
|
85
93
|
interface WordTiming {
|
|
86
94
|
word: string;
|
|
87
95
|
startTime: number;
|
|
@@ -146,6 +154,12 @@ interface StagingSpeechBlock {
|
|
|
146
154
|
wordTimings: WordTiming[];
|
|
147
155
|
/** Exact provider or deterministic timings, local to this audio block. */
|
|
148
156
|
characterTimings?: CharacterTiming[];
|
|
157
|
+
/** Audio-derived estimates on a 20 ms frame grid, when available. */
|
|
158
|
+
phonemeTimings?: PhonemeTiming[];
|
|
159
|
+
/** Decoded audio duration, when acoustic timing was available. */
|
|
160
|
+
audioDuration?: number;
|
|
161
|
+
/** The last live speech block can grow until synthesis and timing finish. */
|
|
162
|
+
pending?: boolean;
|
|
149
163
|
}
|
|
150
164
|
interface StagingPauseBlock {
|
|
151
165
|
id: string;
|
|
@@ -214,6 +228,10 @@ interface StagingStreamPrefixEvent extends Omit<StagingStreamTimelineEvent, 'typ
|
|
|
214
228
|
type: 'staging_prefix';
|
|
215
229
|
complete: false;
|
|
216
230
|
}
|
|
231
|
+
/** Older clients ignore this event and still receive immutable prefixes. */
|
|
232
|
+
interface StagingStreamLiveEvent extends Omit<StagingStreamPrefixEvent, 'type'> {
|
|
233
|
+
type: 'staging_live';
|
|
234
|
+
}
|
|
217
235
|
interface StagingStreamSpeechChunkEvent {
|
|
218
236
|
type: 'speech_chunk';
|
|
219
237
|
taskId: string;
|
|
@@ -259,7 +277,7 @@ interface StagingStreamBillingEvent {
|
|
|
259
277
|
usageCharge?: UsageChargeSummary | null;
|
|
260
278
|
availablePoints?: number;
|
|
261
279
|
}
|
|
262
|
-
type StagingStreamEvent = StagingStreamStartEvent | StagingStreamTextDeltaEvent | StagingStreamTimelineEvent | StagingStreamPrefixEvent | StagingStreamSpeechChunkEvent | StagingStreamSpeechCompleteEvent | StagingStreamSpeechErrorEvent | StagingStreamOptionsEvent | StagingStreamErrorEvent | StagingStreamBillingEvent;
|
|
280
|
+
type StagingStreamEvent = StagingStreamStartEvent | StagingStreamTextDeltaEvent | StagingStreamTimelineEvent | StagingStreamPrefixEvent | StagingStreamLiveEvent | StagingStreamSpeechChunkEvent | StagingStreamSpeechCompleteEvent | StagingStreamSpeechErrorEvent | StagingStreamOptionsEvent | StagingStreamErrorEvent | StagingStreamBillingEvent;
|
|
263
281
|
|
|
264
282
|
declare const BAKED_CONVERSATION_VERSION: 1;
|
|
265
283
|
/** Portable, language-neutral asset. Audio is embedded binary, never a signed URL. */
|
|
@@ -620,6 +638,14 @@ declare class ClientTelemetry {
|
|
|
620
638
|
}): void;
|
|
621
639
|
private readonly createTurnSpan;
|
|
622
640
|
private readonly sendClientLatency;
|
|
641
|
+
reportPhonemePlayback(params: {
|
|
642
|
+
taskId: string;
|
|
643
|
+
stagingId: string;
|
|
644
|
+
speechId: string;
|
|
645
|
+
phonemeCount: number;
|
|
646
|
+
latePhonemeCount: number;
|
|
647
|
+
phonemeLateMs: number;
|
|
648
|
+
}): void;
|
|
623
649
|
}
|
|
624
650
|
|
|
625
651
|
declare class VisemePlayer {
|
|
@@ -640,6 +666,7 @@ declare class VisemePlayer {
|
|
|
640
666
|
attachAudioElement(element: HTMLAudioElement | null): void;
|
|
641
667
|
setVisemes(visemes: ConversationTimelineEntry[]): void;
|
|
642
668
|
cleanup(): void;
|
|
669
|
+
updateVisemes(visemes: ConversationTimelineEntry[]): void;
|
|
643
670
|
capturePerformanceState(): void;
|
|
644
671
|
getAudioPerformanceSnapshot(): AudioPerformanceFrameSnapshot;
|
|
645
672
|
getPerformanceSnapshot(): VisemePerformanceFrameSnapshot;
|
|
@@ -678,6 +705,7 @@ declare class AudioPlayer {
|
|
|
678
705
|
type: 'silent';
|
|
679
706
|
}>, visemes: ConversationTimelineEntry[], options?: SpeechPlaybackOptions): Promise<void>;
|
|
680
707
|
getCurrentTime(): number;
|
|
708
|
+
updateVisemes(visemes: ConversationTimelineEntry[]): void;
|
|
681
709
|
stop(): void;
|
|
682
710
|
reset(): void;
|
|
683
711
|
cleanup(): void;
|
|
@@ -699,6 +727,11 @@ type StagingPlaybackOptions = {
|
|
|
699
727
|
onCharacter?: (speech: StagingSpeechBlock, characterIndex: number, character: string) => void;
|
|
700
728
|
onSpeechStart?: (speech: StagingSpeechBlock) => void;
|
|
701
729
|
onTimelineStart?: (timeline: StagingTimeline) => void;
|
|
730
|
+
onPhonemePlayback?: (speech: StagingSpeechBlock, stats: {
|
|
731
|
+
phonemeCount: number;
|
|
732
|
+
latePhonemeCount: number;
|
|
733
|
+
phonemeLateMs: number;
|
|
734
|
+
}) => void;
|
|
702
735
|
};
|
|
703
736
|
declare class StagingPlayer {
|
|
704
737
|
private readonly audioPlayer;
|
package/dist/index.js
CHANGED
|
@@ -2682,7 +2682,11 @@ var hasDialogueShape = function(value, duration, ids) {
|
|
|
2682
2682
|
ids.add(beat.id);
|
|
2683
2683
|
previousBeatEnd = Number(beat.endTime);
|
|
2684
2684
|
return true;
|
|
2685
|
-
}) && isSameTime(previousBeatEnd, Number(value.endTime)) && value.beats.map((beat) => beat.text).join("") === value.text && Array.isArray(value.visemes) && value.visemes.every((viseme) => hasVisemeShape(viseme, speechDuration)) &&
|
|
2685
|
+
}) && isSameTime(previousBeatEnd, Number(value.endTime)) && value.beats.map((beat) => beat.text).join("") === value.text && Array.isArray(value.visemes) && value.visemes.every((viseme) => hasVisemeShape(viseme, speechDuration)) && (value.pending === void 0 || typeof value.pending === "boolean") && (value.audioDuration === void 0 || isFiniteNonNegative(value.audioDuration) && isSameTime(value.audioDuration, speechDuration)) && (value.phonemeTimings === void 0 || Array.isArray(value.phonemeTimings) && hasOrderedItems(
|
|
2686
|
+
value.phonemeTimings,
|
|
2687
|
+
speechDuration,
|
|
2688
|
+
(phone) => isNonEmptyString(phone.phoneme) && Number(phone.endTime) > Number(phone.startTime) && isFiniteNonNegative(phone.confidence) && phone.confidence <= 1
|
|
2689
|
+
)) && Array.isArray(value.wordTimings) && value.wordTimings.every(
|
|
2686
2690
|
(word) => hasWordTimingShape(word, speechDuration)
|
|
2687
2691
|
) && (value.characterTimings === void 0 || Array.isArray(value.characterTimings) && hasCharacterTimingSequence(value.characterTimings, speechDuration));
|
|
2688
2692
|
};
|
|
@@ -2708,7 +2712,7 @@ var hasAudioShape = function(value) {
|
|
|
2708
2712
|
return false;
|
|
2709
2713
|
};
|
|
2710
2714
|
var hasVisemeShape = function(value, duration) {
|
|
2711
|
-
return isRecord(value) && isNonEmptyString(value.name) && isFiniteNonNegative(value.timestamp) && isWithinDuration(value.timestamp, duration) && (value.weight === void 0 || isFiniteNonNegative(value.weight) && value.weight <= 1);
|
|
2715
|
+
return isRecord(value) && isNonEmptyString(value.name) && isFiniteNonNegative(value.timestamp) && isWithinDuration(value.timestamp, duration) && (value.endTime === void 0 || isFiniteNonNegative(value.endTime) && value.endTime >= value.timestamp && isWithinDuration(value.endTime, duration)) && (value.weight === void 0 || isFiniteNonNegative(value.weight) && value.weight <= 1);
|
|
2712
2716
|
};
|
|
2713
2717
|
var hasWordTimingShape = function(value, duration) {
|
|
2714
2718
|
return isRecord(value) && isNonEmptyString(value.word) && isFiniteNonNegative(value.startTime) && isFiniteNonNegative(value.endTime) && value.endTime >= value.startTime && isWithinDuration(value.endTime, duration);
|
|
@@ -2751,8 +2755,8 @@ var hasTimelineShape = function(value) {
|
|
|
2751
2755
|
}
|
|
2752
2756
|
const ids = /* @__PURE__ */ new Set();
|
|
2753
2757
|
let previousDialogueEnd = 0;
|
|
2754
|
-
return value.dialogue.length > 0 && value.dialogue.every((block) => {
|
|
2755
|
-
if (!isRecord(block) || !isSameTime(Number(block.startTime), previousDialogueEnd) || !hasDialogueShape(block, value.duration, ids)) {
|
|
2758
|
+
return value.dialogue.length > 0 && value.dialogue.every((block, index) => {
|
|
2759
|
+
if (!isRecord(block) || !isSameTime(Number(block.startTime), previousDialogueEnd) || !hasDialogueShape(block, value.duration, ids) || block.pending === true && index !== value.dialogue.length - 1) {
|
|
2756
2760
|
return false;
|
|
2757
2761
|
}
|
|
2758
2762
|
previousDialogueEnd = Number(block.endTime);
|
|
@@ -3477,10 +3481,15 @@ var AVATAR_COLOR_SETTINGS = {
|
|
|
3477
3481
|
toneMapping: NoToneMapping,
|
|
3478
3482
|
outputColorSpace: SRGBColorSpace
|
|
3479
3483
|
};
|
|
3484
|
+
var AVATAR_LIGHTS = {
|
|
3485
|
+
ambientIntensity: 0.7 * Math.PI / 1.9,
|
|
3486
|
+
directionalIntensity: 1.2 * Math.PI / 1.9,
|
|
3487
|
+
directionalPosition: [1, 1.5, 1]
|
|
3488
|
+
};
|
|
3480
3489
|
var addAvatarLights = function(scene, {
|
|
3481
|
-
ambientIntensity =
|
|
3482
|
-
directionalIntensity =
|
|
3483
|
-
directionalPosition = [
|
|
3490
|
+
ambientIntensity = AVATAR_LIGHTS.ambientIntensity,
|
|
3491
|
+
directionalIntensity = AVATAR_LIGHTS.directionalIntensity,
|
|
3492
|
+
directionalPosition = [...AVATAR_LIGHTS.directionalPosition]
|
|
3484
3493
|
} = {}) {
|
|
3485
3494
|
scene.add(new AmbientLight(16777215, ambientIntensity));
|
|
3486
3495
|
const key = new DirectionalLight(16777215, directionalIntensity);
|
|
@@ -3829,6 +3838,12 @@ import {
|
|
|
3829
3838
|
Texture as Texture2
|
|
3830
3839
|
} from "three";
|
|
3831
3840
|
|
|
3841
|
+
// ../vrm-renderer/src/generated-character.ts
|
|
3842
|
+
import {
|
|
3843
|
+
MToonMaterial as MToonMaterial2,
|
|
3844
|
+
VRMExpressionMaterialColorBind
|
|
3845
|
+
} from "@pixiv/three-vrm";
|
|
3846
|
+
|
|
3832
3847
|
// ../vrm-renderer/src/texture-expressions.ts
|
|
3833
3848
|
import {
|
|
3834
3849
|
MToonMaterial,
|
|
@@ -3847,6 +3862,9 @@ var HeldMouthTimeline = class _HeldMouthTimeline {
|
|
|
3847
3862
|
shownAt = -Infinity;
|
|
3848
3863
|
previousTime = -Infinity;
|
|
3849
3864
|
static endTime(cue, next, speechEnd = Infinity) {
|
|
3865
|
+
if (cue.endTime !== void 0 && Number.isFinite(cue.endTime)) {
|
|
3866
|
+
return Math.min(cue.endTime, next?.timestamp ?? Infinity, speechEnd);
|
|
3867
|
+
}
|
|
3850
3868
|
const nextTime = next?.timestamp ?? speechEnd;
|
|
3851
3869
|
return Math.min(
|
|
3852
3870
|
speechEnd,
|
|
@@ -3857,6 +3875,10 @@ var HeldMouthTimeline = class _HeldMouthTimeline {
|
|
|
3857
3875
|
this.cues = [...cues].sort((a, b) => a.timestamp - b.timestamp);
|
|
3858
3876
|
this.reset();
|
|
3859
3877
|
}
|
|
3878
|
+
updateCues(cues) {
|
|
3879
|
+
this.cues = [...cues].sort((a, b) => a.timestamp - b.timestamp);
|
|
3880
|
+
this.next = 0;
|
|
3881
|
+
}
|
|
3860
3882
|
reset() {
|
|
3861
3883
|
this.next = 0;
|
|
3862
3884
|
this.shown = null;
|
|
@@ -3957,13 +3979,9 @@ var findAtlasBindings = function(vrm) {
|
|
|
3957
3979
|
var hasTextureMouthExpressions = function(manager) {
|
|
3958
3980
|
return !!manager && installed.has(manager);
|
|
3959
3981
|
};
|
|
3960
|
-
var
|
|
3982
|
+
var installTextureMouthPlayback = function(vrm, bindings) {
|
|
3961
3983
|
const manager = vrm.expressionManager;
|
|
3962
|
-
if (!manager || installed.has(manager)) {
|
|
3963
|
-
return;
|
|
3964
|
-
}
|
|
3965
|
-
const bindings = findAtlasBindings(vrm);
|
|
3966
|
-
if (!bindings.length) {
|
|
3984
|
+
if (!manager || installed.has(manager) || !bindings.length) {
|
|
3967
3985
|
return;
|
|
3968
3986
|
}
|
|
3969
3987
|
installed.add(manager);
|
|
@@ -4009,6 +4027,75 @@ var enableTextureExpressionPlayback = function(vrm) {
|
|
|
4009
4027
|
}
|
|
4010
4028
|
};
|
|
4011
4029
|
|
|
4030
|
+
// ../vrm-renderer/src/generated-character.ts
|
|
4031
|
+
var TOLERANCE = 1e-4;
|
|
4032
|
+
var prepared = /* @__PURE__ */ new WeakMap();
|
|
4033
|
+
var sameColour = function(a, b) {
|
|
4034
|
+
return Math.abs(a.r - b.r) < TOLERANCE && Math.abs(a.g - b.g) < TOLERANCE && Math.abs(a.b - b.b) < TOLERANCE;
|
|
4035
|
+
};
|
|
4036
|
+
var sceneMToonMaterials = function(scene) {
|
|
4037
|
+
const materials = /* @__PURE__ */ new Set();
|
|
4038
|
+
scene.traverse((object) => {
|
|
4039
|
+
const material = object.material;
|
|
4040
|
+
for (const entry of Array.isArray(material) ? material : [material]) {
|
|
4041
|
+
if (entry instanceof MToonMaterial2) {
|
|
4042
|
+
materials.add(entry);
|
|
4043
|
+
}
|
|
4044
|
+
}
|
|
4045
|
+
});
|
|
4046
|
+
return materials;
|
|
4047
|
+
};
|
|
4048
|
+
var sharesFaceRecipe = function(material, faces) {
|
|
4049
|
+
return !!material.map && material.shadeMultiplyTexture?.image === material.map.image && faces.some(
|
|
4050
|
+
(face) => sameColour(material.color, face.color) && sameColour(material.shadeColorFactor, face.shadeColorFactor)
|
|
4051
|
+
);
|
|
4052
|
+
};
|
|
4053
|
+
var detectGeneratedCharacter = function(vrm) {
|
|
4054
|
+
const known = prepared.get(vrm);
|
|
4055
|
+
if (known) {
|
|
4056
|
+
return known;
|
|
4057
|
+
}
|
|
4058
|
+
const mouths = findAtlasBindings(vrm);
|
|
4059
|
+
if (!mouths.length) {
|
|
4060
|
+
return null;
|
|
4061
|
+
}
|
|
4062
|
+
const faces = [
|
|
4063
|
+
...new Set(mouths.map(({ bind }) => bind.material))
|
|
4064
|
+
];
|
|
4065
|
+
const coloured = /* @__PURE__ */ new Set();
|
|
4066
|
+
for (const expression of vrm.expressionManager?.expressions ?? []) {
|
|
4067
|
+
for (const bind of expression.binds) {
|
|
4068
|
+
if (bind instanceof VRMExpressionMaterialColorBind) {
|
|
4069
|
+
coloured.add(bind.material);
|
|
4070
|
+
}
|
|
4071
|
+
}
|
|
4072
|
+
}
|
|
4073
|
+
const materials = [...sceneMToonMaterials(vrm.scene)].filter(
|
|
4074
|
+
(material) => !coloured.has(material) && (faces.includes(material) || sharesFaceRecipe(material, faces))
|
|
4075
|
+
);
|
|
4076
|
+
return { faces, mouths, materials };
|
|
4077
|
+
};
|
|
4078
|
+
var normaliseLitFactor = function(material) {
|
|
4079
|
+
const { r, g, b } = material.color;
|
|
4080
|
+
if (!(r > 0) || r >= 1 - TOLERANCE || Math.abs(r - g) >= TOLERANCE || Math.abs(r - b) >= TOLERANCE) {
|
|
4081
|
+
return;
|
|
4082
|
+
}
|
|
4083
|
+
material.color.multiplyScalar(1 / r);
|
|
4084
|
+
material.shadeColorFactor.multiplyScalar(1 / r);
|
|
4085
|
+
};
|
|
4086
|
+
var prepareGeneratedCharacter = function(vrm) {
|
|
4087
|
+
const character = detectGeneratedCharacter(vrm);
|
|
4088
|
+
if (!character || prepared.has(vrm)) {
|
|
4089
|
+
return character;
|
|
4090
|
+
}
|
|
4091
|
+
for (const material of character.materials) {
|
|
4092
|
+
normaliseLitFactor(material);
|
|
4093
|
+
}
|
|
4094
|
+
installTextureMouthPlayback(vrm, character.mouths);
|
|
4095
|
+
prepared.set(vrm, character);
|
|
4096
|
+
return character;
|
|
4097
|
+
};
|
|
4098
|
+
|
|
4012
4099
|
// ../vrm-renderer/src/session.ts
|
|
4013
4100
|
import { VRMUtils as VRMUtils2 } from "@pixiv/three-vrm";
|
|
4014
4101
|
import { AnimationMixer, LoopOnce, LoopRepeat } from "three";
|
|
@@ -4333,7 +4420,7 @@ var loadVrm = async ({
|
|
|
4333
4420
|
});
|
|
4334
4421
|
VRMUtils4.removeUnnecessaryVertices(gltf.scene);
|
|
4335
4422
|
VRMUtils4.combineSkeletons(gltf.scene);
|
|
4336
|
-
|
|
4423
|
+
prepareGeneratedCharacter(gltf.userData.vrm);
|
|
4337
4424
|
return gltf.userData.vrm;
|
|
4338
4425
|
};
|
|
4339
4426
|
var loadVrma = async ({
|
|
@@ -4917,6 +5004,9 @@ var AudioPlayer = class {
|
|
|
4917
5004
|
getCurrentTime() {
|
|
4918
5005
|
return this.audioElement?.currentTime ?? 0;
|
|
4919
5006
|
}
|
|
5007
|
+
updateVisemes(visemes) {
|
|
5008
|
+
this.visemePlayer.updateVisemes(visemes);
|
|
5009
|
+
}
|
|
4920
5010
|
stop() {
|
|
4921
5011
|
this.audioElement?.pause();
|
|
4922
5012
|
}
|
|
@@ -5744,11 +5834,21 @@ var IncrementalStaging = class {
|
|
|
5744
5834
|
}
|
|
5745
5835
|
if (timeline.dialogue.length < this.timeline.dialogue.length || this.timeline.dialogue.some((block, index) => {
|
|
5746
5836
|
const next = timeline.dialogue[index];
|
|
5837
|
+
if (block.type === "speech" && block.pending && next?.type === "speech") {
|
|
5838
|
+
return block.id !== next.id || block.text !== next.text || block.startTime !== next.startTime || next.pending && next.endTime < block.endTime;
|
|
5839
|
+
}
|
|
5747
5840
|
return !next || dialogueKey(block) !== dialogueKey(next);
|
|
5748
5841
|
})) {
|
|
5749
5842
|
throw new Error("A staging revision changed published dialogue");
|
|
5750
5843
|
}
|
|
5751
|
-
|
|
5844
|
+
const dialogue = timeline.dialogue.map((next, index) => {
|
|
5845
|
+
const previous = this.timeline.dialogue[index];
|
|
5846
|
+
if (previous?.type === "speech" && previous.pending && next.type === "speech") {
|
|
5847
|
+
return Object.assign(previous, next, { pending: next.pending });
|
|
5848
|
+
}
|
|
5849
|
+
return next;
|
|
5850
|
+
});
|
|
5851
|
+
Object.assign(this.timeline, timeline, { dialogue });
|
|
5752
5852
|
this.complete = complete;
|
|
5753
5853
|
for (const notify of this.listeners) {
|
|
5754
5854
|
notify();
|
|
@@ -5781,6 +5881,29 @@ var IncrementalStaging = class {
|
|
|
5781
5881
|
error.name = "AbortError";
|
|
5782
5882
|
throw error;
|
|
5783
5883
|
}
|
|
5884
|
+
async finishBlock(index, signal) {
|
|
5885
|
+
while (!signal.aborted) {
|
|
5886
|
+
const block = this.timeline.dialogue[index];
|
|
5887
|
+
if (block?.type !== "speech" || !block.pending) {
|
|
5888
|
+
return;
|
|
5889
|
+
}
|
|
5890
|
+
await new Promise((resolve) => {
|
|
5891
|
+
const done = () => {
|
|
5892
|
+
this.listeners.delete(done);
|
|
5893
|
+
signal.removeEventListener("abort", done);
|
|
5894
|
+
resolve();
|
|
5895
|
+
};
|
|
5896
|
+
this.listeners.add(done);
|
|
5897
|
+
signal.addEventListener("abort", done, { once: true });
|
|
5898
|
+
if (signal.aborted || !block.pending) {
|
|
5899
|
+
done();
|
|
5900
|
+
}
|
|
5901
|
+
});
|
|
5902
|
+
}
|
|
5903
|
+
const error = new Error("Staging playback was cancelled");
|
|
5904
|
+
error.name = "AbortError";
|
|
5905
|
+
throw error;
|
|
5906
|
+
}
|
|
5784
5907
|
};
|
|
5785
5908
|
|
|
5786
5909
|
// ../../clients/web-client/src/character/staging-character-cues.ts
|
|
@@ -5798,7 +5921,7 @@ var findCharacterSequence = function(characters, sequence, fromIndex) {
|
|
|
5798
5921
|
}
|
|
5799
5922
|
return -1;
|
|
5800
5923
|
};
|
|
5801
|
-
var buildStagingCharacterCues = function(text, wordTimings, duration, characterTimings = []) {
|
|
5924
|
+
var buildStagingCharacterCues = function(text, wordTimings, duration, characterTimings = [], complete = true) {
|
|
5802
5925
|
const characters = Array.from(text.trim());
|
|
5803
5926
|
if (characters.length === 0) {
|
|
5804
5927
|
return [];
|
|
@@ -5847,7 +5970,7 @@ var buildStagingCharacterCues = function(text, wordTimings, duration, characterT
|
|
|
5847
5970
|
if (startTimes.every((startTime) => startTime === null)) {
|
|
5848
5971
|
return characters.map((character, index2) => ({
|
|
5849
5972
|
character,
|
|
5850
|
-
startTime: safeDuration * index2 / characters.length
|
|
5973
|
+
startTime: complete ? safeDuration * index2 / characters.length : Infinity
|
|
5851
5974
|
}));
|
|
5852
5975
|
}
|
|
5853
5976
|
let index = 0;
|
|
@@ -5861,6 +5984,12 @@ var buildStagingCharacterCues = function(text, wordTimings, duration, characterT
|
|
|
5861
5984
|
index += 1;
|
|
5862
5985
|
}
|
|
5863
5986
|
const gapEnd = index;
|
|
5987
|
+
if (!complete && gapEnd === startTimes.length) {
|
|
5988
|
+
for (let tail = gapStart; tail < gapEnd; tail += 1) {
|
|
5989
|
+
startTimes[tail] = Infinity;
|
|
5990
|
+
}
|
|
5991
|
+
continue;
|
|
5992
|
+
}
|
|
5864
5993
|
const lowerBound = gapStart === 0 ? 0 : startTimes[gapStart - 1] ?? 0;
|
|
5865
5994
|
const upperBound = gapEnd < startTimes.length ? startTimes[gapEnd] ?? safeDuration : safeDuration;
|
|
5866
5995
|
const gapLength = gapEnd - gapStart;
|
|
@@ -5991,6 +6120,7 @@ var StagingPlayer = class {
|
|
|
5991
6120
|
options
|
|
5992
6121
|
);
|
|
5993
6122
|
}
|
|
6123
|
+
await incremental.finishBlock(index, controller.signal);
|
|
5994
6124
|
cursor = block.endTime;
|
|
5995
6125
|
}
|
|
5996
6126
|
if (timeline.duration > cursor) {
|
|
@@ -6025,6 +6155,12 @@ var StagingPlayer = class {
|
|
|
6025
6155
|
}
|
|
6026
6156
|
updateTimeline(timeline, complete = true) {
|
|
6027
6157
|
this.incremental?.update(timeline, complete);
|
|
6158
|
+
const current = this.incremental?.timeline.dialogue.find(
|
|
6159
|
+
(block) => block.id === this.pendingSpeech?.id
|
|
6160
|
+
);
|
|
6161
|
+
if (current?.type === "speech" && this.pendingSpeech?.started) {
|
|
6162
|
+
this.audioPlayer.updateVisemes(current.visemes);
|
|
6163
|
+
}
|
|
6028
6164
|
void this.animationController.prepareAnimations?.(timeline.animations.map((clip) => clip.animation)).catch(() => void 0);
|
|
6029
6165
|
const pending = this.pendingSpeech;
|
|
6030
6166
|
const replacement = timeline.dialogue.find(
|
|
@@ -6122,14 +6258,26 @@ var StagingPlayer = class {
|
|
|
6122
6258
|
throw abortError2();
|
|
6123
6259
|
}
|
|
6124
6260
|
async playSpeechAttempt(speech, timeline, signal, beginTimeline, options) {
|
|
6125
|
-
|
|
6261
|
+
let characterCues = buildStagingCharacterCues(
|
|
6126
6262
|
speech.text,
|
|
6127
6263
|
speech.wordTimings,
|
|
6128
6264
|
speech.endTime - speech.startTime,
|
|
6129
|
-
speech.characterTimings
|
|
6265
|
+
speech.characterTimings,
|
|
6266
|
+
!speech.pending
|
|
6130
6267
|
);
|
|
6268
|
+
let previousTimings = speech.characterTimings;
|
|
6131
6269
|
let nextCharacterIndex = 0;
|
|
6132
6270
|
const revealCharacters = (localTime) => {
|
|
6271
|
+
if (previousTimings !== speech.characterTimings) {
|
|
6272
|
+
previousTimings = speech.characterTimings;
|
|
6273
|
+
characterCues = buildStagingCharacterCues(
|
|
6274
|
+
speech.text,
|
|
6275
|
+
speech.wordTimings,
|
|
6276
|
+
speech.endTime - speech.startTime,
|
|
6277
|
+
speech.characterTimings,
|
|
6278
|
+
!speech.pending
|
|
6279
|
+
);
|
|
6280
|
+
}
|
|
6133
6281
|
let cue = characterCues[nextCharacterIndex];
|
|
6134
6282
|
while (cue && cue.startTime <= localTime) {
|
|
6135
6283
|
options?.onCharacter?.(speech, nextCharacterIndex, cue.character);
|
|
@@ -6138,6 +6286,30 @@ var StagingPlayer = class {
|
|
|
6138
6286
|
}
|
|
6139
6287
|
};
|
|
6140
6288
|
let started = false;
|
|
6289
|
+
let observedThrough = 0;
|
|
6290
|
+
const phonemeStats = {
|
|
6291
|
+
phonemeCount: 0,
|
|
6292
|
+
latePhonemeCount: 0,
|
|
6293
|
+
phonemeLateMs: 0
|
|
6294
|
+
};
|
|
6295
|
+
const observePhonemes = () => {
|
|
6296
|
+
const currentTime = this.audioPlayer.getCurrentTime();
|
|
6297
|
+
for (const phone of speech.phonemeTimings ?? []) {
|
|
6298
|
+
if (phone.endTime <= observedThrough + 1e-6) {
|
|
6299
|
+
continue;
|
|
6300
|
+
}
|
|
6301
|
+
const lateMs = Math.max(0, currentTime - phone.startTime) * 1e3;
|
|
6302
|
+
phonemeStats.phonemeCount += 1;
|
|
6303
|
+
if (lateMs > 20) {
|
|
6304
|
+
phonemeStats.latePhonemeCount += 1;
|
|
6305
|
+
}
|
|
6306
|
+
phonemeStats.phonemeLateMs = Math.max(
|
|
6307
|
+
phonemeStats.phonemeLateMs,
|
|
6308
|
+
lateMs
|
|
6309
|
+
);
|
|
6310
|
+
observedThrough = phone.endTime;
|
|
6311
|
+
}
|
|
6312
|
+
};
|
|
6141
6313
|
const startSpeech = () => {
|
|
6142
6314
|
if (started) {
|
|
6143
6315
|
return;
|
|
@@ -6164,9 +6336,10 @@ var StagingPlayer = class {
|
|
|
6164
6336
|
const tick = async () => {
|
|
6165
6337
|
while (ticking && !signal.aborted) {
|
|
6166
6338
|
if (started) {
|
|
6339
|
+
observePhonemes();
|
|
6167
6340
|
const localTime = Math.min(
|
|
6168
6341
|
Math.max(this.audioPlayer.getCurrentTime(), 0),
|
|
6169
|
-
Math.max(speech.endTime - speech.startTime, 0)
|
|
6342
|
+
speech.pending ? Infinity : Math.max(speech.endTime - speech.startTime, 0)
|
|
6170
6343
|
);
|
|
6171
6344
|
this.applyTracks(speech.startTime + localTime, true, timeline);
|
|
6172
6345
|
revealCharacters(localTime);
|
|
@@ -6188,6 +6361,8 @@ var StagingPlayer = class {
|
|
|
6188
6361
|
startSpeech();
|
|
6189
6362
|
}
|
|
6190
6363
|
revealCharacters(Number.POSITIVE_INFINITY);
|
|
6364
|
+
observePhonemes();
|
|
6365
|
+
options?.onPhonemePlayback?.(speech, phonemeStats);
|
|
6191
6366
|
this.applyTracks(speech.endTime, true, timeline);
|
|
6192
6367
|
} finally {
|
|
6193
6368
|
ticking = false;
|
|
@@ -6314,6 +6489,12 @@ var normalizeVisemeTimeline = function(visemes) {
|
|
|
6314
6489
|
return { entries, droppedSourceIndices, inversionCount };
|
|
6315
6490
|
};
|
|
6316
6491
|
var getVisemeDurationMs = function(viseme, nextViseme) {
|
|
6492
|
+
if (viseme.endTime !== void 0 && Number.isFinite(viseme.endTime)) {
|
|
6493
|
+
return Math.max(
|
|
6494
|
+
0,
|
|
6495
|
+
Math.min(viseme.endTime, nextViseme?.timestamp ?? Infinity) - viseme.timestamp
|
|
6496
|
+
) * 1e3;
|
|
6497
|
+
}
|
|
6317
6498
|
const repeatedDipTime = getRepeatedVisemeDipTimeSeconds(viseme, nextViseme);
|
|
6318
6499
|
if (repeatedDipTime !== null && nextViseme) {
|
|
6319
6500
|
return Math.max((nextViseme.timestamp - viseme.timestamp) * 1e3, 0);
|
|
@@ -6321,6 +6502,9 @@ var getVisemeDurationMs = function(viseme, nextViseme) {
|
|
|
6321
6502
|
return MaximumVisemeDurationMs;
|
|
6322
6503
|
};
|
|
6323
6504
|
var getRepeatedVisemeDipTimeSeconds = function(viseme, nextViseme) {
|
|
6505
|
+
if (viseme.endTime !== void 0) {
|
|
6506
|
+
return null;
|
|
6507
|
+
}
|
|
6324
6508
|
if (!nextViseme || viseme.name !== nextViseme.name) {
|
|
6325
6509
|
return null;
|
|
6326
6510
|
}
|
|
@@ -6412,6 +6596,27 @@ var VisemePlayer = class {
|
|
|
6412
6596
|
this.rawVisemes = [];
|
|
6413
6597
|
this.visemes = [];
|
|
6414
6598
|
}
|
|
6599
|
+
updateVisemes(visemes) {
|
|
6600
|
+
if (!this.audioElement || this.audioElement.ended) {
|
|
6601
|
+
return;
|
|
6602
|
+
}
|
|
6603
|
+
if (visemes.length === this.rawVisemes.length && visemes.every((cue, index) => {
|
|
6604
|
+
const previous = this.rawVisemes[index];
|
|
6605
|
+
return previous?.name === cue.name && previous.timestamp === cue.timestamp && previous.endTime === cue.endTime && previous.weight === cue.weight;
|
|
6606
|
+
})) {
|
|
6607
|
+
return;
|
|
6608
|
+
}
|
|
6609
|
+
this.rawVisemes = visemes.map((viseme) => ({ ...viseme }));
|
|
6610
|
+
this.normalizedTimeline = normalizeVisemeTimeline(this.rawVisemes);
|
|
6611
|
+
this.visemes = this.normalizedTimeline.entries;
|
|
6612
|
+
this.heldMouths.updateCues(this.visemes);
|
|
6613
|
+
this.nextVisemeIndex = 0;
|
|
6614
|
+
this.activeCueIndex = this.activeCueIndex === null ? null : -1;
|
|
6615
|
+
this.repeatedCueDipActive = false;
|
|
6616
|
+
this.timelineRevision += 1;
|
|
6617
|
+
this.running = !this.audioElement.paused;
|
|
6618
|
+
this.recordTimeline();
|
|
6619
|
+
}
|
|
6415
6620
|
capturePerformanceState() {
|
|
6416
6621
|
this.recordTimeline();
|
|
6417
6622
|
}
|
|
@@ -6912,6 +7117,22 @@ var ClientTelemetry = class {
|
|
|
6912
7117
|
} catch {
|
|
6913
7118
|
}
|
|
6914
7119
|
};
|
|
7120
|
+
reportPhonemePlayback(params) {
|
|
7121
|
+
const state = this.state;
|
|
7122
|
+
if (!this.enabled || !state || state.taskId !== params.taskId) {
|
|
7123
|
+
return;
|
|
7124
|
+
}
|
|
7125
|
+
void this.sendClientLatency(
|
|
7126
|
+
state.conversationId,
|
|
7127
|
+
{
|
|
7128
|
+
...params,
|
|
7129
|
+
kind: "phoneme-timing",
|
|
7130
|
+
totalLatencyMs: 0,
|
|
7131
|
+
clientType: "web"
|
|
7132
|
+
},
|
|
7133
|
+
state.traceHeaders ?? void 0
|
|
7134
|
+
);
|
|
7135
|
+
}
|
|
6915
7136
|
};
|
|
6916
7137
|
|
|
6917
7138
|
// ../../clients/web-client/src/conversation-socket.ts
|
|
@@ -7247,7 +7468,7 @@ var StreamPlaybackController = class {
|
|
|
7247
7468
|
this.dependencies.getStagingPlayer().failStreamingSpeech(event.streamId, event.message);
|
|
7248
7469
|
return;
|
|
7249
7470
|
}
|
|
7250
|
-
if (event.type === "staging" || event.type === "staging_prefix") {
|
|
7471
|
+
if (event.type === "staging" || event.type === "staging_prefix" || event.type === "staging_live") {
|
|
7251
7472
|
this.recordStreamEvent(event, "timeline");
|
|
7252
7473
|
this.dependencies.telemetry.recordFirstStaging();
|
|
7253
7474
|
void this.playTimeline({
|
|
@@ -7368,6 +7589,14 @@ var StreamPlaybackController = class {
|
|
|
7368
7589
|
}
|
|
7369
7590
|
this.invokeSpeechPlaybackStartCallback(event);
|
|
7370
7591
|
},
|
|
7592
|
+
onPhonemePlayback: (speech, stats) => {
|
|
7593
|
+
this.dependencies.telemetry.reportPhonemePlayback({
|
|
7594
|
+
taskId: params.taskId,
|
|
7595
|
+
stagingId: params.stagingId,
|
|
7596
|
+
speechId: speech.id,
|
|
7597
|
+
...stats
|
|
7598
|
+
});
|
|
7599
|
+
},
|
|
7371
7600
|
onCharacter: (speech, characterIndex, character) => {
|
|
7372
7601
|
this.invokeCharacterRevealCallback(
|
|
7373
7602
|
this.toSpeechEvent(params, speech),
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@npcify/web",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.6",
|
|
4
4
|
"description": "Framework-independent Npcify characters, live conversations and baked performances for the web",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "UNLICENSED",
|
|
@@ -35,8 +35,8 @@
|
|
|
35
35
|
"tsup": "^8.5.0",
|
|
36
36
|
"typescript": "^5.9.3",
|
|
37
37
|
"@npcify/shared": "^0.1.0",
|
|
38
|
-
"@npcify/
|
|
39
|
-
"@npcify/
|
|
38
|
+
"@npcify/vrm-renderer": "^0.1.0",
|
|
39
|
+
"@npcify/web-client": "^0.0.0"
|
|
40
40
|
},
|
|
41
41
|
"repository": {
|
|
42
42
|
"type": "git",
|