@npcify/web 0.1.5 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +34 -1
- package/dist/index.js +164 -11
- package/package.json +1 -1
package/dist/index.d.ts
CHANGED
|
@@ -80,8 +80,16 @@ interface ConversationWorldline {
|
|
|
80
80
|
interface ConversationTimelineEntry {
|
|
81
81
|
name: string;
|
|
82
82
|
timestamp: number;
|
|
83
|
+
/** Optional measured/estimated release time, local to the speech audio. */
|
|
84
|
+
endTime?: number;
|
|
83
85
|
weight?: number;
|
|
84
86
|
}
|
|
87
|
+
interface PhonemeTiming {
|
|
88
|
+
phoneme: string;
|
|
89
|
+
startTime: number;
|
|
90
|
+
endTime: number;
|
|
91
|
+
confidence: number;
|
|
92
|
+
}
|
|
85
93
|
interface WordTiming {
|
|
86
94
|
word: string;
|
|
87
95
|
startTime: number;
|
|
@@ -146,6 +154,12 @@ interface StagingSpeechBlock {
|
|
|
146
154
|
wordTimings: WordTiming[];
|
|
147
155
|
/** Exact provider or deterministic timings, local to this audio block. */
|
|
148
156
|
characterTimings?: CharacterTiming[];
|
|
157
|
+
/** Audio-derived estimates on a 20 ms frame grid, when available. */
|
|
158
|
+
phonemeTimings?: PhonemeTiming[];
|
|
159
|
+
/** Decoded audio duration, when acoustic timing was available. */
|
|
160
|
+
audioDuration?: number;
|
|
161
|
+
/** The last live speech block can grow until synthesis and timing finish. */
|
|
162
|
+
pending?: boolean;
|
|
149
163
|
}
|
|
150
164
|
interface StagingPauseBlock {
|
|
151
165
|
id: string;
|
|
@@ -214,6 +228,10 @@ interface StagingStreamPrefixEvent extends Omit<StagingStreamTimelineEvent, 'typ
|
|
|
214
228
|
type: 'staging_prefix';
|
|
215
229
|
complete: false;
|
|
216
230
|
}
|
|
231
|
+
/** Older clients ignore this event and still receive immutable prefixes. */
|
|
232
|
+
interface StagingStreamLiveEvent extends Omit<StagingStreamPrefixEvent, 'type'> {
|
|
233
|
+
type: 'staging_live';
|
|
234
|
+
}
|
|
217
235
|
interface StagingStreamSpeechChunkEvent {
|
|
218
236
|
type: 'speech_chunk';
|
|
219
237
|
taskId: string;
|
|
@@ -259,7 +277,7 @@ interface StagingStreamBillingEvent {
|
|
|
259
277
|
usageCharge?: UsageChargeSummary | null;
|
|
260
278
|
availablePoints?: number;
|
|
261
279
|
}
|
|
262
|
-
type StagingStreamEvent = StagingStreamStartEvent | StagingStreamTextDeltaEvent | StagingStreamTimelineEvent | StagingStreamPrefixEvent | StagingStreamSpeechChunkEvent | StagingStreamSpeechCompleteEvent | StagingStreamSpeechErrorEvent | StagingStreamOptionsEvent | StagingStreamErrorEvent | StagingStreamBillingEvent;
|
|
280
|
+
type StagingStreamEvent = StagingStreamStartEvent | StagingStreamTextDeltaEvent | StagingStreamTimelineEvent | StagingStreamPrefixEvent | StagingStreamLiveEvent | StagingStreamSpeechChunkEvent | StagingStreamSpeechCompleteEvent | StagingStreamSpeechErrorEvent | StagingStreamOptionsEvent | StagingStreamErrorEvent | StagingStreamBillingEvent;
|
|
263
281
|
|
|
264
282
|
declare const BAKED_CONVERSATION_VERSION: 1;
|
|
265
283
|
/** Portable, language-neutral asset. Audio is embedded binary, never a signed URL. */
|
|
@@ -620,6 +638,14 @@ declare class ClientTelemetry {
|
|
|
620
638
|
}): void;
|
|
621
639
|
private readonly createTurnSpan;
|
|
622
640
|
private readonly sendClientLatency;
|
|
641
|
+
reportPhonemePlayback(params: {
|
|
642
|
+
taskId: string;
|
|
643
|
+
stagingId: string;
|
|
644
|
+
speechId: string;
|
|
645
|
+
phonemeCount: number;
|
|
646
|
+
latePhonemeCount: number;
|
|
647
|
+
phonemeLateMs: number;
|
|
648
|
+
}): void;
|
|
623
649
|
}
|
|
624
650
|
|
|
625
651
|
declare class VisemePlayer {
|
|
@@ -640,6 +666,7 @@ declare class VisemePlayer {
|
|
|
640
666
|
attachAudioElement(element: HTMLAudioElement | null): void;
|
|
641
667
|
setVisemes(visemes: ConversationTimelineEntry[]): void;
|
|
642
668
|
cleanup(): void;
|
|
669
|
+
updateVisemes(visemes: ConversationTimelineEntry[]): void;
|
|
643
670
|
capturePerformanceState(): void;
|
|
644
671
|
getAudioPerformanceSnapshot(): AudioPerformanceFrameSnapshot;
|
|
645
672
|
getPerformanceSnapshot(): VisemePerformanceFrameSnapshot;
|
|
@@ -678,6 +705,7 @@ declare class AudioPlayer {
|
|
|
678
705
|
type: 'silent';
|
|
679
706
|
}>, visemes: ConversationTimelineEntry[], options?: SpeechPlaybackOptions): Promise<void>;
|
|
680
707
|
getCurrentTime(): number;
|
|
708
|
+
updateVisemes(visemes: ConversationTimelineEntry[]): void;
|
|
681
709
|
stop(): void;
|
|
682
710
|
reset(): void;
|
|
683
711
|
cleanup(): void;
|
|
@@ -699,6 +727,11 @@ type StagingPlaybackOptions = {
|
|
|
699
727
|
onCharacter?: (speech: StagingSpeechBlock, characterIndex: number, character: string) => void;
|
|
700
728
|
onSpeechStart?: (speech: StagingSpeechBlock) => void;
|
|
701
729
|
onTimelineStart?: (timeline: StagingTimeline) => void;
|
|
730
|
+
onPhonemePlayback?: (speech: StagingSpeechBlock, stats: {
|
|
731
|
+
phonemeCount: number;
|
|
732
|
+
latePhonemeCount: number;
|
|
733
|
+
phonemeLateMs: number;
|
|
734
|
+
}) => void;
|
|
702
735
|
};
|
|
703
736
|
declare class StagingPlayer {
|
|
704
737
|
private readonly audioPlayer;
|
package/dist/index.js
CHANGED
|
@@ -2682,7 +2682,11 @@ var hasDialogueShape = function(value, duration, ids) {
|
|
|
2682
2682
|
ids.add(beat.id);
|
|
2683
2683
|
previousBeatEnd = Number(beat.endTime);
|
|
2684
2684
|
return true;
|
|
2685
|
-
}) && isSameTime(previousBeatEnd, Number(value.endTime)) && value.beats.map((beat) => beat.text).join("") === value.text && Array.isArray(value.visemes) && value.visemes.every((viseme) => hasVisemeShape(viseme, speechDuration)) &&
|
|
2685
|
+
}) && isSameTime(previousBeatEnd, Number(value.endTime)) && value.beats.map((beat) => beat.text).join("") === value.text && Array.isArray(value.visemes) && value.visemes.every((viseme) => hasVisemeShape(viseme, speechDuration)) && (value.pending === void 0 || typeof value.pending === "boolean") && (value.audioDuration === void 0 || isFiniteNonNegative(value.audioDuration) && isSameTime(value.audioDuration, speechDuration)) && (value.phonemeTimings === void 0 || Array.isArray(value.phonemeTimings) && hasOrderedItems(
|
|
2686
|
+
value.phonemeTimings,
|
|
2687
|
+
speechDuration,
|
|
2688
|
+
(phone) => isNonEmptyString(phone.phoneme) && Number(phone.endTime) > Number(phone.startTime) && isFiniteNonNegative(phone.confidence) && phone.confidence <= 1
|
|
2689
|
+
)) && Array.isArray(value.wordTimings) && value.wordTimings.every(
|
|
2686
2690
|
(word) => hasWordTimingShape(word, speechDuration)
|
|
2687
2691
|
) && (value.characterTimings === void 0 || Array.isArray(value.characterTimings) && hasCharacterTimingSequence(value.characterTimings, speechDuration));
|
|
2688
2692
|
};
|
|
@@ -2708,7 +2712,7 @@ var hasAudioShape = function(value) {
|
|
|
2708
2712
|
return false;
|
|
2709
2713
|
};
|
|
2710
2714
|
var hasVisemeShape = function(value, duration) {
|
|
2711
|
-
return isRecord(value) && isNonEmptyString(value.name) && isFiniteNonNegative(value.timestamp) && isWithinDuration(value.timestamp, duration) && (value.weight === void 0 || isFiniteNonNegative(value.weight) && value.weight <= 1);
|
|
2715
|
+
return isRecord(value) && isNonEmptyString(value.name) && isFiniteNonNegative(value.timestamp) && isWithinDuration(value.timestamp, duration) && (value.endTime === void 0 || isFiniteNonNegative(value.endTime) && value.endTime >= value.timestamp && isWithinDuration(value.endTime, duration)) && (value.weight === void 0 || isFiniteNonNegative(value.weight) && value.weight <= 1);
|
|
2712
2716
|
};
|
|
2713
2717
|
var hasWordTimingShape = function(value, duration) {
|
|
2714
2718
|
return isRecord(value) && isNonEmptyString(value.word) && isFiniteNonNegative(value.startTime) && isFiniteNonNegative(value.endTime) && value.endTime >= value.startTime && isWithinDuration(value.endTime, duration);
|
|
@@ -2751,8 +2755,8 @@ var hasTimelineShape = function(value) {
|
|
|
2751
2755
|
}
|
|
2752
2756
|
const ids = /* @__PURE__ */ new Set();
|
|
2753
2757
|
let previousDialogueEnd = 0;
|
|
2754
|
-
return value.dialogue.length > 0 && value.dialogue.every((block) => {
|
|
2755
|
-
if (!isRecord(block) || !isSameTime(Number(block.startTime), previousDialogueEnd) || !hasDialogueShape(block, value.duration, ids)) {
|
|
2758
|
+
return value.dialogue.length > 0 && value.dialogue.every((block, index) => {
|
|
2759
|
+
if (!isRecord(block) || !isSameTime(Number(block.startTime), previousDialogueEnd) || !hasDialogueShape(block, value.duration, ids) || block.pending === true && index !== value.dialogue.length - 1) {
|
|
2756
2760
|
return false;
|
|
2757
2761
|
}
|
|
2758
2762
|
previousDialogueEnd = Number(block.endTime);
|
|
@@ -3858,6 +3862,9 @@ var HeldMouthTimeline = class _HeldMouthTimeline {
|
|
|
3858
3862
|
shownAt = -Infinity;
|
|
3859
3863
|
previousTime = -Infinity;
|
|
3860
3864
|
static endTime(cue, next, speechEnd = Infinity) {
|
|
3865
|
+
if (cue.endTime !== void 0 && Number.isFinite(cue.endTime)) {
|
|
3866
|
+
return Math.min(cue.endTime, next?.timestamp ?? Infinity, speechEnd);
|
|
3867
|
+
}
|
|
3861
3868
|
const nextTime = next?.timestamp ?? speechEnd;
|
|
3862
3869
|
return Math.min(
|
|
3863
3870
|
speechEnd,
|
|
@@ -3868,6 +3875,10 @@ var HeldMouthTimeline = class _HeldMouthTimeline {
|
|
|
3868
3875
|
this.cues = [...cues].sort((a, b) => a.timestamp - b.timestamp);
|
|
3869
3876
|
this.reset();
|
|
3870
3877
|
}
|
|
3878
|
+
updateCues(cues) {
|
|
3879
|
+
this.cues = [...cues].sort((a, b) => a.timestamp - b.timestamp);
|
|
3880
|
+
this.next = 0;
|
|
3881
|
+
}
|
|
3871
3882
|
reset() {
|
|
3872
3883
|
this.next = 0;
|
|
3873
3884
|
this.shown = null;
|
|
@@ -4993,6 +5004,9 @@ var AudioPlayer = class {
|
|
|
4993
5004
|
getCurrentTime() {
|
|
4994
5005
|
return this.audioElement?.currentTime ?? 0;
|
|
4995
5006
|
}
|
|
5007
|
+
updateVisemes(visemes) {
|
|
5008
|
+
this.visemePlayer.updateVisemes(visemes);
|
|
5009
|
+
}
|
|
4996
5010
|
stop() {
|
|
4997
5011
|
this.audioElement?.pause();
|
|
4998
5012
|
}
|
|
@@ -5820,11 +5834,21 @@ var IncrementalStaging = class {
|
|
|
5820
5834
|
}
|
|
5821
5835
|
if (timeline.dialogue.length < this.timeline.dialogue.length || this.timeline.dialogue.some((block, index) => {
|
|
5822
5836
|
const next = timeline.dialogue[index];
|
|
5837
|
+
if (block.type === "speech" && block.pending && next?.type === "speech") {
|
|
5838
|
+
return block.id !== next.id || block.text !== next.text || block.startTime !== next.startTime || next.pending && next.endTime < block.endTime;
|
|
5839
|
+
}
|
|
5823
5840
|
return !next || dialogueKey(block) !== dialogueKey(next);
|
|
5824
5841
|
})) {
|
|
5825
5842
|
throw new Error("A staging revision changed published dialogue");
|
|
5826
5843
|
}
|
|
5827
|
-
|
|
5844
|
+
const dialogue = timeline.dialogue.map((next, index) => {
|
|
5845
|
+
const previous = this.timeline.dialogue[index];
|
|
5846
|
+
if (previous?.type === "speech" && previous.pending && next.type === "speech") {
|
|
5847
|
+
return Object.assign(previous, next, { pending: next.pending });
|
|
5848
|
+
}
|
|
5849
|
+
return next;
|
|
5850
|
+
});
|
|
5851
|
+
Object.assign(this.timeline, timeline, { dialogue });
|
|
5828
5852
|
this.complete = complete;
|
|
5829
5853
|
for (const notify of this.listeners) {
|
|
5830
5854
|
notify();
|
|
@@ -5857,6 +5881,29 @@ var IncrementalStaging = class {
|
|
|
5857
5881
|
error.name = "AbortError";
|
|
5858
5882
|
throw error;
|
|
5859
5883
|
}
|
|
5884
|
+
async finishBlock(index, signal) {
|
|
5885
|
+
while (!signal.aborted) {
|
|
5886
|
+
const block = this.timeline.dialogue[index];
|
|
5887
|
+
if (block?.type !== "speech" || !block.pending) {
|
|
5888
|
+
return;
|
|
5889
|
+
}
|
|
5890
|
+
await new Promise((resolve) => {
|
|
5891
|
+
const done = () => {
|
|
5892
|
+
this.listeners.delete(done);
|
|
5893
|
+
signal.removeEventListener("abort", done);
|
|
5894
|
+
resolve();
|
|
5895
|
+
};
|
|
5896
|
+
this.listeners.add(done);
|
|
5897
|
+
signal.addEventListener("abort", done, { once: true });
|
|
5898
|
+
if (signal.aborted || !block.pending) {
|
|
5899
|
+
done();
|
|
5900
|
+
}
|
|
5901
|
+
});
|
|
5902
|
+
}
|
|
5903
|
+
const error = new Error("Staging playback was cancelled");
|
|
5904
|
+
error.name = "AbortError";
|
|
5905
|
+
throw error;
|
|
5906
|
+
}
|
|
5860
5907
|
};
|
|
5861
5908
|
|
|
5862
5909
|
// ../../clients/web-client/src/character/staging-character-cues.ts
|
|
@@ -5874,7 +5921,7 @@ var findCharacterSequence = function(characters, sequence, fromIndex) {
|
|
|
5874
5921
|
}
|
|
5875
5922
|
return -1;
|
|
5876
5923
|
};
|
|
5877
|
-
var buildStagingCharacterCues = function(text, wordTimings, duration, characterTimings = []) {
|
|
5924
|
+
var buildStagingCharacterCues = function(text, wordTimings, duration, characterTimings = [], complete = true) {
|
|
5878
5925
|
const characters = Array.from(text.trim());
|
|
5879
5926
|
if (characters.length === 0) {
|
|
5880
5927
|
return [];
|
|
@@ -5923,7 +5970,7 @@ var buildStagingCharacterCues = function(text, wordTimings, duration, characterT
|
|
|
5923
5970
|
if (startTimes.every((startTime) => startTime === null)) {
|
|
5924
5971
|
return characters.map((character, index2) => ({
|
|
5925
5972
|
character,
|
|
5926
|
-
startTime: safeDuration * index2 / characters.length
|
|
5973
|
+
startTime: complete ? safeDuration * index2 / characters.length : Infinity
|
|
5927
5974
|
}));
|
|
5928
5975
|
}
|
|
5929
5976
|
let index = 0;
|
|
@@ -5937,6 +5984,12 @@ var buildStagingCharacterCues = function(text, wordTimings, duration, characterT
|
|
|
5937
5984
|
index += 1;
|
|
5938
5985
|
}
|
|
5939
5986
|
const gapEnd = index;
|
|
5987
|
+
if (!complete && gapEnd === startTimes.length) {
|
|
5988
|
+
for (let tail = gapStart; tail < gapEnd; tail += 1) {
|
|
5989
|
+
startTimes[tail] = Infinity;
|
|
5990
|
+
}
|
|
5991
|
+
continue;
|
|
5992
|
+
}
|
|
5940
5993
|
const lowerBound = gapStart === 0 ? 0 : startTimes[gapStart - 1] ?? 0;
|
|
5941
5994
|
const upperBound = gapEnd < startTimes.length ? startTimes[gapEnd] ?? safeDuration : safeDuration;
|
|
5942
5995
|
const gapLength = gapEnd - gapStart;
|
|
@@ -6067,6 +6120,7 @@ var StagingPlayer = class {
|
|
|
6067
6120
|
options
|
|
6068
6121
|
);
|
|
6069
6122
|
}
|
|
6123
|
+
await incremental.finishBlock(index, controller.signal);
|
|
6070
6124
|
cursor = block.endTime;
|
|
6071
6125
|
}
|
|
6072
6126
|
if (timeline.duration > cursor) {
|
|
@@ -6101,6 +6155,12 @@ var StagingPlayer = class {
|
|
|
6101
6155
|
}
|
|
6102
6156
|
updateTimeline(timeline, complete = true) {
|
|
6103
6157
|
this.incremental?.update(timeline, complete);
|
|
6158
|
+
const current = this.incremental?.timeline.dialogue.find(
|
|
6159
|
+
(block) => block.id === this.pendingSpeech?.id
|
|
6160
|
+
);
|
|
6161
|
+
if (current?.type === "speech" && this.pendingSpeech?.started) {
|
|
6162
|
+
this.audioPlayer.updateVisemes(current.visemes);
|
|
6163
|
+
}
|
|
6104
6164
|
void this.animationController.prepareAnimations?.(timeline.animations.map((clip) => clip.animation)).catch(() => void 0);
|
|
6105
6165
|
const pending = this.pendingSpeech;
|
|
6106
6166
|
const replacement = timeline.dialogue.find(
|
|
@@ -6198,14 +6258,26 @@ var StagingPlayer = class {
|
|
|
6198
6258
|
throw abortError2();
|
|
6199
6259
|
}
|
|
6200
6260
|
async playSpeechAttempt(speech, timeline, signal, beginTimeline, options) {
|
|
6201
|
-
|
|
6261
|
+
let characterCues = buildStagingCharacterCues(
|
|
6202
6262
|
speech.text,
|
|
6203
6263
|
speech.wordTimings,
|
|
6204
6264
|
speech.endTime - speech.startTime,
|
|
6205
|
-
speech.characterTimings
|
|
6265
|
+
speech.characterTimings,
|
|
6266
|
+
!speech.pending
|
|
6206
6267
|
);
|
|
6268
|
+
let previousTimings = speech.characterTimings;
|
|
6207
6269
|
let nextCharacterIndex = 0;
|
|
6208
6270
|
const revealCharacters = (localTime) => {
|
|
6271
|
+
if (previousTimings !== speech.characterTimings) {
|
|
6272
|
+
previousTimings = speech.characterTimings;
|
|
6273
|
+
characterCues = buildStagingCharacterCues(
|
|
6274
|
+
speech.text,
|
|
6275
|
+
speech.wordTimings,
|
|
6276
|
+
speech.endTime - speech.startTime,
|
|
6277
|
+
speech.characterTimings,
|
|
6278
|
+
!speech.pending
|
|
6279
|
+
);
|
|
6280
|
+
}
|
|
6209
6281
|
let cue = characterCues[nextCharacterIndex];
|
|
6210
6282
|
while (cue && cue.startTime <= localTime) {
|
|
6211
6283
|
options?.onCharacter?.(speech, nextCharacterIndex, cue.character);
|
|
@@ -6214,6 +6286,30 @@ var StagingPlayer = class {
|
|
|
6214
6286
|
}
|
|
6215
6287
|
};
|
|
6216
6288
|
let started = false;
|
|
6289
|
+
let observedThrough = 0;
|
|
6290
|
+
const phonemeStats = {
|
|
6291
|
+
phonemeCount: 0,
|
|
6292
|
+
latePhonemeCount: 0,
|
|
6293
|
+
phonemeLateMs: 0
|
|
6294
|
+
};
|
|
6295
|
+
const observePhonemes = () => {
|
|
6296
|
+
const currentTime = this.audioPlayer.getCurrentTime();
|
|
6297
|
+
for (const phone of speech.phonemeTimings ?? []) {
|
|
6298
|
+
if (phone.endTime <= observedThrough + 1e-6) {
|
|
6299
|
+
continue;
|
|
6300
|
+
}
|
|
6301
|
+
const lateMs = Math.max(0, currentTime - phone.startTime) * 1e3;
|
|
6302
|
+
phonemeStats.phonemeCount += 1;
|
|
6303
|
+
if (lateMs > 20) {
|
|
6304
|
+
phonemeStats.latePhonemeCount += 1;
|
|
6305
|
+
}
|
|
6306
|
+
phonemeStats.phonemeLateMs = Math.max(
|
|
6307
|
+
phonemeStats.phonemeLateMs,
|
|
6308
|
+
lateMs
|
|
6309
|
+
);
|
|
6310
|
+
observedThrough = phone.endTime;
|
|
6311
|
+
}
|
|
6312
|
+
};
|
|
6217
6313
|
const startSpeech = () => {
|
|
6218
6314
|
if (started) {
|
|
6219
6315
|
return;
|
|
@@ -6240,9 +6336,10 @@ var StagingPlayer = class {
|
|
|
6240
6336
|
const tick = async () => {
|
|
6241
6337
|
while (ticking && !signal.aborted) {
|
|
6242
6338
|
if (started) {
|
|
6339
|
+
observePhonemes();
|
|
6243
6340
|
const localTime = Math.min(
|
|
6244
6341
|
Math.max(this.audioPlayer.getCurrentTime(), 0),
|
|
6245
|
-
Math.max(speech.endTime - speech.startTime, 0)
|
|
6342
|
+
speech.pending ? Infinity : Math.max(speech.endTime - speech.startTime, 0)
|
|
6246
6343
|
);
|
|
6247
6344
|
this.applyTracks(speech.startTime + localTime, true, timeline);
|
|
6248
6345
|
revealCharacters(localTime);
|
|
@@ -6264,6 +6361,8 @@ var StagingPlayer = class {
|
|
|
6264
6361
|
startSpeech();
|
|
6265
6362
|
}
|
|
6266
6363
|
revealCharacters(Number.POSITIVE_INFINITY);
|
|
6364
|
+
observePhonemes();
|
|
6365
|
+
options?.onPhonemePlayback?.(speech, phonemeStats);
|
|
6267
6366
|
this.applyTracks(speech.endTime, true, timeline);
|
|
6268
6367
|
} finally {
|
|
6269
6368
|
ticking = false;
|
|
@@ -6390,6 +6489,12 @@ var normalizeVisemeTimeline = function(visemes) {
|
|
|
6390
6489
|
return { entries, droppedSourceIndices, inversionCount };
|
|
6391
6490
|
};
|
|
6392
6491
|
var getVisemeDurationMs = function(viseme, nextViseme) {
|
|
6492
|
+
if (viseme.endTime !== void 0 && Number.isFinite(viseme.endTime)) {
|
|
6493
|
+
return Math.max(
|
|
6494
|
+
0,
|
|
6495
|
+
Math.min(viseme.endTime, nextViseme?.timestamp ?? Infinity) - viseme.timestamp
|
|
6496
|
+
) * 1e3;
|
|
6497
|
+
}
|
|
6393
6498
|
const repeatedDipTime = getRepeatedVisemeDipTimeSeconds(viseme, nextViseme);
|
|
6394
6499
|
if (repeatedDipTime !== null && nextViseme) {
|
|
6395
6500
|
return Math.max((nextViseme.timestamp - viseme.timestamp) * 1e3, 0);
|
|
@@ -6397,6 +6502,9 @@ var getVisemeDurationMs = function(viseme, nextViseme) {
|
|
|
6397
6502
|
return MaximumVisemeDurationMs;
|
|
6398
6503
|
};
|
|
6399
6504
|
var getRepeatedVisemeDipTimeSeconds = function(viseme, nextViseme) {
|
|
6505
|
+
if (viseme.endTime !== void 0) {
|
|
6506
|
+
return null;
|
|
6507
|
+
}
|
|
6400
6508
|
if (!nextViseme || viseme.name !== nextViseme.name) {
|
|
6401
6509
|
return null;
|
|
6402
6510
|
}
|
|
@@ -6488,6 +6596,27 @@ var VisemePlayer = class {
|
|
|
6488
6596
|
this.rawVisemes = [];
|
|
6489
6597
|
this.visemes = [];
|
|
6490
6598
|
}
|
|
6599
|
+
updateVisemes(visemes) {
|
|
6600
|
+
if (!this.audioElement || this.audioElement.ended) {
|
|
6601
|
+
return;
|
|
6602
|
+
}
|
|
6603
|
+
if (visemes.length === this.rawVisemes.length && visemes.every((cue, index) => {
|
|
6604
|
+
const previous = this.rawVisemes[index];
|
|
6605
|
+
return previous?.name === cue.name && previous.timestamp === cue.timestamp && previous.endTime === cue.endTime && previous.weight === cue.weight;
|
|
6606
|
+
})) {
|
|
6607
|
+
return;
|
|
6608
|
+
}
|
|
6609
|
+
this.rawVisemes = visemes.map((viseme) => ({ ...viseme }));
|
|
6610
|
+
this.normalizedTimeline = normalizeVisemeTimeline(this.rawVisemes);
|
|
6611
|
+
this.visemes = this.normalizedTimeline.entries;
|
|
6612
|
+
this.heldMouths.updateCues(this.visemes);
|
|
6613
|
+
this.nextVisemeIndex = 0;
|
|
6614
|
+
this.activeCueIndex = this.activeCueIndex === null ? null : -1;
|
|
6615
|
+
this.repeatedCueDipActive = false;
|
|
6616
|
+
this.timelineRevision += 1;
|
|
6617
|
+
this.running = !this.audioElement.paused;
|
|
6618
|
+
this.recordTimeline();
|
|
6619
|
+
}
|
|
6491
6620
|
capturePerformanceState() {
|
|
6492
6621
|
this.recordTimeline();
|
|
6493
6622
|
}
|
|
@@ -6988,6 +7117,22 @@ var ClientTelemetry = class {
|
|
|
6988
7117
|
} catch {
|
|
6989
7118
|
}
|
|
6990
7119
|
};
|
|
7120
|
+
reportPhonemePlayback(params) {
|
|
7121
|
+
const state = this.state;
|
|
7122
|
+
if (!this.enabled || !state || state.taskId !== params.taskId) {
|
|
7123
|
+
return;
|
|
7124
|
+
}
|
|
7125
|
+
void this.sendClientLatency(
|
|
7126
|
+
state.conversationId,
|
|
7127
|
+
{
|
|
7128
|
+
...params,
|
|
7129
|
+
kind: "phoneme-timing",
|
|
7130
|
+
totalLatencyMs: 0,
|
|
7131
|
+
clientType: "web"
|
|
7132
|
+
},
|
|
7133
|
+
state.traceHeaders ?? void 0
|
|
7134
|
+
);
|
|
7135
|
+
}
|
|
6991
7136
|
};
|
|
6992
7137
|
|
|
6993
7138
|
// ../../clients/web-client/src/conversation-socket.ts
|
|
@@ -7323,7 +7468,7 @@ var StreamPlaybackController = class {
|
|
|
7323
7468
|
this.dependencies.getStagingPlayer().failStreamingSpeech(event.streamId, event.message);
|
|
7324
7469
|
return;
|
|
7325
7470
|
}
|
|
7326
|
-
if (event.type === "staging" || event.type === "staging_prefix") {
|
|
7471
|
+
if (event.type === "staging" || event.type === "staging_prefix" || event.type === "staging_live") {
|
|
7327
7472
|
this.recordStreamEvent(event, "timeline");
|
|
7328
7473
|
this.dependencies.telemetry.recordFirstStaging();
|
|
7329
7474
|
void this.playTimeline({
|
|
@@ -7444,6 +7589,14 @@ var StreamPlaybackController = class {
|
|
|
7444
7589
|
}
|
|
7445
7590
|
this.invokeSpeechPlaybackStartCallback(event);
|
|
7446
7591
|
},
|
|
7592
|
+
onPhonemePlayback: (speech, stats) => {
|
|
7593
|
+
this.dependencies.telemetry.reportPhonemePlayback({
|
|
7594
|
+
taskId: params.taskId,
|
|
7595
|
+
stagingId: params.stagingId,
|
|
7596
|
+
speechId: speech.id,
|
|
7597
|
+
...stats
|
|
7598
|
+
});
|
|
7599
|
+
},
|
|
7447
7600
|
onCharacter: (speech, characterIndex, character) => {
|
|
7448
7601
|
this.invokeCharacterRevealCallback(
|
|
7449
7602
|
this.toSpeechEvent(params, speech),
|