@npcify/web 0.1.4 → 0.1.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.ts CHANGED
@@ -80,8 +80,16 @@ interface ConversationWorldline {
80
80
  interface ConversationTimelineEntry {
81
81
  name: string;
82
82
  timestamp: number;
83
+ /** Optional measured/estimated release time, local to the speech audio. */
84
+ endTime?: number;
83
85
  weight?: number;
84
86
  }
87
+ interface PhonemeTiming {
88
+ phoneme: string;
89
+ startTime: number;
90
+ endTime: number;
91
+ confidence: number;
92
+ }
85
93
  interface WordTiming {
86
94
  word: string;
87
95
  startTime: number;
@@ -146,6 +154,12 @@ interface StagingSpeechBlock {
146
154
  wordTimings: WordTiming[];
147
155
  /** Exact provider or deterministic timings, local to this audio block. */
148
156
  characterTimings?: CharacterTiming[];
157
+ /** Audio-derived estimates on a 20 ms frame grid, when available. */
158
+ phonemeTimings?: PhonemeTiming[];
159
+ /** Decoded audio duration, when acoustic timing was available. */
160
+ audioDuration?: number;
161
+ /** The last live speech block can grow until synthesis and timing finish. */
162
+ pending?: boolean;
149
163
  }
150
164
  interface StagingPauseBlock {
151
165
  id: string;
@@ -214,6 +228,10 @@ interface StagingStreamPrefixEvent extends Omit<StagingStreamTimelineEvent, 'typ
214
228
  type: 'staging_prefix';
215
229
  complete: false;
216
230
  }
231
+ /** Older clients ignore this event and still receive immutable prefixes. */
232
+ interface StagingStreamLiveEvent extends Omit<StagingStreamPrefixEvent, 'type'> {
233
+ type: 'staging_live';
234
+ }
217
235
  interface StagingStreamSpeechChunkEvent {
218
236
  type: 'speech_chunk';
219
237
  taskId: string;
@@ -259,7 +277,7 @@ interface StagingStreamBillingEvent {
259
277
  usageCharge?: UsageChargeSummary | null;
260
278
  availablePoints?: number;
261
279
  }
262
- type StagingStreamEvent = StagingStreamStartEvent | StagingStreamTextDeltaEvent | StagingStreamTimelineEvent | StagingStreamPrefixEvent | StagingStreamSpeechChunkEvent | StagingStreamSpeechCompleteEvent | StagingStreamSpeechErrorEvent | StagingStreamOptionsEvent | StagingStreamErrorEvent | StagingStreamBillingEvent;
280
+ type StagingStreamEvent = StagingStreamStartEvent | StagingStreamTextDeltaEvent | StagingStreamTimelineEvent | StagingStreamPrefixEvent | StagingStreamLiveEvent | StagingStreamSpeechChunkEvent | StagingStreamSpeechCompleteEvent | StagingStreamSpeechErrorEvent | StagingStreamOptionsEvent | StagingStreamErrorEvent | StagingStreamBillingEvent;
263
281
 
264
282
  declare const BAKED_CONVERSATION_VERSION: 1;
265
283
  /** Portable, language-neutral asset. Audio is embedded binary, never a signed URL. */
@@ -620,6 +638,14 @@ declare class ClientTelemetry {
620
638
  }): void;
621
639
  private readonly createTurnSpan;
622
640
  private readonly sendClientLatency;
641
+ reportPhonemePlayback(params: {
642
+ taskId: string;
643
+ stagingId: string;
644
+ speechId: string;
645
+ phonemeCount: number;
646
+ latePhonemeCount: number;
647
+ phonemeLateMs: number;
648
+ }): void;
623
649
  }
624
650
 
625
651
  declare class VisemePlayer {
@@ -640,6 +666,7 @@ declare class VisemePlayer {
640
666
  attachAudioElement(element: HTMLAudioElement | null): void;
641
667
  setVisemes(visemes: ConversationTimelineEntry[]): void;
642
668
  cleanup(): void;
669
+ updateVisemes(visemes: ConversationTimelineEntry[]): void;
643
670
  capturePerformanceState(): void;
644
671
  getAudioPerformanceSnapshot(): AudioPerformanceFrameSnapshot;
645
672
  getPerformanceSnapshot(): VisemePerformanceFrameSnapshot;
@@ -678,6 +705,7 @@ declare class AudioPlayer {
678
705
  type: 'silent';
679
706
  }>, visemes: ConversationTimelineEntry[], options?: SpeechPlaybackOptions): Promise<void>;
680
707
  getCurrentTime(): number;
708
+ updateVisemes(visemes: ConversationTimelineEntry[]): void;
681
709
  stop(): void;
682
710
  reset(): void;
683
711
  cleanup(): void;
@@ -699,6 +727,11 @@ type StagingPlaybackOptions = {
699
727
  onCharacter?: (speech: StagingSpeechBlock, characterIndex: number, character: string) => void;
700
728
  onSpeechStart?: (speech: StagingSpeechBlock) => void;
701
729
  onTimelineStart?: (timeline: StagingTimeline) => void;
730
+ onPhonemePlayback?: (speech: StagingSpeechBlock, stats: {
731
+ phonemeCount: number;
732
+ latePhonemeCount: number;
733
+ phonemeLateMs: number;
734
+ }) => void;
702
735
  };
703
736
  declare class StagingPlayer {
704
737
  private readonly audioPlayer;
package/dist/index.js CHANGED
@@ -2682,7 +2682,11 @@ var hasDialogueShape = function(value, duration, ids) {
2682
2682
  ids.add(beat.id);
2683
2683
  previousBeatEnd = Number(beat.endTime);
2684
2684
  return true;
2685
- }) && isSameTime(previousBeatEnd, Number(value.endTime)) && value.beats.map((beat) => beat.text).join("") === value.text && Array.isArray(value.visemes) && value.visemes.every((viseme) => hasVisemeShape(viseme, speechDuration)) && Array.isArray(value.wordTimings) && value.wordTimings.every(
2685
+ }) && isSameTime(previousBeatEnd, Number(value.endTime)) && value.beats.map((beat) => beat.text).join("") === value.text && Array.isArray(value.visemes) && value.visemes.every((viseme) => hasVisemeShape(viseme, speechDuration)) && (value.pending === void 0 || typeof value.pending === "boolean") && (value.audioDuration === void 0 || isFiniteNonNegative(value.audioDuration) && isSameTime(value.audioDuration, speechDuration)) && (value.phonemeTimings === void 0 || Array.isArray(value.phonemeTimings) && hasOrderedItems(
2686
+ value.phonemeTimings,
2687
+ speechDuration,
2688
+ (phone) => isNonEmptyString(phone.phoneme) && Number(phone.endTime) > Number(phone.startTime) && isFiniteNonNegative(phone.confidence) && phone.confidence <= 1
2689
+ )) && Array.isArray(value.wordTimings) && value.wordTimings.every(
2686
2690
  (word) => hasWordTimingShape(word, speechDuration)
2687
2691
  ) && (value.characterTimings === void 0 || Array.isArray(value.characterTimings) && hasCharacterTimingSequence(value.characterTimings, speechDuration));
2688
2692
  };
@@ -2708,7 +2712,7 @@ var hasAudioShape = function(value) {
2708
2712
  return false;
2709
2713
  };
2710
2714
  var hasVisemeShape = function(value, duration) {
2711
- return isRecord(value) && isNonEmptyString(value.name) && isFiniteNonNegative(value.timestamp) && isWithinDuration(value.timestamp, duration) && (value.weight === void 0 || isFiniteNonNegative(value.weight) && value.weight <= 1);
2715
+ return isRecord(value) && isNonEmptyString(value.name) && isFiniteNonNegative(value.timestamp) && isWithinDuration(value.timestamp, duration) && (value.endTime === void 0 || isFiniteNonNegative(value.endTime) && value.endTime >= value.timestamp && isWithinDuration(value.endTime, duration)) && (value.weight === void 0 || isFiniteNonNegative(value.weight) && value.weight <= 1);
2712
2716
  };
2713
2717
  var hasWordTimingShape = function(value, duration) {
2714
2718
  return isRecord(value) && isNonEmptyString(value.word) && isFiniteNonNegative(value.startTime) && isFiniteNonNegative(value.endTime) && value.endTime >= value.startTime && isWithinDuration(value.endTime, duration);
@@ -2751,8 +2755,8 @@ var hasTimelineShape = function(value) {
2751
2755
  }
2752
2756
  const ids = /* @__PURE__ */ new Set();
2753
2757
  let previousDialogueEnd = 0;
2754
- return value.dialogue.length > 0 && value.dialogue.every((block) => {
2755
- if (!isRecord(block) || !isSameTime(Number(block.startTime), previousDialogueEnd) || !hasDialogueShape(block, value.duration, ids)) {
2758
+ return value.dialogue.length > 0 && value.dialogue.every((block, index) => {
2759
+ if (!isRecord(block) || !isSameTime(Number(block.startTime), previousDialogueEnd) || !hasDialogueShape(block, value.duration, ids) || block.pending === true && index !== value.dialogue.length - 1) {
2756
2760
  return false;
2757
2761
  }
2758
2762
  previousDialogueEnd = Number(block.endTime);
@@ -3477,10 +3481,15 @@ var AVATAR_COLOR_SETTINGS = {
3477
3481
  toneMapping: NoToneMapping,
3478
3482
  outputColorSpace: SRGBColorSpace
3479
3483
  };
3484
+ var AVATAR_LIGHTS = {
3485
+ ambientIntensity: 0.7 * Math.PI / 1.9,
3486
+ directionalIntensity: 1.2 * Math.PI / 1.9,
3487
+ directionalPosition: [1, 1.5, 1]
3488
+ };
3480
3489
  var addAvatarLights = function(scene, {
3481
- ambientIntensity = 0.7,
3482
- directionalIntensity = 1.2,
3483
- directionalPosition = [1, 1.5, 1]
3490
+ ambientIntensity = AVATAR_LIGHTS.ambientIntensity,
3491
+ directionalIntensity = AVATAR_LIGHTS.directionalIntensity,
3492
+ directionalPosition = [...AVATAR_LIGHTS.directionalPosition]
3484
3493
  } = {}) {
3485
3494
  scene.add(new AmbientLight(16777215, ambientIntensity));
3486
3495
  const key = new DirectionalLight(16777215, directionalIntensity);
@@ -3829,6 +3838,12 @@ import {
3829
3838
  Texture as Texture2
3830
3839
  } from "three";
3831
3840
 
3841
+ // ../vrm-renderer/src/generated-character.ts
3842
+ import {
3843
+ MToonMaterial as MToonMaterial2,
3844
+ VRMExpressionMaterialColorBind
3845
+ } from "@pixiv/three-vrm";
3846
+
3832
3847
  // ../vrm-renderer/src/texture-expressions.ts
3833
3848
  import {
3834
3849
  MToonMaterial,
@@ -3847,6 +3862,9 @@ var HeldMouthTimeline = class _HeldMouthTimeline {
3847
3862
  shownAt = -Infinity;
3848
3863
  previousTime = -Infinity;
3849
3864
  static endTime(cue, next, speechEnd = Infinity) {
3865
+ if (cue.endTime !== void 0 && Number.isFinite(cue.endTime)) {
3866
+ return Math.min(cue.endTime, next?.timestamp ?? Infinity, speechEnd);
3867
+ }
3850
3868
  const nextTime = next?.timestamp ?? speechEnd;
3851
3869
  return Math.min(
3852
3870
  speechEnd,
@@ -3857,6 +3875,10 @@ var HeldMouthTimeline = class _HeldMouthTimeline {
3857
3875
  this.cues = [...cues].sort((a, b) => a.timestamp - b.timestamp);
3858
3876
  this.reset();
3859
3877
  }
3878
+ updateCues(cues) {
3879
+ this.cues = [...cues].sort((a, b) => a.timestamp - b.timestamp);
3880
+ this.next = 0;
3881
+ }
3860
3882
  reset() {
3861
3883
  this.next = 0;
3862
3884
  this.shown = null;
@@ -3957,13 +3979,9 @@ var findAtlasBindings = function(vrm) {
3957
3979
  var hasTextureMouthExpressions = function(manager) {
3958
3980
  return !!manager && installed.has(manager);
3959
3981
  };
3960
- var enableTextureExpressionPlayback = function(vrm) {
3982
+ var installTextureMouthPlayback = function(vrm, bindings) {
3961
3983
  const manager = vrm.expressionManager;
3962
- if (!manager || installed.has(manager)) {
3963
- return;
3964
- }
3965
- const bindings = findAtlasBindings(vrm);
3966
- if (!bindings.length) {
3984
+ if (!manager || installed.has(manager) || !bindings.length) {
3967
3985
  return;
3968
3986
  }
3969
3987
  installed.add(manager);
@@ -4009,6 +4027,75 @@ var enableTextureExpressionPlayback = function(vrm) {
4009
4027
  }
4010
4028
  };
4011
4029
 
4030
+ // ../vrm-renderer/src/generated-character.ts
4031
+ var TOLERANCE = 1e-4;
4032
+ var prepared = /* @__PURE__ */ new WeakMap();
4033
+ var sameColour = function(a, b) {
4034
+ return Math.abs(a.r - b.r) < TOLERANCE && Math.abs(a.g - b.g) < TOLERANCE && Math.abs(a.b - b.b) < TOLERANCE;
4035
+ };
4036
+ var sceneMToonMaterials = function(scene) {
4037
+ const materials = /* @__PURE__ */ new Set();
4038
+ scene.traverse((object) => {
4039
+ const material = object.material;
4040
+ for (const entry of Array.isArray(material) ? material : [material]) {
4041
+ if (entry instanceof MToonMaterial2) {
4042
+ materials.add(entry);
4043
+ }
4044
+ }
4045
+ });
4046
+ return materials;
4047
+ };
4048
+ var sharesFaceRecipe = function(material, faces) {
4049
+ return !!material.map && material.shadeMultiplyTexture?.image === material.map.image && faces.some(
4050
+ (face) => sameColour(material.color, face.color) && sameColour(material.shadeColorFactor, face.shadeColorFactor)
4051
+ );
4052
+ };
4053
+ var detectGeneratedCharacter = function(vrm) {
4054
+ const known = prepared.get(vrm);
4055
+ if (known) {
4056
+ return known;
4057
+ }
4058
+ const mouths = findAtlasBindings(vrm);
4059
+ if (!mouths.length) {
4060
+ return null;
4061
+ }
4062
+ const faces = [
4063
+ ...new Set(mouths.map(({ bind }) => bind.material))
4064
+ ];
4065
+ const coloured = /* @__PURE__ */ new Set();
4066
+ for (const expression of vrm.expressionManager?.expressions ?? []) {
4067
+ for (const bind of expression.binds) {
4068
+ if (bind instanceof VRMExpressionMaterialColorBind) {
4069
+ coloured.add(bind.material);
4070
+ }
4071
+ }
4072
+ }
4073
+ const materials = [...sceneMToonMaterials(vrm.scene)].filter(
4074
+ (material) => !coloured.has(material) && (faces.includes(material) || sharesFaceRecipe(material, faces))
4075
+ );
4076
+ return { faces, mouths, materials };
4077
+ };
4078
+ var normaliseLitFactor = function(material) {
4079
+ const { r, g, b } = material.color;
4080
+ if (!(r > 0) || r >= 1 - TOLERANCE || Math.abs(r - g) >= TOLERANCE || Math.abs(r - b) >= TOLERANCE) {
4081
+ return;
4082
+ }
4083
+ material.color.multiplyScalar(1 / r);
4084
+ material.shadeColorFactor.multiplyScalar(1 / r);
4085
+ };
4086
+ var prepareGeneratedCharacter = function(vrm) {
4087
+ const character = detectGeneratedCharacter(vrm);
4088
+ if (!character || prepared.has(vrm)) {
4089
+ return character;
4090
+ }
4091
+ for (const material of character.materials) {
4092
+ normaliseLitFactor(material);
4093
+ }
4094
+ installTextureMouthPlayback(vrm, character.mouths);
4095
+ prepared.set(vrm, character);
4096
+ return character;
4097
+ };
4098
+
4012
4099
  // ../vrm-renderer/src/session.ts
4013
4100
  import { VRMUtils as VRMUtils2 } from "@pixiv/three-vrm";
4014
4101
  import { AnimationMixer, LoopOnce, LoopRepeat } from "three";
@@ -4333,7 +4420,7 @@ var loadVrm = async ({
4333
4420
  });
4334
4421
  VRMUtils4.removeUnnecessaryVertices(gltf.scene);
4335
4422
  VRMUtils4.combineSkeletons(gltf.scene);
4336
- enableTextureExpressionPlayback(gltf.userData.vrm);
4423
+ prepareGeneratedCharacter(gltf.userData.vrm);
4337
4424
  return gltf.userData.vrm;
4338
4425
  };
4339
4426
  var loadVrma = async ({
@@ -4917,6 +5004,9 @@ var AudioPlayer = class {
4917
5004
  getCurrentTime() {
4918
5005
  return this.audioElement?.currentTime ?? 0;
4919
5006
  }
5007
+ updateVisemes(visemes) {
5008
+ this.visemePlayer.updateVisemes(visemes);
5009
+ }
4920
5010
  stop() {
4921
5011
  this.audioElement?.pause();
4922
5012
  }
@@ -5744,11 +5834,21 @@ var IncrementalStaging = class {
5744
5834
  }
5745
5835
  if (timeline.dialogue.length < this.timeline.dialogue.length || this.timeline.dialogue.some((block, index) => {
5746
5836
  const next = timeline.dialogue[index];
5837
+ if (block.type === "speech" && block.pending && next?.type === "speech") {
5838
+ return block.id !== next.id || block.text !== next.text || block.startTime !== next.startTime || next.pending && next.endTime < block.endTime;
5839
+ }
5747
5840
  return !next || dialogueKey(block) !== dialogueKey(next);
5748
5841
  })) {
5749
5842
  throw new Error("A staging revision changed published dialogue");
5750
5843
  }
5751
- Object.assign(this.timeline, timeline, { dialogue: [...timeline.dialogue] });
5844
+ const dialogue = timeline.dialogue.map((next, index) => {
5845
+ const previous = this.timeline.dialogue[index];
5846
+ if (previous?.type === "speech" && previous.pending && next.type === "speech") {
5847
+ return Object.assign(previous, next, { pending: next.pending });
5848
+ }
5849
+ return next;
5850
+ });
5851
+ Object.assign(this.timeline, timeline, { dialogue });
5752
5852
  this.complete = complete;
5753
5853
  for (const notify of this.listeners) {
5754
5854
  notify();
@@ -5781,6 +5881,29 @@ var IncrementalStaging = class {
5781
5881
  error.name = "AbortError";
5782
5882
  throw error;
5783
5883
  }
5884
+ async finishBlock(index, signal) {
5885
+ while (!signal.aborted) {
5886
+ const block = this.timeline.dialogue[index];
5887
+ if (block?.type !== "speech" || !block.pending) {
5888
+ return;
5889
+ }
5890
+ await new Promise((resolve) => {
5891
+ const done = () => {
5892
+ this.listeners.delete(done);
5893
+ signal.removeEventListener("abort", done);
5894
+ resolve();
5895
+ };
5896
+ this.listeners.add(done);
5897
+ signal.addEventListener("abort", done, { once: true });
5898
+ if (signal.aborted || !block.pending) {
5899
+ done();
5900
+ }
5901
+ });
5902
+ }
5903
+ const error = new Error("Staging playback was cancelled");
5904
+ error.name = "AbortError";
5905
+ throw error;
5906
+ }
5784
5907
  };
5785
5908
 
5786
5909
  // ../../clients/web-client/src/character/staging-character-cues.ts
@@ -5798,7 +5921,7 @@ var findCharacterSequence = function(characters, sequence, fromIndex) {
5798
5921
  }
5799
5922
  return -1;
5800
5923
  };
5801
- var buildStagingCharacterCues = function(text, wordTimings, duration, characterTimings = []) {
5924
+ var buildStagingCharacterCues = function(text, wordTimings, duration, characterTimings = [], complete = true) {
5802
5925
  const characters = Array.from(text.trim());
5803
5926
  if (characters.length === 0) {
5804
5927
  return [];
@@ -5847,7 +5970,7 @@ var buildStagingCharacterCues = function(text, wordTimings, duration, characterT
5847
5970
  if (startTimes.every((startTime) => startTime === null)) {
5848
5971
  return characters.map((character, index2) => ({
5849
5972
  character,
5850
- startTime: safeDuration * index2 / characters.length
5973
+ startTime: complete ? safeDuration * index2 / characters.length : Infinity
5851
5974
  }));
5852
5975
  }
5853
5976
  let index = 0;
@@ -5861,6 +5984,12 @@ var buildStagingCharacterCues = function(text, wordTimings, duration, characterT
5861
5984
  index += 1;
5862
5985
  }
5863
5986
  const gapEnd = index;
5987
+ if (!complete && gapEnd === startTimes.length) {
5988
+ for (let tail = gapStart; tail < gapEnd; tail += 1) {
5989
+ startTimes[tail] = Infinity;
5990
+ }
5991
+ continue;
5992
+ }
5864
5993
  const lowerBound = gapStart === 0 ? 0 : startTimes[gapStart - 1] ?? 0;
5865
5994
  const upperBound = gapEnd < startTimes.length ? startTimes[gapEnd] ?? safeDuration : safeDuration;
5866
5995
  const gapLength = gapEnd - gapStart;
@@ -5991,6 +6120,7 @@ var StagingPlayer = class {
5991
6120
  options
5992
6121
  );
5993
6122
  }
6123
+ await incremental.finishBlock(index, controller.signal);
5994
6124
  cursor = block.endTime;
5995
6125
  }
5996
6126
  if (timeline.duration > cursor) {
@@ -6025,6 +6155,12 @@ var StagingPlayer = class {
6025
6155
  }
6026
6156
  updateTimeline(timeline, complete = true) {
6027
6157
  this.incremental?.update(timeline, complete);
6158
+ const current = this.incremental?.timeline.dialogue.find(
6159
+ (block) => block.id === this.pendingSpeech?.id
6160
+ );
6161
+ if (current?.type === "speech" && this.pendingSpeech?.started) {
6162
+ this.audioPlayer.updateVisemes(current.visemes);
6163
+ }
6028
6164
  void this.animationController.prepareAnimations?.(timeline.animations.map((clip) => clip.animation)).catch(() => void 0);
6029
6165
  const pending = this.pendingSpeech;
6030
6166
  const replacement = timeline.dialogue.find(
@@ -6122,14 +6258,26 @@ var StagingPlayer = class {
6122
6258
  throw abortError2();
6123
6259
  }
6124
6260
  async playSpeechAttempt(speech, timeline, signal, beginTimeline, options) {
6125
- const characterCues = buildStagingCharacterCues(
6261
+ let characterCues = buildStagingCharacterCues(
6126
6262
  speech.text,
6127
6263
  speech.wordTimings,
6128
6264
  speech.endTime - speech.startTime,
6129
- speech.characterTimings
6265
+ speech.characterTimings,
6266
+ !speech.pending
6130
6267
  );
6268
+ let previousTimings = speech.characterTimings;
6131
6269
  let nextCharacterIndex = 0;
6132
6270
  const revealCharacters = (localTime) => {
6271
+ if (previousTimings !== speech.characterTimings) {
6272
+ previousTimings = speech.characterTimings;
6273
+ characterCues = buildStagingCharacterCues(
6274
+ speech.text,
6275
+ speech.wordTimings,
6276
+ speech.endTime - speech.startTime,
6277
+ speech.characterTimings,
6278
+ !speech.pending
6279
+ );
6280
+ }
6133
6281
  let cue = characterCues[nextCharacterIndex];
6134
6282
  while (cue && cue.startTime <= localTime) {
6135
6283
  options?.onCharacter?.(speech, nextCharacterIndex, cue.character);
@@ -6138,6 +6286,30 @@ var StagingPlayer = class {
6138
6286
  }
6139
6287
  };
6140
6288
  let started = false;
6289
+ let observedThrough = 0;
6290
+ const phonemeStats = {
6291
+ phonemeCount: 0,
6292
+ latePhonemeCount: 0,
6293
+ phonemeLateMs: 0
6294
+ };
6295
+ const observePhonemes = () => {
6296
+ const currentTime = this.audioPlayer.getCurrentTime();
6297
+ for (const phone of speech.phonemeTimings ?? []) {
6298
+ if (phone.endTime <= observedThrough + 1e-6) {
6299
+ continue;
6300
+ }
6301
+ const lateMs = Math.max(0, currentTime - phone.startTime) * 1e3;
6302
+ phonemeStats.phonemeCount += 1;
6303
+ if (lateMs > 20) {
6304
+ phonemeStats.latePhonemeCount += 1;
6305
+ }
6306
+ phonemeStats.phonemeLateMs = Math.max(
6307
+ phonemeStats.phonemeLateMs,
6308
+ lateMs
6309
+ );
6310
+ observedThrough = phone.endTime;
6311
+ }
6312
+ };
6141
6313
  const startSpeech = () => {
6142
6314
  if (started) {
6143
6315
  return;
@@ -6164,9 +6336,10 @@ var StagingPlayer = class {
6164
6336
  const tick = async () => {
6165
6337
  while (ticking && !signal.aborted) {
6166
6338
  if (started) {
6339
+ observePhonemes();
6167
6340
  const localTime = Math.min(
6168
6341
  Math.max(this.audioPlayer.getCurrentTime(), 0),
6169
- Math.max(speech.endTime - speech.startTime, 0)
6342
+ speech.pending ? Infinity : Math.max(speech.endTime - speech.startTime, 0)
6170
6343
  );
6171
6344
  this.applyTracks(speech.startTime + localTime, true, timeline);
6172
6345
  revealCharacters(localTime);
@@ -6188,6 +6361,8 @@ var StagingPlayer = class {
6188
6361
  startSpeech();
6189
6362
  }
6190
6363
  revealCharacters(Number.POSITIVE_INFINITY);
6364
+ observePhonemes();
6365
+ options?.onPhonemePlayback?.(speech, phonemeStats);
6191
6366
  this.applyTracks(speech.endTime, true, timeline);
6192
6367
  } finally {
6193
6368
  ticking = false;
@@ -6314,6 +6489,12 @@ var normalizeVisemeTimeline = function(visemes) {
6314
6489
  return { entries, droppedSourceIndices, inversionCount };
6315
6490
  };
6316
6491
  var getVisemeDurationMs = function(viseme, nextViseme) {
6492
+ if (viseme.endTime !== void 0 && Number.isFinite(viseme.endTime)) {
6493
+ return Math.max(
6494
+ 0,
6495
+ Math.min(viseme.endTime, nextViseme?.timestamp ?? Infinity) - viseme.timestamp
6496
+ ) * 1e3;
6497
+ }
6317
6498
  const repeatedDipTime = getRepeatedVisemeDipTimeSeconds(viseme, nextViseme);
6318
6499
  if (repeatedDipTime !== null && nextViseme) {
6319
6500
  return Math.max((nextViseme.timestamp - viseme.timestamp) * 1e3, 0);
@@ -6321,6 +6502,9 @@ var getVisemeDurationMs = function(viseme, nextViseme) {
6321
6502
  return MaximumVisemeDurationMs;
6322
6503
  };
6323
6504
  var getRepeatedVisemeDipTimeSeconds = function(viseme, nextViseme) {
6505
+ if (viseme.endTime !== void 0) {
6506
+ return null;
6507
+ }
6324
6508
  if (!nextViseme || viseme.name !== nextViseme.name) {
6325
6509
  return null;
6326
6510
  }
@@ -6412,6 +6596,27 @@ var VisemePlayer = class {
6412
6596
  this.rawVisemes = [];
6413
6597
  this.visemes = [];
6414
6598
  }
6599
+ updateVisemes(visemes) {
6600
+ if (!this.audioElement || this.audioElement.ended) {
6601
+ return;
6602
+ }
6603
+ if (visemes.length === this.rawVisemes.length && visemes.every((cue, index) => {
6604
+ const previous = this.rawVisemes[index];
6605
+ return previous?.name === cue.name && previous.timestamp === cue.timestamp && previous.endTime === cue.endTime && previous.weight === cue.weight;
6606
+ })) {
6607
+ return;
6608
+ }
6609
+ this.rawVisemes = visemes.map((viseme) => ({ ...viseme }));
6610
+ this.normalizedTimeline = normalizeVisemeTimeline(this.rawVisemes);
6611
+ this.visemes = this.normalizedTimeline.entries;
6612
+ this.heldMouths.updateCues(this.visemes);
6613
+ this.nextVisemeIndex = 0;
6614
+ this.activeCueIndex = this.activeCueIndex === null ? null : -1;
6615
+ this.repeatedCueDipActive = false;
6616
+ this.timelineRevision += 1;
6617
+ this.running = !this.audioElement.paused;
6618
+ this.recordTimeline();
6619
+ }
6415
6620
  capturePerformanceState() {
6416
6621
  this.recordTimeline();
6417
6622
  }
@@ -6912,6 +7117,22 @@ var ClientTelemetry = class {
6912
7117
  } catch {
6913
7118
  }
6914
7119
  };
7120
+ reportPhonemePlayback(params) {
7121
+ const state = this.state;
7122
+ if (!this.enabled || !state || state.taskId !== params.taskId) {
7123
+ return;
7124
+ }
7125
+ void this.sendClientLatency(
7126
+ state.conversationId,
7127
+ {
7128
+ ...params,
7129
+ kind: "phoneme-timing",
7130
+ totalLatencyMs: 0,
7131
+ clientType: "web"
7132
+ },
7133
+ state.traceHeaders ?? void 0
7134
+ );
7135
+ }
6915
7136
  };
6916
7137
 
6917
7138
  // ../../clients/web-client/src/conversation-socket.ts
@@ -7247,7 +7468,7 @@ var StreamPlaybackController = class {
7247
7468
  this.dependencies.getStagingPlayer().failStreamingSpeech(event.streamId, event.message);
7248
7469
  return;
7249
7470
  }
7250
- if (event.type === "staging" || event.type === "staging_prefix") {
7471
+ if (event.type === "staging" || event.type === "staging_prefix" || event.type === "staging_live") {
7251
7472
  this.recordStreamEvent(event, "timeline");
7252
7473
  this.dependencies.telemetry.recordFirstStaging();
7253
7474
  void this.playTimeline({
@@ -7368,6 +7589,14 @@ var StreamPlaybackController = class {
7368
7589
  }
7369
7590
  this.invokeSpeechPlaybackStartCallback(event);
7370
7591
  },
7592
+ onPhonemePlayback: (speech, stats) => {
7593
+ this.dependencies.telemetry.reportPhonemePlayback({
7594
+ taskId: params.taskId,
7595
+ stagingId: params.stagingId,
7596
+ speechId: speech.id,
7597
+ ...stats
7598
+ });
7599
+ },
7371
7600
  onCharacter: (speech, characterIndex, character) => {
7372
7601
  this.invokeCharacterRevealCallback(
7373
7602
  this.toSpeechEvent(params, speech),
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@npcify/web",
3
- "version": "0.1.4",
3
+ "version": "0.1.6",
4
4
  "description": "Framework-independent Npcify characters, live conversations and baked performances for the web",
5
5
  "type": "module",
6
6
  "license": "UNLICENSED",
@@ -35,8 +35,8 @@
35
35
  "tsup": "^8.5.0",
36
36
  "typescript": "^5.9.3",
37
37
  "@npcify/shared": "^0.1.0",
38
- "@npcify/web-client": "^0.0.0",
39
- "@npcify/vrm-renderer": "^0.1.0"
38
+ "@npcify/vrm-renderer": "^0.1.0",
39
+ "@npcify/web-client": "^0.0.0"
40
40
  },
41
41
  "repository": {
42
42
  "type": "git",