@spatialwalk/avatarkit-rtc 1.0.0-beta.1 → 1.0.0-beta.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/README.md +2 -410
  2. package/dist/assets/animation-worker-DOGeTjF0.js.map +1 -0
  3. package/dist/core/AvatarPlayer.d.ts +96 -12
  4. package/dist/core/AvatarPlayer.d.ts.map +1 -1
  5. package/dist/core/RTCProvider.d.ts +12 -16
  6. package/dist/core/RTCProvider.d.ts.map +1 -1
  7. package/dist/index.d.ts +3 -3
  8. package/dist/index.d.ts.map +1 -1
  9. package/dist/index.js +3 -2
  10. package/dist/index10.js +91 -39
  11. package/dist/index10.js.map +1 -1
  12. package/dist/index11.js +14 -386
  13. package/dist/index11.js.map +1 -1
  14. package/dist/index12.js +350 -64
  15. package/dist/index12.js.map +1 -1
  16. package/dist/index13.js +44 -14
  17. package/dist/index13.js.map +1 -1
  18. package/dist/index14.js +25 -44
  19. package/dist/index14.js.map +1 -1
  20. package/dist/index2.js +335 -46
  21. package/dist/index2.js.map +1 -1
  22. package/dist/index3.js +265 -54
  23. package/dist/index3.js.map +1 -1
  24. package/dist/index4.js +105 -86
  25. package/dist/index4.js.map +1 -1
  26. package/dist/index5.js +6 -2
  27. package/dist/index5.js.map +1 -1
  28. package/dist/index6.js +603 -39
  29. package/dist/index6.js.map +1 -1
  30. package/dist/index8.js +128 -167
  31. package/dist/index8.js.map +1 -1
  32. package/dist/index9.js +65 -164
  33. package/dist/index9.js.map +1 -1
  34. package/dist/providers/agora/AgoraProvider.d.ts +0 -13
  35. package/dist/providers/agora/AgoraProvider.d.ts.map +1 -1
  36. package/dist/providers/agora/index.d.ts +1 -5
  37. package/dist/providers/agora/index.d.ts.map +1 -1
  38. package/dist/providers/agora/types.d.ts.map +1 -1
  39. package/dist/providers/base/BaseProvider.d.ts +50 -8
  40. package/dist/providers/base/BaseProvider.d.ts.map +1 -1
  41. package/dist/providers/livekit/LiveKitProvider.d.ts +4 -15
  42. package/dist/providers/livekit/LiveKitProvider.d.ts.map +1 -1
  43. package/dist/providers/livekit/animation-worker.d.ts.map +1 -1
  44. package/dist/providers/livekit/index.d.ts +1 -5
  45. package/dist/providers/livekit/index.d.ts.map +1 -1
  46. package/dist/types/index.d.ts +21 -0
  47. package/dist/types/index.d.ts.map +1 -1
  48. package/dist/utils/index.d.ts +2 -0
  49. package/dist/utils/index.d.ts.map +1 -1
  50. package/dist/utils/telemetry.d.ts +22 -0
  51. package/dist/utils/telemetry.d.ts.map +1 -0
  52. package/package.json +21 -12
  53. package/dist/assets/animation-worker-CUXZycUw.js.map +0 -1
  54. package/dist/index15.js +0 -29
  55. package/dist/index15.js.map +0 -1
  56. package/dist/index16.js +0 -144
  57. package/dist/index16.js.map +0 -1
  58. package/dist/index17.js +0 -106
  59. package/dist/index17.js.map +0 -1
  60. package/dist/index18.js +0 -28
  61. package/dist/index18.js.map +0 -1
  62. package/dist/proto/animation.d.ts +0 -12
  63. package/dist/proto/animation.d.ts.map +0 -1
package/dist/index6.js CHANGED
@@ -1,9 +1,9 @@
1
1
  var __defProp = Object.defineProperty;
2
2
  var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
3
3
  var __publicField = (obj, key, value) => __defNormalProp(obj, typeof key !== "symbol" ? key + "" : key, value);
4
- import { decodeAnimationKeyframes } from "./index8.js";
5
4
  import { logger } from "./index7.js";
6
- class AnimationHandler {
5
+ import { trackEvent } from "./index8.js";
6
+ const _AnimationHandler = class _AnimationHandler {
7
7
  /**
8
8
  * @internal
9
9
  */
@@ -11,8 +11,6 @@ class AnimationHandler {
11
11
  /** @internal */
12
12
  __publicField(this, "renderer");
13
13
  /** @internal */
14
- __publicField(this, "decoder");
15
- /** @internal */
16
14
  __publicField(this, "config");
17
15
  // Frame tracking
18
16
  /** @internal */
@@ -43,12 +41,77 @@ class AnimationHandler {
43
41
  __publicField(this, "hasHandledTransitionStart", false);
44
42
  /** @internal */
45
43
  __publicField(this, "hasHandledTransitionEnd", false);
44
+ // Watchdog for detecting stalled data stream
45
+ /** @internal */
46
+ __publicField(this, "lastFrameReceivedTime", 0);
47
+ /** @internal */
48
+ __publicField(this, "isInSession", false);
49
+ /** @internal */
50
+ __publicField(this, "watchdogTimer", null);
51
+ /** @internal */
52
+ __publicField(this, "hasReportedStall", false);
53
+ /** @internal */
54
+ __publicField(this, "onStreamStalledCallback", null);
55
+ /** @internal */
56
+ __publicField(this, "providerName", "");
57
+ /** @internal */
58
+ __publicField(this, "isStalledFallback", false);
59
+ // True when fallback to idle was triggered due to stall
60
+ // Playback stats
61
+ /** @internal */
62
+ __publicField(this, "playbackStatsTimer", null);
63
+ /** @internal */
64
+ __publicField(this, "playbackFrameCount", 0);
65
+ /** @internal */
66
+ __publicField(this, "playbackFrameTimestamps", []);
67
+ /** @internal */
68
+ __publicField(this, "playbackGapCount", 0);
69
+ /** @internal */
70
+ __publicField(this, "playbackExpectedSeq", -1);
71
+ // Jitter buffer
72
+ /** @internal */
73
+ __publicField(this, "bufferState", "direct");
74
+ /** @internal */
75
+ __publicField(this, "frameBuffer", /* @__PURE__ */ new Map());
76
+ /** @internal */
77
+ __publicField(this, "bufferNextSeq", -1);
78
+ /** @internal */
79
+ __publicField(this, "bufferDrainTimer", null);
80
+ /** @internal */
81
+ __publicField(this, "bufferLastDrainTime", 0);
82
+ // Cumulative session stats (never reset until getSessionSummary)
83
+ /** @internal */
84
+ __publicField(this, "cumulativeTotalFrames", 0);
85
+ /** @internal */
86
+ __publicField(this, "cumulativeLost", 0);
87
+ /** @internal */
88
+ __publicField(this, "cumulativeRecovered", 0);
89
+ /** @internal */
90
+ __publicField(this, "cumulativeDropped", 0);
91
+ /** @internal */
92
+ __publicField(this, "cumulativeFpsReadings", []);
93
+ // Per-conversation stats for rtc_playback_stats
94
+ /** @internal */
95
+ __publicField(this, "conversationFrameCount", 0);
96
+ /** @internal */
97
+ __publicField(this, "conversationLost", 0);
98
+ /** @internal */
99
+ __publicField(this, "conversationRecovered", 0);
100
+ /** @internal */
101
+ __publicField(this, "conversationDropped", 0);
102
+ /** @internal */
103
+ __publicField(this, "conversationFpsReadings", []);
104
+ /** @internal */
105
+ __publicField(this, "conversationStartTime", 0);
46
106
  this.renderer = renderer;
47
- this.decoder = config.decoder ?? decodeAnimationKeyframes;
48
107
  this.config = {
49
108
  transitionStartFrameCount: config.transitionStartFrameCount ?? 8,
50
- transitionEndFrameCount: config.transitionEndFrameCount ?? 12
109
+ transitionEndFrameCount: config.transitionEndFrameCount ?? 12,
110
+ enableJitterBuffer: config.enableJitterBuffer ?? true,
111
+ maxBufferDelayMs: config.maxBufferDelayMs ?? 80
51
112
  };
113
+ this.onStreamStalledCallback = config.onStreamStalled ?? null;
114
+ this.providerName = config.providerName ?? "";
52
115
  }
53
116
  /**
54
117
  * Handle animation data received from RTC provider.
@@ -61,17 +124,42 @@ class AnimationHandler {
61
124
  if (this.isPlayingTransition) {
62
125
  this.stopTransition();
63
126
  }
127
+ const now = Date.now();
128
+ if (this.hasReportedStall) {
129
+ const stallDuration = now - this.lastFrameReceivedTime;
130
+ logger.info(
131
+ "AnimationHandler",
132
+ `Data stream resumed after ${stallDuration}ms stall`
133
+ );
134
+ this.hasReportedStall = false;
135
+ }
136
+ if (this.isStalledFallback) {
137
+ logger.info(
138
+ "AnimationHandler",
139
+ "Resuming from stall fallback, rendering directly without transition"
140
+ );
141
+ this.isStalledFallback = false;
142
+ }
143
+ this.lastFrameReceivedTime = now;
64
144
  this.animationFrameCount++;
65
- this.renderedFrameCount++;
145
+ this.ensureSessionActive(frameSeq);
146
+ if (this.config.enableJitterBuffer && frameSeq !== void 0) {
147
+ this.bufferFrame(protobufData, frameSeq);
148
+ return;
149
+ }
66
150
  if (frameSeq !== void 0 && this.lastRenderedFrameSeq !== -1) {
67
- if (frameSeq <= this.lastRenderedFrameSeq) {
68
- logger.info(
151
+ if (frameSeq < this.lastRenderedFrameSeq) {
152
+ logger.warn(
69
153
  "AnimationHandler",
70
- `OUT-OF-ORDER/DUPLICATE: seq=${frameSeq}, lastRendered=${this.lastRenderedFrameSeq}${isRecovered ? " [RECOVERED]" : ""}`
154
+ `OUT-OF-ORDER: seq=${frameSeq}, lastRendered=${this.lastRenderedFrameSeq}${isRecovered ? " [RECOVERED]" : ""}, discarding`
71
155
  );
156
+ this.conversationDropped++;
157
+ return;
158
+ } else if (frameSeq === this.lastRenderedFrameSeq) {
159
+ return;
72
160
  } else if (frameSeq > this.lastRenderedFrameSeq + 1) {
73
161
  const gap = frameSeq - this.lastRenderedFrameSeq - 1;
74
- logger.info(
162
+ logger.warn(
75
163
  "AnimationHandler",
76
164
  `GAP: ${gap} frame(s) between ${this.lastRenderedFrameSeq} and ${frameSeq}${isRecovered ? " [RECOVERED]" : ""}`
77
165
  );
@@ -80,9 +168,17 @@ class AnimationHandler {
80
168
  if (frameSeq !== void 0) {
81
169
  this.lastRenderedFrameSeq = frameSeq;
82
170
  }
83
- const keyframes = this.decoder(protobufData);
84
- if (keyframes && keyframes.length > 0) {
85
- this.renderer.renderFrame(keyframes[0]);
171
+ this.renderedFrameCount++;
172
+ if (isRecovered) this.conversationRecovered++;
173
+ this.renderer.renderFromProtobuf(protobufData);
174
+ this.logRenderedFrame("direct", frameSeq, isRecovered);
175
+ this.playbackFrameTimestamps.push(performance.now());
176
+ this.playbackFrameCount++;
177
+ if (frameSeq !== void 0) {
178
+ if (this.playbackExpectedSeq >= 0 && frameSeq > this.playbackExpectedSeq) {
179
+ this.playbackGapCount += frameSeq - this.playbackExpectedSeq;
180
+ }
181
+ this.playbackExpectedSeq = frameSeq + 1;
86
182
  }
87
183
  }
88
184
  /**
@@ -93,6 +189,10 @@ class AnimationHandler {
93
189
  * @internal
94
190
  */
95
191
  async handleTransitionData(protobufData, frameCount) {
192
+ logger.info(
193
+ "AnimationHandler",
194
+ `Start transition packet received (bytes=${protobufData.byteLength}, requestedFrames=${frameCount ?? this.config.transitionStartFrameCount}, hasHandledStart=${this.hasHandledTransitionStart}, isInSession=${this.isInSession}, isPlayingTransition=${this.isPlayingTransition}, isGeneratingStart=${this.isGeneratingStartTransition}, lastRenderedSeq=${this.lastRenderedFrameSeq}, bufferState=${this.bufferState}, buffered=${this.frameBuffer.size})`
195
+ );
96
196
  if (this.hasHandledTransitionStart) {
97
197
  return;
98
198
  }
@@ -106,23 +206,32 @@ class AnimationHandler {
106
206
  logger.warn("AnimationHandler", "Renderer not ready for transition");
107
207
  return;
108
208
  }
109
- const keyframes = this.decoder(protobufData);
110
- if (!keyframes || keyframes.length === 0) {
111
- logger.warn("AnimationHandler", "No target keyframe in transition data");
209
+ if (this.isInSession && (this.lastRenderedFrameSeq >= 0 || this.frameBuffer.size > 0 || this.bufferState !== "direct")) {
210
+ this.hasHandledTransitionStart = true;
211
+ logger.warn(
212
+ "AnimationHandler",
213
+ `Ignoring late transition packet after playback start (lastRenderedSeq=${this.lastRenderedFrameSeq}, bufferState=${this.bufferState}, buffered=${this.frameBuffer.size})`
214
+ );
112
215
  return;
113
216
  }
114
217
  this.hasHandledTransitionStart = true;
115
218
  this.hasHandledTransitionEnd = false;
116
- const targetFrame = keyframes[0];
219
+ this.ensureSessionActive();
117
220
  const frames = frameCount ?? this.config.transitionStartFrameCount;
118
- logger.info("AnimationHandler", `Generating ${frames} transition frames to target`);
221
+ logger.info(
222
+ "AnimationHandler",
223
+ `Generating ${frames} transition frames to target`
224
+ );
119
225
  this.isGeneratingStartTransition = true;
120
226
  try {
121
- const transitionFrames = await this.renderer.generateTransitionFromIdle(
122
- targetFrame,
227
+ const transitionFrames = await this.renderer.generateTransitionFromProtobuf(
228
+ protobufData,
123
229
  frames
124
230
  );
125
- logger.info("AnimationHandler", `Generated ${transitionFrames.length} transition frames`);
231
+ logger.info(
232
+ "AnimationHandler",
233
+ `Generated ${transitionFrames.length} transition frames`
234
+ );
126
235
  this.isPlayingTransition = true;
127
236
  this.isTransitioningToIdle = false;
128
237
  this.transitionFrames = transitionFrames;
@@ -130,7 +239,8 @@ class AnimationHandler {
130
239
  this.playTransitionFrame();
131
240
  } catch (error) {
132
241
  logger.error("AnimationHandler", "Failed to generate transition:", error);
133
- this.renderer.renderFrame(targetFrame);
242
+ this.renderer.renderFromProtobuf(protobufData);
243
+ this.logRenderedFrame("transition-fallback");
134
244
  } finally {
135
245
  this.isGeneratingStartTransition = false;
136
246
  }
@@ -143,6 +253,13 @@ class AnimationHandler {
143
253
  * @internal
144
254
  */
145
255
  async handleTransitionToIdle(protobufData, frameCount) {
256
+ if (!this.isInSession) {
257
+ logger.info(
258
+ "AnimationHandler",
259
+ "Ignoring transition end packet with no active session"
260
+ );
261
+ return;
262
+ }
146
263
  if (this.hasHandledTransitionEnd) {
147
264
  return;
148
265
  }
@@ -153,27 +270,31 @@ class AnimationHandler {
153
270
  return;
154
271
  }
155
272
  if (!this.renderer.isReady()) {
156
- logger.warn("AnimationHandler", "Renderer not ready for transition to idle");
157
- this.renderer.renderFrame(void 0, true);
158
- return;
159
- }
160
- const keyframes = this.decoder(protobufData);
161
- if (!keyframes || keyframes.length === 0) {
162
- logger.warn("AnimationHandler", "No last keyframe in transition end data, starting idle directly");
273
+ logger.warn(
274
+ "AnimationHandler",
275
+ "Renderer not ready for transition to idle"
276
+ );
163
277
  this.renderer.renderFrame(void 0, true);
278
+ this.logRenderedFrame("idle");
164
279
  return;
165
280
  }
166
281
  this.hasHandledTransitionEnd = true;
167
- const lastFrame = keyframes[0];
282
+ this.flushBuffer();
168
283
  const frames = frameCount ?? this.config.transitionEndFrameCount;
169
- logger.info("AnimationHandler", `Generating ${frames} reverse transition frames to idle`);
284
+ logger.info(
285
+ "AnimationHandler",
286
+ `Generating ${frames} reverse transition frames to idle`
287
+ );
170
288
  this.isGeneratingEndTransition = true;
171
289
  try {
172
- const transitionFrames = await this.renderer.generateTransitionFromIdle(
173
- lastFrame,
290
+ const transitionFrames = await this.renderer.generateTransitionFromProtobuf(
291
+ protobufData,
174
292
  frames
175
293
  );
176
- logger.info("AnimationHandler", `Generated ${transitionFrames.length} transition frames, reversing for playback`);
294
+ logger.info(
295
+ "AnimationHandler",
296
+ `Generated ${transitionFrames.length} transition frames, reversing for playback`
297
+ );
177
298
  const reversedFrames = transitionFrames.slice().reverse();
178
299
  this.isPlayingTransition = true;
179
300
  this.isTransitioningToIdle = true;
@@ -181,8 +302,13 @@ class AnimationHandler {
181
302
  this.transitionFrameIndex = 0;
182
303
  this.playTransitionFrame();
183
304
  } catch (error) {
184
- logger.error("AnimationHandler", "Failed to generate reverse transition:", error);
305
+ logger.error(
306
+ "AnimationHandler",
307
+ "Failed to generate reverse transition:",
308
+ error
309
+ );
185
310
  this.renderer.renderFrame(void 0, true);
311
+ this.logRenderedFrame("idle");
186
312
  } finally {
187
313
  this.isGeneratingEndTransition = false;
188
314
  }
@@ -192,7 +318,34 @@ class AnimationHandler {
192
318
  * @internal
193
319
  */
194
320
  startIdle() {
321
+ this.reportConversationStats();
322
+ this.isInSession = false;
323
+ this.hasReportedStall = false;
195
324
  this.renderer.renderFrame(void 0, true);
325
+ this.logRenderedFrame("idle");
326
+ }
327
+ /**
328
+ * Ensure session-level timers/stats are active.
329
+ * @internal
330
+ */
331
+ ensureSessionActive(frameSeq) {
332
+ if (this.isInSession) {
333
+ return;
334
+ }
335
+ this.isInSession = true;
336
+ this.lastFrameReceivedTime = Date.now();
337
+ this.hasReportedStall = false;
338
+ if (!this.conversationStartTime) {
339
+ this.conversationStartTime = Date.now();
340
+ }
341
+ this.startWatchdog();
342
+ this.startPlaybackStats();
343
+ if (frameSeq !== void 0) {
344
+ logger.info(
345
+ "AnimationHandler",
346
+ `Session started from animation frame seq=${frameSeq}`
347
+ );
348
+ }
196
349
  }
197
350
  /**
198
351
  * Reset animation frame tracking (call on session start).
@@ -204,6 +357,8 @@ class AnimationHandler {
204
357
  this.animationFrameCount = 0;
205
358
  this.hasHandledTransitionStart = false;
206
359
  this.hasHandledTransitionEnd = false;
360
+ this.resetPlaybackStats();
361
+ this.flushBuffer();
207
362
  logger.info("AnimationHandler", "Frame tracking reset");
208
363
  }
209
364
  /**
@@ -211,7 +366,7 @@ class AnimationHandler {
211
366
  * @internal
212
367
  */
213
368
  isInTransition() {
214
- return this.isPlayingTransition;
369
+ return this.isPlayingTransition || this.isGeneratingStartTransition || this.isGeneratingEndTransition;
215
370
  }
216
371
  /**
217
372
  * Stop transition playback.
@@ -231,6 +386,58 @@ class AnimationHandler {
231
386
  clearTimeout(this.transitionTimeoutId);
232
387
  this.transitionTimeoutId = null;
233
388
  }
389
+ this.flushBuffer();
390
+ }
391
+ /**
392
+ * Get cumulative session summary for telemetry reporting.
393
+ * @internal
394
+ */
395
+ getSessionSummary() {
396
+ const avgFps = this.cumulativeFpsReadings.length > 0 ? Math.round(
397
+ this.cumulativeFpsReadings.reduce((a, b) => a + b, 0) / this.cumulativeFpsReadings.length
398
+ ) : 0;
399
+ return {
400
+ totalFrames: this.cumulativeTotalFrames,
401
+ totalLost: this.cumulativeLost,
402
+ totalRecovered: this.cumulativeRecovered,
403
+ totalDropped: this.cumulativeDropped,
404
+ avgFps
405
+ };
406
+ }
407
+ /**
408
+ * Report per-conversation playback stats via telemetry.
409
+ * Called when a conversation ends (transition to idle).
410
+ * @internal
411
+ */
412
+ reportConversationStats() {
413
+ if (this.conversationFrameCount === 0) return;
414
+ const durationMs = this.conversationStartTime ? Date.now() - this.conversationStartTime : 0;
415
+ const avgFps = this.conversationFpsReadings.length > 0 ? Math.round(
416
+ this.conversationFpsReadings.reduce((a, b) => a + b, 0) / this.conversationFpsReadings.length
417
+ ) : 0;
418
+ const totalExpected = this.conversationFrameCount + this.conversationLost;
419
+ const lossRate = totalExpected > 0 ? Number((this.conversationLost / totalExpected * 100).toFixed(1)) : 0;
420
+ trackEvent("rtc_playback_stats", "info", {
421
+ provider: this.providerName,
422
+ avg_fps: avgFps,
423
+ frame_count: this.conversationFrameCount,
424
+ frames_lost: this.conversationLost,
425
+ frames_recovered: this.conversationRecovered,
426
+ frames_dropped: this.conversationDropped,
427
+ loss_rate: lossRate,
428
+ duration_ms: durationMs
429
+ });
430
+ this.cumulativeTotalFrames += this.conversationFrameCount;
431
+ this.cumulativeLost += this.conversationLost;
432
+ this.cumulativeRecovered += this.conversationRecovered;
433
+ this.cumulativeDropped += this.conversationDropped;
434
+ this.cumulativeFpsReadings.push(...this.conversationFpsReadings);
435
+ this.conversationFrameCount = 0;
436
+ this.conversationLost = 0;
437
+ this.conversationRecovered = 0;
438
+ this.conversationDropped = 0;
439
+ this.conversationFpsReadings = [];
440
+ this.conversationStartTime = 0;
234
441
  }
235
442
  /**
236
443
  * Clean up resources.
@@ -238,6 +445,339 @@ class AnimationHandler {
238
445
  */
239
446
  dispose() {
240
447
  this.stopTransition();
448
+ this.stopWatchdog();
449
+ }
450
+ /**
451
+ * Start the watchdog timer to detect stalled data streams.
452
+ * @internal
453
+ */
454
+ startWatchdog() {
455
+ if (this.watchdogTimer) {
456
+ return;
457
+ }
458
+ this.watchdogTimer = setInterval(() => {
459
+ if (!this.isInSession) {
460
+ return;
461
+ }
462
+ if (this.isPlayingTransition) {
463
+ return;
464
+ }
465
+ const elapsed = Date.now() - this.lastFrameReceivedTime;
466
+ if (elapsed > _AnimationHandler.STALL_TIMEOUT_MS && !this.hasReportedStall) {
467
+ logger.error(
468
+ "AnimationHandler",
469
+ `Data stream stalled: no frames received for ${elapsed}ms, falling back to idle`
470
+ );
471
+ this.hasReportedStall = true;
472
+ this.isStalledFallback = true;
473
+ this.startIdle();
474
+ if (this.onStreamStalledCallback) {
475
+ try {
476
+ this.onStreamStalledCallback();
477
+ } catch (e) {
478
+ logger.error(
479
+ "AnimationHandler",
480
+ "Error in onStreamStalled callback:",
481
+ e
482
+ );
483
+ }
484
+ }
485
+ }
486
+ }, 1e3);
487
+ }
488
+ /**
489
+ * Stop the watchdog timer.
490
+ * @internal
491
+ */
492
+ stopWatchdog() {
493
+ if (this.watchdogTimer) {
494
+ clearInterval(this.watchdogTimer);
495
+ this.watchdogTimer = null;
496
+ }
497
+ this.hasReportedStall = false;
498
+ this.isStalledFallback = false;
499
+ this.stopPlaybackStats();
500
+ }
501
+ /**
502
+ * Start the playback stats reporting timer.
503
+ * @internal
504
+ */
505
+ startPlaybackStats() {
506
+ if (this.playbackStatsTimer) {
507
+ return;
508
+ }
509
+ this.resetPlaybackStats();
510
+ this.playbackStatsTimer = setInterval(() => {
511
+ this.reportPlaybackStats();
512
+ }, 1e3);
513
+ }
514
+ /**
515
+ * Stop the playback stats reporting timer.
516
+ * @internal
517
+ */
518
+ stopPlaybackStats() {
519
+ if (this.playbackStatsTimer) {
520
+ clearInterval(this.playbackStatsTimer);
521
+ this.playbackStatsTimer = null;
522
+ }
523
+ this.resetPlaybackStats();
524
+ }
525
+ /**
526
+ * Reset playback stats counters.
527
+ * @internal
528
+ */
529
+ resetPlaybackStats() {
530
+ this.playbackFrameCount = 0;
531
+ this.playbackFrameTimestamps = [];
532
+ this.playbackGapCount = 0;
533
+ this.playbackExpectedSeq = -1;
534
+ }
535
+ /**
536
+ * Report playback stats (called every 1s by timer).
537
+ * Logs FPS, frame loss rate, and playback jitter.
538
+ * @internal
539
+ */
540
+ reportPlaybackStats() {
541
+ if (this.isPlayingTransition) {
542
+ this.resetPlaybackStats();
543
+ return;
544
+ }
545
+ if (this.playbackFrameCount === 0) {
546
+ return;
547
+ }
548
+ const fps = this.playbackFrameCount;
549
+ const totalExpected = this.playbackFrameCount + this.playbackGapCount;
550
+ const lossRate = totalExpected > 0 ? this.playbackGapCount / totalExpected * 100 : 0;
551
+ let jitter = 0;
552
+ if (this.playbackFrameTimestamps.length >= 2) {
553
+ const intervals = [];
554
+ for (let i = 1; i < this.playbackFrameTimestamps.length; i++) {
555
+ intervals.push(
556
+ this.playbackFrameTimestamps[i] - this.playbackFrameTimestamps[i - 1]
557
+ );
558
+ }
559
+ const mean = intervals.reduce((a, b) => a + b, 0) / intervals.length;
560
+ const variance = intervals.reduce((sum, v) => sum + (v - mean) ** 2, 0) / intervals.length;
561
+ jitter = Math.sqrt(variance);
562
+ }
563
+ logger.info(
564
+ "AnimationHandler",
565
+ `Playback stats: fps=${fps}, lossRate=${lossRate.toFixed(1)}%, jitter=${jitter.toFixed(1)}ms`
566
+ );
567
+ this.conversationFrameCount += this.playbackFrameCount;
568
+ this.conversationLost += this.playbackGapCount;
569
+ this.conversationFpsReadings.push(fps);
570
+ this.playbackFrameCount = 0;
571
+ this.playbackFrameTimestamps = [];
572
+ this.playbackGapCount = 0;
573
+ }
574
+ // ── Jitter Buffer ──
575
+ /**
576
+ * Insert a frame into the jitter buffer.
577
+ * Handles dedup, enforces max size, and drives the buffer state machine.
578
+ * @internal
579
+ */
580
+ bufferFrame(protobufData, seq) {
581
+ if (this.lastRenderedFrameSeq >= 0 && seq <= this.lastRenderedFrameSeq) {
582
+ logger.warn(
583
+ "AnimationHandler",
584
+ `Jitter buffer: dropping stale frame seq=${seq} (lastRendered=${this.lastRenderedFrameSeq})`
585
+ );
586
+ this.conversationDropped++;
587
+ return;
588
+ }
589
+ if (this.bufferNextSeq >= 0 && seq < this.bufferNextSeq) {
590
+ logger.warn(
591
+ "AnimationHandler",
592
+ `Jitter buffer: dropping late frame seq=${seq} (nextExpected=${this.bufferNextSeq})`
593
+ );
594
+ this.conversationDropped++;
595
+ return;
596
+ }
597
+ if (this.frameBuffer.has(seq)) {
598
+ return;
599
+ }
600
+ this.frameBuffer.set(seq, { protobufData, seq, receivedAt: performance.now() });
601
+ if (this.frameBuffer.size > _AnimationHandler.BUFFER_MAX_SIZE) {
602
+ let oldestSeq = Infinity;
603
+ for (const k of this.frameBuffer.keys()) {
604
+ if (k < oldestSeq) oldestSeq = k;
605
+ }
606
+ this.frameBuffer.delete(oldestSeq);
607
+ this.conversationDropped++;
608
+ logger.warn(
609
+ "AnimationHandler",
610
+ `Jitter buffer: overflow, dropping seq=${oldestSeq} (nextExpected=${this.bufferNextSeq})`
611
+ );
612
+ }
613
+ switch (this.bufferState) {
614
+ case "direct":
615
+ this.bufferState = "filling";
616
+ if (this.bufferNextSeq < 0) {
617
+ this.bufferNextSeq = seq;
618
+ }
619
+ logger.info(
620
+ "AnimationHandler",
621
+ `Jitter buffer: filling (first frame seq=${seq})`
622
+ );
623
+ if (this.frameBuffer.size >= _AnimationHandler.BUFFER_INITIAL_FILL) {
624
+ this.startBufferDrain();
625
+ }
626
+ break;
627
+ case "filling":
628
+ if (this.frameBuffer.size >= _AnimationHandler.BUFFER_INITIAL_FILL) {
629
+ this.startBufferDrain();
630
+ }
631
+ break;
632
+ case "starved":
633
+ this.startBufferDrain();
634
+ break;
635
+ }
636
+ }
637
+ /**
638
+ * Drop buffered frames that are now too old to ever be rendered in-order.
639
+ * @internal
640
+ */
641
+ dropStaleBufferedFrames() {
642
+ if (this.frameBuffer.size === 0) {
643
+ return;
644
+ }
645
+ const minAllowedSeq = Math.max(
646
+ this.bufferNextSeq,
647
+ this.lastRenderedFrameSeq + 1
648
+ );
649
+ if (minAllowedSeq < 0) {
650
+ return;
651
+ }
652
+ let dropped = 0;
653
+ for (const seq of Array.from(this.frameBuffer.keys())) {
654
+ if (seq < minAllowedSeq) {
655
+ this.frameBuffer.delete(seq);
656
+ dropped++;
657
+ }
658
+ }
659
+ if (dropped > 0) {
660
+ logger.warn(
661
+ "AnimationHandler",
662
+ `Jitter buffer: dropped ${dropped} stale frame(s) older than seq=${minAllowedSeq}`
663
+ );
664
+ }
665
+ }
666
+ /**
667
+ * Find the lowest buffered sequence at or after minSeq.
668
+ * @internal
669
+ */
670
+ findLowestBufferedSeqAtOrAfter(minSeq) {
671
+ let candidate = Infinity;
672
+ for (const seq of this.frameBuffer.keys()) {
673
+ if (seq >= minSeq && seq < candidate) {
674
+ candidate = seq;
675
+ }
676
+ }
677
+ return candidate === Infinity ? null : candidate;
678
+ }
679
+ /**
680
+ * Begin draining the buffer at 25fps.
681
+ * @internal
682
+ */
683
+ startBufferDrain() {
684
+ this.bufferState = "draining";
685
+ if (this.bufferNextSeq < 0) {
686
+ let minSeq = Infinity;
687
+ for (const k of this.frameBuffer.keys()) {
688
+ if (k < minSeq) minSeq = k;
689
+ }
690
+ this.bufferNextSeq = minSeq;
691
+ }
692
+ logger.info(
693
+ "AnimationHandler",
694
+ `Jitter buffer: draining (${this.frameBuffer.size} frames buffered)`
695
+ );
696
+ this.bufferLastDrainTime = performance.now();
697
+ this.drainBufferFrame();
698
+ }
699
+ /**
700
+ * Drain one frame from the buffer and schedule the next drain.
701
+ * Handles missing frames (skip-ahead after maxBufferDelayMs) and starvation.
702
+ * @internal
703
+ */
704
+ drainBufferFrame() {
705
+ this.bufferDrainTimer = null;
706
+ if (this.bufferState !== "draining") {
707
+ return;
708
+ }
709
+ this.dropStaleBufferedFrames();
710
+ const frame = this.frameBuffer.get(this.bufferNextSeq);
711
+ if (frame) {
712
+ this.renderBufferedFrame(frame);
713
+ this.frameBuffer.delete(this.bufferNextSeq);
714
+ this.bufferNextSeq++;
715
+ } else if (this.frameBuffer.size > 0) {
716
+ const nextSeq = this.findLowestBufferedSeqAtOrAfter(this.bufferNextSeq);
717
+ if (nextSeq === null) {
718
+ this.bufferState = "starved";
719
+ logger.warn(
720
+ "AnimationHandler",
721
+ "Jitter buffer: no in-order frames available, pausing drain"
722
+ );
723
+ return;
724
+ }
725
+ const nextFrame = this.frameBuffer.get(nextSeq);
726
+ const waitTime = performance.now() - nextFrame.receivedAt;
727
+ if (waitTime > this.config.maxBufferDelayMs) {
728
+ const gap = Math.max(0, nextSeq - this.bufferNextSeq);
729
+ this.playbackGapCount += gap;
730
+ logger.warn(
731
+ "AnimationHandler",
732
+ `Jitter buffer: skipping ${gap} frame(s) from seq ${this.bufferNextSeq} to ${nextSeq} after ${waitTime.toFixed(1)}ms`
733
+ );
734
+ this.renderBufferedFrame(nextFrame);
735
+ this.frameBuffer.delete(nextSeq);
736
+ this.bufferNextSeq = nextSeq + 1;
737
+ }
738
+ } else {
739
+ this.bufferState = "starved";
740
+ logger.warn("AnimationHandler", "Jitter buffer: starved, pausing drain");
741
+ return;
742
+ }
743
+ const now = performance.now();
744
+ const nextTarget = this.bufferLastDrainTime + _AnimationHandler.BUFFER_FRAME_INTERVAL_MS;
745
+ const delay = Math.max(0, nextTarget - now);
746
+ this.bufferLastDrainTime = nextTarget;
747
+ this.bufferDrainTimer = setTimeout(() => this.drainBufferFrame(), delay);
748
+ }
749
+ /**
750
+ * Render a single frame from the buffer and collect playback stats.
751
+ * @internal
752
+ */
753
+ renderBufferedFrame(frame) {
754
+ if (this.lastRenderedFrameSeq >= 0 && frame.seq <= this.lastRenderedFrameSeq) {
755
+ logger.warn(
756
+ "AnimationHandler",
757
+ `Jitter buffer: refusing out-of-order render seq=${frame.seq} (lastRendered=${this.lastRenderedFrameSeq})`
758
+ );
759
+ return;
760
+ }
761
+ this.renderer.renderFromProtobuf(frame.protobufData);
762
+ this.lastRenderedFrameSeq = frame.seq;
763
+ this.renderedFrameCount++;
764
+ this.logRenderedFrame("buffer", frame.seq);
765
+ this.playbackFrameTimestamps.push(performance.now());
766
+ this.playbackFrameCount++;
767
+ }
768
+ /**
769
+ * Flush the jitter buffer, stop the drain loop, and revert to direct mode.
770
+ * @internal
771
+ */
772
+ flushBuffer() {
773
+ this.frameBuffer.clear();
774
+ this.bufferState = "direct";
775
+ this.bufferNextSeq = -1;
776
+ this.bufferLastDrainTime = 0;
777
+ if (this.bufferDrainTimer !== null) {
778
+ clearTimeout(this.bufferDrainTimer);
779
+ this.bufferDrainTimer = null;
780
+ }
241
781
  }
242
782
  /**
243
783
  * Play a single transition frame and schedule the next one.
@@ -256,8 +796,12 @@ class AnimationHandler {
256
796
  }
257
797
  logger.info("AnimationHandler", "Transition playback complete");
258
798
  if (wasTransitioningToIdle) {
259
- logger.info("AnimationHandler", "Starting idle animation after transition");
799
+ logger.info(
800
+ "AnimationHandler",
801
+ "Starting idle animation after transition"
802
+ );
260
803
  this.renderer.renderFrame(void 0, true);
804
+ this.logRenderedFrame("idle");
261
805
  }
262
806
  return;
263
807
  }
@@ -268,6 +812,7 @@ class AnimationHandler {
268
812
  }
269
813
  const frame = this.transitionFrames[this.transitionFrameIndex];
270
814
  this.renderer.renderFrame(frame);
815
+ this.logRenderedFrame("transition");
271
816
  this.transitionFrameIndex++;
272
817
  this.transitionTimeoutId = setTimeout(() => {
273
818
  if (this.isPlayingTransition) {
@@ -275,7 +820,26 @@ class AnimationHandler {
275
820
  }
276
821
  }, 40);
277
822
  }
278
- }
823
+ /**
824
+ * Emit a per-frame render log for debugging ordering issues.
825
+ * @internal
826
+ */
827
+ logRenderedFrame(source, seq, isRecovered) {
828
+ logger.info(
829
+ "AnimationHandler",
830
+ `Rendered frame: source=${source}, seq=${seq ?? "n/a"}${isRecovered ? " [RECOVERED]" : ""}`
831
+ );
832
+ }
833
+ };
834
+ /** @internal */
835
+ __publicField(_AnimationHandler, "STALL_TIMEOUT_MS", 5e3);
836
+ /** @internal */
837
+ __publicField(_AnimationHandler, "BUFFER_MAX_SIZE", 4);
838
+ /** @internal */
839
+ __publicField(_AnimationHandler, "BUFFER_INITIAL_FILL", 2);
840
+ /** @internal */
841
+ __publicField(_AnimationHandler, "BUFFER_FRAME_INTERVAL_MS", 40);
842
+ let AnimationHandler = _AnimationHandler;
279
843
  export {
280
844
  AnimationHandler
281
845
  };