@spatialwalk/avatarkit-rtc 1.0.0-beta.1 → 1.0.0-beta.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -410
- package/dist/assets/animation-worker-DOGeTjF0.js.map +1 -0
- package/dist/core/AvatarPlayer.d.ts +96 -12
- package/dist/core/AvatarPlayer.d.ts.map +1 -1
- package/dist/core/RTCProvider.d.ts +12 -16
- package/dist/core/RTCProvider.d.ts.map +1 -1
- package/dist/index.d.ts +3 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -2
- package/dist/index10.js +91 -39
- package/dist/index10.js.map +1 -1
- package/dist/index11.js +14 -386
- package/dist/index11.js.map +1 -1
- package/dist/index12.js +350 -64
- package/dist/index12.js.map +1 -1
- package/dist/index13.js +44 -14
- package/dist/index13.js.map +1 -1
- package/dist/index14.js +25 -44
- package/dist/index14.js.map +1 -1
- package/dist/index2.js +335 -46
- package/dist/index2.js.map +1 -1
- package/dist/index3.js +265 -54
- package/dist/index3.js.map +1 -1
- package/dist/index4.js +105 -86
- package/dist/index4.js.map +1 -1
- package/dist/index5.js +6 -2
- package/dist/index5.js.map +1 -1
- package/dist/index6.js +603 -39
- package/dist/index6.js.map +1 -1
- package/dist/index8.js +128 -167
- package/dist/index8.js.map +1 -1
- package/dist/index9.js +65 -164
- package/dist/index9.js.map +1 -1
- package/dist/providers/agora/AgoraProvider.d.ts +0 -13
- package/dist/providers/agora/AgoraProvider.d.ts.map +1 -1
- package/dist/providers/agora/index.d.ts +1 -5
- package/dist/providers/agora/index.d.ts.map +1 -1
- package/dist/providers/agora/types.d.ts.map +1 -1
- package/dist/providers/base/BaseProvider.d.ts +50 -8
- package/dist/providers/base/BaseProvider.d.ts.map +1 -1
- package/dist/providers/livekit/LiveKitProvider.d.ts +4 -15
- package/dist/providers/livekit/LiveKitProvider.d.ts.map +1 -1
- package/dist/providers/livekit/animation-worker.d.ts.map +1 -1
- package/dist/providers/livekit/index.d.ts +1 -5
- package/dist/providers/livekit/index.d.ts.map +1 -1
- package/dist/types/index.d.ts +21 -0
- package/dist/types/index.d.ts.map +1 -1
- package/dist/utils/index.d.ts +2 -0
- package/dist/utils/index.d.ts.map +1 -1
- package/dist/utils/telemetry.d.ts +22 -0
- package/dist/utils/telemetry.d.ts.map +1 -0
- package/package.json +21 -12
- package/dist/assets/animation-worker-CUXZycUw.js.map +0 -1
- package/dist/index15.js +0 -29
- package/dist/index15.js.map +0 -1
- package/dist/index16.js +0 -144
- package/dist/index16.js.map +0 -1
- package/dist/index17.js +0 -106
- package/dist/index17.js.map +0 -1
- package/dist/index18.js +0 -28
- package/dist/index18.js.map +0 -1
- package/dist/proto/animation.d.ts +0 -12
- package/dist/proto/animation.d.ts.map +0 -1
package/dist/index6.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
var __defProp = Object.defineProperty;
|
|
2
2
|
var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
|
|
3
3
|
var __publicField = (obj, key, value) => __defNormalProp(obj, typeof key !== "symbol" ? key + "" : key, value);
|
|
4
|
-
import { decodeAnimationKeyframes } from "./index8.js";
|
|
5
4
|
import { logger } from "./index7.js";
|
|
6
|
-
|
|
5
|
+
import { trackEvent } from "./index8.js";
|
|
6
|
+
const _AnimationHandler = class _AnimationHandler {
|
|
7
7
|
/**
|
|
8
8
|
* @internal
|
|
9
9
|
*/
|
|
@@ -11,8 +11,6 @@ class AnimationHandler {
|
|
|
11
11
|
/** @internal */
|
|
12
12
|
__publicField(this, "renderer");
|
|
13
13
|
/** @internal */
|
|
14
|
-
__publicField(this, "decoder");
|
|
15
|
-
/** @internal */
|
|
16
14
|
__publicField(this, "config");
|
|
17
15
|
// Frame tracking
|
|
18
16
|
/** @internal */
|
|
@@ -43,12 +41,77 @@ class AnimationHandler {
|
|
|
43
41
|
__publicField(this, "hasHandledTransitionStart", false);
|
|
44
42
|
/** @internal */
|
|
45
43
|
__publicField(this, "hasHandledTransitionEnd", false);
|
|
44
|
+
// Watchdog for detecting stalled data stream
|
|
45
|
+
/** @internal */
|
|
46
|
+
__publicField(this, "lastFrameReceivedTime", 0);
|
|
47
|
+
/** @internal */
|
|
48
|
+
__publicField(this, "isInSession", false);
|
|
49
|
+
/** @internal */
|
|
50
|
+
__publicField(this, "watchdogTimer", null);
|
|
51
|
+
/** @internal */
|
|
52
|
+
__publicField(this, "hasReportedStall", false);
|
|
53
|
+
/** @internal */
|
|
54
|
+
__publicField(this, "onStreamStalledCallback", null);
|
|
55
|
+
/** @internal */
|
|
56
|
+
__publicField(this, "providerName", "");
|
|
57
|
+
/** @internal */
|
|
58
|
+
__publicField(this, "isStalledFallback", false);
|
|
59
|
+
// True when fallback to idle was triggered due to stall
|
|
60
|
+
// Playback stats
|
|
61
|
+
/** @internal */
|
|
62
|
+
__publicField(this, "playbackStatsTimer", null);
|
|
63
|
+
/** @internal */
|
|
64
|
+
__publicField(this, "playbackFrameCount", 0);
|
|
65
|
+
/** @internal */
|
|
66
|
+
__publicField(this, "playbackFrameTimestamps", []);
|
|
67
|
+
/** @internal */
|
|
68
|
+
__publicField(this, "playbackGapCount", 0);
|
|
69
|
+
/** @internal */
|
|
70
|
+
__publicField(this, "playbackExpectedSeq", -1);
|
|
71
|
+
// Jitter buffer
|
|
72
|
+
/** @internal */
|
|
73
|
+
__publicField(this, "bufferState", "direct");
|
|
74
|
+
/** @internal */
|
|
75
|
+
__publicField(this, "frameBuffer", /* @__PURE__ */ new Map());
|
|
76
|
+
/** @internal */
|
|
77
|
+
__publicField(this, "bufferNextSeq", -1);
|
|
78
|
+
/** @internal */
|
|
79
|
+
__publicField(this, "bufferDrainTimer", null);
|
|
80
|
+
/** @internal */
|
|
81
|
+
__publicField(this, "bufferLastDrainTime", 0);
|
|
82
|
+
// Cumulative session stats (never reset until getSessionSummary)
|
|
83
|
+
/** @internal */
|
|
84
|
+
__publicField(this, "cumulativeTotalFrames", 0);
|
|
85
|
+
/** @internal */
|
|
86
|
+
__publicField(this, "cumulativeLost", 0);
|
|
87
|
+
/** @internal */
|
|
88
|
+
__publicField(this, "cumulativeRecovered", 0);
|
|
89
|
+
/** @internal */
|
|
90
|
+
__publicField(this, "cumulativeDropped", 0);
|
|
91
|
+
/** @internal */
|
|
92
|
+
__publicField(this, "cumulativeFpsReadings", []);
|
|
93
|
+
// Per-conversation stats for rtc_playback_stats
|
|
94
|
+
/** @internal */
|
|
95
|
+
__publicField(this, "conversationFrameCount", 0);
|
|
96
|
+
/** @internal */
|
|
97
|
+
__publicField(this, "conversationLost", 0);
|
|
98
|
+
/** @internal */
|
|
99
|
+
__publicField(this, "conversationRecovered", 0);
|
|
100
|
+
/** @internal */
|
|
101
|
+
__publicField(this, "conversationDropped", 0);
|
|
102
|
+
/** @internal */
|
|
103
|
+
__publicField(this, "conversationFpsReadings", []);
|
|
104
|
+
/** @internal */
|
|
105
|
+
__publicField(this, "conversationStartTime", 0);
|
|
46
106
|
this.renderer = renderer;
|
|
47
|
-
this.decoder = config.decoder ?? decodeAnimationKeyframes;
|
|
48
107
|
this.config = {
|
|
49
108
|
transitionStartFrameCount: config.transitionStartFrameCount ?? 8,
|
|
50
|
-
transitionEndFrameCount: config.transitionEndFrameCount ?? 12
|
|
109
|
+
transitionEndFrameCount: config.transitionEndFrameCount ?? 12,
|
|
110
|
+
enableJitterBuffer: config.enableJitterBuffer ?? true,
|
|
111
|
+
maxBufferDelayMs: config.maxBufferDelayMs ?? 80
|
|
51
112
|
};
|
|
113
|
+
this.onStreamStalledCallback = config.onStreamStalled ?? null;
|
|
114
|
+
this.providerName = config.providerName ?? "";
|
|
52
115
|
}
|
|
53
116
|
/**
|
|
54
117
|
* Handle animation data received from RTC provider.
|
|
@@ -61,17 +124,42 @@ class AnimationHandler {
|
|
|
61
124
|
if (this.isPlayingTransition) {
|
|
62
125
|
this.stopTransition();
|
|
63
126
|
}
|
|
127
|
+
const now = Date.now();
|
|
128
|
+
if (this.hasReportedStall) {
|
|
129
|
+
const stallDuration = now - this.lastFrameReceivedTime;
|
|
130
|
+
logger.info(
|
|
131
|
+
"AnimationHandler",
|
|
132
|
+
`Data stream resumed after ${stallDuration}ms stall`
|
|
133
|
+
);
|
|
134
|
+
this.hasReportedStall = false;
|
|
135
|
+
}
|
|
136
|
+
if (this.isStalledFallback) {
|
|
137
|
+
logger.info(
|
|
138
|
+
"AnimationHandler",
|
|
139
|
+
"Resuming from stall fallback, rendering directly without transition"
|
|
140
|
+
);
|
|
141
|
+
this.isStalledFallback = false;
|
|
142
|
+
}
|
|
143
|
+
this.lastFrameReceivedTime = now;
|
|
64
144
|
this.animationFrameCount++;
|
|
65
|
-
this.
|
|
145
|
+
this.ensureSessionActive(frameSeq);
|
|
146
|
+
if (this.config.enableJitterBuffer && frameSeq !== void 0) {
|
|
147
|
+
this.bufferFrame(protobufData, frameSeq);
|
|
148
|
+
return;
|
|
149
|
+
}
|
|
66
150
|
if (frameSeq !== void 0 && this.lastRenderedFrameSeq !== -1) {
|
|
67
|
-
if (frameSeq
|
|
68
|
-
logger.
|
|
151
|
+
if (frameSeq < this.lastRenderedFrameSeq) {
|
|
152
|
+
logger.warn(
|
|
69
153
|
"AnimationHandler",
|
|
70
|
-
`OUT-OF-ORDER
|
|
154
|
+
`OUT-OF-ORDER: seq=${frameSeq}, lastRendered=${this.lastRenderedFrameSeq}${isRecovered ? " [RECOVERED]" : ""}, discarding`
|
|
71
155
|
);
|
|
156
|
+
this.conversationDropped++;
|
|
157
|
+
return;
|
|
158
|
+
} else if (frameSeq === this.lastRenderedFrameSeq) {
|
|
159
|
+
return;
|
|
72
160
|
} else if (frameSeq > this.lastRenderedFrameSeq + 1) {
|
|
73
161
|
const gap = frameSeq - this.lastRenderedFrameSeq - 1;
|
|
74
|
-
logger.
|
|
162
|
+
logger.warn(
|
|
75
163
|
"AnimationHandler",
|
|
76
164
|
`GAP: ${gap} frame(s) between ${this.lastRenderedFrameSeq} and ${frameSeq}${isRecovered ? " [RECOVERED]" : ""}`
|
|
77
165
|
);
|
|
@@ -80,9 +168,17 @@ class AnimationHandler {
|
|
|
80
168
|
if (frameSeq !== void 0) {
|
|
81
169
|
this.lastRenderedFrameSeq = frameSeq;
|
|
82
170
|
}
|
|
83
|
-
|
|
84
|
-
if (
|
|
85
|
-
|
|
171
|
+
this.renderedFrameCount++;
|
|
172
|
+
if (isRecovered) this.conversationRecovered++;
|
|
173
|
+
this.renderer.renderFromProtobuf(protobufData);
|
|
174
|
+
this.logRenderedFrame("direct", frameSeq, isRecovered);
|
|
175
|
+
this.playbackFrameTimestamps.push(performance.now());
|
|
176
|
+
this.playbackFrameCount++;
|
|
177
|
+
if (frameSeq !== void 0) {
|
|
178
|
+
if (this.playbackExpectedSeq >= 0 && frameSeq > this.playbackExpectedSeq) {
|
|
179
|
+
this.playbackGapCount += frameSeq - this.playbackExpectedSeq;
|
|
180
|
+
}
|
|
181
|
+
this.playbackExpectedSeq = frameSeq + 1;
|
|
86
182
|
}
|
|
87
183
|
}
|
|
88
184
|
/**
|
|
@@ -93,6 +189,10 @@ class AnimationHandler {
|
|
|
93
189
|
* @internal
|
|
94
190
|
*/
|
|
95
191
|
async handleTransitionData(protobufData, frameCount) {
|
|
192
|
+
logger.info(
|
|
193
|
+
"AnimationHandler",
|
|
194
|
+
`Start transition packet received (bytes=${protobufData.byteLength}, requestedFrames=${frameCount ?? this.config.transitionStartFrameCount}, hasHandledStart=${this.hasHandledTransitionStart}, isInSession=${this.isInSession}, isPlayingTransition=${this.isPlayingTransition}, isGeneratingStart=${this.isGeneratingStartTransition}, lastRenderedSeq=${this.lastRenderedFrameSeq}, bufferState=${this.bufferState}, buffered=${this.frameBuffer.size})`
|
|
195
|
+
);
|
|
96
196
|
if (this.hasHandledTransitionStart) {
|
|
97
197
|
return;
|
|
98
198
|
}
|
|
@@ -106,23 +206,32 @@ class AnimationHandler {
|
|
|
106
206
|
logger.warn("AnimationHandler", "Renderer not ready for transition");
|
|
107
207
|
return;
|
|
108
208
|
}
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
logger.warn(
|
|
209
|
+
if (this.isInSession && (this.lastRenderedFrameSeq >= 0 || this.frameBuffer.size > 0 || this.bufferState !== "direct")) {
|
|
210
|
+
this.hasHandledTransitionStart = true;
|
|
211
|
+
logger.warn(
|
|
212
|
+
"AnimationHandler",
|
|
213
|
+
`Ignoring late transition packet after playback start (lastRenderedSeq=${this.lastRenderedFrameSeq}, bufferState=${this.bufferState}, buffered=${this.frameBuffer.size})`
|
|
214
|
+
);
|
|
112
215
|
return;
|
|
113
216
|
}
|
|
114
217
|
this.hasHandledTransitionStart = true;
|
|
115
218
|
this.hasHandledTransitionEnd = false;
|
|
116
|
-
|
|
219
|
+
this.ensureSessionActive();
|
|
117
220
|
const frames = frameCount ?? this.config.transitionStartFrameCount;
|
|
118
|
-
logger.info(
|
|
221
|
+
logger.info(
|
|
222
|
+
"AnimationHandler",
|
|
223
|
+
`Generating ${frames} transition frames to target`
|
|
224
|
+
);
|
|
119
225
|
this.isGeneratingStartTransition = true;
|
|
120
226
|
try {
|
|
121
|
-
const transitionFrames = await this.renderer.
|
|
122
|
-
|
|
227
|
+
const transitionFrames = await this.renderer.generateTransitionFromProtobuf(
|
|
228
|
+
protobufData,
|
|
123
229
|
frames
|
|
124
230
|
);
|
|
125
|
-
logger.info(
|
|
231
|
+
logger.info(
|
|
232
|
+
"AnimationHandler",
|
|
233
|
+
`Generated ${transitionFrames.length} transition frames`
|
|
234
|
+
);
|
|
126
235
|
this.isPlayingTransition = true;
|
|
127
236
|
this.isTransitioningToIdle = false;
|
|
128
237
|
this.transitionFrames = transitionFrames;
|
|
@@ -130,7 +239,8 @@ class AnimationHandler {
|
|
|
130
239
|
this.playTransitionFrame();
|
|
131
240
|
} catch (error) {
|
|
132
241
|
logger.error("AnimationHandler", "Failed to generate transition:", error);
|
|
133
|
-
this.renderer.
|
|
242
|
+
this.renderer.renderFromProtobuf(protobufData);
|
|
243
|
+
this.logRenderedFrame("transition-fallback");
|
|
134
244
|
} finally {
|
|
135
245
|
this.isGeneratingStartTransition = false;
|
|
136
246
|
}
|
|
@@ -143,6 +253,13 @@ class AnimationHandler {
|
|
|
143
253
|
* @internal
|
|
144
254
|
*/
|
|
145
255
|
async handleTransitionToIdle(protobufData, frameCount) {
|
|
256
|
+
if (!this.isInSession) {
|
|
257
|
+
logger.info(
|
|
258
|
+
"AnimationHandler",
|
|
259
|
+
"Ignoring transition end packet with no active session"
|
|
260
|
+
);
|
|
261
|
+
return;
|
|
262
|
+
}
|
|
146
263
|
if (this.hasHandledTransitionEnd) {
|
|
147
264
|
return;
|
|
148
265
|
}
|
|
@@ -153,27 +270,31 @@ class AnimationHandler {
|
|
|
153
270
|
return;
|
|
154
271
|
}
|
|
155
272
|
if (!this.renderer.isReady()) {
|
|
156
|
-
logger.warn(
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
const keyframes = this.decoder(protobufData);
|
|
161
|
-
if (!keyframes || keyframes.length === 0) {
|
|
162
|
-
logger.warn("AnimationHandler", "No last keyframe in transition end data, starting idle directly");
|
|
273
|
+
logger.warn(
|
|
274
|
+
"AnimationHandler",
|
|
275
|
+
"Renderer not ready for transition to idle"
|
|
276
|
+
);
|
|
163
277
|
this.renderer.renderFrame(void 0, true);
|
|
278
|
+
this.logRenderedFrame("idle");
|
|
164
279
|
return;
|
|
165
280
|
}
|
|
166
281
|
this.hasHandledTransitionEnd = true;
|
|
167
|
-
|
|
282
|
+
this.flushBuffer();
|
|
168
283
|
const frames = frameCount ?? this.config.transitionEndFrameCount;
|
|
169
|
-
logger.info(
|
|
284
|
+
logger.info(
|
|
285
|
+
"AnimationHandler",
|
|
286
|
+
`Generating ${frames} reverse transition frames to idle`
|
|
287
|
+
);
|
|
170
288
|
this.isGeneratingEndTransition = true;
|
|
171
289
|
try {
|
|
172
|
-
const transitionFrames = await this.renderer.
|
|
173
|
-
|
|
290
|
+
const transitionFrames = await this.renderer.generateTransitionFromProtobuf(
|
|
291
|
+
protobufData,
|
|
174
292
|
frames
|
|
175
293
|
);
|
|
176
|
-
logger.info(
|
|
294
|
+
logger.info(
|
|
295
|
+
"AnimationHandler",
|
|
296
|
+
`Generated ${transitionFrames.length} transition frames, reversing for playback`
|
|
297
|
+
);
|
|
177
298
|
const reversedFrames = transitionFrames.slice().reverse();
|
|
178
299
|
this.isPlayingTransition = true;
|
|
179
300
|
this.isTransitioningToIdle = true;
|
|
@@ -181,8 +302,13 @@ class AnimationHandler {
|
|
|
181
302
|
this.transitionFrameIndex = 0;
|
|
182
303
|
this.playTransitionFrame();
|
|
183
304
|
} catch (error) {
|
|
184
|
-
logger.error(
|
|
305
|
+
logger.error(
|
|
306
|
+
"AnimationHandler",
|
|
307
|
+
"Failed to generate reverse transition:",
|
|
308
|
+
error
|
|
309
|
+
);
|
|
185
310
|
this.renderer.renderFrame(void 0, true);
|
|
311
|
+
this.logRenderedFrame("idle");
|
|
186
312
|
} finally {
|
|
187
313
|
this.isGeneratingEndTransition = false;
|
|
188
314
|
}
|
|
@@ -192,7 +318,34 @@ class AnimationHandler {
|
|
|
192
318
|
* @internal
|
|
193
319
|
*/
|
|
194
320
|
startIdle() {
|
|
321
|
+
this.reportConversationStats();
|
|
322
|
+
this.isInSession = false;
|
|
323
|
+
this.hasReportedStall = false;
|
|
195
324
|
this.renderer.renderFrame(void 0, true);
|
|
325
|
+
this.logRenderedFrame("idle");
|
|
326
|
+
}
|
|
327
|
+
/**
|
|
328
|
+
* Ensure session-level timers/stats are active.
|
|
329
|
+
* @internal
|
|
330
|
+
*/
|
|
331
|
+
ensureSessionActive(frameSeq) {
|
|
332
|
+
if (this.isInSession) {
|
|
333
|
+
return;
|
|
334
|
+
}
|
|
335
|
+
this.isInSession = true;
|
|
336
|
+
this.lastFrameReceivedTime = Date.now();
|
|
337
|
+
this.hasReportedStall = false;
|
|
338
|
+
if (!this.conversationStartTime) {
|
|
339
|
+
this.conversationStartTime = Date.now();
|
|
340
|
+
}
|
|
341
|
+
this.startWatchdog();
|
|
342
|
+
this.startPlaybackStats();
|
|
343
|
+
if (frameSeq !== void 0) {
|
|
344
|
+
logger.info(
|
|
345
|
+
"AnimationHandler",
|
|
346
|
+
`Session started from animation frame seq=${frameSeq}`
|
|
347
|
+
);
|
|
348
|
+
}
|
|
196
349
|
}
|
|
197
350
|
/**
|
|
198
351
|
* Reset animation frame tracking (call on session start).
|
|
@@ -204,6 +357,8 @@ class AnimationHandler {
|
|
|
204
357
|
this.animationFrameCount = 0;
|
|
205
358
|
this.hasHandledTransitionStart = false;
|
|
206
359
|
this.hasHandledTransitionEnd = false;
|
|
360
|
+
this.resetPlaybackStats();
|
|
361
|
+
this.flushBuffer();
|
|
207
362
|
logger.info("AnimationHandler", "Frame tracking reset");
|
|
208
363
|
}
|
|
209
364
|
/**
|
|
@@ -211,7 +366,7 @@ class AnimationHandler {
|
|
|
211
366
|
* @internal
|
|
212
367
|
*/
|
|
213
368
|
isInTransition() {
|
|
214
|
-
return this.isPlayingTransition;
|
|
369
|
+
return this.isPlayingTransition || this.isGeneratingStartTransition || this.isGeneratingEndTransition;
|
|
215
370
|
}
|
|
216
371
|
/**
|
|
217
372
|
* Stop transition playback.
|
|
@@ -231,6 +386,58 @@ class AnimationHandler {
|
|
|
231
386
|
clearTimeout(this.transitionTimeoutId);
|
|
232
387
|
this.transitionTimeoutId = null;
|
|
233
388
|
}
|
|
389
|
+
this.flushBuffer();
|
|
390
|
+
}
|
|
391
|
+
/**
|
|
392
|
+
* Get cumulative session summary for telemetry reporting.
|
|
393
|
+
* @internal
|
|
394
|
+
*/
|
|
395
|
+
getSessionSummary() {
|
|
396
|
+
const avgFps = this.cumulativeFpsReadings.length > 0 ? Math.round(
|
|
397
|
+
this.cumulativeFpsReadings.reduce((a, b) => a + b, 0) / this.cumulativeFpsReadings.length
|
|
398
|
+
) : 0;
|
|
399
|
+
return {
|
|
400
|
+
totalFrames: this.cumulativeTotalFrames,
|
|
401
|
+
totalLost: this.cumulativeLost,
|
|
402
|
+
totalRecovered: this.cumulativeRecovered,
|
|
403
|
+
totalDropped: this.cumulativeDropped,
|
|
404
|
+
avgFps
|
|
405
|
+
};
|
|
406
|
+
}
|
|
407
|
+
/**
|
|
408
|
+
* Report per-conversation playback stats via telemetry.
|
|
409
|
+
* Called when a conversation ends (transition to idle).
|
|
410
|
+
* @internal
|
|
411
|
+
*/
|
|
412
|
+
reportConversationStats() {
|
|
413
|
+
if (this.conversationFrameCount === 0) return;
|
|
414
|
+
const durationMs = this.conversationStartTime ? Date.now() - this.conversationStartTime : 0;
|
|
415
|
+
const avgFps = this.conversationFpsReadings.length > 0 ? Math.round(
|
|
416
|
+
this.conversationFpsReadings.reduce((a, b) => a + b, 0) / this.conversationFpsReadings.length
|
|
417
|
+
) : 0;
|
|
418
|
+
const totalExpected = this.conversationFrameCount + this.conversationLost;
|
|
419
|
+
const lossRate = totalExpected > 0 ? Number((this.conversationLost / totalExpected * 100).toFixed(1)) : 0;
|
|
420
|
+
trackEvent("rtc_playback_stats", "info", {
|
|
421
|
+
provider: this.providerName,
|
|
422
|
+
avg_fps: avgFps,
|
|
423
|
+
frame_count: this.conversationFrameCount,
|
|
424
|
+
frames_lost: this.conversationLost,
|
|
425
|
+
frames_recovered: this.conversationRecovered,
|
|
426
|
+
frames_dropped: this.conversationDropped,
|
|
427
|
+
loss_rate: lossRate,
|
|
428
|
+
duration_ms: durationMs
|
|
429
|
+
});
|
|
430
|
+
this.cumulativeTotalFrames += this.conversationFrameCount;
|
|
431
|
+
this.cumulativeLost += this.conversationLost;
|
|
432
|
+
this.cumulativeRecovered += this.conversationRecovered;
|
|
433
|
+
this.cumulativeDropped += this.conversationDropped;
|
|
434
|
+
this.cumulativeFpsReadings.push(...this.conversationFpsReadings);
|
|
435
|
+
this.conversationFrameCount = 0;
|
|
436
|
+
this.conversationLost = 0;
|
|
437
|
+
this.conversationRecovered = 0;
|
|
438
|
+
this.conversationDropped = 0;
|
|
439
|
+
this.conversationFpsReadings = [];
|
|
440
|
+
this.conversationStartTime = 0;
|
|
234
441
|
}
|
|
235
442
|
/**
|
|
236
443
|
* Clean up resources.
|
|
@@ -238,6 +445,339 @@ class AnimationHandler {
|
|
|
238
445
|
*/
|
|
239
446
|
dispose() {
|
|
240
447
|
this.stopTransition();
|
|
448
|
+
this.stopWatchdog();
|
|
449
|
+
}
|
|
450
|
+
/**
|
|
451
|
+
* Start the watchdog timer to detect stalled data streams.
|
|
452
|
+
* @internal
|
|
453
|
+
*/
|
|
454
|
+
startWatchdog() {
|
|
455
|
+
if (this.watchdogTimer) {
|
|
456
|
+
return;
|
|
457
|
+
}
|
|
458
|
+
this.watchdogTimer = setInterval(() => {
|
|
459
|
+
if (!this.isInSession) {
|
|
460
|
+
return;
|
|
461
|
+
}
|
|
462
|
+
if (this.isPlayingTransition) {
|
|
463
|
+
return;
|
|
464
|
+
}
|
|
465
|
+
const elapsed = Date.now() - this.lastFrameReceivedTime;
|
|
466
|
+
if (elapsed > _AnimationHandler.STALL_TIMEOUT_MS && !this.hasReportedStall) {
|
|
467
|
+
logger.error(
|
|
468
|
+
"AnimationHandler",
|
|
469
|
+
`Data stream stalled: no frames received for ${elapsed}ms, falling back to idle`
|
|
470
|
+
);
|
|
471
|
+
this.hasReportedStall = true;
|
|
472
|
+
this.isStalledFallback = true;
|
|
473
|
+
this.startIdle();
|
|
474
|
+
if (this.onStreamStalledCallback) {
|
|
475
|
+
try {
|
|
476
|
+
this.onStreamStalledCallback();
|
|
477
|
+
} catch (e) {
|
|
478
|
+
logger.error(
|
|
479
|
+
"AnimationHandler",
|
|
480
|
+
"Error in onStreamStalled callback:",
|
|
481
|
+
e
|
|
482
|
+
);
|
|
483
|
+
}
|
|
484
|
+
}
|
|
485
|
+
}
|
|
486
|
+
}, 1e3);
|
|
487
|
+
}
|
|
488
|
+
/**
|
|
489
|
+
* Stop the watchdog timer.
|
|
490
|
+
* @internal
|
|
491
|
+
*/
|
|
492
|
+
stopWatchdog() {
|
|
493
|
+
if (this.watchdogTimer) {
|
|
494
|
+
clearInterval(this.watchdogTimer);
|
|
495
|
+
this.watchdogTimer = null;
|
|
496
|
+
}
|
|
497
|
+
this.hasReportedStall = false;
|
|
498
|
+
this.isStalledFallback = false;
|
|
499
|
+
this.stopPlaybackStats();
|
|
500
|
+
}
|
|
501
|
+
/**
|
|
502
|
+
* Start the playback stats reporting timer.
|
|
503
|
+
* @internal
|
|
504
|
+
*/
|
|
505
|
+
startPlaybackStats() {
|
|
506
|
+
if (this.playbackStatsTimer) {
|
|
507
|
+
return;
|
|
508
|
+
}
|
|
509
|
+
this.resetPlaybackStats();
|
|
510
|
+
this.playbackStatsTimer = setInterval(() => {
|
|
511
|
+
this.reportPlaybackStats();
|
|
512
|
+
}, 1e3);
|
|
513
|
+
}
|
|
514
|
+
/**
|
|
515
|
+
* Stop the playback stats reporting timer.
|
|
516
|
+
* @internal
|
|
517
|
+
*/
|
|
518
|
+
stopPlaybackStats() {
|
|
519
|
+
if (this.playbackStatsTimer) {
|
|
520
|
+
clearInterval(this.playbackStatsTimer);
|
|
521
|
+
this.playbackStatsTimer = null;
|
|
522
|
+
}
|
|
523
|
+
this.resetPlaybackStats();
|
|
524
|
+
}
|
|
525
|
+
/**
|
|
526
|
+
* Reset playback stats counters.
|
|
527
|
+
* @internal
|
|
528
|
+
*/
|
|
529
|
+
resetPlaybackStats() {
|
|
530
|
+
this.playbackFrameCount = 0;
|
|
531
|
+
this.playbackFrameTimestamps = [];
|
|
532
|
+
this.playbackGapCount = 0;
|
|
533
|
+
this.playbackExpectedSeq = -1;
|
|
534
|
+
}
|
|
535
|
+
/**
|
|
536
|
+
* Report playback stats (called every 1s by timer).
|
|
537
|
+
* Logs FPS, frame loss rate, and playback jitter.
|
|
538
|
+
* @internal
|
|
539
|
+
*/
|
|
540
|
+
reportPlaybackStats() {
|
|
541
|
+
if (this.isPlayingTransition) {
|
|
542
|
+
this.resetPlaybackStats();
|
|
543
|
+
return;
|
|
544
|
+
}
|
|
545
|
+
if (this.playbackFrameCount === 0) {
|
|
546
|
+
return;
|
|
547
|
+
}
|
|
548
|
+
const fps = this.playbackFrameCount;
|
|
549
|
+
const totalExpected = this.playbackFrameCount + this.playbackGapCount;
|
|
550
|
+
const lossRate = totalExpected > 0 ? this.playbackGapCount / totalExpected * 100 : 0;
|
|
551
|
+
let jitter = 0;
|
|
552
|
+
if (this.playbackFrameTimestamps.length >= 2) {
|
|
553
|
+
const intervals = [];
|
|
554
|
+
for (let i = 1; i < this.playbackFrameTimestamps.length; i++) {
|
|
555
|
+
intervals.push(
|
|
556
|
+
this.playbackFrameTimestamps[i] - this.playbackFrameTimestamps[i - 1]
|
|
557
|
+
);
|
|
558
|
+
}
|
|
559
|
+
const mean = intervals.reduce((a, b) => a + b, 0) / intervals.length;
|
|
560
|
+
const variance = intervals.reduce((sum, v) => sum + (v - mean) ** 2, 0) / intervals.length;
|
|
561
|
+
jitter = Math.sqrt(variance);
|
|
562
|
+
}
|
|
563
|
+
logger.info(
|
|
564
|
+
"AnimationHandler",
|
|
565
|
+
`Playback stats: fps=${fps}, lossRate=${lossRate.toFixed(1)}%, jitter=${jitter.toFixed(1)}ms`
|
|
566
|
+
);
|
|
567
|
+
this.conversationFrameCount += this.playbackFrameCount;
|
|
568
|
+
this.conversationLost += this.playbackGapCount;
|
|
569
|
+
this.conversationFpsReadings.push(fps);
|
|
570
|
+
this.playbackFrameCount = 0;
|
|
571
|
+
this.playbackFrameTimestamps = [];
|
|
572
|
+
this.playbackGapCount = 0;
|
|
573
|
+
}
|
|
574
|
+
// ── Jitter Buffer ──
|
|
575
|
+
/**
|
|
576
|
+
* Insert a frame into the jitter buffer.
|
|
577
|
+
* Handles dedup, enforces max size, and drives the buffer state machine.
|
|
578
|
+
* @internal
|
|
579
|
+
*/
|
|
580
|
+
bufferFrame(protobufData, seq) {
|
|
581
|
+
if (this.lastRenderedFrameSeq >= 0 && seq <= this.lastRenderedFrameSeq) {
|
|
582
|
+
logger.warn(
|
|
583
|
+
"AnimationHandler",
|
|
584
|
+
`Jitter buffer: dropping stale frame seq=${seq} (lastRendered=${this.lastRenderedFrameSeq})`
|
|
585
|
+
);
|
|
586
|
+
this.conversationDropped++;
|
|
587
|
+
return;
|
|
588
|
+
}
|
|
589
|
+
if (this.bufferNextSeq >= 0 && seq < this.bufferNextSeq) {
|
|
590
|
+
logger.warn(
|
|
591
|
+
"AnimationHandler",
|
|
592
|
+
`Jitter buffer: dropping late frame seq=${seq} (nextExpected=${this.bufferNextSeq})`
|
|
593
|
+
);
|
|
594
|
+
this.conversationDropped++;
|
|
595
|
+
return;
|
|
596
|
+
}
|
|
597
|
+
if (this.frameBuffer.has(seq)) {
|
|
598
|
+
return;
|
|
599
|
+
}
|
|
600
|
+
this.frameBuffer.set(seq, { protobufData, seq, receivedAt: performance.now() });
|
|
601
|
+
if (this.frameBuffer.size > _AnimationHandler.BUFFER_MAX_SIZE) {
|
|
602
|
+
let oldestSeq = Infinity;
|
|
603
|
+
for (const k of this.frameBuffer.keys()) {
|
|
604
|
+
if (k < oldestSeq) oldestSeq = k;
|
|
605
|
+
}
|
|
606
|
+
this.frameBuffer.delete(oldestSeq);
|
|
607
|
+
this.conversationDropped++;
|
|
608
|
+
logger.warn(
|
|
609
|
+
"AnimationHandler",
|
|
610
|
+
`Jitter buffer: overflow, dropping seq=${oldestSeq} (nextExpected=${this.bufferNextSeq})`
|
|
611
|
+
);
|
|
612
|
+
}
|
|
613
|
+
switch (this.bufferState) {
|
|
614
|
+
case "direct":
|
|
615
|
+
this.bufferState = "filling";
|
|
616
|
+
if (this.bufferNextSeq < 0) {
|
|
617
|
+
this.bufferNextSeq = seq;
|
|
618
|
+
}
|
|
619
|
+
logger.info(
|
|
620
|
+
"AnimationHandler",
|
|
621
|
+
`Jitter buffer: filling (first frame seq=${seq})`
|
|
622
|
+
);
|
|
623
|
+
if (this.frameBuffer.size >= _AnimationHandler.BUFFER_INITIAL_FILL) {
|
|
624
|
+
this.startBufferDrain();
|
|
625
|
+
}
|
|
626
|
+
break;
|
|
627
|
+
case "filling":
|
|
628
|
+
if (this.frameBuffer.size >= _AnimationHandler.BUFFER_INITIAL_FILL) {
|
|
629
|
+
this.startBufferDrain();
|
|
630
|
+
}
|
|
631
|
+
break;
|
|
632
|
+
case "starved":
|
|
633
|
+
this.startBufferDrain();
|
|
634
|
+
break;
|
|
635
|
+
}
|
|
636
|
+
}
|
|
637
|
+
/**
|
|
638
|
+
* Drop buffered frames that are now too old to ever be rendered in-order.
|
|
639
|
+
* @internal
|
|
640
|
+
*/
|
|
641
|
+
dropStaleBufferedFrames() {
|
|
642
|
+
if (this.frameBuffer.size === 0) {
|
|
643
|
+
return;
|
|
644
|
+
}
|
|
645
|
+
const minAllowedSeq = Math.max(
|
|
646
|
+
this.bufferNextSeq,
|
|
647
|
+
this.lastRenderedFrameSeq + 1
|
|
648
|
+
);
|
|
649
|
+
if (minAllowedSeq < 0) {
|
|
650
|
+
return;
|
|
651
|
+
}
|
|
652
|
+
let dropped = 0;
|
|
653
|
+
for (const seq of Array.from(this.frameBuffer.keys())) {
|
|
654
|
+
if (seq < minAllowedSeq) {
|
|
655
|
+
this.frameBuffer.delete(seq);
|
|
656
|
+
dropped++;
|
|
657
|
+
}
|
|
658
|
+
}
|
|
659
|
+
if (dropped > 0) {
|
|
660
|
+
logger.warn(
|
|
661
|
+
"AnimationHandler",
|
|
662
|
+
`Jitter buffer: dropped ${dropped} stale frame(s) older than seq=${minAllowedSeq}`
|
|
663
|
+
);
|
|
664
|
+
}
|
|
665
|
+
}
|
|
666
|
+
/**
|
|
667
|
+
* Find the lowest buffered sequence at or after minSeq.
|
|
668
|
+
* @internal
|
|
669
|
+
*/
|
|
670
|
+
findLowestBufferedSeqAtOrAfter(minSeq) {
|
|
671
|
+
let candidate = Infinity;
|
|
672
|
+
for (const seq of this.frameBuffer.keys()) {
|
|
673
|
+
if (seq >= minSeq && seq < candidate) {
|
|
674
|
+
candidate = seq;
|
|
675
|
+
}
|
|
676
|
+
}
|
|
677
|
+
return candidate === Infinity ? null : candidate;
|
|
678
|
+
}
|
|
679
|
+
/**
|
|
680
|
+
* Begin draining the buffer at 25fps.
|
|
681
|
+
* @internal
|
|
682
|
+
*/
|
|
683
|
+
startBufferDrain() {
|
|
684
|
+
this.bufferState = "draining";
|
|
685
|
+
if (this.bufferNextSeq < 0) {
|
|
686
|
+
let minSeq = Infinity;
|
|
687
|
+
for (const k of this.frameBuffer.keys()) {
|
|
688
|
+
if (k < minSeq) minSeq = k;
|
|
689
|
+
}
|
|
690
|
+
this.bufferNextSeq = minSeq;
|
|
691
|
+
}
|
|
692
|
+
logger.info(
|
|
693
|
+
"AnimationHandler",
|
|
694
|
+
`Jitter buffer: draining (${this.frameBuffer.size} frames buffered)`
|
|
695
|
+
);
|
|
696
|
+
this.bufferLastDrainTime = performance.now();
|
|
697
|
+
this.drainBufferFrame();
|
|
698
|
+
}
|
|
699
|
+
/**
|
|
700
|
+
* Drain one frame from the buffer and schedule the next drain.
|
|
701
|
+
* Handles missing frames (skip-ahead after maxBufferDelayMs) and starvation.
|
|
702
|
+
* @internal
|
|
703
|
+
*/
|
|
704
|
+
drainBufferFrame() {
|
|
705
|
+
this.bufferDrainTimer = null;
|
|
706
|
+
if (this.bufferState !== "draining") {
|
|
707
|
+
return;
|
|
708
|
+
}
|
|
709
|
+
this.dropStaleBufferedFrames();
|
|
710
|
+
const frame = this.frameBuffer.get(this.bufferNextSeq);
|
|
711
|
+
if (frame) {
|
|
712
|
+
this.renderBufferedFrame(frame);
|
|
713
|
+
this.frameBuffer.delete(this.bufferNextSeq);
|
|
714
|
+
this.bufferNextSeq++;
|
|
715
|
+
} else if (this.frameBuffer.size > 0) {
|
|
716
|
+
const nextSeq = this.findLowestBufferedSeqAtOrAfter(this.bufferNextSeq);
|
|
717
|
+
if (nextSeq === null) {
|
|
718
|
+
this.bufferState = "starved";
|
|
719
|
+
logger.warn(
|
|
720
|
+
"AnimationHandler",
|
|
721
|
+
"Jitter buffer: no in-order frames available, pausing drain"
|
|
722
|
+
);
|
|
723
|
+
return;
|
|
724
|
+
}
|
|
725
|
+
const nextFrame = this.frameBuffer.get(nextSeq);
|
|
726
|
+
const waitTime = performance.now() - nextFrame.receivedAt;
|
|
727
|
+
if (waitTime > this.config.maxBufferDelayMs) {
|
|
728
|
+
const gap = Math.max(0, nextSeq - this.bufferNextSeq);
|
|
729
|
+
this.playbackGapCount += gap;
|
|
730
|
+
logger.warn(
|
|
731
|
+
"AnimationHandler",
|
|
732
|
+
`Jitter buffer: skipping ${gap} frame(s) from seq ${this.bufferNextSeq} to ${nextSeq} after ${waitTime.toFixed(1)}ms`
|
|
733
|
+
);
|
|
734
|
+
this.renderBufferedFrame(nextFrame);
|
|
735
|
+
this.frameBuffer.delete(nextSeq);
|
|
736
|
+
this.bufferNextSeq = nextSeq + 1;
|
|
737
|
+
}
|
|
738
|
+
} else {
|
|
739
|
+
this.bufferState = "starved";
|
|
740
|
+
logger.warn("AnimationHandler", "Jitter buffer: starved, pausing drain");
|
|
741
|
+
return;
|
|
742
|
+
}
|
|
743
|
+
const now = performance.now();
|
|
744
|
+
const nextTarget = this.bufferLastDrainTime + _AnimationHandler.BUFFER_FRAME_INTERVAL_MS;
|
|
745
|
+
const delay = Math.max(0, nextTarget - now);
|
|
746
|
+
this.bufferLastDrainTime = nextTarget;
|
|
747
|
+
this.bufferDrainTimer = setTimeout(() => this.drainBufferFrame(), delay);
|
|
748
|
+
}
|
|
749
|
+
/**
|
|
750
|
+
* Render a single frame from the buffer and collect playback stats.
|
|
751
|
+
* @internal
|
|
752
|
+
*/
|
|
753
|
+
renderBufferedFrame(frame) {
|
|
754
|
+
if (this.lastRenderedFrameSeq >= 0 && frame.seq <= this.lastRenderedFrameSeq) {
|
|
755
|
+
logger.warn(
|
|
756
|
+
"AnimationHandler",
|
|
757
|
+
`Jitter buffer: refusing out-of-order render seq=${frame.seq} (lastRendered=${this.lastRenderedFrameSeq})`
|
|
758
|
+
);
|
|
759
|
+
return;
|
|
760
|
+
}
|
|
761
|
+
this.renderer.renderFromProtobuf(frame.protobufData);
|
|
762
|
+
this.lastRenderedFrameSeq = frame.seq;
|
|
763
|
+
this.renderedFrameCount++;
|
|
764
|
+
this.logRenderedFrame("buffer", frame.seq);
|
|
765
|
+
this.playbackFrameTimestamps.push(performance.now());
|
|
766
|
+
this.playbackFrameCount++;
|
|
767
|
+
}
|
|
768
|
+
/**
|
|
769
|
+
* Flush the jitter buffer, stop the drain loop, and revert to direct mode.
|
|
770
|
+
* @internal
|
|
771
|
+
*/
|
|
772
|
+
flushBuffer() {
|
|
773
|
+
this.frameBuffer.clear();
|
|
774
|
+
this.bufferState = "direct";
|
|
775
|
+
this.bufferNextSeq = -1;
|
|
776
|
+
this.bufferLastDrainTime = 0;
|
|
777
|
+
if (this.bufferDrainTimer !== null) {
|
|
778
|
+
clearTimeout(this.bufferDrainTimer);
|
|
779
|
+
this.bufferDrainTimer = null;
|
|
780
|
+
}
|
|
241
781
|
}
|
|
242
782
|
/**
|
|
243
783
|
* Play a single transition frame and schedule the next one.
|
|
@@ -256,8 +796,12 @@ class AnimationHandler {
|
|
|
256
796
|
}
|
|
257
797
|
logger.info("AnimationHandler", "Transition playback complete");
|
|
258
798
|
if (wasTransitioningToIdle) {
|
|
259
|
-
logger.info(
|
|
799
|
+
logger.info(
|
|
800
|
+
"AnimationHandler",
|
|
801
|
+
"Starting idle animation after transition"
|
|
802
|
+
);
|
|
260
803
|
this.renderer.renderFrame(void 0, true);
|
|
804
|
+
this.logRenderedFrame("idle");
|
|
261
805
|
}
|
|
262
806
|
return;
|
|
263
807
|
}
|
|
@@ -268,6 +812,7 @@ class AnimationHandler {
|
|
|
268
812
|
}
|
|
269
813
|
const frame = this.transitionFrames[this.transitionFrameIndex];
|
|
270
814
|
this.renderer.renderFrame(frame);
|
|
815
|
+
this.logRenderedFrame("transition");
|
|
271
816
|
this.transitionFrameIndex++;
|
|
272
817
|
this.transitionTimeoutId = setTimeout(() => {
|
|
273
818
|
if (this.isPlayingTransition) {
|
|
@@ -275,7 +820,26 @@ class AnimationHandler {
|
|
|
275
820
|
}
|
|
276
821
|
}, 40);
|
|
277
822
|
}
|
|
278
|
-
|
|
823
|
+
/**
|
|
824
|
+
* Emit a per-frame render log for debugging ordering issues.
|
|
825
|
+
* @internal
|
|
826
|
+
*/
|
|
827
|
+
logRenderedFrame(source, seq, isRecovered) {
|
|
828
|
+
logger.info(
|
|
829
|
+
"AnimationHandler",
|
|
830
|
+
`Rendered frame: source=${source}, seq=${seq ?? "n/a"}${isRecovered ? " [RECOVERED]" : ""}`
|
|
831
|
+
);
|
|
832
|
+
}
|
|
833
|
+
};
|
|
834
|
+
/** @internal */
|
|
835
|
+
__publicField(_AnimationHandler, "STALL_TIMEOUT_MS", 5e3);
|
|
836
|
+
/** @internal */
|
|
837
|
+
__publicField(_AnimationHandler, "BUFFER_MAX_SIZE", 4);
|
|
838
|
+
/** @internal */
|
|
839
|
+
__publicField(_AnimationHandler, "BUFFER_INITIAL_FILL", 2);
|
|
840
|
+
/** @internal */
|
|
841
|
+
__publicField(_AnimationHandler, "BUFFER_FRAME_INTERVAL_MS", 40);
|
|
842
|
+
let AnimationHandler = _AnimationHandler;
|
|
279
843
|
export {
|
|
280
844
|
AnimationHandler
|
|
281
845
|
};
|