@spatius/avatarkit 1.3.1-beta.7 → 1.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/dist/{AvatarDownloader-B1tJTnIb.js → AvatarDownloader-BDKn8AjO.js} +98 -43
- package/dist/{AvatarSDK-CTprrfJk.js → AvatarSDK-DqP8n0ex.js} +162 -2560
- package/dist/{OpusCodec-PWv7ynPp.js → OpusCodec-B0YtpenN.js} +43 -9
- package/dist/{OpusDecoderProxy-m0fVN_M2.js → OpusDecoderProxy-vGagApTz.js} +19 -6
- package/dist/{OpusEncoderProxy-DiW21sBK.js → OpusEncoderProxy-sMAwGYqf.js} +42 -7
- package/dist/{StreamingAudioPlayer-C6hgkvbK.js → StreamingAudioPlayer-DXsCXseQ.js} +2 -2
- package/dist/assets/{AvatarDownloader-LO9uuYvm.js → AvatarDownloader-u-f7nYVa.js} +96 -41
- package/dist/assets/{AvatarSDK-CPEeTeLT.js → AvatarSDK-bSFhbb92.js} +149 -12
- package/dist/assets/{OpusDecoderWorker.worker-CSSWn75M.js → OpusDecoderWorker.worker-Cp7jmav3.js} +21 -5
- package/dist/assets/{OpusEncoderWorker.worker-q4TdCVDZ.js → OpusEncoderWorker.worker-CQct4FWr.js} +53 -11
- package/dist/assets/{logger-DCzHWd4N.js → logger-D7nCOkvy.js} +319 -126
- package/dist/core/AvatarController.d.ts +10 -0
- package/dist/{error-utils-BCZDCrc6.js → error-utils-D7aLxFrw.js} +1 -1
- package/dist/index.js +304 -44
- package/dist/internal-telemetry.d.ts +45 -20
- package/dist/internal-telemetry.js +27 -2
- package/dist/{logger-DdmfSEn-.js → logger-gpv4NaPj.js} +336 -126
- package/dist/otel-trace-M2fi3yoE.js +2611 -0
- package/dist/{pwa-cache-manager-BMKyJWXI.js → pwa-cache-manager-CcFR2ARt.js} +1 -1
- package/dist/types/index.d.ts +30 -4
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
|
-
import { a as
|
|
2
|
-
import { At as
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import { t as
|
|
1
|
+
import { a as EgressType, c as TransportCompression, i as AudioFormat, l as BinaryReader, o as Message, r as avatarViewRegistry, s as MessageType, t as AvatarSDK, u as BinaryWriter } from "./AvatarSDK-DqP8n0ex.js";
|
|
2
|
+
import { At as FrameStarvationMode, C as recordMetric, Ct as ConnectionState, Dt as DEFAULT_REGION_REQUEST, Et as DEFAULT_REGION, Ft as TransitionType, Mt as LogLevel, Nt as RENDER_QUALITY_PARAMS, Ot as DrivingServiceMode, Pt as RenderQuality, St as AvatarState, Tt as DEFAULT_OPUS_BITRATE, bt as AnimationType, c as logEvent, g as clockSync, h as resolveMarks, jt as LoadProgress, kt as ErrorCode, l as logMetric, m as monoTimestamp, p as monoMarkFrom, t as logger, vt as idManager, wt as ConversationState, xt as AvatarError } from "./logger-gpv4NaPj.js";
|
|
3
|
+
import { a as startLoadTrace, o as startPlaybackTrace, r as startConnectTrace, s as buildTraceparent } from "./otel-trace-M2fi3yoE.js";
|
|
4
|
+
import { n as APP_CONFIG } from "./error-utils-D7aLxFrw.js";
|
|
5
|
+
import { r as OggMuxer, t as OggPageSplitter } from "./OpusCodec-B0YtpenN.js";
|
|
6
|
+
import { n as setOpusEncodeFrameSink } from "./OpusEncoderProxy-sMAwGYqf.js";
|
|
7
|
+
import { t as AvatarDownloader } from "./AvatarDownloader-BDKn8AjO.js";
|
|
6
8
|
//#region core/Avatar.ts
|
|
7
9
|
var Avatar = class {
|
|
8
10
|
id;
|
|
@@ -309,8 +311,8 @@ var AnimationPlayer = class AnimationPlayer {
|
|
|
309
311
|
*/
|
|
310
312
|
async createAndInitializeStreamingPlayer() {
|
|
311
313
|
if (this.streamingPlayer) return;
|
|
312
|
-
const { StreamingAudioPlayer } = await import("./StreamingAudioPlayer-
|
|
313
|
-
const { AvatarSDK } = await import("./AvatarSDK-
|
|
314
|
+
const { StreamingAudioPlayer } = await import("./StreamingAudioPlayer-DXsCXseQ.js");
|
|
315
|
+
const { AvatarSDK } = await import("./AvatarSDK-DqP8n0ex.js").then((n) => n.n);
|
|
314
316
|
const audioFormat = AvatarSDK.getAudioFormat();
|
|
315
317
|
this.streamingPlayer = new StreamingAudioPlayer({
|
|
316
318
|
sampleRate: audioFormat.sampleRate,
|
|
@@ -930,9 +932,11 @@ var PendingAudioBuffer = class {
|
|
|
930
932
|
};
|
|
931
933
|
//#endregion
|
|
932
934
|
//#region animation/AnimationWebSocketClient.ts
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
935
|
+
var onOpusEncodeSegment = null;
|
|
936
|
+
/** @internal 由 AvatarController 在每轮播放开始时装上、结束时卸下。 */
|
|
937
|
+
function setOpusEncodeSegmentSink(fn) {
|
|
938
|
+
onOpusEncodeSegment = fn;
|
|
939
|
+
}
|
|
936
940
|
var AnimationWebSocketClient = class extends EventEmitter {
|
|
937
941
|
wsUrl;
|
|
938
942
|
reconnectAttempts;
|
|
@@ -948,6 +952,15 @@ var AnimationWebSocketClient = class extends EventEmitter {
|
|
|
948
952
|
sessionConfigured = false;
|
|
949
953
|
connectionStartTime = 0;
|
|
950
954
|
/**
|
|
955
|
+
* 连接发起时刻(`new WebSocket()` 之前)。
|
|
956
|
+
*
|
|
957
|
+
* `ws_connect_latency` 与 connect trace 都以它为起点。与 `connectionStartTime`
|
|
958
|
+
* 分开:那个是「连接建立成功」的时刻,用来算 session 时长,语义不同。
|
|
959
|
+
*/
|
|
960
|
+
connectStartMs = 0;
|
|
961
|
+
/** 握手完成(onopen)时刻,用来把连接拆成握手 / 会话确认两段。 */
|
|
962
|
+
wsOpenMs = 0;
|
|
963
|
+
/**
|
|
951
964
|
* Opus encoder proxy, created only when the SDK-decided uplink is Opus. Null
|
|
952
965
|
* means audio is sent as raw PCM. Scoped to one connection: created
|
|
953
966
|
* on connect, destroyed on disconnect. Runs the encode off the main thread via
|
|
@@ -1194,26 +1207,84 @@ var AnimationWebSocketClient = class extends EventEmitter {
|
|
|
1194
1207
|
* message so the server sees the stream close. Caller ensures socket/session
|
|
1195
1208
|
* are ready.
|
|
1196
1209
|
*/
|
|
1210
|
+
/** 1 秒 PCM 的字节数(16-bit 单声道),用于把编码分段按音频秒数切开。 */
|
|
1211
|
+
get audioBytesPerSecond() {
|
|
1212
|
+
return (AvatarSDK.getAudioFormat().sampleRate || 16e3) * 2;
|
|
1213
|
+
}
|
|
1214
|
+
/**
|
|
1215
|
+
* 编码 span 的跨**调用**累计状态(不足 1 秒的余量留到下一次 encode 调用)。
|
|
1216
|
+
*
|
|
1217
|
+
* `encodeSpanReqId` 用来判轮次切换:这几个字段是实例级的,不跟着 controller 每轮
|
|
1218
|
+
* 重置的 traceActions 走,上一轮的余量留到下一轮会让段数虚增、时刻串到几秒前
|
|
1219
|
+
* (实测 12 秒音频报出 24 段,且落在 send_audio 之前 4 秒)。
|
|
1220
|
+
*/
|
|
1221
|
+
encodeSpanReqId = null;
|
|
1222
|
+
encodeSpanStart = null;
|
|
1223
|
+
encodeSpanBytes = 0;
|
|
1224
|
+
/**
|
|
1225
|
+
* 把逐帧的编码时刻按**音频秒数**切成段,报给 trace。
|
|
1226
|
+
*
|
|
1227
|
+
* 与解码侧同一套口径:累计满 1 秒 PCM 收一段,边界用那一帧编完的真实时刻;
|
|
1228
|
+
* 尾巴不足 1 秒的余量跨调用累计(流式喂时一块往往远不足 1 秒)。
|
|
1229
|
+
*/
|
|
1230
|
+
emitEncodeSegments(reqId, fallbackStart, frames, frameTs) {
|
|
1231
|
+
if (frames.length === 0) return;
|
|
1232
|
+
if (this.encodeSpanReqId !== reqId) {
|
|
1233
|
+
this.encodeSpanReqId = reqId;
|
|
1234
|
+
this.encodeSpanStart = null;
|
|
1235
|
+
this.encodeSpanBytes = 0;
|
|
1236
|
+
}
|
|
1237
|
+
const oneSecondBytes = this.audioBytesPerSecond;
|
|
1238
|
+
for (let i = 0; i < frames.length; i++) {
|
|
1239
|
+
this.encodeSpanStart ??= fallbackStart;
|
|
1240
|
+
this.encodeSpanBytes += frames[i];
|
|
1241
|
+
if (this.encodeSpanBytes < oneSecondBytes) continue;
|
|
1242
|
+
const endTs = monoMarkFrom(frameTs[i]);
|
|
1243
|
+
onOpusEncodeSegment?.({
|
|
1244
|
+
startTs: this.encodeSpanStart,
|
|
1245
|
+
endTs,
|
|
1246
|
+
pcmBytes: this.encodeSpanBytes
|
|
1247
|
+
});
|
|
1248
|
+
this.encodeSpanStart = endTs;
|
|
1249
|
+
this.encodeSpanBytes = 0;
|
|
1250
|
+
}
|
|
1251
|
+
}
|
|
1197
1252
|
async encodeAndSendOpus(conversationId, audioData, end) {
|
|
1198
1253
|
try {
|
|
1199
|
-
const
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1254
|
+
const encodeStartTs = monoTimestamp();
|
|
1255
|
+
let pendingFrames = [];
|
|
1256
|
+
let pendingTs = [];
|
|
1257
|
+
setOpusEncodeFrameSink((frames, ts) => {
|
|
1258
|
+
pendingFrames = frames;
|
|
1259
|
+
pendingTs = ts;
|
|
1260
|
+
});
|
|
1261
|
+
const sendPage = (page, isLast) => {
|
|
1262
|
+
if (!this.ws || this.ws.readyState !== WebSocket.OPEN) return;
|
|
1203
1263
|
const message = {
|
|
1204
1264
|
type: MessageType.MESSAGE_CLIENT_AUDIO_INPUT,
|
|
1205
1265
|
clientAudioInput: {
|
|
1206
1266
|
reqId: conversationId,
|
|
1207
|
-
audio:
|
|
1267
|
+
audio: page,
|
|
1208
1268
|
end: isLast,
|
|
1209
1269
|
traceContext: this.takeTraceContext(conversationId)
|
|
1210
1270
|
}
|
|
1211
1271
|
};
|
|
1212
1272
|
const writer = new BinaryWriter();
|
|
1213
1273
|
Message.encode(message, writer);
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1274
|
+
this.ws.send(writer.finish());
|
|
1275
|
+
};
|
|
1276
|
+
let held = null;
|
|
1277
|
+
const pushPage = (page) => {
|
|
1278
|
+
if (held) sendPage(held, false);
|
|
1279
|
+
held = page;
|
|
1280
|
+
};
|
|
1281
|
+
await this.opusEncoder.encode(conversationId, audioData, end, (batch) => {
|
|
1282
|
+
for (const page of batch) pushPage(page);
|
|
1283
|
+
});
|
|
1284
|
+
setOpusEncodeFrameSink(null);
|
|
1285
|
+
this.emitEncodeSegments(conversationId, encodeStartTs, pendingFrames, pendingTs);
|
|
1286
|
+
if (held === null && end) held = /* @__PURE__ */ new Uint8Array(0);
|
|
1287
|
+
if (held) sendPage(held, end);
|
|
1217
1288
|
} catch (error) {
|
|
1218
1289
|
const message = error instanceof Error ? error.message : String(error);
|
|
1219
1290
|
logger.error("[AnimationWebSocketClient] Failed to send Opus audio:", message);
|
|
@@ -1229,8 +1300,8 @@ var AnimationWebSocketClient = class extends EventEmitter {
|
|
|
1229
1300
|
* Internal decision for the SDK's upstream audio format when the host feeds PCM:
|
|
1230
1301
|
* whether to encode PCM→Opus (true) or send raw PCM (false).
|
|
1231
1302
|
*
|
|
1232
|
-
* Opus uplink is opt-
|
|
1233
|
-
*
|
|
1303
|
+
* Opus uplink is opt-OUT: it is on unless the host passes
|
|
1304
|
+
* `opusUplinkEnabled: false`, which keeps the raw PCM uplink.
|
|
1234
1305
|
*
|
|
1235
1306
|
* Once opted in, the decision is still the SDK's: the backend (v2) supports Opus
|
|
1236
1307
|
* unconditionally and the encoder has a two-tier fallback (worker → main-thread
|
|
@@ -1240,7 +1311,7 @@ var AnimationWebSocketClient = class extends EventEmitter {
|
|
|
1240
1311
|
*/
|
|
1241
1312
|
shouldUseOpusUplink() {
|
|
1242
1313
|
const { opusUplinkEnabled } = AvatarSDK.getAudioFormat();
|
|
1243
|
-
if (opusUplinkEnabled
|
|
1314
|
+
if (opusUplinkEnabled === false) return false;
|
|
1244
1315
|
return true;
|
|
1245
1316
|
}
|
|
1246
1317
|
/**
|
|
@@ -1281,7 +1352,7 @@ var AnimationWebSocketClient = class extends EventEmitter {
|
|
|
1281
1352
|
return;
|
|
1282
1353
|
}
|
|
1283
1354
|
try {
|
|
1284
|
-
const { OpusEncoderProxy } = await import("./OpusEncoderProxy-
|
|
1355
|
+
const { OpusEncoderProxy } = await import("./OpusEncoderProxy-sMAwGYqf.js").then((n) => n.t);
|
|
1285
1356
|
const bitrate = opusBitrate ?? 48e3;
|
|
1286
1357
|
this.opusEncoder = await OpusEncoderProxy.create(sampleRate, bitrate);
|
|
1287
1358
|
this.uplinkIsOpus = true;
|
|
@@ -1337,19 +1408,39 @@ var AnimationWebSocketClient = class extends EventEmitter {
|
|
|
1337
1408
|
if (connectionId) url.searchParams.set("connection_id", connectionId);
|
|
1338
1409
|
return url.toString();
|
|
1339
1410
|
}
|
|
1411
|
+
/**
|
|
1412
|
+
* 发一条建连 trace:`connect` 下挂 `ws_handshake` / `session_confirm` 两段。
|
|
1413
|
+
*
|
|
1414
|
+
* 在会话确认时一次性回填(两段的时刻都已知),而不是实时开 span——建连全程只有
|
|
1415
|
+
* 两个边界点,回填式更简单,也与 init / load trace 的做法一致。
|
|
1416
|
+
*/
|
|
1417
|
+
emitConnectTrace(confirmMs) {
|
|
1418
|
+
const start = this.connectStartMs;
|
|
1419
|
+
const open = this.wsOpenMs;
|
|
1420
|
+
if (start <= 0 || open <= 0) return;
|
|
1421
|
+
const t = startConnectTrace(start, {
|
|
1422
|
+
connection_id: idManager.getConnectionId() ?? "",
|
|
1423
|
+
region: AvatarSDK.configuration?.region ?? ""
|
|
1424
|
+
});
|
|
1425
|
+
t.span("ws_handshake", start, open, { dur_ms: open - start });
|
|
1426
|
+
t.span("session_confirm", open, confirmMs, { dur_ms: confirmMs - open });
|
|
1427
|
+
t.end(true, confirmMs, { dur_ms: confirmMs - start });
|
|
1428
|
+
this.wsOpenMs = 0;
|
|
1429
|
+
}
|
|
1340
1430
|
connectWebSocket(url) {
|
|
1341
1431
|
return new Promise((resolve, reject) => {
|
|
1342
1432
|
try {
|
|
1343
1433
|
const urlForLog = this.sanitizeUrlForLog(url);
|
|
1344
1434
|
logger.log(`[AnimationWebSocketClient] Connecting to WebSocket: ${urlForLog}`);
|
|
1345
|
-
|
|
1435
|
+
this.connectStartMs = Date.now();
|
|
1436
|
+
this.wsOpenMs = 0;
|
|
1346
1437
|
this.ws = new WebSocket(url);
|
|
1347
1438
|
this.ws.onopen = () => {
|
|
1348
1439
|
this.isConnecting = false;
|
|
1349
1440
|
this.currentRetryCount = 0;
|
|
1350
1441
|
this.isManuallyDisconnected = false;
|
|
1351
1442
|
this.connectionStartTime = Date.now();
|
|
1352
|
-
|
|
1443
|
+
this.wsOpenMs = this.connectionStartTime;
|
|
1353
1444
|
logger.log(`[AnimationWebSocketClient] WebSocket connected successfully`);
|
|
1354
1445
|
this.configureSession();
|
|
1355
1446
|
this.emit("connected");
|
|
@@ -1524,6 +1615,12 @@ var AnimationWebSocketClient = class extends EventEmitter {
|
|
|
1524
1615
|
idManager.setConnectionId(connectionId);
|
|
1525
1616
|
logger.log(`[AnimationWebSocketClient] Session confirmed, connection_id updated: ${connectionId}`);
|
|
1526
1617
|
} else logger.log("[AnimationWebSocketClient] Session confirmed by server");
|
|
1618
|
+
if (this.connectStartMs > 0) {
|
|
1619
|
+
const confirmMs = Date.now();
|
|
1620
|
+
logMetric("ws_connect_latency", confirmMs - this.connectStartMs, {}, { connection_id: idManager.getConnectionId() || "" });
|
|
1621
|
+
this.emitConnectTrace(confirmMs);
|
|
1622
|
+
this.connectStartMs = 0;
|
|
1623
|
+
}
|
|
1527
1624
|
this.flushPendingAudio();
|
|
1528
1625
|
this.emit("sessionConfirmed", connectionId);
|
|
1529
1626
|
return;
|
|
@@ -2056,7 +2153,7 @@ function resolveInputAudioFormat() {
|
|
|
2056
2153
|
*/
|
|
2057
2154
|
function resolveUplinkAudioFormat() {
|
|
2058
2155
|
const fmt = AvatarSDK.getAudioFormat();
|
|
2059
|
-
return fmt.inputAudioFormat === "opus" || fmt.opusUplinkEnabled
|
|
2156
|
+
return fmt.inputAudioFormat === "opus" || fmt.opusUplinkEnabled !== false ? "opus" : "pcm";
|
|
2060
2157
|
}
|
|
2061
2158
|
var AvatarController = class {
|
|
2062
2159
|
networkLayer;
|
|
@@ -2185,9 +2282,24 @@ var AvatarController = class {
|
|
|
2185
2282
|
*/
|
|
2186
2283
|
traceActions = {
|
|
2187
2284
|
sends: [],
|
|
2188
|
-
recvs: []
|
|
2285
|
+
recvs: [],
|
|
2286
|
+
decodes: [],
|
|
2287
|
+
encodes: []
|
|
2288
|
+
};
|
|
2289
|
+
/** 上行编码分段的接收钩子,构造时装上、dispose 时卸下(见 setOpusEncodeSegmentSink)。 */
|
|
2290
|
+
onEncodeSegment = (seg) => {
|
|
2291
|
+
this.traceActions.encodes.push(seg);
|
|
2292
|
+
this.markAudioTapAt(seg.pcmBytes, seg.endTs);
|
|
2189
2293
|
};
|
|
2190
2294
|
/**
|
|
2295
|
+
* 解码 span 的聚合状态:累计满 1 秒 PCM 收一段。
|
|
2296
|
+
*
|
|
2297
|
+
* 跨 chunk 累计——宿主流式喂时一块可能远不足 1 秒,按块收段会碎成几百个 span。
|
|
2298
|
+
* `decodeSpanStart` 是上一段的收尾时刻(null = 用本次 decode 的发起时刻当起点)。
|
|
2299
|
+
*/
|
|
2300
|
+
decodeSpanStart = null;
|
|
2301
|
+
decodeSpanBytes = 0;
|
|
2302
|
+
/**
|
|
2191
2303
|
* Highest Ogg granule forwarded so far this round, for Opus input. The granule
|
|
2192
2304
|
* accumulates across a stream, so a send's audio seconds is its own end granule
|
|
2193
2305
|
* minus this. Reset per conversation. -1 = nothing forwarded yet.
|
|
@@ -2208,6 +2320,7 @@ var AvatarController = class {
|
|
|
2208
2320
|
if (this.playbackMode === DrivingServiceMode.direct) this.networkLayer = new NetworkLayer(this);
|
|
2209
2321
|
if (typeof document !== "undefined") document.addEventListener("visibilitychange", this.handleVisibilityChange);
|
|
2210
2322
|
this.ensureInputOpusDecoder();
|
|
2323
|
+
setOpusEncodeSegmentSink(this.onEncodeSegment);
|
|
2211
2324
|
}
|
|
2212
2325
|
handleVisibilityChange = () => {
|
|
2213
2326
|
if (document.visibilityState === "hidden") this.shouldReportPlaybackStats = false;
|
|
@@ -2457,9 +2570,14 @@ var AvatarController = class {
|
|
|
2457
2570
|
const resolved = resolveMarks(latency);
|
|
2458
2571
|
const tap0 = resolved["tap_0"];
|
|
2459
2572
|
if (typeof tap0 === "number") {
|
|
2573
|
+
const latencyLabels = {
|
|
2574
|
+
...labels,
|
|
2575
|
+
input_audio_format: resolveInputAudioFormat(),
|
|
2576
|
+
uplink_audio_format: resolveUplinkAudioFormat()
|
|
2577
|
+
};
|
|
2460
2578
|
const emitDelta = (name, key) => {
|
|
2461
2579
|
const t = resolved[key];
|
|
2462
|
-
if (typeof t === "number") recordMetric(name, t - tap0,
|
|
2580
|
+
if (typeof t === "number") recordMetric(name, t - tap0, latencyLabels);
|
|
2463
2581
|
};
|
|
2464
2582
|
emitDelta("playback_latency_tap_1_ms", "tap_1");
|
|
2465
2583
|
emitDelta("playback_latency_tap_2_ms", "tap_2");
|
|
@@ -2497,6 +2615,14 @@ var AvatarController = class {
|
|
|
2497
2615
|
actionMarks[`recv_start_${i}`] = r.startTs;
|
|
2498
2616
|
actionMarks[`recv_end_${i}`] = r.endTs;
|
|
2499
2617
|
});
|
|
2618
|
+
this.traceActions.decodes.forEach((d, i) => {
|
|
2619
|
+
actionMarks[`decode_start_${i}`] = d.startTs;
|
|
2620
|
+
actionMarks[`decode_end_${i}`] = d.endTs;
|
|
2621
|
+
});
|
|
2622
|
+
this.traceActions.encodes.forEach((e, i) => {
|
|
2623
|
+
actionMarks[`encode_start_${i}`] = e.startTs;
|
|
2624
|
+
actionMarks[`encode_end_${i}`] = e.endTs;
|
|
2625
|
+
});
|
|
2500
2626
|
const rm = resolveMarks(actionMarks);
|
|
2501
2627
|
const allEnds = [...this.traceActions.sends.map((_, i) => rm[`send_end_${i}`]), ...this.traceActions.recvs.map((_, i) => rm[`recv_end_${i}`])].filter((v) => typeof v === "number");
|
|
2502
2628
|
const allStarts = [...this.traceActions.sends.map((_, i) => rm[`send_start_${i}`]), ...this.traceActions.recvs.map((_, i) => rm[`recv_start_${i}`])].filter((v) => typeof v === "number");
|
|
@@ -2524,6 +2650,44 @@ var AvatarController = class {
|
|
|
2524
2650
|
});
|
|
2525
2651
|
});
|
|
2526
2652
|
arrival.end(arrivalEnd);
|
|
2653
|
+
const decodeMarks = this.traceActions.decodes;
|
|
2654
|
+
if (decodeMarks.length > 0) {
|
|
2655
|
+
const firstStart = rm[`decode_start_0`];
|
|
2656
|
+
if (typeof firstStart === "number") {
|
|
2657
|
+
const decodeGroup = t.group("decode", firstStart, { view: "decode" });
|
|
2658
|
+
let decodeEnd = firstStart;
|
|
2659
|
+
decodeMarks.forEach((d, i) => {
|
|
2660
|
+
const start = rm[`decode_start_${i}`];
|
|
2661
|
+
const end = rm[`decode_end_${i}`];
|
|
2662
|
+
if (typeof start !== "number" || typeof end !== "number") return;
|
|
2663
|
+
decodeEnd = Math.max(decodeEnd, end);
|
|
2664
|
+
decodeGroup.span(`decode_audio_${i + 1}`, start, floorEnd(start, end), {
|
|
2665
|
+
dur_ms: Math.round(end - start),
|
|
2666
|
+
audio_ms: Math.round(d.pcmBytes / this.audioBytesPerSecond * 1e3)
|
|
2667
|
+
});
|
|
2668
|
+
});
|
|
2669
|
+
decodeGroup.end(decodeEnd);
|
|
2670
|
+
}
|
|
2671
|
+
}
|
|
2672
|
+
const encodeMarks = this.traceActions.encodes;
|
|
2673
|
+
if (encodeMarks.length > 0) {
|
|
2674
|
+
const firstStart = rm[`encode_start_0`];
|
|
2675
|
+
if (typeof firstStart === "number") {
|
|
2676
|
+
const encodeGroup = t.group("encode", firstStart, { view: "encode" });
|
|
2677
|
+
let encodeEnd = firstStart;
|
|
2678
|
+
encodeMarks.forEach((e, i) => {
|
|
2679
|
+
const start = rm[`encode_start_${i}`];
|
|
2680
|
+
const end = rm[`encode_end_${i}`];
|
|
2681
|
+
if (typeof start !== "number" || typeof end !== "number") return;
|
|
2682
|
+
encodeEnd = Math.max(encodeEnd, end);
|
|
2683
|
+
encodeGroup.span(`encode_audio_${i + 1}`, start, floorEnd(start, end), {
|
|
2684
|
+
dur_ms: Math.round(end - start),
|
|
2685
|
+
audio_ms: Math.round(e.pcmBytes / this.audioBytesPerSecond * 1e3)
|
|
2686
|
+
});
|
|
2687
|
+
});
|
|
2688
|
+
encodeGroup.end(encodeEnd);
|
|
2689
|
+
}
|
|
2690
|
+
}
|
|
2527
2691
|
const renderEnd = this.playbackEndedAt > tap0 ? this.playbackEndedAt : tap0 + durationMs;
|
|
2528
2692
|
const emitPlaybackChain = (into, playLabel, stallLabel, chainStart, chainEnd, events) => {
|
|
2529
2693
|
const stalls = events.filter((e) => e.startTs >= chainStart && e.startTs <= chainEnd).sort((a, b) => a.startTs - b.startTs);
|
|
@@ -2630,7 +2794,7 @@ var AvatarController = class {
|
|
|
2630
2794
|
const { inputAudioFormat } = AvatarSDK.getAudioFormat();
|
|
2631
2795
|
if (inputAudioFormat !== "opus" || this.inputOpusDecoderProxy) return;
|
|
2632
2796
|
try {
|
|
2633
|
-
const { OpusDecoderProxy } = await import("./OpusDecoderProxy-
|
|
2797
|
+
const { OpusDecoderProxy } = await import("./OpusDecoderProxy-vGagApTz.js");
|
|
2634
2798
|
this.inputOpusDecoderProxy = await OpusDecoderProxy.create();
|
|
2635
2799
|
this.inputDecodeChain = Promise.resolve();
|
|
2636
2800
|
} catch (error) {
|
|
@@ -2649,15 +2813,28 @@ var AvatarController = class {
|
|
|
2649
2813
|
if (!this.inputOpusDecoderProxy) {
|
|
2650
2814
|
if (resolveInputAudioFormat() === "opus") {
|
|
2651
2815
|
logger.error("[AvatarController] Opus input but decoder unavailable; dropping chunk to avoid playing raw Opus as PCM");
|
|
2652
|
-
return
|
|
2816
|
+
return {
|
|
2817
|
+
pcm: /* @__PURE__ */ new Uint8Array(0),
|
|
2818
|
+
segments: [],
|
|
2819
|
+
segmentTs: []
|
|
2820
|
+
};
|
|
2653
2821
|
}
|
|
2654
|
-
|
|
2822
|
+
const raw = new Uint8Array(data);
|
|
2823
|
+
return {
|
|
2824
|
+
pcm: raw,
|
|
2825
|
+
segments: raw.length > 0 ? [raw.length] : [],
|
|
2826
|
+
segmentTs: raw.length > 0 ? [performance.now()] : []
|
|
2827
|
+
};
|
|
2655
2828
|
}
|
|
2656
2829
|
try {
|
|
2657
2830
|
return await this.inputOpusDecoderProxy.decode(data);
|
|
2658
2831
|
} catch (error) {
|
|
2659
2832
|
logger.error("[AvatarController] Failed to decode input Opus audio:", error instanceof Error ? error.message : String(error));
|
|
2660
|
-
return
|
|
2833
|
+
return {
|
|
2834
|
+
pcm: /* @__PURE__ */ new Uint8Array(0),
|
|
2835
|
+
segments: [],
|
|
2836
|
+
segmentTs: []
|
|
2837
|
+
};
|
|
2661
2838
|
}
|
|
2662
2839
|
}
|
|
2663
2840
|
/**
|
|
@@ -2705,6 +2882,7 @@ var AvatarController = class {
|
|
|
2705
2882
|
this.currentConversationId = this.generateAndLogNewConversationId();
|
|
2706
2883
|
this.reqEnd = false;
|
|
2707
2884
|
}
|
|
2885
|
+
this.markFirstAudioTap();
|
|
2708
2886
|
const sendStartTs = monoTimestamp();
|
|
2709
2887
|
const isOpusInput = resolveInputAudioFormat() === "opus";
|
|
2710
2888
|
let sendAudioMs = isOpusInput ? 0 : audioData.byteLength / this.audioBytesPerSecond * 1e3;
|
|
@@ -2732,17 +2910,41 @@ var AvatarController = class {
|
|
|
2732
2910
|
const oggEnd = end;
|
|
2733
2911
|
this.inputDecodeChain = this.inputDecodeChain.then(async () => {
|
|
2734
2912
|
await this.ensureInputOpusDecoder();
|
|
2735
|
-
const
|
|
2913
|
+
const decodeStartTs = monoTimestamp();
|
|
2914
|
+
const { pcm, segments, segmentTs } = await this.normalizeInputAudioToPcm(oggForDecode);
|
|
2736
2915
|
this.receivedAudioBytes += pcm.length;
|
|
2737
|
-
|
|
2916
|
+
for (let i = 0; i < segments.length; i++) {
|
|
2917
|
+
this.decodeSpanBytes += segments[i];
|
|
2918
|
+
if (this.decodeSpanBytes < this.audioBytesPerSecond) continue;
|
|
2919
|
+
const endTs = monoMarkFrom(segmentTs[i]);
|
|
2920
|
+
this.traceActions.decodes.push({
|
|
2921
|
+
startTs: this.decodeSpanStart ?? decodeStartTs,
|
|
2922
|
+
endTs,
|
|
2923
|
+
pcmBytes: this.decodeSpanBytes
|
|
2924
|
+
});
|
|
2925
|
+
this.decodeSpanStart = endTs;
|
|
2926
|
+
this.decodeSpanBytes = 0;
|
|
2927
|
+
}
|
|
2928
|
+
if (this.decodeSpanBytes > 0) {
|
|
2929
|
+
const endTs = monoMarkFrom(segmentTs[segments.length - 1]);
|
|
2930
|
+
this.traceActions.decodes.push({
|
|
2931
|
+
startTs: this.decodeSpanStart ?? decodeStartTs,
|
|
2932
|
+
endTs,
|
|
2933
|
+
pcmBytes: this.decodeSpanBytes
|
|
2934
|
+
});
|
|
2935
|
+
this.decodeSpanStart = endTs;
|
|
2936
|
+
this.decodeSpanBytes = 0;
|
|
2937
|
+
}
|
|
2738
2938
|
this.addAudioChunkToBuffer(pcm, oggEnd);
|
|
2739
2939
|
}).catch((error) => {
|
|
2740
2940
|
logger.error("[AvatarController] Opus decode→buffer chain error:", error instanceof Error ? error.message : String(error));
|
|
2741
2941
|
});
|
|
2742
2942
|
this.networkLayer.sendAudioData(this.currentConversationId, audioData, end);
|
|
2943
|
+
this.markAudioTap(sendAudioMs / 1e3 * this.audioBytesPerSecond);
|
|
2743
2944
|
} else {
|
|
2744
2945
|
this.receivedAudioBytes += audioData.byteLength;
|
|
2745
|
-
this.
|
|
2946
|
+
if (resolveUplinkAudioFormat() === "opus") this.markFirstAudioTap();
|
|
2947
|
+
else this.markAudioTap(audioData.byteLength);
|
|
2746
2948
|
this.addAudioChunkToBuffer(new Uint8Array(audioData), end);
|
|
2747
2949
|
this.networkLayer.sendAudioData(this.currentConversationId, audioData, end);
|
|
2748
2950
|
}
|
|
@@ -2854,7 +3056,7 @@ var AvatarController = class {
|
|
|
2854
3056
|
const oggLast = isLast;
|
|
2855
3057
|
this.inputDecodeChain = this.inputDecodeChain.then(async () => {
|
|
2856
3058
|
await this.ensureInputOpusDecoder();
|
|
2857
|
-
const pcm = await this.normalizeInputAudioToPcm(ogg);
|
|
3059
|
+
const { pcm } = await this.normalizeInputAudioToPcm(ogg);
|
|
2858
3060
|
this.bufferHostPcm(pcm, oggLast);
|
|
2859
3061
|
}).catch((error) => {
|
|
2860
3062
|
logger.error("[AvatarController] Opus decode→buffer chain error:", error instanceof Error ? error.message : String(error));
|
|
@@ -2868,24 +3070,50 @@ var AvatarController = class {
|
|
|
2868
3070
|
return this.currentConversationId;
|
|
2869
3071
|
}
|
|
2870
3072
|
/**
|
|
3073
|
+
* 记下 tap_0 —— 宿主送入首包音频的时刻,只抢零点、不参与分秒累计。
|
|
3074
|
+
*
|
|
3075
|
+
* 语义是「用户把第一块音频交给 SDK 的那一刻」,PCM 与 Opus 一视同仁:它是端到端
|
|
3076
|
+
* 延迟的零点,上行发送、本地解码都发生在它之后,本就该被算进后续各段 Δ 里。
|
|
3077
|
+
*
|
|
3078
|
+
* 分秒(tap_N≥1)在各路径的「发出去」那一刻推进,见 markAudioTap / markAudioTapAt
|
|
3079
|
+
* 的调用点。字节口径恒为 PCM(opus 输入由 granule 秒数折算),因为分母
|
|
3080
|
+
* audioBytesPerSecond 按真实采样率算,直接用 Ogg 压缩字节会让 tap_N 几乎不递增。
|
|
3081
|
+
* taps.set(0) 是 set-once,故这里先占、后面那次不会覆盖。
|
|
3082
|
+
* @internal
|
|
3083
|
+
*/
|
|
3084
|
+
markFirstAudioTap() {
|
|
3085
|
+
const marks = this.latencyMarks;
|
|
3086
|
+
if (!marks.taps.has(0)) marks.taps.set(0, monoTimestamp());
|
|
3087
|
+
}
|
|
3088
|
+
/**
|
|
2871
3089
|
* 记一次音频 tap:tap_0=首包送入,之后每累计满整秒记一个 tap_N。
|
|
2872
3090
|
*
|
|
2873
3091
|
* **两种模式都走这里**(direct 由 send() 调、backend 由 bufferHostPcm 调),
|
|
2874
3092
|
* 所以只有这一份分秒逻辑。
|
|
2875
3093
|
*
|
|
2876
|
-
* @param pcmByteLength
|
|
2877
|
-
*
|
|
2878
|
-
*
|
|
3094
|
+
* @param pcmByteLength 本块的 **PCM 口径**字节数。分母 audioBytesPerSecond 按真实
|
|
3095
|
+
* 采样率算,直接传 Opus 压缩字节(比 PCM 小 5-10 倍)会让 tap_N 几乎不递增;
|
|
3096
|
+
* opus 输入不解码,故由 Ogg granule 的秒数折算成等价 PCM 字节再传进来。
|
|
2879
3097
|
* @internal
|
|
2880
3098
|
*/
|
|
2881
3099
|
markAudioTap(pcmByteLength) {
|
|
3100
|
+
this.markAudioTapAt(pcmByteLength, monoTimestamp());
|
|
3101
|
+
}
|
|
3102
|
+
/**
|
|
3103
|
+
* markAudioTap 的指定时刻版:这一秒音频「就绪」的时刻由调用方给,而不是取当下。
|
|
3104
|
+
*
|
|
3105
|
+
* 上行 Opus 的编码分段用它——那一秒的就绪时刻是它编完的时刻(回调触发时已过去
|
|
3106
|
+
* 一小会儿),取 monoTimestamp() 会把编码后的排队/回调延迟算进 tap。
|
|
3107
|
+
* @internal
|
|
3108
|
+
*/
|
|
3109
|
+
markAudioTapAt(pcmByteLength, at) {
|
|
2882
3110
|
const marks = this.latencyMarks;
|
|
2883
|
-
if (!marks.taps.has(0)) marks.taps.set(0,
|
|
3111
|
+
if (!marks.taps.has(0)) marks.taps.set(0, at);
|
|
2884
3112
|
marks.accumulatedBytes += pcmByteLength;
|
|
2885
3113
|
const currentDuration = marks.accumulatedBytes / this.audioBytesPerSecond;
|
|
2886
3114
|
let s = (marks.taps.size > 0 ? Math.max(...marks.taps.keys()) : 0) + 1;
|
|
2887
3115
|
while (currentDuration >= s) {
|
|
2888
|
-
marks.taps.set(s,
|
|
3116
|
+
marks.taps.set(s, at);
|
|
2889
3117
|
s++;
|
|
2890
3118
|
}
|
|
2891
3119
|
}
|
|
@@ -3149,8 +3377,12 @@ var AvatarController = class {
|
|
|
3149
3377
|
};
|
|
3150
3378
|
this.traceActions = {
|
|
3151
3379
|
sends: [],
|
|
3152
|
-
recvs: []
|
|
3380
|
+
recvs: [],
|
|
3381
|
+
decodes: [],
|
|
3382
|
+
encodes: []
|
|
3153
3383
|
};
|
|
3384
|
+
this.decodeSpanStart = null;
|
|
3385
|
+
this.decodeSpanBytes = 0;
|
|
3154
3386
|
this.lastOpusGranule = -1;
|
|
3155
3387
|
}
|
|
3156
3388
|
/**
|
|
@@ -3410,7 +3642,7 @@ var AvatarController = class {
|
|
|
3410
3642
|
const stateChanged = isOutOfBounds !== this.lastOutOfBoundsState;
|
|
3411
3643
|
const timeSinceLastLog = now - this.lastSyncLogTime;
|
|
3412
3644
|
if (stateChanged || timeSinceLastLog >= 1e3) {
|
|
3413
|
-
logger.warn(`[
|
|
3645
|
+
logger.warn(`[AvatarKit] Animation is running behind the audio: playback has reached ${(frameIndex / 25).toFixed(1)}s but only ${((this.currentKeyframes.length + this.keyframesOffset) / 25).toFixed(1)}s of animation has arrived, so the last available frame is being held. Send audio to the SDK faster than real time — at least 2s of audio per second of wall clock, ideally more — so animation frames can stay ahead of playback.`);
|
|
3414
3646
|
this.lastSyncLogTime = now;
|
|
3415
3647
|
}
|
|
3416
3648
|
this.lastOutOfBoundsState = isOutOfBounds;
|
|
@@ -3614,6 +3846,7 @@ var AvatarController = class {
|
|
|
3614
3846
|
this.animationPlayer.dispose();
|
|
3615
3847
|
this.animationPlayer = null;
|
|
3616
3848
|
}
|
|
3849
|
+
setOpusEncodeSegmentSink(null);
|
|
3617
3850
|
if (this.inputOpusDecoderProxy) {
|
|
3618
3851
|
this.inputOpusDecoderProxy.destroy();
|
|
3619
3852
|
this.inputOpusDecoderProxy = null;
|
|
@@ -3799,6 +4032,7 @@ var AvatarManager = class AvatarManager {
|
|
|
3799
4032
|
if (task.status === "cancelled") return;
|
|
3800
4033
|
task.status = "failed";
|
|
3801
4034
|
const err = error instanceof Error ? error : new Error(String(error));
|
|
4035
|
+
task._loadTrace?.end(false, Date.now(), { error: err.message });
|
|
3802
4036
|
this.notifyAllRequests(task, {
|
|
3803
4037
|
type: LoadProgress.failed,
|
|
3804
4038
|
error: err
|
|
@@ -3819,10 +4053,19 @@ var AvatarManager = class AvatarManager {
|
|
|
3819
4053
|
const { id, abortController } = task;
|
|
3820
4054
|
const signal = abortController.signal;
|
|
3821
4055
|
const startTime = Date.now();
|
|
4056
|
+
const loadTrace = startLoadTrace(id, startTime, {
|
|
4057
|
+
use_compressed_model: task.useCompressedModel,
|
|
4058
|
+
source: "network"
|
|
4059
|
+
});
|
|
4060
|
+
task._loadTrace = loadTrace;
|
|
3822
4061
|
if (signal.aborted) throw new Error("Task cancelled");
|
|
3823
4062
|
task.status = "fetching-meta";
|
|
3824
4063
|
logger.log(`[AvatarManager] Fetching metadata for avatar ${id}...`);
|
|
4064
|
+
const metaStart = Date.now();
|
|
4065
|
+
loadTrace.span("prepare", startTime, metaStart);
|
|
3825
4066
|
const characterMeta = await this.avatarDownloader.getCharacterById(id, { signal });
|
|
4067
|
+
const metaEnd = Date.now();
|
|
4068
|
+
loadTrace.span("fetch_metadata", metaStart, metaEnd);
|
|
3826
4069
|
task.characterMeta = characterMeta;
|
|
3827
4070
|
if (signal.aborted) throw new Error("Task cancelled");
|
|
3828
4071
|
const cached = this.avatarCache.get(id);
|
|
@@ -3839,11 +4082,19 @@ var AvatarManager = class AvatarManager {
|
|
|
3839
4082
|
type: LoadProgress.downloading,
|
|
3840
4083
|
progress: 1
|
|
3841
4084
|
});
|
|
4085
|
+
const hitEnd = Date.now();
|
|
4086
|
+
loadTrace.span("prepare_assets", metaEnd, hitEnd, { cache_hit: true });
|
|
4087
|
+
loadTrace.end(true, hitEnd, { cache_hit: true });
|
|
4088
|
+
logMetric("fetch_avatar_latency", Date.now() - startTime, {
|
|
4089
|
+
resolution: "default",
|
|
4090
|
+
use_compressed_model: task.useCompressedModel,
|
|
4091
|
+
cache_hit: true
|
|
4092
|
+
}, { avatar_id: id });
|
|
3842
4093
|
return cached;
|
|
3843
4094
|
} else {
|
|
3844
4095
|
logger.log(`[AvatarManager] Avatar ${id} cache invalid: updatedAt=${cachedUpdatedAt}->${newUpdatedAt}, modelType=${cachedModelType}->${requestedModelType}, reloading...`);
|
|
3845
4096
|
this.avatarCache.delete(id);
|
|
3846
|
-
const { PwaCacheManager } = await import("./pwa-cache-manager-
|
|
4097
|
+
const { PwaCacheManager } = await import("./pwa-cache-manager-CcFR2ARt.js").then((n) => n.n);
|
|
3847
4098
|
await PwaCacheManager.clearCharacterCache(id);
|
|
3848
4099
|
}
|
|
3849
4100
|
}
|
|
@@ -3855,6 +4106,8 @@ var AvatarManager = class AvatarManager {
|
|
|
3855
4106
|
});
|
|
3856
4107
|
if (signal.aborted) throw new Error("Task cancelled");
|
|
3857
4108
|
task.status = "downloading";
|
|
4109
|
+
const assetsStart = Date.now();
|
|
4110
|
+
loadTrace.span("prepare_assets", metaEnd, assetsStart);
|
|
3858
4111
|
logger.log("[AvatarManager] Downloading resources...");
|
|
3859
4112
|
let downloadedCount = 0;
|
|
3860
4113
|
const resources = await this.avatarDownloader.preloadResources(characterMeta, {
|
|
@@ -3874,11 +4127,18 @@ var AvatarManager = class AvatarManager {
|
|
|
3874
4127
|
if (signal.aborted) throw new Error("Task cancelled");
|
|
3875
4128
|
logger.log("[AvatarManager] Creating Avatar instance...");
|
|
3876
4129
|
const avatar = new Avatar(id, characterMeta, resources, requestedModelType, false);
|
|
4130
|
+
const loadEnd = Date.now();
|
|
4131
|
+
loadTrace.span("load_assets", assetsStart, loadEnd, {
|
|
4132
|
+
cache_hit: resources.cacheHit,
|
|
4133
|
+
cache_type: resources.cacheType
|
|
4134
|
+
});
|
|
4135
|
+
loadTrace.end(true, loadEnd, { cache_hit: resources.cacheHit });
|
|
3877
4136
|
this.avatarCache.set(id, avatar);
|
|
3878
4137
|
logger.log("[AvatarManager] Avatar loaded successfully");
|
|
3879
4138
|
logMetric("fetch_avatar_latency", Date.now() - startTime, {
|
|
3880
4139
|
resolution: "default",
|
|
3881
|
-
use_compressed_model: task.useCompressedModel
|
|
4140
|
+
use_compressed_model: task.useCompressedModel,
|
|
4141
|
+
cache_hit: resources.cacheHit
|
|
3882
4142
|
}, { avatar_id: id });
|
|
3883
4143
|
return avatar;
|
|
3884
4144
|
}
|