@spatius/avatarkit 1.3.9 → 1.3.10-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/dist/assets/{OpusDecoderWorker.worker-DWI3dYQC.js → OpusDecoderWorker.worker-6yl1qL0f.js} +217 -10170
- package/dist/assets/{OpusEncoderWorker.worker-BlkaRHli.js → OpusEncoderWorker.worker-BTGeizmj.js} +217 -10170
- package/dist/core/AvatarController.d.ts +68 -2
- package/dist/core/AvatarSDK.d.ts +2 -1
- package/dist/core/AvatarView.d.ts +3 -2
- package/dist/index.js +314 -127
- package/dist/internal-telemetry.js +1 -1
- package/dist/{otel-trace-DqXSWBaw.js → otel-trace-CjdZ3saJ.js} +155 -10247
- package/dist/types/index.d.ts +17 -2
- package/package.json +2 -3
package/dist/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { $ as
|
|
1
|
+
import { $ as FrameStarvationMode, A as recordHttpClientDuration, B as idManager, C as monoTimestamp, D as hostOf, E as fetchBootstrap, F as getSdkPackage, G as ConnectionState, H as AnimationType, I as resolveSdkVersion, J as DEFAULT_OPUS_BITRATE, K as ConversationState, M as OTEL_LOGS_ENDPOINT, N as OTEL_STREAM_NAME, O as cleanupOtelMetrics, P as isDebugMode, Q as ErrorCode, R as logger, S as monoMarkFrom, T as clockSync, U as AvatarError, V as generateTraceId, W as AvatarState, X as DEFAULT_REGION_REQUEST, Y as DEFAULT_REGION, Z as DrivingServiceMode, _ as logMetric, a as startLoadTrace, at as TransitionType, b as initializeOtel, c as AudioFormat, ct as __commonJSMin, d as MessageType, et as LoadProgress, f as TransportCompression, g as logEvent, h as clientContextFields, i as startInitTrace, it as SUPPORTED_SAMPLE_RATES, j as recordMetric, k as initializeOtelMetrics, l as EgressType, lt as __toESM, m as BinaryWriter, n as initializeOtelTrace, nt as RENDER_QUALITY_PARAMS, o as startPlaybackTrace, ot as requireSDK, p as BinaryReader, q as DEFAULT_AUDIO_FORMAT, r as startConnectTrace, rt as RenderQuality, s as buildTraceparent, st as setSDKRef, t as cleanupOtelTrace, tt as LogLevel, u as Message, v as logMetricDetail, w as resolveMarks, y as cleanupOtel, z as setLogLevel } from "./otel-trace-CjdZ3saJ.js";
|
|
2
2
|
//#region core/Avatar.ts
|
|
3
3
|
var Avatar = class {
|
|
4
4
|
id;
|
|
@@ -41893,7 +41893,7 @@ var OpusDecoderProxy = class OpusDecoderProxy {
|
|
|
41893
41893
|
try {
|
|
41894
41894
|
worker = new Worker(new URL(
|
|
41895
41895
|
/* @vite-ignore */
|
|
41896
|
-
"" + new URL("assets/OpusDecoderWorker.worker-
|
|
41896
|
+
"" + new URL("assets/OpusDecoderWorker.worker-6yl1qL0f.js", import.meta.url).href,
|
|
41897
41897
|
"" + import.meta.url
|
|
41898
41898
|
), { type: "module" });
|
|
41899
41899
|
} catch (err) {
|
|
@@ -42892,9 +42892,8 @@ var AnimationPlayer = class AnimationPlayer {
|
|
|
42892
42892
|
* Called when service connection is established
|
|
42893
42893
|
* @internal
|
|
42894
42894
|
*/
|
|
42895
|
-
async createAndInitializeStreamingPlayer() {
|
|
42895
|
+
async createAndInitializeStreamingPlayer(audioFormat) {
|
|
42896
42896
|
if (this.streamingPlayer) return;
|
|
42897
|
-
const audioFormat = requireSDK().getAudioFormat();
|
|
42898
42897
|
this.streamingPlayer = new StreamingAudioPlayer({
|
|
42899
42898
|
sampleRate: audioFormat.sampleRate,
|
|
42900
42899
|
channelCount: audioFormat.channelCount,
|
|
@@ -42915,8 +42914,8 @@ var AnimationPlayer = class AnimationPlayer {
|
|
|
42915
42914
|
* Stop previous playback and update end callback
|
|
42916
42915
|
* @internal
|
|
42917
42916
|
*/
|
|
42918
|
-
async prepareStreamingPlayer(onEnded) {
|
|
42919
|
-
if (!this.streamingPlayer) await this.createAndInitializeStreamingPlayer();
|
|
42917
|
+
async prepareStreamingPlayer(onEnded, audioFormat) {
|
|
42918
|
+
if (!this.streamingPlayer) await this.createAndInitializeStreamingPlayer(audioFormat);
|
|
42920
42919
|
if (!this.streamingPlayer) throw new Error("Failed to create streaming player");
|
|
42921
42920
|
this.streamingPlayer.stop();
|
|
42922
42921
|
this.onEndedCallback = onEnded;
|
|
@@ -44445,7 +44444,7 @@ function emitInitErrorLog(params) {
|
|
|
44445
44444
|
* 这里直接 `fetch(keepalive)`,浏览器保证请求在页面卸载后仍会完成(≤64KB)。
|
|
44446
44445
|
*
|
|
44447
44446
|
* 带上常规通道会自动注入的关联键(client_id / session_id / connection_id)和客户端环境
|
|
44448
|
-
*
|
|
44447
|
+
* 字段,后台按同样的字段名查。
|
|
44449
44448
|
* @internal
|
|
44450
44449
|
*/
|
|
44451
44450
|
function emitDirectLog(event, level, contents, ctx) {
|
|
@@ -45893,17 +45892,11 @@ async function probe() {
|
|
|
45893
45892
|
//#endregion
|
|
45894
45893
|
//#region core/AvatarSDK.ts
|
|
45895
45894
|
/** PCM 输入 + Opus 上行时受支持的采样率(Opus 编码器可用的采样率)。 */
|
|
45896
|
-
var OPUS_UPLINK_SAMPLE_RATES = [
|
|
45897
|
-
8e3,
|
|
45898
|
-
16e3,
|
|
45899
|
-
24e3,
|
|
45900
|
-
48e3
|
|
45901
|
-
];
|
|
45902
45895
|
var AvatarSDK = class {
|
|
45903
45896
|
static _initializationState = "uninitialized";
|
|
45904
45897
|
static _initializingPromise = null;
|
|
45905
45898
|
static _configuration = null;
|
|
45906
|
-
static _version = "1.3.
|
|
45899
|
+
static _version = "1.3.10-beta.1";
|
|
45907
45900
|
static _avatarCore = null;
|
|
45908
45901
|
static _cachedDeviceScore = null;
|
|
45909
45902
|
static _rendererBackend = null;
|
|
@@ -45971,26 +45964,7 @@ var AvatarSDK = class {
|
|
|
45971
45964
|
...configuration,
|
|
45972
45965
|
region
|
|
45973
45966
|
};
|
|
45974
|
-
const af = this._configuration.audioFormat;
|
|
45975
|
-
if (af && af.opusUplinkEnabled !== false && af.inputAudioFormat !== "opus" && !OPUS_UPLINK_SAMPLE_RATES.includes(af.sampleRate)) {
|
|
45976
|
-
logger.warn(`[AvatarSDK] Opus uplink is on but sampleRate=${af.sampleRate} is not Opus-compatible (supported: ${OPUS_UPLINK_SAMPLE_RATES}); disabling Opus uplink, sending raw PCM.`);
|
|
45977
|
-
this._configuration = {
|
|
45978
|
-
...this._configuration,
|
|
45979
|
-
audioFormat: {
|
|
45980
|
-
...af,
|
|
45981
|
-
opusUplinkEnabled: false
|
|
45982
|
-
}
|
|
45983
|
-
};
|
|
45984
|
-
logEvent("opus_uplink_disabled_unsupported_rate", "warning", {
|
|
45985
|
-
sample_rate: af.sampleRate,
|
|
45986
|
-
supported_rates: OPUS_UPLINK_SAMPLE_RATES.join(",")
|
|
45987
|
-
});
|
|
45988
|
-
}
|
|
45989
45967
|
const regionEnd = Date.now();
|
|
45990
|
-
initializePostHog(this._version, {
|
|
45991
|
-
region,
|
|
45992
|
-
dsm
|
|
45993
|
-
});
|
|
45994
45968
|
initializeOtel(this._version, {
|
|
45995
45969
|
region,
|
|
45996
45970
|
dsm,
|
|
@@ -46039,9 +46013,6 @@ var AvatarSDK = class {
|
|
|
46039
46013
|
gpu_renderer: cap.gpuRenderer
|
|
46040
46014
|
});
|
|
46041
46015
|
}).catch(() => {});
|
|
46042
|
-
setTimeout(() => {
|
|
46043
|
-
this.flushLogsIfNeeded().catch(() => {});
|
|
46044
|
-
}, 0);
|
|
46045
46016
|
} catch (error) {
|
|
46046
46017
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
46047
46018
|
try {
|
|
@@ -46130,7 +46101,6 @@ var AvatarSDK = class {
|
|
|
46130
46101
|
*/
|
|
46131
46102
|
static setUserId(userId) {
|
|
46132
46103
|
idManager.setUserId(userId);
|
|
46133
|
-
updatePostHogPersonPropertiesForFlags();
|
|
46134
46104
|
}
|
|
46135
46105
|
/** @internal */
|
|
46136
46106
|
static get isInitialized() {
|
|
@@ -46178,21 +46148,6 @@ var AvatarSDK = class {
|
|
|
46178
46148
|
return this._configuration?.drivingServiceMode ?? DrivingServiceMode.direct;
|
|
46179
46149
|
}
|
|
46180
46150
|
/**
|
|
46181
|
-
* 获取音频格式配置(带默认值)
|
|
46182
|
-
* @internal
|
|
46183
|
-
*/
|
|
46184
|
-
static getAudioFormat() {
|
|
46185
|
-
const format = this._configuration?.audioFormat ?? {
|
|
46186
|
-
channelCount: 1,
|
|
46187
|
-
sampleRate: 16e3
|
|
46188
|
-
};
|
|
46189
|
-
if (format.inputAudioFormat === "opus" && format.sampleRate !== 48e3) return {
|
|
46190
|
-
...format,
|
|
46191
|
-
sampleRate: 48e3
|
|
46192
|
-
};
|
|
46193
|
-
return format;
|
|
46194
|
-
}
|
|
46195
|
-
/**
|
|
46196
46151
|
* 获取AvatarCore实例(仅供 SDK 内部使用)
|
|
46197
46152
|
* @internal
|
|
46198
46153
|
*/
|
|
@@ -46213,21 +46168,6 @@ var AvatarSDK = class {
|
|
|
46213
46168
|
};
|
|
46214
46169
|
}
|
|
46215
46170
|
/**
|
|
46216
|
-
* Override the audio format after init. Used by the Opus integration cases to
|
|
46217
|
-
* flip `inputAudioFormat` between 'pcm' and 'opus' without a full cleanup +
|
|
46218
|
-
* init cycle (the format is otherwise fixed at initialize() time, and the
|
|
46219
|
-
* runner initializes once for the whole suite). Pass undefined to restore the
|
|
46220
|
-
* default PCM format. Throws if SDK isn't initialized.
|
|
46221
|
-
* @internal Test-only SPI; aligned with iOS/Android `setAudioFormatForTesting`.
|
|
46222
|
-
*/
|
|
46223
|
-
static setAudioFormatForTesting(format) {
|
|
46224
|
-
if (!this._configuration) throw new Error("[AvatarSDK] setAudioFormatForTesting: SDK not initialized");
|
|
46225
|
-
this._configuration = {
|
|
46226
|
-
...this._configuration,
|
|
46227
|
-
audioFormat: format
|
|
46228
|
-
};
|
|
46229
|
-
}
|
|
46230
|
-
/**
|
|
46231
46171
|
* Count keyframes in a raw `Message` payload (as captured via
|
|
46232
46172
|
* `AvatarController.onRawAnimationData`). Useful for validating that a
|
|
46233
46173
|
* test fixture decoded correctly. Returns 0 on decode failure.
|
|
@@ -46328,7 +46268,6 @@ var AvatarSDK = class {
|
|
|
46328
46268
|
this._initializationState = "uninitialized";
|
|
46329
46269
|
this._initializingPromise = null;
|
|
46330
46270
|
idManager.clear();
|
|
46331
|
-
cleanupPostHog();
|
|
46332
46271
|
cleanupOtel();
|
|
46333
46272
|
cleanupOtelMetrics();
|
|
46334
46273
|
cleanupOtelTrace();
|
|
@@ -46355,20 +46294,6 @@ var AvatarSDK = class {
|
|
|
46355
46294
|
};
|
|
46356
46295
|
}
|
|
46357
46296
|
/**
|
|
46358
|
-
* Flush logs if feature flag is enabled (private method)
|
|
46359
|
-
* First fetches latest feature flag from server, then decides whether to upload logs
|
|
46360
|
-
* @internal
|
|
46361
|
-
*/
|
|
46362
|
-
static async flushLogsIfNeeded() {
|
|
46363
|
-
try {
|
|
46364
|
-
if (!await getLogsFeatureFlag()) return;
|
|
46365
|
-
await logSink.flush();
|
|
46366
|
-
} catch (error) {
|
|
46367
|
-
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
46368
|
-
logger.warn("[AvatarSDK] Failed to flush logs:", errorMessage);
|
|
46369
|
-
}
|
|
46370
|
-
}
|
|
46371
|
-
/**
|
|
46372
46297
|
* Device performance score (CPU + GPU).
|
|
46373
46298
|
* CPU: matrix multiply throughput in Web Worker (warmup 1s + measure 1s).
|
|
46374
46299
|
* GPU: WebGPU compute shader throughput (warmup 1s + measure 1s).
|
|
@@ -46387,7 +46312,7 @@ var AvatarSDK = class {
|
|
|
46387
46312
|
static GPU_SCORE_THRESHOLD = 100;
|
|
46388
46313
|
/**
|
|
46389
46314
|
* Check if the current device can run the avatar.
|
|
46390
|
-
* Runs a ~2s benchmark, reports device info and scores
|
|
46315
|
+
* Runs a ~2s benchmark, reports device info and scores via telemetry.
|
|
46391
46316
|
* Returns false when neither WebGL2 nor WebGPU can be created — that is deterministic
|
|
46392
46317
|
* and the benchmark is skipped. Otherwise runs the benchmark and returns true; the
|
|
46393
46318
|
* score thresholds are not yet enforced and will be calibrated from production data.
|
|
@@ -46543,7 +46468,7 @@ var OpusEncoderProxy = class OpusEncoderProxy {
|
|
|
46543
46468
|
try {
|
|
46544
46469
|
worker = new Worker(new URL(
|
|
46545
46470
|
/* @vite-ignore */
|
|
46546
|
-
"" + new URL("assets/OpusEncoderWorker.worker-
|
|
46471
|
+
"" + new URL("assets/OpusEncoderWorker.worker-BTGeizmj.js", import.meta.url).href,
|
|
46547
46472
|
"" + import.meta.url
|
|
46548
46473
|
), { type: "module" });
|
|
46549
46474
|
} catch (err) {
|
|
@@ -46823,8 +46748,47 @@ var AnimationWebSocketClient = class extends EventEmitter {
|
|
|
46823
46748
|
* for the rationale and conversation-boundary handling.
|
|
46824
46749
|
*/
|
|
46825
46750
|
pendingAudioBuffer = new PendingAudioBuffer();
|
|
46751
|
+
/**
|
|
46752
|
+
* 本轮 ws 上行写入计数:分母 = 调用了几次 ws.send(),分子 = 其中抛异常的次数。
|
|
46753
|
+
* 统计的是**写 socket 这个动作**本身的成败,不掺宿主时序、降级或服务端处理。
|
|
46754
|
+
*
|
|
46755
|
+
* 必须记在这一层:一次 send() 对 ws.send() 不是一对一。PCM 路径一对一,而 Opus 的
|
|
46756
|
+
* 编码 / 转发两条路径都按 Ogg page 拆,一次调用可能写 socket 多次,所以调用层的
|
|
46757
|
+
* send_audio_N span 数当不了分母。
|
|
46758
|
+
*/
|
|
46759
|
+
wsSendStats = {
|
|
46760
|
+
total: 0,
|
|
46761
|
+
failed: 0
|
|
46762
|
+
};
|
|
46763
|
+
/**
|
|
46764
|
+
* 已就哪一轮发过 `audioSendDropped`。一次断连会让本轮**剩下的每一页**都发不出去,
|
|
46765
|
+
* 而降级只需要通知一次:后续通知既是 no-op(controller 已在降级态),又会因为这些页
|
|
46766
|
+
* 往往属于刚结束的那一轮而撞上 cid 校验,每页刷一条 `keyframes_conversation_id_mismatch`
|
|
46767
|
+
* 告警。故按 req_id 去重,与 `tracedReqId` 的 set-once 模式一致。
|
|
46768
|
+
*/
|
|
46769
|
+
droppedNotifiedReqId = null;
|
|
46770
|
+
/** 通知本轮上行已失败(需要降级),同一轮只发一次。 */
|
|
46771
|
+
notifyAudioSendDropped(conversationId) {
|
|
46772
|
+
if (!conversationId || this.droppedNotifiedReqId === conversationId) return;
|
|
46773
|
+
this.droppedNotifiedReqId = conversationId;
|
|
46774
|
+
this.emit("audioSendDropped", conversationId);
|
|
46775
|
+
}
|
|
46776
|
+
/** 取本轮 ws 上行计数快照,供轮次级上报。 @internal */
|
|
46777
|
+
getWsSendStats() {
|
|
46778
|
+
return { ...this.wsSendStats };
|
|
46779
|
+
}
|
|
46780
|
+
/** 轮次结束、上报完毕后清零。 @internal */
|
|
46781
|
+
resetWsSendStats() {
|
|
46782
|
+
this.wsSendStats = {
|
|
46783
|
+
total: 0,
|
|
46784
|
+
failed: 0
|
|
46785
|
+
};
|
|
46786
|
+
}
|
|
46787
|
+
/** 取本 view 当前生效的音频格式(见 options 注释)。 */
|
|
46788
|
+
getAudioFormat;
|
|
46826
46789
|
constructor(options) {
|
|
46827
46790
|
super();
|
|
46791
|
+
this.getAudioFormat = options.getAudioFormat;
|
|
46828
46792
|
this.wsUrl = options.wsUrl;
|
|
46829
46793
|
this.jwtToken = options.jwtToken;
|
|
46830
46794
|
this.appId = options.appId;
|
|
@@ -46848,6 +46812,7 @@ var AnimationWebSocketClient = class extends EventEmitter {
|
|
|
46848
46812
|
this.currentCharacterId = characterId;
|
|
46849
46813
|
this.sessionConfigured = false;
|
|
46850
46814
|
this.tracedReqId = null;
|
|
46815
|
+
this.droppedNotifiedReqId = null;
|
|
46851
46816
|
await this.ensureOpusEncoder();
|
|
46852
46817
|
idManager.generateConnectionId();
|
|
46853
46818
|
const url = this.buildWebSocketUrl(characterId);
|
|
@@ -46941,9 +46906,11 @@ var AnimationWebSocketClient = class extends EventEmitter {
|
|
|
46941
46906
|
const writer = new BinaryWriter();
|
|
46942
46907
|
Message.encode(message, writer);
|
|
46943
46908
|
const bytes = writer.finish();
|
|
46909
|
+
this.wsSendStats.total++;
|
|
46944
46910
|
this.ws.send(bytes);
|
|
46945
46911
|
return true;
|
|
46946
46912
|
} catch (error) {
|
|
46913
|
+
this.wsSendStats.failed++;
|
|
46947
46914
|
const message = error instanceof Error ? error.message : String(error);
|
|
46948
46915
|
logger.error("[AnimationWebSocketClient] Failed to send audio data:", message);
|
|
46949
46916
|
logEvent("send_audio_failed", "error", {
|
|
@@ -46993,6 +46960,7 @@ var AnimationWebSocketClient = class extends EventEmitter {
|
|
|
46993
46960
|
};
|
|
46994
46961
|
const writer = new BinaryWriter();
|
|
46995
46962
|
Message.encode(message, writer);
|
|
46963
|
+
this.wsSendStats.total++;
|
|
46996
46964
|
this.ws.send(writer.finish());
|
|
46997
46965
|
};
|
|
46998
46966
|
for (let i = 0; i < pages.length; i++) sendOne(pages[i], end && i === pages.length - 1);
|
|
@@ -47003,6 +46971,7 @@ var AnimationWebSocketClient = class extends EventEmitter {
|
|
|
47003
46971
|
}
|
|
47004
46972
|
return true;
|
|
47005
46973
|
} catch (error) {
|
|
46974
|
+
this.wsSendStats.failed++;
|
|
47006
46975
|
const message = error instanceof Error ? error.message : String(error);
|
|
47007
46976
|
logger.error("[AnimationWebSocketClient] Failed to forward Opus audio:", message);
|
|
47008
46977
|
logEvent("send_audio_failed", "error", {
|
|
@@ -47025,7 +46994,7 @@ var AnimationWebSocketClient = class extends EventEmitter {
|
|
|
47025
46994
|
*/
|
|
47026
46995
|
/** 1 秒 PCM 的字节数(16-bit 单声道),用于把编码分段按音频秒数切开。 */
|
|
47027
46996
|
get audioBytesPerSecond() {
|
|
47028
|
-
return (
|
|
46997
|
+
return (this.getAudioFormat().sampleRate || 16e3) * 2;
|
|
47029
46998
|
}
|
|
47030
46999
|
/**
|
|
47031
47000
|
* 编码 span 的跨**调用**累计状态(不足 1 秒的余量留到下一次 encode 调用)。
|
|
@@ -47075,7 +47044,10 @@ var AnimationWebSocketClient = class extends EventEmitter {
|
|
|
47075
47044
|
pendingTs = ts;
|
|
47076
47045
|
});
|
|
47077
47046
|
const sendPage = (page, isLast) => {
|
|
47078
|
-
if (!this.ws || this.ws.readyState !== WebSocket.OPEN)
|
|
47047
|
+
if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
|
|
47048
|
+
this.notifyAudioSendDropped(conversationId);
|
|
47049
|
+
return;
|
|
47050
|
+
}
|
|
47079
47051
|
const message = {
|
|
47080
47052
|
type: MessageType.MESSAGE_CLIENT_AUDIO_INPUT,
|
|
47081
47053
|
clientAudioInput: {
|
|
@@ -47087,6 +47059,7 @@ var AnimationWebSocketClient = class extends EventEmitter {
|
|
|
47087
47059
|
};
|
|
47088
47060
|
const writer = new BinaryWriter();
|
|
47089
47061
|
Message.encode(message, writer);
|
|
47062
|
+
this.wsSendStats.total++;
|
|
47090
47063
|
this.ws.send(writer.finish());
|
|
47091
47064
|
};
|
|
47092
47065
|
let held = null;
|
|
@@ -47102,6 +47075,7 @@ var AnimationWebSocketClient = class extends EventEmitter {
|
|
|
47102
47075
|
if (held === null && end) held = /* @__PURE__ */ new Uint8Array(0);
|
|
47103
47076
|
if (held) sendPage(held, end);
|
|
47104
47077
|
} catch (error) {
|
|
47078
|
+
this.wsSendStats.failed++;
|
|
47105
47079
|
const message = error instanceof Error ? error.message : String(error);
|
|
47106
47080
|
logger.error("[AnimationWebSocketClient] Failed to send Opus audio:", message);
|
|
47107
47081
|
logEvent("send_audio_failed", "error", {
|
|
@@ -47110,6 +47084,7 @@ var AnimationWebSocketClient = class extends EventEmitter {
|
|
|
47110
47084
|
description: message,
|
|
47111
47085
|
format: "opus"
|
|
47112
47086
|
});
|
|
47087
|
+
this.notifyAudioSendDropped(conversationId);
|
|
47113
47088
|
this.emit("error", error);
|
|
47114
47089
|
}
|
|
47115
47090
|
}
|
|
@@ -47127,7 +47102,7 @@ var AnimationWebSocketClient = class extends EventEmitter {
|
|
|
47127
47102
|
* environment that can't do Opus).
|
|
47128
47103
|
*/
|
|
47129
47104
|
shouldUseOpusUplink() {
|
|
47130
|
-
const { opusUplinkEnabled } =
|
|
47105
|
+
const { opusUplinkEnabled } = this.getAudioFormat();
|
|
47131
47106
|
if (opusUplinkEnabled === false) return false;
|
|
47132
47107
|
return true;
|
|
47133
47108
|
}
|
|
@@ -47144,7 +47119,7 @@ var AnimationWebSocketClient = class extends EventEmitter {
|
|
|
47144
47119
|
* the connection.
|
|
47145
47120
|
*/
|
|
47146
47121
|
async ensureOpusEncoder() {
|
|
47147
|
-
const { sampleRate, opusBitrate, inputAudioFormat } =
|
|
47122
|
+
const { sampleRate, opusBitrate, inputAudioFormat } = this.getAudioFormat();
|
|
47148
47123
|
if (this.opusEncoder) {
|
|
47149
47124
|
this.opusEncoder.destroy();
|
|
47150
47125
|
this.opusEncoder = null;
|
|
@@ -47415,7 +47390,7 @@ var AnimationWebSocketClient = class extends EventEmitter {
|
|
|
47415
47390
|
return;
|
|
47416
47391
|
}
|
|
47417
47392
|
try {
|
|
47418
|
-
const audioFormatConfig =
|
|
47393
|
+
const audioFormatConfig = this.getAudioFormat();
|
|
47419
47394
|
const audioFormat = this.uplinkIsOpus ? AudioFormat.AUDIO_FORMAT_OGG_OPUS : AudioFormat.AUDIO_FORMAT_PCM_S16LE;
|
|
47420
47395
|
const bitrate = this.opusEncoder ? audioFormatConfig.opusBitrate ?? 48e3 : 0;
|
|
47421
47396
|
const message = {
|
|
@@ -47522,7 +47497,8 @@ var NetworkLayer = class {
|
|
|
47522
47497
|
wsUrl: config.driveningressWsUrl,
|
|
47523
47498
|
jwtToken: AvatarSDK.sessionToken || void 0,
|
|
47524
47499
|
appId: AvatarSDK.appId || void 0,
|
|
47525
|
-
clientId: idManager.getClientId()
|
|
47500
|
+
clientId: idManager.getClientId(),
|
|
47501
|
+
getAudioFormat: () => this.dataController.audioFormat
|
|
47526
47502
|
});
|
|
47527
47503
|
this.setupWebSocketListeners();
|
|
47528
47504
|
this.startHeartbeatCheck();
|
|
@@ -47600,9 +47576,29 @@ var NetworkLayer = class {
|
|
|
47600
47576
|
return;
|
|
47601
47577
|
}
|
|
47602
47578
|
}
|
|
47579
|
+
/** 取本轮 ws 上行写入计数快照,供 playback_quality 上报。 @internal */
|
|
47580
|
+
getWsSendStats() {
|
|
47581
|
+
return this.wsClient.getWsSendStats();
|
|
47582
|
+
}
|
|
47583
|
+
/** 轮次上报完毕后清零。 @internal */
|
|
47584
|
+
resetWsSendStats() {
|
|
47585
|
+
this.wsClient.resetWsSendStats();
|
|
47586
|
+
}
|
|
47603
47587
|
/**
|
|
47604
47588
|
* 断开连接
|
|
47605
47589
|
*/
|
|
47590
|
+
/**
|
|
47591
|
+
* 主动断连(宿主 close / setAudioFormat / view 销毁都走这里)。
|
|
47592
|
+
*
|
|
47593
|
+
* **通知宿主的责任在这里,不在调用方。** 下面先摘掉 wsClient 的全部监听器再关 socket,
|
|
47594
|
+
* 所以 onclose 不会再经 'disconnected' 监听器转成 onConnectionState——这一句只能由本方法
|
|
47595
|
+
* 自己补。曾经是每个调用方各补各的:close() 补了、setSampleRate 忘了,宿主看到的状态停在
|
|
47596
|
+
* connected 而 socket 早已不在。收拢之后调用方只管调 disconnect()。
|
|
47597
|
+
*
|
|
47598
|
+
* **每次调用都通知**,没连过也发:这是 close() 一直以来的契约(集成用例「Double close」
|
|
47599
|
+
* 锁着它:两次 close 两条 disconnected),Android 的 NetworkLayer.close() 也是无条件发。
|
|
47600
|
+
* iOS 对同状态去重是那一端 didSet 的实现细节,不作为 web 的口径。
|
|
47601
|
+
*/
|
|
47606
47602
|
disconnect() {
|
|
47607
47603
|
this.isFallbackMode = false;
|
|
47608
47604
|
this._fallbackReason = "";
|
|
@@ -47611,6 +47607,8 @@ var NetworkLayer = class {
|
|
|
47611
47607
|
this.wsClient.disconnect();
|
|
47612
47608
|
this.stopHeartbeatCheck();
|
|
47613
47609
|
idManager.clearConnectionId();
|
|
47610
|
+
this.dataController.setConnected(false);
|
|
47611
|
+
this.dataController.onConnectionState?.(ConnectionState.disconnected);
|
|
47614
47612
|
}
|
|
47615
47613
|
/**
|
|
47616
47614
|
* 设置 WebSocket 事件监听器
|
|
@@ -47638,6 +47636,11 @@ var NetworkLayer = class {
|
|
|
47638
47636
|
this.connectionStartTime = null;
|
|
47639
47637
|
this.dataController.onConnectionState?.(ConnectionState.disconnected);
|
|
47640
47638
|
});
|
|
47639
|
+
this.wsClient.on("audioSendDropped", (conversationId) => {
|
|
47640
|
+
if (!conversationId) return;
|
|
47641
|
+
this._fallbackReason = "send_failed";
|
|
47642
|
+
this.dataController.yieldKeyframes([], conversationId);
|
|
47643
|
+
});
|
|
47641
47644
|
this.wsClient.on("error", (error) => {
|
|
47642
47645
|
const message = error instanceof Error ? error.message : String(error);
|
|
47643
47646
|
logger.error("[NetworkLayer] WebSocket error:", message);
|
|
@@ -47990,22 +47993,95 @@ var FrameRateMonitor = class {
|
|
|
47990
47993
|
* Responsible for audio playback and animation rendering synchronization
|
|
47991
47994
|
* Automatically decides whether to compose network layer based on configuration
|
|
47992
47995
|
*/
|
|
47993
|
-
/**
|
|
47994
|
-
|
|
47995
|
-
|
|
47996
|
+
/** PCM 输入 + Opus 上行时,Opus 编码器支持的采样率。 */
|
|
47997
|
+
var OPUS_UPLINK_SAMPLE_RATES = [
|
|
47998
|
+
8e3,
|
|
47999
|
+
16e3,
|
|
48000
|
+
24e3,
|
|
48001
|
+
48e3
|
|
48002
|
+
];
|
|
48003
|
+
/**
|
|
48004
|
+
* 把宿主给的音频格式规范化成 SDK 内部实际生效的那一份。两条规则原先分散在
|
|
48005
|
+
* `AvatarSDK.getAudioFormat()`(每次读都算一遍)和 `initialize()`(只在 init 时校验一次),
|
|
48006
|
+
* 音频格式下放到 view 之后收在这里,构造与 setter 共用同一份判断。
|
|
48007
|
+
*
|
|
48008
|
+
* 1. **Opus 输入归一化到 48 kHz**:合规的 Ogg Opus 一律解码成 48 kHz,宿主传的 `sampleRate`
|
|
48009
|
+
* 是 PCM 输入才有的概念。不归一化的话,解码器、播放链路、以及 SDK 向服务端声明的会话
|
|
48010
|
+
* 配置会各自看到一个不真实的采样率。
|
|
48011
|
+
* 2. **Opus 上行采样率不兼容则关掉 Opus 上行**:编码器只支持 8k/16k/24k/48k,其余值会让
|
|
48012
|
+
* 编码器建不起来后静默降级成 PCM——宿主以为在用 Opus,实际发的是 PCM。这里显式关掉并
|
|
48013
|
+
* 告警,让降级可见。Opus 输入不受此限(已归一化到 48k)。
|
|
48014
|
+
*/
|
|
48015
|
+
function normalizeAudioFormat(format) {
|
|
48016
|
+
let out = format;
|
|
48017
|
+
if (out.inputAudioFormat === "opus" && out.sampleRate !== 48e3) out = {
|
|
48018
|
+
...out,
|
|
48019
|
+
sampleRate: 48e3
|
|
48020
|
+
};
|
|
48021
|
+
if (out.opusUplinkEnabled !== false && out.inputAudioFormat !== "opus" && !OPUS_UPLINK_SAMPLE_RATES.includes(out.sampleRate)) {
|
|
48022
|
+
logger.warn(`[AvatarController] Opus uplink is on but sampleRate=${out.sampleRate} is not Opus-compatible (supported: ${OPUS_UPLINK_SAMPLE_RATES}); disabling Opus uplink, sending raw PCM.`);
|
|
48023
|
+
logEvent("opus_uplink_disabled_unsupported_rate", "warning", {
|
|
48024
|
+
sample_rate: out.sampleRate,
|
|
48025
|
+
supported_rates: OPUS_UPLINK_SAMPLE_RATES.join(",")
|
|
48026
|
+
});
|
|
48027
|
+
out = {
|
|
48028
|
+
...out,
|
|
48029
|
+
opusUplinkEnabled: false
|
|
48030
|
+
};
|
|
48031
|
+
}
|
|
48032
|
+
return out;
|
|
48033
|
+
}
|
|
48034
|
+
/**
|
|
48035
|
+
* 宿主要求的格式里唯一一条「写死」的规则:Opus 输入下采样率恒为解码后的 48 kHz,宿主传的
|
|
48036
|
+
* 其它值没有意义,直接改写——不进 normalize(那是可逆的生效规则),而是改写要求本身,
|
|
48037
|
+
* 与 Android / iOS 在 `AudioFormat` 构造里做的事一致。
|
|
48038
|
+
*/
|
|
48039
|
+
function coerceRequestedAudioFormat(format) {
|
|
48040
|
+
return format.inputAudioFormat === "opus" && format.sampleRate !== 48e3 ? {
|
|
48041
|
+
...format,
|
|
48042
|
+
sampleRate: 48e3
|
|
48043
|
+
} : format;
|
|
48044
|
+
}
|
|
48045
|
+
function sameAudioFormat(a, b) {
|
|
48046
|
+
return a.sampleRate === b.sampleRate && (a.opusBitrate ?? 48e3) === (b.opusBitrate ?? 48e3) && a.opusUplinkEnabled !== false === (b.opusUplinkEnabled !== false) && (a.inputAudioFormat ?? "pcm") === (b.inputAudioFormat ?? "pcm");
|
|
48047
|
+
}
|
|
48048
|
+
/** 用户经 `send()` 喂入 SDK 的音频格式(view 的 inputAudioFormat)。view 级,可经 setAudioFormat 改。 */
|
|
48049
|
+
function resolveInputAudioFormat(fmt) {
|
|
48050
|
+
return fmt.inputAudioFormat === "opus" ? "opus" : "pcm";
|
|
47996
48051
|
}
|
|
47997
48052
|
/**
|
|
47998
48053
|
* SDK 实际发往驱动服务的线上音频格式。opus 输入原样转发为 opus;PCM 输入在 opusUplinkEnabled
|
|
47999
|
-
* 时被 SDK 编码成 opus
|
|
48000
|
-
* 纯 config 推导(session 级、init 写死)。
|
|
48054
|
+
* 时被 SDK 编码成 opus(采样率兼容性由 normalizeAudioFormat 保证,不兼容会降级为 PCM)。
|
|
48001
48055
|
*/
|
|
48002
|
-
function resolveUplinkAudioFormat() {
|
|
48003
|
-
const fmt = AvatarSDK.getAudioFormat();
|
|
48056
|
+
function resolveUplinkAudioFormat(fmt) {
|
|
48004
48057
|
return fmt.inputAudioFormat === "opus" || fmt.opusUplinkEnabled !== false ? "opus" : "pcm";
|
|
48005
48058
|
}
|
|
48006
48059
|
var AvatarController = class {
|
|
48007
48060
|
networkLayer;
|
|
48008
48061
|
playbackMode;
|
|
48062
|
+
/**
|
|
48063
|
+
* 宿主**要求**的音频格式:构造时传入的值,之后由 {@link setAudioFormat} 逐字段合并。
|
|
48064
|
+
* 与 `_audioFormat` 分开存,是为了让规范化可逆:22050 + Opus 上行会被规范化成
|
|
48065
|
+
* 「Opus 上行关」,之后把采样率改回 16000 时上行应当自动恢复——只存规范化后的值就找
|
|
48066
|
+
* 不回宿主原本要的 `opusUplinkEnabled: true` 了。唯一不可逆的一条:切到 Opus 输入时
|
|
48067
|
+
* 采样率写死 48000(与 Android / iOS 的 `AudioFormat` 构造语义一致,三端同形)。
|
|
48068
|
+
*/
|
|
48069
|
+
_requestedAudioFormat;
|
|
48070
|
+
/**
|
|
48071
|
+
* 本 view 生效的音频格式(已规范化)。**不是全局的**:同一页面里两个 view 可以跑不同
|
|
48072
|
+
* 采样率。构造时定下,之后任一字段都可经 {@link setAudioFormat} 改。
|
|
48073
|
+
*/
|
|
48074
|
+
_audioFormat;
|
|
48075
|
+
/**
|
|
48076
|
+
* 本 view 生效的音频格式(已规范化)。只读——改格式走 {@link setAudioFormat}。
|
|
48077
|
+
*
|
|
48078
|
+
* 对外可读,与 iOS `AvatarController.audioFormat` / Android `AvatarController.audioFormat`
|
|
48079
|
+
* 对齐:宿主传进去的值会被规范化(Opus 输入归一到 48 kHz、不兼容的 Opus 上行采样率会关掉
|
|
48080
|
+
* Opus 上行),读回来才能确认实际生效的是什么。
|
|
48081
|
+
*/
|
|
48082
|
+
get audioFormat() {
|
|
48083
|
+
return this._audioFormat;
|
|
48084
|
+
}
|
|
48009
48085
|
/** @internal */
|
|
48010
48086
|
avatar;
|
|
48011
48087
|
/** @internal */
|
|
@@ -48169,15 +48245,17 @@ var AvatarController = class {
|
|
|
48169
48245
|
/**
|
|
48170
48246
|
* 音频字节率(bytes/s),按 **实际配置的采样率** 动态计算,而不是写死 16kHz。
|
|
48171
48247
|
* SDK 支持 8k/16k/22.05k/24k/32k/44.1k/48k,写死会让非 16k 的时长统计整体缩放出错。
|
|
48172
|
-
* Opus 输入已被
|
|
48248
|
+
* Opus 输入已被 normalizeAudioFormat 归一化成 48000,正是解码后 PCM 的真实采样率。
|
|
48173
48249
|
*/
|
|
48174
48250
|
get audioBytesPerSecond() {
|
|
48175
|
-
const { sampleRate } =
|
|
48251
|
+
const { sampleRate } = this._audioFormat;
|
|
48176
48252
|
return sampleRate * 1 * 2;
|
|
48177
48253
|
}
|
|
48178
48254
|
constructor(avatar, options) {
|
|
48179
48255
|
this.avatar = avatar;
|
|
48180
48256
|
this.playbackMode = options?.playbackMode ?? DrivingServiceMode.direct;
|
|
48257
|
+
this._requestedAudioFormat = coerceRequestedAudioFormat(options?.audioFormat ?? DEFAULT_AUDIO_FORMAT);
|
|
48258
|
+
this._audioFormat = normalizeAudioFormat(this._requestedAudioFormat);
|
|
48181
48259
|
if (this.playbackMode === DrivingServiceMode.direct) this.networkLayer = new NetworkLayer(this);
|
|
48182
48260
|
if (typeof document !== "undefined") document.addEventListener("visibilitychange", this.handleVisibilityChange);
|
|
48183
48261
|
this.ensureInputOpusDecoder();
|
|
@@ -48371,13 +48449,18 @@ var AvatarController = class {
|
|
|
48371
48449
|
};
|
|
48372
48450
|
if (AvatarSDK.getRendererBackend()) fps.renderer_backend = AvatarSDK.getRendererBackend();
|
|
48373
48451
|
const latency = this.getPlaybackLatency();
|
|
48452
|
+
const wsSend = this.networkLayer?.getWsSendStats();
|
|
48453
|
+
const wsSendFields = wsSend && wsSend.total > 0 ? {
|
|
48454
|
+
ws_send_total: wsSend.total,
|
|
48455
|
+
ws_send_failed: wsSend.failed
|
|
48456
|
+
} : {};
|
|
48374
48457
|
logEvent("playback_quality", stall.count > 0 || audioStall.count > 0 ? "warning" : "info", {
|
|
48375
48458
|
avatar_id: this.avatar.id,
|
|
48376
48459
|
conversation_id: this.currentConversationId || void 0,
|
|
48377
48460
|
end_reason: this.playbackEndReason,
|
|
48378
48461
|
is_fallback: this.isFallbackMode,
|
|
48379
|
-
input_audio_format: resolveInputAudioFormat(),
|
|
48380
|
-
uplink_audio_format: resolveUplinkAudioFormat(),
|
|
48462
|
+
input_audio_format: resolveInputAudioFormat(this._audioFormat),
|
|
48463
|
+
uplink_audio_format: resolveUplinkAudioFormat(this._audioFormat),
|
|
48381
48464
|
audio_duration_ms: Math.round(this.receivedAudioBytes / this.audioBytesPerSecond * 1e3),
|
|
48382
48465
|
audio_ended: this.reqEnd,
|
|
48383
48466
|
animation_frame_count: this.receivedAnimationFrames,
|
|
@@ -48385,7 +48468,8 @@ var AvatarController = class {
|
|
|
48385
48468
|
fps: JSON.stringify(fps),
|
|
48386
48469
|
stall: JSON.stringify(stall),
|
|
48387
48470
|
audio_stall: JSON.stringify(audioStall),
|
|
48388
|
-
...latency ? { latency: JSON.stringify(resolveMarks(latency)) } : {}
|
|
48471
|
+
...latency ? { latency: JSON.stringify(resolveMarks(latency)) } : {},
|
|
48472
|
+
...wsSendFields
|
|
48389
48473
|
});
|
|
48390
48474
|
this.reportPlaybackMetrics(stats, stall, audioStall, latency);
|
|
48391
48475
|
const traceConversationId = this.currentConversationId || "";
|
|
@@ -48402,6 +48486,7 @@ var AvatarController = class {
|
|
|
48402
48486
|
this.isFrameStarved = false;
|
|
48403
48487
|
this.audioStallEvents = [];
|
|
48404
48488
|
this.isAudioBufferStalled = false;
|
|
48489
|
+
this.networkLayer?.resetWsSendStats();
|
|
48405
48490
|
}
|
|
48406
48491
|
/**
|
|
48407
48492
|
* 从 playback_quality 拆出数值指标,各发一个 Histogram(供后台聚合 P95/均值/趋势)。
|
|
@@ -48436,14 +48521,19 @@ var AvatarController = class {
|
|
|
48436
48521
|
recordMetric("playback_audio_stall_count", audioStall.count, labels);
|
|
48437
48522
|
recordMetric("playback_audio_stall_max_ms", audioStall.max_ms, labels);
|
|
48438
48523
|
recordMetric("playback_audio_stall_total_ms", audioStall.total_ms, labels);
|
|
48524
|
+
const wsSendMetrics = this.networkLayer?.getWsSendStats();
|
|
48525
|
+
if (wsSendMetrics && wsSendMetrics.total > 0) {
|
|
48526
|
+
recordMetric("playback_ws_send_total", wsSendMetrics.total, labels);
|
|
48527
|
+
recordMetric("playback_ws_send_failed", wsSendMetrics.failed, labels);
|
|
48528
|
+
}
|
|
48439
48529
|
if (latency) {
|
|
48440
48530
|
const resolved = resolveMarks(latency);
|
|
48441
48531
|
const tap0 = resolved["tap_0"];
|
|
48442
48532
|
if (typeof tap0 === "number") {
|
|
48443
48533
|
const latencyLabels = {
|
|
48444
48534
|
...labels,
|
|
48445
|
-
input_audio_format: resolveInputAudioFormat(),
|
|
48446
|
-
uplink_audio_format: resolveUplinkAudioFormat()
|
|
48535
|
+
input_audio_format: resolveInputAudioFormat(this._audioFormat),
|
|
48536
|
+
uplink_audio_format: resolveUplinkAudioFormat(this._audioFormat)
|
|
48447
48537
|
};
|
|
48448
48538
|
const emitDelta = (name, key) => {
|
|
48449
48539
|
const t = resolved[key];
|
|
@@ -48472,8 +48562,8 @@ var AvatarController = class {
|
|
|
48472
48562
|
dsm: this.playbackMode,
|
|
48473
48563
|
is_fallback: this.isFallbackMode,
|
|
48474
48564
|
end_reason: this.playbackEndReason,
|
|
48475
|
-
input_audio_format: resolveInputAudioFormat(),
|
|
48476
|
-
uplink_audio_format: resolveUplinkAudioFormat()
|
|
48565
|
+
input_audio_format: resolveInputAudioFormat(this._audioFormat),
|
|
48566
|
+
uplink_audio_format: resolveUplinkAudioFormat(this._audioFormat)
|
|
48477
48567
|
}, this.serverTraceparent ?? void 0);
|
|
48478
48568
|
if (resolved && typeof tap0 === "number") {
|
|
48479
48569
|
const actionMarks = {};
|
|
@@ -48639,7 +48729,7 @@ var AvatarController = class {
|
|
|
48639
48729
|
if (this.animationPlayer?.isStreamingReady()) return;
|
|
48640
48730
|
if (!this.animationPlayer) this.animationPlayer = new AnimationPlayer();
|
|
48641
48731
|
if (!this.animationPlayer.isStreamingReady()) try {
|
|
48642
|
-
await this.animationPlayer.createAndInitializeStreamingPlayer();
|
|
48732
|
+
await this.animationPlayer.createAndInitializeStreamingPlayer(this._audioFormat);
|
|
48643
48733
|
} catch (error) {
|
|
48644
48734
|
const message = error instanceof Error ? error.message : String(error);
|
|
48645
48735
|
logger.error("[AvatarController] Failed to initialize audio context:", message);
|
|
@@ -48661,7 +48751,7 @@ var AvatarController = class {
|
|
|
48661
48751
|
* into PCM16 for the local audio pipeline.
|
|
48662
48752
|
*/
|
|
48663
48753
|
async ensureInputOpusDecoder() {
|
|
48664
|
-
const { inputAudioFormat } =
|
|
48754
|
+
const { inputAudioFormat } = this._audioFormat;
|
|
48665
48755
|
if (inputAudioFormat !== "opus" || this.inputOpusDecoderProxy) return;
|
|
48666
48756
|
try {
|
|
48667
48757
|
this.inputOpusDecoderProxy = await OpusDecoderProxy.create();
|
|
@@ -48673,7 +48763,7 @@ var AvatarController = class {
|
|
|
48673
48763
|
stage: "decoder_init",
|
|
48674
48764
|
reason: message
|
|
48675
48765
|
}, { rateLimit: true });
|
|
48676
|
-
this.
|
|
48766
|
+
this.emitError(new AvatarError(`Failed to initialize Opus decoder: ${message}`, ErrorCode.playbackInitFailed));
|
|
48677
48767
|
}
|
|
48678
48768
|
}
|
|
48679
48769
|
/**
|
|
@@ -48684,7 +48774,7 @@ var AvatarController = class {
|
|
|
48684
48774
|
*/
|
|
48685
48775
|
async normalizeInputAudioToPcm(data) {
|
|
48686
48776
|
if (!this.inputOpusDecoderProxy) {
|
|
48687
|
-
if (resolveInputAudioFormat() === "opus") {
|
|
48777
|
+
if (resolveInputAudioFormat(this._audioFormat) === "opus") {
|
|
48688
48778
|
logger.error("[AvatarController] Opus input but decoder unavailable; dropping chunk to avoid playing raw Opus as PCM");
|
|
48689
48779
|
logEvent("opus_input_decode_failed", "error", { stage: "decoder_unavailable" }, { rateLimit: true });
|
|
48690
48780
|
return {
|
|
@@ -48717,6 +48807,101 @@ var AvatarController = class {
|
|
|
48717
48807
|
}
|
|
48718
48808
|
}
|
|
48719
48809
|
/**
|
|
48810
|
+
* 改本 view 的音频格式——**部分更新**:传什么改什么,没传的字段保持原状。
|
|
48811
|
+
*
|
|
48812
|
+
* 四个字段走同一条路,不区分「重」「轻」:**打断当前轮次 → 销毁音频上下文 → 写入新格式
|
|
48813
|
+
* → direct 模式断连**。之后喂入的音频按新格式解释;下次 `start()` 时编码器、输入解码器与
|
|
48814
|
+
* 会话配置都按新格式重建。
|
|
48815
|
+
*
|
|
48816
|
+
* - 不设 idle 门禁:「音频已喂进来、动画还在路上、尚未起播」这个中间态既不是 idle 也不在
|
|
48817
|
+
* 播,恰恰必须清干净;interrupt 本就是为此存在的,idle 且无音频时它是空操作。
|
|
48818
|
+
* - **只销毁音频上下文,不重建**:`AudioContext` 的采样率是构造参数、Web Audio 不允许改,
|
|
48819
|
+
* 而重建需要用户手势——那是 SDK 保证不了、只有宿主知道的前提。宿主在自己的手势回调里调
|
|
48820
|
+
* {@link initializeAudioContext} 重建;在那之前 `send()` 会经 `onError` 报
|
|
48821
|
+
* `audioContextNotInitialized` 并返回 null,不会静默无声。
|
|
48822
|
+
* - 断连而不自动重连:宿主接下来还要重建上下文,连接由它后续的 `start()` 带起。
|
|
48823
|
+
*
|
|
48824
|
+
* 规则:
|
|
48825
|
+
* - 切到 `inputAudioFormat: 'opus'` 时采样率写死 48000(Opus 解码固定值),传了别的值也
|
|
48826
|
+
* 会被改写。切回 `'pcm'` 时采样率**保持当前值**(可能是 48000)——要换就在同一次调用里
|
|
48827
|
+
* 一起传 `sampleRate`。
|
|
48828
|
+
* - 采样率不兼容 Opus 上行(不是 8k/16k/24k/48k)时 Opus 上行会被关掉;之后把采样率改回
|
|
48829
|
+
* 兼容值,上行按宿主原本的要求自动恢复。
|
|
48830
|
+
* - 传入的值与当前要求完全一致时什么都不做(不打断、不销毁、不断连)。
|
|
48831
|
+
*
|
|
48832
|
+
* **两种非法用法都只记一条 `logger.error` 后原样返回**,不抛异常也不进 `onError`:
|
|
48833
|
+
* 1. `sampleRate` 不在支持列表里(8000 / 16000 / 22050 / 24000 / 32000 / 44100 / 48000)
|
|
48834
|
+
* 2. 在 Opus 输入下传 `sampleRate`(无论是本来就是 Opus 输入,还是这一次一起切过去)
|
|
48835
|
+
*
|
|
48836
|
+
* 都是调用方把代码写错了,不是运行期可恢复的 SDK 故障。走 `onError` 并不更可靠:那是
|
|
48837
|
+
* 可选回调,宿主未必注册、注册了也未必打到控制台;为它们新增 ErrorCode 还会平白扩大
|
|
48838
|
+
* 三端公开错误码面。注释 + 日志 + 文档已经够让写代码的人当场发现。三端一致。
|
|
48839
|
+
*/
|
|
48840
|
+
setAudioFormat(patch) {
|
|
48841
|
+
const current = this._requestedAudioFormat;
|
|
48842
|
+
const merged = {
|
|
48843
|
+
...current,
|
|
48844
|
+
...patch.sampleRate !== void 0 && { sampleRate: patch.sampleRate },
|
|
48845
|
+
...patch.opusBitrate !== void 0 && { opusBitrate: patch.opusBitrate },
|
|
48846
|
+
...patch.opusUplinkEnabled !== void 0 && { opusUplinkEnabled: patch.opusUplinkEnabled },
|
|
48847
|
+
...patch.inputAudioFormat !== void 0 && { inputAudioFormat: patch.inputAudioFormat }
|
|
48848
|
+
};
|
|
48849
|
+
if (patch.sampleRate !== void 0) {
|
|
48850
|
+
if (!SUPPORTED_SAMPLE_RATES.includes(patch.sampleRate)) {
|
|
48851
|
+
logger.error(`[AvatarController] Unsupported sample rate: ${patch.sampleRate}. Supported: ${SUPPORTED_SAMPLE_RATES.join(", ")}`);
|
|
48852
|
+
return;
|
|
48853
|
+
}
|
|
48854
|
+
if (merged.inputAudioFormat === "opus") {
|
|
48855
|
+
logger.error("[AvatarController] sampleRate is not applicable with Opus input: the rate is fixed at 48000 (the Opus decode rate).");
|
|
48856
|
+
return;
|
|
48857
|
+
}
|
|
48858
|
+
}
|
|
48859
|
+
const next = coerceRequestedAudioFormat(merged);
|
|
48860
|
+
if (sameAudioFormat(next, current)) return;
|
|
48861
|
+
const before = this._audioFormat;
|
|
48862
|
+
const after = normalizeAudioFormat(next);
|
|
48863
|
+
logEvent("lifecycle_api_called", "info", {
|
|
48864
|
+
api: "setAudioFormat",
|
|
48865
|
+
avatar_id: this.avatar.id,
|
|
48866
|
+
changed_fields: [
|
|
48867
|
+
"sampleRate",
|
|
48868
|
+
"opusBitrate",
|
|
48869
|
+
"opusUplinkEnabled",
|
|
48870
|
+
"inputAudioFormat"
|
|
48871
|
+
].filter((k) => patch[k] !== void 0 && patch[k] !== current[k]).join(","),
|
|
48872
|
+
from_sample_rate: before.sampleRate,
|
|
48873
|
+
to_sample_rate: after.sampleRate,
|
|
48874
|
+
from_input_audio_format: before.inputAudioFormat ?? "pcm",
|
|
48875
|
+
to_input_audio_format: after.inputAudioFormat ?? "pcm",
|
|
48876
|
+
from_opus_bitrate: before.opusBitrate ?? 48e3,
|
|
48877
|
+
to_opus_bitrate: after.opusBitrate ?? 48e3,
|
|
48878
|
+
from_opus_uplink_enabled: before.opusUplinkEnabled !== false,
|
|
48879
|
+
to_opus_uplink_enabled: after.opusUplinkEnabled !== false,
|
|
48880
|
+
was_playing: this.isPlaying
|
|
48881
|
+
});
|
|
48882
|
+
this.interrupt();
|
|
48883
|
+
this.cleanupPlayers();
|
|
48884
|
+
this._requestedAudioFormat = next;
|
|
48885
|
+
this._audioFormat = after;
|
|
48886
|
+
this.ensureInputOpusDecoder();
|
|
48887
|
+
this.networkLayer?.disconnect();
|
|
48888
|
+
}
|
|
48889
|
+
/**
|
|
48890
|
+
* 把错误交给宿主,同时写一条 `logger.error`。
|
|
48891
|
+
*
|
|
48892
|
+
* **所有 onError 都必须走这里。** 直接调 `this.onError?.()` 的写法曾让 9 个调用点里 7 个
|
|
48893
|
+
* 不打日志:宿主如果没把回调接到 console(demo 的 log() 就只写页面面板),这些错误在
|
|
48894
|
+
* 控制台完全不可见,排查时看起来像「什么都没发生」——一次上下文未初始化的正常拒绝,
|
|
48895
|
+
* 因此被当成功能失效查了很久。
|
|
48896
|
+
*
|
|
48897
|
+
* 日志走 SDK 自己的 logger(受 logLevel 控制),宿主自己再打一条也无妨:重复一行日志的
|
|
48898
|
+
* 代价,远小于一条错误在控制台彻底消失。
|
|
48899
|
+
*/
|
|
48900
|
+
emitError(error) {
|
|
48901
|
+
logger.error(`[AvatarController] ${error.code ?? "error"}: ${error.message}`);
|
|
48902
|
+
this.onError?.(error);
|
|
48903
|
+
}
|
|
48904
|
+
/**
|
|
48720
48905
|
* Check if audio context is initialized
|
|
48721
48906
|
* @throws AvatarError if audio context is not initialized
|
|
48722
48907
|
* @internal
|
|
@@ -48749,11 +48934,11 @@ var AvatarController = class {
|
|
|
48749
48934
|
try {
|
|
48750
48935
|
this.checkAudioContextInitialized();
|
|
48751
48936
|
} catch (error) {
|
|
48752
|
-
this.
|
|
48937
|
+
this.emitError(error);
|
|
48753
48938
|
return null;
|
|
48754
48939
|
}
|
|
48755
48940
|
if (!this.networkLayer) {
|
|
48756
|
-
this.
|
|
48941
|
+
this.emitError(new AvatarError("Network layer not available", ErrorCode.networkLayerNotAvailable));
|
|
48757
48942
|
return null;
|
|
48758
48943
|
}
|
|
48759
48944
|
if (this.reqEnd && this.currentConversationId) if (this.isPlaying) this.interruptForNewRound();
|
|
@@ -48767,7 +48952,7 @@ var AvatarController = class {
|
|
|
48767
48952
|
}
|
|
48768
48953
|
this.markFirstAudioTap();
|
|
48769
48954
|
const sendStartTs = monoTimestamp();
|
|
48770
|
-
const isOpusInput = resolveInputAudioFormat() === "opus";
|
|
48955
|
+
const isOpusInput = resolveInputAudioFormat(this._audioFormat) === "opus";
|
|
48771
48956
|
let sendAudioMs = isOpusInput ? 0 : audioData.byteLength / this.audioBytesPerSecond * 1e3;
|
|
48772
48957
|
if (isOpusInput) {
|
|
48773
48958
|
const normalized = this.opusInputNormalizer.push(this.currentConversationId, new Uint8Array(audioData), end);
|
|
@@ -48780,7 +48965,7 @@ var AvatarController = class {
|
|
|
48780
48965
|
reason: normalized.rejection,
|
|
48781
48966
|
detected_shape: normalized.shape
|
|
48782
48967
|
});
|
|
48783
|
-
this.
|
|
48968
|
+
this.emitError(new AvatarError(message, ErrorCode.invalidAudioInput));
|
|
48784
48969
|
return this.currentConversationId;
|
|
48785
48970
|
}
|
|
48786
48971
|
const granule = oggMaxGranule(normalized.ogg);
|
|
@@ -48831,7 +49016,7 @@ var AvatarController = class {
|
|
|
48831
49016
|
this.markAudioTap(sendAudioMs / 1e3 * this.audioBytesPerSecond);
|
|
48832
49017
|
} else {
|
|
48833
49018
|
this.receivedAudioBytes += audioData.byteLength;
|
|
48834
|
-
if (resolveUplinkAudioFormat() === "opus") this.markFirstAudioTap();
|
|
49019
|
+
if (resolveUplinkAudioFormat(this._audioFormat) === "opus") this.markFirstAudioTap();
|
|
48835
49020
|
else this.markAudioTap(audioData.byteLength);
|
|
48836
49021
|
this.addAudioChunkToBuffer(new Uint8Array(audioData), end);
|
|
48837
49022
|
this.networkLayer.sendAudioData(this.currentConversationId, audioData, end);
|
|
@@ -48862,8 +49047,6 @@ var AvatarController = class {
|
|
|
48862
49047
|
if (this.networkLayer) this.networkLayer.disconnect();
|
|
48863
49048
|
else this.currentConversationId = null;
|
|
48864
49049
|
this.reqEnd = false;
|
|
48865
|
-
this.isConnected = false;
|
|
48866
|
-
this.onConnectionState?.(ConnectionState.disconnected);
|
|
48867
49050
|
}
|
|
48868
49051
|
/**
|
|
48869
49052
|
* Playback existing audio and animation data (host mode)
|
|
@@ -48904,7 +49087,7 @@ var AvatarController = class {
|
|
|
48904
49087
|
try {
|
|
48905
49088
|
this.checkAudioContextInitialized();
|
|
48906
49089
|
} catch (error) {
|
|
48907
|
-
this.
|
|
49090
|
+
this.emitError(error);
|
|
48908
49091
|
return null;
|
|
48909
49092
|
}
|
|
48910
49093
|
if (this.reqEnd && this.currentConversationId) {
|
|
@@ -48923,7 +49106,7 @@ var AvatarController = class {
|
|
|
48923
49106
|
this.clearPlaybackData();
|
|
48924
49107
|
}
|
|
48925
49108
|
const yieldStartTs = monoTimestamp();
|
|
48926
|
-
const isOpusInput = resolveInputAudioFormat() === "opus";
|
|
49109
|
+
const isOpusInput = resolveInputAudioFormat(this._audioFormat) === "opus";
|
|
48927
49110
|
let yieldAudioMs = isOpusInput ? 0 : data.length / this.audioBytesPerSecond * 1e3;
|
|
48928
49111
|
if (isLast) this.reqEnd = true;
|
|
48929
49112
|
if (isOpusInput) {
|
|
@@ -48937,7 +49120,7 @@ var AvatarController = class {
|
|
|
48937
49120
|
reason: normalized.rejection,
|
|
48938
49121
|
detected_shape: normalized.shape
|
|
48939
49122
|
});
|
|
48940
|
-
this.
|
|
49123
|
+
this.emitError(new AvatarError(message, ErrorCode.invalidAudioInput));
|
|
48941
49124
|
return this.currentConversationId;
|
|
48942
49125
|
}
|
|
48943
49126
|
const granule = oggMaxGranule(normalized.ogg);
|
|
@@ -49081,7 +49264,7 @@ var AvatarController = class {
|
|
|
49081
49264
|
try {
|
|
49082
49265
|
const { animation: result, avatarIdMismatch } = decodeAnimation(binaryData, this.avatar.id);
|
|
49083
49266
|
if (avatarIdMismatch) {
|
|
49084
|
-
this.
|
|
49267
|
+
this.emitError(new AvatarError(`Animation data avatar_id mismatch - expected: ${this.avatar.id}`, ErrorCode.avatarIDUnrecognized));
|
|
49085
49268
|
continue;
|
|
49086
49269
|
}
|
|
49087
49270
|
if (!result) {
|
|
@@ -49158,7 +49341,7 @@ var AvatarController = class {
|
|
|
49158
49341
|
path: "auto",
|
|
49159
49342
|
reason: error instanceof Error ? error.message : String(error)
|
|
49160
49343
|
}, { rateLimit: true });
|
|
49161
|
-
this.
|
|
49344
|
+
this.emitError(new AvatarError("Failed to start playback", ErrorCode.playbackStartFailed));
|
|
49162
49345
|
});
|
|
49163
49346
|
this.traceActions.recvs.push({
|
|
49164
49347
|
startTs: recvStartTs,
|
|
@@ -49471,7 +49654,7 @@ var AvatarController = class {
|
|
|
49471
49654
|
if (!this.animationPlayer) throw new AvatarError("Animation player not initialized", ErrorCode.animationPlayerNotInitialized);
|
|
49472
49655
|
await this.animationPlayer.prepareStreamingPlayer(() => {
|
|
49473
49656
|
this.onPlaybackEnded();
|
|
49474
|
-
});
|
|
49657
|
+
}, this._audioFormat);
|
|
49475
49658
|
this.animationPlayer.onAudioStall((stalled) => {
|
|
49476
49659
|
this.trackAudioBufferStall(stalled);
|
|
49477
49660
|
});
|
|
@@ -49502,7 +49685,7 @@ var AvatarController = class {
|
|
|
49502
49685
|
path: "streaming",
|
|
49503
49686
|
reason: message
|
|
49504
49687
|
}, { rateLimit: true });
|
|
49505
|
-
this.
|
|
49688
|
+
this.emitError(new AvatarError("Failed to start streaming playback", ErrorCode.playbackInitFailed));
|
|
49506
49689
|
this.isPlaying = false;
|
|
49507
49690
|
this.isStartingPlayback = false;
|
|
49508
49691
|
}
|
|
@@ -52232,13 +52415,17 @@ var AvatarView = class AvatarView {
|
|
|
52232
52415
|
* Creates a unified AvatarController, internally composes network layer based on configuration
|
|
52233
52416
|
* @param avatar - Avatar instance
|
|
52234
52417
|
* @param container - Canvas container element (required)
|
|
52418
|
+
* @param options - Per-view options; see {@link AvatarViewOptions}
|
|
52235
52419
|
*/
|
|
52236
|
-
constructor(avatar, container) {
|
|
52420
|
+
constructor(avatar, container, options) {
|
|
52237
52421
|
this.avatar = avatar;
|
|
52238
52422
|
this.characterId = `${avatar.id}-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
|
|
52239
52423
|
if (!AvatarSDK.configuration) throw new Error("AvatarSDK must be initialized before creating AvatarView. Call AvatarSDK.initialize() first.");
|
|
52240
52424
|
const playbackMode = AvatarSDK.getPlaybackMode();
|
|
52241
|
-
this.avatarController = new AvatarController(avatar, {
|
|
52425
|
+
this.avatarController = new AvatarController(avatar, {
|
|
52426
|
+
playbackMode,
|
|
52427
|
+
audioFormat: options?.audioFormat
|
|
52428
|
+
});
|
|
52242
52429
|
this.avatarController.setRenderCallback(() => {}, null);
|
|
52243
52430
|
this.canvas = this.createCanvas(container);
|
|
52244
52431
|
avatarViewRegistry.register(this);
|
|
@@ -53476,4 +53663,4 @@ var AvatarView = class AvatarView {
|
|
|
53476
53663
|
}
|
|
53477
53664
|
};
|
|
53478
53665
|
//#endregion
|
|
53479
|
-
export { AnimationType, Avatar, AvatarController, AvatarError, AvatarManager, AvatarSDK, AvatarState, AvatarView, ConnectionState, ConversationState, DEFAULT_OPUS_BITRATE, DEFAULT_REGION, DEFAULT_REGION_REQUEST, DrivingServiceMode, ErrorCode, FrameStarvationMode, LoadProgress, LogLevel, RENDER_QUALITY_PARAMS, RenderQuality, TransitionType };
|
|
53666
|
+
export { AnimationType, Avatar, AvatarController, AvatarError, AvatarManager, AvatarSDK, AvatarState, AvatarView, ConnectionState, ConversationState, DEFAULT_AUDIO_FORMAT, DEFAULT_OPUS_BITRATE, DEFAULT_REGION, DEFAULT_REGION_REQUEST, DrivingServiceMode, ErrorCode, FrameStarvationMode, LoadProgress, LogLevel, RENDER_QUALITY_PARAMS, RenderQuality, SUPPORTED_SAMPLE_RATES, TransitionType };
|