@spatialwalk/avatarkit 1.0.0-beta.7 → 1.0.0-beta.70
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +595 -10
- package/README.md +475 -312
- package/dist/StreamingAudioPlayer-Bi2685bX.js +633 -0
- package/dist/animation/AnimationWebSocketClient.d.ts +18 -7
- package/dist/animation/utils/eventEmitter.d.ts +0 -1
- package/dist/animation/utils/flameConverter.d.ts +0 -1
- package/dist/audio/AnimationPlayer.d.ts +19 -1
- package/dist/audio/StreamingAudioPlayer.d.ts +41 -9
- package/dist/avatar_core_wasm-Dv943JJl.js +2696 -0
- package/dist/{avatar_core_wasm.wasm → avatar_core_wasm-e68766db.wasm} +0 -0
- package/dist/config/app-config.d.ts +3 -4
- package/dist/config/constants.d.ts +10 -18
- package/dist/config/sdk-config-loader.d.ts +4 -10
- package/dist/core/Avatar.d.ts +2 -14
- package/dist/core/AvatarController.d.ts +95 -85
- package/dist/core/AvatarDownloader.d.ts +7 -92
- package/dist/core/AvatarManager.d.ts +22 -12
- package/dist/core/AvatarSDK.d.ts +35 -0
- package/dist/core/AvatarView.d.ts +55 -140
- package/dist/core/NetworkLayer.d.ts +7 -59
- package/dist/generated/common/v1/models.d.ts +36 -0
- package/dist/generated/driveningress/v1/driveningress.d.ts +0 -1
- package/dist/generated/driveningress/v2/driveningress.d.ts +82 -1
- package/dist/generated/google/protobuf/struct.d.ts +0 -1
- package/dist/generated/google/protobuf/timestamp.d.ts +0 -1
- package/dist/index-CvW_c7G-.js +16434 -0
- package/dist/index.d.ts +2 -4
- package/dist/index.js +17 -18
- package/dist/renderer/RenderSystem.d.ts +9 -79
- package/dist/renderer/covariance.d.ts +3 -11
- package/dist/renderer/renderer.d.ts +6 -2
- package/dist/renderer/sortSplats.d.ts +3 -10
- package/dist/renderer/webgl/reorderData.d.ts +4 -11
- package/dist/renderer/webgl/webglRenderer.d.ts +34 -4
- package/dist/renderer/webgpu/webgpuRenderer.d.ts +30 -5
- package/dist/types/character-settings.d.ts +1 -1
- package/dist/types/character.d.ts +3 -15
- package/dist/types/index.d.ts +123 -43
- package/dist/utils/animation-interpolation.d.ts +4 -15
- package/dist/utils/client-id.d.ts +6 -0
- package/dist/utils/conversationId.d.ts +10 -0
- package/dist/utils/error-utils.d.ts +0 -1
- package/dist/utils/id-manager.d.ts +34 -0
- package/dist/utils/logger.d.ts +2 -11
- package/dist/utils/posthog-tracker.d.ts +8 -0
- package/dist/utils/pwa-cache-manager.d.ts +17 -0
- package/dist/utils/usage-tracker.d.ts +6 -0
- package/dist/vanilla/vite.config.d.ts +2 -0
- package/dist/vite.d.ts +19 -0
- package/dist/wasm/avatarCoreAdapter.d.ts +15 -126
- package/dist/wasm/avatarCoreMemory.d.ts +5 -2
- package/package.json +19 -8
- package/vite.d.ts +20 -0
- package/vite.js +126 -0
- package/dist/StreamingAudioPlayer-D7s8q5h0.js +0 -319
- package/dist/StreamingAudioPlayer-D7s8q5h0.js.map +0 -1
- package/dist/animation/AnimationWebSocketClient.d.ts.map +0 -1
- package/dist/animation/utils/eventEmitter.d.ts.map +0 -1
- package/dist/animation/utils/flameConverter.d.ts.map +0 -1
- package/dist/audio/AnimationPlayer.d.ts.map +0 -1
- package/dist/audio/StreamingAudioPlayer.d.ts.map +0 -1
- package/dist/avatar_core_wasm-D4eEi7Eh.js +0 -1666
- package/dist/avatar_core_wasm-D4eEi7Eh.js.map +0 -1
- package/dist/config/app-config.d.ts.map +0 -1
- package/dist/config/constants.d.ts.map +0 -1
- package/dist/config/sdk-config-loader.d.ts.map +0 -1
- package/dist/core/Avatar.d.ts.map +0 -1
- package/dist/core/AvatarController.d.ts.map +0 -1
- package/dist/core/AvatarDownloader.d.ts.map +0 -1
- package/dist/core/AvatarKit.d.ts +0 -66
- package/dist/core/AvatarKit.d.ts.map +0 -1
- package/dist/core/AvatarManager.d.ts.map +0 -1
- package/dist/core/AvatarView.d.ts.map +0 -1
- package/dist/core/NetworkLayer.d.ts.map +0 -1
- package/dist/generated/driveningress/v1/driveningress.d.ts.map +0 -1
- package/dist/generated/driveningress/v2/driveningress.d.ts.map +0 -1
- package/dist/generated/google/protobuf/struct.d.ts.map +0 -1
- package/dist/generated/google/protobuf/timestamp.d.ts.map +0 -1
- package/dist/index-CpSvWi6A.js +0 -6026
- package/dist/index-CpSvWi6A.js.map +0 -1
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/renderer/RenderSystem.d.ts.map +0 -1
- package/dist/renderer/covariance.d.ts.map +0 -1
- package/dist/renderer/renderer.d.ts.map +0 -1
- package/dist/renderer/sortSplats.d.ts.map +0 -1
- package/dist/renderer/webgl/reorderData.d.ts.map +0 -1
- package/dist/renderer/webgl/webglRenderer.d.ts.map +0 -1
- package/dist/renderer/webgpu/webgpuRenderer.d.ts.map +0 -1
- package/dist/types/character-settings.d.ts.map +0 -1
- package/dist/types/character.d.ts.map +0 -1
- package/dist/types/index.d.ts.map +0 -1
- package/dist/utils/animation-interpolation.d.ts.map +0 -1
- package/dist/utils/cls-tracker.d.ts +0 -17
- package/dist/utils/cls-tracker.d.ts.map +0 -1
- package/dist/utils/error-utils.d.ts.map +0 -1
- package/dist/utils/logger.d.ts.map +0 -1
- package/dist/utils/reqId.d.ts +0 -20
- package/dist/utils/reqId.d.ts.map +0 -1
- package/dist/wasm/avatarCoreAdapter.d.ts.map +0 -1
- package/dist/wasm/avatarCoreMemory.d.ts.map +0 -1
|
@@ -0,0 +1,633 @@
|
|
|
1
|
+
var __defProp = Object.defineProperty;
|
|
2
|
+
var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
|
|
3
|
+
var __publicField = (obj, key, value) => __defNormalProp(obj, typeof key !== "symbol" ? key + "" : key, value);
|
|
4
|
+
import { A as APP_CONFIG, l as logger, e as errorToMessage, a as logEvent } from "./index-CvW_c7G-.js";
|
|
5
|
+
class StreamingAudioPlayer {
|
|
6
|
+
// 标记是否正在恢复 AudioContext,避免并发恢复请求
|
|
7
|
+
constructor(options) {
|
|
8
|
+
// AudioContext is managed internally
|
|
9
|
+
__publicField(this, "audioContext", null);
|
|
10
|
+
__publicField(this, "sampleRate");
|
|
11
|
+
__publicField(this, "channelCount");
|
|
12
|
+
__publicField(this, "debug");
|
|
13
|
+
// Session-level state
|
|
14
|
+
__publicField(this, "sessionId");
|
|
15
|
+
__publicField(this, "sessionStartTime", 0);
|
|
16
|
+
// AudioContext time when session started
|
|
17
|
+
__publicField(this, "pausedTimeOffset", 0);
|
|
18
|
+
// Accumulated paused time
|
|
19
|
+
__publicField(this, "pausedAt", 0);
|
|
20
|
+
// Time when paused
|
|
21
|
+
__publicField(this, "pausedAudioContextTime", 0);
|
|
22
|
+
// audioContext.currentTime when paused (for resume calculation)
|
|
23
|
+
__publicField(this, "scheduledTime", 0);
|
|
24
|
+
// Next chunk schedule time in AudioContext time
|
|
25
|
+
// Playback state
|
|
26
|
+
__publicField(this, "isPlaying", false);
|
|
27
|
+
__publicField(this, "isPaused", false);
|
|
28
|
+
__publicField(this, "autoStartEnabled", true);
|
|
29
|
+
// Control whether to auto-start when buffer is ready
|
|
30
|
+
__publicField(this, "autoContinue", false);
|
|
31
|
+
// 标记是否应该自动继续播放(当 end=false 且无数据时自动暂停后使用)
|
|
32
|
+
// Audio buffer queue
|
|
33
|
+
__publicField(this, "audioChunks", []);
|
|
34
|
+
__publicField(this, "scheduledChunks", 0);
|
|
35
|
+
// Number of chunks already scheduled
|
|
36
|
+
__publicField(this, "activeSources", /* @__PURE__ */ new Set());
|
|
37
|
+
__publicField(this, "lastScheduledChunkEndTime", 0);
|
|
38
|
+
// 最后一个已调度 chunk 的结束时间(相对时间)
|
|
39
|
+
__publicField(this, "lastGetCurrentTimeLog", 0);
|
|
40
|
+
// 上次记录 getCurrentTime 日志的时间戳(用于节流)
|
|
41
|
+
// 跟踪每个已调度的 chunk 的开始时间(绝对时间)和持续时间,用于准确计算当前播放时间
|
|
42
|
+
__publicField(this, "scheduledChunkInfo", []);
|
|
43
|
+
// Volume control
|
|
44
|
+
__publicField(this, "gainNode", null);
|
|
45
|
+
__publicField(this, "volume", 1);
|
|
46
|
+
// Default volume 1.0 (0.0 - 1.0)
|
|
47
|
+
// Event callbacks
|
|
48
|
+
__publicField(this, "onEndedCallback");
|
|
49
|
+
// AudioContext state management
|
|
50
|
+
__publicField(this, "stateChangeHandler");
|
|
51
|
+
__publicField(this, "isResuming", false);
|
|
52
|
+
this.sessionId = `session_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
|
|
53
|
+
this.sampleRate = (options == null ? void 0 : options.sampleRate) ?? APP_CONFIG.audio.sampleRate;
|
|
54
|
+
this.channelCount = (options == null ? void 0 : options.channelCount) ?? 1;
|
|
55
|
+
this.debug = (options == null ? void 0 : options.debug) ?? false;
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Initialize audio context (create and ensure it's ready)
|
|
59
|
+
*/
|
|
60
|
+
async initialize() {
|
|
61
|
+
if (this.audioContext) {
|
|
62
|
+
return;
|
|
63
|
+
}
|
|
64
|
+
try {
|
|
65
|
+
this.audioContext = new AudioContext({
|
|
66
|
+
sampleRate: this.sampleRate
|
|
67
|
+
});
|
|
68
|
+
this.gainNode = this.audioContext.createGain();
|
|
69
|
+
this.gainNode.gain.value = this.volume;
|
|
70
|
+
this.gainNode.connect(this.audioContext.destination);
|
|
71
|
+
if (this.audioContext.state === "suspended") {
|
|
72
|
+
await this.audioContext.resume();
|
|
73
|
+
}
|
|
74
|
+
this.stateChangeHandler = (event) => {
|
|
75
|
+
const context = event.target;
|
|
76
|
+
if (context.state === "suspended" && this.isPlaying && !this.isPaused) {
|
|
77
|
+
this.ensureAudioContextRunning().catch((err) => {
|
|
78
|
+
logger.errorWithError("[StreamingAudioPlayer] Failed to auto-resume AudioContext after external suspend:", err);
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
};
|
|
82
|
+
this.audioContext.addEventListener("statechange", this.stateChangeHandler);
|
|
83
|
+
this.log("AudioContext initialized", {
|
|
84
|
+
sessionId: this.sessionId,
|
|
85
|
+
sampleRate: this.audioContext.sampleRate,
|
|
86
|
+
state: this.audioContext.state
|
|
87
|
+
});
|
|
88
|
+
} catch (error) {
|
|
89
|
+
const message = errorToMessage(error);
|
|
90
|
+
logEvent("activeAudioSessionFailed", "warning", {
|
|
91
|
+
sessionId: this.sessionId,
|
|
92
|
+
reason: message
|
|
93
|
+
});
|
|
94
|
+
logger.error("Failed to initialize AudioContext:", message);
|
|
95
|
+
throw error instanceof Error ? error : new Error(message);
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* 确保 AudioContext 正在运行(如果被暂停则自动恢复)
|
|
100
|
+
* 只在正在播放且未暂停时自动恢复,避免干扰正常的暂停/恢复逻辑
|
|
101
|
+
*
|
|
102
|
+
* 优化:
|
|
103
|
+
* - 快速路径:如果已经是 running 状态,直接返回
|
|
104
|
+
* - 避免并发恢复:使用 isResuming 标志防止重复恢复请求
|
|
105
|
+
* - 处理 closed 状态:如果 AudioContext 已关闭,无法恢复
|
|
106
|
+
*/
|
|
107
|
+
async ensureAudioContextRunning() {
|
|
108
|
+
if (!this.audioContext) {
|
|
109
|
+
return;
|
|
110
|
+
}
|
|
111
|
+
const state = this.audioContext.state;
|
|
112
|
+
if (state === "running") {
|
|
113
|
+
return;
|
|
114
|
+
}
|
|
115
|
+
if (state === "closed") {
|
|
116
|
+
this.log("AudioContext is closed, cannot resume", {
|
|
117
|
+
sessionId: this.sessionId,
|
|
118
|
+
state
|
|
119
|
+
});
|
|
120
|
+
return;
|
|
121
|
+
}
|
|
122
|
+
if (state === "suspended" && this.isPlaying && !this.isPaused) {
|
|
123
|
+
if (this.isResuming) {
|
|
124
|
+
this.log("AudioContext resume already in progress, skipping duplicate request", {
|
|
125
|
+
sessionId: this.sessionId,
|
|
126
|
+
state
|
|
127
|
+
});
|
|
128
|
+
return;
|
|
129
|
+
}
|
|
130
|
+
this.isResuming = true;
|
|
131
|
+
try {
|
|
132
|
+
this.log("AudioContext is suspended during playback, resuming...", {
|
|
133
|
+
sessionId: this.sessionId,
|
|
134
|
+
state,
|
|
135
|
+
isPlaying: this.isPlaying,
|
|
136
|
+
isPaused: this.isPaused
|
|
137
|
+
});
|
|
138
|
+
await this.audioContext.resume();
|
|
139
|
+
this.log("AudioContext resumed successfully", {
|
|
140
|
+
sessionId: this.sessionId,
|
|
141
|
+
state: this.audioContext.state
|
|
142
|
+
});
|
|
143
|
+
} catch (err) {
|
|
144
|
+
logger.errorWithError("[StreamingAudioPlayer] Failed to resume AudioContext:", err);
|
|
145
|
+
logEvent("character_player", "error", {
|
|
146
|
+
sessionId: this.sessionId,
|
|
147
|
+
event: "audio_context_resume_failed",
|
|
148
|
+
reason: err instanceof Error ? err.message : String(err)
|
|
149
|
+
});
|
|
150
|
+
} finally {
|
|
151
|
+
this.isResuming = false;
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
/**
|
|
156
|
+
* Add audio chunk (16-bit PCM)
|
|
157
|
+
*/
|
|
158
|
+
addChunk(pcmData, isLast = false) {
|
|
159
|
+
if (!this.audioContext) {
|
|
160
|
+
logger.error("AudioContext not initialized");
|
|
161
|
+
return;
|
|
162
|
+
}
|
|
163
|
+
if (this.isPlaying && !this.isPaused && this.audioContext.state === "suspended") {
|
|
164
|
+
this.ensureAudioContextRunning().catch((err) => {
|
|
165
|
+
logger.errorWithError("[StreamingAudioPlayer] Failed to ensure AudioContext running in addChunk:", err);
|
|
166
|
+
});
|
|
167
|
+
}
|
|
168
|
+
this.audioChunks.push({ data: pcmData, isLast });
|
|
169
|
+
this.log(`Added chunk ${this.audioChunks.length}`, {
|
|
170
|
+
size: pcmData.length,
|
|
171
|
+
totalChunks: this.audioChunks.length,
|
|
172
|
+
isLast,
|
|
173
|
+
isPlaying: this.isPlaying,
|
|
174
|
+
scheduledChunks: this.scheduledChunks
|
|
175
|
+
});
|
|
176
|
+
if (this.autoContinue && this.isPaused) {
|
|
177
|
+
this.log("[StreamingAudioPlayer] autoContinue=true, auto-resuming playback");
|
|
178
|
+
this.autoContinue = false;
|
|
179
|
+
this.resume().catch((err) => {
|
|
180
|
+
logger.errorWithError("Failed to auto-resume playback:", err);
|
|
181
|
+
});
|
|
182
|
+
}
|
|
183
|
+
if (!this.isPlaying && this.autoStartEnabled && this.audioChunks.length > 0) {
|
|
184
|
+
this.log("[StreamingAudioPlayer] Auto-starting playback from addChunk");
|
|
185
|
+
this.startPlayback().catch((err) => {
|
|
186
|
+
logger.errorWithError("[StreamingAudioPlayer] Failed to start playback from addChunk:", err);
|
|
187
|
+
});
|
|
188
|
+
} else if (this.isPlaying && !this.isPaused) {
|
|
189
|
+
this.log("[StreamingAudioPlayer] Already playing, scheduling next chunk");
|
|
190
|
+
this.scheduleNextChunk();
|
|
191
|
+
} else {
|
|
192
|
+
this.log("[StreamingAudioPlayer] Not playing and no chunks, waiting for more chunks");
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
/**
|
|
196
|
+
* Start new session (stop current and start fresh)
|
|
197
|
+
*/
|
|
198
|
+
async startNewSession(audioChunks) {
|
|
199
|
+
this.stop();
|
|
200
|
+
this.sessionId = `session_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
|
|
201
|
+
this.audioChunks = [];
|
|
202
|
+
this.scheduledChunks = 0;
|
|
203
|
+
this.pausedTimeOffset = 0;
|
|
204
|
+
this.pausedAt = 0;
|
|
205
|
+
this.pausedAudioContextTime = 0;
|
|
206
|
+
this.autoContinue = false;
|
|
207
|
+
this.log("Starting new session", {
|
|
208
|
+
chunks: audioChunks.length
|
|
209
|
+
});
|
|
210
|
+
for (const chunk of audioChunks) {
|
|
211
|
+
this.addChunk(chunk.data, chunk.isLast);
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
/**
|
|
215
|
+
* Start playback
|
|
216
|
+
*/
|
|
217
|
+
async startPlayback() {
|
|
218
|
+
if (!this.audioContext) {
|
|
219
|
+
this.log("[StreamingAudioPlayer] Cannot start playback: AudioContext not initialized");
|
|
220
|
+
return;
|
|
221
|
+
}
|
|
222
|
+
if (this.isPlaying) {
|
|
223
|
+
this.log("[StreamingAudioPlayer] Cannot start playback: Already playing");
|
|
224
|
+
return;
|
|
225
|
+
}
|
|
226
|
+
await this.ensureAudioContextRunning();
|
|
227
|
+
this.isPlaying = true;
|
|
228
|
+
this.sessionStartTime = this.audioContext.currentTime;
|
|
229
|
+
this.scheduledTime = this.sessionStartTime;
|
|
230
|
+
this.lastScheduledChunkEndTime = 0;
|
|
231
|
+
this.scheduledChunkInfo = [];
|
|
232
|
+
this.autoContinue = false;
|
|
233
|
+
this.log("[StreamingAudioPlayer] Starting playback", {
|
|
234
|
+
sessionStartTime: this.sessionStartTime,
|
|
235
|
+
bufferedChunks: this.audioChunks.length,
|
|
236
|
+
scheduledChunks: this.scheduledChunks,
|
|
237
|
+
activeSources: this.activeSources.size,
|
|
238
|
+
audioContextState: this.audioContext.state
|
|
239
|
+
});
|
|
240
|
+
this.scheduleAllChunks();
|
|
241
|
+
}
|
|
242
|
+
/**
|
|
243
|
+
* Schedule all pending chunks
|
|
244
|
+
*/
|
|
245
|
+
scheduleAllChunks() {
|
|
246
|
+
while (this.scheduledChunks < this.audioChunks.length) {
|
|
247
|
+
this.scheduleNextChunk();
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
/**
|
|
251
|
+
* Schedule next audio chunk
|
|
252
|
+
*/
|
|
253
|
+
scheduleNextChunk() {
|
|
254
|
+
if (!this.audioContext) {
|
|
255
|
+
this.log("[StreamingAudioPlayer] Cannot schedule chunk: AudioContext not initialized");
|
|
256
|
+
return;
|
|
257
|
+
}
|
|
258
|
+
if (!this.isPlaying || this.isPaused) {
|
|
259
|
+
this.log("[StreamingAudioPlayer] Cannot schedule chunk: Not playing or paused");
|
|
260
|
+
return;
|
|
261
|
+
}
|
|
262
|
+
if (this.audioContext.state === "suspended") {
|
|
263
|
+
this.ensureAudioContextRunning().catch((err) => {
|
|
264
|
+
logger.errorWithError("[StreamingAudioPlayer] Failed to ensure AudioContext running in scheduleNextChunk:", err);
|
|
265
|
+
});
|
|
266
|
+
}
|
|
267
|
+
const chunkIndex = this.scheduledChunks;
|
|
268
|
+
if (chunkIndex >= this.audioChunks.length) {
|
|
269
|
+
this.log(`[StreamingAudioPlayer] No more chunks to schedule (chunkIndex: ${chunkIndex}, totalChunks: ${this.audioChunks.length})`);
|
|
270
|
+
return;
|
|
271
|
+
}
|
|
272
|
+
const chunk = this.audioChunks[chunkIndex];
|
|
273
|
+
if (chunk.data.length === 0 && !chunk.isLast) {
|
|
274
|
+
this.scheduledChunks++;
|
|
275
|
+
return;
|
|
276
|
+
}
|
|
277
|
+
const pcmData = chunk.data;
|
|
278
|
+
const isLast = chunk.isLast;
|
|
279
|
+
const audioBuffer = this.pcmToAudioBuffer(pcmData);
|
|
280
|
+
if (!audioBuffer) {
|
|
281
|
+
const errorMessage = "Failed to create AudioBuffer from PCM data";
|
|
282
|
+
logger.error(errorMessage);
|
|
283
|
+
logEvent("character_player", "error", {
|
|
284
|
+
sessionId: this.sessionId,
|
|
285
|
+
event: "audio_buffer_creation_failed"
|
|
286
|
+
});
|
|
287
|
+
return;
|
|
288
|
+
}
|
|
289
|
+
try {
|
|
290
|
+
const source = this.audioContext.createBufferSource();
|
|
291
|
+
source.buffer = audioBuffer;
|
|
292
|
+
source.connect(this.gainNode);
|
|
293
|
+
const chunkStartTime = this.scheduledTime;
|
|
294
|
+
source.start(chunkStartTime);
|
|
295
|
+
const actualStartTime = Math.max(chunkStartTime, this.audioContext.currentTime);
|
|
296
|
+
this.scheduledChunkInfo.push({
|
|
297
|
+
startTime: actualStartTime,
|
|
298
|
+
duration: audioBuffer.duration
|
|
299
|
+
});
|
|
300
|
+
this.activeSources.add(source);
|
|
301
|
+
source.onended = () => {
|
|
302
|
+
this.activeSources.delete(source);
|
|
303
|
+
if (this.activeSources.size === 0) {
|
|
304
|
+
const lastChunk = this.audioChunks[this.scheduledChunks - 1];
|
|
305
|
+
if (lastChunk && !lastChunk.isLast) {
|
|
306
|
+
this.log("All audio chunks ended but end=false, pausing and setting autoContinue");
|
|
307
|
+
this.autoContinue = true;
|
|
308
|
+
this.pause();
|
|
309
|
+
} else if (isLast) {
|
|
310
|
+
this.log("Last audio chunk ended, marking playback as ended");
|
|
311
|
+
this.markEnded();
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
};
|
|
315
|
+
this.scheduledTime += audioBuffer.duration;
|
|
316
|
+
this.lastScheduledChunkEndTime = this.scheduledTime - this.sessionStartTime - this.pausedTimeOffset;
|
|
317
|
+
this.scheduledChunks++;
|
|
318
|
+
this.log(`[StreamingAudioPlayer] Scheduled chunk ${chunkIndex + 1}/${this.audioChunks.length}`, {
|
|
319
|
+
startTime: this.scheduledTime - audioBuffer.duration,
|
|
320
|
+
duration: audioBuffer.duration,
|
|
321
|
+
nextScheduleTime: this.scheduledTime,
|
|
322
|
+
isLast,
|
|
323
|
+
activeSources: this.activeSources.size
|
|
324
|
+
});
|
|
325
|
+
} catch (err) {
|
|
326
|
+
logger.errorWithError("Failed to schedule audio chunk:", err);
|
|
327
|
+
logEvent("character_player", "error", {
|
|
328
|
+
sessionId: this.sessionId,
|
|
329
|
+
event: "schedule_chunk_failed",
|
|
330
|
+
reason: err instanceof Error ? err.message : String(err)
|
|
331
|
+
});
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
/**
|
|
335
|
+
* Convert PCM data to AudioBuffer
|
|
336
|
+
* Input: 16-bit PCM (int16), Output: AudioBuffer (float32 [-1, 1])
|
|
337
|
+
*/
|
|
338
|
+
pcmToAudioBuffer(pcmData) {
|
|
339
|
+
if (!this.audioContext) {
|
|
340
|
+
return null;
|
|
341
|
+
}
|
|
342
|
+
if (pcmData.length === 0) {
|
|
343
|
+
const silenceDuration = 0.01;
|
|
344
|
+
const numSamples2 = Math.floor(this.sampleRate * silenceDuration);
|
|
345
|
+
const audioBuffer2 = this.audioContext.createBuffer(
|
|
346
|
+
this.channelCount,
|
|
347
|
+
numSamples2,
|
|
348
|
+
this.sampleRate
|
|
349
|
+
);
|
|
350
|
+
for (let channel = 0; channel < this.channelCount; channel++) {
|
|
351
|
+
const channelData = audioBuffer2.getChannelData(channel);
|
|
352
|
+
channelData.fill(0);
|
|
353
|
+
}
|
|
354
|
+
return audioBuffer2;
|
|
355
|
+
}
|
|
356
|
+
const alignedData = new Uint8Array(pcmData);
|
|
357
|
+
const int16Array = new Int16Array(alignedData.buffer, 0, alignedData.length / 2);
|
|
358
|
+
const numSamples = int16Array.length / this.channelCount;
|
|
359
|
+
const audioBuffer = this.audioContext.createBuffer(
|
|
360
|
+
this.channelCount,
|
|
361
|
+
numSamples,
|
|
362
|
+
this.sampleRate
|
|
363
|
+
);
|
|
364
|
+
for (let channel = 0; channel < this.channelCount; channel++) {
|
|
365
|
+
const channelData = audioBuffer.getChannelData(channel);
|
|
366
|
+
for (let i = 0; i < numSamples; i++) {
|
|
367
|
+
const sampleIndex = i * this.channelCount + channel;
|
|
368
|
+
channelData[i] = int16Array[sampleIndex] / 32768;
|
|
369
|
+
}
|
|
370
|
+
}
|
|
371
|
+
return audioBuffer;
|
|
372
|
+
}
|
|
373
|
+
/**
|
|
374
|
+
* Get current playback time (seconds)
|
|
375
|
+
* 返回实际播放的音频总时长
|
|
376
|
+
*/
|
|
377
|
+
getCurrentTime() {
|
|
378
|
+
if (!this.audioContext || !this.isPlaying) {
|
|
379
|
+
return 0;
|
|
380
|
+
}
|
|
381
|
+
if (this.isPaused) {
|
|
382
|
+
return this.pausedAt;
|
|
383
|
+
}
|
|
384
|
+
const currentAudioTime = this.audioContext.currentTime;
|
|
385
|
+
if (this.activeSources.size === 0 && this.scheduledChunks > 0) {
|
|
386
|
+
return Math.max(0, this.lastScheduledChunkEndTime);
|
|
387
|
+
}
|
|
388
|
+
let totalPlayedDuration = 0;
|
|
389
|
+
for (let i = 0; i < this.scheduledChunkInfo.length; i++) {
|
|
390
|
+
const chunkInfo = this.scheduledChunkInfo[i];
|
|
391
|
+
const chunkEndTime = chunkInfo.startTime + chunkInfo.duration;
|
|
392
|
+
if (currentAudioTime < chunkInfo.startTime) {
|
|
393
|
+
break;
|
|
394
|
+
} else if (chunkEndTime <= currentAudioTime) {
|
|
395
|
+
totalPlayedDuration += chunkInfo.duration;
|
|
396
|
+
} else {
|
|
397
|
+
const playedTime = currentAudioTime - chunkInfo.startTime;
|
|
398
|
+
totalPlayedDuration += playedTime;
|
|
399
|
+
break;
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
return Math.max(0, totalPlayedDuration);
|
|
403
|
+
}
|
|
404
|
+
/**
|
|
405
|
+
* Get total duration of buffered audio (seconds)
|
|
406
|
+
* 计算所有已缓冲 chunk 的总时长
|
|
407
|
+
*/
|
|
408
|
+
getBufferedDuration() {
|
|
409
|
+
if (!this.audioContext) {
|
|
410
|
+
return 0;
|
|
411
|
+
}
|
|
412
|
+
let totalDuration = 0;
|
|
413
|
+
for (const chunk of this.audioChunks) {
|
|
414
|
+
const chunkDuration = chunk.data.length / (this.sampleRate * this.channelCount * 2);
|
|
415
|
+
totalDuration += chunkDuration;
|
|
416
|
+
}
|
|
417
|
+
return totalDuration;
|
|
418
|
+
}
|
|
419
|
+
/**
|
|
420
|
+
* Get current AudioContext time
|
|
421
|
+
* @returns Current AudioContext time in seconds, or 0 if AudioContext is not initialized
|
|
422
|
+
*/
|
|
423
|
+
getAudioContextTime() {
|
|
424
|
+
var _a;
|
|
425
|
+
return ((_a = this.audioContext) == null ? void 0 : _a.currentTime) ?? 0;
|
|
426
|
+
}
|
|
427
|
+
/**
|
|
428
|
+
* Pause playback
|
|
429
|
+
*/
|
|
430
|
+
pause() {
|
|
431
|
+
if (!this.isPlaying || this.isPaused || !this.audioContext) {
|
|
432
|
+
return;
|
|
433
|
+
}
|
|
434
|
+
this.pausedAt = this.getCurrentTime();
|
|
435
|
+
this.pausedAudioContextTime = this.audioContext.currentTime;
|
|
436
|
+
this.isPaused = true;
|
|
437
|
+
if (this.audioContext.state === "running") {
|
|
438
|
+
this.audioContext.suspend().catch((err) => {
|
|
439
|
+
logger.errorWithError("Failed to suspend AudioContext:", err);
|
|
440
|
+
this.isPaused = false;
|
|
441
|
+
});
|
|
442
|
+
}
|
|
443
|
+
this.log("Playback paused", {
|
|
444
|
+
pausedAt: this.pausedAt,
|
|
445
|
+
pausedAudioContextTime: this.pausedAudioContextTime,
|
|
446
|
+
audioContextState: this.audioContext.state
|
|
447
|
+
});
|
|
448
|
+
}
|
|
449
|
+
/**
|
|
450
|
+
* Resume playback
|
|
451
|
+
*/
|
|
452
|
+
async resume() {
|
|
453
|
+
if (!this.isPaused || !this.audioContext || !this.isPlaying) {
|
|
454
|
+
return;
|
|
455
|
+
}
|
|
456
|
+
this.autoContinue = false;
|
|
457
|
+
if (this.audioContext.state === "suspended") {
|
|
458
|
+
try {
|
|
459
|
+
await this.audioContext.resume();
|
|
460
|
+
} catch (err) {
|
|
461
|
+
logger.errorWithError("Failed to resume AudioContext:", err);
|
|
462
|
+
throw err;
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
const currentAudioTime = this.audioContext.currentTime;
|
|
466
|
+
this.sessionStartTime = this.pausedAudioContextTime - this.pausedAt - this.pausedTimeOffset;
|
|
467
|
+
this.isPaused = false;
|
|
468
|
+
if (this.scheduledChunks < this.audioChunks.length) {
|
|
469
|
+
this.scheduleAllChunks();
|
|
470
|
+
}
|
|
471
|
+
this.log("Playback resumed", {
|
|
472
|
+
pausedAt: this.pausedAt,
|
|
473
|
+
pausedAudioContextTime: this.pausedAudioContextTime,
|
|
474
|
+
currentAudioContextTime: currentAudioTime,
|
|
475
|
+
adjustedSessionStartTime: this.sessionStartTime,
|
|
476
|
+
audioContextState: this.audioContext.state
|
|
477
|
+
});
|
|
478
|
+
}
|
|
479
|
+
/**
|
|
480
|
+
* Stop playback
|
|
481
|
+
*/
|
|
482
|
+
stop() {
|
|
483
|
+
if (!this.audioContext) {
|
|
484
|
+
return;
|
|
485
|
+
}
|
|
486
|
+
if (this.isPaused && this.audioContext.state === "suspended") {
|
|
487
|
+
this.audioContext.resume().catch(() => {
|
|
488
|
+
});
|
|
489
|
+
this.isPaused = false;
|
|
490
|
+
}
|
|
491
|
+
this.isPlaying = false;
|
|
492
|
+
this.isPaused = false;
|
|
493
|
+
this.isResuming = false;
|
|
494
|
+
this.sessionStartTime = 0;
|
|
495
|
+
this.scheduledTime = 0;
|
|
496
|
+
for (const source of this.activeSources) {
|
|
497
|
+
source.onended = null;
|
|
498
|
+
try {
|
|
499
|
+
source.stop(0);
|
|
500
|
+
} catch {
|
|
501
|
+
}
|
|
502
|
+
try {
|
|
503
|
+
source.disconnect();
|
|
504
|
+
} catch {
|
|
505
|
+
}
|
|
506
|
+
}
|
|
507
|
+
this.activeSources.clear();
|
|
508
|
+
this.audioChunks = [];
|
|
509
|
+
this.scheduledChunks = 0;
|
|
510
|
+
this.autoContinue = false;
|
|
511
|
+
this.log("[StreamingAudioPlayer] Playback stopped, state reset");
|
|
512
|
+
}
|
|
513
|
+
/**
|
|
514
|
+
* Enable or disable auto-start (for delayed start scenarios)
|
|
515
|
+
*/
|
|
516
|
+
setAutoStart(enabled) {
|
|
517
|
+
this.autoStartEnabled = enabled;
|
|
518
|
+
this.log(`Auto-start ${enabled ? "enabled" : "disabled"}`);
|
|
519
|
+
}
|
|
520
|
+
/**
|
|
521
|
+
* Start playback manually (for delayed start scenarios)
|
|
522
|
+
* This allows starting playback after transition animation completes
|
|
523
|
+
*/
|
|
524
|
+
play() {
|
|
525
|
+
if (this.isPlaying) {
|
|
526
|
+
return;
|
|
527
|
+
}
|
|
528
|
+
this.autoStartEnabled = true;
|
|
529
|
+
this.startPlayback().catch((err) => {
|
|
530
|
+
logger.errorWithError("[StreamingAudioPlayer] Failed to start playback from play():", err);
|
|
531
|
+
});
|
|
532
|
+
}
|
|
533
|
+
/**
|
|
534
|
+
* Mark playback as ended
|
|
535
|
+
*/
|
|
536
|
+
markEnded() {
|
|
537
|
+
var _a;
|
|
538
|
+
this.log("Playback ended");
|
|
539
|
+
this.isPlaying = false;
|
|
540
|
+
(_a = this.onEndedCallback) == null ? void 0 : _a.call(this);
|
|
541
|
+
}
|
|
542
|
+
/**
|
|
543
|
+
* Set ended callback
|
|
544
|
+
*/
|
|
545
|
+
onEnded(callback) {
|
|
546
|
+
this.onEndedCallback = callback;
|
|
547
|
+
}
|
|
548
|
+
/**
|
|
549
|
+
* Check if playing
|
|
550
|
+
*/
|
|
551
|
+
isPlayingNow() {
|
|
552
|
+
return this.isPlaying && !this.isPaused;
|
|
553
|
+
}
|
|
554
|
+
/**
|
|
555
|
+
* Dispose and cleanup
|
|
556
|
+
*/
|
|
557
|
+
dispose() {
|
|
558
|
+
this.stop();
|
|
559
|
+
if (this.audioContext && this.stateChangeHandler) {
|
|
560
|
+
this.audioContext.removeEventListener("statechange", this.stateChangeHandler);
|
|
561
|
+
this.stateChangeHandler = void 0;
|
|
562
|
+
}
|
|
563
|
+
if (this.audioContext) {
|
|
564
|
+
this.audioContext.close();
|
|
565
|
+
this.audioContext = null;
|
|
566
|
+
this.gainNode = null;
|
|
567
|
+
}
|
|
568
|
+
this.audioChunks = [];
|
|
569
|
+
this.scheduledChunks = 0;
|
|
570
|
+
this.sessionStartTime = 0;
|
|
571
|
+
this.pausedTimeOffset = 0;
|
|
572
|
+
this.pausedAt = 0;
|
|
573
|
+
this.pausedAudioContextTime = 0;
|
|
574
|
+
this.scheduledTime = 0;
|
|
575
|
+
this.onEndedCallback = void 0;
|
|
576
|
+
this.log("StreamingAudioPlayer disposed");
|
|
577
|
+
}
|
|
578
|
+
/**
|
|
579
|
+
* Flush buffered audio
|
|
580
|
+
* - hard: stops all playing sources and clears all chunks
|
|
581
|
+
* - soft (default): clears UNSCHEDULED chunks only
|
|
582
|
+
*/
|
|
583
|
+
flush(options) {
|
|
584
|
+
const hard = (options == null ? void 0 : options.hard) === true;
|
|
585
|
+
if (hard) {
|
|
586
|
+
this.stop();
|
|
587
|
+
this.audioChunks = [];
|
|
588
|
+
this.scheduledChunks = 0;
|
|
589
|
+
this.sessionStartTime = 0;
|
|
590
|
+
this.pausedAt = 0;
|
|
591
|
+
this.scheduledTime = 0;
|
|
592
|
+
this.log("Flushed (hard)");
|
|
593
|
+
return;
|
|
594
|
+
}
|
|
595
|
+
if (this.scheduledChunks < this.audioChunks.length) {
|
|
596
|
+
this.audioChunks.splice(this.scheduledChunks);
|
|
597
|
+
}
|
|
598
|
+
this.log("Flushed (soft)", { remainingScheduled: this.scheduledChunks });
|
|
599
|
+
}
|
|
600
|
+
/**
|
|
601
|
+
* 设置音量 (0.0 - 1.0)
|
|
602
|
+
* 注意:这仅控制数字人音频播放器的音量,不影响系统音量
|
|
603
|
+
* @param volume 音量值,范围 0.0 到 1.0(0.0 为静音,1.0 为最大音量)
|
|
604
|
+
*/
|
|
605
|
+
setVolume(volume) {
|
|
606
|
+
if (volume < 0 || volume > 1) {
|
|
607
|
+
logger.warn(`[StreamingAudioPlayer] Volume out of range: ${volume}, clamping to [0, 1]`);
|
|
608
|
+
volume = Math.max(0, Math.min(1, volume));
|
|
609
|
+
}
|
|
610
|
+
this.volume = volume;
|
|
611
|
+
if (this.gainNode) {
|
|
612
|
+
this.gainNode.gain.value = volume;
|
|
613
|
+
}
|
|
614
|
+
}
|
|
615
|
+
/**
|
|
616
|
+
* 获取当前音量
|
|
617
|
+
* @returns 当前音量值 (0.0 - 1.0)
|
|
618
|
+
*/
|
|
619
|
+
getVolume() {
|
|
620
|
+
return this.volume;
|
|
621
|
+
}
|
|
622
|
+
/**
|
|
623
|
+
* Debug logging
|
|
624
|
+
*/
|
|
625
|
+
log(message, data) {
|
|
626
|
+
if (this.debug) {
|
|
627
|
+
logger.log(`[StreamingAudioPlayer] ${message}`, data || "");
|
|
628
|
+
}
|
|
629
|
+
}
|
|
630
|
+
}
|
|
631
|
+
export {
|
|
632
|
+
StreamingAudioPlayer
|
|
633
|
+
};
|
|
@@ -2,20 +2,23 @@ import { EventEmitter } from './utils/eventEmitter';
|
|
|
2
2
|
export interface AnimationWebSocketClientOptions {
|
|
3
3
|
wsUrl: string;
|
|
4
4
|
reconnectAttempts?: number;
|
|
5
|
-
debug?: boolean;
|
|
6
5
|
jwtToken?: string;
|
|
6
|
+
appId?: string;
|
|
7
|
+
clientId?: string;
|
|
7
8
|
}
|
|
8
9
|
export declare class AnimationWebSocketClient extends EventEmitter {
|
|
9
10
|
private wsUrl;
|
|
10
11
|
private reconnectAttempts;
|
|
11
|
-
private debug;
|
|
12
12
|
private jwtToken?;
|
|
13
|
+
private appId?;
|
|
14
|
+
private clientId?;
|
|
13
15
|
private ws;
|
|
14
16
|
private currentCharacterId;
|
|
15
17
|
private currentRetryCount;
|
|
16
18
|
private isConnecting;
|
|
17
19
|
private isManuallyDisconnected;
|
|
18
20
|
private reconnectTimer;
|
|
21
|
+
private sessionConfigured;
|
|
19
22
|
constructor(options: AnimationWebSocketClientOptions);
|
|
20
23
|
/**
|
|
21
24
|
* 连接WebSocket
|
|
@@ -27,13 +30,14 @@ export declare class AnimationWebSocketClient extends EventEmitter {
|
|
|
27
30
|
disconnect(): void;
|
|
28
31
|
/**
|
|
29
32
|
* 发送音频数据
|
|
33
|
+
* @param conversationId - 会话ID(在 protobuf 协议中映射为 reqId 字段)
|
|
30
34
|
*/
|
|
31
|
-
sendAudioData(
|
|
35
|
+
sendAudioData(conversationId: string, audioData: ArrayBuffer, end: boolean): boolean;
|
|
32
36
|
/**
|
|
33
|
-
*
|
|
34
|
-
*
|
|
37
|
+
* 生成会话ID
|
|
38
|
+
* 使用统一的会话ID生成规则:YYYYMMDDHHmmss_nanoid
|
|
35
39
|
*/
|
|
36
|
-
|
|
40
|
+
generateConversationId(): string;
|
|
37
41
|
/**
|
|
38
42
|
* 获取连接状态
|
|
39
43
|
*/
|
|
@@ -44,7 +48,14 @@ export declare class AnimationWebSocketClient extends EventEmitter {
|
|
|
44
48
|
getCurrentCharacterId(): string;
|
|
45
49
|
private buildWebSocketUrl;
|
|
46
50
|
private connectWebSocket;
|
|
51
|
+
/**
|
|
52
|
+
* 清理 URL 用于日志记录(隐藏敏感信息)
|
|
53
|
+
*/
|
|
54
|
+
private sanitizeUrlForLog;
|
|
55
|
+
/**
|
|
56
|
+
* v2 协议:配置会话(发送采样率等参数)
|
|
57
|
+
*/
|
|
58
|
+
private configureSession;
|
|
47
59
|
private handleMessage;
|
|
48
60
|
private scheduleReconnect;
|
|
49
61
|
}
|
|
50
|
-
//# sourceMappingURL=AnimationWebSocketClient.d.ts.map
|