@xmov/avatar 2.0.1 → 2.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/README.md +3 -1
  2. package/dist/agent/avatar.cjs +1 -2
  3. package/dist/agent/avatar.modern.js +1 -2
  4. package/dist/agent/avatar.module.js +1 -2
  5. package/dist/agent/avatar.umd.js +1 -2
  6. package/dist/agent/index.cjs +1 -4
  7. package/dist/agent/index.d.ts +12 -5
  8. package/dist/agent/index.umd.js +3 -5
  9. package/dist/agent/types.d.ts +5 -43
  10. package/dist/baseRender/AvatarRenderer.d.ts +0 -1
  11. package/dist/control/RenderScheduler.d.ts +17 -1
  12. package/dist/control/ttsa.d.ts +8 -1
  13. package/dist/index.cjs +1 -1
  14. package/dist/index.cjs.map +1 -1
  15. package/dist/index.d.ts +5 -1
  16. package/dist/index.modern.js +1 -1
  17. package/dist/index.modern.js.map +1 -1
  18. package/dist/index.module.js +1 -1
  19. package/dist/index.module.js.map +1 -1
  20. package/dist/index.umd.js +1 -1
  21. package/dist/index.umd.js.map +1 -1
  22. package/dist/modules/ResourceManager.d.ts +2 -1
  23. package/dist/utils/request.d.ts +1 -0
  24. package/package.json +29 -7
  25. package/src/baseRender/AudioRenderer.ts +16 -5
  26. package/src/baseRender/AvatarRenderer.ts +11 -20
  27. package/src/baseRender/MSEAudioPlayer.ts +18 -7
  28. package/src/control/RenderScheduler.ts +74 -8
  29. package/src/control/ttsa.ts +47 -11
  30. package/src/index.ts +66 -45
  31. package/src/modules/ResourceManager.ts +19 -6
  32. package/src/modules/decoder.ts +34 -1
  33. package/src/utils/capability-checker.ts +95 -21
  34. package/src/utils/request.ts +14 -2
  35. package/src/view/DebugOverlay.ts +9 -4
  36. package/dist/agent/__tests__/agent.test.d.ts +0 -1
  37. package/dist/agent/audio-debug.d.ts +0 -3
  38. package/dist/agent/audio-uplink.d.ts +0 -62
  39. package/dist/agent/avatar.cjs.map +0 -1
  40. package/dist/agent/avatar.modern.js.map +0 -1
  41. package/dist/agent/avatar.module.js.map +0 -1
  42. package/dist/agent/avatar.umd.js.map +0 -1
  43. package/dist/agent/e2e-client.d.ts +0 -32
  44. package/dist/agent/fixed-audio-track.d.ts +0 -48
  45. package/dist/agent/microphone.d.ts +0 -49
  46. package/src/agent/__tests__/agent.test.ts +0 -3787
  47. package/src/agent/audio-debug.ts +0 -36
  48. package/src/agent/audio-uplink.ts +0 -392
  49. package/src/agent/e2e-client.ts +0 -215
  50. package/src/agent/fixed-audio-track.ts +0 -547
  51. package/src/agent/index.ts +0 -1393
  52. package/src/agent/microphone.ts +0 -534
  53. package/src/agent/types.ts +0 -197
@@ -1,3787 +0,0 @@
1
- // @ts-nocheck
2
- import test from "node:test";
3
- import assert from "node:assert/strict";
4
- import XingyunAvatarAgent, {
5
- AgentE2EClient,
6
- MicrophoneController,
7
- } from "../index";
8
- import {
9
- AgentAudioUplink,
10
- AU_STREAM_FAREND,
11
- AU_STREAM_NEAREND,
12
- packAuAudioFrame,
13
- } from "../audio-uplink";
14
- import {
15
- beginWebAudioCaptureSession,
16
- configureWebAudioPlaybackSession,
17
- endWebAudioCaptureSession,
18
- subscribeWebAudioPlaybackSessionRestored,
19
- } from "../../utils/audio-session";
20
- import PCMAudioPlayer from "../../baseRender/AudioWorklet";
21
- import { PcmWebMEncoder } from "../../encoding/pcm-webm-encoder";
22
-
23
- class MockWebSocket {
24
- static CONNECTING = 0;
25
- static OPEN = 1;
26
- static CLOSING = 2;
27
- static CLOSED = 3;
28
- static instances = [];
29
- static failedConnections = 0;
30
- static autoOpen = true;
31
-
32
- readyState = MockWebSocket.CONNECTING;
33
- bufferedAmount = 0;
34
- sent = [];
35
- closed = false;
36
- onopen = null;
37
- onmessage = null;
38
- onerror = null;
39
- onclose = null;
40
-
41
- constructor(url) {
42
- this.url = url;
43
- MockWebSocket.instances.push(this);
44
- if (MockWebSocket.autoOpen) {
45
- queueMicrotask(() => this.open());
46
- }
47
- }
48
-
49
- open() {
50
- if (this.readyState !== MockWebSocket.CONNECTING) {
51
- return;
52
- }
53
- if (MockWebSocket.failedConnections > 0) {
54
- MockWebSocket.failedConnections -= 1;
55
- this.readyState = MockWebSocket.CLOSED;
56
- this.onerror?.({});
57
- this.onclose?.({ code: 1006, reason: "", wasClean: false });
58
- return;
59
- }
60
- this.readyState = MockWebSocket.OPEN;
61
- this.onopen?.({});
62
- }
63
-
64
- send(payload) {
65
- this.sent.push(payload);
66
- }
67
-
68
- close(code, reason) {
69
- this.closed = true;
70
- this.readyState = MockWebSocket.CLOSED;
71
- this.onclose?.({ code, reason });
72
- }
73
-
74
- closeRemote(code, reason = "", wasClean = false) {
75
- this.readyState = MockWebSocket.CLOSED;
76
- this.onclose?.({ code, reason, wasClean });
77
- }
78
-
79
- emit(data) {
80
- this.onmessage?.({ data: JSON.stringify(data) });
81
- }
82
-
83
- emitRaw(data) {
84
- this.onmessage?.({ data });
85
- }
86
- }
87
-
88
- function createAgent(callbacks = {}, options = {}) {
89
- const agent = new XingyunAvatarAgent({
90
- containerId: "#sdk",
91
- appId: "app",
92
- appSecret: "secret",
93
- gatewayServer: "https://gateway.example.com/user/v1/ttsa_v2/session",
94
- asr_id: 1,
95
- llm_id: 1,
96
- onMessage() {},
97
- e2eServer: "wss://e2e.example.com/ws",
98
- authToken: "token-from-nebula",
99
- webSocketCtor: MockWebSocket,
100
- ...options,
101
- agentCallbacks: callbacks,
102
- });
103
- agent.incompleteInitializationCleanups = [];
104
- agent.initializationFailureNotifications = [];
105
- agent.cleanupBeforeInitComplete = async (reason) => {
106
- agent.incompleteInitializationCleanups.push(reason);
107
- agent.destroyed = true;
108
- };
109
- agent.reportInitializationFailure = (message, cause) => {
110
- agent.initializationFailureNotifications.push({ message, cause });
111
- agent.avatarOptions.onMessage?.({ code: 10005, message, cause });
112
- };
113
- return { agent };
114
- }
115
-
116
- function readAuFrame(frame) {
117
- assert.equal(frame instanceof ArrayBuffer, true);
118
- const bytes = new Uint8Array(frame);
119
- const view = new DataView(frame);
120
- return {
121
- magic: String.fromCharCode(bytes[0], bytes[1]),
122
- version: bytes[2],
123
- streamId: bytes[3],
124
- tsMs: view.getUint32(4, false),
125
- payload: new TextDecoder().decode(bytes.subarray(8)),
126
- };
127
- }
128
-
129
- function installMicrophoneBrowser({
130
- getUserMedia,
131
- mediaDevices = true,
132
- webmOpus = true,
133
- sampleRate = 48000,
134
- audioDataFrames = [],
135
- trackTransform = true,
136
- webAudio = true,
137
- audioContextSampleRate = 16000,
138
- audioContextState = "running",
139
- audioContextResume,
140
- audioContextError,
141
- mediaRecorderConstructorError,
142
- mediaRecorderStartError,
143
- audioSession = false,
144
- webCodecsOpus = true,
145
- webCodecsPreservesOpusConfig = true,
146
- audioEncoderSupport,
147
- encodedChunkDurations,
148
- flushEncodedChunkDurations = [],
149
- } = {}) {
150
- const originalNavigator = Object.getOwnPropertyDescriptor(globalThis, "navigator");
151
- const originalMediaRecorder = Object.getOwnPropertyDescriptor(globalThis, "MediaRecorder");
152
- const originalMediaStream = Object.getOwnPropertyDescriptor(globalThis, "MediaStream");
153
- const originalAudioData = Object.getOwnPropertyDescriptor(globalThis, "AudioData");
154
- const originalProcessor = Object.getOwnPropertyDescriptor(globalThis, "MediaStreamTrackProcessor");
155
- const originalGenerator = Object.getOwnPropertyDescriptor(globalThis, "MediaStreamTrackGenerator");
156
- const originalAudioContext = Object.getOwnPropertyDescriptor(globalThis, "AudioContext");
157
- const originalAudioEncoder = Object.getOwnPropertyDescriptor(globalThis, "AudioEncoder");
158
- const originalAudioWorkletNode = Object.getOwnPropertyDescriptor(globalThis, "AudioWorkletNode");
159
- const getUserMediaCalls = [];
160
- const tracks = [];
161
- const streams = [];
162
- const recorders = [];
163
- const processors = [];
164
- const generators = [];
165
- const audioContexts = [];
166
- const audioSources = [];
167
- const audioDestinations = [];
168
- const audioSessionTypes = [];
169
- const audioEncoders = [];
170
- const audioWorkletNodes = [];
171
- const mockedAudioSession = audioSession
172
- ? {
173
- currentType: typeof audioSession === "string" ? audioSession : "ambient",
174
- get type() {
175
- return this.currentType;
176
- },
177
- set type(value) {
178
- this.currentType = value;
179
- audioSessionTypes.push(value);
180
- },
181
- }
182
- : undefined;
183
-
184
- function createStream(trackSampleRate = sampleRate, channelCount = 1) {
185
- const track = {
186
- stopCalls: 0,
187
- readyState: "live",
188
- getSettings() {
189
- return { sampleRate: trackSampleRate, channelCount };
190
- },
191
- stop() {
192
- this.stopCalls += 1;
193
- this.readyState = "ended";
194
- },
195
- clone() {
196
- return createStream(trackSampleRate, channelCount).getAudioTracks()[0];
197
- },
198
- };
199
- const stream = {
200
- getTracks() {
201
- return [track];
202
- },
203
- getAudioTracks() {
204
- return [track];
205
- },
206
- };
207
- tracks.push(track);
208
- streams.push(stream);
209
- return stream;
210
- }
211
-
212
- class MockMediaStream {
213
- constructor(inputTracks = []) {
214
- this.tracks = [...inputTracks];
215
- }
216
-
217
- getTracks() {
218
- return this.tracks;
219
- }
220
-
221
- getAudioTracks() {
222
- return this.tracks.filter((track) => track.kind !== "video");
223
- }
224
- }
225
-
226
- class MockAudioData {
227
- constructor(options) {
228
- this.sampleRate = options.sampleRate;
229
- this.numberOfFrames = options.numberOfFrames;
230
- this.numberOfChannels = options.numberOfChannels;
231
- this.timestamp = options.timestamp;
232
- this.planes = Array.isArray(options.data) ? options.data : [options.data];
233
- this.data = this.planes[0];
234
- this.closed = false;
235
- }
236
-
237
- copyTo(destination, options = {}) {
238
- destination.set(this.planes[options.planeIndex || 0]);
239
- }
240
-
241
- close() {
242
- this.closed = true;
243
- }
244
- }
245
-
246
- class MockMediaStreamTrackProcessor {
247
- constructor({ track }) {
248
- this.track = track;
249
- this.cancelCalls = 0;
250
- this.releaseCalls = 0;
251
- this.readable = new ReadableStream({
252
- start: (controller) => {
253
- for (const frame of audioDataFrames) {
254
- controller.enqueue(new MockAudioData(frame));
255
- }
256
- controller.close();
257
- },
258
- });
259
- processors.push(this);
260
- }
261
- }
262
-
263
- class MockMediaStreamTrackGenerator {
264
- constructor() {
265
- this.kind = "audio";
266
- this.readyState = "live";
267
- this.stopCalls = 0;
268
- this.written = [];
269
- this.writable = new WritableStream({
270
- write: (audioData) => {
271
- this.written.push(audioData);
272
- },
273
- close: () => {
274
- this.writableClosed = true;
275
- },
276
- });
277
- generators.push(this);
278
- }
279
-
280
- getSettings() {
281
- return { sampleRate: 16000, channelCount: 1 };
282
- }
283
-
284
- stop() {
285
- this.stopCalls += 1;
286
- this.readyState = "ended";
287
- }
288
- }
289
-
290
- class MockAudioContext {
291
- constructor(options = {}) {
292
- if (audioContextError) {
293
- throw audioContextError;
294
- }
295
- this.sampleRate = audioContextSampleRate ?? options.sampleRate ?? sampleRate;
296
- this.state = audioContextState;
297
- this.closeCalls = 0;
298
- this.destination = {};
299
- this.audioWorklet = {
300
- addModule: async () => {},
301
- };
302
- audioContexts.push(this);
303
- }
304
-
305
- createMediaStreamSource(stream) {
306
- const source = {
307
- stream,
308
- connectedTo: null,
309
- disconnectCalls: 0,
310
- connect: (destination) => {
311
- source.connectedTo = destination;
312
- },
313
- disconnect: () => {
314
- source.disconnectCalls += 1;
315
- source.connectedTo = null;
316
- },
317
- };
318
- audioSources.push(source);
319
- return source;
320
- }
321
-
322
- createMediaStreamDestination() {
323
- const track = {
324
- kind: "audio",
325
- readyState: "live",
326
- stopCalls: 0,
327
- getSettings: () => ({ sampleRate: this.sampleRate, channelCount: 1 }),
328
- stop() {
329
- this.stopCalls += 1;
330
- this.readyState = "ended";
331
- },
332
- };
333
- const destination = {
334
- channelCount: 2,
335
- channelCountMode: "max",
336
- channelInterpretation: "speakers",
337
- disconnectCalls: 0,
338
- stream: new MockMediaStream([track]),
339
- disconnect() {
340
- this.disconnectCalls += 1;
341
- },
342
- };
343
- audioDestinations.push(destination);
344
- return destination;
345
- }
346
-
347
- async resume() {
348
- await audioContextResume?.(this);
349
- if (this.state !== "closed") {
350
- this.state = "running";
351
- }
352
- }
353
-
354
- async close() {
355
- this.closeCalls += 1;
356
- this.state = "closed";
357
- }
358
- }
359
-
360
- class MockEncodedAudioChunk {
361
- constructor(timestamp, duration, data) {
362
- this.timestamp = timestamp;
363
- this.duration = duration;
364
- this.data = data;
365
- this.byteLength = data.length;
366
- }
367
-
368
- copyTo(destination) {
369
- destination.set(this.data);
370
- }
371
- }
372
-
373
- class MockAudioEncoder {
374
- static async isConfigSupported(config) {
375
- if (audioEncoderSupport) {
376
- return audioEncoderSupport(config);
377
- }
378
- if (!webCodecsPreservesOpusConfig) {
379
- const { opus, ...baseConfig } = config;
380
- return { supported: webCodecsOpus, config: baseConfig };
381
- }
382
- return { supported: webCodecsOpus, config };
383
- }
384
-
385
- constructor(options) {
386
- this.options = options;
387
- this.flushCalls = 0;
388
- this.nextTimestamp = 0;
389
- audioEncoders.push(this);
390
- }
391
-
392
- configure(config) {
393
- this.config = config;
394
- }
395
-
396
- encode(audioData) {
397
- let timestamp = audioData.timestamp;
398
- const inputDuration = Math.round(
399
- (audioData.numberOfFrames * 1_000_000) / audioData.sampleRate,
400
- );
401
- const frameDuration = this.config?.opus?.frameDuration ?? inputDuration;
402
- const durations = encodedChunkDurations
403
- ?? new Array(Math.ceil(inputDuration / frameDuration)).fill(frameDuration);
404
- for (const duration of durations) {
405
- this.options.output(new MockEncodedAudioChunk(
406
- timestamp,
407
- duration,
408
- new Uint8Array([0xf8, 0xff, 0xfe]),
409
- ));
410
- timestamp += duration;
411
- }
412
- this.nextTimestamp = timestamp;
413
- }
414
-
415
- async flush() {
416
- this.flushCalls += 1;
417
- for (const duration of flushEncodedChunkDurations) {
418
- this.options.output(new MockEncodedAudioChunk(
419
- this.nextTimestamp,
420
- duration,
421
- new Uint8Array([0xf8, 0xff, 0xfe]),
422
- ));
423
- this.nextTimestamp += duration;
424
- }
425
- }
426
-
427
- close() {
428
- this.closed = true;
429
- }
430
- }
431
-
432
- class MockAudioWorkletNode {
433
- constructor(context, name, options) {
434
- this.context = context;
435
- this.name = name;
436
- this.options = options;
437
- this.port = {
438
- onmessage: null,
439
- postMessage: (message) => {
440
- if (message?.type === "flush") {
441
- queueMicrotask(() => this.port.onmessage?.({ data: { type: "flushed" } }));
442
- }
443
- },
444
- close() {},
445
- };
446
- audioWorkletNodes.push(this);
447
- }
448
-
449
- connect(destination) {
450
- this.connectedTo = destination;
451
- }
452
-
453
- disconnect() {
454
- this.connectedTo = null;
455
- }
456
-
457
- emitPcm(sequence = 0, fill = 1) {
458
- const pcm = new Int16Array(1600).fill(fill);
459
- this.port.onmessage?.({
460
- data: { type: "frame", pcm: pcm.buffer, sequence },
461
- });
462
- }
463
- }
464
-
465
- class MockMediaRecorder {
466
- static isTypeSupported(type) {
467
- return webmOpus && type === "audio/webm;codecs=opus";
468
- }
469
-
470
- state = "inactive";
471
- mimeType = "audio/webm;codecs=opus";
472
- ondataavailable = null;
473
- onerror = null;
474
- onstop = null;
475
- requestDataCalls = 0;
476
-
477
- constructor(stream, options) {
478
- if (mediaRecorderConstructorError) {
479
- throw mediaRecorderConstructorError;
480
- }
481
- this.stream = stream;
482
- this.options = options;
483
- recorders.push(this);
484
- }
485
-
486
- start(timeslice) {
487
- if (mediaRecorderStartError) {
488
- throw mediaRecorderStartError;
489
- }
490
- this.state = "recording";
491
- this.timeslice = timeslice;
492
- }
493
-
494
- requestData() {
495
- this.requestDataCalls += 1;
496
- }
497
-
498
- stop() {
499
- this.ondataavailable?.({
500
- data: new Blob(["opus-final"], { type: this.mimeType }),
501
- timecode: 200,
502
- });
503
- this.state = "inactive";
504
- this.onstop?.({});
505
- }
506
- }
507
-
508
- const mockedMediaDevices = mediaDevices
509
- ? {
510
- async getUserMedia(constraints) {
511
- getUserMediaCalls.push(constraints);
512
- return getUserMedia ? getUserMedia(constraints, createStream) : createStream();
513
- },
514
- }
515
- : undefined;
516
-
517
- Object.defineProperty(globalThis, "navigator", {
518
- configurable: true,
519
- value: {
520
- mediaDevices: mockedMediaDevices,
521
- ...(mockedAudioSession ? { audioSession: mockedAudioSession } : {}),
522
- },
523
- });
524
- Object.defineProperty(globalThis, "MediaRecorder", {
525
- configurable: true,
526
- value: MockMediaRecorder,
527
- });
528
- Object.defineProperty(globalThis, "MediaStream", {
529
- configurable: true,
530
- value: MockMediaStream,
531
- });
532
- if (trackTransform) {
533
- Object.defineProperty(globalThis, "AudioData", {
534
- configurable: true,
535
- value: MockAudioData,
536
- });
537
- Object.defineProperty(globalThis, "MediaStreamTrackProcessor", {
538
- configurable: true,
539
- value: MockMediaStreamTrackProcessor,
540
- });
541
- Object.defineProperty(globalThis, "MediaStreamTrackGenerator", {
542
- configurable: true,
543
- value: MockMediaStreamTrackGenerator,
544
- });
545
- } else {
546
- delete globalThis.AudioData;
547
- delete globalThis.MediaStreamTrackProcessor;
548
- delete globalThis.MediaStreamTrackGenerator;
549
- }
550
- if (webAudio) {
551
- Object.defineProperty(globalThis, "AudioContext", {
552
- configurable: true,
553
- value: MockAudioContext,
554
- });
555
- } else {
556
- delete globalThis.AudioContext;
557
- }
558
- Object.defineProperty(globalThis, "AudioEncoder", {
559
- configurable: true,
560
- value: MockAudioEncoder,
561
- });
562
- Object.defineProperty(globalThis, "AudioWorkletNode", {
563
- configurable: true,
564
- value: MockAudioWorkletNode,
565
- });
566
-
567
- return {
568
- getUserMediaCalls,
569
- recorders,
570
- processors,
571
- generators,
572
- audioContexts,
573
- audioSources,
574
- audioDestinations,
575
- audioSessionTypes,
576
- audioEncoders,
577
- audioWorkletNodes,
578
- audioSession: mockedAudioSession,
579
- createStream,
580
- streams,
581
- tracks,
582
- restore() {
583
- if (originalNavigator) {
584
- Object.defineProperty(globalThis, "navigator", originalNavigator);
585
- } else {
586
- delete globalThis.navigator;
587
- }
588
- if (originalMediaRecorder) {
589
- Object.defineProperty(globalThis, "MediaRecorder", originalMediaRecorder);
590
- } else {
591
- delete globalThis.MediaRecorder;
592
- }
593
- if (originalMediaStream) {
594
- Object.defineProperty(globalThis, "MediaStream", originalMediaStream);
595
- } else {
596
- delete globalThis.MediaStream;
597
- }
598
- if (originalAudioData) {
599
- Object.defineProperty(globalThis, "AudioData", originalAudioData);
600
- } else {
601
- delete globalThis.AudioData;
602
- }
603
- if (originalProcessor) {
604
- Object.defineProperty(globalThis, "MediaStreamTrackProcessor", originalProcessor);
605
- } else {
606
- delete globalThis.MediaStreamTrackProcessor;
607
- }
608
- if (originalGenerator) {
609
- Object.defineProperty(globalThis, "MediaStreamTrackGenerator", originalGenerator);
610
- } else {
611
- delete globalThis.MediaStreamTrackGenerator;
612
- }
613
- if (originalAudioContext) {
614
- Object.defineProperty(globalThis, "AudioContext", originalAudioContext);
615
- } else {
616
- delete globalThis.AudioContext;
617
- }
618
- if (originalAudioEncoder) {
619
- Object.defineProperty(globalThis, "AudioEncoder", originalAudioEncoder);
620
- } else {
621
- delete globalThis.AudioEncoder;
622
- }
623
- if (originalAudioWorkletNode) {
624
- Object.defineProperty(globalThis, "AudioWorkletNode", originalAudioWorkletNode);
625
- } else {
626
- delete globalThis.AudioWorkletNode;
627
- }
628
- },
629
- };
630
- }
631
-
632
- function namedError(name, message) {
633
- const error = new Error(message);
634
- error.name = name;
635
- return error;
636
- }
637
-
638
- async function waitFor(predicate, timeoutMs = 1000) {
639
- const deadline = Date.now() + timeoutMs;
640
- while (!predicate()) {
641
- if (Date.now() >= deadline) {
642
- throw new Error("Timed out waiting for condition");
643
- }
644
- await new Promise((resolve) => setTimeout(resolve, 0));
645
- }
646
- }
647
-
648
- test("AU v1 packer writes magic, version, stream and big-endian timestamp", () => {
649
- const frame = packAuAudioFrame(
650
- AU_STREAM_FAREND,
651
- 12340,
652
- new TextEncoder().encode("opus"),
653
- );
654
-
655
- assert.deepEqual(readAuFrame(frame), {
656
- magic: "AU",
657
- version: 1,
658
- streamId: AU_STREAM_FAREND,
659
- tsMs: 12340,
660
- payload: "opus",
661
- });
662
- });
663
-
664
- test("AgentAudioUplink sends farend before nearend in the same 100ms bucket", async () => {
665
- const sent = [];
666
- const uplink = new AgentAudioUplink({ sendBinary: (frame) => sent.push(frame) });
667
- uplink.reset(1000);
668
-
669
- uplink.offerFarend({ data: new TextEncoder().encode("far"), timestamp: 1250 });
670
- uplink.offerNearend({ data: new Blob(["near"]), timestamp: 1270 });
671
- await uplink.drain();
672
-
673
- assert.deepEqual(sent.map(readAuFrame), [
674
- { magic: "AU", version: 1, streamId: AU_STREAM_FAREND, tsMs: 250, payload: "far" },
675
- { magic: "AU", version: 1, streamId: AU_STREAM_NEAREND, tsMs: 270, payload: "near" },
676
- ]);
677
- });
678
-
679
- test("AgentAudioUplink waits for a same-bucket farend that arrives after nearend", async () => {
680
- const sent = [];
681
- const uplink = new AgentAudioUplink({ sendBinary: (frame) => sent.push(frame) });
682
- uplink.reset(1000);
683
-
684
- uplink.offerNearend({ data: new Blob(["near"]), timestamp: 1270 });
685
- assert.deepEqual(sent, []);
686
- uplink.offerFarend({ data: new TextEncoder().encode("far"), timestamp: 1250 });
687
- await uplink.drain();
688
-
689
- assert.deepEqual(sent.map(readAuFrame), [
690
- { magic: "AU", version: 1, streamId: AU_STREAM_FAREND, tsMs: 250, payload: "far" },
691
- { magic: "AU", version: 1, streamId: AU_STREAM_NEAREND, tsMs: 270, payload: "near" },
692
- ]);
693
- });
694
-
695
- test("AgentAudioUplink preserves every nearend fragment in the same 100ms bucket", async () => {
696
- const sent = [];
697
- const uplink = new AgentAudioUplink({ sendBinary: (frame) => sent.push(frame) });
698
- uplink.reset(1000);
699
-
700
- uplink.offerNearend({ data: new Blob(["near-a"]), timestamp: 1150 });
701
- uplink.offerFarend({ data: new TextEncoder().encode("far"), timestamp: 1140 });
702
- uplink.offerNearend({ data: new Blob(["near-b"]), timestamp: 1170 });
703
- uplink.offerNearend({ data: new Blob(["next"]), timestamp: 1250 });
704
- await uplink.drain();
705
-
706
- assert.deepEqual(sent.map(readAuFrame), [
707
- { magic: "AU", version: 1, streamId: AU_STREAM_FAREND, tsMs: 140, payload: "far" },
708
- {
709
- magic: "AU",
710
- version: 1,
711
- streamId: AU_STREAM_NEAREND,
712
- tsMs: 150,
713
- payload: "near-anear-b",
714
- },
715
- { magic: "AU", version: 1, streamId: AU_STREAM_NEAREND, tsMs: 250, payload: "next" },
716
- ]);
717
- });
718
-
719
- test("AgentAudioUplink matches farend from the preceding 100ms bucket", async () => {
720
- const sent = [];
721
- const uplink = new AgentAudioUplink({ sendBinary: (frame) => sent.push(frame) });
722
- uplink.reset(1000);
723
-
724
- uplink.offerFarend({
725
- data: new TextEncoder().encode("far"),
726
- timestamp: 1050,
727
- isFirstChunk: true,
728
- });
729
- uplink.offerNearend({ data: new Blob(["near"]), timestamp: 1170 });
730
- await uplink.drain();
731
-
732
- assert.deepEqual(sent.map(readAuFrame), [
733
- { magic: "AU", version: 1, streamId: AU_STREAM_FAREND, tsMs: 50, payload: "far" },
734
- { magic: "AU", version: 1, streamId: AU_STREAM_NEAREND, tsMs: 170, payload: "near" },
735
- ]);
736
- });
737
-
738
- test("AgentAudioUplink matches a late preceding-bucket farend", async () => {
739
- const sent = [];
740
- const uplink = new AgentAudioUplink({ sendBinary: (frame) => sent.push(frame) });
741
- uplink.reset(1000);
742
-
743
- uplink.offerNearend({ data: new Blob(["near"]), timestamp: 1170 });
744
- uplink.offerFarend({
745
- data: new TextEncoder().encode("far"),
746
- timestamp: 1050,
747
- isFirstChunk: true,
748
- });
749
- await uplink.drain();
750
-
751
- assert.deepEqual(sent.map(readAuFrame), [
752
- { magic: "AU", version: 1, streamId: AU_STREAM_FAREND, tsMs: 50, payload: "far" },
753
- { magic: "AU", version: 1, streamId: AU_STREAM_NEAREND, tsMs: 170, payload: "near" },
754
- ]);
755
- });
756
-
757
- test("AgentAudioUplink keeps nearend open for a delayed preceding-bucket farend", async () => {
758
- const sent = [];
759
- const uplink = new AgentAudioUplink({ sendBinary: (frame) => sent.push(frame) });
760
- uplink.reset(1000);
761
-
762
- // Playback capture timestamps farend one bucket earlier, while its callback
763
- // arrives just after the next nearend block. Keep one extra bucket open so
764
- // this stable ordering does not permanently miss the pairing window.
765
- uplink.offerNearend({ data: new Blob(["near-0"]), timestamp: 1200 });
766
- uplink.offerNearend({ data: new Blob(["near-1"]), timestamp: 1300 });
767
- uplink.offerFarend({
768
- data: new TextEncoder().encode("far-0"),
769
- timestamp: 1100,
770
- isFirstChunk: true,
771
- });
772
- uplink.offerNearend({ data: new Blob(["near-2"]), timestamp: 1400 });
773
- uplink.offerFarend({
774
- data: new TextEncoder().encode("far-1"),
775
- timestamp: 1200,
776
- isFirstChunk: false,
777
- });
778
- uplink.offerNearend({ data: new Blob(["near-3"]), timestamp: 1500 });
779
- await uplink.drain();
780
-
781
- assert.deepEqual(sent.map(readAuFrame), [
782
- { magic: "AU", version: 1, streamId: AU_STREAM_FAREND, tsMs: 100, payload: "far-0" },
783
- { magic: "AU", version: 1, streamId: AU_STREAM_NEAREND, tsMs: 200, payload: "near-0" },
784
- { magic: "AU", version: 1, streamId: AU_STREAM_FAREND, tsMs: 200, payload: "far-1" },
785
- { magic: "AU", version: 1, streamId: AU_STREAM_NEAREND, tsMs: 300, payload: "near-1" },
786
- { magic: "AU", version: 1, streamId: AU_STREAM_NEAREND, tsMs: 400, payload: "near-2" },
787
- { magic: "AU", version: 1, streamId: AU_STREAM_NEAREND, tsMs: 500, payload: "near-3" },
788
- ]);
789
- });
790
-
791
- test("AgentAudioUplink coalesces a WebM magic split across MediaRecorder blobs", async () => {
792
- const sent = [];
793
- const uplink = new AgentAudioUplink({ sendBinary: (frame) => sent.push(frame) });
794
- uplink.reset(1000);
795
-
796
- uplink.offerNearend({ data: new Uint8Array([0x1a]), timestamp: 1100 });
797
- uplink.offerNearend({ data: new Uint8Array([0x45, 0xdf]), timestamp: 1200 });
798
- uplink.offerNearend({ data: new Uint8Array([0xa3, 0x9f, 0x42, 0x86]), timestamp: 1300 });
799
- await uplink.drain();
800
-
801
- assert.equal(sent.length, 1);
802
- const bytes = new Uint8Array(sent[0]);
803
- assert.equal(bytes.byteLength, 15);
804
- assert.equal(new DataView(sent[0]).getUint32(4, false), 300);
805
- assert.deepEqual([...bytes.subarray(8)], [0x1a, 0x45, 0xdf, 0xa3, 0x9f, 0x42, 0x86]);
806
- });
807
-
808
- test("AgentAudioUplink keeps the farend WebM header when it precedes the paired bucket", async () => {
809
- const sent = [];
810
- const uplink = new AgentAudioUplink({ sendBinary: (frame) => sent.push(frame) });
811
- uplink.reset(1000);
812
-
813
- uplink.offerFarend({
814
- data: new Uint8Array([0x1a, 0x45, 0xdf, 0xa3, 0x9f]),
815
- timestamp: 1100,
816
- isFirstChunk: true,
817
- });
818
- uplink.offerNearend({ data: new Blob(["near"]), timestamp: 1200 });
819
- uplink.offerFarend({
820
- data: new Uint8Array([0x40, 0xaa, 0x81, 0x00]),
821
- timestamp: 1200,
822
- isFirstChunk: false,
823
- });
824
- await uplink.drain();
825
-
826
- assert.equal(sent.length, 2);
827
- const farend = new Uint8Array(sent[0]);
828
- assert.equal(farend[3], AU_STREAM_FAREND);
829
- assert.equal(new DataView(sent[0]).getUint32(4, false), 200);
830
- assert.deepEqual(
831
- [...farend.subarray(8)],
832
- [0x1a, 0x45, 0xdf, 0xa3, 0x9f, 0x40, 0xaa, 0x81, 0x00],
833
- );
834
- assert.deepEqual(readAuFrame(sent[1]), {
835
- magic: "AU",
836
- version: 1,
837
- streamId: AU_STREAM_NEAREND,
838
- tsMs: 200,
839
- payload: "near",
840
- });
841
- });
842
-
843
- test("AgentAudioUplink drops farend continuation until a WebM header is observed", async () => {
844
- const sent = [];
845
- const uplink = new AgentAudioUplink({ sendBinary: (frame) => sent.push(frame) });
846
- uplink.reset(1000);
847
-
848
- uplink.offerFarend({
849
- data: new Uint8Array([0x40, 0xaa, 0x81, 0x00]),
850
- timestamp: 1100,
851
- isFirstChunk: false,
852
- });
853
- uplink.offerNearend({ data: new Blob(["near"]), timestamp: 1100 });
854
- await uplink.drain();
855
-
856
- assert.deepEqual(sent.map(readAuFrame), [
857
- { magic: "AU", version: 1, streamId: AU_STREAM_NEAREND, tsMs: 100, payload: "near" },
858
- ]);
859
- });
860
-
861
- test("AgentAudioUplink never sends a delayed timestamp after a newer frame", async () => {
862
- const sent = [];
863
- const uplink = new AgentAudioUplink({ sendBinary: (frame) => sent.push(frame) });
864
- uplink.reset(1000);
865
-
866
- uplink.enqueue([{
867
- data: new Blob(["newer"]),
868
- timestamp: 1200,
869
- streamId: AU_STREAM_NEAREND,
870
- tsMs: 200,
871
- frameKey: 2,
872
- }]);
873
- uplink.enqueue([{
874
- data: new Blob(["delayed-older"]),
875
- timestamp: 1100,
876
- streamId: AU_STREAM_NEAREND,
877
- tsMs: 100,
878
- frameKey: 1,
879
- }]);
880
- await uplink.drain();
881
-
882
- assert.deepEqual(sent.map(readAuFrame), [
883
- { magic: "AU", version: 1, streamId: AU_STREAM_NEAREND, tsMs: 200, payload: "newer" },
884
- ]);
885
- });
886
-
887
- test("AgentAudioUplink drops farend-only and stale-connection frames", async () => {
888
- const sent = [];
889
- const errors = [];
890
- let releasePayload;
891
- const delayedPayload = new Blob(["stale"]);
892
- delayedPayload.arrayBuffer = () => new Promise((_, reject) => {
893
- releasePayload = () => reject(new Error("stale payload failed"));
894
- });
895
- const uplink = new AgentAudioUplink({
896
- sendBinary: (frame) => sent.push(frame),
897
- onError: (error) => errors.push(error),
898
- });
899
- uplink.reset(1000);
900
- uplink.offerFarend({ data: new TextEncoder().encode("far-only"), timestamp: 1100 });
901
- uplink.offerNearend({ data: delayedPayload, timestamp: 1300 });
902
- const staleDrain = uplink.drain();
903
- await waitFor(() => typeof releasePayload === "function");
904
- uplink.reset(2000);
905
- releasePayload();
906
- await staleDrain;
907
- uplink.offerNearend({ data: new Blob(["current"]), timestamp: 2000 });
908
- await uplink.drain();
909
-
910
- assert.deepEqual(sent.map(readAuFrame), [
911
- { magic: "AU", version: 1, streamId: AU_STREAM_NEAREND, tsMs: 0, payload: "current" },
912
- ]);
913
- assert.deepEqual(errors, []);
914
- });
915
-
916
- test("AgentAudioUplink fails closed until the next audio-input session", async () => {
917
- const sent = [];
918
- const errors = [];
919
- let sendAttempts = 0;
920
- let shouldFail = true;
921
- const uplink = new AgentAudioUplink({
922
- sendBinary: (frame) => {
923
- sendAttempts += 1;
924
- if (shouldFail) {
925
- throw new Error("backpressure");
926
- }
927
- sent.push(frame);
928
- },
929
- onError: (error) => errors.push(error),
930
- });
931
- uplink.reset(1000);
932
-
933
- uplink.offerNearend({ data: new Blob(["first"]), timestamp: 1000 });
934
- uplink.offerNearend({ data: new Blob(["already-queued"]), timestamp: 1100 });
935
- await uplink.drain();
936
- shouldFail = false;
937
- uplink.offerNearend({ data: new Blob(["dropped"]), timestamp: 1200 });
938
- await uplink.drain();
939
- assert.equal(errors.length, 1);
940
- assert.equal(sendAttempts, 1);
941
- assert.deepEqual(sent, []);
942
-
943
- uplink.startInput();
944
- uplink.offerNearend({ data: new Blob(["next"]), timestamp: 1300 });
945
- await uplink.drain();
946
- assert.deepEqual(sent.map(readAuFrame), [
947
- { magic: "AU", version: 1, streamId: AU_STREAM_NEAREND, tsMs: 300, payload: "next" },
948
- ]);
949
- });
950
-
951
- test("AgentE2EClient connects with token, sends JSON, and dispatches events", async () => {
952
- MockWebSocket.instances = [];
953
- const events = [];
954
- const client = new AgentE2EClient({
955
- wsUrl: "wss://e2e.example.com/ws",
956
- token: "abc",
957
- WebSocketCtor: MockWebSocket,
958
- onEvent: (event) => events.push(event),
959
- });
960
-
961
- await client.connect();
962
- const ws = MockWebSocket.instances[0];
963
- assert.equal(ws.url, "wss://e2e.example.com/ws?token=abc");
964
-
965
- client.send({ type: "ask", message: { text: "hello" } });
966
- assert.deepEqual(JSON.parse(ws.sent[0]), { type: "ask", message: { text: "hello" } });
967
-
968
- ws.emit({ type: "pong" });
969
- assert.deepEqual(events, [{ type: "pong" }]);
970
- });
971
-
972
- test("AgentE2EClient serializes concurrent connection attempts", async () => {
973
- MockWebSocket.instances = [];
974
- MockWebSocket.failedConnections = 0;
975
- const client = new AgentE2EClient({
976
- wsUrl: "wss://e2e.example.com/ws",
977
- token: "token",
978
- WebSocketCtor: MockWebSocket,
979
- });
980
-
981
- const first = client.connect();
982
- const second = client.connect();
983
- await Promise.all([first, second]);
984
-
985
- assert.equal(MockWebSocket.instances.length, 1);
986
- client.disconnect();
987
- });
988
-
989
- test("AgentE2EClient can reconnect immediately after cancelling a pending connection", async () => {
990
- MockWebSocket.instances = [];
991
- MockWebSocket.failedConnections = 0;
992
- const client = new AgentE2EClient({
993
- wsUrl: "wss://e2e.example.com/ws",
994
- token: "token",
995
- WebSocketCtor: MockWebSocket,
996
- });
997
-
998
- const cancelledConnection = client.connect();
999
- client.disconnect();
1000
- const nextConnection = client.connect();
1001
-
1002
- await assert.rejects(cancelledConnection, /连接已取消/);
1003
- await nextConnection;
1004
- assert.equal(MockWebSocket.instances.length, 2);
1005
- client.disconnect();
1006
- });
1007
-
1008
- test("AgentE2EClient sends ping and dispatches pong", async () => {
1009
- MockWebSocket.instances = [];
1010
- const events = [];
1011
- const client = new AgentE2EClient({
1012
- wsUrl: "wss://e2e.example.com/ws",
1013
- token: "token",
1014
- WebSocketCtor: MockWebSocket,
1015
- onEvent: (event) => events.push(event),
1016
- });
1017
-
1018
- await client.connect();
1019
- const ws = MockWebSocket.instances[0];
1020
- client.ping();
1021
- ws.emit({ type: "pong" });
1022
-
1023
- assert.deepEqual(JSON.parse(ws.sent[0]), { type: "ping" });
1024
- assert.deepEqual(events, [{ type: "pong" }]);
1025
- client.disconnect();
1026
- });
1027
-
1028
- test("AgentE2EClient rejects sends before connecting", () => {
1029
- const client = new AgentE2EClient({
1030
- wsUrl: "wss://e2e.example.com/ws",
1031
- token: "token",
1032
- WebSocketCtor: MockWebSocket,
1033
- });
1034
-
1035
- assert.throws(() => client.send({ type: "ping" }), /E2E WebSocket 未连接/);
1036
- assert.throws(() => client.sendBinary(new Blob(["audio"])), /E2E WebSocket 未连接/);
1037
- assert.doesNotThrow(() => client.disconnect());
1038
- });
1039
-
1040
- test("AgentE2EClient reports invalid JSON and ignores non-text messages", async () => {
1041
- MockWebSocket.instances = [];
1042
- const events = [];
1043
- const errors = [];
1044
- const client = new AgentE2EClient({
1045
- wsUrl: "wss://e2e.example.com/ws",
1046
- token: "token",
1047
- WebSocketCtor: MockWebSocket,
1048
- onEvent: (event) => events.push(event),
1049
- onError: (error) => errors.push(error),
1050
- });
1051
-
1052
- await client.connect();
1053
- const ws = MockWebSocket.instances[0];
1054
- ws.emitRaw("not-json");
1055
- ws.emitRaw(new Blob(["binary"]));
1056
- ws.emitRaw(JSON.stringify({ message: "missing type" }));
1057
-
1058
- assert.deepEqual(events, []);
1059
- assert.equal(errors.length, 1);
1060
- assert.match(errors[0].message, /不是合法 JSON/);
1061
- client.disconnect();
1062
- });
1063
-
1064
- test("AgentE2EClient rejects Binary frames when the socket buffer is over limit", async () => {
1065
- MockWebSocket.instances = [];
1066
- const client = new AgentE2EClient({
1067
- wsUrl: "wss://e2e.example.com/ws",
1068
- token: "token",
1069
- WebSocketCtor: MockWebSocket,
1070
- });
1071
-
1072
- await client.connect();
1073
- const ws = MockWebSocket.instances[0];
1074
- ws.bufferedAmount = 2 * 1024 * 1024 + 1;
1075
-
1076
- assert.throws(
1077
- () => client.sendBinary(new Blob(["audio"])),
1078
- (error) => error.name === "WebSocketBackpressureError" && /缓冲区超限/.test(error.message),
1079
- );
1080
- assert.deepEqual(ws.sent, []);
1081
- client.disconnect();
1082
- });
1083
-
1084
- test("AgentE2EClient can drop JSON frames when the socket buffer is over limit", async () => {
1085
- MockWebSocket.instances = [];
1086
- const client = new AgentE2EClient({
1087
- wsUrl: "wss://e2e.example.com/ws",
1088
- token: "token",
1089
- WebSocketCtor: MockWebSocket,
1090
- });
1091
-
1092
- await client.connect();
1093
- const ws = MockWebSocket.instances[0];
1094
- ws.bufferedAmount = 2 * 1024 * 1024 + 1;
1095
-
1096
- assert.equal(client.trySend({ type: "telemetry" }), false);
1097
- assert.deepEqual(ws.sent, []);
1098
- client.disconnect();
1099
- });
1100
-
1101
- test("AgentE2EClient preserves a server URL that already contains its token", async () => {
1102
- MockWebSocket.instances = [];
1103
- const wsUrl = "ws://test-e2emp.xingyun3d.com:8000/ws/session?session_id=test&token=server-token";
1104
- const client = new AgentE2EClient({
1105
- wsUrl,
1106
- token: "separate-token",
1107
- WebSocketCtor: MockWebSocket,
1108
- });
1109
-
1110
- await client.connect();
1111
- assert.equal(MockWebSocket.instances[0].url, wsUrl);
1112
- client.disconnect();
1113
- });
1114
-
1115
- test("AgentE2EClient adds audio_input_off for a disabled initial audio input", async () => {
1116
- MockWebSocket.instances = [];
1117
- const client = new AgentE2EClient({
1118
- wsUrl: "wss://e2e.example.com/ws/session?session_id=test&token=server-token",
1119
- token: "separate-token",
1120
- audioInputEnabled: false,
1121
- WebSocketCtor: MockWebSocket,
1122
- });
1123
-
1124
- await client.connect();
1125
- assert.equal(
1126
- MockWebSocket.instances[0].url,
1127
- "wss://e2e.example.com/ws/session?session_id=test&token=server-token&audio_input_off=1",
1128
- );
1129
- client.disconnect();
1130
- });
1131
-
1132
- test("AgentE2EClient removes stale audio_input_off for an enabled initial audio input", async () => {
1133
- MockWebSocket.instances = [];
1134
- const client = new AgentE2EClient({
1135
- wsUrl: "wss://e2e.example.com/ws/session?session_id=test&audio_input_off=1",
1136
- token: "token",
1137
- audioInputEnabled: true,
1138
- WebSocketCtor: MockWebSocket,
1139
- });
1140
-
1141
- await client.connect();
1142
- assert.equal(MockWebSocket.instances[0].url, "wss://e2e.example.com/ws/session?session_id=test&token=token");
1143
- client.disconnect();
1144
- });
1145
-
1146
- test("AgentE2EClient uses the latest audio input state when reconnecting", async () => {
1147
- MockWebSocket.instances = [];
1148
- const client = new AgentE2EClient({
1149
- wsUrl: "wss://e2e.example.com/ws/session?session_id=test",
1150
- token: "token",
1151
- audioInputEnabled: false,
1152
- WebSocketCtor: MockWebSocket,
1153
- });
1154
-
1155
- await client.connect();
1156
- assert.match(MockWebSocket.instances[0].url, /[?&]audio_input_off=1(?:&|$)/);
1157
- client.setAudioInputEnabled(true);
1158
- client.disconnect();
1159
- await client.connect();
1160
- assert.equal(MockWebSocket.instances[1].url, "wss://e2e.example.com/ws/session?session_id=test&token=token");
1161
- client.disconnect();
1162
- });
1163
-
1164
- test("XingyunAvatarAgent initializes E2E from the unified session e2e_resp", async () => {
1165
- MockWebSocket.instances = [];
1166
- const { agent } = createAgent();
1167
- agent.sessionInfo = {
1168
- session_id: "gateway-session",
1169
- room: "gateway-room",
1170
- token: "gateway-token",
1171
- socket_io_url: "wss://gateway.example.com/socket.io",
1172
- e2e_resp: {
1173
- ws_url: "wss://e2e.example.com/ws/session?session_id=gateway-session",
1174
- e2e_token: "unified-e2e-token",
1175
- },
1176
- };
1177
-
1178
- try {
1179
- const sessionInfo = await agent.init();
1180
- assert.equal(sessionInfo, agent.sessionInfo);
1181
- assert.equal(
1182
- MockWebSocket.instances[0].url,
1183
- "wss://e2e.example.com/ws/session?session_id=gateway-session&token=unified-e2e-token&audio_input_off=1",
1184
- );
1185
- } finally {
1186
- await agent.destroy("test");
1187
- }
1188
- });
1189
-
1190
- test("XingyunAvatarAgent enables dual audio from the returned AEC feature", async () => {
1191
- MockWebSocket.instances = [];
1192
- const { agent } = createAgent();
1193
- agent.sessionInfo = {
1194
- ...agent.sessionInfo,
1195
- features: {
1196
- speech_frontend: { enable_aec: false },
1197
- },
1198
- e2e_resp: {
1199
- ...agent.sessionInfo.e2e_resp,
1200
- features: {
1201
- speech_frontend: { enable_aec: true },
1202
- },
1203
- },
1204
- };
1205
-
1206
- try {
1207
- await agent.init();
1208
- assert.equal(typeof agent.avatarOptions.onAudioPlaybackData, "function");
1209
- assert.equal(agent.audioCaptureEnableCalls, 1);
1210
-
1211
- const callback = agent.avatarOptions.onAudioPlaybackData;
1212
- const timestamp = agent.audioUplink.connectionStartedAt;
1213
- agent.microphone = { isRecording: true, async stop() {} };
1214
- callback({
1215
- data: new TextEncoder().encode("feature-enabled").buffer,
1216
- codec: "opus",
1217
- isFirstChunk: true,
1218
- sampleRate: 16000,
1219
- samples: 1600,
1220
- timestamp,
1221
- speech_id: 1,
1222
- });
1223
- agent.sendAudioFrame({ data: new Blob(["nearend"]), timestamp });
1224
- await agent.audioUplink.drain();
1225
- assert.deepEqual(MockWebSocket.instances[0].sent.map(readAuFrame), [
1226
- { magic: "AU", version: 1, streamId: AU_STREAM_FAREND, tsMs: 0, payload: "feature-enabled" },
1227
- { magic: "AU", version: 1, streamId: AU_STREAM_NEAREND, tsMs: 0, payload: "nearend" },
1228
- ]);
1229
- } finally {
1230
- await agent.destroy("test");
1231
- }
1232
- });
1233
-
1234
- test("XingyunAvatarAgent keeps requested AEC when the session does not echo features", async () => {
1235
- MockWebSocket.instances = [];
1236
- const { agent } = createAgent({}, {
1237
- features: {
1238
- speech_frontend: { enable_aec: true },
1239
- },
1240
- });
1241
-
1242
- try {
1243
- await agent.init();
1244
- assert.equal(typeof agent.avatarOptions.onAudioPlaybackData, "function");
1245
- const timestamp = Date.now();
1246
- agent.microphone = { isRecording: true, async stop() {} };
1247
- agent.sendAudioFrame({ data: new Blob(["nearend"]), timestamp });
1248
- await agent.audioUplink.drain();
1249
-
1250
- assert.deepEqual(readAuFrame(MockWebSocket.instances[0].sent[0]), {
1251
- magic: "AU",
1252
- version: 1,
1253
- streamId: AU_STREAM_NEAREND,
1254
- tsMs: 0,
1255
- payload: "nearend",
1256
- });
1257
- } finally {
1258
- await agent.destroy("test");
1259
- }
1260
- });
1261
-
1262
- test("XingyunAvatarAgent lets the returned false AEC feature disable the legacy audio option", async () => {
1263
- MockWebSocket.instances = [];
1264
- const { agent } = createAgent({}, {
1265
- audio: { echoCancellationEnabled: true },
1266
- features: {
1267
- speech_frontend: { enable_aec: true },
1268
- },
1269
- });
1270
- agent.sessionInfo = {
1271
- ...agent.sessionInfo,
1272
- features: {
1273
- speech_frontend: { enable_aec: false },
1274
- },
1275
- };
1276
-
1277
- try {
1278
- await agent.init();
1279
- assert.equal(agent.avatarOptions.onAudioPlaybackData, undefined);
1280
- assert.equal(agent.audioCaptureEnableCalls, 0);
1281
- const rawNearend = new Blob(["raw-nearend"], { type: "audio/webm;codecs=opus" });
1282
- agent.sendAudioFrame({ data: rawNearend, timestamp: Date.now() });
1283
- assert.equal(MockWebSocket.instances[0].sent[0], rawNearend);
1284
- } finally {
1285
- await agent.destroy("test");
1286
- }
1287
- });
1288
-
1289
- test("XingyunAvatarAgent init starts exactly once when TTSA is ready before E2E", async () => {
1290
- MockWebSocket.instances = [];
1291
- const states = [];
1292
- const { agent } = createAgent({
1293
- onAgentStateChange: (state) => states.push(state),
1294
- });
1295
-
1296
- await agent.init();
1297
-
1298
- assert.equal(agent.getAgentState(), "running");
1299
- assert.equal(agent.started, true);
1300
- assert.equal(agent.startCalls, 1);
1301
- assert.deepEqual(states, ["initializing", "ready", "running"]);
1302
- await agent.destroy("test");
1303
- });
1304
-
1305
- test("XingyunAvatarAgent init waits for TTSA when E2E is ready first", async () => {
1306
- MockWebSocket.instances = [];
1307
- const { agent } = createAgent();
1308
- agent.autoTtsaReady = false;
1309
- let initSettled = false;
1310
-
1311
- const initPromise = agent.init().finally(() => {
1312
- initSettled = true;
1313
- });
1314
- await waitFor(() => MockWebSocket.instances[0]?.readyState === MockWebSocket.OPEN);
1315
-
1316
- assert.equal(initSettled, false);
1317
- assert.equal(agent.getAgentState(), "initializing");
1318
- agent.onTtsaReady();
1319
- await initPromise;
1320
-
1321
- assert.equal(agent.getAgentState(), "running");
1322
- assert.equal(agent.started, true);
1323
- assert.equal(agent.startCalls, 1);
1324
- await agent.destroy("test");
1325
- });
1326
-
1327
- test("XingyunAvatarAgent shares one in-flight initialization", async () => {
1328
- MockWebSocket.instances = [];
1329
- const { agent } = createAgent();
1330
- agent.autoTtsaReady = false;
1331
-
1332
- const firstInit = agent.init();
1333
- const secondInit = agent.init();
1334
-
1335
- assert.equal(secondInit, firstInit);
1336
- await waitFor(() => MockWebSocket.instances[0]?.readyState === MockWebSocket.OPEN);
1337
- assert.equal(MockWebSocket.instances.length, 1);
1338
- agent.onTtsaReady();
1339
- await firstInit;
1340
- assert.equal(agent.startCalls, 1);
1341
- await agent.destroy("test");
1342
- });
1343
-
1344
- test("XingyunAvatarAgent keeps initial init pending across a TTSA session reload", async () => {
1345
- MockWebSocket.instances = [];
1346
- const { agent } = createAgent();
1347
- agent.autoTtsaReady = false;
1348
- const initPromise = agent.init();
1349
-
1350
- await waitFor(() => MockWebSocket.instances[0]?.readyState === MockWebSocket.OPEN);
1351
- await agent.onSessionReloaded({
1352
- session_id: "reloaded-session",
1353
- e2e_resp: {
1354
- ws_url: "wss://e2e.example.com/ws/session?session_id=reloaded-session",
1355
- e2e_token: "reloaded-token",
1356
- },
1357
- });
1358
- assert.equal(agent.getAgentState(), "initializing");
1359
- assert.equal(MockWebSocket.instances.length, 2);
1360
-
1361
- agent.onTtsaReady();
1362
- const sessionInfo = await initPromise;
1363
- assert.equal(sessionInfo.session_id, "reloaded-session");
1364
- assert.equal(agent.getAgentState(), "running");
1365
- assert.equal(agent.startCalls, 1);
1366
- await agent.destroy("test");
1367
- });
1368
-
1369
- test("XingyunAvatarAgent requires TTSA readiness from the latest initial session", async () => {
1370
- MockWebSocket.instances = [];
1371
- const { agent } = createAgent();
1372
- agent.autoTtsaReady = false;
1373
- let initSettled = false;
1374
- const initPromise = agent.init().finally(() => {
1375
- initSettled = true;
1376
- });
1377
-
1378
- await waitFor(() => MockWebSocket.instances[0]?.readyState === MockWebSocket.OPEN);
1379
- agent.onTtsaReady({ session_id: "test-session" });
1380
- await agent.onSessionReloaded({
1381
- session_id: "replacement-session",
1382
- e2e_resp: {
1383
- ws_url: "wss://e2e.example.com/ws/session?session_id=replacement-session",
1384
- e2e_token: "replacement-token",
1385
- },
1386
- });
1387
- await new Promise((resolve) => setTimeout(resolve, 0));
1388
-
1389
- assert.equal(initSettled, false);
1390
- agent.onTtsaReady({ session_id: "test-session" });
1391
- await new Promise((resolve) => setTimeout(resolve, 0));
1392
- assert.equal(initSettled, false);
1393
-
1394
- agent.onTtsaReady({ session_id: "replacement-session" });
1395
- const sessionInfo = await initPromise;
1396
- assert.equal(sessionInfo.session_id, "replacement-session");
1397
- assert.equal(agent.getAgentState(), "running");
1398
- assert.equal(agent.startCalls, 1);
1399
- await agent.destroy("test");
1400
- });
1401
-
1402
- test("XingyunAvatarAgent rejects init when the latest TTSA session reload is exhausted", async () => {
1403
- MockWebSocket.instances = [];
1404
- const { agent } = createAgent();
1405
- agent.autoTtsaReady = false;
1406
- const initPromise = agent.init();
1407
-
1408
- await waitFor(() => MockWebSocket.instances[0]?.readyState === MockWebSocket.OPEN);
1409
- agent.onTtsaReady({ session_id: "test-session" });
1410
- await agent.onSessionReloaded({
1411
- session_id: "replacement-session",
1412
- e2e_resp: {
1413
- ws_url: "wss://e2e.example.com/ws/session?session_id=replacement-session",
1414
- e2e_token: "replacement-token",
1415
- },
1416
- });
1417
- agent.onSessionReloadExhausted(new Error("TTSA reload exhausted"));
1418
-
1419
- await assert.rejects(initPromise, /TTSA reload exhausted/);
1420
- assert.equal(agent.getAgentState(), "failed");
1421
- assert.equal(agent.startCalls, 0);
1422
- });
1423
-
1424
- test("XingyunAvatarAgent supersedes an in-flight E2E attempt during initial session reload", async () => {
1425
- MockWebSocket.instances = [];
1426
- MockWebSocket.autoOpen = false;
1427
- const { agent } = createAgent();
1428
- agent.autoTtsaReady = false;
1429
- const initPromise = agent.init();
1430
-
1431
- try {
1432
- await waitFor(() => MockWebSocket.instances.length === 1);
1433
- const reloadPromise = agent.onSessionReloaded({
1434
- session_id: "replacement-session",
1435
- e2e_resp: {
1436
- ws_url: "wss://e2e.example.com/ws/session?session_id=replacement-session",
1437
- e2e_token: "replacement-token",
1438
- },
1439
- });
1440
- await waitFor(() => MockWebSocket.instances.length === 2);
1441
- MockWebSocket.instances[1].open();
1442
- await reloadPromise;
1443
- agent.onTtsaReady();
1444
-
1445
- const sessionInfo = await initPromise;
1446
- assert.equal(sessionInfo.session_id, "replacement-session");
1447
- assert.equal(agent.getAgentState(), "running");
1448
- assert.equal(agent.startCalls, 1);
1449
- } finally {
1450
- MockWebSocket.autoOpen = true;
1451
- await agent.destroy("test");
1452
- }
1453
- });
1454
-
1455
- test("XingyunAvatarAgent init rejects if E2E closes while waiting for TTSA", async () => {
1456
- MockWebSocket.instances = [];
1457
- const { agent } = createAgent();
1458
- agent.autoTtsaReady = false;
1459
- const initPromise = agent.init();
1460
-
1461
- try {
1462
- await waitFor(() => MockWebSocket.instances[0]?.readyState === MockWebSocket.OPEN);
1463
- MockWebSocket.instances[0].closeRemote(4001, "invalid token", true);
1464
-
1465
- await assert.rejects(
1466
- Promise.race([
1467
- initPromise,
1468
- new Promise((_, reject) => setTimeout(() => reject(new Error("init timeout")), 50)),
1469
- ]),
1470
- /Agent 初始化期间 E2E WebSocket 已关闭/,
1471
- );
1472
- } finally {
1473
- await agent.destroy("test");
1474
- }
1475
- });
1476
-
1477
- test("XingyunAvatarAgent funnels an initial E2E connection failure through shared initialization cleanup", async () => {
1478
- MockWebSocket.instances = [];
1479
- MockWebSocket.failedConnections = 1;
1480
- const legacyErrors = [];
1481
- const agentErrors = [];
1482
- const { agent } = createAgent(
1483
- { onError: (error) => agentErrors.push(error) },
1484
- { onMessage: (error) => legacyErrors.push(error) },
1485
- );
1486
-
1487
- try {
1488
- await assert.rejects(agent.init(), /E2E WebSocket/);
1489
-
1490
- assert.equal(agent.getAgentState(), "failed");
1491
- assert.deepEqual(agent.incompleteInitializationCleanups, ["agent_init_failed"]);
1492
- assert.equal(agent.initializationFailureNotifications.length, 1);
1493
- assert.equal(legacyErrors.length, 1);
1494
- assert.equal(legacyErrors[0].code, 10005);
1495
- assert.deepEqual(agentErrors.map((error) => error.code), ["AGENT_INIT_FAILED"]);
1496
- } finally {
1497
- MockWebSocket.failedConnections = 0;
1498
- await agent.destroy("test");
1499
- }
1500
- });
1501
-
1502
- test("XingyunAvatarAgent funnels a unified session failure through shared initialization cleanup", async () => {
1503
- const legacyErrors = [];
1504
- const agentErrors = [];
1505
- const { agent } = createAgent(
1506
- { onError: (error) => agentErrors.push(error) },
1507
- { onMessage: (error) => legacyErrors.push(error) },
1508
- );
1509
- agent.sessionInfo = null;
1510
-
1511
- await assert.rejects(agent.init(), /未返回 sessionInfo/);
1512
-
1513
- assert.equal(agent.getAgentState(), "failed");
1514
- assert.deepEqual(agent.incompleteInitializationCleanups, ["agent_init_failed"]);
1515
- assert.equal(agent.initializationFailureNotifications.length, 1);
1516
- assert.equal(legacyErrors.length, 1);
1517
- assert.equal(legacyErrors[0].code, 10005);
1518
- assert.deepEqual(agentErrors.map((error) => error.code), ["AGENT_INIT_FAILED"]);
1519
-
1520
- await agent.destroy("test");
1521
- assert.equal(agent.getAgentState(), "destroyed");
1522
- });
1523
-
1524
- test("XingyunAvatarAgent preserves parent diagnostics during initialization", async () => {
1525
- const agentErrors = [];
1526
- const { agent } = createAgent({ onError: (error) => agentErrors.push(error) });
1527
- Object.defineProperty(agent, "sessionInfo", {
1528
- configurable: true,
1529
- get() {
1530
- agent.avatarOptions.onMessage({ code: "SESSION_DETAIL", message: "session detail" });
1531
- return null;
1532
- },
1533
- });
1534
-
1535
- await assert.rejects(agent.init(), /未返回 sessionInfo/);
1536
-
1537
- assert.deepEqual(agentErrors.map((error) => error.code), [
1538
- "SESSION_DETAIL",
1539
- "AGENT_INIT_FAILED",
1540
- ]);
1541
- });
1542
-
1543
- test("XingyunAvatarAgent preserves its initialization failure when callbacks throw", async () => {
1544
- const agentErrors = [];
1545
- const { agent } = createAgent(
1546
- {
1547
- onAgentStateChange(state) {
1548
- if (state === "failed") {
1549
- throw new Error("state callback failed");
1550
- }
1551
- },
1552
- onError(error) {
1553
- agentErrors.push(error);
1554
- throw new Error("error callback failed");
1555
- },
1556
- },
1557
- {
1558
- onMessage() {
1559
- throw new Error("legacy callback failed");
1560
- },
1561
- },
1562
- );
1563
- agent.sessionInfo = null;
1564
-
1565
- await assert.rejects(agent.init(), /未返回 sessionInfo/);
1566
-
1567
- assert.equal(agent.getAgentState(), "failed");
1568
- assert.deepEqual(agentErrors.map((error) => error.code), ["AGENT_INIT_FAILED"]);
1569
- });
1570
-
1571
- test("XingyunAvatarAgent rejects a partial unified session response", async () => {
1572
- const { agent } = createAgent();
1573
- agent.sessionInfo = {
1574
- session_id: "gateway-session",
1575
- room: "gateway-room",
1576
- token: "gateway-token",
1577
- socket_io_url: "wss://gateway.example.com/socket.io",
1578
- e2e_resp: {
1579
- ws_url: "wss://e2e.example.com/ws/session?session_id=gateway-session",
1580
- },
1581
- };
1582
-
1583
- await assert.rejects(agent.init(), /e2e_resp 必须同时返回 ws_url 和 e2e_token/);
1584
- assert.equal(agent.destroyed, true);
1585
- assert.deepEqual(agent.incompleteInitializationCleanups, ["agent_init_failed"]);
1586
- });
1587
-
1588
- test("XingyunAvatarAgent rejects e2e_resp with only an E2E token", async () => {
1589
- const { agent } = createAgent();
1590
- agent.sessionInfo = {
1591
- session_id: "gateway-session",
1592
- room: "gateway-room",
1593
- token: "gateway-token",
1594
- socket_io_url: "wss://gateway.example.com/socket.io",
1595
- e2e_resp: {
1596
- e2e_token: "unified-e2e-token",
1597
- },
1598
- };
1599
-
1600
- await assert.rejects(agent.init(), /e2e_resp 必须同时返回 ws_url 和 e2e_token/);
1601
- assert.equal(agent.destroyed, true);
1602
- assert.deepEqual(agent.incompleteInitializationCleanups, ["agent_init_failed"]);
1603
- });
1604
-
1605
- test("XingyunAvatarAgent rejects an incomplete direct E2E fallback", async () => {
1606
- const { agent } = createAgent({}, { e2eServer: undefined });
1607
-
1608
- await assert.rejects(agent.init(), /e2eServer 和 authToken 必须同时配置/);
1609
- assert.equal(agent.destroyed, true);
1610
- });
1611
-
1612
- test("XingyunAvatarAgent fails when unified session and fallbacks are unavailable", async () => {
1613
- const { agent } = createAgent({}, { e2eServer: undefined, authToken: undefined });
1614
-
1615
- await assert.rejects(agent.init(), /未返回 e2e_resp\.ws_url,且未配置 e2eServer/);
1616
- assert.equal(agent.destroyed, true);
1617
- });
1618
-
1619
- test("XingyunAvatarAgent defaults session_speak_req_id without overwriting zero", () => {
1620
- const { agent: defaultAgent } = createAgent();
1621
- const { agent: zeroAgent } = createAgent({}, { session_speak_req_id: 0 });
1622
-
1623
- assert.equal(defaultAgent.avatarOptions.session_speak_req_id, 1);
1624
- assert.equal(zeroAgent.avatarOptions.session_speak_req_id, 0);
1625
- });
1626
-
1627
- test("XingyunAvatarAgent accepts JSON Config sources", () => {
1628
- const { agent } = createAgent({}, {
1629
- asr_id: undefined,
1630
- asr_config: { provider: "asr-custom" },
1631
- tts_config: { provider: "tts-custom" },
1632
- features: {
1633
- anti_interference: { semantic_judge_enabled: false },
1634
- speech_frontend: {
1635
- enabled: false,
1636
- enable_aec: false,
1637
- enable_speech_separation: false,
1638
- },
1639
- vad_merge_mode: true,
1640
- },
1641
- extras: { trace_context: "agent-test" },
1642
- llm_id: undefined,
1643
- brain_config: { provider: "brain-custom" },
1644
- });
1645
-
1646
- assert.deepEqual(agent.avatarOptions.sessionRequestData, {
1647
- asr_config: { provider: "asr-custom" },
1648
- tts_config: { provider: "tts-custom" },
1649
- features: {
1650
- anti_interference: { semantic_judge_enabled: false },
1651
- speech_frontend: {
1652
- enabled: false,
1653
- enable_aec: false,
1654
- enable_speech_separation: false,
1655
- },
1656
- vad_merge_mode: true,
1657
- },
1658
- extras: { trace_context: "agent-test" },
1659
- brain_config: { provider: "brain-custom" },
1660
- session_speak_req_id: 1,
1661
- });
1662
- });
1663
-
1664
- test("XingyunAvatarAgent omits empty ASR, TTS, Features, Extras and Brain configs", () => {
1665
- const { agent } = createAgent({}, {
1666
- asr_id: undefined,
1667
- asr_config: {},
1668
- tts_config: {},
1669
- features: {},
1670
- extras: {},
1671
- llm_id: undefined,
1672
- brain_config: {},
1673
- });
1674
-
1675
- assert.deepEqual(agent.avatarOptions.sessionRequestData, {
1676
- session_speak_req_id: 1,
1677
- });
1678
- });
1679
-
1680
- test("XingyunAvatarAgent treats configs without serializable fields as empty", () => {
1681
- const { agent } = createAgent({}, {
1682
- asr_id: 3,
1683
- asr_config: { provider: undefined },
1684
- tts_config: { provider: undefined },
1685
- features: { vad_merge_mode: undefined },
1686
- extras: { trace_context: undefined },
1687
- llm_id: 4,
1688
- brain_config: { extra_body: undefined },
1689
- });
1690
-
1691
- assert.deepEqual(agent.avatarOptions.sessionRequestData, {
1692
- asr_id: 3,
1693
- llm_id: 4,
1694
- session_speak_req_id: 1,
1695
- });
1696
- });
1697
-
1698
- test("XingyunAvatarAgent rejects non-object features", () => {
1699
- assert.throws(
1700
- () => createAgent({}, { features: [] as never }),
1701
- /Agent features 必须是对象/,
1702
- );
1703
- });
1704
-
1705
- test("XingyunAvatarAgent rejects non-object TTS config", () => {
1706
- assert.throws(
1707
- () => createAgent({}, { tts_config: [] as never }),
1708
- /Agent TTS JSON Config 必须是对象/,
1709
- );
1710
- });
1711
-
1712
- test("XingyunAvatarAgent rejects non-object extras", () => {
1713
- assert.throws(
1714
- () => createAgent({}, { extras: [] as never }),
1715
- /Agent extras 必须是对象/,
1716
- );
1717
- });
1718
-
1719
- test("XingyunAvatarAgent preserves Brain extra_body", () => {
1720
- const { agent } = createAgent({}, {
1721
- asr_id: undefined,
1722
- asr_config: { provider: "asr-custom" },
1723
- llm_id: undefined,
1724
- brain_config: {
1725
- provider: "openai-compatible",
1726
- extra_body: { thinking: { type: "disabled" } },
1727
- },
1728
- });
1729
-
1730
- assert.deepEqual(agent.avatarOptions.sessionRequestData, {
1731
- asr_config: { provider: "asr-custom" },
1732
- brain_config: {
1733
- provider: "openai-compatible",
1734
- extra_body: { thinking: { type: "disabled" } },
1735
- },
1736
- session_speak_req_id: 1,
1737
- });
1738
- });
1739
-
1740
- test("XingyunAvatarAgent only captures playback audio when AEC or a caller callback needs it", async () => {
1741
- const playbackFrames = [];
1742
- const { agent: defaultAgent } = createAgent();
1743
- const { agent: callbackAgent } = createAgent({}, {
1744
- onAudioPlaybackData: (data) => playbackFrames.push(data),
1745
- });
1746
- const { agent: aecAgent } = createAgent({}, {
1747
- audio: { echoCancellationEnabled: true },
1748
- });
1749
-
1750
- assert.equal(defaultAgent.avatarOptions.onAudioPlaybackData, undefined);
1751
- assert.equal(typeof callbackAgent.avatarOptions.onAudioPlaybackData, "function");
1752
- assert.equal(aecAgent.avatarOptions.onAudioPlaybackData, undefined);
1753
- await aecAgent.init();
1754
- assert.equal(typeof aecAgent.avatarOptions.onAudioPlaybackData, "function");
1755
-
1756
- const frame = { data: new TextEncoder().encode("AAE=").buffer, codec: "opus", isFirstChunk: false, sampleRate: 16000, samples: 2, timestamp: 1, speech_id: 3 };
1757
- callbackAgent.avatarOptions.onAudioPlaybackData(frame);
1758
- assert.deepEqual(playbackFrames, [frame]);
1759
- await aecAgent.destroy("test");
1760
- });
1761
-
1762
- test("XingyunAvatarAgent forwards paired playback and microphone audio as ordered AU frames", async () => {
1763
- MockWebSocket.instances = [];
1764
- const { agent } = createAgent({}, {
1765
- audio: { echoCancellationEnabled: true },
1766
- });
1767
-
1768
- const toBuffer = (s: string) => new TextEncoder().encode(s).buffer;
1769
- await agent.init();
1770
- const callback = agent.avatarOptions.onAudioPlaybackData;
1771
- const ws = MockWebSocket.instances[0];
1772
- agent.microphone = { isRecording: true, async stop() {} };
1773
- const connectionStartedAt = agent.audioUplink.connectionStartedAt;
1774
- const timestamp = connectionStartedAt + 100;
1775
- callback({ data: toBuffer("farend"), codec: "opus", isFirstChunk: true, sampleRate: 16000, samples: 1600, timestamp, speech_id: 1 });
1776
- agent.sendAudioFrame({ data: new Blob(["nearend"]), timestamp });
1777
- await agent.audioUplink.drain();
1778
-
1779
- assert.deepEqual(ws.sent.map(readAuFrame), [
1780
- { magic: "AU", version: 1, streamId: AU_STREAM_FAREND, tsMs: 100, payload: "farend" },
1781
- { magic: "AU", version: 1, streamId: AU_STREAM_NEAREND, tsMs: 100, payload: "nearend" },
1782
- ]);
1783
- await agent.destroy("test");
1784
- });
1785
-
1786
- test("XingyunAvatarAgent restarts playback capture when dual-audio ASR starts", async () => {
1787
- MockWebSocket.instances = [];
1788
- const browser = installMicrophoneBrowser();
1789
- const { agent } = createAgent({}, {
1790
- audio: { echoCancellationEnabled: true },
1791
- });
1792
-
1793
- try {
1794
- await agent.init();
1795
- await agent.startASR();
1796
- assert.equal(agent.audioCaptureRestartCalls, 1);
1797
- await agent.stopASR();
1798
- } finally {
1799
- await agent.destroy("test");
1800
- browser.restore();
1801
- }
1802
- });
1803
-
1804
- test("XingyunAvatarAgent uses a cloned explicit ASR input stream", async () => {
1805
- MockWebSocket.instances = [];
1806
- const browser = installMicrophoneBrowser();
1807
- const inputStream = browser.createStream(16_000, 1);
1808
- const sourceTrack = inputStream.getAudioTracks()[0];
1809
- const { agent } = createAgent();
1810
-
1811
- try {
1812
- await agent.init();
1813
- await agent.startASR({ inputStream });
1814
-
1815
- assert.equal(browser.getUserMediaCalls.length, 0);
1816
- assert.equal(sourceTrack.stopCalls, 0);
1817
- assert.equal(browser.recorders.length, 1);
1818
-
1819
- await agent.stopASR();
1820
-
1821
- assert.equal(sourceTrack.stopCalls, 0);
1822
- assert.equal(browser.tracks[1].stopCalls, 1);
1823
- } finally {
1824
- await agent.destroy("test");
1825
- browser.restore();
1826
- }
1827
- });
1828
-
1829
- test("XingyunAvatarAgent skips ended tracks when cloning an explicit ASR input stream", async () => {
1830
- MockWebSocket.instances = [];
1831
- const browser = installMicrophoneBrowser();
1832
- const endedTrack = browser.createStream(16_000, 1).getAudioTracks()[0];
1833
- const liveTrack = browser.createStream(16_000, 1).getAudioTracks()[0];
1834
- endedTrack.stop();
1835
- const inputStream = new MediaStream([endedTrack, liveTrack]);
1836
- const { agent } = createAgent();
1837
-
1838
- try {
1839
- await agent.init();
1840
- await agent.startASR({ inputStream });
1841
- await agent.stopASR();
1842
-
1843
- assert.equal(endedTrack.stopCalls, 1);
1844
- assert.equal(liveTrack.stopCalls, 0);
1845
- assert.equal(browser.tracks[2].stopCalls, 1);
1846
- } finally {
1847
- await agent.destroy("test");
1848
- browser.restore();
1849
- }
1850
- });
1851
-
1852
- test("XingyunAvatarAgent uses an explicit ASR input stream in dual-audio mode", async () => {
1853
- MockWebSocket.instances = [];
1854
- const browser = installMicrophoneBrowser();
1855
- const inputStream = browser.createStream(16_000, 1);
1856
- const sourceTrack = inputStream.getAudioTracks()[0];
1857
- const { agent } = createAgent({}, {
1858
- audio: { echoCancellationEnabled: true },
1859
- });
1860
-
1861
- try {
1862
- await agent.init();
1863
- await agent.startASR({ inputStream });
1864
-
1865
- assert.equal(browser.getUserMediaCalls.length, 0);
1866
- assert.equal(browser.recorders.length, 0);
1867
- assert.equal(browser.audioWorkletNodes.length, 1);
1868
-
1869
- await agent.stopASR();
1870
-
1871
- assert.equal(sourceTrack.stopCalls, 0);
1872
- assert.equal(browser.tracks[1].stopCalls, 1);
1873
- } finally {
1874
- await agent.destroy("test");
1875
- browser.restore();
1876
- }
1877
- });
1878
-
1879
- test("XingyunAvatarAgent does not request the microphone after a pending explicit-input start is cancelled", async () => {
1880
- MockWebSocket.instances = [];
1881
- let resolveAudioEncoderSupport;
1882
- let supportCheckStarted = false;
1883
- const audioEncoderSupportResult = new Promise((resolve) => {
1884
- resolveAudioEncoderSupport = resolve;
1885
- });
1886
- const browser = installMicrophoneBrowser({
1887
- audioEncoderSupport: async (config) => {
1888
- supportCheckStarted = true;
1889
- await audioEncoderSupportResult;
1890
- return { supported: true, config };
1891
- },
1892
- });
1893
- const inputStream = browser.createStream(16_000, 1);
1894
- const sourceTrack = inputStream.getAudioTracks()[0];
1895
- const { agent } = createAgent({}, {
1896
- audio: { echoCancellationEnabled: true },
1897
- });
1898
-
1899
- try {
1900
- await agent.init();
1901
- const startPromise = agent.startASR({ inputStream });
1902
- await waitFor(() => supportCheckStarted);
1903
-
1904
- await agent.stopASR();
1905
- resolveAudioEncoderSupport();
1906
-
1907
- await assert.rejects(startPromise, (error) => (
1908
- error?.agentCode === "AUDIO_FIXED_TRACK_CANCELLED"
1909
- ));
1910
- assert.equal(browser.getUserMediaCalls.length, 0);
1911
- assert.equal(sourceTrack.stopCalls, 0);
1912
- assert.equal(browser.tracks[1].stopCalls, 1);
1913
- assert.equal(browser.audioWorkletNodes.length, 0);
1914
- } finally {
1915
- resolveAudioEncoderSupport?.();
1916
- await agent.destroy("test");
1917
- browser.restore();
1918
- }
1919
- });
1920
-
1921
- test("XingyunAvatarAgent releases explicit input and restores server state when a startup callback throws", async () => {
1922
- MockWebSocket.instances = [];
1923
- const browser = installMicrophoneBrowser();
1924
- const inputStream = browser.createStream(16_000, 1);
1925
- const sourceTrack = inputStream.getAudioTracks()[0];
1926
- const { agent } = createAgent({
1927
- onConversationChange() {
1928
- throw new Error("conversation callback failed");
1929
- },
1930
- });
1931
-
1932
- try {
1933
- await agent.init();
1934
- const ws = MockWebSocket.instances[0];
1935
-
1936
- await assert.rejects(
1937
- agent.startASR({ inputStream }),
1938
- /conversation callback failed/,
1939
- );
1940
-
1941
- assert.equal(sourceTrack.stopCalls, 0);
1942
- assert.equal(browser.tracks[1].stopCalls, 1);
1943
- assert.equal(agent.microphone, null);
1944
- assert.deepEqual(
1945
- ws.sent.map((payload) => JSON.parse(payload).message),
1946
- ["audio_input_on", "audio_input_off"],
1947
- );
1948
- } finally {
1949
- await agent.destroy("test");
1950
- browser.restore();
1951
- }
1952
- });
1953
-
1954
- test("XingyunAvatarAgent does not continue an explicit-input start stopped from a startup callback", async () => {
1955
- MockWebSocket.instances = [];
1956
- const browser = installMicrophoneBrowser();
1957
- const inputStream = browser.createStream(16_000, 1);
1958
- const sourceTrack = inputStream.getAudioTracks()[0];
1959
- let agent;
1960
- ({ agent } = createAgent({
1961
- onConversationChange(event) {
1962
- if (event.state === "asking") {
1963
- void agent.stopASR();
1964
- }
1965
- },
1966
- }));
1967
-
1968
- try {
1969
- await agent.init();
1970
-
1971
- await assert.rejects(
1972
- agent.startASR({ inputStream }),
1973
- (error) => error?.agentCode === "AUDIO_FIXED_TRACK_CANCELLED",
1974
- );
1975
-
1976
- await agent.stopASR();
1977
- assert.equal(browser.getUserMediaCalls.length, 0);
1978
- assert.equal(browser.recorders.length, 0);
1979
- assert.equal(sourceTrack.stopCalls, 0);
1980
- assert.equal(browser.tracks[1].stopCalls, 1);
1981
- assert.equal(agent.microphone, null);
1982
- } finally {
1983
- await agent.destroy("test");
1984
- browser.restore();
1985
- }
1986
- });
1987
-
1988
- test("XingyunAvatarAgent returns to idle when ASR state callbacks stop a pending explicit-input start", async () => {
1989
- for (const stopState of ["requesting-permission", "starting"]) {
1990
- MockWebSocket.instances = [];
1991
- const browser = installMicrophoneBrowser();
1992
- const inputStream = browser.createStream(16_000, 1);
1993
- let agent;
1994
- ({ agent } = createAgent({
1995
- onASRStateChange(state) {
1996
- if (state === stopState) {
1997
- void agent.stopASR();
1998
- }
1999
- },
2000
- }));
2001
-
2002
- try {
2003
- await agent.init();
2004
- await assert.rejects(
2005
- agent.startASR({ inputStream }),
2006
- (error) => error?.agentCode === "AUDIO_FIXED_TRACK_CANCELLED",
2007
- );
2008
- await agent.stopASR();
2009
-
2010
- assert.equal(agent.getASRState(), "idle", stopState);
2011
- assert.equal(browser.getUserMediaCalls.length, 0, stopState);
2012
- assert.equal(browser.recorders.length, 0, stopState);
2013
- } finally {
2014
- await agent.destroy("test");
2015
- browser.restore();
2016
- }
2017
- }
2018
- });
2019
-
2020
- test("XingyunAvatarAgent releases a cloned explicit input when ASR startup fails", async () => {
2021
- MockWebSocket.instances = [];
2022
- const browser = installMicrophoneBrowser({ webmOpus: false });
2023
- const inputStream = browser.createStream(16_000, 1);
2024
- const sourceTrack = inputStream.getAudioTracks()[0];
2025
- const { agent } = createAgent();
2026
-
2027
- try {
2028
- await agent.init();
2029
- await assert.rejects(
2030
- agent.startASR({ inputStream }),
2031
- /当前浏览器不支持 audio\/webm;codecs=opus/,
2032
- );
2033
-
2034
- assert.equal(sourceTrack.stopCalls, 0);
2035
- assert.equal(browser.tracks[1].stopCalls, 1);
2036
- } finally {
2037
- await agent.destroy("test");
2038
- browser.restore();
2039
- }
2040
- });
2041
-
2042
- test("XingyunAvatarAgent releases a cloned explicit input when enabling ASR transport fails", async () => {
2043
- MockWebSocket.instances = [];
2044
- const browser = installMicrophoneBrowser();
2045
- const inputStream = browser.createStream(16_000, 1);
2046
- const sourceTrack = inputStream.getAudioTracks()[0];
2047
- const { agent } = createAgent();
2048
-
2049
- try {
2050
- await agent.init();
2051
- MockWebSocket.instances[0].send = () => {
2052
- throw new Error("transport send failed");
2053
- };
2054
-
2055
- await assert.rejects(
2056
- agent.startASR({ inputStream }),
2057
- /transport send failed/,
2058
- );
2059
-
2060
- assert.equal(sourceTrack.stopCalls, 0);
2061
- assert.equal(browser.tracks[1].stopCalls, 1);
2062
- assert.equal(agent.microphone, null);
2063
- } finally {
2064
- await agent.destroy("test");
2065
- browser.restore();
2066
- }
2067
- });
2068
-
2069
- test("XingyunAvatarAgent stops the current ASR after an AU send failure", async () => {
2070
- MockWebSocket.instances = [];
2071
- const errors = [];
2072
- const { agent } = createAgent(
2073
- { onError: (error) => errors.push(error) },
2074
- { audio: { echoCancellationEnabled: true } },
2075
- );
2076
-
2077
- await agent.init();
2078
- const ws = MockWebSocket.instances[0];
2079
- let stopCalls = 0;
2080
- agent.microphone = {
2081
- isRecording: true,
2082
- async stop() {
2083
- stopCalls += 1;
2084
- },
2085
- };
2086
- agent.audioUplink.startInput();
2087
- agent.enableAudioInput();
2088
- ws.sent = [];
2089
- ws.bufferedAmount = 2 * 1024 * 1024 + 1;
2090
-
2091
- agent.sendAudioFrame({
2092
- data: new Blob(["nearend"]),
2093
- timestamp: agent.audioUplink.connectionStartedAt,
2094
- });
2095
- await agent.audioUplink.drain();
2096
- await waitFor(() => stopCalls === 1 && agent.stopASRPromise === null);
2097
-
2098
- assert.equal(agent.microphone, null);
2099
- assert.equal(agent.getASRState(), "failed");
2100
- assert.equal(errors.at(-1).code, "E2E_AUDIO_SEND_FAILED");
2101
- assert.deepEqual(ws.sent.map((payload) => JSON.parse(payload)), [
2102
- { type: "event", message: "audio_input_off" },
2103
- ]);
2104
- await agent.destroy("test");
2105
- });
2106
-
2107
- test("XingyunAvatarAgent resets the AU timeline for each E2E connection", async () => {
2108
- MockWebSocket.instances = [];
2109
- const { agent } = createAgent({}, {
2110
- audio: { echoCancellationEnabled: true },
2111
- reconnect: { initialDelayMs: 0, maxDelayMs: 0, maxAttempts: 2 },
2112
- });
2113
- await agent.init();
2114
- const toBuffer = (s: string) => new TextEncoder().encode(s).buffer;
2115
- const callback = agent.avatarOptions.onAudioPlaybackData;
2116
- agent.microphone = { isRecording: true, async stop() {} };
2117
- const firstWs = MockWebSocket.instances[0];
2118
- let timestamp = agent.audioUplink.connectionStartedAt;
2119
- callback({ data: toBuffer("first-far"), codec: "opus", isFirstChunk: true, sampleRate: 16000, samples: 1600, timestamp, speech_id: 1 });
2120
- agent.sendAudioFrame({ data: new Blob(["first-near"]), timestamp });
2121
- await agent.audioUplink.drain();
2122
-
2123
- firstWs.closeRemote(1006);
2124
- await waitFor(() => MockWebSocket.instances.length === 2 && agent.getAgentState() === "running");
2125
- const secondWs = MockWebSocket.instances[1];
2126
- agent.microphone = { isRecording: true, async stop() {} };
2127
- timestamp = agent.audioUplink.connectionStartedAt;
2128
- callback({ data: toBuffer("second-far"), codec: "opus", isFirstChunk: true, sampleRate: 16000, samples: 1600, timestamp, speech_id: 2 });
2129
- agent.sendAudioFrame({ data: new Blob(["second-near"]), timestamp });
2130
- await agent.audioUplink.drain();
2131
-
2132
- assert.equal(readAuFrame(firstWs.sent[0]).tsMs, 0);
2133
- assert.equal(readAuFrame(secondWs.sent[0]).tsMs, 0);
2134
- await agent.destroy("test");
2135
- });
2136
-
2137
- test("XingyunAvatarAgent rejects non-object Brain extra_body", () => {
2138
- assert.throws(
2139
- () => createAgent({}, {
2140
- asr_id: undefined,
2141
- asr_config: { provider: "asr-custom" },
2142
- llm_id: undefined,
2143
- brain_config: { provider: "brain-custom", extra_body: [] as never },
2144
- }),
2145
- /Agent Brain extra_body 必须是 JSON 对象/,
2146
- );
2147
- });
2148
-
2149
- test("XingyunAvatarAgent rejects ambiguous non-empty config sources", () => {
2150
- assert.throws(
2151
- () => createAgent({}, { asr_config: { provider: "asr-custom" } }),
2152
- /Agent ASR 配置不能同时提供 ID 和 JSON Config/,
2153
- );
2154
- });
2155
-
2156
- test("XingyunAvatarAgent extends XmovAvatar and preserves base speak", async () => {
2157
- MockWebSocket.instances = [];
2158
- const { agent } = createAgent();
2159
- const XmovAvatar = require("../../index");
2160
-
2161
- await agent.init();
2162
- const ws = MockWebSocket.instances[0];
2163
- assert.equal(ws.url, "wss://e2e.example.com/ws?token=token-from-nebula&audio_input_off=1");
2164
- assert.equal(agent instanceof XmovAvatar, true);
2165
- assert.equal(agent.started, true);
2166
- assert.equal(agent.startCalls, 1);
2167
-
2168
- agent.start();
2169
- assert.equal(agent.startCalls, 1);
2170
- agent.speak("legacy TTSA speak");
2171
- assert.deepEqual(agent.speeches, [["legacy TTSA speak"]]);
2172
- assert.deepEqual(ws.sent, []);
2173
- agent.avatarOptions.onVoiceStateChange("end");
2174
- assert.deepEqual(ws.sent, []);
2175
- });
2176
-
2177
- test("XingyunAvatarAgent restarts TTSA after same-session ready without leaving stopped state", async () => {
2178
- MockWebSocket.instances = [];
2179
- const { agent } = createAgent();
2180
-
2181
- await agent.init();
2182
- assert.equal(agent.startCalls, 1);
2183
-
2184
- agent.onTtsaReady({ session_id: "test-session" });
2185
- assert.equal(agent.startCalls, 2);
2186
- assert.equal(agent.getAgentState(), "running");
2187
-
2188
- await agent.stop();
2189
- agent.onTtsaReady({ session_id: "test-session" });
2190
- assert.equal(agent.startCalls, 2);
2191
- assert.equal(agent.getAgentState(), "stopped");
2192
- await agent.destroy("test");
2193
- });
2194
-
2195
- test("XingyunAvatarAgent ignores an explicit start while init is pending", () => {
2196
- const { agent } = createAgent();
2197
- agent.agentState = "initializing";
2198
-
2199
- assert.doesNotThrow(() => agent.start());
2200
- assert.equal(agent.started, false);
2201
- });
2202
-
2203
- test("XingyunAvatarAgent sends frozen ask and speak payloads without client IDs or providers", async () => {
2204
- MockWebSocket.instances = [];
2205
- const conversations = [];
2206
- const { agent } = createAgent({
2207
- onConversationChange: (event) => conversations.push(event),
2208
- });
2209
-
2210
- await agent.init();
2211
- const ws = MockWebSocket.instances[0];
2212
-
2213
- await agent.ask("hello");
2214
- const askPayload = JSON.parse(ws.sent[0]);
2215
- assert.deepEqual(askPayload, { type: "ask", message: { text: "hello" } });
2216
- assert.deepEqual(conversations.at(-1), { state: "asking", text: "hello" });
2217
-
2218
- await agent.speakByE2E("welcome");
2219
- const speakPayload = JSON.parse(ws.sent.at(-1));
2220
- assert.deepEqual(speakPayload, {
2221
- type: "speak",
2222
- message: { text: "welcome", is_start: true, is_end: true },
2223
- });
2224
- });
2225
-
2226
- test("XingyunAvatarAgent interrupts E2E and local playback without client IDs", async () => {
2227
- MockWebSocket.instances = [];
2228
- const { agent } = createAgent();
2229
-
2230
- await agent.init();
2231
- const ws = MockWebSocket.instances[0];
2232
- await agent.ask("cancel me");
2233
-
2234
- await agent.interruptConversation("user");
2235
- assert.deepEqual(agent.interrupts, ["speak"]);
2236
- assert.deepEqual(JSON.parse(ws.sent.at(-1)), { type: "interrupt" });
2237
- });
2238
-
2239
- test("XingyunAvatarAgent keeps ASR listening when interrupting a conversation", async () => {
2240
- MockWebSocket.instances = [];
2241
- const browser = installMicrophoneBrowser();
2242
- const { agent } = createAgent();
2243
-
2244
- try {
2245
- await agent.init();
2246
- await agent.startASR();
2247
- const ws = MockWebSocket.instances[0];
2248
- const recorder = browser.recorders[0];
2249
- ws.sent = [];
2250
-
2251
- await agent.interruptConversation("user");
2252
- const audioAfterInterrupt = new Blob(["audio-after-interrupt"], {
2253
- type: "audio/webm;codecs=opus",
2254
- });
2255
- recorder.ondataavailable?.({ data: audioAfterInterrupt, timecode: 100 });
2256
-
2257
- assert.equal(agent.getASRState(), "listening");
2258
- assert.equal(agent.microphone?.isRecording, true);
2259
- assert.equal(recorder.state, "recording");
2260
- assert.equal(browser.tracks[0].readyState, "live");
2261
- assert.equal(ws.sent.at(-1), audioAfterInterrupt);
2262
- assert.deepEqual(
2263
- ws.sent
2264
- .filter((payload) => typeof payload === "string")
2265
- .map((payload) => JSON.parse(payload)),
2266
- [{ type: "interrupt" }],
2267
- );
2268
- } finally {
2269
- await agent.destroy("test");
2270
- browser.restore();
2271
- }
2272
- });
2273
-
2274
- test("XingyunAvatarAgent forwards TTSA voice_end to E2E while running", async () => {
2275
- MockWebSocket.instances = [];
2276
- const voiceStates = [];
2277
- const { agent } = createAgent({}, {
2278
- onVoiceStateChange: (state, duration) => voiceStates.push({ state, duration }),
2279
- });
2280
-
2281
- await agent.init();
2282
- const ws = MockWebSocket.instances[0];
2283
- agent.avatarOptions.onVoiceStateChange("end", 1200);
2284
- assert.deepEqual(JSON.parse(ws.sent.at(-1)), { type: "event", message: "voice_end" });
2285
- assert.deepEqual(voiceStates, [
2286
- { state: "end", duration: 1200 },
2287
- ]);
2288
-
2289
- await agent.destroy("test");
2290
- agent.avatarOptions.onVoiceStateChange("end", 1200);
2291
- const voiceEndMessages = ws.sent
2292
- .filter((payload) => typeof payload === "string")
2293
- .map((payload) => JSON.parse(payload))
2294
- .filter((payload) => payload.message === "voice_end");
2295
- assert.equal(voiceEndMessages.length, 1);
2296
- });
2297
-
2298
- test("XingyunAvatarAgent stops callbacks after destroy", async () => {
2299
- MockWebSocket.instances = [];
2300
- const conversations = [];
2301
- const states = [];
2302
- const { agent } = createAgent({
2303
- onConversationChange: (event) => conversations.push(event),
2304
- onAgentStateChange: (state) => states.push(state),
2305
- });
2306
-
2307
- await agent.init();
2308
- const ws = MockWebSocket.instances[0];
2309
- await agent.ask("hello");
2310
- await agent.destroy("test");
2311
-
2312
- ws.emit({ type: "asr_result", text: "ignored", is_final: true });
2313
- assert.equal(conversations.length, 1);
2314
- assert.equal(states.at(-1), "destroyed");
2315
- assert.equal(agent.destroyed, true);
2316
- });
2317
-
2318
- test("XingyunAvatarAgent cleans up a pending microphone start after destroy", async () => {
2319
- MockWebSocket.instances = [];
2320
- let resolveStream;
2321
- const browser = installMicrophoneBrowser({
2322
- getUserMedia: (_constraints, createStream) => new Promise((resolve) => {
2323
- resolveStream = () => resolve(createStream());
2324
- }),
2325
- });
2326
- const { agent } = createAgent();
2327
-
2328
- try {
2329
- await agent.init();
2330
-
2331
- const startPromise = agent.startASR();
2332
- await Promise.resolve();
2333
- await agent.destroy("test");
2334
- resolveStream();
2335
-
2336
- await assert.rejects(startPromise, /Agent 已销毁/);
2337
- assert.equal(agent.getAgentState(), "destroyed");
2338
- assert.notEqual(agent.getASRState(), "listening");
2339
- assert.equal(agent.microphone, null);
2340
- assert.equal(browser.tracks[0].stopCalls, 1);
2341
- } finally {
2342
- await agent.destroy("test");
2343
- browser.restore();
2344
- }
2345
- });
2346
-
2347
- test("XingyunAvatarAgent cancels a pending microphone permission request on stop", async () => {
2348
- MockWebSocket.instances = [];
2349
- let resolveStream;
2350
- const errors = [];
2351
- const browser = installMicrophoneBrowser({
2352
- getUserMedia: (_constraints, createStream) => new Promise((resolve) => {
2353
- resolveStream = () => resolve(createStream());
2354
- }),
2355
- });
2356
- const { agent } = createAgent({ onError: (error) => errors.push(error) });
2357
-
2358
- try {
2359
- await agent.init();
2360
-
2361
- const startPromise = agent.startASR();
2362
- await Promise.resolve();
2363
- await agent.stopASR();
2364
- assert.equal(agent.getASRState(), "idle");
2365
- assert.equal(agent.microphone, null);
2366
-
2367
- resolveStream();
2368
- await assert.rejects(
2369
- startPromise,
2370
- (error) => error.agentCode === "AUDIO_FIXED_TRACK_CANCELLED",
2371
- );
2372
- assert.equal(agent.getASRState(), "idle");
2373
- assert.equal(errors.length, 0);
2374
- assert.equal(browser.recorders.length, 0);
2375
- assert.equal(browser.tracks[0].stopCalls, 1);
2376
- } finally {
2377
- resolveStream?.();
2378
- await agent.destroy("test");
2379
- browser.restore();
2380
- }
2381
- });
2382
-
2383
- test("XingyunAvatarAgent ignores a stale microphone start after a new start succeeds", async () => {
2384
- MockWebSocket.instances = [];
2385
- const pendingStreams = [];
2386
- const errors = [];
2387
- const browser = installMicrophoneBrowser({
2388
- getUserMedia: (_constraints, createStream) => new Promise((resolve, reject) => {
2389
- pendingStreams.push({
2390
- resolve: () => resolve(createStream()),
2391
- reject,
2392
- });
2393
- }),
2394
- });
2395
- const { agent } = createAgent({ onError: (error) => errors.push(error) });
2396
-
2397
- try {
2398
- await agent.init();
2399
-
2400
- const firstStart = agent.startASR();
2401
- await Promise.resolve();
2402
- await agent.stopASR();
2403
-
2404
- const secondStart = agent.startASR();
2405
- await Promise.resolve();
2406
- pendingStreams[1].resolve();
2407
- await secondStart;
2408
- assert.equal(agent.getASRState(), "listening");
2409
-
2410
- const staleError = new Error("stale permission rejection");
2411
- staleError.name = "NotAllowedError";
2412
- pendingStreams[0].reject(staleError);
2413
- await assert.rejects(
2414
- firstStart,
2415
- (error) => error.agentCode === "AUDIO_FIXED_TRACK_CANCELLED",
2416
- );
2417
-
2418
- assert.equal(agent.getASRState(), "listening");
2419
- assert.equal(errors.length, 0);
2420
- const messages = MockWebSocket.instances[0].sent
2421
- .filter((payload) => typeof payload === "string")
2422
- .map((payload) => JSON.parse(payload).message)
2423
- .filter(Boolean);
2424
- assert.deepEqual(messages, ["audio_input_on", "audio_input_off", "audio_input_on"]);
2425
- } finally {
2426
- pendingStreams.forEach(({ resolve }) => resolve());
2427
- await agent.destroy("test");
2428
- browser.restore();
2429
- }
2430
- });
2431
-
2432
- test("XingyunAvatarAgent serializes concurrent ASR stops and sends audio_input_off once", async () => {
2433
- MockWebSocket.instances = [];
2434
- const { agent } = createAgent({}, { audio: { inputEnabled: true } });
2435
- await agent.init();
2436
- const ws = MockWebSocket.instances[0];
2437
- let releaseStop;
2438
- const microphone = {
2439
- isRecording: true,
2440
- stopCalls: 0,
2441
- stop() {
2442
- this.stopCalls += 1;
2443
- return new Promise((resolve) => {
2444
- releaseStop = resolve;
2445
- });
2446
- },
2447
- };
2448
- agent.microphone = microphone;
2449
-
2450
- const firstStop = agent.stopASR();
2451
- const secondStop = agent.stopASR();
2452
- assert.equal(firstStop, secondStop);
2453
- assert.equal(microphone.stopCalls, 1);
2454
-
2455
- releaseStop();
2456
- await Promise.all([firstStop, secondStop]);
2457
- const audioInputOffMessages = ws.sent
2458
- .filter((payload) => typeof payload === "string")
2459
- .map((payload) => JSON.parse(payload))
2460
- .filter((payload) => payload.message === "audio_input_off");
2461
- assert.equal(audioInputOffMessages.length, 1);
2462
- assert.deepEqual(audioInputOffMessages[0], { type: "event", message: "audio_input_off" });
2463
- assert.equal(agent.getASRState(), "idle");
2464
- await agent.destroy("test");
2465
- });
2466
-
2467
- test("XingyunAvatarAgent fails closed when MediaRecorder reports an asynchronous error", async () => {
2468
- MockWebSocket.instances = [];
2469
- const errors = [];
2470
- let errorCallbackState;
2471
- const { agent } = createAgent({
2472
- onError: (error) => {
2473
- errors.push(error);
2474
- if (error.code === "MEDIA_RECORDER_ERROR") {
2475
- errorCallbackState = {
2476
- microphone: agent.microphone,
2477
- stopASRPromise: agent.stopASRPromise,
2478
- };
2479
- }
2480
- },
2481
- }, { audio: { inputEnabled: true } });
2482
- await agent.init();
2483
- const ws = MockWebSocket.instances[0];
2484
- let releaseStop;
2485
- const microphone = {
2486
- isRecording: true,
2487
- stopCalls: 0,
2488
- stop() {
2489
- this.stopCalls += 1;
2490
- agent.sendAudioFrame({ data: new Blob(["final-opus"]), timestamp: Date.now() });
2491
- return new Promise((resolve) => {
2492
- releaseStop = resolve;
2493
- });
2494
- },
2495
- };
2496
- agent.microphone = microphone;
2497
- const recorderError = new Error("encoder failed");
2498
- recorderError.name = "MediaRecorderError";
2499
-
2500
- agent.handleMicrophoneError(microphone, recorderError);
2501
-
2502
- assert.equal(agent.getASRState(), "failed");
2503
- assert.equal(agent.microphone, null);
2504
- assert.equal(microphone.stopCalls, 1);
2505
- assert.notEqual(agent.stopASRPromise, null);
2506
- assert.equal(errorCallbackState.microphone, null);
2507
- assert.notEqual(errorCallbackState.stopASRPromise, null);
2508
- assert.equal(errors.at(-1).domain, "asr");
2509
- assert.equal(errors.at(-1).code, "MEDIA_RECORDER_ERROR");
2510
- assert.equal(ws.sent.filter((payload) => typeof payload === "string").length, 0);
2511
-
2512
- releaseStop();
2513
- await agent.stopASRPromise;
2514
- const audioInputOffMessages = ws.sent
2515
- .filter((payload) => typeof payload === "string")
2516
- .map((payload) => JSON.parse(payload))
2517
- .filter((payload) => payload.message === "audio_input_off");
2518
- assert.equal(audioInputOffMessages.length, 1);
2519
- assert.deepEqual(audioInputOffMessages[0], { type: "event", message: "audio_input_off" });
2520
- assert.equal(ws.sent[0] instanceof Blob, true);
2521
- assert.equal(await ws.sent[0].text(), "final-opus");
2522
- assert.equal(JSON.parse(ws.sent[1]).message, "audio_input_off");
2523
- await agent.destroy("test");
2524
- });
2525
-
2526
- test("XingyunAvatarAgent uploads raw WebM Opus when dual audio is disabled", async () => {
2527
- MockWebSocket.instances = [];
2528
- const browser = installMicrophoneBrowser();
2529
- const { agent } = createAgent();
2530
-
2531
- try {
2532
- await agent.init();
2533
- const ws = MockWebSocket.instances[0];
2534
-
2535
- await agent.startASR();
2536
- assert.equal(agent.audioCaptureRestartCalls, 0);
2537
- const firstFragment = new Blob(["webm-header-and-cluster-prefix"], { type: "audio/webm;codecs=opus" });
2538
- const secondFragment = new Blob(["cluster-tail-and-opus-payload"], { type: "audio/webm;codecs=opus" });
2539
- browser.recorders[0].ondataavailable?.({ data: firstFragment, timecode: 0 });
2540
- browser.recorders[0].ondataavailable?.({ data: secondFragment, timecode: 100 });
2541
- await agent.stopASR();
2542
-
2543
- const controlMessages = ws.sent
2544
- .filter((payload) => typeof payload === "string")
2545
- .map((payload) => JSON.parse(payload));
2546
- const audioFrames = ws.sent.filter((payload) => payload instanceof Blob);
2547
- assert.equal(audioFrames.length, 3);
2548
- assert.deepEqual(
2549
- await Promise.all(audioFrames.map((frame) => frame.text())),
2550
- ["webm-header-and-cluster-prefix", "cluster-tail-and-opus-payload", "opus-final"],
2551
- );
2552
- assert.deepEqual(controlMessages, [
2553
- { type: "event", message: "audio_input_on" },
2554
- { type: "event", message: "audio_input_off" },
2555
- ]);
2556
- assert.equal(controlMessages.some((payload) => payload.message === "audio_start"), false);
2557
- assert.equal(controlMessages.some((payload) => payload.message === "voice_end"), false);
2558
- } finally {
2559
- await agent.destroy("test");
2560
- browser.restore();
2561
- }
2562
- });
2563
-
2564
- test("XingyunAvatarAgent enables its default-off audio input before uploading and disables it after stop", async () => {
2565
- MockWebSocket.instances = [];
2566
- const browser = installMicrophoneBrowser();
2567
- const { agent } = createAgent();
2568
-
2569
- try {
2570
- await agent.init();
2571
- const ws = MockWebSocket.instances[0];
2572
- assert.match(ws.url, /[?&]audio_input_off=1(?:&|$)/);
2573
-
2574
- await agent.startASR();
2575
- browser.recorders[0].ondataavailable?.({
2576
- data: new Blob(["opus"], { type: "audio/webm;codecs=opus" }),
2577
- timecode: 0,
2578
- });
2579
- await agent.stopASR();
2580
-
2581
- const messages = ws.sent.map((payload) => (
2582
- typeof payload === "string" ? JSON.parse(payload).message : "binary"
2583
- ));
2584
- assert.deepEqual(messages, ["audio_input_on", "binary", "binary", "audio_input_off"]);
2585
- assert.equal(messages.includes("voice_end"), false);
2586
- } finally {
2587
- await agent.destroy("test");
2588
- browser.restore();
2589
- }
2590
- });
2591
-
2592
- test("XingyunAvatarAgent reports a capability error when WebM Opus is unavailable", async () => {
2593
- MockWebSocket.instances = [];
2594
- const browser = installMicrophoneBrowser({ webmOpus: false });
2595
- const errors = [];
2596
- const { agent } = createAgent({ onError: (error) => errors.push(error) });
2597
-
2598
- try {
2599
- await agent.init();
2600
- await assert.rejects(agent.startASR(), /当前浏览器不支持 audio\/webm;codecs=opus/);
2601
- assert.equal(errors.at(-1).domain, "sdk");
2602
- assert.equal(errors.at(-1).code, "AUDIO_WEBM_OPUS_UNSUPPORTED");
2603
- assert.equal(browser.getUserMediaCalls.length, 0);
2604
- } finally {
2605
- await agent.destroy("test");
2606
- browser.restore();
2607
- }
2608
- });
2609
-
2610
- test("XingyunAvatarAgent distinguishes an unavailable fixed audio transform from WebM Opus support", async () => {
2611
- MockWebSocket.instances = [];
2612
- const browser = installMicrophoneBrowser({ trackTransform: false, webAudio: false });
2613
- const errors = [];
2614
- const { agent } = createAgent({ onError: (error) => errors.push(error) });
2615
-
2616
- try {
2617
- await agent.init();
2618
- await assert.rejects(agent.startASR(), /当前浏览器不支持 16 kHz mono 音频处理/);
2619
- assert.equal(errors.at(-1).domain, "sdk");
2620
- assert.equal(errors.at(-1).code, "AUDIO_FIXED_TRACK_UNSUPPORTED");
2621
- assert.equal(errors.at(-1).retryable, false);
2622
- assert.equal(browser.getUserMediaCalls.length, 1);
2623
- assert.equal(browser.recorders.length, 0);
2624
- } finally {
2625
- await agent.destroy("test");
2626
- browser.restore();
2627
- }
2628
- });
2629
-
2630
- test("XingyunAvatarAgent normalizes a MediaRecorder constructor capability failure", async () => {
2631
- MockWebSocket.instances = [];
2632
- const nativeError = new Error("constructor rejected mime type");
2633
- nativeError.name = "NotSupportedError";
2634
- const browser = installMicrophoneBrowser({ mediaRecorderConstructorError: nativeError });
2635
- const errors = [];
2636
- const { agent } = createAgent({ onError: (error) => errors.push(error) });
2637
-
2638
- try {
2639
- await agent.init();
2640
- await assert.rejects(agent.startASR(), /当前浏览器不支持 audio\/webm;codecs=opus/);
2641
- assert.equal(errors.at(-1).domain, "sdk");
2642
- assert.equal(errors.at(-1).code, "AUDIO_WEBM_OPUS_UNSUPPORTED");
2643
- assert.equal(errors.at(-1).retryable, false);
2644
- assert.equal(browser.tracks[0].stopCalls, 1);
2645
- } finally {
2646
- await agent.destroy("test");
2647
- browser.restore();
2648
- }
2649
- });
2650
-
2651
- test("XingyunAvatarAgent restores a default-off audio input when ASR start fails", async () => {
2652
- MockWebSocket.instances = [];
2653
- const browser = installMicrophoneBrowser({ webmOpus: false });
2654
- const { agent } = createAgent({}, { audio: { inputEnabled: false } });
2655
-
2656
- try {
2657
- await agent.init();
2658
- const ws = MockWebSocket.instances[0];
2659
-
2660
- await assert.rejects(agent.startASR(), /当前浏览器不支持 audio\/webm;codecs=opus/);
2661
- const messages = ws.sent.map((payload) => JSON.parse(payload).message);
2662
- assert.deepEqual(messages, ["audio_input_on", "audio_input_off"]);
2663
- } finally {
2664
- await agent.destroy("test");
2665
- browser.restore();
2666
- }
2667
- });
2668
-
2669
- test("XingyunAvatarAgent handles frozen E2E downlink events", async () => {
2670
- MockWebSocket.instances = [];
2671
- const asrResults = [];
2672
- const conversations = [];
2673
- const speakStates = [];
2674
- const errors = [];
2675
- const llmResponses = [];
2676
- const semanticResults = [];
2677
- const { agent } = createAgent({
2678
- onASRResult: (result) => asrResults.push(result),
2679
- onConversationChange: (event) => conversations.push(event),
2680
- onSpeakStateChange: (event) => speakStates.push(event),
2681
- onLLMResponse: (response) => llmResponses.push(response),
2682
- onSemanticJudgeResult: (result) => semanticResults.push(result),
2683
- onError: (error) => errors.push(error),
2684
- });
2685
-
2686
- await agent.init();
2687
- const ws = MockWebSocket.instances[0];
2688
- const microphone = {
2689
- isRecording: true,
2690
- stopCalls: 0,
2691
- async stop() {
2692
- this.stopCalls += 1;
2693
- },
2694
- };
2695
- agent.microphone = microphone;
2696
-
2697
- ws.emit({ type: "asr_result", text: "你好", is_final: false });
2698
- ws.emit({ type: "asr_result", text: "你好世界", is_final: true });
2699
- ws.emit({ type: "llm_response", event: "chunk", text: "今天", is_first: true });
2700
- ws.emit({
2701
- type: "llm_response",
2702
- event: "done",
2703
- usage: {
2704
- prompt_tokens: 150,
2705
- completion_tokens: 80,
2706
- total_tokens: 230,
2707
- cached_tokens: 10,
2708
- },
2709
- });
2710
- ws.emit({
2711
- type: "semantic_judge_round_result",
2712
- message: { query: "今天天气怎么样", meaningful: true, action: "release" },
2713
- });
2714
- ws.emit({ type: "speak_state_change", state: "ignored" });
2715
- agent.avatarOptions.onSpeakStateChange("speak_start", "speak-1");
2716
- agent.avatarOptions.onSpeakStateChange("speak_end", "speak-1");
2717
- ws.emit({ type: "error", domain: "asr", code: 5101, message: "ASR initialization failed" });
2718
- await agent.stopASRPromise;
2719
-
2720
- assert.deepEqual(asrResults.map(({ text, isFinal }) => ({ text, isFinal })), [
2721
- { text: "你好", isFinal: false },
2722
- { text: "你好世界", isFinal: true },
2723
- ]);
2724
- assert.deepEqual(conversations, [{ state: "speaking" }, { state: "completed" }, { state: "failed" }]);
2725
- assert.deepEqual(speakStates, [
2726
- { state: "speak_start", clientSpeakId: "speak-1" },
2727
- { state: "speak_end", clientSpeakId: "speak-1" },
2728
- ]);
2729
- assert.deepEqual(llmResponses.map(({ raw, ...response }) => response), [
2730
- { event: "chunk", text: "今天", isFirst: true },
2731
- {
2732
- event: "done",
2733
- usage: { promptTokens: 150, completionTokens: 80, totalTokens: 230, cachedTokens: 10 },
2734
- },
2735
- ]);
2736
- assert.deepEqual(semanticResults.map(({ raw, ...result }) => result), [
2737
- { query: "今天天气怎么样", meaningful: true, action: "release" },
2738
- ]);
2739
- assert.deepEqual({ domain: errors[0].domain, code: errors[0].code, message: errors[0].message }, {
2740
- domain: "asr",
2741
- code: "5101",
2742
- message: "ASR initialization failed",
2743
- });
2744
- assert.equal(errors[0].retryable, true);
2745
- assert.equal(microphone.stopCalls, 1);
2746
- assert.equal(agent.microphone, null);
2747
- assert.equal(agent.getASRState(), "failed");
2748
- await agent.destroy("test");
2749
- });
2750
-
2751
- test("XingyunAvatarAgent does not reconnect terminal WebSocket close codes", async () => {
2752
- for (const code of [1000, 4000, 4001, 4002, 4009, 4010]) {
2753
- MockWebSocket.instances = [];
2754
- const errors = [];
2755
- const { agent } = createAgent({ onError: (error) => errors.push(error) }, {
2756
- reconnect: { initialDelayMs: 0, maxDelayMs: 0 },
2757
- });
2758
- await agent.init();
2759
- const ws = MockWebSocket.instances[0];
2760
-
2761
- ws.closeRemote(code, code === 1000 ? "normal" : "");
2762
-
2763
- assert.equal(agent.getAgentState(), "failed");
2764
- assert.deepEqual(errors.map((error) => error.code), [`E2E_CLOSE_${code}`]);
2765
- assert.equal(MockWebSocket.instances.length, 1);
2766
- if (code === 4009) {
2767
- assert.equal(errors[0].domain, "quota");
2768
- assert.equal(errors[0].retryable, false);
2769
- }
2770
- await agent.destroy("test");
2771
- }
2772
- });
2773
-
2774
- test("XingyunAvatarAgent reconnects an abnormal E2E close and leaves ASR stopped", async () => {
2775
- MockWebSocket.instances = [];
2776
- MockWebSocket.failedConnections = 0;
2777
- const states = [];
2778
- const errors = [];
2779
- const { agent } = createAgent({
2780
- onAgentStateChange: (state) => states.push(state),
2781
- onError: (error) => errors.push(error),
2782
- }, {
2783
- reconnect: { initialDelayMs: 0, maxDelayMs: 0, maxAttempts: 2 },
2784
- });
2785
- await agent.init();
2786
- const ws = MockWebSocket.instances[0];
2787
- const microphone = {
2788
- isRecording: true,
2789
- stopCalls: 0,
2790
- async stop() {
2791
- this.stopCalls += 1;
2792
- },
2793
- };
2794
- agent.microphone = microphone;
2795
-
2796
- ws.closeRemote(1006);
2797
- await agent.stopASRPromise;
2798
- await waitFor(() => MockWebSocket.instances.length === 2 && agent.getAgentState() === "running");
2799
-
2800
- assert.deepEqual(states.slice(-2), ["reconnecting", "running"]);
2801
- assert.deepEqual(errors.map((error) => error.code), ["E2E_RECONNECTING"]);
2802
- assert.equal(agent.getASRState(), "idle");
2803
- assert.equal(microphone.stopCalls, 1);
2804
- assert.equal(agent.microphone, null);
2805
- assert.match(MockWebSocket.instances[1].url, /audio_input_off=1/);
2806
- assert.equal(ws.sent.length, 0);
2807
- await agent.destroy("test");
2808
- });
2809
-
2810
- test("XingyunAvatarAgent falls back to a fresh unified session after E2E retries are exhausted", async () => {
2811
- MockWebSocket.instances = [];
2812
- MockWebSocket.failedConnections = 0;
2813
- const states = [];
2814
- const { agent } = createAgent({
2815
- onAgentStateChange: (state) => states.push(state),
2816
- }, {
2817
- reconnect: { initialDelayMs: 0, maxDelayMs: 0, maxAttempts: 2 },
2818
- });
2819
- agent.reloadSessionInfo = {
2820
- session_id: "fresh-session",
2821
- room: "fresh-room",
2822
- token: "fresh-ttsa-token",
2823
- socket_io_url: "wss://gateway.example.com/fresh",
2824
- e2e_resp: {
2825
- ws_url: "wss://e2e.example.com/ws/session?session_id=fresh-session",
2826
- e2e_token: "fresh-e2e-token",
2827
- },
2828
- };
2829
-
2830
- await agent.init();
2831
- MockWebSocket.failedConnections = 2;
2832
- MockWebSocket.instances[0].closeRemote(1006);
2833
-
2834
- await waitFor(() => agent.restartReasons.length === 1 && agent.getAgentState() === "running");
2835
- assert.deepEqual(agent.restartReasons, ["e2e_reconnect_exhausted"]);
2836
- assert.equal(agent.reloadSuccessCalls, 1);
2837
- assert.match(MockWebSocket.instances.at(-1).url, /session_id=fresh-session/);
2838
- assert.match(MockWebSocket.instances.at(-1).url, /token=fresh-e2e-token/);
2839
- assert.deepEqual(states.slice(-2), ["reconnecting", "running"]);
2840
- await agent.destroy("test");
2841
- });
2842
-
2843
- test("XingyunAvatarAgent waits for TTSA before completing a unified session reconnect", async () => {
2844
- MockWebSocket.instances = [];
2845
- MockWebSocket.failedConnections = 0;
2846
- const { agent } = createAgent({}, {
2847
- reconnect: { initialDelayMs: 0, maxDelayMs: 0, maxAttempts: 2 },
2848
- });
2849
- await agent.init();
2850
-
2851
- await agent.onSessionReloaded({
2852
- e2e_resp: {
2853
- ws_url: "wss://e2e.example.com/ws/session?session_id=fresh-session",
2854
- e2e_token: "fresh-e2e-token",
2855
- },
2856
- });
2857
- assert.equal(agent.getAgentState(), "reconnecting");
2858
- agent.reloadSuccess();
2859
- assert.equal(agent.getAgentState(), "reconnecting");
2860
- agent.started = false;
2861
- agent.onTtsaReady();
2862
- assert.equal(agent.started, true);
2863
- assert.equal(agent.getAgentState(), "running");
2864
- await agent.destroy("test");
2865
- });
2866
-
2867
- test("XingyunAvatarAgent preserves a stopped state after a unified session reconnect", async () => {
2868
- MockWebSocket.instances = [];
2869
- MockWebSocket.failedConnections = 0;
2870
- const { agent } = createAgent({}, {
2871
- reconnect: { initialDelayMs: 0, maxDelayMs: 0, maxAttempts: 2 },
2872
- });
2873
- agent.reloadSessionInfo = {
2874
- session_id: "fresh-session",
2875
- room: "fresh-room",
2876
- token: "fresh-ttsa-token",
2877
- socket_io_url: "wss://gateway.example.com/fresh",
2878
- e2e_resp: {
2879
- ws_url: "wss://e2e.example.com/ws/session?session_id=fresh-session",
2880
- e2e_token: "fresh-e2e-token",
2881
- },
2882
- };
2883
-
2884
- await agent.init();
2885
- await agent.stop();
2886
- agent.started = false;
2887
- MockWebSocket.failedConnections = 2;
2888
- MockWebSocket.instances[0].closeRemote(1006);
2889
-
2890
- await waitFor(() => agent.restartReasons.length === 1 && agent.getAgentState() === "stopped");
2891
- assert.equal(agent.started, false);
2892
- await agent.destroy("test");
2893
- });
2894
-
2895
- test("XingyunAvatarAgent cancels a scheduled reconnect when destroyed", async () => {
2896
- MockWebSocket.instances = [];
2897
- MockWebSocket.failedConnections = 0;
2898
- const { agent } = createAgent({}, {
2899
- reconnect: { initialDelayMs: 50, maxDelayMs: 50, maxAttempts: 2 },
2900
- });
2901
- await agent.init();
2902
-
2903
- MockWebSocket.instances[0].closeRemote(1006);
2904
- await agent.destroy("test");
2905
- await new Promise((resolve) => setTimeout(resolve, 70));
2906
-
2907
- assert.equal(agent.getAgentState(), "destroyed");
2908
- assert.equal(MockWebSocket.instances.length, 1);
2909
- });
2910
-
2911
- test("XingyunAvatarAgent cancels a pending unified session restart when destroyed", async () => {
2912
- MockWebSocket.instances = [];
2913
- MockWebSocket.failedConnections = 0;
2914
- const { agent } = createAgent({}, {
2915
- reconnect: { initialDelayMs: 0, maxDelayMs: 0, maxAttempts: 1 },
2916
- });
2917
- agent.reloadSessionInfo = {
2918
- e2e_resp: {
2919
- ws_url: "wss://e2e.example.com/ws/session?session_id=late-session",
2920
- e2e_token: "late-e2e-token",
2921
- },
2922
- };
2923
- agent.restartDelayMs = 50;
2924
- await agent.init();
2925
-
2926
- MockWebSocket.failedConnections = 1;
2927
- MockWebSocket.instances[0].closeRemote(1006);
2928
- await waitFor(() => agent.restartReasons.length === 1);
2929
- await agent.destroy("test");
2930
- await new Promise((resolve) => setTimeout(resolve, 70));
2931
-
2932
- assert.equal(agent.getAgentState(), "destroyed");
2933
- assert.equal(MockWebSocket.instances.length, 2);
2934
- });
2935
-
2936
- test("XingyunAvatarAgent rejects a pending microphone start after an abnormal WebSocket close", async () => {
2937
- MockWebSocket.instances = [];
2938
- let resolveStream;
2939
- const browser = installMicrophoneBrowser({
2940
- getUserMedia: (_constraints, createStream) => new Promise((resolve) => {
2941
- resolveStream = () => resolve(createStream());
2942
- }),
2943
- });
2944
- const { agent } = createAgent();
2945
-
2946
- try {
2947
- await agent.init();
2948
- const ws = MockWebSocket.instances[0];
2949
- const startPromise = agent.startASR();
2950
- await Promise.resolve();
2951
-
2952
- ws.closeRemote(1006);
2953
- resolveStream();
2954
-
2955
- await assert.rejects(startPromise, /E2E WebSocket|Agent 尚未启动/);
2956
- await waitFor(() => agent.getAgentState() === "running");
2957
- assert.equal(agent.getASRState(), "idle");
2958
- assert.equal(browser.tracks[0].stopCalls, 1);
2959
- assert.equal(agent.microphone, null);
2960
- assert.deepEqual(ws.sent.map((payload) => JSON.parse(payload)), [
2961
- { type: "event", message: "audio_input_on" },
2962
- ]);
2963
- } finally {
2964
- await agent.destroy("test");
2965
- browser.restore();
2966
- }
2967
- });
2968
-
2969
- test("MicrophoneController uses WebM Opus and flushes its final Blob before stop resolves", async () => {
2970
- const browser = installMicrophoneBrowser({ sampleRate: 48000 });
2971
- const metadata = [];
2972
- const frames = [];
2973
- const audioBlocks = [];
2974
- const microphone = new MicrophoneController({
2975
- onMetadata: (value) => metadata.push(value),
2976
- onFrame: (frame, timestamp) => {
2977
- frames.push(frame);
2978
- audioBlocks.push({ data: frame, timestamp });
2979
- },
2980
- });
2981
-
2982
- try {
2983
- await microphone.start();
2984
- assert.equal(browser.recorders.length, 1);
2985
- assert.equal(browser.recorders[0].timeslice, 100);
2986
- assert.deepEqual(browser.recorders[0].options, {
2987
- mimeType: "audio/webm;codecs=opus",
2988
- audioBitsPerSecond: 24000,
2989
- });
2990
- assert.equal(browser.generators.length, 1);
2991
- assert.equal(browser.recorders[0].stream.getAudioTracks()[0], browser.generators[0]);
2992
- assert.deepEqual(browser.recorders[0].stream.getAudioTracks()[0].getSettings(), {
2993
- sampleRate: 16000,
2994
- channelCount: 1,
2995
- });
2996
- assert.deepEqual(metadata, [{
2997
- format: "webm_opus",
2998
- mimeType: "audio/webm;codecs=opus",
2999
- sampleRate: 16000,
3000
- channels: 1,
3001
- chunkMs: 100,
3002
- }]);
3003
-
3004
- await microphone.stop();
3005
- assert.equal(frames.length, 1);
3006
- assert.equal(frames[0] instanceof Blob, true);
3007
- assert.equal(audioBlocks[0].data, frames[0]);
3008
- assert.equal(Number.isFinite(audioBlocks[0].timestamp), true);
3009
- assert.equal(browser.tracks[0].stopCalls, 1);
3010
- assert.equal(browser.generators[0].writableClosed, true);
3011
- } finally {
3012
- await microphone.stop();
3013
- browser.restore();
3014
- }
3015
- });
3016
-
3017
- test("MicrophoneController normalizes Safari second-based timecodes to 100ms timestamps", async () => {
3018
- const browser = installMicrophoneBrowser();
3019
- const timestamps = [];
3020
- const microphone = new MicrophoneController({
3021
- onFrame: (_frame, timestamp) => timestamps.push(timestamp),
3022
- });
3023
-
3024
- try {
3025
- await microphone.start();
3026
- const recorder = browser.recorders[0];
3027
- recorder.ondataavailable?.({ data: new Blob(["a"]), timecode: 12.5 });
3028
- recorder.ondataavailable?.({ data: new Blob(["b"]), timecode: 12.6 });
3029
- recorder.ondataavailable?.({ data: new Blob(["c"]), timecode: 12.7 });
3030
-
3031
- const firstTimestamp = timestamps[0];
3032
- assert.deepEqual(
3033
- timestamps.map((timestamp) => Math.round(timestamp - firstTimestamp)),
3034
- [0, 100, 200],
3035
- );
3036
- } finally {
3037
- await microphone.stop();
3038
- browser.restore();
3039
- }
3040
- });
3041
-
3042
- test("XingyunAvatarAgent rejects non-100ms AU chunks before requesting permission", async () => {
3043
- MockWebSocket.instances = [];
3044
- const browser = installMicrophoneBrowser();
3045
- const errors = [];
3046
- const { agent } = createAgent(
3047
- { onError: (error) => errors.push(error) },
3048
- { audio: { chunkMs: 200, echoCancellationEnabled: true } },
3049
- );
3050
-
3051
- try {
3052
- await agent.init();
3053
- await assert.rejects(
3054
- agent.startASR(),
3055
- (error) => error.agentCode === "AUDIO_CHUNK_DURATION_UNSUPPORTED",
3056
- );
3057
- assert.equal(browser.getUserMediaCalls.length, 0);
3058
- assert.equal(errors[0].code, "AUDIO_CHUNK_DURATION_UNSUPPORTED");
3059
- } finally {
3060
- await agent.destroy("test");
3061
- browser.restore();
3062
- }
3063
- });
3064
-
3065
- test("XingyunAvatarAgent uses PCM WebCodecs instead of MediaRecorder for dual-audio nearend", async () => {
3066
- MockWebSocket.instances = [];
3067
- const browser = installMicrophoneBrowser({ webmOpus: false });
3068
- const { agent } = createAgent({}, {
3069
- audio: { echoCancellationEnabled: true },
3070
- });
3071
-
3072
- try {
3073
- await agent.init();
3074
- await agent.startASR();
3075
-
3076
- assert.equal(browser.recorders.length, 0);
3077
- assert.equal(browser.audioEncoders.length, 1);
3078
- assert.equal(browser.audioWorkletNodes.length, 1);
3079
- browser.audioWorkletNodes[0].emitPcm(0);
3080
- browser.audioWorkletNodes[0].emitPcm(1);
3081
- browser.audioWorkletNodes[0].emitPcm(2);
3082
- browser.audioWorkletNodes[0].emitPcm(3);
3083
-
3084
- const ws = MockWebSocket.instances[0];
3085
- await waitFor(() => ws.sent.filter((payload) => payload instanceof ArrayBuffer).length >= 2);
3086
- const auFrames = ws.sent.filter((payload) => payload instanceof ArrayBuffer);
3087
- const bytes = new Uint8Array(auFrames[0]);
3088
- assert.equal(bytes[3], AU_STREAM_NEAREND);
3089
- assert.deepEqual([...bytes.slice(8, 12)], [0x1a, 0x45, 0xdf, 0xa3]);
3090
- const firstTimestamp = new DataView(auFrames[0]).getUint32(4, false);
3091
- const secondTimestamp = new DataView(auFrames[1]).getUint32(4, false);
3092
- assert.equal(secondTimestamp - firstTimestamp, 100);
3093
- await agent.stopASR();
3094
- } finally {
3095
- await agent.destroy("test");
3096
- browser.restore();
3097
- }
3098
- });
3099
-
3100
- test("XingyunAvatarAgent fails before microphone permission when dual-audio WebCodecs Opus is unavailable", async () => {
3101
- MockWebSocket.instances = [];
3102
- const browser = installMicrophoneBrowser({ webCodecsOpus: false });
3103
- const errors = [];
3104
- const { agent } = createAgent(
3105
- { onError: (error) => errors.push(error) },
3106
- { audio: { echoCancellationEnabled: true } },
3107
- );
3108
-
3109
- try {
3110
- await agent.init();
3111
- await assert.rejects(
3112
- agent.startASR(),
3113
- (error) => error.agentCode === "AUDIO_WEBCODECS_OPUS_UNSUPPORTED",
3114
- );
3115
- assert.equal(browser.getUserMediaCalls.length, 0);
3116
- assert.equal(browser.recorders.length, 0);
3117
- assert.equal(errors.at(-1).code, "AUDIO_WEBCODECS_OPUS_UNSUPPORTED");
3118
- } finally {
3119
- await agent.destroy("test");
3120
- browser.restore();
3121
- }
3122
- });
3123
-
3124
- test("PcmWebMEncoder preserves every 20ms WebCodecs Opus packet as a WebM block", async () => {
3125
- const browser = installMicrophoneBrowser({
3126
- encodedChunkDurations: [20000, 20000, 20000, 20000, 20000],
3127
- });
3128
- const encoder = new PcmWebMEncoder();
3129
-
3130
- try {
3131
- await encoder.init();
3132
- const webm = new Uint8Array(await encoder.encode(new Int16Array(1600).fill(1)));
3133
- let clusterCount = 0;
3134
- for (let index = 0; index <= webm.length - 4; index += 1) {
3135
- if (
3136
- webm[index] === 0x1f
3137
- && webm[index + 1] === 0x43
3138
- && webm[index + 2] === 0xb6
3139
- && webm[index + 3] === 0x75
3140
- ) {
3141
- clusterCount += 1;
3142
- }
3143
- }
3144
- assert.equal(clusterCount, 5);
3145
- } finally {
3146
- await encoder.destroy();
3147
- browser.restore();
3148
- }
3149
- });
3150
-
3151
- test("PcmWebMEncoder emits a standards-compliant WebM Opus track header", async () => {
3152
- const browser = installMicrophoneBrowser();
3153
- const encoder = new PcmWebMEncoder();
3154
- const includesBytes = (data, expected) => {
3155
- for (let offset = 0; offset <= data.length - expected.length; offset += 1) {
3156
- if (expected.every((byte, index) => data[offset + index] === byte)) {
3157
- return true;
3158
- }
3159
- }
3160
- return false;
3161
- };
3162
-
3163
- try {
3164
- await encoder.init();
3165
- const webm = new Uint8Array(await encoder.encode(new Int16Array(1600).fill(1)));
3166
-
3167
- assert.equal(
3168
- includesBytes(webm, [0x42, 0x87, 0x81, 0x04]),
3169
- true,
3170
- "Opus WebM must declare DocTypeVersion 4",
3171
- );
3172
- assert.equal(
3173
- includesBytes(webm, [0x83, 0x81, 0x02]),
3174
- true,
3175
- "TrackEntry must declare TrackType 2 (audio)",
3176
- );
3177
- assert.equal(
3178
- includesBytes(webm, [0xa3, 0x87, 0x81, 0x00, 0x00, 0x80]),
3179
- true,
3180
- "Opus SimpleBlock must be marked as a key block",
3181
- );
3182
- } finally {
3183
- await encoder.destroy();
3184
- browser.restore();
3185
- }
3186
- });
3187
-
3188
- test("PcmWebMEncoder pins Opus packets to 20ms inside each 100ms AU payload", async () => {
3189
- const browser = installMicrophoneBrowser();
3190
- const encoder = new PcmWebMEncoder();
3191
-
3192
- try {
3193
- await encoder.init();
3194
- const webm = new Uint8Array(await encoder.encode(new Int16Array(1600).fill(1)));
3195
- let clusterCount = 0;
3196
- for (let index = 0; index <= webm.length - 4; index += 1) {
3197
- if (
3198
- webm[index] === 0x1f
3199
- && webm[index + 1] === 0x43
3200
- && webm[index + 2] === 0xb6
3201
- && webm[index + 3] === 0x75
3202
- ) {
3203
- clusterCount += 1;
3204
- }
3205
- }
3206
-
3207
- assert.equal(browser.audioEncoders[0].config.opus.format, "opus");
3208
- assert.equal(browser.audioEncoders[0].config.opus.frameDuration, 20000);
3209
- assert.equal(clusterCount, 5);
3210
- } finally {
3211
- await encoder.destroy();
3212
- browser.restore();
3213
- }
3214
- });
3215
-
3216
- test("PcmWebMEncoder keeps Opus continuous and flushes only when destroyed", async () => {
3217
- const browser = installMicrophoneBrowser({
3218
- flushEncodedChunkDurations: [20000],
3219
- });
3220
- const encoder = new PcmWebMEncoder();
3221
-
3222
- try {
3223
- await encoder.init();
3224
- const webm = new Uint8Array(await encoder.encode(new Int16Array(1600).fill(1)));
3225
- let clusterCount = 0;
3226
- for (let index = 0; index <= webm.length - 4; index += 1) {
3227
- if (
3228
- webm[index] === 0x1f
3229
- && webm[index + 1] === 0x43
3230
- && webm[index + 2] === 0xb6
3231
- && webm[index + 3] === 0x75
3232
- ) {
3233
- clusterCount += 1;
3234
- }
3235
- }
3236
-
3237
- assert.equal(clusterCount, 5);
3238
- assert.equal(browser.audioEncoders[0].flushCalls, 0);
3239
- } finally {
3240
- await encoder.destroy();
3241
- assert.equal(browser.audioEncoders[0].flushCalls, 1);
3242
- browser.restore();
3243
- }
3244
- });
3245
-
3246
- test("PcmWebMEncoder starts a fresh Opus and WebM stream after reset", async () => {
3247
- const browser = installMicrophoneBrowser();
3248
- const encoder = new PcmWebMEncoder();
3249
-
3250
- try {
3251
- await encoder.init();
3252
- await encoder.encode(new Int16Array(1600).fill(1));
3253
- encoder.reset();
3254
- const webm = new Uint8Array(await encoder.encode(new Int16Array(1600).fill(2)));
3255
-
3256
- assert.equal(browser.audioEncoders.length, 2);
3257
- assert.equal(browser.audioEncoders[0].closed, true);
3258
- assert.deepEqual([...webm.subarray(0, 4)], [0x1a, 0x45, 0xdf, 0xa3]);
3259
- } finally {
3260
- await encoder.destroy();
3261
- browser.restore();
3262
- }
3263
- });
3264
-
3265
- test("PcmWebMEncoder rejects browsers that ignore the 20ms Opus extension", async () => {
3266
- const browser = installMicrophoneBrowser({ webCodecsPreservesOpusConfig: false });
3267
-
3268
- try {
3269
- assert.equal(await PcmWebMEncoder.isSupported(), false);
3270
- } finally {
3271
- browser.restore();
3272
- }
3273
- });
3274
-
3275
- test("PcmWebMEncoder rejects non-20ms Opus packets at runtime", async () => {
3276
- const browser = installMicrophoneBrowser({ encodedChunkDurations: [100000] });
3277
- const encoder = new PcmWebMEncoder();
3278
-
3279
- try {
3280
- await encoder.init();
3281
- await assert.rejects(
3282
- encoder.encode(new Int16Array(1600).fill(1)),
3283
- /WebCodecs Opus 未按 20ms 分包/,
3284
- );
3285
- } finally {
3286
- await encoder.destroy();
3287
- browser.restore();
3288
- }
3289
- });
3290
-
3291
- test("MicrophoneController keeps custom chunk duration for raw nearend uploads", async () => {
3292
- const browser = installMicrophoneBrowser();
3293
- const microphone = new MicrophoneController({
3294
- audio: { chunkMs: 200 },
3295
- onFrame() {},
3296
- });
3297
-
3298
- try {
3299
- await microphone.start();
3300
- assert.equal(browser.recorders[0].timeslice, 200);
3301
- } finally {
3302
- await microphone.stop();
3303
- browser.restore();
3304
- }
3305
- });
3306
-
3307
- test("MicrophoneController resamples continuous input to 16 kHz mono before MediaRecorder", async () => {
3308
- const browser = installMicrophoneBrowser({
3309
- sampleRate: 48000,
3310
- audioDataFrames: [
3311
- {
3312
- sampleRate: 48000,
3313
- numberOfFrames: 480,
3314
- numberOfChannels: 2,
3315
- timestamp: 123,
3316
- data: [new Float32Array(480).fill(1), new Float32Array(480).fill(0)],
3317
- },
3318
- {
3319
- sampleRate: 48000,
3320
- numberOfFrames: 480,
3321
- numberOfChannels: 2,
3322
- timestamp: 10123,
3323
- data: [new Float32Array(480).fill(1), new Float32Array(480).fill(0)],
3324
- },
3325
- ],
3326
- });
3327
- const microphone = new MicrophoneController({ onFrame() {} });
3328
-
3329
- try {
3330
- await microphone.start();
3331
- await microphone.stop();
3332
-
3333
- assert.equal(browser.generators.length, 1);
3334
- assert.equal(browser.generators[0].written.length, 1);
3335
- assert.equal(browser.generators[0].written[0].sampleRate, 16000);
3336
- assert.equal(browser.generators[0].written[0].numberOfChannels, 1);
3337
- assert.equal(browser.generators[0].written[0].numberOfFrames, 320);
3338
- assert.equal(browser.generators[0].written[0].data[0], 0.5);
3339
- } finally {
3340
- await microphone.stop();
3341
- browser.restore();
3342
- }
3343
- });
3344
-
3345
- test("MicrophoneController releases tracks when MediaRecorder never dispatches stop", async () => {
3346
- const microphone = new MicrophoneController({ onFrame() {} });
3347
- const track = {
3348
- stopCalls: 0,
3349
- stop() {
3350
- this.stopCalls += 1;
3351
- },
3352
- };
3353
- microphone.stream = {
3354
- getTracks: () => [track],
3355
- };
3356
- const recorder = {
3357
- state: "recording",
3358
- ondataavailable: () => {},
3359
- onerror: null,
3360
- onstop: null,
3361
- stop() {},
3362
- };
3363
- microphone.mediaRecorder = recorder;
3364
-
3365
- const startedAt = Date.now();
3366
- await microphone.stop();
3367
-
3368
- assert.equal(track.stopCalls, 1);
3369
- assert.equal(microphone.isRecording, false);
3370
- assert.equal(recorder.ondataavailable, null);
3371
- assert.ok(Date.now() - startedAt < 1500);
3372
- });
3373
-
3374
- test("MicrophoneController waits for the final Blob when an errored recorder is already inactive", async () => {
3375
- const events = [];
3376
- const microphone = new MicrophoneController({
3377
- onFrame: () => events.push("blob"),
3378
- });
3379
- const track = { stop: () => events.push("track-stop") };
3380
- const recorder = {
3381
- state: "inactive",
3382
- ondataavailable: (event) => microphone.options.onFrame(event.data),
3383
- onerror: null,
3384
- onstop: null,
3385
- stopCalls: 0,
3386
- stop() {
3387
- this.stopCalls += 1;
3388
- },
3389
- };
3390
- microphone.stream = { getTracks: () => [track] };
3391
- microphone.mediaRecorder = recorder;
3392
-
3393
- const stopPromise = microphone.stop().then(() => events.push("stop-resolved"));
3394
- queueMicrotask(() => {
3395
- recorder.ondataavailable?.({ data: new Blob(["final-opus"]) });
3396
- recorder.onstop?.({});
3397
- });
3398
- await stopPromise;
3399
-
3400
- assert.equal(recorder.stopCalls, 0);
3401
- assert.deepEqual(events, ["blob", "track-stop", "stop-resolved"]);
3402
- });
3403
-
3404
- test("MicrophoneController starts with browser audio constraints and releases its track", async () => {
3405
- const browser = installMicrophoneBrowser({ audioSession: "ambient" });
3406
- const metadata = [];
3407
- const microphone = new MicrophoneController({
3408
- onMetadata: (value) => metadata.push(value),
3409
- onFrame() {},
3410
- });
3411
-
3412
- try {
3413
- await microphone.start();
3414
-
3415
- assert.equal(microphone.isRecording, true);
3416
- assert.deepEqual(browser.getUserMediaCalls, [
3417
- {
3418
- audio: {
3419
- sampleRate: { ideal: 16000 },
3420
- channelCount: { ideal: 1 },
3421
- echoCancellation: true,
3422
- noiseSuppression: true,
3423
- autoGainControl: true,
3424
- },
3425
- },
3426
- ]);
3427
- assert.deepEqual(metadata, [microphone.getAudioMetadata()]);
3428
- assert.equal(browser.recorders.length, 1);
3429
- assert.equal(browser.recorders[0].options.mimeType, "audio/webm;codecs=opus");
3430
- assert.deepEqual(browser.audioSessionTypes, ["play-and-record"]);
3431
-
3432
- await microphone.stop();
3433
-
3434
- assert.equal(microphone.isRecording, false);
3435
- assert.equal(browser.tracks[0].stopCalls, 1);
3436
- assert.deepEqual(browser.audioSessionTypes, ["play-and-record", "playback"]);
3437
- } finally {
3438
- await microphone.stop();
3439
- browser.restore();
3440
- }
3441
- });
3442
-
3443
- test("Web Audio session stays capture-safe until all microphone users stop", () => {
3444
- const audioSession = { type: "ambient" };
3445
- const browserNavigator = { audioSession };
3446
-
3447
- assert.equal(configureWebAudioPlaybackSession(browserNavigator), true);
3448
- assert.equal(audioSession.type, "playback");
3449
- assert.equal(beginWebAudioCaptureSession(browserNavigator), true);
3450
- assert.equal(beginWebAudioCaptureSession(browserNavigator), true);
3451
- assert.equal(audioSession.type, "play-and-record");
3452
- assert.equal(configureWebAudioPlaybackSession(browserNavigator), true);
3453
- assert.equal(audioSession.type, "play-and-record");
3454
- assert.equal(endWebAudioCaptureSession(browserNavigator), true);
3455
- assert.equal(audioSession.type, "play-and-record");
3456
- assert.equal(endWebAudioCaptureSession(browserNavigator), true);
3457
- assert.equal(audioSession.type, "playback");
3458
- assert.equal(configureWebAudioPlaybackSession({}), false);
3459
- });
3460
-
3461
- test("Web Audio playback route refresh waits for the final capture session", () => {
3462
- const audioSession = { type: "playback" };
3463
- const browserNavigator = { audioSession };
3464
- const restoredTypes = [];
3465
- const unsubscribe = subscribeWebAudioPlaybackSessionRestored(
3466
- () => restoredTypes.push(audioSession.type),
3467
- browserNavigator,
3468
- );
3469
-
3470
- beginWebAudioCaptureSession(browserNavigator);
3471
- beginWebAudioCaptureSession(browserNavigator);
3472
- endWebAudioCaptureSession(browserNavigator);
3473
- assert.deepEqual(restoredTypes, []);
3474
-
3475
- endWebAudioCaptureSession(browserNavigator);
3476
- assert.deepEqual(restoredTypes, ["playback"]);
3477
-
3478
- unsubscribe();
3479
- beginWebAudioCaptureSession(browserNavigator);
3480
- endWebAudioCaptureSession(browserNavigator);
3481
- assert.deepEqual(restoredTypes, ["playback"]);
3482
- });
3483
-
3484
- test("Web Audio playback restoration listeners cannot break capture cleanup", () => {
3485
- const audioSession = { type: "playback" };
3486
- const browserNavigator = { audioSession };
3487
- const unsubscribe = subscribeWebAudioPlaybackSessionRestored(() => {
3488
- throw new Error("route refresh failed");
3489
- }, browserNavigator);
3490
-
3491
- beginWebAudioCaptureSession(browserNavigator);
3492
- assert.equal(endWebAudioCaptureSession(browserNavigator), true);
3493
- assert.equal(audioSession.type, "playback");
3494
- unsubscribe();
3495
- });
3496
-
3497
- test("Web Audio playback route is not refreshed for a pre-existing capture session", () => {
3498
- const audioSession = { type: "play-and-record" };
3499
- const browserNavigator = { audioSession };
3500
- let restoreCalls = 0;
3501
- const unsubscribe = subscribeWebAudioPlaybackSessionRestored(
3502
- () => {
3503
- restoreCalls += 1;
3504
- },
3505
- browserNavigator,
3506
- );
3507
-
3508
- beginWebAudioCaptureSession(browserNavigator);
3509
- endWebAudioCaptureSession(browserNavigator);
3510
-
3511
- assert.equal(audioSession.type, "play-and-record");
3512
- assert.equal(restoreCalls, 0);
3513
- unsubscribe();
3514
- });
3515
-
3516
- test("PCMAudioPlayer reconnects a running graph when playback routing is restored", () => {
3517
- const hadWindow = "window" in globalThis;
3518
- const previousWindow = globalThis.window;
3519
- globalThis.window = { avatarSDKLogger: { log() {}, warn() {} } };
3520
- const destination = {};
3521
- const connections = [];
3522
- const node = {
3523
- disconnectCalls: 0,
3524
- disconnect() {
3525
- this.disconnectCalls += 1;
3526
- },
3527
- connect(target) {
3528
- connections.push(target);
3529
- },
3530
- };
3531
- const player = new PCMAudioPlayer();
3532
- player.isInitialized = true;
3533
- player.isPlaying = true;
3534
- player.audioCtx = { state: "running", destination };
3535
- player.audioWorkletNode = node;
3536
-
3537
- try {
3538
- assert.equal(player.refreshPlaybackRoute(), true);
3539
- assert.equal(node.disconnectCalls, 1);
3540
- assert.deepEqual(connections, [destination]);
3541
- } finally {
3542
- if (hadWindow) {
3543
- globalThis.window = previousWindow;
3544
- } else {
3545
- delete globalThis.window;
3546
- }
3547
- }
3548
- });
3549
-
3550
- test("MicrophoneController restores playback session when audio drain fails", async () => {
3551
- const browser = installMicrophoneBrowser({ audioSession: "ambient" });
3552
- const microphone = new MicrophoneController({ onFrame() {} });
3553
- const drainError = new Error("drain failed");
3554
-
3555
- try {
3556
- microphone.hasAudioSessionCapture = beginWebAudioCaptureSession();
3557
- microphone.audioTransformer = {
3558
- async drain() {
3559
- throw drainError;
3560
- },
3561
- async stop() {},
3562
- };
3563
-
3564
- await assert.rejects(microphone.stop(), (error) => error === drainError);
3565
- assert.equal(browser.audioSession.type, "playback");
3566
- } finally {
3567
- await microphone.stop();
3568
- browser.restore();
3569
- }
3570
- });
3571
-
3572
- test("MicrophoneController preserves permission rejection and reports it through onError", async () => {
3573
- const permissionError = namedError("NotAllowedError", "Permission denied");
3574
- const browser = installMicrophoneBrowser({
3575
- getUserMedia: async () => {
3576
- throw permissionError;
3577
- },
3578
- });
3579
- const errors = [];
3580
- const microphone = new MicrophoneController({
3581
- onFrame() {},
3582
- onError: (error) => errors.push(error),
3583
- });
3584
-
3585
- try {
3586
- await assert.rejects(microphone.start(), (error) => error === permissionError);
3587
- assert.deepEqual(errors, [permissionError]);
3588
- assert.equal(microphone.isRecording, false);
3589
- } finally {
3590
- await microphone.stop();
3591
- browser.restore();
3592
- }
3593
- });
3594
-
3595
- test("MicrophoneController preserves the no-device error", async () => {
3596
- const noDeviceError = namedError("NotFoundError", "Requested device not found");
3597
- const browser = installMicrophoneBrowser({
3598
- getUserMedia: async () => {
3599
- throw noDeviceError;
3600
- },
3601
- });
3602
- const errors = [];
3603
- const microphone = new MicrophoneController({
3604
- onFrame() {},
3605
- onError: (error) => errors.push(error),
3606
- });
3607
-
3608
- try {
3609
- await assert.rejects(microphone.start(), (error) => error === noDeviceError);
3610
- assert.deepEqual(errors, [noDeviceError]);
3611
- assert.equal(microphone.isRecording, false);
3612
- } finally {
3613
- await microphone.stop();
3614
- browser.restore();
3615
- }
3616
- });
3617
-
3618
- test("MicrophoneController rejects browsers without getUserMedia", async () => {
3619
- const browser = installMicrophoneBrowser({ mediaDevices: false });
3620
- const microphone = new MicrophoneController({ onFrame() {} });
3621
-
3622
- try {
3623
- await assert.rejects(microphone.start(), /当前浏览器不支持麦克风采集/);
3624
- assert.equal(microphone.isRecording, false);
3625
- } finally {
3626
- await microphone.stop();
3627
- browser.restore();
3628
- }
3629
- });
3630
-
3631
- test("MicrophoneController rejects browsers without WebM Opus instead of falling back to PCM", async () => {
3632
- const browser = installMicrophoneBrowser({ webmOpus: false });
3633
- const microphone = new MicrophoneController({ onFrame() {} });
3634
-
3635
- try {
3636
- await assert.rejects(microphone.start(), /当前浏览器不支持 audio\/webm;codecs=opus/);
3637
- assert.equal(microphone.isRecording, false);
3638
- assert.equal(browser.getUserMediaCalls.length, 0);
3639
- assert.equal(browser.recorders.length, 0);
3640
- } finally {
3641
- await microphone.stop();
3642
- browser.restore();
3643
- }
3644
- });
3645
-
3646
- test("MicrophoneController falls back to Web Audio without track processor APIs", async () => {
3647
- const browser = installMicrophoneBrowser({ trackTransform: false });
3648
- const microphone = new MicrophoneController({ onFrame() {} });
3649
-
3650
- try {
3651
- await microphone.start();
3652
-
3653
- assert.equal(browser.audioContexts.length, 1);
3654
- assert.equal(browser.audioContexts[0].sampleRate, 16000);
3655
- assert.equal(browser.audioDestinations[0].channelCount, 1);
3656
- assert.equal(browser.audioDestinations[0].channelCountMode, "explicit");
3657
- assert.equal(browser.recorders[0].stream, browser.audioDestinations[0].stream);
3658
-
3659
- await microphone.stop();
3660
-
3661
- assert.equal(browser.tracks[0].stopCalls, 1);
3662
- assert.equal(browser.audioDestinations[0].stream.getAudioTracks()[0].stopCalls, 1);
3663
- assert.equal(browser.audioSources[0].disconnectCalls, 1);
3664
- assert.equal(browser.audioContexts[0].closeCalls, 1);
3665
- } finally {
3666
- await microphone.stop();
3667
- browser.restore();
3668
- }
3669
- });
3670
-
3671
- test("MicrophoneController rejects a Web Audio fallback that cannot provide 16 kHz", async () => {
3672
- const browser = installMicrophoneBrowser({
3673
- trackTransform: false,
3674
- audioContextSampleRate: 48000,
3675
- });
3676
- const microphone = new MicrophoneController({ onFrame() {} });
3677
-
3678
- try {
3679
- await assert.rejects(microphone.start(), /浏览器无法创建 16 kHz AudioContext/);
3680
- assert.equal(browser.recorders.length, 0);
3681
- assert.equal(browser.tracks[0].stopCalls, 1);
3682
- assert.equal(browser.audioContexts[0].closeCalls, 1);
3683
- } finally {
3684
- await microphone.stop();
3685
- browser.restore();
3686
- }
3687
- });
3688
-
3689
- test("MicrophoneController closes a pending Web Audio fallback when stopped", async () => {
3690
- let releaseResume;
3691
- const resumeGate = new Promise((resolve) => {
3692
- releaseResume = resolve;
3693
- });
3694
- const browser = installMicrophoneBrowser({
3695
- trackTransform: false,
3696
- audioContextState: "suspended",
3697
- audioContextResume: () => resumeGate,
3698
- });
3699
- const microphone = new MicrophoneController({ onFrame() {} });
3700
-
3701
- try {
3702
- const startPromise = microphone.start();
3703
- await Promise.resolve();
3704
- await Promise.resolve();
3705
- assert.equal(browser.audioContexts.length, 1);
3706
-
3707
- await microphone.stop();
3708
- assert.equal(browser.audioContexts[0].closeCalls, 1);
3709
-
3710
- releaseResume();
3711
- await assert.rejects(startPromise, (error) => error.agentCode === "AUDIO_FIXED_TRACK_CANCELLED");
3712
- assert.equal(browser.recorders.length, 0);
3713
- assert.equal(browser.tracks[0].stopCalls, 1);
3714
- } finally {
3715
- releaseResume?.();
3716
- await microphone.stop();
3717
- browser.restore();
3718
- }
3719
- });
3720
-
3721
- test("MicrophoneController normalizes native Web Audio capability failures", async () => {
3722
- const nativeError = new Error("sample rate is unsupported");
3723
- nativeError.name = "NotSupportedError";
3724
- const browser = installMicrophoneBrowser({
3725
- trackTransform: false,
3726
- audioContextError: nativeError,
3727
- });
3728
- const microphone = new MicrophoneController({ onFrame() {} });
3729
-
3730
- try {
3731
- await assert.rejects(
3732
- microphone.start(),
3733
- (error) => error.agentCode === "AUDIO_FIXED_TRACK_UNSUPPORTED" && error.retryable === false,
3734
- );
3735
- assert.equal(browser.recorders.length, 0);
3736
- assert.equal(browser.tracks[0].stopCalls, 1);
3737
- } finally {
3738
- await microphone.stop();
3739
- browser.restore();
3740
- }
3741
- });
3742
-
3743
- test("MicrophoneController makes repeated start and stop calls idempotent", async () => {
3744
- const browser = installMicrophoneBrowser();
3745
- const microphone = new MicrophoneController({ onFrame() {} });
3746
-
3747
- try {
3748
- await microphone.start();
3749
- await microphone.start();
3750
- assert.equal(browser.getUserMediaCalls.length, 1);
3751
- assert.equal(browser.recorders.length, 1);
3752
-
3753
- await microphone.stop();
3754
- await microphone.stop();
3755
- assert.equal(browser.tracks[0].stopCalls, 1);
3756
-
3757
- await microphone.start();
3758
- assert.equal(browser.getUserMediaCalls.length, 2);
3759
- assert.equal(browser.recorders.length, 2);
3760
- await microphone.stop();
3761
- assert.equal(browser.tracks[1].stopCalls, 1);
3762
- } finally {
3763
- await microphone.stop();
3764
- browser.restore();
3765
- }
3766
- });
3767
-
3768
- test("MicrophoneController emits no frames after stop", async () => {
3769
- const browser = installMicrophoneBrowser();
3770
- const frames = [];
3771
- const microphone = new MicrophoneController({ onFrame: (frame) => frames.push(frame) });
3772
-
3773
- try {
3774
- await microphone.start();
3775
- const recorder = browser.recorders[0];
3776
- recorder.ondataavailable?.({ data: new Blob(["opus"]) });
3777
- assert.equal(frames.length, 1);
3778
-
3779
- await microphone.stop();
3780
- assert.equal(frames.length, 2);
3781
- recorder.ondataavailable?.({ data: new Blob(["late-opus"]) });
3782
- assert.equal(frames.length, 2);
3783
- } finally {
3784
- await microphone.stop();
3785
- browser.restore();
3786
- }
3787
- });