@memberjunction/ai-realtime-client 0.0.1 → 5.41.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +148 -28
- package/dist/audio/audioMeter.d.ts +101 -0
- package/dist/audio/audioMeter.d.ts.map +1 -0
- package/dist/audio/audioMeter.js +193 -0
- package/dist/audio/audioMeter.js.map +1 -0
- package/dist/audio/micCapture.d.ts +26 -0
- package/dist/audio/micCapture.d.ts.map +1 -0
- package/dist/audio/micCapture.js +69 -0
- package/dist/audio/micCapture.js.map +1 -0
- package/dist/audio/pcmPlayback.d.ts +73 -0
- package/dist/audio/pcmPlayback.d.ts.map +1 -0
- package/dist/audio/pcmPlayback.js +78 -0
- package/dist/audio/pcmPlayback.js.map +1 -0
- package/dist/audio/pcmUtils.d.ts +17 -0
- package/dist/audio/pcmUtils.d.ts.map +1 -0
- package/dist/audio/pcmUtils.js +46 -0
- package/dist/audio/pcmUtils.js.map +1 -0
- package/dist/drivers/assemblyAIRealtimeClient.d.ts +384 -0
- package/dist/drivers/assemblyAIRealtimeClient.d.ts.map +1 -0
- package/dist/drivers/assemblyAIRealtimeClient.js +732 -0
- package/dist/drivers/assemblyAIRealtimeClient.js.map +1 -0
- package/dist/drivers/elevenLabsRealtimeClient.d.ts +362 -0
- package/dist/drivers/elevenLabsRealtimeClient.d.ts.map +1 -0
- package/dist/drivers/elevenLabsRealtimeClient.js +686 -0
- package/dist/drivers/elevenLabsRealtimeClient.js.map +1 -0
- package/dist/drivers/geminiRealtimeClient.d.ts +406 -0
- package/dist/drivers/geminiRealtimeClient.d.ts.map +1 -0
- package/dist/drivers/geminiRealtimeClient.js +675 -0
- package/dist/drivers/geminiRealtimeClient.js.map +1 -0
- package/dist/drivers/openAIRealtimeClient.d.ts +381 -0
- package/dist/drivers/openAIRealtimeClient.d.ts.map +1 -0
- package/dist/drivers/openAIRealtimeClient.js +602 -0
- package/dist/drivers/openAIRealtimeClient.js.map +1 -0
- package/dist/drivers/xaiRealtimeClient.d.ts +430 -0
- package/dist/drivers/xaiRealtimeClient.d.ts.map +1 -0
- package/dist/drivers/xaiRealtimeClient.js +676 -0
- package/dist/drivers/xaiRealtimeClient.js.map +1 -0
- package/dist/generic/baseRealtimeClient.d.ts +382 -0
- package/dist/generic/baseRealtimeClient.d.ts.map +1 -0
- package/dist/generic/baseRealtimeClient.js +208 -0
- package/dist/generic/baseRealtimeClient.js.map +1 -0
- package/dist/index.d.ts +11 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +11 -0
- package/dist/index.js.map +1 -0
- package/package.json +28 -7
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import { IRealtimeAudioMeter } from './audioMeter.js';
|
|
2
|
+
/**
|
|
3
|
+
* The playback contract for a client-owned realtime audio plane: schedules raw PCM16 chunks
|
|
4
|
+
* for gapless playout and reports whether audio is AUDIBLY playing. Production is
|
|
5
|
+
* {@link RealtimePcmPlayback} (Web Audio, playhead-clock scheduling); tests inject a fake with
|
|
6
|
+
* a controllable `IsPlaying`.
|
|
7
|
+
*
|
|
8
|
+
* Shared by every websocket driver whose audio plane is client-owned (Gemini Live, ElevenLabs
|
|
9
|
+
* Agents) — WebRTC drivers (OpenAI) get playback from the peer connection instead and don't
|
|
10
|
+
* use this.
|
|
11
|
+
*/
|
|
12
|
+
export interface IRealtimePcmPlayback {
|
|
13
|
+
/** Schedules a raw PCM16 mono chunk back-to-back after any already-queued audio. */
|
|
14
|
+
Enqueue(pcm16: ArrayBuffer): void;
|
|
15
|
+
/** Stops + clears every scheduled source (barge-in / interruption). */
|
|
16
|
+
Flush(): void;
|
|
17
|
+
/** `true` while scheduled audio is audibly playing (playhead ahead of the context clock). */
|
|
18
|
+
readonly IsPlaying: boolean;
|
|
19
|
+
/** Flushes and releases the underlying audio context. */
|
|
20
|
+
Close(): void;
|
|
21
|
+
/**
|
|
22
|
+
* OPTIONAL: creates an {@link IRealtimeAudioMeter} tapping this engine's output — the
|
|
23
|
+
* AGENT-audio side of the call UI's audio-reactive visuals. Drivers feed it to
|
|
24
|
+
* `BaseRealtimeClient`'s audio-activity surface. Optional so test fakes (and minimal
|
|
25
|
+
* implementations) stay valid; callers use `playback.CreateMeter?.() ?? null`.
|
|
26
|
+
*/
|
|
27
|
+
CreateMeter?(): IRealtimeAudioMeter | null;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Web Audio playout scheduler for raw PCM16 model audio at a driver-supplied sample rate.
|
|
31
|
+
*
|
|
32
|
+
* Chunks are wrapped in `AudioBuffer`s and scheduled back-to-back against a **playhead clock**:
|
|
33
|
+
* each chunk starts at `max(playheadTime, currentTime)` and advances the playhead by its
|
|
34
|
+
* duration, producing gapless playout regardless of network jitter. {@link IsPlaying} is
|
|
35
|
+
* computed directly from that clock — the playhead being ahead of `currentTime` (with live
|
|
36
|
+
* sources) means audio is audibly coming out of the speaker. On interruption, {@link Flush}
|
|
37
|
+
* stops every scheduled source and rewinds the playhead.
|
|
38
|
+
*
|
|
39
|
+
* Generalized from the Gemini driver's original 24 kHz-fixed engine: the sample rate is now a
|
|
40
|
+
* constructor parameter so providers that negotiate their output format at session start
|
|
41
|
+
* (e.g. ElevenLabs' `agent_output_audio_format`) can construct the playout engine with the
|
|
42
|
+
* negotiated rate.
|
|
43
|
+
*/
|
|
44
|
+
export declare class RealtimePcmPlayback implements IRealtimePcmPlayback {
|
|
45
|
+
private context;
|
|
46
|
+
private sampleRate;
|
|
47
|
+
/**
|
|
48
|
+
* Master gain every source routes through (instead of the destination directly) —
|
|
49
|
+
* the single tap point {@link CreateMeter} analyses without altering the audio path.
|
|
50
|
+
*/
|
|
51
|
+
private masterGain;
|
|
52
|
+
/** The absolute context time up to which audio has been scheduled. */
|
|
53
|
+
private playheadTime;
|
|
54
|
+
/** Sources scheduled and not yet ended (so Flush can stop them). */
|
|
55
|
+
private activeSources;
|
|
56
|
+
/**
|
|
57
|
+
* @param sampleRate The PCM16 sample rate (Hz) of the chunks this engine will play
|
|
58
|
+
* (e.g. 24000 for Gemini Live, the negotiated `agent_output_audio_format` rate for
|
|
59
|
+
* ElevenLabs).
|
|
60
|
+
*/
|
|
61
|
+
constructor(sampleRate: number);
|
|
62
|
+
/** @inheritdoc */
|
|
63
|
+
Enqueue(pcm16: ArrayBuffer): void;
|
|
64
|
+
/** @inheritdoc */
|
|
65
|
+
Flush(): void;
|
|
66
|
+
/** @inheritdoc */
|
|
67
|
+
get IsPlaying(): boolean;
|
|
68
|
+
/** @inheritdoc */
|
|
69
|
+
CreateMeter(): IRealtimeAudioMeter | null;
|
|
70
|
+
/** @inheritdoc */
|
|
71
|
+
Close(): void;
|
|
72
|
+
}
|
|
73
|
+
//# sourceMappingURL=pcmPlayback.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pcmPlayback.d.ts","sourceRoot":"","sources":["../../src/audio/pcmPlayback.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,mBAAmB,EAAsB,MAAM,cAAc,CAAC;AAEvE;;;;;;;;;GASG;AACH,MAAM,WAAW,oBAAoB;IACjC,oFAAoF;IACpF,OAAO,CAAC,KAAK,EAAE,WAAW,GAAG,IAAI,CAAC;IAClC,uEAAuE;IACvE,KAAK,IAAI,IAAI,CAAC;IACd,6FAA6F;IAC7F,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC;IAC5B,yDAAyD;IACzD,KAAK,IAAI,IAAI,CAAC;IACd;;;;;OAKG;IACH,WAAW,CAAC,IAAI,mBAAmB,GAAG,IAAI,CAAC;CAC9C;AAED;;;;;;;;;;;;;;GAcG;AACH,qBAAa,mBAAoB,YAAW,oBAAoB;IAC5D,OAAO,CAAC,OAAO,CAAe;IAC9B,OAAO,CAAC,UAAU,CAAS;IAC3B;;;OAGG;IACH,OAAO,CAAC,UAAU,CAAW;IAC7B,sEAAsE;IACtE,OAAO,CAAC,YAAY,CAAK;IACzB,oEAAoE;IACpE,OAAO,CAAC,aAAa,CAAoC;IAEzD;;;;OAIG;gBACS,UAAU,EAAE,MAAM;IAO9B,kBAAkB;IACX,OAAO,CAAC,KAAK,EAAE,WAAW,GAAG,IAAI;IAiBxC,kBAAkB;IACX,KAAK,IAAI,IAAI;IAYpB,kBAAkB;IAClB,IAAW,SAAS,IAAI,OAAO,CAE9B;IAED,kBAAkB;IACX,WAAW,IAAI,mBAAmB,GAAG,IAAI;IAIhD,kBAAkB;IACX,KAAK,IAAI,IAAI;CAIvB"}
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import { pcm16ToFloat32 } from './pcmUtils.js';
|
|
2
|
+
import { RealtimeAudioMeter } from './audioMeter.js';
|
|
3
|
+
/**
|
|
4
|
+
* Web Audio playout scheduler for raw PCM16 model audio at a driver-supplied sample rate.
|
|
5
|
+
*
|
|
6
|
+
* Chunks are wrapped in `AudioBuffer`s and scheduled back-to-back against a **playhead clock**:
|
|
7
|
+
* each chunk starts at `max(playheadTime, currentTime)` and advances the playhead by its
|
|
8
|
+
* duration, producing gapless playout regardless of network jitter. {@link IsPlaying} is
|
|
9
|
+
* computed directly from that clock — the playhead being ahead of `currentTime` (with live
|
|
10
|
+
* sources) means audio is audibly coming out of the speaker. On interruption, {@link Flush}
|
|
11
|
+
* stops every scheduled source and rewinds the playhead.
|
|
12
|
+
*
|
|
13
|
+
* Generalized from the Gemini driver's original 24 kHz-fixed engine: the sample rate is now a
|
|
14
|
+
* constructor parameter so providers that negotiate their output format at session start
|
|
15
|
+
* (e.g. ElevenLabs' `agent_output_audio_format`) can construct the playout engine with the
|
|
16
|
+
* negotiated rate.
|
|
17
|
+
*/
|
|
18
|
+
export class RealtimePcmPlayback {
|
|
19
|
+
/**
|
|
20
|
+
* @param sampleRate The PCM16 sample rate (Hz) of the chunks this engine will play
|
|
21
|
+
* (e.g. 24000 for Gemini Live, the negotiated `agent_output_audio_format` rate for
|
|
22
|
+
* ElevenLabs).
|
|
23
|
+
*/
|
|
24
|
+
constructor(sampleRate) {
|
|
25
|
+
/** The absolute context time up to which audio has been scheduled. */
|
|
26
|
+
this.playheadTime = 0;
|
|
27
|
+
/** Sources scheduled and not yet ended (so Flush can stop them). */
|
|
28
|
+
this.activeSources = new Set();
|
|
29
|
+
this.sampleRate = sampleRate;
|
|
30
|
+
this.context = new AudioContext({ sampleRate });
|
|
31
|
+
this.masterGain = this.context.createGain();
|
|
32
|
+
this.masterGain.connect(this.context.destination);
|
|
33
|
+
}
|
|
34
|
+
/** @inheritdoc */
|
|
35
|
+
Enqueue(pcm16) {
|
|
36
|
+
const samples = pcm16ToFloat32(pcm16);
|
|
37
|
+
if (samples.length === 0) {
|
|
38
|
+
return;
|
|
39
|
+
}
|
|
40
|
+
const buffer = this.context.createBuffer(1, samples.length, this.sampleRate);
|
|
41
|
+
buffer.copyToChannel(samples, 0);
|
|
42
|
+
const source = this.context.createBufferSource();
|
|
43
|
+
source.buffer = buffer;
|
|
44
|
+
source.connect(this.masterGain);
|
|
45
|
+
source.onended = () => this.activeSources.delete(source);
|
|
46
|
+
this.activeSources.add(source);
|
|
47
|
+
const startAt = Math.max(this.playheadTime, this.context.currentTime);
|
|
48
|
+
source.start(startAt);
|
|
49
|
+
this.playheadTime = startAt + buffer.duration;
|
|
50
|
+
}
|
|
51
|
+
/** @inheritdoc */
|
|
52
|
+
Flush() {
|
|
53
|
+
for (const source of this.activeSources) {
|
|
54
|
+
try {
|
|
55
|
+
source.stop();
|
|
56
|
+
}
|
|
57
|
+
catch {
|
|
58
|
+
/* source never started or already stopped — fine */
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
this.activeSources.clear();
|
|
62
|
+
this.playheadTime = 0;
|
|
63
|
+
}
|
|
64
|
+
/** @inheritdoc */
|
|
65
|
+
get IsPlaying() {
|
|
66
|
+
return this.activeSources.size > 0 && this.playheadTime > this.context.currentTime;
|
|
67
|
+
}
|
|
68
|
+
/** @inheritdoc */
|
|
69
|
+
CreateMeter() {
|
|
70
|
+
return RealtimeAudioMeter.ForContextNode(this.context, this.masterGain);
|
|
71
|
+
}
|
|
72
|
+
/** @inheritdoc */
|
|
73
|
+
Close() {
|
|
74
|
+
this.Flush();
|
|
75
|
+
void this.context.close();
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
//# sourceMappingURL=pcmPlayback.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pcmPlayback.js","sourceRoot":"","sources":["../../src/audio/pcmPlayback.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAuB,kBAAkB,EAAE,MAAM,cAAc,CAAC;AA8BvE;;;;;;;;;;;;;;GAcG;AACH,MAAM,OAAO,mBAAmB;IAa5B;;;;OAIG;IACH,YAAY,UAAkB;QAV9B,sEAAsE;QAC9D,iBAAY,GAAG,CAAC,CAAC;QACzB,oEAAoE;QAC5D,kBAAa,GAAG,IAAI,GAAG,EAAyB,CAAC;QAQrD,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,OAAO,GAAG,IAAI,YAAY,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC;QAChD,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC;QAC5C,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;IACtD,CAAC;IAED,kBAAkB;IACX,OAAO,CAAC,KAAkB;QAC7B,MAAM,OAAO,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;QACtC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO;QACX,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,EAAE,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QAC7E,MAAM,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QACjC,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,kBAAkB,EAAE,CAAC;QACjD,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC;QACvB,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAChC,MAAM,CAAC,OAAO,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACzD,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC/B,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QACtE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACtB,IAAI,CAAC,YAAY,GAAG,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC;IAClD,CAAC;IAED,kBAAkB;IACX,KAAK;QACR,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACtC,IAAI,CAAC;gBACD,MAAM,CAAC,IAAI,EAAE,CAAC;YAClB,CAAC;YAAC,MAAM,CAAC;gBACL,oDAAoD;YACxD,CAAC;QACL,CAAC;QACD,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,CAAC;QAC3B,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;IAC1B,CAAC;IAED,kBAAkB;IAClB,IAAW,SAAS;QAChB,OAAO,IAAI,CAAC,aAAa,CAAC,IAAI,GAAG,CAAC,IAAI,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;IACvF,CAAC;IAED,kBAAkB;IACX,WAAW;QACd,OAAO,kBAAkB,CAAC,cAAc,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;IAC5E,CAAC;IAED,kBAAkB;IACX,KAAK;QACR,IAAI,CAAC,KAAK,EAAE,CAAC;QACb,KAAK,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;IAC9B,CAAC;CACJ"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared base64 / PCM16 codec helpers for client-owned realtime audio planes.
|
|
3
|
+
*
|
|
4
|
+
* Extracted from the Gemini client driver so every websocket-based driver with a
|
|
5
|
+
* client-owned audio plane (Gemini Live, ElevenLabs Agents, future providers) shares one
|
|
6
|
+
* implementation. `atob`/`btoa` are global in both browsers and Node 18+, so these run
|
|
7
|
+
* unchanged in unit tests.
|
|
8
|
+
*/
|
|
9
|
+
/** Decodes a base64 string into a freshly-allocated `ArrayBuffer`. */
|
|
10
|
+
export declare function base64ToArrayBuffer(base64: string): ArrayBuffer;
|
|
11
|
+
/** Encodes raw bytes to base64 in chunks (avoids call-stack limits on large frames). */
|
|
12
|
+
export declare function bytesToBase64(bytes: Uint8Array): string;
|
|
13
|
+
/** Converts a Float32 [-1, 1] sample block to PCM16 little-endian bytes, base64-encoded. */
|
|
14
|
+
export declare function encodeFloat32ToPcm16Base64(samples: Float32Array): string;
|
|
15
|
+
/** Converts raw PCM16 little-endian bytes to Float32 samples (truncates a trailing odd byte). */
|
|
16
|
+
export declare function pcm16ToFloat32(pcm: ArrayBuffer): Float32Array<ArrayBuffer>;
|
|
17
|
+
//# sourceMappingURL=pcmUtils.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pcmUtils.d.ts","sourceRoot":"","sources":["../../src/audio/pcmUtils.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,sEAAsE;AACtE,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW,CAO/D;AAED,wFAAwF;AACxF,wBAAgB,aAAa,CAAC,KAAK,EAAE,UAAU,GAAG,MAAM,CAOvD;AAED,4FAA4F;AAC5F,wBAAgB,0BAA0B,CAAC,OAAO,EAAE,YAAY,GAAG,MAAM,CAOxE;AAED,iGAAiG;AACjG,wBAAgB,cAAc,CAAC,GAAG,EAAE,WAAW,GAAG,YAAY,CAAC,WAAW,CAAC,CAQ1E"}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared base64 / PCM16 codec helpers for client-owned realtime audio planes.
|
|
3
|
+
*
|
|
4
|
+
* Extracted from the Gemini client driver so every websocket-based driver with a
|
|
5
|
+
* client-owned audio plane (Gemini Live, ElevenLabs Agents, future providers) shares one
|
|
6
|
+
* implementation. `atob`/`btoa` are global in both browsers and Node 18+, so these run
|
|
7
|
+
* unchanged in unit tests.
|
|
8
|
+
*/
|
|
9
|
+
/** Decodes a base64 string into a freshly-allocated `ArrayBuffer`. */
|
|
10
|
+
export function base64ToArrayBuffer(base64) {
|
|
11
|
+
const binary = atob(base64);
|
|
12
|
+
const out = new Uint8Array(binary.length);
|
|
13
|
+
for (let i = 0; i < binary.length; i++) {
|
|
14
|
+
out[i] = binary.charCodeAt(i);
|
|
15
|
+
}
|
|
16
|
+
return out.buffer;
|
|
17
|
+
}
|
|
18
|
+
/** Encodes raw bytes to base64 in chunks (avoids call-stack limits on large frames). */
|
|
19
|
+
export function bytesToBase64(bytes) {
|
|
20
|
+
let binary = '';
|
|
21
|
+
const chunkSize = 0x8000;
|
|
22
|
+
for (let i = 0; i < bytes.length; i += chunkSize) {
|
|
23
|
+
binary += String.fromCharCode(...bytes.subarray(i, i + chunkSize));
|
|
24
|
+
}
|
|
25
|
+
return btoa(binary);
|
|
26
|
+
}
|
|
27
|
+
/** Converts a Float32 [-1, 1] sample block to PCM16 little-endian bytes, base64-encoded. */
|
|
28
|
+
export function encodeFloat32ToPcm16Base64(samples) {
|
|
29
|
+
const pcm = new Int16Array(samples.length);
|
|
30
|
+
for (let i = 0; i < samples.length; i++) {
|
|
31
|
+
const s = Math.max(-1, Math.min(1, samples[i]));
|
|
32
|
+
pcm[i] = s < 0 ? s * 0x8000 : s * 0x7fff;
|
|
33
|
+
}
|
|
34
|
+
return bytesToBase64(new Uint8Array(pcm.buffer));
|
|
35
|
+
}
|
|
36
|
+
/** Converts raw PCM16 little-endian bytes to Float32 samples (truncates a trailing odd byte). */
|
|
37
|
+
export function pcm16ToFloat32(pcm) {
|
|
38
|
+
const even = pcm.byteLength - (pcm.byteLength % 2);
|
|
39
|
+
const ints = new Int16Array(pcm.slice(0, even));
|
|
40
|
+
const out = new Float32Array(ints.length);
|
|
41
|
+
for (let i = 0; i < ints.length; i++) {
|
|
42
|
+
out[i] = ints[i] / 0x8000;
|
|
43
|
+
}
|
|
44
|
+
return out;
|
|
45
|
+
}
|
|
46
|
+
//# sourceMappingURL=pcmUtils.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pcmUtils.js","sourceRoot":"","sources":["../../src/audio/pcmUtils.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,sEAAsE;AACtE,MAAM,UAAU,mBAAmB,CAAC,MAAc;IAC9C,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;IAC5B,MAAM,GAAG,GAAG,IAAI,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC1C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,GAAG,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;IAClC,CAAC;IACD,OAAO,GAAG,CAAC,MAAM,CAAC;AACtB,CAAC;AAED,wFAAwF;AACxF,MAAM,UAAU,aAAa,CAAC,KAAiB;IAC3C,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,MAAM,SAAS,GAAG,MAAM,CAAC;IACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;QAC/C,MAAM,IAAI,MAAM,CAAC,YAAY,CAAC,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC;IACvE,CAAC;IACD,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC;AACxB,CAAC;AAED,4FAA4F;AAC5F,MAAM,UAAU,0BAA0B,CAAC,OAAqB;IAC5D,MAAM,GAAG,GAAG,IAAI,UAAU,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAC3C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAChD,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC;IAC7C,CAAC;IACD,OAAO,aAAa,CAAC,IAAI,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;AACrD,CAAC;AAED,iGAAiG;AACjG,MAAM,UAAU,cAAc,CAAC,GAAgB;IAC3C,MAAM,IAAI,GAAG,GAAG,CAAC,UAAU,GAAG,CAAC,GAAG,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;IACnD,MAAM,IAAI,GAAG,IAAI,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;IAChD,MAAM,GAAG,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC1C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACnC,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC;IAC9B,CAAC;IACD,OAAO,GAAG,CAAC;AACf,CAAC"}
|
|
@@ -0,0 +1,384 @@
|
|
|
1
|
+
import { ClientRealtimeSessionConfig, JSONObject } from '@memberjunction/ai';
|
|
2
|
+
import { BaseRealtimeClient } from '../generic/baseRealtimeClient.js';
|
|
3
|
+
import { IRealtimePcmPlayback } from '../audio/pcmPlayback.js';
|
|
4
|
+
import { IPcmMicCapture } from '../audio/micCapture.js';
|
|
5
|
+
/** The single Voice Agent websocket endpoint (auth rides as a `?token=` query parameter). */
|
|
6
|
+
export declare const ASSEMBLYAI_AGENT_WS_URL = "wss://agents.assemblyai.com/v1/ws";
|
|
7
|
+
/**
|
|
8
|
+
* The Voice Agent wire format is FIXED: 16-bit signed little-endian PCM, mono, 24 kHz,
|
|
9
|
+
* base64-encoded, in BOTH directions (`input.audio` up, `reply.audio` down) — no
|
|
10
|
+
* per-session format negotiation exists on this provider.
|
|
11
|
+
*/
|
|
12
|
+
export declare const ASSEMBLYAI_PCM_SAMPLE_RATE = 24000;
|
|
13
|
+
/** A parsed inbound websocket frame. The Voice Agent protocol multiplexes on `type`. */
|
|
14
|
+
export interface AssemblyAIServerEvent {
|
|
15
|
+
type?: string;
|
|
16
|
+
/** `session.ready` — the provider-assigned session id. */
|
|
17
|
+
session_id?: string;
|
|
18
|
+
/** `transcript.user[.delta]` / `transcript.agent` — the transcribed text. */
|
|
19
|
+
text?: string;
|
|
20
|
+
/** `transcript.*` — provider conversation-item id. */
|
|
21
|
+
item_id?: string;
|
|
22
|
+
/** `reply.started` / `transcript.agent` — id of the reply the frame belongs to. */
|
|
23
|
+
reply_id?: string;
|
|
24
|
+
/** `transcript.agent` — true when the turn was cut off by a barge-in. */
|
|
25
|
+
interrupted?: boolean;
|
|
26
|
+
/** `reply.audio` — one base64 PCM16 chunk of the agent's spoken output. */
|
|
27
|
+
data?: string;
|
|
28
|
+
/** `reply.done` — `'interrupted'` when the user barged in mid-reply; absent otherwise. */
|
|
29
|
+
status?: string;
|
|
30
|
+
/** `tool.call` — correlation id the result must echo. */
|
|
31
|
+
call_id?: string;
|
|
32
|
+
/** `tool.call` — the tool name. */
|
|
33
|
+
name?: string;
|
|
34
|
+
/** `tool.call` — the arguments, ALREADY PARSED into an object by the provider. */
|
|
35
|
+
arguments?: JSONObject;
|
|
36
|
+
/** `session.error` — provider error code. */
|
|
37
|
+
code?: string;
|
|
38
|
+
/** `session.error` — human-readable error message. */
|
|
39
|
+
message?: string;
|
|
40
|
+
/** `session.error` — the offending parameter, when applicable. */
|
|
41
|
+
param?: string;
|
|
42
|
+
/** `session.updated` — echo of the applied session object. */
|
|
43
|
+
session?: JSONObject;
|
|
44
|
+
/** `session.ended` — total billable session seconds. */
|
|
45
|
+
session_duration_seconds?: number;
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* The minimal websocket surface this client depends on: assignable lifecycle handlers plus
|
|
49
|
+
* `send`/`close`. Declaring the seam as an interface (rather than the platform `WebSocket`)
|
|
50
|
+
* lets unit tests inject a fully in-memory fake that captures outbound frames and drives the
|
|
51
|
+
* handlers with AssemblyAI-shaped events — no websocket, no network.
|
|
52
|
+
*/
|
|
53
|
+
export interface IAssemblyAIClientSocket {
|
|
54
|
+
/** Invoked once the socket is open. */
|
|
55
|
+
onopen: (() => void) | null;
|
|
56
|
+
/** Invoked with each inbound frame's raw string payload. */
|
|
57
|
+
onmessage: ((data: string) => void) | null;
|
|
58
|
+
/** Invoked on a socket-level error (fatal). */
|
|
59
|
+
onerror: ((message: string) => void) | null;
|
|
60
|
+
/** Invoked when the socket closes (any reason). */
|
|
61
|
+
onclose: (() => void) | null;
|
|
62
|
+
/** Sends one JSON-serialized client frame. */
|
|
63
|
+
send(data: string): void;
|
|
64
|
+
/** Terminates the underlying connection. */
|
|
65
|
+
close(): void;
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* AssemblyAI Voice Agent implementation of {@link BaseRealtimeClient}: a **browser-direct**
|
|
69
|
+
* agent websocket authenticated with the server-minted ONE-TIME client token (the
|
|
70
|
+
* `EphemeralToken` — no API key ever reaches the browser).
|
|
71
|
+
*
|
|
72
|
+
* Registered with the ClassFactory under the key `'assemblyai'` — the `Provider` string the
|
|
73
|
+
* server's `AssemblyAIRealtime` driver stamps on its `ClientRealtimeSessionConfig`.
|
|
74
|
+
*
|
|
75
|
+
* Owns ALL AssemblyAI wire concerns (the behavioral sibling of the ElevenLabs client driver —
|
|
76
|
+
* the audio plane is client-owned over a websocket, no WebRTC):
|
|
77
|
+
* - **Connect handshake**: open `wss://…/v1/ws?token=…` → send the server-authored
|
|
78
|
+
* `session.update` (from the `SessionConfig` pact: system prompt, tools, voice, turn
|
|
79
|
+
* detection) as the FIRST frame → wait for `session.ready` → build the audio plane at the
|
|
80
|
+
* provider's FIXED 24 kHz PCM16 format → `'listening'`. The state is gated on
|
|
81
|
+
* `session.ready` (driver obligation #7): only then has the provider confirmed the session
|
|
82
|
+
* config is applied (audio sent earlier would be dropped anyway).
|
|
83
|
+
* - **Audio in**: mic PCM16 at 24 kHz via the shared {@link createPcmMicCapture} worklet
|
|
84
|
+
* pipeline, streamed as base64 `input.audio` frames.
|
|
85
|
+
* - **Audio out**: `reply.audio` chunks decoded into the shared {@link RealtimePcmPlayback}
|
|
86
|
+
* at 24 kHz; {@link IsAudioPlaying} comes from its playout clock.
|
|
87
|
+
* - **Transcripts**: user turns stream as `transcript.user.delta` fragments
|
|
88
|
+
* (`IsFinal: false` deltas) finalized by `transcript.user`; agent turns arrive as ONE
|
|
89
|
+
* final `transcript.agent` (after a barge-in it carries `interrupted: true` and the text
|
|
90
|
+
* already reflects the truncated turn — no ElevenLabs-style correction event follows).
|
|
91
|
+
* - **Busy mapping**: `IsBusy` is set on `reply.started` / first `reply.audio` (and eagerly
|
|
92
|
+
* when this client triggers a reply) and cleared on `reply.done` and on a `tool.call`
|
|
93
|
+
* frame (the agent has yielded the floor pending the result — deadlock guard,
|
|
94
|
+
* obligation #2).
|
|
95
|
+
* - **Narration is NATIVE**: {@link RequestSpokenUpdate} sends `reply.create` with the
|
|
96
|
+
* instructions (a real per-response-instructions channel — no user-turn emulation). The
|
|
97
|
+
* response kind is stamped `'narration'` at send time (sends are the turn triggers) and
|
|
98
|
+
* reset on the reply boundary; the kind tagging assumes `transcript.agent` precedes its
|
|
99
|
+
* `reply.done`, which is the provider's emission order.
|
|
100
|
+
* - **{@link SendText} is EMULATED via `reply.create`**: the protocol accepts no typed user
|
|
101
|
+
* input, so the typed text rides as response instructions framing it as the user's turn.
|
|
102
|
+
* Fidelity caveat: the text enters as instructions rather than a conversation user turn,
|
|
103
|
+
* so the agent may occasionally reference "the message you sent" — hosts should treat
|
|
104
|
+
* typed input on this provider as best-effort.
|
|
105
|
+
* - **{@link SendContextNote} is EMULATED via the mutable `system_prompt`**: notes are
|
|
106
|
+
* appended under a "Background updates" heading and the full prompt is re-sent via
|
|
107
|
+
* `session.update` — a config write that never triggers or disturbs generation, so it is
|
|
108
|
+
* sent immediately even mid-reply.
|
|
109
|
+
* - **Barge-in**: `input.speech.started` while a reply is active (or audio audibly playing)
|
|
110
|
+
* is the snappy flush point (per the provider's own guidance — ~300 ms faster than waiting
|
|
111
|
+
* for `reply.done`); `reply.done` with `status: 'interrupted'` is the authoritative
|
|
112
|
+
* verdict and is the fallback flush when the speech-start gate missed. A speech start
|
|
113
|
+
* while idle is a user simply taking their turn — NOT an interruption (base contract).
|
|
114
|
+
* - **Cancel**: the protocol has no cancel frame. {@link CancelActiveResponse} flushes the
|
|
115
|
+
* locally-owned playout queue (speech stops immediately), marks the reply inactive, and
|
|
116
|
+
* SUPPRESSES residual `reply.audio` frames of the cancelled reply until the next reply
|
|
117
|
+
* boundary — so late-arriving chunks of a cancelled turn are never played or allowed to
|
|
118
|
+
* re-assert `'speaking'`.
|
|
119
|
+
* - **No usage telemetry**: the streaming socket exposes no token-usage events, so this
|
|
120
|
+
* driver NEVER emits {@link OnUsage} (registering a handler is safe; it just never fires).
|
|
121
|
+
*/
|
|
122
|
+
export declare class AssemblyAIRealtimeClient extends BaseRealtimeClient {
|
|
123
|
+
private socket;
|
|
124
|
+
private micStream;
|
|
125
|
+
private micCapture;
|
|
126
|
+
private playback;
|
|
127
|
+
/** True while an agent reply is in flight; gates (queues) client-triggered sends. */
|
|
128
|
+
private responseActive;
|
|
129
|
+
/** The kind of the reply currently in flight; stamped at send time, reset on boundary. */
|
|
130
|
+
private activeResponseKind;
|
|
131
|
+
/** Sends deferred while a reply is in flight; drained in order at the next boundary. */
|
|
132
|
+
private queuedSends;
|
|
133
|
+
/**
|
|
134
|
+
* True between a {@link CancelActiveResponse} and the next reply boundary: residual
|
|
135
|
+
* `reply.audio` chunks of the cancelled reply are dropped instead of played.
|
|
136
|
+
*/
|
|
137
|
+
private suppressReplyAudio;
|
|
138
|
+
/**
|
|
139
|
+
* Tool calls awaiting their result — used to make {@link SendToolResult} EXACTLY-ONCE:
|
|
140
|
+
* the id is consumed when the result is accepted (queued or sent), so a duplicate send
|
|
141
|
+
* for the same call is dropped with a warning instead of confusing the conversation.
|
|
142
|
+
*/
|
|
143
|
+
private pendingToolCallIds;
|
|
144
|
+
/** True once Disconnect ran — an expected socket close must not surface as fatal. */
|
|
145
|
+
private closedByConsumer;
|
|
146
|
+
/** Provider-assigned session id (from `session.ready`) — the `session.resume` key. */
|
|
147
|
+
private providerSessionId;
|
|
148
|
+
/** The token-authenticated endpoint URL, kept for the resume reattach socket. */
|
|
149
|
+
private connectUrl;
|
|
150
|
+
/** One resume attempt per session: a second unexpected drop is fatal (as before). */
|
|
151
|
+
private resumeAttempted;
|
|
152
|
+
/**
|
|
153
|
+
* The client's own view of the session state — mirrors what was last emitted, EXCEPT after
|
|
154
|
+
* a tool call: the host typically shows its own busy indicator then, so the client silently
|
|
155
|
+
* leaves `'speaking'` (no emission) until the result reply's first output re-asserts it.
|
|
156
|
+
*/
|
|
157
|
+
private currentState;
|
|
158
|
+
/** The server-authored base system prompt (from the SessionConfig pact). */
|
|
159
|
+
private basePrompt;
|
|
160
|
+
/** Accumulated {@link SendContextNote} texts, re-sent with the full prompt each time. */
|
|
161
|
+
private contextNotes;
|
|
162
|
+
/**
|
|
163
|
+
* Opens the client-direct session: socket to the token-authenticated endpoint, the
|
|
164
|
+
* server-authored `session.update` as the first frame, then — once `session.ready`
|
|
165
|
+
* confirms the config is applied — the audio plane at the provider's fixed 24 kHz
|
|
166
|
+
* format. Reports `'listening'` only after all of that (obligation #7).
|
|
167
|
+
*/
|
|
168
|
+
Connect(config: ClientRealtimeSessionConfig, micStream: MediaStream): Promise<void>;
|
|
169
|
+
/**
|
|
170
|
+
* Tears down the socket, mic capture, mic tracks, and playout engine, resets the response
|
|
171
|
+
* state machine, and emits a final `'closed'` (unless already `'error'`). Sends
|
|
172
|
+
* `session.end` first — without it the provider holds the session (billable) for a
|
|
173
|
+
* 30-second resume window. Safe to call more than once.
|
|
174
|
+
*/
|
|
175
|
+
Disconnect(): Promise<void>;
|
|
176
|
+
/**
|
|
177
|
+
* Injects typed text — EMULATED via `reply.create` (the protocol accepts no typed user
|
|
178
|
+
* input), with instructions framing the text as the user's turn. No-op when the session
|
|
179
|
+
* is not open. See the class-level fidelity caveat.
|
|
180
|
+
*
|
|
181
|
+
* **SendText implies barge-in** (base-contract rule): an active spoken reply is cancelled
|
|
182
|
+
* via {@link CancelActiveResponse} first — playback flushed, reply marked inactive,
|
|
183
|
+
* queued sends drained — so the typed turn takes the floor immediately. If a drained
|
|
184
|
+
* queued send starts a new reply, the text queues behind it, preserving delivery order.
|
|
185
|
+
*/
|
|
186
|
+
SendText(text: string): void;
|
|
187
|
+
/**
|
|
188
|
+
* @inheritdoc
|
|
189
|
+
*
|
|
190
|
+
* AssemblyAI has no cancel frame — the client OWNS the audio plane, so cancelling means:
|
|
191
|
+
* flush the local playout queue (speech stops immediately), mark the in-flight reply
|
|
192
|
+
* inactive, SUPPRESS residual `reply.audio` chunks of the cancelled reply until the next
|
|
193
|
+
* reply boundary, and drain queued sends (a queued narration takes the floor next —
|
|
194
|
+
* delivery is never dropped by a cancel). No-op when nothing is active.
|
|
195
|
+
*/
|
|
196
|
+
CancelActiveResponse(): void;
|
|
197
|
+
/**
|
|
198
|
+
* Injects background context — EMULATED via the mutable `system_prompt`: the note is
|
|
199
|
+
* appended under a "Background updates" heading and the FULL prompt is re-sent via
|
|
200
|
+
* `session.update`. A config write never triggers or disturbs generation, so it is sent
|
|
201
|
+
* immediately even while a reply is in flight (no queueing applies).
|
|
202
|
+
*/
|
|
203
|
+
SendContextNote(text: string): void;
|
|
204
|
+
/**
|
|
205
|
+
* Triggers ONE short spoken update — NATIVE via `reply.create` with per-response
|
|
206
|
+
* instructions. The resulting reply is stamped `Kind: 'narration'` at send time (reset
|
|
207
|
+
* on the reply boundary). Queued behind any in-flight reply per the base contract's
|
|
208
|
+
* collision rule (an overlapping `reply.create` would collide with active generation).
|
|
209
|
+
*/
|
|
210
|
+
RequestSpokenUpdate(instructions: string): void;
|
|
211
|
+
/**
|
|
212
|
+
* Feeds an executed tool's result back via `tool.result`, correlated by the provider's
|
|
213
|
+
* `call_id`. EXACTLY-ONCE: the pending id is consumed when the result is accepted, and a
|
|
214
|
+
* duplicate (or unknown) callID is dropped with a warning. Sent immediately when idle —
|
|
215
|
+
* the provider speaks the result as the turn's continuation (no explicit generation
|
|
216
|
+
* trigger exists or is needed) — otherwise queued behind the in-flight reply (e.g. a
|
|
217
|
+
* progress narration) so the trigger is never lost and the narration's kind tagging is
|
|
218
|
+
* not clobbered. The wire `result` slot expects a JSON-STRING, which is exactly the
|
|
219
|
+
* contract's `outputJson` shape — it passes through verbatim.
|
|
220
|
+
*/
|
|
221
|
+
SendToolResult(callID: string, outputJson: string): void;
|
|
222
|
+
/**
|
|
223
|
+
* Mutes / unmutes by toggling the mic tracks' `enabled` flag: the capture pipeline stays
|
|
224
|
+
* up and streams SILENCE while muted (the provider's VAD sees a continuous stream and the
|
|
225
|
+
* un-mute is glitch-free — same policy as the other client drivers).
|
|
226
|
+
*/
|
|
227
|
+
SetMuted(muted: boolean): void;
|
|
228
|
+
/** @inheritdoc */
|
|
229
|
+
get IsBusy(): boolean;
|
|
230
|
+
/**
|
|
231
|
+
* @inheritdoc
|
|
232
|
+
*
|
|
233
|
+
* Computed directly from the playout engine's playhead clock — this client OWNS the
|
|
234
|
+
* output buffer, so "audibly playing" is precisely "scheduled audio extends beyond the
|
|
235
|
+
* audio context's current time".
|
|
236
|
+
*/
|
|
237
|
+
get IsAudioPlaying(): boolean;
|
|
238
|
+
/**
|
|
239
|
+
* Creation seam for the agent websocket. Production wraps the platform-global
|
|
240
|
+
* `WebSocket` opened against the token-authenticated URL; unit tests override this to
|
|
241
|
+
* return an in-memory fake. Handlers are attached by {@link Connect} AFTER this returns,
|
|
242
|
+
* so the implementation must not require them at construction time.
|
|
243
|
+
*/
|
|
244
|
+
protected createSocket(url: string): IAssemblyAIClientSocket;
|
|
245
|
+
/**
|
|
246
|
+
* Creation seam for the mic-capture pipeline at the provider's fixed 24 kHz rate.
|
|
247
|
+
* Production delegates to the shared {@link createPcmMicCapture}; unit tests override
|
|
248
|
+
* this with a no-op fake (and may capture `onPcmChunk` to simulate mic frames).
|
|
249
|
+
*/
|
|
250
|
+
protected createMicCapture(micStream: MediaStream, sampleRate: number, onPcmChunk: (base64Pcm16: string) => void): Promise<IPcmMicCapture>;
|
|
251
|
+
/**
|
|
252
|
+
* Creation seam for the playout engine at the provider's fixed 24 kHz rate. Production
|
|
253
|
+
* returns the shared {@link RealtimePcmPlayback}.
|
|
254
|
+
*/
|
|
255
|
+
protected createPlayback(sampleRate: number): IRealtimePcmPlayback;
|
|
256
|
+
/** Resolver for the in-flight Connect's session.ready wait (null outside Connect). */
|
|
257
|
+
private onSessionReady;
|
|
258
|
+
/**
|
|
259
|
+
* Extracts the wire-shaped `session` object from the server-minted `SessionConfig` pact
|
|
260
|
+
* (`{ session, config }` — authored by the server's `AssemblyAIRealtime`). Falls back to
|
|
261
|
+
* an empty object when absent (the session then runs on provider defaults).
|
|
262
|
+
*/
|
|
263
|
+
private parseSessionObject;
|
|
264
|
+
/** Streams one base64 PCM16 mic chunk as an `input.audio` frame. */
|
|
265
|
+
private sendMicChunk;
|
|
266
|
+
/** Surfaces a fatal socket error and marks the session unusable (obligation #6). */
|
|
267
|
+
private handleSocketError;
|
|
268
|
+
/**
|
|
269
|
+
* A socket close the CONSUMER didn't ask for is fatal: the provider hard-closes at token
|
|
270
|
+
* expiry and when it ends the session itself, so an unexpected close is how credential /
|
|
271
|
+
* session death reaches the host (obligation #6).
|
|
272
|
+
*/
|
|
273
|
+
private handleSocketClose;
|
|
274
|
+
/**
|
|
275
|
+
* ONE-shot reattach inside the provider's 30-second resume window: a fresh socket to
|
|
276
|
+
* the same token-authenticated endpoint whose FIRST frame is `session.resume` with the
|
|
277
|
+
* `session_id` captured from `session.ready`. The provider re-confirms with another
|
|
278
|
+
* `session.ready`, which restores `'listening'` — the mic worklet and playout engine
|
|
279
|
+
* survive untouched (mic chunks simply flow into the new socket). A failed or second
|
|
280
|
+
* drop falls through to the pre-existing fatal path, so the worst case is exactly the
|
|
281
|
+
* old behavior. Returns `true` when a reattach was started.
|
|
282
|
+
*/
|
|
283
|
+
private tryResumeSession;
|
|
284
|
+
/** Parses one raw socket payload; non-JSON frames are ignored. */
|
|
285
|
+
private handleSocketMessage;
|
|
286
|
+
/** Multiplexes one inbound frame to the focused per-concern handlers. */
|
|
287
|
+
private handleServerEvent;
|
|
288
|
+
/** A fresh reply is generating: lift any cancel suppression and mark the agent busy. */
|
|
289
|
+
private handleReplyStarted;
|
|
290
|
+
/**
|
|
291
|
+
* Decodes one base64 reply-audio chunk into the playout queue and marks generation live —
|
|
292
|
+
* unless a local cancel suppressed the remainder of this reply, in which case the chunk
|
|
293
|
+
* is dropped (never played, never re-asserts `'speaking'`).
|
|
294
|
+
*/
|
|
295
|
+
private handleReplyAudio;
|
|
296
|
+
/** User transcript: streaming deltas (`IsFinal: false`) finalized by `transcript.user`. */
|
|
297
|
+
private handleUserTranscript;
|
|
298
|
+
/**
|
|
299
|
+
* Agent transcript: the turn's complete text, emitted FINAL with the ACTIVE reply kind
|
|
300
|
+
* so narration turns are tagged correctly (stamp-at-send — see the class doc). After a
|
|
301
|
+
* barge-in the provider sends the truncated text with `interrupted: true` — already the
|
|
302
|
+
* authoritative record of what was spoken, emitted the same way.
|
|
303
|
+
*/
|
|
304
|
+
private handleAgentTranscript;
|
|
305
|
+
/**
|
|
306
|
+
* Reply boundary. `status: 'interrupted'` is the provider's authoritative true-barge-in
|
|
307
|
+
* verdict — when the snappy `input.speech.started` gate already handled it this is a
|
|
308
|
+
* no-op fallback; when it didn't (e.g. the speech-start frame was missed), the flush +
|
|
309
|
+
* interruption surface here. Then: release the busy lock, lift any cancel suppression,
|
|
310
|
+
* reset the reply kind, drain queued sends (stopping at the first one that starts a new
|
|
311
|
+
* reply), and return the floor.
|
|
312
|
+
*/
|
|
313
|
+
private handleReplyDone;
|
|
314
|
+
/**
|
|
315
|
+
* Surfaces the agent's tool call to the host. Two deliberate behaviors mirror the other
|
|
316
|
+
* drivers: (1) the client silently leaves `'speaking'` (no emission) so a host-rendered
|
|
317
|
+
* busy indicator isn't clobbered by the turn's trailing frames (obligation #1); (2)
|
|
318
|
+
* `responseActive` is CLEARED — the agent has yielded the floor pending the result, so a
|
|
319
|
+
* queued send can never deadlock (obligation #2). The queue is NOT drained here: a queued
|
|
320
|
+
* narration must not trigger a reply between the tool call and its result. The provider
|
|
321
|
+
* emits `arguments` ALREADY PARSED, so it is re-stringified to honor the base contract's
|
|
322
|
+
* JSON-string `ArgumentsJson` shape.
|
|
323
|
+
*/
|
|
324
|
+
private handleToolCall;
|
|
325
|
+
/**
|
|
326
|
+
* The user started speaking. When a reply is in flight or audio is audibly playing this
|
|
327
|
+
* is a TRUE barge-in: flush playout NOW (the provider's own guidance — ~300 ms snappier
|
|
328
|
+
* than waiting for `reply.done`), suppress the cancelled reply's residual audio, surface
|
|
329
|
+
* the interruption, and give the floor back. While idle it is a user simply taking their
|
|
330
|
+
* turn — NOT an interruption (base contract), so nothing is emitted. The queue is NOT
|
|
331
|
+
* drained — the user has the floor; queued sends flush at the next reply boundary.
|
|
332
|
+
*/
|
|
333
|
+
private handleSpeechStarted;
|
|
334
|
+
/**
|
|
335
|
+
* The provider ended the session itself. After a consumer {@link Disconnect} (which
|
|
336
|
+
* sends `session.end`) this is the expected acknowledgment and stays silent; otherwise
|
|
337
|
+
* it is surfaced as FATAL so the host tears down instead of idling on a dying socket.
|
|
338
|
+
*/
|
|
339
|
+
private handleSessionEnded;
|
|
340
|
+
/**
|
|
341
|
+
* First model output of a reply (reply.started or an audio chunk): the agent is busy and
|
|
342
|
+
* the client is audibly / imminently `'speaking'`.
|
|
343
|
+
*/
|
|
344
|
+
private markGenerationStarted;
|
|
345
|
+
/**
|
|
346
|
+
* Runs a send immediately when no reply is in flight; otherwise queues it for the next
|
|
347
|
+
* boundary (`reply.done`). An overlapping `reply.create` would collide with active
|
|
348
|
+
* generation, so deferral is the safe default — mirroring the other drivers' rule.
|
|
349
|
+
*/
|
|
350
|
+
private enqueueOrRun;
|
|
351
|
+
/**
|
|
352
|
+
* Drains queued sends in order at a reply boundary, stopping as soon as one starts a
|
|
353
|
+
* new reply (sets {@link responseActive}) — the rest wait for that reply to finish.
|
|
354
|
+
*/
|
|
355
|
+
private flushQueuedSends;
|
|
356
|
+
/**
|
|
357
|
+
* Sends a `reply.create` that triggers a reply, stamping the upcoming reply's kind at
|
|
358
|
+
* send time and eagerly marking the agent busy. `emitSpeaking` mirrors the other
|
|
359
|
+
* drivers: typed text reflects `'speaking'` immediately; narration waits for the first
|
|
360
|
+
* model output.
|
|
361
|
+
*/
|
|
362
|
+
private sendReplyCreate;
|
|
363
|
+
/**
|
|
364
|
+
* Sends the `tool.result` frame (the provider speaks the result as the turn's
|
|
365
|
+
* continuation — no explicit generation trigger exists or is needed) and eagerly marks
|
|
366
|
+
* the agent busy so a queued narration can't slip in before the spoken result.
|
|
367
|
+
*/
|
|
368
|
+
private sendToolResultFrame;
|
|
369
|
+
/** The base prompt plus every accumulated context note under a "Background updates" heading. */
|
|
370
|
+
private composePromptWithNotes;
|
|
371
|
+
/** JSON-serializes and sends one client frame (no-op once the socket is gone). */
|
|
372
|
+
private sendFrame;
|
|
373
|
+
/** Resets the per-session response state machine (used on Disconnect). */
|
|
374
|
+
private resetResponseState;
|
|
375
|
+
/** Updates the client's own state view and emits the change to the host. */
|
|
376
|
+
private setState;
|
|
377
|
+
}
|
|
378
|
+
/**
|
|
379
|
+
* Tree-shaking prevention: bundlers cannot see that {@link AssemblyAIRealtimeClient} is
|
|
380
|
+
* instantiated dynamically through the ClassFactory, so a consumer must call this no-op
|
|
381
|
+
* to create a static code path that keeps the `@RegisterClass` side effect alive.
|
|
382
|
+
*/
|
|
383
|
+
export declare function LoadAssemblyAIRealtimeClient(): void;
|
|
384
|
+
//# sourceMappingURL=assemblyAIRealtimeClient.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"assemblyAIRealtimeClient.d.ts","sourceRoot":"","sources":["../../src/drivers/assemblyAIRealtimeClient.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,2BAA2B,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAC7E,OAAO,EAAE,kBAAkB,EAAuB,MAAM,+BAA+B,CAAC;AAExF,OAAO,EAAE,oBAAoB,EAAuB,MAAM,sBAAsB,CAAC;AAEjF,OAAO,EAAuB,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAI1E,6FAA6F;AAC7F,eAAO,MAAM,uBAAuB,sCAAsC,CAAC;AAE3E;;;;GAIG;AACH,eAAO,MAAM,0BAA0B,QAAQ,CAAC;AAIhD,wFAAwF;AACxF,MAAM,WAAW,qBAAqB;IAClC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,0DAA0D;IAC1D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,6EAA6E;IAC7E,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,sDAAsD;IACtD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,mFAAmF;IACnF,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,yEAAyE;IACzE,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,2EAA2E;IAC3E,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,0FAA0F;IAC1F,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,yDAAyD;IACzD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,mCAAmC;IACnC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,kFAAkF;IAClF,SAAS,CAAC,EAAE,UAAU,CAAC;IACvB,6CAA6C;IAC7C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,sDAAsD;IACtD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,kEAAkE;IAClE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,8DAA8D;IAC9D,OAAO,CAAC,EAAE,UAAU,CAAC;IACrB,wDAAwD;IACxD,wBAAwB,CAAC,EAAE,MAAM,CAAC;CACrC;AAID;;;;;GAKG;AACH,MAAM,WAAW,uBAAuB;IACpC,uCAAuC;IACvC,MAAM,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,IAAI,CAAC;IAC5B,4DAA4D;IAC5D,SAAS,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC;IAC3C,+CAA+C;IAC/C,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC;IAC5C,mDAAmD;IACnD,OAAO,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,IAAI,CAAC;IAC7B,8CAA8C;IAC9C,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,4CAA4C;IAC5C,KAAK,IAAI,IAAI,CAAC;CACjB;AAkBD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsDG;AACH,qBACa,wBAAyB,SAAQ,kBAAkB;IAE5D,OAAO,CAAC,MAAM,CAAwC;IACtD,OAAO,CAAC,SAAS,CAA4B;IAC7C,OAAO,CAAC,UAAU,CAA+B;IACjD,OAAO,CAAC,QAAQ,CAAqC;IAGrD,qFAAqF;IACrF,OAAO,CAAC,cAAc,CAAS;IAC/B,0FAA0F;IAC1F,OAAO,CAAC,kBAAkB,CAAoC;IAC9D,wFAAwF;IACxF,OAAO,CAAC,WAAW,CAAyB;IAC5C;;;OAGG;IACH,OAAO,CAAC,kBAAkB,CAAS;IACnC;;;;OAIG;IACH,OAAO,CAAC,kBAAkB,CAAqB;IAC/C,qFAAqF;IACrF,OAAO,CAAC,gBAAgB,CAAS;IAGjC,sFAAsF;IACtF,OAAO,CAAC,iBAAiB,CAAuB;IAChD,iFAAiF;IACjF,OAAO,CAAC,UAAU,CAAuB;IACzC,qFAAqF;IACrF,OAAO,CAAC,eAAe,CAAS;IAChC;;;;OAIG;IACH,OAAO,CAAC,YAAY,CAAiC;IAGrD,4EAA4E;IAC5E,OAAO,CAAC,UAAU,CAAM;IACxB,yFAAyF;IACzF,OAAO,CAAC,YAAY,CAAgB;IAIpC;;;;;OAKG;IACU,OAAO,CAAC,MAAM,EAAE,2BAA2B,EAAE,SAAS,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAgEhG;;;;;OAKG;IACU,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IA8BxC;;;;;;;;;OASG;IACI,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAUnC;;;;;;;;OAQG;IACI,oBAAoB,IAAI,IAAI;IAmBnC;;;;;OAKG;IACI,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAQ1C;;;;;OAKG;IACI,mBAAmB,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI;IAOtD;;;;;;;;;OASG;IACI,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,IAAI;IAc/D;;;;OAIG;IACI,QAAQ,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI;IAOrC,kBAAkB;IAClB,IAAW,MAAM,IAAI,OAAO,CAE3B;IAED;;;;;;OAMG;IACH,IAAW,cAAc,IAAI,OAAO,CAEnC;IAID;;;;;OAKG;IACH,SAAS,CAAC,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,uBAAuB;IAqB5D;;;;OAIG;cACa,gBAAgB,CAC5B,SAAS,EAAE,WAAW,EACtB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,CAAC,WAAW,EAAE,MAAM,KAAK,IAAI,GAC1C,OAAO,CAAC,cAAc,CAAC;IAI1B;;;OAGG;IACH,SAAS,CAAC,cAAc,CAAC,UAAU,EAAE,MAAM,GAAG,oBAAoB;IAMlE,sFAAsF;IACtF,OAAO,CAAC,cAAc,CAA6B;IAEnD;;;;OAIG;IACH,OAAO,CAAC,kBAAkB;IAM1B,oEAAoE;IACpE,OAAO,CAAC,YAAY;IAMpB,oFAAoF;IACpF,OAAO,CAAC,iBAAiB;IAQzB;;;;OAIG;IACH,OAAO,CAAC,iBAAiB;IAazB;;;;;;;;OAQG;IACH,OAAO,CAAC,gBAAgB;IAwBxB,kEAAkE;IAClE,OAAO,CAAC,mBAAmB;IAU3B,yEAAyE;IACzE,OAAO,CAAC,iBAAiB;IAmDzB,wFAAwF;IACxF,OAAO,CAAC,kBAAkB;IAK1B;;;;OAIG;IACH,OAAO,CAAC,gBAAgB;IAQxB,2FAA2F;IAC3F,OAAO,CAAC,oBAAoB;IAM5B;;;;;OAKG;IACH,OAAO,CAAC,qBAAqB;IAM7B;;;;;;;OAOG;IACH,OAAO,CAAC,eAAe;IAcvB;;;;;;;;;OASG;IACH,OAAO,CAAC,cAAc;IAUtB;;;;;;;OAOG;IACH,OAAO,CAAC,mBAAmB;IAc3B;;;;OAIG;IACH,OAAO,CAAC,kBAAkB;IAQ1B;;;OAGG;IACH,OAAO,CAAC,qBAAqB;IAS7B;;;;OAIG;IACH,OAAO,CAAC,YAAY;IAQpB;;;OAGG;IACH,OAAO,CAAC,gBAAgB;IAOxB;;;;;OAKG;IACH,OAAO,CAAC,eAAe;IAYvB;;;;OAIG;IACH,OAAO,CAAC,mBAAmB;IAY3B,gGAAgG;IAChG,OAAO,CAAC,sBAAsB;IAO9B,kFAAkF;IAClF,OAAO,CAAC,SAAS;IAIjB,0EAA0E;IAC1E,OAAO,CAAC,kBAAkB;IAU1B,4EAA4E;IAC5E,OAAO,CAAC,QAAQ;CAInB;AAED;;;;GAIG;AACH,wBAAgB,4BAA4B,IAAI,IAAI,CAEnD"}
|