@elevenlabs/client 1.7.1 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/BaseConversation.d.ts +2 -2
- package/dist/BaseConversation.d.ts.map +1 -1
- package/dist/BaseConversation.js +18 -15
- package/dist/BaseConversation.js.map +1 -1
- package/dist/InputController.d.ts +9 -0
- package/dist/InputController.d.ts.map +1 -1
- package/dist/OutputController.d.ts +18 -0
- package/dist/OutputController.d.ts.map +1 -1
- package/dist/TextConversation.js +2 -2
- package/dist/TextConversation.js.map +1 -1
- package/dist/VoiceConversation.d.ts +3 -6
- package/dist/VoiceConversation.d.ts.map +1 -1
- package/dist/VoiceConversation.js +15 -69
- package/dist/VoiceConversation.js.map +1 -1
- package/dist/WebRTCAudioAdapter.d.ts +52 -0
- package/dist/WebRTCAudioAdapter.d.ts.map +1 -0
- package/dist/WebRTCAudioAdapter.js +17 -0
- package/dist/WebRTCAudioAdapter.js.map +1 -0
- package/dist/index.d.ts +3 -5
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -1
- package/dist/index.js.map +1 -1
- package/dist/internal.d.ts +6 -1
- package/dist/internal.d.ts.map +1 -1
- package/dist/internal.js +4 -1
- package/dist/internal.js.map +1 -1
- package/dist/lib.iife.js +3415 -3219
- package/dist/lib.iife.js.map +1 -1
- package/dist/platform/VoiceSessionSetup.d.ts +10 -15
- package/dist/platform/VoiceSessionSetup.d.ts.map +1 -1
- package/dist/platform/VoiceSessionSetup.js +20 -62
- package/dist/platform/VoiceSessionSetup.js.map +1 -1
- package/dist/platform/tsconfig.test.tsbuildinfo +1 -0
- package/dist/platform/web/VoiceSessionSetup.d.ts +9 -0
- package/dist/platform/web/VoiceSessionSetup.d.ts.map +1 -0
- package/dist/platform/web/VoiceSessionSetup.js +154 -0
- package/dist/platform/web/VoiceSessionSetup.js.map +1 -0
- package/dist/platform/web/addLibsamplerateModule.d.ts.map +1 -0
- package/dist/platform/web/addLibsamplerateModule.js.map +1 -0
- package/dist/platform/web/audioConcatProcessor.generated.d.ts.map +1 -0
- package/dist/platform/web/audioConcatProcessor.generated.js.map +1 -0
- package/dist/platform/web/compatibility.d.ts.map +1 -0
- package/dist/platform/web/compatibility.js.map +1 -0
- package/dist/platform/web/createWorkletModuleLoader.d.ts.map +1 -0
- package/dist/platform/web/createWorkletModuleLoader.js.map +1 -0
- package/dist/platform/web/index.d.ts +3 -0
- package/dist/platform/web/index.d.ts.map +1 -0
- package/dist/platform/web/index.js +12 -0
- package/dist/platform/web/index.js.map +1 -0
- package/dist/{utils → platform/web}/input.d.ts +3 -12
- package/dist/platform/web/input.d.ts.map +1 -0
- package/dist/platform/web/input.js.map +1 -0
- package/dist/{utils → platform/web}/output.d.ts +3 -21
- package/dist/platform/web/output.d.ts.map +1 -0
- package/dist/platform/web/output.js.map +1 -0
- package/dist/platform/web/rawAudioProcessor.generated.d.ts.map +1 -0
- package/dist/platform/web/rawAudioProcessor.generated.js.map +1 -0
- package/dist/platform/web/scribeAudioProcessor.generated.d.ts.map +1 -0
- package/dist/platform/web/scribeAudioProcessor.generated.js.map +1 -0
- package/dist/platform/web/scribeMicrophone.d.ts +10 -0
- package/dist/platform/web/scribeMicrophone.d.ts.map +1 -0
- package/dist/platform/web/scribeMicrophone.js +65 -0
- package/dist/platform/web/scribeMicrophone.js.map +1 -0
- package/dist/platform/web/volumeProvider.d.ts +4 -0
- package/dist/platform/web/volumeProvider.d.ts.map +1 -0
- package/dist/platform/web/volumeProvider.js +24 -0
- package/dist/platform/web/volumeProvider.js.map +1 -0
- package/dist/platform/web/webAudioAdapter.d.ts +23 -0
- package/dist/platform/web/webAudioAdapter.d.ts.map +1 -0
- package/dist/platform/web/webAudioAdapter.js +127 -0
- package/dist/platform/web/webAudioAdapter.js.map +1 -0
- package/dist/runtime.d.ts +181 -0
- package/dist/runtime.d.ts.map +1 -0
- package/dist/runtime.js +34 -0
- package/dist/runtime.js.map +1 -0
- package/dist/scribe/microphone.d.ts +31 -0
- package/dist/scribe/microphone.d.ts.map +1 -0
- package/dist/scribe/microphone.js +23 -0
- package/dist/scribe/microphone.js.map +1 -0
- package/dist/scribe/scribe.d.ts +1 -1
- package/dist/scribe/scribe.d.ts.map +1 -1
- package/dist/scribe/scribe.js +6 -62
- package/dist/scribe/scribe.js.map +1 -1
- package/dist/types.d.ts +12 -2
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js.map +1 -1
- package/dist/utils/BaseConnection.d.ts +1 -1
- package/dist/utils/BaseConnection.d.ts.map +1 -1
- package/dist/utils/WebRTCConnection.d.ts +4 -7
- package/dist/utils/WebRTCConnection.d.ts.map +1 -1
- package/dist/utils/WebRTCConnection.js +51 -128
- package/dist/utils/WebRTCConnection.js.map +1 -1
- package/dist/utils/WebSocketConnection.d.ts +1 -1
- package/dist/utils/WebSocketConnection.d.ts.map +1 -1
- package/dist/utils/WebSocketConnection.js +14 -12
- package/dist/utils/WebSocketConnection.js.map +1 -1
- package/dist/utils/applyDelay.d.ts +3 -2
- package/dist/utils/applyDelay.d.ts.map +1 -1
- package/dist/utils/applyDelay.js +18 -12
- package/dist/utils/applyDelay.js.map +1 -1
- package/dist/utils/assert.d.ts +3 -0
- package/dist/utils/assert.d.ts.map +1 -0
- package/dist/utils/assert.js +9 -0
- package/dist/utils/assert.js.map +1 -0
- package/dist/utils/attachConnectionToOutput.d.ts +1 -1
- package/dist/utils/attachConnectionToOutput.d.ts.map +1 -1
- package/dist/utils/attachInputToConnection.d.ts +1 -1
- package/dist/utils/attachInputToConnection.d.ts.map +1 -1
- package/dist/utils/audio.d.ts.map +1 -1
- package/dist/utils/audio.js +2 -3
- package/dist/utils/audio.js.map +1 -1
- package/dist/utils/connection.d.ts +1 -1
- package/dist/utils/connection.d.ts.map +1 -1
- package/dist/utils/connection.js.map +1 -1
- package/dist/utils/errors.d.ts.map +1 -1
- package/dist/utils/errors.js +5 -1
- package/dist/utils/errors.js.map +1 -1
- package/dist/utils/mergeOptions.d.ts.map +1 -1
- package/dist/utils/mergeOptions.js +1 -3
- package/dist/utils/mergeOptions.js.map +1 -1
- package/dist/utils/volumeProvider.d.ts +0 -1
- package/dist/utils/volumeProvider.d.ts.map +1 -1
- package/dist/utils/volumeProvider.js +0 -22
- package/dist/utils/volumeProvider.js.map +1 -1
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +3 -2
- package/dist/utils/addLibsamplerateModule.d.ts.map +0 -1
- package/dist/utils/addLibsamplerateModule.js.map +0 -1
- package/dist/utils/audioConcatProcessor.generated.d.ts.map +0 -1
- package/dist/utils/audioConcatProcessor.generated.js.map +0 -1
- package/dist/utils/compatibility.d.ts.map +0 -1
- package/dist/utils/compatibility.js.map +0 -1
- package/dist/utils/createWorkletModuleLoader.d.ts.map +0 -1
- package/dist/utils/createWorkletModuleLoader.js.map +0 -1
- package/dist/utils/input.d.ts.map +0 -1
- package/dist/utils/input.js.map +0 -1
- package/dist/utils/output.d.ts.map +0 -1
- package/dist/utils/output.js.map +0 -1
- package/dist/utils/rawAudioProcessor.generated.d.ts.map +0 -1
- package/dist/utils/rawAudioProcessor.generated.js.map +0 -1
- package/dist/utils/scribeAudioProcessor.generated.d.ts.map +0 -1
- package/dist/utils/scribeAudioProcessor.generated.js.map +0 -1
- /package/dist/{utils → platform/web}/addLibsamplerateModule.d.ts +0 -0
- /package/dist/{utils → platform/web}/addLibsamplerateModule.js +0 -0
- /package/dist/{utils → platform/web}/audioConcatProcessor.generated.d.ts +0 -0
- /package/dist/{utils → platform/web}/audioConcatProcessor.generated.js +0 -0
- /package/dist/{utils → platform/web}/compatibility.d.ts +0 -0
- /package/dist/{utils → platform/web}/compatibility.js +0 -0
- /package/dist/{utils → platform/web}/createWorkletModuleLoader.d.ts +0 -0
- /package/dist/{utils → platform/web}/createWorkletModuleLoader.js +0 -0
- /package/dist/{utils → platform/web}/input.js +0 -0
- /package/dist/{utils → platform/web}/output.js +0 -0
- /package/dist/{utils → platform/web}/rawAudioProcessor.generated.d.ts +0 -0
- /package/dist/{utils → platform/web}/rawAudioProcessor.generated.js +0 -0
- /package/dist/{utils → platform/web}/scribeAudioProcessor.generated.d.ts +0 -0
- /package/dist/{utils → platform/web}/scribeAudioProcessor.generated.js +0 -0
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Platform-agnostic microphone streaming interface for Scribe.
|
|
3
|
+
*
|
|
4
|
+
* The web implementation (`platform/web/scribeMicrophone.ts`) provides the
|
|
5
|
+
* actual AudioContext + getUserMedia pipeline; other platforms can supply
|
|
6
|
+
* their own implementation via `setScribeMicrophoneSetup`.
|
|
7
|
+
*/
|
|
8
|
+
export interface ScribeMicrophoneConfig {
|
|
9
|
+
deviceId?: MediaDeviceConstraint;
|
|
10
|
+
echoCancellation?: boolean;
|
|
11
|
+
noiseSuppression?: boolean;
|
|
12
|
+
autoGainControl?: boolean;
|
|
13
|
+
channelCount?: number;
|
|
14
|
+
}
|
|
15
|
+
export interface ScribeMicrophoneResult {
|
|
16
|
+
/** Track reference so the connection can toggle `track.enabled` for mute/unmute. */
|
|
17
|
+
mediaStreamTrack: MediaStreamTrack;
|
|
18
|
+
/** Tear down the audio pipeline and release hardware. */
|
|
19
|
+
cleanup: () => void;
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Sets up microphone capture and streams PCM audio to the provided callback.
|
|
23
|
+
*
|
|
24
|
+
* @param config - Microphone constraints
|
|
25
|
+
* @param onAudioData - Called with base64-encoded PCM16 chunks
|
|
26
|
+
* @returns A result containing the track and cleanup function
|
|
27
|
+
*/
|
|
28
|
+
export type ScribeMicrophoneSetup = (config: ScribeMicrophoneConfig, onAudioData: (base64Audio: string) => void) => Promise<ScribeMicrophoneResult>;
|
|
29
|
+
export declare function setScribeMicrophoneSetup(setup: ScribeMicrophoneSetup): void;
|
|
30
|
+
export declare function getScribeMicrophoneSetup(): ScribeMicrophoneSetup;
|
|
31
|
+
//# sourceMappingURL=microphone.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"microphone.d.ts","sourceRoot":"","sources":["../../src/scribe/microphone.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,MAAM,WAAW,sBAAsB;IACrC,QAAQ,CAAC,EAAE,qBAAqB,CAAC;IACjC,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,sBAAsB;IACrC,oFAAoF;IACpF,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,yDAAyD;IACzD,OAAO,EAAE,MAAM,IAAI,CAAC;CACrB;AAED;;;;;;GAMG;AACH,MAAM,MAAM,qBAAqB,GAAG,CAClC,MAAM,EAAE,sBAAsB,EAC9B,WAAW,EAAE,CAAC,WAAW,EAAE,MAAM,KAAK,IAAI,KACvC,OAAO,CAAC,sBAAsB,CAAC,CAAC;AAQrC,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,qBAAqB,GAAG,IAAI,CAE3E;AAED,wBAAgB,wBAAwB,IAAI,qBAAqB,CAShE"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Platform-agnostic microphone streaming interface for Scribe.
|
|
3
|
+
*
|
|
4
|
+
* The web implementation (`platform/web/scribeMicrophone.ts`) provides the
|
|
5
|
+
* actual AudioContext + getUserMedia pipeline; other platforms can supply
|
|
6
|
+
* their own implementation via `setScribeMicrophoneSetup`.
|
|
7
|
+
*/
|
|
8
|
+
// ---------------------------------------------------------------------------
|
|
9
|
+
// Injectable factory
|
|
10
|
+
// ---------------------------------------------------------------------------
|
|
11
|
+
let microphoneSetup = null;
|
|
12
|
+
export function setScribeMicrophoneSetup(setup) {
|
|
13
|
+
microphoneSetup = setup;
|
|
14
|
+
}
|
|
15
|
+
export function getScribeMicrophoneSetup() {
|
|
16
|
+
if (!microphoneSetup) {
|
|
17
|
+
throw new Error("No Scribe microphone implementation registered. " +
|
|
18
|
+
"Import '@elevenlabs/client/platform/web' or provide a custom " +
|
|
19
|
+
"implementation via setScribeMicrophoneSetup().");
|
|
20
|
+
}
|
|
21
|
+
return microphoneSetup;
|
|
22
|
+
}
|
|
23
|
+
//# sourceMappingURL=microphone.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"microphone.js","sourceRoot":"","sources":["../../src/scribe/microphone.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AA6BH,8EAA8E;AAC9E,qBAAqB;AACrB,8EAA8E;AAE9E,IAAI,eAAe,GAAiC,IAAI,CAAC;AAEzD,MAAM,UAAU,wBAAwB,CAAC,KAA4B;IACnE,eAAe,GAAG,KAAK,CAAC;AAC1B,CAAC;AAED,MAAM,UAAU,wBAAwB;IACtC,IAAI,CAAC,eAAe,EAAE,CAAC;QACrB,MAAM,IAAI,KAAK,CACb,kDAAkD;YAChD,+DAA+D;YAC/D,gDAAgD,CACnD,CAAC;IACJ,CAAC;IACD,OAAO,eAAe,CAAC;AACzB,CAAC"}
|
package/dist/scribe/scribe.d.ts
CHANGED
|
@@ -83,7 +83,7 @@ export interface AudioOptions extends BaseOptions {
|
|
|
83
83
|
*/
|
|
84
84
|
export interface MicrophoneOptions extends BaseOptions {
|
|
85
85
|
microphone?: {
|
|
86
|
-
deviceId?:
|
|
86
|
+
deviceId?: MediaDeviceConstraint;
|
|
87
87
|
echoCancellation?: boolean;
|
|
88
88
|
noiseSuppression?: boolean;
|
|
89
89
|
autoGainControl?: boolean;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scribe.d.ts","sourceRoot":"","sources":["../../src/scribe/scribe.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AAGrD,oBAAY,WAAW;IACrB,QAAQ,aAAa;IACrB,SAAS,cAAc;IACvB,SAAS,cAAc;IACvB,SAAS,cAAc;IACvB,SAAS,cAAc;IACvB,SAAS,cAAc;IACvB,SAAS,cAAc;CACxB;AAED,oBAAY,cAAc;IACxB,MAAM,WAAW;IACjB,GAAG,QAAQ;CACZ;AAED,UAAU,WAAW;IACnB;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;;OAGG;IACH,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC;;;OAGG;IACH,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B;;;OAGG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B;;;OAGG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;OAGG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB;;;OAGG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,YAAa,SAAQ,WAAW;IAC/C,WAAW,EAAE,WAAW,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,KAAK,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAkB,SAAQ,WAAW;IACpD,UAAU,CAAC,EAAE;QACX,QAAQ,CAAC,EAAE,
|
|
1
|
+
{"version":3,"file":"scribe.d.ts","sourceRoot":"","sources":["../../src/scribe/scribe.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AAGrD,oBAAY,WAAW;IACrB,QAAQ,aAAa;IACrB,SAAS,cAAc;IACvB,SAAS,cAAc;IACvB,SAAS,cAAc;IACvB,SAAS,cAAc;IACvB,SAAS,cAAc;IACvB,SAAS,cAAc;CACxB;AAED,oBAAY,cAAc;IACxB,MAAM,WAAW;IACjB,GAAG,QAAQ;CACZ;AAED,UAAU,WAAW;IACnB;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;;OAGG;IACH,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC;;;OAGG;IACH,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B;;;OAGG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B;;;OAGG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;OAGG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB;;;OAGG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,YAAa,SAAQ,WAAW;IAC/C,WAAW,EAAE,WAAW,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,KAAK,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAkB,SAAQ,WAAW;IACpD,UAAU,CAAC,EAAE;QACX,QAAQ,CAAC,EAAE,qBAAqB,CAAC;QACjC,gBAAgB,CAAC,EAAE,OAAO,CAAC;QAC3B,gBAAgB,CAAC,EAAE,OAAO,CAAC;QAC3B,eAAe,CAAC,EAAE,OAAO,CAAC;QAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;KACvB,CAAC;IACF,WAAW,CAAC,EAAE,KAAK,CAAC;IACpB,UAAU,CAAC,EAAE,KAAK,CAAC;CACpB;AAED;;;GAGG;AAGH,qBAAa,cAAc;IACzB,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,gBAAgB,CAA6B;IAErE,OAAO,CAAC,MAAM,CAAC,eAAe;IAM9B,OAAO,CAAC,MAAM,CAAC,iBAAiB;IAiFhC;;;;;;;;;;;;;;;;;;;;;;;;;;OA0BG;WACW,OAAO,CACnB,OAAO,EAAE,YAAY,GAAG,iBAAiB,GACxC,kBAAkB;mBAgCA,oBAAoB;CAiB1C"}
|
package/dist/scribe/scribe.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { RealtimeConnection } from "./connection.js";
|
|
2
|
-
import {
|
|
2
|
+
import { getScribeMicrophoneSetup } from "./microphone.js";
|
|
3
3
|
export var AudioFormat;
|
|
4
4
|
(function (AudioFormat) {
|
|
5
5
|
AudioFormat["PCM_8000"] = "pcm_8000";
|
|
@@ -131,69 +131,13 @@ export class ScribeRealtime {
|
|
|
131
131
|
return connection;
|
|
132
132
|
}
|
|
133
133
|
static async streamFromMicrophone(options, connection) {
|
|
134
|
-
const TARGET_SAMPLE_RATE = 16000;
|
|
135
134
|
try {
|
|
136
|
-
|
|
137
|
-
const
|
|
138
|
-
audio: {
|
|
139
|
-
deviceId: options.microphone?.deviceId,
|
|
140
|
-
echoCancellation: options.microphone?.echoCancellation ?? true,
|
|
141
|
-
noiseSuppression: options.microphone?.noiseSuppression ?? true,
|
|
142
|
-
autoGainControl: options.microphone?.autoGainControl ?? true,
|
|
143
|
-
channelCount: options.microphone?.channelCount ?? 1,
|
|
144
|
-
sampleRate: { ideal: TARGET_SAMPLE_RATE },
|
|
145
|
-
},
|
|
146
|
-
});
|
|
147
|
-
// Get the actual sample rate from the stream - the ideal may not have been honored
|
|
148
|
-
const trackSettings = stream.getAudioTracks()[0]?.getSettings();
|
|
149
|
-
const streamSampleRate = trackSettings?.sampleRate;
|
|
150
|
-
// Create audio context matching the stream's sample rate to avoid Firefox errors
|
|
151
|
-
// Firefox requires the AudioContext to match the microphone's native sample rate
|
|
152
|
-
const audioContext = new AudioContext(streamSampleRate ? { sampleRate: streamSampleRate } : {});
|
|
153
|
-
// Load scribe worklet
|
|
154
|
-
await loadScribeAudioProcessor(audioContext.audioWorklet);
|
|
155
|
-
// Set up audio pipeline
|
|
156
|
-
const source = audioContext.createMediaStreamSource(stream);
|
|
157
|
-
const scribeNode = new AudioWorkletNode(audioContext, "scribeAudioProcessor");
|
|
158
|
-
// Configure the worklet with sample rate info for resampling
|
|
159
|
-
// (only needed when AudioContext sample rate differs from target)
|
|
160
|
-
if (audioContext.sampleRate !== TARGET_SAMPLE_RATE) {
|
|
161
|
-
scribeNode.port.postMessage({
|
|
162
|
-
type: "configure",
|
|
163
|
-
inputSampleRate: audioContext.sampleRate,
|
|
164
|
-
outputSampleRate: TARGET_SAMPLE_RATE,
|
|
165
|
-
});
|
|
166
|
-
}
|
|
167
|
-
// Store the track so mute()/unmute() can toggle track.enabled.
|
|
168
|
-
const [audioTrack] = stream.getAudioTracks();
|
|
169
|
-
connection._mediaStreamTrack = audioTrack;
|
|
170
|
-
// Handle audio data from worklet
|
|
171
|
-
scribeNode.port.onmessage = event => {
|
|
172
|
-
const { audioData } = event.data;
|
|
173
|
-
// Convert ArrayBuffer to base64
|
|
174
|
-
const bytes = new Uint8Array(audioData);
|
|
175
|
-
let binary = "";
|
|
176
|
-
for (let i = 0; i < bytes.length; i++) {
|
|
177
|
-
binary += String.fromCharCode(bytes[i]);
|
|
178
|
-
}
|
|
179
|
-
const base64Audio = btoa(binary);
|
|
135
|
+
const setup = getScribeMicrophoneSetup();
|
|
136
|
+
const result = await setup(options.microphone ?? {}, base64Audio => {
|
|
180
137
|
connection.send({ audioBase64: base64Audio });
|
|
181
|
-
};
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
// Resume audio context if needed
|
|
185
|
-
if (audioContext.state === "suspended") {
|
|
186
|
-
await audioContext.resume();
|
|
187
|
-
}
|
|
188
|
-
// Store cleanup function
|
|
189
|
-
connection._audioCleanup = () => {
|
|
190
|
-
stream.getTracks().forEach(track => {
|
|
191
|
-
track.stop();
|
|
192
|
-
});
|
|
193
|
-
source.disconnect();
|
|
194
|
-
scribeNode.disconnect();
|
|
195
|
-
audioContext.close();
|
|
196
|
-
};
|
|
138
|
+
});
|
|
139
|
+
connection._mediaStreamTrack = result.mediaStreamTrack;
|
|
140
|
+
connection._audioCleanup = result.cleanup;
|
|
197
141
|
}
|
|
198
142
|
catch (error) {
|
|
199
143
|
console.error("Failed to start microphone streaming:", error);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scribe.js","sourceRoot":"","sources":["../../src/scribe/scribe.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AACrD,OAAO,EAAE,wBAAwB,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"scribe.js","sourceRoot":"","sources":["../../src/scribe/scribe.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AACrD,OAAO,EAAE,wBAAwB,EAAE,MAAM,iBAAiB,CAAC;AAE3D,MAAM,CAAN,IAAY,WAQX;AARD,WAAY,WAAW;IACrB,oCAAqB,CAAA;IACrB,sCAAuB,CAAA;IACvB,sCAAuB,CAAA;IACvB,sCAAuB,CAAA;IACvB,sCAAuB,CAAA;IACvB,sCAAuB,CAAA;IACvB,sCAAuB,CAAA;AACzB,CAAC,EARW,WAAW,KAAX,WAAW,QAQtB;AAED,MAAM,CAAN,IAAY,cAGX;AAHD,WAAY,cAAc;IACxB,mCAAiB,CAAA;IACjB,6BAAW,CAAA;AACb,CAAC,EAHW,cAAc,KAAd,cAAc,QAGzB;AAqFD;;;GAGG;AAEH,sGAAsG;AACtG,MAAM,OAAO,cAAc;IACjB,MAAM,CAAU,gBAAgB,GAAG,yBAAyB,CAAC;IAE7D,MAAM,CAAC,eAAe,CAC5B,UAAkB,cAAc,CAAC,gBAAgB;QAEjD,OAAO,GAAG,OAAO,6BAA6B,CAAC;IACjD,CAAC;IAEO,MAAM,CAAC,iBAAiB,CAC9B,OAAyC;QAEzC,MAAM,OAAO,GAAG,cAAc,CAAC,eAAe,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAChE,MAAM,MAAM,GAAG,IAAI,eAAe,EAAE,CAAC;QAErC,wDAAwD;QACxD,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;QAC3C,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;QAEtC,uDAAuD;QACvD,IAAI,OAAO,CAAC,cAAc,KAAK,SAAS,EAAE,CAAC;YACzC,MAAM,CAAC,MAAM,CAAC,iBAAiB,EAAE,OAAO,CAAC,cAAc,CAAC,CAAC;QAC3D,CAAC;QACD,IAAI,OAAO,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;YACtC,MAAM,CAAC,MAAM,CAAC,cAAc,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;QACrD,CAAC;QACD,IAAI,OAAO,CAAC,uBAAuB,KAAK,SAAS,EAAE,CAAC;YAClD,IACE,OAAO,CAAC,uBAAuB,IAAI,GAAG;gBACtC,OAAO,CAAC,uBAAuB,GAAG,GAAG,EACrC,CAAC;gBACD,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;YACzE,CAAC;YACD,MAAM,CAAC,MAAM,CACX,4BAA4B,EAC5B,OAAO,CAAC,uBAAuB,CAAC,QAAQ,EAAE,CAC3C,CAAC;QACJ,CAAC;QACD,IAAI,OAAO,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;YACvC,IAAI,OAAO,CAAC,YAAY,GAAG,GAAG,IAAI,OAAO,CAAC,YAAY,GAAG,GAAG,EAAE,CAAC;gBAC7D,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;YAC9D,CAAC;YACD,MAAM,CAAC,MAAM,CAAC,eAAe,EAAE,OAAO,CAAC,YAAY,CAAC,QAAQ,EAAE,CAAC,CAAC;QAClE,CAAC;QACD,IAAI,OAAO,CAAC,mBAAmB,KAAK,SAAS,EAAE,CAAC;YAC9C,IACE,OAAO,CAAC,mBAAmB,IAAI,EAAE;gBACjC,OAAO,CAAC,mBAAmB,GAAG,IAAI,EAClC,CAAC;gBACD,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;YACrE,CAAC;YACD,MAAM,CAAC,MAAM,CACX,wBAAwB,EACxB,OAAO,CAAC,mBAAmB,CAAC,QAAQ,EAAE,CACvC,CAAC;QACJ,CAAC;QACD,IAAI,OAAO,CAAC,oBAAoB,KAAK,SAAS,EAAE,CAAC;YAC/C,IACE,OAAO,CAAC,oBAAoB,IAAI,EAAE;gBAClC,OAAO,CAAC,oBAAoB,GAAG,IAAI,EACnC,CAAC;gBACD,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;YACtE,CAAC;YACD,MAAM,CAAC,MAAM,CACX,yBAAyB,EACzB,OAAO,CAAC,oBAAoB,CAAC,QAAQ,EAAE,CACxC,CAAC;QACJ,CAAC;QACD,IAAI,OAAO,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;YACvC,MAAM,CAAC,MAAM,CAAC,eAAe,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC;QACvD,CAAC;QACD,IAAI,OAAO,CAAC,iBAAiB,KAAK,SAAS,EAAE,CAAC;YAC5C,MAAM,CAAC,MAAM,CACX,oBAAoB,EACpB,OAAO,CAAC,iBAAiB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAC7C,CAAC;QACJ,CAAC;QACD,IAAI,OAAO,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;YACnC,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;gBACpC,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;YAClC,CAAC;QACH,CAAC;QACD,IAAI,OAAO,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;YACrC,MAAM,CAAC,MAAM,CAAC,aAAa,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QACtE,CAAC;QAED,MAAM,WAAW,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC;QACtC,OAAO,WAAW,CAAC,CAAC,CAAC,GAAG,OAAO,IAAI,WAAW,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;IAC7D,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;OA0BG;IACI,MAAM,CAAC,OAAO,CACnB,OAAyC;QAEzC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;YACrB,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;QACzC,CAAC;QAED,4FAA4F;QAC5F,MAAM,UAAU,GACd,YAAY,IAAI,OAAO,IAAI,OAAO,CAAC,UAAU;YAC3C,CAAC,CAAC,KAAK;YACP,CAAC,CAAE,OAAwB,CAAC,UAAU,CAAC;QAC3C,MAAM,UAAU,GAAG,IAAI,kBAAkB,CAAC,UAAU,CAAC,CAAC;QAEtD,4CAA4C;QAC5C,MAAM,GAAG,GAAG,cAAc,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;QAEtD,MAAM,SAAS,GAAG,IAAI,SAAS,CAAC,GAAG,CAAC,CAAC;QAErC,+CAA+C;QAC/C,IAAI,YAAY,IAAI,OAAO,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;YAClD,SAAS,CAAC,gBAAgB,CAAC,MAAM,EAAE,GAAG,EAAE;gBACtC,cAAc,CAAC,oBAAoB,CACjC,OAA4B,EAC5B,UAAU,CACX,CAAC;YACJ,CAAC,CAAC,CAAC;QACL,CAAC;QAED,UAAU,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;QAEnC,OAAO,UAAU,CAAC;IACpB,CAAC;IAEO,MAAM,CAAC,KAAK,CAAC,oBAAoB,CACvC,OAA0B,EAC1B,UAA8B;QAE9B,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,wBAAwB,EAAE,CAAC;YACzC,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,UAAU,IAAI,EAAE,EAAE,WAAW,CAAC,EAAE;gBACjE,UAAU,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,WAAW,EAAE,CAAC,CAAC;YAChD,CAAC,CAAC,CAAC;YAEH,UAAU,CAAC,iBAAiB,GAAG,MAAM,CAAC,gBAAgB,CAAC;YACvD,UAAU,CAAC,aAAa,GAAG,MAAM,CAAC,OAAO,CAAC;QAC5C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,uCAAuC,EAAE,KAAK,CAAC,CAAC;YAC9D,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC"}
|
package/dist/types.d.ts
CHANGED
|
@@ -12,18 +12,28 @@ export type Mode = "speaking" | "listening";
|
|
|
12
12
|
* Connection status of the conversation
|
|
13
13
|
*/
|
|
14
14
|
export type Status = "disconnected" | "connecting" | "connected" | "disconnecting";
|
|
15
|
+
/**
|
|
16
|
+
* Platform-agnostic representation of the event that triggered a disconnection.
|
|
17
|
+
* Replaces the former `Event` / `CloseEvent` DOM constructors which are
|
|
18
|
+
* not available on React Native.
|
|
19
|
+
*/
|
|
20
|
+
export type DisconnectionContext = {
|
|
21
|
+
type: string;
|
|
22
|
+
reason?: string;
|
|
23
|
+
code?: number;
|
|
24
|
+
};
|
|
15
25
|
/**
|
|
16
26
|
* Reason for the disconnection
|
|
17
27
|
*/
|
|
18
28
|
export type DisconnectionDetails = {
|
|
19
29
|
reason: "error";
|
|
20
30
|
message: string;
|
|
21
|
-
context:
|
|
31
|
+
context: DisconnectionContext;
|
|
22
32
|
closeCode?: number;
|
|
23
33
|
closeReason?: string;
|
|
24
34
|
} | {
|
|
25
35
|
reason: "agent";
|
|
26
|
-
context?:
|
|
36
|
+
context?: DisconnectionContext;
|
|
27
37
|
closeCode?: number;
|
|
28
38
|
closeReason?: string;
|
|
29
39
|
} | {
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,mBAAmB,EACnB,yBAAyB,EACzB,sBAAsB,EACtB,8BAA8B,EAC9B,2BAA2B,EAC3B,4BAA4B,EAC5B,uCAAuC,EACvC,oBAAoB,EACpB,0BAA0B,EAC1B,YAAY,EACZ,uBAAuB,EACvB,gCAAgC,EACjC,MAAM,mBAAmB,CAAC;AAE3B;;GAEG;AACH,MAAM,MAAM,IAAI,GAAG,MAAM,GAAG,OAAO,CAAC;AAEpC,MAAM,MAAM,mBAAmB,GAAG,mBAAmB,CAAC;AAEtD;;GAEG;AACH,MAAM,MAAM,IAAI,GAAG,UAAU,GAAG,WAAW,CAAC;AAE5C;;GAEG;AACH,MAAM,MAAM,MAAM,GACd,cAAc,GACd,YAAY,GACZ,WAAW,GACX,eAAe,CAAC;AAEpB;;GAEG;AACH,MAAM,MAAM,oBAAoB,GAC5B;IACE,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,mBAAmB,EACnB,yBAAyB,EACzB,sBAAsB,EACtB,8BAA8B,EAC9B,2BAA2B,EAC3B,4BAA4B,EAC5B,uCAAuC,EACvC,oBAAoB,EACpB,0BAA0B,EAC1B,YAAY,EACZ,uBAAuB,EACvB,gCAAgC,EACjC,MAAM,mBAAmB,CAAC;AAE3B;;GAEG;AACH,MAAM,MAAM,IAAI,GAAG,MAAM,GAAG,OAAO,CAAC;AAEpC,MAAM,MAAM,mBAAmB,GAAG,mBAAmB,CAAC;AAEtD;;GAEG;AACH,MAAM,MAAM,IAAI,GAAG,UAAU,GAAG,WAAW,CAAC;AAE5C;;GAEG;AACH,MAAM,MAAM,MAAM,GACd,cAAc,GACd,YAAY,GACZ,WAAW,GACX,eAAe,CAAC;AAEpB;;;;GAIG;AACH,MAAM,MAAM,oBAAoB,GAAG;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,CAAC;AAEF;;GAEG;AACH,MAAM,MAAM,oBAAoB,GAC5B;IACE,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,oBAAoB,CAAC;IAC9B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,GACD;IACE,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,GACD;IACE,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEN,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,IAAI,EAAE,IAAI,CAAC;CACZ;AAED;;GAEG;AACH,MAAM,MAAM,SAAS,GAAG;IACtB,SAAS,CAAC,EAAE,CAAC,KAAK,EAAE;QAAE,cAAc,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;IACxD,YAAY,CAAC,EAAE,CAAC,OAAO,EAAE,oBAAoB,KAAK,IAAI,CAAC;IACvD,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,GAAG,KAAK,IAAI,CAAC;IACnD,SAAS,CAAC,EAAE,CAAC,KAAK,EAAE,cAAc,KAAK,IAAI,CAAC;IAC5C,OAAO,CAAC,EAAE,CAAC,WAAW,EAAE,MAAM,KAAK,IAAI,CAAC;IACxC,YAAY,CAAC,EAAE,CAAC,IAAI,EAAE;QAAE,IAAI,EAAE,IAAI,CAAA;KAAE,KAAK,IAAI,CAAC;IAC9C,cAAc,CAAC,EAAE,CAAC,IAAI,EAAE;QAAE,MAAM,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;IACpD,uBAAuB,CAAC,EAAE,CAAC,IAAI,EAAE;QAAE,eAAe,EAAE,OAAO,CAAA;KAAE,KAAK,IAAI,CAAC;IACvE,yBAAyB,CAAC,EAAE,CAC1B,MAAM,EAAE,yBAAyB,CAAC,kBAAkB,CAAC,KAClD,IAAI,CAAC;IACV,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;IACnD,aAAa,CAAC,EAAE,CAAC,KAAK,EAAE,sBAAsB,CAAC,eAAe,CAAC,KAAK,IAAI,CAAC;IACzE,qBAAqB,CAAC,EAAE,CACtB,KAAK,EAAE,8BAA8B,CAAC,uBAAuB,CAAC,KAC3D,IAAI,CAAC;IACV,kBAAkB,CAAC,EAAE,CACnB,KAAK,EAAE,2BAA2B,CAAC,oBAAoB,CAAC,KACrD,IAAI,CAAC;IACV,mBAAmB,CAAC,EAAE,CACpB,KAAK,EACD,4BAA4B,CAAC,qBAAqB,CAAC,GACnD,uCAAuC,CAAC,kCAAkC,CAAC,KAC5E,IAAI,CAAC;IACV,sBAAsB,CAAC,EAAE,CACvB,KAAK,EAAE,oBAAoB,CAAC,wCAAwC,CAAC,KAClE,IAAI,CAAC;IACV,uBAAuB,CAAC,EAAE,CACxB,KAAK,EAAE,0BAA0B,CAAC,+BAA+B,CAAC,KAC/D,IAAI,CAAC;IACV,cAAc,CAAC,EAAE,CAAC,KAAK,EAAE,YAAY,CAAC,oBAAoB,CAAC,KAAK,IAAI,CAAC;IACrE,yBAAyB,CAAC,EAAE,CAC1B,KAAK,EAAE,uBAAuB,CAAC,iCAAiC,CAAC,KAC9D,IAAI,CAAC;IACV,uBAAuB,CAAC,EAAE,CACxB,KAAK,EAAE,gCAAgC,CAAC,oBAAoB,CAAC,KAC1D,IAAI,CAAC;IACV,oBAAoB,CAAC,EAAE,MAAM,IAAI,CAAC;IAClC,gBAAgB,CAAC,EAAE,CAAC,KAAK,EAAE,mBAAmB,KAAK,IAAI,CAAC;IAExD,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,KAAK,IAAI,CAAC;CAChC,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,aAAa,gcAuBuB,CAAC"}
|
package/dist/types.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AA6HA;;;GAGG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,WAAW;IACX,cAAc;IACd,SAAS;IACT,WAAW;IACX,SAAS;IACT,cAAc;IACd,gBAAgB;IAChB,yBAAyB;IACzB,2BAA2B;IAC3B,YAAY;IACZ,eAAe;IACf,uBAAuB;IACvB,oBAAoB;IACpB,qBAAqB;IACrB,wBAAwB;IACxB,yBAAyB;IACzB,gBAAgB;IAChB,2BAA2B;IAC3B,yBAAyB;IACzB,kBAAkB;IAClB,sBAAsB;IACtB,SAAS;CACsC,CAAC"}
|
|
@@ -3,7 +3,7 @@ import type { Mode } from "../BaseConversation.js";
|
|
|
3
3
|
import type { ConversationConfigOverrideAgentPrompt, ConversationConfigOverrideAgentLanguage as Language } from "@elevenlabs/types";
|
|
4
4
|
import type { DisconnectionDetails } from "../types.js";
|
|
5
5
|
export type { ConversationConfigOverrideAgentPrompt, ConversationConfigOverrideAgentLanguage as Language, } from "@elevenlabs/types";
|
|
6
|
-
export type { DisconnectionDetails } from "../types.js";
|
|
6
|
+
export type { DisconnectionContext, DisconnectionDetails } from "../types.js";
|
|
7
7
|
export type DelayConfig = {
|
|
8
8
|
default: number;
|
|
9
9
|
android?: number;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"BaseConnection.d.ts","sourceRoot":"","sources":["../../src/utils/BaseConnection.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAC5E,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,wBAAwB,CAAC;AACnD,OAAO,KAAK,EACV,qCAAqC,EACrC,uCAAuC,IAAI,QAAQ,EACpD,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAExD,YAAY,EACV,qCAAqC,EACrC,uCAAuC,IAAI,QAAQ,GACpD,MAAM,mBAAmB,CAAC;AAC3B,YAAY,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"BaseConnection.d.ts","sourceRoot":"","sources":["../../src/utils/BaseConnection.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAC5E,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,wBAAwB,CAAC;AACnD,OAAO,KAAK,EACV,qCAAqC,EACrC,uCAAuC,IAAI,QAAQ,EACpD,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAExD,YAAY,EACV,qCAAqC,EACrC,uCAAuC,IAAI,QAAQ,GACpD,MAAM,mBAAmB,CAAC;AAC3B,YAAY,EAAE,oBAAoB,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAE9E,MAAM,MAAM,WAAW,GAAG;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,GAAG,CAAC,EAAE,MAAM,CAAC;CACd,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACzB,MAAM,EAAE,KAAK,GAAG,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG,CAAC,OAAO,EAAE,oBAAoB,KAAK,IAAI,CAAC;AAC3E,MAAM,MAAM,iBAAiB,GAAG,CAAC,KAAK,EAAE,mBAAmB,KAAK,IAAI,CAAC;AAErE,MAAM,MAAM,iBAAiB,GAAG;IAC9B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE;QACV,KAAK,CAAC,EAAE;YACN,MAAM,CAAC,EAAE,qCAAqC,CAAC;YAC/C,YAAY,CAAC,EAAE,MAAM,CAAC;YACtB,QAAQ,CAAC,EAAE,QAAQ,CAAC;SACrB,CAAC;QACF,GAAG,CAAC,EAAE;YACJ,OAAO,CAAC,EAAE,MAAM,CAAC;YACjB,KAAK,CAAC,EAAE,MAAM,CAAC;YACf,SAAS,CAAC,EAAE,MAAM,CAAC;YACnB,eAAe,CAAC,EAAE,MAAM,CAAC;SAC1B,CAAC;QACF,YAAY,CAAC,EAAE;YACb,QAAQ,CAAC,EAAE,OAAO,CAAC;SACpB,CAAC;KACH,CAAC;IACF,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,gBAAgB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,CAAC;IAC7D,cAAc,CAAC,EAAE;QACf,+CAA+C;QAC/C,eAAe,CAAC,EAAE,MAAM,GAAG,KAAK,GAAG,UAAU,CAAC;QAC9C,6DAA6D;QAC7D,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;QAC3B,iFAAiF;QACjF,gBAAgB,CAAC,EAAE,aAAa,GAAG,gBAAgB,CAAC;KACrD,CAAC;IACF,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,eAAe,CAAC,EAAE,WAAW,CAAC;IAC9B,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG,WAAW,GAAG,QAAQ,CAAC;AAEpD,MAAM,MAAM,mBAAmB,GAAG,iBAAiB,GAAG;IACpD,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,SAAS,CAAC,EAAE,KAAK,CAAC;IAClB,iBAAiB,CAAC,EAAE,KAAK,CAAC;CAC3B,CAAC;AAEF,MAAM,MAAM,6BAA6B,GAAG,iBAAiB,GAAG;IAC9D,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,WAAW,CAAC;IAC7B,OAAO,CAAC,EAAE,KAAK,CAAC;IAChB,iBAAiB,CAAC,EAAE,KAAK,CAAC;CAC3B,CAAC;AAEF,MAAM,MAAM,0BAA0B,GAAG,iBAAiB,GAAG;IAC3D,iBAAiB,EAAE,MAAM,CAAC;IAC1B,cAAc,CAAC,EAAE,QAAQ,CAAC;IAC1B,OAAO,CAAC,EAAE,KAAK,CAAC;IAChB,SAAS,CAAC,EAAE,KAAK,CAAC;CACnB,CAAC;AAGF,MAAM,MAAM,aAAa,GACrB,mBAAmB,GACnB,6BAA6B,GAC7B,0BAA0B,CAAC;AAE/B,8BAAsB,cAAc;IAClC,kBAAyB,cAAc,EAAE,MAAM,CAAC;IAChD,kBAAyB,WAAW,EAAE,YAAY,CAAC;IACnD,kBAAyB,YAAY,EAAE,YAAY,CAAC;IAEpD,SAAS,CAAC,KAAK,EAAE,mBAAmB,EAAE,CAAM;IAC5C,SAAS,CAAC,oBAAoB,EAAE,oBAAoB,GAAG,IAAI,CAAQ;IACnE,SAAS,CAAC,oBAAoB,EAAE,oBAAoB,GAAG,IAAI,CAAQ;IACnE,SAAS,CAAC,iBAAiB,EAAE,iBAAiB,GAAG,IAAI,CAAQ;IAC7D,SAAS,CAAC,oBAAoB,EAAE,CAAC,CAAC,IAAI,EAAE,IAAI,KAAK,IAAI,CAAC,GAAG,IAAI,CAAQ;IACrE,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC,IAAI,EAAE,OAAO,KAAK,IAAI,CAAC;gBAEhC,MAAM,GAAE;QAAE,OAAO,CAAC,EAAE,CAAC,IAAI,EAAE,OAAO,KAAK,IAAI,CAAA;KAAO;IAI9D,SAAS,CAAC,KAAK,CAAC,IAAI,EAAE,OAAO;aAIb,KAAK,IAAI,IAAI;aACb,WAAW,CAAC,OAAO,EAAE,mBAAmB,GAAG,IAAI;IAExD,SAAS,CAAC,QAAQ,EAAE,iBAAiB;IAcrC,YAAY,CAAC,QAAQ,EAAE,oBAAoB;IAY3C,YAAY,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,IAAI,KAAK,IAAI;IAIlD,SAAS,CAAC,UAAU,CAAC,IAAI,EAAE,IAAI;IAI/B,SAAS,CAAC,UAAU,CAAC,OAAO,EAAE,oBAAoB;IAOlD,SAAS,CAAC,aAAa,CAAC,WAAW,EAAE,mBAAmB;CAOzD;AAED,wBAAgB,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,YAAY,CAexD"}
|
|
@@ -14,11 +14,9 @@ export declare class WebRTCConnection extends BaseConnection {
|
|
|
14
14
|
private room;
|
|
15
15
|
private isConnected;
|
|
16
16
|
private audioEventId;
|
|
17
|
-
private audioCaptureContext;
|
|
18
|
-
private audioElements;
|
|
19
17
|
private outputDeviceId;
|
|
18
|
+
private audioAdapter;
|
|
20
19
|
private inputAnalyser;
|
|
21
|
-
private inputAudioContext;
|
|
22
20
|
private inputVolumeProvider;
|
|
23
21
|
private outputAnalyser;
|
|
24
22
|
private outputVolumeProvider;
|
|
@@ -32,10 +30,9 @@ export declare class WebRTCConnection extends BaseConnection {
|
|
|
32
30
|
sendMessage(message: OutgoingSocketEvent): Promise<void>;
|
|
33
31
|
getRoom(): Room;
|
|
34
32
|
/**
|
|
35
|
-
*
|
|
36
|
-
*
|
|
37
|
-
*
|
|
38
|
-
* active mic track.
|
|
33
|
+
* Delegates input volume analysis to the audio adapter (if present).
|
|
34
|
+
* Called once during create() and again after an input device switch
|
|
35
|
+
* so the analyser follows the active mic track.
|
|
39
36
|
*/
|
|
40
37
|
private setupInputAnalyser;
|
|
41
38
|
setInputVolumeProvider(provider: VolumeProvider): void;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"WebRTCConnection.d.ts","sourceRoot":"","sources":["../../src/utils/WebRTCConnection.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,cAAc,EACd,KAAK,aAAa,EAClB,KAAK,YAAY,EAElB,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"WebRTCConnection.d.ts","sourceRoot":"","sources":["../../src/utils/WebRTCConnection.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,cAAc,EACd,KAAK,aAAa,EAClB,KAAK,YAAY,EAElB,MAAM,qBAAqB,CAAC;AAI7B,OAAO,EAAsB,KAAK,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAC3E,OAAO,EACL,IAAI,EAKL,MAAM,gBAAgB,CAAC;AAYxB,OAAO,KAAK,EAAE,eAAe,EAAqB,MAAM,uBAAuB,CAAC;AAChF,OAAO,KAAK,EACV,gBAAgB,EAEjB,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,KAAK,cAAc,EAAa,MAAM,qBAAqB,CAAC;AAerE,MAAM,MAAM,gBAAgB,GAAG,aAAa,GAAG;IAC7C,OAAO,CAAC,EAAE,CAAC,IAAI,EAAE,OAAO,KAAK,IAAI,CAAC;CACnC,CAAC;AAEF,qBAAa,gBAAiB,SAAQ,cAAc;IAC3C,cAAc,EAAE,MAAM,CAAC;IAC9B,SAAgB,WAAW,EAAE,YAAY,CAAC;IAC1C,SAAgB,YAAY,EAAE,YAAY,CAAC;IAE3C,OAAO,CAAC,IAAI,CAAO;IACnB,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,YAAY,CAAK;IACzB,OAAO,CAAC,cAAc,CAAuB;IAE7C,OAAO,CAAC,YAAY,CAA4B;IAEhD,OAAO,CAAC,aAAa,CAAsB;IAC3C,OAAO,CAAC,mBAAmB,CAA6B;IAExD,OAAO,CAAC,cAAc,CAAsB;IAC5C,OAAO,CAAC,oBAAoB,CAA6B;IAGzD,OAAO,CAAC,QAAQ,CAAS;IAGzB,SAAgB,KAAK,EAAE,eAAe,CAgGpC;IAGF,SAAgB,MAAM,EAAE,gBAAgB,CAmCtC;IAEF,OAAO;WAiBa,MAAM,CACxB,MAAM,EAAE,gBAAgB,GACvB,OAAO,CAAC,gBAAgB,CAAC;IA0I5B,OAAO,CAAC,uBAAuB;IAuGxB,KAAK;IA0BC,WAAW,CAAC,OAAO,EAAE,mBAAmB;IAgC9C,OAAO,IAAI,IAAI;IAItB;;;;OAIG;IACH,OAAO,CAAC,kBAAkB;IAiBnB,sBAAsB,CAAC,QAAQ,EAAE,cAAc;IAI/C,uBAAuB,CAAC,QAAQ,EAAE,cAAc;YAIzC,iBAAiB;IAqCxB,cAAc,CAAC,MAAM,EAAE,MAAM;IAIvB,oBAAoB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAarD,mBAAmB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;CAqDlE"}
|
|
@@ -1,14 +1,16 @@
|
|
|
1
1
|
import { BaseConnection, parseFormat, } from "./BaseConnection.js";
|
|
2
|
+
import { isJsonObject } from "./assert.js";
|
|
2
3
|
import { extractApiErrorMessage } from "./errors.js";
|
|
3
4
|
import { sourceInfo } from "../sourceInfo.js";
|
|
4
5
|
import { isValidSocketEvent } from "./events.js";
|
|
5
6
|
import { Room, RoomEvent, Track, ConnectionState, createLocalAudioTrack, } from "livekit-client";
|
|
6
7
|
import { constructOverrides, CONVERSATION_INITIATION_CLIENT_DATA_TYPE, } from "./overrides.js";
|
|
7
8
|
import { arrayBufferToBase64 } from "./audio.js";
|
|
8
|
-
import {
|
|
9
|
-
import {
|
|
9
|
+
import { NO_VOLUME } from "./volumeProvider.js";
|
|
10
|
+
import { createAudioAdapter, } from "../WebRTCAudioAdapter.js";
|
|
10
11
|
const DEFAULT_LIVEKIT_WS_URL = "wss://livekit.rtc.elevenlabs.io";
|
|
11
12
|
const HTTPS_API_ORIGIN = "https://api.elevenlabs.io";
|
|
13
|
+
const AUDIO_VOLUME_THRESHOLD = 0.01;
|
|
12
14
|
// Convert WSS origin to HTTPS for API calls
|
|
13
15
|
function convertWssToHttps(origin) {
|
|
14
16
|
return origin.replace(/^wss:\/\//, "https://");
|
|
@@ -20,13 +22,11 @@ export class WebRTCConnection extends BaseConnection {
|
|
|
20
22
|
room;
|
|
21
23
|
isConnected = false;
|
|
22
24
|
audioEventId = 1;
|
|
23
|
-
audioCaptureContext = null;
|
|
24
|
-
audioElements = [];
|
|
25
25
|
outputDeviceId = null;
|
|
26
|
-
|
|
27
|
-
|
|
26
|
+
audioAdapter;
|
|
27
|
+
inputAnalyser = undefined;
|
|
28
28
|
inputVolumeProvider = NO_VOLUME;
|
|
29
|
-
outputAnalyser =
|
|
29
|
+
outputAnalyser = undefined;
|
|
30
30
|
outputVolumeProvider = NO_VOLUME;
|
|
31
31
|
// InputController state
|
|
32
32
|
_isMuted = false;
|
|
@@ -102,7 +102,7 @@ export class WebRTCConnection extends BaseConnection {
|
|
|
102
102
|
}
|
|
103
103
|
},
|
|
104
104
|
isMuted: () => this._isMuted,
|
|
105
|
-
getAnalyser: () => this.inputAnalyser
|
|
105
|
+
getAnalyser: () => this.inputAnalyser,
|
|
106
106
|
getVolume: () => {
|
|
107
107
|
if (this._isMuted)
|
|
108
108
|
return 0;
|
|
@@ -143,7 +143,7 @@ export class WebRTCConnection extends BaseConnection {
|
|
|
143
143
|
// No-op for WebRTC - LiveKit handles audio playback and interruption
|
|
144
144
|
// Audio interruption is managed by the server/agent
|
|
145
145
|
},
|
|
146
|
-
getAnalyser: () => this.outputAnalyser
|
|
146
|
+
getAnalyser: () => this.outputAnalyser,
|
|
147
147
|
getVolume: () => this.outputVolumeProvider.getVolume(),
|
|
148
148
|
getByteFrequencyData: (buffer) => {
|
|
149
149
|
this.outputVolumeProvider.getByteFrequencyData(buffer);
|
|
@@ -155,6 +155,7 @@ export class WebRTCConnection extends BaseConnection {
|
|
|
155
155
|
this.conversationId = conversationId;
|
|
156
156
|
this.inputFormat = inputFormat;
|
|
157
157
|
this.outputFormat = outputFormat;
|
|
158
|
+
this.audioAdapter = createAudioAdapter();
|
|
158
159
|
this.setupRoomEventListeners();
|
|
159
160
|
}
|
|
160
161
|
static async create(config) {
|
|
@@ -180,6 +181,9 @@ export class WebRTCConnection extends BaseConnection {
|
|
|
180
181
|
throw new Error(`ElevenLabs API returned ${response.status} ${message}`);
|
|
181
182
|
}
|
|
182
183
|
const data = await response.json();
|
|
184
|
+
if (!isJsonObject(data) || typeof data.token !== "string") {
|
|
185
|
+
throw new Error("No conversation token received from API");
|
|
186
|
+
}
|
|
183
187
|
conversationToken = data.token;
|
|
184
188
|
if (!conversationToken) {
|
|
185
189
|
throw new Error("No conversation token received from API");
|
|
@@ -265,14 +269,14 @@ export class WebRTCConnection extends BaseConnection {
|
|
|
265
269
|
}
|
|
266
270
|
}
|
|
267
271
|
setupRoomEventListeners() {
|
|
268
|
-
this.room.on(RoomEvent.Connected,
|
|
272
|
+
this.room.on(RoomEvent.Connected, () => {
|
|
269
273
|
this.isConnected = true;
|
|
270
274
|
});
|
|
271
275
|
this.room.on(RoomEvent.Disconnected, reason => {
|
|
272
276
|
this.isConnected = false;
|
|
273
277
|
this.disconnect({
|
|
274
278
|
reason: "agent",
|
|
275
|
-
context:
|
|
279
|
+
context: { type: "close", reason: reason?.toString() },
|
|
276
280
|
});
|
|
277
281
|
});
|
|
278
282
|
this.room.on(RoomEvent.ConnectionStateChanged, state => {
|
|
@@ -281,7 +285,7 @@ export class WebRTCConnection extends BaseConnection {
|
|
|
281
285
|
this.disconnect({
|
|
282
286
|
reason: "error",
|
|
283
287
|
message: `LiveKit connection state changed to ${state}`,
|
|
284
|
-
context:
|
|
288
|
+
context: { type: "connection_state_changed" },
|
|
285
289
|
});
|
|
286
290
|
}
|
|
287
291
|
});
|
|
@@ -308,32 +312,14 @@ export class WebRTCConnection extends BaseConnection {
|
|
|
308
312
|
this.room.on(RoomEvent.TrackSubscribed, async (track, _publication, participant) => {
|
|
309
313
|
if (track.kind === Track.Kind.Audio &&
|
|
310
314
|
participant.identity.includes("agent")) {
|
|
311
|
-
// Play the audio track
|
|
312
315
|
const remoteAudioTrack = track;
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
try {
|
|
319
|
-
await audioElement.setSinkId(this.outputDeviceId);
|
|
320
|
-
}
|
|
321
|
-
catch (error) {
|
|
322
|
-
console.warn("Failed to set output device for new audio element:", error);
|
|
323
|
-
}
|
|
324
|
-
}
|
|
325
|
-
// Add to DOM (hidden) to ensure it plays
|
|
326
|
-
audioElement.style.display = "none";
|
|
327
|
-
document.body.appendChild(audioElement);
|
|
328
|
-
// Store reference for volume control
|
|
329
|
-
this.audioElements.push(audioElement);
|
|
330
|
-
// Apply current volume if it exists (for when volume was set before audio track arrived)
|
|
331
|
-
if (this.audioElements.length === 1) {
|
|
332
|
-
// First audio element - trigger a callback to sync with current volume
|
|
316
|
+
if (this.audioAdapter) {
|
|
317
|
+
// Delegate playback to the platform-specific adapter
|
|
318
|
+
await this.audioAdapter.attachRemoteTrack(remoteAudioTrack, this.outputDeviceId);
|
|
319
|
+
// Set up output volume analysis and audio capture
|
|
320
|
+
await this.setupAudioCapture(remoteAudioTrack);
|
|
333
321
|
this.onDebug?.({ type: "audio_element_ready" });
|
|
334
322
|
}
|
|
335
|
-
// Set up audio capture for onAudio callback
|
|
336
|
-
await this.setupAudioCapture(remoteAudioTrack);
|
|
337
323
|
}
|
|
338
324
|
});
|
|
339
325
|
this.room.on(RoomEvent.ActiveSpeakersChanged, async (speakers) => {
|
|
@@ -348,7 +334,7 @@ export class WebRTCConnection extends BaseConnection {
|
|
|
348
334
|
if (participant.identity?.startsWith("agent")) {
|
|
349
335
|
this.disconnect({
|
|
350
336
|
reason: "agent",
|
|
351
|
-
context:
|
|
337
|
+
context: { type: "close", reason: "agent disconnected" },
|
|
352
338
|
});
|
|
353
339
|
}
|
|
354
340
|
});
|
|
@@ -366,28 +352,12 @@ export class WebRTCConnection extends BaseConnection {
|
|
|
366
352
|
catch (error) {
|
|
367
353
|
console.warn("Error stopping local tracks:", error);
|
|
368
354
|
}
|
|
369
|
-
//
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
this.inputAnalyser = null;
|
|
376
|
-
}
|
|
377
|
-
// Clean up audio capture context (non-blocking)
|
|
378
|
-
if (this.audioCaptureContext) {
|
|
379
|
-
this.audioCaptureContext.close().catch(error => {
|
|
380
|
-
console.warn("Error closing audio capture context:", error);
|
|
381
|
-
});
|
|
382
|
-
this.audioCaptureContext = null;
|
|
383
|
-
}
|
|
384
|
-
// Clean up audio elements
|
|
385
|
-
this.audioElements.forEach(element => {
|
|
386
|
-
if (element.parentNode) {
|
|
387
|
-
element.parentNode.removeChild(element);
|
|
388
|
-
}
|
|
389
|
-
});
|
|
390
|
-
this.audioElements = [];
|
|
355
|
+
// Delegate all audio cleanup to the adapter
|
|
356
|
+
this.audioAdapter?.cleanup();
|
|
357
|
+
this.inputAnalyser = undefined;
|
|
358
|
+
this.outputAnalyser = undefined;
|
|
359
|
+
this.inputVolumeProvider = NO_VOLUME;
|
|
360
|
+
this.outputVolumeProvider = NO_VOLUME;
|
|
391
361
|
this.room.disconnect();
|
|
392
362
|
}
|
|
393
363
|
}
|
|
@@ -422,26 +392,17 @@ export class WebRTCConnection extends BaseConnection {
|
|
|
422
392
|
return this.room;
|
|
423
393
|
}
|
|
424
394
|
/**
|
|
425
|
-
*
|
|
426
|
-
*
|
|
427
|
-
*
|
|
428
|
-
* active mic track.
|
|
395
|
+
* Delegates input volume analysis to the audio adapter (if present).
|
|
396
|
+
* Called once during create() and again after an input device switch
|
|
397
|
+
* so the analyser follows the active mic track.
|
|
429
398
|
*/
|
|
430
399
|
setupInputAnalyser(mediaStreamTrack) {
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
this.inputAudioContext.close().catch(() => { });
|
|
434
|
-
this.inputAudioContext = null;
|
|
435
|
-
this.inputAnalyser = null;
|
|
436
|
-
}
|
|
400
|
+
if (!this.audioAdapter)
|
|
401
|
+
return;
|
|
437
402
|
try {
|
|
438
|
-
const
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
source.connect(analyser);
|
|
442
|
-
this.inputAnalyser = analyser;
|
|
443
|
-
this.inputAudioContext = ctx;
|
|
444
|
-
this.inputVolumeProvider = createAnalyserVolumeProvider(analyser, ctx.sampleRate);
|
|
403
|
+
const result = this.audioAdapter.setupInputAnalysis(mediaStreamTrack);
|
|
404
|
+
this.inputVolumeProvider = result.volumeProvider;
|
|
405
|
+
this.inputAnalyser = result.analyser;
|
|
445
406
|
}
|
|
446
407
|
catch (error) {
|
|
447
408
|
// Don't reset inputVolumeProvider here — an external provider (e.g.
|
|
@@ -456,39 +417,14 @@ export class WebRTCConnection extends BaseConnection {
|
|
|
456
417
|
this.outputVolumeProvider = provider;
|
|
457
418
|
}
|
|
458
419
|
async setupAudioCapture(track) {
|
|
420
|
+
if (!this.audioAdapter)
|
|
421
|
+
return;
|
|
459
422
|
try {
|
|
460
|
-
|
|
461
|
-
const audioContext = new AudioContext();
|
|
462
|
-
this.audioCaptureContext = audioContext;
|
|
463
|
-
// Create analyser for frequency data
|
|
464
|
-
this.outputAnalyser = audioContext.createAnalyser();
|
|
465
|
-
this.outputAnalyser.fftSize = 2048;
|
|
466
|
-
this.outputAnalyser.smoothingTimeConstant = 0.8;
|
|
467
|
-
// Create MediaStream from the track
|
|
468
|
-
const mediaStream = new MediaStream([track.mediaStreamTrack]);
|
|
469
|
-
// Create audio source from the stream
|
|
470
|
-
const source = audioContext.createMediaStreamSource(mediaStream);
|
|
471
|
-
// Connect source to analyser
|
|
472
|
-
source.connect(this.outputAnalyser);
|
|
473
|
-
this.setOutputVolumeProvider(createAnalyserVolumeProvider(this.outputAnalyser, audioContext.sampleRate));
|
|
474
|
-
await loadRawAudioProcessor(audioContext.audioWorklet);
|
|
475
|
-
const worklet = new AudioWorkletNode(audioContext, "rawAudioProcessor");
|
|
476
|
-
// Connect analyser to worklet for processing
|
|
477
|
-
this.outputAnalyser.connect(worklet);
|
|
478
|
-
// Configure the processor for the output format
|
|
479
|
-
worklet.port.postMessage({
|
|
480
|
-
type: "setFormat",
|
|
481
|
-
format: this.outputFormat.format,
|
|
482
|
-
sampleRate: this.outputFormat.sampleRate,
|
|
483
|
-
});
|
|
484
|
-
// Handle processed audio data
|
|
485
|
-
worklet.port.onmessage = (event) => {
|
|
486
|
-
const [audioData, maxVolume] = event.data;
|
|
423
|
+
const onAudioData = (audioData, maxVolume) => {
|
|
487
424
|
// Only send audio if there's significant volume (not just silence)
|
|
488
|
-
|
|
489
|
-
if (maxVolume > volumeThreshold) {
|
|
425
|
+
if (maxVolume > AUDIO_VOLUME_THRESHOLD) {
|
|
490
426
|
// Convert to base64
|
|
491
|
-
const base64Audio = arrayBufferToBase64(audioData
|
|
427
|
+
const base64Audio = arrayBufferToBase64(audioData);
|
|
492
428
|
// Use sequential event ID for proper feedback tracking
|
|
493
429
|
const eventId = this.audioEventId++;
|
|
494
430
|
// Trigger the onAudio callback by simulating an audio event
|
|
@@ -501,33 +437,22 @@ export class WebRTCConnection extends BaseConnection {
|
|
|
501
437
|
});
|
|
502
438
|
}
|
|
503
439
|
};
|
|
504
|
-
|
|
505
|
-
|
|
440
|
+
const result = await this.audioAdapter.setupOutputAnalysis(track, this.outputFormat, onAudioData);
|
|
441
|
+
this.outputVolumeProvider = result.volumeProvider;
|
|
442
|
+
this.outputAnalyser = result.analyser;
|
|
506
443
|
}
|
|
507
444
|
catch (error) {
|
|
508
445
|
console.warn("Failed to set up audio capture:", error);
|
|
509
446
|
}
|
|
510
447
|
}
|
|
511
448
|
setAudioVolume(volume) {
|
|
512
|
-
this.
|
|
513
|
-
element.volume = volume;
|
|
514
|
-
});
|
|
449
|
+
this.audioAdapter?.setVolume(volume);
|
|
515
450
|
}
|
|
516
451
|
async setAudioOutputDevice(deviceId) {
|
|
517
|
-
if (!
|
|
518
|
-
throw new Error("
|
|
452
|
+
if (!this.audioAdapter) {
|
|
453
|
+
throw new Error("Cannot set output device: no audio adapter available on this platform");
|
|
519
454
|
}
|
|
520
|
-
|
|
521
|
-
const promises = this.audioElements.map(async (element) => {
|
|
522
|
-
try {
|
|
523
|
-
await element.setSinkId(deviceId);
|
|
524
|
-
}
|
|
525
|
-
catch (error) {
|
|
526
|
-
console.error("Failed to set sink ID for audio element:", error);
|
|
527
|
-
throw error;
|
|
528
|
-
}
|
|
529
|
-
});
|
|
530
|
-
await Promise.all(promises);
|
|
455
|
+
await this.audioAdapter.setOutputDevice(deviceId);
|
|
531
456
|
// Store the device ID for future audio elements
|
|
532
457
|
this.outputDeviceId = deviceId;
|
|
533
458
|
}
|
|
@@ -543,16 +468,14 @@ export class WebRTCConnection extends BaseConnection {
|
|
|
543
468
|
await currentMicTrackPublication.track.stop();
|
|
544
469
|
await this.room.localParticipant.unpublishTrack(currentMicTrackPublication.track);
|
|
545
470
|
}
|
|
546
|
-
// Create
|
|
547
|
-
const
|
|
471
|
+
// Create new audio track with the specified device
|
|
472
|
+
const audioTrack = await createLocalAudioTrack({
|
|
548
473
|
deviceId: { exact: deviceId },
|
|
549
474
|
echoCancellation: true,
|
|
550
475
|
noiseSuppression: true,
|
|
551
476
|
autoGainControl: true,
|
|
552
477
|
channelCount: { ideal: 1 },
|
|
553
|
-
};
|
|
554
|
-
// Create new audio track with the specified device
|
|
555
|
-
const audioTrack = await createLocalAudioTrack(audioConstraints);
|
|
478
|
+
});
|
|
556
479
|
// Publish the new microphone track
|
|
557
480
|
await this.room.localParticipant.publishTrack(audioTrack, {
|
|
558
481
|
name: "microphone",
|