@elevenlabs/client 0.15.2 → 1.0.0-rc.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -836
- package/dist/BaseConversation.d.ts +10 -12
- package/dist/BaseConversation.d.ts.map +1 -0
- package/dist/BaseConversation.js +392 -0
- package/dist/BaseConversation.js.map +1 -0
- package/dist/InputController.d.ts +13 -0
- package/dist/InputController.d.ts.map +1 -0
- package/dist/InputController.js +2 -0
- package/dist/InputController.js.map +1 -0
- package/dist/OutputController.d.ts +12 -0
- package/dist/OutputController.d.ts.map +1 -0
- package/dist/OutputController.js +2 -0
- package/dist/OutputController.js.map +1 -0
- package/dist/TextConversation.d.ts +7 -0
- package/dist/TextConversation.d.ts.map +1 -0
- package/dist/TextConversation.js +53 -0
- package/dist/TextConversation.js.map +1 -0
- package/dist/VoiceConversation.d.ts +14 -12
- package/dist/VoiceConversation.d.ts.map +1 -0
- package/dist/VoiceConversation.js +225 -0
- package/dist/VoiceConversation.js.map +1 -0
- package/dist/index.d.ts +11 -6
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +20 -0
- package/dist/index.js.map +1 -0
- package/dist/internal.d.ts +8 -0
- package/dist/internal.d.ts.map +1 -0
- package/dist/internal.js +6 -0
- package/dist/internal.js.map +1 -0
- package/dist/lib.iife.js +23493 -0
- package/dist/lib.iife.js.map +1 -0
- package/dist/platform/VoiceSessionSetup.d.ts +35 -0
- package/dist/platform/VoiceSessionSetup.d.ts.map +1 -0
- package/dist/platform/VoiceSessionSetup.js +74 -0
- package/dist/platform/VoiceSessionSetup.js.map +1 -0
- package/dist/scribe/connection.d.ts +1 -0
- package/dist/scribe/connection.d.ts.map +1 -0
- package/dist/scribe/connection.js +363 -0
- package/dist/scribe/connection.js.map +1 -0
- package/dist/scribe/index.d.ts +1 -0
- package/dist/scribe/index.d.ts.map +1 -0
- package/dist/scribe/index.js +8 -0
- package/dist/scribe/index.js.map +1 -0
- package/dist/scribe/scribe.d.ts +1 -0
- package/dist/scribe/scribe.d.ts.map +1 -0
- package/dist/scribe/scribe.js +193 -0
- package/dist/scribe/scribe.js.map +1 -0
- package/dist/sourceInfo.d.ts +7 -0
- package/dist/sourceInfo.d.ts.map +1 -0
- package/dist/sourceInfo.js +9 -0
- package/dist/sourceInfo.js.map +1 -0
- package/dist/utils/BaseConnection.d.ts +2 -9
- package/dist/utils/BaseConnection.d.ts.map +1 -0
- package/dist/utils/BaseConnection.js +73 -0
- package/dist/utils/BaseConnection.js.map +1 -0
- package/dist/utils/ConnectionFactory.d.ts +1 -0
- package/dist/utils/ConnectionFactory.d.ts.map +1 -0
- package/dist/utils/ConnectionFactory.js +26 -0
- package/dist/utils/ConnectionFactory.js.map +1 -0
- package/dist/utils/WebRTCConnection.d.ts +6 -1
- package/dist/utils/WebRTCConnection.d.ts.map +1 -0
- package/dist/utils/WebRTCConnection.js +494 -0
- package/dist/utils/WebRTCConnection.js.map +1 -0
- package/dist/utils/WebSocketConnection.d.ts +8 -3
- package/dist/utils/WebSocketConnection.d.ts.map +1 -0
- package/dist/utils/WebSocketConnection.js +155 -0
- package/dist/utils/WebSocketConnection.js.map +1 -0
- package/dist/utils/applyDelay.d.ts +1 -0
- package/dist/utils/applyDelay.d.ts.map +1 -0
- package/dist/utils/applyDelay.js +18 -0
- package/dist/utils/applyDelay.js.map +1 -0
- package/dist/utils/attachConnectionToOutput.d.ts +5 -0
- package/dist/utils/attachConnectionToOutput.d.ts.map +1 -0
- package/dist/utils/attachConnectionToOutput.js +11 -0
- package/dist/utils/attachConnectionToOutput.js.map +1 -0
- package/dist/utils/attachInputToConnection.d.ts +6 -0
- package/dist/utils/attachInputToConnection.d.ts.map +1 -0
- package/dist/utils/attachInputToConnection.js +16 -0
- package/dist/utils/attachInputToConnection.js.map +1 -0
- package/dist/utils/audio.d.ts +1 -0
- package/dist/utils/audio.d.ts.map +1 -0
- package/dist/utils/audio.js +16 -0
- package/dist/utils/audio.js.map +1 -0
- package/dist/utils/audioConcatProcessor.generated.d.ts +1 -0
- package/dist/utils/audioConcatProcessor.generated.d.ts.map +1 -0
- package/{worklets/audioConcatProcessor.js → dist/utils/audioConcatProcessor.generated.js} +12 -18
- package/dist/utils/audioConcatProcessor.generated.js.map +1 -0
- package/dist/utils/compatibility.d.ts +1 -0
- package/dist/utils/compatibility.d.ts.map +1 -0
- package/dist/utils/compatibility.js +16 -0
- package/dist/utils/compatibility.js.map +1 -0
- package/dist/utils/connection.d.ts +11 -0
- package/dist/utils/connection.d.ts.map +1 -0
- package/dist/utils/connection.js +14 -0
- package/dist/utils/connection.js.map +1 -0
- package/dist/utils/createWorkletModuleLoader.d.ts +1 -0
- package/dist/utils/createWorkletModuleLoader.d.ts.map +1 -0
- package/dist/utils/createWorkletModuleLoader.js +43 -0
- package/dist/utils/createWorkletModuleLoader.js.map +1 -0
- package/dist/utils/errors.d.ts +1 -0
- package/dist/utils/errors.d.ts.map +1 -0
- package/dist/utils/errors.js +11 -0
- package/dist/utils/errors.js.map +1 -0
- package/dist/utils/events.d.ts +3 -5
- package/dist/utils/events.d.ts.map +1 -0
- package/dist/utils/events.js +4 -0
- package/dist/utils/events.js.map +1 -0
- package/dist/utils/input.d.ts +22 -11
- package/dist/utils/input.d.ts.map +1 -0
- package/dist/utils/input.js +181 -0
- package/dist/utils/input.js.map +1 -0
- package/dist/utils/location.d.ts +5 -0
- package/dist/utils/location.d.ts.map +1 -0
- package/dist/utils/location.js +31 -0
- package/dist/utils/location.js.map +1 -0
- package/dist/utils/mergeOptions.d.ts +11 -0
- package/dist/utils/mergeOptions.d.ts.map +1 -0
- package/dist/utils/mergeOptions.js +38 -0
- package/dist/utils/mergeOptions.js.map +1 -0
- package/dist/utils/output.d.ts +36 -10
- package/dist/utils/output.d.ts.map +1 -0
- package/dist/utils/output.js +127 -0
- package/dist/utils/output.js.map +1 -0
- package/dist/utils/overrides.d.ts +1 -0
- package/dist/utils/overrides.d.ts.map +1 -0
- package/dist/utils/overrides.js +40 -0
- package/dist/utils/overrides.js.map +1 -0
- package/dist/utils/postOverallFeedback.d.ts +1 -0
- package/dist/utils/postOverallFeedback.d.ts.map +1 -0
- package/dist/utils/postOverallFeedback.js +19 -0
- package/dist/utils/postOverallFeedback.js.map +1 -0
- package/dist/utils/rawAudioProcessor.generated.d.ts +1 -0
- package/dist/utils/rawAudioProcessor.generated.d.ts.map +1 -0
- package/{worklets/rawAudioProcessor.js → dist/utils/rawAudioProcessor.generated.js} +7 -1
- package/dist/utils/rawAudioProcessor.generated.js.map +1 -0
- package/dist/utils/scribeAudioProcessor.generated.d.ts +1 -0
- package/dist/utils/scribeAudioProcessor.generated.d.ts.map +1 -0
- package/{worklets/scribeAudioProcessor.js → dist/utils/scribeAudioProcessor.generated.js} +7 -1
- package/dist/utils/scribeAudioProcessor.generated.js.map +1 -0
- package/dist/version.d.ts +2 -1
- package/dist/version.d.ts.map +1 -0
- package/dist/version.js +3 -0
- package/dist/version.js.map +1 -0
- package/package.json +22 -16
- package/.turbo/turbo-build.log +0 -13
- package/.turbo/turbo-generate-version.log +0 -4
- package/.turbo/turbo-generate-worklets.log +0 -9
- package/.turbo/turbo-lint$colon$es.log +0 -6
- package/.turbo/turbo-lint$colon$prettier.log +0 -6
- package/CHANGELOG.md +0 -50
- package/dist/lib.cjs +0 -2
- package/dist/lib.cjs.map +0 -1
- package/dist/lib.modern.js +0 -2
- package/dist/lib.modern.js.map +0 -1
- package/dist/lib.module.js +0 -2
- package/dist/lib.module.js.map +0 -1
- package/dist/lib.umd.js +0 -2
- package/dist/lib.umd.js.map +0 -1
- package/dist/utils/addLibsamplerateModule.d.ts +0 -1
- package/scripts/generateWorklets.js +0 -59
package/README.md
CHANGED
|
@@ -1,872 +1,56 @@
|
|
|
1
1
|

|
|
2
2
|
|
|
3
|
-
# ElevenAgents
|
|
3
|
+
# ElevenAgents TypeScript SDK
|
|
4
4
|
|
|
5
5
|
Build multimodal agents with [ElevenAgents](https://elevenlabs.io/docs/eleven-agents/overview).
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
A TypeScript / JavaScript client library for using ElevenAgents, or as a base for framework-specific libraries. If you're using React, consider using [`@elevenlabs/react`](https://www.npmjs.com/package/@elevenlabs/react) instead.
|
|
8
8
|
|
|
9
9
|

|
|
10
10
|
[](https://discord.gg/elevenlabs)
|
|
11
|
-
[](https://twitter.com/ElevenLabs)
|
|
12
12
|
|
|
13
13
|
## Installation
|
|
14
14
|
|
|
15
|
-
Install the package in your project through package manager.
|
|
16
|
-
|
|
17
15
|
```shell
|
|
18
16
|
npm install @elevenlabs/client
|
|
19
|
-
# or
|
|
20
|
-
yarn add @elevenlabs/client
|
|
21
|
-
# or
|
|
22
|
-
pnpm install @elevenlabs/client
|
|
23
|
-
```
|
|
24
|
-
|
|
25
|
-
## Usage
|
|
26
|
-
|
|
27
|
-
This library is primarily meant for development in vanilla JavaScript projects, or as a base for libraries tailored to specific frameworks.
|
|
28
|
-
It is recommended to check whether your specific framework has it's own library.
|
|
29
|
-
However, you can use this library in any JavaScript-based project.
|
|
30
|
-
|
|
31
|
-
### Connection types
|
|
32
|
-
|
|
33
|
-
A conversation can be started via one of two connection types: WebSockets (the default) or WebRTC.
|
|
34
|
-
|
|
35
|
-
### Initialize conversation
|
|
36
|
-
|
|
37
|
-
First, initialize the Conversation instance:
|
|
38
|
-
|
|
39
|
-
```js
|
|
40
|
-
const conversation = await Conversation.startSession(options);
|
|
41
|
-
```
|
|
42
|
-
|
|
43
|
-
This will kick off the websocket connection and start using microphone to communicate with the ElevenLabs agent. Consider explaining and allowing microphone access in your apps UI before the Conversation kicks off. The microphone may also be blocked for the current page by default, resulting in the allow prompt not showing up at all. You should handle such use case in your application and display appropriate message to the user:
|
|
44
|
-
|
|
45
|
-
```js
|
|
46
|
-
// call after explaning to the user why the microphone access is needed
|
|
47
|
-
// handle errors and show appropriate message to the user
|
|
48
|
-
try {
|
|
49
|
-
await navigator.mediaDevices.getUserMedia();
|
|
50
|
-
} catch {
|
|
51
|
-
// handle error
|
|
52
|
-
}
|
|
53
|
-
```
|
|
54
|
-
|
|
55
|
-
#### Session configuration
|
|
56
|
-
|
|
57
|
-
The options passed to `startSession` specifiy how the session is established. There are three ways to start a session:
|
|
58
|
-
|
|
59
|
-
##### Public agents
|
|
60
|
-
|
|
61
|
-
Agents that don't require any authentication can be used to start a conversation by using the agent ID and the connection type. The agent ID can be acquired through the [ElevenLabs UI](https://elevenlabs.io/app/conversational-ai).
|
|
62
|
-
|
|
63
|
-
For public agents, you can use the ID directly:
|
|
64
|
-
|
|
65
|
-
```js
|
|
66
|
-
const conversation = await Conversation.startSession({
|
|
67
|
-
agentId: "<your-agent-id>",
|
|
68
|
-
connectionType: "webrtc", // 'websocket' is also accepted
|
|
69
|
-
});
|
|
70
|
-
```
|
|
71
|
-
|
|
72
|
-
##### Private agents
|
|
73
|
-
|
|
74
|
-
If the conversation requires authorization, you will need to add a dedicated endpoint to your server that will either request a signed url (if using the WebSockets connection type) or a conversation token (if using WebRTC) using the [ElevenLabs API](https://elevenlabs.io/docs/introduction) and pass it back to the client.
|
|
75
|
-
|
|
76
|
-
Here's an example for a WebSocket connection:
|
|
77
|
-
|
|
78
|
-
```js
|
|
79
|
-
// Node.js server
|
|
80
|
-
|
|
81
|
-
app.get("/signed-url", yourAuthMiddleware, async (req, res) => {
|
|
82
|
-
const response = await fetch(
|
|
83
|
-
`https://api.elevenlabs.io/v1/convai/conversation/get-signed-url?agent_id=${process.env.AGENT_ID}`,
|
|
84
|
-
{
|
|
85
|
-
headers: {
|
|
86
|
-
// Requesting a signed url requires your ElevenLabs API key
|
|
87
|
-
// Do NOT expose your API key to the client!
|
|
88
|
-
"xi-api-key": process.env.ELEVENLABS_API_KEY,
|
|
89
|
-
},
|
|
90
|
-
}
|
|
91
|
-
);
|
|
92
|
-
|
|
93
|
-
if (!response.ok) {
|
|
94
|
-
return res.status(500).send("Failed to get signed URL");
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
const body = await response.json();
|
|
98
|
-
res.send(body.signed_url);
|
|
99
|
-
});
|
|
100
|
-
```
|
|
101
|
-
|
|
102
|
-
```js
|
|
103
|
-
// Client
|
|
104
|
-
|
|
105
|
-
const response = await fetch("/signed-url", yourAuthHeaders);
|
|
106
|
-
const signedUrl = await response.text();
|
|
107
|
-
|
|
108
|
-
const conversation = await Conversation.startSession({
|
|
109
|
-
signedUrl,
|
|
110
|
-
connectionType: "websocket",
|
|
111
|
-
});
|
|
112
|
-
```
|
|
113
|
-
|
|
114
|
-
Here's an example for WebRTC:
|
|
115
|
-
|
|
116
|
-
```js
|
|
117
|
-
// Node.js server
|
|
118
|
-
|
|
119
|
-
app.get("/conversation-token", yourAuthMiddleware, async (req, res) => {
|
|
120
|
-
const response = await fetch(
|
|
121
|
-
`https://api.elevenlabs.io/v1/convai/conversation/token?agent_id=${process.env.AGENT_ID}`,
|
|
122
|
-
{
|
|
123
|
-
headers: {
|
|
124
|
-
// Requesting a conversation token requires your ElevenLabs API key
|
|
125
|
-
// Do NOT expose your API key to the client!
|
|
126
|
-
'xi-api-key': process.env.ELEVENLABS_API_KEY,
|
|
127
|
-
}
|
|
128
|
-
}
|
|
129
|
-
);
|
|
130
|
-
|
|
131
|
-
if (!response.ok) {
|
|
132
|
-
return res.status(500).send("Failed to get conversation token");
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
const body = await response.json();
|
|
136
|
-
res.send(body.token);
|
|
137
|
-
);
|
|
138
|
-
```
|
|
139
|
-
|
|
140
|
-
Once you have the token, providing it to `startSession` will initiate the conversation using WebRTC.
|
|
141
|
-
|
|
142
|
-
```js
|
|
143
|
-
// Client
|
|
144
|
-
|
|
145
|
-
const response = await fetch("/conversation-token", yourAuthHeaders);
|
|
146
|
-
const conversationToken = await response.text();
|
|
147
|
-
|
|
148
|
-
const conversation = await Conversation.startSession({
|
|
149
|
-
conversationToken,
|
|
150
|
-
connectionType: "webrtc",
|
|
151
|
-
});
|
|
152
|
-
```
|
|
153
|
-
|
|
154
|
-
#### Optional callbacks
|
|
155
|
-
|
|
156
|
-
The options passed to `startSession` can also be used to register optional callbacks:
|
|
157
|
-
|
|
158
|
-
- **onConnect** - handler called when the conversation websocket connection is established.
|
|
159
|
-
- **onDisconnect** - handler called when the conversation websocket connection is ended.
|
|
160
|
-
- **onMessage** - handler called when a new text message is received. These can be tentative or final transcriptions of user voice, replies produced by LLM. Primarily used for handling conversation transcription.
|
|
161
|
-
- **onError** - handler called when an error is encountered.
|
|
162
|
-
- **onStatusChange** - handler called whenever connection status changes. Can be `connected`, `connecting` and `disconnected` (initial).
|
|
163
|
-
- **onModeChange** - handler called when a status changes, eg. agent switches from `speaking` to `listening`, or the other way around.
|
|
164
|
-
- **onCanSendFeedbackChange** - handler called when sending feedback becomes available or unavailable.
|
|
165
|
-
- **onUnhandledClientToolCall** - handler called when a client tool is invoked but no corresponding client tool was defined
|
|
166
|
-
- **onDebug** - handler called for debugging events, including tentative agent responses and internal events. Useful for development and troubleshooting.
|
|
167
|
-
- **onAudio** - handler called when audio data is received from the agent. Provides access to raw audio events for custom processing.
|
|
168
|
-
- **onInterruption** - handler called when the conversation is interrupted, typically when the user starts speaking while the agent is talking.
|
|
169
|
-
- **onVadScore** - handler called with voice activity detection scores, indicating the likelihood of speech in the audio input.
|
|
170
|
-
- **onMCPToolCall** - handler called when an MCP (Model Context Protocol) tool is invoked by the agent.
|
|
171
|
-
- **onMCPConnectionStatus** - handler called when the MCP connection status changes, useful for monitoring MCP server connectivity.
|
|
172
|
-
- **onAgentToolRequest** - handler called when the agent begins tool execution.
|
|
173
|
-
- **onAgentToolResponse** - handler called when the agent receives a response from a tool execution.
|
|
174
|
-
- **onConversationMetadata** - handler called with conversation initiation metadata, providing information about the conversation setup.
|
|
175
|
-
- **onAsrInitiationMetadata** - handler called with ASR (Automatic Speech Recognition) initiation metadata, containing configuration details for speech recognition.
|
|
176
|
-
- **onAgentChatResponsePart** - handler called with streaming text chunks during text-only conversations. Provides start, delta, and stop events for real-time text streaming.
|
|
177
|
-
- **onAudioAlignment** - handler called with character-level timing data for synthesized audio. Provides arrays of characters, start times, and durations for text-to-speech synchronization.
|
|
178
|
-
|
|
179
|
-
#### Setting input/output devices
|
|
180
|
-
|
|
181
|
-
You can provide a device ID to start the conversation using the input/output device of your choice. If the device ID is invalid, the default input and output devices will be used.
|
|
182
|
-
|
|
183
|
-
```js
|
|
184
|
-
const conversation = await Conversation.startSession({
|
|
185
|
-
agentId: "<your-agent-id>",
|
|
186
|
-
inputDeviceId: "<new-input-id>",
|
|
187
|
-
outputDeviceId: "<new-output-id>",
|
|
188
|
-
});
|
|
189
|
-
```
|
|
190
|
-
|
|
191
|
-
**Note:** Device switching only works for voice conversations. You can enumerate available devices using the [MediaDevices.enumerateDevices()](https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices/enumerateDevices) API.
|
|
192
|
-
|
|
193
|
-
#### Client Tools
|
|
194
|
-
|
|
195
|
-
Client tools are a way to enabled agent to invoke client-side functionality. This can be used to trigger actions in the client, such as opening a modal or doing an API call on behalf of the user.
|
|
196
|
-
|
|
197
|
-
Client tools definition is an object of functions, and needs to be identical with your configuration within the [ElevenLabs UI](https://elevenlabs.io/app/conversational-ai), where you can name and describe different tools, as well as set up the parameters passed by the agent.
|
|
198
|
-
|
|
199
|
-
```ts
|
|
200
|
-
const conversation = await Conversation.startSession({
|
|
201
|
-
clientTools: {
|
|
202
|
-
displayMessage: async (parameters: { text: string }) => {
|
|
203
|
-
alert(text);
|
|
204
|
-
|
|
205
|
-
return "Message displayed";
|
|
206
|
-
},
|
|
207
|
-
},
|
|
208
|
-
});
|
|
209
|
-
```
|
|
210
|
-
|
|
211
|
-
In case function returns a value, it will be passed back to the agent as a response.
|
|
212
|
-
Note that the tool needs to be explicitly set to be blocking conversation in ElevenLabs UI for the agent to await and react to the response, otherwise agent assumes success and continues the conversation.
|
|
213
|
-
|
|
214
|
-
#### Conversation overrides
|
|
215
|
-
|
|
216
|
-
You may choose to override various settings of the conversation and set them dynamically based other user interactions.
|
|
217
|
-
We support overriding various settings.
|
|
218
|
-
These settings are optional and can be used to customize the conversation experience.
|
|
219
|
-
The following settings are available:
|
|
220
|
-
|
|
221
|
-
```ts
|
|
222
|
-
const conversation = await Conversation.startSession({
|
|
223
|
-
overrides: {
|
|
224
|
-
agent: {
|
|
225
|
-
prompt: {
|
|
226
|
-
prompt: "My custom prompt",
|
|
227
|
-
llm: "gemini-2.5-flash",
|
|
228
|
-
},
|
|
229
|
-
firstMessage: "My custom first message",
|
|
230
|
-
language: "en",
|
|
231
|
-
},
|
|
232
|
-
tts: {
|
|
233
|
-
voiceId: "custom voice id",
|
|
234
|
-
speed: 1.0,
|
|
235
|
-
stability: 0.5,
|
|
236
|
-
similarityBoost: 0.8,
|
|
237
|
-
},
|
|
238
|
-
conversation: {
|
|
239
|
-
textOnly: true,
|
|
240
|
-
},
|
|
241
|
-
},
|
|
242
|
-
});
|
|
243
|
-
```
|
|
244
|
-
|
|
245
|
-
#### User identification
|
|
246
|
-
|
|
247
|
-
You can optionally pass a user ID to identify the user in the conversation. This can be your own customer identifier. This will be included in the conversation initiation data sent to the server:
|
|
248
|
-
|
|
249
|
-
Tracking this ID can be helpful for filtering conversations, tracking analytics on a user level, etc.
|
|
250
|
-
|
|
251
|
-
```ts
|
|
252
|
-
const conversation = await Conversation.startSession({
|
|
253
|
-
agentId: "<your-agent-id>",
|
|
254
|
-
userId: "user-123", // Optional user identifier
|
|
255
|
-
connectionType: "webrtc",
|
|
256
|
-
});
|
|
257
|
-
```
|
|
258
|
-
|
|
259
|
-
#### Text only
|
|
260
|
-
|
|
261
|
-
If your agent is configured to run in text-only mode, i.e. it does not send or receive audio messages,
|
|
262
|
-
you can use this flag to use a lighter version of the conversation. In that case, the
|
|
263
|
-
user will not be asked for microphone permissions and no audio context will be created.
|
|
264
|
-
|
|
265
|
-
```ts
|
|
266
|
-
const conversation = await Conversation.startSession({
|
|
267
|
-
textOnly: true,
|
|
268
|
-
});
|
|
269
17
|
```
|
|
270
18
|
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
While this SDK leaves the choice of audio input/output device to the browser/system, iOS Safari seem to prefer the built-in speaker over headphones even when bluetooth device is in use. If you want to "force" the use of headphones on iOS devices when available, you can use the following option. Please, keep in mind that this is not guaranteed, since this functionality is not provided by the browser. System audio should be the default choice.
|
|
274
|
-
|
|
275
|
-
```ts
|
|
276
|
-
const conversation = await Conversation.startSession({
|
|
277
|
-
preferHeadphonesForIosDevices: true,
|
|
278
|
-
});
|
|
279
|
-
```
|
|
280
|
-
|
|
281
|
-
#### Connection delay
|
|
282
|
-
|
|
283
|
-
You can configure additional delay between when the microphone is activated and when the connection is established.
|
|
284
|
-
On Android, the delay is set to 3 seconds by default to make sure the device has time to switch to the correct audio mode.
|
|
285
|
-
Without it, you may experience issues with the beginning of the first message being cut off.
|
|
286
|
-
|
|
287
|
-
```ts
|
|
288
|
-
const conversation = await Conversation.startSession({
|
|
289
|
-
connectionDelay: {
|
|
290
|
-
android: 3_000,
|
|
291
|
-
ios: 0,
|
|
292
|
-
default: 0,
|
|
293
|
-
},
|
|
294
|
-
});
|
|
295
|
-
```
|
|
296
|
-
|
|
297
|
-
#### Acquiring a Wake Lock
|
|
298
|
-
|
|
299
|
-
By default, the conversation will attempt to acquire a [wake lock](https://developer.mozilla.org/en-US/docs/Web/API/Screen_Wake_Lock_API) to prevent the device from going to sleep during the conversation.
|
|
300
|
-
This can be disabled by setting the `useWakeLock` option to `false`:
|
|
301
|
-
|
|
302
|
-
```ts
|
|
303
|
-
const conversation = await Conversation.startSession({
|
|
304
|
-
useWakeLock: false,
|
|
305
|
-
});
|
|
306
|
-
```
|
|
307
|
-
|
|
308
|
-
#### Return value
|
|
309
|
-
|
|
310
|
-
`startSession` returns a `Conversation` instance that can be used to control the session. The method will throw an error if the session cannot be established. This can happen if the user denies microphone access, or if the websocket connection
|
|
311
|
-
fails.
|
|
312
|
-
|
|
313
|
-
##### endSession
|
|
314
|
-
|
|
315
|
-
A method to manually end the conversation. The method will end the conversation and disconnect from websocket.
|
|
316
|
-
Afterwards the conversation instance will be unusable and can be safely discarded.
|
|
317
|
-
|
|
318
|
-
```js
|
|
319
|
-
await conversation.endSession();
|
|
320
|
-
```
|
|
321
|
-
|
|
322
|
-
##### sendFeedback
|
|
323
|
-
|
|
324
|
-
A method for sending binary feedback to the agent.
|
|
325
|
-
The method accepts a boolean value, where `true` represents positive feedback and `false` negative feedback.
|
|
326
|
-
Feedback is always correlated to the most recent agent response and can be sent only once per response.
|
|
327
|
-
You can listen to `onCanSendFeedbackChange` to know if feedback can be sent at the given moment.
|
|
19
|
+
## Quick Start
|
|
328
20
|
|
|
329
21
|
```js
|
|
330
|
-
|
|
331
|
-
```
|
|
332
|
-
|
|
333
|
-
##### sendContextualUpdate
|
|
334
|
-
|
|
335
|
-
A method to send contextual updates to the agent.
|
|
336
|
-
This can be used to inform the agent about user actions that are not directly related to the conversation, but may influence the agent's responses.
|
|
337
|
-
|
|
338
|
-
```js
|
|
339
|
-
conversation.sendContextualUpdate(
|
|
340
|
-
"User navigated to another page. Consider it for next response, but don't react to this contextual update."
|
|
341
|
-
);
|
|
342
|
-
```
|
|
343
|
-
|
|
344
|
-
##### sendUserMessage
|
|
345
|
-
|
|
346
|
-
Sends a text messages to the agent.
|
|
347
|
-
|
|
348
|
-
Can be used to let the user type in the message instead of using the microphone.
|
|
349
|
-
Unlike `sendContextualUpdate`, this will be treated as a user message and will prompt the agent to take its turn in the conversation.
|
|
350
|
-
|
|
351
|
-
```js
|
|
352
|
-
sendButton.addEventListener("click", e => {
|
|
353
|
-
conversation.sendUserMessage(textInput.value);
|
|
354
|
-
textInput.value = "";
|
|
355
|
-
});
|
|
356
|
-
```
|
|
357
|
-
|
|
358
|
-
##### sendUserActivity
|
|
359
|
-
|
|
360
|
-
Notifies the agent about user activity.
|
|
361
|
-
|
|
362
|
-
The agent will not attempt to speak for at least 2 seconds after the user activity is detected.
|
|
363
|
-
This can be used to prevent the agent from interrupting the user when they are typing.
|
|
364
|
-
|
|
365
|
-
```js
|
|
366
|
-
textInput.addEventListener("input", () => {
|
|
367
|
-
conversation.sendUserActivity();
|
|
368
|
-
});
|
|
369
|
-
```
|
|
370
|
-
|
|
371
|
-
##### getId
|
|
372
|
-
|
|
373
|
-
A method returning the conversation ID.
|
|
374
|
-
|
|
375
|
-
```js
|
|
376
|
-
const id = conversation.getId();
|
|
377
|
-
```
|
|
378
|
-
|
|
379
|
-
##### setVolume
|
|
380
|
-
|
|
381
|
-
A method to set the output volume of the conversation. Accepts object with volume field between 0 and 1.
|
|
382
|
-
|
|
383
|
-
```js
|
|
384
|
-
await conversation.setVolume({ volume: 0.5 });
|
|
385
|
-
```
|
|
386
|
-
|
|
387
|
-
##### muteMic
|
|
388
|
-
|
|
389
|
-
A method to mute/unmute the microphone.
|
|
390
|
-
|
|
391
|
-
```js
|
|
392
|
-
// Mute the microphone
|
|
393
|
-
conversation.setMicMuted(true);
|
|
394
|
-
|
|
395
|
-
// Unmute the microphone
|
|
396
|
-
conversation.setMicMuted(false);
|
|
397
|
-
```
|
|
398
|
-
|
|
399
|
-
##### getInputVolume / getOutputVolume
|
|
400
|
-
|
|
401
|
-
Methods that return the current input/output volume on a scale from `0` to `1` where `0` is -100 dB and `1` is -30 dB.
|
|
402
|
-
|
|
403
|
-
```js
|
|
404
|
-
const inputVolume = await conversation.getInputVolume();
|
|
405
|
-
const outputVolume = await conversation.getOutputVolume();
|
|
406
|
-
```
|
|
407
|
-
|
|
408
|
-
##### getInputByteFrequencyData / getOutputByteFrequencyData
|
|
409
|
-
|
|
410
|
-
Methods that return `Uint8Array`s containing the current input/output frequency data. See [AnalyserNode.getByteFrequencyData](https://developer.mozilla.org/en-US/docs/Web/API/AnalyserNode/getByteFrequencyData) for more information.
|
|
411
|
-
|
|
412
|
-
**Note:** These methods are only available for voice conversations. In WebRTC mode the audio is hardcoded to use `pcm_48000`, meaning any visualization using the returned data might show different patterns to WebSocket connections.
|
|
413
|
-
|
|
414
|
-
##### changeInputDevice
|
|
22
|
+
import { Conversation } from "@elevenlabs/client";
|
|
415
23
|
|
|
416
|
-
Allows you to change the audio input device during an active voice conversation. This method is only available for voice conversations.
|
|
417
|
-
|
|
418
|
-
**Note:** In WebRTC mode the input format and sample rate are hardcoded to `pcm` and `48000` respectively. Changing those values when changing the input device is a no-op.
|
|
419
|
-
|
|
420
|
-
```js
|
|
421
24
|
const conversation = await Conversation.startSession({
|
|
422
|
-
agentId: "
|
|
423
|
-
})
|
|
424
|
-
|
|
425
|
-
// Change to a specific input device
|
|
426
|
-
await conversation.changeInputDevice({
|
|
427
|
-
sampleRate: 16000,
|
|
428
|
-
format: "pcm",
|
|
429
|
-
preferHeadphonesForIosDevices: true,
|
|
430
|
-
inputDeviceId: "your-device-id",
|
|
431
|
-
});
|
|
432
|
-
```
|
|
433
|
-
|
|
434
|
-
##### changeOutputDevice
|
|
435
|
-
|
|
436
|
-
Allows you to change the audio output device during an active voice conversation. This method is only available for voice conversations.
|
|
437
|
-
|
|
438
|
-
**Note:** In WebRTC mode the output format and sample rate are hardcoded to `pcm` and `48000` respectively. Changing those values when changing the output device is a no-op.
|
|
439
|
-
|
|
440
|
-
```js
|
|
441
|
-
// Change to a specific output device
|
|
442
|
-
await conversation.changeOutputDevice({
|
|
443
|
-
sampleRate: 16000,
|
|
444
|
-
format: "pcm",
|
|
445
|
-
outputDeviceId: "your-device-id",
|
|
446
|
-
});
|
|
447
|
-
```
|
|
448
|
-
|
|
449
|
-
**Note:** Device switching only works for voice conversations. If no specific `deviceId` is provided, the browser will use its default device selection. You can enumerate available devices using the [MediaDevices.enumerateDevices()](https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices/enumerateDevices) API.
|
|
450
|
-
|
|
451
|
-
## Scribe - Real-time Speech-to-Text
|
|
452
|
-
|
|
453
|
-
Scribe is ElevenLabs' real-time speech-to-text API that provides low-latency transcription with support for both streaming microphone input and pre-recorded audio files.
|
|
454
|
-
|
|
455
|
-
### Quick Start
|
|
456
|
-
|
|
457
|
-
```js
|
|
458
|
-
import { Scribe, RealtimeEvents } from "@elevenlabs/client";
|
|
459
|
-
|
|
460
|
-
// Connect with microphone streaming
|
|
461
|
-
const connection = Scribe.connect({
|
|
462
|
-
token: "your-token",
|
|
463
|
-
modelId: "scribe_v2_realtime",
|
|
464
|
-
microphone: {
|
|
465
|
-
echoCancellation: true,
|
|
466
|
-
noiseSuppression: true,
|
|
25
|
+
agentId: "agent_7101k5zvyjhmfg983brhmhkd98n6", // replace with your agent's ID
|
|
26
|
+
onConnect: ({ conversationId }) => {
|
|
27
|
+
console.log("Connected:", conversationId);
|
|
467
28
|
},
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
// Listen for transcripts
|
|
471
|
-
connection.on(RealtimeEvents.PARTIAL_TRANSCRIPT, (data) => {
|
|
472
|
-
console.log("Partial:", data.text);
|
|
473
|
-
});
|
|
474
|
-
|
|
475
|
-
connection.on(RealtimeEvents.COMMITTED_TRANSCRIPT, (data) => {
|
|
476
|
-
console.log("Committed:", data.text);
|
|
477
|
-
});
|
|
478
|
-
|
|
479
|
-
// Close connection when done
|
|
480
|
-
connection.close();
|
|
481
|
-
```
|
|
482
|
-
|
|
483
|
-
### Getting a Token
|
|
484
|
-
|
|
485
|
-
Scribe requires a single-use token for authentication. These tokens are generated via the ElevenLabs API on the server.
|
|
486
|
-
|
|
487
|
-
You should create an API endpoint on your server to generate these tokens:
|
|
488
|
-
|
|
489
|
-
```js
|
|
490
|
-
// Node.js server
|
|
491
|
-
app.get("/scribe-token", yourAuthMiddleware, async (req, res) => {
|
|
492
|
-
const response = await fetch(
|
|
493
|
-
"https://api.elevenlabs.io/v1/single-use-token/realtime_scribe",
|
|
494
|
-
{
|
|
495
|
-
method: "POST",
|
|
496
|
-
headers: {
|
|
497
|
-
"xi-api-key": process.env.ELEVENLABS_API_KEY,
|
|
498
|
-
},
|
|
499
|
-
}
|
|
500
|
-
);
|
|
501
|
-
|
|
502
|
-
const data = await response.json();
|
|
503
|
-
res.json({ token: data.token });
|
|
504
|
-
});
|
|
505
|
-
```
|
|
506
|
-
|
|
507
|
-
```js
|
|
508
|
-
// Client
|
|
509
|
-
const response = await fetch("/scribe-token");
|
|
510
|
-
const { token } = await response.json();
|
|
511
|
-
```
|
|
512
|
-
|
|
513
|
-
**Warning:** Your ElevenLabs API key is sensitive, do not leak it to the client. Always generate the token on the server.
|
|
514
|
-
|
|
515
|
-
### Microphone Mode
|
|
516
|
-
|
|
517
|
-
Automatically stream audio from the user's microphone:
|
|
518
|
-
|
|
519
|
-
```js
|
|
520
|
-
import { Scribe, RealtimeEvents } from "@elevenlabs/client";
|
|
521
|
-
|
|
522
|
-
const connection = Scribe.connect({
|
|
523
|
-
token: "your-token",
|
|
524
|
-
modelId: "scribe_v2_realtime",
|
|
525
|
-
microphone: {
|
|
526
|
-
deviceId: "optional-device-id", // Optional: specific microphone
|
|
527
|
-
echoCancellation: true,
|
|
528
|
-
noiseSuppression: true,
|
|
529
|
-
autoGainControl: true,
|
|
530
|
-
channelCount: 1,
|
|
29
|
+
onDisconnect: () => {
|
|
30
|
+
console.log("Disconnected");
|
|
531
31
|
},
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
The microphone stream is automatically converted to PCM16 format required by the API. In this mode audio is automatically committed.
|
|
536
|
-
|
|
537
|
-
### Manual Audio Mode
|
|
538
|
-
|
|
539
|
-
For transcribing pre-recorded audio files or custom audio sources:
|
|
540
|
-
|
|
541
|
-
```js
|
|
542
|
-
import { Scribe, AudioFormat, RealtimeEvents } from "@elevenlabs/client";
|
|
543
|
-
|
|
544
|
-
const connection = Scribe.connect({
|
|
545
|
-
token: "your-token",
|
|
546
|
-
modelId: "scribe_v2_realtime",
|
|
547
|
-
audioFormat: AudioFormat.PCM_16000,
|
|
548
|
-
sampleRate: 16000,
|
|
549
|
-
});
|
|
550
|
-
|
|
551
|
-
// Send audio chunks as base64
|
|
552
|
-
connection.send({ audioBase64: base64AudioChunk });
|
|
553
|
-
|
|
554
|
-
// Signal end of audio segment
|
|
555
|
-
connection.commit();
|
|
556
|
-
```
|
|
557
|
-
|
|
558
|
-
#### Example: Transcribing an Audio File
|
|
559
|
-
|
|
560
|
-
```js
|
|
561
|
-
// Get file from input element
|
|
562
|
-
const fileInput = document.querySelector('input[type="file"]');
|
|
563
|
-
const audioFile = fileInput.files[0];
|
|
564
|
-
|
|
565
|
-
// Read file as ArrayBuffer
|
|
566
|
-
const arrayBuffer = await audioFile.arrayBuffer();
|
|
567
|
-
const audioData = new Uint8Array(arrayBuffer);
|
|
568
|
-
|
|
569
|
-
// Convert to base64 and send in chunks
|
|
570
|
-
const chunkSize = 8192; // 8KB chunks
|
|
571
|
-
for (let i = 0; i < audioData.length; i += chunkSize) {
|
|
572
|
-
const chunk = audioData.slice(i, i + chunkSize);
|
|
573
|
-
const base64 = btoa(String.fromCharCode(...chunk));
|
|
574
|
-
connection.send({ audioBase64: base64 });
|
|
575
|
-
|
|
576
|
-
// Optional: Add delay to simulate real-time streaming
|
|
577
|
-
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
578
|
-
}
|
|
579
|
-
|
|
580
|
-
// Signal end of audio
|
|
581
|
-
connection.commit();
|
|
582
|
-
```
|
|
583
|
-
|
|
584
|
-
### Event Handlers
|
|
585
|
-
|
|
586
|
-
Subscribe to events using the connection instance:
|
|
587
|
-
|
|
588
|
-
```js
|
|
589
|
-
import { RealtimeEvents } from "@elevenlabs/client";
|
|
590
|
-
|
|
591
|
-
// Session started
|
|
592
|
-
connection.on(RealtimeEvents.SESSION_STARTED, () => {
|
|
593
|
-
console.log("Session started");
|
|
594
|
-
});
|
|
595
|
-
|
|
596
|
-
// Partial transcripts (interim results)
|
|
597
|
-
connection.on(RealtimeEvents.PARTIAL_TRANSCRIPT, (data) => {
|
|
598
|
-
console.log("Partial:", data.text);
|
|
599
|
-
});
|
|
600
|
-
|
|
601
|
-
// Committed transcripts
|
|
602
|
-
connection.on(RealtimeEvents.COMMITTED_TRANSCRIPT, (data) => {
|
|
603
|
-
console.log("Committed:", data.text);
|
|
604
|
-
});
|
|
605
|
-
|
|
606
|
-
// Committed transcripts with word-level timestamps
|
|
607
|
-
// Only received when `includeTimestamps = true`
|
|
608
|
-
connection.on(RealtimeEvents.COMMITTED_TRANSCRIPT_WITH_TIMESTAMPS, (data) => {
|
|
609
|
-
console.log("Committed:", data.text);
|
|
610
|
-
console.log("Timestamps:", data.words);
|
|
611
|
-
});
|
|
612
|
-
|
|
613
|
-
// Errors
|
|
614
|
-
connection.on(RealtimeEvents.ERROR, (error) => {
|
|
615
|
-
console.error("Error:", error);
|
|
616
|
-
});
|
|
617
|
-
|
|
618
|
-
// Authentication errors
|
|
619
|
-
connection.on(RealtimeEvents.AUTH_ERROR, (data) => {
|
|
620
|
-
console.error("Auth error:", data.error);
|
|
621
|
-
});
|
|
622
|
-
|
|
623
|
-
// Connection opened
|
|
624
|
-
connection.on(RealtimeEvents.OPEN, () => {
|
|
625
|
-
console.log("Connection opened");
|
|
626
|
-
});
|
|
627
|
-
|
|
628
|
-
// Connection closed
|
|
629
|
-
connection.on(RealtimeEvents.CLOSE, () => {
|
|
630
|
-
console.log("Connection closed");
|
|
631
|
-
});
|
|
632
|
-
|
|
633
|
-
// Quota exceeded
|
|
634
|
-
connection.on(RealtimeEvents.QUOTA_EXCEEDED, (data) => {
|
|
635
|
-
console.log("Quota exceeded:", data.error)
|
|
636
|
-
})
|
|
637
|
-
```
|
|
638
|
-
|
|
639
|
-
### Configuration Options
|
|
640
|
-
|
|
641
|
-
#### Common Options
|
|
642
|
-
|
|
643
|
-
All connection modes support these options:
|
|
644
|
-
|
|
645
|
-
```js
|
|
646
|
-
const connection = await scribe.connect({
|
|
647
|
-
token: "your-token", // Required: Single-use token
|
|
648
|
-
modelId: "scribe_v2_realtime", // Required: Model ID
|
|
649
|
-
baseUri: "wss://api.elevenlabs.io", // Optional: Custom endpoint
|
|
650
|
-
|
|
651
|
-
// Voice Activity Detection (VAD) settings
|
|
652
|
-
commitStrategy: CommitStrategy.MANUAL, // or CommitStrategy.VAD
|
|
653
|
-
vadSilenceThresholdSecs: 0.5, // Seconds of silence before committing
|
|
654
|
-
vadThreshold: 0.5, // VAD sensitivity (0-1)
|
|
655
|
-
minSpeechDurationMs: 100, // Minimum speech duration to process
|
|
656
|
-
minSilenceDurationMs: 500, // Minimum silence to detect pause
|
|
657
|
-
|
|
658
|
-
languageCode: "en", // ISO 639-1 language code
|
|
659
|
-
|
|
660
|
-
includeTimestamps: true // Whether to receive the committed_transcript_with_timestamps event after committing
|
|
661
|
-
});
|
|
662
|
-
```
|
|
663
|
-
|
|
664
|
-
#### Microphone-Specific Options
|
|
665
|
-
|
|
666
|
-
```js
|
|
667
|
-
const connection = await scribe.connect({
|
|
668
|
-
// ... common options
|
|
669
|
-
microphone: {
|
|
670
|
-
deviceId: "optional-device-id",
|
|
671
|
-
echoCancellation: true,
|
|
672
|
-
noiseSuppression: true,
|
|
673
|
-
autoGainControl: true,
|
|
674
|
-
channelCount: 1,
|
|
32
|
+
onMessage: (message) => {
|
|
33
|
+
console.log("Message:", message);
|
|
675
34
|
},
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
#### Manual Audio Options
|
|
680
|
-
|
|
681
|
-
```js
|
|
682
|
-
import { AudioFormat } from "@elevenlabs/client";
|
|
683
|
-
|
|
684
|
-
const connection = Scribe.connect({
|
|
685
|
-
// ... common options
|
|
686
|
-
audioFormat: AudioFormat.PCM_16000, // or AudioFormat.PCM_24000
|
|
687
|
-
sampleRate: 16000, // Must match audioFormat
|
|
688
|
-
});
|
|
689
|
-
```
|
|
690
|
-
|
|
691
|
-
### Commit Strategies
|
|
692
|
-
|
|
693
|
-
Scribe supports two commit strategies when in manual audio mode:
|
|
694
|
-
|
|
695
|
-
#### Manual
|
|
696
|
-
|
|
697
|
-
You explicitly control when to commit transcriptions:
|
|
698
|
-
|
|
699
|
-
```js
|
|
700
|
-
import { Scribe, CommitStrategy, RealtimeEvents } from "@elevenlabs/client";
|
|
701
|
-
|
|
702
|
-
const connection = Scribe.connect({
|
|
703
|
-
token: "your-token",
|
|
704
|
-
modelId: "scribe_v2_realtime",
|
|
705
|
-
commitStrategy: CommitStrategy.MANUAL,
|
|
706
|
-
audioFormat: AudioFormat.PCM_16000,
|
|
707
|
-
sampleRate: 16000,
|
|
708
|
-
});
|
|
709
|
-
|
|
710
|
-
connection.send({ audioBase64: base64Audio });
|
|
711
|
-
|
|
712
|
-
// Later, when you want to commit the segment
|
|
713
|
-
connection.commit();
|
|
714
|
-
```
|
|
715
|
-
|
|
716
|
-
#### Voice Activity Detection (VAD)
|
|
717
|
-
|
|
718
|
-
The API automatically detects when speech ends and commits the transcription:
|
|
719
|
-
|
|
720
|
-
```js
|
|
721
|
-
import { Scribe, CommitStrategy, RealtimeEvents } from "@elevenlabs/client";
|
|
722
|
-
|
|
723
|
-
const connection = Scribe.connect({
|
|
724
|
-
token: "your-token",
|
|
725
|
-
modelId: "scribe_v2_realtime",
|
|
726
|
-
commitStrategy: CommitStrategy.VAD,
|
|
727
|
-
audioFormat: AudioFormat.PCM_16000,
|
|
728
|
-
sampleRate: 16000,
|
|
729
|
-
});
|
|
730
|
-
```
|
|
731
|
-
|
|
732
|
-
### Connection Methods
|
|
733
|
-
|
|
734
|
-
#### close()
|
|
735
|
-
|
|
736
|
-
Close the connection and clean up resources:
|
|
737
|
-
|
|
738
|
-
```js
|
|
739
|
-
connection.close();
|
|
740
|
-
```
|
|
741
|
-
|
|
742
|
-
#### send(options)
|
|
743
|
-
|
|
744
|
-
Send audio data (manual mode only):
|
|
745
|
-
|
|
746
|
-
```js
|
|
747
|
-
connection.send({
|
|
748
|
-
audioBase64: base64AudioData,
|
|
749
|
-
commit: false, // Optional: commit immediately
|
|
750
|
-
sampleRate: 16000, // Optional: override sample rate
|
|
751
|
-
previousText: "Previous transcription text", // Optional: include text from a previous transcription or base64 encoded audio data. Will be used to provide context to the model. Can only be sent in the first audio chunk.
|
|
752
|
-
});
|
|
753
|
-
```
|
|
754
|
-
|
|
755
|
-
**Warning:** The `previousText`field can only be sent in the first audio chunk of a session. If sent in any other chunk an error will be returned.
|
|
756
|
-
|
|
757
|
-
#### commit()
|
|
758
|
-
|
|
759
|
-
Manually commit the current segment:
|
|
760
|
-
|
|
761
|
-
```js
|
|
762
|
-
connection.commit();
|
|
763
|
-
```
|
|
764
|
-
|
|
765
|
-
### TypeScript Support
|
|
766
|
-
|
|
767
|
-
Full TypeScript types are included:
|
|
768
|
-
|
|
769
|
-
```typescript
|
|
770
|
-
import {
|
|
771
|
-
Scribe,
|
|
772
|
-
RealtimeConnection,
|
|
773
|
-
AudioFormat,
|
|
774
|
-
CommitStrategy,
|
|
775
|
-
RealtimeEvents,
|
|
776
|
-
type AudioOptions,
|
|
777
|
-
type MicrophoneOptions,
|
|
778
|
-
type PartialTranscriptMessage,
|
|
779
|
-
type CommittedTranscriptMessage,
|
|
780
|
-
} from "@elevenlabs/client";
|
|
781
|
-
|
|
782
|
-
const connection: RealtimeConnection = await scribe.connect({
|
|
783
|
-
token: "your-token",
|
|
784
|
-
modelId: "scribe_v2_realtime",
|
|
785
|
-
microphone: {
|
|
786
|
-
echoCancellation: true,
|
|
35
|
+
onError: (message) => {
|
|
36
|
+
console.error("Error:", message);
|
|
787
37
|
},
|
|
788
38
|
});
|
|
789
|
-
```
|
|
790
39
|
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
Always handle errors appropriately:
|
|
794
|
-
|
|
795
|
-
```js
|
|
796
|
-
import { Scribe, RealtimeEvents } from "@elevenlabs/client";
|
|
797
|
-
|
|
798
|
-
try {
|
|
799
|
-
const connection = Scribe.connect({
|
|
800
|
-
token: "your-token",
|
|
801
|
-
modelId: "scribe_v2_realtime",
|
|
802
|
-
microphone: {},
|
|
803
|
-
});
|
|
804
|
-
|
|
805
|
-
// Generic event that fires on all errors, including auth and quota exceeded
|
|
806
|
-
connection.on(RealtimeEvents.ERROR, (error) => {
|
|
807
|
-
console.error("Connection error:", error);
|
|
808
|
-
});
|
|
809
|
-
|
|
810
|
-
connection.on(RealtimeEvents.AUTH_ERROR, (data) => {
|
|
811
|
-
console.error("Authentication failed:", data.error);
|
|
812
|
-
});
|
|
813
|
-
|
|
814
|
-
connection.on(RealtimeEvents.QUOTA_EXCEEDED, (data) => {
|
|
815
|
-
console.error("Quota exceeded:", data.error);
|
|
816
|
-
});
|
|
817
|
-
} catch (error) {
|
|
818
|
-
console.error("Failed to connect:", error);
|
|
819
|
-
}
|
|
820
|
-
```
|
|
821
|
-
|
|
822
|
-
## CSP compliance
|
|
823
|
-
|
|
824
|
-
If your application has a tight Content Security Policy and does not allow data: or blob: in the `script-src` (w3.org/TR/CSP2#source-list-guid-matching), you self-host the needed files in the public folder.
|
|
825
|
-
|
|
826
|
-
Whitelisting these values is not recommended w3.org/TR/CSP2#source-list-guid-matching.
|
|
827
|
-
|
|
828
|
-
Add the worklet files to your public folder eg `public/elevenlabs`.
|
|
829
|
-
|
|
830
|
-
```
|
|
831
|
-
@elevenlabs/client/scripts/
|
|
40
|
+
// End the conversation
|
|
41
|
+
await conversation.endSession();
|
|
832
42
|
```
|
|
833
43
|
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
It is recommended to update the scripts with a build script like
|
|
44
|
+
## Documentation
|
|
837
45
|
|
|
838
|
-
|
|
839
|
-
import { viteStaticCopy } from 'vite-plugin-static-copy'
|
|
840
|
-
import { createRequire } from 'node:module';
|
|
841
|
-
import path from 'path';
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
const require = createRequire(import.meta.url);
|
|
845
|
-
|
|
846
|
-
export default {
|
|
847
|
-
plugins: [
|
|
848
|
-
viteStaticCopy({
|
|
849
|
-
targets: [
|
|
850
|
-
{
|
|
851
|
-
src: require.resolve('@elevenlabs/client')/dist/worklets/audioConcatProcessor.js',
|
|
852
|
-
dest: 'dist',
|
|
853
|
-
},
|
|
854
|
-
{
|
|
855
|
-
src: require.resolve('@elevenlabs/client')/dist/worklets/rawAudioProcessor.js',
|
|
856
|
-
dest: 'dist',
|
|
857
|
-
},
|
|
858
|
-
],
|
|
859
|
-
}),
|
|
860
|
-
],
|
|
861
|
-
}
|
|
862
|
-
```
|
|
46
|
+
For the full API reference including connection types, client tools, conversation overrides, and more, see the [JavaScript SDK documentation](https://elevenlabs.io/docs/eleven-agents/libraries/java-script).
|
|
863
47
|
|
|
864
48
|
## Development
|
|
865
49
|
|
|
866
|
-
Please
|
|
50
|
+
Please refer to the README.md file in the root of this repository.
|
|
867
51
|
|
|
868
52
|
## Contributing
|
|
869
53
|
|
|
870
|
-
Please
|
|
54
|
+
Please create an issue first to discuss the proposed changes. Any contributions are welcome!
|
|
871
55
|
|
|
872
56
|
Remember, if merged, your code will be used as part of a MIT licensed project. By submitting a Pull Request, you are giving your consent for your code to be integrated into this library.
|