@d-id/client-sdk 3.0.0-staging.413 → 3.0.0-staging.415
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +2 -0
- package/dist/index-D0K-5SID.js +2037 -0
- package/dist/index.d.ts +3270 -0
- package/dist/index.js +20 -20
- package/dist/index.umd.cjs +13 -13
- package/dist/{index-BOeL7MEm.js → livekit-client.esm-QkbCKsng.js} +6234 -8256
- package/dist/livekit-manager-CU_yynQK.js +557 -0
- package/package.json +33 -9
- package/dist/livekit-manager-D-B3qdMq.js +0 -506
- package/dist/src/api/agents.d.ts +0 -18
- package/dist/src/api/apiClient.d.ts +0 -11
- package/dist/src/api/apiClient.test.d.ts +0 -1
- package/dist/src/api/knowledge.d.ts +0 -15
- package/dist/src/api/streams/index.d.ts +0 -1
- package/dist/src/api/streams/streamApi.d.ts +0 -3
- package/dist/src/api/streams/streamsApiV2.d.ts +0 -5
- package/dist/src/auth/get-auth-header.d.ts +0 -5
- package/dist/src/auth/get-auth-header.test.d.ts +0 -1
- package/dist/src/config/consts.d.ts +0 -3
- package/dist/src/config/environment.d.ts +0 -5
- package/dist/src/errors/base-error.d.ts +0 -182
- package/dist/src/errors/chat/chat-creation-failed.d.ts +0 -30
- package/dist/src/errors/chat/chat-mode-downgraded.d.ts +0 -35
- package/dist/src/errors/errors.test.d.ts +0 -1
- package/dist/src/errors/http-error.d.ts +0 -74
- package/dist/src/errors/index.d.ts +0 -9
- package/dist/src/errors/network-error.d.ts +0 -72
- package/dist/src/errors/request-meta.d.ts +0 -12
- package/dist/src/errors/stream-error.d.ts +0 -37
- package/dist/src/errors/validation-error.d.ts +0 -67
- package/dist/src/errors/ws-error.d.ts +0 -34
- package/dist/src/index.d.ts +0 -20
- package/dist/src/public-api.test.d.ts +0 -1
- package/dist/src/services/agent-manager/connect-to-manager.d.ts +0 -19
- package/dist/src/services/agent-manager/connect-to-manager.test.d.ts +0 -1
- package/dist/src/services/agent-manager/index.d.ts +0 -64
- package/dist/src/services/agent-manager/index.test.d.ts +0 -1
- package/dist/src/services/analytics/mixpanel.d.ts +0 -22
- package/dist/src/services/analytics/mixpanel.test.d.ts +0 -1
- package/dist/src/services/analytics/timestamp-tracker.d.ts +0 -15
- package/dist/src/services/chat/index.d.ts +0 -10
- package/dist/src/services/chat/index.test.d.ts +0 -1
- package/dist/src/services/chat/intial-messages.d.ts +0 -3
- package/dist/src/services/chat/intial-messages.test.d.ts +0 -1
- package/dist/src/services/socket-manager/index.d.ts +0 -12
- package/dist/src/services/socket-manager/message-queue.d.ts +0 -12
- package/dist/src/services/socket-manager/message-queue.test.d.ts +0 -1
- package/dist/src/services/streaming-manager/advanced.test.d.ts +0 -1
- package/dist/src/services/streaming-manager/business-flows.test.d.ts +0 -5
- package/dist/src/services/streaming-manager/common.d.ts +0 -101
- package/dist/src/services/streaming-manager/data-channel-handlers.d.ts +0 -3
- package/dist/src/services/streaming-manager/disconnect.test.d.ts +0 -5
- package/dist/src/services/streaming-manager/edge-cases.test.d.ts +0 -1
- package/dist/src/services/streaming-manager/factory.d.ts +0 -13
- package/dist/src/services/streaming-manager/factory.test.d.ts +0 -1
- package/dist/src/services/streaming-manager/index.d.ts +0 -5
- package/dist/src/services/streaming-manager/livekit-manager.d.ts +0 -10
- package/dist/src/services/streaming-manager/livekit-manager.test.d.ts +0 -1
- package/dist/src/services/streaming-manager/stats/av-sync.d.ts +0 -3
- package/dist/src/services/streaming-manager/stats/av-sync.test.d.ts +0 -1
- package/dist/src/services/streaming-manager/stats/poll.d.ts +0 -16
- package/dist/src/services/streaming-manager/stats/report.d.ts +0 -19
- package/dist/src/services/streaming-manager/stats/report.test.d.ts +0 -1
- package/dist/src/services/streaming-manager/webrtc-core.test.d.ts +0 -1
- package/dist/src/services/streaming-manager/webrtc-manager.d.ts +0 -12
- package/dist/src/services/streaming-manager/webrtc-utils.test.d.ts +0 -5
- package/dist/src/test-utils/factories/agent-manager.factory.d.ts +0 -3
- package/dist/src/test-utils/factories/agent.factory.d.ts +0 -4
- package/dist/src/test-utils/factories/agents-api.factory.d.ts +0 -1
- package/dist/src/test-utils/factories/analytics.factory.d.ts +0 -1
- package/dist/src/test-utils/factories/chat.factory.d.ts +0 -1
- package/dist/src/test-utils/factories/index.d.ts +0 -8
- package/dist/src/test-utils/factories/socket-manager.factory.d.ts +0 -1
- package/dist/src/test-utils/factories/stream-api.factory.d.ts +0 -1
- package/dist/src/test-utils/factories/streaming-manager.factory.d.ts +0 -4
- package/dist/src/types/auth.d.ts +0 -95
- package/dist/src/types/entities/agents/agent.d.ts +0 -202
- package/dist/src/types/entities/agents/chat.d.ts +0 -476
- package/dist/src/types/entities/agents/index.d.ts +0 -3
- package/dist/src/types/entities/agents/manager.d.ts +0 -896
- package/dist/src/types/entities/avatar.d.ts +0 -21
- package/dist/src/types/entities/index.d.ts +0 -3
- package/dist/src/types/entities/knowledge/document.d.ts +0 -31
- package/dist/src/types/entities/knowledge/index.d.ts +0 -4
- package/dist/src/types/entities/knowledge/knowledge.d.ts +0 -56
- package/dist/src/types/entities/knowledge/record.d.ts +0 -24
- package/dist/src/types/entities/knowledge/retrival.d.ts +0 -9
- package/dist/src/types/face-rect.d.ts +0 -6
- package/dist/src/types/index.d.ts +0 -6
- package/dist/src/types/stream/api/clip.d.ts +0 -122
- package/dist/src/types/stream/api/index.d.ts +0 -2
- package/dist/src/types/stream/api/talk.d.ts +0 -77
- package/dist/src/types/stream/data-channel.d.ts +0 -16
- package/dist/src/types/stream/index.d.ts +0 -4
- package/dist/src/types/stream/rtc.d.ts +0 -114
- package/dist/src/types/stream/stream.d.ts +0 -785
- package/dist/src/types/stream/streams-v2.d.ts +0 -28
- package/dist/src/types/stream-script.d.ts +0 -138
- package/dist/src/types/voice/stt.d.ts +0 -16
- package/dist/src/types/voice/tts.d.ts +0 -238
- package/dist/src/utils/agent.d.ts +0 -8
- package/dist/src/utils/analytics.d.ts +0 -31
- package/dist/src/utils/analytics.test.d.ts +0 -1
- package/dist/src/utils/chat.d.ts +0 -4
- package/dist/src/utils/content-parser.d.ts +0 -35
- package/dist/src/utils/content-parser.test.d.ts +0 -1
- package/dist/src/utils/defer.d.ts +0 -7
- package/dist/src/utils/error-analytics.d.ts +0 -4
- package/dist/src/utils/error-analytics.test.d.ts +0 -1
- package/dist/src/utils/index.d.ts +0 -3
- package/dist/src/utils/retry-operation.d.ts +0 -38
- package/dist/src/utils/stream-end.d.ts +0 -5
- package/dist/src/utils/stream-end.test.d.ts +0 -1
- package/dist/src/utils/tool-calls.d.ts +0 -27
- package/dist/src/utils/tool-calls.test.d.ts +0 -1
- package/dist/src/utils/user-context.d.ts +0 -15
- package/dist/src/utils/user-context.test.d.ts +0 -1
- package/dist/src/version.d.ts +0 -7
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,3270 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* An agent's profile, as the Agents API returns it.
|
|
3
|
+
*
|
|
4
|
+
* The SDK fetches it once while {@link createAgentManager} runs and exposes it as
|
|
5
|
+
* {@link AgentManager.agent}, so everything here is available before
|
|
6
|
+
* {@link AgentManager.connect | connect()} — which is what makes it useful for rendering the agent
|
|
7
|
+
* before the video exists. The fields are the agent's configuration, whether it was set in D-ID Studio
|
|
8
|
+
* or through the Agents API; most of them are for the application to use, and each one below says
|
|
9
|
+
* where the SDK itself reads it.
|
|
10
|
+
*
|
|
11
|
+
* @see [Get an agent](https://docs.d-id.com/reference/agent-get)
|
|
12
|
+
* @category Agent Manager
|
|
13
|
+
*/
|
|
14
|
+
export declare interface Agent {
|
|
15
|
+
/** Id of the agent: the same value passed to {@link createAgentManager}. */
|
|
16
|
+
id: string;
|
|
17
|
+
/** Id of the D-ID account that owns the agent. Set by the Agents API. */
|
|
18
|
+
owner_id?: string;
|
|
19
|
+
/** The agent's display name. */
|
|
20
|
+
name?: string;
|
|
21
|
+
/** `'public'` when the agent is configured for public access; omitted otherwise. */
|
|
22
|
+
access?: 'public';
|
|
23
|
+
/** URL of the agent's still image. Use it as the poster of your video element before
|
|
24
|
+
* {@link AgentManager.connect | connect()} has a stream to show. */
|
|
25
|
+
thumbnail?: string;
|
|
26
|
+
/**
|
|
27
|
+
* The greeting lines configured for the agent.
|
|
28
|
+
*
|
|
29
|
+
* The SDK never speaks them by itself: pass one to {@link AgentManager.speak | speak()} once
|
|
30
|
+
* the agent is connected if you want the agent to open the conversation. Check what your own
|
|
31
|
+
* agent does first — an Expressive (V4) session's server side may greet on its own, in which
|
|
32
|
+
* case speaking a greeting as well says it twice.
|
|
33
|
+
*/
|
|
34
|
+
greetings?: string[];
|
|
35
|
+
/**
|
|
36
|
+
* Suggested opening questions to offer the user.
|
|
37
|
+
*
|
|
38
|
+
* The SDK surfaces the same array as {@link AgentManager.starterMessages | starterMessages}.
|
|
39
|
+
*/
|
|
40
|
+
starter_message?: string[];
|
|
41
|
+
/**
|
|
42
|
+
* URL of the agent's looping idle video.
|
|
43
|
+
*
|
|
44
|
+
* Show it while the agent is not speaking: put it on your video element's `src` when
|
|
45
|
+
* {@link AgentManagerCallbacks.onVideoStateChange | onVideoStateChange} reports `STOP`.
|
|
46
|
+
*/
|
|
47
|
+
idle_video?: string;
|
|
48
|
+
/**
|
|
49
|
+
* The knowledge base attached to the agent, when it has one.
|
|
50
|
+
*
|
|
51
|
+
* Its presence is what tells you the agent can cite sources; the citations themselves arrive on
|
|
52
|
+
* each answer as {@link Message.matches | matches}.
|
|
53
|
+
*/
|
|
54
|
+
knowledge?: {
|
|
55
|
+
/**
|
|
56
|
+
* Id of the knowledge base.
|
|
57
|
+
*
|
|
58
|
+
* The SDK sends it with every rating made by {@link AgentManager.rate | rate()}, and it is
|
|
59
|
+
* the {@link RetrievalMetadata.knowledge_id | knowledge_id} of the citations on an answer.
|
|
60
|
+
*/
|
|
61
|
+
id: string;
|
|
62
|
+
/** The embedding model the knowledge base was indexed with. The SDK does not act on it. */
|
|
63
|
+
embedder?: {
|
|
64
|
+
/** Whether that embedding model covers only a limited set of languages, so knowledge
|
|
65
|
+
* written in another language may not be retrieved well. */
|
|
66
|
+
is_limited_language?: boolean;
|
|
67
|
+
};
|
|
68
|
+
};
|
|
69
|
+
/**
|
|
70
|
+
* The avatar the agent is built on.
|
|
71
|
+
*
|
|
72
|
+
* Its `type` is the single most important field on an agent: it decides which transport the
|
|
73
|
+
* session uses and therefore which SDK features work. See {@link AgentAvatar}.
|
|
74
|
+
*/
|
|
75
|
+
avatar: AgentAvatar;
|
|
76
|
+
/**
|
|
77
|
+
* Whether vision is enabled for the agent.
|
|
78
|
+
*
|
|
79
|
+
* Exposed so an application can decide whether to offer the camera; the SDK does not read it,
|
|
80
|
+
* and publishing a camera stream is an Expressive (V4) feature — see
|
|
81
|
+
* {@link AgentManager.publishCameraStream | publishCameraStream()}.
|
|
82
|
+
*/
|
|
83
|
+
vision?: {
|
|
84
|
+
/** `true` when the agent is configured to use the user's camera. */
|
|
85
|
+
enabled: boolean;
|
|
86
|
+
};
|
|
87
|
+
/**
|
|
88
|
+
* The end-of-call feedback form configured for the agent.
|
|
89
|
+
*
|
|
90
|
+
* The SDK does not read it: render your own form from it and submit the result with
|
|
91
|
+
* {@link AgentManager.submitFeedback | submitFeedback()}. See
|
|
92
|
+
* {@link EndOfCallFeedbackConfig}.
|
|
93
|
+
*/
|
|
94
|
+
end_of_call_feedback?: EndOfCallFeedbackConfig;
|
|
95
|
+
/** Whether the agent has triggers configured. The SDK does not act on them. */
|
|
96
|
+
triggers_available?: boolean;
|
|
97
|
+
/** Settings that change how D-ID's own surfaces present the agent. */
|
|
98
|
+
advanced_settings?: {
|
|
99
|
+
/**
|
|
100
|
+
* Whether the SDK logs its streaming lifecycle to the browser console.
|
|
101
|
+
*
|
|
102
|
+
* When it is enabled the SDK turns debug logging on for every visitor, whatever
|
|
103
|
+
* {@link AgentManagerOptions.debug | debug} was passed to {@link createAgentManager}.
|
|
104
|
+
*/
|
|
105
|
+
ui_debug_mode?: boolean;
|
|
106
|
+
/**
|
|
107
|
+
* Account the agent's rendering VM is billed to. D-ID internal; kept in the declarations
|
|
108
|
+
* for D-ID's own applications, hidden from the reference.
|
|
109
|
+
*
|
|
110
|
+
* @hidden
|
|
111
|
+
*/
|
|
112
|
+
vm_account_id?: string;
|
|
113
|
+
/** Whether the embedding UI should show closed captions. The SDK does not act on it. */
|
|
114
|
+
closed_captions_enabled?: boolean;
|
|
115
|
+
};
|
|
116
|
+
/**
|
|
117
|
+
* Feature-flag targeting context for the account this agent belongs to.
|
|
118
|
+
*
|
|
119
|
+
* Used by D-ID's own applications; not part of the SDK contract and may change without notice.
|
|
120
|
+
*/
|
|
121
|
+
ld_context?: {
|
|
122
|
+
/** Key identifying the context. */
|
|
123
|
+
key: string;
|
|
124
|
+
/** Hashed form of the key. */
|
|
125
|
+
hash_key: string;
|
|
126
|
+
/** The plan group the account falls into. */
|
|
127
|
+
plan_group: string;
|
|
128
|
+
};
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/**
|
|
132
|
+
* What the agent is doing right now, for driving the application's UI.
|
|
133
|
+
*
|
|
134
|
+
* The argument of
|
|
135
|
+
* {@link AgentManagerCallbacks.onAgentActivityStateChange | onAgentActivityStateChange}. Expressive
|
|
136
|
+
* (V4) agents move through all four states; Talks (V2) and Clips (V3) agents only alternate between
|
|
137
|
+
* {@link AgentActivityState.Talking | TALKING} and {@link AgentActivityState.Idle | IDLE}, and only
|
|
138
|
+
* on fluent streams.
|
|
139
|
+
*
|
|
140
|
+
* @category Callbacks & Events
|
|
141
|
+
*/
|
|
142
|
+
export declare enum AgentActivityState {
|
|
143
|
+
/** The agent is doing nothing and is waiting for the user — the conversational turn has ended. */
|
|
144
|
+
Idle = "IDLE",
|
|
145
|
+
/**
|
|
146
|
+
* The agent is working on an answer that has not started streaming yet.
|
|
147
|
+
*
|
|
148
|
+
* Expressive (V4) only: set when a turn starts and when the user's speech has just been
|
|
149
|
+
* transcribed. Use it for a typing or thinking indicator.
|
|
150
|
+
*/
|
|
151
|
+
Loading = "LOADING",
|
|
152
|
+
/** The agent's video has started; the agent is speaking. */
|
|
153
|
+
Talking = "TALKING",
|
|
154
|
+
/**
|
|
155
|
+
* The agent is running one or more tool calls.
|
|
156
|
+
*
|
|
157
|
+
* Expressive (V4) only: set when a tool call starts, and returned to after the agent finishes
|
|
158
|
+
* speaking while any call is still outstanding (for example a client tool waiting on the user).
|
|
159
|
+
* The individual calls are reported by
|
|
160
|
+
* {@link AgentManagerCallbacks.onRunningToolCallsChange | onRunningToolCallsChange}.
|
|
161
|
+
*/
|
|
162
|
+
ToolActive = "TOOL_ACTIVE"
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/**
|
|
166
|
+
* The avatar an agent speaks through: its rendering tier and the language of its voice.
|
|
167
|
+
*
|
|
168
|
+
* The value of {@link Agent.avatar}. Its `type` is the single most important field on an agent: it
|
|
169
|
+
* decides which transport the session uses and therefore which SDK features work.
|
|
170
|
+
*
|
|
171
|
+
* @category Agent Manager
|
|
172
|
+
*/
|
|
173
|
+
export declare interface AgentAvatar {
|
|
174
|
+
/**
|
|
175
|
+
* The kind of avatar: {@link AvatarType.Talk | talk} for Talks (V2), {@link AvatarType.Clip | clip}
|
|
176
|
+
* for Clips (V3), {@link AvatarType.Expressive | expressive} for Expressive (V4).
|
|
177
|
+
*
|
|
178
|
+
* Talks (V2) and Clips (V3) agents stream over WebRTC and receive their events on the
|
|
179
|
+
* notifications web socket; Expressive (V4) agents connect to a real-time session instead, which
|
|
180
|
+
* is what makes the microphone, camera, client tool and data-channel methods of
|
|
181
|
+
* {@link AgentManager} available.
|
|
182
|
+
*/
|
|
183
|
+
type: AvatarType;
|
|
184
|
+
/** The voice the agent speaks with. */
|
|
185
|
+
voice?: {
|
|
186
|
+
/**
|
|
187
|
+
* Language of the configured voice. The SDK reports it in analytics; it does not change how
|
|
188
|
+
* the session behaves.
|
|
189
|
+
*/
|
|
190
|
+
language?: string;
|
|
191
|
+
};
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/**
|
|
195
|
+
* A live connection to one agent: its profile, its chat, and the video stream it answers on.
|
|
196
|
+
*
|
|
197
|
+
* Created by {@link createAgentManager}. Nothing happens on the wire until
|
|
198
|
+
* {@link AgentManager.connect | connect()} resolves; after that
|
|
199
|
+
* {@link AgentManager.chat | chat()} makes the agent answer with its own LLM and
|
|
200
|
+
* {@link AgentManager.speak | speak()} makes it say exactly what you give it. Call
|
|
201
|
+
* {@link AgentManager.disconnect | disconnect()} when the user leaves.
|
|
202
|
+
*
|
|
203
|
+
* Two properties, {@link AgentManager.agent | agent} and
|
|
204
|
+
* {@link AgentManager.starterMessages | starterMessages}, are readable as soon as the manager
|
|
205
|
+
* exists; everything else is a method. Some methods work only with some avatar types: each one
|
|
206
|
+
* says so, and {@link AgentAvatar} is where the session's tier is read from.
|
|
207
|
+
*
|
|
208
|
+
* Every method that reaches the Agents API can reject with an {@link HttpError} when the request
|
|
209
|
+
* comes back non-2xx, or a {@link NetworkError} when it never reaches the server; the individual
|
|
210
|
+
* `@throws` entries below name the errors that are specific to each method.
|
|
211
|
+
*
|
|
212
|
+
* @category Agent Manager
|
|
213
|
+
*/
|
|
214
|
+
export declare interface AgentManager {
|
|
215
|
+
/**
|
|
216
|
+
* The agent this manager is connected to, as the Agents API returned it.
|
|
217
|
+
*
|
|
218
|
+
* Fetched once while the manager is created, so it is available before
|
|
219
|
+
* {@link AgentManager.connect | connect()}. Useful for rendering the agent's name, thumbnail
|
|
220
|
+
* and {@link Agent.idle_video | idle_video}. See {@link Agent}.
|
|
221
|
+
*/
|
|
222
|
+
agent: Agent;
|
|
223
|
+
/**
|
|
224
|
+
* Returns the kind of stream the current session negotiated.
|
|
225
|
+
*
|
|
226
|
+
* @returns {@link StreamType.Fluent} for a fluent stream (one video for the idle and talking
|
|
227
|
+
* states), {@link StreamType.Legacy} for the two-element legacy mode, or `undefined` before
|
|
228
|
+
* {@link AgentManager.connect | connect()} has established a session.
|
|
229
|
+
*/
|
|
230
|
+
getStreamType(): StreamType | undefined;
|
|
231
|
+
/**
|
|
232
|
+
* Returns whether the current stream supports interrupting the agent mid-answer.
|
|
233
|
+
*
|
|
234
|
+
* True on a fluent stream — a Clips (V3) agent built on a Pro avatar, or any Expressive (V4)
|
|
235
|
+
* agent — once connected. This says the stream supports interrupting at all;
|
|
236
|
+
* {@link AgentManagerCallbacks.onInterruptibleChange | onInterruptibleChange} says whether
|
|
237
|
+
* interrupting is allowed right now.
|
|
238
|
+
*
|
|
239
|
+
* @returns `true` when {@link AgentManager.interrupt | interrupt()} can do anything.
|
|
240
|
+
*/
|
|
241
|
+
isInterruptAvailable(): boolean;
|
|
242
|
+
/**
|
|
243
|
+
* The agent's starter messages.
|
|
244
|
+
*
|
|
245
|
+
* Suggested openers to offer the user as buttons; empty when the agent defines none. Available
|
|
246
|
+
* before {@link AgentManager.connect | connect()}.
|
|
247
|
+
*/
|
|
248
|
+
starterMessages: string[];
|
|
249
|
+
/**
|
|
250
|
+
* Fetches a short-lived token for the D-ID speech-to-text service.
|
|
251
|
+
*
|
|
252
|
+
* The SDK sends the request whenever it is called, connected or not; the service decides
|
|
253
|
+
* whether to issue a token for the agent.
|
|
254
|
+
*
|
|
255
|
+
* @returns The {@link SttTokenResponse} for this agent.
|
|
256
|
+
* @throws {@link HttpError} When the service does not answer with a token, or the request comes
|
|
257
|
+
* back non-2xx for any other reason.
|
|
258
|
+
* @throws {@link NetworkError} When the request never reaches the server.
|
|
259
|
+
*/
|
|
260
|
+
getSttToken(): Promise<SttTokenResponse>;
|
|
261
|
+
/**
|
|
262
|
+
* Opens a new session with the agent: a new WebRTC connection, a new web socket and a new chat.
|
|
263
|
+
*
|
|
264
|
+
* Resolves once the connection reaches {@link ConnectionState.Connected | 'connected'}, by
|
|
265
|
+
* which point {@link AgentManagerCallbacks.onSrcObjectReady | onSrcObjectReady} has been called
|
|
266
|
+
* with the media stream to render.
|
|
267
|
+
*
|
|
268
|
+
* One session at a time: while a call is still in flight a second call returns that same
|
|
269
|
+
* promise rather than opening a second session, which is what makes it safe in a React
|
|
270
|
+
* StrictMode effect. Once a session exists it rejects instead — call
|
|
271
|
+
* {@link AgentManager.disconnect | disconnect()} first to start a fresh conversation, or
|
|
272
|
+
* {@link AgentManager.reconnect | reconnect()} to keep the current one.
|
|
273
|
+
*
|
|
274
|
+
* @returns Resolves when the agent is connected and ready.
|
|
275
|
+
* @throws {@link ValidationError} When a session is already open.
|
|
276
|
+
* @throws {@link HttpError} When creating the stream or the chat comes back non-2xx — a client
|
|
277
|
+
* key that is not authorized for the agent or the calling domain, or an account out of
|
|
278
|
+
* credits. The SDK tries the initialization up to three times first, except on `429` and on an
|
|
279
|
+
* out-of-credits response.
|
|
280
|
+
* @throws {@link NetworkError} When those requests never reach the server.
|
|
281
|
+
* @example
|
|
282
|
+
* ```ts
|
|
283
|
+
* await agentManager.connect();
|
|
284
|
+
* ```
|
|
285
|
+
*/
|
|
286
|
+
connect(): Promise<void>;
|
|
287
|
+
/**
|
|
288
|
+
* Reopens the stream when the session expires, and continues the conversation on the same chat.
|
|
289
|
+
*
|
|
290
|
+
* The chat id normally does not change, so the agent keeps its context. It starts a new stream
|
|
291
|
+
* — including after the server ended the previous one deliberately — rather than resuming the
|
|
292
|
+
* old one. On Expressive (V4) agents the transport is asked to reconnect first; if that fails
|
|
293
|
+
* the SDK falls back to a disconnect and a fresh connect, which starts a new chat id.
|
|
294
|
+
*
|
|
295
|
+
* @returns Resolves when the new stream is connected.
|
|
296
|
+
* @throws {@link ValidationError} When a {@link AgentManager.connect | connect()} is still in
|
|
297
|
+
* flight; wait for it to settle first.
|
|
298
|
+
* @throws {@link HttpError} When creating the new stream comes back non-2xx.
|
|
299
|
+
* @throws {@link NetworkError} When that request never reaches the server.
|
|
300
|
+
*/
|
|
301
|
+
reconnect(): Promise<void>;
|
|
302
|
+
/**
|
|
303
|
+
* Closes the existing connection and chat with the agent: the stream and the web socket.
|
|
304
|
+
*
|
|
305
|
+
* Call it when the user leaves the page or the conversation, so the session does not keep
|
|
306
|
+
* running. After it resolves, {@link AgentManager.connect | connect()} is needed before the
|
|
307
|
+
* agent can be used again.
|
|
308
|
+
*
|
|
309
|
+
* @returns Resolves once everything is closed.
|
|
310
|
+
*/
|
|
311
|
+
disconnect(): Promise<void>;
|
|
312
|
+
/**
|
|
313
|
+
* Publishes a microphone audio track to the session so the agent can hear the user.
|
|
314
|
+
*
|
|
315
|
+
* Call it after {@link AgentManager.connect | connect()} to enable voice input. Expressive (V4)
|
|
316
|
+
* agents only; on Talks (V2) and Clips (V3) agents, and before `connect()`, the returned
|
|
317
|
+
* promise rejects with a {@link ValidationError}.
|
|
318
|
+
*
|
|
319
|
+
* @param stream - A `MediaStream` whose audio track is published to the session.
|
|
320
|
+
* @returns Resolves once the track is published.
|
|
321
|
+
* @throws {@link ValidationError} When the session is not an Expressive (V4) one, or
|
|
322
|
+
* {@link AgentManager.connect | connect()} has not run yet.
|
|
323
|
+
* @example
|
|
324
|
+
* ```ts
|
|
325
|
+
* const micStream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
|
326
|
+
* await agentManager.publishMicrophoneStream(micStream);
|
|
327
|
+
* ```
|
|
328
|
+
*/
|
|
329
|
+
publishMicrophoneStream(stream: MediaStream): Promise<void>;
|
|
330
|
+
/**
|
|
331
|
+
* Stops and removes the currently published microphone track from the session.
|
|
332
|
+
*
|
|
333
|
+
* The counterpart of {@link AgentManager.publishMicrophoneStream | publishMicrophoneStream()} —
|
|
334
|
+
* use it to mute the user for the rest of the session. Expressive (V4) agents only; on Talks
|
|
335
|
+
* (V2) and Clips (V3) agents, and when nothing is published, it resolves without doing
|
|
336
|
+
* anything.
|
|
337
|
+
*
|
|
338
|
+
* @returns Resolves once the track is removed.
|
|
339
|
+
* @example
|
|
340
|
+
* ```ts
|
|
341
|
+
* await agentManager.unpublishMicrophoneStream();
|
|
342
|
+
* ```
|
|
343
|
+
*/
|
|
344
|
+
unpublishMicrophoneStream(): Promise<void>;
|
|
345
|
+
/**
|
|
346
|
+
* Swaps the live microphone track without unpublishing it.
|
|
347
|
+
*
|
|
348
|
+
* Use it when the user picks a different input device: the publication is preserved — its
|
|
349
|
+
* LiveKit publication id (SID) and SSRC stay the same, though the `MediaStreamTrack` id
|
|
350
|
+
* changes — so the server sees continuous audio across the swap rather than a stop and a
|
|
351
|
+
* restart. Rejects with a plain `Error` from the transport when there is no active publication
|
|
352
|
+
* — fall back to {@link AgentManager.publishMicrophoneStream | publishMicrophoneStream()} in
|
|
353
|
+
* that case. Expressive (V4) agents only; on Talks (V2) and Clips (V3) agents, and before
|
|
354
|
+
* {@link AgentManager.connect | connect()}, the returned promise rejects with a
|
|
355
|
+
* {@link ValidationError}.
|
|
356
|
+
*
|
|
357
|
+
* @param track - The audio track to send from now on.
|
|
358
|
+
* @returns Resolves once the transport has switched to the new track.
|
|
359
|
+
* @throws {@link ValidationError} When the session is not an Expressive (V4) one, or
|
|
360
|
+
* {@link AgentManager.connect | connect()} has not run yet.
|
|
361
|
+
*/
|
|
362
|
+
replaceMicrophoneTrack(track: MediaStreamTrack): Promise<void>;
|
|
363
|
+
/**
|
|
364
|
+
* Publishes a camera video track to the session so the agent can see the user.
|
|
365
|
+
*
|
|
366
|
+
* Call it after {@link AgentManager.connect | connect()} to enable vision. Expressive (V4)
|
|
367
|
+
* agents only; on Talks (V2) and Clips (V3) agents, and before `connect()`, the returned
|
|
368
|
+
* promise rejects with a {@link ValidationError}.
|
|
369
|
+
*
|
|
370
|
+
* @param stream - A `MediaStream` whose video track is published to the session.
|
|
371
|
+
* @returns Resolves once the track is published.
|
|
372
|
+
* @throws {@link ValidationError} When the session is not an Expressive (V4) one, or
|
|
373
|
+
* {@link AgentManager.connect | connect()} has not run yet.
|
|
374
|
+
*/
|
|
375
|
+
publishCameraStream(stream: MediaStream): Promise<void>;
|
|
376
|
+
/**
|
|
377
|
+
* Stops and removes the currently published camera track from the session.
|
|
378
|
+
*
|
|
379
|
+
* Call it to disable vision. Expressive (V4) agents only; on Talks (V2) and Clips (V3) agents,
|
|
380
|
+
* and when nothing is published, it resolves without doing anything.
|
|
381
|
+
*
|
|
382
|
+
* @returns Resolves once the track is removed.
|
|
383
|
+
*/
|
|
384
|
+
unpublishCameraStream(): Promise<void>;
|
|
385
|
+
/**
|
|
386
|
+
* Sends a message to the agent and gets a streamed video based on its answer (LLM).
|
|
387
|
+
*
|
|
388
|
+
* The answer arrives through {@link AgentManagerCallbacks.onNewMessage | onNewMessage} — first
|
|
389
|
+
* as `partial` chunks, then as the full `answer` — while the video plays it. The chat is
|
|
390
|
+
* created on the first call if the session does not have one yet.
|
|
391
|
+
*
|
|
392
|
+
* @param userMessage - The user's message text to send to the agent.
|
|
393
|
+
* @returns The {@link ChatResponse} for this turn.
|
|
394
|
+
* @throws {@link ValidationError} When the message is empty or too long, when the chat mode has
|
|
395
|
+
* chat disabled or is in maintenance, or when the manager is not connected yet.
|
|
396
|
+
* @throws {@link ChatCreationFailed} On Talks (V2) and Clips (V3) agents, when the session has
|
|
397
|
+
* no chat yet and the Agents API answers the creation request without one.
|
|
398
|
+
* @throws {@link HttpError} On Talks (V2) and Clips (V3) agents, and in
|
|
399
|
+
* {@link ChatMode.Playground}, when the message request comes back non-2xx. Expressive (V4)
|
|
400
|
+
* agents send the message over the data channel instead, so no HTTP request is made.
|
|
401
|
+
* @throws {@link NetworkError} On Talks (V2) and Clips (V3) agents, and in
|
|
402
|
+
* {@link ChatMode.Playground}, when that request never reaches the server.
|
|
403
|
+
* @example
|
|
404
|
+
* ```ts
|
|
405
|
+
* const chat = await agentManager.chat('What is the distance to the moon?');
|
|
406
|
+
* ```
|
|
407
|
+
*/
|
|
408
|
+
chat(userMessage: string): Promise<ChatResponse>;
|
|
409
|
+
/**
|
|
410
|
+
* Rates one of the agent's answers in the chat, for future analytics and insights.
|
|
411
|
+
*
|
|
412
|
+
* Pass `rateId` to change a rating the user already gave instead of adding another.
|
|
413
|
+
*
|
|
414
|
+
* @param messageId - Id of the message being rated.
|
|
415
|
+
* @param score - 1 for a positive rating, -1 for a negative one.
|
|
416
|
+
* @param rateId - Id of an existing rating to update; omit to create a new one.
|
|
417
|
+
* @returns The created or updated {@link Rating}.
|
|
418
|
+
* @throws {@link ValidationError} When no chat has started, or when no message with that id is
|
|
419
|
+
* in the transcript.
|
|
420
|
+
* @throws {@link HttpError} When the rating request comes back non-2xx.
|
|
421
|
+
* @throws {@link NetworkError} When the rating request never reaches the server.
|
|
422
|
+
*/
|
|
423
|
+
rate(messageId: string, score: 1 | -1, rateId?: string): Promise<Rating>;
|
|
424
|
+
/**
|
|
425
|
+
* Removes a rating the user gave to an answer in the chat.
|
|
426
|
+
*
|
|
427
|
+
* @param id - Id of the rating to remove, as returned by
|
|
428
|
+
* {@link AgentManager.rate | rate()}.
|
|
429
|
+
* @returns The {@link Rating} that was deleted.
|
|
430
|
+
* @throws {@link ValidationError} When no chat has started.
|
|
431
|
+
* @throws {@link HttpError} When the delete request comes back non-2xx.
|
|
432
|
+
* @throws {@link NetworkError} When the delete request never reaches the server.
|
|
433
|
+
*/
|
|
434
|
+
deleteRate(id: string): Promise<Rating>;
|
|
435
|
+
/**
|
|
436
|
+
* Submits end-of-call feedback for the whole conversation.
|
|
437
|
+
*
|
|
438
|
+
* Separate from {@link AgentManager.rate | rate()}, which scores a single answer. Collect it
|
|
439
|
+
* when the user ends the call, using the agent's end-of-call feedback configuration.
|
|
440
|
+
*
|
|
441
|
+
* @param rating - The user's score for the conversation, a whole number from 1 to 5.
|
|
442
|
+
* @param answer - The user's free-text answer to the follow-up question, when one was asked.
|
|
443
|
+
* @returns The stored {@link SubmitFeedbackResponse}.
|
|
444
|
+
* @throws {@link ValidationError} When no chat has started.
|
|
445
|
+
* @throws {@link HttpError} When the feedback request comes back non-2xx.
|
|
446
|
+
* @throws {@link NetworkError} When the feedback request never reaches the server.
|
|
447
|
+
*/
|
|
448
|
+
submitFeedback(rating: number, answer?: string): Promise<SubmitFeedbackResponse>;
|
|
449
|
+
/**
|
|
450
|
+
* Makes the agent stream back a video based on the text or audio file you provide.
|
|
451
|
+
*
|
|
452
|
+
* Unlike {@link AgentManager.chat | chat()} the agent's LLM is not involved, so this is how you
|
|
453
|
+
* script greetings and canned lines. Pass a plain string as a shorthand for a text script. See
|
|
454
|
+
* {@link SpeakScript}, {@link TextStreamScript} and {@link AudioStreamScript}. Text
|
|
455
|
+
* scripts also accept an optional `sentiment`, for Expressive (V4) agents only; if the
|
|
456
|
+
* requested sentiment is not supported by the agent, the default sentiment is used.
|
|
457
|
+
*
|
|
458
|
+
* @see [Create a video stream](https://docs.d-id.com/reference/createvideoagentstream) — the request
|
|
459
|
+
* sent for Talks (V2) and Clips (V3) agents.
|
|
460
|
+
* @see [Control the Agent](https://docs.d-id.com/docs/livekit-commands) — the `did.speak` command
|
|
461
|
+
* sent over the data channel for Expressive (V4) agents.
|
|
462
|
+
* @param payload - A text or audio script, or a string treated as the text to speak.
|
|
463
|
+
* @returns The {@link SpeakResponse} for the video that was produced, or the same
|
|
464
|
+
* response with `duration` `0` and an empty `videoId` when the call produced no discrete video
|
|
465
|
+
* — on Expressive (V4) agents, and in a text-only chat mode.
|
|
466
|
+
* @throws {@link ValidationError} When the manager is not connected to a stream yet.
|
|
467
|
+
* @throws {@link HttpError} On Talks (V2) and Clips (V3) agents, when the Agents API answers
|
|
468
|
+
* the request non-2xx. Expressive (V4) agents send the script over the data channel instead,
|
|
469
|
+
* so no HTTP request is made.
|
|
470
|
+
* @throws {@link NetworkError} On Talks (V2) and Clips (V3) agents, when that request never
|
|
471
|
+
* reaches the server.
|
|
472
|
+
* @example Text
|
|
473
|
+
* ```ts
|
|
474
|
+
* const speak = await agentManager.speak({
|
|
475
|
+
* type: 'text',
|
|
476
|
+
* input: "Hi! I'm Alice!",
|
|
477
|
+
* });
|
|
478
|
+
* ```
|
|
479
|
+
* @example Text with sentiment (Expressive (V4) agents)
|
|
480
|
+
* ```ts
|
|
481
|
+
* const speak = await agentManager.speak({
|
|
482
|
+
* type: 'text',
|
|
483
|
+
* input: "Hi! I'm Alice!",
|
|
484
|
+
* sentiment: 'friendly',
|
|
485
|
+
* });
|
|
486
|
+
* ```
|
|
487
|
+
* @example Audio file
|
|
488
|
+
* ```ts
|
|
489
|
+
* const speak = await agentManager.speak({
|
|
490
|
+
* type: 'audio',
|
|
491
|
+
* audio_url: 'https://www.yourwebsite.com/audio.mp3',
|
|
492
|
+
* });
|
|
493
|
+
* ```
|
|
494
|
+
*/
|
|
495
|
+
speak(payload: SpeakScript | string): Promise<SpeakResponse>;
|
|
496
|
+
/**
|
|
497
|
+
* Switches the chat to another mode.
|
|
498
|
+
*
|
|
499
|
+
* Anything other than {@link ChatMode.Functional} disconnects the stream, since those modes do
|
|
500
|
+
* not produce video, which is why this is asynchronous: the returned promise resolves once that
|
|
501
|
+
* disconnect has finished. Switching *into* {@link ChatMode.Functional} disconnects it as well
|
|
502
|
+
* when the open session was built for a mode that skipped what a conversation needs — a
|
|
503
|
+
* {@link ChatMode.DirectPlayback} session has neither the notifications web socket the answer
|
|
504
|
+
* arrives on nor a chat to send to. Call {@link AgentManager.connect | connect()} again after
|
|
505
|
+
* a change that tore the session down. {@link AgentManagerCallbacks.onModeChange | onModeChange}
|
|
506
|
+
* fires once the change has been applied; passing the mode already in effect does nothing.
|
|
507
|
+
*
|
|
508
|
+
* @param mode - The {@link ChatMode} to switch to.
|
|
509
|
+
* @returns A promise resolved when the mode is in effect and any disconnect it caused has
|
|
510
|
+
* completed.
|
|
511
|
+
* @throws {@link ValidationError} Rejects on Expressive (V4) agents for {@link ChatMode.Off} and
|
|
512
|
+
* {@link ChatMode.DirectPlayback}, which those agents do not support.
|
|
513
|
+
*/
|
|
514
|
+
changeMode(mode: ChatMode): Promise<void>;
|
|
515
|
+
/**
|
|
516
|
+
* Adds properties to every analytics event the SDK sends from now on.
|
|
517
|
+
*
|
|
518
|
+
* The same thing
|
|
519
|
+
* {@link AgentManagerOptions.mixpanelAdditionalProperties | mixpanelAdditionalProperties} does
|
|
520
|
+
* at creation time, for values you only learn later.
|
|
521
|
+
*
|
|
522
|
+
* Advanced. Calls merge, so a property sent twice takes the later value, and events already
|
|
523
|
+
* sent are not changed. It has no visible effect when analytics is switched off with
|
|
524
|
+
* {@link AgentManagerOptions.enableAnalytics | enableAnalytics: false}, because nothing is
|
|
525
|
+
* sent at all.
|
|
526
|
+
*
|
|
527
|
+
* @param properties - A flat JSON object whose properties are added to every analytics event
|
|
528
|
+
* the SDK sends from now on.
|
|
529
|
+
*/
|
|
530
|
+
enrichAnalytics(properties: Record<string, unknown>): void;
|
|
531
|
+
/**
|
|
532
|
+
* Interrupts the current video stream mid-playback, so the user can talk over the agent.
|
|
533
|
+
*
|
|
534
|
+
* Supported on a fluent stream — a Clips (V3) agent built on a Pro avatar, or any Expressive
|
|
535
|
+
* (V4) agent. It never throws: on every agent type it returns without doing anything when
|
|
536
|
+
* interrupting is not available for the session, when it is not allowed right now, and — on
|
|
537
|
+
* Talks (V2) and Clips (V3) agents — when the stream is not a fluent stream or no video is
|
|
538
|
+
* playing. Check {@link AgentManager.isInterruptAvailable | isInterruptAvailable()}
|
|
539
|
+
* before offering the control at all; on Expressive (V4) agents
|
|
540
|
+
* {@link AgentManagerCallbacks.onInterruptibleChange | onInterruptibleChange} tracks whether it
|
|
541
|
+
* is allowed right now.
|
|
542
|
+
*
|
|
543
|
+
* The interrupted message is marked as such in the next
|
|
544
|
+
* {@link AgentManagerCallbacks.onNewMessage | onNewMessage} — but only when an interrupt was
|
|
545
|
+
* actually sent. A call that finds nothing to interrupt leaves the message untouched and fires
|
|
546
|
+
* no callback.
|
|
547
|
+
*
|
|
548
|
+
* @param options - What caused the interruption, as an {@link InterruptOptions}: `text`,
|
|
549
|
+
* `audio`, `click` or `manual`. Expressive (V4) agents drop `text` interrupts, because the
|
|
550
|
+
* orchestrator does not cancel the in-flight answer for them.
|
|
551
|
+
*/
|
|
552
|
+
interrupt(options: InterruptOptions): void;
|
|
553
|
+
/**
|
|
554
|
+
* Switches the speech-to-text language in the middle of a session.
|
|
555
|
+
*
|
|
556
|
+
* Expressive (V4) agents only, after {@link AgentManager.connect | connect()}; otherwise the
|
|
557
|
+
* returned promise rejects with a {@link ValidationError}.
|
|
558
|
+
*
|
|
559
|
+
* @param language - The language to transcribe in, as a name or a BCP-47 code — `"English"` or
|
|
560
|
+
* `"en-US"`.
|
|
561
|
+
* @returns Resolves once the new language has been sent to the agent. A room that has dropped
|
|
562
|
+
* since {@link AgentManager.connect | connect()} reports a {@link StreamError} through
|
|
563
|
+
* {@link AgentManagerCallbacks.onError | onError} and the promise still resolves.
|
|
564
|
+
* @throws {@link ValidationError} When the session is not an Expressive (V4) one, or
|
|
565
|
+
* {@link AgentManager.connect | connect()} has not run yet.
|
|
566
|
+
*/
|
|
567
|
+
setSttLanguage(language: string): Promise<void>;
|
|
568
|
+
/**
|
|
569
|
+
* Sends a JSON payload to the agent over a data-channel topic.
|
|
570
|
+
*
|
|
571
|
+
* Use it for application-specific messages that are not speech, such as telling a presentation
|
|
572
|
+
* to change slide. Expressive (V4) agents only, after {@link AgentManager.connect | connect()};
|
|
573
|
+
* otherwise the returned promise rejects with a {@link ValidationError}.
|
|
574
|
+
*
|
|
575
|
+
* @param topic - Data-channel topic to send on. {@link DataChannelTopic} is exported from
|
|
576
|
+
* the package root and lists every topic this method accepts.
|
|
577
|
+
* @param payload - A plain object, sent as JSON.
|
|
578
|
+
* @returns Resolves once the payload has been sent. A room that has dropped since
|
|
579
|
+
* {@link AgentManager.connect | connect()} reports a {@link StreamError} through
|
|
580
|
+
* {@link AgentManagerCallbacks.onError | onError} and the promise still resolves.
|
|
581
|
+
* @throws {@link ValidationError} When the session is not an Expressive (V4) one, or
|
|
582
|
+
* {@link AgentManager.connect | connect()} has not run yet.
|
|
583
|
+
* @example
|
|
584
|
+
* ```ts
|
|
585
|
+
* import { DataChannelTopic } from '@d-id/client-sdk';
|
|
586
|
+
*
|
|
587
|
+
* await agentManager.sendDataChannelMessage(DataChannelTopic.Presentation, {
|
|
588
|
+
* type: 'navigate',
|
|
589
|
+
* slide: 3,
|
|
590
|
+
* });
|
|
591
|
+
* ```
|
|
592
|
+
*/
|
|
593
|
+
sendDataChannelMessage(topic: DataChannelTopic, payload: Record<string, unknown>): Promise<void>;
|
|
594
|
+
/**
|
|
595
|
+
* Registers a handler for a client tool, run in the browser when the agent's LLM calls it.
|
|
596
|
+
*
|
|
597
|
+
* The handler executes on the client and its result is returned to the LLM. Register the
|
|
598
|
+
* handlers before {@link AgentManager.connect | connect()} so the agent can call
|
|
599
|
+
* them from the moment the session starts; registering the same name again replaces the
|
|
600
|
+
* handler. Progress is reported through
|
|
601
|
+
* {@link AgentManagerCallbacks.onToolEvent | onToolEvent}.
|
|
602
|
+
*
|
|
603
|
+
* @param name - Name of the tool, which must match the one defined in the agent's
|
|
604
|
+
* configuration.
|
|
605
|
+
* @param handler - The function that runs when the agent calls the tool. It receives the
|
|
606
|
+
* arguments the LLM produced and must resolve to a JSON string of at most 15 KiB. See
|
|
607
|
+
* {@link ClientToolHandler}.
|
|
608
|
+
*/
|
|
609
|
+
registerClientTool(name: string, handler: ClientToolHandler): void;
|
|
610
|
+
/**
|
|
611
|
+
* Removes a previously registered client tool handler.
|
|
612
|
+
*
|
|
613
|
+
* After this the agent's calls to that tool fail rather than reaching your code.
|
|
614
|
+
*
|
|
615
|
+
* @param name - Name of the tool whose handler should be removed.
|
|
616
|
+
*/
|
|
617
|
+
unregisterClientTool(name: string): void;
|
|
618
|
+
}
|
|
619
|
+
|
|
620
|
+
/**
|
|
621
|
+
* Handlers the SDK calls as the connection, the video stream and the chat change state.
|
|
622
|
+
*
|
|
623
|
+
* Pass the object as {@link AgentManagerOptions.callbacks}. Only
|
|
624
|
+
* {@link AgentManagerCallbacks.onSrcObjectReady | onSrcObjectReady} is required — without it there
|
|
625
|
+
* is nothing to render the agent into. Every handler is called from the SDK's own event handling,
|
|
626
|
+
* so keep the work inside short.
|
|
627
|
+
*
|
|
628
|
+
* The handlers fall into four groups: the connection
|
|
629
|
+
* ({@link AgentManagerCallbacks.onConnectionStateChange | onConnectionStateChange},
|
|
630
|
+
* {@link AgentManagerCallbacks.onConnectivityStateChange | onConnectivityStateChange},
|
|
631
|
+
* {@link AgentManagerCallbacks.onError | onError}), the video stream
|
|
632
|
+
* ({@link AgentManagerCallbacks.onSrcObjectReady | onSrcObjectReady},
|
|
633
|
+
* {@link AgentManagerCallbacks.onStreamCreated | onStreamCreated},
|
|
634
|
+
* {@link AgentManagerCallbacks.onVideoStateChange | onVideoStateChange},
|
|
635
|
+
* {@link AgentManagerCallbacks.onAgentActivityStateChange | onAgentActivityStateChange},
|
|
636
|
+
* {@link AgentManagerCallbacks.onInterruptibleChange | onInterruptibleChange}), the chat
|
|
637
|
+
* ({@link AgentManagerCallbacks.onNewChat | onNewChat}, {@link AgentManagerCallbacks.onNewMessage | onNewMessage},
|
|
638
|
+
* {@link AgentManagerCallbacks.onModeChange | onModeChange}) and client tools
|
|
639
|
+
* ({@link AgentManagerCallbacks.onToolEvent | onToolEvent},
|
|
640
|
+
* {@link AgentManagerCallbacks.onRunningToolCallsChange | onRunningToolCallsChange}). A handler that
|
|
641
|
+
* lives outside the object can be typed with an indexed access such as
|
|
642
|
+
* `AgentManagerCallbacks['onNewMessage']`.
|
|
643
|
+
*
|
|
644
|
+
* @category Callbacks & Events
|
|
645
|
+
*/
|
|
646
|
+
export declare interface AgentManagerCallbacks {
|
|
647
|
+
/**
|
|
648
|
+
* Called every time the connection to the agent's stream changes state.
|
|
649
|
+
*
|
|
650
|
+
* Triggered by {@link AgentManager.connect | connect()},
|
|
651
|
+
* {@link AgentManager.reconnect | reconnect()} and
|
|
652
|
+
* {@link AgentManager.disconnect | disconnect()}.
|
|
653
|
+
* {@link ConnectionState.Connected | 'connected'} is the point at which
|
|
654
|
+
* {@link AgentManager.chat | chat()} and {@link AgentManager.speak | speak()} can be called.
|
|
655
|
+
*
|
|
656
|
+
* @param state - The state just reached: one of `new`, `connecting`, `connected`, `completed`,
|
|
657
|
+
* `disconnecting`, `disconnected`, `closed` or `fail`. See {@link ConnectionState}.
|
|
658
|
+
* @param reason - Why that state was reached. On `disconnected` it is a
|
|
659
|
+
* {@link StreamEndReason} value when the server ended the stream on purpose, which is how you
|
|
660
|
+
* tell a deliberate end from a dropped connection. Talks (V2) and Clips (V3) agents report
|
|
661
|
+
* only `ok`, `unknown_error`, `network_issue` and `inactivity`; `message_limit`, `time_limit`
|
|
662
|
+
* and `ended_by_agent` come from Expressive (V4) agents. A close reason the SDK does not
|
|
663
|
+
* recognise is forwarded as-is, so compare `reason` against the enum rather than parsing it;
|
|
664
|
+
* any other value is an opaque transport diagnostic.
|
|
665
|
+
* {@link AgentManager.reconnect | reconnect()} still works after a deliberate end; it starts a
|
|
666
|
+
* new stream rather than resuming the old one.
|
|
667
|
+
* @example
|
|
668
|
+
* ```ts
|
|
669
|
+
* onConnectionStateChange(state, reason) {
|
|
670
|
+
* console.log('onConnectionStateChange(): ', state, reason);
|
|
671
|
+
*
|
|
672
|
+
* if (state === 'connected') {
|
|
673
|
+
* console.log("I'm ready to go!");
|
|
674
|
+
* }
|
|
675
|
+
* }
|
|
676
|
+
* ```
|
|
677
|
+
*/
|
|
678
|
+
onConnectionStateChange?: (state: ConnectionState, reason?: string) => void;
|
|
679
|
+
/**
|
|
680
|
+
* Called when the streamed video starts and stops, so the video element can switch source.
|
|
681
|
+
*
|
|
682
|
+
* Triggered by {@link AgentManager.chat | chat()} and {@link AgentManager.speak | speak()}. On
|
|
683
|
+
* `STOP` point the element at the agent's idle video ({@link Agent.idle_video}); on `START` put
|
|
684
|
+
* the stream handed to {@link AgentManagerCallbacks.onSrcObjectReady | onSrcObjectReady} back
|
|
685
|
+
* on it.
|
|
686
|
+
*
|
|
687
|
+
* @param state - {@link StreamingState.Start | START} while the agent is speaking,
|
|
688
|
+
* {@link StreamingState.Stop | STOP} once it has finished.
|
|
689
|
+
* @example
|
|
690
|
+
* ```ts
|
|
691
|
+
* onVideoStateChange(state) {
|
|
692
|
+
* console.log('onVideoStateChange(): ', state);
|
|
693
|
+
*
|
|
694
|
+
* if (state === 'STOP') {
|
|
695
|
+
* videoElement.srcObject = undefined;
|
|
696
|
+
* videoElement.src = agentManager.agent.idle_video ?? '';
|
|
697
|
+
* } else {
|
|
698
|
+
* videoElement.src = '';
|
|
699
|
+
* videoElement.srcObject = srcObject;
|
|
700
|
+
* }
|
|
701
|
+
* }
|
|
702
|
+
* ```
|
|
703
|
+
*/
|
|
704
|
+
onVideoStateChange?: (state: StreamingState) => void;
|
|
705
|
+
/**
|
|
706
|
+
* Called with the media stream carrying the agent's video and audio. Required.
|
|
707
|
+
*
|
|
708
|
+
* This is the one callback the SDK cannot work without: assign the value to the `srcObject` of
|
|
709
|
+
* your `<video>` element, and keep a reference to it, because
|
|
710
|
+
* {@link AgentManagerCallbacks.onVideoStateChange | onVideoStateChange} has to put it back
|
|
711
|
+
* after the idle video has been shown. Triggered by
|
|
712
|
+
* {@link AgentManager.connect | connect()}, {@link AgentManager.reconnect | reconnect()} and
|
|
713
|
+
* {@link AgentManager.disconnect | disconnect()}.
|
|
714
|
+
*
|
|
715
|
+
* @param srcObject - The live media stream to render.
|
|
716
|
+
* @example
|
|
717
|
+
* ```ts
|
|
718
|
+
* let srcObject;
|
|
719
|
+
*
|
|
720
|
+
* const callbacks = {
|
|
721
|
+
* onSrcObjectReady(value) {
|
|
722
|
+
* videoElement.srcObject = value;
|
|
723
|
+
* srcObject = value;
|
|
724
|
+
* },
|
|
725
|
+
* };
|
|
726
|
+
* ```
|
|
727
|
+
*/
|
|
728
|
+
onSrcObjectReady: (srcObject: MediaStream) => void;
|
|
729
|
+
/**
|
|
730
|
+
* Called with the whole chat transcript every time a message is added or updated.
|
|
731
|
+
*
|
|
732
|
+
* Triggered by {@link AgentManager.chat | chat()}, by {@link AgentManager.speak | speak()} for
|
|
733
|
+
* text scripts, and as the agent's answer streams in. It also fires once while the manager is
|
|
734
|
+
* created, with whatever {@link AgentManagerOptions.initialMessages | initialMessages} were
|
|
735
|
+
* given, and again on every {@link AgentManager.connect | connect()} after the first — both
|
|
736
|
+
* with type `answer`. The array is a fresh copy on each call, oldest message first; every
|
|
737
|
+
* {@link Message} carries an `id`, a `role` of `user` or `assistant` (the agent), its `content`
|
|
738
|
+
* and a `createdAt` timestamp.
|
|
739
|
+
*
|
|
740
|
+
* @param messages - The chat so far.
|
|
741
|
+
* @param type - `partial` while the agent's answer is still streaming in, `answer` for the full
|
|
742
|
+
* answer spoken in the streamed video, `user` when the message just added is the user's own
|
|
743
|
+
* (a message passed to {@link AgentManager.chat | chat()}, or a transcribed utterance).
|
|
744
|
+
* @example
|
|
745
|
+
* ```ts
|
|
746
|
+
* onNewMessage(messages, type) {
|
|
747
|
+
* console.log(messages, type);
|
|
748
|
+
*
|
|
749
|
+
* if (type === 'answer') {
|
|
750
|
+
* console.log(messages[messages.length - 1].content);
|
|
751
|
+
* }
|
|
752
|
+
* }
|
|
753
|
+
* ```
|
|
754
|
+
*/
|
|
755
|
+
onNewMessage?: (messages: Message[], type: 'answer' | 'partial' | 'user') => void;
|
|
756
|
+
/**
|
|
757
|
+
* Called when a chat is created for this session.
|
|
758
|
+
*
|
|
759
|
+
* A chat is created while {@link AgentManager.connect | connect()} runs, and lazily on the
|
|
760
|
+
* first {@link AgentManager.chat | chat()} when none exists yet. Store the id if you want to
|
|
761
|
+
* correlate the conversation with your own records.
|
|
762
|
+
*
|
|
763
|
+
* @param chatId - Id of the chat that was just created.
|
|
764
|
+
*/
|
|
765
|
+
onNewChat?: (chatId: string) => void;
|
|
766
|
+
/**
|
|
767
|
+
* Called when the chat mode changes.
|
|
768
|
+
*
|
|
769
|
+
* Fires after {@link AgentManager.changeMode | changeMode()}, and when the server answers with
|
|
770
|
+
* a different mode than the one requested (for example {@link ChatMode.Maintenance}).
|
|
771
|
+
*
|
|
772
|
+
* @param mode - The {@link ChatMode} now in effect.
|
|
773
|
+
*/
|
|
774
|
+
onModeChange?: (mode: ChatMode) => void;
|
|
775
|
+
/**
|
|
776
|
+
* Called when the quality of the user's internet connection changes.
|
|
777
|
+
*
|
|
778
|
+
* The state is derived from jitter-buffer delay and freeze count on the video stream for
|
|
779
|
+
* Talks (V2) and Clips (V3) agents, and from LiveKit's reported connection quality for
|
|
780
|
+
* Expressive (V4) agents, so it reflects what the session is actually getting rather than the
|
|
781
|
+
* browser's online flag.
|
|
782
|
+
*
|
|
783
|
+
* @param state - {@link ConnectivityState.Strong | STRONG},
|
|
784
|
+
* {@link ConnectivityState.Weak | WEAK} or {@link ConnectivityState.Unknown | UNKNOWN}.
|
|
785
|
+
* @example
|
|
786
|
+
* ```ts
|
|
787
|
+
* onConnectivityStateChange(state) {
|
|
788
|
+
* console.log('onConnectivityStateChange(): ', state);
|
|
789
|
+
* }
|
|
790
|
+
* ```
|
|
791
|
+
*/
|
|
792
|
+
onConnectivityStateChange?: (state: ConnectivityState) => void;
|
|
793
|
+
/**
|
|
794
|
+
* Called when the SDK fails, so the application can surface the problem.
|
|
795
|
+
*
|
|
796
|
+
* Receives failures from the Agents API requests, the stream and the web socket — an
|
|
797
|
+
* {@link HttpError} when a request comes back non-2xx, a {@link NetworkError} when it never
|
|
798
|
+
* reaches the server, a {@link WsError} when the web socket itself fails, and also
|
|
799
|
+
* {@link ChatModeDowngraded} and {@link StreamError}. Two errors do not arrive here:
|
|
800
|
+
* {@link ValidationError} and {@link ChatCreationFailed} are thrown to whoever called the
|
|
801
|
+
* method ({@link AgentManager.chat | chat()}, {@link AgentManager.speak | speak()} and the
|
|
802
|
+
* rating methods), so they surface as a rejected promise rather than through this callback.
|
|
803
|
+
*
|
|
804
|
+
* @param error - The error that occurred.
|
|
805
|
+
* @param errorData - Extra context about the failure, such as the request URL and options.
|
|
806
|
+
* @example
|
|
807
|
+
* ```ts
|
|
808
|
+
* onError(error, errorData) {
|
|
809
|
+
* console.log('Error:', error, 'Error Data', errorData);
|
|
810
|
+
* }
|
|
811
|
+
* ```
|
|
812
|
+
*/
|
|
813
|
+
onError?: (error: Error, errorData?: Record<string, unknown>) => void;
|
|
814
|
+
/**
|
|
815
|
+
* Called when the agent moves between idle, loading, talking and running a tool.
|
|
816
|
+
*
|
|
817
|
+
* Use it to drive a typing indicator or to disable input while the agent is busy. The full set
|
|
818
|
+
* of states is reported for Expressive (V4) agents only. Talks (V2) and Clips (V3) agents
|
|
819
|
+
* report just {@link AgentActivityState.Talking | Talking} and
|
|
820
|
+
* {@link AgentActivityState.Idle | Idle}, and only on fluent streams, plus a final
|
|
821
|
+
* {@link AgentActivityState.Idle | Idle} when the connection closes.
|
|
822
|
+
*
|
|
823
|
+
* @param state - The {@link AgentActivityState} the agent has moved to.
|
|
824
|
+
*/
|
|
825
|
+
onAgentActivityStateChange?: (state: AgentActivityState) => void;
|
|
826
|
+
/**
|
|
827
|
+
* Called once per session when the stream has been created on the server.
|
|
828
|
+
*
|
|
829
|
+
* Fires before the connection reaches {@link ConnectionState.Connected | 'connected'}. The ids
|
|
830
|
+
* are useful when correlating a session with D-ID support or with your own logs.
|
|
831
|
+
*
|
|
832
|
+
* @param stream - The new stream's `streamId`, `sessionId` and `agentId`.
|
|
833
|
+
*/
|
|
834
|
+
onStreamCreated?: (stream: StreamCreatedInfo) => void;
|
|
835
|
+
/**
|
|
836
|
+
* Called when the agent starts, finishes or fails a tool call.
|
|
837
|
+
*
|
|
838
|
+
* Expressive (V4) agents only. The handler takes two arguments and returns nothing: `event`,
|
|
839
|
+
* one of {@link ToolCallEvent.Started}, {@link ToolCallEvent.Done} or
|
|
840
|
+
* {@link ToolCallEvent.Error}; and `data`, the payload that event narrows to —
|
|
841
|
+
* {@link ToolCallStartedPayload}, {@link ToolCallDonePayload} or
|
|
842
|
+
* {@link ToolCallErrorPayload} respectively. The overloads that do the narrowing are on
|
|
843
|
+
* {@link ToolEventCallback}, with an example handler.
|
|
844
|
+
*/
|
|
845
|
+
onToolEvent?: ToolEventCallback;
|
|
846
|
+
/**
|
|
847
|
+
* Called when the agent becomes interruptible, or stops being interruptible.
|
|
848
|
+
*
|
|
849
|
+
* Expressive (V4) agents only, and about whether interrupting is allowed *right now*: it goes
|
|
850
|
+
* `false` while a `blocking` client tool call is outstanding, because the agent is suspended
|
|
851
|
+
* waiting for it, and back to `true` when the call finishes. That is a different question from
|
|
852
|
+
* {@link AgentManager.isInterruptAvailable | isInterruptAvailable()}, which says whether
|
|
853
|
+
* the session supports interrupting at all. Use this one to enable or disable an interrupt
|
|
854
|
+
* button. Talks (V2) and Clips (V3) agents never report a change.
|
|
855
|
+
*/
|
|
856
|
+
onInterruptibleChange?: (
|
|
857
|
+
/** `true` while there is something to interrupt, `false` while there is not. */
|
|
858
|
+
interruptible: boolean) => void;
|
|
859
|
+
/**
|
|
860
|
+
* Called whenever the set of tool calls running in the session changes.
|
|
861
|
+
*
|
|
862
|
+
* Expressive (V4) agents only. Fires with an empty array on disconnect, so a spinner driven by
|
|
863
|
+
* this callback always clears. Each entry is a {@link RunningToolCall}.
|
|
864
|
+
*/
|
|
865
|
+
onRunningToolCallsChange?: (
|
|
866
|
+
/** Every tool call running right now, empty when none is. */
|
|
867
|
+
calls: readonly RunningToolCall[]) => void;
|
|
868
|
+
}
|
|
869
|
+
|
|
870
|
+
/**
|
|
871
|
+
* Everything {@link createAgentManager} needs to reach an agent and report back to the application.
|
|
872
|
+
*
|
|
873
|
+
* {@link AgentManagerOptions.auth | auth} and {@link AgentManagerOptions.callbacks | callbacks} are
|
|
874
|
+
* required; the rest have defaults that suit a browser application talking to D-ID production.
|
|
875
|
+
*
|
|
876
|
+
* {@link createAgentManager} reads the object once, at creation, and never writes to it: the
|
|
877
|
+
* manager works from its own copy, so one options object can create two managers, and a property
|
|
878
|
+
* you read back afterwards is still the one you set. The same means a handler assigned to
|
|
879
|
+
* {@link AgentManagerOptions.callbacks | callbacks} after the manager exists is not picked up —
|
|
880
|
+
* give the callback a stable identity and dispatch inside it instead.
|
|
881
|
+
*
|
|
882
|
+
* @category Agent Manager
|
|
883
|
+
*/
|
|
884
|
+
export declare interface AgentManagerOptions {
|
|
885
|
+
/**
|
|
886
|
+
* Credentials used for every request the SDK makes on the application's behalf.
|
|
887
|
+
*
|
|
888
|
+
* Use `{ type: 'key', clientKey }` in a browser: a client key is scoped to one agent and to
|
|
889
|
+
* the domains you allowed, so it is the only shape that is safe to ship in a page. See
|
|
890
|
+
* {@link Auth}.
|
|
891
|
+
*/
|
|
892
|
+
auth: Auth;
|
|
893
|
+
/**
|
|
894
|
+
* Handlers the SDK calls as the connection, the video stream and the chat change state.
|
|
895
|
+
*
|
|
896
|
+
* See {@link AgentManagerCallbacks}.
|
|
897
|
+
* {@link AgentManagerCallbacks.onSrcObjectReady | onSrcObjectReady} is mandatory — it is what
|
|
898
|
+
* connects the streamed media to your video element; the rest are optional. The handlers are
|
|
899
|
+
* captured at creation, so replacing one on this object later has no effect.
|
|
900
|
+
*/
|
|
901
|
+
callbacks: AgentManagerCallbacks;
|
|
902
|
+
/**
|
|
903
|
+
* How the agent answers: with a streamed video, as text only, or not at all.
|
|
904
|
+
*
|
|
905
|
+
* See {@link ChatMode}. It can be changed later with
|
|
906
|
+
* {@link AgentManager.changeMode | changeMode()}. {@link ChatMode.Off} and
|
|
907
|
+
* {@link ChatMode.DirectPlayback} are supported on Talks (V2) and Clips (V3) agents only.
|
|
908
|
+
*
|
|
909
|
+
* @default ChatMode.Functional
|
|
910
|
+
* @throws {@link ValidationError} From {@link createAgentManager}, when the agent is Expressive
|
|
911
|
+
* (V4) and the mode is {@link ChatMode.Off} or {@link ChatMode.DirectPlayback}.
|
|
912
|
+
*/
|
|
913
|
+
mode?: ChatMode;
|
|
914
|
+
/**
|
|
915
|
+
* Base URL of the D-ID Agents API.
|
|
916
|
+
*
|
|
917
|
+
* Advanced. Points the SDK at another D-ID environment. Production applications do not set
|
|
918
|
+
* this. Every REST call the manager makes — agent lookup, chat, ratings, stream creation — is
|
|
919
|
+
* addressed against it.
|
|
920
|
+
*
|
|
921
|
+
* @defaultValue D-ID's production Agents API.
|
|
922
|
+
*/
|
|
923
|
+
baseURL?: string;
|
|
924
|
+
/**
|
|
925
|
+
* URL of the D-ID notifications web socket.
|
|
926
|
+
*
|
|
927
|
+
* Advanced. Points the SDK at another D-ID environment. Production applications do not set
|
|
928
|
+
* this. Only opened in the chat modes that stream the agent's answer, so it has no effect in
|
|
929
|
+
* {@link ChatMode.DirectPlayback | DirectPlayback} or {@link ChatMode.Off | Off}.
|
|
930
|
+
*
|
|
931
|
+
* @defaultValue D-ID's production notifications web socket.
|
|
932
|
+
*/
|
|
933
|
+
wsURL?: string;
|
|
934
|
+
/**
|
|
935
|
+
* Whether the SDK logs its streaming lifecycle to the browser console.
|
|
936
|
+
*
|
|
937
|
+
* It is also switched on for everyone when the agent's `ui_debug_mode` advanced setting is
|
|
938
|
+
* enabled, regardless of the value passed here.
|
|
939
|
+
*
|
|
940
|
+
* @default false
|
|
941
|
+
*/
|
|
942
|
+
debug?: boolean;
|
|
943
|
+
/**
|
|
944
|
+
* Whether the stream is created in verbose mode, which logs more server-side detail.
|
|
945
|
+
*
|
|
946
|
+
* Independent of {@link AgentManagerOptions.debug | debug}: this one is sent to the streaming
|
|
947
|
+
* service rather than controlling console output. Expressive (V4) agents only.
|
|
948
|
+
*
|
|
949
|
+
* @default false
|
|
950
|
+
*/
|
|
951
|
+
verbose?: boolean;
|
|
952
|
+
/**
|
|
953
|
+
* Whether the SDK reports usage analytics. Set it to `false` to send nothing.
|
|
954
|
+
*
|
|
955
|
+
* It is on unless you turn it off. The SDK posts an event to Mixpanel for each step of a
|
|
956
|
+
* session — connect, reconnect and disconnect, every message sent and answered, every
|
|
957
|
+
* {@link AgentManager.speak | speak()}, mode change, rating, feedback, tool call, video start
|
|
958
|
+
* and stop, interrupt and error. Each event carries the agent id, the visitor id derived from
|
|
959
|
+
* {@link AgentManagerOptions.externalId | externalId}, the page URL, the screen size, the user
|
|
960
|
+
* agent, the SDK version and whatever
|
|
961
|
+
* {@link AgentManagerOptions.mixpanelAdditionalProperties | mixpanelAdditionalProperties}
|
|
962
|
+
* added. Conversation text is included too: the agent's answer on `agent-message-received`,
|
|
963
|
+
* and the script on `agent-speak`.
|
|
964
|
+
*
|
|
965
|
+
* The events go to D-ID's own Mixpanel project unless
|
|
966
|
+
* {@link AgentManagerOptions.mixpanelKey | mixpanelKey} points them at yours. Events that fail
|
|
967
|
+
* to post are buffered in memory (at most 50) and retried when the tab comes back online or
|
|
968
|
+
* becomes visible.
|
|
969
|
+
*
|
|
970
|
+
* @default true
|
|
971
|
+
*/
|
|
972
|
+
enableAnalytics?: boolean;
|
|
973
|
+
/**
|
|
974
|
+
* Mixpanel project token the SDK's analytics events are sent to.
|
|
975
|
+
*
|
|
976
|
+
* Defaults to D-ID's own project. Set it to send the SDK's events to your Mixpanel project
|
|
977
|
+
* instead.
|
|
978
|
+
*/
|
|
979
|
+
mixpanelKey?: string;
|
|
980
|
+
/**
|
|
981
|
+
* Extra properties merged into every analytics event the SDK sends.
|
|
982
|
+
*
|
|
983
|
+
* A flat object — use it to tag events with your own identifiers. More properties can be added
|
|
984
|
+
* later with `enrichAnalytics()`.
|
|
985
|
+
*/
|
|
986
|
+
mixpanelAdditionalProperties?: Record<string, unknown>;
|
|
987
|
+
/**
|
|
988
|
+
* Your own identifier for the end user.
|
|
989
|
+
*
|
|
990
|
+
* It is stored in `localStorage`, sent as part of the `type: 'key'` authorization header and
|
|
991
|
+
* attached to analytics, so the same visitor is recognised across page loads. When it is
|
|
992
|
+
* omitted the SDK generates an id and reuses it.
|
|
993
|
+
*/
|
|
994
|
+
externalId?: string;
|
|
995
|
+
/**
|
|
996
|
+
* Transport settings applied when the stream is created.
|
|
997
|
+
*
|
|
998
|
+
* See {@link StreamOptions}. Talks (V2) and Clips (V3) agents only. Expressive (V4) agents
|
|
999
|
+
* manage transport settings automatically.
|
|
1000
|
+
*/
|
|
1001
|
+
streamOptions?: StreamOptions;
|
|
1002
|
+
/**
|
|
1003
|
+
* Messages the chat starts with, for example a transcript restored from your own storage.
|
|
1004
|
+
*
|
|
1005
|
+
* They are delivered to {@link AgentManagerCallbacks.onNewMessage | onNewMessage} so the UI can
|
|
1006
|
+
* render them. On Talks (V2) and Clips (V3) agents they are also sent as context with the next
|
|
1007
|
+
* {@link AgentManager.chat | chat()} request. See {@link Message}.
|
|
1008
|
+
*
|
|
1009
|
+
* {@link Message.parts | parts} is a required field, but it does not have to be filled: pass an
|
|
1010
|
+
* empty array and the SDK builds the parts from `content` with {@link parseMessageParts}, so a
|
|
1011
|
+
* transcript restored from `content` alone still renders. A non-empty array is kept as given.
|
|
1012
|
+
*
|
|
1013
|
+
* @example Restoring rows that carry only the message text
|
|
1014
|
+
* ```ts
|
|
1015
|
+
* const initialMessages = stored.map(({ id, role, content }) => ({ id, role, content, parts: [] }));
|
|
1016
|
+
* ```
|
|
1017
|
+
*/
|
|
1018
|
+
initialMessages?: Message[];
|
|
1019
|
+
/**
|
|
1020
|
+
* Whether the server keeps the chat so it can be resumed in a later session.
|
|
1021
|
+
*
|
|
1022
|
+
* Without persistence a chat lives only as long as the connection.
|
|
1023
|
+
* {@link AgentManager.reconnect | reconnect()} normally continues the same chat either way —
|
|
1024
|
+
* see the caveat there for Expressive (V4) agents.
|
|
1025
|
+
*
|
|
1026
|
+
* Off unless you set it, on every agent type.
|
|
1027
|
+
*
|
|
1028
|
+
* @default false
|
|
1029
|
+
*/
|
|
1030
|
+
persistentChat?: boolean;
|
|
1031
|
+
}
|
|
1032
|
+
|
|
1033
|
+
/**
|
|
1034
|
+
* Amazon provider details: the provider type and the requested voice id.
|
|
1035
|
+
*
|
|
1036
|
+
* Pass it as {@link TextStreamScript.provider} to have Amazon synthesise the text. It is the one
|
|
1037
|
+
* provider with no `voice_config`: the voice is selected by `voice_id` alone.
|
|
1038
|
+
*
|
|
1039
|
+
* @category Voice
|
|
1040
|
+
*/
|
|
1041
|
+
export declare interface AmazonTtsProvider {
|
|
1042
|
+
/** Selects Amazon. Always {@link Providers.Amazon} (`'amazon'`). */
|
|
1043
|
+
type: Providers.Amazon;
|
|
1044
|
+
/**
|
|
1045
|
+
* Id of the voice to speak with, from D-ID's list of Amazon voices.
|
|
1046
|
+
*
|
|
1047
|
+
* @example "Joanna"
|
|
1048
|
+
* @see [Amazon voices](https://docs.d-id.com/docs/tts-amazon)
|
|
1049
|
+
*/
|
|
1050
|
+
voice_id: string;
|
|
1051
|
+
}
|
|
1052
|
+
|
|
1053
|
+
/**
|
|
1054
|
+
* A script that makes the agent lip-sync an audio file you host, with no text-to-speech involved.
|
|
1055
|
+
*
|
|
1056
|
+
* The other payload {@link AgentManager.speak | speak()} accepts. Use it when the audio already
|
|
1057
|
+
* exists — a recording, or speech you synthesised yourself — instead of having a provider generate
|
|
1058
|
+
* it from text.
|
|
1059
|
+
*
|
|
1060
|
+
* @example Audio file
|
|
1061
|
+
* ```ts
|
|
1062
|
+
* const speak = await agentManager.speak({
|
|
1063
|
+
* type: 'audio',
|
|
1064
|
+
* audio_url: 'https://www.yourwebsite.com/audio.mp3',
|
|
1065
|
+
* });
|
|
1066
|
+
* ```
|
|
1067
|
+
* @category Speak & Scripts
|
|
1068
|
+
*/
|
|
1069
|
+
export declare interface AudioStreamScript {
|
|
1070
|
+
/**
|
|
1071
|
+
* Which kind of script this is. Always `audio` for this variant; a
|
|
1072
|
+
* {@link TextStreamScript} carries `text` instead.
|
|
1073
|
+
*/
|
|
1074
|
+
type: 'audio';
|
|
1075
|
+
/**
|
|
1076
|
+
* URL of the audio file the agent lip-syncs to.
|
|
1077
|
+
*
|
|
1078
|
+
* The URL has to be publicly reachable, since the file is fetched server-side rather than
|
|
1079
|
+
* uploaded from the browser. The file may be at most 15 MB.
|
|
1080
|
+
* @example "https://www.yourwebsite.com/audio.mp3"
|
|
1081
|
+
*/
|
|
1082
|
+
audio_url: string;
|
|
1083
|
+
}
|
|
1084
|
+
|
|
1085
|
+
/**
|
|
1086
|
+
* The credentials {@link createAgentManager} accepts, as
|
|
1087
|
+
* {@link AgentManagerOptions.auth | options.auth}.
|
|
1088
|
+
*
|
|
1089
|
+
* Pick {@link ClientKeyAuth} (`{ type: 'key', clientKey }`) for anything that runs in a browser: it
|
|
1090
|
+
* is the only variant scoped to one agent and to the domains you allowed. {@link BearerToken} and
|
|
1091
|
+
* {@link BasicAuth} carry account-wide credentials and belong in a trusted environment. Whichever
|
|
1092
|
+
* you pass is used for every request the SDK makes — the REST calls, the video stream and the
|
|
1093
|
+
* notifications web socket.
|
|
1094
|
+
*
|
|
1095
|
+
* @category Authentication
|
|
1096
|
+
*/
|
|
1097
|
+
export declare type Auth = BearerToken | BasicAuth | ClientKeyAuth;
|
|
1098
|
+
|
|
1099
|
+
/**
|
|
1100
|
+
* The kind of avatar an agent is built on, which is what decides how its video is produced.
|
|
1101
|
+
*
|
|
1102
|
+
* It is the `type` of the agent's avatar ({@link Agent.avatar}) and so determines which SDK
|
|
1103
|
+
* features a session has: {@link AvatarType.Talk | talk} is Talks (V2),
|
|
1104
|
+
* {@link AvatarType.Clip | clip} is Clips (V3), and {@link AvatarType.Expressive | expressive} is
|
|
1105
|
+
* Expressive (V4).
|
|
1106
|
+
*
|
|
1107
|
+
* @category Agent Manager
|
|
1108
|
+
*/
|
|
1109
|
+
export declare enum AvatarType {
|
|
1110
|
+
/** A Clips (V3) agent: a pre-recorded presenter, driven by the D-ID Clips pipeline. */
|
|
1111
|
+
Clip = "clip",
|
|
1112
|
+
/** A Talks (V2) agent: a still photo animated to speak. */
|
|
1113
|
+
Talk = "talk",
|
|
1114
|
+
/**
|
|
1115
|
+
* An Expressive (V4) avatar: a real-time avatar that also supports microphone and camera
|
|
1116
|
+
* publishing, client tools and interruption.
|
|
1117
|
+
*/
|
|
1118
|
+
Expressive = "expressive"
|
|
1119
|
+
}
|
|
1120
|
+
|
|
1121
|
+
/**
|
|
1122
|
+
* Azure OpenAI provider details: the provider type, the requested voice id and an optional
|
|
1123
|
+
* `voice_config` for style, rate and pitch.
|
|
1124
|
+
*
|
|
1125
|
+
* The same shape as {@link MicrosoftTtsProvider} — a `voice_id` and an optional
|
|
1126
|
+
* {@link VoiceConfigMicrosoft} as `voice_config` — with the `type` naming Azure OpenAI instead.
|
|
1127
|
+
*
|
|
1128
|
+
* @category Voice
|
|
1129
|
+
*/
|
|
1130
|
+
export declare interface AzureOpenAiTtsProvider extends Omit<MicrosoftTtsProvider, 'type'> {
|
|
1131
|
+
/** Selects Azure OpenAI. Always {@link Providers.AzureOpenAi} (`'azure-openai'`). */
|
|
1132
|
+
type: Providers.AzureOpenAi;
|
|
1133
|
+
}
|
|
1134
|
+
|
|
1135
|
+
/**
|
|
1136
|
+
* Base class of every error the SDK itself raises.
|
|
1137
|
+
*
|
|
1138
|
+
* Catching `BaseError` catches everything the SDK classified; branch on
|
|
1139
|
+
* {@link BaseError.kind | kind}, which is stable, rather than on the message, which is not.
|
|
1140
|
+
* Errors reach the application two ways, and each subclass says which applies: some are thrown to
|
|
1141
|
+
* the caller of the method, so they reject the promise it returned; others are handed to
|
|
1142
|
+
* {@link AgentManagerCallbacks.onError | onError} because no call of yours was in flight when they
|
|
1143
|
+
* happened. {@link HttpError} and {@link NetworkError} do both — the request that failed rejects,
|
|
1144
|
+
* and the callback is notified as well.
|
|
1145
|
+
*
|
|
1146
|
+
* Not everything that reaches the application is one of these. The streaming transport is the main
|
|
1147
|
+
* exception: a failure LiveKit raises inside a media call on an Expressive (V4) agent — no
|
|
1148
|
+
* microphone publication to replace, a room that is not connected — rejects with a plain `Error`,
|
|
1149
|
+
* and so do a few low-level guards inside {@link AgentManager.connect | connect()}, such as a
|
|
1150
|
+
* `livekit-client` package that is not installed, a transport the SDK does not recognise, or a
|
|
1151
|
+
* stream the server created without a session id. An unknown
|
|
1152
|
+
* {@link AgentManagerOptions.auth | auth} type is a plain `Error` too. So keep the
|
|
1153
|
+
* `else throw error` branch in a handler built on {@link isDIDError}.
|
|
1154
|
+
*
|
|
1155
|
+
* Subclasses: {@link HttpError}, {@link NetworkError}, {@link WsError}, {@link StreamError},
|
|
1156
|
+
* {@link ValidationError}, {@link ChatCreationFailed} and {@link ChatModeDowngraded}. When the
|
|
1157
|
+
* caught value is typed `unknown`, {@link isDIDError} narrows it to {@link DIDError}, the union of
|
|
1158
|
+
* those seven, so a `switch` on {@link BaseError.kind | kind} then narrows it to one class.
|
|
1159
|
+
*
|
|
1160
|
+
* @category Errors
|
|
1161
|
+
*/
|
|
1162
|
+
export declare class BaseError extends Error {
|
|
1163
|
+
/**
|
|
1164
|
+
* Stable machine-readable code for this failure, and the value to branch on.
|
|
1165
|
+
*
|
|
1166
|
+
* Every subclass redeclares it as the literal it always carries, so a `switch` on `kind`
|
|
1167
|
+
* narrows a value {@link isDIDError} has recognized to that class: `'HttpError'` on
|
|
1168
|
+
* {@link HttpError}, `'NetworkError'` on {@link NetworkError}, `'WSError'` on
|
|
1169
|
+
* {@link WsError}, `'StreamError'` on {@link StreamError}, `'ValidationError'` on
|
|
1170
|
+
* {@link ValidationError}, `'ChatCreationFailed'` on {@link ChatCreationFailed} and
|
|
1171
|
+
* `'ChatModeDowngraded'` on {@link ChatModeDowngraded}. It stays a plain `string` here on
|
|
1172
|
+
* the base class: on a `BaseError` built directly it is whatever the caller passed, and
|
|
1173
|
+
* `'Error'` when nothing was. The Agents API's own classification of a failed request is
|
|
1174
|
+
* {@link HttpError.code | HttpError.code}, not this.
|
|
1175
|
+
*/
|
|
1176
|
+
readonly kind: string;
|
|
1177
|
+
/**
|
|
1178
|
+
* The error or value that caused this one, when the SDK was wrapping something else.
|
|
1179
|
+
*
|
|
1180
|
+
* For a {@link NetworkError} this is the browser's own `fetch` rejection. It is not
|
|
1181
|
+
* serialized as-is: {@link BaseError.toJson | toJson()} takes only an `Error` cause's
|
|
1182
|
+
* message.
|
|
1183
|
+
*/
|
|
1184
|
+
readonly originalError?: unknown | undefined;
|
|
1185
|
+
/**
|
|
1186
|
+
* Creates an SDK error. The SDK builds these itself; applications normally only catch them.
|
|
1187
|
+
*
|
|
1188
|
+
* @param message - Human-readable description of the failure, used as the `Error` message.
|
|
1189
|
+
* @param kind - Stable machine-readable code for the failure.
|
|
1190
|
+
* @param originalError - The underlying error or value this one wraps, when there is one.
|
|
1191
|
+
*/
|
|
1192
|
+
constructor(message: string,
|
|
1193
|
+
/**
|
|
1194
|
+
* Stable machine-readable code for this failure, and the value to branch on.
|
|
1195
|
+
*
|
|
1196
|
+
* Every subclass redeclares it as the literal it always carries, so a `switch` on `kind`
|
|
1197
|
+
* narrows a value {@link isDIDError} has recognized to that class: `'HttpError'` on
|
|
1198
|
+
* {@link HttpError}, `'NetworkError'` on {@link NetworkError}, `'WSError'` on
|
|
1199
|
+
* {@link WsError}, `'StreamError'` on {@link StreamError}, `'ValidationError'` on
|
|
1200
|
+
* {@link ValidationError}, `'ChatCreationFailed'` on {@link ChatCreationFailed} and
|
|
1201
|
+
* `'ChatModeDowngraded'` on {@link ChatModeDowngraded}. It stays a plain `string` here on
|
|
1202
|
+
* the base class: on a `BaseError` built directly it is whatever the caller passed, and
|
|
1203
|
+
* `'Error'` when nothing was. The Agents API's own classification of a failed request is
|
|
1204
|
+
* {@link HttpError.code | HttpError.code}, not this.
|
|
1205
|
+
*/
|
|
1206
|
+
kind?: string,
|
|
1207
|
+
/**
|
|
1208
|
+
* The error or value that caused this one, when the SDK was wrapping something else.
|
|
1209
|
+
*
|
|
1210
|
+
* For a {@link NetworkError} this is the browser's own `fetch` rejection. It is not
|
|
1211
|
+
* serialized as-is: {@link BaseError.toJson | toJson()} takes only an `Error` cause's
|
|
1212
|
+
* message.
|
|
1213
|
+
*/
|
|
1214
|
+
originalError?: unknown | undefined);
|
|
1215
|
+
/**
|
|
1216
|
+
* Serializes the error into the JSON-safe {@link ErrorJson} payload for logging or reporting.
|
|
1217
|
+
*
|
|
1218
|
+
* The payload always carries {@link BaseError.kind | kind} and the message. The cause's message
|
|
1219
|
+
* is added only when {@link BaseError.originalError | originalError} is an `Error` and says
|
|
1220
|
+
* something the message does not, truncated to 256 characters; any other cause is dropped.
|
|
1221
|
+
* Subclasses extend the payload with the fields listed on their own pages.
|
|
1222
|
+
*
|
|
1223
|
+
* @returns The error as plain, JSON-serializable data.
|
|
1224
|
+
* @example An HttpError from a request that was refused
|
|
1225
|
+
* ```json
|
|
1226
|
+
* {
|
|
1227
|
+
* "kind": "InsufficientCreditsError",
|
|
1228
|
+
* "message": "Account has insufficient credits",
|
|
1229
|
+
* "httpStatus": 402,
|
|
1230
|
+
* "endpoint": "/agt_x/chat/cht_y",
|
|
1231
|
+
* "method": "POST"
|
|
1232
|
+
* }
|
|
1233
|
+
* ```
|
|
1234
|
+
* @example A NetworkError from a request that never left the browser
|
|
1235
|
+
* ```json
|
|
1236
|
+
* {
|
|
1237
|
+
* "kind": "NetworkError",
|
|
1238
|
+
* "message": "Network request failed",
|
|
1239
|
+
* "cause": "Failed to fetch",
|
|
1240
|
+
* "endpoint": "/agt_x/chat/cht_y",
|
|
1241
|
+
* "method": "POST",
|
|
1242
|
+
* "durationMs": 12,
|
|
1243
|
+
* "online": false,
|
|
1244
|
+
* "visibility": "visible"
|
|
1245
|
+
* }
|
|
1246
|
+
* ```
|
|
1247
|
+
*/
|
|
1248
|
+
toJson(): ErrorJson;
|
|
1249
|
+
}
|
|
1250
|
+
|
|
1251
|
+
/**
|
|
1252
|
+
* HTTP basic credentials for the Agents API, given either pre-encoded or as a username and
|
|
1253
|
+
* password.
|
|
1254
|
+
*
|
|
1255
|
+
* Like {@link BearerToken} these are account-wide and belong in a trusted environment, never in a
|
|
1256
|
+
* client bundle. The SDK sends them as `Authorization: Basic <credentials>~<connectionId>`, where
|
|
1257
|
+
* the `token` form is used as the credentials as-is and the `username`/`password` form is
|
|
1258
|
+
* base64-encoded as `username:password` first; the trailing per-connection id is stripped by the
|
|
1259
|
+
* D-ID authorizer before the credentials are validated.
|
|
1260
|
+
*
|
|
1261
|
+
* @category Authentication
|
|
1262
|
+
*/
|
|
1263
|
+
export declare type BasicAuth = {
|
|
1264
|
+
/**
|
|
1265
|
+
* Discriminator selecting this variant of {@link Auth}.
|
|
1266
|
+
*/
|
|
1267
|
+
type: 'basic';
|
|
1268
|
+
/**
|
|
1269
|
+
* Credentials that are already base64-encoded, as `username:password`. Used as they
|
|
1270
|
+
* are, so they are not encoded a second time.
|
|
1271
|
+
*/
|
|
1272
|
+
token: string;
|
|
1273
|
+
} | {
|
|
1274
|
+
/**
|
|
1275
|
+
* Discriminator selecting this variant of {@link Auth}.
|
|
1276
|
+
*/
|
|
1277
|
+
type: 'basic';
|
|
1278
|
+
/**
|
|
1279
|
+
* The account's user name; base64-encoded with the password before it is sent.
|
|
1280
|
+
*/
|
|
1281
|
+
username: string;
|
|
1282
|
+
/**
|
|
1283
|
+
* The account's password; base64-encoded with the user name before it is sent.
|
|
1284
|
+
*/
|
|
1285
|
+
password: string;
|
|
1286
|
+
};
|
|
1287
|
+
|
|
1288
|
+
/**
|
|
1289
|
+
* A bearer token for the Agents API.
|
|
1290
|
+
*
|
|
1291
|
+
* Account-wide, so it is meant for a trusted environment — a server, a build step, an internal
|
|
1292
|
+
* tool. Anyone who reads the token can spend the account's credits, so do not ship one in a page
|
|
1293
|
+
* you serve to users; use {@link ClientKeyAuth} there. The SDK sends it as
|
|
1294
|
+
* `Authorization: Bearer <token>~<connectionId>`, appending a per-connection id the D-ID
|
|
1295
|
+
* authorizer strips before it validates the token.
|
|
1296
|
+
*
|
|
1297
|
+
* @category Authentication
|
|
1298
|
+
*/
|
|
1299
|
+
export declare interface BearerToken {
|
|
1300
|
+
/**
|
|
1301
|
+
* Discriminator selecting this variant of {@link Auth}.
|
|
1302
|
+
*/
|
|
1303
|
+
type: 'bearer';
|
|
1304
|
+
/**
|
|
1305
|
+
* The bearer token itself, sent in the `Authorization` header.
|
|
1306
|
+
*/
|
|
1307
|
+
token: string;
|
|
1308
|
+
}
|
|
1309
|
+
|
|
1310
|
+
/**
|
|
1311
|
+
* A chat could not be created for the session.
|
|
1312
|
+
*
|
|
1313
|
+
* {@link AgentManager.chat | chat()} creates a chat on the first message when the session does not
|
|
1314
|
+
* have one yet; this error is thrown when that call comes back without one, so it rejects the
|
|
1315
|
+
* promise `chat()` returned and the user's message is not sent. It is not delivered to
|
|
1316
|
+
* {@link AgentManagerCallbacks.onError | onError} — catch it around `chat()`. A failing request
|
|
1317
|
+
* surfaces as {@link HttpError} or {@link NetworkError} instead; this error is the case where the
|
|
1318
|
+
* request succeeded but produced no chat.
|
|
1319
|
+
*
|
|
1320
|
+
* The message names the {@link ChatMode} in effect and whether a persistent chat was asked for, for
|
|
1321
|
+
* example `Failed to create persistent chat, mode: Functional`.
|
|
1322
|
+
* {@link BaseError.kind | kind} is `'ChatCreationFailed'`.
|
|
1323
|
+
*
|
|
1324
|
+
* @category Errors
|
|
1325
|
+
*/
|
|
1326
|
+
export declare class ChatCreationFailed extends BaseError {
|
|
1327
|
+
/**
|
|
1328
|
+
* Always `'ChatCreationFailed'`. Branch on it to tell this failure from the other SDK errors.
|
|
1329
|
+
*/
|
|
1330
|
+
readonly kind: 'ChatCreationFailed';
|
|
1331
|
+
}
|
|
1332
|
+
|
|
1333
|
+
/**
|
|
1334
|
+
* How the agent answers: with a streamed video, as text only, or not at all.
|
|
1335
|
+
*
|
|
1336
|
+
* Chosen with {@link AgentManagerOptions.mode | mode} and changed later with
|
|
1337
|
+
* {@link AgentManager.changeMode | changeMode()}, which reports the new value through
|
|
1338
|
+
* {@link AgentManagerCallbacks.onModeChange | onModeChange}. Switching to anything other than
|
|
1339
|
+
* {@link ChatMode.Functional} disconnects the stream, and switching *into*
|
|
1340
|
+
* {@link ChatMode.Functional} disconnects it too when the open session cannot carry a conversation
|
|
1341
|
+
* — call {@link AgentManager.connect | connect()} again after either.
|
|
1342
|
+
*
|
|
1343
|
+
* What the mode decides is narrower than it looks: whether a chat is created for the session, and
|
|
1344
|
+
* whether {@link AgentManager.connect | connect()} opens the notifications web socket — which is
|
|
1345
|
+
* exactly why a session built for one mode may not serve another. The video stream is established
|
|
1346
|
+
* in every mode, and every guard reads the mode the session is in at the time of the call, not the
|
|
1347
|
+
* one {@link createAgentManager} was given.
|
|
1348
|
+
*
|
|
1349
|
+
* The server can also answer with a different mode than the one asked for when the chat is created;
|
|
1350
|
+
* the SDK then adopts it and reports a {@link ChatModeDowngraded} error through
|
|
1351
|
+
* {@link AgentManagerCallbacks.onError | onError}.
|
|
1352
|
+
*
|
|
1353
|
+
* @category Chat
|
|
1354
|
+
*/
|
|
1355
|
+
export declare enum ChatMode {
|
|
1356
|
+
/**
|
|
1357
|
+
* Chat and video: the agent answers questions with its own LLM, in a streamed video.
|
|
1358
|
+
*
|
|
1359
|
+
* The default when {@link AgentManagerOptions.mode | mode} is omitted, and the only mode a
|
|
1360
|
+
* session can be in while a stream is connected and chat is available at the same time:
|
|
1361
|
+
* {@link AgentManager.changeMode | changeMode()} to any other value disconnects the stream.
|
|
1362
|
+
*/
|
|
1363
|
+
Functional = "Functional",
|
|
1364
|
+
/**
|
|
1365
|
+
* Text answers only: the chat works, but no video is produced.
|
|
1366
|
+
*
|
|
1367
|
+
* {@link AgentManager.chat | chat()} still returns answers through
|
|
1368
|
+
* {@link AgentManagerCallbacks.onNewMessage | onNewMessage}, and
|
|
1369
|
+
* {@link AgentManager.speak | speak()} adds a text script to the transcript but streams no
|
|
1370
|
+
* video: it resolves with a `duration` of `0` and an empty `videoId`.
|
|
1371
|
+
*/
|
|
1372
|
+
TextOnly = "TextOnly",
|
|
1373
|
+
/**
|
|
1374
|
+
* The agent is unavailable: {@link AgentManager.chat | chat()} throws a
|
|
1375
|
+
* {@link ValidationError}.
|
|
1376
|
+
*
|
|
1377
|
+
* The SDK switches to it by itself when connecting fails after its retries, so a UI can show
|
|
1378
|
+
* that the agent is temporarily out of service; the server can also return it when a chat is
|
|
1379
|
+
* created. Like {@link ChatMode.TextOnly} it produces no video.
|
|
1380
|
+
*/
|
|
1381
|
+
Maintenance = "Maintenance",
|
|
1382
|
+
/**
|
|
1383
|
+
* A text-only test conversation, used by the agent playground in D-ID Studio.
|
|
1384
|
+
*
|
|
1385
|
+
* Produces no video, and marks each chat request with a playground header so the Agents API
|
|
1386
|
+
* can treat it as a test. Chats in this mode always go over the Agents API, even for
|
|
1387
|
+
* Expressive (V4) agents, which otherwise chat over the data channel. Applications normally
|
|
1388
|
+
* use {@link ChatMode.Functional} or {@link ChatMode.TextOnly} instead.
|
|
1389
|
+
*/
|
|
1390
|
+
Playground = "Playground",
|
|
1391
|
+
/**
|
|
1392
|
+
* Speak-only: no chat is created for the session, while video still streams.
|
|
1393
|
+
*
|
|
1394
|
+
* Talks (V2) and Clips (V3) agents only; {@link createAgentManager} and
|
|
1395
|
+
* {@link AgentManager.changeMode | changeMode()} reject it with a {@link ValidationError} for
|
|
1396
|
+
* Expressive (V4) agents.
|
|
1397
|
+
*
|
|
1398
|
+
* {@link AgentManager.chat | chat()} rejects with a {@link ValidationError} while this is the
|
|
1399
|
+
* mode, however the session arrived at it. {@link AgentManager.connect | connect()} also skips
|
|
1400
|
+
* the notifications web socket while it is in force. Use it when the application drives the
|
|
1401
|
+
* agent entirely through {@link AgentManager.speak | speak()} and never asks its LLM anything.
|
|
1402
|
+
* The stream keeps running only when this is the mode {@link createAgentManager} was given;
|
|
1403
|
+
* switching into it later with {@link AgentManager.changeMode | changeMode()} disconnects the
|
|
1404
|
+
* stream like any other non-{@link ChatMode.Functional} mode.
|
|
1405
|
+
*/
|
|
1406
|
+
DirectPlayback = "DirectPlayback",
|
|
1407
|
+
/**
|
|
1408
|
+
* Chat is switched off: no chat is created for the session, while video still streams.
|
|
1409
|
+
*
|
|
1410
|
+
* Talks (V2) and Clips (V3) agents only; {@link createAgentManager} and
|
|
1411
|
+
* {@link AgentManager.changeMode | changeMode()} reject it with a {@link ValidationError} for
|
|
1412
|
+
* Expressive (V4) agents.
|
|
1413
|
+
*
|
|
1414
|
+
* {@link AgentManager.chat | chat()} rejects with a {@link ValidationError} while this is the
|
|
1415
|
+
* mode, exactly as in {@link ChatMode.DirectPlayback}. The one difference between the two is
|
|
1416
|
+
* the notifications web socket: this mode still opens it on a Talks (V2) or Clips (V3) agent,
|
|
1417
|
+
* where {@link ChatMode.DirectPlayback} skips it. As with {@link ChatMode.DirectPlayback}, the
|
|
1418
|
+
* stream keeps running only when this is the mode set at creation; switching into it later with
|
|
1419
|
+
* {@link AgentManager.changeMode | changeMode()} disconnects the stream.
|
|
1420
|
+
*/
|
|
1421
|
+
Off = "Off"
|
|
1422
|
+
}
|
|
1423
|
+
|
|
1424
|
+
/**
|
|
1425
|
+
* The server put the session into a different chat mode from the one that was requested, and the
|
|
1426
|
+
* new mode cannot stream video.
|
|
1427
|
+
*
|
|
1428
|
+
* Raised while {@link AgentManager.connect | connect()} runs, when the chat the server created came
|
|
1429
|
+
* back in a mode other than the requested {@link AgentManagerOptions.mode | mode} and that mode is
|
|
1430
|
+
* not {@link ChatMode.Functional | Functional} — {@link ChatMode.TextOnly | TextOnly} or
|
|
1431
|
+
* {@link ChatMode.Maintenance | Maintenance}, for instance. The comparison only runs when `mode` was
|
|
1432
|
+
* set in the options, so a session that never asked for a mode never sees this error. The SDK
|
|
1433
|
+
* disconnects the stream it had just opened and `connect()` resolves anyway, so no video will play
|
|
1434
|
+
* and {@link AgentManagerCallbacks.onSrcObjectReady | onSrcObjectReady} will not fire again: the
|
|
1435
|
+
* session continues as a text chat.
|
|
1436
|
+
*
|
|
1437
|
+
* It is delivered to {@link AgentManagerCallbacks.onError | onError} and never thrown, so
|
|
1438
|
+
* `connect()` does not reject on it.
|
|
1439
|
+
* {@link AgentManagerCallbacks.onModeChange | onModeChange} fires with the new mode just before it.
|
|
1440
|
+
* Treat it as a signal to switch the interface to text, not as a connection failure.
|
|
1441
|
+
* {@link BaseError.kind | kind} is `'ChatModeDowngraded'`.
|
|
1442
|
+
*
|
|
1443
|
+
* @category Errors
|
|
1444
|
+
*/
|
|
1445
|
+
export declare class ChatModeDowngraded extends BaseError {
|
|
1446
|
+
/**
|
|
1447
|
+
* Always `'ChatModeDowngraded'`. Branch on it to tell this failure from the other SDK errors.
|
|
1448
|
+
*/
|
|
1449
|
+
readonly kind: 'ChatModeDowngraded';
|
|
1450
|
+
}
|
|
1451
|
+
|
|
1452
|
+
/**
|
|
1453
|
+
* What the Agents API answers a chat request with.
|
|
1454
|
+
*
|
|
1455
|
+
* Returned by {@link AgentManager.chat | chat()}. The SDK has already added the answer to the
|
|
1456
|
+
* transcript by the time you get it, so most applications render
|
|
1457
|
+
* {@link AgentManagerCallbacks.onNewMessage | onNewMessage} instead and use this only for the
|
|
1458
|
+
* fields that never reach a {@link Message}. Expressive (V4) agents chat over the data channel
|
|
1459
|
+
* rather than over the Agents API — except in {@link ChatMode.Playground} — so for them the
|
|
1460
|
+
* object is empty and the answer arrives through the callback.
|
|
1461
|
+
*
|
|
1462
|
+
* @category Chat
|
|
1463
|
+
*/
|
|
1464
|
+
export declare interface ChatResponse {
|
|
1465
|
+
/**
|
|
1466
|
+
* The agent's answer.
|
|
1467
|
+
*
|
|
1468
|
+
* Empty or absent when the answer is delivered as it is generated instead, which is what
|
|
1469
|
+
* streaming modes do; the SDK then adds no message for it and the text arrives through
|
|
1470
|
+
* {@link AgentManagerCallbacks.onNewMessage | onNewMessage}.
|
|
1471
|
+
*/
|
|
1472
|
+
result?: string;
|
|
1473
|
+
/**
|
|
1474
|
+
* The Agents API's own list of ids for the documents behind the answer. The SDK never reads it;
|
|
1475
|
+
* {@link ChatResponse.matches | matches} is what it keeps on the message.
|
|
1476
|
+
*/
|
|
1477
|
+
documentIds?: string[];
|
|
1478
|
+
/**
|
|
1479
|
+
* The knowledge citations behind the answer.
|
|
1480
|
+
*
|
|
1481
|
+
* Copied onto the answer as {@link Message.matches | matches}.
|
|
1482
|
+
*/
|
|
1483
|
+
matches?: RetrievalMetadata[];
|
|
1484
|
+
/**
|
|
1485
|
+
* The {@link ChatMode} the server handled the request in.
|
|
1486
|
+
*
|
|
1487
|
+
* It can differ from the mode that was asked for — see {@link ChatMode}.
|
|
1488
|
+
*/
|
|
1489
|
+
chatMode?: ChatMode;
|
|
1490
|
+
/**
|
|
1491
|
+
* The retrieved context the answer was generated from.
|
|
1492
|
+
*
|
|
1493
|
+
* Copied onto the answer as {@link Message.context | context}.
|
|
1494
|
+
*/
|
|
1495
|
+
context?: string;
|
|
1496
|
+
/** Id of the video generated for the answer, when the server reported one. */
|
|
1497
|
+
videoId?: string;
|
|
1498
|
+
}
|
|
1499
|
+
|
|
1500
|
+
/**
|
|
1501
|
+
* A client key for the Agents API — the credential to use in a browser.
|
|
1502
|
+
*
|
|
1503
|
+
* This is the one shape that is safe to ship in a page: a client key is scoped to a single agent
|
|
1504
|
+
* and only works from the domains allowed for it, so it cannot be reused elsewhere. Copy it from the
|
|
1505
|
+
* agent's Embed snippet, where it appears as `data-client-key`, or create one with the Agents API
|
|
1506
|
+
* (see the link below). The SDK sends it
|
|
1507
|
+
* as `Authorization: Client-Key <clientKey>.<externalId>_<connectionId>`, where the external id is
|
|
1508
|
+
* either {@link AgentManagerOptions.externalId | externalId} or a per-browser id the SDK keeps in
|
|
1509
|
+
* `localStorage`.
|
|
1510
|
+
*
|
|
1511
|
+
* @see [Create a client key](https://docs.d-id.com/reference/createclientkey)
|
|
1512
|
+
* @category Authentication
|
|
1513
|
+
*/
|
|
1514
|
+
export declare interface ClientKeyAuth {
|
|
1515
|
+
/**
|
|
1516
|
+
* Discriminator selecting this variant of {@link Auth}.
|
|
1517
|
+
*/
|
|
1518
|
+
type: 'key';
|
|
1519
|
+
/**
|
|
1520
|
+
* The client key: the `data-client-key` value from the agent's Embed snippet, or a key created with
|
|
1521
|
+
* the Agents API.
|
|
1522
|
+
*/
|
|
1523
|
+
clientKey: string;
|
|
1524
|
+
}
|
|
1525
|
+
|
|
1526
|
+
/**
|
|
1527
|
+
* A client tool's implementation: the function that runs in the browser when the agent calls it.
|
|
1528
|
+
*
|
|
1529
|
+
* Register it with {@link AgentManager.registerClientTool | registerClientTool()}, under the tool
|
|
1530
|
+
* name defined in the agent's configuration. The SDK parses the arguments the agent's LLM produced
|
|
1531
|
+
* and passes them in; whatever the handler resolves with is sent back to the agent as the tool's
|
|
1532
|
+
* result. Throwing rejects the call, and the error message is forwarded to the agent. Expressive
|
|
1533
|
+
* (V4) agents only.
|
|
1534
|
+
*
|
|
1535
|
+
* @param args - The arguments the LLM produced for this call, already parsed from JSON.
|
|
1536
|
+
* @returns A JSON string with the tool's result, at most 15 KiB — the LiveKit RPC response limit;
|
|
1537
|
+
* a larger result fails the call with an RPC error.
|
|
1538
|
+
* @example
|
|
1539
|
+
* ```ts
|
|
1540
|
+
* agentManager.registerClientTool('get_cart_total', async args => {
|
|
1541
|
+
* const total = await cart.total(args.currency as string);
|
|
1542
|
+
* return JSON.stringify({ total });
|
|
1543
|
+
* });
|
|
1544
|
+
* ```
|
|
1545
|
+
* @category Callbacks & Events
|
|
1546
|
+
*/
|
|
1547
|
+
export declare type ClientToolHandler = (args: Record<string, unknown>) => Promise<string>;
|
|
1548
|
+
|
|
1549
|
+
/**
|
|
1550
|
+
* Which video codec the stream should negotiate.
|
|
1551
|
+
*
|
|
1552
|
+
* Passed as {@link StreamOptions.compatibilityMode}. `on` forces VP8 and `off` forces H264; with
|
|
1553
|
+
* `auto` the SDK forwards the flag and the codec is selected according to the browser. Talks (V2)
|
|
1554
|
+
* and Clips (V3) agents only — Expressive (V4) agents negotiate the codec themselves and ignore
|
|
1555
|
+
* the setting.
|
|
1556
|
+
*
|
|
1557
|
+
* @category Streaming Options
|
|
1558
|
+
*/
|
|
1559
|
+
export declare type CompatibilityMode = 'on' | 'off' | 'auto';
|
|
1560
|
+
|
|
1561
|
+
/**
|
|
1562
|
+
* The state of the connection between the browser and the agent's stream.
|
|
1563
|
+
*
|
|
1564
|
+
* The first argument of
|
|
1565
|
+
* {@link AgentManagerCallbacks.onConnectionStateChange | onConnectionStateChange}.
|
|
1566
|
+
* {@link ConnectionState.Connected | 'connected'} is the point at which
|
|
1567
|
+
* {@link AgentManager.chat | chat()} and {@link AgentManager.speak | speak()} can be called.
|
|
1568
|
+
*
|
|
1569
|
+
* Every agent type reports `'connecting'`, `'connected'`, `'fail'` and `'disconnected'`; `'new'`,
|
|
1570
|
+
* `'completed'` and `'closed'` are WebRTC ICE states that only Talks (V2) and Clips (V3) agents
|
|
1571
|
+
* reach, and `'disconnecting'` is Expressive (V4) only.
|
|
1572
|
+
*
|
|
1573
|
+
* @category Callbacks & Events
|
|
1574
|
+
*/
|
|
1575
|
+
export declare enum ConnectionState {
|
|
1576
|
+
/** The connection object exists but nothing has been negotiated yet. */
|
|
1577
|
+
New = "new",
|
|
1578
|
+
/** The connection could not be established, or dropped irrecoverably. */
|
|
1579
|
+
Fail = "fail",
|
|
1580
|
+
/** The stream is live: the agent can be spoken to and its video rendered. */
|
|
1581
|
+
Connected = "connected",
|
|
1582
|
+
/** The connection is being established, or re-established after a drop. */
|
|
1583
|
+
Connecting = "connecting",
|
|
1584
|
+
/** The connection has been shut down and cannot be used again. */
|
|
1585
|
+
Closed = "closed",
|
|
1586
|
+
/**
|
|
1587
|
+
* Negotiation finished and the connection is fully established.
|
|
1588
|
+
*
|
|
1589
|
+
* It is reached after {@link ConnectionState.Connected | 'connected'}, so gate the application
|
|
1590
|
+
* on that one instead: it is the state every agent type reports.
|
|
1591
|
+
*/
|
|
1592
|
+
Completed = "completed",
|
|
1593
|
+
/**
|
|
1594
|
+
* {@link AgentManager.disconnect | disconnect()} is in progress.
|
|
1595
|
+
*
|
|
1596
|
+
* Expressive (V4) agents only; Talks (V2) and Clips (V3) go straight from
|
|
1597
|
+
* {@link ConnectionState.Connected | 'connected'} to
|
|
1598
|
+
* {@link ConnectionState.Disconnected | 'disconnected'}.
|
|
1599
|
+
*/
|
|
1600
|
+
Disconnecting = "disconnecting",
|
|
1601
|
+
/**
|
|
1602
|
+
* The stream has ended.
|
|
1603
|
+
*
|
|
1604
|
+
* The `reason` argument of
|
|
1605
|
+
* {@link AgentManagerCallbacks.onConnectionStateChange | onConnectionStateChange} says why —
|
|
1606
|
+
* a {@link StreamEndReason} when the server ended the session deliberately. Reconnecting with
|
|
1607
|
+
* {@link AgentManager.reconnect | reconnect()} starts a new stream.
|
|
1608
|
+
*/
|
|
1609
|
+
Disconnected = "disconnected"
|
|
1610
|
+
}
|
|
1611
|
+
|
|
1612
|
+
/**
|
|
1613
|
+
* How well the streamed media is reaching the user.
|
|
1614
|
+
*
|
|
1615
|
+
* The argument of
|
|
1616
|
+
* {@link AgentManagerCallbacks.onConnectivityStateChange | onConnectivityStateChange}. It is
|
|
1617
|
+
* estimated from the session's own media statistics — jitter-buffer delay and freeze count for
|
|
1618
|
+
* Talks (V2) and Clips (V3), the transport's reported connection quality for Expressive (V4) — so
|
|
1619
|
+
* it reflects what this stream is actually getting rather than whether the browser is online.
|
|
1620
|
+
*
|
|
1621
|
+
* @category Callbacks & Events
|
|
1622
|
+
*/
|
|
1623
|
+
export declare enum ConnectivityState {
|
|
1624
|
+
/** The stream is arriving smoothly. */
|
|
1625
|
+
Strong = "STRONG",
|
|
1626
|
+
/** The stream is degraded: video is stalling or arriving late. */
|
|
1627
|
+
Weak = "WEAK",
|
|
1628
|
+
/** There is not enough data to judge the connection yet, or the stream stopped arriving. */
|
|
1629
|
+
Unknown = "UNKNOWN"
|
|
1630
|
+
}
|
|
1631
|
+
|
|
1632
|
+
/**
|
|
1633
|
+
* Creates an {@link AgentManager} for one agent: its chat, its video stream and its connections.
|
|
1634
|
+
*
|
|
1635
|
+
* Fetches the agent from the Agents API before it resolves, so the returned manager already has
|
|
1636
|
+
* {@link AgentManager.agent | agent} and {@link AgentManager.starterMessages | starterMessages}
|
|
1637
|
+
* filled in. No stream is opened until {@link AgentManager.connect | connect()} is called.
|
|
1638
|
+
* {@link AgentManagerOptions.callbacks | callbacks} must include
|
|
1639
|
+
* {@link AgentManagerCallbacks.onSrcObjectReady | onSrcObjectReady}, which is what attaches the
|
|
1640
|
+
* streamed media to a video element.
|
|
1641
|
+
*
|
|
1642
|
+
* One option is checked here: an Expressive (V4) agent asked for {@link ChatMode.Off} or
|
|
1643
|
+
* {@link ChatMode.DirectPlayback}, which only Talks (V2) and Clips (V3) agents support, rejects
|
|
1644
|
+
* with a {@link ValidationError}. Every other bad argument surfaces later, as a
|
|
1645
|
+
* {@link ValidationError} rejected by the method that was called on the returned manager, such as
|
|
1646
|
+
* {@link AgentManager.chat | chat()} or {@link AgentManager.speak | speak()}.
|
|
1647
|
+
*
|
|
1648
|
+
* @param agent - Id of the agent to talk to: the `data-agent-id` from its Embed snippet, or the `id`
|
|
1649
|
+
* returned by the Agents API.
|
|
1650
|
+
* @param options - Credentials, callbacks and everything else the manager needs. See
|
|
1651
|
+
* {@link AgentManagerOptions}.
|
|
1652
|
+
* @returns A manager for that agent, ready to {@link AgentManager.connect | connect()}.
|
|
1653
|
+
* @throws {@link ValidationError} When an Expressive (V4) agent is created with
|
|
1654
|
+
* {@link ChatMode.Off} or {@link ChatMode.DirectPlayback}.
|
|
1655
|
+
* @throws {@link HttpError} When the agent cannot be fetched — an unknown id, or a client key that
|
|
1656
|
+
* is not authorized for the agent or the calling domain.
|
|
1657
|
+
* @throws {@link NetworkError} When the request for the agent never reaches the server: the browser
|
|
1658
|
+
* is offline, DNS or TLS fails, or the request is blocked.
|
|
1659
|
+
*
|
|
1660
|
+
* @example
|
|
1661
|
+
* ```ts
|
|
1662
|
+
* import * as sdk from '@d-id/client-sdk';
|
|
1663
|
+
*
|
|
1664
|
+
* const agentManager = await sdk.createAgentManager('agt_fumf1234', {
|
|
1665
|
+
* auth: { type: 'key', clientKey: 'YOUR_CLIENT_KEY' },
|
|
1666
|
+
* callbacks: {
|
|
1667
|
+
* // Required: hand the streamed media to your video element.
|
|
1668
|
+
* onSrcObjectReady(value) {
|
|
1669
|
+
* videoElement.srcObject = value;
|
|
1670
|
+
* },
|
|
1671
|
+
* },
|
|
1672
|
+
* });
|
|
1673
|
+
*
|
|
1674
|
+
* await agentManager.connect();
|
|
1675
|
+
* await agentManager.chat('What is the distance to the moon?');
|
|
1676
|
+
* ```
|
|
1677
|
+
*
|
|
1678
|
+
* @category Agent Manager
|
|
1679
|
+
*/
|
|
1680
|
+
export declare function createAgentManager(agent: string, options: AgentManagerOptions): Promise<AgentManager>;
|
|
1681
|
+
|
|
1682
|
+
/**
|
|
1683
|
+
* The data-channel topics an application may send on with
|
|
1684
|
+
* {@link AgentManager.sendDataChannelMessage | sendDataChannelMessage()}.
|
|
1685
|
+
*
|
|
1686
|
+
* The remaining data-channel topics are driven by their own methods
|
|
1687
|
+
* ({@link AgentManager.chat | chat()}, {@link AgentManager.speak | speak()},
|
|
1688
|
+
* {@link AgentManager.interrupt | interrupt()},
|
|
1689
|
+
* {@link AgentManager.setSttLanguage | setSttLanguage()}), which own the payload shape and
|
|
1690
|
+
* bookkeeping those topics expect, so they stay internal.
|
|
1691
|
+
*
|
|
1692
|
+
* The members' values are the wire strings. Take the value from the enum rather than writing the
|
|
1693
|
+
* string out, so the wire string stays in one place. Expressive (V4) agents only.
|
|
1694
|
+
*
|
|
1695
|
+
* @example
|
|
1696
|
+
* ```ts
|
|
1697
|
+
* import { DataChannelTopic } from '@d-id/client-sdk';
|
|
1698
|
+
*
|
|
1699
|
+
* await agentManager.sendDataChannelMessage(DataChannelTopic.Presentation, {
|
|
1700
|
+
* type: 'navigate',
|
|
1701
|
+
* slide: 3,
|
|
1702
|
+
* });
|
|
1703
|
+
* ```
|
|
1704
|
+
* @category Agent Manager
|
|
1705
|
+
*/
|
|
1706
|
+
export declare enum DataChannelTopic {
|
|
1707
|
+
/**
|
|
1708
|
+
* Messages that drive a presentation the agent is showing alongside its video, such as moving
|
|
1709
|
+
* to another slide. Sent on the wire as `did.presentation`; the payload shape is whatever the
|
|
1710
|
+
* presentation the agent is running expects.
|
|
1711
|
+
*/
|
|
1712
|
+
Presentation = "did.presentation"
|
|
1713
|
+
}
|
|
1714
|
+
|
|
1715
|
+
/**
|
|
1716
|
+
* Every error the SDK raises, as a union — what {@link isDIDError} narrows a caught value to.
|
|
1717
|
+
*
|
|
1718
|
+
* Each member declares {@link BaseError.kind | kind} as a different literal, so a `switch` on
|
|
1719
|
+
* `kind` inside an `isDIDError` branch narrows the value to one class and gives typed access to
|
|
1720
|
+
* that class's own fields — {@link HttpError.status | status} and {@link HttpError.code | code} on
|
|
1721
|
+
* an {@link HttpError}, {@link ValidationError.key | key} on a {@link ValidationError}, and so on.
|
|
1722
|
+
* The `default` branch of an exhaustive `switch` is `never` to the type checker.
|
|
1723
|
+
*
|
|
1724
|
+
* Keep that branch total anyway. {@link isDIDError} recognizes any `Error` carrying a string
|
|
1725
|
+
* `kind`, which is what lets it see across a bundle boundary, so a value built by a different copy
|
|
1726
|
+
* of the SDK — a 2.x `HttpError`, whose `kind` was the server's classification — can reach the
|
|
1727
|
+
* `default` branch at runtime with a `kind` outside this union.
|
|
1728
|
+
*
|
|
1729
|
+
* {@link BaseError} itself is deliberately not a member: its `kind` is a plain `string`, which
|
|
1730
|
+
* matches every literal and would stop the narrowing from working.
|
|
1731
|
+
*
|
|
1732
|
+
* @example Branching on a caught error
|
|
1733
|
+
* ```ts
|
|
1734
|
+
* import { isDIDError, type DIDError } from '@d-id/client-sdk';
|
|
1735
|
+
*
|
|
1736
|
+
* try {
|
|
1737
|
+
* await agentManager.connect();
|
|
1738
|
+
* } catch (error) {
|
|
1739
|
+
* if (!isDIDError(error)) {
|
|
1740
|
+
* throw error;
|
|
1741
|
+
* }
|
|
1742
|
+
*
|
|
1743
|
+
* switch (error.kind) {
|
|
1744
|
+
* case 'HttpError':
|
|
1745
|
+
* console.error(error.status, error.code);
|
|
1746
|
+
* break;
|
|
1747
|
+
* case 'NetworkError':
|
|
1748
|
+
* console.error('offline?', error.endpoint);
|
|
1749
|
+
* break;
|
|
1750
|
+
* default:
|
|
1751
|
+
* // Total on purpose: `kind` is `never` here to the type checker, but a value from
|
|
1752
|
+
* // another copy of the SDK can still land in this branch.
|
|
1753
|
+
* console.error(error.kind, error.message);
|
|
1754
|
+
* }
|
|
1755
|
+
* }
|
|
1756
|
+
* ```
|
|
1757
|
+
* @category Errors
|
|
1758
|
+
*/
|
|
1759
|
+
export declare type DIDError = HttpError | NetworkError | StreamError | WsError | ValidationError | ChatCreationFailed | ChatModeDowngraded;
|
|
1760
|
+
|
|
1761
|
+
/**
|
|
1762
|
+
* ElevenLabs provider details: the provider type and the requested voice id. Available to premium users.
|
|
1763
|
+
*
|
|
1764
|
+
* Pass it as {@link TextStreamScript.provider} to have ElevenLabs synthesise the text, optionally
|
|
1765
|
+
* with {@link VoiceConfigElevenlabs} to control how closely the voice is reproduced.
|
|
1766
|
+
*
|
|
1767
|
+
* @category Voice
|
|
1768
|
+
*/
|
|
1769
|
+
export declare interface ElevenlabsTtsProvider {
|
|
1770
|
+
/** Selects ElevenLabs. Always {@link Providers.Elevenlabs} (`'elevenlabs'`). */
|
|
1771
|
+
type: Providers.Elevenlabs;
|
|
1772
|
+
/**
|
|
1773
|
+
* Id of the voice to speak with, from D-ID's list of ElevenLabs voices.
|
|
1774
|
+
*
|
|
1775
|
+
* @example "21m00Tcm4TlvDq8ikWAM"
|
|
1776
|
+
* @see [ElevenLabs voices](https://docs.d-id.com/docs/tts-elevenlabs)
|
|
1777
|
+
*/
|
|
1778
|
+
voice_id: string;
|
|
1779
|
+
/**
|
|
1780
|
+
* How closely the voice should follow the original it was cloned from. See
|
|
1781
|
+
* {@link VoiceConfigElevenlabs}.
|
|
1782
|
+
*
|
|
1783
|
+
* @see [ElevenLabs voice settings](https://elevenlabs.io/docs/best-practices/prompting/controls)
|
|
1784
|
+
*/
|
|
1785
|
+
voice_config?: VoiceConfigElevenlabs;
|
|
1786
|
+
}
|
|
1787
|
+
|
|
1788
|
+
/**
|
|
1789
|
+
* The end-of-call feedback form configured on an agent.
|
|
1790
|
+
*
|
|
1791
|
+
* It describes what to ask the user when the conversation ends; the SDK does not render anything
|
|
1792
|
+
* itself. Read it from {@link Agent.end_of_call_feedback}, show your own form when
|
|
1793
|
+
* {@link EndOfCallFeedbackConfig.enabled | enabled} is set, and send the result with
|
|
1794
|
+
* {@link AgentManager.submitFeedback | submitFeedback()}.
|
|
1795
|
+
*
|
|
1796
|
+
* @category Agent Manager
|
|
1797
|
+
*/
|
|
1798
|
+
export declare interface EndOfCallFeedbackConfig {
|
|
1799
|
+
/** Whether the agent is configured to ask for feedback when a call ends. */
|
|
1800
|
+
enabled: boolean;
|
|
1801
|
+
/** The line to show the user as the call ends, before the rating is asked for. */
|
|
1802
|
+
closing_message?: string;
|
|
1803
|
+
/** Whether a free-text follow-up question should be asked after the rating. */
|
|
1804
|
+
follow_up_enabled?: boolean;
|
|
1805
|
+
/**
|
|
1806
|
+
* The follow-up question to ask, chosen by the score the user gave.
|
|
1807
|
+
*
|
|
1808
|
+
* Whichever one you show is what {@link AgentManager.submitFeedback | submitFeedback()} records
|
|
1809
|
+
* as {@link SubmitFeedbackResponse.question_shown | question_shown}.
|
|
1810
|
+
*/
|
|
1811
|
+
follow_up_messages?: {
|
|
1812
|
+
/** Question for a score at the low end of the scale. */
|
|
1813
|
+
low?: string;
|
|
1814
|
+
/** Question for a score of 4. */
|
|
1815
|
+
four?: string;
|
|
1816
|
+
/** Question for a score of 5. */
|
|
1817
|
+
five?: string;
|
|
1818
|
+
};
|
|
1819
|
+
}
|
|
1820
|
+
|
|
1821
|
+
/**
|
|
1822
|
+
* The JSON-safe payload {@link BaseError.toJson | toJson()} produces, for logs and error reports.
|
|
1823
|
+
*
|
|
1824
|
+
* Only the fields the SDK deliberately exposes are present, so the payload can be forwarded to a
|
|
1825
|
+
* logging service as it is. Subclasses add their own keys: {@link HttpError} adds `code` and
|
|
1826
|
+
* `httpStatus` and, when the failing call is known, `endpoint` and `method`; {@link NetworkError}
|
|
1827
|
+
* adds `endpoint`, `method`, `durationMs`, `online` and `visibility`.
|
|
1828
|
+
*
|
|
1829
|
+
* @example An HttpError from a request that was refused
|
|
1830
|
+
* ```json
|
|
1831
|
+
* {
|
|
1832
|
+
* "kind": "HttpError",
|
|
1833
|
+
* "code": "InsufficientCreditsError",
|
|
1834
|
+
* "message": "Account has insufficient credits",
|
|
1835
|
+
* "httpStatus": 402,
|
|
1836
|
+
* "endpoint": "/agt_x/chat/cht_y",
|
|
1837
|
+
* "method": "POST"
|
|
1838
|
+
* }
|
|
1839
|
+
* ```
|
|
1840
|
+
* @category Errors
|
|
1841
|
+
*/
|
|
1842
|
+
export declare interface ErrorJson {
|
|
1843
|
+
/**
|
|
1844
|
+
* Stable machine-readable code for the failure — the error's {@link BaseError.kind | kind}.
|
|
1845
|
+
*/
|
|
1846
|
+
kind: string;
|
|
1847
|
+
/**
|
|
1848
|
+
* Human-readable description of the failure — the error's `message`.
|
|
1849
|
+
*/
|
|
1850
|
+
message: string;
|
|
1851
|
+
/**
|
|
1852
|
+
* Message of the error that caused this one, truncated to 256 characters.
|
|
1853
|
+
*
|
|
1854
|
+
* Present only when {@link BaseError.originalError | originalError} is an `Error` and its
|
|
1855
|
+
* message differs from {@link ErrorJson.message | message}. A cause that is not an `Error` is
|
|
1856
|
+
* never serialized, so an arbitrary object carrying credentials cannot end up in the payload.
|
|
1857
|
+
*/
|
|
1858
|
+
cause?: string;
|
|
1859
|
+
/**
|
|
1860
|
+
* The Agents API's own classification of the failure, from {@link HttpError.code | code}.
|
|
1861
|
+
*
|
|
1862
|
+
* Present on an {@link HttpError} only.
|
|
1863
|
+
*/
|
|
1864
|
+
code?: string;
|
|
1865
|
+
/**
|
|
1866
|
+
* Additional fields contributed by a subclass, such as `httpStatus` on {@link HttpError}.
|
|
1867
|
+
*
|
|
1868
|
+
* Typed `unknown`, so a key that is not declared above has to be narrowed before it is used.
|
|
1869
|
+
*/
|
|
1870
|
+
[key: string]: unknown;
|
|
1871
|
+
}
|
|
1872
|
+
|
|
1873
|
+
/**
|
|
1874
|
+
* A request to the Agents API came back with a non-2xx status.
|
|
1875
|
+
*
|
|
1876
|
+
* Raised for every REST call the SDK makes on the application's behalf: fetching the agent in
|
|
1877
|
+
* {@link createAgentManager}, creating the stream and the chat during
|
|
1878
|
+
* {@link AgentManager.connect | connect()}, sending a message with
|
|
1879
|
+
* {@link AgentManager.chat | chat()}, and the rating and feedback calls. The same error is both
|
|
1880
|
+
* handed to {@link AgentManagerCallbacks.onError | onError} and thrown, so it also rejects the
|
|
1881
|
+
* promise of whichever method made the request — catch it around that call, or handle it centrally
|
|
1882
|
+
* in the callback. For the message-send request behind {@link AgentManager.chat | chat()} the
|
|
1883
|
+
* callback may not fire; the error is still thrown. Typical cases are `401` or `403` for a client
|
|
1884
|
+
* key that is not authorized for the agent or the calling domain, and `404` for an unknown agent
|
|
1885
|
+
* id; an account that is out of credits comes back with {@link HttpError.code | code}
|
|
1886
|
+
* `'InsufficientCreditsError'`. A `429` is retried twice, one second apart (three attempts in
|
|
1887
|
+
* total), before it surfaces.
|
|
1888
|
+
*
|
|
1889
|
+
* {@link BaseError.kind | kind} is always `'HttpError'`, so a `switch` on it narrows a caught value
|
|
1890
|
+
* to this class. The server's own classification lives on {@link HttpError.code | code} — the
|
|
1891
|
+
* `'InsufficientCreditsError'` above is one — so a message-independent branch on a specific API
|
|
1892
|
+
* failure is possible.
|
|
1893
|
+
*
|
|
1894
|
+
* The message is the envelope's `description` when the body is that JSON, and the raw body
|
|
1895
|
+
* otherwise — truncated to 256 characters either way, because a gateway can answer a 5xx with a
|
|
1896
|
+
* whole HTML page.
|
|
1897
|
+
*
|
|
1898
|
+
* @category Errors
|
|
1899
|
+
*/
|
|
1900
|
+
export declare class HttpError extends BaseError {
|
|
1901
|
+
/**
|
|
1902
|
+
* Always `'HttpError'`. Branch on it to tell this failure from the other SDK errors.
|
|
1903
|
+
*/
|
|
1904
|
+
readonly kind: 'HttpError';
|
|
1905
|
+
/**
|
|
1906
|
+
* The Agents API's own classification of the failure — the `kind` from D-ID's
|
|
1907
|
+
* `{ kind, description }` error body — and `'HttpError'` when the response was not that
|
|
1908
|
+
* envelope.
|
|
1909
|
+
*
|
|
1910
|
+
* An account out of credits comes back as `'InsufficientCreditsError'`, for instance. It is a
|
|
1911
|
+
* plain `string` because the set is the API's, not the SDK's, and grows without an SDK release.
|
|
1912
|
+
* Use {@link HttpError.status | status} for the transport-level branch, this for the API-level
|
|
1913
|
+
* one, and {@link BaseError.kind | kind} to tell an `HttpError` from the SDK's other errors.
|
|
1914
|
+
*/
|
|
1915
|
+
readonly code: string;
|
|
1916
|
+
/**
|
|
1917
|
+
* HTTP status code of the response, such as `401`, `404` or `500`.
|
|
1918
|
+
*/
|
|
1919
|
+
readonly status: number;
|
|
1920
|
+
/**
|
|
1921
|
+
* Path of the request that failed, relative to the API client's base path — for example
|
|
1922
|
+
* `/agt_x/chat/cht_y` for a message sent to a chat.
|
|
1923
|
+
*
|
|
1924
|
+
* Absent when the error was constructed without call context. {@link NetworkError.endpoint}
|
|
1925
|
+
* is the same value on a transport failure.
|
|
1926
|
+
*/
|
|
1927
|
+
readonly endpoint?: string;
|
|
1928
|
+
/**
|
|
1929
|
+
* HTTP method of the request that failed, such as `GET` or `POST`.
|
|
1930
|
+
*
|
|
1931
|
+
* Absent when the error was constructed without call context.
|
|
1932
|
+
*/
|
|
1933
|
+
readonly method?: string;
|
|
1934
|
+
/**
|
|
1935
|
+
* Serializes the error, adding the failing call to what
|
|
1936
|
+
* {@link BaseError.toJson | BaseError.toJson()} already returns.
|
|
1937
|
+
*
|
|
1938
|
+
* Adds {@link HttpError.code | code}, `httpStatus` from {@link HttpError.status | status}, and
|
|
1939
|
+
* `endpoint` and `method` when the call context is known. The raw `status` property is
|
|
1940
|
+
* serialized as `httpStatus`.
|
|
1941
|
+
*
|
|
1942
|
+
* @returns The error as plain, JSON-serializable data.
|
|
1943
|
+
*/
|
|
1944
|
+
toJson(): ErrorJson;
|
|
1945
|
+
}
|
|
1946
|
+
|
|
1947
|
+
/**
|
|
1948
|
+
* What caused the user to interrupt the agent, passed to
|
|
1949
|
+
* {@link AgentManager.interrupt | interrupt()}.
|
|
1950
|
+
*
|
|
1951
|
+
* @category Chat
|
|
1952
|
+
*/
|
|
1953
|
+
export declare interface InterruptOptions {
|
|
1954
|
+
/**
|
|
1955
|
+
* The cause: `text` when the user typed over the answer, `audio` when they started speaking,
|
|
1956
|
+
* `click` when they pressed a stop control, and `manual` for an interruption the application
|
|
1957
|
+
* decided on itself.
|
|
1958
|
+
*
|
|
1959
|
+
* The SDK records it with the interruption for analytics. It changes behaviour in one case
|
|
1960
|
+
* only: Expressive (V4) agents ignore a `text` interrupt, because their orchestrator does not
|
|
1961
|
+
* cancel the answer already in flight. Talks (V2) and Clips (V3) agents interrupt the current
|
|
1962
|
+
* video whatever the value.
|
|
1963
|
+
*/
|
|
1964
|
+
type: 'text' | 'audio' | 'click' | 'manual';
|
|
1965
|
+
}
|
|
1966
|
+
|
|
1967
|
+
/**
|
|
1968
|
+
* Whether the agent is waiting on a tool call before it can continue.
|
|
1969
|
+
*
|
|
1970
|
+
* True while any of the running calls is `blocking`; false when nothing is running, or when
|
|
1971
|
+
* everything running is `async` and the agent can keep talking. Pass it the array
|
|
1972
|
+
* {@link AgentManagerCallbacks.onRunningToolCallsChange | onRunningToolCallsChange} hands you, to
|
|
1973
|
+
* decide whether to show the user that the agent is busy.
|
|
1974
|
+
*
|
|
1975
|
+
* @param calls - The tool calls running right now, as reported by
|
|
1976
|
+
* {@link AgentManagerCallbacks.onRunningToolCallsChange | onRunningToolCallsChange}. See
|
|
1977
|
+
* {@link RunningToolCall}.
|
|
1978
|
+
* @returns `true` while the agent is suspended on a blocking tool call.
|
|
1979
|
+
* @example
|
|
1980
|
+
* ```ts
|
|
1981
|
+
* import { isAwaitingTool } from '@d-id/client-sdk';
|
|
1982
|
+
*
|
|
1983
|
+
* const callbacks = {
|
|
1984
|
+
* onRunningToolCallsChange(calls) {
|
|
1985
|
+
* setSpinnerVisible(isAwaitingTool(calls));
|
|
1986
|
+
* },
|
|
1987
|
+
* };
|
|
1988
|
+
* ```
|
|
1989
|
+
* @category Callbacks & Events
|
|
1990
|
+
*/
|
|
1991
|
+
export declare function isAwaitingTool(calls: readonly RunningToolCall[]): boolean;
|
|
1992
|
+
|
|
1993
|
+
/**
|
|
1994
|
+
* Type guard that reports whether a caught value is an error raised by the SDK.
|
|
1995
|
+
*
|
|
1996
|
+
* It accepts any `Error` carrying a string `kind`, which every {@link BaseError} has, so it also
|
|
1997
|
+
* recognizes SDK errors that crossed a bundle boundary, where `instanceof BaseError` can fail
|
|
1998
|
+
* because two copies of the class are loaded. It narrows to {@link DIDError}, the union of the
|
|
1999
|
+
* seven subclasses, so a `switch` on {@link BaseError.kind | kind} inside the branch narrows
|
|
2000
|
+
* further to one class and its own fields — `error.status` on an {@link HttpError},
|
|
2001
|
+
* `error.key` on a {@link ValidationError}.
|
|
2002
|
+
*
|
|
2003
|
+
* The duck-typing is wider than the union it promises, so keep the `default` branch of a `switch`
|
|
2004
|
+
* total — {@link DIDError} says why.
|
|
2005
|
+
*
|
|
2006
|
+
* @param error - The caught value to test.
|
|
2007
|
+
* @returns `true` when `error` is an error the SDK raised.
|
|
2008
|
+
* @example
|
|
2009
|
+
* ```ts
|
|
2010
|
+
* try {
|
|
2011
|
+
* await agentManager.chat('What is the distance to the moon?');
|
|
2012
|
+
* } catch (error) {
|
|
2013
|
+
* if (isDIDError(error)) {
|
|
2014
|
+
* console.error(error.kind, error.toJson());
|
|
2015
|
+
*
|
|
2016
|
+
* if (error.kind === 'HttpError' && error.status === 401) {
|
|
2017
|
+
* refreshCredentials();
|
|
2018
|
+
* }
|
|
2019
|
+
* } else {
|
|
2020
|
+
* throw error;
|
|
2021
|
+
* }
|
|
2022
|
+
* }
|
|
2023
|
+
* ```
|
|
2024
|
+
* @category Errors
|
|
2025
|
+
*/
|
|
2026
|
+
export declare function isDIDError(error: unknown): error is DIDError;
|
|
2027
|
+
|
|
2028
|
+
/**
|
|
2029
|
+
* One message of a chat: what the user asked, or what the agent answered.
|
|
2030
|
+
*
|
|
2031
|
+
* The SDK keeps the whole transcript and hands a fresh copy of it to
|
|
2032
|
+
* {@link AgentManagerCallbacks.onNewMessage | onNewMessage} every time a message is added or
|
|
2033
|
+
* changed, oldest first. While an answer streams in, the last message's
|
|
2034
|
+
* {@link Message.content | content} and {@link Message.parts | parts} are replaced with the latest
|
|
2035
|
+
* text on each `partial` callback — not appended to — and are final on `answer`. The same shape is
|
|
2036
|
+
* accepted by {@link AgentManagerOptions.initialMessages | initialMessages} to seed a transcript.
|
|
2037
|
+
*
|
|
2038
|
+
* @category Chat
|
|
2039
|
+
*/
|
|
2040
|
+
export declare interface Message {
|
|
2041
|
+
/**
|
|
2042
|
+
* Id of this message.
|
|
2043
|
+
*
|
|
2044
|
+
* Pass it to {@link AgentManager.rate | rate()} to rate the answer. The SDK generates a random
|
|
2045
|
+
* id for the messages it creates locally, and uses the id sent with the message for those that
|
|
2046
|
+
* arrive from the agent.
|
|
2047
|
+
*/
|
|
2048
|
+
id: string;
|
|
2049
|
+
/**
|
|
2050
|
+
* Who the message is from.
|
|
2051
|
+
*
|
|
2052
|
+
* The SDK itself only creates `user` (the end user) and `assistant` (the agent) messages; the
|
|
2053
|
+
* value on a transcribed or streamed message is whatever the server payload carried. `system`,
|
|
2054
|
+
* `function` and `tool` exist because the underlying chat protocol allows them.
|
|
2055
|
+
*/
|
|
2056
|
+
role?: 'system' | 'assistant' | 'user' | 'function' | 'tool';
|
|
2057
|
+
/**
|
|
2058
|
+
* The message text.
|
|
2059
|
+
*
|
|
2060
|
+
* Plain text, which for agent answers may contain markdown. It is replaced by the latest text
|
|
2061
|
+
* on every `partial` callback while the answer streams in — and the final `answer` may be
|
|
2062
|
+
* shorter than the partials before it, which is exactly how the SDK spots an interruption — so
|
|
2063
|
+
* render it as it is rather than appending to what you rendered before.
|
|
2064
|
+
*/
|
|
2065
|
+
content: string;
|
|
2066
|
+
/**
|
|
2067
|
+
* {@link Message.content | content} split into renderable pieces by {@link parseMessageParts}.
|
|
2068
|
+
*
|
|
2069
|
+
* Kept in step with `content` on every message the SDK creates, including while an answer
|
|
2070
|
+
* streams in. Render these instead of the raw string when the agent may answer with images,
|
|
2071
|
+
* videos or links; a message with no markup is a single `text` part, and an empty message is
|
|
2072
|
+
* an empty array.
|
|
2073
|
+
*
|
|
2074
|
+
* {@link AgentManagerOptions.initialMessages | initialMessages} are treated the same way: the
|
|
2075
|
+
* SDK runs the parser over the `content` of any of them that arrives with empty or missing
|
|
2076
|
+
* `parts`, so a restored transcript can pass `parts: []` and still render. A non-empty array
|
|
2077
|
+
* you built yourself is kept exactly as given.
|
|
2078
|
+
*/
|
|
2079
|
+
parts: MessagePart[];
|
|
2080
|
+
/** When the message was added, as an ISO 8601 timestamp. */
|
|
2081
|
+
createdAt?: string;
|
|
2082
|
+
/**
|
|
2083
|
+
* The knowledge citations the answer was drawn from, as {@link RetrievalMetadata} entries.
|
|
2084
|
+
*
|
|
2085
|
+
* Present on an agent answer when the chat response carried them, so a UI can show its sources;
|
|
2086
|
+
* {@link AgentManager.rate | rate()} also sends them with a rating. The SDK strips this field
|
|
2087
|
+
* from the transcript it sends back to the Agents API with the next
|
|
2088
|
+
* {@link AgentManager.chat | chat()}.
|
|
2089
|
+
*/
|
|
2090
|
+
matches?: ChatResponse['matches'];
|
|
2091
|
+
/**
|
|
2092
|
+
* The retrieved context the answer was generated from, as
|
|
2093
|
+
* {@link ChatResponse.context | the chat response} supplied it.
|
|
2094
|
+
*
|
|
2095
|
+
* Only ever set on an agent answer, and only when the answer came back from the Agents API.
|
|
2096
|
+
* Expressive (V4) agents chat over the data channel instead, so their answers carry no context
|
|
2097
|
+
* — except in {@link ChatMode.Playground}, which always takes the Agents API path.
|
|
2098
|
+
*/
|
|
2099
|
+
context?: string;
|
|
2100
|
+
/**
|
|
2101
|
+
* `true` when the answer was cut short instead of being spoken to the end.
|
|
2102
|
+
*
|
|
2103
|
+
* Set on the last message by {@link AgentManager.interrupt | interrupt()}, and by the SDK when
|
|
2104
|
+
* the final answer turns out to be shorter than the partial text already received — which is
|
|
2105
|
+
* how it detects that the agent was interrupted mid-utterance.
|
|
2106
|
+
*/
|
|
2107
|
+
interrupted?: boolean;
|
|
2108
|
+
/**
|
|
2109
|
+
* `true` when this user message came from speech-to-text rather than from
|
|
2110
|
+
* {@link AgentManager.chat | chat()}.
|
|
2111
|
+
*
|
|
2112
|
+
* It is what distinguishes an utterance the user spoke from one they typed.
|
|
2113
|
+
*/
|
|
2114
|
+
transcribed?: boolean;
|
|
2115
|
+
/**
|
|
2116
|
+
* The sentiment the agent delivered this answer with, when the stream reported one.
|
|
2117
|
+
*
|
|
2118
|
+
* Taken from the metadata the server sends with the video it created, and attached to the most
|
|
2119
|
+
* recent agent answer. The SDK only ever fills it in from an Expressive (V4) session, and only
|
|
2120
|
+
* while {@link AgentManagerOptions.debug | debug} is enabled.
|
|
2121
|
+
*/
|
|
2122
|
+
sentiment?: MessageSentiment;
|
|
2123
|
+
}
|
|
2124
|
+
|
|
2125
|
+
/**
|
|
2126
|
+
* One renderable piece of a message: a run of text, an image, a video or a link.
|
|
2127
|
+
*
|
|
2128
|
+
* {@link parseMessageParts} splits a message's {@link Message.content | content} into these, and
|
|
2129
|
+
* the SDK keeps the result on {@link Message.parts}. Switch on `type` when rendering. Anything the
|
|
2130
|
+
* parser did not recognise is preserved verbatim as a `text` part, in its original order, and
|
|
2131
|
+
* nothing is dropped or reordered; a recognised image, video or link replaces the markup it was
|
|
2132
|
+
* written as, so the parts are not a concatenation of the original string.
|
|
2133
|
+
*
|
|
2134
|
+
* @category Chat
|
|
2135
|
+
*/
|
|
2136
|
+
export declare type MessagePart = {
|
|
2137
|
+
/** Discriminant: plain text that the parser found no markup in. */
|
|
2138
|
+
type: 'text';
|
|
2139
|
+
/** The text itself, exactly as it appeared in {@link Message.content}. */
|
|
2140
|
+
text: string;
|
|
2141
|
+
} | {
|
|
2142
|
+
/** Discriminant: an image, from markdown image syntax such as ``. */
|
|
2143
|
+
type: 'image';
|
|
2144
|
+
/** URL of the image. */
|
|
2145
|
+
src: string;
|
|
2146
|
+
/** The alt text from the markdown, or an empty string when it had none. */
|
|
2147
|
+
alt: string;
|
|
2148
|
+
/**
|
|
2149
|
+
* `image/gif` when the URL itself ends in `.gif`, ignoring case; absent otherwise.
|
|
2150
|
+
*
|
|
2151
|
+
* The check looks at the whole URL, so a query string or fragment suppresses it —
|
|
2152
|
+
* `cat.gif?v=2` carries no `mimeType`. Video detection differs: it strips both before
|
|
2153
|
+
* looking at the extension.
|
|
2154
|
+
*/
|
|
2155
|
+
mimeType?: string;
|
|
2156
|
+
} | {
|
|
2157
|
+
/**
|
|
2158
|
+
* Discriminant: a video, either from thumbnail syntax (`[](video)`) or from
|
|
2159
|
+
* an image whose URL ends in `.mp4`, `.webm`, `.mkv`, `.mov`, `.m4v` or `.ogv` once any
|
|
2160
|
+
* query string and fragment have been stripped.
|
|
2161
|
+
*/
|
|
2162
|
+
type: 'video';
|
|
2163
|
+
/** URL of the video file. */
|
|
2164
|
+
src: string;
|
|
2165
|
+
/** The alt text from the markdown, or an empty string when it had none. */
|
|
2166
|
+
alt: string;
|
|
2167
|
+
/** URL of the poster image, when the thumbnail syntax supplied one. */
|
|
2168
|
+
thumbnail?: string;
|
|
2169
|
+
} | {
|
|
2170
|
+
/** Discriminant: a link, from markdown `[label](url)` or from an HTML `<a href>`. */
|
|
2171
|
+
type: 'link';
|
|
2172
|
+
/** Target of the link. */
|
|
2173
|
+
href: string;
|
|
2174
|
+
/** The text shown for the link. */
|
|
2175
|
+
label: string;
|
|
2176
|
+
};
|
|
2177
|
+
|
|
2178
|
+
/**
|
|
2179
|
+
* The sentiment an agent answer was delivered with, as the server reported it.
|
|
2180
|
+
*
|
|
2181
|
+
* The shape of {@link Message.sentiment}. It is an object rather than a string because the server
|
|
2182
|
+
* reports both the id of the sentiment it used and its name, where
|
|
2183
|
+
* {@link TextStreamScript.sentiment} — the sentiment the application *asks* for — is just the
|
|
2184
|
+
* name.
|
|
2185
|
+
*
|
|
2186
|
+
* @category Chat
|
|
2187
|
+
*/
|
|
2188
|
+
export declare interface MessageSentiment {
|
|
2189
|
+
/** Id of the sentiment, as the server sent it. */
|
|
2190
|
+
id: string;
|
|
2191
|
+
/** Name of the sentiment, as the server sent it. */
|
|
2192
|
+
name: string;
|
|
2193
|
+
}
|
|
2194
|
+
|
|
2195
|
+
/**
|
|
2196
|
+
* Microsoft Azure provider details: the provider type, the requested voice id and an optional
|
|
2197
|
+
* `voice_config` for style, rate and pitch.
|
|
2198
|
+
*
|
|
2199
|
+
* Pass it as {@link TextStreamScript.provider} to pick a Microsoft Azure voice explicitly,
|
|
2200
|
+
* optionally with {@link VoiceConfigMicrosoft} to set style, rate and pitch. The Agents API
|
|
2201
|
+
* documents Microsoft TTS as its default when a script names no provider, so this is also the
|
|
2202
|
+
* provider a script without one ends up using.
|
|
2203
|
+
*
|
|
2204
|
+
* @category Voice
|
|
2205
|
+
*/
|
|
2206
|
+
export declare interface MicrosoftTtsProvider {
|
|
2207
|
+
/** Selects Microsoft Azure. Always {@link Providers.Microsoft} (`'microsoft'`). */
|
|
2208
|
+
type: Providers.Microsoft;
|
|
2209
|
+
/**
|
|
2210
|
+
* Id of the voice to speak with, from D-ID's list of Microsoft Azure voices.
|
|
2211
|
+
*
|
|
2212
|
+
* @example "en-US-JennyNeural"
|
|
2213
|
+
* @see [Microsoft Azure voices](https://docs.d-id.com/docs/tts-microsoft)
|
|
2214
|
+
*/
|
|
2215
|
+
voice_id: string;
|
|
2216
|
+
/**
|
|
2217
|
+
* How the voice should deliver the text: style, rate and pitch. See
|
|
2218
|
+
* {@link VoiceConfigMicrosoft}.
|
|
2219
|
+
*/
|
|
2220
|
+
voice_config?: VoiceConfigMicrosoft;
|
|
2221
|
+
}
|
|
2222
|
+
|
|
2223
|
+
/**
|
|
2224
|
+
* A request to the Agents API never reached the server, so no response came back.
|
|
2225
|
+
*
|
|
2226
|
+
* This is the transport failure counterpart of {@link HttpError}, where the server did answer: the
|
|
2227
|
+
* browser's `fetch` rejected outright because the device is offline, DNS or TLS failed, the
|
|
2228
|
+
* connection was refused or the request was blocked, for instance by CORS or an extension. Like
|
|
2229
|
+
* {@link HttpError} it is both handed to {@link AgentManagerCallbacks.onError | onError} and
|
|
2230
|
+
* thrown, so it also rejects the promise of the method that made the request; for the message-send
|
|
2231
|
+
* request behind {@link AgentManager.chat | chat()} the callback may not fire, but the error is
|
|
2232
|
+
* still thrown. A request the SDK cancelled itself is not reported this way — the browser's own
|
|
2233
|
+
* `AbortError` is rethrown unchanged.
|
|
2234
|
+
*
|
|
2235
|
+
* The message is always `'Network request failed'`; the browser's own wording, such as
|
|
2236
|
+
* `'Failed to fetch'`, is kept in {@link BaseError.originalError | originalError} and appears as
|
|
2237
|
+
* `cause` in {@link NetworkError.toJson | toJson()}. The remaining properties record what was
|
|
2238
|
+
* happening at the time, which is usually what distinguishes a dropped connection from a tab that
|
|
2239
|
+
* was backgrounded.
|
|
2240
|
+
*
|
|
2241
|
+
* @category Errors
|
|
2242
|
+
*/
|
|
2243
|
+
export declare class NetworkError extends BaseError {
|
|
2244
|
+
/**
|
|
2245
|
+
* Always `'NetworkError'`. Branch on it to tell this failure from the other SDK errors.
|
|
2246
|
+
*/
|
|
2247
|
+
readonly kind: 'NetworkError';
|
|
2248
|
+
/**
|
|
2249
|
+
* Path of the request that failed, relative to the API client's base path — for example
|
|
2250
|
+
* `/agt_x/chat/cht_y`.
|
|
2251
|
+
*/
|
|
2252
|
+
readonly endpoint?: string;
|
|
2253
|
+
/**
|
|
2254
|
+
* HTTP method of the request that failed, such as `GET` or `POST`.
|
|
2255
|
+
*/
|
|
2256
|
+
readonly method?: string;
|
|
2257
|
+
/**
|
|
2258
|
+
* How long the request ran before it failed, in milliseconds.
|
|
2259
|
+
*
|
|
2260
|
+
* A value close to zero usually means the request never left the browser; a large one points
|
|
2261
|
+
* at a connection that was established and then lost.
|
|
2262
|
+
*/
|
|
2263
|
+
readonly durationMs?: number;
|
|
2264
|
+
/**
|
|
2265
|
+
* The browser's `navigator.onLine` at the moment of the failure — `false` when the device
|
|
2266
|
+
* reported itself offline.
|
|
2267
|
+
*/
|
|
2268
|
+
readonly online?: boolean;
|
|
2269
|
+
/**
|
|
2270
|
+
* The page's `document.visibilityState` at the moment of the failure.
|
|
2271
|
+
*
|
|
2272
|
+
* `'hidden'` means the tab was in the background, where browsers throttle or suspend network
|
|
2273
|
+
* activity.
|
|
2274
|
+
*/
|
|
2275
|
+
readonly visibility?: DocumentVisibilityState;
|
|
2276
|
+
/**
|
|
2277
|
+
* Serializes the error, adding the captured context to what
|
|
2278
|
+
* {@link BaseError.toJson | BaseError.toJson()} already returns.
|
|
2279
|
+
*
|
|
2280
|
+
* Each of `endpoint`, `method`, `durationMs`, `online` and `visibility` is included only when it
|
|
2281
|
+
* was captured.
|
|
2282
|
+
*
|
|
2283
|
+
* @returns The error as plain, JSON-serializable data.
|
|
2284
|
+
*/
|
|
2285
|
+
toJson(): ErrorJson;
|
|
2286
|
+
}
|
|
2287
|
+
|
|
2288
|
+
/**
|
|
2289
|
+
* Splits a message's text into the typed {@link MessagePart | parts} a UI can render.
|
|
2290
|
+
*
|
|
2291
|
+
* The SDK already runs it over every message it builds and keeps the result on
|
|
2292
|
+
* {@link Message.parts}, so this is for content that comes from somewhere else — a transcript you
|
|
2293
|
+
* render outside the SDK, or text you assembled before handing it to a renderer.
|
|
2294
|
+
* {@link AgentManagerOptions.initialMessages | initialMessages} no longer need it: pass `parts: []`
|
|
2295
|
+
* and the SDK runs this for you. It recognises markdown
|
|
2296
|
+
* images (``), the video thumbnail form (`[](video)`), markdown links
|
|
2297
|
+
* (`[label](url)`) and HTML anchors (`<a href="url">label</a>`); an image whose URL looks like a
|
|
2298
|
+
* video becomes a `video` part instead, and a GIF is tagged with a `mimeType`, both under the rules
|
|
2299
|
+
* {@link MessagePart} sets out. Everything the parser does not recognise is preserved verbatim as a
|
|
2300
|
+
* `text` part, in its original order, and nothing is dropped or reordered; each recognised part
|
|
2301
|
+
* replaces the markup it was written as, so the parts are not a concatenation of the input.
|
|
2302
|
+
*
|
|
2303
|
+
* @param content - The message text to split, as it appears in {@link Message.content}.
|
|
2304
|
+
* @returns The parts in the order they appear in `content`: a single `text` part when there is no
|
|
2305
|
+
* markup to find, and an empty array for an empty string.
|
|
2306
|
+
* @example
|
|
2307
|
+
* ```ts
|
|
2308
|
+
* import { parseMessageParts } from '@d-id/client-sdk';
|
|
2309
|
+
*
|
|
2310
|
+
* parseMessageParts('Here it is:  — enjoy!');
|
|
2311
|
+
* // [
|
|
2312
|
+
* // { type: 'text', text: 'Here it is: ' },
|
|
2313
|
+
* // { type: 'image', src: 'https://example.com/cat.png', alt: 'a cat' },
|
|
2314
|
+
* // { type: 'text', text: ' — enjoy!' },
|
|
2315
|
+
* // ]
|
|
2316
|
+
* ```
|
|
2317
|
+
* @category Chat
|
|
2318
|
+
*/
|
|
2319
|
+
export declare function parseMessageParts(content: string): MessagePart[];
|
|
2320
|
+
|
|
2321
|
+
/**
|
|
2322
|
+
* The text-to-speech engines a voice can be served by.
|
|
2323
|
+
*
|
|
2324
|
+
* The value of the `type` field that discriminates the provider objects — {@link Providers.Microsoft}
|
|
2325
|
+
* selects a {@link MicrosoftTtsProvider}, {@link Providers.Elevenlabs} an
|
|
2326
|
+
* {@link ElevenlabsTtsProvider}, and so on — and of {@link Voice.provider} when you look a voice up
|
|
2327
|
+
* in D-ID's catalogue. Which providers an account may use depends on its plan.
|
|
2328
|
+
*
|
|
2329
|
+
* @category Voice
|
|
2330
|
+
*/
|
|
2331
|
+
export declare enum Providers {
|
|
2332
|
+
/** Amazon's text-to-speech service; see {@link AmazonTtsProvider}. */
|
|
2333
|
+
Amazon = "amazon",
|
|
2334
|
+
/** Azure OpenAI text-to-speech; see {@link AzureOpenAiTtsProvider}. */
|
|
2335
|
+
AzureOpenAi = "azure-openai",
|
|
2336
|
+
/** Microsoft Azure text-to-speech, which the Agents API documents as its default when a
|
|
2337
|
+
* script names no provider; see {@link MicrosoftTtsProvider}. */
|
|
2338
|
+
Microsoft = "microsoft",
|
|
2339
|
+
/** ElevenLabs text-to-speech; see {@link ElevenlabsTtsProvider}. */
|
|
2340
|
+
Elevenlabs = "elevenlabs"
|
|
2341
|
+
}
|
|
2342
|
+
|
|
2343
|
+
/**
|
|
2344
|
+
* A rating stored against one message of a chat, as the Agents API returns it.
|
|
2345
|
+
*
|
|
2346
|
+
* Produced by {@link AgentManager.rate | rate()} and returned again by
|
|
2347
|
+
* {@link AgentManager.deleteRate | deleteRate()}. Keep {@link Rating.id | id} if the user may
|
|
2348
|
+
* change their mind: pass it back to {@link AgentManager.rate | rate()} as `rateId` to update the
|
|
2349
|
+
* rating, or to {@link AgentManager.deleteRate | deleteRate()} to remove it.
|
|
2350
|
+
*
|
|
2351
|
+
* @category Chat
|
|
2352
|
+
*/
|
|
2353
|
+
export declare interface Rating {
|
|
2354
|
+
/**
|
|
2355
|
+
* Id of this rating.
|
|
2356
|
+
*
|
|
2357
|
+
* Pass it as the `rateId` argument of {@link AgentManager.rate | rate()} to update the rating,
|
|
2358
|
+
* or to {@link AgentManager.deleteRate | deleteRate()} to remove it.
|
|
2359
|
+
*/
|
|
2360
|
+
id: string;
|
|
2361
|
+
/** Id of the D-ID account that owns the agent the rating was left on. Set by the Agents API. */
|
|
2362
|
+
owner_id: string;
|
|
2363
|
+
/** Id of the agent whose answer was rated. */
|
|
2364
|
+
agent_id: string;
|
|
2365
|
+
/**
|
|
2366
|
+
* The knowledge citations that were attached to the rated answer, as `[document_id, id]` pairs.
|
|
2367
|
+
*
|
|
2368
|
+
* The SDK builds them from the {@link Message.matches | matches} of the rated message, so a
|
|
2369
|
+
* negative rating can be traced back to the documents the answer was drawn from. Empty when the
|
|
2370
|
+
* answer cited nothing.
|
|
2371
|
+
*/
|
|
2372
|
+
matches: [string, string][];
|
|
2373
|
+
/**
|
|
2374
|
+
* Id of the agent's knowledge base ({@link Agent.knowledge}) at the time of the rating.
|
|
2375
|
+
*
|
|
2376
|
+
* An empty string when the agent has no knowledge base.
|
|
2377
|
+
*/
|
|
2378
|
+
knowledge_id: string;
|
|
2379
|
+
/**
|
|
2380
|
+
* The end-user identifier the Agents API recorded for the request.
|
|
2381
|
+
*
|
|
2382
|
+
* Derived from the credentials the SDK sends, which carry
|
|
2383
|
+
* {@link AgentManagerOptions.externalId | externalId} for `type: 'key'` authorization.
|
|
2384
|
+
*/
|
|
2385
|
+
external_id: string;
|
|
2386
|
+
/** Identity the Agents API attributed the rating to. Set by the Agents API. */
|
|
2387
|
+
created_by: string;
|
|
2388
|
+
/** Id of the chat the rated message belongs to — the value
|
|
2389
|
+
* {@link AgentManagerCallbacks.onNewChat | onNewChat} reported. */
|
|
2390
|
+
chat_id: string;
|
|
2391
|
+
/** The score itself: `1` for a positive rating, `-1` for a negative one. */
|
|
2392
|
+
score: 1 | -1;
|
|
2393
|
+
/** When the rating was created, as an ISO 8601 timestamp. */
|
|
2394
|
+
created_at: string;
|
|
2395
|
+
/** When the rating was last changed, as an ISO 8601 timestamp. */
|
|
2396
|
+
modified_at: string;
|
|
2397
|
+
/** {@link Message.id | Id of the message} that was rated. */
|
|
2398
|
+
message_id: string;
|
|
2399
|
+
}
|
|
2400
|
+
|
|
2401
|
+
/**
|
|
2402
|
+
* One knowledge citation: the passage of the agent's knowledge base an answer was drawn from.
|
|
2403
|
+
*
|
|
2404
|
+
* The Agents API returns these with an answer and the SDK keeps them on
|
|
2405
|
+
* {@link Message.matches | matches}, so a UI can show the sources behind a reply.
|
|
2406
|
+
*
|
|
2407
|
+
* @category Chat
|
|
2408
|
+
*/
|
|
2409
|
+
export declare interface RetrievalMetadata {
|
|
2410
|
+
/** Id of this match. Sent back with a rating as the second half of a
|
|
2411
|
+
* {@link Rating.matches} pair. */
|
|
2412
|
+
id: string;
|
|
2413
|
+
/** The matched passage itself, as stored in the knowledge base. */
|
|
2414
|
+
data: string;
|
|
2415
|
+
/** Title of the document the passage comes from. */
|
|
2416
|
+
title: string;
|
|
2417
|
+
/** Id of the source document. Sent back with a rating as the first half of a
|
|
2418
|
+
* {@link Rating.matches} pair. */
|
|
2419
|
+
document_id: string;
|
|
2420
|
+
/** Id of the knowledge base the document belongs to — the agent's {@link Agent.knowledge}. */
|
|
2421
|
+
knowledge_id: string;
|
|
2422
|
+
/** URL of the original source, for linking the citation out of the chat. */
|
|
2423
|
+
source_url: string;
|
|
2424
|
+
}
|
|
2425
|
+
|
|
2426
|
+
/**
|
|
2427
|
+
* A tool call currently running in the session.
|
|
2428
|
+
*
|
|
2429
|
+
* The entries of the array given to
|
|
2430
|
+
* {@link AgentManagerCallbacks.onRunningToolCallsChange | onRunningToolCallsChange}. A call appears
|
|
2431
|
+
* when it starts and disappears when it finishes, fails, or — for a `blocking` call — when its turn
|
|
2432
|
+
* ends; an `async` call outlives its turn.
|
|
2433
|
+
*
|
|
2434
|
+
* @category Callbacks & Events
|
|
2435
|
+
*/
|
|
2436
|
+
export declare interface RunningToolCall {
|
|
2437
|
+
/**
|
|
2438
|
+
* Id of this call, matching the {@link ToolCallStartedPayload.callId | callId} of the
|
|
2439
|
+
* {@link ToolCallStartedPayload} that announced it.
|
|
2440
|
+
*/
|
|
2441
|
+
callId: string;
|
|
2442
|
+
/** Name of the tool being called, as configured on the agent. */
|
|
2443
|
+
name: string;
|
|
2444
|
+
/**
|
|
2445
|
+
* Whether the agent is suspended while this call runs.
|
|
2446
|
+
*
|
|
2447
|
+
* `blocking` means the agent waits for the result before it can continue; `async` means it
|
|
2448
|
+
* keeps talking while the call runs. See {@link ToolExecutionMode}.
|
|
2449
|
+
*/
|
|
2450
|
+
executionMode: ToolExecutionMode;
|
|
2451
|
+
}
|
|
2452
|
+
|
|
2453
|
+
/**
|
|
2454
|
+
* What {@link AgentManager.speak | speak()} resolves with: the video the agent is about to stream.
|
|
2455
|
+
*
|
|
2456
|
+
* The fields come from the Talks (V2) and Clips (V3) API, which generates a discrete video while
|
|
2457
|
+
* answering the request. Where no such video exists the call still resolves, with the same stub —
|
|
2458
|
+
* `status` `'success'`, `duration` `0` and an empty `videoId`: on Expressive (V4) agents, whose
|
|
2459
|
+
* speech is streamed over the data channel rather than rendered as a separate video, and in a
|
|
2460
|
+
* text-only chat mode ({@link ChatMode.TextOnly}, {@link ChatMode.Playground} or
|
|
2461
|
+
* {@link ChatMode.Maintenance}), which produces no video at all.
|
|
2462
|
+
*
|
|
2463
|
+
* @category Speak & Scripts
|
|
2464
|
+
*/
|
|
2465
|
+
export declare interface SpeakResponse {
|
|
2466
|
+
/**
|
|
2467
|
+
* Whether the server accepted the speak request — `'success'` when it did.
|
|
2468
|
+
*/
|
|
2469
|
+
status: string;
|
|
2470
|
+
/**
|
|
2471
|
+
* Id of the session this call was made on; the SDK sends it back on later streaming requests.
|
|
2472
|
+
*/
|
|
2473
|
+
sessionId?: string;
|
|
2474
|
+
/**
|
|
2475
|
+
* Duration of the generated video as reported by the Agents API for Talks (V2) and Clips (V3)
|
|
2476
|
+
* agents. `0` when the call produced no discrete video — on Expressive (V4) agents, and in a
|
|
2477
|
+
* text-only chat mode.
|
|
2478
|
+
*/
|
|
2479
|
+
duration: number;
|
|
2480
|
+
/**
|
|
2481
|
+
* Id of the generated video.
|
|
2482
|
+
*
|
|
2483
|
+
* Use it to correlate this call with the video the agent then plays, which
|
|
2484
|
+
* {@link AgentManagerCallbacks.onVideoStateChange | onVideoStateChange} reports the start and
|
|
2485
|
+
* end of, and which {@link AgentManager.interrupt | interrupt()} cancels. Empty when the call
|
|
2486
|
+
* produced no discrete video — on Expressive (V4) agents, and in a text-only chat mode.
|
|
2487
|
+
*/
|
|
2488
|
+
videoId: string;
|
|
2489
|
+
}
|
|
2490
|
+
|
|
2491
|
+
/**
|
|
2492
|
+
* The script payload {@link AgentManager.speak | speak()} accepts: text or audio.
|
|
2493
|
+
*
|
|
2494
|
+
* Discriminated by its `type` field, so `'text'` narrows the object to {@link TextStreamScript}
|
|
2495
|
+
* and `'audio'` to {@link AudioStreamScript}.
|
|
2496
|
+
* {@link AgentManager.speak | speak()} additionally takes a plain string, as shorthand for a
|
|
2497
|
+
* {@link TextStreamScript} with that string as its `input`.
|
|
2498
|
+
*
|
|
2499
|
+
* @category Speak & Scripts
|
|
2500
|
+
*/
|
|
2501
|
+
export declare type SpeakScript = TextStreamScript | AudioStreamScript;
|
|
2502
|
+
|
|
2503
|
+
/**
|
|
2504
|
+
* Identifiers of the stream the SDK has just opened for the session.
|
|
2505
|
+
*
|
|
2506
|
+
* Handed to {@link AgentManagerCallbacks.onStreamCreated | onStreamCreated} once the server has
|
|
2507
|
+
* accepted the stream, before the first video frame arrives. Log the three ids together: they are
|
|
2508
|
+
* what identifies the session in D-ID's own records.
|
|
2509
|
+
*
|
|
2510
|
+
* @category Callbacks & Events
|
|
2511
|
+
*/
|
|
2512
|
+
export declare interface StreamCreatedInfo {
|
|
2513
|
+
/**
|
|
2514
|
+
* Id of the agent the stream was opened for.
|
|
2515
|
+
*/
|
|
2516
|
+
agentId: string;
|
|
2517
|
+
/**
|
|
2518
|
+
* Id of the session; the SDK sends it back on every subsequent request for this stream.
|
|
2519
|
+
* On Expressive (V4) agents it is the same value as
|
|
2520
|
+
* {@link StreamCreatedInfo.streamId | streamId}.
|
|
2521
|
+
*/
|
|
2522
|
+
sessionId: string;
|
|
2523
|
+
/**
|
|
2524
|
+
* Id of the stream itself.
|
|
2525
|
+
*/
|
|
2526
|
+
streamId: string;
|
|
2527
|
+
}
|
|
2528
|
+
|
|
2529
|
+
/**
|
|
2530
|
+
* Why the server ended the session.
|
|
2531
|
+
*
|
|
2532
|
+
* Arrives as the `reason` argument of
|
|
2533
|
+
* {@link AgentManagerCallbacks.onConnectionStateChange | onConnectionStateChange} together with
|
|
2534
|
+
* {@link ConnectionState.Disconnected | 'disconnected'}, and is how you tell a session the server
|
|
2535
|
+
* closed on purpose from a connection that simply dropped — `reason` is an opaque transport
|
|
2536
|
+
* diagnostic in the latter case, so compare it against these values rather than parsing it. A
|
|
2537
|
+
* close reason the SDK does not recognise is forwarded as-is, which is the other reason to compare
|
|
2538
|
+
* against the enum. Which members can arrive depends on the agent type: Talks (V2) and Clips (V3)
|
|
2539
|
+
* agents report only {@link StreamEndReason.Ok | Ok},
|
|
2540
|
+
* {@link StreamEndReason.UnknownError | UnknownError},
|
|
2541
|
+
* {@link StreamEndReason.NetworkIssue | NetworkIssue} and
|
|
2542
|
+
* {@link StreamEndReason.Inactivity | Inactivity}.
|
|
2543
|
+
* {@link AgentManager.reconnect | reconnect()} still works afterwards; it starts a new stream
|
|
2544
|
+
* rather than resuming the ended one.
|
|
2545
|
+
*
|
|
2546
|
+
* @example Reacting to the limits an Expressive (V4) session reports
|
|
2547
|
+
* ```ts
|
|
2548
|
+
* import { ConnectionState, StreamEndReason } from '@d-id/client-sdk';
|
|
2549
|
+
*
|
|
2550
|
+
* const callbacks = {
|
|
2551
|
+
* onConnectionStateChange(state, reason) {
|
|
2552
|
+
* if (state !== ConnectionState.Disconnected) return;
|
|
2553
|
+
*
|
|
2554
|
+
* if (reason === StreamEndReason.TimeLimit || reason === StreamEndReason.MessageLimit) {
|
|
2555
|
+
* showMessage('This session has reached its limit.');
|
|
2556
|
+
* } else if (reason === StreamEndReason.Inactivity) {
|
|
2557
|
+
* showMessage('The session timed out.');
|
|
2558
|
+
* }
|
|
2559
|
+
* },
|
|
2560
|
+
* };
|
|
2561
|
+
* ```
|
|
2562
|
+
* @category Callbacks & Events
|
|
2563
|
+
*/
|
|
2564
|
+
export declare enum StreamEndReason {
|
|
2565
|
+
/** The session ended normally, because it was closed deliberately. */
|
|
2566
|
+
Ok = "ok",
|
|
2567
|
+
/** The session ended because of a failure the server could not classify. */
|
|
2568
|
+
UnknownError = "unknown_error",
|
|
2569
|
+
/** The session ended because the connection to the browser broke down. */
|
|
2570
|
+
NetworkIssue = "network_issue",
|
|
2571
|
+
/**
|
|
2572
|
+
* The session ended because it reached the message limit configured for the agent.
|
|
2573
|
+
*
|
|
2574
|
+
* Expressive (V4) agents.
|
|
2575
|
+
*/
|
|
2576
|
+
MessageLimit = "message_limit",
|
|
2577
|
+
/**
|
|
2578
|
+
* The session ended because it reached its maximum duration.
|
|
2579
|
+
*
|
|
2580
|
+
* Expressive (V4) agents.
|
|
2581
|
+
*/
|
|
2582
|
+
TimeLimit = "time_limit",
|
|
2583
|
+
/**
|
|
2584
|
+
* The session ended because nothing happened for too long.
|
|
2585
|
+
*
|
|
2586
|
+
* The window is {@link StreamOptions.sessionTimeout | sessionTimeout} for Talks (V2) and
|
|
2587
|
+
* Clips (V3).
|
|
2588
|
+
*/
|
|
2589
|
+
Inactivity = "inactivity",
|
|
2590
|
+
/**
|
|
2591
|
+
* The agent itself ended the call.
|
|
2592
|
+
*
|
|
2593
|
+
* Expressive (V4) agents.
|
|
2594
|
+
*/
|
|
2595
|
+
EndedByAgent = "ended_by_agent"
|
|
2596
|
+
}
|
|
2597
|
+
|
|
2598
|
+
/**
|
|
2599
|
+
* Something went wrong inside the live audio and video stream.
|
|
2600
|
+
*
|
|
2601
|
+
* Unlike {@link HttpError} and {@link NetworkError}, which come from REST calls, this one comes
|
|
2602
|
+
* from the transport carrying the agent's video, so it is only ever delivered to
|
|
2603
|
+
* {@link AgentManagerCallbacks.onError | onError} — no call of yours is necessarily in flight when
|
|
2604
|
+
* it happens, and nothing throws it. The SDK raises it when:
|
|
2605
|
+
*
|
|
2606
|
+
* - no media track is subscribed within the timeout after the LiveKit room used by Expressive (V4)
|
|
2607
|
+
* agents connects, after which the SDK disconnects the stream;
|
|
2608
|
+
* - the LiveKit room reports a media-device or encryption failure;
|
|
2609
|
+
* - a message cannot be put on the data channel — the WebRTC channel used by Talks (V2) and Clips
|
|
2610
|
+
* (V3) is not open, or the LiveKit room is not connected or rejects the send;
|
|
2611
|
+
* - the server reports that the stream itself failed, with the message
|
|
2612
|
+
* `Stream failed with event <event>`.
|
|
2613
|
+
*
|
|
2614
|
+
* Most of these leave the session unusable, so the useful response is to tell the user and call
|
|
2615
|
+
* {@link AgentManager.reconnect | reconnect()}. {@link BaseError.kind | kind} is `'StreamError'`.
|
|
2616
|
+
* The message describes the failure, and is the plain `'Stream Error'` when the transport gave no
|
|
2617
|
+
* detail; the underlying error, where there was one, is kept as
|
|
2618
|
+
* {@link BaseError.originalError | originalError}.
|
|
2619
|
+
*
|
|
2620
|
+
* @category Errors
|
|
2621
|
+
*/
|
|
2622
|
+
export declare class StreamError extends BaseError {
|
|
2623
|
+
/**
|
|
2624
|
+
* Always `'StreamError'`. Branch on it to tell this failure from the other SDK errors.
|
|
2625
|
+
*/
|
|
2626
|
+
readonly kind: 'StreamError';
|
|
2627
|
+
}
|
|
2628
|
+
|
|
2629
|
+
/**
|
|
2630
|
+
* Whether the agent's video is currently playing.
|
|
2631
|
+
*
|
|
2632
|
+
* The argument of {@link AgentManagerCallbacks.onVideoStateChange | onVideoStateChange}: it tells
|
|
2633
|
+
* the application when to show the streamed video and when to fall back to the agent's idle video.
|
|
2634
|
+
*
|
|
2635
|
+
* @category Callbacks & Events
|
|
2636
|
+
*/
|
|
2637
|
+
export declare enum StreamingState {
|
|
2638
|
+
/** The agent has started speaking; render the streamed media. */
|
|
2639
|
+
Start = "START",
|
|
2640
|
+
/** The agent has finished speaking; the application can switch back to the idle video. */
|
|
2641
|
+
Stop = "STOP"
|
|
2642
|
+
}
|
|
2643
|
+
|
|
2644
|
+
/**
|
|
2645
|
+
* Transport settings applied when the stream is created.
|
|
2646
|
+
*
|
|
2647
|
+
* Talks (V2) and Clips (V3) agents only. Expressive (V4) agents manage transport settings
|
|
2648
|
+
* automatically and ignore these options. Pass the object as
|
|
2649
|
+
* {@link AgentManagerOptions.streamOptions}; every field is optional.
|
|
2650
|
+
*
|
|
2651
|
+
* @category Streaming Options
|
|
2652
|
+
*/
|
|
2653
|
+
export declare interface StreamOptions {
|
|
2654
|
+
/**
|
|
2655
|
+
* Which video codec the stream negotiates.
|
|
2656
|
+
*
|
|
2657
|
+
* See {@link CompatibilityMode}.
|
|
2658
|
+
*
|
|
2659
|
+
* @default auto
|
|
2660
|
+
*/
|
|
2661
|
+
compatibilityMode?: CompatibilityMode;
|
|
2662
|
+
/**
|
|
2663
|
+
* Whether the stream plays a warmup video while the connection settles.
|
|
2664
|
+
*
|
|
2665
|
+
* With it on the server plays a short warmup video as soon as the connection is up, and the
|
|
2666
|
+
* SDK holds back {@link ConnectionState.Connected | 'connected'} — and so the resolution of
|
|
2667
|
+
* {@link AgentManager.connect | connect()} — until that video has played out. Without it,
|
|
2668
|
+
* `'connected'` is reported as soon as the data channel opens. The application sees nothing of
|
|
2669
|
+
* the warmup video itself beyond the delay; it is not distinguishable from any other. A fluent
|
|
2670
|
+
* stream ignores the option — the warmup only runs on legacy streams (see
|
|
2671
|
+
* {@link StreamOptions.fluent | fluent}).
|
|
2672
|
+
*
|
|
2673
|
+
* @default false
|
|
2674
|
+
*/
|
|
2675
|
+
streamWarmup?: boolean;
|
|
2676
|
+
/**
|
|
2677
|
+
* How long the session may sit idle between messages before the server ends it, in seconds.
|
|
2678
|
+
*
|
|
2679
|
+
* Up to 300, and only for accounts whose plan allows it. When the session does time out,
|
|
2680
|
+
* {@link AgentManagerCallbacks.onConnectionStateChange | onConnectionStateChange} reports
|
|
2681
|
+
* {@link ConnectionState.Disconnected | 'disconnected'} with the reason
|
|
2682
|
+
* {@link StreamEndReason.Inactivity | 'inactivity'}. Leave it out and the server decides.
|
|
2683
|
+
*/
|
|
2684
|
+
sessionTimeout?: number;
|
|
2685
|
+
/**
|
|
2686
|
+
* Whether to request a fluent stream.
|
|
2687
|
+
*
|
|
2688
|
+
* `true` streams one video for both the idle and talking states; `false` uses the legacy mode,
|
|
2689
|
+
* where the application swaps between two video elements. Clips (V3) agents built on a Pro
|
|
2690
|
+
* avatar; always on for Expressive (V4). A fluent stream is also what makes
|
|
2691
|
+
* {@link AgentManager.interrupt | interrupt()} available.
|
|
2692
|
+
*
|
|
2693
|
+
* @default false
|
|
2694
|
+
*/
|
|
2695
|
+
fluent?: boolean;
|
|
2696
|
+
}
|
|
2697
|
+
|
|
2698
|
+
/**
|
|
2699
|
+
* How the agent's idle and talking video are delivered.
|
|
2700
|
+
*
|
|
2701
|
+
* Returned by {@link AgentManager.getStreamType | getStreamType()}. Which one a session gets
|
|
2702
|
+
* follows from the agent and from {@link StreamOptions.fluent | fluent}.
|
|
2703
|
+
*
|
|
2704
|
+
* @category Streaming Options
|
|
2705
|
+
*/
|
|
2706
|
+
export declare enum StreamType {
|
|
2707
|
+
/**
|
|
2708
|
+
* Two videos: the agent's idle video, which the application plays itself, and the streamed
|
|
2709
|
+
* talking video. Switch between them in
|
|
2710
|
+
* {@link AgentManagerCallbacks.onVideoStateChange | onVideoStateChange}.
|
|
2711
|
+
*/
|
|
2712
|
+
Legacy = "legacy",
|
|
2713
|
+
/**
|
|
2714
|
+
* One continuous video for both the idle and the talking state, so there is nothing to swap.
|
|
2715
|
+
* Expressive (V4) agents always stream this way, and it is what makes
|
|
2716
|
+
* {@link AgentManager.interrupt | interrupt()} possible.
|
|
2717
|
+
*/
|
|
2718
|
+
Fluent = "fluent"
|
|
2719
|
+
}
|
|
2720
|
+
|
|
2721
|
+
/**
|
|
2722
|
+
* A short-lived credential for the D-ID speech-to-text service.
|
|
2723
|
+
*
|
|
2724
|
+
* What {@link AgentManager.getSttToken | getSttToken()} resolves with, fetched fresh from the
|
|
2725
|
+
* Agents API each time. It lets an application drive its own speech recognition against the
|
|
2726
|
+
* service the agent uses without putting D-ID credentials in the browser. The token expires, so request one per recognition session
|
|
2727
|
+
* rather than holding on to it.
|
|
2728
|
+
*
|
|
2729
|
+
* @category Agent Manager
|
|
2730
|
+
*/
|
|
2731
|
+
export declare interface SttTokenResponse {
|
|
2732
|
+
/** The authorization token to present to the speech service. */
|
|
2733
|
+
token: string;
|
|
2734
|
+
/** The speech service region the token is valid in. */
|
|
2735
|
+
region: string;
|
|
2736
|
+
}
|
|
2737
|
+
|
|
2738
|
+
/**
|
|
2739
|
+
* What the Agents API stored when end-of-call feedback was submitted.
|
|
2740
|
+
*
|
|
2741
|
+
* Returned by {@link AgentManager.submitFeedback | submitFeedback()}, which rates the conversation
|
|
2742
|
+
* as a whole rather than a single answer.
|
|
2743
|
+
*
|
|
2744
|
+
* @category Chat
|
|
2745
|
+
*/
|
|
2746
|
+
export declare interface SubmitFeedbackResponse {
|
|
2747
|
+
/** Id of the chat the feedback belongs to — the value
|
|
2748
|
+
* {@link AgentManagerCallbacks.onNewChat | onNewChat} reported. */
|
|
2749
|
+
chat_id: string;
|
|
2750
|
+
/** The score that was submitted, on the agent's end-of-call scale of 1 to 5. */
|
|
2751
|
+
rating: number;
|
|
2752
|
+
/**
|
|
2753
|
+
* The follow-up question the user was asked, when one was recorded.
|
|
2754
|
+
*
|
|
2755
|
+
* Applications pick it from {@link EndOfCallFeedbackConfig.follow_up_messages}.
|
|
2756
|
+
*/
|
|
2757
|
+
question_shown?: string;
|
|
2758
|
+
/** When the feedback was stored, as an ISO 8601 timestamp. */
|
|
2759
|
+
submitted_at: string;
|
|
2760
|
+
}
|
|
2761
|
+
|
|
2762
|
+
/**
|
|
2763
|
+
* A script that makes the agent say text you supply, synthesised by a text-to-speech provider.
|
|
2764
|
+
*
|
|
2765
|
+
* The usual payload for {@link AgentManager.speak | speak()}. The agent's LLM is not involved, so
|
|
2766
|
+
* the agent says exactly what {@link TextStreamScript.input | input} contains — which is what makes
|
|
2767
|
+
* it the right script for greetings and other canned lines. Passing a plain string to
|
|
2768
|
+
* {@link AgentManager.speak | speak()} is shorthand for this script with `ssml` set to `false`.
|
|
2769
|
+
*
|
|
2770
|
+
* @example Text
|
|
2771
|
+
* ```ts
|
|
2772
|
+
* const speak = await agentManager.speak({
|
|
2773
|
+
* type: 'text',
|
|
2774
|
+
* input: "Hi! I'm Alice!",
|
|
2775
|
+
* });
|
|
2776
|
+
* ```
|
|
2777
|
+
* @example Text with sentiment
|
|
2778
|
+
* `sentiment` is for Expressive (V4) agents only. If the requested sentiment is not supported by
|
|
2779
|
+
* the agent, the default sentiment is used.
|
|
2780
|
+
* ```ts
|
|
2781
|
+
* const speak = await agentManager.speak({
|
|
2782
|
+
* type: 'text',
|
|
2783
|
+
* input: "Hi! I'm Alice!",
|
|
2784
|
+
* sentiment: 'friendly',
|
|
2785
|
+
* });
|
|
2786
|
+
* ```
|
|
2787
|
+
* @category Speak & Scripts
|
|
2788
|
+
*/
|
|
2789
|
+
export declare interface TextStreamScript {
|
|
2790
|
+
/**
|
|
2791
|
+
* Which kind of script this is. Always `text` for this variant; an
|
|
2792
|
+
* {@link AudioStreamScript} carries `audio` instead.
|
|
2793
|
+
*/
|
|
2794
|
+
type: 'text';
|
|
2795
|
+
/**
|
|
2796
|
+
* The text-to-speech provider and voice that synthesise
|
|
2797
|
+
* {@link TextStreamScript.input | input}, from the list of supported providers.
|
|
2798
|
+
*
|
|
2799
|
+
* One of the objects in {@link TtsProvider}: a `type` naming the provider, the
|
|
2800
|
+
* `voice_id` to speak with, and optional provider-specific `voice_config`. Leave it out and the
|
|
2801
|
+
* SDK sends the script without a provider, so the voice is chosen server-side; the Agents API
|
|
2802
|
+
* documents Microsoft TTS as its default when no provider is given.
|
|
2803
|
+
*/
|
|
2804
|
+
provider?: TtsProvider;
|
|
2805
|
+
/**
|
|
2806
|
+
* The text to be synthesised into speech.
|
|
2807
|
+
*
|
|
2808
|
+
* Each provider has its own limit on the text length.
|
|
2809
|
+
* @example "This is an example text"
|
|
2810
|
+
* @maxLength 40000
|
|
2811
|
+
* @minLength 3
|
|
2812
|
+
*/
|
|
2813
|
+
input: string;
|
|
2814
|
+
/**
|
|
2815
|
+
* Is the text provided in ssml form.
|
|
2816
|
+
*
|
|
2817
|
+
* Set it to `true` when {@link TextStreamScript.input | input} is SSML markup rather than plain
|
|
2818
|
+
* text, so the provider reads the tags instead of speaking them.
|
|
2819
|
+
* @default false
|
|
2820
|
+
*/
|
|
2821
|
+
ssml?: boolean;
|
|
2822
|
+
/**
|
|
2823
|
+
* Queue this speak behind the current speech instead of interrupting it. Expressive (V4)
|
|
2824
|
+
* agents only.
|
|
2825
|
+
* @default false
|
|
2826
|
+
*/
|
|
2827
|
+
should_queue_speaks?: boolean;
|
|
2828
|
+
/**
|
|
2829
|
+
* Sentiment name to speak with. Expressive (V4) agents only; if the agent does not support the
|
|
2830
|
+
* requested sentiment, its default sentiment is used.
|
|
2831
|
+
* @example "friendly"
|
|
2832
|
+
*/
|
|
2833
|
+
sentiment?: string;
|
|
2834
|
+
}
|
|
2835
|
+
|
|
2836
|
+
/**
|
|
2837
|
+
* The payload of a {@link ToolCallEvent.Done} event: a tool call finished successfully.
|
|
2838
|
+
*
|
|
2839
|
+
* Delivered to {@link AgentManagerCallbacks.onToolEvent | onToolEvent} — see
|
|
2840
|
+
* {@link ToolEventCallback}.
|
|
2841
|
+
*
|
|
2842
|
+
* @category Callbacks & Events
|
|
2843
|
+
*/
|
|
2844
|
+
export declare interface ToolCallDonePayload {
|
|
2845
|
+
/** Id of the call that finished, matching the {@link ToolCallStartedPayload} that announced it. */
|
|
2846
|
+
callId: string;
|
|
2847
|
+
/** Name of the tool that was called. */
|
|
2848
|
+
name: string;
|
|
2849
|
+
/** The arguments the call was made with. */
|
|
2850
|
+
input: Record<string, unknown>;
|
|
2851
|
+
/** The result the tool returned. */
|
|
2852
|
+
output: Record<string, unknown>;
|
|
2853
|
+
/** How long the call took, in milliseconds. */
|
|
2854
|
+
durationMs: number;
|
|
2855
|
+
/** Any additional metadata the tool reported alongside its result. */
|
|
2856
|
+
extra: Record<string, unknown>;
|
|
2857
|
+
/** When the call finished, as reported by the server. */
|
|
2858
|
+
timestamp: string;
|
|
2859
|
+
}
|
|
2860
|
+
|
|
2861
|
+
/**
|
|
2862
|
+
* The payload of a {@link ToolCallEvent.Error} event: a tool call failed.
|
|
2863
|
+
*
|
|
2864
|
+
* Delivered to {@link AgentManagerCallbacks.onToolEvent | onToolEvent} — see
|
|
2865
|
+
* {@link ToolEventCallback}. Read {@link ToolCallErrorPayload.error | error} for what went wrong.
|
|
2866
|
+
* The agent carries on with the conversation; the SDK does not retry.
|
|
2867
|
+
*
|
|
2868
|
+
* @category Callbacks & Events
|
|
2869
|
+
*/
|
|
2870
|
+
export declare interface ToolCallErrorPayload {
|
|
2871
|
+
/** Id of the call that failed, matching the {@link ToolCallStartedPayload} that announced it. */
|
|
2872
|
+
callId: string;
|
|
2873
|
+
/** Name of the tool that was called. */
|
|
2874
|
+
name: string;
|
|
2875
|
+
/** The arguments the call was made with. */
|
|
2876
|
+
input: Record<string, unknown>;
|
|
2877
|
+
/**
|
|
2878
|
+
* Whatever the failed call produced, if anything.
|
|
2879
|
+
*
|
|
2880
|
+
* A `string` when the server answers a failure with the reason as plain text rather than a
|
|
2881
|
+
* structured result — that text is also given to you as
|
|
2882
|
+
* {@link ToolCallErrorPayload.error | error}, so narrow this one before you index into it.
|
|
2883
|
+
*/
|
|
2884
|
+
output: Record<string, unknown> | string;
|
|
2885
|
+
/** How long the call ran before failing, in milliseconds. */
|
|
2886
|
+
durationMs: number;
|
|
2887
|
+
/** Any additional metadata the server reported with the failure. */
|
|
2888
|
+
extra: Record<string, unknown>;
|
|
2889
|
+
/**
|
|
2890
|
+
* What went wrong, in one line, when the server said.
|
|
2891
|
+
*
|
|
2892
|
+
* Taken from the failure the server reports in {@link ToolCallErrorPayload.extra | extra}
|
|
2893
|
+
* (`extra.error.message`), and from {@link ToolCallErrorPayload.output | output} when that is
|
|
2894
|
+
* a plain string instead. Absent when the server sent neither, which is why an application
|
|
2895
|
+
* that shows the reason needs a fallback of its own; the full structured failure — its kind,
|
|
2896
|
+
* code and any data — is in `extra.error`.
|
|
2897
|
+
*/
|
|
2898
|
+
error?: string;
|
|
2899
|
+
/** When the call failed, as reported by the server. */
|
|
2900
|
+
timestamp: string;
|
|
2901
|
+
}
|
|
2902
|
+
|
|
2903
|
+
/**
|
|
2904
|
+
* The tool-call events delivered to {@link AgentManagerCallbacks.onToolEvent | onToolEvent}.
|
|
2905
|
+
*
|
|
2906
|
+
* Switch on the first argument of the handler to tell them apart: it narrows the second argument to
|
|
2907
|
+
* the payload that event carries — see {@link ToolEventCallback}. The members' values are the wire
|
|
2908
|
+
* strings the server sends, so compare against the enum rather than writing the string out.
|
|
2909
|
+
* Expressive (V4) agents only.
|
|
2910
|
+
*
|
|
2911
|
+
* @example
|
|
2912
|
+
* ```ts
|
|
2913
|
+
* import { ToolCallEvent } from '@d-id/client-sdk';
|
|
2914
|
+
*
|
|
2915
|
+
* const callbacks = {
|
|
2916
|
+
* onToolEvent(event, data) {
|
|
2917
|
+
* if (event === ToolCallEvent.Started) {
|
|
2918
|
+
* console.log('started', data.name, data.input);
|
|
2919
|
+
* }
|
|
2920
|
+
* },
|
|
2921
|
+
* };
|
|
2922
|
+
* ```
|
|
2923
|
+
* @category Callbacks & Events
|
|
2924
|
+
*/
|
|
2925
|
+
export declare enum ToolCallEvent {
|
|
2926
|
+
/**
|
|
2927
|
+
* The agent has begun a tool call; the payload is a {@link ToolCallStartedPayload}, carrying the
|
|
2928
|
+
* call's id, the tool's name and the arguments the agent's LLM produced.
|
|
2929
|
+
*/
|
|
2930
|
+
Started = "tool-call/started",
|
|
2931
|
+
/**
|
|
2932
|
+
* A tool call finished successfully; the payload is a {@link ToolCallDonePayload}, carrying the
|
|
2933
|
+
* result the tool returned and how long the call took.
|
|
2934
|
+
*/
|
|
2935
|
+
Done = "tool-call/done",
|
|
2936
|
+
/**
|
|
2937
|
+
* A tool call failed; the payload is a {@link ToolCallErrorPayload}, carrying the reason in
|
|
2938
|
+
* {@link ToolCallErrorPayload.error | error} when the server gave one. The agent carries on
|
|
2939
|
+
* with the conversation.
|
|
2940
|
+
*/
|
|
2941
|
+
Error = "tool-call/error"
|
|
2942
|
+
}
|
|
2943
|
+
|
|
2944
|
+
/**
|
|
2945
|
+
* The payload of a {@link ToolCallEvent.Started} event: the agent has begun a tool call.
|
|
2946
|
+
*
|
|
2947
|
+
* Delivered to {@link AgentManagerCallbacks.onToolEvent | onToolEvent} — see
|
|
2948
|
+
* {@link ToolEventCallback}.
|
|
2949
|
+
*
|
|
2950
|
+
* @category Callbacks & Events
|
|
2951
|
+
*/
|
|
2952
|
+
export declare interface ToolCallStartedPayload {
|
|
2953
|
+
/** Id of this call. The matching done or error payload carries the same id. */
|
|
2954
|
+
callId: string;
|
|
2955
|
+
/** Name of the tool the agent is calling, as configured on the agent. */
|
|
2956
|
+
name: string;
|
|
2957
|
+
/** The arguments the agent's LLM produced for this call. */
|
|
2958
|
+
input: Record<string, unknown>;
|
|
2959
|
+
/**
|
|
2960
|
+
* The tool's result, when the server sent one.
|
|
2961
|
+
*
|
|
2962
|
+
* The event is emitted before the tool has run, so it is normally absent; read the
|
|
2963
|
+
* {@link ToolCallDonePayload.output | output} of the matching done event instead.
|
|
2964
|
+
*/
|
|
2965
|
+
output?: Record<string, unknown>;
|
|
2966
|
+
/**
|
|
2967
|
+
* Whether the server considers the agent interruptible while this call is outstanding.
|
|
2968
|
+
*
|
|
2969
|
+
* Informational: the SDK does not forward it.
|
|
2970
|
+
* {@link AgentManagerCallbacks.onInterruptibleChange | onInterruptibleChange} is derived from
|
|
2971
|
+
* the {@link ToolCallStartedPayload.executionMode | executionMode} of the calls still
|
|
2972
|
+
* running, not from this field.
|
|
2973
|
+
*/
|
|
2974
|
+
interruptible: boolean;
|
|
2975
|
+
/**
|
|
2976
|
+
* Whether the agent waits for this call. See {@link ToolExecutionMode}.
|
|
2977
|
+
*
|
|
2978
|
+
* Always present: a server event that omits it, or carries anything other than `async`, is
|
|
2979
|
+
* reported as `'blocking'` — the same normalization
|
|
2980
|
+
* {@link RunningToolCall.executionMode | RunningToolCall.executionMode} applies, so the two
|
|
2981
|
+
* agree about the same call.
|
|
2982
|
+
*/
|
|
2983
|
+
executionMode: ToolExecutionMode;
|
|
2984
|
+
/** The conversational turn this call belongs to, or `null` when it belongs to no turn. */
|
|
2985
|
+
turnId?: number | null;
|
|
2986
|
+
/** When the call started, as reported by the server. */
|
|
2987
|
+
timestamp: string;
|
|
2988
|
+
}
|
|
2989
|
+
|
|
2990
|
+
/**
|
|
2991
|
+
* The overloaded handler type for
|
|
2992
|
+
* {@link AgentManagerCallbacks.onToolEvent | onToolEvent}: the event argument narrows the payload
|
|
2993
|
+
* argument.
|
|
2994
|
+
*
|
|
2995
|
+
* Three overloads, one per tool-call event, so the first argument narrows the second: a handler
|
|
2996
|
+
* written against this type sees exactly one of {@link ToolCallStartedPayload},
|
|
2997
|
+
* {@link ToolCallDonePayload} and {@link ToolCallErrorPayload}, never a union of the three.
|
|
2998
|
+
* Expressive (V4) agents only.
|
|
2999
|
+
*
|
|
3000
|
+
* @example
|
|
3001
|
+
* ```ts
|
|
3002
|
+
* import { ToolCallEvent } from '@d-id/client-sdk';
|
|
3003
|
+
*
|
|
3004
|
+
* const callbacks = {
|
|
3005
|
+
* onToolEvent(event, data) {
|
|
3006
|
+
* if (event === ToolCallEvent.Started) {
|
|
3007
|
+
* console.log('started', data.name, data.input);
|
|
3008
|
+
* } else if (event === ToolCallEvent.Done) {
|
|
3009
|
+
* console.log('done', data.name, data.output, data.durationMs);
|
|
3010
|
+
* } else {
|
|
3011
|
+
* console.log('failed', data.name, data.error ?? 'no reason given');
|
|
3012
|
+
* }
|
|
3013
|
+
* },
|
|
3014
|
+
* };
|
|
3015
|
+
* ```
|
|
3016
|
+
* @category Callbacks & Events
|
|
3017
|
+
*/
|
|
3018
|
+
export declare type ToolEventCallback = {
|
|
3019
|
+
/**
|
|
3020
|
+
* @param event - Always {@link ToolCallEvent.Started} in this overload.
|
|
3021
|
+
* @param data - The call the agent has just begun, with the arguments its LLM produced.
|
|
3022
|
+
*/
|
|
3023
|
+
(event: ToolCallEvent.Started, data: ToolCallStartedPayload): void;
|
|
3024
|
+
/**
|
|
3025
|
+
* @param event - Always {@link ToolCallEvent.Done} in this overload.
|
|
3026
|
+
* @param data - The call that has just finished, with the result the tool returned.
|
|
3027
|
+
*/
|
|
3028
|
+
(event: ToolCallEvent.Done, data: ToolCallDonePayload): void;
|
|
3029
|
+
/**
|
|
3030
|
+
* @param event - Always {@link ToolCallEvent.Error} in this overload.
|
|
3031
|
+
* @param data - The call that has just failed, with the reason in
|
|
3032
|
+
* {@link ToolCallErrorPayload.error | error} when the server gave one.
|
|
3033
|
+
*/
|
|
3034
|
+
(event: ToolCallEvent.Error, data: ToolCallErrorPayload): void;
|
|
3035
|
+
};
|
|
3036
|
+
|
|
3037
|
+
/**
|
|
3038
|
+
* Whether the agent waits for a tool call to finish before it carries on.
|
|
3039
|
+
*
|
|
3040
|
+
* Set on the agent's tool configuration, and reported on {@link RunningToolCall.executionMode} and
|
|
3041
|
+
* {@link ToolCallStartedPayload.executionMode}. Only a `blocking` call suspends the agent, which
|
|
3042
|
+
* is why one being outstanding is what makes the agent uninterruptible — see
|
|
3043
|
+
* {@link AgentManagerCallbacks.onInterruptibleChange | onInterruptibleChange}.
|
|
3044
|
+
*
|
|
3045
|
+
* @category Callbacks & Events
|
|
3046
|
+
*/
|
|
3047
|
+
export declare type ToolExecutionMode = 'blocking' | 'async';
|
|
3048
|
+
|
|
3049
|
+
/**
|
|
3050
|
+
* The provider object a speak script accepts: any of the four text-to-speech providers.
|
|
3051
|
+
*
|
|
3052
|
+
* This is the type of {@link TextStreamScript.provider}. Pick the variant for the provider you
|
|
3053
|
+
* want, give it the `voice_id` to speak with and, except for {@link AmazonTtsProvider}, an optional
|
|
3054
|
+
* `voice_config`; `type` discriminates the union.
|
|
3055
|
+
*
|
|
3056
|
+
* @example
|
|
3057
|
+
* ```ts
|
|
3058
|
+
* import { Providers } from '@d-id/client-sdk';
|
|
3059
|
+
*
|
|
3060
|
+
* const speak = await agentManager.speak({
|
|
3061
|
+
* type: 'text',
|
|
3062
|
+
* input: "Hi! I'm Alice!",
|
|
3063
|
+
* provider: { type: Providers.Microsoft, voice_id: 'en-US-JennyNeural' },
|
|
3064
|
+
* });
|
|
3065
|
+
* ```
|
|
3066
|
+
* @category Voice
|
|
3067
|
+
*/
|
|
3068
|
+
export declare type TtsProvider = MicrosoftTtsProvider | AzureOpenAiTtsProvider | ElevenlabsTtsProvider | AmazonTtsProvider;
|
|
3069
|
+
|
|
3070
|
+
/**
|
|
3071
|
+
* A method was called with arguments, or in a state, that the SDK will not send to the Agents
|
|
3072
|
+
* API.
|
|
3073
|
+
*
|
|
3074
|
+
* This is one of the two SDK errors that only ever reach the caller and are never delivered to
|
|
3075
|
+
* {@link AgentManagerCallbacks.onError | onError} (the other is {@link ChatCreationFailed}): it is
|
|
3076
|
+
* the direct answer to the call you just made, and every method that raises it rejects the promise
|
|
3077
|
+
* it returned, so `await` the call inside a `try`/`catch` or attach a `.catch()`. The SDK raises it
|
|
3078
|
+
* from
|
|
3079
|
+
* {@link AgentManager.chat | chat()} when the message is empty or 2000 characters or longer, when
|
|
3080
|
+
* the session's {@link ChatMode} has chat disabled or is in maintenance, and when the chat or the
|
|
3081
|
+
* stream has not been initialized yet; from {@link AgentManager.speak | speak()} when it is called
|
|
3082
|
+
* before {@link AgentManager.connect | connect()}; and from
|
|
3083
|
+
* {@link AgentManager.rate | rate()}, {@link AgentManager.deleteRate | deleteRate()} and
|
|
3084
|
+
* {@link AgentManager.submitFeedback | submitFeedback()} when no chat has started, plus, for
|
|
3085
|
+
* `rate()` alone, when no message with the given id is in the transcript. It is also raised from
|
|
3086
|
+
* {@link createAgentManager} and {@link AgentManager.changeMode | changeMode()} when an Expressive
|
|
3087
|
+
* (V4) agent is asked for {@link ChatMode.Off} or {@link ChatMode.DirectPlayback}, which only Talks
|
|
3088
|
+
* (V2) and Clips (V3) agents support. The five Expressive-only media methods —
|
|
3089
|
+
* {@link AgentManager.publishMicrophoneStream | publishMicrophoneStream()},
|
|
3090
|
+
* {@link AgentManager.replaceMicrophoneTrack | replaceMicrophoneTrack()},
|
|
3091
|
+
* {@link AgentManager.publishCameraStream | publishCameraStream()},
|
|
3092
|
+
* {@link AgentManager.setSttLanguage | setSttLanguage()} and
|
|
3093
|
+
* {@link AgentManager.sendDataChannelMessage | sendDataChannelMessage()} — raise it the other way
|
|
3094
|
+
* round: when the session is a Talks (V2) or Clips (V3) one, or
|
|
3095
|
+
* {@link AgentManager.connect | connect()} has not run yet.
|
|
3096
|
+
*
|
|
3097
|
+
* Nothing was sent to the Agents API when this is raised, so the session stays usable: fix the
|
|
3098
|
+
* argument or wait for {@link ConnectionState.Connected | 'connected'} and call again.
|
|
3099
|
+
* {@link BaseError.kind | kind} is `'ValidationError'`.
|
|
3100
|
+
*
|
|
3101
|
+
* @category Errors
|
|
3102
|
+
*/
|
|
3103
|
+
export declare class ValidationError extends BaseError {
|
|
3104
|
+
/**
|
|
3105
|
+
* Name of the field the failure is about.
|
|
3106
|
+
*
|
|
3107
|
+
* The SDK does not populate it — every validation it raises passes the message alone — so it
|
|
3108
|
+
* is only set on a `ValidationError` an application constructed itself. It is also left out
|
|
3109
|
+
* of {@link BaseError.toJson | toJson()}, so a serialized validation error carries only its
|
|
3110
|
+
* kind and message.
|
|
3111
|
+
*/
|
|
3112
|
+
readonly key?: string | undefined;
|
|
3113
|
+
/**
|
|
3114
|
+
* Always `'ValidationError'`. Branch on it to tell this failure from the other SDK errors.
|
|
3115
|
+
*/
|
|
3116
|
+
readonly kind: 'ValidationError';
|
|
3117
|
+
/**
|
|
3118
|
+
* Builds a validation failure. The SDK does this itself before it performs a request.
|
|
3119
|
+
*
|
|
3120
|
+
* @param message - What was wrong with the call, such as `'Message cannot be empty'`.
|
|
3121
|
+
* @param key - Name of the field the failure is about. The SDK's own validations leave it
|
|
3122
|
+
* unset.
|
|
3123
|
+
*/
|
|
3124
|
+
constructor(message: string,
|
|
3125
|
+
/**
|
|
3126
|
+
* Name of the field the failure is about.
|
|
3127
|
+
*
|
|
3128
|
+
* The SDK does not populate it — every validation it raises passes the message alone — so it
|
|
3129
|
+
* is only set on a `ValidationError` an application constructed itself. It is also left out
|
|
3130
|
+
* of {@link BaseError.toJson | toJson()}, so a serialized validation error carries only its
|
|
3131
|
+
* kind and message.
|
|
3132
|
+
*/
|
|
3133
|
+
key?: string | undefined);
|
|
3134
|
+
}
|
|
3135
|
+
|
|
3136
|
+
/**
|
|
3137
|
+
* One voice in D-ID's catalogue of text-to-speech voices.
|
|
3138
|
+
*
|
|
3139
|
+
* The SDK does not fetch voices itself; the type is exported so an application that lists them —
|
|
3140
|
+
* to build a voice picker, say — can type the result and then feed {@link Voice.id | id} and
|
|
3141
|
+
* {@link Voice.provider | provider} into the provider object it passes as
|
|
3142
|
+
* {@link TextStreamScript.provider}. {@link Voice.provider | provider} is the wide
|
|
3143
|
+
* {@link Providers} enum, while each provider variant requires its own literal `type` — so switch
|
|
3144
|
+
* on it to build the matching variant, for example {@link Providers.Elevenlabs} to an
|
|
3145
|
+
* {@link ElevenlabsTtsProvider}.
|
|
3146
|
+
*
|
|
3147
|
+
* @category Voice
|
|
3148
|
+
*/
|
|
3149
|
+
export declare interface Voice {
|
|
3150
|
+
/** Identifier of the voice: the value to pass as a provider object's `voice_id`. */
|
|
3151
|
+
id: string;
|
|
3152
|
+
/** Human-readable name of the voice. */
|
|
3153
|
+
name: string;
|
|
3154
|
+
/** The gender the voice is labelled with. */
|
|
3155
|
+
gender: string;
|
|
3156
|
+
/** The locale the voice speaks, such as `en-US`. */
|
|
3157
|
+
locale: string;
|
|
3158
|
+
/** Which accounts may use the voice. See {@link VoiceAccess}. */
|
|
3159
|
+
access: VoiceAccess;
|
|
3160
|
+
/** The provider that serves the voice. See {@link Providers}. */
|
|
3161
|
+
provider: Providers;
|
|
3162
|
+
/** The speaking styles this particular voice offers, for example as a
|
|
3163
|
+
* {@link VoiceConfigMicrosoft.style | style}. Styles differ from voice to voice. */
|
|
3164
|
+
styles: string[];
|
|
3165
|
+
/** The language of the voice, as a name rather than a locale code. */
|
|
3166
|
+
language: string;
|
|
3167
|
+
}
|
|
3168
|
+
|
|
3169
|
+
/**
|
|
3170
|
+
* Who may use a voice.
|
|
3171
|
+
*
|
|
3172
|
+
* The value of {@link Voice.access}, so an application listing voices can show or hide the ones the
|
|
3173
|
+
* account cannot select.
|
|
3174
|
+
*
|
|
3175
|
+
* @category Voice
|
|
3176
|
+
*/
|
|
3177
|
+
export declare enum VoiceAccess {
|
|
3178
|
+
/** As exposed by the voices API: available to every account. */
|
|
3179
|
+
Public = "public",
|
|
3180
|
+
/** As exposed by the voices API: available to accounts whose plan includes premium voices. */
|
|
3181
|
+
Premium = "premium",
|
|
3182
|
+
/** As exposed by the voices API: available only to the account the voice belongs to. */
|
|
3183
|
+
Private = "private"
|
|
3184
|
+
}
|
|
3185
|
+
|
|
3186
|
+
/**
|
|
3187
|
+
* How closely an ElevenLabs voice should follow the original it was built from.
|
|
3188
|
+
*
|
|
3189
|
+
* The `voice_config` of {@link ElevenlabsTtsProvider}, mirroring ElevenLabs' own voice settings.
|
|
3190
|
+
* Both fields are optional; when one is omitted the Agents API applies its own default.
|
|
3191
|
+
*
|
|
3192
|
+
* @category Voice
|
|
3193
|
+
*/
|
|
3194
|
+
export declare interface VoiceConfigElevenlabs {
|
|
3195
|
+
/**
|
|
3196
|
+
* How stable the voice is, and how much each generation varies.
|
|
3197
|
+
*
|
|
3198
|
+
* A number from 0 to 1: lower is more expressive and more variable between generations, higher
|
|
3199
|
+
* is flatter and more repeatable.
|
|
3200
|
+
* @example 0.5
|
|
3201
|
+
*/
|
|
3202
|
+
stability?: number;
|
|
3203
|
+
/**
|
|
3204
|
+
* How closely the synthesised speech should adhere to the original voice it was cloned from.
|
|
3205
|
+
*
|
|
3206
|
+
* A number from 0 to 1: higher follows the original more strictly.
|
|
3207
|
+
* @example 0.5
|
|
3208
|
+
*/
|
|
3209
|
+
similarity_boost?: number;
|
|
3210
|
+
}
|
|
3211
|
+
|
|
3212
|
+
/**
|
|
3213
|
+
* How a Microsoft Azure or Azure OpenAI voice should deliver the text.
|
|
3214
|
+
*
|
|
3215
|
+
* The `voice_config` of {@link MicrosoftTtsProvider} and {@link AzureOpenAiTtsProvider}. Every
|
|
3216
|
+
* field is optional; when one is omitted the Agents API applies its own default.
|
|
3217
|
+
*
|
|
3218
|
+
* @category Voice
|
|
3219
|
+
*/
|
|
3220
|
+
export declare interface VoiceConfigMicrosoft {
|
|
3221
|
+
/**
|
|
3222
|
+
* The style of the voice.
|
|
3223
|
+
* Available styles change between voices.
|
|
3224
|
+
*/
|
|
3225
|
+
style?: string;
|
|
3226
|
+
/**
|
|
3227
|
+
* The speed of the voice.
|
|
3228
|
+
* The value is relative to 1, 0.5 being half speed, 2 being twice as fast, etc.
|
|
3229
|
+
* Another option is a constant value from x-slow/slow/medium/fast/x-fast.
|
|
3230
|
+
* @example "0.5"
|
|
3231
|
+
*/
|
|
3232
|
+
rate?: string;
|
|
3233
|
+
/**
|
|
3234
|
+
* The pitch of the voice.
|
|
3235
|
+
* Value could be an absolute value in Hz (including units), a relative value in Hz or st(semitones)
|
|
3236
|
+
* or a constant value from x-low/low/medium/high/x-high.
|
|
3237
|
+
* @example "+2st"
|
|
3238
|
+
*/
|
|
3239
|
+
pitch?: string;
|
|
3240
|
+
}
|
|
3241
|
+
|
|
3242
|
+
/**
|
|
3243
|
+
* The web socket the SDK uses for chat notifications failed to connect.
|
|
3244
|
+
*
|
|
3245
|
+
* Talks (V2) and Clips (V3) sessions open a web socket alongside the video stream; the agent's
|
|
3246
|
+
* answer arrives over it as `partial` and `answer` events, which is what drives
|
|
3247
|
+
* {@link AgentManagerCallbacks.onNewMessage | onNewMessage}. This error is raised when that socket
|
|
3248
|
+
* emits an `error` event, once per failed attempt while
|
|
3249
|
+
* {@link AgentManager.connect | connect()} is retrying. It is only delivered to
|
|
3250
|
+
* {@link AgentManagerCallbacks.onError | onError} and never thrown, so there is nothing to catch
|
|
3251
|
+
* around `connect()` for it; if every attempt fails, `connect()` rejects with the browser's own
|
|
3252
|
+
* socket event instead.
|
|
3253
|
+
*
|
|
3254
|
+
* The socket is not opened at all for Expressive (V4) agents, which receive the same events over
|
|
3255
|
+
* the LiveKit data channel, nor in {@link ChatMode.DirectPlayback | DirectPlayback} mode, so the
|
|
3256
|
+
* error cannot occur in either case.
|
|
3257
|
+
*
|
|
3258
|
+
* {@link BaseError.kind | kind} is `'WSError'`, and the message is
|
|
3259
|
+
* `'Websocket failed to connect'`.
|
|
3260
|
+
*
|
|
3261
|
+
* @category Errors
|
|
3262
|
+
*/
|
|
3263
|
+
export declare class WsError extends BaseError {
|
|
3264
|
+
/**
|
|
3265
|
+
* Always `'WSError'` — note the capitalization, which does not match the class name.
|
|
3266
|
+
*/
|
|
3267
|
+
readonly kind: 'WSError';
|
|
3268
|
+
}
|
|
3269
|
+
|
|
3270
|
+
export { }
|