@d-id/client-sdk 3.0.0-staging.413 → 3.0.0-staging.414
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{index-BOeL7MEm.js → index-DBWnR72S.js} +1628 -1615
- package/dist/index.js +20 -20
- package/dist/index.umd.cjs +7 -7
- package/dist/livekit-manager-BM3JMzT5.js +531 -0
- package/dist/src/api/agents.d.ts +5 -5
- package/dist/src/api/streams/streamApi.test.d.ts +1 -0
- package/dist/src/index.d.ts +6 -6
- package/dist/src/services/streaming-manager/common.d.ts +4 -4
- package/dist/src/types/entities/agents/chat.d.ts +9 -9
- package/dist/src/types/entities/agents/manager.d.ts +31 -30
- package/dist/src/types/stream/data-channel.d.ts +2 -2
- package/dist/src/types/stream/rtc.d.ts +14 -3
- package/dist/src/types/stream/stream.d.ts +114 -43
- package/dist/src/types/stream-script.d.ts +5 -5
- package/dist/src/types/voice/stt.d.ts +2 -2
- package/dist/src/types/voice/tts.d.ts +1 -1
- package/package.json +1 -1
- package/dist/livekit-manager-D-B3qdMq.js +0 -506
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { SttTokenResponse } from '../..';
|
|
2
2
|
import { Auth } from '../../auth';
|
|
3
|
-
import { AgentActivityState, ClientToolHandler, CompatibilityMode, ConnectionState, ConnectivityState,
|
|
4
|
-
import {
|
|
3
|
+
import { AgentActivityState, ClientToolHandler, CompatibilityMode, ConnectionState, ConnectivityState, DataChannelTopic, SpeakResponse, StreamCreatedInfo, StreamEvents, StreamType, StreamingState } from '../../stream';
|
|
4
|
+
import { SpeakScript } from '../../stream-script';
|
|
5
5
|
import { StreamingManagerCallbacks as StreamManagerCallbacks } from '../../stream/stream';
|
|
6
6
|
import { Agent } from './agent';
|
|
7
|
-
import { ChatMode, ChatResponse,
|
|
7
|
+
import { ChatMode, ChatResponse, InterruptOptions, Message, Rating, SubmitFeedbackResponse } from './chat';
|
|
8
8
|
|
|
9
9
|
/**
|
|
10
10
|
* Types of events provided in Chat Progress Callback
|
|
@@ -159,7 +159,7 @@ export interface AgentManagerCallbacks {
|
|
|
159
159
|
* given, and again on every {@link AgentManager.connect | connect()} after the first — both
|
|
160
160
|
* with type `answer`. The array is a fresh copy on each call, oldest message first; every
|
|
161
161
|
* {@link Message} carries an `id`, a `role` of `user` or `assistant` (the agent), its `content`
|
|
162
|
-
* and a `
|
|
162
|
+
* and a `createdAt` timestamp.
|
|
163
163
|
*
|
|
164
164
|
* @param messages - The chat so far.
|
|
165
165
|
* @param type - `partial` while the agent's answer is still streaming in, `answer` for the full
|
|
@@ -253,15 +253,15 @@ export interface AgentManagerCallbacks {
|
|
|
253
253
|
* Fires before the connection reaches {@link ConnectionState.Connected | 'connected'}. The ids
|
|
254
254
|
* are useful when correlating a session with D-ID support or with your own logs.
|
|
255
255
|
*
|
|
256
|
-
* @param stream - The new stream's `
|
|
256
|
+
* @param stream - The new stream's `streamId`, `sessionId` and `agentId`.
|
|
257
257
|
*/
|
|
258
258
|
onStreamCreated?: (stream: StreamCreatedInfo) => void;
|
|
259
259
|
/**
|
|
260
260
|
* Called when the agent starts, finishes or fails a tool call.
|
|
261
261
|
*
|
|
262
262
|
* Expressive (V4) agents only. The handler takes two arguments and returns nothing: `event`,
|
|
263
|
-
* one of {@link
|
|
264
|
-
* {@link
|
|
263
|
+
* one of {@link ToolCallEvent.Started}, {@link ToolCallEvent.Done} or
|
|
264
|
+
* {@link ToolCallEvent.Error}; and `data`, the payload that event narrows to —
|
|
265
265
|
* {@link ToolCallStartedPayload}, {@link ToolCallDonePayload} or
|
|
266
266
|
* {@link ToolCallErrorPayload} respectively. The overloads that do the narrowing are on
|
|
267
267
|
* {@link ToolEventCallback}, with an example handler.
|
|
@@ -273,7 +273,7 @@ export interface AgentManagerCallbacks {
|
|
|
273
273
|
* Expressive (V4) agents only, and about whether interrupting is allowed *right now*: it goes
|
|
274
274
|
* `false` while a `blocking` client tool call is outstanding, because the agent is suspended
|
|
275
275
|
* waiting for it, and back to `true` when the call finishes. That is a different question from
|
|
276
|
-
* {@link AgentManager.
|
|
276
|
+
* {@link AgentManager.isInterruptAvailable | isInterruptAvailable()}, which says whether
|
|
277
277
|
* the session supports interrupting at all. Use this one to enable or disable an interrupt
|
|
278
278
|
* button. Talks (V2) and Clips (V3) agents never report a change.
|
|
279
279
|
*/
|
|
@@ -438,7 +438,7 @@ export interface AgentManagerOptions {
|
|
|
438
438
|
* Extra properties merged into every analytics event the SDK sends.
|
|
439
439
|
*
|
|
440
440
|
* A flat object — use it to tag events with your own identifiers. More properties can be added
|
|
441
|
-
* later with
|
|
441
|
+
* later with `enrichAnalytics()`.
|
|
442
442
|
*/
|
|
443
443
|
mixpanelAdditionalProperties?: Record<string, unknown>;
|
|
444
444
|
/**
|
|
@@ -533,7 +533,7 @@ export interface AgentManager {
|
|
|
533
533
|
*
|
|
534
534
|
* @returns `true` when {@link AgentManager.interrupt | interrupt()} can do anything.
|
|
535
535
|
*/
|
|
536
|
-
|
|
536
|
+
isInterruptAvailable(): boolean;
|
|
537
537
|
/**
|
|
538
538
|
* The agent's starter messages.
|
|
539
539
|
*
|
|
@@ -547,12 +547,12 @@ export interface AgentManager {
|
|
|
547
547
|
* The SDK sends the request whenever it is called, connected or not; the service decides
|
|
548
548
|
* whether to issue a token for the agent.
|
|
549
549
|
*
|
|
550
|
-
* @returns The {@link
|
|
550
|
+
* @returns The {@link SttTokenResponse} for this agent.
|
|
551
551
|
* @throws {@link HttpError} When the service does not answer with a token, or the request comes
|
|
552
552
|
* back non-2xx for any other reason.
|
|
553
553
|
* @throws {@link NetworkError} When the request never reaches the server.
|
|
554
554
|
*/
|
|
555
|
-
|
|
555
|
+
getSttToken(): Promise<SttTokenResponse>;
|
|
556
556
|
/**
|
|
557
557
|
* Opens a new session with the agent: a new WebRTC connection, a new web socket and a new chat.
|
|
558
558
|
*
|
|
@@ -701,24 +701,24 @@ export interface AgentManager {
|
|
|
701
701
|
* @param messageId - Id of the message being rated.
|
|
702
702
|
* @param score - 1 for a positive rating, -1 for a negative one.
|
|
703
703
|
* @param rateId - Id of an existing rating to update; omit to create a new one.
|
|
704
|
-
* @returns The created or updated {@link
|
|
704
|
+
* @returns The created or updated {@link Rating}.
|
|
705
705
|
* @throws {@link ValidationError} When no chat has started, or when no message with that id is
|
|
706
706
|
* in the transcript.
|
|
707
707
|
* @throws {@link HttpError} When the rating request comes back non-2xx.
|
|
708
708
|
* @throws {@link NetworkError} When the rating request never reaches the server.
|
|
709
709
|
*/
|
|
710
|
-
rate(messageId: string, score: 1 | -1, rateId?: string): Promise<
|
|
710
|
+
rate(messageId: string, score: 1 | -1, rateId?: string): Promise<Rating>;
|
|
711
711
|
/**
|
|
712
712
|
* Removes a rating the user gave to an answer in the chat.
|
|
713
713
|
*
|
|
714
714
|
* @param id - Id of the rating to remove, as returned by
|
|
715
715
|
* {@link AgentManager.rate | rate()}.
|
|
716
|
-
* @returns The {@link
|
|
716
|
+
* @returns The {@link Rating} that was deleted.
|
|
717
717
|
* @throws {@link ValidationError} When no chat has started.
|
|
718
718
|
* @throws {@link HttpError} When the delete request comes back non-2xx.
|
|
719
719
|
* @throws {@link NetworkError} When the delete request never reaches the server.
|
|
720
720
|
*/
|
|
721
|
-
deleteRate(id: string): Promise<
|
|
721
|
+
deleteRate(id: string): Promise<Rating>;
|
|
722
722
|
/**
|
|
723
723
|
* Submits end-of-call feedback for the whole conversation.
|
|
724
724
|
*
|
|
@@ -738,7 +738,7 @@ export interface AgentManager {
|
|
|
738
738
|
*
|
|
739
739
|
* Unlike {@link AgentManager.chat | chat()} the agent's LLM is not involved, so this is how you
|
|
740
740
|
* script greetings and canned lines. Pass a plain string as a shorthand for a text script. See
|
|
741
|
-
* {@link
|
|
741
|
+
* {@link SpeakScript}, {@link TextStreamScript} and {@link AudioStreamScript}. Text
|
|
742
742
|
* scripts also accept an optional `sentiment`, for Expressive (V4) agents only; if the
|
|
743
743
|
* requested sentiment is not supported by the agent, the default sentiment is used.
|
|
744
744
|
*
|
|
@@ -747,8 +747,8 @@ export interface AgentManager {
|
|
|
747
747
|
* @see [Control the Agent](https://docs.d-id.com/docs/livekit-commands) — the `did.speak` command
|
|
748
748
|
* sent over the data channel for Expressive (V4) agents.
|
|
749
749
|
* @param payload - A text or audio script, or a string treated as the text to speak.
|
|
750
|
-
* @returns The {@link
|
|
751
|
-
* response with `duration` `0` and an empty `
|
|
750
|
+
* @returns The {@link SpeakResponse} for the video that was produced, or the same
|
|
751
|
+
* response with `duration` `0` and an empty `videoId` when the call produced no discrete video
|
|
752
752
|
* — on Expressive (V4) agents, and in a text-only chat mode.
|
|
753
753
|
* @throws {@link ValidationError} When the manager is not connected to a stream yet.
|
|
754
754
|
* @throws {@link HttpError} On Talks (V2) and Clips (V3) agents, when the Agents API answers
|
|
@@ -779,7 +779,7 @@ export interface AgentManager {
|
|
|
779
779
|
* });
|
|
780
780
|
* ```
|
|
781
781
|
*/
|
|
782
|
-
speak(payload:
|
|
782
|
+
speak(payload: SpeakScript | string): Promise<SpeakResponse>;
|
|
783
783
|
/**
|
|
784
784
|
* Switches the chat to another mode.
|
|
785
785
|
*
|
|
@@ -804,6 +804,7 @@ export interface AgentManager {
|
|
|
804
804
|
*
|
|
805
805
|
* @param properties - A flat JSON object whose properties are added to every analytics event
|
|
806
806
|
* the SDK sends from now on.
|
|
807
|
+
* @internal Used by D-ID's own embedded widget; not part of the public SDK surface.
|
|
807
808
|
*/
|
|
808
809
|
enrichAnalytics(properties: Record<string, unknown>): void;
|
|
809
810
|
/**
|
|
@@ -813,7 +814,7 @@ export interface AgentManager {
|
|
|
813
814
|
* (V4) agent. It never throws: on every agent type it returns without doing anything when
|
|
814
815
|
* interrupting is not available for the session, when it is not allowed right now, and — on
|
|
815
816
|
* Talks (V2) and Clips (V3) agents — when the stream is not a fluent stream or no video is
|
|
816
|
-
* playing. Check {@link AgentManager.
|
|
817
|
+
* playing. Check {@link AgentManager.isInterruptAvailable | isInterruptAvailable()}
|
|
817
818
|
* before offering the control at all; on Expressive (V4) agents
|
|
818
819
|
* {@link AgentManagerCallbacks.onInterruptibleChange | onInterruptibleChange} tracks whether it
|
|
819
820
|
* is allowed right now.
|
|
@@ -823,11 +824,11 @@ export interface AgentManager {
|
|
|
823
824
|
* actually sent. A call that finds nothing to interrupt leaves the message untouched and fires
|
|
824
825
|
* no callback.
|
|
825
826
|
*
|
|
826
|
-
* @param
|
|
827
|
-
* `click` or `manual`. Expressive (V4) agents drop `text` interrupts, because the
|
|
828
|
-
* does not cancel the in-flight answer for them.
|
|
827
|
+
* @param options - What caused the interruption, as an {@link InterruptOptions}: `text`,
|
|
828
|
+
* `audio`, `click` or `manual`. Expressive (V4) agents drop `text` interrupts, because the
|
|
829
|
+
* orchestrator does not cancel the in-flight answer for them.
|
|
829
830
|
*/
|
|
830
|
-
interrupt(
|
|
831
|
+
interrupt(options: InterruptOptions): void;
|
|
831
832
|
/**
|
|
832
833
|
* Switches the speech-to-text language in the middle of a session.
|
|
833
834
|
*
|
|
@@ -850,7 +851,7 @@ export interface AgentManager {
|
|
|
850
851
|
* to change slide. Expressive (V4) agents only, after {@link AgentManager.connect | connect()};
|
|
851
852
|
* otherwise the returned promise rejects with a {@link ValidationError}.
|
|
852
853
|
*
|
|
853
|
-
* @param topic - Data-channel topic to send on. {@link
|
|
854
|
+
* @param topic - Data-channel topic to send on. {@link DataChannelTopic} is exported from
|
|
854
855
|
* the package root and lists every topic this method accepts.
|
|
855
856
|
* @param payload - A plain object, sent as JSON.
|
|
856
857
|
* @returns Resolves once the payload has been sent. A room that has dropped since
|
|
@@ -860,15 +861,15 @@ export interface AgentManager {
|
|
|
860
861
|
* {@link AgentManager.connect | connect()} has not run yet.
|
|
861
862
|
* @example
|
|
862
863
|
* ```ts
|
|
863
|
-
* import {
|
|
864
|
+
* import { DataChannelTopic } from '@d-id/client-sdk';
|
|
864
865
|
*
|
|
865
|
-
* await agentManager.sendDataChannelMessage(
|
|
866
|
+
* await agentManager.sendDataChannelMessage(DataChannelTopic.Presentation, {
|
|
866
867
|
* type: 'navigate',
|
|
867
868
|
* slide: 3,
|
|
868
869
|
* });
|
|
869
870
|
* ```
|
|
870
871
|
*/
|
|
871
|
-
sendDataChannelMessage(topic:
|
|
872
|
+
sendDataChannelMessage(topic: DataChannelTopic, payload: Record<string, unknown>): Promise<void>;
|
|
872
873
|
/**
|
|
873
874
|
* Registers a handler for a client tool, run in the browser when the agent's LLM calls it.
|
|
874
875
|
*
|
|
@@ -5,9 +5,9 @@
|
|
|
5
5
|
* Internal on purpose: this module is deliberately left out of the `stream`
|
|
6
6
|
* barrel so the topics driven by dedicated methods (`chat`, `speak`,
|
|
7
7
|
* `interrupt`, `setSttLanguage`) never reach the package's public API.
|
|
8
|
-
* Customers get `
|
|
8
|
+
* Customers get the public `DataChannelTopic` enum instead.
|
|
9
9
|
*/
|
|
10
|
-
export declare enum
|
|
10
|
+
export declare enum InternalDataChannelTopic {
|
|
11
11
|
Chat = "lk.chat",
|
|
12
12
|
Speak = "did.speak",
|
|
13
13
|
Interrupt = "did.interrupt",
|
|
@@ -79,14 +79,14 @@ export interface Status {
|
|
|
79
79
|
*
|
|
80
80
|
* The fields come from the Talks (V2) and Clips (V3) API, which generates a discrete video while
|
|
81
81
|
* answering the request. Where no such video exists the call still resolves, with the same stub —
|
|
82
|
-
* `status` `'success'`, `duration` `0` and an empty `
|
|
82
|
+
* `status` `'success'`, `duration` `0` and an empty `videoId`: on Expressive (V4) agents, whose
|
|
83
83
|
* speech is streamed over the data channel rather than rendered as a separate video, and in a
|
|
84
84
|
* text-only chat mode ({@link ChatMode.TextOnly}, {@link ChatMode.Playground} or
|
|
85
85
|
* {@link ChatMode.Maintenance}), which produces no video at all.
|
|
86
86
|
*
|
|
87
87
|
* @category Speak & Scripts
|
|
88
88
|
*/
|
|
89
|
-
export interface
|
|
89
|
+
export interface SpeakResponse {
|
|
90
90
|
/**
|
|
91
91
|
* Whether the server accepted the speak request — `'success'` when it did.
|
|
92
92
|
*/
|
|
@@ -94,7 +94,7 @@ export interface SendStreamPayloadResponse {
|
|
|
94
94
|
/**
|
|
95
95
|
* Id of the session this call was made on; the SDK sends it back on later streaming requests.
|
|
96
96
|
*/
|
|
97
|
-
|
|
97
|
+
sessionId?: string;
|
|
98
98
|
/**
|
|
99
99
|
* Duration of the generated video as reported by the Agents API for Talks (V2) and Clips (V3)
|
|
100
100
|
* agents. `0` when the call produced no discrete video — on Expressive (V4) agents, and in a
|
|
@@ -109,6 +109,17 @@ export interface SendStreamPayloadResponse {
|
|
|
109
109
|
* end of, and which {@link AgentManager.interrupt | interrupt()} cancels. Empty when the call
|
|
110
110
|
* produced no discrete video — on Expressive (V4) agents, and in a text-only chat mode.
|
|
111
111
|
*/
|
|
112
|
+
videoId: string;
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* What the Agents API answers a speak request with, converted to {@link SpeakResponse} before it
|
|
116
|
+
* reaches the application.
|
|
117
|
+
* @internal Wire type of the streaming transport; not part of the public SDK surface.
|
|
118
|
+
*/
|
|
119
|
+
export interface SpeakWireResponse {
|
|
120
|
+
status: string;
|
|
121
|
+
session_id?: string;
|
|
122
|
+
duration: number;
|
|
112
123
|
video_id: string;
|
|
113
124
|
}
|
|
114
125
|
export {};
|
|
@@ -3,7 +3,7 @@ import { VideoRTCStatsReport } from '../../services/streaming-manager/stats/repo
|
|
|
3
3
|
import { Auth } from '../auth';
|
|
4
4
|
import { ChatProgressCallback } from '../entities/agents/manager';
|
|
5
5
|
import { CreateClipStreamRequest, CreateTalkStreamRequest, SendClipStreamPayload, SendTalkStreamPayload } from './api';
|
|
6
|
-
import { ICreateStreamRequestResponse, IceCandidate,
|
|
6
|
+
import { ICreateStreamRequestResponse, IceCandidate, SpeakResponse, Status } from './rtc';
|
|
7
7
|
|
|
8
8
|
/**
|
|
9
9
|
* Which video codec the stream should negotiate.
|
|
@@ -83,14 +83,11 @@ export declare enum AgentActivityState {
|
|
|
83
83
|
ToolActive = "TOOL_ACTIVE"
|
|
84
84
|
}
|
|
85
85
|
/**
|
|
86
|
-
*
|
|
87
|
-
* {@link AgentManagerCallbacks.onToolEvent | onToolEvent}.
|
|
86
|
+
* Every event the SDK receives on a stream's data channel or web socket.
|
|
88
87
|
*
|
|
89
|
-
*
|
|
90
|
-
* {@link
|
|
91
|
-
*
|
|
92
|
-
*
|
|
93
|
-
* @category Callbacks & Events
|
|
88
|
+
* The three tool-call members are the only ones an application ever sees, and they reach it as
|
|
89
|
+
* {@link ToolCallEvent} instead; the rest are consumed by the SDK and surfaced as callbacks.
|
|
90
|
+
* @internal Implementation type; not part of the public SDK surface.
|
|
94
91
|
*/
|
|
95
92
|
export declare enum StreamEvents {
|
|
96
93
|
/**
|
|
@@ -192,6 +189,45 @@ export declare enum StreamEvents {
|
|
|
192
189
|
*/
|
|
193
190
|
TurnEnded = "turn/ended"
|
|
194
191
|
}
|
|
192
|
+
/**
|
|
193
|
+
* The tool-call events delivered to {@link AgentManagerCallbacks.onToolEvent | onToolEvent}.
|
|
194
|
+
*
|
|
195
|
+
* Switch on the first argument of the handler to tell them apart: it narrows the second argument to
|
|
196
|
+
* the payload that event carries — see {@link ToolEventCallback}. The members' values are the wire
|
|
197
|
+
* strings the server sends, so compare against the enum rather than writing the string out.
|
|
198
|
+
* Expressive (V4) agents only.
|
|
199
|
+
*
|
|
200
|
+
* @example
|
|
201
|
+
* ```ts
|
|
202
|
+
* import { ToolCallEvent } from '@d-id/client-sdk';
|
|
203
|
+
*
|
|
204
|
+
* const callbacks = {
|
|
205
|
+
* onToolEvent(event, data) {
|
|
206
|
+
* if (event === ToolCallEvent.Started) {
|
|
207
|
+
* console.log('started', data.name, data.input);
|
|
208
|
+
* }
|
|
209
|
+
* },
|
|
210
|
+
* };
|
|
211
|
+
* ```
|
|
212
|
+
* @category Callbacks & Events
|
|
213
|
+
*/
|
|
214
|
+
export declare enum ToolCallEvent {
|
|
215
|
+
/**
|
|
216
|
+
* The agent has begun a tool call; the payload is a {@link ToolCallStartedPayload}, carrying the
|
|
217
|
+
* call's id, the tool's name and the arguments the agent's LLM produced.
|
|
218
|
+
*/
|
|
219
|
+
Started = "tool-call/started",
|
|
220
|
+
/**
|
|
221
|
+
* A tool call finished successfully; the payload is a {@link ToolCallDonePayload}, carrying the
|
|
222
|
+
* result the tool returned and how long the call took.
|
|
223
|
+
*/
|
|
224
|
+
Done = "tool-call/done",
|
|
225
|
+
/**
|
|
226
|
+
* A tool call failed; the payload is a {@link ToolCallErrorPayload}, carrying whatever the
|
|
227
|
+
* server reported about the failure. The agent carries on with the conversation.
|
|
228
|
+
*/
|
|
229
|
+
Error = "tool-call/error"
|
|
230
|
+
}
|
|
195
231
|
/**
|
|
196
232
|
* The data-channel topics an application may send on with
|
|
197
233
|
* {@link AgentManager.sendDataChannelMessage | sendDataChannelMessage()}.
|
|
@@ -207,16 +243,16 @@ export declare enum StreamEvents {
|
|
|
207
243
|
*
|
|
208
244
|
* @example
|
|
209
245
|
* ```ts
|
|
210
|
-
* import {
|
|
246
|
+
* import { DataChannelTopic } from '@d-id/client-sdk';
|
|
211
247
|
*
|
|
212
|
-
* await agentManager.sendDataChannelMessage(
|
|
248
|
+
* await agentManager.sendDataChannelMessage(DataChannelTopic.Presentation, {
|
|
213
249
|
* type: 'navigate',
|
|
214
250
|
* slide: 3,
|
|
215
251
|
* });
|
|
216
252
|
* ```
|
|
217
253
|
* @category Agent Manager
|
|
218
254
|
*/
|
|
219
|
-
export declare enum
|
|
255
|
+
export declare enum DataChannelTopic {
|
|
220
256
|
/**
|
|
221
257
|
* Messages that drive a presentation the agent is showing alongside its video, such as moving
|
|
222
258
|
* to another slide. Sent on the wire as `did.presentation`; the payload shape is whatever the
|
|
@@ -316,16 +352,17 @@ export interface StreamCreatedInfo {
|
|
|
316
352
|
/**
|
|
317
353
|
* Id of the agent the stream was opened for.
|
|
318
354
|
*/
|
|
319
|
-
|
|
355
|
+
agentId: string;
|
|
320
356
|
/**
|
|
321
357
|
* Id of the session; the SDK sends it back on every subsequent request for this stream.
|
|
322
|
-
* On Expressive (V4) agents it is the same value as
|
|
358
|
+
* On Expressive (V4) agents it is the same value as
|
|
359
|
+
* {@link StreamCreatedInfo.streamId | streamId}.
|
|
323
360
|
*/
|
|
324
|
-
|
|
361
|
+
sessionId: string;
|
|
325
362
|
/**
|
|
326
363
|
* Id of the stream itself.
|
|
327
364
|
*/
|
|
328
|
-
|
|
365
|
+
streamId: string;
|
|
329
366
|
}
|
|
330
367
|
/**
|
|
331
368
|
* Callback set consumed by the streaming managers (WebRTC and LiveKit).
|
|
@@ -406,7 +443,7 @@ export interface RtcApi {
|
|
|
406
443
|
createStream(options: CreateStreamOptions, signal?: AbortSignal): Promise<ICreateStreamRequestResponse>;
|
|
407
444
|
startConnection(streamId: string, answer: RTCSessionDescriptionInit, sessionId?: string, signal?: AbortSignal): Promise<Status>;
|
|
408
445
|
addIceCandidate(streamId: string, candidate: IceCandidate, sessionId: string, signal?: AbortSignal): Promise<Status>;
|
|
409
|
-
sendStreamRequest(streamId: string, sessionId: string, payload: SendClipStreamPayload | SendTalkStreamPayload): Promise<
|
|
446
|
+
sendStreamRequest(streamId: string, sessionId: string, payload: SendClipStreamPayload | SendTalkStreamPayload): Promise<SpeakResponse>;
|
|
410
447
|
close(streamId: string, sessionId: string): Promise<Status>;
|
|
411
448
|
}
|
|
412
449
|
/**
|
|
@@ -540,7 +577,7 @@ export type ClientToolHandler = (args: Record<string, unknown>) => Promise<strin
|
|
|
540
577
|
* Whether the agent waits for a tool call to finish before it carries on.
|
|
541
578
|
*
|
|
542
579
|
* Set on the agent's tool configuration, and reported on {@link RunningToolCall.executionMode} and
|
|
543
|
-
* {@link ToolCallStartedPayload.
|
|
580
|
+
* {@link ToolCallStartedPayload.executionMode}. Only a `blocking` call suspends the agent, which
|
|
544
581
|
* is why one being outstanding is what makes the agent uninterruptible — see
|
|
545
582
|
* {@link AgentManagerCallbacks.onInterruptibleChange | onInterruptibleChange}.
|
|
546
583
|
*
|
|
@@ -555,14 +592,13 @@ export type ToolExecutionMode = 'blocking' | 'async';
|
|
|
555
592
|
* when it starts and disappears when it finishes, fails, or — for a `blocking` call — when its turn
|
|
556
593
|
* ends; an `async` call outlives its turn.
|
|
557
594
|
*
|
|
558
|
-
* The fields are camelCase because this is the SDK's own view of a call; the same call arrives in
|
|
559
|
-
* {@link AgentManagerCallbacks.onToolEvent | onToolEvent} as a raw server payload, where the
|
|
560
|
-
* equivalent fields are snake_case — `callId` here is `call_id` there.
|
|
561
|
-
*
|
|
562
595
|
* @category Callbacks & Events
|
|
563
596
|
*/
|
|
564
597
|
export interface RunningToolCall {
|
|
565
|
-
/**
|
|
598
|
+
/**
|
|
599
|
+
* Id of this call, matching the {@link ToolCallStartedPayload.callId | callId} of the
|
|
600
|
+
* {@link ToolCallStartedPayload} that announced it.
|
|
601
|
+
*/
|
|
566
602
|
callId: string;
|
|
567
603
|
/** Name of the tool being called, as configured on the agent. */
|
|
568
604
|
name: string;
|
|
@@ -575,7 +611,7 @@ export interface RunningToolCall {
|
|
|
575
611
|
executionMode: ToolExecutionMode;
|
|
576
612
|
}
|
|
577
613
|
/**
|
|
578
|
-
* The payload of a {@link
|
|
614
|
+
* The payload of a {@link ToolCallEvent.Started} event: the agent has begun a tool call.
|
|
579
615
|
*
|
|
580
616
|
* Delivered to {@link AgentManagerCallbacks.onToolEvent | onToolEvent} — see
|
|
581
617
|
* {@link ToolEventCallback}.
|
|
@@ -584,7 +620,7 @@ export interface RunningToolCall {
|
|
|
584
620
|
*/
|
|
585
621
|
export interface ToolCallStartedPayload {
|
|
586
622
|
/** Id of this call. The matching done or error payload carries the same id. */
|
|
587
|
-
|
|
623
|
+
callId: string;
|
|
588
624
|
/** Name of the tool the agent is calling, as configured on the agent. */
|
|
589
625
|
name: string;
|
|
590
626
|
/** The arguments the agent's LLM produced for this call. */
|
|
@@ -596,7 +632,7 @@ export interface ToolCallStartedPayload {
|
|
|
596
632
|
*
|
|
597
633
|
* Informational: the SDK does not forward it.
|
|
598
634
|
* {@link AgentManagerCallbacks.onInterruptibleChange | onInterruptibleChange} is derived from
|
|
599
|
-
* the {@link ToolCallStartedPayload.
|
|
635
|
+
* the {@link ToolCallStartedPayload.executionMode | executionMode} of the calls still
|
|
600
636
|
* running, not from this field.
|
|
601
637
|
*/
|
|
602
638
|
interruptible: boolean;
|
|
@@ -604,14 +640,14 @@ export interface ToolCallStartedPayload {
|
|
|
604
640
|
* Whether the agent waits for this call. See {@link ToolExecutionMode}; anything other than
|
|
605
641
|
* `async` is treated as `blocking`.
|
|
606
642
|
*/
|
|
607
|
-
|
|
643
|
+
executionMode?: ToolExecutionMode;
|
|
608
644
|
/** The conversational turn this call belongs to, or `null` when it belongs to no turn. */
|
|
609
|
-
|
|
645
|
+
turnId?: number | null;
|
|
610
646
|
/** When the call started, as reported by the server. */
|
|
611
647
|
timestamp: string;
|
|
612
648
|
}
|
|
613
649
|
/**
|
|
614
|
-
* The payload of a {@link
|
|
650
|
+
* The payload of a {@link ToolCallEvent.Done} event: a tool call finished successfully.
|
|
615
651
|
*
|
|
616
652
|
* Delivered to {@link AgentManagerCallbacks.onToolEvent | onToolEvent} — see
|
|
617
653
|
* {@link ToolEventCallback}.
|
|
@@ -620,7 +656,7 @@ export interface ToolCallStartedPayload {
|
|
|
620
656
|
*/
|
|
621
657
|
export interface ToolCallDonePayload {
|
|
622
658
|
/** Id of the call that finished, matching the {@link ToolCallStartedPayload} that announced it. */
|
|
623
|
-
|
|
659
|
+
callId: string;
|
|
624
660
|
/** Name of the tool that was called. */
|
|
625
661
|
name: string;
|
|
626
662
|
/** The arguments the call was made with. */
|
|
@@ -628,14 +664,14 @@ export interface ToolCallDonePayload {
|
|
|
628
664
|
/** The result the tool returned. */
|
|
629
665
|
output: Record<string, unknown>;
|
|
630
666
|
/** How long the call took, in milliseconds. */
|
|
631
|
-
|
|
667
|
+
durationMs: number;
|
|
632
668
|
/** Any additional metadata the tool reported alongside its result. */
|
|
633
669
|
extra: Record<string, unknown>;
|
|
634
670
|
/** When the call finished, as reported by the server. */
|
|
635
671
|
timestamp: string;
|
|
636
672
|
}
|
|
637
673
|
/**
|
|
638
|
-
* The payload of a {@link
|
|
674
|
+
* The payload of a {@link ToolCallEvent.Error} event: a tool call failed.
|
|
639
675
|
*
|
|
640
676
|
* Delivered to {@link AgentManagerCallbacks.onToolEvent | onToolEvent} — see
|
|
641
677
|
* {@link ToolEventCallback}. The agent carries on with the conversation; the SDK does not retry.
|
|
@@ -644,7 +680,7 @@ export interface ToolCallDonePayload {
|
|
|
644
680
|
*/
|
|
645
681
|
export interface ToolCallErrorPayload {
|
|
646
682
|
/** Id of the call that failed, matching the {@link ToolCallStartedPayload} that announced it. */
|
|
647
|
-
|
|
683
|
+
callId: string;
|
|
648
684
|
/** Name of the tool that was called. */
|
|
649
685
|
name: string;
|
|
650
686
|
/** The arguments the call was made with. */
|
|
@@ -652,7 +688,7 @@ export interface ToolCallErrorPayload {
|
|
|
652
688
|
/** Whatever the failed call produced, if anything. */
|
|
653
689
|
output: Record<string, unknown>;
|
|
654
690
|
/** How long the call ran before failing, in milliseconds. */
|
|
655
|
-
|
|
691
|
+
durationMs: number;
|
|
656
692
|
/** Any additional metadata the server reported with the failure. */
|
|
657
693
|
extra: Record<string, unknown>;
|
|
658
694
|
/** When the call failed, as reported by the server. */
|
|
@@ -663,6 +699,41 @@ export interface ToolCallErrorPayload {
|
|
|
663
699
|
* @internal Implementation type; not part of the public SDK surface.
|
|
664
700
|
*/
|
|
665
701
|
export type ToolEventPayload = ToolCallStartedPayload | ToolCallDonePayload | ToolCallErrorPayload;
|
|
702
|
+
/**
|
|
703
|
+
* Data-channel wire shape of a `tool-call/started` event, converted to
|
|
704
|
+
* {@link ToolCallStartedPayload} before it reaches the application.
|
|
705
|
+
* @internal Wire type of the streaming transport; not part of the public SDK surface.
|
|
706
|
+
*/
|
|
707
|
+
export interface ToolCallStartedWirePayload {
|
|
708
|
+
call_id: string;
|
|
709
|
+
name: string;
|
|
710
|
+
input: Record<string, unknown>;
|
|
711
|
+
output: Record<string, unknown>;
|
|
712
|
+
interruptible?: boolean;
|
|
713
|
+
execution_mode?: ToolExecutionMode;
|
|
714
|
+
turn_id?: number | null;
|
|
715
|
+
timestamp: string;
|
|
716
|
+
}
|
|
717
|
+
/**
|
|
718
|
+
* Data-channel wire shape of a `tool-call/done` event, converted to {@link ToolCallDonePayload}
|
|
719
|
+
* before it reaches the application.
|
|
720
|
+
* @internal Wire type of the streaming transport; not part of the public SDK surface.
|
|
721
|
+
*/
|
|
722
|
+
export interface ToolCallDoneWirePayload {
|
|
723
|
+
call_id: string;
|
|
724
|
+
name: string;
|
|
725
|
+
input: Record<string, unknown>;
|
|
726
|
+
output: Record<string, unknown>;
|
|
727
|
+
duration_ms: number;
|
|
728
|
+
extra: Record<string, unknown>;
|
|
729
|
+
timestamp: string;
|
|
730
|
+
}
|
|
731
|
+
/**
|
|
732
|
+
* Data-channel wire shape of a `tool-call/error` event, converted to {@link ToolCallErrorPayload}
|
|
733
|
+
* before it reaches the application.
|
|
734
|
+
* @internal Wire type of the streaming transport; not part of the public SDK surface.
|
|
735
|
+
*/
|
|
736
|
+
export type ToolCallErrorWirePayload = ToolCallDoneWirePayload;
|
|
666
737
|
/**
|
|
667
738
|
* Data-channel payload identifying the conversational turn a `turn/started` or `turn/ended` event belongs to.
|
|
668
739
|
* @internal Implementation type; not part of the public SDK surface.
|
|
@@ -750,14 +821,14 @@ export declare enum StreamEndReason {
|
|
|
750
821
|
*
|
|
751
822
|
* @example
|
|
752
823
|
* ```ts
|
|
753
|
-
* import {
|
|
824
|
+
* import { ToolCallEvent } from '@d-id/client-sdk';
|
|
754
825
|
*
|
|
755
826
|
* const callbacks = {
|
|
756
827
|
* onToolEvent(event, data) {
|
|
757
|
-
* if (event ===
|
|
828
|
+
* if (event === ToolCallEvent.Started) {
|
|
758
829
|
* console.log('started', data.name, data.input);
|
|
759
|
-
* } else if (event ===
|
|
760
|
-
* console.log('done', data.name, data.output, data.
|
|
830
|
+
* } else if (event === ToolCallEvent.Done) {
|
|
831
|
+
* console.log('done', data.name, data.output, data.durationMs);
|
|
761
832
|
* } else {
|
|
762
833
|
* console.log('failed', data.name, data.extra);
|
|
763
834
|
* }
|
|
@@ -768,18 +839,18 @@ export declare enum StreamEndReason {
|
|
|
768
839
|
*/
|
|
769
840
|
export type ToolEventCallback = {
|
|
770
841
|
/**
|
|
771
|
-
* @param event - Always {@link
|
|
842
|
+
* @param event - Always {@link ToolCallEvent.Started} in this overload.
|
|
772
843
|
* @param data - The call the agent has just begun, with the arguments its LLM produced.
|
|
773
844
|
*/
|
|
774
|
-
(event:
|
|
845
|
+
(event: ToolCallEvent.Started, data: ToolCallStartedPayload): void;
|
|
775
846
|
/**
|
|
776
|
-
* @param event - Always {@link
|
|
847
|
+
* @param event - Always {@link ToolCallEvent.Done} in this overload.
|
|
777
848
|
* @param data - The call that has just finished, with the result the tool returned.
|
|
778
849
|
*/
|
|
779
|
-
(event:
|
|
850
|
+
(event: ToolCallEvent.Done, data: ToolCallDonePayload): void;
|
|
780
851
|
/**
|
|
781
|
-
* @param event - Always {@link
|
|
852
|
+
* @param event - Always {@link ToolCallEvent.Error} in this overload.
|
|
782
853
|
* @param data - The call that has just failed, with whatever the server reported about it.
|
|
783
854
|
*/
|
|
784
|
-
(event:
|
|
855
|
+
(event: ToolCallEvent.Error, data: ToolCallErrorPayload): void;
|
|
785
856
|
};
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { Message } from './entities';
|
|
2
|
-
import {
|
|
2
|
+
import { TtsProvider } from './voice/tts';
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
5
|
* A script that makes the agent say text you supply, synthesised by a text-to-speech provider.
|
|
@@ -38,12 +38,12 @@ export interface TextStreamScript {
|
|
|
38
38
|
* The text-to-speech provider and voice that synthesise
|
|
39
39
|
* {@link TextStreamScript.input | input}, from the list of supported providers.
|
|
40
40
|
*
|
|
41
|
-
* One of the objects in {@link
|
|
41
|
+
* One of the objects in {@link TtsProvider}: a `type` naming the provider, the
|
|
42
42
|
* `voice_id` to speak with, and optional provider-specific `voice_config`. Leave it out and the
|
|
43
43
|
* SDK sends the script without a provider, so the voice is chosen server-side; the Agents API
|
|
44
44
|
* documents Microsoft TTS as its default when no provider is given.
|
|
45
45
|
*/
|
|
46
|
-
provider?:
|
|
46
|
+
provider?: TtsProvider;
|
|
47
47
|
/**
|
|
48
48
|
* The text to be synthesised into speech.
|
|
49
49
|
*
|
|
@@ -111,7 +111,7 @@ export interface AudioStreamScript {
|
|
|
111
111
|
*/
|
|
112
112
|
export interface LlmStreamScript {
|
|
113
113
|
type: 'llm';
|
|
114
|
-
provider:
|
|
114
|
+
provider: TtsProvider;
|
|
115
115
|
ssml?: boolean;
|
|
116
116
|
llm: {
|
|
117
117
|
messages: Message[];
|
|
@@ -135,4 +135,4 @@ export type StreamScript = TextStreamScript | AudioStreamScript | LlmStreamScrip
|
|
|
135
135
|
*
|
|
136
136
|
* @category Speak & Scripts
|
|
137
137
|
*/
|
|
138
|
-
export type
|
|
138
|
+
export type SpeakScript = TextStreamScript | AudioStreamScript;
|
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* A short-lived credential for the D-ID speech-to-text service.
|
|
3
3
|
*
|
|
4
|
-
* What {@link AgentManager.
|
|
4
|
+
* What {@link AgentManager.getSttToken | getSttToken()} resolves with, fetched fresh from the
|
|
5
5
|
* Agents API each time. It lets an application drive its own speech recognition against the
|
|
6
6
|
* service the agent uses without putting D-ID credentials in the browser. The token expires, so request one per recognition session
|
|
7
7
|
* rather than holding on to it.
|
|
8
8
|
*
|
|
9
9
|
* @category Agent Manager
|
|
10
10
|
*/
|
|
11
|
-
export interface
|
|
11
|
+
export interface SttTokenResponse {
|
|
12
12
|
/** The authorization token to present to the speech service. */
|
|
13
13
|
token: string;
|
|
14
14
|
/** The speech service region the token is valid in. */
|
|
@@ -235,4 +235,4 @@ export interface VoiceConfigElevenlabs {
|
|
|
235
235
|
* ```
|
|
236
236
|
* @category Voice
|
|
237
237
|
*/
|
|
238
|
-
export type
|
|
238
|
+
export type TtsProvider = MicrosoftTtsProvider | AzureOpenAiTtsProvider | ElevenlabsTtsProvider | AmazonTtsProvider;
|