@d-id/client-sdk 3.0.0-staging.419 → 3.0.0-staging.420

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (10) hide show
  1. package/README.md +4 -4
  2. package/dist/{index-D-HEM_62.js → index-BrPt8AuN.js} +5 -5
  3. package/dist/index-BrPt8AuN.js.map +1 -0
  4. package/dist/index.d.ts +172 -199
  5. package/dist/index.js +1 -1
  6. package/dist/index.umd.cjs +1 -1
  7. package/dist/{livekit-manager-BY7TXE80.js → livekit-manager-me8pKdKj.js} +2 -2
  8. package/dist/{livekit-manager-BY7TXE80.js.map → livekit-manager-me8pKdKj.js.map} +1 -1
  9. package/package.json +1 -1
  10. package/dist/index-D-HEM_62.js.map +0 -1
package/dist/index.d.ts CHANGED
@@ -256,8 +256,10 @@ export declare interface AgentManager {
256
256
  /**
257
257
  * Returns whether the current stream supports interrupting the agent mid-answer.
258
258
  *
259
- * True on a fluent stream — a Clips (V3) agent built on a Pro avatar, or any Expressive (V4)
260
- * agent — once connected. This says the stream supports interrupting at all;
259
+ * `true` once connected on a fluent stream (every Expressive (V4) session, or a fluent Clips
260
+ * (V3) one) whose agent has not turned interrupting off in its advanced settings. It reads the
261
+ * session {@link AgentManager.connect | connect()} attached, so ask it after `connect()`
262
+ * resolves. This says the stream supports interrupting at all;
261
263
  * {@link AgentManagerCallbacks.onInterruptibleChange | onInterruptibleChange} says whether
262
264
  * interrupting is allowed right now.
263
265
  *
@@ -287,14 +289,14 @@ export declare interface AgentManager {
287
289
  */
288
290
  readonly starterMessages: readonly string[];
289
291
  /**
290
- * Fetches a short-lived token for the D-ID speech-to-text service.
292
+ * Fetches a short-lived Azure Speech token, for running your own speech recognition.
291
293
  *
292
- * The SDK sends the request whenever it is called, connected or not; the service decides
293
- * whether to issue a token for the agent.
294
+ * Works with every {@link Auth} type, connected or not. The agent must be public or belong to
295
+ * the account the credentials are for.
294
296
  *
295
- * @returns The {@link SttTokenResponse} for this agent.
296
- * @throws {@link HttpError} When the service does not answer with a token, or the request comes
297
- * back non-2xx for any other reason.
297
+ * @returns The {@link SttTokenResponse}: the token and its Azure region.
298
+ * @throws {@link HttpError} `400` when the agent is neither public nor the caller's, `404` when
299
+ * it does not exist, or any other non-2xx answer.
298
300
  * @throws {@link NetworkError} When the request never reaches the server.
299
301
  * @example
300
302
  * ```ts
@@ -365,10 +367,13 @@ export declare interface AgentManager {
365
367
  * with the media stream to render.
366
368
  *
367
369
  * One session at a time: while a call is still in flight a second call returns that same
368
- * promise rather than opening a second session, which is what makes it safe in a React
369
- * StrictMode effect. Once a session exists it rejects instead — call
370
- * {@link AgentManager.disconnect | disconnect()} first to start a fresh conversation, or
371
- * {@link AgentManager.reconnect | reconnect()} to keep the current one. A call that joins an
370
+ * promise rather than opening a second session, and cancels a
371
+ * {@link AgentManager.disconnect | disconnect()} made in between, which is what makes it safe
372
+ * in a React StrictMode effect. While a session is open it rejects instead: call
373
+ * `disconnect()` first to start a fresh conversation, or
374
+ * {@link AgentManager.reconnect | reconnect()} to keep the current one. After the server ended
375
+ * the session (`'disconnected'` with a {@link StreamEndReason}), `connect()` opens a new one. A
376
+ * call that joins an
372
377
  * operation a {@link AgentManager.disconnect | disconnect()} later cancels resolves without a
373
378
  * session open; read {@link AgentManager.getConnectionState | getConnectionState()} for what
374
379
  * happened.
@@ -455,12 +460,12 @@ export declare interface AgentManager {
455
460
  */
456
461
  publishMicrophoneStream(stream: MediaStream): Promise<void>;
457
462
  /**
458
- * Stops and removes the currently published microphone track from the session.
463
+ * Removes the published microphone track from the session.
459
464
  *
460
- * The counterpart of {@link AgentManager.publishMicrophoneStream | publishMicrophoneStream()} —
461
- * use it to mute the user for the rest of the session. Expressive (V4) agents only; on Talks
462
- * (V2) and Clips (V3) agents, and when nothing is published, it resolves without doing
463
- * anything.
465
+ * The counterpart of {@link AgentManager.publishMicrophoneStream | publishMicrophoneStream()}.
466
+ * The `MediaStreamTrack` is not stopped, because the stream is the caller's: stop it to release
467
+ * the microphone. Expressive (V4) agents only; on Talks (V2) and Clips (V3) agents, and when
468
+ * nothing is published, it resolves without doing anything.
464
469
  *
465
470
  * @returns Resolves once the track is removed.
466
471
  * @example
@@ -475,9 +480,10 @@ export declare interface AgentManager {
475
480
  * Use it when the user picks a different input device: the publication is preserved — its
476
481
  * LiveKit publication id (SID) and SSRC stay the same, though the `MediaStreamTrack` id
477
482
  * changes — so the server sees continuous audio across the swap rather than a stop and a
478
- * restart. Rejects with a plain `Error` from the transport when there is no active publication
479
- * — fall back to {@link AgentManager.publishMicrophoneStream | publishMicrophoneStream()} in
480
- * that case. Expressive (V4) agents only; on Talks (V2) and Clips (V3) agents, and before
483
+ * restart. The previous track is not stopped; stop it once the swap resolves. Rejects with a
484
+ * plain `Error` from the transport when there is no active publication: fall back to
485
+ * {@link AgentManager.publishMicrophoneStream | publishMicrophoneStream()} in that case.
486
+ * Expressive (V4) agents only; on Talks (V2) and Clips (V3) agents, and before
481
487
  * {@link AgentManager.connect | connect()}, the returned promise rejects with a
482
488
  * {@link ValidationError}.
483
489
  *
@@ -520,9 +526,10 @@ export declare interface AgentManager {
520
526
  */
521
527
  publishCameraStream(stream: MediaStream): Promise<void>;
522
528
  /**
523
- * Stops and removes the currently published camera track from the session.
529
+ * Removes the published camera track from the session.
524
530
  *
525
- * Call it to disable vision. Expressive (V4) agents only; on Talks (V2) and Clips (V3) agents,
531
+ * Call it to disable vision. The `MediaStreamTrack` is not stopped, because the stream is the
532
+ * caller's: stop it to release the camera. Expressive (V4) agents only; on Talks (V2) and Clips (V3) agents,
526
533
  * and when nothing is published, it resolves without doing anything.
527
534
  *
528
535
  * @returns Resolves once the track is removed.
@@ -691,7 +698,7 @@ export declare interface AgentManager {
691
698
  * ```ts
692
699
  * import { ChatMode } from '@d-id/client-sdk';
693
700
  *
694
- * await agentManager.changeMode(ChatMode.TextOnly); // Disconnects the stream.
701
+ * await agentManager.changeMode(ChatMode.Off); // Disconnects the stream.
695
702
  *
696
703
  * await agentManager.changeMode(ChatMode.Functional);
697
704
  * await agentManager.connect();
@@ -731,8 +738,8 @@ export declare interface AgentManager {
731
738
  * {@link AgentManagerCallbacks.onInterruptibleChange | onInterruptibleChange} tracks whether it
732
739
  * is allowed right now.
733
740
  *
734
- * The interrupted message is marked as such in the next
735
- * {@link AgentManagerCallbacks.onNewMessage | onNewMessage} — but only when an interrupt was
741
+ * The last message in the transcript is marked `interrupted` in the next
742
+ * {@link AgentManagerCallbacks.onNewMessage | onNewMessage}, but only when an interrupt was
736
743
  * actually sent. A call that finds nothing to interrupt leaves the message untouched and fires
737
744
  * no callback.
738
745
  *
@@ -1015,7 +1022,7 @@ export declare interface AgentManager {
1015
1022
  * onNewMessage(messages, type) {
1016
1023
  * // `partial` fires repeatedly as the answer streams in; `answer` is the final one.
1017
1024
  * if (type === 'answer') {
1018
- * console.log(messages[messages.length - 1].content);
1025
+ * console.log(messages[messages.length - 1]?.content);
1019
1026
  * }
1020
1027
  * },
1021
1028
  * };
@@ -1206,8 +1213,9 @@ export declare interface AgentManager {
1206
1213
  * Called when the agent becomes interruptible, or stops being interruptible.
1207
1214
  *
1208
1215
  * Expressive (V4) agents only, and about whether interrupting is allowed *right now*: it goes
1209
- * `false` while a `blocking` client tool call is outstanding, because the agent is suspended
1210
- * waiting for it, and back to `true` when the call finishes. That is a different question from
1216
+ * `false` while any `blocking` tool call is outstanding (client or server), and back to `true`
1217
+ * once none is: finished, failed, its turn ended, or the session disconnected. That is a
1218
+ * different question from
1211
1219
  * {@link AgentManager.isInterruptAvailable | isInterruptAvailable()}, which says whether
1212
1220
  * the session supports interrupting at all. Use this one to enable or disable an interrupt
1213
1221
  * button. Talks (V2) and Clips (V3) agents never report a change.
@@ -1223,7 +1231,7 @@ export declare interface AgentManager {
1223
1231
  * ```
1224
1232
  */
1225
1233
  onInterruptibleChange?: (
1226
- /** `true` while there is something to interrupt, `false` while there is not. */
1234
+ /** `true` when interrupting is allowed, `false` while a blocking tool call is pending. */
1227
1235
  interruptible: boolean) => void;
1228
1236
  /**
1229
1237
  * Called whenever the set of tool calls running in the session changes.
@@ -1266,8 +1274,8 @@ export declare interface AgentManager {
1266
1274
  /**
1267
1275
  * Credentials used for every request the SDK makes on the application's behalf.
1268
1276
  *
1269
- * Use `{ type: 'key', clientKey }` in a browser: a client key is scoped to one agent and to
1270
- * the domains you allowed, so it is the only shape that is safe to ship in a page. See
1277
+ * Use `{ type: 'key', clientKey }` in a browser: a client key works for one agent and only from
1278
+ * the domains you allowed, so it is the only credential that is safe to ship in a page. See
1271
1279
  * {@link Auth}.
1272
1280
  */
1273
1281
  auth: Auth;
@@ -1322,10 +1330,13 @@ export declare interface AgentManager {
1322
1330
  */
1323
1331
  debug?: boolean;
1324
1332
  /**
1325
- * Whether the stream is created in verbose mode, which logs more server-side detail.
1333
+ * Whether tool-call events carry the call's input, output, duration and failure details.
1326
1334
  *
1327
- * Independent of {@link AgentManagerOptions.debug | debug}: this one is sent to the streaming
1328
- * service rather than controlling console output. Expressive (V4) agents only.
1335
+ * Without it, {@link AgentManagerCallbacks.onToolEvent | onToolEvent} payloads carry only the
1336
+ * call's id, name and timestamp. The Agents API accepts it only from the agent's owner with
1337
+ * {@link BasicAuth} or {@link BearerToken} auth, and rejects it with a `403` otherwise, so a
1338
+ * browser application on a client key cannot use it. Independent of
1339
+ * {@link AgentManagerOptions.debug | debug}. Expressive (V4) agents only.
1329
1340
  *
1330
1341
  * @default false
1331
1342
  */
@@ -1385,29 +1396,6 @@ export declare interface AgentManager {
1385
1396
  persistentChat?: boolean;
1386
1397
  }
1387
1398
 
1388
- /**
1389
- * Amazon provider details: the provider type and the requested voice id.
1390
- *
1391
- * Pass it as {@link TextStreamScript.provider} to have Amazon synthesize the text. It is the one
1392
- * provider with no `voice_config`: the voice is selected by `voice_id` alone.
1393
- *
1394
- * @category Voice
1395
- */
1396
- export declare interface AmazonTtsProvider {
1397
- /**
1398
- * Selects Amazon. Either {@link Providers.Amazon} or its string value `'amazon'` — both are
1399
- * accepted, so the provider object can be written inline without importing the enum.
1400
- */
1401
- type: `${Providers.Amazon}`;
1402
- /**
1403
- * Id of the voice to speak with, from D-ID's list of Amazon voices.
1404
- *
1405
- * @example "Joanna"
1406
- * @see [Amazon voices](https://docs.d-id.com/docs/tts-amazon)
1407
- */
1408
- voice_id: string;
1409
- }
1410
-
1411
1399
  /**
1412
1400
  * What the SDK reports about the session, and where it goes.
1413
1401
  *
@@ -1466,8 +1454,9 @@ export declare interface AgentManager {
1466
1454
  * A script that makes the agent lip-sync an audio file you host, with no text-to-speech involved.
1467
1455
  *
1468
1456
  * The other payload {@link AgentManager.speak | speak()} accepts. Use it when the audio already
1469
- * exists — a recording, or speech you synthesized yourself — instead of having a provider generate
1470
- * it from text.
1457
+ * exists (a recording, or speech you synthesized yourself) instead of having a provider generate
1458
+ * it from text. Talks (V2) and Clips (V3) agents only: an Expressive (V4) agent drops an audio
1459
+ * script without an error.
1471
1460
  *
1472
1461
  * @example Audio file
1473
1462
  * ```ts
@@ -1499,10 +1488,10 @@ export declare interface AgentManager {
1499
1488
  * {@link AgentManagerOptions.auth | options.auth}.
1500
1489
  *
1501
1490
  * Pick {@link ClientKeyAuth} (`{ type: 'key', clientKey }`) for anything that runs in a browser: it
1502
- * is the only variant scoped to one agent and to the domains you allowed. {@link BearerToken} and
1503
- * {@link BasicAuth} carry account-wide credentials and belong in a trusted environment. Whichever
1504
- * you pass is used for every request the SDK makes — the REST calls, the video stream and the
1505
- * notifications web socket.
1491
+ * is the only variant limited to one agent and to the domains you allowed. {@link BasicAuth} is an
1492
+ * API key with access to the whole account, for a server. {@link BearerToken} is a D-ID user access
1493
+ * token. Whichever you pass is used for every request the SDK makes: the REST calls, the video
1494
+ * stream and the notifications web socket.
1506
1495
  *
1507
1496
  * @category Authentication
1508
1497
  */
@@ -1531,20 +1520,15 @@ export declare interface AgentManager {
1531
1520
  }
1532
1521
 
1533
1522
  /**
1534
- * Azure OpenAI provider details: the provider type, the requested voice id and an optional
1535
- * `voice_config` for style, rate and pitch.
1523
+ * Selects an Azure OpenAI voice for {@link TextStreamScript.provider}.
1536
1524
  *
1537
- * The same shape as {@link MicrosoftTtsProvider} — a `voice_id` and an optional
1538
- * {@link VoiceConfigMicrosoft} as `voice_config` — with the `type` naming Azure OpenAI instead.
1525
+ * The same shape as {@link MicrosoftTtsProvider}, with the `type` naming Azure OpenAI. Azure
1526
+ * ignores `voice_config.rate` for these voices.
1539
1527
  *
1540
1528
  * @category Voice
1541
1529
  */
1542
1530
  export declare interface AzureOpenAiTtsProvider extends Omit<MicrosoftTtsProvider, 'type'> {
1543
- /**
1544
- * Selects Azure OpenAI. Either {@link Providers.AzureOpenAi} or its string value
1545
- * `'azure-openai'` — both are accepted, so the provider object can be written inline without
1546
- * importing the enum.
1547
- */
1531
+ /** `'azure-openai'`, or {@link Providers.AzureOpenAi}. */
1548
1532
  type: `${Providers.AzureOpenAi}`;
1549
1533
  }
1550
1534
 
@@ -1666,14 +1650,15 @@ export declare interface AgentManager {
1666
1650
  }
1667
1651
 
1668
1652
  /**
1669
- * HTTP basic credentials for the Agents API, given either pre-encoded or as a username and
1670
- * password.
1653
+ * A D-ID API key, sent with HTTP basic authentication.
1654
+ *
1655
+ * An API key from D-ID Studio has the form `API_USERNAME:API_PASSWORD`. Pass it whole as `token`,
1656
+ * or split at the colon into `username` and `password`. It gives access to the whole account, so
1657
+ * keep it on a server and never ship it in a page; use {@link ClientKeyAuth} in a browser. The SDK
1658
+ * sends it as `Authorization: Basic <credentials>~<connectionId>`; the D-ID authorizer strips the
1659
+ * per-connection id before it validates the key.
1671
1660
  *
1672
- * Like {@link BearerToken} these are account-wide and belong in a trusted environment, never in a
1673
- * client bundle. The SDK sends them as `Authorization: Basic <credentials>~<connectionId>`, where
1674
- * the `token` form is used as the credentials as-is and the `username`/`password` form is
1675
- * base64-encoded as `username:password` first; the trailing per-connection id is stripped by the
1676
- * D-ID authorizer before the credentials are validated.
1661
+ * @see [Basic authentication](https://docs.d-id.com/reference/basic-authentication)
1677
1662
  *
1678
1663
  * @category Authentication
1679
1664
  */
@@ -1683,8 +1668,8 @@ export declare interface AgentManager {
1683
1668
  */
1684
1669
  type: 'basic';
1685
1670
  /**
1686
- * Credentials that are already base64-encoded, as `username:password`. Used as they
1687
- * are, so they are not encoded a second time.
1671
+ * The API key as D-ID Studio shows it (`API_USERNAME:API_PASSWORD`), or that string
1672
+ * base64-encoded. Sent as given.
1688
1673
  */
1689
1674
  token: string;
1690
1675
  } | {
@@ -1693,23 +1678,23 @@ export declare interface AgentManager {
1693
1678
  */
1694
1679
  type: 'basic';
1695
1680
  /**
1696
- * The account's user name; base64-encoded with the password before it is sent.
1681
+ * The part of the API key before the colon (`API_USERNAME`). It is not the account's
1682
+ * email address.
1697
1683
  */
1698
1684
  username: string;
1699
1685
  /**
1700
- * The account's password; base64-encoded with the user name before it is sent.
1686
+ * The part of the API key after the colon (`API_PASSWORD`).
1701
1687
  */
1702
1688
  password: string;
1703
1689
  };
1704
1690
 
1705
1691
  /**
1706
- * A bearer token for the Agents API.
1692
+ * A D-ID user access token: the short-lived token a signed-in D-ID user holds.
1707
1693
  *
1708
- * Account-wide, so it is meant for a trusted environment — a server, a build step, an internal
1709
- * tool. Anyone who reads the token can spend the account's credits, so do not ship one in a page
1710
- * you serve to users; use {@link ClientKeyAuth} there. The SDK sends it as
1711
- * `Authorization: Bearer <token>~<connectionId>`, appending a per-connection id the D-ID
1712
- * authorizer strips before it validates the token.
1694
+ * It is what D-ID's own applications use. An integration authenticates with an API key
1695
+ * ({@link BasicAuth}) on a server, or with a client key ({@link ClientKeyAuth}) in a browser. The
1696
+ * SDK sends it as `Authorization: Bearer <token>~<connectionId>`; the D-ID authorizer strips the
1697
+ * per-connection id before it validates the token.
1713
1698
  *
1714
1699
  * @category Authentication
1715
1700
  */
@@ -1781,8 +1766,9 @@ export declare interface AgentManager {
1781
1766
  /**
1782
1767
  * Text answers only: the chat works over the Agents API, but no video is produced.
1783
1768
  *
1784
- * Talks (V2) and Clips (V3) agents. Expressive (V4) agents chat over the streaming session,
1785
- * which {@link AgentManager.connect | connect()} always opens in {@link ChatMode.Functional}.
1769
+ * A fallback the server sets, for example when the account is out of credits; the SDK adopts
1770
+ * it and reports it through {@link AgentManagerCallbacks.onModeChange | onModeChange}. Talks
1771
+ * (V2) and Clips (V3) agents only.
1786
1772
  */
1787
1773
  TextOnly = "TextOnly",
1788
1774
  /**
@@ -1825,21 +1811,19 @@ export declare interface AgentManager {
1825
1811
  }
1826
1812
 
1827
1813
  /**
1828
- * The server put the session into a different chat mode from the one that was requested, and the
1829
- * new mode cannot stream video.
1830
- *
1831
- * Raised while {@link AgentManager.connect | connect()} runs, when the chat the server created came
1832
- * back in a mode other than the requested {@link AgentManagerOptions.mode | mode} and that mode is
1833
- * not {@link ChatMode.Functional | Functional} — {@link ChatMode.TextOnly | TextOnly} or
1834
- * {@link ChatMode.Maintenance | Maintenance}, for instance. The comparison only runs when `mode` was
1835
- * set in the options, so a session that never asked for a mode never sees this error. The SDK
1814
+ * The server created the chat in a mode that cannot stream video.
1815
+ *
1816
+ * Raised while {@link AgentManager.connect | connect()} runs on a Talks (V2) or Clips (V3) agent,
1817
+ * when the chat the server created is in a mode other than the session's (`Functional` by default)
1818
+ * and that mode is not {@link ChatMode.Functional | Functional}: {@link ChatMode.TextOnly | TextOnly}
1819
+ * or {@link ChatMode.Maintenance | Maintenance}, for instance. The SDK
1836
1820
  * disconnects the stream it had just opened and `connect()` resolves anyway, so no video will play
1837
1821
  * and {@link AgentManagerCallbacks.onSrcObjectReady | onSrcObjectReady} will not fire again: the
1838
1822
  * session continues as a text chat.
1839
1823
  *
1840
1824
  * It is delivered to {@link AgentManagerCallbacks.onError | onError} and never thrown, so
1841
1825
  * `connect()` does not reject on it.
1842
- * {@link AgentManagerCallbacks.onModeChange | onModeChange} fires with the new mode just before it.
1826
+ * {@link AgentManagerCallbacks.onModeChange | onModeChange} fires with the new mode just after it.
1843
1827
  * Treat it as a signal to switch the interface to text, not as a connection failure.
1844
1828
  * {@link BaseError.kind | kind} is `'ChatModeDowngraded'`.
1845
1829
  *
@@ -1903,15 +1887,14 @@ export declare interface AgentManager {
1903
1887
  /**
1904
1888
  * A client key for the Agents API — the credential to use in a browser.
1905
1889
  *
1906
- * This is the one shape that is safe to ship in a page: a client key is scoped to a single agent
1907
- * and only works from the domains allowed for it, so it cannot be reused elsewhere. Copy it from the
1908
- * agent's Embed snippet, where it appears as `data-client-key`, or create one with the Agents API
1909
- * (see the link below). The SDK sends it
1910
- * as `Authorization: Client-Key <clientKey>.<externalId>_<connectionId>`, where the external id is
1890
+ * This is the one credential that is safe to ship in a page: a client key works for one agent,
1891
+ * and only from the domains allowed for it. Copy it from the agent's Embed snippet, where it
1892
+ * appears as `data-client-key`, or create one with the Agents API (see the link below). The SDK
1893
+ * sends it as `Authorization: Client-Key <clientKey>.<externalId>_<connectionId>`, where the external id is
1911
1894
  * either {@link AgentManagerOptions.externalId | externalId} or a per-browser id the SDK keeps in
1912
1895
  * `localStorage`.
1913
1896
  *
1914
- * @see [Create a client key](https://docs.d-id.com/reference/createclientkey)
1897
+ * @see [Create an agent client key](https://docs.d-id.com/reference/createresourceclientkey)
1915
1898
  * @category Authentication
1916
1899
  */
1917
1900
  export declare interface ClientKeyAuth {
@@ -1932,13 +1915,14 @@ export declare interface AgentManager {
1932
1915
  * Register it with {@link AgentManager.registerClientTool | registerClientTool()}, under the tool
1933
1916
  * name defined in the agent's configuration. The SDK parses the arguments the agent's LLM produced
1934
1917
  * and passes them in; whatever the handler resolves with is sent back to the agent as the tool's
1935
- * result. Throwing rejects the call, and the error message is forwarded to the agent. Expressive
1936
- * (V4) agents only — {@link AgentManager.registerClientTool | registerClientTool()} throws a
1937
- * {@link ValidationError} on a Talks (V2) or Clips (V3) agent.
1918
+ * result. Throwing rejects the call, and the error message (at most 256 bytes) is forwarded to the
1919
+ * agent. The server gives a handler 10 seconds unless the tool's configuration sets a `timeout`.
1920
+ * Expressive (V4) agents only: {@link AgentManager.registerClientTool | registerClientTool()}
1921
+ * throws a {@link ValidationError} on a Talks (V2) or Clips (V3) agent.
1938
1922
  *
1939
1923
  * @param args - The arguments the LLM produced for this call, already parsed from JSON.
1940
- * @returns A JSON string with the tool's result, at most 15 KiB — the LiveKit RPC response limit;
1941
- * a larger result fails the call with an RPC error. A synchronous handler may return the string
1924
+ * @returns The tool's result as a string, at most 15 KiB (the LiveKit RPC response limit; a larger
1925
+ * result fails the call). Return JSON: the server parses it, and passes other text on as it is. A synchronous handler may return the string
1942
1926
  * directly; the SDK awaits the result either way.
1943
1927
  * @example
1944
1928
  * ```ts
@@ -2117,9 +2101,10 @@ export declare interface AgentManager {
2117
2101
  */
2118
2102
  export declare enum DataChannelTopic {
2119
2103
  /**
2120
- * Messages that drive a presentation the agent is showing alongside its video, such as moving
2121
- * to another slide. Sent on the wire as `did.presentation`; the payload shape is whatever the
2122
- * presentation the agent is running expects.
2104
+ * Tells the agent which slide the user moved to in the deck it is presenting. Sent on the wire
2105
+ * as `did.presentation`. The only payload the agent reads is
2106
+ * `{ type: 'navigate', slide: <integer> }`; anything else, or any message while the agent has
2107
+ * no deck, is ignored.
2123
2108
  */
2124
2109
  Presentation = "did.presentation"
2125
2110
  }
@@ -2171,18 +2156,12 @@ export declare interface AgentManager {
2171
2156
  export declare type DIDError = HttpError | NetworkError | StreamError | WsError | ValidationError | ChatCreationFailed | ChatModeDowngraded;
2172
2157
 
2173
2158
  /**
2174
- * ElevenLabs provider details: the provider type and the requested voice id. Available to premium users.
2175
- *
2176
- * Pass it as {@link TextStreamScript.provider} to have ElevenLabs synthesize the text, optionally
2177
- * with {@link VoiceConfigElevenlabs} to control how closely the voice is reproduced.
2159
+ * Selects an ElevenLabs voice for {@link TextStreamScript.provider}. Available on paid plans.
2178
2160
  *
2179
2161
  * @category Voice
2180
2162
  */
2181
2163
  export declare interface ElevenlabsTtsProvider {
2182
- /**
2183
- * Selects ElevenLabs. Either {@link Providers.Elevenlabs} or its string value `'elevenlabs'` —
2184
- * both are accepted, so the provider object can be written inline without importing the enum.
2185
- */
2164
+ /** `'elevenlabs'`, or {@link Providers.Elevenlabs}. */
2186
2165
  type: `${Providers.Elevenlabs}`;
2187
2166
  /**
2188
2167
  * Id of the voice to speak with, from D-ID's list of ElevenLabs voices.
@@ -2191,12 +2170,7 @@ export declare interface AgentManager {
2191
2170
  * @see [ElevenLabs voices](https://docs.d-id.com/docs/tts-elevenlabs)
2192
2171
  */
2193
2172
  voice_id: string;
2194
- /**
2195
- * How closely the voice should follow the original it was cloned from. See
2196
- * {@link VoiceConfigElevenlabs}.
2197
- *
2198
- * @see [ElevenLabs voice settings](https://elevenlabs.io/docs/best-practices/prompting/controls)
2199
- */
2173
+ /** Voice settings; see {@link VoiceConfigElevenlabs}. */
2200
2174
  voice_config?: VoiceConfigElevenlabs;
2201
2175
  }
2202
2176
 
@@ -2666,22 +2640,12 @@ export declare interface AgentManager {
2666
2640
  }
2667
2641
 
2668
2642
  /**
2669
- * Microsoft Azure provider details: the provider type, the requested voice id and an optional
2670
- * `voice_config` for style, rate and pitch.
2671
- *
2672
- * Pass it as {@link TextStreamScript.provider} to pick a Microsoft Azure voice explicitly,
2673
- * optionally with {@link VoiceConfigMicrosoft} to set style, rate and pitch. The Agents API
2674
- * documents Microsoft TTS as its default when a script names no provider, so this is also the
2675
- * provider a script without one ends up using.
2643
+ * Selects a Microsoft Azure voice for {@link TextStreamScript.provider}.
2676
2644
  *
2677
2645
  * @category Voice
2678
2646
  */
2679
2647
  export declare interface MicrosoftTtsProvider {
2680
- /**
2681
- * Selects Microsoft Azure. Either {@link Providers.Microsoft} or its string value
2682
- * `'microsoft'` — both are accepted, so the provider object can be written inline without
2683
- * importing the enum.
2684
- */
2648
+ /** `'microsoft'`, or {@link Providers.Microsoft}. */
2685
2649
  type: `${Providers.Microsoft}`;
2686
2650
  /**
2687
2651
  * Id of the voice to speak with, from D-ID's list of Microsoft Azure voices.
@@ -2796,25 +2760,26 @@ export declare interface AgentManager {
2796
2760
  export declare function parseMessageParts(content: string): MessagePart[];
2797
2761
 
2798
2762
  /**
2799
- * The text-to-speech engines a voice can be served by.
2763
+ * The text-to-speech engines a voice in D-ID's catalog can be served by.
2800
2764
  *
2801
- * The value of the `type` field that discriminates the provider objects — {@link Providers.Microsoft}
2802
- * selects a {@link MicrosoftTtsProvider}, {@link Providers.Elevenlabs} an
2803
- * {@link ElevenlabsTtsProvider}, and so on — and of {@link Voice.provider} when you look a voice up
2804
- * in D-ID's catalog. Which providers an account may use depends on its plan.
2765
+ * The value of {@link Voice.provider}, and of the `type` field that discriminates the provider
2766
+ * objects a speak script accepts: {@link Providers.Microsoft} selects a {@link MicrosoftTtsProvider},
2767
+ * {@link Providers.Elevenlabs} an {@link ElevenlabsTtsProvider} and {@link Providers.AzureOpenAi}
2768
+ * an {@link AzureOpenAiTtsProvider}. The other values appear only in the catalog.
2805
2769
  *
2806
2770
  * @category Voice
2807
2771
  */
2808
2772
  export declare enum Providers {
2809
- /** Amazon's text-to-speech service; see {@link AmazonTtsProvider}. */
2773
+ /** Amazon Polly. Catalog only: agents cannot speak with an Amazon voice. */
2810
2774
  Amazon = "amazon",
2811
2775
  /** Azure OpenAI text-to-speech; see {@link AzureOpenAiTtsProvider}. */
2812
2776
  AzureOpenAi = "azure-openai",
2813
- /** Microsoft Azure text-to-speech, which the Agents API documents as its default when a
2814
- * script names no provider; see {@link MicrosoftTtsProvider}. */
2777
+ /** Microsoft Azure text-to-speech; see {@link MicrosoftTtsProvider}. */
2815
2778
  Microsoft = "microsoft",
2816
2779
  /** ElevenLabs text-to-speech; see {@link ElevenlabsTtsProvider}. */
2817
- Elevenlabs = "elevenlabs"
2780
+ Elevenlabs = "elevenlabs",
2781
+ /** Cartesia text-to-speech. Available as the configured voice of Expressive (V4) agents only. */
2782
+ Cartesia = "cartesia"
2818
2783
  }
2819
2784
 
2820
2785
  /**
@@ -3198,19 +3163,18 @@ export declare interface AgentManager {
3198
3163
  }
3199
3164
 
3200
3165
  /**
3201
- * A short-lived credential for the D-ID speech-to-text service.
3166
+ * A short-lived Azure Speech authorization token, issued by the Agents API.
3202
3167
  *
3203
- * What {@link AgentManager.getSttToken | getSttToken()} resolves with, fetched fresh from the
3204
- * Agents API each time. It lets an application drive its own speech recognition against the
3205
- * service the agent uses without putting D-ID credentials in the browser. The token expires, so request one per recognition session
3206
- * rather than holding on to it.
3168
+ * What {@link AgentManager.getSttToken | getSttToken()} resolves with. It lets an application run
3169
+ * Azure speech recognition in the browser without holding a speech subscription key. Azure tokens
3170
+ * expire after about ten minutes, so request one per recognition session.
3207
3171
  *
3208
3172
  * @category Agent Manager
3209
3173
  */
3210
3174
  export declare interface SttTokenResponse {
3211
- /** The authorization token to present to the speech service. */
3175
+ /** The authorization token to present to Azure Speech. */
3212
3176
  token: string;
3213
- /** The speech service region the token is valid in. */
3177
+ /** The Azure region the token is valid in. */
3214
3178
  region: string;
3215
3179
  }
3216
3180
 
@@ -3277,8 +3241,9 @@ export declare interface AgentManager {
3277
3241
  *
3278
3242
  * One of the objects in {@link TtsProvider}: a `type` naming the provider, the
3279
3243
  * `voice_id` to speak with, and optional provider-specific `voice_config`. Leave it out and the
3280
- * SDK sends the script without a provider, so the voice is chosen server-side; the Agents API
3281
- * documents Microsoft TTS as its default when no provider is given.
3244
+ * agent speaks with its own configured voice; that voice's `voice_config` is not applied.
3245
+ * Talks (V2) and Clips (V3) agents only: Expressive (V4) agents always speak with their
3246
+ * configured voice and ignore this field.
3282
3247
  */
3283
3248
  provider?: TtsProvider;
3284
3249
  /**
@@ -3325,14 +3290,14 @@ export declare interface AgentManager {
3325
3290
  callId: string;
3326
3291
  /** Name of the tool that was called. */
3327
3292
  name: string;
3328
- /** The arguments the call was made with. */
3329
- input: Record<string, unknown>;
3330
- /** The result the tool returned. */
3331
- output: Record<string, unknown>;
3332
- /** How long the call took, in milliseconds. */
3333
- durationMs: number;
3334
- /** Any additional metadata the tool reported alongside its result. */
3335
- extra: Record<string, unknown>;
3293
+ /** The arguments the call was made with. Sent only in {@link AgentManagerOptions.verbose | verbose} sessions. */
3294
+ input?: Record<string, unknown>;
3295
+ /** The result the tool returned. Sent only in {@link AgentManagerOptions.verbose | verbose} sessions. */
3296
+ output?: Record<string, unknown>;
3297
+ /** How long the call took, in milliseconds. Sent only in {@link AgentManagerOptions.verbose | verbose} sessions. */
3298
+ durationMs?: number;
3299
+ /** Any additional metadata the tool reported alongside its result. Sent only in {@link AgentManagerOptions.verbose | verbose} sessions. */
3300
+ extra?: Record<string, unknown>;
3336
3301
  /** When the call finished, as reported by the server. */
3337
3302
  timestamp: string;
3338
3303
  }
@@ -3351,23 +3316,23 @@ export declare interface AgentManager {
3351
3316
  callId: string;
3352
3317
  /** Name of the tool that was called. */
3353
3318
  name: string;
3354
- /** The arguments the call was made with. */
3355
- input: Record<string, unknown>;
3319
+ /** The arguments the call was made with. Sent only in {@link AgentManagerOptions.verbose | verbose} sessions. */
3320
+ input?: Record<string, unknown>;
3356
3321
  /**
3357
- * Whatever the failed call produced, if anything.
3322
+ * Whatever the failed call produced, if anything. Sent only in {@link AgentManagerOptions.verbose | verbose} sessions.
3358
3323
  *
3359
3324
  * Any JSON value — the server's own type for a tool result is unconstrained, and on a failure
3360
3325
  * it is commonly the reason as a plain string rather than a structured result. Narrow it
3361
3326
  * before use. The string case is also given to you as
3362
3327
  * {@link ToolCallErrorPayload.error | error}, which is what to show.
3363
3328
  */
3364
- output: unknown;
3365
- /** How long the call ran before failing, in milliseconds. */
3366
- durationMs: number;
3367
- /** Any additional metadata the server reported with the failure. */
3368
- extra: Record<string, unknown>;
3329
+ output?: unknown;
3330
+ /** How long the call ran before failing, in milliseconds. Sent only in {@link AgentManagerOptions.verbose | verbose} sessions. */
3331
+ durationMs?: number;
3332
+ /** Any additional metadata the server reported with the failure. Sent only in {@link AgentManagerOptions.verbose | verbose} sessions. */
3333
+ extra?: Record<string, unknown>;
3369
3334
  /**
3370
- * What went wrong, in one line, when the server said.
3335
+ * What went wrong, in one line, when the server said. Only verbose sessions carry it.
3371
3336
  *
3372
3337
  * Taken from the failure the server reports in {@link ToolCallErrorPayload.extra | extra}
3373
3338
  * (`extra.error.message`), and from {@link ToolCallErrorPayload.output | output} when that is
@@ -3434,8 +3399,8 @@ export declare interface AgentManager {
3434
3399
  callId: string;
3435
3400
  /** Name of the tool the agent is calling, as configured on the agent. */
3436
3401
  name: string;
3437
- /** The arguments the agent's LLM produced for this call. */
3438
- input: Record<string, unknown>;
3402
+ /** The arguments the agent's LLM produced for this call. Sent only in {@link AgentManagerOptions.verbose | verbose} sessions. */
3403
+ input?: Record<string, unknown>;
3439
3404
  /**
3440
3405
  * The tool's result, when the server sent one.
3441
3406
  *
@@ -3518,11 +3483,11 @@ export declare interface AgentManager {
3518
3483
  export declare type ToolExecutionMode = 'blocking' | 'async';
3519
3484
 
3520
3485
  /**
3521
- * The provider object a speak script accepts: any of the four text-to-speech providers.
3486
+ * The provider object a speak script accepts.
3522
3487
  *
3523
3488
  * This is the type of {@link TextStreamScript.provider}. Pick the variant for the provider you
3524
- * want, give it the `voice_id` to speak with and, except for {@link AmazonTtsProvider}, an optional
3525
- * `voice_config`; `type` discriminates the union.
3489
+ * want and give it the `voice_id` to speak with and an optional `voice_config`; `type`
3490
+ * discriminates the union.
3526
3491
  *
3527
3492
  * @example
3528
3493
  * ```ts
@@ -3536,7 +3501,7 @@ export declare interface AgentManager {
3536
3501
  * ```
3537
3502
  * @category Voice
3538
3503
  */
3539
- export declare type TtsProvider = MicrosoftTtsProvider | AzureOpenAiTtsProvider | ElevenlabsTtsProvider | AmazonTtsProvider;
3504
+ export declare type TtsProvider = MicrosoftTtsProvider | AzureOpenAiTtsProvider | ElevenlabsTtsProvider;
3540
3505
 
3541
3506
  /**
3542
3507
  * A method was called with arguments, or in a state, that the SDK will not send to the Agents
@@ -3611,21 +3576,29 @@ export declare interface AgentManager {
3611
3576
  * to build a voice picker, say — can type the result and then feed {@link Voice.id | id} and
3612
3577
  * {@link Voice.provider | provider} into the provider object it passes as
3613
3578
  * {@link TextStreamScript.provider}. {@link Voice.provider | provider} is the wide
3614
- * {@link Providers} enum, while each provider variant requires its own literal `type` — so switch
3579
+ * {@link Providers} enum, while each provider variant requires its own literal `type`, so switch
3615
3580
  * on it to build the matching variant, for example {@link Providers.Elevenlabs} to an
3616
- * {@link ElevenlabsTtsProvider}.
3581
+ * {@link ElevenlabsTtsProvider}. The catalog also lists voices agents cannot speak with (Google
3582
+ * voices, for example), so keep only the providers {@link TtsProvider} has a variant for.
3617
3583
  *
3618
3584
  * @category Voice
3619
3585
  */
3620
3586
  export declare interface Voice {
3621
3587
  /** Identifier of the voice: the value to pass as a provider object's `voice_id`. */
3622
3588
  id: string;
3623
- /** Human-readable name of the voice. */
3589
+ /** Display name of the voice. */
3624
3590
  name: string;
3625
3591
  /** The gender the voice is labelled with. */
3626
3592
  gender: string;
3627
- /** The locale the voice speaks, such as `en-US`. */
3628
- locale: string;
3593
+ /** Each language the voice speaks, with its locale code such as `en-US`. */
3594
+ languages: {
3595
+ /** The language, as a name such as `English`. */
3596
+ language: string;
3597
+ /** Its locale code, such as `en-US`. */
3598
+ locale: string;
3599
+ /** The accent, when the catalog names one. */
3600
+ accent?: string;
3601
+ }[];
3629
3602
  /** Which accounts may use the voice. See {@link VoiceAccess}. */
3630
3603
  access: VoiceAccess;
3631
3604
  /** The provider that serves the voice. See {@link Providers}. */
@@ -3633,8 +3606,8 @@ export declare interface AgentManager {
3633
3606
  /** The speaking styles this particular voice offers, for example as a
3634
3607
  * {@link VoiceConfigMicrosoft.style | style}. Styles differ from voice to voice. */
3635
3608
  styles: string[];
3636
- /** The language of the voice, as a name rather than a locale code. */
3637
- language: string;
3609
+ /** The language of the voice, as a name rather than a locale code, when the catalog sets one. */
3610
+ language?: string;
3638
3611
  }
3639
3612
 
3640
3613
  /**
@@ -3646,19 +3619,22 @@ export declare interface AgentManager {
3646
3619
  * @category Voice
3647
3620
  */
3648
3621
  export declare enum VoiceAccess {
3649
- /** As exposed by the voices API: available to every account. */
3622
+ /** Available to every account. */
3650
3623
  Public = "public",
3651
- /** As exposed by the voices API: available to accounts whose plan includes premium voices. */
3624
+ /** Available to accounts whose plan includes premium voices. */
3652
3625
  Premium = "premium",
3653
- /** As exposed by the voices API: available only to the account the voice belongs to. */
3654
- Private = "private"
3626
+ /** Available only to the account the voice belongs to. */
3627
+ Private = "private",
3628
+ /** A voice from the account's own ElevenLabs account, used with its ElevenLabs API key. */
3629
+ ExternalPrivate = "external-private"
3655
3630
  }
3656
3631
 
3657
3632
  /**
3658
- * How closely an ElevenLabs voice should follow the original it was built from.
3633
+ * ElevenLabs voice settings: how stable the voice is and how closely it adheres to the source voice.
3659
3634
  *
3660
- * The `voice_config` of {@link ElevenlabsTtsProvider}, mirroring ElevenLabs' own voice settings.
3661
- * Both fields are optional; when one is omitted the Agents API applies its own default.
3635
+ * The `voice_config` of {@link ElevenlabsTtsProvider}. Every field is optional.
3636
+ *
3637
+ * @see [ElevenLabs on D-ID](https://docs.d-id.com/docs/tts-elevenlabs)
3662
3638
  *
3663
3639
  * @category Voice
3664
3640
  */
@@ -3689,10 +3665,7 @@ export declare interface AgentManager {
3689
3665
  * @category Voice
3690
3666
  */
3691
3667
  export declare interface VoiceConfigMicrosoft {
3692
- /**
3693
- * The style of the voice.
3694
- * Available styles change between voices.
3695
- */
3668
+ /** Speaking style, such as `cheerful`; {@link Voice.styles} lists what a voice offers. */
3696
3669
  style?: string;
3697
3670
  /**
3698
3671
  * The speed of the voice.