@convai/web-sdk 1.8.0-beta.5 → 1.8.0-beta.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/README.md +333 -13
  2. package/dist/core/CharacterRoster.d.ts +84 -0
  3. package/dist/core/CharacterRoster.d.ts.map +1 -0
  4. package/dist/core/CharacterRoster.js +348 -0
  5. package/dist/core/CharacterRoster.js.map +1 -0
  6. package/dist/core/ConvaiClient.d.ts +152 -2
  7. package/dist/core/ConvaiClient.d.ts.map +1 -1
  8. package/dist/core/ConvaiClient.js +1036 -49
  9. package/dist/core/ConvaiClient.js.map +1 -1
  10. package/dist/core/ConvaiRoomError.d.ts +48 -0
  11. package/dist/core/ConvaiRoomError.d.ts.map +1 -0
  12. package/dist/core/ConvaiRoomError.js +111 -0
  13. package/dist/core/ConvaiRoomError.js.map +1 -0
  14. package/dist/core/MessageHandler.d.ts +25 -0
  15. package/dist/core/MessageHandler.d.ts.map +1 -1
  16. package/dist/core/MessageHandler.js +431 -30
  17. package/dist/core/MessageHandler.js.map +1 -1
  18. package/dist/core/SSESession.d.ts +15 -0
  19. package/dist/core/SSESession.d.ts.map +1 -1
  20. package/dist/core/SSESession.js +92 -3
  21. package/dist/core/SSESession.js.map +1 -1
  22. package/dist/core/connectRequest.d.ts +38 -2
  23. package/dist/core/connectRequest.d.ts.map +1 -1
  24. package/dist/core/connectRequest.js +57 -0
  25. package/dist/core/connectRequest.js.map +1 -1
  26. package/dist/core/index.d.ts +6 -1
  27. package/dist/core/index.d.ts.map +1 -1
  28. package/dist/core/index.js +5 -1
  29. package/dist/core/index.js.map +1 -1
  30. package/dist/core/logicalTurn.d.ts +4 -1
  31. package/dist/core/logicalTurn.d.ts.map +1 -1
  32. package/dist/core/logicalTurn.js +35 -4
  33. package/dist/core/logicalTurn.js.map +1 -1
  34. package/dist/core/modelOutput.d.ts +40 -0
  35. package/dist/core/modelOutput.d.ts.map +1 -0
  36. package/dist/core/modelOutput.js +187 -0
  37. package/dist/core/modelOutput.js.map +1 -0
  38. package/dist/core/publishedChat.d.ts +9 -0
  39. package/dist/core/publishedChat.d.ts.map +1 -0
  40. package/dist/core/publishedChat.js +13 -0
  41. package/dist/core/publishedChat.js.map +1 -0
  42. package/dist/core/rosterRequest.d.ts +57 -0
  43. package/dist/core/rosterRequest.d.ts.map +1 -0
  44. package/dist/core/rosterRequest.js +210 -0
  45. package/dist/core/rosterRequest.js.map +1 -0
  46. package/dist/core/types.d.ts +667 -10
  47. package/dist/core/types.d.ts.map +1 -1
  48. package/dist/core/types.js.map +1 -1
  49. package/dist/embed/ConvaiChatElement.d.ts +27 -0
  50. package/dist/embed/ConvaiChatElement.d.ts.map +1 -0
  51. package/dist/embed/ConvaiChatElement.js +300 -0
  52. package/dist/embed/ConvaiChatElement.js.map +1 -0
  53. package/dist/embed/browser.d.ts +3 -0
  54. package/dist/embed/browser.d.ts.map +1 -0
  55. package/dist/embed/browser.js +5 -0
  56. package/dist/embed/browser.js.map +1 -0
  57. package/dist/embed/chat-embed-v1.js +100 -0
  58. package/dist/embed/chat-embed-v1.js.map +7 -0
  59. package/dist/embed/index.d.ts +9 -0
  60. package/dist/embed/index.d.ts.map +1 -0
  61. package/dist/embed/index.js +9 -0
  62. package/dist/embed/index.js.map +1 -0
  63. package/dist/embed/publicationApi.d.ts +23 -0
  64. package/dist/embed/publicationApi.d.ts.map +1 -0
  65. package/dist/embed/publicationApi.js +141 -0
  66. package/dist/embed/publicationApi.js.map +1 -0
  67. package/dist/react/components/ConvaiWidget.d.ts +11 -1
  68. package/dist/react/components/ConvaiWidget.d.ts.map +1 -1
  69. package/dist/react/components/ConvaiWidget.js +67 -11
  70. package/dist/react/components/ConvaiWidget.js.map +1 -1
  71. package/dist/react/components/rtc-widget/components/MarkdownRenderer.d.ts.map +1 -1
  72. package/dist/react/components/rtc-widget/components/MarkdownRenderer.js +39 -44
  73. package/dist/react/components/rtc-widget/components/MarkdownRenderer.js.map +1 -1
  74. package/dist/react/components/rtc-widget/components/conviComponents/SettingsTray.d.ts +1 -1
  75. package/dist/react/components/rtc-widget/components/conviComponents/SettingsTray.d.ts.map +1 -1
  76. package/dist/react/components/rtc-widget/components/conviComponents/SettingsTray.js +2 -2
  77. package/dist/react/components/rtc-widget/components/conviComponents/SettingsTray.js.map +1 -1
  78. package/dist/react/hooks/useConvaiClient.d.ts +2 -0
  79. package/dist/react/hooks/useConvaiClient.d.ts.map +1 -1
  80. package/dist/react/hooks/useConvaiClient.js +16 -0
  81. package/dist/react/hooks/useConvaiClient.js.map +1 -1
  82. package/dist/react/index.d.ts +1 -1
  83. package/dist/react/index.d.ts.map +1 -1
  84. package/dist/react/index.js +1 -1
  85. package/dist/react/index.js.map +1 -1
  86. package/dist/utils/inlineMarkdown.d.ts +38 -0
  87. package/dist/utils/inlineMarkdown.d.ts.map +1 -0
  88. package/dist/utils/inlineMarkdown.js +106 -0
  89. package/dist/utils/inlineMarkdown.js.map +1 -0
  90. package/dist/vanilla/AudioRenderer.d.ts.map +1 -1
  91. package/dist/vanilla/AudioRenderer.js +6 -0
  92. package/dist/vanilla/AudioRenderer.js.map +1 -1
  93. package/dist/vanilla/ConvaiWidget.d.ts.map +1 -1
  94. package/dist/vanilla/ConvaiWidget.js +224 -89
  95. package/dist/vanilla/ConvaiWidget.js.map +1 -1
  96. package/dist/vanilla/WidgetEventSubscriptions.d.ts +15 -0
  97. package/dist/vanilla/WidgetEventSubscriptions.d.ts.map +1 -0
  98. package/dist/vanilla/WidgetEventSubscriptions.js +37 -0
  99. package/dist/vanilla/WidgetEventSubscriptions.js.map +1 -0
  100. package/dist/vanilla/index.d.ts +2 -1
  101. package/dist/vanilla/index.d.ts.map +1 -1
  102. package/dist/vanilla/index.js +2 -1
  103. package/dist/vanilla/index.js.map +1 -1
  104. package/dist/vanilla/types.d.ts +8 -1
  105. package/dist/vanilla/types.d.ts.map +1 -1
  106. package/dist/vanilla/types.js.map +1 -1
  107. package/dist/version.d.ts +1 -1
  108. package/dist/version.js +1 -1
  109. package/dist/version.js.map +1 -1
  110. package/package.json +19 -6
package/README.md CHANGED
@@ -2,7 +2,7 @@
2
2
 
3
3
  **Real-time conversational AI characters for the web.**
4
4
 
5
- [![npm](https://img.shields.io/badge/npm-1.7.0-blue)](https://www.npmjs.com/package/@convai/web-sdk)
5
+ [![npm](https://img.shields.io/badge/npm-1.8.0--beta.7-blue)](https://www.npmjs.com/package/@convai/web-sdk)
6
6
  [![TypeScript](https://img.shields.io/badge/TypeScript-first-3178C6)](https://www.typescriptlang.org/)
7
7
  [![License](https://img.shields.io/badge/license-Apache--2.0-green)](./LICENSE)
8
8
 
@@ -10,14 +10,28 @@ TypeScript-first SDK for embedding Convai AI characters into React and vanilla J
10
10
 
11
11
  ---
12
12
 
13
- ## What's new in 1.7.0
13
+ ## What's new in 1.8.0 (beta)
14
+
15
+ Install the beta with `npm install @convai/web-sdk@beta`.
16
+
17
+ - **Multi-character rooms** — connect a roster of characters with the `characters` config option (two copies of one character are two independent members), switch who answers with `setInteractionTarget()`, add or remove members live with `updateCharacterRoster()`, and let other participants join the same room with `joinRoom()`. Every transcript line, audio track and readiness event is attributed to a `membershipId`. [Usage below.](#multi-character-rooms-beta)
18
+ - **Agentic actions and canonical model output (v2)** — opt in with `capabilities`, declare client-executed tools in `actionConfig.tools`, consume typed `modelOutput` envelopes, return results with `sendActionResultAndWait()` for a correlated server acknowledgement, and read the raw provider stream through `botLlmTextRaw`. `visionPromptMode: "disabled"` keeps scene context without advertising visual perception. [Usage below.](#agentic-actions-and-canonical-model-output-v2)
19
+ - **Published text chat** — a credential-free, text-only connection for public chat pages: `connectWithPublicationGrant()`, `connectionType: "text"`, and the self-contained `<convai-chat>` browser bundle. [Usage below.](#published-text-chat)
20
+ - **Typed events** — `client.on(event, callback)` infers each callback's payload from the exported `ConvaiEventMap`, so `botReady`, `modelOutput`, `characterStatus` and the rest are checked at compile time. See the [events reference](docs/convai_events.md).
21
+ - **Local transcript lines and usage updates** — `appendMessage()` inserts a line into the transcript without sending anything, and `toggleUsageUpdates(true)` opts into per-turn `usageUpdate` cost breakdowns.
22
+ - **Character versioning** — connect to a character's editable draft, its promoted latest release, or an immutable tag with the `characterVersion` config option, and list, compare, release, promote, fork and discard versions through `client.characterVersions`. [Usage below.](#character-versioning-beta)
23
+ - **SSE interaction transport** — text-only streamed interactions through the interaction API, selected by `interactionApiUrl`; the one transport that runs under Node. [Usage below.](#sse-interaction-transport-beta)
24
+ - **Embeddable vanilla widget** — the vanilla widget runs inside a shadow root, with a character header and a connecting overlay, so it can be dropped into any page.
25
+ - **Node-importable dist** — `@convai/web-sdk/core`, `/vanilla`, `/vanilla/websocket` and `/lipsync-helpers` load in plain Node, SSR and Next.js server components.
26
+
27
+ ### Previously in 1.7.0
14
28
 
15
29
  - **Character state of mind** — set a temporary generation mood with the `stateOfMind` config option, and change it mid-session with `updateEmotion()` without triggering a response. [Usage below.](#character-state-of-mind)
16
30
  - **Send-ahead enabled by default** — NeuroSync lipsync ahead-delivery is now on by default, reversing the 1.6.0 opt-in. Fall back to the legacy paced path with `blendshapeConfig.deliver_chunks_ahead: false`.
17
31
  - **Lipsync naturalness pipeline** — modular naturalness processing with a tuned MetaHuman profile.
18
32
  - **Adaptive glass widget styling** — `ConvaiWidget` adapts to light and dark backgrounds.
19
33
 
20
- ### Previously in 1.6.0
34
+ ### 1.6.0
21
35
 
22
36
  - **Narrative Design template keys** — personalize one narrative graph per session: seed values at connect with the `narrativeTemplateKeys` config option, replace them mid-session with `updateTemplateKeys()`. [Usage below.](#narrative-design-template-keys)
23
37
  - **Typed parameterized actions** — `actionResponse` is now typed via the exported `ConvaiAction` / `ActionResponseEvent`, including the `target` of parameterized actions. [Usage below.](#actions)
@@ -34,6 +48,11 @@ TypeScript-first SDK for embedding Convai AI characters into React and vanilla J
34
48
  - **Dynamic context & vision** — inject text state, scene metadata, and LiveKit video frames mid-session
35
49
  - **Actions & Narrative Design** — typed action decisions with parameterized targets, named triggers, and per-session template keys
36
50
  - **Long-term memory** — persistent cross-session memories scoped to each end user
51
+ - **Multi-character rooms** — a roster of characters in one room, per-member attribution, live roster changes
52
+ - **Agentic actions (v2)** — client-executed tools, canonical model output, acknowledged action results, raw provider stream
53
+ - **Published text chat** — grant-based, credential-free text sessions and a drop-in `<convai-chat>` element
54
+ - **Typed events** — every `client.on` payload typed through `ConvaiEventMap`
55
+ - **Character versioning** — connect to a draft, latest, or tagged version, and manage releases from the client
37
56
  - **File upload** — send images to the character during a live session
38
57
  - **WebSocket transport** — opt-in alternative to WebRTC for constrained networks
39
58
  - **Auth tokens** — server-side token exchange for production deployments
@@ -91,6 +110,114 @@ window.addEventListener("beforeunload", () => {
91
110
  });
92
111
  ```
93
112
 
113
+ ### Published text chat
114
+
115
+ See the [Published text chat guide](./docs/convai_published_chat.md) for
116
+ React and vanilla integrations, lifecycle behavior, and the complete API
117
+ reference.
118
+
119
+ Published chat pages use a short-lived launch grant instead of the publisher's
120
+ API key. Native Web SDK integration is the recommended embed path; an iframe is
121
+ not required. Obtain the grant from the public Character API and pass it to the
122
+ core client:
123
+
124
+ ```ts
125
+ import { ConvaiClient } from '@convai/web-sdk/core';
126
+ import { requestPublishedChatGrant } from '@convai/web-sdk/embed';
127
+
128
+ const launchToken = await requestPublishedChatGrant(
129
+ `${characterApiUrl}/chat-publications/${publicationId}/launch-grants`,
130
+ );
131
+ const client = new ConvaiClient();
132
+ const unsubscribe = client.on('botReady', () => {
133
+ unsubscribe();
134
+ client.sendUserTextMessage('Hello!');
135
+ });
136
+ await client.connectWithPublicationGrant(launchToken);
137
+ ```
138
+
139
+ The grant is single-use and is not retained for reconnect. Request a new grant
140
+ for each new session. The SDK attaches a browser-generated logical attempt ID
141
+ and retries one transient grant exchange with that same ID, so a response lost
142
+ after server-side consumption does not strand the token or create a duplicate
143
+ session. An application performing its own retry may pass the same
144
+ `connectAttemptId` option with the same launch token. Published chat is
145
+ text-only; the server-issued room token does not permit publishing microphone
146
+ or camera tracks.
147
+
148
+ The `@convai/web-sdk/embed` browser element creates its own per-browser
149
+ requester identifier. If a server or SSR integration calls its published-chat
150
+ grant helpers directly, it must pass a caller-scoped requester identifier; do
151
+ not reuse a process-wide identifier across visitors.
152
+
153
+ When using `ConvaiWidget`, provide `requestPublicationGrant` so Reset,
154
+ Disconnect, and later reopen operations mint a new grant instead of attempting
155
+ to reuse the consumed token:
156
+
157
+ ```tsx
158
+ <ConvaiWidget
159
+ convaiClient={client}
160
+ publishedChatDescriptor={{
161
+ display_name: publication.display_name,
162
+ image_url: publication.image_url,
163
+ }}
164
+ requestPublicationGrant={async () => {
165
+ const response = await fetch(`/api/chat/${publicationId}/grant`, {
166
+ method: 'POST',
167
+ });
168
+ if (!response.ok) throw new Error('Unable to relaunch published chat');
169
+ return (await response.json()).launch_token;
170
+ }}
171
+ />
172
+ ```
173
+
174
+ For copy/paste installation without an application build step, deploy the
175
+ versioned browser artifact at `dist/embed/chat-embed-v1.js` and use its native
176
+ custom element:
177
+
178
+ ```html
179
+ <script
180
+ type="module"
181
+ src="https://cdn.jsdelivr.net/npm/@convai/web-sdk@<SDK_VERSION>/dist/embed/chat-embed-v1.js"
182
+ ></script>
183
+ <convai-chat publication-id="PUBLICATION_UUID"></convai-chat>
184
+ ```
185
+
186
+ The element renders directly in the host page with Shadow DOM style isolation,
187
+ automatically mints a fresh grant per connection, and emits `convai-ready`,
188
+ `convai-disconnect`, and `convai-error` DOM events. The production bundle uses
189
+ `https://api.convai.com` and `https://realtime-api.convai.com` only as defaults
190
+ for a hand-written SaaS integration. Platform-generated snippets always emit
191
+ the deployment's public Character API and Core endpoints explicitly, so an
192
+ on-prem or non-production embed never falls through to another deployment and
193
+ still requires no credential:
194
+
195
+ ```html
196
+ <convai-chat
197
+ publication-id="PUBLICATION_UUID"
198
+ publication-api-url="https://character-api.example.com"
199
+ core-url="https://realtime.example.com"
200
+ ></convai-chat>
201
+ ```
202
+
203
+ Replace `<SDK_VERSION>` with the exact published package version. Do not use
204
+ `latest`: generated embed snippets must stay bound to the SDK version validated
205
+ for their target environment.
206
+
207
+ Descriptor and grant endpoints must allow credential-free cross-origin `GET`
208
+ and `POST` requests. The embed never receives a publisher API key, character
209
+ configuration, or reusable launch token.
210
+
211
+ For customer-managed and GovCloud deployments, the repository also builds a
212
+ browser-only image from `deploy/published-chat/Dockerfile`. It serves the exact
213
+ released bundle at `/sdk/chat-embed-v1.js` on port `8080`, plus `/healthz`, and
214
+ contains no API key or cloud identity. The manual `build-onprem` workflow
215
+ requires an immutable semver tag matching `package.json`, refuses to overwrite
216
+ an existing Docker Hub tag, and publishes both amd64 and arm64 images. Pin the
217
+ resulting digest in the deployment repository; do not make an on-prem browser
218
+ depend on jsDelivr, npm availability, GCP workload identity, or the Experience
219
+ application image.
220
+
94
221
  ---
95
222
 
96
223
  ## Documentation
@@ -115,6 +242,9 @@ Full documentation is at **<a href="https://docs.convai.com/api-docs/plugins-and
115
242
  | <a href="https://docs.convai.com/api-docs/plugins-and-integrations/web-plugins/convai-web-sdk/websocket-transport-layer" target="_blank" rel="noopener noreferrer">WebSocket Transport</a> | Alternative transport for WebRTC-constrained environments |
116
243
  | <a href="https://docs.convai.com/api-docs/plugins-and-integrations/web-plugins/convai-web-sdk" target="_blank" rel="noopener noreferrer">SSE Transport</a> | Text-only interaction transport; the one that runs under Node |
117
244
  | [Character Versioning](docs/convai_character_versioning.md) | Connect to a draft, latest, or tagged version; list, compare, release and promote versions |
245
+ | [Multi-Character Rooms](docs/convai_multi_character.md) | Rosters, `joinRoom`, interaction targets, live roster updates, per-member attribution |
246
+ | [Published Text Chat](docs/convai_published_chat.md) | Grant-based text-only sessions, `<convai-chat>`, lifecycle and API reference |
247
+ | [Events Reference (local)](docs/convai_events.md) | Every event with its typed payload, including the v2 and multi-character events |
118
248
 
119
249
  ---
120
250
 
@@ -126,6 +256,8 @@ Full documentation is at **<a href="https://docs.convai.com/api-docs/plugins-and
126
256
  | `@convai/web-sdk/react` | Same as default (React-explicit alias) |
127
257
  | `@convai/web-sdk/vanilla` | `ConvaiClient`, `createConvaiWidget`, `AudioRenderer` |
128
258
  | `@convai/web-sdk/core` | Framework-agnostic `ConvaiClient`, managers, and all types |
259
+ | `@convai/web-sdk/embed` | Server-safe publication descriptor and grant helpers |
260
+ | `@convai/web-sdk/embed/browser` | Self-contained browser bundle that registers `<convai-chat>` |
129
261
  | `@convai/web-sdk/lipsync-helpers` | Blendshape format utilities and queue helpers |
130
262
  | `@convai/web-sdk/vanilla/websocket` | **Opt-in.** Registers the WebSocket transport. Import alongside `/vanilla` when using `transport: "websocket"`. |
131
263
 
@@ -184,28 +316,103 @@ Full guide: [SSE Interaction Transport](docs/convai_sse_transport.md).
184
316
 
185
317
  ## Character versioning (beta)
186
318
 
187
- Connect to a character's editable draft, its promoted latest release, or an immutable tag, and
188
- manage those versions from the same client:
319
+ Every character has an editable **draft** and, once released, immutable **tagged versions**
320
+ (`1.0`, `1.1`, …). A movable `latest` pointer names the release the runtime uses when a client
321
+ connects without a selector. Pick the version to run with `characterVersion`:
189
322
 
190
- ```ts
191
- const client = new ConvaiClient({
323
+ | `characterVersion` | Runs |
324
+ | ------------------ | ----------------------------------------------------------------- |
325
+ | *(omitted)* | The effective latest — how characters that predate versioning behave |
326
+ | `"draft"` | The editable draft, for testing unreleased changes |
327
+ | `"latest"` | The promoted latest release, explicitly |
328
+ | `"1.2"` / `"1.2.3"` | An immutable tagged version |
329
+
330
+ ```tsx
331
+ const client = useConvaiClient({
192
332
  apiKey: "...",
193
333
  characterId: "...",
194
- characterVersion: "draft", // or "latest", "1.2" — omit for the effective latest
334
+ characterVersion: "draft",
195
335
  });
336
+ ```
337
+
338
+ On the wire the selector is joined to the id as `<uuid>-draft`; `client.characterId` keeps
339
+ returning the bare UUID, and `client.characterReference` returns the joined form.
340
+
341
+ Manage versions from the same client — `client.characterVersions` is available as soon as the
342
+ config has an `apiKey`, before connecting:
196
343
 
344
+ ```ts
197
345
  const versions = client.characterVersions!;
198
- const { has_unpublished_changes } = await versions.list();
199
- if (has_unpublished_changes) await versions.create("1.1", { makeLatest: true });
200
- await client.connect();
346
+
347
+ const { has_unpublished_changes, draft_revision_id } = await versions.list();
348
+ const diff = await versions.diff("latest", "draft", { view: "semantic" });
349
+
350
+ await versions.create("1.1", { makeLatest: true }); // release the draft as a tag
351
+ await versions.promote("1.0"); // roll back: latest → 1.0
352
+ await versions.discardDraft(draft_revision_id!); // throw away unreleased edits
201
353
  ```
202
354
 
203
- Explicit selectors are resolved by the runtime through the Character REST platform; staging has
204
- this today and production returns `503` for explicit selectors until it is promoted there.
355
+ `CharacterVersionManager` can also be constructed standalone with an `apiKey` or a Convai
356
+ personal access token; every method rejects with `CharacterApiError` (`status`, `detail`) on a
357
+ non-2xx answer. Point `characterApiUrl` at `https://api2-stg.convai.com` to author against
358
+ staging.
359
+
360
+ Explicit selectors are resolved by the runtime through the Character REST platform. Staging has
361
+ this today; production answers explicit selectors with `503` until it is promoted there.
205
362
  Full guide: [Character Versioning](docs/convai_character_versioning.md).
206
363
 
207
364
  ---
208
365
 
366
+
367
+ ## Multi-character rooms (beta)
368
+
369
+ Pass `characters` instead of `characterId` to open a room with several members.
370
+ `characters[0]` answers first; the same character id may appear more than once
371
+ and each entry is an independent member with its own `membershipId`. Roster
372
+ rooms need the LiveKit transport and a nonblank `endUserId`.
373
+
374
+ ```ts
375
+ const client = new ConvaiClient({
376
+ apiKey: "YOUR_API_KEY",
377
+ endUserId: "player-42",
378
+ characters: [
379
+ { characterId: GUIDE }, // initial active member
380
+ { characterId: ASSESSOR },
381
+ { characterId: GUIDE }, // a second, independent copy of the guide
382
+ ],
383
+ maxNumParticipants: 2,
384
+ });
385
+
386
+ client.on("characterReady", (member) => console.log("ready", member.membershipId));
387
+ client.on("characterStatus", ({ membershipId, status, failureCode }) => {
388
+ if (status === "failed") console.warn(membershipId, failureCode);
389
+ });
390
+
391
+ await client.connect();
392
+
393
+ // Route the next user turns to another member; null closes every gate.
394
+ await client.setInteractionTarget(client.characters[1].membershipId);
395
+
396
+ // Change the roster on a live room.
397
+ await client.updateCharacterRoster({ add: [{ characterId: GUIDE }] });
398
+ await client.updateCharacterRoster({ remove: [client.characters[0].membershipId] });
399
+
400
+ // Another participant joins the same room without sending any topology.
401
+ await other.joinRoom({ roomSessionId: client.roomSession!.roomSessionId, endUserId: "player-7" });
402
+ ```
403
+
404
+ Key points:
405
+
406
+ - `characters` and `characterId` are mutually exclusive, and roster entries take a bare character UUID (no `-draft` / `-1.2` selectors).
407
+ - `provisioningStatus` describes dispatch only; a member is usable when its own `characterReady` arrives.
408
+ - Transcript rows, `characterAudioTrack` events and `botReady` payloads carry `membershipId`, so two copies of one character never merge.
409
+ - Room-topology failures throw `ConvaiRoomError` with `status`, `code`, `requestTraceId` and `isRetryable`; the SDK retries `ROSTER_PROVISIONING_IN_PROGRESS` for you (`rosterProvisioningRetry`).
410
+ - Action protocol v2 and `actionConfig.tools` are single-character only and cannot be combined with a roster.
411
+
412
+ Full reference: [docs/convai_multi_character.md](docs/convai_multi_character.md).
413
+
414
+ ---
415
+
209
416
  ## Vision dynamic context beta
210
417
 
211
418
  Vision dynamic context is the default WebRTC/LiveKit vision path when `enableVideo: true`. Camera, screen, canvas, and custom video tracks can feed unified vision context; set `visionInputConfig.enabled: false` only when you need to keep the video channel while opting out.
@@ -273,6 +480,119 @@ client.on("actionResponse", ({ actions }: ActionResponseEvent) => {
273
480
  });
274
481
  ```
275
482
 
483
+ ### Agentic actions and canonical model output (v2)
484
+
485
+ The v2 protocol is an explicit opt-in. Existing connections continue to receive
486
+ the v1 `actionResponse` contract. A v2 client declares tools at connect time,
487
+ dispatches only Core-validated `items`, and returns one correlated terminal
488
+ result for every `tool_call`.
489
+
490
+ ```ts
491
+ import { ConvaiClient, type ActionResult } from "@convai/web-sdk/core";
492
+
493
+ const client = new ConvaiClient({
494
+ apiKey: "...",
495
+ characterId: "...",
496
+ capabilities: {
497
+ actionProtocolVersion: 2,
498
+ modelOutputVersion: 2,
499
+ },
500
+ actionConfig: {
501
+ actions: [],
502
+ objects: [],
503
+ characters: [],
504
+ tools: [
505
+ {
506
+ name: "search_documents",
507
+ description: "Search the user's approved document collection",
508
+ inputSchema: {
509
+ type: "object",
510
+ properties: { query: { type: "string" } },
511
+ required: ["query"],
512
+ },
513
+ },
514
+ ],
515
+ },
516
+ });
517
+
518
+ client.on("modelOutput", async (output) => {
519
+ for (const item of output.items) {
520
+ if (item.type === "message") renderAssistantText(item.content);
521
+ if (item.type !== "tool_call") continue;
522
+
523
+ let terminalResult: ActionResult;
524
+ try {
525
+ const result = await executeApprovedTool(item.name, item.arguments);
526
+ terminalResult = {
527
+ id: item.id,
528
+ status: "completed",
529
+ output: result,
530
+ };
531
+ } catch (error) {
532
+ terminalResult = {
533
+ id: item.id,
534
+ status: "error",
535
+ error: { message: String(error) },
536
+ };
537
+ }
538
+
539
+ try {
540
+ const ack = await client.sendActionResultAndWait(terminalResult, {
541
+ timeoutMs: 10_000,
542
+ });
543
+ markResultReported(item.id, ack.idempotent);
544
+ } catch (deliveryError) {
545
+ markResultDeliveryUncertain(item.id, deliveryError);
546
+ // If appropriate for your session lifecycle, retry `terminalResult`
547
+ // unchanged. Never execute the tool again merely because its ACK was lost.
548
+ }
549
+ }
550
+ });
551
+ ```
552
+
553
+ ### Parallel raw provider output (v2)
554
+
555
+ Model output v2 may provide a separate diagnostics-only raw stream alongside
556
+ the unchanged `bot-llm-text` chat stream. Each chunk carries Core's stable
557
+ logical-turn correlation id and, on newer Core versions, an invocation id:
558
+
559
+ ```ts
560
+ const client = new ConvaiClient({
561
+ apiKey: "...",
562
+ characterId: "...",
563
+ capabilities: { modelOutputVersion: 2 },
564
+ });
565
+
566
+ client.on("botLlmTextRaw", ({ text, logical_turn_id, output_id }) => {
567
+ appendRawDiagnostics(output_id ?? logical_turn_id, text);
568
+ });
569
+ ```
570
+
571
+ The SDK ignores this channel unless Core negotiated `modelOutputVersion: 2`.
572
+ Raw chunks never enter `chatMessages`, bot output, TTS, speech, or response
573
+ lifecycle state. They are retained for diagnostics and must never be executed
574
+ or reparsed. The ordinary `bot-llm-text` text and cadence remain unchanged. On
575
+ v2 connections, `ChatMessage.outputId` matches the raw event's optional
576
+ `output_id` and the final `modelOutput.output_id`; `logicalTurnId`/
577
+ `logical_turn_id` still groups every invocation in the user turn. Older v2 Core
578
+ omits the invocation id, so applications can fall back to one turn-level raw
579
+ aggregate. Never correlate by arrival order. When v2 is selected, the SDK suppresses the legacy `actionResponse` compatibility
580
+ projection so an application cannot execute the same operation twice.
581
+
582
+ Developer-defined visual output uses data-only `extension` items. Applications
583
+ register a renderer for a specific schema and version; unknown schemas use the
584
+ server-provided text fallback or remain unhandled. The SDK never evaluates code
585
+ from a model output.
586
+
587
+ ```ts
588
+ import { ModelOutputExtensionRegistry } from "@convai/web-sdk/core";
589
+
590
+ const extensions = new ModelOutputExtensionRegistry<HTMLElement>();
591
+ extensions.register("com.example/status-card", 1, ({ payload }) =>
592
+ renderStatusCard(payload),
593
+ );
594
+ ```
595
+
276
596
  ---
277
597
 
278
598
  ## Narrative Design template keys
@@ -0,0 +1,84 @@
1
+ import type { ActiveCharacterChange, CharacterRosterChange, ConvaiCharacterInstance, ConvaiRoomSession, ServerResponseExtras, UpdateCharacterRosterOptions } from "./types.js";
2
+ export declare class CharacterRoster {
3
+ private _characters;
4
+ private _activeMembershipId;
5
+ private _roomSessionId;
6
+ private _roomName;
7
+ private _requestTraceId;
8
+ private _routeEpoch;
9
+ private _rosterEpoch;
10
+ private _partialDispatch;
11
+ /** Character instances in roster order. Empty in single-character mode. */
12
+ get characters(): ConvaiCharacterInstance[];
13
+ /** Membership currently receiving user turns, or null. */
14
+ get activeMembershipId(): string | null;
15
+ get roomSessionId(): string | null;
16
+ get routeEpoch(): number;
17
+ get rosterEpoch(): number;
18
+ /** True once a multi-character connect response has been applied. */
19
+ get isMultiCharacter(): boolean;
20
+ /** The server-owned snapshot, or null for a single-character session. */
21
+ get roomSession(): ConvaiRoomSession | null;
22
+ find(membershipId: string): ConvaiCharacterInstance | undefined;
23
+ /** Members that have sent their own `bot-ready`. */
24
+ get readyCharacters(): ConvaiCharacterInstance[];
25
+ /**
26
+ * Mark a member ready from its own `bot-ready`, filling in any detail the
27
+ * connect response did not yet have.
28
+ *
29
+ * Returns whether anything changed, so the caller only pushes state when it
30
+ * did. A single-character `bot-ready` also carries these fields, so nothing
31
+ * is tracked until a roster is actually seeded.
32
+ */
33
+ applyBotReady(about: Record<string, unknown> | undefined): boolean;
34
+ /**
35
+ * Apply a `character-status` transition. Readiness recorded here is
36
+ * bookkeeping for replacement selection; `bot-ready` remains what gates
37
+ * interaction.
38
+ */
39
+ applyCharacterStatus(status: {
40
+ membershipId: string;
41
+ status: "starting" | "ready" | "failed";
42
+ failureCode: string | null;
43
+ }): boolean;
44
+ /**
45
+ * Pick a member to take over routing when the active one is removed.
46
+ *
47
+ * The runtime refuses a replacement that has not sent `bot-ready`
48
+ * (`replacement_target_unavailable`), so ready survivors come first. Falling
49
+ * back to any survivor at least produces a nameable rejection rather than
50
+ * `replacement_target_required`.
51
+ */
52
+ pickReplacement(removing: string[]): string | undefined;
53
+ /** Payload that switches the active member. */
54
+ buildInteractionTarget(membershipId: string | null, commandId: string): Record<string, unknown>;
55
+ /** Payload that adds and removes members. */
56
+ buildRosterUpdate(options: UpdateCharacterRosterOptions, commandId: string): Record<string, unknown>;
57
+ /**
58
+ * Adopt the server's authoritative epochs and active member from any
59
+ * response, errors included.
60
+ *
61
+ * A `stale_route_epoch` rejection carries the real epoch, and adopting it is
62
+ * precisely what makes the single retry meaningful rather than a repeat of
63
+ * the same losing bet.
64
+ */
65
+ adoptEpochs(extras: ServerResponseExtras | null | undefined): void;
66
+ /**
67
+ * Apply a successful `interaction-target` ack. Returns the change even when
68
+ * the target did not move, so a caller awaiting the promise always learns the
69
+ * authoritative state.
70
+ */
71
+ applyInteractionTarget(extras: ServerResponseExtras | null | undefined): ActiveCharacterChange;
72
+ /** Apply a successful `character-roster-update` ack, mutating the roster. */
73
+ applyRosterUpdate(extras: ServerResponseExtras | null | undefined): CharacterRosterChange;
74
+ reset(): void;
75
+ /**
76
+ * Seed the roster from a `/connect` response, for either a create or a join.
77
+ *
78
+ * Returns false for a response with no `characters` array — that is an
79
+ * ordinary single-character session, and the roster stays empty so
80
+ * `isMultiCharacter` keeps distinguishing the two.
81
+ */
82
+ applyConnectResponse(data: Record<string, unknown>): boolean;
83
+ }
84
+ //# sourceMappingURL=CharacterRoster.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"CharacterRoster.d.ts","sourceRoot":"","sources":["../../src/core/CharacterRoster.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,qBAAqB,EAErB,qBAAqB,EACrB,uBAAuB,EAEvB,iBAAiB,EACjB,oBAAoB,EACpB,4BAA4B,EAC7B,MAAM,SAAS,CAAC;AA8EjB,qBAAa,eAAe;IAC1B,OAAO,CAAC,WAAW,CAAiC;IACpD,OAAO,CAAC,mBAAmB,CAAuB;IAClD,OAAO,CAAC,cAAc,CAAuB;IAC7C,OAAO,CAAC,SAAS,CAAuB;IACxC,OAAO,CAAC,eAAe,CAAuB;IAC9C,OAAO,CAAC,WAAW,CAAK;IACxB,OAAO,CAAC,YAAY,CAAK;IACzB,OAAO,CAAC,gBAAgB,CAAS;IAEjC,2EAA2E;IAC3E,IAAI,UAAU,IAAI,uBAAuB,EAAE,CAE1C;IAED,0DAA0D;IAC1D,IAAI,kBAAkB,IAAI,MAAM,GAAG,IAAI,CAEtC;IAED,IAAI,aAAa,IAAI,MAAM,GAAG,IAAI,CAEjC;IAED,IAAI,UAAU,IAAI,MAAM,CAEvB;IAED,IAAI,WAAW,IAAI,MAAM,CAExB;IAED,qEAAqE;IACrE,IAAI,gBAAgB,IAAI,OAAO,CAE9B;IAED,yEAAyE;IACzE,IAAI,WAAW,IAAI,iBAAiB,GAAG,IAAI,CAY1C;IAED,IAAI,CAAC,YAAY,EAAE,MAAM,GAAG,uBAAuB,GAAG,SAAS;IAI/D,oDAAoD;IACpD,IAAI,eAAe,IAAI,uBAAuB,EAAE,CAE/C;IAED;;;;;;;OAOG;IACH,aAAa,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,GAAG,OAAO;IAyClE;;;;OAIG;IACH,oBAAoB,CAAC,MAAM,EAAE;QAC3B,YAAY,EAAE,MAAM,CAAC;QACrB,MAAM,EAAE,UAAU,GAAG,OAAO,GAAG,QAAQ,CAAC;QACxC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;KAC5B,GAAG,OAAO;IAiCX;;;;;;;OAOG;IACH,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,GAAG,SAAS;IAUvD,+CAA+C;IAC/C,sBAAsB,CACpB,YAAY,EAAE,MAAM,GAAG,IAAI,EAC3B,SAAS,EAAE,MAAM,GAChB,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;IAS1B,6CAA6C;IAC7C,iBAAiB,CACf,OAAO,EAAE,4BAA4B,EACrC,SAAS,EAAE,MAAM,GAChB,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;IAmB1B;;;;;;;OAOG;IACH,WAAW,CAAC,MAAM,EAAE,oBAAoB,GAAG,IAAI,GAAG,SAAS,GAAG,IAAI;IASlE;;;;OAIG;IACH,sBAAsB,CACpB,MAAM,EAAE,oBAAoB,GAAG,IAAI,GAAG,SAAS,GAC9C,qBAAqB;IAaxB,6EAA6E;IAC7E,iBAAiB,CACf,MAAM,EAAE,oBAAoB,GAAG,IAAI,GAAG,SAAS,GAC9C,qBAAqB;IAuDxB,KAAK,IAAI,IAAI;IAWb;;;;;;OAMG;IACH,oBAAoB,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO;CA2B7D"}