@convai/web-sdk 1.8.0-beta.2 → 1.8.0-beta.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -0
- package/dist/core/AudioManager.d.ts +3 -3
- package/dist/core/AudioManager.js +1 -1
- package/dist/core/ConnectionStateHandler.d.ts +1 -1
- package/dist/core/ConvaiClient.d.ts +6 -5
- package/dist/core/ConvaiClient.d.ts.map +1 -1
- package/dist/core/ConvaiClient.js +126 -43
- package/dist/core/ConvaiClient.js.map +1 -1
- package/dist/core/MemoryManager.d.ts +1 -1
- package/dist/core/MessageHandler.d.ts +4 -3
- package/dist/core/MessageHandler.d.ts.map +1 -1
- package/dist/core/MessageHandler.js +28 -11
- package/dist/core/MessageHandler.js.map +1 -1
- package/dist/core/SSESession.d.ts +44 -0
- package/dist/core/SSESession.d.ts.map +1 -0
- package/dist/core/SSESession.js +200 -0
- package/dist/core/SSESession.js.map +1 -0
- package/dist/core/ScreenShareManager.d.ts +2 -2
- package/dist/core/ScreenShareManager.js +2 -2
- package/dist/core/VideoManager.d.ts +2 -2
- package/dist/core/VideoManager.js +3 -3
- package/dist/core/connectRequest.d.ts +3 -1
- package/dist/core/connectRequest.d.ts.map +1 -1
- package/dist/core/connectRequest.js +27 -0
- package/dist/core/connectRequest.js.map +1 -1
- package/dist/core/contextUpdateRequest.d.ts +1 -1
- package/dist/core/index.d.ts +12 -12
- package/dist/core/index.js +10 -10
- package/dist/core/logicalTurn.d.ts +5 -0
- package/dist/core/logicalTurn.d.ts.map +1 -0
- package/dist/core/logicalTurn.js +10 -0
- package/dist/core/logicalTurn.js.map +1 -0
- package/dist/core/types.d.ts +14 -5
- package/dist/core/types.d.ts.map +1 -1
- package/dist/core/types.js.map +1 -1
- package/dist/core/videoSourceState.d.ts +1 -1
- package/dist/core/visionRequest.d.ts +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.js +2 -2
- package/dist/lipsync-helpers/createBlendshapeQueue.d.ts +2 -2
- package/dist/lipsync-helpers/createBlendshapeQueue.js +2 -2
- package/dist/lipsync-helpers/declarativeMapping.d.ts +1 -1
- package/dist/lipsync-helpers/declarativeMapping.js +2 -2
- package/dist/lipsync-helpers/index.d.ts +12 -12
- package/dist/lipsync-helpers/index.js +9 -9
- package/dist/lipsync-helpers/mhaLimitDrivers.js +2 -2
- package/dist/lipsync-helpers/naturalness/processor.d.ts +1 -1
- package/dist/lipsync-helpers/naturalness/processor.js +1 -1
- package/dist/lipsync-helpers/naturalness/profiles/mha.d.ts +1 -1
- package/dist/lipsync-helpers/naturalness/profiles/mha.js +1 -1
- package/dist/lipsync-helpers/presetMappers.d.ts +1 -1
- package/dist/react/components/ConvaiWidget.d.ts +1 -1
- package/dist/react/components/ConvaiWidget.js +4 -4
- package/dist/react/components/index.d.ts +1 -1
- package/dist/react/components/index.js +1 -1
- package/dist/react/components/rtc-widget/components/AudioSettingsPanel.d.ts +1 -1
- package/dist/react/components/rtc-widget/components/ConviMessage.js +3 -3
- package/dist/react/components/rtc-widget/components/FloatingVideo.js +3 -3
- package/dist/react/components/rtc-widget/components/MessageBubble.js +1 -1
- package/dist/react/components/rtc-widget/components/MessageList.d.ts +1 -1
- package/dist/react/components/rtc-widget/components/MessageList.js +4 -4
- package/dist/react/components/rtc-widget/components/UserMessage.js +3 -3
- package/dist/react/components/rtc-widget/components/conviComponents/ConviButton.js +4 -4
- package/dist/react/components/rtc-widget/components/conviComponents/ConviFooter.js +3 -3
- package/dist/react/components/rtc-widget/components/conviComponents/ConviHeader.js +4 -4
- package/dist/react/components/rtc-widget/components/conviComponents/SettingsTray.js +3 -3
- package/dist/react/components/rtc-widget/components/conviComponents/VoiceModeOverlay.js +3 -3
- package/dist/react/components/rtc-widget/components/conviComponents/index.d.ts +5 -5
- package/dist/react/components/rtc-widget/components/conviComponents/index.js +5 -5
- package/dist/react/components/rtc-widget/components/index.d.ts +8 -8
- package/dist/react/components/rtc-widget/components/index.js +9 -9
- package/dist/react/components/rtc-widget/index.d.ts +5 -5
- package/dist/react/components/rtc-widget/index.js +3 -3
- package/dist/react/components/rtc-widget/styles/icons.d.ts +1 -1
- package/dist/react/components/rtc-widget/styles/icons.js +1 -1
- package/dist/react/components/rtc-widget/styles/index.d.ts +5 -5
- package/dist/react/components/rtc-widget/styles/index.js +4 -4
- package/dist/react/components/rtc-widget/styles/styledComponents.js +1 -1
- package/dist/react/hooks/index.d.ts +3 -3
- package/dist/react/hooks/index.js +3 -3
- package/dist/react/hooks/useCharacterInfo.js +1 -1
- package/dist/react/hooks/useConvaiClient.d.ts +1 -1
- package/dist/react/hooks/useConvaiClient.js +1 -1
- package/dist/react/index.d.ts +6 -6
- package/dist/react/index.js +6 -6
- package/dist/vanilla/AudioRenderer.js +1 -1
- package/dist/vanilla/ConvaiWidget.d.ts +1 -1
- package/dist/vanilla/ConvaiWidget.js +4 -4
- package/dist/vanilla/WebSocketSession.d.ts +1 -1
- package/dist/vanilla/WebSocketSession.js +1 -1
- package/dist/vanilla/index.d.ts +5 -5
- package/dist/vanilla/index.js +4 -4
- package/dist/vanilla/types.d.ts +1 -1
- package/dist/vanilla/websocket.d.ts +1 -1
- package/dist/vanilla/websocket.js +2 -2
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/version.js.map +1 -1
- package/package.json +5 -4
package/README.md
CHANGED
|
@@ -113,6 +113,7 @@ Full documentation is at **<a href="https://docs.convai.com/api-docs/plugins-and
|
|
|
113
113
|
| <a href="https://docs.convai.com/api-docs/plugins-and-integrations/web-plugins/convai-web-sdk/error-handling" target="_blank" rel="noopener noreferrer">Error Handling</a> | `error`, `disconnect`, `serverResponse`, retry patterns |
|
|
114
114
|
| <a href="https://docs.convai.com/api-docs/plugins-and-integrations/web-plugins/convai-web-sdk/auth-tokens" target="_blank" rel="noopener noreferrer">Auth Tokens</a> | Server-side token exchange for production |
|
|
115
115
|
| <a href="https://docs.convai.com/api-docs/plugins-and-integrations/web-plugins/convai-web-sdk/websocket-transport-layer" target="_blank" rel="noopener noreferrer">WebSocket Transport</a> | Alternative transport for WebRTC-constrained environments |
|
|
116
|
+
| <a href="https://docs.convai.com/api-docs/plugins-and-integrations/web-plugins/convai-web-sdk" target="_blank" rel="noopener noreferrer">SSE Transport</a> | Text-only interaction transport; the one that runs under Node |
|
|
116
117
|
|
|
117
118
|
---
|
|
118
119
|
|
|
@@ -151,6 +152,35 @@ See the <a href="https://docs.convai.com/api-docs/plugins-and-integrations/web-p
|
|
|
151
152
|
|
|
152
153
|
---
|
|
153
154
|
|
|
155
|
+
## SSE interaction transport (beta)
|
|
156
|
+
|
|
157
|
+
Use the interaction API for text-only integrations that need a streamed response:
|
|
158
|
+
|
|
159
|
+
```ts
|
|
160
|
+
const client = new ConvaiClient({
|
|
161
|
+
apiKey: "...",
|
|
162
|
+
characterId: "...",
|
|
163
|
+
interactionApiUrl: "https://interaction-api-stg.convai.com/v1/interactions",
|
|
164
|
+
});
|
|
165
|
+
|
|
166
|
+
await client.connect();
|
|
167
|
+
client.sendUserTextMessage("Hello!");
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
This sends each message as an SSE `POST` with `Authorization: Bearer ...`. Set `transport: "sse"`
|
|
171
|
+
explicitly if preferred. SSE carries text only — no voice, video, or lipsync — and it never calls
|
|
172
|
+
`/connect`, so `connect()` makes no network request and the session id arrives on the first
|
|
173
|
+
interaction via the `characterSessionId` event.
|
|
174
|
+
|
|
175
|
+
Because it needs neither WebRTC nor a DOM, this is the only transport that runs outside a browser.
|
|
176
|
+
Import `@convai/web-sdk/core` from Node, SSR, or a serverless function — not the root or `/react`
|
|
177
|
+
entry points, which pull in browser-only dependencies.
|
|
178
|
+
|
|
179
|
+
The interaction API is currently reachable at staging only; a production host is planned.
|
|
180
|
+
Full guide: [SSE Interaction Transport](docs/convai_sse_transport.md).
|
|
181
|
+
|
|
182
|
+
---
|
|
183
|
+
|
|
154
184
|
## Vision dynamic context beta
|
|
155
185
|
|
|
156
186
|
Vision dynamic context is the default WebRTC/LiveKit vision path when `enableVideo: true`. Camera, screen, canvas, and custom video tracks can feed unified vision context; set `visionInputConfig.enabled: false` only when you need to keep the video channel while opting out.
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { Room } from "livekit-client";
|
|
2
|
-
import { AudioControls } from "./types";
|
|
3
|
-
import { EventEmitter } from "./EventEmitter";
|
|
4
|
-
import type { IWebSocketSession } from "./types";
|
|
2
|
+
import { AudioControls } from "./types.js";
|
|
3
|
+
import { EventEmitter } from "./EventEmitter.js";
|
|
4
|
+
import type { IWebSocketSession } from "./types.js";
|
|
5
5
|
/**
|
|
6
6
|
* Manages audio controls for LiveKit room
|
|
7
7
|
* Provides methods to enable/disable microphone, manage audio devices,
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { ConnectionState, DisconnectReason } from "livekit-client";
|
|
2
|
-
import { ConvaiClientState } from "./types";
|
|
2
|
+
import { ConvaiClientState } from "./types.js";
|
|
3
3
|
/**
|
|
4
4
|
* Handles connection state changes and disconnection logic for ConvaiClient.
|
|
5
5
|
* Centralizes error handling and state management for better maintainability.
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { Room } from "livekit-client";
|
|
2
|
-
import { ConvaiConfig, ConvaiClientState, ChatMessage, IConvaiClient, AudioControls, VideoControls, ScreenShareControls, DynamicInfo, ContextUpdateOptions, UploadFileOptions, WebSocketSessionFactory, RespondModeUpdateOptions, UpdateSceneMetadataOptions, VisionStatusOptions, VisionTriggerOptions, ConnectionData } from "./types";
|
|
3
|
-
import { EventEmitter } from "./EventEmitter";
|
|
4
|
-
import { BlendshapeQueue } from "./BlendshapeQueue";
|
|
5
|
-
import { MemoryManager } from "./MemoryManager";
|
|
2
|
+
import { ConvaiConfig, ConvaiClientState, ChatMessage, IConvaiClient, AudioControls, VideoControls, ScreenShareControls, DynamicInfo, ContextUpdateOptions, UploadFileOptions, WebSocketSessionFactory, RespondModeUpdateOptions, UpdateSceneMetadataOptions, VisionStatusOptions, VisionTriggerOptions, ConnectionData, SendUserTextMessageOptions } from "./types.js";
|
|
3
|
+
import { EventEmitter } from "./EventEmitter.js";
|
|
4
|
+
import { BlendshapeQueue } from "./BlendshapeQueue.js";
|
|
5
|
+
import { MemoryManager } from "./MemoryManager.js";
|
|
6
6
|
/**
|
|
7
7
|
* Main Convai client class for managing AI voice assistant connections
|
|
8
8
|
* Provides complete interface for connecting to Convai's voice assistants,
|
|
@@ -60,6 +60,7 @@ export declare class ConvaiClient extends EventEmitter implements IConvaiClient
|
|
|
60
60
|
private _endUserMetadata;
|
|
61
61
|
private _wsSession;
|
|
62
62
|
private _activeTransport;
|
|
63
|
+
private _sseSession;
|
|
63
64
|
private _logRtviMessages;
|
|
64
65
|
private _isDisconnecting;
|
|
65
66
|
private _clientReadyRetryTimer;
|
|
@@ -193,7 +194,7 @@ export declare class ConvaiClient extends EventEmitter implements IConvaiClient
|
|
|
193
194
|
/**
|
|
194
195
|
* Send a text message to the character
|
|
195
196
|
*/
|
|
196
|
-
sendUserTextMessage(text: string): void;
|
|
197
|
+
sendUserTextMessage(text: string, options?: SendUserTextMessageOptions): void;
|
|
197
198
|
/**
|
|
198
199
|
* Send a trigger message to invoke specific character actions
|
|
199
200
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ConvaiClient.d.ts","sourceRoot":"","sources":["../../src/core/ConvaiClient.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,IAAI,EAML,MAAM,gBAAgB,CAAC;AAGxB,OAAO,EACL,YAAY,EACZ,iBAAiB,EACjB,WAAW,EAEX,aAAa,EACb,aAAa,EACb,aAAa,EACb,mBAAmB,EACnB,WAAW,EACX,oBAAoB,EACpB,iBAAiB,EAIjB,uBAAuB,EACvB,wBAAwB,EAExB,0BAA0B,EAC1B,mBAAmB,EACnB,oBAAoB,EAGpB,cAAc,
|
|
1
|
+
{"version":3,"file":"ConvaiClient.d.ts","sourceRoot":"","sources":["../../src/core/ConvaiClient.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,IAAI,EAML,MAAM,gBAAgB,CAAC;AAGxB,OAAO,EACL,YAAY,EACZ,iBAAiB,EACjB,WAAW,EAEX,aAAa,EACb,aAAa,EACb,aAAa,EACb,mBAAmB,EACnB,WAAW,EACX,oBAAoB,EACpB,iBAAiB,EAIjB,uBAAuB,EACvB,wBAAwB,EAExB,0BAA0B,EAC1B,mBAAmB,EACnB,oBAAoB,EAGpB,cAAc,EACd,0BAA0B,EAC3B,MAAM,SAAS,CAAC;AAMjB,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAoChD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AACH,qBAAa,YAAa,SAAQ,YAAa,YAAW,aAAa;IACrE,OAAO,CAAC,KAAK,CAAO;IACpB,OAAO,CAAC,MAAM,CAAoB;IAClC,OAAO,CAAC,eAAe,CAAkC;IACzD,OAAO,CAAC,OAAO,CAAuB;IACtC,OAAO,CAAC,UAAU,CAAuB;IACzC,OAAO,CAAC,YAAY,CAAuB;IAC3C,OAAO,CAAC,mBAAmB,CAAuB;IAClD,OAAO,CAAC,WAAW,CAAkB;IACrC,OAAO,CAAC,eAAe,CAAc;IACrC,OAAO,CAAC,cAAc,CAAgB;IACtC,OAAO,CAAC,aAAa,CAA6B;IAClD;;;OAGG;IACH,OAAO,CAAC,YAAY,CAAuB;IAC3C,OAAO,CAAC,UAAU,CAAuB;IACzC,OAAO,CAAC,gBAAgB,CAAwC;IAChE,OAAO,CAAC,UAAU,CAAkC;IACpD,OAAO,CAAC,gBAAgB,CAAgD;IACxE,OAAO,CAAC,WAAW,CAA2B;IAC9C,OAAO,CAAC,gBAAgB,CAAiB;IACzC,OAAO,CAAC,gBAAgB,CAAkB;IAG1C,OAAO,CAAC,sBAAsB,CAA8C;IAC5E,OAAO,CAAC,wBAAwB,CAA8C;IAC9E,OAAO,CAAC,iBAAiB,CAAuB;IAChD,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,qBAAqB,CAAO;IACpD,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAuB,CAAU;IAEzD;;;;OAIG;IACH,MAAM,CAAC,0BAA0B,CAAC,OAAO,EAAE,uBAAuB,GAAG,IAAI;IAKzE,OAAO,CAAC,aAAa,CAAe;IACpC,OAAO,CAAC,aAAa,CAAe;IACpC,OAAO,CAAC,mBAAmB,CAAqB;IAChD,OAAO,CAAC,eAAe,CAAiB;IACxC,OAAO,CAAC,uBAAuB,CAAyB;IAGxD,OAAO,CAAC,sBAAsB,CAAa;IAC3C,OAAO,CAAC,sBAAsB,CAAa;IAG3C,OAAO,CAAC,cAAc,CAA8B;gBAExC,MAAM,CAAC,EAAE,YAAY;IA+DjC,IAAI,KAAK,IAAI,iBAAiB,CAE7B;IAED,IAAI,cAAc,IAAI,OAAO,GAAG,OAAO,GAAG,IAAI,CAE7C;IAED,IAAI,MAAM,IAAI,MAAM,GAAG,IAAI,CAE1B;IAED,IAAI,SAAS,IAAI,MAAM,GAAG,IAAI,CAE7B;IAED,IAAI,WAAW,IAAI,MAAM,GAAG,IAAI,CAE/B;IAED,IAAI,IAAI,IAAI,IAAI,CAEf;IAED,IAAI,YAAY,IAAI,WAAW,EAAE,CAEhC;IAED,IAAI,iBAAiB,IAAI,MAAM,CAE9B;IAED,IAAI,kBAAkB,IAAI,MAAM,GAAG,IAAI,CAEtC;IAED,IAAI,UAAU,IAAI,OAAO,CAExB;IAED,IAAI,aAAa,IAAI,aAAa,CAEjC;IAED,IAAI,aAAa,IAAI,aAAa,CAEjC;IAED,IAAI,mBAAmB,IAAI,mBAAmB,CAE7C;IAED,IAAI,eAAe,IAAI,eAAe,CAErC;IAED,IAAI,qBAAqB,IAAI,MAAM,CAElC;IAED,IAAI,aAAa,IAAI,aAAa,GAAG,IAAI,CAExC;IAED;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAqL3B;;;;OAIG;IACH,OAAO,CAAC,yBAAyB;IASjC;;OAEG;IACH,OAAO,CAAC,WAAW;IAuBnB;;;;OAIG;IACH,OAAO,CAAC,cAAc;IAkBtB;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAYxB;;OAEG;IACG,OAAO,CAAC,MAAM,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAmOnD;;;;;;;;;;;;;;;;;;OAkBG;IACG,yBAAyB,CAC7B,IAAI,EAAE,cAAc,EACpB,MAAM,CAAC,EAAE,YAAY,GACpB,OAAO,CAAC,IAAI,CAAC;IAwChB;;;;;;;;;;;OAWG;IACH,OAAO,CAAC,mBAAmB;IA4B3B;;;;OAIG;YACW,qBAAqB;YA2PrB,0CAA0C;IAgBxD;;;;;OAKG;IACH,OAAO,CAAC,0BAA0B;IAgDlC,OAAO,CAAC,yBAAyB;IAWjC;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAiCjC;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAkBxB;;OAEG;IACG,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IAOhC;;OAEG;IACH,YAAY,IAAI,IAAI;IAQpB;;OAEG;IACH,mBAAmB,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,0BAA+B,GAAG,IAAI;IAgEjF;;OAEG;IACH,kBAAkB,CAAC,WAAW,CAAC,EAAE,MAAM,EAAE,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI;IA2BvE;;;;OAIG;IACH,OAAO,CAAC,8BAA8B;IAmBtC,OAAO,CAAC,wBAAwB;IAkBhC;;OAEG;IACH,oBAAoB,IAAI,IAAI;IAwB5B;;;;OAIG;IACH,kBAAkB,CAAC,YAAY,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,GAAG,IAAI;IAUjE;;OAEG;IACH,iBAAiB,CAAC,WAAW,EAAE,WAAW,GAAG,IAAI;IASjD;;;;;;;;;;;;;;;;;;;;;;;;;OAyBG;IACH,aAAa,CAAC,OAAO,EAAE,oBAAoB,GAAG,IAAI;IAmClD,YAAY,CAAC,OAAO,GAAE,mBAAwB,GAAG,MAAM,GAAG,IAAI;IAU9D,aAAa,CAAC,OAAO,GAAE,oBAAyB,GAAG,MAAM,GAAG,IAAI;IAkBhE,iBAAiB,CAAC,OAAO,EAAE,wBAAwB,GAAG,MAAM,GAAG,IAAI;IAYnE;;;OAGG;IACH,aAAa,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,GAAG,IAAI;IAiB/C;;;OAGG;IACH,mBAAmB,CACjB,KAAK,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC,EACnD,OAAO,GAAE,0BAA+B,GACvC,IAAI;IAaP,OAAO,CAAC,cAAc;IAOtB,OAAO,CAAC,qBAAqB;IAe7B;;;;;OAKG;IACG,UAAU,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,GAAE,iBAAsB,GAAG,OAAO,CAAC,IAAI,CAAC;IAgB5E;;OAEG;IACH,SAAS,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI;IAKjC;;OAEG;IACH,SAAS,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI;IAKjC;;;;OAIG;IACH,cAAc,IAAI,IAAI;CASvB"}
|
|
@@ -1,17 +1,19 @@
|
|
|
1
1
|
import { Room, RoomEvent, DefaultReconnectPolicy, setLogLevel, } from "livekit-client";
|
|
2
2
|
setLogLevel("warn");
|
|
3
|
-
import { DisconnectReasonEnum, } from "./types";
|
|
4
|
-
import { AudioManager } from "./AudioManager";
|
|
5
|
-
import { VideoManager } from "./VideoManager";
|
|
6
|
-
import { ScreenShareManager } from "./ScreenShareManager";
|
|
7
|
-
import { MessageHandler } from "./MessageHandler";
|
|
8
|
-
import { SDK_VERSION } from "../version";
|
|
9
|
-
import { EventEmitter } from "./EventEmitter";
|
|
10
|
-
import { MemoryManager } from "./MemoryManager";
|
|
11
|
-
import { ConnectionStateHandler } from "./ConnectionStateHandler";
|
|
12
|
-
import { buildBlendshapeConnectConfig } from "./connectRequest";
|
|
13
|
-
import { shouldPreemptForContextUpdateResponse, shouldPreemptForExplicitRespondMode, } from "./contextUpdateRequest";
|
|
14
|
-
import { resolveVisionInputConfig, serializeVisionInputConfig, } from "./visionRequest";
|
|
3
|
+
import { DisconnectReasonEnum, } from "./types.js";
|
|
4
|
+
import { AudioManager } from "./AudioManager.js";
|
|
5
|
+
import { VideoManager } from "./VideoManager.js";
|
|
6
|
+
import { ScreenShareManager } from "./ScreenShareManager.js";
|
|
7
|
+
import { MessageHandler } from "./MessageHandler.js";
|
|
8
|
+
import { SDK_VERSION } from "../version.js";
|
|
9
|
+
import { EventEmitter } from "./EventEmitter.js";
|
|
10
|
+
import { MemoryManager } from "./MemoryManager.js";
|
|
11
|
+
import { ConnectionStateHandler } from "./ConnectionStateHandler.js";
|
|
12
|
+
import { buildActionConnectConfig, buildBlendshapeConnectConfig, buildEmotionConnectConfig, } from "./connectRequest.js";
|
|
13
|
+
import { shouldPreemptForContextUpdateResponse, shouldPreemptForExplicitRespondMode, } from "./contextUpdateRequest.js";
|
|
14
|
+
import { resolveVisionInputConfig, serializeVisionInputConfig, } from "./visionRequest.js";
|
|
15
|
+
import { SSESession } from "./SSESession.js";
|
|
16
|
+
import { isValidLogicalTurnId } from "./logicalTurn.js";
|
|
15
17
|
// Module-level registry — populated by importing @convai/web-sdk/vanilla/websocket
|
|
16
18
|
let _wsFactory = null;
|
|
17
19
|
function normalizeStateOfMind(value) {
|
|
@@ -92,6 +94,7 @@ export class ConvaiClient extends EventEmitter {
|
|
|
92
94
|
this._endUserMetadata = null;
|
|
93
95
|
this._wsSession = null;
|
|
94
96
|
this._activeTransport = null;
|
|
97
|
+
this._sseSession = null;
|
|
95
98
|
this._logRtviMessages = true;
|
|
96
99
|
this._isDisconnecting = false;
|
|
97
100
|
// RTVI client-ready handshake (race-proof bot-ready delivery)
|
|
@@ -377,8 +380,12 @@ export class ConvaiClient extends EventEmitter {
|
|
|
377
380
|
* For LiveKit: publishes via the data channel.
|
|
378
381
|
* For WebSocket: sends via PipecatClient.sendClientMessage (wrapped in {t,d} envelope).
|
|
379
382
|
*/
|
|
380
|
-
publishMessage(type, data) {
|
|
381
|
-
const message = { type, data };
|
|
383
|
+
publishMessage(type, data, id) {
|
|
384
|
+
const message = id === undefined ? { type, data } : { type, id, data };
|
|
385
|
+
if (this._activeTransport === "sse" && this._sseSession?.isConnected) {
|
|
386
|
+
this._sseSession.sendMessage(type, data, id);
|
|
387
|
+
return;
|
|
388
|
+
}
|
|
382
389
|
if (this._activeTransport === "websocket" && this._wsSession?.isConnected) {
|
|
383
390
|
this._wsSession.sendMessage(type, data);
|
|
384
391
|
}
|
|
@@ -392,6 +399,9 @@ export class ConvaiClient extends EventEmitter {
|
|
|
392
399
|
* Returns true when a message can be sent on the active transport.
|
|
393
400
|
*/
|
|
394
401
|
isTransportReady() {
|
|
402
|
+
if (this._activeTransport === "sse") {
|
|
403
|
+
return this._sseSession?.isConnected ?? false;
|
|
404
|
+
}
|
|
395
405
|
if (this._activeTransport === "websocket") {
|
|
396
406
|
return this._wsSession?.isConnected ?? false;
|
|
397
407
|
}
|
|
@@ -423,16 +433,67 @@ export class ConvaiClient extends EventEmitter {
|
|
|
423
433
|
// Determine connection type based on enableVideo
|
|
424
434
|
const connType = configWithDefaults.enableVideo ? "video" : "audio";
|
|
425
435
|
this._connectionType = connType;
|
|
426
|
-
const transportType = configWithDefaults.transport ??
|
|
436
|
+
const transportType = configWithDefaults.transport ??
|
|
437
|
+
(configWithDefaults.interactionApiUrl ? "sse" : "livekit");
|
|
438
|
+
if (transportType === "sse" && !configWithDefaults.interactionApiUrl) {
|
|
439
|
+
throw new Error("interactionApiUrl is required when using the SSE transport");
|
|
440
|
+
}
|
|
441
|
+
if (transportType === "sse" && configWithDefaults.enableVideo) {
|
|
442
|
+
throw new Error("The SSE transport supports text interactions only");
|
|
443
|
+
}
|
|
427
444
|
this._activeTransport = transportType;
|
|
428
445
|
const effectiveVisionInputConfig = resolveVisionInputConfig(configWithDefaults);
|
|
429
|
-
// Prepare request body with required parameters
|
|
430
446
|
// An explicit stateOfMind in the config wins; otherwise carry whatever
|
|
431
447
|
// updateEmotion() last set, including before the first connect.
|
|
432
448
|
const stateOfMind = configWithDefaults.stateOfMind !== undefined
|
|
433
449
|
? normalizeStateOfMind(configWithDefaults.stateOfMind) ?? null
|
|
434
450
|
: this._stateOfMind;
|
|
435
451
|
this._stateOfMind = stateOfMind;
|
|
452
|
+
if (transportType === "sse") {
|
|
453
|
+
this._sseSession = new SSESession({
|
|
454
|
+
interactionApiUrl: configWithDefaults.interactionApiUrl,
|
|
455
|
+
authorization: configWithDefaults.authToken ?? configWithDefaults.apiKey,
|
|
456
|
+
characterId: configWithDefaults.characterId,
|
|
457
|
+
getCharacterSessionId: () => this._characterSessionId,
|
|
458
|
+
takeStateOfMind: () => {
|
|
459
|
+
const stateOfMind = this._stateOfMind;
|
|
460
|
+
this._stateOfMind = null;
|
|
461
|
+
if (this._storedConfig?.stateOfMind !== undefined) {
|
|
462
|
+
this._storedConfig = { ...this._storedConfig, stateOfMind: null };
|
|
463
|
+
}
|
|
464
|
+
return stateOfMind;
|
|
465
|
+
},
|
|
466
|
+
onCharacterSessionId: (characterSessionId) => {
|
|
467
|
+
this._characterSessionId = characterSessionId;
|
|
468
|
+
this.emit("characterSessionId", characterSessionId);
|
|
469
|
+
this._storedConfig = {
|
|
470
|
+
...this._storedConfig,
|
|
471
|
+
characterSessionId,
|
|
472
|
+
};
|
|
473
|
+
},
|
|
474
|
+
endUserId: configWithDefaults.endUserId,
|
|
475
|
+
endUserMetadata: configWithDefaults.endUserMetadata,
|
|
476
|
+
enableEmotion: configWithDefaults.enableEmotion,
|
|
477
|
+
emotionConfig: configWithDefaults.emotionConfig,
|
|
478
|
+
actionConfig: configWithDefaults.actionConfig,
|
|
479
|
+
debug: configWithDefaults.debug,
|
|
480
|
+
onMessage: (payload) => this._messageHandler.handleDataReceivedPublic(payload),
|
|
481
|
+
onError: (error) => this.emit("error", error),
|
|
482
|
+
});
|
|
483
|
+
this._sseSession.connect();
|
|
484
|
+
this._connectionType = "audio";
|
|
485
|
+
this._isBotReady = true;
|
|
486
|
+
this.updateState({
|
|
487
|
+
isConnected: true,
|
|
488
|
+
isConnecting: false,
|
|
489
|
+
endUserId: configWithDefaults.endUserId ?? null,
|
|
490
|
+
endUserMetadata: configWithDefaults.endUserMetadata ?? null,
|
|
491
|
+
});
|
|
492
|
+
this.emit("connect");
|
|
493
|
+
this.emit("botReady");
|
|
494
|
+
return;
|
|
495
|
+
}
|
|
496
|
+
// Prepare request body with required parameters
|
|
436
497
|
const characterSessionIdToSend = configWithDefaults.characterSessionId ?? this._characterSessionId;
|
|
437
498
|
const requestBody = {
|
|
438
499
|
character_id: configWithDefaults.characterId,
|
|
@@ -447,20 +508,8 @@ export class ConvaiClient extends EventEmitter {
|
|
|
447
508
|
connection_type: connType,
|
|
448
509
|
...buildBlendshapeConnectConfig(configWithDefaults),
|
|
449
510
|
llm_provider: "dynamic",
|
|
450
|
-
...(configWithDefaults
|
|
451
|
-
emotion_config: (
|
|
452
|
-
const cfg = configWithDefaults.emotionConfig;
|
|
453
|
-
const provider = cfg?.provider ?? "llm";
|
|
454
|
-
if (provider === "nrclex") {
|
|
455
|
-
return {
|
|
456
|
-
provider,
|
|
457
|
-
min_word_threshold: cfg.min_word_threshold ?? 3,
|
|
458
|
-
low_intensity_threshold: cfg.low_intensity_threshold ?? 0.33,
|
|
459
|
-
high_intensity_threshold: cfg.high_intensity_threshold ?? 0.66,
|
|
460
|
-
};
|
|
461
|
-
}
|
|
462
|
-
return { provider };
|
|
463
|
-
})(),
|
|
511
|
+
...(buildEmotionConnectConfig(configWithDefaults) && {
|
|
512
|
+
emotion_config: buildEmotionConnectConfig(configWithDefaults),
|
|
464
513
|
}),
|
|
465
514
|
default_tts_enabled: configWithDefaults.ttsEnabled !== undefined
|
|
466
515
|
? configWithDefaults.ttsEnabled
|
|
@@ -476,15 +525,8 @@ export class ConvaiClient extends EventEmitter {
|
|
|
476
525
|
characterSessionIdToSend !== "-1" && {
|
|
477
526
|
character_session_id: characterSessionIdToSend,
|
|
478
527
|
}),
|
|
479
|
-
...(configWithDefaults
|
|
480
|
-
action_config:
|
|
481
|
-
actions: configWithDefaults.actionConfig.actions,
|
|
482
|
-
objects: configWithDefaults.actionConfig.objects,
|
|
483
|
-
characters: configWithDefaults.actionConfig.characters,
|
|
484
|
-
...(configWithDefaults.actionConfig.current_attention_object && {
|
|
485
|
-
current_attention_object: configWithDefaults.actionConfig.current_attention_object,
|
|
486
|
-
}),
|
|
487
|
-
},
|
|
528
|
+
...(buildActionConnectConfig(configWithDefaults) && {
|
|
529
|
+
action_config: buildActionConnectConfig(configWithDefaults),
|
|
488
530
|
}),
|
|
489
531
|
...(configWithDefaults.dynamicInfo && {
|
|
490
532
|
dynamic_info: {
|
|
@@ -684,6 +726,10 @@ export class ConvaiClient extends EventEmitter {
|
|
|
684
726
|
// Capture session identifiers from /connect response
|
|
685
727
|
if (connectionData.character_session_id) {
|
|
686
728
|
this._characterSessionId = connectionData.character_session_id;
|
|
729
|
+
// Emitted here rather than in connect(), so the proxy-auth path
|
|
730
|
+
// (connectWithConnectionData) surfaces the session id too -- both routes
|
|
731
|
+
// establish it at exactly this point.
|
|
732
|
+
this.emit("characterSessionId", connectionData.character_session_id);
|
|
687
733
|
this._storedConfig = {
|
|
688
734
|
...this._storedConfig,
|
|
689
735
|
characterSessionId: connectionData.character_session_id,
|
|
@@ -926,7 +972,12 @@ export class ConvaiClient extends EventEmitter {
|
|
|
926
972
|
*/
|
|
927
973
|
async disconnect() {
|
|
928
974
|
this._stopClientReadyHandshake();
|
|
929
|
-
if (this._activeTransport === "
|
|
975
|
+
if (this._activeTransport === "sse" && this._sseSession) {
|
|
976
|
+
await this._sseSession.disconnect();
|
|
977
|
+
this._connectionStateHandler.resetConnectionState();
|
|
978
|
+
this.clearAllManagers();
|
|
979
|
+
}
|
|
980
|
+
else if (this._activeTransport === "websocket" && this._wsSession) {
|
|
930
981
|
this._isDisconnecting = true;
|
|
931
982
|
try {
|
|
932
983
|
await this._wsSession.disconnect();
|
|
@@ -961,6 +1012,7 @@ export class ConvaiClient extends EventEmitter {
|
|
|
961
1012
|
this._connectionType = null;
|
|
962
1013
|
this._activeTransport = null;
|
|
963
1014
|
this._wsSession = null;
|
|
1015
|
+
this._sseSession = null;
|
|
964
1016
|
this._apiKey = null;
|
|
965
1017
|
this._authToken = null;
|
|
966
1018
|
this._characterId = null;
|
|
@@ -987,6 +1039,7 @@ export class ConvaiClient extends EventEmitter {
|
|
|
987
1039
|
*/
|
|
988
1040
|
resetSession() {
|
|
989
1041
|
this._characterSessionId = "-1";
|
|
1042
|
+
this.emit("characterSessionId", "-1");
|
|
990
1043
|
this._messageHandler.reset();
|
|
991
1044
|
// Clear metrics and disconnect reason when resetting session
|
|
992
1045
|
this.updateState({ metrics: [], disconnectReason: null });
|
|
@@ -994,11 +1047,15 @@ export class ConvaiClient extends EventEmitter {
|
|
|
994
1047
|
/**
|
|
995
1048
|
* Send a text message to the character
|
|
996
1049
|
*/
|
|
997
|
-
sendUserTextMessage(text) {
|
|
1050
|
+
sendUserTextMessage(text, options = {}) {
|
|
998
1051
|
if (!this.isTransportReady())
|
|
999
1052
|
return;
|
|
1000
1053
|
if (!text || !text.trim())
|
|
1001
1054
|
return;
|
|
1055
|
+
const logicalTurnId = options?.logicalTurnId;
|
|
1056
|
+
if (logicalTurnId !== undefined && !isValidLogicalTurnId(logicalTurnId)) {
|
|
1057
|
+
throw new Error("logicalTurnId must be non-empty and at most 128 UTF-8 bytes");
|
|
1058
|
+
}
|
|
1002
1059
|
try {
|
|
1003
1060
|
// Reset blendshape queue if mid-conversation, but don't interrupt the server
|
|
1004
1061
|
if (this.blendshapeQueue.isConversationActive() ||
|
|
@@ -1015,10 +1072,32 @@ export class ConvaiClient extends EventEmitter {
|
|
|
1015
1072
|
userMessage: text.trim(),
|
|
1016
1073
|
timestamp: this._conversationStartTime,
|
|
1017
1074
|
});
|
|
1018
|
-
|
|
1075
|
+
const userTextMessage = {
|
|
1019
1076
|
text: text.trim(),
|
|
1020
1077
|
participant_sid: this._participantSid || this._room.localParticipant?.sid,
|
|
1021
|
-
}
|
|
1078
|
+
};
|
|
1079
|
+
const stateOfMind = normalizeStateOfMind(options.stateOfMind) ?? null;
|
|
1080
|
+
if (stateOfMind)
|
|
1081
|
+
this._stateOfMind = stateOfMind;
|
|
1082
|
+
if (stateOfMind && this._activeTransport === "sse") {
|
|
1083
|
+
this._sseSession?.sendMessages([
|
|
1084
|
+
{ type: "update-emotion", data: { state_of_mind: stateOfMind } },
|
|
1085
|
+
{
|
|
1086
|
+
type: "user_text_message",
|
|
1087
|
+
data: userTextMessage,
|
|
1088
|
+
...(logicalTurnId !== undefined ? { id: logicalTurnId } : {}),
|
|
1089
|
+
},
|
|
1090
|
+
]);
|
|
1091
|
+
}
|
|
1092
|
+
else {
|
|
1093
|
+
if (stateOfMind) {
|
|
1094
|
+
this.publishMessage("update-emotion", {
|
|
1095
|
+
state_of_mind: stateOfMind,
|
|
1096
|
+
participant_sid: this._participantSid || this._room.localParticipant?.sid,
|
|
1097
|
+
});
|
|
1098
|
+
}
|
|
1099
|
+
this.publishMessage("user_text_message", userTextMessage, logicalTurnId);
|
|
1100
|
+
}
|
|
1022
1101
|
}
|
|
1023
1102
|
catch (error) {
|
|
1024
1103
|
throw error;
|
|
@@ -1315,6 +1394,10 @@ export class ConvaiClient extends EventEmitter {
|
|
|
1315
1394
|
* the session from being disconnected due to inactivity.
|
|
1316
1395
|
*/
|
|
1317
1396
|
resetIdleTimer() {
|
|
1397
|
+
// SSE requests are one-turn streams; there is no persistent server-side
|
|
1398
|
+
// realtime session for an idle timer to keep alive.
|
|
1399
|
+
if (this._activeTransport === "sse")
|
|
1400
|
+
return;
|
|
1318
1401
|
if (!this.isTransportReady())
|
|
1319
1402
|
return;
|
|
1320
1403
|
this.publishMessage("reset-idle-timer", {
|