@convai/web-sdk 1.8.0-beta.8 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +45 -213
- package/dist/core/ConvaiClient.d.ts +1 -190
- package/dist/core/ConvaiClient.d.ts.map +1 -1
- package/dist/core/ConvaiClient.js +286 -1306
- package/dist/core/ConvaiClient.js.map +1 -1
- package/dist/core/MessageHandler.d.ts +1 -19
- package/dist/core/MessageHandler.d.ts.map +1 -1
- package/dist/core/MessageHandler.js +27 -378
- package/dist/core/MessageHandler.js.map +1 -1
- package/dist/core/connectRequest.d.ts +4 -17
- package/dist/core/connectRequest.d.ts.map +1 -1
- package/dist/core/connectRequest.js +17 -28
- package/dist/core/connectRequest.js.map +1 -1
- package/dist/core/index.d.ts +1 -8
- package/dist/core/index.d.ts.map +1 -1
- package/dist/core/index.js +1 -13
- package/dist/core/index.js.map +1 -1
- package/dist/core/logicalTurn.d.ts +6 -4
- package/dist/core/logicalTurn.d.ts.map +1 -1
- package/dist/core/logicalTurn.js +4 -29
- package/dist/core/logicalTurn.js.map +1 -1
- package/dist/core/modelOutput.d.ts.map +1 -1
- package/dist/core/modelOutput.js +2 -2
- package/dist/core/modelOutput.js.map +1 -1
- package/dist/core/types.d.ts +22 -860
- package/dist/core/types.d.ts.map +1 -1
- package/dist/core/types.js.map +1 -1
- package/dist/embed/ConvaiChatElement.d.ts +1 -77
- package/dist/embed/ConvaiChatElement.d.ts.map +1 -1
- package/dist/embed/ConvaiChatElement.js +54 -433
- package/dist/embed/ConvaiChatElement.js.map +1 -1
- package/dist/embed/chat-embed-v1.js +35 -136
- package/dist/embed/chat-embed-v1.js.map +4 -4
- package/dist/react/components/ConvaiWidget.d.ts.map +1 -1
- package/dist/react/components/ConvaiWidget.js +1 -6
- package/dist/react/components/ConvaiWidget.js.map +1 -1
- package/dist/react/components/rtc-widget/components/AudioVisualizer.d.ts.map +1 -1
- package/dist/react/components/rtc-widget/components/AudioVisualizer.js +1 -28
- package/dist/react/components/rtc-widget/components/AudioVisualizer.js.map +1 -1
- package/dist/react/components/rtc-widget/components/MarkdownRenderer.d.ts +0 -7
- package/dist/react/components/rtc-widget/components/MarkdownRenderer.d.ts.map +1 -1
- package/dist/react/components/rtc-widget/components/MarkdownRenderer.js +46 -135
- package/dist/react/components/rtc-widget/components/MarkdownRenderer.js.map +1 -1
- package/dist/react/components/rtc-widget/components/UserMessage.js +1 -1
- package/dist/react/components/rtc-widget/components/UserMessage.js.map +1 -1
- package/dist/react/components/rtc-widget/components/conviComponents/VoiceModeOverlay.d.ts.map +1 -1
- package/dist/react/components/rtc-widget/components/conviComponents/VoiceModeOverlay.js +49 -93
- package/dist/react/components/rtc-widget/components/conviComponents/VoiceModeOverlay.js.map +1 -1
- package/dist/react/hooks/useConvaiClient.d.ts.map +1 -1
- package/dist/react/hooks/useConvaiClient.js +2 -17
- package/dist/react/hooks/useConvaiClient.js.map +1 -1
- package/dist/react/index.d.ts +1 -4
- package/dist/react/index.d.ts.map +1 -1
- package/dist/react/index.js +1 -8
- package/dist/react/index.js.map +1 -1
- package/dist/vanilla/AudioRenderer.d.ts +1 -3
- package/dist/vanilla/AudioRenderer.d.ts.map +1 -1
- package/dist/vanilla/AudioRenderer.js +2 -8
- package/dist/vanilla/AudioRenderer.js.map +1 -1
- package/dist/vanilla/ConvaiWidget.d.ts.map +1 -1
- package/dist/vanilla/ConvaiWidget.js +302 -1122
- package/dist/vanilla/ConvaiWidget.js.map +1 -1
- package/dist/vanilla/WidgetEventSubscriptions.d.ts +1 -1
- package/dist/vanilla/WidgetEventSubscriptions.d.ts.map +1 -1
- package/dist/vanilla/WidgetEventSubscriptions.js +0 -5
- package/dist/vanilla/WidgetEventSubscriptions.js.map +1 -1
- package/dist/vanilla/icons.d.ts.map +1 -1
- package/dist/vanilla/icons.js +15 -62
- package/dist/vanilla/icons.js.map +1 -1
- package/dist/vanilla/index.d.ts +2 -6
- package/dist/vanilla/index.d.ts.map +1 -1
- package/dist/vanilla/index.js +1 -5
- package/dist/vanilla/index.js.map +1 -1
- package/dist/vanilla/styles.d.ts +1 -18
- package/dist/vanilla/styles.d.ts.map +1 -1
- package/dist/vanilla/styles.js +29 -141
- package/dist/vanilla/styles.js.map +1 -1
- package/dist/vanilla/types.d.ts +0 -74
- package/dist/vanilla/types.d.ts.map +1 -1
- package/dist/vanilla/types.js.map +1 -1
- package/dist/version.d.ts +1 -1
- package/dist/version.d.ts.map +1 -1
- package/dist/version.js +1 -1
- package/dist/version.js.map +1 -1
- package/package.json +17 -44
- package/dist/core/CharacterRoster.d.ts +0 -84
- package/dist/core/CharacterRoster.d.ts.map +0 -1
- package/dist/core/CharacterRoster.js +0 -348
- package/dist/core/CharacterRoster.js.map +0 -1
- package/dist/core/CharacterVersionManager.d.ts +0 -100
- package/dist/core/CharacterVersionManager.d.ts.map +0 -1
- package/dist/core/CharacterVersionManager.js +0 -227
- package/dist/core/CharacterVersionManager.js.map +0 -1
- package/dist/core/ConvaiRoomError.d.ts +0 -48
- package/dist/core/ConvaiRoomError.d.ts.map +0 -1
- package/dist/core/ConvaiRoomError.js +0 -111
- package/dist/core/ConvaiRoomError.js.map +0 -1
- package/dist/core/SSESession.d.ts +0 -59
- package/dist/core/SSESession.d.ts.map +0 -1
- package/dist/core/SSESession.js +0 -289
- package/dist/core/SSESession.js.map +0 -1
- package/dist/core/characterReference.d.ts +0 -40
- package/dist/core/characterReference.d.ts.map +0 -1
- package/dist/core/characterReference.js +0 -72
- package/dist/core/characterReference.js.map +0 -1
- package/dist/core/rosterRequest.d.ts +0 -57
- package/dist/core/rosterRequest.d.ts.map +0 -1
- package/dist/core/rosterRequest.js +0 -210
- package/dist/core/rosterRequest.js.map +0 -1
- package/dist/utils/inlineMarkdown.d.ts +0 -60
- package/dist/utils/inlineMarkdown.d.ts.map +0 -1
- package/dist/utils/inlineMarkdown.js +0 -183
- package/dist/utils/inlineMarkdown.js.map +0 -1
|
@@ -5,19 +5,13 @@ import { AudioManager } from "./AudioManager.js";
|
|
|
5
5
|
import { VideoManager } from "./VideoManager.js";
|
|
6
6
|
import { ScreenShareManager } from "./ScreenShareManager.js";
|
|
7
7
|
import { MessageHandler } from "./MessageHandler.js";
|
|
8
|
-
import { CharacterRoster } from "./CharacterRoster.js";
|
|
9
|
-
import { buildJoinRequestBody, buildRosterConnectFields, isRosterConfig, validateJoinOptions, validateProtocolTopology, validateRosterConfig, } from "./rosterRequest.js";
|
|
10
|
-
import { ConvaiRoomError, roomErrorFromResponse } from "./ConvaiRoomError.js";
|
|
11
8
|
import { SDK_VERSION } from "../version.js";
|
|
12
9
|
import { EventEmitter } from "./EventEmitter.js";
|
|
13
10
|
import { MemoryManager } from "./MemoryManager.js";
|
|
14
|
-
import { CharacterVersionManager } from "./CharacterVersionManager.js";
|
|
15
|
-
import { resolveCharacterReference, } from "./characterReference.js";
|
|
16
11
|
import { ConnectionStateHandler } from "./ConnectionStateHandler.js";
|
|
17
|
-
import {
|
|
12
|
+
import { buildBlendshapeConnectConfig, serializeClientTools, serializeProtocolCapabilities, validateActionProtocolSelection, validateBotLlmTextModeSelection, validateModelOutputVersionSelection, } from "./connectRequest.js";
|
|
18
13
|
import { shouldPreemptForContextUpdateResponse, shouldPreemptForExplicitRespondMode, } from "./contextUpdateRequest.js";
|
|
19
14
|
import { resolveVisionInputConfig, serializeVisionInputConfig, } from "./visionRequest.js";
|
|
20
|
-
import { SSESession } from "./SSESession.js";
|
|
21
15
|
import { isJsonValue } from "./modelOutput.js";
|
|
22
16
|
import { isValidLogicalTurnId } from "./logicalTurn.js";
|
|
23
17
|
// Module-level registry — populated by importing @convai/web-sdk/vanilla/websocket
|
|
@@ -28,33 +22,6 @@ function normalizeStateOfMind(value) {
|
|
|
28
22
|
const normalized = value.trim().toLowerCase();
|
|
29
23
|
return normalized === "neutral" || normalized === "" ? null : normalized;
|
|
30
24
|
}
|
|
31
|
-
/**
|
|
32
|
-
* Rejection from {@link ConvaiClient.sendActionResultAndWait} after local
|
|
33
|
-
* validation succeeds. Inspect `reason` rather than parsing the message.
|
|
34
|
-
*/
|
|
35
|
-
export class ActionResultAckError extends Error {
|
|
36
|
-
constructor(message, options) {
|
|
37
|
-
super(message);
|
|
38
|
-
this.name =
|
|
39
|
-
options.reason === "aborted"
|
|
40
|
-
? "AbortError"
|
|
41
|
-
: options.reason === "timeout"
|
|
42
|
-
? "TimeoutError"
|
|
43
|
-
: "ActionResultAckError";
|
|
44
|
-
this.reason = options.reason;
|
|
45
|
-
this.toolCallId = options.toolCallId;
|
|
46
|
-
this.errorCode = options.errorCode;
|
|
47
|
-
this.response = options.response;
|
|
48
|
-
this.cause = options.cause;
|
|
49
|
-
}
|
|
50
|
-
}
|
|
51
|
-
/**
|
|
52
|
-
* Marks a publication exchange failure that is safe to retry with the same
|
|
53
|
-
* logical connect attempt. The launch grant itself is single-use, so callers
|
|
54
|
-
* must never mint or submit a second grant for this recovery path.
|
|
55
|
-
*/
|
|
56
|
-
class RetryablePublishedChatExchangeError extends Error {
|
|
57
|
-
}
|
|
58
25
|
/**
|
|
59
26
|
* Custom reconnect policy that disables automatic reconnection
|
|
60
27
|
*/
|
|
@@ -114,16 +81,7 @@ export class ConvaiClient extends EventEmitter {
|
|
|
114
81
|
this._apiKey = null;
|
|
115
82
|
this._authToken = null;
|
|
116
83
|
this._characterId = null;
|
|
117
|
-
this._characterVersion = null;
|
|
118
|
-
this._characterReference = null;
|
|
119
|
-
this._characterVersionManager = null;
|
|
120
|
-
this._characterVersionManagerKey = null;
|
|
121
84
|
this._characterSessionId = "-1";
|
|
122
|
-
/** Roster state for multi-character rooms. Empty for single-character sessions. */
|
|
123
|
-
this._roster = new CharacterRoster();
|
|
124
|
-
/** Roster commands awaiting their ack, keyed by command id. */
|
|
125
|
-
this._pendingRosterCommands = new Map();
|
|
126
|
-
this._rosterCommandSeq = 0;
|
|
127
85
|
this._requestTraceId = null;
|
|
128
86
|
this._isBotReady = false;
|
|
129
87
|
this._participantSid = "";
|
|
@@ -137,7 +95,6 @@ export class ConvaiClient extends EventEmitter {
|
|
|
137
95
|
this._endUserMetadata = null;
|
|
138
96
|
this._wsSession = null;
|
|
139
97
|
this._activeTransport = null;
|
|
140
|
-
this._sseSession = null;
|
|
141
98
|
this._logRtviMessages = true;
|
|
142
99
|
this._isDisconnecting = false;
|
|
143
100
|
this._isPublishedChatSession = false;
|
|
@@ -145,7 +102,6 @@ export class ConvaiClient extends EventEmitter {
|
|
|
145
102
|
this._clientReadyRetryTimer = null;
|
|
146
103
|
this._clientReadyTimeoutTimer = null;
|
|
147
104
|
this._clientReadyMsgId = null;
|
|
148
|
-
this._pendingActionResultAcks = new Map();
|
|
149
105
|
// Conversation session tracking
|
|
150
106
|
this._conversationSessionId = 0;
|
|
151
107
|
this._conversationStartTime = 0;
|
|
@@ -155,23 +111,6 @@ export class ConvaiClient extends EventEmitter {
|
|
|
155
111
|
if (config) {
|
|
156
112
|
this._storedConfig = config;
|
|
157
113
|
this._logRtviMessages = config.logRtviMessages !== false;
|
|
158
|
-
// Derive the version readouts from the config eagerly, as
|
|
159
|
-
// characterVersions already is, so a React consumer destructuring
|
|
160
|
-
// characterReference off the hook sees the selector on first render
|
|
161
|
-
// rather than null until connect(). A malformed selector is left for
|
|
162
|
-
// connect() to reject -- the constructor has never thrown on config and
|
|
163
|
-
// starting to would be a behaviour change.
|
|
164
|
-
if (config.characterId && !isRosterConfig(config)) {
|
|
165
|
-
try {
|
|
166
|
-
const ref = resolveCharacterReference(config.characterId, config.characterVersion);
|
|
167
|
-
this._characterId = ref.characterId;
|
|
168
|
-
this._characterVersion = ref.version;
|
|
169
|
-
this._characterReference = ref.reference;
|
|
170
|
-
}
|
|
171
|
-
catch {
|
|
172
|
-
/* reported by connect() */
|
|
173
|
-
}
|
|
174
|
-
}
|
|
175
114
|
}
|
|
176
115
|
// Initialize room with no reconnect policy
|
|
177
116
|
this._room = new Room({
|
|
@@ -189,8 +128,6 @@ export class ConvaiClient extends EventEmitter {
|
|
|
189
128
|
endUserId: null,
|
|
190
129
|
endUserMetadata: null,
|
|
191
130
|
metrics: [],
|
|
192
|
-
characters: [],
|
|
193
|
-
activeMembershipId: null,
|
|
194
131
|
disconnectReason: null,
|
|
195
132
|
};
|
|
196
133
|
// Default audio settings for optimal interruption handling
|
|
@@ -236,42 +173,6 @@ export class ConvaiClient extends EventEmitter {
|
|
|
236
173
|
get characterId() {
|
|
237
174
|
return this._characterId;
|
|
238
175
|
}
|
|
239
|
-
get characterVersion() {
|
|
240
|
-
return this._characterVersion;
|
|
241
|
-
}
|
|
242
|
-
get characterReference() {
|
|
243
|
-
return this._characterReference;
|
|
244
|
-
}
|
|
245
|
-
/**
|
|
246
|
-
* Character version manager for the configured character. Built from the
|
|
247
|
-
* live connection when there is one, otherwise from the stored config, so
|
|
248
|
-
* versions can be listed and released before connecting. Null without an
|
|
249
|
-
* API key (the Character REST platform does not accept realtime auth tokens).
|
|
250
|
-
*/
|
|
251
|
-
get characterVersions() {
|
|
252
|
-
const apiKey = this._apiKey ?? this._storedConfig?.apiKey ?? null;
|
|
253
|
-
const rawCharacterId = this._characterId ?? this._storedConfig?.characterId ?? null;
|
|
254
|
-
if (!apiKey || !rawCharacterId)
|
|
255
|
-
return null;
|
|
256
|
-
let characterId = rawCharacterId;
|
|
257
|
-
try {
|
|
258
|
-
characterId = resolveCharacterReference(rawCharacterId).characterId;
|
|
259
|
-
}
|
|
260
|
-
catch {
|
|
261
|
-
// A malformed id is reported by connect(); use it verbatim here.
|
|
262
|
-
}
|
|
263
|
-
const baseUrl = this._storedConfig?.characterApiUrl ?? undefined;
|
|
264
|
-
const key = `${apiKey}\u0000${characterId}\u0000${baseUrl ?? ""}`;
|
|
265
|
-
if (this._characterVersionManagerKey !== key) {
|
|
266
|
-
this._characterVersionManager = new CharacterVersionManager({
|
|
267
|
-
apiKey,
|
|
268
|
-
characterId,
|
|
269
|
-
baseUrl,
|
|
270
|
-
});
|
|
271
|
-
this._characterVersionManagerKey = key;
|
|
272
|
-
}
|
|
273
|
-
return this._characterVersionManager;
|
|
274
|
-
}
|
|
275
176
|
get room() {
|
|
276
177
|
return this._room;
|
|
277
178
|
}
|
|
@@ -284,45 +185,6 @@ export class ConvaiClient extends EventEmitter {
|
|
|
284
185
|
get characterSessionId() {
|
|
285
186
|
return this._characterSessionId;
|
|
286
187
|
}
|
|
287
|
-
on(event, callback) {
|
|
288
|
-
return super.on(event, callback);
|
|
289
|
-
}
|
|
290
|
-
/**
|
|
291
|
-
* Server-owned snapshot of a multi-character room, or null for a
|
|
292
|
-
* single-character session. Carries the room id, epochs and roster.
|
|
293
|
-
*/
|
|
294
|
-
get roomSession() {
|
|
295
|
-
return this._roster.roomSession;
|
|
296
|
-
}
|
|
297
|
-
/**
|
|
298
|
-
* Character instances in a multi-character room, in roster order. Empty for a
|
|
299
|
-
* single-character session, so `length` distinguishes the two.
|
|
300
|
-
*/
|
|
301
|
-
get characters() {
|
|
302
|
-
return this._roster.characters;
|
|
303
|
-
}
|
|
304
|
-
/** Membership currently receiving user turns, or null. */
|
|
305
|
-
get activeMembershipId() {
|
|
306
|
-
return this._roster.activeMembershipId;
|
|
307
|
-
}
|
|
308
|
-
/**
|
|
309
|
-
* Start or stop the server streaming per-turn usage/cost breakdowns as
|
|
310
|
-
* `usageUpdate` events. Informational only; server-side usage tracking and
|
|
311
|
-
* billing are unaffected either way.
|
|
312
|
-
*/
|
|
313
|
-
toggleUsageUpdates(enabled) {
|
|
314
|
-
if (!this.isTransportReady())
|
|
315
|
-
return;
|
|
316
|
-
this.publishMessage("usage-toggle", { enabled });
|
|
317
|
-
}
|
|
318
|
-
/** Insert a transcript line locally. Nothing is sent; nothing is billed. */
|
|
319
|
-
appendMessage(message) {
|
|
320
|
-
if (!message?.content?.trim()) {
|
|
321
|
-
throw new Error("appendMessage requires non-empty content");
|
|
322
|
-
}
|
|
323
|
-
return this._messageHandler.appendLocalMessage(message);
|
|
324
|
-
}
|
|
325
|
-
/** Trace id of the /connect that produced this session; quote it to backend support. */
|
|
326
188
|
get requestTraceId() {
|
|
327
189
|
return this._requestTraceId;
|
|
328
190
|
}
|
|
@@ -352,60 +214,12 @@ export class ConvaiClient extends EventEmitter {
|
|
|
352
214
|
*/
|
|
353
215
|
setupEventListeners() {
|
|
354
216
|
// Room event listeners
|
|
355
|
-
// Per-membership audio binding. In a roster room every character is its
|
|
356
|
-
// own participant with identity `character:{membership_id}`; the guide's
|
|
357
|
-
// rule is to bind a track to the exact member or drop it, never to fall
|
|
358
|
-
// back to the active character. Single-character rooms keep the legacy
|
|
359
|
-
// behaviour: the track is surfaced with no attribution.
|
|
360
|
-
this._room.on(RoomEvent.TrackSubscribed, (track, _publication, participant) => {
|
|
361
|
-
if (track.kind !== "audio")
|
|
362
|
-
return;
|
|
363
|
-
const identity = participant.identity;
|
|
364
|
-
if (!this._roster.isMultiCharacter) {
|
|
365
|
-
this.emit("characterAudioTrack", {
|
|
366
|
-
membershipId: null,
|
|
367
|
-
characterId: this._characterId,
|
|
368
|
-
participantIdentity: identity,
|
|
369
|
-
track: track.mediaStreamTrack,
|
|
370
|
-
});
|
|
371
|
-
return;
|
|
372
|
-
}
|
|
373
|
-
const membershipId = identity.startsWith("character:")
|
|
374
|
-
? identity.slice("character:".length)
|
|
375
|
-
: null;
|
|
376
|
-
const member = membershipId ? this._roster.find(membershipId) : undefined;
|
|
377
|
-
if (!member) {
|
|
378
|
-
// Unattributable in a roster room: quarantine rather than guess.
|
|
379
|
-
if (this._logRtviMessages) {
|
|
380
|
-
console.warn(`[ConvaiClient] dropped audio track from unattributed participant "${identity}"`);
|
|
381
|
-
}
|
|
382
|
-
return;
|
|
383
|
-
}
|
|
384
|
-
this.emit("characterAudioTrack", {
|
|
385
|
-
membershipId: member.membershipId,
|
|
386
|
-
characterId: member.characterId,
|
|
387
|
-
participantIdentity: identity,
|
|
388
|
-
track: track.mediaStreamTrack,
|
|
389
|
-
});
|
|
390
|
-
});
|
|
391
217
|
this._room.on(RoomEvent.Disconnected, (reason) => {
|
|
392
218
|
this._connectionStateHandler.handleDisconnected(reason);
|
|
393
219
|
});
|
|
394
220
|
this._room.on(RoomEvent.ConnectionStateChanged, this._connectionStateHandler.handleConnectionStateChanged.bind(this._connectionStateHandler));
|
|
395
221
|
// Message handler events
|
|
396
|
-
this._messageHandler.on("botReady", (
|
|
397
|
-
// Each roster member sends its own bot-ready. Feed the roster before the
|
|
398
|
-
// de-dupe below, which only guards the single consumer-facing event --
|
|
399
|
-
// otherwise every member after the first would be dropped.
|
|
400
|
-
if (this._roster.applyBotReady(about)) {
|
|
401
|
-
this.updateState({
|
|
402
|
-
characters: this._roster.characters,
|
|
403
|
-
activeMembershipId: this._roster.activeMembershipId,
|
|
404
|
-
});
|
|
405
|
-
const membershipId = about?.membership_id;
|
|
406
|
-
if (membershipId)
|
|
407
|
-
this.emit("characterReady", this._roster.find(membershipId));
|
|
408
|
-
}
|
|
222
|
+
this._messageHandler.on("botReady", () => {
|
|
409
223
|
// Server may send multiple bot-ready frames (one per client-ready
|
|
410
224
|
// we retried). Only surface the first to consumers per connect.
|
|
411
225
|
if (this._isBotReady) {
|
|
@@ -416,17 +230,8 @@ export class ConvaiClient extends EventEmitter {
|
|
|
416
230
|
this.updateState({ isConnected: true });
|
|
417
231
|
this.emit("botReady");
|
|
418
232
|
});
|
|
419
|
-
this._messageHandler.on("messagesChange", (messages
|
|
420
|
-
|
|
421
|
-
// earlier streaming row. A v2 stop can finalize several rows at once.
|
|
422
|
-
if (updatedMessages?.length) {
|
|
423
|
-
for (const message of updatedMessages) {
|
|
424
|
-
this.emit("message", message);
|
|
425
|
-
}
|
|
426
|
-
}
|
|
427
|
-
else {
|
|
428
|
-
this.emit("message", updatedMessage ?? messages[messages.length - 1]);
|
|
429
|
-
}
|
|
233
|
+
this._messageHandler.on("messagesChange", (messages) => {
|
|
234
|
+
this.emit("message", messages[messages.length - 1]);
|
|
430
235
|
this.emit("messagesChange", messages);
|
|
431
236
|
});
|
|
432
237
|
this._messageHandler.on("userTranscriptionChange", (transcription) => {
|
|
@@ -508,9 +313,6 @@ export class ConvaiClient extends EventEmitter {
|
|
|
508
313
|
this._messageHandler.on("llmNoResponse", () => {
|
|
509
314
|
this.emit("llmNoResponse");
|
|
510
315
|
});
|
|
511
|
-
this._messageHandler.on("botTurnCompleted", (data) => {
|
|
512
|
-
this.emit("botTurnCompleted", data);
|
|
513
|
-
});
|
|
514
316
|
// Forward bot-output event (aggregated output with spoken status)
|
|
515
317
|
this._messageHandler.on("botOutput", (data) => {
|
|
516
318
|
this.emit("botOutput", data);
|
|
@@ -536,29 +338,13 @@ export class ConvaiClient extends EventEmitter {
|
|
|
536
338
|
this._messageHandler.on("actionResponse", (data) => {
|
|
537
339
|
this.emit("actionResponse", data);
|
|
538
340
|
});
|
|
539
|
-
this._messageHandler.on("usageUpdate", (data) => {
|
|
540
|
-
this.emit("usageUpdate", data);
|
|
541
|
-
});
|
|
542
|
-
this._messageHandler.on("characterStatus", (status) => {
|
|
543
|
-
if (this._roster.applyCharacterStatus(status)) {
|
|
544
|
-
this.updateState({
|
|
545
|
-
characters: this._roster.characters,
|
|
546
|
-
activeMembershipId: this._roster.activeMembershipId,
|
|
547
|
-
});
|
|
548
|
-
}
|
|
549
|
-
this.emit("characterStatus", status);
|
|
550
|
-
});
|
|
551
341
|
this._messageHandler.on("modelOutput", (data) => {
|
|
552
342
|
this.emit("modelOutput", data);
|
|
553
343
|
});
|
|
554
|
-
this._messageHandler.on("botLlmTextRaw", (data) => {
|
|
555
|
-
this.emit("botLlmTextRaw", data);
|
|
556
|
-
});
|
|
557
344
|
this._messageHandler.on("modelOutputProtocolError", (data) => {
|
|
558
345
|
this.emit("modelOutputProtocolError", data);
|
|
559
346
|
});
|
|
560
347
|
this._messageHandler.on("serverResponse", (response) => {
|
|
561
|
-
this._handleRosterServerResponse(response);
|
|
562
348
|
this.emit("serverResponse", response);
|
|
563
349
|
});
|
|
564
350
|
this._messageHandler.on("interactionCreated", (data) => {
|
|
@@ -609,10 +395,6 @@ export class ConvaiClient extends EventEmitter {
|
|
|
609
395
|
*/
|
|
610
396
|
publishMessage(type, data, id) {
|
|
611
397
|
const message = id === undefined ? { type, data } : { type, id, data };
|
|
612
|
-
if (this._activeTransport === "sse" && this._sseSession?.isConnected) {
|
|
613
|
-
this._sseSession.sendMessage(type, data, id);
|
|
614
|
-
return;
|
|
615
|
-
}
|
|
616
398
|
if (this._activeTransport === "websocket" && this._wsSession?.isConnected) {
|
|
617
399
|
this._wsSession.sendMessage(type, data);
|
|
618
400
|
}
|
|
@@ -622,37 +404,10 @@ export class ConvaiClient extends EventEmitter {
|
|
|
622
404
|
this._room.localParticipant.publishData(encodedData, { reliable: true });
|
|
623
405
|
}
|
|
624
406
|
}
|
|
625
|
-
/**
|
|
626
|
-
* Publish exactly one message while retaining the transport's local completion
|
|
627
|
-
* signal. Server acceptance is still proved separately by `serverResponse`.
|
|
628
|
-
*/
|
|
629
|
-
async publishMessageAndWaitForLocalPublication(type, data, id) {
|
|
630
|
-
const message = id === undefined ? { type, data } : { type, id, data };
|
|
631
|
-
if (this._activeTransport === "websocket") {
|
|
632
|
-
if (!this._wsSession?.isConnected) {
|
|
633
|
-
throw new Error("The action-result WebSocket transport is no longer available");
|
|
634
|
-
}
|
|
635
|
-
this._wsSession.sendMessage(type, data);
|
|
636
|
-
return;
|
|
637
|
-
}
|
|
638
|
-
if (this._activeTransport === "livekit" &&
|
|
639
|
-
this._room?.localParticipant &&
|
|
640
|
-
this._room.state !== "disconnected") {
|
|
641
|
-
const encodedData = new TextEncoder().encode(JSON.stringify(message));
|
|
642
|
-
await this._room.localParticipant.publishData(encodedData, {
|
|
643
|
-
reliable: true,
|
|
644
|
-
});
|
|
645
|
-
return;
|
|
646
|
-
}
|
|
647
|
-
throw new Error("The action-result transport is no longer available");
|
|
648
|
-
}
|
|
649
407
|
/**
|
|
650
408
|
* Returns true when a message can be sent on the active transport.
|
|
651
409
|
*/
|
|
652
410
|
isTransportReady() {
|
|
653
|
-
if (this._activeTransport === "sse") {
|
|
654
|
-
return this._sseSession?.isConnected ?? false;
|
|
655
|
-
}
|
|
656
411
|
if (this._activeTransport === "websocket") {
|
|
657
412
|
return this._wsSession?.isConnected ?? false;
|
|
658
413
|
}
|
|
@@ -675,10 +430,19 @@ export class ConvaiClient extends EventEmitter {
|
|
|
675
430
|
finalConfig.characterSessionId !== "-1") {
|
|
676
431
|
this._characterSessionId = finalConfig.characterSessionId;
|
|
677
432
|
}
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
433
|
+
// Add default URL if not provided
|
|
434
|
+
const configWithDefaults = {
|
|
435
|
+
url: "https://realtime-api.convai.com",
|
|
436
|
+
...finalConfig,
|
|
437
|
+
};
|
|
438
|
+
this._logRtviMessages = configWithDefaults.logRtviMessages !== false;
|
|
439
|
+
this._messageHandler.setRtviMessageLogging(this._logRtviMessages);
|
|
681
440
|
this._messageHandler.setModelOutputVersion(1);
|
|
441
|
+
const hasApiKey = Boolean(configWithDefaults.apiKey);
|
|
442
|
+
const hasAuthToken = Boolean(configWithDefaults.authToken);
|
|
443
|
+
if ((!hasApiKey && !hasAuthToken) || !configWithDefaults.characterId) {
|
|
444
|
+
throw new Error("Either apiKey or authToken is required, and characterId is required");
|
|
445
|
+
}
|
|
682
446
|
this.updateState({ isConnecting: true });
|
|
683
447
|
this._requestTraceId = null;
|
|
684
448
|
try {
|
|
@@ -686,87 +450,23 @@ export class ConvaiClient extends EventEmitter {
|
|
|
686
450
|
this._apiKey = configWithDefaults.apiKey ?? null;
|
|
687
451
|
this._authToken = configWithDefaults.authToken ?? null;
|
|
688
452
|
this._characterId = configWithDefaults.characterId;
|
|
689
|
-
this._characterVersion = configWithDefaults.characterVersion ?? null;
|
|
690
|
-
this._characterReference = configWithDefaults.characterReference;
|
|
691
453
|
// Determine connection type based on enableVideo
|
|
692
454
|
const connType = configWithDefaults.enableVideo ? "video" : "audio";
|
|
693
455
|
this._connectionType = connType;
|
|
694
|
-
const transportType = configWithDefaults.transport ??
|
|
695
|
-
(configWithDefaults.interactionApiUrl ? "sse" : "livekit");
|
|
696
|
-
if (transportType === "sse" && !configWithDefaults.interactionApiUrl) {
|
|
697
|
-
throw new Error("interactionApiUrl is required when using the SSE transport");
|
|
698
|
-
}
|
|
699
|
-
if (transportType === "sse" && configWithDefaults.enableVideo) {
|
|
700
|
-
throw new Error("The SSE transport supports text interactions only");
|
|
701
|
-
}
|
|
456
|
+
const transportType = configWithDefaults.transport ?? "livekit";
|
|
702
457
|
this._activeTransport = transportType;
|
|
703
458
|
const effectiveVisionInputConfig = resolveVisionInputConfig(configWithDefaults);
|
|
459
|
+
// Prepare request body with required parameters
|
|
704
460
|
// An explicit stateOfMind in the config wins; otherwise carry whatever
|
|
705
461
|
// updateEmotion() last set, including before the first connect.
|
|
706
462
|
const stateOfMind = configWithDefaults.stateOfMind !== undefined
|
|
707
463
|
? normalizeStateOfMind(configWithDefaults.stateOfMind) ?? null
|
|
708
464
|
: this._stateOfMind;
|
|
709
465
|
this._stateOfMind = stateOfMind;
|
|
710
|
-
if (transportType === "sse") {
|
|
711
|
-
this._sseSession = new SSESession({
|
|
712
|
-
interactionApiUrl: configWithDefaults.interactionApiUrl,
|
|
713
|
-
authorization: configWithDefaults.authToken ?? configWithDefaults.apiKey,
|
|
714
|
-
characterId: configWithDefaults.characterReference,
|
|
715
|
-
getCharacterSessionId: () => this._characterSessionId,
|
|
716
|
-
takeStateOfMind: () => {
|
|
717
|
-
const stateOfMind = this._stateOfMind;
|
|
718
|
-
this._stateOfMind = null;
|
|
719
|
-
if (this._storedConfig?.stateOfMind !== undefined) {
|
|
720
|
-
this._storedConfig = { ...this._storedConfig, stateOfMind: null };
|
|
721
|
-
}
|
|
722
|
-
return stateOfMind;
|
|
723
|
-
},
|
|
724
|
-
onCharacterSessionId: (characterSessionId) => {
|
|
725
|
-
this._characterSessionId = characterSessionId;
|
|
726
|
-
this.emit("characterSessionId", characterSessionId);
|
|
727
|
-
this._storedConfig = {
|
|
728
|
-
...this._storedConfig,
|
|
729
|
-
characterSessionId,
|
|
730
|
-
};
|
|
731
|
-
},
|
|
732
|
-
endUserId: configWithDefaults.endUserId,
|
|
733
|
-
endUserMetadata: configWithDefaults.endUserMetadata,
|
|
734
|
-
enableEmotion: configWithDefaults.enableEmotion,
|
|
735
|
-
emotionConfig: configWithDefaults.emotionConfig,
|
|
736
|
-
actionConfig: configWithDefaults.actionConfig,
|
|
737
|
-
capabilities: configWithDefaults.capabilities,
|
|
738
|
-
dynamicInfo: configWithDefaults.dynamicInfo,
|
|
739
|
-
keepInContext: configWithDefaults.keepInContext,
|
|
740
|
-
narrativeTemplateKeys: configWithDefaults.narrativeTemplateKeys,
|
|
741
|
-
sceneDescription: configWithDefaults.sceneDescription,
|
|
742
|
-
respondModes: configWithDefaults.respondModes,
|
|
743
|
-
useResponseLifecycleContract: configWithDefaults.useResponseLifecycleContract,
|
|
744
|
-
invocationMetadata: configWithDefaults.invocationMetadata,
|
|
745
|
-
debug: configWithDefaults.debug,
|
|
746
|
-
onMessage: (payload) => this._messageHandler.handleDataReceivedPublic(payload),
|
|
747
|
-
onError: (error) => this.emit("error", error),
|
|
748
|
-
});
|
|
749
|
-
this._sseSession.connect();
|
|
750
|
-
this._connectionType = "audio";
|
|
751
|
-
this._isBotReady = true;
|
|
752
|
-
this.updateState({
|
|
753
|
-
isConnected: true,
|
|
754
|
-
isConnecting: false,
|
|
755
|
-
endUserId: configWithDefaults.endUserId ?? null,
|
|
756
|
-
endUserMetadata: configWithDefaults.endUserMetadata ?? null,
|
|
757
|
-
});
|
|
758
|
-
this.emit("connect");
|
|
759
|
-
this.emit("botReady");
|
|
760
|
-
return;
|
|
761
|
-
}
|
|
762
|
-
// Prepare request body with required parameters
|
|
763
466
|
const characterSessionIdToSend = configWithDefaults.characterSessionId ?? this._characterSessionId;
|
|
764
|
-
const isRoster = isRosterConfig(configWithDefaults);
|
|
765
467
|
const serializedCapabilities = serializeProtocolCapabilities(configWithDefaults.capabilities);
|
|
766
468
|
const requestBody = {
|
|
767
|
-
|
|
768
|
-
? buildRosterConnectFields(configWithDefaults)
|
|
769
|
-
: { character_id: configWithDefaults.characterReference }),
|
|
469
|
+
character_id: configWithDefaults.characterId,
|
|
770
470
|
...(stateOfMind ? { state_of_mind: stateOfMind } : {}),
|
|
771
471
|
...(configWithDefaults.endUserId && {
|
|
772
472
|
end_user_id: configWithDefaults.endUserId,
|
|
@@ -780,8 +480,21 @@ export class ConvaiClient extends EventEmitter {
|
|
|
780
480
|
}),
|
|
781
481
|
connection_type: connType,
|
|
782
482
|
...buildBlendshapeConnectConfig(configWithDefaults),
|
|
783
|
-
|
|
784
|
-
|
|
483
|
+
llm_provider: "dynamic",
|
|
484
|
+
...(configWithDefaults.enableEmotion === true && {
|
|
485
|
+
emotion_config: (() => {
|
|
486
|
+
const cfg = configWithDefaults.emotionConfig;
|
|
487
|
+
const provider = cfg?.provider ?? "llm";
|
|
488
|
+
if (provider === "nrclex") {
|
|
489
|
+
return {
|
|
490
|
+
provider,
|
|
491
|
+
min_word_threshold: cfg.min_word_threshold ?? 3,
|
|
492
|
+
low_intensity_threshold: cfg.low_intensity_threshold ?? 0.33,
|
|
493
|
+
high_intensity_threshold: cfg.high_intensity_threshold ?? 0.66,
|
|
494
|
+
};
|
|
495
|
+
}
|
|
496
|
+
return { provider };
|
|
497
|
+
})(),
|
|
785
498
|
}),
|
|
786
499
|
default_tts_enabled: configWithDefaults.ttsEnabled !== undefined
|
|
787
500
|
? configWithDefaults.ttsEnabled
|
|
@@ -790,25 +503,25 @@ export class ConvaiClient extends EventEmitter {
|
|
|
790
503
|
...(effectiveVisionInputConfig && {
|
|
791
504
|
vision_input_config: serializeVisionInputConfig(effectiveVisionInputConfig),
|
|
792
505
|
}),
|
|
793
|
-
...(configWithDefaults.visionPromptMode !== undefined && {
|
|
794
|
-
vision_prompt_mode: configWithDefaults.visionPromptMode,
|
|
795
|
-
}),
|
|
796
506
|
...(configWithDefaults.respondModes && {
|
|
797
507
|
respond_modes: this.serializeRespondModes(configWithDefaults.respondModes),
|
|
798
508
|
}),
|
|
799
|
-
...(
|
|
800
|
-
...(configWithDefaults.sceneDescription && {
|
|
801
|
-
scene_description: configWithDefaults.sceneDescription,
|
|
802
|
-
}),
|
|
803
|
-
// A roster room owns its members' sessions; the singular resume id is
|
|
804
|
-
// a compatibility mirror and must not go back on the wire.
|
|
805
|
-
...(!isRoster &&
|
|
806
|
-
characterSessionIdToSend &&
|
|
509
|
+
...(characterSessionIdToSend &&
|
|
807
510
|
characterSessionIdToSend !== "-1" && {
|
|
808
511
|
character_session_id: characterSessionIdToSend,
|
|
809
512
|
}),
|
|
810
|
-
...(
|
|
811
|
-
action_config:
|
|
513
|
+
...(configWithDefaults.actionConfig && {
|
|
514
|
+
action_config: {
|
|
515
|
+
actions: configWithDefaults.actionConfig.actions,
|
|
516
|
+
objects: configWithDefaults.actionConfig.objects,
|
|
517
|
+
characters: configWithDefaults.actionConfig.characters,
|
|
518
|
+
...(configWithDefaults.actionConfig.current_attention_object && {
|
|
519
|
+
current_attention_object: configWithDefaults.actionConfig.current_attention_object,
|
|
520
|
+
}),
|
|
521
|
+
...(configWithDefaults.actionConfig.tools && {
|
|
522
|
+
tools: serializeClientTools(configWithDefaults.actionConfig.tools),
|
|
523
|
+
}),
|
|
524
|
+
},
|
|
812
525
|
}),
|
|
813
526
|
...(configWithDefaults.dynamicInfo && {
|
|
814
527
|
dynamic_info: {
|
|
@@ -825,12 +538,6 @@ export class ConvaiClient extends EventEmitter {
|
|
|
825
538
|
...(configWithDefaults.debug !== undefined && {
|
|
826
539
|
debug: configWithDefaults.debug,
|
|
827
540
|
}),
|
|
828
|
-
...(configWithDefaults.preserveLinksInOutput !== undefined && {
|
|
829
|
-
preserve_links_in_output: configWithDefaults.preserveLinksInOutput,
|
|
830
|
-
}),
|
|
831
|
-
...(configWithDefaults.useResponseLifecycleContract !== undefined && {
|
|
832
|
-
use_response_lifecycle_contract: configWithDefaults.useResponseLifecycleContract,
|
|
833
|
-
}),
|
|
834
541
|
invocation_metadata: {
|
|
835
542
|
source: configWithDefaults.invocationMetadata?.source ?? "web_sdk",
|
|
836
543
|
client_version: configWithDefaults.invocationMetadata?.clientVersion ??
|
|
@@ -849,676 +556,217 @@ export class ConvaiClient extends EventEmitter {
|
|
|
849
556
|
else if (configWithDefaults.apiKey) {
|
|
850
557
|
connectHeaders["X-API-Key"] = configWithDefaults.apiKey;
|
|
851
558
|
}
|
|
852
|
-
// HTTP POST /connect — same request for both transports
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
const
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
const errorData = JSON.parse(errorText);
|
|
867
|
-
const raw = errorData.message ?? errorData.error ?? errorData.detail;
|
|
868
|
-
if (typeof raw === "string") {
|
|
869
|
-
errorMessage = raw;
|
|
870
|
-
}
|
|
871
|
-
else if (Array.isArray(raw)) {
|
|
872
|
-
// FastAPI/Pydantic 422 — detail is an array of validation error objects
|
|
873
|
-
errorMessage = raw
|
|
874
|
-
.map((e) => [e.loc?.join("."), e.msg].filter(Boolean).join(": "))
|
|
875
|
-
.join("; ") || errorMessage;
|
|
876
|
-
}
|
|
877
|
-
else if (raw != null) {
|
|
878
|
-
errorMessage = JSON.stringify(raw);
|
|
879
|
-
}
|
|
880
|
-
else if (errorText) {
|
|
881
|
-
errorMessage = `${errorMessage}: ${errorText}`;
|
|
882
|
-
}
|
|
559
|
+
// HTTP POST /connect — same request for both transports
|
|
560
|
+
const response = await fetch(`${configWithDefaults.url}/connect`, {
|
|
561
|
+
method: "POST",
|
|
562
|
+
headers: connectHeaders,
|
|
563
|
+
body: JSON.stringify(requestBody),
|
|
564
|
+
});
|
|
565
|
+
if (!response.ok) {
|
|
566
|
+
const errorText = await response.text();
|
|
567
|
+
let errorMessage = `HTTP ${response.status}`;
|
|
568
|
+
try {
|
|
569
|
+
const errorData = JSON.parse(errorText);
|
|
570
|
+
const raw = errorData.message ?? errorData.error ?? errorData.detail;
|
|
571
|
+
if (typeof raw === "string") {
|
|
572
|
+
errorMessage = raw;
|
|
883
573
|
}
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
574
|
+
else if (Array.isArray(raw)) {
|
|
575
|
+
// FastAPI/Pydantic 422 — detail is an array of validation error objects
|
|
576
|
+
errorMessage = raw
|
|
577
|
+
.map((e) => [e.loc?.join("."), e.msg].filter(Boolean).join(": "))
|
|
578
|
+
.join("; ") || errorMessage;
|
|
887
579
|
}
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
// error. Single-character behaviour is unchanged.
|
|
891
|
-
if (isRoster) {
|
|
892
|
-
let parsed = errorText;
|
|
893
|
-
try {
|
|
894
|
-
parsed = JSON.parse(errorText);
|
|
895
|
-
}
|
|
896
|
-
catch {
|
|
897
|
-
/* keep the raw text as the detail */
|
|
898
|
-
}
|
|
899
|
-
throw roomErrorFromResponse(response.status, parsed);
|
|
580
|
+
else if (raw != null) {
|
|
581
|
+
errorMessage = JSON.stringify(raw);
|
|
900
582
|
}
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
return await response.json();
|
|
904
|
-
};
|
|
905
|
-
const connectionData = isRoster
|
|
906
|
-
? await this._withProvisioningRetry(configWithDefaults, connectOnce)
|
|
907
|
-
: await connectOnce();
|
|
908
|
-
return await this.consumeConnectionData(connectionData, configWithDefaults);
|
|
909
|
-
}
|
|
910
|
-
catch (error) {
|
|
911
|
-
await this.cleanupWebSocketSessionAfterConnectFailure();
|
|
912
|
-
this.updateState({
|
|
913
|
-
isConnected: false,
|
|
914
|
-
isConnecting: false,
|
|
915
|
-
});
|
|
916
|
-
this.emit("error", error);
|
|
917
|
-
throw error;
|
|
918
|
-
}
|
|
919
|
-
}
|
|
920
|
-
/**
|
|
921
|
-
* Complete a connection from an already-fetched /connect response body.
|
|
922
|
-
*
|
|
923
|
-
* `connect()` fetches /connect and then calls this internally. Call it
|
|
924
|
-
* directly when the response was obtained elsewhere — a server-side session
|
|
925
|
-
* manager, or the embed's connect-proxy flow, where the API key must never
|
|
926
|
-
* reach the browser.
|
|
927
|
-
*
|
|
928
|
-
* @example
|
|
929
|
-
* // Server (customer's backend): holds the API key, calls Convai's
|
|
930
|
-
* // /connect, and relays the response body verbatim.
|
|
931
|
-
* // Browser: never sees the API key, only the relayed response.
|
|
932
|
-
* const data = await fetch('/api/convai-connect', {
|
|
933
|
-
* method: 'POST',
|
|
934
|
-
* headers: { 'Content-Type': 'application/json' },
|
|
935
|
-
* body: JSON.stringify({ characterId: 'your-character-id' }),
|
|
936
|
-
* }).then(r => r.json())
|
|
937
|
-
* await client.connectWithConnectionData(data)
|
|
938
|
-
*/
|
|
939
|
-
async connectWithConnectionData(data, config) {
|
|
940
|
-
// A client can be reused after a published session. This is a normal
|
|
941
|
-
// credentialed/proxy connection path, so never leave published-only UI
|
|
942
|
-
// semantics (fresh grant requirement, text-only controls) enabled.
|
|
943
|
-
this._isPublishedChatSession = false;
|
|
944
|
-
const finalConfig = config || this._storedConfig;
|
|
945
|
-
if (!finalConfig) {
|
|
946
|
-
throw new Error("No configuration provided. Pass config to connectWithConnectionData() or store it in the client");
|
|
947
|
-
}
|
|
948
|
-
this._storedConfig = { ...finalConfig };
|
|
949
|
-
// Use characterSessionId from config when reconnecting (so we resume the same session)
|
|
950
|
-
if (finalConfig.characterSessionId &&
|
|
951
|
-
finalConfig.characterSessionId !== "-1") {
|
|
952
|
-
this._characterSessionId = finalConfig.characterSessionId;
|
|
953
|
-
}
|
|
954
|
-
const configWithDefaults = this.applyConfigDefaults(finalConfig, {
|
|
955
|
-
requireCredential: false,
|
|
956
|
-
});
|
|
957
|
-
// Symmetric with connect(): the catch below emits isConnecting: false, so
|
|
958
|
-
// without this a UI bound to stateChange never sees a connecting state on
|
|
959
|
-
// the proxy path -- and on failure receives a transition out of a state it
|
|
960
|
-
// was never told about. consumeConnectionData() clears the flag on success.
|
|
961
|
-
this.updateState({ isConnecting: true });
|
|
962
|
-
try {
|
|
963
|
-
return await this.consumeConnectionData(data, configWithDefaults);
|
|
964
|
-
}
|
|
965
|
-
catch (error) {
|
|
966
|
-
await this.cleanupWebSocketSessionAfterConnectFailure();
|
|
967
|
-
this.updateState({
|
|
968
|
-
isConnected: false,
|
|
969
|
-
isConnecting: false,
|
|
970
|
-
});
|
|
971
|
-
this.emit("error", error);
|
|
972
|
-
throw error;
|
|
973
|
-
}
|
|
974
|
-
}
|
|
975
|
-
/** Correlate roster acks back to the promise that issued the command. */
|
|
976
|
-
_handleRosterServerResponse(response) {
|
|
977
|
-
const eventType = response.event_type;
|
|
978
|
-
if (eventType !== "interaction-target" &&
|
|
979
|
-
eventType !== "character-roster-update") {
|
|
980
|
-
return;
|
|
981
|
-
}
|
|
982
|
-
const extras = response.extras ?? null;
|
|
983
|
-
const commandId = extras?.command_id;
|
|
984
|
-
const pending = commandId
|
|
985
|
-
? this._pendingRosterCommands.get(commandId)
|
|
986
|
-
: undefined;
|
|
987
|
-
if (response.status === "error") {
|
|
988
|
-
// Adopt the server's epochs even from a rejection -- a stale-epoch error
|
|
989
|
-
// carries the authoritative value, and that is what makes a retry
|
|
990
|
-
// meaningful.
|
|
991
|
-
this._roster.adoptEpochs(extras);
|
|
992
|
-
this._pushRosterState();
|
|
993
|
-
if (pending && commandId) {
|
|
994
|
-
this._pendingRosterCommands.delete(commandId);
|
|
995
|
-
clearTimeout(pending.timer);
|
|
996
|
-
pending.reject(Object.assign(new Error(response.message ?? `${eventType} was rejected`), { code: extras?.code ?? null, extras }));
|
|
997
|
-
}
|
|
998
|
-
return;
|
|
999
|
-
}
|
|
1000
|
-
// Apply the change whether or not this client asked for it: another
|
|
1001
|
-
// participant's switch reaches us through the same event.
|
|
1002
|
-
let change;
|
|
1003
|
-
if (eventType === "interaction-target") {
|
|
1004
|
-
change = this._roster.applyInteractionTarget(extras);
|
|
1005
|
-
this._pushRosterState();
|
|
1006
|
-
this.emit("activeCharacterChange", change);
|
|
1007
|
-
}
|
|
1008
|
-
else {
|
|
1009
|
-
change = this._roster.applyRosterUpdate(extras);
|
|
1010
|
-
this._pushRosterState();
|
|
1011
|
-
this.emit("rosterChange", change);
|
|
1012
|
-
}
|
|
1013
|
-
if (pending && commandId) {
|
|
1014
|
-
this._pendingRosterCommands.delete(commandId);
|
|
1015
|
-
clearTimeout(pending.timer);
|
|
1016
|
-
pending.resolve(change);
|
|
1017
|
-
}
|
|
1018
|
-
}
|
|
1019
|
-
/**
|
|
1020
|
-
* Settle every in-flight roster command.
|
|
1021
|
-
*
|
|
1022
|
-
* Without this a caller who disconnects mid-command keeps a live 10s timer
|
|
1023
|
-
* and gets a rejection long after the session ended -- an unhandled rejection
|
|
1024
|
-
* if they stopped awaiting when they tore the client down. The roster is
|
|
1025
|
-
* reset alongside, so the command could not be applied meaningfully anyway.
|
|
1026
|
-
*/
|
|
1027
|
-
_failPendingRosterCommands(reason) {
|
|
1028
|
-
for (const [, pending] of this._pendingRosterCommands) {
|
|
1029
|
-
clearTimeout(pending.timer);
|
|
1030
|
-
pending.reject(new Error(reason));
|
|
1031
|
-
}
|
|
1032
|
-
this._pendingRosterCommands.clear();
|
|
1033
|
-
}
|
|
1034
|
-
_pushRosterState() {
|
|
1035
|
-
this.updateState({
|
|
1036
|
-
characters: this._roster.characters,
|
|
1037
|
-
activeMembershipId: this._roster.activeMembershipId,
|
|
1038
|
-
});
|
|
1039
|
-
}
|
|
1040
|
-
/** Publish a roster command and settle when its ack arrives. */
|
|
1041
|
-
_sendRosterCommand(type, build) {
|
|
1042
|
-
if (!this._roster.isMultiCharacter) {
|
|
1043
|
-
return Promise.reject(new Error(`Cannot send ${type}: this session was not created with a characters array`));
|
|
1044
|
-
}
|
|
1045
|
-
if (!this.isTransportReady()) {
|
|
1046
|
-
return Promise.reject(new Error(`Cannot send ${type}: not connected`));
|
|
1047
|
-
}
|
|
1048
|
-
const commandId = `${type}-${++this._rosterCommandSeq}-${Date.now()}`;
|
|
1049
|
-
return new Promise((resolve, reject) => {
|
|
1050
|
-
const timer = setTimeout(() => {
|
|
1051
|
-
this._pendingRosterCommands.delete(commandId);
|
|
1052
|
-
reject(new Error(`${type} was not acknowledged within ${ConvaiClient.ROSTER_COMMAND_TIMEOUT_MS}ms. ` +
|
|
1053
|
-
`The outcome is unknown -- acks can be suppressed as duplicates. ` +
|
|
1054
|
-
`Reconcile from the next authoritative room state before sending a conflicting command.`));
|
|
1055
|
-
}, ConvaiClient.ROSTER_COMMAND_TIMEOUT_MS);
|
|
1056
|
-
this._pendingRosterCommands.set(commandId, { resolve, reject, timer });
|
|
1057
|
-
this.publishMessage(type, build(commandId), commandId);
|
|
1058
|
-
});
|
|
1059
|
-
}
|
|
1060
|
-
/**
|
|
1061
|
-
* Route subsequent user turns to a different character instance.
|
|
1062
|
-
*
|
|
1063
|
-
* Pass null to close every gate, so no member consumes the user until a
|
|
1064
|
-
* target is set again.
|
|
1065
|
-
*/
|
|
1066
|
-
async setInteractionTarget(membershipId) {
|
|
1067
|
-
if (!this._roster.isMultiCharacter) {
|
|
1068
|
-
throw new Error("Cannot send interaction-target: this session was not created with a characters array");
|
|
1069
|
-
}
|
|
1070
|
-
if (membershipId !== null) {
|
|
1071
|
-
if (!membershipId) {
|
|
1072
|
-
throw new Error("setInteractionTarget requires a membershipId, or null to close all gates");
|
|
1073
|
-
}
|
|
1074
|
-
const instance = this._roster.find(membershipId);
|
|
1075
|
-
if (this._roster.isMultiCharacter && !instance) {
|
|
1076
|
-
throw new Error(`Unknown membershipId "${membershipId}". Address instances by membershipId, not characterId.`);
|
|
1077
|
-
}
|
|
1078
|
-
if (instance && !instance.isReady) {
|
|
1079
|
-
throw new Error(`Membership "${membershipId}" has not sent bot-ready yet; the runtime refuses an unready target`);
|
|
1080
|
-
}
|
|
1081
|
-
}
|
|
1082
|
-
const send = () => this._sendRosterCommand("interaction-target", (commandId) => this._roster.buildInteractionTarget(membershipId, commandId));
|
|
1083
|
-
try {
|
|
1084
|
-
return (await send());
|
|
1085
|
-
}
|
|
1086
|
-
catch (error) {
|
|
1087
|
-
// Another participant switched first. The roster adopted the server's
|
|
1088
|
-
// epoch when the rejection arrived, so exactly one retry is now fenced
|
|
1089
|
-
// correctly; a second would mean we are losing a race we should report.
|
|
1090
|
-
if (error?.code !== "stale_route_epoch")
|
|
1091
|
-
throw error;
|
|
1092
|
-
return (await send());
|
|
1093
|
-
}
|
|
1094
|
-
}
|
|
1095
|
-
/** Add or remove character instances on a live room, without reconnecting. */
|
|
1096
|
-
async updateCharacterRoster(options) {
|
|
1097
|
-
// Report the wrong-session-shape first. Otherwise a single-character
|
|
1098
|
-
// client, whose roster is empty, trips the "cannot be emptied" rule below
|
|
1099
|
-
// and gets an error about a room it never had.
|
|
1100
|
-
if (!this._roster.isMultiCharacter) {
|
|
1101
|
-
throw new Error("Cannot send character-roster-update: this session was not created with a characters array");
|
|
1102
|
-
}
|
|
1103
|
-
const add = options?.add ?? [];
|
|
1104
|
-
const remove = options?.remove ?? [];
|
|
1105
|
-
if (add.length === 0 && remove.length === 0) {
|
|
1106
|
-
throw new Error("updateCharacterRoster requires at least one add or remove");
|
|
1107
|
-
}
|
|
1108
|
-
if (add.some((c) => !c?.characterId?.trim())) {
|
|
1109
|
-
throw new Error("Every entry in add requires a characterId");
|
|
1110
|
-
}
|
|
1111
|
-
if (remove.length && remove.length >= this._roster.characters.length && add.length === 0) {
|
|
1112
|
-
throw new Error("A roster room cannot be emptied; add a replacement or keep one member");
|
|
1113
|
-
}
|
|
1114
|
-
// Removing the active member needs an explicit, ready replacement --
|
|
1115
|
-
// the runtime answers `replacement_target_required` otherwise, and
|
|
1116
|
-
// `replacement_target_unavailable` for one that has not sent bot-ready.
|
|
1117
|
-
const active = this._roster.activeMembershipId;
|
|
1118
|
-
const effective = !options.replacementTarget && active && remove.includes(active)
|
|
1119
|
-
? { ...options, replacementTarget: this._roster.pickReplacement(remove) }
|
|
1120
|
-
: options;
|
|
1121
|
-
return (await this._sendRosterCommand("character-roster-update", (commandId) => this._roster.buildRosterUpdate(effective, commandId)));
|
|
1122
|
-
}
|
|
1123
|
-
/**
|
|
1124
|
-
* Run a room request, retrying only the conflicts that mean "still working".
|
|
1125
|
-
*
|
|
1126
|
-
* The runtime answers a create that collides with an in-flight identical
|
|
1127
|
-
* create with 409 `ROSTER_PROVISIONING_IN_PROGRESS`, and the documented
|
|
1128
|
-
* recovery is to retry the same key with bounded backoff. Left to callers,
|
|
1129
|
-
* every integration reimplements that, usually without a cap. The payload is
|
|
1130
|
-
* replayed verbatim -- changing it would turn the retry into a fingerprint
|
|
1131
|
-
* mismatch.
|
|
1132
|
-
*/
|
|
1133
|
-
async _withProvisioningRetry(config, run) {
|
|
1134
|
-
const setting = config.rosterProvisioningRetry;
|
|
1135
|
-
if (setting === false)
|
|
1136
|
-
return run();
|
|
1137
|
-
const attempts = Math.max(1, setting?.attempts ?? ConvaiClient.ROSTER_RETRY_ATTEMPTS);
|
|
1138
|
-
const initialDelayMs = Math.max(0, setting?.initialDelayMs ?? ConvaiClient.ROSTER_RETRY_INITIAL_MS);
|
|
1139
|
-
let lastError;
|
|
1140
|
-
for (let attempt = 0; attempt < attempts; attempt++) {
|
|
1141
|
-
try {
|
|
1142
|
-
return await run();
|
|
1143
|
-
}
|
|
1144
|
-
catch (error) {
|
|
1145
|
-
lastError = error;
|
|
1146
|
-
const retryable = error instanceof ConvaiRoomError && error.isRetryable;
|
|
1147
|
-
if (!retryable || attempt === attempts - 1)
|
|
1148
|
-
throw error;
|
|
1149
|
-
// Jitter so a lobby of clients waking on the same room does not
|
|
1150
|
-
// retry in lockstep.
|
|
1151
|
-
const base = initialDelayMs * 2 ** attempt;
|
|
1152
|
-
const jittered = base * (0.8 + Math.random() * 0.4);
|
|
1153
|
-
await new Promise((resolve) => setTimeout(resolve, jittered));
|
|
1154
|
-
}
|
|
1155
|
-
}
|
|
1156
|
-
throw lastError;
|
|
1157
|
-
}
|
|
1158
|
-
/**
|
|
1159
|
-
* Join a multi-character room that already exists.
|
|
1160
|
-
*
|
|
1161
|
-
* A join supplies a locator and the joining human, and nothing else: the
|
|
1162
|
-
* roster is server-owned, so resending topology is rejected. It never spawns,
|
|
1163
|
-
* removes or repairs a character -- only room creation does that.
|
|
1164
|
-
*
|
|
1165
|
-
* Capacity counts distinct speakers derived from `endUserId`, and defaults to
|
|
1166
|
-
* one, so raise `maxNumParticipants` at creation before expecting a second
|
|
1167
|
-
* human to get in.
|
|
1168
|
-
*
|
|
1169
|
-
* @example
|
|
1170
|
-
* ```ts
|
|
1171
|
-
* const client = new ConvaiClient({ apiKey: 'YOUR_API_KEY' })
|
|
1172
|
-
* await client.joinRoom({
|
|
1173
|
-
* roomSessionId: 'the-room-session-id',
|
|
1174
|
-
* endUserId: 'player-99',
|
|
1175
|
-
* })
|
|
1176
|
-
* ```
|
|
1177
|
-
*/
|
|
1178
|
-
async joinRoom(options) {
|
|
1179
|
-
validateJoinOptions(options);
|
|
1180
|
-
// joinRoom is a credentialed Workspace connection, not a publication.
|
|
1181
|
-
// Reset this before further validation so a rejected normal attempt also
|
|
1182
|
-
// cannot leave a reused client in published-session mode.
|
|
1183
|
-
this._isPublishedChatSession = false;
|
|
1184
|
-
const stored = this._storedConfig ?? {};
|
|
1185
|
-
const url = stored.url ?? "https://realtime-api.convai.com";
|
|
1186
|
-
if (!stored.apiKey && !stored.authToken) {
|
|
1187
|
-
throw new Error("Either apiKey or authToken is required to join a room");
|
|
1188
|
-
}
|
|
1189
|
-
this._logRtviMessages = stored.logRtviMessages !== false;
|
|
1190
|
-
this._messageHandler.setRtviMessageLogging(this._logRtviMessages);
|
|
1191
|
-
this.updateState({ isConnecting: true });
|
|
1192
|
-
try {
|
|
1193
|
-
const connectionData = await this._withProvisioningRetry(stored, async () => {
|
|
1194
|
-
const response = await fetch(`${url}/connect`, {
|
|
1195
|
-
method: "POST",
|
|
1196
|
-
headers: {
|
|
1197
|
-
"Content-Type": "application/json",
|
|
1198
|
-
...(stored.apiKey
|
|
1199
|
-
? { "X-API-Key": stored.apiKey }
|
|
1200
|
-
: { "API-AUTH-TOKEN": stored.authToken }),
|
|
1201
|
-
},
|
|
1202
|
-
body: JSON.stringify(buildJoinRequestBody(options)),
|
|
1203
|
-
});
|
|
1204
|
-
if (!response.ok) {
|
|
1205
|
-
const text = await response.text();
|
|
1206
|
-
let parsed = text;
|
|
1207
|
-
try {
|
|
1208
|
-
parsed = JSON.parse(text);
|
|
583
|
+
else if (errorText) {
|
|
584
|
+
errorMessage = `${errorMessage}: ${errorText}`;
|
|
1209
585
|
}
|
|
1210
|
-
catch {
|
|
1211
|
-
/* keep the raw text as the detail */
|
|
1212
|
-
}
|
|
1213
|
-
throw roomErrorFromResponse(response.status, parsed);
|
|
1214
586
|
}
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
const initialCharacterId = rosterCharacters?.[0]?.character_id;
|
|
1221
|
-
const joinConfig = {
|
|
1222
|
-
...stored,
|
|
1223
|
-
characters: undefined,
|
|
1224
|
-
characterId: initialCharacterId ?? stored.characterId,
|
|
1225
|
-
endUserId: options.endUserId,
|
|
1226
|
-
...(options.endUserMetadata
|
|
1227
|
-
? { endUserMetadata: options.endUserMetadata }
|
|
1228
|
-
: {}),
|
|
1229
|
-
};
|
|
1230
|
-
this._storedConfig = joinConfig;
|
|
1231
|
-
const configWithDefaults = this.applyConfigDefaults(joinConfig, {
|
|
1232
|
-
requireCredential: false,
|
|
1233
|
-
});
|
|
1234
|
-
return await this.consumeConnectionData(connectionData, configWithDefaults);
|
|
1235
|
-
}
|
|
1236
|
-
catch (error) {
|
|
1237
|
-
await this.cleanupWebSocketSessionAfterConnectFailure();
|
|
1238
|
-
this.updateState({ isConnected: false, isConnecting: false });
|
|
1239
|
-
this.emit("error", error);
|
|
1240
|
-
throw error;
|
|
1241
|
-
}
|
|
1242
|
-
}
|
|
1243
|
-
/**
|
|
1244
|
-
* Add default URL, apply RTVI-logging config, and validate that either an
|
|
1245
|
-
* API key or auth token plus a character ID are present.
|
|
1246
|
-
* Shared by connect() and connectWithConnectionData() so both derive
|
|
1247
|
-
* `configWithDefaults` identically.
|
|
1248
|
-
*
|
|
1249
|
-
* `connect()` performs its own /connect fetch and always needs a credential
|
|
1250
|
-
* to do so, so it uses the default `requireCredential: true`.
|
|
1251
|
-
* `connectWithConnectionData()` consumes a response obtained elsewhere (e.g.
|
|
1252
|
-
* a proxy that holds the credential server-side) and passes
|
|
1253
|
-
* `requireCredential: false` — a character ID is still required either way.
|
|
1254
|
-
*/
|
|
1255
|
-
applyConfigDefaults(finalConfig, opts = {}) {
|
|
1256
|
-
const { requireCredential = true } = opts;
|
|
1257
|
-
// Add default URL if not provided
|
|
1258
|
-
const configWithDefaults = {
|
|
1259
|
-
url: "https://realtime-api.convai.com",
|
|
1260
|
-
...finalConfig,
|
|
1261
|
-
};
|
|
1262
|
-
this._logRtviMessages = configWithDefaults.logRtviMessages !== false;
|
|
1263
|
-
this._messageHandler.setRtviMessageLogging(this._logRtviMessages);
|
|
1264
|
-
const hasApiKey = Boolean(configWithDefaults.apiKey);
|
|
1265
|
-
const hasAuthToken = Boolean(configWithDefaults.authToken);
|
|
1266
|
-
const missingCredential = requireCredential && !hasApiKey && !hasAuthToken;
|
|
1267
|
-
// Runs for every topology: the v2 protocol constraints rule out
|
|
1268
|
-
// sharedSessionKey and extra human capacity in single-character rooms too.
|
|
1269
|
-
validateProtocolTopology(configWithDefaults);
|
|
1270
|
-
const roster = isRosterConfig(configWithDefaults);
|
|
1271
|
-
if (missingCredential || (!roster && !configWithDefaults.characterId)) {
|
|
1272
|
-
throw new Error("Either apiKey or authToken is required, and characterId is required");
|
|
1273
|
-
}
|
|
1274
|
-
// A roster names its members individually; every roster rule is checked
|
|
1275
|
-
// before a request is built.
|
|
1276
|
-
if (roster) {
|
|
1277
|
-
validateRosterConfig(configWithDefaults);
|
|
1278
|
-
const initial = configWithDefaults.characters[0].characterId.trim();
|
|
1279
|
-
return {
|
|
1280
|
-
...configWithDefaults,
|
|
1281
|
-
// Singular readouts mirror characters[0], exactly as the wire's own
|
|
1282
|
-
// top-level session_id/character_session_id aliases do.
|
|
1283
|
-
characterId: initial,
|
|
1284
|
-
characterVersion: null,
|
|
1285
|
-
characterReference: initial,
|
|
1286
|
-
};
|
|
1287
|
-
}
|
|
1288
|
-
// Split the character id from its version selector. The bare UUID names
|
|
1289
|
-
// the character everywhere inside the SDK (memory, character info); the
|
|
1290
|
-
// joined reference is what the runtime receives.
|
|
1291
|
-
const reference = resolveCharacterReference(configWithDefaults.characterId, configWithDefaults.characterVersion);
|
|
1292
|
-
return {
|
|
1293
|
-
...configWithDefaults,
|
|
1294
|
-
characterId: reference.characterId,
|
|
1295
|
-
characterVersion: reference.version,
|
|
1296
|
-
characterReference: reference.reference,
|
|
1297
|
-
};
|
|
1298
|
-
}
|
|
1299
|
-
/**
|
|
1300
|
-
* Consume a /connect response and bring up the transport.
|
|
1301
|
-
* `connect()` calls this after its own fetch; external callers use it
|
|
1302
|
-
* when the response came from a proxy.
|
|
1303
|
-
*/
|
|
1304
|
-
async consumeConnectionData(connectionData, configWithDefaults) {
|
|
1305
|
-
// Store connection config (mirrors connect()'s pre-fetch setup so this
|
|
1306
|
-
// method is self-sufficient when called directly, without connect()
|
|
1307
|
-
// having run first).
|
|
1308
|
-
//
|
|
1309
|
-
// Known race, judged acceptable: when called from connect(), that method
|
|
1310
|
-
// sets these same fields once, before the /connect fetch, then awaits
|
|
1311
|
-
// the network round trip. If disconnect() -> clearAllManagers() lands in
|
|
1312
|
-
// that await window, the pre-split code left these fields null (nothing
|
|
1313
|
-
// re-set them post-fetch), whereas re-deriving them here re-arms them
|
|
1314
|
-
// once the response arrives. Checked `_isDisconnecting` as a guard: it
|
|
1315
|
-
// doesn't cover this window — it's flipped back to false (in a
|
|
1316
|
-
// `finally`) before clearAllManagers() runs on the websocket path, and
|
|
1317
|
-
// disconnect()'s LiveKit-room path never sets it at all — so it can't
|
|
1318
|
-
// reliably detect "a disconnect landed mid-connect" here. Even
|
|
1319
|
-
// pre-split, connect() never checked for a concurrent disconnect after
|
|
1320
|
-
// the fetch resolved; it always went on to establish a new room/session
|
|
1321
|
-
// regardless. Re-arming these fields keeps them consistent with the
|
|
1322
|
-
// connection actually being brought up in that case, rather than
|
|
1323
|
-
// silently stale. Building a proper cancellation token for
|
|
1324
|
-
// connect()-vs-disconnect() races is out of scope for this additive
|
|
1325
|
-
// split.
|
|
1326
|
-
this._apiKey = configWithDefaults.apiKey ?? null;
|
|
1327
|
-
this._authToken = configWithDefaults.authToken ?? null;
|
|
1328
|
-
this._characterId = configWithDefaults.characterId;
|
|
1329
|
-
this._characterVersion = configWithDefaults.characterVersion ?? null;
|
|
1330
|
-
this._characterReference = configWithDefaults.characterReference;
|
|
1331
|
-
const connType = configWithDefaults.enableVideo ? "video" : "audio";
|
|
1332
|
-
this._connectionType = connType;
|
|
1333
|
-
const transportType = configWithDefaults.transport ?? "livekit";
|
|
1334
|
-
this._activeTransport = transportType;
|
|
1335
|
-
const effectiveVisionInputConfig = resolveVisionInputConfig(configWithDefaults);
|
|
1336
|
-
// Capture session identifiers from /connect response
|
|
1337
|
-
if (connectionData.character_session_id) {
|
|
1338
|
-
this._characterSessionId = connectionData.character_session_id;
|
|
1339
|
-
// Emitted here rather than in connect(), so the proxy-auth path
|
|
1340
|
-
// (connectWithConnectionData) surfaces the session id too -- both routes
|
|
1341
|
-
// establish it at exactly this point.
|
|
1342
|
-
this.emit("characterSessionId", connectionData.character_session_id);
|
|
1343
|
-
this._storedConfig = {
|
|
1344
|
-
...this._storedConfig,
|
|
1345
|
-
characterSessionId: connectionData.character_session_id,
|
|
1346
|
-
};
|
|
1347
|
-
}
|
|
1348
|
-
// The server answers a capability request with what it actually selected.
|
|
1349
|
-
// A silent downgrade would leave the client parsing v2 frames as v1 (or the
|
|
1350
|
-
// reverse), so each selection is validated and the model-output version the
|
|
1351
|
-
// message handler decodes is set from the server's answer, not the request.
|
|
1352
|
-
const selectedCapabilities = connectionData.capabilities ?? {};
|
|
1353
|
-
const requestedCapabilities = configWithDefaults.capabilities;
|
|
1354
|
-
validateActionProtocolSelection(requestedCapabilities?.actionProtocolVersion, selectedCapabilities.action_protocol_version);
|
|
1355
|
-
this._messageHandler.setModelOutputVersion(validateModelOutputVersionSelection(requestedCapabilities?.modelOutputVersion, selectedCapabilities.model_output_version));
|
|
1356
|
-
this._requestTraceId =
|
|
1357
|
-
connectionData.request_trace_id ?? null;
|
|
1358
|
-
// Seed the roster before the transport comes up, so the first bot-ready and
|
|
1359
|
-
// any early roster change already have instances to match against.
|
|
1360
|
-
if (this._roster.applyConnectResponse(connectionData)) {
|
|
1361
|
-
this.updateState({
|
|
1362
|
-
characters: this._roster.characters,
|
|
1363
|
-
activeMembershipId: this._roster.activeMembershipId,
|
|
1364
|
-
});
|
|
1365
|
-
}
|
|
1366
|
-
if (connectionData.end_user_id) {
|
|
1367
|
-
this._endUserId = connectionData.end_user_id;
|
|
1368
|
-
}
|
|
1369
|
-
if (connectionData.end_user_metadata) {
|
|
1370
|
-
this._endUserMetadata = connectionData.end_user_metadata;
|
|
1371
|
-
}
|
|
1372
|
-
if (transportType === "websocket") {
|
|
1373
|
-
// ── WebSocket (Pipecat) transport path ────────────────────────────
|
|
1374
|
-
// Server returns the WebSocket URL in room_url (same field as LiveKit).
|
|
1375
|
-
const wsUrl = connectionData.room_url;
|
|
1376
|
-
if (!wsUrl) {
|
|
1377
|
-
throw new Error("No WebSocket URL returned from /connect endpoint (expected room_url)");
|
|
587
|
+
catch {
|
|
588
|
+
if (errorText)
|
|
589
|
+
errorMessage = `${errorMessage}: ${errorText}`;
|
|
590
|
+
}
|
|
591
|
+
throw new Error(errorMessage);
|
|
1378
592
|
}
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
593
|
+
const connectionData = await response.json();
|
|
594
|
+
const selectedCapabilities = connectionData.capabilities ?? {};
|
|
595
|
+
const requestedCapabilities = configWithDefaults.capabilities;
|
|
596
|
+
validateActionProtocolSelection(requestedCapabilities?.actionProtocolVersion, selectedCapabilities.action_protocol_version);
|
|
597
|
+
validateBotLlmTextModeSelection(requestedCapabilities?.botLlmTextMode, selectedCapabilities.bot_llm_text_mode);
|
|
598
|
+
this._messageHandler.setModelOutputVersion(validateModelOutputVersionSelection(requestedCapabilities?.modelOutputVersion, selectedCapabilities.model_output_version));
|
|
599
|
+
// Capture session identifiers from /connect response
|
|
600
|
+
if (connectionData.character_session_id) {
|
|
601
|
+
this._characterSessionId = connectionData.character_session_id;
|
|
602
|
+
this._storedConfig = {
|
|
603
|
+
...this._storedConfig,
|
|
604
|
+
characterSessionId: connectionData.character_session_id,
|
|
605
|
+
};
|
|
1382
606
|
}
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
// after the connection is established (enableMic(false) works once the
|
|
1386
|
-
// stream exists, but not before it is initialized).
|
|
1387
|
-
this._wsSession = _wsFactory((payload) => this._messageHandler.handleDataReceivedPublic(payload), true);
|
|
1388
|
-
const webSocketVisionEnabled = configWithDefaults.enableVideo === true &&
|
|
1389
|
-
effectiveVisionInputConfig?.enabled !== false;
|
|
1390
|
-
if (webSocketVisionEnabled) {
|
|
1391
|
-
this._videoManager.setWebSocketVisionMessageSender((type, data) => {
|
|
1392
|
-
if (this._activeTransport !== "websocket" ||
|
|
1393
|
-
!this._wsSession?.isConnected) {
|
|
1394
|
-
return;
|
|
1395
|
-
}
|
|
1396
|
-
this._wsSession.sendMessage(type, data);
|
|
1397
|
-
});
|
|
607
|
+
if (connectionData.end_user_id) {
|
|
608
|
+
this._endUserId = connectionData.end_user_id;
|
|
1398
609
|
}
|
|
1399
|
-
|
|
1400
|
-
this.
|
|
610
|
+
if (connectionData.end_user_metadata) {
|
|
611
|
+
this._endUserMetadata = connectionData.end_user_metadata;
|
|
1401
612
|
}
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
return;
|
|
613
|
+
if (transportType === "websocket") {
|
|
614
|
+
// ── WebSocket (Pipecat) transport path ────────────────────────────
|
|
615
|
+
// Server returns the WebSocket URL in room_url (same field as LiveKit).
|
|
616
|
+
const wsUrl = connectionData.room_url;
|
|
617
|
+
if (!wsUrl) {
|
|
618
|
+
throw new Error("No WebSocket URL returned from /connect endpoint (expected room_url)");
|
|
1409
619
|
}
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
620
|
+
if (!_wsFactory) {
|
|
621
|
+
throw new Error("[ConvaiClient] WebSocket transport is not registered. " +
|
|
622
|
+
"Add `import '@convai/web-sdk/vanilla/websocket'` before calling connect().");
|
|
623
|
+
}
|
|
624
|
+
// Always enable mic at construction so WavMediaManager initializes the
|
|
625
|
+
// audio stream during connect(). If startWithAudioOn is false, we mute
|
|
626
|
+
// after the connection is established (enableMic(false) works once the
|
|
627
|
+
// stream exists, but not before it is initialized).
|
|
628
|
+
this._wsSession = _wsFactory((payload) => this._messageHandler.handleDataReceivedPublic(payload), true);
|
|
629
|
+
const webSocketVisionEnabled = configWithDefaults.enableVideo === true &&
|
|
630
|
+
effectiveVisionInputConfig?.enabled !== false;
|
|
631
|
+
if (webSocketVisionEnabled) {
|
|
632
|
+
this._videoManager.setWebSocketVisionMessageSender((type, data) => {
|
|
633
|
+
if (this._activeTransport !== "websocket" ||
|
|
634
|
+
!this._wsSession?.isConnected) {
|
|
635
|
+
return;
|
|
636
|
+
}
|
|
637
|
+
this._wsSession.sendMessage(type, data);
|
|
638
|
+
});
|
|
639
|
+
}
|
|
640
|
+
else {
|
|
641
|
+
this._videoManager.setWebSocketVisionDisabled();
|
|
642
|
+
}
|
|
643
|
+
const wsSession = this._wsSession;
|
|
644
|
+
wsSession.on("botAudioTrack", (track) => {
|
|
645
|
+
this.emit("botAudioTrack", track);
|
|
646
|
+
});
|
|
647
|
+
wsSession.on("disconnected", () => {
|
|
648
|
+
if (this._wsSession !== wsSession || this._isDisconnecting) {
|
|
1424
649
|
return;
|
|
1425
650
|
}
|
|
1426
|
-
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
|
|
651
|
+
this._connectionStateHandler.handleDisconnected();
|
|
652
|
+
this.clearAllManagers();
|
|
653
|
+
});
|
|
654
|
+
this._audioManager.setWebSocketSession(this._wsSession);
|
|
655
|
+
// Wait for WebSocket transport to connect (fires before bot-ready)
|
|
656
|
+
await new Promise((resolve, reject) => {
|
|
657
|
+
let settled = false;
|
|
658
|
+
const cleanupConnectListeners = () => {
|
|
659
|
+
wsSession.off("connected", onConnected);
|
|
660
|
+
wsSession.off("error", onError);
|
|
661
|
+
wsSession.off("disconnected", onDisconnected);
|
|
662
|
+
};
|
|
663
|
+
const settle = (callback) => {
|
|
664
|
+
if (settled) {
|
|
665
|
+
return;
|
|
666
|
+
}
|
|
667
|
+
settled = true;
|
|
668
|
+
cleanupConnectListeners();
|
|
669
|
+
callback();
|
|
670
|
+
};
|
|
671
|
+
const onConnected = () => {
|
|
672
|
+
settle(resolve);
|
|
673
|
+
};
|
|
674
|
+
const onError = (err) => {
|
|
675
|
+
settle(() => {
|
|
676
|
+
reject(err instanceof Error ? err : new Error(String(err)));
|
|
677
|
+
});
|
|
678
|
+
};
|
|
679
|
+
const onDisconnected = () => {
|
|
680
|
+
settle(() => {
|
|
681
|
+
reject(new Error("WebSocket disconnected before connect completed"));
|
|
682
|
+
});
|
|
683
|
+
};
|
|
684
|
+
wsSession.on("connected", onConnected);
|
|
685
|
+
wsSession.on("error", onError);
|
|
686
|
+
wsSession.on("disconnected", onDisconnected);
|
|
687
|
+
// connectWithUrl calls initDevices() then connect({ wsUrl }) — matches sandbox flow
|
|
688
|
+
wsSession.connectWithUrl(wsUrl).catch((err) => {
|
|
689
|
+
settle(() => {
|
|
690
|
+
reject(err instanceof Error ? err : new Error(String(err)));
|
|
691
|
+
});
|
|
1450
692
|
});
|
|
1451
693
|
});
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
694
|
+
// For WebSocket, mic is already streaming after connect.
|
|
695
|
+
// Only mute if the caller explicitly sets startWithAudioOn: false.
|
|
696
|
+
// Defaulting to on matches the sandbox behavior and ensures the bot
|
|
697
|
+
// sees audio when it initializes (avoids server "tap mic to talk" prompt).
|
|
698
|
+
if (configWithDefaults.startWithAudioOn === false) {
|
|
699
|
+
this._wsSession.enableMic(false);
|
|
700
|
+
this._audioManager.syncWsAudioState(false);
|
|
701
|
+
}
|
|
702
|
+
else {
|
|
703
|
+
this._audioManager.syncWsAudioState(true);
|
|
704
|
+
}
|
|
705
|
+
if (webSocketVisionEnabled &&
|
|
706
|
+
configWithDefaults.enableVideo &&
|
|
707
|
+
configWithDefaults.startWithVideoOn) {
|
|
708
|
+
await this._videoManager.enableVideo();
|
|
709
|
+
}
|
|
1460
710
|
}
|
|
1461
711
|
else {
|
|
1462
|
-
this.
|
|
712
|
+
this._videoManager.setWebSocketVisionMessageSender(null);
|
|
713
|
+
// ── LiveKit transport path (default) ─────────────────────────────
|
|
714
|
+
await this._room.connect(connectionData.room_url, connectionData.token, {
|
|
715
|
+
rtcConfig: {
|
|
716
|
+
iceTransportPolicy: "relay",
|
|
717
|
+
},
|
|
718
|
+
});
|
|
719
|
+
if (configWithDefaults.startWithAudioOn) {
|
|
720
|
+
await this._room.localParticipant.setMicrophoneEnabled(true, {
|
|
721
|
+
echoCancellation: this._audioSettings.echoCancellation,
|
|
722
|
+
noiseSuppression: this._audioSettings.noiseSuppression,
|
|
723
|
+
autoGainControl: this._audioSettings.autoGainControl,
|
|
724
|
+
sampleRate: this._audioSettings.sampleRate,
|
|
725
|
+
channelCount: this._audioSettings.channelCount,
|
|
726
|
+
});
|
|
727
|
+
}
|
|
728
|
+
if (configWithDefaults.enableVideo &&
|
|
729
|
+
configWithDefaults.startWithVideoOn) {
|
|
730
|
+
await this._room.localParticipant.setCameraEnabled(true);
|
|
731
|
+
}
|
|
732
|
+
this._audioManager.syncStateFromRoom({ emit: true });
|
|
733
|
+
this._participantSid = this._room.localParticipant.sid;
|
|
1463
734
|
}
|
|
1464
|
-
if
|
|
1465
|
-
|
|
1466
|
-
configWithDefaults.
|
|
1467
|
-
await this._videoManager.enableVideo();
|
|
735
|
+
// Apply custom mapper to blendshape queue if provided
|
|
736
|
+
if (configWithDefaults.blendshapeConfig?.customMapper) {
|
|
737
|
+
this.blendshapeQueue.setMapper(configWithDefaults.blendshapeConfig.customMapper);
|
|
1468
738
|
}
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
rtcConfig: {
|
|
1475
|
-
// Defaults to "relay" (TURN-only) for reliable NAT traversal
|
|
1476
|
-
// against Convai's hosted LiveKit. Set iceTransportPolicy: "all"
|
|
1477
|
-
// when pointing at a LiveKit without TURN (e.g. local dev).
|
|
1478
|
-
iceTransportPolicy: configWithDefaults.iceTransportPolicy ?? "relay",
|
|
1479
|
-
},
|
|
739
|
+
this.updateState({
|
|
740
|
+
isConnected: true,
|
|
741
|
+
isConnecting: false,
|
|
742
|
+
endUserId: this._endUserId,
|
|
743
|
+
endUserMetadata: this._endUserMetadata,
|
|
1480
744
|
});
|
|
1481
|
-
if
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
745
|
+
// Initialize memory manager if we have authentication and endUserId
|
|
746
|
+
if (this._endUserId) {
|
|
747
|
+
if (this._apiKey) {
|
|
748
|
+
// Use API key authentication
|
|
749
|
+
this._memoryManager = new MemoryManager(this._apiKey, this._characterId, this._endUserId, undefined, // Use default base URL
|
|
750
|
+
false);
|
|
751
|
+
}
|
|
752
|
+
else if (this._authToken) {
|
|
753
|
+
// Use auth token authentication
|
|
754
|
+
this._memoryManager = new MemoryManager(this._authToken, this._characterId, this._endUserId, undefined, // Use default base URL
|
|
755
|
+
true);
|
|
756
|
+
}
|
|
1493
757
|
}
|
|
1494
|
-
this.
|
|
1495
|
-
this.
|
|
1496
|
-
}
|
|
1497
|
-
// Apply custom mapper to blendshape queue if provided
|
|
1498
|
-
if (configWithDefaults.blendshapeConfig?.customMapper) {
|
|
1499
|
-
this.blendshapeQueue.setMapper(configWithDefaults.blendshapeConfig.customMapper);
|
|
758
|
+
this.emit("connect");
|
|
759
|
+
this._startClientReadyHandshake();
|
|
1500
760
|
}
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
if (this._apiKey) {
|
|
1510
|
-
// Use API key authentication
|
|
1511
|
-
this._memoryManager = new MemoryManager(this._apiKey, this._characterId, this._endUserId, undefined, // Use default base URL
|
|
1512
|
-
false);
|
|
1513
|
-
}
|
|
1514
|
-
else if (this._authToken) {
|
|
1515
|
-
// Use auth token authentication
|
|
1516
|
-
this._memoryManager = new MemoryManager(this._authToken, this._characterId, this._endUserId, undefined, // Use default base URL
|
|
1517
|
-
true);
|
|
1518
|
-
}
|
|
761
|
+
catch (error) {
|
|
762
|
+
await this.cleanupWebSocketSessionAfterConnectFailure();
|
|
763
|
+
this.updateState({
|
|
764
|
+
isConnected: false,
|
|
765
|
+
isConnecting: false,
|
|
766
|
+
});
|
|
767
|
+
this.emit("error", error);
|
|
768
|
+
throw error;
|
|
1519
769
|
}
|
|
1520
|
-
this.emit("connect");
|
|
1521
|
-
this._startClientReadyHandshake();
|
|
1522
770
|
}
|
|
1523
771
|
async cleanupWebSocketSessionAfterConnectFailure() {
|
|
1524
772
|
if (this._activeTransport !== "websocket" || !this._wsSession) {
|
|
@@ -1579,7 +827,6 @@ export class ConvaiClient extends EventEmitter {
|
|
|
1579
827
|
this._characterSessionId = "-1";
|
|
1580
828
|
this._endUserId = null;
|
|
1581
829
|
this._endUserMetadata = null;
|
|
1582
|
-
this._requestTraceId = null;
|
|
1583
830
|
this._memoryManager = null;
|
|
1584
831
|
this._connectionType = "text";
|
|
1585
832
|
this._activeTransport = "livekit";
|
|
@@ -1604,95 +851,58 @@ export class ConvaiClient extends EventEmitter {
|
|
|
1604
851
|
}),
|
|
1605
852
|
signal: options.signal,
|
|
1606
853
|
});
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
if (retryTransientFailure && response.status >= 500) {
|
|
1610
|
-
throw new RetryablePublishedChatExchangeError();
|
|
1611
|
-
}
|
|
1612
|
-
const errorText = await response.text();
|
|
1613
|
-
let errorMessage = `HTTP ${response.status}`;
|
|
1614
|
-
try {
|
|
1615
|
-
const errorData = JSON.parse(errorText);
|
|
1616
|
-
this._requestTraceId =
|
|
1617
|
-
errorData.request_trace_id ?? errorData.requestTraceId ?? null;
|
|
1618
|
-
const raw = errorData.message ?? errorData.error ?? errorData.detail;
|
|
1619
|
-
errorMessage =
|
|
1620
|
-
typeof raw === "string"
|
|
1621
|
-
? raw
|
|
1622
|
-
: raw == null
|
|
1623
|
-
? errorMessage
|
|
1624
|
-
: JSON.stringify(raw);
|
|
1625
|
-
}
|
|
1626
|
-
catch {
|
|
1627
|
-
if (errorText)
|
|
1628
|
-
errorMessage = `${errorMessage}: ${errorText}`;
|
|
1629
|
-
}
|
|
1630
|
-
throw new Error(errorMessage);
|
|
1631
|
-
}
|
|
1632
|
-
let connectionData;
|
|
1633
|
-
try {
|
|
1634
|
-
connectionData = await response.json();
|
|
1635
|
-
}
|
|
1636
|
-
catch {
|
|
1637
|
-
if (retryTransientFailure) {
|
|
1638
|
-
throw new RetryablePublishedChatExchangeError();
|
|
1639
|
-
}
|
|
1640
|
-
throw new Error("Published chat returned an invalid connection response");
|
|
1641
|
-
}
|
|
1642
|
-
throwIfAborted();
|
|
1643
|
-
if (typeof connectionData !== "object" ||
|
|
1644
|
-
connectionData === null ||
|
|
1645
|
-
typeof connectionData.room_url !== "string" ||
|
|
1646
|
-
typeof connectionData.token !== "string" ||
|
|
1647
|
-
typeof connectionData.character_session_id !==
|
|
1648
|
-
"string") {
|
|
1649
|
-
if (retryTransientFailure) {
|
|
1650
|
-
throw new RetryablePublishedChatExchangeError();
|
|
1651
|
-
}
|
|
1652
|
-
throw new Error("Published chat returned an invalid connection response");
|
|
1653
|
-
}
|
|
1654
|
-
return connectionData;
|
|
1655
|
-
};
|
|
1656
|
-
const requestConnection = async (retryTransientFailure) => {
|
|
1657
|
-
let response;
|
|
1658
|
-
try {
|
|
1659
|
-
response = await exchangeGrant();
|
|
1660
|
-
}
|
|
1661
|
-
catch (error) {
|
|
1662
|
-
throwIfAborted();
|
|
1663
|
-
if (retryTransientFailure) {
|
|
1664
|
-
throw new RetryablePublishedChatExchangeError();
|
|
1665
|
-
}
|
|
1666
|
-
throw error;
|
|
1667
|
-
}
|
|
1668
|
-
return readConnectionResponse(response, retryTransientFailure);
|
|
1669
|
-
};
|
|
1670
|
-
let connectionData;
|
|
854
|
+
let response;
|
|
855
|
+
let exchangeRetried = false;
|
|
1671
856
|
try {
|
|
1672
|
-
|
|
857
|
+
response = await exchangeGrant();
|
|
1673
858
|
}
|
|
1674
859
|
catch (error) {
|
|
1675
860
|
throwIfAborted();
|
|
1676
|
-
|
|
1677
|
-
|
|
861
|
+
// A response can be lost after Core consumed the single-use grant.
|
|
862
|
+
// Retry exactly once with the same attempt key; Core/Character API
|
|
863
|
+
// replay that logical attempt instead of creating a second session.
|
|
864
|
+
exchangeRetried = true;
|
|
865
|
+
response = await exchangeGrant();
|
|
866
|
+
}
|
|
867
|
+
if (!exchangeRetried && response.status >= 500) {
|
|
868
|
+
throwIfAborted();
|
|
869
|
+
response = await exchangeGrant();
|
|
870
|
+
}
|
|
871
|
+
throwIfAborted();
|
|
872
|
+
if (!response.ok) {
|
|
873
|
+
const errorText = await response.text();
|
|
874
|
+
let errorMessage = `HTTP ${response.status}`;
|
|
875
|
+
try {
|
|
876
|
+
const errorData = JSON.parse(errorText);
|
|
877
|
+
this._requestTraceId =
|
|
878
|
+
errorData.request_trace_id ?? errorData.requestTraceId ?? null;
|
|
879
|
+
const raw = errorData.message ?? errorData.error ?? errorData.detail;
|
|
880
|
+
errorMessage =
|
|
881
|
+
typeof raw === "string"
|
|
882
|
+
? raw
|
|
883
|
+
: raw == null
|
|
884
|
+
? errorMessage
|
|
885
|
+
: JSON.stringify(raw);
|
|
886
|
+
}
|
|
887
|
+
catch {
|
|
888
|
+
if (errorText)
|
|
889
|
+
errorMessage = `${errorMessage}: ${errorText}`;
|
|
1678
890
|
}
|
|
1679
|
-
|
|
1680
|
-
// single-use grant. Retry exactly once with the same attempt key;
|
|
1681
|
-
// Core/Character API replay that logical attempt instead of creating
|
|
1682
|
-
// a second session.
|
|
1683
|
-
connectionData = await requestConnection(false);
|
|
891
|
+
throw new Error(errorMessage);
|
|
1684
892
|
}
|
|
893
|
+
const connectionData = await response.json();
|
|
1685
894
|
throwIfAborted();
|
|
895
|
+
if (typeof connectionData.room_url !== "string" ||
|
|
896
|
+
typeof connectionData.token !== "string" ||
|
|
897
|
+
typeof connectionData.character_session_id !== "string") {
|
|
898
|
+
throw new Error("Published chat returned an invalid connection response");
|
|
899
|
+
}
|
|
1686
900
|
this._requestTraceId = connectionData.request_trace_id ?? null;
|
|
1687
901
|
this._characterSessionId = connectionData.character_session_id;
|
|
1688
902
|
this._endUserId = connectionData.end_user_id ?? null;
|
|
1689
903
|
this._endUserMetadata = connectionData.end_user_metadata ?? null;
|
|
1690
904
|
await this._room.connect(connectionData.room_url, connectionData.token, {
|
|
1691
|
-
rtcConfig: {
|
|
1692
|
-
// Hosted Core has TURN, so relay is the safe default. On-prem
|
|
1693
|
-
// deployments without TURN may explicitly allow host/srflx paths.
|
|
1694
|
-
iceTransportPolicy: options.iceTransportPolicy ?? "relay",
|
|
1695
|
-
},
|
|
905
|
+
rtcConfig: { iceTransportPolicy: "relay" },
|
|
1696
906
|
});
|
|
1697
907
|
throwIfAborted();
|
|
1698
908
|
this._audioManager.syncStateFromRoom({ emit: true });
|
|
@@ -1783,16 +993,7 @@ export class ConvaiClient extends EventEmitter {
|
|
|
1783
993
|
*/
|
|
1784
994
|
async disconnect() {
|
|
1785
995
|
this._stopClientReadyHandshake();
|
|
1786
|
-
|
|
1787
|
-
// matched, and none of them runs when there is nothing to tear down. A
|
|
1788
|
-
// command in flight must still be settled either way.
|
|
1789
|
-
this._failPendingRosterCommands("Disconnected before the roster command was acknowledged; its outcome is unknown");
|
|
1790
|
-
if (this._activeTransport === "sse" && this._sseSession) {
|
|
1791
|
-
await this._sseSession.disconnect();
|
|
1792
|
-
this._connectionStateHandler.resetConnectionState();
|
|
1793
|
-
this.clearAllManagers();
|
|
1794
|
-
}
|
|
1795
|
-
else if (this._activeTransport === "websocket" && this._wsSession) {
|
|
996
|
+
if (this._activeTransport === "websocket" && this._wsSession) {
|
|
1796
997
|
this._isDisconnecting = true;
|
|
1797
998
|
try {
|
|
1798
999
|
await this._wsSession.disconnect();
|
|
@@ -1819,15 +1020,6 @@ export class ConvaiClient extends EventEmitter {
|
|
|
1819
1020
|
this.clearAllManagers();
|
|
1820
1021
|
}
|
|
1821
1022
|
}
|
|
1822
|
-
else if (this._state.isConnected || this._state.isConnecting) {
|
|
1823
|
-
// No transport matched, but the client believes it is (or is becoming)
|
|
1824
|
-
// connected -- a connect() that failed after flipping isConnecting, or
|
|
1825
|
-
// was torn down underneath us. Without this branch disconnect() was a
|
|
1826
|
-
// silent no-op here and the flags stayed stuck, so a UI bound to
|
|
1827
|
-
// stateChange kept showing a session that did not exist.
|
|
1828
|
-
this._connectionStateHandler.resetConnectionState();
|
|
1829
|
-
this.clearAllManagers();
|
|
1830
|
-
}
|
|
1831
1023
|
}
|
|
1832
1024
|
/**
|
|
1833
1025
|
* Clear all manager states
|
|
@@ -1836,14 +1028,9 @@ export class ConvaiClient extends EventEmitter {
|
|
|
1836
1028
|
this._connectionType = null;
|
|
1837
1029
|
this._activeTransport = null;
|
|
1838
1030
|
this._wsSession = null;
|
|
1839
|
-
this._failPendingRosterCommands("Disconnected before the roster command was acknowledged; its outcome is unknown");
|
|
1840
|
-
this._roster.reset();
|
|
1841
|
-
this._sseSession = null;
|
|
1842
1031
|
this._apiKey = null;
|
|
1843
1032
|
this._authToken = null;
|
|
1844
1033
|
this._characterId = null;
|
|
1845
|
-
this._characterVersion = null;
|
|
1846
|
-
this._characterReference = null;
|
|
1847
1034
|
this._endUserId = null;
|
|
1848
1035
|
this._endUserMetadata = null;
|
|
1849
1036
|
this._memoryManager = null; // Clear memory manager on disconnect
|
|
@@ -1866,15 +1053,7 @@ export class ConvaiClient extends EventEmitter {
|
|
|
1866
1053
|
* Reset the session ID to start a new conversation
|
|
1867
1054
|
*/
|
|
1868
1055
|
resetSession() {
|
|
1869
|
-
// In a roster room the singular session id is a compatibility mirror of
|
|
1870
|
-
// characters[0], and character sessions are server-owned -- there is no
|
|
1871
|
-
// client-side reset for them. Throwing names the situation; a silent no-op
|
|
1872
|
-
// would look like it worked.
|
|
1873
|
-
if (this._roster.isMultiCharacter) {
|
|
1874
|
-
throw new Error("resetSession() is not supported in a multi-character room; character sessions are server-owned");
|
|
1875
|
-
}
|
|
1876
1056
|
this._characterSessionId = "-1";
|
|
1877
|
-
this.emit("characterSessionId", "-1");
|
|
1878
1057
|
this._messageHandler.reset();
|
|
1879
1058
|
// Clear metrics and disconnect reason when resetting session
|
|
1880
1059
|
this.updateState({ metrics: [], disconnectReason: null });
|
|
@@ -1882,7 +1061,7 @@ export class ConvaiClient extends EventEmitter {
|
|
|
1882
1061
|
/**
|
|
1883
1062
|
* Send a text message to the character
|
|
1884
1063
|
*/
|
|
1885
|
-
sendUserTextMessage(text, options
|
|
1064
|
+
sendUserTextMessage(text, options) {
|
|
1886
1065
|
if (!this.isTransportReady())
|
|
1887
1066
|
return;
|
|
1888
1067
|
if (!text || !text.trim())
|
|
@@ -1907,48 +1086,10 @@ export class ConvaiClient extends EventEmitter {
|
|
|
1907
1086
|
userMessage: text.trim(),
|
|
1908
1087
|
timestamp: this._conversationStartTime,
|
|
1909
1088
|
});
|
|
1910
|
-
|
|
1089
|
+
this.publishMessage("user_text_message", {
|
|
1911
1090
|
text: text.trim(),
|
|
1912
1091
|
participant_sid: this._participantSid || this._room.localParticipant?.sid,
|
|
1913
|
-
};
|
|
1914
|
-
const stateOfMind = normalizeStateOfMind(options.stateOfMind) ?? null;
|
|
1915
|
-
if (stateOfMind)
|
|
1916
|
-
this._stateOfMind = stateOfMind;
|
|
1917
|
-
if (this._activeTransport === "sse") {
|
|
1918
|
-
const requestedDynamicInfo = options.dynamicInfo?.trim();
|
|
1919
|
-
if (requestedDynamicInfo) {
|
|
1920
|
-
this._sseSession?.updateDynamicInfo(requestedDynamicInfo);
|
|
1921
|
-
}
|
|
1922
|
-
const messages = [];
|
|
1923
|
-
const dynamicInfo = this._sseSession?.getDynamicInfo()?.trim();
|
|
1924
|
-
if (dynamicInfo) {
|
|
1925
|
-
messages.push({
|
|
1926
|
-
type: "update-dynamic-info",
|
|
1927
|
-
data: { dynamic_info: { text: dynamicInfo } },
|
|
1928
|
-
});
|
|
1929
|
-
}
|
|
1930
|
-
if (stateOfMind) {
|
|
1931
|
-
messages.push({ type: "update-emotion", data: { state_of_mind: stateOfMind } });
|
|
1932
|
-
}
|
|
1933
|
-
messages.push({
|
|
1934
|
-
type: "user_text_message",
|
|
1935
|
-
data: userTextMessage,
|
|
1936
|
-
...(logicalTurnId !== undefined ? { id: logicalTurnId } : {}),
|
|
1937
|
-
});
|
|
1938
|
-
this._sseSession?.sendMessages(messages);
|
|
1939
|
-
}
|
|
1940
|
-
else {
|
|
1941
|
-
if (stateOfMind) {
|
|
1942
|
-
this.publishMessage("update-emotion", {
|
|
1943
|
-
state_of_mind: stateOfMind,
|
|
1944
|
-
participant_sid: this._participantSid || this._room.localParticipant?.sid,
|
|
1945
|
-
});
|
|
1946
|
-
}
|
|
1947
|
-
this.publishMessage("user_text_message", userTextMessage, logicalTurnId);
|
|
1948
|
-
}
|
|
1949
|
-
// The submitted line is appended locally on every transport, so it shows
|
|
1950
|
-
// without waiting for a server echo. Sessions that do echo it back as a
|
|
1951
|
-
// final user-transcription are deduplicated in the message handler.
|
|
1092
|
+
}, logicalTurnId);
|
|
1952
1093
|
this._messageHandler.addUserTextMessage(text);
|
|
1953
1094
|
}
|
|
1954
1095
|
catch (error) {
|
|
@@ -1959,123 +1100,6 @@ export class ConvaiClient extends EventEmitter {
|
|
|
1959
1100
|
* Return a correlated terminal result for a client-executed tool call.
|
|
1960
1101
|
*/
|
|
1961
1102
|
sendActionResult(result) {
|
|
1962
|
-
const payload = this.buildActionResultPayload(result);
|
|
1963
|
-
this.publishMessage("action-result", payload);
|
|
1964
|
-
}
|
|
1965
|
-
/**
|
|
1966
|
-
* Publish one correlated terminal result and wait for Core's exact
|
|
1967
|
-
* `server-response` acknowledgement. This method performs one publication;
|
|
1968
|
-
* callers own any idempotent retry policy.
|
|
1969
|
-
*/
|
|
1970
|
-
async sendActionResultAndWait(result, options = {}) {
|
|
1971
|
-
const timeoutMs = options.timeoutMs ?? ConvaiClient.ACTION_RESULT_ACK_TIMEOUT_MS;
|
|
1972
|
-
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) {
|
|
1973
|
-
throw new Error("Action result acknowledgement timeout must be greater than zero");
|
|
1974
|
-
}
|
|
1975
|
-
if (options.signal?.aborted) {
|
|
1976
|
-
throw new ActionResultAckError("Action result acknowledgement was aborted", {
|
|
1977
|
-
reason: "aborted",
|
|
1978
|
-
toolCallId: result.id,
|
|
1979
|
-
});
|
|
1980
|
-
}
|
|
1981
|
-
const payload = this.buildActionResultPayload(result);
|
|
1982
|
-
const toolCallId = payload.id;
|
|
1983
|
-
if (this._pendingActionResultAcks.has(toolCallId)) {
|
|
1984
|
-
throw new ActionResultAckError(`An action result acknowledgement is already pending for ${toolCallId}`, {
|
|
1985
|
-
reason: "duplicate_in_flight",
|
|
1986
|
-
toolCallId,
|
|
1987
|
-
});
|
|
1988
|
-
}
|
|
1989
|
-
const reservation = Symbol(toolCallId);
|
|
1990
|
-
this._pendingActionResultAcks.set(toolCallId, reservation);
|
|
1991
|
-
return new Promise((resolve, reject) => {
|
|
1992
|
-
let settled = false;
|
|
1993
|
-
let timeout = null;
|
|
1994
|
-
let unsubscribe = null;
|
|
1995
|
-
const cleanup = () => {
|
|
1996
|
-
if (timeout !== null) {
|
|
1997
|
-
clearTimeout(timeout);
|
|
1998
|
-
timeout = null;
|
|
1999
|
-
}
|
|
2000
|
-
if (unsubscribe) {
|
|
2001
|
-
unsubscribe();
|
|
2002
|
-
unsubscribe = null;
|
|
2003
|
-
}
|
|
2004
|
-
options.signal?.removeEventListener("abort", handleAbort);
|
|
2005
|
-
if (this._pendingActionResultAcks.get(toolCallId) === reservation) {
|
|
2006
|
-
this._pendingActionResultAcks.delete(toolCallId);
|
|
2007
|
-
}
|
|
2008
|
-
};
|
|
2009
|
-
const resolveOnce = (ack) => {
|
|
2010
|
-
if (settled)
|
|
2011
|
-
return;
|
|
2012
|
-
settled = true;
|
|
2013
|
-
cleanup();
|
|
2014
|
-
resolve(ack);
|
|
2015
|
-
};
|
|
2016
|
-
const rejectOnce = (error) => {
|
|
2017
|
-
if (settled)
|
|
2018
|
-
return;
|
|
2019
|
-
settled = true;
|
|
2020
|
-
cleanup();
|
|
2021
|
-
reject(error);
|
|
2022
|
-
};
|
|
2023
|
-
const handleAbort = () => {
|
|
2024
|
-
rejectOnce(new ActionResultAckError("Action result acknowledgement was aborted", {
|
|
2025
|
-
reason: "aborted",
|
|
2026
|
-
toolCallId,
|
|
2027
|
-
}));
|
|
2028
|
-
};
|
|
2029
|
-
const handleServerResponse = (response) => {
|
|
2030
|
-
if (!response || typeof response !== "object")
|
|
2031
|
-
return;
|
|
2032
|
-
const candidate = response;
|
|
2033
|
-
if (candidate.event_type !== "action-result")
|
|
2034
|
-
return;
|
|
2035
|
-
const extras = candidate.extras;
|
|
2036
|
-
if (!extras || extras.tool_call_id !== toolCallId)
|
|
2037
|
-
return;
|
|
2038
|
-
const message = typeof candidate.message === "string" ? candidate.message : null;
|
|
2039
|
-
if (candidate.status === "success") {
|
|
2040
|
-
resolveOnce({
|
|
2041
|
-
toolCallId,
|
|
2042
|
-
status: "success",
|
|
2043
|
-
message,
|
|
2044
|
-
idempotent: extras.idempotent === true,
|
|
2045
|
-
});
|
|
2046
|
-
return;
|
|
2047
|
-
}
|
|
2048
|
-
if (candidate.status !== "error")
|
|
2049
|
-
return;
|
|
2050
|
-
const errorCode = typeof extras.error_code === "string" ? extras.error_code : undefined;
|
|
2051
|
-
rejectOnce(new ActionResultAckError(message ?? errorCode ?? "Core rejected the action result", {
|
|
2052
|
-
reason: "server_error",
|
|
2053
|
-
toolCallId,
|
|
2054
|
-
...(errorCode ? { errorCode } : {}),
|
|
2055
|
-
response: candidate,
|
|
2056
|
-
}));
|
|
2057
|
-
};
|
|
2058
|
-
// Register every terminal observer before publication so a synchronous
|
|
2059
|
-
// WebSocket acknowledgement cannot outrun its waiter.
|
|
2060
|
-
unsubscribe = this.on("serverResponse", handleServerResponse);
|
|
2061
|
-
options.signal?.addEventListener("abort", handleAbort, { once: true });
|
|
2062
|
-
timeout = setTimeout(() => {
|
|
2063
|
-
rejectOnce(new ActionResultAckError(`Timed out waiting for action result acknowledgement for ${toolCallId}`, {
|
|
2064
|
-
reason: "timeout",
|
|
2065
|
-
toolCallId,
|
|
2066
|
-
}));
|
|
2067
|
-
}, timeoutMs);
|
|
2068
|
-
void this.publishMessageAndWaitForLocalPublication("action-result", payload).catch((cause) => {
|
|
2069
|
-
const detail = cause instanceof Error ? `: ${cause.message}` : "";
|
|
2070
|
-
rejectOnce(new ActionResultAckError(`Failed to publish action result ${toolCallId}${detail}`, {
|
|
2071
|
-
reason: "publication_error",
|
|
2072
|
-
toolCallId,
|
|
2073
|
-
cause,
|
|
2074
|
-
}));
|
|
2075
|
-
});
|
|
2076
|
-
});
|
|
2077
|
-
}
|
|
2078
|
-
buildActionResultPayload(result) {
|
|
2079
1103
|
if (!this.isTransportReady()) {
|
|
2080
1104
|
throw new Error("Cannot send an action result before the session is ready");
|
|
2081
1105
|
}
|
|
@@ -2095,17 +1119,17 @@ export class ConvaiClient extends EventEmitter {
|
|
|
2095
1119
|
if (result.error !== undefined && !isJsonValue(result.error)) {
|
|
2096
1120
|
throw new Error("Action result error must contain only finite JSON values");
|
|
2097
1121
|
}
|
|
2098
|
-
const
|
|
2099
|
-
return {
|
|
1122
|
+
const payload = {
|
|
2100
1123
|
id: result.id,
|
|
2101
1124
|
status: result.status,
|
|
2102
1125
|
...(result.output !== undefined && { output: result.output }),
|
|
2103
1126
|
...(result.error !== undefined && { error: result.error }),
|
|
2104
|
-
...(characterSessionId &&
|
|
2105
|
-
characterSessionId !== "-1" && {
|
|
2106
|
-
character_session_id: characterSessionId,
|
|
1127
|
+
...((result.characterSessionId ?? this._characterSessionId) &&
|
|
1128
|
+
(result.characterSessionId ?? this._characterSessionId) !== "-1" && {
|
|
1129
|
+
character_session_id: result.characterSessionId ?? this._characterSessionId,
|
|
2107
1130
|
}),
|
|
2108
1131
|
};
|
|
1132
|
+
this.publishMessage("action-result", payload);
|
|
2109
1133
|
}
|
|
2110
1134
|
/**
|
|
2111
1135
|
* Send a trigger message to invoke specific character actions
|
|
@@ -2207,16 +1231,8 @@ export class ConvaiClient extends EventEmitter {
|
|
|
2207
1231
|
updateDynamicInfo(dynamicInfo) {
|
|
2208
1232
|
if (!this.isTransportReady() || !dynamicInfo?.trim())
|
|
2209
1233
|
return;
|
|
2210
|
-
const nextDynamicInfo = dynamicInfo.trim();
|
|
2211
|
-
if (this._storedConfig) {
|
|
2212
|
-
this._storedConfig = { ...this._storedConfig, dynamicInfo: nextDynamicInfo };
|
|
2213
|
-
}
|
|
2214
|
-
if (this._activeTransport === "sse" && this._sseSession) {
|
|
2215
|
-
this._sseSession.updateDynamicInfo(nextDynamicInfo);
|
|
2216
|
-
return;
|
|
2217
|
-
}
|
|
2218
1234
|
this.publishMessage("update-dynamic-info", {
|
|
2219
|
-
dynamic_info: { text:
|
|
1235
|
+
dynamic_info: { text: dynamicInfo },
|
|
2220
1236
|
participant_sid: this._participantSid || this._room.localParticipant?.sid,
|
|
2221
1237
|
});
|
|
2222
1238
|
}
|
|
@@ -2264,22 +1280,6 @@ export class ConvaiClient extends EventEmitter {
|
|
|
2264
1280
|
if (shouldPreemptLipsync) {
|
|
2265
1281
|
this.startTrackedConversation(`[context-update:${options.mode ?? "append"}]`);
|
|
2266
1282
|
}
|
|
2267
|
-
if (this._activeTransport === "sse" && this._sseSession) {
|
|
2268
|
-
const data = {
|
|
2269
|
-
...(hasText && { text: options.text.trim() }),
|
|
2270
|
-
...(options.mode && { mode: options.mode }),
|
|
2271
|
-
...(options.run_llm && { run_llm: options.run_llm }),
|
|
2272
|
-
...(options.respond_mode && { respond_mode: options.respond_mode }),
|
|
2273
|
-
...(hasAttention && { current_attention_object: options.current_attention_object }),
|
|
2274
|
-
};
|
|
2275
|
-
if (options.run_llm === "false") {
|
|
2276
|
-
this._sseSession.queueMessage("context-update", data);
|
|
2277
|
-
}
|
|
2278
|
-
else {
|
|
2279
|
-
this._sseSession.sendMessage("context-update", data);
|
|
2280
|
-
}
|
|
2281
|
-
return;
|
|
2282
|
-
}
|
|
2283
1283
|
this.publishMessage("context-update", {
|
|
2284
1284
|
...(hasText && { text: options.text.trim() }),
|
|
2285
1285
|
...(options.mode && { mode: options.mode }),
|
|
@@ -2341,8 +1341,6 @@ export class ConvaiClient extends EventEmitter {
|
|
|
2341
1341
|
}
|
|
2342
1342
|
if (!this.isTransportReady())
|
|
2343
1343
|
return;
|
|
2344
|
-
if (this._activeTransport === "sse")
|
|
2345
|
-
return;
|
|
2346
1344
|
this.publishMessage("update-emotion", {
|
|
2347
1345
|
state_of_mind: normalized,
|
|
2348
1346
|
participant_sid: this._participantSid || this._room.localParticipant?.sid,
|
|
@@ -2424,10 +1422,6 @@ export class ConvaiClient extends EventEmitter {
|
|
|
2424
1422
|
* the session from being disconnected due to inactivity.
|
|
2425
1423
|
*/
|
|
2426
1424
|
resetIdleTimer() {
|
|
2427
|
-
// SSE requests are one-turn streams; there is no persistent server-side
|
|
2428
|
-
// realtime session for an idle timer to keep alive.
|
|
2429
|
-
if (this._activeTransport === "sse")
|
|
2430
|
-
return;
|
|
2431
1425
|
if (!this.isTransportReady())
|
|
2432
1426
|
return;
|
|
2433
1427
|
this.publishMessage("reset-idle-timer", {
|
|
@@ -2437,18 +1431,4 @@ export class ConvaiClient extends EventEmitter {
|
|
|
2437
1431
|
}
|
|
2438
1432
|
ConvaiClient.CLIENT_READY_RETRY_MS = 500;
|
|
2439
1433
|
ConvaiClient.CLIENT_READY_TIMEOUT_MS = 45000;
|
|
2440
|
-
ConvaiClient.ACTION_RESULT_ACK_TIMEOUT_MS = 10000;
|
|
2441
|
-
/**
|
|
2442
|
-
* How long to wait for a roster command's ack.
|
|
2443
|
-
*
|
|
2444
|
-
* A timeout is *not* a failure. Every character participant observes the
|
|
2445
|
-
* broadcast and duplicate forwards are response-suppressed after one actor
|
|
2446
|
-
* answers, so a lost ack is an expected outcome. The contract's rule is to
|
|
2447
|
-
* treat it as unknown and reconcile from the next authoritative snapshot,
|
|
2448
|
-
* which is what the rejection below tells the caller to do.
|
|
2449
|
-
*/
|
|
2450
|
-
ConvaiClient.ROSTER_COMMAND_TIMEOUT_MS = 10000;
|
|
2451
|
-
/** Default bounded retry while a roster room is still provisioning. */
|
|
2452
|
-
ConvaiClient.ROSTER_RETRY_ATTEMPTS = 4;
|
|
2453
|
-
ConvaiClient.ROSTER_RETRY_INITIAL_MS = 500;
|
|
2454
1434
|
//# sourceMappingURL=ConvaiClient.js.map
|