@convai/web-sdk 1.8.0-beta.7 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +44 -194
- package/dist/core/ConvaiClient.d.ts +1 -190
- package/dist/core/ConvaiClient.d.ts.map +1 -1
- package/dist/core/ConvaiClient.js +286 -1306
- package/dist/core/ConvaiClient.js.map +1 -1
- package/dist/core/MessageHandler.d.ts +1 -19
- package/dist/core/MessageHandler.d.ts.map +1 -1
- package/dist/core/MessageHandler.js +27 -378
- package/dist/core/MessageHandler.js.map +1 -1
- package/dist/core/connectRequest.d.ts +4 -17
- package/dist/core/connectRequest.d.ts.map +1 -1
- package/dist/core/connectRequest.js +17 -28
- package/dist/core/connectRequest.js.map +1 -1
- package/dist/core/index.d.ts +1 -6
- package/dist/core/index.d.ts.map +1 -1
- package/dist/core/index.js +1 -7
- package/dist/core/index.js.map +1 -1
- package/dist/core/logicalTurn.d.ts +6 -4
- package/dist/core/logicalTurn.d.ts.map +1 -1
- package/dist/core/logicalTurn.js +4 -29
- package/dist/core/logicalTurn.js.map +1 -1
- package/dist/core/modelOutput.d.ts.map +1 -1
- package/dist/core/modelOutput.js +2 -2
- package/dist/core/modelOutput.js.map +1 -1
- package/dist/core/types.d.ts +22 -860
- package/dist/core/types.d.ts.map +1 -1
- package/dist/core/types.js.map +1 -1
- package/dist/embed/chat-embed-v1.js +9 -10
- package/dist/embed/chat-embed-v1.js.map +4 -4
- package/dist/react/components/ConvaiWidget.d.ts.map +1 -1
- package/dist/react/components/ConvaiWidget.js +1 -6
- package/dist/react/components/ConvaiWidget.js.map +1 -1
- package/dist/react/components/rtc-widget/components/AudioVisualizer.d.ts.map +1 -1
- package/dist/react/components/rtc-widget/components/AudioVisualizer.js +1 -28
- package/dist/react/components/rtc-widget/components/AudioVisualizer.js.map +1 -1
- package/dist/react/components/rtc-widget/components/MarkdownRenderer.d.ts.map +1 -1
- package/dist/react/components/rtc-widget/components/MarkdownRenderer.js +44 -39
- package/dist/react/components/rtc-widget/components/MarkdownRenderer.js.map +1 -1
- package/dist/react/components/rtc-widget/components/conviComponents/VoiceModeOverlay.d.ts.map +1 -1
- package/dist/react/components/rtc-widget/components/conviComponents/VoiceModeOverlay.js +49 -93
- package/dist/react/components/rtc-widget/components/conviComponents/VoiceModeOverlay.js.map +1 -1
- package/dist/react/hooks/useConvaiClient.d.ts.map +1 -1
- package/dist/react/hooks/useConvaiClient.js +2 -17
- package/dist/react/hooks/useConvaiClient.js.map +1 -1
- package/dist/react/index.d.ts +1 -1
- package/dist/react/index.d.ts.map +1 -1
- package/dist/react/index.js +1 -1
- package/dist/react/index.js.map +1 -1
- package/dist/vanilla/AudioRenderer.d.ts +1 -3
- package/dist/vanilla/AudioRenderer.d.ts.map +1 -1
- package/dist/vanilla/AudioRenderer.js +2 -8
- package/dist/vanilla/AudioRenderer.js.map +1 -1
- package/dist/vanilla/ConvaiWidget.d.ts.map +1 -1
- package/dist/vanilla/ConvaiWidget.js +300 -994
- package/dist/vanilla/ConvaiWidget.js.map +1 -1
- package/dist/vanilla/WidgetEventSubscriptions.d.ts +1 -1
- package/dist/vanilla/WidgetEventSubscriptions.d.ts.map +1 -1
- package/dist/vanilla/WidgetEventSubscriptions.js +0 -5
- package/dist/vanilla/WidgetEventSubscriptions.js.map +1 -1
- package/dist/vanilla/icons.d.ts.map +1 -1
- package/dist/vanilla/icons.js +15 -62
- package/dist/vanilla/icons.js.map +1 -1
- package/dist/vanilla/index.d.ts +2 -4
- package/dist/vanilla/index.d.ts.map +1 -1
- package/dist/vanilla/index.js +1 -3
- package/dist/vanilla/index.js.map +1 -1
- package/dist/vanilla/styles.d.ts +1 -18
- package/dist/vanilla/styles.d.ts.map +1 -1
- package/dist/vanilla/styles.js +29 -129
- package/dist/vanilla/styles.js.map +1 -1
- package/dist/vanilla/types.d.ts +0 -74
- package/dist/vanilla/types.d.ts.map +1 -1
- package/dist/vanilla/types.js.map +1 -1
- package/dist/version.d.ts +1 -1
- package/dist/version.d.ts.map +1 -1
- package/dist/version.js +1 -1
- package/dist/version.js.map +1 -1
- package/package.json +8 -13
- package/dist/core/CharacterRoster.d.ts +0 -84
- package/dist/core/CharacterRoster.d.ts.map +0 -1
- package/dist/core/CharacterRoster.js +0 -348
- package/dist/core/CharacterRoster.js.map +0 -1
- package/dist/core/CharacterVersionManager.d.ts +0 -100
- package/dist/core/CharacterVersionManager.d.ts.map +0 -1
- package/dist/core/CharacterVersionManager.js +0 -227
- package/dist/core/CharacterVersionManager.js.map +0 -1
- package/dist/core/ConvaiRoomError.d.ts +0 -48
- package/dist/core/ConvaiRoomError.d.ts.map +0 -1
- package/dist/core/ConvaiRoomError.js +0 -111
- package/dist/core/ConvaiRoomError.js.map +0 -1
- package/dist/core/SSESession.d.ts +0 -59
- package/dist/core/SSESession.d.ts.map +0 -1
- package/dist/core/SSESession.js +0 -289
- package/dist/core/SSESession.js.map +0 -1
- package/dist/core/characterReference.d.ts +0 -40
- package/dist/core/characterReference.d.ts.map +0 -1
- package/dist/core/characterReference.js +0 -72
- package/dist/core/characterReference.js.map +0 -1
- package/dist/core/rosterRequest.d.ts +0 -57
- package/dist/core/rosterRequest.d.ts.map +0 -1
- package/dist/core/rosterRequest.js +0 -210
- package/dist/core/rosterRequest.js.map +0 -1
- package/dist/utils/inlineMarkdown.d.ts +0 -38
- package/dist/utils/inlineMarkdown.d.ts.map +0 -1
- package/dist/utils/inlineMarkdown.js +0 -106
- package/dist/utils/inlineMarkdown.js.map +0 -1
|
@@ -9,7 +9,6 @@ import { BlendshapeQueue } from "./BlendshapeQueue.js";
|
|
|
9
9
|
export declare class MessageHandler extends EventEmitter {
|
|
10
10
|
private room;
|
|
11
11
|
private chatMessages;
|
|
12
|
-
private activeBotLlmMessageId;
|
|
13
12
|
private userTranscription;
|
|
14
13
|
private logRtviMessages;
|
|
15
14
|
private isBotResponding;
|
|
@@ -19,8 +18,7 @@ export declare class MessageHandler extends EventEmitter {
|
|
|
19
18
|
private blendshapeQueue;
|
|
20
19
|
private _currentEmotion;
|
|
21
20
|
private modelOutputVersion;
|
|
22
|
-
private
|
|
23
|
-
private seenModelOutputPayloadCharacters;
|
|
21
|
+
private seenModelOutputIds;
|
|
24
22
|
private seenBotOutputSegments;
|
|
25
23
|
private readonly _boundHandleDataReceived;
|
|
26
24
|
constructor(room?: Room);
|
|
@@ -86,8 +84,6 @@ export declare class MessageHandler extends EventEmitter {
|
|
|
86
84
|
private handleDataReceived;
|
|
87
85
|
private _handleActionResponse;
|
|
88
86
|
private _handleModelOutput;
|
|
89
|
-
private _clearModelOutputReplayLedger;
|
|
90
|
-
private _handleBotLlmTextRaw;
|
|
91
87
|
private lifecycleOwner;
|
|
92
88
|
private numberOrUndefined;
|
|
93
89
|
private transcriptionLogicalTurnId;
|
|
@@ -97,20 +93,6 @@ export declare class MessageHandler extends EventEmitter {
|
|
|
97
93
|
handleDataReceivedPublic(payload: Uint8Array): void;
|
|
98
94
|
/** Append a submitted text message without waiting for a server echo. */
|
|
99
95
|
addUserTextMessage(text: string): void;
|
|
100
|
-
/**
|
|
101
|
-
* Insert a message into the transcript locally. Nothing is transmitted and
|
|
102
|
-
* no LLM turn is billed -- this is for client-side lines (a greeting, a
|
|
103
|
-
* system notice, a replayed history entry) that must appear in the chat.
|
|
104
|
-
*
|
|
105
|
-
* Subscribers see it through the same `messagesChange` as a live message,
|
|
106
|
-
* so React and the widgets pick it up with no special casing.
|
|
107
|
-
*/
|
|
108
|
-
appendLocalMessage(message: {
|
|
109
|
-
sender: "user" | "bot";
|
|
110
|
-
content: string;
|
|
111
|
-
membershipId?: string;
|
|
112
|
-
characterId?: string;
|
|
113
|
-
}): ChatMessage;
|
|
114
96
|
/**
|
|
115
97
|
* Reset message state
|
|
116
98
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MessageHandler.d.ts","sourceRoot":"","sources":["../../src/core/MessageHandler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAa,MAAM,gBAAgB,CAAC;AACjD,OAAO,EACL,WAAW,
|
|
1
|
+
{"version":3,"file":"MessageHandler.d.ts","sourceRoot":"","sources":["../../src/core/MessageHandler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAa,MAAM,gBAAgB,CAAC;AACjD,OAAO,EACL,WAAW,EAMZ,MAAM,SAAS,CAAC;AACjB,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAM9C,OAAO,EACL,eAAe,EAEhB,MAAM,mBAAmB,CAAC;AAS3B;;;GAGG;AACH,qBAAa,cAAe,SAAQ,YAAY;IAC9C,OAAO,CAAC,IAAI,CAAqB;IACjC,OAAO,CAAC,YAAY,CAAqB;IACzC,OAAO,CAAC,iBAAiB,CAAc;IACvC,OAAO,CAAC,eAAe,CAAiB;IACxC,OAAO,CAAC,eAAe,CAAkB;IACzC,OAAO,CAAC,UAAU,CAAkB;IACpC,OAAO,CAAC,WAAW,CAAkB;IACrC,OAAO,CAAC,wBAAwB,CAAa;IAC7C,OAAO,CAAC,eAAe,CAAkB;IACzC,OAAO,CAAC,eAAe,CAAoD;IAC3E,OAAO,CAAC,kBAAkB,CAAY;IACtC,OAAO,CAAC,kBAAkB,CAAqB;IAI/C,OAAO,CAAC,qBAAqB,CAA8B;IAK3D,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAG/B;gBAEE,IAAI,CAAC,EAAE,IAAI;IASvB;;OAEG;IACH,kBAAkB,IAAI,eAAe;IAIrC;;OAEG;IACH,qBAAqB,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI;IAI7C,8EAA8E;IAC9E,qBAAqB,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,GAAG,IAAI;IAK3C;;OAEG;IACH,eAAe,IAAI,WAAW,EAAE;IAIhC;;OAEG;IACH,oBAAoB,IAAI,MAAM;IAI9B;;OAEG;IACH,kBAAkB,IAAI,OAAO;IAI7B;;OAEG;IACH,aAAa,IAAI,OAAO;IAIxB;;OAEG;IACH,UAAU,IAAI;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI;IAIxD;;;OAGG;IACH,cAAc,IAAI,OAAO;IAIzB;;;;OAIG;IACH,wBAAwB,IAAI,IAAI;IAShC;;OAEG;IACH,OAAO,CAAC,IAAI,EAAE,IAAI,GAAG,IAAI;IAUzB;;OAEG;IACH,OAAO,CAAC,cAAc;IAMtB;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAMxB;;OAEG;IACH,OAAO,CAAC,kBAAkB;IA6gB1B,OAAO,CAAC,qBAAqB;IAsB7B,OAAO,CAAC,kBAAkB;IAe1B,OAAO,CAAC,cAAc;IAkBtB,OAAO,CAAC,iBAAiB;IAMzB,OAAO,CAAC,0BAA0B;IAMlC;;OAEG;IACH,wBAAwB,CAAC,OAAO,EAAE,UAAU,GAAG,IAAI;IAInD,yEAAyE;IACzE,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAYtC;;OAEG;IACH,KAAK,IAAI,IAAI;CAkBd"}
|
|
@@ -1,24 +1,8 @@
|
|
|
1
1
|
import { RoomEvent } from "livekit-client";
|
|
2
2
|
import { EventEmitter } from "./EventEmitter.js";
|
|
3
3
|
import { extractModelOutputPayload, parseModelOutputMessage, } from "./modelOutput.js";
|
|
4
|
-
import { isValidLogicalTurnId
|
|
4
|
+
import { isValidLogicalTurnId } from "./logicalTurn.js";
|
|
5
5
|
import { BlendshapeQueue, } from "./BlendshapeQueue.js";
|
|
6
|
-
function stableJsonFingerprint(value) {
|
|
7
|
-
if (Array.isArray(value)) {
|
|
8
|
-
return `[${value.map(stableJsonFingerprint).join(",")}]`;
|
|
9
|
-
}
|
|
10
|
-
if (value && typeof value === "object") {
|
|
11
|
-
const entries = Object.entries(value)
|
|
12
|
-
.sort(([left], [right]) => left.localeCompare(right))
|
|
13
|
-
.map(([key, item]) => `${JSON.stringify(key)}:${stableJsonFingerprint(item)}`);
|
|
14
|
-
return `{${entries.join(",")}}`;
|
|
15
|
-
}
|
|
16
|
-
return JSON.stringify(value);
|
|
17
|
-
}
|
|
18
|
-
const MAX_MODEL_OUTPUT_REPLAY_FINGERPRINTS = 400;
|
|
19
|
-
// Preserve exact replay matching instead of accepting hash collisions, while
|
|
20
|
-
// bounding the aggregate strings retained by a long-lived v2 session.
|
|
21
|
-
const MAX_MODEL_OUTPUT_REPLAY_RETAINED_CHARS = 4 * 1024 * 1024;
|
|
22
6
|
/**
|
|
23
7
|
* Handles incoming messages from Convai
|
|
24
8
|
* Processes various message types from the AI assistant and updates chat history
|
|
@@ -28,7 +12,6 @@ export class MessageHandler extends EventEmitter {
|
|
|
28
12
|
super();
|
|
29
13
|
this.room = null;
|
|
30
14
|
this.chatMessages = [];
|
|
31
|
-
this.activeBotLlmMessageId = null;
|
|
32
15
|
this.userTranscription = "";
|
|
33
16
|
this.logRtviMessages = true;
|
|
34
17
|
this.isBotResponding = false;
|
|
@@ -37,8 +20,7 @@ export class MessageHandler extends EventEmitter {
|
|
|
37
20
|
this.totalBlendshapesReceived = 0;
|
|
38
21
|
this._currentEmotion = null;
|
|
39
22
|
this.modelOutputVersion = 1;
|
|
40
|
-
this.
|
|
41
|
-
this.seenModelOutputPayloadCharacters = 0;
|
|
23
|
+
this.seenModelOutputIds = new Set();
|
|
42
24
|
// bot-output segments already added to the chat. The server re-sends the
|
|
43
25
|
// same segment (full text) as TTS progress advances (spoken_status
|
|
44
26
|
// in-progress → completed); only the first sighting may touch the chat.
|
|
@@ -64,7 +46,7 @@ export class MessageHandler extends EventEmitter {
|
|
|
64
46
|
/** Select the server-negotiated canonical output version for this session. */
|
|
65
47
|
setModelOutputVersion(version) {
|
|
66
48
|
this.modelOutputVersion = version;
|
|
67
|
-
this.
|
|
49
|
+
this.seenModelOutputIds.clear();
|
|
68
50
|
}
|
|
69
51
|
/**
|
|
70
52
|
* Get current chat messages
|
|
@@ -166,23 +148,6 @@ export class MessageHandler extends EventEmitter {
|
|
|
166
148
|
// Extract and categorize messages for chat display
|
|
167
149
|
const timestamp = new Date().toISOString();
|
|
168
150
|
const messageId = `${messageData.type}-${Date.now()}-${Math.random()}`;
|
|
169
|
-
// Multi-character routing identity, carried inline on every
|
|
170
|
-
// character-originated event. Undefined in single-character sessions, so
|
|
171
|
-
// these spread to nothing and every existing message keeps its shape.
|
|
172
|
-
const attribution = {
|
|
173
|
-
...(messageData.data?.membership_id && {
|
|
174
|
-
membershipId: messageData.data.membership_id,
|
|
175
|
-
}),
|
|
176
|
-
...(messageData.data?.character_id && {
|
|
177
|
-
characterId: messageData.data.character_id,
|
|
178
|
-
}),
|
|
179
|
-
};
|
|
180
|
-
/**
|
|
181
|
-
* Two instances must never share a streaming bubble. Without this a
|
|
182
|
-
* target switch mid-turn splices one character's text onto another's,
|
|
183
|
-
* because the merge would only be checking `isStreaming`.
|
|
184
|
-
*/
|
|
185
|
-
const sameSpeaker = (msg) => !!msg && msg.membershipId === attribution.membershipId;
|
|
186
151
|
// Handle different message types
|
|
187
152
|
switch (messageData.type) {
|
|
188
153
|
// Submitted text is appended locally by sendUserTextMessage. Ignore
|
|
@@ -199,40 +164,21 @@ export class MessageHandler extends EventEmitter {
|
|
|
199
164
|
break;
|
|
200
165
|
// Bot Ready - Bot is ready, connection should be true
|
|
201
166
|
case "bot-ready":
|
|
202
|
-
|
|
203
|
-
// and the same fields are also flattened onto `data`. Prefer `about`,
|
|
204
|
-
// fall back to the flattened copy, and pass undefined for a
|
|
205
|
-
// single-character session.
|
|
206
|
-
this.emit("botReady", messageData.data?.about ??
|
|
207
|
-
(messageData.data?.membership_id ? messageData.data : undefined));
|
|
167
|
+
this.emit("botReady");
|
|
208
168
|
break;
|
|
209
169
|
// Bot LLM Started - Begin streaming response
|
|
210
170
|
case "bot-llm-started":
|
|
211
171
|
this.isBotResponding = true;
|
|
212
172
|
this.emit("botRespondingChange", true);
|
|
213
173
|
this.seenBotOutputSegments.clear();
|
|
214
|
-
//
|
|
215
|
-
// unowned placeholder so concurrent starts cannot create anonymous
|
|
216
|
-
// rows that later get adopted in reverse order. Invocation-correlated
|
|
217
|
-
// chunks establish additional rows in authoritative arrival order.
|
|
218
|
-
const hasUnownedV2Starter = this.modelOutputVersion === 2 &&
|
|
219
|
-
this.chatMessages.some((message) => message.type === "bot-llm-text" &&
|
|
220
|
-
message.isStreaming &&
|
|
221
|
-
message.content === "" &&
|
|
222
|
-
message.outputId === undefined &&
|
|
223
|
-
message.logicalTurnId === undefined);
|
|
224
|
-
if (hasUnownedV2Starter)
|
|
225
|
-
break;
|
|
226
|
-
// Add initial streaming message to chat.
|
|
174
|
+
// Add initial streaming message to chat
|
|
227
175
|
const streamingMessage = {
|
|
228
176
|
id: messageId,
|
|
229
177
|
type: "bot-llm-text",
|
|
230
178
|
content: "",
|
|
231
179
|
timestamp: timestamp,
|
|
232
180
|
isStreaming: true,
|
|
233
|
-
...attribution,
|
|
234
181
|
};
|
|
235
|
-
this.activeBotLlmMessageId = messageId;
|
|
236
182
|
this.chatMessages.push(streamingMessage);
|
|
237
183
|
this.emit("messagesChange", this.chatMessages);
|
|
238
184
|
break;
|
|
@@ -259,24 +205,17 @@ export class MessageHandler extends EventEmitter {
|
|
|
259
205
|
break;
|
|
260
206
|
this.seenBotOutputSegments.add(segmentId);
|
|
261
207
|
}
|
|
262
|
-
const activeBotLlmMessage = this.activeBotLlmMessageId
|
|
263
|
-
? this.chatMessages.find((message) => message.id === this.activeBotLlmMessageId)
|
|
264
|
-
: undefined;
|
|
265
208
|
const lastMsg = this.chatMessages[this.chatMessages.length - 1];
|
|
266
209
|
// When the reply is rendered by the LLM text stream
|
|
267
210
|
// (bot-llm-text), the TTS echo adds nothing — never mix
|
|
268
211
|
// bot-output into an LLM bubble (streaming or complete).
|
|
269
|
-
if (activeBotLlmMessage?.type === "bot-llm-text")
|
|
270
|
-
break;
|
|
271
212
|
if (lastMsg?.type === "bot-llm-text")
|
|
272
213
|
break;
|
|
273
214
|
if (lastMsg?.type === "bot-output" &&
|
|
274
215
|
lastMsg.content.includes(newText)) {
|
|
275
216
|
break;
|
|
276
217
|
}
|
|
277
|
-
if (lastMsg?.type === "bot-output" &&
|
|
278
|
-
lastMsg.isStreaming &&
|
|
279
|
-
sameSpeaker(lastMsg)) {
|
|
218
|
+
if (lastMsg?.type === "bot-output" && lastMsg.isStreaming) {
|
|
280
219
|
this.chatMessages[this.chatMessages.length - 1] = {
|
|
281
220
|
...lastMsg,
|
|
282
221
|
content: lastMsg.content + newText,
|
|
@@ -289,7 +228,6 @@ export class MessageHandler extends EventEmitter {
|
|
|
289
228
|
content: newText,
|
|
290
229
|
timestamp: new Date().toISOString(),
|
|
291
230
|
isStreaming: true,
|
|
292
|
-
...attribution,
|
|
293
231
|
});
|
|
294
232
|
}
|
|
295
233
|
this.emit("messagesChange", this.chatMessages);
|
|
@@ -299,206 +237,41 @@ export class MessageHandler extends EventEmitter {
|
|
|
299
237
|
case "bot-llm-text":
|
|
300
238
|
if (messageData.data?.text) {
|
|
301
239
|
const newChunk = messageData.data.text;
|
|
302
|
-
//
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
const logicalTurnId = this.modelOutputVersion === 2 &&
|
|
309
|
-
isValidLogicalTurnId(messageData.data.logical_turn_id)
|
|
310
|
-
? messageData.data.logical_turn_id
|
|
311
|
-
: undefined;
|
|
312
|
-
// A logical turn can contain multiple provider invocations (for
|
|
313
|
-
// example, the response before a tool call and the continuation
|
|
314
|
-
// after its result). Prefer the invocation id, then fall back to
|
|
315
|
-
// the older v2 logical-turn id when Core does not send one.
|
|
316
|
-
let targetMessageIndex = -1;
|
|
317
|
-
if (outputId !== undefined) {
|
|
318
|
-
for (let index = this.chatMessages.length - 1; index >= 0; index--) {
|
|
319
|
-
const message = this.chatMessages[index];
|
|
320
|
-
if (message.type === "bot-llm-text" &&
|
|
321
|
-
message.isStreaming &&
|
|
322
|
-
message.outputId === outputId &&
|
|
323
|
-
sameSpeaker(message)) {
|
|
324
|
-
targetMessageIndex = index;
|
|
325
|
-
break;
|
|
326
|
-
}
|
|
327
|
-
}
|
|
328
|
-
// bot-llm-started has no correlation metadata. Let one empty,
|
|
329
|
-
// unowned row adopt the first authoritative invocation id.
|
|
330
|
-
if (targetMessageIndex === -1) {
|
|
331
|
-
for (let index = this.chatMessages.length - 1; index >= 0; index--) {
|
|
332
|
-
const message = this.chatMessages[index];
|
|
333
|
-
if (message.type === "bot-llm-text" &&
|
|
334
|
-
message.isStreaming &&
|
|
335
|
-
message.content === "" &&
|
|
336
|
-
message.outputId === undefined &&
|
|
337
|
-
message.logicalTurnId === undefined &&
|
|
338
|
-
sameSpeaker(message)) {
|
|
339
|
-
targetMessageIndex = index;
|
|
340
|
-
break;
|
|
341
|
-
}
|
|
342
|
-
}
|
|
343
|
-
}
|
|
344
|
-
}
|
|
345
|
-
else if (logicalTurnId !== undefined) {
|
|
346
|
-
for (let index = this.chatMessages.length - 1; index >= 0; index--) {
|
|
347
|
-
const message = this.chatMessages[index];
|
|
348
|
-
if (message.type === "bot-llm-text" &&
|
|
349
|
-
message.isStreaming &&
|
|
350
|
-
message.outputId === undefined &&
|
|
351
|
-
message.logicalTurnId === logicalTurnId &&
|
|
352
|
-
sameSpeaker(message)) {
|
|
353
|
-
targetMessageIndex = index;
|
|
354
|
-
break;
|
|
355
|
-
}
|
|
356
|
-
}
|
|
357
|
-
// Older v2 Core still needs the starter-row behavior that was
|
|
358
|
-
// introduced with logical-turn correlation.
|
|
359
|
-
if (targetMessageIndex === -1) {
|
|
360
|
-
const lastIndex = this.chatMessages.length - 1;
|
|
361
|
-
const lastMessage = this.chatMessages[lastIndex];
|
|
362
|
-
if (lastMessage?.type === "bot-llm-text" &&
|
|
363
|
-
lastMessage.isStreaming &&
|
|
364
|
-
lastMessage.content === "" &&
|
|
365
|
-
lastMessage.outputId === undefined &&
|
|
366
|
-
lastMessage.logicalTurnId === undefined &&
|
|
367
|
-
sameSpeaker(lastMessage)) {
|
|
368
|
-
targetMessageIndex = lastIndex;
|
|
369
|
-
}
|
|
370
|
-
}
|
|
371
|
-
}
|
|
372
|
-
else {
|
|
373
|
-
// Uncorrelated (v1) chunk: it belongs to the bubble bot-llm-started
|
|
374
|
-
// opened for this speaker. Only when no such bubble exists does the
|
|
375
|
-
// historical "last streaming row" rule apply -- and still only for
|
|
376
|
-
// the same speaker, so a target switch mid-turn cannot splice one
|
|
377
|
-
// member's text onto another's row.
|
|
378
|
-
// Ownership applies among uncorrelated rows only: an uncorrelated
|
|
379
|
-
// chunk must never join a row that carries an output or turn id,
|
|
380
|
-
// or a v2 correlated stream would absorb legacy text.
|
|
381
|
-
if (this.activeBotLlmMessageId) {
|
|
382
|
-
targetMessageIndex = this.chatMessages.findIndex((message) => message.id === this.activeBotLlmMessageId &&
|
|
383
|
-
message.type === "bot-llm-text" &&
|
|
384
|
-
message.isStreaming &&
|
|
385
|
-
!message.outputId &&
|
|
386
|
-
!message.logicalTurnId &&
|
|
387
|
-
sameSpeaker(message));
|
|
388
|
-
}
|
|
389
|
-
if (targetMessageIndex === -1) {
|
|
390
|
-
const lastIndex = this.chatMessages.length - 1;
|
|
391
|
-
const lastMessage = this.chatMessages[lastIndex];
|
|
392
|
-
if (lastMessage &&
|
|
393
|
-
lastMessage.type === "bot-llm-text" &&
|
|
394
|
-
lastMessage.isStreaming &&
|
|
395
|
-
!lastMessage.outputId &&
|
|
396
|
-
!lastMessage.logicalTurnId &&
|
|
397
|
-
sameSpeaker(lastMessage)) {
|
|
398
|
-
targetMessageIndex = lastIndex;
|
|
399
|
-
}
|
|
400
|
-
}
|
|
401
|
-
}
|
|
402
|
-
let updatedMessage;
|
|
403
|
-
if (targetMessageIndex !== -1) {
|
|
404
|
-
const targetMessage = this.chatMessages[targetMessageIndex];
|
|
405
|
-
updatedMessage = {
|
|
406
|
-
...targetMessage,
|
|
407
|
-
content: targetMessage.content + newChunk,
|
|
408
|
-
...(outputId !== undefined &&
|
|
409
|
-
targetMessage.outputId === undefined
|
|
410
|
-
? { outputId }
|
|
411
|
-
: {}),
|
|
412
|
-
...(logicalTurnId !== undefined &&
|
|
413
|
-
targetMessage.logicalTurnId === undefined
|
|
414
|
-
? { logicalTurnId }
|
|
415
|
-
: {}),
|
|
240
|
+
// Update the streaming message in chat
|
|
241
|
+
const lastMessage = this.chatMessages[this.chatMessages.length - 1];
|
|
242
|
+
if (lastMessage && lastMessage.isStreaming) {
|
|
243
|
+
this.chatMessages[this.chatMessages.length - 1] = {
|
|
244
|
+
...lastMessage,
|
|
245
|
+
content: lastMessage.content + newChunk,
|
|
416
246
|
};
|
|
417
|
-
this.chatMessages[targetMessageIndex] = updatedMessage;
|
|
418
247
|
}
|
|
419
248
|
else {
|
|
420
249
|
// If no streaming message exists yet, create one
|
|
421
|
-
|
|
250
|
+
const streamingMsg = {
|
|
422
251
|
id: `${messageData.type}-${Date.now()}-${Math.random()}`,
|
|
423
252
|
type: "bot-llm-text",
|
|
424
253
|
content: newChunk,
|
|
425
254
|
timestamp: new Date().toISOString(),
|
|
426
255
|
isStreaming: true,
|
|
427
|
-
...(outputId ? { outputId } : {}),
|
|
428
|
-
...(logicalTurnId ? { logicalTurnId } : {}),
|
|
429
|
-
...attribution,
|
|
430
256
|
};
|
|
431
|
-
this.
|
|
432
|
-
this.chatMessages.push(updatedMessage);
|
|
257
|
+
this.chatMessages.push(streamingMsg);
|
|
433
258
|
}
|
|
434
|
-
this.emit("messagesChange", this.chatMessages
|
|
259
|
+
this.emit("messagesChange", this.chatMessages);
|
|
435
260
|
}
|
|
436
261
|
break;
|
|
437
|
-
// Parallel diagnostics-only provider output. This must never affect
|
|
438
|
-
// normal chat, speech, TTS, or response lifecycle state.
|
|
439
|
-
case "bot-llm-text-raw":
|
|
440
|
-
this._handleBotLlmTextRaw(messageData.data);
|
|
441
|
-
break;
|
|
442
262
|
// Bot LLM Stopped - Complete streaming response
|
|
443
|
-
case "bot-llm-stopped":
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
for (let index = 0; index < this.chatMessages.length; index++) {
|
|
449
|
-
const message = this.chatMessages[index];
|
|
450
|
-
if ((message.type === "bot-llm-text" ||
|
|
451
|
-
message.type === "bot-output") &&
|
|
452
|
-
message.isStreaming) {
|
|
453
|
-
const finalizedMessage = {
|
|
454
|
-
...message,
|
|
455
|
-
isStreaming: false,
|
|
456
|
-
};
|
|
457
|
-
this.chatMessages[index] = finalizedMessage;
|
|
458
|
-
finalizedMessages.push(finalizedMessage);
|
|
459
|
-
}
|
|
460
|
-
}
|
|
461
|
-
}
|
|
462
|
-
else {
|
|
463
|
-
// v1: a stop frame finalizes the bubble this turn owns; only when no
|
|
464
|
-
// bubble was opened does the historical last-row rule apply.
|
|
465
|
-
let stopIndex = this.activeBotLlmMessageId
|
|
466
|
-
? this.chatMessages.findIndex((message) => message.id === this.activeBotLlmMessageId)
|
|
467
|
-
: -1;
|
|
468
|
-
if (stopIndex === -1)
|
|
469
|
-
stopIndex = this.chatMessages.length - 1;
|
|
470
|
-
const lastBotMsg = this.chatMessages[stopIndex];
|
|
471
|
-
if (lastBotMsg && lastBotMsg.isStreaming) {
|
|
472
|
-
const finalizedMessage = { ...lastBotMsg, isStreaming: false };
|
|
473
|
-
this.chatMessages[stopIndex] = finalizedMessage;
|
|
474
|
-
finalizedMessages.push(finalizedMessage);
|
|
475
|
-
}
|
|
476
|
-
}
|
|
477
|
-
// Clear streaming state
|
|
478
|
-
this.isBotResponding = false;
|
|
479
|
-
this.emit("botRespondingChange", false);
|
|
480
|
-
// Keep the full-list event at one emission per stop while carrying
|
|
481
|
-
// every v2 terminal delta for singular-message consumers.
|
|
482
|
-
this.emit("messagesChange", this.chatMessages, undefined, finalizedMessages);
|
|
483
|
-
break;
|
|
484
|
-
}
|
|
485
|
-
// Text-only Interactions API terminal marker. This is deliberately a
|
|
486
|
-
// separate event: a completed REST turn must not be represented as a
|
|
487
|
-
// synthetic bot-llm-stopped message.
|
|
488
|
-
case "bot-turn-completed":
|
|
489
|
-
const completedIndex = this.activeBotLlmMessageId
|
|
490
|
-
? this.chatMessages.findIndex((message) => message.id === this.activeBotLlmMessageId)
|
|
491
|
-
: -1;
|
|
492
|
-
if (completedIndex >= 0 && this.chatMessages[completedIndex].isStreaming) {
|
|
493
|
-
this.chatMessages[completedIndex] = {
|
|
494
|
-
...this.chatMessages[completedIndex],
|
|
263
|
+
case "bot-llm-stopped":
|
|
264
|
+
const lastBotMsg = this.chatMessages[this.chatMessages.length - 1];
|
|
265
|
+
if (lastBotMsg && lastBotMsg.isStreaming) {
|
|
266
|
+
this.chatMessages[this.chatMessages.length - 1] = {
|
|
267
|
+
...lastBotMsg,
|
|
495
268
|
isStreaming: false,
|
|
496
269
|
};
|
|
497
|
-
this.emit("messagesChange", this.chatMessages);
|
|
498
270
|
}
|
|
271
|
+
// Clear streaming state
|
|
499
272
|
this.isBotResponding = false;
|
|
500
273
|
this.emit("botRespondingChange", false);
|
|
501
|
-
this.emit("
|
|
274
|
+
this.emit("messagesChange", this.chatMessages);
|
|
502
275
|
break;
|
|
503
276
|
// User Transcription Messages
|
|
504
277
|
case "user-transcription":
|
|
@@ -508,14 +281,6 @@ export class MessageHandler extends EventEmitter {
|
|
|
508
281
|
const logicalTurnId = this.transcriptionLogicalTurnId(messageData.data.logical_turn_id);
|
|
509
282
|
// Final transcription - accumulate in single bubble until bot responds
|
|
510
283
|
const lastUserMsg = this.chatMessages[this.chatMessages.length - 1];
|
|
511
|
-
// A final transcription identical to the line appended locally on send is
|
|
512
|
-
// the server echoing that send back: drop it, or the user sees it twice.
|
|
513
|
-
if (lastUserMsg?.type === "user-llm-text" &&
|
|
514
|
-
lastUserMsg.content.trim() === String(messageContent).trim()) {
|
|
515
|
-
this.userTranscription = "";
|
|
516
|
-
this.emit("userTranscriptionChange", "");
|
|
517
|
-
break;
|
|
518
|
-
}
|
|
519
284
|
// Check if last message is an accumulating user message (isStreaming=true)
|
|
520
285
|
if (lastUserMsg &&
|
|
521
286
|
lastUserMsg.type === "user-transcription" &&
|
|
@@ -560,8 +325,7 @@ export class MessageHandler extends EventEmitter {
|
|
|
560
325
|
emotion: messageData.data.emotion,
|
|
561
326
|
scale: messageData.data.scale,
|
|
562
327
|
};
|
|
563
|
-
|
|
564
|
-
console.log("[SDK bot-emotion]", messageData.data);
|
|
328
|
+
console.log("[SDK bot-emotion]", messageData.data);
|
|
565
329
|
this.emit("emotionChange", this._currentEmotion);
|
|
566
330
|
const emotionMessage = {
|
|
567
331
|
id: messageId,
|
|
@@ -635,14 +399,6 @@ export class MessageHandler extends EventEmitter {
|
|
|
635
399
|
this.emit("metrics", messageData.data);
|
|
636
400
|
}
|
|
637
401
|
break;
|
|
638
|
-
// Per-turn usage/cost breakdown. Informational only: the server streams
|
|
639
|
-
// it once the client opts in with usage-toggle, and it never affects
|
|
640
|
-
// server-side usage tracking or billing.
|
|
641
|
-
case "usage-update":
|
|
642
|
-
if (messageData.data) {
|
|
643
|
-
this.emit("usageUpdate", messageData.data);
|
|
644
|
-
}
|
|
645
|
-
break;
|
|
646
402
|
// Behavior Tree Response Messages
|
|
647
403
|
case "behavior-tree-response":
|
|
648
404
|
if (messageData.data?.narrative_section_id) {
|
|
@@ -661,22 +417,11 @@ export class MessageHandler extends EventEmitter {
|
|
|
661
417
|
if (messageData.data) {
|
|
662
418
|
const serverData = messageData.data;
|
|
663
419
|
switch (serverData.type) {
|
|
664
|
-
case "bot-llm-text-raw":
|
|
665
|
-
this._handleBotLlmTextRaw(serverData.data ?? serverData);
|
|
666
|
-
break;
|
|
667
420
|
case "final-user-transcription":
|
|
668
421
|
if (serverData.text) {
|
|
669
422
|
const finalText = serverData.text;
|
|
670
423
|
const logicalTurnId = this.transcriptionLogicalTurnId(serverData.logical_turn_id);
|
|
671
424
|
const lastUserMsg = this.chatMessages[this.chatMessages.length - 1];
|
|
672
|
-
// A final transcription identical to the line appended locally on send is
|
|
673
|
-
// the server echoing that send back: drop it, or the user sees it twice.
|
|
674
|
-
if (lastUserMsg?.type === "user-llm-text" &&
|
|
675
|
-
lastUserMsg.content.trim() === String(finalText).trim()) {
|
|
676
|
-
this.userTranscription = "";
|
|
677
|
-
this.emit("userTranscriptionChange", "");
|
|
678
|
-
break;
|
|
679
|
-
}
|
|
680
425
|
if (lastUserMsg &&
|
|
681
426
|
lastUserMsg.type === "user-transcription" &&
|
|
682
427
|
lastUserMsg.isStreaming === true) {
|
|
@@ -714,8 +459,7 @@ export class MessageHandler extends EventEmitter {
|
|
|
714
459
|
emotion: serverData.emotion,
|
|
715
460
|
scale: serverData.scale,
|
|
716
461
|
};
|
|
717
|
-
|
|
718
|
-
console.log("[SDK bot-emotion (ws)]", serverData);
|
|
462
|
+
console.log("[SDK bot-emotion (ws)]", serverData);
|
|
719
463
|
this.emit("emotionChange", this._currentEmotion);
|
|
720
464
|
}
|
|
721
465
|
break;
|
|
@@ -802,35 +546,6 @@ export class MessageHandler extends EventEmitter {
|
|
|
802
546
|
this.emit("serverResponse", serverResponse);
|
|
803
547
|
break;
|
|
804
548
|
}
|
|
805
|
-
/**
|
|
806
|
-
* Lifecycle of one roster member. Emitted when a member is added
|
|
807
|
-
* at runtime, becomes ready, or fails after /connect already
|
|
808
|
-
* returned 200. `bot-ready` stays the readiness authority --
|
|
809
|
-
* a `character-status` of "ready" must not enable interaction on
|
|
810
|
-
* its own -- but this is the only way a background dispatch
|
|
811
|
-
* failure is ever reported.
|
|
812
|
-
*/
|
|
813
|
-
case "character-status": {
|
|
814
|
-
const status = {
|
|
815
|
-
status: serverData.status ?? "starting",
|
|
816
|
-
membershipId: serverData.membership_id ?? "",
|
|
817
|
-
characterId: serverData.character_id ?? "",
|
|
818
|
-
characterSessionId: serverData.character_session_id ?? null,
|
|
819
|
-
participantIdentity: serverData.participant_identity ??
|
|
820
|
-
(serverData.membership_id
|
|
821
|
-
? `character:${serverData.membership_id}`
|
|
822
|
-
: ""),
|
|
823
|
-
roomSessionId: serverData.room_session_id ?? null,
|
|
824
|
-
rosterEpoch: typeof serverData.roster_epoch === "number"
|
|
825
|
-
? serverData.roster_epoch
|
|
826
|
-
: null,
|
|
827
|
-
failureCode: serverData.failure_code ?? null,
|
|
828
|
-
};
|
|
829
|
-
// Unattributable status is dropped rather than guessed at.
|
|
830
|
-
if (status.membershipId)
|
|
831
|
-
this.emit("characterStatus", status);
|
|
832
|
-
break;
|
|
833
|
-
}
|
|
834
549
|
// interaction-created: session IDs for analytics/tracking
|
|
835
550
|
case "interaction-created": {
|
|
836
551
|
const interactionCreated = {
|
|
@@ -898,55 +613,11 @@ export class MessageHandler extends EventEmitter {
|
|
|
898
613
|
});
|
|
899
614
|
return;
|
|
900
615
|
}
|
|
901
|
-
|
|
902
|
-
// One invocation can produce distinct text and tool-call projections with
|
|
903
|
-
// the same id, and consumers must receive both in transport arrival order.
|
|
904
|
-
// Suppress only an exact transport replay of an already delivered payload.
|
|
905
|
-
const fingerprint = stableJsonFingerprint(output);
|
|
906
|
-
if (this.seenModelOutputPayloads.has(fingerprint))
|
|
907
|
-
return;
|
|
908
|
-
if (fingerprint.length > MAX_MODEL_OUTPUT_REPLAY_RETAINED_CHARS) {
|
|
909
|
-
this.emit("modelOutputProtocolError", {
|
|
910
|
-
code: "invalid_model_output",
|
|
911
|
-
message: "Core sent an oversized model-output envelope",
|
|
912
|
-
});
|
|
616
|
+
if (this.seenModelOutputIds.has(output.output_id))
|
|
913
617
|
return;
|
|
914
|
-
|
|
915
|
-
this.seenModelOutputPayloads.add(fingerprint);
|
|
916
|
-
this.seenModelOutputPayloadCharacters += fingerprint.length;
|
|
917
|
-
while (this.seenModelOutputPayloads.size > MAX_MODEL_OUTPUT_REPLAY_FINGERPRINTS ||
|
|
918
|
-
this.seenModelOutputPayloadCharacters >
|
|
919
|
-
MAX_MODEL_OUTPUT_REPLAY_RETAINED_CHARS) {
|
|
920
|
-
const oldestFingerprint = this.seenModelOutputPayloads.values().next()
|
|
921
|
-
.value;
|
|
922
|
-
if (!oldestFingerprint)
|
|
923
|
-
break;
|
|
924
|
-
this.seenModelOutputPayloads.delete(oldestFingerprint);
|
|
925
|
-
this.seenModelOutputPayloadCharacters -= oldestFingerprint.length;
|
|
926
|
-
}
|
|
618
|
+
this.seenModelOutputIds.add(output.output_id);
|
|
927
619
|
this.emit("modelOutput", output);
|
|
928
620
|
}
|
|
929
|
-
_clearModelOutputReplayLedger() {
|
|
930
|
-
this.seenModelOutputPayloads.clear();
|
|
931
|
-
this.seenModelOutputPayloadCharacters = 0;
|
|
932
|
-
}
|
|
933
|
-
_handleBotLlmTextRaw(value) {
|
|
934
|
-
if (this.modelOutputVersion !== 2 || !value || typeof value !== "object") {
|
|
935
|
-
return;
|
|
936
|
-
}
|
|
937
|
-
const data = value;
|
|
938
|
-
if (typeof data.text !== "string" ||
|
|
939
|
-
!isValidLogicalTurnId(data.logical_turn_id)) {
|
|
940
|
-
return;
|
|
941
|
-
}
|
|
942
|
-
this.emit("botLlmTextRaw", {
|
|
943
|
-
text: data.text,
|
|
944
|
-
logical_turn_id: data.logical_turn_id,
|
|
945
|
-
...(isValidOutputId(data.output_id)
|
|
946
|
-
? { output_id: data.output_id }
|
|
947
|
-
: {}),
|
|
948
|
-
});
|
|
949
|
-
}
|
|
950
621
|
lifecycleOwner(data) {
|
|
951
622
|
if (!data) {
|
|
952
623
|
return {};
|
|
@@ -988,41 +659,19 @@ export class MessageHandler extends EventEmitter {
|
|
|
988
659
|
});
|
|
989
660
|
this.emit("messagesChange", this.chatMessages);
|
|
990
661
|
}
|
|
991
|
-
/**
|
|
992
|
-
* Insert a message into the transcript locally. Nothing is transmitted and
|
|
993
|
-
* no LLM turn is billed -- this is for client-side lines (a greeting, a
|
|
994
|
-
* system notice, a replayed history entry) that must appear in the chat.
|
|
995
|
-
*
|
|
996
|
-
* Subscribers see it through the same `messagesChange` as a live message,
|
|
997
|
-
* so React and the widgets pick it up with no special casing.
|
|
998
|
-
*/
|
|
999
|
-
appendLocalMessage(message) {
|
|
1000
|
-
const chatMessage = {
|
|
1001
|
-
id: `local-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`,
|
|
1002
|
-
type: message.sender === "user" ? "user-llm-text" : "bot-llm-text",
|
|
1003
|
-
content: message.content,
|
|
1004
|
-
timestamp: new Date().toISOString(),
|
|
1005
|
-
...(message.membershipId && { membershipId: message.membershipId }),
|
|
1006
|
-
...(message.characterId && { characterId: message.characterId }),
|
|
1007
|
-
};
|
|
1008
|
-
this.chatMessages.push(chatMessage);
|
|
1009
|
-
this.emit("messagesChange", this.chatMessages);
|
|
1010
|
-
return chatMessage;
|
|
1011
|
-
}
|
|
1012
662
|
/**
|
|
1013
663
|
* Reset message state
|
|
1014
664
|
*/
|
|
1015
665
|
reset() {
|
|
1016
666
|
this.chatMessages = [];
|
|
1017
|
-
this.activeBotLlmMessageId = null;
|
|
1018
667
|
this.userTranscription = "";
|
|
1019
668
|
this.isBotResponding = false;
|
|
1020
669
|
this.isSpeaking = false;
|
|
1021
670
|
this.isListening = false;
|
|
1022
671
|
this.totalBlendshapesReceived = 0;
|
|
1023
672
|
this._currentEmotion = null;
|
|
673
|
+
this.seenModelOutputIds.clear();
|
|
1024
674
|
this.seenBotOutputSegments.clear();
|
|
1025
|
-
this._clearModelOutputReplayLedger();
|
|
1026
675
|
this.blendshapeQueue.reset();
|
|
1027
676
|
this.emit("messagesChange", this.chatMessages);
|
|
1028
677
|
this.emit("userTranscriptionChange", "");
|