@charivo/realtime 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +148 -0
- package/dist/chunk-BNN3F7VN.mjs +433 -0
- package/dist/chunk-BNN3F7VN.mjs.map +1 -0
- package/dist/chunk-DTNZCAFN.mjs +522 -0
- package/dist/chunk-DTNZCAFN.mjs.map +1 -0
- package/dist/chunk-XJNT4ZTF.mjs +41 -0
- package/dist/chunk-XJNT4ZTF.mjs.map +1 -0
- package/dist/index.d.mts +88 -0
- package/dist/index.d.ts +88 -0
- package/dist/index.js +874 -0
- package/dist/index.js.map +1 -0
- package/dist/index.mjs +864 -0
- package/dist/index.mjs.map +1 -0
- package/dist/openai/index.d.mts +54 -0
- package/dist/openai/index.d.ts +54 -0
- package/dist/openai/index.js +476 -0
- package/dist/openai/index.js.map +1 -0
- package/dist/openai/index.mjs +4 -0
- package/dist/openai/index.mjs.map +1 -0
- package/dist/openai-agents/index.d.mts +49 -0
- package/dist/openai-agents/index.d.ts +49 -0
- package/dist/openai-agents/index.js +564 -0
- package/dist/openai-agents/index.js.map +1 -0
- package/dist/openai-agents/index.mjs +4 -0
- package/dist/openai-agents/index.mjs.map +1 -0
- package/dist/remote/index.d.mts +45 -0
- package/dist/remote/index.d.ts +45 -0
- package/dist/remote/index.js +1115 -0
- package/dist/remote/index.js.map +1 -0
- package/dist/remote/index.mjs +131 -0
- package/dist/remote/index.mjs.map +1 -0
- package/dist/types-DntfJo0o.d.mts +54 -0
- package/dist/types-DntfJo0o.d.ts +54 -0
- package/package.json +64 -0
package/dist/index.mjs
ADDED
|
@@ -0,0 +1,864 @@
|
|
|
1
|
+
// src/instructions.ts
|
|
2
|
+
var DEFAULT_PROVIDER = "openai";
|
|
3
|
+
var DEFAULT_MODEL = "gpt-realtime-mini";
|
|
4
|
+
var DEFAULT_VOICE = "marin";
|
|
5
|
+
var SET_EXPRESSION_TOOL_DESCRIPTION = "Change the avatar's facial expression when the emotional beat clearly shifts. Usually keep the same expression for a few turns.";
|
|
6
|
+
var PLAY_MOTION_TOOL_DESCRIPTION = "Play a noticeable body motion for greetings, emphasis, or bigger reaction beats. Usually use at most one motion in a reply.";
|
|
7
|
+
var LOOK_AT_TOOL_DESCRIPTION = "Shift the avatar's gaze for subtle attention changes or conversational focus. Prefer this when a lightweight reaction is enough.";
|
|
8
|
+
var DEFAULT_REALTIME_AGENT_INSTRUCTIONS = `
|
|
9
|
+
You are speaking in a realtime voice conversation through a Live2D avatar.
|
|
10
|
+
Respond naturally and keep replies concise enough for spoken delivery.
|
|
11
|
+
Use avatar tools only when they add something meaningful to the moment.
|
|
12
|
+
Use "setExpression" only when the emotional beat clearly shifts, usually not more than once in a few turns.
|
|
13
|
+
Use "playMotion" for greetings, emphasis, or bigger reaction beats, usually at most once in a reply.
|
|
14
|
+
Use "lookAt" for subtle attention shifts or conversational focus, and prefer it when a lightweight reaction is enough.
|
|
15
|
+
Prefer a single meaningful avatar action over chaining multiple actions on simple replies.
|
|
16
|
+
Many turns should use no avatar tool at all.
|
|
17
|
+
Never break character. Never refer to yourself as an AI, model, or assistant.
|
|
18
|
+
Do not mention tool calls in the spoken response.
|
|
19
|
+
Do not narrate your facial expressions, motions, or gaze. Let the avatar show them.
|
|
20
|
+
`.trim();
|
|
21
|
+
function buildRealtimeSessionConfig({
|
|
22
|
+
character,
|
|
23
|
+
baseConfig
|
|
24
|
+
} = {}) {
|
|
25
|
+
return {
|
|
26
|
+
provider: baseConfig?.provider ?? DEFAULT_PROVIDER,
|
|
27
|
+
transport: baseConfig?.transport ?? "webrtc",
|
|
28
|
+
model: baseConfig?.model ?? DEFAULT_MODEL,
|
|
29
|
+
voice: baseConfig?.voice ?? character?.voice?.voiceId ?? DEFAULT_VOICE,
|
|
30
|
+
instructions: baseConfig?.instructions ?? buildCharacterInstructions(character),
|
|
31
|
+
temperature: baseConfig?.temperature,
|
|
32
|
+
maxTokens: baseConfig?.maxTokens,
|
|
33
|
+
tools: baseConfig?.tools,
|
|
34
|
+
toolChoice: baseConfig?.toolChoice ?? "auto"
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
function buildCharacterInstructions(character) {
|
|
38
|
+
if (!character) {
|
|
39
|
+
return DEFAULT_REALTIME_AGENT_INSTRUCTIONS;
|
|
40
|
+
}
|
|
41
|
+
const identityLines = [
|
|
42
|
+
`You are ${character.name}.`,
|
|
43
|
+
"Stay fully in character during the conversation."
|
|
44
|
+
];
|
|
45
|
+
if (character.description) {
|
|
46
|
+
identityLines.push(ensureSentence(character.description));
|
|
47
|
+
}
|
|
48
|
+
if (character.personality) {
|
|
49
|
+
identityLines.push(
|
|
50
|
+
`Your personality is ${ensureSentenceFragment(character.personality)}.`
|
|
51
|
+
);
|
|
52
|
+
}
|
|
53
|
+
return [...identityLines, DEFAULT_REALTIME_AGENT_INSTRUCTIONS].join("\n");
|
|
54
|
+
}
|
|
55
|
+
function ensureSentence(text) {
|
|
56
|
+
return /[.!?]$/.test(text) ? text : `${text}.`;
|
|
57
|
+
}
|
|
58
|
+
function ensureSentenceFragment(text) {
|
|
59
|
+
return text.trim().replace(/[.!?]+$/, "");
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
// src/tools.ts
|
|
63
|
+
var MIN_GAZE = -1;
|
|
64
|
+
var MAX_GAZE = 1;
|
|
65
|
+
var SET_EXPRESSION_TOOL_NAME = "setExpression";
|
|
66
|
+
var PLAY_MOTION_TOOL_NAME = "playMotion";
|
|
67
|
+
var LOOK_AT_TOOL_NAME = "lookAt";
|
|
68
|
+
var AVATAR_CONTROL_TOOL_NAMES = [
|
|
69
|
+
SET_EXPRESSION_TOOL_NAME,
|
|
70
|
+
PLAY_MOTION_TOOL_NAME,
|
|
71
|
+
LOOK_AT_TOOL_NAME
|
|
72
|
+
];
|
|
73
|
+
function createAvatarControlTools(catalog) {
|
|
74
|
+
const tools = [];
|
|
75
|
+
if (catalog.expressions.length > 0) {
|
|
76
|
+
const expressionValues = [...catalog.expressions];
|
|
77
|
+
tools.push({
|
|
78
|
+
definition: {
|
|
79
|
+
type: "function",
|
|
80
|
+
name: SET_EXPRESSION_TOOL_NAME,
|
|
81
|
+
description: SET_EXPRESSION_TOOL_DESCRIPTION,
|
|
82
|
+
parameters: {
|
|
83
|
+
type: "object",
|
|
84
|
+
properties: {
|
|
85
|
+
expressionId: {
|
|
86
|
+
type: "string",
|
|
87
|
+
description: "Expression ID from the loaded Live2D model.",
|
|
88
|
+
enum: expressionValues
|
|
89
|
+
}
|
|
90
|
+
},
|
|
91
|
+
required: ["expressionId"]
|
|
92
|
+
}
|
|
93
|
+
},
|
|
94
|
+
async handler(args) {
|
|
95
|
+
const expressionId = args.expressionId;
|
|
96
|
+
if (typeof expressionId !== "string" || !expressionValues.includes(expressionId)) {
|
|
97
|
+
throw new Error(
|
|
98
|
+
'setExpression requires a valid "expressionId" from the model catalog'
|
|
99
|
+
);
|
|
100
|
+
}
|
|
101
|
+
return {
|
|
102
|
+
success: true,
|
|
103
|
+
expressionId
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
});
|
|
107
|
+
}
|
|
108
|
+
const motionGroups = Object.keys(catalog.motions);
|
|
109
|
+
if (motionGroups.length > 0) {
|
|
110
|
+
tools.push({
|
|
111
|
+
definition: {
|
|
112
|
+
type: "function",
|
|
113
|
+
name: PLAY_MOTION_TOOL_NAME,
|
|
114
|
+
description: PLAY_MOTION_TOOL_DESCRIPTION,
|
|
115
|
+
parameters: {
|
|
116
|
+
type: "object",
|
|
117
|
+
properties: {
|
|
118
|
+
group: {
|
|
119
|
+
type: "string",
|
|
120
|
+
description: "Motion group from the loaded Live2D model.",
|
|
121
|
+
enum: motionGroups
|
|
122
|
+
},
|
|
123
|
+
index: {
|
|
124
|
+
type: "number",
|
|
125
|
+
description: "Zero-based motion index within the selected motion group."
|
|
126
|
+
}
|
|
127
|
+
},
|
|
128
|
+
required: ["group", "index"]
|
|
129
|
+
}
|
|
130
|
+
},
|
|
131
|
+
async handler(args) {
|
|
132
|
+
const group = args.group;
|
|
133
|
+
const index = args.index;
|
|
134
|
+
if (typeof group !== "string" || !motionGroups.includes(group)) {
|
|
135
|
+
throw new Error(
|
|
136
|
+
'playMotion requires a valid "group" from the model catalog'
|
|
137
|
+
);
|
|
138
|
+
}
|
|
139
|
+
if (!Number.isInteger(index)) {
|
|
140
|
+
throw new Error('playMotion requires an integer "index"');
|
|
141
|
+
}
|
|
142
|
+
const count = catalog.motions[group] ?? 0;
|
|
143
|
+
const motionIndex = index;
|
|
144
|
+
if (motionIndex < 0 || motionIndex >= count) {
|
|
145
|
+
throw new Error(
|
|
146
|
+
`playMotion index ${String(motionIndex)} is out of range for group "${group}"`
|
|
147
|
+
);
|
|
148
|
+
}
|
|
149
|
+
return {
|
|
150
|
+
success: true,
|
|
151
|
+
group,
|
|
152
|
+
index: motionIndex
|
|
153
|
+
};
|
|
154
|
+
}
|
|
155
|
+
});
|
|
156
|
+
}
|
|
157
|
+
tools.push({
|
|
158
|
+
definition: {
|
|
159
|
+
type: "function",
|
|
160
|
+
name: LOOK_AT_TOOL_NAME,
|
|
161
|
+
description: LOOK_AT_TOOL_DESCRIPTION,
|
|
162
|
+
parameters: {
|
|
163
|
+
type: "object",
|
|
164
|
+
properties: {
|
|
165
|
+
x: {
|
|
166
|
+
type: "number",
|
|
167
|
+
description: "Normalized horizontal gaze target from -1 to 1."
|
|
168
|
+
},
|
|
169
|
+
y: {
|
|
170
|
+
type: "number",
|
|
171
|
+
description: "Normalized vertical gaze target from -1 to 1."
|
|
172
|
+
}
|
|
173
|
+
},
|
|
174
|
+
required: ["x", "y"]
|
|
175
|
+
}
|
|
176
|
+
},
|
|
177
|
+
async handler(args) {
|
|
178
|
+
const x = args.x;
|
|
179
|
+
const y = args.y;
|
|
180
|
+
if (typeof x !== "number" || Number.isNaN(x)) {
|
|
181
|
+
throw new Error('lookAt requires a numeric "x"');
|
|
182
|
+
}
|
|
183
|
+
if (typeof y !== "number" || Number.isNaN(y)) {
|
|
184
|
+
throw new Error('lookAt requires a numeric "y"');
|
|
185
|
+
}
|
|
186
|
+
return {
|
|
187
|
+
success: true,
|
|
188
|
+
x: clamp(x, MIN_GAZE, MAX_GAZE),
|
|
189
|
+
y: clamp(y, MIN_GAZE, MAX_GAZE)
|
|
190
|
+
};
|
|
191
|
+
}
|
|
192
|
+
});
|
|
193
|
+
return tools;
|
|
194
|
+
}
|
|
195
|
+
function clamp(value, min, max) {
|
|
196
|
+
return Math.max(min, Math.min(max, value));
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
// src/realtime-manager.ts
|
|
200
|
+
var DEFAULT_TOOL_TIMEOUT_MS = 1e4;
|
|
201
|
+
var RealtimeManagerImpl = class {
|
|
202
|
+
constructor(client, options = {}) {
|
|
203
|
+
this.client = client;
|
|
204
|
+
this.options = options;
|
|
205
|
+
for (const tool of options.tools ?? []) {
|
|
206
|
+
this.registerTool(tool);
|
|
207
|
+
}
|
|
208
|
+
this.client.onEvent((event) => {
|
|
209
|
+
void this.handleClientEvent(event).catch((error) => {
|
|
210
|
+
this.applyError(
|
|
211
|
+
error instanceof Error ? error : new Error(String(error)),
|
|
212
|
+
this.state.connection === "idle" ? "error" : this.state.connection
|
|
213
|
+
);
|
|
214
|
+
});
|
|
215
|
+
});
|
|
216
|
+
}
|
|
217
|
+
eventEmitter;
|
|
218
|
+
character = null;
|
|
219
|
+
isStoppingSession = false;
|
|
220
|
+
isRefreshingSession = false;
|
|
221
|
+
isAudioPlaybackActive = false;
|
|
222
|
+
sessionBaseConfig;
|
|
223
|
+
refreshInFlight = null;
|
|
224
|
+
hasQueuedRefresh = false;
|
|
225
|
+
stopRequestedDuringRefresh = false;
|
|
226
|
+
emittedUserStopDuringRefresh = false;
|
|
227
|
+
toolRegistry = /* @__PURE__ */ new Map();
|
|
228
|
+
state = {
|
|
229
|
+
connection: "idle",
|
|
230
|
+
session: {
|
|
231
|
+
status: "idle",
|
|
232
|
+
config: null
|
|
233
|
+
},
|
|
234
|
+
response: {
|
|
235
|
+
status: "idle",
|
|
236
|
+
text: ""
|
|
237
|
+
},
|
|
238
|
+
lastError: null
|
|
239
|
+
};
|
|
240
|
+
setEventEmitter(eventEmitter) {
|
|
241
|
+
this.eventEmitter = eventEmitter;
|
|
242
|
+
}
|
|
243
|
+
setCharacter(character) {
|
|
244
|
+
this.character = character;
|
|
245
|
+
this.state = {
|
|
246
|
+
...this.state,
|
|
247
|
+
session: {
|
|
248
|
+
...this.state.session,
|
|
249
|
+
characterId: character.id
|
|
250
|
+
}
|
|
251
|
+
};
|
|
252
|
+
this.emitState();
|
|
253
|
+
}
|
|
254
|
+
registerTool(tool) {
|
|
255
|
+
this.toolRegistry.set(tool.definition.name, tool);
|
|
256
|
+
}
|
|
257
|
+
unregisterTool(name) {
|
|
258
|
+
this.toolRegistry.delete(name);
|
|
259
|
+
}
|
|
260
|
+
getRegisteredTools() {
|
|
261
|
+
return Array.from(this.toolRegistry.values(), (tool) => ({
|
|
262
|
+
...tool.definition,
|
|
263
|
+
parameters: {
|
|
264
|
+
...tool.definition.parameters,
|
|
265
|
+
properties: { ...tool.definition.parameters.properties },
|
|
266
|
+
required: tool.definition.parameters.required ? [...tool.definition.parameters.required] : void 0
|
|
267
|
+
}
|
|
268
|
+
}));
|
|
269
|
+
}
|
|
270
|
+
getState() {
|
|
271
|
+
return {
|
|
272
|
+
...this.state,
|
|
273
|
+
session: {
|
|
274
|
+
...this.state.session,
|
|
275
|
+
config: this.state.session.config ? {
|
|
276
|
+
...this.state.session.config,
|
|
277
|
+
tools: this.state.session.config.tools ? [...this.state.session.config.tools] : void 0
|
|
278
|
+
} : null
|
|
279
|
+
},
|
|
280
|
+
response: {
|
|
281
|
+
...this.state.response
|
|
282
|
+
}
|
|
283
|
+
};
|
|
284
|
+
}
|
|
285
|
+
async startSession(config) {
|
|
286
|
+
if (this.state.session.status === "active" || this.state.session.status === "starting" || this.refreshInFlight) {
|
|
287
|
+
throw new Error("Realtime session already active");
|
|
288
|
+
}
|
|
289
|
+
this.sessionBaseConfig = this.resolveNextSessionBaseConfig(config);
|
|
290
|
+
const sessionConfig = this.buildEffectiveSessionConfig();
|
|
291
|
+
this.state = {
|
|
292
|
+
...this.state,
|
|
293
|
+
connection: "connecting",
|
|
294
|
+
session: {
|
|
295
|
+
status: "starting",
|
|
296
|
+
config: null,
|
|
297
|
+
characterId: this.character?.id
|
|
298
|
+
},
|
|
299
|
+
response: {
|
|
300
|
+
status: "idle",
|
|
301
|
+
text: ""
|
|
302
|
+
},
|
|
303
|
+
lastError: null
|
|
304
|
+
};
|
|
305
|
+
this.emitState();
|
|
306
|
+
try {
|
|
307
|
+
await this.client.connect(sessionConfig);
|
|
308
|
+
this.commitActiveSession(sessionConfig);
|
|
309
|
+
this.emitSessionStart("user");
|
|
310
|
+
} catch (error) {
|
|
311
|
+
this.finalizeFailedSession(
|
|
312
|
+
error instanceof Error ? error : new Error(String(error))
|
|
313
|
+
);
|
|
314
|
+
throw error;
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
updateSession(config) {
|
|
318
|
+
this.sessionBaseConfig = this.resolveNextSessionBaseConfig(config);
|
|
319
|
+
if (this.state.session.status !== "active") {
|
|
320
|
+
return Promise.resolve();
|
|
321
|
+
}
|
|
322
|
+
if (this.refreshInFlight) {
|
|
323
|
+
this.hasQueuedRefresh = true;
|
|
324
|
+
return this.refreshInFlight;
|
|
325
|
+
}
|
|
326
|
+
this.stopRequestedDuringRefresh = false;
|
|
327
|
+
this.hasQueuedRefresh = false;
|
|
328
|
+
this.emittedUserStopDuringRefresh = false;
|
|
329
|
+
this.refreshInFlight = this.runRefreshLoop().finally(() => {
|
|
330
|
+
this.refreshInFlight = null;
|
|
331
|
+
this.hasQueuedRefresh = false;
|
|
332
|
+
this.stopRequestedDuringRefresh = false;
|
|
333
|
+
this.emittedUserStopDuringRefresh = false;
|
|
334
|
+
});
|
|
335
|
+
return this.refreshInFlight;
|
|
336
|
+
}
|
|
337
|
+
async stopSession() {
|
|
338
|
+
if (this.refreshInFlight) {
|
|
339
|
+
this.stopRequestedDuringRefresh = true;
|
|
340
|
+
this.hasQueuedRefresh = false;
|
|
341
|
+
try {
|
|
342
|
+
await this.refreshInFlight;
|
|
343
|
+
} catch {
|
|
344
|
+
}
|
|
345
|
+
if (this.state.session.status === "active") {
|
|
346
|
+
await this.performStopSession();
|
|
347
|
+
} else if (this.stopRequestedDuringRefresh && !this.emittedUserStopDuringRefresh) {
|
|
348
|
+
this.emitSessionEnd("user");
|
|
349
|
+
}
|
|
350
|
+
return;
|
|
351
|
+
}
|
|
352
|
+
if (this.state.session.status !== "active") {
|
|
353
|
+
return;
|
|
354
|
+
}
|
|
355
|
+
await this.performStopSession();
|
|
356
|
+
}
|
|
357
|
+
async sendMessage(text) {
|
|
358
|
+
if (this.state.session.status !== "active") {
|
|
359
|
+
throw new Error("Realtime session not active");
|
|
360
|
+
}
|
|
361
|
+
await this.client.sendText(text);
|
|
362
|
+
}
|
|
363
|
+
async sendAudioChunk(audio) {
|
|
364
|
+
if (this.state.session.status !== "active") {
|
|
365
|
+
throw new Error("Realtime session not active");
|
|
366
|
+
}
|
|
367
|
+
await this.client.sendAudio(audio);
|
|
368
|
+
}
|
|
369
|
+
async interrupt() {
|
|
370
|
+
if (this.state.session.status !== "active") {
|
|
371
|
+
throw new Error("Realtime session not active");
|
|
372
|
+
}
|
|
373
|
+
await this.client.interrupt();
|
|
374
|
+
this.state = {
|
|
375
|
+
...this.state,
|
|
376
|
+
response: {
|
|
377
|
+
...this.state.response,
|
|
378
|
+
status: "interrupted"
|
|
379
|
+
}
|
|
380
|
+
};
|
|
381
|
+
this.emitAudioEnd();
|
|
382
|
+
this.emitState();
|
|
383
|
+
}
|
|
384
|
+
async handleClientEvent(event) {
|
|
385
|
+
switch (event.type) {
|
|
386
|
+
case "session.started":
|
|
387
|
+
if (this.state.connection === "connecting" || this.isRefreshingSession) {
|
|
388
|
+
return;
|
|
389
|
+
}
|
|
390
|
+
this.state = mergeRealtimeState(this.state, {
|
|
391
|
+
connection: "connected",
|
|
392
|
+
session: {
|
|
393
|
+
status: this.state.session.status === "starting" ? "active" : this.state.session.status,
|
|
394
|
+
config: this.state.session.config,
|
|
395
|
+
characterId: this.character?.id
|
|
396
|
+
}
|
|
397
|
+
});
|
|
398
|
+
this.emitState();
|
|
399
|
+
return;
|
|
400
|
+
case "session.ended":
|
|
401
|
+
if (this.isStoppingSession || this.isRefreshingSession || this.state.connection === "disconnecting") {
|
|
402
|
+
return;
|
|
403
|
+
}
|
|
404
|
+
this.finalizeStoppedSession(true, "user");
|
|
405
|
+
return;
|
|
406
|
+
case "user.transcript":
|
|
407
|
+
this.eventEmitter?.emit("realtime:user:transcript", {
|
|
408
|
+
text: event.text
|
|
409
|
+
});
|
|
410
|
+
return;
|
|
411
|
+
case "assistant.response.started":
|
|
412
|
+
this.state = {
|
|
413
|
+
...this.state,
|
|
414
|
+
response: {
|
|
415
|
+
status: "responding",
|
|
416
|
+
text: ""
|
|
417
|
+
}
|
|
418
|
+
};
|
|
419
|
+
this.eventEmitter?.emit("realtime:assistant:start", {
|
|
420
|
+
state: this.getState()
|
|
421
|
+
});
|
|
422
|
+
this.emitState();
|
|
423
|
+
return;
|
|
424
|
+
case "assistant.text.delta":
|
|
425
|
+
this.ensureResponseStarted();
|
|
426
|
+
this.state = {
|
|
427
|
+
...this.state,
|
|
428
|
+
response: {
|
|
429
|
+
status: "responding",
|
|
430
|
+
text: this.state.response.text + event.text
|
|
431
|
+
}
|
|
432
|
+
};
|
|
433
|
+
this.eventEmitter?.emit("realtime:text:delta", { text: event.text });
|
|
434
|
+
this.eventEmitter?.emit("realtime:assistant:delta", {
|
|
435
|
+
text: event.text
|
|
436
|
+
});
|
|
437
|
+
this.emitState();
|
|
438
|
+
return;
|
|
439
|
+
case "assistant.response.completed": {
|
|
440
|
+
if (this.state.response.status === "interrupted") {
|
|
441
|
+
return;
|
|
442
|
+
}
|
|
443
|
+
const text = event.text || this.state.response.text;
|
|
444
|
+
this.state = {
|
|
445
|
+
...this.state,
|
|
446
|
+
response: {
|
|
447
|
+
status: "completed",
|
|
448
|
+
text
|
|
449
|
+
}
|
|
450
|
+
};
|
|
451
|
+
this.eventEmitter?.emit("realtime:assistant:done", { text });
|
|
452
|
+
this.emitState();
|
|
453
|
+
return;
|
|
454
|
+
}
|
|
455
|
+
case "audio.output.started":
|
|
456
|
+
this.emitAudioStart();
|
|
457
|
+
return;
|
|
458
|
+
case "audio.output.ended":
|
|
459
|
+
this.emitAudioEnd();
|
|
460
|
+
return;
|
|
461
|
+
case "audio.lipsync":
|
|
462
|
+
if (event.rms > 1e-3 && !this.isAudioPlaybackActive) {
|
|
463
|
+
this.emitAudioStart();
|
|
464
|
+
}
|
|
465
|
+
this.eventEmitter?.emit("tts:lipsync:update", { rms: event.rms });
|
|
466
|
+
return;
|
|
467
|
+
case "tool.call":
|
|
468
|
+
this.eventEmitter?.emit("realtime:tool:call", {
|
|
469
|
+
name: event.name,
|
|
470
|
+
args: event.args,
|
|
471
|
+
callId: event.callId
|
|
472
|
+
});
|
|
473
|
+
await this.executeToolCall(event);
|
|
474
|
+
return;
|
|
475
|
+
case "tool.result":
|
|
476
|
+
this.eventEmitter?.emit("realtime:tool:result", {
|
|
477
|
+
name: event.name,
|
|
478
|
+
output: event.output,
|
|
479
|
+
callId: event.callId
|
|
480
|
+
});
|
|
481
|
+
return;
|
|
482
|
+
case "state":
|
|
483
|
+
this.state = mergeRealtimeState(this.state, event.state);
|
|
484
|
+
this.emitState();
|
|
485
|
+
return;
|
|
486
|
+
case "error":
|
|
487
|
+
this.applyError(event.error, this.state.connection);
|
|
488
|
+
return;
|
|
489
|
+
}
|
|
490
|
+
}
|
|
491
|
+
async executeToolCall(event) {
|
|
492
|
+
const tool = this.toolRegistry.get(event.name);
|
|
493
|
+
if (!event.callId) {
|
|
494
|
+
this.emitToolError(
|
|
495
|
+
event.name,
|
|
496
|
+
new Error(`Tool "${event.name}" is missing a call ID`),
|
|
497
|
+
event.callId
|
|
498
|
+
);
|
|
499
|
+
return;
|
|
500
|
+
}
|
|
501
|
+
if (!tool) {
|
|
502
|
+
await this.handleToolExecutionFailure(
|
|
503
|
+
event.name,
|
|
504
|
+
event.callId,
|
|
505
|
+
new Error(`No realtime tool registered for "${event.name}"`)
|
|
506
|
+
);
|
|
507
|
+
return;
|
|
508
|
+
}
|
|
509
|
+
try {
|
|
510
|
+
const output = await this.runToolHandler(tool, event);
|
|
511
|
+
await this.client.sendToolResult(event.callId, output);
|
|
512
|
+
this.postProcessToolResult(event.name, output);
|
|
513
|
+
this.eventEmitter?.emit("realtime:tool:result", {
|
|
514
|
+
name: event.name,
|
|
515
|
+
output,
|
|
516
|
+
callId: event.callId
|
|
517
|
+
});
|
|
518
|
+
} catch (error) {
|
|
519
|
+
await this.handleToolExecutionFailure(
|
|
520
|
+
event.name,
|
|
521
|
+
event.callId,
|
|
522
|
+
error instanceof Error ? error : new Error(String(error))
|
|
523
|
+
);
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
async runToolHandler(tool, event) {
|
|
527
|
+
const timeoutMs = tool.timeoutMs ?? this.options.defaultToolTimeoutMs ?? DEFAULT_TOOL_TIMEOUT_MS;
|
|
528
|
+
const result = await withTimeout(
|
|
529
|
+
tool.handler(event.args, {
|
|
530
|
+
character: this.character,
|
|
531
|
+
state: this.getState(),
|
|
532
|
+
callId: event.callId
|
|
533
|
+
}),
|
|
534
|
+
timeoutMs,
|
|
535
|
+
tool.definition.name
|
|
536
|
+
);
|
|
537
|
+
if (!isRecord(result)) {
|
|
538
|
+
throw new Error(
|
|
539
|
+
`Realtime tool "${tool.definition.name}" must return an object`
|
|
540
|
+
);
|
|
541
|
+
}
|
|
542
|
+
return result;
|
|
543
|
+
}
|
|
544
|
+
async handleToolExecutionFailure(name, callId, error) {
|
|
545
|
+
this.emitToolError(name, error, callId);
|
|
546
|
+
try {
|
|
547
|
+
await this.client.sendToolResult(callId, createFailureOutput(error));
|
|
548
|
+
} catch (sendError) {
|
|
549
|
+
this.emitToolError(
|
|
550
|
+
name,
|
|
551
|
+
sendError instanceof Error ? sendError : new Error(String(sendError)),
|
|
552
|
+
callId
|
|
553
|
+
);
|
|
554
|
+
}
|
|
555
|
+
}
|
|
556
|
+
postProcessToolResult(name, output) {
|
|
557
|
+
switch (name) {
|
|
558
|
+
case SET_EXPRESSION_TOOL_NAME: {
|
|
559
|
+
const expressionId = output.expressionId;
|
|
560
|
+
if (typeof expressionId === "string") {
|
|
561
|
+
this.eventEmitter?.emit("realtime:expression", { expressionId });
|
|
562
|
+
}
|
|
563
|
+
return;
|
|
564
|
+
}
|
|
565
|
+
case PLAY_MOTION_TOOL_NAME: {
|
|
566
|
+
const group = output.group;
|
|
567
|
+
const index = output.index;
|
|
568
|
+
if (typeof group === "string" && Number.isInteger(index)) {
|
|
569
|
+
const motionIndex = index;
|
|
570
|
+
this.eventEmitter?.emit("realtime:motion", {
|
|
571
|
+
group,
|
|
572
|
+
index: motionIndex
|
|
573
|
+
});
|
|
574
|
+
}
|
|
575
|
+
return;
|
|
576
|
+
}
|
|
577
|
+
case LOOK_AT_TOOL_NAME: {
|
|
578
|
+
const coords = readGazeCoordinates(output);
|
|
579
|
+
if (coords) {
|
|
580
|
+
this.eventEmitter?.emit("realtime:gaze", coords);
|
|
581
|
+
}
|
|
582
|
+
return;
|
|
583
|
+
}
|
|
584
|
+
default:
|
|
585
|
+
return;
|
|
586
|
+
}
|
|
587
|
+
}
|
|
588
|
+
emitToolError(name, error, callId) {
|
|
589
|
+
this.eventEmitter?.emit("realtime:tool:error", {
|
|
590
|
+
name,
|
|
591
|
+
error,
|
|
592
|
+
callId
|
|
593
|
+
});
|
|
594
|
+
}
|
|
595
|
+
ensureResponseStarted() {
|
|
596
|
+
if (this.state.response.status === "responding") {
|
|
597
|
+
return;
|
|
598
|
+
}
|
|
599
|
+
this.state = {
|
|
600
|
+
...this.state,
|
|
601
|
+
response: {
|
|
602
|
+
status: "responding",
|
|
603
|
+
text: this.state.response.text
|
|
604
|
+
}
|
|
605
|
+
};
|
|
606
|
+
this.eventEmitter?.emit("realtime:assistant:start", {
|
|
607
|
+
state: this.getState()
|
|
608
|
+
});
|
|
609
|
+
}
|
|
610
|
+
emitState() {
|
|
611
|
+
this.eventEmitter?.emit("realtime:state", {
|
|
612
|
+
state: this.getState()
|
|
613
|
+
});
|
|
614
|
+
}
|
|
615
|
+
applyError(error, connection) {
|
|
616
|
+
this.emitAudioEnd();
|
|
617
|
+
this.state = {
|
|
618
|
+
...this.state,
|
|
619
|
+
connection,
|
|
620
|
+
lastError: error
|
|
621
|
+
};
|
|
622
|
+
this.eventEmitter?.emit("realtime:error", { error });
|
|
623
|
+
this.emitState();
|
|
624
|
+
}
|
|
625
|
+
emitAudioStart() {
|
|
626
|
+
if (this.isAudioPlaybackActive) {
|
|
627
|
+
return;
|
|
628
|
+
}
|
|
629
|
+
this.isAudioPlaybackActive = true;
|
|
630
|
+
this.eventEmitter?.emit("tts:audio:start", {});
|
|
631
|
+
}
|
|
632
|
+
emitAudioEnd() {
|
|
633
|
+
if (!this.isAudioPlaybackActive) {
|
|
634
|
+
return;
|
|
635
|
+
}
|
|
636
|
+
this.isAudioPlaybackActive = false;
|
|
637
|
+
this.eventEmitter?.emit("tts:lipsync:update", { rms: 0 });
|
|
638
|
+
this.eventEmitter?.emit("tts:audio:end", {});
|
|
639
|
+
}
|
|
640
|
+
resolveNextSessionBaseConfig(config) {
|
|
641
|
+
return mergeSessionConfig(
|
|
642
|
+
this.sessionBaseConfig ?? this.options.defaultSessionConfig,
|
|
643
|
+
config
|
|
644
|
+
);
|
|
645
|
+
}
|
|
646
|
+
buildEffectiveSessionConfig() {
|
|
647
|
+
return buildRealtimeSessionConfig({
|
|
648
|
+
character: this.character,
|
|
649
|
+
baseConfig: {
|
|
650
|
+
...this.sessionBaseConfig ?? {},
|
|
651
|
+
tools: this.getRegisteredTools()
|
|
652
|
+
}
|
|
653
|
+
});
|
|
654
|
+
}
|
|
655
|
+
commitActiveSession(config) {
|
|
656
|
+
this.state = {
|
|
657
|
+
...this.state,
|
|
658
|
+
connection: "connected",
|
|
659
|
+
session: {
|
|
660
|
+
status: "active",
|
|
661
|
+
config,
|
|
662
|
+
characterId: this.character?.id
|
|
663
|
+
},
|
|
664
|
+
response: {
|
|
665
|
+
status: "idle",
|
|
666
|
+
text: ""
|
|
667
|
+
},
|
|
668
|
+
lastError: null
|
|
669
|
+
};
|
|
670
|
+
this.emitState();
|
|
671
|
+
}
|
|
672
|
+
async performStopSession() {
|
|
673
|
+
this.isStoppingSession = true;
|
|
674
|
+
this.state = {
|
|
675
|
+
...this.state,
|
|
676
|
+
connection: "disconnecting"
|
|
677
|
+
};
|
|
678
|
+
this.emitState();
|
|
679
|
+
try {
|
|
680
|
+
await this.client.disconnect();
|
|
681
|
+
this.finalizeStoppedSession(true, "user");
|
|
682
|
+
} catch (error) {
|
|
683
|
+
this.finalizeFailedSession(
|
|
684
|
+
error instanceof Error ? error : new Error(String(error))
|
|
685
|
+
);
|
|
686
|
+
throw error;
|
|
687
|
+
} finally {
|
|
688
|
+
this.isStoppingSession = false;
|
|
689
|
+
}
|
|
690
|
+
}
|
|
691
|
+
async runRefreshLoop() {
|
|
692
|
+
do {
|
|
693
|
+
this.hasQueuedRefresh = false;
|
|
694
|
+
await this.performSessionRefresh();
|
|
695
|
+
} while (this.hasQueuedRefresh && !this.stopRequestedDuringRefresh);
|
|
696
|
+
}
|
|
697
|
+
async performSessionRefresh() {
|
|
698
|
+
const previousState = this.getState();
|
|
699
|
+
const sessionConfig = this.buildEffectiveSessionConfig();
|
|
700
|
+
this.isRefreshingSession = true;
|
|
701
|
+
this.state = {
|
|
702
|
+
...this.state,
|
|
703
|
+
connection: "disconnecting",
|
|
704
|
+
lastError: null
|
|
705
|
+
};
|
|
706
|
+
this.emitState();
|
|
707
|
+
try {
|
|
708
|
+
await this.client.disconnect();
|
|
709
|
+
if (this.stopRequestedDuringRefresh) {
|
|
710
|
+
this.emittedUserStopDuringRefresh = true;
|
|
711
|
+
this.finalizeStoppedSession(true, "user");
|
|
712
|
+
return;
|
|
713
|
+
}
|
|
714
|
+
this.state = {
|
|
715
|
+
...this.state,
|
|
716
|
+
connection: "connecting",
|
|
717
|
+
response: {
|
|
718
|
+
status: "idle",
|
|
719
|
+
text: ""
|
|
720
|
+
}
|
|
721
|
+
};
|
|
722
|
+
this.emitState();
|
|
723
|
+
await this.client.connect(sessionConfig);
|
|
724
|
+
if (this.stopRequestedDuringRefresh) {
|
|
725
|
+
await this.client.disconnect();
|
|
726
|
+
this.emittedUserStopDuringRefresh = true;
|
|
727
|
+
this.finalizeStoppedSession(true, "user");
|
|
728
|
+
return;
|
|
729
|
+
}
|
|
730
|
+
this.commitActiveSession(sessionConfig);
|
|
731
|
+
this.emitSessionEnd("refresh", previousState);
|
|
732
|
+
this.emitSessionStart("refresh");
|
|
733
|
+
} catch (error) {
|
|
734
|
+
this.finalizeFailedSession(
|
|
735
|
+
error instanceof Error ? error : new Error(String(error))
|
|
736
|
+
);
|
|
737
|
+
throw error;
|
|
738
|
+
} finally {
|
|
739
|
+
this.isRefreshingSession = false;
|
|
740
|
+
}
|
|
741
|
+
}
|
|
742
|
+
emitSessionStart(reason, state = this.getState()) {
|
|
743
|
+
this.eventEmitter?.emit("realtime:session:start", {
|
|
744
|
+
state,
|
|
745
|
+
reason
|
|
746
|
+
});
|
|
747
|
+
}
|
|
748
|
+
emitSessionEnd(reason, state = this.getState()) {
|
|
749
|
+
this.eventEmitter?.emit("realtime:session:end", {
|
|
750
|
+
state,
|
|
751
|
+
reason
|
|
752
|
+
});
|
|
753
|
+
}
|
|
754
|
+
finalizeFailedSession(error) {
|
|
755
|
+
this.emitAudioEnd();
|
|
756
|
+
this.state = {
|
|
757
|
+
...this.state,
|
|
758
|
+
connection: "error",
|
|
759
|
+
session: {
|
|
760
|
+
status: "stopped",
|
|
761
|
+
config: null,
|
|
762
|
+
characterId: this.character?.id
|
|
763
|
+
},
|
|
764
|
+
response: {
|
|
765
|
+
status: "idle",
|
|
766
|
+
text: ""
|
|
767
|
+
},
|
|
768
|
+
lastError: error
|
|
769
|
+
};
|
|
770
|
+
this.eventEmitter?.emit("realtime:error", { error });
|
|
771
|
+
this.emitState();
|
|
772
|
+
}
|
|
773
|
+
finalizeStoppedSession(emitSessionEnd, reason) {
|
|
774
|
+
this.emitAudioEnd();
|
|
775
|
+
this.state = {
|
|
776
|
+
...this.state,
|
|
777
|
+
connection: "idle",
|
|
778
|
+
session: {
|
|
779
|
+
status: "stopped",
|
|
780
|
+
config: null,
|
|
781
|
+
characterId: this.character?.id
|
|
782
|
+
},
|
|
783
|
+
response: {
|
|
784
|
+
status: "idle",
|
|
785
|
+
text: ""
|
|
786
|
+
},
|
|
787
|
+
lastError: null
|
|
788
|
+
};
|
|
789
|
+
this.emitState();
|
|
790
|
+
if (emitSessionEnd) {
|
|
791
|
+
this.emitSessionEnd(reason);
|
|
792
|
+
}
|
|
793
|
+
}
|
|
794
|
+
};
|
|
795
|
+
function readGazeCoordinates(output) {
|
|
796
|
+
const x = output.x;
|
|
797
|
+
const y = output.y;
|
|
798
|
+
if (typeof x !== "number" || typeof y !== "number") {
|
|
799
|
+
return null;
|
|
800
|
+
}
|
|
801
|
+
return { x, y };
|
|
802
|
+
}
|
|
803
|
+
function createRealtimeManager(client, options) {
|
|
804
|
+
return new RealtimeManagerImpl(client, options);
|
|
805
|
+
}
|
|
806
|
+
function mergeSessionConfig(baseConfig, overrideConfig) {
|
|
807
|
+
if (!baseConfig && !overrideConfig) {
|
|
808
|
+
return void 0;
|
|
809
|
+
}
|
|
810
|
+
const { tools: _ignoredOverrideTools, ...overrideWithoutTools } = overrideConfig ?? {};
|
|
811
|
+
return {
|
|
812
|
+
...baseConfig,
|
|
813
|
+
...overrideWithoutTools
|
|
814
|
+
};
|
|
815
|
+
}
|
|
816
|
+
function mergeRealtimeState(current, partial) {
|
|
817
|
+
return {
|
|
818
|
+
connection: partial.connection ?? current.connection,
|
|
819
|
+
session: {
|
|
820
|
+
status: partial.session?.status ?? current.session.status,
|
|
821
|
+
config: partial.session?.config ?? current.session.config,
|
|
822
|
+
characterId: partial.session?.characterId ?? current.session.characterId
|
|
823
|
+
},
|
|
824
|
+
response: {
|
|
825
|
+
status: partial.response?.status ?? current.response.status,
|
|
826
|
+
text: partial.response?.text ?? current.response.text
|
|
827
|
+
},
|
|
828
|
+
lastError: partial.lastError ?? current.lastError
|
|
829
|
+
};
|
|
830
|
+
}
|
|
831
|
+
function isRecord(value) {
|
|
832
|
+
return typeof value === "object" && value !== null;
|
|
833
|
+
}
|
|
834
|
+
function createFailureOutput(error) {
|
|
835
|
+
return {
|
|
836
|
+
success: false,
|
|
837
|
+
error: error.message
|
|
838
|
+
};
|
|
839
|
+
}
|
|
840
|
+
async function withTimeout(promise, timeoutMs, toolName) {
|
|
841
|
+
let timeoutId;
|
|
842
|
+
try {
|
|
843
|
+
return await Promise.race([
|
|
844
|
+
promise,
|
|
845
|
+
new Promise((_, reject) => {
|
|
846
|
+
timeoutId = setTimeout(() => {
|
|
847
|
+
reject(
|
|
848
|
+
new Error(
|
|
849
|
+
`Realtime tool "${toolName}" timed out after ${timeoutMs}ms`
|
|
850
|
+
)
|
|
851
|
+
);
|
|
852
|
+
}, timeoutMs);
|
|
853
|
+
})
|
|
854
|
+
]);
|
|
855
|
+
} finally {
|
|
856
|
+
if (timeoutId !== void 0) {
|
|
857
|
+
clearTimeout(timeoutId);
|
|
858
|
+
}
|
|
859
|
+
}
|
|
860
|
+
}
|
|
861
|
+
|
|
862
|
+
export { AVATAR_CONTROL_TOOL_NAMES, DEFAULT_REALTIME_AGENT_INSTRUCTIONS, LOOK_AT_TOOL_NAME, PLAY_MOTION_TOOL_NAME, RealtimeManagerImpl, SET_EXPRESSION_TOOL_NAME, buildRealtimeSessionConfig, createAvatarControlTools, createRealtimeManager };
|
|
863
|
+
//# sourceMappingURL=index.mjs.map
|
|
864
|
+
//# sourceMappingURL=index.mjs.map
|