@charivo/realtime 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -10
- package/dist/chunk-3K733UIP.mjs +58 -0
- package/dist/chunk-3K733UIP.mjs.map +1 -0
- package/dist/{chunk-DHNRDHST.mjs → chunk-WPZF3VXZ.mjs} +8 -5
- package/dist/chunk-WPZF3VXZ.mjs.map +1 -0
- package/dist/{chunk-PTFZ7CML.mjs → chunk-YLMDWMWQ.mjs} +23 -10
- package/dist/chunk-YLMDWMWQ.mjs.map +1 -0
- package/dist/{chunk-Y7JO2FFX.mjs → chunk-ZJZO4QYA.mjs} +7 -5
- package/dist/chunk-ZJZO4QYA.mjs.map +1 -0
- package/dist/index.d.mts +23 -17
- package/dist/index.d.ts +23 -17
- package/dist/index.js +129 -214
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +128 -202
- package/dist/index.mjs.map +1 -1
- package/dist/openai/index.d.mts +1 -1
- package/dist/openai/index.d.ts +1 -1
- package/dist/openai/index.js +8 -5
- package/dist/openai/index.js.map +1 -1
- package/dist/openai/index.mjs +3 -3
- package/dist/openai-agents/index.d.mts +2 -1
- package/dist/openai-agents/index.d.ts +2 -1
- package/dist/openai-agents/index.js +22 -10
- package/dist/openai-agents/index.js.map +1 -1
- package/dist/openai-agents/index.mjs +3 -3
- package/dist/remote/index.d.mts +1 -1
- package/dist/remote/index.d.ts +1 -1
- package/dist/remote/index.js +58 -39
- package/dist/remote/index.js.map +1 -1
- package/dist/remote/index.mjs +37 -33
- package/dist/remote/index.mjs.map +1 -1
- package/dist/{types-bmYuc_Vs.d.mts → types-BRya38bF.d.mts} +3 -0
- package/dist/{types-bmYuc_Vs.d.ts → types-BRya38bF.d.ts} +3 -0
- package/package.json +2 -2
- package/dist/chunk-DHNRDHST.mjs.map +0 -1
- package/dist/chunk-PDODY2T7.mjs +0 -65
- package/dist/chunk-PDODY2T7.mjs.map +0 -1
- package/dist/chunk-PTFZ7CML.mjs.map +0 -1
- package/dist/chunk-Y7JO2FFX.mjs.map +0 -1
package/dist/index.d.ts
CHANGED
|
@@ -1,13 +1,28 @@
|
|
|
1
|
-
import { R as RealtimeTransportClient } from './types-
|
|
2
|
-
export { b as RealtimeClient, a as RealtimeTransportEvent } from './types-
|
|
3
|
-
import { RealtimeSessionConfig, RealtimeToolRegistration, RealtimeManager, CharivoEventEmitter, Character, RealtimeTool, RealtimeState
|
|
4
|
-
export { Character, CharivoEventEmitter, RealtimeConnectionState, RealtimeManager, RealtimeProvider, RealtimeReconnectCause, RealtimeResponseStatus, RealtimeSessionBootstrap, RealtimeSessionConfig, RealtimeSessionRequest, RealtimeSessionStatus, RealtimeSessionTransitionReason, RealtimeState, RealtimeTool, RealtimeToolChoice, RealtimeToolContext, RealtimeToolHandler, RealtimeToolRegistration, RealtimeTransportKind } from '@charivo/core';
|
|
1
|
+
import { R as RealtimeTransportClient } from './types-BRya38bF.js';
|
|
2
|
+
export { b as RealtimeClient, a as RealtimeTransportEvent } from './types-BRya38bF.js';
|
|
3
|
+
import { RealtimeSessionConfig, RealtimeToolRegistration, EventMap, RealtimeManager, CharivoEventEmitter, Character, RealtimeTool, RealtimeState } from '@charivo/core';
|
|
4
|
+
export { Character, CharivoEventEmitter, RealtimeConnectionState, RealtimeManager, RealtimeProvider, RealtimeReconnectCause, RealtimeResponseStatus, RealtimeSessionBootstrap, RealtimeSessionConfig, RealtimeSessionRequest, RealtimeSessionStatus, RealtimeSessionTransitionReason, RealtimeState, RealtimeTool, RealtimeToolChoice, RealtimeToolContext, RealtimeToolHandler, RealtimeToolRegistration, RealtimeTransportKind, RealtimeUsageEvent } from '@charivo/core';
|
|
5
5
|
|
|
6
6
|
interface RealtimeManagerOptions {
|
|
7
7
|
defaultSessionConfig?: Omit<RealtimeSessionConfig, "tools">;
|
|
8
8
|
tools?: RealtimeToolRegistration[];
|
|
9
9
|
defaultToolTimeoutMs?: number;
|
|
10
|
+
resultProjectors?: RealtimeToolResultProjector[];
|
|
11
|
+
logger?: RealtimeLogger;
|
|
10
12
|
}
|
|
13
|
+
interface RealtimeLogger {
|
|
14
|
+
debug?(message: string, context?: Record<string, unknown>): void;
|
|
15
|
+
info?(message: string, context?: Record<string, unknown>): void;
|
|
16
|
+
warn?(message: string, context?: Record<string, unknown>): void;
|
|
17
|
+
error?(message: string, context?: Record<string, unknown>): void;
|
|
18
|
+
}
|
|
19
|
+
interface RealtimeToolResultProjectorContext {
|
|
20
|
+
name: string;
|
|
21
|
+
output: Record<string, unknown>;
|
|
22
|
+
callId?: string;
|
|
23
|
+
emit<K extends keyof EventMap>(event: K, payload: EventMap[K]): void;
|
|
24
|
+
}
|
|
25
|
+
type RealtimeToolResultProjector = (context: RealtimeToolResultProjectorContext) => void;
|
|
11
26
|
/**
|
|
12
27
|
* Provider-agnostic realtime manager.
|
|
13
28
|
*
|
|
@@ -51,7 +66,7 @@ declare class RealtimeManagerImpl implements RealtimeManager {
|
|
|
51
66
|
private executeToolCall;
|
|
52
67
|
private runToolHandler;
|
|
53
68
|
private handleToolExecutionFailure;
|
|
54
|
-
private
|
|
69
|
+
private projectToolResult;
|
|
55
70
|
private emitToolError;
|
|
56
71
|
private ensureResponseStarted;
|
|
57
72
|
private beginReconnect;
|
|
@@ -76,20 +91,11 @@ declare class RealtimeManagerImpl implements RealtimeManager {
|
|
|
76
91
|
private emitSessionEnd;
|
|
77
92
|
private finalizeFailedSession;
|
|
78
93
|
private finalizeStoppedSession;
|
|
94
|
+
private emitUsageEvent;
|
|
95
|
+
private log;
|
|
79
96
|
}
|
|
80
97
|
declare function createRealtimeManager(client: RealtimeTransportClient, options?: RealtimeManagerOptions): RealtimeManager;
|
|
81
98
|
|
|
82
|
-
declare const SET_EXPRESSION_TOOL_NAME = "setExpression";
|
|
83
|
-
declare const PLAY_MOTION_TOOL_NAME = "playMotion";
|
|
84
|
-
declare const LOOK_AT_TOOL_NAME = "lookAt";
|
|
85
|
-
declare const AVATAR_CONTROL_TOOL_NAMES: readonly ["setExpression", "playMotion", "lookAt"];
|
|
86
|
-
interface ExpressionArgs {
|
|
87
|
-
expressionId: string;
|
|
88
|
-
}
|
|
89
|
-
type MotionArgs = MotionSelection;
|
|
90
|
-
type LookAtArgs = GazeCoordinates;
|
|
91
|
-
declare function createAvatarControlTools(catalog: AvatarControlCatalog): RealtimeToolRegistration[];
|
|
92
|
-
|
|
93
99
|
declare const DEFAULT_REALTIME_AGENT_INSTRUCTIONS: string;
|
|
94
100
|
interface BuildRealtimeSessionConfigOptions {
|
|
95
101
|
character?: Character | null;
|
|
@@ -97,4 +103,4 @@ interface BuildRealtimeSessionConfigOptions {
|
|
|
97
103
|
}
|
|
98
104
|
declare function buildRealtimeSessionConfig({ character, baseConfig, }?: BuildRealtimeSessionConfigOptions): RealtimeSessionConfig;
|
|
99
105
|
|
|
100
|
-
export {
|
|
106
|
+
export { type BuildRealtimeSessionConfigOptions, DEFAULT_REALTIME_AGENT_INSTRUCTIONS, type RealtimeLogger, RealtimeManagerImpl, type RealtimeManagerOptions, type RealtimeToolResultProjector, type RealtimeToolResultProjectorContext, RealtimeTransportClient, buildRealtimeSessionConfig, createRealtimeManager };
|
package/dist/index.js
CHANGED
|
@@ -8,22 +8,15 @@ var core = require('@charivo/core');
|
|
|
8
8
|
var DEFAULT_PROVIDER = "openai";
|
|
9
9
|
var DEFAULT_MODEL = "gpt-realtime-mini";
|
|
10
10
|
var DEFAULT_REALTIME_VOICE = "marin";
|
|
11
|
-
var SET_EXPRESSION_TOOL_DESCRIPTION = "Change the avatar's facial expression only when the emotional beat clearly shifts or should linger across the reply. Do not use this for every polite or lightweight reaction.";
|
|
12
|
-
var PLAY_MOTION_TOOL_DESCRIPTION = "Play a noticeable body motion for greetings, emphasis, or bigger reaction beats. Prefer this over stacking multiple smaller actions when the moment needs one clear accent. Usually use at most one motion in a reply.";
|
|
13
|
-
var LOOK_AT_TOOL_DESCRIPTION = `Shift the avatar's gaze for subtle attention changes or conversational focus. Trigger this on natural phrases like "glance", "look over", "peek at", or directional cues. Prefer this before "setExpression" when a lightweight reaction is enough.`;
|
|
14
11
|
var DEFAULT_REALTIME_AGENT_INSTRUCTIONS = `
|
|
15
|
-
You are speaking in a realtime voice conversation
|
|
12
|
+
You are speaking in a realtime voice conversation.
|
|
16
13
|
Respond naturally and keep replies concise enough for spoken delivery.
|
|
17
|
-
Use
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
Use "playMotion" for greetings, emphasis, or bigger reaction beats, usually at most once in a reply.
|
|
21
|
-
Use "lookAt" for subtle attention shifts or conversational focus, and prefer it before "setExpression" when a lightweight reaction is enough.
|
|
22
|
-
Prefer a single meaningful avatar action over chaining multiple actions on simple replies.
|
|
23
|
-
Many turns should use no avatar tool at all.
|
|
14
|
+
Use tools only when they add something meaningful to the moment.
|
|
15
|
+
Prefer a single meaningful action over chaining multiple actions on simple replies.
|
|
16
|
+
Many turns should use no tool at all.
|
|
24
17
|
Never break character. Never refer to yourself as an AI, model, or assistant.
|
|
25
18
|
Do not mention tool calls in the spoken response.
|
|
26
|
-
Speak only the words you want heard aloud. Use
|
|
19
|
+
Speak only the words you want heard aloud. Use tools for actions. Never say tool names or tool arguments out loud. Do not output bracketed or parenthetical action notes. If a tool is unavailable, omit the action entirely rather than narrating it.
|
|
27
20
|
`.trim();
|
|
28
21
|
function buildRealtimeSessionConfig({
|
|
29
22
|
character,
|
|
@@ -66,143 +59,6 @@ function ensureSentenceFragment(text) {
|
|
|
66
59
|
return text.trim().replace(/[.!?]+$/, "");
|
|
67
60
|
}
|
|
68
61
|
|
|
69
|
-
// src/tools.ts
|
|
70
|
-
var MIN_GAZE = -1;
|
|
71
|
-
var MAX_GAZE = 1;
|
|
72
|
-
var SET_EXPRESSION_TOOL_NAME = "setExpression";
|
|
73
|
-
var PLAY_MOTION_TOOL_NAME = "playMotion";
|
|
74
|
-
var LOOK_AT_TOOL_NAME = "lookAt";
|
|
75
|
-
var AVATAR_CONTROL_TOOL_NAMES = [
|
|
76
|
-
SET_EXPRESSION_TOOL_NAME,
|
|
77
|
-
PLAY_MOTION_TOOL_NAME,
|
|
78
|
-
LOOK_AT_TOOL_NAME
|
|
79
|
-
];
|
|
80
|
-
function createAvatarControlTools(catalog) {
|
|
81
|
-
const tools = [];
|
|
82
|
-
if (catalog.expressions.length > 0) {
|
|
83
|
-
const expressionValues = [...catalog.expressions];
|
|
84
|
-
tools.push({
|
|
85
|
-
definition: {
|
|
86
|
-
type: "function",
|
|
87
|
-
name: SET_EXPRESSION_TOOL_NAME,
|
|
88
|
-
description: SET_EXPRESSION_TOOL_DESCRIPTION,
|
|
89
|
-
parameters: {
|
|
90
|
-
type: "object",
|
|
91
|
-
properties: {
|
|
92
|
-
expressionId: {
|
|
93
|
-
type: "string",
|
|
94
|
-
description: "Expression ID from the loaded Live2D model.",
|
|
95
|
-
enum: expressionValues
|
|
96
|
-
}
|
|
97
|
-
},
|
|
98
|
-
required: ["expressionId"]
|
|
99
|
-
}
|
|
100
|
-
},
|
|
101
|
-
async handler(args) {
|
|
102
|
-
const expressionId = args.expressionId;
|
|
103
|
-
if (typeof expressionId !== "string" || !expressionValues.includes(expressionId)) {
|
|
104
|
-
throw new Error(
|
|
105
|
-
'setExpression requires a valid "expressionId" from the model catalog'
|
|
106
|
-
);
|
|
107
|
-
}
|
|
108
|
-
return {
|
|
109
|
-
success: true,
|
|
110
|
-
expressionId
|
|
111
|
-
};
|
|
112
|
-
}
|
|
113
|
-
});
|
|
114
|
-
}
|
|
115
|
-
const motionGroups = Object.keys(catalog.motions);
|
|
116
|
-
if (motionGroups.length > 0) {
|
|
117
|
-
tools.push({
|
|
118
|
-
definition: {
|
|
119
|
-
type: "function",
|
|
120
|
-
name: PLAY_MOTION_TOOL_NAME,
|
|
121
|
-
description: PLAY_MOTION_TOOL_DESCRIPTION,
|
|
122
|
-
parameters: {
|
|
123
|
-
type: "object",
|
|
124
|
-
properties: {
|
|
125
|
-
group: {
|
|
126
|
-
type: "string",
|
|
127
|
-
description: "Motion group from the loaded Live2D model.",
|
|
128
|
-
enum: motionGroups
|
|
129
|
-
},
|
|
130
|
-
index: {
|
|
131
|
-
type: "number",
|
|
132
|
-
description: "Zero-based motion index within the selected motion group."
|
|
133
|
-
}
|
|
134
|
-
},
|
|
135
|
-
required: ["group", "index"]
|
|
136
|
-
}
|
|
137
|
-
},
|
|
138
|
-
async handler(args) {
|
|
139
|
-
const group = args.group;
|
|
140
|
-
const index = args.index;
|
|
141
|
-
if (typeof group !== "string" || !motionGroups.includes(group)) {
|
|
142
|
-
throw new Error(
|
|
143
|
-
'playMotion requires a valid "group" from the model catalog'
|
|
144
|
-
);
|
|
145
|
-
}
|
|
146
|
-
if (!Number.isInteger(index)) {
|
|
147
|
-
throw new Error('playMotion requires an integer "index"');
|
|
148
|
-
}
|
|
149
|
-
const count = catalog.motions[group] ?? 0;
|
|
150
|
-
const motionIndex = index;
|
|
151
|
-
if (motionIndex < 0 || motionIndex >= count) {
|
|
152
|
-
throw new Error(
|
|
153
|
-
`playMotion index ${String(motionIndex)} is out of range for group "${group}"`
|
|
154
|
-
);
|
|
155
|
-
}
|
|
156
|
-
return {
|
|
157
|
-
success: true,
|
|
158
|
-
group,
|
|
159
|
-
index: motionIndex
|
|
160
|
-
};
|
|
161
|
-
}
|
|
162
|
-
});
|
|
163
|
-
}
|
|
164
|
-
tools.push({
|
|
165
|
-
definition: {
|
|
166
|
-
type: "function",
|
|
167
|
-
name: LOOK_AT_TOOL_NAME,
|
|
168
|
-
description: LOOK_AT_TOOL_DESCRIPTION,
|
|
169
|
-
parameters: {
|
|
170
|
-
type: "object",
|
|
171
|
-
properties: {
|
|
172
|
-
x: {
|
|
173
|
-
type: "number",
|
|
174
|
-
description: "Normalized horizontal gaze target. -1 looks left, 0 is center, 1 looks right."
|
|
175
|
-
},
|
|
176
|
-
y: {
|
|
177
|
-
type: "number",
|
|
178
|
-
description: "Normalized vertical gaze target. -1 looks down, 0 is center, 1 looks up."
|
|
179
|
-
}
|
|
180
|
-
},
|
|
181
|
-
required: ["x", "y"]
|
|
182
|
-
}
|
|
183
|
-
},
|
|
184
|
-
async handler(args) {
|
|
185
|
-
const x = args.x;
|
|
186
|
-
const y = args.y;
|
|
187
|
-
if (typeof x !== "number" || Number.isNaN(x)) {
|
|
188
|
-
throw new Error('lookAt requires a numeric "x"');
|
|
189
|
-
}
|
|
190
|
-
if (typeof y !== "number" || Number.isNaN(y)) {
|
|
191
|
-
throw new Error('lookAt requires a numeric "y"');
|
|
192
|
-
}
|
|
193
|
-
return {
|
|
194
|
-
success: true,
|
|
195
|
-
x: clamp(x, MIN_GAZE, MAX_GAZE),
|
|
196
|
-
y: clamp(y, MIN_GAZE, MAX_GAZE)
|
|
197
|
-
};
|
|
198
|
-
}
|
|
199
|
-
});
|
|
200
|
-
return tools;
|
|
201
|
-
}
|
|
202
|
-
function clamp(value, min, max) {
|
|
203
|
-
return Math.max(min, Math.min(max, value));
|
|
204
|
-
}
|
|
205
|
-
|
|
206
62
|
// src/realtime-manager.ts
|
|
207
63
|
var DEFAULT_TOOL_TIMEOUT_MS = 1e4;
|
|
208
64
|
var RECONNECT_DELAYS_MS = [500, 1e3, 2e3, 4e3, 5e3];
|
|
@@ -297,7 +153,7 @@ var RealtimeManagerImpl = class {
|
|
|
297
153
|
}
|
|
298
154
|
async startSession(config) {
|
|
299
155
|
if (this.state.session.status === "active" || this.state.session.status === "starting" || this.refreshInFlight || this.reconnectInFlight) {
|
|
300
|
-
throw new
|
|
156
|
+
throw new core.CharivoStateError("Realtime session already active");
|
|
301
157
|
}
|
|
302
158
|
this.sessionBaseConfig = this.resolveNextSessionBaseConfig(config);
|
|
303
159
|
const sessionConfig = this.buildEffectiveSessionConfig();
|
|
@@ -323,10 +179,9 @@ var RealtimeManagerImpl = class {
|
|
|
323
179
|
void this.requestWakeLock();
|
|
324
180
|
this.emitSessionStart("user");
|
|
325
181
|
} catch (error) {
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
throw error;
|
|
182
|
+
const typedError = core.toCharivoError("transport", error);
|
|
183
|
+
this.finalizeFailedSession(typedError);
|
|
184
|
+
throw typedError;
|
|
330
185
|
}
|
|
331
186
|
}
|
|
332
187
|
updateSession(config) {
|
|
@@ -367,41 +222,45 @@ var RealtimeManagerImpl = class {
|
|
|
367
222
|
} catch {
|
|
368
223
|
}
|
|
369
224
|
if (this.state.session.status === "active") {
|
|
370
|
-
await this.performStopSession()
|
|
225
|
+
await this.performStopSession().catch((error) => {
|
|
226
|
+
throw core.toCharivoError("transport", error);
|
|
227
|
+
});
|
|
371
228
|
}
|
|
372
229
|
return;
|
|
373
230
|
}
|
|
374
231
|
if (this.state.session.status !== "active") {
|
|
375
232
|
return;
|
|
376
233
|
}
|
|
377
|
-
await this.performStopSession()
|
|
234
|
+
await this.performStopSession().catch((error) => {
|
|
235
|
+
throw core.toCharivoError("transport", error);
|
|
236
|
+
});
|
|
378
237
|
}
|
|
379
238
|
async sendMessage(text) {
|
|
380
239
|
if (this.state.session.status !== "active") {
|
|
381
|
-
throw new
|
|
240
|
+
throw new core.CharivoStateError("Realtime session not active");
|
|
382
241
|
}
|
|
383
242
|
if (this.state.connection !== "connected") {
|
|
384
|
-
throw new
|
|
243
|
+
throw new core.CharivoStateError("Realtime session is reconnecting");
|
|
385
244
|
}
|
|
386
|
-
await this.client.sendText(text);
|
|
245
|
+
await this.client.sendText(text).catch((error) => Promise.reject(core.toCharivoError("transport", error)));
|
|
387
246
|
}
|
|
388
247
|
async sendAudioChunk(audio) {
|
|
389
248
|
if (this.state.session.status !== "active") {
|
|
390
|
-
throw new
|
|
249
|
+
throw new core.CharivoStateError("Realtime session not active");
|
|
391
250
|
}
|
|
392
251
|
if (this.state.connection !== "connected") {
|
|
393
|
-
throw new
|
|
252
|
+
throw new core.CharivoStateError("Realtime session is reconnecting");
|
|
394
253
|
}
|
|
395
|
-
await this.client.sendAudio(audio);
|
|
254
|
+
await this.client.sendAudio(audio).catch((error) => Promise.reject(core.toCharivoError("transport", error)));
|
|
396
255
|
}
|
|
397
256
|
async interrupt() {
|
|
398
257
|
if (this.state.session.status !== "active") {
|
|
399
|
-
throw new
|
|
258
|
+
throw new core.CharivoStateError("Realtime session not active");
|
|
400
259
|
}
|
|
401
260
|
if (this.state.connection !== "connected") {
|
|
402
|
-
throw new
|
|
261
|
+
throw new core.CharivoStateError("Realtime session is reconnecting");
|
|
403
262
|
}
|
|
404
|
-
await this.client.interrupt();
|
|
263
|
+
await this.client.interrupt().catch((error) => Promise.reject(core.toCharivoError("transport", error)));
|
|
405
264
|
this.state = {
|
|
406
265
|
...this.state,
|
|
407
266
|
response: {
|
|
@@ -475,6 +334,7 @@ var RealtimeManagerImpl = class {
|
|
|
475
334
|
return;
|
|
476
335
|
}
|
|
477
336
|
const text = event.text || this.state.response.text;
|
|
337
|
+
this.emitUsageEvent(event);
|
|
478
338
|
this.state = {
|
|
479
339
|
...this.state,
|
|
480
340
|
response: {
|
|
@@ -549,14 +409,22 @@ var RealtimeManagerImpl = class {
|
|
|
549
409
|
return;
|
|
550
410
|
}
|
|
551
411
|
try {
|
|
412
|
+
this.log("debug", "Realtime tool execution started", {
|
|
413
|
+
name: event.name,
|
|
414
|
+
callId: event.callId
|
|
415
|
+
});
|
|
552
416
|
const output = await this.runToolHandler(tool, event);
|
|
553
417
|
await this.client.sendToolResult(event.callId, output);
|
|
554
|
-
this.postProcessToolResult(event.name, output);
|
|
555
418
|
this.eventEmitter?.emit("realtime:tool:result", {
|
|
556
419
|
name: event.name,
|
|
557
420
|
output,
|
|
558
421
|
callId: event.callId
|
|
559
422
|
});
|
|
423
|
+
this.projectToolResult(event.name, output, event.callId);
|
|
424
|
+
this.log("info", "Realtime tool execution succeeded", {
|
|
425
|
+
name: event.name,
|
|
426
|
+
callId: event.callId
|
|
427
|
+
});
|
|
560
428
|
} catch (error) {
|
|
561
429
|
await this.handleToolExecutionFailure(
|
|
562
430
|
event.name,
|
|
@@ -595,39 +463,33 @@ var RealtimeManagerImpl = class {
|
|
|
595
463
|
);
|
|
596
464
|
}
|
|
597
465
|
}
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
}
|
|
617
|
-
return;
|
|
618
|
-
}
|
|
619
|
-
case LOOK_AT_TOOL_NAME: {
|
|
620
|
-
const coords = readGazeCoordinates(output);
|
|
621
|
-
if (coords) {
|
|
622
|
-
this.eventEmitter?.emit("realtime:gaze", coords);
|
|
623
|
-
}
|
|
624
|
-
return;
|
|
466
|
+
projectToolResult(name, output, callId) {
|
|
467
|
+
for (const projector of this.options.resultProjectors ?? []) {
|
|
468
|
+
try {
|
|
469
|
+
projector({
|
|
470
|
+
name,
|
|
471
|
+
output,
|
|
472
|
+
callId,
|
|
473
|
+
emit: (event, payload) => {
|
|
474
|
+
this.eventEmitter?.emit(event, payload);
|
|
475
|
+
}
|
|
476
|
+
});
|
|
477
|
+
} catch (error) {
|
|
478
|
+
const projectorError = error instanceof Error ? error : new Error(String(error));
|
|
479
|
+
this.log("warn", "Realtime result projector failed", {
|
|
480
|
+
name,
|
|
481
|
+
callId,
|
|
482
|
+
error: projectorError.message
|
|
483
|
+
});
|
|
625
484
|
}
|
|
626
|
-
default:
|
|
627
|
-
return;
|
|
628
485
|
}
|
|
629
486
|
}
|
|
630
487
|
emitToolError(name, error, callId) {
|
|
488
|
+
this.log("warn", "Realtime tool execution failed", {
|
|
489
|
+
name,
|
|
490
|
+
callId,
|
|
491
|
+
error: error.message
|
|
492
|
+
});
|
|
631
493
|
this.eventEmitter?.emit("realtime:tool:error", {
|
|
632
494
|
name,
|
|
633
495
|
error,
|
|
@@ -675,6 +537,10 @@ var RealtimeManagerImpl = class {
|
|
|
675
537
|
};
|
|
676
538
|
}
|
|
677
539
|
this.emitState();
|
|
540
|
+
this.log("warn", "Realtime reconnect started", {
|
|
541
|
+
cause,
|
|
542
|
+
connection: this.state.connection
|
|
543
|
+
});
|
|
678
544
|
this.reconnectInFlight = this.runReconnectLoop(reconnectToken, cause).catch((reconnectError) => {
|
|
679
545
|
this.finalizeFailedSession(reconnectError);
|
|
680
546
|
}).finally(() => {
|
|
@@ -695,6 +561,11 @@ var RealtimeManagerImpl = class {
|
|
|
695
561
|
delayMs,
|
|
696
562
|
cause
|
|
697
563
|
});
|
|
564
|
+
this.log("info", "Realtime reconnect attempt", {
|
|
565
|
+
attempt,
|
|
566
|
+
delayMs,
|
|
567
|
+
cause
|
|
568
|
+
});
|
|
698
569
|
await delay(delayMs);
|
|
699
570
|
if (!this.isReconnectTokenActive(reconnectToken)) {
|
|
700
571
|
return;
|
|
@@ -723,6 +594,11 @@ var RealtimeManagerImpl = class {
|
|
|
723
594
|
totalMs: Date.now() - startedAt,
|
|
724
595
|
cause
|
|
725
596
|
});
|
|
597
|
+
this.log("info", "Realtime reconnect succeeded", {
|
|
598
|
+
attempts: attempt,
|
|
599
|
+
totalMs: Date.now() - startedAt,
|
|
600
|
+
cause
|
|
601
|
+
});
|
|
726
602
|
return;
|
|
727
603
|
} catch (error) {
|
|
728
604
|
lastError = error instanceof Error ? error : new Error(String(error));
|
|
@@ -743,6 +619,12 @@ var RealtimeManagerImpl = class {
|
|
|
743
619
|
cause,
|
|
744
620
|
lastError
|
|
745
621
|
});
|
|
622
|
+
this.log("error", "Realtime reconnect exhausted", {
|
|
623
|
+
attempts: RECONNECT_DELAYS_MS.length,
|
|
624
|
+
totalMs: Date.now() - startedAt,
|
|
625
|
+
cause,
|
|
626
|
+
error: lastError.message
|
|
627
|
+
});
|
|
746
628
|
throw lastError;
|
|
747
629
|
}
|
|
748
630
|
cancelReconnectLoop() {
|
|
@@ -759,14 +641,19 @@ var RealtimeManagerImpl = class {
|
|
|
759
641
|
});
|
|
760
642
|
}
|
|
761
643
|
applyError(error, connection) {
|
|
644
|
+
const typedError = core.toCharivoError("transport", error);
|
|
762
645
|
this.emitAudioEnd();
|
|
763
646
|
this.state = {
|
|
764
647
|
...this.state,
|
|
765
648
|
connection,
|
|
766
|
-
lastError:
|
|
649
|
+
lastError: typedError
|
|
767
650
|
};
|
|
768
|
-
this.eventEmitter?.emit("realtime:error", { error });
|
|
651
|
+
this.eventEmitter?.emit("realtime:error", { error: typedError });
|
|
769
652
|
this.emitState();
|
|
653
|
+
this.log("error", "Realtime transport error surfaced", {
|
|
654
|
+
connection,
|
|
655
|
+
error: typedError.message
|
|
656
|
+
});
|
|
770
657
|
}
|
|
771
658
|
emitAudioStart() {
|
|
772
659
|
if (this.isAudioPlaybackActive) {
|
|
@@ -912,32 +799,52 @@ var RealtimeManagerImpl = class {
|
|
|
912
799
|
lastError: null
|
|
913
800
|
};
|
|
914
801
|
this.emitState();
|
|
802
|
+
this.log("debug", "Realtime session patch started", {
|
|
803
|
+
voice: sessionConfig.voice,
|
|
804
|
+
model: sessionConfig.model
|
|
805
|
+
});
|
|
915
806
|
try {
|
|
916
807
|
await this.client.updateSession(sessionConfig);
|
|
917
808
|
if (this.stopRequestedDuringRefresh) {
|
|
918
809
|
return;
|
|
919
810
|
}
|
|
920
811
|
this.commitPatchedSession(sessionConfig);
|
|
812
|
+
this.log("info", "Realtime session patch succeeded", {
|
|
813
|
+
voice: sessionConfig.voice,
|
|
814
|
+
model: sessionConfig.model
|
|
815
|
+
});
|
|
921
816
|
} catch (error) {
|
|
922
817
|
const refreshError = error instanceof Error ? error : new Error(String(error));
|
|
818
|
+
const typedRefreshError = core.toCharivoError("transport", refreshError);
|
|
923
819
|
if (this.isRecoveringConnection) {
|
|
924
|
-
throw
|
|
820
|
+
throw typedRefreshError;
|
|
925
821
|
}
|
|
926
|
-
if (this.state.lastError !== refreshError) {
|
|
927
|
-
this.applyError(
|
|
822
|
+
if (this.state.lastError !== typedRefreshError && this.state.lastError?.cause !== refreshError) {
|
|
823
|
+
this.applyError(typedRefreshError, "connected");
|
|
928
824
|
}
|
|
929
|
-
|
|
825
|
+
this.log("error", "Realtime session patch failed", {
|
|
826
|
+
error: typedRefreshError.message
|
|
827
|
+
});
|
|
828
|
+
throw typedRefreshError;
|
|
930
829
|
} finally {
|
|
931
830
|
this.isRefreshingSession = false;
|
|
932
831
|
}
|
|
933
832
|
}
|
|
934
833
|
emitSessionStart(reason, state = this.getState()) {
|
|
834
|
+
this.log("info", "Realtime session started", {
|
|
835
|
+
reason,
|
|
836
|
+
connection: state.connection
|
|
837
|
+
});
|
|
935
838
|
this.eventEmitter?.emit("realtime:session:start", {
|
|
936
839
|
state,
|
|
937
840
|
reason
|
|
938
841
|
});
|
|
939
842
|
}
|
|
940
843
|
emitSessionEnd(reason, state = this.getState()) {
|
|
844
|
+
this.log("info", "Realtime session ended", {
|
|
845
|
+
reason,
|
|
846
|
+
connection: state.connection
|
|
847
|
+
});
|
|
941
848
|
this.eventEmitter?.emit("realtime:session:end", {
|
|
942
849
|
state,
|
|
943
850
|
reason
|
|
@@ -965,6 +872,9 @@ var RealtimeManagerImpl = class {
|
|
|
965
872
|
};
|
|
966
873
|
this.eventEmitter?.emit("realtime:error", { error });
|
|
967
874
|
this.emitState();
|
|
875
|
+
this.log("error", "Realtime session failed", {
|
|
876
|
+
error: error.message
|
|
877
|
+
});
|
|
968
878
|
}
|
|
969
879
|
finalizeStoppedSession(emitSessionEnd, reason) {
|
|
970
880
|
this.cancelReconnectLoop();
|
|
@@ -991,15 +901,25 @@ var RealtimeManagerImpl = class {
|
|
|
991
901
|
this.emitSessionEnd(reason);
|
|
992
902
|
}
|
|
993
903
|
}
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
904
|
+
emitUsageEvent(event) {
|
|
905
|
+
if (!event.usage) {
|
|
906
|
+
return;
|
|
907
|
+
}
|
|
908
|
+
const payload = {
|
|
909
|
+
usage: event.usage,
|
|
910
|
+
model: event.model,
|
|
911
|
+
responseId: event.responseId
|
|
912
|
+
};
|
|
913
|
+
this.eventEmitter?.emit("realtime:usage", payload);
|
|
914
|
+
this.log("debug", "Realtime usage reported", {
|
|
915
|
+
model: event.model,
|
|
916
|
+
responseId: event.responseId
|
|
917
|
+
});
|
|
1000
918
|
}
|
|
1001
|
-
|
|
1002
|
-
|
|
919
|
+
log(level, message, context) {
|
|
920
|
+
this.options.logger?.[level]?.(message, context);
|
|
921
|
+
}
|
|
922
|
+
};
|
|
1003
923
|
function createRealtimeManager(client, options) {
|
|
1004
924
|
return new RealtimeManagerImpl(client, options);
|
|
1005
925
|
}
|
|
@@ -1064,14 +984,9 @@ function delay(ms) {
|
|
|
1064
984
|
});
|
|
1065
985
|
}
|
|
1066
986
|
|
|
1067
|
-
exports.AVATAR_CONTROL_TOOL_NAMES = AVATAR_CONTROL_TOOL_NAMES;
|
|
1068
987
|
exports.DEFAULT_REALTIME_AGENT_INSTRUCTIONS = DEFAULT_REALTIME_AGENT_INSTRUCTIONS;
|
|
1069
|
-
exports.LOOK_AT_TOOL_NAME = LOOK_AT_TOOL_NAME;
|
|
1070
|
-
exports.PLAY_MOTION_TOOL_NAME = PLAY_MOTION_TOOL_NAME;
|
|
1071
988
|
exports.RealtimeManagerImpl = RealtimeManagerImpl;
|
|
1072
|
-
exports.SET_EXPRESSION_TOOL_NAME = SET_EXPRESSION_TOOL_NAME;
|
|
1073
989
|
exports.buildRealtimeSessionConfig = buildRealtimeSessionConfig;
|
|
1074
|
-
exports.createAvatarControlTools = createAvatarControlTools;
|
|
1075
990
|
exports.createRealtimeManager = createRealtimeManager;
|
|
1076
991
|
//# sourceMappingURL=index.js.map
|
|
1077
992
|
//# sourceMappingURL=index.js.map
|