@charivo/realtime 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +51 -12
- package/dist/chunk-LBKX5PJ6.mjs +54 -0
- package/dist/chunk-LBKX5PJ6.mjs.map +1 -0
- package/dist/{chunk-J5L3BPMG.mjs → chunk-LOHYIBEP.mjs} +8 -6
- package/dist/chunk-LOHYIBEP.mjs.map +1 -0
- package/dist/{chunk-3O3COMUY.mjs → chunk-M7TPQNHF.mjs} +18 -11
- package/dist/chunk-M7TPQNHF.mjs.map +1 -0
- package/dist/{chunk-ZJZO4QYA.mjs → chunk-RRXU3IC5.mjs} +9 -4
- package/dist/chunk-RRXU3IC5.mjs.map +1 -0
- package/dist/index.d.mts +24 -17
- package/dist/index.d.ts +24 -17
- package/dist/index.js +121 -198
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +117 -179
- package/dist/index.mjs.map +1 -1
- package/dist/openai/index.d.mts +1 -1
- package/dist/openai/index.d.ts +1 -1
- package/dist/openai/index.js +8 -5
- package/dist/openai/index.js.map +1 -1
- package/dist/openai/index.mjs +3 -3
- package/dist/openai-agents/index.d.mts +2 -1
- package/dist/openai-agents/index.d.ts +2 -1
- package/dist/openai-agents/index.js +29 -8
- package/dist/openai-agents/index.js.map +1 -1
- package/dist/openai-agents/index.mjs +3 -3
- package/dist/remote/index.d.mts +1 -1
- package/dist/remote/index.d.ts +1 -1
- package/dist/remote/index.js +34 -10
- package/dist/remote/index.js.map +1 -1
- package/dist/remote/index.mjs +4 -4
- package/dist/{types-bmYuc_Vs.d.mts → types-BRya38bF.d.mts} +3 -0
- package/dist/{types-bmYuc_Vs.d.ts → types-BRya38bF.d.ts} +3 -0
- package/package.json +2 -2
- package/dist/chunk-3O3COMUY.mjs.map +0 -1
- package/dist/chunk-J5L3BPMG.mjs.map +0 -1
- package/dist/chunk-PDODY2T7.mjs +0 -65
- package/dist/chunk-PDODY2T7.mjs.map +0 -1
- package/dist/chunk-ZJZO4QYA.mjs.map +0 -1
package/dist/index.d.mts
CHANGED
|
@@ -1,13 +1,28 @@
|
|
|
1
|
-
import { R as RealtimeTransportClient } from './types-
|
|
2
|
-
export { b as RealtimeClient, a as RealtimeTransportEvent } from './types-
|
|
3
|
-
import { RealtimeSessionConfig, RealtimeToolRegistration, RealtimeManager, CharivoEventEmitter, Character, RealtimeTool, RealtimeState
|
|
4
|
-
export { Character, CharivoEventEmitter, RealtimeConnectionState, RealtimeManager, RealtimeProvider, RealtimeReconnectCause, RealtimeResponseStatus, RealtimeSessionBootstrap, RealtimeSessionConfig, RealtimeSessionRequest, RealtimeSessionStatus, RealtimeSessionTransitionReason, RealtimeState, RealtimeTool, RealtimeToolChoice, RealtimeToolContext, RealtimeToolHandler, RealtimeToolRegistration, RealtimeTransportKind } from '@charivo/core';
|
|
1
|
+
import { R as RealtimeTransportClient } from './types-BRya38bF.mjs';
|
|
2
|
+
export { b as RealtimeClient, a as RealtimeTransportEvent } from './types-BRya38bF.mjs';
|
|
3
|
+
import { RealtimeSessionConfig, RealtimeToolRegistration, EventMap, RealtimeManager, CharivoEventEmitter, Character, RealtimeTool, RealtimeState } from '@charivo/core';
|
|
4
|
+
export { Character, CharivoEventEmitter, RealtimeConnectionState, RealtimeManager, RealtimeProvider, RealtimeReconnectCause, RealtimeResponseStatus, RealtimeSessionBootstrap, RealtimeSessionConfig, RealtimeSessionRequest, RealtimeSessionStatus, RealtimeSessionTransitionReason, RealtimeState, RealtimeTool, RealtimeToolChoice, RealtimeToolContext, RealtimeToolHandler, RealtimeToolRegistration, RealtimeTransportKind, RealtimeUsageEvent } from '@charivo/core';
|
|
5
5
|
|
|
6
6
|
interface RealtimeManagerOptions {
|
|
7
7
|
defaultSessionConfig?: Omit<RealtimeSessionConfig, "tools">;
|
|
8
8
|
tools?: RealtimeToolRegistration[];
|
|
9
9
|
defaultToolTimeoutMs?: number;
|
|
10
|
+
resultProjectors?: RealtimeToolResultProjector[];
|
|
11
|
+
logger?: RealtimeLogger;
|
|
10
12
|
}
|
|
13
|
+
interface RealtimeLogger {
|
|
14
|
+
debug?(message: string, context?: Record<string, unknown>): void;
|
|
15
|
+
info?(message: string, context?: Record<string, unknown>): void;
|
|
16
|
+
warn?(message: string, context?: Record<string, unknown>): void;
|
|
17
|
+
error?(message: string, context?: Record<string, unknown>): void;
|
|
18
|
+
}
|
|
19
|
+
interface RealtimeToolResultProjectorContext {
|
|
20
|
+
name: string;
|
|
21
|
+
output: Record<string, unknown>;
|
|
22
|
+
callId?: string;
|
|
23
|
+
emit<K extends keyof EventMap>(event: K, payload: EventMap[K]): void;
|
|
24
|
+
}
|
|
25
|
+
type RealtimeToolResultProjector = (context: RealtimeToolResultProjectorContext) => void;
|
|
11
26
|
/**
|
|
12
27
|
* Provider-agnostic realtime manager.
|
|
13
28
|
*
|
|
@@ -32,6 +47,7 @@ declare class RealtimeManagerImpl implements RealtimeManager {
|
|
|
32
47
|
private reconnectToken;
|
|
33
48
|
private wakeLockSentinel;
|
|
34
49
|
private teardownBrowserLifecycle?;
|
|
50
|
+
private sessionId?;
|
|
35
51
|
private readonly toolRegistry;
|
|
36
52
|
private state;
|
|
37
53
|
constructor(client: RealtimeTransportClient, options?: RealtimeManagerOptions);
|
|
@@ -51,7 +67,7 @@ declare class RealtimeManagerImpl implements RealtimeManager {
|
|
|
51
67
|
private executeToolCall;
|
|
52
68
|
private runToolHandler;
|
|
53
69
|
private handleToolExecutionFailure;
|
|
54
|
-
private
|
|
70
|
+
private projectToolResult;
|
|
55
71
|
private emitToolError;
|
|
56
72
|
private ensureResponseStarted;
|
|
57
73
|
private beginReconnect;
|
|
@@ -76,20 +92,11 @@ declare class RealtimeManagerImpl implements RealtimeManager {
|
|
|
76
92
|
private emitSessionEnd;
|
|
77
93
|
private finalizeFailedSession;
|
|
78
94
|
private finalizeStoppedSession;
|
|
95
|
+
private emitUsageEvent;
|
|
96
|
+
private log;
|
|
79
97
|
}
|
|
80
98
|
declare function createRealtimeManager(client: RealtimeTransportClient, options?: RealtimeManagerOptions): RealtimeManager;
|
|
81
99
|
|
|
82
|
-
declare const SET_EXPRESSION_TOOL_NAME = "setExpression";
|
|
83
|
-
declare const PLAY_MOTION_TOOL_NAME = "playMotion";
|
|
84
|
-
declare const LOOK_AT_TOOL_NAME = "lookAt";
|
|
85
|
-
declare const AVATAR_CONTROL_TOOL_NAMES: readonly ["setExpression", "playMotion", "lookAt"];
|
|
86
|
-
interface ExpressionArgs {
|
|
87
|
-
expressionId: string;
|
|
88
|
-
}
|
|
89
|
-
type MotionArgs = MotionSelection;
|
|
90
|
-
type LookAtArgs = GazeCoordinates;
|
|
91
|
-
declare function createAvatarControlTools(catalog: AvatarControlCatalog): RealtimeToolRegistration[];
|
|
92
|
-
|
|
93
100
|
declare const DEFAULT_REALTIME_AGENT_INSTRUCTIONS: string;
|
|
94
101
|
interface BuildRealtimeSessionConfigOptions {
|
|
95
102
|
character?: Character | null;
|
|
@@ -97,4 +104,4 @@ interface BuildRealtimeSessionConfigOptions {
|
|
|
97
104
|
}
|
|
98
105
|
declare function buildRealtimeSessionConfig({ character, baseConfig, }?: BuildRealtimeSessionConfigOptions): RealtimeSessionConfig;
|
|
99
106
|
|
|
100
|
-
export {
|
|
107
|
+
export { type BuildRealtimeSessionConfigOptions, DEFAULT_REALTIME_AGENT_INSTRUCTIONS, type RealtimeLogger, RealtimeManagerImpl, type RealtimeManagerOptions, type RealtimeToolResultProjector, type RealtimeToolResultProjectorContext, RealtimeTransportClient, buildRealtimeSessionConfig, createRealtimeManager };
|
package/dist/index.d.ts
CHANGED
|
@@ -1,13 +1,28 @@
|
|
|
1
|
-
import { R as RealtimeTransportClient } from './types-
|
|
2
|
-
export { b as RealtimeClient, a as RealtimeTransportEvent } from './types-
|
|
3
|
-
import { RealtimeSessionConfig, RealtimeToolRegistration, RealtimeManager, CharivoEventEmitter, Character, RealtimeTool, RealtimeState
|
|
4
|
-
export { Character, CharivoEventEmitter, RealtimeConnectionState, RealtimeManager, RealtimeProvider, RealtimeReconnectCause, RealtimeResponseStatus, RealtimeSessionBootstrap, RealtimeSessionConfig, RealtimeSessionRequest, RealtimeSessionStatus, RealtimeSessionTransitionReason, RealtimeState, RealtimeTool, RealtimeToolChoice, RealtimeToolContext, RealtimeToolHandler, RealtimeToolRegistration, RealtimeTransportKind } from '@charivo/core';
|
|
1
|
+
import { R as RealtimeTransportClient } from './types-BRya38bF.js';
|
|
2
|
+
export { b as RealtimeClient, a as RealtimeTransportEvent } from './types-BRya38bF.js';
|
|
3
|
+
import { RealtimeSessionConfig, RealtimeToolRegistration, EventMap, RealtimeManager, CharivoEventEmitter, Character, RealtimeTool, RealtimeState } from '@charivo/core';
|
|
4
|
+
export { Character, CharivoEventEmitter, RealtimeConnectionState, RealtimeManager, RealtimeProvider, RealtimeReconnectCause, RealtimeResponseStatus, RealtimeSessionBootstrap, RealtimeSessionConfig, RealtimeSessionRequest, RealtimeSessionStatus, RealtimeSessionTransitionReason, RealtimeState, RealtimeTool, RealtimeToolChoice, RealtimeToolContext, RealtimeToolHandler, RealtimeToolRegistration, RealtimeTransportKind, RealtimeUsageEvent } from '@charivo/core';
|
|
5
5
|
|
|
6
6
|
interface RealtimeManagerOptions {
|
|
7
7
|
defaultSessionConfig?: Omit<RealtimeSessionConfig, "tools">;
|
|
8
8
|
tools?: RealtimeToolRegistration[];
|
|
9
9
|
defaultToolTimeoutMs?: number;
|
|
10
|
+
resultProjectors?: RealtimeToolResultProjector[];
|
|
11
|
+
logger?: RealtimeLogger;
|
|
10
12
|
}
|
|
13
|
+
interface RealtimeLogger {
|
|
14
|
+
debug?(message: string, context?: Record<string, unknown>): void;
|
|
15
|
+
info?(message: string, context?: Record<string, unknown>): void;
|
|
16
|
+
warn?(message: string, context?: Record<string, unknown>): void;
|
|
17
|
+
error?(message: string, context?: Record<string, unknown>): void;
|
|
18
|
+
}
|
|
19
|
+
interface RealtimeToolResultProjectorContext {
|
|
20
|
+
name: string;
|
|
21
|
+
output: Record<string, unknown>;
|
|
22
|
+
callId?: string;
|
|
23
|
+
emit<K extends keyof EventMap>(event: K, payload: EventMap[K]): void;
|
|
24
|
+
}
|
|
25
|
+
type RealtimeToolResultProjector = (context: RealtimeToolResultProjectorContext) => void;
|
|
11
26
|
/**
|
|
12
27
|
* Provider-agnostic realtime manager.
|
|
13
28
|
*
|
|
@@ -32,6 +47,7 @@ declare class RealtimeManagerImpl implements RealtimeManager {
|
|
|
32
47
|
private reconnectToken;
|
|
33
48
|
private wakeLockSentinel;
|
|
34
49
|
private teardownBrowserLifecycle?;
|
|
50
|
+
private sessionId?;
|
|
35
51
|
private readonly toolRegistry;
|
|
36
52
|
private state;
|
|
37
53
|
constructor(client: RealtimeTransportClient, options?: RealtimeManagerOptions);
|
|
@@ -51,7 +67,7 @@ declare class RealtimeManagerImpl implements RealtimeManager {
|
|
|
51
67
|
private executeToolCall;
|
|
52
68
|
private runToolHandler;
|
|
53
69
|
private handleToolExecutionFailure;
|
|
54
|
-
private
|
|
70
|
+
private projectToolResult;
|
|
55
71
|
private emitToolError;
|
|
56
72
|
private ensureResponseStarted;
|
|
57
73
|
private beginReconnect;
|
|
@@ -76,20 +92,11 @@ declare class RealtimeManagerImpl implements RealtimeManager {
|
|
|
76
92
|
private emitSessionEnd;
|
|
77
93
|
private finalizeFailedSession;
|
|
78
94
|
private finalizeStoppedSession;
|
|
95
|
+
private emitUsageEvent;
|
|
96
|
+
private log;
|
|
79
97
|
}
|
|
80
98
|
declare function createRealtimeManager(client: RealtimeTransportClient, options?: RealtimeManagerOptions): RealtimeManager;
|
|
81
99
|
|
|
82
|
-
declare const SET_EXPRESSION_TOOL_NAME = "setExpression";
|
|
83
|
-
declare const PLAY_MOTION_TOOL_NAME = "playMotion";
|
|
84
|
-
declare const LOOK_AT_TOOL_NAME = "lookAt";
|
|
85
|
-
declare const AVATAR_CONTROL_TOOL_NAMES: readonly ["setExpression", "playMotion", "lookAt"];
|
|
86
|
-
interface ExpressionArgs {
|
|
87
|
-
expressionId: string;
|
|
88
|
-
}
|
|
89
|
-
type MotionArgs = MotionSelection;
|
|
90
|
-
type LookAtArgs = GazeCoordinates;
|
|
91
|
-
declare function createAvatarControlTools(catalog: AvatarControlCatalog): RealtimeToolRegistration[];
|
|
92
|
-
|
|
93
100
|
declare const DEFAULT_REALTIME_AGENT_INSTRUCTIONS: string;
|
|
94
101
|
interface BuildRealtimeSessionConfigOptions {
|
|
95
102
|
character?: Character | null;
|
|
@@ -97,4 +104,4 @@ interface BuildRealtimeSessionConfigOptions {
|
|
|
97
104
|
}
|
|
98
105
|
declare function buildRealtimeSessionConfig({ character, baseConfig, }?: BuildRealtimeSessionConfigOptions): RealtimeSessionConfig;
|
|
99
106
|
|
|
100
|
-
export {
|
|
107
|
+
export { type BuildRealtimeSessionConfigOptions, DEFAULT_REALTIME_AGENT_INSTRUCTIONS, type RealtimeLogger, RealtimeManagerImpl, type RealtimeManagerOptions, type RealtimeToolResultProjector, type RealtimeToolResultProjectorContext, RealtimeTransportClient, buildRealtimeSessionConfig, createRealtimeManager };
|
package/dist/index.js
CHANGED
|
@@ -5,35 +5,24 @@ var core = require('@charivo/core');
|
|
|
5
5
|
// src/realtime-manager.ts
|
|
6
6
|
|
|
7
7
|
// src/instructions.ts
|
|
8
|
-
var DEFAULT_PROVIDER = "openai";
|
|
9
|
-
var DEFAULT_MODEL = "gpt-realtime-mini";
|
|
10
|
-
var DEFAULT_REALTIME_VOICE = "marin";
|
|
11
|
-
var SET_EXPRESSION_TOOL_DESCRIPTION = "Change the avatar's facial expression only when the emotional beat clearly shifts or should linger across the reply. Do not use this for every polite or lightweight reaction.";
|
|
12
|
-
var PLAY_MOTION_TOOL_DESCRIPTION = "Play a noticeable body motion for greetings, emphasis, or bigger reaction beats. Prefer this over stacking multiple smaller actions when the moment needs one clear accent. Usually use at most one motion in a reply.";
|
|
13
|
-
var LOOK_AT_TOOL_DESCRIPTION = `Shift the avatar's gaze for subtle attention changes or conversational focus. Trigger this on natural phrases like "glance", "look over", "peek at", or directional cues. Prefer this before "setExpression" when a lightweight reaction is enough.`;
|
|
14
8
|
var DEFAULT_REALTIME_AGENT_INSTRUCTIONS = `
|
|
15
|
-
You are speaking in a realtime voice conversation
|
|
9
|
+
You are speaking in a realtime voice conversation.
|
|
16
10
|
Respond naturally and keep replies concise enough for spoken delivery.
|
|
17
|
-
Use
|
|
18
|
-
|
|
19
|
-
Do not use "setExpression" for every friendly, polite, or mildly positive reply.
|
|
20
|
-
Use "playMotion" for greetings, emphasis, or bigger reaction beats, usually at most once in a reply.
|
|
21
|
-
Use "lookAt" for subtle attention shifts or conversational focus, and prefer it before "setExpression" when a lightweight reaction is enough.
|
|
22
|
-
Prefer a single meaningful avatar action over chaining multiple actions on simple replies.
|
|
23
|
-
Many turns should use no avatar tool at all.
|
|
11
|
+
Use available tools proactively when they make the conversation clearer or more present.
|
|
12
|
+
Prefer one well-timed tool action over chaining multiple actions on simple replies.
|
|
24
13
|
Never break character. Never refer to yourself as an AI, model, or assistant.
|
|
25
14
|
Do not mention tool calls in the spoken response.
|
|
26
|
-
Speak only the words you want heard aloud. Use
|
|
15
|
+
Speak only the words you want heard aloud. Use tools for actions. Never say tool names or tool arguments out loud. Do not output bracketed or parenthetical action notes. If a tool is unavailable, omit the action entirely rather than narrating it.
|
|
27
16
|
`.trim();
|
|
28
17
|
function buildRealtimeSessionConfig({
|
|
29
18
|
character,
|
|
30
19
|
baseConfig
|
|
31
20
|
} = {}) {
|
|
32
21
|
return {
|
|
33
|
-
provider: baseConfig?.provider ?? DEFAULT_PROVIDER,
|
|
34
22
|
transport: baseConfig?.transport ?? "webrtc",
|
|
35
|
-
|
|
36
|
-
|
|
23
|
+
provider: baseConfig?.provider,
|
|
24
|
+
model: baseConfig?.model,
|
|
25
|
+
voice: baseConfig?.voice ?? character?.voice?.voiceId,
|
|
37
26
|
instructions: baseConfig?.instructions ?? buildCharacterInstructions(character),
|
|
38
27
|
temperature: baseConfig?.temperature,
|
|
39
28
|
maxTokens: baseConfig?.maxTokens,
|
|
@@ -66,146 +55,10 @@ function ensureSentenceFragment(text) {
|
|
|
66
55
|
return text.trim().replace(/[.!?]+$/, "");
|
|
67
56
|
}
|
|
68
57
|
|
|
69
|
-
// src/tools.ts
|
|
70
|
-
var MIN_GAZE = -1;
|
|
71
|
-
var MAX_GAZE = 1;
|
|
72
|
-
var SET_EXPRESSION_TOOL_NAME = "setExpression";
|
|
73
|
-
var PLAY_MOTION_TOOL_NAME = "playMotion";
|
|
74
|
-
var LOOK_AT_TOOL_NAME = "lookAt";
|
|
75
|
-
var AVATAR_CONTROL_TOOL_NAMES = [
|
|
76
|
-
SET_EXPRESSION_TOOL_NAME,
|
|
77
|
-
PLAY_MOTION_TOOL_NAME,
|
|
78
|
-
LOOK_AT_TOOL_NAME
|
|
79
|
-
];
|
|
80
|
-
function createAvatarControlTools(catalog) {
|
|
81
|
-
const tools = [];
|
|
82
|
-
if (catalog.expressions.length > 0) {
|
|
83
|
-
const expressionValues = [...catalog.expressions];
|
|
84
|
-
tools.push({
|
|
85
|
-
definition: {
|
|
86
|
-
type: "function",
|
|
87
|
-
name: SET_EXPRESSION_TOOL_NAME,
|
|
88
|
-
description: SET_EXPRESSION_TOOL_DESCRIPTION,
|
|
89
|
-
parameters: {
|
|
90
|
-
type: "object",
|
|
91
|
-
properties: {
|
|
92
|
-
expressionId: {
|
|
93
|
-
type: "string",
|
|
94
|
-
description: "Expression ID from the loaded Live2D model.",
|
|
95
|
-
enum: expressionValues
|
|
96
|
-
}
|
|
97
|
-
},
|
|
98
|
-
required: ["expressionId"]
|
|
99
|
-
}
|
|
100
|
-
},
|
|
101
|
-
async handler(args) {
|
|
102
|
-
const expressionId = args.expressionId;
|
|
103
|
-
if (typeof expressionId !== "string" || !expressionValues.includes(expressionId)) {
|
|
104
|
-
throw new Error(
|
|
105
|
-
'setExpression requires a valid "expressionId" from the model catalog'
|
|
106
|
-
);
|
|
107
|
-
}
|
|
108
|
-
return {
|
|
109
|
-
success: true,
|
|
110
|
-
expressionId
|
|
111
|
-
};
|
|
112
|
-
}
|
|
113
|
-
});
|
|
114
|
-
}
|
|
115
|
-
const motionGroups = Object.keys(catalog.motions);
|
|
116
|
-
if (motionGroups.length > 0) {
|
|
117
|
-
tools.push({
|
|
118
|
-
definition: {
|
|
119
|
-
type: "function",
|
|
120
|
-
name: PLAY_MOTION_TOOL_NAME,
|
|
121
|
-
description: PLAY_MOTION_TOOL_DESCRIPTION,
|
|
122
|
-
parameters: {
|
|
123
|
-
type: "object",
|
|
124
|
-
properties: {
|
|
125
|
-
group: {
|
|
126
|
-
type: "string",
|
|
127
|
-
description: "Motion group from the loaded Live2D model.",
|
|
128
|
-
enum: motionGroups
|
|
129
|
-
},
|
|
130
|
-
index: {
|
|
131
|
-
type: "number",
|
|
132
|
-
description: "Zero-based motion index within the selected motion group."
|
|
133
|
-
}
|
|
134
|
-
},
|
|
135
|
-
required: ["group", "index"]
|
|
136
|
-
}
|
|
137
|
-
},
|
|
138
|
-
async handler(args) {
|
|
139
|
-
const group = args.group;
|
|
140
|
-
const index = args.index;
|
|
141
|
-
if (typeof group !== "string" || !motionGroups.includes(group)) {
|
|
142
|
-
throw new Error(
|
|
143
|
-
'playMotion requires a valid "group" from the model catalog'
|
|
144
|
-
);
|
|
145
|
-
}
|
|
146
|
-
if (!Number.isInteger(index)) {
|
|
147
|
-
throw new Error('playMotion requires an integer "index"');
|
|
148
|
-
}
|
|
149
|
-
const count = catalog.motions[group] ?? 0;
|
|
150
|
-
const motionIndex = index;
|
|
151
|
-
if (motionIndex < 0 || motionIndex >= count) {
|
|
152
|
-
throw new Error(
|
|
153
|
-
`playMotion index ${String(motionIndex)} is out of range for group "${group}"`
|
|
154
|
-
);
|
|
155
|
-
}
|
|
156
|
-
return {
|
|
157
|
-
success: true,
|
|
158
|
-
group,
|
|
159
|
-
index: motionIndex
|
|
160
|
-
};
|
|
161
|
-
}
|
|
162
|
-
});
|
|
163
|
-
}
|
|
164
|
-
tools.push({
|
|
165
|
-
definition: {
|
|
166
|
-
type: "function",
|
|
167
|
-
name: LOOK_AT_TOOL_NAME,
|
|
168
|
-
description: LOOK_AT_TOOL_DESCRIPTION,
|
|
169
|
-
parameters: {
|
|
170
|
-
type: "object",
|
|
171
|
-
properties: {
|
|
172
|
-
x: {
|
|
173
|
-
type: "number",
|
|
174
|
-
description: "Normalized horizontal gaze target. -1 looks left, 0 is center, 1 looks right."
|
|
175
|
-
},
|
|
176
|
-
y: {
|
|
177
|
-
type: "number",
|
|
178
|
-
description: "Normalized vertical gaze target. -1 looks down, 0 is center, 1 looks up."
|
|
179
|
-
}
|
|
180
|
-
},
|
|
181
|
-
required: ["x", "y"]
|
|
182
|
-
}
|
|
183
|
-
},
|
|
184
|
-
async handler(args) {
|
|
185
|
-
const x = args.x;
|
|
186
|
-
const y = args.y;
|
|
187
|
-
if (typeof x !== "number" || Number.isNaN(x)) {
|
|
188
|
-
throw new Error('lookAt requires a numeric "x"');
|
|
189
|
-
}
|
|
190
|
-
if (typeof y !== "number" || Number.isNaN(y)) {
|
|
191
|
-
throw new Error('lookAt requires a numeric "y"');
|
|
192
|
-
}
|
|
193
|
-
return {
|
|
194
|
-
success: true,
|
|
195
|
-
x: clamp(x, MIN_GAZE, MAX_GAZE),
|
|
196
|
-
y: clamp(y, MIN_GAZE, MAX_GAZE)
|
|
197
|
-
};
|
|
198
|
-
}
|
|
199
|
-
});
|
|
200
|
-
return tools;
|
|
201
|
-
}
|
|
202
|
-
function clamp(value, min, max) {
|
|
203
|
-
return Math.max(min, Math.min(max, value));
|
|
204
|
-
}
|
|
205
|
-
|
|
206
58
|
// src/realtime-manager.ts
|
|
207
59
|
var DEFAULT_TOOL_TIMEOUT_MS = 1e4;
|
|
208
60
|
var RECONNECT_DELAYS_MS = [500, 1e3, 2e3, 4e3, 5e3];
|
|
61
|
+
var nextFallbackSessionId = 1;
|
|
209
62
|
var RealtimeManagerImpl = class {
|
|
210
63
|
constructor(client, options = {}) {
|
|
211
64
|
this.client = client;
|
|
@@ -237,6 +90,7 @@ var RealtimeManagerImpl = class {
|
|
|
237
90
|
reconnectToken = 0;
|
|
238
91
|
wakeLockSentinel = null;
|
|
239
92
|
teardownBrowserLifecycle;
|
|
93
|
+
sessionId;
|
|
240
94
|
toolRegistry = /* @__PURE__ */ new Map();
|
|
241
95
|
state = {
|
|
242
96
|
connection: "idle",
|
|
@@ -301,6 +155,7 @@ var RealtimeManagerImpl = class {
|
|
|
301
155
|
}
|
|
302
156
|
this.sessionBaseConfig = this.resolveNextSessionBaseConfig(config);
|
|
303
157
|
const sessionConfig = this.buildEffectiveSessionConfig();
|
|
158
|
+
this.sessionId = createRealtimeSessionId();
|
|
304
159
|
this.state = {
|
|
305
160
|
...this.state,
|
|
306
161
|
connection: "connecting",
|
|
@@ -478,6 +333,7 @@ var RealtimeManagerImpl = class {
|
|
|
478
333
|
return;
|
|
479
334
|
}
|
|
480
335
|
const text = event.text || this.state.response.text;
|
|
336
|
+
this.emitUsageEvent(event);
|
|
481
337
|
this.state = {
|
|
482
338
|
...this.state,
|
|
483
339
|
response: {
|
|
@@ -552,14 +408,22 @@ var RealtimeManagerImpl = class {
|
|
|
552
408
|
return;
|
|
553
409
|
}
|
|
554
410
|
try {
|
|
411
|
+
this.log("debug", "Realtime tool execution started", {
|
|
412
|
+
name: event.name,
|
|
413
|
+
callId: event.callId
|
|
414
|
+
});
|
|
555
415
|
const output = await this.runToolHandler(tool, event);
|
|
556
416
|
await this.client.sendToolResult(event.callId, output);
|
|
557
|
-
this.postProcessToolResult(event.name, output);
|
|
558
417
|
this.eventEmitter?.emit("realtime:tool:result", {
|
|
559
418
|
name: event.name,
|
|
560
419
|
output,
|
|
561
420
|
callId: event.callId
|
|
562
421
|
});
|
|
422
|
+
this.projectToolResult(event.name, output, event.callId);
|
|
423
|
+
this.log("info", "Realtime tool execution succeeded", {
|
|
424
|
+
name: event.name,
|
|
425
|
+
callId: event.callId
|
|
426
|
+
});
|
|
563
427
|
} catch (error) {
|
|
564
428
|
await this.handleToolExecutionFailure(
|
|
565
429
|
event.name,
|
|
@@ -598,39 +462,33 @@ var RealtimeManagerImpl = class {
|
|
|
598
462
|
);
|
|
599
463
|
}
|
|
600
464
|
}
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
}
|
|
620
|
-
return;
|
|
621
|
-
}
|
|
622
|
-
case LOOK_AT_TOOL_NAME: {
|
|
623
|
-
const coords = readGazeCoordinates(output);
|
|
624
|
-
if (coords) {
|
|
625
|
-
this.eventEmitter?.emit("realtime:gaze", coords);
|
|
626
|
-
}
|
|
627
|
-
return;
|
|
465
|
+
projectToolResult(name, output, callId) {
|
|
466
|
+
for (const projector of this.options.resultProjectors ?? []) {
|
|
467
|
+
try {
|
|
468
|
+
projector({
|
|
469
|
+
name,
|
|
470
|
+
output,
|
|
471
|
+
callId,
|
|
472
|
+
emit: (event, payload) => {
|
|
473
|
+
this.eventEmitter?.emit(event, payload);
|
|
474
|
+
}
|
|
475
|
+
});
|
|
476
|
+
} catch (error) {
|
|
477
|
+
const projectorError = error instanceof Error ? error : new Error(String(error));
|
|
478
|
+
this.log("warn", "Realtime result projector failed", {
|
|
479
|
+
name,
|
|
480
|
+
callId,
|
|
481
|
+
error: projectorError.message
|
|
482
|
+
});
|
|
628
483
|
}
|
|
629
|
-
default:
|
|
630
|
-
return;
|
|
631
484
|
}
|
|
632
485
|
}
|
|
633
486
|
emitToolError(name, error, callId) {
|
|
487
|
+
this.log("warn", "Realtime tool execution failed", {
|
|
488
|
+
name,
|
|
489
|
+
callId,
|
|
490
|
+
error: error.message
|
|
491
|
+
});
|
|
634
492
|
this.eventEmitter?.emit("realtime:tool:error", {
|
|
635
493
|
name,
|
|
636
494
|
error,
|
|
@@ -678,6 +536,10 @@ var RealtimeManagerImpl = class {
|
|
|
678
536
|
};
|
|
679
537
|
}
|
|
680
538
|
this.emitState();
|
|
539
|
+
this.log("warn", "Realtime reconnect started", {
|
|
540
|
+
cause,
|
|
541
|
+
connection: this.state.connection
|
|
542
|
+
});
|
|
681
543
|
this.reconnectInFlight = this.runReconnectLoop(reconnectToken, cause).catch((reconnectError) => {
|
|
682
544
|
this.finalizeFailedSession(reconnectError);
|
|
683
545
|
}).finally(() => {
|
|
@@ -698,6 +560,11 @@ var RealtimeManagerImpl = class {
|
|
|
698
560
|
delayMs,
|
|
699
561
|
cause
|
|
700
562
|
});
|
|
563
|
+
this.log("info", "Realtime reconnect attempt", {
|
|
564
|
+
attempt,
|
|
565
|
+
delayMs,
|
|
566
|
+
cause
|
|
567
|
+
});
|
|
701
568
|
await delay(delayMs);
|
|
702
569
|
if (!this.isReconnectTokenActive(reconnectToken)) {
|
|
703
570
|
return;
|
|
@@ -726,6 +593,11 @@ var RealtimeManagerImpl = class {
|
|
|
726
593
|
totalMs: Date.now() - startedAt,
|
|
727
594
|
cause
|
|
728
595
|
});
|
|
596
|
+
this.log("info", "Realtime reconnect succeeded", {
|
|
597
|
+
attempts: attempt,
|
|
598
|
+
totalMs: Date.now() - startedAt,
|
|
599
|
+
cause
|
|
600
|
+
});
|
|
729
601
|
return;
|
|
730
602
|
} catch (error) {
|
|
731
603
|
lastError = error instanceof Error ? error : new Error(String(error));
|
|
@@ -746,6 +618,12 @@ var RealtimeManagerImpl = class {
|
|
|
746
618
|
cause,
|
|
747
619
|
lastError
|
|
748
620
|
});
|
|
621
|
+
this.log("error", "Realtime reconnect exhausted", {
|
|
622
|
+
attempts: RECONNECT_DELAYS_MS.length,
|
|
623
|
+
totalMs: Date.now() - startedAt,
|
|
624
|
+
cause,
|
|
625
|
+
error: lastError.message
|
|
626
|
+
});
|
|
749
627
|
throw lastError;
|
|
750
628
|
}
|
|
751
629
|
cancelReconnectLoop() {
|
|
@@ -771,6 +649,10 @@ var RealtimeManagerImpl = class {
|
|
|
771
649
|
};
|
|
772
650
|
this.eventEmitter?.emit("realtime:error", { error: typedError });
|
|
773
651
|
this.emitState();
|
|
652
|
+
this.log("error", "Realtime transport error surfaced", {
|
|
653
|
+
connection,
|
|
654
|
+
error: typedError.message
|
|
655
|
+
});
|
|
774
656
|
}
|
|
775
657
|
emitAudioStart() {
|
|
776
658
|
if (this.isAudioPlaybackActive) {
|
|
@@ -916,12 +798,20 @@ var RealtimeManagerImpl = class {
|
|
|
916
798
|
lastError: null
|
|
917
799
|
};
|
|
918
800
|
this.emitState();
|
|
801
|
+
this.log("debug", "Realtime session patch started", {
|
|
802
|
+
voice: sessionConfig.voice,
|
|
803
|
+
model: sessionConfig.model
|
|
804
|
+
});
|
|
919
805
|
try {
|
|
920
806
|
await this.client.updateSession(sessionConfig);
|
|
921
807
|
if (this.stopRequestedDuringRefresh) {
|
|
922
808
|
return;
|
|
923
809
|
}
|
|
924
810
|
this.commitPatchedSession(sessionConfig);
|
|
811
|
+
this.log("info", "Realtime session patch succeeded", {
|
|
812
|
+
voice: sessionConfig.voice,
|
|
813
|
+
model: sessionConfig.model
|
|
814
|
+
});
|
|
925
815
|
} catch (error) {
|
|
926
816
|
const refreshError = error instanceof Error ? error : new Error(String(error));
|
|
927
817
|
const typedRefreshError = core.toCharivoError("transport", refreshError);
|
|
@@ -931,18 +821,29 @@ var RealtimeManagerImpl = class {
|
|
|
931
821
|
if (this.state.lastError !== typedRefreshError && this.state.lastError?.cause !== refreshError) {
|
|
932
822
|
this.applyError(typedRefreshError, "connected");
|
|
933
823
|
}
|
|
824
|
+
this.log("error", "Realtime session patch failed", {
|
|
825
|
+
error: typedRefreshError.message
|
|
826
|
+
});
|
|
934
827
|
throw typedRefreshError;
|
|
935
828
|
} finally {
|
|
936
829
|
this.isRefreshingSession = false;
|
|
937
830
|
}
|
|
938
831
|
}
|
|
939
832
|
emitSessionStart(reason, state = this.getState()) {
|
|
833
|
+
this.log("info", "Realtime session started", {
|
|
834
|
+
reason,
|
|
835
|
+
connection: state.connection
|
|
836
|
+
});
|
|
940
837
|
this.eventEmitter?.emit("realtime:session:start", {
|
|
941
838
|
state,
|
|
942
839
|
reason
|
|
943
840
|
});
|
|
944
841
|
}
|
|
945
842
|
emitSessionEnd(reason, state = this.getState()) {
|
|
843
|
+
this.log("info", "Realtime session ended", {
|
|
844
|
+
reason,
|
|
845
|
+
connection: state.connection
|
|
846
|
+
});
|
|
946
847
|
this.eventEmitter?.emit("realtime:session:end", {
|
|
947
848
|
state,
|
|
948
849
|
reason
|
|
@@ -970,6 +871,10 @@ var RealtimeManagerImpl = class {
|
|
|
970
871
|
};
|
|
971
872
|
this.eventEmitter?.emit("realtime:error", { error });
|
|
972
873
|
this.emitState();
|
|
874
|
+
this.log("error", "Realtime session failed", {
|
|
875
|
+
error: error.message
|
|
876
|
+
});
|
|
877
|
+
this.sessionId = void 0;
|
|
973
878
|
}
|
|
974
879
|
finalizeStoppedSession(emitSessionEnd, reason) {
|
|
975
880
|
this.cancelReconnectLoop();
|
|
@@ -995,16 +900,32 @@ var RealtimeManagerImpl = class {
|
|
|
995
900
|
if (emitSessionEnd) {
|
|
996
901
|
this.emitSessionEnd(reason);
|
|
997
902
|
}
|
|
903
|
+
this.sessionId = void 0;
|
|
998
904
|
}
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
905
|
+
emitUsageEvent(event) {
|
|
906
|
+
if (!event.usage) {
|
|
907
|
+
return;
|
|
908
|
+
}
|
|
909
|
+
const payload = {
|
|
910
|
+
usage: event.usage,
|
|
911
|
+
model: event.model,
|
|
912
|
+
responseId: event.responseId,
|
|
913
|
+
sessionId: this.sessionId
|
|
914
|
+
};
|
|
915
|
+
this.eventEmitter?.emit("realtime:usage", payload);
|
|
916
|
+
this.log("debug", "Realtime usage reported", {
|
|
917
|
+
model: event.model,
|
|
918
|
+
responseId: event.responseId
|
|
919
|
+
});
|
|
1005
920
|
}
|
|
1006
|
-
|
|
1007
|
-
|
|
921
|
+
log(level, message, context) {
|
|
922
|
+
const mergedContext = this.sessionId === void 0 ? context : {
|
|
923
|
+
...context,
|
|
924
|
+
sessionId: this.sessionId
|
|
925
|
+
};
|
|
926
|
+
this.options.logger?.[level]?.(message, mergedContext);
|
|
927
|
+
}
|
|
928
|
+
};
|
|
1008
929
|
function createRealtimeManager(client, options) {
|
|
1009
930
|
return new RealtimeManagerImpl(client, options);
|
|
1010
931
|
}
|
|
@@ -1033,6 +954,13 @@ function mergeRealtimeState(current, partial) {
|
|
|
1033
954
|
lastError: partial.lastError ?? current.lastError
|
|
1034
955
|
};
|
|
1035
956
|
}
|
|
957
|
+
function createRealtimeSessionId() {
|
|
958
|
+
const randomUuid = globalThis.crypto?.randomUUID?.();
|
|
959
|
+
if (randomUuid) {
|
|
960
|
+
return randomUuid;
|
|
961
|
+
}
|
|
962
|
+
return `realtime-session-${Date.now()}-${nextFallbackSessionId++}`;
|
|
963
|
+
}
|
|
1036
964
|
function isRecord(value) {
|
|
1037
965
|
return typeof value === "object" && value !== null;
|
|
1038
966
|
}
|
|
@@ -1069,14 +997,9 @@ function delay(ms) {
|
|
|
1069
997
|
});
|
|
1070
998
|
}
|
|
1071
999
|
|
|
1072
|
-
exports.AVATAR_CONTROL_TOOL_NAMES = AVATAR_CONTROL_TOOL_NAMES;
|
|
1073
1000
|
exports.DEFAULT_REALTIME_AGENT_INSTRUCTIONS = DEFAULT_REALTIME_AGENT_INSTRUCTIONS;
|
|
1074
|
-
exports.LOOK_AT_TOOL_NAME = LOOK_AT_TOOL_NAME;
|
|
1075
|
-
exports.PLAY_MOTION_TOOL_NAME = PLAY_MOTION_TOOL_NAME;
|
|
1076
1001
|
exports.RealtimeManagerImpl = RealtimeManagerImpl;
|
|
1077
|
-
exports.SET_EXPRESSION_TOOL_NAME = SET_EXPRESSION_TOOL_NAME;
|
|
1078
1002
|
exports.buildRealtimeSessionConfig = buildRealtimeSessionConfig;
|
|
1079
|
-
exports.createAvatarControlTools = createAvatarControlTools;
|
|
1080
1003
|
exports.createRealtimeManager = createRealtimeManager;
|
|
1081
1004
|
//# sourceMappingURL=index.js.map
|
|
1082
1005
|
//# sourceMappingURL=index.js.map
|