@charivo/realtime 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -10
- package/dist/chunk-3K733UIP.mjs +58 -0
- package/dist/chunk-3K733UIP.mjs.map +1 -0
- package/dist/{chunk-J5L3BPMG.mjs → chunk-WPZF3VXZ.mjs} +7 -4
- package/dist/chunk-WPZF3VXZ.mjs.map +1 -0
- package/dist/{chunk-3O3COMUY.mjs → chunk-YLMDWMWQ.mjs} +15 -5
- package/dist/chunk-YLMDWMWQ.mjs.map +1 -0
- package/dist/index.d.mts +23 -17
- package/dist/index.d.ts +23 -17
- package/dist/index.js +102 -192
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +100 -179
- package/dist/index.mjs.map +1 -1
- package/dist/openai/index.d.mts +1 -1
- package/dist/openai/index.d.ts +1 -1
- package/dist/openai/index.js +4 -1
- package/dist/openai/index.js.map +1 -1
- package/dist/openai/index.mjs +2 -2
- package/dist/openai-agents/index.d.mts +2 -1
- package/dist/openai-agents/index.d.ts +2 -1
- package/dist/openai-agents/index.js +11 -1
- package/dist/openai-agents/index.js.map +1 -1
- package/dist/openai-agents/index.mjs +2 -2
- package/dist/remote/index.d.mts +1 -1
- package/dist/remote/index.d.ts +1 -1
- package/dist/remote/index.js +15 -2
- package/dist/remote/index.js.map +1 -1
- package/dist/remote/index.mjs +3 -3
- package/dist/{types-bmYuc_Vs.d.mts → types-BRya38bF.d.mts} +3 -0
- package/dist/{types-bmYuc_Vs.d.ts → types-BRya38bF.d.ts} +3 -0
- package/package.json +2 -2
- package/dist/chunk-3O3COMUY.mjs.map +0 -1
- package/dist/chunk-J5L3BPMG.mjs.map +0 -1
- package/dist/chunk-PDODY2T7.mjs +0 -65
- package/dist/chunk-PDODY2T7.mjs.map +0 -1
package/dist/index.mjs
CHANGED
|
@@ -1,145 +1,7 @@
|
|
|
1
|
-
import {
|
|
2
|
-
export { DEFAULT_REALTIME_AGENT_INSTRUCTIONS, buildRealtimeSessionConfig } from './chunk-
|
|
1
|
+
import { buildRealtimeSessionConfig } from './chunk-3K733UIP.mjs';
|
|
2
|
+
export { DEFAULT_REALTIME_AGENT_INSTRUCTIONS, buildRealtimeSessionConfig } from './chunk-3K733UIP.mjs';
|
|
3
3
|
import { CharivoStateError, toCharivoError, subscribeBrowserLifecycle } from '@charivo/core';
|
|
4
4
|
|
|
5
|
-
// src/tools.ts
|
|
6
|
-
var MIN_GAZE = -1;
|
|
7
|
-
var MAX_GAZE = 1;
|
|
8
|
-
var SET_EXPRESSION_TOOL_NAME = "setExpression";
|
|
9
|
-
var PLAY_MOTION_TOOL_NAME = "playMotion";
|
|
10
|
-
var LOOK_AT_TOOL_NAME = "lookAt";
|
|
11
|
-
var AVATAR_CONTROL_TOOL_NAMES = [
|
|
12
|
-
SET_EXPRESSION_TOOL_NAME,
|
|
13
|
-
PLAY_MOTION_TOOL_NAME,
|
|
14
|
-
LOOK_AT_TOOL_NAME
|
|
15
|
-
];
|
|
16
|
-
function createAvatarControlTools(catalog) {
|
|
17
|
-
const tools = [];
|
|
18
|
-
if (catalog.expressions.length > 0) {
|
|
19
|
-
const expressionValues = [...catalog.expressions];
|
|
20
|
-
tools.push({
|
|
21
|
-
definition: {
|
|
22
|
-
type: "function",
|
|
23
|
-
name: SET_EXPRESSION_TOOL_NAME,
|
|
24
|
-
description: SET_EXPRESSION_TOOL_DESCRIPTION,
|
|
25
|
-
parameters: {
|
|
26
|
-
type: "object",
|
|
27
|
-
properties: {
|
|
28
|
-
expressionId: {
|
|
29
|
-
type: "string",
|
|
30
|
-
description: "Expression ID from the loaded Live2D model.",
|
|
31
|
-
enum: expressionValues
|
|
32
|
-
}
|
|
33
|
-
},
|
|
34
|
-
required: ["expressionId"]
|
|
35
|
-
}
|
|
36
|
-
},
|
|
37
|
-
async handler(args) {
|
|
38
|
-
const expressionId = args.expressionId;
|
|
39
|
-
if (typeof expressionId !== "string" || !expressionValues.includes(expressionId)) {
|
|
40
|
-
throw new Error(
|
|
41
|
-
'setExpression requires a valid "expressionId" from the model catalog'
|
|
42
|
-
);
|
|
43
|
-
}
|
|
44
|
-
return {
|
|
45
|
-
success: true,
|
|
46
|
-
expressionId
|
|
47
|
-
};
|
|
48
|
-
}
|
|
49
|
-
});
|
|
50
|
-
}
|
|
51
|
-
const motionGroups = Object.keys(catalog.motions);
|
|
52
|
-
if (motionGroups.length > 0) {
|
|
53
|
-
tools.push({
|
|
54
|
-
definition: {
|
|
55
|
-
type: "function",
|
|
56
|
-
name: PLAY_MOTION_TOOL_NAME,
|
|
57
|
-
description: PLAY_MOTION_TOOL_DESCRIPTION,
|
|
58
|
-
parameters: {
|
|
59
|
-
type: "object",
|
|
60
|
-
properties: {
|
|
61
|
-
group: {
|
|
62
|
-
type: "string",
|
|
63
|
-
description: "Motion group from the loaded Live2D model.",
|
|
64
|
-
enum: motionGroups
|
|
65
|
-
},
|
|
66
|
-
index: {
|
|
67
|
-
type: "number",
|
|
68
|
-
description: "Zero-based motion index within the selected motion group."
|
|
69
|
-
}
|
|
70
|
-
},
|
|
71
|
-
required: ["group", "index"]
|
|
72
|
-
}
|
|
73
|
-
},
|
|
74
|
-
async handler(args) {
|
|
75
|
-
const group = args.group;
|
|
76
|
-
const index = args.index;
|
|
77
|
-
if (typeof group !== "string" || !motionGroups.includes(group)) {
|
|
78
|
-
throw new Error(
|
|
79
|
-
'playMotion requires a valid "group" from the model catalog'
|
|
80
|
-
);
|
|
81
|
-
}
|
|
82
|
-
if (!Number.isInteger(index)) {
|
|
83
|
-
throw new Error('playMotion requires an integer "index"');
|
|
84
|
-
}
|
|
85
|
-
const count = catalog.motions[group] ?? 0;
|
|
86
|
-
const motionIndex = index;
|
|
87
|
-
if (motionIndex < 0 || motionIndex >= count) {
|
|
88
|
-
throw new Error(
|
|
89
|
-
`playMotion index ${String(motionIndex)} is out of range for group "${group}"`
|
|
90
|
-
);
|
|
91
|
-
}
|
|
92
|
-
return {
|
|
93
|
-
success: true,
|
|
94
|
-
group,
|
|
95
|
-
index: motionIndex
|
|
96
|
-
};
|
|
97
|
-
}
|
|
98
|
-
});
|
|
99
|
-
}
|
|
100
|
-
tools.push({
|
|
101
|
-
definition: {
|
|
102
|
-
type: "function",
|
|
103
|
-
name: LOOK_AT_TOOL_NAME,
|
|
104
|
-
description: LOOK_AT_TOOL_DESCRIPTION,
|
|
105
|
-
parameters: {
|
|
106
|
-
type: "object",
|
|
107
|
-
properties: {
|
|
108
|
-
x: {
|
|
109
|
-
type: "number",
|
|
110
|
-
description: "Normalized horizontal gaze target. -1 looks left, 0 is center, 1 looks right."
|
|
111
|
-
},
|
|
112
|
-
y: {
|
|
113
|
-
type: "number",
|
|
114
|
-
description: "Normalized vertical gaze target. -1 looks down, 0 is center, 1 looks up."
|
|
115
|
-
}
|
|
116
|
-
},
|
|
117
|
-
required: ["x", "y"]
|
|
118
|
-
}
|
|
119
|
-
},
|
|
120
|
-
async handler(args) {
|
|
121
|
-
const x = args.x;
|
|
122
|
-
const y = args.y;
|
|
123
|
-
if (typeof x !== "number" || Number.isNaN(x)) {
|
|
124
|
-
throw new Error('lookAt requires a numeric "x"');
|
|
125
|
-
}
|
|
126
|
-
if (typeof y !== "number" || Number.isNaN(y)) {
|
|
127
|
-
throw new Error('lookAt requires a numeric "y"');
|
|
128
|
-
}
|
|
129
|
-
return {
|
|
130
|
-
success: true,
|
|
131
|
-
x: clamp(x, MIN_GAZE, MAX_GAZE),
|
|
132
|
-
y: clamp(y, MIN_GAZE, MAX_GAZE)
|
|
133
|
-
};
|
|
134
|
-
}
|
|
135
|
-
});
|
|
136
|
-
return tools;
|
|
137
|
-
}
|
|
138
|
-
function clamp(value, min, max) {
|
|
139
|
-
return Math.max(min, Math.min(max, value));
|
|
140
|
-
}
|
|
141
|
-
|
|
142
|
-
// src/realtime-manager.ts
|
|
143
5
|
var DEFAULT_TOOL_TIMEOUT_MS = 1e4;
|
|
144
6
|
var RECONNECT_DELAYS_MS = [500, 1e3, 2e3, 4e3, 5e3];
|
|
145
7
|
var RealtimeManagerImpl = class {
|
|
@@ -414,6 +276,7 @@ var RealtimeManagerImpl = class {
|
|
|
414
276
|
return;
|
|
415
277
|
}
|
|
416
278
|
const text = event.text || this.state.response.text;
|
|
279
|
+
this.emitUsageEvent(event);
|
|
417
280
|
this.state = {
|
|
418
281
|
...this.state,
|
|
419
282
|
response: {
|
|
@@ -488,14 +351,22 @@ var RealtimeManagerImpl = class {
|
|
|
488
351
|
return;
|
|
489
352
|
}
|
|
490
353
|
try {
|
|
354
|
+
this.log("debug", "Realtime tool execution started", {
|
|
355
|
+
name: event.name,
|
|
356
|
+
callId: event.callId
|
|
357
|
+
});
|
|
491
358
|
const output = await this.runToolHandler(tool, event);
|
|
492
359
|
await this.client.sendToolResult(event.callId, output);
|
|
493
|
-
this.postProcessToolResult(event.name, output);
|
|
494
360
|
this.eventEmitter?.emit("realtime:tool:result", {
|
|
495
361
|
name: event.name,
|
|
496
362
|
output,
|
|
497
363
|
callId: event.callId
|
|
498
364
|
});
|
|
365
|
+
this.projectToolResult(event.name, output, event.callId);
|
|
366
|
+
this.log("info", "Realtime tool execution succeeded", {
|
|
367
|
+
name: event.name,
|
|
368
|
+
callId: event.callId
|
|
369
|
+
});
|
|
499
370
|
} catch (error) {
|
|
500
371
|
await this.handleToolExecutionFailure(
|
|
501
372
|
event.name,
|
|
@@ -534,39 +405,33 @@ var RealtimeManagerImpl = class {
|
|
|
534
405
|
);
|
|
535
406
|
}
|
|
536
407
|
}
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
}
|
|
556
|
-
return;
|
|
557
|
-
}
|
|
558
|
-
case LOOK_AT_TOOL_NAME: {
|
|
559
|
-
const coords = readGazeCoordinates(output);
|
|
560
|
-
if (coords) {
|
|
561
|
-
this.eventEmitter?.emit("realtime:gaze", coords);
|
|
562
|
-
}
|
|
563
|
-
return;
|
|
408
|
+
projectToolResult(name, output, callId) {
|
|
409
|
+
for (const projector of this.options.resultProjectors ?? []) {
|
|
410
|
+
try {
|
|
411
|
+
projector({
|
|
412
|
+
name,
|
|
413
|
+
output,
|
|
414
|
+
callId,
|
|
415
|
+
emit: (event, payload) => {
|
|
416
|
+
this.eventEmitter?.emit(event, payload);
|
|
417
|
+
}
|
|
418
|
+
});
|
|
419
|
+
} catch (error) {
|
|
420
|
+
const projectorError = error instanceof Error ? error : new Error(String(error));
|
|
421
|
+
this.log("warn", "Realtime result projector failed", {
|
|
422
|
+
name,
|
|
423
|
+
callId,
|
|
424
|
+
error: projectorError.message
|
|
425
|
+
});
|
|
564
426
|
}
|
|
565
|
-
default:
|
|
566
|
-
return;
|
|
567
427
|
}
|
|
568
428
|
}
|
|
569
429
|
emitToolError(name, error, callId) {
|
|
430
|
+
this.log("warn", "Realtime tool execution failed", {
|
|
431
|
+
name,
|
|
432
|
+
callId,
|
|
433
|
+
error: error.message
|
|
434
|
+
});
|
|
570
435
|
this.eventEmitter?.emit("realtime:tool:error", {
|
|
571
436
|
name,
|
|
572
437
|
error,
|
|
@@ -614,6 +479,10 @@ var RealtimeManagerImpl = class {
|
|
|
614
479
|
};
|
|
615
480
|
}
|
|
616
481
|
this.emitState();
|
|
482
|
+
this.log("warn", "Realtime reconnect started", {
|
|
483
|
+
cause,
|
|
484
|
+
connection: this.state.connection
|
|
485
|
+
});
|
|
617
486
|
this.reconnectInFlight = this.runReconnectLoop(reconnectToken, cause).catch((reconnectError) => {
|
|
618
487
|
this.finalizeFailedSession(reconnectError);
|
|
619
488
|
}).finally(() => {
|
|
@@ -634,6 +503,11 @@ var RealtimeManagerImpl = class {
|
|
|
634
503
|
delayMs,
|
|
635
504
|
cause
|
|
636
505
|
});
|
|
506
|
+
this.log("info", "Realtime reconnect attempt", {
|
|
507
|
+
attempt,
|
|
508
|
+
delayMs,
|
|
509
|
+
cause
|
|
510
|
+
});
|
|
637
511
|
await delay(delayMs);
|
|
638
512
|
if (!this.isReconnectTokenActive(reconnectToken)) {
|
|
639
513
|
return;
|
|
@@ -662,6 +536,11 @@ var RealtimeManagerImpl = class {
|
|
|
662
536
|
totalMs: Date.now() - startedAt,
|
|
663
537
|
cause
|
|
664
538
|
});
|
|
539
|
+
this.log("info", "Realtime reconnect succeeded", {
|
|
540
|
+
attempts: attempt,
|
|
541
|
+
totalMs: Date.now() - startedAt,
|
|
542
|
+
cause
|
|
543
|
+
});
|
|
665
544
|
return;
|
|
666
545
|
} catch (error) {
|
|
667
546
|
lastError = error instanceof Error ? error : new Error(String(error));
|
|
@@ -682,6 +561,12 @@ var RealtimeManagerImpl = class {
|
|
|
682
561
|
cause,
|
|
683
562
|
lastError
|
|
684
563
|
});
|
|
564
|
+
this.log("error", "Realtime reconnect exhausted", {
|
|
565
|
+
attempts: RECONNECT_DELAYS_MS.length,
|
|
566
|
+
totalMs: Date.now() - startedAt,
|
|
567
|
+
cause,
|
|
568
|
+
error: lastError.message
|
|
569
|
+
});
|
|
685
570
|
throw lastError;
|
|
686
571
|
}
|
|
687
572
|
cancelReconnectLoop() {
|
|
@@ -707,6 +592,10 @@ var RealtimeManagerImpl = class {
|
|
|
707
592
|
};
|
|
708
593
|
this.eventEmitter?.emit("realtime:error", { error: typedError });
|
|
709
594
|
this.emitState();
|
|
595
|
+
this.log("error", "Realtime transport error surfaced", {
|
|
596
|
+
connection,
|
|
597
|
+
error: typedError.message
|
|
598
|
+
});
|
|
710
599
|
}
|
|
711
600
|
emitAudioStart() {
|
|
712
601
|
if (this.isAudioPlaybackActive) {
|
|
@@ -852,12 +741,20 @@ var RealtimeManagerImpl = class {
|
|
|
852
741
|
lastError: null
|
|
853
742
|
};
|
|
854
743
|
this.emitState();
|
|
744
|
+
this.log("debug", "Realtime session patch started", {
|
|
745
|
+
voice: sessionConfig.voice,
|
|
746
|
+
model: sessionConfig.model
|
|
747
|
+
});
|
|
855
748
|
try {
|
|
856
749
|
await this.client.updateSession(sessionConfig);
|
|
857
750
|
if (this.stopRequestedDuringRefresh) {
|
|
858
751
|
return;
|
|
859
752
|
}
|
|
860
753
|
this.commitPatchedSession(sessionConfig);
|
|
754
|
+
this.log("info", "Realtime session patch succeeded", {
|
|
755
|
+
voice: sessionConfig.voice,
|
|
756
|
+
model: sessionConfig.model
|
|
757
|
+
});
|
|
861
758
|
} catch (error) {
|
|
862
759
|
const refreshError = error instanceof Error ? error : new Error(String(error));
|
|
863
760
|
const typedRefreshError = toCharivoError("transport", refreshError);
|
|
@@ -867,18 +764,29 @@ var RealtimeManagerImpl = class {
|
|
|
867
764
|
if (this.state.lastError !== typedRefreshError && this.state.lastError?.cause !== refreshError) {
|
|
868
765
|
this.applyError(typedRefreshError, "connected");
|
|
869
766
|
}
|
|
767
|
+
this.log("error", "Realtime session patch failed", {
|
|
768
|
+
error: typedRefreshError.message
|
|
769
|
+
});
|
|
870
770
|
throw typedRefreshError;
|
|
871
771
|
} finally {
|
|
872
772
|
this.isRefreshingSession = false;
|
|
873
773
|
}
|
|
874
774
|
}
|
|
875
775
|
emitSessionStart(reason, state = this.getState()) {
|
|
776
|
+
this.log("info", "Realtime session started", {
|
|
777
|
+
reason,
|
|
778
|
+
connection: state.connection
|
|
779
|
+
});
|
|
876
780
|
this.eventEmitter?.emit("realtime:session:start", {
|
|
877
781
|
state,
|
|
878
782
|
reason
|
|
879
783
|
});
|
|
880
784
|
}
|
|
881
785
|
emitSessionEnd(reason, state = this.getState()) {
|
|
786
|
+
this.log("info", "Realtime session ended", {
|
|
787
|
+
reason,
|
|
788
|
+
connection: state.connection
|
|
789
|
+
});
|
|
882
790
|
this.eventEmitter?.emit("realtime:session:end", {
|
|
883
791
|
state,
|
|
884
792
|
reason
|
|
@@ -906,6 +814,9 @@ var RealtimeManagerImpl = class {
|
|
|
906
814
|
};
|
|
907
815
|
this.eventEmitter?.emit("realtime:error", { error });
|
|
908
816
|
this.emitState();
|
|
817
|
+
this.log("error", "Realtime session failed", {
|
|
818
|
+
error: error.message
|
|
819
|
+
});
|
|
909
820
|
}
|
|
910
821
|
finalizeStoppedSession(emitSessionEnd, reason) {
|
|
911
822
|
this.cancelReconnectLoop();
|
|
@@ -932,15 +843,25 @@ var RealtimeManagerImpl = class {
|
|
|
932
843
|
this.emitSessionEnd(reason);
|
|
933
844
|
}
|
|
934
845
|
}
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
846
|
+
emitUsageEvent(event) {
|
|
847
|
+
if (!event.usage) {
|
|
848
|
+
return;
|
|
849
|
+
}
|
|
850
|
+
const payload = {
|
|
851
|
+
usage: event.usage,
|
|
852
|
+
model: event.model,
|
|
853
|
+
responseId: event.responseId
|
|
854
|
+
};
|
|
855
|
+
this.eventEmitter?.emit("realtime:usage", payload);
|
|
856
|
+
this.log("debug", "Realtime usage reported", {
|
|
857
|
+
model: event.model,
|
|
858
|
+
responseId: event.responseId
|
|
859
|
+
});
|
|
941
860
|
}
|
|
942
|
-
|
|
943
|
-
|
|
861
|
+
log(level, message, context) {
|
|
862
|
+
this.options.logger?.[level]?.(message, context);
|
|
863
|
+
}
|
|
864
|
+
};
|
|
944
865
|
function createRealtimeManager(client, options) {
|
|
945
866
|
return new RealtimeManagerImpl(client, options);
|
|
946
867
|
}
|
|
@@ -1005,6 +926,6 @@ function delay(ms) {
|
|
|
1005
926
|
});
|
|
1006
927
|
}
|
|
1007
928
|
|
|
1008
|
-
export {
|
|
929
|
+
export { RealtimeManagerImpl, createRealtimeManager };
|
|
1009
930
|
//# sourceMappingURL=index.mjs.map
|
|
1010
931
|
//# sourceMappingURL=index.mjs.map
|