@charivo/realtime 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +51 -12
- package/dist/chunk-LBKX5PJ6.mjs +54 -0
- package/dist/chunk-LBKX5PJ6.mjs.map +1 -0
- package/dist/{chunk-J5L3BPMG.mjs → chunk-LOHYIBEP.mjs} +8 -6
- package/dist/chunk-LOHYIBEP.mjs.map +1 -0
- package/dist/{chunk-3O3COMUY.mjs → chunk-M7TPQNHF.mjs} +18 -11
- package/dist/chunk-M7TPQNHF.mjs.map +1 -0
- package/dist/{chunk-ZJZO4QYA.mjs → chunk-RRXU3IC5.mjs} +9 -4
- package/dist/chunk-RRXU3IC5.mjs.map +1 -0
- package/dist/index.d.mts +24 -17
- package/dist/index.d.ts +24 -17
- package/dist/index.js +121 -198
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +117 -179
- package/dist/index.mjs.map +1 -1
- package/dist/openai/index.d.mts +1 -1
- package/dist/openai/index.d.ts +1 -1
- package/dist/openai/index.js +8 -5
- package/dist/openai/index.js.map +1 -1
- package/dist/openai/index.mjs +3 -3
- package/dist/openai-agents/index.d.mts +2 -1
- package/dist/openai-agents/index.d.ts +2 -1
- package/dist/openai-agents/index.js +29 -8
- package/dist/openai-agents/index.js.map +1 -1
- package/dist/openai-agents/index.mjs +3 -3
- package/dist/remote/index.d.mts +1 -1
- package/dist/remote/index.d.ts +1 -1
- package/dist/remote/index.js +34 -10
- package/dist/remote/index.js.map +1 -1
- package/dist/remote/index.mjs +4 -4
- package/dist/{types-bmYuc_Vs.d.mts → types-BRya38bF.d.mts} +3 -0
- package/dist/{types-bmYuc_Vs.d.ts → types-BRya38bF.d.ts} +3 -0
- package/package.json +2 -2
- package/dist/chunk-3O3COMUY.mjs.map +0 -1
- package/dist/chunk-J5L3BPMG.mjs.map +0 -1
- package/dist/chunk-PDODY2T7.mjs +0 -65
- package/dist/chunk-PDODY2T7.mjs.map +0 -1
- package/dist/chunk-ZJZO4QYA.mjs.map +0 -1
package/dist/index.mjs
CHANGED
|
@@ -1,147 +1,10 @@
|
|
|
1
|
-
import {
|
|
2
|
-
export { DEFAULT_REALTIME_AGENT_INSTRUCTIONS, buildRealtimeSessionConfig } from './chunk-
|
|
1
|
+
import { buildRealtimeSessionConfig } from './chunk-LBKX5PJ6.mjs';
|
|
2
|
+
export { DEFAULT_REALTIME_AGENT_INSTRUCTIONS, buildRealtimeSessionConfig } from './chunk-LBKX5PJ6.mjs';
|
|
3
3
|
import { CharivoStateError, toCharivoError, subscribeBrowserLifecycle } from '@charivo/core';
|
|
4
4
|
|
|
5
|
-
// src/tools.ts
|
|
6
|
-
var MIN_GAZE = -1;
|
|
7
|
-
var MAX_GAZE = 1;
|
|
8
|
-
var SET_EXPRESSION_TOOL_NAME = "setExpression";
|
|
9
|
-
var PLAY_MOTION_TOOL_NAME = "playMotion";
|
|
10
|
-
var LOOK_AT_TOOL_NAME = "lookAt";
|
|
11
|
-
var AVATAR_CONTROL_TOOL_NAMES = [
|
|
12
|
-
SET_EXPRESSION_TOOL_NAME,
|
|
13
|
-
PLAY_MOTION_TOOL_NAME,
|
|
14
|
-
LOOK_AT_TOOL_NAME
|
|
15
|
-
];
|
|
16
|
-
function createAvatarControlTools(catalog) {
|
|
17
|
-
const tools = [];
|
|
18
|
-
if (catalog.expressions.length > 0) {
|
|
19
|
-
const expressionValues = [...catalog.expressions];
|
|
20
|
-
tools.push({
|
|
21
|
-
definition: {
|
|
22
|
-
type: "function",
|
|
23
|
-
name: SET_EXPRESSION_TOOL_NAME,
|
|
24
|
-
description: SET_EXPRESSION_TOOL_DESCRIPTION,
|
|
25
|
-
parameters: {
|
|
26
|
-
type: "object",
|
|
27
|
-
properties: {
|
|
28
|
-
expressionId: {
|
|
29
|
-
type: "string",
|
|
30
|
-
description: "Expression ID from the loaded Live2D model.",
|
|
31
|
-
enum: expressionValues
|
|
32
|
-
}
|
|
33
|
-
},
|
|
34
|
-
required: ["expressionId"]
|
|
35
|
-
}
|
|
36
|
-
},
|
|
37
|
-
async handler(args) {
|
|
38
|
-
const expressionId = args.expressionId;
|
|
39
|
-
if (typeof expressionId !== "string" || !expressionValues.includes(expressionId)) {
|
|
40
|
-
throw new Error(
|
|
41
|
-
'setExpression requires a valid "expressionId" from the model catalog'
|
|
42
|
-
);
|
|
43
|
-
}
|
|
44
|
-
return {
|
|
45
|
-
success: true,
|
|
46
|
-
expressionId
|
|
47
|
-
};
|
|
48
|
-
}
|
|
49
|
-
});
|
|
50
|
-
}
|
|
51
|
-
const motionGroups = Object.keys(catalog.motions);
|
|
52
|
-
if (motionGroups.length > 0) {
|
|
53
|
-
tools.push({
|
|
54
|
-
definition: {
|
|
55
|
-
type: "function",
|
|
56
|
-
name: PLAY_MOTION_TOOL_NAME,
|
|
57
|
-
description: PLAY_MOTION_TOOL_DESCRIPTION,
|
|
58
|
-
parameters: {
|
|
59
|
-
type: "object",
|
|
60
|
-
properties: {
|
|
61
|
-
group: {
|
|
62
|
-
type: "string",
|
|
63
|
-
description: "Motion group from the loaded Live2D model.",
|
|
64
|
-
enum: motionGroups
|
|
65
|
-
},
|
|
66
|
-
index: {
|
|
67
|
-
type: "number",
|
|
68
|
-
description: "Zero-based motion index within the selected motion group."
|
|
69
|
-
}
|
|
70
|
-
},
|
|
71
|
-
required: ["group", "index"]
|
|
72
|
-
}
|
|
73
|
-
},
|
|
74
|
-
async handler(args) {
|
|
75
|
-
const group = args.group;
|
|
76
|
-
const index = args.index;
|
|
77
|
-
if (typeof group !== "string" || !motionGroups.includes(group)) {
|
|
78
|
-
throw new Error(
|
|
79
|
-
'playMotion requires a valid "group" from the model catalog'
|
|
80
|
-
);
|
|
81
|
-
}
|
|
82
|
-
if (!Number.isInteger(index)) {
|
|
83
|
-
throw new Error('playMotion requires an integer "index"');
|
|
84
|
-
}
|
|
85
|
-
const count = catalog.motions[group] ?? 0;
|
|
86
|
-
const motionIndex = index;
|
|
87
|
-
if (motionIndex < 0 || motionIndex >= count) {
|
|
88
|
-
throw new Error(
|
|
89
|
-
`playMotion index ${String(motionIndex)} is out of range for group "${group}"`
|
|
90
|
-
);
|
|
91
|
-
}
|
|
92
|
-
return {
|
|
93
|
-
success: true,
|
|
94
|
-
group,
|
|
95
|
-
index: motionIndex
|
|
96
|
-
};
|
|
97
|
-
}
|
|
98
|
-
});
|
|
99
|
-
}
|
|
100
|
-
tools.push({
|
|
101
|
-
definition: {
|
|
102
|
-
type: "function",
|
|
103
|
-
name: LOOK_AT_TOOL_NAME,
|
|
104
|
-
description: LOOK_AT_TOOL_DESCRIPTION,
|
|
105
|
-
parameters: {
|
|
106
|
-
type: "object",
|
|
107
|
-
properties: {
|
|
108
|
-
x: {
|
|
109
|
-
type: "number",
|
|
110
|
-
description: "Normalized horizontal gaze target. -1 looks left, 0 is center, 1 looks right."
|
|
111
|
-
},
|
|
112
|
-
y: {
|
|
113
|
-
type: "number",
|
|
114
|
-
description: "Normalized vertical gaze target. -1 looks down, 0 is center, 1 looks up."
|
|
115
|
-
}
|
|
116
|
-
},
|
|
117
|
-
required: ["x", "y"]
|
|
118
|
-
}
|
|
119
|
-
},
|
|
120
|
-
async handler(args) {
|
|
121
|
-
const x = args.x;
|
|
122
|
-
const y = args.y;
|
|
123
|
-
if (typeof x !== "number" || Number.isNaN(x)) {
|
|
124
|
-
throw new Error('lookAt requires a numeric "x"');
|
|
125
|
-
}
|
|
126
|
-
if (typeof y !== "number" || Number.isNaN(y)) {
|
|
127
|
-
throw new Error('lookAt requires a numeric "y"');
|
|
128
|
-
}
|
|
129
|
-
return {
|
|
130
|
-
success: true,
|
|
131
|
-
x: clamp(x, MIN_GAZE, MAX_GAZE),
|
|
132
|
-
y: clamp(y, MIN_GAZE, MAX_GAZE)
|
|
133
|
-
};
|
|
134
|
-
}
|
|
135
|
-
});
|
|
136
|
-
return tools;
|
|
137
|
-
}
|
|
138
|
-
function clamp(value, min, max) {
|
|
139
|
-
return Math.max(min, Math.min(max, value));
|
|
140
|
-
}
|
|
141
|
-
|
|
142
|
-
// src/realtime-manager.ts
|
|
143
5
|
var DEFAULT_TOOL_TIMEOUT_MS = 1e4;
|
|
144
6
|
var RECONNECT_DELAYS_MS = [500, 1e3, 2e3, 4e3, 5e3];
|
|
7
|
+
var nextFallbackSessionId = 1;
|
|
145
8
|
var RealtimeManagerImpl = class {
|
|
146
9
|
constructor(client, options = {}) {
|
|
147
10
|
this.client = client;
|
|
@@ -173,6 +36,7 @@ var RealtimeManagerImpl = class {
|
|
|
173
36
|
reconnectToken = 0;
|
|
174
37
|
wakeLockSentinel = null;
|
|
175
38
|
teardownBrowserLifecycle;
|
|
39
|
+
sessionId;
|
|
176
40
|
toolRegistry = /* @__PURE__ */ new Map();
|
|
177
41
|
state = {
|
|
178
42
|
connection: "idle",
|
|
@@ -237,6 +101,7 @@ var RealtimeManagerImpl = class {
|
|
|
237
101
|
}
|
|
238
102
|
this.sessionBaseConfig = this.resolveNextSessionBaseConfig(config);
|
|
239
103
|
const sessionConfig = this.buildEffectiveSessionConfig();
|
|
104
|
+
this.sessionId = createRealtimeSessionId();
|
|
240
105
|
this.state = {
|
|
241
106
|
...this.state,
|
|
242
107
|
connection: "connecting",
|
|
@@ -414,6 +279,7 @@ var RealtimeManagerImpl = class {
|
|
|
414
279
|
return;
|
|
415
280
|
}
|
|
416
281
|
const text = event.text || this.state.response.text;
|
|
282
|
+
this.emitUsageEvent(event);
|
|
417
283
|
this.state = {
|
|
418
284
|
...this.state,
|
|
419
285
|
response: {
|
|
@@ -488,14 +354,22 @@ var RealtimeManagerImpl = class {
|
|
|
488
354
|
return;
|
|
489
355
|
}
|
|
490
356
|
try {
|
|
357
|
+
this.log("debug", "Realtime tool execution started", {
|
|
358
|
+
name: event.name,
|
|
359
|
+
callId: event.callId
|
|
360
|
+
});
|
|
491
361
|
const output = await this.runToolHandler(tool, event);
|
|
492
362
|
await this.client.sendToolResult(event.callId, output);
|
|
493
|
-
this.postProcessToolResult(event.name, output);
|
|
494
363
|
this.eventEmitter?.emit("realtime:tool:result", {
|
|
495
364
|
name: event.name,
|
|
496
365
|
output,
|
|
497
366
|
callId: event.callId
|
|
498
367
|
});
|
|
368
|
+
this.projectToolResult(event.name, output, event.callId);
|
|
369
|
+
this.log("info", "Realtime tool execution succeeded", {
|
|
370
|
+
name: event.name,
|
|
371
|
+
callId: event.callId
|
|
372
|
+
});
|
|
499
373
|
} catch (error) {
|
|
500
374
|
await this.handleToolExecutionFailure(
|
|
501
375
|
event.name,
|
|
@@ -534,39 +408,33 @@ var RealtimeManagerImpl = class {
|
|
|
534
408
|
);
|
|
535
409
|
}
|
|
536
410
|
}
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
}
|
|
556
|
-
return;
|
|
557
|
-
}
|
|
558
|
-
case LOOK_AT_TOOL_NAME: {
|
|
559
|
-
const coords = readGazeCoordinates(output);
|
|
560
|
-
if (coords) {
|
|
561
|
-
this.eventEmitter?.emit("realtime:gaze", coords);
|
|
562
|
-
}
|
|
563
|
-
return;
|
|
411
|
+
projectToolResult(name, output, callId) {
|
|
412
|
+
for (const projector of this.options.resultProjectors ?? []) {
|
|
413
|
+
try {
|
|
414
|
+
projector({
|
|
415
|
+
name,
|
|
416
|
+
output,
|
|
417
|
+
callId,
|
|
418
|
+
emit: (event, payload) => {
|
|
419
|
+
this.eventEmitter?.emit(event, payload);
|
|
420
|
+
}
|
|
421
|
+
});
|
|
422
|
+
} catch (error) {
|
|
423
|
+
const projectorError = error instanceof Error ? error : new Error(String(error));
|
|
424
|
+
this.log("warn", "Realtime result projector failed", {
|
|
425
|
+
name,
|
|
426
|
+
callId,
|
|
427
|
+
error: projectorError.message
|
|
428
|
+
});
|
|
564
429
|
}
|
|
565
|
-
default:
|
|
566
|
-
return;
|
|
567
430
|
}
|
|
568
431
|
}
|
|
569
432
|
emitToolError(name, error, callId) {
|
|
433
|
+
this.log("warn", "Realtime tool execution failed", {
|
|
434
|
+
name,
|
|
435
|
+
callId,
|
|
436
|
+
error: error.message
|
|
437
|
+
});
|
|
570
438
|
this.eventEmitter?.emit("realtime:tool:error", {
|
|
571
439
|
name,
|
|
572
440
|
error,
|
|
@@ -614,6 +482,10 @@ var RealtimeManagerImpl = class {
|
|
|
614
482
|
};
|
|
615
483
|
}
|
|
616
484
|
this.emitState();
|
|
485
|
+
this.log("warn", "Realtime reconnect started", {
|
|
486
|
+
cause,
|
|
487
|
+
connection: this.state.connection
|
|
488
|
+
});
|
|
617
489
|
this.reconnectInFlight = this.runReconnectLoop(reconnectToken, cause).catch((reconnectError) => {
|
|
618
490
|
this.finalizeFailedSession(reconnectError);
|
|
619
491
|
}).finally(() => {
|
|
@@ -634,6 +506,11 @@ var RealtimeManagerImpl = class {
|
|
|
634
506
|
delayMs,
|
|
635
507
|
cause
|
|
636
508
|
});
|
|
509
|
+
this.log("info", "Realtime reconnect attempt", {
|
|
510
|
+
attempt,
|
|
511
|
+
delayMs,
|
|
512
|
+
cause
|
|
513
|
+
});
|
|
637
514
|
await delay(delayMs);
|
|
638
515
|
if (!this.isReconnectTokenActive(reconnectToken)) {
|
|
639
516
|
return;
|
|
@@ -662,6 +539,11 @@ var RealtimeManagerImpl = class {
|
|
|
662
539
|
totalMs: Date.now() - startedAt,
|
|
663
540
|
cause
|
|
664
541
|
});
|
|
542
|
+
this.log("info", "Realtime reconnect succeeded", {
|
|
543
|
+
attempts: attempt,
|
|
544
|
+
totalMs: Date.now() - startedAt,
|
|
545
|
+
cause
|
|
546
|
+
});
|
|
665
547
|
return;
|
|
666
548
|
} catch (error) {
|
|
667
549
|
lastError = error instanceof Error ? error : new Error(String(error));
|
|
@@ -682,6 +564,12 @@ var RealtimeManagerImpl = class {
|
|
|
682
564
|
cause,
|
|
683
565
|
lastError
|
|
684
566
|
});
|
|
567
|
+
this.log("error", "Realtime reconnect exhausted", {
|
|
568
|
+
attempts: RECONNECT_DELAYS_MS.length,
|
|
569
|
+
totalMs: Date.now() - startedAt,
|
|
570
|
+
cause,
|
|
571
|
+
error: lastError.message
|
|
572
|
+
});
|
|
685
573
|
throw lastError;
|
|
686
574
|
}
|
|
687
575
|
cancelReconnectLoop() {
|
|
@@ -707,6 +595,10 @@ var RealtimeManagerImpl = class {
|
|
|
707
595
|
};
|
|
708
596
|
this.eventEmitter?.emit("realtime:error", { error: typedError });
|
|
709
597
|
this.emitState();
|
|
598
|
+
this.log("error", "Realtime transport error surfaced", {
|
|
599
|
+
connection,
|
|
600
|
+
error: typedError.message
|
|
601
|
+
});
|
|
710
602
|
}
|
|
711
603
|
emitAudioStart() {
|
|
712
604
|
if (this.isAudioPlaybackActive) {
|
|
@@ -852,12 +744,20 @@ var RealtimeManagerImpl = class {
|
|
|
852
744
|
lastError: null
|
|
853
745
|
};
|
|
854
746
|
this.emitState();
|
|
747
|
+
this.log("debug", "Realtime session patch started", {
|
|
748
|
+
voice: sessionConfig.voice,
|
|
749
|
+
model: sessionConfig.model
|
|
750
|
+
});
|
|
855
751
|
try {
|
|
856
752
|
await this.client.updateSession(sessionConfig);
|
|
857
753
|
if (this.stopRequestedDuringRefresh) {
|
|
858
754
|
return;
|
|
859
755
|
}
|
|
860
756
|
this.commitPatchedSession(sessionConfig);
|
|
757
|
+
this.log("info", "Realtime session patch succeeded", {
|
|
758
|
+
voice: sessionConfig.voice,
|
|
759
|
+
model: sessionConfig.model
|
|
760
|
+
});
|
|
861
761
|
} catch (error) {
|
|
862
762
|
const refreshError = error instanceof Error ? error : new Error(String(error));
|
|
863
763
|
const typedRefreshError = toCharivoError("transport", refreshError);
|
|
@@ -867,18 +767,29 @@ var RealtimeManagerImpl = class {
|
|
|
867
767
|
if (this.state.lastError !== typedRefreshError && this.state.lastError?.cause !== refreshError) {
|
|
868
768
|
this.applyError(typedRefreshError, "connected");
|
|
869
769
|
}
|
|
770
|
+
this.log("error", "Realtime session patch failed", {
|
|
771
|
+
error: typedRefreshError.message
|
|
772
|
+
});
|
|
870
773
|
throw typedRefreshError;
|
|
871
774
|
} finally {
|
|
872
775
|
this.isRefreshingSession = false;
|
|
873
776
|
}
|
|
874
777
|
}
|
|
875
778
|
emitSessionStart(reason, state = this.getState()) {
|
|
779
|
+
this.log("info", "Realtime session started", {
|
|
780
|
+
reason,
|
|
781
|
+
connection: state.connection
|
|
782
|
+
});
|
|
876
783
|
this.eventEmitter?.emit("realtime:session:start", {
|
|
877
784
|
state,
|
|
878
785
|
reason
|
|
879
786
|
});
|
|
880
787
|
}
|
|
881
788
|
emitSessionEnd(reason, state = this.getState()) {
|
|
789
|
+
this.log("info", "Realtime session ended", {
|
|
790
|
+
reason,
|
|
791
|
+
connection: state.connection
|
|
792
|
+
});
|
|
882
793
|
this.eventEmitter?.emit("realtime:session:end", {
|
|
883
794
|
state,
|
|
884
795
|
reason
|
|
@@ -906,6 +817,10 @@ var RealtimeManagerImpl = class {
|
|
|
906
817
|
};
|
|
907
818
|
this.eventEmitter?.emit("realtime:error", { error });
|
|
908
819
|
this.emitState();
|
|
820
|
+
this.log("error", "Realtime session failed", {
|
|
821
|
+
error: error.message
|
|
822
|
+
});
|
|
823
|
+
this.sessionId = void 0;
|
|
909
824
|
}
|
|
910
825
|
finalizeStoppedSession(emitSessionEnd, reason) {
|
|
911
826
|
this.cancelReconnectLoop();
|
|
@@ -931,16 +846,32 @@ var RealtimeManagerImpl = class {
|
|
|
931
846
|
if (emitSessionEnd) {
|
|
932
847
|
this.emitSessionEnd(reason);
|
|
933
848
|
}
|
|
849
|
+
this.sessionId = void 0;
|
|
934
850
|
}
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
851
|
+
emitUsageEvent(event) {
|
|
852
|
+
if (!event.usage) {
|
|
853
|
+
return;
|
|
854
|
+
}
|
|
855
|
+
const payload = {
|
|
856
|
+
usage: event.usage,
|
|
857
|
+
model: event.model,
|
|
858
|
+
responseId: event.responseId,
|
|
859
|
+
sessionId: this.sessionId
|
|
860
|
+
};
|
|
861
|
+
this.eventEmitter?.emit("realtime:usage", payload);
|
|
862
|
+
this.log("debug", "Realtime usage reported", {
|
|
863
|
+
model: event.model,
|
|
864
|
+
responseId: event.responseId
|
|
865
|
+
});
|
|
941
866
|
}
|
|
942
|
-
|
|
943
|
-
|
|
867
|
+
log(level, message, context) {
|
|
868
|
+
const mergedContext = this.sessionId === void 0 ? context : {
|
|
869
|
+
...context,
|
|
870
|
+
sessionId: this.sessionId
|
|
871
|
+
};
|
|
872
|
+
this.options.logger?.[level]?.(message, mergedContext);
|
|
873
|
+
}
|
|
874
|
+
};
|
|
944
875
|
function createRealtimeManager(client, options) {
|
|
945
876
|
return new RealtimeManagerImpl(client, options);
|
|
946
877
|
}
|
|
@@ -969,6 +900,13 @@ function mergeRealtimeState(current, partial) {
|
|
|
969
900
|
lastError: partial.lastError ?? current.lastError
|
|
970
901
|
};
|
|
971
902
|
}
|
|
903
|
+
function createRealtimeSessionId() {
|
|
904
|
+
const randomUuid = globalThis.crypto?.randomUUID?.();
|
|
905
|
+
if (randomUuid) {
|
|
906
|
+
return randomUuid;
|
|
907
|
+
}
|
|
908
|
+
return `realtime-session-${Date.now()}-${nextFallbackSessionId++}`;
|
|
909
|
+
}
|
|
972
910
|
function isRecord(value) {
|
|
973
911
|
return typeof value === "object" && value !== null;
|
|
974
912
|
}
|
|
@@ -1005,6 +943,6 @@ function delay(ms) {
|
|
|
1005
943
|
});
|
|
1006
944
|
}
|
|
1007
945
|
|
|
1008
|
-
export {
|
|
946
|
+
export { RealtimeManagerImpl, createRealtimeManager };
|
|
1009
947
|
//# sourceMappingURL=index.mjs.map
|
|
1010
948
|
//# sourceMappingURL=index.mjs.map
|