@charivo/realtime 0.5.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -8,22 +8,15 @@ var core = require('@charivo/core');
8
8
  var DEFAULT_PROVIDER = "openai";
9
9
  var DEFAULT_MODEL = "gpt-realtime-mini";
10
10
  var DEFAULT_REALTIME_VOICE = "marin";
11
- var SET_EXPRESSION_TOOL_DESCRIPTION = "Change the avatar's facial expression only when the emotional beat clearly shifts or should linger across the reply. Do not use this for every polite or lightweight reaction.";
12
- var PLAY_MOTION_TOOL_DESCRIPTION = "Play a noticeable body motion for greetings, emphasis, or bigger reaction beats. Prefer this over stacking multiple smaller actions when the moment needs one clear accent. Usually use at most one motion in a reply.";
13
- var LOOK_AT_TOOL_DESCRIPTION = `Shift the avatar's gaze for subtle attention changes or conversational focus. Trigger this on natural phrases like "glance", "look over", "peek at", or directional cues. Prefer this before "setExpression" when a lightweight reaction is enough.`;
14
11
  var DEFAULT_REALTIME_AGENT_INSTRUCTIONS = `
15
- You are speaking in a realtime voice conversation through a Live2D avatar.
12
+ You are speaking in a realtime voice conversation.
16
13
  Respond naturally and keep replies concise enough for spoken delivery.
17
- Use avatar tools only when they add something meaningful to the moment.
18
- Use "setExpression" when the emotional beat clearly shifts, especially when the expression should linger across the reply.
19
- Do not use "setExpression" for every friendly, polite, or mildly positive reply.
20
- Use "playMotion" for greetings, emphasis, or bigger reaction beats, usually at most once in a reply.
21
- Use "lookAt" for subtle attention shifts or conversational focus, and prefer it before "setExpression" when a lightweight reaction is enough.
22
- Prefer a single meaningful avatar action over chaining multiple actions on simple replies.
23
- Many turns should use no avatar tool at all.
14
+ Use tools only when they add something meaningful to the moment.
15
+ Prefer a single meaningful action over chaining multiple actions on simple replies.
16
+ Many turns should use no tool at all.
24
17
  Never break character. Never refer to yourself as an AI, model, or assistant.
25
18
  Do not mention tool calls in the spoken response.
26
- Speak only the words you want heard aloud. Use avatar tools for actions. Never say tool names or tool arguments out loud. Do not output bracketed or parenthetical action notes. If a tool is unavailable, omit the action entirely rather than narrating it.
19
+ Speak only the words you want heard aloud. Use tools for actions. Never say tool names or tool arguments out loud. Do not output bracketed or parenthetical action notes. If a tool is unavailable, omit the action entirely rather than narrating it.
27
20
  `.trim();
28
21
  function buildRealtimeSessionConfig({
29
22
  character,
@@ -66,143 +59,6 @@ function ensureSentenceFragment(text) {
66
59
  return text.trim().replace(/[.!?]+$/, "");
67
60
  }
68
61
 
69
- // src/tools.ts
70
- var MIN_GAZE = -1;
71
- var MAX_GAZE = 1;
72
- var SET_EXPRESSION_TOOL_NAME = "setExpression";
73
- var PLAY_MOTION_TOOL_NAME = "playMotion";
74
- var LOOK_AT_TOOL_NAME = "lookAt";
75
- var AVATAR_CONTROL_TOOL_NAMES = [
76
- SET_EXPRESSION_TOOL_NAME,
77
- PLAY_MOTION_TOOL_NAME,
78
- LOOK_AT_TOOL_NAME
79
- ];
80
- function createAvatarControlTools(catalog) {
81
- const tools = [];
82
- if (catalog.expressions.length > 0) {
83
- const expressionValues = [...catalog.expressions];
84
- tools.push({
85
- definition: {
86
- type: "function",
87
- name: SET_EXPRESSION_TOOL_NAME,
88
- description: SET_EXPRESSION_TOOL_DESCRIPTION,
89
- parameters: {
90
- type: "object",
91
- properties: {
92
- expressionId: {
93
- type: "string",
94
- description: "Expression ID from the loaded Live2D model.",
95
- enum: expressionValues
96
- }
97
- },
98
- required: ["expressionId"]
99
- }
100
- },
101
- async handler(args) {
102
- const expressionId = args.expressionId;
103
- if (typeof expressionId !== "string" || !expressionValues.includes(expressionId)) {
104
- throw new Error(
105
- 'setExpression requires a valid "expressionId" from the model catalog'
106
- );
107
- }
108
- return {
109
- success: true,
110
- expressionId
111
- };
112
- }
113
- });
114
- }
115
- const motionGroups = Object.keys(catalog.motions);
116
- if (motionGroups.length > 0) {
117
- tools.push({
118
- definition: {
119
- type: "function",
120
- name: PLAY_MOTION_TOOL_NAME,
121
- description: PLAY_MOTION_TOOL_DESCRIPTION,
122
- parameters: {
123
- type: "object",
124
- properties: {
125
- group: {
126
- type: "string",
127
- description: "Motion group from the loaded Live2D model.",
128
- enum: motionGroups
129
- },
130
- index: {
131
- type: "number",
132
- description: "Zero-based motion index within the selected motion group."
133
- }
134
- },
135
- required: ["group", "index"]
136
- }
137
- },
138
- async handler(args) {
139
- const group = args.group;
140
- const index = args.index;
141
- if (typeof group !== "string" || !motionGroups.includes(group)) {
142
- throw new Error(
143
- 'playMotion requires a valid "group" from the model catalog'
144
- );
145
- }
146
- if (!Number.isInteger(index)) {
147
- throw new Error('playMotion requires an integer "index"');
148
- }
149
- const count = catalog.motions[group] ?? 0;
150
- const motionIndex = index;
151
- if (motionIndex < 0 || motionIndex >= count) {
152
- throw new Error(
153
- `playMotion index ${String(motionIndex)} is out of range for group "${group}"`
154
- );
155
- }
156
- return {
157
- success: true,
158
- group,
159
- index: motionIndex
160
- };
161
- }
162
- });
163
- }
164
- tools.push({
165
- definition: {
166
- type: "function",
167
- name: LOOK_AT_TOOL_NAME,
168
- description: LOOK_AT_TOOL_DESCRIPTION,
169
- parameters: {
170
- type: "object",
171
- properties: {
172
- x: {
173
- type: "number",
174
- description: "Normalized horizontal gaze target. -1 looks left, 0 is center, 1 looks right."
175
- },
176
- y: {
177
- type: "number",
178
- description: "Normalized vertical gaze target. -1 looks down, 0 is center, 1 looks up."
179
- }
180
- },
181
- required: ["x", "y"]
182
- }
183
- },
184
- async handler(args) {
185
- const x = args.x;
186
- const y = args.y;
187
- if (typeof x !== "number" || Number.isNaN(x)) {
188
- throw new Error('lookAt requires a numeric "x"');
189
- }
190
- if (typeof y !== "number" || Number.isNaN(y)) {
191
- throw new Error('lookAt requires a numeric "y"');
192
- }
193
- return {
194
- success: true,
195
- x: clamp(x, MIN_GAZE, MAX_GAZE),
196
- y: clamp(y, MIN_GAZE, MAX_GAZE)
197
- };
198
- }
199
- });
200
- return tools;
201
- }
202
- function clamp(value, min, max) {
203
- return Math.max(min, Math.min(max, value));
204
- }
205
-
206
62
  // src/realtime-manager.ts
207
63
  var DEFAULT_TOOL_TIMEOUT_MS = 1e4;
208
64
  var RECONNECT_DELAYS_MS = [500, 1e3, 2e3, 4e3, 5e3];
@@ -478,6 +334,7 @@ var RealtimeManagerImpl = class {
478
334
  return;
479
335
  }
480
336
  const text = event.text || this.state.response.text;
337
+ this.emitUsageEvent(event);
481
338
  this.state = {
482
339
  ...this.state,
483
340
  response: {
@@ -552,14 +409,22 @@ var RealtimeManagerImpl = class {
552
409
  return;
553
410
  }
554
411
  try {
412
+ this.log("debug", "Realtime tool execution started", {
413
+ name: event.name,
414
+ callId: event.callId
415
+ });
555
416
  const output = await this.runToolHandler(tool, event);
556
417
  await this.client.sendToolResult(event.callId, output);
557
- this.postProcessToolResult(event.name, output);
558
418
  this.eventEmitter?.emit("realtime:tool:result", {
559
419
  name: event.name,
560
420
  output,
561
421
  callId: event.callId
562
422
  });
423
+ this.projectToolResult(event.name, output, event.callId);
424
+ this.log("info", "Realtime tool execution succeeded", {
425
+ name: event.name,
426
+ callId: event.callId
427
+ });
563
428
  } catch (error) {
564
429
  await this.handleToolExecutionFailure(
565
430
  event.name,
@@ -598,39 +463,33 @@ var RealtimeManagerImpl = class {
598
463
  );
599
464
  }
600
465
  }
601
- postProcessToolResult(name, output) {
602
- switch (name) {
603
- case SET_EXPRESSION_TOOL_NAME: {
604
- const expressionId = output.expressionId;
605
- if (typeof expressionId === "string") {
606
- this.eventEmitter?.emit("realtime:expression", { expressionId });
607
- }
608
- return;
609
- }
610
- case PLAY_MOTION_TOOL_NAME: {
611
- const group = output.group;
612
- const index = output.index;
613
- if (typeof group === "string" && Number.isInteger(index)) {
614
- const motionIndex = index;
615
- this.eventEmitter?.emit("realtime:motion", {
616
- group,
617
- index: motionIndex
618
- });
619
- }
620
- return;
621
- }
622
- case LOOK_AT_TOOL_NAME: {
623
- const coords = readGazeCoordinates(output);
624
- if (coords) {
625
- this.eventEmitter?.emit("realtime:gaze", coords);
626
- }
627
- return;
466
+ projectToolResult(name, output, callId) {
467
+ for (const projector of this.options.resultProjectors ?? []) {
468
+ try {
469
+ projector({
470
+ name,
471
+ output,
472
+ callId,
473
+ emit: (event, payload) => {
474
+ this.eventEmitter?.emit(event, payload);
475
+ }
476
+ });
477
+ } catch (error) {
478
+ const projectorError = error instanceof Error ? error : new Error(String(error));
479
+ this.log("warn", "Realtime result projector failed", {
480
+ name,
481
+ callId,
482
+ error: projectorError.message
483
+ });
628
484
  }
629
- default:
630
- return;
631
485
  }
632
486
  }
633
487
  emitToolError(name, error, callId) {
488
+ this.log("warn", "Realtime tool execution failed", {
489
+ name,
490
+ callId,
491
+ error: error.message
492
+ });
634
493
  this.eventEmitter?.emit("realtime:tool:error", {
635
494
  name,
636
495
  error,
@@ -678,6 +537,10 @@ var RealtimeManagerImpl = class {
678
537
  };
679
538
  }
680
539
  this.emitState();
540
+ this.log("warn", "Realtime reconnect started", {
541
+ cause,
542
+ connection: this.state.connection
543
+ });
681
544
  this.reconnectInFlight = this.runReconnectLoop(reconnectToken, cause).catch((reconnectError) => {
682
545
  this.finalizeFailedSession(reconnectError);
683
546
  }).finally(() => {
@@ -698,6 +561,11 @@ var RealtimeManagerImpl = class {
698
561
  delayMs,
699
562
  cause
700
563
  });
564
+ this.log("info", "Realtime reconnect attempt", {
565
+ attempt,
566
+ delayMs,
567
+ cause
568
+ });
701
569
  await delay(delayMs);
702
570
  if (!this.isReconnectTokenActive(reconnectToken)) {
703
571
  return;
@@ -726,6 +594,11 @@ var RealtimeManagerImpl = class {
726
594
  totalMs: Date.now() - startedAt,
727
595
  cause
728
596
  });
597
+ this.log("info", "Realtime reconnect succeeded", {
598
+ attempts: attempt,
599
+ totalMs: Date.now() - startedAt,
600
+ cause
601
+ });
729
602
  return;
730
603
  } catch (error) {
731
604
  lastError = error instanceof Error ? error : new Error(String(error));
@@ -746,6 +619,12 @@ var RealtimeManagerImpl = class {
746
619
  cause,
747
620
  lastError
748
621
  });
622
+ this.log("error", "Realtime reconnect exhausted", {
623
+ attempts: RECONNECT_DELAYS_MS.length,
624
+ totalMs: Date.now() - startedAt,
625
+ cause,
626
+ error: lastError.message
627
+ });
749
628
  throw lastError;
750
629
  }
751
630
  cancelReconnectLoop() {
@@ -771,6 +650,10 @@ var RealtimeManagerImpl = class {
771
650
  };
772
651
  this.eventEmitter?.emit("realtime:error", { error: typedError });
773
652
  this.emitState();
653
+ this.log("error", "Realtime transport error surfaced", {
654
+ connection,
655
+ error: typedError.message
656
+ });
774
657
  }
775
658
  emitAudioStart() {
776
659
  if (this.isAudioPlaybackActive) {
@@ -916,12 +799,20 @@ var RealtimeManagerImpl = class {
916
799
  lastError: null
917
800
  };
918
801
  this.emitState();
802
+ this.log("debug", "Realtime session patch started", {
803
+ voice: sessionConfig.voice,
804
+ model: sessionConfig.model
805
+ });
919
806
  try {
920
807
  await this.client.updateSession(sessionConfig);
921
808
  if (this.stopRequestedDuringRefresh) {
922
809
  return;
923
810
  }
924
811
  this.commitPatchedSession(sessionConfig);
812
+ this.log("info", "Realtime session patch succeeded", {
813
+ voice: sessionConfig.voice,
814
+ model: sessionConfig.model
815
+ });
925
816
  } catch (error) {
926
817
  const refreshError = error instanceof Error ? error : new Error(String(error));
927
818
  const typedRefreshError = core.toCharivoError("transport", refreshError);
@@ -931,18 +822,29 @@ var RealtimeManagerImpl = class {
931
822
  if (this.state.lastError !== typedRefreshError && this.state.lastError?.cause !== refreshError) {
932
823
  this.applyError(typedRefreshError, "connected");
933
824
  }
825
+ this.log("error", "Realtime session patch failed", {
826
+ error: typedRefreshError.message
827
+ });
934
828
  throw typedRefreshError;
935
829
  } finally {
936
830
  this.isRefreshingSession = false;
937
831
  }
938
832
  }
939
833
  emitSessionStart(reason, state = this.getState()) {
834
+ this.log("info", "Realtime session started", {
835
+ reason,
836
+ connection: state.connection
837
+ });
940
838
  this.eventEmitter?.emit("realtime:session:start", {
941
839
  state,
942
840
  reason
943
841
  });
944
842
  }
945
843
  emitSessionEnd(reason, state = this.getState()) {
844
+ this.log("info", "Realtime session ended", {
845
+ reason,
846
+ connection: state.connection
847
+ });
946
848
  this.eventEmitter?.emit("realtime:session:end", {
947
849
  state,
948
850
  reason
@@ -970,6 +872,9 @@ var RealtimeManagerImpl = class {
970
872
  };
971
873
  this.eventEmitter?.emit("realtime:error", { error });
972
874
  this.emitState();
875
+ this.log("error", "Realtime session failed", {
876
+ error: error.message
877
+ });
973
878
  }
974
879
  finalizeStoppedSession(emitSessionEnd, reason) {
975
880
  this.cancelReconnectLoop();
@@ -996,15 +901,25 @@ var RealtimeManagerImpl = class {
996
901
  this.emitSessionEnd(reason);
997
902
  }
998
903
  }
999
- };
1000
- function readGazeCoordinates(output) {
1001
- const x = output.x;
1002
- const y = output.y;
1003
- if (typeof x !== "number" || typeof y !== "number") {
1004
- return null;
904
+ emitUsageEvent(event) {
905
+ if (!event.usage) {
906
+ return;
907
+ }
908
+ const payload = {
909
+ usage: event.usage,
910
+ model: event.model,
911
+ responseId: event.responseId
912
+ };
913
+ this.eventEmitter?.emit("realtime:usage", payload);
914
+ this.log("debug", "Realtime usage reported", {
915
+ model: event.model,
916
+ responseId: event.responseId
917
+ });
1005
918
  }
1006
- return { x, y };
1007
- }
919
+ log(level, message, context) {
920
+ this.options.logger?.[level]?.(message, context);
921
+ }
922
+ };
1008
923
  function createRealtimeManager(client, options) {
1009
924
  return new RealtimeManagerImpl(client, options);
1010
925
  }
@@ -1069,14 +984,9 @@ function delay(ms) {
1069
984
  });
1070
985
  }
1071
986
 
1072
- exports.AVATAR_CONTROL_TOOL_NAMES = AVATAR_CONTROL_TOOL_NAMES;
1073
987
  exports.DEFAULT_REALTIME_AGENT_INSTRUCTIONS = DEFAULT_REALTIME_AGENT_INSTRUCTIONS;
1074
- exports.LOOK_AT_TOOL_NAME = LOOK_AT_TOOL_NAME;
1075
- exports.PLAY_MOTION_TOOL_NAME = PLAY_MOTION_TOOL_NAME;
1076
988
  exports.RealtimeManagerImpl = RealtimeManagerImpl;
1077
- exports.SET_EXPRESSION_TOOL_NAME = SET_EXPRESSION_TOOL_NAME;
1078
989
  exports.buildRealtimeSessionConfig = buildRealtimeSessionConfig;
1079
- exports.createAvatarControlTools = createAvatarControlTools;
1080
990
  exports.createRealtimeManager = createRealtimeManager;
1081
991
  //# sourceMappingURL=index.js.map
1082
992
  //# sourceMappingURL=index.js.map