@eka-care/medassist-core 1.0.65 → 1.0.67

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (147) hide show
  1. package/dist/Synapse.d.ts +0 -1
  2. package/dist/Synapse.js +2 -25
  3. package/dist/connection/ConnectionFactory.d.ts +0 -1
  4. package/dist/connection/SSE.d.ts +0 -1
  5. package/dist/connection/Websocket.d.ts +0 -1
  6. package/dist/constants/index.d.ts +0 -1
  7. package/dist/constants/types.d.ts +0 -1
  8. package/dist/conversation.d.ts +0 -1
  9. package/dist/esm/Synapse.js +612 -0
  10. package/dist/esm/connection/ConnectionFactory.js +27 -0
  11. package/dist/esm/connection/SSE.js +212 -0
  12. package/dist/esm/connection/Websocket.js +178 -0
  13. package/dist/esm/constants/index.js +25 -0
  14. package/dist/esm/constants/types.js +1 -0
  15. package/dist/esm/conversation.js +7 -0
  16. package/dist/esm/events/Events.js +41 -0
  17. package/dist/esm/events/Incoming.js +1 -0
  18. package/dist/esm/events/Outgoing.js +1 -0
  19. package/dist/esm/events/index.js +2 -0
  20. package/dist/esm/events/types.js +5 -0
  21. package/dist/esm/index.js +34 -0
  22. package/dist/esm/internal/Api/BaseResource.js +50 -0
  23. package/dist/esm/internal/Api/HttpClient.js +131 -0
  24. package/dist/esm/internal/Api/types.js +1 -0
  25. package/dist/esm/internal/Error/Error.js +229 -0
  26. package/dist/esm/internal/Error/types.js +9 -0
  27. package/dist/esm/internal/connection/BaseConnection.js +134 -0
  28. package/dist/esm/internal/connection/types.js +17 -0
  29. package/dist/esm/internal/events/EventEmitter.js +26 -0
  30. package/dist/esm/internal/store/index.js +5 -0
  31. package/dist/esm/media/audio/Audio.copy.js +363 -0
  32. package/dist/esm/media/audio/Audio.js +310 -0
  33. package/dist/esm/media/audio/types.js +13 -0
  34. package/dist/esm/media/file/File.js +159 -0
  35. package/dist/esm/messages/MessageManager.js +476 -0
  36. package/dist/esm/messages/types.js +35 -0
  37. package/dist/esm/resources/config/Config.js +11 -0
  38. package/dist/esm/resources/feedback/Feedback.js +9 -0
  39. package/dist/esm/resources/feedback/types.js +7 -0
  40. package/dist/esm/resources/index.js +152 -0
  41. package/dist/esm/resources/session/Session.js +44 -0
  42. package/dist/esm/resources/session/types.js +5 -0
  43. package/dist/esm/resources/toolCall/ToolCall.js +12 -0
  44. package/dist/esm/resources/toolCall/types.js +34 -0
  45. package/dist/esm/resources/types.js +4 -0
  46. package/dist/esm/resources/voice/VoiceResource.js +14 -0
  47. package/dist/esm/resources/voice/types.js +1 -0
  48. package/dist/esm/types/index.js +8 -0
  49. package/dist/esm/utils/Error.js +110 -0
  50. package/dist/esm/voice/VoiceAgent.js +305 -0
  51. package/dist/esm/voice/VoiceAudioAnalyser.js +32 -0
  52. package/dist/esm/voice/index.js +1 -0
  53. package/dist/esm/voice/types.js +15 -0
  54. package/dist/events/Events.d.ts +0 -1
  55. package/dist/events/Incoming.d.ts +0 -1
  56. package/dist/events/Outgoing.d.ts +0 -1
  57. package/dist/events/index.d.ts +0 -1
  58. package/dist/events/types.d.ts +0 -1
  59. package/dist/index.d.ts +0 -1
  60. package/dist/internal/Api/BaseResource.d.ts +0 -1
  61. package/dist/internal/Api/HttpClient.d.ts +0 -1
  62. package/dist/internal/Api/types.d.ts +0 -1
  63. package/dist/internal/Error/Error.d.ts +0 -1
  64. package/dist/internal/Error/types.d.ts +0 -1
  65. package/dist/internal/connection/BaseConnection.d.ts +0 -1
  66. package/dist/internal/connection/types.d.ts +0 -1
  67. package/dist/internal/events/EventEmitter.d.ts +0 -1
  68. package/dist/internal/store/index.d.ts +0 -1
  69. package/dist/media/audio/Audio.copy.d.ts +0 -1
  70. package/dist/media/audio/Audio.d.ts +0 -1
  71. package/dist/media/audio/types.d.ts +0 -1
  72. package/dist/media/file/File.d.ts +0 -1
  73. package/dist/messages/MessageManager.d.ts +4 -2
  74. package/dist/messages/MessageManager.js +29 -5
  75. package/dist/messages/types.d.ts +0 -1
  76. package/dist/resources/config/Config.d.ts +0 -1
  77. package/dist/resources/feedback/Feedback.d.ts +0 -1
  78. package/dist/resources/feedback/types.d.ts +0 -1
  79. package/dist/resources/index.d.ts +0 -1
  80. package/dist/resources/session/Session.d.ts +0 -1
  81. package/dist/resources/session/types.d.ts +0 -1
  82. package/dist/resources/toolCall/ToolCall.d.ts +0 -1
  83. package/dist/resources/toolCall/types.d.ts +0 -1
  84. package/dist/resources/types.d.ts +0 -1
  85. package/dist/resources/voice/VoiceResource.d.ts +0 -1
  86. package/dist/resources/voice/types.d.ts +0 -1
  87. package/dist/types/index.d.ts +0 -1
  88. package/dist/utils/Error.d.ts +0 -1
  89. package/dist/voice/VoiceAgent.d.ts +0 -1
  90. package/dist/voice/VoiceAudioAnalyser.d.ts +0 -1
  91. package/dist/voice/index.d.ts +0 -1
  92. package/dist/voice/types.d.ts +0 -1
  93. package/package.json +4 -2
  94. package/dist/Synapse.d.ts.map +0 -1
  95. package/dist/auth/constants.d.ts +0 -12
  96. package/dist/auth/constants.d.ts.map +0 -1
  97. package/dist/auth/constants.js +0 -10
  98. package/dist/auth/index.d.ts +0 -3
  99. package/dist/auth/index.d.ts.map +0 -1
  100. package/dist/auth/index.js +0 -18
  101. package/dist/auth/session.d.ts +0 -5
  102. package/dist/auth/session.d.ts.map +0 -1
  103. package/dist/auth/session.js +0 -36
  104. package/dist/connection/ConnectionFactory.d.ts.map +0 -1
  105. package/dist/connection/SSE.d.ts.map +0 -1
  106. package/dist/connection/Websocket.d.ts.map +0 -1
  107. package/dist/constants/index.d.ts.map +0 -1
  108. package/dist/constants/types.d.ts.map +0 -1
  109. package/dist/conversation.d.ts.map +0 -1
  110. package/dist/events/Events.d.ts.map +0 -1
  111. package/dist/events/Incoming.d.ts.map +0 -1
  112. package/dist/events/Outgoing.d.ts.map +0 -1
  113. package/dist/events/index.d.ts.map +0 -1
  114. package/dist/events/types.d.ts.map +0 -1
  115. package/dist/index.d.ts.map +0 -1
  116. package/dist/internal/Api/BaseResource.d.ts.map +0 -1
  117. package/dist/internal/Api/HttpClient.d.ts.map +0 -1
  118. package/dist/internal/Api/types.d.ts.map +0 -1
  119. package/dist/internal/Error/Error.d.ts.map +0 -1
  120. package/dist/internal/Error/types.d.ts.map +0 -1
  121. package/dist/internal/connection/BaseConnection.d.ts.map +0 -1
  122. package/dist/internal/connection/types.d.ts.map +0 -1
  123. package/dist/internal/events/EventEmitter.d.ts.map +0 -1
  124. package/dist/internal/store/index.d.ts.map +0 -1
  125. package/dist/media/audio/Audio.copy.d.ts.map +0 -1
  126. package/dist/media/audio/Audio.d.ts.map +0 -1
  127. package/dist/media/audio/types.d.ts.map +0 -1
  128. package/dist/media/file/File.d.ts.map +0 -1
  129. package/dist/messages/MessageManager.d.ts.map +0 -1
  130. package/dist/messages/types.d.ts.map +0 -1
  131. package/dist/resources/config/Config.d.ts.map +0 -1
  132. package/dist/resources/feedback/Feedback.d.ts.map +0 -1
  133. package/dist/resources/feedback/types.d.ts.map +0 -1
  134. package/dist/resources/index.d.ts.map +0 -1
  135. package/dist/resources/session/Session.d.ts.map +0 -1
  136. package/dist/resources/session/types.d.ts.map +0 -1
  137. package/dist/resources/toolCall/ToolCall.d.ts.map +0 -1
  138. package/dist/resources/toolCall/types.d.ts.map +0 -1
  139. package/dist/resources/types.d.ts.map +0 -1
  140. package/dist/resources/voice/VoiceResource.d.ts.map +0 -1
  141. package/dist/resources/voice/types.d.ts.map +0 -1
  142. package/dist/types/index.d.ts.map +0 -1
  143. package/dist/utils/Error.d.ts.map +0 -1
  144. package/dist/voice/VoiceAgent.d.ts.map +0 -1
  145. package/dist/voice/VoiceAudioAnalyser.d.ts.map +0 -1
  146. package/dist/voice/index.d.ts.map +0 -1
  147. package/dist/voice/types.d.ts.map +0 -1
@@ -0,0 +1,305 @@
1
+ import { EventEmitter } from "../internal/events/EventEmitter";
2
+ import { VoiceAudioAnalyser } from "./VoiceAudioAnalyser";
3
+ import { APIError } from "../internal/Error/Error";
4
+ import { VOICE_AGENT_STATE, VOICE_AGENT_EVENTS, } from "./types";
5
+ const DEFAULT_ICE_GATHERING_TIMEOUT_MS = 3000;
6
+ const DEFAULT_SPEAKING_THRESHOLD = 8;
7
+ const ICE_SERVERS_CACHE_TTL_MS = 60 * 60 * 1000; // 60 minutes
8
+ const USER_SILENCE_DEBOUNCE_MS = 600;
9
+ function waitForIceGathering(pc, timeoutMs) {
10
+ return new Promise((resolve) => {
11
+ if (pc.iceGatheringState === "complete") {
12
+ resolve();
13
+ return;
14
+ }
15
+ const timeout = setTimeout(resolve, timeoutMs);
16
+ pc.onicegatheringstatechange = () => {
17
+ if (pc.iceGatheringState === "complete") {
18
+ clearTimeout(timeout);
19
+ resolve();
20
+ }
21
+ };
22
+ });
23
+ }
24
+ export class VoiceAgent {
25
+ config;
26
+ resourceManager;
27
+ getCredentials;
28
+ emitter = new EventEmitter();
29
+ remoteAnalyser = new VoiceAudioAnalyser();
30
+ localAnalyser = new VoiceAudioAnalyser();
31
+ pc = null;
32
+ pcId = null;
33
+ dataChannel = null;
34
+ localStream = null;
35
+ remoteAudio = null;
36
+ _state = VOICE_AGENT_STATE.IDLE;
37
+ _isMuted = false;
38
+ manuallyDisconnected = false;
39
+ _hasConnectionFailureNotified = false;
40
+ _userSpeaking = false;
41
+ _silenceTimer = null;
42
+ _cachedIceServers = null;
43
+ _iceServersCachedAt = 0;
44
+ constructor(config, resourceManager, getCredentials) {
45
+ this.config = config;
46
+ this.resourceManager = resourceManager;
47
+ this.getCredentials = getCredentials;
48
+ }
49
+ get state() {
50
+ return this._state;
51
+ }
52
+ get isMuted() {
53
+ return this._isMuted;
54
+ }
55
+ on(event, listener) {
56
+ this.emitter.on(event, listener);
57
+ }
58
+ off(event, listener) {
59
+ this.emitter.off(event, listener);
60
+ }
61
+ async connect() {
62
+ const credentials = await this.getCredentials();
63
+ this.setState(VOICE_AGENT_STATE.CONNECTING);
64
+ this._hasConnectionFailureNotified = false;
65
+ try {
66
+ if (!navigator.mediaDevices?.getUserMedia) {
67
+ throw new Error("Microphone access is not available. Please ensure you are using a secure (HTTPS) connection.");
68
+ }
69
+ const stream = await navigator.mediaDevices.getUserMedia({
70
+ audio: {
71
+ channelCount: 1,
72
+ echoCancellation: true,
73
+ noiseSuppression: true,
74
+ autoGainControl: true,
75
+ sampleRate: 16000,
76
+ },
77
+ video: false,
78
+ });
79
+ this.localStream = stream;
80
+ const iceServers = await this.fetchIceServers(credentials);
81
+ const pc = new RTCPeerConnection({ iceServers, iceTransportPolicy: "all" });
82
+ this.pc = pc;
83
+ // Data channel required by SmallWebRTCTransport — without it the server
84
+ // connects ICE but drops queued greeting/audio.
85
+ this.dataChannel = pc.createDataChannel("pipecat-events", { ordered: true });
86
+ this.dataChannel.onopen = () => console.log("data channel open");
87
+ this.dataChannel.onclose = () => this.handleConnectionFailure(new Error("Data channel closed"), "Voice connection failed");
88
+ this.dataChannel.onerror = () => this.handleConnectionFailure(new Error("Data channel error"), "Voice connection failed");
89
+ stream.getAudioTracks().forEach((track) => pc.addTrack(track, stream));
90
+ pc.ontrack = (event) => this.handleRemoteTrack(event);
91
+ pc.onconnectionstatechange = () => {
92
+ if (pc.connectionState === "connected") {
93
+ this.setState(VOICE_AGENT_STATE.LISTENING);
94
+ this.startLocalAudioAnalysis(stream);
95
+ }
96
+ else if (pc.connectionState === "failed" || pc.connectionState === "closed") {
97
+ this.releaseResources();
98
+ this.setState(VOICE_AGENT_STATE.IDLE);
99
+ }
100
+ };
101
+ pc.oniceconnectionstatechange = () => {
102
+ if ((pc.iceConnectionState === "connected" || pc.iceConnectionState === "completed") &&
103
+ this._state === VOICE_AGENT_STATE.CONNECTING) {
104
+ this.setState(VOICE_AGENT_STATE.LISTENING);
105
+ this.startLocalAudioAnalysis(stream);
106
+ }
107
+ else if (pc.iceConnectionState === "failed") {
108
+ this.releaseResources();
109
+ this.setState(VOICE_AGENT_STATE.IDLE);
110
+ }
111
+ };
112
+ const offer = await pc.createOffer();
113
+ await pc.setLocalDescription(offer);
114
+ const timeoutMs = this.config.iceGatheringTimeoutMs ?? DEFAULT_ICE_GATHERING_TIMEOUT_MS;
115
+ await waitForIceGathering(pc, timeoutMs);
116
+ const buildOfferBody = (creds) => ({
117
+ sdp: pc.localDescription.sdp,
118
+ type: pc.localDescription.type,
119
+ pc_id: this.pcId,
120
+ request_data: {
121
+ session_id: creds.sessionId,
122
+ token: creds.sessionToken,
123
+ },
124
+ });
125
+ let answer;
126
+ try {
127
+ answer = await this.resourceManager.sendVoiceOffer(buildOfferBody(credentials));
128
+ }
129
+ catch (err) {
130
+ if (err instanceof APIError && err.status === 401 && this.config.handleRefresh) {
131
+ await this.config.handleRefresh();
132
+ const refreshed = await this.getCredentials();
133
+ // If the retry also fails, let it propagate to the outer catch → handleConnectionFailure
134
+ answer = await this.resourceManager.sendVoiceOffer(buildOfferBody(refreshed));
135
+ }
136
+ else {
137
+ throw err;
138
+ }
139
+ }
140
+ if (!answer) {
141
+ throw new Error("No answer received from voice offer");
142
+ }
143
+ this.pcId = answer.pc_id;
144
+ if (pc.signalingState === "closed")
145
+ return;
146
+ await pc.setRemoteDescription(new RTCSessionDescription({ sdp: answer.sdp, type: answer.type }));
147
+ }
148
+ catch (err) {
149
+ this.handleConnectionFailure(err);
150
+ }
151
+ }
152
+ disconnect() {
153
+ this.manuallyDisconnected = true;
154
+ this.setState(VOICE_AGENT_STATE.DISCONNECTING);
155
+ this.releaseResources();
156
+ this.setState(VOICE_AGENT_STATE.IDLE);
157
+ }
158
+ toggleMute() {
159
+ if (!this.localStream)
160
+ return;
161
+ const track = this.localStream.getAudioTracks()[0];
162
+ if (!track)
163
+ return;
164
+ const wasEnabled = track.enabled;
165
+ this.localStream.getAudioTracks().forEach((t) => { t.enabled = !wasEnabled; });
166
+ this._isMuted = wasEnabled;
167
+ this.clearSilenceTimer();
168
+ this._userSpeaking = false;
169
+ this.setState(wasEnabled ? VOICE_AGENT_STATE.MUTED : VOICE_AGENT_STATE.LISTENING);
170
+ }
171
+ reset() {
172
+ this.setState(VOICE_AGENT_STATE.IDLE);
173
+ }
174
+ destroy() {
175
+ this.releaseResources();
176
+ this.emitter.removeAllListeners();
177
+ this._state = VOICE_AGENT_STATE.IDLE;
178
+ }
179
+ // ──────────────────────────── private ────────────────────────────
180
+ async fetchIceServers(credentials) {
181
+ if (this.config.iceServers)
182
+ return this.config.iceServers;
183
+ const now = Date.now();
184
+ if (this._cachedIceServers && now - this._iceServersCachedAt < ICE_SERVERS_CACHE_TTL_MS) {
185
+ return this._cachedIceServers;
186
+ }
187
+ let resp;
188
+ try {
189
+ resp = await this.resourceManager.getVoiceIceServers(credentials.sessionId, credentials.sessionToken);
190
+ }
191
+ catch (err) {
192
+ if (err instanceof APIError && err.status === 401 && this.config.handleRefresh) {
193
+ await this.config.handleRefresh();
194
+ const refreshed = await this.getCredentials();
195
+ // If the retry also fails, propagate — caller's outer catch → handleConnectionFailure
196
+ resp = await this.resourceManager.getVoiceIceServers(refreshed.sessionId, refreshed.sessionToken);
197
+ }
198
+ else {
199
+ throw err;
200
+ }
201
+ }
202
+ if (!Array.isArray(resp?.ice_servers) || resp.ice_servers.length === 0) {
203
+ throw new Error("ICE servers response is empty");
204
+ }
205
+ this._cachedIceServers = resp.ice_servers;
206
+ this._iceServersCachedAt = Date.now();
207
+ return this._cachedIceServers;
208
+ }
209
+ setState(next) {
210
+ this._state = next;
211
+ this.emitter.emit(VOICE_AGENT_EVENTS.STATE_CHANGED, next);
212
+ }
213
+ clearSilenceTimer() {
214
+ if (this._silenceTimer !== null) {
215
+ clearTimeout(this._silenceTimer);
216
+ this._silenceTimer = null;
217
+ }
218
+ }
219
+ handleConnectionFailure(cause, message = "Voice connection failed") {
220
+ if (this._hasConnectionFailureNotified)
221
+ return;
222
+ if (this._state === VOICE_AGENT_STATE.DISCONNECTING || this.manuallyDisconnected)
223
+ return;
224
+ this._hasConnectionFailureNotified = true;
225
+ console.error("voice connection failure", cause);
226
+ this.releaseResources();
227
+ this.setState(VOICE_AGENT_STATE.ERROR);
228
+ const voiceError = { message, cause };
229
+ this.emitter.emit(VOICE_AGENT_EVENTS.ERROR, voiceError);
230
+ }
231
+ releaseResources() {
232
+ this.remoteAnalyser.stop();
233
+ this.localAnalyser.stop();
234
+ this.clearSilenceTimer();
235
+ this._userSpeaking = false;
236
+ if (this.localStream) {
237
+ this.localStream.getTracks().forEach((t) => t.stop());
238
+ this.localStream = null;
239
+ }
240
+ if (this.dataChannel) {
241
+ this.dataChannel.close();
242
+ this.dataChannel = null;
243
+ }
244
+ if (this.pc) {
245
+ this.pc.onconnectionstatechange = null;
246
+ this.pc.oniceconnectionstatechange = null;
247
+ this.pc.ontrack = null;
248
+ this.pc.onicecandidate = null;
249
+ this.pc.ondatachannel = null;
250
+ this.pc.close();
251
+ this.pc = null;
252
+ }
253
+ this.pcId = null;
254
+ this._isMuted = false;
255
+ if (this.remoteAudio) {
256
+ this.remoteAudio.srcObject = null;
257
+ }
258
+ }
259
+ startLocalAudioAnalysis(stream) {
260
+ const threshold = this.config.speakingThreshold ?? DEFAULT_SPEAKING_THRESHOLD;
261
+ this.localAnalyser.start(stream, threshold, (isUserSpeaking) => {
262
+ if (this._state !== VOICE_AGENT_STATE.LISTENING &&
263
+ this._state !== VOICE_AGENT_STATE.THINKING) {
264
+ return;
265
+ }
266
+ if (isUserSpeaking) {
267
+ this._userSpeaking = true;
268
+ this.clearSilenceTimer();
269
+ if (this._state === VOICE_AGENT_STATE.THINKING) {
270
+ this.setState(VOICE_AGENT_STATE.LISTENING);
271
+ }
272
+ }
273
+ else if (this._userSpeaking && this._silenceTimer === null) {
274
+ this._silenceTimer = setTimeout(() => {
275
+ this._silenceTimer = null;
276
+ if (this._userSpeaking && this._state === VOICE_AGENT_STATE.LISTENING) {
277
+ this._userSpeaking = false;
278
+ this.setState(VOICE_AGENT_STATE.THINKING);
279
+ }
280
+ }, USER_SILENCE_DEBOUNCE_MS);
281
+ }
282
+ });
283
+ }
284
+ handleRemoteTrack(event) {
285
+ if (!this.remoteAudio) {
286
+ this.remoteAudio = new Audio();
287
+ this.remoteAudio.autoplay = true;
288
+ }
289
+ const remoteStream = event.streams[0];
290
+ this.remoteAudio.srcObject = remoteStream;
291
+ const threshold = this.config.speakingThreshold ?? DEFAULT_SPEAKING_THRESHOLD;
292
+ this.remoteAnalyser.start(remoteStream, threshold, (isSpeaking) => {
293
+ if (isSpeaking) {
294
+ this.clearSilenceTimer();
295
+ this._userSpeaking = false;
296
+ if (this._state !== VOICE_AGENT_STATE.SPEAKING) {
297
+ this.setState(VOICE_AGENT_STATE.SPEAKING);
298
+ }
299
+ }
300
+ else if (this._state === VOICE_AGENT_STATE.SPEAKING) {
301
+ this.setState(VOICE_AGENT_STATE.LISTENING);
302
+ }
303
+ });
304
+ }
305
+ }
@@ -0,0 +1,32 @@
1
+ const DEFAULT_FFT_SIZE = 256;
2
+ export class VoiceAudioAnalyser {
3
+ audioCtx = null;
4
+ animFrame = null;
5
+ start(stream, threshold, onSpeakingChange) {
6
+ const audioCtx = new AudioContext();
7
+ this.audioCtx = audioCtx;
8
+ const analyser = audioCtx.createAnalyser();
9
+ analyser.fftSize = DEFAULT_FFT_SIZE;
10
+ audioCtx.createMediaStreamSource(stream).connect(analyser);
11
+ const data = new Uint8Array(analyser.frequencyBinCount);
12
+ const poll = () => {
13
+ if (!this.audioCtx)
14
+ return;
15
+ analyser.getByteFrequencyData(data);
16
+ const avg = data.reduce((s, v) => s + v, 0) / data.length;
17
+ onSpeakingChange(avg > threshold);
18
+ this.animFrame = requestAnimationFrame(poll);
19
+ };
20
+ poll();
21
+ }
22
+ stop() {
23
+ if (this.animFrame !== null) {
24
+ cancelAnimationFrame(this.animFrame);
25
+ this.animFrame = null;
26
+ }
27
+ if (this.audioCtx) {
28
+ this.audioCtx.close().catch(() => { });
29
+ this.audioCtx = null;
30
+ }
31
+ }
32
+ }
@@ -0,0 +1 @@
1
+ export * from "./types";
@@ -0,0 +1,15 @@
1
+ export const VOICE_AGENT_STATE = {
2
+ IDLE: "idle",
3
+ CONNECTING: "connecting",
4
+ CONNECTED: "connected",
5
+ LISTENING: "listening",
6
+ THINKING: "thinking",
7
+ SPEAKING: "speaking",
8
+ MUTED: "muted",
9
+ DISCONNECTING: "disconnecting",
10
+ ERROR: "error",
11
+ };
12
+ export const VOICE_AGENT_EVENTS = {
13
+ STATE_CHANGED: "voice:state_changed",
14
+ ERROR: "voice:error",
15
+ };
@@ -43,4 +43,3 @@ export type OutgoingSocketStreamMessage = Outgoing.StreamSynapseToMatrixMessage;
43
43
  export type OutgoingSocketEndOfStreamMessage = Outgoing.EndOfStreamSynapseToMatrixMessage;
44
44
  export type OutgoingSocket = Outgoing.PingMessage;
45
45
  export declare function IsValidSocketMesssage(message: any): message is IncomingSocketMessage;
46
- //# sourceMappingURL=Events.d.ts.map
@@ -1,2 +1 @@
1
1
  export type { PingMessage, PongMessage, ErrorMessage, SyncMessage, ConnectionEstablishedMatrixToSynapseMessage, ChatMatrixToSynapseMessage, StreamMatrixToSynapseMessage, EndOfStreamMatrixToSynapseMessage, } from "./types";
2
- //# sourceMappingURL=Incoming.d.ts.map
@@ -1,2 +1 @@
1
1
  export type { PingMessage, PongMessage, ErrorMessage, SyncMessage, AuthSynapseToMatrixMessage, ChatSynapseToMatrixMessage, StreamSynapseToMatrixMessage, EndOfStreamSynapseToMatrixMessage, } from "./types";
2
- //# sourceMappingURL=Outgoing.d.ts.map
@@ -1,3 +1,2 @@
1
1
  export * as Outgoing from "../events/Outgoing";
2
2
  export * as Incoming from "../events/Incoming";
3
- //# sourceMappingURL=index.d.ts.map
@@ -96,4 +96,3 @@ export interface EndOfStreamMatrixToSynapseMessage extends BaseMessage {
96
96
  ev: (typeof SOCKET_EVENTS)["END_OF_STREAM"];
97
97
  ct: (typeof SOCKET_CONTENT_TYPES)["TEXT"];
98
98
  }
99
- //# sourceMappingURL=types.d.ts.map
package/dist/index.d.ts CHANGED
@@ -31,4 +31,3 @@ export type { SendMessageOptions, TSynapseError } from "./types";
31
31
  export { SYNAPSE_MESSAGE_TYPES } from "./messages/types";
32
32
  export * from "./conversation";
33
33
  export * from "./voice";
34
- //# sourceMappingURL=index.d.ts.map
@@ -32,4 +32,3 @@ export declare abstract class BaseResource {
32
32
  */
33
33
  protected buildPath(template: string, params: Record<string, string>): string;
34
34
  }
35
- //# sourceMappingURL=BaseResource.d.ts.map
@@ -22,4 +22,3 @@ export declare class HttpClient {
22
22
  private buildUrl;
23
23
  private handleError;
24
24
  }
25
- //# sourceMappingURL=HttpClient.d.ts.map
@@ -4,4 +4,3 @@ export interface RequestOptions {
4
4
  params?: Record<string, string>;
5
5
  retries?: number;
6
6
  }
7
- //# sourceMappingURL=types.d.ts.map
@@ -99,4 +99,3 @@ export declare class ValidationError extends SynapseError {
99
99
  type SynapseErrorCtor<T extends SynapseError> = new (message: string, options?: SynapseErrorOptions) => T;
100
100
  export declare function normalizeError<T extends SynapseError>(error: unknown, ctor: SynapseErrorCtor<T>, fallbackMessage: string, options?: SynapseErrorOptions): T;
101
101
  export {};
102
- //# sourceMappingURL=Error.d.ts.map
@@ -14,4 +14,3 @@ export interface ErrorContext {
14
14
  sessionId: string;
15
15
  timestamp: string;
16
16
  }
17
- //# sourceMappingURL=types.d.ts.map
@@ -97,4 +97,3 @@ export declare abstract class BaseConnection {
97
97
  */
98
98
  abstract handleConnected(): void;
99
99
  }
100
- //# sourceMappingURL=BaseConnection.d.ts.map
@@ -40,4 +40,3 @@ export type OnMessageCallback = (event: IncomingMessage) => void;
40
40
  export type OnStatusChangeCallback = (status: ConnectionStatus) => void;
41
41
  export type OnOpenCallback = () => void;
42
42
  export type OnErrorCallback = (error: Error) => void;
43
- //# sourceMappingURL=types.d.ts.map
@@ -5,4 +5,3 @@ export declare class EventEmitter {
5
5
  emit(event: string, ...args: unknown[]): void;
6
6
  removeAllListeners(): void;
7
7
  }
8
- //# sourceMappingURL=EventEmitter.d.ts.map
@@ -7,4 +7,3 @@ export declare abstract class Storage {
7
7
  abstract delete(key: string): void;
8
8
  abstract clear(): void;
9
9
  }
10
- //# sourceMappingURL=index.d.ts.map
@@ -41,4 +41,3 @@ export declare class AudioManager {
41
41
  cancel(): void;
42
42
  destroy(): void;
43
43
  }
44
- //# sourceMappingURL=Audio.copy.d.ts.map
@@ -33,4 +33,3 @@ export declare class AudioManager {
33
33
  cancel(): void;
34
34
  destroy(): void;
35
35
  }
36
- //# sourceMappingURL=Audio.d.ts.map
@@ -13,4 +13,3 @@ export interface AudioConfig {
13
13
  }
14
14
  export type AudioDataCallback = (data: AudioMetaData) => void;
15
15
  export type AudioErrorCallback = (error: RecordingError) => void;
16
- //# sourceMappingURL=types.d.ts.map
@@ -38,4 +38,3 @@ export declare class Filemanager {
38
38
  */
39
39
  clearPendingFilesState(): void;
40
40
  }
41
- //# sourceMappingURL=File.d.ts.map
@@ -9,7 +9,9 @@ export declare class MessageManager {
9
9
  private callbacks;
10
10
  private fileManager;
11
11
  private audioManager;
12
- constructor(connection: BaseConnection, callbacks?: SynapseSDKCallbacks);
12
+ private outgoingBuffer;
13
+ constructor(callbacks?: SynapseSDKCallbacks);
14
+ setConnection(connection: BaseConnection): void;
13
15
  /**
14
16
  * Handle file upload process
15
17
  */
@@ -59,6 +61,7 @@ export declare class MessageManager {
59
61
  */
60
62
  sendSocketPongMessage(): void;
61
63
  handleConnectionEstablished(): void;
64
+ private flushOutgoingBuffer;
62
65
  /**
63
66
  * Cleanup message service
64
67
  */
@@ -68,4 +71,3 @@ export declare class MessageManager {
68
71
  private toRecordingError;
69
72
  private assertConnection;
70
73
  }
71
- //# sourceMappingURL=MessageManager.d.ts.map
@@ -11,10 +11,13 @@ class MessageManager {
11
11
  callbacks = null;
12
12
  fileManager = null;
13
13
  audioManager = null;
14
- constructor(connection, callbacks) {
15
- this.connection = connection || null;
14
+ outgoingBuffer = null;
15
+ constructor(callbacks) {
16
16
  this.callbacks = callbacks || null;
17
17
  }
18
+ setConnection(connection) {
19
+ this.connection = connection;
20
+ }
18
21
  /**
19
22
  * Handle file upload process
20
23
  */
@@ -64,7 +67,20 @@ class MessageManager {
64
67
  * send chat message through socket
65
68
  */
66
69
  sendSocketMessage({ message, files, audio, urls, tool_declined, tool_id, tool_result, initial_prompts }) {
67
- const connection = this.assertConnection("sendSocketChatMessage");
70
+ if (!this.connection?.isConnected()) {
71
+ this.outgoingBuffer = {
72
+ ...(message !== undefined && { message }),
73
+ ...(files && { files }),
74
+ ...(audio && { audio }),
75
+ ...(urls && { urls }),
76
+ ...(tool_declined && { tool_declined }),
77
+ ...(tool_id && { tool_id }),
78
+ ...(tool_result && { tool_result }),
79
+ ...(initial_prompts && { initial_prompts }),
80
+ };
81
+ return;
82
+ }
83
+ const connection = this.connection;
68
84
  let outMessage;
69
85
  if ((files && files.length > 0) || (urls && urls.length > 0)) {
70
86
  if (files && files.length > 3) {
@@ -355,7 +371,6 @@ class MessageManager {
355
371
  this.emitError(error);
356
372
  throw error;
357
373
  }
358
- this.assertConnection("startRecording");
359
374
  if (!this.audioManager) {
360
375
  this.audioManager = new Audio_1.AudioManager();
361
376
  }
@@ -406,12 +421,21 @@ class MessageManager {
406
421
  handleConnectionEstablished() {
407
422
  this.connection?.handleConnected(); //this will call the onConnectionStatusChange callback with connected
408
423
  this.connection?.emit(types_1.SYNAPSE_REALTIME_EVENTS.CONNECTED);
409
- // this.connection?.handleConnected();
424
+ this.flushOutgoingBuffer();
425
+ }
426
+ flushOutgoingBuffer() {
427
+ if (!this.connection?.isConnected() || !this.outgoingBuffer)
428
+ return;
429
+ const buf = this.outgoingBuffer;
430
+ this.outgoingBuffer = null;
431
+ this.sendSocketMessage(buf);
410
432
  }
411
433
  /**
412
434
  * Cleanup message service
413
435
  */
414
436
  cleanupMessageServerState() {
437
+ this.outgoingBuffer = null;
438
+ this.connection = null;
415
439
  if (this.fileManager) {
416
440
  this.fileManager.clearPendingFilesState();
417
441
  this.fileManager = null;
@@ -149,4 +149,3 @@ export type TMediaCard = {
149
149
  link?: string;
150
150
  };
151
151
  export type SynapseContentTypes = SOCKET_CONTENT_TYPES;
152
- //# sourceMappingURL=types.d.ts.map
@@ -8,4 +8,3 @@ export declare class Config extends BaseResource {
8
8
  private basePath;
9
9
  retrieve(agentId: string): Promise<AgentConfigResponse>;
10
10
  }
11
- //# sourceMappingURL=Config.d.ts.map
@@ -1 +0,0 @@
1
- //# sourceMappingURL=Feedback.d.ts.map
@@ -1 +0,0 @@
1
- //# sourceMappingURL=types.d.ts.map
@@ -44,4 +44,3 @@ export declare class ResourceManager {
44
44
  getVoiceIceServers(sessionId: string, sessionToken: string): Promise<IceServersResponse>;
45
45
  sendVoiceOffer(body: VoiceOfferRequest): Promise<VoiceOfferResponse>;
46
46
  }
47
- //# sourceMappingURL=index.d.ts.map
@@ -20,4 +20,3 @@ export declare class Session extends BaseResource {
20
20
  feedback(sessionId: string, messageId: string, feedback: USER_FEEDBACK, feedback_reason?: string): Promise<void>;
21
21
  callTool(sessionId: string, toolId: string, messageId: string, sessionToken: string, toolParams?: Record<string, unknown>): Promise<ToolCallResponse>;
22
22
  }
23
- //# sourceMappingURL=Session.d.ts.map
@@ -66,4 +66,3 @@ export declare const USER_FEEDBACK: {
66
66
  readonly NONE: "NONE";
67
67
  };
68
68
  export type USER_FEEDBACK = (typeof USER_FEEDBACK)[keyof typeof USER_FEEDBACK];
69
- //# sourceMappingURL=types.d.ts.map
@@ -4,4 +4,3 @@ export declare class ToolCall extends BaseResource {
4
4
  private basePath;
5
5
  callTool(sessionId: string, toolName: string, toolParams?: Record<string, unknown>): Promise<ToolCallResponse>;
6
6
  }
7
- //# sourceMappingURL=ToolCall.d.ts.map
@@ -143,4 +143,3 @@ export declare enum SYNAPSE_EICITATION_STATUS {
143
143
  }
144
144
  export type ToolCallResponse = StreamMatrixToSynapseMessage & ResourceResponse;
145
145
  export {};
146
- //# sourceMappingURL=types.d.ts.map
@@ -26,4 +26,3 @@ export type ResourceResponse = {
26
26
  msg: string;
27
27
  };
28
28
  };
29
- //# sourceMappingURL=types.d.ts.map
@@ -6,4 +6,3 @@ export declare class VoiceResource extends BaseResource {
6
6
  getIceServers(sessionId: string, sessionToken: string): Promise<IceServersResponse>;
7
7
  sendOffer(body: VoiceOfferRequest): Promise<VoiceOfferResponse>;
8
8
  }
9
- //# sourceMappingURL=VoiceResource.d.ts.map
@@ -16,4 +16,3 @@ export interface VoiceOfferResponse extends ResourceResponse {
16
16
  type: string;
17
17
  pc_id: string;
18
18
  }
19
- //# sourceMappingURL=types.d.ts.map
@@ -75,4 +75,3 @@ export interface ExistingSessionOptions {
75
75
  session_id: string;
76
76
  session_token: string;
77
77
  }
78
- //# sourceMappingURL=index.d.ts.map
@@ -42,4 +42,3 @@ export declare class ErrorUtils {
42
42
  hint?: string;
43
43
  };
44
44
  }
45
- //# sourceMappingURL=Error.d.ts.map
@@ -38,4 +38,3 @@ export declare class VoiceAgent {
38
38
  private startLocalAudioAnalysis;
39
39
  private handleRemoteTrack;
40
40
  }
41
- //# sourceMappingURL=VoiceAgent.d.ts.map
@@ -4,4 +4,3 @@ export declare class VoiceAudioAnalyser {
4
4
  start(stream: MediaStream, threshold: number, onSpeakingChange: (isSpeaking: boolean) => void): void;
5
5
  stop(): void;
6
6
  }
7
- //# sourceMappingURL=VoiceAudioAnalyser.d.ts.map