@eka-care/medassist-core 1.0.66 → 1.0.67
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/Synapse.d.ts +0 -1
- package/dist/connection/ConnectionFactory.d.ts +0 -1
- package/dist/connection/SSE.d.ts +0 -1
- package/dist/connection/Websocket.d.ts +0 -1
- package/dist/constants/index.d.ts +0 -1
- package/dist/constants/types.d.ts +0 -1
- package/dist/conversation.d.ts +0 -1
- package/dist/esm/Synapse.js +612 -0
- package/dist/esm/connection/ConnectionFactory.js +27 -0
- package/dist/esm/connection/SSE.js +212 -0
- package/dist/esm/connection/Websocket.js +178 -0
- package/dist/esm/constants/index.js +25 -0
- package/dist/esm/constants/types.js +1 -0
- package/dist/esm/conversation.js +7 -0
- package/dist/esm/events/Events.js +41 -0
- package/dist/esm/events/Incoming.js +1 -0
- package/dist/esm/events/Outgoing.js +1 -0
- package/dist/esm/events/index.js +2 -0
- package/dist/esm/events/types.js +5 -0
- package/dist/esm/index.js +34 -0
- package/dist/esm/internal/Api/BaseResource.js +50 -0
- package/dist/esm/internal/Api/HttpClient.js +131 -0
- package/dist/esm/internal/Api/types.js +1 -0
- package/dist/esm/internal/Error/Error.js +229 -0
- package/dist/esm/internal/Error/types.js +9 -0
- package/dist/esm/internal/connection/BaseConnection.js +134 -0
- package/dist/esm/internal/connection/types.js +17 -0
- package/dist/esm/internal/events/EventEmitter.js +26 -0
- package/dist/esm/internal/store/index.js +5 -0
- package/dist/esm/media/audio/Audio.copy.js +363 -0
- package/dist/esm/media/audio/Audio.js +310 -0
- package/dist/esm/media/audio/types.js +13 -0
- package/dist/esm/media/file/File.js +159 -0
- package/dist/esm/messages/MessageManager.js +476 -0
- package/dist/esm/messages/types.js +35 -0
- package/dist/esm/resources/config/Config.js +11 -0
- package/dist/esm/resources/feedback/Feedback.js +9 -0
- package/dist/esm/resources/feedback/types.js +7 -0
- package/dist/esm/resources/index.js +152 -0
- package/dist/esm/resources/session/Session.js +44 -0
- package/dist/esm/resources/session/types.js +5 -0
- package/dist/esm/resources/toolCall/ToolCall.js +12 -0
- package/dist/esm/resources/toolCall/types.js +34 -0
- package/dist/esm/resources/types.js +4 -0
- package/dist/esm/resources/voice/VoiceResource.js +14 -0
- package/dist/esm/resources/voice/types.js +1 -0
- package/dist/esm/types/index.js +8 -0
- package/dist/esm/utils/Error.js +110 -0
- package/dist/esm/voice/VoiceAgent.js +305 -0
- package/dist/esm/voice/VoiceAudioAnalyser.js +32 -0
- package/dist/esm/voice/index.js +1 -0
- package/dist/esm/voice/types.js +15 -0
- package/dist/events/Events.d.ts +0 -1
- package/dist/events/Incoming.d.ts +0 -1
- package/dist/events/Outgoing.d.ts +0 -1
- package/dist/events/index.d.ts +0 -1
- package/dist/events/types.d.ts +0 -1
- package/dist/index.d.ts +0 -1
- package/dist/internal/Api/BaseResource.d.ts +0 -1
- package/dist/internal/Api/HttpClient.d.ts +0 -1
- package/dist/internal/Api/types.d.ts +0 -1
- package/dist/internal/Error/Error.d.ts +0 -1
- package/dist/internal/Error/types.d.ts +0 -1
- package/dist/internal/connection/BaseConnection.d.ts +0 -1
- package/dist/internal/connection/types.d.ts +0 -1
- package/dist/internal/events/EventEmitter.d.ts +0 -1
- package/dist/internal/store/index.d.ts +0 -1
- package/dist/media/audio/Audio.copy.d.ts +0 -1
- package/dist/media/audio/Audio.d.ts +0 -1
- package/dist/media/audio/types.d.ts +0 -1
- package/dist/media/file/File.d.ts +0 -1
- package/dist/messages/MessageManager.d.ts +0 -1
- package/dist/messages/types.d.ts +0 -1
- package/dist/resources/config/Config.d.ts +0 -1
- package/dist/resources/feedback/Feedback.d.ts +0 -1
- package/dist/resources/feedback/types.d.ts +0 -1
- package/dist/resources/index.d.ts +0 -1
- package/dist/resources/session/Session.d.ts +0 -1
- package/dist/resources/session/types.d.ts +0 -1
- package/dist/resources/toolCall/ToolCall.d.ts +0 -1
- package/dist/resources/toolCall/types.d.ts +0 -1
- package/dist/resources/types.d.ts +0 -1
- package/dist/resources/voice/VoiceResource.d.ts +0 -1
- package/dist/resources/voice/types.d.ts +0 -1
- package/dist/types/index.d.ts +0 -1
- package/dist/utils/Error.d.ts +0 -1
- package/dist/voice/VoiceAgent.d.ts +0 -1
- package/dist/voice/VoiceAudioAnalyser.d.ts +0 -1
- package/dist/voice/index.d.ts +0 -1
- package/dist/voice/types.d.ts +0 -1
- package/package.json +4 -2
- package/dist/Synapse.d.ts.map +0 -1
- package/dist/auth/constants.d.ts +0 -12
- package/dist/auth/constants.d.ts.map +0 -1
- package/dist/auth/constants.js +0 -10
- package/dist/auth/index.d.ts +0 -3
- package/dist/auth/index.d.ts.map +0 -1
- package/dist/auth/index.js +0 -18
- package/dist/auth/session.d.ts +0 -5
- package/dist/auth/session.d.ts.map +0 -1
- package/dist/auth/session.js +0 -36
- package/dist/connection/ConnectionFactory.d.ts.map +0 -1
- package/dist/connection/SSE.d.ts.map +0 -1
- package/dist/connection/Websocket.d.ts.map +0 -1
- package/dist/constants/index.d.ts.map +0 -1
- package/dist/constants/types.d.ts.map +0 -1
- package/dist/conversation.d.ts.map +0 -1
- package/dist/events/Events.d.ts.map +0 -1
- package/dist/events/Incoming.d.ts.map +0 -1
- package/dist/events/Outgoing.d.ts.map +0 -1
- package/dist/events/index.d.ts.map +0 -1
- package/dist/events/types.d.ts.map +0 -1
- package/dist/index.d.ts.map +0 -1
- package/dist/internal/Api/BaseResource.d.ts.map +0 -1
- package/dist/internal/Api/HttpClient.d.ts.map +0 -1
- package/dist/internal/Api/types.d.ts.map +0 -1
- package/dist/internal/Error/Error.d.ts.map +0 -1
- package/dist/internal/Error/types.d.ts.map +0 -1
- package/dist/internal/connection/BaseConnection.d.ts.map +0 -1
- package/dist/internal/connection/types.d.ts.map +0 -1
- package/dist/internal/events/EventEmitter.d.ts.map +0 -1
- package/dist/internal/store/index.d.ts.map +0 -1
- package/dist/media/audio/Audio.copy.d.ts.map +0 -1
- package/dist/media/audio/Audio.d.ts.map +0 -1
- package/dist/media/audio/types.d.ts.map +0 -1
- package/dist/media/file/File.d.ts.map +0 -1
- package/dist/messages/MessageManager.d.ts.map +0 -1
- package/dist/messages/types.d.ts.map +0 -1
- package/dist/resources/config/Config.d.ts.map +0 -1
- package/dist/resources/feedback/Feedback.d.ts.map +0 -1
- package/dist/resources/feedback/types.d.ts.map +0 -1
- package/dist/resources/index.d.ts.map +0 -1
- package/dist/resources/session/Session.d.ts.map +0 -1
- package/dist/resources/session/types.d.ts.map +0 -1
- package/dist/resources/toolCall/ToolCall.d.ts.map +0 -1
- package/dist/resources/toolCall/types.d.ts.map +0 -1
- package/dist/resources/types.d.ts.map +0 -1
- package/dist/resources/voice/VoiceResource.d.ts.map +0 -1
- package/dist/resources/voice/types.d.ts.map +0 -1
- package/dist/types/index.d.ts.map +0 -1
- package/dist/utils/Error.d.ts.map +0 -1
- package/dist/voice/VoiceAgent.d.ts.map +0 -1
- package/dist/voice/VoiceAudioAnalyser.d.ts.map +0 -1
- package/dist/voice/index.d.ts.map +0 -1
- package/dist/voice/types.d.ts.map +0 -1
|
@@ -0,0 +1,305 @@
|
|
|
1
|
+
import { EventEmitter } from "../internal/events/EventEmitter";
|
|
2
|
+
import { VoiceAudioAnalyser } from "./VoiceAudioAnalyser";
|
|
3
|
+
import { APIError } from "../internal/Error/Error";
|
|
4
|
+
import { VOICE_AGENT_STATE, VOICE_AGENT_EVENTS, } from "./types";
|
|
5
|
+
const DEFAULT_ICE_GATHERING_TIMEOUT_MS = 3000;
|
|
6
|
+
const DEFAULT_SPEAKING_THRESHOLD = 8;
|
|
7
|
+
const ICE_SERVERS_CACHE_TTL_MS = 60 * 60 * 1000; // 60 minutes
|
|
8
|
+
const USER_SILENCE_DEBOUNCE_MS = 600;
|
|
9
|
+
function waitForIceGathering(pc, timeoutMs) {
|
|
10
|
+
return new Promise((resolve) => {
|
|
11
|
+
if (pc.iceGatheringState === "complete") {
|
|
12
|
+
resolve();
|
|
13
|
+
return;
|
|
14
|
+
}
|
|
15
|
+
const timeout = setTimeout(resolve, timeoutMs);
|
|
16
|
+
pc.onicegatheringstatechange = () => {
|
|
17
|
+
if (pc.iceGatheringState === "complete") {
|
|
18
|
+
clearTimeout(timeout);
|
|
19
|
+
resolve();
|
|
20
|
+
}
|
|
21
|
+
};
|
|
22
|
+
});
|
|
23
|
+
}
|
|
24
|
+
export class VoiceAgent {
|
|
25
|
+
config;
|
|
26
|
+
resourceManager;
|
|
27
|
+
getCredentials;
|
|
28
|
+
emitter = new EventEmitter();
|
|
29
|
+
remoteAnalyser = new VoiceAudioAnalyser();
|
|
30
|
+
localAnalyser = new VoiceAudioAnalyser();
|
|
31
|
+
pc = null;
|
|
32
|
+
pcId = null;
|
|
33
|
+
dataChannel = null;
|
|
34
|
+
localStream = null;
|
|
35
|
+
remoteAudio = null;
|
|
36
|
+
_state = VOICE_AGENT_STATE.IDLE;
|
|
37
|
+
_isMuted = false;
|
|
38
|
+
manuallyDisconnected = false;
|
|
39
|
+
_hasConnectionFailureNotified = false;
|
|
40
|
+
_userSpeaking = false;
|
|
41
|
+
_silenceTimer = null;
|
|
42
|
+
_cachedIceServers = null;
|
|
43
|
+
_iceServersCachedAt = 0;
|
|
44
|
+
constructor(config, resourceManager, getCredentials) {
|
|
45
|
+
this.config = config;
|
|
46
|
+
this.resourceManager = resourceManager;
|
|
47
|
+
this.getCredentials = getCredentials;
|
|
48
|
+
}
|
|
49
|
+
get state() {
|
|
50
|
+
return this._state;
|
|
51
|
+
}
|
|
52
|
+
get isMuted() {
|
|
53
|
+
return this._isMuted;
|
|
54
|
+
}
|
|
55
|
+
on(event, listener) {
|
|
56
|
+
this.emitter.on(event, listener);
|
|
57
|
+
}
|
|
58
|
+
off(event, listener) {
|
|
59
|
+
this.emitter.off(event, listener);
|
|
60
|
+
}
|
|
61
|
+
async connect() {
|
|
62
|
+
const credentials = await this.getCredentials();
|
|
63
|
+
this.setState(VOICE_AGENT_STATE.CONNECTING);
|
|
64
|
+
this._hasConnectionFailureNotified = false;
|
|
65
|
+
try {
|
|
66
|
+
if (!navigator.mediaDevices?.getUserMedia) {
|
|
67
|
+
throw new Error("Microphone access is not available. Please ensure you are using a secure (HTTPS) connection.");
|
|
68
|
+
}
|
|
69
|
+
const stream = await navigator.mediaDevices.getUserMedia({
|
|
70
|
+
audio: {
|
|
71
|
+
channelCount: 1,
|
|
72
|
+
echoCancellation: true,
|
|
73
|
+
noiseSuppression: true,
|
|
74
|
+
autoGainControl: true,
|
|
75
|
+
sampleRate: 16000,
|
|
76
|
+
},
|
|
77
|
+
video: false,
|
|
78
|
+
});
|
|
79
|
+
this.localStream = stream;
|
|
80
|
+
const iceServers = await this.fetchIceServers(credentials);
|
|
81
|
+
const pc = new RTCPeerConnection({ iceServers, iceTransportPolicy: "all" });
|
|
82
|
+
this.pc = pc;
|
|
83
|
+
// Data channel required by SmallWebRTCTransport — without it the server
|
|
84
|
+
// connects ICE but drops queued greeting/audio.
|
|
85
|
+
this.dataChannel = pc.createDataChannel("pipecat-events", { ordered: true });
|
|
86
|
+
this.dataChannel.onopen = () => console.log("data channel open");
|
|
87
|
+
this.dataChannel.onclose = () => this.handleConnectionFailure(new Error("Data channel closed"), "Voice connection failed");
|
|
88
|
+
this.dataChannel.onerror = () => this.handleConnectionFailure(new Error("Data channel error"), "Voice connection failed");
|
|
89
|
+
stream.getAudioTracks().forEach((track) => pc.addTrack(track, stream));
|
|
90
|
+
pc.ontrack = (event) => this.handleRemoteTrack(event);
|
|
91
|
+
pc.onconnectionstatechange = () => {
|
|
92
|
+
if (pc.connectionState === "connected") {
|
|
93
|
+
this.setState(VOICE_AGENT_STATE.LISTENING);
|
|
94
|
+
this.startLocalAudioAnalysis(stream);
|
|
95
|
+
}
|
|
96
|
+
else if (pc.connectionState === "failed" || pc.connectionState === "closed") {
|
|
97
|
+
this.releaseResources();
|
|
98
|
+
this.setState(VOICE_AGENT_STATE.IDLE);
|
|
99
|
+
}
|
|
100
|
+
};
|
|
101
|
+
pc.oniceconnectionstatechange = () => {
|
|
102
|
+
if ((pc.iceConnectionState === "connected" || pc.iceConnectionState === "completed") &&
|
|
103
|
+
this._state === VOICE_AGENT_STATE.CONNECTING) {
|
|
104
|
+
this.setState(VOICE_AGENT_STATE.LISTENING);
|
|
105
|
+
this.startLocalAudioAnalysis(stream);
|
|
106
|
+
}
|
|
107
|
+
else if (pc.iceConnectionState === "failed") {
|
|
108
|
+
this.releaseResources();
|
|
109
|
+
this.setState(VOICE_AGENT_STATE.IDLE);
|
|
110
|
+
}
|
|
111
|
+
};
|
|
112
|
+
const offer = await pc.createOffer();
|
|
113
|
+
await pc.setLocalDescription(offer);
|
|
114
|
+
const timeoutMs = this.config.iceGatheringTimeoutMs ?? DEFAULT_ICE_GATHERING_TIMEOUT_MS;
|
|
115
|
+
await waitForIceGathering(pc, timeoutMs);
|
|
116
|
+
const buildOfferBody = (creds) => ({
|
|
117
|
+
sdp: pc.localDescription.sdp,
|
|
118
|
+
type: pc.localDescription.type,
|
|
119
|
+
pc_id: this.pcId,
|
|
120
|
+
request_data: {
|
|
121
|
+
session_id: creds.sessionId,
|
|
122
|
+
token: creds.sessionToken,
|
|
123
|
+
},
|
|
124
|
+
});
|
|
125
|
+
let answer;
|
|
126
|
+
try {
|
|
127
|
+
answer = await this.resourceManager.sendVoiceOffer(buildOfferBody(credentials));
|
|
128
|
+
}
|
|
129
|
+
catch (err) {
|
|
130
|
+
if (err instanceof APIError && err.status === 401 && this.config.handleRefresh) {
|
|
131
|
+
await this.config.handleRefresh();
|
|
132
|
+
const refreshed = await this.getCredentials();
|
|
133
|
+
// If the retry also fails, let it propagate to the outer catch → handleConnectionFailure
|
|
134
|
+
answer = await this.resourceManager.sendVoiceOffer(buildOfferBody(refreshed));
|
|
135
|
+
}
|
|
136
|
+
else {
|
|
137
|
+
throw err;
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
if (!answer) {
|
|
141
|
+
throw new Error("No answer received from voice offer");
|
|
142
|
+
}
|
|
143
|
+
this.pcId = answer.pc_id;
|
|
144
|
+
if (pc.signalingState === "closed")
|
|
145
|
+
return;
|
|
146
|
+
await pc.setRemoteDescription(new RTCSessionDescription({ sdp: answer.sdp, type: answer.type }));
|
|
147
|
+
}
|
|
148
|
+
catch (err) {
|
|
149
|
+
this.handleConnectionFailure(err);
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
disconnect() {
|
|
153
|
+
this.manuallyDisconnected = true;
|
|
154
|
+
this.setState(VOICE_AGENT_STATE.DISCONNECTING);
|
|
155
|
+
this.releaseResources();
|
|
156
|
+
this.setState(VOICE_AGENT_STATE.IDLE);
|
|
157
|
+
}
|
|
158
|
+
toggleMute() {
|
|
159
|
+
if (!this.localStream)
|
|
160
|
+
return;
|
|
161
|
+
const track = this.localStream.getAudioTracks()[0];
|
|
162
|
+
if (!track)
|
|
163
|
+
return;
|
|
164
|
+
const wasEnabled = track.enabled;
|
|
165
|
+
this.localStream.getAudioTracks().forEach((t) => { t.enabled = !wasEnabled; });
|
|
166
|
+
this._isMuted = wasEnabled;
|
|
167
|
+
this.clearSilenceTimer();
|
|
168
|
+
this._userSpeaking = false;
|
|
169
|
+
this.setState(wasEnabled ? VOICE_AGENT_STATE.MUTED : VOICE_AGENT_STATE.LISTENING);
|
|
170
|
+
}
|
|
171
|
+
reset() {
|
|
172
|
+
this.setState(VOICE_AGENT_STATE.IDLE);
|
|
173
|
+
}
|
|
174
|
+
destroy() {
|
|
175
|
+
this.releaseResources();
|
|
176
|
+
this.emitter.removeAllListeners();
|
|
177
|
+
this._state = VOICE_AGENT_STATE.IDLE;
|
|
178
|
+
}
|
|
179
|
+
// ──────────────────────────── private ────────────────────────────
|
|
180
|
+
async fetchIceServers(credentials) {
|
|
181
|
+
if (this.config.iceServers)
|
|
182
|
+
return this.config.iceServers;
|
|
183
|
+
const now = Date.now();
|
|
184
|
+
if (this._cachedIceServers && now - this._iceServersCachedAt < ICE_SERVERS_CACHE_TTL_MS) {
|
|
185
|
+
return this._cachedIceServers;
|
|
186
|
+
}
|
|
187
|
+
let resp;
|
|
188
|
+
try {
|
|
189
|
+
resp = await this.resourceManager.getVoiceIceServers(credentials.sessionId, credentials.sessionToken);
|
|
190
|
+
}
|
|
191
|
+
catch (err) {
|
|
192
|
+
if (err instanceof APIError && err.status === 401 && this.config.handleRefresh) {
|
|
193
|
+
await this.config.handleRefresh();
|
|
194
|
+
const refreshed = await this.getCredentials();
|
|
195
|
+
// If the retry also fails, propagate — caller's outer catch → handleConnectionFailure
|
|
196
|
+
resp = await this.resourceManager.getVoiceIceServers(refreshed.sessionId, refreshed.sessionToken);
|
|
197
|
+
}
|
|
198
|
+
else {
|
|
199
|
+
throw err;
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
if (!Array.isArray(resp?.ice_servers) || resp.ice_servers.length === 0) {
|
|
203
|
+
throw new Error("ICE servers response is empty");
|
|
204
|
+
}
|
|
205
|
+
this._cachedIceServers = resp.ice_servers;
|
|
206
|
+
this._iceServersCachedAt = Date.now();
|
|
207
|
+
return this._cachedIceServers;
|
|
208
|
+
}
|
|
209
|
+
setState(next) {
|
|
210
|
+
this._state = next;
|
|
211
|
+
this.emitter.emit(VOICE_AGENT_EVENTS.STATE_CHANGED, next);
|
|
212
|
+
}
|
|
213
|
+
clearSilenceTimer() {
|
|
214
|
+
if (this._silenceTimer !== null) {
|
|
215
|
+
clearTimeout(this._silenceTimer);
|
|
216
|
+
this._silenceTimer = null;
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
handleConnectionFailure(cause, message = "Voice connection failed") {
|
|
220
|
+
if (this._hasConnectionFailureNotified)
|
|
221
|
+
return;
|
|
222
|
+
if (this._state === VOICE_AGENT_STATE.DISCONNECTING || this.manuallyDisconnected)
|
|
223
|
+
return;
|
|
224
|
+
this._hasConnectionFailureNotified = true;
|
|
225
|
+
console.error("voice connection failure", cause);
|
|
226
|
+
this.releaseResources();
|
|
227
|
+
this.setState(VOICE_AGENT_STATE.ERROR);
|
|
228
|
+
const voiceError = { message, cause };
|
|
229
|
+
this.emitter.emit(VOICE_AGENT_EVENTS.ERROR, voiceError);
|
|
230
|
+
}
|
|
231
|
+
releaseResources() {
|
|
232
|
+
this.remoteAnalyser.stop();
|
|
233
|
+
this.localAnalyser.stop();
|
|
234
|
+
this.clearSilenceTimer();
|
|
235
|
+
this._userSpeaking = false;
|
|
236
|
+
if (this.localStream) {
|
|
237
|
+
this.localStream.getTracks().forEach((t) => t.stop());
|
|
238
|
+
this.localStream = null;
|
|
239
|
+
}
|
|
240
|
+
if (this.dataChannel) {
|
|
241
|
+
this.dataChannel.close();
|
|
242
|
+
this.dataChannel = null;
|
|
243
|
+
}
|
|
244
|
+
if (this.pc) {
|
|
245
|
+
this.pc.onconnectionstatechange = null;
|
|
246
|
+
this.pc.oniceconnectionstatechange = null;
|
|
247
|
+
this.pc.ontrack = null;
|
|
248
|
+
this.pc.onicecandidate = null;
|
|
249
|
+
this.pc.ondatachannel = null;
|
|
250
|
+
this.pc.close();
|
|
251
|
+
this.pc = null;
|
|
252
|
+
}
|
|
253
|
+
this.pcId = null;
|
|
254
|
+
this._isMuted = false;
|
|
255
|
+
if (this.remoteAudio) {
|
|
256
|
+
this.remoteAudio.srcObject = null;
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
startLocalAudioAnalysis(stream) {
|
|
260
|
+
const threshold = this.config.speakingThreshold ?? DEFAULT_SPEAKING_THRESHOLD;
|
|
261
|
+
this.localAnalyser.start(stream, threshold, (isUserSpeaking) => {
|
|
262
|
+
if (this._state !== VOICE_AGENT_STATE.LISTENING &&
|
|
263
|
+
this._state !== VOICE_AGENT_STATE.THINKING) {
|
|
264
|
+
return;
|
|
265
|
+
}
|
|
266
|
+
if (isUserSpeaking) {
|
|
267
|
+
this._userSpeaking = true;
|
|
268
|
+
this.clearSilenceTimer();
|
|
269
|
+
if (this._state === VOICE_AGENT_STATE.THINKING) {
|
|
270
|
+
this.setState(VOICE_AGENT_STATE.LISTENING);
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
else if (this._userSpeaking && this._silenceTimer === null) {
|
|
274
|
+
this._silenceTimer = setTimeout(() => {
|
|
275
|
+
this._silenceTimer = null;
|
|
276
|
+
if (this._userSpeaking && this._state === VOICE_AGENT_STATE.LISTENING) {
|
|
277
|
+
this._userSpeaking = false;
|
|
278
|
+
this.setState(VOICE_AGENT_STATE.THINKING);
|
|
279
|
+
}
|
|
280
|
+
}, USER_SILENCE_DEBOUNCE_MS);
|
|
281
|
+
}
|
|
282
|
+
});
|
|
283
|
+
}
|
|
284
|
+
handleRemoteTrack(event) {
|
|
285
|
+
if (!this.remoteAudio) {
|
|
286
|
+
this.remoteAudio = new Audio();
|
|
287
|
+
this.remoteAudio.autoplay = true;
|
|
288
|
+
}
|
|
289
|
+
const remoteStream = event.streams[0];
|
|
290
|
+
this.remoteAudio.srcObject = remoteStream;
|
|
291
|
+
const threshold = this.config.speakingThreshold ?? DEFAULT_SPEAKING_THRESHOLD;
|
|
292
|
+
this.remoteAnalyser.start(remoteStream, threshold, (isSpeaking) => {
|
|
293
|
+
if (isSpeaking) {
|
|
294
|
+
this.clearSilenceTimer();
|
|
295
|
+
this._userSpeaking = false;
|
|
296
|
+
if (this._state !== VOICE_AGENT_STATE.SPEAKING) {
|
|
297
|
+
this.setState(VOICE_AGENT_STATE.SPEAKING);
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
else if (this._state === VOICE_AGENT_STATE.SPEAKING) {
|
|
301
|
+
this.setState(VOICE_AGENT_STATE.LISTENING);
|
|
302
|
+
}
|
|
303
|
+
});
|
|
304
|
+
}
|
|
305
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
const DEFAULT_FFT_SIZE = 256;
|
|
2
|
+
export class VoiceAudioAnalyser {
|
|
3
|
+
audioCtx = null;
|
|
4
|
+
animFrame = null;
|
|
5
|
+
start(stream, threshold, onSpeakingChange) {
|
|
6
|
+
const audioCtx = new AudioContext();
|
|
7
|
+
this.audioCtx = audioCtx;
|
|
8
|
+
const analyser = audioCtx.createAnalyser();
|
|
9
|
+
analyser.fftSize = DEFAULT_FFT_SIZE;
|
|
10
|
+
audioCtx.createMediaStreamSource(stream).connect(analyser);
|
|
11
|
+
const data = new Uint8Array(analyser.frequencyBinCount);
|
|
12
|
+
const poll = () => {
|
|
13
|
+
if (!this.audioCtx)
|
|
14
|
+
return;
|
|
15
|
+
analyser.getByteFrequencyData(data);
|
|
16
|
+
const avg = data.reduce((s, v) => s + v, 0) / data.length;
|
|
17
|
+
onSpeakingChange(avg > threshold);
|
|
18
|
+
this.animFrame = requestAnimationFrame(poll);
|
|
19
|
+
};
|
|
20
|
+
poll();
|
|
21
|
+
}
|
|
22
|
+
stop() {
|
|
23
|
+
if (this.animFrame !== null) {
|
|
24
|
+
cancelAnimationFrame(this.animFrame);
|
|
25
|
+
this.animFrame = null;
|
|
26
|
+
}
|
|
27
|
+
if (this.audioCtx) {
|
|
28
|
+
this.audioCtx.close().catch(() => { });
|
|
29
|
+
this.audioCtx = null;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from "./types";
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
export const VOICE_AGENT_STATE = {
|
|
2
|
+
IDLE: "idle",
|
|
3
|
+
CONNECTING: "connecting",
|
|
4
|
+
CONNECTED: "connected",
|
|
5
|
+
LISTENING: "listening",
|
|
6
|
+
THINKING: "thinking",
|
|
7
|
+
SPEAKING: "speaking",
|
|
8
|
+
MUTED: "muted",
|
|
9
|
+
DISCONNECTING: "disconnecting",
|
|
10
|
+
ERROR: "error",
|
|
11
|
+
};
|
|
12
|
+
export const VOICE_AGENT_EVENTS = {
|
|
13
|
+
STATE_CHANGED: "voice:state_changed",
|
|
14
|
+
ERROR: "voice:error",
|
|
15
|
+
};
|
package/dist/events/Events.d.ts
CHANGED
|
@@ -43,4 +43,3 @@ export type OutgoingSocketStreamMessage = Outgoing.StreamSynapseToMatrixMessage;
|
|
|
43
43
|
export type OutgoingSocketEndOfStreamMessage = Outgoing.EndOfStreamSynapseToMatrixMessage;
|
|
44
44
|
export type OutgoingSocket = Outgoing.PingMessage;
|
|
45
45
|
export declare function IsValidSocketMesssage(message: any): message is IncomingSocketMessage;
|
|
46
|
-
//# sourceMappingURL=Events.d.ts.map
|
package/dist/events/index.d.ts
CHANGED
package/dist/events/types.d.ts
CHANGED
package/dist/index.d.ts
CHANGED
|
@@ -99,4 +99,3 @@ export declare class ValidationError extends SynapseError {
|
|
|
99
99
|
type SynapseErrorCtor<T extends SynapseError> = new (message: string, options?: SynapseErrorOptions) => T;
|
|
100
100
|
export declare function normalizeError<T extends SynapseError>(error: unknown, ctor: SynapseErrorCtor<T>, fallbackMessage: string, options?: SynapseErrorOptions): T;
|
|
101
101
|
export {};
|
|
102
|
-
//# sourceMappingURL=Error.d.ts.map
|
|
@@ -40,4 +40,3 @@ export type OnMessageCallback = (event: IncomingMessage) => void;
|
|
|
40
40
|
export type OnStatusChangeCallback = (status: ConnectionStatus) => void;
|
|
41
41
|
export type OnOpenCallback = () => void;
|
|
42
42
|
export type OnErrorCallback = (error: Error) => void;
|
|
43
|
-
//# sourceMappingURL=types.d.ts.map
|
package/dist/messages/types.d.ts
CHANGED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
//# sourceMappingURL=Feedback.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
//# sourceMappingURL=types.d.ts.map
|
|
@@ -20,4 +20,3 @@ export declare class Session extends BaseResource {
|
|
|
20
20
|
feedback(sessionId: string, messageId: string, feedback: USER_FEEDBACK, feedback_reason?: string): Promise<void>;
|
|
21
21
|
callTool(sessionId: string, toolId: string, messageId: string, sessionToken: string, toolParams?: Record<string, unknown>): Promise<ToolCallResponse>;
|
|
22
22
|
}
|
|
23
|
-
//# sourceMappingURL=Session.d.ts.map
|
package/dist/types/index.d.ts
CHANGED
package/dist/utils/Error.d.ts
CHANGED
package/dist/voice/index.d.ts
CHANGED
package/dist/voice/types.d.ts
CHANGED
package/package.json
CHANGED
|
@@ -1,15 +1,17 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@eka-care/medassist-core",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.67",
|
|
4
4
|
"description": "TypeScript SDK for real-time medical chatbot experiences with session management, WebSocket connectivity, and media handling",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
|
+
"module": "dist/esm/index.js",
|
|
6
7
|
"types": "dist/index.d.ts",
|
|
8
|
+
"sideEffects": false,
|
|
7
9
|
"files": [
|
|
8
10
|
"dist",
|
|
9
11
|
"README.md"
|
|
10
12
|
],
|
|
11
13
|
"scripts": {
|
|
12
|
-
"build": "tsc --project tsconfig.build.json",
|
|
14
|
+
"build": "tsc --project tsconfig.build.json && tsc --project tsconfig.esm.json",
|
|
13
15
|
"prepublishOnly": "npm run build",
|
|
14
16
|
"clean": "rm -rf dist"
|
|
15
17
|
},
|
package/dist/Synapse.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"Synapse.d.ts","sourceRoot":"","sources":["../src/Synapse.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAgBH,OAAO,EACL,uBAAuB,EAExB,MAAM,kBAAkB,CAAC;AAuB1B,OAAO,EACL,KAAK,eAAe,EACrB,MAAM,2BAA2B,CAAC;AACnC,OAAO,EAAE,KAAK,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,KAAK,EAAE,mBAAmB,EAAE,sBAAsB,EAAE,kBAAkB,EAA2C,gBAAgB,EAAiB,aAAa,EAAE,MAAM,SAAS,CAAC;AACxL,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAKhD,qBAAa,UAAU;IACrB,OAAO,CAAC,UAAU,CAA+B;IACjD,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,eAAe,CAAkB;IAEzC,OAAO,CAAC,aAAa,CAA8B;IACnD,OAAO,CAAC,MAAM,CAAmB;IACjC,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,mBAAmB,CAAkB;IAC7C,OAAO,CAAC,kBAAkB,CAAa;IACvC,OAAO,CAAC,eAAe,CAAoC;IAC3D,OAAO,CAAC,MAAM,CAA2B;gBAE7B,MAAM,EAAE,gBAAgB;IAgBpC;;;;;;OAMG;IACU,YAAY,CACvB,eAAe,CAAC,EAAE,sBAAsB,GACvC,OAAO,CAAC,eAAe,CAAC;IA6B3B;;OAEG;IACU,WAAW,CAAC,EACvB,OAAO,EACP,KAAK,EACL,KAAK,EACL,aAAa,EACb,OAAO,EACP,eAAe,EACf,WAAW,EACZ,EAAE,kBAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;IA0BrC;;OAEG;IACU,YAAY,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAwBrG;;OAEG;IACI,EAAE,CACP,KAAK,EAAE,uBAAuB,EAC9B,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,KAAK,IAAI,GACrC,IAAI;IAIP;;OAEG;IACI,GAAG,CACR,KAAK,EAAE,uBAAuB,EAC9B,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,KAAK,IAAI,GACrC,IAAI;IAIP;;OAEG;IACI,gBAAgB,IAAI,eAAe;IAU1C;;OAEG;IACU,QAAQ,CACnB,OAAO,EAAE,MAAM,EACf,UAAU,EAAE,MAAM,EAClB,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAC/B,OAAO,CAAC,IAAI,CAAC;IA0DhB;;SAEK;IACQ,cAAc,CAAC,EAC1B,QAAQ,EACR,OAAO,GACR,EAAE;QACD,QAAQ,EAAE,CAAC,MAAM,EAAE,aAAa,KAAK,IAAI,CAAC;QAC1C,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC;KAClC,GAAG,OAAO,CAAC,IAAI,CAAC;IAsBjB;;OAEG;IACI,YAAY,IAAI,IAAI;IAqBd,cAAc,IAAI,OAAO,CAAC,mBAAmB,CAAC;IAIpD,WAAW,IAAI,OAAO;IAI7B,IAAW,KAAK,IAAI,UAAU,CAe7B;IAED;;OAEG;IACI,UAAU,IAAI,IAAI;IAOzB,OAAO,CAAC,gBAAgB;IAYxB,OAAO,CAAC,aAAa;IAuBrB,OAAO,CAAC,SAAS;IAIjB,OAAO,CAAC,cAAc;IAYtB;;;OAGG;YACW,oBAAoB;IA4ClC;;;OAGG;YACW,aAAa;IAqC3B;;OAEG;YACW,cAAc;IAuB5B;;OAEG;YACW,gBAAgB;IAO9B;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAsE1B;;OAEG;IACH,OAAO,CAAC,uBAAuB;IAW/B;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAe9B;;OAEG;IACH,OAAO,CAAC,uBAAuB;IAgD/B;;OAEG;YACW,mBAAmB;IA0CjC,OAAO,CAAC,iBAAiB;IAYzB,OAAO,CAAC,eAAe;IAOvB,OAAO,CAAC,OAAO;CAKhB"}
|
package/dist/auth/constants.d.ts
DELETED
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
export declare const EKA_AUTH: {
|
|
2
|
-
readonly SESSION_STORAGE_KEY: "eka_auth_session";
|
|
3
|
-
/** postMessage type sent from auth popup to opener on successful login */
|
|
4
|
-
readonly POST_MESSAGE_SUCCESS_TYPE: "EKA_AUTH_SUCCESS";
|
|
5
|
-
/** Default URL for the Eka Care auth popup */
|
|
6
|
-
readonly DEFAULT_AUTH_URL: "https://aortago.dev.eka.care/auth/login?audience=doc-web";
|
|
7
|
-
};
|
|
8
|
-
export type EkaAuthSession = {
|
|
9
|
-
sess: string;
|
|
10
|
-
refresh: string;
|
|
11
|
-
};
|
|
12
|
-
//# sourceMappingURL=constants.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../src/auth/constants.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,QAAQ;;IAEnB,0EAA0E;;IAE1E,8CAA8C;;CAEtC,CAAC;AAEX,MAAM,MAAM,cAAc,GAAG;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC"}
|
package/dist/auth/constants.js
DELETED
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.EKA_AUTH = void 0;
|
|
4
|
-
exports.EKA_AUTH = {
|
|
5
|
-
SESSION_STORAGE_KEY: "eka_auth_session",
|
|
6
|
-
/** postMessage type sent from auth popup to opener on successful login */
|
|
7
|
-
POST_MESSAGE_SUCCESS_TYPE: "EKA_AUTH_SUCCESS",
|
|
8
|
-
/** Default URL for the Eka Care auth popup */
|
|
9
|
-
DEFAULT_AUTH_URL: "https://aortago.dev.eka.care/auth/login?audience=doc-web",
|
|
10
|
-
};
|
package/dist/auth/index.d.ts
DELETED
package/dist/auth/index.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/auth/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,WAAW,CAAC"}
|