@autoscriber/core 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +147 -0
- package/dist/audio/audio-manager.d.ts +27 -0
- package/dist/audio/audio-manager.d.ts.map +1 -0
- package/dist/audio/permissions.d.ts +8 -0
- package/dist/audio/permissions.d.ts.map +1 -0
- package/dist/audio/resampler.d.ts +5 -0
- package/dist/audio/resampler.d.ts.map +1 -0
- package/dist/audio/silence-detector.d.ts +16 -0
- package/dist/audio/silence-detector.d.ts.map +1 -0
- package/dist/audio/wake-lock.d.ts +29 -0
- package/dist/audio/wake-lock.d.ts.map +1 -0
- package/dist/audio/worklets.d.ts +3 -0
- package/dist/audio/worklets.d.ts.map +1 -0
- package/dist/core/assistant-client.d.ts +78 -0
- package/dist/core/assistant-client.d.ts.map +1 -0
- package/dist/core/message-store.d.ts +28 -0
- package/dist/core/message-store.d.ts.map +1 -0
- package/dist/core/offline-queue.d.ts +9 -0
- package/dist/core/offline-queue.d.ts.map +1 -0
- package/dist/core/websocket-client.d.ts +31 -0
- package/dist/core/websocket-client.d.ts.map +1 -0
- package/dist/headless/create-assistant.d.ts +9 -0
- package/dist/headless/create-assistant.d.ts.map +1 -0
- package/dist/index.cjs +2005 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +1977 -0
- package/dist/types/index.d.ts +103 -0
- package/dist/types/index.d.ts.map +1 -0
- package/dist/utils/base64.d.ts +3 -0
- package/dist/utils/base64.d.ts.map +1 -0
- package/dist/utils/event-bus.d.ts +11 -0
- package/dist/utils/event-bus.d.ts.map +1 -0
- package/dist/utils/i18n.d.ts +5 -0
- package/dist/utils/i18n.d.ts.map +1 -0
- package/dist/utils/id.d.ts +2 -0
- package/dist/utils/id.d.ts.map +1 -0
- package/dist/utils/logger.d.ts +8 -0
- package/dist/utils/logger.d.ts.map +1 -0
- package/dist/utils/token.d.ts +4 -0
- package/dist/utils/token.d.ts.map +1 -0
- package/package.json +60 -0
package/dist/index.js
ADDED
|
@@ -0,0 +1,1977 @@
|
|
|
1
|
+
// src/utils/event-bus.ts
|
|
2
|
+
var EventBus = class {
|
|
3
|
+
constructor() {
|
|
4
|
+
this.listeners = /* @__PURE__ */ new Map();
|
|
5
|
+
}
|
|
6
|
+
on(event, listener) {
|
|
7
|
+
const listeners = this.listeners.get(event) ?? /* @__PURE__ */ new Set();
|
|
8
|
+
listeners.add(listener);
|
|
9
|
+
this.listeners.set(event, listeners);
|
|
10
|
+
return () => this.off(event, listener);
|
|
11
|
+
}
|
|
12
|
+
once(event, listener) {
|
|
13
|
+
const off = this.on(event, (payload) => {
|
|
14
|
+
off();
|
|
15
|
+
listener(payload);
|
|
16
|
+
});
|
|
17
|
+
return off;
|
|
18
|
+
}
|
|
19
|
+
off(event, listener) {
|
|
20
|
+
const listeners = this.listeners.get(event);
|
|
21
|
+
if (!listeners)
|
|
22
|
+
return;
|
|
23
|
+
listeners.delete(listener);
|
|
24
|
+
if (listeners.size === 0) {
|
|
25
|
+
this.listeners.delete(event);
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
emit(event, payload) {
|
|
29
|
+
const listeners = this.listeners.get(event);
|
|
30
|
+
if (!listeners)
|
|
31
|
+
return;
|
|
32
|
+
listeners.forEach((listener) => {
|
|
33
|
+
try {
|
|
34
|
+
listener(payload);
|
|
35
|
+
} catch (error) {
|
|
36
|
+
console.error("[EventBus] listener error", error);
|
|
37
|
+
}
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
clear() {
|
|
41
|
+
this.listeners.clear();
|
|
42
|
+
}
|
|
43
|
+
};
|
|
44
|
+
|
|
45
|
+
// src/utils/logger.ts
|
|
46
|
+
var noop = () => {
|
|
47
|
+
};
|
|
48
|
+
function createLogger(overrides) {
|
|
49
|
+
return {
|
|
50
|
+
debug: overrides?.debug?.bind(overrides) ?? noop,
|
|
51
|
+
info: overrides?.info?.bind(overrides) ?? noop,
|
|
52
|
+
warn: overrides?.warn?.bind(overrides) ?? console.warn,
|
|
53
|
+
error: overrides?.error?.bind(overrides) ?? console.error
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
// src/utils/i18n.ts
|
|
58
|
+
var TRANSLATIONS = {
|
|
59
|
+
en: {
|
|
60
|
+
askAnything: "Ask anything...",
|
|
61
|
+
typeOrHoldSpace: 'Type or hold "space" to talk',
|
|
62
|
+
paused: "Paused...",
|
|
63
|
+
listening: "Listening...",
|
|
64
|
+
title: "Assistant",
|
|
65
|
+
aiDisclaimer: "AI generated answers can be inaccurate.",
|
|
66
|
+
newChat: "New chat",
|
|
67
|
+
emergencyStop: "Recording emergency stopped",
|
|
68
|
+
transcriptionTimeout: "Transcription timeout - please try again",
|
|
69
|
+
voiceCommandFailed: "\u26A0\uFE0F Voice command failed to start - microphone unavailable",
|
|
70
|
+
transcriptionFailed: "\u26A0\uFE0F Transcription failed to start - microphone unavailable",
|
|
71
|
+
microphoneRequired: "\u26A0\uFE0F Microphone access required - please allow microphone access when prompted",
|
|
72
|
+
microphoneNotFound: "\u26A0\uFE0F No microphone found - please connect a microphone",
|
|
73
|
+
microphonePermissionError: "\u26A0\uFE0F Unable to check microphone permissions",
|
|
74
|
+
microphoneAccessError: "\u26A0\uFE0F Unable to access microphone - please check your browser settings",
|
|
75
|
+
microphoneAccessDenied: "\u26A0\uFE0F Microphone access denied - please enable in browser settings",
|
|
76
|
+
settings: "Settings",
|
|
77
|
+
microphone: "Microphone",
|
|
78
|
+
noMicrophonesFound: "No microphones found",
|
|
79
|
+
transcriptionStarted: "Transcription started",
|
|
80
|
+
transcriptionStopped: "Transcription stopped",
|
|
81
|
+
voiceCommandSilence: "Voice command cancelled - silence detected",
|
|
82
|
+
networkOffline: "\u26A0\uFE0F Network offline - messages will be sent when connection returns",
|
|
83
|
+
transcriptionStoppedOffline: "Transcription stopped - Network offline",
|
|
84
|
+
transcriptionBufferingOffline: "Transcription buffering - will resume when connection returns",
|
|
85
|
+
voiceCommandCancelledOffline: "Voice command cancelled - Network offline",
|
|
86
|
+
connectionRestored: "Connection restored",
|
|
87
|
+
offlineBuffering: "Message buffered - will resend once connected"
|
|
88
|
+
},
|
|
89
|
+
nl: {
|
|
90
|
+
askAnything: "Stel je vraag...",
|
|
91
|
+
typeOrHoldSpace: 'Typ of houd "spatie" ingedrukt om te praten',
|
|
92
|
+
paused: "Gepauzeerd...",
|
|
93
|
+
listening: "Luisteren...",
|
|
94
|
+
title: "Assistent",
|
|
95
|
+
aiDisclaimer: "AI-gegenereerde antwoorden kunnen onnauwkeurig zijn.",
|
|
96
|
+
newChat: "Nieuwe chat",
|
|
97
|
+
emergencyStop: "Opname noodstop uitgevoerd",
|
|
98
|
+
transcriptionTimeout: "Transcriptie timeout - probeer opnieuw",
|
|
99
|
+
voiceCommandFailed: "\u26A0\uFE0F Spraakopdracht starten mislukt - microfoon niet beschikbaar",
|
|
100
|
+
transcriptionFailed: "\u26A0\uFE0F Transcriptie starten mislukt - microfoon niet beschikbaar",
|
|
101
|
+
microphoneRequired: "\u26A0\uFE0F Microfoon toegang vereist - sta microfoon toegang toe wanneer gevraagd",
|
|
102
|
+
microphoneNotFound: "\u26A0\uFE0F Geen microfoon gevonden - sluit een microfoon aan",
|
|
103
|
+
microphonePermissionError: "\u26A0\uFE0F Kan microfoon machtigingen niet controleren",
|
|
104
|
+
microphoneAccessError: "\u26A0\uFE0F Kan microfoon niet benaderen - controleer uw browserinstellingen",
|
|
105
|
+
microphoneAccessDenied: "\u26A0\uFE0F Microfoon toegang geweigerd - schakel in browserinstellingen in",
|
|
106
|
+
settings: "Instellingen",
|
|
107
|
+
microphone: "Microfoon",
|
|
108
|
+
noMicrophonesFound: "Geen microfoons gevonden",
|
|
109
|
+
transcriptionStarted: "Transcriptie gestart",
|
|
110
|
+
transcriptionStopped: "Transcriptie gestopt",
|
|
111
|
+
voiceCommandSilence: "Spraakopdracht geannuleerd - stilte gedetecteerd",
|
|
112
|
+
networkOffline: "\u26A0\uFE0F Netwerk offline - berichten worden verzonden zodra de verbinding terug is",
|
|
113
|
+
transcriptionStoppedOffline: "Transcriptie gestopt - Netwerk offline",
|
|
114
|
+
transcriptionBufferingOffline: "Transcriptie wordt gebufferd - gaat verder zodra we weer online zijn",
|
|
115
|
+
voiceCommandCancelledOffline: "Spraakopdracht geannuleerd - Netwerk offline",
|
|
116
|
+
connectionRestored: "Verbinding hersteld",
|
|
117
|
+
offlineBuffering: "Bericht gebufferd - wordt verzonden zodra we weer online zijn"
|
|
118
|
+
},
|
|
119
|
+
de: {
|
|
120
|
+
askAnything: "Stellen Sie eine Frage...",
|
|
121
|
+
typeOrHoldSpace: "Tippen zum Schreiben oder Leertaste halten zum Sprechen",
|
|
122
|
+
paused: "Pausiert...",
|
|
123
|
+
listening: "H\xF6rt zu...",
|
|
124
|
+
title: "Assistent",
|
|
125
|
+
aiDisclaimer: "KI-generierte Antworten k\xF6nnen ungenau sein.",
|
|
126
|
+
newChat: "Neuer Chat",
|
|
127
|
+
emergencyStop: "Notfallstopp der Aufnahme ausgef\xFChrt",
|
|
128
|
+
transcriptionTimeout: "Transkriptions-Timeout \u2013 bitte erneut versuchen",
|
|
129
|
+
voiceCommandFailed: "\u26A0\uFE0F Sprachbefehl konnte nicht gestartet werden \u2013 Mikrofon nicht verf\xFCgbar",
|
|
130
|
+
transcriptionFailed: "\u26A0\uFE0F Transkription konnte nicht gestartet werden \u2013 Mikrofon nicht verf\xFCgbar",
|
|
131
|
+
microphoneRequired: "\u26A0\uFE0F Mikrofonzugriff erforderlich \u2013 bitte Zugriff bei Aufforderung erlauben",
|
|
132
|
+
microphoneNotFound: "\u26A0\uFE0F Kein Mikrofon gefunden \u2013 bitte Mikrofon anschlie\xDFen",
|
|
133
|
+
microphonePermissionError: "\u26A0\uFE0F Mikrofonberechtigungen k\xF6nnen nicht \xFCberpr\xFCft werden",
|
|
134
|
+
microphoneAccessError: "\u26A0\uFE0F Auf Mikrofon kann nicht zugegriffen werden \u2013 bitte Browsereinstellungen \xFCberpr\xFCfen",
|
|
135
|
+
microphoneAccessDenied: "\u26A0\uFE0F Mikrofonzugriff verweigert \u2013 bitte in den Browsereinstellungen aktivieren",
|
|
136
|
+
settings: "Einstellungen",
|
|
137
|
+
microphone: "Mikrofon",
|
|
138
|
+
noMicrophonesFound: "Keine Mikrofone gefunden",
|
|
139
|
+
transcriptionStarted: "Transkription gestartet",
|
|
140
|
+
transcriptionStopped: "Transkription gestoppt",
|
|
141
|
+
voiceCommandSilence: "Sprachbefehl abgebrochen \u2013 Stille erkannt",
|
|
142
|
+
networkOffline: "\u26A0\uFE0F Netzwerk offline \u2013 Nachrichten werden gesendet, sobald die Verbindung zur\xFCck ist",
|
|
143
|
+
transcriptionStoppedOffline: "Transkription gestoppt \u2013 Netzwerk offline",
|
|
144
|
+
transcriptionBufferingOffline: "Transkription wird gepuffert \u2013 l\xE4uft weiter, sobald die Verbindung zur\xFCck ist",
|
|
145
|
+
voiceCommandCancelledOffline: "Sprachbefehl abgebrochen \u2013 Netzwerk offline",
|
|
146
|
+
connectionRestored: "Verbindung wiederhergestellt",
|
|
147
|
+
offlineBuffering: "Nachricht gepuffert \u2013 wird gesendet, sobald wir wieder online sind"
|
|
148
|
+
}
|
|
149
|
+
};
|
|
150
|
+
var DEFAULT_LANGUAGE = "en";
|
|
151
|
+
function translate(language, key, params = {}) {
|
|
152
|
+
const dictionary = TRANSLATIONS[language] ?? TRANSLATIONS[DEFAULT_LANGUAGE];
|
|
153
|
+
let text = dictionary[key] ?? TRANSLATIONS[DEFAULT_LANGUAGE][key] ?? key;
|
|
154
|
+
Object.entries(params).forEach(([placeholder, value]) => {
|
|
155
|
+
text = text.replace(new RegExp(`{{${placeholder}}}`, "g"), String(value));
|
|
156
|
+
});
|
|
157
|
+
return text;
|
|
158
|
+
}
|
|
159
|
+
function validateLanguage(lang) {
|
|
160
|
+
if (!lang)
|
|
161
|
+
return DEFAULT_LANGUAGE;
|
|
162
|
+
if (["en", "nl", "de"].includes(lang)) {
|
|
163
|
+
return lang;
|
|
164
|
+
}
|
|
165
|
+
console.warn(
|
|
166
|
+
`[Autoscriber] Unsupported language "${lang}" provided. Falling back to "${DEFAULT_LANGUAGE}".`
|
|
167
|
+
);
|
|
168
|
+
return DEFAULT_LANGUAGE;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
// src/utils/id.ts
|
|
172
|
+
function generateMessageId() {
|
|
173
|
+
return Math.random().toString(36).substring(2, 10);
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
// src/core/message-store.ts
|
|
177
|
+
var MessageStore = class {
|
|
178
|
+
constructor() {
|
|
179
|
+
this.messages = [];
|
|
180
|
+
this.interactions = [];
|
|
181
|
+
}
|
|
182
|
+
createMessage(input = {}) {
|
|
183
|
+
const message = {
|
|
184
|
+
id: input.id ?? generateMessageId(),
|
|
185
|
+
request: input.request ?? null,
|
|
186
|
+
partialResults: [],
|
|
187
|
+
result: null,
|
|
188
|
+
type: input.type ?? "chat",
|
|
189
|
+
metadata: input.metadata ?? {}
|
|
190
|
+
};
|
|
191
|
+
this.messages.push(message);
|
|
192
|
+
return message;
|
|
193
|
+
}
|
|
194
|
+
addMessage(message) {
|
|
195
|
+
this.messages.push(message);
|
|
196
|
+
}
|
|
197
|
+
getMessageById(id) {
|
|
198
|
+
return this.messages.find((msg) => msg.id === id);
|
|
199
|
+
}
|
|
200
|
+
setMessageResult(id, result) {
|
|
201
|
+
const target = this.getMessageById(id);
|
|
202
|
+
if (!target)
|
|
203
|
+
return;
|
|
204
|
+
target.result = result;
|
|
205
|
+
target.partialResults = [];
|
|
206
|
+
}
|
|
207
|
+
setMessagePartialResult(id, payload, append = true) {
|
|
208
|
+
const target = this.getMessageById(id);
|
|
209
|
+
if (!target)
|
|
210
|
+
return;
|
|
211
|
+
if (!append) {
|
|
212
|
+
target.partialResults = [payload];
|
|
213
|
+
} else {
|
|
214
|
+
target.partialResults.push(payload);
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
applyJsonPatch(id, patchOperations) {
|
|
218
|
+
const target = this.getMessageById(id);
|
|
219
|
+
if (!target)
|
|
220
|
+
return;
|
|
221
|
+
if (!target.metadata)
|
|
222
|
+
target.metadata = {};
|
|
223
|
+
if (!target.metadata.transcriptData) {
|
|
224
|
+
target.metadata.transcriptData = {
|
|
225
|
+
resourceType: "Transcript",
|
|
226
|
+
entries: []
|
|
227
|
+
};
|
|
228
|
+
}
|
|
229
|
+
const transcriptData = target.metadata.transcriptData;
|
|
230
|
+
patchOperations.forEach((operation) => {
|
|
231
|
+
if (operation.op === "add" && operation.path === "/entries") {
|
|
232
|
+
if (Array.isArray(operation.value)) {
|
|
233
|
+
transcriptData.entries.push(...operation.value);
|
|
234
|
+
} else {
|
|
235
|
+
transcriptData.entries.push(operation.value);
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
});
|
|
239
|
+
const transcriptResult = {
|
|
240
|
+
mediaType: "application/fhir+json",
|
|
241
|
+
data: { ...transcriptData }
|
|
242
|
+
};
|
|
243
|
+
target.result = [transcriptResult];
|
|
244
|
+
}
|
|
245
|
+
setError(id, errorMessage, result) {
|
|
246
|
+
const target = this.getMessageById(id);
|
|
247
|
+
if (!target)
|
|
248
|
+
return;
|
|
249
|
+
target.errorMessage = errorMessage;
|
|
250
|
+
if (result) {
|
|
251
|
+
target.result = result;
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
upsertInteraction(pill) {
|
|
255
|
+
const existing = this.interactions.find((item) => item.id === pill.id);
|
|
256
|
+
if (existing) {
|
|
257
|
+
Object.assign(existing, pill);
|
|
258
|
+
return existing;
|
|
259
|
+
}
|
|
260
|
+
this.interactions.push(pill);
|
|
261
|
+
return pill;
|
|
262
|
+
}
|
|
263
|
+
updateInteractionStatus(id, status, metadata) {
|
|
264
|
+
const target = this.interactions.find((item) => item.id === id);
|
|
265
|
+
if (!target)
|
|
266
|
+
return;
|
|
267
|
+
target.status = status;
|
|
268
|
+
if (metadata) {
|
|
269
|
+
target.metadata = { ...target.metadata ?? {}, ...metadata };
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
getInteractions() {
|
|
273
|
+
return [...this.interactions];
|
|
274
|
+
}
|
|
275
|
+
getMessages() {
|
|
276
|
+
return [...this.messages];
|
|
277
|
+
}
|
|
278
|
+
reset() {
|
|
279
|
+
this.messages = [];
|
|
280
|
+
this.interactions = [];
|
|
281
|
+
}
|
|
282
|
+
};
|
|
283
|
+
|
|
284
|
+
// src/core/offline-queue.ts
|
|
285
|
+
var OfflineQueue = class {
|
|
286
|
+
constructor() {
|
|
287
|
+
this.queue = [];
|
|
288
|
+
}
|
|
289
|
+
enqueue(item) {
|
|
290
|
+
this.queue.push(item);
|
|
291
|
+
}
|
|
292
|
+
drain() {
|
|
293
|
+
const items = [...this.queue];
|
|
294
|
+
this.queue = [];
|
|
295
|
+
return items;
|
|
296
|
+
}
|
|
297
|
+
clear() {
|
|
298
|
+
this.queue = [];
|
|
299
|
+
}
|
|
300
|
+
get size() {
|
|
301
|
+
return this.queue.length;
|
|
302
|
+
}
|
|
303
|
+
get items() {
|
|
304
|
+
return [...this.queue];
|
|
305
|
+
}
|
|
306
|
+
};
|
|
307
|
+
|
|
308
|
+
// src/core/websocket-client.ts
|
|
309
|
+
var WebSocketClient = class {
|
|
310
|
+
constructor(options = {}) {
|
|
311
|
+
this.socket = null;
|
|
312
|
+
this.manualClose = false;
|
|
313
|
+
this.reconnectAttempts = 0;
|
|
314
|
+
this.bus = new EventBus();
|
|
315
|
+
this.reconnectDelay = options.reconnectDelay ?? 3e3;
|
|
316
|
+
this.autoReconnect = options.autoReconnect ?? true;
|
|
317
|
+
this.logger = {
|
|
318
|
+
...console,
|
|
319
|
+
...options.logger
|
|
320
|
+
};
|
|
321
|
+
}
|
|
322
|
+
on(event, listener) {
|
|
323
|
+
return this.bus.on(event, listener);
|
|
324
|
+
}
|
|
325
|
+
async connect(url) {
|
|
326
|
+
this.manualClose = false;
|
|
327
|
+
if (this.socket) {
|
|
328
|
+
this.logger.warn("[Autoscriber] WebSocket already connected - closing existing connection");
|
|
329
|
+
this.socket.close(4e3, "Reconnecting");
|
|
330
|
+
this.socket = null;
|
|
331
|
+
}
|
|
332
|
+
await this.openSocket(url);
|
|
333
|
+
}
|
|
334
|
+
openSocket(url) {
|
|
335
|
+
return new Promise((resolve, reject) => {
|
|
336
|
+
try {
|
|
337
|
+
const socket = new WebSocket(url);
|
|
338
|
+
this.socket = socket;
|
|
339
|
+
socket.onopen = (event) => {
|
|
340
|
+
this.logger.info("[Autoscriber] websocket open");
|
|
341
|
+
this.reconnectAttempts = 0;
|
|
342
|
+
this.bus.emit("open", event);
|
|
343
|
+
resolve();
|
|
344
|
+
};
|
|
345
|
+
socket.onclose = (event) => {
|
|
346
|
+
this.logger.warn("[Autoscriber] websocket closed", event.code, event.reason);
|
|
347
|
+
this.socket = null;
|
|
348
|
+
this.bus.emit("close", event);
|
|
349
|
+
if (!this.manualClose && this.autoReconnect) {
|
|
350
|
+
this.scheduleReconnect(url);
|
|
351
|
+
}
|
|
352
|
+
};
|
|
353
|
+
socket.onerror = (event) => {
|
|
354
|
+
const errorInfo = {
|
|
355
|
+
type: event.type,
|
|
356
|
+
timeStamp: event.timeStamp,
|
|
357
|
+
readyState: socket.readyState,
|
|
358
|
+
url: socket.url
|
|
359
|
+
};
|
|
360
|
+
if (event instanceof ErrorEvent) {
|
|
361
|
+
errorInfo.message = event.message;
|
|
362
|
+
errorInfo.filename = event.filename;
|
|
363
|
+
errorInfo.lineno = event.lineno;
|
|
364
|
+
errorInfo.colno = event.colno;
|
|
365
|
+
if (event.error) {
|
|
366
|
+
errorInfo.error = event.error instanceof Error ? { name: event.error.name, message: event.error.message, stack: event.error.stack } : String(event.error);
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
this.logger.error("[Autoscriber] websocket error", errorInfo);
|
|
370
|
+
this.bus.emit("error", event);
|
|
371
|
+
const errorMessage = event instanceof ErrorEvent && event.message ? event.message : `WebSocket error (readyState: ${socket.readyState}, url: ${socket.url})`;
|
|
372
|
+
const enhancedError = new Error(errorMessage);
|
|
373
|
+
enhancedError.originalEvent = event;
|
|
374
|
+
enhancedError.readyState = socket.readyState;
|
|
375
|
+
enhancedError.url = socket.url;
|
|
376
|
+
reject(enhancedError);
|
|
377
|
+
};
|
|
378
|
+
socket.onmessage = (event) => {
|
|
379
|
+
this.bus.emit("message", event);
|
|
380
|
+
};
|
|
381
|
+
} catch (error) {
|
|
382
|
+
reject(error);
|
|
383
|
+
}
|
|
384
|
+
});
|
|
385
|
+
}
|
|
386
|
+
scheduleReconnect(url) {
|
|
387
|
+
this.reconnectAttempts += 1;
|
|
388
|
+
const delay = this.reconnectDelay;
|
|
389
|
+
if (!this.autoReconnect) {
|
|
390
|
+
return;
|
|
391
|
+
}
|
|
392
|
+
this.logger.info("[Autoscriber] scheduling reconnect", { attempt: this.reconnectAttempts, delay });
|
|
393
|
+
this.bus.emit("reconnecting", this.reconnectAttempts);
|
|
394
|
+
setTimeout(() => {
|
|
395
|
+
if (this.manualClose)
|
|
396
|
+
return;
|
|
397
|
+
this.openSocket(url).catch((error) => {
|
|
398
|
+
this.logger.error("[Autoscriber] reconnect attempt failed", error);
|
|
399
|
+
this.scheduleReconnect(url);
|
|
400
|
+
});
|
|
401
|
+
}, delay);
|
|
402
|
+
}
|
|
403
|
+
send(payload) {
|
|
404
|
+
if (!this.socket || this.socket.readyState !== WebSocket.OPEN) {
|
|
405
|
+
throw new Error("WebSocket not connected");
|
|
406
|
+
}
|
|
407
|
+
this.socket.send(payload);
|
|
408
|
+
}
|
|
409
|
+
close(code, reason) {
|
|
410
|
+
this.manualClose = true;
|
|
411
|
+
this.socket?.close(code, reason);
|
|
412
|
+
this.socket = null;
|
|
413
|
+
}
|
|
414
|
+
get readyState() {
|
|
415
|
+
return this.socket?.readyState ?? WebSocket.CLOSED;
|
|
416
|
+
}
|
|
417
|
+
};
|
|
418
|
+
|
|
419
|
+
// src/utils/base64.ts
|
|
420
|
+
function arrayBufferToBase64(buffer) {
|
|
421
|
+
let binary = "";
|
|
422
|
+
const bytes = new Uint8Array(buffer);
|
|
423
|
+
const len = bytes.byteLength;
|
|
424
|
+
for (let i = 0; i < len; i += 1) {
|
|
425
|
+
binary += String.fromCharCode(bytes[i]);
|
|
426
|
+
}
|
|
427
|
+
return btoa(binary);
|
|
428
|
+
}
|
|
429
|
+
function base64ToArray(base64) {
|
|
430
|
+
const binaryString = atob(base64);
|
|
431
|
+
const len = binaryString.length;
|
|
432
|
+
const bytes = new Uint8Array(len);
|
|
433
|
+
for (let i = 0; i < len; i += 1) {
|
|
434
|
+
bytes[i] = binaryString.charCodeAt(i);
|
|
435
|
+
}
|
|
436
|
+
return bytes.buffer;
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
// src/audio/worklets.ts
|
|
440
|
+
var PCM_PLAYER_PROCESSOR = `
|
|
441
|
+
class PCMPlayerProcessor extends AudioWorkletProcessor {
|
|
442
|
+
constructor() {
|
|
443
|
+
super();
|
|
444
|
+
this.bufferSize = 24000 * 180;
|
|
445
|
+
this.buffer = new Float32Array(this.bufferSize);
|
|
446
|
+
this.writeIndex = 0;
|
|
447
|
+
this.readIndex = 0;
|
|
448
|
+
this.port.onmessage = (event) => {
|
|
449
|
+
if (event.data.command === 'endOfAudio') {
|
|
450
|
+
this.readIndex = this.writeIndex;
|
|
451
|
+
return;
|
|
452
|
+
}
|
|
453
|
+
const int16Samples = new Int16Array(event.data);
|
|
454
|
+
this._enqueue(int16Samples);
|
|
455
|
+
};
|
|
456
|
+
}
|
|
457
|
+
_enqueue(int16Samples) {
|
|
458
|
+
for (let i = 0; i < int16Samples.length; i++) {
|
|
459
|
+
const floatVal = int16Samples[i] / 32768;
|
|
460
|
+
this.buffer[this.writeIndex] = floatVal;
|
|
461
|
+
this.writeIndex = (this.writeIndex + 1) % this.bufferSize;
|
|
462
|
+
if (this.writeIndex === this.readIndex) {
|
|
463
|
+
this.readIndex = (this.readIndex + 1) % this.bufferSize;
|
|
464
|
+
}
|
|
465
|
+
}
|
|
466
|
+
}
|
|
467
|
+
process(inputs, outputs) {
|
|
468
|
+
const output = outputs[0];
|
|
469
|
+
const framesPerBlock = output[0].length;
|
|
470
|
+
for (let frame = 0; frame < framesPerBlock; frame++) {
|
|
471
|
+
output[0][frame] = this.buffer[this.readIndex];
|
|
472
|
+
if (output.length > 1) {
|
|
473
|
+
output[1][frame] = this.buffer[this.readIndex];
|
|
474
|
+
}
|
|
475
|
+
if (this.readIndex !== this.writeIndex) {
|
|
476
|
+
this.readIndex = (this.readIndex + 1) % this.bufferSize;
|
|
477
|
+
}
|
|
478
|
+
}
|
|
479
|
+
return true;
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
registerProcessor('pcm-player-processor', PCMPlayerProcessor);
|
|
483
|
+
`;
|
|
484
|
+
var PCM_RECORDER_PROCESSOR = `
|
|
485
|
+
class PCMProcessor extends AudioWorkletProcessor {
|
|
486
|
+
process(inputs) {
|
|
487
|
+
if (inputs.length > 0 && inputs[0].length > 0) {
|
|
488
|
+
const inputChannel = inputs[0][0];
|
|
489
|
+
const inputCopy = new Float32Array(inputChannel);
|
|
490
|
+
this.port.postMessage(inputCopy);
|
|
491
|
+
}
|
|
492
|
+
return true;
|
|
493
|
+
}
|
|
494
|
+
}
|
|
495
|
+
registerProcessor("pcm-recorder-processor", PCMProcessor);
|
|
496
|
+
`;
|
|
497
|
+
|
|
498
|
+
// src/audio/resampler.ts
|
|
499
|
+
function resamplePCM16(inputBuffer, inputSampleRate, outputSampleRate) {
|
|
500
|
+
if (inputSampleRate === outputSampleRate) {
|
|
501
|
+
return inputBuffer;
|
|
502
|
+
}
|
|
503
|
+
const inputSamples = new Int16Array(inputBuffer.buffer, inputBuffer.byteOffset, inputBuffer.length / 2);
|
|
504
|
+
const inputLength = inputSamples.length;
|
|
505
|
+
const ratio = inputSampleRate / outputSampleRate;
|
|
506
|
+
const outputLength = Math.round(inputLength / ratio);
|
|
507
|
+
const outputSamples = new Int16Array(outputLength);
|
|
508
|
+
for (let i = 0; i < outputLength; i++) {
|
|
509
|
+
const srcIndex = i * ratio;
|
|
510
|
+
const srcIndexFloor = Math.floor(srcIndex);
|
|
511
|
+
const srcIndexCeil = Math.min(srcIndexFloor + 1, inputLength - 1);
|
|
512
|
+
const fraction = srcIndex - srcIndexFloor;
|
|
513
|
+
const sample1 = inputSamples[srcIndexFloor];
|
|
514
|
+
const sample2 = inputSamples[srcIndexCeil];
|
|
515
|
+
outputSamples[i] = Math.round(sample1 + (sample2 - sample1) * fraction);
|
|
516
|
+
}
|
|
517
|
+
return new Uint8Array(outputSamples.buffer);
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
// src/audio/audio-manager.ts
|
|
521
|
+
var AudioManager = class {
|
|
522
|
+
constructor() {
|
|
523
|
+
this.recorderContext = null;
|
|
524
|
+
this.recorderNode = null;
|
|
525
|
+
this.playerContext = null;
|
|
526
|
+
this.playerNode = null;
|
|
527
|
+
this.micStream = null;
|
|
528
|
+
this.actualSampleRate = null;
|
|
529
|
+
this.targetSampleRate = 16e3;
|
|
530
|
+
}
|
|
531
|
+
async init(options = {}) {
|
|
532
|
+
this.targetSampleRate = options.targetSampleRate ?? 16e3;
|
|
533
|
+
await this.setupRecorder(options);
|
|
534
|
+
await this.setupPlayer(options);
|
|
535
|
+
}
|
|
536
|
+
async setupPlayer(options = {}) {
|
|
537
|
+
if (this.playerContext)
|
|
538
|
+
return;
|
|
539
|
+
if (this.actualSampleRate === null) {
|
|
540
|
+
const tempContext = new AudioContext();
|
|
541
|
+
this.actualSampleRate = tempContext.sampleRate;
|
|
542
|
+
tempContext.close();
|
|
543
|
+
}
|
|
544
|
+
const audioContext = new AudioContext({
|
|
545
|
+
sampleRate: this.actualSampleRate
|
|
546
|
+
});
|
|
547
|
+
const blob = new Blob([PCM_PLAYER_PROCESSOR], { type: "application/javascript" });
|
|
548
|
+
const workletURL = URL.createObjectURL(blob);
|
|
549
|
+
await audioContext.audioWorklet.addModule(workletURL);
|
|
550
|
+
URL.revokeObjectURL(workletURL);
|
|
551
|
+
const playerNode = new AudioWorkletNode(audioContext, "pcm-player-processor");
|
|
552
|
+
playerNode.connect(audioContext.destination);
|
|
553
|
+
this.playerContext = audioContext;
|
|
554
|
+
this.playerNode = playerNode;
|
|
555
|
+
}
|
|
556
|
+
async setupRecorder(options = {}) {
|
|
557
|
+
if (this.recorderContext)
|
|
558
|
+
return;
|
|
559
|
+
const constraints = { channelCount: 1 };
|
|
560
|
+
if (options.deviceId) {
|
|
561
|
+
constraints.deviceId = { exact: options.deviceId };
|
|
562
|
+
}
|
|
563
|
+
const stream = await navigator.mediaDevices.getUserMedia({
|
|
564
|
+
audio: constraints
|
|
565
|
+
});
|
|
566
|
+
if (this.actualSampleRate === null) {
|
|
567
|
+
const tempContext = new AudioContext();
|
|
568
|
+
const tempSource = tempContext.createMediaStreamSource(stream);
|
|
569
|
+
this.actualSampleRate = tempContext.sampleRate;
|
|
570
|
+
tempSource.disconnect();
|
|
571
|
+
tempContext.close();
|
|
572
|
+
}
|
|
573
|
+
const audioContext = new AudioContext({
|
|
574
|
+
sampleRate: this.actualSampleRate
|
|
575
|
+
});
|
|
576
|
+
const blob = new Blob([PCM_RECORDER_PROCESSOR], { type: "application/javascript" });
|
|
577
|
+
const workletURL = URL.createObjectURL(blob);
|
|
578
|
+
await audioContext.audioWorklet.addModule(workletURL);
|
|
579
|
+
URL.revokeObjectURL(workletURL);
|
|
580
|
+
const source = audioContext.createMediaStreamSource(stream);
|
|
581
|
+
const recorderNode = new AudioWorkletNode(audioContext, "pcm-recorder-processor");
|
|
582
|
+
recorderNode.port.onmessage = (event) => {
|
|
583
|
+
const float32 = event.data;
|
|
584
|
+
let pcm16 = this.convertFloat32ToPCM(float32);
|
|
585
|
+
if (this.actualSampleRate && this.actualSampleRate !== this.targetSampleRate) {
|
|
586
|
+
pcm16 = resamplePCM16(pcm16, this.actualSampleRate, this.targetSampleRate);
|
|
587
|
+
}
|
|
588
|
+
options.onRecorderData?.(pcm16);
|
|
589
|
+
};
|
|
590
|
+
source.connect(recorderNode);
|
|
591
|
+
this.recorderContext = audioContext;
|
|
592
|
+
this.recorderNode = recorderNode;
|
|
593
|
+
this.micStream = stream;
|
|
594
|
+
}
|
|
595
|
+
stop() {
|
|
596
|
+
this.stopRecorder();
|
|
597
|
+
this.stopPlayer();
|
|
598
|
+
}
|
|
599
|
+
stopRecorder() {
|
|
600
|
+
this.recorderNode?.disconnect();
|
|
601
|
+
this.recorderNode = null;
|
|
602
|
+
this.recorderContext?.close();
|
|
603
|
+
this.recorderContext = null;
|
|
604
|
+
if (this.micStream) {
|
|
605
|
+
this.micStream.getTracks().forEach((track) => track.stop());
|
|
606
|
+
}
|
|
607
|
+
this.micStream = null;
|
|
608
|
+
if (!this.playerContext) {
|
|
609
|
+
this.actualSampleRate = null;
|
|
610
|
+
}
|
|
611
|
+
}
|
|
612
|
+
stopPlayer() {
|
|
613
|
+
this.playerNode?.disconnect();
|
|
614
|
+
this.playerNode = null;
|
|
615
|
+
this.playerContext?.close();
|
|
616
|
+
this.playerContext = null;
|
|
617
|
+
if (!this.recorderContext) {
|
|
618
|
+
this.actualSampleRate = null;
|
|
619
|
+
}
|
|
620
|
+
}
|
|
621
|
+
teardown() {
|
|
622
|
+
this.stop();
|
|
623
|
+
}
|
|
624
|
+
playbackPcm(buffer) {
|
|
625
|
+
if (!this.playerNode)
|
|
626
|
+
return;
|
|
627
|
+
this.playerNode.port.postMessage(buffer);
|
|
628
|
+
}
|
|
629
|
+
endOfAudio() {
|
|
630
|
+
this.playerNode?.port.postMessage({ command: "endOfAudio" });
|
|
631
|
+
}
|
|
632
|
+
convertFloat32ToPCM(inputData) {
|
|
633
|
+
const pcm16 = new Int16Array(inputData.length);
|
|
634
|
+
for (let i = 0; i < inputData.length; i += 1) {
|
|
635
|
+
pcm16[i] = Math.max(-1, Math.min(1, inputData[i])) * 32767;
|
|
636
|
+
}
|
|
637
|
+
return new Uint8Array(pcm16.buffer);
|
|
638
|
+
}
|
|
639
|
+
};
|
|
640
|
+
|
|
641
|
+
// src/audio/silence-detector.ts
|
|
642
|
+
function convertPCMToFloat32(pcmBuffer) {
|
|
643
|
+
const int16View = new Int16Array(
|
|
644
|
+
pcmBuffer.buffer,
|
|
645
|
+
pcmBuffer.byteOffset,
|
|
646
|
+
pcmBuffer.byteLength / 2
|
|
647
|
+
);
|
|
648
|
+
const float32Array = new Float32Array(int16View.length);
|
|
649
|
+
for (let i = 0; i < int16View.length; i += 1) {
|
|
650
|
+
float32Array[i] = int16View[i] / 32768;
|
|
651
|
+
}
|
|
652
|
+
return float32Array;
|
|
653
|
+
}
|
|
654
|
+
function analyseAudioAmplitude(channelData, silenceThreshold = 0.01, silenceRatio = 0.95) {
|
|
655
|
+
let silentSamples = 0;
|
|
656
|
+
const totalSamples = channelData.length;
|
|
657
|
+
for (let i = 0; i < channelData.length; i += 1) {
|
|
658
|
+
const amplitude = Math.abs(channelData[i]);
|
|
659
|
+
if (amplitude < silenceThreshold) {
|
|
660
|
+
silentSamples += 1;
|
|
661
|
+
}
|
|
662
|
+
}
|
|
663
|
+
const silencePercentage = silentSamples / totalSamples;
|
|
664
|
+
const isSilence = silencePercentage > silenceRatio;
|
|
665
|
+
return {
|
|
666
|
+
isSilence,
|
|
667
|
+
silencePercentage,
|
|
668
|
+
totalSamples,
|
|
669
|
+
silentSamples
|
|
670
|
+
};
|
|
671
|
+
}
|
|
672
|
+
function analyseAudioEnergy(channelData, energyThreshold = 1e-3, silenceRatio = 0.9) {
|
|
673
|
+
const windowSize = 1024;
|
|
674
|
+
const energyLevels = [];
|
|
675
|
+
for (let i = 0; i < channelData.length; i += windowSize) {
|
|
676
|
+
let energy = 0;
|
|
677
|
+
const windowEnd = Math.min(i + windowSize, channelData.length);
|
|
678
|
+
for (let j = i; j < windowEnd; j += 1) {
|
|
679
|
+
energy += channelData[j] * channelData[j];
|
|
680
|
+
}
|
|
681
|
+
energy = energy / (windowEnd - i);
|
|
682
|
+
energyLevels.push(energy);
|
|
683
|
+
}
|
|
684
|
+
const silentWindows = energyLevels.filter((energy) => energy < energyThreshold).length;
|
|
685
|
+
const silencePercentage = energyLevels.length === 0 ? 1 : silentWindows / energyLevels.length;
|
|
686
|
+
const averageEnergy = energyLevels.length === 0 ? 0 : energyLevels.reduce((a, b) => a + b, 0) / energyLevels.length;
|
|
687
|
+
return {
|
|
688
|
+
isSilence: silencePercentage > silenceRatio,
|
|
689
|
+
silencePercentage,
|
|
690
|
+
averageEnergy,
|
|
691
|
+
energyLevels
|
|
692
|
+
};
|
|
693
|
+
}
|
|
694
|
+
function detectSilence(pcmBuffer) {
|
|
695
|
+
try {
|
|
696
|
+
const channelData = convertPCMToFloat32(pcmBuffer);
|
|
697
|
+
const amplitudeResult = analyseAudioAmplitude(channelData);
|
|
698
|
+
if (amplitudeResult.isSilence) {
|
|
699
|
+
const energyResult = analyseAudioEnergy(channelData);
|
|
700
|
+
if (energyResult.isSilence) {
|
|
701
|
+
return {
|
|
702
|
+
isSilence: true,
|
|
703
|
+
amplitudeAnalysis: amplitudeResult,
|
|
704
|
+
energyAnalysis: {
|
|
705
|
+
silencePercentage: energyResult.silencePercentage,
|
|
706
|
+
averageEnergy: energyResult.averageEnergy
|
|
707
|
+
}
|
|
708
|
+
};
|
|
709
|
+
}
|
|
710
|
+
return {
|
|
711
|
+
isSilence: false,
|
|
712
|
+
amplitudeAnalysis: amplitudeResult,
|
|
713
|
+
energyAnalysis: {
|
|
714
|
+
silencePercentage: energyResult.silencePercentage,
|
|
715
|
+
averageEnergy: energyResult.averageEnergy
|
|
716
|
+
}
|
|
717
|
+
};
|
|
718
|
+
}
|
|
719
|
+
return {
|
|
720
|
+
isSilence: false,
|
|
721
|
+
amplitudeAnalysis: amplitudeResult
|
|
722
|
+
};
|
|
723
|
+
} catch (error) {
|
|
724
|
+
console.error("[Autoscriber] silence detection failed", error);
|
|
725
|
+
return {
|
|
726
|
+
isSilence: false,
|
|
727
|
+
error: error instanceof Error ? error.message : String(error)
|
|
728
|
+
};
|
|
729
|
+
}
|
|
730
|
+
}
|
|
731
|
+
|
|
732
|
+
// src/audio/permissions.ts
|
|
733
|
+
async function checkMicrophonePermission(language) {
|
|
734
|
+
const lang = validateLanguage(language);
|
|
735
|
+
if (!("navigator" in globalThis) || !navigator.mediaDevices) {
|
|
736
|
+
return {
|
|
737
|
+
granted: false,
|
|
738
|
+
state: "denied",
|
|
739
|
+
message: translate(lang, "microphoneAccessError")
|
|
740
|
+
};
|
|
741
|
+
}
|
|
742
|
+
try {
|
|
743
|
+
if ("permissions" in navigator && "query" in navigator.permissions) {
|
|
744
|
+
try {
|
|
745
|
+
const permissionResult = await navigator.permissions.query({ name: "microphone" });
|
|
746
|
+
if (permissionResult.state === "granted") {
|
|
747
|
+
return { granted: true, state: "granted" };
|
|
748
|
+
}
|
|
749
|
+
if (permissionResult.state === "denied") {
|
|
750
|
+
return {
|
|
751
|
+
granted: false,
|
|
752
|
+
state: "denied",
|
|
753
|
+
message: translate(lang, "microphoneAccessDenied")
|
|
754
|
+
};
|
|
755
|
+
}
|
|
756
|
+
} catch (error) {
|
|
757
|
+
console.warn("[Autoscriber] permissions API error", error);
|
|
758
|
+
}
|
|
759
|
+
}
|
|
760
|
+
const stream = await navigator.mediaDevices.getUserMedia({ audio: { channelCount: 1 } });
|
|
761
|
+
stream.getTracks().forEach((track) => track.stop());
|
|
762
|
+
return { granted: true, state: "granted" };
|
|
763
|
+
} catch (error) {
|
|
764
|
+
const err = error;
|
|
765
|
+
if (err?.name === "NotAllowedError" || err?.name === "PermissionDeniedError") {
|
|
766
|
+
return {
|
|
767
|
+
granted: false,
|
|
768
|
+
state: "denied",
|
|
769
|
+
message: translate(lang, "microphoneRequired")
|
|
770
|
+
};
|
|
771
|
+
}
|
|
772
|
+
if (err?.name === "NotFoundError" || err?.name === "DevicesNotFoundError") {
|
|
773
|
+
return {
|
|
774
|
+
granted: false,
|
|
775
|
+
state: "denied",
|
|
776
|
+
message: translate(lang, "microphoneNotFound")
|
|
777
|
+
};
|
|
778
|
+
}
|
|
779
|
+
return {
|
|
780
|
+
granted: false,
|
|
781
|
+
state: "denied",
|
|
782
|
+
message: translate(lang, "microphoneAccessError")
|
|
783
|
+
};
|
|
784
|
+
}
|
|
785
|
+
}
|
|
786
|
+
async function getAvailableMicrophones(language) {
|
|
787
|
+
const lang = validateLanguage(language);
|
|
788
|
+
if (!("navigator" in globalThis) || !navigator.mediaDevices?.enumerateDevices) {
|
|
789
|
+
console.warn("[Autoscriber] media devices API unavailable");
|
|
790
|
+
return [];
|
|
791
|
+
}
|
|
792
|
+
try {
|
|
793
|
+
const devices = await navigator.mediaDevices.enumerateDevices();
|
|
794
|
+
return devices.filter((device) => device.kind === "audioinput").map((device) => ({
|
|
795
|
+
deviceId: device.deviceId,
|
|
796
|
+
label: device.label || `${translate(lang, "microphone")} ${device.deviceId.slice(0, 8)}`,
|
|
797
|
+
groupId: device.groupId
|
|
798
|
+
}));
|
|
799
|
+
} catch (error) {
|
|
800
|
+
console.error("[Autoscriber] failed to enumerate microphones", error);
|
|
801
|
+
return [];
|
|
802
|
+
}
|
|
803
|
+
}
|
|
804
|
+
|
|
805
|
+
// src/utils/token.ts
|
|
806
|
+
function decodeJwtPayload(token) {
|
|
807
|
+
const parts = token.split(".");
|
|
808
|
+
if (parts.length !== 3) {
|
|
809
|
+
return null;
|
|
810
|
+
}
|
|
811
|
+
try {
|
|
812
|
+
const payload = JSON.parse(atob(parts[1]));
|
|
813
|
+
return payload;
|
|
814
|
+
} catch (error) {
|
|
815
|
+
console.error("[Autoscriber] failed to decode JWT payload", error);
|
|
816
|
+
return null;
|
|
817
|
+
}
|
|
818
|
+
}
|
|
819
|
+
function getSessionIdFromToken(token) {
|
|
820
|
+
const payload = decodeJwtPayload(token);
|
|
821
|
+
if (!payload)
|
|
822
|
+
return null;
|
|
823
|
+
const sessionId = payload.sessionId ?? payload.session_id;
|
|
824
|
+
return typeof sessionId === "string" ? sessionId : null;
|
|
825
|
+
}
|
|
826
|
+
function isTokenExpiredOrExpiring(token, secondsBuffer = 10) {
|
|
827
|
+
const payload = decodeJwtPayload(token);
|
|
828
|
+
if (!payload)
|
|
829
|
+
return true;
|
|
830
|
+
const exp = payload.exp;
|
|
831
|
+
if (typeof exp !== "number")
|
|
832
|
+
return true;
|
|
833
|
+
const current = Math.floor(Date.now() / 1e3);
|
|
834
|
+
return current >= exp - secondsBuffer;
|
|
835
|
+
}
|
|
836
|
+
|
|
837
|
+
// src/audio/wake-lock.ts
|
|
838
|
+
var WakeLockManager = class _WakeLockManager {
|
|
839
|
+
constructor() {
|
|
840
|
+
this.wakeLock = null;
|
|
841
|
+
}
|
|
842
|
+
/**
|
|
843
|
+
* Check if Wake Lock API is supported in the current browser
|
|
844
|
+
*/
|
|
845
|
+
static isSupported() {
|
|
846
|
+
return typeof navigator !== "undefined" && "wakeLock" in navigator;
|
|
847
|
+
}
|
|
848
|
+
/**
|
|
849
|
+
* Request a wake lock to prevent the screen from sleeping
|
|
850
|
+
* @returns Promise that resolves when wake lock is acquired, or rejects if not supported/failed
|
|
851
|
+
*/
|
|
852
|
+
async request() {
|
|
853
|
+
if (!_WakeLockManager.isSupported()) {
|
|
854
|
+
throw new Error("Wake Lock API is not supported in this browser");
|
|
855
|
+
}
|
|
856
|
+
try {
|
|
857
|
+
this.wakeLock = await navigator.wakeLock.request("screen");
|
|
858
|
+
this.wakeLock.addEventListener("release", () => {
|
|
859
|
+
this.wakeLock = null;
|
|
860
|
+
});
|
|
861
|
+
} catch (error) {
|
|
862
|
+
throw new Error(`Failed to acquire wake lock: ${error instanceof Error ? error.message : String(error)}`);
|
|
863
|
+
}
|
|
864
|
+
}
|
|
865
|
+
/**
|
|
866
|
+
* Release the wake lock if it's currently active
|
|
867
|
+
*/
|
|
868
|
+
async release() {
|
|
869
|
+
if (this.wakeLock) {
|
|
870
|
+
try {
|
|
871
|
+
await this.wakeLock.release();
|
|
872
|
+
} catch (error) {
|
|
873
|
+
console.warn("[WakeLock] Error releasing wake lock", error);
|
|
874
|
+
} finally {
|
|
875
|
+
this.wakeLock = null;
|
|
876
|
+
}
|
|
877
|
+
}
|
|
878
|
+
}
|
|
879
|
+
/**
|
|
880
|
+
* Check if a wake lock is currently active
|
|
881
|
+
*/
|
|
882
|
+
isActive() {
|
|
883
|
+
return this.wakeLock !== null;
|
|
884
|
+
}
|
|
885
|
+
/**
|
|
886
|
+
* Clean up resources (release wake lock and remove listeners)
|
|
887
|
+
*/
|
|
888
|
+
async destroy() {
|
|
889
|
+
await this.release();
|
|
890
|
+
}
|
|
891
|
+
};
|
|
892
|
+
|
|
893
|
+
// src/core/assistant-client.ts
|
|
894
|
+
var TOKEN_REFRESH_THROTTLE_MS = 15e3;
|
|
895
|
+
var AssistantClient = class {
|
|
896
|
+
constructor(options) {
|
|
897
|
+
this.bus = new EventBus();
|
|
898
|
+
this.logger = createLogger();
|
|
899
|
+
this.messageStore = new MessageStore();
|
|
900
|
+
this.offlineQueue = new OfflineQueue();
|
|
901
|
+
this.audioManager = new AudioManager();
|
|
902
|
+
this.wakeLockManager = new WakeLockManager();
|
|
903
|
+
this.bufferTimer = null;
|
|
904
|
+
this.audioBuffer = [];
|
|
905
|
+
this.voiceCommandBuffer = [];
|
|
906
|
+
this.manualDisconnect = false;
|
|
907
|
+
this.pendingRequests = /* @__PURE__ */ new Map();
|
|
908
|
+
this.reconnectTimer = null;
|
|
909
|
+
this.lastTokenRefreshAttempt = null;
|
|
910
|
+
this.tokenRefreshInFlight = false;
|
|
911
|
+
this.hasConnected = false;
|
|
912
|
+
this.handleBrowserOnline = () => {
|
|
913
|
+
this.snapshot.connection.online = true;
|
|
914
|
+
if (this.snapshot.connection.reason === "offline") {
|
|
915
|
+
this.snapshot.connection.reason = "reconnecting";
|
|
916
|
+
}
|
|
917
|
+
this.emitConnection();
|
|
918
|
+
this.manualDisconnect = false;
|
|
919
|
+
if (this.websocket.readyState !== WebSocket.OPEN) {
|
|
920
|
+
void this.connectWebsocket();
|
|
921
|
+
} else {
|
|
922
|
+
this.flushOfflineQueue();
|
|
923
|
+
}
|
|
924
|
+
};
|
|
925
|
+
this.handleBrowserOffline = () => {
|
|
926
|
+
if (!this.snapshot.connection.online && this.snapshot.connection.reason === "offline") {
|
|
927
|
+
return;
|
|
928
|
+
}
|
|
929
|
+
this.snapshot.connection.online = false;
|
|
930
|
+
this.snapshot.connection.reason = "offline";
|
|
931
|
+
if (this.snapshot.connection.websocketStatus === "open") {
|
|
932
|
+
this.snapshot.connection.websocketStatus = "closing";
|
|
933
|
+
}
|
|
934
|
+
this.snapshot.connection.buffering = this.offlineQueue.size > 0 || this.snapshot.offlineQueueLength > 0;
|
|
935
|
+
this.emitConnection();
|
|
936
|
+
if (this.snapshot.recording.isRecording) {
|
|
937
|
+
if (this.snapshot.recording.mode === "transcription") {
|
|
938
|
+
this.pushInteraction({
|
|
939
|
+
id: generateMessageId(),
|
|
940
|
+
label: translate(this.snapshot.language, "transcriptionBufferingOffline"),
|
|
941
|
+
status: "pending",
|
|
942
|
+
type: "transcription",
|
|
943
|
+
metadata: { reason: "offline" }
|
|
944
|
+
});
|
|
945
|
+
} else if (this.snapshot.recording.mode === "voice_command") {
|
|
946
|
+
this.snapshot.recording.wasTranscriptionActive = false;
|
|
947
|
+
this.snapshot.recording.transcriptionPausedState = false;
|
|
948
|
+
this.pushInteraction({
|
|
949
|
+
id: generateMessageId(),
|
|
950
|
+
label: translate(this.snapshot.language, "voiceCommandCancelledOffline"),
|
|
951
|
+
status: "error",
|
|
952
|
+
type: "voice"
|
|
953
|
+
});
|
|
954
|
+
this.stopRecording();
|
|
955
|
+
}
|
|
956
|
+
}
|
|
957
|
+
this.pushInteraction({
|
|
958
|
+
id: generateMessageId(),
|
|
959
|
+
label: translate(this.snapshot.language, "networkOffline"),
|
|
960
|
+
status: "error",
|
|
961
|
+
type: "info"
|
|
962
|
+
});
|
|
963
|
+
const readyState = this.websocket.readyState;
|
|
964
|
+
if (readyState === WebSocket.OPEN || readyState === WebSocket.CONNECTING) {
|
|
965
|
+
this.manualDisconnect = true;
|
|
966
|
+
this.websocket.close(4001, "Browser offline");
|
|
967
|
+
}
|
|
968
|
+
};
|
|
969
|
+
if (!options?.token) {
|
|
970
|
+
throw new Error("Autoscriber Assistant requires a token");
|
|
971
|
+
}
|
|
972
|
+
this.options = options;
|
|
973
|
+
this.logger = createLogger(options.logger);
|
|
974
|
+
this.token = options.token;
|
|
975
|
+
const language = validateLanguage(options.language);
|
|
976
|
+
this.snapshot = {
|
|
977
|
+
language,
|
|
978
|
+
debugMode: options.debugMode ?? false,
|
|
979
|
+
messages: [],
|
|
980
|
+
interactions: [],
|
|
981
|
+
connection: {
|
|
982
|
+
websocketStatus: "idle",
|
|
983
|
+
online: typeof navigator !== "undefined" ? navigator.onLine : true,
|
|
984
|
+
buffering: false
|
|
985
|
+
},
|
|
986
|
+
microphones: {
|
|
987
|
+
available: [],
|
|
988
|
+
permission: "unknown"
|
|
989
|
+
},
|
|
990
|
+
recording: {
|
|
991
|
+
mode: "idle",
|
|
992
|
+
transcriptionId: null,
|
|
993
|
+
isRecording: false,
|
|
994
|
+
isPaused: false,
|
|
995
|
+
bufferedChunks: 0,
|
|
996
|
+
wasTranscriptionActive: false,
|
|
997
|
+
transcriptionPausedState: false
|
|
998
|
+
},
|
|
999
|
+
offlineQueueLength: 0,
|
|
1000
|
+
lastUpdatedAt: Date.now()
|
|
1001
|
+
};
|
|
1002
|
+
this.websocket = new WebSocketClient({
|
|
1003
|
+
logger: options.logger,
|
|
1004
|
+
reconnectDelay: options.websocketReconnectDelay,
|
|
1005
|
+
autoReconnect: false
|
|
1006
|
+
});
|
|
1007
|
+
this.registerWebsocketEvents();
|
|
1008
|
+
this.setupNetworkListeners();
|
|
1009
|
+
this.initializeMicrophones();
|
|
1010
|
+
this.connectWebsocket();
|
|
1011
|
+
const sessionId = getSessionIdFromToken(this.token);
|
|
1012
|
+
this.logStructured("info", "Session created", {
|
|
1013
|
+
sessionId: sessionId ?? "unknown",
|
|
1014
|
+
language,
|
|
1015
|
+
websocketUrl: options.websocketUrl ?? "default",
|
|
1016
|
+
debugMode: options.debugMode ?? false
|
|
1017
|
+
});
|
|
1018
|
+
}
|
|
1019
|
+
/**
|
|
1020
|
+
* Internal structured logging helper for Datadog monitoring
|
|
1021
|
+
* Formats logs as structured JSON for better parsing and filtering
|
|
1022
|
+
*/
|
|
1023
|
+
logStructured(level, message, metadata) {
|
|
1024
|
+
try {
|
|
1025
|
+
const structuredLog = {
|
|
1026
|
+
message,
|
|
1027
|
+
level,
|
|
1028
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1029
|
+
sdk: "autoscriber-core",
|
|
1030
|
+
...metadata
|
|
1031
|
+
};
|
|
1032
|
+
const logMessage = `[Autoscriber] ${message}${metadata ? ` ${JSON.stringify(metadata)}` : ""}`;
|
|
1033
|
+
if (this.options.onLog) {
|
|
1034
|
+
try {
|
|
1035
|
+
this.options.onLog(level, message, structuredLog);
|
|
1036
|
+
} catch (error) {
|
|
1037
|
+
this.logger.warn("[Autoscriber] onLog callback failed", error);
|
|
1038
|
+
}
|
|
1039
|
+
}
|
|
1040
|
+
switch (level) {
|
|
1041
|
+
case "info":
|
|
1042
|
+
this.logger.info(logMessage, structuredLog);
|
|
1043
|
+
break;
|
|
1044
|
+
case "warn":
|
|
1045
|
+
this.logger.warn(logMessage, structuredLog);
|
|
1046
|
+
break;
|
|
1047
|
+
case "error":
|
|
1048
|
+
this.logger.error(logMessage, structuredLog);
|
|
1049
|
+
break;
|
|
1050
|
+
}
|
|
1051
|
+
} catch (error) {
|
|
1052
|
+
this.logger.error(`[Autoscriber] Failed to log structured message: ${message}`, error);
|
|
1053
|
+
}
|
|
1054
|
+
}
|
|
1055
|
+
on(event, listener) {
|
|
1056
|
+
return this.bus.on(event, listener);
|
|
1057
|
+
}
|
|
1058
|
+
getSnapshot() {
|
|
1059
|
+
return { ...this.snapshot, messages: this.messageStore.getMessages(), interactions: this.messageStore.getInteractions() };
|
|
1060
|
+
}
|
|
1061
|
+
setDebugMode(enabled) {
|
|
1062
|
+
const next = Boolean(enabled);
|
|
1063
|
+
if (this.snapshot.debugMode === next) {
|
|
1064
|
+
return;
|
|
1065
|
+
}
|
|
1066
|
+
this.snapshot.debugMode = next;
|
|
1067
|
+
this.emitSnapshot();
|
|
1068
|
+
}
|
|
1069
|
+
setLanguage(language) {
|
|
1070
|
+
const validatedLanguage = validateLanguage(language);
|
|
1071
|
+
if (this.snapshot.language === validatedLanguage) {
|
|
1072
|
+
return;
|
|
1073
|
+
}
|
|
1074
|
+
const oldLanguage = this.snapshot.language;
|
|
1075
|
+
this.snapshot.language = validatedLanguage;
|
|
1076
|
+
this.emitSnapshot();
|
|
1077
|
+
this.bus.emit("language:changed", validatedLanguage);
|
|
1078
|
+
const sessionId = getSessionIdFromToken(this.token);
|
|
1079
|
+
this.logStructured("info", "Language changed", {
|
|
1080
|
+
oldLanguage,
|
|
1081
|
+
newLanguage: validatedLanguage,
|
|
1082
|
+
sessionId: sessionId ?? "unknown"
|
|
1083
|
+
});
|
|
1084
|
+
}
|
|
1085
|
+
destroy() {
|
|
1086
|
+
this.manualDisconnect = true;
|
|
1087
|
+
if (this.reconnectTimer) {
|
|
1088
|
+
clearTimeout(this.reconnectTimer);
|
|
1089
|
+
this.reconnectTimer = null;
|
|
1090
|
+
}
|
|
1091
|
+
this.websocket.close(1e3, "Destroyed");
|
|
1092
|
+
this.audioManager.teardown();
|
|
1093
|
+
void this.wakeLockManager.destroy().catch((error) => {
|
|
1094
|
+
this.logger.warn("[Autoscriber] Error releasing wake lock on destroy", error);
|
|
1095
|
+
});
|
|
1096
|
+
if (typeof window !== "undefined") {
|
|
1097
|
+
window.removeEventListener("online", this.handleBrowserOnline);
|
|
1098
|
+
window.removeEventListener("offline", this.handleBrowserOffline);
|
|
1099
|
+
}
|
|
1100
|
+
if (this.bufferTimer) {
|
|
1101
|
+
clearInterval(this.bufferTimer);
|
|
1102
|
+
this.bufferTimer = null;
|
|
1103
|
+
}
|
|
1104
|
+
this.messageStore.reset();
|
|
1105
|
+
this.bus.clear();
|
|
1106
|
+
}
|
|
1107
|
+
/** Chat related methods */
|
|
1108
|
+
sendTextMessage(text) {
|
|
1109
|
+
const trimmed = text.trim();
|
|
1110
|
+
if (!trimmed) {
|
|
1111
|
+
throw new Error("Cannot send empty message");
|
|
1112
|
+
}
|
|
1113
|
+
const id = generateMessageId();
|
|
1114
|
+
const message = this.messageStore.createMessage({
|
|
1115
|
+
id,
|
|
1116
|
+
request: trimmed,
|
|
1117
|
+
type: "chat"
|
|
1118
|
+
});
|
|
1119
|
+
this.emitMessages();
|
|
1120
|
+
this.sendMessage({
|
|
1121
|
+
jsonrpc: "2.0",
|
|
1122
|
+
id,
|
|
1123
|
+
method: "chat",
|
|
1124
|
+
params: {
|
|
1125
|
+
mediaType: "text/plain",
|
|
1126
|
+
data: trimmed
|
|
1127
|
+
}
|
|
1128
|
+
});
|
|
1129
|
+
return message;
|
|
1130
|
+
}
|
|
1131
|
+
/** Recording controls */
|
|
1132
|
+
async startTranscription(customId) {
|
|
1133
|
+
if (this.snapshot.recording.isRecording) {
|
|
1134
|
+
this.logger.warn("[Autoscriber] startTranscription called while recording");
|
|
1135
|
+
return this.snapshot.recording.transcriptionId ?? "";
|
|
1136
|
+
}
|
|
1137
|
+
const permission = await checkMicrophonePermission(this.snapshot.language);
|
|
1138
|
+
this.updateMicrophonePermission(permission.state);
|
|
1139
|
+
if (!permission.granted) {
|
|
1140
|
+
if (permission.message) {
|
|
1141
|
+
this.pushInteraction({
|
|
1142
|
+
id: generateMessageId(),
|
|
1143
|
+
label: permission.message,
|
|
1144
|
+
status: "error",
|
|
1145
|
+
type: "transcription"
|
|
1146
|
+
});
|
|
1147
|
+
}
|
|
1148
|
+
return "";
|
|
1149
|
+
}
|
|
1150
|
+
const transcriptionId = customId ?? generateMessageId();
|
|
1151
|
+
await this.audioManager.init({
|
|
1152
|
+
deviceId: this.snapshot.microphones.selected?.deviceId,
|
|
1153
|
+
targetSampleRate: this.options.targetSampleRate,
|
|
1154
|
+
onRecorderData: (pcm16) => this.handleRecorderData(pcm16, "transcription")
|
|
1155
|
+
});
|
|
1156
|
+
try {
|
|
1157
|
+
await this.wakeLockManager.request();
|
|
1158
|
+
} catch (error) {
|
|
1159
|
+
this.logger.warn("[Autoscriber] Failed to acquire wake lock", error);
|
|
1160
|
+
}
|
|
1161
|
+
this.audioBuffer = [];
|
|
1162
|
+
this.snapshot.recording = {
|
|
1163
|
+
...this.snapshot.recording,
|
|
1164
|
+
mode: "transcription",
|
|
1165
|
+
transcriptionId,
|
|
1166
|
+
isRecording: true,
|
|
1167
|
+
isPaused: false,
|
|
1168
|
+
bufferedChunks: 0
|
|
1169
|
+
};
|
|
1170
|
+
this.emitRecording();
|
|
1171
|
+
const sessionId = getSessionIdFromToken(this.token);
|
|
1172
|
+
this.logStructured("info", "Transcription started", {
|
|
1173
|
+
transcriptionId,
|
|
1174
|
+
mode: "transcription",
|
|
1175
|
+
sessionId: sessionId ?? "unknown",
|
|
1176
|
+
deviceId: this.snapshot.microphones.selected?.deviceId
|
|
1177
|
+
});
|
|
1178
|
+
this.pushInteraction({
|
|
1179
|
+
id: transcriptionId,
|
|
1180
|
+
label: translate(this.snapshot.language, "transcriptionStarted"),
|
|
1181
|
+
status: "pending",
|
|
1182
|
+
type: "transcription"
|
|
1183
|
+
});
|
|
1184
|
+
if (this.bufferTimer) {
|
|
1185
|
+
clearInterval(this.bufferTimer);
|
|
1186
|
+
}
|
|
1187
|
+
this.bufferTimer = window.setInterval(() => {
|
|
1188
|
+
if (this.snapshot.recording.mode === "transcription" && this.audioBuffer.length > 0 && !this.snapshot.recording.isPaused) {
|
|
1189
|
+
this.sendBufferedAudio();
|
|
1190
|
+
}
|
|
1191
|
+
}, 5e3);
|
|
1192
|
+
return transcriptionId;
|
|
1193
|
+
}
|
|
1194
|
+
pauseTranscription() {
|
|
1195
|
+
if (this.snapshot.recording.mode !== "transcription")
|
|
1196
|
+
return;
|
|
1197
|
+
this.snapshot.recording.isPaused = !this.snapshot.recording.isPaused;
|
|
1198
|
+
this.emitRecording();
|
|
1199
|
+
}
|
|
1200
|
+
stopRecording() {
|
|
1201
|
+
if (!this.snapshot.recording.isRecording)
|
|
1202
|
+
return;
|
|
1203
|
+
const currentMode = this.snapshot.recording.mode;
|
|
1204
|
+
const shouldResumeTranscription = currentMode === "voice_command" && this.snapshot.recording.wasTranscriptionActive;
|
|
1205
|
+
const resumePausedState = this.snapshot.recording.transcriptionPausedState;
|
|
1206
|
+
if (currentMode === "transcription") {
|
|
1207
|
+
const transcriptionId = this.snapshot.recording.transcriptionId ?? generateMessageId();
|
|
1208
|
+
this.pushInteraction({
|
|
1209
|
+
id: transcriptionId,
|
|
1210
|
+
label: translate(this.snapshot.language, "transcriptionStopped"),
|
|
1211
|
+
status: "success",
|
|
1212
|
+
type: "transcription"
|
|
1213
|
+
});
|
|
1214
|
+
if (this.audioBuffer.length > 0) {
|
|
1215
|
+
this.sendBufferedAudio(true);
|
|
1216
|
+
}
|
|
1217
|
+
const sessionId = getSessionIdFromToken(this.token);
|
|
1218
|
+
this.logStructured("info", "Transcription stopped", {
|
|
1219
|
+
transcriptionId,
|
|
1220
|
+
mode: "transcription",
|
|
1221
|
+
sessionId: sessionId ?? "unknown",
|
|
1222
|
+
bufferedChunks: this.snapshot.recording.bufferedChunks
|
|
1223
|
+
});
|
|
1224
|
+
} else if (currentMode === "voice_command") {
|
|
1225
|
+
if (this.voiceCommandBuffer.length > 0) {
|
|
1226
|
+
this.sendVoiceCommandAudio();
|
|
1227
|
+
}
|
|
1228
|
+
}
|
|
1229
|
+
this.audioManager.stop();
|
|
1230
|
+
void this.wakeLockManager.release().catch((error) => {
|
|
1231
|
+
this.logger.warn("[Autoscriber] Failed to release wake lock", error);
|
|
1232
|
+
});
|
|
1233
|
+
if (shouldResumeTranscription) {
|
|
1234
|
+
void this.resumeTranscriptionAfterVoiceCommand(resumePausedState);
|
|
1235
|
+
} else {
|
|
1236
|
+
this.resetRecordingState();
|
|
1237
|
+
}
|
|
1238
|
+
}
|
|
1239
|
+
async startVoiceCommand() {
|
|
1240
|
+
if (this.snapshot.recording.mode === "transcription") {
|
|
1241
|
+
this.snapshot.recording.wasTranscriptionActive = true;
|
|
1242
|
+
this.snapshot.recording.transcriptionPausedState = this.snapshot.recording.isPaused;
|
|
1243
|
+
this.snapshot.recording.isPaused = true;
|
|
1244
|
+
}
|
|
1245
|
+
const permission = await checkMicrophonePermission(this.snapshot.language);
|
|
1246
|
+
this.updateMicrophonePermission(permission.state);
|
|
1247
|
+
if (!permission.granted) {
|
|
1248
|
+
if (permission.message) {
|
|
1249
|
+
this.pushInteraction({
|
|
1250
|
+
id: generateMessageId(),
|
|
1251
|
+
label: permission.message,
|
|
1252
|
+
status: "error",
|
|
1253
|
+
type: "voice"
|
|
1254
|
+
});
|
|
1255
|
+
}
|
|
1256
|
+
return;
|
|
1257
|
+
}
|
|
1258
|
+
await this.audioManager.init({
|
|
1259
|
+
deviceId: this.snapshot.microphones.selected?.deviceId,
|
|
1260
|
+
targetSampleRate: this.options.targetSampleRate,
|
|
1261
|
+
onRecorderData: (pcm16) => this.handleRecorderData(pcm16, "voice_command")
|
|
1262
|
+
});
|
|
1263
|
+
try {
|
|
1264
|
+
await this.wakeLockManager.request();
|
|
1265
|
+
} catch (error) {
|
|
1266
|
+
this.logger.warn("[Autoscriber] Failed to acquire wake lock", error);
|
|
1267
|
+
}
|
|
1268
|
+
this.voiceCommandBuffer = [];
|
|
1269
|
+
this.snapshot.recording = {
|
|
1270
|
+
...this.snapshot.recording,
|
|
1271
|
+
mode: "voice_command",
|
|
1272
|
+
isRecording: true,
|
|
1273
|
+
isPaused: false,
|
|
1274
|
+
bufferedChunks: 0
|
|
1275
|
+
};
|
|
1276
|
+
this.emitRecording();
|
|
1277
|
+
const sessionId = getSessionIdFromToken(this.token);
|
|
1278
|
+
this.logStructured("info", "Voice command started", {
|
|
1279
|
+
mode: "voice_command",
|
|
1280
|
+
sessionId: sessionId ?? "unknown",
|
|
1281
|
+
deviceId: this.snapshot.microphones.selected?.deviceId
|
|
1282
|
+
});
|
|
1283
|
+
}
|
|
1284
|
+
/** FHIR context */
|
|
1285
|
+
setFhirContext(resource) {
|
|
1286
|
+
const id = generateMessageId();
|
|
1287
|
+
const resourceType = typeof resource === "object" && resource && "resourceType" in resource ? String(resource.resourceType) : "Resource";
|
|
1288
|
+
this.pushInteraction({
|
|
1289
|
+
id,
|
|
1290
|
+
label: `Updating ${resourceType} context`,
|
|
1291
|
+
status: "pending",
|
|
1292
|
+
type: "fhir",
|
|
1293
|
+
metadata: { resourceType }
|
|
1294
|
+
});
|
|
1295
|
+
this.sendMessage({
|
|
1296
|
+
jsonrpc: "2.0",
|
|
1297
|
+
id,
|
|
1298
|
+
method: "fhirContext",
|
|
1299
|
+
params: {
|
|
1300
|
+
mediaType: "application/fhir+json",
|
|
1301
|
+
data: resource
|
|
1302
|
+
}
|
|
1303
|
+
});
|
|
1304
|
+
this.pendingRequests.set(id, { id, method: "fhirContext" });
|
|
1305
|
+
return id;
|
|
1306
|
+
}
|
|
1307
|
+
fillQuestionnaire(context) {
|
|
1308
|
+
const id = generateMessageId();
|
|
1309
|
+
const message = this.messageStore.createMessage({
|
|
1310
|
+
id,
|
|
1311
|
+
request: null,
|
|
1312
|
+
type: "questionnaire"
|
|
1313
|
+
});
|
|
1314
|
+
this.emitMessages();
|
|
1315
|
+
this.pushInteraction({
|
|
1316
|
+
id,
|
|
1317
|
+
label: "Summarising",
|
|
1318
|
+
status: "pending",
|
|
1319
|
+
type: "questionnaire"
|
|
1320
|
+
});
|
|
1321
|
+
this.sendMessage({
|
|
1322
|
+
jsonrpc: "2.0",
|
|
1323
|
+
id,
|
|
1324
|
+
method: "tool",
|
|
1325
|
+
params: {
|
|
1326
|
+
name: "fillQuestionnaire",
|
|
1327
|
+
args: context ? { context } : {}
|
|
1328
|
+
}
|
|
1329
|
+
});
|
|
1330
|
+
this.pendingRequests.set(id, { id, method: "fillQuestionnaire" });
|
|
1331
|
+
return id;
|
|
1332
|
+
}
|
|
1333
|
+
/** Microphone controls */
|
|
1334
|
+
async refreshMicrophones() {
|
|
1335
|
+
const microphones = await getAvailableMicrophones(this.snapshot.language);
|
|
1336
|
+
const selected = this.snapshot.microphones.selected;
|
|
1337
|
+
const stillAvailable = selected && microphones.find((mic) => mic.deviceId === selected.deviceId);
|
|
1338
|
+
this.snapshot.microphones = {
|
|
1339
|
+
...this.snapshot.microphones,
|
|
1340
|
+
available: microphones,
|
|
1341
|
+
selected: stillAvailable ?? microphones[0] ?? selected
|
|
1342
|
+
};
|
|
1343
|
+
this.emitMicrophones();
|
|
1344
|
+
}
|
|
1345
|
+
setSelectedMicrophone(device) {
|
|
1346
|
+
this.snapshot.microphones = {
|
|
1347
|
+
...this.snapshot.microphones,
|
|
1348
|
+
selected: device
|
|
1349
|
+
};
|
|
1350
|
+
if (typeof window !== "undefined" && window.localStorage) {
|
|
1351
|
+
window.localStorage.setItem(
|
|
1352
|
+
"autoscriber_selected_microphone",
|
|
1353
|
+
JSON.stringify(device)
|
|
1354
|
+
);
|
|
1355
|
+
}
|
|
1356
|
+
this.emitMicrophones();
|
|
1357
|
+
}
|
|
1358
|
+
/** Internal plumbing */
|
|
1359
|
+
registerWebsocketEvents() {
|
|
1360
|
+
this.websocket.on("open", () => {
|
|
1361
|
+
const wasOffline = this.hasConnected && (this.snapshot.connection.reason === "offline" || this.snapshot.connection.reason === "reconnecting");
|
|
1362
|
+
if (this.reconnectTimer) {
|
|
1363
|
+
clearTimeout(this.reconnectTimer);
|
|
1364
|
+
this.reconnectTimer = null;
|
|
1365
|
+
}
|
|
1366
|
+
this.snapshot.connection.websocketStatus = "open";
|
|
1367
|
+
this.snapshot.connection.reason = void 0;
|
|
1368
|
+
this.snapshot.connection.lastError = void 0;
|
|
1369
|
+
this.snapshot.connection.buffering = this.offlineQueue.size > 0;
|
|
1370
|
+
this.emitConnection();
|
|
1371
|
+
if (wasOffline) {
|
|
1372
|
+
this.pushInteraction({
|
|
1373
|
+
id: generateMessageId(),
|
|
1374
|
+
label: translate(this.snapshot.language, "connectionRestored"),
|
|
1375
|
+
status: "success",
|
|
1376
|
+
type: "info"
|
|
1377
|
+
});
|
|
1378
|
+
}
|
|
1379
|
+
this.flushOfflineQueue();
|
|
1380
|
+
this.hasConnected = true;
|
|
1381
|
+
});
|
|
1382
|
+
this.websocket.on("close", () => {
|
|
1383
|
+
const preserveOffline = this.snapshot.connection.reason === "offline";
|
|
1384
|
+
this.snapshot.connection.websocketStatus = "closed";
|
|
1385
|
+
if (!preserveOffline) {
|
|
1386
|
+
this.snapshot.connection.reason = this.manualDisconnect ? "manual" : "closed";
|
|
1387
|
+
}
|
|
1388
|
+
this.emitConnection();
|
|
1389
|
+
this.scheduleReconnectAttempt();
|
|
1390
|
+
});
|
|
1391
|
+
this.websocket.on("error", (event) => {
|
|
1392
|
+
this.snapshot.connection.websocketStatus = "closed";
|
|
1393
|
+
if (this.snapshot.connection.reason !== "offline") {
|
|
1394
|
+
this.snapshot.connection.reason = "error";
|
|
1395
|
+
}
|
|
1396
|
+
let errorMessage;
|
|
1397
|
+
let errorDetails = {};
|
|
1398
|
+
if (event instanceof ErrorEvent) {
|
|
1399
|
+
errorMessage = event.message || "WebSocket error event";
|
|
1400
|
+
errorDetails = {
|
|
1401
|
+
type: event.type,
|
|
1402
|
+
filename: event.filename,
|
|
1403
|
+
lineno: event.lineno,
|
|
1404
|
+
colno: event.colno,
|
|
1405
|
+
error: event.error ? String(event.error) : void 0
|
|
1406
|
+
};
|
|
1407
|
+
} else if (event instanceof Event) {
|
|
1408
|
+
errorMessage = `WebSocket error: ${event.type}`;
|
|
1409
|
+
errorDetails = {
|
|
1410
|
+
type: event.type,
|
|
1411
|
+
target: event.target ? String(event.target) : void 0,
|
|
1412
|
+
currentTarget: event.currentTarget ? String(event.currentTarget) : void 0,
|
|
1413
|
+
timeStamp: event.timeStamp
|
|
1414
|
+
};
|
|
1415
|
+
} else {
|
|
1416
|
+
errorMessage = "WebSocket error";
|
|
1417
|
+
errorDetails = {
|
|
1418
|
+
errorType: typeof event,
|
|
1419
|
+
errorValue: String(event)
|
|
1420
|
+
};
|
|
1421
|
+
}
|
|
1422
|
+
this.snapshot.connection.lastError = errorMessage;
|
|
1423
|
+
this.emitConnection();
|
|
1424
|
+
const sessionId = getSessionIdFromToken(this.token);
|
|
1425
|
+
const wsUrl = this.buildWebsocketUrl();
|
|
1426
|
+
this.logStructured("error", "WebSocket error event", {
|
|
1427
|
+
error: errorMessage,
|
|
1428
|
+
...errorDetails,
|
|
1429
|
+
websocketUrl: wsUrl,
|
|
1430
|
+
sessionId: sessionId ?? "unknown",
|
|
1431
|
+
readyState: this.websocket.readyState
|
|
1432
|
+
});
|
|
1433
|
+
this.scheduleReconnectAttempt();
|
|
1434
|
+
});
|
|
1435
|
+
this.websocket.on("reconnecting", () => {
|
|
1436
|
+
if (this.snapshot.connection.reason === "offline")
|
|
1437
|
+
return;
|
|
1438
|
+
this.snapshot.connection.websocketStatus = "connecting";
|
|
1439
|
+
this.snapshot.connection.reason = "reconnecting";
|
|
1440
|
+
this.emitConnection();
|
|
1441
|
+
});
|
|
1442
|
+
this.websocket.on("message", (event) => {
|
|
1443
|
+
this.handleMessage(event.data);
|
|
1444
|
+
});
|
|
1445
|
+
}
|
|
1446
|
+
setupNetworkListeners() {
|
|
1447
|
+
if (typeof window === "undefined")
|
|
1448
|
+
return;
|
|
1449
|
+
window.addEventListener("online", this.handleBrowserOnline);
|
|
1450
|
+
window.addEventListener("offline", this.handleBrowserOffline);
|
|
1451
|
+
if (!navigator.onLine) {
|
|
1452
|
+
this.handleBrowserOffline();
|
|
1453
|
+
}
|
|
1454
|
+
}
|
|
1455
|
+
scheduleReconnectAttempt() {
|
|
1456
|
+
if (this.manualDisconnect) {
|
|
1457
|
+
return;
|
|
1458
|
+
}
|
|
1459
|
+
if (!this.snapshot.connection.online) {
|
|
1460
|
+
return;
|
|
1461
|
+
}
|
|
1462
|
+
if (this.reconnectTimer) {
|
|
1463
|
+
return;
|
|
1464
|
+
}
|
|
1465
|
+
const delay = this.options.websocketReconnectDelay ?? 3e3;
|
|
1466
|
+
if (this.hasConnected && this.snapshot.connection.reason !== "offline") {
|
|
1467
|
+
this.snapshot.connection.reason = "reconnecting";
|
|
1468
|
+
}
|
|
1469
|
+
if (this.snapshot.connection.websocketStatus !== "open") {
|
|
1470
|
+
this.snapshot.connection.websocketStatus = "connecting";
|
|
1471
|
+
}
|
|
1472
|
+
this.emitConnection();
|
|
1473
|
+
this.reconnectTimer = setTimeout(() => {
|
|
1474
|
+
this.reconnectTimer = null;
|
|
1475
|
+
if (this.manualDisconnect || !this.snapshot.connection.online) {
|
|
1476
|
+
return;
|
|
1477
|
+
}
|
|
1478
|
+
void this.connectWebsocket();
|
|
1479
|
+
}, delay);
|
|
1480
|
+
}
|
|
1481
|
+
async initializeMicrophones() {
|
|
1482
|
+
if (typeof window !== "undefined" && window.localStorage) {
|
|
1483
|
+
try {
|
|
1484
|
+
const stored = window.localStorage.getItem("autoscriber_selected_microphone");
|
|
1485
|
+
if (stored) {
|
|
1486
|
+
const parsed = JSON.parse(stored);
|
|
1487
|
+
this.snapshot.microphones.selected = parsed;
|
|
1488
|
+
}
|
|
1489
|
+
} catch (error) {
|
|
1490
|
+
this.logger.warn("[Autoscriber] unable to parse stored microphone", error);
|
|
1491
|
+
}
|
|
1492
|
+
}
|
|
1493
|
+
await this.refreshMicrophones();
|
|
1494
|
+
}
|
|
1495
|
+
async connectWebsocket() {
|
|
1496
|
+
try {
|
|
1497
|
+
if (this.options.refreshToken && !this.tokenRefreshInFlight) {
|
|
1498
|
+
const isExpired = isTokenExpiredOrExpiring(this.token, 0);
|
|
1499
|
+
const isExpiringSoon = isTokenExpiredOrExpiring(this.token);
|
|
1500
|
+
if (isExpired || isExpiringSoon) {
|
|
1501
|
+
const now = Date.now();
|
|
1502
|
+
const throttleElapsed = !this.lastTokenRefreshAttempt || now - this.lastTokenRefreshAttempt >= TOKEN_REFRESH_THROTTLE_MS;
|
|
1503
|
+
if (isExpired || throttleElapsed) {
|
|
1504
|
+
this.lastTokenRefreshAttempt = now;
|
|
1505
|
+
const sessionId = getSessionIdFromToken(this.token);
|
|
1506
|
+
if (sessionId) {
|
|
1507
|
+
this.tokenRefreshInFlight = true;
|
|
1508
|
+
try {
|
|
1509
|
+
const newToken = await this.options.refreshToken(sessionId);
|
|
1510
|
+
if (newToken) {
|
|
1511
|
+
this.token = newToken;
|
|
1512
|
+
this.lastTokenRefreshAttempt = null;
|
|
1513
|
+
this.logStructured("info", "Token refresh succeeded", {
|
|
1514
|
+
sessionId,
|
|
1515
|
+
wasExpired: isExpired
|
|
1516
|
+
});
|
|
1517
|
+
} else if (isExpired) {
|
|
1518
|
+
this.logger.warn("[Autoscriber] refresh returned no token while current token expired");
|
|
1519
|
+
this.lastTokenRefreshAttempt = null;
|
|
1520
|
+
this.logStructured("warn", "Token refresh returned no token", {
|
|
1521
|
+
sessionId,
|
|
1522
|
+
wasExpired: true
|
|
1523
|
+
});
|
|
1524
|
+
}
|
|
1525
|
+
} catch (refreshError) {
|
|
1526
|
+
this.logger.error("[Autoscriber] token refresh failed", refreshError);
|
|
1527
|
+
this.logStructured("error", "Token refresh failed", {
|
|
1528
|
+
sessionId,
|
|
1529
|
+
wasExpired: isExpired,
|
|
1530
|
+
error: refreshError instanceof Error ? refreshError.message : String(refreshError)
|
|
1531
|
+
});
|
|
1532
|
+
if (isExpired) {
|
|
1533
|
+
this.lastTokenRefreshAttempt = null;
|
|
1534
|
+
}
|
|
1535
|
+
} finally {
|
|
1536
|
+
this.tokenRefreshInFlight = false;
|
|
1537
|
+
}
|
|
1538
|
+
}
|
|
1539
|
+
}
|
|
1540
|
+
if (isExpired && isTokenExpiredOrExpiring(this.token, 0)) {
|
|
1541
|
+
throw new Error("Token expired and refresh unavailable");
|
|
1542
|
+
}
|
|
1543
|
+
}
|
|
1544
|
+
}
|
|
1545
|
+
const wsUrl = this.buildWebsocketUrl();
|
|
1546
|
+
this.snapshot.connection.websocketStatus = "connecting";
|
|
1547
|
+
if (this.hasConnected && this.snapshot.connection.reason !== "offline") {
|
|
1548
|
+
this.snapshot.connection.reason = "reconnecting";
|
|
1549
|
+
}
|
|
1550
|
+
this.emitConnection();
|
|
1551
|
+
await this.websocket.connect(wsUrl);
|
|
1552
|
+
} catch (error) {
|
|
1553
|
+
this.logger.error("[Autoscriber] failed to connect websocket", error);
|
|
1554
|
+
let errorMessage;
|
|
1555
|
+
let errorDetails = {};
|
|
1556
|
+
if (error instanceof Error) {
|
|
1557
|
+
errorMessage = error.message;
|
|
1558
|
+
errorDetails = {
|
|
1559
|
+
name: error.name,
|
|
1560
|
+
stack: error.stack
|
|
1561
|
+
};
|
|
1562
|
+
} else if (error instanceof ErrorEvent) {
|
|
1563
|
+
errorMessage = error.message || "WebSocket error event";
|
|
1564
|
+
errorDetails = {
|
|
1565
|
+
type: error.type,
|
|
1566
|
+
filename: error.filename,
|
|
1567
|
+
lineno: error.lineno,
|
|
1568
|
+
colno: error.colno,
|
|
1569
|
+
error: error.error ? String(error.error) : void 0
|
|
1570
|
+
};
|
|
1571
|
+
} else if (error instanceof Event) {
|
|
1572
|
+
errorMessage = `WebSocket error: ${error.type}`;
|
|
1573
|
+
errorDetails = {
|
|
1574
|
+
type: error.type,
|
|
1575
|
+
target: error.target ? String(error.target) : void 0,
|
|
1576
|
+
currentTarget: error.currentTarget ? String(error.currentTarget) : void 0,
|
|
1577
|
+
timeStamp: error.timeStamp
|
|
1578
|
+
};
|
|
1579
|
+
} else {
|
|
1580
|
+
errorMessage = String(error);
|
|
1581
|
+
errorDetails = {
|
|
1582
|
+
errorType: typeof error,
|
|
1583
|
+
errorValue: String(error)
|
|
1584
|
+
};
|
|
1585
|
+
}
|
|
1586
|
+
this.snapshot.connection.lastError = errorMessage;
|
|
1587
|
+
this.snapshot.connection.websocketStatus = "closed";
|
|
1588
|
+
if (this.snapshot.connection.reason !== "offline") {
|
|
1589
|
+
this.snapshot.connection.reason = "error";
|
|
1590
|
+
}
|
|
1591
|
+
this.emitConnection();
|
|
1592
|
+
const sessionId = getSessionIdFromToken(this.token);
|
|
1593
|
+
const wsUrl = this.buildWebsocketUrl();
|
|
1594
|
+
this.logStructured("error", "WebSocket connection failed", {
|
|
1595
|
+
error: errorMessage,
|
|
1596
|
+
...errorDetails,
|
|
1597
|
+
websocketUrl: wsUrl,
|
|
1598
|
+
sessionId: sessionId ?? "unknown"
|
|
1599
|
+
});
|
|
1600
|
+
this.scheduleReconnectAttempt();
|
|
1601
|
+
}
|
|
1602
|
+
}
|
|
1603
|
+
buildWebsocketUrl() {
|
|
1604
|
+
if (this.options.websocketUrl) {
|
|
1605
|
+
return `${this.options.websocketUrl}?token=${this.token}`;
|
|
1606
|
+
}
|
|
1607
|
+
if (typeof window === "undefined") {
|
|
1608
|
+
throw new Error("Window is not available to resolve WebSocket URL");
|
|
1609
|
+
}
|
|
1610
|
+
const origin = window.location.origin;
|
|
1611
|
+
const wsBase = origin.replace(/^http/, "ws");
|
|
1612
|
+
return `${wsBase}/ws?token=${this.token}`;
|
|
1613
|
+
}
|
|
1614
|
+
sendMessage(payload) {
|
|
1615
|
+
if (this.snapshot.connection.online && this.websocket.readyState === WebSocket.OPEN) {
|
|
1616
|
+
const message = JSON.stringify(payload);
|
|
1617
|
+
this.websocket.send(message);
|
|
1618
|
+
return;
|
|
1619
|
+
}
|
|
1620
|
+
const wasEmpty = this.offlineQueue.size === 0;
|
|
1621
|
+
this.offlineQueue.enqueue(payload);
|
|
1622
|
+
this.snapshot.offlineQueueLength = this.offlineQueue.size;
|
|
1623
|
+
this.snapshot.connection.buffering = true;
|
|
1624
|
+
if (wasEmpty) {
|
|
1625
|
+
this.pushInteraction({
|
|
1626
|
+
id: generateMessageId(),
|
|
1627
|
+
label: translate(this.snapshot.language, "offlineBuffering"),
|
|
1628
|
+
status: "pending",
|
|
1629
|
+
type: "info"
|
|
1630
|
+
});
|
|
1631
|
+
}
|
|
1632
|
+
this.emitConnection();
|
|
1633
|
+
}
|
|
1634
|
+
flushOfflineQueue() {
|
|
1635
|
+
if (this.offlineQueue.size === 0) {
|
|
1636
|
+
if (this.snapshot.connection.buffering) {
|
|
1637
|
+
this.snapshot.connection.buffering = false;
|
|
1638
|
+
this.emitConnection();
|
|
1639
|
+
}
|
|
1640
|
+
return;
|
|
1641
|
+
}
|
|
1642
|
+
const items = this.offlineQueue.drain();
|
|
1643
|
+
this.snapshot.offlineQueueLength = 0;
|
|
1644
|
+
items.forEach((payload) => this.sendMessage(payload));
|
|
1645
|
+
const hasQueued = this.offlineQueue.size > 0;
|
|
1646
|
+
this.snapshot.connection.buffering = hasQueued;
|
|
1647
|
+
this.snapshot.offlineQueueLength = hasQueued ? this.offlineQueue.size : 0;
|
|
1648
|
+
this.emitConnection();
|
|
1649
|
+
}
|
|
1650
|
+
emitSnapshot() {
|
|
1651
|
+
this.snapshot.lastUpdatedAt = Date.now();
|
|
1652
|
+
this.bus.emit("snapshot", this.getSnapshot());
|
|
1653
|
+
}
|
|
1654
|
+
emitMessages() {
|
|
1655
|
+
const messages = this.messageStore.getMessages();
|
|
1656
|
+
this.snapshot.messages = messages;
|
|
1657
|
+
this.bus.emit("messages:updated", messages);
|
|
1658
|
+
this.emitSnapshot();
|
|
1659
|
+
}
|
|
1660
|
+
emitInteractions() {
|
|
1661
|
+
const interactions = this.messageStore.getInteractions();
|
|
1662
|
+
this.snapshot.interactions = interactions;
|
|
1663
|
+
this.bus.emit("interactions:updated", interactions);
|
|
1664
|
+
this.emitSnapshot();
|
|
1665
|
+
}
|
|
1666
|
+
emitConnection() {
|
|
1667
|
+
this.bus.emit("connection:changed", this.snapshot.connection);
|
|
1668
|
+
this.emitSnapshot();
|
|
1669
|
+
}
|
|
1670
|
+
emitRecording() {
|
|
1671
|
+
this.bus.emit("recording:changed", this.snapshot.recording);
|
|
1672
|
+
this.emitSnapshot();
|
|
1673
|
+
}
|
|
1674
|
+
emitMicrophones() {
|
|
1675
|
+
this.bus.emit("microphones:changed", this.snapshot.microphones);
|
|
1676
|
+
this.emitSnapshot();
|
|
1677
|
+
}
|
|
1678
|
+
pushInteraction(pill) {
|
|
1679
|
+
this.messageStore.upsertInteraction(pill);
|
|
1680
|
+
this.emitInteractions();
|
|
1681
|
+
}
|
|
1682
|
+
updateMicrophonePermission(state) {
|
|
1683
|
+
this.snapshot.microphones.permission = state;
|
|
1684
|
+
this.emitMicrophones();
|
|
1685
|
+
}
|
|
1686
|
+
handleRecorderData(pcm16, mode) {
|
|
1687
|
+
if (mode === "transcription") {
|
|
1688
|
+
if (!this.snapshot.recording.isRecording || this.snapshot.recording.mode !== "transcription") {
|
|
1689
|
+
return;
|
|
1690
|
+
}
|
|
1691
|
+
if (!this.snapshot.recording.isPaused) {
|
|
1692
|
+
this.audioBuffer.push(pcm16);
|
|
1693
|
+
this.snapshot.recording.bufferedChunks = this.audioBuffer.length;
|
|
1694
|
+
}
|
|
1695
|
+
} else if (mode === "voice_command") {
|
|
1696
|
+
if (!this.snapshot.recording.isRecording || this.snapshot.recording.mode !== "voice_command") {
|
|
1697
|
+
return;
|
|
1698
|
+
}
|
|
1699
|
+
this.voiceCommandBuffer.push(pcm16);
|
|
1700
|
+
}
|
|
1701
|
+
}
|
|
1702
|
+
sendBufferedAudio(isFinal = false) {
|
|
1703
|
+
if (!this.snapshot.recording.transcriptionId || this.audioBuffer.length === 0) {
|
|
1704
|
+
return;
|
|
1705
|
+
}
|
|
1706
|
+
const combined = this.concatPcmChunks(this.audioBuffer);
|
|
1707
|
+
this.audioBuffer = [];
|
|
1708
|
+
this.snapshot.recording.bufferedChunks = 0;
|
|
1709
|
+
this.emitRecording();
|
|
1710
|
+
const payload = {
|
|
1711
|
+
jsonrpc: "2.0",
|
|
1712
|
+
method: "transcribe",
|
|
1713
|
+
params: {
|
|
1714
|
+
id: this.snapshot.recording.transcriptionId,
|
|
1715
|
+
mediaType: "audio/pcm",
|
|
1716
|
+
data: arrayBufferToBase64(combined.buffer),
|
|
1717
|
+
isFinal
|
|
1718
|
+
}
|
|
1719
|
+
};
|
|
1720
|
+
if (!this.messageStore.getMessageById(this.snapshot.recording.transcriptionId)) {
|
|
1721
|
+
const message = {
|
|
1722
|
+
id: this.snapshot.recording.transcriptionId,
|
|
1723
|
+
request: null,
|
|
1724
|
+
partialResults: [],
|
|
1725
|
+
result: null,
|
|
1726
|
+
type: "transcription"
|
|
1727
|
+
};
|
|
1728
|
+
this.messageStore.addMessage(message);
|
|
1729
|
+
payload.id = this.snapshot.recording.transcriptionId;
|
|
1730
|
+
}
|
|
1731
|
+
this.sendMessage(payload);
|
|
1732
|
+
}
|
|
1733
|
+
concatPcmChunks(chunks) {
|
|
1734
|
+
const totalLength = chunks.reduce((acc, chunk) => acc + chunk.length, 0);
|
|
1735
|
+
const combined = new Uint8Array(totalLength);
|
|
1736
|
+
let offset = 0;
|
|
1737
|
+
chunks.forEach((chunk) => {
|
|
1738
|
+
combined.set(chunk, offset);
|
|
1739
|
+
offset += chunk.length;
|
|
1740
|
+
});
|
|
1741
|
+
return combined;
|
|
1742
|
+
}
|
|
1743
|
+
async resumeTranscriptionAfterVoiceCommand(shouldBePaused) {
|
|
1744
|
+
this.audioBuffer = [];
|
|
1745
|
+
const transcriptionId = generateMessageId();
|
|
1746
|
+
try {
|
|
1747
|
+
await this.audioManager.init({
|
|
1748
|
+
deviceId: this.snapshot.microphones.selected?.deviceId,
|
|
1749
|
+
targetSampleRate: this.options.targetSampleRate,
|
|
1750
|
+
onRecorderData: (pcm16) => this.handleRecorderData(pcm16, "transcription")
|
|
1751
|
+
});
|
|
1752
|
+
try {
|
|
1753
|
+
await this.wakeLockManager.request();
|
|
1754
|
+
} catch (error) {
|
|
1755
|
+
this.logger.warn("[Autoscriber] Failed to acquire wake lock", error);
|
|
1756
|
+
}
|
|
1757
|
+
this.snapshot.recording = {
|
|
1758
|
+
mode: "transcription",
|
|
1759
|
+
transcriptionId,
|
|
1760
|
+
isRecording: true,
|
|
1761
|
+
isPaused: shouldBePaused,
|
|
1762
|
+
bufferedChunks: 0,
|
|
1763
|
+
wasTranscriptionActive: false,
|
|
1764
|
+
transcriptionPausedState: false
|
|
1765
|
+
};
|
|
1766
|
+
this.emitRecording();
|
|
1767
|
+
if (this.bufferTimer) {
|
|
1768
|
+
clearInterval(this.bufferTimer);
|
|
1769
|
+
}
|
|
1770
|
+
this.bufferTimer = window.setInterval(() => {
|
|
1771
|
+
if (this.snapshot.recording.mode === "transcription" && this.audioBuffer.length > 0 && !this.snapshot.recording.isPaused) {
|
|
1772
|
+
this.sendBufferedAudio();
|
|
1773
|
+
}
|
|
1774
|
+
}, 5e3);
|
|
1775
|
+
} catch (error) {
|
|
1776
|
+
this.logger.error("[Autoscriber] failed to resume transcription", error);
|
|
1777
|
+
this.resetRecordingState();
|
|
1778
|
+
}
|
|
1779
|
+
}
|
|
1780
|
+
sendVoiceCommandAudio() {
|
|
1781
|
+
if (this.voiceCommandBuffer.length === 0)
|
|
1782
|
+
return;
|
|
1783
|
+
const combined = this.concatPcmChunks(this.voiceCommandBuffer);
|
|
1784
|
+
this.voiceCommandBuffer = [];
|
|
1785
|
+
const silence = detectSilence(combined);
|
|
1786
|
+
if (silence.isSilence) {
|
|
1787
|
+
this.pushInteraction({
|
|
1788
|
+
id: generateMessageId(),
|
|
1789
|
+
label: translate(this.snapshot.language, "voiceCommandSilence"),
|
|
1790
|
+
status: "error",
|
|
1791
|
+
type: "voice"
|
|
1792
|
+
});
|
|
1793
|
+
return;
|
|
1794
|
+
}
|
|
1795
|
+
const messageId = generateMessageId();
|
|
1796
|
+
const message = this.messageStore.createMessage({
|
|
1797
|
+
id: messageId,
|
|
1798
|
+
request: "VOICE_COMMAND_ICON",
|
|
1799
|
+
type: "voice"
|
|
1800
|
+
});
|
|
1801
|
+
this.emitMessages();
|
|
1802
|
+
this.sendMessage({
|
|
1803
|
+
jsonrpc: "2.0",
|
|
1804
|
+
id: messageId,
|
|
1805
|
+
method: "voiceCommand",
|
|
1806
|
+
params: {
|
|
1807
|
+
mediaType: "audio/pcm",
|
|
1808
|
+
data: arrayBufferToBase64(combined.buffer)
|
|
1809
|
+
}
|
|
1810
|
+
});
|
|
1811
|
+
this.pendingRequests.set(messageId, { id: messageId, method: "voiceCommand" });
|
|
1812
|
+
}
|
|
1813
|
+
handleMessage(data) {
|
|
1814
|
+
let payload;
|
|
1815
|
+
try {
|
|
1816
|
+
payload = JSON.parse(data);
|
|
1817
|
+
} catch (error) {
|
|
1818
|
+
this.logger.error("[Autoscriber] failed to parse websocket payload", error);
|
|
1819
|
+
return;
|
|
1820
|
+
}
|
|
1821
|
+
if (payload.error) {
|
|
1822
|
+
this.handleErrorResponse(payload);
|
|
1823
|
+
return;
|
|
1824
|
+
}
|
|
1825
|
+
if (payload.result) {
|
|
1826
|
+
this.handleResultResponse(payload);
|
|
1827
|
+
return;
|
|
1828
|
+
}
|
|
1829
|
+
if (payload.method === "partialResult") {
|
|
1830
|
+
this.handlePartialResult(payload.params);
|
|
1831
|
+
return;
|
|
1832
|
+
}
|
|
1833
|
+
if (payload.method === "voiceTranscription") {
|
|
1834
|
+
this.handleVoiceTranscription(payload.params);
|
|
1835
|
+
}
|
|
1836
|
+
}
|
|
1837
|
+
handleErrorResponse(payload) {
|
|
1838
|
+
const id = payload.id;
|
|
1839
|
+
const message = this.messageStore.getMessageById(id);
|
|
1840
|
+
const errorMessage = payload.error?.message ?? "Unknown error";
|
|
1841
|
+
const errorCode = payload.error?.code;
|
|
1842
|
+
if (message) {
|
|
1843
|
+
this.messageStore.setError(id, `Error: ${errorMessage}`, {
|
|
1844
|
+
code: String(errorCode ?? ""),
|
|
1845
|
+
message: errorMessage
|
|
1846
|
+
});
|
|
1847
|
+
this.emitMessages();
|
|
1848
|
+
} else {
|
|
1849
|
+
this.logger.error("[Autoscriber] error for unknown message", payload);
|
|
1850
|
+
}
|
|
1851
|
+
const pending = this.pendingRequests.get(id);
|
|
1852
|
+
if (pending) {
|
|
1853
|
+
this.updateInteractionFromResult(id, { code: String(errorCode ?? ""), message: errorMessage });
|
|
1854
|
+
}
|
|
1855
|
+
this.bus.emit("error", { code: errorCode, message: errorMessage });
|
|
1856
|
+
const sessionId = getSessionIdFromToken(this.token);
|
|
1857
|
+
this.logStructured("error", "Error response received", {
|
|
1858
|
+
messageId: id,
|
|
1859
|
+
errorCode: String(errorCode ?? ""),
|
|
1860
|
+
errorMessage,
|
|
1861
|
+
method: pending?.method,
|
|
1862
|
+
sessionId: sessionId ?? "unknown"
|
|
1863
|
+
});
|
|
1864
|
+
}
|
|
1865
|
+
handleResultResponse(payload) {
|
|
1866
|
+
this.logStructured("info", "Result response received", {
|
|
1867
|
+
messageId: payload.id,
|
|
1868
|
+
result: payload.result
|
|
1869
|
+
});
|
|
1870
|
+
const id = payload.id;
|
|
1871
|
+
const result = payload.result;
|
|
1872
|
+
if (result?.mediaType === "audio/pcm" && result?.data) {
|
|
1873
|
+
const buffer = base64ToArray(result.data);
|
|
1874
|
+
this.audioManager.playbackPcm(buffer);
|
|
1875
|
+
return;
|
|
1876
|
+
}
|
|
1877
|
+
const message = this.messageStore.getMessageById(id);
|
|
1878
|
+
if (message) {
|
|
1879
|
+
const data = result?.data ?? result;
|
|
1880
|
+
if (Array.isArray(data) || typeof data === "string") {
|
|
1881
|
+
this.messageStore.setMessageResult(id, data);
|
|
1882
|
+
} else if (result?.mediaType === "application/fhir+json") {
|
|
1883
|
+
this.messageStore.setMessageResult(id, [
|
|
1884
|
+
{
|
|
1885
|
+
mediaType: "application/fhir+json",
|
|
1886
|
+
data
|
|
1887
|
+
}
|
|
1888
|
+
]);
|
|
1889
|
+
this.bus.emit("fhir:data", data);
|
|
1890
|
+
const sessionId = getSessionIdFromToken(this.token);
|
|
1891
|
+
const resourceType = typeof data === "object" && data && "resourceType" in data ? String(data.resourceType) : "unknown";
|
|
1892
|
+
this.logStructured("info", "FHIR data received", {
|
|
1893
|
+
messageId: id,
|
|
1894
|
+
resourceType,
|
|
1895
|
+
sessionId: sessionId ?? "unknown"
|
|
1896
|
+
});
|
|
1897
|
+
} else {
|
|
1898
|
+
this.messageStore.setMessageResult(id, data);
|
|
1899
|
+
}
|
|
1900
|
+
this.emitMessages();
|
|
1901
|
+
}
|
|
1902
|
+
this.updateInteractionFromResult(id, result);
|
|
1903
|
+
}
|
|
1904
|
+
updateInteractionFromResult(id, result) {
|
|
1905
|
+
const pending = this.pendingRequests.get(id);
|
|
1906
|
+
if (!pending)
|
|
1907
|
+
return;
|
|
1908
|
+
const isSuccess = String(result?.code ?? "200") === "200";
|
|
1909
|
+
this.messageStore.updateInteractionStatus(id, isSuccess ? "success" : "error", {
|
|
1910
|
+
result
|
|
1911
|
+
});
|
|
1912
|
+
this.emitInteractions();
|
|
1913
|
+
this.pendingRequests.delete(id);
|
|
1914
|
+
}
|
|
1915
|
+
handlePartialResult(params) {
|
|
1916
|
+
const id = params.id;
|
|
1917
|
+
const message = this.messageStore.getMessageById(id);
|
|
1918
|
+
if (!message)
|
|
1919
|
+
return;
|
|
1920
|
+
if (params.mediaType === "application/json-patch+json") {
|
|
1921
|
+
this.messageStore.applyJsonPatch(id, params.data);
|
|
1922
|
+
} else if (params.mediaType === "application/fhir+json" && params.data?.resourceType === "Transcript") {
|
|
1923
|
+
this.messageStore.setMessageResult(id, [
|
|
1924
|
+
{
|
|
1925
|
+
mediaType: "application/fhir+json",
|
|
1926
|
+
data: params.data
|
|
1927
|
+
}
|
|
1928
|
+
]);
|
|
1929
|
+
} else if (typeof params.data === "string") {
|
|
1930
|
+
this.messageStore.setMessagePartialResult(id, params.data);
|
|
1931
|
+
}
|
|
1932
|
+
this.emitMessages();
|
|
1933
|
+
}
|
|
1934
|
+
handleVoiceTranscription(params) {
|
|
1935
|
+
const id = params.id;
|
|
1936
|
+
const text = params.text;
|
|
1937
|
+
const message = this.messageStore.getMessageById(id);
|
|
1938
|
+
if (!message)
|
|
1939
|
+
return;
|
|
1940
|
+
message.request = text;
|
|
1941
|
+
this.emitMessages();
|
|
1942
|
+
this.bus.emit("voice:transcription", {
|
|
1943
|
+
transcriptionId: id,
|
|
1944
|
+
finalText: text
|
|
1945
|
+
});
|
|
1946
|
+
}
|
|
1947
|
+
resetRecordingState() {
|
|
1948
|
+
if (this.bufferTimer) {
|
|
1949
|
+
clearInterval(this.bufferTimer);
|
|
1950
|
+
this.bufferTimer = null;
|
|
1951
|
+
}
|
|
1952
|
+
this.snapshot.recording = {
|
|
1953
|
+
mode: "idle",
|
|
1954
|
+
transcriptionId: null,
|
|
1955
|
+
isRecording: false,
|
|
1956
|
+
isPaused: false,
|
|
1957
|
+
bufferedChunks: 0,
|
|
1958
|
+
wasTranscriptionActive: false,
|
|
1959
|
+
transcriptionPausedState: false
|
|
1960
|
+
};
|
|
1961
|
+
this.emitRecording();
|
|
1962
|
+
}
|
|
1963
|
+
};
|
|
1964
|
+
|
|
1965
|
+
// src/headless/create-assistant.ts
|
|
1966
|
+
function createAssistant(options) {
|
|
1967
|
+
const client = new AssistantClient(options);
|
|
1968
|
+
return {
|
|
1969
|
+
client,
|
|
1970
|
+
getSnapshot: () => client.getSnapshot(),
|
|
1971
|
+
subscribe: (event, listener) => client.on(event, listener)
|
|
1972
|
+
};
|
|
1973
|
+
}
|
|
1974
|
+
export {
|
|
1975
|
+
AssistantClient,
|
|
1976
|
+
createAssistant
|
|
1977
|
+
};
|