@charivo/stt 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -6
- package/dist/gemini/index.d.mts +4 -2
- package/dist/gemini/index.d.ts +4 -2
- package/dist/gemini-live/index.d.mts +19 -0
- package/dist/gemini-live/index.d.ts +19 -0
- package/dist/gemini-live/index.js +807 -0
- package/dist/gemini-live/index.mjs +786 -0
- package/dist/remote/index.js +14 -6
- package/dist/remote/index.mjs +16 -7
- package/package.json +8 -3
|
@@ -0,0 +1,786 @@
|
|
|
1
|
+
// src/gemini-live/index.ts
|
|
2
|
+
import {
|
|
3
|
+
CharivoProviderError,
|
|
4
|
+
CharivoStateError,
|
|
5
|
+
CharivoTimeoutError
|
|
6
|
+
} from "@charivo/core";
|
|
7
|
+
|
|
8
|
+
// src/gemini-live/capture.ts
|
|
9
|
+
var INPUT_SAMPLE_RATE = 16e3;
|
|
10
|
+
var CAPTURE_FRAME_SAMPLES = 1600;
|
|
11
|
+
var CAPTURE_PROCESSOR_NAME = "charivo-gemini-live-stt-capture";
|
|
12
|
+
var FLUSH_TIMEOUT_MS = 250;
|
|
13
|
+
var CAPTURE_WORKLET_SOURCE = `
|
|
14
|
+
class CaptureProcessor extends AudioWorkletProcessor {
|
|
15
|
+
constructor(options) {
|
|
16
|
+
super();
|
|
17
|
+
this.ratio = sampleRate / options.processorOptions.targetRate;
|
|
18
|
+
this.frameSamples = options.processorOptions.frameSamples;
|
|
19
|
+
this.position = 0;
|
|
20
|
+
this.sum = 0;
|
|
21
|
+
this.count = 0;
|
|
22
|
+
this.pending = [];
|
|
23
|
+
// The samples captured after the last full frame are the end of what the
|
|
24
|
+
// user said. Nothing else ever posts them: without this answer they sit in
|
|
25
|
+
// \`pending\` until the context closes and the transcript loses its tail.
|
|
26
|
+
this.port.onmessage = (event) => {
|
|
27
|
+
if (!event.data || event.data.type !== "flush") {
|
|
28
|
+
return;
|
|
29
|
+
}
|
|
30
|
+
if (this.pending.length > 0) {
|
|
31
|
+
this.postFrame(this.pending.splice(0, this.pending.length));
|
|
32
|
+
}
|
|
33
|
+
this.port.postMessage({ type: "flushed" });
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
postFrame(chunk) {
|
|
38
|
+
const frame = new Int16Array(chunk.length);
|
|
39
|
+
for (let i = 0; i < chunk.length; i += 1) {
|
|
40
|
+
const clamped = Math.max(-1, Math.min(1, chunk[i]));
|
|
41
|
+
frame[i] = clamped < 0 ? clamped * 0x8000 : clamped * 0x7fff;
|
|
42
|
+
}
|
|
43
|
+
this.port.postMessage(frame.buffer, [frame.buffer]);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
process(inputs) {
|
|
47
|
+
const channel = inputs[0] && inputs[0][0];
|
|
48
|
+
if (!channel) {
|
|
49
|
+
return true;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
// Box-average decimation. Not a proper anti-alias filter, but far better
|
|
53
|
+
// than picking every Nth sample, which folds speech harmonics back down.
|
|
54
|
+
// The position accumulator carries the fractional remainder across groups,
|
|
55
|
+
// so a device rate that is not a whole multiple of the target \u2014 44.1 kHz,
|
|
56
|
+
// say \u2014 alternates group sizes rather than drifting out of step.
|
|
57
|
+
for (let i = 0; i < channel.length; i += 1) {
|
|
58
|
+
this.sum += channel[i];
|
|
59
|
+
this.count += 1;
|
|
60
|
+
this.position += 1;
|
|
61
|
+
if (this.position >= this.ratio) {
|
|
62
|
+
this.position -= this.ratio;
|
|
63
|
+
this.pending.push(this.count > 0 ? this.sum / this.count : 0);
|
|
64
|
+
this.sum = 0;
|
|
65
|
+
this.count = 0;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
while (this.pending.length >= this.frameSamples) {
|
|
70
|
+
this.postFrame(this.pending.splice(0, this.frameSamples));
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
return true;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
registerProcessor("${CAPTURE_PROCESSOR_NAME}", CaptureProcessor);
|
|
78
|
+
`;
|
|
79
|
+
async function createCapturePipeline({
|
|
80
|
+
stream,
|
|
81
|
+
onFrame,
|
|
82
|
+
onError
|
|
83
|
+
}) {
|
|
84
|
+
const context = new AudioContext();
|
|
85
|
+
const workletUrl = URL.createObjectURL(
|
|
86
|
+
new Blob([CAPTURE_WORKLET_SOURCE], { type: "application/javascript" })
|
|
87
|
+
);
|
|
88
|
+
try {
|
|
89
|
+
await context.audioWorklet.addModule(workletUrl);
|
|
90
|
+
const source = context.createMediaStreamSource(stream);
|
|
91
|
+
const capture = new AudioWorkletNode(context, CAPTURE_PROCESSOR_NAME, {
|
|
92
|
+
processorOptions: {
|
|
93
|
+
targetRate: INPUT_SAMPLE_RATE,
|
|
94
|
+
frameSamples: CAPTURE_FRAME_SAMPLES
|
|
95
|
+
}
|
|
96
|
+
});
|
|
97
|
+
source.connect(capture);
|
|
98
|
+
let pendingFlush = null;
|
|
99
|
+
const settleFlush = (drained) => {
|
|
100
|
+
const flushing = pendingFlush;
|
|
101
|
+
if (!flushing) {
|
|
102
|
+
return;
|
|
103
|
+
}
|
|
104
|
+
pendingFlush = null;
|
|
105
|
+
clearTimeout(flushing.timer);
|
|
106
|
+
flushing.resolve({ drained });
|
|
107
|
+
};
|
|
108
|
+
capture.port.onmessage = (event) => {
|
|
109
|
+
if (event.data instanceof ArrayBuffer) {
|
|
110
|
+
onFrame(new Uint8Array(event.data));
|
|
111
|
+
return;
|
|
112
|
+
}
|
|
113
|
+
if (event.data?.type === "flushed") {
|
|
114
|
+
settleFlush(true);
|
|
115
|
+
}
|
|
116
|
+
};
|
|
117
|
+
capture.onprocessorerror = () => {
|
|
118
|
+
onError();
|
|
119
|
+
};
|
|
120
|
+
return {
|
|
121
|
+
flush() {
|
|
122
|
+
return new Promise((resolve) => {
|
|
123
|
+
pendingFlush = {
|
|
124
|
+
resolve,
|
|
125
|
+
timer: setTimeout(() => settleFlush(false), FLUSH_TIMEOUT_MS)
|
|
126
|
+
};
|
|
127
|
+
capture.port.postMessage({ type: "flush" });
|
|
128
|
+
});
|
|
129
|
+
},
|
|
130
|
+
stop() {
|
|
131
|
+
capture.port.onmessage = null;
|
|
132
|
+
capture.onprocessorerror = null;
|
|
133
|
+
capture.port.close();
|
|
134
|
+
source.disconnect();
|
|
135
|
+
capture.disconnect();
|
|
136
|
+
void closeContext(context);
|
|
137
|
+
}
|
|
138
|
+
};
|
|
139
|
+
} catch (error) {
|
|
140
|
+
void closeContext(context);
|
|
141
|
+
throw error;
|
|
142
|
+
} finally {
|
|
143
|
+
URL.revokeObjectURL(workletUrl);
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
function closeContext(context) {
|
|
147
|
+
return context.close().catch(
|
|
148
|
+
(error) => console.error("Failed to close the capture audio context:", error)
|
|
149
|
+
);
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
// src/gemini-live/index.ts
|
|
153
|
+
var TRANSCRIPTION_MODEL = "gemini-3.5-transcribe-live";
|
|
154
|
+
var OPEN_TIMEOUT_MS = 1e4;
|
|
155
|
+
var BOOTSTRAP_TIMEOUT_MS = 15e3;
|
|
156
|
+
var FINALIZE_TIMEOUT_MS = 5e3;
|
|
157
|
+
var INPUT_MIME_TYPE = "audio/pcm;rate=16000";
|
|
158
|
+
var GeminiLiveSTTTranscriber = class {
|
|
159
|
+
constructor(config) {
|
|
160
|
+
this.config = config;
|
|
161
|
+
}
|
|
162
|
+
mediaStream = null;
|
|
163
|
+
binding = null;
|
|
164
|
+
capture = null;
|
|
165
|
+
// Session epoch. Every per-session callback captures the value from its
|
|
166
|
+
// startRecording() call and ignores events once cleanup() has bumped it, so
|
|
167
|
+
// listeners still attached to a closing socket cannot mutate a newer session.
|
|
168
|
+
generation = 0;
|
|
169
|
+
connecting = false;
|
|
170
|
+
recording = false;
|
|
171
|
+
stopPending = false;
|
|
172
|
+
isCleaningUp = false;
|
|
173
|
+
terminalError = null;
|
|
174
|
+
setupError = null;
|
|
175
|
+
setupReject = null;
|
|
176
|
+
pendingStop = null;
|
|
177
|
+
openGate = null;
|
|
178
|
+
openTimer;
|
|
179
|
+
stopTimer;
|
|
180
|
+
// `snapshot` is what the server last said, `lastEmitted` what subscribers
|
|
181
|
+
// last saw.
|
|
182
|
+
snapshot = "";
|
|
183
|
+
lastEmitted = "";
|
|
184
|
+
activityEnded = false;
|
|
185
|
+
audioSent = false;
|
|
186
|
+
partialSubscribers = [];
|
|
187
|
+
handlePageHide = () => this.cleanup();
|
|
188
|
+
/**
|
|
189
|
+
* Open the Live API session and start streaming microphone audio
|
|
190
|
+
*/
|
|
191
|
+
async startRecording(options) {
|
|
192
|
+
if (this.recording || this.connecting) {
|
|
193
|
+
throw new CharivoStateError("already recording");
|
|
194
|
+
}
|
|
195
|
+
this.connecting = true;
|
|
196
|
+
this.stopPending = false;
|
|
197
|
+
this.terminalError = null;
|
|
198
|
+
this.setupError = null;
|
|
199
|
+
this.pendingStop = null;
|
|
200
|
+
this.snapshot = "";
|
|
201
|
+
this.lastEmitted = "";
|
|
202
|
+
this.activityEnded = false;
|
|
203
|
+
this.audioSent = false;
|
|
204
|
+
const gen = ++this.generation;
|
|
205
|
+
const setupFailure = new Promise((_, reject) => {
|
|
206
|
+
this.setupReject = reject;
|
|
207
|
+
});
|
|
208
|
+
void setupFailure.catch(() => {
|
|
209
|
+
});
|
|
210
|
+
try {
|
|
211
|
+
const micPromise = this.acquireMic();
|
|
212
|
+
void micPromise.then(
|
|
213
|
+
(stream2) => {
|
|
214
|
+
if (gen !== this.generation) {
|
|
215
|
+
stream2.getTracks().forEach((track) => track.stop());
|
|
216
|
+
}
|
|
217
|
+
},
|
|
218
|
+
() => {
|
|
219
|
+
}
|
|
220
|
+
);
|
|
221
|
+
const stream = await Promise.race([micPromise, setupFailure]);
|
|
222
|
+
if (gen !== this.generation) {
|
|
223
|
+
stream.getTracks().forEach((track) => track.stop());
|
|
224
|
+
throw new CharivoStateError(
|
|
225
|
+
"start canceled because stop was called while connecting"
|
|
226
|
+
);
|
|
227
|
+
}
|
|
228
|
+
this.mediaStream = stream;
|
|
229
|
+
const { url, token } = resolveBootstrap(
|
|
230
|
+
await this.raceBootstrap(
|
|
231
|
+
this.config.bootstrap({
|
|
232
|
+
session: {
|
|
233
|
+
model: TRANSCRIPTION_MODEL,
|
|
234
|
+
language: options?.language
|
|
235
|
+
}
|
|
236
|
+
}),
|
|
237
|
+
setupFailure
|
|
238
|
+
)
|
|
239
|
+
);
|
|
240
|
+
this.openSocket(url, token, gen);
|
|
241
|
+
await Promise.race([this.waitForSetupComplete(gen), setupFailure]);
|
|
242
|
+
if (gen !== this.generation) {
|
|
243
|
+
throw new CharivoStateError(
|
|
244
|
+
"start canceled because stop was called while connecting"
|
|
245
|
+
);
|
|
246
|
+
}
|
|
247
|
+
this.binding.socket.send(
|
|
248
|
+
JSON.stringify({ realtimeInput: { activityStart: {} } })
|
|
249
|
+
);
|
|
250
|
+
const capturePromise = createCapturePipeline({
|
|
251
|
+
stream,
|
|
252
|
+
onFrame: (frame) => this.sendAudioFrame(frame),
|
|
253
|
+
// Routed like every other failure source in this class: the pipeline is
|
|
254
|
+
// built while the session is still connecting, so a failure inside that
|
|
255
|
+
// window has to fail the in-flight start rather than become a terminal
|
|
256
|
+
// error the start then walks over by going live on a stopped pipeline.
|
|
257
|
+
onError: () => this.failOpenOrTerminal(
|
|
258
|
+
new CharivoProviderError("capture worklet stopped processing")
|
|
259
|
+
)
|
|
260
|
+
});
|
|
261
|
+
void capturePromise.then(
|
|
262
|
+
(pipeline) => {
|
|
263
|
+
if (gen !== this.generation) {
|
|
264
|
+
pipeline.stop();
|
|
265
|
+
}
|
|
266
|
+
},
|
|
267
|
+
() => {
|
|
268
|
+
}
|
|
269
|
+
);
|
|
270
|
+
const capture = await Promise.race([capturePromise, setupFailure]);
|
|
271
|
+
if (gen !== this.generation) {
|
|
272
|
+
capture.stop();
|
|
273
|
+
throw new CharivoStateError(
|
|
274
|
+
"start canceled because stop was called while connecting"
|
|
275
|
+
);
|
|
276
|
+
}
|
|
277
|
+
this.capture = capture;
|
|
278
|
+
if (typeof window !== "undefined") {
|
|
279
|
+
window.addEventListener("pagehide", this.handlePageHide);
|
|
280
|
+
}
|
|
281
|
+
this.recording = true;
|
|
282
|
+
} catch (error) {
|
|
283
|
+
const failure = (gen === this.generation ? this.setupError : null) ?? error;
|
|
284
|
+
if (gen === this.generation) {
|
|
285
|
+
this.cleanup();
|
|
286
|
+
this.terminalError = null;
|
|
287
|
+
this.setupError = null;
|
|
288
|
+
}
|
|
289
|
+
throw failure;
|
|
290
|
+
} finally {
|
|
291
|
+
if (gen === this.generation) {
|
|
292
|
+
this.connecting = false;
|
|
293
|
+
this.setupReject = null;
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
/**
|
|
298
|
+
* Close the activity and resolve with the transcript of the whole recording
|
|
299
|
+
*/
|
|
300
|
+
async stopRecording() {
|
|
301
|
+
if (this.terminalError) {
|
|
302
|
+
const error = this.terminalError;
|
|
303
|
+
this.terminalError = null;
|
|
304
|
+
this.cleanup();
|
|
305
|
+
throw error;
|
|
306
|
+
}
|
|
307
|
+
if (this.stopPending) {
|
|
308
|
+
throw new CharivoStateError("stop already in progress");
|
|
309
|
+
}
|
|
310
|
+
if (this.connecting) {
|
|
311
|
+
this.cancelConnecting();
|
|
312
|
+
return "";
|
|
313
|
+
}
|
|
314
|
+
if (!this.recording) {
|
|
315
|
+
return "";
|
|
316
|
+
}
|
|
317
|
+
this.stopPending = true;
|
|
318
|
+
const gen = this.generation;
|
|
319
|
+
const stopped = new Promise((resolve, reject) => {
|
|
320
|
+
this.pendingStop = { resolve, reject };
|
|
321
|
+
this.stopTimer = setTimeout(
|
|
322
|
+
() => this.rejectPendingStop(
|
|
323
|
+
new CharivoTimeoutError(
|
|
324
|
+
`stop timed out after ${FINALIZE_TIMEOUT_MS}ms waiting for the final transcript`
|
|
325
|
+
)
|
|
326
|
+
),
|
|
327
|
+
FINALIZE_TIMEOUT_MS
|
|
328
|
+
);
|
|
329
|
+
}).then(
|
|
330
|
+
(text) => {
|
|
331
|
+
this.cleanup();
|
|
332
|
+
return text;
|
|
333
|
+
},
|
|
334
|
+
(error) => {
|
|
335
|
+
this.terminalError = null;
|
|
336
|
+
this.cleanup();
|
|
337
|
+
throw error;
|
|
338
|
+
}
|
|
339
|
+
);
|
|
340
|
+
void this.finalizeStop(gen).catch(() => {
|
|
341
|
+
});
|
|
342
|
+
return stopped;
|
|
343
|
+
}
|
|
344
|
+
/**
|
|
345
|
+
* Check if currently recording, including while a session is still
|
|
346
|
+
* connecting: lifecycle owners like Charivo.dispose() gate their stop call
|
|
347
|
+
* on this, so a connecting session must count as active.
|
|
348
|
+
*/
|
|
349
|
+
isRecording() {
|
|
350
|
+
return this.recording || this.connecting;
|
|
351
|
+
}
|
|
352
|
+
/**
|
|
353
|
+
* Subscribe to cumulative interim transcript snapshots
|
|
354
|
+
*/
|
|
355
|
+
onPartial(callback) {
|
|
356
|
+
this.partialSubscribers.push(callback);
|
|
357
|
+
}
|
|
358
|
+
/**
|
|
359
|
+
* Hand one capture frame to the server, or drop it. The capture pipeline is
|
|
360
|
+
* stopped before the socket is torn down, so a frame can only arrive here
|
|
361
|
+
* while the socket is still open.
|
|
362
|
+
*/
|
|
363
|
+
sendAudioFrame(frame) {
|
|
364
|
+
const socket = this.binding?.socket;
|
|
365
|
+
if (!socket || socket.readyState !== WebSocket.OPEN) {
|
|
366
|
+
return;
|
|
367
|
+
}
|
|
368
|
+
socket.send(
|
|
369
|
+
JSON.stringify({
|
|
370
|
+
realtimeInput: {
|
|
371
|
+
audio: { data: bytesToBase64(frame), mimeType: INPUT_MIME_TYPE }
|
|
372
|
+
}
|
|
373
|
+
})
|
|
374
|
+
);
|
|
375
|
+
this.audioSent = true;
|
|
376
|
+
}
|
|
377
|
+
async acquireMic() {
|
|
378
|
+
return navigator.mediaDevices.getUserMedia({
|
|
379
|
+
audio: {
|
|
380
|
+
echoCancellation: true,
|
|
381
|
+
noiseSuppression: true,
|
|
382
|
+
autoGainControl: true
|
|
383
|
+
}
|
|
384
|
+
});
|
|
385
|
+
}
|
|
386
|
+
async raceBootstrap(bootstrapPromise, setupFailure) {
|
|
387
|
+
let timer;
|
|
388
|
+
const timeout = new Promise((_, reject) => {
|
|
389
|
+
timer = setTimeout(
|
|
390
|
+
() => reject(
|
|
391
|
+
new CharivoTimeoutError(
|
|
392
|
+
`streaming STT bootstrap timed out after ${BOOTSTRAP_TIMEOUT_MS}ms`
|
|
393
|
+
)
|
|
394
|
+
),
|
|
395
|
+
BOOTSTRAP_TIMEOUT_MS
|
|
396
|
+
);
|
|
397
|
+
});
|
|
398
|
+
try {
|
|
399
|
+
return await Promise.race([bootstrapPromise, setupFailure, timeout]);
|
|
400
|
+
} finally {
|
|
401
|
+
clearTimeout(timer);
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
openSocket(url, token, gen) {
|
|
405
|
+
let endpoint;
|
|
406
|
+
try {
|
|
407
|
+
endpoint = new URL(url);
|
|
408
|
+
} catch {
|
|
409
|
+
throw new CharivoProviderError(
|
|
410
|
+
"bootstrap url is not a valid websocket URL"
|
|
411
|
+
);
|
|
412
|
+
}
|
|
413
|
+
if (endpoint.protocol !== "ws:" && endpoint.protocol !== "wss:") {
|
|
414
|
+
throw new CharivoProviderError("bootstrap url must use ws: or wss:");
|
|
415
|
+
}
|
|
416
|
+
let socket;
|
|
417
|
+
try {
|
|
418
|
+
endpoint.searchParams.set("access_token", token);
|
|
419
|
+
socket = new WebSocket(endpoint.toString());
|
|
420
|
+
} catch {
|
|
421
|
+
throw new CharivoProviderError(
|
|
422
|
+
"failed to open the Gemini Live websocket"
|
|
423
|
+
);
|
|
424
|
+
}
|
|
425
|
+
if (gen !== this.generation) {
|
|
426
|
+
socket.close();
|
|
427
|
+
throw new CharivoStateError(
|
|
428
|
+
"start canceled because stop was called while connecting"
|
|
429
|
+
);
|
|
430
|
+
}
|
|
431
|
+
const binding = { socket, gen, pump: Promise.resolve() };
|
|
432
|
+
this.binding = binding;
|
|
433
|
+
socket.onopen = () => {
|
|
434
|
+
if (gen !== this.generation) {
|
|
435
|
+
return;
|
|
436
|
+
}
|
|
437
|
+
socket.send(
|
|
438
|
+
JSON.stringify({
|
|
439
|
+
setup: {
|
|
440
|
+
model: `models/${TRANSCRIPTION_MODEL}`,
|
|
441
|
+
generationConfig: { responseModalities: ["TEXT"] },
|
|
442
|
+
inputAudioTranscription: { mode: "VERBATIM" },
|
|
443
|
+
realtimeInputConfig: {
|
|
444
|
+
automaticActivityDetection: { disabled: true }
|
|
445
|
+
}
|
|
446
|
+
}
|
|
447
|
+
})
|
|
448
|
+
);
|
|
449
|
+
};
|
|
450
|
+
socket.onmessage = (event) => {
|
|
451
|
+
this.enqueueServerMessage(binding, event.data);
|
|
452
|
+
};
|
|
453
|
+
socket.onerror = () => {
|
|
454
|
+
if (gen !== this.generation || this.isCleaningUp) {
|
|
455
|
+
return;
|
|
456
|
+
}
|
|
457
|
+
this.failOpenOrTerminal(
|
|
458
|
+
new CharivoProviderError("Gemini Live websocket error")
|
|
459
|
+
);
|
|
460
|
+
};
|
|
461
|
+
socket.onclose = (event) => {
|
|
462
|
+
if (gen !== this.generation || this.isCleaningUp) {
|
|
463
|
+
return;
|
|
464
|
+
}
|
|
465
|
+
this.failOpenOrTerminal(
|
|
466
|
+
new CharivoProviderError(
|
|
467
|
+
`Gemini Live websocket closed (code ${event.code})`
|
|
468
|
+
)
|
|
469
|
+
);
|
|
470
|
+
};
|
|
471
|
+
}
|
|
472
|
+
enqueueServerMessage(binding, data) {
|
|
473
|
+
if (binding.gen !== this.generation) {
|
|
474
|
+
return;
|
|
475
|
+
}
|
|
476
|
+
const arrivedAfterActivityEnd = this.activityEnded;
|
|
477
|
+
binding.pump = binding.pump.then(async () => {
|
|
478
|
+
let payload;
|
|
479
|
+
try {
|
|
480
|
+
const raw = data instanceof Blob ? await data.text() : data;
|
|
481
|
+
if (binding.gen !== this.generation) {
|
|
482
|
+
return;
|
|
483
|
+
}
|
|
484
|
+
if (typeof raw !== "string") {
|
|
485
|
+
throw new TypeError("unreadable frame");
|
|
486
|
+
}
|
|
487
|
+
payload = JSON.parse(raw);
|
|
488
|
+
} catch {
|
|
489
|
+
this.failOpenOrTerminal(
|
|
490
|
+
new CharivoProviderError("malformed Gemini Live message")
|
|
491
|
+
);
|
|
492
|
+
return;
|
|
493
|
+
}
|
|
494
|
+
if (typeof payload !== "object" || payload === null) {
|
|
495
|
+
this.failOpenOrTerminal(
|
|
496
|
+
new CharivoProviderError("malformed Gemini Live message")
|
|
497
|
+
);
|
|
498
|
+
return;
|
|
499
|
+
}
|
|
500
|
+
this.handleServerMessage(
|
|
501
|
+
payload,
|
|
502
|
+
arrivedAfterActivityEnd
|
|
503
|
+
);
|
|
504
|
+
});
|
|
505
|
+
}
|
|
506
|
+
handleServerMessage(message, arrivedAfterActivityEnd) {
|
|
507
|
+
if (message.setupComplete) {
|
|
508
|
+
this.resolveOpenGate();
|
|
509
|
+
return;
|
|
510
|
+
}
|
|
511
|
+
const content = message.serverContent;
|
|
512
|
+
if (!content) {
|
|
513
|
+
return;
|
|
514
|
+
}
|
|
515
|
+
const interim = content.interimInputTranscription;
|
|
516
|
+
if (interim) {
|
|
517
|
+
const text = decodeTranscriptText(interim.text);
|
|
518
|
+
if (text === null) {
|
|
519
|
+
this.failOpenOrTerminal(
|
|
520
|
+
new CharivoProviderError("malformed Gemini Live message")
|
|
521
|
+
);
|
|
522
|
+
return;
|
|
523
|
+
}
|
|
524
|
+
this.snapshot = text;
|
|
525
|
+
this.emitPartial();
|
|
526
|
+
}
|
|
527
|
+
const final = content.inputTranscription;
|
|
528
|
+
if (final) {
|
|
529
|
+
const text = decodeTranscriptText(final.text);
|
|
530
|
+
if (text === null) {
|
|
531
|
+
this.failOpenOrTerminal(
|
|
532
|
+
new CharivoProviderError("malformed Gemini Live message")
|
|
533
|
+
);
|
|
534
|
+
return;
|
|
535
|
+
}
|
|
536
|
+
this.snapshot = text;
|
|
537
|
+
this.emitPartial();
|
|
538
|
+
this.maybeResolveStop(arrivedAfterActivityEnd);
|
|
539
|
+
}
|
|
540
|
+
}
|
|
541
|
+
/**
|
|
542
|
+
* Hand the snapshot to subscribers, but only when it changed. The server
|
|
543
|
+
* repeats the interim unchanged while the speaker is silent (8 identical
|
|
544
|
+
* frames across a measured 1.5 s gap), and a subscriber redrawing a draft
|
|
545
|
+
* has nothing to do with those. What is recorded is what was EMITTED, so a
|
|
546
|
+
* final that repeats the text its last interim already showed emits nothing.
|
|
547
|
+
*
|
|
548
|
+
* A shorter snapshot is emitted as given: nothing here enforces
|
|
549
|
+
* monotonicity, because suppressing a revision would leave subscribers
|
|
550
|
+
* holding text the transcript stop returns no longer contains.
|
|
551
|
+
*/
|
|
552
|
+
emitPartial() {
|
|
553
|
+
if (this.snapshot === this.lastEmitted) {
|
|
554
|
+
return;
|
|
555
|
+
}
|
|
556
|
+
this.lastEmitted = this.snapshot;
|
|
557
|
+
for (const callback of this.partialSubscribers) {
|
|
558
|
+
callback(this.snapshot);
|
|
559
|
+
}
|
|
560
|
+
}
|
|
561
|
+
/**
|
|
562
|
+
* Settle a stop that is already in flight with the final just applied, but
|
|
563
|
+
* only if that final is eligible: eligibility is having ARRIVED after
|
|
564
|
+
* `activityEnd` went out, because an earlier one cannot be known to cover the
|
|
565
|
+
* audio sent after it. That is the only rule, and the flag is the only place
|
|
566
|
+
* it is read — `activityEnded` says where the session is now, which for a
|
|
567
|
+
* frame the pump held is a different question.
|
|
568
|
+
*
|
|
569
|
+
* The snapshot is trimmed here and nowhere else. Partials stay exactly as the
|
|
570
|
+
* server sent them, and the transcript a caller keeps is the trimmed one —
|
|
571
|
+
* the same split `buildFinal()` makes in `@charivo/stt/openai-realtime`. So
|
|
572
|
+
* the last partial equals what stop resolves MODULO that trim, and nothing
|
|
573
|
+
* else: a final that revises the tail is emitted before it is trimmed, so no
|
|
574
|
+
* subscriber is left holding text the transcript contradicts.
|
|
575
|
+
*/
|
|
576
|
+
maybeResolveStop(arrivedAfterActivityEnd) {
|
|
577
|
+
if (!this.stopPending || !this.pendingStop || !arrivedAfterActivityEnd) {
|
|
578
|
+
return;
|
|
579
|
+
}
|
|
580
|
+
this.resolvePendingStop(this.snapshot.trim());
|
|
581
|
+
}
|
|
582
|
+
waitForSetupComplete(gen) {
|
|
583
|
+
return new Promise((resolve, reject) => {
|
|
584
|
+
this.openGate = { resolve, reject, settled: false };
|
|
585
|
+
this.openTimer = setTimeout(() => {
|
|
586
|
+
if (gen !== this.generation) {
|
|
587
|
+
return;
|
|
588
|
+
}
|
|
589
|
+
this.rejectOpenGate(
|
|
590
|
+
new CharivoTimeoutError(
|
|
591
|
+
`streaming STT session setup did not complete within ${OPEN_TIMEOUT_MS}ms`
|
|
592
|
+
)
|
|
593
|
+
);
|
|
594
|
+
}, OPEN_TIMEOUT_MS);
|
|
595
|
+
});
|
|
596
|
+
}
|
|
597
|
+
resolveOpenGate() {
|
|
598
|
+
clearTimeout(this.openTimer);
|
|
599
|
+
this.openTimer = void 0;
|
|
600
|
+
const gate = this.openGate;
|
|
601
|
+
this.openGate = null;
|
|
602
|
+
if (!gate || gate.settled) {
|
|
603
|
+
return;
|
|
604
|
+
}
|
|
605
|
+
gate.settled = true;
|
|
606
|
+
gate.resolve();
|
|
607
|
+
}
|
|
608
|
+
rejectOpenGate(error) {
|
|
609
|
+
clearTimeout(this.openTimer);
|
|
610
|
+
this.openTimer = void 0;
|
|
611
|
+
const gate = this.openGate;
|
|
612
|
+
this.openGate = null;
|
|
613
|
+
if (!gate || gate.settled) {
|
|
614
|
+
return;
|
|
615
|
+
}
|
|
616
|
+
gate.settled = true;
|
|
617
|
+
gate.reject(error);
|
|
618
|
+
}
|
|
619
|
+
failOpenOrTerminal(error) {
|
|
620
|
+
if (this.openGate && !this.openGate.settled) {
|
|
621
|
+
this.rejectOpenGate(error);
|
|
622
|
+
} else if (this.connecting) {
|
|
623
|
+
this.setupError ??= error;
|
|
624
|
+
this.setupReject?.(error);
|
|
625
|
+
} else {
|
|
626
|
+
this.failTerminal(error);
|
|
627
|
+
}
|
|
628
|
+
}
|
|
629
|
+
failTerminal(error) {
|
|
630
|
+
if (this.terminalError) {
|
|
631
|
+
return;
|
|
632
|
+
}
|
|
633
|
+
this.terminalError = error;
|
|
634
|
+
this.cleanup();
|
|
635
|
+
}
|
|
636
|
+
// Cancels the in-flight startRecording(). Runs cleanup() synchronously so a
|
|
637
|
+
// caller awaiting stopRecording() sees every resource acquired so far (mic,
|
|
638
|
+
// socket, capture, timers, pagehide listener) already released by the time
|
|
639
|
+
// it returns. Every step in startRecording()'s try block re-checks the
|
|
640
|
+
// generation before touching `this` again, so the setup step it is
|
|
641
|
+
// currently awaiting rejects with this cancel message instead of resuming
|
|
642
|
+
// and acquiring or writing one more thing. The open gate is rejected with
|
|
643
|
+
// the cancel reason FIRST, before cleanup() reaches for its own generic
|
|
644
|
+
// "recording ended": cleanup()'s `if (this.openGate && !this.openGate.settled)`
|
|
645
|
+
// is then a no-op, since rejectOpenGate() already settled and cleared it.
|
|
646
|
+
// setupReject is captured before cleanup() runs because cleanup() nulls it
|
|
647
|
+
// as part of tearing this generation down.
|
|
648
|
+
cancelConnecting() {
|
|
649
|
+
const error = new CharivoStateError(
|
|
650
|
+
"start canceled because stop was called while connecting"
|
|
651
|
+
);
|
|
652
|
+
this.setupError ??= error;
|
|
653
|
+
const reject = this.setupReject;
|
|
654
|
+
this.rejectOpenGate(error);
|
|
655
|
+
this.cleanup();
|
|
656
|
+
reject?.(error);
|
|
657
|
+
}
|
|
658
|
+
/**
|
|
659
|
+
* Drain the capture worklet, then close the activity the recording belongs
|
|
660
|
+
* to. Settles the pending stop itself on every path that cannot get as far
|
|
661
|
+
* as `activityEnd`.
|
|
662
|
+
*/
|
|
663
|
+
async finalizeStop(gen) {
|
|
664
|
+
const capture = this.capture;
|
|
665
|
+
const { drained } = await capture.flush();
|
|
666
|
+
if (gen !== this.generation || !this.pendingStop) {
|
|
667
|
+
return;
|
|
668
|
+
}
|
|
669
|
+
this.capture = null;
|
|
670
|
+
capture.stop();
|
|
671
|
+
if (!drained) {
|
|
672
|
+
this.rejectPendingStop(
|
|
673
|
+
new CharivoProviderError("capture worklet did not drain before stop")
|
|
674
|
+
);
|
|
675
|
+
return;
|
|
676
|
+
}
|
|
677
|
+
if (!this.audioSent) {
|
|
678
|
+
this.resolvePendingStop("");
|
|
679
|
+
return;
|
|
680
|
+
}
|
|
681
|
+
try {
|
|
682
|
+
this.binding.socket.send(
|
|
683
|
+
JSON.stringify({ realtimeInput: { activityEnd: {} } })
|
|
684
|
+
);
|
|
685
|
+
} catch (error) {
|
|
686
|
+
this.failTerminal(
|
|
687
|
+
error instanceof Error ? error : new CharivoProviderError("failed to end the activity")
|
|
688
|
+
);
|
|
689
|
+
return;
|
|
690
|
+
}
|
|
691
|
+
this.activityEnded = true;
|
|
692
|
+
}
|
|
693
|
+
resolvePendingStop(transcription) {
|
|
694
|
+
const pending = this.pendingStop;
|
|
695
|
+
if (!pending) {
|
|
696
|
+
return;
|
|
697
|
+
}
|
|
698
|
+
clearTimeout(this.stopTimer);
|
|
699
|
+
this.stopTimer = void 0;
|
|
700
|
+
this.pendingStop = null;
|
|
701
|
+
pending.resolve(transcription);
|
|
702
|
+
}
|
|
703
|
+
rejectPendingStop(error) {
|
|
704
|
+
const pending = this.pendingStop;
|
|
705
|
+
if (!pending) {
|
|
706
|
+
return;
|
|
707
|
+
}
|
|
708
|
+
clearTimeout(this.stopTimer);
|
|
709
|
+
this.stopTimer = void 0;
|
|
710
|
+
this.pendingStop = null;
|
|
711
|
+
pending.reject(error);
|
|
712
|
+
}
|
|
713
|
+
cleanup() {
|
|
714
|
+
if (this.isCleaningUp) {
|
|
715
|
+
return;
|
|
716
|
+
}
|
|
717
|
+
this.isCleaningUp = true;
|
|
718
|
+
try {
|
|
719
|
+
clearTimeout(this.stopTimer);
|
|
720
|
+
this.stopTimer = void 0;
|
|
721
|
+
clearTimeout(this.openTimer);
|
|
722
|
+
this.openTimer = void 0;
|
|
723
|
+
if (this.openGate && !this.openGate.settled) {
|
|
724
|
+
const gate = this.openGate;
|
|
725
|
+
gate.settled = true;
|
|
726
|
+
gate.reject(new CharivoStateError("recording ended"));
|
|
727
|
+
}
|
|
728
|
+
this.openGate = null;
|
|
729
|
+
if (this.pendingStop) {
|
|
730
|
+
this.rejectPendingStop(
|
|
731
|
+
this.terminalError ?? new CharivoStateError("recording ended")
|
|
732
|
+
);
|
|
733
|
+
}
|
|
734
|
+
const capture = this.capture;
|
|
735
|
+
this.capture = null;
|
|
736
|
+
capture?.stop();
|
|
737
|
+
this.mediaStream?.getTracks().forEach((track) => track.stop());
|
|
738
|
+
this.binding?.socket.close();
|
|
739
|
+
this.binding = null;
|
|
740
|
+
if (typeof window !== "undefined") {
|
|
741
|
+
window.removeEventListener("pagehide", this.handlePageHide);
|
|
742
|
+
}
|
|
743
|
+
this.mediaStream = null;
|
|
744
|
+
this.recording = false;
|
|
745
|
+
this.connecting = false;
|
|
746
|
+
this.setupReject = null;
|
|
747
|
+
this.stopPending = false;
|
|
748
|
+
this.activityEnded = false;
|
|
749
|
+
this.audioSent = false;
|
|
750
|
+
this.snapshot = "";
|
|
751
|
+
this.lastEmitted = "";
|
|
752
|
+
this.generation++;
|
|
753
|
+
} finally {
|
|
754
|
+
this.isCleaningUp = false;
|
|
755
|
+
}
|
|
756
|
+
}
|
|
757
|
+
};
|
|
758
|
+
function bytesToBase64(bytes) {
|
|
759
|
+
let binary = "";
|
|
760
|
+
const stride = 32768;
|
|
761
|
+
for (let offset = 0; offset < bytes.length; offset += stride) {
|
|
762
|
+
binary += String.fromCharCode(...bytes.subarray(offset, offset + stride));
|
|
763
|
+
}
|
|
764
|
+
return btoa(binary);
|
|
765
|
+
}
|
|
766
|
+
function decodeTranscriptText(text) {
|
|
767
|
+
if (text === void 0) {
|
|
768
|
+
return "";
|
|
769
|
+
}
|
|
770
|
+
return typeof text === "string" ? text : null;
|
|
771
|
+
}
|
|
772
|
+
function resolveBootstrap(bootstrap) {
|
|
773
|
+
const { url, token } = bootstrap ?? {};
|
|
774
|
+
if (typeof url !== "string" || !url || typeof token !== "string" || !token) {
|
|
775
|
+
throw new CharivoProviderError(
|
|
776
|
+
"streaming STT bootstrap is missing its websocket url or token"
|
|
777
|
+
);
|
|
778
|
+
}
|
|
779
|
+
return { url, token };
|
|
780
|
+
}
|
|
781
|
+
function createGeminiLiveSTTTranscriber(config) {
|
|
782
|
+
return new GeminiLiveSTTTranscriber(config);
|
|
783
|
+
}
|
|
784
|
+
export {
|
|
785
|
+
createGeminiLiveSTTTranscriber
|
|
786
|
+
};
|