@micdrop/openai 1.3.2 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +112 -3
- package/dist/index.d.mts +63 -4
- package/dist/index.d.ts +63 -4
- package/dist/index.js +383 -88
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +384 -87
- package/dist/index.mjs.map +1 -1
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -31,6 +31,7 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
|
|
|
31
31
|
var index_exports = {};
|
|
32
32
|
__export(index_exports, {
|
|
33
33
|
OpenaiAgent: () => OpenaiAgent,
|
|
34
|
+
OpenaiRealtime: () => OpenaiRealtime,
|
|
34
35
|
OpenaiSTT: () => OpenaiSTT,
|
|
35
36
|
OpenaiTTS: () => OpenaiTTS
|
|
36
37
|
});
|
|
@@ -183,69 +184,379 @@ var OpenaiAgent = class extends import_server.Agent {
|
|
|
183
184
|
}
|
|
184
185
|
};
|
|
185
186
|
|
|
186
|
-
// src/
|
|
187
|
+
// src/OpenaiRealtime.ts
|
|
187
188
|
var import_server2 = require("@micdrop/server");
|
|
188
189
|
var import_ws = __toESM(require("ws"));
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
var
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
this.
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
190
|
+
var import_zod2 = __toESM(require("zod"));
|
|
191
|
+
var DEFAULT_MODEL2 = "gpt-realtime-2.1";
|
|
192
|
+
var DEFAULT_VOICE = "marin";
|
|
193
|
+
var DEFAULT_TRANSCRIPTION_MODEL = "gpt-4o-transcribe";
|
|
194
|
+
var SAMPLE_RATE = 16e3;
|
|
195
|
+
var OPENAI_SAMPLE_RATE = 24e3;
|
|
196
|
+
var OPENAI_BYTES_PER_MS = OPENAI_SAMPLE_RATE * 2 / 1e3;
|
|
197
|
+
var DEFAULT_CONNECTION_TIMEOUT = 5e3;
|
|
198
|
+
var DEFAULT_RETRY_DELAY2 = 1e3;
|
|
199
|
+
var DEFAULT_MAX_RETRY = 3;
|
|
200
|
+
var OpenaiRealtime = class extends import_server2.Realtime {
|
|
201
|
+
constructor(options) {
|
|
202
|
+
super(options);
|
|
203
|
+
this.connected = false;
|
|
204
|
+
this.everConnected = false;
|
|
205
|
+
this.destroyed = false;
|
|
206
|
+
// Events sent while the connection opens
|
|
207
|
+
this.pending = [];
|
|
208
|
+
this.retryCount = 0;
|
|
209
|
+
this.inputResampler = new import_server2.Pcm16Resampler(SAMPLE_RATE, OPENAI_SAMPLE_RATE);
|
|
210
|
+
this.outputResampler = new import_server2.Pcm16Resampler(OPENAI_SAMPLE_RATE, SAMPLE_RATE);
|
|
211
|
+
// Bumped by every interruption, so late responses can tell they are stale
|
|
212
|
+
this.generation = 0;
|
|
213
|
+
this.requested = [];
|
|
214
|
+
this.ignoredResponses = /* @__PURE__ */ new Set();
|
|
215
|
+
this.audioStartedAt = 0;
|
|
216
|
+
this.audioBytes = 0;
|
|
217
|
+
this.connect();
|
|
218
|
+
}
|
|
219
|
+
destroy() {
|
|
220
|
+
this.destroyed = true;
|
|
221
|
+
super.destroy();
|
|
222
|
+
if (this.reconnectTimeout) {
|
|
223
|
+
clearTimeout(this.reconnectTimeout);
|
|
224
|
+
this.reconnectTimeout = void 0;
|
|
225
|
+
}
|
|
226
|
+
this.closeSocket();
|
|
227
|
+
}
|
|
228
|
+
openTurn() {
|
|
229
|
+
this.inputResampler.reset();
|
|
230
|
+
}
|
|
231
|
+
appendAudio(chunk) {
|
|
232
|
+
const audio = this.inputResampler.process(chunk);
|
|
233
|
+
if (audio.length === 0) return;
|
|
234
|
+
this.send({
|
|
235
|
+
type: "input_audio_buffer.append",
|
|
236
|
+
audio: audio.toString("base64")
|
|
237
|
+
});
|
|
238
|
+
}
|
|
239
|
+
closeTurn() {
|
|
240
|
+
this.send({ type: "input_audio_buffer.commit" });
|
|
241
|
+
this.createResponse();
|
|
242
|
+
}
|
|
243
|
+
clearTurn() {
|
|
244
|
+
this.send({ type: "input_audio_buffer.clear" });
|
|
245
|
+
}
|
|
246
|
+
generate() {
|
|
247
|
+
this.createResponse();
|
|
248
|
+
}
|
|
249
|
+
speakText(text) {
|
|
250
|
+
this.createResponse({
|
|
251
|
+
conversation: "none",
|
|
252
|
+
instructions: `Say exactly the following text, word for word, and nothing else: ${text}`
|
|
253
|
+
});
|
|
254
|
+
}
|
|
255
|
+
cancelAnswer() {
|
|
256
|
+
this.generation++;
|
|
257
|
+
this.outputResampler.reset();
|
|
258
|
+
const response = this.response;
|
|
259
|
+
if (!response) return;
|
|
260
|
+
this.response = void 0;
|
|
261
|
+
this.ignoredResponses.add(response.id);
|
|
262
|
+
this.send({ type: "response.cancel", response_id: response.id });
|
|
263
|
+
if (!response.outOfBand && this.audioItemId) {
|
|
264
|
+
const played = Math.min(
|
|
265
|
+
Date.now() - this.audioStartedAt,
|
|
266
|
+
this.audioBytes / OPENAI_BYTES_PER_MS
|
|
267
|
+
);
|
|
268
|
+
this.send({
|
|
269
|
+
type: "conversation.item.truncate",
|
|
270
|
+
item_id: this.audioItemId,
|
|
271
|
+
content_index: 0,
|
|
272
|
+
audio_end_ms: Math.max(0, Math.floor(played))
|
|
273
|
+
});
|
|
274
|
+
}
|
|
275
|
+
this.audioItemId = void 0;
|
|
276
|
+
}
|
|
277
|
+
updateTools() {
|
|
278
|
+
if (this.connected) this.sendSessionUpdate();
|
|
279
|
+
}
|
|
280
|
+
injectMessage(role, text) {
|
|
281
|
+
if (!this.connected && this.everConnected) return;
|
|
282
|
+
this.send(this.messageItem(role, text));
|
|
283
|
+
}
|
|
284
|
+
createResponse(response) {
|
|
285
|
+
this.requested.push({
|
|
286
|
+
generation: this.generation,
|
|
287
|
+
outOfBand: !!response
|
|
288
|
+
});
|
|
289
|
+
this.send({ type: "response.create", ...response ? { response } : {} });
|
|
290
|
+
}
|
|
291
|
+
send(event) {
|
|
292
|
+
if (this.connected && this.socket?.readyState === import_ws.default.OPEN) {
|
|
293
|
+
this.socket.send(JSON.stringify(event));
|
|
294
|
+
} else {
|
|
295
|
+
this.pending.push(event);
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
messageItem(role, text) {
|
|
299
|
+
return {
|
|
300
|
+
type: "conversation.item.create",
|
|
301
|
+
item: {
|
|
302
|
+
type: "message",
|
|
303
|
+
role,
|
|
304
|
+
content: [
|
|
305
|
+
{
|
|
306
|
+
type: role === "assistant" ? "output_text" : "input_text",
|
|
307
|
+
text
|
|
308
|
+
}
|
|
309
|
+
]
|
|
310
|
+
}
|
|
311
|
+
};
|
|
312
|
+
}
|
|
313
|
+
sendSessionUpdate() {
|
|
314
|
+
this.socket?.send(
|
|
315
|
+
JSON.stringify({
|
|
316
|
+
type: "session.update",
|
|
317
|
+
session: {
|
|
318
|
+
type: "realtime",
|
|
319
|
+
model: this.options.model || DEFAULT_MODEL2,
|
|
320
|
+
instructions: this.options.systemPrompt,
|
|
321
|
+
output_modalities: ["audio"],
|
|
322
|
+
audio: {
|
|
323
|
+
input: {
|
|
324
|
+
format: { type: "audio/pcm", rate: OPENAI_SAMPLE_RATE },
|
|
325
|
+
// The Micdrop client detects when the user speaks
|
|
326
|
+
turn_detection: null,
|
|
327
|
+
transcription: {
|
|
328
|
+
model: this.options.transcriptionModel || DEFAULT_TRANSCRIPTION_MODEL,
|
|
329
|
+
...this.options.language ? { language: this.options.language } : {}
|
|
330
|
+
},
|
|
331
|
+
noise_reduction: { type: "near_field" }
|
|
332
|
+
},
|
|
333
|
+
output: {
|
|
334
|
+
format: { type: "audio/pcm", rate: OPENAI_SAMPLE_RATE },
|
|
335
|
+
voice: this.options.voice || DEFAULT_VOICE
|
|
336
|
+
}
|
|
337
|
+
},
|
|
338
|
+
tools: this.tools.map((tool) => ({
|
|
339
|
+
type: "function",
|
|
340
|
+
name: tool.name,
|
|
341
|
+
description: tool.description,
|
|
342
|
+
parameters: (0, import_zod2.toJSONSchema)(tool.inputSchema || import_zod2.default.object())
|
|
343
|
+
}))
|
|
344
|
+
}
|
|
345
|
+
})
|
|
346
|
+
);
|
|
347
|
+
}
|
|
348
|
+
connect() {
|
|
349
|
+
this.initWS().catch((error) => {
|
|
350
|
+
this.log("Connection error:", error);
|
|
351
|
+
this.reconnect();
|
|
352
|
+
});
|
|
353
|
+
}
|
|
354
|
+
initWS() {
|
|
355
|
+
return new Promise((resolve, reject) => {
|
|
356
|
+
const model = this.options.model || DEFAULT_MODEL2;
|
|
357
|
+
const socket = new import_ws.default(
|
|
358
|
+
`wss://api.openai.com/v1/realtime?model=${encodeURIComponent(model)}`,
|
|
359
|
+
{ headers: { Authorization: `Bearer ${this.options.apiKey}` } }
|
|
360
|
+
);
|
|
361
|
+
this.socket = socket;
|
|
362
|
+
const timeout = setTimeout(() => {
|
|
363
|
+
this.log("Connection timeout");
|
|
364
|
+
this.closeSocket();
|
|
365
|
+
reject(new Error("WebSocket connection timeout"));
|
|
366
|
+
}, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT);
|
|
367
|
+
socket.on("open", () => {
|
|
368
|
+
clearTimeout(timeout);
|
|
369
|
+
this.log("Connection opened");
|
|
370
|
+
this.sendSessionUpdate();
|
|
371
|
+
if (this.everConnected) {
|
|
372
|
+
for (const message of this.conversation) {
|
|
373
|
+
if (message.role !== "user" && message.role !== "assistant") {
|
|
374
|
+
continue;
|
|
375
|
+
}
|
|
376
|
+
socket.send(
|
|
377
|
+
JSON.stringify(this.messageItem(message.role, message.content))
|
|
378
|
+
);
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
this.connected = true;
|
|
382
|
+
this.everConnected = true;
|
|
383
|
+
this.retryCount = 0;
|
|
384
|
+
const pending = this.pending;
|
|
385
|
+
this.pending = [];
|
|
386
|
+
pending.forEach((event) => this.send(event));
|
|
387
|
+
resolve();
|
|
388
|
+
});
|
|
389
|
+
socket.on("error", (error) => {
|
|
390
|
+
clearTimeout(timeout);
|
|
391
|
+
this.log("WebSocket error:", error);
|
|
392
|
+
reject(new Error("WebSocket connection error"));
|
|
393
|
+
});
|
|
394
|
+
socket.on("close", (code, reason) => {
|
|
395
|
+
clearTimeout(timeout);
|
|
396
|
+
const wasConnected = this.connected;
|
|
397
|
+
this.closeSocket();
|
|
398
|
+
if (this.destroyed) return;
|
|
399
|
+
this.log("Connection closed", { code, reason: reason.toString() });
|
|
400
|
+
if (wasConnected) this.reconnect();
|
|
401
|
+
});
|
|
402
|
+
socket.on("message", (data) => {
|
|
403
|
+
this.handleMessage(JSON.parse(data.toString()));
|
|
404
|
+
});
|
|
405
|
+
});
|
|
406
|
+
}
|
|
407
|
+
closeSocket() {
|
|
408
|
+
const socket = this.socket;
|
|
409
|
+
this.socket = void 0;
|
|
410
|
+
this.connected = false;
|
|
411
|
+
this.pending = [];
|
|
412
|
+
this.requested = [];
|
|
413
|
+
this.response = void 0;
|
|
414
|
+
if (!socket) return;
|
|
415
|
+
socket.removeAllListeners();
|
|
416
|
+
socket.on("error", () => {
|
|
417
|
+
});
|
|
418
|
+
if (socket.readyState === import_ws.default.OPEN || socket.readyState === import_ws.default.CONNECTING) {
|
|
419
|
+
socket.close(1e3);
|
|
420
|
+
}
|
|
421
|
+
this.endAnswer();
|
|
422
|
+
}
|
|
423
|
+
reconnect() {
|
|
424
|
+
if (this.destroyed) return;
|
|
425
|
+
this.retryCount++;
|
|
426
|
+
if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY)) {
|
|
427
|
+
this.log("Max retries reached, giving up");
|
|
428
|
+
this.emit("Failed");
|
|
429
|
+
return;
|
|
430
|
+
}
|
|
431
|
+
this.log("Reconnecting...");
|
|
432
|
+
this.reconnectTimeout = setTimeout(() => {
|
|
433
|
+
this.reconnectTimeout = void 0;
|
|
434
|
+
this.connect();
|
|
435
|
+
}, this.options.retryDelay ?? DEFAULT_RETRY_DELAY2);
|
|
436
|
+
}
|
|
437
|
+
isIgnored(responseId) {
|
|
438
|
+
return !responseId || this.ignoredResponses.has(responseId) || this.response?.id !== responseId;
|
|
439
|
+
}
|
|
440
|
+
handleMessage(message) {
|
|
441
|
+
switch (message.type) {
|
|
442
|
+
case "response.created": {
|
|
443
|
+
const requested = this.requested.shift() ?? {
|
|
444
|
+
generation: this.generation,
|
|
445
|
+
outOfBand: false
|
|
446
|
+
};
|
|
447
|
+
const id = message.response.id;
|
|
448
|
+
if (requested.generation !== this.generation) {
|
|
449
|
+
this.ignoredResponses.add(id);
|
|
450
|
+
this.socket?.send(
|
|
451
|
+
JSON.stringify({ type: "response.cancel", response_id: id })
|
|
452
|
+
);
|
|
453
|
+
break;
|
|
454
|
+
}
|
|
455
|
+
this.response = { id, ...requested };
|
|
456
|
+
this.audioItemId = void 0;
|
|
457
|
+
this.audioBytes = 0;
|
|
458
|
+
this.outputResampler.reset();
|
|
459
|
+
break;
|
|
460
|
+
}
|
|
461
|
+
case "response.output_audio.delta": {
|
|
462
|
+
if (this.isIgnored(message.response_id)) break;
|
|
463
|
+
const audio = Buffer.from(message.delta, "base64");
|
|
464
|
+
if (this.audioItemId !== message.item_id) {
|
|
465
|
+
this.audioItemId = message.item_id;
|
|
466
|
+
this.audioStartedAt = Date.now();
|
|
467
|
+
this.audioBytes = 0;
|
|
468
|
+
}
|
|
469
|
+
this.audioBytes += audio.length;
|
|
470
|
+
const output = this.outputResampler.process(audio);
|
|
471
|
+
if (output.length > 0) this.emitAudio(output);
|
|
472
|
+
break;
|
|
473
|
+
}
|
|
474
|
+
case "response.output_audio_transcript.delta":
|
|
475
|
+
if (this.isIgnored(message.response_id)) break;
|
|
476
|
+
this.addAnswerTranscript(message.delta);
|
|
477
|
+
break;
|
|
478
|
+
case "conversation.item.input_audio_transcription.completed":
|
|
479
|
+
this.log(`User transcript: "${message.transcript}"`);
|
|
480
|
+
this.addUserTranscript(message.transcript || "");
|
|
481
|
+
this.commitUserTranscript();
|
|
482
|
+
break;
|
|
483
|
+
case "conversation.item.input_audio_transcription.failed":
|
|
484
|
+
this.log("Transcription failed:", message.error);
|
|
485
|
+
this.commitUserTranscript();
|
|
486
|
+
break;
|
|
487
|
+
case "response.done":
|
|
488
|
+
this.onResponseDone(message.response);
|
|
489
|
+
break;
|
|
490
|
+
case "error":
|
|
491
|
+
if (message.error?.code === "response_cancel_not_active") break;
|
|
492
|
+
this.log("Error:", message.error);
|
|
493
|
+
break;
|
|
494
|
+
default:
|
|
495
|
+
break;
|
|
496
|
+
}
|
|
497
|
+
}
|
|
498
|
+
async onResponseDone(response) {
|
|
499
|
+
if (this.ignoredResponses.delete(response.id)) return;
|
|
500
|
+
if (this.response?.id !== response.id) return;
|
|
501
|
+
this.response = void 0;
|
|
502
|
+
const generation = this.generation;
|
|
503
|
+
if (response.status === "failed") {
|
|
504
|
+
this.log("Response failed:", response.status_details);
|
|
210
505
|
}
|
|
211
|
-
const
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
506
|
+
const calls = (response.output ?? []).filter(
|
|
507
|
+
(item) => item.type === "function_call"
|
|
508
|
+
);
|
|
509
|
+
if (calls.length === 0) {
|
|
510
|
+
this.endAnswer();
|
|
511
|
+
return;
|
|
512
|
+
}
|
|
513
|
+
let skipAnswer = true;
|
|
514
|
+
for (const call of calls) {
|
|
515
|
+
const toolCall = {
|
|
516
|
+
role: "tool_call",
|
|
517
|
+
toolCallId: call.call_id,
|
|
518
|
+
toolName: call.name,
|
|
519
|
+
parameters: call.arguments
|
|
520
|
+
};
|
|
521
|
+
const result = await this.runTool(toolCall);
|
|
522
|
+
if (generation !== this.generation) return;
|
|
523
|
+
if (!result.skipAnswer) skipAnswer = false;
|
|
524
|
+
this.send({
|
|
525
|
+
type: "conversation.item.create",
|
|
526
|
+
item: {
|
|
527
|
+
type: "function_call_output",
|
|
528
|
+
call_id: call.call_id,
|
|
529
|
+
output: JSON.stringify(result.output ?? null)
|
|
530
|
+
}
|
|
531
|
+
});
|
|
220
532
|
}
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
for (let k = 0; k < out.length; k++) {
|
|
226
|
-
result.writeInt16LE(out[k], k * 2);
|
|
533
|
+
if (skipAnswer) {
|
|
534
|
+
this.endAnswer();
|
|
535
|
+
} else {
|
|
536
|
+
this.createResponse();
|
|
227
537
|
}
|
|
228
|
-
return result;
|
|
229
538
|
}
|
|
230
539
|
};
|
|
231
540
|
|
|
232
541
|
// src/OpenaiSTT.ts
|
|
233
|
-
var
|
|
542
|
+
var import_server3 = require("@micdrop/server");
|
|
543
|
+
var import_ws2 = __toESM(require("ws"));
|
|
544
|
+
var DEFAULT_MODEL3 = "gpt-4o-transcribe";
|
|
234
545
|
var DEFAULT_LANGUAGE = "en";
|
|
235
|
-
var
|
|
236
|
-
var
|
|
237
|
-
var
|
|
546
|
+
var SAMPLE_RATE2 = 16e3;
|
|
547
|
+
var OPENAI_SAMPLE_RATE2 = 24e3;
|
|
548
|
+
var DEFAULT_CONNECTION_TIMEOUT2 = 5e3;
|
|
238
549
|
var DEFAULT_TRANSCRIPTION_TIMEOUT = 4e3;
|
|
239
|
-
var
|
|
240
|
-
var
|
|
241
|
-
var OpenaiSTT = class extends
|
|
550
|
+
var DEFAULT_RETRY_DELAY3 = 1e3;
|
|
551
|
+
var DEFAULT_MAX_RETRY2 = 3;
|
|
552
|
+
var OpenaiSTT = class extends import_server3.STT {
|
|
242
553
|
constructor(options) {
|
|
243
554
|
super();
|
|
244
555
|
this.options = options;
|
|
245
556
|
this.retryCount = 0;
|
|
246
557
|
this.audioChunksPending = [];
|
|
247
558
|
// Store audio chunks to send them again if reconnecting
|
|
248
|
-
this.resampler = new Pcm16Resampler(
|
|
559
|
+
this.resampler = new import_server3.Pcm16Resampler(SAMPLE_RATE2, OPENAI_SAMPLE_RATE2);
|
|
249
560
|
this.initPromise = this.initWS().catch((error) => {
|
|
250
561
|
console.error("[OpenaiSTT] Connection error:", error);
|
|
251
562
|
this.reconnect();
|
|
@@ -282,7 +593,7 @@ var OpenaiSTT = class extends import_server2.STT {
|
|
|
282
593
|
this.transcriptionTimeout = void 0;
|
|
283
594
|
}
|
|
284
595
|
this.socket?.removeAllListeners();
|
|
285
|
-
if (this.socket?.readyState ===
|
|
596
|
+
if (this.socket?.readyState === import_ws2.default.OPEN) {
|
|
286
597
|
this.socket?.close(1e3);
|
|
287
598
|
}
|
|
288
599
|
this.socket = void 0;
|
|
@@ -298,10 +609,10 @@ var OpenaiSTT = class extends import_server2.STT {
|
|
|
298
609
|
input: {
|
|
299
610
|
format: {
|
|
300
611
|
type: "audio/pcm",
|
|
301
|
-
rate:
|
|
612
|
+
rate: OPENAI_SAMPLE_RATE2
|
|
302
613
|
},
|
|
303
614
|
transcription: {
|
|
304
|
-
model: this.options.model ||
|
|
615
|
+
model: this.options.model || DEFAULT_MODEL3,
|
|
305
616
|
language: this.options.language || DEFAULT_LANGUAGE,
|
|
306
617
|
prompt: this.options.prompt || "Transcribe the incoming audio in real time."
|
|
307
618
|
},
|
|
@@ -320,7 +631,7 @@ var OpenaiSTT = class extends import_server2.STT {
|
|
|
320
631
|
}
|
|
321
632
|
async initWS() {
|
|
322
633
|
return new Promise((resolve, reject) => {
|
|
323
|
-
const socket = new
|
|
634
|
+
const socket = new import_ws2.default(
|
|
324
635
|
"wss://api.openai.com/v1/realtime?intent=transcription",
|
|
325
636
|
{
|
|
326
637
|
headers: {
|
|
@@ -335,7 +646,7 @@ var OpenaiSTT = class extends import_server2.STT {
|
|
|
335
646
|
socket.close();
|
|
336
647
|
this.socket = void 0;
|
|
337
648
|
reject(new Error("WebSocket connection timeout"));
|
|
338
|
-
}, this.options.connectionTimeout ??
|
|
649
|
+
}, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT2);
|
|
339
650
|
socket.addEventListener("open", () => {
|
|
340
651
|
clearTimeout(timeout);
|
|
341
652
|
this.sendSessionUpdate();
|
|
@@ -409,7 +720,7 @@ var OpenaiSTT = class extends import_server2.STT {
|
|
|
409
720
|
}
|
|
410
721
|
reconnect() {
|
|
411
722
|
this.retryCount++;
|
|
412
|
-
if (this.retryCount > (this.options.maxRetry ??
|
|
723
|
+
if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY2)) {
|
|
413
724
|
this.log("Max retries reached, giving up");
|
|
414
725
|
this.emit("Failed", this.audioChunksPending);
|
|
415
726
|
return;
|
|
@@ -431,7 +742,7 @@ var OpenaiSTT = class extends import_server2.STT {
|
|
|
431
742
|
this.log("Reconnection error:", error);
|
|
432
743
|
this.reconnect();
|
|
433
744
|
});
|
|
434
|
-
}, this.options.retryDelay ??
|
|
745
|
+
}, this.options.retryDelay ?? DEFAULT_RETRY_DELAY3);
|
|
435
746
|
});
|
|
436
747
|
}
|
|
437
748
|
commitAudio() {
|
|
@@ -442,15 +753,15 @@ var OpenaiSTT = class extends import_server2.STT {
|
|
|
442
753
|
};
|
|
443
754
|
|
|
444
755
|
// src/OpenaiTTS.ts
|
|
445
|
-
var
|
|
756
|
+
var import_server4 = require("@micdrop/server");
|
|
446
757
|
var import_openai2 = __toESM(require("openai"));
|
|
447
|
-
var
|
|
448
|
-
var
|
|
449
|
-
var
|
|
758
|
+
var DEFAULT_MODEL4 = "gpt-4o-mini-tts";
|
|
759
|
+
var DEFAULT_VOICE2 = "alloy";
|
|
760
|
+
var OPENAI_SAMPLE_RATE3 = 24e3;
|
|
450
761
|
var OUTPUT_SAMPLE_RATE = 16e3;
|
|
451
762
|
var MAX_IN_FLIGHT = 2;
|
|
452
763
|
var OPENING_CUSHION = 300 * 32;
|
|
453
|
-
var OpenaiTTS = class extends
|
|
764
|
+
var OpenaiTTS = class extends import_server4.TTS {
|
|
454
765
|
constructor(options) {
|
|
455
766
|
super();
|
|
456
767
|
this.options = options;
|
|
@@ -461,8 +772,7 @@ var OpenaiTTS = class extends import_server3.TTS {
|
|
|
461
772
|
// `counter`, which numbers the synthesis calls and must only move when there
|
|
462
773
|
// is something to synthesize.
|
|
463
774
|
this.generation = 0;
|
|
464
|
-
this.
|
|
465
|
-
// Incomplete sentence waiting for more text
|
|
775
|
+
this.splitter = new import_server4.SentenceSplitter();
|
|
466
776
|
this.queue = [];
|
|
467
777
|
// Sentences whose synthesis has not started
|
|
468
778
|
this.pending = [];
|
|
@@ -483,7 +793,7 @@ var OpenaiTTS = class extends import_server3.TTS {
|
|
|
483
793
|
if (this.generation !== generation) return false;
|
|
484
794
|
this.counter++;
|
|
485
795
|
counter = this.counter;
|
|
486
|
-
this.
|
|
796
|
+
this.splitter.reset();
|
|
487
797
|
this.cushion = [];
|
|
488
798
|
this.cushionBytes = 0;
|
|
489
799
|
return true;
|
|
@@ -491,22 +801,25 @@ var OpenaiTTS = class extends import_server3.TTS {
|
|
|
491
801
|
textStream.on("data", (chunk) => {
|
|
492
802
|
if (!claimCall()) return;
|
|
493
803
|
if (counter !== this.counter) return;
|
|
494
|
-
this.
|
|
495
|
-
|
|
804
|
+
for (const sentence of this.splitter.push(chunk.toString("utf-8"))) {
|
|
805
|
+
this.enqueue(counter, sentence);
|
|
806
|
+
}
|
|
496
807
|
});
|
|
497
808
|
textStream.on("error", (error) => {
|
|
498
809
|
this.log("Error in text stream", error);
|
|
499
810
|
});
|
|
500
811
|
textStream.on("end", () => {
|
|
501
812
|
if (!counter || counter !== this.counter) return;
|
|
502
|
-
this.
|
|
813
|
+
for (const sentence of this.splitter.flush()) {
|
|
814
|
+
this.enqueue(counter, sentence);
|
|
815
|
+
}
|
|
503
816
|
});
|
|
504
817
|
}
|
|
505
818
|
cancel() {
|
|
506
819
|
this.generation++;
|
|
507
820
|
this.log("Cancel");
|
|
508
821
|
this.counter++;
|
|
509
|
-
this.
|
|
822
|
+
this.splitter.reset();
|
|
510
823
|
this.queue = [];
|
|
511
824
|
this.pending = [];
|
|
512
825
|
this.cushion = [];
|
|
@@ -514,25 +827,6 @@ var OpenaiTTS = class extends import_server3.TTS {
|
|
|
514
827
|
this.abortControllers.forEach((controller) => controller.abort());
|
|
515
828
|
this.abortControllers.clear();
|
|
516
829
|
}
|
|
517
|
-
// Extract complete sentences from the buffer and enqueue them.
|
|
518
|
-
// On `end`, whatever remains in the buffer is flushed as a last sentence.
|
|
519
|
-
flushSentences(counter, end) {
|
|
520
|
-
const regex = /[\s\S]*?[.!?…\n]+(?=\s|$)/g;
|
|
521
|
-
let match;
|
|
522
|
-
let lastIndex = 0;
|
|
523
|
-
while ((match = regex.exec(this.buffer)) !== null) {
|
|
524
|
-
if (!end && regex.lastIndex === this.buffer.length) break;
|
|
525
|
-
const sentence = match[0].trim();
|
|
526
|
-
if (sentence) this.enqueue(counter, sentence);
|
|
527
|
-
lastIndex = regex.lastIndex;
|
|
528
|
-
}
|
|
529
|
-
this.buffer = this.buffer.slice(lastIndex);
|
|
530
|
-
if (end) {
|
|
531
|
-
const rest = this.buffer.trim();
|
|
532
|
-
if (rest) this.enqueue(counter, rest);
|
|
533
|
-
this.buffer = "";
|
|
534
|
-
}
|
|
535
|
-
}
|
|
536
830
|
enqueue(counter, text) {
|
|
537
831
|
this.queue.push({ counter, text });
|
|
538
832
|
this.pump();
|
|
@@ -577,8 +871,8 @@ var OpenaiTTS = class extends import_server3.TTS {
|
|
|
577
871
|
this.abortControllers.add(controller);
|
|
578
872
|
const response = this.openai.audio.speech.create(
|
|
579
873
|
{
|
|
580
|
-
model: this.options.model ||
|
|
581
|
-
voice: this.options.voice ||
|
|
874
|
+
model: this.options.model || DEFAULT_MODEL4,
|
|
875
|
+
voice: this.options.voice || DEFAULT_VOICE2,
|
|
582
876
|
input: item.text,
|
|
583
877
|
response_format: "pcm",
|
|
584
878
|
...this.options.instructions ? { instructions: this.options.instructions } : {},
|
|
@@ -617,8 +911,8 @@ var OpenaiTTS = class extends import_server3.TTS {
|
|
|
617
911
|
if (!response?.body) return;
|
|
618
912
|
if (item.counter !== this.counter) return;
|
|
619
913
|
this.log(`Synthesizing: "${item.text}"`);
|
|
620
|
-
const resampler = new Pcm16Resampler(
|
|
621
|
-
|
|
914
|
+
const resampler = new import_server4.Pcm16Resampler(
|
|
915
|
+
OPENAI_SAMPLE_RATE3,
|
|
622
916
|
OUTPUT_SAMPLE_RATE
|
|
623
917
|
);
|
|
624
918
|
const reader = response.body.getReader();
|
|
@@ -640,6 +934,7 @@ var OpenaiTTS = class extends import_server3.TTS {
|
|
|
640
934
|
// Annotate the CommonJS export names for ESM import in node:
|
|
641
935
|
0 && (module.exports = {
|
|
642
936
|
OpenaiAgent,
|
|
937
|
+
OpenaiRealtime,
|
|
643
938
|
OpenaiSTT,
|
|
644
939
|
OpenaiTTS
|
|
645
940
|
});
|