@micdrop/openai 1.3.2 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +112 -3
- package/dist/index.d.mts +63 -4
- package/dist/index.d.ts +63 -4
- package/dist/index.js +383 -88
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +384 -87
- package/dist/index.mjs.map +1 -1
- package/package.json +2 -2
package/dist/index.mjs
CHANGED
|
@@ -145,61 +145,374 @@ var OpenaiAgent = class extends Agent {
|
|
|
145
145
|
}
|
|
146
146
|
};
|
|
147
147
|
|
|
148
|
-
// src/
|
|
149
|
-
import {
|
|
148
|
+
// src/OpenaiRealtime.ts
|
|
149
|
+
import {
|
|
150
|
+
Pcm16Resampler,
|
|
151
|
+
Realtime
|
|
152
|
+
} from "@micdrop/server";
|
|
150
153
|
import WebSocket from "ws";
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
var
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
this.
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
154
|
+
import z2, { toJSONSchema as toJSONSchema2 } from "zod";
|
|
155
|
+
var DEFAULT_MODEL2 = "gpt-realtime-2.1";
|
|
156
|
+
var DEFAULT_VOICE = "marin";
|
|
157
|
+
var DEFAULT_TRANSCRIPTION_MODEL = "gpt-4o-transcribe";
|
|
158
|
+
var SAMPLE_RATE = 16e3;
|
|
159
|
+
var OPENAI_SAMPLE_RATE = 24e3;
|
|
160
|
+
var OPENAI_BYTES_PER_MS = OPENAI_SAMPLE_RATE * 2 / 1e3;
|
|
161
|
+
var DEFAULT_CONNECTION_TIMEOUT = 5e3;
|
|
162
|
+
var DEFAULT_RETRY_DELAY2 = 1e3;
|
|
163
|
+
var DEFAULT_MAX_RETRY = 3;
|
|
164
|
+
var OpenaiRealtime = class extends Realtime {
|
|
165
|
+
constructor(options) {
|
|
166
|
+
super(options);
|
|
167
|
+
this.connected = false;
|
|
168
|
+
this.everConnected = false;
|
|
169
|
+
this.destroyed = false;
|
|
170
|
+
// Events sent while the connection opens
|
|
171
|
+
this.pending = [];
|
|
172
|
+
this.retryCount = 0;
|
|
173
|
+
this.inputResampler = new Pcm16Resampler(SAMPLE_RATE, OPENAI_SAMPLE_RATE);
|
|
174
|
+
this.outputResampler = new Pcm16Resampler(OPENAI_SAMPLE_RATE, SAMPLE_RATE);
|
|
175
|
+
// Bumped by every interruption, so late responses can tell they are stale
|
|
176
|
+
this.generation = 0;
|
|
177
|
+
this.requested = [];
|
|
178
|
+
this.ignoredResponses = /* @__PURE__ */ new Set();
|
|
179
|
+
this.audioStartedAt = 0;
|
|
180
|
+
this.audioBytes = 0;
|
|
181
|
+
this.connect();
|
|
182
|
+
}
|
|
183
|
+
destroy() {
|
|
184
|
+
this.destroyed = true;
|
|
185
|
+
super.destroy();
|
|
186
|
+
if (this.reconnectTimeout) {
|
|
187
|
+
clearTimeout(this.reconnectTimeout);
|
|
188
|
+
this.reconnectTimeout = void 0;
|
|
189
|
+
}
|
|
190
|
+
this.closeSocket();
|
|
191
|
+
}
|
|
192
|
+
openTurn() {
|
|
193
|
+
this.inputResampler.reset();
|
|
194
|
+
}
|
|
195
|
+
appendAudio(chunk) {
|
|
196
|
+
const audio = this.inputResampler.process(chunk);
|
|
197
|
+
if (audio.length === 0) return;
|
|
198
|
+
this.send({
|
|
199
|
+
type: "input_audio_buffer.append",
|
|
200
|
+
audio: audio.toString("base64")
|
|
201
|
+
});
|
|
202
|
+
}
|
|
203
|
+
closeTurn() {
|
|
204
|
+
this.send({ type: "input_audio_buffer.commit" });
|
|
205
|
+
this.createResponse();
|
|
206
|
+
}
|
|
207
|
+
clearTurn() {
|
|
208
|
+
this.send({ type: "input_audio_buffer.clear" });
|
|
209
|
+
}
|
|
210
|
+
generate() {
|
|
211
|
+
this.createResponse();
|
|
212
|
+
}
|
|
213
|
+
speakText(text) {
|
|
214
|
+
this.createResponse({
|
|
215
|
+
conversation: "none",
|
|
216
|
+
instructions: `Say exactly the following text, word for word, and nothing else: ${text}`
|
|
217
|
+
});
|
|
218
|
+
}
|
|
219
|
+
cancelAnswer() {
|
|
220
|
+
this.generation++;
|
|
221
|
+
this.outputResampler.reset();
|
|
222
|
+
const response = this.response;
|
|
223
|
+
if (!response) return;
|
|
224
|
+
this.response = void 0;
|
|
225
|
+
this.ignoredResponses.add(response.id);
|
|
226
|
+
this.send({ type: "response.cancel", response_id: response.id });
|
|
227
|
+
if (!response.outOfBand && this.audioItemId) {
|
|
228
|
+
const played = Math.min(
|
|
229
|
+
Date.now() - this.audioStartedAt,
|
|
230
|
+
this.audioBytes / OPENAI_BYTES_PER_MS
|
|
231
|
+
);
|
|
232
|
+
this.send({
|
|
233
|
+
type: "conversation.item.truncate",
|
|
234
|
+
item_id: this.audioItemId,
|
|
235
|
+
content_index: 0,
|
|
236
|
+
audio_end_ms: Math.max(0, Math.floor(played))
|
|
237
|
+
});
|
|
238
|
+
}
|
|
239
|
+
this.audioItemId = void 0;
|
|
240
|
+
}
|
|
241
|
+
updateTools() {
|
|
242
|
+
if (this.connected) this.sendSessionUpdate();
|
|
243
|
+
}
|
|
244
|
+
injectMessage(role, text) {
|
|
245
|
+
if (!this.connected && this.everConnected) return;
|
|
246
|
+
this.send(this.messageItem(role, text));
|
|
247
|
+
}
|
|
248
|
+
createResponse(response) {
|
|
249
|
+
this.requested.push({
|
|
250
|
+
generation: this.generation,
|
|
251
|
+
outOfBand: !!response
|
|
252
|
+
});
|
|
253
|
+
this.send({ type: "response.create", ...response ? { response } : {} });
|
|
254
|
+
}
|
|
255
|
+
send(event) {
|
|
256
|
+
if (this.connected && this.socket?.readyState === WebSocket.OPEN) {
|
|
257
|
+
this.socket.send(JSON.stringify(event));
|
|
258
|
+
} else {
|
|
259
|
+
this.pending.push(event);
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
messageItem(role, text) {
|
|
263
|
+
return {
|
|
264
|
+
type: "conversation.item.create",
|
|
265
|
+
item: {
|
|
266
|
+
type: "message",
|
|
267
|
+
role,
|
|
268
|
+
content: [
|
|
269
|
+
{
|
|
270
|
+
type: role === "assistant" ? "output_text" : "input_text",
|
|
271
|
+
text
|
|
272
|
+
}
|
|
273
|
+
]
|
|
274
|
+
}
|
|
275
|
+
};
|
|
276
|
+
}
|
|
277
|
+
sendSessionUpdate() {
|
|
278
|
+
this.socket?.send(
|
|
279
|
+
JSON.stringify({
|
|
280
|
+
type: "session.update",
|
|
281
|
+
session: {
|
|
282
|
+
type: "realtime",
|
|
283
|
+
model: this.options.model || DEFAULT_MODEL2,
|
|
284
|
+
instructions: this.options.systemPrompt,
|
|
285
|
+
output_modalities: ["audio"],
|
|
286
|
+
audio: {
|
|
287
|
+
input: {
|
|
288
|
+
format: { type: "audio/pcm", rate: OPENAI_SAMPLE_RATE },
|
|
289
|
+
// The Micdrop client detects when the user speaks
|
|
290
|
+
turn_detection: null,
|
|
291
|
+
transcription: {
|
|
292
|
+
model: this.options.transcriptionModel || DEFAULT_TRANSCRIPTION_MODEL,
|
|
293
|
+
...this.options.language ? { language: this.options.language } : {}
|
|
294
|
+
},
|
|
295
|
+
noise_reduction: { type: "near_field" }
|
|
296
|
+
},
|
|
297
|
+
output: {
|
|
298
|
+
format: { type: "audio/pcm", rate: OPENAI_SAMPLE_RATE },
|
|
299
|
+
voice: this.options.voice || DEFAULT_VOICE
|
|
300
|
+
}
|
|
301
|
+
},
|
|
302
|
+
tools: this.tools.map((tool) => ({
|
|
303
|
+
type: "function",
|
|
304
|
+
name: tool.name,
|
|
305
|
+
description: tool.description,
|
|
306
|
+
parameters: toJSONSchema2(tool.inputSchema || z2.object())
|
|
307
|
+
}))
|
|
308
|
+
}
|
|
309
|
+
})
|
|
310
|
+
);
|
|
311
|
+
}
|
|
312
|
+
connect() {
|
|
313
|
+
this.initWS().catch((error) => {
|
|
314
|
+
this.log("Connection error:", error);
|
|
315
|
+
this.reconnect();
|
|
316
|
+
});
|
|
317
|
+
}
|
|
318
|
+
initWS() {
|
|
319
|
+
return new Promise((resolve, reject) => {
|
|
320
|
+
const model = this.options.model || DEFAULT_MODEL2;
|
|
321
|
+
const socket = new WebSocket(
|
|
322
|
+
`wss://api.openai.com/v1/realtime?model=${encodeURIComponent(model)}`,
|
|
323
|
+
{ headers: { Authorization: `Bearer ${this.options.apiKey}` } }
|
|
324
|
+
);
|
|
325
|
+
this.socket = socket;
|
|
326
|
+
const timeout = setTimeout(() => {
|
|
327
|
+
this.log("Connection timeout");
|
|
328
|
+
this.closeSocket();
|
|
329
|
+
reject(new Error("WebSocket connection timeout"));
|
|
330
|
+
}, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT);
|
|
331
|
+
socket.on("open", () => {
|
|
332
|
+
clearTimeout(timeout);
|
|
333
|
+
this.log("Connection opened");
|
|
334
|
+
this.sendSessionUpdate();
|
|
335
|
+
if (this.everConnected) {
|
|
336
|
+
for (const message of this.conversation) {
|
|
337
|
+
if (message.role !== "user" && message.role !== "assistant") {
|
|
338
|
+
continue;
|
|
339
|
+
}
|
|
340
|
+
socket.send(
|
|
341
|
+
JSON.stringify(this.messageItem(message.role, message.content))
|
|
342
|
+
);
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
this.connected = true;
|
|
346
|
+
this.everConnected = true;
|
|
347
|
+
this.retryCount = 0;
|
|
348
|
+
const pending = this.pending;
|
|
349
|
+
this.pending = [];
|
|
350
|
+
pending.forEach((event) => this.send(event));
|
|
351
|
+
resolve();
|
|
352
|
+
});
|
|
353
|
+
socket.on("error", (error) => {
|
|
354
|
+
clearTimeout(timeout);
|
|
355
|
+
this.log("WebSocket error:", error);
|
|
356
|
+
reject(new Error("WebSocket connection error"));
|
|
357
|
+
});
|
|
358
|
+
socket.on("close", (code, reason) => {
|
|
359
|
+
clearTimeout(timeout);
|
|
360
|
+
const wasConnected = this.connected;
|
|
361
|
+
this.closeSocket();
|
|
362
|
+
if (this.destroyed) return;
|
|
363
|
+
this.log("Connection closed", { code, reason: reason.toString() });
|
|
364
|
+
if (wasConnected) this.reconnect();
|
|
365
|
+
});
|
|
366
|
+
socket.on("message", (data) => {
|
|
367
|
+
this.handleMessage(JSON.parse(data.toString()));
|
|
368
|
+
});
|
|
369
|
+
});
|
|
370
|
+
}
|
|
371
|
+
closeSocket() {
|
|
372
|
+
const socket = this.socket;
|
|
373
|
+
this.socket = void 0;
|
|
374
|
+
this.connected = false;
|
|
375
|
+
this.pending = [];
|
|
376
|
+
this.requested = [];
|
|
377
|
+
this.response = void 0;
|
|
378
|
+
if (!socket) return;
|
|
379
|
+
socket.removeAllListeners();
|
|
380
|
+
socket.on("error", () => {
|
|
381
|
+
});
|
|
382
|
+
if (socket.readyState === WebSocket.OPEN || socket.readyState === WebSocket.CONNECTING) {
|
|
383
|
+
socket.close(1e3);
|
|
172
384
|
}
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
385
|
+
this.endAnswer();
|
|
386
|
+
}
|
|
387
|
+
reconnect() {
|
|
388
|
+
if (this.destroyed) return;
|
|
389
|
+
this.retryCount++;
|
|
390
|
+
if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY)) {
|
|
391
|
+
this.log("Max retries reached, giving up");
|
|
392
|
+
this.emit("Failed");
|
|
393
|
+
return;
|
|
182
394
|
}
|
|
183
|
-
|
|
184
|
-
this.
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
395
|
+
this.log("Reconnecting...");
|
|
396
|
+
this.reconnectTimeout = setTimeout(() => {
|
|
397
|
+
this.reconnectTimeout = void 0;
|
|
398
|
+
this.connect();
|
|
399
|
+
}, this.options.retryDelay ?? DEFAULT_RETRY_DELAY2);
|
|
400
|
+
}
|
|
401
|
+
isIgnored(responseId) {
|
|
402
|
+
return !responseId || this.ignoredResponses.has(responseId) || this.response?.id !== responseId;
|
|
403
|
+
}
|
|
404
|
+
handleMessage(message) {
|
|
405
|
+
switch (message.type) {
|
|
406
|
+
case "response.created": {
|
|
407
|
+
const requested = this.requested.shift() ?? {
|
|
408
|
+
generation: this.generation,
|
|
409
|
+
outOfBand: false
|
|
410
|
+
};
|
|
411
|
+
const id = message.response.id;
|
|
412
|
+
if (requested.generation !== this.generation) {
|
|
413
|
+
this.ignoredResponses.add(id);
|
|
414
|
+
this.socket?.send(
|
|
415
|
+
JSON.stringify({ type: "response.cancel", response_id: id })
|
|
416
|
+
);
|
|
417
|
+
break;
|
|
418
|
+
}
|
|
419
|
+
this.response = { id, ...requested };
|
|
420
|
+
this.audioItemId = void 0;
|
|
421
|
+
this.audioBytes = 0;
|
|
422
|
+
this.outputResampler.reset();
|
|
423
|
+
break;
|
|
424
|
+
}
|
|
425
|
+
case "response.output_audio.delta": {
|
|
426
|
+
if (this.isIgnored(message.response_id)) break;
|
|
427
|
+
const audio = Buffer.from(message.delta, "base64");
|
|
428
|
+
if (this.audioItemId !== message.item_id) {
|
|
429
|
+
this.audioItemId = message.item_id;
|
|
430
|
+
this.audioStartedAt = Date.now();
|
|
431
|
+
this.audioBytes = 0;
|
|
432
|
+
}
|
|
433
|
+
this.audioBytes += audio.length;
|
|
434
|
+
const output = this.outputResampler.process(audio);
|
|
435
|
+
if (output.length > 0) this.emitAudio(output);
|
|
436
|
+
break;
|
|
437
|
+
}
|
|
438
|
+
case "response.output_audio_transcript.delta":
|
|
439
|
+
if (this.isIgnored(message.response_id)) break;
|
|
440
|
+
this.addAnswerTranscript(message.delta);
|
|
441
|
+
break;
|
|
442
|
+
case "conversation.item.input_audio_transcription.completed":
|
|
443
|
+
this.log(`User transcript: "${message.transcript}"`);
|
|
444
|
+
this.addUserTranscript(message.transcript || "");
|
|
445
|
+
this.commitUserTranscript();
|
|
446
|
+
break;
|
|
447
|
+
case "conversation.item.input_audio_transcription.failed":
|
|
448
|
+
this.log("Transcription failed:", message.error);
|
|
449
|
+
this.commitUserTranscript();
|
|
450
|
+
break;
|
|
451
|
+
case "response.done":
|
|
452
|
+
this.onResponseDone(message.response);
|
|
453
|
+
break;
|
|
454
|
+
case "error":
|
|
455
|
+
if (message.error?.code === "response_cancel_not_active") break;
|
|
456
|
+
this.log("Error:", message.error);
|
|
457
|
+
break;
|
|
458
|
+
default:
|
|
459
|
+
break;
|
|
460
|
+
}
|
|
461
|
+
}
|
|
462
|
+
async onResponseDone(response) {
|
|
463
|
+
if (this.ignoredResponses.delete(response.id)) return;
|
|
464
|
+
if (this.response?.id !== response.id) return;
|
|
465
|
+
this.response = void 0;
|
|
466
|
+
const generation = this.generation;
|
|
467
|
+
if (response.status === "failed") {
|
|
468
|
+
this.log("Response failed:", response.status_details);
|
|
469
|
+
}
|
|
470
|
+
const calls = (response.output ?? []).filter(
|
|
471
|
+
(item) => item.type === "function_call"
|
|
472
|
+
);
|
|
473
|
+
if (calls.length === 0) {
|
|
474
|
+
this.endAnswer();
|
|
475
|
+
return;
|
|
476
|
+
}
|
|
477
|
+
let skipAnswer = true;
|
|
478
|
+
for (const call of calls) {
|
|
479
|
+
const toolCall = {
|
|
480
|
+
role: "tool_call",
|
|
481
|
+
toolCallId: call.call_id,
|
|
482
|
+
toolName: call.name,
|
|
483
|
+
parameters: call.arguments
|
|
484
|
+
};
|
|
485
|
+
const result = await this.runTool(toolCall);
|
|
486
|
+
if (generation !== this.generation) return;
|
|
487
|
+
if (!result.skipAnswer) skipAnswer = false;
|
|
488
|
+
this.send({
|
|
489
|
+
type: "conversation.item.create",
|
|
490
|
+
item: {
|
|
491
|
+
type: "function_call_output",
|
|
492
|
+
call_id: call.call_id,
|
|
493
|
+
output: JSON.stringify(result.output ?? null)
|
|
494
|
+
}
|
|
495
|
+
});
|
|
496
|
+
}
|
|
497
|
+
if (skipAnswer) {
|
|
498
|
+
this.endAnswer();
|
|
499
|
+
} else {
|
|
500
|
+
this.createResponse();
|
|
189
501
|
}
|
|
190
|
-
return result;
|
|
191
502
|
}
|
|
192
503
|
};
|
|
193
504
|
|
|
194
505
|
// src/OpenaiSTT.ts
|
|
195
|
-
|
|
506
|
+
import { Pcm16Resampler as Pcm16Resampler2, STT } from "@micdrop/server";
|
|
507
|
+
import WebSocket2 from "ws";
|
|
508
|
+
var DEFAULT_MODEL3 = "gpt-4o-transcribe";
|
|
196
509
|
var DEFAULT_LANGUAGE = "en";
|
|
197
|
-
var
|
|
198
|
-
var
|
|
199
|
-
var
|
|
510
|
+
var SAMPLE_RATE2 = 16e3;
|
|
511
|
+
var OPENAI_SAMPLE_RATE2 = 24e3;
|
|
512
|
+
var DEFAULT_CONNECTION_TIMEOUT2 = 5e3;
|
|
200
513
|
var DEFAULT_TRANSCRIPTION_TIMEOUT = 4e3;
|
|
201
|
-
var
|
|
202
|
-
var
|
|
514
|
+
var DEFAULT_RETRY_DELAY3 = 1e3;
|
|
515
|
+
var DEFAULT_MAX_RETRY2 = 3;
|
|
203
516
|
var OpenaiSTT = class extends STT {
|
|
204
517
|
constructor(options) {
|
|
205
518
|
super();
|
|
@@ -207,7 +520,7 @@ var OpenaiSTT = class extends STT {
|
|
|
207
520
|
this.retryCount = 0;
|
|
208
521
|
this.audioChunksPending = [];
|
|
209
522
|
// Store audio chunks to send them again if reconnecting
|
|
210
|
-
this.resampler = new
|
|
523
|
+
this.resampler = new Pcm16Resampler2(SAMPLE_RATE2, OPENAI_SAMPLE_RATE2);
|
|
211
524
|
this.initPromise = this.initWS().catch((error) => {
|
|
212
525
|
console.error("[OpenaiSTT] Connection error:", error);
|
|
213
526
|
this.reconnect();
|
|
@@ -244,7 +557,7 @@ var OpenaiSTT = class extends STT {
|
|
|
244
557
|
this.transcriptionTimeout = void 0;
|
|
245
558
|
}
|
|
246
559
|
this.socket?.removeAllListeners();
|
|
247
|
-
if (this.socket?.readyState ===
|
|
560
|
+
if (this.socket?.readyState === WebSocket2.OPEN) {
|
|
248
561
|
this.socket?.close(1e3);
|
|
249
562
|
}
|
|
250
563
|
this.socket = void 0;
|
|
@@ -260,10 +573,10 @@ var OpenaiSTT = class extends STT {
|
|
|
260
573
|
input: {
|
|
261
574
|
format: {
|
|
262
575
|
type: "audio/pcm",
|
|
263
|
-
rate:
|
|
576
|
+
rate: OPENAI_SAMPLE_RATE2
|
|
264
577
|
},
|
|
265
578
|
transcription: {
|
|
266
|
-
model: this.options.model ||
|
|
579
|
+
model: this.options.model || DEFAULT_MODEL3,
|
|
267
580
|
language: this.options.language || DEFAULT_LANGUAGE,
|
|
268
581
|
prompt: this.options.prompt || "Transcribe the incoming audio in real time."
|
|
269
582
|
},
|
|
@@ -282,7 +595,7 @@ var OpenaiSTT = class extends STT {
|
|
|
282
595
|
}
|
|
283
596
|
async initWS() {
|
|
284
597
|
return new Promise((resolve, reject) => {
|
|
285
|
-
const socket = new
|
|
598
|
+
const socket = new WebSocket2(
|
|
286
599
|
"wss://api.openai.com/v1/realtime?intent=transcription",
|
|
287
600
|
{
|
|
288
601
|
headers: {
|
|
@@ -297,7 +610,7 @@ var OpenaiSTT = class extends STT {
|
|
|
297
610
|
socket.close();
|
|
298
611
|
this.socket = void 0;
|
|
299
612
|
reject(new Error("WebSocket connection timeout"));
|
|
300
|
-
}, this.options.connectionTimeout ??
|
|
613
|
+
}, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT2);
|
|
301
614
|
socket.addEventListener("open", () => {
|
|
302
615
|
clearTimeout(timeout);
|
|
303
616
|
this.sendSessionUpdate();
|
|
@@ -371,7 +684,7 @@ var OpenaiSTT = class extends STT {
|
|
|
371
684
|
}
|
|
372
685
|
reconnect() {
|
|
373
686
|
this.retryCount++;
|
|
374
|
-
if (this.retryCount > (this.options.maxRetry ??
|
|
687
|
+
if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY2)) {
|
|
375
688
|
this.log("Max retries reached, giving up");
|
|
376
689
|
this.emit("Failed", this.audioChunksPending);
|
|
377
690
|
return;
|
|
@@ -393,7 +706,7 @@ var OpenaiSTT = class extends STT {
|
|
|
393
706
|
this.log("Reconnection error:", error);
|
|
394
707
|
this.reconnect();
|
|
395
708
|
});
|
|
396
|
-
}, this.options.retryDelay ??
|
|
709
|
+
}, this.options.retryDelay ?? DEFAULT_RETRY_DELAY3);
|
|
397
710
|
});
|
|
398
711
|
}
|
|
399
712
|
commitAudio() {
|
|
@@ -404,11 +717,11 @@ var OpenaiSTT = class extends STT {
|
|
|
404
717
|
};
|
|
405
718
|
|
|
406
719
|
// src/OpenaiTTS.ts
|
|
407
|
-
import { TTS } from "@micdrop/server";
|
|
720
|
+
import { Pcm16Resampler as Pcm16Resampler3, SentenceSplitter, TTS } from "@micdrop/server";
|
|
408
721
|
import OpenAI2 from "openai";
|
|
409
|
-
var
|
|
410
|
-
var
|
|
411
|
-
var
|
|
722
|
+
var DEFAULT_MODEL4 = "gpt-4o-mini-tts";
|
|
723
|
+
var DEFAULT_VOICE2 = "alloy";
|
|
724
|
+
var OPENAI_SAMPLE_RATE3 = 24e3;
|
|
412
725
|
var OUTPUT_SAMPLE_RATE = 16e3;
|
|
413
726
|
var MAX_IN_FLIGHT = 2;
|
|
414
727
|
var OPENING_CUSHION = 300 * 32;
|
|
@@ -423,8 +736,7 @@ var OpenaiTTS = class extends TTS {
|
|
|
423
736
|
// `counter`, which numbers the synthesis calls and must only move when there
|
|
424
737
|
// is something to synthesize.
|
|
425
738
|
this.generation = 0;
|
|
426
|
-
this.
|
|
427
|
-
// Incomplete sentence waiting for more text
|
|
739
|
+
this.splitter = new SentenceSplitter();
|
|
428
740
|
this.queue = [];
|
|
429
741
|
// Sentences whose synthesis has not started
|
|
430
742
|
this.pending = [];
|
|
@@ -445,7 +757,7 @@ var OpenaiTTS = class extends TTS {
|
|
|
445
757
|
if (this.generation !== generation) return false;
|
|
446
758
|
this.counter++;
|
|
447
759
|
counter = this.counter;
|
|
448
|
-
this.
|
|
760
|
+
this.splitter.reset();
|
|
449
761
|
this.cushion = [];
|
|
450
762
|
this.cushionBytes = 0;
|
|
451
763
|
return true;
|
|
@@ -453,22 +765,25 @@ var OpenaiTTS = class extends TTS {
|
|
|
453
765
|
textStream.on("data", (chunk) => {
|
|
454
766
|
if (!claimCall()) return;
|
|
455
767
|
if (counter !== this.counter) return;
|
|
456
|
-
this.
|
|
457
|
-
|
|
768
|
+
for (const sentence of this.splitter.push(chunk.toString("utf-8"))) {
|
|
769
|
+
this.enqueue(counter, sentence);
|
|
770
|
+
}
|
|
458
771
|
});
|
|
459
772
|
textStream.on("error", (error) => {
|
|
460
773
|
this.log("Error in text stream", error);
|
|
461
774
|
});
|
|
462
775
|
textStream.on("end", () => {
|
|
463
776
|
if (!counter || counter !== this.counter) return;
|
|
464
|
-
this.
|
|
777
|
+
for (const sentence of this.splitter.flush()) {
|
|
778
|
+
this.enqueue(counter, sentence);
|
|
779
|
+
}
|
|
465
780
|
});
|
|
466
781
|
}
|
|
467
782
|
cancel() {
|
|
468
783
|
this.generation++;
|
|
469
784
|
this.log("Cancel");
|
|
470
785
|
this.counter++;
|
|
471
|
-
this.
|
|
786
|
+
this.splitter.reset();
|
|
472
787
|
this.queue = [];
|
|
473
788
|
this.pending = [];
|
|
474
789
|
this.cushion = [];
|
|
@@ -476,25 +791,6 @@ var OpenaiTTS = class extends TTS {
|
|
|
476
791
|
this.abortControllers.forEach((controller) => controller.abort());
|
|
477
792
|
this.abortControllers.clear();
|
|
478
793
|
}
|
|
479
|
-
// Extract complete sentences from the buffer and enqueue them.
|
|
480
|
-
// On `end`, whatever remains in the buffer is flushed as a last sentence.
|
|
481
|
-
flushSentences(counter, end) {
|
|
482
|
-
const regex = /[\s\S]*?[.!?…\n]+(?=\s|$)/g;
|
|
483
|
-
let match;
|
|
484
|
-
let lastIndex = 0;
|
|
485
|
-
while ((match = regex.exec(this.buffer)) !== null) {
|
|
486
|
-
if (!end && regex.lastIndex === this.buffer.length) break;
|
|
487
|
-
const sentence = match[0].trim();
|
|
488
|
-
if (sentence) this.enqueue(counter, sentence);
|
|
489
|
-
lastIndex = regex.lastIndex;
|
|
490
|
-
}
|
|
491
|
-
this.buffer = this.buffer.slice(lastIndex);
|
|
492
|
-
if (end) {
|
|
493
|
-
const rest = this.buffer.trim();
|
|
494
|
-
if (rest) this.enqueue(counter, rest);
|
|
495
|
-
this.buffer = "";
|
|
496
|
-
}
|
|
497
|
-
}
|
|
498
794
|
enqueue(counter, text) {
|
|
499
795
|
this.queue.push({ counter, text });
|
|
500
796
|
this.pump();
|
|
@@ -539,8 +835,8 @@ var OpenaiTTS = class extends TTS {
|
|
|
539
835
|
this.abortControllers.add(controller);
|
|
540
836
|
const response = this.openai.audio.speech.create(
|
|
541
837
|
{
|
|
542
|
-
model: this.options.model ||
|
|
543
|
-
voice: this.options.voice ||
|
|
838
|
+
model: this.options.model || DEFAULT_MODEL4,
|
|
839
|
+
voice: this.options.voice || DEFAULT_VOICE2,
|
|
544
840
|
input: item.text,
|
|
545
841
|
response_format: "pcm",
|
|
546
842
|
...this.options.instructions ? { instructions: this.options.instructions } : {},
|
|
@@ -579,8 +875,8 @@ var OpenaiTTS = class extends TTS {
|
|
|
579
875
|
if (!response?.body) return;
|
|
580
876
|
if (item.counter !== this.counter) return;
|
|
581
877
|
this.log(`Synthesizing: "${item.text}"`);
|
|
582
|
-
const resampler = new
|
|
583
|
-
|
|
878
|
+
const resampler = new Pcm16Resampler3(
|
|
879
|
+
OPENAI_SAMPLE_RATE3,
|
|
584
880
|
OUTPUT_SAMPLE_RATE
|
|
585
881
|
);
|
|
586
882
|
const reader = response.body.getReader();
|
|
@@ -601,6 +897,7 @@ var OpenaiTTS = class extends TTS {
|
|
|
601
897
|
};
|
|
602
898
|
export {
|
|
603
899
|
OpenaiAgent,
|
|
900
|
+
OpenaiRealtime,
|
|
604
901
|
OpenaiSTT,
|
|
605
902
|
OpenaiTTS
|
|
606
903
|
};
|