@micdrop/openai 1.7.0 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +66 -25
- package/dist/index.d.ts +66 -25
- package/dist/index.js +443 -417
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +447 -420
- package/dist/index.mjs.map +1 -1
- package/package.json +3 -3
package/dist/index.mjs
CHANGED
|
@@ -6,19 +6,69 @@ var DEFAULT_MODEL = "gpt-4o";
|
|
|
6
6
|
var DEFAULT_MAX_STEPS = 5;
|
|
7
7
|
var DEFAULT_MAX_RETRIES = 3;
|
|
8
8
|
var DEFAULT_RETRY_DELAY = 1e3;
|
|
9
|
+
var StepInterruptedError = class extends Error {
|
|
10
|
+
constructor(outcome, reason) {
|
|
11
|
+
super("Answer interrupted");
|
|
12
|
+
this.outcome = outcome;
|
|
13
|
+
this.reason = reason;
|
|
14
|
+
}
|
|
15
|
+
};
|
|
9
16
|
var OpenaiAgent = class extends Agent {
|
|
10
17
|
constructor(options) {
|
|
11
18
|
super(options);
|
|
12
19
|
this.openai = "openai" in options ? options.openai : new OpenAI({ apiKey: options.apiKey });
|
|
13
20
|
}
|
|
14
|
-
async generateAnswer(stream,
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
21
|
+
async generateAnswer(stream, signal) {
|
|
22
|
+
const maxSteps = this.options.maxSteps || DEFAULT_MAX_STEPS;
|
|
23
|
+
const maxRetry = this.options.maxRetry ?? DEFAULT_MAX_RETRIES;
|
|
24
|
+
for (let step = 0; step < maxSteps; step++) {
|
|
25
|
+
let outcome;
|
|
26
|
+
for (let tryCount = 0; !outcome; tryCount++) {
|
|
27
|
+
try {
|
|
28
|
+
outcome = await this.generateStep(stream, signal);
|
|
29
|
+
} catch (error) {
|
|
30
|
+
if (signal.aborted) return;
|
|
31
|
+
if (error instanceof StepInterruptedError) {
|
|
32
|
+
console.error("[OpenaiAgent] Answer interrupted:", error.reason);
|
|
33
|
+
outcome = error.outcome;
|
|
34
|
+
break;
|
|
35
|
+
}
|
|
36
|
+
console.error("[OpenaiAgent] Error answering:", error);
|
|
37
|
+
if (tryCount >= maxRetry) {
|
|
38
|
+
this.log("Max retries reached, giving up");
|
|
39
|
+
this.emit("Failed");
|
|
40
|
+
return;
|
|
41
|
+
}
|
|
42
|
+
await new Promise(
|
|
43
|
+
(resolve) => setTimeout(resolve, this.options.retryDelay ?? DEFAULT_RETRY_DELAY)
|
|
44
|
+
);
|
|
45
|
+
if (signal.aborted) return;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
if (signal.aborted) return;
|
|
49
|
+
if (outcome.recorded || outcome.skipAnswer) return;
|
|
50
|
+
if (outcome.text) {
|
|
51
|
+
const { message, metadata } = this.extract(outcome.text);
|
|
52
|
+
this.addAssistantMessage(message, metadata);
|
|
53
|
+
return;
|
|
54
|
+
}
|
|
18
55
|
}
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
56
|
+
console.error("[OpenaiAgent] Max steps reached");
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Runs one request, writing its text to the stream and running its tools.
|
|
60
|
+
*
|
|
61
|
+
* It throws the error of the request when it failed before producing
|
|
62
|
+
* anything, which can then be retried, and a {@link StepInterruptedError}
|
|
63
|
+
* once something was written or done.
|
|
64
|
+
*/
|
|
65
|
+
async generateStep(stream, signal) {
|
|
66
|
+
const outcome = {
|
|
67
|
+
text: "",
|
|
68
|
+
recorded: false,
|
|
69
|
+
skipAnswer: false
|
|
70
|
+
};
|
|
71
|
+
let produced = false;
|
|
22
72
|
let extracting = false;
|
|
23
73
|
const extractOptions = this.getExtractOptions();
|
|
24
74
|
try {
|
|
@@ -32,14 +82,14 @@ var OpenaiAgent = class extends Agent {
|
|
|
32
82
|
},
|
|
33
83
|
{ signal }
|
|
34
84
|
);
|
|
35
|
-
let skipAnswer = false;
|
|
36
|
-
let hasAnswer = false;
|
|
37
85
|
for await (const event of response) {
|
|
38
|
-
if (
|
|
86
|
+
if (signal.aborted) return outcome;
|
|
39
87
|
switch (event.type) {
|
|
40
88
|
case "response.output_text.delta":
|
|
41
|
-
if (
|
|
89
|
+
if (outcome.recorded) break;
|
|
42
90
|
this.log(`Answer chunk: "${event.delta}"`);
|
|
91
|
+
outcome.text += event.delta;
|
|
92
|
+
produced = true;
|
|
43
93
|
if (extractOptions) {
|
|
44
94
|
if (!extracting) {
|
|
45
95
|
const startTagIndex = event.delta.indexOf(
|
|
@@ -57,15 +107,17 @@ var OpenaiAgent = class extends Agent {
|
|
|
57
107
|
}
|
|
58
108
|
stream.write(event.delta);
|
|
59
109
|
break;
|
|
60
|
-
case "response.output_text.done":
|
|
61
|
-
if (
|
|
110
|
+
case "response.output_text.done": {
|
|
111
|
+
if (outcome.recorded) break;
|
|
62
112
|
const { message, metadata } = this.extract(event.text);
|
|
63
113
|
this.addAssistantMessage(message, metadata);
|
|
64
|
-
|
|
114
|
+
outcome.recorded = true;
|
|
65
115
|
break;
|
|
116
|
+
}
|
|
66
117
|
case "response.output_item.done":
|
|
67
118
|
if (event.item.type === "function_call") {
|
|
68
119
|
const toolCall = event.item;
|
|
120
|
+
produced = true;
|
|
69
121
|
const result = await this.executeTool({
|
|
70
122
|
role: "tool_call",
|
|
71
123
|
toolCallId: toolCall.call_id,
|
|
@@ -73,7 +125,7 @@ var OpenaiAgent = class extends Agent {
|
|
|
73
125
|
parameters: toolCall.arguments
|
|
74
126
|
});
|
|
75
127
|
if (result.skipAnswer) {
|
|
76
|
-
skipAnswer = true;
|
|
128
|
+
outcome.skipAnswer = true;
|
|
77
129
|
}
|
|
78
130
|
}
|
|
79
131
|
break;
|
|
@@ -81,23 +133,12 @@ var OpenaiAgent = class extends Agent {
|
|
|
81
133
|
break;
|
|
82
134
|
}
|
|
83
135
|
}
|
|
84
|
-
|
|
85
|
-
if (!skipAnswer && !hasAnswer) {
|
|
86
|
-
await this.generateAnswer(stream, stepCount + 1);
|
|
87
|
-
}
|
|
88
|
-
this.abortController = void 0;
|
|
136
|
+
return outcome;
|
|
89
137
|
} catch (error) {
|
|
90
|
-
if (
|
|
91
|
-
|
|
92
|
-
if (tryCount < (this.options.maxRetry ?? DEFAULT_MAX_RETRIES)) {
|
|
93
|
-
await new Promise(
|
|
94
|
-
(resolve) => setTimeout(resolve, this.options.retryDelay ?? DEFAULT_RETRY_DELAY)
|
|
95
|
-
);
|
|
96
|
-
await this.generateAnswer(stream, stepCount, tryCount + 1);
|
|
97
|
-
} else {
|
|
98
|
-
this.log("Max retries reached, giving up");
|
|
99
|
-
this.emit("Failed");
|
|
138
|
+
if (produced && !signal.aborted) {
|
|
139
|
+
throw new StepInterruptedError(outcome, error);
|
|
100
140
|
}
|
|
141
|
+
throw error;
|
|
101
142
|
}
|
|
102
143
|
}
|
|
103
144
|
buildInput() {
|
|
@@ -115,7 +156,8 @@ var OpenaiAgent = class extends Agent {
|
|
|
115
156
|
type: "function_call",
|
|
116
157
|
call_id: message.toolCallId,
|
|
117
158
|
name: message.toolName,
|
|
118
|
-
|
|
159
|
+
// Already the JSON the model wrote
|
|
160
|
+
arguments: message.parameters
|
|
119
161
|
};
|
|
120
162
|
case "tool_result":
|
|
121
163
|
return {
|
|
@@ -137,12 +179,6 @@ var OpenaiAgent = class extends Agent {
|
|
|
137
179
|
parameters: toJSONSchema(tool.inputSchema || z.object())
|
|
138
180
|
}));
|
|
139
181
|
}
|
|
140
|
-
cancel() {
|
|
141
|
-
if (!this.abortController) return;
|
|
142
|
-
this.log("Cancel");
|
|
143
|
-
this.abortController.abort();
|
|
144
|
-
this.abortController = void 0;
|
|
145
|
-
}
|
|
146
182
|
};
|
|
147
183
|
|
|
148
184
|
// src/OpenaiClassifier.ts
|
|
@@ -248,11 +284,29 @@ function buildQuestion(name, question) {
|
|
|
248
284
|
// src/OpenaiRealtime.ts
|
|
249
285
|
import {
|
|
250
286
|
Pcm16Resampler,
|
|
251
|
-
Realtime
|
|
287
|
+
Realtime,
|
|
288
|
+
ReconnectingSocket
|
|
252
289
|
} from "@micdrop/server";
|
|
253
|
-
import WebSocket from "ws";
|
|
254
290
|
import z2, { toJSONSchema as toJSONSchema2 } from "zod";
|
|
255
291
|
|
|
292
|
+
// src/session.ts
|
|
293
|
+
var sessionHandshake = {
|
|
294
|
+
read(data) {
|
|
295
|
+
const message = parseMessage(data);
|
|
296
|
+
if (message?.type === "error") {
|
|
297
|
+
throw new Error(message.error?.message || "Session refused");
|
|
298
|
+
}
|
|
299
|
+
return message?.type === "session.created" || message?.type === "transcription_session.created";
|
|
300
|
+
}
|
|
301
|
+
};
|
|
302
|
+
function parseMessage(data) {
|
|
303
|
+
try {
|
|
304
|
+
return JSON.parse(data.toString());
|
|
305
|
+
} catch {
|
|
306
|
+
return void 0;
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
|
|
256
310
|
// src/transcription.ts
|
|
257
311
|
function transcriptionLanguage(model, language) {
|
|
258
312
|
if (!language) return {};
|
|
@@ -269,36 +323,56 @@ var DEFAULT_TRANSCRIPTION_MODEL = "gpt-4o-transcribe";
|
|
|
269
323
|
var SAMPLE_RATE = 16e3;
|
|
270
324
|
var OPENAI_SAMPLE_RATE = 24e3;
|
|
271
325
|
var OPENAI_BYTES_PER_MS = OPENAI_SAMPLE_RATE * 2 / 1e3;
|
|
272
|
-
var
|
|
273
|
-
var
|
|
274
|
-
var DEFAULT_MAX_RETRY = 3;
|
|
326
|
+
var MAX_PENDING_EVENTS = 5e3;
|
|
327
|
+
var REQUEST_METADATA_KEY = "micdrop_request";
|
|
275
328
|
var OpenaiRealtime = class extends Realtime {
|
|
276
329
|
constructor(options) {
|
|
277
330
|
super(options);
|
|
278
|
-
this.
|
|
279
|
-
this.everConnected = false;
|
|
280
|
-
this.destroyed = false;
|
|
331
|
+
this.connection = "connecting";
|
|
281
332
|
// Events sent while the connection opens
|
|
282
333
|
this.pending = [];
|
|
283
|
-
this.
|
|
334
|
+
this.requestCount = 0;
|
|
284
335
|
this.inputResampler = new Pcm16Resampler(SAMPLE_RATE, OPENAI_SAMPLE_RATE);
|
|
285
336
|
this.outputResampler = new Pcm16Resampler(OPENAI_SAMPLE_RATE, SAMPLE_RATE);
|
|
286
337
|
// Bumped by every interruption, so late responses can tell they are stale
|
|
287
338
|
this.generation = 0;
|
|
288
|
-
|
|
339
|
+
// Responses asked for and not created yet, by the id sent as their metadata
|
|
340
|
+
// and as the event id of their request, which an error names
|
|
341
|
+
this.requested = /* @__PURE__ */ new Map();
|
|
289
342
|
this.ignoredResponses = /* @__PURE__ */ new Set();
|
|
290
343
|
this.audioStartedAt = 0;
|
|
291
344
|
this.audioBytes = 0;
|
|
292
|
-
|
|
345
|
+
const model = options.model || DEFAULT_MODEL3;
|
|
346
|
+
this.socket = new ReconnectingSocket({
|
|
347
|
+
url: `wss://api.openai.com/v1/realtime?model=${encodeURIComponent(model)}`,
|
|
348
|
+
clientOptions: {
|
|
349
|
+
headers: { Authorization: `Bearer ${options.apiKey}` }
|
|
350
|
+
},
|
|
351
|
+
connectionTimeout: options.connectionTimeout,
|
|
352
|
+
retryDelay: options.retryDelay,
|
|
353
|
+
maxRetry: options.maxRetry,
|
|
354
|
+
handshake: sessionHandshake,
|
|
355
|
+
log: (...message) => this.log(...message)
|
|
356
|
+
});
|
|
357
|
+
this.socket.on("Open", ({ reconnected }) => this.onOpen(reconnected));
|
|
358
|
+
this.socket.on("Close", () => {
|
|
359
|
+
this.connection = "reconnecting";
|
|
360
|
+
this.resetSession();
|
|
361
|
+
});
|
|
362
|
+
this.socket.on("Failed", () => {
|
|
363
|
+
this.connection = "closed";
|
|
364
|
+
this.pending = [];
|
|
365
|
+
this.resetSession();
|
|
366
|
+
this.emit("Failed");
|
|
367
|
+
});
|
|
368
|
+
this.socket.on("Message", (data) => this.handleMessage(parseMessage(data)));
|
|
369
|
+
this.socket.connect();
|
|
293
370
|
}
|
|
294
371
|
destroy() {
|
|
295
|
-
this.destroyed = true;
|
|
296
372
|
super.destroy();
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
}
|
|
301
|
-
this.closeSocket();
|
|
373
|
+
this.connection = "closed";
|
|
374
|
+
this.pending = [];
|
|
375
|
+
this.socket.destroy();
|
|
302
376
|
}
|
|
303
377
|
openTurn() {
|
|
304
378
|
this.inputResampler.reset();
|
|
@@ -350,25 +424,49 @@ var OpenaiRealtime = class extends Realtime {
|
|
|
350
424
|
this.audioItemId = void 0;
|
|
351
425
|
}
|
|
352
426
|
updateTools() {
|
|
353
|
-
if (this.connected) this.sendSessionUpdate();
|
|
427
|
+
if (this.connection === "connected") this.sendSessionUpdate();
|
|
354
428
|
}
|
|
355
429
|
injectMessage(role, text) {
|
|
356
|
-
if (
|
|
430
|
+
if (this.connection === "reconnecting") return;
|
|
357
431
|
this.send(this.messageItem(role, text));
|
|
358
432
|
}
|
|
433
|
+
/**
|
|
434
|
+
* Asks for a response, tagged so its `response.created` tells which request
|
|
435
|
+
* it answers. Matching them by order alone would shift every later response
|
|
436
|
+
* onto the wrong request as soon as one is refused.
|
|
437
|
+
*/
|
|
359
438
|
createResponse(response) {
|
|
360
|
-
this.
|
|
361
|
-
|
|
362
|
-
|
|
439
|
+
const id = `micdrop-response-${++this.requestCount}`;
|
|
440
|
+
const generation = this.generation;
|
|
441
|
+
const accepted = this.send({
|
|
442
|
+
type: "response.create",
|
|
443
|
+
// Errors caused by this event carry its id
|
|
444
|
+
event_id: id,
|
|
445
|
+
response: { ...response, metadata: { [REQUEST_METADATA_KEY]: id } }
|
|
363
446
|
});
|
|
364
|
-
|
|
447
|
+
if (!accepted) {
|
|
448
|
+
queueMicrotask(() => {
|
|
449
|
+
if (generation === this.generation) this.endAnswer();
|
|
450
|
+
});
|
|
451
|
+
return;
|
|
452
|
+
}
|
|
453
|
+
this.requested.set(id, { generation, outOfBand: !!response });
|
|
365
454
|
}
|
|
455
|
+
/**
|
|
456
|
+
* Sends an event, or holds it until the connection opens.
|
|
457
|
+
* @returns False when it was dropped, the connection being given up on
|
|
458
|
+
*/
|
|
366
459
|
send(event) {
|
|
367
|
-
if (this.
|
|
368
|
-
this.socket.send(JSON.stringify(event));
|
|
369
|
-
} else {
|
|
370
|
-
this.pending.push(event);
|
|
460
|
+
if (this.connection === "connected") {
|
|
461
|
+
return this.socket.send(JSON.stringify(event));
|
|
371
462
|
}
|
|
463
|
+
if (this.connection === "closed") return false;
|
|
464
|
+
if (this.pending.length >= MAX_PENDING_EVENTS) {
|
|
465
|
+
this.log("Too many events waiting for the connection, dropped");
|
|
466
|
+
return false;
|
|
467
|
+
}
|
|
468
|
+
this.pending.push(event);
|
|
469
|
+
return true;
|
|
372
470
|
}
|
|
373
471
|
messageItem(role, text) {
|
|
374
472
|
return {
|
|
@@ -387,7 +485,7 @@ var OpenaiRealtime = class extends Realtime {
|
|
|
387
485
|
}
|
|
388
486
|
sendSessionUpdate() {
|
|
389
487
|
const transcriptionModel = this.options.transcriptionModel || DEFAULT_TRANSCRIPTION_MODEL;
|
|
390
|
-
this.socket
|
|
488
|
+
this.socket.send(
|
|
391
489
|
JSON.stringify({
|
|
392
490
|
type: "session.update",
|
|
393
491
|
session: {
|
|
@@ -424,111 +522,50 @@ var OpenaiRealtime = class extends Realtime {
|
|
|
424
522
|
})
|
|
425
523
|
);
|
|
426
524
|
}
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
const
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
const timeout = setTimeout(() => {
|
|
442
|
-
this.log("Connection timeout");
|
|
443
|
-
this.closeSocket();
|
|
444
|
-
reject(new Error("WebSocket connection timeout"));
|
|
445
|
-
}, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT);
|
|
446
|
-
socket.on("open", () => {
|
|
447
|
-
clearTimeout(timeout);
|
|
448
|
-
this.log("Connection opened");
|
|
449
|
-
this.sendSessionUpdate();
|
|
450
|
-
if (this.everConnected) {
|
|
451
|
-
for (const message of this.conversation) {
|
|
452
|
-
if (message.role !== "user" && message.role !== "assistant") {
|
|
453
|
-
continue;
|
|
454
|
-
}
|
|
455
|
-
socket.send(
|
|
456
|
-
JSON.stringify(this.messageItem(message.role, message.content))
|
|
457
|
-
);
|
|
458
|
-
}
|
|
459
|
-
}
|
|
460
|
-
this.connected = true;
|
|
461
|
-
this.everConnected = true;
|
|
462
|
-
this.retryCount = 0;
|
|
463
|
-
const pending = this.pending;
|
|
464
|
-
this.pending = [];
|
|
465
|
-
pending.forEach((event) => this.send(event));
|
|
466
|
-
resolve();
|
|
467
|
-
});
|
|
468
|
-
socket.on("error", (error) => {
|
|
469
|
-
clearTimeout(timeout);
|
|
470
|
-
this.log("WebSocket error:", error);
|
|
471
|
-
reject(new Error("WebSocket connection error"));
|
|
472
|
-
});
|
|
473
|
-
socket.on("close", (code, reason) => {
|
|
474
|
-
clearTimeout(timeout);
|
|
475
|
-
const wasConnected = this.connected;
|
|
476
|
-
this.closeSocket();
|
|
477
|
-
if (this.destroyed) return;
|
|
478
|
-
this.log("Connection closed", { code, reason: reason.toString() });
|
|
479
|
-
if (wasConnected) this.reconnect();
|
|
480
|
-
});
|
|
481
|
-
socket.on("message", (data) => {
|
|
482
|
-
this.handleMessage(JSON.parse(data.toString()));
|
|
483
|
-
});
|
|
484
|
-
});
|
|
485
|
-
}
|
|
486
|
-
closeSocket() {
|
|
487
|
-
const socket = this.socket;
|
|
488
|
-
this.socket = void 0;
|
|
489
|
-
this.connected = false;
|
|
525
|
+
/**
|
|
526
|
+
* Sets the session up, then sends what waited for it. A new connection
|
|
527
|
+
* starts from an empty conversation, so a reconnection sends it again first.
|
|
528
|
+
*/
|
|
529
|
+
onOpen(reconnected) {
|
|
530
|
+
this.connection = "connected";
|
|
531
|
+
this.sendSessionUpdate();
|
|
532
|
+
if (reconnected) {
|
|
533
|
+
for (const message of this.conversation) {
|
|
534
|
+
if (message.role !== "user" && message.role !== "assistant") continue;
|
|
535
|
+
this.send(this.messageItem(message.role, message.content));
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
const pending = this.pending;
|
|
490
539
|
this.pending = [];
|
|
491
|
-
|
|
540
|
+
pending.forEach((event) => this.send(event));
|
|
541
|
+
}
|
|
542
|
+
/**
|
|
543
|
+
* Forgets what belonged to the session that is gone: its responses, and the
|
|
544
|
+
* answer in progress, which is lost with it.
|
|
545
|
+
*/
|
|
546
|
+
resetSession() {
|
|
547
|
+
this.requested.clear();
|
|
492
548
|
this.response = void 0;
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
socket.close(1e3);
|
|
499
|
-
}
|
|
549
|
+
this.ignoredResponses.clear();
|
|
550
|
+
this.audioItemId = void 0;
|
|
551
|
+
this.audioBytes = 0;
|
|
552
|
+
this.outputResampler.reset();
|
|
553
|
+
this.generation++;
|
|
500
554
|
this.endAnswer();
|
|
501
555
|
}
|
|
502
|
-
reconnect() {
|
|
503
|
-
if (this.destroyed) return;
|
|
504
|
-
this.retryCount++;
|
|
505
|
-
if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY)) {
|
|
506
|
-
this.log("Max retries reached, giving up");
|
|
507
|
-
this.emit("Failed");
|
|
508
|
-
return;
|
|
509
|
-
}
|
|
510
|
-
this.log("Reconnecting...");
|
|
511
|
-
this.reconnectTimeout = setTimeout(() => {
|
|
512
|
-
this.reconnectTimeout = void 0;
|
|
513
|
-
this.connect();
|
|
514
|
-
}, this.options.retryDelay ?? DEFAULT_RETRY_DELAY2);
|
|
515
|
-
}
|
|
516
556
|
isIgnored(responseId) {
|
|
517
557
|
return !responseId || this.ignoredResponses.has(responseId) || this.response?.id !== responseId;
|
|
518
558
|
}
|
|
519
559
|
handleMessage(message) {
|
|
520
|
-
switch (message
|
|
560
|
+
switch (message?.type) {
|
|
521
561
|
case "response.created": {
|
|
522
|
-
const requested = this.requested.shift() ?? {
|
|
523
|
-
generation: this.generation,
|
|
524
|
-
outOfBand: false
|
|
525
|
-
};
|
|
526
562
|
const id = message.response.id;
|
|
527
|
-
|
|
563
|
+
const requestId = message.response.metadata?.[REQUEST_METADATA_KEY];
|
|
564
|
+
const requested = requestId ? this.requested.get(requestId) : { generation: this.generation, outOfBand: false };
|
|
565
|
+
if (requestId) this.requested.delete(requestId);
|
|
566
|
+
if (!requested || requested.generation !== this.generation) {
|
|
528
567
|
this.ignoredResponses.add(id);
|
|
529
|
-
this.
|
|
530
|
-
JSON.stringify({ type: "response.cancel", response_id: id })
|
|
531
|
-
);
|
|
568
|
+
this.send({ type: "response.cancel", response_id: id });
|
|
532
569
|
break;
|
|
533
570
|
}
|
|
534
571
|
this.response = { id, ...requested };
|
|
@@ -566,10 +603,18 @@ var OpenaiRealtime = class extends Realtime {
|
|
|
566
603
|
case "response.done":
|
|
567
604
|
this.onResponseDone(message.response);
|
|
568
605
|
break;
|
|
569
|
-
case "error":
|
|
606
|
+
case "error": {
|
|
570
607
|
if (message.error?.code === "response_cancel_not_active") break;
|
|
571
608
|
this.log("Error:", message.error);
|
|
609
|
+
const requestId = message.error?.event_id;
|
|
610
|
+
const requested = this.requested.get(requestId);
|
|
611
|
+
if (!requested) break;
|
|
612
|
+
this.requested.delete(requestId);
|
|
613
|
+
if (requested.generation === this.generation && !this.response) {
|
|
614
|
+
this.endAnswer();
|
|
615
|
+
}
|
|
572
616
|
break;
|
|
617
|
+
}
|
|
573
618
|
default:
|
|
574
619
|
break;
|
|
575
620
|
}
|
|
@@ -618,70 +663,79 @@ var OpenaiRealtime = class extends Realtime {
|
|
|
618
663
|
};
|
|
619
664
|
|
|
620
665
|
// src/OpenaiSTT.ts
|
|
621
|
-
import { Pcm16Resampler as Pcm16Resampler2, STT } from "@micdrop/server";
|
|
622
|
-
import WebSocket2 from "ws";
|
|
666
|
+
import { Pcm16Resampler as Pcm16Resampler2, ReconnectingSocket as ReconnectingSocket2, STT } from "@micdrop/server";
|
|
623
667
|
var DEFAULT_MODEL4 = "gpt-4o-transcribe";
|
|
624
668
|
var DEFAULT_LANGUAGE = "en";
|
|
625
669
|
var SAMPLE_RATE2 = 16e3;
|
|
626
670
|
var OPENAI_SAMPLE_RATE2 = 24e3;
|
|
627
|
-
var DEFAULT_CONNECTION_TIMEOUT2 = 5e3;
|
|
628
671
|
var DEFAULT_TRANSCRIPTION_TIMEOUT = 4e3;
|
|
629
|
-
var DEFAULT_RETRY_DELAY3 = 1e3;
|
|
630
|
-
var DEFAULT_MAX_RETRY2 = 3;
|
|
631
672
|
var OpenaiSTT = class extends STT {
|
|
632
673
|
constructor(options) {
|
|
633
674
|
super();
|
|
634
675
|
this.options = options;
|
|
635
|
-
this.
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
this.
|
|
641
|
-
|
|
642
|
-
|
|
676
|
+
this.utteranceCount = 0;
|
|
677
|
+
// Utterances not transcribed yet, in the order they were spoken. Each one
|
|
678
|
+
// leaves the list with the single transcript it emits.
|
|
679
|
+
this.utterances = [];
|
|
680
|
+
// Commits sent and not acknowledged yet, in the order the API handles them
|
|
681
|
+
this.unacknowledged = [];
|
|
682
|
+
this.socket = new ReconnectingSocket2({
|
|
683
|
+
url: "wss://api.openai.com/v1/realtime?intent=transcription",
|
|
684
|
+
clientOptions: {
|
|
685
|
+
headers: { Authorization: `Bearer ${this.options.apiKey}` }
|
|
686
|
+
},
|
|
687
|
+
connectionTimeout: options.connectionTimeout,
|
|
688
|
+
retryDelay: options.retryDelay,
|
|
689
|
+
maxRetry: options.maxRetry,
|
|
690
|
+
handshake: sessionHandshake,
|
|
691
|
+
log: (...message) => this.log(...message)
|
|
692
|
+
});
|
|
693
|
+
this.socket.on("Open", () => this.onOpen());
|
|
694
|
+
this.socket.on("Close", () => {
|
|
695
|
+
this.unacknowledged = [];
|
|
643
696
|
});
|
|
697
|
+
this.socket.on("Message", (data) => this.handleMessage(parseMessage(data)));
|
|
698
|
+
this.socket.on("Failed", () => this.onFailed());
|
|
699
|
+
this.socket.connect();
|
|
644
700
|
}
|
|
701
|
+
/**
|
|
702
|
+
* Transcribes one utterance, and emits its transcript once, possibly empty
|
|
703
|
+
* when none comes in time.
|
|
704
|
+
*
|
|
705
|
+
* After the connection was given up on, the next utterance tries it again.
|
|
706
|
+
*/
|
|
645
707
|
transcribe(audioStream) {
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
708
|
+
const utterance = {
|
|
709
|
+
id: `micdrop-stt-${++this.utteranceCount}`,
|
|
710
|
+
status: "streaming",
|
|
711
|
+
chunks: [],
|
|
712
|
+
resampler: new Pcm16Resampler2(SAMPLE_RATE2, OPENAI_SAMPLE_RATE2)
|
|
713
|
+
};
|
|
714
|
+
this.utterances.push(utterance);
|
|
715
|
+
this.socket.connect();
|
|
716
|
+
audioStream.on("data", (chunk) => {
|
|
717
|
+
if (!this.utterances.includes(utterance)) return;
|
|
718
|
+
utterance.chunks.push(chunk);
|
|
719
|
+
this.sendAudioChunk(utterance, chunk);
|
|
652
720
|
});
|
|
653
|
-
audioStream.on("end",
|
|
654
|
-
|
|
655
|
-
if (
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
this.audioChunksPending.length = 0;
|
|
662
|
-
}, this.options.transcriptionTimeout || DEFAULT_TRANSCRIPTION_TIMEOUT);
|
|
721
|
+
audioStream.on("end", () => {
|
|
722
|
+
if (!this.utterances.includes(utterance)) return;
|
|
723
|
+
if (utterance.chunks.length === 0) {
|
|
724
|
+
this.remove(utterance);
|
|
725
|
+
return;
|
|
726
|
+
}
|
|
727
|
+
utterance.status = "committed";
|
|
728
|
+
this.commitAudio(utterance);
|
|
663
729
|
});
|
|
664
730
|
}
|
|
665
731
|
destroy() {
|
|
666
732
|
super.destroy();
|
|
667
|
-
this.
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
}
|
|
672
|
-
if (this.transcriptionTimeout) {
|
|
673
|
-
clearTimeout(this.transcriptionTimeout);
|
|
674
|
-
this.transcriptionTimeout = void 0;
|
|
675
|
-
}
|
|
676
|
-
if (this.connectionTimeout) {
|
|
677
|
-
clearTimeout(this.connectionTimeout);
|
|
678
|
-
this.connectionTimeout = void 0;
|
|
679
|
-
}
|
|
680
|
-
if (this.socket) closeSocket(this.socket);
|
|
681
|
-
this.socket = void 0;
|
|
733
|
+
this.socket.destroy();
|
|
734
|
+
this.utterances.forEach((utterance) => clearTimeout(utterance.timer));
|
|
735
|
+
this.utterances = [];
|
|
736
|
+
this.unacknowledged = [];
|
|
682
737
|
}
|
|
683
738
|
sendSessionUpdate() {
|
|
684
|
-
if (!this.socket) return;
|
|
685
739
|
const model = this.options.model || DEFAULT_MODEL4;
|
|
686
740
|
this.socket.send(
|
|
687
741
|
JSON.stringify({
|
|
@@ -715,56 +769,35 @@ var OpenaiSTT = class extends STT {
|
|
|
715
769
|
})
|
|
716
770
|
);
|
|
717
771
|
}
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
this.connectionTimeout = timeout;
|
|
736
|
-
socket.addEventListener("open", () => {
|
|
737
|
-
clearTimeout(timeout);
|
|
738
|
-
this.sendSessionUpdate();
|
|
739
|
-
this.log("Connection opened");
|
|
740
|
-
resolve();
|
|
741
|
-
});
|
|
742
|
-
socket.addEventListener("error", (error) => {
|
|
743
|
-
clearTimeout(timeout);
|
|
744
|
-
this.log("WebSocket error:", error);
|
|
745
|
-
reject(new Error("WebSocket connection error"));
|
|
746
|
-
});
|
|
747
|
-
socket.addEventListener(
|
|
748
|
-
"close",
|
|
749
|
-
({ code, reason }) => {
|
|
750
|
-
clearTimeout(timeout);
|
|
751
|
-
this.socket?.removeAllListeners();
|
|
752
|
-
this.socket = void 0;
|
|
753
|
-
if (code !== 1e3) {
|
|
754
|
-
this.reconnect();
|
|
755
|
-
} else {
|
|
756
|
-
this.log("Connection closed", { code, reason });
|
|
757
|
-
}
|
|
758
|
-
}
|
|
759
|
-
);
|
|
760
|
-
socket.addEventListener("message", (event) => {
|
|
761
|
-
this.handleMessage(JSON.parse(event.data.toString()));
|
|
762
|
-
});
|
|
763
|
-
});
|
|
772
|
+
/**
|
|
773
|
+
* Configures the session, then sends again what a previous connection did
|
|
774
|
+
* not get transcribed. The OpenAI buffer of a new session is empty, so each
|
|
775
|
+
* utterance is replayed through a fresh resampler, which reproduces the exact
|
|
776
|
+
* same stream, and committed again if it was.
|
|
777
|
+
*/
|
|
778
|
+
onOpen() {
|
|
779
|
+
this.sendSessionUpdate();
|
|
780
|
+
this.unacknowledged = [];
|
|
781
|
+
if (this.utterances.length > 0) this.log("Sending audio chunks again");
|
|
782
|
+
for (const utterance of this.utterances) {
|
|
783
|
+
if (utterance.transcript !== void 0) continue;
|
|
784
|
+
utterance.itemId = void 0;
|
|
785
|
+
utterance.resampler = new Pcm16Resampler2(SAMPLE_RATE2, OPENAI_SAMPLE_RATE2);
|
|
786
|
+
utterance.chunks.forEach((chunk) => this.sendAudioChunk(utterance, chunk));
|
|
787
|
+
if (utterance.status === "committed") this.commitAudio(utterance);
|
|
788
|
+
}
|
|
764
789
|
}
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
const
|
|
790
|
+
/** Every retry failed: the audio waiting for a transcript is handed back */
|
|
791
|
+
onFailed() {
|
|
792
|
+
const chunks = this.utterances.flatMap((utterance) => utterance.chunks);
|
|
793
|
+
this.utterances.forEach((utterance) => clearTimeout(utterance.timer));
|
|
794
|
+
this.utterances = [];
|
|
795
|
+
this.unacknowledged = [];
|
|
796
|
+
this.emit("Failed", chunks);
|
|
797
|
+
}
|
|
798
|
+
sendAudioChunk(utterance, chunk) {
|
|
799
|
+
if (!this.socket.isOpen) return;
|
|
800
|
+
const resampled = utterance.resampler.process(chunk);
|
|
768
801
|
if (resampled.length === 0) return;
|
|
769
802
|
this.socket.send(
|
|
770
803
|
JSON.stringify({
|
|
@@ -773,91 +806,108 @@ var OpenaiSTT = class extends STT {
|
|
|
773
806
|
})
|
|
774
807
|
);
|
|
775
808
|
}
|
|
809
|
+
commitAudio(utterance) {
|
|
810
|
+
if (!this.socket.isOpen) return;
|
|
811
|
+
this.socket.send(
|
|
812
|
+
JSON.stringify({
|
|
813
|
+
type: "input_audio_buffer.commit",
|
|
814
|
+
event_id: utterance.id
|
|
815
|
+
})
|
|
816
|
+
);
|
|
817
|
+
this.unacknowledged.push(utterance);
|
|
818
|
+
this.log("Committed audio buffer");
|
|
819
|
+
clearTimeout(utterance.timer);
|
|
820
|
+
utterance.timer = setTimeout(() => {
|
|
821
|
+
utterance.timer = void 0;
|
|
822
|
+
this.log("Transcription timeout");
|
|
823
|
+
this.finish(utterance, "");
|
|
824
|
+
}, this.options.transcriptionTimeout || DEFAULT_TRANSCRIPTION_TIMEOUT);
|
|
825
|
+
}
|
|
826
|
+
/**
|
|
827
|
+
* Settles the transcript of an utterance, the only one it gets.
|
|
828
|
+
*
|
|
829
|
+
* The API transcribes each utterance on its own and may finish a short one
|
|
830
|
+
* before a long one spoken earlier, so transcripts go out in the order the
|
|
831
|
+
* utterances were spoken.
|
|
832
|
+
*/
|
|
833
|
+
finish(utterance, transcript) {
|
|
834
|
+
if (!this.utterances.includes(utterance)) return;
|
|
835
|
+
if (utterance.transcript !== void 0) return;
|
|
836
|
+
clearTimeout(utterance.timer);
|
|
837
|
+
utterance.timer = void 0;
|
|
838
|
+
utterance.transcript = transcript;
|
|
839
|
+
while (this.utterances[0]?.transcript !== void 0) {
|
|
840
|
+
const first = this.utterances[0];
|
|
841
|
+
this.remove(first);
|
|
842
|
+
this.emit("Transcript", first.transcript);
|
|
843
|
+
}
|
|
844
|
+
}
|
|
845
|
+
remove(utterance) {
|
|
846
|
+
clearTimeout(utterance.timer);
|
|
847
|
+
utterance.timer = void 0;
|
|
848
|
+
this.utterances = this.utterances.filter((other) => other !== utterance);
|
|
849
|
+
this.unacknowledged = this.unacknowledged.filter(
|
|
850
|
+
(other) => other !== utterance
|
|
851
|
+
);
|
|
852
|
+
}
|
|
853
|
+
/**
|
|
854
|
+
* The utterance a transcript event is about. The API names the item, and
|
|
855
|
+
* the item of each utterance is known from the acknowledgement of its
|
|
856
|
+
* commit. A transcript of an utterance that timed out matches none.
|
|
857
|
+
*/
|
|
858
|
+
findUtterance(itemId) {
|
|
859
|
+
if (!itemId) {
|
|
860
|
+
return this.utterances.find(
|
|
861
|
+
({ status, transcript }) => status === "committed" && transcript === void 0
|
|
862
|
+
);
|
|
863
|
+
}
|
|
864
|
+
return this.utterances.find((utterance) => utterance.itemId === itemId);
|
|
865
|
+
}
|
|
776
866
|
handleMessage(message) {
|
|
777
|
-
switch (message
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
// case 'input_audio_buffer.speech_stopped':
|
|
785
|
-
// this.log('Speech stopped')
|
|
786
|
-
// break
|
|
787
|
-
// case 'conversation.item.input_audio_transcription.delta':
|
|
788
|
-
// this.log(`Received transcript delta: "${message.delta}"`)
|
|
789
|
-
// break
|
|
790
|
-
case "conversation.item.input_audio_transcription.completed":
|
|
867
|
+
switch (message?.type) {
|
|
868
|
+
case "input_audio_buffer.committed": {
|
|
869
|
+
const utterance = this.unacknowledged.shift();
|
|
870
|
+
if (utterance) utterance.itemId = message.item_id;
|
|
871
|
+
break;
|
|
872
|
+
}
|
|
873
|
+
case "conversation.item.input_audio_transcription.completed": {
|
|
791
874
|
const transcript = message.transcript || "";
|
|
792
875
|
this.log(`Received completed transcript: "${transcript}"`);
|
|
793
|
-
this.
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
this.transcriptionTimeout = void 0;
|
|
876
|
+
const utterance = this.findUtterance(message.item_id);
|
|
877
|
+
if (!utterance) {
|
|
878
|
+
this.log("Transcript of an utterance already given up on, ignored");
|
|
879
|
+
break;
|
|
798
880
|
}
|
|
881
|
+
this.finish(utterance, transcript);
|
|
799
882
|
break;
|
|
800
|
-
|
|
801
|
-
|
|
883
|
+
}
|
|
884
|
+
case "conversation.item.input_audio_transcription.failed": {
|
|
885
|
+
this.log("Transcription failed:", message.error);
|
|
886
|
+
const utterance = this.findUtterance(message.item_id);
|
|
887
|
+
if (utterance) this.finish(utterance, "");
|
|
802
888
|
break;
|
|
803
|
-
|
|
889
|
+
}
|
|
890
|
+
case "error": {
|
|
804
891
|
this.log("Error:", message.error);
|
|
892
|
+
const utterance = this.utterances.find(
|
|
893
|
+
({ id }) => id === message.error?.event_id
|
|
894
|
+
);
|
|
895
|
+
if (utterance) this.finish(utterance, "");
|
|
805
896
|
break;
|
|
897
|
+
}
|
|
806
898
|
default:
|
|
807
899
|
break;
|
|
808
900
|
}
|
|
809
901
|
}
|
|
810
|
-
reconnect() {
|
|
811
|
-
if (this.destroyed) return;
|
|
812
|
-
this.retryCount++;
|
|
813
|
-
if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY2)) {
|
|
814
|
-
this.log("Max retries reached, giving up");
|
|
815
|
-
this.emit("Failed", this.audioChunksPending);
|
|
816
|
-
return;
|
|
817
|
-
}
|
|
818
|
-
this.initPromise = new Promise((resolve) => {
|
|
819
|
-
this.log("Reconnecting...");
|
|
820
|
-
this.reconnectTimeout = setTimeout(() => {
|
|
821
|
-
this.reconnectTimeout = void 0;
|
|
822
|
-
this.initWS().then(() => {
|
|
823
|
-
if (this.audioChunksPending.length > 0) {
|
|
824
|
-
this.log("Sending audio chunks again");
|
|
825
|
-
this.resampler.reset();
|
|
826
|
-
this.audioChunksPending.forEach(
|
|
827
|
-
(chunk) => this.sendAudioChunk(chunk)
|
|
828
|
-
);
|
|
829
|
-
}
|
|
830
|
-
}).then(resolve).catch((error) => {
|
|
831
|
-
this.log("Reconnection error:", error);
|
|
832
|
-
this.reconnect();
|
|
833
|
-
});
|
|
834
|
-
}, this.options.retryDelay ?? DEFAULT_RETRY_DELAY3);
|
|
835
|
-
});
|
|
836
|
-
}
|
|
837
|
-
commitAudio() {
|
|
838
|
-
if (!this.socket) return;
|
|
839
|
-
this.socket.send(JSON.stringify({ type: "input_audio_buffer.commit" }));
|
|
840
|
-
this.log("Committed audio buffer");
|
|
841
|
-
}
|
|
842
902
|
};
|
|
843
|
-
function closeSocket(socket) {
|
|
844
|
-
socket.removeAllListeners();
|
|
845
|
-
socket.on("error", () => {
|
|
846
|
-
});
|
|
847
|
-
if (socket.readyState === WebSocket2.OPEN || socket.readyState === WebSocket2.CONNECTING) {
|
|
848
|
-
socket.close(1e3);
|
|
849
|
-
}
|
|
850
|
-
}
|
|
851
903
|
|
|
852
904
|
// src/OpenaiTTS.ts
|
|
853
905
|
import { Pcm16Resampler as Pcm16Resampler4, SentenceSplitter, TTS } from "@micdrop/server";
|
|
854
906
|
import OpenAI3 from "openai";
|
|
855
907
|
|
|
856
908
|
// src/OpenaiRealtimeSpeech.ts
|
|
857
|
-
import { Pcm16Resampler as Pcm16Resampler3 } from "@micdrop/server";
|
|
858
|
-
import WebSocket3 from "ws";
|
|
909
|
+
import { Pcm16Resampler as Pcm16Resampler3, ReconnectingSocket as ReconnectingSocket3 } from "@micdrop/server";
|
|
859
910
|
var DEFAULT_REASONING_EFFORT = "minimal";
|
|
860
|
-
var DEFAULT_CONNECTION_TIMEOUT3 = 5e3;
|
|
861
911
|
var MIN_SPEED = 0.25;
|
|
862
912
|
var MAX_SPEED = 1.5;
|
|
863
913
|
var OPENAI_SAMPLE_RATE3 = 24e3;
|
|
@@ -870,22 +920,48 @@ var OpenaiRealtimeSpeech = class {
|
|
|
870
920
|
constructor(options, log) {
|
|
871
921
|
this.options = options;
|
|
872
922
|
this.log = log;
|
|
873
|
-
this.destroyed = false;
|
|
874
923
|
this.itemCount = 0;
|
|
875
924
|
// Responses asked for and not finished, by their metadata id and by the id
|
|
876
|
-
// the API gave them.
|
|
925
|
+
// the API gave them. Finished ones leave both maps, which mutes them.
|
|
877
926
|
this.items = /* @__PURE__ */ new Map();
|
|
878
927
|
this.responses = /* @__PURE__ */ new Map();
|
|
879
|
-
|
|
928
|
+
const { model, apiKey } = options;
|
|
929
|
+
this.socket = new ReconnectingSocket3({
|
|
930
|
+
url: `wss://api.openai.com/v1/realtime?model=${encodeURIComponent(model)}`,
|
|
931
|
+
clientOptions: { headers: { Authorization: `Bearer ${apiKey}` } },
|
|
932
|
+
connectionTimeout: options.connectionTimeout,
|
|
933
|
+
handshake: sessionHandshake,
|
|
934
|
+
log
|
|
935
|
+
});
|
|
936
|
+
this.socket.on("Open", () => {
|
|
937
|
+
this.socket.send(JSON.stringify(this.sessionUpdate()));
|
|
880
938
|
});
|
|
939
|
+
this.socket.on("Close", () => {
|
|
940
|
+
for (const item of [...this.items.values()]) {
|
|
941
|
+
if (item.status === "created") {
|
|
942
|
+
this.finish(item, "failed", new Error("Connection lost"));
|
|
943
|
+
} else if (item.status === "requested") {
|
|
944
|
+
item.status = "waiting";
|
|
945
|
+
this.request(item);
|
|
946
|
+
}
|
|
947
|
+
}
|
|
948
|
+
});
|
|
949
|
+
this.socket.on("Message", (data) => {
|
|
950
|
+
try {
|
|
951
|
+
this.handleMessage(parseMessage(data));
|
|
952
|
+
} catch (error) {
|
|
953
|
+
this.log("Error handling message", error);
|
|
954
|
+
}
|
|
955
|
+
});
|
|
956
|
+
this.socket.connect();
|
|
881
957
|
}
|
|
882
958
|
synthesize(text) {
|
|
883
959
|
const item = {
|
|
884
960
|
id: `micdrop-tts-${++this.itemCount}`,
|
|
885
961
|
text,
|
|
962
|
+
status: "waiting",
|
|
886
963
|
resampler: new Pcm16Resampler3(OPENAI_SAMPLE_RATE3, OUTPUT_SAMPLE_RATE),
|
|
887
|
-
audio: []
|
|
888
|
-
done: false
|
|
964
|
+
audio: []
|
|
889
965
|
};
|
|
890
966
|
this.items.set(item.id, item);
|
|
891
967
|
this.request(item);
|
|
@@ -895,13 +971,28 @@ var OpenaiRealtimeSpeech = class {
|
|
|
895
971
|
};
|
|
896
972
|
}
|
|
897
973
|
destroy() {
|
|
898
|
-
this.
|
|
899
|
-
this.
|
|
974
|
+
this.socket.destroy();
|
|
975
|
+
for (const item of [...this.items.values()]) {
|
|
976
|
+
this.finish(item, "failed", new Error("TTS destroyed"));
|
|
977
|
+
}
|
|
900
978
|
}
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
979
|
+
/**
|
|
980
|
+
* Sends the sentence once the connection is open.
|
|
981
|
+
*
|
|
982
|
+
* A Realtime session lasts an hour at most, and the connection opens again
|
|
983
|
+
* on its own when the API closes it. After every retry failed, the next
|
|
984
|
+
* sentence starts over with a full retry budget.
|
|
985
|
+
*/
|
|
986
|
+
async request(item) {
|
|
987
|
+
this.socket.connect();
|
|
988
|
+
const open = await this.socket.ready();
|
|
989
|
+
if (item.status !== "waiting") return;
|
|
990
|
+
if (!open) {
|
|
991
|
+
this.finish(item, "failed", new Error("WebSocket connection failed"));
|
|
992
|
+
return;
|
|
993
|
+
}
|
|
994
|
+
const sent = this.socket.send(
|
|
995
|
+
JSON.stringify({
|
|
905
996
|
type: "response.create",
|
|
906
997
|
// Errors caused by this event carry its id
|
|
907
998
|
event_id: item.id,
|
|
@@ -927,8 +1018,13 @@ var OpenaiRealtimeSpeech = class {
|
|
|
927
1018
|
output_modalities: ["audio"],
|
|
928
1019
|
metadata: { micdrop_id: item.id }
|
|
929
1020
|
}
|
|
930
|
-
})
|
|
931
|
-
|
|
1021
|
+
})
|
|
1022
|
+
);
|
|
1023
|
+
if (sent) {
|
|
1024
|
+
item.status = "requested";
|
|
1025
|
+
} else {
|
|
1026
|
+
this.finish(item, "failed", new Error("WebSocket connection lost"));
|
|
1027
|
+
}
|
|
932
1028
|
}
|
|
933
1029
|
async *read(item) {
|
|
934
1030
|
while (true) {
|
|
@@ -937,21 +1033,22 @@ var OpenaiRealtimeSpeech = class {
|
|
|
937
1033
|
yield chunk;
|
|
938
1034
|
continue;
|
|
939
1035
|
}
|
|
940
|
-
if (item.
|
|
941
|
-
if (item.done) return;
|
|
1036
|
+
if (item.status === "failed") throw item.error;
|
|
1037
|
+
if (item.status === "done" || item.status === "cancelled") return;
|
|
942
1038
|
await new Promise((resolve) => item.wake = resolve);
|
|
943
1039
|
}
|
|
944
1040
|
}
|
|
945
1041
|
cancel(item) {
|
|
946
|
-
if (item.
|
|
1042
|
+
if (item.status === "created" && item.responseId) {
|
|
947
1043
|
this.send({ type: "response.cancel", response_id: item.responseId });
|
|
948
1044
|
}
|
|
949
1045
|
item.audio = [];
|
|
950
|
-
this.finish(item);
|
|
1046
|
+
this.finish(item, "cancelled");
|
|
951
1047
|
}
|
|
952
|
-
|
|
1048
|
+
/** Ends a sentence, the first end being the only one that counts */
|
|
1049
|
+
finish(item, status, error) {
|
|
953
1050
|
if (!this.items.has(item.id)) return;
|
|
954
|
-
item.
|
|
1051
|
+
item.status = status;
|
|
955
1052
|
item.error = error;
|
|
956
1053
|
this.items.delete(item.id);
|
|
957
1054
|
if (item.responseId) this.responses.delete(item.responseId);
|
|
@@ -960,9 +1057,7 @@ var OpenaiRealtimeSpeech = class {
|
|
|
960
1057
|
wake?.();
|
|
961
1058
|
}
|
|
962
1059
|
send(event) {
|
|
963
|
-
|
|
964
|
-
this.socket.send(JSON.stringify(event));
|
|
965
|
-
}
|
|
1060
|
+
this.socket.send(JSON.stringify(event));
|
|
966
1061
|
}
|
|
967
1062
|
sessionUpdate() {
|
|
968
1063
|
const { instructions, speed } = this.options;
|
|
@@ -989,80 +1084,8 @@ Delivery: ${instructions}` : READER_INSTRUCTIONS,
|
|
|
989
1084
|
}
|
|
990
1085
|
};
|
|
991
1086
|
}
|
|
992
|
-
/**
|
|
993
|
-
* Opens the session, or hands over the one already open.
|
|
994
|
-
*
|
|
995
|
-
* A Realtime session lasts an hour at most. A closed one is opened again by
|
|
996
|
-
* the next sentence rather than on a timer, so an idle agent holds no
|
|
997
|
-
* connection.
|
|
998
|
-
*/
|
|
999
|
-
connect() {
|
|
1000
|
-
if (this.destroyed) return Promise.reject(new Error("TTS destroyed"));
|
|
1001
|
-
if (this.socket?.readyState === WebSocket3.OPEN) {
|
|
1002
|
-
return Promise.resolve(this.socket);
|
|
1003
|
-
}
|
|
1004
|
-
if (this.connecting) return this.connecting;
|
|
1005
|
-
const { model, apiKey } = this.options;
|
|
1006
|
-
const connecting = new Promise((resolve, reject) => {
|
|
1007
|
-
const socket = new WebSocket3(
|
|
1008
|
-
`wss://api.openai.com/v1/realtime?model=${encodeURIComponent(model)}`,
|
|
1009
|
-
{ headers: { Authorization: `Bearer ${apiKey}` } }
|
|
1010
|
-
);
|
|
1011
|
-
this.socket = socket;
|
|
1012
|
-
const timeout = setTimeout(() => {
|
|
1013
|
-
this.log("Connection timeout");
|
|
1014
|
-
this.closeSocket();
|
|
1015
|
-
reject(new Error("WebSocket connection timeout"));
|
|
1016
|
-
}, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT3);
|
|
1017
|
-
socket.on("open", () => {
|
|
1018
|
-
clearTimeout(timeout);
|
|
1019
|
-
this.log("Connection opened");
|
|
1020
|
-
socket.send(JSON.stringify(this.sessionUpdate()));
|
|
1021
|
-
resolve(socket);
|
|
1022
|
-
});
|
|
1023
|
-
socket.on("error", (error) => {
|
|
1024
|
-
clearTimeout(timeout);
|
|
1025
|
-
this.log("WebSocket error:", error);
|
|
1026
|
-
reject(new Error("WebSocket connection error"));
|
|
1027
|
-
});
|
|
1028
|
-
socket.on("close", (code, reason) => {
|
|
1029
|
-
clearTimeout(timeout);
|
|
1030
|
-
this.log("Connection closed", { code, reason: reason.toString() });
|
|
1031
|
-
reject(new Error("WebSocket connection closed"));
|
|
1032
|
-
this.closeSocket();
|
|
1033
|
-
});
|
|
1034
|
-
socket.on("message", (data) => {
|
|
1035
|
-
try {
|
|
1036
|
-
this.handleMessage(JSON.parse(data.toString()));
|
|
1037
|
-
} catch (error) {
|
|
1038
|
-
this.log("Error handling message", error);
|
|
1039
|
-
}
|
|
1040
|
-
});
|
|
1041
|
-
});
|
|
1042
|
-
this.connecting = connecting;
|
|
1043
|
-
const settle = () => {
|
|
1044
|
-
if (this.connecting === connecting) this.connecting = void 0;
|
|
1045
|
-
};
|
|
1046
|
-
connecting.then(settle, settle);
|
|
1047
|
-
return connecting;
|
|
1048
|
-
}
|
|
1049
|
-
closeSocket() {
|
|
1050
|
-
const socket = this.socket;
|
|
1051
|
-
this.socket = void 0;
|
|
1052
|
-
this.connecting = void 0;
|
|
1053
|
-
for (const item of [...this.items.values()]) {
|
|
1054
|
-
this.finish(item, new Error("Connection lost"));
|
|
1055
|
-
}
|
|
1056
|
-
if (!socket) return;
|
|
1057
|
-
socket.removeAllListeners();
|
|
1058
|
-
socket.on("error", () => {
|
|
1059
|
-
});
|
|
1060
|
-
if (socket.readyState === WebSocket3.OPEN || socket.readyState === WebSocket3.CONNECTING) {
|
|
1061
|
-
socket.close(1e3);
|
|
1062
|
-
}
|
|
1063
|
-
}
|
|
1064
1087
|
handleMessage(message) {
|
|
1065
|
-
switch (message
|
|
1088
|
+
switch (message?.type) {
|
|
1066
1089
|
case "response.created": {
|
|
1067
1090
|
const id = message.response.id;
|
|
1068
1091
|
const item = this.items.get(message.response.metadata?.micdrop_id);
|
|
@@ -1070,6 +1093,7 @@ Delivery: ${instructions}` : READER_INSTRUCTIONS,
|
|
|
1070
1093
|
this.send({ type: "response.cancel", response_id: id });
|
|
1071
1094
|
break;
|
|
1072
1095
|
}
|
|
1096
|
+
item.status = "created";
|
|
1073
1097
|
item.responseId = id;
|
|
1074
1098
|
this.responses.set(id, item);
|
|
1075
1099
|
break;
|
|
@@ -1101,12 +1125,13 @@ Delivery: ${instructions}` : READER_INSTRUCTIONS,
|
|
|
1101
1125
|
if (response.status === "failed") {
|
|
1102
1126
|
this.finish(
|
|
1103
1127
|
item,
|
|
1128
|
+
"failed",
|
|
1104
1129
|
new Error(
|
|
1105
1130
|
`Response failed: ${JSON.stringify(response.status_details)}`
|
|
1106
1131
|
)
|
|
1107
1132
|
);
|
|
1108
1133
|
} else {
|
|
1109
|
-
this.finish(item);
|
|
1134
|
+
this.finish(item, "done");
|
|
1110
1135
|
}
|
|
1111
1136
|
break;
|
|
1112
1137
|
}
|
|
@@ -1114,7 +1139,7 @@ Delivery: ${instructions}` : READER_INSTRUCTIONS,
|
|
|
1114
1139
|
if (message.error?.code === "response_cancel_not_active") break;
|
|
1115
1140
|
this.log("Error:", message.error);
|
|
1116
1141
|
const item = this.items.get(message.error?.event_id);
|
|
1117
|
-
if (item) this.finish(item, new Error(message.error?.message));
|
|
1142
|
+
if (item) this.finish(item, "failed", new Error(message.error?.message));
|
|
1118
1143
|
break;
|
|
1119
1144
|
}
|
|
1120
1145
|
default:
|
|
@@ -1194,15 +1219,17 @@ var OpenaiTTS = class extends TTS {
|
|
|
1194
1219
|
this.enqueue(counter, sentence);
|
|
1195
1220
|
}
|
|
1196
1221
|
});
|
|
1197
|
-
|
|
1198
|
-
this.log("Error in text stream", error);
|
|
1199
|
-
});
|
|
1200
|
-
textStream.on("end", () => {
|
|
1222
|
+
const flush = () => {
|
|
1201
1223
|
if (!counter || counter !== this.counter) return;
|
|
1202
1224
|
for (const sentence of this.splitter.flush()) {
|
|
1203
1225
|
this.enqueue(counter, sentence);
|
|
1204
1226
|
}
|
|
1227
|
+
};
|
|
1228
|
+
textStream.on("error", (error) => {
|
|
1229
|
+
this.log("Error in text stream", error);
|
|
1230
|
+
flush();
|
|
1205
1231
|
});
|
|
1232
|
+
textStream.on("end", flush);
|
|
1206
1233
|
}
|
|
1207
1234
|
cancel() {
|
|
1208
1235
|
this.generation++;
|