@micdrop/openai 1.7.0 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +66 -25
- package/dist/index.d.ts +66 -25
- package/dist/index.js +443 -417
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +447 -420
- package/dist/index.mjs.map +1 -1
- package/package.json +3 -3
package/dist/index.js
CHANGED
|
@@ -49,19 +49,69 @@ var DEFAULT_MODEL = "gpt-4o";
|
|
|
49
49
|
var DEFAULT_MAX_STEPS = 5;
|
|
50
50
|
var DEFAULT_MAX_RETRIES = 3;
|
|
51
51
|
var DEFAULT_RETRY_DELAY = 1e3;
|
|
52
|
+
var StepInterruptedError = class extends Error {
|
|
53
|
+
constructor(outcome, reason) {
|
|
54
|
+
super("Answer interrupted");
|
|
55
|
+
this.outcome = outcome;
|
|
56
|
+
this.reason = reason;
|
|
57
|
+
}
|
|
58
|
+
};
|
|
52
59
|
var OpenaiAgent = class extends import_server.Agent {
|
|
53
60
|
constructor(options) {
|
|
54
61
|
super(options);
|
|
55
62
|
this.openai = "openai" in options ? options.openai : new import_openai.default({ apiKey: options.apiKey });
|
|
56
63
|
}
|
|
57
|
-
async generateAnswer(stream,
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
64
|
+
async generateAnswer(stream, signal) {
|
|
65
|
+
const maxSteps = this.options.maxSteps || DEFAULT_MAX_STEPS;
|
|
66
|
+
const maxRetry = this.options.maxRetry ?? DEFAULT_MAX_RETRIES;
|
|
67
|
+
for (let step = 0; step < maxSteps; step++) {
|
|
68
|
+
let outcome;
|
|
69
|
+
for (let tryCount = 0; !outcome; tryCount++) {
|
|
70
|
+
try {
|
|
71
|
+
outcome = await this.generateStep(stream, signal);
|
|
72
|
+
} catch (error) {
|
|
73
|
+
if (signal.aborted) return;
|
|
74
|
+
if (error instanceof StepInterruptedError) {
|
|
75
|
+
console.error("[OpenaiAgent] Answer interrupted:", error.reason);
|
|
76
|
+
outcome = error.outcome;
|
|
77
|
+
break;
|
|
78
|
+
}
|
|
79
|
+
console.error("[OpenaiAgent] Error answering:", error);
|
|
80
|
+
if (tryCount >= maxRetry) {
|
|
81
|
+
this.log("Max retries reached, giving up");
|
|
82
|
+
this.emit("Failed");
|
|
83
|
+
return;
|
|
84
|
+
}
|
|
85
|
+
await new Promise(
|
|
86
|
+
(resolve) => setTimeout(resolve, this.options.retryDelay ?? DEFAULT_RETRY_DELAY)
|
|
87
|
+
);
|
|
88
|
+
if (signal.aborted) return;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
if (signal.aborted) return;
|
|
92
|
+
if (outcome.recorded || outcome.skipAnswer) return;
|
|
93
|
+
if (outcome.text) {
|
|
94
|
+
const { message, metadata } = this.extract(outcome.text);
|
|
95
|
+
this.addAssistantMessage(message, metadata);
|
|
96
|
+
return;
|
|
97
|
+
}
|
|
61
98
|
}
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
99
|
+
console.error("[OpenaiAgent] Max steps reached");
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* Runs one request, writing its text to the stream and running its tools.
|
|
103
|
+
*
|
|
104
|
+
* It throws the error of the request when it failed before producing
|
|
105
|
+
* anything, which can then be retried, and a {@link StepInterruptedError}
|
|
106
|
+
* once something was written or done.
|
|
107
|
+
*/
|
|
108
|
+
async generateStep(stream, signal) {
|
|
109
|
+
const outcome = {
|
|
110
|
+
text: "",
|
|
111
|
+
recorded: false,
|
|
112
|
+
skipAnswer: false
|
|
113
|
+
};
|
|
114
|
+
let produced = false;
|
|
65
115
|
let extracting = false;
|
|
66
116
|
const extractOptions = this.getExtractOptions();
|
|
67
117
|
try {
|
|
@@ -75,14 +125,14 @@ var OpenaiAgent = class extends import_server.Agent {
|
|
|
75
125
|
},
|
|
76
126
|
{ signal }
|
|
77
127
|
);
|
|
78
|
-
let skipAnswer = false;
|
|
79
|
-
let hasAnswer = false;
|
|
80
128
|
for await (const event of response) {
|
|
81
|
-
if (
|
|
129
|
+
if (signal.aborted) return outcome;
|
|
82
130
|
switch (event.type) {
|
|
83
131
|
case "response.output_text.delta":
|
|
84
|
-
if (
|
|
132
|
+
if (outcome.recorded) break;
|
|
85
133
|
this.log(`Answer chunk: "${event.delta}"`);
|
|
134
|
+
outcome.text += event.delta;
|
|
135
|
+
produced = true;
|
|
86
136
|
if (extractOptions) {
|
|
87
137
|
if (!extracting) {
|
|
88
138
|
const startTagIndex = event.delta.indexOf(
|
|
@@ -100,15 +150,17 @@ var OpenaiAgent = class extends import_server.Agent {
|
|
|
100
150
|
}
|
|
101
151
|
stream.write(event.delta);
|
|
102
152
|
break;
|
|
103
|
-
case "response.output_text.done":
|
|
104
|
-
if (
|
|
153
|
+
case "response.output_text.done": {
|
|
154
|
+
if (outcome.recorded) break;
|
|
105
155
|
const { message, metadata } = this.extract(event.text);
|
|
106
156
|
this.addAssistantMessage(message, metadata);
|
|
107
|
-
|
|
157
|
+
outcome.recorded = true;
|
|
108
158
|
break;
|
|
159
|
+
}
|
|
109
160
|
case "response.output_item.done":
|
|
110
161
|
if (event.item.type === "function_call") {
|
|
111
162
|
const toolCall = event.item;
|
|
163
|
+
produced = true;
|
|
112
164
|
const result = await this.executeTool({
|
|
113
165
|
role: "tool_call",
|
|
114
166
|
toolCallId: toolCall.call_id,
|
|
@@ -116,7 +168,7 @@ var OpenaiAgent = class extends import_server.Agent {
|
|
|
116
168
|
parameters: toolCall.arguments
|
|
117
169
|
});
|
|
118
170
|
if (result.skipAnswer) {
|
|
119
|
-
skipAnswer = true;
|
|
171
|
+
outcome.skipAnswer = true;
|
|
120
172
|
}
|
|
121
173
|
}
|
|
122
174
|
break;
|
|
@@ -124,23 +176,12 @@ var OpenaiAgent = class extends import_server.Agent {
|
|
|
124
176
|
break;
|
|
125
177
|
}
|
|
126
178
|
}
|
|
127
|
-
|
|
128
|
-
if (!skipAnswer && !hasAnswer) {
|
|
129
|
-
await this.generateAnswer(stream, stepCount + 1);
|
|
130
|
-
}
|
|
131
|
-
this.abortController = void 0;
|
|
179
|
+
return outcome;
|
|
132
180
|
} catch (error) {
|
|
133
|
-
if (
|
|
134
|
-
|
|
135
|
-
if (tryCount < (this.options.maxRetry ?? DEFAULT_MAX_RETRIES)) {
|
|
136
|
-
await new Promise(
|
|
137
|
-
(resolve) => setTimeout(resolve, this.options.retryDelay ?? DEFAULT_RETRY_DELAY)
|
|
138
|
-
);
|
|
139
|
-
await this.generateAnswer(stream, stepCount, tryCount + 1);
|
|
140
|
-
} else {
|
|
141
|
-
this.log("Max retries reached, giving up");
|
|
142
|
-
this.emit("Failed");
|
|
181
|
+
if (produced && !signal.aborted) {
|
|
182
|
+
throw new StepInterruptedError(outcome, error);
|
|
143
183
|
}
|
|
184
|
+
throw error;
|
|
144
185
|
}
|
|
145
186
|
}
|
|
146
187
|
buildInput() {
|
|
@@ -158,7 +199,8 @@ var OpenaiAgent = class extends import_server.Agent {
|
|
|
158
199
|
type: "function_call",
|
|
159
200
|
call_id: message.toolCallId,
|
|
160
201
|
name: message.toolName,
|
|
161
|
-
|
|
202
|
+
// Already the JSON the model wrote
|
|
203
|
+
arguments: message.parameters
|
|
162
204
|
};
|
|
163
205
|
case "tool_result":
|
|
164
206
|
return {
|
|
@@ -180,12 +222,6 @@ var OpenaiAgent = class extends import_server.Agent {
|
|
|
180
222
|
parameters: (0, import_zod.toJSONSchema)(tool.inputSchema || import_zod.default.object())
|
|
181
223
|
}));
|
|
182
224
|
}
|
|
183
|
-
cancel() {
|
|
184
|
-
if (!this.abortController) return;
|
|
185
|
-
this.log("Cancel");
|
|
186
|
-
this.abortController.abort();
|
|
187
|
-
this.abortController = void 0;
|
|
188
|
-
}
|
|
189
225
|
};
|
|
190
226
|
|
|
191
227
|
// src/OpenaiClassifier.ts
|
|
@@ -290,9 +326,26 @@ function buildQuestion(name, question) {
|
|
|
290
326
|
|
|
291
327
|
// src/OpenaiRealtime.ts
|
|
292
328
|
var import_server3 = require("@micdrop/server");
|
|
293
|
-
var import_ws = __toESM(require("ws"));
|
|
294
329
|
var import_zod2 = __toESM(require("zod"));
|
|
295
330
|
|
|
331
|
+
// src/session.ts
|
|
332
|
+
var sessionHandshake = {
|
|
333
|
+
read(data) {
|
|
334
|
+
const message = parseMessage(data);
|
|
335
|
+
if (message?.type === "error") {
|
|
336
|
+
throw new Error(message.error?.message || "Session refused");
|
|
337
|
+
}
|
|
338
|
+
return message?.type === "session.created" || message?.type === "transcription_session.created";
|
|
339
|
+
}
|
|
340
|
+
};
|
|
341
|
+
function parseMessage(data) {
|
|
342
|
+
try {
|
|
343
|
+
return JSON.parse(data.toString());
|
|
344
|
+
} catch {
|
|
345
|
+
return void 0;
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
|
|
296
349
|
// src/transcription.ts
|
|
297
350
|
function transcriptionLanguage(model, language) {
|
|
298
351
|
if (!language) return {};
|
|
@@ -309,36 +362,56 @@ var DEFAULT_TRANSCRIPTION_MODEL = "gpt-4o-transcribe";
|
|
|
309
362
|
var SAMPLE_RATE = 16e3;
|
|
310
363
|
var OPENAI_SAMPLE_RATE = 24e3;
|
|
311
364
|
var OPENAI_BYTES_PER_MS = OPENAI_SAMPLE_RATE * 2 / 1e3;
|
|
312
|
-
var
|
|
313
|
-
var
|
|
314
|
-
var DEFAULT_MAX_RETRY = 3;
|
|
365
|
+
var MAX_PENDING_EVENTS = 5e3;
|
|
366
|
+
var REQUEST_METADATA_KEY = "micdrop_request";
|
|
315
367
|
var OpenaiRealtime = class extends import_server3.Realtime {
|
|
316
368
|
constructor(options) {
|
|
317
369
|
super(options);
|
|
318
|
-
this.
|
|
319
|
-
this.everConnected = false;
|
|
320
|
-
this.destroyed = false;
|
|
370
|
+
this.connection = "connecting";
|
|
321
371
|
// Events sent while the connection opens
|
|
322
372
|
this.pending = [];
|
|
323
|
-
this.
|
|
373
|
+
this.requestCount = 0;
|
|
324
374
|
this.inputResampler = new import_server3.Pcm16Resampler(SAMPLE_RATE, OPENAI_SAMPLE_RATE);
|
|
325
375
|
this.outputResampler = new import_server3.Pcm16Resampler(OPENAI_SAMPLE_RATE, SAMPLE_RATE);
|
|
326
376
|
// Bumped by every interruption, so late responses can tell they are stale
|
|
327
377
|
this.generation = 0;
|
|
328
|
-
|
|
378
|
+
// Responses asked for and not created yet, by the id sent as their metadata
|
|
379
|
+
// and as the event id of their request, which an error names
|
|
380
|
+
this.requested = /* @__PURE__ */ new Map();
|
|
329
381
|
this.ignoredResponses = /* @__PURE__ */ new Set();
|
|
330
382
|
this.audioStartedAt = 0;
|
|
331
383
|
this.audioBytes = 0;
|
|
332
|
-
|
|
384
|
+
const model = options.model || DEFAULT_MODEL3;
|
|
385
|
+
this.socket = new import_server3.ReconnectingSocket({
|
|
386
|
+
url: `wss://api.openai.com/v1/realtime?model=${encodeURIComponent(model)}`,
|
|
387
|
+
clientOptions: {
|
|
388
|
+
headers: { Authorization: `Bearer ${options.apiKey}` }
|
|
389
|
+
},
|
|
390
|
+
connectionTimeout: options.connectionTimeout,
|
|
391
|
+
retryDelay: options.retryDelay,
|
|
392
|
+
maxRetry: options.maxRetry,
|
|
393
|
+
handshake: sessionHandshake,
|
|
394
|
+
log: (...message) => this.log(...message)
|
|
395
|
+
});
|
|
396
|
+
this.socket.on("Open", ({ reconnected }) => this.onOpen(reconnected));
|
|
397
|
+
this.socket.on("Close", () => {
|
|
398
|
+
this.connection = "reconnecting";
|
|
399
|
+
this.resetSession();
|
|
400
|
+
});
|
|
401
|
+
this.socket.on("Failed", () => {
|
|
402
|
+
this.connection = "closed";
|
|
403
|
+
this.pending = [];
|
|
404
|
+
this.resetSession();
|
|
405
|
+
this.emit("Failed");
|
|
406
|
+
});
|
|
407
|
+
this.socket.on("Message", (data) => this.handleMessage(parseMessage(data)));
|
|
408
|
+
this.socket.connect();
|
|
333
409
|
}
|
|
334
410
|
destroy() {
|
|
335
|
-
this.destroyed = true;
|
|
336
411
|
super.destroy();
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
}
|
|
341
|
-
this.closeSocket();
|
|
412
|
+
this.connection = "closed";
|
|
413
|
+
this.pending = [];
|
|
414
|
+
this.socket.destroy();
|
|
342
415
|
}
|
|
343
416
|
openTurn() {
|
|
344
417
|
this.inputResampler.reset();
|
|
@@ -390,25 +463,49 @@ var OpenaiRealtime = class extends import_server3.Realtime {
|
|
|
390
463
|
this.audioItemId = void 0;
|
|
391
464
|
}
|
|
392
465
|
updateTools() {
|
|
393
|
-
if (this.connected) this.sendSessionUpdate();
|
|
466
|
+
if (this.connection === "connected") this.sendSessionUpdate();
|
|
394
467
|
}
|
|
395
468
|
injectMessage(role, text) {
|
|
396
|
-
if (
|
|
469
|
+
if (this.connection === "reconnecting") return;
|
|
397
470
|
this.send(this.messageItem(role, text));
|
|
398
471
|
}
|
|
472
|
+
/**
|
|
473
|
+
* Asks for a response, tagged so its `response.created` tells which request
|
|
474
|
+
* it answers. Matching them by order alone would shift every later response
|
|
475
|
+
* onto the wrong request as soon as one is refused.
|
|
476
|
+
*/
|
|
399
477
|
createResponse(response) {
|
|
400
|
-
this.
|
|
401
|
-
|
|
402
|
-
|
|
478
|
+
const id = `micdrop-response-${++this.requestCount}`;
|
|
479
|
+
const generation = this.generation;
|
|
480
|
+
const accepted = this.send({
|
|
481
|
+
type: "response.create",
|
|
482
|
+
// Errors caused by this event carry its id
|
|
483
|
+
event_id: id,
|
|
484
|
+
response: { ...response, metadata: { [REQUEST_METADATA_KEY]: id } }
|
|
403
485
|
});
|
|
404
|
-
|
|
486
|
+
if (!accepted) {
|
|
487
|
+
queueMicrotask(() => {
|
|
488
|
+
if (generation === this.generation) this.endAnswer();
|
|
489
|
+
});
|
|
490
|
+
return;
|
|
491
|
+
}
|
|
492
|
+
this.requested.set(id, { generation, outOfBand: !!response });
|
|
405
493
|
}
|
|
494
|
+
/**
|
|
495
|
+
* Sends an event, or holds it until the connection opens.
|
|
496
|
+
* @returns False when it was dropped, the connection being given up on
|
|
497
|
+
*/
|
|
406
498
|
send(event) {
|
|
407
|
-
if (this.
|
|
408
|
-
this.socket.send(JSON.stringify(event));
|
|
409
|
-
} else {
|
|
410
|
-
this.pending.push(event);
|
|
499
|
+
if (this.connection === "connected") {
|
|
500
|
+
return this.socket.send(JSON.stringify(event));
|
|
411
501
|
}
|
|
502
|
+
if (this.connection === "closed") return false;
|
|
503
|
+
if (this.pending.length >= MAX_PENDING_EVENTS) {
|
|
504
|
+
this.log("Too many events waiting for the connection, dropped");
|
|
505
|
+
return false;
|
|
506
|
+
}
|
|
507
|
+
this.pending.push(event);
|
|
508
|
+
return true;
|
|
412
509
|
}
|
|
413
510
|
messageItem(role, text) {
|
|
414
511
|
return {
|
|
@@ -427,7 +524,7 @@ var OpenaiRealtime = class extends import_server3.Realtime {
|
|
|
427
524
|
}
|
|
428
525
|
sendSessionUpdate() {
|
|
429
526
|
const transcriptionModel = this.options.transcriptionModel || DEFAULT_TRANSCRIPTION_MODEL;
|
|
430
|
-
this.socket
|
|
527
|
+
this.socket.send(
|
|
431
528
|
JSON.stringify({
|
|
432
529
|
type: "session.update",
|
|
433
530
|
session: {
|
|
@@ -464,111 +561,50 @@ var OpenaiRealtime = class extends import_server3.Realtime {
|
|
|
464
561
|
})
|
|
465
562
|
);
|
|
466
563
|
}
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
const
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
const timeout = setTimeout(() => {
|
|
482
|
-
this.log("Connection timeout");
|
|
483
|
-
this.closeSocket();
|
|
484
|
-
reject(new Error("WebSocket connection timeout"));
|
|
485
|
-
}, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT);
|
|
486
|
-
socket.on("open", () => {
|
|
487
|
-
clearTimeout(timeout);
|
|
488
|
-
this.log("Connection opened");
|
|
489
|
-
this.sendSessionUpdate();
|
|
490
|
-
if (this.everConnected) {
|
|
491
|
-
for (const message of this.conversation) {
|
|
492
|
-
if (message.role !== "user" && message.role !== "assistant") {
|
|
493
|
-
continue;
|
|
494
|
-
}
|
|
495
|
-
socket.send(
|
|
496
|
-
JSON.stringify(this.messageItem(message.role, message.content))
|
|
497
|
-
);
|
|
498
|
-
}
|
|
499
|
-
}
|
|
500
|
-
this.connected = true;
|
|
501
|
-
this.everConnected = true;
|
|
502
|
-
this.retryCount = 0;
|
|
503
|
-
const pending = this.pending;
|
|
504
|
-
this.pending = [];
|
|
505
|
-
pending.forEach((event) => this.send(event));
|
|
506
|
-
resolve();
|
|
507
|
-
});
|
|
508
|
-
socket.on("error", (error) => {
|
|
509
|
-
clearTimeout(timeout);
|
|
510
|
-
this.log("WebSocket error:", error);
|
|
511
|
-
reject(new Error("WebSocket connection error"));
|
|
512
|
-
});
|
|
513
|
-
socket.on("close", (code, reason) => {
|
|
514
|
-
clearTimeout(timeout);
|
|
515
|
-
const wasConnected = this.connected;
|
|
516
|
-
this.closeSocket();
|
|
517
|
-
if (this.destroyed) return;
|
|
518
|
-
this.log("Connection closed", { code, reason: reason.toString() });
|
|
519
|
-
if (wasConnected) this.reconnect();
|
|
520
|
-
});
|
|
521
|
-
socket.on("message", (data) => {
|
|
522
|
-
this.handleMessage(JSON.parse(data.toString()));
|
|
523
|
-
});
|
|
524
|
-
});
|
|
525
|
-
}
|
|
526
|
-
closeSocket() {
|
|
527
|
-
const socket = this.socket;
|
|
528
|
-
this.socket = void 0;
|
|
529
|
-
this.connected = false;
|
|
564
|
+
/**
|
|
565
|
+
* Sets the session up, then sends what waited for it. A new connection
|
|
566
|
+
* starts from an empty conversation, so a reconnection sends it again first.
|
|
567
|
+
*/
|
|
568
|
+
onOpen(reconnected) {
|
|
569
|
+
this.connection = "connected";
|
|
570
|
+
this.sendSessionUpdate();
|
|
571
|
+
if (reconnected) {
|
|
572
|
+
for (const message of this.conversation) {
|
|
573
|
+
if (message.role !== "user" && message.role !== "assistant") continue;
|
|
574
|
+
this.send(this.messageItem(message.role, message.content));
|
|
575
|
+
}
|
|
576
|
+
}
|
|
577
|
+
const pending = this.pending;
|
|
530
578
|
this.pending = [];
|
|
531
|
-
|
|
579
|
+
pending.forEach((event) => this.send(event));
|
|
580
|
+
}
|
|
581
|
+
/**
|
|
582
|
+
* Forgets what belonged to the session that is gone: its responses, and the
|
|
583
|
+
* answer in progress, which is lost with it.
|
|
584
|
+
*/
|
|
585
|
+
resetSession() {
|
|
586
|
+
this.requested.clear();
|
|
532
587
|
this.response = void 0;
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
socket.close(1e3);
|
|
539
|
-
}
|
|
588
|
+
this.ignoredResponses.clear();
|
|
589
|
+
this.audioItemId = void 0;
|
|
590
|
+
this.audioBytes = 0;
|
|
591
|
+
this.outputResampler.reset();
|
|
592
|
+
this.generation++;
|
|
540
593
|
this.endAnswer();
|
|
541
594
|
}
|
|
542
|
-
reconnect() {
|
|
543
|
-
if (this.destroyed) return;
|
|
544
|
-
this.retryCount++;
|
|
545
|
-
if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY)) {
|
|
546
|
-
this.log("Max retries reached, giving up");
|
|
547
|
-
this.emit("Failed");
|
|
548
|
-
return;
|
|
549
|
-
}
|
|
550
|
-
this.log("Reconnecting...");
|
|
551
|
-
this.reconnectTimeout = setTimeout(() => {
|
|
552
|
-
this.reconnectTimeout = void 0;
|
|
553
|
-
this.connect();
|
|
554
|
-
}, this.options.retryDelay ?? DEFAULT_RETRY_DELAY2);
|
|
555
|
-
}
|
|
556
595
|
isIgnored(responseId) {
|
|
557
596
|
return !responseId || this.ignoredResponses.has(responseId) || this.response?.id !== responseId;
|
|
558
597
|
}
|
|
559
598
|
handleMessage(message) {
|
|
560
|
-
switch (message
|
|
599
|
+
switch (message?.type) {
|
|
561
600
|
case "response.created": {
|
|
562
|
-
const requested = this.requested.shift() ?? {
|
|
563
|
-
generation: this.generation,
|
|
564
|
-
outOfBand: false
|
|
565
|
-
};
|
|
566
601
|
const id = message.response.id;
|
|
567
|
-
|
|
602
|
+
const requestId = message.response.metadata?.[REQUEST_METADATA_KEY];
|
|
603
|
+
const requested = requestId ? this.requested.get(requestId) : { generation: this.generation, outOfBand: false };
|
|
604
|
+
if (requestId) this.requested.delete(requestId);
|
|
605
|
+
if (!requested || requested.generation !== this.generation) {
|
|
568
606
|
this.ignoredResponses.add(id);
|
|
569
|
-
this.
|
|
570
|
-
JSON.stringify({ type: "response.cancel", response_id: id })
|
|
571
|
-
);
|
|
607
|
+
this.send({ type: "response.cancel", response_id: id });
|
|
572
608
|
break;
|
|
573
609
|
}
|
|
574
610
|
this.response = { id, ...requested };
|
|
@@ -606,10 +642,18 @@ var OpenaiRealtime = class extends import_server3.Realtime {
|
|
|
606
642
|
case "response.done":
|
|
607
643
|
this.onResponseDone(message.response);
|
|
608
644
|
break;
|
|
609
|
-
case "error":
|
|
645
|
+
case "error": {
|
|
610
646
|
if (message.error?.code === "response_cancel_not_active") break;
|
|
611
647
|
this.log("Error:", message.error);
|
|
648
|
+
const requestId = message.error?.event_id;
|
|
649
|
+
const requested = this.requested.get(requestId);
|
|
650
|
+
if (!requested) break;
|
|
651
|
+
this.requested.delete(requestId);
|
|
652
|
+
if (requested.generation === this.generation && !this.response) {
|
|
653
|
+
this.endAnswer();
|
|
654
|
+
}
|
|
612
655
|
break;
|
|
656
|
+
}
|
|
613
657
|
default:
|
|
614
658
|
break;
|
|
615
659
|
}
|
|
@@ -659,69 +703,78 @@ var OpenaiRealtime = class extends import_server3.Realtime {
|
|
|
659
703
|
|
|
660
704
|
// src/OpenaiSTT.ts
|
|
661
705
|
var import_server4 = require("@micdrop/server");
|
|
662
|
-
var import_ws2 = __toESM(require("ws"));
|
|
663
706
|
var DEFAULT_MODEL4 = "gpt-4o-transcribe";
|
|
664
707
|
var DEFAULT_LANGUAGE = "en";
|
|
665
708
|
var SAMPLE_RATE2 = 16e3;
|
|
666
709
|
var OPENAI_SAMPLE_RATE2 = 24e3;
|
|
667
|
-
var DEFAULT_CONNECTION_TIMEOUT2 = 5e3;
|
|
668
710
|
var DEFAULT_TRANSCRIPTION_TIMEOUT = 4e3;
|
|
669
|
-
var DEFAULT_RETRY_DELAY3 = 1e3;
|
|
670
|
-
var DEFAULT_MAX_RETRY2 = 3;
|
|
671
711
|
var OpenaiSTT = class extends import_server4.STT {
|
|
672
712
|
constructor(options) {
|
|
673
713
|
super();
|
|
674
714
|
this.options = options;
|
|
675
|
-
this.
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
this.
|
|
681
|
-
|
|
682
|
-
|
|
715
|
+
this.utteranceCount = 0;
|
|
716
|
+
// Utterances not transcribed yet, in the order they were spoken. Each one
|
|
717
|
+
// leaves the list with the single transcript it emits.
|
|
718
|
+
this.utterances = [];
|
|
719
|
+
// Commits sent and not acknowledged yet, in the order the API handles them
|
|
720
|
+
this.unacknowledged = [];
|
|
721
|
+
this.socket = new import_server4.ReconnectingSocket({
|
|
722
|
+
url: "wss://api.openai.com/v1/realtime?intent=transcription",
|
|
723
|
+
clientOptions: {
|
|
724
|
+
headers: { Authorization: `Bearer ${this.options.apiKey}` }
|
|
725
|
+
},
|
|
726
|
+
connectionTimeout: options.connectionTimeout,
|
|
727
|
+
retryDelay: options.retryDelay,
|
|
728
|
+
maxRetry: options.maxRetry,
|
|
729
|
+
handshake: sessionHandshake,
|
|
730
|
+
log: (...message) => this.log(...message)
|
|
731
|
+
});
|
|
732
|
+
this.socket.on("Open", () => this.onOpen());
|
|
733
|
+
this.socket.on("Close", () => {
|
|
734
|
+
this.unacknowledged = [];
|
|
683
735
|
});
|
|
736
|
+
this.socket.on("Message", (data) => this.handleMessage(parseMessage(data)));
|
|
737
|
+
this.socket.on("Failed", () => this.onFailed());
|
|
738
|
+
this.socket.connect();
|
|
684
739
|
}
|
|
740
|
+
/**
|
|
741
|
+
* Transcribes one utterance, and emits its transcript once, possibly empty
|
|
742
|
+
* when none comes in time.
|
|
743
|
+
*
|
|
744
|
+
* After the connection was given up on, the next utterance tries it again.
|
|
745
|
+
*/
|
|
685
746
|
transcribe(audioStream) {
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
747
|
+
const utterance = {
|
|
748
|
+
id: `micdrop-stt-${++this.utteranceCount}`,
|
|
749
|
+
status: "streaming",
|
|
750
|
+
chunks: [],
|
|
751
|
+
resampler: new import_server4.Pcm16Resampler(SAMPLE_RATE2, OPENAI_SAMPLE_RATE2)
|
|
752
|
+
};
|
|
753
|
+
this.utterances.push(utterance);
|
|
754
|
+
this.socket.connect();
|
|
755
|
+
audioStream.on("data", (chunk) => {
|
|
756
|
+
if (!this.utterances.includes(utterance)) return;
|
|
757
|
+
utterance.chunks.push(chunk);
|
|
758
|
+
this.sendAudioChunk(utterance, chunk);
|
|
692
759
|
});
|
|
693
|
-
audioStream.on("end",
|
|
694
|
-
|
|
695
|
-
if (
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
this.audioChunksPending.length = 0;
|
|
702
|
-
}, this.options.transcriptionTimeout || DEFAULT_TRANSCRIPTION_TIMEOUT);
|
|
760
|
+
audioStream.on("end", () => {
|
|
761
|
+
if (!this.utterances.includes(utterance)) return;
|
|
762
|
+
if (utterance.chunks.length === 0) {
|
|
763
|
+
this.remove(utterance);
|
|
764
|
+
return;
|
|
765
|
+
}
|
|
766
|
+
utterance.status = "committed";
|
|
767
|
+
this.commitAudio(utterance);
|
|
703
768
|
});
|
|
704
769
|
}
|
|
705
770
|
destroy() {
|
|
706
771
|
super.destroy();
|
|
707
|
-
this.
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
}
|
|
712
|
-
if (this.transcriptionTimeout) {
|
|
713
|
-
clearTimeout(this.transcriptionTimeout);
|
|
714
|
-
this.transcriptionTimeout = void 0;
|
|
715
|
-
}
|
|
716
|
-
if (this.connectionTimeout) {
|
|
717
|
-
clearTimeout(this.connectionTimeout);
|
|
718
|
-
this.connectionTimeout = void 0;
|
|
719
|
-
}
|
|
720
|
-
if (this.socket) closeSocket(this.socket);
|
|
721
|
-
this.socket = void 0;
|
|
772
|
+
this.socket.destroy();
|
|
773
|
+
this.utterances.forEach((utterance) => clearTimeout(utterance.timer));
|
|
774
|
+
this.utterances = [];
|
|
775
|
+
this.unacknowledged = [];
|
|
722
776
|
}
|
|
723
777
|
sendSessionUpdate() {
|
|
724
|
-
if (!this.socket) return;
|
|
725
778
|
const model = this.options.model || DEFAULT_MODEL4;
|
|
726
779
|
this.socket.send(
|
|
727
780
|
JSON.stringify({
|
|
@@ -755,56 +808,35 @@ var OpenaiSTT = class extends import_server4.STT {
|
|
|
755
808
|
})
|
|
756
809
|
);
|
|
757
810
|
}
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
this.connectionTimeout = timeout;
|
|
776
|
-
socket.addEventListener("open", () => {
|
|
777
|
-
clearTimeout(timeout);
|
|
778
|
-
this.sendSessionUpdate();
|
|
779
|
-
this.log("Connection opened");
|
|
780
|
-
resolve();
|
|
781
|
-
});
|
|
782
|
-
socket.addEventListener("error", (error) => {
|
|
783
|
-
clearTimeout(timeout);
|
|
784
|
-
this.log("WebSocket error:", error);
|
|
785
|
-
reject(new Error("WebSocket connection error"));
|
|
786
|
-
});
|
|
787
|
-
socket.addEventListener(
|
|
788
|
-
"close",
|
|
789
|
-
({ code, reason }) => {
|
|
790
|
-
clearTimeout(timeout);
|
|
791
|
-
this.socket?.removeAllListeners();
|
|
792
|
-
this.socket = void 0;
|
|
793
|
-
if (code !== 1e3) {
|
|
794
|
-
this.reconnect();
|
|
795
|
-
} else {
|
|
796
|
-
this.log("Connection closed", { code, reason });
|
|
797
|
-
}
|
|
798
|
-
}
|
|
799
|
-
);
|
|
800
|
-
socket.addEventListener("message", (event) => {
|
|
801
|
-
this.handleMessage(JSON.parse(event.data.toString()));
|
|
802
|
-
});
|
|
803
|
-
});
|
|
811
|
+
/**
|
|
812
|
+
* Configures the session, then sends again what a previous connection did
|
|
813
|
+
* not get transcribed. The OpenAI buffer of a new session is empty, so each
|
|
814
|
+
* utterance is replayed through a fresh resampler, which reproduces the exact
|
|
815
|
+
* same stream, and committed again if it was.
|
|
816
|
+
*/
|
|
817
|
+
onOpen() {
|
|
818
|
+
this.sendSessionUpdate();
|
|
819
|
+
this.unacknowledged = [];
|
|
820
|
+
if (this.utterances.length > 0) this.log("Sending audio chunks again");
|
|
821
|
+
for (const utterance of this.utterances) {
|
|
822
|
+
if (utterance.transcript !== void 0) continue;
|
|
823
|
+
utterance.itemId = void 0;
|
|
824
|
+
utterance.resampler = new import_server4.Pcm16Resampler(SAMPLE_RATE2, OPENAI_SAMPLE_RATE2);
|
|
825
|
+
utterance.chunks.forEach((chunk) => this.sendAudioChunk(utterance, chunk));
|
|
826
|
+
if (utterance.status === "committed") this.commitAudio(utterance);
|
|
827
|
+
}
|
|
804
828
|
}
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
const
|
|
829
|
+
/** Every retry failed: the audio waiting for a transcript is handed back */
|
|
830
|
+
onFailed() {
|
|
831
|
+
const chunks = this.utterances.flatMap((utterance) => utterance.chunks);
|
|
832
|
+
this.utterances.forEach((utterance) => clearTimeout(utterance.timer));
|
|
833
|
+
this.utterances = [];
|
|
834
|
+
this.unacknowledged = [];
|
|
835
|
+
this.emit("Failed", chunks);
|
|
836
|
+
}
|
|
837
|
+
sendAudioChunk(utterance, chunk) {
|
|
838
|
+
if (!this.socket.isOpen) return;
|
|
839
|
+
const resampled = utterance.resampler.process(chunk);
|
|
808
840
|
if (resampled.length === 0) return;
|
|
809
841
|
this.socket.send(
|
|
810
842
|
JSON.stringify({
|
|
@@ -813,81 +845,100 @@ var OpenaiSTT = class extends import_server4.STT {
|
|
|
813
845
|
})
|
|
814
846
|
);
|
|
815
847
|
}
|
|
848
|
+
commitAudio(utterance) {
|
|
849
|
+
if (!this.socket.isOpen) return;
|
|
850
|
+
this.socket.send(
|
|
851
|
+
JSON.stringify({
|
|
852
|
+
type: "input_audio_buffer.commit",
|
|
853
|
+
event_id: utterance.id
|
|
854
|
+
})
|
|
855
|
+
);
|
|
856
|
+
this.unacknowledged.push(utterance);
|
|
857
|
+
this.log("Committed audio buffer");
|
|
858
|
+
clearTimeout(utterance.timer);
|
|
859
|
+
utterance.timer = setTimeout(() => {
|
|
860
|
+
utterance.timer = void 0;
|
|
861
|
+
this.log("Transcription timeout");
|
|
862
|
+
this.finish(utterance, "");
|
|
863
|
+
}, this.options.transcriptionTimeout || DEFAULT_TRANSCRIPTION_TIMEOUT);
|
|
864
|
+
}
|
|
865
|
+
/**
|
|
866
|
+
* Settles the transcript of an utterance, the only one it gets.
|
|
867
|
+
*
|
|
868
|
+
* The API transcribes each utterance on its own and may finish a short one
|
|
869
|
+
* before a long one spoken earlier, so transcripts go out in the order the
|
|
870
|
+
* utterances were spoken.
|
|
871
|
+
*/
|
|
872
|
+
finish(utterance, transcript) {
|
|
873
|
+
if (!this.utterances.includes(utterance)) return;
|
|
874
|
+
if (utterance.transcript !== void 0) return;
|
|
875
|
+
clearTimeout(utterance.timer);
|
|
876
|
+
utterance.timer = void 0;
|
|
877
|
+
utterance.transcript = transcript;
|
|
878
|
+
while (this.utterances[0]?.transcript !== void 0) {
|
|
879
|
+
const first = this.utterances[0];
|
|
880
|
+
this.remove(first);
|
|
881
|
+
this.emit("Transcript", first.transcript);
|
|
882
|
+
}
|
|
883
|
+
}
|
|
884
|
+
remove(utterance) {
|
|
885
|
+
clearTimeout(utterance.timer);
|
|
886
|
+
utterance.timer = void 0;
|
|
887
|
+
this.utterances = this.utterances.filter((other) => other !== utterance);
|
|
888
|
+
this.unacknowledged = this.unacknowledged.filter(
|
|
889
|
+
(other) => other !== utterance
|
|
890
|
+
);
|
|
891
|
+
}
|
|
892
|
+
/**
|
|
893
|
+
* The utterance a transcript event is about. The API names the item, and
|
|
894
|
+
* the item of each utterance is known from the acknowledgement of its
|
|
895
|
+
* commit. A transcript of an utterance that timed out matches none.
|
|
896
|
+
*/
|
|
897
|
+
findUtterance(itemId) {
|
|
898
|
+
if (!itemId) {
|
|
899
|
+
return this.utterances.find(
|
|
900
|
+
({ status, transcript }) => status === "committed" && transcript === void 0
|
|
901
|
+
);
|
|
902
|
+
}
|
|
903
|
+
return this.utterances.find((utterance) => utterance.itemId === itemId);
|
|
904
|
+
}
|
|
816
905
|
handleMessage(message) {
|
|
817
|
-
switch (message
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
// case 'input_audio_buffer.speech_stopped':
|
|
825
|
-
// this.log('Speech stopped')
|
|
826
|
-
// break
|
|
827
|
-
// case 'conversation.item.input_audio_transcription.delta':
|
|
828
|
-
// this.log(`Received transcript delta: "${message.delta}"`)
|
|
829
|
-
// break
|
|
830
|
-
case "conversation.item.input_audio_transcription.completed":
|
|
906
|
+
switch (message?.type) {
|
|
907
|
+
case "input_audio_buffer.committed": {
|
|
908
|
+
const utterance = this.unacknowledged.shift();
|
|
909
|
+
if (utterance) utterance.itemId = message.item_id;
|
|
910
|
+
break;
|
|
911
|
+
}
|
|
912
|
+
case "conversation.item.input_audio_transcription.completed": {
|
|
831
913
|
const transcript = message.transcript || "";
|
|
832
914
|
this.log(`Received completed transcript: "${transcript}"`);
|
|
833
|
-
this.
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
this.transcriptionTimeout = void 0;
|
|
915
|
+
const utterance = this.findUtterance(message.item_id);
|
|
916
|
+
if (!utterance) {
|
|
917
|
+
this.log("Transcript of an utterance already given up on, ignored");
|
|
918
|
+
break;
|
|
838
919
|
}
|
|
920
|
+
this.finish(utterance, transcript);
|
|
839
921
|
break;
|
|
840
|
-
|
|
841
|
-
|
|
922
|
+
}
|
|
923
|
+
case "conversation.item.input_audio_transcription.failed": {
|
|
924
|
+
this.log("Transcription failed:", message.error);
|
|
925
|
+
const utterance = this.findUtterance(message.item_id);
|
|
926
|
+
if (utterance) this.finish(utterance, "");
|
|
842
927
|
break;
|
|
843
|
-
|
|
928
|
+
}
|
|
929
|
+
case "error": {
|
|
844
930
|
this.log("Error:", message.error);
|
|
931
|
+
const utterance = this.utterances.find(
|
|
932
|
+
({ id }) => id === message.error?.event_id
|
|
933
|
+
);
|
|
934
|
+
if (utterance) this.finish(utterance, "");
|
|
845
935
|
break;
|
|
936
|
+
}
|
|
846
937
|
default:
|
|
847
938
|
break;
|
|
848
939
|
}
|
|
849
940
|
}
|
|
850
|
-
reconnect() {
|
|
851
|
-
if (this.destroyed) return;
|
|
852
|
-
this.retryCount++;
|
|
853
|
-
if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY2)) {
|
|
854
|
-
this.log("Max retries reached, giving up");
|
|
855
|
-
this.emit("Failed", this.audioChunksPending);
|
|
856
|
-
return;
|
|
857
|
-
}
|
|
858
|
-
this.initPromise = new Promise((resolve) => {
|
|
859
|
-
this.log("Reconnecting...");
|
|
860
|
-
this.reconnectTimeout = setTimeout(() => {
|
|
861
|
-
this.reconnectTimeout = void 0;
|
|
862
|
-
this.initWS().then(() => {
|
|
863
|
-
if (this.audioChunksPending.length > 0) {
|
|
864
|
-
this.log("Sending audio chunks again");
|
|
865
|
-
this.resampler.reset();
|
|
866
|
-
this.audioChunksPending.forEach(
|
|
867
|
-
(chunk) => this.sendAudioChunk(chunk)
|
|
868
|
-
);
|
|
869
|
-
}
|
|
870
|
-
}).then(resolve).catch((error) => {
|
|
871
|
-
this.log("Reconnection error:", error);
|
|
872
|
-
this.reconnect();
|
|
873
|
-
});
|
|
874
|
-
}, this.options.retryDelay ?? DEFAULT_RETRY_DELAY3);
|
|
875
|
-
});
|
|
876
|
-
}
|
|
877
|
-
commitAudio() {
|
|
878
|
-
if (!this.socket) return;
|
|
879
|
-
this.socket.send(JSON.stringify({ type: "input_audio_buffer.commit" }));
|
|
880
|
-
this.log("Committed audio buffer");
|
|
881
|
-
}
|
|
882
941
|
};
|
|
883
|
-
function closeSocket(socket) {
|
|
884
|
-
socket.removeAllListeners();
|
|
885
|
-
socket.on("error", () => {
|
|
886
|
-
});
|
|
887
|
-
if (socket.readyState === import_ws2.default.OPEN || socket.readyState === import_ws2.default.CONNECTING) {
|
|
888
|
-
socket.close(1e3);
|
|
889
|
-
}
|
|
890
|
-
}
|
|
891
942
|
|
|
892
943
|
// src/OpenaiTTS.ts
|
|
893
944
|
var import_server6 = require("@micdrop/server");
|
|
@@ -895,9 +946,7 @@ var import_openai3 = __toESM(require("openai"));
|
|
|
895
946
|
|
|
896
947
|
// src/OpenaiRealtimeSpeech.ts
|
|
897
948
|
var import_server5 = require("@micdrop/server");
|
|
898
|
-
var import_ws3 = __toESM(require("ws"));
|
|
899
949
|
var DEFAULT_REASONING_EFFORT = "minimal";
|
|
900
|
-
var DEFAULT_CONNECTION_TIMEOUT3 = 5e3;
|
|
901
950
|
var MIN_SPEED = 0.25;
|
|
902
951
|
var MAX_SPEED = 1.5;
|
|
903
952
|
var OPENAI_SAMPLE_RATE3 = 24e3;
|
|
@@ -910,22 +959,48 @@ var OpenaiRealtimeSpeech = class {
|
|
|
910
959
|
constructor(options, log) {
|
|
911
960
|
this.options = options;
|
|
912
961
|
this.log = log;
|
|
913
|
-
this.destroyed = false;
|
|
914
962
|
this.itemCount = 0;
|
|
915
963
|
// Responses asked for and not finished, by their metadata id and by the id
|
|
916
|
-
// the API gave them.
|
|
964
|
+
// the API gave them. Finished ones leave both maps, which mutes them.
|
|
917
965
|
this.items = /* @__PURE__ */ new Map();
|
|
918
966
|
this.responses = /* @__PURE__ */ new Map();
|
|
919
|
-
|
|
967
|
+
const { model, apiKey } = options;
|
|
968
|
+
this.socket = new import_server5.ReconnectingSocket({
|
|
969
|
+
url: `wss://api.openai.com/v1/realtime?model=${encodeURIComponent(model)}`,
|
|
970
|
+
clientOptions: { headers: { Authorization: `Bearer ${apiKey}` } },
|
|
971
|
+
connectionTimeout: options.connectionTimeout,
|
|
972
|
+
handshake: sessionHandshake,
|
|
973
|
+
log
|
|
974
|
+
});
|
|
975
|
+
this.socket.on("Open", () => {
|
|
976
|
+
this.socket.send(JSON.stringify(this.sessionUpdate()));
|
|
920
977
|
});
|
|
978
|
+
this.socket.on("Close", () => {
|
|
979
|
+
for (const item of [...this.items.values()]) {
|
|
980
|
+
if (item.status === "created") {
|
|
981
|
+
this.finish(item, "failed", new Error("Connection lost"));
|
|
982
|
+
} else if (item.status === "requested") {
|
|
983
|
+
item.status = "waiting";
|
|
984
|
+
this.request(item);
|
|
985
|
+
}
|
|
986
|
+
}
|
|
987
|
+
});
|
|
988
|
+
this.socket.on("Message", (data) => {
|
|
989
|
+
try {
|
|
990
|
+
this.handleMessage(parseMessage(data));
|
|
991
|
+
} catch (error) {
|
|
992
|
+
this.log("Error handling message", error);
|
|
993
|
+
}
|
|
994
|
+
});
|
|
995
|
+
this.socket.connect();
|
|
921
996
|
}
|
|
922
997
|
synthesize(text) {
|
|
923
998
|
const item = {
|
|
924
999
|
id: `micdrop-tts-${++this.itemCount}`,
|
|
925
1000
|
text,
|
|
1001
|
+
status: "waiting",
|
|
926
1002
|
resampler: new import_server5.Pcm16Resampler(OPENAI_SAMPLE_RATE3, OUTPUT_SAMPLE_RATE),
|
|
927
|
-
audio: []
|
|
928
|
-
done: false
|
|
1003
|
+
audio: []
|
|
929
1004
|
};
|
|
930
1005
|
this.items.set(item.id, item);
|
|
931
1006
|
this.request(item);
|
|
@@ -935,13 +1010,28 @@ var OpenaiRealtimeSpeech = class {
|
|
|
935
1010
|
};
|
|
936
1011
|
}
|
|
937
1012
|
destroy() {
|
|
938
|
-
this.
|
|
939
|
-
this.
|
|
1013
|
+
this.socket.destroy();
|
|
1014
|
+
for (const item of [...this.items.values()]) {
|
|
1015
|
+
this.finish(item, "failed", new Error("TTS destroyed"));
|
|
1016
|
+
}
|
|
940
1017
|
}
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
1018
|
+
/**
|
|
1019
|
+
* Sends the sentence once the connection is open.
|
|
1020
|
+
*
|
|
1021
|
+
* A Realtime session lasts an hour at most, and the connection opens again
|
|
1022
|
+
* on its own when the API closes it. After every retry failed, the next
|
|
1023
|
+
* sentence starts over with a full retry budget.
|
|
1024
|
+
*/
|
|
1025
|
+
async request(item) {
|
|
1026
|
+
this.socket.connect();
|
|
1027
|
+
const open = await this.socket.ready();
|
|
1028
|
+
if (item.status !== "waiting") return;
|
|
1029
|
+
if (!open) {
|
|
1030
|
+
this.finish(item, "failed", new Error("WebSocket connection failed"));
|
|
1031
|
+
return;
|
|
1032
|
+
}
|
|
1033
|
+
const sent = this.socket.send(
|
|
1034
|
+
JSON.stringify({
|
|
945
1035
|
type: "response.create",
|
|
946
1036
|
// Errors caused by this event carry its id
|
|
947
1037
|
event_id: item.id,
|
|
@@ -967,8 +1057,13 @@ var OpenaiRealtimeSpeech = class {
|
|
|
967
1057
|
output_modalities: ["audio"],
|
|
968
1058
|
metadata: { micdrop_id: item.id }
|
|
969
1059
|
}
|
|
970
|
-
})
|
|
971
|
-
|
|
1060
|
+
})
|
|
1061
|
+
);
|
|
1062
|
+
if (sent) {
|
|
1063
|
+
item.status = "requested";
|
|
1064
|
+
} else {
|
|
1065
|
+
this.finish(item, "failed", new Error("WebSocket connection lost"));
|
|
1066
|
+
}
|
|
972
1067
|
}
|
|
973
1068
|
async *read(item) {
|
|
974
1069
|
while (true) {
|
|
@@ -977,21 +1072,22 @@ var OpenaiRealtimeSpeech = class {
|
|
|
977
1072
|
yield chunk;
|
|
978
1073
|
continue;
|
|
979
1074
|
}
|
|
980
|
-
if (item.
|
|
981
|
-
if (item.done) return;
|
|
1075
|
+
if (item.status === "failed") throw item.error;
|
|
1076
|
+
if (item.status === "done" || item.status === "cancelled") return;
|
|
982
1077
|
await new Promise((resolve) => item.wake = resolve);
|
|
983
1078
|
}
|
|
984
1079
|
}
|
|
985
1080
|
cancel(item) {
|
|
986
|
-
if (item.
|
|
1081
|
+
if (item.status === "created" && item.responseId) {
|
|
987
1082
|
this.send({ type: "response.cancel", response_id: item.responseId });
|
|
988
1083
|
}
|
|
989
1084
|
item.audio = [];
|
|
990
|
-
this.finish(item);
|
|
1085
|
+
this.finish(item, "cancelled");
|
|
991
1086
|
}
|
|
992
|
-
|
|
1087
|
+
/** Ends a sentence, the first end being the only one that counts */
|
|
1088
|
+
finish(item, status, error) {
|
|
993
1089
|
if (!this.items.has(item.id)) return;
|
|
994
|
-
item.
|
|
1090
|
+
item.status = status;
|
|
995
1091
|
item.error = error;
|
|
996
1092
|
this.items.delete(item.id);
|
|
997
1093
|
if (item.responseId) this.responses.delete(item.responseId);
|
|
@@ -1000,9 +1096,7 @@ var OpenaiRealtimeSpeech = class {
|
|
|
1000
1096
|
wake?.();
|
|
1001
1097
|
}
|
|
1002
1098
|
send(event) {
|
|
1003
|
-
|
|
1004
|
-
this.socket.send(JSON.stringify(event));
|
|
1005
|
-
}
|
|
1099
|
+
this.socket.send(JSON.stringify(event));
|
|
1006
1100
|
}
|
|
1007
1101
|
sessionUpdate() {
|
|
1008
1102
|
const { instructions, speed } = this.options;
|
|
@@ -1029,80 +1123,8 @@ Delivery: ${instructions}` : READER_INSTRUCTIONS,
|
|
|
1029
1123
|
}
|
|
1030
1124
|
};
|
|
1031
1125
|
}
|
|
1032
|
-
/**
|
|
1033
|
-
* Opens the session, or hands over the one already open.
|
|
1034
|
-
*
|
|
1035
|
-
* A Realtime session lasts an hour at most. A closed one is opened again by
|
|
1036
|
-
* the next sentence rather than on a timer, so an idle agent holds no
|
|
1037
|
-
* connection.
|
|
1038
|
-
*/
|
|
1039
|
-
connect() {
|
|
1040
|
-
if (this.destroyed) return Promise.reject(new Error("TTS destroyed"));
|
|
1041
|
-
if (this.socket?.readyState === import_ws3.default.OPEN) {
|
|
1042
|
-
return Promise.resolve(this.socket);
|
|
1043
|
-
}
|
|
1044
|
-
if (this.connecting) return this.connecting;
|
|
1045
|
-
const { model, apiKey } = this.options;
|
|
1046
|
-
const connecting = new Promise((resolve, reject) => {
|
|
1047
|
-
const socket = new import_ws3.default(
|
|
1048
|
-
`wss://api.openai.com/v1/realtime?model=${encodeURIComponent(model)}`,
|
|
1049
|
-
{ headers: { Authorization: `Bearer ${apiKey}` } }
|
|
1050
|
-
);
|
|
1051
|
-
this.socket = socket;
|
|
1052
|
-
const timeout = setTimeout(() => {
|
|
1053
|
-
this.log("Connection timeout");
|
|
1054
|
-
this.closeSocket();
|
|
1055
|
-
reject(new Error("WebSocket connection timeout"));
|
|
1056
|
-
}, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT3);
|
|
1057
|
-
socket.on("open", () => {
|
|
1058
|
-
clearTimeout(timeout);
|
|
1059
|
-
this.log("Connection opened");
|
|
1060
|
-
socket.send(JSON.stringify(this.sessionUpdate()));
|
|
1061
|
-
resolve(socket);
|
|
1062
|
-
});
|
|
1063
|
-
socket.on("error", (error) => {
|
|
1064
|
-
clearTimeout(timeout);
|
|
1065
|
-
this.log("WebSocket error:", error);
|
|
1066
|
-
reject(new Error("WebSocket connection error"));
|
|
1067
|
-
});
|
|
1068
|
-
socket.on("close", (code, reason) => {
|
|
1069
|
-
clearTimeout(timeout);
|
|
1070
|
-
this.log("Connection closed", { code, reason: reason.toString() });
|
|
1071
|
-
reject(new Error("WebSocket connection closed"));
|
|
1072
|
-
this.closeSocket();
|
|
1073
|
-
});
|
|
1074
|
-
socket.on("message", (data) => {
|
|
1075
|
-
try {
|
|
1076
|
-
this.handleMessage(JSON.parse(data.toString()));
|
|
1077
|
-
} catch (error) {
|
|
1078
|
-
this.log("Error handling message", error);
|
|
1079
|
-
}
|
|
1080
|
-
});
|
|
1081
|
-
});
|
|
1082
|
-
this.connecting = connecting;
|
|
1083
|
-
const settle = () => {
|
|
1084
|
-
if (this.connecting === connecting) this.connecting = void 0;
|
|
1085
|
-
};
|
|
1086
|
-
connecting.then(settle, settle);
|
|
1087
|
-
return connecting;
|
|
1088
|
-
}
|
|
1089
|
-
closeSocket() {
|
|
1090
|
-
const socket = this.socket;
|
|
1091
|
-
this.socket = void 0;
|
|
1092
|
-
this.connecting = void 0;
|
|
1093
|
-
for (const item of [...this.items.values()]) {
|
|
1094
|
-
this.finish(item, new Error("Connection lost"));
|
|
1095
|
-
}
|
|
1096
|
-
if (!socket) return;
|
|
1097
|
-
socket.removeAllListeners();
|
|
1098
|
-
socket.on("error", () => {
|
|
1099
|
-
});
|
|
1100
|
-
if (socket.readyState === import_ws3.default.OPEN || socket.readyState === import_ws3.default.CONNECTING) {
|
|
1101
|
-
socket.close(1e3);
|
|
1102
|
-
}
|
|
1103
|
-
}
|
|
1104
1126
|
handleMessage(message) {
|
|
1105
|
-
switch (message
|
|
1127
|
+
switch (message?.type) {
|
|
1106
1128
|
case "response.created": {
|
|
1107
1129
|
const id = message.response.id;
|
|
1108
1130
|
const item = this.items.get(message.response.metadata?.micdrop_id);
|
|
@@ -1110,6 +1132,7 @@ Delivery: ${instructions}` : READER_INSTRUCTIONS,
|
|
|
1110
1132
|
this.send({ type: "response.cancel", response_id: id });
|
|
1111
1133
|
break;
|
|
1112
1134
|
}
|
|
1135
|
+
item.status = "created";
|
|
1113
1136
|
item.responseId = id;
|
|
1114
1137
|
this.responses.set(id, item);
|
|
1115
1138
|
break;
|
|
@@ -1141,12 +1164,13 @@ Delivery: ${instructions}` : READER_INSTRUCTIONS,
|
|
|
1141
1164
|
if (response.status === "failed") {
|
|
1142
1165
|
this.finish(
|
|
1143
1166
|
item,
|
|
1167
|
+
"failed",
|
|
1144
1168
|
new Error(
|
|
1145
1169
|
`Response failed: ${JSON.stringify(response.status_details)}`
|
|
1146
1170
|
)
|
|
1147
1171
|
);
|
|
1148
1172
|
} else {
|
|
1149
|
-
this.finish(item);
|
|
1173
|
+
this.finish(item, "done");
|
|
1150
1174
|
}
|
|
1151
1175
|
break;
|
|
1152
1176
|
}
|
|
@@ -1154,7 +1178,7 @@ Delivery: ${instructions}` : READER_INSTRUCTIONS,
|
|
|
1154
1178
|
if (message.error?.code === "response_cancel_not_active") break;
|
|
1155
1179
|
this.log("Error:", message.error);
|
|
1156
1180
|
const item = this.items.get(message.error?.event_id);
|
|
1157
|
-
if (item) this.finish(item, new Error(message.error?.message));
|
|
1181
|
+
if (item) this.finish(item, "failed", new Error(message.error?.message));
|
|
1158
1182
|
break;
|
|
1159
1183
|
}
|
|
1160
1184
|
default:
|
|
@@ -1234,15 +1258,17 @@ var OpenaiTTS = class extends import_server6.TTS {
|
|
|
1234
1258
|
this.enqueue(counter, sentence);
|
|
1235
1259
|
}
|
|
1236
1260
|
});
|
|
1237
|
-
|
|
1238
|
-
this.log("Error in text stream", error);
|
|
1239
|
-
});
|
|
1240
|
-
textStream.on("end", () => {
|
|
1261
|
+
const flush = () => {
|
|
1241
1262
|
if (!counter || counter !== this.counter) return;
|
|
1242
1263
|
for (const sentence of this.splitter.flush()) {
|
|
1243
1264
|
this.enqueue(counter, sentence);
|
|
1244
1265
|
}
|
|
1266
|
+
};
|
|
1267
|
+
textStream.on("error", (error) => {
|
|
1268
|
+
this.log("Error in text stream", error);
|
|
1269
|
+
flush();
|
|
1245
1270
|
});
|
|
1271
|
+
textStream.on("end", flush);
|
|
1246
1272
|
}
|
|
1247
1273
|
cancel() {
|
|
1248
1274
|
this.generation++;
|