@micdrop/openai 1.1.3 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +9 -0
- package/dist/index.d.mts +3 -1
- package/dist/index.d.ts +3 -1
- package/dist/index.js +15 -8
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +15 -8
- package/dist/index.mjs.map +1 -1
- package/package.json +13 -14
package/LICENSE
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2022 Rolebase
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
|
6
|
+
|
|
7
|
+
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
|
8
|
+
|
|
9
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
package/dist/index.d.mts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { Agent, AgentOptions, STT } from '@micdrop/server';
|
|
2
2
|
import OpenAI from 'openai';
|
|
3
3
|
import { PassThrough, Readable } from 'stream';
|
|
4
4
|
|
|
@@ -36,12 +36,14 @@ interface OpenaiSTTOptions {
|
|
|
36
36
|
prompt?: string;
|
|
37
37
|
transcriptionTimeout?: number;
|
|
38
38
|
retryDelay?: number;
|
|
39
|
+
maxRetry?: number;
|
|
39
40
|
}
|
|
40
41
|
declare class OpenaiSTT extends STT {
|
|
41
42
|
private options;
|
|
42
43
|
private socket?;
|
|
43
44
|
private initPromise;
|
|
44
45
|
private reconnectTimeout?;
|
|
46
|
+
private retryCount;
|
|
45
47
|
private transcriptionTimeout?;
|
|
46
48
|
private audioChunksPending;
|
|
47
49
|
private ephemeralToken?;
|
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { Agent, AgentOptions, STT } from '@micdrop/server';
|
|
2
2
|
import OpenAI from 'openai';
|
|
3
3
|
import { PassThrough, Readable } from 'stream';
|
|
4
4
|
|
|
@@ -36,12 +36,14 @@ interface OpenaiSTTOptions {
|
|
|
36
36
|
prompt?: string;
|
|
37
37
|
transcriptionTimeout?: number;
|
|
38
38
|
retryDelay?: number;
|
|
39
|
+
maxRetry?: number;
|
|
39
40
|
}
|
|
40
41
|
declare class OpenaiSTT extends STT {
|
|
41
42
|
private options;
|
|
42
43
|
private socket?;
|
|
43
44
|
private initPromise;
|
|
44
45
|
private reconnectTimeout?;
|
|
46
|
+
private retryCount;
|
|
45
47
|
private transcriptionTimeout?;
|
|
46
48
|
private audioChunksPending;
|
|
47
49
|
private ephemeralToken?;
|
package/dist/index.js
CHANGED
|
@@ -185,10 +185,12 @@ var SAMPLE_RATE = 16e3;
|
|
|
185
185
|
var BIT_DEPTH = 16;
|
|
186
186
|
var DEFAULT_TRANSCRIPTION_TIMEOUT = 4e3;
|
|
187
187
|
var DEFAULT_RETRY_DELAY2 = 1e3;
|
|
188
|
+
var DEFAULT_MAX_RETRY = 3;
|
|
188
189
|
var OpenaiSTT = class extends import_server2.STT {
|
|
189
190
|
constructor(options) {
|
|
190
191
|
super();
|
|
191
192
|
this.options = options;
|
|
193
|
+
this.retryCount = 0;
|
|
192
194
|
this.audioChunksPending = [];
|
|
193
195
|
this.initPromise = this.createTranscriptionSession().then(() => this.initWS()).catch((error) => {
|
|
194
196
|
console.error("[OpenaiSTT] Connection error:", error);
|
|
@@ -197,10 +199,9 @@ var OpenaiSTT = class extends import_server2.STT {
|
|
|
197
199
|
}
|
|
198
200
|
transcribe(audioStream) {
|
|
199
201
|
audioStream.on("data", async (chunk) => {
|
|
202
|
+
this.audioChunksPending.push(chunk);
|
|
200
203
|
await this.initPromise;
|
|
201
|
-
|
|
202
|
-
this.audioChunksPending.push(base64Audio);
|
|
203
|
-
this.sendAudioChunk(base64Audio);
|
|
204
|
+
this.sendAudioChunk(chunk);
|
|
204
205
|
this.log(`Sent audio chunk (${chunk.byteLength} bytes)`);
|
|
205
206
|
});
|
|
206
207
|
audioStream.on("end", async () => {
|
|
@@ -312,11 +313,11 @@ var OpenaiSTT = class extends import_server2.STT {
|
|
|
312
313
|
});
|
|
313
314
|
});
|
|
314
315
|
}
|
|
315
|
-
sendAudioChunk(
|
|
316
|
+
sendAudioChunk(chunk) {
|
|
316
317
|
if (!this.socket) return;
|
|
317
318
|
const audioMessage = {
|
|
318
319
|
type: "input_audio_buffer.append",
|
|
319
|
-
audio:
|
|
320
|
+
audio: chunk.toString("base64")
|
|
320
321
|
};
|
|
321
322
|
this.socket.send(JSON.stringify(audioMessage));
|
|
322
323
|
}
|
|
@@ -352,11 +353,18 @@ var OpenaiSTT = class extends import_server2.STT {
|
|
|
352
353
|
}
|
|
353
354
|
}
|
|
354
355
|
reconnect() {
|
|
356
|
+
this.retryCount++;
|
|
357
|
+
if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY)) {
|
|
358
|
+
this.log("Max retries reached, giving up");
|
|
359
|
+
this.emit("Failed", this.audioChunksPending);
|
|
360
|
+
return;
|
|
361
|
+
}
|
|
355
362
|
this.initPromise = new Promise((resolve) => {
|
|
356
363
|
this.log("Reconnecting...");
|
|
357
364
|
this.reconnectTimeout = setTimeout(() => {
|
|
358
365
|
this.reconnectTimeout = void 0;
|
|
359
366
|
this.createTranscriptionSession().then(() => this.initWS()).then(() => {
|
|
367
|
+
this.retryCount = 0;
|
|
360
368
|
if (this.audioChunksPending.length > 0) {
|
|
361
369
|
this.log("Sending audio chunks again");
|
|
362
370
|
this.audioChunksPending.forEach(
|
|
@@ -375,9 +383,8 @@ var OpenaiSTT = class extends import_server2.STT {
|
|
|
375
383
|
const numSamples = Math.round(SAMPLE_RATE * durationSeconds);
|
|
376
384
|
const bytesPerSample = BIT_DEPTH / 8;
|
|
377
385
|
const silenceBuffer = Buffer.alloc(numSamples * bytesPerSample);
|
|
378
|
-
|
|
379
|
-
this.
|
|
380
|
-
this.sendAudioChunk(base64Audio);
|
|
386
|
+
this.audioChunksPending.push(silenceBuffer);
|
|
387
|
+
this.sendAudioChunk(silenceBuffer);
|
|
381
388
|
this.log(
|
|
382
389
|
`Sent ${durationSeconds * 1e3}ms of silence (${silenceBuffer.byteLength} bytes) after stream end`
|
|
383
390
|
);
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/index.ts","../src/OpenaiAgent.ts","../src/OpenaiSTT.ts"],"sourcesContent":["export * from './OpenaiAgent'\nexport * from './OpenaiSTT'\n","import { Agent, AgentOptions } from '@micdrop/server'\nimport OpenAI from 'openai'\nimport { PassThrough } from 'stream'\nimport z, { toJSONSchema } from 'zod'\n\nexport type OpenaiAgentOptions = AgentOptions &\n OpenaiOptions & {\n model?: string\n settings?: Omit<\n OpenAI.Responses.ResponseCreateParamsStreaming,\n 'input' | 'model' | 'tools' | 'stream'\n >\n retryDelay?: number\n maxRetry?: number\n maxSteps?: number\n }\n\nexport type OpenaiOptions = { apiKey: string } | { openai: OpenAI }\n\nconst DEFAULT_MODEL = 'gpt-4o'\nconst DEFAULT_MAX_STEPS = 5\nconst DEFAULT_MAX_RETRIES = 3\nconst DEFAULT_RETRY_DELAY = 1000\n\nexport class OpenaiAgent extends Agent<OpenaiAgentOptions> {\n private openai: OpenAI\n private abortController?: AbortController\n\n constructor(options: OpenaiAgentOptions) {\n super(options)\n this.openai =\n 'openai' in options\n ? options.openai\n : new OpenAI({ apiKey: options.apiKey })\n }\n\n protected async generateAnswer(\n stream: PassThrough,\n stepCount = 0,\n tryCount = 0\n ): Promise<void> {\n if (stepCount >= (this.options.maxSteps || DEFAULT_MAX_STEPS)) {\n console.error('[OpenaiAgent] Max steps reached')\n return\n }\n\n const abortController = new AbortController()\n this.abortController = abortController\n const signal = abortController.signal\n\n // Prepare extracting\n let extracting = false\n const extractOptions = this.getExtractOptions()\n\n try {\n // Generate answer\n const response = await this.openai.responses.create(\n {\n model: this.options.model || DEFAULT_MODEL,\n input: this.buildInput(),\n tools: this.buildTools(),\n stream: true,\n ...this.options.settings,\n },\n { signal }\n )\n let skipAnswer = false\n\n // Handle response events\n for await (const event of response) {\n if (abortController !== this.abortController) return\n switch (event.type) {\n case 'response.output_text.delta':\n this.log(`Answer chunk: \"${event.delta}\"`)\n\n // Extracting value?\n if (extractOptions) {\n if (!extracting) {\n const startTagIndex = event.delta.indexOf(\n extractOptions.startTag\n )\n if (startTagIndex !== -1) {\n extracting = true\n const messagePart = event.delta\n .slice(0, startTagIndex)\n .trimEnd()\n stream.write(messagePart)\n break\n }\n } else {\n // Extracting, don't write to stream\n break\n }\n }\n\n // Send chunk\n stream.write(event.delta)\n break\n\n case 'response.output_text.done':\n const { message, metadata } = this.extract(event.text)\n this.addAssistantMessage(message, metadata)\n this.abortController = undefined\n // Stop query now that we have a complete answer\n // Note: OpenAI can sometimes return multiple outputs.\n // See https://community.openai.com/t/how-to-prevent-the-api-returning-multiple-outputs/1251365/28\n // We can prompt it like this to avoid consuming too many tokens:\n // `Return one complete answer and then stop`\n return\n\n case 'response.output_item.done':\n if (event.item.type === 'function_call') {\n const toolCall = event.item\n const result = await this.executeTool({\n role: 'tool_call',\n toolCallId: toolCall.call_id,\n toolName: toolCall.name,\n parameters: toolCall.arguments,\n })\n if (result.skipAnswer) {\n skipAnswer = true\n }\n }\n break\n\n default:\n break\n }\n }\n\n if (abortController !== this.abortController) return\n\n // Query again in case of tool call\n if (!skipAnswer) {\n await this.generateAnswer(stream, stepCount + 1)\n }\n this.abortController = undefined\n } catch (error) {\n if (error instanceof OpenAI.APIUserAbortError) return\n console.error('[OpenaiAgent] Error answering:', error)\n\n if (tryCount < (this.options.maxRetry ?? DEFAULT_MAX_RETRIES)) {\n await new Promise((resolve) =>\n setTimeout(resolve, this.options.retryDelay ?? DEFAULT_RETRY_DELAY)\n )\n await this.generateAnswer(stream, stepCount, tryCount + 1)\n }\n }\n }\n\n private buildInput(): OpenAI.Responses.ResponseInput {\n return this.conversation.map((message) => {\n switch (message.role) {\n case 'user':\n case 'assistant':\n case 'system':\n return {\n role: message.role,\n content: message.content,\n }\n case 'tool_call':\n return {\n type: 'function_call',\n call_id: message.toolCallId,\n name: message.toolName,\n arguments: JSON.stringify(message.parameters),\n }\n case 'tool_result':\n return {\n type: 'function_call_output',\n call_id: message.toolCallId,\n output: message.output,\n }\n }\n })\n }\n\n private buildTools(): OpenAI.Responses.Tool[] | undefined {\n // Disable tools if first message\n const enableTools = this.conversation.length > 1\n if (!enableTools) return undefined\n\n return this.tools.map((tool) => ({\n type: 'function',\n name: tool.name,\n description: tool.description,\n strict: true,\n parameters: toJSONSchema(tool.inputSchema || z.object()),\n }))\n }\n\n cancel() {\n if (!this.abortController) return\n this.log('Cancel')\n this.abortController.abort()\n this.abortController = undefined\n }\n}\n","import { STT } from '@micdrop/server'\nimport { Readable } from 'stream'\nimport WebSocket from 'ws'\n\n/**\n * OpenAI Real-time STT\n *\n * @see https://platform.openai.com/docs/guides/speech-to-text\n */\n\nexport interface OpenaiSTTOptions {\n apiKey: string\n model?: string\n language?: string\n prompt?: string\n transcriptionTimeout?: number\n retryDelay?: number\n}\n\nconst DEFAULT_MODEL = 'gpt-4o-transcribe'\nconst DEFAULT_LANGUAGE = 'en'\nconst SAMPLE_RATE = 16000\nconst BIT_DEPTH = 16\nconst DEFAULT_TRANSCRIPTION_TIMEOUT = 4000\nconst DEFAULT_RETRY_DELAY = 1000\n\nexport class OpenaiSTT extends STT {\n private socket?: WebSocket\n private initPromise: Promise<void>\n private reconnectTimeout?: NodeJS.Timeout\n private transcriptionTimeout?: NodeJS.Timeout\n private audioChunksPending: string[] = [] // Store Base64 encoded audio chunks\n private ephemeralToken?: string\n\n constructor(private options: OpenaiSTTOptions) {\n super()\n\n // Setup WebSocket connection (first create session, then connect)\n this.initPromise = this.createTranscriptionSession()\n .then(() => this.initWS())\n .catch((error) => {\n console.error('[OpenaiSTT] Connection error:', error)\n this.reconnect()\n })\n }\n\n transcribe(audioStream: Readable) {\n // Read audio stream and send to OpenAI\n audioStream.on('data', async (chunk: Buffer) => {\n await this.initPromise\n const base64Audio = chunk.toString('base64')\n this.audioChunksPending.push(base64Audio)\n this.sendAudioChunk(base64Audio)\n this.log(`Sent audio chunk (${chunk.byteLength} bytes)`)\n })\n\n // Handle stream end\n audioStream.on('end', async () => {\n await this.initPromise\n if (this.audioChunksPending.length === 0) return\n this.sendSilence(2)\n\n // Timeout transcription if no transcript is received\n this.transcriptionTimeout = setTimeout(() => {\n this.transcriptionTimeout = undefined\n this.log('Transcription timeout')\n this.emit('Transcript', '')\n this.audioChunksPending.length = 0\n }, this.options.transcriptionTimeout || DEFAULT_TRANSCRIPTION_TIMEOUT)\n })\n }\n\n destroy() {\n super.destroy()\n if (this.reconnectTimeout) {\n clearTimeout(this.reconnectTimeout)\n this.reconnectTimeout = undefined\n }\n if (this.transcriptionTimeout) {\n clearTimeout(this.transcriptionTimeout)\n this.transcriptionTimeout = undefined\n }\n this.socket?.removeAllListeners()\n if (this.socket?.readyState === WebSocket.OPEN) {\n this.socket?.close(1000)\n }\n this.socket = undefined\n }\n\n private async createTranscriptionSession(): Promise<void> {\n const payload = {\n input_audio_format: 'pcm16',\n input_audio_transcription: {\n model: this.options.model || DEFAULT_MODEL,\n language: this.options.language || DEFAULT_LANGUAGE,\n prompt:\n this.options.prompt || 'Transcribe the incoming audio in real time.',\n },\n turn_detection: {\n type: 'server_vad',\n silence_duration_ms: 1000,\n },\n input_audio_noise_reduction: {\n type: 'near_field',\n },\n }\n\n const response = await fetch(\n 'https://api.openai.com/v1/realtime/transcription_sessions',\n {\n method: 'POST',\n headers: {\n Authorization: `Bearer ${this.options.apiKey}`,\n 'Content-Type': 'application/json',\n 'OpenAI-Beta': 'realtime=v1',\n },\n body: JSON.stringify(payload),\n }\n )\n\n if (!response.ok) {\n const status = response.status\n\n // Don't retry on 4xx errors\n if (status >= 400 && status < 500) {\n throw new Error(\n `Failed to create transcription session: ${status} ${response.statusText}`\n )\n }\n\n // Retry on other errors\n this.log('Error creating transcription session, retrying...', {\n status,\n text: response.statusText,\n })\n await new Promise((resolve) =>\n setTimeout(resolve, this.options.retryDelay ?? DEFAULT_RETRY_DELAY)\n )\n return this.createTranscriptionSession()\n }\n\n const data = (await response.json()) as { client_secret: { value: string } }\n this.ephemeralToken = data.client_secret.value\n this.log('Transcription session created')\n }\n\n private async initWS(): Promise<void> {\n return new Promise<void>((resolve) => {\n const socket = new WebSocket('wss://api.openai.com/v1/realtime', {\n headers: {\n Authorization: `Bearer ${this.ephemeralToken}`,\n 'OpenAI-Beta': 'realtime=v1',\n },\n })\n this.socket = socket\n\n socket.addEventListener('open', () => {\n this.log('Connection opened')\n resolve()\n })\n\n socket.addEventListener('error', (error: any) => {\n this.log('WebSocket error:', error)\n })\n\n socket.addEventListener(\n 'close',\n ({ code, reason }: { code: number; reason: string }) => {\n this.socket?.removeAllListeners()\n this.socket = undefined\n\n if (code !== 1000) {\n this.reconnect()\n } else {\n this.log('Connection closed', { code, reason })\n }\n }\n )\n\n socket.addEventListener('message', (event: any) => {\n this.handleMessage(JSON.parse(event.data.toString()))\n })\n })\n }\n\n private sendAudioChunk(base64Audio: string) {\n if (!this.socket) return\n\n const audioMessage = {\n type: 'input_audio_buffer.append',\n audio: base64Audio,\n }\n\n this.socket.send(JSON.stringify(audioMessage))\n }\n\n private handleMessage(message: any) {\n switch (message.type) {\n // case 'input_audio_buffer.committed':\n // this.log('Audio buffer committed')\n // break\n\n // case 'input_audio_buffer.speech_started':\n // this.log('Speech started')\n // break\n\n // case 'input_audio_buffer.speech_stopped':\n // this.log('Speech stopped')\n // break\n\n // case 'conversation.item.input_audio_transcription.delta':\n // this.log(`Received transcript delta: \"${message.delta}\"`)\n // break\n\n case 'conversation.item.input_audio_transcription.completed':\n const transcript = message.transcript || ''\n this.log(`Received completed transcript: \"${transcript}\"`)\n this.emit('Transcript', transcript)\n // Reset audio chunks and clear timeout\n this.audioChunksPending.length = 0\n if (this.transcriptionTimeout) {\n clearTimeout(this.transcriptionTimeout)\n this.transcriptionTimeout = undefined\n }\n break\n\n case 'error':\n this.log('Error:', message.error)\n break\n\n default:\n break\n }\n }\n\n private reconnect() {\n this.initPromise = new Promise((resolve) => {\n this.log('Reconnecting...')\n this.reconnectTimeout = setTimeout(() => {\n this.reconnectTimeout = undefined\n this.createTranscriptionSession()\n .then(() => this.initWS())\n .then(() => {\n // Send audio chunks again if reconnecting during transcription\n if (this.audioChunksPending.length > 0) {\n this.log('Sending audio chunks again')\n this.audioChunksPending.forEach((chunk) =>\n this.sendAudioChunk(chunk)\n )\n }\n })\n .then(resolve)\n .catch((error) => {\n this.log('Reconnection error:', error)\n this.reconnect()\n })\n }, this.options.retryDelay ?? DEFAULT_RETRY_DELAY)\n })\n }\n\n private sendSilence(durationSeconds: number) {\n if (!this.socket) return\n const numSamples = Math.round(SAMPLE_RATE * durationSeconds)\n const bytesPerSample = BIT_DEPTH / 8\n const silenceBuffer = Buffer.alloc(numSamples * bytesPerSample)\n const base64Audio = silenceBuffer.toString('base64')\n this.audioChunksPending.push(base64Audio)\n this.sendAudioChunk(base64Audio)\n this.log(\n `Sent ${durationSeconds * 1000}ms of silence (${silenceBuffer.byteLength} bytes) after stream end`\n )\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACAA,oBAAoC;AACpC,oBAAmB;AAEnB,iBAAgC;AAgBhC,IAAM,gBAAgB;AACtB,IAAM,oBAAoB;AAC1B,IAAM,sBAAsB;AAC5B,IAAM,sBAAsB;AAErB,IAAM,cAAN,cAA0B,oBAA0B;AAAA,EAIzD,YAAY,SAA6B;AACvC,UAAM,OAAO;AACb,SAAK,SACH,YAAY,UACR,QAAQ,SACR,IAAI,cAAAA,QAAO,EAAE,QAAQ,QAAQ,OAAO,CAAC;AAAA,EAC7C;AAAA,EAEA,MAAgB,eACd,QACA,YAAY,GACZ,WAAW,GACI;AACf,QAAI,cAAc,KAAK,QAAQ,YAAY,oBAAoB;AAC7D,cAAQ,MAAM,iCAAiC;AAC/C;AAAA,IACF;AAEA,UAAM,kBAAkB,IAAI,gBAAgB;AAC5C,SAAK,kBAAkB;AACvB,UAAM,SAAS,gBAAgB;AAG/B,QAAI,aAAa;AACjB,UAAM,iBAAiB,KAAK,kBAAkB;AAE9C,QAAI;AAEF,YAAM,WAAW,MAAM,KAAK,OAAO,UAAU;AAAA,QAC3C;AAAA,UACE,OAAO,KAAK,QAAQ,SAAS;AAAA,UAC7B,OAAO,KAAK,WAAW;AAAA,UACvB,OAAO,KAAK,WAAW;AAAA,UACvB,QAAQ;AAAA,UACR,GAAG,KAAK,QAAQ;AAAA,QAClB;AAAA,QACA,EAAE,OAAO;AAAA,MACX;AACA,UAAI,aAAa;AAGjB,uBAAiB,SAAS,UAAU;AAClC,YAAI,oBAAoB,KAAK,gBAAiB;AAC9C,gBAAQ,MAAM,MAAM;AAAA,UAClB,KAAK;AACH,iBAAK,IAAI,kBAAkB,MAAM,KAAK,GAAG;AAGzC,gBAAI,gBAAgB;AAClB,kBAAI,CAAC,YAAY;AACf,sBAAM,gBAAgB,MAAM,MAAM;AAAA,kBAChC,eAAe;AAAA,gBACjB;AACA,oBAAI,kBAAkB,IAAI;AACxB,+BAAa;AACb,wBAAM,cAAc,MAAM,MACvB,MAAM,GAAG,aAAa,EACtB,QAAQ;AACX,yBAAO,MAAM,WAAW;AACxB;AAAA,gBACF;AAAA,cACF,OAAO;AAEL;AAAA,cACF;AAAA,YACF;AAGA,mBAAO,MAAM,MAAM,KAAK;AACxB;AAAA,UAEF,KAAK;AACH,kBAAM,EAAE,SAAS,SAAS,IAAI,KAAK,QAAQ,MAAM,IAAI;AACrD,iBAAK,oBAAoB,SAAS,QAAQ;AAC1C,iBAAK,kBAAkB;AAMvB;AAAA,UAEF,KAAK;AACH,gBAAI,MAAM,KAAK,SAAS,iBAAiB;AACvC,oBAAM,WAAW,MAAM;AACvB,oBAAM,SAAS,MAAM,KAAK,YAAY;AAAA,gBACpC,MAAM;AAAA,gBACN,YAAY,SAAS;AAAA,gBACrB,UAAU,SAAS;AAAA,gBACnB,YAAY,SAAS;AAAA,cACvB,CAAC;AACD,kBAAI,OAAO,YAAY;AACrB,6BAAa;AAAA,cACf;AAAA,YACF;AACA;AAAA,UAEF;AACE;AAAA,QACJ;AAAA,MACF;AAEA,UAAI,oBAAoB,KAAK,gBAAiB;AAG9C,UAAI,CAAC,YAAY;AACf,cAAM,KAAK,eAAe,QAAQ,YAAY,CAAC;AAAA,MACjD;AACA,WAAK,kBAAkB;AAAA,IACzB,SAAS,OAAO;AACd,UAAI,iBAAiB,cAAAA,QAAO,kBAAmB;AAC/C,cAAQ,MAAM,kCAAkC,KAAK;AAErD,UAAI,YAAY,KAAK,QAAQ,YAAY,sBAAsB;AAC7D,cAAM,IAAI;AAAA,UAAQ,CAAC,YACjB,WAAW,SAAS,KAAK,QAAQ,cAAc,mBAAmB;AAAA,QACpE;AACA,cAAM,KAAK,eAAe,QAAQ,WAAW,WAAW,CAAC;AAAA,MAC3D;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,aAA6C;AACnD,WAAO,KAAK,aAAa,IAAI,CAAC,YAAY;AACxC,cAAQ,QAAQ,MAAM;AAAA,QACpB,KAAK;AAAA,QACL,KAAK;AAAA,QACL,KAAK;AACH,iBAAO;AAAA,YACL,MAAM,QAAQ;AAAA,YACd,SAAS,QAAQ;AAAA,UACnB;AAAA,QACF,KAAK;AACH,iBAAO;AAAA,YACL,MAAM;AAAA,YACN,SAAS,QAAQ;AAAA,YACjB,MAAM,QAAQ;AAAA,YACd,WAAW,KAAK,UAAU,QAAQ,UAAU;AAAA,UAC9C;AAAA,QACF,KAAK;AACH,iBAAO;AAAA,YACL,MAAM;AAAA,YACN,SAAS,QAAQ;AAAA,YACjB,QAAQ,QAAQ;AAAA,UAClB;AAAA,MACJ;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEQ,aAAkD;AAExD,UAAM,cAAc,KAAK,aAAa,SAAS;AAC/C,QAAI,CAAC,YAAa,QAAO;AAEzB,WAAO,KAAK,MAAM,IAAI,CAAC,UAAU;AAAA,MAC/B,MAAM;AAAA,MACN,MAAM,KAAK;AAAA,MACX,aAAa,KAAK;AAAA,MAClB,QAAQ;AAAA,MACR,gBAAY,yBAAa,KAAK,eAAe,WAAAC,QAAE,OAAO,CAAC;AAAA,IACzD,EAAE;AAAA,EACJ;AAAA,EAEA,SAAS;AACP,QAAI,CAAC,KAAK,gBAAiB;AAC3B,SAAK,IAAI,QAAQ;AACjB,SAAK,gBAAgB,MAAM;AAC3B,SAAK,kBAAkB;AAAA,EACzB;AACF;;;ACrMA,IAAAC,iBAAoB;AAEpB,gBAAsB;AAiBtB,IAAMC,iBAAgB;AACtB,IAAM,mBAAmB;AACzB,IAAM,cAAc;AACpB,IAAM,YAAY;AAClB,IAAM,gCAAgC;AACtC,IAAMC,uBAAsB;AAErB,IAAM,YAAN,cAAwB,mBAAI;AAAA,EAQjC,YAAoB,SAA2B;AAC7C,UAAM;AADY;AAHpB,SAAQ,qBAA+B,CAAC;AAOtC,SAAK,cAAc,KAAK,2BAA2B,EAChD,KAAK,MAAM,KAAK,OAAO,CAAC,EACxB,MAAM,CAAC,UAAU;AAChB,cAAQ,MAAM,iCAAiC,KAAK;AACpD,WAAK,UAAU;AAAA,IACjB,CAAC;AAAA,EACL;AAAA,EAEA,WAAW,aAAuB;AAEhC,gBAAY,GAAG,QAAQ,OAAO,UAAkB;AAC9C,YAAM,KAAK;AACX,YAAM,cAAc,MAAM,SAAS,QAAQ;AAC3C,WAAK,mBAAmB,KAAK,WAAW;AACxC,WAAK,eAAe,WAAW;AAC/B,WAAK,IAAI,qBAAqB,MAAM,UAAU,SAAS;AAAA,IACzD,CAAC;AAGD,gBAAY,GAAG,OAAO,YAAY;AAChC,YAAM,KAAK;AACX,UAAI,KAAK,mBAAmB,WAAW,EAAG;AAC1C,WAAK,YAAY,CAAC;AAGlB,WAAK,uBAAuB,WAAW,MAAM;AAC3C,aAAK,uBAAuB;AAC5B,aAAK,IAAI,uBAAuB;AAChC,aAAK,KAAK,cAAc,EAAE;AAC1B,aAAK,mBAAmB,SAAS;AAAA,MACnC,GAAG,KAAK,QAAQ,wBAAwB,6BAA6B;AAAA,IACvE,CAAC;AAAA,EACH;AAAA,EAEA,UAAU;AACR,UAAM,QAAQ;AACd,QAAI,KAAK,kBAAkB;AACzB,mBAAa,KAAK,gBAAgB;AAClC,WAAK,mBAAmB;AAAA,IAC1B;AACA,QAAI,KAAK,sBAAsB;AAC7B,mBAAa,KAAK,oBAAoB;AACtC,WAAK,uBAAuB;AAAA,IAC9B;AACA,SAAK,QAAQ,mBAAmB;AAChC,QAAI,KAAK,QAAQ,eAAe,UAAAC,QAAU,MAAM;AAC9C,WAAK,QAAQ,MAAM,GAAI;AAAA,IACzB;AACA,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,MAAc,6BAA4C;AACxD,UAAM,UAAU;AAAA,MACd,oBAAoB;AAAA,MACpB,2BAA2B;AAAA,QACzB,OAAO,KAAK,QAAQ,SAASF;AAAA,QAC7B,UAAU,KAAK,QAAQ,YAAY;AAAA,QACnC,QACE,KAAK,QAAQ,UAAU;AAAA,MAC3B;AAAA,MACA,gBAAgB;AAAA,QACd,MAAM;AAAA,QACN,qBAAqB;AAAA,MACvB;AAAA,MACA,6BAA6B;AAAA,QAC3B,MAAM;AAAA,MACR;AAAA,IACF;AAEA,UAAM,WAAW,MAAM;AAAA,MACrB;AAAA,MACA;AAAA,QACE,QAAQ;AAAA,QACR,SAAS;AAAA,UACP,eAAe,UAAU,KAAK,QAAQ,MAAM;AAAA,UAC5C,gBAAgB;AAAA,UAChB,eAAe;AAAA,QACjB;AAAA,QACA,MAAM,KAAK,UAAU,OAAO;AAAA,MAC9B;AAAA,IACF;AAEA,QAAI,CAAC,SAAS,IAAI;AAChB,YAAM,SAAS,SAAS;AAGxB,UAAI,UAAU,OAAO,SAAS,KAAK;AACjC,cAAM,IAAI;AAAA,UACR,2CAA2C,MAAM,IAAI,SAAS,UAAU;AAAA,QAC1E;AAAA,MACF;AAGA,WAAK,IAAI,qDAAqD;AAAA,QAC5D;AAAA,QACA,MAAM,SAAS;AAAA,MACjB,CAAC;AACD,YAAM,IAAI;AAAA,QAAQ,CAAC,YACjB,WAAW,SAAS,KAAK,QAAQ,cAAcC,oBAAmB;AAAA,MACpE;AACA,aAAO,KAAK,2BAA2B;AAAA,IACzC;AAEA,UAAM,OAAQ,MAAM,SAAS,KAAK;AAClC,SAAK,iBAAiB,KAAK,cAAc;AACzC,SAAK,IAAI,+BAA+B;AAAA,EAC1C;AAAA,EAEA,MAAc,SAAwB;AACpC,WAAO,IAAI,QAAc,CAAC,YAAY;AACpC,YAAM,SAAS,IAAI,UAAAC,QAAU,oCAAoC;AAAA,QAC/D,SAAS;AAAA,UACP,eAAe,UAAU,KAAK,cAAc;AAAA,UAC5C,eAAe;AAAA,QACjB;AAAA,MACF,CAAC;AACD,WAAK,SAAS;AAEd,aAAO,iBAAiB,QAAQ,MAAM;AACpC,aAAK,IAAI,mBAAmB;AAC5B,gBAAQ;AAAA,MACV,CAAC;AAED,aAAO,iBAAiB,SAAS,CAAC,UAAe;AAC/C,aAAK,IAAI,oBAAoB,KAAK;AAAA,MACpC,CAAC;AAED,aAAO;AAAA,QACL;AAAA,QACA,CAAC,EAAE,MAAM,OAAO,MAAwC;AACtD,eAAK,QAAQ,mBAAmB;AAChC,eAAK,SAAS;AAEd,cAAI,SAAS,KAAM;AACjB,iBAAK,UAAU;AAAA,UACjB,OAAO;AACL,iBAAK,IAAI,qBAAqB,EAAE,MAAM,OAAO,CAAC;AAAA,UAChD;AAAA,QACF;AAAA,MACF;AAEA,aAAO,iBAAiB,WAAW,CAAC,UAAe;AACjD,aAAK,cAAc,KAAK,MAAM,MAAM,KAAK,SAAS,CAAC,CAAC;AAAA,MACtD,CAAC;AAAA,IACH,CAAC;AAAA,EACH;AAAA,EAEQ,eAAe,aAAqB;AAC1C,QAAI,CAAC,KAAK,OAAQ;AAElB,UAAM,eAAe;AAAA,MACnB,MAAM;AAAA,MACN,OAAO;AAAA,IACT;AAEA,SAAK,OAAO,KAAK,KAAK,UAAU,YAAY,CAAC;AAAA,EAC/C;AAAA,EAEQ,cAAc,SAAc;AAClC,YAAQ,QAAQ,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAiBpB,KAAK;AACH,cAAM,aAAa,QAAQ,cAAc;AACzC,aAAK,IAAI,mCAAmC,UAAU,GAAG;AACzD,aAAK,KAAK,cAAc,UAAU;AAElC,aAAK,mBAAmB,SAAS;AACjC,YAAI,KAAK,sBAAsB;AAC7B,uBAAa,KAAK,oBAAoB;AACtC,eAAK,uBAAuB;AAAA,QAC9B;AACA;AAAA,MAEF,KAAK;AACH,aAAK,IAAI,UAAU,QAAQ,KAAK;AAChC;AAAA,MAEF;AACE;AAAA,IACJ;AAAA,EACF;AAAA,EAEQ,YAAY;AAClB,SAAK,cAAc,IAAI,QAAQ,CAAC,YAAY;AAC1C,WAAK,IAAI,iBAAiB;AAC1B,WAAK,mBAAmB,WAAW,MAAM;AACvC,aAAK,mBAAmB;AACxB,aAAK,2BAA2B,EAC7B,KAAK,MAAM,KAAK,OAAO,CAAC,EACxB,KAAK,MAAM;AAEV,cAAI,KAAK,mBAAmB,SAAS,GAAG;AACtC,iBAAK,IAAI,4BAA4B;AACrC,iBAAK,mBAAmB;AAAA,cAAQ,CAAC,UAC/B,KAAK,eAAe,KAAK;AAAA,YAC3B;AAAA,UACF;AAAA,QACF,CAAC,EACA,KAAK,OAAO,EACZ,MAAM,CAAC,UAAU;AAChB,eAAK,IAAI,uBAAuB,KAAK;AACrC,eAAK,UAAU;AAAA,QACjB,CAAC;AAAA,MACL,GAAG,KAAK,QAAQ,cAAcD,oBAAmB;AAAA,IACnD,CAAC;AAAA,EACH;AAAA,EAEQ,YAAY,iBAAyB;AAC3C,QAAI,CAAC,KAAK,OAAQ;AAClB,UAAM,aAAa,KAAK,MAAM,cAAc,eAAe;AAC3D,UAAM,iBAAiB,YAAY;AACnC,UAAM,gBAAgB,OAAO,MAAM,aAAa,cAAc;AAC9D,UAAM,cAAc,cAAc,SAAS,QAAQ;AACnD,SAAK,mBAAmB,KAAK,WAAW;AACxC,SAAK,eAAe,WAAW;AAC/B,SAAK;AAAA,MACH,QAAQ,kBAAkB,GAAI,kBAAkB,cAAc,UAAU;AAAA,IAC1E;AAAA,EACF;AACF;","names":["OpenAI","z","import_server","DEFAULT_MODEL","DEFAULT_RETRY_DELAY","WebSocket"]}
|
|
1
|
+
{"version":3,"sources":["../src/index.ts","../src/OpenaiAgent.ts","../src/OpenaiSTT.ts"],"sourcesContent":["export * from './OpenaiAgent'\nexport * from './OpenaiSTT'\n","import { Agent, AgentOptions } from '@micdrop/server'\nimport OpenAI from 'openai'\nimport { PassThrough } from 'stream'\nimport z, { toJSONSchema } from 'zod'\n\nexport type OpenaiAgentOptions = AgentOptions &\n OpenaiOptions & {\n model?: string\n settings?: Omit<\n OpenAI.Responses.ResponseCreateParamsStreaming,\n 'input' | 'model' | 'tools' | 'stream'\n >\n retryDelay?: number\n maxRetry?: number\n maxSteps?: number\n }\n\nexport type OpenaiOptions = { apiKey: string } | { openai: OpenAI }\n\nconst DEFAULT_MODEL = 'gpt-4o'\nconst DEFAULT_MAX_STEPS = 5\nconst DEFAULT_MAX_RETRIES = 3\nconst DEFAULT_RETRY_DELAY = 1000\n\nexport class OpenaiAgent extends Agent<OpenaiAgentOptions> {\n private openai: OpenAI\n private abortController?: AbortController\n\n constructor(options: OpenaiAgentOptions) {\n super(options)\n this.openai =\n 'openai' in options\n ? options.openai\n : new OpenAI({ apiKey: options.apiKey })\n }\n\n protected async generateAnswer(\n stream: PassThrough,\n stepCount = 0,\n tryCount = 0\n ): Promise<void> {\n if (stepCount >= (this.options.maxSteps || DEFAULT_MAX_STEPS)) {\n console.error('[OpenaiAgent] Max steps reached')\n return\n }\n\n const abortController = new AbortController()\n this.abortController = abortController\n const signal = abortController.signal\n\n // Prepare extracting\n let extracting = false\n const extractOptions = this.getExtractOptions()\n\n try {\n // Generate answer\n const response = await this.openai.responses.create(\n {\n model: this.options.model || DEFAULT_MODEL,\n input: this.buildInput(),\n tools: this.buildTools(),\n stream: true,\n ...this.options.settings,\n },\n { signal }\n )\n let skipAnswer = false\n\n // Handle response events\n for await (const event of response) {\n if (abortController !== this.abortController) return\n switch (event.type) {\n case 'response.output_text.delta':\n this.log(`Answer chunk: \"${event.delta}\"`)\n\n // Extracting value?\n if (extractOptions) {\n if (!extracting) {\n const startTagIndex = event.delta.indexOf(\n extractOptions.startTag\n )\n if (startTagIndex !== -1) {\n extracting = true\n const messagePart = event.delta\n .slice(0, startTagIndex)\n .trimEnd()\n stream.write(messagePart)\n break\n }\n } else {\n // Extracting, don't write to stream\n break\n }\n }\n\n // Send chunk\n stream.write(event.delta)\n break\n\n case 'response.output_text.done':\n const { message, metadata } = this.extract(event.text)\n this.addAssistantMessage(message, metadata)\n this.abortController = undefined\n // Stop query now that we have a complete answer\n // Note: OpenAI can sometimes return multiple outputs.\n // See https://community.openai.com/t/how-to-prevent-the-api-returning-multiple-outputs/1251365/28\n // We can prompt it like this to avoid consuming too many tokens:\n // `Return one complete answer and then stop`\n return\n\n case 'response.output_item.done':\n if (event.item.type === 'function_call') {\n const toolCall = event.item\n const result = await this.executeTool({\n role: 'tool_call',\n toolCallId: toolCall.call_id,\n toolName: toolCall.name,\n parameters: toolCall.arguments,\n })\n if (result.skipAnswer) {\n skipAnswer = true\n }\n }\n break\n\n default:\n break\n }\n }\n\n if (abortController !== this.abortController) return\n\n // Query again in case of tool call\n if (!skipAnswer) {\n await this.generateAnswer(stream, stepCount + 1)\n }\n this.abortController = undefined\n } catch (error) {\n if (error instanceof OpenAI.APIUserAbortError) return\n console.error('[OpenaiAgent] Error answering:', error)\n\n if (tryCount < (this.options.maxRetry ?? DEFAULT_MAX_RETRIES)) {\n await new Promise((resolve) =>\n setTimeout(resolve, this.options.retryDelay ?? DEFAULT_RETRY_DELAY)\n )\n await this.generateAnswer(stream, stepCount, tryCount + 1)\n }\n }\n }\n\n private buildInput(): OpenAI.Responses.ResponseInput {\n return this.conversation.map((message) => {\n switch (message.role) {\n case 'user':\n case 'assistant':\n case 'system':\n return {\n role: message.role,\n content: message.content,\n }\n case 'tool_call':\n return {\n type: 'function_call',\n call_id: message.toolCallId,\n name: message.toolName,\n arguments: JSON.stringify(message.parameters),\n }\n case 'tool_result':\n return {\n type: 'function_call_output',\n call_id: message.toolCallId,\n output: message.output,\n }\n }\n })\n }\n\n private buildTools(): OpenAI.Responses.Tool[] | undefined {\n // Disable tools if first message\n const enableTools = this.conversation.length > 1\n if (!enableTools) return undefined\n\n return this.tools.map((tool) => ({\n type: 'function',\n name: tool.name,\n description: tool.description,\n strict: true,\n parameters: toJSONSchema(tool.inputSchema || z.object()),\n }))\n }\n\n cancel() {\n if (!this.abortController) return\n this.log('Cancel')\n this.abortController.abort()\n this.abortController = undefined\n }\n}\n","import { STT } from '@micdrop/server'\nimport { Readable } from 'stream'\nimport WebSocket from 'ws'\n\n/**\n * OpenAI Real-time STT\n *\n * @see https://platform.openai.com/docs/guides/speech-to-text\n */\n\nexport interface OpenaiSTTOptions {\n apiKey: string\n model?: string\n language?: string\n prompt?: string\n transcriptionTimeout?: number\n retryDelay?: number\n maxRetry?: number\n}\n\nconst DEFAULT_MODEL = 'gpt-4o-transcribe'\nconst DEFAULT_LANGUAGE = 'en'\nconst SAMPLE_RATE = 16000\nconst BIT_DEPTH = 16\nconst DEFAULT_TRANSCRIPTION_TIMEOUT = 4000\nconst DEFAULT_RETRY_DELAY = 1000\nconst DEFAULT_MAX_RETRY = 3\n\nexport class OpenaiSTT extends STT {\n private socket?: WebSocket\n private initPromise: Promise<void>\n private reconnectTimeout?: NodeJS.Timeout\n private retryCount = 0\n private transcriptionTimeout?: NodeJS.Timeout\n private audioChunksPending: Buffer[] = [] // Store audio chunks to send them again if reconnecting\n private ephemeralToken?: string\n\n constructor(private options: OpenaiSTTOptions) {\n super()\n\n // Setup WebSocket connection (first create session, then connect)\n this.initPromise = this.createTranscriptionSession()\n .then(() => this.initWS())\n .catch((error) => {\n console.error('[OpenaiSTT] Connection error:', error)\n this.reconnect()\n })\n }\n\n transcribe(audioStream: Readable) {\n // Read audio stream and send to OpenAI\n audioStream.on('data', async (chunk: Buffer) => {\n this.audioChunksPending.push(chunk)\n await this.initPromise\n this.sendAudioChunk(chunk)\n this.log(`Sent audio chunk (${chunk.byteLength} bytes)`)\n })\n\n // Handle stream end\n audioStream.on('end', async () => {\n await this.initPromise\n if (this.audioChunksPending.length === 0) return\n this.sendSilence(2)\n\n // Timeout transcription if no transcript is received\n this.transcriptionTimeout = setTimeout(() => {\n this.transcriptionTimeout = undefined\n this.log('Transcription timeout')\n this.emit('Transcript', '')\n this.audioChunksPending.length = 0\n }, this.options.transcriptionTimeout || DEFAULT_TRANSCRIPTION_TIMEOUT)\n })\n }\n\n destroy() {\n super.destroy()\n if (this.reconnectTimeout) {\n clearTimeout(this.reconnectTimeout)\n this.reconnectTimeout = undefined\n }\n if (this.transcriptionTimeout) {\n clearTimeout(this.transcriptionTimeout)\n this.transcriptionTimeout = undefined\n }\n this.socket?.removeAllListeners()\n if (this.socket?.readyState === WebSocket.OPEN) {\n this.socket?.close(1000)\n }\n this.socket = undefined\n }\n\n private async createTranscriptionSession(): Promise<void> {\n const payload = {\n input_audio_format: 'pcm16',\n input_audio_transcription: {\n model: this.options.model || DEFAULT_MODEL,\n language: this.options.language || DEFAULT_LANGUAGE,\n prompt:\n this.options.prompt || 'Transcribe the incoming audio in real time.',\n },\n turn_detection: {\n type: 'server_vad',\n silence_duration_ms: 1000,\n },\n input_audio_noise_reduction: {\n type: 'near_field',\n },\n }\n\n const response = await fetch(\n 'https://api.openai.com/v1/realtime/transcription_sessions',\n {\n method: 'POST',\n headers: {\n Authorization: `Bearer ${this.options.apiKey}`,\n 'Content-Type': 'application/json',\n 'OpenAI-Beta': 'realtime=v1',\n },\n body: JSON.stringify(payload),\n }\n )\n\n if (!response.ok) {\n const status = response.status\n\n // Don't retry on 4xx errors\n if (status >= 400 && status < 500) {\n throw new Error(\n `Failed to create transcription session: ${status} ${response.statusText}`\n )\n }\n\n // Retry on other errors\n this.log('Error creating transcription session, retrying...', {\n status,\n text: response.statusText,\n })\n await new Promise((resolve) =>\n setTimeout(resolve, this.options.retryDelay ?? DEFAULT_RETRY_DELAY)\n )\n return this.createTranscriptionSession()\n }\n\n const data = (await response.json()) as { client_secret: { value: string } }\n this.ephemeralToken = data.client_secret.value\n this.log('Transcription session created')\n }\n\n private async initWS(): Promise<void> {\n return new Promise<void>((resolve) => {\n const socket = new WebSocket('wss://api.openai.com/v1/realtime', {\n headers: {\n Authorization: `Bearer ${this.ephemeralToken}`,\n 'OpenAI-Beta': 'realtime=v1',\n },\n })\n this.socket = socket\n\n socket.addEventListener('open', () => {\n this.log('Connection opened')\n resolve()\n })\n\n socket.addEventListener('error', (error: any) => {\n this.log('WebSocket error:', error)\n })\n\n socket.addEventListener(\n 'close',\n ({ code, reason }: { code: number; reason: string }) => {\n this.socket?.removeAllListeners()\n this.socket = undefined\n\n if (code !== 1000) {\n this.reconnect()\n } else {\n this.log('Connection closed', { code, reason })\n }\n }\n )\n\n socket.addEventListener('message', (event: any) => {\n this.handleMessage(JSON.parse(event.data.toString()))\n })\n })\n }\n\n private sendAudioChunk(chunk: Buffer) {\n if (!this.socket) return\n\n const audioMessage = {\n type: 'input_audio_buffer.append',\n audio: chunk.toString('base64'),\n }\n\n this.socket.send(JSON.stringify(audioMessage))\n }\n\n private handleMessage(message: any) {\n switch (message.type) {\n // case 'input_audio_buffer.committed':\n // this.log('Audio buffer committed')\n // break\n\n // case 'input_audio_buffer.speech_started':\n // this.log('Speech started')\n // break\n\n // case 'input_audio_buffer.speech_stopped':\n // this.log('Speech stopped')\n // break\n\n // case 'conversation.item.input_audio_transcription.delta':\n // this.log(`Received transcript delta: \"${message.delta}\"`)\n // break\n\n case 'conversation.item.input_audio_transcription.completed':\n const transcript = message.transcript || ''\n this.log(`Received completed transcript: \"${transcript}\"`)\n this.emit('Transcript', transcript)\n // Reset audio chunks and clear timeout\n this.audioChunksPending.length = 0\n if (this.transcriptionTimeout) {\n clearTimeout(this.transcriptionTimeout)\n this.transcriptionTimeout = undefined\n }\n break\n\n case 'error':\n this.log('Error:', message.error)\n break\n\n default:\n break\n }\n }\n\n private reconnect() {\n this.retryCount++\n if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY)) {\n this.log('Max retries reached, giving up')\n this.emit('Failed', this.audioChunksPending)\n return\n }\n\n this.initPromise = new Promise((resolve) => {\n this.log('Reconnecting...')\n this.reconnectTimeout = setTimeout(() => {\n this.reconnectTimeout = undefined\n this.createTranscriptionSession()\n .then(() => this.initWS())\n .then(() => {\n this.retryCount = 0\n\n // Send audio chunks again if reconnecting during transcription\n if (this.audioChunksPending.length > 0) {\n this.log('Sending audio chunks again')\n this.audioChunksPending.forEach((chunk) =>\n this.sendAudioChunk(chunk)\n )\n }\n })\n .then(resolve)\n .catch((error) => {\n this.log('Reconnection error:', error)\n this.reconnect()\n })\n }, this.options.retryDelay ?? DEFAULT_RETRY_DELAY)\n })\n }\n\n private sendSilence(durationSeconds: number) {\n if (!this.socket) return\n const numSamples = Math.round(SAMPLE_RATE * durationSeconds)\n const bytesPerSample = BIT_DEPTH / 8\n const silenceBuffer = Buffer.alloc(numSamples * bytesPerSample)\n this.audioChunksPending.push(silenceBuffer)\n this.sendAudioChunk(silenceBuffer)\n this.log(\n `Sent ${durationSeconds * 1000}ms of silence (${silenceBuffer.byteLength} bytes) after stream end`\n )\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACAA,oBAAoC;AACpC,oBAAmB;AAEnB,iBAAgC;AAgBhC,IAAM,gBAAgB;AACtB,IAAM,oBAAoB;AAC1B,IAAM,sBAAsB;AAC5B,IAAM,sBAAsB;AAErB,IAAM,cAAN,cAA0B,oBAA0B;AAAA,EAIzD,YAAY,SAA6B;AACvC,UAAM,OAAO;AACb,SAAK,SACH,YAAY,UACR,QAAQ,SACR,IAAI,cAAAA,QAAO,EAAE,QAAQ,QAAQ,OAAO,CAAC;AAAA,EAC7C;AAAA,EAEA,MAAgB,eACd,QACA,YAAY,GACZ,WAAW,GACI;AACf,QAAI,cAAc,KAAK,QAAQ,YAAY,oBAAoB;AAC7D,cAAQ,MAAM,iCAAiC;AAC/C;AAAA,IACF;AAEA,UAAM,kBAAkB,IAAI,gBAAgB;AAC5C,SAAK,kBAAkB;AACvB,UAAM,SAAS,gBAAgB;AAG/B,QAAI,aAAa;AACjB,UAAM,iBAAiB,KAAK,kBAAkB;AAE9C,QAAI;AAEF,YAAM,WAAW,MAAM,KAAK,OAAO,UAAU;AAAA,QAC3C;AAAA,UACE,OAAO,KAAK,QAAQ,SAAS;AAAA,UAC7B,OAAO,KAAK,WAAW;AAAA,UACvB,OAAO,KAAK,WAAW;AAAA,UACvB,QAAQ;AAAA,UACR,GAAG,KAAK,QAAQ;AAAA,QAClB;AAAA,QACA,EAAE,OAAO;AAAA,MACX;AACA,UAAI,aAAa;AAGjB,uBAAiB,SAAS,UAAU;AAClC,YAAI,oBAAoB,KAAK,gBAAiB;AAC9C,gBAAQ,MAAM,MAAM;AAAA,UAClB,KAAK;AACH,iBAAK,IAAI,kBAAkB,MAAM,KAAK,GAAG;AAGzC,gBAAI,gBAAgB;AAClB,kBAAI,CAAC,YAAY;AACf,sBAAM,gBAAgB,MAAM,MAAM;AAAA,kBAChC,eAAe;AAAA,gBACjB;AACA,oBAAI,kBAAkB,IAAI;AACxB,+BAAa;AACb,wBAAM,cAAc,MAAM,MACvB,MAAM,GAAG,aAAa,EACtB,QAAQ;AACX,yBAAO,MAAM,WAAW;AACxB;AAAA,gBACF;AAAA,cACF,OAAO;AAEL;AAAA,cACF;AAAA,YACF;AAGA,mBAAO,MAAM,MAAM,KAAK;AACxB;AAAA,UAEF,KAAK;AACH,kBAAM,EAAE,SAAS,SAAS,IAAI,KAAK,QAAQ,MAAM,IAAI;AACrD,iBAAK,oBAAoB,SAAS,QAAQ;AAC1C,iBAAK,kBAAkB;AAMvB;AAAA,UAEF,KAAK;AACH,gBAAI,MAAM,KAAK,SAAS,iBAAiB;AACvC,oBAAM,WAAW,MAAM;AACvB,oBAAM,SAAS,MAAM,KAAK,YAAY;AAAA,gBACpC,MAAM;AAAA,gBACN,YAAY,SAAS;AAAA,gBACrB,UAAU,SAAS;AAAA,gBACnB,YAAY,SAAS;AAAA,cACvB,CAAC;AACD,kBAAI,OAAO,YAAY;AACrB,6BAAa;AAAA,cACf;AAAA,YACF;AACA;AAAA,UAEF;AACE;AAAA,QACJ;AAAA,MACF;AAEA,UAAI,oBAAoB,KAAK,gBAAiB;AAG9C,UAAI,CAAC,YAAY;AACf,cAAM,KAAK,eAAe,QAAQ,YAAY,CAAC;AAAA,MACjD;AACA,WAAK,kBAAkB;AAAA,IACzB,SAAS,OAAO;AACd,UAAI,iBAAiB,cAAAA,QAAO,kBAAmB;AAC/C,cAAQ,MAAM,kCAAkC,KAAK;AAErD,UAAI,YAAY,KAAK,QAAQ,YAAY,sBAAsB;AAC7D,cAAM,IAAI;AAAA,UAAQ,CAAC,YACjB,WAAW,SAAS,KAAK,QAAQ,cAAc,mBAAmB;AAAA,QACpE;AACA,cAAM,KAAK,eAAe,QAAQ,WAAW,WAAW,CAAC;AAAA,MAC3D;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,aAA6C;AACnD,WAAO,KAAK,aAAa,IAAI,CAAC,YAAY;AACxC,cAAQ,QAAQ,MAAM;AAAA,QACpB,KAAK;AAAA,QACL,KAAK;AAAA,QACL,KAAK;AACH,iBAAO;AAAA,YACL,MAAM,QAAQ;AAAA,YACd,SAAS,QAAQ;AAAA,UACnB;AAAA,QACF,KAAK;AACH,iBAAO;AAAA,YACL,MAAM;AAAA,YACN,SAAS,QAAQ;AAAA,YACjB,MAAM,QAAQ;AAAA,YACd,WAAW,KAAK,UAAU,QAAQ,UAAU;AAAA,UAC9C;AAAA,QACF,KAAK;AACH,iBAAO;AAAA,YACL,MAAM;AAAA,YACN,SAAS,QAAQ;AAAA,YACjB,QAAQ,QAAQ;AAAA,UAClB;AAAA,MACJ;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEQ,aAAkD;AAExD,UAAM,cAAc,KAAK,aAAa,SAAS;AAC/C,QAAI,CAAC,YAAa,QAAO;AAEzB,WAAO,KAAK,MAAM,IAAI,CAAC,UAAU;AAAA,MAC/B,MAAM;AAAA,MACN,MAAM,KAAK;AAAA,MACX,aAAa,KAAK;AAAA,MAClB,QAAQ;AAAA,MACR,gBAAY,yBAAa,KAAK,eAAe,WAAAC,QAAE,OAAO,CAAC;AAAA,IACzD,EAAE;AAAA,EACJ;AAAA,EAEA,SAAS;AACP,QAAI,CAAC,KAAK,gBAAiB;AAC3B,SAAK,IAAI,QAAQ;AACjB,SAAK,gBAAgB,MAAM;AAC3B,SAAK,kBAAkB;AAAA,EACzB;AACF;;;ACrMA,IAAAC,iBAAoB;AAEpB,gBAAsB;AAkBtB,IAAMC,iBAAgB;AACtB,IAAM,mBAAmB;AACzB,IAAM,cAAc;AACpB,IAAM,YAAY;AAClB,IAAM,gCAAgC;AACtC,IAAMC,uBAAsB;AAC5B,IAAM,oBAAoB;AAEnB,IAAM,YAAN,cAAwB,mBAAI;AAAA,EASjC,YAAoB,SAA2B;AAC7C,UAAM;AADY;AALpB,SAAQ,aAAa;AAErB,SAAQ,qBAA+B,CAAC;AAOtC,SAAK,cAAc,KAAK,2BAA2B,EAChD,KAAK,MAAM,KAAK,OAAO,CAAC,EACxB,MAAM,CAAC,UAAU;AAChB,cAAQ,MAAM,iCAAiC,KAAK;AACpD,WAAK,UAAU;AAAA,IACjB,CAAC;AAAA,EACL;AAAA,EAEA,WAAW,aAAuB;AAEhC,gBAAY,GAAG,QAAQ,OAAO,UAAkB;AAC9C,WAAK,mBAAmB,KAAK,KAAK;AAClC,YAAM,KAAK;AACX,WAAK,eAAe,KAAK;AACzB,WAAK,IAAI,qBAAqB,MAAM,UAAU,SAAS;AAAA,IACzD,CAAC;AAGD,gBAAY,GAAG,OAAO,YAAY;AAChC,YAAM,KAAK;AACX,UAAI,KAAK,mBAAmB,WAAW,EAAG;AAC1C,WAAK,YAAY,CAAC;AAGlB,WAAK,uBAAuB,WAAW,MAAM;AAC3C,aAAK,uBAAuB;AAC5B,aAAK,IAAI,uBAAuB;AAChC,aAAK,KAAK,cAAc,EAAE;AAC1B,aAAK,mBAAmB,SAAS;AAAA,MACnC,GAAG,KAAK,QAAQ,wBAAwB,6BAA6B;AAAA,IACvE,CAAC;AAAA,EACH;AAAA,EAEA,UAAU;AACR,UAAM,QAAQ;AACd,QAAI,KAAK,kBAAkB;AACzB,mBAAa,KAAK,gBAAgB;AAClC,WAAK,mBAAmB;AAAA,IAC1B;AACA,QAAI,KAAK,sBAAsB;AAC7B,mBAAa,KAAK,oBAAoB;AACtC,WAAK,uBAAuB;AAAA,IAC9B;AACA,SAAK,QAAQ,mBAAmB;AAChC,QAAI,KAAK,QAAQ,eAAe,UAAAC,QAAU,MAAM;AAC9C,WAAK,QAAQ,MAAM,GAAI;AAAA,IACzB;AACA,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,MAAc,6BAA4C;AACxD,UAAM,UAAU;AAAA,MACd,oBAAoB;AAAA,MACpB,2BAA2B;AAAA,QACzB,OAAO,KAAK,QAAQ,SAASF;AAAA,QAC7B,UAAU,KAAK,QAAQ,YAAY;AAAA,QACnC,QACE,KAAK,QAAQ,UAAU;AAAA,MAC3B;AAAA,MACA,gBAAgB;AAAA,QACd,MAAM;AAAA,QACN,qBAAqB;AAAA,MACvB;AAAA,MACA,6BAA6B;AAAA,QAC3B,MAAM;AAAA,MACR;AAAA,IACF;AAEA,UAAM,WAAW,MAAM;AAAA,MACrB;AAAA,MACA;AAAA,QACE,QAAQ;AAAA,QACR,SAAS;AAAA,UACP,eAAe,UAAU,KAAK,QAAQ,MAAM;AAAA,UAC5C,gBAAgB;AAAA,UAChB,eAAe;AAAA,QACjB;AAAA,QACA,MAAM,KAAK,UAAU,OAAO;AAAA,MAC9B;AAAA,IACF;AAEA,QAAI,CAAC,SAAS,IAAI;AAChB,YAAM,SAAS,SAAS;AAGxB,UAAI,UAAU,OAAO,SAAS,KAAK;AACjC,cAAM,IAAI;AAAA,UACR,2CAA2C,MAAM,IAAI,SAAS,UAAU;AAAA,QAC1E;AAAA,MACF;AAGA,WAAK,IAAI,qDAAqD;AAAA,QAC5D;AAAA,QACA,MAAM,SAAS;AAAA,MACjB,CAAC;AACD,YAAM,IAAI;AAAA,QAAQ,CAAC,YACjB,WAAW,SAAS,KAAK,QAAQ,cAAcC,oBAAmB;AAAA,MACpE;AACA,aAAO,KAAK,2BAA2B;AAAA,IACzC;AAEA,UAAM,OAAQ,MAAM,SAAS,KAAK;AAClC,SAAK,iBAAiB,KAAK,cAAc;AACzC,SAAK,IAAI,+BAA+B;AAAA,EAC1C;AAAA,EAEA,MAAc,SAAwB;AACpC,WAAO,IAAI,QAAc,CAAC,YAAY;AACpC,YAAM,SAAS,IAAI,UAAAC,QAAU,oCAAoC;AAAA,QAC/D,SAAS;AAAA,UACP,eAAe,UAAU,KAAK,cAAc;AAAA,UAC5C,eAAe;AAAA,QACjB;AAAA,MACF,CAAC;AACD,WAAK,SAAS;AAEd,aAAO,iBAAiB,QAAQ,MAAM;AACpC,aAAK,IAAI,mBAAmB;AAC5B,gBAAQ;AAAA,MACV,CAAC;AAED,aAAO,iBAAiB,SAAS,CAAC,UAAe;AAC/C,aAAK,IAAI,oBAAoB,KAAK;AAAA,MACpC,CAAC;AAED,aAAO;AAAA,QACL;AAAA,QACA,CAAC,EAAE,MAAM,OAAO,MAAwC;AACtD,eAAK,QAAQ,mBAAmB;AAChC,eAAK,SAAS;AAEd,cAAI,SAAS,KAAM;AACjB,iBAAK,UAAU;AAAA,UACjB,OAAO;AACL,iBAAK,IAAI,qBAAqB,EAAE,MAAM,OAAO,CAAC;AAAA,UAChD;AAAA,QACF;AAAA,MACF;AAEA,aAAO,iBAAiB,WAAW,CAAC,UAAe;AACjD,aAAK,cAAc,KAAK,MAAM,MAAM,KAAK,SAAS,CAAC,CAAC;AAAA,MACtD,CAAC;AAAA,IACH,CAAC;AAAA,EACH;AAAA,EAEQ,eAAe,OAAe;AACpC,QAAI,CAAC,KAAK,OAAQ;AAElB,UAAM,eAAe;AAAA,MACnB,MAAM;AAAA,MACN,OAAO,MAAM,SAAS,QAAQ;AAAA,IAChC;AAEA,SAAK,OAAO,KAAK,KAAK,UAAU,YAAY,CAAC;AAAA,EAC/C;AAAA,EAEQ,cAAc,SAAc;AAClC,YAAQ,QAAQ,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAiBpB,KAAK;AACH,cAAM,aAAa,QAAQ,cAAc;AACzC,aAAK,IAAI,mCAAmC,UAAU,GAAG;AACzD,aAAK,KAAK,cAAc,UAAU;AAElC,aAAK,mBAAmB,SAAS;AACjC,YAAI,KAAK,sBAAsB;AAC7B,uBAAa,KAAK,oBAAoB;AACtC,eAAK,uBAAuB;AAAA,QAC9B;AACA;AAAA,MAEF,KAAK;AACH,aAAK,IAAI,UAAU,QAAQ,KAAK;AAChC;AAAA,MAEF;AACE;AAAA,IACJ;AAAA,EACF;AAAA,EAEQ,YAAY;AAClB,SAAK;AACL,QAAI,KAAK,cAAc,KAAK,QAAQ,YAAY,oBAAoB;AAClE,WAAK,IAAI,gCAAgC;AACzC,WAAK,KAAK,UAAU,KAAK,kBAAkB;AAC3C;AAAA,IACF;AAEA,SAAK,cAAc,IAAI,QAAQ,CAAC,YAAY;AAC1C,WAAK,IAAI,iBAAiB;AAC1B,WAAK,mBAAmB,WAAW,MAAM;AACvC,aAAK,mBAAmB;AACxB,aAAK,2BAA2B,EAC7B,KAAK,MAAM,KAAK,OAAO,CAAC,EACxB,KAAK,MAAM;AACV,eAAK,aAAa;AAGlB,cAAI,KAAK,mBAAmB,SAAS,GAAG;AACtC,iBAAK,IAAI,4BAA4B;AACrC,iBAAK,mBAAmB;AAAA,cAAQ,CAAC,UAC/B,KAAK,eAAe,KAAK;AAAA,YAC3B;AAAA,UACF;AAAA,QACF,CAAC,EACA,KAAK,OAAO,EACZ,MAAM,CAAC,UAAU;AAChB,eAAK,IAAI,uBAAuB,KAAK;AACrC,eAAK,UAAU;AAAA,QACjB,CAAC;AAAA,MACL,GAAG,KAAK,QAAQ,cAAcD,oBAAmB;AAAA,IACnD,CAAC;AAAA,EACH;AAAA,EAEQ,YAAY,iBAAyB;AAC3C,QAAI,CAAC,KAAK,OAAQ;AAClB,UAAM,aAAa,KAAK,MAAM,cAAc,eAAe;AAC3D,UAAM,iBAAiB,YAAY;AACnC,UAAM,gBAAgB,OAAO,MAAM,aAAa,cAAc;AAC9D,SAAK,mBAAmB,KAAK,aAAa;AAC1C,SAAK,eAAe,aAAa;AACjC,SAAK;AAAA,MACH,QAAQ,kBAAkB,GAAI,kBAAkB,cAAc,UAAU;AAAA,IAC1E;AAAA,EACF;AACF;","names":["OpenAI","z","import_server","DEFAULT_MODEL","DEFAULT_RETRY_DELAY","WebSocket"]}
|
package/dist/index.mjs
CHANGED
|
@@ -148,10 +148,12 @@ var SAMPLE_RATE = 16e3;
|
|
|
148
148
|
var BIT_DEPTH = 16;
|
|
149
149
|
var DEFAULT_TRANSCRIPTION_TIMEOUT = 4e3;
|
|
150
150
|
var DEFAULT_RETRY_DELAY2 = 1e3;
|
|
151
|
+
var DEFAULT_MAX_RETRY = 3;
|
|
151
152
|
var OpenaiSTT = class extends STT {
|
|
152
153
|
constructor(options) {
|
|
153
154
|
super();
|
|
154
155
|
this.options = options;
|
|
156
|
+
this.retryCount = 0;
|
|
155
157
|
this.audioChunksPending = [];
|
|
156
158
|
this.initPromise = this.createTranscriptionSession().then(() => this.initWS()).catch((error) => {
|
|
157
159
|
console.error("[OpenaiSTT] Connection error:", error);
|
|
@@ -160,10 +162,9 @@ var OpenaiSTT = class extends STT {
|
|
|
160
162
|
}
|
|
161
163
|
transcribe(audioStream) {
|
|
162
164
|
audioStream.on("data", async (chunk) => {
|
|
165
|
+
this.audioChunksPending.push(chunk);
|
|
163
166
|
await this.initPromise;
|
|
164
|
-
|
|
165
|
-
this.audioChunksPending.push(base64Audio);
|
|
166
|
-
this.sendAudioChunk(base64Audio);
|
|
167
|
+
this.sendAudioChunk(chunk);
|
|
167
168
|
this.log(`Sent audio chunk (${chunk.byteLength} bytes)`);
|
|
168
169
|
});
|
|
169
170
|
audioStream.on("end", async () => {
|
|
@@ -275,11 +276,11 @@ var OpenaiSTT = class extends STT {
|
|
|
275
276
|
});
|
|
276
277
|
});
|
|
277
278
|
}
|
|
278
|
-
sendAudioChunk(
|
|
279
|
+
sendAudioChunk(chunk) {
|
|
279
280
|
if (!this.socket) return;
|
|
280
281
|
const audioMessage = {
|
|
281
282
|
type: "input_audio_buffer.append",
|
|
282
|
-
audio:
|
|
283
|
+
audio: chunk.toString("base64")
|
|
283
284
|
};
|
|
284
285
|
this.socket.send(JSON.stringify(audioMessage));
|
|
285
286
|
}
|
|
@@ -315,11 +316,18 @@ var OpenaiSTT = class extends STT {
|
|
|
315
316
|
}
|
|
316
317
|
}
|
|
317
318
|
reconnect() {
|
|
319
|
+
this.retryCount++;
|
|
320
|
+
if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY)) {
|
|
321
|
+
this.log("Max retries reached, giving up");
|
|
322
|
+
this.emit("Failed", this.audioChunksPending);
|
|
323
|
+
return;
|
|
324
|
+
}
|
|
318
325
|
this.initPromise = new Promise((resolve) => {
|
|
319
326
|
this.log("Reconnecting...");
|
|
320
327
|
this.reconnectTimeout = setTimeout(() => {
|
|
321
328
|
this.reconnectTimeout = void 0;
|
|
322
329
|
this.createTranscriptionSession().then(() => this.initWS()).then(() => {
|
|
330
|
+
this.retryCount = 0;
|
|
323
331
|
if (this.audioChunksPending.length > 0) {
|
|
324
332
|
this.log("Sending audio chunks again");
|
|
325
333
|
this.audioChunksPending.forEach(
|
|
@@ -338,9 +346,8 @@ var OpenaiSTT = class extends STT {
|
|
|
338
346
|
const numSamples = Math.round(SAMPLE_RATE * durationSeconds);
|
|
339
347
|
const bytesPerSample = BIT_DEPTH / 8;
|
|
340
348
|
const silenceBuffer = Buffer.alloc(numSamples * bytesPerSample);
|
|
341
|
-
|
|
342
|
-
this.
|
|
343
|
-
this.sendAudioChunk(base64Audio);
|
|
349
|
+
this.audioChunksPending.push(silenceBuffer);
|
|
350
|
+
this.sendAudioChunk(silenceBuffer);
|
|
344
351
|
this.log(
|
|
345
352
|
`Sent ${durationSeconds * 1e3}ms of silence (${silenceBuffer.byteLength} bytes) after stream end`
|
|
346
353
|
);
|
package/dist/index.mjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/OpenaiAgent.ts","../src/OpenaiSTT.ts"],"sourcesContent":["import { Agent, AgentOptions } from '@micdrop/server'\nimport OpenAI from 'openai'\nimport { PassThrough } from 'stream'\nimport z, { toJSONSchema } from 'zod'\n\nexport type OpenaiAgentOptions = AgentOptions &\n OpenaiOptions & {\n model?: string\n settings?: Omit<\n OpenAI.Responses.ResponseCreateParamsStreaming,\n 'input' | 'model' | 'tools' | 'stream'\n >\n retryDelay?: number\n maxRetry?: number\n maxSteps?: number\n }\n\nexport type OpenaiOptions = { apiKey: string } | { openai: OpenAI }\n\nconst DEFAULT_MODEL = 'gpt-4o'\nconst DEFAULT_MAX_STEPS = 5\nconst DEFAULT_MAX_RETRIES = 3\nconst DEFAULT_RETRY_DELAY = 1000\n\nexport class OpenaiAgent extends Agent<OpenaiAgentOptions> {\n private openai: OpenAI\n private abortController?: AbortController\n\n constructor(options: OpenaiAgentOptions) {\n super(options)\n this.openai =\n 'openai' in options\n ? options.openai\n : new OpenAI({ apiKey: options.apiKey })\n }\n\n protected async generateAnswer(\n stream: PassThrough,\n stepCount = 0,\n tryCount = 0\n ): Promise<void> {\n if (stepCount >= (this.options.maxSteps || DEFAULT_MAX_STEPS)) {\n console.error('[OpenaiAgent] Max steps reached')\n return\n }\n\n const abortController = new AbortController()\n this.abortController = abortController\n const signal = abortController.signal\n\n // Prepare extracting\n let extracting = false\n const extractOptions = this.getExtractOptions()\n\n try {\n // Generate answer\n const response = await this.openai.responses.create(\n {\n model: this.options.model || DEFAULT_MODEL,\n input: this.buildInput(),\n tools: this.buildTools(),\n stream: true,\n ...this.options.settings,\n },\n { signal }\n )\n let skipAnswer = false\n\n // Handle response events\n for await (const event of response) {\n if (abortController !== this.abortController) return\n switch (event.type) {\n case 'response.output_text.delta':\n this.log(`Answer chunk: \"${event.delta}\"`)\n\n // Extracting value?\n if (extractOptions) {\n if (!extracting) {\n const startTagIndex = event.delta.indexOf(\n extractOptions.startTag\n )\n if (startTagIndex !== -1) {\n extracting = true\n const messagePart = event.delta\n .slice(0, startTagIndex)\n .trimEnd()\n stream.write(messagePart)\n break\n }\n } else {\n // Extracting, don't write to stream\n break\n }\n }\n\n // Send chunk\n stream.write(event.delta)\n break\n\n case 'response.output_text.done':\n const { message, metadata } = this.extract(event.text)\n this.addAssistantMessage(message, metadata)\n this.abortController = undefined\n // Stop query now that we have a complete answer\n // Note: OpenAI can sometimes return multiple outputs.\n // See https://community.openai.com/t/how-to-prevent-the-api-returning-multiple-outputs/1251365/28\n // We can prompt it like this to avoid consuming too many tokens:\n // `Return one complete answer and then stop`\n return\n\n case 'response.output_item.done':\n if (event.item.type === 'function_call') {\n const toolCall = event.item\n const result = await this.executeTool({\n role: 'tool_call',\n toolCallId: toolCall.call_id,\n toolName: toolCall.name,\n parameters: toolCall.arguments,\n })\n if (result.skipAnswer) {\n skipAnswer = true\n }\n }\n break\n\n default:\n break\n }\n }\n\n if (abortController !== this.abortController) return\n\n // Query again in case of tool call\n if (!skipAnswer) {\n await this.generateAnswer(stream, stepCount + 1)\n }\n this.abortController = undefined\n } catch (error) {\n if (error instanceof OpenAI.APIUserAbortError) return\n console.error('[OpenaiAgent] Error answering:', error)\n\n if (tryCount < (this.options.maxRetry ?? DEFAULT_MAX_RETRIES)) {\n await new Promise((resolve) =>\n setTimeout(resolve, this.options.retryDelay ?? DEFAULT_RETRY_DELAY)\n )\n await this.generateAnswer(stream, stepCount, tryCount + 1)\n }\n }\n }\n\n private buildInput(): OpenAI.Responses.ResponseInput {\n return this.conversation.map((message) => {\n switch (message.role) {\n case 'user':\n case 'assistant':\n case 'system':\n return {\n role: message.role,\n content: message.content,\n }\n case 'tool_call':\n return {\n type: 'function_call',\n call_id: message.toolCallId,\n name: message.toolName,\n arguments: JSON.stringify(message.parameters),\n }\n case 'tool_result':\n return {\n type: 'function_call_output',\n call_id: message.toolCallId,\n output: message.output,\n }\n }\n })\n }\n\n private buildTools(): OpenAI.Responses.Tool[] | undefined {\n // Disable tools if first message\n const enableTools = this.conversation.length > 1\n if (!enableTools) return undefined\n\n return this.tools.map((tool) => ({\n type: 'function',\n name: tool.name,\n description: tool.description,\n strict: true,\n parameters: toJSONSchema(tool.inputSchema || z.object()),\n }))\n }\n\n cancel() {\n if (!this.abortController) return\n this.log('Cancel')\n this.abortController.abort()\n this.abortController = undefined\n }\n}\n","import { STT } from '@micdrop/server'\nimport { Readable } from 'stream'\nimport WebSocket from 'ws'\n\n/**\n * OpenAI Real-time STT\n *\n * @see https://platform.openai.com/docs/guides/speech-to-text\n */\n\nexport interface OpenaiSTTOptions {\n apiKey: string\n model?: string\n language?: string\n prompt?: string\n transcriptionTimeout?: number\n retryDelay?: number\n}\n\nconst DEFAULT_MODEL = 'gpt-4o-transcribe'\nconst DEFAULT_LANGUAGE = 'en'\nconst SAMPLE_RATE = 16000\nconst BIT_DEPTH = 16\nconst DEFAULT_TRANSCRIPTION_TIMEOUT = 4000\nconst DEFAULT_RETRY_DELAY = 1000\n\nexport class OpenaiSTT extends STT {\n private socket?: WebSocket\n private initPromise: Promise<void>\n private reconnectTimeout?: NodeJS.Timeout\n private transcriptionTimeout?: NodeJS.Timeout\n private audioChunksPending: string[] = [] // Store Base64 encoded audio chunks\n private ephemeralToken?: string\n\n constructor(private options: OpenaiSTTOptions) {\n super()\n\n // Setup WebSocket connection (first create session, then connect)\n this.initPromise = this.createTranscriptionSession()\n .then(() => this.initWS())\n .catch((error) => {\n console.error('[OpenaiSTT] Connection error:', error)\n this.reconnect()\n })\n }\n\n transcribe(audioStream: Readable) {\n // Read audio stream and send to OpenAI\n audioStream.on('data', async (chunk: Buffer) => {\n await this.initPromise\n const base64Audio = chunk.toString('base64')\n this.audioChunksPending.push(base64Audio)\n this.sendAudioChunk(base64Audio)\n this.log(`Sent audio chunk (${chunk.byteLength} bytes)`)\n })\n\n // Handle stream end\n audioStream.on('end', async () => {\n await this.initPromise\n if (this.audioChunksPending.length === 0) return\n this.sendSilence(2)\n\n // Timeout transcription if no transcript is received\n this.transcriptionTimeout = setTimeout(() => {\n this.transcriptionTimeout = undefined\n this.log('Transcription timeout')\n this.emit('Transcript', '')\n this.audioChunksPending.length = 0\n }, this.options.transcriptionTimeout || DEFAULT_TRANSCRIPTION_TIMEOUT)\n })\n }\n\n destroy() {\n super.destroy()\n if (this.reconnectTimeout) {\n clearTimeout(this.reconnectTimeout)\n this.reconnectTimeout = undefined\n }\n if (this.transcriptionTimeout) {\n clearTimeout(this.transcriptionTimeout)\n this.transcriptionTimeout = undefined\n }\n this.socket?.removeAllListeners()\n if (this.socket?.readyState === WebSocket.OPEN) {\n this.socket?.close(1000)\n }\n this.socket = undefined\n }\n\n private async createTranscriptionSession(): Promise<void> {\n const payload = {\n input_audio_format: 'pcm16',\n input_audio_transcription: {\n model: this.options.model || DEFAULT_MODEL,\n language: this.options.language || DEFAULT_LANGUAGE,\n prompt:\n this.options.prompt || 'Transcribe the incoming audio in real time.',\n },\n turn_detection: {\n type: 'server_vad',\n silence_duration_ms: 1000,\n },\n input_audio_noise_reduction: {\n type: 'near_field',\n },\n }\n\n const response = await fetch(\n 'https://api.openai.com/v1/realtime/transcription_sessions',\n {\n method: 'POST',\n headers: {\n Authorization: `Bearer ${this.options.apiKey}`,\n 'Content-Type': 'application/json',\n 'OpenAI-Beta': 'realtime=v1',\n },\n body: JSON.stringify(payload),\n }\n )\n\n if (!response.ok) {\n const status = response.status\n\n // Don't retry on 4xx errors\n if (status >= 400 && status < 500) {\n throw new Error(\n `Failed to create transcription session: ${status} ${response.statusText}`\n )\n }\n\n // Retry on other errors\n this.log('Error creating transcription session, retrying...', {\n status,\n text: response.statusText,\n })\n await new Promise((resolve) =>\n setTimeout(resolve, this.options.retryDelay ?? DEFAULT_RETRY_DELAY)\n )\n return this.createTranscriptionSession()\n }\n\n const data = (await response.json()) as { client_secret: { value: string } }\n this.ephemeralToken = data.client_secret.value\n this.log('Transcription session created')\n }\n\n private async initWS(): Promise<void> {\n return new Promise<void>((resolve) => {\n const socket = new WebSocket('wss://api.openai.com/v1/realtime', {\n headers: {\n Authorization: `Bearer ${this.ephemeralToken}`,\n 'OpenAI-Beta': 'realtime=v1',\n },\n })\n this.socket = socket\n\n socket.addEventListener('open', () => {\n this.log('Connection opened')\n resolve()\n })\n\n socket.addEventListener('error', (error: any) => {\n this.log('WebSocket error:', error)\n })\n\n socket.addEventListener(\n 'close',\n ({ code, reason }: { code: number; reason: string }) => {\n this.socket?.removeAllListeners()\n this.socket = undefined\n\n if (code !== 1000) {\n this.reconnect()\n } else {\n this.log('Connection closed', { code, reason })\n }\n }\n )\n\n socket.addEventListener('message', (event: any) => {\n this.handleMessage(JSON.parse(event.data.toString()))\n })\n })\n }\n\n private sendAudioChunk(base64Audio: string) {\n if (!this.socket) return\n\n const audioMessage = {\n type: 'input_audio_buffer.append',\n audio: base64Audio,\n }\n\n this.socket.send(JSON.stringify(audioMessage))\n }\n\n private handleMessage(message: any) {\n switch (message.type) {\n // case 'input_audio_buffer.committed':\n // this.log('Audio buffer committed')\n // break\n\n // case 'input_audio_buffer.speech_started':\n // this.log('Speech started')\n // break\n\n // case 'input_audio_buffer.speech_stopped':\n // this.log('Speech stopped')\n // break\n\n // case 'conversation.item.input_audio_transcription.delta':\n // this.log(`Received transcript delta: \"${message.delta}\"`)\n // break\n\n case 'conversation.item.input_audio_transcription.completed':\n const transcript = message.transcript || ''\n this.log(`Received completed transcript: \"${transcript}\"`)\n this.emit('Transcript', transcript)\n // Reset audio chunks and clear timeout\n this.audioChunksPending.length = 0\n if (this.transcriptionTimeout) {\n clearTimeout(this.transcriptionTimeout)\n this.transcriptionTimeout = undefined\n }\n break\n\n case 'error':\n this.log('Error:', message.error)\n break\n\n default:\n break\n }\n }\n\n private reconnect() {\n this.initPromise = new Promise((resolve) => {\n this.log('Reconnecting...')\n this.reconnectTimeout = setTimeout(() => {\n this.reconnectTimeout = undefined\n this.createTranscriptionSession()\n .then(() => this.initWS())\n .then(() => {\n // Send audio chunks again if reconnecting during transcription\n if (this.audioChunksPending.length > 0) {\n this.log('Sending audio chunks again')\n this.audioChunksPending.forEach((chunk) =>\n this.sendAudioChunk(chunk)\n )\n }\n })\n .then(resolve)\n .catch((error) => {\n this.log('Reconnection error:', error)\n this.reconnect()\n })\n }, this.options.retryDelay ?? DEFAULT_RETRY_DELAY)\n })\n }\n\n private sendSilence(durationSeconds: number) {\n if (!this.socket) return\n const numSamples = Math.round(SAMPLE_RATE * durationSeconds)\n const bytesPerSample = BIT_DEPTH / 8\n const silenceBuffer = Buffer.alloc(numSamples * bytesPerSample)\n const base64Audio = silenceBuffer.toString('base64')\n this.audioChunksPending.push(base64Audio)\n this.sendAudioChunk(base64Audio)\n this.log(\n `Sent ${durationSeconds * 1000}ms of silence (${silenceBuffer.byteLength} bytes) after stream end`\n )\n }\n}\n"],"mappings":";AAAA,SAAS,aAA2B;AACpC,OAAO,YAAY;AAEnB,OAAO,KAAK,oBAAoB;AAgBhC,IAAM,gBAAgB;AACtB,IAAM,oBAAoB;AAC1B,IAAM,sBAAsB;AAC5B,IAAM,sBAAsB;AAErB,IAAM,cAAN,cAA0B,MAA0B;AAAA,EAIzD,YAAY,SAA6B;AACvC,UAAM,OAAO;AACb,SAAK,SACH,YAAY,UACR,QAAQ,SACR,IAAI,OAAO,EAAE,QAAQ,QAAQ,OAAO,CAAC;AAAA,EAC7C;AAAA,EAEA,MAAgB,eACd,QACA,YAAY,GACZ,WAAW,GACI;AACf,QAAI,cAAc,KAAK,QAAQ,YAAY,oBAAoB;AAC7D,cAAQ,MAAM,iCAAiC;AAC/C;AAAA,IACF;AAEA,UAAM,kBAAkB,IAAI,gBAAgB;AAC5C,SAAK,kBAAkB;AACvB,UAAM,SAAS,gBAAgB;AAG/B,QAAI,aAAa;AACjB,UAAM,iBAAiB,KAAK,kBAAkB;AAE9C,QAAI;AAEF,YAAM,WAAW,MAAM,KAAK,OAAO,UAAU;AAAA,QAC3C;AAAA,UACE,OAAO,KAAK,QAAQ,SAAS;AAAA,UAC7B,OAAO,KAAK,WAAW;AAAA,UACvB,OAAO,KAAK,WAAW;AAAA,UACvB,QAAQ;AAAA,UACR,GAAG,KAAK,QAAQ;AAAA,QAClB;AAAA,QACA,EAAE,OAAO;AAAA,MACX;AACA,UAAI,aAAa;AAGjB,uBAAiB,SAAS,UAAU;AAClC,YAAI,oBAAoB,KAAK,gBAAiB;AAC9C,gBAAQ,MAAM,MAAM;AAAA,UAClB,KAAK;AACH,iBAAK,IAAI,kBAAkB,MAAM,KAAK,GAAG;AAGzC,gBAAI,gBAAgB;AAClB,kBAAI,CAAC,YAAY;AACf,sBAAM,gBAAgB,MAAM,MAAM;AAAA,kBAChC,eAAe;AAAA,gBACjB;AACA,oBAAI,kBAAkB,IAAI;AACxB,+BAAa;AACb,wBAAM,cAAc,MAAM,MACvB,MAAM,GAAG,aAAa,EACtB,QAAQ;AACX,yBAAO,MAAM,WAAW;AACxB;AAAA,gBACF;AAAA,cACF,OAAO;AAEL;AAAA,cACF;AAAA,YACF;AAGA,mBAAO,MAAM,MAAM,KAAK;AACxB;AAAA,UAEF,KAAK;AACH,kBAAM,EAAE,SAAS,SAAS,IAAI,KAAK,QAAQ,MAAM,IAAI;AACrD,iBAAK,oBAAoB,SAAS,QAAQ;AAC1C,iBAAK,kBAAkB;AAMvB;AAAA,UAEF,KAAK;AACH,gBAAI,MAAM,KAAK,SAAS,iBAAiB;AACvC,oBAAM,WAAW,MAAM;AACvB,oBAAM,SAAS,MAAM,KAAK,YAAY;AAAA,gBACpC,MAAM;AAAA,gBACN,YAAY,SAAS;AAAA,gBACrB,UAAU,SAAS;AAAA,gBACnB,YAAY,SAAS;AAAA,cACvB,CAAC;AACD,kBAAI,OAAO,YAAY;AACrB,6BAAa;AAAA,cACf;AAAA,YACF;AACA;AAAA,UAEF;AACE;AAAA,QACJ;AAAA,MACF;AAEA,UAAI,oBAAoB,KAAK,gBAAiB;AAG9C,UAAI,CAAC,YAAY;AACf,cAAM,KAAK,eAAe,QAAQ,YAAY,CAAC;AAAA,MACjD;AACA,WAAK,kBAAkB;AAAA,IACzB,SAAS,OAAO;AACd,UAAI,iBAAiB,OAAO,kBAAmB;AAC/C,cAAQ,MAAM,kCAAkC,KAAK;AAErD,UAAI,YAAY,KAAK,QAAQ,YAAY,sBAAsB;AAC7D,cAAM,IAAI;AAAA,UAAQ,CAAC,YACjB,WAAW,SAAS,KAAK,QAAQ,cAAc,mBAAmB;AAAA,QACpE;AACA,cAAM,KAAK,eAAe,QAAQ,WAAW,WAAW,CAAC;AAAA,MAC3D;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,aAA6C;AACnD,WAAO,KAAK,aAAa,IAAI,CAAC,YAAY;AACxC,cAAQ,QAAQ,MAAM;AAAA,QACpB,KAAK;AAAA,QACL,KAAK;AAAA,QACL,KAAK;AACH,iBAAO;AAAA,YACL,MAAM,QAAQ;AAAA,YACd,SAAS,QAAQ;AAAA,UACnB;AAAA,QACF,KAAK;AACH,iBAAO;AAAA,YACL,MAAM;AAAA,YACN,SAAS,QAAQ;AAAA,YACjB,MAAM,QAAQ;AAAA,YACd,WAAW,KAAK,UAAU,QAAQ,UAAU;AAAA,UAC9C;AAAA,QACF,KAAK;AACH,iBAAO;AAAA,YACL,MAAM;AAAA,YACN,SAAS,QAAQ;AAAA,YACjB,QAAQ,QAAQ;AAAA,UAClB;AAAA,MACJ;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEQ,aAAkD;AAExD,UAAM,cAAc,KAAK,aAAa,SAAS;AAC/C,QAAI,CAAC,YAAa,QAAO;AAEzB,WAAO,KAAK,MAAM,IAAI,CAAC,UAAU;AAAA,MAC/B,MAAM;AAAA,MACN,MAAM,KAAK;AAAA,MACX,aAAa,KAAK;AAAA,MAClB,QAAQ;AAAA,MACR,YAAY,aAAa,KAAK,eAAe,EAAE,OAAO,CAAC;AAAA,IACzD,EAAE;AAAA,EACJ;AAAA,EAEA,SAAS;AACP,QAAI,CAAC,KAAK,gBAAiB;AAC3B,SAAK,IAAI,QAAQ;AACjB,SAAK,gBAAgB,MAAM;AAC3B,SAAK,kBAAkB;AAAA,EACzB;AACF;;;ACrMA,SAAS,WAAW;AAEpB,OAAO,eAAe;AAiBtB,IAAMA,iBAAgB;AACtB,IAAM,mBAAmB;AACzB,IAAM,cAAc;AACpB,IAAM,YAAY;AAClB,IAAM,gCAAgC;AACtC,IAAMC,uBAAsB;AAErB,IAAM,YAAN,cAAwB,IAAI;AAAA,EAQjC,YAAoB,SAA2B;AAC7C,UAAM;AADY;AAHpB,SAAQ,qBAA+B,CAAC;AAOtC,SAAK,cAAc,KAAK,2BAA2B,EAChD,KAAK,MAAM,KAAK,OAAO,CAAC,EACxB,MAAM,CAAC,UAAU;AAChB,cAAQ,MAAM,iCAAiC,KAAK;AACpD,WAAK,UAAU;AAAA,IACjB,CAAC;AAAA,EACL;AAAA,EAEA,WAAW,aAAuB;AAEhC,gBAAY,GAAG,QAAQ,OAAO,UAAkB;AAC9C,YAAM,KAAK;AACX,YAAM,cAAc,MAAM,SAAS,QAAQ;AAC3C,WAAK,mBAAmB,KAAK,WAAW;AACxC,WAAK,eAAe,WAAW;AAC/B,WAAK,IAAI,qBAAqB,MAAM,UAAU,SAAS;AAAA,IACzD,CAAC;AAGD,gBAAY,GAAG,OAAO,YAAY;AAChC,YAAM,KAAK;AACX,UAAI,KAAK,mBAAmB,WAAW,EAAG;AAC1C,WAAK,YAAY,CAAC;AAGlB,WAAK,uBAAuB,WAAW,MAAM;AAC3C,aAAK,uBAAuB;AAC5B,aAAK,IAAI,uBAAuB;AAChC,aAAK,KAAK,cAAc,EAAE;AAC1B,aAAK,mBAAmB,SAAS;AAAA,MACnC,GAAG,KAAK,QAAQ,wBAAwB,6BAA6B;AAAA,IACvE,CAAC;AAAA,EACH;AAAA,EAEA,UAAU;AACR,UAAM,QAAQ;AACd,QAAI,KAAK,kBAAkB;AACzB,mBAAa,KAAK,gBAAgB;AAClC,WAAK,mBAAmB;AAAA,IAC1B;AACA,QAAI,KAAK,sBAAsB;AAC7B,mBAAa,KAAK,oBAAoB;AACtC,WAAK,uBAAuB;AAAA,IAC9B;AACA,SAAK,QAAQ,mBAAmB;AAChC,QAAI,KAAK,QAAQ,eAAe,UAAU,MAAM;AAC9C,WAAK,QAAQ,MAAM,GAAI;AAAA,IACzB;AACA,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,MAAc,6BAA4C;AACxD,UAAM,UAAU;AAAA,MACd,oBAAoB;AAAA,MACpB,2BAA2B;AAAA,QACzB,OAAO,KAAK,QAAQ,SAASD;AAAA,QAC7B,UAAU,KAAK,QAAQ,YAAY;AAAA,QACnC,QACE,KAAK,QAAQ,UAAU;AAAA,MAC3B;AAAA,MACA,gBAAgB;AAAA,QACd,MAAM;AAAA,QACN,qBAAqB;AAAA,MACvB;AAAA,MACA,6BAA6B;AAAA,QAC3B,MAAM;AAAA,MACR;AAAA,IACF;AAEA,UAAM,WAAW,MAAM;AAAA,MACrB;AAAA,MACA;AAAA,QACE,QAAQ;AAAA,QACR,SAAS;AAAA,UACP,eAAe,UAAU,KAAK,QAAQ,MAAM;AAAA,UAC5C,gBAAgB;AAAA,UAChB,eAAe;AAAA,QACjB;AAAA,QACA,MAAM,KAAK,UAAU,OAAO;AAAA,MAC9B;AAAA,IACF;AAEA,QAAI,CAAC,SAAS,IAAI;AAChB,YAAM,SAAS,SAAS;AAGxB,UAAI,UAAU,OAAO,SAAS,KAAK;AACjC,cAAM,IAAI;AAAA,UACR,2CAA2C,MAAM,IAAI,SAAS,UAAU;AAAA,QAC1E;AAAA,MACF;AAGA,WAAK,IAAI,qDAAqD;AAAA,QAC5D;AAAA,QACA,MAAM,SAAS;AAAA,MACjB,CAAC;AACD,YAAM,IAAI;AAAA,QAAQ,CAAC,YACjB,WAAW,SAAS,KAAK,QAAQ,cAAcC,oBAAmB;AAAA,MACpE;AACA,aAAO,KAAK,2BAA2B;AAAA,IACzC;AAEA,UAAM,OAAQ,MAAM,SAAS,KAAK;AAClC,SAAK,iBAAiB,KAAK,cAAc;AACzC,SAAK,IAAI,+BAA+B;AAAA,EAC1C;AAAA,EAEA,MAAc,SAAwB;AACpC,WAAO,IAAI,QAAc,CAAC,YAAY;AACpC,YAAM,SAAS,IAAI,UAAU,oCAAoC;AAAA,QAC/D,SAAS;AAAA,UACP,eAAe,UAAU,KAAK,cAAc;AAAA,UAC5C,eAAe;AAAA,QACjB;AAAA,MACF,CAAC;AACD,WAAK,SAAS;AAEd,aAAO,iBAAiB,QAAQ,MAAM;AACpC,aAAK,IAAI,mBAAmB;AAC5B,gBAAQ;AAAA,MACV,CAAC;AAED,aAAO,iBAAiB,SAAS,CAAC,UAAe;AAC/C,aAAK,IAAI,oBAAoB,KAAK;AAAA,MACpC,CAAC;AAED,aAAO;AAAA,QACL;AAAA,QACA,CAAC,EAAE,MAAM,OAAO,MAAwC;AACtD,eAAK,QAAQ,mBAAmB;AAChC,eAAK,SAAS;AAEd,cAAI,SAAS,KAAM;AACjB,iBAAK,UAAU;AAAA,UACjB,OAAO;AACL,iBAAK,IAAI,qBAAqB,EAAE,MAAM,OAAO,CAAC;AAAA,UAChD;AAAA,QACF;AAAA,MACF;AAEA,aAAO,iBAAiB,WAAW,CAAC,UAAe;AACjD,aAAK,cAAc,KAAK,MAAM,MAAM,KAAK,SAAS,CAAC,CAAC;AAAA,MACtD,CAAC;AAAA,IACH,CAAC;AAAA,EACH;AAAA,EAEQ,eAAe,aAAqB;AAC1C,QAAI,CAAC,KAAK,OAAQ;AAElB,UAAM,eAAe;AAAA,MACnB,MAAM;AAAA,MACN,OAAO;AAAA,IACT;AAEA,SAAK,OAAO,KAAK,KAAK,UAAU,YAAY,CAAC;AAAA,EAC/C;AAAA,EAEQ,cAAc,SAAc;AAClC,YAAQ,QAAQ,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAiBpB,KAAK;AACH,cAAM,aAAa,QAAQ,cAAc;AACzC,aAAK,IAAI,mCAAmC,UAAU,GAAG;AACzD,aAAK,KAAK,cAAc,UAAU;AAElC,aAAK,mBAAmB,SAAS;AACjC,YAAI,KAAK,sBAAsB;AAC7B,uBAAa,KAAK,oBAAoB;AACtC,eAAK,uBAAuB;AAAA,QAC9B;AACA;AAAA,MAEF,KAAK;AACH,aAAK,IAAI,UAAU,QAAQ,KAAK;AAChC;AAAA,MAEF;AACE;AAAA,IACJ;AAAA,EACF;AAAA,EAEQ,YAAY;AAClB,SAAK,cAAc,IAAI,QAAQ,CAAC,YAAY;AAC1C,WAAK,IAAI,iBAAiB;AAC1B,WAAK,mBAAmB,WAAW,MAAM;AACvC,aAAK,mBAAmB;AACxB,aAAK,2BAA2B,EAC7B,KAAK,MAAM,KAAK,OAAO,CAAC,EACxB,KAAK,MAAM;AAEV,cAAI,KAAK,mBAAmB,SAAS,GAAG;AACtC,iBAAK,IAAI,4BAA4B;AACrC,iBAAK,mBAAmB;AAAA,cAAQ,CAAC,UAC/B,KAAK,eAAe,KAAK;AAAA,YAC3B;AAAA,UACF;AAAA,QACF,CAAC,EACA,KAAK,OAAO,EACZ,MAAM,CAAC,UAAU;AAChB,eAAK,IAAI,uBAAuB,KAAK;AACrC,eAAK,UAAU;AAAA,QACjB,CAAC;AAAA,MACL,GAAG,KAAK,QAAQ,cAAcA,oBAAmB;AAAA,IACnD,CAAC;AAAA,EACH;AAAA,EAEQ,YAAY,iBAAyB;AAC3C,QAAI,CAAC,KAAK,OAAQ;AAClB,UAAM,aAAa,KAAK,MAAM,cAAc,eAAe;AAC3D,UAAM,iBAAiB,YAAY;AACnC,UAAM,gBAAgB,OAAO,MAAM,aAAa,cAAc;AAC9D,UAAM,cAAc,cAAc,SAAS,QAAQ;AACnD,SAAK,mBAAmB,KAAK,WAAW;AACxC,SAAK,eAAe,WAAW;AAC/B,SAAK;AAAA,MACH,QAAQ,kBAAkB,GAAI,kBAAkB,cAAc,UAAU;AAAA,IAC1E;AAAA,EACF;AACF;","names":["DEFAULT_MODEL","DEFAULT_RETRY_DELAY"]}
|
|
1
|
+
{"version":3,"sources":["../src/OpenaiAgent.ts","../src/OpenaiSTT.ts"],"sourcesContent":["import { Agent, AgentOptions } from '@micdrop/server'\nimport OpenAI from 'openai'\nimport { PassThrough } from 'stream'\nimport z, { toJSONSchema } from 'zod'\n\nexport type OpenaiAgentOptions = AgentOptions &\n OpenaiOptions & {\n model?: string\n settings?: Omit<\n OpenAI.Responses.ResponseCreateParamsStreaming,\n 'input' | 'model' | 'tools' | 'stream'\n >\n retryDelay?: number\n maxRetry?: number\n maxSteps?: number\n }\n\nexport type OpenaiOptions = { apiKey: string } | { openai: OpenAI }\n\nconst DEFAULT_MODEL = 'gpt-4o'\nconst DEFAULT_MAX_STEPS = 5\nconst DEFAULT_MAX_RETRIES = 3\nconst DEFAULT_RETRY_DELAY = 1000\n\nexport class OpenaiAgent extends Agent<OpenaiAgentOptions> {\n private openai: OpenAI\n private abortController?: AbortController\n\n constructor(options: OpenaiAgentOptions) {\n super(options)\n this.openai =\n 'openai' in options\n ? options.openai\n : new OpenAI({ apiKey: options.apiKey })\n }\n\n protected async generateAnswer(\n stream: PassThrough,\n stepCount = 0,\n tryCount = 0\n ): Promise<void> {\n if (stepCount >= (this.options.maxSteps || DEFAULT_MAX_STEPS)) {\n console.error('[OpenaiAgent] Max steps reached')\n return\n }\n\n const abortController = new AbortController()\n this.abortController = abortController\n const signal = abortController.signal\n\n // Prepare extracting\n let extracting = false\n const extractOptions = this.getExtractOptions()\n\n try {\n // Generate answer\n const response = await this.openai.responses.create(\n {\n model: this.options.model || DEFAULT_MODEL,\n input: this.buildInput(),\n tools: this.buildTools(),\n stream: true,\n ...this.options.settings,\n },\n { signal }\n )\n let skipAnswer = false\n\n // Handle response events\n for await (const event of response) {\n if (abortController !== this.abortController) return\n switch (event.type) {\n case 'response.output_text.delta':\n this.log(`Answer chunk: \"${event.delta}\"`)\n\n // Extracting value?\n if (extractOptions) {\n if (!extracting) {\n const startTagIndex = event.delta.indexOf(\n extractOptions.startTag\n )\n if (startTagIndex !== -1) {\n extracting = true\n const messagePart = event.delta\n .slice(0, startTagIndex)\n .trimEnd()\n stream.write(messagePart)\n break\n }\n } else {\n // Extracting, don't write to stream\n break\n }\n }\n\n // Send chunk\n stream.write(event.delta)\n break\n\n case 'response.output_text.done':\n const { message, metadata } = this.extract(event.text)\n this.addAssistantMessage(message, metadata)\n this.abortController = undefined\n // Stop query now that we have a complete answer\n // Note: OpenAI can sometimes return multiple outputs.\n // See https://community.openai.com/t/how-to-prevent-the-api-returning-multiple-outputs/1251365/28\n // We can prompt it like this to avoid consuming too many tokens:\n // `Return one complete answer and then stop`\n return\n\n case 'response.output_item.done':\n if (event.item.type === 'function_call') {\n const toolCall = event.item\n const result = await this.executeTool({\n role: 'tool_call',\n toolCallId: toolCall.call_id,\n toolName: toolCall.name,\n parameters: toolCall.arguments,\n })\n if (result.skipAnswer) {\n skipAnswer = true\n }\n }\n break\n\n default:\n break\n }\n }\n\n if (abortController !== this.abortController) return\n\n // Query again in case of tool call\n if (!skipAnswer) {\n await this.generateAnswer(stream, stepCount + 1)\n }\n this.abortController = undefined\n } catch (error) {\n if (error instanceof OpenAI.APIUserAbortError) return\n console.error('[OpenaiAgent] Error answering:', error)\n\n if (tryCount < (this.options.maxRetry ?? DEFAULT_MAX_RETRIES)) {\n await new Promise((resolve) =>\n setTimeout(resolve, this.options.retryDelay ?? DEFAULT_RETRY_DELAY)\n )\n await this.generateAnswer(stream, stepCount, tryCount + 1)\n }\n }\n }\n\n private buildInput(): OpenAI.Responses.ResponseInput {\n return this.conversation.map((message) => {\n switch (message.role) {\n case 'user':\n case 'assistant':\n case 'system':\n return {\n role: message.role,\n content: message.content,\n }\n case 'tool_call':\n return {\n type: 'function_call',\n call_id: message.toolCallId,\n name: message.toolName,\n arguments: JSON.stringify(message.parameters),\n }\n case 'tool_result':\n return {\n type: 'function_call_output',\n call_id: message.toolCallId,\n output: message.output,\n }\n }\n })\n }\n\n private buildTools(): OpenAI.Responses.Tool[] | undefined {\n // Disable tools if first message\n const enableTools = this.conversation.length > 1\n if (!enableTools) return undefined\n\n return this.tools.map((tool) => ({\n type: 'function',\n name: tool.name,\n description: tool.description,\n strict: true,\n parameters: toJSONSchema(tool.inputSchema || z.object()),\n }))\n }\n\n cancel() {\n if (!this.abortController) return\n this.log('Cancel')\n this.abortController.abort()\n this.abortController = undefined\n }\n}\n","import { STT } from '@micdrop/server'\nimport { Readable } from 'stream'\nimport WebSocket from 'ws'\n\n/**\n * OpenAI Real-time STT\n *\n * @see https://platform.openai.com/docs/guides/speech-to-text\n */\n\nexport interface OpenaiSTTOptions {\n apiKey: string\n model?: string\n language?: string\n prompt?: string\n transcriptionTimeout?: number\n retryDelay?: number\n maxRetry?: number\n}\n\nconst DEFAULT_MODEL = 'gpt-4o-transcribe'\nconst DEFAULT_LANGUAGE = 'en'\nconst SAMPLE_RATE = 16000\nconst BIT_DEPTH = 16\nconst DEFAULT_TRANSCRIPTION_TIMEOUT = 4000\nconst DEFAULT_RETRY_DELAY = 1000\nconst DEFAULT_MAX_RETRY = 3\n\nexport class OpenaiSTT extends STT {\n private socket?: WebSocket\n private initPromise: Promise<void>\n private reconnectTimeout?: NodeJS.Timeout\n private retryCount = 0\n private transcriptionTimeout?: NodeJS.Timeout\n private audioChunksPending: Buffer[] = [] // Store audio chunks to send them again if reconnecting\n private ephemeralToken?: string\n\n constructor(private options: OpenaiSTTOptions) {\n super()\n\n // Setup WebSocket connection (first create session, then connect)\n this.initPromise = this.createTranscriptionSession()\n .then(() => this.initWS())\n .catch((error) => {\n console.error('[OpenaiSTT] Connection error:', error)\n this.reconnect()\n })\n }\n\n transcribe(audioStream: Readable) {\n // Read audio stream and send to OpenAI\n audioStream.on('data', async (chunk: Buffer) => {\n this.audioChunksPending.push(chunk)\n await this.initPromise\n this.sendAudioChunk(chunk)\n this.log(`Sent audio chunk (${chunk.byteLength} bytes)`)\n })\n\n // Handle stream end\n audioStream.on('end', async () => {\n await this.initPromise\n if (this.audioChunksPending.length === 0) return\n this.sendSilence(2)\n\n // Timeout transcription if no transcript is received\n this.transcriptionTimeout = setTimeout(() => {\n this.transcriptionTimeout = undefined\n this.log('Transcription timeout')\n this.emit('Transcript', '')\n this.audioChunksPending.length = 0\n }, this.options.transcriptionTimeout || DEFAULT_TRANSCRIPTION_TIMEOUT)\n })\n }\n\n destroy() {\n super.destroy()\n if (this.reconnectTimeout) {\n clearTimeout(this.reconnectTimeout)\n this.reconnectTimeout = undefined\n }\n if (this.transcriptionTimeout) {\n clearTimeout(this.transcriptionTimeout)\n this.transcriptionTimeout = undefined\n }\n this.socket?.removeAllListeners()\n if (this.socket?.readyState === WebSocket.OPEN) {\n this.socket?.close(1000)\n }\n this.socket = undefined\n }\n\n private async createTranscriptionSession(): Promise<void> {\n const payload = {\n input_audio_format: 'pcm16',\n input_audio_transcription: {\n model: this.options.model || DEFAULT_MODEL,\n language: this.options.language || DEFAULT_LANGUAGE,\n prompt:\n this.options.prompt || 'Transcribe the incoming audio in real time.',\n },\n turn_detection: {\n type: 'server_vad',\n silence_duration_ms: 1000,\n },\n input_audio_noise_reduction: {\n type: 'near_field',\n },\n }\n\n const response = await fetch(\n 'https://api.openai.com/v1/realtime/transcription_sessions',\n {\n method: 'POST',\n headers: {\n Authorization: `Bearer ${this.options.apiKey}`,\n 'Content-Type': 'application/json',\n 'OpenAI-Beta': 'realtime=v1',\n },\n body: JSON.stringify(payload),\n }\n )\n\n if (!response.ok) {\n const status = response.status\n\n // Don't retry on 4xx errors\n if (status >= 400 && status < 500) {\n throw new Error(\n `Failed to create transcription session: ${status} ${response.statusText}`\n )\n }\n\n // Retry on other errors\n this.log('Error creating transcription session, retrying...', {\n status,\n text: response.statusText,\n })\n await new Promise((resolve) =>\n setTimeout(resolve, this.options.retryDelay ?? DEFAULT_RETRY_DELAY)\n )\n return this.createTranscriptionSession()\n }\n\n const data = (await response.json()) as { client_secret: { value: string } }\n this.ephemeralToken = data.client_secret.value\n this.log('Transcription session created')\n }\n\n private async initWS(): Promise<void> {\n return new Promise<void>((resolve) => {\n const socket = new WebSocket('wss://api.openai.com/v1/realtime', {\n headers: {\n Authorization: `Bearer ${this.ephemeralToken}`,\n 'OpenAI-Beta': 'realtime=v1',\n },\n })\n this.socket = socket\n\n socket.addEventListener('open', () => {\n this.log('Connection opened')\n resolve()\n })\n\n socket.addEventListener('error', (error: any) => {\n this.log('WebSocket error:', error)\n })\n\n socket.addEventListener(\n 'close',\n ({ code, reason }: { code: number; reason: string }) => {\n this.socket?.removeAllListeners()\n this.socket = undefined\n\n if (code !== 1000) {\n this.reconnect()\n } else {\n this.log('Connection closed', { code, reason })\n }\n }\n )\n\n socket.addEventListener('message', (event: any) => {\n this.handleMessage(JSON.parse(event.data.toString()))\n })\n })\n }\n\n private sendAudioChunk(chunk: Buffer) {\n if (!this.socket) return\n\n const audioMessage = {\n type: 'input_audio_buffer.append',\n audio: chunk.toString('base64'),\n }\n\n this.socket.send(JSON.stringify(audioMessage))\n }\n\n private handleMessage(message: any) {\n switch (message.type) {\n // case 'input_audio_buffer.committed':\n // this.log('Audio buffer committed')\n // break\n\n // case 'input_audio_buffer.speech_started':\n // this.log('Speech started')\n // break\n\n // case 'input_audio_buffer.speech_stopped':\n // this.log('Speech stopped')\n // break\n\n // case 'conversation.item.input_audio_transcription.delta':\n // this.log(`Received transcript delta: \"${message.delta}\"`)\n // break\n\n case 'conversation.item.input_audio_transcription.completed':\n const transcript = message.transcript || ''\n this.log(`Received completed transcript: \"${transcript}\"`)\n this.emit('Transcript', transcript)\n // Reset audio chunks and clear timeout\n this.audioChunksPending.length = 0\n if (this.transcriptionTimeout) {\n clearTimeout(this.transcriptionTimeout)\n this.transcriptionTimeout = undefined\n }\n break\n\n case 'error':\n this.log('Error:', message.error)\n break\n\n default:\n break\n }\n }\n\n private reconnect() {\n this.retryCount++\n if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY)) {\n this.log('Max retries reached, giving up')\n this.emit('Failed', this.audioChunksPending)\n return\n }\n\n this.initPromise = new Promise((resolve) => {\n this.log('Reconnecting...')\n this.reconnectTimeout = setTimeout(() => {\n this.reconnectTimeout = undefined\n this.createTranscriptionSession()\n .then(() => this.initWS())\n .then(() => {\n this.retryCount = 0\n\n // Send audio chunks again if reconnecting during transcription\n if (this.audioChunksPending.length > 0) {\n this.log('Sending audio chunks again')\n this.audioChunksPending.forEach((chunk) =>\n this.sendAudioChunk(chunk)\n )\n }\n })\n .then(resolve)\n .catch((error) => {\n this.log('Reconnection error:', error)\n this.reconnect()\n })\n }, this.options.retryDelay ?? DEFAULT_RETRY_DELAY)\n })\n }\n\n private sendSilence(durationSeconds: number) {\n if (!this.socket) return\n const numSamples = Math.round(SAMPLE_RATE * durationSeconds)\n const bytesPerSample = BIT_DEPTH / 8\n const silenceBuffer = Buffer.alloc(numSamples * bytesPerSample)\n this.audioChunksPending.push(silenceBuffer)\n this.sendAudioChunk(silenceBuffer)\n this.log(\n `Sent ${durationSeconds * 1000}ms of silence (${silenceBuffer.byteLength} bytes) after stream end`\n )\n }\n}\n"],"mappings":";AAAA,SAAS,aAA2B;AACpC,OAAO,YAAY;AAEnB,OAAO,KAAK,oBAAoB;AAgBhC,IAAM,gBAAgB;AACtB,IAAM,oBAAoB;AAC1B,IAAM,sBAAsB;AAC5B,IAAM,sBAAsB;AAErB,IAAM,cAAN,cAA0B,MAA0B;AAAA,EAIzD,YAAY,SAA6B;AACvC,UAAM,OAAO;AACb,SAAK,SACH,YAAY,UACR,QAAQ,SACR,IAAI,OAAO,EAAE,QAAQ,QAAQ,OAAO,CAAC;AAAA,EAC7C;AAAA,EAEA,MAAgB,eACd,QACA,YAAY,GACZ,WAAW,GACI;AACf,QAAI,cAAc,KAAK,QAAQ,YAAY,oBAAoB;AAC7D,cAAQ,MAAM,iCAAiC;AAC/C;AAAA,IACF;AAEA,UAAM,kBAAkB,IAAI,gBAAgB;AAC5C,SAAK,kBAAkB;AACvB,UAAM,SAAS,gBAAgB;AAG/B,QAAI,aAAa;AACjB,UAAM,iBAAiB,KAAK,kBAAkB;AAE9C,QAAI;AAEF,YAAM,WAAW,MAAM,KAAK,OAAO,UAAU;AAAA,QAC3C;AAAA,UACE,OAAO,KAAK,QAAQ,SAAS;AAAA,UAC7B,OAAO,KAAK,WAAW;AAAA,UACvB,OAAO,KAAK,WAAW;AAAA,UACvB,QAAQ;AAAA,UACR,GAAG,KAAK,QAAQ;AAAA,QAClB;AAAA,QACA,EAAE,OAAO;AAAA,MACX;AACA,UAAI,aAAa;AAGjB,uBAAiB,SAAS,UAAU;AAClC,YAAI,oBAAoB,KAAK,gBAAiB;AAC9C,gBAAQ,MAAM,MAAM;AAAA,UAClB,KAAK;AACH,iBAAK,IAAI,kBAAkB,MAAM,KAAK,GAAG;AAGzC,gBAAI,gBAAgB;AAClB,kBAAI,CAAC,YAAY;AACf,sBAAM,gBAAgB,MAAM,MAAM;AAAA,kBAChC,eAAe;AAAA,gBACjB;AACA,oBAAI,kBAAkB,IAAI;AACxB,+BAAa;AACb,wBAAM,cAAc,MAAM,MACvB,MAAM,GAAG,aAAa,EACtB,QAAQ;AACX,yBAAO,MAAM,WAAW;AACxB;AAAA,gBACF;AAAA,cACF,OAAO;AAEL;AAAA,cACF;AAAA,YACF;AAGA,mBAAO,MAAM,MAAM,KAAK;AACxB;AAAA,UAEF,KAAK;AACH,kBAAM,EAAE,SAAS,SAAS,IAAI,KAAK,QAAQ,MAAM,IAAI;AACrD,iBAAK,oBAAoB,SAAS,QAAQ;AAC1C,iBAAK,kBAAkB;AAMvB;AAAA,UAEF,KAAK;AACH,gBAAI,MAAM,KAAK,SAAS,iBAAiB;AACvC,oBAAM,WAAW,MAAM;AACvB,oBAAM,SAAS,MAAM,KAAK,YAAY;AAAA,gBACpC,MAAM;AAAA,gBACN,YAAY,SAAS;AAAA,gBACrB,UAAU,SAAS;AAAA,gBACnB,YAAY,SAAS;AAAA,cACvB,CAAC;AACD,kBAAI,OAAO,YAAY;AACrB,6BAAa;AAAA,cACf;AAAA,YACF;AACA;AAAA,UAEF;AACE;AAAA,QACJ;AAAA,MACF;AAEA,UAAI,oBAAoB,KAAK,gBAAiB;AAG9C,UAAI,CAAC,YAAY;AACf,cAAM,KAAK,eAAe,QAAQ,YAAY,CAAC;AAAA,MACjD;AACA,WAAK,kBAAkB;AAAA,IACzB,SAAS,OAAO;AACd,UAAI,iBAAiB,OAAO,kBAAmB;AAC/C,cAAQ,MAAM,kCAAkC,KAAK;AAErD,UAAI,YAAY,KAAK,QAAQ,YAAY,sBAAsB;AAC7D,cAAM,IAAI;AAAA,UAAQ,CAAC,YACjB,WAAW,SAAS,KAAK,QAAQ,cAAc,mBAAmB;AAAA,QACpE;AACA,cAAM,KAAK,eAAe,QAAQ,WAAW,WAAW,CAAC;AAAA,MAC3D;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,aAA6C;AACnD,WAAO,KAAK,aAAa,IAAI,CAAC,YAAY;AACxC,cAAQ,QAAQ,MAAM;AAAA,QACpB,KAAK;AAAA,QACL,KAAK;AAAA,QACL,KAAK;AACH,iBAAO;AAAA,YACL,MAAM,QAAQ;AAAA,YACd,SAAS,QAAQ;AAAA,UACnB;AAAA,QACF,KAAK;AACH,iBAAO;AAAA,YACL,MAAM;AAAA,YACN,SAAS,QAAQ;AAAA,YACjB,MAAM,QAAQ;AAAA,YACd,WAAW,KAAK,UAAU,QAAQ,UAAU;AAAA,UAC9C;AAAA,QACF,KAAK;AACH,iBAAO;AAAA,YACL,MAAM;AAAA,YACN,SAAS,QAAQ;AAAA,YACjB,QAAQ,QAAQ;AAAA,UAClB;AAAA,MACJ;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEQ,aAAkD;AAExD,UAAM,cAAc,KAAK,aAAa,SAAS;AAC/C,QAAI,CAAC,YAAa,QAAO;AAEzB,WAAO,KAAK,MAAM,IAAI,CAAC,UAAU;AAAA,MAC/B,MAAM;AAAA,MACN,MAAM,KAAK;AAAA,MACX,aAAa,KAAK;AAAA,MAClB,QAAQ;AAAA,MACR,YAAY,aAAa,KAAK,eAAe,EAAE,OAAO,CAAC;AAAA,IACzD,EAAE;AAAA,EACJ;AAAA,EAEA,SAAS;AACP,QAAI,CAAC,KAAK,gBAAiB;AAC3B,SAAK,IAAI,QAAQ;AACjB,SAAK,gBAAgB,MAAM;AAC3B,SAAK,kBAAkB;AAAA,EACzB;AACF;;;ACrMA,SAAS,WAAW;AAEpB,OAAO,eAAe;AAkBtB,IAAMA,iBAAgB;AACtB,IAAM,mBAAmB;AACzB,IAAM,cAAc;AACpB,IAAM,YAAY;AAClB,IAAM,gCAAgC;AACtC,IAAMC,uBAAsB;AAC5B,IAAM,oBAAoB;AAEnB,IAAM,YAAN,cAAwB,IAAI;AAAA,EASjC,YAAoB,SAA2B;AAC7C,UAAM;AADY;AALpB,SAAQ,aAAa;AAErB,SAAQ,qBAA+B,CAAC;AAOtC,SAAK,cAAc,KAAK,2BAA2B,EAChD,KAAK,MAAM,KAAK,OAAO,CAAC,EACxB,MAAM,CAAC,UAAU;AAChB,cAAQ,MAAM,iCAAiC,KAAK;AACpD,WAAK,UAAU;AAAA,IACjB,CAAC;AAAA,EACL;AAAA,EAEA,WAAW,aAAuB;AAEhC,gBAAY,GAAG,QAAQ,OAAO,UAAkB;AAC9C,WAAK,mBAAmB,KAAK,KAAK;AAClC,YAAM,KAAK;AACX,WAAK,eAAe,KAAK;AACzB,WAAK,IAAI,qBAAqB,MAAM,UAAU,SAAS;AAAA,IACzD,CAAC;AAGD,gBAAY,GAAG,OAAO,YAAY;AAChC,YAAM,KAAK;AACX,UAAI,KAAK,mBAAmB,WAAW,EAAG;AAC1C,WAAK,YAAY,CAAC;AAGlB,WAAK,uBAAuB,WAAW,MAAM;AAC3C,aAAK,uBAAuB;AAC5B,aAAK,IAAI,uBAAuB;AAChC,aAAK,KAAK,cAAc,EAAE;AAC1B,aAAK,mBAAmB,SAAS;AAAA,MACnC,GAAG,KAAK,QAAQ,wBAAwB,6BAA6B;AAAA,IACvE,CAAC;AAAA,EACH;AAAA,EAEA,UAAU;AACR,UAAM,QAAQ;AACd,QAAI,KAAK,kBAAkB;AACzB,mBAAa,KAAK,gBAAgB;AAClC,WAAK,mBAAmB;AAAA,IAC1B;AACA,QAAI,KAAK,sBAAsB;AAC7B,mBAAa,KAAK,oBAAoB;AACtC,WAAK,uBAAuB;AAAA,IAC9B;AACA,SAAK,QAAQ,mBAAmB;AAChC,QAAI,KAAK,QAAQ,eAAe,UAAU,MAAM;AAC9C,WAAK,QAAQ,MAAM,GAAI;AAAA,IACzB;AACA,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,MAAc,6BAA4C;AACxD,UAAM,UAAU;AAAA,MACd,oBAAoB;AAAA,MACpB,2BAA2B;AAAA,QACzB,OAAO,KAAK,QAAQ,SAASD;AAAA,QAC7B,UAAU,KAAK,QAAQ,YAAY;AAAA,QACnC,QACE,KAAK,QAAQ,UAAU;AAAA,MAC3B;AAAA,MACA,gBAAgB;AAAA,QACd,MAAM;AAAA,QACN,qBAAqB;AAAA,MACvB;AAAA,MACA,6BAA6B;AAAA,QAC3B,MAAM;AAAA,MACR;AAAA,IACF;AAEA,UAAM,WAAW,MAAM;AAAA,MACrB;AAAA,MACA;AAAA,QACE,QAAQ;AAAA,QACR,SAAS;AAAA,UACP,eAAe,UAAU,KAAK,QAAQ,MAAM;AAAA,UAC5C,gBAAgB;AAAA,UAChB,eAAe;AAAA,QACjB;AAAA,QACA,MAAM,KAAK,UAAU,OAAO;AAAA,MAC9B;AAAA,IACF;AAEA,QAAI,CAAC,SAAS,IAAI;AAChB,YAAM,SAAS,SAAS;AAGxB,UAAI,UAAU,OAAO,SAAS,KAAK;AACjC,cAAM,IAAI;AAAA,UACR,2CAA2C,MAAM,IAAI,SAAS,UAAU;AAAA,QAC1E;AAAA,MACF;AAGA,WAAK,IAAI,qDAAqD;AAAA,QAC5D;AAAA,QACA,MAAM,SAAS;AAAA,MACjB,CAAC;AACD,YAAM,IAAI;AAAA,QAAQ,CAAC,YACjB,WAAW,SAAS,KAAK,QAAQ,cAAcC,oBAAmB;AAAA,MACpE;AACA,aAAO,KAAK,2BAA2B;AAAA,IACzC;AAEA,UAAM,OAAQ,MAAM,SAAS,KAAK;AAClC,SAAK,iBAAiB,KAAK,cAAc;AACzC,SAAK,IAAI,+BAA+B;AAAA,EAC1C;AAAA,EAEA,MAAc,SAAwB;AACpC,WAAO,IAAI,QAAc,CAAC,YAAY;AACpC,YAAM,SAAS,IAAI,UAAU,oCAAoC;AAAA,QAC/D,SAAS;AAAA,UACP,eAAe,UAAU,KAAK,cAAc;AAAA,UAC5C,eAAe;AAAA,QACjB;AAAA,MACF,CAAC;AACD,WAAK,SAAS;AAEd,aAAO,iBAAiB,QAAQ,MAAM;AACpC,aAAK,IAAI,mBAAmB;AAC5B,gBAAQ;AAAA,MACV,CAAC;AAED,aAAO,iBAAiB,SAAS,CAAC,UAAe;AAC/C,aAAK,IAAI,oBAAoB,KAAK;AAAA,MACpC,CAAC;AAED,aAAO;AAAA,QACL;AAAA,QACA,CAAC,EAAE,MAAM,OAAO,MAAwC;AACtD,eAAK,QAAQ,mBAAmB;AAChC,eAAK,SAAS;AAEd,cAAI,SAAS,KAAM;AACjB,iBAAK,UAAU;AAAA,UACjB,OAAO;AACL,iBAAK,IAAI,qBAAqB,EAAE,MAAM,OAAO,CAAC;AAAA,UAChD;AAAA,QACF;AAAA,MACF;AAEA,aAAO,iBAAiB,WAAW,CAAC,UAAe;AACjD,aAAK,cAAc,KAAK,MAAM,MAAM,KAAK,SAAS,CAAC,CAAC;AAAA,MACtD,CAAC;AAAA,IACH,CAAC;AAAA,EACH;AAAA,EAEQ,eAAe,OAAe;AACpC,QAAI,CAAC,KAAK,OAAQ;AAElB,UAAM,eAAe;AAAA,MACnB,MAAM;AAAA,MACN,OAAO,MAAM,SAAS,QAAQ;AAAA,IAChC;AAEA,SAAK,OAAO,KAAK,KAAK,UAAU,YAAY,CAAC;AAAA,EAC/C;AAAA,EAEQ,cAAc,SAAc;AAClC,YAAQ,QAAQ,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAiBpB,KAAK;AACH,cAAM,aAAa,QAAQ,cAAc;AACzC,aAAK,IAAI,mCAAmC,UAAU,GAAG;AACzD,aAAK,KAAK,cAAc,UAAU;AAElC,aAAK,mBAAmB,SAAS;AACjC,YAAI,KAAK,sBAAsB;AAC7B,uBAAa,KAAK,oBAAoB;AACtC,eAAK,uBAAuB;AAAA,QAC9B;AACA;AAAA,MAEF,KAAK;AACH,aAAK,IAAI,UAAU,QAAQ,KAAK;AAChC;AAAA,MAEF;AACE;AAAA,IACJ;AAAA,EACF;AAAA,EAEQ,YAAY;AAClB,SAAK;AACL,QAAI,KAAK,cAAc,KAAK,QAAQ,YAAY,oBAAoB;AAClE,WAAK,IAAI,gCAAgC;AACzC,WAAK,KAAK,UAAU,KAAK,kBAAkB;AAC3C;AAAA,IACF;AAEA,SAAK,cAAc,IAAI,QAAQ,CAAC,YAAY;AAC1C,WAAK,IAAI,iBAAiB;AAC1B,WAAK,mBAAmB,WAAW,MAAM;AACvC,aAAK,mBAAmB;AACxB,aAAK,2BAA2B,EAC7B,KAAK,MAAM,KAAK,OAAO,CAAC,EACxB,KAAK,MAAM;AACV,eAAK,aAAa;AAGlB,cAAI,KAAK,mBAAmB,SAAS,GAAG;AACtC,iBAAK,IAAI,4BAA4B;AACrC,iBAAK,mBAAmB;AAAA,cAAQ,CAAC,UAC/B,KAAK,eAAe,KAAK;AAAA,YAC3B;AAAA,UACF;AAAA,QACF,CAAC,EACA,KAAK,OAAO,EACZ,MAAM,CAAC,UAAU;AAChB,eAAK,IAAI,uBAAuB,KAAK;AACrC,eAAK,UAAU;AAAA,QACjB,CAAC;AAAA,MACL,GAAG,KAAK,QAAQ,cAAcA,oBAAmB;AAAA,IACnD,CAAC;AAAA,EACH;AAAA,EAEQ,YAAY,iBAAyB;AAC3C,QAAI,CAAC,KAAK,OAAQ;AAClB,UAAM,aAAa,KAAK,MAAM,cAAc,eAAe;AAC3D,UAAM,iBAAiB,YAAY;AACnC,UAAM,gBAAgB,OAAO,MAAM,aAAa,cAAc;AAC9D,SAAK,mBAAmB,KAAK,aAAa;AAC1C,SAAK,eAAe,aAAa;AACjC,SAAK;AAAA,MACH,QAAQ,kBAAkB,GAAI,kBAAkB,cAAc,UAAU;AAAA,IAC1E;AAAA,EACF;AACF;","names":["DEFAULT_MODEL","DEFAULT_RETRY_DELAY"]}
|
package/package.json
CHANGED
|
@@ -1,25 +1,18 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@micdrop/openai",
|
|
3
|
-
"version": "1.1
|
|
3
|
+
"version": "1.2.1",
|
|
4
4
|
"description": "OpenAI implementation for @micdrop/server",
|
|
5
5
|
"author": "Lonestone",
|
|
6
6
|
"license": "MIT",
|
|
7
7
|
"main": "./dist/index.js",
|
|
8
8
|
"module": "./dist/index.mjs",
|
|
9
9
|
"types": "./dist/index.d.ts",
|
|
10
|
-
"scripts": {
|
|
11
|
-
"build": "tsup",
|
|
12
|
-
"dev": "tsup --watch",
|
|
13
|
-
"clean": "rm -rf dist",
|
|
14
|
-
"typecheck": "tsc --noEmit",
|
|
15
|
-
"prepublishOnly": "npm run build"
|
|
16
|
-
},
|
|
17
10
|
"files": [
|
|
18
11
|
"dist"
|
|
19
12
|
],
|
|
20
13
|
"repository": {
|
|
21
14
|
"type": "git",
|
|
22
|
-
"url": "git+https://github.com/
|
|
15
|
+
"url": "git+https://github.com/Godefroy/micdrop.git",
|
|
23
16
|
"directory": "packages/openai"
|
|
24
17
|
},
|
|
25
18
|
"publishConfig": {
|
|
@@ -34,17 +27,23 @@
|
|
|
34
27
|
"openai"
|
|
35
28
|
],
|
|
36
29
|
"dependencies": {
|
|
37
|
-
"openai": "^
|
|
38
|
-
"ws": "^8.
|
|
30
|
+
"openai": "^6.0.0",
|
|
31
|
+
"ws": "^8.19.0"
|
|
39
32
|
},
|
|
40
33
|
"peerDependencies": {
|
|
41
34
|
"@micdrop/server": "*",
|
|
42
35
|
"zod": "^4"
|
|
43
36
|
},
|
|
44
37
|
"devDependencies": {
|
|
45
|
-
"@types/node": "^22.13
|
|
38
|
+
"@types/node": "^22.19.13",
|
|
46
39
|
"@types/ws": "^8.18.1",
|
|
47
|
-
"tsup": "^8.
|
|
48
|
-
"typescript": "^5.
|
|
40
|
+
"tsup": "^8.5.1",
|
|
41
|
+
"typescript": "^5.9.3"
|
|
42
|
+
},
|
|
43
|
+
"scripts": {
|
|
44
|
+
"build": "tsup",
|
|
45
|
+
"dev": "tsup --watch",
|
|
46
|
+
"clean": "rm -rf dist",
|
|
47
|
+
"typecheck": "tsc --noEmit"
|
|
49
48
|
}
|
|
50
49
|
}
|