@micdrop/openai 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md ADDED
@@ -0,0 +1,220 @@
1
+ # @micdrop/openai
2
+
3
+ OpenAI implementation for [@micdrop/server](../server/README.md).
4
+
5
+ This package provides AI agent and speech-to-text implementations using OpenAI's API.
6
+
7
+ ## Features
8
+
9
+ - 🤖 **OpenAI Agent** - GPT-powered conversational AI with advanced features:
10
+ - Streaming responses for real-time conversation
11
+ - Auto end call detection when user requests to end conversation
12
+ - Semantic turn detection to handle incomplete sentences
13
+ - User noise filtering to ignore meaningless interjections
14
+ - Tool calling support for enhanced interactions
15
+ - 🎙️ **OpenAI STT** - Whisper-powered speech-to-text with:
16
+ - High-quality transcription using Whisper models
17
+ - Multi-language support
18
+ - Configurable model selection
19
+
20
+ ## Installation
21
+
22
+ ```bash
23
+ npm install @micdrop/openai
24
+ # or
25
+ yarn add @micdrop/openai
26
+ # or
27
+ pnpm add @micdrop/openai
28
+ ```
29
+
30
+ ## OpenAI Agent
31
+
32
+ ### Usage
33
+
34
+ ```typescript
35
+ import { OpenaiAgent } from '@micdrop/openai'
36
+ import { MicdropServer } from '@micdrop/server'
37
+
38
+ const agent = new OpenaiAgent({
39
+ apiKey: process.env.OPENAI_API_KEY || '',
40
+ model: 'gpt-4o', // Default model
41
+ systemPrompt: 'You are a helpful assistant',
42
+
43
+ // Advanced features (optional)
44
+ autoEndCall: true, // Automatically end call when user requests
45
+ autoSemanticTurn: true, // Handle incomplete sentences
46
+ autoIgnoreUserNoise: true, // Filter out meaningless sounds
47
+
48
+ // Custom OpenAI settings (optional)
49
+ settings: {
50
+ temperature: 0.7,
51
+ max_output_tokens: 150,
52
+ },
53
+ })
54
+
55
+ // Use with MicdropServer
56
+ new MicdropServer(socket, {
57
+ agent,
58
+ // ... other options
59
+ })
60
+ ```
61
+
62
+ ### Options
63
+
64
+ | Option | Type | Default | Description |
65
+ | --------------------- | ------------------- | ---------- | --------------------------------------- |
66
+ | `apiKey` | `string` | Required | Your OpenAI API key |
67
+ | `model` | `string` | `'gpt-4o'` | OpenAI model to use |
68
+ | `systemPrompt` | `string` | Required | System prompt for the agent |
69
+ | `autoEndCall` | `boolean \| string` | `false` | Auto-detect when user wants to end call |
70
+ | `autoSemanticTurn` | `boolean \| string` | `false` | Handle incomplete user sentences |
71
+ | `autoIgnoreUserNoise` | `boolean \| string` | `false` | Filter meaningless user sounds |
72
+ | `settings` | `object` | `{}` | Additional OpenAI API parameters |
73
+
74
+ ## OpenAI STT (Speech-to-Text)
75
+
76
+ ### Usage
77
+
78
+ ```typescript
79
+ import { OpenaiSTT } from '@micdrop/openai'
80
+ import { MicdropServer } from '@micdrop/server'
81
+
82
+ const stt = new OpenaiSTT({
83
+ apiKey: process.env.OPENAI_API_KEY || '',
84
+ model: 'whisper-1', // Default Whisper model
85
+ language: 'en', // Optional: specify language for better accuracy
86
+ })
87
+
88
+ // Use with MicdropServer
89
+ new MicdropServer(socket, {
90
+ stt,
91
+ // ... other options
92
+ })
93
+ ```
94
+
95
+ ### Options
96
+
97
+ | Option | Type | Default | Description |
98
+ | ---------- | -------- | ------------- | ------------------------------- |
99
+ | `apiKey` | `string` | Required | Your OpenAI API key |
100
+ | `model` | `string` | `'whisper-1'` | Whisper model to use |
101
+ | `language` | `string` | Optional | Language code for transcription |
102
+
103
+ ## Tool Management
104
+
105
+ The OpenAI Agent supports adding and removing custom tools to extend its capabilities.
106
+
107
+ ### Adding Tools
108
+
109
+ Use `addTool()` to add custom functions that the agent can call during conversations:
110
+
111
+ ```typescript
112
+ import { z } from 'zod'
113
+
114
+ const agent = new OpenaiAgent({
115
+ apiKey: process.env.OPENAI_API_KEY || '',
116
+ systemPrompt: 'You are a helpful assistant that can manage user information.',
117
+ })
118
+
119
+ // Add a simple tool without parameters
120
+ agent.addTool({
121
+ name: 'get_time',
122
+ description: 'Get the current time',
123
+ callback: () => new Date().toLocaleTimeString(),
124
+ })
125
+
126
+ // Add a tool with typed parameters using Zod schema
127
+ agent.addTool({
128
+ name: 'set_user_info',
129
+ description: 'Save user information to the database',
130
+ parameters: z.object({
131
+ city: z.string().describe('City'),
132
+ jobTitle: z.string().describe('Job title').nullable(),
133
+ experience: z
134
+ .number()
135
+ .describe('Number of years of experience of the user')
136
+ .nullable(),
137
+ }),
138
+ callback: ({ city, jobTitle, experience }) => {
139
+ // Your implementation here
140
+ console.log('Saving user:', { city, jobTitle, experience })
141
+ return { success: true, message: 'User information saved' }
142
+ },
143
+ })
144
+
145
+ // Add a tool that returns data for the conversation
146
+ agent.addTool({
147
+ name: 'search_database',
148
+ description: 'Search for items in the database',
149
+ parameters: z.object({
150
+ query: z.string().describe('Search query'),
151
+ limit: z.number().default(10).describe('Maximum number of results'),
152
+ }),
153
+ callback: async ({ query, limit }) => {
154
+ // Your search implementation
155
+ const results = await searchDatabase(query, limit)
156
+ return { results, total: results.length }
157
+ },
158
+ })
159
+ ```
160
+
161
+ ### Removing Tools
162
+
163
+ Use `removeTool()` to remove tools by name:
164
+
165
+ ```typescript
166
+ // Remove a specific tool
167
+ agent.removeTool('get_time')
168
+ ```
169
+
170
+ ## Advanced Features
171
+
172
+ ### Auto End Call
173
+
174
+ When enabled, the agent automatically detects when the user wants to end the conversation and triggers the call termination:
175
+
176
+ ```typescript
177
+ const agent = new OpenaiAgent({
178
+ apiKey: process.env.OPENAI_API_KEY || '',
179
+ systemPrompt: 'You are a helpful assistant',
180
+ autoEndCall: true, // Use default detection
181
+ // or provide custom prompt:
182
+ // autoEndCall: 'User is saying goodbye or wants to hang up',
183
+ })
184
+ ```
185
+
186
+ ### Semantic Turn Detection
187
+
188
+ Handles cases where users speak incomplete sentences, allowing for more natural conversation flow:
189
+
190
+ ```typescript
191
+ const agent = new OpenaiAgent({
192
+ apiKey: process.env.OPENAI_API_KEY || '',
193
+ systemPrompt: 'You are a helpful assistant',
194
+ autoSemanticTurn: true, // Wait for complete thoughts
195
+ // or provide custom prompt:
196
+ // autoSemanticTurn: 'Last user message is an incomplete sentence',
197
+ })
198
+ ```
199
+
200
+ ### User Noise Filtering
201
+
202
+ Filters out meaningless sounds like "uh", "hmm", "ahem" that don't carry conversational meaning:
203
+
204
+ ```typescript
205
+ const agent = new OpenaiAgent({
206
+ apiKey: process.env.OPENAI_API_KEY || '',
207
+ systemPrompt: 'You are a helpful assistant',
208
+ autoIgnoreUserNoise: true, // Ignore filler sounds
209
+ // or provide custom prompt:
210
+ // autoIgnoreUserNoise: 'Last user message is just an interjection',
211
+ })
212
+ ```
213
+
214
+ ## License
215
+
216
+ MIT
217
+
218
+ ## Author
219
+
220
+ Originally developed for [Raconte.ai](https://www.raconte.ai) and open sourced by [Lonestone](https://www.lonestone.io) ([GitHub](https://github.com/lonestone))
@@ -0,0 +1,55 @@
1
+ import { AgentOptions, Agent, FileSTT } from '@micdrop/server';
2
+ import OpenAI from 'openai';
3
+ import { PassThrough } from 'stream';
4
+ import { z } from 'zod';
5
+
6
+ interface Tool<Schema extends z.ZodObject = z.ZodObject> {
7
+ name: string;
8
+ description: string;
9
+ parameters?: Schema;
10
+ callback: (input: z.infer<Schema>) => any | Promise<any>;
11
+ }
12
+
13
+ interface OpenaiAgentOptions extends AgentOptions {
14
+ apiKey: string;
15
+ model?: string;
16
+ maxCalls?: number;
17
+ settings?: Omit<OpenAI.Responses.ResponseCreateParamsStreaming, 'input' | 'model'>;
18
+ autoEndCall?: boolean | string;
19
+ autoSemanticTurn?: boolean | string;
20
+ autoIgnoreUserNoise?: boolean | string;
21
+ }
22
+ declare class OpenaiAgent extends Agent<OpenaiAgentOptions> {
23
+ private openai;
24
+ private tools;
25
+ private abortController?;
26
+ constructor(options: OpenaiAgentOptions);
27
+ answer(): {
28
+ message: Promise<string>;
29
+ stream: PassThrough;
30
+ };
31
+ private generateAnswer;
32
+ addTool<Schema extends z.ZodObject>(tool: Tool<Schema>): void;
33
+ removeTool(name: string): void;
34
+ private getDefaultTools;
35
+ private executeTool;
36
+ cancel(): void;
37
+ }
38
+
39
+ /**
40
+ * OpenAI Whisper STT
41
+ *
42
+ * @see https://platform.openai.com/docs/guides/speech-to-text?lang=curl
43
+ */
44
+ interface OpenaiSTTOptions {
45
+ apiKey: string;
46
+ model?: string;
47
+ language?: string;
48
+ }
49
+ declare class OpenaiSTT extends FileSTT {
50
+ private options;
51
+ constructor(options: OpenaiSTTOptions);
52
+ transcribeFile(file: File): Promise<string>;
53
+ }
54
+
55
+ export { OpenaiAgent, type OpenaiAgentOptions, OpenaiSTT, type OpenaiSTTOptions };
@@ -0,0 +1,55 @@
1
+ import { AgentOptions, Agent, FileSTT } from '@micdrop/server';
2
+ import OpenAI from 'openai';
3
+ import { PassThrough } from 'stream';
4
+ import { z } from 'zod';
5
+
6
+ interface Tool<Schema extends z.ZodObject = z.ZodObject> {
7
+ name: string;
8
+ description: string;
9
+ parameters?: Schema;
10
+ callback: (input: z.infer<Schema>) => any | Promise<any>;
11
+ }
12
+
13
+ interface OpenaiAgentOptions extends AgentOptions {
14
+ apiKey: string;
15
+ model?: string;
16
+ maxCalls?: number;
17
+ settings?: Omit<OpenAI.Responses.ResponseCreateParamsStreaming, 'input' | 'model'>;
18
+ autoEndCall?: boolean | string;
19
+ autoSemanticTurn?: boolean | string;
20
+ autoIgnoreUserNoise?: boolean | string;
21
+ }
22
+ declare class OpenaiAgent extends Agent<OpenaiAgentOptions> {
23
+ private openai;
24
+ private tools;
25
+ private abortController?;
26
+ constructor(options: OpenaiAgentOptions);
27
+ answer(): {
28
+ message: Promise<string>;
29
+ stream: PassThrough;
30
+ };
31
+ private generateAnswer;
32
+ addTool<Schema extends z.ZodObject>(tool: Tool<Schema>): void;
33
+ removeTool(name: string): void;
34
+ private getDefaultTools;
35
+ private executeTool;
36
+ cancel(): void;
37
+ }
38
+
39
+ /**
40
+ * OpenAI Whisper STT
41
+ *
42
+ * @see https://platform.openai.com/docs/guides/speech-to-text?lang=curl
43
+ */
44
+ interface OpenaiSTTOptions {
45
+ apiKey: string;
46
+ model?: string;
47
+ language?: string;
48
+ }
49
+ declare class OpenaiSTT extends FileSTT {
50
+ private options;
51
+ constructor(options: OpenaiSTTOptions);
52
+ transcribeFile(file: File): Promise<string>;
53
+ }
54
+
55
+ export { OpenaiAgent, type OpenaiAgentOptions, OpenaiSTT, type OpenaiSTTOptions };
package/dist/index.js ADDED
@@ -0,0 +1,253 @@
1
+ "use strict";
2
+ var __create = Object.create;
3
+ var __defProp = Object.defineProperty;
4
+ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
5
+ var __getOwnPropNames = Object.getOwnPropertyNames;
6
+ var __getProtoOf = Object.getPrototypeOf;
7
+ var __hasOwnProp = Object.prototype.hasOwnProperty;
8
+ var __export = (target, all) => {
9
+ for (var name in all)
10
+ __defProp(target, name, { get: all[name], enumerable: true });
11
+ };
12
+ var __copyProps = (to, from, except, desc) => {
13
+ if (from && typeof from === "object" || typeof from === "function") {
14
+ for (let key of __getOwnPropNames(from))
15
+ if (!__hasOwnProp.call(to, key) && key !== except)
16
+ __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
17
+ }
18
+ return to;
19
+ };
20
+ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
21
+ // If the importer is in node compatibility mode or this is not an ESM
22
+ // file that has been converted to a CommonJS file using a Babel-
23
+ // compatible transform (i.e. "__esModule" has not been set), then set
24
+ // "default" to the CommonJS "module.exports" for node compatibility.
25
+ isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
26
+ mod
27
+ ));
28
+ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
29
+
30
+ // src/index.ts
31
+ var index_exports = {};
32
+ __export(index_exports, {
33
+ OpenaiAgent: () => OpenaiAgent,
34
+ OpenaiSTT: () => OpenaiSTT
35
+ });
36
+ module.exports = __toCommonJS(index_exports);
37
+
38
+ // src/OpenaiAgent.ts
39
+ var import_server = require("@micdrop/server");
40
+ var import_openai = __toESM(require("openai"));
41
+ var import_stream = require("stream");
42
+
43
+ // src/tools.ts
44
+ var import_zod = require("zod");
45
+ function createToolSchema(tool) {
46
+ return {
47
+ type: "function",
48
+ name: tool.name,
49
+ description: tool.description,
50
+ strict: true,
51
+ parameters: (0, import_zod.toJSONSchema)(tool.parameters || import_zod.z.object())
52
+ };
53
+ }
54
+ function createToolCallOutput(toolCall, args) {
55
+ return {
56
+ type: "function_call_output",
57
+ call_id: toolCall.call_id,
58
+ output: JSON.stringify(args)
59
+ };
60
+ }
61
+
62
+ // src/OpenaiAgent.ts
63
+ var DEFAULT_MODEL = "gpt-4o";
64
+ var DEFAULT_MAX_CALLS = 5;
65
+ var AUTO_END_CALL_TOOL_NAME = "end_call";
66
+ var AUTO_END_CALL_PROMPT = "User asks to end the call";
67
+ var AUTO_SEMANTIC_TURN_TOOL_NAME = "semantic_turn";
68
+ var AUTO_SEMANTIC_TURN_PROMPT = "Last user message is an incomplete sentence";
69
+ var AUTO_IGNORE_USER_NOISE_TOOL_NAME = "ignore_user_noise";
70
+ var AUTO_IGNORE_USER_NOISE_PROMPT = `Last user message is just an interjection or a sound that expresses emotion, hesitation, or reaction (ex: "Uh", "Ahem", "Hmm", "Ah") but doesn't carry any clear meaning like agreeing, refusing, or commanding`;
71
+ var OpenaiAgent = class extends import_server.Agent {
72
+ constructor(options) {
73
+ super(options);
74
+ this.openai = new import_openai.default({ apiKey: options.apiKey });
75
+ this.tools = this.getDefaultTools();
76
+ }
77
+ answer() {
78
+ this.cancel();
79
+ this.log("Start answering");
80
+ const stream = new import_stream.PassThrough();
81
+ const textPromise = this.createTextPromise();
82
+ this.generateAnswer(stream, textPromise).finally(() => {
83
+ this.abortController = void 0;
84
+ if (stream.writable) {
85
+ stream.end();
86
+ }
87
+ });
88
+ return { message: textPromise.promise, stream };
89
+ }
90
+ async generateAnswer(stream, textPromise, callCount = 0, toolInputs) {
91
+ if (callCount >= (this.options.maxCalls || DEFAULT_MAX_CALLS)) {
92
+ console.error("[OpenaiAgent] Max calls reached");
93
+ textPromise.reject(new Error("Max calls reached"));
94
+ return;
95
+ }
96
+ this.abortController = new AbortController();
97
+ const signal = this.abortController.signal;
98
+ const enableTools = this.conversation.length > 1;
99
+ const input = this.conversation.map(
100
+ (message) => ({
101
+ role: message.role,
102
+ content: message.content
103
+ })
104
+ );
105
+ if (toolInputs) {
106
+ input.push(...toolInputs);
107
+ }
108
+ try {
109
+ const response = await this.openai.responses.create(
110
+ {
111
+ model: this.options.model || DEFAULT_MODEL,
112
+ input,
113
+ temperature: 0.5,
114
+ max_output_tokens: 250,
115
+ stream: true,
116
+ tools: enableTools ? this.tools.map(createToolSchema) : void 0,
117
+ ...this.options.settings
118
+ },
119
+ { signal }
120
+ );
121
+ const toolOutputs = [];
122
+ for await (const event of response) {
123
+ switch (event.type) {
124
+ case "response.output_text.delta":
125
+ this.log(`Answer chunk: "${event.delta}"`);
126
+ stream.write(event.delta);
127
+ break;
128
+ case "response.output_text.done":
129
+ this.addAssistantMessage(event.text);
130
+ stream.end();
131
+ textPromise.resolve(event.text);
132
+ break;
133
+ case "response.output_item.done":
134
+ if (event.item.type === "function_call") {
135
+ toolOutputs.push(event.item);
136
+ const toolOutput = await this.executeTool(event.item);
137
+ toolOutputs.push(toolOutput);
138
+ }
139
+ break;
140
+ default:
141
+ break;
142
+ }
143
+ }
144
+ if (toolOutputs.length > 0) {
145
+ await this.generateAnswer(
146
+ stream,
147
+ textPromise,
148
+ callCount + 1,
149
+ toolOutputs
150
+ );
151
+ }
152
+ } catch (error) {
153
+ textPromise.reject(error);
154
+ if (!(error instanceof import_openai.default.APIUserAbortError)) {
155
+ console.error("[OpenaiAgent] Error answering:", error);
156
+ }
157
+ }
158
+ }
159
+ addTool(tool) {
160
+ this.tools.push(tool);
161
+ }
162
+ removeTool(name) {
163
+ this.tools = this.tools.filter((tool) => tool.name !== name);
164
+ }
165
+ getDefaultTools() {
166
+ const tools = [];
167
+ if (this.options.autoEndCall) {
168
+ tools.push({
169
+ name: AUTO_END_CALL_TOOL_NAME,
170
+ description: typeof this.options.autoEndCall === "string" ? this.options.autoEndCall : AUTO_END_CALL_PROMPT,
171
+ callback: () => this.endCall()
172
+ });
173
+ }
174
+ if (this.options.autoSemanticTurn) {
175
+ tools.push({
176
+ name: AUTO_SEMANTIC_TURN_TOOL_NAME,
177
+ description: typeof this.options.autoSemanticTurn === "string" ? this.options.autoSemanticTurn : AUTO_SEMANTIC_TURN_PROMPT,
178
+ callback: () => this.skipAnswer()
179
+ });
180
+ }
181
+ if (this.options.autoIgnoreUserNoise) {
182
+ tools.push({
183
+ name: AUTO_IGNORE_USER_NOISE_TOOL_NAME,
184
+ description: typeof this.options.autoIgnoreUserNoise === "string" ? this.options.autoIgnoreUserNoise : AUTO_IGNORE_USER_NOISE_PROMPT,
185
+ callback: () => this.cancelLastUserMessage()
186
+ });
187
+ }
188
+ return tools;
189
+ }
190
+ async executeTool(toolCall) {
191
+ try {
192
+ const tool = this.tools.find((tool2) => tool2.name === toolCall.name);
193
+ if (!tool) {
194
+ throw new Error(`Tool not found "${toolCall.name}"`);
195
+ }
196
+ this.log("Executing tool:", toolCall.name, toolCall.arguments);
197
+ const args = JSON.parse(toolCall.arguments);
198
+ const result = await tool.callback(args);
199
+ return createToolCallOutput(toolCall, result || {});
200
+ } catch (error) {
201
+ console.error("[OpenaiAgent] Error executing tool:", error);
202
+ return createToolCallOutput(toolCall, { error: error.message });
203
+ }
204
+ }
205
+ cancel() {
206
+ if (!this.abortController) return;
207
+ this.log("Cancel");
208
+ this.abortController.abort();
209
+ this.abortController = void 0;
210
+ }
211
+ };
212
+
213
+ // src/OpenaiSTT.ts
214
+ var import_server2 = require("@micdrop/server");
215
+ var DEFAULT_MODEL2 = "whisper-1";
216
+ var OpenaiSTT = class extends import_server2.FileSTT {
217
+ constructor(options) {
218
+ super();
219
+ this.options = options;
220
+ }
221
+ async transcribeFile(file) {
222
+ const formData = new FormData();
223
+ formData.append("file", file);
224
+ formData.append("model", this.options.model || DEFAULT_MODEL2);
225
+ formData.append("response_format", "text");
226
+ if (this.options.language) {
227
+ formData.append("language", this.options.language);
228
+ }
229
+ const response = await fetch(
230
+ "https://api.openai.com/v1/audio/transcriptions",
231
+ {
232
+ method: "POST",
233
+ headers: {
234
+ Authorization: `Bearer ${this.options.apiKey}`
235
+ },
236
+ body: formData
237
+ }
238
+ );
239
+ if (!response.ok) {
240
+ this.log("Failed to transcribe audio", await response.json());
241
+ throw new Error(`Failed to transcribe audio: ${response.statusText}`);
242
+ }
243
+ const transcript = await response.text();
244
+ this.log(`Received transcript: "${transcript}"`);
245
+ return transcript;
246
+ }
247
+ };
248
+ // Annotate the CommonJS export names for ESM import in node:
249
+ 0 && (module.exports = {
250
+ OpenaiAgent,
251
+ OpenaiSTT
252
+ });
253
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"sources":["../src/index.ts","../src/OpenaiAgent.ts","../src/tools.ts","../src/OpenaiSTT.ts"],"sourcesContent":["export * from './OpenaiAgent'\nexport * from './OpenaiSTT'\n","import { Agent, AgentOptions, TextPromise } from '@micdrop/server'\nimport OpenAI from 'openai'\nimport { PassThrough } from 'stream'\nimport { z } from 'zod'\nimport {\n createToolCallOutput,\n createToolSchema,\n Tool,\n ToolCall,\n ToolCallOutput,\n} from './tools'\n\nexport interface OpenaiAgentOptions extends AgentOptions {\n apiKey: string\n model?: string\n maxCalls?: number\n settings?: Omit<\n OpenAI.Responses.ResponseCreateParamsStreaming,\n 'input' | 'model'\n >\n\n // Enable auto ending of the call when user asks to end the call\n // You can provide a custom prompt to use instead of the default one by passing a string\n autoEndCall?: boolean | string\n\n // Enable detection of an incomplete sentence, and skip the answer (assistant waits)\n // You can provide a custom prompt to use instead of the default one by passing a string\n autoSemanticTurn?: boolean | string\n\n // Ignore of the last user message when it's meaningless\n // You can provide a custom prompt to use instead of the default one by passing a string\n autoIgnoreUserNoise?: boolean | string\n}\n\nconst DEFAULT_MODEL = 'gpt-4o'\nconst DEFAULT_MAX_CALLS = 5\n\nconst AUTO_END_CALL_TOOL_NAME = 'end_call'\nconst AUTO_END_CALL_PROMPT = 'User asks to end the call'\n\nconst AUTO_SEMANTIC_TURN_TOOL_NAME = 'semantic_turn'\nconst AUTO_SEMANTIC_TURN_PROMPT = 'Last user message is an incomplete sentence'\n\nconst AUTO_IGNORE_USER_NOISE_TOOL_NAME = 'ignore_user_noise'\nconst AUTO_IGNORE_USER_NOISE_PROMPT =\n 'Last user message is just an interjection or a sound that expresses emotion, hesitation, or reaction (ex: \"Uh\", \"Ahem\", \"Hmm\", \"Ah\") but doesn\\'t carry any clear meaning like agreeing, refusing, or commanding'\n\nexport class OpenaiAgent extends Agent<OpenaiAgentOptions> {\n private openai: OpenAI\n private tools: Tool[]\n private abortController?: AbortController\n\n constructor(options: OpenaiAgentOptions) {\n super(options)\n this.openai = new OpenAI({ apiKey: options.apiKey })\n this.tools = this.getDefaultTools()\n }\n\n answer() {\n this.cancel()\n this.log('Start answering')\n const stream = new PassThrough()\n const textPromise = this.createTextPromise()\n\n this.generateAnswer(stream, textPromise).finally(() => {\n this.abortController = undefined\n if (stream.writable) {\n stream.end()\n }\n })\n\n return { message: textPromise.promise, stream }\n }\n\n private async generateAnswer(\n stream: PassThrough,\n textPromise: TextPromise,\n callCount = 0,\n toolInputs?: OpenAI.Responses.ResponseInputItem[]\n ): Promise<void> {\n if (callCount >= (this.options.maxCalls || DEFAULT_MAX_CALLS)) {\n console.error('[OpenaiAgent] Max calls reached')\n textPromise.reject(new Error('Max calls reached'))\n return\n }\n\n this.abortController = new AbortController()\n const signal = this.abortController.signal\n\n // Disable tools if first message\n const enableTools = this.conversation.length > 1\n\n // Build input\n const input: OpenAI.Responses.ResponseInput = this.conversation.map(\n (message) => ({\n role: message.role,\n content: message.content,\n })\n )\n if (toolInputs) {\n input.push(...toolInputs)\n }\n\n try {\n // Generate answer\n const response = await this.openai.responses.create(\n {\n model: this.options.model || DEFAULT_MODEL,\n input,\n temperature: 0.5,\n max_output_tokens: 250,\n stream: true,\n tools: enableTools ? this.tools.map(createToolSchema) : undefined,\n ...this.options.settings,\n },\n { signal }\n )\n const toolOutputs: OpenAI.Responses.ResponseInputItem[] = []\n\n // Handle response events\n for await (const event of response) {\n switch (event.type) {\n case 'response.output_text.delta':\n this.log(`Answer chunk: \"${event.delta}\"`)\n stream.write(event.delta)\n break\n\n case 'response.output_text.done':\n this.addAssistantMessage(event.text)\n stream.end()\n textPromise.resolve(event.text)\n break\n\n case 'response.output_item.done':\n if (event.item.type === 'function_call') {\n toolOutputs.push(event.item)\n const toolOutput = await this.executeTool(event.item)\n toolOutputs.push(toolOutput)\n }\n break\n\n default:\n break\n }\n }\n\n // Query again in case of tool call\n if (toolOutputs.length > 0) {\n await this.generateAnswer(\n stream,\n textPromise,\n callCount + 1,\n toolOutputs\n )\n }\n } catch (error) {\n textPromise.reject(error)\n if (!(error instanceof OpenAI.APIUserAbortError)) {\n console.error('[OpenaiAgent] Error answering:', error)\n }\n }\n }\n\n addTool<Schema extends z.ZodObject>(tool: Tool<Schema>) {\n this.tools.push(tool)\n }\n\n removeTool(name: string) {\n this.tools = this.tools.filter((tool) => tool.name !== name)\n }\n\n private getDefaultTools() {\n const tools: Tool[] = []\n if (this.options.autoEndCall) {\n tools.push({\n name: AUTO_END_CALL_TOOL_NAME,\n description:\n typeof this.options.autoEndCall === 'string'\n ? this.options.autoEndCall\n : AUTO_END_CALL_PROMPT,\n callback: () => this.endCall(),\n })\n }\n if (this.options.autoSemanticTurn) {\n tools.push({\n name: AUTO_SEMANTIC_TURN_TOOL_NAME,\n description:\n typeof this.options.autoSemanticTurn === 'string'\n ? this.options.autoSemanticTurn\n : AUTO_SEMANTIC_TURN_PROMPT,\n callback: () => this.skipAnswer(),\n })\n }\n if (this.options.autoIgnoreUserNoise) {\n tools.push({\n name: AUTO_IGNORE_USER_NOISE_TOOL_NAME,\n description:\n typeof this.options.autoIgnoreUserNoise === 'string'\n ? this.options.autoIgnoreUserNoise\n : AUTO_IGNORE_USER_NOISE_PROMPT,\n callback: () => this.cancelLastUserMessage(),\n })\n }\n return tools\n }\n\n private async executeTool(toolCall: ToolCall): Promise<ToolCallOutput> {\n try {\n const tool = this.tools.find((tool) => tool.name === toolCall.name)\n if (!tool) {\n throw new Error(`Tool not found \"${toolCall.name}\"`)\n }\n\n this.log('Executing tool:', toolCall.name, toolCall.arguments)\n const args = JSON.parse(toolCall.arguments)\n const result = await tool.callback(args)\n return createToolCallOutput(toolCall, result || {})\n } catch (error: any) {\n console.error('[OpenaiAgent] Error executing tool:', error)\n return createToolCallOutput(toolCall, { error: error.message })\n }\n }\n\n cancel() {\n if (!this.abortController) return\n this.log('Cancel')\n this.abortController.abort()\n this.abortController = undefined\n }\n}\n","import {\n FunctionTool,\n ResponseInputItem,\n Responses,\n} from 'openai/resources/responses/responses'\nimport { toJSONSchema, z } from 'zod'\n\nexport interface Tool<Schema extends z.ZodObject = z.ZodObject> {\n name: string\n description: string\n parameters?: Schema\n callback: (input: z.infer<Schema>) => any | Promise<any>\n}\n\nexport type ToolCall = Responses.ResponseFunctionToolCall\nexport type ToolCallOutput = ResponseInputItem.FunctionCallOutput\n\nexport function createToolSchema<Schema extends z.ZodObject>(\n tool: Tool<Schema>\n): FunctionTool {\n return {\n type: 'function',\n name: tool.name,\n description: tool.description,\n strict: true,\n parameters: toJSONSchema(tool.parameters || z.object()),\n }\n}\n\nexport function createToolCallOutput(\n toolCall: ToolCall,\n args: any\n): ToolCallOutput {\n return {\n type: 'function_call_output',\n call_id: toolCall.call_id,\n output: JSON.stringify(args),\n }\n}\n","import { FileSTT } from '@micdrop/server'\n\n/**\n * OpenAI Whisper STT\n *\n * @see https://platform.openai.com/docs/guides/speech-to-text?lang=curl\n */\n\nexport interface OpenaiSTTOptions {\n apiKey: string\n model?: string\n language?: string\n}\n\nconst DEFAULT_MODEL = 'whisper-1'\n\nexport class OpenaiSTT extends FileSTT {\n constructor(private options: OpenaiSTTOptions) {\n super()\n }\n\n async transcribeFile(file: File) {\n // Prepare form data\n const formData = new FormData()\n formData.append('file', file)\n formData.append('model', this.options.model || DEFAULT_MODEL)\n formData.append('response_format', 'text')\n\n if (this.options.language) {\n formData.append('language', this.options.language)\n }\n\n const response = await fetch(\n 'https://api.openai.com/v1/audio/transcriptions',\n {\n method: 'POST',\n headers: {\n Authorization: `Bearer ${this.options.apiKey}`,\n },\n body: formData,\n }\n )\n\n if (!response.ok) {\n this.log('Failed to transcribe audio', await response.json())\n throw new Error(`Failed to transcribe audio: ${response.statusText}`)\n }\n\n const transcript = await response.text()\n this.log(`Received transcript: \"${transcript}\"`)\n\n return transcript\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACAA,oBAAiD;AACjD,oBAAmB;AACnB,oBAA4B;;;ACG5B,iBAAgC;AAYzB,SAAS,iBACd,MACc;AACd,SAAO;AAAA,IACL,MAAM;AAAA,IACN,MAAM,KAAK;AAAA,IACX,aAAa,KAAK;AAAA,IAClB,QAAQ;AAAA,IACR,gBAAY,yBAAa,KAAK,cAAc,aAAE,OAAO,CAAC;AAAA,EACxD;AACF;AAEO,SAAS,qBACd,UACA,MACgB;AAChB,SAAO;AAAA,IACL,MAAM;AAAA,IACN,SAAS,SAAS;AAAA,IAClB,QAAQ,KAAK,UAAU,IAAI;AAAA,EAC7B;AACF;;;ADJA,IAAM,gBAAgB;AACtB,IAAM,oBAAoB;AAE1B,IAAM,0BAA0B;AAChC,IAAM,uBAAuB;AAE7B,IAAM,+BAA+B;AACrC,IAAM,4BAA4B;AAElC,IAAM,mCAAmC;AACzC,IAAM,gCACJ;AAEK,IAAM,cAAN,cAA0B,oBAA0B;AAAA,EAKzD,YAAY,SAA6B;AACvC,UAAM,OAAO;AACb,SAAK,SAAS,IAAI,cAAAA,QAAO,EAAE,QAAQ,QAAQ,OAAO,CAAC;AACnD,SAAK,QAAQ,KAAK,gBAAgB;AAAA,EACpC;AAAA,EAEA,SAAS;AACP,SAAK,OAAO;AACZ,SAAK,IAAI,iBAAiB;AAC1B,UAAM,SAAS,IAAI,0BAAY;AAC/B,UAAM,cAAc,KAAK,kBAAkB;AAE3C,SAAK,eAAe,QAAQ,WAAW,EAAE,QAAQ,MAAM;AACrD,WAAK,kBAAkB;AACvB,UAAI,OAAO,UAAU;AACnB,eAAO,IAAI;AAAA,MACb;AAAA,IACF,CAAC;AAED,WAAO,EAAE,SAAS,YAAY,SAAS,OAAO;AAAA,EAChD;AAAA,EAEA,MAAc,eACZ,QACA,aACA,YAAY,GACZ,YACe;AACf,QAAI,cAAc,KAAK,QAAQ,YAAY,oBAAoB;AAC7D,cAAQ,MAAM,iCAAiC;AAC/C,kBAAY,OAAO,IAAI,MAAM,mBAAmB,CAAC;AACjD;AAAA,IACF;AAEA,SAAK,kBAAkB,IAAI,gBAAgB;AAC3C,UAAM,SAAS,KAAK,gBAAgB;AAGpC,UAAM,cAAc,KAAK,aAAa,SAAS;AAG/C,UAAM,QAAwC,KAAK,aAAa;AAAA,MAC9D,CAAC,aAAa;AAAA,QACZ,MAAM,QAAQ;AAAA,QACd,SAAS,QAAQ;AAAA,MACnB;AAAA,IACF;AACA,QAAI,YAAY;AACd,YAAM,KAAK,GAAG,UAAU;AAAA,IAC1B;AAEA,QAAI;AAEF,YAAM,WAAW,MAAM,KAAK,OAAO,UAAU;AAAA,QAC3C;AAAA,UACE,OAAO,KAAK,QAAQ,SAAS;AAAA,UAC7B;AAAA,UACA,aAAa;AAAA,UACb,mBAAmB;AAAA,UACnB,QAAQ;AAAA,UACR,OAAO,cAAc,KAAK,MAAM,IAAI,gBAAgB,IAAI;AAAA,UACxD,GAAG,KAAK,QAAQ;AAAA,QAClB;AAAA,QACA,EAAE,OAAO;AAAA,MACX;AACA,YAAM,cAAoD,CAAC;AAG3D,uBAAiB,SAAS,UAAU;AAClC,gBAAQ,MAAM,MAAM;AAAA,UAClB,KAAK;AACH,iBAAK,IAAI,kBAAkB,MAAM,KAAK,GAAG;AACzC,mBAAO,MAAM,MAAM,KAAK;AACxB;AAAA,UAEF,KAAK;AACH,iBAAK,oBAAoB,MAAM,IAAI;AACnC,mBAAO,IAAI;AACX,wBAAY,QAAQ,MAAM,IAAI;AAC9B;AAAA,UAEF,KAAK;AACH,gBAAI,MAAM,KAAK,SAAS,iBAAiB;AACvC,0BAAY,KAAK,MAAM,IAAI;AAC3B,oBAAM,aAAa,MAAM,KAAK,YAAY,MAAM,IAAI;AACpD,0BAAY,KAAK,UAAU;AAAA,YAC7B;AACA;AAAA,UAEF;AACE;AAAA,QACJ;AAAA,MACF;AAGA,UAAI,YAAY,SAAS,GAAG;AAC1B,cAAM,KAAK;AAAA,UACT;AAAA,UACA;AAAA,UACA,YAAY;AAAA,UACZ;AAAA,QACF;AAAA,MACF;AAAA,IACF,SAAS,OAAO;AACd,kBAAY,OAAO,KAAK;AACxB,UAAI,EAAE,iBAAiB,cAAAA,QAAO,oBAAoB;AAChD,gBAAQ,MAAM,kCAAkC,KAAK;AAAA,MACvD;AAAA,IACF;AAAA,EACF;AAAA,EAEA,QAAoC,MAAoB;AACtD,SAAK,MAAM,KAAK,IAAI;AAAA,EACtB;AAAA,EAEA,WAAW,MAAc;AACvB,SAAK,QAAQ,KAAK,MAAM,OAAO,CAAC,SAAS,KAAK,SAAS,IAAI;AAAA,EAC7D;AAAA,EAEQ,kBAAkB;AACxB,UAAM,QAAgB,CAAC;AACvB,QAAI,KAAK,QAAQ,aAAa;AAC5B,YAAM,KAAK;AAAA,QACT,MAAM;AAAA,QACN,aACE,OAAO,KAAK,QAAQ,gBAAgB,WAChC,KAAK,QAAQ,cACb;AAAA,QACN,UAAU,MAAM,KAAK,QAAQ;AAAA,MAC/B,CAAC;AAAA,IACH;AACA,QAAI,KAAK,QAAQ,kBAAkB;AACjC,YAAM,KAAK;AAAA,QACT,MAAM;AAAA,QACN,aACE,OAAO,KAAK,QAAQ,qBAAqB,WACrC,KAAK,QAAQ,mBACb;AAAA,QACN,UAAU,MAAM,KAAK,WAAW;AAAA,MAClC,CAAC;AAAA,IACH;AACA,QAAI,KAAK,QAAQ,qBAAqB;AACpC,YAAM,KAAK;AAAA,QACT,MAAM;AAAA,QACN,aACE,OAAO,KAAK,QAAQ,wBAAwB,WACxC,KAAK,QAAQ,sBACb;AAAA,QACN,UAAU,MAAM,KAAK,sBAAsB;AAAA,MAC7C,CAAC;AAAA,IACH;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAc,YAAY,UAA6C;AACrE,QAAI;AACF,YAAM,OAAO,KAAK,MAAM,KAAK,CAACC,UAASA,MAAK,SAAS,SAAS,IAAI;AAClE,UAAI,CAAC,MAAM;AACT,cAAM,IAAI,MAAM,mBAAmB,SAAS,IAAI,GAAG;AAAA,MACrD;AAEA,WAAK,IAAI,mBAAmB,SAAS,MAAM,SAAS,SAAS;AAC7D,YAAM,OAAO,KAAK,MAAM,SAAS,SAAS;AAC1C,YAAM,SAAS,MAAM,KAAK,SAAS,IAAI;AACvC,aAAO,qBAAqB,UAAU,UAAU,CAAC,CAAC;AAAA,IACpD,SAAS,OAAY;AACnB,cAAQ,MAAM,uCAAuC,KAAK;AAC1D,aAAO,qBAAqB,UAAU,EAAE,OAAO,MAAM,QAAQ,CAAC;AAAA,IAChE;AAAA,EACF;AAAA,EAEA,SAAS;AACP,QAAI,CAAC,KAAK,gBAAiB;AAC3B,SAAK,IAAI,QAAQ;AACjB,SAAK,gBAAgB,MAAM;AAC3B,SAAK,kBAAkB;AAAA,EACzB;AACF;;;AErOA,IAAAC,iBAAwB;AAcxB,IAAMC,iBAAgB;AAEf,IAAM,YAAN,cAAwB,uBAAQ;AAAA,EACrC,YAAoB,SAA2B;AAC7C,UAAM;AADY;AAAA,EAEpB;AAAA,EAEA,MAAM,eAAe,MAAY;AAE/B,UAAM,WAAW,IAAI,SAAS;AAC9B,aAAS,OAAO,QAAQ,IAAI;AAC5B,aAAS,OAAO,SAAS,KAAK,QAAQ,SAASA,cAAa;AAC5D,aAAS,OAAO,mBAAmB,MAAM;AAEzC,QAAI,KAAK,QAAQ,UAAU;AACzB,eAAS,OAAO,YAAY,KAAK,QAAQ,QAAQ;AAAA,IACnD;AAEA,UAAM,WAAW,MAAM;AAAA,MACrB;AAAA,MACA;AAAA,QACE,QAAQ;AAAA,QACR,SAAS;AAAA,UACP,eAAe,UAAU,KAAK,QAAQ,MAAM;AAAA,QAC9C;AAAA,QACA,MAAM;AAAA,MACR;AAAA,IACF;AAEA,QAAI,CAAC,SAAS,IAAI;AAChB,WAAK,IAAI,8BAA8B,MAAM,SAAS,KAAK,CAAC;AAC5D,YAAM,IAAI,MAAM,+BAA+B,SAAS,UAAU,EAAE;AAAA,IACtE;AAEA,UAAM,aAAa,MAAM,SAAS,KAAK;AACvC,SAAK,IAAI,yBAAyB,UAAU,GAAG;AAE/C,WAAO;AAAA,EACT;AACF;","names":["OpenAI","tool","import_server","DEFAULT_MODEL"]}
package/dist/index.mjs ADDED
@@ -0,0 +1,215 @@
1
+ // src/OpenaiAgent.ts
2
+ import { Agent } from "@micdrop/server";
3
+ import OpenAI from "openai";
4
+ import { PassThrough } from "stream";
5
+
6
+ // src/tools.ts
7
+ import { toJSONSchema, z } from "zod";
8
+ function createToolSchema(tool) {
9
+ return {
10
+ type: "function",
11
+ name: tool.name,
12
+ description: tool.description,
13
+ strict: true,
14
+ parameters: toJSONSchema(tool.parameters || z.object())
15
+ };
16
+ }
17
+ function createToolCallOutput(toolCall, args) {
18
+ return {
19
+ type: "function_call_output",
20
+ call_id: toolCall.call_id,
21
+ output: JSON.stringify(args)
22
+ };
23
+ }
24
+
25
+ // src/OpenaiAgent.ts
26
+ var DEFAULT_MODEL = "gpt-4o";
27
+ var DEFAULT_MAX_CALLS = 5;
28
+ var AUTO_END_CALL_TOOL_NAME = "end_call";
29
+ var AUTO_END_CALL_PROMPT = "User asks to end the call";
30
+ var AUTO_SEMANTIC_TURN_TOOL_NAME = "semantic_turn";
31
+ var AUTO_SEMANTIC_TURN_PROMPT = "Last user message is an incomplete sentence";
32
+ var AUTO_IGNORE_USER_NOISE_TOOL_NAME = "ignore_user_noise";
33
+ var AUTO_IGNORE_USER_NOISE_PROMPT = `Last user message is just an interjection or a sound that expresses emotion, hesitation, or reaction (ex: "Uh", "Ahem", "Hmm", "Ah") but doesn't carry any clear meaning like agreeing, refusing, or commanding`;
34
+ var OpenaiAgent = class extends Agent {
35
+ constructor(options) {
36
+ super(options);
37
+ this.openai = new OpenAI({ apiKey: options.apiKey });
38
+ this.tools = this.getDefaultTools();
39
+ }
40
+ answer() {
41
+ this.cancel();
42
+ this.log("Start answering");
43
+ const stream = new PassThrough();
44
+ const textPromise = this.createTextPromise();
45
+ this.generateAnswer(stream, textPromise).finally(() => {
46
+ this.abortController = void 0;
47
+ if (stream.writable) {
48
+ stream.end();
49
+ }
50
+ });
51
+ return { message: textPromise.promise, stream };
52
+ }
53
+ async generateAnswer(stream, textPromise, callCount = 0, toolInputs) {
54
+ if (callCount >= (this.options.maxCalls || DEFAULT_MAX_CALLS)) {
55
+ console.error("[OpenaiAgent] Max calls reached");
56
+ textPromise.reject(new Error("Max calls reached"));
57
+ return;
58
+ }
59
+ this.abortController = new AbortController();
60
+ const signal = this.abortController.signal;
61
+ const enableTools = this.conversation.length > 1;
62
+ const input = this.conversation.map(
63
+ (message) => ({
64
+ role: message.role,
65
+ content: message.content
66
+ })
67
+ );
68
+ if (toolInputs) {
69
+ input.push(...toolInputs);
70
+ }
71
+ try {
72
+ const response = await this.openai.responses.create(
73
+ {
74
+ model: this.options.model || DEFAULT_MODEL,
75
+ input,
76
+ temperature: 0.5,
77
+ max_output_tokens: 250,
78
+ stream: true,
79
+ tools: enableTools ? this.tools.map(createToolSchema) : void 0,
80
+ ...this.options.settings
81
+ },
82
+ { signal }
83
+ );
84
+ const toolOutputs = [];
85
+ for await (const event of response) {
86
+ switch (event.type) {
87
+ case "response.output_text.delta":
88
+ this.log(`Answer chunk: "${event.delta}"`);
89
+ stream.write(event.delta);
90
+ break;
91
+ case "response.output_text.done":
92
+ this.addAssistantMessage(event.text);
93
+ stream.end();
94
+ textPromise.resolve(event.text);
95
+ break;
96
+ case "response.output_item.done":
97
+ if (event.item.type === "function_call") {
98
+ toolOutputs.push(event.item);
99
+ const toolOutput = await this.executeTool(event.item);
100
+ toolOutputs.push(toolOutput);
101
+ }
102
+ break;
103
+ default:
104
+ break;
105
+ }
106
+ }
107
+ if (toolOutputs.length > 0) {
108
+ await this.generateAnswer(
109
+ stream,
110
+ textPromise,
111
+ callCount + 1,
112
+ toolOutputs
113
+ );
114
+ }
115
+ } catch (error) {
116
+ textPromise.reject(error);
117
+ if (!(error instanceof OpenAI.APIUserAbortError)) {
118
+ console.error("[OpenaiAgent] Error answering:", error);
119
+ }
120
+ }
121
+ }
122
+ addTool(tool) {
123
+ this.tools.push(tool);
124
+ }
125
+ removeTool(name) {
126
+ this.tools = this.tools.filter((tool) => tool.name !== name);
127
+ }
128
+ getDefaultTools() {
129
+ const tools = [];
130
+ if (this.options.autoEndCall) {
131
+ tools.push({
132
+ name: AUTO_END_CALL_TOOL_NAME,
133
+ description: typeof this.options.autoEndCall === "string" ? this.options.autoEndCall : AUTO_END_CALL_PROMPT,
134
+ callback: () => this.endCall()
135
+ });
136
+ }
137
+ if (this.options.autoSemanticTurn) {
138
+ tools.push({
139
+ name: AUTO_SEMANTIC_TURN_TOOL_NAME,
140
+ description: typeof this.options.autoSemanticTurn === "string" ? this.options.autoSemanticTurn : AUTO_SEMANTIC_TURN_PROMPT,
141
+ callback: () => this.skipAnswer()
142
+ });
143
+ }
144
+ if (this.options.autoIgnoreUserNoise) {
145
+ tools.push({
146
+ name: AUTO_IGNORE_USER_NOISE_TOOL_NAME,
147
+ description: typeof this.options.autoIgnoreUserNoise === "string" ? this.options.autoIgnoreUserNoise : AUTO_IGNORE_USER_NOISE_PROMPT,
148
+ callback: () => this.cancelLastUserMessage()
149
+ });
150
+ }
151
+ return tools;
152
+ }
153
+ async executeTool(toolCall) {
154
+ try {
155
+ const tool = this.tools.find((tool2) => tool2.name === toolCall.name);
156
+ if (!tool) {
157
+ throw new Error(`Tool not found "${toolCall.name}"`);
158
+ }
159
+ this.log("Executing tool:", toolCall.name, toolCall.arguments);
160
+ const args = JSON.parse(toolCall.arguments);
161
+ const result = await tool.callback(args);
162
+ return createToolCallOutput(toolCall, result || {});
163
+ } catch (error) {
164
+ console.error("[OpenaiAgent] Error executing tool:", error);
165
+ return createToolCallOutput(toolCall, { error: error.message });
166
+ }
167
+ }
168
+ cancel() {
169
+ if (!this.abortController) return;
170
+ this.log("Cancel");
171
+ this.abortController.abort();
172
+ this.abortController = void 0;
173
+ }
174
+ };
175
+
176
+ // src/OpenaiSTT.ts
177
+ import { FileSTT } from "@micdrop/server";
178
+ var DEFAULT_MODEL2 = "whisper-1";
179
+ var OpenaiSTT = class extends FileSTT {
180
+ constructor(options) {
181
+ super();
182
+ this.options = options;
183
+ }
184
+ async transcribeFile(file) {
185
+ const formData = new FormData();
186
+ formData.append("file", file);
187
+ formData.append("model", this.options.model || DEFAULT_MODEL2);
188
+ formData.append("response_format", "text");
189
+ if (this.options.language) {
190
+ formData.append("language", this.options.language);
191
+ }
192
+ const response = await fetch(
193
+ "https://api.openai.com/v1/audio/transcriptions",
194
+ {
195
+ method: "POST",
196
+ headers: {
197
+ Authorization: `Bearer ${this.options.apiKey}`
198
+ },
199
+ body: formData
200
+ }
201
+ );
202
+ if (!response.ok) {
203
+ this.log("Failed to transcribe audio", await response.json());
204
+ throw new Error(`Failed to transcribe audio: ${response.statusText}`);
205
+ }
206
+ const transcript = await response.text();
207
+ this.log(`Received transcript: "${transcript}"`);
208
+ return transcript;
209
+ }
210
+ };
211
+ export {
212
+ OpenaiAgent,
213
+ OpenaiSTT
214
+ };
215
+ //# sourceMappingURL=index.mjs.map
@@ -0,0 +1 @@
1
+ {"version":3,"sources":["../src/OpenaiAgent.ts","../src/tools.ts","../src/OpenaiSTT.ts"],"sourcesContent":["import { Agent, AgentOptions, TextPromise } from '@micdrop/server'\nimport OpenAI from 'openai'\nimport { PassThrough } from 'stream'\nimport { z } from 'zod'\nimport {\n createToolCallOutput,\n createToolSchema,\n Tool,\n ToolCall,\n ToolCallOutput,\n} from './tools'\n\nexport interface OpenaiAgentOptions extends AgentOptions {\n apiKey: string\n model?: string\n maxCalls?: number\n settings?: Omit<\n OpenAI.Responses.ResponseCreateParamsStreaming,\n 'input' | 'model'\n >\n\n // Enable auto ending of the call when user asks to end the call\n // You can provide a custom prompt to use instead of the default one by passing a string\n autoEndCall?: boolean | string\n\n // Enable detection of an incomplete sentence, and skip the answer (assistant waits)\n // You can provide a custom prompt to use instead of the default one by passing a string\n autoSemanticTurn?: boolean | string\n\n // Ignore of the last user message when it's meaningless\n // You can provide a custom prompt to use instead of the default one by passing a string\n autoIgnoreUserNoise?: boolean | string\n}\n\nconst DEFAULT_MODEL = 'gpt-4o'\nconst DEFAULT_MAX_CALLS = 5\n\nconst AUTO_END_CALL_TOOL_NAME = 'end_call'\nconst AUTO_END_CALL_PROMPT = 'User asks to end the call'\n\nconst AUTO_SEMANTIC_TURN_TOOL_NAME = 'semantic_turn'\nconst AUTO_SEMANTIC_TURN_PROMPT = 'Last user message is an incomplete sentence'\n\nconst AUTO_IGNORE_USER_NOISE_TOOL_NAME = 'ignore_user_noise'\nconst AUTO_IGNORE_USER_NOISE_PROMPT =\n 'Last user message is just an interjection or a sound that expresses emotion, hesitation, or reaction (ex: \"Uh\", \"Ahem\", \"Hmm\", \"Ah\") but doesn\\'t carry any clear meaning like agreeing, refusing, or commanding'\n\nexport class OpenaiAgent extends Agent<OpenaiAgentOptions> {\n private openai: OpenAI\n private tools: Tool[]\n private abortController?: AbortController\n\n constructor(options: OpenaiAgentOptions) {\n super(options)\n this.openai = new OpenAI({ apiKey: options.apiKey })\n this.tools = this.getDefaultTools()\n }\n\n answer() {\n this.cancel()\n this.log('Start answering')\n const stream = new PassThrough()\n const textPromise = this.createTextPromise()\n\n this.generateAnswer(stream, textPromise).finally(() => {\n this.abortController = undefined\n if (stream.writable) {\n stream.end()\n }\n })\n\n return { message: textPromise.promise, stream }\n }\n\n private async generateAnswer(\n stream: PassThrough,\n textPromise: TextPromise,\n callCount = 0,\n toolInputs?: OpenAI.Responses.ResponseInputItem[]\n ): Promise<void> {\n if (callCount >= (this.options.maxCalls || DEFAULT_MAX_CALLS)) {\n console.error('[OpenaiAgent] Max calls reached')\n textPromise.reject(new Error('Max calls reached'))\n return\n }\n\n this.abortController = new AbortController()\n const signal = this.abortController.signal\n\n // Disable tools if first message\n const enableTools = this.conversation.length > 1\n\n // Build input\n const input: OpenAI.Responses.ResponseInput = this.conversation.map(\n (message) => ({\n role: message.role,\n content: message.content,\n })\n )\n if (toolInputs) {\n input.push(...toolInputs)\n }\n\n try {\n // Generate answer\n const response = await this.openai.responses.create(\n {\n model: this.options.model || DEFAULT_MODEL,\n input,\n temperature: 0.5,\n max_output_tokens: 250,\n stream: true,\n tools: enableTools ? this.tools.map(createToolSchema) : undefined,\n ...this.options.settings,\n },\n { signal }\n )\n const toolOutputs: OpenAI.Responses.ResponseInputItem[] = []\n\n // Handle response events\n for await (const event of response) {\n switch (event.type) {\n case 'response.output_text.delta':\n this.log(`Answer chunk: \"${event.delta}\"`)\n stream.write(event.delta)\n break\n\n case 'response.output_text.done':\n this.addAssistantMessage(event.text)\n stream.end()\n textPromise.resolve(event.text)\n break\n\n case 'response.output_item.done':\n if (event.item.type === 'function_call') {\n toolOutputs.push(event.item)\n const toolOutput = await this.executeTool(event.item)\n toolOutputs.push(toolOutput)\n }\n break\n\n default:\n break\n }\n }\n\n // Query again in case of tool call\n if (toolOutputs.length > 0) {\n await this.generateAnswer(\n stream,\n textPromise,\n callCount + 1,\n toolOutputs\n )\n }\n } catch (error) {\n textPromise.reject(error)\n if (!(error instanceof OpenAI.APIUserAbortError)) {\n console.error('[OpenaiAgent] Error answering:', error)\n }\n }\n }\n\n addTool<Schema extends z.ZodObject>(tool: Tool<Schema>) {\n this.tools.push(tool)\n }\n\n removeTool(name: string) {\n this.tools = this.tools.filter((tool) => tool.name !== name)\n }\n\n private getDefaultTools() {\n const tools: Tool[] = []\n if (this.options.autoEndCall) {\n tools.push({\n name: AUTO_END_CALL_TOOL_NAME,\n description:\n typeof this.options.autoEndCall === 'string'\n ? this.options.autoEndCall\n : AUTO_END_CALL_PROMPT,\n callback: () => this.endCall(),\n })\n }\n if (this.options.autoSemanticTurn) {\n tools.push({\n name: AUTO_SEMANTIC_TURN_TOOL_NAME,\n description:\n typeof this.options.autoSemanticTurn === 'string'\n ? this.options.autoSemanticTurn\n : AUTO_SEMANTIC_TURN_PROMPT,\n callback: () => this.skipAnswer(),\n })\n }\n if (this.options.autoIgnoreUserNoise) {\n tools.push({\n name: AUTO_IGNORE_USER_NOISE_TOOL_NAME,\n description:\n typeof this.options.autoIgnoreUserNoise === 'string'\n ? this.options.autoIgnoreUserNoise\n : AUTO_IGNORE_USER_NOISE_PROMPT,\n callback: () => this.cancelLastUserMessage(),\n })\n }\n return tools\n }\n\n private async executeTool(toolCall: ToolCall): Promise<ToolCallOutput> {\n try {\n const tool = this.tools.find((tool) => tool.name === toolCall.name)\n if (!tool) {\n throw new Error(`Tool not found \"${toolCall.name}\"`)\n }\n\n this.log('Executing tool:', toolCall.name, toolCall.arguments)\n const args = JSON.parse(toolCall.arguments)\n const result = await tool.callback(args)\n return createToolCallOutput(toolCall, result || {})\n } catch (error: any) {\n console.error('[OpenaiAgent] Error executing tool:', error)\n return createToolCallOutput(toolCall, { error: error.message })\n }\n }\n\n cancel() {\n if (!this.abortController) return\n this.log('Cancel')\n this.abortController.abort()\n this.abortController = undefined\n }\n}\n","import {\n FunctionTool,\n ResponseInputItem,\n Responses,\n} from 'openai/resources/responses/responses'\nimport { toJSONSchema, z } from 'zod'\n\nexport interface Tool<Schema extends z.ZodObject = z.ZodObject> {\n name: string\n description: string\n parameters?: Schema\n callback: (input: z.infer<Schema>) => any | Promise<any>\n}\n\nexport type ToolCall = Responses.ResponseFunctionToolCall\nexport type ToolCallOutput = ResponseInputItem.FunctionCallOutput\n\nexport function createToolSchema<Schema extends z.ZodObject>(\n tool: Tool<Schema>\n): FunctionTool {\n return {\n type: 'function',\n name: tool.name,\n description: tool.description,\n strict: true,\n parameters: toJSONSchema(tool.parameters || z.object()),\n }\n}\n\nexport function createToolCallOutput(\n toolCall: ToolCall,\n args: any\n): ToolCallOutput {\n return {\n type: 'function_call_output',\n call_id: toolCall.call_id,\n output: JSON.stringify(args),\n }\n}\n","import { FileSTT } from '@micdrop/server'\n\n/**\n * OpenAI Whisper STT\n *\n * @see https://platform.openai.com/docs/guides/speech-to-text?lang=curl\n */\n\nexport interface OpenaiSTTOptions {\n apiKey: string\n model?: string\n language?: string\n}\n\nconst DEFAULT_MODEL = 'whisper-1'\n\nexport class OpenaiSTT extends FileSTT {\n constructor(private options: OpenaiSTTOptions) {\n super()\n }\n\n async transcribeFile(file: File) {\n // Prepare form data\n const formData = new FormData()\n formData.append('file', file)\n formData.append('model', this.options.model || DEFAULT_MODEL)\n formData.append('response_format', 'text')\n\n if (this.options.language) {\n formData.append('language', this.options.language)\n }\n\n const response = await fetch(\n 'https://api.openai.com/v1/audio/transcriptions',\n {\n method: 'POST',\n headers: {\n Authorization: `Bearer ${this.options.apiKey}`,\n },\n body: formData,\n }\n )\n\n if (!response.ok) {\n this.log('Failed to transcribe audio', await response.json())\n throw new Error(`Failed to transcribe audio: ${response.statusText}`)\n }\n\n const transcript = await response.text()\n this.log(`Received transcript: \"${transcript}\"`)\n\n return transcript\n }\n}\n"],"mappings":";AAAA,SAAS,aAAwC;AACjD,OAAO,YAAY;AACnB,SAAS,mBAAmB;;;ACG5B,SAAS,cAAc,SAAS;AAYzB,SAAS,iBACd,MACc;AACd,SAAO;AAAA,IACL,MAAM;AAAA,IACN,MAAM,KAAK;AAAA,IACX,aAAa,KAAK;AAAA,IAClB,QAAQ;AAAA,IACR,YAAY,aAAa,KAAK,cAAc,EAAE,OAAO,CAAC;AAAA,EACxD;AACF;AAEO,SAAS,qBACd,UACA,MACgB;AAChB,SAAO;AAAA,IACL,MAAM;AAAA,IACN,SAAS,SAAS;AAAA,IAClB,QAAQ,KAAK,UAAU,IAAI;AAAA,EAC7B;AACF;;;ADJA,IAAM,gBAAgB;AACtB,IAAM,oBAAoB;AAE1B,IAAM,0BAA0B;AAChC,IAAM,uBAAuB;AAE7B,IAAM,+BAA+B;AACrC,IAAM,4BAA4B;AAElC,IAAM,mCAAmC;AACzC,IAAM,gCACJ;AAEK,IAAM,cAAN,cAA0B,MAA0B;AAAA,EAKzD,YAAY,SAA6B;AACvC,UAAM,OAAO;AACb,SAAK,SAAS,IAAI,OAAO,EAAE,QAAQ,QAAQ,OAAO,CAAC;AACnD,SAAK,QAAQ,KAAK,gBAAgB;AAAA,EACpC;AAAA,EAEA,SAAS;AACP,SAAK,OAAO;AACZ,SAAK,IAAI,iBAAiB;AAC1B,UAAM,SAAS,IAAI,YAAY;AAC/B,UAAM,cAAc,KAAK,kBAAkB;AAE3C,SAAK,eAAe,QAAQ,WAAW,EAAE,QAAQ,MAAM;AACrD,WAAK,kBAAkB;AACvB,UAAI,OAAO,UAAU;AACnB,eAAO,IAAI;AAAA,MACb;AAAA,IACF,CAAC;AAED,WAAO,EAAE,SAAS,YAAY,SAAS,OAAO;AAAA,EAChD;AAAA,EAEA,MAAc,eACZ,QACA,aACA,YAAY,GACZ,YACe;AACf,QAAI,cAAc,KAAK,QAAQ,YAAY,oBAAoB;AAC7D,cAAQ,MAAM,iCAAiC;AAC/C,kBAAY,OAAO,IAAI,MAAM,mBAAmB,CAAC;AACjD;AAAA,IACF;AAEA,SAAK,kBAAkB,IAAI,gBAAgB;AAC3C,UAAM,SAAS,KAAK,gBAAgB;AAGpC,UAAM,cAAc,KAAK,aAAa,SAAS;AAG/C,UAAM,QAAwC,KAAK,aAAa;AAAA,MAC9D,CAAC,aAAa;AAAA,QACZ,MAAM,QAAQ;AAAA,QACd,SAAS,QAAQ;AAAA,MACnB;AAAA,IACF;AACA,QAAI,YAAY;AACd,YAAM,KAAK,GAAG,UAAU;AAAA,IAC1B;AAEA,QAAI;AAEF,YAAM,WAAW,MAAM,KAAK,OAAO,UAAU;AAAA,QAC3C;AAAA,UACE,OAAO,KAAK,QAAQ,SAAS;AAAA,UAC7B;AAAA,UACA,aAAa;AAAA,UACb,mBAAmB;AAAA,UACnB,QAAQ;AAAA,UACR,OAAO,cAAc,KAAK,MAAM,IAAI,gBAAgB,IAAI;AAAA,UACxD,GAAG,KAAK,QAAQ;AAAA,QAClB;AAAA,QACA,EAAE,OAAO;AAAA,MACX;AACA,YAAM,cAAoD,CAAC;AAG3D,uBAAiB,SAAS,UAAU;AAClC,gBAAQ,MAAM,MAAM;AAAA,UAClB,KAAK;AACH,iBAAK,IAAI,kBAAkB,MAAM,KAAK,GAAG;AACzC,mBAAO,MAAM,MAAM,KAAK;AACxB;AAAA,UAEF,KAAK;AACH,iBAAK,oBAAoB,MAAM,IAAI;AACnC,mBAAO,IAAI;AACX,wBAAY,QAAQ,MAAM,IAAI;AAC9B;AAAA,UAEF,KAAK;AACH,gBAAI,MAAM,KAAK,SAAS,iBAAiB;AACvC,0BAAY,KAAK,MAAM,IAAI;AAC3B,oBAAM,aAAa,MAAM,KAAK,YAAY,MAAM,IAAI;AACpD,0BAAY,KAAK,UAAU;AAAA,YAC7B;AACA;AAAA,UAEF;AACE;AAAA,QACJ;AAAA,MACF;AAGA,UAAI,YAAY,SAAS,GAAG;AAC1B,cAAM,KAAK;AAAA,UACT;AAAA,UACA;AAAA,UACA,YAAY;AAAA,UACZ;AAAA,QACF;AAAA,MACF;AAAA,IACF,SAAS,OAAO;AACd,kBAAY,OAAO,KAAK;AACxB,UAAI,EAAE,iBAAiB,OAAO,oBAAoB;AAChD,gBAAQ,MAAM,kCAAkC,KAAK;AAAA,MACvD;AAAA,IACF;AAAA,EACF;AAAA,EAEA,QAAoC,MAAoB;AACtD,SAAK,MAAM,KAAK,IAAI;AAAA,EACtB;AAAA,EAEA,WAAW,MAAc;AACvB,SAAK,QAAQ,KAAK,MAAM,OAAO,CAAC,SAAS,KAAK,SAAS,IAAI;AAAA,EAC7D;AAAA,EAEQ,kBAAkB;AACxB,UAAM,QAAgB,CAAC;AACvB,QAAI,KAAK,QAAQ,aAAa;AAC5B,YAAM,KAAK;AAAA,QACT,MAAM;AAAA,QACN,aACE,OAAO,KAAK,QAAQ,gBAAgB,WAChC,KAAK,QAAQ,cACb;AAAA,QACN,UAAU,MAAM,KAAK,QAAQ;AAAA,MAC/B,CAAC;AAAA,IACH;AACA,QAAI,KAAK,QAAQ,kBAAkB;AACjC,YAAM,KAAK;AAAA,QACT,MAAM;AAAA,QACN,aACE,OAAO,KAAK,QAAQ,qBAAqB,WACrC,KAAK,QAAQ,mBACb;AAAA,QACN,UAAU,MAAM,KAAK,WAAW;AAAA,MAClC,CAAC;AAAA,IACH;AACA,QAAI,KAAK,QAAQ,qBAAqB;AACpC,YAAM,KAAK;AAAA,QACT,MAAM;AAAA,QACN,aACE,OAAO,KAAK,QAAQ,wBAAwB,WACxC,KAAK,QAAQ,sBACb;AAAA,QACN,UAAU,MAAM,KAAK,sBAAsB;AAAA,MAC7C,CAAC;AAAA,IACH;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAc,YAAY,UAA6C;AACrE,QAAI;AACF,YAAM,OAAO,KAAK,MAAM,KAAK,CAACA,UAASA,MAAK,SAAS,SAAS,IAAI;AAClE,UAAI,CAAC,MAAM;AACT,cAAM,IAAI,MAAM,mBAAmB,SAAS,IAAI,GAAG;AAAA,MACrD;AAEA,WAAK,IAAI,mBAAmB,SAAS,MAAM,SAAS,SAAS;AAC7D,YAAM,OAAO,KAAK,MAAM,SAAS,SAAS;AAC1C,YAAM,SAAS,MAAM,KAAK,SAAS,IAAI;AACvC,aAAO,qBAAqB,UAAU,UAAU,CAAC,CAAC;AAAA,IACpD,SAAS,OAAY;AACnB,cAAQ,MAAM,uCAAuC,KAAK;AAC1D,aAAO,qBAAqB,UAAU,EAAE,OAAO,MAAM,QAAQ,CAAC;AAAA,IAChE;AAAA,EACF;AAAA,EAEA,SAAS;AACP,QAAI,CAAC,KAAK,gBAAiB;AAC3B,SAAK,IAAI,QAAQ;AACjB,SAAK,gBAAgB,MAAM;AAC3B,SAAK,kBAAkB;AAAA,EACzB;AACF;;;AErOA,SAAS,eAAe;AAcxB,IAAMC,iBAAgB;AAEf,IAAM,YAAN,cAAwB,QAAQ;AAAA,EACrC,YAAoB,SAA2B;AAC7C,UAAM;AADY;AAAA,EAEpB;AAAA,EAEA,MAAM,eAAe,MAAY;AAE/B,UAAM,WAAW,IAAI,SAAS;AAC9B,aAAS,OAAO,QAAQ,IAAI;AAC5B,aAAS,OAAO,SAAS,KAAK,QAAQ,SAASA,cAAa;AAC5D,aAAS,OAAO,mBAAmB,MAAM;AAEzC,QAAI,KAAK,QAAQ,UAAU;AACzB,eAAS,OAAO,YAAY,KAAK,QAAQ,QAAQ;AAAA,IACnD;AAEA,UAAM,WAAW,MAAM;AAAA,MACrB;AAAA,MACA;AAAA,QACE,QAAQ;AAAA,QACR,SAAS;AAAA,UACP,eAAe,UAAU,KAAK,QAAQ,MAAM;AAAA,QAC9C;AAAA,QACA,MAAM;AAAA,MACR;AAAA,IACF;AAEA,QAAI,CAAC,SAAS,IAAI;AAChB,WAAK,IAAI,8BAA8B,MAAM,SAAS,KAAK,CAAC;AAC5D,YAAM,IAAI,MAAM,+BAA+B,SAAS,UAAU,EAAE;AAAA,IACtE;AAEA,UAAM,aAAa,MAAM,SAAS,KAAK;AACvC,SAAK,IAAI,yBAAyB,UAAU,GAAG;AAE/C,WAAO;AAAA,EACT;AACF;","names":["tool","DEFAULT_MODEL"]}
package/package.json ADDED
@@ -0,0 +1,48 @@
1
+ {
2
+ "name": "@micdrop/openai",
3
+ "version": "1.0.0",
4
+ "description": "OpenAI implementation for @micdrop/server",
5
+ "author": "Lonestone",
6
+ "license": "MIT",
7
+ "main": "./dist/index.js",
8
+ "module": "./dist/index.mjs",
9
+ "types": "./dist/index.d.ts",
10
+ "scripts": {
11
+ "build": "tsup",
12
+ "dev": "tsup --watch",
13
+ "clean": "rm -rf dist",
14
+ "typecheck": "tsc --noEmit",
15
+ "prepublishOnly": "npm run build"
16
+ },
17
+ "files": [
18
+ "dist"
19
+ ],
20
+ "repository": {
21
+ "type": "git",
22
+ "url": "git+https://github.com/lonestone/micdrop.git",
23
+ "directory": "packages/openai"
24
+ },
25
+ "publishConfig": {
26
+ "access": "public"
27
+ },
28
+ "keywords": [
29
+ "microphone",
30
+ "audio",
31
+ "browser",
32
+ "voice",
33
+ "conversation",
34
+ "openai"
35
+ ],
36
+ "dependencies": {
37
+ "openai": "^5.8.2",
38
+ "zod": "^4.0.5"
39
+ },
40
+ "peerDependencies": {
41
+ "@micdrop/server": "*"
42
+ },
43
+ "devDependencies": {
44
+ "@types/node": "^22.13.4",
45
+ "tsup": "^8.3.6",
46
+ "typescript": "^5.7.3"
47
+ }
48
+ }