@elizaos/plugin-openai 1.6.0 → 2.0.0-alpha.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. package/LICENSE +1 -1
  2. package/dist/browser/index.browser.js +2 -2
  3. package/dist/browser/index.browser.js.map +18 -17
  4. package/dist/build.d.ts +13 -0
  5. package/dist/build.d.ts.map +1 -0
  6. package/dist/cjs/index.node.cjs +998 -658
  7. package/dist/cjs/index.node.js.map +18 -17
  8. package/dist/generated/specs/specs.d.ts +55 -0
  9. package/dist/generated/specs/specs.d.ts.map +1 -0
  10. package/dist/index.browser.d.ts +1 -0
  11. package/dist/index.browser.d.ts.map +1 -0
  12. package/dist/index.d.ts +1 -5
  13. package/dist/index.d.ts.map +1 -0
  14. package/dist/index.node.d.ts +1 -0
  15. package/dist/index.node.d.ts.map +1 -0
  16. package/dist/init.d.ts +4 -5
  17. package/dist/init.d.ts.map +1 -0
  18. package/dist/models/audio.d.ts +9 -10
  19. package/dist/models/audio.d.ts.map +1 -0
  20. package/dist/models/embedding.d.ts +1 -3
  21. package/dist/models/embedding.d.ts.map +1 -0
  22. package/dist/models/image.d.ts +4 -13
  23. package/dist/models/image.d.ts.map +1 -0
  24. package/dist/models/index.d.ts +7 -5
  25. package/dist/models/index.d.ts.map +1 -0
  26. package/dist/models/object.d.ts +4 -9
  27. package/dist/models/object.d.ts.map +1 -0
  28. package/dist/models/research.d.ts +34 -0
  29. package/dist/models/research.d.ts.map +1 -0
  30. package/dist/models/text.d.ts +22 -3
  31. package/dist/models/text.d.ts.map +1 -0
  32. package/dist/models/tokenizer.d.ts +4 -9
  33. package/dist/models/tokenizer.d.ts.map +1 -0
  34. package/dist/node/index.node.js +987 -644
  35. package/dist/node/index.node.js.map +18 -17
  36. package/dist/providers/index.d.ts +2 -1
  37. package/dist/providers/index.d.ts.map +1 -0
  38. package/dist/providers/openai.d.ts +3 -7
  39. package/dist/providers/openai.d.ts.map +1 -0
  40. package/dist/types/index.d.ts +313 -10
  41. package/dist/types/index.d.ts.map +1 -0
  42. package/dist/utils/audio.d.ts +6 -12
  43. package/dist/utils/audio.d.ts.map +1 -0
  44. package/dist/utils/config.d.ts +16 -59
  45. package/dist/utils/config.d.ts.map +1 -0
  46. package/dist/utils/events.d.ts +14 -9
  47. package/dist/utils/events.d.ts.map +1 -0
  48. package/dist/utils/index.d.ts +2 -1
  49. package/dist/utils/index.d.ts.map +1 -0
  50. package/dist/utils/json.d.ts +9 -6
  51. package/dist/utils/json.d.ts.map +1 -0
  52. package/dist/utils/tokenization.d.ts +5 -16
  53. package/dist/utils/tokenization.d.ts.map +1 -0
  54. package/package.json +24 -28
  55. package/README.md +0 -160
@@ -1,373 +1,162 @@
1
- // src/index.ts
2
- import { logger as logger10, ModelType as ModelType7 } from "@elizaos/core";
1
+ // index.ts
2
+ import { logger as logger11, ModelType as ModelType7 } from "@elizaos/core";
3
3
 
4
- // src/init.ts
4
+ // init.ts
5
5
  import { logger as logger2 } from "@elizaos/core";
6
6
 
7
- // src/utils/config.ts
7
+ // utils/config.ts
8
8
  import { logger } from "@elizaos/core";
9
+ function getEnvValue(key) {
10
+ if (typeof process === "undefined" || !process.env) {
11
+ return;
12
+ }
13
+ const value = process.env[key];
14
+ return value === undefined ? undefined : String(value);
15
+ }
9
16
  function getSetting(runtime, key, defaultValue) {
10
17
  const value = runtime.getSetting(key);
11
18
  if (value !== undefined && value !== null) {
12
19
  return String(value);
13
20
  }
14
- return process.env[key] ?? defaultValue;
21
+ return getEnvValue(key) ?? defaultValue;
22
+ }
23
+ function getNumericSetting(runtime, key, defaultValue) {
24
+ const value = getSetting(runtime, key);
25
+ if (value === undefined) {
26
+ return defaultValue;
27
+ }
28
+ const parsed = Number.parseInt(value, 10);
29
+ if (!Number.isFinite(parsed)) {
30
+ throw new Error(`Setting '${key}' must be a valid integer, got: ${value}`);
31
+ }
32
+ return parsed;
33
+ }
34
+ function getBooleanSetting(runtime, key, defaultValue) {
35
+ const value = getSetting(runtime, key);
36
+ if (value === undefined) {
37
+ return defaultValue;
38
+ }
39
+ const normalized = value.toLowerCase();
40
+ return normalized === "true" || normalized === "1" || normalized === "yes";
15
41
  }
16
42
  function isBrowser() {
17
- return typeof globalThis !== "undefined" && "document" in globalThis && typeof globalThis.document !== "undefined";
43
+ return typeof globalThis !== "undefined" && typeof globalThis.document !== "undefined";
18
44
  }
19
45
  function isProxyMode(runtime) {
20
46
  return isBrowser() && !!getSetting(runtime, "OPENAI_BROWSER_BASE_URL");
21
47
  }
48
+ function getApiKey(runtime) {
49
+ return getSetting(runtime, "OPENAI_API_KEY");
50
+ }
51
+ function getEmbeddingApiKey(runtime) {
52
+ const embeddingApiKey = getSetting(runtime, "OPENAI_EMBEDDING_API_KEY");
53
+ if (embeddingApiKey) {
54
+ logger.debug("[OpenAI] Using specific embedding API key");
55
+ return embeddingApiKey;
56
+ }
57
+ logger.debug("[OpenAI] Falling back to general API key for embeddings");
58
+ return getApiKey(runtime);
59
+ }
22
60
  function getAuthHeader(runtime, forEmbedding = false) {
23
- if (isBrowser())
61
+ if (isBrowser() && !getBooleanSetting(runtime, "OPENAI_ALLOW_BROWSER_API_KEY", false)) {
24
62
  return {};
63
+ }
25
64
  const key = forEmbedding ? getEmbeddingApiKey(runtime) : getApiKey(runtime);
26
65
  return key ? { Authorization: `Bearer ${key}` } : {};
27
66
  }
28
67
  function getBaseURL(runtime) {
29
68
  const browserURL = getSetting(runtime, "OPENAI_BROWSER_BASE_URL");
30
- const baseURL = isBrowser() && browserURL ? browserURL : getSetting(runtime, "OPENAI_BASE_URL", "https://api.openai.com/v1");
31
- logger.debug(`[OpenAI] Default base URL: ${baseURL}`);
69
+ const baseURL = isBrowser() && browserURL ? browserURL : getSetting(runtime, "OPENAI_BASE_URL") ?? "https://api.openai.com/v1";
70
+ logger.debug(`[OpenAI] Base URL: ${baseURL}`);
32
71
  return baseURL;
33
72
  }
34
73
  function getEmbeddingBaseURL(runtime) {
35
- const embeddingURL = isBrowser() ? getSetting(runtime, "OPENAI_BROWSER_EMBEDDING_URL") || getSetting(runtime, "OPENAI_BROWSER_BASE_URL") : getSetting(runtime, "OPENAI_EMBEDDING_URL");
74
+ const embeddingURL = isBrowser() ? getSetting(runtime, "OPENAI_BROWSER_EMBEDDING_URL") ?? getSetting(runtime, "OPENAI_BROWSER_BASE_URL") : getSetting(runtime, "OPENAI_EMBEDDING_URL");
36
75
  if (embeddingURL) {
37
- logger.debug(`[OpenAI] Using specific embedding base URL: ${embeddingURL}`);
76
+ logger.debug(`[OpenAI] Using embedding base URL: ${embeddingURL}`);
38
77
  return embeddingURL;
39
78
  }
40
- logger.debug("[OpenAI] Falling back to general base URL for embeddings.");
79
+ logger.debug("[OpenAI] Falling back to general base URL for embeddings");
41
80
  return getBaseURL(runtime);
42
81
  }
43
- function getApiKey(runtime) {
44
- return getSetting(runtime, "OPENAI_API_KEY");
45
- }
46
- function getEmbeddingApiKey(runtime) {
47
- const embeddingApiKey = getSetting(runtime, "OPENAI_EMBEDDING_API_KEY");
48
- if (embeddingApiKey) {
49
- logger.debug("[OpenAI] Using specific embedding API key (present)");
50
- return embeddingApiKey;
51
- }
52
- logger.debug("[OpenAI] Falling back to general API key for embeddings.");
53
- return getApiKey(runtime);
54
- }
55
82
  function getSmallModel(runtime) {
56
- return getSetting(runtime, "OPENAI_SMALL_MODEL") ?? getSetting(runtime, "SMALL_MODEL", "gpt-4o-mini");
83
+ return getSetting(runtime, "OPENAI_SMALL_MODEL") ?? getSetting(runtime, "SMALL_MODEL") ?? "gpt-5-mini";
57
84
  }
58
85
  function getLargeModel(runtime) {
59
- return getSetting(runtime, "OPENAI_LARGE_MODEL") ?? getSetting(runtime, "LARGE_MODEL", "gpt-4o");
86
+ return getSetting(runtime, "OPENAI_LARGE_MODEL") ?? getSetting(runtime, "LARGE_MODEL") ?? "gpt-5";
87
+ }
88
+ function getEmbeddingModel(runtime) {
89
+ return getSetting(runtime, "OPENAI_EMBEDDING_MODEL") ?? "text-embedding-3-small";
60
90
  }
61
91
  function getImageDescriptionModel(runtime) {
62
- return getSetting(runtime, "OPENAI_IMAGE_DESCRIPTION_MODEL", "gpt-5-nano");
92
+ return getSetting(runtime, "OPENAI_IMAGE_DESCRIPTION_MODEL") ?? "gpt-5-mini";
63
93
  }
64
- function getExperimentalTelemetry(runtime) {
65
- const setting = getSetting(runtime, "OPENAI_EXPERIMENTAL_TELEMETRY", "false");
66
- const normalizedSetting = String(setting).toLowerCase();
67
- const result = normalizedSetting === "true";
68
- logger.debug(`[OpenAI] Experimental telemetry in function: "${setting}" (type: ${typeof setting}, normalized: "${normalizedSetting}", result: ${result})`);
69
- return result;
94
+ function getTranscriptionModel(runtime) {
95
+ return getSetting(runtime, "OPENAI_TRANSCRIPTION_MODEL") ?? "gpt-5-mini-transcribe";
70
96
  }
71
-
72
- // src/init.ts
73
- function initializeOpenAI(_config, runtime) {
74
- (async () => {
75
- try {
76
- if (!getApiKey(runtime) && !isBrowser()) {
77
- logger2.warn("OPENAI_API_KEY is not set in environment - OpenAI functionality will be limited");
78
- return;
79
- }
80
- try {
81
- const baseURL = getBaseURL(runtime);
82
- const response = await fetch(`${baseURL}/models`, {
83
- headers: getAuthHeader(runtime)
84
- });
85
- if (!response.ok) {
86
- logger2.warn(`OpenAI API key validation failed: ${response.statusText}`);
87
- logger2.warn("OpenAI functionality will be limited until a valid API key is provided");
88
- } else {
89
- logger2.log("OpenAI API key validated successfully");
90
- }
91
- } catch (fetchError) {
92
- const message = fetchError instanceof Error ? fetchError.message : String(fetchError);
93
- logger2.warn(`Error validating OpenAI API key: ${message}`);
94
- logger2.warn("OpenAI functionality will be limited until a valid API key is provided");
95
- }
96
- } catch (error) {
97
- const message = error?.errors?.map((e) => e.message).join(", ") || (error instanceof Error ? error.message : String(error));
98
- logger2.warn(`OpenAI plugin configuration issue: ${message} - You need to configure the OPENAI_API_KEY in your environment variables`);
99
- }
100
- })();
97
+ function getTTSModel(runtime) {
98
+ return getSetting(runtime, "OPENAI_TTS_MODEL") ?? "tts-1";
101
99
  }
102
-
103
- // src/models/text.ts
104
- import { logger as logger3, ModelType } from "@elizaos/core";
105
- import { generateText, streamText } from "ai";
106
-
107
- // src/providers/openai.ts
108
- import { createOpenAI } from "@ai-sdk/openai";
109
- function createOpenAIClient(runtime) {
110
- const baseURL = getBaseURL(runtime);
111
- const apiKey = getApiKey(runtime) ?? (isProxyMode(runtime) ? "sk-proxy" : undefined);
112
- return createOpenAI({ apiKey: apiKey ?? "", baseURL });
100
+ function getTTSVoice(runtime) {
101
+ return getSetting(runtime, "OPENAI_TTS_VOICE") ?? "nova";
113
102
  }
114
-
115
- // src/utils/events.ts
116
- import { EventType } from "@elizaos/core";
117
- function emitModelUsageEvent(runtime, type, prompt, usage) {
118
- const promptTokens = ("promptTokens" in usage ? usage.promptTokens : undefined) ?? ("inputTokens" in usage ? usage.inputTokens : undefined) ?? 0;
119
- const completionTokens = ("completionTokens" in usage ? usage.completionTokens : undefined) ?? ("outputTokens" in usage ? usage.outputTokens : undefined) ?? 0;
120
- const totalTokens = ("totalTokens" in usage ? usage.totalTokens : undefined) ?? promptTokens + completionTokens;
121
- const truncatedPrompt = typeof prompt === "string" ? prompt.length > 200 ? `${prompt.slice(0, 200)}…` : prompt : "";
122
- runtime.emitEvent(EventType.MODEL_USED, {
123
- runtime,
124
- source: "openai",
125
- provider: "openai",
126
- type,
127
- prompt: truncatedPrompt,
128
- tokens: {
129
- prompt: promptTokens,
130
- completion: completionTokens,
131
- total: totalTokens
132
- }
133
- });
103
+ function getTTSInstructions(runtime) {
104
+ return getSetting(runtime, "OPENAI_TTS_INSTRUCTIONS") ?? "";
134
105
  }
135
-
136
- // src/models/text.ts
137
- async function generateTextByModelType(runtime, params, modelType, getModelFn) {
138
- const openai = createOpenAIClient(runtime);
139
- const modelName = getModelFn(runtime);
140
- logger3.debug(`[OpenAI] ${modelType} model: ${modelName}`);
141
- const generateParams = {
142
- model: openai.languageModel(modelName),
143
- prompt: params.prompt,
144
- system: runtime.character.system ?? undefined,
145
- temperature: params.temperature ?? 0.7,
146
- maxOutputTokens: params.maxTokens ?? 8192,
147
- frequencyPenalty: params.frequencyPenalty ?? 0.7,
148
- presencePenalty: params.presencePenalty ?? 0.7,
149
- stopSequences: params.stopSequences ?? [],
150
- experimental_telemetry: { isEnabled: getExperimentalTelemetry(runtime) }
151
- };
152
- if (params.stream) {
153
- const result = streamText(generateParams);
154
- return {
155
- textStream: result.textStream,
156
- text: result.text,
157
- usage: result.usage.then((u) => u ? {
158
- promptTokens: u.inputTokens ?? 0,
159
- completionTokens: u.outputTokens ?? 0,
160
- totalTokens: (u.inputTokens ?? 0) + (u.outputTokens ?? 0)
161
- } : undefined),
162
- finishReason: result.finishReason
163
- };
164
- }
165
- const { text, usage } = await generateText(generateParams);
166
- if (usage)
167
- emitModelUsageEvent(runtime, modelType, params.prompt, usage);
168
- return text;
106
+ function getImageModel(runtime) {
107
+ return getSetting(runtime, "OPENAI_IMAGE_MODEL") ?? "dall-e-3";
169
108
  }
170
- async function handleTextSmall(runtime, params) {
171
- return generateTextByModelType(runtime, params, ModelType.TEXT_SMALL, getSmallModel);
109
+ function getExperimentalTelemetry(runtime) {
110
+ return getBooleanSetting(runtime, "OPENAI_EXPERIMENTAL_TELEMETRY", false);
172
111
  }
173
- async function handleTextLarge(runtime, params) {
174
- return generateTextByModelType(runtime, params, ModelType.TEXT_LARGE, getLargeModel);
112
+ function getEmbeddingDimensions(runtime) {
113
+ return getNumericSetting(runtime, "OPENAI_EMBEDDING_DIMENSIONS", 1536);
175
114
  }
176
- // src/models/embedding.ts
177
- import { logger as logger4, ModelType as ModelType2, VECTOR_DIMS } from "@elizaos/core";
178
- async function handleTextEmbedding(runtime, params) {
179
- const embeddingModelName = getSetting(runtime, "OPENAI_EMBEDDING_MODEL", "text-embedding-3-small");
180
- const embeddingDimension = Number.parseInt(getSetting(runtime, "OPENAI_EMBEDDING_DIMENSIONS", "1536") || "1536", 10);
181
- if (!Object.values(VECTOR_DIMS).includes(embeddingDimension)) {
182
- const errorMsg = `Invalid embedding dimension: ${embeddingDimension}. Must be one of: ${Object.values(VECTOR_DIMS).join(", ")}`;
183
- logger4.error(errorMsg);
184
- throw new Error(errorMsg);
185
- }
186
- if (params === null) {
187
- logger4.debug("Creating test embedding for initialization");
188
- const testVector = Array(embeddingDimension).fill(0);
189
- testVector[0] = 0.1;
190
- return testVector;
191
- }
192
- let text;
193
- if (typeof params === "string") {
194
- text = params;
195
- } else if (typeof params === "object" && params.text) {
196
- text = params.text;
197
- } else {
198
- const errorMsg = "Invalid input format for embedding";
199
- logger4.warn(errorMsg);
200
- const fallbackVector = Array(embeddingDimension).fill(0);
201
- fallbackVector[0] = 0.2;
202
- return fallbackVector;
203
- }
204
- if (!text.trim()) {
205
- const errorMsg = "Empty text for embedding";
206
- logger4.warn(errorMsg);
207
- const fallbackVector = Array(embeddingDimension).fill(0);
208
- fallbackVector[0] = 0.3;
209
- return fallbackVector;
210
- }
211
- const embeddingBaseURL = getEmbeddingBaseURL(runtime);
212
- try {
213
- const response = await fetch(`${embeddingBaseURL}/embeddings`, {
214
- method: "POST",
215
- headers: {
216
- ...getAuthHeader(runtime, true),
217
- "Content-Type": "application/json"
218
- },
219
- body: JSON.stringify({
220
- model: embeddingModelName,
221
- input: text
222
- })
223
- });
224
- if (!response.ok) {
225
- logger4.error(`OpenAI API error: ${response.status} - ${response.statusText}`);
226
- throw new Error(`OpenAI API error: ${response.status} - ${response.statusText}`);
227
- }
228
- const data = await response.json();
229
- if (!data?.data?.[0]?.embedding) {
230
- logger4.error("API returned invalid structure");
231
- throw new Error("API returned invalid structure");
232
- }
233
- const embedding = data.data[0].embedding;
234
- if (!Array.isArray(embedding) || embedding.length !== embeddingDimension) {
235
- const errorMsg = `Embedding length ${embedding?.length ?? 0} does not match configured dimension ${embeddingDimension}`;
236
- logger4.error(errorMsg);
237
- const fallbackVector = Array(embeddingDimension).fill(0);
238
- fallbackVector[0] = 0.4;
239
- return fallbackVector;
240
- }
241
- if (data.usage) {
242
- const usage = {
243
- inputTokens: data.usage.prompt_tokens,
244
- outputTokens: 0,
245
- totalTokens: data.usage.total_tokens
246
- };
247
- emitModelUsageEvent(runtime, ModelType2.TEXT_EMBEDDING, text, usage);
248
- }
249
- logger4.log(`Got valid embedding with length ${embedding.length}`);
250
- return embedding;
251
- } catch (error) {
252
- const message = error instanceof Error ? error.message : String(error);
253
- logger4.error(`Error generating embedding: ${message}`);
254
- throw error instanceof Error ? error : new Error(message);
255
- }
115
+ function getImageDescriptionMaxTokens(runtime) {
116
+ return getNumericSetting(runtime, "OPENAI_IMAGE_DESCRIPTION_MAX_TOKENS", 8192);
256
117
  }
257
- // src/models/image.ts
258
- import { logger as logger5, ModelType as ModelType3 } from "@elizaos/core";
259
- async function handleImageGeneration(runtime, params) {
260
- const n = params.count || 1;
261
- const size = params.size || "1024x1024";
262
- const prompt = params.prompt;
263
- const modelName = getSetting(runtime, "OPENAI_IMAGE_MODEL", "gpt-image-1");
264
- logger5.log(`[OpenAI] Using IMAGE model: ${modelName}`);
265
- const baseURL = getBaseURL(runtime);
266
- try {
267
- const response = await fetch(`${baseURL}/images/generations`, {
268
- method: "POST",
269
- headers: {
270
- ...getAuthHeader(runtime),
271
- "Content-Type": "application/json"
272
- },
273
- body: JSON.stringify({
274
- model: modelName,
275
- prompt,
276
- n,
277
- size
278
- })
279
- });
280
- if (!response.ok) {
281
- throw new Error(`Failed to generate image: ${response.statusText}`);
282
- }
283
- const data = await response.json();
284
- const typedData = data;
285
- return typedData.data;
286
- } catch (error) {
287
- const message = error instanceof Error ? error.message : String(error);
288
- throw error;
289
- }
118
+ function getResearchModel(runtime) {
119
+ return getSetting(runtime, "OPENAI_RESEARCH_MODEL") ?? "o3-deep-research";
290
120
  }
291
- async function handleImageDescription(runtime, params) {
292
- let imageUrl;
293
- let promptText;
294
- const modelName = getImageDescriptionModel(runtime);
295
- logger5.log(`[OpenAI] Using IMAGE_DESCRIPTION model: ${modelName}`);
296
- const maxTokens = Number.parseInt(getSetting(runtime, "OPENAI_IMAGE_DESCRIPTION_MAX_TOKENS", "8192") || "8192", 10);
297
- const DEFAULT_PROMPT = "Please analyze this image and provide a title and detailed description.";
298
- if (typeof params === "string") {
299
- imageUrl = params;
300
- promptText = DEFAULT_PROMPT;
301
- } else {
302
- imageUrl = params.imageUrl;
303
- promptText = params.prompt || DEFAULT_PROMPT;
121
+ function getResearchTimeout(runtime) {
122
+ return getNumericSetting(runtime, "OPENAI_RESEARCH_TIMEOUT", 3600000);
123
+ }
124
+
125
+ // init.ts
126
+ globalThis.AI_SDK_LOG_WARNINGS ??= false;
127
+ function initializeOpenAI(_config, runtime) {
128
+ validateOpenAIConfiguration(runtime);
129
+ }
130
+ async function validateOpenAIConfiguration(runtime) {
131
+ if (isBrowser()) {
132
+ logger2.debug("[OpenAI] Skipping API validation in browser environment");
133
+ return;
134
+ }
135
+ const apiKey = getApiKey(runtime);
136
+ if (!apiKey) {
137
+ logger2.warn("[OpenAI] OPENAI_API_KEY is not configured. " + "OpenAI functionality will fail until a valid API key is provided.");
138
+ return;
304
139
  }
305
- const messages = [
306
- {
307
- role: "user",
308
- content: [
309
- { type: "text", text: promptText },
310
- { type: "image_url", image_url: { url: imageUrl } }
311
- ]
312
- }
313
- ];
314
- const baseURL = getBaseURL(runtime);
315
140
  try {
316
- const requestBody = {
317
- model: modelName,
318
- messages,
319
- max_tokens: maxTokens
320
- };
321
- const response = await fetch(`${baseURL}/chat/completions`, {
322
- method: "POST",
323
- headers: {
324
- "Content-Type": "application/json",
325
- ...getAuthHeader(runtime)
326
- },
327
- body: JSON.stringify(requestBody)
141
+ const baseURL = getBaseURL(runtime);
142
+ const response = await fetch(`${baseURL}/models`, {
143
+ headers: getAuthHeader(runtime)
328
144
  });
329
145
  if (!response.ok) {
330
- throw new Error(`OpenAI API error: ${response.status}`);
146
+ logger2.warn(`[OpenAI] API key validation failed: ${response.status} ${response.statusText}. ` + "Please verify your OPENAI_API_KEY is correct.");
147
+ return;
331
148
  }
332
- const result = await response.json();
333
- const typedResult = result;
334
- const content = typedResult.choices?.[0]?.message?.content;
335
- if (typedResult.usage) {
336
- emitModelUsageEvent(runtime, ModelType3.IMAGE_DESCRIPTION, typeof params === "string" ? params : params.prompt || "", {
337
- inputTokens: typedResult.usage.prompt_tokens,
338
- outputTokens: typedResult.usage.completion_tokens,
339
- totalTokens: typedResult.usage.total_tokens
340
- });
341
- }
342
- if (!content) {
343
- return {
344
- title: "Failed to analyze image",
345
- description: "No response from API"
346
- };
347
- }
348
- const titleMatch = content.match(/title[:\s]+(.+?)(?:\n|$)/i);
349
- const title = titleMatch?.[1]?.trim();
350
- if (!title) {
351
- logger5.warn("Could not extract title from image description response");
352
- }
353
- const finalTitle = title || "Image Analysis";
354
- const description = content.replace(/title[:\s]+(.+?)(?:\n|$)/i, "").trim();
355
- const processedResult = { title: finalTitle, description };
356
- return processedResult;
357
149
  } catch (error) {
358
150
  const message = error instanceof Error ? error.message : String(error);
359
- logger5.error(`Error analyzing image: ${message}`);
360
- return {
361
- title: "Failed to analyze image",
362
- description: `Error: ${message}`
363
- };
151
+ logger2.warn(`[OpenAI] API validation error: ${message}. OpenAI functionality may be limited.`);
364
152
  }
365
153
  }
366
- // src/models/audio.ts
367
- import { logger as logger7 } from "@elizaos/core";
368
154
 
369
- // src/utils/audio.ts
370
- import { logger as logger6 } from "@elizaos/core";
155
+ // models/audio.ts
156
+ import { logger as logger4 } from "@elizaos/core";
157
+
158
+ // utils/audio.ts
159
+ import { logger as logger3 } from "@elizaos/core";
371
160
  var MAGIC_BYTES = {
372
161
  WAV: {
373
162
  HEADER: [82, 73, 70, 70],
@@ -379,21 +168,26 @@ var MAGIC_BYTES = {
379
168
  FTYP: [102, 116, 121, 112],
380
169
  WEBM_EBML: [26, 69, 223, 163]
381
170
  };
382
- function matchBytes(buffer, offset, bytes) {
383
- for (let i = 0;i < bytes.length; i++) {
384
- if (buffer[offset + i] !== bytes[i])
171
+ var MIN_DETECTION_BUFFER_SIZE = 12;
172
+ function matchBytes(buffer, offset, expected) {
173
+ for (let i = 0;i < expected.length; i++) {
174
+ const expectedByte = expected[i];
175
+ if (expectedByte === undefined || buffer[offset + i] !== expectedByte) {
385
176
  return false;
177
+ }
386
178
  }
387
179
  return true;
388
180
  }
389
181
  function detectAudioMimeType(buffer) {
390
- if (buffer.length < 12) {
182
+ if (buffer.length < MIN_DETECTION_BUFFER_SIZE) {
391
183
  return "application/octet-stream";
392
184
  }
393
185
  if (matchBytes(buffer, 0, MAGIC_BYTES.WAV.HEADER) && matchBytes(buffer, 8, MAGIC_BYTES.WAV.IDENTIFIER)) {
394
186
  return "audio/wav";
395
187
  }
396
- if (matchBytes(buffer, 0, MAGIC_BYTES.MP3_ID3) || buffer[0] === 255 && (buffer[1] & 224) === 224) {
188
+ const firstByte = buffer[0];
189
+ const secondByte = buffer[1];
190
+ if (matchBytes(buffer, 0, MAGIC_BYTES.MP3_ID3) || firstByte === 255 && secondByte !== undefined && (secondByte & 224) === 224) {
397
191
  return "audio/mpeg";
398
192
  }
399
193
  if (matchBytes(buffer, 0, MAGIC_BYTES.OGG)) {
@@ -408,250 +202,847 @@ function detectAudioMimeType(buffer) {
408
202
  if (matchBytes(buffer, 0, MAGIC_BYTES.WEBM_EBML)) {
409
203
  return "audio/webm";
410
204
  }
411
- logger6.warn("Could not detect audio format from buffer, using generic binary type");
205
+ logger3.warn("Could not detect audio format from buffer, using generic binary type");
412
206
  return "application/octet-stream";
413
207
  }
208
+ function getExtensionForMimeType(mimeType) {
209
+ switch (mimeType) {
210
+ case "audio/wav":
211
+ return "wav";
212
+ case "audio/mpeg":
213
+ return "mp3";
214
+ case "audio/ogg":
215
+ return "ogg";
216
+ case "audio/flac":
217
+ return "flac";
218
+ case "audio/mp4":
219
+ return "m4a";
220
+ case "audio/webm":
221
+ return "webm";
222
+ case "application/octet-stream":
223
+ return "bin";
224
+ }
225
+ }
226
+ function getFilenameForMimeType(mimeType) {
227
+ const ext = getExtensionForMimeType(mimeType);
228
+ return `recording.${ext}`;
229
+ }
414
230
 
415
- // src/models/audio.ts
416
- async function fetchTextToSpeech(runtime, options) {
417
- const defaultModel = getSetting(runtime, "OPENAI_TTS_MODEL", "gpt-4o-mini-tts");
418
- const defaultVoice = getSetting(runtime, "OPENAI_TTS_VOICE", "nova");
419
- const defaultInstructions = getSetting(runtime, "OPENAI_TTS_INSTRUCTIONS", "");
420
- const baseURL = getBaseURL(runtime);
421
- const model = options.model || defaultModel;
422
- const voice = options.voice || defaultVoice;
423
- const instructions = options.instructions ?? defaultInstructions;
424
- const format = options.format || "mp3";
425
- try {
426
- const res = await fetch(`${baseURL}/audio/speech`, {
427
- method: "POST",
428
- headers: {
429
- ...getAuthHeader(runtime),
430
- "Content-Type": "application/json",
431
- ...format === "mp3" ? { Accept: "audio/mpeg" } : {}
432
- },
433
- body: JSON.stringify({
434
- model,
435
- voice,
436
- input: options.text,
437
- format,
438
- ...instructions && { instructions }
439
- })
440
- });
441
- if (!res.ok) {
442
- const err = await res.text();
443
- throw new Error(`OpenAI TTS error ${res.status}: ${err}`);
444
- }
445
- return await res.arrayBuffer();
446
- } catch (err) {
447
- const message = err instanceof Error ? err.message : String(err);
448
- throw new Error(`Failed to fetch speech from OpenAI TTS: ${message}`);
231
+ // models/audio.ts
232
+ function isBlobOrFile(value) {
233
+ return value instanceof Blob || value instanceof File;
234
+ }
235
+ function isBuffer(value) {
236
+ return Buffer.isBuffer(value);
237
+ }
238
+ function isLocalTranscriptionParams(value) {
239
+ return typeof value === "object" && value !== null && "audio" in value && (isBlobOrFile(value.audio) || isBuffer(value.audio));
240
+ }
241
+ function isCoreTranscriptionParams(value) {
242
+ return typeof value === "object" && value !== null && "audioUrl" in value && typeof value.audioUrl === "string";
243
+ }
244
+ async function fetchAudioFromUrl(url) {
245
+ const response = await fetch(url);
246
+ if (!response.ok) {
247
+ throw new Error(`Failed to fetch audio from URL: ${response.status}`);
449
248
  }
249
+ return response.blob();
450
250
  }
451
251
  async function handleTranscription(runtime, input) {
452
- let modelName = getSetting(runtime, "OPENAI_TRANSCRIPTION_MODEL", "gpt-4o-mini-transcribe");
453
- logger7.log(`[OpenAI] Using TRANSCRIPTION model: ${modelName}`);
454
- const baseURL = getBaseURL(runtime);
252
+ let modelName = getTranscriptionModel(runtime);
455
253
  let blob;
456
- let extraParams = null;
457
- if (input instanceof Blob || input instanceof File) {
254
+ let extraParams = {};
255
+ if (typeof input === "string") {
256
+ logger4.debug(`[OpenAI] Fetching audio from URL: ${input}`);
257
+ blob = await fetchAudioFromUrl(input);
258
+ } else if (isBlobOrFile(input)) {
458
259
  blob = input;
459
- } else if (Buffer.isBuffer(input)) {
460
- const detectedMimeType = detectAudioMimeType(input);
461
- logger7.debug(`Auto-detected audio MIME type: ${detectedMimeType}`);
462
- const uint8Array = new Uint8Array(input);
463
- blob = new Blob([uint8Array], { type: detectedMimeType });
464
- } else if (typeof input === "object" && input !== null && input.audio != null) {
465
- const params = input;
466
- if (!(params.audio instanceof Blob) && !(params.audio instanceof File) && !Buffer.isBuffer(params.audio)) {
467
- throw new Error("TRANSCRIPTION param 'audio' must be a Blob/File/Buffer.");
260
+ } else if (isBuffer(input)) {
261
+ const mimeType2 = detectAudioMimeType(input);
262
+ logger4.debug(`[OpenAI] Auto-detected audio MIME type: ${mimeType2}`);
263
+ blob = new Blob([new Uint8Array(input)], { type: mimeType2 });
264
+ } else if (isLocalTranscriptionParams(input)) {
265
+ extraParams = input;
266
+ if (input.model) {
267
+ modelName = input.model;
468
268
  }
469
- if (Buffer.isBuffer(params.audio)) {
470
- let mimeType = params.mimeType;
471
- if (!mimeType) {
472
- mimeType = detectAudioMimeType(params.audio);
473
- logger7.debug(`Auto-detected audio MIME type: ${mimeType}`);
474
- } else {
475
- logger7.debug(`Using provided MIME type: ${mimeType}`);
476
- }
477
- const uint8Array = new Uint8Array(params.audio);
478
- blob = new Blob([uint8Array], { type: mimeType });
269
+ if (isBuffer(input.audio)) {
270
+ const mimeType2 = input.mimeType ?? detectAudioMimeType(input.audio);
271
+ logger4.debug(`[OpenAI] Using MIME type: ${mimeType2}`);
272
+ blob = new Blob([new Uint8Array(input.audio)], { type: mimeType2 });
479
273
  } else {
480
- blob = params.audio;
481
- }
482
- extraParams = params;
483
- if (typeof params.model === "string" && params.model) {
484
- modelName = params.model;
274
+ blob = input.audio;
485
275
  }
276
+ } else if (isCoreTranscriptionParams(input)) {
277
+ logger4.debug(`[OpenAI] Fetching audio from URL: ${input.audioUrl}`);
278
+ blob = await fetchAudioFromUrl(input.audioUrl);
279
+ extraParams = { prompt: input.prompt };
486
280
  } else {
487
- throw new Error("TRANSCRIPTION expects a Blob/File/Buffer or an object { audio: Blob/File/Buffer, mimeType?, language?, response_format?, timestampGranularities?, prompt?, temperature?, model? }");
281
+ throw new Error("TRANSCRIPTION expects Blob, File, Buffer, URL string, or TranscriptionParams object");
488
282
  }
489
- const mime = blob.type || "audio/webm";
490
- const filename = blob.name || (mime.includes("mp3") || mime.includes("mpeg") ? "recording.mp3" : mime.includes("ogg") ? "recording.ogg" : mime.includes("wav") ? "recording.wav" : mime.includes("webm") ? "recording.webm" : "recording.bin");
283
+ logger4.debug(`[OpenAI] Using TRANSCRIPTION model: ${modelName}`);
284
+ const mimeType = blob.type || "audio/webm";
285
+ const filename = blob.name || getFilenameForMimeType(mimeType.startsWith("audio/") ? mimeType : "audio/webm");
491
286
  const formData = new FormData;
492
287
  formData.append("file", blob, filename);
493
- formData.append("model", String(modelName));
494
- if (extraParams) {
495
- if (typeof extraParams.language === "string") {
496
- formData.append("language", String(extraParams.language));
288
+ formData.append("model", modelName);
289
+ if (extraParams.language) {
290
+ formData.append("language", extraParams.language);
291
+ }
292
+ if (extraParams.responseFormat) {
293
+ formData.append("response_format", extraParams.responseFormat);
294
+ }
295
+ if (extraParams.prompt) {
296
+ formData.append("prompt", extraParams.prompt);
297
+ }
298
+ if (extraParams.temperature !== undefined) {
299
+ formData.append("temperature", String(extraParams.temperature));
300
+ }
301
+ if (extraParams.timestampGranularities) {
302
+ for (const granularity of extraParams.timestampGranularities) {
303
+ formData.append("timestamp_granularities[]", granularity);
497
304
  }
498
- if (typeof extraParams.response_format === "string") {
499
- formData.append("response_format", String(extraParams.response_format));
305
+ }
306
+ const baseURL = getBaseURL(runtime);
307
+ const response = await fetch(`${baseURL}/audio/transcriptions`, {
308
+ method: "POST",
309
+ headers: getAuthHeader(runtime),
310
+ body: formData
311
+ });
312
+ if (!response.ok) {
313
+ const errorText = await response.text().catch(() => "Unknown error");
314
+ throw new Error(`OpenAI transcription failed: ${response.status} ${response.statusText} - ${errorText}`);
315
+ }
316
+ const data = await response.json();
317
+ return data.text;
318
+ }
319
+ async function handleTextToSpeech(runtime, input) {
320
+ let text;
321
+ let voice;
322
+ let format = "mp3";
323
+ let model;
324
+ let instructions;
325
+ if (typeof input === "string") {
326
+ text = input;
327
+ voice = undefined;
328
+ } else {
329
+ text = input.text;
330
+ voice = input.voice;
331
+ if ("format" in input && input.format) {
332
+ format = input.format;
500
333
  }
501
- if (typeof extraParams.prompt === "string") {
502
- formData.append("prompt", String(extraParams.prompt));
334
+ if ("model" in input && input.model) {
335
+ model = input.model;
503
336
  }
504
- if (typeof extraParams.temperature === "number") {
505
- formData.append("temperature", String(extraParams.temperature));
337
+ if ("instructions" in input && input.instructions) {
338
+ instructions = input.instructions;
506
339
  }
507
- if (Array.isArray(extraParams.timestampGranularities)) {
508
- for (const g of extraParams.timestampGranularities) {
509
- formData.append("timestamp_granularities[]", String(g));
510
- }
340
+ }
341
+ model = model ?? getTTSModel(runtime);
342
+ voice = voice ?? getTTSVoice(runtime);
343
+ instructions = instructions ?? getTTSInstructions(runtime);
344
+ logger4.debug(`[OpenAI] Using TEXT_TO_SPEECH model: ${model}`);
345
+ if (!text || text.trim().length === 0) {
346
+ throw new Error("TEXT_TO_SPEECH requires non-empty text");
347
+ }
348
+ if (text.length > 4096) {
349
+ throw new Error("TEXT_TO_SPEECH text exceeds 4096 character limit");
350
+ }
351
+ const validVoices = ["alloy", "echo", "fable", "onyx", "nova", "shimmer"];
352
+ if (voice && !validVoices.includes(voice)) {
353
+ throw new Error(`Invalid voice: ${voice}. Must be one of: ${validVoices.join(", ")}`);
354
+ }
355
+ const baseURL = getBaseURL(runtime);
356
+ const requestBody = {
357
+ model,
358
+ voice,
359
+ input: text,
360
+ response_format: format
361
+ };
362
+ if (instructions && instructions.length > 0) {
363
+ requestBody.instructions = instructions;
364
+ }
365
+ const response = await fetch(`${baseURL}/audio/speech`, {
366
+ method: "POST",
367
+ headers: {
368
+ ...getAuthHeader(runtime),
369
+ "Content-Type": "application/json",
370
+ ...format === "mp3" ? { Accept: "audio/mpeg" } : {}
371
+ },
372
+ body: JSON.stringify(requestBody)
373
+ });
374
+ if (!response.ok) {
375
+ const errorText = await response.text().catch(() => "Unknown error");
376
+ throw new Error(`OpenAI TTS failed: ${response.status} ${response.statusText} - ${errorText}`);
377
+ }
378
+ return response.arrayBuffer();
379
+ }
380
+ // models/embedding.ts
381
+ import { logger as logger5, ModelType, VECTOR_DIMS } from "@elizaos/core";
382
+
383
+ // utils/events.ts
384
+ import { EventType } from "@elizaos/core";
385
+ var MAX_PROMPT_LENGTH = 200;
386
+ function truncatePrompt(prompt) {
387
+ if (prompt.length <= MAX_PROMPT_LENGTH) {
388
+ return prompt;
389
+ }
390
+ return `${prompt.slice(0, MAX_PROMPT_LENGTH)}…`;
391
+ }
392
+ function normalizeUsage(usage) {
393
+ if ("promptTokens" in usage) {
394
+ return {
395
+ promptTokens: usage.promptTokens ?? 0,
396
+ completionTokens: usage.completionTokens ?? 0,
397
+ totalTokens: usage.totalTokens ?? (usage.promptTokens ?? 0) + (usage.completionTokens ?? 0)
398
+ };
399
+ }
400
+ if ("inputTokens" in usage || "outputTokens" in usage) {
401
+ const input = usage.inputTokens ?? 0;
402
+ const output = usage.outputTokens ?? 0;
403
+ return {
404
+ promptTokens: input,
405
+ completionTokens: output,
406
+ totalTokens: input + output
407
+ };
408
+ }
409
+ return {
410
+ promptTokens: 0,
411
+ completionTokens: 0,
412
+ totalTokens: 0
413
+ };
414
+ }
415
+ function emitModelUsageEvent(runtime, type, prompt, usage) {
416
+ const normalized = normalizeUsage(usage);
417
+ const payload = {
418
+ runtime,
419
+ source: "openai",
420
+ provider: "openai",
421
+ type,
422
+ prompt: truncatePrompt(prompt),
423
+ tokens: {
424
+ prompt: normalized.promptTokens,
425
+ completion: normalized.completionTokens,
426
+ total: normalized.totalTokens
511
427
  }
428
+ };
429
+ runtime.emitEvent(EventType.MODEL_USED, payload);
430
+ }
431
+
432
+ // models/embedding.ts
433
+ function validateDimension(dimension) {
434
+ const validDimensions = Object.values(VECTOR_DIMS);
435
+ if (!validDimensions.includes(dimension)) {
436
+ throw new Error(`Invalid embedding dimension: ${dimension}. Must be one of: ${validDimensions.join(", ")}`);
512
437
  }
513
- try {
514
- const response = await fetch(`${baseURL}/audio/transcriptions`, {
515
- method: "POST",
516
- headers: {
517
- ...getAuthHeader(runtime)
518
- },
519
- body: formData
438
+ return dimension;
439
+ }
440
+ function extractText(params) {
441
+ if (params === null) {
442
+ return null;
443
+ }
444
+ if (typeof params === "string") {
445
+ return params;
446
+ }
447
+ if (typeof params === "object" && typeof params.text === "string") {
448
+ return params.text;
449
+ }
450
+ throw new Error("Invalid embedding params: expected string, { text: string }, or null");
451
+ }
452
+ async function handleTextEmbedding(runtime, params) {
453
+ const embeddingModel = getEmbeddingModel(runtime);
454
+ const embeddingDimension = validateDimension(getEmbeddingDimensions(runtime));
455
+ const text = extractText(params);
456
+ if (text === null) {
457
+ logger5.debug("[OpenAI] Creating test embedding for initialization");
458
+ const testVector = new Array(embeddingDimension).fill(0);
459
+ testVector[0] = 0.1;
460
+ return testVector;
461
+ }
462
+ const trimmedText = text.trim();
463
+ if (trimmedText.length === 0) {
464
+ throw new Error("Cannot generate embedding for empty text");
465
+ }
466
+ const baseURL = getEmbeddingBaseURL(runtime);
467
+ const url = `${baseURL}/embeddings`;
468
+ logger5.debug(`[OpenAI] Generating embedding with model: ${embeddingModel}`);
469
+ const response = await fetch(url, {
470
+ method: "POST",
471
+ headers: {
472
+ ...getAuthHeader(runtime, true),
473
+ "Content-Type": "application/json"
474
+ },
475
+ body: JSON.stringify({
476
+ model: embeddingModel,
477
+ input: trimmedText
478
+ })
479
+ });
480
+ if (!response.ok) {
481
+ const errorText = await response.text().catch(() => "Unknown error");
482
+ throw new Error(`OpenAI embedding API error: ${response.status} ${response.statusText} - ${errorText}`);
483
+ }
484
+ const data = await response.json();
485
+ const firstResult = data?.data?.[0];
486
+ if (!firstResult || !firstResult.embedding) {
487
+ throw new Error("OpenAI API returned invalid embedding response structure");
488
+ }
489
+ const embedding = firstResult.embedding;
490
+ if (embedding.length !== embeddingDimension) {
491
+ throw new Error(`Embedding dimension mismatch: got ${embedding.length}, expected ${embeddingDimension}. ` + `Check OPENAI_EMBEDDING_DIMENSIONS setting.`);
492
+ }
493
+ if (data.usage) {
494
+ emitModelUsageEvent(runtime, ModelType.TEXT_EMBEDDING, trimmedText, {
495
+ promptTokens: data.usage.prompt_tokens,
496
+ completionTokens: 0,
497
+ totalTokens: data.usage.total_tokens
520
498
  });
521
- if (!response.ok) {
522
- throw new Error(`Failed to transcribe audio: ${response.status} ${response.statusText}`);
523
- }
524
- const data = await response.json();
525
- return data.text || "";
526
- } catch (error) {
527
- const message = error instanceof Error ? error.message : String(error);
528
- logger7.error(`TRANSCRIPTION error: ${message}`);
529
- throw error;
530
499
  }
500
+ logger5.debug(`[OpenAI] Generated embedding with ${embedding.length} dimensions`);
501
+ return embedding;
531
502
  }
532
- async function handleTextToSpeech(runtime, input) {
533
- const options = typeof input === "string" ? { text: input } : input;
534
- const resolvedModel = options.model || getSetting(runtime, "OPENAI_TTS_MODEL", "gpt-4o-mini-tts");
535
- logger7.log(`[OpenAI] Using TEXT_TO_SPEECH model: ${resolvedModel}`);
536
- try {
537
- return await fetchTextToSpeech(runtime, options);
538
- } catch (error) {
539
- const message = error instanceof Error ? error.message : String(error);
540
- logger7.error(`Error in TEXT_TO_SPEECH: ${message}`);
541
- throw error;
503
+ // models/image.ts
504
+ import { logger as logger6, ModelType as ModelType2 } from "@elizaos/core";
505
+ var DEFAULT_IMAGE_DESCRIPTION_PROMPT = "Please analyze this image and provide a title and detailed description.";
506
+ async function handleImageGeneration(runtime, params) {
507
+ const modelName = getImageModel(runtime);
508
+ const count = params.count ?? 1;
509
+ const size = params.size ?? "1024x1024";
510
+ const extendedParams = params;
511
+ logger6.debug(`[OpenAI] Using IMAGE model: ${modelName}`);
512
+ if (!params.prompt || params.prompt.trim().length === 0) {
513
+ throw new Error("IMAGE generation requires a non-empty prompt");
514
+ }
515
+ if (count < 1 || count > 10) {
516
+ throw new Error("IMAGE count must be between 1 and 10");
517
+ }
518
+ const baseURL = getBaseURL(runtime);
519
+ const requestBody = {
520
+ model: modelName,
521
+ prompt: params.prompt,
522
+ n: count,
523
+ size
524
+ };
525
+ if (extendedParams.quality) {
526
+ requestBody.quality = extendedParams.quality;
542
527
  }
528
+ if (extendedParams.style) {
529
+ requestBody.style = extendedParams.style;
530
+ }
531
+ const response = await fetch(`${baseURL}/images/generations`, {
532
+ method: "POST",
533
+ headers: {
534
+ ...getAuthHeader(runtime),
535
+ "Content-Type": "application/json"
536
+ },
537
+ body: JSON.stringify(requestBody)
538
+ });
539
+ if (!response.ok) {
540
+ const errorText = await response.text().catch(() => "Unknown error");
541
+ throw new Error(`OpenAI image generation failed: ${response.status} ${response.statusText} - ${errorText}`);
542
+ }
543
+ const data = await response.json();
544
+ if (!data.data || data.data.length === 0) {
545
+ throw new Error("OpenAI API returned no images");
546
+ }
547
+ return data.data.map((item) => ({
548
+ url: item.url,
549
+ revisedPrompt: item.revised_prompt
550
+ }));
551
+ }
552
+ function parseTitleFromResponse(content) {
553
+ const titleMatch = content.match(/title[:\s]+(.+?)(?:\n|$)/i);
554
+ return titleMatch?.[1]?.trim() ?? "Image Analysis";
555
+ }
556
+ function parseDescriptionFromResponse(content) {
557
+ return content.replace(/title[:\s]+(.+?)(?:\n|$)/i, "").trim();
543
558
  }
544
- // src/models/object.ts
545
- import { logger as logger9, ModelType as ModelType4 } from "@elizaos/core";
559
+ async function handleImageDescription(runtime, params) {
560
+ const modelName = getImageDescriptionModel(runtime);
561
+ const maxTokens = getImageDescriptionMaxTokens(runtime);
562
+ logger6.debug(`[OpenAI] Using IMAGE_DESCRIPTION model: ${modelName}`);
563
+ let imageUrl;
564
+ let promptText;
565
+ if (typeof params === "string") {
566
+ imageUrl = params;
567
+ promptText = DEFAULT_IMAGE_DESCRIPTION_PROMPT;
568
+ } else {
569
+ imageUrl = params.imageUrl;
570
+ promptText = params.prompt ?? DEFAULT_IMAGE_DESCRIPTION_PROMPT;
571
+ }
572
+ if (!imageUrl || imageUrl.trim().length === 0) {
573
+ throw new Error("IMAGE_DESCRIPTION requires a valid image URL");
574
+ }
575
+ const baseURL = getBaseURL(runtime);
576
+ const requestBody = {
577
+ model: modelName,
578
+ messages: [
579
+ {
580
+ role: "user",
581
+ content: [
582
+ { type: "text", text: promptText },
583
+ { type: "image_url", image_url: { url: imageUrl } }
584
+ ]
585
+ }
586
+ ],
587
+ max_tokens: maxTokens
588
+ };
589
+ const response = await fetch(`${baseURL}/chat/completions`, {
590
+ method: "POST",
591
+ headers: {
592
+ ...getAuthHeader(runtime),
593
+ "Content-Type": "application/json"
594
+ },
595
+ body: JSON.stringify(requestBody)
596
+ });
597
+ if (!response.ok) {
598
+ const errorText = await response.text().catch(() => "Unknown error");
599
+ throw new Error(`OpenAI image description failed: ${response.status} ${response.statusText} - ${errorText}`);
600
+ }
601
+ const data = await response.json();
602
+ if (data.usage) {
603
+ emitModelUsageEvent(runtime, ModelType2.IMAGE_DESCRIPTION, typeof params === "string" ? params : params.prompt ?? "", {
604
+ promptTokens: data.usage.prompt_tokens,
605
+ completionTokens: data.usage.completion_tokens,
606
+ totalTokens: data.usage.total_tokens
607
+ });
608
+ }
609
+ const firstChoice = data.choices?.[0];
610
+ const content = firstChoice?.message?.content;
611
+ if (!content) {
612
+ throw new Error("OpenAI API returned empty image description");
613
+ }
614
+ return {
615
+ title: parseTitleFromResponse(content),
616
+ description: parseDescriptionFromResponse(content)
617
+ };
618
+ }
619
+ // models/object.ts
620
+ import { logger as logger8, ModelType as ModelType3 } from "@elizaos/core";
546
621
  import { generateObject } from "ai";
547
622
 
548
- // src/utils/json.ts
549
- import { logger as logger8 } from "@elizaos/core";
623
+ // providers/openai.ts
624
+ import { createOpenAI } from "@ai-sdk/openai";
625
+ var PROXY_API_KEY = "sk-proxy";
626
+ function createOpenAIClient(runtime) {
627
+ const baseURL = getBaseURL(runtime);
628
+ const apiKey = getApiKey(runtime);
629
+ if (!apiKey && isProxyMode(runtime)) {
630
+ return createOpenAI({
631
+ apiKey: PROXY_API_KEY,
632
+ baseURL
633
+ });
634
+ }
635
+ if (!apiKey) {
636
+ throw new Error("OPENAI_API_KEY is required. Set it in your environment variables or runtime settings.");
637
+ }
638
+ return createOpenAI({
639
+ apiKey,
640
+ baseURL
641
+ });
642
+ }
643
+ // utils/json.ts
644
+ import { logger as logger7 } from "@elizaos/core";
550
645
  import { JSONParseError } from "ai";
646
+ var JSON_CLEANUP_PATTERNS = {
647
+ MARKDOWN_JSON: /```json\n|\n```|```/g,
648
+ WHITESPACE: /^\s+|\s+$/g
649
+ };
551
650
  function getJsonRepairFunction() {
552
651
  return async ({ text, error }) => {
553
- try {
554
- if (error instanceof JSONParseError) {
555
- const cleanedText = text.replace(/```json\n|\n```|```/g, "");
556
- JSON.parse(cleanedText);
557
- return cleanedText;
558
- }
652
+ if (!(error instanceof JSONParseError)) {
559
653
  return null;
560
- } catch (jsonError) {
561
- const message = jsonError instanceof Error ? jsonError.message : String(jsonError);
562
- logger8.warn(`Failed to repair JSON text: ${message}`);
654
+ }
655
+ try {
656
+ const cleanedText = text.replace(JSON_CLEANUP_PATTERNS.MARKDOWN_JSON, "");
657
+ JSON.parse(cleanedText);
658
+ logger7.debug("[JSON Repair] Successfully repaired JSON by removing markdown wrappers");
659
+ return cleanedText;
660
+ } catch {
661
+ logger7.warn("[JSON Repair] Unable to repair JSON text");
563
662
  return null;
564
663
  }
565
664
  };
566
665
  }
567
666
 
568
- // src/models/object.ts
667
+ // models/object.ts
569
668
  async function generateObjectByModelType(runtime, params, modelType, getModelFn) {
570
669
  const openai = createOpenAIClient(runtime);
571
670
  const modelName = getModelFn(runtime);
572
- logger9.log(`[OpenAI] Using ${modelType} model: ${modelName}`);
573
- const temperature = params.temperature ?? 0;
574
- const schemaPresent = !!params.schema;
575
- if (schemaPresent) {
576
- logger9.warn(`Schema provided but ignored: OpenAI object generation currently uses output=no-schema. The schema parameter has no effect.`);
671
+ logger8.debug(`[OpenAI] Using ${modelType} model: ${modelName}`);
672
+ if (!params.prompt || params.prompt.trim().length === 0) {
673
+ throw new Error("Object generation requires a non-empty prompt");
577
674
  }
578
- try {
579
- const { object, usage } = await generateObject({
580
- model: openai.languageModel(modelName),
581
- output: "no-schema",
582
- prompt: params.prompt,
583
- temperature,
584
- experimental_repairText: getJsonRepairFunction()
585
- });
586
- if (usage) {
587
- emitModelUsageEvent(runtime, modelType, params.prompt, usage);
588
- }
589
- return object;
590
- } catch (error) {
591
- const message = error instanceof Error ? error.message : String(error);
592
- logger9.error(`[generateObject] Error: ${message}`);
593
- throw error;
675
+ if (params.schema) {
676
+ logger8.debug("[OpenAI] Schema provided but using no-schema mode. " + "Structure is determined by prompt instructions.");
677
+ }
678
+ const model = openai.chat(modelName);
679
+ const { object, usage } = await generateObject({
680
+ model,
681
+ output: "no-schema",
682
+ prompt: params.prompt,
683
+ experimental_repairText: getJsonRepairFunction()
684
+ });
685
+ if (usage) {
686
+ emitModelUsageEvent(runtime, modelType, params.prompt, usage);
687
+ }
688
+ if (typeof object !== "object" || object === null) {
689
+ throw new Error(`Object generation returned ${typeof object}, expected object`);
594
690
  }
691
+ return object;
595
692
  }
596
693
  async function handleObjectSmall(runtime, params) {
597
- return generateObjectByModelType(runtime, params, ModelType4.OBJECT_SMALL, getSmallModel);
694
+ return generateObjectByModelType(runtime, params, ModelType3.OBJECT_SMALL, getSmallModel);
598
695
  }
599
696
  async function handleObjectLarge(runtime, params) {
600
- return generateObjectByModelType(runtime, params, ModelType4.OBJECT_LARGE, getLargeModel);
697
+ return generateObjectByModelType(runtime, params, ModelType3.OBJECT_LARGE, getLargeModel);
698
+ }
699
+ // models/research.ts
700
+ import { logger as logger9 } from "@elizaos/core";
701
+ function convertToolToApi(tool) {
702
+ switch (tool.type) {
703
+ case "web_search_preview":
704
+ return { type: "web_search_preview" };
705
+ case "file_search":
706
+ return {
707
+ type: "file_search",
708
+ vector_store_ids: tool.vectorStoreIds
709
+ };
710
+ case "code_interpreter":
711
+ return {
712
+ type: "code_interpreter",
713
+ container: tool.container ?? { type: "auto" }
714
+ };
715
+ case "mcp":
716
+ return {
717
+ type: "mcp",
718
+ server_label: tool.serverLabel,
719
+ server_url: tool.serverUrl,
720
+ require_approval: tool.requireApproval ?? "never"
721
+ };
722
+ default:
723
+ throw new Error(`Unknown research tool type: ${tool.type}`);
724
+ }
725
+ }
726
+ function convertOutputItem(item) {
727
+ switch (item.type) {
728
+ case "web_search_call":
729
+ return {
730
+ id: item.id ?? "",
731
+ type: "web_search_call",
732
+ status: item.status ?? "completed",
733
+ action: {
734
+ type: item.action?.type ?? "search",
735
+ query: item.action?.query,
736
+ url: item.action?.url
737
+ }
738
+ };
739
+ case "file_search_call":
740
+ return {
741
+ id: item.id ?? "",
742
+ type: "file_search_call",
743
+ status: item.status ?? "completed",
744
+ query: item.query ?? "",
745
+ results: item.results?.map((r) => ({
746
+ fileId: r.file_id,
747
+ fileName: r.file_name,
748
+ score: r.score
749
+ }))
750
+ };
751
+ case "code_interpreter_call":
752
+ return {
753
+ id: item.id ?? "",
754
+ type: "code_interpreter_call",
755
+ status: item.status ?? "completed",
756
+ code: item.code ?? "",
757
+ output: item.output
758
+ };
759
+ case "mcp_tool_call":
760
+ return {
761
+ id: item.id ?? "",
762
+ type: "mcp_tool_call",
763
+ status: item.status ?? "completed",
764
+ serverLabel: item.server_label ?? "",
765
+ toolName: item.tool_name ?? "",
766
+ arguments: item.arguments ?? {},
767
+ result: item.result
768
+ };
769
+ case "message":
770
+ return {
771
+ type: "message",
772
+ content: item.content?.map((c) => ({
773
+ type: "output_text",
774
+ text: c.text,
775
+ annotations: c.annotations?.map((a) => ({
776
+ url: a.url,
777
+ title: a.title,
778
+ startIndex: a.start_index,
779
+ endIndex: a.end_index
780
+ })) ?? []
781
+ })) ?? []
782
+ };
783
+ default:
784
+ return null;
785
+ }
786
+ }
787
+ function extractTextAndAnnotations(response) {
788
+ if (response.output_text) {
789
+ const annotations2 = [];
790
+ if (response.output) {
791
+ for (const item of response.output) {
792
+ if (item.type === "message" && item.content) {
793
+ for (const content of item.content) {
794
+ if (content.annotations) {
795
+ for (const ann of content.annotations) {
796
+ annotations2.push({
797
+ url: ann.url,
798
+ title: ann.title,
799
+ startIndex: ann.start_index,
800
+ endIndex: ann.end_index
801
+ });
802
+ }
803
+ }
804
+ }
805
+ }
806
+ }
807
+ }
808
+ return { text: response.output_text, annotations: annotations2 };
809
+ }
810
+ let text = "";
811
+ const annotations = [];
812
+ if (response.output) {
813
+ for (const item of response.output) {
814
+ if (item.type === "message" && item.content) {
815
+ for (const content of item.content) {
816
+ text += content.text;
817
+ if (content.annotations) {
818
+ for (const ann of content.annotations) {
819
+ annotations.push({
820
+ url: ann.url,
821
+ title: ann.title,
822
+ startIndex: ann.start_index,
823
+ endIndex: ann.end_index
824
+ });
825
+ }
826
+ }
827
+ }
828
+ }
829
+ }
830
+ }
831
+ return { text, annotations };
601
832
  }
602
- // src/models/tokenizer.ts
833
+ async function handleResearch(runtime, params) {
834
+ const apiKey = getApiKey(runtime);
835
+ if (!apiKey) {
836
+ throw new Error("OPENAI_API_KEY is required for deep research. Set it in your environment variables or runtime settings.");
837
+ }
838
+ const baseURL = getBaseURL(runtime);
839
+ const modelName = params.model ?? getResearchModel(runtime);
840
+ const timeout = getResearchTimeout(runtime);
841
+ logger9.debug(`[OpenAI] Starting deep research with model: ${modelName}`);
842
+ logger9.debug(`[OpenAI] Research input: ${params.input.substring(0, 100)}...`);
843
+ const dataSourceTools = params.tools?.filter((t) => t.type === "web_search_preview" || t.type === "file_search" || t.type === "mcp");
844
+ if (!dataSourceTools || dataSourceTools.length === 0) {
845
+ logger9.debug("[OpenAI] No data source tools specified, defaulting to web_search_preview");
846
+ params.tools = [{ type: "web_search_preview" }, ...params.tools ?? []];
847
+ }
848
+ const requestBody = {
849
+ model: modelName,
850
+ input: params.input
851
+ };
852
+ if (params.instructions) {
853
+ requestBody.instructions = params.instructions;
854
+ }
855
+ if (params.background !== undefined) {
856
+ requestBody.background = params.background;
857
+ }
858
+ if (params.tools && params.tools.length > 0) {
859
+ requestBody.tools = params.tools.map(convertToolToApi);
860
+ }
861
+ if (params.maxToolCalls !== undefined) {
862
+ requestBody.max_tool_calls = params.maxToolCalls;
863
+ }
864
+ if (params.reasoningSummary) {
865
+ requestBody.reasoning = { summary: params.reasoningSummary };
866
+ }
867
+ logger9.debug(`[OpenAI] Research request body: ${JSON.stringify(requestBody, null, 2)}`);
868
+ const response = await fetch(`${baseURL}/responses`, {
869
+ method: "POST",
870
+ headers: {
871
+ Authorization: `Bearer ${apiKey}`,
872
+ "Content-Type": "application/json"
873
+ },
874
+ body: JSON.stringify(requestBody),
875
+ signal: AbortSignal.timeout(timeout)
876
+ });
877
+ if (!response.ok) {
878
+ const errorText = await response.text();
879
+ logger9.error(`[OpenAI] Research request failed: ${response.status} ${errorText}`);
880
+ throw new Error(`Deep research request failed: ${response.status} ${response.statusText}`);
881
+ }
882
+ const data = await response.json();
883
+ if (data.error) {
884
+ logger9.error(`[OpenAI] Research API error: ${data.error.message}`);
885
+ throw new Error(`Deep research error: ${data.error.message}`);
886
+ }
887
+ logger9.debug(`[OpenAI] Research response received. Status: ${data.status ?? "completed"}`);
888
+ const { text, annotations } = extractTextAndAnnotations(data);
889
+ const outputItems = [];
890
+ if (data.output) {
891
+ for (const item of data.output) {
892
+ const converted = convertOutputItem(item);
893
+ if (converted) {
894
+ outputItems.push(converted);
895
+ }
896
+ }
897
+ }
898
+ const result = {
899
+ id: data.id,
900
+ text,
901
+ annotations,
902
+ outputItems,
903
+ status: data.status
904
+ };
905
+ logger9.info(`[OpenAI] Research completed. Text length: ${text.length}, Annotations: ${annotations.length}, Output items: ${outputItems.length}`);
906
+ return result;
907
+ }
908
+ // models/text.ts
909
+ import { logger as logger10, ModelType as ModelType4 } from "@elizaos/core";
910
+ import { generateText, streamText } from "ai";
911
+ function convertUsage(usage) {
912
+ if (!usage) {
913
+ return;
914
+ }
915
+ const promptTokens = usage.inputTokens ?? 0;
916
+ const completionTokens = usage.outputTokens ?? 0;
917
+ return {
918
+ promptTokens,
919
+ completionTokens,
920
+ totalTokens: promptTokens + completionTokens
921
+ };
922
+ }
923
+ async function generateTextByModelType(runtime, params, modelType, getModelFn) {
924
+ const openai = createOpenAIClient(runtime);
925
+ const modelName = getModelFn(runtime);
926
+ logger10.debug(`[OpenAI] Using ${modelType} model: ${modelName}`);
927
+ const systemPrompt = runtime.character.system ?? undefined;
928
+ const model = openai.chat(modelName);
929
+ const generateParams = {
930
+ model,
931
+ prompt: params.prompt,
932
+ system: systemPrompt,
933
+ maxOutputTokens: params.maxTokens ?? 8192,
934
+ experimental_telemetry: { isEnabled: getExperimentalTelemetry(runtime) }
935
+ };
936
+ if (params.stream) {
937
+ const result = streamText(generateParams);
938
+ return {
939
+ textStream: result.textStream,
940
+ text: Promise.resolve(result.text),
941
+ usage: Promise.resolve(result.usage).then(convertUsage),
942
+ finishReason: Promise.resolve(result.finishReason).then((r) => r)
943
+ };
944
+ }
945
+ const { text, usage } = await generateText(generateParams);
946
+ if (usage) {
947
+ emitModelUsageEvent(runtime, modelType, params.prompt, usage);
948
+ }
949
+ return text;
950
+ }
951
+ async function handleTextSmall(runtime, params) {
952
+ return generateTextByModelType(runtime, params, ModelType4.TEXT_SMALL, getSmallModel);
953
+ }
954
+ async function handleTextLarge(runtime, params) {
955
+ return generateTextByModelType(runtime, params, ModelType4.TEXT_LARGE, getLargeModel);
956
+ }
957
+ // models/tokenizer.ts
603
958
  import { ModelType as ModelType6 } from "@elizaos/core";
604
959
 
605
- // src/utils/tokenization.ts
960
+ // utils/tokenization.ts
606
961
  import { ModelType as ModelType5 } from "@elizaos/core";
607
- import { encodingForModel, getEncoding } from "js-tiktoken";
962
+ import {
963
+ encodingForModel,
964
+ getEncoding
965
+ } from "js-tiktoken";
608
966
  function resolveTokenizerEncoding(modelName) {
609
967
  const normalized = modelName.toLowerCase();
610
968
  const fallbackEncoding = normalized.includes("4o") ? "o200k_base" : "cl100k_base";
611
969
  try {
612
970
  return encodingForModel(modelName);
613
- } catch (error) {
971
+ } catch {
614
972
  return getEncoding(fallbackEncoding);
615
973
  }
616
974
  }
617
- async function tokenizeText(runtime, model, prompt) {
618
- const modelName = model === ModelType5.TEXT_SMALL ? getSmallModel(runtime) : getLargeModel(runtime);
619
- const tokens = resolveTokenizerEncoding(modelName).encode(prompt);
620
- return tokens;
975
+ function getModelName(runtime, modelType) {
976
+ if (modelType === ModelType5.TEXT_SMALL) {
977
+ return getSmallModel(runtime);
978
+ }
979
+ return getLargeModel(runtime);
621
980
  }
622
- async function detokenizeText(runtime, model, tokens) {
623
- const modelName = model === ModelType5.TEXT_SMALL ? getSmallModel(runtime) : getLargeModel(runtime);
624
- return resolveTokenizerEncoding(modelName).decode(tokens);
981
+ function tokenizeText(runtime, modelType, text) {
982
+ const modelName = getModelName(runtime, modelType);
983
+ const encoder = resolveTokenizerEncoding(modelName);
984
+ return encoder.encode(text);
985
+ }
986
+ function detokenizeText(runtime, modelType, tokens) {
987
+ const modelName = getModelName(runtime, modelType);
988
+ const encoder = resolveTokenizerEncoding(modelName);
989
+ return encoder.decode(tokens);
625
990
  }
626
991
 
627
- // src/models/tokenizer.ts
628
- async function handleTokenizerEncode(runtime, { prompt, modelType = ModelType6.TEXT_LARGE }) {
629
- return await tokenizeText(runtime, modelType, prompt);
992
+ // models/tokenizer.ts
993
+ async function handleTokenizerEncode(runtime, params) {
994
+ if (!params.prompt) {
995
+ throw new Error("Tokenization requires a non-empty prompt");
996
+ }
997
+ const modelType = params.modelType ?? ModelType6.TEXT_LARGE;
998
+ return tokenizeText(runtime, modelType, params.prompt);
999
+ }
1000
+ async function handleTokenizerDecode(runtime, params) {
1001
+ if (!params.tokens || !Array.isArray(params.tokens)) {
1002
+ throw new Error("Detokenization requires a valid tokens array");
1003
+ }
1004
+ if (params.tokens.length === 0) {
1005
+ return "";
1006
+ }
1007
+ for (let i = 0;i < params.tokens.length; i++) {
1008
+ const token = params.tokens[i];
1009
+ if (typeof token !== "number" || !Number.isFinite(token)) {
1010
+ throw new Error(`Invalid token at index ${i}: expected number`);
1011
+ }
1012
+ }
1013
+ const modelType = params.modelType ?? ModelType6.TEXT_LARGE;
1014
+ return detokenizeText(runtime, modelType, params.tokens);
630
1015
  }
631
- async function handleTokenizerDecode(runtime, { tokens, modelType = ModelType6.TEXT_LARGE }) {
632
- return await detokenizeText(runtime, modelType, tokens);
1016
+ // index.ts
1017
+ function getProcessEnv() {
1018
+ if (typeof process === "undefined") {
1019
+ return {};
1020
+ }
1021
+ return process.env;
633
1022
  }
634
- // src/index.ts
1023
+ var env = getProcessEnv();
635
1024
  var openaiPlugin = {
636
1025
  name: "openai",
637
- description: "OpenAI plugin",
1026
+ description: "OpenAI API integration for text, image, audio, and embedding models",
638
1027
  config: {
639
- OPENAI_API_KEY: process.env.OPENAI_API_KEY,
640
- OPENAI_BASE_URL: process.env.OPENAI_BASE_URL,
641
- OPENAI_SMALL_MODEL: process.env.OPENAI_SMALL_MODEL,
642
- OPENAI_LARGE_MODEL: process.env.OPENAI_LARGE_MODEL,
643
- SMALL_MODEL: process.env.SMALL_MODEL,
644
- LARGE_MODEL: process.env.LARGE_MODEL,
645
- OPENAI_EMBEDDING_MODEL: process.env.OPENAI_EMBEDDING_MODEL,
646
- OPENAI_EMBEDDING_API_KEY: process.env.OPENAI_EMBEDDING_API_KEY,
647
- OPENAI_EMBEDDING_URL: process.env.OPENAI_EMBEDDING_URL,
648
- OPENAI_EMBEDDING_DIMENSIONS: process.env.OPENAI_EMBEDDING_DIMENSIONS,
649
- OPENAI_IMAGE_DESCRIPTION_MODEL: process.env.OPENAI_IMAGE_DESCRIPTION_MODEL,
650
- OPENAI_IMAGE_DESCRIPTION_MAX_TOKENS: process.env.OPENAI_IMAGE_DESCRIPTION_MAX_TOKENS,
651
- OPENAI_EXPERIMENTAL_TELEMETRY: process.env.OPENAI_EXPERIMENTAL_TELEMETRY
1028
+ OPENAI_API_KEY: env.OPENAI_API_KEY ?? null,
1029
+ OPENAI_BASE_URL: env.OPENAI_BASE_URL ?? null,
1030
+ OPENAI_SMALL_MODEL: env.OPENAI_SMALL_MODEL ?? null,
1031
+ OPENAI_LARGE_MODEL: env.OPENAI_LARGE_MODEL ?? null,
1032
+ SMALL_MODEL: env.SMALL_MODEL ?? null,
1033
+ LARGE_MODEL: env.LARGE_MODEL ?? null,
1034
+ OPENAI_EMBEDDING_MODEL: env.OPENAI_EMBEDDING_MODEL ?? null,
1035
+ OPENAI_EMBEDDING_API_KEY: env.OPENAI_EMBEDDING_API_KEY ?? null,
1036
+ OPENAI_EMBEDDING_URL: env.OPENAI_EMBEDDING_URL ?? null,
1037
+ OPENAI_EMBEDDING_DIMENSIONS: env.OPENAI_EMBEDDING_DIMENSIONS ?? null,
1038
+ OPENAI_IMAGE_DESCRIPTION_MODEL: env.OPENAI_IMAGE_DESCRIPTION_MODEL ?? null,
1039
+ OPENAI_IMAGE_DESCRIPTION_MAX_TOKENS: env.OPENAI_IMAGE_DESCRIPTION_MAX_TOKENS ?? null,
1040
+ OPENAI_EXPERIMENTAL_TELEMETRY: env.OPENAI_EXPERIMENTAL_TELEMETRY ?? null,
1041
+ OPENAI_RESEARCH_MODEL: env.OPENAI_RESEARCH_MODEL ?? null,
1042
+ OPENAI_RESEARCH_TIMEOUT: env.OPENAI_RESEARCH_TIMEOUT ?? null
652
1043
  },
653
- async init(_config, runtime) {
654
- initializeOpenAI(_config, runtime);
1044
+ async init(config, runtime) {
1045
+ initializeOpenAI(config, runtime);
655
1046
  },
656
1047
  models: {
657
1048
  [ModelType7.TEXT_EMBEDDING]: async (runtime, params) => {
@@ -686,6 +1077,9 @@ var openaiPlugin = {
686
1077
  },
687
1078
  [ModelType7.OBJECT_LARGE]: async (runtime, params) => {
688
1079
  return handleObjectLarge(runtime, params);
1080
+ },
1081
+ [ModelType7.RESEARCH]: async (runtime, params) => {
1082
+ return handleResearch(runtime, params);
689
1083
  }
690
1084
  },
691
1085
  tests: [
@@ -693,221 +1087,170 @@ var openaiPlugin = {
693
1087
  name: "openai_plugin_tests",
694
1088
  tests: [
695
1089
  {
696
- name: "openai_test_url_and_api_key_validation",
1090
+ name: "openai_test_api_connectivity",
697
1091
  fn: async (runtime) => {
698
1092
  const baseURL = getBaseURL(runtime);
699
1093
  const response = await fetch(`${baseURL}/models`, {
700
1094
  headers: getAuthHeader(runtime)
701
1095
  });
702
- const data = await response.json();
703
- logger10.log({ data: data?.data?.length ?? "N/A" }, "Models Available");
704
1096
  if (!response.ok) {
705
- throw new Error(`Failed to validate OpenAI API key: ${response.statusText}`);
1097
+ throw new Error(`API connectivity test failed: ${response.status} ${response.statusText}`);
706
1098
  }
1099
+ const data = await response.json();
1100
+ logger11.info(`[OpenAI Test] API connected. ${data.data?.length ?? 0} models available.`);
707
1101
  }
708
1102
  },
709
1103
  {
710
1104
  name: "openai_test_text_embedding",
711
1105
  fn: async (runtime) => {
712
- try {
713
- const embedding = await runtime.useModel(ModelType7.TEXT_EMBEDDING, {
714
- text: "Hello, world!"
715
- });
716
- logger10.log({ embedding }, "embedding");
717
- } catch (error) {
718
- const message = error instanceof Error ? error.message : String(error);
719
- logger10.error(`Error in test_text_embedding: ${message}`);
720
- throw error;
1106
+ const embedding = await runtime.useModel(ModelType7.TEXT_EMBEDDING, {
1107
+ text: "Hello, world!"
1108
+ });
1109
+ if (!Array.isArray(embedding) || embedding.length === 0) {
1110
+ throw new Error("Embedding should return a non-empty array");
721
1111
  }
1112
+ logger11.info(`[OpenAI Test] Generated embedding with ${embedding.length} dimensions`);
722
1113
  }
723
1114
  },
724
1115
  {
725
- name: "openai_test_text_large",
1116
+ name: "openai_test_text_small",
726
1117
  fn: async (runtime) => {
727
- try {
728
- const text = await runtime.useModel(ModelType7.TEXT_LARGE, {
729
- prompt: "What is the nature of reality in 10 words?"
730
- });
731
- if (text.length === 0) {
732
- throw new Error("Failed to generate text");
733
- }
734
- logger10.log({ text }, "generated with test_text_large");
735
- } catch (error) {
736
- const message = error instanceof Error ? error.message : String(error);
737
- logger10.error(`Error in test_text_large: ${message}`);
738
- throw error;
1118
+ const text = await runtime.useModel(ModelType7.TEXT_SMALL, {
1119
+ prompt: "Say hello in exactly 5 words."
1120
+ });
1121
+ if (typeof text !== "string" || text.length === 0) {
1122
+ throw new Error("TEXT_SMALL should return non-empty string");
739
1123
  }
1124
+ logger11.info(`[OpenAI Test] TEXT_SMALL generated: "${text.substring(0, 50)}..."`);
740
1125
  }
741
1126
  },
742
1127
  {
743
- name: "openai_test_text_small",
1128
+ name: "openai_test_text_large",
744
1129
  fn: async (runtime) => {
745
- try {
746
- const text = await runtime.useModel(ModelType7.TEXT_SMALL, {
747
- prompt: "What is the nature of reality in 10 words?"
748
- });
749
- if (text.length === 0) {
750
- throw new Error("Failed to generate text");
751
- }
752
- logger10.log({ text }, "generated with test_text_small");
753
- } catch (error) {
754
- const message = error instanceof Error ? error.message : String(error);
755
- logger10.error(`Error in test_text_small: ${message}`);
756
- throw error;
1130
+ const text = await runtime.useModel(ModelType7.TEXT_LARGE, {
1131
+ prompt: "Explain quantum computing in 2 sentences."
1132
+ });
1133
+ if (typeof text !== "string" || text.length === 0) {
1134
+ throw new Error("TEXT_LARGE should return non-empty string");
757
1135
  }
1136
+ logger11.info(`[OpenAI Test] TEXT_LARGE generated: "${text.substring(0, 50)}..."`);
758
1137
  }
759
1138
  },
760
1139
  {
761
- name: "openai_test_image_generation",
1140
+ name: "openai_test_tokenizer_roundtrip",
762
1141
  fn: async (runtime) => {
763
- logger10.log("openai_test_image_generation");
764
- try {
765
- const image = await runtime.useModel(ModelType7.IMAGE, {
766
- prompt: "A beautiful sunset over a calm ocean",
767
- count: 1,
768
- size: "1024x1024"
769
- });
770
- logger10.log({ image }, "generated with test_image_generation");
771
- } catch (error) {
772
- const message = error instanceof Error ? error.message : String(error);
773
- logger10.error(`Error in test_image_generation: ${message}`);
774
- throw error;
1142
+ const originalText = "Hello, tokenizer test!";
1143
+ const tokens = await runtime.useModel(ModelType7.TEXT_TOKENIZER_ENCODE, {
1144
+ prompt: originalText,
1145
+ modelType: ModelType7.TEXT_SMALL
1146
+ });
1147
+ if (!Array.isArray(tokens) || tokens.length === 0) {
1148
+ throw new Error("Tokenization should return non-empty token array");
1149
+ }
1150
+ const decodedText = await runtime.useModel(ModelType7.TEXT_TOKENIZER_DECODE, {
1151
+ tokens,
1152
+ modelType: ModelType7.TEXT_SMALL
1153
+ });
1154
+ if (decodedText !== originalText) {
1155
+ throw new Error(`Tokenizer roundtrip failed: expected "${originalText}", got "${decodedText}"`);
775
1156
  }
1157
+ logger11.info(`[OpenAI Test] Tokenizer roundtrip successful (${tokens.length} tokens)`);
776
1158
  }
777
1159
  },
778
1160
  {
779
- name: "image-description",
1161
+ name: "openai_test_streaming",
780
1162
  fn: async (runtime) => {
781
- try {
782
- logger10.log("openai_test_image_description");
783
- try {
784
- const result = await runtime.useModel(ModelType7.IMAGE_DESCRIPTION, "https://upload.wikimedia.org/wikipedia/commons/thumb/1/1c/Vitalik_Buterin_TechCrunch_London_2015_%28cropped%29.jpg/537px-Vitalik_Buterin_TechCrunch_London_2015_%28cropped%29.jpg");
785
- if (result && typeof result === "object" && "title" in result && "description" in result) {
786
- logger10.log({ result }, "Image description");
787
- } else {
788
- logger10.error("Invalid image description result format:", result);
789
- }
790
- } catch (e) {
791
- const message = e instanceof Error ? e.message : String(e);
792
- logger10.error(`Error in image description test: ${message}`);
1163
+ const chunks = [];
1164
+ const result = await runtime.useModel(ModelType7.TEXT_LARGE, {
1165
+ prompt: "Count from 1 to 5, one number per line.",
1166
+ stream: true,
1167
+ onStreamChunk: (chunk) => {
1168
+ chunks.push(chunk);
793
1169
  }
794
- } catch (e) {
795
- const message = e instanceof Error ? e.message : String(e);
796
- logger10.error(`Error in openai_test_image_description: ${message}`);
1170
+ });
1171
+ if (typeof result !== "string" || result.length === 0) {
1172
+ throw new Error("Streaming should return non-empty result");
797
1173
  }
798
- }
799
- },
800
- {
801
- name: "openai_test_transcription",
802
- fn: async (runtime) => {
803
- logger10.log("openai_test_transcription");
804
- try {
805
- const response = await fetch("https://upload.wikimedia.org/wikipedia/en/4/40/Chris_Benoit_Voice_Message.ogg");
806
- const arrayBuffer = await response.arrayBuffer();
807
- const transcription = await runtime.useModel(ModelType7.TRANSCRIPTION, Buffer.from(new Uint8Array(arrayBuffer)));
808
- logger10.log({ transcription }, "generated with test_transcription");
809
- } catch (error) {
810
- const message = error instanceof Error ? error.message : String(error);
811
- logger10.error(`Error in test_transcription: ${message}`);
812
- throw error;
1174
+ if (chunks.length === 0) {
1175
+ throw new Error("No streaming chunks received");
813
1176
  }
1177
+ logger11.info(`[OpenAI Test] Streaming test: ${chunks.length} chunks received`);
814
1178
  }
815
1179
  },
816
1180
  {
817
- name: "openai_test_text_tokenizer_encode",
1181
+ name: "openai_test_image_description",
818
1182
  fn: async (runtime) => {
819
- const prompt = "Hello tokenizer encode!";
820
- const tokens = await runtime.useModel(ModelType7.TEXT_TOKENIZER_ENCODE, { prompt, modelType: ModelType7.TEXT_SMALL });
821
- if (!Array.isArray(tokens) || tokens.length === 0) {
822
- throw new Error("Failed to tokenize text: expected non-empty array of tokens");
1183
+ const testImageUrl = "https://upload.wikimedia.org/wikipedia/commons/thumb/a/a7/Camponotus_flavomarginatus_ant.jpg/440px-Camponotus_flavomarginatus_ant.jpg";
1184
+ const result = await runtime.useModel(ModelType7.IMAGE_DESCRIPTION, testImageUrl);
1185
+ if (!result || typeof result !== "object" || !("title" in result) || !("description" in result)) {
1186
+ throw new Error("Image description should return { title, description }");
823
1187
  }
824
- logger10.log({ tokens }, "Tokenized output");
1188
+ logger11.info(`[OpenAI Test] Image described: "${result.title}"`);
825
1189
  }
826
1190
  },
827
1191
  {
828
- name: "openai_test_text_tokenizer_decode",
1192
+ name: "openai_test_transcription",
829
1193
  fn: async (runtime) => {
830
- const prompt = "Hello tokenizer decode!";
831
- const tokens = await runtime.useModel(ModelType7.TEXT_TOKENIZER_ENCODE, { prompt, modelType: ModelType7.TEXT_SMALL });
832
- const decodedText = await runtime.useModel(ModelType7.TEXT_TOKENIZER_DECODE, {
833
- tokens,
834
- modelType: ModelType7.TEXT_SMALL
835
- });
836
- if (decodedText !== prompt) {
837
- throw new Error(`Decoded text does not match original. Expected "${prompt}", got "${decodedText}"`);
1194
+ const audioUrl = "https://upload.wikimedia.org/wikipedia/commons/2/25/En-Open_Source.ogg";
1195
+ const response = await fetch(audioUrl);
1196
+ const arrayBuffer = await response.arrayBuffer();
1197
+ const audioBuffer = Buffer.from(new Uint8Array(arrayBuffer));
1198
+ const transcription = await runtime.useModel(ModelType7.TRANSCRIPTION, audioBuffer);
1199
+ if (typeof transcription !== "string") {
1200
+ throw new Error("Transcription should return a string");
838
1201
  }
839
- logger10.log({ decodedText }, "Decoded text");
1202
+ logger11.info(`[OpenAI Test] Transcription: "${transcription.substring(0, 50)}..."`);
840
1203
  }
841
1204
  },
842
1205
  {
843
1206
  name: "openai_test_text_to_speech",
844
1207
  fn: async (runtime) => {
845
- try {
846
- const response = await runtime.useModel(ModelType7.TEXT_TO_SPEECH, {
847
- text: "Hello, this is a test for text-to-speech."
848
- });
849
- if (!response) {
850
- throw new Error("Failed to generate speech");
851
- }
852
- logger10.log("Generated speech successfully");
853
- } catch (error) {
854
- const message = error instanceof Error ? error.message : String(error);
855
- logger10.error(`Error in openai_test_text_to_speech: ${message}`);
856
- throw error;
1208
+ const audioData = await runtime.useModel(ModelType7.TEXT_TO_SPEECH, {
1209
+ text: "Hello, this is a text-to-speech test."
1210
+ });
1211
+ if (!(audioData instanceof ArrayBuffer) || audioData.byteLength === 0) {
1212
+ throw new Error("TTS should return non-empty ArrayBuffer");
857
1213
  }
1214
+ logger11.info(`[OpenAI Test] TTS generated ${audioData.byteLength} bytes of audio`);
858
1215
  }
859
1216
  },
860
1217
  {
861
- name: "openai_test_text_generation_large",
1218
+ name: "openai_test_object_generation",
862
1219
  fn: async (runtime) => {
863
- try {
864
- const result = await runtime.useModel(ModelType7.TEXT_LARGE, {
865
- prompt: "Say hello in 5 words."
866
- });
867
- if (!result || result.length === 0) {
868
- throw new Error("Text generation returned empty result");
869
- }
870
- logger10.log({ result }, "Text generation test completed");
871
- } catch (error) {
872
- const message = error instanceof Error ? error.message : String(error);
873
- logger10.error(`Error in openai_test_text_generation_large: ${message}`);
874
- throw error;
1220
+ const result = await runtime.useModel(ModelType7.OBJECT_SMALL, {
1221
+ prompt: "Return a JSON object with exactly these fields: name (string), age (number), active (boolean)"
1222
+ });
1223
+ if (!result || typeof result !== "object") {
1224
+ throw new Error("Object generation should return an object");
875
1225
  }
1226
+ logger11.info(`[OpenAI Test] Object generated: ${JSON.stringify(result).substring(0, 100)}`);
876
1227
  }
877
1228
  },
878
1229
  {
879
- name: "openai_test_streaming",
1230
+ name: "openai_test_research",
880
1231
  fn: async (runtime) => {
881
- try {
882
- const chunks = [];
883
- const result = await runtime.useModel(ModelType7.TEXT_LARGE, {
884
- prompt: "Count from 1 to 5.",
885
- onStreamChunk: (chunk) => {
886
- chunks.push(chunk);
887
- }
888
- });
889
- if (!result || result.length === 0) {
890
- throw new Error("Streaming returned empty result");
891
- }
892
- if (chunks.length === 0) {
893
- throw new Error("No streaming chunks received");
894
- }
895
- logger10.log({ chunks: chunks.length, result: result.substring(0, 50) }, "Streaming test completed");
896
- } catch (error) {
897
- const message = error instanceof Error ? error.message : String(error);
898
- logger10.error(`Error in openai_test_streaming: ${message}`);
899
- throw error;
1232
+ const result = await runtime.useModel(ModelType7.RESEARCH, {
1233
+ input: "What is the current date and time?",
1234
+ tools: [{ type: "web_search_preview" }],
1235
+ maxToolCalls: 3
1236
+ });
1237
+ if (!result || typeof result !== "object" || !("text" in result)) {
1238
+ throw new Error("Research should return an object with text property");
1239
+ }
1240
+ if (typeof result.text !== "string" || result.text.length === 0) {
1241
+ throw new Error("Research result text should be a non-empty string");
900
1242
  }
1243
+ logger11.info(`[OpenAI Test] Research completed. Text length: ${result.text.length}, Annotations: ${result.annotations?.length ?? 0}`);
901
1244
  }
902
1245
  }
903
1246
  ]
904
1247
  }
905
1248
  ]
906
1249
  };
907
- var src_default = openaiPlugin;
1250
+ var typescript_default = openaiPlugin;
908
1251
  export {
909
1252
  openaiPlugin,
910
- src_default as default
1253
+ typescript_default as default
911
1254
  };
912
1255
 
913
- //# debugId=382E411BFB8DEF2564756E2164756E21
1256
+ //# debugId=AE97436E4960EAED64756E2164756E21