@elizaos/plugin-openai 1.6.0 → 2.0.0-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/dist/browser/index.browser.js +2 -2
- package/dist/browser/index.browser.js.map +18 -17
- package/dist/build.d.ts +13 -0
- package/dist/build.d.ts.map +1 -0
- package/dist/cjs/index.node.cjs +998 -658
- package/dist/cjs/index.node.js.map +18 -17
- package/dist/generated/specs/specs.d.ts +55 -0
- package/dist/generated/specs/specs.d.ts.map +1 -0
- package/dist/index.browser.d.ts +1 -0
- package/dist/index.browser.d.ts.map +1 -0
- package/dist/index.d.ts +1 -5
- package/dist/index.d.ts.map +1 -0
- package/dist/index.node.d.ts +1 -0
- package/dist/index.node.d.ts.map +1 -0
- package/dist/init.d.ts +4 -5
- package/dist/init.d.ts.map +1 -0
- package/dist/models/audio.d.ts +9 -10
- package/dist/models/audio.d.ts.map +1 -0
- package/dist/models/embedding.d.ts +1 -3
- package/dist/models/embedding.d.ts.map +1 -0
- package/dist/models/image.d.ts +4 -13
- package/dist/models/image.d.ts.map +1 -0
- package/dist/models/index.d.ts +7 -5
- package/dist/models/index.d.ts.map +1 -0
- package/dist/models/object.d.ts +4 -9
- package/dist/models/object.d.ts.map +1 -0
- package/dist/models/research.d.ts +34 -0
- package/dist/models/research.d.ts.map +1 -0
- package/dist/models/text.d.ts +22 -3
- package/dist/models/text.d.ts.map +1 -0
- package/dist/models/tokenizer.d.ts +4 -9
- package/dist/models/tokenizer.d.ts.map +1 -0
- package/dist/node/index.node.js +987 -644
- package/dist/node/index.node.js.map +18 -17
- package/dist/providers/index.d.ts +2 -1
- package/dist/providers/index.d.ts.map +1 -0
- package/dist/providers/openai.d.ts +3 -7
- package/dist/providers/openai.d.ts.map +1 -0
- package/dist/types/index.d.ts +313 -10
- package/dist/types/index.d.ts.map +1 -0
- package/dist/utils/audio.d.ts +6 -12
- package/dist/utils/audio.d.ts.map +1 -0
- package/dist/utils/config.d.ts +16 -59
- package/dist/utils/config.d.ts.map +1 -0
- package/dist/utils/events.d.ts +14 -9
- package/dist/utils/events.d.ts.map +1 -0
- package/dist/utils/index.d.ts +2 -1
- package/dist/utils/index.d.ts.map +1 -0
- package/dist/utils/json.d.ts +9 -6
- package/dist/utils/json.d.ts.map +1 -0
- package/dist/utils/tokenization.d.ts +5 -16
- package/dist/utils/tokenization.d.ts.map +1 -0
- package/package.json +24 -28
- package/README.md +0 -160
package/dist/node/index.node.js
CHANGED
|
@@ -1,373 +1,162 @@
|
|
|
1
|
-
//
|
|
2
|
-
import { logger as
|
|
1
|
+
// index.ts
|
|
2
|
+
import { logger as logger11, ModelType as ModelType7 } from "@elizaos/core";
|
|
3
3
|
|
|
4
|
-
//
|
|
4
|
+
// init.ts
|
|
5
5
|
import { logger as logger2 } from "@elizaos/core";
|
|
6
6
|
|
|
7
|
-
//
|
|
7
|
+
// utils/config.ts
|
|
8
8
|
import { logger } from "@elizaos/core";
|
|
9
|
+
function getEnvValue(key) {
|
|
10
|
+
if (typeof process === "undefined" || !process.env) {
|
|
11
|
+
return;
|
|
12
|
+
}
|
|
13
|
+
const value = process.env[key];
|
|
14
|
+
return value === undefined ? undefined : String(value);
|
|
15
|
+
}
|
|
9
16
|
function getSetting(runtime, key, defaultValue) {
|
|
10
17
|
const value = runtime.getSetting(key);
|
|
11
18
|
if (value !== undefined && value !== null) {
|
|
12
19
|
return String(value);
|
|
13
20
|
}
|
|
14
|
-
return
|
|
21
|
+
return getEnvValue(key) ?? defaultValue;
|
|
22
|
+
}
|
|
23
|
+
function getNumericSetting(runtime, key, defaultValue) {
|
|
24
|
+
const value = getSetting(runtime, key);
|
|
25
|
+
if (value === undefined) {
|
|
26
|
+
return defaultValue;
|
|
27
|
+
}
|
|
28
|
+
const parsed = Number.parseInt(value, 10);
|
|
29
|
+
if (!Number.isFinite(parsed)) {
|
|
30
|
+
throw new Error(`Setting '${key}' must be a valid integer, got: ${value}`);
|
|
31
|
+
}
|
|
32
|
+
return parsed;
|
|
33
|
+
}
|
|
34
|
+
function getBooleanSetting(runtime, key, defaultValue) {
|
|
35
|
+
const value = getSetting(runtime, key);
|
|
36
|
+
if (value === undefined) {
|
|
37
|
+
return defaultValue;
|
|
38
|
+
}
|
|
39
|
+
const normalized = value.toLowerCase();
|
|
40
|
+
return normalized === "true" || normalized === "1" || normalized === "yes";
|
|
15
41
|
}
|
|
16
42
|
function isBrowser() {
|
|
17
|
-
return typeof globalThis !== "undefined" &&
|
|
43
|
+
return typeof globalThis !== "undefined" && typeof globalThis.document !== "undefined";
|
|
18
44
|
}
|
|
19
45
|
function isProxyMode(runtime) {
|
|
20
46
|
return isBrowser() && !!getSetting(runtime, "OPENAI_BROWSER_BASE_URL");
|
|
21
47
|
}
|
|
48
|
+
function getApiKey(runtime) {
|
|
49
|
+
return getSetting(runtime, "OPENAI_API_KEY");
|
|
50
|
+
}
|
|
51
|
+
function getEmbeddingApiKey(runtime) {
|
|
52
|
+
const embeddingApiKey = getSetting(runtime, "OPENAI_EMBEDDING_API_KEY");
|
|
53
|
+
if (embeddingApiKey) {
|
|
54
|
+
logger.debug("[OpenAI] Using specific embedding API key");
|
|
55
|
+
return embeddingApiKey;
|
|
56
|
+
}
|
|
57
|
+
logger.debug("[OpenAI] Falling back to general API key for embeddings");
|
|
58
|
+
return getApiKey(runtime);
|
|
59
|
+
}
|
|
22
60
|
function getAuthHeader(runtime, forEmbedding = false) {
|
|
23
|
-
if (isBrowser())
|
|
61
|
+
if (isBrowser() && !getBooleanSetting(runtime, "OPENAI_ALLOW_BROWSER_API_KEY", false)) {
|
|
24
62
|
return {};
|
|
63
|
+
}
|
|
25
64
|
const key = forEmbedding ? getEmbeddingApiKey(runtime) : getApiKey(runtime);
|
|
26
65
|
return key ? { Authorization: `Bearer ${key}` } : {};
|
|
27
66
|
}
|
|
28
67
|
function getBaseURL(runtime) {
|
|
29
68
|
const browserURL = getSetting(runtime, "OPENAI_BROWSER_BASE_URL");
|
|
30
|
-
const baseURL = isBrowser() && browserURL ? browserURL : getSetting(runtime, "OPENAI_BASE_URL"
|
|
31
|
-
logger.debug(`[OpenAI]
|
|
69
|
+
const baseURL = isBrowser() && browserURL ? browserURL : getSetting(runtime, "OPENAI_BASE_URL") ?? "https://api.openai.com/v1";
|
|
70
|
+
logger.debug(`[OpenAI] Base URL: ${baseURL}`);
|
|
32
71
|
return baseURL;
|
|
33
72
|
}
|
|
34
73
|
function getEmbeddingBaseURL(runtime) {
|
|
35
|
-
const embeddingURL = isBrowser() ? getSetting(runtime, "OPENAI_BROWSER_EMBEDDING_URL")
|
|
74
|
+
const embeddingURL = isBrowser() ? getSetting(runtime, "OPENAI_BROWSER_EMBEDDING_URL") ?? getSetting(runtime, "OPENAI_BROWSER_BASE_URL") : getSetting(runtime, "OPENAI_EMBEDDING_URL");
|
|
36
75
|
if (embeddingURL) {
|
|
37
|
-
logger.debug(`[OpenAI] Using
|
|
76
|
+
logger.debug(`[OpenAI] Using embedding base URL: ${embeddingURL}`);
|
|
38
77
|
return embeddingURL;
|
|
39
78
|
}
|
|
40
|
-
logger.debug("[OpenAI] Falling back to general base URL for embeddings
|
|
79
|
+
logger.debug("[OpenAI] Falling back to general base URL for embeddings");
|
|
41
80
|
return getBaseURL(runtime);
|
|
42
81
|
}
|
|
43
|
-
function getApiKey(runtime) {
|
|
44
|
-
return getSetting(runtime, "OPENAI_API_KEY");
|
|
45
|
-
}
|
|
46
|
-
function getEmbeddingApiKey(runtime) {
|
|
47
|
-
const embeddingApiKey = getSetting(runtime, "OPENAI_EMBEDDING_API_KEY");
|
|
48
|
-
if (embeddingApiKey) {
|
|
49
|
-
logger.debug("[OpenAI] Using specific embedding API key (present)");
|
|
50
|
-
return embeddingApiKey;
|
|
51
|
-
}
|
|
52
|
-
logger.debug("[OpenAI] Falling back to general API key for embeddings.");
|
|
53
|
-
return getApiKey(runtime);
|
|
54
|
-
}
|
|
55
82
|
function getSmallModel(runtime) {
|
|
56
|
-
return getSetting(runtime, "OPENAI_SMALL_MODEL") ?? getSetting(runtime, "SMALL_MODEL"
|
|
83
|
+
return getSetting(runtime, "OPENAI_SMALL_MODEL") ?? getSetting(runtime, "SMALL_MODEL") ?? "gpt-5-mini";
|
|
57
84
|
}
|
|
58
85
|
function getLargeModel(runtime) {
|
|
59
|
-
return getSetting(runtime, "OPENAI_LARGE_MODEL") ?? getSetting(runtime, "LARGE_MODEL"
|
|
86
|
+
return getSetting(runtime, "OPENAI_LARGE_MODEL") ?? getSetting(runtime, "LARGE_MODEL") ?? "gpt-5";
|
|
87
|
+
}
|
|
88
|
+
function getEmbeddingModel(runtime) {
|
|
89
|
+
return getSetting(runtime, "OPENAI_EMBEDDING_MODEL") ?? "text-embedding-3-small";
|
|
60
90
|
}
|
|
61
91
|
function getImageDescriptionModel(runtime) {
|
|
62
|
-
return getSetting(runtime, "OPENAI_IMAGE_DESCRIPTION_MODEL"
|
|
92
|
+
return getSetting(runtime, "OPENAI_IMAGE_DESCRIPTION_MODEL") ?? "gpt-5-mini";
|
|
63
93
|
}
|
|
64
|
-
function
|
|
65
|
-
|
|
66
|
-
const normalizedSetting = String(setting).toLowerCase();
|
|
67
|
-
const result = normalizedSetting === "true";
|
|
68
|
-
logger.debug(`[OpenAI] Experimental telemetry in function: "${setting}" (type: ${typeof setting}, normalized: "${normalizedSetting}", result: ${result})`);
|
|
69
|
-
return result;
|
|
94
|
+
function getTranscriptionModel(runtime) {
|
|
95
|
+
return getSetting(runtime, "OPENAI_TRANSCRIPTION_MODEL") ?? "gpt-5-mini-transcribe";
|
|
70
96
|
}
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
function initializeOpenAI(_config, runtime) {
|
|
74
|
-
(async () => {
|
|
75
|
-
try {
|
|
76
|
-
if (!getApiKey(runtime) && !isBrowser()) {
|
|
77
|
-
logger2.warn("OPENAI_API_KEY is not set in environment - OpenAI functionality will be limited");
|
|
78
|
-
return;
|
|
79
|
-
}
|
|
80
|
-
try {
|
|
81
|
-
const baseURL = getBaseURL(runtime);
|
|
82
|
-
const response = await fetch(`${baseURL}/models`, {
|
|
83
|
-
headers: getAuthHeader(runtime)
|
|
84
|
-
});
|
|
85
|
-
if (!response.ok) {
|
|
86
|
-
logger2.warn(`OpenAI API key validation failed: ${response.statusText}`);
|
|
87
|
-
logger2.warn("OpenAI functionality will be limited until a valid API key is provided");
|
|
88
|
-
} else {
|
|
89
|
-
logger2.log("OpenAI API key validated successfully");
|
|
90
|
-
}
|
|
91
|
-
} catch (fetchError) {
|
|
92
|
-
const message = fetchError instanceof Error ? fetchError.message : String(fetchError);
|
|
93
|
-
logger2.warn(`Error validating OpenAI API key: ${message}`);
|
|
94
|
-
logger2.warn("OpenAI functionality will be limited until a valid API key is provided");
|
|
95
|
-
}
|
|
96
|
-
} catch (error) {
|
|
97
|
-
const message = error?.errors?.map((e) => e.message).join(", ") || (error instanceof Error ? error.message : String(error));
|
|
98
|
-
logger2.warn(`OpenAI plugin configuration issue: ${message} - You need to configure the OPENAI_API_KEY in your environment variables`);
|
|
99
|
-
}
|
|
100
|
-
})();
|
|
97
|
+
function getTTSModel(runtime) {
|
|
98
|
+
return getSetting(runtime, "OPENAI_TTS_MODEL") ?? "tts-1";
|
|
101
99
|
}
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
import { logger as logger3, ModelType } from "@elizaos/core";
|
|
105
|
-
import { generateText, streamText } from "ai";
|
|
106
|
-
|
|
107
|
-
// src/providers/openai.ts
|
|
108
|
-
import { createOpenAI } from "@ai-sdk/openai";
|
|
109
|
-
function createOpenAIClient(runtime) {
|
|
110
|
-
const baseURL = getBaseURL(runtime);
|
|
111
|
-
const apiKey = getApiKey(runtime) ?? (isProxyMode(runtime) ? "sk-proxy" : undefined);
|
|
112
|
-
return createOpenAI({ apiKey: apiKey ?? "", baseURL });
|
|
100
|
+
function getTTSVoice(runtime) {
|
|
101
|
+
return getSetting(runtime, "OPENAI_TTS_VOICE") ?? "nova";
|
|
113
102
|
}
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
import { EventType } from "@elizaos/core";
|
|
117
|
-
function emitModelUsageEvent(runtime, type, prompt, usage) {
|
|
118
|
-
const promptTokens = ("promptTokens" in usage ? usage.promptTokens : undefined) ?? ("inputTokens" in usage ? usage.inputTokens : undefined) ?? 0;
|
|
119
|
-
const completionTokens = ("completionTokens" in usage ? usage.completionTokens : undefined) ?? ("outputTokens" in usage ? usage.outputTokens : undefined) ?? 0;
|
|
120
|
-
const totalTokens = ("totalTokens" in usage ? usage.totalTokens : undefined) ?? promptTokens + completionTokens;
|
|
121
|
-
const truncatedPrompt = typeof prompt === "string" ? prompt.length > 200 ? `${prompt.slice(0, 200)}…` : prompt : "";
|
|
122
|
-
runtime.emitEvent(EventType.MODEL_USED, {
|
|
123
|
-
runtime,
|
|
124
|
-
source: "openai",
|
|
125
|
-
provider: "openai",
|
|
126
|
-
type,
|
|
127
|
-
prompt: truncatedPrompt,
|
|
128
|
-
tokens: {
|
|
129
|
-
prompt: promptTokens,
|
|
130
|
-
completion: completionTokens,
|
|
131
|
-
total: totalTokens
|
|
132
|
-
}
|
|
133
|
-
});
|
|
103
|
+
function getTTSInstructions(runtime) {
|
|
104
|
+
return getSetting(runtime, "OPENAI_TTS_INSTRUCTIONS") ?? "";
|
|
134
105
|
}
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
async function generateTextByModelType(runtime, params, modelType, getModelFn) {
|
|
138
|
-
const openai = createOpenAIClient(runtime);
|
|
139
|
-
const modelName = getModelFn(runtime);
|
|
140
|
-
logger3.debug(`[OpenAI] ${modelType} model: ${modelName}`);
|
|
141
|
-
const generateParams = {
|
|
142
|
-
model: openai.languageModel(modelName),
|
|
143
|
-
prompt: params.prompt,
|
|
144
|
-
system: runtime.character.system ?? undefined,
|
|
145
|
-
temperature: params.temperature ?? 0.7,
|
|
146
|
-
maxOutputTokens: params.maxTokens ?? 8192,
|
|
147
|
-
frequencyPenalty: params.frequencyPenalty ?? 0.7,
|
|
148
|
-
presencePenalty: params.presencePenalty ?? 0.7,
|
|
149
|
-
stopSequences: params.stopSequences ?? [],
|
|
150
|
-
experimental_telemetry: { isEnabled: getExperimentalTelemetry(runtime) }
|
|
151
|
-
};
|
|
152
|
-
if (params.stream) {
|
|
153
|
-
const result = streamText(generateParams);
|
|
154
|
-
return {
|
|
155
|
-
textStream: result.textStream,
|
|
156
|
-
text: result.text,
|
|
157
|
-
usage: result.usage.then((u) => u ? {
|
|
158
|
-
promptTokens: u.inputTokens ?? 0,
|
|
159
|
-
completionTokens: u.outputTokens ?? 0,
|
|
160
|
-
totalTokens: (u.inputTokens ?? 0) + (u.outputTokens ?? 0)
|
|
161
|
-
} : undefined),
|
|
162
|
-
finishReason: result.finishReason
|
|
163
|
-
};
|
|
164
|
-
}
|
|
165
|
-
const { text, usage } = await generateText(generateParams);
|
|
166
|
-
if (usage)
|
|
167
|
-
emitModelUsageEvent(runtime, modelType, params.prompt, usage);
|
|
168
|
-
return text;
|
|
106
|
+
function getImageModel(runtime) {
|
|
107
|
+
return getSetting(runtime, "OPENAI_IMAGE_MODEL") ?? "dall-e-3";
|
|
169
108
|
}
|
|
170
|
-
|
|
171
|
-
return
|
|
109
|
+
function getExperimentalTelemetry(runtime) {
|
|
110
|
+
return getBooleanSetting(runtime, "OPENAI_EXPERIMENTAL_TELEMETRY", false);
|
|
172
111
|
}
|
|
173
|
-
|
|
174
|
-
return
|
|
112
|
+
function getEmbeddingDimensions(runtime) {
|
|
113
|
+
return getNumericSetting(runtime, "OPENAI_EMBEDDING_DIMENSIONS", 1536);
|
|
175
114
|
}
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
async function handleTextEmbedding(runtime, params) {
|
|
179
|
-
const embeddingModelName = getSetting(runtime, "OPENAI_EMBEDDING_MODEL", "text-embedding-3-small");
|
|
180
|
-
const embeddingDimension = Number.parseInt(getSetting(runtime, "OPENAI_EMBEDDING_DIMENSIONS", "1536") || "1536", 10);
|
|
181
|
-
if (!Object.values(VECTOR_DIMS).includes(embeddingDimension)) {
|
|
182
|
-
const errorMsg = `Invalid embedding dimension: ${embeddingDimension}. Must be one of: ${Object.values(VECTOR_DIMS).join(", ")}`;
|
|
183
|
-
logger4.error(errorMsg);
|
|
184
|
-
throw new Error(errorMsg);
|
|
185
|
-
}
|
|
186
|
-
if (params === null) {
|
|
187
|
-
logger4.debug("Creating test embedding for initialization");
|
|
188
|
-
const testVector = Array(embeddingDimension).fill(0);
|
|
189
|
-
testVector[0] = 0.1;
|
|
190
|
-
return testVector;
|
|
191
|
-
}
|
|
192
|
-
let text;
|
|
193
|
-
if (typeof params === "string") {
|
|
194
|
-
text = params;
|
|
195
|
-
} else if (typeof params === "object" && params.text) {
|
|
196
|
-
text = params.text;
|
|
197
|
-
} else {
|
|
198
|
-
const errorMsg = "Invalid input format for embedding";
|
|
199
|
-
logger4.warn(errorMsg);
|
|
200
|
-
const fallbackVector = Array(embeddingDimension).fill(0);
|
|
201
|
-
fallbackVector[0] = 0.2;
|
|
202
|
-
return fallbackVector;
|
|
203
|
-
}
|
|
204
|
-
if (!text.trim()) {
|
|
205
|
-
const errorMsg = "Empty text for embedding";
|
|
206
|
-
logger4.warn(errorMsg);
|
|
207
|
-
const fallbackVector = Array(embeddingDimension).fill(0);
|
|
208
|
-
fallbackVector[0] = 0.3;
|
|
209
|
-
return fallbackVector;
|
|
210
|
-
}
|
|
211
|
-
const embeddingBaseURL = getEmbeddingBaseURL(runtime);
|
|
212
|
-
try {
|
|
213
|
-
const response = await fetch(`${embeddingBaseURL}/embeddings`, {
|
|
214
|
-
method: "POST",
|
|
215
|
-
headers: {
|
|
216
|
-
...getAuthHeader(runtime, true),
|
|
217
|
-
"Content-Type": "application/json"
|
|
218
|
-
},
|
|
219
|
-
body: JSON.stringify({
|
|
220
|
-
model: embeddingModelName,
|
|
221
|
-
input: text
|
|
222
|
-
})
|
|
223
|
-
});
|
|
224
|
-
if (!response.ok) {
|
|
225
|
-
logger4.error(`OpenAI API error: ${response.status} - ${response.statusText}`);
|
|
226
|
-
throw new Error(`OpenAI API error: ${response.status} - ${response.statusText}`);
|
|
227
|
-
}
|
|
228
|
-
const data = await response.json();
|
|
229
|
-
if (!data?.data?.[0]?.embedding) {
|
|
230
|
-
logger4.error("API returned invalid structure");
|
|
231
|
-
throw new Error("API returned invalid structure");
|
|
232
|
-
}
|
|
233
|
-
const embedding = data.data[0].embedding;
|
|
234
|
-
if (!Array.isArray(embedding) || embedding.length !== embeddingDimension) {
|
|
235
|
-
const errorMsg = `Embedding length ${embedding?.length ?? 0} does not match configured dimension ${embeddingDimension}`;
|
|
236
|
-
logger4.error(errorMsg);
|
|
237
|
-
const fallbackVector = Array(embeddingDimension).fill(0);
|
|
238
|
-
fallbackVector[0] = 0.4;
|
|
239
|
-
return fallbackVector;
|
|
240
|
-
}
|
|
241
|
-
if (data.usage) {
|
|
242
|
-
const usage = {
|
|
243
|
-
inputTokens: data.usage.prompt_tokens,
|
|
244
|
-
outputTokens: 0,
|
|
245
|
-
totalTokens: data.usage.total_tokens
|
|
246
|
-
};
|
|
247
|
-
emitModelUsageEvent(runtime, ModelType2.TEXT_EMBEDDING, text, usage);
|
|
248
|
-
}
|
|
249
|
-
logger4.log(`Got valid embedding with length ${embedding.length}`);
|
|
250
|
-
return embedding;
|
|
251
|
-
} catch (error) {
|
|
252
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
253
|
-
logger4.error(`Error generating embedding: ${message}`);
|
|
254
|
-
throw error instanceof Error ? error : new Error(message);
|
|
255
|
-
}
|
|
115
|
+
function getImageDescriptionMaxTokens(runtime) {
|
|
116
|
+
return getNumericSetting(runtime, "OPENAI_IMAGE_DESCRIPTION_MAX_TOKENS", 8192);
|
|
256
117
|
}
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
async function handleImageGeneration(runtime, params) {
|
|
260
|
-
const n = params.count || 1;
|
|
261
|
-
const size = params.size || "1024x1024";
|
|
262
|
-
const prompt = params.prompt;
|
|
263
|
-
const modelName = getSetting(runtime, "OPENAI_IMAGE_MODEL", "gpt-image-1");
|
|
264
|
-
logger5.log(`[OpenAI] Using IMAGE model: ${modelName}`);
|
|
265
|
-
const baseURL = getBaseURL(runtime);
|
|
266
|
-
try {
|
|
267
|
-
const response = await fetch(`${baseURL}/images/generations`, {
|
|
268
|
-
method: "POST",
|
|
269
|
-
headers: {
|
|
270
|
-
...getAuthHeader(runtime),
|
|
271
|
-
"Content-Type": "application/json"
|
|
272
|
-
},
|
|
273
|
-
body: JSON.stringify({
|
|
274
|
-
model: modelName,
|
|
275
|
-
prompt,
|
|
276
|
-
n,
|
|
277
|
-
size
|
|
278
|
-
})
|
|
279
|
-
});
|
|
280
|
-
if (!response.ok) {
|
|
281
|
-
throw new Error(`Failed to generate image: ${response.statusText}`);
|
|
282
|
-
}
|
|
283
|
-
const data = await response.json();
|
|
284
|
-
const typedData = data;
|
|
285
|
-
return typedData.data;
|
|
286
|
-
} catch (error) {
|
|
287
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
288
|
-
throw error;
|
|
289
|
-
}
|
|
118
|
+
function getResearchModel(runtime) {
|
|
119
|
+
return getSetting(runtime, "OPENAI_RESEARCH_MODEL") ?? "o3-deep-research";
|
|
290
120
|
}
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
121
|
+
function getResearchTimeout(runtime) {
|
|
122
|
+
return getNumericSetting(runtime, "OPENAI_RESEARCH_TIMEOUT", 3600000);
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
// init.ts
|
|
126
|
+
globalThis.AI_SDK_LOG_WARNINGS ??= false;
|
|
127
|
+
function initializeOpenAI(_config, runtime) {
|
|
128
|
+
validateOpenAIConfiguration(runtime);
|
|
129
|
+
}
|
|
130
|
+
async function validateOpenAIConfiguration(runtime) {
|
|
131
|
+
if (isBrowser()) {
|
|
132
|
+
logger2.debug("[OpenAI] Skipping API validation in browser environment");
|
|
133
|
+
return;
|
|
134
|
+
}
|
|
135
|
+
const apiKey = getApiKey(runtime);
|
|
136
|
+
if (!apiKey) {
|
|
137
|
+
logger2.warn("[OpenAI] OPENAI_API_KEY is not configured. " + "OpenAI functionality will fail until a valid API key is provided.");
|
|
138
|
+
return;
|
|
304
139
|
}
|
|
305
|
-
const messages = [
|
|
306
|
-
{
|
|
307
|
-
role: "user",
|
|
308
|
-
content: [
|
|
309
|
-
{ type: "text", text: promptText },
|
|
310
|
-
{ type: "image_url", image_url: { url: imageUrl } }
|
|
311
|
-
]
|
|
312
|
-
}
|
|
313
|
-
];
|
|
314
|
-
const baseURL = getBaseURL(runtime);
|
|
315
140
|
try {
|
|
316
|
-
const
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
max_tokens: maxTokens
|
|
320
|
-
};
|
|
321
|
-
const response = await fetch(`${baseURL}/chat/completions`, {
|
|
322
|
-
method: "POST",
|
|
323
|
-
headers: {
|
|
324
|
-
"Content-Type": "application/json",
|
|
325
|
-
...getAuthHeader(runtime)
|
|
326
|
-
},
|
|
327
|
-
body: JSON.stringify(requestBody)
|
|
141
|
+
const baseURL = getBaseURL(runtime);
|
|
142
|
+
const response = await fetch(`${baseURL}/models`, {
|
|
143
|
+
headers: getAuthHeader(runtime)
|
|
328
144
|
});
|
|
329
145
|
if (!response.ok) {
|
|
330
|
-
|
|
146
|
+
logger2.warn(`[OpenAI] API key validation failed: ${response.status} ${response.statusText}. ` + "Please verify your OPENAI_API_KEY is correct.");
|
|
147
|
+
return;
|
|
331
148
|
}
|
|
332
|
-
const result = await response.json();
|
|
333
|
-
const typedResult = result;
|
|
334
|
-
const content = typedResult.choices?.[0]?.message?.content;
|
|
335
|
-
if (typedResult.usage) {
|
|
336
|
-
emitModelUsageEvent(runtime, ModelType3.IMAGE_DESCRIPTION, typeof params === "string" ? params : params.prompt || "", {
|
|
337
|
-
inputTokens: typedResult.usage.prompt_tokens,
|
|
338
|
-
outputTokens: typedResult.usage.completion_tokens,
|
|
339
|
-
totalTokens: typedResult.usage.total_tokens
|
|
340
|
-
});
|
|
341
|
-
}
|
|
342
|
-
if (!content) {
|
|
343
|
-
return {
|
|
344
|
-
title: "Failed to analyze image",
|
|
345
|
-
description: "No response from API"
|
|
346
|
-
};
|
|
347
|
-
}
|
|
348
|
-
const titleMatch = content.match(/title[:\s]+(.+?)(?:\n|$)/i);
|
|
349
|
-
const title = titleMatch?.[1]?.trim();
|
|
350
|
-
if (!title) {
|
|
351
|
-
logger5.warn("Could not extract title from image description response");
|
|
352
|
-
}
|
|
353
|
-
const finalTitle = title || "Image Analysis";
|
|
354
|
-
const description = content.replace(/title[:\s]+(.+?)(?:\n|$)/i, "").trim();
|
|
355
|
-
const processedResult = { title: finalTitle, description };
|
|
356
|
-
return processedResult;
|
|
357
149
|
} catch (error) {
|
|
358
150
|
const message = error instanceof Error ? error.message : String(error);
|
|
359
|
-
|
|
360
|
-
return {
|
|
361
|
-
title: "Failed to analyze image",
|
|
362
|
-
description: `Error: ${message}`
|
|
363
|
-
};
|
|
151
|
+
logger2.warn(`[OpenAI] API validation error: ${message}. OpenAI functionality may be limited.`);
|
|
364
152
|
}
|
|
365
153
|
}
|
|
366
|
-
// src/models/audio.ts
|
|
367
|
-
import { logger as logger7 } from "@elizaos/core";
|
|
368
154
|
|
|
369
|
-
//
|
|
370
|
-
import { logger as
|
|
155
|
+
// models/audio.ts
|
|
156
|
+
import { logger as logger4 } from "@elizaos/core";
|
|
157
|
+
|
|
158
|
+
// utils/audio.ts
|
|
159
|
+
import { logger as logger3 } from "@elizaos/core";
|
|
371
160
|
var MAGIC_BYTES = {
|
|
372
161
|
WAV: {
|
|
373
162
|
HEADER: [82, 73, 70, 70],
|
|
@@ -379,21 +168,26 @@ var MAGIC_BYTES = {
|
|
|
379
168
|
FTYP: [102, 116, 121, 112],
|
|
380
169
|
WEBM_EBML: [26, 69, 223, 163]
|
|
381
170
|
};
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
171
|
+
var MIN_DETECTION_BUFFER_SIZE = 12;
|
|
172
|
+
function matchBytes(buffer, offset, expected) {
|
|
173
|
+
for (let i = 0;i < expected.length; i++) {
|
|
174
|
+
const expectedByte = expected[i];
|
|
175
|
+
if (expectedByte === undefined || buffer[offset + i] !== expectedByte) {
|
|
385
176
|
return false;
|
|
177
|
+
}
|
|
386
178
|
}
|
|
387
179
|
return true;
|
|
388
180
|
}
|
|
389
181
|
function detectAudioMimeType(buffer) {
|
|
390
|
-
if (buffer.length <
|
|
182
|
+
if (buffer.length < MIN_DETECTION_BUFFER_SIZE) {
|
|
391
183
|
return "application/octet-stream";
|
|
392
184
|
}
|
|
393
185
|
if (matchBytes(buffer, 0, MAGIC_BYTES.WAV.HEADER) && matchBytes(buffer, 8, MAGIC_BYTES.WAV.IDENTIFIER)) {
|
|
394
186
|
return "audio/wav";
|
|
395
187
|
}
|
|
396
|
-
|
|
188
|
+
const firstByte = buffer[0];
|
|
189
|
+
const secondByte = buffer[1];
|
|
190
|
+
if (matchBytes(buffer, 0, MAGIC_BYTES.MP3_ID3) || firstByte === 255 && secondByte !== undefined && (secondByte & 224) === 224) {
|
|
397
191
|
return "audio/mpeg";
|
|
398
192
|
}
|
|
399
193
|
if (matchBytes(buffer, 0, MAGIC_BYTES.OGG)) {
|
|
@@ -408,250 +202,847 @@ function detectAudioMimeType(buffer) {
|
|
|
408
202
|
if (matchBytes(buffer, 0, MAGIC_BYTES.WEBM_EBML)) {
|
|
409
203
|
return "audio/webm";
|
|
410
204
|
}
|
|
411
|
-
|
|
205
|
+
logger3.warn("Could not detect audio format from buffer, using generic binary type");
|
|
412
206
|
return "application/octet-stream";
|
|
413
207
|
}
|
|
208
|
+
function getExtensionForMimeType(mimeType) {
|
|
209
|
+
switch (mimeType) {
|
|
210
|
+
case "audio/wav":
|
|
211
|
+
return "wav";
|
|
212
|
+
case "audio/mpeg":
|
|
213
|
+
return "mp3";
|
|
214
|
+
case "audio/ogg":
|
|
215
|
+
return "ogg";
|
|
216
|
+
case "audio/flac":
|
|
217
|
+
return "flac";
|
|
218
|
+
case "audio/mp4":
|
|
219
|
+
return "m4a";
|
|
220
|
+
case "audio/webm":
|
|
221
|
+
return "webm";
|
|
222
|
+
case "application/octet-stream":
|
|
223
|
+
return "bin";
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
function getFilenameForMimeType(mimeType) {
|
|
227
|
+
const ext = getExtensionForMimeType(mimeType);
|
|
228
|
+
return `recording.${ext}`;
|
|
229
|
+
}
|
|
414
230
|
|
|
415
|
-
//
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
},
|
|
433
|
-
body: JSON.stringify({
|
|
434
|
-
model,
|
|
435
|
-
voice,
|
|
436
|
-
input: options.text,
|
|
437
|
-
format,
|
|
438
|
-
...instructions && { instructions }
|
|
439
|
-
})
|
|
440
|
-
});
|
|
441
|
-
if (!res.ok) {
|
|
442
|
-
const err = await res.text();
|
|
443
|
-
throw new Error(`OpenAI TTS error ${res.status}: ${err}`);
|
|
444
|
-
}
|
|
445
|
-
return await res.arrayBuffer();
|
|
446
|
-
} catch (err) {
|
|
447
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
448
|
-
throw new Error(`Failed to fetch speech from OpenAI TTS: ${message}`);
|
|
231
|
+
// models/audio.ts
|
|
232
|
+
function isBlobOrFile(value) {
|
|
233
|
+
return value instanceof Blob || value instanceof File;
|
|
234
|
+
}
|
|
235
|
+
function isBuffer(value) {
|
|
236
|
+
return Buffer.isBuffer(value);
|
|
237
|
+
}
|
|
238
|
+
function isLocalTranscriptionParams(value) {
|
|
239
|
+
return typeof value === "object" && value !== null && "audio" in value && (isBlobOrFile(value.audio) || isBuffer(value.audio));
|
|
240
|
+
}
|
|
241
|
+
function isCoreTranscriptionParams(value) {
|
|
242
|
+
return typeof value === "object" && value !== null && "audioUrl" in value && typeof value.audioUrl === "string";
|
|
243
|
+
}
|
|
244
|
+
async function fetchAudioFromUrl(url) {
|
|
245
|
+
const response = await fetch(url);
|
|
246
|
+
if (!response.ok) {
|
|
247
|
+
throw new Error(`Failed to fetch audio from URL: ${response.status}`);
|
|
449
248
|
}
|
|
249
|
+
return response.blob();
|
|
450
250
|
}
|
|
451
251
|
async function handleTranscription(runtime, input) {
|
|
452
|
-
let modelName =
|
|
453
|
-
logger7.log(`[OpenAI] Using TRANSCRIPTION model: ${modelName}`);
|
|
454
|
-
const baseURL = getBaseURL(runtime);
|
|
252
|
+
let modelName = getTranscriptionModel(runtime);
|
|
455
253
|
let blob;
|
|
456
|
-
let extraParams =
|
|
457
|
-
if (
|
|
254
|
+
let extraParams = {};
|
|
255
|
+
if (typeof input === "string") {
|
|
256
|
+
logger4.debug(`[OpenAI] Fetching audio from URL: ${input}`);
|
|
257
|
+
blob = await fetchAudioFromUrl(input);
|
|
258
|
+
} else if (isBlobOrFile(input)) {
|
|
458
259
|
blob = input;
|
|
459
|
-
} else if (
|
|
460
|
-
const
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
throw new Error("TRANSCRIPTION param 'audio' must be a Blob/File/Buffer.");
|
|
260
|
+
} else if (isBuffer(input)) {
|
|
261
|
+
const mimeType2 = detectAudioMimeType(input);
|
|
262
|
+
logger4.debug(`[OpenAI] Auto-detected audio MIME type: ${mimeType2}`);
|
|
263
|
+
blob = new Blob([new Uint8Array(input)], { type: mimeType2 });
|
|
264
|
+
} else if (isLocalTranscriptionParams(input)) {
|
|
265
|
+
extraParams = input;
|
|
266
|
+
if (input.model) {
|
|
267
|
+
modelName = input.model;
|
|
468
268
|
}
|
|
469
|
-
if (
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
logger7.debug(`Auto-detected audio MIME type: ${mimeType}`);
|
|
474
|
-
} else {
|
|
475
|
-
logger7.debug(`Using provided MIME type: ${mimeType}`);
|
|
476
|
-
}
|
|
477
|
-
const uint8Array = new Uint8Array(params.audio);
|
|
478
|
-
blob = new Blob([uint8Array], { type: mimeType });
|
|
269
|
+
if (isBuffer(input.audio)) {
|
|
270
|
+
const mimeType2 = input.mimeType ?? detectAudioMimeType(input.audio);
|
|
271
|
+
logger4.debug(`[OpenAI] Using MIME type: ${mimeType2}`);
|
|
272
|
+
blob = new Blob([new Uint8Array(input.audio)], { type: mimeType2 });
|
|
479
273
|
} else {
|
|
480
|
-
blob =
|
|
481
|
-
}
|
|
482
|
-
extraParams = params;
|
|
483
|
-
if (typeof params.model === "string" && params.model) {
|
|
484
|
-
modelName = params.model;
|
|
274
|
+
blob = input.audio;
|
|
485
275
|
}
|
|
276
|
+
} else if (isCoreTranscriptionParams(input)) {
|
|
277
|
+
logger4.debug(`[OpenAI] Fetching audio from URL: ${input.audioUrl}`);
|
|
278
|
+
blob = await fetchAudioFromUrl(input.audioUrl);
|
|
279
|
+
extraParams = { prompt: input.prompt };
|
|
486
280
|
} else {
|
|
487
|
-
throw new Error("TRANSCRIPTION expects
|
|
281
|
+
throw new Error("TRANSCRIPTION expects Blob, File, Buffer, URL string, or TranscriptionParams object");
|
|
488
282
|
}
|
|
489
|
-
|
|
490
|
-
const
|
|
283
|
+
logger4.debug(`[OpenAI] Using TRANSCRIPTION model: ${modelName}`);
|
|
284
|
+
const mimeType = blob.type || "audio/webm";
|
|
285
|
+
const filename = blob.name || getFilenameForMimeType(mimeType.startsWith("audio/") ? mimeType : "audio/webm");
|
|
491
286
|
const formData = new FormData;
|
|
492
287
|
formData.append("file", blob, filename);
|
|
493
|
-
formData.append("model",
|
|
494
|
-
if (extraParams) {
|
|
495
|
-
|
|
496
|
-
|
|
288
|
+
formData.append("model", modelName);
|
|
289
|
+
if (extraParams.language) {
|
|
290
|
+
formData.append("language", extraParams.language);
|
|
291
|
+
}
|
|
292
|
+
if (extraParams.responseFormat) {
|
|
293
|
+
formData.append("response_format", extraParams.responseFormat);
|
|
294
|
+
}
|
|
295
|
+
if (extraParams.prompt) {
|
|
296
|
+
formData.append("prompt", extraParams.prompt);
|
|
297
|
+
}
|
|
298
|
+
if (extraParams.temperature !== undefined) {
|
|
299
|
+
formData.append("temperature", String(extraParams.temperature));
|
|
300
|
+
}
|
|
301
|
+
if (extraParams.timestampGranularities) {
|
|
302
|
+
for (const granularity of extraParams.timestampGranularities) {
|
|
303
|
+
formData.append("timestamp_granularities[]", granularity);
|
|
497
304
|
}
|
|
498
|
-
|
|
499
|
-
|
|
305
|
+
}
|
|
306
|
+
const baseURL = getBaseURL(runtime);
|
|
307
|
+
const response = await fetch(`${baseURL}/audio/transcriptions`, {
|
|
308
|
+
method: "POST",
|
|
309
|
+
headers: getAuthHeader(runtime),
|
|
310
|
+
body: formData
|
|
311
|
+
});
|
|
312
|
+
if (!response.ok) {
|
|
313
|
+
const errorText = await response.text().catch(() => "Unknown error");
|
|
314
|
+
throw new Error(`OpenAI transcription failed: ${response.status} ${response.statusText} - ${errorText}`);
|
|
315
|
+
}
|
|
316
|
+
const data = await response.json();
|
|
317
|
+
return data.text;
|
|
318
|
+
}
|
|
319
|
+
async function handleTextToSpeech(runtime, input) {
|
|
320
|
+
let text;
|
|
321
|
+
let voice;
|
|
322
|
+
let format = "mp3";
|
|
323
|
+
let model;
|
|
324
|
+
let instructions;
|
|
325
|
+
if (typeof input === "string") {
|
|
326
|
+
text = input;
|
|
327
|
+
voice = undefined;
|
|
328
|
+
} else {
|
|
329
|
+
text = input.text;
|
|
330
|
+
voice = input.voice;
|
|
331
|
+
if ("format" in input && input.format) {
|
|
332
|
+
format = input.format;
|
|
500
333
|
}
|
|
501
|
-
if (
|
|
502
|
-
|
|
334
|
+
if ("model" in input && input.model) {
|
|
335
|
+
model = input.model;
|
|
503
336
|
}
|
|
504
|
-
if (
|
|
505
|
-
|
|
337
|
+
if ("instructions" in input && input.instructions) {
|
|
338
|
+
instructions = input.instructions;
|
|
506
339
|
}
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
340
|
+
}
|
|
341
|
+
model = model ?? getTTSModel(runtime);
|
|
342
|
+
voice = voice ?? getTTSVoice(runtime);
|
|
343
|
+
instructions = instructions ?? getTTSInstructions(runtime);
|
|
344
|
+
logger4.debug(`[OpenAI] Using TEXT_TO_SPEECH model: ${model}`);
|
|
345
|
+
if (!text || text.trim().length === 0) {
|
|
346
|
+
throw new Error("TEXT_TO_SPEECH requires non-empty text");
|
|
347
|
+
}
|
|
348
|
+
if (text.length > 4096) {
|
|
349
|
+
throw new Error("TEXT_TO_SPEECH text exceeds 4096 character limit");
|
|
350
|
+
}
|
|
351
|
+
const validVoices = ["alloy", "echo", "fable", "onyx", "nova", "shimmer"];
|
|
352
|
+
if (voice && !validVoices.includes(voice)) {
|
|
353
|
+
throw new Error(`Invalid voice: ${voice}. Must be one of: ${validVoices.join(", ")}`);
|
|
354
|
+
}
|
|
355
|
+
const baseURL = getBaseURL(runtime);
|
|
356
|
+
const requestBody = {
|
|
357
|
+
model,
|
|
358
|
+
voice,
|
|
359
|
+
input: text,
|
|
360
|
+
response_format: format
|
|
361
|
+
};
|
|
362
|
+
if (instructions && instructions.length > 0) {
|
|
363
|
+
requestBody.instructions = instructions;
|
|
364
|
+
}
|
|
365
|
+
const response = await fetch(`${baseURL}/audio/speech`, {
|
|
366
|
+
method: "POST",
|
|
367
|
+
headers: {
|
|
368
|
+
...getAuthHeader(runtime),
|
|
369
|
+
"Content-Type": "application/json",
|
|
370
|
+
...format === "mp3" ? { Accept: "audio/mpeg" } : {}
|
|
371
|
+
},
|
|
372
|
+
body: JSON.stringify(requestBody)
|
|
373
|
+
});
|
|
374
|
+
if (!response.ok) {
|
|
375
|
+
const errorText = await response.text().catch(() => "Unknown error");
|
|
376
|
+
throw new Error(`OpenAI TTS failed: ${response.status} ${response.statusText} - ${errorText}`);
|
|
377
|
+
}
|
|
378
|
+
return response.arrayBuffer();
|
|
379
|
+
}
|
|
380
|
+
// models/embedding.ts
|
|
381
|
+
import { logger as logger5, ModelType, VECTOR_DIMS } from "@elizaos/core";
|
|
382
|
+
|
|
383
|
+
// utils/events.ts
|
|
384
|
+
import { EventType } from "@elizaos/core";
|
|
385
|
+
var MAX_PROMPT_LENGTH = 200;
|
|
386
|
+
function truncatePrompt(prompt) {
|
|
387
|
+
if (prompt.length <= MAX_PROMPT_LENGTH) {
|
|
388
|
+
return prompt;
|
|
389
|
+
}
|
|
390
|
+
return `${prompt.slice(0, MAX_PROMPT_LENGTH)}…`;
|
|
391
|
+
}
|
|
392
|
+
function normalizeUsage(usage) {
|
|
393
|
+
if ("promptTokens" in usage) {
|
|
394
|
+
return {
|
|
395
|
+
promptTokens: usage.promptTokens ?? 0,
|
|
396
|
+
completionTokens: usage.completionTokens ?? 0,
|
|
397
|
+
totalTokens: usage.totalTokens ?? (usage.promptTokens ?? 0) + (usage.completionTokens ?? 0)
|
|
398
|
+
};
|
|
399
|
+
}
|
|
400
|
+
if ("inputTokens" in usage || "outputTokens" in usage) {
|
|
401
|
+
const input = usage.inputTokens ?? 0;
|
|
402
|
+
const output = usage.outputTokens ?? 0;
|
|
403
|
+
return {
|
|
404
|
+
promptTokens: input,
|
|
405
|
+
completionTokens: output,
|
|
406
|
+
totalTokens: input + output
|
|
407
|
+
};
|
|
408
|
+
}
|
|
409
|
+
return {
|
|
410
|
+
promptTokens: 0,
|
|
411
|
+
completionTokens: 0,
|
|
412
|
+
totalTokens: 0
|
|
413
|
+
};
|
|
414
|
+
}
|
|
415
|
+
function emitModelUsageEvent(runtime, type, prompt, usage) {
|
|
416
|
+
const normalized = normalizeUsage(usage);
|
|
417
|
+
const payload = {
|
|
418
|
+
runtime,
|
|
419
|
+
source: "openai",
|
|
420
|
+
provider: "openai",
|
|
421
|
+
type,
|
|
422
|
+
prompt: truncatePrompt(prompt),
|
|
423
|
+
tokens: {
|
|
424
|
+
prompt: normalized.promptTokens,
|
|
425
|
+
completion: normalized.completionTokens,
|
|
426
|
+
total: normalized.totalTokens
|
|
511
427
|
}
|
|
428
|
+
};
|
|
429
|
+
runtime.emitEvent(EventType.MODEL_USED, payload);
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
// models/embedding.ts
|
|
433
|
+
function validateDimension(dimension) {
|
|
434
|
+
const validDimensions = Object.values(VECTOR_DIMS);
|
|
435
|
+
if (!validDimensions.includes(dimension)) {
|
|
436
|
+
throw new Error(`Invalid embedding dimension: ${dimension}. Must be one of: ${validDimensions.join(", ")}`);
|
|
512
437
|
}
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
438
|
+
return dimension;
|
|
439
|
+
}
|
|
440
|
+
function extractText(params) {
|
|
441
|
+
if (params === null) {
|
|
442
|
+
return null;
|
|
443
|
+
}
|
|
444
|
+
if (typeof params === "string") {
|
|
445
|
+
return params;
|
|
446
|
+
}
|
|
447
|
+
if (typeof params === "object" && typeof params.text === "string") {
|
|
448
|
+
return params.text;
|
|
449
|
+
}
|
|
450
|
+
throw new Error("Invalid embedding params: expected string, { text: string }, or null");
|
|
451
|
+
}
|
|
452
|
+
async function handleTextEmbedding(runtime, params) {
|
|
453
|
+
const embeddingModel = getEmbeddingModel(runtime);
|
|
454
|
+
const embeddingDimension = validateDimension(getEmbeddingDimensions(runtime));
|
|
455
|
+
const text = extractText(params);
|
|
456
|
+
if (text === null) {
|
|
457
|
+
logger5.debug("[OpenAI] Creating test embedding for initialization");
|
|
458
|
+
const testVector = new Array(embeddingDimension).fill(0);
|
|
459
|
+
testVector[0] = 0.1;
|
|
460
|
+
return testVector;
|
|
461
|
+
}
|
|
462
|
+
const trimmedText = text.trim();
|
|
463
|
+
if (trimmedText.length === 0) {
|
|
464
|
+
throw new Error("Cannot generate embedding for empty text");
|
|
465
|
+
}
|
|
466
|
+
const baseURL = getEmbeddingBaseURL(runtime);
|
|
467
|
+
const url = `${baseURL}/embeddings`;
|
|
468
|
+
logger5.debug(`[OpenAI] Generating embedding with model: ${embeddingModel}`);
|
|
469
|
+
const response = await fetch(url, {
|
|
470
|
+
method: "POST",
|
|
471
|
+
headers: {
|
|
472
|
+
...getAuthHeader(runtime, true),
|
|
473
|
+
"Content-Type": "application/json"
|
|
474
|
+
},
|
|
475
|
+
body: JSON.stringify({
|
|
476
|
+
model: embeddingModel,
|
|
477
|
+
input: trimmedText
|
|
478
|
+
})
|
|
479
|
+
});
|
|
480
|
+
if (!response.ok) {
|
|
481
|
+
const errorText = await response.text().catch(() => "Unknown error");
|
|
482
|
+
throw new Error(`OpenAI embedding API error: ${response.status} ${response.statusText} - ${errorText}`);
|
|
483
|
+
}
|
|
484
|
+
const data = await response.json();
|
|
485
|
+
const firstResult = data?.data?.[0];
|
|
486
|
+
if (!firstResult || !firstResult.embedding) {
|
|
487
|
+
throw new Error("OpenAI API returned invalid embedding response structure");
|
|
488
|
+
}
|
|
489
|
+
const embedding = firstResult.embedding;
|
|
490
|
+
if (embedding.length !== embeddingDimension) {
|
|
491
|
+
throw new Error(`Embedding dimension mismatch: got ${embedding.length}, expected ${embeddingDimension}. ` + `Check OPENAI_EMBEDDING_DIMENSIONS setting.`);
|
|
492
|
+
}
|
|
493
|
+
if (data.usage) {
|
|
494
|
+
emitModelUsageEvent(runtime, ModelType.TEXT_EMBEDDING, trimmedText, {
|
|
495
|
+
promptTokens: data.usage.prompt_tokens,
|
|
496
|
+
completionTokens: 0,
|
|
497
|
+
totalTokens: data.usage.total_tokens
|
|
520
498
|
});
|
|
521
|
-
if (!response.ok) {
|
|
522
|
-
throw new Error(`Failed to transcribe audio: ${response.status} ${response.statusText}`);
|
|
523
|
-
}
|
|
524
|
-
const data = await response.json();
|
|
525
|
-
return data.text || "";
|
|
526
|
-
} catch (error) {
|
|
527
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
528
|
-
logger7.error(`TRANSCRIPTION error: ${message}`);
|
|
529
|
-
throw error;
|
|
530
499
|
}
|
|
500
|
+
logger5.debug(`[OpenAI] Generated embedding with ${embedding.length} dimensions`);
|
|
501
|
+
return embedding;
|
|
531
502
|
}
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
503
|
+
// models/image.ts
|
|
504
|
+
import { logger as logger6, ModelType as ModelType2 } from "@elizaos/core";
|
|
505
|
+
var DEFAULT_IMAGE_DESCRIPTION_PROMPT = "Please analyze this image and provide a title and detailed description.";
|
|
506
|
+
async function handleImageGeneration(runtime, params) {
|
|
507
|
+
const modelName = getImageModel(runtime);
|
|
508
|
+
const count = params.count ?? 1;
|
|
509
|
+
const size = params.size ?? "1024x1024";
|
|
510
|
+
const extendedParams = params;
|
|
511
|
+
logger6.debug(`[OpenAI] Using IMAGE model: ${modelName}`);
|
|
512
|
+
if (!params.prompt || params.prompt.trim().length === 0) {
|
|
513
|
+
throw new Error("IMAGE generation requires a non-empty prompt");
|
|
514
|
+
}
|
|
515
|
+
if (count < 1 || count > 10) {
|
|
516
|
+
throw new Error("IMAGE count must be between 1 and 10");
|
|
517
|
+
}
|
|
518
|
+
const baseURL = getBaseURL(runtime);
|
|
519
|
+
const requestBody = {
|
|
520
|
+
model: modelName,
|
|
521
|
+
prompt: params.prompt,
|
|
522
|
+
n: count,
|
|
523
|
+
size
|
|
524
|
+
};
|
|
525
|
+
if (extendedParams.quality) {
|
|
526
|
+
requestBody.quality = extendedParams.quality;
|
|
542
527
|
}
|
|
528
|
+
if (extendedParams.style) {
|
|
529
|
+
requestBody.style = extendedParams.style;
|
|
530
|
+
}
|
|
531
|
+
const response = await fetch(`${baseURL}/images/generations`, {
|
|
532
|
+
method: "POST",
|
|
533
|
+
headers: {
|
|
534
|
+
...getAuthHeader(runtime),
|
|
535
|
+
"Content-Type": "application/json"
|
|
536
|
+
},
|
|
537
|
+
body: JSON.stringify(requestBody)
|
|
538
|
+
});
|
|
539
|
+
if (!response.ok) {
|
|
540
|
+
const errorText = await response.text().catch(() => "Unknown error");
|
|
541
|
+
throw new Error(`OpenAI image generation failed: ${response.status} ${response.statusText} - ${errorText}`);
|
|
542
|
+
}
|
|
543
|
+
const data = await response.json();
|
|
544
|
+
if (!data.data || data.data.length === 0) {
|
|
545
|
+
throw new Error("OpenAI API returned no images");
|
|
546
|
+
}
|
|
547
|
+
return data.data.map((item) => ({
|
|
548
|
+
url: item.url,
|
|
549
|
+
revisedPrompt: item.revised_prompt
|
|
550
|
+
}));
|
|
551
|
+
}
|
|
552
|
+
function parseTitleFromResponse(content) {
|
|
553
|
+
const titleMatch = content.match(/title[:\s]+(.+?)(?:\n|$)/i);
|
|
554
|
+
return titleMatch?.[1]?.trim() ?? "Image Analysis";
|
|
555
|
+
}
|
|
556
|
+
function parseDescriptionFromResponse(content) {
|
|
557
|
+
return content.replace(/title[:\s]+(.+?)(?:\n|$)/i, "").trim();
|
|
543
558
|
}
|
|
544
|
-
|
|
545
|
-
|
|
559
|
+
async function handleImageDescription(runtime, params) {
|
|
560
|
+
const modelName = getImageDescriptionModel(runtime);
|
|
561
|
+
const maxTokens = getImageDescriptionMaxTokens(runtime);
|
|
562
|
+
logger6.debug(`[OpenAI] Using IMAGE_DESCRIPTION model: ${modelName}`);
|
|
563
|
+
let imageUrl;
|
|
564
|
+
let promptText;
|
|
565
|
+
if (typeof params === "string") {
|
|
566
|
+
imageUrl = params;
|
|
567
|
+
promptText = DEFAULT_IMAGE_DESCRIPTION_PROMPT;
|
|
568
|
+
} else {
|
|
569
|
+
imageUrl = params.imageUrl;
|
|
570
|
+
promptText = params.prompt ?? DEFAULT_IMAGE_DESCRIPTION_PROMPT;
|
|
571
|
+
}
|
|
572
|
+
if (!imageUrl || imageUrl.trim().length === 0) {
|
|
573
|
+
throw new Error("IMAGE_DESCRIPTION requires a valid image URL");
|
|
574
|
+
}
|
|
575
|
+
const baseURL = getBaseURL(runtime);
|
|
576
|
+
const requestBody = {
|
|
577
|
+
model: modelName,
|
|
578
|
+
messages: [
|
|
579
|
+
{
|
|
580
|
+
role: "user",
|
|
581
|
+
content: [
|
|
582
|
+
{ type: "text", text: promptText },
|
|
583
|
+
{ type: "image_url", image_url: { url: imageUrl } }
|
|
584
|
+
]
|
|
585
|
+
}
|
|
586
|
+
],
|
|
587
|
+
max_tokens: maxTokens
|
|
588
|
+
};
|
|
589
|
+
const response = await fetch(`${baseURL}/chat/completions`, {
|
|
590
|
+
method: "POST",
|
|
591
|
+
headers: {
|
|
592
|
+
...getAuthHeader(runtime),
|
|
593
|
+
"Content-Type": "application/json"
|
|
594
|
+
},
|
|
595
|
+
body: JSON.stringify(requestBody)
|
|
596
|
+
});
|
|
597
|
+
if (!response.ok) {
|
|
598
|
+
const errorText = await response.text().catch(() => "Unknown error");
|
|
599
|
+
throw new Error(`OpenAI image description failed: ${response.status} ${response.statusText} - ${errorText}`);
|
|
600
|
+
}
|
|
601
|
+
const data = await response.json();
|
|
602
|
+
if (data.usage) {
|
|
603
|
+
emitModelUsageEvent(runtime, ModelType2.IMAGE_DESCRIPTION, typeof params === "string" ? params : params.prompt ?? "", {
|
|
604
|
+
promptTokens: data.usage.prompt_tokens,
|
|
605
|
+
completionTokens: data.usage.completion_tokens,
|
|
606
|
+
totalTokens: data.usage.total_tokens
|
|
607
|
+
});
|
|
608
|
+
}
|
|
609
|
+
const firstChoice = data.choices?.[0];
|
|
610
|
+
const content = firstChoice?.message?.content;
|
|
611
|
+
if (!content) {
|
|
612
|
+
throw new Error("OpenAI API returned empty image description");
|
|
613
|
+
}
|
|
614
|
+
return {
|
|
615
|
+
title: parseTitleFromResponse(content),
|
|
616
|
+
description: parseDescriptionFromResponse(content)
|
|
617
|
+
};
|
|
618
|
+
}
|
|
619
|
+
// models/object.ts
|
|
620
|
+
import { logger as logger8, ModelType as ModelType3 } from "@elizaos/core";
|
|
546
621
|
import { generateObject } from "ai";
|
|
547
622
|
|
|
548
|
-
//
|
|
549
|
-
import {
|
|
623
|
+
// providers/openai.ts
|
|
624
|
+
import { createOpenAI } from "@ai-sdk/openai";
|
|
625
|
+
var PROXY_API_KEY = "sk-proxy";
|
|
626
|
+
function createOpenAIClient(runtime) {
|
|
627
|
+
const baseURL = getBaseURL(runtime);
|
|
628
|
+
const apiKey = getApiKey(runtime);
|
|
629
|
+
if (!apiKey && isProxyMode(runtime)) {
|
|
630
|
+
return createOpenAI({
|
|
631
|
+
apiKey: PROXY_API_KEY,
|
|
632
|
+
baseURL
|
|
633
|
+
});
|
|
634
|
+
}
|
|
635
|
+
if (!apiKey) {
|
|
636
|
+
throw new Error("OPENAI_API_KEY is required. Set it in your environment variables or runtime settings.");
|
|
637
|
+
}
|
|
638
|
+
return createOpenAI({
|
|
639
|
+
apiKey,
|
|
640
|
+
baseURL
|
|
641
|
+
});
|
|
642
|
+
}
|
|
643
|
+
// utils/json.ts
|
|
644
|
+
import { logger as logger7 } from "@elizaos/core";
|
|
550
645
|
import { JSONParseError } from "ai";
|
|
646
|
+
var JSON_CLEANUP_PATTERNS = {
|
|
647
|
+
MARKDOWN_JSON: /```json\n|\n```|```/g,
|
|
648
|
+
WHITESPACE: /^\s+|\s+$/g
|
|
649
|
+
};
|
|
551
650
|
function getJsonRepairFunction() {
|
|
552
651
|
return async ({ text, error }) => {
|
|
553
|
-
|
|
554
|
-
if (error instanceof JSONParseError) {
|
|
555
|
-
const cleanedText = text.replace(/```json\n|\n```|```/g, "");
|
|
556
|
-
JSON.parse(cleanedText);
|
|
557
|
-
return cleanedText;
|
|
558
|
-
}
|
|
652
|
+
if (!(error instanceof JSONParseError)) {
|
|
559
653
|
return null;
|
|
560
|
-
}
|
|
561
|
-
|
|
562
|
-
|
|
654
|
+
}
|
|
655
|
+
try {
|
|
656
|
+
const cleanedText = text.replace(JSON_CLEANUP_PATTERNS.MARKDOWN_JSON, "");
|
|
657
|
+
JSON.parse(cleanedText);
|
|
658
|
+
logger7.debug("[JSON Repair] Successfully repaired JSON by removing markdown wrappers");
|
|
659
|
+
return cleanedText;
|
|
660
|
+
} catch {
|
|
661
|
+
logger7.warn("[JSON Repair] Unable to repair JSON text");
|
|
563
662
|
return null;
|
|
564
663
|
}
|
|
565
664
|
};
|
|
566
665
|
}
|
|
567
666
|
|
|
568
|
-
//
|
|
667
|
+
// models/object.ts
|
|
569
668
|
async function generateObjectByModelType(runtime, params, modelType, getModelFn) {
|
|
570
669
|
const openai = createOpenAIClient(runtime);
|
|
571
670
|
const modelName = getModelFn(runtime);
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
if (schemaPresent) {
|
|
576
|
-
logger9.warn(`Schema provided but ignored: OpenAI object generation currently uses output=no-schema. The schema parameter has no effect.`);
|
|
671
|
+
logger8.debug(`[OpenAI] Using ${modelType} model: ${modelName}`);
|
|
672
|
+
if (!params.prompt || params.prompt.trim().length === 0) {
|
|
673
|
+
throw new Error("Object generation requires a non-empty prompt");
|
|
577
674
|
}
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
}
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
throw error;
|
|
675
|
+
if (params.schema) {
|
|
676
|
+
logger8.debug("[OpenAI] Schema provided but using no-schema mode. " + "Structure is determined by prompt instructions.");
|
|
677
|
+
}
|
|
678
|
+
const model = openai.chat(modelName);
|
|
679
|
+
const { object, usage } = await generateObject({
|
|
680
|
+
model,
|
|
681
|
+
output: "no-schema",
|
|
682
|
+
prompt: params.prompt,
|
|
683
|
+
experimental_repairText: getJsonRepairFunction()
|
|
684
|
+
});
|
|
685
|
+
if (usage) {
|
|
686
|
+
emitModelUsageEvent(runtime, modelType, params.prompt, usage);
|
|
687
|
+
}
|
|
688
|
+
if (typeof object !== "object" || object === null) {
|
|
689
|
+
throw new Error(`Object generation returned ${typeof object}, expected object`);
|
|
594
690
|
}
|
|
691
|
+
return object;
|
|
595
692
|
}
|
|
596
693
|
async function handleObjectSmall(runtime, params) {
|
|
597
|
-
return generateObjectByModelType(runtime, params,
|
|
694
|
+
return generateObjectByModelType(runtime, params, ModelType3.OBJECT_SMALL, getSmallModel);
|
|
598
695
|
}
|
|
599
696
|
async function handleObjectLarge(runtime, params) {
|
|
600
|
-
return generateObjectByModelType(runtime, params,
|
|
697
|
+
return generateObjectByModelType(runtime, params, ModelType3.OBJECT_LARGE, getLargeModel);
|
|
698
|
+
}
|
|
699
|
+
// models/research.ts
|
|
700
|
+
import { logger as logger9 } from "@elizaos/core";
|
|
701
|
+
function convertToolToApi(tool) {
|
|
702
|
+
switch (tool.type) {
|
|
703
|
+
case "web_search_preview":
|
|
704
|
+
return { type: "web_search_preview" };
|
|
705
|
+
case "file_search":
|
|
706
|
+
return {
|
|
707
|
+
type: "file_search",
|
|
708
|
+
vector_store_ids: tool.vectorStoreIds
|
|
709
|
+
};
|
|
710
|
+
case "code_interpreter":
|
|
711
|
+
return {
|
|
712
|
+
type: "code_interpreter",
|
|
713
|
+
container: tool.container ?? { type: "auto" }
|
|
714
|
+
};
|
|
715
|
+
case "mcp":
|
|
716
|
+
return {
|
|
717
|
+
type: "mcp",
|
|
718
|
+
server_label: tool.serverLabel,
|
|
719
|
+
server_url: tool.serverUrl,
|
|
720
|
+
require_approval: tool.requireApproval ?? "never"
|
|
721
|
+
};
|
|
722
|
+
default:
|
|
723
|
+
throw new Error(`Unknown research tool type: ${tool.type}`);
|
|
724
|
+
}
|
|
725
|
+
}
|
|
726
|
+
function convertOutputItem(item) {
|
|
727
|
+
switch (item.type) {
|
|
728
|
+
case "web_search_call":
|
|
729
|
+
return {
|
|
730
|
+
id: item.id ?? "",
|
|
731
|
+
type: "web_search_call",
|
|
732
|
+
status: item.status ?? "completed",
|
|
733
|
+
action: {
|
|
734
|
+
type: item.action?.type ?? "search",
|
|
735
|
+
query: item.action?.query,
|
|
736
|
+
url: item.action?.url
|
|
737
|
+
}
|
|
738
|
+
};
|
|
739
|
+
case "file_search_call":
|
|
740
|
+
return {
|
|
741
|
+
id: item.id ?? "",
|
|
742
|
+
type: "file_search_call",
|
|
743
|
+
status: item.status ?? "completed",
|
|
744
|
+
query: item.query ?? "",
|
|
745
|
+
results: item.results?.map((r) => ({
|
|
746
|
+
fileId: r.file_id,
|
|
747
|
+
fileName: r.file_name,
|
|
748
|
+
score: r.score
|
|
749
|
+
}))
|
|
750
|
+
};
|
|
751
|
+
case "code_interpreter_call":
|
|
752
|
+
return {
|
|
753
|
+
id: item.id ?? "",
|
|
754
|
+
type: "code_interpreter_call",
|
|
755
|
+
status: item.status ?? "completed",
|
|
756
|
+
code: item.code ?? "",
|
|
757
|
+
output: item.output
|
|
758
|
+
};
|
|
759
|
+
case "mcp_tool_call":
|
|
760
|
+
return {
|
|
761
|
+
id: item.id ?? "",
|
|
762
|
+
type: "mcp_tool_call",
|
|
763
|
+
status: item.status ?? "completed",
|
|
764
|
+
serverLabel: item.server_label ?? "",
|
|
765
|
+
toolName: item.tool_name ?? "",
|
|
766
|
+
arguments: item.arguments ?? {},
|
|
767
|
+
result: item.result
|
|
768
|
+
};
|
|
769
|
+
case "message":
|
|
770
|
+
return {
|
|
771
|
+
type: "message",
|
|
772
|
+
content: item.content?.map((c) => ({
|
|
773
|
+
type: "output_text",
|
|
774
|
+
text: c.text,
|
|
775
|
+
annotations: c.annotations?.map((a) => ({
|
|
776
|
+
url: a.url,
|
|
777
|
+
title: a.title,
|
|
778
|
+
startIndex: a.start_index,
|
|
779
|
+
endIndex: a.end_index
|
|
780
|
+
})) ?? []
|
|
781
|
+
})) ?? []
|
|
782
|
+
};
|
|
783
|
+
default:
|
|
784
|
+
return null;
|
|
785
|
+
}
|
|
786
|
+
}
|
|
787
|
+
function extractTextAndAnnotations(response) {
|
|
788
|
+
if (response.output_text) {
|
|
789
|
+
const annotations2 = [];
|
|
790
|
+
if (response.output) {
|
|
791
|
+
for (const item of response.output) {
|
|
792
|
+
if (item.type === "message" && item.content) {
|
|
793
|
+
for (const content of item.content) {
|
|
794
|
+
if (content.annotations) {
|
|
795
|
+
for (const ann of content.annotations) {
|
|
796
|
+
annotations2.push({
|
|
797
|
+
url: ann.url,
|
|
798
|
+
title: ann.title,
|
|
799
|
+
startIndex: ann.start_index,
|
|
800
|
+
endIndex: ann.end_index
|
|
801
|
+
});
|
|
802
|
+
}
|
|
803
|
+
}
|
|
804
|
+
}
|
|
805
|
+
}
|
|
806
|
+
}
|
|
807
|
+
}
|
|
808
|
+
return { text: response.output_text, annotations: annotations2 };
|
|
809
|
+
}
|
|
810
|
+
let text = "";
|
|
811
|
+
const annotations = [];
|
|
812
|
+
if (response.output) {
|
|
813
|
+
for (const item of response.output) {
|
|
814
|
+
if (item.type === "message" && item.content) {
|
|
815
|
+
for (const content of item.content) {
|
|
816
|
+
text += content.text;
|
|
817
|
+
if (content.annotations) {
|
|
818
|
+
for (const ann of content.annotations) {
|
|
819
|
+
annotations.push({
|
|
820
|
+
url: ann.url,
|
|
821
|
+
title: ann.title,
|
|
822
|
+
startIndex: ann.start_index,
|
|
823
|
+
endIndex: ann.end_index
|
|
824
|
+
});
|
|
825
|
+
}
|
|
826
|
+
}
|
|
827
|
+
}
|
|
828
|
+
}
|
|
829
|
+
}
|
|
830
|
+
}
|
|
831
|
+
return { text, annotations };
|
|
601
832
|
}
|
|
602
|
-
|
|
833
|
+
async function handleResearch(runtime, params) {
|
|
834
|
+
const apiKey = getApiKey(runtime);
|
|
835
|
+
if (!apiKey) {
|
|
836
|
+
throw new Error("OPENAI_API_KEY is required for deep research. Set it in your environment variables or runtime settings.");
|
|
837
|
+
}
|
|
838
|
+
const baseURL = getBaseURL(runtime);
|
|
839
|
+
const modelName = params.model ?? getResearchModel(runtime);
|
|
840
|
+
const timeout = getResearchTimeout(runtime);
|
|
841
|
+
logger9.debug(`[OpenAI] Starting deep research with model: ${modelName}`);
|
|
842
|
+
logger9.debug(`[OpenAI] Research input: ${params.input.substring(0, 100)}...`);
|
|
843
|
+
const dataSourceTools = params.tools?.filter((t) => t.type === "web_search_preview" || t.type === "file_search" || t.type === "mcp");
|
|
844
|
+
if (!dataSourceTools || dataSourceTools.length === 0) {
|
|
845
|
+
logger9.debug("[OpenAI] No data source tools specified, defaulting to web_search_preview");
|
|
846
|
+
params.tools = [{ type: "web_search_preview" }, ...params.tools ?? []];
|
|
847
|
+
}
|
|
848
|
+
const requestBody = {
|
|
849
|
+
model: modelName,
|
|
850
|
+
input: params.input
|
|
851
|
+
};
|
|
852
|
+
if (params.instructions) {
|
|
853
|
+
requestBody.instructions = params.instructions;
|
|
854
|
+
}
|
|
855
|
+
if (params.background !== undefined) {
|
|
856
|
+
requestBody.background = params.background;
|
|
857
|
+
}
|
|
858
|
+
if (params.tools && params.tools.length > 0) {
|
|
859
|
+
requestBody.tools = params.tools.map(convertToolToApi);
|
|
860
|
+
}
|
|
861
|
+
if (params.maxToolCalls !== undefined) {
|
|
862
|
+
requestBody.max_tool_calls = params.maxToolCalls;
|
|
863
|
+
}
|
|
864
|
+
if (params.reasoningSummary) {
|
|
865
|
+
requestBody.reasoning = { summary: params.reasoningSummary };
|
|
866
|
+
}
|
|
867
|
+
logger9.debug(`[OpenAI] Research request body: ${JSON.stringify(requestBody, null, 2)}`);
|
|
868
|
+
const response = await fetch(`${baseURL}/responses`, {
|
|
869
|
+
method: "POST",
|
|
870
|
+
headers: {
|
|
871
|
+
Authorization: `Bearer ${apiKey}`,
|
|
872
|
+
"Content-Type": "application/json"
|
|
873
|
+
},
|
|
874
|
+
body: JSON.stringify(requestBody),
|
|
875
|
+
signal: AbortSignal.timeout(timeout)
|
|
876
|
+
});
|
|
877
|
+
if (!response.ok) {
|
|
878
|
+
const errorText = await response.text();
|
|
879
|
+
logger9.error(`[OpenAI] Research request failed: ${response.status} ${errorText}`);
|
|
880
|
+
throw new Error(`Deep research request failed: ${response.status} ${response.statusText}`);
|
|
881
|
+
}
|
|
882
|
+
const data = await response.json();
|
|
883
|
+
if (data.error) {
|
|
884
|
+
logger9.error(`[OpenAI] Research API error: ${data.error.message}`);
|
|
885
|
+
throw new Error(`Deep research error: ${data.error.message}`);
|
|
886
|
+
}
|
|
887
|
+
logger9.debug(`[OpenAI] Research response received. Status: ${data.status ?? "completed"}`);
|
|
888
|
+
const { text, annotations } = extractTextAndAnnotations(data);
|
|
889
|
+
const outputItems = [];
|
|
890
|
+
if (data.output) {
|
|
891
|
+
for (const item of data.output) {
|
|
892
|
+
const converted = convertOutputItem(item);
|
|
893
|
+
if (converted) {
|
|
894
|
+
outputItems.push(converted);
|
|
895
|
+
}
|
|
896
|
+
}
|
|
897
|
+
}
|
|
898
|
+
const result = {
|
|
899
|
+
id: data.id,
|
|
900
|
+
text,
|
|
901
|
+
annotations,
|
|
902
|
+
outputItems,
|
|
903
|
+
status: data.status
|
|
904
|
+
};
|
|
905
|
+
logger9.info(`[OpenAI] Research completed. Text length: ${text.length}, Annotations: ${annotations.length}, Output items: ${outputItems.length}`);
|
|
906
|
+
return result;
|
|
907
|
+
}
|
|
908
|
+
// models/text.ts
|
|
909
|
+
import { logger as logger10, ModelType as ModelType4 } from "@elizaos/core";
|
|
910
|
+
import { generateText, streamText } from "ai";
|
|
911
|
+
function convertUsage(usage) {
|
|
912
|
+
if (!usage) {
|
|
913
|
+
return;
|
|
914
|
+
}
|
|
915
|
+
const promptTokens = usage.inputTokens ?? 0;
|
|
916
|
+
const completionTokens = usage.outputTokens ?? 0;
|
|
917
|
+
return {
|
|
918
|
+
promptTokens,
|
|
919
|
+
completionTokens,
|
|
920
|
+
totalTokens: promptTokens + completionTokens
|
|
921
|
+
};
|
|
922
|
+
}
|
|
923
|
+
async function generateTextByModelType(runtime, params, modelType, getModelFn) {
|
|
924
|
+
const openai = createOpenAIClient(runtime);
|
|
925
|
+
const modelName = getModelFn(runtime);
|
|
926
|
+
logger10.debug(`[OpenAI] Using ${modelType} model: ${modelName}`);
|
|
927
|
+
const systemPrompt = runtime.character.system ?? undefined;
|
|
928
|
+
const model = openai.chat(modelName);
|
|
929
|
+
const generateParams = {
|
|
930
|
+
model,
|
|
931
|
+
prompt: params.prompt,
|
|
932
|
+
system: systemPrompt,
|
|
933
|
+
maxOutputTokens: params.maxTokens ?? 8192,
|
|
934
|
+
experimental_telemetry: { isEnabled: getExperimentalTelemetry(runtime) }
|
|
935
|
+
};
|
|
936
|
+
if (params.stream) {
|
|
937
|
+
const result = streamText(generateParams);
|
|
938
|
+
return {
|
|
939
|
+
textStream: result.textStream,
|
|
940
|
+
text: Promise.resolve(result.text),
|
|
941
|
+
usage: Promise.resolve(result.usage).then(convertUsage),
|
|
942
|
+
finishReason: Promise.resolve(result.finishReason).then((r) => r)
|
|
943
|
+
};
|
|
944
|
+
}
|
|
945
|
+
const { text, usage } = await generateText(generateParams);
|
|
946
|
+
if (usage) {
|
|
947
|
+
emitModelUsageEvent(runtime, modelType, params.prompt, usage);
|
|
948
|
+
}
|
|
949
|
+
return text;
|
|
950
|
+
}
|
|
951
|
+
async function handleTextSmall(runtime, params) {
|
|
952
|
+
return generateTextByModelType(runtime, params, ModelType4.TEXT_SMALL, getSmallModel);
|
|
953
|
+
}
|
|
954
|
+
async function handleTextLarge(runtime, params) {
|
|
955
|
+
return generateTextByModelType(runtime, params, ModelType4.TEXT_LARGE, getLargeModel);
|
|
956
|
+
}
|
|
957
|
+
// models/tokenizer.ts
|
|
603
958
|
import { ModelType as ModelType6 } from "@elizaos/core";
|
|
604
959
|
|
|
605
|
-
//
|
|
960
|
+
// utils/tokenization.ts
|
|
606
961
|
import { ModelType as ModelType5 } from "@elizaos/core";
|
|
607
|
-
import {
|
|
962
|
+
import {
|
|
963
|
+
encodingForModel,
|
|
964
|
+
getEncoding
|
|
965
|
+
} from "js-tiktoken";
|
|
608
966
|
function resolveTokenizerEncoding(modelName) {
|
|
609
967
|
const normalized = modelName.toLowerCase();
|
|
610
968
|
const fallbackEncoding = normalized.includes("4o") ? "o200k_base" : "cl100k_base";
|
|
611
969
|
try {
|
|
612
970
|
return encodingForModel(modelName);
|
|
613
|
-
} catch
|
|
971
|
+
} catch {
|
|
614
972
|
return getEncoding(fallbackEncoding);
|
|
615
973
|
}
|
|
616
974
|
}
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
975
|
+
function getModelName(runtime, modelType) {
|
|
976
|
+
if (modelType === ModelType5.TEXT_SMALL) {
|
|
977
|
+
return getSmallModel(runtime);
|
|
978
|
+
}
|
|
979
|
+
return getLargeModel(runtime);
|
|
621
980
|
}
|
|
622
|
-
|
|
623
|
-
const modelName =
|
|
624
|
-
|
|
981
|
+
function tokenizeText(runtime, modelType, text) {
|
|
982
|
+
const modelName = getModelName(runtime, modelType);
|
|
983
|
+
const encoder = resolveTokenizerEncoding(modelName);
|
|
984
|
+
return encoder.encode(text);
|
|
985
|
+
}
|
|
986
|
+
function detokenizeText(runtime, modelType, tokens) {
|
|
987
|
+
const modelName = getModelName(runtime, modelType);
|
|
988
|
+
const encoder = resolveTokenizerEncoding(modelName);
|
|
989
|
+
return encoder.decode(tokens);
|
|
625
990
|
}
|
|
626
991
|
|
|
627
|
-
//
|
|
628
|
-
async function handleTokenizerEncode(runtime,
|
|
629
|
-
|
|
992
|
+
// models/tokenizer.ts
|
|
993
|
+
async function handleTokenizerEncode(runtime, params) {
|
|
994
|
+
if (!params.prompt) {
|
|
995
|
+
throw new Error("Tokenization requires a non-empty prompt");
|
|
996
|
+
}
|
|
997
|
+
const modelType = params.modelType ?? ModelType6.TEXT_LARGE;
|
|
998
|
+
return tokenizeText(runtime, modelType, params.prompt);
|
|
999
|
+
}
|
|
1000
|
+
async function handleTokenizerDecode(runtime, params) {
|
|
1001
|
+
if (!params.tokens || !Array.isArray(params.tokens)) {
|
|
1002
|
+
throw new Error("Detokenization requires a valid tokens array");
|
|
1003
|
+
}
|
|
1004
|
+
if (params.tokens.length === 0) {
|
|
1005
|
+
return "";
|
|
1006
|
+
}
|
|
1007
|
+
for (let i = 0;i < params.tokens.length; i++) {
|
|
1008
|
+
const token = params.tokens[i];
|
|
1009
|
+
if (typeof token !== "number" || !Number.isFinite(token)) {
|
|
1010
|
+
throw new Error(`Invalid token at index ${i}: expected number`);
|
|
1011
|
+
}
|
|
1012
|
+
}
|
|
1013
|
+
const modelType = params.modelType ?? ModelType6.TEXT_LARGE;
|
|
1014
|
+
return detokenizeText(runtime, modelType, params.tokens);
|
|
630
1015
|
}
|
|
631
|
-
|
|
632
|
-
|
|
1016
|
+
// index.ts
|
|
1017
|
+
function getProcessEnv() {
|
|
1018
|
+
if (typeof process === "undefined") {
|
|
1019
|
+
return {};
|
|
1020
|
+
}
|
|
1021
|
+
return process.env;
|
|
633
1022
|
}
|
|
634
|
-
|
|
1023
|
+
var env = getProcessEnv();
|
|
635
1024
|
var openaiPlugin = {
|
|
636
1025
|
name: "openai",
|
|
637
|
-
description: "OpenAI
|
|
1026
|
+
description: "OpenAI API integration for text, image, audio, and embedding models",
|
|
638
1027
|
config: {
|
|
639
|
-
OPENAI_API_KEY:
|
|
640
|
-
OPENAI_BASE_URL:
|
|
641
|
-
OPENAI_SMALL_MODEL:
|
|
642
|
-
OPENAI_LARGE_MODEL:
|
|
643
|
-
SMALL_MODEL:
|
|
644
|
-
LARGE_MODEL:
|
|
645
|
-
OPENAI_EMBEDDING_MODEL:
|
|
646
|
-
OPENAI_EMBEDDING_API_KEY:
|
|
647
|
-
OPENAI_EMBEDDING_URL:
|
|
648
|
-
OPENAI_EMBEDDING_DIMENSIONS:
|
|
649
|
-
OPENAI_IMAGE_DESCRIPTION_MODEL:
|
|
650
|
-
OPENAI_IMAGE_DESCRIPTION_MAX_TOKENS:
|
|
651
|
-
OPENAI_EXPERIMENTAL_TELEMETRY:
|
|
1028
|
+
OPENAI_API_KEY: env.OPENAI_API_KEY ?? null,
|
|
1029
|
+
OPENAI_BASE_URL: env.OPENAI_BASE_URL ?? null,
|
|
1030
|
+
OPENAI_SMALL_MODEL: env.OPENAI_SMALL_MODEL ?? null,
|
|
1031
|
+
OPENAI_LARGE_MODEL: env.OPENAI_LARGE_MODEL ?? null,
|
|
1032
|
+
SMALL_MODEL: env.SMALL_MODEL ?? null,
|
|
1033
|
+
LARGE_MODEL: env.LARGE_MODEL ?? null,
|
|
1034
|
+
OPENAI_EMBEDDING_MODEL: env.OPENAI_EMBEDDING_MODEL ?? null,
|
|
1035
|
+
OPENAI_EMBEDDING_API_KEY: env.OPENAI_EMBEDDING_API_KEY ?? null,
|
|
1036
|
+
OPENAI_EMBEDDING_URL: env.OPENAI_EMBEDDING_URL ?? null,
|
|
1037
|
+
OPENAI_EMBEDDING_DIMENSIONS: env.OPENAI_EMBEDDING_DIMENSIONS ?? null,
|
|
1038
|
+
OPENAI_IMAGE_DESCRIPTION_MODEL: env.OPENAI_IMAGE_DESCRIPTION_MODEL ?? null,
|
|
1039
|
+
OPENAI_IMAGE_DESCRIPTION_MAX_TOKENS: env.OPENAI_IMAGE_DESCRIPTION_MAX_TOKENS ?? null,
|
|
1040
|
+
OPENAI_EXPERIMENTAL_TELEMETRY: env.OPENAI_EXPERIMENTAL_TELEMETRY ?? null,
|
|
1041
|
+
OPENAI_RESEARCH_MODEL: env.OPENAI_RESEARCH_MODEL ?? null,
|
|
1042
|
+
OPENAI_RESEARCH_TIMEOUT: env.OPENAI_RESEARCH_TIMEOUT ?? null
|
|
652
1043
|
},
|
|
653
|
-
async init(
|
|
654
|
-
initializeOpenAI(
|
|
1044
|
+
async init(config, runtime) {
|
|
1045
|
+
initializeOpenAI(config, runtime);
|
|
655
1046
|
},
|
|
656
1047
|
models: {
|
|
657
1048
|
[ModelType7.TEXT_EMBEDDING]: async (runtime, params) => {
|
|
@@ -686,6 +1077,9 @@ var openaiPlugin = {
|
|
|
686
1077
|
},
|
|
687
1078
|
[ModelType7.OBJECT_LARGE]: async (runtime, params) => {
|
|
688
1079
|
return handleObjectLarge(runtime, params);
|
|
1080
|
+
},
|
|
1081
|
+
[ModelType7.RESEARCH]: async (runtime, params) => {
|
|
1082
|
+
return handleResearch(runtime, params);
|
|
689
1083
|
}
|
|
690
1084
|
},
|
|
691
1085
|
tests: [
|
|
@@ -693,221 +1087,170 @@ var openaiPlugin = {
|
|
|
693
1087
|
name: "openai_plugin_tests",
|
|
694
1088
|
tests: [
|
|
695
1089
|
{
|
|
696
|
-
name: "
|
|
1090
|
+
name: "openai_test_api_connectivity",
|
|
697
1091
|
fn: async (runtime) => {
|
|
698
1092
|
const baseURL = getBaseURL(runtime);
|
|
699
1093
|
const response = await fetch(`${baseURL}/models`, {
|
|
700
1094
|
headers: getAuthHeader(runtime)
|
|
701
1095
|
});
|
|
702
|
-
const data = await response.json();
|
|
703
|
-
logger10.log({ data: data?.data?.length ?? "N/A" }, "Models Available");
|
|
704
1096
|
if (!response.ok) {
|
|
705
|
-
throw new Error(`
|
|
1097
|
+
throw new Error(`API connectivity test failed: ${response.status} ${response.statusText}`);
|
|
706
1098
|
}
|
|
1099
|
+
const data = await response.json();
|
|
1100
|
+
logger11.info(`[OpenAI Test] API connected. ${data.data?.length ?? 0} models available.`);
|
|
707
1101
|
}
|
|
708
1102
|
},
|
|
709
1103
|
{
|
|
710
1104
|
name: "openai_test_text_embedding",
|
|
711
1105
|
fn: async (runtime) => {
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
} catch (error) {
|
|
718
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
719
|
-
logger10.error(`Error in test_text_embedding: ${message}`);
|
|
720
|
-
throw error;
|
|
1106
|
+
const embedding = await runtime.useModel(ModelType7.TEXT_EMBEDDING, {
|
|
1107
|
+
text: "Hello, world!"
|
|
1108
|
+
});
|
|
1109
|
+
if (!Array.isArray(embedding) || embedding.length === 0) {
|
|
1110
|
+
throw new Error("Embedding should return a non-empty array");
|
|
721
1111
|
}
|
|
1112
|
+
logger11.info(`[OpenAI Test] Generated embedding with ${embedding.length} dimensions`);
|
|
722
1113
|
}
|
|
723
1114
|
},
|
|
724
1115
|
{
|
|
725
|
-
name: "
|
|
1116
|
+
name: "openai_test_text_small",
|
|
726
1117
|
fn: async (runtime) => {
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
throw new Error("Failed to generate text");
|
|
733
|
-
}
|
|
734
|
-
logger10.log({ text }, "generated with test_text_large");
|
|
735
|
-
} catch (error) {
|
|
736
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
737
|
-
logger10.error(`Error in test_text_large: ${message}`);
|
|
738
|
-
throw error;
|
|
1118
|
+
const text = await runtime.useModel(ModelType7.TEXT_SMALL, {
|
|
1119
|
+
prompt: "Say hello in exactly 5 words."
|
|
1120
|
+
});
|
|
1121
|
+
if (typeof text !== "string" || text.length === 0) {
|
|
1122
|
+
throw new Error("TEXT_SMALL should return non-empty string");
|
|
739
1123
|
}
|
|
1124
|
+
logger11.info(`[OpenAI Test] TEXT_SMALL generated: "${text.substring(0, 50)}..."`);
|
|
740
1125
|
}
|
|
741
1126
|
},
|
|
742
1127
|
{
|
|
743
|
-
name: "
|
|
1128
|
+
name: "openai_test_text_large",
|
|
744
1129
|
fn: async (runtime) => {
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
throw new Error("Failed to generate text");
|
|
751
|
-
}
|
|
752
|
-
logger10.log({ text }, "generated with test_text_small");
|
|
753
|
-
} catch (error) {
|
|
754
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
755
|
-
logger10.error(`Error in test_text_small: ${message}`);
|
|
756
|
-
throw error;
|
|
1130
|
+
const text = await runtime.useModel(ModelType7.TEXT_LARGE, {
|
|
1131
|
+
prompt: "Explain quantum computing in 2 sentences."
|
|
1132
|
+
});
|
|
1133
|
+
if (typeof text !== "string" || text.length === 0) {
|
|
1134
|
+
throw new Error("TEXT_LARGE should return non-empty string");
|
|
757
1135
|
}
|
|
1136
|
+
logger11.info(`[OpenAI Test] TEXT_LARGE generated: "${text.substring(0, 50)}..."`);
|
|
758
1137
|
}
|
|
759
1138
|
},
|
|
760
1139
|
{
|
|
761
|
-
name: "
|
|
1140
|
+
name: "openai_test_tokenizer_roundtrip",
|
|
762
1141
|
fn: async (runtime) => {
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
1142
|
+
const originalText = "Hello, tokenizer test!";
|
|
1143
|
+
const tokens = await runtime.useModel(ModelType7.TEXT_TOKENIZER_ENCODE, {
|
|
1144
|
+
prompt: originalText,
|
|
1145
|
+
modelType: ModelType7.TEXT_SMALL
|
|
1146
|
+
});
|
|
1147
|
+
if (!Array.isArray(tokens) || tokens.length === 0) {
|
|
1148
|
+
throw new Error("Tokenization should return non-empty token array");
|
|
1149
|
+
}
|
|
1150
|
+
const decodedText = await runtime.useModel(ModelType7.TEXT_TOKENIZER_DECODE, {
|
|
1151
|
+
tokens,
|
|
1152
|
+
modelType: ModelType7.TEXT_SMALL
|
|
1153
|
+
});
|
|
1154
|
+
if (decodedText !== originalText) {
|
|
1155
|
+
throw new Error(`Tokenizer roundtrip failed: expected "${originalText}", got "${decodedText}"`);
|
|
775
1156
|
}
|
|
1157
|
+
logger11.info(`[OpenAI Test] Tokenizer roundtrip successful (${tokens.length} tokens)`);
|
|
776
1158
|
}
|
|
777
1159
|
},
|
|
778
1160
|
{
|
|
779
|
-
name: "
|
|
1161
|
+
name: "openai_test_streaming",
|
|
780
1162
|
fn: async (runtime) => {
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
} else {
|
|
788
|
-
logger10.error("Invalid image description result format:", result);
|
|
789
|
-
}
|
|
790
|
-
} catch (e) {
|
|
791
|
-
const message = e instanceof Error ? e.message : String(e);
|
|
792
|
-
logger10.error(`Error in image description test: ${message}`);
|
|
1163
|
+
const chunks = [];
|
|
1164
|
+
const result = await runtime.useModel(ModelType7.TEXT_LARGE, {
|
|
1165
|
+
prompt: "Count from 1 to 5, one number per line.",
|
|
1166
|
+
stream: true,
|
|
1167
|
+
onStreamChunk: (chunk) => {
|
|
1168
|
+
chunks.push(chunk);
|
|
793
1169
|
}
|
|
794
|
-
}
|
|
795
|
-
|
|
796
|
-
|
|
1170
|
+
});
|
|
1171
|
+
if (typeof result !== "string" || result.length === 0) {
|
|
1172
|
+
throw new Error("Streaming should return non-empty result");
|
|
797
1173
|
}
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
{
|
|
801
|
-
name: "openai_test_transcription",
|
|
802
|
-
fn: async (runtime) => {
|
|
803
|
-
logger10.log("openai_test_transcription");
|
|
804
|
-
try {
|
|
805
|
-
const response = await fetch("https://upload.wikimedia.org/wikipedia/en/4/40/Chris_Benoit_Voice_Message.ogg");
|
|
806
|
-
const arrayBuffer = await response.arrayBuffer();
|
|
807
|
-
const transcription = await runtime.useModel(ModelType7.TRANSCRIPTION, Buffer.from(new Uint8Array(arrayBuffer)));
|
|
808
|
-
logger10.log({ transcription }, "generated with test_transcription");
|
|
809
|
-
} catch (error) {
|
|
810
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
811
|
-
logger10.error(`Error in test_transcription: ${message}`);
|
|
812
|
-
throw error;
|
|
1174
|
+
if (chunks.length === 0) {
|
|
1175
|
+
throw new Error("No streaming chunks received");
|
|
813
1176
|
}
|
|
1177
|
+
logger11.info(`[OpenAI Test] Streaming test: ${chunks.length} chunks received`);
|
|
814
1178
|
}
|
|
815
1179
|
},
|
|
816
1180
|
{
|
|
817
|
-
name: "
|
|
1181
|
+
name: "openai_test_image_description",
|
|
818
1182
|
fn: async (runtime) => {
|
|
819
|
-
const
|
|
820
|
-
const
|
|
821
|
-
if (!
|
|
822
|
-
throw new Error("
|
|
1183
|
+
const testImageUrl = "https://upload.wikimedia.org/wikipedia/commons/thumb/a/a7/Camponotus_flavomarginatus_ant.jpg/440px-Camponotus_flavomarginatus_ant.jpg";
|
|
1184
|
+
const result = await runtime.useModel(ModelType7.IMAGE_DESCRIPTION, testImageUrl);
|
|
1185
|
+
if (!result || typeof result !== "object" || !("title" in result) || !("description" in result)) {
|
|
1186
|
+
throw new Error("Image description should return { title, description }");
|
|
823
1187
|
}
|
|
824
|
-
|
|
1188
|
+
logger11.info(`[OpenAI Test] Image described: "${result.title}"`);
|
|
825
1189
|
}
|
|
826
1190
|
},
|
|
827
1191
|
{
|
|
828
|
-
name: "
|
|
1192
|
+
name: "openai_test_transcription",
|
|
829
1193
|
fn: async (runtime) => {
|
|
830
|
-
const
|
|
831
|
-
const
|
|
832
|
-
const
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
throw new Error(`Decoded text does not match original. Expected "${prompt}", got "${decodedText}"`);
|
|
1194
|
+
const audioUrl = "https://upload.wikimedia.org/wikipedia/commons/2/25/En-Open_Source.ogg";
|
|
1195
|
+
const response = await fetch(audioUrl);
|
|
1196
|
+
const arrayBuffer = await response.arrayBuffer();
|
|
1197
|
+
const audioBuffer = Buffer.from(new Uint8Array(arrayBuffer));
|
|
1198
|
+
const transcription = await runtime.useModel(ModelType7.TRANSCRIPTION, audioBuffer);
|
|
1199
|
+
if (typeof transcription !== "string") {
|
|
1200
|
+
throw new Error("Transcription should return a string");
|
|
838
1201
|
}
|
|
839
|
-
|
|
1202
|
+
logger11.info(`[OpenAI Test] Transcription: "${transcription.substring(0, 50)}..."`);
|
|
840
1203
|
}
|
|
841
1204
|
},
|
|
842
1205
|
{
|
|
843
1206
|
name: "openai_test_text_to_speech",
|
|
844
1207
|
fn: async (runtime) => {
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
throw new Error("Failed to generate speech");
|
|
851
|
-
}
|
|
852
|
-
logger10.log("Generated speech successfully");
|
|
853
|
-
} catch (error) {
|
|
854
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
855
|
-
logger10.error(`Error in openai_test_text_to_speech: ${message}`);
|
|
856
|
-
throw error;
|
|
1208
|
+
const audioData = await runtime.useModel(ModelType7.TEXT_TO_SPEECH, {
|
|
1209
|
+
text: "Hello, this is a text-to-speech test."
|
|
1210
|
+
});
|
|
1211
|
+
if (!(audioData instanceof ArrayBuffer) || audioData.byteLength === 0) {
|
|
1212
|
+
throw new Error("TTS should return non-empty ArrayBuffer");
|
|
857
1213
|
}
|
|
1214
|
+
logger11.info(`[OpenAI Test] TTS generated ${audioData.byteLength} bytes of audio`);
|
|
858
1215
|
}
|
|
859
1216
|
},
|
|
860
1217
|
{
|
|
861
|
-
name: "
|
|
1218
|
+
name: "openai_test_object_generation",
|
|
862
1219
|
fn: async (runtime) => {
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
throw new Error("Text generation returned empty result");
|
|
869
|
-
}
|
|
870
|
-
logger10.log({ result }, "Text generation test completed");
|
|
871
|
-
} catch (error) {
|
|
872
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
873
|
-
logger10.error(`Error in openai_test_text_generation_large: ${message}`);
|
|
874
|
-
throw error;
|
|
1220
|
+
const result = await runtime.useModel(ModelType7.OBJECT_SMALL, {
|
|
1221
|
+
prompt: "Return a JSON object with exactly these fields: name (string), age (number), active (boolean)"
|
|
1222
|
+
});
|
|
1223
|
+
if (!result || typeof result !== "object") {
|
|
1224
|
+
throw new Error("Object generation should return an object");
|
|
875
1225
|
}
|
|
1226
|
+
logger11.info(`[OpenAI Test] Object generated: ${JSON.stringify(result).substring(0, 100)}`);
|
|
876
1227
|
}
|
|
877
1228
|
},
|
|
878
1229
|
{
|
|
879
|
-
name: "
|
|
1230
|
+
name: "openai_test_research",
|
|
880
1231
|
fn: async (runtime) => {
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
}
|
|
892
|
-
if (chunks.length === 0) {
|
|
893
|
-
throw new Error("No streaming chunks received");
|
|
894
|
-
}
|
|
895
|
-
logger10.log({ chunks: chunks.length, result: result.substring(0, 50) }, "Streaming test completed");
|
|
896
|
-
} catch (error) {
|
|
897
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
898
|
-
logger10.error(`Error in openai_test_streaming: ${message}`);
|
|
899
|
-
throw error;
|
|
1232
|
+
const result = await runtime.useModel(ModelType7.RESEARCH, {
|
|
1233
|
+
input: "What is the current date and time?",
|
|
1234
|
+
tools: [{ type: "web_search_preview" }],
|
|
1235
|
+
maxToolCalls: 3
|
|
1236
|
+
});
|
|
1237
|
+
if (!result || typeof result !== "object" || !("text" in result)) {
|
|
1238
|
+
throw new Error("Research should return an object with text property");
|
|
1239
|
+
}
|
|
1240
|
+
if (typeof result.text !== "string" || result.text.length === 0) {
|
|
1241
|
+
throw new Error("Research result text should be a non-empty string");
|
|
900
1242
|
}
|
|
1243
|
+
logger11.info(`[OpenAI Test] Research completed. Text length: ${result.text.length}, Annotations: ${result.annotations?.length ?? 0}`);
|
|
901
1244
|
}
|
|
902
1245
|
}
|
|
903
1246
|
]
|
|
904
1247
|
}
|
|
905
1248
|
]
|
|
906
1249
|
};
|
|
907
|
-
var
|
|
1250
|
+
var typescript_default = openaiPlugin;
|
|
908
1251
|
export {
|
|
909
1252
|
openaiPlugin,
|
|
910
|
-
|
|
1253
|
+
typescript_default as default
|
|
911
1254
|
};
|
|
912
1255
|
|
|
913
|
-
//# debugId=
|
|
1256
|
+
//# debugId=AE97436E4960EAED64756E2164756E21
|