@elizaos/plugin-openai 2.0.3-beta.2 → 2.0.3-beta.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/index.browser.js +3 -0
- package/dist/browser/index.browser.js.map +23 -0
- package/dist/browser/index.d.ts +2 -0
- package/dist/index.browser.d.ts +4 -0
- package/dist/index.browser.d.ts.map +1 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.node.d.ts +4 -0
- package/dist/index.node.d.ts.map +1 -0
- package/dist/init.d.ts +4 -0
- package/dist/init.d.ts.map +1 -0
- package/dist/models/audio.d.ts +9 -0
- package/dist/models/audio.d.ts.map +1 -0
- package/dist/models/embedding.d.ts +3 -0
- package/dist/models/embedding.d.ts.map +1 -0
- package/dist/models/image.d.ts +5 -0
- package/dist/models/image.d.ts.map +1 -0
- package/dist/models/index.d.ts +7 -0
- package/dist/models/index.d.ts.map +1 -0
- package/dist/models/research.d.ts +34 -0
- package/dist/models/research.d.ts.map +1 -0
- package/dist/models/text.d.ts +62 -0
- package/dist/models/text.d.ts.map +1 -0
- package/dist/models/tokenizer.d.ts +4 -0
- package/dist/models/tokenizer.d.ts.map +1 -0
- package/dist/node/index.d.ts +2 -0
- package/dist/node/index.node.js +2057 -0
- package/dist/node/index.node.js.map +23 -0
- package/dist/providers/index.d.ts +2 -0
- package/dist/providers/index.d.ts.map +1 -0
- package/dist/providers/openai.d.ts +4 -0
- package/dist/providers/openai.d.ts.map +1 -0
- package/dist/types/index.d.ts +367 -0
- package/dist/types/index.d.ts.map +1 -0
- package/dist/utils/audio.d.ts +6 -0
- package/dist/utils/audio.d.ts.map +1 -0
- package/dist/utils/config.d.ts +47 -0
- package/dist/utils/config.d.ts.map +1 -0
- package/dist/utils/events.d.ts +21 -0
- package/dist/utils/events.d.ts.map +1 -0
- package/dist/utils/tokenization.d.ts +6 -0
- package/dist/utils/tokenization.d.ts.map +1 -0
- package/package.json +7 -4
- package/registry-entry.json +189 -0
|
@@ -0,0 +1,2057 @@
|
|
|
1
|
+
// index.ts
|
|
2
|
+
import { logger as logger9, ModelType as ModelType5 } from "@elizaos/core";
|
|
3
|
+
|
|
4
|
+
// init.ts
|
|
5
|
+
import { logger as logger2 } from "@elizaos/core";
|
|
6
|
+
|
|
7
|
+
// utils/config.ts
|
|
8
|
+
import { logger } from "@elizaos/core";
|
|
9
|
+
function getEnvValue(key) {
|
|
10
|
+
if (typeof process === "undefined" || !process.env) {
|
|
11
|
+
return;
|
|
12
|
+
}
|
|
13
|
+
const value = process.env[key];
|
|
14
|
+
return value === undefined ? undefined : String(value);
|
|
15
|
+
}
|
|
16
|
+
function getSetting(runtime, key, defaultValue) {
|
|
17
|
+
const value = runtime.getSetting(key);
|
|
18
|
+
if (value !== undefined && value !== null) {
|
|
19
|
+
return String(value);
|
|
20
|
+
}
|
|
21
|
+
return getEnvValue(key) ?? defaultValue;
|
|
22
|
+
}
|
|
23
|
+
function getNumericSetting(runtime, key, defaultValue) {
|
|
24
|
+
const value = getSetting(runtime, key);
|
|
25
|
+
if (value === undefined) {
|
|
26
|
+
return defaultValue;
|
|
27
|
+
}
|
|
28
|
+
const parsed = Number.parseInt(value, 10);
|
|
29
|
+
if (!Number.isFinite(parsed)) {
|
|
30
|
+
throw new Error(`Setting '${key}' must be a valid integer, got: ${value}`);
|
|
31
|
+
}
|
|
32
|
+
return parsed;
|
|
33
|
+
}
|
|
34
|
+
function getBooleanSetting(runtime, key, defaultValue) {
|
|
35
|
+
const value = getSetting(runtime, key);
|
|
36
|
+
if (value === undefined) {
|
|
37
|
+
return defaultValue;
|
|
38
|
+
}
|
|
39
|
+
const normalized = value.toLowerCase();
|
|
40
|
+
return normalized === "true" || normalized === "1" || normalized === "yes";
|
|
41
|
+
}
|
|
42
|
+
function isBrowser() {
|
|
43
|
+
return typeof globalThis !== "undefined" && typeof globalThis.document !== "undefined";
|
|
44
|
+
}
|
|
45
|
+
function isProxyMode(runtime) {
|
|
46
|
+
return isBrowser() && !!getSetting(runtime, "OPENAI_BROWSER_BASE_URL");
|
|
47
|
+
}
|
|
48
|
+
function isCerebrasMode(runtime) {
|
|
49
|
+
const explicitProvider = getSetting(runtime, "ELIZA_PROVIDER");
|
|
50
|
+
if (explicitProvider && explicitProvider.toLowerCase() === "cerebras") {
|
|
51
|
+
return true;
|
|
52
|
+
}
|
|
53
|
+
const baseURL = getSetting(runtime, "OPENAI_BASE_URL");
|
|
54
|
+
if (baseURL && /(^|\.)cerebras\.ai(\/|$)/i.test(baseURL)) {
|
|
55
|
+
return true;
|
|
56
|
+
}
|
|
57
|
+
const cerebrasKey = getSetting(runtime, "CEREBRAS_API_KEY");
|
|
58
|
+
if (cerebrasKey && !getSetting(runtime, "OPENAI_API_KEY") && !getSetting(runtime, "OPENAI_BASE_URL")) {
|
|
59
|
+
return true;
|
|
60
|
+
}
|
|
61
|
+
return false;
|
|
62
|
+
}
|
|
63
|
+
function isEvoLinkMode(runtime) {
|
|
64
|
+
const explicitProvider = getSetting(runtime, "ELIZA_PROVIDER");
|
|
65
|
+
if (explicitProvider && explicitProvider.toLowerCase() === "evolink") {
|
|
66
|
+
return true;
|
|
67
|
+
}
|
|
68
|
+
const baseURL = getSetting(runtime, "OPENAI_BASE_URL");
|
|
69
|
+
if (baseURL && /(^|\.)evolink\.ai(\/|$)/i.test(baseURL)) {
|
|
70
|
+
return true;
|
|
71
|
+
}
|
|
72
|
+
const evolinkKey = getSetting(runtime, "EVOLINK_API_KEY");
|
|
73
|
+
if (evolinkKey && !getSetting(runtime, "OPENAI_API_KEY") && !getSetting(runtime, "OPENAI_BASE_URL")) {
|
|
74
|
+
return true;
|
|
75
|
+
}
|
|
76
|
+
return false;
|
|
77
|
+
}
|
|
78
|
+
function getApiKey(runtime) {
|
|
79
|
+
if (isCerebrasMode(runtime)) {
|
|
80
|
+
const cerebrasKey = getSetting(runtime, "CEREBRAS_API_KEY");
|
|
81
|
+
if (cerebrasKey) {
|
|
82
|
+
return cerebrasKey;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
if (isEvoLinkMode(runtime)) {
|
|
86
|
+
const evolinkKey = getSetting(runtime, "EVOLINK_API_KEY");
|
|
87
|
+
if (evolinkKey) {
|
|
88
|
+
return evolinkKey;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
return getSetting(runtime, "OPENAI_API_KEY");
|
|
92
|
+
}
|
|
93
|
+
function getEmbeddingApiKey(runtime) {
|
|
94
|
+
const embeddingApiKey = getSetting(runtime, "OPENAI_EMBEDDING_API_KEY");
|
|
95
|
+
if (embeddingApiKey) {
|
|
96
|
+
logger.debug("[OpenAI] Using specific embedding API key");
|
|
97
|
+
return embeddingApiKey;
|
|
98
|
+
}
|
|
99
|
+
logger.debug("[OpenAI] Falling back to general API key for embeddings");
|
|
100
|
+
return getApiKey(runtime);
|
|
101
|
+
}
|
|
102
|
+
function getAuthHeader(runtime, forEmbedding = false) {
|
|
103
|
+
if (isBrowser() && !getBooleanSetting(runtime, "OPENAI_ALLOW_BROWSER_API_KEY", false)) {
|
|
104
|
+
return {};
|
|
105
|
+
}
|
|
106
|
+
const key = forEmbedding ? getEmbeddingApiKey(runtime) : getApiKey(runtime);
|
|
107
|
+
return key ? { Authorization: `Bearer ${key}` } : {};
|
|
108
|
+
}
|
|
109
|
+
function authHeaderForKey(runtime, key) {
|
|
110
|
+
if (isBrowser() && !getBooleanSetting(runtime, "OPENAI_ALLOW_BROWSER_API_KEY", false)) {
|
|
111
|
+
return {};
|
|
112
|
+
}
|
|
113
|
+
return key ? { Authorization: `Bearer ${key}` } : {};
|
|
114
|
+
}
|
|
115
|
+
function getMockBaseURL() {
|
|
116
|
+
const base = getEnvValue("ELIZA_MOCK_OPENAI_BASE")?.trim();
|
|
117
|
+
return base ? base : undefined;
|
|
118
|
+
}
|
|
119
|
+
function getBaseURL(runtime) {
|
|
120
|
+
const browserURL = getSetting(runtime, "OPENAI_BROWSER_BASE_URL");
|
|
121
|
+
const mockBaseURL = getMockBaseURL();
|
|
122
|
+
const cerebrasBaseURL = isCerebrasMode(runtime) && !getSetting(runtime, "OPENAI_BASE_URL") ? getSetting(runtime, "CEREBRAS_BASE_URL") ?? "https://api.cerebras.ai/v1" : undefined;
|
|
123
|
+
const evolinkBaseURL = isEvoLinkMode(runtime) && !getSetting(runtime, "OPENAI_BASE_URL") ? getSetting(runtime, "EVOLINK_BASE_URL") ?? "https://direct.evolink.ai/v1" : undefined;
|
|
124
|
+
const baseURL = isBrowser() && browserURL ? browserURL : mockBaseURL ?? getSetting(runtime, "OPENAI_BASE_URL") ?? cerebrasBaseURL ?? evolinkBaseURL ?? "https://api.openai.com/v1";
|
|
125
|
+
logger.debug(`[OpenAI] Base URL: ${baseURL}`);
|
|
126
|
+
return baseURL;
|
|
127
|
+
}
|
|
128
|
+
function getEmbeddingBaseURL(runtime) {
|
|
129
|
+
const embeddingURL = isBrowser() ? getSetting(runtime, "OPENAI_BROWSER_EMBEDDING_URL") ?? getSetting(runtime, "OPENAI_BROWSER_BASE_URL") : getSetting(runtime, "OPENAI_EMBEDDING_URL");
|
|
130
|
+
if (embeddingURL) {
|
|
131
|
+
logger.debug(`[OpenAI] Using embedding base URL: ${embeddingURL}`);
|
|
132
|
+
return embeddingURL;
|
|
133
|
+
}
|
|
134
|
+
logger.debug("[OpenAI] Falling back to general base URL for embeddings");
|
|
135
|
+
return getBaseURL(runtime);
|
|
136
|
+
}
|
|
137
|
+
function getImageDescriptionApiKey(runtime) {
|
|
138
|
+
const imageDescriptionApiKey = getSetting(runtime, "OPENAI_IMAGE_DESCRIPTION_API_KEY");
|
|
139
|
+
if (imageDescriptionApiKey) {
|
|
140
|
+
return imageDescriptionApiKey;
|
|
141
|
+
}
|
|
142
|
+
const imageDescriptionURL = getSetting(runtime, "OPENAI_IMAGE_DESCRIPTION_BASE_URL");
|
|
143
|
+
if (imageDescriptionURL && /(^|\.)openai\.com(\/|$)/i.test(imageDescriptionURL)) {
|
|
144
|
+
return getSetting(runtime, "OPENAI_API_KEY");
|
|
145
|
+
}
|
|
146
|
+
return getApiKey(runtime);
|
|
147
|
+
}
|
|
148
|
+
function getImageDescriptionAuthHeader(runtime) {
|
|
149
|
+
return authHeaderForKey(runtime, getImageDescriptionApiKey(runtime));
|
|
150
|
+
}
|
|
151
|
+
function getImageDescriptionBaseURL(runtime) {
|
|
152
|
+
const imageDescriptionURL = getSetting(runtime, "OPENAI_IMAGE_DESCRIPTION_BASE_URL");
|
|
153
|
+
if (imageDescriptionURL) {
|
|
154
|
+
logger.debug(`[OpenAI] Using image-description base URL: ${imageDescriptionURL}`);
|
|
155
|
+
return imageDescriptionURL;
|
|
156
|
+
}
|
|
157
|
+
return getBaseURL(runtime);
|
|
158
|
+
}
|
|
159
|
+
function getCerebrasModel(runtime) {
|
|
160
|
+
return isCerebrasMode(runtime) ? getSetting(runtime, "CEREBRAS_MODEL") : undefined;
|
|
161
|
+
}
|
|
162
|
+
function getEvoLinkModel(runtime) {
|
|
163
|
+
return isEvoLinkMode(runtime) ? getSetting(runtime, "EVOLINK_MODEL") ?? "gpt-5.2" : undefined;
|
|
164
|
+
}
|
|
165
|
+
function getSmallModel(runtime) {
|
|
166
|
+
return getSetting(runtime, "OPENAI_SMALL_MODEL") ?? getCerebrasModel(runtime) ?? getEvoLinkModel(runtime) ?? getSetting(runtime, "SMALL_MODEL") ?? "gpt-5.4-mini";
|
|
167
|
+
}
|
|
168
|
+
function getNanoModel(runtime) {
|
|
169
|
+
return getSetting(runtime, "OPENAI_NANO_MODEL") ?? getCerebrasModel(runtime) ?? getEvoLinkModel(runtime) ?? getSetting(runtime, "NANO_MODEL") ?? getSmallModel(runtime);
|
|
170
|
+
}
|
|
171
|
+
function getMediumModel(runtime) {
|
|
172
|
+
return getSetting(runtime, "OPENAI_MEDIUM_MODEL") ?? getCerebrasModel(runtime) ?? getEvoLinkModel(runtime) ?? getSetting(runtime, "MEDIUM_MODEL") ?? getSmallModel(runtime);
|
|
173
|
+
}
|
|
174
|
+
function getLargeModel(runtime) {
|
|
175
|
+
return getSetting(runtime, "OPENAI_LARGE_MODEL") ?? getCerebrasModel(runtime) ?? getEvoLinkModel(runtime) ?? getSetting(runtime, "LARGE_MODEL") ?? "gpt-5";
|
|
176
|
+
}
|
|
177
|
+
function getMegaModel(runtime) {
|
|
178
|
+
return getSetting(runtime, "OPENAI_MEGA_MODEL") ?? getSetting(runtime, "MEGA_MODEL") ?? getLargeModel(runtime);
|
|
179
|
+
}
|
|
180
|
+
function getResponseHandlerModel(runtime) {
|
|
181
|
+
return getSetting(runtime, "OPENAI_RESPONSE_HANDLER_MODEL") ?? getSetting(runtime, "OPENAI_SHOULD_RESPOND_MODEL") ?? getCerebrasModel(runtime) ?? getEvoLinkModel(runtime) ?? getSetting(runtime, "RESPONSE_HANDLER_MODEL") ?? getSetting(runtime, "SHOULD_RESPOND_MODEL") ?? getSmallModel(runtime);
|
|
182
|
+
}
|
|
183
|
+
function getActionPlannerModel(runtime) {
|
|
184
|
+
return getSetting(runtime, "OPENAI_ACTION_PLANNER_MODEL") ?? getSetting(runtime, "OPENAI_PLANNER_MODEL") ?? getCerebrasModel(runtime) ?? getEvoLinkModel(runtime) ?? getSetting(runtime, "ACTION_PLANNER_MODEL") ?? getSetting(runtime, "PLANNER_MODEL") ?? getMediumModel(runtime);
|
|
185
|
+
}
|
|
186
|
+
function getEmbeddingModel(runtime) {
|
|
187
|
+
return getSetting(runtime, "OPENAI_EMBEDDING_MODEL") ?? "text-embedding-3-small";
|
|
188
|
+
}
|
|
189
|
+
function getImageDescriptionModel(runtime) {
|
|
190
|
+
return getSetting(runtime, "OPENAI_IMAGE_DESCRIPTION_MODEL") ?? "gpt-5-mini";
|
|
191
|
+
}
|
|
192
|
+
function getTranscriptionModel(runtime) {
|
|
193
|
+
return getSetting(runtime, "OPENAI_TRANSCRIPTION_MODEL") ?? "gpt-5-mini-transcribe";
|
|
194
|
+
}
|
|
195
|
+
function getTTSModel(runtime) {
|
|
196
|
+
return getSetting(runtime, "OPENAI_TTS_MODEL") ?? "gpt-5-mini-tts";
|
|
197
|
+
}
|
|
198
|
+
function getTTSVoice(runtime) {
|
|
199
|
+
return getSetting(runtime, "OPENAI_TTS_VOICE") ?? "nova";
|
|
200
|
+
}
|
|
201
|
+
function getTTSInstructions(runtime) {
|
|
202
|
+
return getSetting(runtime, "OPENAI_TTS_INSTRUCTIONS") ?? "";
|
|
203
|
+
}
|
|
204
|
+
function getImageModel(runtime) {
|
|
205
|
+
return getSetting(runtime, "OPENAI_IMAGE_MODEL") ?? "dall-e-3";
|
|
206
|
+
}
|
|
207
|
+
function getExperimentalTelemetry(runtime) {
|
|
208
|
+
return getBooleanSetting(runtime, "OPENAI_EXPERIMENTAL_TELEMETRY", false);
|
|
209
|
+
}
|
|
210
|
+
function getEmbeddingDimensions(runtime) {
|
|
211
|
+
return getNumericSetting(runtime, "OPENAI_EMBEDDING_DIMENSIONS", 1536);
|
|
212
|
+
}
|
|
213
|
+
function getImageDescriptionMaxTokens(runtime) {
|
|
214
|
+
return getNumericSetting(runtime, "OPENAI_IMAGE_DESCRIPTION_MAX_TOKENS", 8192);
|
|
215
|
+
}
|
|
216
|
+
function getResearchModel(runtime) {
|
|
217
|
+
return getSetting(runtime, "OPENAI_RESEARCH_MODEL") ?? "o3-deep-research";
|
|
218
|
+
}
|
|
219
|
+
function getResearchTimeout(runtime) {
|
|
220
|
+
return getNumericSetting(runtime, "OPENAI_RESEARCH_TIMEOUT", 3600000);
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
// init.ts
|
|
224
|
+
globalThis.AI_SDK_LOG_WARNINGS ??= false;
|
|
225
|
+
function initializeOpenAI(_config, runtime) {
|
|
226
|
+
validateOpenAIConfiguration(runtime);
|
|
227
|
+
}
|
|
228
|
+
async function validateOpenAIConfiguration(runtime) {
|
|
229
|
+
if (isBrowser()) {
|
|
230
|
+
logger2.debug("[OpenAI] Skipping API validation in browser environment");
|
|
231
|
+
return;
|
|
232
|
+
}
|
|
233
|
+
const apiKey = getApiKey(runtime);
|
|
234
|
+
if (!apiKey) {
|
|
235
|
+
logger2.warn("[OpenAI] OPENAI_API_KEY is not configured. " + "OpenAI functionality will fail until a valid API key is provided.");
|
|
236
|
+
return;
|
|
237
|
+
}
|
|
238
|
+
try {
|
|
239
|
+
const baseURL = getBaseURL(runtime);
|
|
240
|
+
const response = await fetch(`${baseURL}/models`, {
|
|
241
|
+
headers: getAuthHeader(runtime)
|
|
242
|
+
});
|
|
243
|
+
if (!response.ok) {
|
|
244
|
+
logger2.warn(`[OpenAI] API key validation failed: ${response.status} ${response.statusText}. ` + "Please verify your OPENAI_API_KEY is correct.");
|
|
245
|
+
return;
|
|
246
|
+
}
|
|
247
|
+
} catch (error) {
|
|
248
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
249
|
+
logger2.warn(`[OpenAI] API validation error: ${message}. OpenAI functionality may be limited.`);
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
// models/audio.ts
|
|
254
|
+
import { logger as logger4, recordLlmCall } from "@elizaos/core";
|
|
255
|
+
|
|
256
|
+
// utils/audio.ts
|
|
257
|
+
import { logger as logger3 } from "@elizaos/core";
|
|
258
|
+
var MAGIC_BYTES = {
|
|
259
|
+
WAV: {
|
|
260
|
+
HEADER: [82, 73, 70, 70],
|
|
261
|
+
IDENTIFIER: [87, 65, 86, 69]
|
|
262
|
+
},
|
|
263
|
+
MP3_ID3: [73, 68, 51],
|
|
264
|
+
OGG: [79, 103, 103, 83],
|
|
265
|
+
FLAC: [102, 76, 97, 67],
|
|
266
|
+
FTYP: [102, 116, 121, 112],
|
|
267
|
+
WEBM_EBML: [26, 69, 223, 163]
|
|
268
|
+
};
|
|
269
|
+
var MIN_DETECTION_BUFFER_SIZE = 12;
|
|
270
|
+
function matchBytes(buffer, offset, expected) {
|
|
271
|
+
for (let i = 0;i < expected.length; i++) {
|
|
272
|
+
const expectedByte = expected[i];
|
|
273
|
+
if (expectedByte === undefined || buffer[offset + i] !== expectedByte) {
|
|
274
|
+
return false;
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
return true;
|
|
278
|
+
}
|
|
279
|
+
function detectAudioMimeType(buffer) {
|
|
280
|
+
if (buffer.length < MIN_DETECTION_BUFFER_SIZE) {
|
|
281
|
+
return "application/octet-stream";
|
|
282
|
+
}
|
|
283
|
+
if (matchBytes(buffer, 0, MAGIC_BYTES.WAV.HEADER) && matchBytes(buffer, 8, MAGIC_BYTES.WAV.IDENTIFIER)) {
|
|
284
|
+
return "audio/wav";
|
|
285
|
+
}
|
|
286
|
+
const firstByte = buffer[0];
|
|
287
|
+
const secondByte = buffer[1];
|
|
288
|
+
if (matchBytes(buffer, 0, MAGIC_BYTES.MP3_ID3) || firstByte === 255 && secondByte !== undefined && (secondByte & 224) === 224) {
|
|
289
|
+
return "audio/mpeg";
|
|
290
|
+
}
|
|
291
|
+
if (matchBytes(buffer, 0, MAGIC_BYTES.OGG)) {
|
|
292
|
+
return "audio/ogg";
|
|
293
|
+
}
|
|
294
|
+
if (matchBytes(buffer, 0, MAGIC_BYTES.FLAC)) {
|
|
295
|
+
return "audio/flac";
|
|
296
|
+
}
|
|
297
|
+
if (matchBytes(buffer, 4, MAGIC_BYTES.FTYP)) {
|
|
298
|
+
return "audio/mp4";
|
|
299
|
+
}
|
|
300
|
+
if (matchBytes(buffer, 0, MAGIC_BYTES.WEBM_EBML)) {
|
|
301
|
+
return "audio/webm";
|
|
302
|
+
}
|
|
303
|
+
logger3.warn("Could not detect audio format from buffer, using generic binary type");
|
|
304
|
+
return "application/octet-stream";
|
|
305
|
+
}
|
|
306
|
+
function getExtensionForMimeType(mimeType) {
|
|
307
|
+
switch (mimeType) {
|
|
308
|
+
case "audio/wav":
|
|
309
|
+
return "wav";
|
|
310
|
+
case "audio/mpeg":
|
|
311
|
+
return "mp3";
|
|
312
|
+
case "audio/ogg":
|
|
313
|
+
return "ogg";
|
|
314
|
+
case "audio/flac":
|
|
315
|
+
return "flac";
|
|
316
|
+
case "audio/mp4":
|
|
317
|
+
return "m4a";
|
|
318
|
+
case "audio/webm":
|
|
319
|
+
return "webm";
|
|
320
|
+
case "application/octet-stream":
|
|
321
|
+
return "bin";
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
function getFilenameForMimeType(mimeType) {
|
|
325
|
+
const ext = getExtensionForMimeType(mimeType);
|
|
326
|
+
return `recording.${ext}`;
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
// models/audio.ts
|
|
330
|
+
function isBlobOrFile(value) {
|
|
331
|
+
return value instanceof Blob || value instanceof File;
|
|
332
|
+
}
|
|
333
|
+
function isBuffer(value) {
|
|
334
|
+
return Buffer.isBuffer(value);
|
|
335
|
+
}
|
|
336
|
+
function isLocalTranscriptionParams(value) {
|
|
337
|
+
return typeof value === "object" && value !== null && "audio" in value && (isBlobOrFile(value.audio) || isBuffer(value.audio));
|
|
338
|
+
}
|
|
339
|
+
function isCoreTranscriptionParams(value) {
|
|
340
|
+
return typeof value === "object" && value !== null && "audioUrl" in value && typeof value.audioUrl === "string";
|
|
341
|
+
}
|
|
342
|
+
async function fetchAudioFromUrl(url) {
|
|
343
|
+
const response = await fetch(url);
|
|
344
|
+
if (!response.ok) {
|
|
345
|
+
throw new Error(`Failed to fetch audio from URL: ${response.status}`);
|
|
346
|
+
}
|
|
347
|
+
return response.blob();
|
|
348
|
+
}
|
|
349
|
+
async function handleTranscription(runtime, input) {
|
|
350
|
+
let modelName = getTranscriptionModel(runtime);
|
|
351
|
+
let blob;
|
|
352
|
+
let extraParams = {};
|
|
353
|
+
if (typeof input === "string") {
|
|
354
|
+
logger4.debug(`[OpenAI] Fetching audio from URL: ${input}`);
|
|
355
|
+
blob = await fetchAudioFromUrl(input);
|
|
356
|
+
} else if (isBlobOrFile(input)) {
|
|
357
|
+
blob = input;
|
|
358
|
+
} else if (isBuffer(input)) {
|
|
359
|
+
const mimeType2 = detectAudioMimeType(input);
|
|
360
|
+
logger4.debug(`[OpenAI] Auto-detected audio MIME type: ${mimeType2}`);
|
|
361
|
+
blob = new Blob([new Uint8Array(input)], { type: mimeType2 });
|
|
362
|
+
} else if (isLocalTranscriptionParams(input)) {
|
|
363
|
+
extraParams = input;
|
|
364
|
+
if (input.model) {
|
|
365
|
+
modelName = input.model;
|
|
366
|
+
}
|
|
367
|
+
if (isBuffer(input.audio)) {
|
|
368
|
+
const mimeType2 = input.mimeType ?? detectAudioMimeType(input.audio);
|
|
369
|
+
logger4.debug(`[OpenAI] Using MIME type: ${mimeType2}`);
|
|
370
|
+
blob = new Blob([new Uint8Array(input.audio)], { type: mimeType2 });
|
|
371
|
+
} else {
|
|
372
|
+
blob = input.audio;
|
|
373
|
+
}
|
|
374
|
+
} else if (isCoreTranscriptionParams(input)) {
|
|
375
|
+
logger4.debug(`[OpenAI] Fetching audio from URL: ${input.audioUrl}`);
|
|
376
|
+
blob = await fetchAudioFromUrl(input.audioUrl);
|
|
377
|
+
extraParams = { prompt: input.prompt };
|
|
378
|
+
} else {
|
|
379
|
+
throw new Error("TRANSCRIPTION expects Blob, File, Buffer, URL string, or TranscriptionParams object");
|
|
380
|
+
}
|
|
381
|
+
logger4.debug(`[OpenAI] Using TRANSCRIPTION model: ${modelName}`);
|
|
382
|
+
const mimeType = blob.type || "audio/webm";
|
|
383
|
+
const filename = blob.name || getFilenameForMimeType(mimeType.startsWith("audio/") ? mimeType : "audio/webm");
|
|
384
|
+
const formData = new FormData;
|
|
385
|
+
formData.append("file", blob, filename);
|
|
386
|
+
formData.append("model", modelName);
|
|
387
|
+
if (extraParams.language) {
|
|
388
|
+
formData.append("language", extraParams.language);
|
|
389
|
+
}
|
|
390
|
+
if (extraParams.responseFormat) {
|
|
391
|
+
formData.append("response_format", extraParams.responseFormat);
|
|
392
|
+
}
|
|
393
|
+
if (extraParams.prompt) {
|
|
394
|
+
formData.append("prompt", extraParams.prompt);
|
|
395
|
+
}
|
|
396
|
+
if (extraParams.temperature !== undefined) {
|
|
397
|
+
formData.append("temperature", String(extraParams.temperature));
|
|
398
|
+
}
|
|
399
|
+
if (extraParams.timestampGranularities) {
|
|
400
|
+
for (const granularity of extraParams.timestampGranularities) {
|
|
401
|
+
formData.append("timestamp_granularities[]", granularity);
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
const baseURL = getBaseURL(runtime);
|
|
405
|
+
const details = {
|
|
406
|
+
model: modelName,
|
|
407
|
+
systemPrompt: extraParams.prompt ?? "",
|
|
408
|
+
userPrompt: [
|
|
409
|
+
`audio transcription request: filename=${filename}`,
|
|
410
|
+
`mimeType=${mimeType}`,
|
|
411
|
+
extraParams.language ? `language=${extraParams.language}` : "",
|
|
412
|
+
extraParams.responseFormat ? `responseFormat=${extraParams.responseFormat}` : ""
|
|
413
|
+
].filter(Boolean).join(" "),
|
|
414
|
+
temperature: extraParams.temperature ?? 0,
|
|
415
|
+
maxTokens: 0,
|
|
416
|
+
purpose: "external_llm",
|
|
417
|
+
actionType: "openai.audio.transcriptions.create"
|
|
418
|
+
};
|
|
419
|
+
const data = await recordLlmCall(runtime, details, async () => {
|
|
420
|
+
const response = await fetch(`${baseURL}/audio/transcriptions`, {
|
|
421
|
+
method: "POST",
|
|
422
|
+
headers: getAuthHeader(runtime),
|
|
423
|
+
body: formData
|
|
424
|
+
});
|
|
425
|
+
if (!response.ok) {
|
|
426
|
+
const errorText = await response.text().catch(() => "Unknown error");
|
|
427
|
+
throw new Error(`OpenAI transcription failed: ${response.status} ${response.statusText} - ${errorText}`);
|
|
428
|
+
}
|
|
429
|
+
const result = await response.json();
|
|
430
|
+
details.response = result.text;
|
|
431
|
+
return result;
|
|
432
|
+
});
|
|
433
|
+
return data.text;
|
|
434
|
+
}
|
|
435
|
+
async function handleTextToSpeech(runtime, input) {
|
|
436
|
+
let text;
|
|
437
|
+
let voice;
|
|
438
|
+
let format = "mp3";
|
|
439
|
+
let model;
|
|
440
|
+
let instructions;
|
|
441
|
+
if (typeof input === "string") {
|
|
442
|
+
text = input;
|
|
443
|
+
voice = undefined;
|
|
444
|
+
} else {
|
|
445
|
+
text = input.text;
|
|
446
|
+
voice = input.voice;
|
|
447
|
+
if ("format" in input && input.format) {
|
|
448
|
+
format = input.format;
|
|
449
|
+
}
|
|
450
|
+
if ("model" in input && input.model) {
|
|
451
|
+
model = input.model;
|
|
452
|
+
}
|
|
453
|
+
if ("instructions" in input && input.instructions) {
|
|
454
|
+
instructions = input.instructions;
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
model = model ?? getTTSModel(runtime);
|
|
458
|
+
voice = voice ?? getTTSVoice(runtime);
|
|
459
|
+
instructions = instructions ?? getTTSInstructions(runtime);
|
|
460
|
+
logger4.debug(`[OpenAI] Using TEXT_TO_SPEECH model: ${model}`);
|
|
461
|
+
if (!text || text.trim().length === 0) {
|
|
462
|
+
throw new Error("TEXT_TO_SPEECH requires non-empty text");
|
|
463
|
+
}
|
|
464
|
+
if (text.length > 4096) {
|
|
465
|
+
throw new Error("TEXT_TO_SPEECH text exceeds 4096 character limit");
|
|
466
|
+
}
|
|
467
|
+
const validVoices = ["alloy", "echo", "fable", "onyx", "nova", "shimmer"];
|
|
468
|
+
if (voice && !validVoices.includes(voice)) {
|
|
469
|
+
throw new Error(`Invalid voice: ${voice}. Must be one of: ${validVoices.join(", ")}`);
|
|
470
|
+
}
|
|
471
|
+
const baseURL = getBaseURL(runtime);
|
|
472
|
+
const requestBody = {
|
|
473
|
+
model,
|
|
474
|
+
voice,
|
|
475
|
+
input: text,
|
|
476
|
+
response_format: format
|
|
477
|
+
};
|
|
478
|
+
if (instructions && instructions.length > 0) {
|
|
479
|
+
requestBody.instructions = instructions;
|
|
480
|
+
}
|
|
481
|
+
const details = {
|
|
482
|
+
model,
|
|
483
|
+
systemPrompt: instructions,
|
|
484
|
+
userPrompt: text,
|
|
485
|
+
temperature: 0,
|
|
486
|
+
maxTokens: 0,
|
|
487
|
+
purpose: "external_llm",
|
|
488
|
+
actionType: "openai.audio.speech.create"
|
|
489
|
+
};
|
|
490
|
+
return recordLlmCall(runtime, details, async () => {
|
|
491
|
+
const response = await fetch(`${baseURL}/audio/speech`, {
|
|
492
|
+
method: "POST",
|
|
493
|
+
headers: {
|
|
494
|
+
...getAuthHeader(runtime),
|
|
495
|
+
"Content-Type": "application/json",
|
|
496
|
+
...format === "mp3" ? { Accept: "audio/mpeg" } : {}
|
|
497
|
+
},
|
|
498
|
+
body: JSON.stringify(requestBody)
|
|
499
|
+
});
|
|
500
|
+
if (!response.ok) {
|
|
501
|
+
const errorText = await response.text().catch(() => "Unknown error");
|
|
502
|
+
throw new Error(`OpenAI TTS failed: ${response.status} ${response.statusText} - ${errorText}`);
|
|
503
|
+
}
|
|
504
|
+
const audioBuffer = await response.arrayBuffer();
|
|
505
|
+
details.response = `[audio bytes=${audioBuffer.byteLength} format=${format}]`;
|
|
506
|
+
return audioBuffer;
|
|
507
|
+
});
|
|
508
|
+
}
|
|
509
|
+
// models/embedding.ts
|
|
510
|
+
import { logger as logger5, ModelType, VECTOR_DIMS } from "@elizaos/core";
|
|
511
|
+
|
|
512
|
+
// utils/events.ts
|
|
513
|
+
import { EventType } from "@elizaos/core";
|
|
514
|
+
var MAX_PROMPT_LENGTH = 200;
|
|
515
|
+
function truncatePrompt(prompt) {
|
|
516
|
+
if (prompt.length <= MAX_PROMPT_LENGTH) {
|
|
517
|
+
return prompt;
|
|
518
|
+
}
|
|
519
|
+
return `${prompt.slice(0, MAX_PROMPT_LENGTH)}…`;
|
|
520
|
+
}
|
|
521
|
+
function normalizeUsage(usage) {
|
|
522
|
+
if ("promptTokens" in usage) {
|
|
523
|
+
const promptTokensDetails = "promptTokensDetails" in usage ? usage.promptTokensDetails : undefined;
|
|
524
|
+
const cachedPromptTokens = usage.cachedPromptTokens ?? promptTokensDetails?.cachedTokens;
|
|
525
|
+
return {
|
|
526
|
+
promptTokens: usage.promptTokens ?? 0,
|
|
527
|
+
completionTokens: usage.completionTokens ?? 0,
|
|
528
|
+
totalTokens: usage.totalTokens ?? (usage.promptTokens ?? 0) + (usage.completionTokens ?? 0),
|
|
529
|
+
cachedPromptTokens
|
|
530
|
+
};
|
|
531
|
+
}
|
|
532
|
+
if ("inputTokens" in usage || "outputTokens" in usage) {
|
|
533
|
+
const input = usage.inputTokens ?? 0;
|
|
534
|
+
const output = usage.outputTokens ?? 0;
|
|
535
|
+
const total = usage.totalTokens ?? input + output;
|
|
536
|
+
return {
|
|
537
|
+
promptTokens: input,
|
|
538
|
+
completionTokens: output,
|
|
539
|
+
totalTokens: total,
|
|
540
|
+
cachedPromptTokens: usage.cachedInputTokens
|
|
541
|
+
};
|
|
542
|
+
}
|
|
543
|
+
return {
|
|
544
|
+
promptTokens: 0,
|
|
545
|
+
completionTokens: 0,
|
|
546
|
+
totalTokens: 0
|
|
547
|
+
};
|
|
548
|
+
}
|
|
549
|
+
function emitModelUsageEvent(runtime, type, prompt, usage) {
|
|
550
|
+
const normalized = normalizeUsage(usage);
|
|
551
|
+
const payload = {
|
|
552
|
+
runtime,
|
|
553
|
+
source: "openai",
|
|
554
|
+
provider: "openai",
|
|
555
|
+
type,
|
|
556
|
+
prompt: truncatePrompt(prompt),
|
|
557
|
+
tokens: {
|
|
558
|
+
prompt: normalized.promptTokens,
|
|
559
|
+
completion: normalized.completionTokens,
|
|
560
|
+
total: normalized.totalTokens,
|
|
561
|
+
...normalized.cachedPromptTokens !== undefined ? { cached: normalized.cachedPromptTokens } : {}
|
|
562
|
+
}
|
|
563
|
+
};
|
|
564
|
+
runtime.emitEvent(EventType.MODEL_USED, payload);
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
// models/embedding.ts
|
|
568
|
+
function validateDimension(dimension) {
|
|
569
|
+
const validDimensions = Object.values(VECTOR_DIMS);
|
|
570
|
+
if (!validDimensions.includes(dimension)) {
|
|
571
|
+
throw new Error(`Invalid embedding dimension: ${dimension}. Must be one of: ${validDimensions.join(", ")}`);
|
|
572
|
+
}
|
|
573
|
+
return dimension;
|
|
574
|
+
}
|
|
575
|
+
function extractText(params) {
|
|
576
|
+
if (params === null) {
|
|
577
|
+
return null;
|
|
578
|
+
}
|
|
579
|
+
if (typeof params === "string") {
|
|
580
|
+
return params;
|
|
581
|
+
}
|
|
582
|
+
if (typeof params === "object" && typeof params.text === "string") {
|
|
583
|
+
return params.text;
|
|
584
|
+
}
|
|
585
|
+
throw new Error("Invalid embedding params: expected string, { text: string }, or null");
|
|
586
|
+
}
|
|
587
|
+
function hasExplicitEmbeddingEndpoint(runtime) {
|
|
588
|
+
const key = isBrowser() ? "OPENAI_BROWSER_EMBEDDING_URL" : "OPENAI_EMBEDDING_URL";
|
|
589
|
+
const value = getSetting(runtime, key);
|
|
590
|
+
return typeof value === "string" && value.trim().length > 0;
|
|
591
|
+
}
|
|
592
|
+
function hasExplicitEmbeddingDimensions(runtime) {
|
|
593
|
+
const value = getSetting(runtime, "OPENAI_EMBEDDING_DIMENSIONS");
|
|
594
|
+
return typeof value === "string" && value.trim().length > 0;
|
|
595
|
+
}
|
|
596
|
+
function shouldUseLocalEmbeddingFallback(runtime) {
|
|
597
|
+
return isCerebrasMode(runtime) && !hasExplicitEmbeddingEndpoint(runtime);
|
|
598
|
+
}
|
|
599
|
+
function hashFeature(feature) {
|
|
600
|
+
let hash = 2166136261;
|
|
601
|
+
for (let i = 0;i < feature.length; i += 1) {
|
|
602
|
+
hash ^= feature.charCodeAt(i);
|
|
603
|
+
hash = Math.imul(hash, 16777619);
|
|
604
|
+
}
|
|
605
|
+
return hash >>> 0;
|
|
606
|
+
}
|
|
607
|
+
function createDeterministicEmbedding(text, dimension) {
|
|
608
|
+
const vector = new Array(dimension).fill(0);
|
|
609
|
+
const normalized = text.toLowerCase();
|
|
610
|
+
const tokens = normalized.match(/[a-z0-9]+(?:[_-][a-z0-9]+)*/g) ?? [normalized];
|
|
611
|
+
const addFeature = (feature, weight) => {
|
|
612
|
+
const hash = hashFeature(feature);
|
|
613
|
+
const idx = hash % dimension;
|
|
614
|
+
const sign = (hash & 1) === 0 ? 1 : -1;
|
|
615
|
+
vector[idx] += sign * weight;
|
|
616
|
+
const secondHash = hashFeature(`b:${feature}`);
|
|
617
|
+
const secondIdx = secondHash % dimension;
|
|
618
|
+
const secondSign = (secondHash & 1) === 0 ? 1 : -1;
|
|
619
|
+
vector[secondIdx] += secondSign * weight * 0.5;
|
|
620
|
+
};
|
|
621
|
+
tokens.forEach((token, index) => {
|
|
622
|
+
addFeature(token, 1);
|
|
623
|
+
if (index > 0) {
|
|
624
|
+
addFeature(`${tokens[index - 1]} ${token}`, 0.35);
|
|
625
|
+
}
|
|
626
|
+
});
|
|
627
|
+
addFeature(normalized.slice(0, 512), 0.15);
|
|
628
|
+
const norm = Math.sqrt(vector.reduce((sum, value) => sum + value * value, 0));
|
|
629
|
+
if (norm === 0) {
|
|
630
|
+
vector[0] = 1;
|
|
631
|
+
return vector;
|
|
632
|
+
}
|
|
633
|
+
return vector.map((value) => value / norm);
|
|
634
|
+
}
|
|
635
|
+
async function handleTextEmbedding(runtime, params) {
|
|
636
|
+
const embeddingModel = getEmbeddingModel(runtime);
|
|
637
|
+
const embeddingDimension = validateDimension(getEmbeddingDimensions(runtime));
|
|
638
|
+
const text = extractText(params);
|
|
639
|
+
if (text === null) {
|
|
640
|
+
logger5.debug("[OpenAI] Creating test embedding for initialization");
|
|
641
|
+
const testVector = new Array(embeddingDimension).fill(0);
|
|
642
|
+
testVector[0] = 0.1;
|
|
643
|
+
return testVector;
|
|
644
|
+
}
|
|
645
|
+
let trimmedText = text.trim();
|
|
646
|
+
if (trimmedText.length === 0) {
|
|
647
|
+
throw new Error("Cannot generate embedding for empty text");
|
|
648
|
+
}
|
|
649
|
+
const maxChars = 8000 * 4;
|
|
650
|
+
if (trimmedText.length > maxChars) {
|
|
651
|
+
logger5.warn(`[OpenAI] Embedding input too long (~${Math.ceil(trimmedText.length / 4)} tokens), truncating to ~8000 tokens`);
|
|
652
|
+
trimmedText = trimmedText.slice(0, maxChars);
|
|
653
|
+
}
|
|
654
|
+
if (shouldUseLocalEmbeddingFallback(runtime)) {
|
|
655
|
+
logger5.debug("[OpenAI] Using deterministic local embedding fallback for Cerebras mode");
|
|
656
|
+
return createDeterministicEmbedding(trimmedText, embeddingDimension);
|
|
657
|
+
}
|
|
658
|
+
const baseURL = getEmbeddingBaseURL(runtime);
|
|
659
|
+
const url = `${baseURL}/embeddings`;
|
|
660
|
+
logger5.debug(`[OpenAI] Generating embedding with model: ${embeddingModel}`);
|
|
661
|
+
const response = await fetch(url, {
|
|
662
|
+
method: "POST",
|
|
663
|
+
headers: {
|
|
664
|
+
...getAuthHeader(runtime, true),
|
|
665
|
+
"Content-Type": "application/json"
|
|
666
|
+
},
|
|
667
|
+
body: JSON.stringify({
|
|
668
|
+
model: embeddingModel,
|
|
669
|
+
input: trimmedText,
|
|
670
|
+
...hasExplicitEmbeddingDimensions(runtime) ? { dimensions: embeddingDimension } : {}
|
|
671
|
+
})
|
|
672
|
+
});
|
|
673
|
+
if (!response.ok) {
|
|
674
|
+
const errorText = await response.text().catch(() => "Unknown error");
|
|
675
|
+
throw new Error(`OpenAI embedding API error: ${response.status} ${response.statusText} - ${errorText}`);
|
|
676
|
+
}
|
|
677
|
+
const data = await response.json();
|
|
678
|
+
const firstResult = Array.isArray(data.data) ? data.data[0] : undefined;
|
|
679
|
+
if (!firstResult?.embedding) {
|
|
680
|
+
throw new Error("OpenAI API returned invalid embedding response structure");
|
|
681
|
+
}
|
|
682
|
+
const embedding = firstResult.embedding;
|
|
683
|
+
if (embedding.length !== embeddingDimension) {
|
|
684
|
+
throw new Error(`Embedding dimension mismatch: got ${embedding.length}, expected ${embeddingDimension}. ` + `Check OPENAI_EMBEDDING_DIMENSIONS setting.`);
|
|
685
|
+
}
|
|
686
|
+
if (data.usage) {
|
|
687
|
+
emitModelUsageEvent(runtime, ModelType.TEXT_EMBEDDING, trimmedText, {
|
|
688
|
+
promptTokens: data.usage.prompt_tokens,
|
|
689
|
+
completionTokens: 0,
|
|
690
|
+
totalTokens: data.usage.total_tokens
|
|
691
|
+
});
|
|
692
|
+
}
|
|
693
|
+
logger5.debug(`[OpenAI] Generated embedding with ${embedding.length} dimensions`);
|
|
694
|
+
return embedding;
|
|
695
|
+
}
|
|
696
|
+
// models/image.ts
|
|
697
|
+
import { logger as logger6, ModelType as ModelType2, recordLlmCall as recordLlmCall2 } from "@elizaos/core";
|
|
698
|
+
var DEFAULT_IMAGE_DESCRIPTION_PROMPT = "Please analyze this image and provide a title and detailed description.";
|
|
699
|
+
async function handleImageGeneration(runtime, params) {
|
|
700
|
+
const modelName = getImageModel(runtime);
|
|
701
|
+
const count = params.count ?? 1;
|
|
702
|
+
const size = params.size ?? "1024x1024";
|
|
703
|
+
const extendedParams = params;
|
|
704
|
+
logger6.debug(`[OpenAI] Using IMAGE model: ${modelName}`);
|
|
705
|
+
if (typeof params.prompt !== "string" || params.prompt.trim().length === 0) {
|
|
706
|
+
throw new Error("IMAGE generation requires a non-empty prompt");
|
|
707
|
+
}
|
|
708
|
+
if (count < 1 || count > 10) {
|
|
709
|
+
throw new Error("IMAGE count must be between 1 and 10");
|
|
710
|
+
}
|
|
711
|
+
const baseURL = getBaseURL(runtime);
|
|
712
|
+
const requestBody = {
|
|
713
|
+
model: modelName,
|
|
714
|
+
prompt: params.prompt,
|
|
715
|
+
n: count,
|
|
716
|
+
size
|
|
717
|
+
};
|
|
718
|
+
if (extendedParams.quality) {
|
|
719
|
+
requestBody.quality = extendedParams.quality;
|
|
720
|
+
}
|
|
721
|
+
if (extendedParams.style) {
|
|
722
|
+
requestBody.style = extendedParams.style;
|
|
723
|
+
}
|
|
724
|
+
const details = {
|
|
725
|
+
model: modelName,
|
|
726
|
+
systemPrompt: "",
|
|
727
|
+
userPrompt: params.prompt,
|
|
728
|
+
temperature: 0,
|
|
729
|
+
maxTokens: 0,
|
|
730
|
+
purpose: "external_llm",
|
|
731
|
+
actionType: "openai.images.generate"
|
|
732
|
+
};
|
|
733
|
+
const data = await recordLlmCall2(runtime, details, async () => {
|
|
734
|
+
const response = await fetch(`${baseURL}/images/generations`, {
|
|
735
|
+
method: "POST",
|
|
736
|
+
headers: {
|
|
737
|
+
...getAuthHeader(runtime),
|
|
738
|
+
"Content-Type": "application/json"
|
|
739
|
+
},
|
|
740
|
+
body: JSON.stringify(requestBody)
|
|
741
|
+
});
|
|
742
|
+
if (!response.ok) {
|
|
743
|
+
const errorText = await response.text().catch(() => "Unknown error");
|
|
744
|
+
throw new Error(`OpenAI image generation failed: ${response.status} ${response.statusText} - ${errorText}`);
|
|
745
|
+
}
|
|
746
|
+
const responseData = await response.json();
|
|
747
|
+
details.response = JSON.stringify(responseData.data);
|
|
748
|
+
return responseData;
|
|
749
|
+
});
|
|
750
|
+
if (data.data.length === 0) {
|
|
751
|
+
throw new Error("OpenAI API returned no images");
|
|
752
|
+
}
|
|
753
|
+
return data.data.map((item) => ({
|
|
754
|
+
url: item.url,
|
|
755
|
+
revisedPrompt: item.revised_prompt
|
|
756
|
+
}));
|
|
757
|
+
}
|
|
758
|
+
function parseTitleFromResponse(content) {
|
|
759
|
+
const titleMatch = content.match(/title[:\s]+(.+?)(?:\n|$)/i);
|
|
760
|
+
return titleMatch?.[1]?.trim() ?? "Image Analysis";
|
|
761
|
+
}
|
|
762
|
+
function parseDescriptionFromResponse(content) {
|
|
763
|
+
return content.replace(/title[:\s]+(.+?)(?:\n|$)/i, "").trim();
|
|
764
|
+
}
|
|
765
|
+
async function handleImageDescription(runtime, params) {
|
|
766
|
+
const modelName = getImageDescriptionModel(runtime);
|
|
767
|
+
const paramsWithMaxTokens = params;
|
|
768
|
+
const maxTokens = typeof params === "object" && typeof paramsWithMaxTokens.maxTokens === "number" ? paramsWithMaxTokens.maxTokens : getImageDescriptionMaxTokens(runtime);
|
|
769
|
+
logger6.debug(`[OpenAI] Using IMAGE_DESCRIPTION model: ${modelName}`);
|
|
770
|
+
let imageUrl;
|
|
771
|
+
let promptText;
|
|
772
|
+
if (typeof params === "string") {
|
|
773
|
+
imageUrl = params;
|
|
774
|
+
promptText = DEFAULT_IMAGE_DESCRIPTION_PROMPT;
|
|
775
|
+
} else {
|
|
776
|
+
imageUrl = params.imageUrl;
|
|
777
|
+
promptText = params.prompt ?? DEFAULT_IMAGE_DESCRIPTION_PROMPT;
|
|
778
|
+
}
|
|
779
|
+
if (!imageUrl || imageUrl.trim().length === 0) {
|
|
780
|
+
throw new Error("IMAGE_DESCRIPTION requires a valid image URL");
|
|
781
|
+
}
|
|
782
|
+
const baseURL = getImageDescriptionBaseURL(runtime);
|
|
783
|
+
const requestBody = {
|
|
784
|
+
model: modelName,
|
|
785
|
+
messages: [
|
|
786
|
+
{
|
|
787
|
+
role: "user",
|
|
788
|
+
content: [
|
|
789
|
+
{ type: "text", text: promptText },
|
|
790
|
+
{ type: "image_url", image_url: { url: imageUrl } }
|
|
791
|
+
]
|
|
792
|
+
}
|
|
793
|
+
],
|
|
794
|
+
max_tokens: maxTokens
|
|
795
|
+
};
|
|
796
|
+
const details = {
|
|
797
|
+
model: modelName,
|
|
798
|
+
systemPrompt: "",
|
|
799
|
+
userPrompt: promptText,
|
|
800
|
+
temperature: 0,
|
|
801
|
+
maxTokens,
|
|
802
|
+
purpose: "external_llm",
|
|
803
|
+
actionType: "openai.chat.completions.create"
|
|
804
|
+
};
|
|
805
|
+
const data = await recordLlmCall2(runtime, details, async () => {
|
|
806
|
+
const response = await fetch(`${baseURL}/chat/completions`, {
|
|
807
|
+
method: "POST",
|
|
808
|
+
headers: {
|
|
809
|
+
...getImageDescriptionAuthHeader(runtime),
|
|
810
|
+
"Content-Type": "application/json"
|
|
811
|
+
},
|
|
812
|
+
body: JSON.stringify(requestBody)
|
|
813
|
+
});
|
|
814
|
+
if (!response.ok) {
|
|
815
|
+
const errorText = await response.text().catch(() => "Unknown error");
|
|
816
|
+
throw new Error(`OpenAI image description failed: ${response.status} ${response.statusText} - ${errorText}`);
|
|
817
|
+
}
|
|
818
|
+
const responseData = await response.json();
|
|
819
|
+
const responseContent = responseData.choices[0]?.message.content;
|
|
820
|
+
if (!responseContent) {
|
|
821
|
+
throw new Error("OpenAI API returned empty image description");
|
|
822
|
+
}
|
|
823
|
+
details.response = responseContent;
|
|
824
|
+
if (responseData.usage) {
|
|
825
|
+
details.promptTokens = responseData.usage.prompt_tokens;
|
|
826
|
+
details.completionTokens = responseData.usage.completion_tokens;
|
|
827
|
+
}
|
|
828
|
+
return responseData;
|
|
829
|
+
});
|
|
830
|
+
if (data.usage) {
|
|
831
|
+
emitModelUsageEvent(runtime, ModelType2.IMAGE_DESCRIPTION, typeof params === "string" ? params : params.prompt ?? "", {
|
|
832
|
+
promptTokens: data.usage.prompt_tokens,
|
|
833
|
+
completionTokens: data.usage.completion_tokens,
|
|
834
|
+
totalTokens: data.usage.total_tokens
|
|
835
|
+
});
|
|
836
|
+
}
|
|
837
|
+
const firstChoice = data.choices[0];
|
|
838
|
+
const content = firstChoice?.message.content;
|
|
839
|
+
if (!content) {
|
|
840
|
+
throw new Error("OpenAI API returned empty image description");
|
|
841
|
+
}
|
|
842
|
+
return {
|
|
843
|
+
title: parseTitleFromResponse(content),
|
|
844
|
+
description: parseDescriptionFromResponse(content)
|
|
845
|
+
};
|
|
846
|
+
}
|
|
847
|
+
// models/research.ts
|
|
848
|
+
import { logger as logger7, recordLlmCall as recordLlmCall3 } from "@elizaos/core";
|
|
849
|
+
function convertToolToApi(tool) {
|
|
850
|
+
switch (tool.type) {
|
|
851
|
+
case "web_search_preview":
|
|
852
|
+
return { type: "web_search_preview" };
|
|
853
|
+
case "file_search":
|
|
854
|
+
return {
|
|
855
|
+
type: "file_search",
|
|
856
|
+
vector_store_ids: tool.vectorStoreIds
|
|
857
|
+
};
|
|
858
|
+
case "code_interpreter":
|
|
859
|
+
return {
|
|
860
|
+
type: "code_interpreter",
|
|
861
|
+
container: tool.container ?? { type: "auto" }
|
|
862
|
+
};
|
|
863
|
+
case "mcp":
|
|
864
|
+
return {
|
|
865
|
+
type: "mcp",
|
|
866
|
+
server_label: tool.serverLabel,
|
|
867
|
+
server_url: tool.serverUrl,
|
|
868
|
+
require_approval: tool.requireApproval ?? "never"
|
|
869
|
+
};
|
|
870
|
+
default:
|
|
871
|
+
throw new Error(`Unknown research tool type: ${tool.type}`);
|
|
872
|
+
}
|
|
873
|
+
}
|
|
874
|
+
function convertOutputItem(item) {
|
|
875
|
+
switch (item.type) {
|
|
876
|
+
case "web_search_call":
|
|
877
|
+
return {
|
|
878
|
+
id: item.id ?? "",
|
|
879
|
+
type: "web_search_call",
|
|
880
|
+
status: item.status ?? "completed",
|
|
881
|
+
action: {
|
|
882
|
+
type: item.action?.type ?? "search",
|
|
883
|
+
query: item.action?.query,
|
|
884
|
+
url: item.action?.url
|
|
885
|
+
}
|
|
886
|
+
};
|
|
887
|
+
case "file_search_call":
|
|
888
|
+
return {
|
|
889
|
+
id: item.id ?? "",
|
|
890
|
+
type: "file_search_call",
|
|
891
|
+
status: item.status ?? "completed",
|
|
892
|
+
query: item.query ?? "",
|
|
893
|
+
results: item.results?.map((r) => ({
|
|
894
|
+
fileId: r.file_id,
|
|
895
|
+
fileName: r.file_name,
|
|
896
|
+
score: r.score
|
|
897
|
+
}))
|
|
898
|
+
};
|
|
899
|
+
case "code_interpreter_call":
|
|
900
|
+
return {
|
|
901
|
+
id: item.id ?? "",
|
|
902
|
+
type: "code_interpreter_call",
|
|
903
|
+
status: item.status ?? "completed",
|
|
904
|
+
code: item.code ?? "",
|
|
905
|
+
output: item.output
|
|
906
|
+
};
|
|
907
|
+
case "mcp_tool_call":
|
|
908
|
+
return {
|
|
909
|
+
id: item.id ?? "",
|
|
910
|
+
type: "mcp_tool_call",
|
|
911
|
+
status: item.status ?? "completed",
|
|
912
|
+
serverLabel: item.server_label ?? "",
|
|
913
|
+
toolName: item.tool_name ?? "",
|
|
914
|
+
arguments: item.arguments ?? {},
|
|
915
|
+
result: item.result
|
|
916
|
+
};
|
|
917
|
+
case "message":
|
|
918
|
+
return {
|
|
919
|
+
type: "message",
|
|
920
|
+
content: item.content?.map((c) => ({
|
|
921
|
+
type: "output_text",
|
|
922
|
+
text: c.text,
|
|
923
|
+
annotations: c.annotations?.map((a) => ({
|
|
924
|
+
url: a.url,
|
|
925
|
+
title: a.title,
|
|
926
|
+
startIndex: a.start_index,
|
|
927
|
+
endIndex: a.end_index
|
|
928
|
+
})) ?? []
|
|
929
|
+
})) ?? []
|
|
930
|
+
};
|
|
931
|
+
default:
|
|
932
|
+
return null;
|
|
933
|
+
}
|
|
934
|
+
}
|
|
935
|
+
function extractTextAndAnnotations(response) {
|
|
936
|
+
if (response.output_text) {
|
|
937
|
+
const annotations2 = [];
|
|
938
|
+
if (response.output) {
|
|
939
|
+
for (const item of response.output) {
|
|
940
|
+
if (item.type === "message" && item.content) {
|
|
941
|
+
for (const content of item.content) {
|
|
942
|
+
if (content.annotations) {
|
|
943
|
+
for (const ann of content.annotations) {
|
|
944
|
+
annotations2.push({
|
|
945
|
+
url: ann.url,
|
|
946
|
+
title: ann.title,
|
|
947
|
+
startIndex: ann.start_index,
|
|
948
|
+
endIndex: ann.end_index
|
|
949
|
+
});
|
|
950
|
+
}
|
|
951
|
+
}
|
|
952
|
+
}
|
|
953
|
+
}
|
|
954
|
+
}
|
|
955
|
+
}
|
|
956
|
+
return { text: response.output_text, annotations: annotations2 };
|
|
957
|
+
}
|
|
958
|
+
let text = "";
|
|
959
|
+
const annotations = [];
|
|
960
|
+
if (response.output) {
|
|
961
|
+
for (const item of response.output) {
|
|
962
|
+
if (item.type === "message" && item.content) {
|
|
963
|
+
for (const content of item.content) {
|
|
964
|
+
text += content.text;
|
|
965
|
+
if (content.annotations) {
|
|
966
|
+
for (const ann of content.annotations) {
|
|
967
|
+
annotations.push({
|
|
968
|
+
url: ann.url,
|
|
969
|
+
title: ann.title,
|
|
970
|
+
startIndex: ann.start_index,
|
|
971
|
+
endIndex: ann.end_index
|
|
972
|
+
});
|
|
973
|
+
}
|
|
974
|
+
}
|
|
975
|
+
}
|
|
976
|
+
}
|
|
977
|
+
}
|
|
978
|
+
}
|
|
979
|
+
return { text, annotations };
|
|
980
|
+
}
|
|
981
|
+
async function handleResearch(runtime, params) {
|
|
982
|
+
const apiKey = getApiKey(runtime);
|
|
983
|
+
if (!apiKey) {
|
|
984
|
+
throw new Error("OPENAI_API_KEY is required for deep research. Set it in your environment variables or runtime settings.");
|
|
985
|
+
}
|
|
986
|
+
const baseURL = getBaseURL(runtime);
|
|
987
|
+
const modelName = params.model ?? getResearchModel(runtime);
|
|
988
|
+
const timeout = getResearchTimeout(runtime);
|
|
989
|
+
logger7.debug(`[OpenAI] Starting deep research with model: ${modelName}`);
|
|
990
|
+
logger7.debug(`[OpenAI] Research input: ${params.input.substring(0, 100)}...`);
|
|
991
|
+
const dataSourceTools = params.tools?.filter((t) => t.type === "web_search_preview" || t.type === "file_search" || t.type === "mcp");
|
|
992
|
+
if (!dataSourceTools || dataSourceTools.length === 0) {
|
|
993
|
+
logger7.debug("[OpenAI] No data source tools specified, defaulting to web_search_preview");
|
|
994
|
+
params.tools = [{ type: "web_search_preview" }, ...params.tools ?? []];
|
|
995
|
+
}
|
|
996
|
+
const requestBody = {
|
|
997
|
+
model: modelName,
|
|
998
|
+
input: params.input
|
|
999
|
+
};
|
|
1000
|
+
if (params.instructions) {
|
|
1001
|
+
requestBody.instructions = params.instructions;
|
|
1002
|
+
}
|
|
1003
|
+
if (params.background !== undefined) {
|
|
1004
|
+
requestBody.background = params.background;
|
|
1005
|
+
}
|
|
1006
|
+
if (params.tools && params.tools.length > 0) {
|
|
1007
|
+
requestBody.tools = params.tools.map(convertToolToApi);
|
|
1008
|
+
}
|
|
1009
|
+
if (params.maxToolCalls !== undefined) {
|
|
1010
|
+
requestBody.max_tool_calls = params.maxToolCalls;
|
|
1011
|
+
}
|
|
1012
|
+
if (params.reasoningSummary) {
|
|
1013
|
+
requestBody.reasoning = { summary: params.reasoningSummary };
|
|
1014
|
+
}
|
|
1015
|
+
logger7.debug(`[OpenAI] Research request body: ${JSON.stringify(requestBody, null, 2)}`);
|
|
1016
|
+
const details = {
|
|
1017
|
+
model: modelName,
|
|
1018
|
+
systemPrompt: params.instructions ?? "",
|
|
1019
|
+
userPrompt: params.input,
|
|
1020
|
+
temperature: 0,
|
|
1021
|
+
maxTokens: 0,
|
|
1022
|
+
purpose: "external_llm",
|
|
1023
|
+
actionType: "openai.responses.create"
|
|
1024
|
+
};
|
|
1025
|
+
const data = await recordLlmCall3(runtime, details, async () => {
|
|
1026
|
+
const response = await fetch(`${baseURL}/responses`, {
|
|
1027
|
+
method: "POST",
|
|
1028
|
+
headers: {
|
|
1029
|
+
Authorization: `Bearer ${apiKey}`,
|
|
1030
|
+
"Content-Type": "application/json"
|
|
1031
|
+
},
|
|
1032
|
+
body: JSON.stringify(requestBody),
|
|
1033
|
+
signal: AbortSignal.timeout(timeout)
|
|
1034
|
+
});
|
|
1035
|
+
if (!response.ok) {
|
|
1036
|
+
const errorText = await response.text();
|
|
1037
|
+
logger7.error(`[OpenAI] Research request failed: ${response.status} ${errorText}`);
|
|
1038
|
+
throw new Error(`Deep research request failed: ${response.status} ${response.statusText}`);
|
|
1039
|
+
}
|
|
1040
|
+
const responseData = await response.json();
|
|
1041
|
+
details.response = responseData.output_text ?? "";
|
|
1042
|
+
return responseData;
|
|
1043
|
+
});
|
|
1044
|
+
if (data.error) {
|
|
1045
|
+
logger7.error(`[OpenAI] Research API error: ${data.error.message}`);
|
|
1046
|
+
throw new Error(`Deep research error: ${data.error.message}`);
|
|
1047
|
+
}
|
|
1048
|
+
logger7.debug(`[OpenAI] Research response received. Status: ${data.status ?? "completed"}`);
|
|
1049
|
+
const { text, annotations } = extractTextAndAnnotations(data);
|
|
1050
|
+
const outputItems = [];
|
|
1051
|
+
if (data.output) {
|
|
1052
|
+
for (const item of data.output) {
|
|
1053
|
+
const converted = convertOutputItem(item);
|
|
1054
|
+
if (converted) {
|
|
1055
|
+
outputItems.push(converted);
|
|
1056
|
+
}
|
|
1057
|
+
}
|
|
1058
|
+
}
|
|
1059
|
+
const result = {
|
|
1060
|
+
id: data.id,
|
|
1061
|
+
text,
|
|
1062
|
+
annotations,
|
|
1063
|
+
outputItems,
|
|
1064
|
+
status: data.status
|
|
1065
|
+
};
|
|
1066
|
+
logger7.info(`[OpenAI] Research completed. Text length: ${text.length}, Annotations: ${annotations.length}, Output items: ${outputItems.length}`);
|
|
1067
|
+
return result;
|
|
1068
|
+
}
|
|
1069
|
+
// models/text.ts
|
|
1070
|
+
import {
|
|
1071
|
+
buildCanonicalSystemPrompt,
|
|
1072
|
+
dropDuplicateLeadingSystemMessage,
|
|
1073
|
+
logger as logger8,
|
|
1074
|
+
ModelType as ModelType3,
|
|
1075
|
+
normalizeSchemaForCerebras,
|
|
1076
|
+
recordLlmCall as recordLlmCall4,
|
|
1077
|
+
resolveEffectiveSystemPrompt,
|
|
1078
|
+
sanitizeFunctionNameForCerebras
|
|
1079
|
+
} from "@elizaos/core";
|
|
1080
|
+
import {
|
|
1081
|
+
generateText,
|
|
1082
|
+
jsonSchema,
|
|
1083
|
+
Output,
|
|
1084
|
+
streamText
|
|
1085
|
+
} from "ai";
|
|
1086
|
+
|
|
1087
|
+
// providers/openai.ts
|
|
1088
|
+
import { createOpenAI } from "@ai-sdk/openai";
|
|
1089
|
+
var PROXY_API_KEY = "sk-proxy";
|
|
1090
|
+
function createOpenAIClient(runtime) {
|
|
1091
|
+
const baseURL = getBaseURL(runtime);
|
|
1092
|
+
const apiKey = getApiKey(runtime);
|
|
1093
|
+
if (!apiKey && isProxyMode(runtime)) {
|
|
1094
|
+
return createOpenAI({
|
|
1095
|
+
apiKey: PROXY_API_KEY,
|
|
1096
|
+
baseURL
|
|
1097
|
+
});
|
|
1098
|
+
}
|
|
1099
|
+
if (!apiKey) {
|
|
1100
|
+
throw new Error("OPENAI_API_KEY is required. Set it in your environment variables or runtime settings.");
|
|
1101
|
+
}
|
|
1102
|
+
return createOpenAI({
|
|
1103
|
+
apiKey,
|
|
1104
|
+
baseURL
|
|
1105
|
+
});
|
|
1106
|
+
}
|
|
1107
|
+
// models/text.ts
|
|
1108
|
+
var TEXT_NANO_MODEL_TYPE = ModelType3.TEXT_NANO;
|
|
1109
|
+
var TEXT_MEDIUM_MODEL_TYPE = ModelType3.TEXT_MEDIUM;
|
|
1110
|
+
var TEXT_MEGA_MODEL_TYPE = ModelType3.TEXT_MEGA;
|
|
1111
|
+
var RESPONSE_HANDLER_MODEL_TYPE = ModelType3.RESPONSE_HANDLER;
|
|
1112
|
+
var ACTION_PLANNER_MODEL_TYPE = ModelType3.ACTION_PLANNER;
|
|
1113
|
+
function resolveRequestedModelName(params, runtime, getModelFn) {
|
|
1114
|
+
return typeof params.model === "string" && params.model.trim().length > 0 ? params.model.trim() : getModelFn(runtime);
|
|
1115
|
+
}
|
|
1116
|
+
function buildUserContent(params) {
|
|
1117
|
+
const content = [{ type: "text", text: params.prompt ?? "" }];
|
|
1118
|
+
for (const attachment of params.attachments ?? []) {
|
|
1119
|
+
content.push({
|
|
1120
|
+
type: "file",
|
|
1121
|
+
data: attachment.data,
|
|
1122
|
+
mediaType: attachment.mediaType,
|
|
1123
|
+
...attachment.filename ? { filename: attachment.filename } : {}
|
|
1124
|
+
});
|
|
1125
|
+
}
|
|
1126
|
+
return content;
|
|
1127
|
+
}
|
|
1128
|
+
function convertUsage(usage) {
|
|
1129
|
+
if (!usage) {
|
|
1130
|
+
return;
|
|
1131
|
+
}
|
|
1132
|
+
const promptTokens = usage.inputTokens ?? 0;
|
|
1133
|
+
const completionTokens = usage.outputTokens ?? 0;
|
|
1134
|
+
const usageWithCache = usage;
|
|
1135
|
+
const cachedInput = firstNumber(usageWithCache.cacheReadInputTokens, usageWithCache.cachedInputTokens, usageWithCache.inputTokenDetails?.cacheReadTokens, usageWithCache.inputTokenDetails?.cachedInputTokens, usageWithCache.input_tokens_details?.cache_read_input_tokens, usageWithCache.input_tokens_details?.cached_tokens, usageWithCache.prompt_tokens_details?.cached_tokens) ?? undefined;
|
|
1136
|
+
const cacheCreationInput = firstNumber(usageWithCache.cacheCreationInputTokens, usageWithCache.cacheWriteInputTokens, usageWithCache.inputTokenDetails?.cacheCreationInputTokens, usageWithCache.inputTokenDetails?.cacheCreationTokens, usageWithCache.inputTokenDetails?.cacheWriteTokens, usageWithCache.input_tokens_details?.cache_creation_input_tokens);
|
|
1137
|
+
return {
|
|
1138
|
+
promptTokens,
|
|
1139
|
+
completionTokens,
|
|
1140
|
+
totalTokens: promptTokens + completionTokens,
|
|
1141
|
+
cachedPromptTokens: cachedInput,
|
|
1142
|
+
cacheReadInputTokens: cachedInput,
|
|
1143
|
+
cacheCreationInputTokens: cacheCreationInput
|
|
1144
|
+
};
|
|
1145
|
+
}
|
|
1146
|
+
function firstNumber(...values) {
|
|
1147
|
+
for (const value of values) {
|
|
1148
|
+
if (typeof value === "number" && Number.isFinite(value)) {
|
|
1149
|
+
return value;
|
|
1150
|
+
}
|
|
1151
|
+
if (typeof value === "string" && value.trim().length > 0) {
|
|
1152
|
+
const parsed = Number(value);
|
|
1153
|
+
if (Number.isFinite(parsed)) {
|
|
1154
|
+
return parsed;
|
|
1155
|
+
}
|
|
1156
|
+
}
|
|
1157
|
+
}
|
|
1158
|
+
return;
|
|
1159
|
+
}
|
|
1160
|
+
function resolvePromptCacheOptions(params) {
|
|
1161
|
+
const withOpenAIOptions = params;
|
|
1162
|
+
return {
|
|
1163
|
+
promptCacheKey: withOpenAIOptions.providerOptions?.openai?.promptCacheKey,
|
|
1164
|
+
promptCacheRetention: withOpenAIOptions.providerOptions?.openai?.promptCacheRetention
|
|
1165
|
+
};
|
|
1166
|
+
}
|
|
1167
|
+
var VALID_REASONING_EFFORTS = ["minimal", "low", "medium", "high"];
|
|
1168
|
+
function isReasoningModel(modelName) {
|
|
1169
|
+
if (!modelName)
|
|
1170
|
+
return false;
|
|
1171
|
+
const m = modelName.toLowerCase();
|
|
1172
|
+
return m.includes("gpt-oss") || m.includes("o1") || m.includes("o3") || m.includes("o4") || m.includes("deepseek-r1") || m.includes("thinking") || m.includes("reasoning") || m.includes("qwq");
|
|
1173
|
+
}
|
|
1174
|
+
function resolveReasoningEffort(runtime, modelName) {
|
|
1175
|
+
const raw = runtime.getSetting("OPENAI_REASONING_EFFORT");
|
|
1176
|
+
const normalized = typeof raw === "string" ? raw.trim().toLowerCase() : "";
|
|
1177
|
+
if (normalized) {
|
|
1178
|
+
if (VALID_REASONING_EFFORTS.includes(normalized)) {
|
|
1179
|
+
return normalized;
|
|
1180
|
+
}
|
|
1181
|
+
logger8.warn(`[OpenAI] OPENAI_REASONING_EFFORT=${raw} is not a valid reasoning effort; ignoring. Expected one of: ${VALID_REASONING_EFFORTS.join(", ")}.`);
|
|
1182
|
+
}
|
|
1183
|
+
if (isCerebrasMode(runtime) && isReasoningModel(modelName)) {
|
|
1184
|
+
return "low";
|
|
1185
|
+
}
|
|
1186
|
+
return;
|
|
1187
|
+
}
|
|
1188
|
+
function resolveProviderOptions(params, runtime, modelName) {
|
|
1189
|
+
const withOpenAIOptions = params;
|
|
1190
|
+
const rawProviderOptions = withOpenAIOptions.providerOptions;
|
|
1191
|
+
const promptCacheOptions = resolvePromptCacheOptions(params);
|
|
1192
|
+
const reasoningEffort = resolveReasoningEffort(runtime, modelName);
|
|
1193
|
+
if (!rawProviderOptions && !promptCacheOptions.promptCacheKey && !promptCacheOptions.promptCacheRetention && !reasoningEffort) {
|
|
1194
|
+
return;
|
|
1195
|
+
}
|
|
1196
|
+
const skipCacheRetention = isCerebrasMode(runtime);
|
|
1197
|
+
const { agentName: _agentName, openai: rawOpenAIOptions, ...rest } = rawProviderOptions ?? {};
|
|
1198
|
+
const sanitizedRawOpenAIOptions = (() => {
|
|
1199
|
+
if (!rawOpenAIOptions || typeof rawOpenAIOptions !== "object")
|
|
1200
|
+
return rawOpenAIOptions;
|
|
1201
|
+
if (!skipCacheRetention)
|
|
1202
|
+
return rawOpenAIOptions;
|
|
1203
|
+
const { promptCacheRetention: _drop, ...rest2 } = rawOpenAIOptions;
|
|
1204
|
+
return rest2;
|
|
1205
|
+
})();
|
|
1206
|
+
const openaiOptions = {
|
|
1207
|
+
...sanitizedRawOpenAIOptions ?? {},
|
|
1208
|
+
...promptCacheOptions.promptCacheKey ? { promptCacheKey: promptCacheOptions.promptCacheKey } : {},
|
|
1209
|
+
...!skipCacheRetention && promptCacheOptions.promptCacheRetention ? { promptCacheRetention: promptCacheOptions.promptCacheRetention } : {},
|
|
1210
|
+
...sanitizedRawOpenAIOptions?.reasoningEffort === undefined && reasoningEffort ? { reasoningEffort } : {}
|
|
1211
|
+
};
|
|
1212
|
+
const providerOptions = {
|
|
1213
|
+
...rest,
|
|
1214
|
+
...Object.keys(openaiOptions).length > 0 ? { openai: openaiOptions } : {}
|
|
1215
|
+
};
|
|
1216
|
+
return Object.keys(providerOptions).length > 0 ? providerOptions : undefined;
|
|
1217
|
+
}
|
|
1218
|
+
function buildStructuredOutput(responseSchema) {
|
|
1219
|
+
if (responseSchema && typeof responseSchema === "object" && "responseFormat" in responseSchema && "parseCompleteOutput" in responseSchema) {
|
|
1220
|
+
return responseSchema;
|
|
1221
|
+
}
|
|
1222
|
+
const schemaOptions = responseSchema && typeof responseSchema === "object" && "schema" in responseSchema ? responseSchema : { schema: responseSchema };
|
|
1223
|
+
return Output.object({
|
|
1224
|
+
schema: jsonSchema(sanitizeJsonSchema(schemaOptions.schema, true)),
|
|
1225
|
+
...schemaOptions.name ? { name: schemaOptions.name } : {},
|
|
1226
|
+
...schemaOptions.description ? { description: schemaOptions.description } : {}
|
|
1227
|
+
});
|
|
1228
|
+
}
|
|
1229
|
+
function normalizeNativeTools(tools, options = {}) {
|
|
1230
|
+
if (!tools) {
|
|
1231
|
+
return;
|
|
1232
|
+
}
|
|
1233
|
+
if (!Array.isArray(tools)) {
|
|
1234
|
+
return tools;
|
|
1235
|
+
}
|
|
1236
|
+
const toolSet = {};
|
|
1237
|
+
for (const rawTool of tools) {
|
|
1238
|
+
const tool = asRecord(rawTool);
|
|
1239
|
+
const functionTool = asRecord(tool.function);
|
|
1240
|
+
const name = firstString(tool.name, functionTool.name);
|
|
1241
|
+
if (!name) {
|
|
1242
|
+
throw new Error("[OpenAI] Native tool definition is missing a name.");
|
|
1243
|
+
}
|
|
1244
|
+
const description = firstString(tool.description, functionTool.description);
|
|
1245
|
+
const rawSchema = tool.parameters ?? functionTool.parameters ?? { type: "object" };
|
|
1246
|
+
let inputSchema = sanitizeJsonSchema(rawSchema, true);
|
|
1247
|
+
if (options.cerebrasMode) {
|
|
1248
|
+
inputSchema = normalizeSchemaForCerebras(inputSchema, true);
|
|
1249
|
+
}
|
|
1250
|
+
const registeredName = options.cerebrasMode ? sanitizeFunctionNameForCerebras(name) : name;
|
|
1251
|
+
toolSet[registeredName] = {
|
|
1252
|
+
...description ? { description } : {},
|
|
1253
|
+
inputSchema: jsonSchema(inputSchema)
|
|
1254
|
+
};
|
|
1255
|
+
}
|
|
1256
|
+
return Object.keys(toolSet).length > 0 ? toolSet : undefined;
|
|
1257
|
+
}
|
|
1258
|
+
function normalizeNativeMessages(messages) {
|
|
1259
|
+
if (!Array.isArray(messages)) {
|
|
1260
|
+
return;
|
|
1261
|
+
}
|
|
1262
|
+
return messages.map((message) => normalizeNativeMessage(message));
|
|
1263
|
+
}
|
|
1264
|
+
function normalizeNativeMessage(message) {
|
|
1265
|
+
const raw = asRecord(message);
|
|
1266
|
+
const providerOptions = asOptionalRecord(raw.providerOptions);
|
|
1267
|
+
if (raw.role === "system") {
|
|
1268
|
+
return {
|
|
1269
|
+
role: "system",
|
|
1270
|
+
content: stringifyMessageContent(raw.content),
|
|
1271
|
+
...providerOptions ? { providerOptions } : {}
|
|
1272
|
+
};
|
|
1273
|
+
}
|
|
1274
|
+
if (raw.role === "assistant") {
|
|
1275
|
+
return {
|
|
1276
|
+
role: "assistant",
|
|
1277
|
+
content: normalizeAssistantContent(raw),
|
|
1278
|
+
...providerOptions ? { providerOptions } : {}
|
|
1279
|
+
};
|
|
1280
|
+
}
|
|
1281
|
+
if (raw.role === "tool") {
|
|
1282
|
+
return {
|
|
1283
|
+
role: "tool",
|
|
1284
|
+
content: normalizeToolContent(raw),
|
|
1285
|
+
...providerOptions ? { providerOptions } : {}
|
|
1286
|
+
};
|
|
1287
|
+
}
|
|
1288
|
+
return {
|
|
1289
|
+
role: "user",
|
|
1290
|
+
content: normalizeUserContent(raw.content),
|
|
1291
|
+
...providerOptions ? { providerOptions } : {}
|
|
1292
|
+
};
|
|
1293
|
+
}
|
|
1294
|
+
function stripReasoningParts(content) {
|
|
1295
|
+
return content.filter((part) => {
|
|
1296
|
+
if (!part || typeof part !== "object")
|
|
1297
|
+
return true;
|
|
1298
|
+
const type = part.type;
|
|
1299
|
+
return type !== "reasoning" && type !== "thinking";
|
|
1300
|
+
});
|
|
1301
|
+
}
|
|
1302
|
+
function normalizeAssistantContent(message) {
|
|
1303
|
+
const toolCalls = Array.isArray(message.toolCalls) ? message.toolCalls : [];
|
|
1304
|
+
if (toolCalls.length === 0) {
|
|
1305
|
+
if (Array.isArray(message.content)) {
|
|
1306
|
+
return stripReasoningParts(message.content);
|
|
1307
|
+
}
|
|
1308
|
+
if (typeof message.content === "string") {
|
|
1309
|
+
return message.content;
|
|
1310
|
+
}
|
|
1311
|
+
return "";
|
|
1312
|
+
}
|
|
1313
|
+
const parts = [];
|
|
1314
|
+
if (typeof message.content === "string" && message.content.length > 0) {
|
|
1315
|
+
parts.push({ type: "text", text: message.content });
|
|
1316
|
+
} else if (Array.isArray(message.content)) {
|
|
1317
|
+
parts.push(...stripReasoningParts(message.content));
|
|
1318
|
+
}
|
|
1319
|
+
for (const toolCall of toolCalls) {
|
|
1320
|
+
const rawCall = asRecord(toolCall);
|
|
1321
|
+
const rawFunction = asRecord(rawCall.function);
|
|
1322
|
+
const toolCallId = firstString(rawCall.toolCallId, rawCall.id);
|
|
1323
|
+
const toolName = firstString(rawCall.toolName, rawCall.name, rawFunction.name);
|
|
1324
|
+
if (!toolCallId || !toolName) {
|
|
1325
|
+
continue;
|
|
1326
|
+
}
|
|
1327
|
+
parts.push({
|
|
1328
|
+
type: "tool-call",
|
|
1329
|
+
toolCallId,
|
|
1330
|
+
toolName,
|
|
1331
|
+
input: parseToolCallInput(rawCall, rawFunction)
|
|
1332
|
+
});
|
|
1333
|
+
}
|
|
1334
|
+
return parts;
|
|
1335
|
+
}
|
|
1336
|
+
function normalizeToolContent(message) {
|
|
1337
|
+
if (Array.isArray(message.content)) {
|
|
1338
|
+
return message.content;
|
|
1339
|
+
}
|
|
1340
|
+
const toolCallId = firstString(message.toolCallId, message.id) ?? "tool-call";
|
|
1341
|
+
const toolName = firstString(message.toolName, message.name) ?? "tool";
|
|
1342
|
+
const parsed = parseJsonIfPossible(message.content);
|
|
1343
|
+
return [
|
|
1344
|
+
{
|
|
1345
|
+
type: "tool-result",
|
|
1346
|
+
toolCallId,
|
|
1347
|
+
toolName,
|
|
1348
|
+
output: typeof parsed === "string" ? { type: "text", value: parsed } : { type: "json", value: parsed }
|
|
1349
|
+
}
|
|
1350
|
+
];
|
|
1351
|
+
}
|
|
1352
|
+
function normalizeUserContent(content) {
|
|
1353
|
+
if (Array.isArray(content)) {
|
|
1354
|
+
return content;
|
|
1355
|
+
}
|
|
1356
|
+
return stringifyMessageContent(content);
|
|
1357
|
+
}
|
|
1358
|
+
function stringifyMessageContent(content) {
|
|
1359
|
+
if (typeof content === "string") {
|
|
1360
|
+
return content;
|
|
1361
|
+
}
|
|
1362
|
+
if (content == null) {
|
|
1363
|
+
return "";
|
|
1364
|
+
}
|
|
1365
|
+
return typeof content === "object" ? JSON.stringify(content) : String(content);
|
|
1366
|
+
}
|
|
1367
|
+
function parseToolCallInput(rawCall, rawFunction) {
|
|
1368
|
+
if ("input" in rawCall) {
|
|
1369
|
+
return rawCall.input;
|
|
1370
|
+
}
|
|
1371
|
+
return parseJsonIfPossible(rawCall.arguments ?? rawFunction.arguments ?? {});
|
|
1372
|
+
}
|
|
1373
|
+
function parseJsonIfPossible(value) {
|
|
1374
|
+
if (typeof value !== "string") {
|
|
1375
|
+
return value ?? "";
|
|
1376
|
+
}
|
|
1377
|
+
try {
|
|
1378
|
+
return JSON.parse(value);
|
|
1379
|
+
} catch {
|
|
1380
|
+
return value;
|
|
1381
|
+
}
|
|
1382
|
+
}
|
|
1383
|
+
function normalizeToolChoice(toolChoice) {
|
|
1384
|
+
if (!toolChoice) {
|
|
1385
|
+
return;
|
|
1386
|
+
}
|
|
1387
|
+
if (typeof toolChoice === "string" && (toolChoice === "auto" || toolChoice === "none" || toolChoice === "required")) {
|
|
1388
|
+
return toolChoice;
|
|
1389
|
+
}
|
|
1390
|
+
const choice = asRecord(toolChoice);
|
|
1391
|
+
if (choice.type === "tool") {
|
|
1392
|
+
if (typeof choice.toolName === "string" && choice.toolName.length > 0) {
|
|
1393
|
+
return toolChoice;
|
|
1394
|
+
}
|
|
1395
|
+
const toolName = firstString(choice.toolName, choice.name);
|
|
1396
|
+
if (toolName) {
|
|
1397
|
+
return { type: "tool", toolName };
|
|
1398
|
+
}
|
|
1399
|
+
}
|
|
1400
|
+
if (choice.type === "function") {
|
|
1401
|
+
const fn = asRecord(choice.function);
|
|
1402
|
+
const toolName = firstString(fn.name);
|
|
1403
|
+
if (toolName) {
|
|
1404
|
+
return { type: "tool", toolName };
|
|
1405
|
+
}
|
|
1406
|
+
}
|
|
1407
|
+
const namedTool = firstString(choice.name);
|
|
1408
|
+
if (namedTool) {
|
|
1409
|
+
return { type: "tool", toolName: namedTool };
|
|
1410
|
+
}
|
|
1411
|
+
return toolChoice;
|
|
1412
|
+
}
|
|
1413
|
+
function hasIllegalStrictRoot(node) {
|
|
1414
|
+
if (node.type !== "object")
|
|
1415
|
+
return true;
|
|
1416
|
+
if (Array.isArray(node.oneOf) && node.oneOf.length > 0)
|
|
1417
|
+
return true;
|
|
1418
|
+
if (Array.isArray(node.anyOf) && node.anyOf.length > 0)
|
|
1419
|
+
return true;
|
|
1420
|
+
if (Array.isArray(node.enum))
|
|
1421
|
+
return true;
|
|
1422
|
+
if (node.not !== undefined)
|
|
1423
|
+
return true;
|
|
1424
|
+
return false;
|
|
1425
|
+
}
|
|
1426
|
+
function sanitizeJsonSchema(schema, isRoot = false) {
|
|
1427
|
+
if (!schema || typeof schema !== "object" || Array.isArray(schema)) {
|
|
1428
|
+
return { type: "object" };
|
|
1429
|
+
}
|
|
1430
|
+
const record = schema;
|
|
1431
|
+
let sanitized = { ...record };
|
|
1432
|
+
if (typeof sanitized.type !== "string") {
|
|
1433
|
+
const inferredType = inferJsonSchemaType(sanitized, isRoot);
|
|
1434
|
+
if (inferredType) {
|
|
1435
|
+
sanitized.type = inferredType;
|
|
1436
|
+
}
|
|
1437
|
+
}
|
|
1438
|
+
if (isRoot && hasIllegalStrictRoot(sanitized)) {
|
|
1439
|
+
sanitized = {
|
|
1440
|
+
type: "object",
|
|
1441
|
+
properties: { value: { ...record } },
|
|
1442
|
+
required: ["value"],
|
|
1443
|
+
additionalProperties: false
|
|
1444
|
+
};
|
|
1445
|
+
}
|
|
1446
|
+
if (sanitized.properties && typeof sanitized.properties === "object" && !Array.isArray(sanitized.properties)) {
|
|
1447
|
+
const properties = {};
|
|
1448
|
+
for (const [key, value] of Object.entries(sanitized.properties)) {
|
|
1449
|
+
properties[key] = sanitizeJsonSchema(value);
|
|
1450
|
+
}
|
|
1451
|
+
sanitized.properties = properties;
|
|
1452
|
+
const propertyKeys = Object.keys(properties);
|
|
1453
|
+
const existingRequired = Array.isArray(sanitized.required) ? sanitized.required.filter((key) => typeof key === "string") : [];
|
|
1454
|
+
sanitized.required = [...new Set([...existingRequired, ...propertyKeys])];
|
|
1455
|
+
}
|
|
1456
|
+
if (sanitized.type === "object" && sanitized.additionalProperties !== false) {
|
|
1457
|
+
sanitized.additionalProperties = false;
|
|
1458
|
+
}
|
|
1459
|
+
if (sanitized.items) {
|
|
1460
|
+
sanitized.items = Array.isArray(sanitized.items) ? sanitized.items.map((item) => sanitizeJsonSchema(item)) : sanitizeJsonSchema(sanitized.items);
|
|
1461
|
+
}
|
|
1462
|
+
for (const unionKey of ["anyOf", "oneOf", "allOf"]) {
|
|
1463
|
+
const value = sanitized[unionKey];
|
|
1464
|
+
if (Array.isArray(value)) {
|
|
1465
|
+
sanitized[unionKey] = value.map((item) => sanitizeJsonSchema(item));
|
|
1466
|
+
}
|
|
1467
|
+
}
|
|
1468
|
+
return sanitized;
|
|
1469
|
+
}
|
|
1470
|
+
function inferJsonSchemaType(schema, isRoot) {
|
|
1471
|
+
if ("properties" in schema || "required" in schema || "additionalProperties" in schema || isRoot) {
|
|
1472
|
+
return "object";
|
|
1473
|
+
}
|
|
1474
|
+
if ("items" in schema) {
|
|
1475
|
+
return "array";
|
|
1476
|
+
}
|
|
1477
|
+
if (Array.isArray(schema.enum) && schema.enum.length > 0) {
|
|
1478
|
+
const types = new Set(schema.enum.map((value) => typeof value));
|
|
1479
|
+
if (types.size === 1) {
|
|
1480
|
+
const [type] = [...types];
|
|
1481
|
+
if (type === "string" || type === "number" || type === "boolean") {
|
|
1482
|
+
return type;
|
|
1483
|
+
}
|
|
1484
|
+
}
|
|
1485
|
+
}
|
|
1486
|
+
return;
|
|
1487
|
+
}
|
|
1488
|
+
function asRecord(value) {
|
|
1489
|
+
return value && typeof value === "object" && !Array.isArray(value) ? value : {};
|
|
1490
|
+
}
|
|
1491
|
+
function asOptionalRecord(value) {
|
|
1492
|
+
return value && typeof value === "object" && !Array.isArray(value) ? value : undefined;
|
|
1493
|
+
}
|
|
1494
|
+
function firstString(...values) {
|
|
1495
|
+
for (const value of values) {
|
|
1496
|
+
if (typeof value === "string" && value.length > 0) {
|
|
1497
|
+
return value;
|
|
1498
|
+
}
|
|
1499
|
+
}
|
|
1500
|
+
return;
|
|
1501
|
+
}
|
|
1502
|
+
function usesNativeTextResult(params) {
|
|
1503
|
+
return Boolean(params.messages || params.tools || params.toolChoice || params.responseSchema);
|
|
1504
|
+
}
|
|
1505
|
+
function buildNativeTextResult(result, modelName) {
|
|
1506
|
+
return {
|
|
1507
|
+
text: result.text,
|
|
1508
|
+
toolCalls: result.toolCalls ?? [],
|
|
1509
|
+
finishReason: result.finishReason,
|
|
1510
|
+
usage: convertUsage(result.usage),
|
|
1511
|
+
providerMetadata: mergeProviderModelName(result.providerMetadata, modelName)
|
|
1512
|
+
};
|
|
1513
|
+
}
|
|
1514
|
+
function handledPromise(value) {
|
|
1515
|
+
const promise = Promise.resolve(value);
|
|
1516
|
+
promise.catch(() => {});
|
|
1517
|
+
return promise;
|
|
1518
|
+
}
|
|
1519
|
+
function handledMappedPromise(value, mapper) {
|
|
1520
|
+
return handledPromise(handledPromise(value).then(mapper));
|
|
1521
|
+
}
|
|
1522
|
+
function mergeProviderModelName(providerMetadata, modelName) {
|
|
1523
|
+
if (!modelName) {
|
|
1524
|
+
return providerMetadata;
|
|
1525
|
+
}
|
|
1526
|
+
if (providerMetadata && typeof providerMetadata === "object" && !Array.isArray(providerMetadata)) {
|
|
1527
|
+
return {
|
|
1528
|
+
...providerMetadata,
|
|
1529
|
+
modelName
|
|
1530
|
+
};
|
|
1531
|
+
}
|
|
1532
|
+
return { modelName };
|
|
1533
|
+
}
|
|
1534
|
+
function createLlmCallDetails(modelName, params, systemPrompt, actionType, modelType, providerOptions, generateParams) {
|
|
1535
|
+
const originalParams = params;
|
|
1536
|
+
const nativeParams = generateParams;
|
|
1537
|
+
const nativePrompt = nativeParams && "prompt" in nativeParams ? nativeParams.prompt : undefined;
|
|
1538
|
+
const nativeMessages = nativeParams && "messages" in nativeParams && Array.isArray(nativeParams.messages) ? nativeParams.messages : undefined;
|
|
1539
|
+
const nativeSystem = typeof nativeParams?.system === "string" ? nativeParams.system : systemPrompt;
|
|
1540
|
+
return {
|
|
1541
|
+
model: modelName,
|
|
1542
|
+
modelType,
|
|
1543
|
+
provider: "vercel-ai-sdk",
|
|
1544
|
+
systemPrompt: nativeSystem ?? "",
|
|
1545
|
+
userPrompt: typeof nativePrompt === "string" ? nativePrompt : typeof params.prompt === "string" ? params.prompt : "",
|
|
1546
|
+
prompt: typeof nativePrompt === "string" ? nativePrompt : undefined,
|
|
1547
|
+
messages: nativeMessages,
|
|
1548
|
+
tools: nativeParams?.tools ?? originalParams.tools,
|
|
1549
|
+
toolChoice: nativeParams?.toolChoice ?? originalParams.toolChoice,
|
|
1550
|
+
output: nativeParams?.output !== undefined ? buildTrajectoryOutputDescriptor(originalParams.responseSchema, nativeParams.output) : undefined,
|
|
1551
|
+
responseSchema: originalParams.responseSchema,
|
|
1552
|
+
providerOptions: providerOptions ?? nativeParams?.providerOptions ?? originalParams.providerOptions,
|
|
1553
|
+
temperature: params.temperature ?? 0,
|
|
1554
|
+
maxTokens: typeof nativeParams?.maxOutputTokens === "number" ? nativeParams.maxOutputTokens : params.omitMaxTokens ? 0 : params.maxTokens ?? 8192,
|
|
1555
|
+
maxTokensOmitted: params.omitMaxTokens && typeof nativeParams?.maxOutputTokens !== "number" ? true : undefined,
|
|
1556
|
+
purpose: "external_llm",
|
|
1557
|
+
actionType
|
|
1558
|
+
};
|
|
1559
|
+
}
|
|
1560
|
+
function buildTrajectoryOutputDescriptor(responseSchema, output) {
|
|
1561
|
+
if (responseSchema !== undefined) {
|
|
1562
|
+
return {
|
|
1563
|
+
type: "object",
|
|
1564
|
+
schema: responseSchema
|
|
1565
|
+
};
|
|
1566
|
+
}
|
|
1567
|
+
return toTrajectoryJsonSafe(output);
|
|
1568
|
+
}
|
|
1569
|
+
function toTrajectoryJsonSafe(value) {
|
|
1570
|
+
try {
|
|
1571
|
+
return JSON.parse(JSON.stringify(value, (_key, nested) => {
|
|
1572
|
+
if (typeof nested === "function")
|
|
1573
|
+
return;
|
|
1574
|
+
if (typeof nested === "bigint")
|
|
1575
|
+
return nested.toString();
|
|
1576
|
+
return nested;
|
|
1577
|
+
}));
|
|
1578
|
+
} catch {
|
|
1579
|
+
return String(value);
|
|
1580
|
+
}
|
|
1581
|
+
}
|
|
1582
|
+
function applyUsageToDetails(details, usage) {
|
|
1583
|
+
if (!usage) {
|
|
1584
|
+
return;
|
|
1585
|
+
}
|
|
1586
|
+
details.promptTokens = usage.inputTokens ?? 0;
|
|
1587
|
+
details.completionTokens = usage.outputTokens ?? 0;
|
|
1588
|
+
}
|
|
1589
|
+
async function generateTextByModelType(runtime, params, modelType, getModelFn) {
|
|
1590
|
+
const paramsWithAttachments = params;
|
|
1591
|
+
const openai = createOpenAIClient(runtime);
|
|
1592
|
+
const modelName = resolveRequestedModelName(paramsWithAttachments, runtime, getModelFn);
|
|
1593
|
+
logger8.debug(`[OpenAI] Using ${modelType} model: ${modelName}`);
|
|
1594
|
+
const providerOptions = resolveProviderOptions(params, runtime, modelName);
|
|
1595
|
+
const hasAttachments = (paramsWithAttachments.attachments?.length ?? 0) > 0;
|
|
1596
|
+
const userContent = hasAttachments ? buildUserContent(paramsWithAttachments) : undefined;
|
|
1597
|
+
const shouldReturnNativeResult = usesNativeTextResult(paramsWithAttachments);
|
|
1598
|
+
const systemPrompt = resolveEffectiveSystemPrompt({
|
|
1599
|
+
params: paramsWithAttachments,
|
|
1600
|
+
fallback: buildCanonicalSystemPrompt({ character: runtime.character })
|
|
1601
|
+
});
|
|
1602
|
+
const agentName = paramsWithAttachments.providerOptions?.agentName;
|
|
1603
|
+
const telemetryConfig = {
|
|
1604
|
+
isEnabled: getExperimentalTelemetry(runtime),
|
|
1605
|
+
functionId: agentName ? `agent:${agentName}` : undefined,
|
|
1606
|
+
metadata: agentName ? { agentName } : undefined
|
|
1607
|
+
};
|
|
1608
|
+
const model = openai.chat(modelName);
|
|
1609
|
+
const cerebrasMode = isCerebrasMode(runtime);
|
|
1610
|
+
const normalizedTools = normalizeNativeTools(paramsWithAttachments.tools, {
|
|
1611
|
+
cerebrasMode
|
|
1612
|
+
});
|
|
1613
|
+
const normalizedToolChoice = normalizeToolChoice(paramsWithAttachments.toolChoice);
|
|
1614
|
+
const normalizedMessages = normalizeNativeMessages(paramsWithAttachments.messages);
|
|
1615
|
+
const wireMessages = dropDuplicateLeadingSystemMessage(normalizedMessages, systemPrompt);
|
|
1616
|
+
const effectiveMessages = wireMessages && wireMessages.length > 0 ? wireMessages : normalizedMessages;
|
|
1617
|
+
const promptText = typeof params.prompt === "string" && params.prompt.length > 0 ? params.prompt : "";
|
|
1618
|
+
const promptOrMessages = effectiveMessages && effectiveMessages.length > 0 ? { messages: effectiveMessages } : userContent ? { messages: [{ role: "user", content: userContent }] } : { prompt: promptText };
|
|
1619
|
+
const callerResponseFormat = paramsWithAttachments.responseFormat;
|
|
1620
|
+
const responseFormatType = typeof callerResponseFormat === "string" ? callerResponseFormat : callerResponseFormat && typeof callerResponseFormat === "object" && ("type" in callerResponseFormat) ? callerResponseFormat.type : undefined;
|
|
1621
|
+
const wireResponseFormat = responseFormatType === "json_object" ? { type: "json" } : responseFormatType === "text" ? { type: "text" } : undefined;
|
|
1622
|
+
const generateParams = {
|
|
1623
|
+
model,
|
|
1624
|
+
...promptOrMessages,
|
|
1625
|
+
system: systemPrompt,
|
|
1626
|
+
allowSystemInMessages: true,
|
|
1627
|
+
...params.omitMaxTokens ? {} : { maxOutputTokens: params.maxTokens ?? 8192 },
|
|
1628
|
+
experimental_telemetry: telemetryConfig,
|
|
1629
|
+
...normalizedTools ? { tools: normalizedTools } : {},
|
|
1630
|
+
...normalizedToolChoice ? { toolChoice: normalizedToolChoice } : {},
|
|
1631
|
+
...paramsWithAttachments.responseSchema && !isCerebrasMode(runtime) ? { output: buildStructuredOutput(paramsWithAttachments.responseSchema) } : {},
|
|
1632
|
+
...wireResponseFormat ? { responseFormat: wireResponseFormat } : {},
|
|
1633
|
+
...providerOptions ? { providerOptions } : {}
|
|
1634
|
+
};
|
|
1635
|
+
if (params.stream) {
|
|
1636
|
+
const details2 = createLlmCallDetails(modelName, params, systemPrompt, "ai.streamText", modelType, providerOptions, generateParams);
|
|
1637
|
+
details2.response = "";
|
|
1638
|
+
const result2 = await recordLlmCall4(runtime, details2, () => streamText(generateParams));
|
|
1639
|
+
return {
|
|
1640
|
+
textStream: async function* textStreamWithCallback() {
|
|
1641
|
+
for await (const chunk of result2.textStream) {
|
|
1642
|
+
params.onStreamChunk?.(chunk);
|
|
1643
|
+
yield chunk;
|
|
1644
|
+
}
|
|
1645
|
+
}(),
|
|
1646
|
+
text: handledPromise(result2.text),
|
|
1647
|
+
...shouldReturnNativeResult ? { toolCalls: handledPromise(result2.toolCalls) } : {},
|
|
1648
|
+
usage: handledMappedPromise(result2.usage, convertUsage),
|
|
1649
|
+
finishReason: handledMappedPromise(result2.finishReason, (r) => r)
|
|
1650
|
+
};
|
|
1651
|
+
}
|
|
1652
|
+
const details = createLlmCallDetails(modelName, params, systemPrompt, "ai.generateText", modelType, providerOptions, generateParams);
|
|
1653
|
+
const result = await recordLlmCall4(runtime, details, async () => {
|
|
1654
|
+
const result2 = await generateText(generateParams);
|
|
1655
|
+
details.response = result2.text;
|
|
1656
|
+
details.toolCalls = result2.toolCalls;
|
|
1657
|
+
details.finishReason = result2.finishReason;
|
|
1658
|
+
details.providerMetadata = result2.providerMetadata;
|
|
1659
|
+
applyUsageToDetails(details, result2.usage);
|
|
1660
|
+
return result2;
|
|
1661
|
+
});
|
|
1662
|
+
if (result.usage) {
|
|
1663
|
+
emitModelUsageEvent(runtime, modelType, params.prompt ?? "", result.usage);
|
|
1664
|
+
}
|
|
1665
|
+
if (shouldReturnNativeResult) {
|
|
1666
|
+
return buildNativeTextResult(result, modelName);
|
|
1667
|
+
}
|
|
1668
|
+
return result.text;
|
|
1669
|
+
}
|
|
1670
|
+
async function handleTextSmall(runtime, params) {
|
|
1671
|
+
return generateTextByModelType(runtime, params, ModelType3.TEXT_SMALL, getSmallModel);
|
|
1672
|
+
}
|
|
1673
|
+
async function handleTextNano(runtime, params) {
|
|
1674
|
+
return generateTextByModelType(runtime, params, TEXT_NANO_MODEL_TYPE, getNanoModel);
|
|
1675
|
+
}
|
|
1676
|
+
async function handleTextMedium(runtime, params) {
|
|
1677
|
+
return generateTextByModelType(runtime, params, TEXT_MEDIUM_MODEL_TYPE, getMediumModel);
|
|
1678
|
+
}
|
|
1679
|
+
async function handleTextLarge(runtime, params) {
|
|
1680
|
+
return generateTextByModelType(runtime, params, ModelType3.TEXT_LARGE, getLargeModel);
|
|
1681
|
+
}
|
|
1682
|
+
async function handleTextMega(runtime, params) {
|
|
1683
|
+
return generateTextByModelType(runtime, params, TEXT_MEGA_MODEL_TYPE, getMegaModel);
|
|
1684
|
+
}
|
|
1685
|
+
async function handleResponseHandler(runtime, params) {
|
|
1686
|
+
return generateTextByModelType(runtime, params, RESPONSE_HANDLER_MODEL_TYPE, getResponseHandlerModel);
|
|
1687
|
+
}
|
|
1688
|
+
async function handleActionPlanner(runtime, params) {
|
|
1689
|
+
return generateTextByModelType(runtime, params, ACTION_PLANNER_MODEL_TYPE, getActionPlannerModel);
|
|
1690
|
+
}
|
|
1691
|
+
// utils/tokenization.ts
|
|
1692
|
+
import { ModelType as ModelType4 } from "@elizaos/core";
|
|
1693
|
+
import {
|
|
1694
|
+
encodingForModel,
|
|
1695
|
+
getEncoding
|
|
1696
|
+
} from "js-tiktoken";
|
|
1697
|
+
function resolveTokenizerEncoding(modelName) {
|
|
1698
|
+
const normalized = modelName.toLowerCase();
|
|
1699
|
+
const fallbackEncoding = normalized.includes("4o") ? "o200k_base" : "cl100k_base";
|
|
1700
|
+
try {
|
|
1701
|
+
return encodingForModel(modelName);
|
|
1702
|
+
} catch {
|
|
1703
|
+
return getEncoding(fallbackEncoding);
|
|
1704
|
+
}
|
|
1705
|
+
}
|
|
1706
|
+
function getModelName(runtime, modelType) {
|
|
1707
|
+
if (modelType === ModelType4.TEXT_SMALL) {
|
|
1708
|
+
return getSmallModel(runtime);
|
|
1709
|
+
}
|
|
1710
|
+
return getLargeModel(runtime);
|
|
1711
|
+
}
|
|
1712
|
+
function tokenizeText(runtime, modelType, text) {
|
|
1713
|
+
const modelName = getModelName(runtime, modelType);
|
|
1714
|
+
const encoder = resolveTokenizerEncoding(modelName);
|
|
1715
|
+
return encoder.encode(text);
|
|
1716
|
+
}
|
|
1717
|
+
function detokenizeText(runtime, modelType, tokens) {
|
|
1718
|
+
const modelName = getModelName(runtime, modelType);
|
|
1719
|
+
const encoder = resolveTokenizerEncoding(modelName);
|
|
1720
|
+
return encoder.decode(tokens);
|
|
1721
|
+
}
|
|
1722
|
+
|
|
1723
|
+
// models/tokenizer.ts
|
|
1724
|
+
async function handleTokenizerEncode(runtime, params) {
|
|
1725
|
+
if (!params.prompt) {
|
|
1726
|
+
throw new Error("Tokenization requires a non-empty prompt");
|
|
1727
|
+
}
|
|
1728
|
+
const modelType = params.modelType;
|
|
1729
|
+
return tokenizeText(runtime, modelType, params.prompt);
|
|
1730
|
+
}
|
|
1731
|
+
async function handleTokenizerDecode(runtime, params) {
|
|
1732
|
+
if (!params.tokens || !Array.isArray(params.tokens)) {
|
|
1733
|
+
throw new Error("Detokenization requires a valid tokens array");
|
|
1734
|
+
}
|
|
1735
|
+
if (params.tokens.length === 0) {
|
|
1736
|
+
return "";
|
|
1737
|
+
}
|
|
1738
|
+
for (let i = 0;i < params.tokens.length; i++) {
|
|
1739
|
+
const token = params.tokens[i];
|
|
1740
|
+
if (typeof token !== "number" || !Number.isFinite(token)) {
|
|
1741
|
+
throw new Error(`Invalid token at index ${i}: expected number`);
|
|
1742
|
+
}
|
|
1743
|
+
}
|
|
1744
|
+
const modelType = params.modelType;
|
|
1745
|
+
return detokenizeText(runtime, modelType, params.tokens);
|
|
1746
|
+
}
|
|
1747
|
+
// index.ts
|
|
1748
|
+
function getProcessEnv() {
|
|
1749
|
+
if (typeof process === "undefined") {
|
|
1750
|
+
return {};
|
|
1751
|
+
}
|
|
1752
|
+
return process.env;
|
|
1753
|
+
}
|
|
1754
|
+
var env = getProcessEnv();
|
|
1755
|
+
var TEXT_NANO_MODEL_TYPE2 = ModelType5.TEXT_NANO;
|
|
1756
|
+
var TEXT_MEDIUM_MODEL_TYPE2 = ModelType5.TEXT_MEDIUM;
|
|
1757
|
+
var TEXT_MEGA_MODEL_TYPE2 = ModelType5.TEXT_MEGA;
|
|
1758
|
+
var RESPONSE_HANDLER_MODEL_TYPE2 = ModelType5.RESPONSE_HANDLER;
|
|
1759
|
+
var ACTION_PLANNER_MODEL_TYPE2 = ModelType5.ACTION_PLANNER;
|
|
1760
|
+
function hasExplicitCapabilityOverride(runtime, overrideKeys) {
|
|
1761
|
+
return overrideKeys.some((key) => {
|
|
1762
|
+
const value = getSetting(runtime, key);
|
|
1763
|
+
return typeof value === "string" && value.trim().length > 0;
|
|
1764
|
+
});
|
|
1765
|
+
}
|
|
1766
|
+
var mediaModelOverrideKeys = {
|
|
1767
|
+
[ModelType5.IMAGE_DESCRIPTION]: ["OPENAI_IMAGE_DESCRIPTION_BASE_URL"]
|
|
1768
|
+
};
|
|
1769
|
+
var mediaModels = {
|
|
1770
|
+
[ModelType5.IMAGE]: async (runtime, params) => {
|
|
1771
|
+
return handleImageGeneration(runtime, params);
|
|
1772
|
+
},
|
|
1773
|
+
[ModelType5.IMAGE_DESCRIPTION]: async (runtime, params) => {
|
|
1774
|
+
return handleImageDescription(runtime, params);
|
|
1775
|
+
},
|
|
1776
|
+
[ModelType5.TRANSCRIPTION]: async (runtime, input) => {
|
|
1777
|
+
return handleTranscription(runtime, input);
|
|
1778
|
+
},
|
|
1779
|
+
[ModelType5.TEXT_TO_SPEECH]: async (runtime, input) => {
|
|
1780
|
+
return handleTextToSpeech(runtime, input);
|
|
1781
|
+
}
|
|
1782
|
+
};
|
|
1783
|
+
function registerMediaModels(runtime) {
|
|
1784
|
+
const cerebras = isCerebrasMode(runtime);
|
|
1785
|
+
for (const [modelType, handler] of Object.entries(mediaModels)) {
|
|
1786
|
+
if (cerebras && !hasExplicitCapabilityOverride(runtime, mediaModelOverrideKeys[modelType] ?? [])) {
|
|
1787
|
+
logger9.info(`[OpenAI] Not registering ${modelType}: the Cerebras endpoint does not serve it`);
|
|
1788
|
+
continue;
|
|
1789
|
+
}
|
|
1790
|
+
runtime.registerModel(modelType, handler, openaiPlugin.name, openaiPlugin.priority);
|
|
1791
|
+
}
|
|
1792
|
+
}
|
|
1793
|
+
var openaiPlugin = {
|
|
1794
|
+
name: "openai",
|
|
1795
|
+
description: "OpenAI API integration for text, image, audio, and embedding models",
|
|
1796
|
+
autoEnable: {
|
|
1797
|
+
envKeys: ["OPENAI_API_KEY", "CEREBRAS_API_KEY", "EVOLINK_API_KEY"]
|
|
1798
|
+
},
|
|
1799
|
+
config: {
|
|
1800
|
+
OPENAI_API_KEY: env.OPENAI_API_KEY ?? null,
|
|
1801
|
+
OPENAI_BASE_URL: env.OPENAI_BASE_URL ?? null,
|
|
1802
|
+
EVOLINK_API_KEY: env.EVOLINK_API_KEY ?? null,
|
|
1803
|
+
EVOLINK_BASE_URL: env.EVOLINK_BASE_URL ?? null,
|
|
1804
|
+
EVOLINK_MODEL: env.EVOLINK_MODEL ?? null,
|
|
1805
|
+
OPENAI_NANO_MODEL: env.OPENAI_NANO_MODEL ?? null,
|
|
1806
|
+
OPENAI_MEDIUM_MODEL: env.OPENAI_MEDIUM_MODEL ?? null,
|
|
1807
|
+
OPENAI_SMALL_MODEL: env.OPENAI_SMALL_MODEL ?? null,
|
|
1808
|
+
OPENAI_LARGE_MODEL: env.OPENAI_LARGE_MODEL ?? null,
|
|
1809
|
+
OPENAI_MEGA_MODEL: env.OPENAI_MEGA_MODEL ?? null,
|
|
1810
|
+
OPENAI_RESPONSE_HANDLER_MODEL: env.OPENAI_RESPONSE_HANDLER_MODEL ?? null,
|
|
1811
|
+
OPENAI_SHOULD_RESPOND_MODEL: env.OPENAI_SHOULD_RESPOND_MODEL ?? null,
|
|
1812
|
+
OPENAI_ACTION_PLANNER_MODEL: env.OPENAI_ACTION_PLANNER_MODEL ?? null,
|
|
1813
|
+
OPENAI_PLANNER_MODEL: env.OPENAI_PLANNER_MODEL ?? null,
|
|
1814
|
+
NANO_MODEL: env.NANO_MODEL ?? null,
|
|
1815
|
+
MEDIUM_MODEL: env.MEDIUM_MODEL ?? null,
|
|
1816
|
+
SMALL_MODEL: env.SMALL_MODEL ?? null,
|
|
1817
|
+
LARGE_MODEL: env.LARGE_MODEL ?? null,
|
|
1818
|
+
MEGA_MODEL: env.MEGA_MODEL ?? null,
|
|
1819
|
+
RESPONSE_HANDLER_MODEL: env.RESPONSE_HANDLER_MODEL ?? null,
|
|
1820
|
+
SHOULD_RESPOND_MODEL: env.SHOULD_RESPOND_MODEL ?? null,
|
|
1821
|
+
ACTION_PLANNER_MODEL: env.ACTION_PLANNER_MODEL ?? null,
|
|
1822
|
+
PLANNER_MODEL: env.PLANNER_MODEL ?? null,
|
|
1823
|
+
OPENAI_EMBEDDING_MODEL: env.OPENAI_EMBEDDING_MODEL ?? null,
|
|
1824
|
+
OPENAI_EMBEDDING_API_KEY: env.OPENAI_EMBEDDING_API_KEY ?? null,
|
|
1825
|
+
OPENAI_EMBEDDING_URL: env.OPENAI_EMBEDDING_URL ?? null,
|
|
1826
|
+
OPENAI_EMBEDDING_DIMENSIONS: env.OPENAI_EMBEDDING_DIMENSIONS ?? null,
|
|
1827
|
+
OPENAI_IMAGE_DESCRIPTION_API_KEY: env.OPENAI_IMAGE_DESCRIPTION_API_KEY ?? null,
|
|
1828
|
+
OPENAI_IMAGE_DESCRIPTION_BASE_URL: env.OPENAI_IMAGE_DESCRIPTION_BASE_URL ?? null,
|
|
1829
|
+
OPENAI_IMAGE_DESCRIPTION_MODEL: env.OPENAI_IMAGE_DESCRIPTION_MODEL ?? null,
|
|
1830
|
+
OPENAI_IMAGE_DESCRIPTION_MAX_TOKENS: env.OPENAI_IMAGE_DESCRIPTION_MAX_TOKENS ?? null,
|
|
1831
|
+
OPENAI_EXPERIMENTAL_TELEMETRY: env.OPENAI_EXPERIMENTAL_TELEMETRY ?? null,
|
|
1832
|
+
OPENAI_RESEARCH_MODEL: env.OPENAI_RESEARCH_MODEL ?? null,
|
|
1833
|
+
OPENAI_RESEARCH_TIMEOUT: env.OPENAI_RESEARCH_TIMEOUT ?? null
|
|
1834
|
+
},
|
|
1835
|
+
async init(config, runtime) {
|
|
1836
|
+
initializeOpenAI(config, runtime);
|
|
1837
|
+
registerMediaModels(runtime);
|
|
1838
|
+
},
|
|
1839
|
+
models: {
|
|
1840
|
+
[ModelType5.TEXT_EMBEDDING]: async (runtime, params) => {
|
|
1841
|
+
return handleTextEmbedding(runtime, params);
|
|
1842
|
+
},
|
|
1843
|
+
[ModelType5.TEXT_TOKENIZER_ENCODE]: async (runtime, params) => {
|
|
1844
|
+
return handleTokenizerEncode(runtime, params);
|
|
1845
|
+
},
|
|
1846
|
+
[ModelType5.TEXT_TOKENIZER_DECODE]: async (runtime, params) => {
|
|
1847
|
+
return handleTokenizerDecode(runtime, params);
|
|
1848
|
+
},
|
|
1849
|
+
[ModelType5.TEXT_SMALL]: async (runtime, params) => {
|
|
1850
|
+
return handleTextSmall(runtime, params);
|
|
1851
|
+
},
|
|
1852
|
+
[TEXT_NANO_MODEL_TYPE2]: async (runtime, params) => {
|
|
1853
|
+
return handleTextNano(runtime, params);
|
|
1854
|
+
},
|
|
1855
|
+
[TEXT_MEDIUM_MODEL_TYPE2]: async (runtime, params) => {
|
|
1856
|
+
return handleTextMedium(runtime, params);
|
|
1857
|
+
},
|
|
1858
|
+
[ModelType5.TEXT_LARGE]: async (runtime, params) => {
|
|
1859
|
+
return handleTextLarge(runtime, params);
|
|
1860
|
+
},
|
|
1861
|
+
[TEXT_MEGA_MODEL_TYPE2]: async (runtime, params) => {
|
|
1862
|
+
return handleTextMega(runtime, params);
|
|
1863
|
+
},
|
|
1864
|
+
[RESPONSE_HANDLER_MODEL_TYPE2]: async (runtime, params) => {
|
|
1865
|
+
return handleResponseHandler(runtime, params);
|
|
1866
|
+
},
|
|
1867
|
+
[ACTION_PLANNER_MODEL_TYPE2]: async (runtime, params) => {
|
|
1868
|
+
return handleActionPlanner(runtime, params);
|
|
1869
|
+
},
|
|
1870
|
+
[ModelType5.RESEARCH]: async (runtime, params) => {
|
|
1871
|
+
return handleResearch(runtime, params);
|
|
1872
|
+
}
|
|
1873
|
+
},
|
|
1874
|
+
tests: [
|
|
1875
|
+
{
|
|
1876
|
+
name: "openai_plugin_tests",
|
|
1877
|
+
tests: [
|
|
1878
|
+
{
|
|
1879
|
+
name: "openai_test_api_connectivity",
|
|
1880
|
+
fn: async (runtime) => {
|
|
1881
|
+
const baseURL = getBaseURL(runtime);
|
|
1882
|
+
const response = await fetch(`${baseURL}/models`, {
|
|
1883
|
+
headers: getAuthHeader(runtime)
|
|
1884
|
+
});
|
|
1885
|
+
if (!response.ok) {
|
|
1886
|
+
throw new Error(`API connectivity test failed: ${response.status} ${response.statusText}`);
|
|
1887
|
+
}
|
|
1888
|
+
const data = await response.json();
|
|
1889
|
+
logger9.info(`[OpenAI Test] API connected. ${data.data?.length ?? 0} models available.`);
|
|
1890
|
+
}
|
|
1891
|
+
},
|
|
1892
|
+
{
|
|
1893
|
+
name: "openai_test_text_embedding",
|
|
1894
|
+
fn: async (runtime) => {
|
|
1895
|
+
const embedding = await runtime.useModel(ModelType5.TEXT_EMBEDDING, {
|
|
1896
|
+
text: "Hello, world!"
|
|
1897
|
+
});
|
|
1898
|
+
if (!Array.isArray(embedding) || embedding.length === 0) {
|
|
1899
|
+
throw new Error("Embedding should return a non-empty array");
|
|
1900
|
+
}
|
|
1901
|
+
logger9.info(`[OpenAI Test] Generated embedding with ${embedding.length} dimensions`);
|
|
1902
|
+
}
|
|
1903
|
+
},
|
|
1904
|
+
{
|
|
1905
|
+
name: "openai_test_text_small",
|
|
1906
|
+
fn: async (runtime) => {
|
|
1907
|
+
const text = await runtime.useModel(ModelType5.TEXT_SMALL, {
|
|
1908
|
+
prompt: "Say hello in exactly 5 words."
|
|
1909
|
+
});
|
|
1910
|
+
if (typeof text !== "string" || text.length === 0) {
|
|
1911
|
+
throw new Error("TEXT_SMALL should return non-empty string");
|
|
1912
|
+
}
|
|
1913
|
+
logger9.info(`[OpenAI Test] TEXT_SMALL generated: "${text.substring(0, 50)}..."`);
|
|
1914
|
+
}
|
|
1915
|
+
},
|
|
1916
|
+
{
|
|
1917
|
+
name: "openai_test_text_large",
|
|
1918
|
+
fn: async (runtime) => {
|
|
1919
|
+
const text = await runtime.useModel(ModelType5.TEXT_LARGE, {
|
|
1920
|
+
prompt: "Explain quantum computing in 2 sentences."
|
|
1921
|
+
});
|
|
1922
|
+
if (typeof text !== "string" || text.length === 0) {
|
|
1923
|
+
throw new Error("TEXT_LARGE should return non-empty string");
|
|
1924
|
+
}
|
|
1925
|
+
logger9.info(`[OpenAI Test] TEXT_LARGE generated: "${text.substring(0, 50)}..."`);
|
|
1926
|
+
}
|
|
1927
|
+
},
|
|
1928
|
+
{
|
|
1929
|
+
name: "openai_test_tokenizer_roundtrip",
|
|
1930
|
+
fn: async (runtime) => {
|
|
1931
|
+
const originalText = "Hello, tokenizer test!";
|
|
1932
|
+
const tokens = await runtime.useModel(ModelType5.TEXT_TOKENIZER_ENCODE, {
|
|
1933
|
+
prompt: originalText,
|
|
1934
|
+
modelType: ModelType5.TEXT_SMALL
|
|
1935
|
+
});
|
|
1936
|
+
if (!Array.isArray(tokens) || tokens.length === 0) {
|
|
1937
|
+
throw new Error("Tokenization should return non-empty token array");
|
|
1938
|
+
}
|
|
1939
|
+
const decodedText = await runtime.useModel(ModelType5.TEXT_TOKENIZER_DECODE, {
|
|
1940
|
+
tokens,
|
|
1941
|
+
modelType: ModelType5.TEXT_SMALL
|
|
1942
|
+
});
|
|
1943
|
+
if (decodedText !== originalText) {
|
|
1944
|
+
throw new Error(`Tokenizer roundtrip failed: expected "${originalText}", got "${decodedText}"`);
|
|
1945
|
+
}
|
|
1946
|
+
logger9.info(`[OpenAI Test] Tokenizer roundtrip successful (${tokens.length} tokens)`);
|
|
1947
|
+
}
|
|
1948
|
+
},
|
|
1949
|
+
{
|
|
1950
|
+
name: "openai_test_streaming",
|
|
1951
|
+
fn: async (runtime) => {
|
|
1952
|
+
const chunks = [];
|
|
1953
|
+
const result = await runtime.useModel(ModelType5.TEXT_LARGE, {
|
|
1954
|
+
prompt: "Count from 1 to 5, one number per line.",
|
|
1955
|
+
stream: true,
|
|
1956
|
+
onStreamChunk: (chunk) => {
|
|
1957
|
+
chunks.push(chunk);
|
|
1958
|
+
}
|
|
1959
|
+
});
|
|
1960
|
+
if (typeof result !== "string" || result.length === 0) {
|
|
1961
|
+
throw new Error("Streaming should return non-empty result");
|
|
1962
|
+
}
|
|
1963
|
+
if (chunks.length === 0) {
|
|
1964
|
+
throw new Error("No streaming chunks received");
|
|
1965
|
+
}
|
|
1966
|
+
logger9.info(`[OpenAI Test] Streaming test: ${chunks.length} chunks received`);
|
|
1967
|
+
}
|
|
1968
|
+
},
|
|
1969
|
+
{
|
|
1970
|
+
name: "openai_test_image_description",
|
|
1971
|
+
fn: async (runtime) => {
|
|
1972
|
+
const testImageUrl = "https://upload.wikimedia.org/wikipedia/commons/thumb/a/a7/Camponotus_flavomarginatus_ant.jpg/440px-Camponotus_flavomarginatus_ant.jpg";
|
|
1973
|
+
const result = await runtime.useModel(ModelType5.IMAGE_DESCRIPTION, testImageUrl);
|
|
1974
|
+
if (!result || typeof result !== "object" || !("title" in result) || !("description" in result)) {
|
|
1975
|
+
throw new Error("Image description should return { title, description }");
|
|
1976
|
+
}
|
|
1977
|
+
logger9.info(`[OpenAI Test] Image described: "${result.title}"`);
|
|
1978
|
+
}
|
|
1979
|
+
},
|
|
1980
|
+
{
|
|
1981
|
+
name: "openai_test_transcription",
|
|
1982
|
+
fn: async (runtime) => {
|
|
1983
|
+
const audioUrl = "https://upload.wikimedia.org/wikipedia/commons/2/25/En-Open_Source.ogg";
|
|
1984
|
+
const response = await fetch(audioUrl);
|
|
1985
|
+
const arrayBuffer = await response.arrayBuffer();
|
|
1986
|
+
const audioBuffer = Buffer.from(new Uint8Array(arrayBuffer));
|
|
1987
|
+
const transcription = await runtime.useModel(ModelType5.TRANSCRIPTION, audioBuffer);
|
|
1988
|
+
if (typeof transcription !== "string") {
|
|
1989
|
+
throw new Error("Transcription should return a string");
|
|
1990
|
+
}
|
|
1991
|
+
logger9.info(`[OpenAI Test] Transcription: "${transcription.substring(0, 50)}..."`);
|
|
1992
|
+
}
|
|
1993
|
+
},
|
|
1994
|
+
{
|
|
1995
|
+
name: "openai_test_text_to_speech",
|
|
1996
|
+
fn: async (runtime) => {
|
|
1997
|
+
const audioData = await runtime.useModel(ModelType5.TEXT_TO_SPEECH, {
|
|
1998
|
+
text: "Hello, this is a text-to-speech test."
|
|
1999
|
+
});
|
|
2000
|
+
if (!(audioData instanceof ArrayBuffer) || audioData.byteLength === 0) {
|
|
2001
|
+
throw new Error("TTS should return non-empty ArrayBuffer");
|
|
2002
|
+
}
|
|
2003
|
+
logger9.info(`[OpenAI Test] TTS generated ${audioData.byteLength} bytes of audio`);
|
|
2004
|
+
}
|
|
2005
|
+
},
|
|
2006
|
+
{
|
|
2007
|
+
name: "openai_test_structured_output_via_text_large",
|
|
2008
|
+
fn: async (runtime) => {
|
|
2009
|
+
const result = await runtime.useModel(ModelType5.TEXT_LARGE, {
|
|
2010
|
+
prompt: "Return a JSON object with exactly these fields: name (string), age (number), active (boolean)",
|
|
2011
|
+
responseSchema: {
|
|
2012
|
+
type: "object",
|
|
2013
|
+
properties: {
|
|
2014
|
+
name: { type: "string" },
|
|
2015
|
+
age: { type: "number" },
|
|
2016
|
+
active: { type: "boolean" }
|
|
2017
|
+
},
|
|
2018
|
+
required: ["name", "age", "active"]
|
|
2019
|
+
}
|
|
2020
|
+
});
|
|
2021
|
+
if (!result || typeof result !== "object" && typeof result !== "string") {
|
|
2022
|
+
throw new Error("Structured output should return an object or text");
|
|
2023
|
+
}
|
|
2024
|
+
logger9.info(`[OpenAI Test] Structured output: ${JSON.stringify(result).substring(0, 100)}`);
|
|
2025
|
+
}
|
|
2026
|
+
},
|
|
2027
|
+
{
|
|
2028
|
+
name: "openai_test_research",
|
|
2029
|
+
fn: async (runtime) => {
|
|
2030
|
+
const result = await runtime.useModel(ModelType5.RESEARCH, {
|
|
2031
|
+
input: "What is the current date and time?",
|
|
2032
|
+
tools: [{ type: "web_search_preview" }],
|
|
2033
|
+
maxToolCalls: 3
|
|
2034
|
+
});
|
|
2035
|
+
if (!result || typeof result !== "object" || !("text" in result)) {
|
|
2036
|
+
throw new Error("Research should return an object with text property");
|
|
2037
|
+
}
|
|
2038
|
+
if (typeof result.text !== "string" || result.text.length === 0) {
|
|
2039
|
+
throw new Error("Research result text should be a non-empty string");
|
|
2040
|
+
}
|
|
2041
|
+
logger9.info(`[OpenAI Test] Research completed. Text length: ${result.text.length}, Annotations: ${result.annotations.length}`);
|
|
2042
|
+
}
|
|
2043
|
+
}
|
|
2044
|
+
]
|
|
2045
|
+
}
|
|
2046
|
+
]
|
|
2047
|
+
};
|
|
2048
|
+
var plugin_openai_default = openaiPlugin;
|
|
2049
|
+
|
|
2050
|
+
// index.node.ts
|
|
2051
|
+
var index_node_default = plugin_openai_default;
|
|
2052
|
+
export {
|
|
2053
|
+
openaiPlugin,
|
|
2054
|
+
index_node_default as default
|
|
2055
|
+
};
|
|
2056
|
+
|
|
2057
|
+
//# debugId=BCCE52F3E7079C4E64756E2164756E21
|