@elizaos/plugin-openai 1.6.0 → 2.0.0-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/dist/browser/index.browser.js +2 -2
- package/dist/browser/index.browser.js.map +18 -17
- package/dist/build.d.ts +13 -0
- package/dist/build.d.ts.map +1 -0
- package/dist/cjs/index.node.cjs +998 -658
- package/dist/cjs/index.node.js.map +18 -17
- package/dist/generated/specs/specs.d.ts +55 -0
- package/dist/generated/specs/specs.d.ts.map +1 -0
- package/dist/index.browser.d.ts +1 -0
- package/dist/index.browser.d.ts.map +1 -0
- package/dist/index.d.ts +1 -5
- package/dist/index.d.ts.map +1 -0
- package/dist/index.node.d.ts +1 -0
- package/dist/index.node.d.ts.map +1 -0
- package/dist/init.d.ts +4 -5
- package/dist/init.d.ts.map +1 -0
- package/dist/models/audio.d.ts +9 -10
- package/dist/models/audio.d.ts.map +1 -0
- package/dist/models/embedding.d.ts +1 -3
- package/dist/models/embedding.d.ts.map +1 -0
- package/dist/models/image.d.ts +4 -13
- package/dist/models/image.d.ts.map +1 -0
- package/dist/models/index.d.ts +7 -5
- package/dist/models/index.d.ts.map +1 -0
- package/dist/models/object.d.ts +4 -9
- package/dist/models/object.d.ts.map +1 -0
- package/dist/models/research.d.ts +34 -0
- package/dist/models/research.d.ts.map +1 -0
- package/dist/models/text.d.ts +22 -3
- package/dist/models/text.d.ts.map +1 -0
- package/dist/models/tokenizer.d.ts +4 -9
- package/dist/models/tokenizer.d.ts.map +1 -0
- package/dist/node/index.node.js +987 -644
- package/dist/node/index.node.js.map +18 -17
- package/dist/providers/index.d.ts +2 -1
- package/dist/providers/index.d.ts.map +1 -0
- package/dist/providers/openai.d.ts +3 -7
- package/dist/providers/openai.d.ts.map +1 -0
- package/dist/types/index.d.ts +313 -10
- package/dist/types/index.d.ts.map +1 -0
- package/dist/utils/audio.d.ts +6 -12
- package/dist/utils/audio.d.ts.map +1 -0
- package/dist/utils/config.d.ts +16 -59
- package/dist/utils/config.d.ts.map +1 -0
- package/dist/utils/events.d.ts +14 -9
- package/dist/utils/events.d.ts.map +1 -0
- package/dist/utils/index.d.ts +2 -1
- package/dist/utils/index.d.ts.map +1 -0
- package/dist/utils/json.d.ts +9 -6
- package/dist/utils/json.d.ts.map +1 -0
- package/dist/utils/tokenization.d.ts +5 -16
- package/dist/utils/tokenization.d.ts.map +1 -0
- package/package.json +24 -28
- package/README.md +0 -160
package/dist/cjs/index.node.cjs
CHANGED
|
@@ -26,384 +26,173 @@ var __export = (target, all) => {
|
|
|
26
26
|
});
|
|
27
27
|
};
|
|
28
28
|
|
|
29
|
-
//
|
|
29
|
+
// index.node.ts
|
|
30
30
|
var exports_index_node = {};
|
|
31
31
|
__export(exports_index_node, {
|
|
32
32
|
openaiPlugin: () => openaiPlugin,
|
|
33
|
-
default: () =>
|
|
33
|
+
default: () => typescript_default
|
|
34
34
|
});
|
|
35
35
|
module.exports = __toCommonJS(exports_index_node);
|
|
36
36
|
|
|
37
|
-
//
|
|
38
|
-
var
|
|
37
|
+
// index.ts
|
|
38
|
+
var import_core14 = require("@elizaos/core");
|
|
39
39
|
|
|
40
|
-
//
|
|
40
|
+
// init.ts
|
|
41
41
|
var import_core2 = require("@elizaos/core");
|
|
42
42
|
|
|
43
|
-
//
|
|
43
|
+
// utils/config.ts
|
|
44
44
|
var import_core = require("@elizaos/core");
|
|
45
|
+
function getEnvValue(key) {
|
|
46
|
+
if (typeof process === "undefined" || !process.env) {
|
|
47
|
+
return;
|
|
48
|
+
}
|
|
49
|
+
const value = process.env[key];
|
|
50
|
+
return value === undefined ? undefined : String(value);
|
|
51
|
+
}
|
|
45
52
|
function getSetting(runtime, key, defaultValue) {
|
|
46
53
|
const value = runtime.getSetting(key);
|
|
47
54
|
if (value !== undefined && value !== null) {
|
|
48
55
|
return String(value);
|
|
49
56
|
}
|
|
50
|
-
return
|
|
57
|
+
return getEnvValue(key) ?? defaultValue;
|
|
58
|
+
}
|
|
59
|
+
function getNumericSetting(runtime, key, defaultValue) {
|
|
60
|
+
const value = getSetting(runtime, key);
|
|
61
|
+
if (value === undefined) {
|
|
62
|
+
return defaultValue;
|
|
63
|
+
}
|
|
64
|
+
const parsed = Number.parseInt(value, 10);
|
|
65
|
+
if (!Number.isFinite(parsed)) {
|
|
66
|
+
throw new Error(`Setting '${key}' must be a valid integer, got: ${value}`);
|
|
67
|
+
}
|
|
68
|
+
return parsed;
|
|
69
|
+
}
|
|
70
|
+
function getBooleanSetting(runtime, key, defaultValue) {
|
|
71
|
+
const value = getSetting(runtime, key);
|
|
72
|
+
if (value === undefined) {
|
|
73
|
+
return defaultValue;
|
|
74
|
+
}
|
|
75
|
+
const normalized = value.toLowerCase();
|
|
76
|
+
return normalized === "true" || normalized === "1" || normalized === "yes";
|
|
51
77
|
}
|
|
52
78
|
function isBrowser() {
|
|
53
|
-
return typeof globalThis !== "undefined" &&
|
|
79
|
+
return typeof globalThis !== "undefined" && typeof globalThis.document !== "undefined";
|
|
54
80
|
}
|
|
55
81
|
function isProxyMode(runtime) {
|
|
56
82
|
return isBrowser() && !!getSetting(runtime, "OPENAI_BROWSER_BASE_URL");
|
|
57
83
|
}
|
|
84
|
+
function getApiKey(runtime) {
|
|
85
|
+
return getSetting(runtime, "OPENAI_API_KEY");
|
|
86
|
+
}
|
|
87
|
+
function getEmbeddingApiKey(runtime) {
|
|
88
|
+
const embeddingApiKey = getSetting(runtime, "OPENAI_EMBEDDING_API_KEY");
|
|
89
|
+
if (embeddingApiKey) {
|
|
90
|
+
import_core.logger.debug("[OpenAI] Using specific embedding API key");
|
|
91
|
+
return embeddingApiKey;
|
|
92
|
+
}
|
|
93
|
+
import_core.logger.debug("[OpenAI] Falling back to general API key for embeddings");
|
|
94
|
+
return getApiKey(runtime);
|
|
95
|
+
}
|
|
58
96
|
function getAuthHeader(runtime, forEmbedding = false) {
|
|
59
|
-
if (isBrowser())
|
|
97
|
+
if (isBrowser() && !getBooleanSetting(runtime, "OPENAI_ALLOW_BROWSER_API_KEY", false)) {
|
|
60
98
|
return {};
|
|
99
|
+
}
|
|
61
100
|
const key = forEmbedding ? getEmbeddingApiKey(runtime) : getApiKey(runtime);
|
|
62
101
|
return key ? { Authorization: `Bearer ${key}` } : {};
|
|
63
102
|
}
|
|
64
103
|
function getBaseURL(runtime) {
|
|
65
104
|
const browserURL = getSetting(runtime, "OPENAI_BROWSER_BASE_URL");
|
|
66
|
-
const baseURL = isBrowser() && browserURL ? browserURL : getSetting(runtime, "OPENAI_BASE_URL"
|
|
67
|
-
import_core.logger.debug(`[OpenAI]
|
|
105
|
+
const baseURL = isBrowser() && browserURL ? browserURL : getSetting(runtime, "OPENAI_BASE_URL") ?? "https://api.openai.com/v1";
|
|
106
|
+
import_core.logger.debug(`[OpenAI] Base URL: ${baseURL}`);
|
|
68
107
|
return baseURL;
|
|
69
108
|
}
|
|
70
109
|
function getEmbeddingBaseURL(runtime) {
|
|
71
|
-
const embeddingURL = isBrowser() ? getSetting(runtime, "OPENAI_BROWSER_EMBEDDING_URL")
|
|
110
|
+
const embeddingURL = isBrowser() ? getSetting(runtime, "OPENAI_BROWSER_EMBEDDING_URL") ?? getSetting(runtime, "OPENAI_BROWSER_BASE_URL") : getSetting(runtime, "OPENAI_EMBEDDING_URL");
|
|
72
111
|
if (embeddingURL) {
|
|
73
|
-
import_core.logger.debug(`[OpenAI] Using
|
|
112
|
+
import_core.logger.debug(`[OpenAI] Using embedding base URL: ${embeddingURL}`);
|
|
74
113
|
return embeddingURL;
|
|
75
114
|
}
|
|
76
|
-
import_core.logger.debug("[OpenAI] Falling back to general base URL for embeddings
|
|
115
|
+
import_core.logger.debug("[OpenAI] Falling back to general base URL for embeddings");
|
|
77
116
|
return getBaseURL(runtime);
|
|
78
117
|
}
|
|
79
|
-
function getApiKey(runtime) {
|
|
80
|
-
return getSetting(runtime, "OPENAI_API_KEY");
|
|
81
|
-
}
|
|
82
|
-
function getEmbeddingApiKey(runtime) {
|
|
83
|
-
const embeddingApiKey = getSetting(runtime, "OPENAI_EMBEDDING_API_KEY");
|
|
84
|
-
if (embeddingApiKey) {
|
|
85
|
-
import_core.logger.debug("[OpenAI] Using specific embedding API key (present)");
|
|
86
|
-
return embeddingApiKey;
|
|
87
|
-
}
|
|
88
|
-
import_core.logger.debug("[OpenAI] Falling back to general API key for embeddings.");
|
|
89
|
-
return getApiKey(runtime);
|
|
90
|
-
}
|
|
91
118
|
function getSmallModel(runtime) {
|
|
92
|
-
return getSetting(runtime, "OPENAI_SMALL_MODEL") ?? getSetting(runtime, "SMALL_MODEL"
|
|
119
|
+
return getSetting(runtime, "OPENAI_SMALL_MODEL") ?? getSetting(runtime, "SMALL_MODEL") ?? "gpt-5-mini";
|
|
93
120
|
}
|
|
94
121
|
function getLargeModel(runtime) {
|
|
95
|
-
return getSetting(runtime, "OPENAI_LARGE_MODEL") ?? getSetting(runtime, "LARGE_MODEL"
|
|
122
|
+
return getSetting(runtime, "OPENAI_LARGE_MODEL") ?? getSetting(runtime, "LARGE_MODEL") ?? "gpt-5";
|
|
123
|
+
}
|
|
124
|
+
function getEmbeddingModel(runtime) {
|
|
125
|
+
return getSetting(runtime, "OPENAI_EMBEDDING_MODEL") ?? "text-embedding-3-small";
|
|
96
126
|
}
|
|
97
127
|
function getImageDescriptionModel(runtime) {
|
|
98
|
-
return getSetting(runtime, "OPENAI_IMAGE_DESCRIPTION_MODEL"
|
|
128
|
+
return getSetting(runtime, "OPENAI_IMAGE_DESCRIPTION_MODEL") ?? "gpt-5-mini";
|
|
99
129
|
}
|
|
100
|
-
function
|
|
101
|
-
|
|
102
|
-
const normalizedSetting = String(setting).toLowerCase();
|
|
103
|
-
const result = normalizedSetting === "true";
|
|
104
|
-
import_core.logger.debug(`[OpenAI] Experimental telemetry in function: "${setting}" (type: ${typeof setting}, normalized: "${normalizedSetting}", result: ${result})`);
|
|
105
|
-
return result;
|
|
130
|
+
function getTranscriptionModel(runtime) {
|
|
131
|
+
return getSetting(runtime, "OPENAI_TRANSCRIPTION_MODEL") ?? "gpt-5-mini-transcribe";
|
|
106
132
|
}
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
function initializeOpenAI(_config, runtime) {
|
|
110
|
-
(async () => {
|
|
111
|
-
try {
|
|
112
|
-
if (!getApiKey(runtime) && !isBrowser()) {
|
|
113
|
-
import_core2.logger.warn("OPENAI_API_KEY is not set in environment - OpenAI functionality will be limited");
|
|
114
|
-
return;
|
|
115
|
-
}
|
|
116
|
-
try {
|
|
117
|
-
const baseURL = getBaseURL(runtime);
|
|
118
|
-
const response = await fetch(`${baseURL}/models`, {
|
|
119
|
-
headers: getAuthHeader(runtime)
|
|
120
|
-
});
|
|
121
|
-
if (!response.ok) {
|
|
122
|
-
import_core2.logger.warn(`OpenAI API key validation failed: ${response.statusText}`);
|
|
123
|
-
import_core2.logger.warn("OpenAI functionality will be limited until a valid API key is provided");
|
|
124
|
-
} else {
|
|
125
|
-
import_core2.logger.log("OpenAI API key validated successfully");
|
|
126
|
-
}
|
|
127
|
-
} catch (fetchError) {
|
|
128
|
-
const message = fetchError instanceof Error ? fetchError.message : String(fetchError);
|
|
129
|
-
import_core2.logger.warn(`Error validating OpenAI API key: ${message}`);
|
|
130
|
-
import_core2.logger.warn("OpenAI functionality will be limited until a valid API key is provided");
|
|
131
|
-
}
|
|
132
|
-
} catch (error) {
|
|
133
|
-
const message = error?.errors?.map((e) => e.message).join(", ") || (error instanceof Error ? error.message : String(error));
|
|
134
|
-
import_core2.logger.warn(`OpenAI plugin configuration issue: ${message} - You need to configure the OPENAI_API_KEY in your environment variables`);
|
|
135
|
-
}
|
|
136
|
-
})();
|
|
133
|
+
function getTTSModel(runtime) {
|
|
134
|
+
return getSetting(runtime, "OPENAI_TTS_MODEL") ?? "tts-1";
|
|
137
135
|
}
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
var import_core4 = require("@elizaos/core");
|
|
141
|
-
var import_ai = require("ai");
|
|
142
|
-
|
|
143
|
-
// src/providers/openai.ts
|
|
144
|
-
var import_openai = require("@ai-sdk/openai");
|
|
145
|
-
function createOpenAIClient(runtime) {
|
|
146
|
-
const baseURL = getBaseURL(runtime);
|
|
147
|
-
const apiKey = getApiKey(runtime) ?? (isProxyMode(runtime) ? "sk-proxy" : undefined);
|
|
148
|
-
return import_openai.createOpenAI({ apiKey: apiKey ?? "", baseURL });
|
|
136
|
+
function getTTSVoice(runtime) {
|
|
137
|
+
return getSetting(runtime, "OPENAI_TTS_VOICE") ?? "nova";
|
|
149
138
|
}
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
var import_core3 = require("@elizaos/core");
|
|
153
|
-
function emitModelUsageEvent(runtime, type, prompt, usage) {
|
|
154
|
-
const promptTokens = ("promptTokens" in usage ? usage.promptTokens : undefined) ?? ("inputTokens" in usage ? usage.inputTokens : undefined) ?? 0;
|
|
155
|
-
const completionTokens = ("completionTokens" in usage ? usage.completionTokens : undefined) ?? ("outputTokens" in usage ? usage.outputTokens : undefined) ?? 0;
|
|
156
|
-
const totalTokens = ("totalTokens" in usage ? usage.totalTokens : undefined) ?? promptTokens + completionTokens;
|
|
157
|
-
const truncatedPrompt = typeof prompt === "string" ? prompt.length > 200 ? `${prompt.slice(0, 200)}…` : prompt : "";
|
|
158
|
-
runtime.emitEvent(import_core3.EventType.MODEL_USED, {
|
|
159
|
-
runtime,
|
|
160
|
-
source: "openai",
|
|
161
|
-
provider: "openai",
|
|
162
|
-
type,
|
|
163
|
-
prompt: truncatedPrompt,
|
|
164
|
-
tokens: {
|
|
165
|
-
prompt: promptTokens,
|
|
166
|
-
completion: completionTokens,
|
|
167
|
-
total: totalTokens
|
|
168
|
-
}
|
|
169
|
-
});
|
|
139
|
+
function getTTSInstructions(runtime) {
|
|
140
|
+
return getSetting(runtime, "OPENAI_TTS_INSTRUCTIONS") ?? "";
|
|
170
141
|
}
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
async function generateTextByModelType(runtime, params, modelType, getModelFn) {
|
|
174
|
-
const openai = createOpenAIClient(runtime);
|
|
175
|
-
const modelName = getModelFn(runtime);
|
|
176
|
-
import_core4.logger.debug(`[OpenAI] ${modelType} model: ${modelName}`);
|
|
177
|
-
const generateParams = {
|
|
178
|
-
model: openai.languageModel(modelName),
|
|
179
|
-
prompt: params.prompt,
|
|
180
|
-
system: runtime.character.system ?? undefined,
|
|
181
|
-
temperature: params.temperature ?? 0.7,
|
|
182
|
-
maxOutputTokens: params.maxTokens ?? 8192,
|
|
183
|
-
frequencyPenalty: params.frequencyPenalty ?? 0.7,
|
|
184
|
-
presencePenalty: params.presencePenalty ?? 0.7,
|
|
185
|
-
stopSequences: params.stopSequences ?? [],
|
|
186
|
-
experimental_telemetry: { isEnabled: getExperimentalTelemetry(runtime) }
|
|
187
|
-
};
|
|
188
|
-
if (params.stream) {
|
|
189
|
-
const result = import_ai.streamText(generateParams);
|
|
190
|
-
return {
|
|
191
|
-
textStream: result.textStream,
|
|
192
|
-
text: result.text,
|
|
193
|
-
usage: result.usage.then((u) => u ? {
|
|
194
|
-
promptTokens: u.inputTokens ?? 0,
|
|
195
|
-
completionTokens: u.outputTokens ?? 0,
|
|
196
|
-
totalTokens: (u.inputTokens ?? 0) + (u.outputTokens ?? 0)
|
|
197
|
-
} : undefined),
|
|
198
|
-
finishReason: result.finishReason
|
|
199
|
-
};
|
|
200
|
-
}
|
|
201
|
-
const { text, usage } = await import_ai.generateText(generateParams);
|
|
202
|
-
if (usage)
|
|
203
|
-
emitModelUsageEvent(runtime, modelType, params.prompt, usage);
|
|
204
|
-
return text;
|
|
142
|
+
function getImageModel(runtime) {
|
|
143
|
+
return getSetting(runtime, "OPENAI_IMAGE_MODEL") ?? "dall-e-3";
|
|
205
144
|
}
|
|
206
|
-
|
|
207
|
-
return
|
|
145
|
+
function getExperimentalTelemetry(runtime) {
|
|
146
|
+
return getBooleanSetting(runtime, "OPENAI_EXPERIMENTAL_TELEMETRY", false);
|
|
208
147
|
}
|
|
209
|
-
|
|
210
|
-
return
|
|
148
|
+
function getEmbeddingDimensions(runtime) {
|
|
149
|
+
return getNumericSetting(runtime, "OPENAI_EMBEDDING_DIMENSIONS", 1536);
|
|
211
150
|
}
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
async function handleTextEmbedding(runtime, params) {
|
|
215
|
-
const embeddingModelName = getSetting(runtime, "OPENAI_EMBEDDING_MODEL", "text-embedding-3-small");
|
|
216
|
-
const embeddingDimension = Number.parseInt(getSetting(runtime, "OPENAI_EMBEDDING_DIMENSIONS", "1536") || "1536", 10);
|
|
217
|
-
if (!Object.values(import_core5.VECTOR_DIMS).includes(embeddingDimension)) {
|
|
218
|
-
const errorMsg = `Invalid embedding dimension: ${embeddingDimension}. Must be one of: ${Object.values(import_core5.VECTOR_DIMS).join(", ")}`;
|
|
219
|
-
import_core5.logger.error(errorMsg);
|
|
220
|
-
throw new Error(errorMsg);
|
|
221
|
-
}
|
|
222
|
-
if (params === null) {
|
|
223
|
-
import_core5.logger.debug("Creating test embedding for initialization");
|
|
224
|
-
const testVector = Array(embeddingDimension).fill(0);
|
|
225
|
-
testVector[0] = 0.1;
|
|
226
|
-
return testVector;
|
|
227
|
-
}
|
|
228
|
-
let text;
|
|
229
|
-
if (typeof params === "string") {
|
|
230
|
-
text = params;
|
|
231
|
-
} else if (typeof params === "object" && params.text) {
|
|
232
|
-
text = params.text;
|
|
233
|
-
} else {
|
|
234
|
-
const errorMsg = "Invalid input format for embedding";
|
|
235
|
-
import_core5.logger.warn(errorMsg);
|
|
236
|
-
const fallbackVector = Array(embeddingDimension).fill(0);
|
|
237
|
-
fallbackVector[0] = 0.2;
|
|
238
|
-
return fallbackVector;
|
|
239
|
-
}
|
|
240
|
-
if (!text.trim()) {
|
|
241
|
-
const errorMsg = "Empty text for embedding";
|
|
242
|
-
import_core5.logger.warn(errorMsg);
|
|
243
|
-
const fallbackVector = Array(embeddingDimension).fill(0);
|
|
244
|
-
fallbackVector[0] = 0.3;
|
|
245
|
-
return fallbackVector;
|
|
246
|
-
}
|
|
247
|
-
const embeddingBaseURL = getEmbeddingBaseURL(runtime);
|
|
248
|
-
try {
|
|
249
|
-
const response = await fetch(`${embeddingBaseURL}/embeddings`, {
|
|
250
|
-
method: "POST",
|
|
251
|
-
headers: {
|
|
252
|
-
...getAuthHeader(runtime, true),
|
|
253
|
-
"Content-Type": "application/json"
|
|
254
|
-
},
|
|
255
|
-
body: JSON.stringify({
|
|
256
|
-
model: embeddingModelName,
|
|
257
|
-
input: text
|
|
258
|
-
})
|
|
259
|
-
});
|
|
260
|
-
if (!response.ok) {
|
|
261
|
-
import_core5.logger.error(`OpenAI API error: ${response.status} - ${response.statusText}`);
|
|
262
|
-
throw new Error(`OpenAI API error: ${response.status} - ${response.statusText}`);
|
|
263
|
-
}
|
|
264
|
-
const data = await response.json();
|
|
265
|
-
if (!data?.data?.[0]?.embedding) {
|
|
266
|
-
import_core5.logger.error("API returned invalid structure");
|
|
267
|
-
throw new Error("API returned invalid structure");
|
|
268
|
-
}
|
|
269
|
-
const embedding = data.data[0].embedding;
|
|
270
|
-
if (!Array.isArray(embedding) || embedding.length !== embeddingDimension) {
|
|
271
|
-
const errorMsg = `Embedding length ${embedding?.length ?? 0} does not match configured dimension ${embeddingDimension}`;
|
|
272
|
-
import_core5.logger.error(errorMsg);
|
|
273
|
-
const fallbackVector = Array(embeddingDimension).fill(0);
|
|
274
|
-
fallbackVector[0] = 0.4;
|
|
275
|
-
return fallbackVector;
|
|
276
|
-
}
|
|
277
|
-
if (data.usage) {
|
|
278
|
-
const usage = {
|
|
279
|
-
inputTokens: data.usage.prompt_tokens,
|
|
280
|
-
outputTokens: 0,
|
|
281
|
-
totalTokens: data.usage.total_tokens
|
|
282
|
-
};
|
|
283
|
-
emitModelUsageEvent(runtime, import_core5.ModelType.TEXT_EMBEDDING, text, usage);
|
|
284
|
-
}
|
|
285
|
-
import_core5.logger.log(`Got valid embedding with length ${embedding.length}`);
|
|
286
|
-
return embedding;
|
|
287
|
-
} catch (error) {
|
|
288
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
289
|
-
import_core5.logger.error(`Error generating embedding: ${message}`);
|
|
290
|
-
throw error instanceof Error ? error : new Error(message);
|
|
291
|
-
}
|
|
151
|
+
function getImageDescriptionMaxTokens(runtime) {
|
|
152
|
+
return getNumericSetting(runtime, "OPENAI_IMAGE_DESCRIPTION_MAX_TOKENS", 8192);
|
|
292
153
|
}
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
async function handleImageGeneration(runtime, params) {
|
|
296
|
-
const n = params.count || 1;
|
|
297
|
-
const size = params.size || "1024x1024";
|
|
298
|
-
const prompt = params.prompt;
|
|
299
|
-
const modelName = getSetting(runtime, "OPENAI_IMAGE_MODEL", "gpt-image-1");
|
|
300
|
-
import_core6.logger.log(`[OpenAI] Using IMAGE model: ${modelName}`);
|
|
301
|
-
const baseURL = getBaseURL(runtime);
|
|
302
|
-
try {
|
|
303
|
-
const response = await fetch(`${baseURL}/images/generations`, {
|
|
304
|
-
method: "POST",
|
|
305
|
-
headers: {
|
|
306
|
-
...getAuthHeader(runtime),
|
|
307
|
-
"Content-Type": "application/json"
|
|
308
|
-
},
|
|
309
|
-
body: JSON.stringify({
|
|
310
|
-
model: modelName,
|
|
311
|
-
prompt,
|
|
312
|
-
n,
|
|
313
|
-
size
|
|
314
|
-
})
|
|
315
|
-
});
|
|
316
|
-
if (!response.ok) {
|
|
317
|
-
throw new Error(`Failed to generate image: ${response.statusText}`);
|
|
318
|
-
}
|
|
319
|
-
const data = await response.json();
|
|
320
|
-
const typedData = data;
|
|
321
|
-
return typedData.data;
|
|
322
|
-
} catch (error) {
|
|
323
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
324
|
-
throw error;
|
|
325
|
-
}
|
|
154
|
+
function getResearchModel(runtime) {
|
|
155
|
+
return getSetting(runtime, "OPENAI_RESEARCH_MODEL") ?? "o3-deep-research";
|
|
326
156
|
}
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
157
|
+
function getResearchTimeout(runtime) {
|
|
158
|
+
return getNumericSetting(runtime, "OPENAI_RESEARCH_TIMEOUT", 3600000);
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
// init.ts
|
|
162
|
+
globalThis.AI_SDK_LOG_WARNINGS ??= false;
|
|
163
|
+
function initializeOpenAI(_config, runtime) {
|
|
164
|
+
validateOpenAIConfiguration(runtime);
|
|
165
|
+
}
|
|
166
|
+
async function validateOpenAIConfiguration(runtime) {
|
|
167
|
+
if (isBrowser()) {
|
|
168
|
+
import_core2.logger.debug("[OpenAI] Skipping API validation in browser environment");
|
|
169
|
+
return;
|
|
170
|
+
}
|
|
171
|
+
const apiKey = getApiKey(runtime);
|
|
172
|
+
if (!apiKey) {
|
|
173
|
+
import_core2.logger.warn("[OpenAI] OPENAI_API_KEY is not configured. " + "OpenAI functionality will fail until a valid API key is provided.");
|
|
174
|
+
return;
|
|
340
175
|
}
|
|
341
|
-
const messages = [
|
|
342
|
-
{
|
|
343
|
-
role: "user",
|
|
344
|
-
content: [
|
|
345
|
-
{ type: "text", text: promptText },
|
|
346
|
-
{ type: "image_url", image_url: { url: imageUrl } }
|
|
347
|
-
]
|
|
348
|
-
}
|
|
349
|
-
];
|
|
350
|
-
const baseURL = getBaseURL(runtime);
|
|
351
176
|
try {
|
|
352
|
-
const
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
max_tokens: maxTokens
|
|
356
|
-
};
|
|
357
|
-
const response = await fetch(`${baseURL}/chat/completions`, {
|
|
358
|
-
method: "POST",
|
|
359
|
-
headers: {
|
|
360
|
-
"Content-Type": "application/json",
|
|
361
|
-
...getAuthHeader(runtime)
|
|
362
|
-
},
|
|
363
|
-
body: JSON.stringify(requestBody)
|
|
177
|
+
const baseURL = getBaseURL(runtime);
|
|
178
|
+
const response = await fetch(`${baseURL}/models`, {
|
|
179
|
+
headers: getAuthHeader(runtime)
|
|
364
180
|
});
|
|
365
181
|
if (!response.ok) {
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
const result = await response.json();
|
|
369
|
-
const typedResult = result;
|
|
370
|
-
const content = typedResult.choices?.[0]?.message?.content;
|
|
371
|
-
if (typedResult.usage) {
|
|
372
|
-
emitModelUsageEvent(runtime, import_core6.ModelType.IMAGE_DESCRIPTION, typeof params === "string" ? params : params.prompt || "", {
|
|
373
|
-
inputTokens: typedResult.usage.prompt_tokens,
|
|
374
|
-
outputTokens: typedResult.usage.completion_tokens,
|
|
375
|
-
totalTokens: typedResult.usage.total_tokens
|
|
376
|
-
});
|
|
377
|
-
}
|
|
378
|
-
if (!content) {
|
|
379
|
-
return {
|
|
380
|
-
title: "Failed to analyze image",
|
|
381
|
-
description: "No response from API"
|
|
382
|
-
};
|
|
383
|
-
}
|
|
384
|
-
const titleMatch = content.match(/title[:\s]+(.+?)(?:\n|$)/i);
|
|
385
|
-
const title = titleMatch?.[1]?.trim();
|
|
386
|
-
if (!title) {
|
|
387
|
-
import_core6.logger.warn("Could not extract title from image description response");
|
|
182
|
+
import_core2.logger.warn(`[OpenAI] API key validation failed: ${response.status} ${response.statusText}. ` + "Please verify your OPENAI_API_KEY is correct.");
|
|
183
|
+
return;
|
|
388
184
|
}
|
|
389
|
-
const finalTitle = title || "Image Analysis";
|
|
390
|
-
const description = content.replace(/title[:\s]+(.+?)(?:\n|$)/i, "").trim();
|
|
391
|
-
const processedResult = { title: finalTitle, description };
|
|
392
|
-
return processedResult;
|
|
393
185
|
} catch (error) {
|
|
394
186
|
const message = error instanceof Error ? error.message : String(error);
|
|
395
|
-
|
|
396
|
-
return {
|
|
397
|
-
title: "Failed to analyze image",
|
|
398
|
-
description: `Error: ${message}`
|
|
399
|
-
};
|
|
187
|
+
import_core2.logger.warn(`[OpenAI] API validation error: ${message}. OpenAI functionality may be limited.`);
|
|
400
188
|
}
|
|
401
189
|
}
|
|
402
|
-
// src/models/audio.ts
|
|
403
|
-
var import_core8 = require("@elizaos/core");
|
|
404
190
|
|
|
405
|
-
//
|
|
406
|
-
var
|
|
191
|
+
// models/audio.ts
|
|
192
|
+
var import_core4 = require("@elizaos/core");
|
|
193
|
+
|
|
194
|
+
// utils/audio.ts
|
|
195
|
+
var import_core3 = require("@elizaos/core");
|
|
407
196
|
var MAGIC_BYTES = {
|
|
408
197
|
WAV: {
|
|
409
198
|
HEADER: [82, 73, 70, 70],
|
|
@@ -415,21 +204,26 @@ var MAGIC_BYTES = {
|
|
|
415
204
|
FTYP: [102, 116, 121, 112],
|
|
416
205
|
WEBM_EBML: [26, 69, 223, 163]
|
|
417
206
|
};
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
207
|
+
var MIN_DETECTION_BUFFER_SIZE = 12;
|
|
208
|
+
function matchBytes(buffer, offset, expected) {
|
|
209
|
+
for (let i = 0;i < expected.length; i++) {
|
|
210
|
+
const expectedByte = expected[i];
|
|
211
|
+
if (expectedByte === undefined || buffer[offset + i] !== expectedByte) {
|
|
421
212
|
return false;
|
|
213
|
+
}
|
|
422
214
|
}
|
|
423
215
|
return true;
|
|
424
216
|
}
|
|
425
217
|
function detectAudioMimeType(buffer) {
|
|
426
|
-
if (buffer.length <
|
|
218
|
+
if (buffer.length < MIN_DETECTION_BUFFER_SIZE) {
|
|
427
219
|
return "application/octet-stream";
|
|
428
220
|
}
|
|
429
221
|
if (matchBytes(buffer, 0, MAGIC_BYTES.WAV.HEADER) && matchBytes(buffer, 8, MAGIC_BYTES.WAV.IDENTIFIER)) {
|
|
430
222
|
return "audio/wav";
|
|
431
223
|
}
|
|
432
|
-
|
|
224
|
+
const firstByte = buffer[0];
|
|
225
|
+
const secondByte = buffer[1];
|
|
226
|
+
if (matchBytes(buffer, 0, MAGIC_BYTES.MP3_ID3) || firstByte === 255 && secondByte !== undefined && (secondByte & 224) === 224) {
|
|
433
227
|
return "audio/mpeg";
|
|
434
228
|
}
|
|
435
229
|
if (matchBytes(buffer, 0, MAGIC_BYTES.OGG)) {
|
|
@@ -444,284 +238,881 @@ function detectAudioMimeType(buffer) {
|
|
|
444
238
|
if (matchBytes(buffer, 0, MAGIC_BYTES.WEBM_EBML)) {
|
|
445
239
|
return "audio/webm";
|
|
446
240
|
}
|
|
447
|
-
|
|
241
|
+
import_core3.logger.warn("Could not detect audio format from buffer, using generic binary type");
|
|
448
242
|
return "application/octet-stream";
|
|
449
243
|
}
|
|
244
|
+
function getExtensionForMimeType(mimeType) {
|
|
245
|
+
switch (mimeType) {
|
|
246
|
+
case "audio/wav":
|
|
247
|
+
return "wav";
|
|
248
|
+
case "audio/mpeg":
|
|
249
|
+
return "mp3";
|
|
250
|
+
case "audio/ogg":
|
|
251
|
+
return "ogg";
|
|
252
|
+
case "audio/flac":
|
|
253
|
+
return "flac";
|
|
254
|
+
case "audio/mp4":
|
|
255
|
+
return "m4a";
|
|
256
|
+
case "audio/webm":
|
|
257
|
+
return "webm";
|
|
258
|
+
case "application/octet-stream":
|
|
259
|
+
return "bin";
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
function getFilenameForMimeType(mimeType) {
|
|
263
|
+
const ext = getExtensionForMimeType(mimeType);
|
|
264
|
+
return `recording.${ext}`;
|
|
265
|
+
}
|
|
450
266
|
|
|
451
|
-
//
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
},
|
|
469
|
-
body: JSON.stringify({
|
|
470
|
-
model,
|
|
471
|
-
voice,
|
|
472
|
-
input: options.text,
|
|
473
|
-
format,
|
|
474
|
-
...instructions && { instructions }
|
|
475
|
-
})
|
|
476
|
-
});
|
|
477
|
-
if (!res.ok) {
|
|
478
|
-
const err = await res.text();
|
|
479
|
-
throw new Error(`OpenAI TTS error ${res.status}: ${err}`);
|
|
480
|
-
}
|
|
481
|
-
return await res.arrayBuffer();
|
|
482
|
-
} catch (err) {
|
|
483
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
484
|
-
throw new Error(`Failed to fetch speech from OpenAI TTS: ${message}`);
|
|
267
|
+
// models/audio.ts
|
|
268
|
+
function isBlobOrFile(value) {
|
|
269
|
+
return value instanceof Blob || value instanceof File;
|
|
270
|
+
}
|
|
271
|
+
function isBuffer(value) {
|
|
272
|
+
return Buffer.isBuffer(value);
|
|
273
|
+
}
|
|
274
|
+
function isLocalTranscriptionParams(value) {
|
|
275
|
+
return typeof value === "object" && value !== null && "audio" in value && (isBlobOrFile(value.audio) || isBuffer(value.audio));
|
|
276
|
+
}
|
|
277
|
+
function isCoreTranscriptionParams(value) {
|
|
278
|
+
return typeof value === "object" && value !== null && "audioUrl" in value && typeof value.audioUrl === "string";
|
|
279
|
+
}
|
|
280
|
+
async function fetchAudioFromUrl(url) {
|
|
281
|
+
const response = await fetch(url);
|
|
282
|
+
if (!response.ok) {
|
|
283
|
+
throw new Error(`Failed to fetch audio from URL: ${response.status}`);
|
|
485
284
|
}
|
|
285
|
+
return response.blob();
|
|
486
286
|
}
|
|
487
287
|
async function handleTranscription(runtime, input) {
|
|
488
|
-
let modelName =
|
|
489
|
-
import_core8.logger.log(`[OpenAI] Using TRANSCRIPTION model: ${modelName}`);
|
|
490
|
-
const baseURL = getBaseURL(runtime);
|
|
288
|
+
let modelName = getTranscriptionModel(runtime);
|
|
491
289
|
let blob;
|
|
492
|
-
let extraParams =
|
|
493
|
-
if (
|
|
290
|
+
let extraParams = {};
|
|
291
|
+
if (typeof input === "string") {
|
|
292
|
+
import_core4.logger.debug(`[OpenAI] Fetching audio from URL: ${input}`);
|
|
293
|
+
blob = await fetchAudioFromUrl(input);
|
|
294
|
+
} else if (isBlobOrFile(input)) {
|
|
494
295
|
blob = input;
|
|
495
|
-
} else if (
|
|
496
|
-
const
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
throw new Error("TRANSCRIPTION param 'audio' must be a Blob/File/Buffer.");
|
|
296
|
+
} else if (isBuffer(input)) {
|
|
297
|
+
const mimeType2 = detectAudioMimeType(input);
|
|
298
|
+
import_core4.logger.debug(`[OpenAI] Auto-detected audio MIME type: ${mimeType2}`);
|
|
299
|
+
blob = new Blob([new Uint8Array(input)], { type: mimeType2 });
|
|
300
|
+
} else if (isLocalTranscriptionParams(input)) {
|
|
301
|
+
extraParams = input;
|
|
302
|
+
if (input.model) {
|
|
303
|
+
modelName = input.model;
|
|
504
304
|
}
|
|
505
|
-
if (
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
import_core8.logger.debug(`Auto-detected audio MIME type: ${mimeType}`);
|
|
510
|
-
} else {
|
|
511
|
-
import_core8.logger.debug(`Using provided MIME type: ${mimeType}`);
|
|
512
|
-
}
|
|
513
|
-
const uint8Array = new Uint8Array(params.audio);
|
|
514
|
-
blob = new Blob([uint8Array], { type: mimeType });
|
|
305
|
+
if (isBuffer(input.audio)) {
|
|
306
|
+
const mimeType2 = input.mimeType ?? detectAudioMimeType(input.audio);
|
|
307
|
+
import_core4.logger.debug(`[OpenAI] Using MIME type: ${mimeType2}`);
|
|
308
|
+
blob = new Blob([new Uint8Array(input.audio)], { type: mimeType2 });
|
|
515
309
|
} else {
|
|
516
|
-
blob =
|
|
517
|
-
}
|
|
518
|
-
extraParams = params;
|
|
519
|
-
if (typeof params.model === "string" && params.model) {
|
|
520
|
-
modelName = params.model;
|
|
310
|
+
blob = input.audio;
|
|
521
311
|
}
|
|
312
|
+
} else if (isCoreTranscriptionParams(input)) {
|
|
313
|
+
import_core4.logger.debug(`[OpenAI] Fetching audio from URL: ${input.audioUrl}`);
|
|
314
|
+
blob = await fetchAudioFromUrl(input.audioUrl);
|
|
315
|
+
extraParams = { prompt: input.prompt };
|
|
522
316
|
} else {
|
|
523
|
-
throw new Error("TRANSCRIPTION expects
|
|
317
|
+
throw new Error("TRANSCRIPTION expects Blob, File, Buffer, URL string, or TranscriptionParams object");
|
|
524
318
|
}
|
|
525
|
-
|
|
526
|
-
const
|
|
319
|
+
import_core4.logger.debug(`[OpenAI] Using TRANSCRIPTION model: ${modelName}`);
|
|
320
|
+
const mimeType = blob.type || "audio/webm";
|
|
321
|
+
const filename = blob.name || getFilenameForMimeType(mimeType.startsWith("audio/") ? mimeType : "audio/webm");
|
|
527
322
|
const formData = new FormData;
|
|
528
323
|
formData.append("file", blob, filename);
|
|
529
|
-
formData.append("model",
|
|
530
|
-
if (extraParams) {
|
|
531
|
-
|
|
532
|
-
|
|
324
|
+
formData.append("model", modelName);
|
|
325
|
+
if (extraParams.language) {
|
|
326
|
+
formData.append("language", extraParams.language);
|
|
327
|
+
}
|
|
328
|
+
if (extraParams.responseFormat) {
|
|
329
|
+
formData.append("response_format", extraParams.responseFormat);
|
|
330
|
+
}
|
|
331
|
+
if (extraParams.prompt) {
|
|
332
|
+
formData.append("prompt", extraParams.prompt);
|
|
333
|
+
}
|
|
334
|
+
if (extraParams.temperature !== undefined) {
|
|
335
|
+
formData.append("temperature", String(extraParams.temperature));
|
|
336
|
+
}
|
|
337
|
+
if (extraParams.timestampGranularities) {
|
|
338
|
+
for (const granularity of extraParams.timestampGranularities) {
|
|
339
|
+
formData.append("timestamp_granularities[]", granularity);
|
|
533
340
|
}
|
|
534
|
-
|
|
535
|
-
|
|
341
|
+
}
|
|
342
|
+
const baseURL = getBaseURL(runtime);
|
|
343
|
+
const response = await fetch(`${baseURL}/audio/transcriptions`, {
|
|
344
|
+
method: "POST",
|
|
345
|
+
headers: getAuthHeader(runtime),
|
|
346
|
+
body: formData
|
|
347
|
+
});
|
|
348
|
+
if (!response.ok) {
|
|
349
|
+
const errorText = await response.text().catch(() => "Unknown error");
|
|
350
|
+
throw new Error(`OpenAI transcription failed: ${response.status} ${response.statusText} - ${errorText}`);
|
|
351
|
+
}
|
|
352
|
+
const data = await response.json();
|
|
353
|
+
return data.text;
|
|
354
|
+
}
|
|
355
|
+
async function handleTextToSpeech(runtime, input) {
|
|
356
|
+
let text;
|
|
357
|
+
let voice;
|
|
358
|
+
let format = "mp3";
|
|
359
|
+
let model;
|
|
360
|
+
let instructions;
|
|
361
|
+
if (typeof input === "string") {
|
|
362
|
+
text = input;
|
|
363
|
+
voice = undefined;
|
|
364
|
+
} else {
|
|
365
|
+
text = input.text;
|
|
366
|
+
voice = input.voice;
|
|
367
|
+
if ("format" in input && input.format) {
|
|
368
|
+
format = input.format;
|
|
536
369
|
}
|
|
537
|
-
if (
|
|
538
|
-
|
|
370
|
+
if ("model" in input && input.model) {
|
|
371
|
+
model = input.model;
|
|
539
372
|
}
|
|
540
|
-
if (
|
|
541
|
-
|
|
373
|
+
if ("instructions" in input && input.instructions) {
|
|
374
|
+
instructions = input.instructions;
|
|
542
375
|
}
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
376
|
+
}
|
|
377
|
+
model = model ?? getTTSModel(runtime);
|
|
378
|
+
voice = voice ?? getTTSVoice(runtime);
|
|
379
|
+
instructions = instructions ?? getTTSInstructions(runtime);
|
|
380
|
+
import_core4.logger.debug(`[OpenAI] Using TEXT_TO_SPEECH model: ${model}`);
|
|
381
|
+
if (!text || text.trim().length === 0) {
|
|
382
|
+
throw new Error("TEXT_TO_SPEECH requires non-empty text");
|
|
383
|
+
}
|
|
384
|
+
if (text.length > 4096) {
|
|
385
|
+
throw new Error("TEXT_TO_SPEECH text exceeds 4096 character limit");
|
|
386
|
+
}
|
|
387
|
+
const validVoices = ["alloy", "echo", "fable", "onyx", "nova", "shimmer"];
|
|
388
|
+
if (voice && !validVoices.includes(voice)) {
|
|
389
|
+
throw new Error(`Invalid voice: ${voice}. Must be one of: ${validVoices.join(", ")}`);
|
|
390
|
+
}
|
|
391
|
+
const baseURL = getBaseURL(runtime);
|
|
392
|
+
const requestBody = {
|
|
393
|
+
model,
|
|
394
|
+
voice,
|
|
395
|
+
input: text,
|
|
396
|
+
response_format: format
|
|
397
|
+
};
|
|
398
|
+
if (instructions && instructions.length > 0) {
|
|
399
|
+
requestBody.instructions = instructions;
|
|
400
|
+
}
|
|
401
|
+
const response = await fetch(`${baseURL}/audio/speech`, {
|
|
402
|
+
method: "POST",
|
|
403
|
+
headers: {
|
|
404
|
+
...getAuthHeader(runtime),
|
|
405
|
+
"Content-Type": "application/json",
|
|
406
|
+
...format === "mp3" ? { Accept: "audio/mpeg" } : {}
|
|
407
|
+
},
|
|
408
|
+
body: JSON.stringify(requestBody)
|
|
409
|
+
});
|
|
410
|
+
if (!response.ok) {
|
|
411
|
+
const errorText = await response.text().catch(() => "Unknown error");
|
|
412
|
+
throw new Error(`OpenAI TTS failed: ${response.status} ${response.statusText} - ${errorText}`);
|
|
413
|
+
}
|
|
414
|
+
return response.arrayBuffer();
|
|
415
|
+
}
|
|
416
|
+
// models/embedding.ts
|
|
417
|
+
var import_core6 = require("@elizaos/core");
|
|
418
|
+
|
|
419
|
+
// utils/events.ts
|
|
420
|
+
var import_core5 = require("@elizaos/core");
|
|
421
|
+
var MAX_PROMPT_LENGTH = 200;
|
|
422
|
+
function truncatePrompt(prompt) {
|
|
423
|
+
if (prompt.length <= MAX_PROMPT_LENGTH) {
|
|
424
|
+
return prompt;
|
|
425
|
+
}
|
|
426
|
+
return `${prompt.slice(0, MAX_PROMPT_LENGTH)}…`;
|
|
427
|
+
}
|
|
428
|
+
function normalizeUsage(usage) {
|
|
429
|
+
if ("promptTokens" in usage) {
|
|
430
|
+
return {
|
|
431
|
+
promptTokens: usage.promptTokens ?? 0,
|
|
432
|
+
completionTokens: usage.completionTokens ?? 0,
|
|
433
|
+
totalTokens: usage.totalTokens ?? (usage.promptTokens ?? 0) + (usage.completionTokens ?? 0)
|
|
434
|
+
};
|
|
435
|
+
}
|
|
436
|
+
if ("inputTokens" in usage || "outputTokens" in usage) {
|
|
437
|
+
const input = usage.inputTokens ?? 0;
|
|
438
|
+
const output = usage.outputTokens ?? 0;
|
|
439
|
+
return {
|
|
440
|
+
promptTokens: input,
|
|
441
|
+
completionTokens: output,
|
|
442
|
+
totalTokens: input + output
|
|
443
|
+
};
|
|
444
|
+
}
|
|
445
|
+
return {
|
|
446
|
+
promptTokens: 0,
|
|
447
|
+
completionTokens: 0,
|
|
448
|
+
totalTokens: 0
|
|
449
|
+
};
|
|
450
|
+
}
|
|
451
|
+
function emitModelUsageEvent(runtime, type, prompt, usage) {
|
|
452
|
+
const normalized = normalizeUsage(usage);
|
|
453
|
+
const payload = {
|
|
454
|
+
runtime,
|
|
455
|
+
source: "openai",
|
|
456
|
+
provider: "openai",
|
|
457
|
+
type,
|
|
458
|
+
prompt: truncatePrompt(prompt),
|
|
459
|
+
tokens: {
|
|
460
|
+
prompt: normalized.promptTokens,
|
|
461
|
+
completion: normalized.completionTokens,
|
|
462
|
+
total: normalized.totalTokens
|
|
547
463
|
}
|
|
464
|
+
};
|
|
465
|
+
runtime.emitEvent(import_core5.EventType.MODEL_USED, payload);
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
// models/embedding.ts
|
|
469
|
+
function validateDimension(dimension) {
|
|
470
|
+
const validDimensions = Object.values(import_core6.VECTOR_DIMS);
|
|
471
|
+
if (!validDimensions.includes(dimension)) {
|
|
472
|
+
throw new Error(`Invalid embedding dimension: ${dimension}. Must be one of: ${validDimensions.join(", ")}`);
|
|
548
473
|
}
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
474
|
+
return dimension;
|
|
475
|
+
}
|
|
476
|
+
function extractText(params) {
|
|
477
|
+
if (params === null) {
|
|
478
|
+
return null;
|
|
479
|
+
}
|
|
480
|
+
if (typeof params === "string") {
|
|
481
|
+
return params;
|
|
482
|
+
}
|
|
483
|
+
if (typeof params === "object" && typeof params.text === "string") {
|
|
484
|
+
return params.text;
|
|
485
|
+
}
|
|
486
|
+
throw new Error("Invalid embedding params: expected string, { text: string }, or null");
|
|
487
|
+
}
|
|
488
|
+
async function handleTextEmbedding(runtime, params) {
|
|
489
|
+
const embeddingModel = getEmbeddingModel(runtime);
|
|
490
|
+
const embeddingDimension = validateDimension(getEmbeddingDimensions(runtime));
|
|
491
|
+
const text = extractText(params);
|
|
492
|
+
if (text === null) {
|
|
493
|
+
import_core6.logger.debug("[OpenAI] Creating test embedding for initialization");
|
|
494
|
+
const testVector = new Array(embeddingDimension).fill(0);
|
|
495
|
+
testVector[0] = 0.1;
|
|
496
|
+
return testVector;
|
|
497
|
+
}
|
|
498
|
+
const trimmedText = text.trim();
|
|
499
|
+
if (trimmedText.length === 0) {
|
|
500
|
+
throw new Error("Cannot generate embedding for empty text");
|
|
501
|
+
}
|
|
502
|
+
const baseURL = getEmbeddingBaseURL(runtime);
|
|
503
|
+
const url = `${baseURL}/embeddings`;
|
|
504
|
+
import_core6.logger.debug(`[OpenAI] Generating embedding with model: ${embeddingModel}`);
|
|
505
|
+
const response = await fetch(url, {
|
|
506
|
+
method: "POST",
|
|
507
|
+
headers: {
|
|
508
|
+
...getAuthHeader(runtime, true),
|
|
509
|
+
"Content-Type": "application/json"
|
|
510
|
+
},
|
|
511
|
+
body: JSON.stringify({
|
|
512
|
+
model: embeddingModel,
|
|
513
|
+
input: trimmedText
|
|
514
|
+
})
|
|
515
|
+
});
|
|
516
|
+
if (!response.ok) {
|
|
517
|
+
const errorText = await response.text().catch(() => "Unknown error");
|
|
518
|
+
throw new Error(`OpenAI embedding API error: ${response.status} ${response.statusText} - ${errorText}`);
|
|
519
|
+
}
|
|
520
|
+
const data = await response.json();
|
|
521
|
+
const firstResult = data?.data?.[0];
|
|
522
|
+
if (!firstResult || !firstResult.embedding) {
|
|
523
|
+
throw new Error("OpenAI API returned invalid embedding response structure");
|
|
524
|
+
}
|
|
525
|
+
const embedding = firstResult.embedding;
|
|
526
|
+
if (embedding.length !== embeddingDimension) {
|
|
527
|
+
throw new Error(`Embedding dimension mismatch: got ${embedding.length}, expected ${embeddingDimension}. ` + `Check OPENAI_EMBEDDING_DIMENSIONS setting.`);
|
|
528
|
+
}
|
|
529
|
+
if (data.usage) {
|
|
530
|
+
emitModelUsageEvent(runtime, import_core6.ModelType.TEXT_EMBEDDING, trimmedText, {
|
|
531
|
+
promptTokens: data.usage.prompt_tokens,
|
|
532
|
+
completionTokens: 0,
|
|
533
|
+
totalTokens: data.usage.total_tokens
|
|
556
534
|
});
|
|
557
|
-
if (!response.ok) {
|
|
558
|
-
throw new Error(`Failed to transcribe audio: ${response.status} ${response.statusText}`);
|
|
559
|
-
}
|
|
560
|
-
const data = await response.json();
|
|
561
|
-
return data.text || "";
|
|
562
|
-
} catch (error) {
|
|
563
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
564
|
-
import_core8.logger.error(`TRANSCRIPTION error: ${message}`);
|
|
565
|
-
throw error;
|
|
566
535
|
}
|
|
536
|
+
import_core6.logger.debug(`[OpenAI] Generated embedding with ${embedding.length} dimensions`);
|
|
537
|
+
return embedding;
|
|
567
538
|
}
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
539
|
+
// models/image.ts
|
|
540
|
+
var import_core7 = require("@elizaos/core");
|
|
541
|
+
var DEFAULT_IMAGE_DESCRIPTION_PROMPT = "Please analyze this image and provide a title and detailed description.";
|
|
542
|
+
async function handleImageGeneration(runtime, params) {
|
|
543
|
+
const modelName = getImageModel(runtime);
|
|
544
|
+
const count = params.count ?? 1;
|
|
545
|
+
const size = params.size ?? "1024x1024";
|
|
546
|
+
const extendedParams = params;
|
|
547
|
+
import_core7.logger.debug(`[OpenAI] Using IMAGE model: ${modelName}`);
|
|
548
|
+
if (!params.prompt || params.prompt.trim().length === 0) {
|
|
549
|
+
throw new Error("IMAGE generation requires a non-empty prompt");
|
|
550
|
+
}
|
|
551
|
+
if (count < 1 || count > 10) {
|
|
552
|
+
throw new Error("IMAGE count must be between 1 and 10");
|
|
578
553
|
}
|
|
554
|
+
const baseURL = getBaseURL(runtime);
|
|
555
|
+
const requestBody = {
|
|
556
|
+
model: modelName,
|
|
557
|
+
prompt: params.prompt,
|
|
558
|
+
n: count,
|
|
559
|
+
size
|
|
560
|
+
};
|
|
561
|
+
if (extendedParams.quality) {
|
|
562
|
+
requestBody.quality = extendedParams.quality;
|
|
563
|
+
}
|
|
564
|
+
if (extendedParams.style) {
|
|
565
|
+
requestBody.style = extendedParams.style;
|
|
566
|
+
}
|
|
567
|
+
const response = await fetch(`${baseURL}/images/generations`, {
|
|
568
|
+
method: "POST",
|
|
569
|
+
headers: {
|
|
570
|
+
...getAuthHeader(runtime),
|
|
571
|
+
"Content-Type": "application/json"
|
|
572
|
+
},
|
|
573
|
+
body: JSON.stringify(requestBody)
|
|
574
|
+
});
|
|
575
|
+
if (!response.ok) {
|
|
576
|
+
const errorText = await response.text().catch(() => "Unknown error");
|
|
577
|
+
throw new Error(`OpenAI image generation failed: ${response.status} ${response.statusText} - ${errorText}`);
|
|
578
|
+
}
|
|
579
|
+
const data = await response.json();
|
|
580
|
+
if (!data.data || data.data.length === 0) {
|
|
581
|
+
throw new Error("OpenAI API returned no images");
|
|
582
|
+
}
|
|
583
|
+
return data.data.map((item) => ({
|
|
584
|
+
url: item.url,
|
|
585
|
+
revisedPrompt: item.revised_prompt
|
|
586
|
+
}));
|
|
579
587
|
}
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
588
|
+
function parseTitleFromResponse(content) {
|
|
589
|
+
const titleMatch = content.match(/title[:\s]+(.+?)(?:\n|$)/i);
|
|
590
|
+
return titleMatch?.[1]?.trim() ?? "Image Analysis";
|
|
591
|
+
}
|
|
592
|
+
function parseDescriptionFromResponse(content) {
|
|
593
|
+
return content.replace(/title[:\s]+(.+?)(?:\n|$)/i, "").trim();
|
|
594
|
+
}
|
|
595
|
+
async function handleImageDescription(runtime, params) {
|
|
596
|
+
const modelName = getImageDescriptionModel(runtime);
|
|
597
|
+
const maxTokens = getImageDescriptionMaxTokens(runtime);
|
|
598
|
+
import_core7.logger.debug(`[OpenAI] Using IMAGE_DESCRIPTION model: ${modelName}`);
|
|
599
|
+
let imageUrl;
|
|
600
|
+
let promptText;
|
|
601
|
+
if (typeof params === "string") {
|
|
602
|
+
imageUrl = params;
|
|
603
|
+
promptText = DEFAULT_IMAGE_DESCRIPTION_PROMPT;
|
|
604
|
+
} else {
|
|
605
|
+
imageUrl = params.imageUrl;
|
|
606
|
+
promptText = params.prompt ?? DEFAULT_IMAGE_DESCRIPTION_PROMPT;
|
|
607
|
+
}
|
|
608
|
+
if (!imageUrl || imageUrl.trim().length === 0) {
|
|
609
|
+
throw new Error("IMAGE_DESCRIPTION requires a valid image URL");
|
|
610
|
+
}
|
|
611
|
+
const baseURL = getBaseURL(runtime);
|
|
612
|
+
const requestBody = {
|
|
613
|
+
model: modelName,
|
|
614
|
+
messages: [
|
|
615
|
+
{
|
|
616
|
+
role: "user",
|
|
617
|
+
content: [
|
|
618
|
+
{ type: "text", text: promptText },
|
|
619
|
+
{ type: "image_url", image_url: { url: imageUrl } }
|
|
620
|
+
]
|
|
621
|
+
}
|
|
622
|
+
],
|
|
623
|
+
max_tokens: maxTokens
|
|
624
|
+
};
|
|
625
|
+
const response = await fetch(`${baseURL}/chat/completions`, {
|
|
626
|
+
method: "POST",
|
|
627
|
+
headers: {
|
|
628
|
+
...getAuthHeader(runtime),
|
|
629
|
+
"Content-Type": "application/json"
|
|
630
|
+
},
|
|
631
|
+
body: JSON.stringify(requestBody)
|
|
632
|
+
});
|
|
633
|
+
if (!response.ok) {
|
|
634
|
+
const errorText = await response.text().catch(() => "Unknown error");
|
|
635
|
+
throw new Error(`OpenAI image description failed: ${response.status} ${response.statusText} - ${errorText}`);
|
|
636
|
+
}
|
|
637
|
+
const data = await response.json();
|
|
638
|
+
if (data.usage) {
|
|
639
|
+
emitModelUsageEvent(runtime, import_core7.ModelType.IMAGE_DESCRIPTION, typeof params === "string" ? params : params.prompt ?? "", {
|
|
640
|
+
promptTokens: data.usage.prompt_tokens,
|
|
641
|
+
completionTokens: data.usage.completion_tokens,
|
|
642
|
+
totalTokens: data.usage.total_tokens
|
|
643
|
+
});
|
|
644
|
+
}
|
|
645
|
+
const firstChoice = data.choices?.[0];
|
|
646
|
+
const content = firstChoice?.message?.content;
|
|
647
|
+
if (!content) {
|
|
648
|
+
throw new Error("OpenAI API returned empty image description");
|
|
649
|
+
}
|
|
650
|
+
return {
|
|
651
|
+
title: parseTitleFromResponse(content),
|
|
652
|
+
description: parseDescriptionFromResponse(content)
|
|
653
|
+
};
|
|
654
|
+
}
|
|
655
|
+
// models/object.ts
|
|
585
656
|
var import_core9 = require("@elizaos/core");
|
|
586
657
|
var import_ai2 = require("ai");
|
|
658
|
+
|
|
659
|
+
// providers/openai.ts
|
|
660
|
+
var import_openai = require("@ai-sdk/openai");
|
|
661
|
+
var PROXY_API_KEY = "sk-proxy";
|
|
662
|
+
function createOpenAIClient(runtime) {
|
|
663
|
+
const baseURL = getBaseURL(runtime);
|
|
664
|
+
const apiKey = getApiKey(runtime);
|
|
665
|
+
if (!apiKey && isProxyMode(runtime)) {
|
|
666
|
+
return import_openai.createOpenAI({
|
|
667
|
+
apiKey: PROXY_API_KEY,
|
|
668
|
+
baseURL
|
|
669
|
+
});
|
|
670
|
+
}
|
|
671
|
+
if (!apiKey) {
|
|
672
|
+
throw new Error("OPENAI_API_KEY is required. Set it in your environment variables or runtime settings.");
|
|
673
|
+
}
|
|
674
|
+
return import_openai.createOpenAI({
|
|
675
|
+
apiKey,
|
|
676
|
+
baseURL
|
|
677
|
+
});
|
|
678
|
+
}
|
|
679
|
+
// utils/json.ts
|
|
680
|
+
var import_core8 = require("@elizaos/core");
|
|
681
|
+
var import_ai = require("ai");
|
|
682
|
+
var JSON_CLEANUP_PATTERNS = {
|
|
683
|
+
MARKDOWN_JSON: /```json\n|\n```|```/g,
|
|
684
|
+
WHITESPACE: /^\s+|\s+$/g
|
|
685
|
+
};
|
|
587
686
|
function getJsonRepairFunction() {
|
|
588
687
|
return async ({ text, error }) => {
|
|
589
|
-
|
|
590
|
-
if (error instanceof import_ai2.JSONParseError) {
|
|
591
|
-
const cleanedText = text.replace(/```json\n|\n```|```/g, "");
|
|
592
|
-
JSON.parse(cleanedText);
|
|
593
|
-
return cleanedText;
|
|
594
|
-
}
|
|
688
|
+
if (!(error instanceof import_ai.JSONParseError)) {
|
|
595
689
|
return null;
|
|
596
|
-
}
|
|
597
|
-
|
|
598
|
-
|
|
690
|
+
}
|
|
691
|
+
try {
|
|
692
|
+
const cleanedText = text.replace(JSON_CLEANUP_PATTERNS.MARKDOWN_JSON, "");
|
|
693
|
+
JSON.parse(cleanedText);
|
|
694
|
+
import_core8.logger.debug("[JSON Repair] Successfully repaired JSON by removing markdown wrappers");
|
|
695
|
+
return cleanedText;
|
|
696
|
+
} catch {
|
|
697
|
+
import_core8.logger.warn("[JSON Repair] Unable to repair JSON text");
|
|
599
698
|
return null;
|
|
600
699
|
}
|
|
601
700
|
};
|
|
602
701
|
}
|
|
603
702
|
|
|
604
|
-
//
|
|
703
|
+
// models/object.ts
|
|
605
704
|
async function generateObjectByModelType(runtime, params, modelType, getModelFn) {
|
|
606
705
|
const openai = createOpenAIClient(runtime);
|
|
607
706
|
const modelName = getModelFn(runtime);
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
if (schemaPresent) {
|
|
612
|
-
import_core10.logger.warn(`Schema provided but ignored: OpenAI object generation currently uses output=no-schema. The schema parameter has no effect.`);
|
|
707
|
+
import_core9.logger.debug(`[OpenAI] Using ${modelType} model: ${modelName}`);
|
|
708
|
+
if (!params.prompt || params.prompt.trim().length === 0) {
|
|
709
|
+
throw new Error("Object generation requires a non-empty prompt");
|
|
613
710
|
}
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
model: openai.languageModel(modelName),
|
|
617
|
-
output: "no-schema",
|
|
618
|
-
prompt: params.prompt,
|
|
619
|
-
temperature,
|
|
620
|
-
experimental_repairText: getJsonRepairFunction()
|
|
621
|
-
});
|
|
622
|
-
if (usage) {
|
|
623
|
-
emitModelUsageEvent(runtime, modelType, params.prompt, usage);
|
|
624
|
-
}
|
|
625
|
-
return object;
|
|
626
|
-
} catch (error) {
|
|
627
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
628
|
-
import_core10.logger.error(`[generateObject] Error: ${message}`);
|
|
629
|
-
throw error;
|
|
711
|
+
if (params.schema) {
|
|
712
|
+
import_core9.logger.debug("[OpenAI] Schema provided but using no-schema mode. " + "Structure is determined by prompt instructions.");
|
|
630
713
|
}
|
|
714
|
+
const model = openai.chat(modelName);
|
|
715
|
+
const { object, usage } = await import_ai2.generateObject({
|
|
716
|
+
model,
|
|
717
|
+
output: "no-schema",
|
|
718
|
+
prompt: params.prompt,
|
|
719
|
+
experimental_repairText: getJsonRepairFunction()
|
|
720
|
+
});
|
|
721
|
+
if (usage) {
|
|
722
|
+
emitModelUsageEvent(runtime, modelType, params.prompt, usage);
|
|
723
|
+
}
|
|
724
|
+
if (typeof object !== "object" || object === null) {
|
|
725
|
+
throw new Error(`Object generation returned ${typeof object}, expected object`);
|
|
726
|
+
}
|
|
727
|
+
return object;
|
|
631
728
|
}
|
|
632
729
|
async function handleObjectSmall(runtime, params) {
|
|
633
|
-
return generateObjectByModelType(runtime, params,
|
|
730
|
+
return generateObjectByModelType(runtime, params, import_core9.ModelType.OBJECT_SMALL, getSmallModel);
|
|
634
731
|
}
|
|
635
732
|
async function handleObjectLarge(runtime, params) {
|
|
636
|
-
return generateObjectByModelType(runtime, params,
|
|
733
|
+
return generateObjectByModelType(runtime, params, import_core9.ModelType.OBJECT_LARGE, getLargeModel);
|
|
637
734
|
}
|
|
638
|
-
//
|
|
639
|
-
var
|
|
640
|
-
|
|
641
|
-
|
|
735
|
+
// models/research.ts
|
|
736
|
+
var import_core10 = require("@elizaos/core");
|
|
737
|
+
function convertToolToApi(tool) {
|
|
738
|
+
switch (tool.type) {
|
|
739
|
+
case "web_search_preview":
|
|
740
|
+
return { type: "web_search_preview" };
|
|
741
|
+
case "file_search":
|
|
742
|
+
return {
|
|
743
|
+
type: "file_search",
|
|
744
|
+
vector_store_ids: tool.vectorStoreIds
|
|
745
|
+
};
|
|
746
|
+
case "code_interpreter":
|
|
747
|
+
return {
|
|
748
|
+
type: "code_interpreter",
|
|
749
|
+
container: tool.container ?? { type: "auto" }
|
|
750
|
+
};
|
|
751
|
+
case "mcp":
|
|
752
|
+
return {
|
|
753
|
+
type: "mcp",
|
|
754
|
+
server_label: tool.serverLabel,
|
|
755
|
+
server_url: tool.serverUrl,
|
|
756
|
+
require_approval: tool.requireApproval ?? "never"
|
|
757
|
+
};
|
|
758
|
+
default:
|
|
759
|
+
throw new Error(`Unknown research tool type: ${tool.type}`);
|
|
760
|
+
}
|
|
761
|
+
}
|
|
762
|
+
function convertOutputItem(item) {
|
|
763
|
+
switch (item.type) {
|
|
764
|
+
case "web_search_call":
|
|
765
|
+
return {
|
|
766
|
+
id: item.id ?? "",
|
|
767
|
+
type: "web_search_call",
|
|
768
|
+
status: item.status ?? "completed",
|
|
769
|
+
action: {
|
|
770
|
+
type: item.action?.type ?? "search",
|
|
771
|
+
query: item.action?.query,
|
|
772
|
+
url: item.action?.url
|
|
773
|
+
}
|
|
774
|
+
};
|
|
775
|
+
case "file_search_call":
|
|
776
|
+
return {
|
|
777
|
+
id: item.id ?? "",
|
|
778
|
+
type: "file_search_call",
|
|
779
|
+
status: item.status ?? "completed",
|
|
780
|
+
query: item.query ?? "",
|
|
781
|
+
results: item.results?.map((r) => ({
|
|
782
|
+
fileId: r.file_id,
|
|
783
|
+
fileName: r.file_name,
|
|
784
|
+
score: r.score
|
|
785
|
+
}))
|
|
786
|
+
};
|
|
787
|
+
case "code_interpreter_call":
|
|
788
|
+
return {
|
|
789
|
+
id: item.id ?? "",
|
|
790
|
+
type: "code_interpreter_call",
|
|
791
|
+
status: item.status ?? "completed",
|
|
792
|
+
code: item.code ?? "",
|
|
793
|
+
output: item.output
|
|
794
|
+
};
|
|
795
|
+
case "mcp_tool_call":
|
|
796
|
+
return {
|
|
797
|
+
id: item.id ?? "",
|
|
798
|
+
type: "mcp_tool_call",
|
|
799
|
+
status: item.status ?? "completed",
|
|
800
|
+
serverLabel: item.server_label ?? "",
|
|
801
|
+
toolName: item.tool_name ?? "",
|
|
802
|
+
arguments: item.arguments ?? {},
|
|
803
|
+
result: item.result
|
|
804
|
+
};
|
|
805
|
+
case "message":
|
|
806
|
+
return {
|
|
807
|
+
type: "message",
|
|
808
|
+
content: item.content?.map((c) => ({
|
|
809
|
+
type: "output_text",
|
|
810
|
+
text: c.text,
|
|
811
|
+
annotations: c.annotations?.map((a) => ({
|
|
812
|
+
url: a.url,
|
|
813
|
+
title: a.title,
|
|
814
|
+
startIndex: a.start_index,
|
|
815
|
+
endIndex: a.end_index
|
|
816
|
+
})) ?? []
|
|
817
|
+
})) ?? []
|
|
818
|
+
};
|
|
819
|
+
default:
|
|
820
|
+
return null;
|
|
821
|
+
}
|
|
822
|
+
}
|
|
823
|
+
function extractTextAndAnnotations(response) {
|
|
824
|
+
if (response.output_text) {
|
|
825
|
+
const annotations2 = [];
|
|
826
|
+
if (response.output) {
|
|
827
|
+
for (const item of response.output) {
|
|
828
|
+
if (item.type === "message" && item.content) {
|
|
829
|
+
for (const content of item.content) {
|
|
830
|
+
if (content.annotations) {
|
|
831
|
+
for (const ann of content.annotations) {
|
|
832
|
+
annotations2.push({
|
|
833
|
+
url: ann.url,
|
|
834
|
+
title: ann.title,
|
|
835
|
+
startIndex: ann.start_index,
|
|
836
|
+
endIndex: ann.end_index
|
|
837
|
+
});
|
|
838
|
+
}
|
|
839
|
+
}
|
|
840
|
+
}
|
|
841
|
+
}
|
|
842
|
+
}
|
|
843
|
+
}
|
|
844
|
+
return { text: response.output_text, annotations: annotations2 };
|
|
845
|
+
}
|
|
846
|
+
let text = "";
|
|
847
|
+
const annotations = [];
|
|
848
|
+
if (response.output) {
|
|
849
|
+
for (const item of response.output) {
|
|
850
|
+
if (item.type === "message" && item.content) {
|
|
851
|
+
for (const content of item.content) {
|
|
852
|
+
text += content.text;
|
|
853
|
+
if (content.annotations) {
|
|
854
|
+
for (const ann of content.annotations) {
|
|
855
|
+
annotations.push({
|
|
856
|
+
url: ann.url,
|
|
857
|
+
title: ann.title,
|
|
858
|
+
startIndex: ann.start_index,
|
|
859
|
+
endIndex: ann.end_index
|
|
860
|
+
});
|
|
861
|
+
}
|
|
862
|
+
}
|
|
863
|
+
}
|
|
864
|
+
}
|
|
865
|
+
}
|
|
866
|
+
}
|
|
867
|
+
return { text, annotations };
|
|
868
|
+
}
|
|
869
|
+
async function handleResearch(runtime, params) {
|
|
870
|
+
const apiKey = getApiKey(runtime);
|
|
871
|
+
if (!apiKey) {
|
|
872
|
+
throw new Error("OPENAI_API_KEY is required for deep research. Set it in your environment variables or runtime settings.");
|
|
873
|
+
}
|
|
874
|
+
const baseURL = getBaseURL(runtime);
|
|
875
|
+
const modelName = params.model ?? getResearchModel(runtime);
|
|
876
|
+
const timeout = getResearchTimeout(runtime);
|
|
877
|
+
import_core10.logger.debug(`[OpenAI] Starting deep research with model: ${modelName}`);
|
|
878
|
+
import_core10.logger.debug(`[OpenAI] Research input: ${params.input.substring(0, 100)}...`);
|
|
879
|
+
const dataSourceTools = params.tools?.filter((t) => t.type === "web_search_preview" || t.type === "file_search" || t.type === "mcp");
|
|
880
|
+
if (!dataSourceTools || dataSourceTools.length === 0) {
|
|
881
|
+
import_core10.logger.debug("[OpenAI] No data source tools specified, defaulting to web_search_preview");
|
|
882
|
+
params.tools = [{ type: "web_search_preview" }, ...params.tools ?? []];
|
|
883
|
+
}
|
|
884
|
+
const requestBody = {
|
|
885
|
+
model: modelName,
|
|
886
|
+
input: params.input
|
|
887
|
+
};
|
|
888
|
+
if (params.instructions) {
|
|
889
|
+
requestBody.instructions = params.instructions;
|
|
890
|
+
}
|
|
891
|
+
if (params.background !== undefined) {
|
|
892
|
+
requestBody.background = params.background;
|
|
893
|
+
}
|
|
894
|
+
if (params.tools && params.tools.length > 0) {
|
|
895
|
+
requestBody.tools = params.tools.map(convertToolToApi);
|
|
896
|
+
}
|
|
897
|
+
if (params.maxToolCalls !== undefined) {
|
|
898
|
+
requestBody.max_tool_calls = params.maxToolCalls;
|
|
899
|
+
}
|
|
900
|
+
if (params.reasoningSummary) {
|
|
901
|
+
requestBody.reasoning = { summary: params.reasoningSummary };
|
|
902
|
+
}
|
|
903
|
+
import_core10.logger.debug(`[OpenAI] Research request body: ${JSON.stringify(requestBody, null, 2)}`);
|
|
904
|
+
const response = await fetch(`${baseURL}/responses`, {
|
|
905
|
+
method: "POST",
|
|
906
|
+
headers: {
|
|
907
|
+
Authorization: `Bearer ${apiKey}`,
|
|
908
|
+
"Content-Type": "application/json"
|
|
909
|
+
},
|
|
910
|
+
body: JSON.stringify(requestBody),
|
|
911
|
+
signal: AbortSignal.timeout(timeout)
|
|
912
|
+
});
|
|
913
|
+
if (!response.ok) {
|
|
914
|
+
const errorText = await response.text();
|
|
915
|
+
import_core10.logger.error(`[OpenAI] Research request failed: ${response.status} ${errorText}`);
|
|
916
|
+
throw new Error(`Deep research request failed: ${response.status} ${response.statusText}`);
|
|
917
|
+
}
|
|
918
|
+
const data = await response.json();
|
|
919
|
+
if (data.error) {
|
|
920
|
+
import_core10.logger.error(`[OpenAI] Research API error: ${data.error.message}`);
|
|
921
|
+
throw new Error(`Deep research error: ${data.error.message}`);
|
|
922
|
+
}
|
|
923
|
+
import_core10.logger.debug(`[OpenAI] Research response received. Status: ${data.status ?? "completed"}`);
|
|
924
|
+
const { text, annotations } = extractTextAndAnnotations(data);
|
|
925
|
+
const outputItems = [];
|
|
926
|
+
if (data.output) {
|
|
927
|
+
for (const item of data.output) {
|
|
928
|
+
const converted = convertOutputItem(item);
|
|
929
|
+
if (converted) {
|
|
930
|
+
outputItems.push(converted);
|
|
931
|
+
}
|
|
932
|
+
}
|
|
933
|
+
}
|
|
934
|
+
const result = {
|
|
935
|
+
id: data.id,
|
|
936
|
+
text,
|
|
937
|
+
annotations,
|
|
938
|
+
outputItems,
|
|
939
|
+
status: data.status
|
|
940
|
+
};
|
|
941
|
+
import_core10.logger.info(`[OpenAI] Research completed. Text length: ${text.length}, Annotations: ${annotations.length}, Output items: ${outputItems.length}`);
|
|
942
|
+
return result;
|
|
943
|
+
}
|
|
944
|
+
// models/text.ts
|
|
642
945
|
var import_core11 = require("@elizaos/core");
|
|
946
|
+
var import_ai3 = require("ai");
|
|
947
|
+
function convertUsage(usage) {
|
|
948
|
+
if (!usage) {
|
|
949
|
+
return;
|
|
950
|
+
}
|
|
951
|
+
const promptTokens = usage.inputTokens ?? 0;
|
|
952
|
+
const completionTokens = usage.outputTokens ?? 0;
|
|
953
|
+
return {
|
|
954
|
+
promptTokens,
|
|
955
|
+
completionTokens,
|
|
956
|
+
totalTokens: promptTokens + completionTokens
|
|
957
|
+
};
|
|
958
|
+
}
|
|
959
|
+
async function generateTextByModelType(runtime, params, modelType, getModelFn) {
|
|
960
|
+
const openai = createOpenAIClient(runtime);
|
|
961
|
+
const modelName = getModelFn(runtime);
|
|
962
|
+
import_core11.logger.debug(`[OpenAI] Using ${modelType} model: ${modelName}`);
|
|
963
|
+
const systemPrompt = runtime.character.system ?? undefined;
|
|
964
|
+
const model = openai.chat(modelName);
|
|
965
|
+
const generateParams = {
|
|
966
|
+
model,
|
|
967
|
+
prompt: params.prompt,
|
|
968
|
+
system: systemPrompt,
|
|
969
|
+
maxOutputTokens: params.maxTokens ?? 8192,
|
|
970
|
+
experimental_telemetry: { isEnabled: getExperimentalTelemetry(runtime) }
|
|
971
|
+
};
|
|
972
|
+
if (params.stream) {
|
|
973
|
+
const result = import_ai3.streamText(generateParams);
|
|
974
|
+
return {
|
|
975
|
+
textStream: result.textStream,
|
|
976
|
+
text: Promise.resolve(result.text),
|
|
977
|
+
usage: Promise.resolve(result.usage).then(convertUsage),
|
|
978
|
+
finishReason: Promise.resolve(result.finishReason).then((r) => r)
|
|
979
|
+
};
|
|
980
|
+
}
|
|
981
|
+
const { text, usage } = await import_ai3.generateText(generateParams);
|
|
982
|
+
if (usage) {
|
|
983
|
+
emitModelUsageEvent(runtime, modelType, params.prompt, usage);
|
|
984
|
+
}
|
|
985
|
+
return text;
|
|
986
|
+
}
|
|
987
|
+
async function handleTextSmall(runtime, params) {
|
|
988
|
+
return generateTextByModelType(runtime, params, import_core11.ModelType.TEXT_SMALL, getSmallModel);
|
|
989
|
+
}
|
|
990
|
+
async function handleTextLarge(runtime, params) {
|
|
991
|
+
return generateTextByModelType(runtime, params, import_core11.ModelType.TEXT_LARGE, getLargeModel);
|
|
992
|
+
}
|
|
993
|
+
// models/tokenizer.ts
|
|
994
|
+
var import_core13 = require("@elizaos/core");
|
|
995
|
+
|
|
996
|
+
// utils/tokenization.ts
|
|
997
|
+
var import_core12 = require("@elizaos/core");
|
|
643
998
|
var import_js_tiktoken = require("js-tiktoken");
|
|
644
999
|
function resolveTokenizerEncoding(modelName) {
|
|
645
1000
|
const normalized = modelName.toLowerCase();
|
|
646
1001
|
const fallbackEncoding = normalized.includes("4o") ? "o200k_base" : "cl100k_base";
|
|
647
1002
|
try {
|
|
648
1003
|
return import_js_tiktoken.encodingForModel(modelName);
|
|
649
|
-
} catch
|
|
1004
|
+
} catch {
|
|
650
1005
|
return import_js_tiktoken.getEncoding(fallbackEncoding);
|
|
651
1006
|
}
|
|
652
1007
|
}
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
1008
|
+
function getModelName(runtime, modelType) {
|
|
1009
|
+
if (modelType === import_core12.ModelType.TEXT_SMALL) {
|
|
1010
|
+
return getSmallModel(runtime);
|
|
1011
|
+
}
|
|
1012
|
+
return getLargeModel(runtime);
|
|
657
1013
|
}
|
|
658
|
-
|
|
659
|
-
const modelName =
|
|
660
|
-
|
|
1014
|
+
function tokenizeText(runtime, modelType, text) {
|
|
1015
|
+
const modelName = getModelName(runtime, modelType);
|
|
1016
|
+
const encoder = resolveTokenizerEncoding(modelName);
|
|
1017
|
+
return encoder.encode(text);
|
|
1018
|
+
}
|
|
1019
|
+
function detokenizeText(runtime, modelType, tokens) {
|
|
1020
|
+
const modelName = getModelName(runtime, modelType);
|
|
1021
|
+
const encoder = resolveTokenizerEncoding(modelName);
|
|
1022
|
+
return encoder.decode(tokens);
|
|
661
1023
|
}
|
|
662
1024
|
|
|
663
|
-
//
|
|
664
|
-
async function handleTokenizerEncode(runtime,
|
|
665
|
-
|
|
1025
|
+
// models/tokenizer.ts
|
|
1026
|
+
async function handleTokenizerEncode(runtime, params) {
|
|
1027
|
+
if (!params.prompt) {
|
|
1028
|
+
throw new Error("Tokenization requires a non-empty prompt");
|
|
1029
|
+
}
|
|
1030
|
+
const modelType = params.modelType ?? import_core13.ModelType.TEXT_LARGE;
|
|
1031
|
+
return tokenizeText(runtime, modelType, params.prompt);
|
|
1032
|
+
}
|
|
1033
|
+
async function handleTokenizerDecode(runtime, params) {
|
|
1034
|
+
if (!params.tokens || !Array.isArray(params.tokens)) {
|
|
1035
|
+
throw new Error("Detokenization requires a valid tokens array");
|
|
1036
|
+
}
|
|
1037
|
+
if (params.tokens.length === 0) {
|
|
1038
|
+
return "";
|
|
1039
|
+
}
|
|
1040
|
+
for (let i = 0;i < params.tokens.length; i++) {
|
|
1041
|
+
const token = params.tokens[i];
|
|
1042
|
+
if (typeof token !== "number" || !Number.isFinite(token)) {
|
|
1043
|
+
throw new Error(`Invalid token at index ${i}: expected number`);
|
|
1044
|
+
}
|
|
1045
|
+
}
|
|
1046
|
+
const modelType = params.modelType ?? import_core13.ModelType.TEXT_LARGE;
|
|
1047
|
+
return detokenizeText(runtime, modelType, params.tokens);
|
|
666
1048
|
}
|
|
667
|
-
|
|
668
|
-
|
|
1049
|
+
// index.ts
|
|
1050
|
+
function getProcessEnv() {
|
|
1051
|
+
if (typeof process === "undefined") {
|
|
1052
|
+
return {};
|
|
1053
|
+
}
|
|
1054
|
+
return process.env;
|
|
669
1055
|
}
|
|
670
|
-
|
|
1056
|
+
var env = getProcessEnv();
|
|
671
1057
|
var openaiPlugin = {
|
|
672
1058
|
name: "openai",
|
|
673
|
-
description: "OpenAI
|
|
1059
|
+
description: "OpenAI API integration for text, image, audio, and embedding models",
|
|
674
1060
|
config: {
|
|
675
|
-
OPENAI_API_KEY:
|
|
676
|
-
OPENAI_BASE_URL:
|
|
677
|
-
OPENAI_SMALL_MODEL:
|
|
678
|
-
OPENAI_LARGE_MODEL:
|
|
679
|
-
SMALL_MODEL:
|
|
680
|
-
LARGE_MODEL:
|
|
681
|
-
OPENAI_EMBEDDING_MODEL:
|
|
682
|
-
OPENAI_EMBEDDING_API_KEY:
|
|
683
|
-
OPENAI_EMBEDDING_URL:
|
|
684
|
-
OPENAI_EMBEDDING_DIMENSIONS:
|
|
685
|
-
OPENAI_IMAGE_DESCRIPTION_MODEL:
|
|
686
|
-
OPENAI_IMAGE_DESCRIPTION_MAX_TOKENS:
|
|
687
|
-
OPENAI_EXPERIMENTAL_TELEMETRY:
|
|
1061
|
+
OPENAI_API_KEY: env.OPENAI_API_KEY ?? null,
|
|
1062
|
+
OPENAI_BASE_URL: env.OPENAI_BASE_URL ?? null,
|
|
1063
|
+
OPENAI_SMALL_MODEL: env.OPENAI_SMALL_MODEL ?? null,
|
|
1064
|
+
OPENAI_LARGE_MODEL: env.OPENAI_LARGE_MODEL ?? null,
|
|
1065
|
+
SMALL_MODEL: env.SMALL_MODEL ?? null,
|
|
1066
|
+
LARGE_MODEL: env.LARGE_MODEL ?? null,
|
|
1067
|
+
OPENAI_EMBEDDING_MODEL: env.OPENAI_EMBEDDING_MODEL ?? null,
|
|
1068
|
+
OPENAI_EMBEDDING_API_KEY: env.OPENAI_EMBEDDING_API_KEY ?? null,
|
|
1069
|
+
OPENAI_EMBEDDING_URL: env.OPENAI_EMBEDDING_URL ?? null,
|
|
1070
|
+
OPENAI_EMBEDDING_DIMENSIONS: env.OPENAI_EMBEDDING_DIMENSIONS ?? null,
|
|
1071
|
+
OPENAI_IMAGE_DESCRIPTION_MODEL: env.OPENAI_IMAGE_DESCRIPTION_MODEL ?? null,
|
|
1072
|
+
OPENAI_IMAGE_DESCRIPTION_MAX_TOKENS: env.OPENAI_IMAGE_DESCRIPTION_MAX_TOKENS ?? null,
|
|
1073
|
+
OPENAI_EXPERIMENTAL_TELEMETRY: env.OPENAI_EXPERIMENTAL_TELEMETRY ?? null,
|
|
1074
|
+
OPENAI_RESEARCH_MODEL: env.OPENAI_RESEARCH_MODEL ?? null,
|
|
1075
|
+
OPENAI_RESEARCH_TIMEOUT: env.OPENAI_RESEARCH_TIMEOUT ?? null
|
|
688
1076
|
},
|
|
689
|
-
async init(
|
|
690
|
-
initializeOpenAI(
|
|
1077
|
+
async init(config, runtime) {
|
|
1078
|
+
initializeOpenAI(config, runtime);
|
|
691
1079
|
},
|
|
692
1080
|
models: {
|
|
693
|
-
[
|
|
1081
|
+
[import_core14.ModelType.TEXT_EMBEDDING]: async (runtime, params) => {
|
|
694
1082
|
return handleTextEmbedding(runtime, params);
|
|
695
1083
|
},
|
|
696
|
-
[
|
|
1084
|
+
[import_core14.ModelType.TEXT_TOKENIZER_ENCODE]: async (runtime, params) => {
|
|
697
1085
|
return handleTokenizerEncode(runtime, params);
|
|
698
1086
|
},
|
|
699
|
-
[
|
|
1087
|
+
[import_core14.ModelType.TEXT_TOKENIZER_DECODE]: async (runtime, params) => {
|
|
700
1088
|
return handleTokenizerDecode(runtime, params);
|
|
701
1089
|
},
|
|
702
|
-
[
|
|
1090
|
+
[import_core14.ModelType.TEXT_SMALL]: async (runtime, params) => {
|
|
703
1091
|
return handleTextSmall(runtime, params);
|
|
704
1092
|
},
|
|
705
|
-
[
|
|
1093
|
+
[import_core14.ModelType.TEXT_LARGE]: async (runtime, params) => {
|
|
706
1094
|
return handleTextLarge(runtime, params);
|
|
707
1095
|
},
|
|
708
|
-
[
|
|
1096
|
+
[import_core14.ModelType.IMAGE]: async (runtime, params) => {
|
|
709
1097
|
return handleImageGeneration(runtime, params);
|
|
710
1098
|
},
|
|
711
|
-
[
|
|
1099
|
+
[import_core14.ModelType.IMAGE_DESCRIPTION]: async (runtime, params) => {
|
|
712
1100
|
return handleImageDescription(runtime, params);
|
|
713
1101
|
},
|
|
714
|
-
[
|
|
1102
|
+
[import_core14.ModelType.TRANSCRIPTION]: async (runtime, input) => {
|
|
715
1103
|
return handleTranscription(runtime, input);
|
|
716
1104
|
},
|
|
717
|
-
[
|
|
1105
|
+
[import_core14.ModelType.TEXT_TO_SPEECH]: async (runtime, input) => {
|
|
718
1106
|
return handleTextToSpeech(runtime, input);
|
|
719
1107
|
},
|
|
720
|
-
[
|
|
1108
|
+
[import_core14.ModelType.OBJECT_SMALL]: async (runtime, params) => {
|
|
721
1109
|
return handleObjectSmall(runtime, params);
|
|
722
1110
|
},
|
|
723
|
-
[
|
|
1111
|
+
[import_core14.ModelType.OBJECT_LARGE]: async (runtime, params) => {
|
|
724
1112
|
return handleObjectLarge(runtime, params);
|
|
1113
|
+
},
|
|
1114
|
+
[import_core14.ModelType.RESEARCH]: async (runtime, params) => {
|
|
1115
|
+
return handleResearch(runtime, params);
|
|
725
1116
|
}
|
|
726
1117
|
},
|
|
727
1118
|
tests: [
|
|
@@ -729,217 +1120,166 @@ var openaiPlugin = {
|
|
|
729
1120
|
name: "openai_plugin_tests",
|
|
730
1121
|
tests: [
|
|
731
1122
|
{
|
|
732
|
-
name: "
|
|
1123
|
+
name: "openai_test_api_connectivity",
|
|
733
1124
|
fn: async (runtime) => {
|
|
734
1125
|
const baseURL = getBaseURL(runtime);
|
|
735
1126
|
const response = await fetch(`${baseURL}/models`, {
|
|
736
1127
|
headers: getAuthHeader(runtime)
|
|
737
1128
|
});
|
|
738
|
-
const data = await response.json();
|
|
739
|
-
import_core13.logger.log({ data: data?.data?.length ?? "N/A" }, "Models Available");
|
|
740
1129
|
if (!response.ok) {
|
|
741
|
-
throw new Error(`
|
|
1130
|
+
throw new Error(`API connectivity test failed: ${response.status} ${response.statusText}`);
|
|
742
1131
|
}
|
|
1132
|
+
const data = await response.json();
|
|
1133
|
+
import_core14.logger.info(`[OpenAI Test] API connected. ${data.data?.length ?? 0} models available.`);
|
|
743
1134
|
}
|
|
744
1135
|
},
|
|
745
1136
|
{
|
|
746
1137
|
name: "openai_test_text_embedding",
|
|
747
1138
|
fn: async (runtime) => {
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
} catch (error) {
|
|
754
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
755
|
-
import_core13.logger.error(`Error in test_text_embedding: ${message}`);
|
|
756
|
-
throw error;
|
|
1139
|
+
const embedding = await runtime.useModel(import_core14.ModelType.TEXT_EMBEDDING, {
|
|
1140
|
+
text: "Hello, world!"
|
|
1141
|
+
});
|
|
1142
|
+
if (!Array.isArray(embedding) || embedding.length === 0) {
|
|
1143
|
+
throw new Error("Embedding should return a non-empty array");
|
|
757
1144
|
}
|
|
1145
|
+
import_core14.logger.info(`[OpenAI Test] Generated embedding with ${embedding.length} dimensions`);
|
|
758
1146
|
}
|
|
759
1147
|
},
|
|
760
1148
|
{
|
|
761
|
-
name: "
|
|
1149
|
+
name: "openai_test_text_small",
|
|
762
1150
|
fn: async (runtime) => {
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
throw new Error("Failed to generate text");
|
|
769
|
-
}
|
|
770
|
-
import_core13.logger.log({ text }, "generated with test_text_large");
|
|
771
|
-
} catch (error) {
|
|
772
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
773
|
-
import_core13.logger.error(`Error in test_text_large: ${message}`);
|
|
774
|
-
throw error;
|
|
1151
|
+
const text = await runtime.useModel(import_core14.ModelType.TEXT_SMALL, {
|
|
1152
|
+
prompt: "Say hello in exactly 5 words."
|
|
1153
|
+
});
|
|
1154
|
+
if (typeof text !== "string" || text.length === 0) {
|
|
1155
|
+
throw new Error("TEXT_SMALL should return non-empty string");
|
|
775
1156
|
}
|
|
1157
|
+
import_core14.logger.info(`[OpenAI Test] TEXT_SMALL generated: "${text.substring(0, 50)}..."`);
|
|
776
1158
|
}
|
|
777
1159
|
},
|
|
778
1160
|
{
|
|
779
|
-
name: "
|
|
1161
|
+
name: "openai_test_text_large",
|
|
780
1162
|
fn: async (runtime) => {
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
throw new Error("Failed to generate text");
|
|
787
|
-
}
|
|
788
|
-
import_core13.logger.log({ text }, "generated with test_text_small");
|
|
789
|
-
} catch (error) {
|
|
790
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
791
|
-
import_core13.logger.error(`Error in test_text_small: ${message}`);
|
|
792
|
-
throw error;
|
|
1163
|
+
const text = await runtime.useModel(import_core14.ModelType.TEXT_LARGE, {
|
|
1164
|
+
prompt: "Explain quantum computing in 2 sentences."
|
|
1165
|
+
});
|
|
1166
|
+
if (typeof text !== "string" || text.length === 0) {
|
|
1167
|
+
throw new Error("TEXT_LARGE should return non-empty string");
|
|
793
1168
|
}
|
|
1169
|
+
import_core14.logger.info(`[OpenAI Test] TEXT_LARGE generated: "${text.substring(0, 50)}..."`);
|
|
794
1170
|
}
|
|
795
1171
|
},
|
|
796
1172
|
{
|
|
797
|
-
name: "
|
|
1173
|
+
name: "openai_test_tokenizer_roundtrip",
|
|
798
1174
|
fn: async (runtime) => {
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
import_core13.logger.log({ image }, "generated with test_image_generation");
|
|
807
|
-
} catch (error) {
|
|
808
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
809
|
-
import_core13.logger.error(`Error in test_image_generation: ${message}`);
|
|
810
|
-
throw error;
|
|
1175
|
+
const originalText = "Hello, tokenizer test!";
|
|
1176
|
+
const tokens = await runtime.useModel(import_core14.ModelType.TEXT_TOKENIZER_ENCODE, {
|
|
1177
|
+
prompt: originalText,
|
|
1178
|
+
modelType: import_core14.ModelType.TEXT_SMALL
|
|
1179
|
+
});
|
|
1180
|
+
if (!Array.isArray(tokens) || tokens.length === 0) {
|
|
1181
|
+
throw new Error("Tokenization should return non-empty token array");
|
|
811
1182
|
}
|
|
1183
|
+
const decodedText = await runtime.useModel(import_core14.ModelType.TEXT_TOKENIZER_DECODE, {
|
|
1184
|
+
tokens,
|
|
1185
|
+
modelType: import_core14.ModelType.TEXT_SMALL
|
|
1186
|
+
});
|
|
1187
|
+
if (decodedText !== originalText) {
|
|
1188
|
+
throw new Error(`Tokenizer roundtrip failed: expected "${originalText}", got "${decodedText}"`);
|
|
1189
|
+
}
|
|
1190
|
+
import_core14.logger.info(`[OpenAI Test] Tokenizer roundtrip successful (${tokens.length} tokens)`);
|
|
812
1191
|
}
|
|
813
1192
|
},
|
|
814
1193
|
{
|
|
815
|
-
name: "
|
|
1194
|
+
name: "openai_test_streaming",
|
|
816
1195
|
fn: async (runtime) => {
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
} else {
|
|
824
|
-
import_core13.logger.error("Invalid image description result format:", result);
|
|
825
|
-
}
|
|
826
|
-
} catch (e) {
|
|
827
|
-
const message = e instanceof Error ? e.message : String(e);
|
|
828
|
-
import_core13.logger.error(`Error in image description test: ${message}`);
|
|
1196
|
+
const chunks = [];
|
|
1197
|
+
const result = await runtime.useModel(import_core14.ModelType.TEXT_LARGE, {
|
|
1198
|
+
prompt: "Count from 1 to 5, one number per line.",
|
|
1199
|
+
stream: true,
|
|
1200
|
+
onStreamChunk: (chunk) => {
|
|
1201
|
+
chunks.push(chunk);
|
|
829
1202
|
}
|
|
830
|
-
}
|
|
831
|
-
|
|
832
|
-
|
|
1203
|
+
});
|
|
1204
|
+
if (typeof result !== "string" || result.length === 0) {
|
|
1205
|
+
throw new Error("Streaming should return non-empty result");
|
|
833
1206
|
}
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
{
|
|
837
|
-
name: "openai_test_transcription",
|
|
838
|
-
fn: async (runtime) => {
|
|
839
|
-
import_core13.logger.log("openai_test_transcription");
|
|
840
|
-
try {
|
|
841
|
-
const response = await fetch("https://upload.wikimedia.org/wikipedia/en/4/40/Chris_Benoit_Voice_Message.ogg");
|
|
842
|
-
const arrayBuffer = await response.arrayBuffer();
|
|
843
|
-
const transcription = await runtime.useModel(import_core13.ModelType.TRANSCRIPTION, Buffer.from(new Uint8Array(arrayBuffer)));
|
|
844
|
-
import_core13.logger.log({ transcription }, "generated with test_transcription");
|
|
845
|
-
} catch (error) {
|
|
846
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
847
|
-
import_core13.logger.error(`Error in test_transcription: ${message}`);
|
|
848
|
-
throw error;
|
|
1207
|
+
if (chunks.length === 0) {
|
|
1208
|
+
throw new Error("No streaming chunks received");
|
|
849
1209
|
}
|
|
1210
|
+
import_core14.logger.info(`[OpenAI Test] Streaming test: ${chunks.length} chunks received`);
|
|
850
1211
|
}
|
|
851
1212
|
},
|
|
852
1213
|
{
|
|
853
|
-
name: "
|
|
1214
|
+
name: "openai_test_image_description",
|
|
854
1215
|
fn: async (runtime) => {
|
|
855
|
-
const
|
|
856
|
-
const
|
|
857
|
-
if (!
|
|
858
|
-
throw new Error("
|
|
1216
|
+
const testImageUrl = "https://upload.wikimedia.org/wikipedia/commons/thumb/a/a7/Camponotus_flavomarginatus_ant.jpg/440px-Camponotus_flavomarginatus_ant.jpg";
|
|
1217
|
+
const result = await runtime.useModel(import_core14.ModelType.IMAGE_DESCRIPTION, testImageUrl);
|
|
1218
|
+
if (!result || typeof result !== "object" || !("title" in result) || !("description" in result)) {
|
|
1219
|
+
throw new Error("Image description should return { title, description }");
|
|
859
1220
|
}
|
|
860
|
-
|
|
1221
|
+
import_core14.logger.info(`[OpenAI Test] Image described: "${result.title}"`);
|
|
861
1222
|
}
|
|
862
1223
|
},
|
|
863
1224
|
{
|
|
864
|
-
name: "
|
|
1225
|
+
name: "openai_test_transcription",
|
|
865
1226
|
fn: async (runtime) => {
|
|
866
|
-
const
|
|
867
|
-
const
|
|
868
|
-
const
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
throw new Error(`Decoded text does not match original. Expected "${prompt}", got "${decodedText}"`);
|
|
1227
|
+
const audioUrl = "https://upload.wikimedia.org/wikipedia/commons/2/25/En-Open_Source.ogg";
|
|
1228
|
+
const response = await fetch(audioUrl);
|
|
1229
|
+
const arrayBuffer = await response.arrayBuffer();
|
|
1230
|
+
const audioBuffer = Buffer.from(new Uint8Array(arrayBuffer));
|
|
1231
|
+
const transcription = await runtime.useModel(import_core14.ModelType.TRANSCRIPTION, audioBuffer);
|
|
1232
|
+
if (typeof transcription !== "string") {
|
|
1233
|
+
throw new Error("Transcription should return a string");
|
|
874
1234
|
}
|
|
875
|
-
|
|
1235
|
+
import_core14.logger.info(`[OpenAI Test] Transcription: "${transcription.substring(0, 50)}..."`);
|
|
876
1236
|
}
|
|
877
1237
|
},
|
|
878
1238
|
{
|
|
879
1239
|
name: "openai_test_text_to_speech",
|
|
880
1240
|
fn: async (runtime) => {
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
throw new Error("Failed to generate speech");
|
|
887
|
-
}
|
|
888
|
-
import_core13.logger.log("Generated speech successfully");
|
|
889
|
-
} catch (error) {
|
|
890
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
891
|
-
import_core13.logger.error(`Error in openai_test_text_to_speech: ${message}`);
|
|
892
|
-
throw error;
|
|
1241
|
+
const audioData = await runtime.useModel(import_core14.ModelType.TEXT_TO_SPEECH, {
|
|
1242
|
+
text: "Hello, this is a text-to-speech test."
|
|
1243
|
+
});
|
|
1244
|
+
if (!(audioData instanceof ArrayBuffer) || audioData.byteLength === 0) {
|
|
1245
|
+
throw new Error("TTS should return non-empty ArrayBuffer");
|
|
893
1246
|
}
|
|
1247
|
+
import_core14.logger.info(`[OpenAI Test] TTS generated ${audioData.byteLength} bytes of audio`);
|
|
894
1248
|
}
|
|
895
1249
|
},
|
|
896
1250
|
{
|
|
897
|
-
name: "
|
|
1251
|
+
name: "openai_test_object_generation",
|
|
898
1252
|
fn: async (runtime) => {
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
throw new Error("Text generation returned empty result");
|
|
905
|
-
}
|
|
906
|
-
import_core13.logger.log({ result }, "Text generation test completed");
|
|
907
|
-
} catch (error) {
|
|
908
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
909
|
-
import_core13.logger.error(`Error in openai_test_text_generation_large: ${message}`);
|
|
910
|
-
throw error;
|
|
1253
|
+
const result = await runtime.useModel(import_core14.ModelType.OBJECT_SMALL, {
|
|
1254
|
+
prompt: "Return a JSON object with exactly these fields: name (string), age (number), active (boolean)"
|
|
1255
|
+
});
|
|
1256
|
+
if (!result || typeof result !== "object") {
|
|
1257
|
+
throw new Error("Object generation should return an object");
|
|
911
1258
|
}
|
|
1259
|
+
import_core14.logger.info(`[OpenAI Test] Object generated: ${JSON.stringify(result).substring(0, 100)}`);
|
|
912
1260
|
}
|
|
913
1261
|
},
|
|
914
1262
|
{
|
|
915
|
-
name: "
|
|
1263
|
+
name: "openai_test_research",
|
|
916
1264
|
fn: async (runtime) => {
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
}
|
|
928
|
-
if (chunks.length === 0) {
|
|
929
|
-
throw new Error("No streaming chunks received");
|
|
930
|
-
}
|
|
931
|
-
import_core13.logger.log({ chunks: chunks.length, result: result.substring(0, 50) }, "Streaming test completed");
|
|
932
|
-
} catch (error) {
|
|
933
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
934
|
-
import_core13.logger.error(`Error in openai_test_streaming: ${message}`);
|
|
935
|
-
throw error;
|
|
1265
|
+
const result = await runtime.useModel(import_core14.ModelType.RESEARCH, {
|
|
1266
|
+
input: "What is the current date and time?",
|
|
1267
|
+
tools: [{ type: "web_search_preview" }],
|
|
1268
|
+
maxToolCalls: 3
|
|
1269
|
+
});
|
|
1270
|
+
if (!result || typeof result !== "object" || !("text" in result)) {
|
|
1271
|
+
throw new Error("Research should return an object with text property");
|
|
1272
|
+
}
|
|
1273
|
+
if (typeof result.text !== "string" || result.text.length === 0) {
|
|
1274
|
+
throw new Error("Research result text should be a non-empty string");
|
|
936
1275
|
}
|
|
1276
|
+
import_core14.logger.info(`[OpenAI Test] Research completed. Text length: ${result.text.length}, Annotations: ${result.annotations?.length ?? 0}`);
|
|
937
1277
|
}
|
|
938
1278
|
}
|
|
939
1279
|
]
|
|
940
1280
|
}
|
|
941
1281
|
]
|
|
942
1282
|
};
|
|
943
|
-
var
|
|
1283
|
+
var typescript_default = openaiPlugin;
|
|
944
1284
|
|
|
945
|
-
//# debugId=
|
|
1285
|
+
//# debugId=43ABA7B59A12377C64756E2164756E21
|