@elizaos/plugin-openai 1.5.16 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/index.browser.js +2 -3
- package/dist/browser/index.browser.js.map +17 -7
- package/dist/cjs/index.node.cjs +612 -537
- package/dist/cjs/index.node.js.map +17 -4
- package/dist/index.d.ts +2 -18
- package/dist/init.d.ts +5 -0
- package/dist/models/audio.d.ts +10 -0
- package/dist/models/embedding.d.ts +5 -0
- package/dist/models/image.d.ts +14 -0
- package/dist/models/index.d.ts +6 -0
- package/dist/models/object.d.ts +9 -0
- package/dist/models/text.d.ts +9 -0
- package/dist/models/tokenizer.d.ts +9 -0
- package/dist/node/index.node.js +613 -548
- package/dist/node/index.node.js.map +17 -4
- package/dist/providers/index.d.ts +1 -0
- package/dist/providers/openai.d.ts +8 -0
- package/dist/types/index.d.ts +26 -0
- package/dist/utils/audio.d.ts +12 -0
- package/dist/utils/config.d.ts +70 -0
- package/dist/utils/events.d.ts +10 -0
- package/dist/utils/index.d.ts +5 -0
- package/dist/utils/json.d.ts +7 -0
- package/dist/utils/tokenization.d.ts +17 -0
- package/package.json +3 -2
package/dist/node/index.node.js
CHANGED
|
@@ -1,36 +1,20 @@
|
|
|
1
|
-
import { createRequire } from "node:module";
|
|
2
|
-
var __create = Object.create;
|
|
3
|
-
var __getProtoOf = Object.getPrototypeOf;
|
|
4
|
-
var __defProp = Object.defineProperty;
|
|
5
|
-
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
-
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
7
|
-
var __toESM = (mod, isNodeMode, target) => {
|
|
8
|
-
target = mod != null ? __create(__getProtoOf(mod)) : {};
|
|
9
|
-
const to = isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target;
|
|
10
|
-
for (let key of __getOwnPropNames(mod))
|
|
11
|
-
if (!__hasOwnProp.call(to, key))
|
|
12
|
-
__defProp(to, key, {
|
|
13
|
-
get: () => mod[key],
|
|
14
|
-
enumerable: true
|
|
15
|
-
});
|
|
16
|
-
return to;
|
|
17
|
-
};
|
|
18
|
-
var __require = /* @__PURE__ */ createRequire(import.meta.url);
|
|
19
|
-
|
|
20
1
|
// src/index.ts
|
|
21
|
-
import {
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
} from "
|
|
28
|
-
import { encodingForModel } from "js-tiktoken";
|
|
2
|
+
import { logger as logger10, ModelType as ModelType7 } from "@elizaos/core";
|
|
3
|
+
|
|
4
|
+
// src/init.ts
|
|
5
|
+
import { logger as logger2 } from "@elizaos/core";
|
|
6
|
+
|
|
7
|
+
// src/utils/config.ts
|
|
8
|
+
import { logger } from "@elizaos/core";
|
|
29
9
|
function getSetting(runtime, key, defaultValue) {
|
|
30
|
-
|
|
10
|
+
const value = runtime.getSetting(key);
|
|
11
|
+
if (value !== undefined && value !== null) {
|
|
12
|
+
return String(value);
|
|
13
|
+
}
|
|
14
|
+
return process.env[key] ?? defaultValue;
|
|
31
15
|
}
|
|
32
16
|
function isBrowser() {
|
|
33
|
-
return typeof globalThis !== "undefined" && typeof globalThis.document !== "undefined";
|
|
17
|
+
return typeof globalThis !== "undefined" && "document" in globalThis && typeof globalThis.document !== "undefined";
|
|
34
18
|
}
|
|
35
19
|
function isProxyMode(runtime) {
|
|
36
20
|
return isBrowser() && !!getSetting(runtime, "OPENAI_BROWSER_BASE_URL");
|
|
@@ -69,13 +53,13 @@ function getEmbeddingApiKey(runtime) {
|
|
|
69
53
|
return getApiKey(runtime);
|
|
70
54
|
}
|
|
71
55
|
function getSmallModel(runtime) {
|
|
72
|
-
return getSetting(runtime, "OPENAI_SMALL_MODEL") ?? getSetting(runtime, "SMALL_MODEL", "gpt-
|
|
56
|
+
return getSetting(runtime, "OPENAI_SMALL_MODEL") ?? getSetting(runtime, "SMALL_MODEL", "gpt-4o-mini");
|
|
73
57
|
}
|
|
74
58
|
function getLargeModel(runtime) {
|
|
75
|
-
return getSetting(runtime, "OPENAI_LARGE_MODEL") ?? getSetting(runtime, "LARGE_MODEL", "gpt-
|
|
59
|
+
return getSetting(runtime, "OPENAI_LARGE_MODEL") ?? getSetting(runtime, "LARGE_MODEL", "gpt-4o");
|
|
76
60
|
}
|
|
77
61
|
function getImageDescriptionModel(runtime) {
|
|
78
|
-
return getSetting(runtime, "OPENAI_IMAGE_DESCRIPTION_MODEL", "gpt-5-nano")
|
|
62
|
+
return getSetting(runtime, "OPENAI_IMAGE_DESCRIPTION_MODEL", "gpt-5-nano");
|
|
79
63
|
}
|
|
80
64
|
function getExperimentalTelemetry(runtime) {
|
|
81
65
|
const setting = getSetting(runtime, "OPENAI_EXPERIMENTAL_TELEMETRY", "false");
|
|
@@ -84,150 +68,351 @@ function getExperimentalTelemetry(runtime) {
|
|
|
84
68
|
logger.debug(`[OpenAI] Experimental telemetry in function: "${setting}" (type: ${typeof setting}, normalized: "${normalizedSetting}", result: ${result})`);
|
|
85
69
|
return result;
|
|
86
70
|
}
|
|
71
|
+
|
|
72
|
+
// src/init.ts
|
|
73
|
+
function initializeOpenAI(_config, runtime) {
|
|
74
|
+
(async () => {
|
|
75
|
+
try {
|
|
76
|
+
if (!getApiKey(runtime) && !isBrowser()) {
|
|
77
|
+
logger2.warn("OPENAI_API_KEY is not set in environment - OpenAI functionality will be limited");
|
|
78
|
+
return;
|
|
79
|
+
}
|
|
80
|
+
try {
|
|
81
|
+
const baseURL = getBaseURL(runtime);
|
|
82
|
+
const response = await fetch(`${baseURL}/models`, {
|
|
83
|
+
headers: getAuthHeader(runtime)
|
|
84
|
+
});
|
|
85
|
+
if (!response.ok) {
|
|
86
|
+
logger2.warn(`OpenAI API key validation failed: ${response.statusText}`);
|
|
87
|
+
logger2.warn("OpenAI functionality will be limited until a valid API key is provided");
|
|
88
|
+
} else {
|
|
89
|
+
logger2.log("OpenAI API key validated successfully");
|
|
90
|
+
}
|
|
91
|
+
} catch (fetchError) {
|
|
92
|
+
const message = fetchError instanceof Error ? fetchError.message : String(fetchError);
|
|
93
|
+
logger2.warn(`Error validating OpenAI API key: ${message}`);
|
|
94
|
+
logger2.warn("OpenAI functionality will be limited until a valid API key is provided");
|
|
95
|
+
}
|
|
96
|
+
} catch (error) {
|
|
97
|
+
const message = error?.errors?.map((e) => e.message).join(", ") || (error instanceof Error ? error.message : String(error));
|
|
98
|
+
logger2.warn(`OpenAI plugin configuration issue: ${message} - You need to configure the OPENAI_API_KEY in your environment variables`);
|
|
99
|
+
}
|
|
100
|
+
})();
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// src/models/text.ts
|
|
104
|
+
import { logger as logger3, ModelType } from "@elizaos/core";
|
|
105
|
+
import { generateText, streamText } from "ai";
|
|
106
|
+
|
|
107
|
+
// src/providers/openai.ts
|
|
108
|
+
import { createOpenAI } from "@ai-sdk/openai";
|
|
87
109
|
function createOpenAIClient(runtime) {
|
|
88
110
|
const baseURL = getBaseURL(runtime);
|
|
89
111
|
const apiKey = getApiKey(runtime) ?? (isProxyMode(runtime) ? "sk-proxy" : undefined);
|
|
90
112
|
return createOpenAI({ apiKey: apiKey ?? "", baseURL });
|
|
91
113
|
}
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
const
|
|
99
|
-
|
|
114
|
+
|
|
115
|
+
// src/utils/events.ts
|
|
116
|
+
import { EventType } from "@elizaos/core";
|
|
117
|
+
function emitModelUsageEvent(runtime, type, prompt, usage) {
|
|
118
|
+
const promptTokens = ("promptTokens" in usage ? usage.promptTokens : undefined) ?? ("inputTokens" in usage ? usage.inputTokens : undefined) ?? 0;
|
|
119
|
+
const completionTokens = ("completionTokens" in usage ? usage.completionTokens : undefined) ?? ("outputTokens" in usage ? usage.outputTokens : undefined) ?? 0;
|
|
120
|
+
const totalTokens = ("totalTokens" in usage ? usage.totalTokens : undefined) ?? promptTokens + completionTokens;
|
|
121
|
+
const truncatedPrompt = typeof prompt === "string" ? prompt.length > 200 ? `${prompt.slice(0, 200)}…` : prompt : "";
|
|
122
|
+
runtime.emitEvent(EventType.MODEL_USED, {
|
|
123
|
+
runtime,
|
|
124
|
+
source: "openai",
|
|
125
|
+
provider: "openai",
|
|
126
|
+
type,
|
|
127
|
+
prompt: truncatedPrompt,
|
|
128
|
+
tokens: {
|
|
129
|
+
prompt: promptTokens,
|
|
130
|
+
completion: completionTokens,
|
|
131
|
+
total: totalTokens
|
|
132
|
+
}
|
|
133
|
+
});
|
|
100
134
|
}
|
|
101
|
-
|
|
135
|
+
|
|
136
|
+
// src/models/text.ts
|
|
137
|
+
async function generateTextByModelType(runtime, params, modelType, getModelFn) {
|
|
102
138
|
const openai = createOpenAIClient(runtime);
|
|
103
139
|
const modelName = getModelFn(runtime);
|
|
104
|
-
|
|
105
|
-
const
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
140
|
+
logger3.debug(`[OpenAI] ${modelType} model: ${modelName}`);
|
|
141
|
+
const generateParams = {
|
|
142
|
+
model: openai.languageModel(modelName),
|
|
143
|
+
prompt: params.prompt,
|
|
144
|
+
system: runtime.character.system ?? undefined,
|
|
145
|
+
temperature: params.temperature ?? 0.7,
|
|
146
|
+
maxOutputTokens: params.maxTokens ?? 8192,
|
|
147
|
+
frequencyPenalty: params.frequencyPenalty ?? 0.7,
|
|
148
|
+
presencePenalty: params.presencePenalty ?? 0.7,
|
|
149
|
+
stopSequences: params.stopSequences ?? [],
|
|
150
|
+
experimental_telemetry: { isEnabled: getExperimentalTelemetry(runtime) }
|
|
151
|
+
};
|
|
152
|
+
if (params.stream) {
|
|
153
|
+
const result = streamText(generateParams);
|
|
154
|
+
return {
|
|
155
|
+
textStream: result.textStream,
|
|
156
|
+
text: result.text,
|
|
157
|
+
usage: result.usage.then((u) => u ? {
|
|
158
|
+
promptTokens: u.inputTokens ?? 0,
|
|
159
|
+
completionTokens: u.outputTokens ?? 0,
|
|
160
|
+
totalTokens: (u.inputTokens ?? 0) + (u.outputTokens ?? 0)
|
|
161
|
+
} : undefined),
|
|
162
|
+
finishReason: result.finishReason
|
|
163
|
+
};
|
|
164
|
+
}
|
|
165
|
+
const { text, usage } = await generateText(generateParams);
|
|
166
|
+
if (usage)
|
|
167
|
+
emitModelUsageEvent(runtime, modelType, params.prompt, usage);
|
|
168
|
+
return text;
|
|
169
|
+
}
|
|
170
|
+
async function handleTextSmall(runtime, params) {
|
|
171
|
+
return generateTextByModelType(runtime, params, ModelType.TEXT_SMALL, getSmallModel);
|
|
172
|
+
}
|
|
173
|
+
async function handleTextLarge(runtime, params) {
|
|
174
|
+
return generateTextByModelType(runtime, params, ModelType.TEXT_LARGE, getLargeModel);
|
|
175
|
+
}
|
|
176
|
+
// src/models/embedding.ts
|
|
177
|
+
import { logger as logger4, ModelType as ModelType2, VECTOR_DIMS } from "@elizaos/core";
|
|
178
|
+
async function handleTextEmbedding(runtime, params) {
|
|
179
|
+
const embeddingModelName = getSetting(runtime, "OPENAI_EMBEDDING_MODEL", "text-embedding-3-small");
|
|
180
|
+
const embeddingDimension = Number.parseInt(getSetting(runtime, "OPENAI_EMBEDDING_DIMENSIONS", "1536") || "1536", 10);
|
|
181
|
+
if (!Object.values(VECTOR_DIMS).includes(embeddingDimension)) {
|
|
182
|
+
const errorMsg = `Invalid embedding dimension: ${embeddingDimension}. Must be one of: ${Object.values(VECTOR_DIMS).join(", ")}`;
|
|
183
|
+
logger4.error(errorMsg);
|
|
184
|
+
throw new Error(errorMsg);
|
|
185
|
+
}
|
|
186
|
+
if (params === null) {
|
|
187
|
+
logger4.debug("Creating test embedding for initialization");
|
|
188
|
+
const testVector = Array(embeddingDimension).fill(0);
|
|
189
|
+
testVector[0] = 0.1;
|
|
190
|
+
return testVector;
|
|
191
|
+
}
|
|
192
|
+
let text;
|
|
193
|
+
if (typeof params === "string") {
|
|
194
|
+
text = params;
|
|
195
|
+
} else if (typeof params === "object" && params.text) {
|
|
196
|
+
text = params.text;
|
|
197
|
+
} else {
|
|
198
|
+
const errorMsg = "Invalid input format for embedding";
|
|
199
|
+
logger4.warn(errorMsg);
|
|
200
|
+
const fallbackVector = Array(embeddingDimension).fill(0);
|
|
201
|
+
fallbackVector[0] = 0.2;
|
|
202
|
+
return fallbackVector;
|
|
203
|
+
}
|
|
204
|
+
if (!text.trim()) {
|
|
205
|
+
const errorMsg = "Empty text for embedding";
|
|
206
|
+
logger4.warn(errorMsg);
|
|
207
|
+
const fallbackVector = Array(embeddingDimension).fill(0);
|
|
208
|
+
fallbackVector[0] = 0.3;
|
|
209
|
+
return fallbackVector;
|
|
109
210
|
}
|
|
211
|
+
const embeddingBaseURL = getEmbeddingBaseURL(runtime);
|
|
110
212
|
try {
|
|
111
|
-
const
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
213
|
+
const response = await fetch(`${embeddingBaseURL}/embeddings`, {
|
|
214
|
+
method: "POST",
|
|
215
|
+
headers: {
|
|
216
|
+
...getAuthHeader(runtime, true),
|
|
217
|
+
"Content-Type": "application/json"
|
|
218
|
+
},
|
|
219
|
+
body: JSON.stringify({
|
|
220
|
+
model: embeddingModelName,
|
|
221
|
+
input: text
|
|
222
|
+
})
|
|
117
223
|
});
|
|
118
|
-
if (
|
|
119
|
-
|
|
224
|
+
if (!response.ok) {
|
|
225
|
+
logger4.error(`OpenAI API error: ${response.status} - ${response.statusText}`);
|
|
226
|
+
throw new Error(`OpenAI API error: ${response.status} - ${response.statusText}`);
|
|
120
227
|
}
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
const repairFunction = getJsonRepairFunction();
|
|
126
|
-
const repairedJsonString = await repairFunction({
|
|
127
|
-
text: error.text,
|
|
128
|
-
error
|
|
129
|
-
});
|
|
130
|
-
if (repairedJsonString) {
|
|
131
|
-
try {
|
|
132
|
-
const repairedObject = JSON.parse(repairedJsonString);
|
|
133
|
-
logger.info("[generateObject] Successfully repaired JSON.");
|
|
134
|
-
return repairedObject;
|
|
135
|
-
} catch (repairParseError) {
|
|
136
|
-
const message = repairParseError instanceof Error ? repairParseError.message : String(repairParseError);
|
|
137
|
-
logger.error(`[generateObject] Failed to parse repaired JSON: ${message}`);
|
|
138
|
-
throw repairParseError;
|
|
139
|
-
}
|
|
140
|
-
} else {
|
|
141
|
-
logger.error("[generateObject] JSON repair failed.");
|
|
142
|
-
throw error;
|
|
143
|
-
}
|
|
144
|
-
} else {
|
|
145
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
146
|
-
logger.error(`[generateObject] Unknown error: ${message}`);
|
|
147
|
-
throw error;
|
|
228
|
+
const data = await response.json();
|
|
229
|
+
if (!data?.data?.[0]?.embedding) {
|
|
230
|
+
logger4.error("API returned invalid structure");
|
|
231
|
+
throw new Error("API returned invalid structure");
|
|
148
232
|
}
|
|
233
|
+
const embedding = data.data[0].embedding;
|
|
234
|
+
if (!Array.isArray(embedding) || embedding.length !== embeddingDimension) {
|
|
235
|
+
const errorMsg = `Embedding length ${embedding?.length ?? 0} does not match configured dimension ${embeddingDimension}`;
|
|
236
|
+
logger4.error(errorMsg);
|
|
237
|
+
const fallbackVector = Array(embeddingDimension).fill(0);
|
|
238
|
+
fallbackVector[0] = 0.4;
|
|
239
|
+
return fallbackVector;
|
|
240
|
+
}
|
|
241
|
+
if (data.usage) {
|
|
242
|
+
const usage = {
|
|
243
|
+
inputTokens: data.usage.prompt_tokens,
|
|
244
|
+
outputTokens: 0,
|
|
245
|
+
totalTokens: data.usage.total_tokens
|
|
246
|
+
};
|
|
247
|
+
emitModelUsageEvent(runtime, ModelType2.TEXT_EMBEDDING, text, usage);
|
|
248
|
+
}
|
|
249
|
+
logger4.log(`Got valid embedding with length ${embedding.length}`);
|
|
250
|
+
return embedding;
|
|
251
|
+
} catch (error) {
|
|
252
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
253
|
+
logger4.error(`Error generating embedding: ${message}`);
|
|
254
|
+
throw error instanceof Error ? error : new Error(message);
|
|
149
255
|
}
|
|
150
256
|
}
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
257
|
+
// src/models/image.ts
|
|
258
|
+
import { logger as logger5, ModelType as ModelType3 } from "@elizaos/core";
|
|
259
|
+
async function handleImageGeneration(runtime, params) {
|
|
260
|
+
const n = params.count || 1;
|
|
261
|
+
const size = params.size || "1024x1024";
|
|
262
|
+
const prompt = params.prompt;
|
|
263
|
+
const modelName = getSetting(runtime, "OPENAI_IMAGE_MODEL", "gpt-image-1");
|
|
264
|
+
logger5.log(`[OpenAI] Using IMAGE model: ${modelName}`);
|
|
265
|
+
const baseURL = getBaseURL(runtime);
|
|
266
|
+
try {
|
|
267
|
+
const response = await fetch(`${baseURL}/images/generations`, {
|
|
268
|
+
method: "POST",
|
|
269
|
+
headers: {
|
|
270
|
+
...getAuthHeader(runtime),
|
|
271
|
+
"Content-Type": "application/json"
|
|
272
|
+
},
|
|
273
|
+
body: JSON.stringify({
|
|
274
|
+
model: modelName,
|
|
275
|
+
prompt,
|
|
276
|
+
n,
|
|
277
|
+
size
|
|
278
|
+
})
|
|
279
|
+
});
|
|
280
|
+
if (!response.ok) {
|
|
281
|
+
throw new Error(`Failed to generate image: ${response.statusText}`);
|
|
164
282
|
}
|
|
165
|
-
|
|
283
|
+
const data = await response.json();
|
|
284
|
+
const typedData = data;
|
|
285
|
+
return typedData.data;
|
|
286
|
+
} catch (error) {
|
|
287
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
288
|
+
throw error;
|
|
289
|
+
}
|
|
166
290
|
}
|
|
167
|
-
function
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
291
|
+
async function handleImageDescription(runtime, params) {
|
|
292
|
+
let imageUrl;
|
|
293
|
+
let promptText;
|
|
294
|
+
const modelName = getImageDescriptionModel(runtime);
|
|
295
|
+
logger5.log(`[OpenAI] Using IMAGE_DESCRIPTION model: ${modelName}`);
|
|
296
|
+
const maxTokens = Number.parseInt(getSetting(runtime, "OPENAI_IMAGE_DESCRIPTION_MAX_TOKENS", "8192") || "8192", 10);
|
|
297
|
+
const DEFAULT_PROMPT = "Please analyze this image and provide a title and detailed description.";
|
|
298
|
+
if (typeof params === "string") {
|
|
299
|
+
imageUrl = params;
|
|
300
|
+
promptText = DEFAULT_PROMPT;
|
|
301
|
+
} else {
|
|
302
|
+
imageUrl = params.imageUrl;
|
|
303
|
+
promptText = params.prompt || DEFAULT_PROMPT;
|
|
304
|
+
}
|
|
305
|
+
const messages = [
|
|
306
|
+
{
|
|
307
|
+
role: "user",
|
|
308
|
+
content: [
|
|
309
|
+
{ type: "text", text: promptText },
|
|
310
|
+
{ type: "image_url", image_url: { url: imageUrl } }
|
|
311
|
+
]
|
|
176
312
|
}
|
|
177
|
-
|
|
313
|
+
];
|
|
314
|
+
const baseURL = getBaseURL(runtime);
|
|
315
|
+
try {
|
|
316
|
+
const requestBody = {
|
|
317
|
+
model: modelName,
|
|
318
|
+
messages,
|
|
319
|
+
max_tokens: maxTokens
|
|
320
|
+
};
|
|
321
|
+
const response = await fetch(`${baseURL}/chat/completions`, {
|
|
322
|
+
method: "POST",
|
|
323
|
+
headers: {
|
|
324
|
+
"Content-Type": "application/json",
|
|
325
|
+
...getAuthHeader(runtime)
|
|
326
|
+
},
|
|
327
|
+
body: JSON.stringify(requestBody)
|
|
328
|
+
});
|
|
329
|
+
if (!response.ok) {
|
|
330
|
+
throw new Error(`OpenAI API error: ${response.status}`);
|
|
331
|
+
}
|
|
332
|
+
const result = await response.json();
|
|
333
|
+
const typedResult = result;
|
|
334
|
+
const content = typedResult.choices?.[0]?.message?.content;
|
|
335
|
+
if (typedResult.usage) {
|
|
336
|
+
emitModelUsageEvent(runtime, ModelType3.IMAGE_DESCRIPTION, typeof params === "string" ? params : params.prompt || "", {
|
|
337
|
+
inputTokens: typedResult.usage.prompt_tokens,
|
|
338
|
+
outputTokens: typedResult.usage.completion_tokens,
|
|
339
|
+
totalTokens: typedResult.usage.total_tokens
|
|
340
|
+
});
|
|
341
|
+
}
|
|
342
|
+
if (!content) {
|
|
343
|
+
return {
|
|
344
|
+
title: "Failed to analyze image",
|
|
345
|
+
description: "No response from API"
|
|
346
|
+
};
|
|
347
|
+
}
|
|
348
|
+
const titleMatch = content.match(/title[:\s]+(.+?)(?:\n|$)/i);
|
|
349
|
+
const title = titleMatch?.[1]?.trim();
|
|
350
|
+
if (!title) {
|
|
351
|
+
logger5.warn("Could not extract title from image description response");
|
|
352
|
+
}
|
|
353
|
+
const finalTitle = title || "Image Analysis";
|
|
354
|
+
const description = content.replace(/title[:\s]+(.+?)(?:\n|$)/i, "").trim();
|
|
355
|
+
const processedResult = { title: finalTitle, description };
|
|
356
|
+
return processedResult;
|
|
357
|
+
} catch (error) {
|
|
358
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
359
|
+
logger5.error(`Error analyzing image: ${message}`);
|
|
360
|
+
return {
|
|
361
|
+
title: "Failed to analyze image",
|
|
362
|
+
description: `Error: ${message}`
|
|
363
|
+
};
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
// src/models/audio.ts
|
|
367
|
+
import { logger as logger7 } from "@elizaos/core";
|
|
368
|
+
|
|
369
|
+
// src/utils/audio.ts
|
|
370
|
+
import { logger as logger6 } from "@elizaos/core";
|
|
371
|
+
var MAGIC_BYTES = {
|
|
372
|
+
WAV: {
|
|
373
|
+
HEADER: [82, 73, 70, 70],
|
|
374
|
+
IDENTIFIER: [87, 65, 86, 69]
|
|
375
|
+
},
|
|
376
|
+
MP3_ID3: [73, 68, 51],
|
|
377
|
+
OGG: [79, 103, 103, 83],
|
|
378
|
+
FLAC: [102, 76, 97, 67],
|
|
379
|
+
FTYP: [102, 116, 121, 112],
|
|
380
|
+
WEBM_EBML: [26, 69, 223, 163]
|
|
381
|
+
};
|
|
382
|
+
function matchBytes(buffer, offset, bytes) {
|
|
383
|
+
for (let i = 0;i < bytes.length; i++) {
|
|
384
|
+
if (buffer[offset + i] !== bytes[i])
|
|
385
|
+
return false;
|
|
386
|
+
}
|
|
387
|
+
return true;
|
|
178
388
|
}
|
|
179
389
|
function detectAudioMimeType(buffer) {
|
|
180
390
|
if (buffer.length < 12) {
|
|
181
391
|
return "application/octet-stream";
|
|
182
392
|
}
|
|
183
|
-
if (buffer
|
|
393
|
+
if (matchBytes(buffer, 0, MAGIC_BYTES.WAV.HEADER) && matchBytes(buffer, 8, MAGIC_BYTES.WAV.IDENTIFIER)) {
|
|
184
394
|
return "audio/wav";
|
|
185
395
|
}
|
|
186
|
-
if (buffer
|
|
396
|
+
if (matchBytes(buffer, 0, MAGIC_BYTES.MP3_ID3) || buffer[0] === 255 && (buffer[1] & 224) === 224) {
|
|
187
397
|
return "audio/mpeg";
|
|
188
398
|
}
|
|
189
|
-
if (buffer
|
|
399
|
+
if (matchBytes(buffer, 0, MAGIC_BYTES.OGG)) {
|
|
190
400
|
return "audio/ogg";
|
|
191
401
|
}
|
|
192
|
-
if (buffer
|
|
402
|
+
if (matchBytes(buffer, 0, MAGIC_BYTES.FLAC)) {
|
|
193
403
|
return "audio/flac";
|
|
194
404
|
}
|
|
195
|
-
if (buffer
|
|
405
|
+
if (matchBytes(buffer, 4, MAGIC_BYTES.FTYP)) {
|
|
196
406
|
return "audio/mp4";
|
|
197
407
|
}
|
|
198
|
-
if (buffer
|
|
408
|
+
if (matchBytes(buffer, 0, MAGIC_BYTES.WEBM_EBML)) {
|
|
199
409
|
return "audio/webm";
|
|
200
410
|
}
|
|
201
|
-
|
|
411
|
+
logger6.warn("Could not detect audio format from buffer, using generic binary type");
|
|
202
412
|
return "application/octet-stream";
|
|
203
413
|
}
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
const { Readable } = await import("node:stream");
|
|
207
|
-
const reader = webStream.getReader();
|
|
208
|
-
return new Readable({
|
|
209
|
-
async read() {
|
|
210
|
-
try {
|
|
211
|
-
const { done, value } = await reader.read();
|
|
212
|
-
if (done) {
|
|
213
|
-
this.push(null);
|
|
214
|
-
} else {
|
|
215
|
-
this.push(value);
|
|
216
|
-
}
|
|
217
|
-
} catch (error) {
|
|
218
|
-
this.destroy(error);
|
|
219
|
-
}
|
|
220
|
-
},
|
|
221
|
-
destroy(error, callback) {
|
|
222
|
-
reader.cancel().finally(() => callback(error));
|
|
223
|
-
}
|
|
224
|
-
});
|
|
225
|
-
} catch (error) {
|
|
226
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
227
|
-
logger.error(`Failed to load node:stream module: ${message}`);
|
|
228
|
-
throw new Error(`Cannot convert stream: node:stream module unavailable. This feature requires a Node.js environment.`);
|
|
229
|
-
}
|
|
230
|
-
}
|
|
414
|
+
|
|
415
|
+
// src/models/audio.ts
|
|
231
416
|
async function fetchTextToSpeech(runtime, options) {
|
|
232
417
|
const defaultModel = getSetting(runtime, "OPENAI_TTS_MODEL", "gpt-4o-mini-tts");
|
|
233
418
|
const defaultVoice = getSetting(runtime, "OPENAI_TTS_VOICE", "nova");
|
|
@@ -257,18 +442,196 @@ async function fetchTextToSpeech(runtime, options) {
|
|
|
257
442
|
const err = await res.text();
|
|
258
443
|
throw new Error(`OpenAI TTS error ${res.status}: ${err}`);
|
|
259
444
|
}
|
|
260
|
-
|
|
261
|
-
throw new Error("OpenAI TTS response body is null");
|
|
262
|
-
}
|
|
263
|
-
if (!isBrowser()) {
|
|
264
|
-
return await webStreamToNodeStream(res.body);
|
|
265
|
-
}
|
|
266
|
-
return res.body;
|
|
445
|
+
return await res.arrayBuffer();
|
|
267
446
|
} catch (err) {
|
|
268
447
|
const message = err instanceof Error ? err.message : String(err);
|
|
269
448
|
throw new Error(`Failed to fetch speech from OpenAI TTS: ${message}`);
|
|
270
449
|
}
|
|
271
450
|
}
|
|
451
|
+
async function handleTranscription(runtime, input) {
|
|
452
|
+
let modelName = getSetting(runtime, "OPENAI_TRANSCRIPTION_MODEL", "gpt-4o-mini-transcribe");
|
|
453
|
+
logger7.log(`[OpenAI] Using TRANSCRIPTION model: ${modelName}`);
|
|
454
|
+
const baseURL = getBaseURL(runtime);
|
|
455
|
+
let blob;
|
|
456
|
+
let extraParams = null;
|
|
457
|
+
if (input instanceof Blob || input instanceof File) {
|
|
458
|
+
blob = input;
|
|
459
|
+
} else if (Buffer.isBuffer(input)) {
|
|
460
|
+
const detectedMimeType = detectAudioMimeType(input);
|
|
461
|
+
logger7.debug(`Auto-detected audio MIME type: ${detectedMimeType}`);
|
|
462
|
+
const uint8Array = new Uint8Array(input);
|
|
463
|
+
blob = new Blob([uint8Array], { type: detectedMimeType });
|
|
464
|
+
} else if (typeof input === "object" && input !== null && input.audio != null) {
|
|
465
|
+
const params = input;
|
|
466
|
+
if (!(params.audio instanceof Blob) && !(params.audio instanceof File) && !Buffer.isBuffer(params.audio)) {
|
|
467
|
+
throw new Error("TRANSCRIPTION param 'audio' must be a Blob/File/Buffer.");
|
|
468
|
+
}
|
|
469
|
+
if (Buffer.isBuffer(params.audio)) {
|
|
470
|
+
let mimeType = params.mimeType;
|
|
471
|
+
if (!mimeType) {
|
|
472
|
+
mimeType = detectAudioMimeType(params.audio);
|
|
473
|
+
logger7.debug(`Auto-detected audio MIME type: ${mimeType}`);
|
|
474
|
+
} else {
|
|
475
|
+
logger7.debug(`Using provided MIME type: ${mimeType}`);
|
|
476
|
+
}
|
|
477
|
+
const uint8Array = new Uint8Array(params.audio);
|
|
478
|
+
blob = new Blob([uint8Array], { type: mimeType });
|
|
479
|
+
} else {
|
|
480
|
+
blob = params.audio;
|
|
481
|
+
}
|
|
482
|
+
extraParams = params;
|
|
483
|
+
if (typeof params.model === "string" && params.model) {
|
|
484
|
+
modelName = params.model;
|
|
485
|
+
}
|
|
486
|
+
} else {
|
|
487
|
+
throw new Error("TRANSCRIPTION expects a Blob/File/Buffer or an object { audio: Blob/File/Buffer, mimeType?, language?, response_format?, timestampGranularities?, prompt?, temperature?, model? }");
|
|
488
|
+
}
|
|
489
|
+
const mime = blob.type || "audio/webm";
|
|
490
|
+
const filename = blob.name || (mime.includes("mp3") || mime.includes("mpeg") ? "recording.mp3" : mime.includes("ogg") ? "recording.ogg" : mime.includes("wav") ? "recording.wav" : mime.includes("webm") ? "recording.webm" : "recording.bin");
|
|
491
|
+
const formData = new FormData;
|
|
492
|
+
formData.append("file", blob, filename);
|
|
493
|
+
formData.append("model", String(modelName));
|
|
494
|
+
if (extraParams) {
|
|
495
|
+
if (typeof extraParams.language === "string") {
|
|
496
|
+
formData.append("language", String(extraParams.language));
|
|
497
|
+
}
|
|
498
|
+
if (typeof extraParams.response_format === "string") {
|
|
499
|
+
formData.append("response_format", String(extraParams.response_format));
|
|
500
|
+
}
|
|
501
|
+
if (typeof extraParams.prompt === "string") {
|
|
502
|
+
formData.append("prompt", String(extraParams.prompt));
|
|
503
|
+
}
|
|
504
|
+
if (typeof extraParams.temperature === "number") {
|
|
505
|
+
formData.append("temperature", String(extraParams.temperature));
|
|
506
|
+
}
|
|
507
|
+
if (Array.isArray(extraParams.timestampGranularities)) {
|
|
508
|
+
for (const g of extraParams.timestampGranularities) {
|
|
509
|
+
formData.append("timestamp_granularities[]", String(g));
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
}
|
|
513
|
+
try {
|
|
514
|
+
const response = await fetch(`${baseURL}/audio/transcriptions`, {
|
|
515
|
+
method: "POST",
|
|
516
|
+
headers: {
|
|
517
|
+
...getAuthHeader(runtime)
|
|
518
|
+
},
|
|
519
|
+
body: formData
|
|
520
|
+
});
|
|
521
|
+
if (!response.ok) {
|
|
522
|
+
throw new Error(`Failed to transcribe audio: ${response.status} ${response.statusText}`);
|
|
523
|
+
}
|
|
524
|
+
const data = await response.json();
|
|
525
|
+
return data.text || "";
|
|
526
|
+
} catch (error) {
|
|
527
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
528
|
+
logger7.error(`TRANSCRIPTION error: ${message}`);
|
|
529
|
+
throw error;
|
|
530
|
+
}
|
|
531
|
+
}
|
|
532
|
+
async function handleTextToSpeech(runtime, input) {
|
|
533
|
+
const options = typeof input === "string" ? { text: input } : input;
|
|
534
|
+
const resolvedModel = options.model || getSetting(runtime, "OPENAI_TTS_MODEL", "gpt-4o-mini-tts");
|
|
535
|
+
logger7.log(`[OpenAI] Using TEXT_TO_SPEECH model: ${resolvedModel}`);
|
|
536
|
+
try {
|
|
537
|
+
return await fetchTextToSpeech(runtime, options);
|
|
538
|
+
} catch (error) {
|
|
539
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
540
|
+
logger7.error(`Error in TEXT_TO_SPEECH: ${message}`);
|
|
541
|
+
throw error;
|
|
542
|
+
}
|
|
543
|
+
}
|
|
544
|
+
// src/models/object.ts
|
|
545
|
+
import { logger as logger9, ModelType as ModelType4 } from "@elizaos/core";
|
|
546
|
+
import { generateObject } from "ai";
|
|
547
|
+
|
|
548
|
+
// src/utils/json.ts
|
|
549
|
+
import { logger as logger8 } from "@elizaos/core";
|
|
550
|
+
import { JSONParseError } from "ai";
|
|
551
|
+
function getJsonRepairFunction() {
|
|
552
|
+
return async ({ text, error }) => {
|
|
553
|
+
try {
|
|
554
|
+
if (error instanceof JSONParseError) {
|
|
555
|
+
const cleanedText = text.replace(/```json\n|\n```|```/g, "");
|
|
556
|
+
JSON.parse(cleanedText);
|
|
557
|
+
return cleanedText;
|
|
558
|
+
}
|
|
559
|
+
return null;
|
|
560
|
+
} catch (jsonError) {
|
|
561
|
+
const message = jsonError instanceof Error ? jsonError.message : String(jsonError);
|
|
562
|
+
logger8.warn(`Failed to repair JSON text: ${message}`);
|
|
563
|
+
return null;
|
|
564
|
+
}
|
|
565
|
+
};
|
|
566
|
+
}
|
|
567
|
+
|
|
568
|
+
// src/models/object.ts
|
|
569
|
+
async function generateObjectByModelType(runtime, params, modelType, getModelFn) {
|
|
570
|
+
const openai = createOpenAIClient(runtime);
|
|
571
|
+
const modelName = getModelFn(runtime);
|
|
572
|
+
logger9.log(`[OpenAI] Using ${modelType} model: ${modelName}`);
|
|
573
|
+
const temperature = params.temperature ?? 0;
|
|
574
|
+
const schemaPresent = !!params.schema;
|
|
575
|
+
if (schemaPresent) {
|
|
576
|
+
logger9.warn(`Schema provided but ignored: OpenAI object generation currently uses output=no-schema. The schema parameter has no effect.`);
|
|
577
|
+
}
|
|
578
|
+
try {
|
|
579
|
+
const { object, usage } = await generateObject({
|
|
580
|
+
model: openai.languageModel(modelName),
|
|
581
|
+
output: "no-schema",
|
|
582
|
+
prompt: params.prompt,
|
|
583
|
+
temperature,
|
|
584
|
+
experimental_repairText: getJsonRepairFunction()
|
|
585
|
+
});
|
|
586
|
+
if (usage) {
|
|
587
|
+
emitModelUsageEvent(runtime, modelType, params.prompt, usage);
|
|
588
|
+
}
|
|
589
|
+
return object;
|
|
590
|
+
} catch (error) {
|
|
591
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
592
|
+
logger9.error(`[generateObject] Error: ${message}`);
|
|
593
|
+
throw error;
|
|
594
|
+
}
|
|
595
|
+
}
|
|
596
|
+
async function handleObjectSmall(runtime, params) {
|
|
597
|
+
return generateObjectByModelType(runtime, params, ModelType4.OBJECT_SMALL, getSmallModel);
|
|
598
|
+
}
|
|
599
|
+
async function handleObjectLarge(runtime, params) {
|
|
600
|
+
return generateObjectByModelType(runtime, params, ModelType4.OBJECT_LARGE, getLargeModel);
|
|
601
|
+
}
|
|
602
|
+
// src/models/tokenizer.ts
|
|
603
|
+
import { ModelType as ModelType6 } from "@elizaos/core";
|
|
604
|
+
|
|
605
|
+
// src/utils/tokenization.ts
|
|
606
|
+
import { ModelType as ModelType5 } from "@elizaos/core";
|
|
607
|
+
import { encodingForModel, getEncoding } from "js-tiktoken";
|
|
608
|
+
function resolveTokenizerEncoding(modelName) {
|
|
609
|
+
const normalized = modelName.toLowerCase();
|
|
610
|
+
const fallbackEncoding = normalized.includes("4o") ? "o200k_base" : "cl100k_base";
|
|
611
|
+
try {
|
|
612
|
+
return encodingForModel(modelName);
|
|
613
|
+
} catch (error) {
|
|
614
|
+
return getEncoding(fallbackEncoding);
|
|
615
|
+
}
|
|
616
|
+
}
|
|
617
|
+
async function tokenizeText(runtime, model, prompt) {
|
|
618
|
+
const modelName = model === ModelType5.TEXT_SMALL ? getSmallModel(runtime) : getLargeModel(runtime);
|
|
619
|
+
const tokens = resolveTokenizerEncoding(modelName).encode(prompt);
|
|
620
|
+
return tokens;
|
|
621
|
+
}
|
|
622
|
+
async function detokenizeText(runtime, model, tokens) {
|
|
623
|
+
const modelName = model === ModelType5.TEXT_SMALL ? getSmallModel(runtime) : getLargeModel(runtime);
|
|
624
|
+
return resolveTokenizerEncoding(modelName).decode(tokens);
|
|
625
|
+
}
|
|
626
|
+
|
|
627
|
+
// src/models/tokenizer.ts
|
|
628
|
+
async function handleTokenizerEncode(runtime, { prompt, modelType = ModelType6.TEXT_LARGE }) {
|
|
629
|
+
return await tokenizeText(runtime, modelType, prompt);
|
|
630
|
+
}
|
|
631
|
+
async function handleTokenizerDecode(runtime, { tokens, modelType = ModelType6.TEXT_LARGE }) {
|
|
632
|
+
return await detokenizeText(runtime, modelType, tokens);
|
|
633
|
+
}
|
|
634
|
+
// src/index.ts
|
|
272
635
|
var openaiPlugin = {
|
|
273
636
|
name: "openai",
|
|
274
637
|
description: "OpenAI plugin",
|
|
@@ -288,383 +651,41 @@ var openaiPlugin = {
|
|
|
288
651
|
OPENAI_EXPERIMENTAL_TELEMETRY: process.env.OPENAI_EXPERIMENTAL_TELEMETRY
|
|
289
652
|
},
|
|
290
653
|
async init(_config, runtime) {
|
|
291
|
-
|
|
292
|
-
resolve();
|
|
293
|
-
try {
|
|
294
|
-
if (!getApiKey(runtime) && !isBrowser()) {
|
|
295
|
-
logger.warn("OPENAI_API_KEY is not set in environment - OpenAI functionality will be limited");
|
|
296
|
-
return;
|
|
297
|
-
}
|
|
298
|
-
try {
|
|
299
|
-
const baseURL = getBaseURL(runtime);
|
|
300
|
-
const response = await fetch(`${baseURL}/models`, {
|
|
301
|
-
headers: { ...getAuthHeader(runtime) }
|
|
302
|
-
});
|
|
303
|
-
if (!response.ok) {
|
|
304
|
-
logger.warn(`OpenAI API key validation failed: ${response.statusText}`);
|
|
305
|
-
logger.warn("OpenAI functionality will be limited until a valid API key is provided");
|
|
306
|
-
} else {
|
|
307
|
-
logger.log("OpenAI API key validated successfully");
|
|
308
|
-
}
|
|
309
|
-
} catch (fetchError) {
|
|
310
|
-
const message = fetchError instanceof Error ? fetchError.message : String(fetchError);
|
|
311
|
-
logger.warn(`Error validating OpenAI API key: ${message}`);
|
|
312
|
-
logger.warn("OpenAI functionality will be limited until a valid API key is provided");
|
|
313
|
-
}
|
|
314
|
-
} catch (error) {
|
|
315
|
-
const message = error?.errors?.map((e) => e.message).join(", ") || (error instanceof Error ? error.message : String(error));
|
|
316
|
-
logger.warn(`OpenAI plugin configuration issue: ${message} - You need to configure the OPENAI_API_KEY in your environment variables`);
|
|
317
|
-
}
|
|
318
|
-
});
|
|
654
|
+
initializeOpenAI(_config, runtime);
|
|
319
655
|
},
|
|
320
656
|
models: {
|
|
321
|
-
[
|
|
322
|
-
|
|
323
|
-
const embeddingDimension = Number.parseInt(getSetting(runtime, "OPENAI_EMBEDDING_DIMENSIONS", "1536") || "1536", 10);
|
|
324
|
-
if (!Object.values(VECTOR_DIMS).includes(embeddingDimension)) {
|
|
325
|
-
const errorMsg = `Invalid embedding dimension: ${embeddingDimension}. Must be one of: ${Object.values(VECTOR_DIMS).join(", ")}`;
|
|
326
|
-
logger.error(errorMsg);
|
|
327
|
-
throw new Error(errorMsg);
|
|
328
|
-
}
|
|
329
|
-
if (params === null) {
|
|
330
|
-
logger.debug("Creating test embedding for initialization");
|
|
331
|
-
const testVector = Array(embeddingDimension).fill(0);
|
|
332
|
-
testVector[0] = 0.1;
|
|
333
|
-
return testVector;
|
|
334
|
-
}
|
|
335
|
-
let text;
|
|
336
|
-
if (typeof params === "string") {
|
|
337
|
-
text = params;
|
|
338
|
-
} else if (typeof params === "object" && params.text) {
|
|
339
|
-
text = params.text;
|
|
340
|
-
} else {
|
|
341
|
-
logger.warn("Invalid input format for embedding");
|
|
342
|
-
const fallbackVector = Array(embeddingDimension).fill(0);
|
|
343
|
-
fallbackVector[0] = 0.2;
|
|
344
|
-
return fallbackVector;
|
|
345
|
-
}
|
|
346
|
-
if (!text.trim()) {
|
|
347
|
-
logger.warn("Empty text for embedding");
|
|
348
|
-
const emptyVector = Array(embeddingDimension).fill(0);
|
|
349
|
-
emptyVector[0] = 0.3;
|
|
350
|
-
return emptyVector;
|
|
351
|
-
}
|
|
352
|
-
const embeddingBaseURL = getEmbeddingBaseURL(runtime);
|
|
353
|
-
try {
|
|
354
|
-
const response = await fetch(`${embeddingBaseURL}/embeddings`, {
|
|
355
|
-
method: "POST",
|
|
356
|
-
headers: {
|
|
357
|
-
...getAuthHeader(runtime, true),
|
|
358
|
-
"Content-Type": "application/json"
|
|
359
|
-
},
|
|
360
|
-
body: JSON.stringify({
|
|
361
|
-
model: embeddingModelName,
|
|
362
|
-
input: text
|
|
363
|
-
})
|
|
364
|
-
});
|
|
365
|
-
if (!response.ok) {
|
|
366
|
-
logger.error(`OpenAI API error: ${response.status} - ${response.statusText}`);
|
|
367
|
-
const errorVector = Array(embeddingDimension).fill(0);
|
|
368
|
-
errorVector[0] = 0.4;
|
|
369
|
-
return errorVector;
|
|
370
|
-
}
|
|
371
|
-
const data = await response.json();
|
|
372
|
-
if (!data?.data?.[0]?.embedding) {
|
|
373
|
-
logger.error("API returned invalid structure");
|
|
374
|
-
const errorVector = Array(embeddingDimension).fill(0);
|
|
375
|
-
errorVector[0] = 0.5;
|
|
376
|
-
return errorVector;
|
|
377
|
-
}
|
|
378
|
-
const embedding = data.data[0].embedding;
|
|
379
|
-
if (data.usage) {
|
|
380
|
-
const usage = {
|
|
381
|
-
inputTokens: data.usage.prompt_tokens,
|
|
382
|
-
outputTokens: 0,
|
|
383
|
-
totalTokens: data.usage.total_tokens
|
|
384
|
-
};
|
|
385
|
-
emitModelUsageEvent(runtime, ModelType.TEXT_EMBEDDING, text, usage);
|
|
386
|
-
}
|
|
387
|
-
logger.log(`Got valid embedding with length ${embedding.length}`);
|
|
388
|
-
return embedding;
|
|
389
|
-
} catch (error) {
|
|
390
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
391
|
-
logger.error(`Error generating embedding: ${message}`);
|
|
392
|
-
const errorVector = Array(embeddingDimension).fill(0);
|
|
393
|
-
errorVector[0] = 0.6;
|
|
394
|
-
return errorVector;
|
|
395
|
-
}
|
|
657
|
+
[ModelType7.TEXT_EMBEDDING]: async (runtime, params) => {
|
|
658
|
+
return handleTextEmbedding(runtime, params);
|
|
396
659
|
},
|
|
397
|
-
[
|
|
398
|
-
return
|
|
660
|
+
[ModelType7.TEXT_TOKENIZER_ENCODE]: async (runtime, params) => {
|
|
661
|
+
return handleTokenizerEncode(runtime, params);
|
|
399
662
|
},
|
|
400
|
-
[
|
|
401
|
-
return
|
|
663
|
+
[ModelType7.TEXT_TOKENIZER_DECODE]: async (runtime, params) => {
|
|
664
|
+
return handleTokenizerDecode(runtime, params);
|
|
402
665
|
},
|
|
403
|
-
[
|
|
404
|
-
|
|
405
|
-
stopSequences = [],
|
|
406
|
-
maxTokens = 8192,
|
|
407
|
-
temperature = 0.7,
|
|
408
|
-
frequencyPenalty = 0.7,
|
|
409
|
-
presencePenalty = 0.7
|
|
410
|
-
}) => {
|
|
411
|
-
const openai = createOpenAIClient(runtime);
|
|
412
|
-
const modelName = getSmallModel(runtime);
|
|
413
|
-
const experimentalTelemetry = getExperimentalTelemetry(runtime);
|
|
414
|
-
logger.log(`[OpenAI] Using TEXT_SMALL model: ${modelName}`);
|
|
415
|
-
logger.log(prompt);
|
|
416
|
-
const { text: openaiResponse, usage } = await generateText({
|
|
417
|
-
model: openai.languageModel(modelName),
|
|
418
|
-
prompt,
|
|
419
|
-
system: runtime.character.system ?? undefined,
|
|
420
|
-
temperature,
|
|
421
|
-
maxOutputTokens: maxTokens,
|
|
422
|
-
frequencyPenalty,
|
|
423
|
-
presencePenalty,
|
|
424
|
-
stopSequences,
|
|
425
|
-
experimental_telemetry: {
|
|
426
|
-
isEnabled: experimentalTelemetry
|
|
427
|
-
}
|
|
428
|
-
});
|
|
429
|
-
if (usage) {
|
|
430
|
-
emitModelUsageEvent(runtime, ModelType.TEXT_SMALL, prompt, usage);
|
|
431
|
-
}
|
|
432
|
-
return openaiResponse;
|
|
666
|
+
[ModelType7.TEXT_SMALL]: async (runtime, params) => {
|
|
667
|
+
return handleTextSmall(runtime, params);
|
|
433
668
|
},
|
|
434
|
-
[
|
|
435
|
-
|
|
436
|
-
stopSequences = [],
|
|
437
|
-
maxTokens = 8192,
|
|
438
|
-
temperature = 0.7,
|
|
439
|
-
frequencyPenalty = 0.7,
|
|
440
|
-
presencePenalty = 0.7
|
|
441
|
-
}) => {
|
|
442
|
-
const openai = createOpenAIClient(runtime);
|
|
443
|
-
const modelName = getLargeModel(runtime);
|
|
444
|
-
const experimentalTelemetry = getExperimentalTelemetry(runtime);
|
|
445
|
-
logger.log(`[OpenAI] Using TEXT_LARGE model: ${modelName}`);
|
|
446
|
-
logger.log(prompt);
|
|
447
|
-
const { text: openaiResponse, usage } = await generateText({
|
|
448
|
-
model: openai.languageModel(modelName),
|
|
449
|
-
prompt,
|
|
450
|
-
system: runtime.character.system ?? undefined,
|
|
451
|
-
temperature,
|
|
452
|
-
maxOutputTokens: maxTokens,
|
|
453
|
-
frequencyPenalty,
|
|
454
|
-
presencePenalty,
|
|
455
|
-
stopSequences,
|
|
456
|
-
experimental_telemetry: {
|
|
457
|
-
isEnabled: experimentalTelemetry
|
|
458
|
-
}
|
|
459
|
-
});
|
|
460
|
-
if (usage) {
|
|
461
|
-
emitModelUsageEvent(runtime, ModelType.TEXT_LARGE, prompt, usage);
|
|
462
|
-
}
|
|
463
|
-
return openaiResponse;
|
|
669
|
+
[ModelType7.TEXT_LARGE]: async (runtime, params) => {
|
|
670
|
+
return handleTextLarge(runtime, params);
|
|
464
671
|
},
|
|
465
|
-
[
|
|
466
|
-
|
|
467
|
-
const size = params.size || "1024x1024";
|
|
468
|
-
const prompt = params.prompt;
|
|
469
|
-
const modelName = "gpt-image-1";
|
|
470
|
-
logger.log(`[OpenAI] Using IMAGE model: ${modelName}`);
|
|
471
|
-
const baseURL = getBaseURL(runtime);
|
|
472
|
-
try {
|
|
473
|
-
const response = await fetch(`${baseURL}/images/generations`, {
|
|
474
|
-
method: "POST",
|
|
475
|
-
headers: {
|
|
476
|
-
...getAuthHeader(runtime),
|
|
477
|
-
"Content-Type": "application/json"
|
|
478
|
-
},
|
|
479
|
-
body: JSON.stringify({
|
|
480
|
-
model: modelName,
|
|
481
|
-
prompt,
|
|
482
|
-
n,
|
|
483
|
-
size
|
|
484
|
-
})
|
|
485
|
-
});
|
|
486
|
-
if (!response.ok) {
|
|
487
|
-
throw new Error(`Failed to generate image: ${response.statusText}`);
|
|
488
|
-
}
|
|
489
|
-
const data = await response.json();
|
|
490
|
-
const typedData = data;
|
|
491
|
-
return typedData.data;
|
|
492
|
-
} catch (error) {
|
|
493
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
494
|
-
throw error;
|
|
495
|
-
}
|
|
672
|
+
[ModelType7.IMAGE]: async (runtime, params) => {
|
|
673
|
+
return handleImageGeneration(runtime, params);
|
|
496
674
|
},
|
|
497
|
-
[
|
|
498
|
-
|
|
499
|
-
let promptText;
|
|
500
|
-
const modelName = getImageDescriptionModel(runtime);
|
|
501
|
-
logger.log(`[OpenAI] Using IMAGE_DESCRIPTION model: ${modelName}`);
|
|
502
|
-
const maxTokens = Number.parseInt(getSetting(runtime, "OPENAI_IMAGE_DESCRIPTION_MAX_TOKENS", "8192") || "8192", 10);
|
|
503
|
-
if (typeof params === "string") {
|
|
504
|
-
imageUrl = params;
|
|
505
|
-
promptText = "Please analyze this image and provide a title and detailed description.";
|
|
506
|
-
} else {
|
|
507
|
-
imageUrl = params.imageUrl;
|
|
508
|
-
promptText = params.prompt || "Please analyze this image and provide a title and detailed description.";
|
|
509
|
-
}
|
|
510
|
-
const messages = [
|
|
511
|
-
{
|
|
512
|
-
role: "user",
|
|
513
|
-
content: [
|
|
514
|
-
{ type: "text", text: promptText },
|
|
515
|
-
{ type: "image_url", image_url: { url: imageUrl } }
|
|
516
|
-
]
|
|
517
|
-
}
|
|
518
|
-
];
|
|
519
|
-
const baseURL = getBaseURL(runtime);
|
|
520
|
-
try {
|
|
521
|
-
const requestBody = {
|
|
522
|
-
model: modelName,
|
|
523
|
-
messages,
|
|
524
|
-
max_tokens: maxTokens
|
|
525
|
-
};
|
|
526
|
-
const response = await fetch(`${baseURL}/chat/completions`, {
|
|
527
|
-
method: "POST",
|
|
528
|
-
headers: {
|
|
529
|
-
"Content-Type": "application/json",
|
|
530
|
-
...getAuthHeader(runtime)
|
|
531
|
-
},
|
|
532
|
-
body: JSON.stringify(requestBody)
|
|
533
|
-
});
|
|
534
|
-
if (!response.ok) {
|
|
535
|
-
throw new Error(`OpenAI API error: ${response.status}`);
|
|
536
|
-
}
|
|
537
|
-
const result = await response.json();
|
|
538
|
-
const typedResult = result;
|
|
539
|
-
const content = typedResult.choices?.[0]?.message?.content;
|
|
540
|
-
if (typedResult.usage) {
|
|
541
|
-
emitModelUsageEvent(runtime, ModelType.IMAGE_DESCRIPTION, typeof params === "string" ? params : params.prompt || "", {
|
|
542
|
-
inputTokens: typedResult.usage.prompt_tokens,
|
|
543
|
-
outputTokens: typedResult.usage.completion_tokens,
|
|
544
|
-
totalTokens: typedResult.usage.total_tokens
|
|
545
|
-
});
|
|
546
|
-
}
|
|
547
|
-
if (!content) {
|
|
548
|
-
return {
|
|
549
|
-
title: "Failed to analyze image",
|
|
550
|
-
description: "No response from API"
|
|
551
|
-
};
|
|
552
|
-
}
|
|
553
|
-
const isCustomPrompt = typeof params === "object" && params.prompt && params.prompt !== "Please analyze this image and provide a title and detailed description.";
|
|
554
|
-
if (isCustomPrompt) {
|
|
555
|
-
return content;
|
|
556
|
-
}
|
|
557
|
-
const titleMatch = content.match(/title[:\s]+(.+?)(?:\n|$)/i);
|
|
558
|
-
const title = titleMatch?.[1]?.trim() || "Image Analysis";
|
|
559
|
-
const description = content.replace(/title[:\s]+(.+?)(?:\n|$)/i, "").trim();
|
|
560
|
-
const processedResult = { title, description };
|
|
561
|
-
return processedResult;
|
|
562
|
-
} catch (error) {
|
|
563
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
564
|
-
logger.error(`Error analyzing image: ${message}`);
|
|
565
|
-
return {
|
|
566
|
-
title: "Failed to analyze image",
|
|
567
|
-
description: `Error: ${message}`
|
|
568
|
-
};
|
|
569
|
-
}
|
|
675
|
+
[ModelType7.IMAGE_DESCRIPTION]: async (runtime, params) => {
|
|
676
|
+
return handleImageDescription(runtime, params);
|
|
570
677
|
},
|
|
571
|
-
[
|
|
572
|
-
|
|
573
|
-
logger.log(`[OpenAI] Using TRANSCRIPTION model: ${modelName}`);
|
|
574
|
-
const baseURL = getBaseURL(runtime);
|
|
575
|
-
let blob;
|
|
576
|
-
let extraParams = null;
|
|
577
|
-
if (input instanceof Blob || input instanceof File) {
|
|
578
|
-
blob = input;
|
|
579
|
-
} else if (Buffer.isBuffer(input)) {
|
|
580
|
-
const detectedMimeType = detectAudioMimeType(input);
|
|
581
|
-
logger.debug(`Auto-detected audio MIME type: ${detectedMimeType}`);
|
|
582
|
-
blob = new Blob([input], { type: detectedMimeType });
|
|
583
|
-
} else if (typeof input === "object" && input !== null && input.audio != null) {
|
|
584
|
-
const params = input;
|
|
585
|
-
if (!(params.audio instanceof Blob) && !(params.audio instanceof File) && !Buffer.isBuffer(params.audio)) {
|
|
586
|
-
throw new Error("TRANSCRIPTION param 'audio' must be a Blob/File/Buffer.");
|
|
587
|
-
}
|
|
588
|
-
if (Buffer.isBuffer(params.audio)) {
|
|
589
|
-
let mimeType = params.mimeType;
|
|
590
|
-
if (!mimeType) {
|
|
591
|
-
mimeType = detectAudioMimeType(params.audio);
|
|
592
|
-
logger.debug(`Auto-detected audio MIME type: ${mimeType}`);
|
|
593
|
-
} else {
|
|
594
|
-
logger.debug(`Using provided MIME type: ${mimeType}`);
|
|
595
|
-
}
|
|
596
|
-
blob = new Blob([params.audio], { type: mimeType });
|
|
597
|
-
} else {
|
|
598
|
-
blob = params.audio;
|
|
599
|
-
}
|
|
600
|
-
extraParams = params;
|
|
601
|
-
if (typeof params.model === "string" && params.model) {
|
|
602
|
-
modelName = params.model;
|
|
603
|
-
}
|
|
604
|
-
} else {
|
|
605
|
-
throw new Error("TRANSCRIPTION expects a Blob/File/Buffer or an object { audio: Blob/File/Buffer, mimeType?, language?, response_format?, timestampGranularities?, prompt?, temperature?, model? }");
|
|
606
|
-
}
|
|
607
|
-
const mime = blob.type || "audio/webm";
|
|
608
|
-
const filename = blob.name || (mime.includes("mp3") || mime.includes("mpeg") ? "recording.mp3" : mime.includes("ogg") ? "recording.ogg" : mime.includes("wav") ? "recording.wav" : mime.includes("webm") ? "recording.webm" : "recording.bin");
|
|
609
|
-
const formData = new FormData;
|
|
610
|
-
formData.append("file", blob, filename);
|
|
611
|
-
formData.append("model", String(modelName));
|
|
612
|
-
if (extraParams) {
|
|
613
|
-
if (typeof extraParams.language === "string") {
|
|
614
|
-
formData.append("language", String(extraParams.language));
|
|
615
|
-
}
|
|
616
|
-
if (typeof extraParams.response_format === "string") {
|
|
617
|
-
formData.append("response_format", String(extraParams.response_format));
|
|
618
|
-
}
|
|
619
|
-
if (typeof extraParams.prompt === "string") {
|
|
620
|
-
formData.append("prompt", String(extraParams.prompt));
|
|
621
|
-
}
|
|
622
|
-
if (typeof extraParams.temperature === "number") {
|
|
623
|
-
formData.append("temperature", String(extraParams.temperature));
|
|
624
|
-
}
|
|
625
|
-
if (Array.isArray(extraParams.timestampGranularities)) {
|
|
626
|
-
for (const g of extraParams.timestampGranularities) {
|
|
627
|
-
formData.append("timestamp_granularities[]", String(g));
|
|
628
|
-
}
|
|
629
|
-
}
|
|
630
|
-
}
|
|
631
|
-
try {
|
|
632
|
-
const response = await fetch(`${baseURL}/audio/transcriptions`, {
|
|
633
|
-
method: "POST",
|
|
634
|
-
headers: {
|
|
635
|
-
...getAuthHeader(runtime)
|
|
636
|
-
},
|
|
637
|
-
body: formData
|
|
638
|
-
});
|
|
639
|
-
if (!response.ok) {
|
|
640
|
-
throw new Error(`Failed to transcribe audio: ${response.status} ${response.statusText}`);
|
|
641
|
-
}
|
|
642
|
-
const data = await response.json();
|
|
643
|
-
return data.text || "";
|
|
644
|
-
} catch (error) {
|
|
645
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
646
|
-
logger.error(`TRANSCRIPTION error: ${message}`);
|
|
647
|
-
throw error;
|
|
648
|
-
}
|
|
678
|
+
[ModelType7.TRANSCRIPTION]: async (runtime, input) => {
|
|
679
|
+
return handleTranscription(runtime, input);
|
|
649
680
|
},
|
|
650
|
-
[
|
|
651
|
-
|
|
652
|
-
const resolvedModel = options.model || getSetting(runtime, "OPENAI_TTS_MODEL", "gpt-4o-mini-tts");
|
|
653
|
-
logger.log(`[OpenAI] Using TEXT_TO_SPEECH model: ${resolvedModel}`);
|
|
654
|
-
try {
|
|
655
|
-
const speechStream = await fetchTextToSpeech(runtime, options);
|
|
656
|
-
return speechStream;
|
|
657
|
-
} catch (error) {
|
|
658
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
659
|
-
logger.error(`Error in TEXT_TO_SPEECH: ${message}`);
|
|
660
|
-
throw error;
|
|
661
|
-
}
|
|
681
|
+
[ModelType7.TEXT_TO_SPEECH]: async (runtime, input) => {
|
|
682
|
+
return handleTextToSpeech(runtime, input);
|
|
662
683
|
},
|
|
663
|
-
[
|
|
664
|
-
return
|
|
684
|
+
[ModelType7.OBJECT_SMALL]: async (runtime, params) => {
|
|
685
|
+
return handleObjectSmall(runtime, params);
|
|
665
686
|
},
|
|
666
|
-
[
|
|
667
|
-
return
|
|
687
|
+
[ModelType7.OBJECT_LARGE]: async (runtime, params) => {
|
|
688
|
+
return handleObjectLarge(runtime, params);
|
|
668
689
|
}
|
|
669
690
|
},
|
|
670
691
|
tests: [
|
|
@@ -676,12 +697,10 @@ var openaiPlugin = {
|
|
|
676
697
|
fn: async (runtime) => {
|
|
677
698
|
const baseURL = getBaseURL(runtime);
|
|
678
699
|
const response = await fetch(`${baseURL}/models`, {
|
|
679
|
-
headers:
|
|
680
|
-
Authorization: `Bearer ${getApiKey(runtime)}`
|
|
681
|
-
}
|
|
700
|
+
headers: getAuthHeader(runtime)
|
|
682
701
|
});
|
|
683
702
|
const data = await response.json();
|
|
684
|
-
|
|
703
|
+
logger10.log({ data: data?.data?.length ?? "N/A" }, "Models Available");
|
|
685
704
|
if (!response.ok) {
|
|
686
705
|
throw new Error(`Failed to validate OpenAI API key: ${response.statusText}`);
|
|
687
706
|
}
|
|
@@ -691,13 +710,13 @@ var openaiPlugin = {
|
|
|
691
710
|
name: "openai_test_text_embedding",
|
|
692
711
|
fn: async (runtime) => {
|
|
693
712
|
try {
|
|
694
|
-
const embedding = await runtime.useModel(
|
|
713
|
+
const embedding = await runtime.useModel(ModelType7.TEXT_EMBEDDING, {
|
|
695
714
|
text: "Hello, world!"
|
|
696
715
|
});
|
|
697
|
-
|
|
716
|
+
logger10.log({ embedding }, "embedding");
|
|
698
717
|
} catch (error) {
|
|
699
718
|
const message = error instanceof Error ? error.message : String(error);
|
|
700
|
-
|
|
719
|
+
logger10.error(`Error in test_text_embedding: ${message}`);
|
|
701
720
|
throw error;
|
|
702
721
|
}
|
|
703
722
|
}
|
|
@@ -706,16 +725,16 @@ var openaiPlugin = {
|
|
|
706
725
|
name: "openai_test_text_large",
|
|
707
726
|
fn: async (runtime) => {
|
|
708
727
|
try {
|
|
709
|
-
const text = await runtime.useModel(
|
|
728
|
+
const text = await runtime.useModel(ModelType7.TEXT_LARGE, {
|
|
710
729
|
prompt: "What is the nature of reality in 10 words?"
|
|
711
730
|
});
|
|
712
731
|
if (text.length === 0) {
|
|
713
732
|
throw new Error("Failed to generate text");
|
|
714
733
|
}
|
|
715
|
-
|
|
734
|
+
logger10.log({ text }, "generated with test_text_large");
|
|
716
735
|
} catch (error) {
|
|
717
736
|
const message = error instanceof Error ? error.message : String(error);
|
|
718
|
-
|
|
737
|
+
logger10.error(`Error in test_text_large: ${message}`);
|
|
719
738
|
throw error;
|
|
720
739
|
}
|
|
721
740
|
}
|
|
@@ -724,16 +743,16 @@ var openaiPlugin = {
|
|
|
724
743
|
name: "openai_test_text_small",
|
|
725
744
|
fn: async (runtime) => {
|
|
726
745
|
try {
|
|
727
|
-
const text = await runtime.useModel(
|
|
746
|
+
const text = await runtime.useModel(ModelType7.TEXT_SMALL, {
|
|
728
747
|
prompt: "What is the nature of reality in 10 words?"
|
|
729
748
|
});
|
|
730
749
|
if (text.length === 0) {
|
|
731
750
|
throw new Error("Failed to generate text");
|
|
732
751
|
}
|
|
733
|
-
|
|
752
|
+
logger10.log({ text }, "generated with test_text_small");
|
|
734
753
|
} catch (error) {
|
|
735
754
|
const message = error instanceof Error ? error.message : String(error);
|
|
736
|
-
|
|
755
|
+
logger10.error(`Error in test_text_small: ${message}`);
|
|
737
756
|
throw error;
|
|
738
757
|
}
|
|
739
758
|
}
|
|
@@ -741,17 +760,17 @@ var openaiPlugin = {
|
|
|
741
760
|
{
|
|
742
761
|
name: "openai_test_image_generation",
|
|
743
762
|
fn: async (runtime) => {
|
|
744
|
-
|
|
763
|
+
logger10.log("openai_test_image_generation");
|
|
745
764
|
try {
|
|
746
|
-
const image = await runtime.useModel(
|
|
765
|
+
const image = await runtime.useModel(ModelType7.IMAGE, {
|
|
747
766
|
prompt: "A beautiful sunset over a calm ocean",
|
|
748
|
-
|
|
767
|
+
count: 1,
|
|
749
768
|
size: "1024x1024"
|
|
750
769
|
});
|
|
751
|
-
|
|
770
|
+
logger10.log({ image }, "generated with test_image_generation");
|
|
752
771
|
} catch (error) {
|
|
753
772
|
const message = error instanceof Error ? error.message : String(error);
|
|
754
|
-
|
|
773
|
+
logger10.error(`Error in test_image_generation: ${message}`);
|
|
755
774
|
throw error;
|
|
756
775
|
}
|
|
757
776
|
}
|
|
@@ -760,36 +779,36 @@ var openaiPlugin = {
|
|
|
760
779
|
name: "image-description",
|
|
761
780
|
fn: async (runtime) => {
|
|
762
781
|
try {
|
|
763
|
-
|
|
782
|
+
logger10.log("openai_test_image_description");
|
|
764
783
|
try {
|
|
765
|
-
const result = await runtime.useModel(
|
|
784
|
+
const result = await runtime.useModel(ModelType7.IMAGE_DESCRIPTION, "https://upload.wikimedia.org/wikipedia/commons/thumb/1/1c/Vitalik_Buterin_TechCrunch_London_2015_%28cropped%29.jpg/537px-Vitalik_Buterin_TechCrunch_London_2015_%28cropped%29.jpg");
|
|
766
785
|
if (result && typeof result === "object" && "title" in result && "description" in result) {
|
|
767
|
-
|
|
786
|
+
logger10.log({ result }, "Image description");
|
|
768
787
|
} else {
|
|
769
|
-
|
|
788
|
+
logger10.error("Invalid image description result format:", result);
|
|
770
789
|
}
|
|
771
790
|
} catch (e) {
|
|
772
791
|
const message = e instanceof Error ? e.message : String(e);
|
|
773
|
-
|
|
792
|
+
logger10.error(`Error in image description test: ${message}`);
|
|
774
793
|
}
|
|
775
794
|
} catch (e) {
|
|
776
795
|
const message = e instanceof Error ? e.message : String(e);
|
|
777
|
-
|
|
796
|
+
logger10.error(`Error in openai_test_image_description: ${message}`);
|
|
778
797
|
}
|
|
779
798
|
}
|
|
780
799
|
},
|
|
781
800
|
{
|
|
782
801
|
name: "openai_test_transcription",
|
|
783
802
|
fn: async (runtime) => {
|
|
784
|
-
|
|
803
|
+
logger10.log("openai_test_transcription");
|
|
785
804
|
try {
|
|
786
805
|
const response = await fetch("https://upload.wikimedia.org/wikipedia/en/4/40/Chris_Benoit_Voice_Message.ogg");
|
|
787
806
|
const arrayBuffer = await response.arrayBuffer();
|
|
788
|
-
const transcription = await runtime.useModel(
|
|
789
|
-
|
|
807
|
+
const transcription = await runtime.useModel(ModelType7.TRANSCRIPTION, Buffer.from(new Uint8Array(arrayBuffer)));
|
|
808
|
+
logger10.log({ transcription }, "generated with test_transcription");
|
|
790
809
|
} catch (error) {
|
|
791
810
|
const message = error instanceof Error ? error.message : String(error);
|
|
792
|
-
|
|
811
|
+
logger10.error(`Error in test_transcription: ${message}`);
|
|
793
812
|
throw error;
|
|
794
813
|
}
|
|
795
814
|
}
|
|
@@ -798,39 +817,85 @@ var openaiPlugin = {
|
|
|
798
817
|
name: "openai_test_text_tokenizer_encode",
|
|
799
818
|
fn: async (runtime) => {
|
|
800
819
|
const prompt = "Hello tokenizer encode!";
|
|
801
|
-
const tokens = await runtime.useModel(
|
|
820
|
+
const tokens = await runtime.useModel(ModelType7.TEXT_TOKENIZER_ENCODE, { prompt, modelType: ModelType7.TEXT_SMALL });
|
|
802
821
|
if (!Array.isArray(tokens) || tokens.length === 0) {
|
|
803
822
|
throw new Error("Failed to tokenize text: expected non-empty array of tokens");
|
|
804
823
|
}
|
|
805
|
-
|
|
824
|
+
logger10.log({ tokens }, "Tokenized output");
|
|
806
825
|
}
|
|
807
826
|
},
|
|
808
827
|
{
|
|
809
828
|
name: "openai_test_text_tokenizer_decode",
|
|
810
829
|
fn: async (runtime) => {
|
|
811
830
|
const prompt = "Hello tokenizer decode!";
|
|
812
|
-
const tokens = await runtime.useModel(
|
|
813
|
-
const decodedText = await runtime.useModel(
|
|
831
|
+
const tokens = await runtime.useModel(ModelType7.TEXT_TOKENIZER_ENCODE, { prompt, modelType: ModelType7.TEXT_SMALL });
|
|
832
|
+
const decodedText = await runtime.useModel(ModelType7.TEXT_TOKENIZER_DECODE, {
|
|
833
|
+
tokens,
|
|
834
|
+
modelType: ModelType7.TEXT_SMALL
|
|
835
|
+
});
|
|
814
836
|
if (decodedText !== prompt) {
|
|
815
837
|
throw new Error(`Decoded text does not match original. Expected "${prompt}", got "${decodedText}"`);
|
|
816
838
|
}
|
|
817
|
-
|
|
839
|
+
logger10.log({ decodedText }, "Decoded text");
|
|
818
840
|
}
|
|
819
841
|
},
|
|
820
842
|
{
|
|
821
843
|
name: "openai_test_text_to_speech",
|
|
822
844
|
fn: async (runtime) => {
|
|
823
845
|
try {
|
|
824
|
-
const response = await
|
|
846
|
+
const response = await runtime.useModel(ModelType7.TEXT_TO_SPEECH, {
|
|
825
847
|
text: "Hello, this is a test for text-to-speech."
|
|
826
848
|
});
|
|
827
849
|
if (!response) {
|
|
828
850
|
throw new Error("Failed to generate speech");
|
|
829
851
|
}
|
|
830
|
-
|
|
852
|
+
logger10.log("Generated speech successfully");
|
|
853
|
+
} catch (error) {
|
|
854
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
855
|
+
logger10.error(`Error in openai_test_text_to_speech: ${message}`);
|
|
856
|
+
throw error;
|
|
857
|
+
}
|
|
858
|
+
}
|
|
859
|
+
},
|
|
860
|
+
{
|
|
861
|
+
name: "openai_test_text_generation_large",
|
|
862
|
+
fn: async (runtime) => {
|
|
863
|
+
try {
|
|
864
|
+
const result = await runtime.useModel(ModelType7.TEXT_LARGE, {
|
|
865
|
+
prompt: "Say hello in 5 words."
|
|
866
|
+
});
|
|
867
|
+
if (!result || result.length === 0) {
|
|
868
|
+
throw new Error("Text generation returned empty result");
|
|
869
|
+
}
|
|
870
|
+
logger10.log({ result }, "Text generation test completed");
|
|
871
|
+
} catch (error) {
|
|
872
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
873
|
+
logger10.error(`Error in openai_test_text_generation_large: ${message}`);
|
|
874
|
+
throw error;
|
|
875
|
+
}
|
|
876
|
+
}
|
|
877
|
+
},
|
|
878
|
+
{
|
|
879
|
+
name: "openai_test_streaming",
|
|
880
|
+
fn: async (runtime) => {
|
|
881
|
+
try {
|
|
882
|
+
const chunks = [];
|
|
883
|
+
const result = await runtime.useModel(ModelType7.TEXT_LARGE, {
|
|
884
|
+
prompt: "Count from 1 to 5.",
|
|
885
|
+
onStreamChunk: (chunk) => {
|
|
886
|
+
chunks.push(chunk);
|
|
887
|
+
}
|
|
888
|
+
});
|
|
889
|
+
if (!result || result.length === 0) {
|
|
890
|
+
throw new Error("Streaming returned empty result");
|
|
891
|
+
}
|
|
892
|
+
if (chunks.length === 0) {
|
|
893
|
+
throw new Error("No streaming chunks received");
|
|
894
|
+
}
|
|
895
|
+
logger10.log({ chunks: chunks.length, result: result.substring(0, 50) }, "Streaming test completed");
|
|
831
896
|
} catch (error) {
|
|
832
897
|
const message = error instanceof Error ? error.message : String(error);
|
|
833
|
-
|
|
898
|
+
logger10.error(`Error in openai_test_streaming: ${message}`);
|
|
834
899
|
throw error;
|
|
835
900
|
}
|
|
836
901
|
}
|
|
@@ -845,4 +910,4 @@ export {
|
|
|
845
910
|
src_default as default
|
|
846
911
|
};
|
|
847
912
|
|
|
848
|
-
//# debugId=
|
|
913
|
+
//# debugId=382E411BFB8DEF2564756E2164756E21
|