@elizaos/plugin-local-ai 1.0.0-beta.48 → 1.0.0-beta.49
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +81 -59
- package/dist/index.js +502 -1019
- package/dist/index.js.map +1 -1
- package/package.json +5 -4
package/dist/index.js
CHANGED
|
@@ -3,52 +3,53 @@ import fs5 from "node:fs";
|
|
|
3
3
|
import os3 from "node:os";
|
|
4
4
|
import path5 from "node:path";
|
|
5
5
|
import { Readable as Readable2 } from "node:stream";
|
|
6
|
-
import {
|
|
7
|
-
import { ModelType as ModelType2, logger as logger9 } from "@elizaos/core";
|
|
6
|
+
import { ModelType, logger as logger8 } from "@elizaos/core";
|
|
8
7
|
import {
|
|
9
8
|
LlamaChatSession,
|
|
10
|
-
getLlama
|
|
9
|
+
getLlama
|
|
11
10
|
} from "node-llama-cpp";
|
|
12
11
|
|
|
13
12
|
// src/environment.ts
|
|
14
13
|
import { logger } from "@elizaos/core";
|
|
15
14
|
import { z } from "zod";
|
|
15
|
+
var DEFAULT_SMALL_MODEL = "DeepHermes-3-Llama-3-3B-Preview-q4.gguf";
|
|
16
|
+
var DEFAULT_LARGE_MODEL = "DeepHermes-3-Llama-3-8B-q4.gguf";
|
|
17
|
+
var DEFAULT_EMBEDDING_MODEL = "bge-small-en-v1.5.Q4_K_M.gguf";
|
|
16
18
|
var configSchema = z.object({
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
19
|
+
LOCAL_SMALL_MODEL: z.string().optional().default(DEFAULT_SMALL_MODEL),
|
|
20
|
+
LOCAL_LARGE_MODEL: z.string().optional().default(DEFAULT_LARGE_MODEL),
|
|
21
|
+
LOCAL_EMBEDDING_MODEL: z.string().optional().default(DEFAULT_EMBEDDING_MODEL),
|
|
22
|
+
MODELS_DIR: z.string().optional(),
|
|
23
|
+
// Path for the models directory
|
|
24
|
+
CACHE_DIR: z.string().optional(),
|
|
25
|
+
// Path for the cache directory
|
|
26
|
+
LOCAL_EMBEDDING_DIMENSIONS: z.string().optional().default("384").transform((val) => parseInt(val, 10))
|
|
27
|
+
// Transform to number
|
|
24
28
|
});
|
|
25
|
-
function
|
|
26
|
-
logger.info("Validating model configuration with values:", {
|
|
27
|
-
USE_LOCAL_AI: config.USE_LOCAL_AI,
|
|
28
|
-
USE_STUDIOLM_TEXT_MODELS: config.USE_STUDIOLM_TEXT_MODELS
|
|
29
|
-
});
|
|
30
|
-
if (!config.USE_LOCAL_AI) {
|
|
31
|
-
config.USE_LOCAL_AI = true;
|
|
32
|
-
logger.info("Setting USE_LOCAL_AI to true as it's required");
|
|
33
|
-
}
|
|
34
|
-
logger.info("Configuration is valid");
|
|
35
|
-
}
|
|
36
|
-
async function validateConfig(config) {
|
|
29
|
+
function validateConfig() {
|
|
37
30
|
try {
|
|
38
|
-
const
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
STUDIOLM_EMBEDDING_MODEL: config.STUDIOLM_EMBEDDING_MODEL || "BAAI/bge-small-en-v1.5"
|
|
31
|
+
const configToParse = {
|
|
32
|
+
// Read model filenames from environment variables or use undefined (so zod defaults apply)
|
|
33
|
+
LOCAL_SMALL_MODEL: process.env.LOCAL_SMALL_MODEL,
|
|
34
|
+
LOCAL_LARGE_MODEL: process.env.LOCAL_LARGE_MODEL,
|
|
35
|
+
LOCAL_EMBEDDING_MODEL: process.env.LOCAL_EMBEDDING_MODEL,
|
|
36
|
+
MODELS_DIR: process.env.MODELS_DIR,
|
|
37
|
+
// Read models directory path from env
|
|
38
|
+
CACHE_DIR: process.env.CACHE_DIR,
|
|
39
|
+
// Read cache directory path from env
|
|
40
|
+
LOCAL_EMBEDDING_DIMENSIONS: process.env.LOCAL_EMBEDDING_DIMENSIONS
|
|
41
|
+
// Read embedding dimensions
|
|
50
42
|
};
|
|
51
|
-
|
|
43
|
+
logger.debug("Validating configuration for local AI plugin from env:", {
|
|
44
|
+
LOCAL_SMALL_MODEL: configToParse.LOCAL_SMALL_MODEL,
|
|
45
|
+
LOCAL_LARGE_MODEL: configToParse.LOCAL_LARGE_MODEL,
|
|
46
|
+
LOCAL_EMBEDDING_MODEL: configToParse.LOCAL_EMBEDDING_MODEL,
|
|
47
|
+
MODELS_DIR: configToParse.MODELS_DIR,
|
|
48
|
+
CACHE_DIR: configToParse.CACHE_DIR,
|
|
49
|
+
LOCAL_EMBEDDING_DIMENSIONS: configToParse.LOCAL_EMBEDDING_DIMENSIONS
|
|
50
|
+
});
|
|
51
|
+
const validatedConfig = configSchema.parse(configToParse);
|
|
52
|
+
logger.info("Using local AI configuration:", validatedConfig);
|
|
52
53
|
return validatedConfig;
|
|
53
54
|
} catch (error) {
|
|
54
55
|
if (error instanceof z.ZodError) {
|
|
@@ -138,78 +139,12 @@ var MODEL_SPECS = {
|
|
|
138
139
|
]
|
|
139
140
|
},
|
|
140
141
|
tts: {
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
languages: ["en"],
|
|
148
|
-
features: ["MULTI_SPEAKER", "VOICE_CLONING", "EMOTION_CONTROL", "SPEED_CONTROL"],
|
|
149
|
-
maxInputLength: 4096,
|
|
150
|
-
sampleRate: 24e3,
|
|
151
|
-
contextSize: 2048,
|
|
152
|
-
tokenizer: {
|
|
153
|
-
name: "OuteAI/OuteTTS-0.2-500M",
|
|
154
|
-
type: "llama"
|
|
155
|
-
}
|
|
156
|
-
},
|
|
157
|
-
medium: {
|
|
158
|
-
name: "OuteTTS-0.3-1B.gguf",
|
|
159
|
-
repo: "OuteAI/OuteTTS-0.3-1B-GGUF",
|
|
160
|
-
size: "1B",
|
|
161
|
-
quantization: "Q8_0",
|
|
162
|
-
speakers: ["male_1", "male_2", "male_3", "female_1", "female_2", "female_3"],
|
|
163
|
-
languages: ["en", "es", "fr", "de", "it"],
|
|
164
|
-
features: [
|
|
165
|
-
"MULTI_SPEAKER",
|
|
166
|
-
"VOICE_CLONING",
|
|
167
|
-
"EMOTION_CONTROL",
|
|
168
|
-
"SPEED_CONTROL",
|
|
169
|
-
"MULTILINGUAL",
|
|
170
|
-
"ACCENT_CONTROL"
|
|
171
|
-
],
|
|
172
|
-
maxInputLength: 8192,
|
|
173
|
-
sampleRate: 32e3,
|
|
174
|
-
contextSize: 4096,
|
|
175
|
-
tokenizer: {
|
|
176
|
-
name: "OuteAI/OuteTTS-0.3-1B",
|
|
177
|
-
type: "llama"
|
|
178
|
-
}
|
|
179
|
-
},
|
|
180
|
-
large: {
|
|
181
|
-
name: "OuteTTS-0.3-3B.gguf",
|
|
182
|
-
repo: "OuteAI/OuteTTS-0.3-3B-GGUF",
|
|
183
|
-
size: "3B",
|
|
184
|
-
quantization: "Q8_0",
|
|
185
|
-
speakers: [
|
|
186
|
-
"male_1",
|
|
187
|
-
"male_2",
|
|
188
|
-
"male_3",
|
|
189
|
-
"male_4",
|
|
190
|
-
"female_1",
|
|
191
|
-
"female_2",
|
|
192
|
-
"female_3",
|
|
193
|
-
"female_4"
|
|
194
|
-
],
|
|
195
|
-
languages: ["en", "es", "fr", "de", "it", "pt", "nl", "pl", "ru", "ja", "ko", "zh"],
|
|
196
|
-
features: [
|
|
197
|
-
"MULTI_SPEAKER",
|
|
198
|
-
"VOICE_CLONING",
|
|
199
|
-
"EMOTION_CONTROL",
|
|
200
|
-
"SPEED_CONTROL",
|
|
201
|
-
"MULTILINGUAL",
|
|
202
|
-
"ACCENT_CONTROL",
|
|
203
|
-
"STYLE_TRANSFER",
|
|
204
|
-
"PROSODY_CONTROL"
|
|
205
|
-
],
|
|
206
|
-
maxInputLength: 16384,
|
|
207
|
-
sampleRate: 48e3,
|
|
208
|
-
contextSize: 8192,
|
|
209
|
-
tokenizer: {
|
|
210
|
-
name: "OuteAI/OuteTTS-0.3-3B",
|
|
211
|
-
type: "llama"
|
|
212
|
-
}
|
|
142
|
+
default: {
|
|
143
|
+
modelId: "Xenova/speecht5_tts",
|
|
144
|
+
defaultSampleRate: 16e3,
|
|
145
|
+
// SpeechT5 default
|
|
146
|
+
// Use the standard embedding URL
|
|
147
|
+
defaultSpeakerEmbeddingUrl: "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/speaker_embeddings.bin"
|
|
213
148
|
}
|
|
214
149
|
}
|
|
215
150
|
};
|
|
@@ -918,297 +853,8 @@ var getPlatformManager = () => {
|
|
|
918
853
|
return PlatformManager.getInstance();
|
|
919
854
|
};
|
|
920
855
|
|
|
921
|
-
// src/utils/studiolmManager.ts
|
|
922
|
-
import { ModelType, logger as logger4 } from "@elizaos/core";
|
|
923
|
-
var StudioLMManager = class _StudioLMManager {
|
|
924
|
-
static instance = null;
|
|
925
|
-
serverUrl;
|
|
926
|
-
initialized = false;
|
|
927
|
-
availableModels = [];
|
|
928
|
-
configuredModels = {
|
|
929
|
-
small: process.env.STUDIOLM_SMALL_MODEL || "lmstudio-community/deepseek-r1-distill-qwen-1.5b",
|
|
930
|
-
medium: process.env.STUDIOLM_MEDIUM_MODEL || "deepseek-r1-distill-qwen-7b"
|
|
931
|
-
};
|
|
932
|
-
/**
|
|
933
|
-
* Private constructor for StudioLMManager.
|
|
934
|
-
* Initializes with default serverUrl if not provided in environment variables.
|
|
935
|
-
* Logs initialization information including serverUrl, configuredModels, and timestamp.
|
|
936
|
-
*/
|
|
937
|
-
constructor() {
|
|
938
|
-
this.serverUrl = process.env.STUDIOLM_SERVER_URL || "http://localhost:1234";
|
|
939
|
-
logger4.info("StudioLMManager initialized with configuration:", {
|
|
940
|
-
serverUrl: this.serverUrl,
|
|
941
|
-
configuredModels: this.configuredModels,
|
|
942
|
-
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
943
|
-
});
|
|
944
|
-
}
|
|
945
|
-
/**
|
|
946
|
-
* Returns an instance of StudioLMManager. If an instance already exists, it returns the existing instance.
|
|
947
|
-
* @returns {StudioLMManager} The instance of StudioLMManager
|
|
948
|
-
*/
|
|
949
|
-
static getInstance() {
|
|
950
|
-
if (!_StudioLMManager.instance) {
|
|
951
|
-
_StudioLMManager.instance = new _StudioLMManager();
|
|
952
|
-
}
|
|
953
|
-
return _StudioLMManager.instance;
|
|
954
|
-
}
|
|
955
|
-
/**
|
|
956
|
-
* Check the status of the server by sending a request to the /v1/models endpoint.
|
|
957
|
-
* @returns {Promise<boolean>} A Promise that resolves to true if the server responds with success status, false otherwise.
|
|
958
|
-
*/
|
|
959
|
-
async checkServerStatus() {
|
|
960
|
-
try {
|
|
961
|
-
const response = await fetch(`${this.serverUrl}/v1/models`);
|
|
962
|
-
if (!response.ok) {
|
|
963
|
-
throw new Error(`Server responded with status: ${response.status}`);
|
|
964
|
-
}
|
|
965
|
-
return true;
|
|
966
|
-
} catch (error) {
|
|
967
|
-
logger4.error("LM Studio server check failed:", {
|
|
968
|
-
error: error instanceof Error ? error.message : String(error),
|
|
969
|
-
serverUrl: this.serverUrl,
|
|
970
|
-
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
971
|
-
});
|
|
972
|
-
return false;
|
|
973
|
-
}
|
|
974
|
-
}
|
|
975
|
-
/**
|
|
976
|
-
* Fetches the available models from the server and stores them in the 'availableModels' property.
|
|
977
|
-
*
|
|
978
|
-
* @returns {Promise<void>} A Promise that resolves when the models are fetched successfully or rejects with an error.
|
|
979
|
-
*/
|
|
980
|
-
async fetchAvailableModels() {
|
|
981
|
-
try {
|
|
982
|
-
const response = await fetch(`${this.serverUrl}/v1/models`);
|
|
983
|
-
if (!response.ok) {
|
|
984
|
-
throw new Error(`Failed to fetch models: ${response.status}`);
|
|
985
|
-
}
|
|
986
|
-
const data = await response.json();
|
|
987
|
-
this.availableModels = data.data;
|
|
988
|
-
logger4.info("LM Studio available models:", {
|
|
989
|
-
count: this.availableModels.length,
|
|
990
|
-
models: this.availableModels.map((m) => m.id),
|
|
991
|
-
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
992
|
-
});
|
|
993
|
-
} catch (error) {
|
|
994
|
-
logger4.error("Failed to fetch LM Studio models:", {
|
|
995
|
-
error: error instanceof Error ? error.message : String(error),
|
|
996
|
-
serverUrl: this.serverUrl,
|
|
997
|
-
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
998
|
-
});
|
|
999
|
-
throw error;
|
|
1000
|
-
}
|
|
1001
|
-
}
|
|
1002
|
-
/**
|
|
1003
|
-
* Asynchronously tests the specified model with a chat completion request.
|
|
1004
|
-
* @param {string} modelId - The ID of the model to test.
|
|
1005
|
-
* @returns {Promise<boolean>} - A promise that resolves to true if the model test was successful, false otherwise.
|
|
1006
|
-
*/
|
|
1007
|
-
async testModel(modelId) {
|
|
1008
|
-
try {
|
|
1009
|
-
const testRequest = {
|
|
1010
|
-
model: modelId,
|
|
1011
|
-
messages: [
|
|
1012
|
-
{
|
|
1013
|
-
role: "system",
|
|
1014
|
-
content: "Always answer in rhymes. Today is Thursday"
|
|
1015
|
-
},
|
|
1016
|
-
{ role: "user", content: "What day is it today?" }
|
|
1017
|
-
],
|
|
1018
|
-
temperature: 0.7,
|
|
1019
|
-
max_tokens: -1,
|
|
1020
|
-
stream: false
|
|
1021
|
-
};
|
|
1022
|
-
logger4.info(`Testing model ${modelId}...`);
|
|
1023
|
-
const response = await fetch(`${this.serverUrl}/v1/chat/completions`, {
|
|
1024
|
-
method: "POST",
|
|
1025
|
-
headers: {
|
|
1026
|
-
"Content-Type": "application/json"
|
|
1027
|
-
},
|
|
1028
|
-
body: JSON.stringify(testRequest)
|
|
1029
|
-
});
|
|
1030
|
-
if (!response.ok) {
|
|
1031
|
-
throw new Error(`Model test failed with status: ${response.status}`);
|
|
1032
|
-
}
|
|
1033
|
-
const result = await response.json();
|
|
1034
|
-
if (!result.choices?.[0]?.message?.content) {
|
|
1035
|
-
throw new Error("No valid response content received");
|
|
1036
|
-
}
|
|
1037
|
-
logger4.info(`Model ${modelId} test response:`, {
|
|
1038
|
-
content: result.choices[0].message.content,
|
|
1039
|
-
model: result.model,
|
|
1040
|
-
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
1041
|
-
});
|
|
1042
|
-
return true;
|
|
1043
|
-
} catch (error) {
|
|
1044
|
-
logger4.error(`Model ${modelId} test failed:`, {
|
|
1045
|
-
error: error instanceof Error ? error.message : String(error),
|
|
1046
|
-
stack: error instanceof Error ? error.stack : void 0,
|
|
1047
|
-
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
1048
|
-
});
|
|
1049
|
-
return false;
|
|
1050
|
-
}
|
|
1051
|
-
}
|
|
1052
|
-
/**
|
|
1053
|
-
* Tests the configured text models to ensure they are working properly.
|
|
1054
|
-
* Logs the results of the test and any failed models.
|
|
1055
|
-
* @returns {Promise<void>} A promise that resolves when the test is complete.
|
|
1056
|
-
*/
|
|
1057
|
-
async testTextModels() {
|
|
1058
|
-
logger4.info("Testing configured text models...");
|
|
1059
|
-
const results = await Promise.all([
|
|
1060
|
-
this.testModel(this.configuredModels.small),
|
|
1061
|
-
this.testModel(this.configuredModels.medium)
|
|
1062
|
-
]);
|
|
1063
|
-
const [smallWorking, mediumWorking] = results;
|
|
1064
|
-
if (!smallWorking || !mediumWorking) {
|
|
1065
|
-
const failedModels = [];
|
|
1066
|
-
if (!smallWorking) failedModels.push("small");
|
|
1067
|
-
if (!mediumWorking) failedModels.push("medium");
|
|
1068
|
-
logger4.warn("Some models failed the test:", {
|
|
1069
|
-
failedModels,
|
|
1070
|
-
small: this.configuredModels.small,
|
|
1071
|
-
medium: this.configuredModels.medium
|
|
1072
|
-
});
|
|
1073
|
-
} else {
|
|
1074
|
-
logger4.success("All configured models passed the test");
|
|
1075
|
-
}
|
|
1076
|
-
}
|
|
1077
|
-
/**
|
|
1078
|
-
* Initializes StudioLM by checking server status, fetching available models,
|
|
1079
|
-
* and testing text models.
|
|
1080
|
-
*
|
|
1081
|
-
* @returns {Promise<void>} A Promise that resolves when initialization is complete
|
|
1082
|
-
*/
|
|
1083
|
-
async initialize() {
|
|
1084
|
-
try {
|
|
1085
|
-
if (this.initialized) {
|
|
1086
|
-
logger4.info("StudioLM already initialized, skipping initialization");
|
|
1087
|
-
return;
|
|
1088
|
-
}
|
|
1089
|
-
logger4.info("Starting StudioLM initialization...");
|
|
1090
|
-
const serverAvailable = await this.checkServerStatus();
|
|
1091
|
-
if (!serverAvailable) {
|
|
1092
|
-
throw new Error("LM Studio server is not available");
|
|
1093
|
-
}
|
|
1094
|
-
await this.fetchAvailableModels();
|
|
1095
|
-
await this.testTextModels();
|
|
1096
|
-
this.initialized = true;
|
|
1097
|
-
logger4.success("StudioLM initialization complete");
|
|
1098
|
-
} catch (error) {
|
|
1099
|
-
logger4.error("StudioLM initialization failed:", {
|
|
1100
|
-
error: error instanceof Error ? error.message : String(error),
|
|
1101
|
-
stack: error instanceof Error ? error.stack : void 0
|
|
1102
|
-
});
|
|
1103
|
-
throw error;
|
|
1104
|
-
}
|
|
1105
|
-
}
|
|
1106
|
-
/**
|
|
1107
|
-
* Retrieves the available models in the studio.
|
|
1108
|
-
*
|
|
1109
|
-
* @returns {StudioLMModel[]} An array of StudioLMModel objects representing the available models.
|
|
1110
|
-
*/
|
|
1111
|
-
getAvailableModels() {
|
|
1112
|
-
return this.availableModels;
|
|
1113
|
-
}
|
|
1114
|
-
/**
|
|
1115
|
-
* Check if the object is initialized.
|
|
1116
|
-
*
|
|
1117
|
-
* @returns {boolean} Returns true if the object is initialized, otherwise false.
|
|
1118
|
-
*/
|
|
1119
|
-
isInitialized() {
|
|
1120
|
-
return this.initialized;
|
|
1121
|
-
}
|
|
1122
|
-
/**
|
|
1123
|
-
* Asynchronously generates text using StudioLM based on provided parameters.
|
|
1124
|
-
*
|
|
1125
|
-
* @param {GenerateTextParams} params - The parameters for generating text.
|
|
1126
|
-
* @param {boolean} [isInitialized=false] - Flag to indicate if the model is already initialized.
|
|
1127
|
-
* @returns {Promise<string>} The generated text as a Promise.
|
|
1128
|
-
*/
|
|
1129
|
-
async generateText(params, isInitialized = false) {
|
|
1130
|
-
try {
|
|
1131
|
-
logger4.info("StudioLM generateText entry:", {
|
|
1132
|
-
isInitialized,
|
|
1133
|
-
currentInitState: this.initialized,
|
|
1134
|
-
managerInitState: this.isInitialized(),
|
|
1135
|
-
modelType: params.modelType,
|
|
1136
|
-
contextLength: params.prompt?.length,
|
|
1137
|
-
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
1138
|
-
});
|
|
1139
|
-
if (!this.initialized && !isInitialized) {
|
|
1140
|
-
throw new Error("StudioLM not initialized. Please initialize before generating text.");
|
|
1141
|
-
}
|
|
1142
|
-
const messages = [
|
|
1143
|
-
{
|
|
1144
|
-
role: "system",
|
|
1145
|
-
content: "You are a helpful AI assistant. Respond to the current request only."
|
|
1146
|
-
},
|
|
1147
|
-
{ role: "user", content: params.prompt }
|
|
1148
|
-
];
|
|
1149
|
-
logger4.info("StudioLM preparing request:", {
|
|
1150
|
-
model: params.modelType === ModelType.TEXT_LARGE ? this.configuredModels.medium : this.configuredModels.small,
|
|
1151
|
-
messageCount: messages.length,
|
|
1152
|
-
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
1153
|
-
});
|
|
1154
|
-
logger4.info("Incoming context structure:", {
|
|
1155
|
-
contextLength: params.prompt.length,
|
|
1156
|
-
hasAction: params.prompt.includes("action"),
|
|
1157
|
-
runtime: !!params.runtime,
|
|
1158
|
-
stopSequences: params.stopSequences
|
|
1159
|
-
});
|
|
1160
|
-
const request = {
|
|
1161
|
-
model: params.modelType === ModelType.TEXT_LARGE ? this.configuredModels.medium : this.configuredModels.small,
|
|
1162
|
-
messages,
|
|
1163
|
-
temperature: 0.7,
|
|
1164
|
-
max_tokens: 8192,
|
|
1165
|
-
stream: false
|
|
1166
|
-
};
|
|
1167
|
-
const response = await fetch(`${this.serverUrl}/v1/chat/completions`, {
|
|
1168
|
-
method: "POST",
|
|
1169
|
-
headers: {
|
|
1170
|
-
"Content-Type": "application/json"
|
|
1171
|
-
},
|
|
1172
|
-
body: JSON.stringify(request)
|
|
1173
|
-
});
|
|
1174
|
-
if (!response.ok) {
|
|
1175
|
-
throw new Error(`StudioLM request failed: ${response.status}`);
|
|
1176
|
-
}
|
|
1177
|
-
const result = await response.json();
|
|
1178
|
-
if (!result.choices?.[0]?.message?.content) {
|
|
1179
|
-
throw new Error("No valid response content received from StudioLM");
|
|
1180
|
-
}
|
|
1181
|
-
let responseText = result.choices[0].message.content;
|
|
1182
|
-
logger4.info("Raw response structure:", {
|
|
1183
|
-
responseLength: responseText.length,
|
|
1184
|
-
hasAction: responseText.includes("action"),
|
|
1185
|
-
hasThinkTag: responseText.includes("<think>")
|
|
1186
|
-
});
|
|
1187
|
-
if (responseText.includes("<think>")) {
|
|
1188
|
-
logger4.info("Cleaning think tags from response");
|
|
1189
|
-
responseText = responseText.replace(/<think>[\s\S]*?<\/think>\n?/g, "");
|
|
1190
|
-
logger4.info("Think tags removed from response");
|
|
1191
|
-
}
|
|
1192
|
-
logger4.info("StudioLM request completed successfully:", {
|
|
1193
|
-
responseLength: responseText.length,
|
|
1194
|
-
hasThinkTags: responseText.includes("<think>"),
|
|
1195
|
-
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
1196
|
-
});
|
|
1197
|
-
return responseText;
|
|
1198
|
-
} catch (error) {
|
|
1199
|
-
logger4.error("StudioLM text generation error:", {
|
|
1200
|
-
error: error instanceof Error ? error.message : String(error),
|
|
1201
|
-
stack: error instanceof Error ? error.stack : void 0,
|
|
1202
|
-
phase: "text generation",
|
|
1203
|
-
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
1204
|
-
});
|
|
1205
|
-
throw error;
|
|
1206
|
-
}
|
|
1207
|
-
}
|
|
1208
|
-
};
|
|
1209
|
-
|
|
1210
856
|
// src/utils/tokenizerManager.ts
|
|
1211
|
-
import { logger as
|
|
857
|
+
import { logger as logger4 } from "@elizaos/core";
|
|
1212
858
|
import { AutoTokenizer } from "@huggingface/transformers";
|
|
1213
859
|
var TokenizerManager = class _TokenizerManager {
|
|
1214
860
|
static instance = null;
|
|
@@ -1248,7 +894,7 @@ var TokenizerManager = class _TokenizerManager {
|
|
|
1248
894
|
async loadTokenizer(modelConfig) {
|
|
1249
895
|
try {
|
|
1250
896
|
const tokenizerKey = `${modelConfig.tokenizer.type}-${modelConfig.tokenizer.name}`;
|
|
1251
|
-
|
|
897
|
+
logger4.info("Loading tokenizer:", {
|
|
1252
898
|
key: tokenizerKey,
|
|
1253
899
|
name: modelConfig.tokenizer.name,
|
|
1254
900
|
type: modelConfig.tokenizer.type,
|
|
@@ -1256,7 +902,7 @@ var TokenizerManager = class _TokenizerManager {
|
|
|
1256
902
|
cacheDir: this.cacheDir
|
|
1257
903
|
});
|
|
1258
904
|
if (this.tokenizers.has(tokenizerKey)) {
|
|
1259
|
-
|
|
905
|
+
logger4.info("Using cached tokenizer:", { key: tokenizerKey });
|
|
1260
906
|
const cachedTokenizer = this.tokenizers.get(tokenizerKey);
|
|
1261
907
|
if (!cachedTokenizer) {
|
|
1262
908
|
throw new Error(`Tokenizer ${tokenizerKey} exists in map but returned undefined`);
|
|
@@ -1265,10 +911,10 @@ var TokenizerManager = class _TokenizerManager {
|
|
|
1265
911
|
}
|
|
1266
912
|
const fs6 = await import("node:fs");
|
|
1267
913
|
if (!fs6.existsSync(this.modelsDir)) {
|
|
1268
|
-
|
|
914
|
+
logger4.warn("Models directory does not exist, creating it:", this.modelsDir);
|
|
1269
915
|
fs6.mkdirSync(this.modelsDir, { recursive: true });
|
|
1270
916
|
}
|
|
1271
|
-
|
|
917
|
+
logger4.info(
|
|
1272
918
|
"Initializing new tokenizer from HuggingFace with models directory:",
|
|
1273
919
|
this.modelsDir
|
|
1274
920
|
);
|
|
@@ -1278,28 +924,28 @@ var TokenizerManager = class _TokenizerManager {
|
|
|
1278
924
|
local_files_only: false
|
|
1279
925
|
});
|
|
1280
926
|
this.tokenizers.set(tokenizerKey, tokenizer);
|
|
1281
|
-
|
|
927
|
+
logger4.success("Tokenizer loaded successfully:", { key: tokenizerKey });
|
|
1282
928
|
return tokenizer;
|
|
1283
929
|
} catch (tokenizeError) {
|
|
1284
|
-
|
|
930
|
+
logger4.error("Failed to load tokenizer from HuggingFace:", {
|
|
1285
931
|
error: tokenizeError instanceof Error ? tokenizeError.message : String(tokenizeError),
|
|
1286
932
|
stack: tokenizeError instanceof Error ? tokenizeError.stack : void 0,
|
|
1287
933
|
tokenizer: modelConfig.tokenizer.name,
|
|
1288
934
|
modelsDir: this.modelsDir
|
|
1289
935
|
});
|
|
1290
|
-
|
|
936
|
+
logger4.info("Retrying tokenizer loading...");
|
|
1291
937
|
const tokenizer = await AutoTokenizer.from_pretrained(modelConfig.tokenizer.name, {
|
|
1292
938
|
cache_dir: this.modelsDir,
|
|
1293
939
|
local_files_only: false
|
|
1294
940
|
});
|
|
1295
941
|
this.tokenizers.set(tokenizerKey, tokenizer);
|
|
1296
|
-
|
|
942
|
+
logger4.success("Tokenizer loaded successfully on retry:", {
|
|
1297
943
|
key: tokenizerKey
|
|
1298
944
|
});
|
|
1299
945
|
return tokenizer;
|
|
1300
946
|
}
|
|
1301
947
|
} catch (error) {
|
|
1302
|
-
|
|
948
|
+
logger4.error("Failed to load tokenizer:", {
|
|
1303
949
|
error: error instanceof Error ? error.message : String(error),
|
|
1304
950
|
stack: error instanceof Error ? error.stack : void 0,
|
|
1305
951
|
model: modelConfig.name,
|
|
@@ -1319,23 +965,23 @@ var TokenizerManager = class _TokenizerManager {
|
|
|
1319
965
|
*/
|
|
1320
966
|
async encode(text, modelConfig) {
|
|
1321
967
|
try {
|
|
1322
|
-
|
|
968
|
+
logger4.info("Encoding text with tokenizer:", {
|
|
1323
969
|
length: text.length,
|
|
1324
970
|
tokenizer: modelConfig.tokenizer.name
|
|
1325
971
|
});
|
|
1326
972
|
const tokenizer = await this.loadTokenizer(modelConfig);
|
|
1327
|
-
|
|
973
|
+
logger4.info("Tokenizer loaded, encoding text...");
|
|
1328
974
|
const encoded = await tokenizer.encode(text, {
|
|
1329
975
|
add_special_tokens: true,
|
|
1330
976
|
return_token_type_ids: false
|
|
1331
977
|
});
|
|
1332
|
-
|
|
978
|
+
logger4.info("Text encoded successfully:", {
|
|
1333
979
|
tokenCount: encoded.length,
|
|
1334
980
|
tokenizer: modelConfig.tokenizer.name
|
|
1335
981
|
});
|
|
1336
982
|
return encoded;
|
|
1337
983
|
} catch (error) {
|
|
1338
|
-
|
|
984
|
+
logger4.error("Text encoding failed:", {
|
|
1339
985
|
error: error instanceof Error ? error.message : String(error),
|
|
1340
986
|
stack: error instanceof Error ? error.stack : void 0,
|
|
1341
987
|
textLength: text.length,
|
|
@@ -1355,23 +1001,23 @@ var TokenizerManager = class _TokenizerManager {
|
|
|
1355
1001
|
*/
|
|
1356
1002
|
async decode(tokens, modelConfig) {
|
|
1357
1003
|
try {
|
|
1358
|
-
|
|
1004
|
+
logger4.info("Decoding tokens with tokenizer:", {
|
|
1359
1005
|
count: tokens.length,
|
|
1360
1006
|
tokenizer: modelConfig.tokenizer.name
|
|
1361
1007
|
});
|
|
1362
1008
|
const tokenizer = await this.loadTokenizer(modelConfig);
|
|
1363
|
-
|
|
1009
|
+
logger4.info("Tokenizer loaded, decoding tokens...");
|
|
1364
1010
|
const decoded = await tokenizer.decode(tokens, {
|
|
1365
1011
|
skip_special_tokens: true,
|
|
1366
1012
|
clean_up_tokenization_spaces: true
|
|
1367
1013
|
});
|
|
1368
|
-
|
|
1014
|
+
logger4.info("Tokens decoded successfully:", {
|
|
1369
1015
|
textLength: decoded.length,
|
|
1370
1016
|
tokenizer: modelConfig.tokenizer.name
|
|
1371
1017
|
});
|
|
1372
1018
|
return decoded;
|
|
1373
1019
|
} catch (error) {
|
|
1374
|
-
|
|
1020
|
+
logger4.error("Token decoding failed:", {
|
|
1375
1021
|
error: error instanceof Error ? error.message : String(error),
|
|
1376
1022
|
stack: error instanceof Error ? error.stack : void 0,
|
|
1377
1023
|
tokenCount: tokens.length,
|
|
@@ -1388,7 +1034,7 @@ import { exec as exec2 } from "node:child_process";
|
|
|
1388
1034
|
import fs2 from "node:fs";
|
|
1389
1035
|
import path2 from "node:path";
|
|
1390
1036
|
import { promisify as promisify2 } from "node:util";
|
|
1391
|
-
import { logger as
|
|
1037
|
+
import { logger as logger5 } from "@elizaos/core";
|
|
1392
1038
|
import { nodewhisper } from "nodejs-whisper";
|
|
1393
1039
|
var execAsync2 = promisify2(exec2);
|
|
1394
1040
|
var TranscribeManager = class _TranscribeManager {
|
|
@@ -1405,7 +1051,7 @@ var TranscribeManager = class _TranscribeManager {
|
|
|
1405
1051
|
*/
|
|
1406
1052
|
constructor(cacheDir) {
|
|
1407
1053
|
this.cacheDir = path2.join(cacheDir, "whisper");
|
|
1408
|
-
|
|
1054
|
+
logger5.debug("Initializing TranscribeManager", {
|
|
1409
1055
|
cacheDir: this.cacheDir,
|
|
1410
1056
|
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
1411
1057
|
});
|
|
@@ -1421,7 +1067,7 @@ var TranscribeManager = class _TranscribeManager {
|
|
|
1421
1067
|
await this.initializeFFmpeg();
|
|
1422
1068
|
this.ffmpegInitialized = true;
|
|
1423
1069
|
} catch (error) {
|
|
1424
|
-
|
|
1070
|
+
logger5.error("FFmpeg initialization failed:", {
|
|
1425
1071
|
error: error instanceof Error ? error.message : String(error),
|
|
1426
1072
|
stack: error instanceof Error ? error.stack : void 0,
|
|
1427
1073
|
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
@@ -1459,13 +1105,13 @@ var TranscribeManager = class _TranscribeManager {
|
|
|
1459
1105
|
try {
|
|
1460
1106
|
const { stdout } = await execAsync2("ffmpeg -version");
|
|
1461
1107
|
this.ffmpegVersion = stdout.split("\n")[0];
|
|
1462
|
-
|
|
1108
|
+
logger5.info("FFmpeg version:", {
|
|
1463
1109
|
version: this.ffmpegVersion,
|
|
1464
1110
|
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
1465
1111
|
});
|
|
1466
1112
|
} catch (error) {
|
|
1467
1113
|
this.ffmpegVersion = null;
|
|
1468
|
-
|
|
1114
|
+
logger5.error("Failed to get FFmpeg version:", {
|
|
1469
1115
|
error: error instanceof Error ? error.message : String(error),
|
|
1470
1116
|
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
1471
1117
|
});
|
|
@@ -1488,12 +1134,17 @@ var TranscribeManager = class _TranscribeManager {
|
|
|
1488
1134
|
if (this.ffmpegAvailable) {
|
|
1489
1135
|
await this.fetchFFmpegVersion();
|
|
1490
1136
|
await this.verifyFFmpegCapabilities();
|
|
1137
|
+
logger5.success("FFmpeg initialized successfully", {
|
|
1138
|
+
version: this.ffmpegVersion,
|
|
1139
|
+
path: this.ffmpegPath,
|
|
1140
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
1141
|
+
});
|
|
1491
1142
|
} else {
|
|
1492
1143
|
this.logFFmpegInstallInstructions();
|
|
1493
1144
|
}
|
|
1494
1145
|
} catch (error) {
|
|
1495
1146
|
this.ffmpegAvailable = false;
|
|
1496
|
-
|
|
1147
|
+
logger5.error("FFmpeg initialization failed:", {
|
|
1497
1148
|
error: error instanceof Error ? error.message : String(error),
|
|
1498
1149
|
stack: error instanceof Error ? error.stack : void 0,
|
|
1499
1150
|
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
@@ -1513,7 +1164,7 @@ var TranscribeManager = class _TranscribeManager {
|
|
|
1513
1164
|
const { stdout, stderr } = await execAsync2("which ffmpeg || where ffmpeg");
|
|
1514
1165
|
this.ffmpegPath = stdout.trim();
|
|
1515
1166
|
this.ffmpegAvailable = true;
|
|
1516
|
-
|
|
1167
|
+
logger5.info("FFmpeg found at:", {
|
|
1517
1168
|
path: this.ffmpegPath,
|
|
1518
1169
|
stderr: stderr ? stderr.trim() : void 0,
|
|
1519
1170
|
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
@@ -1521,7 +1172,7 @@ var TranscribeManager = class _TranscribeManager {
|
|
|
1521
1172
|
} catch (error) {
|
|
1522
1173
|
this.ffmpegAvailable = false;
|
|
1523
1174
|
this.ffmpegPath = null;
|
|
1524
|
-
|
|
1175
|
+
logger5.error("FFmpeg not found in PATH:", {
|
|
1525
1176
|
error: error instanceof Error ? error.message : String(error),
|
|
1526
1177
|
stderr: error instanceof Error && "stderr" in error ? error.stderr : void 0,
|
|
1527
1178
|
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
@@ -1541,7 +1192,7 @@ var TranscribeManager = class _TranscribeManager {
|
|
|
1541
1192
|
throw new Error("FFmpeg installation missing required codecs (pcm_s16le, wav)");
|
|
1542
1193
|
}
|
|
1543
1194
|
} catch (error) {
|
|
1544
|
-
|
|
1195
|
+
logger5.error("FFmpeg capabilities verification failed:", {
|
|
1545
1196
|
error: error instanceof Error ? error.message : String(error),
|
|
1546
1197
|
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
1547
1198
|
});
|
|
@@ -1552,7 +1203,7 @@ var TranscribeManager = class _TranscribeManager {
|
|
|
1552
1203
|
* Logs instructions on how to install FFmpeg if it is not properly installed.
|
|
1553
1204
|
*/
|
|
1554
1205
|
logFFmpegInstallInstructions() {
|
|
1555
|
-
|
|
1206
|
+
logger5.warn("FFmpeg is required but not properly installed. Please install FFmpeg:", {
|
|
1556
1207
|
instructions: {
|
|
1557
1208
|
mac: "brew install ffmpeg",
|
|
1558
1209
|
ubuntu: "sudo apt-get install ffmpeg",
|
|
@@ -1605,7 +1256,7 @@ var TranscribeManager = class _TranscribeManager {
|
|
|
1605
1256
|
`ffmpeg -y -loglevel error -i "${inputPath}" -acodec pcm_s16le -ar 16000 -ac 1 "${outputPath}"`
|
|
1606
1257
|
);
|
|
1607
1258
|
if (stderr) {
|
|
1608
|
-
|
|
1259
|
+
logger5.warn("FFmpeg conversion error:", {
|
|
1609
1260
|
stderr,
|
|
1610
1261
|
inputPath,
|
|
1611
1262
|
outputPath,
|
|
@@ -1616,7 +1267,7 @@ var TranscribeManager = class _TranscribeManager {
|
|
|
1616
1267
|
throw new Error("WAV file was not created successfully");
|
|
1617
1268
|
}
|
|
1618
1269
|
} catch (error) {
|
|
1619
|
-
|
|
1270
|
+
logger5.error("Audio conversion failed:", {
|
|
1620
1271
|
error: error instanceof Error ? error.message : String(error),
|
|
1621
1272
|
stack: error instanceof Error ? error.stack : void 0,
|
|
1622
1273
|
command: `ffmpeg -y -loglevel error -i "${inputPath}" -acodec pcm_s16le -ar 16000 -ac 1 "${outputPath}"`,
|
|
@@ -1652,7 +1303,7 @@ var TranscribeManager = class _TranscribeManager {
|
|
|
1652
1303
|
}
|
|
1653
1304
|
return tempWavFile;
|
|
1654
1305
|
} catch (error) {
|
|
1655
|
-
|
|
1306
|
+
logger5.error("Audio preprocessing failed:", {
|
|
1656
1307
|
error: error instanceof Error ? error.message : String(error),
|
|
1657
1308
|
stack: error instanceof Error ? error.stack : void 0,
|
|
1658
1309
|
ffmpegAvailable: this.ffmpegAvailable,
|
|
@@ -1679,7 +1330,7 @@ var TranscribeManager = class _TranscribeManager {
|
|
|
1679
1330
|
}
|
|
1680
1331
|
try {
|
|
1681
1332
|
const wavFile = await this.preprocessAudio(audioBuffer);
|
|
1682
|
-
|
|
1333
|
+
logger5.info("Starting transcription with whisper...");
|
|
1683
1334
|
const originalStdoutWrite = process.stdout.write;
|
|
1684
1335
|
const originalStderrWrite = process.stderr.write;
|
|
1685
1336
|
const noopWrite = () => true;
|
|
@@ -1702,19 +1353,19 @@ var TranscribeManager = class _TranscribeManager {
|
|
|
1702
1353
|
}
|
|
1703
1354
|
if (fs2.existsSync(wavFile)) {
|
|
1704
1355
|
fs2.unlinkSync(wavFile);
|
|
1705
|
-
|
|
1356
|
+
logger5.info("Temporary WAV file cleaned up");
|
|
1706
1357
|
}
|
|
1707
1358
|
const cleanText = output.split("\n").map((line) => {
|
|
1708
1359
|
const textMatch = line.match(/](.+)$/);
|
|
1709
1360
|
return textMatch ? textMatch[1].trim() : line.trim();
|
|
1710
1361
|
}).filter((line) => line).join(" ");
|
|
1711
|
-
|
|
1362
|
+
logger5.success("Transcription complete:", {
|
|
1712
1363
|
textLength: cleanText.length,
|
|
1713
1364
|
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
1714
1365
|
});
|
|
1715
1366
|
return { text: cleanText };
|
|
1716
1367
|
} catch (error) {
|
|
1717
|
-
|
|
1368
|
+
logger5.error("Transcription failed:", {
|
|
1718
1369
|
error: error instanceof Error ? error.message : String(error),
|
|
1719
1370
|
stack: error instanceof Error ? error.stack : void 0,
|
|
1720
1371
|
ffmpegAvailable: this.ffmpegAvailable
|
|
@@ -1728,244 +1379,162 @@ var TranscribeManager = class _TranscribeManager {
|
|
|
1728
1379
|
import fs3 from "node:fs";
|
|
1729
1380
|
import path3 from "node:path";
|
|
1730
1381
|
import { Readable } from "node:stream";
|
|
1731
|
-
import { logger as
|
|
1732
|
-
import {
|
|
1733
|
-
|
|
1734
|
-
} from "node-llama-cpp";
|
|
1382
|
+
import { logger as logger6, prependWavHeader } from "@elizaos/core";
|
|
1383
|
+
import { pipeline } from "@huggingface/transformers";
|
|
1384
|
+
import { fetch as fetch2 } from "undici";
|
|
1735
1385
|
var TTSManager = class _TTSManager {
|
|
1736
1386
|
static instance = null;
|
|
1737
1387
|
cacheDir;
|
|
1738
|
-
|
|
1739
|
-
|
|
1740
|
-
sequence = null;
|
|
1388
|
+
synthesizer = null;
|
|
1389
|
+
defaultSpeakerEmbedding = null;
|
|
1741
1390
|
initialized = false;
|
|
1742
|
-
|
|
1743
|
-
modelsDir;
|
|
1744
|
-
/**
|
|
1745
|
-
* Creates a new instance of TTSManager with the provided cache directory.
|
|
1746
|
-
*
|
|
1747
|
-
* @param {string} cacheDir - The directory where cached data will be stored.
|
|
1748
|
-
*/
|
|
1391
|
+
initializingPromise = null;
|
|
1749
1392
|
constructor(cacheDir) {
|
|
1750
1393
|
this.cacheDir = path3.join(cacheDir, "tts");
|
|
1751
|
-
this.modelsDir = process.env.LLAMALOCAL_PATH?.trim() ? path3.resolve(process.env.LLAMALOCAL_PATH.trim()) : path3.join(process.cwd(), "models");
|
|
1752
|
-
this.downloadManager = DownloadManager.getInstance(this.cacheDir, this.modelsDir);
|
|
1753
1394
|
this.ensureCacheDirectory();
|
|
1754
|
-
|
|
1395
|
+
logger6.debug("TTSManager using Transformers.js initialized");
|
|
1755
1396
|
}
|
|
1756
|
-
/**
|
|
1757
|
-
* Returns an instance of TTSManager, creating a new one if none exist.
|
|
1758
|
-
*
|
|
1759
|
-
* @param {string} cacheDir - The directory path to store cached audio files.
|
|
1760
|
-
* @returns {TTSManager} An instance of TTSManager.
|
|
1761
|
-
*/
|
|
1762
1397
|
static getInstance(cacheDir) {
|
|
1763
1398
|
if (!_TTSManager.instance) {
|
|
1764
1399
|
_TTSManager.instance = new _TTSManager(cacheDir);
|
|
1765
1400
|
}
|
|
1766
1401
|
return _TTSManager.instance;
|
|
1767
1402
|
}
|
|
1768
|
-
/**
|
|
1769
|
-
* Ensures that the cache directory exists. If it does not exist, the directory will be created.
|
|
1770
|
-
*/
|
|
1771
1403
|
ensureCacheDirectory() {
|
|
1772
1404
|
if (!fs3.existsSync(this.cacheDir)) {
|
|
1773
1405
|
fs3.mkdirSync(this.cacheDir, { recursive: true });
|
|
1774
|
-
|
|
1406
|
+
logger6.debug("Created TTS cache directory:", this.cacheDir);
|
|
1775
1407
|
}
|
|
1776
1408
|
}
|
|
1777
|
-
/**
|
|
1778
|
-
* Asynchronously initializes the TTS module with GGUF backend.
|
|
1779
|
-
* If already initialized or missing necessary components (model and context), it returns early.
|
|
1780
|
-
* Handles model download using different URL patterns as fallback if model not found locally.
|
|
1781
|
-
* Initializes the TTS model, creates context, and sets the sequence for TTS generation.
|
|
1782
|
-
* Logs detailed steps and final output of initialization.
|
|
1783
|
-
*
|
|
1784
|
-
* @returns {Promise<void>} A promise that resolves once the TTS module is fully initialized.
|
|
1785
|
-
*/
|
|
1786
1409
|
async initialize() {
|
|
1787
|
-
|
|
1788
|
-
|
|
1789
|
-
|
|
1790
|
-
|
|
1791
|
-
|
|
1792
|
-
|
|
1793
|
-
|
|
1794
|
-
|
|
1795
|
-
|
|
1796
|
-
|
|
1797
|
-
|
|
1798
|
-
|
|
1799
|
-
|
|
1800
|
-
|
|
1801
|
-
|
|
1802
|
-
|
|
1803
|
-
|
|
1804
|
-
|
|
1805
|
-
|
|
1806
|
-
|
|
1807
|
-
|
|
1808
|
-
|
|
1809
|
-
|
|
1810
|
-
|
|
1811
|
-
|
|
1812
|
-
|
|
1813
|
-
|
|
1814
|
-
|
|
1815
|
-
|
|
1816
|
-
|
|
1817
|
-
|
|
1818
|
-
|
|
1819
|
-
|
|
1820
|
-
|
|
1821
|
-
|
|
1822
|
-
|
|
1823
|
-
|
|
1824
|
-
|
|
1825
|
-
await
|
|
1826
|
-
|
|
1827
|
-
|
|
1828
|
-
|
|
1829
|
-
break;
|
|
1830
|
-
} catch (error) {
|
|
1831
|
-
lastError = error;
|
|
1832
|
-
logger7.warn("TTS model download attempt failed:", {
|
|
1833
|
-
description: attempt.description,
|
|
1834
|
-
error: error instanceof Error ? error.message : String(error),
|
|
1835
|
-
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
1836
|
-
});
|
|
1410
|
+
if (this.initializingPromise) {
|
|
1411
|
+
logger6.debug("TTS initialization already in progress, awaiting existing promise.");
|
|
1412
|
+
return this.initializingPromise;
|
|
1413
|
+
}
|
|
1414
|
+
if (this.initialized) {
|
|
1415
|
+
logger6.debug("TTS already initialized.");
|
|
1416
|
+
return;
|
|
1417
|
+
}
|
|
1418
|
+
this.initializingPromise = (async () => {
|
|
1419
|
+
try {
|
|
1420
|
+
logger6.info("Initializing TTS with Transformers.js backend...");
|
|
1421
|
+
const ttsModelSpec = MODEL_SPECS.tts.default;
|
|
1422
|
+
if (!ttsModelSpec) {
|
|
1423
|
+
throw new Error("Default TTS model specification not found in MODEL_SPECS.");
|
|
1424
|
+
}
|
|
1425
|
+
const modelName = ttsModelSpec.modelId;
|
|
1426
|
+
const speakerEmbeddingUrl = ttsModelSpec.defaultSpeakerEmbeddingUrl;
|
|
1427
|
+
logger6.info(`Loading TTS pipeline for model: ${modelName}`);
|
|
1428
|
+
this.synthesizer = await pipeline("text-to-audio", modelName);
|
|
1429
|
+
logger6.success(`TTS pipeline loaded successfully for model: ${modelName}`);
|
|
1430
|
+
if (speakerEmbeddingUrl) {
|
|
1431
|
+
const embeddingFilename = path3.basename(new URL(speakerEmbeddingUrl).pathname);
|
|
1432
|
+
const embeddingPath = path3.join(this.cacheDir, embeddingFilename);
|
|
1433
|
+
if (fs3.existsSync(embeddingPath)) {
|
|
1434
|
+
logger6.info("Loading default speaker embedding from cache...");
|
|
1435
|
+
const buffer = fs3.readFileSync(embeddingPath);
|
|
1436
|
+
this.defaultSpeakerEmbedding = new Float32Array(
|
|
1437
|
+
buffer.buffer,
|
|
1438
|
+
buffer.byteOffset,
|
|
1439
|
+
buffer.length / Float32Array.BYTES_PER_ELEMENT
|
|
1440
|
+
);
|
|
1441
|
+
logger6.success("Default speaker embedding loaded from cache.");
|
|
1442
|
+
} else {
|
|
1443
|
+
logger6.info(`Downloading default speaker embedding from: ${speakerEmbeddingUrl}`);
|
|
1444
|
+
const response = await fetch2(speakerEmbeddingUrl);
|
|
1445
|
+
if (!response.ok) {
|
|
1446
|
+
throw new Error(`Failed to download speaker embedding: ${response.statusText}`);
|
|
1447
|
+
}
|
|
1448
|
+
const buffer = await response.arrayBuffer();
|
|
1449
|
+
this.defaultSpeakerEmbedding = new Float32Array(buffer);
|
|
1450
|
+
fs3.writeFileSync(embeddingPath, Buffer.from(buffer));
|
|
1451
|
+
logger6.success("Default speaker embedding downloaded and cached.");
|
|
1837
1452
|
}
|
|
1453
|
+
} else {
|
|
1454
|
+
logger6.warn(
|
|
1455
|
+
`No default speaker embedding URL specified for model ${modelName}. Speaker control may be limited.`
|
|
1456
|
+
);
|
|
1457
|
+
this.defaultSpeakerEmbedding = null;
|
|
1838
1458
|
}
|
|
1839
|
-
if (!
|
|
1840
|
-
throw
|
|
1459
|
+
if (!this.synthesizer) {
|
|
1460
|
+
throw new Error("TTS initialization failed: Pipeline not loaded.");
|
|
1841
1461
|
}
|
|
1462
|
+
logger6.success("TTS initialization complete (Transformers.js)");
|
|
1463
|
+
this.initialized = true;
|
|
1464
|
+
} catch (error) {
|
|
1465
|
+
logger6.error("TTS (Transformers.js) initialization failed:", {
|
|
1466
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1467
|
+
stack: error instanceof Error ? error.stack : void 0
|
|
1468
|
+
});
|
|
1469
|
+
this.initialized = false;
|
|
1470
|
+
this.synthesizer = null;
|
|
1471
|
+
this.defaultSpeakerEmbedding = null;
|
|
1472
|
+
throw error;
|
|
1473
|
+
} finally {
|
|
1474
|
+
this.initializingPromise = null;
|
|
1475
|
+
logger6.debug("TTS initializingPromise cleared after completion/failure.");
|
|
1842
1476
|
}
|
|
1843
|
-
|
|
1844
|
-
|
|
1845
|
-
this.model = await llama.loadModel({
|
|
1846
|
-
modelPath,
|
|
1847
|
-
gpuLayers: 0
|
|
1848
|
-
// Force CPU for now until we add GPU support
|
|
1849
|
-
});
|
|
1850
|
-
this.ctx = await this.model.createContext({
|
|
1851
|
-
contextSize: modelSpec.contextSize
|
|
1852
|
-
});
|
|
1853
|
-
this.sequence = this.ctx.getSequence();
|
|
1854
|
-
logger7.success("TTS initialization complete", {
|
|
1855
|
-
modelPath,
|
|
1856
|
-
contextSize: modelSpec.contextSize,
|
|
1857
|
-
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
1858
|
-
});
|
|
1859
|
-
this.initialized = true;
|
|
1860
|
-
} catch (error) {
|
|
1861
|
-
logger7.error("TTS initialization failed:", {
|
|
1862
|
-
error: error instanceof Error ? error.message : String(error),
|
|
1863
|
-
model: MODEL_SPECS.tts.base.name,
|
|
1864
|
-
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
1865
|
-
});
|
|
1866
|
-
throw error;
|
|
1867
|
-
}
|
|
1477
|
+
})();
|
|
1478
|
+
return this.initializingPromise;
|
|
1868
1479
|
}
|
|
1869
1480
|
/**
|
|
1870
|
-
* Asynchronously generates speech from a given text using the
|
|
1481
|
+
* Asynchronously generates speech from a given text using the Transformers.js pipeline.
|
|
1871
1482
|
* @param {string} text - The text to generate speech from.
|
|
1872
|
-
* @returns {Promise<Readable>} A promise that resolves to a Readable stream containing the generated audio data.
|
|
1873
|
-
* @throws {Error} If the TTS model is not initialized or if
|
|
1483
|
+
* @returns {Promise<Readable>} A promise that resolves to a Readable stream containing the generated WAV audio data.
|
|
1484
|
+
* @throws {Error} If the TTS model is not initialized or if generation fails.
|
|
1874
1485
|
*/
|
|
1875
1486
|
async generateSpeech(text) {
|
|
1876
1487
|
try {
|
|
1877
1488
|
await this.initialize();
|
|
1878
|
-
if (!this.
|
|
1879
|
-
throw new Error("TTS
|
|
1489
|
+
if (!this.synthesizer) {
|
|
1490
|
+
throw new Error("TTS Manager not properly initialized.");
|
|
1880
1491
|
}
|
|
1881
|
-
|
|
1882
|
-
|
|
1883
|
-
logger7.info("Formatted prompt:", { prompt });
|
|
1884
|
-
logger7.info("Tokenizing input...");
|
|
1885
|
-
const inputTokens = this.model.tokenize(prompt);
|
|
1886
|
-
logger7.info("Input tokenized:", { tokenCount: inputTokens.length });
|
|
1887
|
-
const maxTokens = inputTokens.length * 2;
|
|
1888
|
-
logger7.info("Starting token generation with optimized limit:", {
|
|
1889
|
-
maxTokens
|
|
1492
|
+
logger6.info("Starting speech generation with Transformers.js for text:", {
|
|
1493
|
+
text: text.substring(0, 50) + "..."
|
|
1890
1494
|
});
|
|
1891
|
-
const
|
|
1892
|
-
|
|
1893
|
-
|
|
1894
|
-
|
|
1895
|
-
|
|
1896
|
-
|
|
1897
|
-
|
|
1898
|
-
|
|
1899
|
-
|
|
1900
|
-
|
|
1901
|
-
|
|
1902
|
-
|
|
1903
|
-
`Token generation: ${progressBar} ${percent}% (${responseTokens.length}/${maxTokens})`
|
|
1904
|
-
);
|
|
1905
|
-
if (responseTokens.length >= maxTokens) {
|
|
1906
|
-
logger7.info("Token generation complete");
|
|
1907
|
-
break;
|
|
1908
|
-
}
|
|
1909
|
-
}
|
|
1910
|
-
} catch (error) {
|
|
1911
|
-
logger7.error("Token generation error:", error);
|
|
1912
|
-
throw error;
|
|
1495
|
+
const output = await this.synthesizer(text, {
|
|
1496
|
+
// Pass embedding only if it was loaded
|
|
1497
|
+
...this.defaultSpeakerEmbedding && { speaker_embeddings: this.defaultSpeakerEmbedding }
|
|
1498
|
+
});
|
|
1499
|
+
const audioFloat32 = output.audio;
|
|
1500
|
+
const samplingRate = output.sampling_rate;
|
|
1501
|
+
logger6.info("Raw audio data received from pipeline:", {
|
|
1502
|
+
samplingRate,
|
|
1503
|
+
length: audioFloat32.length
|
|
1504
|
+
});
|
|
1505
|
+
if (!audioFloat32 || audioFloat32.length === 0) {
|
|
1506
|
+
throw new Error("TTS pipeline generated empty audio output.");
|
|
1913
1507
|
}
|
|
1914
|
-
|
|
1915
|
-
|
|
1508
|
+
const pcmData = new Int16Array(audioFloat32.length);
|
|
1509
|
+
for (let i = 0; i < audioFloat32.length; i++) {
|
|
1510
|
+
const s = Math.max(-1, Math.min(1, audioFloat32[i]));
|
|
1511
|
+
pcmData[i] = s < 0 ? s * 32768 : s * 32767;
|
|
1916
1512
|
}
|
|
1917
|
-
|
|
1918
|
-
|
|
1919
|
-
|
|
1920
|
-
});
|
|
1921
|
-
logger7.info("Audio data generated:", {
|
|
1922
|
-
byteLength: audioData.length,
|
|
1923
|
-
sampleRate: MODEL_SPECS.tts.base.sampleRate
|
|
1513
|
+
const audioBuffer = Buffer.from(pcmData.buffer);
|
|
1514
|
+
logger6.info("Audio data converted to 16-bit PCM Buffer:", {
|
|
1515
|
+
byteLength: audioBuffer.length
|
|
1924
1516
|
});
|
|
1925
1517
|
const audioStream = prependWavHeader(
|
|
1926
|
-
Readable.from(
|
|
1927
|
-
|
|
1928
|
-
|
|
1518
|
+
Readable.from(audioBuffer),
|
|
1519
|
+
audioBuffer.length,
|
|
1520
|
+
// Pass buffer length in bytes
|
|
1521
|
+
samplingRate,
|
|
1929
1522
|
1,
|
|
1523
|
+
// Number of channels (assuming mono)
|
|
1930
1524
|
16
|
|
1525
|
+
// Bit depth
|
|
1931
1526
|
);
|
|
1932
|
-
|
|
1527
|
+
logger6.success("Speech generation complete (Transformers.js)");
|
|
1933
1528
|
return audioStream;
|
|
1934
1529
|
} catch (error) {
|
|
1935
|
-
|
|
1530
|
+
logger6.error("Transformers.js speech generation failed:", {
|
|
1936
1531
|
error: error instanceof Error ? error.message : String(error),
|
|
1937
|
-
text
|
|
1532
|
+
text: text.substring(0, 50) + "...",
|
|
1533
|
+
stack: error instanceof Error ? error.stack : void 0
|
|
1938
1534
|
});
|
|
1939
1535
|
throw error;
|
|
1940
1536
|
}
|
|
1941
1537
|
}
|
|
1942
|
-
/**
|
|
1943
|
-
* Processes the audio response from the TTS service by converting
|
|
1944
|
-
* the data to 16-bit PCM format.
|
|
1945
|
-
* If the response contains direct audio data, it converts Float32Array
|
|
1946
|
-
* to 16-bit PCM. If the response only contains tokens, it converts
|
|
1947
|
-
* them to PCM data. The actual conversion process may vary depending
|
|
1948
|
-
* on the model used.
|
|
1949
|
-
*
|
|
1950
|
-
* @param {TTSResponse} response - The response object from the TTS service
|
|
1951
|
-
* @returns {Buffer} The processed audio data in 16-bit PCM format
|
|
1952
|
-
*/
|
|
1953
|
-
processAudioResponse(response) {
|
|
1954
|
-
if (response.audio) {
|
|
1955
|
-
const pcmData2 = new Int16Array(response.audio.length);
|
|
1956
|
-
for (let i = 0; i < response.audio.length; i++) {
|
|
1957
|
-
const s = Math.max(-1, Math.min(1, response.audio[i]));
|
|
1958
|
-
pcmData2[i] = s < 0 ? s * 32768 : s * 32767;
|
|
1959
|
-
}
|
|
1960
|
-
return Buffer.from(pcmData2.buffer);
|
|
1961
|
-
}
|
|
1962
|
-
const pcmData = new Int16Array(response.tokens.length * 2);
|
|
1963
|
-
for (let i = 0; i < response.tokens.length; i++) {
|
|
1964
|
-
pcmData[i * 2] = response.tokens[i] & 65535;
|
|
1965
|
-
pcmData[i * 2 + 1] = response.tokens[i] >> 16 & 65535;
|
|
1966
|
-
}
|
|
1967
|
-
return Buffer.from(pcmData.buffer);
|
|
1968
|
-
}
|
|
1969
1538
|
};
|
|
1970
1539
|
|
|
1971
1540
|
// src/utils/visionManager.ts
|
|
@@ -1974,7 +1543,7 @@ import fs4 from "node:fs";
|
|
|
1974
1543
|
import os2 from "node:os";
|
|
1975
1544
|
import path4 from "node:path";
|
|
1976
1545
|
import process2 from "node:process";
|
|
1977
|
-
import { logger as
|
|
1546
|
+
import { logger as logger7 } from "@elizaos/core";
|
|
1978
1547
|
import {
|
|
1979
1548
|
AutoProcessor,
|
|
1980
1549
|
AutoTokenizer as AutoTokenizer2,
|
|
@@ -2012,7 +1581,7 @@ var VisionManager = class _VisionManager {
|
|
|
2012
1581
|
this.ensureModelsDirExists();
|
|
2013
1582
|
this.downloadManager = DownloadManager.getInstance(this.cacheDir, this.modelsDir);
|
|
2014
1583
|
this.platformConfig = this.getPlatformConfig();
|
|
2015
|
-
|
|
1584
|
+
logger7.debug("VisionManager initialized");
|
|
2016
1585
|
}
|
|
2017
1586
|
/**
|
|
2018
1587
|
* Retrieves the platform configuration based on the operating system and architecture.
|
|
@@ -2049,7 +1618,7 @@ var VisionManager = class _VisionManager {
|
|
|
2049
1618
|
*/
|
|
2050
1619
|
ensureModelsDirExists() {
|
|
2051
1620
|
if (!existsSync(this.modelsDir)) {
|
|
2052
|
-
|
|
1621
|
+
logger7.debug(`Creating models directory at: ${this.modelsDir}`);
|
|
2053
1622
|
fs4.mkdirSync(this.modelsDir, { recursive: true });
|
|
2054
1623
|
}
|
|
2055
1624
|
}
|
|
@@ -2076,7 +1645,7 @@ var VisionManager = class _VisionManager {
|
|
|
2076
1645
|
checkCacheExists(modelId, type) {
|
|
2077
1646
|
const modelPath = path4.join(this.modelsDir, modelId.replace("/", "--"), type);
|
|
2078
1647
|
if (existsSync(modelPath)) {
|
|
2079
|
-
|
|
1648
|
+
logger7.info(`${type} found at: ${modelPath}`);
|
|
2080
1649
|
return true;
|
|
2081
1650
|
}
|
|
2082
1651
|
return false;
|
|
@@ -2099,7 +1668,7 @@ var VisionManager = class _VisionManager {
|
|
|
2099
1668
|
...component,
|
|
2100
1669
|
dtype: defaultDtype
|
|
2101
1670
|
}));
|
|
2102
|
-
|
|
1671
|
+
logger7.info("Model components configured with dtype:", {
|
|
2103
1672
|
platform,
|
|
2104
1673
|
arch,
|
|
2105
1674
|
defaultDtype,
|
|
@@ -2128,19 +1697,19 @@ var VisionManager = class _VisionManager {
|
|
|
2128
1697
|
async initialize() {
|
|
2129
1698
|
try {
|
|
2130
1699
|
if (this.initialized) {
|
|
2131
|
-
|
|
1700
|
+
logger7.info("Vision model already initialized, skipping initialization");
|
|
2132
1701
|
return;
|
|
2133
1702
|
}
|
|
2134
|
-
|
|
1703
|
+
logger7.info("Starting vision model initialization...");
|
|
2135
1704
|
const modelSpec = MODEL_SPECS.vision;
|
|
2136
|
-
|
|
1705
|
+
logger7.info("Configuring environment for vision model...");
|
|
2137
1706
|
env.allowLocalModels = true;
|
|
2138
1707
|
env.allowRemoteModels = true;
|
|
2139
1708
|
if (this.platformConfig.useOnnx) {
|
|
2140
1709
|
env.backends.onnx.enabled = true;
|
|
2141
1710
|
env.backends.onnx.logLevel = "info";
|
|
2142
1711
|
}
|
|
2143
|
-
|
|
1712
|
+
logger7.info("Loading Florence2 model...");
|
|
2144
1713
|
try {
|
|
2145
1714
|
let lastProgress = -1;
|
|
2146
1715
|
const modelCached = this.checkCacheExists(modelSpec.modelId, "model");
|
|
@@ -2158,22 +1727,22 @@ var VisionManager = class _VisionManager {
|
|
|
2158
1727
|
const barLength = 30;
|
|
2159
1728
|
const filledLength = Math.floor(currentProgress / 100 * barLength);
|
|
2160
1729
|
const progressBar = "\u25B0".repeat(filledLength) + "\u25B1".repeat(barLength - filledLength);
|
|
2161
|
-
|
|
1730
|
+
logger7.info(`Downloading vision model: ${progressBar} ${currentProgress}%`);
|
|
2162
1731
|
if (currentProgress === 100) this.modelDownloaded = true;
|
|
2163
1732
|
}
|
|
2164
1733
|
}
|
|
2165
1734
|
});
|
|
2166
1735
|
this.model = model;
|
|
2167
|
-
|
|
1736
|
+
logger7.success("Florence2 model loaded successfully");
|
|
2168
1737
|
} catch (error) {
|
|
2169
|
-
|
|
1738
|
+
logger7.error("Failed to load Florence2 model:", {
|
|
2170
1739
|
error: error instanceof Error ? error.message : String(error),
|
|
2171
1740
|
stack: error instanceof Error ? error.stack : void 0,
|
|
2172
1741
|
modelId: modelSpec.modelId
|
|
2173
1742
|
});
|
|
2174
1743
|
throw error;
|
|
2175
1744
|
}
|
|
2176
|
-
|
|
1745
|
+
logger7.info("Loading vision tokenizer...");
|
|
2177
1746
|
try {
|
|
2178
1747
|
const tokenizerCached = this.checkCacheExists(modelSpec.modelId, "tokenizer");
|
|
2179
1748
|
let tokenizerProgress = -1;
|
|
@@ -2189,21 +1758,21 @@ var VisionManager = class _VisionManager {
|
|
|
2189
1758
|
const barLength = 30;
|
|
2190
1759
|
const filledLength = Math.floor(currentProgress / 100 * barLength);
|
|
2191
1760
|
const progressBar = "\u25B0".repeat(filledLength) + "\u25B1".repeat(barLength - filledLength);
|
|
2192
|
-
|
|
1761
|
+
logger7.info(`Downloading vision tokenizer: ${progressBar} ${currentProgress}%`);
|
|
2193
1762
|
if (currentProgress === 100) this.tokenizerDownloaded = true;
|
|
2194
1763
|
}
|
|
2195
1764
|
}
|
|
2196
1765
|
});
|
|
2197
|
-
|
|
1766
|
+
logger7.success("Vision tokenizer loaded successfully");
|
|
2198
1767
|
} catch (error) {
|
|
2199
|
-
|
|
1768
|
+
logger7.error("Failed to load tokenizer:", {
|
|
2200
1769
|
error: error instanceof Error ? error.message : String(error),
|
|
2201
1770
|
stack: error instanceof Error ? error.stack : void 0,
|
|
2202
1771
|
modelId: modelSpec.modelId
|
|
2203
1772
|
});
|
|
2204
1773
|
throw error;
|
|
2205
1774
|
}
|
|
2206
|
-
|
|
1775
|
+
logger7.info("Loading vision processor...");
|
|
2207
1776
|
try {
|
|
2208
1777
|
const processorCached = this.checkCacheExists(modelSpec.modelId, "processor");
|
|
2209
1778
|
let processorProgress = -1;
|
|
@@ -2220,14 +1789,14 @@ var VisionManager = class _VisionManager {
|
|
|
2220
1789
|
const barLength = 30;
|
|
2221
1790
|
const filledLength = Math.floor(currentProgress / 100 * barLength);
|
|
2222
1791
|
const progressBar = "\u25B0".repeat(filledLength) + "\u25B1".repeat(barLength - filledLength);
|
|
2223
|
-
|
|
1792
|
+
logger7.info(`Downloading vision processor: ${progressBar} ${currentProgress}%`);
|
|
2224
1793
|
if (currentProgress === 100) this.processorDownloaded = true;
|
|
2225
1794
|
}
|
|
2226
1795
|
}
|
|
2227
1796
|
});
|
|
2228
|
-
|
|
1797
|
+
logger7.success("Vision processor loaded successfully");
|
|
2229
1798
|
} catch (error) {
|
|
2230
|
-
|
|
1799
|
+
logger7.error("Failed to load vision processor:", {
|
|
2231
1800
|
error: error instanceof Error ? error.message : String(error),
|
|
2232
1801
|
stack: error instanceof Error ? error.stack : void 0,
|
|
2233
1802
|
modelId: modelSpec.modelId
|
|
@@ -2235,9 +1804,9 @@ var VisionManager = class _VisionManager {
|
|
|
2235
1804
|
throw error;
|
|
2236
1805
|
}
|
|
2237
1806
|
this.initialized = true;
|
|
2238
|
-
|
|
1807
|
+
logger7.success("Vision model initialization complete");
|
|
2239
1808
|
} catch (error) {
|
|
2240
|
-
|
|
1809
|
+
logger7.error("Vision model initialization failed:", {
|
|
2241
1810
|
error: error instanceof Error ? error.message : String(error),
|
|
2242
1811
|
stack: error instanceof Error ? error.stack : void 0,
|
|
2243
1812
|
modelsDir: this.modelsDir
|
|
@@ -2253,13 +1822,13 @@ var VisionManager = class _VisionManager {
|
|
|
2253
1822
|
*/
|
|
2254
1823
|
async fetchImage(url) {
|
|
2255
1824
|
try {
|
|
2256
|
-
|
|
1825
|
+
logger7.info(`Fetching image from URL: ${url.slice(0, 100)}...`);
|
|
2257
1826
|
if (url.startsWith("data:")) {
|
|
2258
|
-
|
|
1827
|
+
logger7.info("Processing data URL...");
|
|
2259
1828
|
const [header, base64Data] = url.split(",");
|
|
2260
1829
|
const mimeType2 = header.split(";")[0].split(":")[1];
|
|
2261
1830
|
const buffer2 = Buffer.from(base64Data, "base64");
|
|
2262
|
-
|
|
1831
|
+
logger7.info("Data URL processed successfully");
|
|
2263
1832
|
return { buffer: buffer2, mimeType: mimeType2 };
|
|
2264
1833
|
}
|
|
2265
1834
|
const response = await fetch(url);
|
|
@@ -2268,14 +1837,14 @@ var VisionManager = class _VisionManager {
|
|
|
2268
1837
|
}
|
|
2269
1838
|
const buffer = Buffer.from(await response.arrayBuffer());
|
|
2270
1839
|
const mimeType = response.headers.get("content-type") || "image/jpeg";
|
|
2271
|
-
|
|
1840
|
+
logger7.info("Image fetched successfully:", {
|
|
2272
1841
|
mimeType,
|
|
2273
1842
|
bufferSize: buffer.length,
|
|
2274
1843
|
status: response.status
|
|
2275
1844
|
});
|
|
2276
1845
|
return { buffer, mimeType };
|
|
2277
1846
|
} catch (error) {
|
|
2278
|
-
|
|
1847
|
+
logger7.error("Failed to fetch image:", {
|
|
2279
1848
|
error: error instanceof Error ? error.message : String(error),
|
|
2280
1849
|
stack: error instanceof Error ? error.stack : void 0,
|
|
2281
1850
|
url
|
|
@@ -2290,37 +1859,37 @@ var VisionManager = class _VisionManager {
|
|
|
2290
1859
|
*/
|
|
2291
1860
|
async processImage(imageUrl) {
|
|
2292
1861
|
try {
|
|
2293
|
-
|
|
1862
|
+
logger7.info("Starting image processing...");
|
|
2294
1863
|
if (!this.initialized) {
|
|
2295
|
-
|
|
1864
|
+
logger7.info("Vision model not initialized, initializing now...");
|
|
2296
1865
|
await this.initialize();
|
|
2297
1866
|
}
|
|
2298
1867
|
if (!this.model || !this.processor || !this.tokenizer) {
|
|
2299
1868
|
throw new Error("Vision model components not properly initialized");
|
|
2300
1869
|
}
|
|
2301
|
-
|
|
1870
|
+
logger7.info("Fetching image...");
|
|
2302
1871
|
const { buffer, mimeType } = await this.fetchImage(imageUrl);
|
|
2303
|
-
|
|
1872
|
+
logger7.info("Creating image blob...");
|
|
2304
1873
|
const blob = new Blob([buffer], { type: mimeType });
|
|
2305
|
-
|
|
1874
|
+
logger7.info("Converting blob to RawImage...");
|
|
2306
1875
|
const image = await RawImage.fromBlob(blob);
|
|
2307
|
-
|
|
1876
|
+
logger7.info("Processing image with vision processor...");
|
|
2308
1877
|
const visionInputs = await this.processor(image);
|
|
2309
|
-
|
|
1878
|
+
logger7.info("Constructing prompts...");
|
|
2310
1879
|
const prompts = this.processor.construct_prompts("<DETAILED_CAPTION>");
|
|
2311
|
-
|
|
1880
|
+
logger7.info("Tokenizing prompts...");
|
|
2312
1881
|
const textInputs = this.tokenizer(prompts);
|
|
2313
|
-
|
|
1882
|
+
logger7.info("Generating image description...");
|
|
2314
1883
|
const generatedIds = await this.model.generate({
|
|
2315
1884
|
...textInputs,
|
|
2316
1885
|
...visionInputs,
|
|
2317
1886
|
max_new_tokens: MODEL_SPECS.vision.maxTokens
|
|
2318
1887
|
});
|
|
2319
|
-
|
|
1888
|
+
logger7.info("Decoding generated text...");
|
|
2320
1889
|
const generatedText = this.tokenizer.batch_decode(generatedIds, {
|
|
2321
1890
|
skip_special_tokens: false
|
|
2322
1891
|
})[0];
|
|
2323
|
-
|
|
1892
|
+
logger7.info("Post-processing generation...");
|
|
2324
1893
|
const result = this.processor.post_process_generation(
|
|
2325
1894
|
generatedText,
|
|
2326
1895
|
"<DETAILED_CAPTION>",
|
|
@@ -2331,13 +1900,13 @@ var VisionManager = class _VisionManager {
|
|
|
2331
1900
|
title: `${detailedCaption.split(".")[0]}.`,
|
|
2332
1901
|
description: detailedCaption
|
|
2333
1902
|
};
|
|
2334
|
-
|
|
1903
|
+
logger7.success("Image processing complete:", {
|
|
2335
1904
|
titleLength: response.title.length,
|
|
2336
1905
|
descriptionLength: response.description.length
|
|
2337
1906
|
});
|
|
2338
1907
|
return response;
|
|
2339
1908
|
} catch (error) {
|
|
2340
|
-
|
|
1909
|
+
logger7.error("Image processing failed:", {
|
|
2341
1910
|
error: error instanceof Error ? error.message : String(error),
|
|
2342
1911
|
stack: error instanceof Error ? error.stack : void 0,
|
|
2343
1912
|
imageUrl,
|
|
@@ -2352,8 +1921,7 @@ var VisionManager = class _VisionManager {
|
|
|
2352
1921
|
};
|
|
2353
1922
|
|
|
2354
1923
|
// src/index.ts
|
|
2355
|
-
|
|
2356
|
-
var __dirname = path5.dirname(__filename);
|
|
1924
|
+
import { basename } from "path";
|
|
2357
1925
|
var wordsToPunish = [
|
|
2358
1926
|
" please",
|
|
2359
1927
|
" feel",
|
|
@@ -2425,15 +1993,17 @@ var LocalAIManager = class _LocalAIManager {
|
|
|
2425
1993
|
embeddingModelConfig;
|
|
2426
1994
|
transcribeManager;
|
|
2427
1995
|
ttsManager;
|
|
2428
|
-
|
|
1996
|
+
config = null;
|
|
1997
|
+
// Store validated config
|
|
2429
1998
|
// Initialization state flag
|
|
2430
|
-
studioLMInitialized = false;
|
|
2431
1999
|
smallModelInitialized = false;
|
|
2432
2000
|
mediumModelInitialized = false;
|
|
2433
2001
|
embeddingInitialized = false;
|
|
2434
2002
|
visionInitialized = false;
|
|
2435
2003
|
transcriptionInitialized = false;
|
|
2436
2004
|
ttsInitialized = false;
|
|
2005
|
+
environmentInitialized = false;
|
|
2006
|
+
// Add flag for environment initialization
|
|
2437
2007
|
// Initialization promises to prevent duplicate initialization
|
|
2438
2008
|
smallModelInitializingPromise = null;
|
|
2439
2009
|
mediumModelInitializingPromise = null;
|
|
@@ -2441,47 +2011,81 @@ var LocalAIManager = class _LocalAIManager {
|
|
|
2441
2011
|
visionInitializingPromise = null;
|
|
2442
2012
|
transcriptionInitializingPromise = null;
|
|
2443
2013
|
ttsInitializingPromise = null;
|
|
2444
|
-
|
|
2014
|
+
environmentInitializingPromise = null;
|
|
2015
|
+
// Add promise for environment
|
|
2445
2016
|
modelsDir;
|
|
2446
2017
|
/**
|
|
2447
2018
|
* Private constructor function to initialize base managers and paths.
|
|
2448
|
-
*
|
|
2019
|
+
* Model paths are set after environment initialization.
|
|
2449
2020
|
*/
|
|
2450
2021
|
constructor() {
|
|
2451
|
-
|
|
2452
|
-
|
|
2453
|
-
|
|
2022
|
+
this.config = validateConfig();
|
|
2023
|
+
this._setupCacheDir();
|
|
2024
|
+
this.activeModelConfig = MODEL_SPECS.small;
|
|
2025
|
+
this.embeddingModelConfig = MODEL_SPECS.embedding;
|
|
2026
|
+
}
|
|
2027
|
+
/**
|
|
2028
|
+
* Post-validation initialization steps that require config to be set.
|
|
2029
|
+
* Called after config validation in initializeEnvironment.
|
|
2030
|
+
*/
|
|
2031
|
+
_postValidateInit() {
|
|
2032
|
+
this._setupModelsDir();
|
|
2033
|
+
this.downloadManager = DownloadManager.getInstance(this.cacheDir, this.modelsDir);
|
|
2034
|
+
this.tokenizerManager = TokenizerManager.getInstance(this.cacheDir, this.modelsDir);
|
|
2035
|
+
this.visionManager = VisionManager.getInstance(this.cacheDir);
|
|
2036
|
+
this.transcribeManager = TranscribeManager.getInstance(this.cacheDir);
|
|
2037
|
+
this.ttsManager = TTSManager.getInstance(this.cacheDir);
|
|
2038
|
+
}
|
|
2039
|
+
/**
|
|
2040
|
+
* Sets up the models directory, reading from config or environment variables,
|
|
2041
|
+
* and ensures the directory exists.
|
|
2042
|
+
*/
|
|
2043
|
+
_setupModelsDir() {
|
|
2044
|
+
const modelsDirEnv = this.config?.MODELS_DIR?.trim() || process.env.MODELS_DIR?.trim();
|
|
2045
|
+
if (modelsDirEnv) {
|
|
2046
|
+
this.modelsDir = path5.resolve(modelsDirEnv);
|
|
2047
|
+
logger8.info("Using models directory from MODELS_DIR environment variable:", this.modelsDir);
|
|
2454
2048
|
} else {
|
|
2455
|
-
|
|
2456
|
-
|
|
2457
|
-
|
|
2458
|
-
|
|
2459
|
-
|
|
2049
|
+
this.modelsDir = path5.join(os3.homedir(), ".eliza", "models");
|
|
2050
|
+
logger8.info(
|
|
2051
|
+
"MODELS_DIR environment variable not set, using default models directory:",
|
|
2052
|
+
this.modelsDir
|
|
2053
|
+
);
|
|
2054
|
+
}
|
|
2055
|
+
if (!fs5.existsSync(this.modelsDir)) {
|
|
2056
|
+
fs5.mkdirSync(this.modelsDir, { recursive: true });
|
|
2057
|
+
logger8.debug("Ensured models directory exists (created):", this.modelsDir);
|
|
2058
|
+
} else {
|
|
2059
|
+
logger8.debug("Models directory already exists:", this.modelsDir);
|
|
2460
2060
|
}
|
|
2461
|
-
|
|
2462
|
-
|
|
2463
|
-
|
|
2464
|
-
|
|
2061
|
+
}
|
|
2062
|
+
/**
|
|
2063
|
+
* Sets up the cache directory, reading from config or environment variables,
|
|
2064
|
+
* and ensures the directory exists.
|
|
2065
|
+
*/
|
|
2066
|
+
_setupCacheDir() {
|
|
2067
|
+
const cacheDirEnv = this.config?.CACHE_DIR?.trim() || process.env.CACHE_DIR?.trim();
|
|
2465
2068
|
if (cacheDirEnv) {
|
|
2466
2069
|
this.cacheDir = path5.resolve(cacheDirEnv);
|
|
2070
|
+
logger8.info("Using cache directory from CACHE_DIR environment variable:", this.cacheDir);
|
|
2467
2071
|
} else {
|
|
2468
2072
|
const cacheDir = path5.join(os3.homedir(), ".eliza", "cache");
|
|
2469
2073
|
if (!fs5.existsSync(cacheDir)) {
|
|
2470
2074
|
fs5.mkdirSync(cacheDir, { recursive: true });
|
|
2471
|
-
|
|
2075
|
+
logger8.debug("Ensuring cache directory exists (created):", cacheDir);
|
|
2472
2076
|
}
|
|
2473
2077
|
this.cacheDir = cacheDir;
|
|
2078
|
+
logger8.info(
|
|
2079
|
+
"CACHE_DIR environment variable not set, using default cache directory:",
|
|
2080
|
+
this.cacheDir
|
|
2081
|
+
);
|
|
2474
2082
|
}
|
|
2475
|
-
|
|
2476
|
-
|
|
2477
|
-
|
|
2478
|
-
|
|
2479
|
-
|
|
2480
|
-
if (process.env.USE_STUDIOLM_TEXT_MODELS === "true") {
|
|
2481
|
-
this.studioLMManager = StudioLMManager.getInstance();
|
|
2083
|
+
if (!fs5.existsSync(this.cacheDir)) {
|
|
2084
|
+
fs5.mkdirSync(this.cacheDir, { recursive: true });
|
|
2085
|
+
logger8.debug("Ensured cache directory exists (created):", this.cacheDir);
|
|
2086
|
+
} else {
|
|
2087
|
+
logger8.debug("Cache directory already exists:", this.cacheDir);
|
|
2482
2088
|
}
|
|
2483
|
-
this.activeModelConfig = MODEL_SPECS.small;
|
|
2484
|
-
this.embeddingModelConfig = MODEL_SPECS.embedding;
|
|
2485
2089
|
}
|
|
2486
2090
|
/**
|
|
2487
2091
|
* Retrieves the singleton instance of LocalAIManager. If an instance does not already exist, a new one is created and returned.
|
|
@@ -2494,55 +2098,41 @@ var LocalAIManager = class _LocalAIManager {
|
|
|
2494
2098
|
return _LocalAIManager.instance;
|
|
2495
2099
|
}
|
|
2496
2100
|
/**
|
|
2497
|
-
* Initializes the environment by validating the configuration and setting
|
|
2101
|
+
* Initializes the environment by validating the configuration and setting model paths.
|
|
2102
|
+
* Now public to be callable from plugin init and model handlers.
|
|
2498
2103
|
*
|
|
2499
2104
|
* @returns {Promise<void>} A Promise that resolves once the environment has been successfully initialized.
|
|
2500
2105
|
*/
|
|
2501
2106
|
async initializeEnvironment() {
|
|
2502
|
-
|
|
2503
|
-
|
|
2504
|
-
|
|
2505
|
-
|
|
2506
|
-
USE_STUDIOLM_TEXT_MODELS: process.env.USE_STUDIOLM_TEXT_MODELS
|
|
2507
|
-
};
|
|
2508
|
-
const validatedConfig = await validateConfig(config);
|
|
2509
|
-
logger9.info("Environment configuration validated");
|
|
2510
|
-
process.env.USE_LOCAL_AI = String(validatedConfig.USE_LOCAL_AI);
|
|
2511
|
-
process.env.USE_STUDIOLM_TEXT_MODELS = String(validatedConfig.USE_STUDIOLM_TEXT_MODELS);
|
|
2512
|
-
logger9.success("Environment initialization complete");
|
|
2513
|
-
} catch (error) {
|
|
2514
|
-
logger9.error("Environment validation failed:", {
|
|
2515
|
-
error: error instanceof Error ? error.message : String(error),
|
|
2516
|
-
stack: error instanceof Error ? error.stack : void 0
|
|
2517
|
-
});
|
|
2518
|
-
throw error;
|
|
2107
|
+
if (this.environmentInitialized) return;
|
|
2108
|
+
if (this.environmentInitializingPromise) {
|
|
2109
|
+
await this.environmentInitializingPromise;
|
|
2110
|
+
return;
|
|
2519
2111
|
}
|
|
2520
|
-
|
|
2521
|
-
|
|
2522
|
-
|
|
2523
|
-
|
|
2524
|
-
|
|
2525
|
-
|
|
2526
|
-
|
|
2527
|
-
|
|
2528
|
-
|
|
2529
|
-
|
|
2530
|
-
|
|
2531
|
-
|
|
2532
|
-
|
|
2533
|
-
|
|
2534
|
-
|
|
2112
|
+
this.environmentInitializingPromise = (async () => {
|
|
2113
|
+
try {
|
|
2114
|
+
logger8.info("Initializing environment configuration...");
|
|
2115
|
+
this.config = await validateConfig();
|
|
2116
|
+
this._postValidateInit();
|
|
2117
|
+
this.modelPath = path5.join(this.modelsDir, this.config.LOCAL_SMALL_MODEL);
|
|
2118
|
+
this.mediumModelPath = path5.join(this.modelsDir, this.config.LOCAL_LARGE_MODEL);
|
|
2119
|
+
this.embeddingModelPath = path5.join(this.modelsDir, this.config.LOCAL_EMBEDDING_MODEL);
|
|
2120
|
+
logger8.info("Using small model path:", basename(this.modelPath));
|
|
2121
|
+
logger8.info("Using medium model path:", basename(this.mediumModelPath));
|
|
2122
|
+
logger8.info("Using embedding model path:", basename(this.embeddingModelPath));
|
|
2123
|
+
logger8.info("Environment configuration validated and model paths set");
|
|
2124
|
+
this.environmentInitialized = true;
|
|
2125
|
+
logger8.success("Environment initialization complete");
|
|
2126
|
+
} catch (error) {
|
|
2127
|
+
logger8.error("Environment validation failed:", {
|
|
2128
|
+
error: error instanceof Error ? error.message : String(error),
|
|
2129
|
+
stack: error instanceof Error ? error.stack : void 0
|
|
2130
|
+
});
|
|
2131
|
+
this.environmentInitializingPromise = null;
|
|
2132
|
+
throw error;
|
|
2535
2133
|
}
|
|
2536
|
-
|
|
2537
|
-
|
|
2538
|
-
} catch (error) {
|
|
2539
|
-
logger9.error("StudioLM initialization failed:", {
|
|
2540
|
-
error: error instanceof Error ? error.message : String(error),
|
|
2541
|
-
stack: error instanceof Error ? error.stack : void 0,
|
|
2542
|
-
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
2543
|
-
});
|
|
2544
|
-
throw error;
|
|
2545
|
-
}
|
|
2134
|
+
})();
|
|
2135
|
+
await this.environmentInitializingPromise;
|
|
2546
2136
|
}
|
|
2547
2137
|
/**
|
|
2548
2138
|
* Downloads the model based on the modelPath provided.
|
|
@@ -2554,24 +2144,25 @@ var LocalAIManager = class _LocalAIManager {
|
|
|
2554
2144
|
*/
|
|
2555
2145
|
async downloadModel(modelType, customModelSpec) {
|
|
2556
2146
|
let modelSpec;
|
|
2557
|
-
let
|
|
2147
|
+
let modelPathToDownload;
|
|
2148
|
+
await this.initializeEnvironment();
|
|
2558
2149
|
if (customModelSpec) {
|
|
2559
2150
|
modelSpec = customModelSpec;
|
|
2560
|
-
|
|
2561
|
-
} else if (modelType ===
|
|
2151
|
+
modelPathToDownload = modelType === ModelType.TEXT_EMBEDDING ? this.embeddingModelPath : modelType === ModelType.TEXT_LARGE ? this.mediumModelPath : this.modelPath;
|
|
2152
|
+
} else if (modelType === ModelType.TEXT_EMBEDDING) {
|
|
2562
2153
|
modelSpec = MODEL_SPECS.embedding;
|
|
2563
|
-
|
|
2154
|
+
modelPathToDownload = this.embeddingModelPath;
|
|
2564
2155
|
} else {
|
|
2565
|
-
modelSpec = modelType ===
|
|
2566
|
-
|
|
2156
|
+
modelSpec = modelType === ModelType.TEXT_LARGE ? MODEL_SPECS.medium : MODEL_SPECS.small;
|
|
2157
|
+
modelPathToDownload = modelType === ModelType.TEXT_LARGE ? this.mediumModelPath : this.modelPath;
|
|
2567
2158
|
}
|
|
2568
2159
|
try {
|
|
2569
|
-
return await this.downloadManager.downloadModel(modelSpec,
|
|
2160
|
+
return await this.downloadManager.downloadModel(modelSpec, modelPathToDownload);
|
|
2570
2161
|
} catch (error) {
|
|
2571
|
-
|
|
2162
|
+
logger8.error("Model download failed:", {
|
|
2572
2163
|
error: error instanceof Error ? error.message : String(error),
|
|
2573
2164
|
modelType,
|
|
2574
|
-
modelPath
|
|
2165
|
+
modelPath: modelPathToDownload
|
|
2575
2166
|
});
|
|
2576
2167
|
throw error;
|
|
2577
2168
|
}
|
|
@@ -2586,14 +2177,14 @@ var LocalAIManager = class _LocalAIManager {
|
|
|
2586
2177
|
const platformManager = getPlatformManager();
|
|
2587
2178
|
await platformManager.initialize();
|
|
2588
2179
|
const capabilities = platformManager.getCapabilities();
|
|
2589
|
-
|
|
2180
|
+
logger8.info("Platform capabilities detected:", {
|
|
2590
2181
|
platform: capabilities.platform,
|
|
2591
2182
|
gpu: capabilities.gpu?.type || "none",
|
|
2592
2183
|
recommendedModel: capabilities.recommendedModelSize,
|
|
2593
2184
|
supportedBackends: capabilities.supportedBackends
|
|
2594
2185
|
});
|
|
2595
2186
|
} catch (error) {
|
|
2596
|
-
|
|
2187
|
+
logger8.warn("Platform detection failed:", error);
|
|
2597
2188
|
}
|
|
2598
2189
|
}
|
|
2599
2190
|
/**
|
|
@@ -2602,8 +2193,9 @@ var LocalAIManager = class _LocalAIManager {
|
|
|
2602
2193
|
* @param {ModelTypeName} modelType - The type of model to initialize (default: ModelType.TEXT_SMALL)
|
|
2603
2194
|
* @returns {Promise<void>} A promise that resolves when initialization is complete or rejects if an error occurs
|
|
2604
2195
|
*/
|
|
2605
|
-
async initialize(modelType =
|
|
2606
|
-
|
|
2196
|
+
async initialize(modelType = ModelType.TEXT_SMALL) {
|
|
2197
|
+
await this.initializeEnvironment();
|
|
2198
|
+
if (modelType === ModelType.TEXT_LARGE) {
|
|
2607
2199
|
await this.lazyInitMediumModel();
|
|
2608
2200
|
} else {
|
|
2609
2201
|
await this.lazyInitSmallModel();
|
|
@@ -2616,20 +2208,22 @@ var LocalAIManager = class _LocalAIManager {
|
|
|
2616
2208
|
*/
|
|
2617
2209
|
async initializeEmbedding() {
|
|
2618
2210
|
try {
|
|
2619
|
-
|
|
2620
|
-
|
|
2211
|
+
await this.initializeEnvironment();
|
|
2212
|
+
logger8.info("Initializing embedding model...");
|
|
2213
|
+
logger8.info("Models directory:", this.modelsDir);
|
|
2621
2214
|
if (!fs5.existsSync(this.modelsDir)) {
|
|
2622
|
-
|
|
2215
|
+
logger8.warn("Models directory does not exist, creating it:", this.modelsDir);
|
|
2623
2216
|
fs5.mkdirSync(this.modelsDir, { recursive: true });
|
|
2624
2217
|
}
|
|
2625
|
-
await this.downloadModel(
|
|
2218
|
+
await this.downloadModel(ModelType.TEXT_EMBEDDING);
|
|
2626
2219
|
if (!this.llama) {
|
|
2627
|
-
this.llama = await
|
|
2220
|
+
this.llama = await getLlama();
|
|
2628
2221
|
}
|
|
2629
2222
|
if (!this.embeddingModel) {
|
|
2630
|
-
|
|
2223
|
+
logger8.info("Loading embedding model:", this.embeddingModelPath);
|
|
2631
2224
|
this.embeddingModel = await this.llama.loadModel({
|
|
2632
2225
|
modelPath: this.embeddingModelPath,
|
|
2226
|
+
// Use the correct path
|
|
2633
2227
|
gpuLayers: 0,
|
|
2634
2228
|
// Embedding models are typically small enough to run on CPU
|
|
2635
2229
|
vocabOnly: false
|
|
@@ -2638,14 +2232,15 @@ var LocalAIManager = class _LocalAIManager {
|
|
|
2638
2232
|
contextSize: this.embeddingModelConfig.contextSize,
|
|
2639
2233
|
batchSize: 512
|
|
2640
2234
|
});
|
|
2641
|
-
|
|
2235
|
+
logger8.success("Embedding model initialized successfully");
|
|
2642
2236
|
}
|
|
2643
2237
|
} catch (error) {
|
|
2644
|
-
|
|
2238
|
+
logger8.error("Embedding initialization failed with details:", {
|
|
2645
2239
|
error: error instanceof Error ? error.message : String(error),
|
|
2646
2240
|
stack: error instanceof Error ? error.stack : void 0,
|
|
2647
2241
|
modelsDir: this.modelsDir,
|
|
2648
2242
|
embeddingModelPath: this.embeddingModelPath
|
|
2243
|
+
// Log the path being used
|
|
2649
2244
|
});
|
|
2650
2245
|
throw error;
|
|
2651
2246
|
}
|
|
@@ -2659,19 +2254,19 @@ var LocalAIManager = class _LocalAIManager {
|
|
|
2659
2254
|
if (!this.embeddingModel || !this.embeddingContext) {
|
|
2660
2255
|
throw new Error("Failed to initialize embedding model");
|
|
2661
2256
|
}
|
|
2662
|
-
|
|
2257
|
+
logger8.info("Generating embedding for text", { textLength: text.length });
|
|
2663
2258
|
const embeddingResult = await this.embeddingContext.getEmbeddingFor(text);
|
|
2664
2259
|
const mutableEmbedding = [...embeddingResult.vector];
|
|
2665
2260
|
const normalizedEmbedding = this.normalizeEmbedding(mutableEmbedding);
|
|
2666
|
-
|
|
2261
|
+
logger8.info("Embedding generation complete", { dimensions: normalizedEmbedding.length });
|
|
2667
2262
|
return normalizedEmbedding;
|
|
2668
2263
|
} catch (error) {
|
|
2669
|
-
|
|
2264
|
+
logger8.error("Embedding generation failed:", {
|
|
2670
2265
|
error: error instanceof Error ? error.message : String(error),
|
|
2671
2266
|
stack: error instanceof Error ? error.stack : void 0,
|
|
2672
2267
|
textLength: text?.length ?? "text is null"
|
|
2673
2268
|
});
|
|
2674
|
-
const zeroDimensions =
|
|
2269
|
+
const zeroDimensions = this.config?.LOCAL_EMBEDDING_DIMENSIONS ? this.config.LOCAL_EMBEDDING_DIMENSIONS : this.embeddingModelConfig.dimensions;
|
|
2675
2270
|
return new Array(zeroDimensions).fill(0);
|
|
2676
2271
|
}
|
|
2677
2272
|
}
|
|
@@ -2698,9 +2293,9 @@ var LocalAIManager = class _LocalAIManager {
|
|
|
2698
2293
|
this.embeddingInitializingPromise = (async () => {
|
|
2699
2294
|
try {
|
|
2700
2295
|
await this.initializeEnvironment();
|
|
2701
|
-
await this.downloadModel(
|
|
2296
|
+
await this.downloadModel(ModelType.TEXT_EMBEDDING);
|
|
2702
2297
|
if (!this.llama) {
|
|
2703
|
-
this.llama = await
|
|
2298
|
+
this.llama = await getLlama();
|
|
2704
2299
|
}
|
|
2705
2300
|
this.embeddingModel = await this.llama.loadModel({
|
|
2706
2301
|
modelPath: this.embeddingModelPath,
|
|
@@ -2713,9 +2308,9 @@ var LocalAIManager = class _LocalAIManager {
|
|
|
2713
2308
|
batchSize: 512
|
|
2714
2309
|
});
|
|
2715
2310
|
this.embeddingInitialized = true;
|
|
2716
|
-
|
|
2311
|
+
logger8.info("Embedding model initialized successfully");
|
|
2717
2312
|
} catch (error) {
|
|
2718
|
-
|
|
2313
|
+
logger8.error("Failed to initialize embedding model:", error);
|
|
2719
2314
|
this.embeddingInitializingPromise = null;
|
|
2720
2315
|
throw error;
|
|
2721
2316
|
}
|
|
@@ -2723,55 +2318,15 @@ var LocalAIManager = class _LocalAIManager {
|
|
|
2723
2318
|
}
|
|
2724
2319
|
await this.embeddingInitializingPromise;
|
|
2725
2320
|
}
|
|
2726
|
-
/**
|
|
2727
|
-
* Asynchronously generates text using StudioLM models based on the specified parameters.
|
|
2728
|
-
*
|
|
2729
|
-
* @param {GenerateTextParams} params - The parameters for generating the text.
|
|
2730
|
-
* @returns {Promise<string>} - A promise that resolves to the generated text.
|
|
2731
|
-
*/
|
|
2732
|
-
async generateTextLMStudio(params) {
|
|
2733
|
-
try {
|
|
2734
|
-
const modelConfig = this.getTextModelSource();
|
|
2735
|
-
logger9.info("generateTextLMStudio called with:", {
|
|
2736
|
-
modelSource: modelConfig.source,
|
|
2737
|
-
modelType: params.modelType,
|
|
2738
|
-
studioLMInitialized: this.studioLMInitialized,
|
|
2739
|
-
studioLMEnabled: process.env.USE_STUDIOLM_TEXT_MODELS === "true"
|
|
2740
|
-
});
|
|
2741
|
-
if (modelConfig.source === "studiolm") {
|
|
2742
|
-
if (process.env.USE_STUDIOLM_TEXT_MODELS !== "true") {
|
|
2743
|
-
logger9.warn(
|
|
2744
|
-
"StudioLM requested but disabled in environment, falling back to local models"
|
|
2745
|
-
);
|
|
2746
|
-
return this.generateText(params);
|
|
2747
|
-
}
|
|
2748
|
-
if (!this.studioLMManager) {
|
|
2749
|
-
logger9.warn("StudioLM manager not initialized, falling back to local models");
|
|
2750
|
-
return this.generateText(params);
|
|
2751
|
-
}
|
|
2752
|
-
if (!this.studioLMInitialized) {
|
|
2753
|
-
logger9.info("StudioLM not initialized, initializing now...");
|
|
2754
|
-
await this.initializeStudioLM();
|
|
2755
|
-
}
|
|
2756
|
-
return await this.studioLMManager.generateText(params, this.studioLMInitialized);
|
|
2757
|
-
}
|
|
2758
|
-
return this.generateText(params);
|
|
2759
|
-
} catch (error) {
|
|
2760
|
-
logger9.error("Text generation with StudioLM failed:", {
|
|
2761
|
-
error: error instanceof Error ? error.message : String(error),
|
|
2762
|
-
stack: error instanceof Error ? error.stack : void 0,
|
|
2763
|
-
modelSource: this.getTextModelSource().source
|
|
2764
|
-
});
|
|
2765
|
-
return this.generateText(params);
|
|
2766
|
-
}
|
|
2767
|
-
}
|
|
2768
2321
|
/**
|
|
2769
2322
|
* Asynchronously generates text based on the provided parameters.
|
|
2770
2323
|
* Now uses lazy initialization for models
|
|
2771
2324
|
*/
|
|
2772
2325
|
async generateText(params) {
|
|
2773
2326
|
try {
|
|
2774
|
-
|
|
2327
|
+
await this.initializeEnvironment();
|
|
2328
|
+
logger8.info("Generating text with model:", params.modelType);
|
|
2329
|
+
if (params.modelType === ModelType.TEXT_LARGE) {
|
|
2775
2330
|
await this.lazyInitMediumModel();
|
|
2776
2331
|
if (!this.mediumModel) {
|
|
2777
2332
|
throw new Error("Medium model initialization failed");
|
|
@@ -2802,15 +2357,15 @@ var LocalAIManager = class _LocalAIManager {
|
|
|
2802
2357
|
if (!this.chatSession) {
|
|
2803
2358
|
throw new Error("Failed to create chat session");
|
|
2804
2359
|
}
|
|
2805
|
-
|
|
2806
|
-
|
|
2360
|
+
logger8.info("Created new chat session for model:", params.modelType);
|
|
2361
|
+
logger8.info("Incoming prompt structure:", {
|
|
2807
2362
|
contextLength: params.prompt.length,
|
|
2808
2363
|
hasAction: params.prompt.includes("action"),
|
|
2809
2364
|
runtime: !!params.runtime,
|
|
2810
2365
|
stopSequences: params.stopSequences
|
|
2811
2366
|
});
|
|
2812
2367
|
const tokens = await this.tokenizerManager.encode(params.prompt, this.activeModelConfig);
|
|
2813
|
-
|
|
2368
|
+
logger8.info("Input tokens:", { count: tokens.length });
|
|
2814
2369
|
const systemMessage = "You are a helpful AI assistant. Respond to the current request only.";
|
|
2815
2370
|
await this.chatSession.prompt(systemMessage, {
|
|
2816
2371
|
maxTokens: 1,
|
|
@@ -2828,19 +2383,19 @@ var LocalAIManager = class _LocalAIManager {
|
|
|
2828
2383
|
presencePenalty: 0.7
|
|
2829
2384
|
}
|
|
2830
2385
|
});
|
|
2831
|
-
|
|
2386
|
+
logger8.info("Raw response structure:", {
|
|
2832
2387
|
responseLength: response.length,
|
|
2833
2388
|
hasAction: response.includes("action"),
|
|
2834
2389
|
hasThinkTag: response.includes("<think>")
|
|
2835
2390
|
});
|
|
2836
2391
|
if (response.includes("<think>")) {
|
|
2837
|
-
|
|
2392
|
+
logger8.info("Cleaning think tags from response");
|
|
2838
2393
|
response = response.replace(/<think>[\s\S]*?<\/think>\n?/g, "");
|
|
2839
|
-
|
|
2394
|
+
logger8.info("Think tags removed from response");
|
|
2840
2395
|
}
|
|
2841
2396
|
return response;
|
|
2842
2397
|
} catch (error) {
|
|
2843
|
-
|
|
2398
|
+
logger8.error("Text generation failed:", error);
|
|
2844
2399
|
throw error;
|
|
2845
2400
|
}
|
|
2846
2401
|
}
|
|
@@ -2854,7 +2409,7 @@ var LocalAIManager = class _LocalAIManager {
|
|
|
2854
2409
|
const dataUrl = `data:${mimeType};base64,${base64}`;
|
|
2855
2410
|
return await this.visionManager.processImage(dataUrl);
|
|
2856
2411
|
} catch (error) {
|
|
2857
|
-
|
|
2412
|
+
logger8.error("Image description failed:", error);
|
|
2858
2413
|
throw error;
|
|
2859
2414
|
}
|
|
2860
2415
|
}
|
|
@@ -2867,7 +2422,7 @@ var LocalAIManager = class _LocalAIManager {
|
|
|
2867
2422
|
const result = await this.transcribeManager.transcribe(audioBuffer);
|
|
2868
2423
|
return result.text;
|
|
2869
2424
|
} catch (error) {
|
|
2870
|
-
|
|
2425
|
+
logger8.error("Audio transcription failed:", {
|
|
2871
2426
|
error: error instanceof Error ? error.message : String(error),
|
|
2872
2427
|
bufferSize: audioBuffer.length
|
|
2873
2428
|
});
|
|
@@ -2882,14 +2437,13 @@ var LocalAIManager = class _LocalAIManager {
|
|
|
2882
2437
|
await this.lazyInitTTS();
|
|
2883
2438
|
return await this.ttsManager.generateSpeech(text);
|
|
2884
2439
|
} catch (error) {
|
|
2885
|
-
|
|
2440
|
+
logger8.error("Speech generation failed:", {
|
|
2886
2441
|
error: error instanceof Error ? error.message : String(error),
|
|
2887
2442
|
textLength: text.length
|
|
2888
2443
|
});
|
|
2889
2444
|
throw error;
|
|
2890
2445
|
}
|
|
2891
2446
|
}
|
|
2892
|
-
// Add public accessor methods
|
|
2893
2447
|
/**
|
|
2894
2448
|
* Returns the TokenizerManager associated with this object.
|
|
2895
2449
|
*
|
|
@@ -2905,41 +2459,6 @@ var LocalAIManager = class _LocalAIManager {
|
|
|
2905
2459
|
getActiveModelConfig() {
|
|
2906
2460
|
return this.activeModelConfig;
|
|
2907
2461
|
}
|
|
2908
|
-
/**
|
|
2909
|
-
* Retrieves the source configuration for the text model based on environment variables and manager existence.
|
|
2910
|
-
* @returns {TextModelConfig} The configuration object containing the text model source and type.
|
|
2911
|
-
*/
|
|
2912
|
-
getTextModelSource() {
|
|
2913
|
-
try {
|
|
2914
|
-
const config = {
|
|
2915
|
-
source: "local",
|
|
2916
|
-
modelType: ModelType2.TEXT_SMALL
|
|
2917
|
-
};
|
|
2918
|
-
if (process.env.USE_STUDIOLM_TEXT_MODELS === "true" && this.studioLMManager) {
|
|
2919
|
-
config.source = "studiolm";
|
|
2920
|
-
}
|
|
2921
|
-
logger9.info("Selected text model source:", config);
|
|
2922
|
-
return config;
|
|
2923
|
-
} catch (error) {
|
|
2924
|
-
logger9.error("Error determining text model source:", error);
|
|
2925
|
-
return { source: "local", modelType: ModelType2.TEXT_SMALL };
|
|
2926
|
-
}
|
|
2927
|
-
}
|
|
2928
|
-
/**
|
|
2929
|
-
* Generic lazy initialization handler for any model type
|
|
2930
|
-
*/
|
|
2931
|
-
async lazyInitialize(modelType, isInitialized, initPromise, initFunction) {
|
|
2932
|
-
if (isInitialized) {
|
|
2933
|
-
return Promise.resolve(null);
|
|
2934
|
-
}
|
|
2935
|
-
if (initPromise) {
|
|
2936
|
-
logger9.info(`Waiting for ${modelType} initialization to complete...`);
|
|
2937
|
-
await initPromise;
|
|
2938
|
-
return Promise.resolve(null);
|
|
2939
|
-
}
|
|
2940
|
-
logger9.info(`Lazy initializing ${modelType}...`);
|
|
2941
|
-
return initFunction();
|
|
2942
|
-
}
|
|
2943
2462
|
/**
|
|
2944
2463
|
* Lazy initialize the small text model
|
|
2945
2464
|
*/
|
|
@@ -2949,12 +2468,13 @@ var LocalAIManager = class _LocalAIManager {
|
|
|
2949
2468
|
this.smallModelInitializingPromise = (async () => {
|
|
2950
2469
|
await this.initializeEnvironment();
|
|
2951
2470
|
await this.checkPlatformCapabilities();
|
|
2952
|
-
await this.downloadModel(
|
|
2471
|
+
await this.downloadModel(ModelType.TEXT_SMALL);
|
|
2953
2472
|
try {
|
|
2954
|
-
this.llama = await
|
|
2473
|
+
this.llama = await getLlama();
|
|
2955
2474
|
const smallModel = await this.llama.loadModel({
|
|
2956
2475
|
gpuLayers: 43,
|
|
2957
2476
|
modelPath: this.modelPath,
|
|
2477
|
+
// Use the potentially overridden path
|
|
2958
2478
|
vocabOnly: false
|
|
2959
2479
|
});
|
|
2960
2480
|
this.smallModel = smallModel;
|
|
@@ -2964,9 +2484,9 @@ var LocalAIManager = class _LocalAIManager {
|
|
|
2964
2484
|
this.ctx = ctx;
|
|
2965
2485
|
this.sequence = void 0;
|
|
2966
2486
|
this.smallModelInitialized = true;
|
|
2967
|
-
|
|
2487
|
+
logger8.info("Small model initialized successfully");
|
|
2968
2488
|
} catch (error) {
|
|
2969
|
-
|
|
2489
|
+
logger8.error("Failed to initialize small model:", error);
|
|
2970
2490
|
this.smallModelInitializingPromise = null;
|
|
2971
2491
|
throw error;
|
|
2972
2492
|
}
|
|
@@ -2981,21 +2501,23 @@ var LocalAIManager = class _LocalAIManager {
|
|
|
2981
2501
|
if (this.mediumModelInitialized) return;
|
|
2982
2502
|
if (!this.mediumModelInitializingPromise) {
|
|
2983
2503
|
this.mediumModelInitializingPromise = (async () => {
|
|
2504
|
+
await this.initializeEnvironment();
|
|
2984
2505
|
if (!this.llama) {
|
|
2985
2506
|
await this.lazyInitSmallModel();
|
|
2986
2507
|
}
|
|
2987
|
-
await this.downloadModel(
|
|
2508
|
+
await this.downloadModel(ModelType.TEXT_LARGE);
|
|
2988
2509
|
try {
|
|
2989
2510
|
const mediumModel = await this.llama.loadModel({
|
|
2990
2511
|
gpuLayers: 43,
|
|
2991
2512
|
modelPath: this.mediumModelPath,
|
|
2513
|
+
// Use the potentially overridden path
|
|
2992
2514
|
vocabOnly: false
|
|
2993
2515
|
});
|
|
2994
2516
|
this.mediumModel = mediumModel;
|
|
2995
2517
|
this.mediumModelInitialized = true;
|
|
2996
|
-
|
|
2518
|
+
logger8.info("Medium model initialized successfully");
|
|
2997
2519
|
} catch (error) {
|
|
2998
|
-
|
|
2520
|
+
logger8.error("Failed to initialize medium model:", error);
|
|
2999
2521
|
this.mediumModelInitializingPromise = null;
|
|
3000
2522
|
throw error;
|
|
3001
2523
|
}
|
|
@@ -3012,9 +2534,9 @@ var LocalAIManager = class _LocalAIManager {
|
|
|
3012
2534
|
this.visionInitializingPromise = (async () => {
|
|
3013
2535
|
try {
|
|
3014
2536
|
this.visionInitialized = true;
|
|
3015
|
-
|
|
2537
|
+
logger8.info("Vision model initialized successfully");
|
|
3016
2538
|
} catch (error) {
|
|
3017
|
-
|
|
2539
|
+
logger8.error("Failed to initialize vision model:", error);
|
|
3018
2540
|
this.visionInitializingPromise = null;
|
|
3019
2541
|
throw error;
|
|
3020
2542
|
}
|
|
@@ -3030,10 +2552,24 @@ var LocalAIManager = class _LocalAIManager {
|
|
|
3030
2552
|
if (!this.transcriptionInitializingPromise) {
|
|
3031
2553
|
this.transcriptionInitializingPromise = (async () => {
|
|
3032
2554
|
try {
|
|
2555
|
+
await this.initializeEnvironment();
|
|
2556
|
+
if (!this.transcribeManager) {
|
|
2557
|
+
this.transcribeManager = TranscribeManager.getInstance(this.cacheDir);
|
|
2558
|
+
}
|
|
2559
|
+
const ffmpegReady = await this.transcribeManager.ensureFFmpeg();
|
|
2560
|
+
if (!ffmpegReady) {
|
|
2561
|
+
logger8.error(
|
|
2562
|
+
"FFmpeg is not available or not configured correctly. Cannot proceed with transcription."
|
|
2563
|
+
);
|
|
2564
|
+
throw new Error(
|
|
2565
|
+
"FFmpeg is required for transcription but is not available. Please see server logs for installation instructions."
|
|
2566
|
+
);
|
|
2567
|
+
}
|
|
3033
2568
|
this.transcriptionInitialized = true;
|
|
3034
|
-
|
|
2569
|
+
logger8.info("Transcription prerequisites (FFmpeg) checked and ready.");
|
|
2570
|
+
logger8.info("Transcription model initialized successfully");
|
|
3035
2571
|
} catch (error) {
|
|
3036
|
-
|
|
2572
|
+
logger8.error("Failed to initialize transcription model:", error);
|
|
3037
2573
|
this.transcriptionInitializingPromise = null;
|
|
3038
2574
|
throw error;
|
|
3039
2575
|
}
|
|
@@ -3049,10 +2585,12 @@ var LocalAIManager = class _LocalAIManager {
|
|
|
3049
2585
|
if (!this.ttsInitializingPromise) {
|
|
3050
2586
|
this.ttsInitializingPromise = (async () => {
|
|
3051
2587
|
try {
|
|
2588
|
+
await this.initializeEnvironment();
|
|
2589
|
+
this.ttsManager = TTSManager.getInstance(this.cacheDir);
|
|
3052
2590
|
this.ttsInitialized = true;
|
|
3053
|
-
|
|
2591
|
+
logger8.info("TTS model initialized successfully");
|
|
3054
2592
|
} catch (error) {
|
|
3055
|
-
|
|
2593
|
+
logger8.error("Failed to lazy initialize TTS components:", error);
|
|
3056
2594
|
this.ttsInitializingPromise = null;
|
|
3057
2595
|
throw error;
|
|
3058
2596
|
}
|
|
@@ -3060,26 +2598,6 @@ var LocalAIManager = class _LocalAIManager {
|
|
|
3060
2598
|
}
|
|
3061
2599
|
await this.ttsInitializingPromise;
|
|
3062
2600
|
}
|
|
3063
|
-
/**
|
|
3064
|
-
* Lazy initialize the StudioLM integration
|
|
3065
|
-
*/
|
|
3066
|
-
async lazyInitStudioLM() {
|
|
3067
|
-
if (this.studioLMInitialized) return;
|
|
3068
|
-
if (!this.studioLMInitializingPromise) {
|
|
3069
|
-
this.studioLMInitializingPromise = (async () => {
|
|
3070
|
-
try {
|
|
3071
|
-
await this.initializeStudioLM();
|
|
3072
|
-
this.studioLMInitialized = true;
|
|
3073
|
-
logger9.info("StudioLM initialized successfully");
|
|
3074
|
-
} catch (error) {
|
|
3075
|
-
logger9.error("Failed to initialize StudioLM:", error);
|
|
3076
|
-
this.studioLMInitializingPromise = null;
|
|
3077
|
-
throw error;
|
|
3078
|
-
}
|
|
3079
|
-
})();
|
|
3080
|
-
}
|
|
3081
|
-
await this.studioLMInitializingPromise;
|
|
3082
|
-
}
|
|
3083
2601
|
};
|
|
3084
2602
|
var localAIManager = LocalAIManager.getInstance();
|
|
3085
2603
|
var localAiPlugin = {
|
|
@@ -3087,10 +2605,11 @@ var localAiPlugin = {
|
|
|
3087
2605
|
description: "Local AI plugin using LLaMA models",
|
|
3088
2606
|
async init() {
|
|
3089
2607
|
try {
|
|
3090
|
-
|
|
3091
|
-
|
|
2608
|
+
logger8.debug("Initializing local-ai plugin environment...");
|
|
2609
|
+
await localAIManager.initializeEnvironment();
|
|
2610
|
+
logger8.success("Local AI plugin configuration validated and initialized");
|
|
3092
2611
|
} catch (error) {
|
|
3093
|
-
|
|
2612
|
+
logger8.error("Plugin initialization failed:", {
|
|
3094
2613
|
error: error instanceof Error ? error.message : String(error),
|
|
3095
2614
|
stack: error instanceof Error ? error.stack : void 0
|
|
3096
2615
|
});
|
|
@@ -3098,60 +2617,44 @@ var localAiPlugin = {
|
|
|
3098
2617
|
}
|
|
3099
2618
|
},
|
|
3100
2619
|
models: {
|
|
3101
|
-
[
|
|
2620
|
+
[ModelType.TEXT_SMALL]: async (runtime, { prompt, stopSequences = [] }) => {
|
|
3102
2621
|
try {
|
|
3103
|
-
|
|
3104
|
-
if (modelConfig.source !== "local") {
|
|
3105
|
-
return await localAIManager.generateTextLMStudio({
|
|
3106
|
-
prompt,
|
|
3107
|
-
stopSequences,
|
|
3108
|
-
runtime,
|
|
3109
|
-
modelType: ModelType2.TEXT_SMALL
|
|
3110
|
-
});
|
|
3111
|
-
}
|
|
2622
|
+
await localAIManager.initializeEnvironment();
|
|
3112
2623
|
return await localAIManager.generateText({
|
|
3113
2624
|
prompt,
|
|
3114
2625
|
stopSequences,
|
|
3115
2626
|
runtime,
|
|
3116
|
-
modelType:
|
|
2627
|
+
modelType: ModelType.TEXT_SMALL
|
|
3117
2628
|
});
|
|
3118
2629
|
} catch (error) {
|
|
3119
|
-
|
|
2630
|
+
logger8.error("Error in TEXT_SMALL handler:", error);
|
|
3120
2631
|
throw error;
|
|
3121
2632
|
}
|
|
3122
2633
|
},
|
|
3123
|
-
[
|
|
2634
|
+
[ModelType.TEXT_LARGE]: async (runtime, { prompt, stopSequences = [] }) => {
|
|
3124
2635
|
try {
|
|
3125
|
-
|
|
3126
|
-
if (modelConfig.source !== "local") {
|
|
3127
|
-
return await localAIManager.generateTextLMStudio({
|
|
3128
|
-
prompt,
|
|
3129
|
-
stopSequences,
|
|
3130
|
-
runtime,
|
|
3131
|
-
modelType: ModelType2.TEXT_LARGE
|
|
3132
|
-
});
|
|
3133
|
-
}
|
|
2636
|
+
await localAIManager.initializeEnvironment();
|
|
3134
2637
|
return await localAIManager.generateText({
|
|
3135
2638
|
prompt,
|
|
3136
2639
|
stopSequences,
|
|
3137
2640
|
runtime,
|
|
3138
|
-
modelType:
|
|
2641
|
+
modelType: ModelType.TEXT_LARGE
|
|
3139
2642
|
});
|
|
3140
2643
|
} catch (error) {
|
|
3141
|
-
|
|
2644
|
+
logger8.error("Error in TEXT_LARGE handler:", error);
|
|
3142
2645
|
throw error;
|
|
3143
2646
|
}
|
|
3144
2647
|
},
|
|
3145
|
-
[
|
|
2648
|
+
[ModelType.TEXT_EMBEDDING]: async (_runtime, params) => {
|
|
3146
2649
|
const text = params?.text;
|
|
3147
2650
|
try {
|
|
3148
2651
|
if (!text) {
|
|
3149
|
-
|
|
2652
|
+
logger8.debug("Null or empty text input for embedding, returning zero vector");
|
|
3150
2653
|
return new Array(384).fill(0);
|
|
3151
2654
|
}
|
|
3152
2655
|
return await localAIManager.generateEmbedding(text);
|
|
3153
2656
|
} catch (error) {
|
|
3154
|
-
|
|
2657
|
+
logger8.error("Error in TEXT_EMBEDDING handler:", {
|
|
3155
2658
|
error: error instanceof Error ? error.message : String(error),
|
|
3156
2659
|
fullText: text,
|
|
3157
2660
|
textType: typeof text,
|
|
@@ -3160,9 +2663,10 @@ var localAiPlugin = {
|
|
|
3160
2663
|
return new Array(384).fill(0);
|
|
3161
2664
|
}
|
|
3162
2665
|
},
|
|
3163
|
-
[
|
|
2666
|
+
[ModelType.OBJECT_SMALL]: async (runtime, params) => {
|
|
3164
2667
|
try {
|
|
3165
|
-
|
|
2668
|
+
await localAIManager.initializeEnvironment();
|
|
2669
|
+
logger8.info("OBJECT_SMALL handler - Processing request:", {
|
|
3166
2670
|
prompt: params.prompt,
|
|
3167
2671
|
hasSchema: !!params.schema,
|
|
3168
2672
|
temperature: params.temperature
|
|
@@ -3171,23 +2675,12 @@ var localAiPlugin = {
|
|
|
3171
2675
|
if (!jsonPrompt.includes("```json") && !jsonPrompt.includes("respond with valid JSON")) {
|
|
3172
2676
|
jsonPrompt += "\nPlease respond with valid JSON only, without any explanations, markdown formatting, or additional text.";
|
|
3173
2677
|
}
|
|
3174
|
-
const
|
|
3175
|
-
|
|
3176
|
-
|
|
3177
|
-
|
|
3178
|
-
|
|
3179
|
-
|
|
3180
|
-
runtime,
|
|
3181
|
-
modelType: ModelType2.TEXT_SMALL
|
|
3182
|
-
});
|
|
3183
|
-
} else {
|
|
3184
|
-
textResponse = await localAIManager.generateText({
|
|
3185
|
-
prompt: jsonPrompt,
|
|
3186
|
-
stopSequences: params.stopSequences,
|
|
3187
|
-
runtime,
|
|
3188
|
-
modelType: ModelType2.TEXT_SMALL
|
|
3189
|
-
});
|
|
3190
|
-
}
|
|
2678
|
+
const textResponse = await localAIManager.generateText({
|
|
2679
|
+
prompt: jsonPrompt,
|
|
2680
|
+
stopSequences: params.stopSequences,
|
|
2681
|
+
runtime,
|
|
2682
|
+
modelType: ModelType.TEXT_SMALL
|
|
2683
|
+
});
|
|
3191
2684
|
try {
|
|
3192
2685
|
const extractJSON = (text) => {
|
|
3193
2686
|
const jsonBlockRegex = /```(?:json)?\s*([\s\S]*?)\s*```/;
|
|
@@ -3203,17 +2696,17 @@ var localAiPlugin = {
|
|
|
3203
2696
|
return text.trim();
|
|
3204
2697
|
};
|
|
3205
2698
|
const extractedJsonText = extractJSON(textResponse);
|
|
3206
|
-
|
|
2699
|
+
logger8.debug("Extracted JSON text:", extractedJsonText);
|
|
3207
2700
|
let jsonObject;
|
|
3208
2701
|
try {
|
|
3209
2702
|
jsonObject = JSON.parse(extractedJsonText);
|
|
3210
2703
|
} catch (parseError) {
|
|
3211
|
-
|
|
2704
|
+
logger8.debug("Initial JSON parse failed, attempting to fix common issues");
|
|
3212
2705
|
const fixedJson = extractedJsonText.replace(/:\s*"([^"]*)(?:\n)([^"]*)"/g, ': "$1\\n$2"').replace(/"([^"]*?)[^a-zA-Z0-9\s\.,;:\-_\(\)"'\[\]{}]([^"]*?)"/g, '"$1$2"').replace(/(\s*)(\w+)(\s*):/g, '$1"$2"$3:').replace(/,(\s*[\]}])/g, "$1");
|
|
3213
2706
|
try {
|
|
3214
2707
|
jsonObject = JSON.parse(fixedJson);
|
|
3215
2708
|
} catch (finalError) {
|
|
3216
|
-
|
|
2709
|
+
logger8.error("Failed to parse JSON after fixing:", finalError);
|
|
3217
2710
|
throw new Error("Invalid JSON returned from model");
|
|
3218
2711
|
}
|
|
3219
2712
|
}
|
|
@@ -3225,23 +2718,24 @@ var localAiPlugin = {
|
|
|
3225
2718
|
}
|
|
3226
2719
|
}
|
|
3227
2720
|
} catch (schemaError) {
|
|
3228
|
-
|
|
2721
|
+
logger8.error("Schema validation failed:", schemaError);
|
|
3229
2722
|
}
|
|
3230
2723
|
}
|
|
3231
2724
|
return jsonObject;
|
|
3232
2725
|
} catch (parseError) {
|
|
3233
|
-
|
|
3234
|
-
|
|
2726
|
+
logger8.error("Failed to parse JSON:", parseError);
|
|
2727
|
+
logger8.error("Raw response:", textResponse);
|
|
3235
2728
|
throw new Error("Invalid JSON returned from model");
|
|
3236
2729
|
}
|
|
3237
2730
|
} catch (error) {
|
|
3238
|
-
|
|
2731
|
+
logger8.error("Error in OBJECT_SMALL handler:", error);
|
|
3239
2732
|
throw error;
|
|
3240
2733
|
}
|
|
3241
2734
|
},
|
|
3242
|
-
[
|
|
2735
|
+
[ModelType.OBJECT_LARGE]: async (runtime, params) => {
|
|
3243
2736
|
try {
|
|
3244
|
-
|
|
2737
|
+
await localAIManager.initializeEnvironment();
|
|
2738
|
+
logger8.info("OBJECT_LARGE handler - Processing request:", {
|
|
3245
2739
|
prompt: params.prompt,
|
|
3246
2740
|
hasSchema: !!params.schema,
|
|
3247
2741
|
temperature: params.temperature
|
|
@@ -3250,23 +2744,12 @@ var localAiPlugin = {
|
|
|
3250
2744
|
if (!jsonPrompt.includes("```json") && !jsonPrompt.includes("respond with valid JSON")) {
|
|
3251
2745
|
jsonPrompt += "\nPlease respond with valid JSON only, without any explanations, markdown formatting, or additional text.";
|
|
3252
2746
|
}
|
|
3253
|
-
const
|
|
3254
|
-
|
|
3255
|
-
|
|
3256
|
-
|
|
3257
|
-
|
|
3258
|
-
|
|
3259
|
-
runtime,
|
|
3260
|
-
modelType: ModelType2.TEXT_LARGE
|
|
3261
|
-
});
|
|
3262
|
-
} else {
|
|
3263
|
-
textResponse = await localAIManager.generateText({
|
|
3264
|
-
prompt: jsonPrompt,
|
|
3265
|
-
stopSequences: params.stopSequences,
|
|
3266
|
-
runtime,
|
|
3267
|
-
modelType: ModelType2.TEXT_LARGE
|
|
3268
|
-
});
|
|
3269
|
-
}
|
|
2747
|
+
const textResponse = await localAIManager.generateText({
|
|
2748
|
+
prompt: jsonPrompt,
|
|
2749
|
+
stopSequences: params.stopSequences,
|
|
2750
|
+
runtime,
|
|
2751
|
+
modelType: ModelType.TEXT_LARGE
|
|
2752
|
+
});
|
|
3270
2753
|
try {
|
|
3271
2754
|
const extractJSON = (text) => {
|
|
3272
2755
|
const jsonBlockRegex = /```(?:json)?\s*([\s\S]*?)\s*```/;
|
|
@@ -3286,17 +2769,17 @@ var localAiPlugin = {
|
|
|
3286
2769
|
};
|
|
3287
2770
|
const extractedJsonText = extractJSON(textResponse);
|
|
3288
2771
|
const cleanedJsonText = cleanupJSON(extractedJsonText);
|
|
3289
|
-
|
|
2772
|
+
logger8.debug("Extracted JSON text:", cleanedJsonText);
|
|
3290
2773
|
let jsonObject;
|
|
3291
2774
|
try {
|
|
3292
2775
|
jsonObject = JSON.parse(cleanedJsonText);
|
|
3293
2776
|
} catch (parseError) {
|
|
3294
|
-
|
|
2777
|
+
logger8.debug("Initial JSON parse failed, attempting to fix common issues");
|
|
3295
2778
|
const fixedJson = cleanedJsonText.replace(/:\s*"([^"]*)(?:\n)([^"]*)"/g, ': "$1\\n$2"').replace(/"([^"]*?)[^a-zA-Z0-9\s\.,;:\-_\(\)"'\[\]{}]([^"]*?)"/g, '"$1$2"').replace(/(\s*)(\w+)(\s*):/g, '$1"$2"$3:').replace(/,(\s*[\]}])/g, "$1");
|
|
3296
2779
|
try {
|
|
3297
2780
|
jsonObject = JSON.parse(fixedJson);
|
|
3298
2781
|
} catch (finalError) {
|
|
3299
|
-
|
|
2782
|
+
logger8.error("Failed to parse JSON after fixing:", finalError);
|
|
3300
2783
|
throw new Error("Invalid JSON returned from model");
|
|
3301
2784
|
}
|
|
3302
2785
|
}
|
|
@@ -3308,43 +2791,43 @@ var localAiPlugin = {
|
|
|
3308
2791
|
}
|
|
3309
2792
|
}
|
|
3310
2793
|
} catch (schemaError) {
|
|
3311
|
-
|
|
2794
|
+
logger8.error("Schema validation failed:", schemaError);
|
|
3312
2795
|
}
|
|
3313
2796
|
}
|
|
3314
2797
|
return jsonObject;
|
|
3315
2798
|
} catch (parseError) {
|
|
3316
|
-
|
|
3317
|
-
|
|
2799
|
+
logger8.error("Failed to parse JSON:", parseError);
|
|
2800
|
+
logger8.error("Raw response:", textResponse);
|
|
3318
2801
|
throw new Error("Invalid JSON returned from model");
|
|
3319
2802
|
}
|
|
3320
2803
|
} catch (error) {
|
|
3321
|
-
|
|
2804
|
+
logger8.error("Error in OBJECT_LARGE handler:", error);
|
|
3322
2805
|
throw error;
|
|
3323
2806
|
}
|
|
3324
2807
|
},
|
|
3325
|
-
[
|
|
2808
|
+
[ModelType.TEXT_TOKENIZER_ENCODE]: async (_runtime, { text }) => {
|
|
3326
2809
|
try {
|
|
3327
2810
|
const manager = localAIManager.getTokenizerManager();
|
|
3328
2811
|
const config = localAIManager.getActiveModelConfig();
|
|
3329
2812
|
return await manager.encode(text, config);
|
|
3330
2813
|
} catch (error) {
|
|
3331
|
-
|
|
2814
|
+
logger8.error("Error in TEXT_TOKENIZER_ENCODE handler:", error);
|
|
3332
2815
|
throw error;
|
|
3333
2816
|
}
|
|
3334
2817
|
},
|
|
3335
|
-
[
|
|
2818
|
+
[ModelType.TEXT_TOKENIZER_DECODE]: async (_runtime, { tokens }) => {
|
|
3336
2819
|
try {
|
|
3337
2820
|
const manager = localAIManager.getTokenizerManager();
|
|
3338
2821
|
const config = localAIManager.getActiveModelConfig();
|
|
3339
2822
|
return await manager.decode(tokens, config);
|
|
3340
2823
|
} catch (error) {
|
|
3341
|
-
|
|
2824
|
+
logger8.error("Error in TEXT_TOKENIZER_DECODE handler:", error);
|
|
3342
2825
|
throw error;
|
|
3343
2826
|
}
|
|
3344
2827
|
},
|
|
3345
|
-
[
|
|
2828
|
+
[ModelType.IMAGE_DESCRIPTION]: async (_runtime, imageUrl) => {
|
|
3346
2829
|
try {
|
|
3347
|
-
|
|
2830
|
+
logger8.info("Processing image from URL:", imageUrl);
|
|
3348
2831
|
const response = await fetch(imageUrl);
|
|
3349
2832
|
if (!response.ok) {
|
|
3350
2833
|
throw new Error(`Failed to fetch image: ${response.statusText}`);
|
|
@@ -3353,32 +2836,32 @@ var localAiPlugin = {
|
|
|
3353
2836
|
const mimeType = response.headers.get("content-type") || "image/jpeg";
|
|
3354
2837
|
return await localAIManager.describeImage(buffer, mimeType);
|
|
3355
2838
|
} catch (error) {
|
|
3356
|
-
|
|
2839
|
+
logger8.error("Error in IMAGE_DESCRIPTION handler:", {
|
|
3357
2840
|
error: error instanceof Error ? error.message : String(error),
|
|
3358
2841
|
imageUrl
|
|
3359
2842
|
});
|
|
3360
2843
|
throw error;
|
|
3361
2844
|
}
|
|
3362
2845
|
},
|
|
3363
|
-
[
|
|
2846
|
+
[ModelType.TRANSCRIPTION]: async (_runtime, audioBuffer) => {
|
|
3364
2847
|
try {
|
|
3365
|
-
|
|
2848
|
+
logger8.info("Processing audio transcription:", {
|
|
3366
2849
|
bufferSize: audioBuffer.length
|
|
3367
2850
|
});
|
|
3368
2851
|
return await localAIManager.transcribeAudio(audioBuffer);
|
|
3369
2852
|
} catch (error) {
|
|
3370
|
-
|
|
2853
|
+
logger8.error("Error in TRANSCRIPTION handler:", {
|
|
3371
2854
|
error: error instanceof Error ? error.message : String(error),
|
|
3372
2855
|
bufferSize: audioBuffer.length
|
|
3373
2856
|
});
|
|
3374
2857
|
throw error;
|
|
3375
2858
|
}
|
|
3376
2859
|
},
|
|
3377
|
-
[
|
|
2860
|
+
[ModelType.TEXT_TO_SPEECH]: async (_runtime, text) => {
|
|
3378
2861
|
try {
|
|
3379
2862
|
return await localAIManager.generateSpeech(text);
|
|
3380
2863
|
} catch (error) {
|
|
3381
|
-
|
|
2864
|
+
logger8.error("Error in TEXT_TO_SPEECH handler:", {
|
|
3382
2865
|
error: error instanceof Error ? error.message : String(error),
|
|
3383
2866
|
textLength: text.length
|
|
3384
2867
|
});
|
|
@@ -3394,21 +2877,21 @@ var localAiPlugin = {
|
|
|
3394
2877
|
name: "local_ai_test_initialization",
|
|
3395
2878
|
fn: async (runtime) => {
|
|
3396
2879
|
try {
|
|
3397
|
-
|
|
3398
|
-
const result = await runtime.useModel(
|
|
2880
|
+
logger8.info("Starting initialization test");
|
|
2881
|
+
const result = await runtime.useModel(ModelType.TEXT_SMALL, {
|
|
3399
2882
|
prompt: "Debug Mode: Test initialization. Respond with 'Initialization successful' if you can read this.",
|
|
3400
2883
|
stopSequences: []
|
|
3401
2884
|
});
|
|
3402
|
-
|
|
2885
|
+
logger8.info("Model response:", result);
|
|
3403
2886
|
if (!result || typeof result !== "string") {
|
|
3404
2887
|
throw new Error("Invalid response from model");
|
|
3405
2888
|
}
|
|
3406
2889
|
if (!result.includes("successful")) {
|
|
3407
2890
|
throw new Error("Model response does not indicate success");
|
|
3408
2891
|
}
|
|
3409
|
-
|
|
2892
|
+
logger8.success("Initialization test completed successfully");
|
|
3410
2893
|
} catch (error) {
|
|
3411
|
-
|
|
2894
|
+
logger8.error("Initialization test failed:", {
|
|
3412
2895
|
error: error instanceof Error ? error.message : String(error),
|
|
3413
2896
|
stack: error instanceof Error ? error.stack : void 0
|
|
3414
2897
|
});
|
|
@@ -3420,21 +2903,21 @@ var localAiPlugin = {
|
|
|
3420
2903
|
name: "local_ai_test_text_large",
|
|
3421
2904
|
fn: async (runtime) => {
|
|
3422
2905
|
try {
|
|
3423
|
-
|
|
3424
|
-
const result = await runtime.useModel(
|
|
2906
|
+
logger8.info("Starting TEXT_LARGE model test");
|
|
2907
|
+
const result = await runtime.useModel(ModelType.TEXT_LARGE, {
|
|
3425
2908
|
prompt: "Debug Mode: Generate a one-sentence response about artificial intelligence.",
|
|
3426
2909
|
stopSequences: []
|
|
3427
2910
|
});
|
|
3428
|
-
|
|
2911
|
+
logger8.info("Large model response:", result);
|
|
3429
2912
|
if (!result || typeof result !== "string") {
|
|
3430
2913
|
throw new Error("Invalid response from large model");
|
|
3431
2914
|
}
|
|
3432
2915
|
if (result.length < 10) {
|
|
3433
2916
|
throw new Error("Response too short, possible model failure");
|
|
3434
2917
|
}
|
|
3435
|
-
|
|
2918
|
+
logger8.success("TEXT_LARGE test completed successfully");
|
|
3436
2919
|
} catch (error) {
|
|
3437
|
-
|
|
2920
|
+
logger8.error("TEXT_LARGE test failed:", {
|
|
3438
2921
|
error: error instanceof Error ? error.message : String(error),
|
|
3439
2922
|
stack: error instanceof Error ? error.stack : void 0
|
|
3440
2923
|
});
|
|
@@ -3446,11 +2929,11 @@ var localAiPlugin = {
|
|
|
3446
2929
|
name: "local_ai_test_text_embedding",
|
|
3447
2930
|
fn: async (runtime) => {
|
|
3448
2931
|
try {
|
|
3449
|
-
|
|
3450
|
-
const embedding = await runtime.useModel(
|
|
2932
|
+
logger8.info("Starting TEXT_EMBEDDING test");
|
|
2933
|
+
const embedding = await runtime.useModel(ModelType.TEXT_EMBEDDING, {
|
|
3451
2934
|
text: "This is a test of the text embedding model."
|
|
3452
2935
|
});
|
|
3453
|
-
|
|
2936
|
+
logger8.info("Embedding generated with dimensions:", embedding.length);
|
|
3454
2937
|
if (!Array.isArray(embedding)) {
|
|
3455
2938
|
throw new Error("Embedding is not an array");
|
|
3456
2939
|
}
|
|
@@ -3460,13 +2943,13 @@ var localAiPlugin = {
|
|
|
3460
2943
|
if (embedding.some((val) => typeof val !== "number")) {
|
|
3461
2944
|
throw new Error("Embedding contains non-numeric values");
|
|
3462
2945
|
}
|
|
3463
|
-
const nullEmbedding = await runtime.useModel(
|
|
2946
|
+
const nullEmbedding = await runtime.useModel(ModelType.TEXT_EMBEDDING, null);
|
|
3464
2947
|
if (!Array.isArray(nullEmbedding) || nullEmbedding.some((val) => val !== 0)) {
|
|
3465
2948
|
throw new Error("Null input did not return zero vector");
|
|
3466
2949
|
}
|
|
3467
|
-
|
|
2950
|
+
logger8.success("TEXT_EMBEDDING test completed successfully");
|
|
3468
2951
|
} catch (error) {
|
|
3469
|
-
|
|
2952
|
+
logger8.error("TEXT_EMBEDDING test failed:", {
|
|
3470
2953
|
error: error instanceof Error ? error.message : String(error),
|
|
3471
2954
|
stack: error instanceof Error ? error.stack : void 0
|
|
3472
2955
|
});
|
|
@@ -3478,10 +2961,10 @@ var localAiPlugin = {
|
|
|
3478
2961
|
name: "local_ai_test_tokenizer_encode",
|
|
3479
2962
|
fn: async (runtime) => {
|
|
3480
2963
|
try {
|
|
3481
|
-
|
|
2964
|
+
logger8.info("Starting TEXT_TOKENIZER_ENCODE test");
|
|
3482
2965
|
const text = "Hello tokenizer test!";
|
|
3483
|
-
const tokens = await runtime.useModel(
|
|
3484
|
-
|
|
2966
|
+
const tokens = await runtime.useModel(ModelType.TEXT_TOKENIZER_ENCODE, { text });
|
|
2967
|
+
logger8.info("Encoded tokens:", { count: tokens.length });
|
|
3485
2968
|
if (!Array.isArray(tokens)) {
|
|
3486
2969
|
throw new Error("Tokens output is not an array");
|
|
3487
2970
|
}
|
|
@@ -3491,9 +2974,9 @@ var localAiPlugin = {
|
|
|
3491
2974
|
if (tokens.some((token) => !Number.isInteger(token))) {
|
|
3492
2975
|
throw new Error("Tokens contain non-integer values");
|
|
3493
2976
|
}
|
|
3494
|
-
|
|
2977
|
+
logger8.success("TEXT_TOKENIZER_ENCODE test completed successfully");
|
|
3495
2978
|
} catch (error) {
|
|
3496
|
-
|
|
2979
|
+
logger8.error("TEXT_TOKENIZER_ENCODE test failed:", {
|
|
3497
2980
|
error: error instanceof Error ? error.message : String(error),
|
|
3498
2981
|
stack: error instanceof Error ? error.stack : void 0
|
|
3499
2982
|
});
|
|
@@ -3505,24 +2988,24 @@ var localAiPlugin = {
|
|
|
3505
2988
|
name: "local_ai_test_tokenizer_decode",
|
|
3506
2989
|
fn: async (runtime) => {
|
|
3507
2990
|
try {
|
|
3508
|
-
|
|
2991
|
+
logger8.info("Starting TEXT_TOKENIZER_DECODE test");
|
|
3509
2992
|
const originalText = "Hello tokenizer test!";
|
|
3510
|
-
const tokens = await runtime.useModel(
|
|
2993
|
+
const tokens = await runtime.useModel(ModelType.TEXT_TOKENIZER_ENCODE, {
|
|
3511
2994
|
text: originalText
|
|
3512
2995
|
});
|
|
3513
|
-
const decodedText = await runtime.useModel(
|
|
2996
|
+
const decodedText = await runtime.useModel(ModelType.TEXT_TOKENIZER_DECODE, {
|
|
3514
2997
|
tokens
|
|
3515
2998
|
});
|
|
3516
|
-
|
|
2999
|
+
logger8.info("Round trip tokenization:", {
|
|
3517
3000
|
original: originalText,
|
|
3518
3001
|
decoded: decodedText
|
|
3519
3002
|
});
|
|
3520
3003
|
if (typeof decodedText !== "string") {
|
|
3521
3004
|
throw new Error("Decoded output is not a string");
|
|
3522
3005
|
}
|
|
3523
|
-
|
|
3006
|
+
logger8.success("TEXT_TOKENIZER_DECODE test completed successfully");
|
|
3524
3007
|
} catch (error) {
|
|
3525
|
-
|
|
3008
|
+
logger8.error("TEXT_TOKENIZER_DECODE test failed:", {
|
|
3526
3009
|
error: error instanceof Error ? error.message : String(error),
|
|
3527
3010
|
stack: error instanceof Error ? error.stack : void 0
|
|
3528
3011
|
});
|
|
@@ -3534,10 +3017,10 @@ var localAiPlugin = {
|
|
|
3534
3017
|
name: "local_ai_test_image_description",
|
|
3535
3018
|
fn: async (runtime) => {
|
|
3536
3019
|
try {
|
|
3537
|
-
|
|
3020
|
+
logger8.info("Starting IMAGE_DESCRIPTION test");
|
|
3538
3021
|
const imageUrl = "https://raw.githubusercontent.com/microsoft/FLAML/main/website/static/img/flaml.png";
|
|
3539
|
-
const result = await runtime.useModel(
|
|
3540
|
-
|
|
3022
|
+
const result = await runtime.useModel(ModelType.IMAGE_DESCRIPTION, imageUrl);
|
|
3023
|
+
logger8.info("Image description result:", result);
|
|
3541
3024
|
if (!result || typeof result !== "object") {
|
|
3542
3025
|
throw new Error("Invalid response format");
|
|
3543
3026
|
}
|
|
@@ -3547,9 +3030,9 @@ var localAiPlugin = {
|
|
|
3547
3030
|
if (typeof result.title !== "string" || typeof result.description !== "string") {
|
|
3548
3031
|
throw new Error("Title or description is not a string");
|
|
3549
3032
|
}
|
|
3550
|
-
|
|
3033
|
+
logger8.success("IMAGE_DESCRIPTION test completed successfully");
|
|
3551
3034
|
} catch (error) {
|
|
3552
|
-
|
|
3035
|
+
logger8.error("IMAGE_DESCRIPTION test failed:", {
|
|
3553
3036
|
error: error instanceof Error ? error.message : String(error),
|
|
3554
3037
|
stack: error instanceof Error ? error.stack : void 0
|
|
3555
3038
|
});
|
|
@@ -3561,7 +3044,7 @@ var localAiPlugin = {
|
|
|
3561
3044
|
name: "local_ai_test_transcription",
|
|
3562
3045
|
fn: async (runtime) => {
|
|
3563
3046
|
try {
|
|
3564
|
-
|
|
3047
|
+
logger8.info("Starting TRANSCRIPTION test");
|
|
3565
3048
|
const audioData = new Uint8Array([
|
|
3566
3049
|
82,
|
|
3567
3050
|
73,
|
|
@@ -3585,14 +3068,14 @@ var localAiPlugin = {
|
|
|
3585
3068
|
// "fmt "
|
|
3586
3069
|
]);
|
|
3587
3070
|
const audioBuffer = Buffer.from(audioData);
|
|
3588
|
-
const transcription = await runtime.useModel(
|
|
3589
|
-
|
|
3071
|
+
const transcription = await runtime.useModel(ModelType.TRANSCRIPTION, audioBuffer);
|
|
3072
|
+
logger8.info("Transcription result:", transcription);
|
|
3590
3073
|
if (typeof transcription !== "string") {
|
|
3591
3074
|
throw new Error("Transcription result is not a string");
|
|
3592
3075
|
}
|
|
3593
|
-
|
|
3076
|
+
logger8.success("TRANSCRIPTION test completed successfully");
|
|
3594
3077
|
} catch (error) {
|
|
3595
|
-
|
|
3078
|
+
logger8.error("TRANSCRIPTION test failed:", {
|
|
3596
3079
|
error: error instanceof Error ? error.message : String(error),
|
|
3597
3080
|
stack: error instanceof Error ? error.stack : void 0
|
|
3598
3081
|
});
|
|
@@ -3604,9 +3087,9 @@ var localAiPlugin = {
|
|
|
3604
3087
|
name: "local_ai_test_text_to_speech",
|
|
3605
3088
|
fn: async (runtime) => {
|
|
3606
3089
|
try {
|
|
3607
|
-
|
|
3090
|
+
logger8.info("Starting TEXT_TO_SPEECH test");
|
|
3608
3091
|
const testText = "This is a test of the text to speech system.";
|
|
3609
|
-
const audioStream = await runtime.useModel(
|
|
3092
|
+
const audioStream = await runtime.useModel(ModelType.TEXT_TO_SPEECH, testText);
|
|
3610
3093
|
if (!(audioStream instanceof Readable2)) {
|
|
3611
3094
|
throw new Error("TTS output is not a readable stream");
|
|
3612
3095
|
}
|
|
@@ -3624,9 +3107,9 @@ var localAiPlugin = {
|
|
|
3624
3107
|
});
|
|
3625
3108
|
audioStream.on("error", reject);
|
|
3626
3109
|
});
|
|
3627
|
-
|
|
3110
|
+
logger8.success("TEXT_TO_SPEECH test completed successfully");
|
|
3628
3111
|
} catch (error) {
|
|
3629
|
-
|
|
3112
|
+
logger8.error("TEXT_TO_SPEECH test failed:", {
|
|
3630
3113
|
error: error instanceof Error ? error.message : String(error),
|
|
3631
3114
|
stack: error instanceof Error ? error.stack : void 0
|
|
3632
3115
|
});
|