@elizaos/plugin-local-ai 1.0.0-beta.49 → 1.0.0-beta.51
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -44
- package/dist/index.d.ts +9 -0
- package/dist/index.js +333 -139
- package/dist/index.js.map +1 -1
- package/package.json +14 -13
package/dist/index.js
CHANGED
|
@@ -3,7 +3,10 @@ import fs5 from "node:fs";
|
|
|
3
3
|
import os3 from "node:os";
|
|
4
4
|
import path5 from "node:path";
|
|
5
5
|
import { Readable as Readable2 } from "node:stream";
|
|
6
|
-
import {
|
|
6
|
+
import {
|
|
7
|
+
ModelType,
|
|
8
|
+
logger as logger8
|
|
9
|
+
} from "@elizaos/core";
|
|
7
10
|
import {
|
|
8
11
|
LlamaChatSession,
|
|
9
12
|
getLlama
|
|
@@ -250,7 +253,10 @@ var DownloadManager = class _DownloadManager {
|
|
|
250
253
|
reject(new Error(`Failed to download: ${response.statusCode}`));
|
|
251
254
|
return;
|
|
252
255
|
}
|
|
253
|
-
const totalSize = Number.parseInt(
|
|
256
|
+
const totalSize = Number.parseInt(
|
|
257
|
+
response.headers["content-length"] || "0",
|
|
258
|
+
10
|
|
259
|
+
);
|
|
254
260
|
let downloadedSize = 0;
|
|
255
261
|
let lastLoggedPercent = 0;
|
|
256
262
|
const barLength = 30;
|
|
@@ -261,9 +267,13 @@ var DownloadManager = class _DownloadManager {
|
|
|
261
267
|
downloadedSize += chunk.length;
|
|
262
268
|
const percent = Math.round(downloadedSize / totalSize * 100);
|
|
263
269
|
if (percent >= lastLoggedPercent + 5) {
|
|
264
|
-
const filledLength = Math.floor(
|
|
270
|
+
const filledLength = Math.floor(
|
|
271
|
+
downloadedSize / totalSize * barLength
|
|
272
|
+
);
|
|
265
273
|
const progressBar = "\u25B0".repeat(filledLength) + "\u25B1".repeat(barLength - filledLength);
|
|
266
|
-
logger2.info(
|
|
274
|
+
logger2.info(
|
|
275
|
+
`Downloading ${fileName}: ${progressBar} ${percent}%`
|
|
276
|
+
);
|
|
267
277
|
lastLoggedPercent = percent;
|
|
268
278
|
}
|
|
269
279
|
});
|
|
@@ -278,18 +288,24 @@ var DownloadManager = class _DownloadManager {
|
|
|
278
288
|
fs.mkdirSync(destDir, { recursive: true });
|
|
279
289
|
}
|
|
280
290
|
if (!fs.existsSync(tempPath)) {
|
|
281
|
-
reject(
|
|
291
|
+
reject(
|
|
292
|
+
new Error(`Temporary file ${tempPath} does not exist`)
|
|
293
|
+
);
|
|
282
294
|
return;
|
|
283
295
|
}
|
|
284
296
|
if (fs.existsSync(destPath)) {
|
|
285
297
|
try {
|
|
286
298
|
const backupPath = `${destPath}.bak`;
|
|
287
299
|
fs.renameSync(destPath, backupPath);
|
|
288
|
-
logger2.info(
|
|
300
|
+
logger2.info(
|
|
301
|
+
`Created backup of existing file: ${backupPath}`
|
|
302
|
+
);
|
|
289
303
|
fs.renameSync(tempPath, destPath);
|
|
290
304
|
if (fs.existsSync(backupPath)) {
|
|
291
305
|
fs.unlinkSync(backupPath);
|
|
292
|
-
logger2.info(
|
|
306
|
+
logger2.info(
|
|
307
|
+
`Removed backup file after successful update: ${backupPath}`
|
|
308
|
+
);
|
|
293
309
|
}
|
|
294
310
|
} catch (moveErr) {
|
|
295
311
|
logger2.error(
|
|
@@ -299,7 +315,9 @@ var DownloadManager = class _DownloadManager {
|
|
|
299
315
|
if (fs.existsSync(backupPath)) {
|
|
300
316
|
try {
|
|
301
317
|
fs.renameSync(backupPath, destPath);
|
|
302
|
-
logger2.info(
|
|
318
|
+
logger2.info(
|
|
319
|
+
`Restored from backup after failed update: ${backupPath}`
|
|
320
|
+
);
|
|
303
321
|
} catch (restoreErr) {
|
|
304
322
|
logger2.error(
|
|
305
323
|
`Failed to restore from backup: ${restoreErr instanceof Error ? restoreErr.message : String(restoreErr)}`
|
|
@@ -321,7 +339,9 @@ var DownloadManager = class _DownloadManager {
|
|
|
321
339
|
} else {
|
|
322
340
|
fs.renameSync(tempPath, destPath);
|
|
323
341
|
}
|
|
324
|
-
logger2.success(
|
|
342
|
+
logger2.success(
|
|
343
|
+
`Download of ${fileName} completed successfully`
|
|
344
|
+
);
|
|
325
345
|
this.activeDownloads.delete(destPath);
|
|
326
346
|
resolve();
|
|
327
347
|
} catch (err) {
|
|
@@ -343,7 +363,9 @@ var DownloadManager = class _DownloadManager {
|
|
|
343
363
|
});
|
|
344
364
|
});
|
|
345
365
|
file.on("error", (err) => {
|
|
346
|
-
logger2.error(
|
|
366
|
+
logger2.error(
|
|
367
|
+
`File write error: ${err instanceof Error ? err.message : String(err)}`
|
|
368
|
+
);
|
|
347
369
|
file.close(() => {
|
|
348
370
|
if (fs.existsSync(tempPath)) {
|
|
349
371
|
try {
|
|
@@ -361,7 +383,9 @@ var DownloadManager = class _DownloadManager {
|
|
|
361
383
|
}
|
|
362
384
|
);
|
|
363
385
|
request.on("error", (err) => {
|
|
364
|
-
logger2.error(
|
|
386
|
+
logger2.error(
|
|
387
|
+
`Request error: ${err instanceof Error ? err.message : String(err)}`
|
|
388
|
+
);
|
|
365
389
|
if (fs.existsSync(tempPath)) {
|
|
366
390
|
try {
|
|
367
391
|
fs.unlinkSync(tempPath);
|
|
@@ -400,7 +424,9 @@ var DownloadManager = class _DownloadManager {
|
|
|
400
424
|
*/
|
|
401
425
|
async downloadFile(url, destPath) {
|
|
402
426
|
if (this.activeDownloads.has(destPath)) {
|
|
403
|
-
logger2.info(
|
|
427
|
+
logger2.info(
|
|
428
|
+
`Download for ${destPath} already in progress, waiting for it to complete...`
|
|
429
|
+
);
|
|
404
430
|
const existingDownload = this.activeDownloads.get(destPath);
|
|
405
431
|
if (existingDownload) {
|
|
406
432
|
return existingDownload;
|
|
@@ -639,7 +665,9 @@ var PlatformManager = class _PlatformManager {
|
|
|
639
665
|
isAppleSilicon: true
|
|
640
666
|
};
|
|
641
667
|
}
|
|
642
|
-
const { stdout: gpuInfo } = await execAsync(
|
|
668
|
+
const { stdout: gpuInfo } = await execAsync(
|
|
669
|
+
"system_profiler SPDisplaysDataType"
|
|
670
|
+
);
|
|
643
671
|
return {
|
|
644
672
|
name: gpuInfo.split("Chipset Model:")[1]?.split("\n")[0]?.trim() || "Unknown GPU",
|
|
645
673
|
type: "metal",
|
|
@@ -661,7 +689,9 @@ var PlatformManager = class _PlatformManager {
|
|
|
661
689
|
*/
|
|
662
690
|
async detectWindowsGPU() {
|
|
663
691
|
try {
|
|
664
|
-
const { stdout } = await execAsync(
|
|
692
|
+
const { stdout } = await execAsync(
|
|
693
|
+
"wmic path win32_VideoController get name"
|
|
694
|
+
);
|
|
665
695
|
const gpuName = stdout.split("\n")[1].trim();
|
|
666
696
|
if (gpuName.toLowerCase().includes("nvidia")) {
|
|
667
697
|
const { stdout: nvidiaInfo } = await execAsync(
|
|
@@ -855,7 +885,9 @@ var getPlatformManager = () => {
|
|
|
855
885
|
|
|
856
886
|
// src/utils/tokenizerManager.ts
|
|
857
887
|
import { logger as logger4 } from "@elizaos/core";
|
|
858
|
-
import {
|
|
888
|
+
import {
|
|
889
|
+
AutoTokenizer
|
|
890
|
+
} from "@huggingface/transformers";
|
|
859
891
|
var TokenizerManager = class _TokenizerManager {
|
|
860
892
|
static instance = null;
|
|
861
893
|
tokenizers;
|
|
@@ -905,13 +937,18 @@ var TokenizerManager = class _TokenizerManager {
|
|
|
905
937
|
logger4.info("Using cached tokenizer:", { key: tokenizerKey });
|
|
906
938
|
const cachedTokenizer = this.tokenizers.get(tokenizerKey);
|
|
907
939
|
if (!cachedTokenizer) {
|
|
908
|
-
throw new Error(
|
|
940
|
+
throw new Error(
|
|
941
|
+
`Tokenizer ${tokenizerKey} exists in map but returned undefined`
|
|
942
|
+
);
|
|
909
943
|
}
|
|
910
944
|
return cachedTokenizer;
|
|
911
945
|
}
|
|
912
946
|
const fs6 = await import("node:fs");
|
|
913
947
|
if (!fs6.existsSync(this.modelsDir)) {
|
|
914
|
-
logger4.warn(
|
|
948
|
+
logger4.warn(
|
|
949
|
+
"Models directory does not exist, creating it:",
|
|
950
|
+
this.modelsDir
|
|
951
|
+
);
|
|
915
952
|
fs6.mkdirSync(this.modelsDir, { recursive: true });
|
|
916
953
|
}
|
|
917
954
|
logger4.info(
|
|
@@ -919,10 +956,13 @@ var TokenizerManager = class _TokenizerManager {
|
|
|
919
956
|
this.modelsDir
|
|
920
957
|
);
|
|
921
958
|
try {
|
|
922
|
-
const tokenizer = await AutoTokenizer.from_pretrained(
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
959
|
+
const tokenizer = await AutoTokenizer.from_pretrained(
|
|
960
|
+
modelConfig.tokenizer.name,
|
|
961
|
+
{
|
|
962
|
+
cache_dir: this.modelsDir,
|
|
963
|
+
local_files_only: false
|
|
964
|
+
}
|
|
965
|
+
);
|
|
926
966
|
this.tokenizers.set(tokenizerKey, tokenizer);
|
|
927
967
|
logger4.success("Tokenizer loaded successfully:", { key: tokenizerKey });
|
|
928
968
|
return tokenizer;
|
|
@@ -934,10 +974,13 @@ var TokenizerManager = class _TokenizerManager {
|
|
|
934
974
|
modelsDir: this.modelsDir
|
|
935
975
|
});
|
|
936
976
|
logger4.info("Retrying tokenizer loading...");
|
|
937
|
-
const tokenizer = await AutoTokenizer.from_pretrained(
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
977
|
+
const tokenizer = await AutoTokenizer.from_pretrained(
|
|
978
|
+
modelConfig.tokenizer.name,
|
|
979
|
+
{
|
|
980
|
+
cache_dir: this.modelsDir,
|
|
981
|
+
local_files_only: false
|
|
982
|
+
}
|
|
983
|
+
);
|
|
941
984
|
this.tokenizers.set(tokenizerKey, tokenizer);
|
|
942
985
|
logger4.success("Tokenizer loaded successfully on retry:", {
|
|
943
986
|
key: tokenizerKey
|
|
@@ -1161,7 +1204,9 @@ var TranscribeManager = class _TranscribeManager {
|
|
|
1161
1204
|
*/
|
|
1162
1205
|
async checkFFmpegAvailability() {
|
|
1163
1206
|
try {
|
|
1164
|
-
const { stdout, stderr } = await execAsync2(
|
|
1207
|
+
const { stdout, stderr } = await execAsync2(
|
|
1208
|
+
"which ffmpeg || where ffmpeg"
|
|
1209
|
+
);
|
|
1165
1210
|
this.ffmpegPath = stdout.trim();
|
|
1166
1211
|
this.ffmpegAvailable = true;
|
|
1167
1212
|
logger5.info("FFmpeg found at:", {
|
|
@@ -1189,7 +1234,9 @@ var TranscribeManager = class _TranscribeManager {
|
|
|
1189
1234
|
const { stdout } = await execAsync2("ffmpeg -codecs");
|
|
1190
1235
|
const hasRequiredCodecs = stdout.includes("pcm_s16le") && stdout.includes("wav");
|
|
1191
1236
|
if (!hasRequiredCodecs) {
|
|
1192
|
-
throw new Error(
|
|
1237
|
+
throw new Error(
|
|
1238
|
+
"FFmpeg installation missing required codecs (pcm_s16le, wav)"
|
|
1239
|
+
);
|
|
1193
1240
|
}
|
|
1194
1241
|
} catch (error) {
|
|
1195
1242
|
logger5.error("FFmpeg capabilities verification failed:", {
|
|
@@ -1203,17 +1250,20 @@ var TranscribeManager = class _TranscribeManager {
|
|
|
1203
1250
|
* Logs instructions on how to install FFmpeg if it is not properly installed.
|
|
1204
1251
|
*/
|
|
1205
1252
|
logFFmpegInstallInstructions() {
|
|
1206
|
-
logger5.warn(
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1253
|
+
logger5.warn(
|
|
1254
|
+
"FFmpeg is required but not properly installed. Please install FFmpeg:",
|
|
1255
|
+
{
|
|
1256
|
+
instructions: {
|
|
1257
|
+
mac: "brew install ffmpeg",
|
|
1258
|
+
ubuntu: "sudo apt-get install ffmpeg",
|
|
1259
|
+
windows: "choco install ffmpeg",
|
|
1260
|
+
manual: "Download from https://ffmpeg.org/download.html"
|
|
1261
|
+
},
|
|
1262
|
+
requiredVersion: "4.0 or later",
|
|
1263
|
+
requiredCodecs: ["pcm_s16le", "wav"],
|
|
1264
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
1265
|
+
}
|
|
1266
|
+
);
|
|
1217
1267
|
}
|
|
1218
1268
|
/**
|
|
1219
1269
|
* Gets the singleton instance of TranscribeManager, creates a new instance if it doesn't exist.
|
|
@@ -1291,10 +1341,15 @@ var TranscribeManager = class _TranscribeManager {
|
|
|
1291
1341
|
*/
|
|
1292
1342
|
async preprocessAudio(audioBuffer) {
|
|
1293
1343
|
if (!this.ffmpegAvailable) {
|
|
1294
|
-
throw new Error(
|
|
1344
|
+
throw new Error(
|
|
1345
|
+
"FFmpeg is not installed. Please install FFmpeg to use audio transcription."
|
|
1346
|
+
);
|
|
1295
1347
|
}
|
|
1296
1348
|
try {
|
|
1297
|
-
const tempInputFile = path2.join(
|
|
1349
|
+
const tempInputFile = path2.join(
|
|
1350
|
+
this.cacheDir,
|
|
1351
|
+
`temp_input_${Date.now()}`
|
|
1352
|
+
);
|
|
1298
1353
|
const tempWavFile = path2.join(this.cacheDir, `temp_${Date.now()}.wav`);
|
|
1299
1354
|
fs2.writeFileSync(tempInputFile, audioBuffer);
|
|
1300
1355
|
await this.convertToWav(tempInputFile, tempWavFile);
|
|
@@ -1408,7 +1463,9 @@ var TTSManager = class _TTSManager {
|
|
|
1408
1463
|
}
|
|
1409
1464
|
async initialize() {
|
|
1410
1465
|
if (this.initializingPromise) {
|
|
1411
|
-
logger6.debug(
|
|
1466
|
+
logger6.debug(
|
|
1467
|
+
"TTS initialization already in progress, awaiting existing promise."
|
|
1468
|
+
);
|
|
1412
1469
|
return this.initializingPromise;
|
|
1413
1470
|
}
|
|
1414
1471
|
if (this.initialized) {
|
|
@@ -1420,15 +1477,21 @@ var TTSManager = class _TTSManager {
|
|
|
1420
1477
|
logger6.info("Initializing TTS with Transformers.js backend...");
|
|
1421
1478
|
const ttsModelSpec = MODEL_SPECS.tts.default;
|
|
1422
1479
|
if (!ttsModelSpec) {
|
|
1423
|
-
throw new Error(
|
|
1480
|
+
throw new Error(
|
|
1481
|
+
"Default TTS model specification not found in MODEL_SPECS."
|
|
1482
|
+
);
|
|
1424
1483
|
}
|
|
1425
1484
|
const modelName = ttsModelSpec.modelId;
|
|
1426
1485
|
const speakerEmbeddingUrl = ttsModelSpec.defaultSpeakerEmbeddingUrl;
|
|
1427
1486
|
logger6.info(`Loading TTS pipeline for model: ${modelName}`);
|
|
1428
1487
|
this.synthesizer = await pipeline("text-to-audio", modelName);
|
|
1429
|
-
logger6.success(
|
|
1488
|
+
logger6.success(
|
|
1489
|
+
`TTS pipeline loaded successfully for model: ${modelName}`
|
|
1490
|
+
);
|
|
1430
1491
|
if (speakerEmbeddingUrl) {
|
|
1431
|
-
const embeddingFilename = path3.basename(
|
|
1492
|
+
const embeddingFilename = path3.basename(
|
|
1493
|
+
new URL(speakerEmbeddingUrl).pathname
|
|
1494
|
+
);
|
|
1432
1495
|
const embeddingPath = path3.join(this.cacheDir, embeddingFilename);
|
|
1433
1496
|
if (fs3.existsSync(embeddingPath)) {
|
|
1434
1497
|
logger6.info("Loading default speaker embedding from cache...");
|
|
@@ -1440,10 +1503,14 @@ var TTSManager = class _TTSManager {
|
|
|
1440
1503
|
);
|
|
1441
1504
|
logger6.success("Default speaker embedding loaded from cache.");
|
|
1442
1505
|
} else {
|
|
1443
|
-
logger6.info(
|
|
1506
|
+
logger6.info(
|
|
1507
|
+
`Downloading default speaker embedding from: ${speakerEmbeddingUrl}`
|
|
1508
|
+
);
|
|
1444
1509
|
const response = await fetch2(speakerEmbeddingUrl);
|
|
1445
1510
|
if (!response.ok) {
|
|
1446
|
-
throw new Error(
|
|
1511
|
+
throw new Error(
|
|
1512
|
+
`Failed to download speaker embedding: ${response.statusText}`
|
|
1513
|
+
);
|
|
1447
1514
|
}
|
|
1448
1515
|
const buffer = await response.arrayBuffer();
|
|
1449
1516
|
this.defaultSpeakerEmbedding = new Float32Array(buffer);
|
|
@@ -1472,7 +1539,9 @@ var TTSManager = class _TTSManager {
|
|
|
1472
1539
|
throw error;
|
|
1473
1540
|
} finally {
|
|
1474
1541
|
this.initializingPromise = null;
|
|
1475
|
-
logger6.debug(
|
|
1542
|
+
logger6.debug(
|
|
1543
|
+
"TTS initializingPromise cleared after completion/failure."
|
|
1544
|
+
);
|
|
1476
1545
|
}
|
|
1477
1546
|
})();
|
|
1478
1547
|
return this.initializingPromise;
|
|
@@ -1494,7 +1563,9 @@ var TTSManager = class _TTSManager {
|
|
|
1494
1563
|
});
|
|
1495
1564
|
const output = await this.synthesizer(text, {
|
|
1496
1565
|
// Pass embedding only if it was loaded
|
|
1497
|
-
...this.defaultSpeakerEmbedding && {
|
|
1566
|
+
...this.defaultSpeakerEmbedding && {
|
|
1567
|
+
speaker_embeddings: this.defaultSpeakerEmbedding
|
|
1568
|
+
}
|
|
1498
1569
|
});
|
|
1499
1570
|
const audioFloat32 = output.audio;
|
|
1500
1571
|
const samplingRate = output.sampling_rate;
|
|
@@ -1579,7 +1650,10 @@ var VisionManager = class _VisionManager {
|
|
|
1579
1650
|
this.modelsDir = path4.join(path4.dirname(cacheDir), "models", "vision");
|
|
1580
1651
|
this.cacheDir = cacheDir;
|
|
1581
1652
|
this.ensureModelsDirExists();
|
|
1582
|
-
this.downloadManager = DownloadManager.getInstance(
|
|
1653
|
+
this.downloadManager = DownloadManager.getInstance(
|
|
1654
|
+
this.cacheDir,
|
|
1655
|
+
this.modelsDir
|
|
1656
|
+
);
|
|
1583
1657
|
this.platformConfig = this.getPlatformConfig();
|
|
1584
1658
|
logger7.debug("VisionManager initialized");
|
|
1585
1659
|
}
|
|
@@ -1643,7 +1717,11 @@ var VisionManager = class _VisionManager {
|
|
|
1643
1717
|
* @returns {boolean} - Returns true if cache exists, otherwise returns false.
|
|
1644
1718
|
*/
|
|
1645
1719
|
checkCacheExists(modelId, type) {
|
|
1646
|
-
const modelPath = path4.join(
|
|
1720
|
+
const modelPath = path4.join(
|
|
1721
|
+
this.modelsDir,
|
|
1722
|
+
modelId.replace("/", "--"),
|
|
1723
|
+
type
|
|
1724
|
+
);
|
|
1647
1725
|
if (existsSync(modelPath)) {
|
|
1648
1726
|
logger7.info(`${type} found at: ${modelPath}`);
|
|
1649
1727
|
return true;
|
|
@@ -1681,7 +1759,9 @@ var VisionManager = class _VisionManager {
|
|
|
1681
1759
|
* @returns {object} The model configuration object containing device, dtype, and cache_dir.
|
|
1682
1760
|
*/
|
|
1683
1761
|
getModelConfig(componentName) {
|
|
1684
|
-
const component = this.modelComponents.find(
|
|
1762
|
+
const component = this.modelComponents.find(
|
|
1763
|
+
(c) => c.name === componentName
|
|
1764
|
+
);
|
|
1685
1765
|
return {
|
|
1686
1766
|
device: this.platformConfig.device,
|
|
1687
1767
|
dtype: component?.dtype || "fp32",
|
|
@@ -1697,7 +1777,9 @@ var VisionManager = class _VisionManager {
|
|
|
1697
1777
|
async initialize() {
|
|
1698
1778
|
try {
|
|
1699
1779
|
if (this.initialized) {
|
|
1700
|
-
logger7.info(
|
|
1780
|
+
logger7.info(
|
|
1781
|
+
"Vision model already initialized, skipping initialization"
|
|
1782
|
+
);
|
|
1701
1783
|
return;
|
|
1702
1784
|
}
|
|
1703
1785
|
logger7.info("Starting vision model initialization...");
|
|
@@ -1713,25 +1795,32 @@ var VisionManager = class _VisionManager {
|
|
|
1713
1795
|
try {
|
|
1714
1796
|
let lastProgress = -1;
|
|
1715
1797
|
const modelCached = this.checkCacheExists(modelSpec.modelId, "model");
|
|
1716
|
-
const model = await Florence2ForConditionalGeneration.from_pretrained(
|
|
1717
|
-
|
|
1718
|
-
|
|
1719
|
-
|
|
1720
|
-
|
|
1721
|
-
|
|
1722
|
-
|
|
1723
|
-
|
|
1724
|
-
|
|
1725
|
-
|
|
1726
|
-
|
|
1727
|
-
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
1798
|
+
const model = await Florence2ForConditionalGeneration.from_pretrained(
|
|
1799
|
+
modelSpec.modelId,
|
|
1800
|
+
{
|
|
1801
|
+
device: "cpu",
|
|
1802
|
+
cache_dir: this.modelsDir,
|
|
1803
|
+
local_files_only: modelCached,
|
|
1804
|
+
revision: "main",
|
|
1805
|
+
progress_callback: (progressInfo) => {
|
|
1806
|
+
if (modelCached || this.modelDownloaded) return;
|
|
1807
|
+
const progress = "progress" in progressInfo ? Math.max(0, Math.min(1, progressInfo.progress)) : 0;
|
|
1808
|
+
const currentProgress = Math.round(progress * 100);
|
|
1809
|
+
if (currentProgress > lastProgress + 9 || currentProgress === 100) {
|
|
1810
|
+
lastProgress = currentProgress;
|
|
1811
|
+
const barLength = 30;
|
|
1812
|
+
const filledLength = Math.floor(
|
|
1813
|
+
currentProgress / 100 * barLength
|
|
1814
|
+
);
|
|
1815
|
+
const progressBar = "\u25B0".repeat(filledLength) + "\u25B1".repeat(barLength - filledLength);
|
|
1816
|
+
logger7.info(
|
|
1817
|
+
`Downloading vision model: ${progressBar} ${currentProgress}%`
|
|
1818
|
+
);
|
|
1819
|
+
if (currentProgress === 100) this.modelDownloaded = true;
|
|
1820
|
+
}
|
|
1732
1821
|
}
|
|
1733
1822
|
}
|
|
1734
|
-
|
|
1823
|
+
);
|
|
1735
1824
|
this.model = model;
|
|
1736
1825
|
logger7.success("Florence2 model loaded successfully");
|
|
1737
1826
|
} catch (error) {
|
|
@@ -1744,25 +1833,35 @@ var VisionManager = class _VisionManager {
|
|
|
1744
1833
|
}
|
|
1745
1834
|
logger7.info("Loading vision tokenizer...");
|
|
1746
1835
|
try {
|
|
1747
|
-
const tokenizerCached = this.checkCacheExists(
|
|
1836
|
+
const tokenizerCached = this.checkCacheExists(
|
|
1837
|
+
modelSpec.modelId,
|
|
1838
|
+
"tokenizer"
|
|
1839
|
+
);
|
|
1748
1840
|
let tokenizerProgress = -1;
|
|
1749
|
-
this.tokenizer = await AutoTokenizer2.from_pretrained(
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
|
|
1759
|
-
|
|
1760
|
-
|
|
1761
|
-
|
|
1762
|
-
|
|
1841
|
+
this.tokenizer = await AutoTokenizer2.from_pretrained(
|
|
1842
|
+
modelSpec.modelId,
|
|
1843
|
+
{
|
|
1844
|
+
cache_dir: this.modelsDir,
|
|
1845
|
+
local_files_only: tokenizerCached,
|
|
1846
|
+
progress_callback: (progressInfo) => {
|
|
1847
|
+
if (tokenizerCached || this.tokenizerDownloaded) return;
|
|
1848
|
+
const progress = "progress" in progressInfo ? Math.max(0, Math.min(1, progressInfo.progress)) : 0;
|
|
1849
|
+
const currentProgress = Math.round(progress * 100);
|
|
1850
|
+
if (currentProgress !== tokenizerProgress) {
|
|
1851
|
+
tokenizerProgress = currentProgress;
|
|
1852
|
+
const barLength = 30;
|
|
1853
|
+
const filledLength = Math.floor(
|
|
1854
|
+
currentProgress / 100 * barLength
|
|
1855
|
+
);
|
|
1856
|
+
const progressBar = "\u25B0".repeat(filledLength) + "\u25B1".repeat(barLength - filledLength);
|
|
1857
|
+
logger7.info(
|
|
1858
|
+
`Downloading vision tokenizer: ${progressBar} ${currentProgress}%`
|
|
1859
|
+
);
|
|
1860
|
+
if (currentProgress === 100) this.tokenizerDownloaded = true;
|
|
1861
|
+
}
|
|
1763
1862
|
}
|
|
1764
1863
|
}
|
|
1765
|
-
|
|
1864
|
+
);
|
|
1766
1865
|
logger7.success("Vision tokenizer loaded successfully");
|
|
1767
1866
|
} catch (error) {
|
|
1768
1867
|
logger7.error("Failed to load tokenizer:", {
|
|
@@ -1774,26 +1873,36 @@ var VisionManager = class _VisionManager {
|
|
|
1774
1873
|
}
|
|
1775
1874
|
logger7.info("Loading vision processor...");
|
|
1776
1875
|
try {
|
|
1777
|
-
const processorCached = this.checkCacheExists(
|
|
1876
|
+
const processorCached = this.checkCacheExists(
|
|
1877
|
+
modelSpec.modelId,
|
|
1878
|
+
"processor"
|
|
1879
|
+
);
|
|
1778
1880
|
let processorProgress = -1;
|
|
1779
|
-
this.processor = await AutoProcessor.from_pretrained(
|
|
1780
|
-
|
|
1781
|
-
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
|
|
1785
|
-
|
|
1786
|
-
|
|
1787
|
-
|
|
1788
|
-
|
|
1789
|
-
|
|
1790
|
-
|
|
1791
|
-
|
|
1792
|
-
|
|
1793
|
-
|
|
1881
|
+
this.processor = await AutoProcessor.from_pretrained(
|
|
1882
|
+
modelSpec.modelId,
|
|
1883
|
+
{
|
|
1884
|
+
device: "cpu",
|
|
1885
|
+
cache_dir: this.modelsDir,
|
|
1886
|
+
local_files_only: processorCached,
|
|
1887
|
+
progress_callback: (progressInfo) => {
|
|
1888
|
+
if (processorCached || this.processorDownloaded) return;
|
|
1889
|
+
const progress = "progress" in progressInfo ? Math.max(0, Math.min(1, progressInfo.progress)) : 0;
|
|
1890
|
+
const currentProgress = Math.round(progress * 100);
|
|
1891
|
+
if (currentProgress !== processorProgress) {
|
|
1892
|
+
processorProgress = currentProgress;
|
|
1893
|
+
const barLength = 30;
|
|
1894
|
+
const filledLength = Math.floor(
|
|
1895
|
+
currentProgress / 100 * barLength
|
|
1896
|
+
);
|
|
1897
|
+
const progressBar = "\u25B0".repeat(filledLength) + "\u25B1".repeat(barLength - filledLength);
|
|
1898
|
+
logger7.info(
|
|
1899
|
+
`Downloading vision processor: ${progressBar} ${currentProgress}%`
|
|
1900
|
+
);
|
|
1901
|
+
if (currentProgress === 100) this.processorDownloaded = true;
|
|
1902
|
+
}
|
|
1794
1903
|
}
|
|
1795
1904
|
}
|
|
1796
|
-
|
|
1905
|
+
);
|
|
1797
1906
|
logger7.success("Vision processor loaded successfully");
|
|
1798
1907
|
} catch (error) {
|
|
1799
1908
|
logger7.error("Failed to load vision processor:", {
|
|
@@ -2030,8 +2139,14 @@ var LocalAIManager = class _LocalAIManager {
|
|
|
2030
2139
|
*/
|
|
2031
2140
|
_postValidateInit() {
|
|
2032
2141
|
this._setupModelsDir();
|
|
2033
|
-
this.downloadManager = DownloadManager.getInstance(
|
|
2034
|
-
|
|
2142
|
+
this.downloadManager = DownloadManager.getInstance(
|
|
2143
|
+
this.cacheDir,
|
|
2144
|
+
this.modelsDir
|
|
2145
|
+
);
|
|
2146
|
+
this.tokenizerManager = TokenizerManager.getInstance(
|
|
2147
|
+
this.cacheDir,
|
|
2148
|
+
this.modelsDir
|
|
2149
|
+
);
|
|
2035
2150
|
this.visionManager = VisionManager.getInstance(this.cacheDir);
|
|
2036
2151
|
this.transcribeManager = TranscribeManager.getInstance(this.cacheDir);
|
|
2037
2152
|
this.ttsManager = TTSManager.getInstance(this.cacheDir);
|
|
@@ -2044,7 +2159,10 @@ var LocalAIManager = class _LocalAIManager {
|
|
|
2044
2159
|
const modelsDirEnv = this.config?.MODELS_DIR?.trim() || process.env.MODELS_DIR?.trim();
|
|
2045
2160
|
if (modelsDirEnv) {
|
|
2046
2161
|
this.modelsDir = path5.resolve(modelsDirEnv);
|
|
2047
|
-
logger8.info(
|
|
2162
|
+
logger8.info(
|
|
2163
|
+
"Using models directory from MODELS_DIR environment variable:",
|
|
2164
|
+
this.modelsDir
|
|
2165
|
+
);
|
|
2048
2166
|
} else {
|
|
2049
2167
|
this.modelsDir = path5.join(os3.homedir(), ".eliza", "models");
|
|
2050
2168
|
logger8.info(
|
|
@@ -2054,7 +2172,10 @@ var LocalAIManager = class _LocalAIManager {
|
|
|
2054
2172
|
}
|
|
2055
2173
|
if (!fs5.existsSync(this.modelsDir)) {
|
|
2056
2174
|
fs5.mkdirSync(this.modelsDir, { recursive: true });
|
|
2057
|
-
logger8.debug(
|
|
2175
|
+
logger8.debug(
|
|
2176
|
+
"Ensured models directory exists (created):",
|
|
2177
|
+
this.modelsDir
|
|
2178
|
+
);
|
|
2058
2179
|
} else {
|
|
2059
2180
|
logger8.debug("Models directory already exists:", this.modelsDir);
|
|
2060
2181
|
}
|
|
@@ -2067,7 +2188,10 @@ var LocalAIManager = class _LocalAIManager {
|
|
|
2067
2188
|
const cacheDirEnv = this.config?.CACHE_DIR?.trim() || process.env.CACHE_DIR?.trim();
|
|
2068
2189
|
if (cacheDirEnv) {
|
|
2069
2190
|
this.cacheDir = path5.resolve(cacheDirEnv);
|
|
2070
|
-
logger8.info(
|
|
2191
|
+
logger8.info(
|
|
2192
|
+
"Using cache directory from CACHE_DIR environment variable:",
|
|
2193
|
+
this.cacheDir
|
|
2194
|
+
);
|
|
2071
2195
|
} else {
|
|
2072
2196
|
const cacheDir = path5.join(os3.homedir(), ".eliza", "cache");
|
|
2073
2197
|
if (!fs5.existsSync(cacheDir)) {
|
|
@@ -2114,12 +2238,24 @@ var LocalAIManager = class _LocalAIManager {
|
|
|
2114
2238
|
logger8.info("Initializing environment configuration...");
|
|
2115
2239
|
this.config = await validateConfig();
|
|
2116
2240
|
this._postValidateInit();
|
|
2117
|
-
this.modelPath = path5.join(
|
|
2118
|
-
|
|
2119
|
-
|
|
2241
|
+
this.modelPath = path5.join(
|
|
2242
|
+
this.modelsDir,
|
|
2243
|
+
this.config.LOCAL_SMALL_MODEL
|
|
2244
|
+
);
|
|
2245
|
+
this.mediumModelPath = path5.join(
|
|
2246
|
+
this.modelsDir,
|
|
2247
|
+
this.config.LOCAL_LARGE_MODEL
|
|
2248
|
+
);
|
|
2249
|
+
this.embeddingModelPath = path5.join(
|
|
2250
|
+
this.modelsDir,
|
|
2251
|
+
this.config.LOCAL_EMBEDDING_MODEL
|
|
2252
|
+
);
|
|
2120
2253
|
logger8.info("Using small model path:", basename(this.modelPath));
|
|
2121
2254
|
logger8.info("Using medium model path:", basename(this.mediumModelPath));
|
|
2122
|
-
logger8.info(
|
|
2255
|
+
logger8.info(
|
|
2256
|
+
"Using embedding model path:",
|
|
2257
|
+
basename(this.embeddingModelPath)
|
|
2258
|
+
);
|
|
2123
2259
|
logger8.info("Environment configuration validated and model paths set");
|
|
2124
2260
|
this.environmentInitialized = true;
|
|
2125
2261
|
logger8.success("Environment initialization complete");
|
|
@@ -2157,7 +2293,10 @@ var LocalAIManager = class _LocalAIManager {
|
|
|
2157
2293
|
modelPathToDownload = modelType === ModelType.TEXT_LARGE ? this.mediumModelPath : this.modelPath;
|
|
2158
2294
|
}
|
|
2159
2295
|
try {
|
|
2160
|
-
return await this.downloadManager.downloadModel(
|
|
2296
|
+
return await this.downloadManager.downloadModel(
|
|
2297
|
+
modelSpec,
|
|
2298
|
+
modelPathToDownload
|
|
2299
|
+
);
|
|
2161
2300
|
} catch (error) {
|
|
2162
2301
|
logger8.error("Model download failed:", {
|
|
2163
2302
|
error: error instanceof Error ? error.message : String(error),
|
|
@@ -2212,7 +2351,10 @@ var LocalAIManager = class _LocalAIManager {
|
|
|
2212
2351
|
logger8.info("Initializing embedding model...");
|
|
2213
2352
|
logger8.info("Models directory:", this.modelsDir);
|
|
2214
2353
|
if (!fs5.existsSync(this.modelsDir)) {
|
|
2215
|
-
logger8.warn(
|
|
2354
|
+
logger8.warn(
|
|
2355
|
+
"Models directory does not exist, creating it:",
|
|
2356
|
+
this.modelsDir
|
|
2357
|
+
);
|
|
2216
2358
|
fs5.mkdirSync(this.modelsDir, { recursive: true });
|
|
2217
2359
|
}
|
|
2218
2360
|
await this.downloadModel(ModelType.TEXT_EMBEDDING);
|
|
@@ -2258,7 +2400,9 @@ var LocalAIManager = class _LocalAIManager {
|
|
|
2258
2400
|
const embeddingResult = await this.embeddingContext.getEmbeddingFor(text);
|
|
2259
2401
|
const mutableEmbedding = [...embeddingResult.vector];
|
|
2260
2402
|
const normalizedEmbedding = this.normalizeEmbedding(mutableEmbedding);
|
|
2261
|
-
logger8.info("Embedding generation complete", {
|
|
2403
|
+
logger8.info("Embedding generation complete", {
|
|
2404
|
+
dimensions: normalizedEmbedding.length
|
|
2405
|
+
});
|
|
2262
2406
|
return normalizedEmbedding;
|
|
2263
2407
|
} catch (error) {
|
|
2264
2408
|
logger8.error("Embedding generation failed:", {
|
|
@@ -2364,7 +2508,10 @@ var LocalAIManager = class _LocalAIManager {
|
|
|
2364
2508
|
runtime: !!params.runtime,
|
|
2365
2509
|
stopSequences: params.stopSequences
|
|
2366
2510
|
});
|
|
2367
|
-
const tokens = await this.tokenizerManager.encode(
|
|
2511
|
+
const tokens = await this.tokenizerManager.encode(
|
|
2512
|
+
params.prompt,
|
|
2513
|
+
this.activeModelConfig
|
|
2514
|
+
);
|
|
2368
2515
|
logger8.info("Input tokens:", { count: tokens.length });
|
|
2369
2516
|
const systemMessage = "You are a helpful AI assistant. Respond to the current request only.";
|
|
2370
2517
|
await this.chatSession.prompt(systemMessage, {
|
|
@@ -2554,7 +2701,9 @@ var LocalAIManager = class _LocalAIManager {
|
|
|
2554
2701
|
try {
|
|
2555
2702
|
await this.initializeEnvironment();
|
|
2556
2703
|
if (!this.transcribeManager) {
|
|
2557
|
-
this.transcribeManager = TranscribeManager.getInstance(
|
|
2704
|
+
this.transcribeManager = TranscribeManager.getInstance(
|
|
2705
|
+
this.cacheDir
|
|
2706
|
+
);
|
|
2558
2707
|
}
|
|
2559
2708
|
const ffmpegReady = await this.transcribeManager.ensureFFmpeg();
|
|
2560
2709
|
if (!ffmpegReady) {
|
|
@@ -2566,7 +2715,9 @@ var LocalAIManager = class _LocalAIManager {
|
|
|
2566
2715
|
);
|
|
2567
2716
|
}
|
|
2568
2717
|
this.transcriptionInitialized = true;
|
|
2569
|
-
logger8.info(
|
|
2718
|
+
logger8.info(
|
|
2719
|
+
"Transcription prerequisites (FFmpeg) checked and ready."
|
|
2720
|
+
);
|
|
2570
2721
|
logger8.info("Transcription model initialized successfully");
|
|
2571
2722
|
} catch (error) {
|
|
2572
2723
|
logger8.error("Failed to initialize transcription model:", error);
|
|
@@ -2649,7 +2800,9 @@ var localAiPlugin = {
|
|
|
2649
2800
|
const text = params?.text;
|
|
2650
2801
|
try {
|
|
2651
2802
|
if (!text) {
|
|
2652
|
-
logger8.debug(
|
|
2803
|
+
logger8.debug(
|
|
2804
|
+
"Null or empty text input for embedding, returning zero vector"
|
|
2805
|
+
);
|
|
2653
2806
|
return new Array(384).fill(0);
|
|
2654
2807
|
}
|
|
2655
2808
|
return await localAIManager.generateEmbedding(text);
|
|
@@ -2701,8 +2854,13 @@ var localAiPlugin = {
|
|
|
2701
2854
|
try {
|
|
2702
2855
|
jsonObject = JSON.parse(extractedJsonText);
|
|
2703
2856
|
} catch (parseError) {
|
|
2704
|
-
logger8.debug(
|
|
2705
|
-
|
|
2857
|
+
logger8.debug(
|
|
2858
|
+
"Initial JSON parse failed, attempting to fix common issues"
|
|
2859
|
+
);
|
|
2860
|
+
const fixedJson = extractedJsonText.replace(/:\s*"([^"]*)(?:\n)([^"]*)"/g, ': "$1\\n$2"').replace(
|
|
2861
|
+
/"([^"]*?)[^a-zA-Z0-9\s\.,;:\-_\(\)"'\[\]{}]([^"]*?)"/g,
|
|
2862
|
+
'"$1$2"'
|
|
2863
|
+
).replace(/(\s*)(\w+)(\s*):/g, '$1"$2"$3:').replace(/,(\s*[\]}])/g, "$1");
|
|
2706
2864
|
try {
|
|
2707
2865
|
jsonObject = JSON.parse(fixedJson);
|
|
2708
2866
|
} catch (finalError) {
|
|
@@ -2774,8 +2932,13 @@ var localAiPlugin = {
|
|
|
2774
2932
|
try {
|
|
2775
2933
|
jsonObject = JSON.parse(cleanedJsonText);
|
|
2776
2934
|
} catch (parseError) {
|
|
2777
|
-
logger8.debug(
|
|
2778
|
-
|
|
2935
|
+
logger8.debug(
|
|
2936
|
+
"Initial JSON parse failed, attempting to fix common issues"
|
|
2937
|
+
);
|
|
2938
|
+
const fixedJson = cleanedJsonText.replace(/:\s*"([^"]*)(?:\n)([^"]*)"/g, ': "$1\\n$2"').replace(
|
|
2939
|
+
/"([^"]*?)[^a-zA-Z0-9\s\.,;:\-_\(\)"'\[\]{}]([^"]*?)"/g,
|
|
2940
|
+
'"$1$2"'
|
|
2941
|
+
).replace(/(\s*)(\w+)(\s*):/g, '$1"$2"$3:').replace(/,(\s*[\]}])/g, "$1");
|
|
2779
2942
|
try {
|
|
2780
2943
|
jsonObject = JSON.parse(fixedJson);
|
|
2781
2944
|
} catch (finalError) {
|
|
@@ -2930,10 +3093,16 @@ var localAiPlugin = {
|
|
|
2930
3093
|
fn: async (runtime) => {
|
|
2931
3094
|
try {
|
|
2932
3095
|
logger8.info("Starting TEXT_EMBEDDING test");
|
|
2933
|
-
const embedding = await runtime.useModel(
|
|
2934
|
-
|
|
2935
|
-
|
|
2936
|
-
|
|
3096
|
+
const embedding = await runtime.useModel(
|
|
3097
|
+
ModelType.TEXT_EMBEDDING,
|
|
3098
|
+
{
|
|
3099
|
+
text: "This is a test of the text embedding model."
|
|
3100
|
+
}
|
|
3101
|
+
);
|
|
3102
|
+
logger8.info(
|
|
3103
|
+
"Embedding generated with dimensions:",
|
|
3104
|
+
embedding.length
|
|
3105
|
+
);
|
|
2937
3106
|
if (!Array.isArray(embedding)) {
|
|
2938
3107
|
throw new Error("Embedding is not an array");
|
|
2939
3108
|
}
|
|
@@ -2943,7 +3112,10 @@ var localAiPlugin = {
|
|
|
2943
3112
|
if (embedding.some((val) => typeof val !== "number")) {
|
|
2944
3113
|
throw new Error("Embedding contains non-numeric values");
|
|
2945
3114
|
}
|
|
2946
|
-
const nullEmbedding = await runtime.useModel(
|
|
3115
|
+
const nullEmbedding = await runtime.useModel(
|
|
3116
|
+
ModelType.TEXT_EMBEDDING,
|
|
3117
|
+
null
|
|
3118
|
+
);
|
|
2947
3119
|
if (!Array.isArray(nullEmbedding) || nullEmbedding.some((val) => val !== 0)) {
|
|
2948
3120
|
throw new Error("Null input did not return zero vector");
|
|
2949
3121
|
}
|
|
@@ -2963,7 +3135,10 @@ var localAiPlugin = {
|
|
|
2963
3135
|
try {
|
|
2964
3136
|
logger8.info("Starting TEXT_TOKENIZER_ENCODE test");
|
|
2965
3137
|
const text = "Hello tokenizer test!";
|
|
2966
|
-
const tokens = await runtime.useModel(
|
|
3138
|
+
const tokens = await runtime.useModel(
|
|
3139
|
+
ModelType.TEXT_TOKENIZER_ENCODE,
|
|
3140
|
+
{ text }
|
|
3141
|
+
);
|
|
2967
3142
|
logger8.info("Encoded tokens:", { count: tokens.length });
|
|
2968
3143
|
if (!Array.isArray(tokens)) {
|
|
2969
3144
|
throw new Error("Tokens output is not an array");
|
|
@@ -2974,7 +3149,9 @@ var localAiPlugin = {
|
|
|
2974
3149
|
if (tokens.some((token) => !Number.isInteger(token))) {
|
|
2975
3150
|
throw new Error("Tokens contain non-integer values");
|
|
2976
3151
|
}
|
|
2977
|
-
logger8.success(
|
|
3152
|
+
logger8.success(
|
|
3153
|
+
"TEXT_TOKENIZER_ENCODE test completed successfully"
|
|
3154
|
+
);
|
|
2978
3155
|
} catch (error) {
|
|
2979
3156
|
logger8.error("TEXT_TOKENIZER_ENCODE test failed:", {
|
|
2980
3157
|
error: error instanceof Error ? error.message : String(error),
|
|
@@ -2990,12 +3167,18 @@ var localAiPlugin = {
|
|
|
2990
3167
|
try {
|
|
2991
3168
|
logger8.info("Starting TEXT_TOKENIZER_DECODE test");
|
|
2992
3169
|
const originalText = "Hello tokenizer test!";
|
|
2993
|
-
const tokens = await runtime.useModel(
|
|
2994
|
-
|
|
2995
|
-
|
|
2996
|
-
|
|
2997
|
-
|
|
2998
|
-
|
|
3170
|
+
const tokens = await runtime.useModel(
|
|
3171
|
+
ModelType.TEXT_TOKENIZER_ENCODE,
|
|
3172
|
+
{
|
|
3173
|
+
text: originalText
|
|
3174
|
+
}
|
|
3175
|
+
);
|
|
3176
|
+
const decodedText = await runtime.useModel(
|
|
3177
|
+
ModelType.TEXT_TOKENIZER_DECODE,
|
|
3178
|
+
{
|
|
3179
|
+
tokens
|
|
3180
|
+
}
|
|
3181
|
+
);
|
|
2999
3182
|
logger8.info("Round trip tokenization:", {
|
|
3000
3183
|
original: originalText,
|
|
3001
3184
|
decoded: decodedText
|
|
@@ -3003,7 +3186,9 @@ var localAiPlugin = {
|
|
|
3003
3186
|
if (typeof decodedText !== "string") {
|
|
3004
3187
|
throw new Error("Decoded output is not a string");
|
|
3005
3188
|
}
|
|
3006
|
-
logger8.success(
|
|
3189
|
+
logger8.success(
|
|
3190
|
+
"TEXT_TOKENIZER_DECODE test completed successfully"
|
|
3191
|
+
);
|
|
3007
3192
|
} catch (error) {
|
|
3008
3193
|
logger8.error("TEXT_TOKENIZER_DECODE test failed:", {
|
|
3009
3194
|
error: error instanceof Error ? error.message : String(error),
|
|
@@ -3019,7 +3204,10 @@ var localAiPlugin = {
|
|
|
3019
3204
|
try {
|
|
3020
3205
|
logger8.info("Starting IMAGE_DESCRIPTION test");
|
|
3021
3206
|
const imageUrl = "https://raw.githubusercontent.com/microsoft/FLAML/main/website/static/img/flaml.png";
|
|
3022
|
-
const result = await runtime.useModel(
|
|
3207
|
+
const result = await runtime.useModel(
|
|
3208
|
+
ModelType.IMAGE_DESCRIPTION,
|
|
3209
|
+
imageUrl
|
|
3210
|
+
);
|
|
3023
3211
|
logger8.info("Image description result:", result);
|
|
3024
3212
|
if (!result || typeof result !== "object") {
|
|
3025
3213
|
throw new Error("Invalid response format");
|
|
@@ -3068,7 +3256,10 @@ var localAiPlugin = {
|
|
|
3068
3256
|
// "fmt "
|
|
3069
3257
|
]);
|
|
3070
3258
|
const audioBuffer = Buffer.from(audioData);
|
|
3071
|
-
const transcription = await runtime.useModel(
|
|
3259
|
+
const transcription = await runtime.useModel(
|
|
3260
|
+
ModelType.TRANSCRIPTION,
|
|
3261
|
+
audioBuffer
|
|
3262
|
+
);
|
|
3072
3263
|
logger8.info("Transcription result:", transcription);
|
|
3073
3264
|
if (typeof transcription !== "string") {
|
|
3074
3265
|
throw new Error("Transcription result is not a string");
|
|
@@ -3089,7 +3280,10 @@ var localAiPlugin = {
|
|
|
3089
3280
|
try {
|
|
3090
3281
|
logger8.info("Starting TEXT_TO_SPEECH test");
|
|
3091
3282
|
const testText = "This is a test of the text to speech system.";
|
|
3092
|
-
const audioStream = await runtime.useModel(
|
|
3283
|
+
const audioStream = await runtime.useModel(
|
|
3284
|
+
ModelType.TEXT_TO_SPEECH,
|
|
3285
|
+
testText
|
|
3286
|
+
);
|
|
3093
3287
|
if (!(audioStream instanceof Readable2)) {
|
|
3094
3288
|
throw new Error("TTS output is not a readable stream");
|
|
3095
3289
|
}
|