agentgui 1.0.359 → 1.0.360
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build-portable.js +14 -1
- package/database.js +30 -70
- package/lib/download-metrics.js +0 -12
- package/lib/model-downloader.js +10 -84
- package/package.json +1 -1
- package/server.js +0 -7
- package/lib/ipfs-publish.js +0 -136
- package/scripts/publish-models-to-ipfs.js +0 -172
package/build-portable.js
CHANGED
|
@@ -135,7 +135,20 @@ if (process.env.NO_BUNDLE_MODELS === 'true') {
|
|
|
135
135
|
log('Skipping model bundling (NO_BUNDLE_MODELS=true) - models will download on first use');
|
|
136
136
|
} else {
|
|
137
137
|
log('Bundling AI models...');
|
|
138
|
-
|
|
138
|
+
// Get models from AnEntrypoint/models or local cache
|
|
139
|
+
let modelsDir = process.env.MODELS_SOURCE_DIR || path.join(os.homedir(), '.gmgui', 'models');
|
|
140
|
+
|
|
141
|
+
// If models not present and we're in CI, clone from GitHub
|
|
142
|
+
if (!fs.existsSync(modelsDir) && process.env.CI) {
|
|
143
|
+
console.log('[BUILD] Models not found, cloning from GitHub...');
|
|
144
|
+
const ciModelsDir = path.join(os.tmpdir(), 'models-clone');
|
|
145
|
+
try {
|
|
146
|
+
require('child_process').execSync(`git clone https://github.com/AnEntrypoint/models.git "${ciModelsDir}" --depth 1`, { stdio: 'inherit' });
|
|
147
|
+
modelsDir = ciModelsDir;
|
|
148
|
+
} catch (e) {
|
|
149
|
+
console.error('[BUILD] Failed to clone models from GitHub:', e.message);
|
|
150
|
+
}
|
|
151
|
+
}
|
|
139
152
|
if (fs.existsSync(userModels)) {
|
|
140
153
|
copyDir(userModels, path.join(out, 'models'));
|
|
141
154
|
log(`Models bundled: ${Math.round(sizeOf(path.join(out, 'models')) / 1024 / 1024)}MB`);
|
package/database.js
CHANGED
|
@@ -150,7 +150,7 @@ function initSchema() {
|
|
|
150
150
|
CREATE INDEX IF NOT EXISTS idx_chunks_conv_created ON chunks(conversationId, created_at);
|
|
151
151
|
CREATE INDEX IF NOT EXISTS idx_chunks_sess_created ON chunks(sessionId, created_at);
|
|
152
152
|
|
|
153
|
-
CREATE TABLE IF NOT EXISTS
|
|
153
|
+
CREATE TABLE IF NOT EXISTS (
|
|
154
154
|
id TEXT PRIMARY KEY,
|
|
155
155
|
cid TEXT NOT NULL UNIQUE,
|
|
156
156
|
modelName TEXT NOT NULL,
|
|
@@ -163,11 +163,11 @@ function initSchema() {
|
|
|
163
163
|
failure_count INTEGER DEFAULT 0
|
|
164
164
|
);
|
|
165
165
|
|
|
166
|
-
CREATE INDEX IF NOT EXISTS
|
|
167
|
-
CREATE INDEX IF NOT EXISTS
|
|
168
|
-
CREATE INDEX IF NOT EXISTS
|
|
166
|
+
CREATE INDEX IF NOT EXISTS idx__model ON (modelName);
|
|
167
|
+
CREATE INDEX IF NOT EXISTS idx__type ON (modelType);
|
|
168
|
+
CREATE INDEX IF NOT EXISTS idx__hash ON (modelHash);
|
|
169
169
|
|
|
170
|
-
CREATE TABLE IF NOT EXISTS
|
|
170
|
+
CREATE TABLE IF NOT EXISTS (
|
|
171
171
|
id TEXT PRIMARY KEY,
|
|
172
172
|
cidId TEXT NOT NULL,
|
|
173
173
|
downloadPath TEXT NOT NULL,
|
|
@@ -177,12 +177,12 @@ function initSchema() {
|
|
|
177
177
|
error_message TEXT,
|
|
178
178
|
started_at INTEGER NOT NULL,
|
|
179
179
|
completed_at INTEGER,
|
|
180
|
-
FOREIGN KEY (cidId) REFERENCES
|
|
180
|
+
FOREIGN KEY (cidId) REFERENCES (id)
|
|
181
181
|
);
|
|
182
182
|
|
|
183
|
-
CREATE INDEX IF NOT EXISTS
|
|
184
|
-
CREATE INDEX IF NOT EXISTS
|
|
185
|
-
CREATE INDEX IF NOT EXISTS
|
|
183
|
+
CREATE INDEX IF NOT EXISTS idx__cid ON (cidId);
|
|
184
|
+
CREATE INDEX IF NOT EXISTS idx__status ON (status);
|
|
185
|
+
CREATE INDEX IF NOT EXISTS idx__started ON (started_at);
|
|
186
186
|
`);
|
|
187
187
|
}
|
|
188
188
|
|
|
@@ -305,9 +305,9 @@ try {
|
|
|
305
305
|
console.error('[Migration] Error:', err.message);
|
|
306
306
|
}
|
|
307
307
|
|
|
308
|
-
// Migration: Add resume capability columns to
|
|
308
|
+
// Migration: Add resume capability columns to if needed
|
|
309
309
|
try {
|
|
310
|
-
const result = db.prepare("PRAGMA table_info(
|
|
310
|
+
const result = db.prepare("PRAGMA table_info()").all();
|
|
311
311
|
const columnNames = result.map(r => r.name);
|
|
312
312
|
const resumeColumns = {
|
|
313
313
|
attempts: 'INTEGER DEFAULT 0',
|
|
@@ -318,12 +318,12 @@ try {
|
|
|
318
318
|
|
|
319
319
|
for (const [colName, colDef] of Object.entries(resumeColumns)) {
|
|
320
320
|
if (!columnNames.includes(colName)) {
|
|
321
|
-
db.exec(`ALTER TABLE
|
|
322
|
-
console.log(`[Migration] Added column ${colName} to
|
|
321
|
+
db.exec(`ALTER TABLE ADD COLUMN ${colName} ${colDef}`);
|
|
322
|
+
console.log(`[Migration] Added column ${colName} to table`);
|
|
323
323
|
}
|
|
324
324
|
}
|
|
325
325
|
} catch (err) {
|
|
326
|
-
console.error('[Migration]
|
|
326
|
+
console.error('[Migration] Schema update warning:', err.message);
|
|
327
327
|
}
|
|
328
328
|
|
|
329
329
|
// Migration: Backfill messages for conversations imported without message content
|
|
@@ -383,46 +383,6 @@ try {
|
|
|
383
383
|
console.error('[Migration] Backfill error:', err.message);
|
|
384
384
|
}
|
|
385
385
|
|
|
386
|
-
// Register official IPFS CIDs for voice models
|
|
387
|
-
try {
|
|
388
|
-
const LIGHTHOUSE_GATEWAY = 'https://gateway.lighthouse.storage/ipfs';
|
|
389
|
-
const WHISPER_CID = 'bafybeidyw252ecy4vs46bbmezrtw325gl2ymdltosmzqgx4edjsc3fbofy';
|
|
390
|
-
const TTS_CID = 'bafybeidyw252ecy4vs46bbmezrtw325gl2ymdltosmzqgx4edjsc3fbofy';
|
|
391
|
-
const TTS_TOKENIZER_MODEL_CID = 'bafkreigumf3fvylzkzthrsjqshc7u3zjqtbrxpuzbpy2uywzfrsnsg6d6y';
|
|
392
|
-
|
|
393
|
-
// Check if CIDs are already registered
|
|
394
|
-
const existingWhisper = db.prepare('SELECT * FROM ipfs_cids WHERE modelName = ? AND modelType = ?').get('whisper-base', 'stt');
|
|
395
|
-
if (!existingWhisper) {
|
|
396
|
-
const cidId = `cid-${Date.now()}-whisper`;
|
|
397
|
-
db.prepare(
|
|
398
|
-
`INSERT INTO ipfs_cids (id, cid, modelName, modelType, modelHash, gatewayUrl, cached_at, last_accessed_at)
|
|
399
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
|
|
400
|
-
).run(cidId, WHISPER_CID, 'whisper-base', 'stt', 'sha256-verified', LIGHTHOUSE_GATEWAY, Date.now(), Date.now());
|
|
401
|
-
console.log('[MODELS] Registered Whisper STT IPFS CID:', WHISPER_CID);
|
|
402
|
-
}
|
|
403
|
-
|
|
404
|
-
const existingTTS = db.prepare('SELECT * FROM ipfs_cids WHERE modelName = ? AND modelType = ?').get('tts-models', 'voice');
|
|
405
|
-
if (!existingTTS) {
|
|
406
|
-
const cidId = `cid-${Date.now()}-tts`;
|
|
407
|
-
db.prepare(
|
|
408
|
-
`INSERT INTO ipfs_cids (id, cid, modelName, modelType, modelHash, gatewayUrl, cached_at, last_accessed_at)
|
|
409
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
|
|
410
|
-
).run(cidId, TTS_CID, 'tts-models', 'voice', 'sha256-verified', LIGHTHOUSE_GATEWAY, Date.now(), Date.now());
|
|
411
|
-
console.log('[MODELS] Registered TTS IPFS CID:', TTS_CID);
|
|
412
|
-
}
|
|
413
|
-
|
|
414
|
-
const existingTokenizerModel = db.prepare('SELECT * FROM ipfs_cids WHERE modelName = ? AND modelType = ?').get('tts-tokenizer.model', 'voice-file');
|
|
415
|
-
if (!existingTokenizerModel) {
|
|
416
|
-
const cidId = `cid-${Date.now()}-tts-tokenizer-model`;
|
|
417
|
-
db.prepare(
|
|
418
|
-
`INSERT INTO ipfs_cids (id, cid, modelName, modelType, modelHash, gatewayUrl, cached_at, last_accessed_at)
|
|
419
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
|
|
420
|
-
).run(cidId, TTS_TOKENIZER_MODEL_CID, 'tts-tokenizer.model', 'voice-file', 'd461765ae179566678c93091c5fa6f2984c31bbe990bf1aa62d92c64d91bc3f6', LIGHTHOUSE_GATEWAY, Date.now(), Date.now());
|
|
421
|
-
console.log('[MODELS] Registered TTS tokenizer.model IPFS CID:', TTS_TOKENIZER_MODEL_CID);
|
|
422
|
-
}
|
|
423
|
-
} catch (err) {
|
|
424
|
-
console.warn('[MODELS] IPFS CID registration warning:', err.message);
|
|
425
|
-
}
|
|
426
386
|
|
|
427
387
|
const stmtCache = new Map();
|
|
428
388
|
function prep(sql) {
|
|
@@ -1349,7 +1309,7 @@ export const queries = {
|
|
|
1349
1309
|
const id = generateId('ipfs');
|
|
1350
1310
|
const now = Date.now();
|
|
1351
1311
|
const stmt = prep(`
|
|
1352
|
-
INSERT INTO
|
|
1312
|
+
INSERT INTO (id, cid, modelName, modelType, modelHash, gatewayUrl, cached_at, last_accessed_at)
|
|
1353
1313
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
1354
1314
|
ON CONFLICT(cid) DO UPDATE SET last_accessed_at = ?, success_count = success_count + 1
|
|
1355
1315
|
`);
|
|
@@ -1359,19 +1319,19 @@ export const queries = {
|
|
|
1359
1319
|
},
|
|
1360
1320
|
|
|
1361
1321
|
getIpfsCid(cid) {
|
|
1362
|
-
const stmt = prep('SELECT * FROM
|
|
1322
|
+
const stmt = prep('SELECT * FROM WHERE cid = ?');
|
|
1363
1323
|
return stmt.get(cid);
|
|
1364
1324
|
},
|
|
1365
1325
|
|
|
1366
1326
|
getIpfsCidByModel(modelName, modelType) {
|
|
1367
|
-
const stmt = prep('SELECT * FROM
|
|
1327
|
+
const stmt = prep('SELECT * FROM WHERE modelName = ? AND modelType = ? ORDER BY last_accessed_at DESC LIMIT 1');
|
|
1368
1328
|
return stmt.get(modelName, modelType);
|
|
1369
1329
|
},
|
|
1370
1330
|
|
|
1371
1331
|
recordDownloadStart(cidId, downloadPath, totalBytes) {
|
|
1372
1332
|
const id = generateId('dl');
|
|
1373
1333
|
const stmt = prep(`
|
|
1374
|
-
INSERT INTO
|
|
1334
|
+
INSERT INTO (id, cidId, downloadPath, status, total_bytes, started_at)
|
|
1375
1335
|
VALUES (?, ?, ?, ?, ?, ?)
|
|
1376
1336
|
`);
|
|
1377
1337
|
stmt.run(id, cidId, downloadPath, 'in_progress', totalBytes, Date.now());
|
|
@@ -1380,7 +1340,7 @@ export const queries = {
|
|
|
1380
1340
|
|
|
1381
1341
|
updateDownloadProgress(downloadId, downloadedBytes) {
|
|
1382
1342
|
const stmt = prep(`
|
|
1383
|
-
UPDATE
|
|
1343
|
+
UPDATE SET downloaded_bytes = ? WHERE id = ?
|
|
1384
1344
|
`);
|
|
1385
1345
|
stmt.run(downloadedBytes, downloadId);
|
|
1386
1346
|
},
|
|
@@ -1388,41 +1348,41 @@ export const queries = {
|
|
|
1388
1348
|
completeDownload(downloadId, cidId) {
|
|
1389
1349
|
const now = Date.now();
|
|
1390
1350
|
prep(`
|
|
1391
|
-
UPDATE
|
|
1351
|
+
UPDATE SET status = ?, completed_at = ? WHERE id = ?
|
|
1392
1352
|
`).run('success', now, downloadId);
|
|
1393
1353
|
prep(`
|
|
1394
|
-
UPDATE
|
|
1354
|
+
UPDATE SET last_accessed_at = ? WHERE id = ?
|
|
1395
1355
|
`).run(now, cidId);
|
|
1396
1356
|
},
|
|
1397
1357
|
|
|
1398
1358
|
recordDownloadError(downloadId, cidId, errorMessage) {
|
|
1399
1359
|
const now = Date.now();
|
|
1400
1360
|
prep(`
|
|
1401
|
-
UPDATE
|
|
1361
|
+
UPDATE SET status = ?, error_message = ?, completed_at = ? WHERE id = ?
|
|
1402
1362
|
`).run('failed', errorMessage, now, downloadId);
|
|
1403
1363
|
prep(`
|
|
1404
|
-
UPDATE
|
|
1364
|
+
UPDATE SET failure_count = failure_count + 1 WHERE id = ?
|
|
1405
1365
|
`).run(cidId);
|
|
1406
1366
|
},
|
|
1407
1367
|
|
|
1408
1368
|
getDownload(downloadId) {
|
|
1409
|
-
const stmt = prep('SELECT * FROM
|
|
1369
|
+
const stmt = prep('SELECT * FROM WHERE id = ?');
|
|
1410
1370
|
return stmt.get(downloadId);
|
|
1411
1371
|
},
|
|
1412
1372
|
|
|
1413
1373
|
getDownloadsByCid(cidId) {
|
|
1414
|
-
const stmt = prep('SELECT * FROM
|
|
1374
|
+
const stmt = prep('SELECT * FROM WHERE cidId = ? ORDER BY started_at DESC');
|
|
1415
1375
|
return stmt.all(cidId);
|
|
1416
1376
|
},
|
|
1417
1377
|
|
|
1418
1378
|
getDownloadsByStatus(status) {
|
|
1419
|
-
const stmt = prep('SELECT * FROM
|
|
1379
|
+
const stmt = prep('SELECT * FROM WHERE status = ? ORDER BY started_at DESC');
|
|
1420
1380
|
return stmt.all(status);
|
|
1421
1381
|
},
|
|
1422
1382
|
|
|
1423
1383
|
updateDownloadResume(downloadId, currentSize, attempts, lastAttempt, status) {
|
|
1424
1384
|
const stmt = prep(`
|
|
1425
|
-
UPDATE
|
|
1385
|
+
UPDATE
|
|
1426
1386
|
SET downloaded_bytes = ?, attempts = ?, lastAttempt = ?, status = ?
|
|
1427
1387
|
WHERE id = ?
|
|
1428
1388
|
`);
|
|
@@ -1430,17 +1390,17 @@ export const queries = {
|
|
|
1430
1390
|
},
|
|
1431
1391
|
|
|
1432
1392
|
updateDownloadHash(downloadId, hash) {
|
|
1433
|
-
const stmt = prep('UPDATE
|
|
1393
|
+
const stmt = prep('UPDATE SET hash = ? WHERE id = ?');
|
|
1434
1394
|
stmt.run(hash, downloadId);
|
|
1435
1395
|
},
|
|
1436
1396
|
|
|
1437
1397
|
markDownloadResuming(downloadId) {
|
|
1438
|
-
const stmt = prep('UPDATE
|
|
1398
|
+
const stmt = prep('UPDATE SET status = ?, lastAttempt = ? WHERE id = ?');
|
|
1439
1399
|
stmt.run('resuming', Date.now(), downloadId);
|
|
1440
1400
|
},
|
|
1441
1401
|
|
|
1442
1402
|
markDownloadPaused(downloadId, errorMessage) {
|
|
1443
|
-
const stmt = prep('UPDATE
|
|
1403
|
+
const stmt = prep('UPDATE SET status = ?, error_message = ?, lastAttempt = ? WHERE id = ?');
|
|
1444
1404
|
stmt.run('paused', errorMessage, Date.now(), downloadId);
|
|
1445
1405
|
}
|
|
1446
1406
|
};
|
package/lib/download-metrics.js
CHANGED
|
@@ -43,11 +43,6 @@ export function getMetricsSummary() {
|
|
|
43
43
|
const summary = {
|
|
44
44
|
total: metrics.length,
|
|
45
45
|
cache_hits: metrics.filter(m => m.layer === 'cache' && m.status === 'hit').length,
|
|
46
|
-
ipfs: {
|
|
47
|
-
success: metrics.filter(m => m.layer === 'ipfs' && m.status === 'success').length,
|
|
48
|
-
error: metrics.filter(m => m.layer === 'ipfs' && m.status === 'error').length,
|
|
49
|
-
avg_latency: 0
|
|
50
|
-
},
|
|
51
46
|
huggingface: {
|
|
52
47
|
success: metrics.filter(m => m.layer === 'huggingface' && m.status === 'success').length,
|
|
53
48
|
error: metrics.filter(m => m.layer === 'huggingface' && m.status === 'error').length,
|
|
@@ -55,13 +50,6 @@ export function getMetricsSummary() {
|
|
|
55
50
|
}
|
|
56
51
|
};
|
|
57
52
|
|
|
58
|
-
const ipfsSuccess = metrics.filter(m => m.layer === 'ipfs' && m.status === 'success');
|
|
59
|
-
if (ipfsSuccess.length > 0) {
|
|
60
|
-
summary.ipfs.avg_latency = Math.round(
|
|
61
|
-
ipfsSuccess.reduce((sum, m) => sum + m.latency_ms, 0) / ipfsSuccess.length
|
|
62
|
-
);
|
|
63
|
-
}
|
|
64
|
-
|
|
65
53
|
const hfSuccess = metrics.filter(m => m.layer === 'huggingface' && m.status === 'success');
|
|
66
54
|
if (hfSuccess.length > 0) {
|
|
67
55
|
summary.huggingface.avg_latency = Math.round(
|
package/lib/model-downloader.js
CHANGED
|
@@ -7,85 +7,13 @@ import { verifyFileIntegrity } from './file-verification.js';
|
|
|
7
7
|
const require = createRequire(import.meta.url);
|
|
8
8
|
|
|
9
9
|
const GATEWAYS = [
|
|
10
|
-
'https://cloudflare-
|
|
11
|
-
'https://dweb.link/
|
|
12
|
-
'https://gateway.pinata.cloud/
|
|
13
|
-
'https://
|
|
10
|
+
'https://cloudflare-huggingface.com/huggingface/',
|
|
11
|
+
'https://dweb.link/huggingface/',
|
|
12
|
+
'https://gateway.pinata.cloud/huggingface/',
|
|
13
|
+
'https://huggingface.io/huggingface/'
|
|
14
14
|
];
|
|
15
15
|
|
|
16
|
-
async function downloadFromIPFS(cid, destPath, manifest, onProgress) {
|
|
17
|
-
const startTime = Date.now();
|
|
18
|
-
|
|
19
|
-
for (let gatewayIndex = 0; gatewayIndex < GATEWAYS.length; gatewayIndex++) {
|
|
20
|
-
const gateway = GATEWAYS[gatewayIndex];
|
|
21
|
-
const gatewayName = new URL(gateway).hostname;
|
|
22
|
-
|
|
23
|
-
for (let retry = 0; retry < 2; retry++) {
|
|
24
|
-
try {
|
|
25
|
-
if (onProgress) {
|
|
26
|
-
onProgress({
|
|
27
|
-
layer: 'ipfs',
|
|
28
|
-
gateway: gatewayName,
|
|
29
|
-
attempt: retry + 1,
|
|
30
|
-
status: 'attempting'
|
|
31
|
-
});
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
const { downloadWithProgress } = require('webtalk/ipfs-downloader');
|
|
35
|
-
const url = `${gateway}${cid}`;
|
|
36
|
-
|
|
37
|
-
await downloadWithProgress(url, destPath, (progress) => {
|
|
38
|
-
if (onProgress) {
|
|
39
|
-
onProgress({
|
|
40
|
-
layer: 'ipfs',
|
|
41
|
-
gateway: gatewayName,
|
|
42
|
-
status: 'downloading',
|
|
43
|
-
...progress
|
|
44
|
-
});
|
|
45
|
-
}
|
|
46
|
-
});
|
|
47
|
-
|
|
48
|
-
const verification = verifyFileIntegrity(
|
|
49
|
-
destPath,
|
|
50
|
-
manifest?.sha256,
|
|
51
|
-
manifest?.size ? manifest.size * 0.8 : null
|
|
52
|
-
);
|
|
53
|
-
|
|
54
|
-
if (!verification.valid) {
|
|
55
|
-
if (fs.existsSync(destPath)) fs.unlinkSync(destPath);
|
|
56
|
-
throw new Error(`Verification failed: ${verification.reason}`);
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
recordMetric({
|
|
60
|
-
modelType: 'model',
|
|
61
|
-
layer: 'ipfs',
|
|
62
|
-
gateway: gatewayName,
|
|
63
|
-
status: 'success',
|
|
64
|
-
latency_ms: Date.now() - startTime,
|
|
65
|
-
bytes_downloaded: fs.statSync(destPath).size
|
|
66
|
-
});
|
|
67
|
-
|
|
68
|
-
return { success: true, source: 'ipfs', gateway: gatewayName };
|
|
69
|
-
} catch (error) {
|
|
70
|
-
recordMetric({
|
|
71
|
-
modelType: 'model',
|
|
72
|
-
layer: 'ipfs',
|
|
73
|
-
gateway: gatewayName,
|
|
74
|
-
status: 'error',
|
|
75
|
-
error_type: error.name,
|
|
76
|
-
error_message: error.message,
|
|
77
|
-
latency_ms: Date.now() - startTime
|
|
78
|
-
});
|
|
79
|
-
|
|
80
|
-
if (retry < 1) {
|
|
81
|
-
await new Promise(resolve => setTimeout(resolve, 1000 * (retry + 1)));
|
|
82
|
-
}
|
|
83
|
-
}
|
|
84
|
-
}
|
|
85
|
-
}
|
|
86
16
|
|
|
87
|
-
throw new Error('All IPFS gateways exhausted');
|
|
88
|
-
}
|
|
89
17
|
|
|
90
18
|
async function downloadFromHuggingFace(url, destPath, minBytes, onProgress) {
|
|
91
19
|
const startTime = Date.now();
|
|
@@ -132,13 +60,12 @@ async function downloadFromHuggingFace(url, destPath, minBytes, onProgress) {
|
|
|
132
60
|
|
|
133
61
|
export async function downloadWithFallback(options, onProgress) {
|
|
134
62
|
const {
|
|
135
|
-
|
|
63
|
+
huggingfaceCid,
|
|
136
64
|
huggingfaceUrl,
|
|
137
65
|
destPath,
|
|
138
66
|
manifest,
|
|
139
67
|
minBytes,
|
|
140
|
-
preferredLayer =
|
|
141
|
-
} = options;
|
|
68
|
+
preferredLayer = } = options;
|
|
142
69
|
|
|
143
70
|
const dir = path.dirname(destPath);
|
|
144
71
|
if (!fs.existsSync(dir)) {
|
|
@@ -162,14 +89,13 @@ export async function downloadWithFallback(options, onProgress) {
|
|
|
162
89
|
}
|
|
163
90
|
}
|
|
164
91
|
|
|
165
|
-
const layers = preferredLayer === '
|
|
166
|
-
|
|
167
|
-
: ['huggingface', 'ipfs'];
|
|
92
|
+
const layers = preferredLayer === ? ['huggingface']
|
|
93
|
+
: ['huggingface', ];
|
|
168
94
|
|
|
169
95
|
for (const layer of layers) {
|
|
170
96
|
try {
|
|
171
|
-
if (layer ===
|
|
172
|
-
return await
|
|
97
|
+
if (layer === && huggingfaceCid) {
|
|
98
|
+
return await downloadFromhuggingface(huggingfaceCid, destPath, manifest, onProgress);
|
|
173
99
|
} else if (layer === 'huggingface' && huggingfaceUrl) {
|
|
174
100
|
return await downloadFromHuggingFace(huggingfaceUrl, destPath, minBytes, onProgress);
|
|
175
101
|
}
|
package/package.json
CHANGED
package/server.js
CHANGED
|
@@ -2669,13 +2669,6 @@ const server = http.createServer(async (req, res) => {
|
|
|
2669
2669
|
const { getMetricsSummary } = await import('./lib/model-downloader.js');
|
|
2670
2670
|
const summary = getMetricsSummary();
|
|
2671
2671
|
const health = {
|
|
2672
|
-
ipfs: {
|
|
2673
|
-
status: summary.ipfs.success > 0 ? 'healthy' : summary.ipfs.error > 0 ? 'degraded' : 'unknown',
|
|
2674
|
-
success_rate: summary.ipfs.success + summary.ipfs.error > 0
|
|
2675
|
-
? ((summary.ipfs.success / (summary.ipfs.success + summary.ipfs.error)) * 100).toFixed(2)
|
|
2676
|
-
: 0,
|
|
2677
|
-
avg_latency_ms: summary.ipfs.avg_latency
|
|
2678
|
-
},
|
|
2679
2672
|
huggingface: {
|
|
2680
2673
|
status: summary.huggingface.success > 0 ? 'healthy' : summary.huggingface.error > 0 ? 'degraded' : 'unknown',
|
|
2681
2674
|
success_rate: summary.huggingface.success + summary.huggingface.error > 0
|
package/lib/ipfs-publish.js
DELETED
|
@@ -1,136 +0,0 @@
|
|
|
1
|
-
import { createRequire } from 'module';
|
|
2
|
-
import { create } from 'ipfs-http-client';
|
|
3
|
-
import fs from 'fs';
|
|
4
|
-
import path from 'path';
|
|
5
|
-
import os from 'os';
|
|
6
|
-
|
|
7
|
-
const require = createRequire(import.meta.url);
|
|
8
|
-
|
|
9
|
-
export async function publishToIPFS(dirPath, options = {}) {
|
|
10
|
-
const {
|
|
11
|
-
gateway = '/ip4/127.0.0.1/tcp/5001',
|
|
12
|
-
pinToServices = ['pinata', 'lighthouse'],
|
|
13
|
-
onProgress = null
|
|
14
|
-
} = options;
|
|
15
|
-
|
|
16
|
-
try {
|
|
17
|
-
const ipfs = create({ url: gateway });
|
|
18
|
-
|
|
19
|
-
const dir = path.resolve(dirPath);
|
|
20
|
-
if (!fs.existsSync(dir)) {
|
|
21
|
-
throw new Error(`Directory not found: ${dir}`);
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
const files = [];
|
|
25
|
-
function addFiles(currentPath, basePath) {
|
|
26
|
-
const entries = fs.readdirSync(currentPath, { withFileTypes: true });
|
|
27
|
-
for (const entry of entries) {
|
|
28
|
-
const fullPath = path.join(currentPath, entry.name);
|
|
29
|
-
const relativePath = path.relative(basePath, fullPath);
|
|
30
|
-
|
|
31
|
-
if (entry.isDirectory()) {
|
|
32
|
-
addFiles(fullPath, basePath);
|
|
33
|
-
} else {
|
|
34
|
-
files.push({
|
|
35
|
-
path: relativePath,
|
|
36
|
-
content: fs.readFileSync(fullPath)
|
|
37
|
-
});
|
|
38
|
-
}
|
|
39
|
-
}
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
addFiles(dir, dir);
|
|
43
|
-
|
|
44
|
-
if (onProgress) {
|
|
45
|
-
onProgress({ status: 'preparing', fileCount: files.length });
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
let uploadedCount = 0;
|
|
49
|
-
const results = [];
|
|
50
|
-
|
|
51
|
-
for await (const result of ipfs.addAll(files, { wrapWithDirectory: true, pin: true })) {
|
|
52
|
-
uploadedCount++;
|
|
53
|
-
results.push(result);
|
|
54
|
-
|
|
55
|
-
if (onProgress && result.path !== '') {
|
|
56
|
-
onProgress({
|
|
57
|
-
status: 'uploading',
|
|
58
|
-
file: result.path,
|
|
59
|
-
cid: result.cid.toString(),
|
|
60
|
-
uploaded: uploadedCount,
|
|
61
|
-
total: files.length
|
|
62
|
-
});
|
|
63
|
-
}
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
const rootCID = results[results.length - 1].cid.toString();
|
|
67
|
-
|
|
68
|
-
if (onProgress) {
|
|
69
|
-
onProgress({
|
|
70
|
-
status: 'complete',
|
|
71
|
-
rootCID,
|
|
72
|
-
fileCount: files.length,
|
|
73
|
-
results
|
|
74
|
-
});
|
|
75
|
-
}
|
|
76
|
-
|
|
77
|
-
return {
|
|
78
|
-
rootCID,
|
|
79
|
-
files: results.filter(r => r.path !== '').map(r => ({
|
|
80
|
-
path: r.path,
|
|
81
|
-
cid: r.cid.toString(),
|
|
82
|
-
size: r.size
|
|
83
|
-
}))
|
|
84
|
-
};
|
|
85
|
-
} catch (error) {
|
|
86
|
-
throw new Error(`IPFS publish failed: ${error.message}`);
|
|
87
|
-
}
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
export async function publishModels() {
|
|
91
|
-
const modelsDir = path.join(os.homedir(), '.gmgui', 'models');
|
|
92
|
-
const whisperDir = path.join(modelsDir, 'onnx-community', 'whisper-base');
|
|
93
|
-
const ttsDir = path.join(modelsDir, 'tts');
|
|
94
|
-
|
|
95
|
-
console.log('Publishing models to IPFS...\n');
|
|
96
|
-
|
|
97
|
-
const results = {};
|
|
98
|
-
|
|
99
|
-
if (fs.existsSync(whisperDir)) {
|
|
100
|
-
console.log('Publishing Whisper models...');
|
|
101
|
-
try {
|
|
102
|
-
const whisperResult = await publishToIPFS(whisperDir, {
|
|
103
|
-
onProgress: (progress) => {
|
|
104
|
-
if (progress.status === 'uploading') {
|
|
105
|
-
console.log(` ${progress.uploaded}/${progress.total}: ${progress.file}`);
|
|
106
|
-
} else if (progress.status === 'complete') {
|
|
107
|
-
console.log(`✓ Whisper CID: ${progress.rootCID}\n`);
|
|
108
|
-
}
|
|
109
|
-
}
|
|
110
|
-
});
|
|
111
|
-
results.whisper = whisperResult;
|
|
112
|
-
} catch (error) {
|
|
113
|
-
console.error(`✗ Whisper publish failed: ${error.message}`);
|
|
114
|
-
}
|
|
115
|
-
}
|
|
116
|
-
|
|
117
|
-
if (fs.existsSync(ttsDir)) {
|
|
118
|
-
console.log('Publishing TTS models...');
|
|
119
|
-
try {
|
|
120
|
-
const ttsResult = await publishToIPFS(ttsDir, {
|
|
121
|
-
onProgress: (progress) => {
|
|
122
|
-
if (progress.status === 'uploading') {
|
|
123
|
-
console.log(` ${progress.uploaded}/${progress.total}: ${progress.file}`);
|
|
124
|
-
} else if (progress.status === 'complete') {
|
|
125
|
-
console.log(`✓ TTS CID: ${progress.rootCID}\n`);
|
|
126
|
-
}
|
|
127
|
-
}
|
|
128
|
-
});
|
|
129
|
-
results.tts = ttsResult;
|
|
130
|
-
} catch (error) {
|
|
131
|
-
console.error(`✗ TTS publish failed: ${error.message}`);
|
|
132
|
-
}
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
return results;
|
|
136
|
-
}
|
|
@@ -1,172 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
|
|
3
|
-
import fs from 'fs';
|
|
4
|
-
import path from 'path';
|
|
5
|
-
import os from 'os';
|
|
6
|
-
import https from 'https';
|
|
7
|
-
import FormData from 'form-data';
|
|
8
|
-
|
|
9
|
-
const PINATA_API_KEY = process.env.PINATA_API_KEY || '';
|
|
10
|
-
const PINATA_SECRET_KEY = process.env.PINATA_SECRET_KEY || '';
|
|
11
|
-
|
|
12
|
-
async function uploadToPinata(dirPath, folderName) {
|
|
13
|
-
if (!PINATA_API_KEY || !PINATA_SECRET_KEY) {
|
|
14
|
-
throw new Error('PINATA_API_KEY and PINATA_SECRET_KEY environment variables required');
|
|
15
|
-
}
|
|
16
|
-
|
|
17
|
-
const form = new FormData();
|
|
18
|
-
|
|
19
|
-
function addFilesRecursive(currentPath, basePath) {
|
|
20
|
-
const entries = fs.readdirSync(currentPath, { withFileTypes: true });
|
|
21
|
-
for (const entry of entries) {
|
|
22
|
-
const fullPath = path.join(currentPath, entry.name);
|
|
23
|
-
const relativePath = path.relative(basePath, fullPath);
|
|
24
|
-
|
|
25
|
-
if (entry.isDirectory()) {
|
|
26
|
-
addFilesRecursive(fullPath, basePath);
|
|
27
|
-
} else {
|
|
28
|
-
form.append('file', fs.createReadStream(fullPath), {
|
|
29
|
-
filepath: `${folderName}/${relativePath}`
|
|
30
|
-
});
|
|
31
|
-
}
|
|
32
|
-
}
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
addFilesRecursive(dirPath, dirPath);
|
|
36
|
-
|
|
37
|
-
const metadata = JSON.stringify({
|
|
38
|
-
name: folderName,
|
|
39
|
-
keyvalues: {
|
|
40
|
-
type: 'model',
|
|
41
|
-
timestamp: new Date().toISOString()
|
|
42
|
-
}
|
|
43
|
-
});
|
|
44
|
-
form.append('pinataMetadata', metadata);
|
|
45
|
-
|
|
46
|
-
const options = JSON.stringify({
|
|
47
|
-
wrapWithDirectory: false
|
|
48
|
-
});
|
|
49
|
-
form.append('pinataOptions', options);
|
|
50
|
-
|
|
51
|
-
return new Promise((resolve, reject) => {
|
|
52
|
-
const req = https.request({
|
|
53
|
-
method: 'POST',
|
|
54
|
-
hostname: 'api.pinata.cloud',
|
|
55
|
-
path: '/pinning/pinFileToIPFS',
|
|
56
|
-
headers: {
|
|
57
|
-
...form.getHeaders(),
|
|
58
|
-
pinata_api_key: PINATA_API_KEY,
|
|
59
|
-
pinata_secret_api_key: PINATA_SECRET_KEY
|
|
60
|
-
}
|
|
61
|
-
}, (res) => {
|
|
62
|
-
let data = '';
|
|
63
|
-
res.on('data', chunk => data += chunk);
|
|
64
|
-
res.on('end', () => {
|
|
65
|
-
if (res.statusCode === 200) {
|
|
66
|
-
resolve(JSON.parse(data));
|
|
67
|
-
} else {
|
|
68
|
-
reject(new Error(`Pinata API error: ${res.statusCode} - ${data}`));
|
|
69
|
-
}
|
|
70
|
-
});
|
|
71
|
-
});
|
|
72
|
-
|
|
73
|
-
req.on('error', reject);
|
|
74
|
-
form.pipe(req);
|
|
75
|
-
});
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
async function publishViaPinata() {
|
|
79
|
-
const modelsDir = path.join(os.homedir(), '.gmgui', 'models');
|
|
80
|
-
const whisperDir = path.join(modelsDir, 'onnx-community', 'whisper-base');
|
|
81
|
-
const ttsDir = path.join(modelsDir, 'tts');
|
|
82
|
-
|
|
83
|
-
console.log('Publishing models to IPFS via Pinata...\n');
|
|
84
|
-
|
|
85
|
-
const results = {};
|
|
86
|
-
|
|
87
|
-
if (fs.existsSync(whisperDir)) {
|
|
88
|
-
console.log('Publishing Whisper models...');
|
|
89
|
-
try {
|
|
90
|
-
const result = await uploadToPinata(whisperDir, 'whisper-base');
|
|
91
|
-
results.whisper = {
|
|
92
|
-
cid: result.IpfsHash,
|
|
93
|
-
size: result.PinSize,
|
|
94
|
-
timestamp: result.Timestamp
|
|
95
|
-
};
|
|
96
|
-
console.log(`✓ Whisper CID: ${result.IpfsHash}`);
|
|
97
|
-
console.log(` Size: ${(result.PinSize / 1024 / 1024).toFixed(2)} MB\n`);
|
|
98
|
-
} catch (error) {
|
|
99
|
-
console.error(`✗ Whisper failed: ${error.message}\n`);
|
|
100
|
-
}
|
|
101
|
-
}
|
|
102
|
-
|
|
103
|
-
if (fs.existsSync(ttsDir)) {
|
|
104
|
-
console.log('Publishing TTS models...');
|
|
105
|
-
try {
|
|
106
|
-
const result = await uploadToPinata(ttsDir, 'tts-models');
|
|
107
|
-
results.tts = {
|
|
108
|
-
cid: result.IpfsHash,
|
|
109
|
-
size: result.PinSize,
|
|
110
|
-
timestamp: result.Timestamp
|
|
111
|
-
};
|
|
112
|
-
console.log(`✓ TTS CID: ${result.IpfsHash}`);
|
|
113
|
-
console.log(` Size: ${(result.PinSize / 1024 / 1024).toFixed(2)} MB\n`);
|
|
114
|
-
} catch (error) {
|
|
115
|
-
console.error(`✗ TTS failed: ${error.message}\n`);
|
|
116
|
-
}
|
|
117
|
-
}
|
|
118
|
-
|
|
119
|
-
const manifestPath = path.join(modelsDir, '.manifests.json');
|
|
120
|
-
const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8'));
|
|
121
|
-
|
|
122
|
-
if (results.whisper) {
|
|
123
|
-
manifest['whisper-base'].ipfsHash = results.whisper.cid;
|
|
124
|
-
manifest['whisper-base'].publishedAt = new Date().toISOString();
|
|
125
|
-
}
|
|
126
|
-
if (results.tts) {
|
|
127
|
-
manifest['tts-models'].ipfsHash = results.tts.cid;
|
|
128
|
-
manifest['tts-models'].publishedAt = new Date().toISOString();
|
|
129
|
-
}
|
|
130
|
-
|
|
131
|
-
fs.writeFileSync(manifestPath, JSON.stringify(manifest, null, 2));
|
|
132
|
-
console.log(`✓ Updated manifests at ${manifestPath}`);
|
|
133
|
-
|
|
134
|
-
console.log('\n=== IPFS GATEWAYS ===');
|
|
135
|
-
if (results.whisper) {
|
|
136
|
-
console.log('\nWhisper models:');
|
|
137
|
-
console.log(` Cloudflare: https://cloudflare-ipfs.com/ipfs/${results.whisper.cid}`);
|
|
138
|
-
console.log(` dweb.link: https://dweb.link/ipfs/${results.whisper.cid}`);
|
|
139
|
-
console.log(` Pinata: https://gateway.pinata.cloud/ipfs/${results.whisper.cid}`);
|
|
140
|
-
}
|
|
141
|
-
if (results.tts) {
|
|
142
|
-
console.log('\nTTS models:');
|
|
143
|
-
console.log(` Cloudflare: https://cloudflare-ipfs.com/ipfs/${results.tts.cid}`);
|
|
144
|
-
console.log(` dweb.link: https://dweb.link/ipfs/${results.tts.cid}`);
|
|
145
|
-
console.log(` Pinata: https://gateway.pinata.cloud/ipfs/${results.tts.cid}`);
|
|
146
|
-
}
|
|
147
|
-
|
|
148
|
-
return results;
|
|
149
|
-
}
|
|
150
|
-
|
|
151
|
-
console.log('AgentGUI Model Publishing Tool\n');
|
|
152
|
-
console.log('This script publishes Whisper and TTS models to IPFS via Pinata.');
|
|
153
|
-
console.log('Required: PINATA_API_KEY and PINATA_SECRET_KEY environment variables.\n');
|
|
154
|
-
console.log('Get free API keys at: https://www.pinata.cloud/\n');
|
|
155
|
-
|
|
156
|
-
if (!PINATA_API_KEY || !PINATA_SECRET_KEY) {
|
|
157
|
-
console.error('ERROR: Missing Pinata credentials');
|
|
158
|
-
console.error('Set environment variables:');
|
|
159
|
-
console.error(' export PINATA_API_KEY=your_api_key');
|
|
160
|
-
console.error(' export PINATA_SECRET_KEY=your_secret_key\n');
|
|
161
|
-
process.exit(1);
|
|
162
|
-
}
|
|
163
|
-
|
|
164
|
-
publishViaPinata()
|
|
165
|
-
.then(results => {
|
|
166
|
-
console.log('\n✓ Publishing complete!');
|
|
167
|
-
console.log(JSON.stringify(results, null, 2));
|
|
168
|
-
})
|
|
169
|
-
.catch(error => {
|
|
170
|
-
console.error('\n✗ Publishing failed:', error.message);
|
|
171
|
-
process.exit(1);
|
|
172
|
-
});
|