agentgui 1.0.359 → 1.0.361
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build-portable.js +14 -1
- package/database.js +10 -152
- package/lib/download-metrics.js +0 -12
- package/lib/model-downloader.js +10 -84
- package/package.json +1 -1
- package/server.js +0 -7
- package/lib/ipfs-publish.js +0 -136
- package/scripts/publish-models-to-ipfs.js +0 -172
package/build-portable.js
CHANGED
|
@@ -135,7 +135,20 @@ if (process.env.NO_BUNDLE_MODELS === 'true') {
|
|
|
135
135
|
log('Skipping model bundling (NO_BUNDLE_MODELS=true) - models will download on first use');
|
|
136
136
|
} else {
|
|
137
137
|
log('Bundling AI models...');
|
|
138
|
-
|
|
138
|
+
// Get models from AnEntrypoint/models or local cache
|
|
139
|
+
let modelsDir = process.env.MODELS_SOURCE_DIR || path.join(os.homedir(), '.gmgui', 'models');
|
|
140
|
+
|
|
141
|
+
// If models not present and we're in CI, clone from GitHub
|
|
142
|
+
if (!fs.existsSync(modelsDir) && process.env.CI) {
|
|
143
|
+
console.log('[BUILD] Models not found, cloning from GitHub...');
|
|
144
|
+
const ciModelsDir = path.join(os.tmpdir(), 'models-clone');
|
|
145
|
+
try {
|
|
146
|
+
require('child_process').execSync(`git clone https://github.com/AnEntrypoint/models.git "${ciModelsDir}" --depth 1`, { stdio: 'inherit' });
|
|
147
|
+
modelsDir = ciModelsDir;
|
|
148
|
+
} catch (e) {
|
|
149
|
+
console.error('[BUILD] Failed to clone models from GitHub:', e.message);
|
|
150
|
+
}
|
|
151
|
+
}
|
|
139
152
|
if (fs.existsSync(userModels)) {
|
|
140
153
|
copyDir(userModels, path.join(out, 'models'));
|
|
141
154
|
log(`Models bundled: ${Math.round(sizeOf(path.join(out, 'models')) / 1024 / 1024)}MB`);
|
package/database.js
CHANGED
|
@@ -150,39 +150,6 @@ function initSchema() {
|
|
|
150
150
|
CREATE INDEX IF NOT EXISTS idx_chunks_conv_created ON chunks(conversationId, created_at);
|
|
151
151
|
CREATE INDEX IF NOT EXISTS idx_chunks_sess_created ON chunks(sessionId, created_at);
|
|
152
152
|
|
|
153
|
-
CREATE TABLE IF NOT EXISTS ipfs_cids (
|
|
154
|
-
id TEXT PRIMARY KEY,
|
|
155
|
-
cid TEXT NOT NULL UNIQUE,
|
|
156
|
-
modelName TEXT NOT NULL,
|
|
157
|
-
modelType TEXT NOT NULL,
|
|
158
|
-
modelHash TEXT,
|
|
159
|
-
gatewayUrl TEXT,
|
|
160
|
-
cached_at INTEGER NOT NULL,
|
|
161
|
-
last_accessed_at INTEGER NOT NULL,
|
|
162
|
-
success_count INTEGER DEFAULT 0,
|
|
163
|
-
failure_count INTEGER DEFAULT 0
|
|
164
|
-
);
|
|
165
|
-
|
|
166
|
-
CREATE INDEX IF NOT EXISTS idx_ipfs_cids_model ON ipfs_cids(modelName);
|
|
167
|
-
CREATE INDEX IF NOT EXISTS idx_ipfs_cids_type ON ipfs_cids(modelType);
|
|
168
|
-
CREATE INDEX IF NOT EXISTS idx_ipfs_cids_hash ON ipfs_cids(modelHash);
|
|
169
|
-
|
|
170
|
-
CREATE TABLE IF NOT EXISTS ipfs_downloads (
|
|
171
|
-
id TEXT PRIMARY KEY,
|
|
172
|
-
cidId TEXT NOT NULL,
|
|
173
|
-
downloadPath TEXT NOT NULL,
|
|
174
|
-
status TEXT DEFAULT 'pending',
|
|
175
|
-
downloaded_bytes INTEGER DEFAULT 0,
|
|
176
|
-
total_bytes INTEGER,
|
|
177
|
-
error_message TEXT,
|
|
178
|
-
started_at INTEGER NOT NULL,
|
|
179
|
-
completed_at INTEGER,
|
|
180
|
-
FOREIGN KEY (cidId) REFERENCES ipfs_cids(id)
|
|
181
|
-
);
|
|
182
|
-
|
|
183
|
-
CREATE INDEX IF NOT EXISTS idx_ipfs_downloads_cid ON ipfs_downloads(cidId);
|
|
184
|
-
CREATE INDEX IF NOT EXISTS idx_ipfs_downloads_status ON ipfs_downloads(status);
|
|
185
|
-
CREATE INDEX IF NOT EXISTS idx_ipfs_downloads_started ON ipfs_downloads(started_at);
|
|
186
153
|
`);
|
|
187
154
|
}
|
|
188
155
|
|
|
@@ -305,9 +272,9 @@ try {
|
|
|
305
272
|
console.error('[Migration] Error:', err.message);
|
|
306
273
|
}
|
|
307
274
|
|
|
308
|
-
// Migration: Add resume capability columns to
|
|
275
|
+
// Migration: Add resume capability columns to if needed
|
|
309
276
|
try {
|
|
310
|
-
const result = db.prepare("PRAGMA table_info(
|
|
277
|
+
const result = db.prepare("PRAGMA table_info()").all();
|
|
311
278
|
const columnNames = result.map(r => r.name);
|
|
312
279
|
const resumeColumns = {
|
|
313
280
|
attempts: 'INTEGER DEFAULT 0',
|
|
@@ -318,12 +285,12 @@ try {
|
|
|
318
285
|
|
|
319
286
|
for (const [colName, colDef] of Object.entries(resumeColumns)) {
|
|
320
287
|
if (!columnNames.includes(colName)) {
|
|
321
|
-
db.exec(`ALTER TABLE
|
|
322
|
-
console.log(`[Migration] Added column ${colName} to
|
|
288
|
+
db.exec(`ALTER TABLE ADD COLUMN ${colName} ${colDef}`);
|
|
289
|
+
console.log(`[Migration] Added column ${colName} to table`);
|
|
323
290
|
}
|
|
324
291
|
}
|
|
325
292
|
} catch (err) {
|
|
326
|
-
console.error('[Migration]
|
|
293
|
+
console.error('[Migration] Schema update warning:', err.message);
|
|
327
294
|
}
|
|
328
295
|
|
|
329
296
|
// Migration: Backfill messages for conversations imported without message content
|
|
@@ -383,46 +350,6 @@ try {
|
|
|
383
350
|
console.error('[Migration] Backfill error:', err.message);
|
|
384
351
|
}
|
|
385
352
|
|
|
386
|
-
// Register official IPFS CIDs for voice models
|
|
387
|
-
try {
|
|
388
|
-
const LIGHTHOUSE_GATEWAY = 'https://gateway.lighthouse.storage/ipfs';
|
|
389
|
-
const WHISPER_CID = 'bafybeidyw252ecy4vs46bbmezrtw325gl2ymdltosmzqgx4edjsc3fbofy';
|
|
390
|
-
const TTS_CID = 'bafybeidyw252ecy4vs46bbmezrtw325gl2ymdltosmzqgx4edjsc3fbofy';
|
|
391
|
-
const TTS_TOKENIZER_MODEL_CID = 'bafkreigumf3fvylzkzthrsjqshc7u3zjqtbrxpuzbpy2uywzfrsnsg6d6y';
|
|
392
|
-
|
|
393
|
-
// Check if CIDs are already registered
|
|
394
|
-
const existingWhisper = db.prepare('SELECT * FROM ipfs_cids WHERE modelName = ? AND modelType = ?').get('whisper-base', 'stt');
|
|
395
|
-
if (!existingWhisper) {
|
|
396
|
-
const cidId = `cid-${Date.now()}-whisper`;
|
|
397
|
-
db.prepare(
|
|
398
|
-
`INSERT INTO ipfs_cids (id, cid, modelName, modelType, modelHash, gatewayUrl, cached_at, last_accessed_at)
|
|
399
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
|
|
400
|
-
).run(cidId, WHISPER_CID, 'whisper-base', 'stt', 'sha256-verified', LIGHTHOUSE_GATEWAY, Date.now(), Date.now());
|
|
401
|
-
console.log('[MODELS] Registered Whisper STT IPFS CID:', WHISPER_CID);
|
|
402
|
-
}
|
|
403
|
-
|
|
404
|
-
const existingTTS = db.prepare('SELECT * FROM ipfs_cids WHERE modelName = ? AND modelType = ?').get('tts-models', 'voice');
|
|
405
|
-
if (!existingTTS) {
|
|
406
|
-
const cidId = `cid-${Date.now()}-tts`;
|
|
407
|
-
db.prepare(
|
|
408
|
-
`INSERT INTO ipfs_cids (id, cid, modelName, modelType, modelHash, gatewayUrl, cached_at, last_accessed_at)
|
|
409
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
|
|
410
|
-
).run(cidId, TTS_CID, 'tts-models', 'voice', 'sha256-verified', LIGHTHOUSE_GATEWAY, Date.now(), Date.now());
|
|
411
|
-
console.log('[MODELS] Registered TTS IPFS CID:', TTS_CID);
|
|
412
|
-
}
|
|
413
|
-
|
|
414
|
-
const existingTokenizerModel = db.prepare('SELECT * FROM ipfs_cids WHERE modelName = ? AND modelType = ?').get('tts-tokenizer.model', 'voice-file');
|
|
415
|
-
if (!existingTokenizerModel) {
|
|
416
|
-
const cidId = `cid-${Date.now()}-tts-tokenizer-model`;
|
|
417
|
-
db.prepare(
|
|
418
|
-
`INSERT INTO ipfs_cids (id, cid, modelName, modelType, modelHash, gatewayUrl, cached_at, last_accessed_at)
|
|
419
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
|
|
420
|
-
).run(cidId, TTS_TOKENIZER_MODEL_CID, 'tts-tokenizer.model', 'voice-file', 'd461765ae179566678c93091c5fa6f2984c31bbe990bf1aa62d92c64d91bc3f6', LIGHTHOUSE_GATEWAY, Date.now(), Date.now());
|
|
421
|
-
console.log('[MODELS] Registered TTS tokenizer.model IPFS CID:', TTS_TOKENIZER_MODEL_CID);
|
|
422
|
-
}
|
|
423
|
-
} catch (err) {
|
|
424
|
-
console.warn('[MODELS] IPFS CID registration warning:', err.message);
|
|
425
|
-
}
|
|
426
353
|
|
|
427
354
|
const stmtCache = new Map();
|
|
428
355
|
function prep(sql) {
|
|
@@ -1345,84 +1272,15 @@ export const queries = {
|
|
|
1345
1272
|
return deletedCount;
|
|
1346
1273
|
},
|
|
1347
1274
|
|
|
1348
|
-
recordIpfsCid(cid, modelName, modelType, modelHash, gatewayUrl) {
|
|
1349
|
-
const id = generateId('ipfs');
|
|
1350
|
-
const now = Date.now();
|
|
1351
|
-
const stmt = prep(`
|
|
1352
|
-
INSERT INTO ipfs_cids (id, cid, modelName, modelType, modelHash, gatewayUrl, cached_at, last_accessed_at)
|
|
1353
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
1354
|
-
ON CONFLICT(cid) DO UPDATE SET last_accessed_at = ?, success_count = success_count + 1
|
|
1355
|
-
`);
|
|
1356
|
-
stmt.run(id, cid, modelName, modelType, modelHash, gatewayUrl, now, now, now);
|
|
1357
|
-
const record = this.getIpfsCid(cid);
|
|
1358
|
-
return record ? record.id : id;
|
|
1359
|
-
},
|
|
1360
|
-
|
|
1361
|
-
getIpfsCid(cid) {
|
|
1362
|
-
const stmt = prep('SELECT * FROM ipfs_cids WHERE cid = ?');
|
|
1363
|
-
return stmt.get(cid);
|
|
1364
|
-
},
|
|
1365
|
-
|
|
1366
|
-
getIpfsCidByModel(modelName, modelType) {
|
|
1367
|
-
const stmt = prep('SELECT * FROM ipfs_cids WHERE modelName = ? AND modelType = ? ORDER BY last_accessed_at DESC LIMIT 1');
|
|
1368
|
-
return stmt.get(modelName, modelType);
|
|
1369
|
-
},
|
|
1370
|
-
|
|
1371
|
-
recordDownloadStart(cidId, downloadPath, totalBytes) {
|
|
1372
|
-
const id = generateId('dl');
|
|
1373
|
-
const stmt = prep(`
|
|
1374
|
-
INSERT INTO ipfs_downloads (id, cidId, downloadPath, status, total_bytes, started_at)
|
|
1375
|
-
VALUES (?, ?, ?, ?, ?, ?)
|
|
1376
|
-
`);
|
|
1377
|
-
stmt.run(id, cidId, downloadPath, 'in_progress', totalBytes, Date.now());
|
|
1378
|
-
return id;
|
|
1379
|
-
},
|
|
1380
|
-
|
|
1381
|
-
updateDownloadProgress(downloadId, downloadedBytes) {
|
|
1382
|
-
const stmt = prep(`
|
|
1383
|
-
UPDATE ipfs_downloads SET downloaded_bytes = ? WHERE id = ?
|
|
1384
|
-
`);
|
|
1385
|
-
stmt.run(downloadedBytes, downloadId);
|
|
1386
|
-
},
|
|
1387
|
-
|
|
1388
|
-
completeDownload(downloadId, cidId) {
|
|
1389
|
-
const now = Date.now();
|
|
1390
|
-
prep(`
|
|
1391
|
-
UPDATE ipfs_downloads SET status = ?, completed_at = ? WHERE id = ?
|
|
1392
|
-
`).run('success', now, downloadId);
|
|
1393
|
-
prep(`
|
|
1394
|
-
UPDATE ipfs_cids SET last_accessed_at = ? WHERE id = ?
|
|
1395
|
-
`).run(now, cidId);
|
|
1396
|
-
},
|
|
1397
|
-
|
|
1398
|
-
recordDownloadError(downloadId, cidId, errorMessage) {
|
|
1399
|
-
const now = Date.now();
|
|
1400
|
-
prep(`
|
|
1401
|
-
UPDATE ipfs_downloads SET status = ?, error_message = ?, completed_at = ? WHERE id = ?
|
|
1402
|
-
`).run('failed', errorMessage, now, downloadId);
|
|
1403
|
-
prep(`
|
|
1404
|
-
UPDATE ipfs_cids SET failure_count = failure_count + 1 WHERE id = ?
|
|
1405
|
-
`).run(cidId);
|
|
1406
|
-
},
|
|
1407
|
-
|
|
1408
|
-
getDownload(downloadId) {
|
|
1409
|
-
const stmt = prep('SELECT * FROM ipfs_downloads WHERE id = ?');
|
|
1410
|
-
return stmt.get(downloadId);
|
|
1411
|
-
},
|
|
1412
|
-
|
|
1413
|
-
getDownloadsByCid(cidId) {
|
|
1414
|
-
const stmt = prep('SELECT * FROM ipfs_downloads WHERE cidId = ? ORDER BY started_at DESC');
|
|
1415
|
-
return stmt.all(cidId);
|
|
1416
|
-
},
|
|
1417
1275
|
|
|
1418
1276
|
getDownloadsByStatus(status) {
|
|
1419
|
-
const stmt = prep('SELECT * FROM
|
|
1277
|
+
const stmt = prep('SELECT * FROM WHERE status = ? ORDER BY started_at DESC');
|
|
1420
1278
|
return stmt.all(status);
|
|
1421
1279
|
},
|
|
1422
1280
|
|
|
1423
1281
|
updateDownloadResume(downloadId, currentSize, attempts, lastAttempt, status) {
|
|
1424
1282
|
const stmt = prep(`
|
|
1425
|
-
UPDATE
|
|
1283
|
+
UPDATE
|
|
1426
1284
|
SET downloaded_bytes = ?, attempts = ?, lastAttempt = ?, status = ?
|
|
1427
1285
|
WHERE id = ?
|
|
1428
1286
|
`);
|
|
@@ -1430,17 +1288,17 @@ export const queries = {
|
|
|
1430
1288
|
},
|
|
1431
1289
|
|
|
1432
1290
|
updateDownloadHash(downloadId, hash) {
|
|
1433
|
-
const stmt = prep('UPDATE
|
|
1291
|
+
const stmt = prep('UPDATE SET hash = ? WHERE id = ?');
|
|
1434
1292
|
stmt.run(hash, downloadId);
|
|
1435
1293
|
},
|
|
1436
1294
|
|
|
1437
1295
|
markDownloadResuming(downloadId) {
|
|
1438
|
-
const stmt = prep('UPDATE
|
|
1296
|
+
const stmt = prep('UPDATE SET status = ?, lastAttempt = ? WHERE id = ?');
|
|
1439
1297
|
stmt.run('resuming', Date.now(), downloadId);
|
|
1440
1298
|
},
|
|
1441
1299
|
|
|
1442
1300
|
markDownloadPaused(downloadId, errorMessage) {
|
|
1443
|
-
const stmt = prep('UPDATE
|
|
1301
|
+
const stmt = prep('UPDATE SET status = ?, error_message = ?, lastAttempt = ? WHERE id = ?');
|
|
1444
1302
|
stmt.run('paused', errorMessage, Date.now(), downloadId);
|
|
1445
1303
|
}
|
|
1446
1304
|
};
|
package/lib/download-metrics.js
CHANGED
|
@@ -43,11 +43,6 @@ export function getMetricsSummary() {
|
|
|
43
43
|
const summary = {
|
|
44
44
|
total: metrics.length,
|
|
45
45
|
cache_hits: metrics.filter(m => m.layer === 'cache' && m.status === 'hit').length,
|
|
46
|
-
ipfs: {
|
|
47
|
-
success: metrics.filter(m => m.layer === 'ipfs' && m.status === 'success').length,
|
|
48
|
-
error: metrics.filter(m => m.layer === 'ipfs' && m.status === 'error').length,
|
|
49
|
-
avg_latency: 0
|
|
50
|
-
},
|
|
51
46
|
huggingface: {
|
|
52
47
|
success: metrics.filter(m => m.layer === 'huggingface' && m.status === 'success').length,
|
|
53
48
|
error: metrics.filter(m => m.layer === 'huggingface' && m.status === 'error').length,
|
|
@@ -55,13 +50,6 @@ export function getMetricsSummary() {
|
|
|
55
50
|
}
|
|
56
51
|
};
|
|
57
52
|
|
|
58
|
-
const ipfsSuccess = metrics.filter(m => m.layer === 'ipfs' && m.status === 'success');
|
|
59
|
-
if (ipfsSuccess.length > 0) {
|
|
60
|
-
summary.ipfs.avg_latency = Math.round(
|
|
61
|
-
ipfsSuccess.reduce((sum, m) => sum + m.latency_ms, 0) / ipfsSuccess.length
|
|
62
|
-
);
|
|
63
|
-
}
|
|
64
|
-
|
|
65
53
|
const hfSuccess = metrics.filter(m => m.layer === 'huggingface' && m.status === 'success');
|
|
66
54
|
if (hfSuccess.length > 0) {
|
|
67
55
|
summary.huggingface.avg_latency = Math.round(
|
package/lib/model-downloader.js
CHANGED
|
@@ -7,85 +7,13 @@ import { verifyFileIntegrity } from './file-verification.js';
|
|
|
7
7
|
const require = createRequire(import.meta.url);
|
|
8
8
|
|
|
9
9
|
const GATEWAYS = [
|
|
10
|
-
'https://cloudflare-
|
|
11
|
-
'https://dweb.link/
|
|
12
|
-
'https://gateway.pinata.cloud/
|
|
13
|
-
'https://
|
|
10
|
+
'https://cloudflare-huggingface.com/huggingface/',
|
|
11
|
+
'https://dweb.link/huggingface/',
|
|
12
|
+
'https://gateway.pinata.cloud/huggingface/',
|
|
13
|
+
'https://huggingface.io/huggingface/'
|
|
14
14
|
];
|
|
15
15
|
|
|
16
|
-
async function downloadFromIPFS(cid, destPath, manifest, onProgress) {
|
|
17
|
-
const startTime = Date.now();
|
|
18
|
-
|
|
19
|
-
for (let gatewayIndex = 0; gatewayIndex < GATEWAYS.length; gatewayIndex++) {
|
|
20
|
-
const gateway = GATEWAYS[gatewayIndex];
|
|
21
|
-
const gatewayName = new URL(gateway).hostname;
|
|
22
|
-
|
|
23
|
-
for (let retry = 0; retry < 2; retry++) {
|
|
24
|
-
try {
|
|
25
|
-
if (onProgress) {
|
|
26
|
-
onProgress({
|
|
27
|
-
layer: 'ipfs',
|
|
28
|
-
gateway: gatewayName,
|
|
29
|
-
attempt: retry + 1,
|
|
30
|
-
status: 'attempting'
|
|
31
|
-
});
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
const { downloadWithProgress } = require('webtalk/ipfs-downloader');
|
|
35
|
-
const url = `${gateway}${cid}`;
|
|
36
|
-
|
|
37
|
-
await downloadWithProgress(url, destPath, (progress) => {
|
|
38
|
-
if (onProgress) {
|
|
39
|
-
onProgress({
|
|
40
|
-
layer: 'ipfs',
|
|
41
|
-
gateway: gatewayName,
|
|
42
|
-
status: 'downloading',
|
|
43
|
-
...progress
|
|
44
|
-
});
|
|
45
|
-
}
|
|
46
|
-
});
|
|
47
|
-
|
|
48
|
-
const verification = verifyFileIntegrity(
|
|
49
|
-
destPath,
|
|
50
|
-
manifest?.sha256,
|
|
51
|
-
manifest?.size ? manifest.size * 0.8 : null
|
|
52
|
-
);
|
|
53
|
-
|
|
54
|
-
if (!verification.valid) {
|
|
55
|
-
if (fs.existsSync(destPath)) fs.unlinkSync(destPath);
|
|
56
|
-
throw new Error(`Verification failed: ${verification.reason}`);
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
recordMetric({
|
|
60
|
-
modelType: 'model',
|
|
61
|
-
layer: 'ipfs',
|
|
62
|
-
gateway: gatewayName,
|
|
63
|
-
status: 'success',
|
|
64
|
-
latency_ms: Date.now() - startTime,
|
|
65
|
-
bytes_downloaded: fs.statSync(destPath).size
|
|
66
|
-
});
|
|
67
|
-
|
|
68
|
-
return { success: true, source: 'ipfs', gateway: gatewayName };
|
|
69
|
-
} catch (error) {
|
|
70
|
-
recordMetric({
|
|
71
|
-
modelType: 'model',
|
|
72
|
-
layer: 'ipfs',
|
|
73
|
-
gateway: gatewayName,
|
|
74
|
-
status: 'error',
|
|
75
|
-
error_type: error.name,
|
|
76
|
-
error_message: error.message,
|
|
77
|
-
latency_ms: Date.now() - startTime
|
|
78
|
-
});
|
|
79
|
-
|
|
80
|
-
if (retry < 1) {
|
|
81
|
-
await new Promise(resolve => setTimeout(resolve, 1000 * (retry + 1)));
|
|
82
|
-
}
|
|
83
|
-
}
|
|
84
|
-
}
|
|
85
|
-
}
|
|
86
16
|
|
|
87
|
-
throw new Error('All IPFS gateways exhausted');
|
|
88
|
-
}
|
|
89
17
|
|
|
90
18
|
async function downloadFromHuggingFace(url, destPath, minBytes, onProgress) {
|
|
91
19
|
const startTime = Date.now();
|
|
@@ -132,13 +60,12 @@ async function downloadFromHuggingFace(url, destPath, minBytes, onProgress) {
|
|
|
132
60
|
|
|
133
61
|
export async function downloadWithFallback(options, onProgress) {
|
|
134
62
|
const {
|
|
135
|
-
|
|
63
|
+
huggingfaceCid,
|
|
136
64
|
huggingfaceUrl,
|
|
137
65
|
destPath,
|
|
138
66
|
manifest,
|
|
139
67
|
minBytes,
|
|
140
|
-
preferredLayer =
|
|
141
|
-
} = options;
|
|
68
|
+
preferredLayer = } = options;
|
|
142
69
|
|
|
143
70
|
const dir = path.dirname(destPath);
|
|
144
71
|
if (!fs.existsSync(dir)) {
|
|
@@ -162,14 +89,13 @@ export async function downloadWithFallback(options, onProgress) {
|
|
|
162
89
|
}
|
|
163
90
|
}
|
|
164
91
|
|
|
165
|
-
const layers = preferredLayer === '
|
|
166
|
-
|
|
167
|
-
: ['huggingface', 'ipfs'];
|
|
92
|
+
const layers = preferredLayer === ? ['huggingface']
|
|
93
|
+
: ['huggingface', ];
|
|
168
94
|
|
|
169
95
|
for (const layer of layers) {
|
|
170
96
|
try {
|
|
171
|
-
if (layer ===
|
|
172
|
-
return await
|
|
97
|
+
if (layer === && huggingfaceCid) {
|
|
98
|
+
return await downloadFromhuggingface(huggingfaceCid, destPath, manifest, onProgress);
|
|
173
99
|
} else if (layer === 'huggingface' && huggingfaceUrl) {
|
|
174
100
|
return await downloadFromHuggingFace(huggingfaceUrl, destPath, minBytes, onProgress);
|
|
175
101
|
}
|
package/package.json
CHANGED
package/server.js
CHANGED
|
@@ -2669,13 +2669,6 @@ const server = http.createServer(async (req, res) => {
|
|
|
2669
2669
|
const { getMetricsSummary } = await import('./lib/model-downloader.js');
|
|
2670
2670
|
const summary = getMetricsSummary();
|
|
2671
2671
|
const health = {
|
|
2672
|
-
ipfs: {
|
|
2673
|
-
status: summary.ipfs.success > 0 ? 'healthy' : summary.ipfs.error > 0 ? 'degraded' : 'unknown',
|
|
2674
|
-
success_rate: summary.ipfs.success + summary.ipfs.error > 0
|
|
2675
|
-
? ((summary.ipfs.success / (summary.ipfs.success + summary.ipfs.error)) * 100).toFixed(2)
|
|
2676
|
-
: 0,
|
|
2677
|
-
avg_latency_ms: summary.ipfs.avg_latency
|
|
2678
|
-
},
|
|
2679
2672
|
huggingface: {
|
|
2680
2673
|
status: summary.huggingface.success > 0 ? 'healthy' : summary.huggingface.error > 0 ? 'degraded' : 'unknown',
|
|
2681
2674
|
success_rate: summary.huggingface.success + summary.huggingface.error > 0
|
package/lib/ipfs-publish.js
DELETED
|
@@ -1,136 +0,0 @@
|
|
|
1
|
-
import { createRequire } from 'module';
|
|
2
|
-
import { create } from 'ipfs-http-client';
|
|
3
|
-
import fs from 'fs';
|
|
4
|
-
import path from 'path';
|
|
5
|
-
import os from 'os';
|
|
6
|
-
|
|
7
|
-
const require = createRequire(import.meta.url);
|
|
8
|
-
|
|
9
|
-
export async function publishToIPFS(dirPath, options = {}) {
|
|
10
|
-
const {
|
|
11
|
-
gateway = '/ip4/127.0.0.1/tcp/5001',
|
|
12
|
-
pinToServices = ['pinata', 'lighthouse'],
|
|
13
|
-
onProgress = null
|
|
14
|
-
} = options;
|
|
15
|
-
|
|
16
|
-
try {
|
|
17
|
-
const ipfs = create({ url: gateway });
|
|
18
|
-
|
|
19
|
-
const dir = path.resolve(dirPath);
|
|
20
|
-
if (!fs.existsSync(dir)) {
|
|
21
|
-
throw new Error(`Directory not found: ${dir}`);
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
const files = [];
|
|
25
|
-
function addFiles(currentPath, basePath) {
|
|
26
|
-
const entries = fs.readdirSync(currentPath, { withFileTypes: true });
|
|
27
|
-
for (const entry of entries) {
|
|
28
|
-
const fullPath = path.join(currentPath, entry.name);
|
|
29
|
-
const relativePath = path.relative(basePath, fullPath);
|
|
30
|
-
|
|
31
|
-
if (entry.isDirectory()) {
|
|
32
|
-
addFiles(fullPath, basePath);
|
|
33
|
-
} else {
|
|
34
|
-
files.push({
|
|
35
|
-
path: relativePath,
|
|
36
|
-
content: fs.readFileSync(fullPath)
|
|
37
|
-
});
|
|
38
|
-
}
|
|
39
|
-
}
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
addFiles(dir, dir);
|
|
43
|
-
|
|
44
|
-
if (onProgress) {
|
|
45
|
-
onProgress({ status: 'preparing', fileCount: files.length });
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
let uploadedCount = 0;
|
|
49
|
-
const results = [];
|
|
50
|
-
|
|
51
|
-
for await (const result of ipfs.addAll(files, { wrapWithDirectory: true, pin: true })) {
|
|
52
|
-
uploadedCount++;
|
|
53
|
-
results.push(result);
|
|
54
|
-
|
|
55
|
-
if (onProgress && result.path !== '') {
|
|
56
|
-
onProgress({
|
|
57
|
-
status: 'uploading',
|
|
58
|
-
file: result.path,
|
|
59
|
-
cid: result.cid.toString(),
|
|
60
|
-
uploaded: uploadedCount,
|
|
61
|
-
total: files.length
|
|
62
|
-
});
|
|
63
|
-
}
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
const rootCID = results[results.length - 1].cid.toString();
|
|
67
|
-
|
|
68
|
-
if (onProgress) {
|
|
69
|
-
onProgress({
|
|
70
|
-
status: 'complete',
|
|
71
|
-
rootCID,
|
|
72
|
-
fileCount: files.length,
|
|
73
|
-
results
|
|
74
|
-
});
|
|
75
|
-
}
|
|
76
|
-
|
|
77
|
-
return {
|
|
78
|
-
rootCID,
|
|
79
|
-
files: results.filter(r => r.path !== '').map(r => ({
|
|
80
|
-
path: r.path,
|
|
81
|
-
cid: r.cid.toString(),
|
|
82
|
-
size: r.size
|
|
83
|
-
}))
|
|
84
|
-
};
|
|
85
|
-
} catch (error) {
|
|
86
|
-
throw new Error(`IPFS publish failed: ${error.message}`);
|
|
87
|
-
}
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
export async function publishModels() {
|
|
91
|
-
const modelsDir = path.join(os.homedir(), '.gmgui', 'models');
|
|
92
|
-
const whisperDir = path.join(modelsDir, 'onnx-community', 'whisper-base');
|
|
93
|
-
const ttsDir = path.join(modelsDir, 'tts');
|
|
94
|
-
|
|
95
|
-
console.log('Publishing models to IPFS...\n');
|
|
96
|
-
|
|
97
|
-
const results = {};
|
|
98
|
-
|
|
99
|
-
if (fs.existsSync(whisperDir)) {
|
|
100
|
-
console.log('Publishing Whisper models...');
|
|
101
|
-
try {
|
|
102
|
-
const whisperResult = await publishToIPFS(whisperDir, {
|
|
103
|
-
onProgress: (progress) => {
|
|
104
|
-
if (progress.status === 'uploading') {
|
|
105
|
-
console.log(` ${progress.uploaded}/${progress.total}: ${progress.file}`);
|
|
106
|
-
} else if (progress.status === 'complete') {
|
|
107
|
-
console.log(`✓ Whisper CID: ${progress.rootCID}\n`);
|
|
108
|
-
}
|
|
109
|
-
}
|
|
110
|
-
});
|
|
111
|
-
results.whisper = whisperResult;
|
|
112
|
-
} catch (error) {
|
|
113
|
-
console.error(`✗ Whisper publish failed: ${error.message}`);
|
|
114
|
-
}
|
|
115
|
-
}
|
|
116
|
-
|
|
117
|
-
if (fs.existsSync(ttsDir)) {
|
|
118
|
-
console.log('Publishing TTS models...');
|
|
119
|
-
try {
|
|
120
|
-
const ttsResult = await publishToIPFS(ttsDir, {
|
|
121
|
-
onProgress: (progress) => {
|
|
122
|
-
if (progress.status === 'uploading') {
|
|
123
|
-
console.log(` ${progress.uploaded}/${progress.total}: ${progress.file}`);
|
|
124
|
-
} else if (progress.status === 'complete') {
|
|
125
|
-
console.log(`✓ TTS CID: ${progress.rootCID}\n`);
|
|
126
|
-
}
|
|
127
|
-
}
|
|
128
|
-
});
|
|
129
|
-
results.tts = ttsResult;
|
|
130
|
-
} catch (error) {
|
|
131
|
-
console.error(`✗ TTS publish failed: ${error.message}`);
|
|
132
|
-
}
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
return results;
|
|
136
|
-
}
|
|
@@ -1,172 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
|
|
3
|
-
import fs from 'fs';
|
|
4
|
-
import path from 'path';
|
|
5
|
-
import os from 'os';
|
|
6
|
-
import https from 'https';
|
|
7
|
-
import FormData from 'form-data';
|
|
8
|
-
|
|
9
|
-
const PINATA_API_KEY = process.env.PINATA_API_KEY || '';
|
|
10
|
-
const PINATA_SECRET_KEY = process.env.PINATA_SECRET_KEY || '';
|
|
11
|
-
|
|
12
|
-
async function uploadToPinata(dirPath, folderName) {
|
|
13
|
-
if (!PINATA_API_KEY || !PINATA_SECRET_KEY) {
|
|
14
|
-
throw new Error('PINATA_API_KEY and PINATA_SECRET_KEY environment variables required');
|
|
15
|
-
}
|
|
16
|
-
|
|
17
|
-
const form = new FormData();
|
|
18
|
-
|
|
19
|
-
function addFilesRecursive(currentPath, basePath) {
|
|
20
|
-
const entries = fs.readdirSync(currentPath, { withFileTypes: true });
|
|
21
|
-
for (const entry of entries) {
|
|
22
|
-
const fullPath = path.join(currentPath, entry.name);
|
|
23
|
-
const relativePath = path.relative(basePath, fullPath);
|
|
24
|
-
|
|
25
|
-
if (entry.isDirectory()) {
|
|
26
|
-
addFilesRecursive(fullPath, basePath);
|
|
27
|
-
} else {
|
|
28
|
-
form.append('file', fs.createReadStream(fullPath), {
|
|
29
|
-
filepath: `${folderName}/${relativePath}`
|
|
30
|
-
});
|
|
31
|
-
}
|
|
32
|
-
}
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
addFilesRecursive(dirPath, dirPath);
|
|
36
|
-
|
|
37
|
-
const metadata = JSON.stringify({
|
|
38
|
-
name: folderName,
|
|
39
|
-
keyvalues: {
|
|
40
|
-
type: 'model',
|
|
41
|
-
timestamp: new Date().toISOString()
|
|
42
|
-
}
|
|
43
|
-
});
|
|
44
|
-
form.append('pinataMetadata', metadata);
|
|
45
|
-
|
|
46
|
-
const options = JSON.stringify({
|
|
47
|
-
wrapWithDirectory: false
|
|
48
|
-
});
|
|
49
|
-
form.append('pinataOptions', options);
|
|
50
|
-
|
|
51
|
-
return new Promise((resolve, reject) => {
|
|
52
|
-
const req = https.request({
|
|
53
|
-
method: 'POST',
|
|
54
|
-
hostname: 'api.pinata.cloud',
|
|
55
|
-
path: '/pinning/pinFileToIPFS',
|
|
56
|
-
headers: {
|
|
57
|
-
...form.getHeaders(),
|
|
58
|
-
pinata_api_key: PINATA_API_KEY,
|
|
59
|
-
pinata_secret_api_key: PINATA_SECRET_KEY
|
|
60
|
-
}
|
|
61
|
-
}, (res) => {
|
|
62
|
-
let data = '';
|
|
63
|
-
res.on('data', chunk => data += chunk);
|
|
64
|
-
res.on('end', () => {
|
|
65
|
-
if (res.statusCode === 200) {
|
|
66
|
-
resolve(JSON.parse(data));
|
|
67
|
-
} else {
|
|
68
|
-
reject(new Error(`Pinata API error: ${res.statusCode} - ${data}`));
|
|
69
|
-
}
|
|
70
|
-
});
|
|
71
|
-
});
|
|
72
|
-
|
|
73
|
-
req.on('error', reject);
|
|
74
|
-
form.pipe(req);
|
|
75
|
-
});
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
async function publishViaPinata() {
|
|
79
|
-
const modelsDir = path.join(os.homedir(), '.gmgui', 'models');
|
|
80
|
-
const whisperDir = path.join(modelsDir, 'onnx-community', 'whisper-base');
|
|
81
|
-
const ttsDir = path.join(modelsDir, 'tts');
|
|
82
|
-
|
|
83
|
-
console.log('Publishing models to IPFS via Pinata...\n');
|
|
84
|
-
|
|
85
|
-
const results = {};
|
|
86
|
-
|
|
87
|
-
if (fs.existsSync(whisperDir)) {
|
|
88
|
-
console.log('Publishing Whisper models...');
|
|
89
|
-
try {
|
|
90
|
-
const result = await uploadToPinata(whisperDir, 'whisper-base');
|
|
91
|
-
results.whisper = {
|
|
92
|
-
cid: result.IpfsHash,
|
|
93
|
-
size: result.PinSize,
|
|
94
|
-
timestamp: result.Timestamp
|
|
95
|
-
};
|
|
96
|
-
console.log(`✓ Whisper CID: ${result.IpfsHash}`);
|
|
97
|
-
console.log(` Size: ${(result.PinSize / 1024 / 1024).toFixed(2)} MB\n`);
|
|
98
|
-
} catch (error) {
|
|
99
|
-
console.error(`✗ Whisper failed: ${error.message}\n`);
|
|
100
|
-
}
|
|
101
|
-
}
|
|
102
|
-
|
|
103
|
-
if (fs.existsSync(ttsDir)) {
|
|
104
|
-
console.log('Publishing TTS models...');
|
|
105
|
-
try {
|
|
106
|
-
const result = await uploadToPinata(ttsDir, 'tts-models');
|
|
107
|
-
results.tts = {
|
|
108
|
-
cid: result.IpfsHash,
|
|
109
|
-
size: result.PinSize,
|
|
110
|
-
timestamp: result.Timestamp
|
|
111
|
-
};
|
|
112
|
-
console.log(`✓ TTS CID: ${result.IpfsHash}`);
|
|
113
|
-
console.log(` Size: ${(result.PinSize / 1024 / 1024).toFixed(2)} MB\n`);
|
|
114
|
-
} catch (error) {
|
|
115
|
-
console.error(`✗ TTS failed: ${error.message}\n`);
|
|
116
|
-
}
|
|
117
|
-
}
|
|
118
|
-
|
|
119
|
-
const manifestPath = path.join(modelsDir, '.manifests.json');
|
|
120
|
-
const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8'));
|
|
121
|
-
|
|
122
|
-
if (results.whisper) {
|
|
123
|
-
manifest['whisper-base'].ipfsHash = results.whisper.cid;
|
|
124
|
-
manifest['whisper-base'].publishedAt = new Date().toISOString();
|
|
125
|
-
}
|
|
126
|
-
if (results.tts) {
|
|
127
|
-
manifest['tts-models'].ipfsHash = results.tts.cid;
|
|
128
|
-
manifest['tts-models'].publishedAt = new Date().toISOString();
|
|
129
|
-
}
|
|
130
|
-
|
|
131
|
-
fs.writeFileSync(manifestPath, JSON.stringify(manifest, null, 2));
|
|
132
|
-
console.log(`✓ Updated manifests at ${manifestPath}`);
|
|
133
|
-
|
|
134
|
-
console.log('\n=== IPFS GATEWAYS ===');
|
|
135
|
-
if (results.whisper) {
|
|
136
|
-
console.log('\nWhisper models:');
|
|
137
|
-
console.log(` Cloudflare: https://cloudflare-ipfs.com/ipfs/${results.whisper.cid}`);
|
|
138
|
-
console.log(` dweb.link: https://dweb.link/ipfs/${results.whisper.cid}`);
|
|
139
|
-
console.log(` Pinata: https://gateway.pinata.cloud/ipfs/${results.whisper.cid}`);
|
|
140
|
-
}
|
|
141
|
-
if (results.tts) {
|
|
142
|
-
console.log('\nTTS models:');
|
|
143
|
-
console.log(` Cloudflare: https://cloudflare-ipfs.com/ipfs/${results.tts.cid}`);
|
|
144
|
-
console.log(` dweb.link: https://dweb.link/ipfs/${results.tts.cid}`);
|
|
145
|
-
console.log(` Pinata: https://gateway.pinata.cloud/ipfs/${results.tts.cid}`);
|
|
146
|
-
}
|
|
147
|
-
|
|
148
|
-
return results;
|
|
149
|
-
}
|
|
150
|
-
|
|
151
|
-
console.log('AgentGUI Model Publishing Tool\n');
|
|
152
|
-
console.log('This script publishes Whisper and TTS models to IPFS via Pinata.');
|
|
153
|
-
console.log('Required: PINATA_API_KEY and PINATA_SECRET_KEY environment variables.\n');
|
|
154
|
-
console.log('Get free API keys at: https://www.pinata.cloud/\n');
|
|
155
|
-
|
|
156
|
-
if (!PINATA_API_KEY || !PINATA_SECRET_KEY) {
|
|
157
|
-
console.error('ERROR: Missing Pinata credentials');
|
|
158
|
-
console.error('Set environment variables:');
|
|
159
|
-
console.error(' export PINATA_API_KEY=your_api_key');
|
|
160
|
-
console.error(' export PINATA_SECRET_KEY=your_secret_key\n');
|
|
161
|
-
process.exit(1);
|
|
162
|
-
}
|
|
163
|
-
|
|
164
|
-
publishViaPinata()
|
|
165
|
-
.then(results => {
|
|
166
|
-
console.log('\n✓ Publishing complete!');
|
|
167
|
-
console.log(JSON.stringify(results, null, 2));
|
|
168
|
-
})
|
|
169
|
-
.catch(error => {
|
|
170
|
-
console.error('\n✗ Publishing failed:', error.message);
|
|
171
|
-
process.exit(1);
|
|
172
|
-
});
|