@ruvector/edge-net 0.5.0 → 0.5.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +281 -10
- package/core-invariants.js +942 -0
- package/models/adapter-hub.js +1008 -0
- package/models/adapter-security.js +792 -0
- package/models/benchmark.js +688 -0
- package/models/distribution.js +791 -0
- package/models/index.js +109 -0
- package/models/integrity.js +753 -0
- package/models/loader.js +725 -0
- package/models/microlora.js +1298 -0
- package/models/model-loader.js +922 -0
- package/models/model-optimizer.js +1245 -0
- package/models/model-registry.js +696 -0
- package/models/model-utils.js +548 -0
- package/models/models-cli.js +914 -0
- package/models/registry.json +214 -0
- package/models/training-utils.js +1418 -0
- package/models/wasm-core.js +1025 -0
- package/network-genesis.js +2847 -0
- package/onnx-worker.js +462 -8
- package/package.json +33 -3
- package/plugins/SECURITY-AUDIT.md +654 -0
- package/plugins/cli.js +43 -3
- package/plugins/implementations/e2e-encryption.js +57 -12
- package/plugins/plugin-loader.js +610 -21
- package/tests/model-optimizer.test.js +644 -0
- package/tests/network-genesis.test.js +562 -0
- package/tests/plugin-benchmark.js +1239 -0
- package/tests/plugin-system-test.js +163 -0
- package/tests/wasm-core.test.js +368 -0
|
@@ -0,0 +1,922 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @ruvector/edge-net Model Loader
|
|
3
|
+
*
|
|
4
|
+
* Smart model loading with:
|
|
5
|
+
* - IndexedDB caching
|
|
6
|
+
* - Automatic source selection (CDN -> GCS -> IPFS -> fallback)
|
|
7
|
+
* - Streaming download with progress
|
|
8
|
+
* - Model validation before use
|
|
9
|
+
* - Lazy loading support
|
|
10
|
+
*
|
|
11
|
+
* @module @ruvector/edge-net/models/model-loader
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import { EventEmitter } from 'events';
|
|
15
|
+
import { createHash, randomBytes } from 'crypto';
|
|
16
|
+
import { promises as fs } from 'fs';
|
|
17
|
+
import path from 'path';
|
|
18
|
+
|
|
19
|
+
import { ModelRegistry } from './model-registry.js';
|
|
20
|
+
import { DistributionManager, ProgressTracker } from './distribution.js';
|
|
21
|
+
|
|
22
|
+
// ============================================
|
|
23
|
+
// CONSTANTS
|
|
24
|
+
// ============================================
|
|
25
|
+
|
|
26
|
+
const DEFAULT_CACHE_DIR = process.env.HOME
|
|
27
|
+
? `${process.env.HOME}/.ruvector/models/cache`
|
|
28
|
+
: '/tmp/.ruvector/models/cache';
|
|
29
|
+
|
|
30
|
+
const CACHE_VERSION = 1;
|
|
31
|
+
const MAX_CACHE_SIZE_BYTES = 10 * 1024 * 1024 * 1024; // 10GB default
|
|
32
|
+
const CACHE_CLEANUP_THRESHOLD = 0.9; // Cleanup when 90% full
|
|
33
|
+
|
|
34
|
+
// ============================================
|
|
35
|
+
// CACHE STORAGE INTERFACE
|
|
36
|
+
// ============================================
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Cache storage interface for different backends
|
|
40
|
+
*/
|
|
41
|
+
class CacheStorage {
|
|
42
|
+
async get(key) { throw new Error('Not implemented'); }
|
|
43
|
+
async set(key, value, metadata) { throw new Error('Not implemented'); }
|
|
44
|
+
async delete(key) { throw new Error('Not implemented'); }
|
|
45
|
+
async has(key) { throw new Error('Not implemented'); }
|
|
46
|
+
async list() { throw new Error('Not implemented'); }
|
|
47
|
+
async getMetadata(key) { throw new Error('Not implemented'); }
|
|
48
|
+
async clear() { throw new Error('Not implemented'); }
|
|
49
|
+
async getSize() { throw new Error('Not implemented'); }
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
// ============================================
|
|
53
|
+
// FILE SYSTEM CACHE
|
|
54
|
+
// ============================================
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* File system-based cache storage for Node.js
|
|
58
|
+
*/
|
|
59
|
+
class FileSystemCache extends CacheStorage {
|
|
60
|
+
constructor(cacheDir) {
|
|
61
|
+
super();
|
|
62
|
+
this.cacheDir = cacheDir;
|
|
63
|
+
this.metadataDir = path.join(cacheDir, '.metadata');
|
|
64
|
+
this.initialized = false;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
async init() {
|
|
68
|
+
if (this.initialized) return;
|
|
69
|
+
await fs.mkdir(this.cacheDir, { recursive: true });
|
|
70
|
+
await fs.mkdir(this.metadataDir, { recursive: true });
|
|
71
|
+
this.initialized = true;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
_getFilePath(key) {
|
|
75
|
+
// Sanitize key for filesystem
|
|
76
|
+
const safeKey = key.replace(/[^a-zA-Z0-9._-]/g, '_');
|
|
77
|
+
return path.join(this.cacheDir, safeKey);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
_getMetadataPath(key) {
|
|
81
|
+
const safeKey = key.replace(/[^a-zA-Z0-9._-]/g, '_');
|
|
82
|
+
return path.join(this.metadataDir, `${safeKey}.json`);
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
async get(key) {
|
|
86
|
+
await this.init();
|
|
87
|
+
const filePath = this._getFilePath(key);
|
|
88
|
+
|
|
89
|
+
try {
|
|
90
|
+
const data = await fs.readFile(filePath);
|
|
91
|
+
|
|
92
|
+
// Update access time in metadata
|
|
93
|
+
await this._updateAccessTime(key);
|
|
94
|
+
|
|
95
|
+
return data;
|
|
96
|
+
} catch (error) {
|
|
97
|
+
if (error.code === 'ENOENT') return null;
|
|
98
|
+
throw error;
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
async set(key, value, metadata = {}) {
|
|
103
|
+
await this.init();
|
|
104
|
+
const filePath = this._getFilePath(key);
|
|
105
|
+
const metadataPath = this._getMetadataPath(key);
|
|
106
|
+
|
|
107
|
+
// Write data
|
|
108
|
+
await fs.writeFile(filePath, value);
|
|
109
|
+
|
|
110
|
+
// Write metadata
|
|
111
|
+
const fullMetadata = {
|
|
112
|
+
key,
|
|
113
|
+
size: value.length,
|
|
114
|
+
hash: `sha256:${createHash('sha256').update(value).digest('hex')}`,
|
|
115
|
+
createdAt: new Date().toISOString(),
|
|
116
|
+
accessedAt: new Date().toISOString(),
|
|
117
|
+
accessCount: 1,
|
|
118
|
+
cacheVersion: CACHE_VERSION,
|
|
119
|
+
...metadata,
|
|
120
|
+
};
|
|
121
|
+
|
|
122
|
+
await fs.writeFile(metadataPath, JSON.stringify(fullMetadata, null, 2));
|
|
123
|
+
|
|
124
|
+
return fullMetadata;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
async delete(key) {
|
|
128
|
+
await this.init();
|
|
129
|
+
const filePath = this._getFilePath(key);
|
|
130
|
+
const metadataPath = this._getMetadataPath(key);
|
|
131
|
+
|
|
132
|
+
try {
|
|
133
|
+
await fs.unlink(filePath);
|
|
134
|
+
await fs.unlink(metadataPath).catch(() => {});
|
|
135
|
+
return true;
|
|
136
|
+
} catch (error) {
|
|
137
|
+
if (error.code === 'ENOENT') return false;
|
|
138
|
+
throw error;
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
async has(key) {
|
|
143
|
+
await this.init();
|
|
144
|
+
const filePath = this._getFilePath(key);
|
|
145
|
+
|
|
146
|
+
try {
|
|
147
|
+
await fs.access(filePath);
|
|
148
|
+
return true;
|
|
149
|
+
} catch {
|
|
150
|
+
return false;
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
async list() {
|
|
155
|
+
await this.init();
|
|
156
|
+
|
|
157
|
+
try {
|
|
158
|
+
const files = await fs.readdir(this.cacheDir);
|
|
159
|
+
return files.filter(f => !f.startsWith('.'));
|
|
160
|
+
} catch {
|
|
161
|
+
return [];
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
async getMetadata(key) {
|
|
166
|
+
await this.init();
|
|
167
|
+
const metadataPath = this._getMetadataPath(key);
|
|
168
|
+
|
|
169
|
+
try {
|
|
170
|
+
const data = await fs.readFile(metadataPath, 'utf-8');
|
|
171
|
+
return JSON.parse(data);
|
|
172
|
+
} catch {
|
|
173
|
+
return null;
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
async _updateAccessTime(key) {
|
|
178
|
+
const metadataPath = this._getMetadataPath(key);
|
|
179
|
+
|
|
180
|
+
try {
|
|
181
|
+
const data = await fs.readFile(metadataPath, 'utf-8');
|
|
182
|
+
const metadata = JSON.parse(data);
|
|
183
|
+
|
|
184
|
+
metadata.accessedAt = new Date().toISOString();
|
|
185
|
+
metadata.accessCount = (metadata.accessCount || 0) + 1;
|
|
186
|
+
|
|
187
|
+
await fs.writeFile(metadataPath, JSON.stringify(metadata, null, 2));
|
|
188
|
+
} catch {
|
|
189
|
+
// Ignore metadata update errors
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
async clear() {
|
|
194
|
+
await this.init();
|
|
195
|
+
const files = await this.list();
|
|
196
|
+
|
|
197
|
+
for (const file of files) {
|
|
198
|
+
await this.delete(file);
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
async getSize() {
|
|
203
|
+
await this.init();
|
|
204
|
+
const files = await this.list();
|
|
205
|
+
let totalSize = 0;
|
|
206
|
+
|
|
207
|
+
for (const file of files) {
|
|
208
|
+
const filePath = this._getFilePath(file);
|
|
209
|
+
try {
|
|
210
|
+
const stats = await fs.stat(filePath);
|
|
211
|
+
totalSize += stats.size;
|
|
212
|
+
} catch {
|
|
213
|
+
// Ignore missing files
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
return totalSize;
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
async getEntriesWithMetadata() {
|
|
221
|
+
await this.init();
|
|
222
|
+
const files = await this.list();
|
|
223
|
+
const entries = [];
|
|
224
|
+
|
|
225
|
+
for (const file of files) {
|
|
226
|
+
const metadata = await this.getMetadata(file);
|
|
227
|
+
if (metadata) {
|
|
228
|
+
entries.push(metadata);
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
return entries;
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
// ============================================
|
|
237
|
+
// INDEXEDDB CACHE (BROWSER)
|
|
238
|
+
// ============================================
|
|
239
|
+
|
|
240
|
+
/**
|
|
241
|
+
* IndexedDB-based cache storage for browsers
|
|
242
|
+
*/
|
|
243
|
+
class IndexedDBCache extends CacheStorage {
|
|
244
|
+
constructor(dbName = 'ruvector-models') {
|
|
245
|
+
super();
|
|
246
|
+
this.dbName = dbName;
|
|
247
|
+
this.storeName = 'models';
|
|
248
|
+
this.metadataStoreName = 'metadata';
|
|
249
|
+
this.db = null;
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
async init() {
|
|
253
|
+
if (this.db) return;
|
|
254
|
+
|
|
255
|
+
if (typeof indexedDB === 'undefined') {
|
|
256
|
+
throw new Error('IndexedDB not available');
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
return new Promise((resolve, reject) => {
|
|
260
|
+
const request = indexedDB.open(this.dbName, CACHE_VERSION);
|
|
261
|
+
|
|
262
|
+
request.onerror = () => reject(request.error);
|
|
263
|
+
|
|
264
|
+
request.onsuccess = () => {
|
|
265
|
+
this.db = request.result;
|
|
266
|
+
resolve();
|
|
267
|
+
};
|
|
268
|
+
|
|
269
|
+
request.onupgradeneeded = (event) => {
|
|
270
|
+
const db = event.target.result;
|
|
271
|
+
|
|
272
|
+
// Models store
|
|
273
|
+
if (!db.objectStoreNames.contains(this.storeName)) {
|
|
274
|
+
db.createObjectStore(this.storeName);
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
// Metadata store
|
|
278
|
+
if (!db.objectStoreNames.contains(this.metadataStoreName)) {
|
|
279
|
+
const metaStore = db.createObjectStore(this.metadataStoreName);
|
|
280
|
+
metaStore.createIndex('accessedAt', 'accessedAt');
|
|
281
|
+
metaStore.createIndex('size', 'size');
|
|
282
|
+
}
|
|
283
|
+
};
|
|
284
|
+
});
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
async get(key) {
|
|
288
|
+
await this.init();
|
|
289
|
+
|
|
290
|
+
return new Promise((resolve, reject) => {
|
|
291
|
+
const transaction = this.db.transaction([this.storeName], 'readonly');
|
|
292
|
+
const store = transaction.objectStore(this.storeName);
|
|
293
|
+
const request = store.get(key);
|
|
294
|
+
|
|
295
|
+
request.onerror = () => reject(request.error);
|
|
296
|
+
request.onsuccess = () => {
|
|
297
|
+
if (request.result) {
|
|
298
|
+
this._updateAccessTime(key);
|
|
299
|
+
}
|
|
300
|
+
resolve(request.result || null);
|
|
301
|
+
};
|
|
302
|
+
});
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
async set(key, value, metadata = {}) {
|
|
306
|
+
await this.init();
|
|
307
|
+
|
|
308
|
+
const fullMetadata = {
|
|
309
|
+
key,
|
|
310
|
+
size: value.length || value.byteLength,
|
|
311
|
+
hash: await this._computeHash(value),
|
|
312
|
+
createdAt: new Date().toISOString(),
|
|
313
|
+
accessedAt: new Date().toISOString(),
|
|
314
|
+
accessCount: 1,
|
|
315
|
+
cacheVersion: CACHE_VERSION,
|
|
316
|
+
...metadata,
|
|
317
|
+
};
|
|
318
|
+
|
|
319
|
+
return new Promise((resolve, reject) => {
|
|
320
|
+
const transaction = this.db.transaction(
|
|
321
|
+
[this.storeName, this.metadataStoreName],
|
|
322
|
+
'readwrite'
|
|
323
|
+
);
|
|
324
|
+
|
|
325
|
+
const modelStore = transaction.objectStore(this.storeName);
|
|
326
|
+
const metaStore = transaction.objectStore(this.metadataStoreName);
|
|
327
|
+
|
|
328
|
+
modelStore.put(value, key);
|
|
329
|
+
metaStore.put(fullMetadata, key);
|
|
330
|
+
|
|
331
|
+
transaction.oncomplete = () => resolve(fullMetadata);
|
|
332
|
+
transaction.onerror = () => reject(transaction.error);
|
|
333
|
+
});
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
async _computeHash(data) {
|
|
337
|
+
if (typeof crypto !== 'undefined' && crypto.subtle) {
|
|
338
|
+
const buffer = data instanceof ArrayBuffer ? data : data.buffer;
|
|
339
|
+
const hashBuffer = await crypto.subtle.digest('SHA-256', buffer);
|
|
340
|
+
const hashArray = Array.from(new Uint8Array(hashBuffer));
|
|
341
|
+
const hashHex = hashArray.map(b => b.toString(16).padStart(2, '0')).join('');
|
|
342
|
+
return `sha256:${hashHex}`;
|
|
343
|
+
}
|
|
344
|
+
return null;
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
async delete(key) {
|
|
348
|
+
await this.init();
|
|
349
|
+
|
|
350
|
+
return new Promise((resolve, reject) => {
|
|
351
|
+
const transaction = this.db.transaction(
|
|
352
|
+
[this.storeName, this.metadataStoreName],
|
|
353
|
+
'readwrite'
|
|
354
|
+
);
|
|
355
|
+
|
|
356
|
+
transaction.objectStore(this.storeName).delete(key);
|
|
357
|
+
transaction.objectStore(this.metadataStoreName).delete(key);
|
|
358
|
+
|
|
359
|
+
transaction.oncomplete = () => resolve(true);
|
|
360
|
+
transaction.onerror = () => reject(transaction.error);
|
|
361
|
+
});
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
async has(key) {
|
|
365
|
+
const value = await this.get(key);
|
|
366
|
+
return value !== null;
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
async list() {
|
|
370
|
+
await this.init();
|
|
371
|
+
|
|
372
|
+
return new Promise((resolve, reject) => {
|
|
373
|
+
const transaction = this.db.transaction([this.storeName], 'readonly');
|
|
374
|
+
const store = transaction.objectStore(this.storeName);
|
|
375
|
+
const request = store.getAllKeys();
|
|
376
|
+
|
|
377
|
+
request.onerror = () => reject(request.error);
|
|
378
|
+
request.onsuccess = () => resolve(request.result);
|
|
379
|
+
});
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
async getMetadata(key) {
|
|
383
|
+
await this.init();
|
|
384
|
+
|
|
385
|
+
return new Promise((resolve, reject) => {
|
|
386
|
+
const transaction = this.db.transaction([this.metadataStoreName], 'readonly');
|
|
387
|
+
const store = transaction.objectStore(this.metadataStoreName);
|
|
388
|
+
const request = store.get(key);
|
|
389
|
+
|
|
390
|
+
request.onerror = () => reject(request.error);
|
|
391
|
+
request.onsuccess = () => resolve(request.result || null);
|
|
392
|
+
});
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
async _updateAccessTime(key) {
|
|
396
|
+
const metadata = await this.getMetadata(key);
|
|
397
|
+
if (!metadata) return;
|
|
398
|
+
|
|
399
|
+
metadata.accessedAt = new Date().toISOString();
|
|
400
|
+
metadata.accessCount = (metadata.accessCount || 0) + 1;
|
|
401
|
+
|
|
402
|
+
const transaction = this.db.transaction([this.metadataStoreName], 'readwrite');
|
|
403
|
+
transaction.objectStore(this.metadataStoreName).put(metadata, key);
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
async clear() {
|
|
407
|
+
await this.init();
|
|
408
|
+
|
|
409
|
+
return new Promise((resolve, reject) => {
|
|
410
|
+
const transaction = this.db.transaction(
|
|
411
|
+
[this.storeName, this.metadataStoreName],
|
|
412
|
+
'readwrite'
|
|
413
|
+
);
|
|
414
|
+
|
|
415
|
+
transaction.objectStore(this.storeName).clear();
|
|
416
|
+
transaction.objectStore(this.metadataStoreName).clear();
|
|
417
|
+
|
|
418
|
+
transaction.oncomplete = () => resolve();
|
|
419
|
+
transaction.onerror = () => reject(transaction.error);
|
|
420
|
+
});
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
async getSize() {
|
|
424
|
+
await this.init();
|
|
425
|
+
|
|
426
|
+
return new Promise((resolve, reject) => {
|
|
427
|
+
const transaction = this.db.transaction([this.metadataStoreName], 'readonly');
|
|
428
|
+
const store = transaction.objectStore(this.metadataStoreName);
|
|
429
|
+
const request = store.getAll();
|
|
430
|
+
|
|
431
|
+
request.onerror = () => reject(request.error);
|
|
432
|
+
request.onsuccess = () => {
|
|
433
|
+
const totalSize = request.result.reduce((sum, meta) => sum + (meta.size || 0), 0);
|
|
434
|
+
resolve(totalSize);
|
|
435
|
+
};
|
|
436
|
+
});
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
// ============================================
|
|
441
|
+
// MODEL LOADER
|
|
442
|
+
// ============================================
|
|
443
|
+
|
|
444
|
+
/**
|
|
445
|
+
* ModelLoader - Smart model loading with caching
|
|
446
|
+
*/
|
|
447
|
+
export class ModelLoader extends EventEmitter {
|
|
448
|
+
/**
|
|
449
|
+
* Create a new ModelLoader
|
|
450
|
+
* @param {object} options - Configuration options
|
|
451
|
+
*/
|
|
452
|
+
constructor(options = {}) {
|
|
453
|
+
super();
|
|
454
|
+
|
|
455
|
+
this.id = `loader-${randomBytes(6).toString('hex')}`;
|
|
456
|
+
|
|
457
|
+
// Create registry if not provided
|
|
458
|
+
this.registry = options.registry || new ModelRegistry({
|
|
459
|
+
registryPath: options.registryPath,
|
|
460
|
+
});
|
|
461
|
+
|
|
462
|
+
// Create distribution manager if not provided
|
|
463
|
+
this.distribution = options.distribution || new DistributionManager({
|
|
464
|
+
gcsBucket: options.gcsBucket,
|
|
465
|
+
gcsProjectId: options.gcsProjectId,
|
|
466
|
+
cdnBaseUrl: options.cdnBaseUrl,
|
|
467
|
+
ipfsGateway: options.ipfsGateway,
|
|
468
|
+
});
|
|
469
|
+
|
|
470
|
+
// Cache configuration
|
|
471
|
+
this.cacheDir = options.cacheDir || DEFAULT_CACHE_DIR;
|
|
472
|
+
this.maxCacheSize = options.maxCacheSize || MAX_CACHE_SIZE_BYTES;
|
|
473
|
+
|
|
474
|
+
// Initialize cache storage based on environment
|
|
475
|
+
this.cache = this._createCacheStorage(options);
|
|
476
|
+
|
|
477
|
+
// Loaded models (in-memory)
|
|
478
|
+
this.loadedModels = new Map();
|
|
479
|
+
|
|
480
|
+
// Loading promises (prevent duplicate loads)
|
|
481
|
+
this.loadingPromises = new Map();
|
|
482
|
+
|
|
483
|
+
// Lazy load queue
|
|
484
|
+
this.lazyLoadQueue = [];
|
|
485
|
+
this.lazyLoadActive = false;
|
|
486
|
+
|
|
487
|
+
// Stats
|
|
488
|
+
this.stats = {
|
|
489
|
+
cacheHits: 0,
|
|
490
|
+
cacheMisses: 0,
|
|
491
|
+
downloads: 0,
|
|
492
|
+
validationErrors: 0,
|
|
493
|
+
lazyLoads: 0,
|
|
494
|
+
};
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
/**
|
|
498
|
+
* Create appropriate cache storage for environment
|
|
499
|
+
* @private
|
|
500
|
+
*/
|
|
501
|
+
_createCacheStorage(options) {
|
|
502
|
+
// Browser environment
|
|
503
|
+
if (typeof window !== 'undefined' && typeof indexedDB !== 'undefined') {
|
|
504
|
+
return new IndexedDBCache(options.dbName || 'ruvector-models');
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
// Node.js environment
|
|
508
|
+
return new FileSystemCache(this.cacheDir);
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
/**
|
|
512
|
+
* Initialize the loader
|
|
513
|
+
*/
|
|
514
|
+
async initialize() {
|
|
515
|
+
// Initialize cache
|
|
516
|
+
if (this.cache.init) {
|
|
517
|
+
await this.cache.init();
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
// Load registry if path provided
|
|
521
|
+
if (this.registry.registryPath) {
|
|
522
|
+
try {
|
|
523
|
+
await this.registry.load();
|
|
524
|
+
} catch (error) {
|
|
525
|
+
this.emit('warning', { message: 'Failed to load registry', error });
|
|
526
|
+
}
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
this.emit('initialized', { loaderId: this.id });
|
|
530
|
+
|
|
531
|
+
return this;
|
|
532
|
+
}
|
|
533
|
+
|
|
534
|
+
/**
|
|
535
|
+
* Get cache key for a model
|
|
536
|
+
* @private
|
|
537
|
+
*/
|
|
538
|
+
_getCacheKey(name, version) {
|
|
539
|
+
return `${name}@${version}`;
|
|
540
|
+
}
|
|
541
|
+
|
|
542
|
+
/**
|
|
543
|
+
* Load a model
|
|
544
|
+
* @param {string} name - Model name
|
|
545
|
+
* @param {string} version - Version (default: latest)
|
|
546
|
+
* @param {object} options - Load options
|
|
547
|
+
* @returns {Promise<Buffer|Uint8Array>}
|
|
548
|
+
*/
|
|
549
|
+
async load(name, version = 'latest', options = {}) {
|
|
550
|
+
const key = this._getCacheKey(name, version);
|
|
551
|
+
|
|
552
|
+
// Return cached in-memory model
|
|
553
|
+
if (this.loadedModels.has(key) && !options.forceReload) {
|
|
554
|
+
this.stats.cacheHits++;
|
|
555
|
+
return this.loadedModels.get(key);
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
// Return existing loading promise
|
|
559
|
+
if (this.loadingPromises.has(key)) {
|
|
560
|
+
return this.loadingPromises.get(key);
|
|
561
|
+
}
|
|
562
|
+
|
|
563
|
+
// Start loading
|
|
564
|
+
const loadPromise = this._loadModel(name, version, options);
|
|
565
|
+
this.loadingPromises.set(key, loadPromise);
|
|
566
|
+
|
|
567
|
+
try {
|
|
568
|
+
const model = await loadPromise;
|
|
569
|
+
this.loadedModels.set(key, model);
|
|
570
|
+
return model;
|
|
571
|
+
} finally {
|
|
572
|
+
this.loadingPromises.delete(key);
|
|
573
|
+
}
|
|
574
|
+
}
|
|
575
|
+
|
|
576
|
+
/**
|
|
577
|
+
* Internal model loading logic
|
|
578
|
+
* @private
|
|
579
|
+
*/
|
|
580
|
+
async _loadModel(name, version, options = {}) {
|
|
581
|
+
const { onProgress, skipCache = false, skipValidation = false } = options;
|
|
582
|
+
|
|
583
|
+
// Get metadata from registry
|
|
584
|
+
let metadata = this.registry.get(name, version);
|
|
585
|
+
|
|
586
|
+
if (!metadata) {
|
|
587
|
+
// Try to fetch from remote registry
|
|
588
|
+
this.emit('warning', { message: `Model ${name}@${version} not in local registry` });
|
|
589
|
+
throw new Error(`Model not found: ${name}@${version}`);
|
|
590
|
+
}
|
|
591
|
+
|
|
592
|
+
const resolvedVersion = metadata.version;
|
|
593
|
+
const key = this._getCacheKey(name, resolvedVersion);
|
|
594
|
+
|
|
595
|
+
// Check cache first (unless skipped)
|
|
596
|
+
if (!skipCache) {
|
|
597
|
+
const cached = await this.cache.get(key);
|
|
598
|
+
if (cached) {
|
|
599
|
+
// Validate cached data
|
|
600
|
+
if (!skipValidation && metadata.hash) {
|
|
601
|
+
const isValid = this.distribution.verifyIntegrity(cached, metadata.hash);
|
|
602
|
+
if (isValid) {
|
|
603
|
+
this.stats.cacheHits++;
|
|
604
|
+
this.emit('cache_hit', { name, version: resolvedVersion });
|
|
605
|
+
return cached;
|
|
606
|
+
} else {
|
|
607
|
+
this.stats.validationErrors++;
|
|
608
|
+
this.emit('cache_invalid', { name, version: resolvedVersion });
|
|
609
|
+
await this.cache.delete(key);
|
|
610
|
+
}
|
|
611
|
+
} else {
|
|
612
|
+
this.stats.cacheHits++;
|
|
613
|
+
return cached;
|
|
614
|
+
}
|
|
615
|
+
}
|
|
616
|
+
}
|
|
617
|
+
|
|
618
|
+
this.stats.cacheMisses++;
|
|
619
|
+
|
|
620
|
+
// Download model
|
|
621
|
+
this.emit('download_start', { name, version: resolvedVersion });
|
|
622
|
+
|
|
623
|
+
const data = await this.distribution.download(metadata, {
|
|
624
|
+
onProgress: (progress) => {
|
|
625
|
+
this.emit('progress', { name, version: resolvedVersion, ...progress });
|
|
626
|
+
if (onProgress) onProgress(progress);
|
|
627
|
+
},
|
|
628
|
+
});
|
|
629
|
+
|
|
630
|
+
// Validate downloaded data
|
|
631
|
+
if (!skipValidation) {
|
|
632
|
+
const validation = this.distribution.verifyModel(data, metadata);
|
|
633
|
+
if (!validation.valid) {
|
|
634
|
+
this.stats.validationErrors++;
|
|
635
|
+
throw new Error(`Model validation failed: ${JSON.stringify(validation.checks)}`);
|
|
636
|
+
}
|
|
637
|
+
}
|
|
638
|
+
|
|
639
|
+
// Store in cache
|
|
640
|
+
await this.cache.set(key, data, {
|
|
641
|
+
modelName: name,
|
|
642
|
+
version: resolvedVersion,
|
|
643
|
+
format: metadata.format,
|
|
644
|
+
});
|
|
645
|
+
|
|
646
|
+
this.stats.downloads++;
|
|
647
|
+
this.emit('loaded', { name, version: resolvedVersion, size: data.length });
|
|
648
|
+
|
|
649
|
+
// Cleanup cache if needed
|
|
650
|
+
await this._cleanupCacheIfNeeded();
|
|
651
|
+
|
|
652
|
+
return data;
|
|
653
|
+
}
|
|
654
|
+
|
|
655
|
+
/**
|
|
656
|
+
* Lazy load a model (load in background)
|
|
657
|
+
* @param {string} name - Model name
|
|
658
|
+
* @param {string} version - Version
|
|
659
|
+
* @param {object} options - Load options
|
|
660
|
+
*/
|
|
661
|
+
async lazyLoad(name, version = 'latest', options = {}) {
|
|
662
|
+
const key = this._getCacheKey(name, version);
|
|
663
|
+
|
|
664
|
+
// Already loaded or loading
|
|
665
|
+
if (this.loadedModels.has(key) || this.loadingPromises.has(key)) {
|
|
666
|
+
return;
|
|
667
|
+
}
|
|
668
|
+
|
|
669
|
+
// Check cache
|
|
670
|
+
const cached = await this.cache.has(key);
|
|
671
|
+
if (cached) {
|
|
672
|
+
return; // Already in cache
|
|
673
|
+
}
|
|
674
|
+
|
|
675
|
+
// Add to queue
|
|
676
|
+
this.lazyLoadQueue.push({ name, version, options });
|
|
677
|
+
this.stats.lazyLoads++;
|
|
678
|
+
|
|
679
|
+
this.emit('lazy_queued', { name, version, queueLength: this.lazyLoadQueue.length });
|
|
680
|
+
|
|
681
|
+
// Start processing if not active
|
|
682
|
+
if (!this.lazyLoadActive) {
|
|
683
|
+
this._processLazyLoadQueue();
|
|
684
|
+
}
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
/**
|
|
688
|
+
* Process lazy load queue
|
|
689
|
+
* @private
|
|
690
|
+
*/
|
|
691
|
+
async _processLazyLoadQueue() {
|
|
692
|
+
if (this.lazyLoadActive || this.lazyLoadQueue.length === 0) return;
|
|
693
|
+
|
|
694
|
+
this.lazyLoadActive = true;
|
|
695
|
+
|
|
696
|
+
while (this.lazyLoadQueue.length > 0) {
|
|
697
|
+
const { name, version, options } = this.lazyLoadQueue.shift();
|
|
698
|
+
|
|
699
|
+
try {
|
|
700
|
+
await this.load(name, version, {
|
|
701
|
+
...options,
|
|
702
|
+
lazy: true,
|
|
703
|
+
});
|
|
704
|
+
} catch (error) {
|
|
705
|
+
this.emit('lazy_error', { name, version, error: error.message });
|
|
706
|
+
}
|
|
707
|
+
|
|
708
|
+
// Small delay between lazy loads
|
|
709
|
+
await new Promise(resolve => setTimeout(resolve, 100));
|
|
710
|
+
}
|
|
711
|
+
|
|
712
|
+
this.lazyLoadActive = false;
|
|
713
|
+
}
|
|
714
|
+
|
|
715
|
+
/**
|
|
716
|
+
* Preload multiple models
|
|
717
|
+
* @param {Array<{name: string, version?: string}>} models - Models to preload
|
|
718
|
+
*/
|
|
719
|
+
async preload(models) {
|
|
720
|
+
const results = await Promise.allSettled(
|
|
721
|
+
models.map(({ name, version }) => this.load(name, version || 'latest'))
|
|
722
|
+
);
|
|
723
|
+
|
|
724
|
+
return {
|
|
725
|
+
total: models.length,
|
|
726
|
+
loaded: results.filter(r => r.status === 'fulfilled').length,
|
|
727
|
+
failed: results.filter(r => r.status === 'rejected').length,
|
|
728
|
+
results,
|
|
729
|
+
};
|
|
730
|
+
}
|
|
731
|
+
|
|
732
|
+
/**
|
|
733
|
+
* Check if a model is loaded in memory
|
|
734
|
+
* @param {string} name - Model name
|
|
735
|
+
* @param {string} version - Version
|
|
736
|
+
* @returns {boolean}
|
|
737
|
+
*/
|
|
738
|
+
isLoaded(name, version = 'latest') {
|
|
739
|
+
const metadata = this.registry.get(name, version);
|
|
740
|
+
if (!metadata) return false;
|
|
741
|
+
|
|
742
|
+
const key = this._getCacheKey(name, metadata.version);
|
|
743
|
+
return this.loadedModels.has(key);
|
|
744
|
+
}
|
|
745
|
+
|
|
746
|
+
/**
|
|
747
|
+
* Check if a model is cached on disk
|
|
748
|
+
* @param {string} name - Model name
|
|
749
|
+
* @param {string} version - Version
|
|
750
|
+
* @returns {Promise<boolean>}
|
|
751
|
+
*/
|
|
752
|
+
async isCached(name, version = 'latest') {
|
|
753
|
+
const metadata = this.registry.get(name, version);
|
|
754
|
+
if (!metadata) return false;
|
|
755
|
+
|
|
756
|
+
const key = this._getCacheKey(name, metadata.version);
|
|
757
|
+
return this.cache.has(key);
|
|
758
|
+
}
|
|
759
|
+
|
|
760
|
+
/**
|
|
761
|
+
* Unload a model from memory
|
|
762
|
+
* @param {string} name - Model name
|
|
763
|
+
* @param {string} version - Version
|
|
764
|
+
*/
|
|
765
|
+
unload(name, version = 'latest') {
|
|
766
|
+
const metadata = this.registry.get(name, version);
|
|
767
|
+
if (!metadata) return false;
|
|
768
|
+
|
|
769
|
+
const key = this._getCacheKey(name, metadata.version);
|
|
770
|
+
return this.loadedModels.delete(key);
|
|
771
|
+
}
|
|
772
|
+
|
|
773
|
+
/**
|
|
774
|
+
* Unload all models from memory
|
|
775
|
+
*/
|
|
776
|
+
unloadAll() {
|
|
777
|
+
const count = this.loadedModels.size;
|
|
778
|
+
this.loadedModels.clear();
|
|
779
|
+
return count;
|
|
780
|
+
}
|
|
781
|
+
|
|
782
|
+
/**
|
|
783
|
+
* Remove a model from cache
|
|
784
|
+
* @param {string} name - Model name
|
|
785
|
+
* @param {string} version - Version
|
|
786
|
+
*/
|
|
787
|
+
async removeFromCache(name, version = 'latest') {
|
|
788
|
+
const metadata = this.registry.get(name, version);
|
|
789
|
+
if (!metadata) return false;
|
|
790
|
+
|
|
791
|
+
const key = this._getCacheKey(name, metadata.version);
|
|
792
|
+
return this.cache.delete(key);
|
|
793
|
+
}
|
|
794
|
+
|
|
795
|
+
/**
|
|
796
|
+
* Clear all cached models
|
|
797
|
+
*/
|
|
798
|
+
async clearCache() {
|
|
799
|
+
await this.cache.clear();
|
|
800
|
+
this.emit('cache_cleared');
|
|
801
|
+
}
|
|
802
|
+
|
|
803
|
+
/**
|
|
804
|
+
* Cleanup cache if over size limit
|
|
805
|
+
* @private
|
|
806
|
+
*/
|
|
807
|
+
async _cleanupCacheIfNeeded() {
|
|
808
|
+
const currentSize = await this.cache.getSize();
|
|
809
|
+
const threshold = this.maxCacheSize * CACHE_CLEANUP_THRESHOLD;
|
|
810
|
+
|
|
811
|
+
if (currentSize < threshold) return;
|
|
812
|
+
|
|
813
|
+
this.emit('cache_cleanup_start', { currentSize, maxSize: this.maxCacheSize });
|
|
814
|
+
|
|
815
|
+
// Get entries sorted by last access time
|
|
816
|
+
let entries;
|
|
817
|
+
if (this.cache.getEntriesWithMetadata) {
|
|
818
|
+
entries = await this.cache.getEntriesWithMetadata();
|
|
819
|
+
} else {
|
|
820
|
+
const keys = await this.cache.list();
|
|
821
|
+
entries = [];
|
|
822
|
+
for (const key of keys) {
|
|
823
|
+
const meta = await this.cache.getMetadata(key);
|
|
824
|
+
if (meta) entries.push(meta);
|
|
825
|
+
}
|
|
826
|
+
}
|
|
827
|
+
|
|
828
|
+
// Sort by access time (oldest first)
|
|
829
|
+
entries.sort((a, b) =>
|
|
830
|
+
new Date(a.accessedAt).getTime() - new Date(b.accessedAt).getTime()
|
|
831
|
+
);
|
|
832
|
+
|
|
833
|
+
// Remove oldest entries until under 80% capacity
|
|
834
|
+
const targetSize = this.maxCacheSize * 0.8;
|
|
835
|
+
let removedSize = 0;
|
|
836
|
+
let removedCount = 0;
|
|
837
|
+
|
|
838
|
+
for (const entry of entries) {
|
|
839
|
+
if (currentSize - removedSize <= targetSize) break;
|
|
840
|
+
|
|
841
|
+
await this.cache.delete(entry.key);
|
|
842
|
+
removedSize += entry.size;
|
|
843
|
+
removedCount++;
|
|
844
|
+
}
|
|
845
|
+
|
|
846
|
+
this.emit('cache_cleanup_complete', {
|
|
847
|
+
removedCount,
|
|
848
|
+
removedSize,
|
|
849
|
+
newSize: currentSize - removedSize,
|
|
850
|
+
});
|
|
851
|
+
}
|
|
852
|
+
|
|
853
|
+
/**
|
|
854
|
+
* Get cache statistics
|
|
855
|
+
* @returns {Promise<object>}
|
|
856
|
+
*/
|
|
857
|
+
async getCacheStats() {
|
|
858
|
+
const size = await this.cache.getSize();
|
|
859
|
+
const keys = await this.cache.list();
|
|
860
|
+
|
|
861
|
+
return {
|
|
862
|
+
entries: keys.length,
|
|
863
|
+
sizeBytes: size,
|
|
864
|
+
sizeMB: Math.round(size / (1024 * 1024) * 100) / 100,
|
|
865
|
+
maxSizeBytes: this.maxCacheSize,
|
|
866
|
+
usagePercent: Math.round((size / this.maxCacheSize) * 100),
|
|
867
|
+
};
|
|
868
|
+
}
|
|
869
|
+
|
|
870
|
+
/**
|
|
871
|
+
* Get loader statistics
|
|
872
|
+
* @returns {object}
|
|
873
|
+
*/
|
|
874
|
+
getStats() {
|
|
875
|
+
return {
|
|
876
|
+
...this.stats,
|
|
877
|
+
loadedModels: this.loadedModels.size,
|
|
878
|
+
pendingLoads: this.loadingPromises.size,
|
|
879
|
+
lazyQueueLength: this.lazyLoadQueue.length,
|
|
880
|
+
hitRate: this.stats.cacheHits + this.stats.cacheMisses > 0
|
|
881
|
+
? Math.round(
|
|
882
|
+
(this.stats.cacheHits / (this.stats.cacheHits + this.stats.cacheMisses)) * 100
|
|
883
|
+
)
|
|
884
|
+
: 0,
|
|
885
|
+
};
|
|
886
|
+
}
|
|
887
|
+
|
|
888
|
+
/**
|
|
889
|
+
* Get model from registry (without loading)
|
|
890
|
+
* @param {string} name - Model name
|
|
891
|
+
* @param {string} version - Version
|
|
892
|
+
* @returns {object|null}
|
|
893
|
+
*/
|
|
894
|
+
getModelInfo(name, version = 'latest') {
|
|
895
|
+
return this.registry.get(name, version);
|
|
896
|
+
}
|
|
897
|
+
|
|
898
|
+
/**
|
|
899
|
+
* Search for models
|
|
900
|
+
* @param {object} criteria - Search criteria
|
|
901
|
+
* @returns {Array}
|
|
902
|
+
*/
|
|
903
|
+
searchModels(criteria) {
|
|
904
|
+
return this.registry.search(criteria);
|
|
905
|
+
}
|
|
906
|
+
|
|
907
|
+
/**
|
|
908
|
+
* List all available models
|
|
909
|
+
* @returns {string[]}
|
|
910
|
+
*/
|
|
911
|
+
listModels() {
|
|
912
|
+
return this.registry.listModels();
|
|
913
|
+
}
|
|
914
|
+
}
|
|
915
|
+
|
|
916
|
+
// ============================================
|
|
917
|
+
// EXPORTS
|
|
918
|
+
// ============================================
|
|
919
|
+
|
|
920
|
+
export { FileSystemCache, IndexedDBCache, CacheStorage };
|
|
921
|
+
|
|
922
|
+
export default ModelLoader;
|