@ruvector/ruvllm 0.2.4 → 2.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +370 -286
- package/bin/cli.js +619 -2
- package/dist/cjs/benchmarks/embedding-benchmark.d.ts +102 -0
- package/dist/cjs/benchmarks/embedding-benchmark.d.ts.map +1 -0
- package/dist/cjs/benchmarks/embedding-benchmark.js +436 -0
- package/dist/cjs/benchmarks/index.d.ts +34 -0
- package/dist/cjs/benchmarks/index.d.ts.map +1 -0
- package/dist/cjs/benchmarks/index.js +132 -0
- package/dist/cjs/benchmarks/model-comparison.d.ts +71 -0
- package/dist/cjs/benchmarks/model-comparison.d.ts.map +1 -0
- package/dist/cjs/benchmarks/model-comparison.js +476 -0
- package/dist/cjs/benchmarks/routing-benchmark.d.ts +70 -0
- package/dist/cjs/benchmarks/routing-benchmark.d.ts.map +1 -0
- package/dist/cjs/benchmarks/routing-benchmark.js +289 -0
- package/dist/cjs/contrastive.d.ts +229 -0
- package/dist/cjs/contrastive.d.ts.map +1 -0
- package/dist/cjs/contrastive.js +589 -0
- package/dist/cjs/engine.d.ts +3 -0
- package/dist/cjs/engine.d.ts.map +1 -1
- package/dist/cjs/engine.js +26 -6
- package/dist/cjs/index.d.ts +4 -0
- package/dist/cjs/index.d.ts.map +1 -1
- package/dist/cjs/index.js +9 -1
- package/dist/cjs/models.d.ts +129 -0
- package/dist/cjs/models.d.ts.map +1 -0
- package/dist/cjs/models.js +323 -0
- package/dist/cjs/native.d.ts +3 -3
- package/dist/cjs/native.d.ts.map +1 -1
- package/dist/cjs/native.js +1 -1
- package/dist/cjs/rlm/controller.d.ts +245 -0
- package/dist/cjs/rlm/controller.d.ts.map +1 -0
- package/dist/cjs/rlm/controller.js +554 -0
- package/dist/cjs/rlm/index.d.ts +62 -0
- package/dist/cjs/rlm/index.d.ts.map +1 -0
- package/dist/cjs/rlm/index.js +97 -0
- package/dist/cjs/rlm/training.d.ts +333 -0
- package/dist/cjs/rlm/training.d.ts.map +1 -0
- package/dist/cjs/rlm/training.js +680 -0
- package/dist/cjs/rlm/types.d.ts +189 -0
- package/dist/cjs/rlm/types.d.ts.map +1 -0
- package/dist/cjs/rlm/types.js +10 -0
- package/dist/esm/benchmarks/embedding-benchmark.d.ts +102 -0
- package/dist/esm/benchmarks/embedding-benchmark.d.ts.map +1 -0
- package/dist/esm/benchmarks/embedding-benchmark.js +427 -0
- package/dist/esm/benchmarks/index.d.ts +34 -0
- package/dist/esm/benchmarks/index.d.ts.map +1 -0
- package/dist/esm/benchmarks/index.js +110 -0
- package/dist/esm/benchmarks/model-comparison.d.ts +71 -0
- package/dist/esm/benchmarks/model-comparison.d.ts.map +1 -0
- package/dist/esm/benchmarks/model-comparison.js +434 -0
- package/dist/esm/benchmarks/routing-benchmark.d.ts +70 -0
- package/dist/esm/benchmarks/routing-benchmark.d.ts.map +1 -0
- package/dist/esm/benchmarks/routing-benchmark.js +283 -0
- package/dist/esm/contrastive.d.ts +229 -0
- package/dist/esm/contrastive.d.ts.map +1 -0
- package/dist/esm/contrastive.js +578 -0
- package/dist/esm/engine.d.ts +3 -0
- package/dist/esm/engine.d.ts.map +1 -1
- package/dist/esm/engine.js +26 -6
- package/dist/esm/index.d.ts +4 -0
- package/dist/esm/index.d.ts.map +1 -1
- package/dist/esm/index.js +9 -1
- package/dist/esm/models.d.ts +129 -0
- package/dist/esm/models.d.ts.map +1 -0
- package/dist/esm/models.js +282 -0
- package/dist/esm/native.d.ts +3 -3
- package/dist/esm/native.d.ts.map +1 -1
- package/dist/esm/native.js +1 -1
- package/dist/esm/rlm/controller.d.ts +245 -0
- package/dist/esm/rlm/controller.d.ts.map +1 -0
- package/dist/esm/rlm/controller.js +550 -0
- package/dist/esm/rlm/index.d.ts +62 -0
- package/dist/esm/rlm/index.d.ts.map +1 -0
- package/dist/esm/rlm/index.js +71 -0
- package/dist/esm/rlm/training.d.ts +333 -0
- package/dist/esm/rlm/training.d.ts.map +1 -0
- package/dist/esm/rlm/training.js +672 -0
- package/dist/esm/rlm/types.d.ts +189 -0
- package/dist/esm/rlm/types.d.ts.map +1 -0
- package/dist/esm/rlm/types.js +9 -0
- package/package.json +13 -8
- package/scripts/ensemble-model-compare.js +287 -0
- package/scripts/huggingface/README.md +537 -0
- package/scripts/huggingface/publish.sh +112 -0
- package/scripts/hybrid-model-compare.js +373 -0
- package/scripts/improved-model-compare.js +288 -0
- package/scripts/optimized-model-compare.js +364 -0
- package/scripts/real-model-compare.js +280 -0
- package/scripts/training/agentic-flow-capabilities.json +329 -0
- package/scripts/training/claude-code-synth.js +505 -0
- package/scripts/training/claude-flow-capabilities.json +1322 -0
- package/scripts/training/claude-hard-negatives.js +222 -0
- package/scripts/training/contrastive-finetune.js +517 -0
- package/scripts/training/ecosystem-triplets.jsonl +1467 -0
- package/scripts/training/generate-ecosystem-triplets.js +1324 -0
- package/scripts/training/rlm-dataset.js +671 -0
- package/scripts/training/routing-dataset.js +634 -0
- package/scripts/training/ruvector-capabilities.json +798 -0
- package/scripts/training/validate-ecosystem.js +381 -0
- package/scripts/training/validation-results.json +23 -0
package/bin/cli.js
CHANGED
|
@@ -7,11 +7,13 @@
|
|
|
7
7
|
* ruvllm generate "Write a haiku about AI"
|
|
8
8
|
* ruvllm memory add "Important context"
|
|
9
9
|
* ruvllm memory search "context"
|
|
10
|
+
* ruvllm models list
|
|
11
|
+
* ruvllm models download claude-code
|
|
10
12
|
* ruvllm stats
|
|
11
13
|
* ruvllm benchmark
|
|
12
14
|
*/
|
|
13
15
|
|
|
14
|
-
const { RuvLLM, SimdOps, version, hasSimdSupport } = require('../dist/cjs/index.js');
|
|
16
|
+
const { RuvLLM, SimdOps, version, hasSimdSupport, ModelDownloader, listModels, getModelInfo, RUVLTRA_MODELS, getDefaultModelsDir, runRoutingBenchmark, formatRoutingResults, baselineKeywordRouter, runEmbeddingBenchmark, formatEmbeddingResults, runFullBenchmark, formatFullResults, ROUTING_TEST_CASES, runFullComparison, formatComparisonResults, ContrastiveTrainer, AGENT_TRAINING_DATA, generateTrainingDataset, generateContrastivePairs, getDatasetStats, tripletLoss, cosineSimilarity } = require('../dist/cjs/index.js');
|
|
15
17
|
|
|
16
18
|
const args = process.argv.slice(2);
|
|
17
19
|
const command = args[0];
|
|
@@ -235,6 +237,526 @@ async function runInfo(flags) {
|
|
|
235
237
|
}
|
|
236
238
|
}
|
|
237
239
|
|
|
240
|
+
// Model management commands
|
|
241
|
+
async function runModelsList(flags) {
|
|
242
|
+
const downloader = new ModelDownloader();
|
|
243
|
+
const status = downloader.getStatus();
|
|
244
|
+
|
|
245
|
+
if (flags.json) {
|
|
246
|
+
console.log(formatJson(status));
|
|
247
|
+
} else {
|
|
248
|
+
console.log('\n╔══════════════════════════════════════════════════════════════════════════╗');
|
|
249
|
+
console.log('║ RuvLTRA Models ║');
|
|
250
|
+
console.log('║ https://huggingface.co/ruv/ruvltra ║');
|
|
251
|
+
console.log('╠══════════════════════════════════════════════════════════════════════════╣');
|
|
252
|
+
console.log('║ Model │ Size │ Params │ Status │ Use Case ║');
|
|
253
|
+
console.log('╠══════════════════════════════════════════════════════════════════════════╣');
|
|
254
|
+
|
|
255
|
+
for (const { model, downloaded } of status) {
|
|
256
|
+
const statusIcon = downloaded ? '✓ Ready ' : '○ Not DL ';
|
|
257
|
+
const name = model.id.padEnd(12);
|
|
258
|
+
const size = model.size.padEnd(7);
|
|
259
|
+
const params = model.parameters.padEnd(6);
|
|
260
|
+
const useCase = model.useCase.slice(0, 24).padEnd(24);
|
|
261
|
+
console.log(`║ ${name} │ ${size} │ ${params} │ ${statusIcon} │ ${useCase} ║`);
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
console.log('╚══════════════════════════════════════════════════════════════════════════╝');
|
|
265
|
+
console.log(`\nModels directory: ${getDefaultModelsDir()}`);
|
|
266
|
+
console.log('\nDownload with: ruvllm models download <model-id>');
|
|
267
|
+
console.log(' Examples: ruvllm models download claude-code');
|
|
268
|
+
console.log(' ruvllm models download --all');
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
async function runModelsDownload(modelId, flags) {
|
|
273
|
+
const downloader = new ModelDownloader();
|
|
274
|
+
|
|
275
|
+
if (flags.all) {
|
|
276
|
+
console.log('\nDownloading all RuvLTRA models...\n');
|
|
277
|
+
const models = listModels();
|
|
278
|
+
|
|
279
|
+
for (const model of models) {
|
|
280
|
+
console.log(`\n[${model.id}] ${model.name} (${model.size})`);
|
|
281
|
+
|
|
282
|
+
if (downloader.isDownloaded(model.id) && !flags.force) {
|
|
283
|
+
console.log(' Already downloaded, skipping (use --force to re-download)');
|
|
284
|
+
continue;
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
try {
|
|
288
|
+
const lastPercent = { value: -1 };
|
|
289
|
+
const path = await downloader.download(model.id, {
|
|
290
|
+
force: flags.force,
|
|
291
|
+
onProgress: (p) => {
|
|
292
|
+
const percent = Math.floor(p.percent / 5) * 5; // Round to 5%
|
|
293
|
+
if (percent !== lastPercent.value) {
|
|
294
|
+
const bar = '█'.repeat(percent / 5) + '░'.repeat(20 - percent / 5);
|
|
295
|
+
const speed = (p.speedBps / 1024 / 1024).toFixed(1);
|
|
296
|
+
const eta = p.etaSeconds < 60
|
|
297
|
+
? `${Math.ceil(p.etaSeconds)}s`
|
|
298
|
+
: `${Math.ceil(p.etaSeconds / 60)}m`;
|
|
299
|
+
process.stdout.write(`\r [${bar}] ${p.percent}% | ${speed} MB/s | ETA: ${eta} `);
|
|
300
|
+
lastPercent.value = percent;
|
|
301
|
+
}
|
|
302
|
+
},
|
|
303
|
+
});
|
|
304
|
+
console.log(`\n ✓ Downloaded to: ${path}`);
|
|
305
|
+
} catch (error) {
|
|
306
|
+
console.error(`\n ✗ Failed: ${error.message}`);
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
console.log('\n\nDownload complete!');
|
|
311
|
+
return;
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
if (!modelId) {
|
|
315
|
+
console.error('Error: model ID required. Use --all to download all models.');
|
|
316
|
+
console.error('\nAvailable models:');
|
|
317
|
+
listModels().forEach(m => console.error(` - ${m.id}: ${m.name} (${m.size})`));
|
|
318
|
+
process.exit(1);
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
const model = getModelInfo(modelId);
|
|
322
|
+
if (!model) {
|
|
323
|
+
console.error(`Error: Unknown model "${modelId}"`);
|
|
324
|
+
console.error('\nAvailable models:');
|
|
325
|
+
listModels().forEach(m => console.error(` - ${m.id}: ${m.name} (${m.size})`));
|
|
326
|
+
process.exit(1);
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
console.log(`\nDownloading ${model.name} (${model.size})...`);
|
|
330
|
+
console.log(`From: ${model.url}\n`);
|
|
331
|
+
|
|
332
|
+
if (downloader.isDownloaded(modelId) && !flags.force) {
|
|
333
|
+
const path = downloader.getModelPath(modelId);
|
|
334
|
+
console.log(`Model already downloaded at: ${path}`);
|
|
335
|
+
console.log('Use --force to re-download.');
|
|
336
|
+
return;
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
const lastPercent = { value: -1 };
|
|
340
|
+
try {
|
|
341
|
+
const path = await downloader.download(modelId, {
|
|
342
|
+
force: flags.force,
|
|
343
|
+
onProgress: (p) => {
|
|
344
|
+
const percent = Math.floor(p.percent / 2) * 2; // Round to 2%
|
|
345
|
+
if (percent !== lastPercent.value) {
|
|
346
|
+
const bar = '█'.repeat(Math.floor(percent / 5)) + '░'.repeat(20 - Math.floor(percent / 5));
|
|
347
|
+
const downloaded = (p.downloaded / 1024 / 1024).toFixed(1);
|
|
348
|
+
const total = (p.total / 1024 / 1024).toFixed(1);
|
|
349
|
+
const speed = (p.speedBps / 1024 / 1024).toFixed(1);
|
|
350
|
+
const eta = p.etaSeconds < 60
|
|
351
|
+
? `${Math.ceil(p.etaSeconds)}s`
|
|
352
|
+
: `${Math.ceil(p.etaSeconds / 60)}m`;
|
|
353
|
+
process.stdout.write(`\r[${bar}] ${p.percent}% | ${downloaded}/${total} MB | ${speed} MB/s | ETA: ${eta} `);
|
|
354
|
+
lastPercent.value = percent;
|
|
355
|
+
}
|
|
356
|
+
},
|
|
357
|
+
});
|
|
358
|
+
console.log(`\n\n✓ Downloaded to: ${path}`);
|
|
359
|
+
console.log(`\nModel ready to use!`);
|
|
360
|
+
console.log(` Context length: ${model.contextLength} tokens`);
|
|
361
|
+
console.log(` Quantization: ${model.quantization}`);
|
|
362
|
+
} catch (error) {
|
|
363
|
+
console.error(`\n\n✗ Download failed: ${error.message}`);
|
|
364
|
+
process.exit(1);
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
async function runModelsStatus(flags) {
|
|
369
|
+
const downloader = new ModelDownloader();
|
|
370
|
+
const status = downloader.getStatus();
|
|
371
|
+
|
|
372
|
+
if (flags.json) {
|
|
373
|
+
console.log(formatJson(status.map(s => ({
|
|
374
|
+
id: s.model.id,
|
|
375
|
+
name: s.model.name,
|
|
376
|
+
downloaded: s.downloaded,
|
|
377
|
+
path: s.path,
|
|
378
|
+
size: s.model.size,
|
|
379
|
+
}))));
|
|
380
|
+
} else {
|
|
381
|
+
console.log('\nModel Status:');
|
|
382
|
+
console.log(`Directory: ${getDefaultModelsDir()}\n`);
|
|
383
|
+
|
|
384
|
+
for (const { model, downloaded, path } of status) {
|
|
385
|
+
const icon = downloaded ? '✓' : '○';
|
|
386
|
+
const status = downloaded ? 'Ready' : 'Not downloaded';
|
|
387
|
+
console.log(` ${icon} ${model.name.padEnd(25)} ${status.padEnd(15)} ${model.size}`);
|
|
388
|
+
if (downloaded) {
|
|
389
|
+
console.log(` Path: ${path}`);
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
async function runModelsDelete(modelId, flags) {
|
|
396
|
+
const downloader = new ModelDownloader();
|
|
397
|
+
|
|
398
|
+
if (flags.all) {
|
|
399
|
+
const count = downloader.deleteAll();
|
|
400
|
+
console.log(`Deleted ${count} model(s).`);
|
|
401
|
+
return;
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
if (!modelId) {
|
|
405
|
+
console.error('Error: model ID required. Use --all to delete all models.');
|
|
406
|
+
process.exit(1);
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
if (downloader.delete(modelId)) {
|
|
410
|
+
console.log(`Deleted model: ${modelId}`);
|
|
411
|
+
} else {
|
|
412
|
+
console.log(`Model not found or not downloaded: ${modelId}`);
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
// Benchmark commands for Claude Code use cases
|
|
417
|
+
async function runBenchmarkRouting(flags) {
|
|
418
|
+
console.log('\nRunning Routing Benchmark...');
|
|
419
|
+
console.log(`Testing ${ROUTING_TEST_CASES.length} task routing scenarios\n`);
|
|
420
|
+
|
|
421
|
+
const llm = new RuvLLM({ embeddingDim: 768, learningEnabled: false });
|
|
422
|
+
|
|
423
|
+
// Router function using the model
|
|
424
|
+
const modelRouter = (task) => {
|
|
425
|
+
try {
|
|
426
|
+
const decision = llm.route(task);
|
|
427
|
+
return { agent: decision.model, confidence: decision.confidence };
|
|
428
|
+
} catch (e) {
|
|
429
|
+
// Fallback to keyword router if model not available
|
|
430
|
+
return baselineKeywordRouter(task);
|
|
431
|
+
}
|
|
432
|
+
};
|
|
433
|
+
|
|
434
|
+
// Run with baseline (keyword) router
|
|
435
|
+
console.log('Baseline (keyword matching):');
|
|
436
|
+
const baselineResults = runRoutingBenchmark(baselineKeywordRouter);
|
|
437
|
+
|
|
438
|
+
if (flags.json) {
|
|
439
|
+
console.log(JSON.stringify(baselineResults, null, 2));
|
|
440
|
+
} else {
|
|
441
|
+
console.log(formatRoutingResults(baselineResults));
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
// Try model router if native is available
|
|
445
|
+
if (llm.isNativeLoaded() && !flags['baseline-only']) {
|
|
446
|
+
console.log('\nModel Router (RuvLTRA):');
|
|
447
|
+
const modelResults = runRoutingBenchmark(modelRouter);
|
|
448
|
+
if (flags.json) {
|
|
449
|
+
console.log(JSON.stringify(modelResults, null, 2));
|
|
450
|
+
} else {
|
|
451
|
+
console.log(formatRoutingResults(modelResults));
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
// Comparison
|
|
455
|
+
const improvement = modelResults.accuracy - baselineResults.accuracy;
|
|
456
|
+
console.log(`\nComparison: Model ${improvement >= 0 ? '+' : ''}${(improvement * 100).toFixed(1)}% vs baseline`);
|
|
457
|
+
}
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
async function runBenchmarkEmbedding(flags) {
|
|
461
|
+
console.log('\nRunning Embedding Benchmark...');
|
|
462
|
+
console.log('Testing similarity detection, clustering, and search quality\n');
|
|
463
|
+
|
|
464
|
+
const llm = new RuvLLM({ embeddingDim: 768, learningEnabled: false });
|
|
465
|
+
const simd = new SimdOps();
|
|
466
|
+
|
|
467
|
+
// Embedder function
|
|
468
|
+
const embedder = (text) => {
|
|
469
|
+
try {
|
|
470
|
+
return llm.embed(text);
|
|
471
|
+
} catch (e) {
|
|
472
|
+
// Fallback: simple hash-based embedding for testing
|
|
473
|
+
const hash = text.split('').reduce((h, c) => ((h << 5) - h + c.charCodeAt(0)) | 0, 0);
|
|
474
|
+
return Array.from({ length: 768 }, (_, i) => Math.sin(hash + i) * 0.5);
|
|
475
|
+
}
|
|
476
|
+
};
|
|
477
|
+
|
|
478
|
+
// Similarity function
|
|
479
|
+
const similarity = (a, b) => {
|
|
480
|
+
try {
|
|
481
|
+
return simd.cosineSimilarity(a, b);
|
|
482
|
+
} catch (e) {
|
|
483
|
+
// Fallback cosine similarity
|
|
484
|
+
let dot = 0, normA = 0, normB = 0;
|
|
485
|
+
for (let i = 0; i < a.length; i++) {
|
|
486
|
+
dot += a[i] * b[i];
|
|
487
|
+
normA += a[i] * a[i];
|
|
488
|
+
normB += b[i] * b[i];
|
|
489
|
+
}
|
|
490
|
+
return dot / (Math.sqrt(normA) * Math.sqrt(normB));
|
|
491
|
+
}
|
|
492
|
+
};
|
|
493
|
+
|
|
494
|
+
const results = runEmbeddingBenchmark(embedder, similarity);
|
|
495
|
+
|
|
496
|
+
if (flags.json) {
|
|
497
|
+
console.log(JSON.stringify(results, null, 2));
|
|
498
|
+
} else {
|
|
499
|
+
console.log(formatEmbeddingResults(results));
|
|
500
|
+
}
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
async function runBenchmarkCompare(flags) {
|
|
504
|
+
try {
|
|
505
|
+
const results = await runFullComparison();
|
|
506
|
+
|
|
507
|
+
if (flags.json) {
|
|
508
|
+
console.log(JSON.stringify(results, null, 2));
|
|
509
|
+
} else {
|
|
510
|
+
console.log(formatComparisonResults(results));
|
|
511
|
+
}
|
|
512
|
+
} catch (error) {
|
|
513
|
+
console.error('Comparison failed:', error.message);
|
|
514
|
+
process.exit(1);
|
|
515
|
+
}
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
// Training commands
|
|
519
|
+
async function runTrainContrastive(flags) {
|
|
520
|
+
console.log('\n╔═══════════════════════════════════════════════════════════════════════════╗');
|
|
521
|
+
console.log('║ RuvLTRA Contrastive Fine-tuning Pipeline ║');
|
|
522
|
+
console.log('╚═══════════════════════════════════════════════════════════════════════════╝\n');
|
|
523
|
+
|
|
524
|
+
const stats = getDatasetStats();
|
|
525
|
+
const outputPath = flags.output || './training-output';
|
|
526
|
+
|
|
527
|
+
console.log(`Training examples: ${stats.totalExamples}`);
|
|
528
|
+
console.log(`Contrastive pairs: ${stats.contrastivePairs}`);
|
|
529
|
+
console.log(`Agent types: ${stats.agentTypes}`);
|
|
530
|
+
console.log(`Output: ${outputPath}\n`);
|
|
531
|
+
|
|
532
|
+
// Create trainer
|
|
533
|
+
const trainer = new ContrastiveTrainer({
|
|
534
|
+
epochs: parseInt(flags.epochs) || 10,
|
|
535
|
+
batchSize: parseInt(flags['batch-size']) || 16,
|
|
536
|
+
learningRate: parseFloat(flags.lr) || 0.0001,
|
|
537
|
+
margin: parseFloat(flags.margin) || 0.5,
|
|
538
|
+
outputPath,
|
|
539
|
+
});
|
|
540
|
+
|
|
541
|
+
// Get embeddings for agents
|
|
542
|
+
const llm = new RuvLLM({ embeddingDim: 768, learningEnabled: false });
|
|
543
|
+
const agentEmbeddings = {};
|
|
544
|
+
|
|
545
|
+
console.log('Computing agent embeddings...');
|
|
546
|
+
for (const [agent, data] of Object.entries(AGENT_TRAINING_DATA)) {
|
|
547
|
+
process.stdout.write(` ${agent}... `);
|
|
548
|
+
try {
|
|
549
|
+
const emb = llm.embed(data.description);
|
|
550
|
+
agentEmbeddings[agent] = emb;
|
|
551
|
+
trainer.addAgentEmbedding(agent, emb);
|
|
552
|
+
console.log('done');
|
|
553
|
+
} catch (e) {
|
|
554
|
+
// Fallback embedding
|
|
555
|
+
const hash = data.description.split('').reduce((h, c) => ((h << 5) - h + c.charCodeAt(0)) | 0, 0);
|
|
556
|
+
const emb = Array.from({ length: 768 }, (_, i) => Math.sin(hash + i) * 0.5);
|
|
557
|
+
agentEmbeddings[agent] = emb;
|
|
558
|
+
trainer.addAgentEmbedding(agent, emb);
|
|
559
|
+
console.log('done (fallback)');
|
|
560
|
+
}
|
|
561
|
+
}
|
|
562
|
+
|
|
563
|
+
// Generate triplets
|
|
564
|
+
console.log('\nGenerating training triplets...');
|
|
565
|
+
const examples = generateTrainingDataset();
|
|
566
|
+
const agents = Object.keys(AGENT_TRAINING_DATA);
|
|
567
|
+
|
|
568
|
+
let tripletCount = 0;
|
|
569
|
+
for (const example of examples.slice(0, 200)) { // Limit for speed
|
|
570
|
+
let taskEmb;
|
|
571
|
+
try {
|
|
572
|
+
taskEmb = llm.embed(example.task);
|
|
573
|
+
} catch (e) {
|
|
574
|
+
const hash = example.task.split('').reduce((h, c) => ((h << 5) - h + c.charCodeAt(0)) | 0, 0);
|
|
575
|
+
taskEmb = Array.from({ length: 768 }, (_, i) => Math.sin(hash + i) * 0.5);
|
|
576
|
+
}
|
|
577
|
+
|
|
578
|
+
const positiveAgent = example.agent;
|
|
579
|
+
const positiveEmb = agentEmbeddings[positiveAgent];
|
|
580
|
+
|
|
581
|
+
// Hard negatives
|
|
582
|
+
const hardNegatives = example.confusing_with
|
|
583
|
+
? [example.confusing_with]
|
|
584
|
+
: agents.filter(a => a !== positiveAgent).slice(0, 2);
|
|
585
|
+
|
|
586
|
+
for (const negAgent of hardNegatives) {
|
|
587
|
+
const negEmb = agentEmbeddings[negAgent];
|
|
588
|
+
if (negEmb) {
|
|
589
|
+
trainer.addTriplet(
|
|
590
|
+
example.task, taskEmb,
|
|
591
|
+
positiveAgent, positiveEmb,
|
|
592
|
+
negAgent, negEmb,
|
|
593
|
+
!!example.confusing_with
|
|
594
|
+
);
|
|
595
|
+
tripletCount++;
|
|
596
|
+
}
|
|
597
|
+
}
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
console.log(`Created ${tripletCount} triplets\n`);
|
|
601
|
+
|
|
602
|
+
// Train
|
|
603
|
+
console.log('Training...');
|
|
604
|
+
const result = trainer.train();
|
|
605
|
+
|
|
606
|
+
// Export data
|
|
607
|
+
console.log('\nExporting training data...');
|
|
608
|
+
const exportPath = trainer.exportTrainingData();
|
|
609
|
+
const loraConfig = trainer.generateLoRAConfig();
|
|
610
|
+
const scriptPath = trainer.generateTrainingScript();
|
|
611
|
+
|
|
612
|
+
// Summary
|
|
613
|
+
console.log('\n═══════════════════════════════════════════════════════════════════════════');
|
|
614
|
+
console.log(' TRAINING SUMMARY');
|
|
615
|
+
console.log('═══════════════════════════════════════════════════════════════════════════\n');
|
|
616
|
+
|
|
617
|
+
console.log('Training data exported:');
|
|
618
|
+
console.log(` - ${exportPath}/triplets.jsonl (${tripletCount} triplets)`);
|
|
619
|
+
console.log(` - ${exportPath}/triplets.csv (spreadsheet format)`);
|
|
620
|
+
console.log(` - ${exportPath}/embeddings.json (precomputed embeddings)`);
|
|
621
|
+
console.log(` - ${exportPath}/lora_config.json (LoRA configuration)`);
|
|
622
|
+
console.log(` - ${exportPath}/train.sh (training script)\n`);
|
|
623
|
+
|
|
624
|
+
console.log('Training loss (simulated):');
|
|
625
|
+
console.log(` Initial: ${result.initialLoss.toFixed(4)}`);
|
|
626
|
+
console.log(` Final: ${result.finalLoss.toFixed(4)}`);
|
|
627
|
+
console.log(` Improvement: ${result.improvement.toFixed(1)}%\n`);
|
|
628
|
+
|
|
629
|
+
console.log('To fine-tune on GPU:');
|
|
630
|
+
console.log(` cd ${exportPath}`);
|
|
631
|
+
console.log(' chmod +x train.sh && ./train.sh\n');
|
|
632
|
+
}
|
|
633
|
+
|
|
634
|
+
async function runTrainDataset(flags) {
|
|
635
|
+
console.log('\n╔═══════════════════════════════════════════════════════════════════════════╗');
|
|
636
|
+
console.log('║ RuvLTRA Training Dataset Generator ║');
|
|
637
|
+
console.log('╚═══════════════════════════════════════════════════════════════════════════╝\n');
|
|
638
|
+
|
|
639
|
+
const stats = getDatasetStats();
|
|
640
|
+
const outputPath = flags.output || './data/training';
|
|
641
|
+
|
|
642
|
+
console.log(`Generating training data for ${stats.agentTypes} agent types...\n`);
|
|
643
|
+
|
|
644
|
+
// Generate dataset
|
|
645
|
+
const examples = generateTrainingDataset();
|
|
646
|
+
const pairs = generateContrastivePairs();
|
|
647
|
+
|
|
648
|
+
// Export
|
|
649
|
+
const { existsSync, mkdirSync, writeFileSync } = require('fs');
|
|
650
|
+
const { join } = require('path');
|
|
651
|
+
|
|
652
|
+
if (!existsSync(outputPath)) {
|
|
653
|
+
mkdirSync(outputPath, { recursive: true });
|
|
654
|
+
}
|
|
655
|
+
|
|
656
|
+
// JSONL format
|
|
657
|
+
writeFileSync(
|
|
658
|
+
join(outputPath, 'routing-examples.jsonl'),
|
|
659
|
+
examples.map(e => JSON.stringify(e)).join('\n')
|
|
660
|
+
);
|
|
661
|
+
|
|
662
|
+
// Contrastive pairs
|
|
663
|
+
writeFileSync(
|
|
664
|
+
join(outputPath, 'contrastive-pairs.jsonl'),
|
|
665
|
+
pairs.map(p => JSON.stringify(p)).join('\n')
|
|
666
|
+
);
|
|
667
|
+
|
|
668
|
+
// CSV format
|
|
669
|
+
const csv = [
|
|
670
|
+
'task,agent,complexity,confusing_with',
|
|
671
|
+
...examples.map(e => `"${e.task.replace(/"/g, '""')}",${e.agent},${e.complexity || ''},${e.confusing_with || ''}`)
|
|
672
|
+
].join('\n');
|
|
673
|
+
writeFileSync(join(outputPath, 'routing-examples.csv'), csv);
|
|
674
|
+
|
|
675
|
+
console.log('Export complete:');
|
|
676
|
+
console.log(` - ${join(outputPath, 'routing-examples.jsonl')} (${examples.length} examples)`);
|
|
677
|
+
console.log(` - ${join(outputPath, 'contrastive-pairs.jsonl')} (${pairs.length} pairs)`);
|
|
678
|
+
console.log(` - ${join(outputPath, 'routing-examples.csv')} (spreadsheet format)\n`);
|
|
679
|
+
|
|
680
|
+
console.log('Dataset statistics:');
|
|
681
|
+
console.log(` Total examples: ${stats.totalExamples}`);
|
|
682
|
+
console.log(` Contrastive pairs: ${stats.contrastivePairs}`);
|
|
683
|
+
console.log(` Agent types: ${stats.agentTypes}`);
|
|
684
|
+
console.log(` Agents: ${stats.agents.join(', ')}\n`);
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
async function runTrainStats(flags) {
|
|
688
|
+
const stats = getDatasetStats();
|
|
689
|
+
|
|
690
|
+
if (flags.json) {
|
|
691
|
+
console.log(JSON.stringify(stats, null, 2));
|
|
692
|
+
} else {
|
|
693
|
+
console.log('\nTraining Dataset Statistics:');
|
|
694
|
+
console.log(` Total examples: ${stats.totalExamples}`);
|
|
695
|
+
console.log(` Contrastive pairs: ${stats.contrastivePairs}`);
|
|
696
|
+
console.log(` Agent types: ${stats.agentTypes}`);
|
|
697
|
+
console.log('\nAgents:');
|
|
698
|
+
for (const agent of stats.agents) {
|
|
699
|
+
const data = AGENT_TRAINING_DATA[agent];
|
|
700
|
+
console.log(` - ${agent}: ${data.examples.length} examples, ${data.keywords.length} keywords`);
|
|
701
|
+
}
|
|
702
|
+
}
|
|
703
|
+
}
|
|
704
|
+
|
|
705
|
+
async function runBenchmarkFull(flags) {
|
|
706
|
+
console.log('\n╔═══════════════════════════════════════════════════════════════════════════╗');
|
|
707
|
+
console.log('║ RUVLTRA FULL BENCHMARK SUITE ║');
|
|
708
|
+
console.log('║ Evaluating for Claude Code Use Cases ║');
|
|
709
|
+
console.log('╚═══════════════════════════════════════════════════════════════════════════╝\n');
|
|
710
|
+
|
|
711
|
+
const llm = new RuvLLM({ embeddingDim: 768, learningEnabled: false });
|
|
712
|
+
const simd = new SimdOps();
|
|
713
|
+
|
|
714
|
+
const modelName = llm.isNativeLoaded() ? 'RuvLTRA (native)' : 'RuvLTRA (JS fallback)';
|
|
715
|
+
|
|
716
|
+
// Router
|
|
717
|
+
const router = (task) => {
|
|
718
|
+
try {
|
|
719
|
+
const decision = llm.route(task);
|
|
720
|
+
return { agent: decision.model, confidence: decision.confidence };
|
|
721
|
+
} catch (e) {
|
|
722
|
+
return baselineKeywordRouter(task);
|
|
723
|
+
}
|
|
724
|
+
};
|
|
725
|
+
|
|
726
|
+
// Embedder
|
|
727
|
+
const embedder = (text) => {
|
|
728
|
+
try {
|
|
729
|
+
return llm.embed(text);
|
|
730
|
+
} catch (e) {
|
|
731
|
+
const hash = text.split('').reduce((h, c) => ((h << 5) - h + c.charCodeAt(0)) | 0, 0);
|
|
732
|
+
return Array.from({ length: 768 }, (_, i) => Math.sin(hash + i) * 0.5);
|
|
733
|
+
}
|
|
734
|
+
};
|
|
735
|
+
|
|
736
|
+
// Similarity
|
|
737
|
+
const similarity = (a, b) => {
|
|
738
|
+
try {
|
|
739
|
+
return simd.cosineSimilarity(a, b);
|
|
740
|
+
} catch (e) {
|
|
741
|
+
let dot = 0, normA = 0, normB = 0;
|
|
742
|
+
for (let i = 0; i < a.length; i++) {
|
|
743
|
+
dot += a[i] * b[i];
|
|
744
|
+
normA += a[i] * a[i];
|
|
745
|
+
normB += b[i] * b[i];
|
|
746
|
+
}
|
|
747
|
+
return dot / (Math.sqrt(normA) * Math.sqrt(normB));
|
|
748
|
+
}
|
|
749
|
+
};
|
|
750
|
+
|
|
751
|
+
const results = runFullBenchmark(router, embedder, similarity, modelName);
|
|
752
|
+
|
|
753
|
+
if (flags.json) {
|
|
754
|
+
console.log(JSON.stringify(results, null, 2));
|
|
755
|
+
} else {
|
|
756
|
+
console.log(formatFullResults(results));
|
|
757
|
+
}
|
|
758
|
+
}
|
|
759
|
+
|
|
238
760
|
function printHelp() {
|
|
239
761
|
console.log(`
|
|
240
762
|
RuvLLM - Self-learning LLM Orchestration
|
|
@@ -254,6 +776,25 @@ Commands:
|
|
|
254
776
|
info Show system information
|
|
255
777
|
help Show this help message
|
|
256
778
|
|
|
779
|
+
Model Management:
|
|
780
|
+
models list List available RuvLTRA models
|
|
781
|
+
models download <id> Download a model from HuggingFace
|
|
782
|
+
models download --all Download all available models
|
|
783
|
+
models status Check which models are downloaded
|
|
784
|
+
models delete <id> Delete a downloaded model
|
|
785
|
+
|
|
786
|
+
Claude Code Benchmarks:
|
|
787
|
+
benchmark routing Test agent routing accuracy (100 tasks)
|
|
788
|
+
benchmark embedding Test embedding quality (similarity, search, clustering)
|
|
789
|
+
benchmark full Run complete benchmark suite
|
|
790
|
+
benchmark compare Compare Qwen base vs RuvLTRA Claude Code
|
|
791
|
+
benchmark simd Run SIMD performance benchmark
|
|
792
|
+
|
|
793
|
+
Training & Fine-tuning:
|
|
794
|
+
train contrastive Run contrastive fine-tuning pipeline
|
|
795
|
+
train dataset Generate training dataset (JSONL, CSV)
|
|
796
|
+
train stats Show training dataset statistics
|
|
797
|
+
|
|
257
798
|
Options:
|
|
258
799
|
--json Output as JSON
|
|
259
800
|
--temperature <float> Sampling temperature (0.0-2.0)
|
|
@@ -264,6 +805,18 @@ Options:
|
|
|
264
805
|
--metadata <json> Metadata for memory add
|
|
265
806
|
--dims <int> Dimensions for benchmark (default: 768)
|
|
266
807
|
--iterations <int> Iterations for benchmark (default: 1000)
|
|
808
|
+
--force Force re-download even if model exists
|
|
809
|
+
--all Apply to all models (download/delete)
|
|
810
|
+
--output <path> Output directory for training data
|
|
811
|
+
--epochs <int> Number of training epochs (default: 10)
|
|
812
|
+
--batch-size <int> Training batch size (default: 16)
|
|
813
|
+
--lr <float> Learning rate (default: 0.0001)
|
|
814
|
+
--margin <float> Triplet loss margin (default: 0.5)
|
|
815
|
+
|
|
816
|
+
Available Models (from https://huggingface.co/ruv/ruvltra):
|
|
817
|
+
claude-code RuvLTRA Claude Code (398MB) - Claude Code workflows
|
|
818
|
+
small RuvLTRA Small (398MB) - Edge devices, IoT
|
|
819
|
+
medium RuvLTRA Medium (669MB) - General purpose
|
|
267
820
|
|
|
268
821
|
Examples:
|
|
269
822
|
ruvllm query "What is machine learning?"
|
|
@@ -273,6 +826,23 @@ Examples:
|
|
|
273
826
|
ruvllm similarity "hello world" "hi there"
|
|
274
827
|
ruvllm benchmark --dims 1024 --iterations 5000
|
|
275
828
|
|
|
829
|
+
# Model management
|
|
830
|
+
ruvllm models list
|
|
831
|
+
ruvllm models download claude-code
|
|
832
|
+
ruvllm models download --all
|
|
833
|
+
ruvllm models status
|
|
834
|
+
|
|
835
|
+
# Claude Code benchmarks
|
|
836
|
+
ruvllm benchmark routing # Test task routing accuracy
|
|
837
|
+
ruvllm benchmark embedding # Test embedding quality
|
|
838
|
+
ruvllm benchmark full # Run complete benchmark suite
|
|
839
|
+
|
|
840
|
+
# Training & fine-tuning
|
|
841
|
+
ruvllm train contrastive # Run contrastive fine-tuning
|
|
842
|
+
ruvllm train dataset # Generate training dataset
|
|
843
|
+
ruvllm train stats # Show dataset statistics
|
|
844
|
+
ruvllm train contrastive --epochs 20 --output ./my-training
|
|
845
|
+
|
|
276
846
|
Learn more: https://github.com/ruvnet/ruvector
|
|
277
847
|
`);
|
|
278
848
|
}
|
|
@@ -360,13 +930,60 @@ async function main() {
|
|
|
360
930
|
break;
|
|
361
931
|
|
|
362
932
|
case 'benchmark':
|
|
363
|
-
|
|
933
|
+
const benchSubcmd = positional[0];
|
|
934
|
+
if (benchSubcmd === 'routing') {
|
|
935
|
+
await runBenchmarkRouting(flags);
|
|
936
|
+
} else if (benchSubcmd === 'embedding' || benchSubcmd === 'embeddings') {
|
|
937
|
+
await runBenchmarkEmbedding(flags);
|
|
938
|
+
} else if (benchSubcmd === 'full' || benchSubcmd === 'all') {
|
|
939
|
+
await runBenchmarkFull(flags);
|
|
940
|
+
} else if (benchSubcmd === 'compare') {
|
|
941
|
+
await runBenchmarkCompare(flags);
|
|
942
|
+
} else if (benchSubcmd === 'simd' || !benchSubcmd) {
|
|
943
|
+
// Default to SIMD benchmark for backwards compatibility
|
|
944
|
+
await runBenchmark(flags);
|
|
945
|
+
} else {
|
|
946
|
+
console.error(`Unknown benchmark type: ${benchSubcmd}`);
|
|
947
|
+
console.error('Available: routing, embedding, full, simd, compare');
|
|
948
|
+
process.exit(1);
|
|
949
|
+
}
|
|
364
950
|
break;
|
|
365
951
|
|
|
366
952
|
case 'info':
|
|
367
953
|
await runInfo(flags);
|
|
368
954
|
break;
|
|
369
955
|
|
|
956
|
+
case 'models':
|
|
957
|
+
const modelsSubcmd = positional[0];
|
|
958
|
+
if (!modelsSubcmd || modelsSubcmd === 'list') {
|
|
959
|
+
await runModelsList(flags);
|
|
960
|
+
} else if (modelsSubcmd === 'download') {
|
|
961
|
+
await runModelsDownload(positional[1], flags);
|
|
962
|
+
} else if (modelsSubcmd === 'status') {
|
|
963
|
+
await runModelsStatus(flags);
|
|
964
|
+
} else if (modelsSubcmd === 'delete' || modelsSubcmd === 'remove') {
|
|
965
|
+
await runModelsDelete(positional[1], flags);
|
|
966
|
+
} else {
|
|
967
|
+
// Treat subcommand as model ID for download
|
|
968
|
+
await runModelsDownload(modelsSubcmd, flags);
|
|
969
|
+
}
|
|
970
|
+
break;
|
|
971
|
+
|
|
972
|
+
case 'train':
|
|
973
|
+
const trainSubcmd = positional[0];
|
|
974
|
+
if (!trainSubcmd || trainSubcmd === 'contrastive') {
|
|
975
|
+
await runTrainContrastive(flags);
|
|
976
|
+
} else if (trainSubcmd === 'dataset') {
|
|
977
|
+
await runTrainDataset(flags);
|
|
978
|
+
} else if (trainSubcmd === 'stats') {
|
|
979
|
+
await runTrainStats(flags);
|
|
980
|
+
} else {
|
|
981
|
+
console.error(`Unknown train subcommand: ${trainSubcmd}`);
|
|
982
|
+
console.error('Available: contrastive, dataset, stats');
|
|
983
|
+
process.exit(1);
|
|
984
|
+
}
|
|
985
|
+
break;
|
|
986
|
+
|
|
370
987
|
default:
|
|
371
988
|
console.error(`Unknown command: ${command}`);
|
|
372
989
|
console.error('Run "ruvllm help" for usage information.');
|