@ruvector/ruvllm 0.2.4 → 2.4.0

package/dist/esm/benchmarks/index.js

@@ -0,0 +1,110 @@
+/**
+ * RuvLTRA Benchmark Suite
+ *
+ * Comprehensive benchmarks for evaluating RuvLTRA models
+ * on Claude Code-specific use cases.
+ */
+export * from './routing-benchmark';
+export * from './embedding-benchmark';
+export * from './model-comparison';
+import { runRoutingBenchmark, formatRoutingResults, ROUTING_TEST_CASES, } from './routing-benchmark';
+import { runEmbeddingBenchmark, formatEmbeddingResults, SIMILARITY_TEST_PAIRS, SEARCH_TEST_CASES, CLUSTER_TEST_CASES, } from './embedding-benchmark';
+/**
+ * Run all benchmarks with a given model
+ */
+export function runFullBenchmark(router, embedder, similarityFn, modelName = 'unknown') {
+    const routing = runRoutingBenchmark(router);
+    const embedding = runEmbeddingBenchmark(embedder, similarityFn);
+    return {
+        routing,
+        embedding,
+        timestamp: new Date().toISOString(),
+        model: modelName,
+    };
+}
+/**
+ * Format full benchmark results
+ */
+export function formatFullResults(results) {
+    const lines = [];
+    lines.push('');
+    lines.push('╔═══════════════════════════════════════════════════════════════════════════╗');
+    lines.push('║                    RUVLTRA BENCHMARK SUITE                                ║');
+    lines.push('║            Claude Code Use Case Evaluation                                ║');
+    lines.push('╠═══════════════════════════════════════════════════════════════════════════╣');
+    lines.push(`║  Model: ${results.model.padEnd(64)}║`);
+    lines.push(`║  Date:  ${results.timestamp.padEnd(64)}║`);
+    lines.push('╚═══════════════════════════════════════════════════════════════════════════╝');
+    lines.push(formatRoutingResults(results.routing));
+    lines.push(formatEmbeddingResults(results.embedding));
+    // Overall assessment
+    lines.push('');
+    lines.push('═══════════════════════════════════════════════════════════════');
+    lines.push('                      OVERALL ASSESSMENT');
+    lines.push('═══════════════════════════════════════════════════════════════');
+    const routingScore = results.routing.accuracy;
+    const embeddingScore = (results.embedding.similarityAccuracy +
+        results.embedding.searchMRR +
+        results.embedding.clusterPurity) / 3;
+    const overallScore = (routingScore + embeddingScore) / 2;
+    lines.push('');
+    lines.push(`  Routing Score:   ${(routingScore * 100).toFixed(1)}%`);
+    lines.push(`  Embedding Score: ${(embeddingScore * 100).toFixed(1)}%`);
+    lines.push(`  ─────────────────────────`);
+    lines.push(`  Overall Score:   ${(overallScore * 100).toFixed(1)}%`);
+    lines.push('');
+    if (overallScore >= 0.8) {
+        lines.push('  ✓ EXCELLENT - Highly suitable for Claude Code workflows');
+    }
+    else if (overallScore >= 0.6) {
+        lines.push('  ~ GOOD - Suitable for most Claude Code use cases');
+    }
+    else if (overallScore >= 0.4) {
+        lines.push('  ~ ACCEPTABLE - May work but consider alternatives');
+    }
+    else {
+        lines.push('  ✗ NEEDS IMPROVEMENT - Consider different model or fine-tuning');
+    }
+    lines.push('');
+    lines.push('═══════════════════════════════════════════════════════════════');
+    return lines.join('\n');
+}
+/**
+ * Compare two models
+ */
+export function compareModels(results1, results2) {
+    const lines = [];
+    lines.push('');
+    lines.push('╔═══════════════════════════════════════════════════════════════════════════╗');
+    lines.push('║                       MODEL COMPARISON                                    ║');
+    lines.push('╚═══════════════════════════════════════════════════════════════════════════╝');
+    lines.push('');
+    const metrics = [
+        { name: 'Routing Accuracy', v1: results1.routing.accuracy, v2: results2.routing.accuracy },
+        { name: 'Similarity Detection', v1: results1.embedding.similarityAccuracy, v2: results2.embedding.similarityAccuracy },
+        { name: 'Search MRR', v1: results1.embedding.searchMRR, v2: results2.embedding.searchMRR },
+        { name: 'Search NDCG', v1: results1.embedding.searchNDCG, v2: results2.embedding.searchNDCG },
+        { name: 'Cluster Purity', v1: results1.embedding.clusterPurity, v2: results2.embedding.clusterPurity },
+        { name: 'Routing Latency (ms)', v1: results1.routing.avgLatencyMs, v2: results2.routing.avgLatencyMs, lowerBetter: true },
+    ];
+    lines.push(`${'Metric'.padEnd(25)} ${results1.model.padEnd(15)} ${results2.model.padEnd(15)} Winner`);
+    lines.push('─'.repeat(70));
+    for (const m of metrics) {
+        const val1 = m.lowerBetter ? m.v1 : m.v1;
+        const val2 = m.lowerBetter ? m.v2 : m.v2;
+        let winner;
+        if (m.lowerBetter) {
+            winner = val1 < val2 ? results1.model : val2 < val1 ? results2.model : 'tie';
+        }
+        else {
+            winner = val1 > val2 ? results1.model : val2 > val1 ? results2.model : 'tie';
+        }
+        const v1Str = m.lowerBetter ? val1.toFixed(2) : (val1 * 100).toFixed(1) + '%';
+        const v2Str = m.lowerBetter ? val2.toFixed(2) : (val2 * 100).toFixed(1) + '%';
+        lines.push(`${m.name.padEnd(25)} ${v1Str.padEnd(15)} ${v2Str.padEnd(15)} ${winner}`);
+    }
+    return lines.join('\n');
+}
+// Export constants for external use
+export { ROUTING_TEST_CASES, SIMILARITY_TEST_PAIRS, SEARCH_TEST_CASES, CLUSTER_TEST_CASES, };
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5kZXguanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvYmVuY2htYXJrcy9pbmRleC50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQTs7Ozs7R0FLRztBQUVILGNBQWMscUJBQXFCLENBQUM7QUFDcEMsY0FBYyx1QkFBdUIsQ0FBQztBQUN0QyxjQUFjLG9CQUFvQixDQUFDO0FBRW5DLE9BQU8sRUFDTCxtQkFBbUIsRUFDbkIsb0JBQW9CLEVBRXBCLGtCQUFrQixHQUVuQixNQUFNLHFCQUFxQixDQUFDO0FBRTdCLE9BQU8sRUFDTCxxQkFBcUIsRUFDckIsc0JBQXNCLEVBQ3RCLHFCQUFxQixFQUNyQixpQkFBaUIsRUFDakIsa0JBQWtCLEdBRW5CLE1BQU0sdUJBQXVCLENBQUM7QUFTL0I7O0dBRUc7QUFDSCxNQUFNLFVBQVUsZ0JBQWdCLENBQzlCLE1BQStELEVBQy9ELFFBQW9DLEVBQ3BDLFlBQWtELEVBQ2xELFlBQW9CLFNBQVM7SUFFN0IsTUFBTSxPQUFPLEdBQUcsbUJBQW1CLENBQUMsTUFBTSxDQUFDLENBQUM7SUFDNUMsTUFBTSxTQUFTLEdBQUcscUJBQXFCLENBQUMsUUFBUSxFQUFFLFlBQVksQ0FBQyxDQUFDO0lBRWhFLE9BQU87UUFDTCxPQUFPO1FBQ1AsU0FBUztRQUNULFNBQVMsRUFBRSxJQUFJLElBQUksRUFBRSxDQUFDLFdBQVcsRUFBRTtRQUNuQyxLQUFLLEVBQUUsU0FBUztLQUNqQixDQUFDO0FBQ0osQ0FBQztBQUVEOztHQUVHO0FBQ0gsTUFBTSxVQUFVLGlCQUFpQixDQUFDLE9BQTZCO0lBQzdELE1BQU0sS0FBSyxHQUFhLEVBQUUsQ0FBQztJQUUzQixLQUFLLENBQUMsSUFBSSxDQUFDLEVBQUUsQ0FBQyxDQUFDO0lBQ2YsS0FBSyxDQUFDLElBQUksQ0FBQywrRUFBK0UsQ0FBQyxDQUFDO0lBQzVGLEtBQUssQ0FBQyxJQUFJLENBQUMsK0VBQStFLENBQUMsQ0FBQztJQUM1RixLQUFLLENBQUMsSUFBSSxDQUFDLCtFQUErRSxDQUFDLENBQUM7SUFDNUYsS0FBSyxDQUFDLElBQUksQ0FBQywrRUFBK0UsQ0FBQyxDQUFDO0lBQzVGLEtBQUssQ0FBQyxJQUFJLENBQUMsYUFBYSxPQUFPLENBQUMsS0FBSyxDQUFDLE1BQU0sQ0FBQyxFQUFFLENBQUMsR0FBRyxDQUFDLENBQUM7SUFDckQsS0FBSyxDQUFDLElBQUksQ0FBQyxhQUFhLE9BQU8sQ0FBQyxTQUFTLENBQUMsTUFBTSxDQUFDLEVBQUUsQ0FBQyxHQUFHLENBQUMsQ0FBQztJQUN6RCxLQUFLLENBQUMsSUFBSSxDQUFDLCtFQUErRSxDQUFDLENBQUM7SUFFNUYsS0FBSyxDQUFDLElBQUksQ0FBQyxvQkFBb0IsQ0FBQyxPQUFPLENBQUMsT0FBTyxDQUFDLENBQUMsQ0FBQztJQUNsRCxLQUFLLENBQUMsSUFBSSxDQUFDLHNCQUFzQixDQUFDLE9BQU8sQ0FBQyxTQUFTLENBQUMsQ0FBQyxDQUFDO0lBRXRELHFCQUFxQjtJQUNyQixLQUFLLENBQUMsSUFBSSxDQUFDLEVBQUUsQ0FBQyxDQUFDO0lBQ2YsS0FBSyxDQUFDLElBQUksQ0FBQyxpRUFBaUUsQ0FBQyxDQUFDO0lBQzlFLEtBQUssQ0FBQyxJQUFJLENBQUMsMENBQTBDLENBQUMsQ0FBQztJQUN2RCxLQUFLLENBQUMsSUFBSSxDQUFDLGlFQUFpRSxDQUFDLENBQUM7SUFFOUUsTUFBTSxZQUFZLEdBQUcsT0FBTyxDQUFDLE9BQU8sQ0FBQyxRQUFRLENBQUM7SUFDOUMsTUFBTSxjQUFjLEdBQUcsQ0FDckIsT0FBTyxDQUFDLFNBQVMsQ0FBQyxrQkFBa0I7UUFDcEMsT0FBTyxDQUFDLFNBQVMsQ0FBQyxTQUFTO1FBQzNCLE9BQU8sQ0FBQyxTQUFTLENBQUMsYUFBYSxDQUNoQyxHQUFHLENBQUMsQ0FBQztJQUVOLE1BQU0sWUFBWSxHQUFHLENBQUMsWUFBWSxHQUFHLGNBQWMsQ0FBQyxHQUFHLENBQUMsQ0FBQztJQUV6RCxLQUFLLENBQUMsSUFBSSxDQUFDLEVBQUUsQ0FBQyxDQUFDO0lBQ2YsS0FBSyxDQUFDLElBQUksQ0FBQyxzQkFBc0IsQ0FBQyxZQUFZLEdBQUcsR0FBRyxDQUFDLENBQUMsT0FBTyxDQUFDLENBQUMsQ0FBQyxHQUFHLENBQUMsQ0FBQztJQUNyRSxLQUFLLENBQUMsSUFBSSxDQUFDLHNCQUFzQixDQUFDLGNBQWMsR0FBRyxHQUFHLENBQUMsQ0FBQyxPQUFPLENBQUMsQ0FBQyxDQUFDLEdBQUcsQ0FBQyxDQUFDO0lBQ3ZFLEtBQUssQ0FBQyxJQUFJLENBQUMsNkJBQTZCLENBQUMsQ0FBQztJQUMxQyxLQUFLLENBQUMsSUFBSSxDQUFDLHNCQUFzQixDQUFDLFlBQVksR0FBRyxHQUFHLENBQUMsQ0FBQyxPQUFPLENBQUMsQ0FBQyxDQUFDLEdBQUcsQ0FBQyxDQUFDO0lBQ3JFLEtBQUssQ0FBQyxJQUFJLENBQUMsRUFBRSxDQUFDLENBQUM7SUFFZixJQUFJLFlBQVksSUFBSSxHQUFHLEVBQUUsQ0FBQztRQUN4QixLQUFLLENBQUMsSUFBSSxDQUFDLDJEQUEyRCxDQUFDLENBQUM7SUFDMUUsQ0FBQztTQUFNLElBQUksWUFBWSxJQUFJLEdBQUcsRUFBRSxDQUFDO1FBQy9CLEtBQUssQ0FBQyxJQUFJLENBQUMsb0RBQW9ELENBQUMsQ0FBQztJQUNuRSxDQUFDO1NBQU0sSUFBSSxZQUFZLElBQUksR0FBRyxFQUFFLENBQUM7UUFDL0IsS0FBSyxDQUFDLElBQUksQ0FBQyxxREFBcUQsQ0FBQyxDQUFDO0lBQ3BFLENBQUM7U0FBTSxDQUFDO1FBQ04sS0FBSyxDQUFDLElBQUksQ0FBQyxpRUFBaUUsQ0FBQyxDQUFDO0lBQ2hGLENBQUM7SUFFRCxLQUFLLENBQUMsSUFBSSxDQUFDLEVBQUUsQ0FBQyxDQUFDO0lBQ2YsS0FBSyxDQUFDLElBQUksQ0FBQyxpRUFBaUUsQ0FBQyxDQUFDO0lBRTlFLE9BQU8sS0FBSyxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsQ0FBQztBQUMxQixDQUFDO0FBRUQ7O0dBRUc7QUFDSCxNQUFNLFVBQVUsYUFBYSxDQUMzQixRQUE4QixFQUM5QixRQUE4QjtJQUU5QixNQUFNLEtBQUssR0FBYSxFQUFFLENBQUM7SUFFM0IsS0FBSyxDQUFDLElBQUksQ0FBQyxFQUFFLENBQUMsQ0FBQztJQUNmLEtBQUssQ0FBQyxJQUFJLENBQUMsK0VBQStFLENBQUMsQ0FBQztJQUM1RixLQUFLLENBQUMsSUFBSSxDQUFDLCtFQUErRSxDQUFDLENBQUM7SUFDNUYsS0FBSyxDQUFDLElBQUksQ0FBQywrRUFBK0UsQ0FBQyxDQUFDO0lBQzVGLEtBQUssQ0FBQyxJQUFJLENBQUMsRUFBRSxDQUFDLENBQUM7SUFFZixNQUFNLE9BQU8sR0FBRztRQUNkLEVBQUUsSUFBSSxFQUFFLGtCQUFrQixFQUFFLEVBQUUsRUFBRSxRQUFRLENBQUMsT0FBTyxDQUFDLFFBQVEsRUFBRSxFQUFFLEVBQUUsUUFBUSxDQUFDLE9BQU8sQ0FBQyxRQUFRLEVBQUU7UUFDMUYsRUFBRSxJQUFJLEVBQUUsc0JBQXNCLEVBQUUsRUFBRSxFQUFFLFFBQVEsQ0FBQyxTQUFTLENBQUMsa0JBQWtCLEVBQUUsRUFBRSxFQUFFLFFBQVEsQ0FBQyxTQUFTLENBQUMsa0JBQWtCLEVBQUU7UUFDdEgsRUFBRSxJQUFJLEVBQUUsWUFBWSxFQUFFLEVBQUUsRUFBRSxRQUFRLENBQUMsU0FBUyxDQUFDLFNBQVMsRUFBRSxFQUFFLEVBQUUsUUFBUSxDQUFDLFNBQVMsQ0FBQyxTQUFTLEVBQUU7UUFDMUYsRUFBRSxJQUFJLEVBQUUsYUFBYSxFQUFFLEVBQUUsRUFBRSxRQUFRLENBQUMsU0FBUyxDQUFDLFVBQVUsRUFBRSxFQUFFLEVBQUUsUUFBUSxDQUFDLFNBQVMsQ0FBQyxVQUFVLEVBQUU7UUFDN0YsRUFBRSxJQUFJLEVBQUUsZ0JBQWdCLEVBQUUsRUFBRSxFQUFFLFFBQVEsQ0FBQyxTQUFTLENBQUMsYUFBYSxFQUFFLEVBQUUsRUFBRSxRQUFRLENBQUMsU0FBUyxDQUFDLGFBQWEsRUFBRTtRQUN0RyxFQUFFLElBQUksRUFBRSxzQkFBc0IsRUFBRSxFQUFFLEVBQUUsUUFBUSxDQUFDLE9BQU8sQ0FBQyxZQUFZLEVBQUUsRUFBRSxFQUFFLFFBQVEsQ0FBQyxPQUFPLENBQUMsWUFBWSxFQUFFLFdBQVcsRUFBRSxJQUFJLEVBQUU7S0FDMUgsQ0FBQztJQUVGLEtBQUssQ0FBQyxJQUFJLENBQUMsR0FBRyxRQUFRLENBQUMsTUFBTSxDQUFDLEVBQUUsQ0FBQyxJQUFJLFFBQVEsQ0FBQyxLQUFLLENBQUMsTUFBTSxDQUFDLEVBQUUsQ0FBQyxJQUFJLFFBQVEsQ0FBQyxLQUFLLENBQUMsTUFBTSxDQUFDLEVBQUUsQ0FBQyxTQUFTLENBQUMsQ0FBQztJQUN0RyxLQUFLLENBQUMsSUFBSSxDQUFDLEdBQUcsQ0FBQyxNQUFNLENBQUMsRUFBRSxDQUFDLENBQUMsQ0FBQztJQUUzQixLQUFLLE1BQU0sQ0FBQyxJQUFJLE9BQU8sRUFBRSxDQUFDO1FBQ3hCLE1BQU0sSUFBSSxHQUFHLENBQUMsQ0FBQyxXQUFXLENBQUMsQ0FBQyxDQUFDLENBQUMsQ0FBQyxFQUFFLENBQUMsQ0FBQyxDQUFDLENBQUMsQ0FBQyxFQUFFLENBQUM7UUFDekMsTUFBTSxJQUFJLEdBQUcsQ0FBQyxDQUFDLFdBQVcsQ0FBQyxDQUFDLENBQUMsQ0FBQyxDQUFDLEVBQUUsQ0FBQyxDQUFDLENBQUMsQ0FBQyxDQUFDLEVBQUUsQ0FBQztRQUV6QyxJQUFJLE1BQWMsQ0FBQztRQUNuQixJQUFJLENBQUMsQ0FBQyxXQUFXLEVBQUUsQ0FBQztZQUNsQixNQUFNLEdBQUcsSUFBSSxHQUFHLElBQUksQ0FBQyxDQUFDLENBQUMsUUFBUSxDQUFDLEtBQUssQ0FBQyxDQUFDLENBQUMsSUFBSSxHQUFHLElBQUksQ0FBQyxDQUFDLENBQUMsUUFBUSxDQUFDLEtBQUssQ0FBQyxDQUFDLENBQUMsS0FBSyxDQUFDO1FBQy9FLENBQUM7YUFBTSxDQUFDO1lBQ04sTUFBTSxHQUFHLElBQUksR0FBRyxJQUFJLENBQUMsQ0FBQyxDQUFDLFFBQVEsQ0FBQyxLQUFLLENBQUMsQ0FBQyxDQUFDLElBQUksR0FBRyxJQUFJLENBQUMsQ0FBQyxDQUFDLFFBQVEsQ0FBQyxLQUFLLENBQUMsQ0FBQyxDQUFDLEtBQUssQ0FBQztRQUMvRSxDQUFDO1FBRUQsTUFBTSxLQUFLLEdBQUcsQ0FBQyxDQUFDLFdBQVcsQ0FBQyxDQUFDLENBQUMsSUFBSSxDQUFDLE9BQU8sQ0FBQyxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUMsQ0FBQyxJQUFJLEdBQUcsR0FBRyxDQUFDLENBQUMsT0FBTyxDQUFDLENBQUMsQ0FBQyxHQUFHLEdBQUcsQ0FBQztRQUM5RSxNQUFNLEtBQUssR0FBRyxDQUFDLENBQUMsV0FBVyxDQUFDLENBQUMsQ0FBQyxJQUFJLENBQUMsT0FBTyxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUMsQ0FBQyxDQUFDLElBQUksR0FBRyxHQUFHLENBQUMsQ0FBQyxPQUFPLENBQUMsQ0FBQyxDQUFDLEdBQUcsR0FBRyxDQUFDO1FBRTlFLEtBQUssQ0FBQyxJQUFJLENBQUMsR0FBRyxDQUFDLENBQUMsSUFBSSxDQUFDLE1BQU0sQ0FBQyxFQUFFLENBQUMsSUFBSSxLQUFLLENBQUMsTUFBTSxDQUFDLEVBQUUsQ0FBQyxJQUFJLEtBQUssQ0FBQyxNQUFNLENBQUMsRUFBRSxDQUFDLElBQUksTUFBTSxFQUFFLENBQUMsQ0FBQztJQUN2RixDQUFDO0lBRUQsT0FBTyxLQUFLLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxDQUFDO0FBQzFCLENBQUM7QUFFRCxvQ0FBb0M7QUFDcEMsT0FBTyxFQUNMLGtCQUFrQixFQUNsQixxQkFBcUIsRUFDckIsaUJBQWlCLEVBQ2pCLGtCQUFrQixHQUNuQixDQUFDIiwic291cmNlc0NvbnRlbnQiOlsiLyoqXG4gKiBSdXZMVFJBIEJlbmNobWFyayBTdWl0ZVxuICpcbiAqIENvbXByZWhlbnNpdmUgYmVuY2htYXJrcyBmb3IgZXZhbHVhdGluZyBSdXZMVFJBIG1vZGVsc1xuICogb24gQ2xhdWRlIENvZGUtc3BlY2lmaWMgdXNlIGNhc2VzLlxuICovXG5cbmV4cG9ydCAqIGZyb20gJy4vcm91dGluZy1iZW5jaG1hcmsnO1xuZXhwb3J0ICogZnJvbSAnLi9lbWJlZGRpbmctYmVuY2htYXJrJztcbmV4cG9ydCAqIGZyb20gJy4vbW9kZWwtY29tcGFyaXNvbic7XG5cbmltcG9ydCB7XG4gIHJ1blJvdXRpbmdCZW5jaG1hcmssXG4gIGZvcm1hdFJvdXRpbmdSZXN1bHRzLFxuICBiYXNlbGluZUtleXdvcmRSb3V0ZXIsXG4gIFJPVVRJTkdfVEVTVF9DQVNFUyxcbiAgdHlwZSBSb3V0aW5nQmVuY2htYXJrUmVzdWx0cyxcbn0gZnJvbSAnLi9yb3V0aW5nLWJlbmNobWFyayc7XG5cbmltcG9ydCB7XG4gIHJ1bkVtYmVkZGluZ0JlbmNobWFyayxcbiAgZm9ybWF0RW1iZWRkaW5nUmVzdWx0cyxcbiAgU0lNSUxBUklUWV9URVNUX1BBSVJTLFxuICBTRUFSQ0hfVEVTVF9DQVNFUyxcbiAgQ0xVU1RFUl9URVNUX0NBU0VTLFxuICB0eXBlIEVtYmVkZGluZ0JlbmNobWFya1Jlc3VsdHMsXG59IGZyb20gJy4vZW1iZWRkaW5nLWJlbmNobWFyayc7XG5cbmV4cG9ydCBpbnRlcmZhY2UgRnVsbEJlbmNobWFya1Jlc3VsdHMge1xuICByb3V0aW5nOiBSb3V0aW5nQmVuY2htYXJrUmVzdWx0cztcbiAgZW1iZWRkaW5nOiBFbWJlZGRpbmdCZW5jaG1hcmtSZXN1bHRzO1xuICB0aW1lc3RhbXA6IHN0cmluZztcbiAgbW9kZWw6IHN0cmluZztcbn1cblxuLyoqXG4gKiBSdW4gYWxsIGJlbmNobWFya3Mgd2l0aCBhIGdpdmVuIG1vZGVsXG4gKi9cbmV4cG9ydCBmdW5jdGlvbiBydW5GdWxsQmVuY2htYXJrKFxuICByb3V0ZXI6ICh0YXNrOiBzdHJpbmcpID0+IHsgYWdlbnQ6IHN0cmluZzsgY29uZmlkZW5jZTogbnVtYmVyIH0sXG4gIGVtYmVkZGVyOiAodGV4dDogc3RyaW5nKSA9PiBudW1iZXJbXSxcbiAgc2ltaWxhcml0eUZuOiAoYTogbnVtYmVyW10sIGI6IG51bWJlcltdKSA9PiBudW1iZXIsXG4gIG1vZGVsTmFtZTogc3RyaW5nID0gJ3Vua25vd24nXG4pOiBGdWxsQmVuY2htYXJrUmVzdWx0cyB7XG4gIGNvbnN0IHJvdXRpbmcgPSBydW5Sb3V0aW5nQmVuY2htYXJrKHJvdXRlcik7XG4gIGNvbnN0IGVtYmVkZGluZyA9IHJ1bkVtYmVkZGluZ0JlbmNobWFyayhlbWJlZGRlciwgc2ltaWxhcml0eUZuKTtcblxuICByZXR1cm4ge1xuICAgIHJvdXRpbmcsXG4gICAgZW1iZWRkaW5nLFxuICAgIHRpbWVzdGFtcDogbmV3IERhdGUoKS50b0lTT1N0cmluZygpLFxuICAgIG1vZGVsOiBtb2RlbE5hbWUsXG4gIH07XG59XG5cbi8qKlxuICogRm9ybWF0IGZ1bGwgYmVuY2htYXJrIHJlc3VsdHNcbiAqL1xuZXhwb3J0IGZ1bmN0aW9uIGZvcm1hdEZ1bGxSZXN1bHRzKHJlc3VsdHM6IEZ1bGxCZW5jaG1hcmtSZXN1bHRzKTogc3RyaW5nIHtcbiAgY29uc3QgbGluZXM6IHN0cmluZ1tdID0gW107XG5cbiAgbGluZXMucHVzaCgnJyk7XG4gIGxpbmVzLnB1c2goJ+KVlOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVlycpO1xuICBsaW5lcy5wdXNoKCfilZEgICAgICAgICAgICAgICAgICAgIFJVVkxUUkEgQkVOQ0hNQVJLIFNVSVRFICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICDilZEnKTtcbiAgbGluZXMucHVzaCgn4pWRICAgICAgICAgICAgQ2xhdWRlIENvZGUgVXNlIENhc2UgRXZhbHVhdGlvbiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg4pWRJyk7XG4gIGxpbmVzLnB1c2goJ+KVoOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVoycpO1xuICBsaW5lcy5wdXNoKGDilZEgIE1vZGVsOiAke3Jlc3VsdHMubW9kZWwucGFkRW5kKDY0KX3ilZFgKTtcbiAgbGluZXMucHVzaChg4pWRICBEYXRlOiAgJHtyZXN1bHRzLnRpbWVzdGFtcC5wYWRFbmQoNjQpfeKVkWApO1xuICBsaW5lcy5wdXNoKCfilZrilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZ0nKTtcblxuICBsaW5lcy5wdXNoKGZvcm1hdFJvdXRpbmdSZXN1bHRzKHJlc3VsdHMucm91dGluZykpO1xuICBsaW5lcy5wdXNoKGZvcm1hdEVtYmVkZGluZ1Jlc3VsdHMocmVzdWx0cy5lbWJlZGRpbmcpKTtcblxuICAvLyBPdmVyYWxsIGFzc2Vzc21lbnRcbiAgbGluZXMucHVzaCgnJyk7XG4gIGxpbmVzLnB1c2goJ+KVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkCcpO1xuICBsaW5lcy5wdXNoKCcgICAgICAgICAgICAgICAgICAgICAgT1ZFUkFMTCBBU1NFU1NNRU5UJyk7XG4gIGxpbmVzLnB1c2goJ+KVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkCcpO1xuXG4gIGNvbnN0IHJvdXRpbmdTY29yZSA9IHJlc3VsdHMucm91dGluZy5hY2N1cmFjeTtcbiAgY29uc3QgZW1iZWRkaW5nU2NvcmUgPSAoXG4gICAgcmVzdWx0cy5lbWJlZGRpbmcuc2ltaWxhcml0eUFjY3VyYWN5ICtcbiAgICByZXN1bHRzLmVtYmVkZGluZy5zZWFyY2hNUlIgK1xuICAgIHJlc3VsdHMuZW1iZWRkaW5nLmNsdXN0ZXJQdXJpdHlcbiAgKSAvIDM7XG5cbiAgY29uc3Qgb3ZlcmFsbFNjb3JlID0gKHJvdXRpbmdTY29yZSArIGVtYmVkZGluZ1Njb3JlKSAvIDI7XG5cbiAgbGluZXMucHVzaCgnJyk7XG4gIGxpbmVzLnB1c2goYCAgUm91dGluZyBTY29yZTogICAkeyhyb3V0aW5nU2NvcmUgKiAxMDApLnRvRml4ZWQoMSl9JWApO1xuICBsaW5lcy5wdXNoKGAgIEVtYmVkZGluZyBTY29yZTogJHsoZW1iZWRkaW5nU2NvcmUgKiAxMDApLnRvRml4ZWQoMSl9JWApO1xuICBsaW5lcy5wdXNoKGAgIOKUgOKUgOKUgOKUgOKUgOKUgOKUgOKUgOKUgOKUgOKUgOKUgOKUgOKUgOKUgOKUgOKUgOKUgOKUgOKUgOKUgOKUgOKUgOKUgOKUgGApO1xuICBsaW5lcy5wdXNoKGAgIE92ZXJhbGwgU2NvcmU6ICAgJHsob3ZlcmFsbFNjb3JlICogMTAwKS50b0ZpeGVkKDEpfSVgKTtcbiAgbGluZXMucHVzaCgnJyk7XG5cbiAgaWYgKG92ZXJhbGxTY29yZSA+PSAwLjgpIHtcbiAgICBsaW5lcy5wdXNoKCcgIOKckyBFWENFTExFTlQgLSBIaWdobHkgc3VpdGFibGUgZm9yIENsYXVkZSBDb2RlIHdvcmtmbG93cycpO1xuICB9IGVsc2UgaWYgKG92ZXJhbGxTY29yZSA+PSAwLjYpIHtcbiAgICBsaW5lcy5wdXNoKCcgIH4gR09PRCAtIFN1aXRhYmxlIGZvciBtb3N0IENsYXVkZSBDb2RlIHVzZSBjYXNlcycpO1xuICB9IGVsc2UgaWYgKG92ZXJhbGxTY29yZSA+PSAwLjQpIHtcbiAgICBsaW5lcy5wdXNoKCcgIH4gQUNDRVBUQUJMRSAtIE1heSB3b3JrIGJ1dCBjb25zaWRlciBhbHRlcm5hdGl2ZXMnKTtcbiAgfSBlbHNlIHtcbiAgICBsaW5lcy5wdXNoKCcgIOKclyBORUVEUyBJTVBST1ZFTUVOVCAtIENvbnNpZGVyIGRpZmZlcmVudCBtb2RlbCBvciBmaW5lLXR1bmluZycpO1xuICB9XG5cbiAgbGluZXMucHVzaCgnJyk7XG4gIGxpbmVzLnB1c2goJ+KVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkOKVkCcpO1xuXG4gIHJldHVybiBsaW5lcy5qb2luKCdcXG4nKTtcbn1cblxuLyoqXG4gKiBDb21wYXJlIHR3byBtb2RlbHNcbiAqL1xuZXhwb3J0IGZ1bmN0aW9uIGNvbXBhcmVNb2RlbHMoXG4gIHJlc3VsdHMxOiBGdWxsQmVuY2htYXJrUmVzdWx0cyxcbiAgcmVzdWx0czI6IEZ1bGxCZW5jaG1hcmtSZXN1bHRzXG4pOiBzdHJpbmcge1xuICBjb25zdCBsaW5lczogc3RyaW5nW10gPSBbXTtcblxuICBsaW5lcy5wdXNoKCcnKTtcbiAgbGluZXMucHVzaCgn4pWU4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWXJyk7XG4gIGxpbmVzLnB1c2goJ+KVkSAgICAgICAgICAgICAgICAgICAgICAgTU9ERUwgQ09NUEFSSVNPTiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIOKVkScpO1xuICBsaW5lcy5wdXNoKCfilZrilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZDilZ0nKTtcbiAgbGluZXMucHVzaCgnJyk7XG5cbiAgY29uc3QgbWV0cmljcyA9IFtcbiAgICB7IG5hbWU6ICdSb3V0aW5nIEFjY3VyYWN5JywgdjE6IHJlc3VsdHMxLnJvdXRpbmcuYWNjdXJhY3ksIHYyOiByZXN1bHRzMi5yb3V0aW5nLmFjY3VyYWN5IH0sXG4gICAgeyBuYW1lOiAnU2ltaWxhcml0eSBEZXRlY3Rpb24nLCB2MTogcmVzdWx0czEuZW1iZWRkaW5nLnNpbWlsYXJpdHlBY2N1cmFjeSwgdjI6IHJlc3VsdHMyLmVtYmVkZGluZy5zaW1pbGFyaXR5QWNjdXJhY3kgfSxcbiAgICB7IG5hbWU6ICdTZWFyY2ggTVJSJywgdjE6IHJlc3VsdHMxLmVtYmVkZGluZy5zZWFyY2hNUlIsIHYyOiByZXN1bHRzMi5lbWJlZGRpbmcuc2VhcmNoTVJSIH0sXG4gICAgeyBuYW1lOiAnU2VhcmNoIE5EQ0cnLCB2MTogcmVzdWx0czEuZW1iZWRkaW5nLnNlYXJjaE5EQ0csIHYyOiByZXN1bHRzMi5lbWJlZGRpbmcuc2VhcmNoTkRDRyB9LFxuICAgIHsgbmFtZTogJ0NsdXN0ZXIgUHVyaXR5JywgdjE6IHJlc3VsdHMxLmVtYmVkZGluZy5jbHVzdGVyUHVyaXR5LCB2MjogcmVzdWx0czIuZW1iZWRkaW5nLmNsdXN0ZXJQdXJpdHkgfSxcbiAgICB7IG5hbWU6ICdSb3V0aW5nIExhdGVuY3kgKG1zKScsIHYxOiByZXN1bHRzMS5yb3V0aW5nLmF2Z0xhdGVuY3lNcywgdjI6IHJlc3VsdHMyLnJvdXRpbmcuYXZnTGF0ZW5jeU1zLCBsb3dlckJldHRlcjogdHJ1ZSB9LFxuICBdO1xuXG4gIGxpbmVzLnB1c2goYCR7J01ldHJpYycucGFkRW5kKDI1KX0gJHtyZXN1bHRzMS5tb2RlbC5wYWRFbmQoMTUpfSAke3Jlc3VsdHMyLm1vZGVsLnBhZEVuZCgxNSl9IFdpbm5lcmApO1xuICBsaW5lcy5wdXNoKCfilIAnLnJlcGVhdCg3MCkpO1xuXG4gIGZvciAoY29uc3QgbSBvZiBtZXRyaWNzKSB7XG4gICAgY29uc3QgdmFsMSA9IG0ubG93ZXJCZXR0ZXIgPyBtLnYxIDogbS52MTtcbiAgICBjb25zdCB2YWwyID0gbS5sb3dlckJldHRlciA/IG0udjIgOiBtLnYyO1xuXG4gICAgbGV0IHdpbm5lcjogc3RyaW5nO1xuICAgIGlmIChtLmxvd2VyQmV0dGVyKSB7XG4gICAgICB3aW5uZXIgPSB2YWwxIDwgdmFsMiA/IHJlc3VsdHMxLm1vZGVsIDogdmFsMiA8IHZhbDEgPyByZXN1bHRzMi5tb2RlbCA6ICd0aWUnO1xuICAgIH0gZWxzZSB7XG4gICAgICB3aW5uZXIgPSB2YWwxID4gdmFsMiA/IHJlc3VsdHMxLm1vZGVsIDogdmFsMiA+IHZhbDEgPyByZXN1bHRzMi5tb2RlbCA6ICd0aWUnO1xuICAgIH1cblxuICAgIGNvbnN0IHYxU3RyID0gbS5sb3dlckJldHRlciA/IHZhbDEudG9GaXhlZCgyKSA6ICh2YWwxICogMTAwKS50b0ZpeGVkKDEpICsgJyUnO1xuICAgIGNvbnN0IHYyU3RyID0gbS5sb3dlckJldHRlciA/IHZhbDIudG9GaXhlZCgyKSA6ICh2YWwyICogMTAwKS50b0ZpeGVkKDEpICsgJyUnO1xuXG4gICAgbGluZXMucHVzaChgJHttLm5hbWUucGFkRW5kKDI1KX0gJHt2MVN0ci5wYWRFbmQoMTUpfSAke3YyU3RyLnBhZEVuZCgxNSl9ICR7d2lubmVyfWApO1xuICB9XG5cbiAgcmV0dXJuIGxpbmVzLmpvaW4oJ1xcbicpO1xufVxuXG4vLyBFeHBvcnQgY29uc3RhbnRzIGZvciBleHRlcm5hbCB1c2VcbmV4cG9ydCB7XG4gIFJPVVRJTkdfVEVTVF9DQVNFUyxcbiAgU0lNSUxBUklUWV9URVNUX1BBSVJTLFxuICBTRUFSQ0hfVEVTVF9DQVNFUyxcbiAgQ0xVU1RFUl9URVNUX0NBU0VTLFxufTtcbiJdfQ==

package/dist/esm/benchmarks/model-comparison.d.ts

@@ -0,0 +1,71 @@
+/**
+ * Model Comparison Benchmark
+ *
+ * Head-to-head comparison between:
+ * - Qwen2.5-0.5B-Instruct (base model)
+ * - RuvLTRA Claude Code 0.5B (fine-tuned for Claude Code)
+ *
+ * Tests routing accuracy and embedding quality for Claude Code use cases.
+ */
+import { type RoutingBenchmarkResults } from './routing-benchmark';
+import { type EmbeddingBenchmarkResults } from './embedding-benchmark';
+/** Model configuration */
+export interface ModelConfig {
+    id: string;
+    name: string;
+    url: string;
+    filename: string;
+    sizeBytes: number;
+    description: string;
+}
+/** Comparison models */
+export declare const COMPARISON_MODELS: Record<string, ModelConfig>;
+/** Comparison result */
+export interface ComparisonResult {
+    modelId: string;
+    modelName: string;
+    routing: RoutingBenchmarkResults;
+    embedding: EmbeddingBenchmarkResults;
+    overallScore: number;
+}
+/** Full comparison results */
+export interface FullComparisonResults {
+    timestamp: string;
+    baseline: ComparisonResult;
+    models: ComparisonResult[];
+    winner: string;
+    summary: string;
+}
+/**
+ * Get models directory
+ */
+export declare function getModelsDir(): string;
+/**
+ * Check if model is downloaded
+ */
+export declare function isModelDownloaded(modelId: string): boolean;
+/**
+ * Download a model with progress
+ */
+export declare function downloadModel(modelId: string, onProgress?: (percent: number, speed: number) => void): Promise<string>;
+/**
+ * Run comparison for a single model
+ */
+export declare function runModelComparison(modelId: string, modelName: string, embedder: (text: string) => number[]): ComparisonResult;
+/**
+ * Format comparison results
+ */
+export declare function formatComparisonResults(results: FullComparisonResults): string;
+/**
+ * Run full comparison
+ */
+export declare function runFullComparison(): Promise<FullComparisonResults>;
+declare const _default: {
+    COMPARISON_MODELS: Record<string, ModelConfig>;
+    runFullComparison: typeof runFullComparison;
+    formatComparisonResults: typeof formatComparisonResults;
+    downloadModel: typeof downloadModel;
+    isModelDownloaded: typeof isModelDownloaded;
+};
+export default _default;
+//# sourceMappingURL=model-comparison.d.ts.map

package/dist/esm/benchmarks/model-comparison.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"model-comparison.d.ts","sourceRoot":"","sources":["../../../src/benchmarks/model-comparison.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAQH,OAAO,EAML,KAAK,uBAAuB,EAC7B,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAGL,KAAK,yBAAyB,EAC/B,MAAM,uBAAuB,CAAC;AAE/B,0BAA0B;AAC1B,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,wBAAwB;AACxB,eAAO,MAAM,iBAAiB,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAiBzD,CAAC;AAEF,wBAAwB;AACxB,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,uBAAuB,CAAC;IACjC,SAAS,EAAE,yBAAyB,CAAC;IACrC,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,8BAA8B;AAC9B,MAAM,WAAW,qBAAqB;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,gBAAgB,CAAC;IAC3B,MAAM,EAAE,gBAAgB,EAAE,CAAC;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,wBAAgB,YAAY,IAAI,MAAM,CAErC;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAS1D;AAED;;GAEG;AACH,wBAAsB,aAAa,CACjC,OAAO,EAAE,MAAM,EACf,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,GACpD,OAAO,CAAC,MAAM,CAAC,CA2EjB;AAyJD;;GAEG;AACH,wBAAgB,kBAAkB,CAChC,OAAO,EAAE,MAAM,EACf,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,EAAE,GACnC,gBAAgB,CAyBlB;AAED;;GAEG;AACH,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,qBAAqB,GAAG,MAAM,CA8E9E;AAED;;GAEG;AACH,wBAAsB,iBAAiB,IAAI,OAAO,CAAC,qBAAqB,CAAC,CAqGxE;;;;;;;;AAED,wBAME"}