@cachly-dev/mcp-server 0.10.89 → 0.10.90

package/dist/src/index.js

@@ -1754,6 +1754,84 @@ if (process.argv[2] === 'upgrade') {
     }
     process.exit(0);
 }
+// ── CLI: cachly bench ─────────────────────────────────────────────────────────
+// Usage: npx @cachly-dev/mcp-server@latest bench [--json] [corpus.json]
+// Runs the three-ranker recall quality benchmark (cachly vs BM25 vs flat-file)
+// on the bundled external corpus. No auth required. One shareable headline.
+if (process.argv[2] === 'bench') {
+    const { fileURLToPath } = await import('node:url');
+    const { dirname: _bd, resolve: _br } = await import('node:path');
+    const { loadExternalCorpus, runExternalBenchmark } = await import('./bench/external-corpus.js');
+    const asJson = process.argv.includes('--json');
+    const customPath = process.argv.slice(3).find(a => !a.startsWith('--'));
+    const here = _bd(fileURLToPath(import.meta.url));
+    const corpusPath = customPath
+        ? _br(process.cwd(), customPath)
+        : _br(here, 'bench', 'external', 'sample-corpus.json');
+    try {
+        const corpus = await loadExternalCorpus(corpusPath);
+        const r = await runExternalBenchmark(corpus);
+        if (asJson) {
+            console.log(JSON.stringify({
+                corpus: corpus.name,
+                lessons: r.corpusSize,
+                queries: r.queryCount,
+                cachly_p1: Math.round(r.cachly.precisionAt1 * 1000) / 10,
+                flatfile_p1: Math.round(r.flatfile.precisionAt1 * 1000) / 10,
+                cachly_mrr: Math.round(r.cachly.mrr * 1000) / 10,
+                flatfile_mrr: Math.round(r.flatfile.mrr * 1000) / 10,
+                p1_lift_vs_flatfile_pct: Math.round((r.cachly.precisionAt1 - r.flatfile.precisionAt1) * 1000) / 10,
+                mrr_lift_vs_flatfile_pct: Math.round((r.cachly.mrr - r.flatfile.mrr) * 1000) / 10,
+            }, null, 2));
+            process.exit(0);
+        }
+        const p1Lift = ((r.cachly.precisionAt1 - r.flatfile.precisionAt1) * 100).toFixed(1);
+        const mrrLift = ((r.cachly.mrr - r.flatfile.mrr) * 100).toFixed(1);
+        const p1Bm25 = ((r.cachly.precisionAt1 - r.baseline.precisionAt1) * 100).toFixed(1);
+        const p1Sign = r.cachly.precisionAt1 >= r.flatfile.precisionAt1 ? '\x1b[32m+' : '\x1b[31m';
+        const mrrSign = r.cachly.mrr >= r.flatfile.mrr ? '\x1b[32m+' : '\x1b[31m';
+        console.log('');
+        console.log('\x1b[35m  ╔══════════════════════════════════════════════════════════════╗\x1b[0m');
+        console.log('\x1b[35m  ║\x1b[0m  \x1b[1m🧠 cachly recall quality benchmark\x1b[0m                       \x1b[35m║\x1b[0m');
+        console.log('\x1b[35m  ╚══════════════════════════════════════════════════════════════╝\x1b[0m');
+        console.log('');
+        console.log(`  Corpus : ${corpus.name ?? corpusPath}`);
+        console.log(`  Dataset: ${r.corpusSize} lessons · ${r.queryCount} queries · 10 engineering domains`);
+        console.log(`           (k8s, DB, auth, CI, frontend, API, payments, observability, Node.js, infra)`);
+        console.log('');
+        console.log('  \x1b[1mPrecision@1\x1b[0m (finds the right lesson at rank #1):');
+        console.log(`    flat-file memory : \x1b[33m${(r.flatfile.precisionAt1 * 100).toFixed(1)}%\x1b[0m  (naive keyword overlap — no quality signal)`);
+        console.log(`    raw BM25         : \x1b[33m${(r.baseline.precisionAt1 * 100).toFixed(1)}%\x1b[0m  (keyword ranking)`);
+        console.log(`    cachly           : \x1b[1m\x1b[32m${(r.cachly.precisionAt1 * 100).toFixed(1)}%\x1b[0m  (BM25 + quality reranking)`);
+        console.log('');
+        console.log('  \x1b[1mMean Reciprocal Rank\x1b[0m (average rank of the right answer):');
+        console.log(`    flat-file memory : \x1b[33m${(r.flatfile.mrr * 100).toFixed(1)}%\x1b[0m`);
+        console.log(`    raw BM25         : \x1b[33m${(r.baseline.mrr * 100).toFixed(1)}%\x1b[0m`);
+        console.log(`    cachly           : \x1b[1m\x1b[32m${(r.cachly.mrr * 100).toFixed(1)}%\x1b[0m`);
+        console.log('');
+        console.log('  \x1b[1mLift vs flat-file memory\x1b[0m (what you get by switching from plain memory files):');
+        console.log(`    Precision@1 : ${p1Sign}${p1Lift}%\x1b[0m`);
+        console.log(`    MRR         : ${mrrSign}${mrrLift}%\x1b[0m`);
+        console.log(`    vs BM25     : \x1b[32m+${p1Bm25}%\x1b[0m Precision@1 (quality reranking on top of keyword search)`);
+        console.log('');
+        console.log('  \x1b[2mFlat-file = lexical overlap over raw text files (how Anthropic/Claude Memory works).\x1b[0m');
+        console.log('  \x1b[2mAdversarial distractors included — failure lessons that share the exact query vocabulary.\x1b[0m');
+        console.log('  \x1b[2mcachly ignores them because they have low confidence / failed outcome.\x1b[0m');
+        console.log('');
+        console.log('  Share your results:');
+        console.log(`  \x1b[36m"My Brain: Precision@1 ${(r.cachly.precisionAt1 * 100).toFixed(1)}% (${p1Sign}${p1Lift}% vs flat-file memory, ${r.queryCount} queries)"\x1b[0m`);
+        console.log('');
+        console.log('  Run on a custom corpus:');
+        console.log('  \x1b[2m  npx @cachly-dev/mcp-server@latest bench my-incidents.json\x1b[0m');
+        console.log('  \x1b[2m  npx @cachly-dev/mcp-server@latest bench --json   # machine-readable\x1b[0m');
+        console.log('');
+    }
+    catch (e) {
+        console.error(`\n❌ bench failed: ${e.message}\n`);
+        process.exit(1);
+    }
+    process.exit(0);
+}
 // ── CLI: cachly demo ──────────────────────────────────────────────────────────
 // Usage: npx @cachly-dev/mcp-server@latest demo
 // Zero-signup. Reads local git history and shows what the AI Brain would know.
@@ -2130,6 +2208,7 @@ if (!process.argv[2] && process.stdout.isTTY) {
     console.log('  \x1b[36m  npx @cachly-dev/mcp-server@latest badge\x1b[0m    ← README badge for your Brain');
     console.log('  \x1b[36m  npx @cachly-dev/mcp-server@latest invite\x1b[0m   ← Invite a teammate');
     console.log('  \x1b[36m  npx @cachly-dev/mcp-server@latest join <token>\x1b[0m ← Accept a Brain invite');
+    console.log('  \x1b[36m  npx @cachly-dev/mcp-server@latest bench\x1b[0m     ← Recall quality vs flat-file memory');
     console.log('  \x1b[36m  npx @cachly-dev/mcp-server@latest upgrade\x1b[0m  ← Check for updates');
     console.log('');
     console.log('  \x1b[90mWorks with: Claude Code · Cursor · Windsurf · GitHub Copilot · Cline · Zed\x1b[0m');
@@ -3312,7 +3391,7 @@ if (!JWT) {
 //   • Editor as an MCP stdio server (non-TTY)  → ONE stderr hint, then KEEP RUNNING
 //     so tools/list works and the first tool call starts the browser sign-in.
 // Skip entirely for CLI subcommands that intentionally run without credentials.
-const _cliNoAuthCommands = ['demo', 'share', 'publish', 'health', 'setup', 'autopilot', 'init', 'digest', 'invite', 'badge', 'join', 'upgrade', 'tool-specs', 'openapi'];
+const _cliNoAuthCommands = ['demo', 'share', 'publish', 'health', 'setup', 'autopilot', 'init', 'digest', 'invite', 'badge', 'join', 'upgrade', 'bench', 'tool-specs', 'openapi'];
 if (!JWT && !_cliNoAuthCommands.includes(process.argv[2] ?? '')) {
     const runningInTerminal = !process.argv[2] && process.stdout.isTTY === true && _isMain;
     if (runningInTerminal) {