@cachly-dev/mcp-server 0.10.88 → 0.10.90

package/dist/src/index.js

@@ -792,7 +792,7 @@ async function handleTool(name, args) {
             '',
             'Run the setup wizard once in your terminal:',
             '   ```',
-            '   npx @cachly-dev/mcp-server@latest setup',
+            '   npx @cachly-dev/mcp-server@latest autopilot',
             '   ```',
             '',
             'Or get your API key at: https://cachly.dev/setup-ai',
@@ -1406,7 +1406,7 @@ if (process.argv[2] === 'digest') {
     const instanceId = process.env.CACHLY_BRAIN_INSTANCE_ID ?? '';
     if (!apiKey || !instanceId) {
         console.log('\n⚠️  CACHLY_JWT and CACHLY_BRAIN_INSTANCE_ID must be set.');
-        console.log('   Run: npx @cachly-dev/mcp-server@latest setup\n');
+        console.log('   Run: npx @cachly-dev/mcp-server@latest autopilot\n');
         process.exit(1);
     }
     process.stdout.write('\n🧠 Fetching your weekly Brain digest...\n\n');
@@ -1495,7 +1495,7 @@ if (process.argv[2] === 'invite') {
     const apiKey = process.env.CACHLY_JWT ?? '';
     if (!apiKey) {
         console.log('\n⚠️  CACHLY_JWT must be set.');
-        console.log('   Run: npx @cachly-dev/mcp-server@latest setup\n');
+        console.log('   Run: npx @cachly-dev/mcp-server@latest autopilot\n');
         process.exit(1);
     }
     try {
@@ -1742,7 +1742,7 @@ if (process.argv[2] === 'upgrade') {
             console.log('');
             console.log('  Update your editor configs to pick up the new version:');
             console.log('');
-            console.log('  \x1b[32m  npx @cachly-dev/mcp-server@latest setup\x1b[0m');
+            console.log('  \x1b[32m  npx @cachly-dev/mcp-server@latest autopilot\x1b[0m');
             console.log('');
             console.log('  \x1b[2m(npx always fetches the latest when @latest is specified)\x1b[0m');
             console.log('');
@@ -1754,6 +1754,84 @@ if (process.argv[2] === 'upgrade') {
     }
     process.exit(0);
 }
+// ── CLI: cachly bench ─────────────────────────────────────────────────────────
+// Usage: npx @cachly-dev/mcp-server@latest bench [--json] [corpus.json]
+// Runs the three-ranker recall quality benchmark (cachly vs BM25 vs flat-file)
+// on the bundled external corpus. No auth required. One shareable headline.
+if (process.argv[2] === 'bench') {
+    const { fileURLToPath } = await import('node:url');
+    const { dirname: _bd, resolve: _br } = await import('node:path');
+    const { loadExternalCorpus, runExternalBenchmark } = await import('./bench/external-corpus.js');
+    const asJson = process.argv.includes('--json');
+    const customPath = process.argv.slice(3).find(a => !a.startsWith('--'));
+    const here = _bd(fileURLToPath(import.meta.url));
+    const corpusPath = customPath
+        ? _br(process.cwd(), customPath)
+        : _br(here, 'bench', 'external', 'sample-corpus.json');
+    try {
+        const corpus = await loadExternalCorpus(corpusPath);
+        const r = await runExternalBenchmark(corpus);
+        if (asJson) {
+            console.log(JSON.stringify({
+                corpus: corpus.name,
+                lessons: r.corpusSize,
+                queries: r.queryCount,
+                cachly_p1: Math.round(r.cachly.precisionAt1 * 1000) / 10,
+                flatfile_p1: Math.round(r.flatfile.precisionAt1 * 1000) / 10,
+                cachly_mrr: Math.round(r.cachly.mrr * 1000) / 10,
+                flatfile_mrr: Math.round(r.flatfile.mrr * 1000) / 10,
+                p1_lift_vs_flatfile_pct: Math.round((r.cachly.precisionAt1 - r.flatfile.precisionAt1) * 1000) / 10,
+                mrr_lift_vs_flatfile_pct: Math.round((r.cachly.mrr - r.flatfile.mrr) * 1000) / 10,
+            }, null, 2));
+            process.exit(0);
+        }
+        const p1Lift = ((r.cachly.precisionAt1 - r.flatfile.precisionAt1) * 100).toFixed(1);
+        const mrrLift = ((r.cachly.mrr - r.flatfile.mrr) * 100).toFixed(1);
+        const p1Bm25 = ((r.cachly.precisionAt1 - r.baseline.precisionAt1) * 100).toFixed(1);
+        const p1Sign = r.cachly.precisionAt1 >= r.flatfile.precisionAt1 ? '\x1b[32m+' : '\x1b[31m';
+        const mrrSign = r.cachly.mrr >= r.flatfile.mrr ? '\x1b[32m+' : '\x1b[31m';
+        console.log('');
+        console.log('\x1b[35m  ╔══════════════════════════════════════════════════════════════╗\x1b[0m');
+        console.log('\x1b[35m  ║\x1b[0m  \x1b[1m🧠 cachly recall quality benchmark\x1b[0m                       \x1b[35m║\x1b[0m');
+        console.log('\x1b[35m  ╚══════════════════════════════════════════════════════════════╝\x1b[0m');
+        console.log('');
+        console.log(`  Corpus : ${corpus.name ?? corpusPath}`);
+        console.log(`  Dataset: ${r.corpusSize} lessons · ${r.queryCount} queries · 10 engineering domains`);
+        console.log(`           (k8s, DB, auth, CI, frontend, API, payments, observability, Node.js, infra)`);
+        console.log('');
+        console.log('  \x1b[1mPrecision@1\x1b[0m (finds the right lesson at rank #1):');
+        console.log(`    flat-file memory : \x1b[33m${(r.flatfile.precisionAt1 * 100).toFixed(1)}%\x1b[0m  (naive keyword overlap — no quality signal)`);
+        console.log(`    raw BM25         : \x1b[33m${(r.baseline.precisionAt1 * 100).toFixed(1)}%\x1b[0m  (keyword ranking)`);
+        console.log(`    cachly           : \x1b[1m\x1b[32m${(r.cachly.precisionAt1 * 100).toFixed(1)}%\x1b[0m  (BM25 + quality reranking)`);
+        console.log('');
+        console.log('  \x1b[1mMean Reciprocal Rank\x1b[0m (average rank of the right answer):');
+        console.log(`    flat-file memory : \x1b[33m${(r.flatfile.mrr * 100).toFixed(1)}%\x1b[0m`);
+        console.log(`    raw BM25         : \x1b[33m${(r.baseline.mrr * 100).toFixed(1)}%\x1b[0m`);
+        console.log(`    cachly           : \x1b[1m\x1b[32m${(r.cachly.mrr * 100).toFixed(1)}%\x1b[0m`);
+        console.log('');
+        console.log('  \x1b[1mLift vs flat-file memory\x1b[0m (what you get by switching from plain memory files):');
+        console.log(`    Precision@1 : ${p1Sign}${p1Lift}%\x1b[0m`);
+        console.log(`    MRR         : ${mrrSign}${mrrLift}%\x1b[0m`);
+        console.log(`    vs BM25     : \x1b[32m+${p1Bm25}%\x1b[0m Precision@1 (quality reranking on top of keyword search)`);
+        console.log('');
+        console.log('  \x1b[2mFlat-file = lexical overlap over raw text files (how Anthropic/Claude Memory works).\x1b[0m');
+        console.log('  \x1b[2mAdversarial distractors included — failure lessons that share the exact query vocabulary.\x1b[0m');
+        console.log('  \x1b[2mcachly ignores them because they have low confidence / failed outcome.\x1b[0m');
+        console.log('');
+        console.log('  Share your results:');
+        console.log(`  \x1b[36m"My Brain: Precision@1 ${(r.cachly.precisionAt1 * 100).toFixed(1)}% (${p1Sign}${p1Lift}% vs flat-file memory, ${r.queryCount} queries)"\x1b[0m`);
+        console.log('');
+        console.log('  Run on a custom corpus:');
+        console.log('  \x1b[2m  npx @cachly-dev/mcp-server@latest bench my-incidents.json\x1b[0m');
+        console.log('  \x1b[2m  npx @cachly-dev/mcp-server@latest bench --json   # machine-readable\x1b[0m');
+        console.log('');
+    }
+    catch (e) {
+        console.error(`\n❌ bench failed: ${e.message}\n`);
+        process.exit(1);
+    }
+    process.exit(0);
+}
 // ── CLI: cachly demo ──────────────────────────────────────────────────────────
 // Usage: npx @cachly-dev/mcp-server@latest demo
 // Zero-signup. Reads local git history and shows what the AI Brain would know.
@@ -1905,7 +1983,7 @@ if (process.argv[2] === 'demo') {
     });
     const previewURL = `https://cachly.dev/preview?${previewParams.toString()}`;
     console.log('  \x1b[1mMake this permanent (free, 1–5 minutes):\x1b[0m');
-    console.log('  \x1b[32m$ npx @cachly-dev/mcp-server@latest setup\x1b[0m');
+    console.log('  \x1b[32m$ npx @cachly-dev/mcp-server@latest autopilot\x1b[0m');
     console.log('');
     console.log(`  \x1b[90m🔗 Shareable preview:\x1b[0m \x1b[36m${previewURL}\x1b[0m`);
     console.log('');
@@ -1922,7 +2000,7 @@ if (process.argv[2] === 'share') {
     const instanceId = process.env.CACHLY_BRAIN_INSTANCE_ID ?? '';
     if (!apiKey || !instanceId) {
         console.log('\n⚠️  CACHLY_JWT and CACHLY_BRAIN_INSTANCE_ID must be set.');
-        console.log('   Run: npx @cachly-dev/mcp-server@latest setup  (takes 1–5 minutes)\n');
+        console.log('   Run: npx @cachly-dev/mcp-server@latest autopilot  (takes 1–5 minutes)\n');
         process.exit(1);
     }
     process.stdout.write('\n🧠 Fetching your Brain stats...\n');
@@ -1954,7 +2032,7 @@ if (process.argv[2] === 'share') {
         console.log('├─────────────────────────────────────────────────────────────┤');
         console.log('│  Works with: Claude Code · Cursor · Windsurf · Copilot      │');
         console.log('│  Free · GDPR · German servers                                │');
-        console.log('│  \x1b[35mnpx @cachly-dev/mcp-server@latest setup\x1b[0m                     │');
+        console.log('│  \x1b[35mnpx @cachly-dev/mcp-server@latest autopilot\x1b[0m                 │');
         console.log('└─────────────────────────────────────────────────────────────┘');
         console.log('');
         // Tweet text
@@ -1984,7 +2062,7 @@ if (process.argv[2] === 'publish') {
     const instanceId = process.env.CACHLY_BRAIN_INSTANCE_ID ?? '';
     if (!apiKey || !instanceId) {
         console.log('\n⚠️  CACHLY_JWT and CACHLY_BRAIN_INSTANCE_ID must be set.');
-        console.log('   Run: npx @cachly-dev/mcp-server@latest setup\n');
+        console.log('   Run: npx @cachly-dev/mcp-server@latest autopilot\n');
         process.exit(1);
     }
     const titleIdx = process.argv.indexOf('--title');
@@ -2079,7 +2157,7 @@ if (process.argv[2] === 'badge') {
     const instanceId = process.env.CACHLY_BRAIN_INSTANCE_ID ?? '';
     if (!instanceId) {
         console.log('\n⚠️  CACHLY_BRAIN_INSTANCE_ID must be set.');
-        console.log('   Run: npx @cachly-dev/mcp-server@latest setup  (takes 1–5 minutes)\n');
+        console.log('   Run: npx @cachly-dev/mcp-server@latest autopilot  (takes 1–5 minutes)\n');
         process.exit(1);
     }
     const badgeUrl = `${API_URL}/api/v1/badge/${instanceId}`;
@@ -2130,6 +2208,7 @@ if (!process.argv[2] && process.stdout.isTTY) {
     console.log('  \x1b[36m  npx @cachly-dev/mcp-server@latest badge\x1b[0m    ← README badge for your Brain');
     console.log('  \x1b[36m  npx @cachly-dev/mcp-server@latest invite\x1b[0m   ← Invite a teammate');
     console.log('  \x1b[36m  npx @cachly-dev/mcp-server@latest join <token>\x1b[0m ← Accept a Brain invite');
+    console.log('  \x1b[36m  npx @cachly-dev/mcp-server@latest bench\x1b[0m     ← Recall quality vs flat-file memory');
     console.log('  \x1b[36m  npx @cachly-dev/mcp-server@latest upgrade\x1b[0m  ← Check for updates');
     console.log('');
     console.log('  \x1b[90mWorks with: Claude Code · Cursor · Windsurf · GitHub Copilot · Cline · Zed\x1b[0m');
@@ -2176,7 +2255,7 @@ if (process.argv[2] === 'init') {
     }
     if (!instanceId || !apiKey) {
         console.error('\nUsage: npx @cachly-dev/mcp-server@latest init --instance-id <uuid> --api-key <cky_live_...> [--editor claude|cursor|windsurf|copilot|continue] [--project-dir /path] [--api-url https://your-self-hosted-backend]\n');
-        console.error('First time? Run the zero-config wizard (signs you in, auto-provisions): npx @cachly-dev/mcp-server@latest setup\n');
+        console.error('First time? Run the zero-config wizard (signs you in, auto-provisions): npx @cachly-dev/mcp-server@latest autopilot\n');
         console.error('Get your credentials from: https://cachly.dev/setup-ai\n');
         process.exit(1);
     }
@@ -2259,7 +2338,7 @@ if (process.argv[2] === 'status') {
     const instanceId = process.env.CACHLY_BRAIN_INSTANCE_ID ?? '';
     if (!apiKey || !instanceId) {
         console.log('\n⚠️  CACHLY_JWT and CACHLY_BRAIN_INSTANCE_ID must be set.');
-        console.log('   Run: npx @cachly-dev/mcp-server@latest setup\n');
+        console.log('   Run: npx @cachly-dev/mcp-server@latest autopilot\n');
         process.exit(1);
     }
     try {
@@ -2337,7 +2416,7 @@ if (process.argv[2] === 'health') {
     console.log('🔑 Auth token');
     const jwt = process.env.CACHLY_JWT ?? '';
     if (!jwt) {
-        fail('CACHLY_JWT not set — run: npx @cachly-dev/mcp-server@latest setup');
+        fail('CACHLY_JWT not set — run: npx @cachly-dev/mcp-server@latest autopilot');
     }
     else if (jwt.startsWith('cky_')) {
         // Long-lived API key (the credential `setup` provisions). It is not a JWT and
@@ -2486,7 +2565,7 @@ if (process.argv[2] === 'health') {
         }
     }
     if (configsFound === 0) {
-        fail('No editor MCP configs found — run: npx @cachly-dev/mcp-server@latest setup');
+        fail('No editor MCP configs found — run: npx @cachly-dev/mcp-server@latest autopilot');
     }
     // ── 5. Git hook ─────────────────────────────────────────────────────────────
     console.log('\n🪝 Git hook (ambient learning)');
@@ -2509,7 +2588,7 @@ if (process.argv[2] === 'health') {
         }
     }
     else {
-        warn('.git/hooks/post-commit not found — run: npx @cachly-dev/mcp-server@latest setup');
+        warn('.git/hooks/post-commit not found — run: npx @cachly-dev/mcp-server@latest autopilot');
     }
     // ── 6. Embedding provider (BYOK) ─────────────────────────────────────────────
     console.log('\n🧬 Embedding provider (semantic search)');
@@ -2542,7 +2621,7 @@ if (process.argv[2] === 'health') {
     }
     else {
         console.log(`❌ ${failed} check(s) failed, ${passed} passed\n`);
-        console.log(`💡 Fix issues with: npx @cachly-dev/mcp-server@latest setup\n`);
+        console.log(`💡 Fix issues with: npx @cachly-dev/mcp-server@latest autopilot\n`);
         process.exit(1);
     }
     process.exit(0);
@@ -2732,7 +2811,7 @@ if (process.argv[2] === 'setup' || _isAutopilotCli) {
                 sendFunnelEvent('device_flow_failed', { reason: 'non_interactive_no_key' });
                 console.error('\n   ⚠️  This terminal is non-interactive, so the key cannot be pasted here.');
                 console.error('   Get your key at the URL above, then set it and re-run setup:\n');
-                console.error('     \x1b[1mCACHLY_JWT=cky_live_xxx npx @cachly-dev/mcp-server@latest setup\x1b[0m\n');
+                console.error('     \x1b[1mCACHLY_JWT=cky_live_xxx npx @cachly-dev/mcp-server@latest autopilot\x1b[0m\n');
                 console.error('   Or add it to your editor\'s MCP config under env.CACHLY_JWT.\n');
                 rl.close();
                 process.exit(1);
@@ -3312,7 +3391,7 @@ if (!JWT) {
 //   • Editor as an MCP stdio server (non-TTY)  → ONE stderr hint, then KEEP RUNNING
 //     so tools/list works and the first tool call starts the browser sign-in.
 // Skip entirely for CLI subcommands that intentionally run without credentials.
-const _cliNoAuthCommands = ['demo', 'share', 'publish', 'health', 'setup', 'autopilot', 'init', 'digest', 'invite', 'badge', 'join', 'upgrade', 'tool-specs', 'openapi'];
+const _cliNoAuthCommands = ['demo', 'share', 'publish', 'health', 'setup', 'autopilot', 'init', 'digest', 'invite', 'badge', 'join', 'upgrade', 'bench', 'tool-specs', 'openapi'];
 if (!JWT && !_cliNoAuthCommands.includes(process.argv[2] ?? '')) {
     const runningInTerminal = !process.argv[2] && process.stdout.isTTY === true && _isMain;
     if (runningInTerminal) {
@@ -3327,9 +3406,9 @@ if (!JWT && !_cliNoAuthCommands.includes(process.argv[2] ?? '')) {
             '║                                                                  ║\n' +
             '║    👉  https://cachly.dev/setup-ai                              ║\n' +
             '║                                                                  ║\n' +
-            '║  Then run the interactive setup wizard:                         ║\n' +
+            '║  Then run the one-command setup wizard:                         ║\n' +
             '║                                                                  ║\n' +
-            '║    npx @cachly-dev/mcp-server@latest setup                      ║\n' +
+            '║    npx @cachly-dev/mcp-server@latest autopilot                  ║\n' +
             '║                                                                  ║\n' +
             '║  Free tier — no credit card required.                           ║\n' +
             '╚══════════════════════════════════════════════════════════════════╝\n' +
@@ -3343,7 +3422,7 @@ if (!JWT && !_cliNoAuthCommands.includes(process.argv[2] ?? '')) {
         // hint to stderr (shown in the editor's MCP log) and fall through so the
         // server starts and the zero-credential device flow can run on first tool call.
         process.stderr.write('\n🧠 cachly: no CACHLY_JWT set yet — call any cachly tool and a 10-second browser sign-in starts automatically.\n' +
-            '   (Or run once: npx @cachly-dev/mcp-server@latest setup)\n\n');
+            '   (Or run once: npx @cachly-dev/mcp-server@latest autopilot)\n\n');
     }
 }
 else {
@@ -3373,7 +3452,7 @@ if (!process.env.CACHLY_NO_UPDATE_CHECK) {
                 const latest = data?.version ?? '';
                 if (latest && latest !== CURRENT_VERSION) {
                     process.stderr.write(`\n⚡ cachly update available: ${CURRENT_VERSION} → ${latest}\n` +
-                        `   Run: npx @cachly-dev/mcp-server@latest setup\n\n`);
+                        `   Run: npx @cachly-dev/mcp-server@latest autopilot\n\n`);
                 }
             }
         }