@cachly-dev/mcp-server 0.10.88 → 0.10.90

package/README.md

@@ -22,7 +22,7 @@
     <img src="https://img.shields.io/badge/GDPR-EU%20servers-green" alt="GDPR: EU servers" />
   </a>
   &nbsp;
-  <img src="https://img.shields.io/badge/107%20MCP%20tools-violet" alt="107 MCP tools" />
+  <img src="https://img.shields.io/badge/114%20MCP%20tools-violet" alt="114 MCP tools" />
   &nbsp;
   <img src="https://img.shields.io/badge/License-Apache--2.0-yellow" alt="License: Apache-2.0" />
 </p>
@@ -101,30 +101,20 @@ Like what you see? Make it permanent in the next step.
 
 ---
 
-## The plan — three steps, then it's automatic
-
-### 1. Run one command
+## Setup — one command
 
 ```bash
-npx @cachly-dev/mcp-server@latest setup
+npx @cachly-dev/mcp-server@latest autopilot
 ```
 
-It signs you in (one browser click, no password, no credit card), detects every AI
-editor you use, writes the MCP config, seeds your brain from git history, and installs
-a git hook so it keeps learning.
-
-### 2. Restart your editor
-
-That's it. From now on your AI arrives pre-briefed — every session.
-
-### 3. Just work
-
-cachly learns in the background. You never have to "remember to save." Every fix,
-every commit, every session feeds the brain automatically.
+Autopilot does everything in a single command: it auto-detects every AI editor you use,
+writes the MCP config, signs you in via browser device-flow (one click, no password, no
+credit card), and bootstraps your brain from git history. Restart your editor and your AI
+arrives pre-briefed — every session, automatically.
 
 > **Already inside Claude / Cursor / Copilot?** Paste this to your AI and it configures everything itself:
 > ```
-> Set up cachly for this project. Run: npx @cachly-dev/mcp-server@latest setup
+> Set up cachly for this project. Run: npx @cachly-dev/mcp-server@latest autopilot
 > It gives my AI persistent memory across sessions. Follow the browser login
 > (one click, no credit card), then restart the editor.
 > ```
@@ -234,8 +224,10 @@ causal_trace(problem="auth breaks after restart")
 ## CLI Commands
 
 ```bash
+npx @cachly-dev/mcp-server@latest autopilot # One command — signs in, configures every editor, bootstraps from git
 npx @cachly-dev/mcp-server@latest demo      # Preview your Brain (no account needed)
-npx @cachly-dev/mcp-server@latest setup     # Wire up all your AI editors (1–5 minutes)
+npx @cachly-dev/mcp-server@latest bench     # Recall quality vs flat-file memory (no auth required)
+npx @cachly-dev/mcp-server@latest setup     # Interactive variant — pick editors yourself
 npx @cachly-dev/mcp-server@latest health    # Check token, API, editors, git hook
 npx @cachly-dev/mcp-server@latest digest    # Weekly Brain summary — shareable
 npx @cachly-dev/mcp-server@latest share     # Generate a shareable stats card + tweet
@@ -252,7 +244,7 @@ npx @cachly-dev/mcp-server@latest learn-git # Auto-learn lessons from recent git
 
 ---
 
-## MCP Tools (113 total)
+## MCP Tools (114 total)
 
 ### 🧠 Session & Memory (most used)
 
@@ -344,7 +336,7 @@ sessions + WIP registry — typically one tool call.
 
 ## Editor support matrix
 
-`npx @cachly-dev/mcp-server@latest setup` auto-detects and configures all of the
+`npx @cachly-dev/mcp-server@latest autopilot` auto-detects and configures all of the
 following. Manual snippets are in the **Manual Setup** section below.
 
 | Editor / Client | Auto-setup | Config file written | Global config | Notes |
@@ -370,7 +362,7 @@ following. Manual snippets are in the **Manual Setup** section below.
 
 > **Tip — fastest per-project setup from inside Claude Code:**
 > ```
-> Set up cachly for this project: npx @cachly-dev/mcp-server@latest setup
+> Set up cachly for this project: npx @cachly-dev/mcp-server@latest autopilot
 > ```
 > Claude runs it and restarts automatically.
 
@@ -455,7 +447,7 @@ private backend instead of `api.cachly.dev`:
 
 ```bash
 # One-shot: wire up the wizard against your self-hosted backend
-npx @cachly-dev/mcp-server@latest setup --api-url https://cachly.mycorp.internal
+npx @cachly-dev/mcp-server@latest autopilot --api-url https://cachly.mycorp.internal
 
 # Or non-interactively
 npx @cachly-dev/mcp-server@latest init \
@@ -513,5 +505,5 @@ keep talking to your backend on every editor launch. All data stays in your infr
 > brain that remembers, learns, and gets sharper with every commit.
 >
 > ```bash
-> npx @cachly-dev/mcp-server@latest setup
+> npx @cachly-dev/mcp-server@latest autopilot
 > ```

package/dist/src/bench/cachly-bench.d.ts

@@ -0,0 +1,46 @@
+/**
+ * Cachly-Bench — the moat proof.
+ *
+ * Measures recall quality on a fixed, labeled corpus and compares two rankers
+ * over the *same* BM25 candidate set, so the comparison isolates ranking quality:
+ *
+ *   baseline  — raw BM25+ keyword ranking (what a text-only / flat-file memory does)
+ *   cachly    — BM25+ followed by quality-aware reranking (rerank.ts)
+ *
+ * It reports standard IR metrics (Precision@k, Recall@k, MRR, nDCG@k) and the
+ * relative lift. The corpus (fixtures.ts) is fair: gold answers are the lessons
+ * that actually solve each problem, while text-similar failed attempts act as
+ * distractors — exactly where ranking by quality should pull ahead.
+ *
+ * Run:  npm run bench        (pretty report)
+ *       npm run bench -- --json   (machine-readable)
+ */
+import { type BenchLesson, type BenchQuery } from './fixtures.js';
+export interface BenchMetrics {
+    precisionAt1: number;
+    precisionAt3: number;
+    recallAt3: number;
+    mrr: number;
+    ndcgAt5: number;
+}
+export interface BenchResult {
+    /** Flat-file memory simulation: naive term-overlap, no IDF, no quality signal. */
+    flatfile: BenchMetrics;
+    baseline: BenchMetrics;
+    cachly: BenchMetrics;
+    /** Relative lift of cachly vs the BM25 baseline. */
+    lift: BenchMetrics;
+    /** Relative lift of cachly vs a flat-file memory (the head-to-head that matters). */
+    liftVsFlatfile: BenchMetrics;
+    queryCount: number;
+    corpusSize: number;
+}
+/**
+ * Run the three-ranker benchmark over any labeled corpus + query set. Pure-ish:
+ * no network, no real Redis. Shared by the built-in fixture bench and the external
+ * labeled-corpus bench (external-corpus.ts) so both prove lift the same way.
+ */
+export declare function runBenchmarkOn(lessons: BenchLesson[], queries: BenchQuery[]): Promise<BenchResult>;
+/** Run the benchmark over the built-in fixture corpus. */
+export declare function runBenchmark(): Promise<BenchResult>;
+export declare function formatReport(r: BenchResult): string;

package/dist/src/bench/cachly-bench.js

@@ -0,0 +1,231 @@
+/**
+ * Cachly-Bench — the moat proof.
+ *
+ * Measures recall quality on a fixed, labeled corpus and compares two rankers
+ * over the *same* BM25 candidate set, so the comparison isolates ranking quality:
+ *
+ *   baseline  — raw BM25+ keyword ranking (what a text-only / flat-file memory does)
+ *   cachly    — BM25+ followed by quality-aware reranking (rerank.ts)
+ *
+ * It reports standard IR metrics (Precision@k, Recall@k, MRR, nDCG@k) and the
+ * relative lift. The corpus (fixtures.ts) is fair: gold answers are the lessons
+ * that actually solve each problem, while text-similar failed attempts act as
+ * distractors — exactly where ranking by quality should pull ahead.
+ *
+ * Run:  npm run bench        (pretty report)
+ *       npm run bench -- --json   (machine-readable)
+ */
+import { EventEmitter } from 'node:events';
+import { keywordSearch } from '../search.js';
+import { rerankByQuality } from '../rerank.js';
+import { BENCH_LESSONS, BENCH_QUERIES } from './fixtures.js';
+const LESSON_PREFIX = 'cachly:lesson:best:';
+// ── Minimal in-memory Redis (only what keywordSearch needs) ────────────────────
+class MiniRedis {
+    store = new Map();
+    set(key, value) { this.store.set(key, value); }
+    async get(key) { return this.store.get(key) ?? null; }
+    scanStream(opts) {
+        const emitter = new EventEmitter();
+        const pattern = '^' + opts.match.replace(/[.+?^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '.*') + '$';
+        const regex = new RegExp(pattern);
+        const matches = [...this.store.keys()].filter(k => regex.test(k));
+        setImmediate(() => { emitter.emit('data', matches); emitter.emit('end'); });
+        return emitter;
+    }
+    pipeline() {
+        const cmds = [];
+        const store = this.store;
+        return {
+            get(key) { cmds.push(key); return this; },
+            async exec() {
+                return cmds.map(k => [null, store.get(k) ?? null]);
+            },
+        };
+    }
+}
+function indexCorpus(lessons) {
+    const redis = new MiniRedis();
+    for (const l of lessons) {
+        redis.set(`${LESSON_PREFIX}${l.topic}`, JSON.stringify(l));
+    }
+    return redis;
+}
+// ── IR metrics ─────────────────────────────────────────────────────────────────
+function precisionAtK(ranked, relevant, k) {
+    const top = ranked.slice(0, k);
+    if (top.length === 0)
+        return 0;
+    const hits = top.filter(t => relevant.has(t)).length;
+    return hits / Math.min(k, top.length);
+}
+function recallAtK(ranked, relevant, k) {
+    if (relevant.size === 0)
+        return 0;
+    const top = ranked.slice(0, k);
+    const hits = top.filter(t => relevant.has(t)).length;
+    return hits / relevant.size;
+}
+function reciprocalRank(ranked, relevant) {
+    for (let i = 0; i < ranked.length; i++) {
+        if (relevant.has(ranked[i]))
+            return 1 / (i + 1);
+    }
+    return 0;
+}
+function ndcgAtK(ranked, relevant, k) {
+    let dcg = 0;
+    for (let i = 0; i < Math.min(k, ranked.length); i++) {
+        if (relevant.has(ranked[i]))
+            dcg += 1 / Math.log2(i + 2);
+    }
+    let idcg = 0;
+    for (let i = 0; i < Math.min(k, relevant.size); i++) {
+        idcg += 1 / Math.log2(i + 2);
+    }
+    return idcg === 0 ? 0 : dcg / idcg;
+}
+/**
+ * Simulates how a flat-file memory (e.g. an LLM reading a `/memories` directory)
+ * actually surfaces knowledge: it has NO ranking engine. Practical retrieval is
+ * naive lexical overlap — count how many distinct query terms appear in the file's
+ * text — with NO inverse-document-frequency, NO length normalization, and crucially
+ * NO quality signal (a proven fix and a failed attempt are just text). Ties break by
+ * recency, mimicking "most recently edited note first". This is a fair, and honestly
+ * weaker, stand-in for "Claude reads its own memory files".
+ */
+function flatFileRank(lessons, query) {
+    const qTerms = new Set(query.toLowerCase().replace(/[^a-z0-9\s]/g, ' ').split(/\s+/).filter(t => t.length >= 3));
+    const scored = lessons.map(l => {
+        const blob = [l.topic, l.what_worked, l.what_failed ?? '', l.context ?? '', (l.tags ?? []).join(' ')]
+            .join(' ').toLowerCase();
+        let overlap = 0;
+        for (const t of qTerms)
+            if (blob.includes(t))
+                overlap++;
+        const recency = l.ts ? new Date(l.ts).getTime() : 0;
+        return { topic: l.topic, overlap, recency };
+    });
+    return scored
+        .filter(s => s.overlap > 0)
+        .sort((a, b) => (b.overlap - a.overlap) || (b.recency - a.recency))
+        .map(s => s.topic);
+}
+const matchTopic = (m) => m.key.replace(LESSON_PREFIX, '');
+function mean(xs) {
+    return xs.length === 0 ? 0 : xs.reduce((a, b) => a + b, 0) / xs.length;
+}
+function aggregate(rows) {
+    return {
+        precisionAt1: mean(rows.map(r => r.precisionAt1)),
+        precisionAt3: mean(rows.map(r => r.precisionAt3)),
+        recallAt3: mean(rows.map(r => r.recallAt3)),
+        mrr: mean(rows.map(r => r.mrr)),
+        ndcgAt5: mean(rows.map(r => r.ndcgAt5)),
+    };
+}
+function scoreRanking(ranked, relevant) {
+    return {
+        precisionAt1: precisionAtK(ranked, relevant, 1),
+        precisionAt3: precisionAtK(ranked, relevant, 3),
+        recallAt3: recallAtK(ranked, relevant, 3),
+        mrr: reciprocalRank(ranked, relevant),
+        ndcgAt5: ndcgAtK(ranked, relevant, 5),
+    };
+}
+/**
+ * Run the three-ranker benchmark over any labeled corpus + query set. Pure-ish:
+ * no network, no real Redis. Shared by the built-in fixture bench and the external
+ * labeled-corpus bench (external-corpus.ts) so both prove lift the same way.
+ */
+export async function runBenchmarkOn(lessons, queries) {
+    const redis = indexCorpus(lessons);
+    const flatfileRows = [];
+    const baselineRows = [];
+    const cachlyRows = [];
+    for (const q of queries) {
+        const relevant = new Set(q.relevant);
+        // BM25 baseline + cachly share the same candidate set — isolates ranking quality.
+        const matches = await keywordSearch(redis, [`${LESSON_PREFIX}*`], q.query, 10);
+        const baselineRanked = matches.map(matchTopic);
+        const cachlyRanked = rerankByQuality(matches).map(matchTopic);
+        // Flat-file memory sees the whole corpus but has no ranking engine.
+        const flatfileRanked = flatFileRank(lessons, q.query);
+        flatfileRows.push(scoreRanking(flatfileRanked, relevant));
+        baselineRows.push(scoreRanking(baselineRanked, relevant));
+        cachlyRows.push(scoreRanking(cachlyRanked, relevant));
+    }
+    const flatfile = aggregate(flatfileRows);
+    const baseline = aggregate(baselineRows);
+    const cachly = aggregate(cachlyRows);
+    const liftOf = (b, c) => (b === 0 ? (c > 0 ? 1 : 0) : (c - b) / b);
+    const liftBetween = (from) => ({
+        precisionAt1: liftOf(from.precisionAt1, cachly.precisionAt1),
+        precisionAt3: liftOf(from.precisionAt3, cachly.precisionAt3),
+        recallAt3: liftOf(from.recallAt3, cachly.recallAt3),
+        mrr: liftOf(from.mrr, cachly.mrr),
+        ndcgAt5: liftOf(from.ndcgAt5, cachly.ndcgAt5),
+    });
+    return {
+        flatfile, baseline, cachly,
+        lift: liftBetween(baseline),
+        liftVsFlatfile: liftBetween(flatfile),
+        queryCount: queries.length,
+        corpusSize: lessons.length,
+    };
+}
+/** Run the benchmark over the built-in fixture corpus. */
+export async function runBenchmark() {
+    return runBenchmarkOn(BENCH_LESSONS, BENCH_QUERIES);
+}
+// ── Reporting ──────────────────────────────────────────────────────────────────
+const pct = (x) => `${(x * 100).toFixed(1)}%`;
+const signedPct = (x) => `${x >= 0 ? '+' : ''}${(x * 100).toFixed(1)}%`;
+export function formatReport(r) {
+    const row = (label, f, b, c, lf) => `  ${label.padEnd(12)} ${pct(f).padStart(8)} ${pct(b).padStart(8)} ${pct(c).padStart(8)}   ${signedPct(lf).padStart(8)}`;
+    return [
+        '',
+        '🧠  Cachly-Bench — recall quality head-to-head',
+        '──────────────────────────────────────────────────────────────────────────',
+        `  corpus: ${r.corpusSize} lessons · queries: ${r.queryCount}`,
+        '  flat-file = naive term-overlap, no quality signal (≈ LLM reading memory files)',
+        '  baseline  = raw BM25+ keyword ranking',
+        '  cachly    = BM25+ followed by quality-aware reranking',
+        '',
+        `  ${'metric'.padEnd(12)} ${'flatfile'.padStart(8)} ${'baseline'.padStart(8)} ${'cachly'.padStart(8)}   ${'vs flat'.padStart(8)}`,
+        row('Precision@1', r.flatfile.precisionAt1, r.baseline.precisionAt1, r.cachly.precisionAt1, r.liftVsFlatfile.precisionAt1),
+        row('Precision@3', r.flatfile.precisionAt3, r.baseline.precisionAt3, r.cachly.precisionAt3, r.liftVsFlatfile.precisionAt3),
+        row('Recall@3', r.flatfile.recallAt3, r.baseline.recallAt3, r.cachly.recallAt3, r.liftVsFlatfile.recallAt3),
+        row('MRR', r.flatfile.mrr, r.baseline.mrr, r.cachly.mrr, r.liftVsFlatfile.mrr),
+        row('nDCG@5', r.flatfile.ndcgAt5, r.baseline.ndcgAt5, r.cachly.ndcgAt5, r.liftVsFlatfile.ndcgAt5),
+        '──────────────────────────────────────────────────────────────────────────',
+        `  vs BM25 baseline : MRR ${signedPct(r.lift.mrr)} · Precision@1 ${signedPct(r.lift.precisionAt1)}`,
+        `  vs flat-file mem : MRR ${signedPct(r.liftVsFlatfile.mrr)} · Precision@1 ${signedPct(r.liftVsFlatfile.precisionAt1)}`,
+        '',
+    ].join('\n');
+}
+// ── CLI entry ───────────────────────────────────────────────────────────────────
+const isMain = (() => {
+    try {
+        return import.meta.url === `file://${process.argv[1]}`
+            || process.argv[1]?.endsWith('cachly-bench.ts')
+            || process.argv[1]?.endsWith('cachly-bench.js');
+    }
+    catch {
+        return false;
+    }
+})();
+if (isMain) {
+    runBenchmark().then(result => {
+        if (process.argv.includes('--json')) {
+            process.stdout.write(JSON.stringify(result, null, 2) + '\n');
+        }
+        else {
+            process.stdout.write(formatReport(result));
+        }
+    }).catch(err => {
+        process.stderr.write(`bench failed: ${err instanceof Error ? err.stack : String(err)}\n`);
+        process.exit(1);
+    });
+}
+//# sourceMappingURL=cachly-bench.js.map

package/dist/src/bench/cachly-bench.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cachly-bench.js","sourceRoot":"","sources":["../../../src/bench/cachly-bench.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAqB,MAAM,cAAc,CAAC;AAChE,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC/C,OAAO,EAAE,aAAa,EAAE,aAAa,EAAqC,MAAM,eAAe,CAAC;AAEhG,MAAM,aAAa,GAAG,qBAAqB,CAAC;AAE5C,kFAAkF;AAClF,MAAM,SAAS;IACL,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAC;IAE1C,GAAG,CAAC,GAAW,EAAE,KAAa,IAAU,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;IACrE,KAAK,CAAC,GAAG,CAAC,GAAW,IAA4B,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC;IAEtF,UAAU,CAAC,IAAuC;QAChD,MAAM,OAAO,GAAG,IAAI,YAAY,EAAE,CAAC;QACnC,MAAM,OAAO,GAAG,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,oBAAoB,EAAE,MAAM,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,GAAG,CAAC;QAClG,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC;QAClC,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QAClE,YAAY,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5E,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,QAAQ;QACN,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACzB,OAAO;YACL,GAAG,CAAC,GAAW,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC;YACjD,KAAK,CAAC,IAAI;gBACR,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,CAA0B,CAAC,CAAC;YAC9E,CAAC;SACF,CAAC;IACJ,CAAC;CACF;AAED,SAAS,WAAW,CAAC,OAAsB;IACzC,MAAM,KAAK,GAAG,IAAI,SAAS,EAAE,CAAC;IAC9B,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,KAAK,CAAC,GAAG,CAAC,GAAG,aAAa,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7D,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,kFAAkF;AAClF,SAAS,YAAY,CAAC,MAAgB,EAAE,QAAqB,EAAE,CAAS;IACtE,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC/B,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAC/B,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IACrD,OAAO,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;AACxC,CAAC;AAED,SAAS,SAAS,CAAC,MAAgB,EAAE,QAAqB,EAAE,CAAS;IACnE,IAAI,QAAQ,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAClC,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC/B,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IACrD,OAAO,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC;AAC9B,CAAC;AAED,SAAS,cAAc,CAAC,MAAgB,EAAE,QAAqB;IAC7D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,IAAI,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YAAE,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAClD,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC;AAED,SAAS,OAAO,CAAC,MAAgB,EAAE,QAAqB,EAAE,CAAS;IACjE,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACpD,IAAI,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YAAE,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC3D,CAAC;IACD,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACpD,IAAI,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC/B,CAAC;IACD,OAAO,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,IAAI,CAAC;AACrC,CAAC;AAuBD;;;;;;;;GAQG;AACH,SAAS,YAAY,CAAC,OAAsB,EAAE,KAAa;IACzD,MAAM,MAAM,GAAG,IAAI,GAAG,CACpB,KAAK,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,CACzF,CAAC;IACF,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE;QAC7B,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,WAAW,IAAI,EAAE,EAAE,CAAC,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;aAClG,IAAI,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;QAC3B,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,MAAM,CAAC,IAAI,MAAM;YAAE,IAAI,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;gBAAE,OAAO,EAAE,CAAC;QACxD,MAAM,OAAO,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACpD,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;IAC9C,CAAC,CAAC,CAAC;IACH,OAAO,MAAM;SACV,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC;SAC1B,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC;SAClE,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;AACvB,CAAC;AAED,MAAM,UAAU,GAAG,CAAC,CAAe,EAAU,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC;AAEjF,SAAS,IAAI,CAAC,EAAY;IACxB,OAAO,EAAE,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC;AACzE,CAAC;AAED,SAAS,SAAS,CAAC,IAAoB;IACrC,OAAO;QACL,YAAY,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;QACjD,YAAY,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;QACjD,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAC3C,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC/B,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;KACxC,CAAC;AACJ,CAAC;AAED,SAAS,YAAY,CAAC,MAAgB,EAAE,QAAqB;IAC3D,OAAO;QACL,YAAY,EAAE,YAAY,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC/C,YAAY,EAAE,YAAY,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC/C,SAAS,EAAE,SAAS,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;QACzC,GAAG,EAAE,cAAc,CAAC,MAAM,EAAE,QAAQ,CAAC;QACrC,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;KACtC,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,OAAsB,EAAE,OAAqB;IAChF,MAAM,KAAK,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;IACnC,MAAM,YAAY,GAAmB,EAAE,CAAC;IACxC,MAAM,YAAY,GAAmB,EAAE,CAAC;IACxC,MAAM,UAAU,GAAmB,EAAE,CAAC;IAEtC,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QAErC,kFAAkF;QAClF,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,KAAc,EAAE,CAAC,GAAG,aAAa,GAAG,CAAC,EAAE,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACxF,MAAM,cAAc,GAAG,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAC/C,MAAM,YAAY,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAE9D,oEAAoE;QACpE,MAAM,cAAc,GAAG,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;QAEtD,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,cAAc,EAAE,QAAQ,CAAC,CAAC,CAAC;QAC1D,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,cAAc,EAAE,QAAQ,CAAC,CAAC,CAAC;QAC1D,UAAU,CAAC,IAAI,CAAC,YAAY,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC,CAAC;IACxD,CAAC;IAED,MAAM,QAAQ,GAAG,SAAS,CAAC,YAAY,CAAC,CAAC;IACzC,MAAM,QAAQ,GAAG,SAAS,CAAC,YAAY,CAAC,CAAC;IACzC,MAAM,MAAM,GAAG,SAAS,CAAC,UAAU,CAAC,CAAC;IACrC,MAAM,MAAM,GAAG,CAAC,CAAS,EAAE,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACnF,MAAM,WAAW,GAAG,CAAC,IAAkB,EAAgB,EAAE,CAAC,CAAC;QACzD,YAAY,EAAE,MAAM,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,YAAY,CAAC;QAC5D,YAAY,EAAE,MAAM,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,YAAY,CAAC;QAC5D,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC;QACnD,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC;QACjC,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC;KAC9C,CAAC,CAAC;IAEH,OAAO;QACL,QAAQ,EAAE,QAAQ,EAAE,MAAM;QAC1B,IAAI,EAAE,WAAW,CAAC,QAAQ,CAAC;QAC3B,cAAc,EAAE,WAAW,CAAC,QAAQ,CAAC;QACrC,UAAU,EAAE,OAAO,CAAC,MAAM;QAC1B,UAAU,EAAE,OAAO,CAAC,MAAM;KAC3B,CAAC;AACJ,CAAC;AAED,0DAA0D;AAC1D,MAAM,CAAC,KAAK,UAAU,YAAY;IAChC,OAAO,cAAc,CAAC,aAAa,EAAE,aAAa,CAAC,CAAC;AACtD,CAAC;AAED,kFAAkF;AAClF,MAAM,GAAG,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;AACtD,MAAM,SAAS,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;AAEhF,MAAM,UAAU,YAAY,CAAC,CAAc;IACzC,MAAM,GAAG,GAAG,CAAC,KAAa,EAAE,CAAS,EAAE,CAAS,EAAE,CAAS,EAAE,EAAU,EAAE,EAAE,CACzE,KAAK,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,SAAS,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;IAC3H,OAAO;QACL,EAAE;QACF,gDAAgD;QAChD,4EAA4E;QAC5E,aAAa,CAAC,CAAC,UAAU,uBAAuB,CAAC,CAAC,UAAU,EAAE;QAC9D,kFAAkF;QAClF,yCAAyC;QACzC,yDAAyD;QACzD,EAAE;QACF,KAAK,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE;QACjI,GAAG,CAAC,aAAa,EAAE,CAAC,CAAC,QAAQ,CAAC,YAAY,EAAE,CAAC,CAAC,QAAQ,CAAC,YAAY,EAAE,CAAC,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC,CAAC,cAAc,CAAC,YAAY,CAAC;QAC1H,GAAG,CAAC,aAAa,EAAE,CAAC,CAAC,QAAQ,CAAC,YAAY,EAAE,CAAC,CAAC,QAAQ,CAAC,YAAY,EAAE,CAAC,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC,CAAC,cAAc,CAAC,YAAY,CAAC;QAC1H,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,cAAc,CAAC,SAAS,CAAC;QAC3G,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,cAAc,CAAC,GAAG,CAAC;QAC9E,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,cAAc,CAAC,OAAO,CAAC;QACjG,4EAA4E;QAC5E,4BAA4B,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,kBAAkB,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE;QACnG,4BAA4B,SAAS,CAAC,CAAC,CAAC,cAAc,CAAC,GAAG,CAAC,kBAAkB,SAAS,CAAC,CAAC,CAAC,cAAc,CAAC,YAAY,CAAC,EAAE;QACvH,EAAE;KACH,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED,mFAAmF;AACnF,MAAM,MAAM,GAAG,CAAC,GAAG,EAAE;IACnB,IAAI,CAAC;QACH,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,UAAU,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE;eACjD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,iBAAiB,CAAC;eAC5C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,iBAAiB,CAAC,CAAC;IACpD,CAAC;IAAC,MAAM,CAAC;QAAC,OAAO,KAAK,CAAC;IAAC,CAAC;AAC3B,CAAC,CAAC,EAAE,CAAC;AAEL,IAAI,MAAM,EAAE,CAAC;IACX,YAAY,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE;QAC3B,IAAI,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;YACpC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;QAC/D,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE;QACb,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,iBAAiB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC1F,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/src/bench/external-corpus.d.ts

@@ -0,0 +1,44 @@
+/**
+ * External labeled-corpus benchmark.
+ *
+ * The built-in fixture bench (fixtures.ts) proves lift on a corpus *we* wrote.
+ * The honest next step is to prove the same lift on a corpus authored by a third
+ * party. This module loads a labeled corpus from a portable JSON file and runs the
+ * exact same three-ranker comparison (flat-file vs BM25 baseline vs cachly).
+ *
+ * Portable format (BEIR-ish, but with the optional quality metadata our reranker
+ * uses — a generic IR corpus without it will simply show cachly ≈ BM25, which is
+ * itself an honest, documented result):
+ *
+ *   {
+ *     "name": "my-team-incidents-2025",
+ *     "lessons": [
+ *       { "topic": "deploy:oom", "outcome": "success", "what_worked": "...",
+ *         "confidence": 0.9, "recall_count": 4, "severity": "critical",
+ *         "review_level": "senior", "reviewed_by": "alice", "endorsements": 2,
+ *         "ts": "2025-01-02T00:00:00.000Z" },
+ *       ...
+ *     ],
+ *     "queries": [
+ *       { "query": "pod keeps restarting out of memory", "relevant": ["deploy:oom"] },
+ *       ...
+ *     ]
+ *   }
+ *
+ * Run:  npm run bench:external                       (uses the bundled sample)
+ *       npm run bench:external -- ./path/corpus.json (your own labeled corpus)
+ *       npm run bench:external -- ./corpus.json --json
+ */
+import { type BenchResult } from './cachly-bench.js';
+import type { BenchLesson, BenchQuery } from './fixtures.js';
+export interface ExternalCorpus {
+    name?: string;
+    lessons: BenchLesson[];
+    queries: BenchQuery[];
+}
+/** Parse + validate a portable corpus object. Throws a clear error on bad shape. */
+export declare function parseExternalCorpus(raw: unknown): ExternalCorpus;
+/** Load + validate a corpus from a JSON file path. */
+export declare function loadExternalCorpus(path: string): Promise<ExternalCorpus>;
+/** Run the three-ranker benchmark on an external corpus. */
+export declare function runExternalBenchmark(corpus: ExternalCorpus): Promise<BenchResult>;

package/dist/src/bench/external-corpus.js

@@ -0,0 +1,135 @@
+/**
+ * External labeled-corpus benchmark.
+ *
+ * The built-in fixture bench (fixtures.ts) proves lift on a corpus *we* wrote.
+ * The honest next step is to prove the same lift on a corpus authored by a third
+ * party. This module loads a labeled corpus from a portable JSON file and runs the
+ * exact same three-ranker comparison (flat-file vs BM25 baseline vs cachly).
+ *
+ * Portable format (BEIR-ish, but with the optional quality metadata our reranker
+ * uses — a generic IR corpus without it will simply show cachly ≈ BM25, which is
+ * itself an honest, documented result):
+ *
+ *   {
+ *     "name": "my-team-incidents-2025",
+ *     "lessons": [
+ *       { "topic": "deploy:oom", "outcome": "success", "what_worked": "...",
+ *         "confidence": 0.9, "recall_count": 4, "severity": "critical",
+ *         "review_level": "senior", "reviewed_by": "alice", "endorsements": 2,
+ *         "ts": "2025-01-02T00:00:00.000Z" },
+ *       ...
+ *     ],
+ *     "queries": [
+ *       { "query": "pod keeps restarting out of memory", "relevant": ["deploy:oom"] },
+ *       ...
+ *     ]
+ *   }
+ *
+ * Run:  npm run bench:external                       (uses the bundled sample)
+ *       npm run bench:external -- ./path/corpus.json (your own labeled corpus)
+ *       npm run bench:external -- ./corpus.json --json
+ */
+import { readFile } from 'node:fs/promises';
+import { fileURLToPath } from 'node:url';
+import { dirname, resolve } from 'node:path';
+import { runBenchmarkOn, formatReport } from './cachly-bench.js';
+/** Parse + validate a portable corpus object. Throws a clear error on bad shape. */
+export function parseExternalCorpus(raw) {
+    if (!raw || typeof raw !== 'object') {
+        throw new Error('Corpus must be a JSON object with `lessons` and `queries`.');
+    }
+    const obj = raw;
+    const lessons = obj.lessons;
+    const queries = obj.queries;
+    if (!Array.isArray(lessons) || lessons.length === 0) {
+        throw new Error('Corpus.lessons must be a non-empty array.');
+    }
+    if (!Array.isArray(queries) || queries.length === 0) {
+        throw new Error('Corpus.queries must be a non-empty array.');
+    }
+    const topics = new Set();
+    for (const [i, l] of lessons.entries()) {
+        const lesson = l;
+        if (typeof lesson.topic !== 'string' || !lesson.topic.trim()) {
+            throw new Error(`lessons[${i}].topic must be a non-empty string.`);
+        }
+        if (typeof lesson.what_worked !== 'string') {
+            throw new Error(`lessons[${i}].what_worked must be a string.`);
+        }
+        if (topics.has(lesson.topic)) {
+            throw new Error(`Duplicate lesson topic "${lesson.topic}" — topics must be unique.`);
+        }
+        topics.add(lesson.topic);
+    }
+    for (const [i, q] of queries.entries()) {
+        const query = q;
+        if (typeof query.query !== 'string' || !query.query.trim()) {
+            throw new Error(`queries[${i}].query must be a non-empty string.`);
+        }
+        if (!Array.isArray(query.relevant) || query.relevant.length === 0) {
+            throw new Error(`queries[${i}].relevant must be a non-empty array of topics.`);
+        }
+        for (const rel of query.relevant) {
+            if (typeof rel !== 'string' || !topics.has(rel)) {
+                throw new Error(`queries[${i}] references unknown relevant topic "${String(rel)}".`);
+            }
+        }
+    }
+    return {
+        name: typeof obj.name === 'string' ? obj.name : undefined,
+        lessons: lessons,
+        queries: queries,
+    };
+}
+/** Load + validate a corpus from a JSON file path. */
+export async function loadExternalCorpus(path) {
+    const text = await readFile(path, 'utf-8');
+    let raw;
+    try {
+        raw = JSON.parse(text);
+    }
+    catch (e) {
+        throw new Error(`Corpus file is not valid JSON: ${e.message}`);
+    }
+    return parseExternalCorpus(raw);
+}
+/** Run the three-ranker benchmark on an external corpus. */
+export async function runExternalBenchmark(corpus) {
+    return runBenchmarkOn(corpus.lessons, corpus.queries);
+}
+// ── CLI entry ───────────────────────────────────────────────────────────────────
+const isMain = (() => {
+    try {
+        return import.meta.url === `file://${process.argv[1]}`
+            || process.argv[1]?.endsWith('external-corpus.ts')
+            || process.argv[1]?.endsWith('external-corpus.js');
+    }
+    catch {
+        return false;
+    }
+})();
+if (isMain) {
+    const args = process.argv.slice(2).filter(a => a !== '--json');
+    const asJson = process.argv.includes('--json');
+    const here = dirname(fileURLToPath(import.meta.url));
+    // Default to the bundled sample so `npm run bench:external` works out of the box.
+    const corpusPath = args[0]
+        ? resolve(process.cwd(), args[0])
+        : resolve(here, 'external', 'sample-corpus.json');
+    loadExternalCorpus(corpusPath)
+        .then(async (corpus) => {
+        const result = await runExternalBenchmark(corpus);
+        if (asJson) {
+            console.log(JSON.stringify({ corpus: corpus.name ?? corpusPath, ...result }, null, 2));
+        }
+        else {
+            console.log(`\n📦  External corpus: ${corpus.name ?? corpusPath}`);
+            console.log(formatReport(result));
+        }
+    })
+        .catch((e) => {
+        console.error(`\n❌ External bench failed: ${e.message}\n`);
+        process.exit(1);
+    });
+}
+//# sourceMappingURL=external-corpus.js.map

package/dist/src/bench/external-corpus.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"external-corpus.js","sourceRoot":"","sources":["../../../src/bench/external-corpus.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,cAAc,EAAE,YAAY,EAAoB,MAAM,mBAAmB,CAAC;AASnF,oFAAoF;AACpF,MAAM,UAAU,mBAAmB,CAAC,GAAY;IAC9C,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QACpC,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAC;IAChF,CAAC;IACD,MAAM,GAAG,GAAG,GAA8B,CAAC;IAC3C,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC;IAC5B,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC;IAC5B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACpD,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;IAC/D,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACpD,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;IAC/D,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;IACjC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;QACvC,MAAM,MAAM,GAAG,CAA4B,CAAC;QAC5C,IAAI,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;YAC7D,MAAM,IAAI,KAAK,CAAC,WAAW,CAAC,qCAAqC,CAAC,CAAC;QACrE,CAAC;QACD,IAAI,OAAO,MAAM,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;YAC3C,MAAM,IAAI,KAAK,CAAC,WAAW,CAAC,iCAAiC,CAAC,CAAC;QACjE,CAAC;QACD,IAAI,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CAAC,2BAA2B,MAAM,CAAC,KAAK,4BAA4B,CAAC,CAAC;QACvF,CAAC;QACD,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC3B,CAAC;IAED,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;QACvC,MAAM,KAAK,GAAG,CAA4B,CAAC;QAC3C,IAAI,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;YAC3D,MAAM,IAAI,KAAK,CAAC,WAAW,CAAC,qCAAqC,CAAC,CAAC;QACrE,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAClE,MAAM,IAAI,KAAK,CAAC,WAAW,CAAC,iDAAiD,CAAC,CAAC;QACjF,CAAC;QACD,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,QAAqB,EAAE,CAAC;YAC9C,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;gBAChD,MAAM,IAAI,KAAK,CAAC,WAAW,CAAC,wCAAwC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACvF,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,IAAI,EAAE,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;QACzD,OAAO,EAAE,OAAwB;QACjC,OAAO,EAAE,OAAuB;KACjC,CAAC;AACJ,CAAC;AAED,sDAAsD;AACtD,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,IAAY;IACnD,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC3C,IAAI,GAAY,CAAC;IACjB,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACzB,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,kCAAmC,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;IAC5E,CAAC;IACD,OAAO,mBAAmB,CAAC,GAAG,CAAC,CAAC;AAClC,CAAC;AAED,4DAA4D;AAC5D,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,MAAsB;IAC/D,OAAO,cAAc,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;AACxD,CAAC;AAED,mFAAmF;AACnF,MAAM,MAAM,GAAG,CAAC,GAAG,EAAE;IACnB,IAAI,CAAC;QACH,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,UAAU,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE;eACjD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,oBAAoB,CAAC;eAC/C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,oBAAoB,CAAC,CAAC;IACvD,CAAC;IAAC,MAAM,CAAC;QAAC,OAAO,KAAK,CAAC;IAAC,CAAC;AAC3B,CAAC,CAAC,EAAE,CAAC;AAEL,IAAI,MAAM,EAAE,CAAC;IACX,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,QAAQ,CAAC,CAAC;IAC/D,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAC/C,MAAM,IAAI,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IACrD,kFAAkF;IAClF,MAAM,UAAU,GAAG,IAAI,CAAC,CAAC,CAAC;QACxB,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;QACjC,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,UAAU,EAAE,oBAAoB,CAAC,CAAC;IAEpD,kBAAkB,CAAC,UAAU,CAAC;SAC3B,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QACrB,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAAC,MAAM,CAAC,CAAC;QAClD,IAAI,MAAM,EAAE,CAAC;YACX,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,IAAI,UAAU,EAAE,GAAG,MAAM,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QACzF,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,0BAA0B,MAAM,CAAC,IAAI,IAAI,UAAU,EAAE,CAAC,CAAC;YACnE,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC;QACpC,CAAC;IACH,CAAC,CAAC;SACD,KAAK,CAAC,CAAC,CAAQ,EAAE,EAAE;QAClB,OAAO,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC;QAC3D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;AACP,CAAC"}

package/dist/src/bench/fixtures.d.ts

@@ -0,0 +1,37 @@
+/**
+ * Cachly-Bench fixture corpus.
+ *
+ * A small but realistic corpus of engineering "lessons" plus a set of labeled
+ * natural-language queries. Each query declares which lesson topics are *truly*
+ * relevant — the answer an agent actually wants.
+ *
+ * The corpus is deliberately adversarial in a fair way: for several queries there
+ * is a text-similar but LOW-QUALITY distractor (a failed attempt, an unverified
+ * note) competing with the proven success lesson. This is exactly the situation
+ * where raw text ranking (and a flat-file memory) goes wrong, and where Cachly's
+ * quality-aware reranking should win. The labels reward the lesson that solves the
+ * problem, not the one that merely shares vocabulary.
+ */
+export interface BenchLesson {
+    /** Stored under cachly:lesson:best:{topic} */
+    topic: string;
+    outcome: 'success' | 'partial' | 'failure';
+    what_worked: string;
+    what_failed?: string;
+    context?: string;
+    severity?: 'critical' | 'major' | 'minor';
+    confidence?: number;
+    recall_count?: number;
+    reviewed_by?: string;
+    review_level?: 'senior' | 'peer';
+    endorsements?: number;
+    tags?: string[];
+    ts?: string;
+}
+export interface BenchQuery {
+    query: string;
+    /** Topics that genuinely answer the query (the gold set). */
+    relevant: string[];
+}
+export declare const BENCH_LESSONS: BenchLesson[];
+export declare const BENCH_QUERIES: BenchQuery[];