@cachly-dev/mcp-server 0.10.129 → 0.10.131

package/dist/src/bench/cachly-bench.js

@@ -1,249 +1,4 @@
-/**
- * ⚠ DIESER MESSSTAND MISST FALSCH HERUM. Er heisst seit dem 20.08.2026
- * `npm run bench:spielzeug` und nicht mehr `npm run bench`.
- *
- * Belegt an diesem Tag: mit 17 Lektionen und 13 Fragen sortiert er die beiden
- * Rangfolge-Formeln GENAU VERKEHRT HERUM ein — 92,3 gegen 69,2 Prozent fuer die
- * Fassung, die auf 498 echten Lektionen 15 statt 30 Prozent erreichte.
- *
- * Er bleibt hier, weil seine gegnerischen Beispielpaare (zwei fast gleiche
- * CSRF-Lektionen, von denen nur eine stimmt) etwas pruefen, was im grossen
- * Korpus untergeht. Als Beleg fuer Rangfolge-Qualitaet taugt er NICHT.
- *
- * Der Messstand, der zaehlt: src/bench/echter-korpus.ts — 499 echte Lektionen,
- * 100 von Hand geschriebene Fragen, mit Untergrenzen.
- */
-/**
- * Cachly-Bench — the moat proof.
- *
- * Measures recall quality on a fixed, labeled corpus and compares two rankers
- * over the *same* BM25 candidate set, so the comparison isolates ranking quality:
- *
- *   baseline  — raw BM25+ keyword ranking (what a text-only / flat-file memory does)
- *   cachly    — BM25+ followed by quality-aware reranking (rerank.ts)
- *
- * It reports standard IR metrics (Precision@k, Recall@k, MRR, nDCG@k) and the
- * relative lift. The corpus (fixtures.ts) is fair: gold answers are the lessons
- * that actually solve each problem, while text-similar failed attempts act as
- * distractors — exactly where ranking by quality should pull ahead.
- *
- * Run:  npm run bench        (pretty report)
- *       npm run bench -- --json   (machine-readable)
- */
-import { EventEmitter } from 'node:events';
-import { keywordSearch } from '../search.js';
-import { rerankByQuality } from '../rerank.js';
-import { BENCH_LESSONS, BENCH_QUERIES } from './fixtures.js';
-const LESSON_PREFIX = 'cachly:lesson:best:';
-// ── Minimal in-memory Redis (only what keywordSearch needs) ──────────────────
-class MiniRedis {
-    store = new Map();
-    set(key, value) { this.store.set(key, value); }
-    async get(key) { return this.store.get(key) ?? null; }
-    scanStream(opts) {
-        const emitter = new EventEmitter();
-        const pattern = '^' + opts.match.replace(/[.+?^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '.*') + '$';
-        const regex = new RegExp(pattern);
-        const matches = [...this.store.keys()].filter(k => regex.test(k));
-        setImmediate(() => { emitter.emit('data', matches); emitter.emit('end'); });
-        return emitter;
-    }
-    pipeline() {
-        const cmds = [];
-        const store = this.store;
-        return {
-            get(key) { cmds.push(key); return this; },
-            async exec() {
-                return cmds.map(k => [null, store.get(k) ?? null]);
-            },
-        };
-    }
-}
-function indexCorpus(lessons) {
-    const redis = new MiniRedis();
-    for (const l of lessons) {
-        redis.set(`${LESSON_PREFIX}${l.topic}`, JSON.stringify(l));
-    }
-    return redis;
-}
-// ── IR metrics ──────────────────────────────────────────────────────────────────
-function precisionAtK(ranked, relevant, k) {
-    const top = ranked.slice(0, k);
-    if (top.length === 0)
-        return 0;
-    const hits = top.filter(t => relevant.has(t)).length;
-    return hits / Math.min(k, top.length);
-}
-function recallAtK(ranked, relevant, k) {
-    if (relevant.size === 0)
-        return 0;
-    const top = ranked.slice(0, k);
-    const hits = top.filter(t => relevant.has(t)).length;
-    return hits / relevant.size;
-}
-function reciprocalRank(ranked, relevant) {
-    for (let i = 0; i < ranked.length; i++) {
-        if (relevant.has(ranked[i]))
-            return 1 / (i + 1);
-    }
-    return 0;
-}
-function ndcgAtK(ranked, relevant, k) {
-    let dcg = 0;
-    for (let i = 0; i < Math.min(k, ranked.length); i++) {
-        if (relevant.has(ranked[i]))
-            dcg += 1 / Math.log2(i + 2);
-    }
-    let idcg = 0;
-    for (let i = 0; i < Math.min(k, relevant.size); i++) {
-        idcg += 1 / Math.log2(i + 2);
-    }
-    return idcg === 0 ? 0 : dcg / idcg;
-}
-/**
- * Simulates how a flat-file memory (e.g. an LLM reading a `/memories` directory)
- * actually surfaces knowledge: it has NO ranking engine. Practical retrieval is
- * naive lexical overlap — count how many distinct query terms appear in the file's
- * text — with NO inverse-document-frequency, NO length normalization, and crucially
- * NO quality signal (a proven fix and a failed attempt are just text). Ties break by
- * recency, mimicking "most recently edited note first". This is a fair, and honestly
- * weaker, stand-in for "Claude reads its own memory files".
- */
-function flatFileRank(lessons, query) {
-    const qTerms = new Set(query.toLowerCase().replace(/[^a-z0-9\s]/g, ' ').split(/\s+/).filter(t => t.length >= 3));
-    const scored = lessons.map(l => {
-        const blob = [l.topic, l.what_worked, l.what_failed ?? '', l.context ?? '', (l.tags ?? []).join(' ')]
-            .join(' ').toLowerCase();
-        let overlap = 0;
-        for (const t of qTerms)
-            if (blob.includes(t))
-                overlap++;
-        const recency = l.ts ? new Date(l.ts).getTime() : 0;
-        return { topic: l.topic, overlap, recency };
-    });
-    return scored
-        .filter(s => s.overlap > 0)
-        .sort((a, b) => (b.overlap - a.overlap) || (b.recency - a.recency))
-        .map(s => s.topic);
-}
-const matchTopic = (m) => m.key.replace(LESSON_PREFIX, '');
-function mean(xs) {
-    return xs.length === 0 ? 0 : xs.reduce((a, b) => a + b, 0) / xs.length;
-}
-function aggregate(rows) {
-    return {
-        precisionAt1: mean(rows.map(r => r.precisionAt1)),
-        precisionAt3: mean(rows.map(r => r.precisionAt3)),
-        recallAt3: mean(rows.map(r => r.recallAt3)),
-        mrr: mean(rows.map(r => r.mrr)),
-        ndcgAt5: mean(rows.map(r => r.ndcgAt5)),
-    };
-}
-function scoreRanking(ranked, relevant) {
-    return {
-        precisionAt1: precisionAtK(ranked, relevant, 1),
-        precisionAt3: precisionAtK(ranked, relevant, 3),
-        recallAt3: recallAtK(ranked, relevant, 3),
-        mrr: reciprocalRank(ranked, relevant),
-        ndcgAt5: ndcgAtK(ranked, relevant, 5),
-    };
-}
-/**
- * Run the three-ranker benchmark over any labeled corpus + query set. Pure-ish:
- * no network, no real Redis. Shared by the built-in fixture bench and the external
- * labeled-corpus bench (external-corpus.ts) so both prove lift the same way.
- */
-export async function runBenchmarkOn(lessons, queries) {
-    const redis = indexCorpus(lessons);
-    const flatfileRows = [];
-    const baselineRows = [];
-    const cachlyRows = [];
-    for (const q of queries) {
-        const relevant = new Set(q.relevant);
-        // Baseline: BM25 top-10 only (no reranking), mirrors what a plain keyword search gives.
-        const baselineMatches = await keywordSearch(redis, [`${LESSON_PREFIX}*`], q.query, 10);
-        const baselineRanked = baselineMatches.map(matchTopic);
-        // Cachly: wider candidate pool (top-25) → quality reranker selects the best 3–5.
-        // Wider pool lets the reranker rescue relevant items that BM25 pushed to ranks 11–25.
-        const cachlyMatches = await keywordSearch(redis, [`${LESSON_PREFIX}*`], q.query, 25);
-        const cachlyRanked = rerankByQuality(cachlyMatches).map(matchTopic);
-        // Flat-file memory sees the whole corpus but has no ranking engine.
-        const flatfileRanked = flatFileRank(lessons, q.query);
-        flatfileRows.push(scoreRanking(flatfileRanked, relevant));
-        baselineRows.push(scoreRanking(baselineRanked, relevant));
-        cachlyRows.push(scoreRanking(cachlyRanked, relevant));
-    }
-    const flatfile = aggregate(flatfileRows);
-    const baseline = aggregate(baselineRows);
-    const cachly = aggregate(cachlyRows);
-    const liftOf = (b, c) => (b === 0 ? (c > 0 ? 1 : 0) : (c - b) / b);
-    const liftBetween = (from) => ({
-        precisionAt1: liftOf(from.precisionAt1, cachly.precisionAt1),
-        precisionAt3: liftOf(from.precisionAt3, cachly.precisionAt3),
-        recallAt3: liftOf(from.recallAt3, cachly.recallAt3),
-        mrr: liftOf(from.mrr, cachly.mrr),
-        ndcgAt5: liftOf(from.ndcgAt5, cachly.ndcgAt5),
-    });
-    return {
-        flatfile, baseline, cachly,
-        lift: liftBetween(baseline),
-        liftVsFlatfile: liftBetween(flatfile),
-        queryCount: queries.length,
-        corpusSize: lessons.length,
-    };
-}
-/** Run the benchmark over the built-in fixture corpus. */
-export async function runBenchmark() {
-    return runBenchmarkOn(BENCH_LESSONS, BENCH_QUERIES);
-}
-// ── Reporting ───────────────────────────────────────────────────────────────────────────
-const pct = (x) => `${(x * 100).toFixed(1)}%`;
-const signedPct = (x) => `${x >= 0 ? '+' : ''}${(x * 100).toFixed(1)}%`;
-export function formatReport(r) {
-    const row = (label, f, b, c, lf) => `  ${label.padEnd(12)} ${pct(f).padStart(8)} ${pct(b).padStart(8)} ${pct(c).padStart(8)}   ${signedPct(lf).padStart(8)}`;
-    return [
-        '',
-        '🧠  Cachly-Bench — recall quality head-to-head',
-        '──────────────────────────────────────────────────────────────────────────',
-        `  corpus: ${r.corpusSize} lessons · queries: ${r.queryCount}`,
-        '  flat-file = naive term-overlap, no quality signal (≈ LLM reading memory files)',
-        '  baseline  = raw BM25+ keyword ranking',
-        '  cachly    = BM25+ followed by quality-aware reranking',
-        '',
-        `  ${'metric'.padEnd(12)} ${'flatfile'.padStart(8)} ${'baseline'.padStart(8)} ${'cachly'.padStart(8)}   ${'vs flat'.padStart(8)}`,
-        row('Precision@1', r.flatfile.precisionAt1, r.baseline.precisionAt1, r.cachly.precisionAt1, r.liftVsFlatfile.precisionAt1),
-        row('Precision@3', r.flatfile.precisionAt3, r.baseline.precisionAt3, r.cachly.precisionAt3, r.liftVsFlatfile.precisionAt3),
-        row('Recall@3', r.flatfile.recallAt3, r.baseline.recallAt3, r.cachly.recallAt3, r.liftVsFlatfile.recallAt3),
-        row('MRR', r.flatfile.mrr, r.baseline.mrr, r.cachly.mrr, r.liftVsFlatfile.mrr),
-        row('nDCG@5', r.flatfile.ndcgAt5, r.baseline.ndcgAt5, r.cachly.ndcgAt5, r.liftVsFlatfile.ndcgAt5),
-        '──────────────────────────────────────────────────────────────────────────',
-        `  vs BM25 baseline : MRR ${signedPct(r.lift.mrr)} · Precision@1 ${signedPct(r.lift.precisionAt1)}`,
-        `  vs flat-file mem : MRR ${signedPct(r.liftVsFlatfile.mrr)} · Precision@1 ${signedPct(r.liftVsFlatfile.precisionAt1)}`,
-        '',
-    ].join('\n');
-}
-// ── CLI entry ───────────────────────────────────────────────────────────────────────────
-const isMain = (() => {
-    try {
-        return import.meta.url === `file://${process.argv[1]}`
-            || process.argv[1]?.endsWith('cachly-bench.ts')
-            || process.argv[1]?.endsWith('cachly-bench.js');
-    }
-    catch {
-        return false;
-    }
-})();
-if (isMain) {
-    runBenchmark().then(result => {
-        if (process.argv.includes('--json')) {
-            process.stdout.write(JSON.stringify(result, null, 2) + '\n');
-        }
-        else {
-            process.stdout.write(formatReport(result));
-        }
-    }).catch(err => {
-        process.stderr.write(`bench failed: ${err instanceof Error ? err.stack : String(err)}\n`);
-        process.exit(1);
-    });
-}
-//# sourceMappingURL=cachly-bench.js.map
+import{EventEmitter as F}from"node:events";import{keywordSearch as $}from"../search.js";import{rerankByQuality as q}from"../rerank.js";import{BENCH_LESSONS as B,BENCH_QUERIES as k}from"./fixtures.js";const m="cachly:lesson:best:";class v{store=new Map;set(e,n){this.store.set(e,n)}async get(e){return this.store.get(e)??null}scanStream(e){const n=new F,s="^"+e.match.replace(/[.+?^${}()|[\]\\]/g,"\\$&").replace(/\*/g,".*")+"$",i=new RegExp(s),c=[...this.store.keys()].filter(a=>i.test(a));return setImmediate(()=>{n.emit("data",c),n.emit("end")}),n}pipeline(){const e=[],n=this.store;return{get(s){return e.push(s),this},async exec(){return e.map(s=>[null,n.get(s)??null])}}}}function C(t){const e=new v;for(const n of t)e.set(`${m}${n.topic}`,JSON.stringify(n));return e}function w(t,e,n){const s=t.slice(0,n);return s.length===0?0:s.filter(c=>e.has(c)).length/Math.min(n,s.length)}function x(t,e,n){return e.size===0?0:t.slice(0,n).filter(c=>e.has(c)).length/e.size}function V(t,e){for(let n=0;n<t.length;n++)if(e.has(t[n]))return 1/(n+1);return 0}function z(t,e,n){let s=0;for(let c=0;c<Math.min(n,t.length);c++)e.has(t[c])&&(s+=1/Math.log2(c+2));let i=0;for(let c=0;c<Math.min(n,e.size);c++)i+=1/Math.log2(c+2);return i===0?0:s/i}function L(t,e){const n=new Set(e.toLowerCase().replace(/[^a-z0-9\s]/g," ").split(/\s+/).filter(i=>i.length>=3));return t.map(i=>{const c=[i.topic,i.what_worked,i.what_failed??"",i.context??"",(i.tags??[]).join(" ")].join(" ").toLowerCase();let a=0;for(const o of n)c.includes(o)&&a++;const u=i.ts?new Date(i.ts).getTime():0;return{topic:i.topic,overlap:a,recency:u}}).filter(i=>i.overlap>0).sort((i,c)=>c.overlap-i.overlap||c.recency-i.recency).map(i=>i.topic)}const S=t=>t.key.replace(m,"");function f(t){return t.length===0?0:t.reduce((e,n)=>e+n,0)/t.length}function g(t){return{precisionAt1:f(t.map(e=>e.precisionAt1)),precisionAt3:f(t.map(e=>e.precisionAt3)),recallAt3:f(t.map(e=>e.recallAt3)),mrr:f(t.map(e=>e.mrr)),ndcgAt5:f(t.map(e=>e.ndcgAt5))}}function d(t,e){return{precisionAt1:w(t,e,1),precisionAt3:w(t,e,3),recallAt3:x(t,e,3),mrr:V(t,e),ndcgAt5:z(t,e,5)}}async function N(t,e){const n=C(t),s=[],i=[],c=[];for(const r of e){const l=new Set(r.relevant),R=(await $(n,[`${m}*`],r.query,10)).map(S),M=await $(n,[`${m}*`],r.query,25),b=q(M).map(S),E=L(t,r.query);s.push(d(E,l)),i.push(d(R,l)),c.push(d(b,l))}const a=g(s),u=g(i),o=g(c),p=(r,l)=>r===0?l>0?1:0:(l-r)/r,y=r=>({precisionAt1:p(r.precisionAt1,o.precisionAt1),precisionAt3:p(r.precisionAt3,o.precisionAt3),recallAt3:p(r.recallAt3,o.recallAt3),mrr:p(r.mrr,o.mrr),ndcgAt5:p(r.ndcgAt5,o.ndcgAt5)});return{flatfile:a,baseline:u,cachly:o,lift:y(u),liftVsFlatfile:y(a),queryCount:e.length,corpusSize:t.length}}async function O(){return N(B,k)}const A=t=>`${(t*100).toFixed(1)}%`,h=t=>`${t>=0?"+":""}${(t*100).toFixed(1)}%`;function P(t){const e=(n,s,i,c,a)=>`  ${n.padEnd(12)} ${A(s).padStart(8)} ${A(i).padStart(8)} ${A(c).padStart(8)}   ${h(a).padStart(8)}`;return["","\u{1F9E0}  Cachly-Bench \u2014 recall quality head-to-head","\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500",`  corpus: ${t.corpusSize} lessons \xB7 queries: ${t.queryCount}`,"  flat-file = naive term-overlap, no quality signal (\u2248 LLM reading memory files)","  baseline  = raw BM25+ keyword ranking","  cachly    = BM25+ followed by quality-aware reranking","",`  ${"metric".padEnd(12)} ${"flatfile".padStart(8)} ${"baseline".padStart(8)} ${"cachly".padStart(8)}   ${"vs flat".padStart(8)}`,e("Precision@1",t.flatfile.precisionAt1,t.baseline.precisionAt1,t.cachly.precisionAt1,t.liftVsFlatfile.precisionAt1),e("Precision@3",t.flatfile.precisionAt3,t.baseline.precisionAt3,t.cachly.precisionAt3,t.liftVsFlatfile.precisionAt3),e("Recall@3",t.flatfile.recallAt3,t.baseline.recallAt3,t.cachly.recallAt3,t.liftVsFlatfile.recallAt3),e("MRR",t.flatfile.mrr,t.baseline.mrr,t.cachly.mrr,t.liftVsFlatfile.mrr),e("nDCG@5",t.flatfile.ndcgAt5,t.baseline.ndcgAt5,t.cachly.ndcgAt5,t.liftVsFlatfile.ndcgAt5),"\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500",`  vs BM25 baseline : MRR ${h(t.lift.mrr)} \xB7 Precision@1 ${h(t.lift.precisionAt1)}`,`  vs flat-file mem : MRR ${h(t.liftVsFlatfile.mrr)} \xB7 Precision@1 ${h(t.liftVsFlatfile.precisionAt1)}`,""].join(`
+`)}const j=(()=>{try{return import.meta.url===`file://${process.argv[1]}`||process.argv[1]?.endsWith("cachly-bench.ts")||process.argv[1]?.endsWith("cachly-bench.js")}catch{return!1}})();j&&O().then(t=>{process.argv.includes("--json")?process.stdout.write(JSON.stringify(t,null,2)+`
+`):process.stdout.write(P(t))}).catch(t=>{process.stderr.write(`bench failed: ${t instanceof Error?t.stack:String(t)}
+`),process.exit(1)});export{P as formatReport,O as runBenchmark,N as runBenchmarkOn};

package/dist/src/bench/external-corpus.js

@@ -1,135 +1,4 @@
-/**
- * External labeled-corpus benchmark.
- *
- * The built-in fixture bench (fixtures.ts) proves lift on a corpus *we* wrote.
- * The honest next step is to prove the same lift on a corpus authored by a third
- * party. This module loads a labeled corpus from a portable JSON file and runs the
- * exact same three-ranker comparison (flat-file vs BM25 baseline vs cachly).
- *
- * Portable format (BEIR-ish, but with the optional quality metadata our reranker
- * uses — a generic IR corpus without it will simply show cachly ≈ BM25, which is
- * itself an honest, documented result):
- *
- *   {
- *     "name": "my-team-incidents-2025",
- *     "lessons": [
- *       { "topic": "deploy:oom", "outcome": "success", "what_worked": "...",
- *         "confidence": 0.9, "recall_count": 4, "severity": "critical",
- *         "review_level": "senior", "reviewed_by": "alice", "endorsements": 2,
- *         "ts": "2025-01-02T00:00:00.000Z" },
- *       ...
- *     ],
- *     "queries": [
- *       { "query": "pod keeps restarting out of memory", "relevant": ["deploy:oom"] },
- *       ...
- *     ]
- *   }
- *
- * Run:  npm run bench:external                       (uses the bundled sample)
- *       npm run bench:external -- ./path/corpus.json (your own labeled corpus)
- *       npm run bench:external -- ./corpus.json --json
- */
-import { readFile } from 'node:fs/promises';
-import { fileURLToPath } from 'node:url';
-import { dirname, resolve } from 'node:path';
-import { runBenchmarkOn, formatReport } from './cachly-bench.js';
-/** Parse + validate a portable corpus object. Throws a clear error on bad shape. */
-export function parseExternalCorpus(raw) {
-    if (!raw || typeof raw !== 'object') {
-        throw new Error('Corpus must be a JSON object with `lessons` and `queries`.');
-    }
-    const obj = raw;
-    const lessons = obj.lessons;
-    const queries = obj.queries;
-    if (!Array.isArray(lessons) || lessons.length === 0) {
-        throw new Error('Corpus.lessons must be a non-empty array.');
-    }
-    if (!Array.isArray(queries) || queries.length === 0) {
-        throw new Error('Corpus.queries must be a non-empty array.');
-    }
-    const topics = new Set();
-    for (const [i, l] of lessons.entries()) {
-        const lesson = l;
-        if (typeof lesson.topic !== 'string' || !lesson.topic.trim()) {
-            throw new Error(`lessons[${i}].topic must be a non-empty string.`);
-        }
-        if (typeof lesson.what_worked !== 'string') {
-            throw new Error(`lessons[${i}].what_worked must be a string.`);
-        }
-        if (topics.has(lesson.topic)) {
-            throw new Error(`Duplicate lesson topic "${lesson.topic}" — topics must be unique.`);
-        }
-        topics.add(lesson.topic);
-    }
-    for (const [i, q] of queries.entries()) {
-        const query = q;
-        if (typeof query.query !== 'string' || !query.query.trim()) {
-            throw new Error(`queries[${i}].query must be a non-empty string.`);
-        }
-        if (!Array.isArray(query.relevant) || query.relevant.length === 0) {
-            throw new Error(`queries[${i}].relevant must be a non-empty array of topics.`);
-        }
-        for (const rel of query.relevant) {
-            if (typeof rel !== 'string' || !topics.has(rel)) {
-                throw new Error(`queries[${i}] references unknown relevant topic "${String(rel)}".`);
-            }
-        }
-    }
-    return {
-        name: typeof obj.name === 'string' ? obj.name : undefined,
-        lessons: lessons,
-        queries: queries,
-    };
-}
-/** Load + validate a corpus from a JSON file path. */
-export async function loadExternalCorpus(path) {
-    const text = await readFile(path, 'utf-8');
-    let raw;
-    try {
-        raw = JSON.parse(text);
-    }
-    catch (e) {
-        throw new Error(`Corpus file is not valid JSON: ${e.message}`);
-    }
-    return parseExternalCorpus(raw);
-}
-/** Run the three-ranker benchmark on an external corpus. */
-export async function runExternalBenchmark(corpus) {
-    return runBenchmarkOn(corpus.lessons, corpus.queries);
-}
-// ── CLI entry ───────────────────────────────────────────────────────────────────
-const isMain = (() => {
-    try {
-        return import.meta.url === `file://${process.argv[1]}`
-            || process.argv[1]?.endsWith('external-corpus.ts')
-            || process.argv[1]?.endsWith('external-corpus.js');
-    }
-    catch {
-        return false;
-    }
-})();
-if (isMain) {
-    const args = process.argv.slice(2).filter(a => a !== '--json');
-    const asJson = process.argv.includes('--json');
-    const here = dirname(fileURLToPath(import.meta.url));
-    // Default to the bundled sample so `npm run bench:external` works out of the box.
-    const corpusPath = args[0]
-        ? resolve(process.cwd(), args[0])
-        : resolve(here, 'external', 'sample-corpus.json');
-    loadExternalCorpus(corpusPath)
-        .then(async (corpus) => {
-        const result = await runExternalBenchmark(corpus);
-        if (asJson) {
-            console.log(JSON.stringify({ corpus: corpus.name ?? corpusPath, ...result }, null, 2));
-        }
-        else {
-            console.log(`\n📦  External corpus: ${corpus.name ?? corpusPath}`);
-            console.log(formatReport(result));
-        }
-    })
-        .catch((e) => {
-        console.error(`\n❌ External bench failed: ${e.message}\n`);
-        process.exit(1);
-    });
-}
-//# sourceMappingURL=external-corpus.js.map
+import{readFile as p}from"node:fs/promises";import{fileURLToPath as u}from"node:url";import{dirname as f,resolve as l}from"node:path";import{runBenchmarkOn as m,formatReport as h}from"./cachly-bench.js";function y(e){if(!e||typeof e!="object")throw new Error("Corpus must be a JSON object with `lessons` and `queries`.");const i=e,t=i.lessons,o=i.queries;if(!Array.isArray(t)||t.length===0)throw new Error("Corpus.lessons must be a non-empty array.");if(!Array.isArray(o)||o.length===0)throw new Error("Corpus.queries must be a non-empty array.");const s=new Set;for(const[n,a]of t.entries()){const r=a;if(typeof r.topic!="string"||!r.topic.trim())throw new Error(`lessons[${n}].topic must be a non-empty string.`);if(typeof r.what_worked!="string")throw new Error(`lessons[${n}].what_worked must be a string.`);if(s.has(r.topic))throw new Error(`Duplicate lesson topic "${r.topic}" \u2014 topics must be unique.`);s.add(r.topic)}for(const[n,a]of o.entries()){const r=a;if(typeof r.query!="string"||!r.query.trim())throw new Error(`queries[${n}].query must be a non-empty string.`);if(!Array.isArray(r.relevant)||r.relevant.length===0)throw new Error(`queries[${n}].relevant must be a non-empty array of topics.`);for(const c of r.relevant)if(typeof c!="string"||!s.has(c))throw new Error(`queries[${n}] references unknown relevant topic "${String(c)}".`)}return{name:typeof i.name=="string"?i.name:void 0,lessons:t,queries:o}}async function w(e){const i=await p(e,"utf-8");let t;try{t=JSON.parse(i)}catch(o){throw new Error(`Corpus file is not valid JSON: ${o.message}`)}return y(t)}async function g(e){return m(e.lessons,e.queries)}const d=(()=>{try{return import.meta.url===`file://${process.argv[1]}`||process.argv[1]?.endsWith("external-corpus.ts")||process.argv[1]?.endsWith("external-corpus.js")}catch{return!1}})();if(d){const e=process.argv.slice(2).filter(s=>s!=="--json"),i=process.argv.includes("--json"),t=f(u(import.meta.url)),o=e[0]?l(process.cwd(),e[0]):l(t,"external","sample-corpus.json");w(o).then(async s=>{const n=await g(s);i?console.log(JSON.stringify({corpus:s.name??o,...n},null,2)):(console.log(`
+\u{1F4E6}  External corpus: ${s.name??o}`),console.log(h(n)))}).catch(s=>{console.error(`
+\u274C External bench failed: ${s.message}
+`),process.exit(1)})}export{w as loadExternalCorpus,y as parseExternalCorpus,g as runExternalBenchmark};

package/dist/src/bench/fixtures.js

@@ -1,225 +1 @@
-/**
- * Cachly-Bench fixture corpus.
- *
- * A small but realistic corpus of engineering "lessons" plus a set of labeled
- * natural-language queries. Each query declares which lesson topics are *truly*
- * relevant — the answer an agent actually wants.
- *
- * The corpus is deliberately adversarial in a fair way: for several queries there
- * is a text-similar but LOW-QUALITY distractor (a failed attempt, an unverified
- * note) competing with the proven success lesson. This is exactly the situation
- * where raw text ranking (and a flat-file memory) goes wrong, and where Cachly's
- * quality-aware reranking should win. The labels reward the lesson that solves the
- * problem, not the one that merely shares vocabulary.
- */
-const NOW = Date.now();
-const daysAgo = (d) => new Date(NOW - d * 86400_000).toISOString();
-export const BENCH_LESSONS = [
-    // ── Deploy domain ──────────────────────────────────────────────────────────
-    {
-        topic: 'deploy:k8s:rollout-stuck',
-        outcome: 'success',
-        what_worked: 'Set a readinessProbe with a longer initialDelaySeconds and a higher failureThreshold so the liveness probe stopped terminating the container before startup completed. The pod then reached ready and traffic flowed.',
-        context: 'Production release never became healthy.',
-        severity: 'critical', confidence: 0.95, recall_count: 12,
-        tags: ['kubernetes', 'deploy', 'probe'], ts: daysAgo(8),
-    },
-    {
-        topic: 'deploy:k8s:rollout-attempt-restart',
-        outcome: 'failure',
-        what_worked: '',
-        what_failed: 'The kubernetes deploy was stuck, the rollout was not finishing, so I restarted the deploy. The kubernetes deploy stayed stuck and the rollout still was not finishing. Restarted the stuck rollout again, deploy still stuck.',
-        context: 'Kubernetes deploy stuck, rollout not finishing.',
-        severity: 'major', confidence: 0.3, recall_count: 0,
-        tags: ['kubernetes', 'deploy'], ts: daysAgo(9),
-    },
-    {
-        topic: 'deploy:docker:image-too-large',
-        outcome: 'success',
-        what_worked: 'Reduced Docker image size dramatically with a multi-stage build and a slim base image, moving build-only deps out of the final layer.',
-        context: 'CI pushes were slow due to a 1.8GB image.',
-        severity: 'minor', confidence: 0.9, recall_count: 6,
-        tags: ['docker', 'deploy', 'optimization'], ts: daysAgo(20),
-    },
-    // ── Database domain ────────────────────────────────────────────────────────
-    {
-        topic: 'db:postgres:connection-pool-exhausted',
-        outcome: 'success',
-        what_worked: 'Added PgBouncer in transaction pooling mode and lowered the per-service max pool size so backends were shared. Peak traffic stopped exhausting backends.',
-        context: 'API returned 500s at peak traffic.',
-        severity: 'critical', confidence: 0.92, recall_count: 9,
-        tags: ['postgres', 'database', 'pool'], ts: daysAgo(15),
-    },
-    {
-        topic: 'db:postgres:slow-query-missing-index',
-        outcome: 'success',
-        what_worked: 'A slow Postgres query was fixed by adding a composite index on (tenant_id, created_at); query dropped from 4s to 12ms.',
-        context: 'Dashboard timed out loading recent events.',
-        severity: 'major', confidence: 0.88, recall_count: 4,
-        tags: ['postgres', 'database', 'index', 'performance'], ts: daysAgo(30),
-    },
-    {
-        topic: 'db:postgres:pool-raise-limit-attempt',
-        outcome: 'failure',
-        what_worked: '',
-        what_failed: 'Postgres had too many connections under load, so I raised max_connections. Postgres still had too many connections under load and the load made it worse — too many connections again under load.',
-        context: 'Postgres too many connections under load.',
-        severity: 'major', confidence: 0.25, recall_count: 0,
-        tags: ['postgres', 'database', 'pool'], ts: daysAgo(16),
-    },
-    // ── Auth domain ────────────────────────────────────────────────────────────
-    {
-        topic: 'auth:jwt:expired-on-restart',
-        outcome: 'success',
-        what_worked: 'JWT was lost on server restart because it was only kept in memory; persisting the token to disk after the device flow fixed recall counts staying at zero.',
-        context: 'Telemetry showed zero authenticated calls after restarts.',
-        severity: 'critical', confidence: 0.93, recall_count: 7,
-        tags: ['jwt', 'auth', 'persistence'], ts: daysAgo(3),
-    },
-    {
-        topic: 'auth:pkce:keycloak-rejects-exchange',
-        outcome: 'success',
-        what_worked: 'Keycloak rejected the PKCE code exchange because the verifier used non-URL-safe Base64; switching to RFC 7636 Base64URL random bytes fixed the login.',
-        context: 'Mobile login failed at the token exchange step.',
-        severity: 'major', confidence: 0.9, recall_count: 3,
-        tags: ['pkce', 'auth', 'keycloak', 'oauth'], ts: daysAgo(2),
-    },
-    // ── Redis / cache domain ───────────────────────────────────────────────────
-    {
-        topic: 'redis:noauth-on-every-call',
-        outcome: 'success',
-        what_worked: 'The connection endpoint omitted the password field, so the client connected unauthenticated; returning the credential explicitly in the response let the client authenticate and the calls succeeded.',
-        context: 'Hosted Valkey requires a credential.',
-        severity: 'critical', confidence: 0.94, recall_count: 11,
-        tags: ['redis', 'valkey', 'auth'], ts: daysAgo(4),
-    },
-    {
-        topic: 'redis:noauth-disable-auth-attempt',
-        outcome: 'failure',
-        what_worked: '',
-        what_failed: 'Redis threw NOAUTH on every call, so I tried disabling auth on Redis. Redis still threw NOAUTH on every call. Tried again to stop the NOAUTH error on every Redis call — no luck.',
-        context: 'Redis NOAUTH error on every call.',
-        severity: 'major', confidence: 0.2, recall_count: 0,
-        tags: ['redis', 'auth'], ts: daysAgo(5),
-    },
-    {
-        topic: 'redis:keys-no-ttl-memory-leak',
-        outcome: 'success',
-        what_worked: 'Redis memory grew unbounded because dependency-index and history keys had no TTL; adding a 90-day expire and an ltrim bounded the memory.',
-        context: 'Instance memory crept up over weeks of use.',
-        severity: 'major', confidence: 0.85, recall_count: 2,
-        tags: ['redis', 'ttl', 'memory'], ts: daysAgo(1),
-    },
-    // ── CI / build domain ──────────────────────────────────────────────────────
-    {
-        topic: 'ci:npm-ci-lockfile-mismatch',
-        outcome: 'success',
-        what_worked: 'CI "npm ci" failed because a dependency was added to package.json but the lockfile was never regenerated; running npm install to update package-lock.json fixed the build.',
-        context: 'GitHub Actions install step failed after adding a dep.',
-        severity: 'major', confidence: 0.91, recall_count: 5,
-        tags: ['ci', 'npm', 'lockfile'], ts: daysAgo(0),
-    },
-    {
-        topic: 'ci:golangci-lint-errcheck',
-        outcome: 'success',
-        what_worked: 'golangci-lint failed on an unchecked recover() return value; assigning it to _ satisfied errcheck.',
-        context: 'Go lint job blocked the deploy pipeline.',
-        severity: 'minor', confidence: 0.8, recall_count: 1,
-        tags: ['ci', 'go', 'lint'], ts: daysAgo(1),
-    },
-    // ── Frontend domain ────────────────────────────────────────────────────────
-    {
-        topic: 'frontend:cors-preflight-blocked',
-        outcome: 'success',
-        what_worked: 'Dashboard mutations failed because the CORS middleware did not allow PUT/PATCH; adding them to AllowMethods fixed the preflight.',
-        context: 'BYOK and webhook settings could not be saved from the browser.',
-        severity: 'major', confidence: 0.87, recall_count: 4,
-        tags: ['cors', 'frontend', 'api'], ts: daysAgo(6),
-    },
-    {
-        topic: 'frontend:hydration-mismatch',
-        outcome: 'partial',
-        what_worked: 'A Next.js hydration mismatch was reduced by guarding a date render with a client-only effect, though the root cause (locale formatting) still needs work.',
-        context: 'Console hydration warnings on the landing page.',
-        severity: 'minor', confidence: 0.6, recall_count: 1,
-        tags: ['nextjs', 'frontend', 'hydration'], ts: daysAgo(25),
-    },
-    // ── Governance adversarial pair: two similar-text success lessons ──────────
-    // Only the reviewed one should win; baseline cannot distinguish them by text.
-    {
-        topic: 'auth:csrf:double-submit-cookie',
-        outcome: 'success',
-        what_worked: 'CSRF protection was added via the double-submit cookie pattern: the server sets a random CSRF token cookie; the client echoes it as a request header. Requests with a mismatched or absent header are rejected.',
-        context: 'API mutations lacked CSRF protection; security audit required a fix.',
-        severity: 'critical', confidence: 0.91, recall_count: 2,
-        reviewed_by: 'senior-alice', review_level: 'senior', endorsements: 2,
-        tags: ['csrf', 'auth', 'security', 'cookie'], ts: daysAgo(12),
-    },
-    {
-        topic: 'auth:csrf:origin-header-check',
-        outcome: 'success',
-        what_worked: 'CSRF mitigation attempted by checking the Origin header on API requests, but this was bypassed in a penetration test because same-site requests from subdomains also sent the header.',
-        context: 'API mutations lacked CSRF protection; origin header check was insufficient.',
-        severity: 'major', confidence: 0.55, recall_count: 1,
-        tags: ['csrf', 'auth', 'security'], ts: daysAgo(14),
-    },
-];
-export const BENCH_QUERIES = [
-    {
-        query: 'kubernetes deploy stuck rollout not finishing',
-        relevant: ['deploy:k8s:rollout-stuck'],
-    },
-    {
-        query: 'postgres too many connections under load',
-        relevant: ['db:postgres:connection-pool-exhausted'],
-    },
-    {
-        query: 'dashboard query is slow and times out',
-        relevant: ['db:postgres:slow-query-missing-index'],
-    },
-    {
-        query: 'jwt lost after restart recall count zero',
-        relevant: ['auth:jwt:expired-on-restart'],
-    },
-    {
-        query: 'keycloak rejects pkce token exchange on login',
-        relevant: ['auth:pkce:keycloak-rejects-exchange'],
-    },
-    {
-        query: 'redis noauth error on every call',
-        relevant: ['redis:noauth-on-every-call'],
-    },
-    {
-        query: 'redis memory keeps growing unbounded',
-        relevant: ['redis:keys-no-ttl-memory-leak'],
-    },
-    {
-        query: 'npm ci fails in github actions after adding dependency',
-        relevant: ['ci:npm-ci-lockfile-mismatch'],
-    },
-    {
-        query: 'cannot save settings cors preflight blocked',
-        relevant: ['frontend:cors-preflight-blocked'],
-    },
-    {
-        query: 'docker image too big slow ci push',
-        relevant: ['deploy:docker:image-too-large'],
-    },
-    {
-        query: 'go lint failing on unchecked return value',
-        relevant: ['ci:golangci-lint-errcheck'],
-    },
-    // Multi-relevant: both postgres performance lessons are reasonable answers.
-    {
-        query: 'postgres performance problem in production',
-        relevant: ['db:postgres:connection-pool-exhausted', 'db:postgres:slow-query-missing-index'],
-    },
-    // Governance adversarial: two CSRF lessons have almost identical text, but only
-    // the senior-reviewed canonical one is a proven complete solution. Quality
-    // reranking must prefer the reviewed+endorsed lesson over the unreviewed partial.
-    {
-        query: 'CSRF protection missing on API mutations security',
-        relevant: ['auth:csrf:double-submit-cookie'],
-    },
-];
-//# sourceMappingURL=fixtures.js.map
+const o=Date.now(),e=t=>new Date(o-t*864e5).toISOString(),a=[{topic:"deploy:k8s:rollout-stuck",outcome:"success",what_worked:"Set a readinessProbe with a longer initialDelaySeconds and a higher failureThreshold so the liveness probe stopped terminating the container before startup completed. The pod then reached ready and traffic flowed.",context:"Production release never became healthy.",severity:"critical",confidence:.95,recall_count:12,tags:["kubernetes","deploy","probe"],ts:e(8)},{topic:"deploy:k8s:rollout-attempt-restart",outcome:"failure",what_worked:"",what_failed:"The kubernetes deploy was stuck, the rollout was not finishing, so I restarted the deploy. The kubernetes deploy stayed stuck and the rollout still was not finishing. Restarted the stuck rollout again, deploy still stuck.",context:"Kubernetes deploy stuck, rollout not finishing.",severity:"major",confidence:.3,recall_count:0,tags:["kubernetes","deploy"],ts:e(9)},{topic:"deploy:docker:image-too-large",outcome:"success",what_worked:"Reduced Docker image size dramatically with a multi-stage build and a slim base image, moving build-only deps out of the final layer.",context:"CI pushes were slow due to a 1.8GB image.",severity:"minor",confidence:.9,recall_count:6,tags:["docker","deploy","optimization"],ts:e(20)},{topic:"db:postgres:connection-pool-exhausted",outcome:"success",what_worked:"Added PgBouncer in transaction pooling mode and lowered the per-service max pool size so backends were shared. Peak traffic stopped exhausting backends.",context:"API returned 500s at peak traffic.",severity:"critical",confidence:.92,recall_count:9,tags:["postgres","database","pool"],ts:e(15)},{topic:"db:postgres:slow-query-missing-index",outcome:"success",what_worked:"A slow Postgres query was fixed by adding a composite index on (tenant_id, created_at); query dropped from 4s to 12ms.",context:"Dashboard timed out loading recent events.",severity:"major",confidence:.88,recall_count:4,tags:["postgres","database","index","performance"],ts:e(30)},{topic:"db:postgres:pool-raise-limit-attempt",outcome:"failure",what_worked:"",what_failed:"Postgres had too many connections under load, so I raised max_connections. Postgres still had too many connections under load and the load made it worse \u2014 too many connections again under load.",context:"Postgres too many connections under load.",severity:"major",confidence:.25,recall_count:0,tags:["postgres","database","pool"],ts:e(16)},{topic:"auth:jwt:expired-on-restart",outcome:"success",what_worked:"JWT was lost on server restart because it was only kept in memory; persisting the token to disk after the device flow fixed recall counts staying at zero.",context:"Telemetry showed zero authenticated calls after restarts.",severity:"critical",confidence:.93,recall_count:7,tags:["jwt","auth","persistence"],ts:e(3)},{topic:"auth:pkce:keycloak-rejects-exchange",outcome:"success",what_worked:"Keycloak rejected the PKCE code exchange because the verifier used non-URL-safe Base64; switching to RFC 7636 Base64URL random bytes fixed the login.",context:"Mobile login failed at the token exchange step.",severity:"major",confidence:.9,recall_count:3,tags:["pkce","auth","keycloak","oauth"],ts:e(2)},{topic:"redis:noauth-on-every-call",outcome:"success",what_worked:"The connection endpoint omitted the password field, so the client connected unauthenticated; returning the credential explicitly in the response let the client authenticate and the calls succeeded.",context:"Hosted Valkey requires a credential.",severity:"critical",confidence:.94,recall_count:11,tags:["redis","valkey","auth"],ts:e(4)},{topic:"redis:noauth-disable-auth-attempt",outcome:"failure",what_worked:"",what_failed:"Redis threw NOAUTH on every call, so I tried disabling auth on Redis. Redis still threw NOAUTH on every call. Tried again to stop the NOAUTH error on every Redis call \u2014 no luck.",context:"Redis NOAUTH error on every call.",severity:"major",confidence:.2,recall_count:0,tags:["redis","auth"],ts:e(5)},{topic:"redis:keys-no-ttl-memory-leak",outcome:"success",what_worked:"Redis memory grew unbounded because dependency-index and history keys had no TTL; adding a 90-day expire and an ltrim bounded the memory.",context:"Instance memory crept up over weeks of use.",severity:"major",confidence:.85,recall_count:2,tags:["redis","ttl","memory"],ts:e(1)},{topic:"ci:npm-ci-lockfile-mismatch",outcome:"success",what_worked:'CI "npm ci" failed because a dependency was added to package.json but the lockfile was never regenerated; running npm install to update package-lock.json fixed the build.',context:"GitHub Actions install step failed after adding a dep.",severity:"major",confidence:.91,recall_count:5,tags:["ci","npm","lockfile"],ts:e(0)},{topic:"ci:golangci-lint-errcheck",outcome:"success",what_worked:"golangci-lint failed on an unchecked recover() return value; assigning it to _ satisfied errcheck.",context:"Go lint job blocked the deploy pipeline.",severity:"minor",confidence:.8,recall_count:1,tags:["ci","go","lint"],ts:e(1)},{topic:"frontend:cors-preflight-blocked",outcome:"success",what_worked:"Dashboard mutations failed because the CORS middleware did not allow PUT/PATCH; adding them to AllowMethods fixed the preflight.",context:"BYOK and webhook settings could not be saved from the browser.",severity:"major",confidence:.87,recall_count:4,tags:["cors","frontend","api"],ts:e(6)},{topic:"frontend:hydration-mismatch",outcome:"partial",what_worked:"A Next.js hydration mismatch was reduced by guarding a date render with a client-only effect, though the root cause (locale formatting) still needs work.",context:"Console hydration warnings on the landing page.",severity:"minor",confidence:.6,recall_count:1,tags:["nextjs","frontend","hydration"],ts:e(25)},{topic:"auth:csrf:double-submit-cookie",outcome:"success",what_worked:"CSRF protection was added via the double-submit cookie pattern: the server sets a random CSRF token cookie; the client echoes it as a request header. Requests with a mismatched or absent header are rejected.",context:"API mutations lacked CSRF protection; security audit required a fix.",severity:"critical",confidence:.91,recall_count:2,reviewed_by:"senior-alice",review_level:"senior",endorsements:2,tags:["csrf","auth","security","cookie"],ts:e(12)},{topic:"auth:csrf:origin-header-check",outcome:"success",what_worked:"CSRF mitigation attempted by checking the Origin header on API requests, but this was bypassed in a penetration test because same-site requests from subdomains also sent the header.",context:"API mutations lacked CSRF protection; origin header check was insufficient.",severity:"major",confidence:.55,recall_count:1,tags:["csrf","auth","security"],ts:e(14)}],n=[{query:"kubernetes deploy stuck rollout not finishing",relevant:["deploy:k8s:rollout-stuck"]},{query:"postgres too many connections under load",relevant:["db:postgres:connection-pool-exhausted"]},{query:"dashboard query is slow and times out",relevant:["db:postgres:slow-query-missing-index"]},{query:"jwt lost after restart recall count zero",relevant:["auth:jwt:expired-on-restart"]},{query:"keycloak rejects pkce token exchange on login",relevant:["auth:pkce:keycloak-rejects-exchange"]},{query:"redis noauth error on every call",relevant:["redis:noauth-on-every-call"]},{query:"redis memory keeps growing unbounded",relevant:["redis:keys-no-ttl-memory-leak"]},{query:"npm ci fails in github actions after adding dependency",relevant:["ci:npm-ci-lockfile-mismatch"]},{query:"cannot save settings cors preflight blocked",relevant:["frontend:cors-preflight-blocked"]},{query:"docker image too big slow ci push",relevant:["deploy:docker:image-too-large"]},{query:"go lint failing on unchecked return value",relevant:["ci:golangci-lint-errcheck"]},{query:"postgres performance problem in production",relevant:["db:postgres:connection-pool-exhausted","db:postgres:slow-query-missing-index"]},{query:"CSRF protection missing on API mutations security",relevant:["auth:csrf:double-submit-cookie"]}];export{a as BENCH_LESSONS,n as BENCH_QUERIES};