@cachly-dev/mcp-server 0.10.49 → 0.10.50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/BENCH.md +80 -0
- package/STRATEGY.md +21 -12
- package/apps/travel-chaos-organizer/frontend/package-lock.json +184 -0
- package/dist/src/__tests__/rerank.test.d.ts +1 -0
- package/dist/src/__tests__/rerank.test.js +98 -0
- package/dist/src/__tests__/rerank.test.js.map +1 -0
- package/dist/src/bench/cachly-bench.d.ts +35 -0
- package/dist/src/bench/cachly-bench.js +182 -0
- package/dist/src/bench/cachly-bench.js.map +1 -0
- package/dist/src/bench/fixtures.d.ts +34 -0
- package/dist/src/bench/fixtures.js +199 -0
- package/dist/src/bench/fixtures.js.map +1 -0
- package/dist/src/handlers/brain.js +135 -65
- package/dist/src/handlers/brain.js.map +1 -1
- package/dist/src/rerank.d.ts +50 -0
- package/dist/src/rerank.js +92 -0
- package/dist/src/rerank.js.map +1 -0
- package/package.json +2 -1
- package/src/__tests__/rerank.test.ts +111 -0
- package/src/bench/cachly-bench.ts +217 -0
- package/src/bench/fixtures.ts +226 -0
- package/src/handlers/brain.ts +150 -73
- package/src/rerank.ts +123 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cachly-bench.js","sourceRoot":"","sources":["../../../src/bench/cachly-bench.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAqB,MAAM,cAAc,CAAC;AAChE,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC/C,OAAO,EAAE,aAAa,EAAE,aAAa,EAAoB,MAAM,eAAe,CAAC;AAE/E,MAAM,aAAa,GAAG,qBAAqB,CAAC;AAE5C,kFAAkF;AAClF,MAAM,SAAS;IACL,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAC;IAE1C,GAAG,CAAC,GAAW,EAAE,KAAa,IAAU,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;IACrE,KAAK,CAAC,GAAG,CAAC,GAAW,IAA4B,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC;IAEtF,UAAU,CAAC,IAAuC;QAChD,MAAM,OAAO,GAAG,IAAI,YAAY,EAAE,CAAC;QACnC,MAAM,OAAO,GAAG,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,oBAAoB,EAAE,MAAM,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,GAAG,CAAC;QAClG,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC;QAClC,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QAClE,YAAY,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5E,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,QAAQ;QACN,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACzB,OAAO;YACL,GAAG,CAAC,GAAW,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC;YACjD,KAAK,CAAC,IAAI;gBACR,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,CAA0B,CAAC,CAAC;YAC9E,CAAC;SACF,CAAC;IACJ,CAAC;CACF;AAED,SAAS,WAAW,CAAC,OAAsB;IACzC,MAAM,KAAK,GAAG,IAAI,SAAS,EAAE,CAAC;IAC9B,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,KAAK,CAAC,GAAG,CAAC,GAAG,aAAa,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7D,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,kFAAkF;AAClF,SAAS,YAAY,CAAC,MAAgB,EAAE,QAAqB,EAAE,CAAS;IACtE,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC/B,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAC/B,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IACrD,OAAO,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;AACxC,CAAC;AAED,SAAS,SAAS,CAAC,MAAgB,EAAE,QAAqB,EAAE,CAAS;IACnE,IAAI,QAAQ,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAClC,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC/B,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IACrD,OAAO,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC;AAC9B,CAAC;AAED,SAAS,cAAc,CAAC,MAAgB,EAAE,QAAqB;IAC7D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,IAAI,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YAAE,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAClD,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC;AAED,SAAS,OAAO,CAAC,MAAgB,EAAE,QAAqB,EAAE,CAAS;IACjE,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACpD,IAAI,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YAAE,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC3D,CAAC;IACD,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACpD,IAAI,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC/B,CAAC;IACD,OAAO,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,IAAI,CAAC;AACrC,CAAC;AAmBD,MAAM,UAAU,GAAG,CAAC,CAAe,EAAU,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC;AAEjF,SAAS,IAAI,CAAC,EAAY;IACxB,OAAO,EAAE,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC;AACzE,CAAC;AAED,SAAS,SAAS,CAAC,IAAoB;IACrC,OAAO;QACL,YAAY,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;QACjD,YAAY,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;QACjD,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAC3C,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC/B,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;KACxC,CAAC;AACJ,CAAC;AAED,SAAS,YAAY,CAAC,MAAgB,EAAE,QAAqB;IAC3D,OAAO;QACL,YAAY,EAAE,YAAY,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC/C,YAAY,EAAE,YAAY,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC/C,SAAS,EAAE,SAAS,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;QACzC,GAAG,EAAE,cAAc,CAAC,MAAM,EAAE,QAAQ,CAAC;QACrC,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;KACtC,CAAC;AACJ,CAAC;AAED,sFAAsF;AACtF,MAAM,CAAC,KAAK,UAAU,YAAY;IAChC,MAAM,KAAK,GAAG,WAAW,CAAC,aAAa,CAAC,CAAC;IACzC,MAAM,YAAY,GAAmB,EAAE,CAAC;IACxC,MAAM,UAAU,GAAmB,EAAE,CAAC;IAEtC,KAAK,MAAM,CAAC,IAAI,aAAa,EAAE,CAAC;QAC9B,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QAErC,2EAA2E;QAC3E,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,KAAc,EAAE,CAAC,GAAG,aAAa,GAAG,CAAC,EAAE,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAExF,MAAM,cAAc,GAAG,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAC/C,MAAM,YAAY,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAE9D,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,cAAc,EAAE,QAAQ,CAAC,CAAC,CAAC;QAC1D,UAAU,CAAC,IAAI,CAAC,YAAY,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC,CAAC;IACxD,CAAC;IAED,MAAM,QAAQ,GAAG,SAAS,CAAC,YAAY,CAAC,CAAC;IACzC,MAAM,MAAM,GAAG,SAAS,CAAC,UAAU,CAAC,CAAC;IACrC,MAAM,MAAM,GAAG,CAAC,CAAS,EAAE,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACnF,MAAM,IAAI,GAAiB;QACzB,YAAY,EAAE,MAAM,CAAC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC,YAAY,CAAC;QAChE,YAAY,EAAE,MAAM,CAAC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC,YAAY,CAAC;QAChE,SAAS,EAAE,MAAM,CAAC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC;QACvD,GAAG,EAAE,MAAM,CAAC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC;QACrC,OAAO,EAAE,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC;KAClD,CAAC;IAEF,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,aAAa,CAAC,MAAM,EAAE,UAAU,EAAE,aAAa,CAAC,MAAM,EAAE,CAAC;AACxG,CAAC;AAED,kFAAkF;AAClF,MAAM,GAAG,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;AACtD,MAAM,SAAS,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;AAEhF,MAAM,UAAU,YAAY,CAAC,CAAc;IACzC,MAAM,GAAG,GAAG,CAAC,KAAa,EAAE,CAAS,EAAE,CAAS,EAAE,CAAS,EAAE,EAAE,CAC7D,KAAK,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,SAAS,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;IACxG,OAAO;QACL,EAAE;QACF,2EAA2E;QAC3E,0EAA0E;QAC1E,aAAa,CAAC,CAAC,UAAU,uBAAuB,CAAC,CAAC,UAAU,EAAE;QAC9D,EAAE;QACF,KAAK,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE;QAC3G,GAAG,CAAC,aAAa,EAAE,CAAC,CAAC,QAAQ,CAAC,YAAY,EAAE,CAAC,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC;QACvF,GAAG,CAAC,aAAa,EAAE,CAAC,CAAC,QAAQ,CAAC,YAAY,EAAE,CAAC,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC;QACvF,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC;QAC3E,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;QACpD,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC;QACnE,0EAA0E;QAC1E,mBAAmB,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,kBAAkB,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE;QAC1F,EAAE;KACH,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED,mFAAmF;AACnF,MAAM,MAAM,GAAG,CAAC,GAAG,EAAE;IACnB,IAAI,CAAC;QACH,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,UAAU,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE;eACjD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,iBAAiB,CAAC;eAC5C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,iBAAiB,CAAC,CAAC;IACpD,CAAC;IAAC,MAAM,CAAC;QAAC,OAAO,KAAK,CAAC;IAAC,CAAC;AAC3B,CAAC,CAAC,EAAE,CAAC;AAEL,IAAI,MAAM,EAAE,CAAC;IACX,YAAY,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE;QAC3B,IAAI,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;YACpC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;QAC/D,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE;QACb,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,iBAAiB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC1F,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cachly-Bench fixture corpus.
|
|
3
|
+
*
|
|
4
|
+
* A small but realistic corpus of engineering "lessons" plus a set of labeled
|
|
5
|
+
* natural-language queries. Each query declares which lesson topics are *truly*
|
|
6
|
+
* relevant — the answer an agent actually wants.
|
|
7
|
+
*
|
|
8
|
+
* The corpus is deliberately adversarial in a fair way: for several queries there
|
|
9
|
+
* is a text-similar but LOW-QUALITY distractor (a failed attempt, an unverified
|
|
10
|
+
* note) competing with the proven success lesson. This is exactly the situation
|
|
11
|
+
* where raw text ranking (and a flat-file memory) goes wrong, and where Cachly's
|
|
12
|
+
* quality-aware reranking should win. The labels reward the lesson that solves the
|
|
13
|
+
* problem, not the one that merely shares vocabulary.
|
|
14
|
+
*/
|
|
15
|
+
export interface BenchLesson {
|
|
16
|
+
/** Stored under cachly:lesson:best:{topic} */
|
|
17
|
+
topic: string;
|
|
18
|
+
outcome: 'success' | 'partial' | 'failure';
|
|
19
|
+
what_worked: string;
|
|
20
|
+
what_failed?: string;
|
|
21
|
+
context?: string;
|
|
22
|
+
severity?: 'critical' | 'major' | 'minor';
|
|
23
|
+
confidence?: number;
|
|
24
|
+
recall_count?: number;
|
|
25
|
+
tags?: string[];
|
|
26
|
+
ts?: string;
|
|
27
|
+
}
|
|
28
|
+
export interface BenchQuery {
|
|
29
|
+
query: string;
|
|
30
|
+
/** Topics that genuinely answer the query (the gold set). */
|
|
31
|
+
relevant: string[];
|
|
32
|
+
}
|
|
33
|
+
export declare const BENCH_LESSONS: BenchLesson[];
|
|
34
|
+
export declare const BENCH_QUERIES: BenchQuery[];
|
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cachly-Bench fixture corpus.
|
|
3
|
+
*
|
|
4
|
+
* A small but realistic corpus of engineering "lessons" plus a set of labeled
|
|
5
|
+
* natural-language queries. Each query declares which lesson topics are *truly*
|
|
6
|
+
* relevant — the answer an agent actually wants.
|
|
7
|
+
*
|
|
8
|
+
* The corpus is deliberately adversarial in a fair way: for several queries there
|
|
9
|
+
* is a text-similar but LOW-QUALITY distractor (a failed attempt, an unverified
|
|
10
|
+
* note) competing with the proven success lesson. This is exactly the situation
|
|
11
|
+
* where raw text ranking (and a flat-file memory) goes wrong, and where Cachly's
|
|
12
|
+
* quality-aware reranking should win. The labels reward the lesson that solves the
|
|
13
|
+
* problem, not the one that merely shares vocabulary.
|
|
14
|
+
*/
|
|
15
|
+
const NOW = Date.now();
|
|
16
|
+
const daysAgo = (d) => new Date(NOW - d * 86400_000).toISOString();
|
|
17
|
+
export const BENCH_LESSONS = [
|
|
18
|
+
// ── Deploy domain ──────────────────────────────────────────────────────────
|
|
19
|
+
{
|
|
20
|
+
topic: 'deploy:k8s:rollout-stuck',
|
|
21
|
+
outcome: 'success',
|
|
22
|
+
what_worked: 'Set a readinessProbe with a longer initialDelaySeconds and a higher failureThreshold so the liveness probe stopped terminating the container before startup completed. The pod then reached ready and traffic flowed.',
|
|
23
|
+
context: 'Production release never became healthy.',
|
|
24
|
+
severity: 'critical', confidence: 0.95, recall_count: 12,
|
|
25
|
+
tags: ['kubernetes', 'deploy', 'probe'], ts: daysAgo(8),
|
|
26
|
+
},
|
|
27
|
+
{
|
|
28
|
+
topic: 'deploy:k8s:rollout-attempt-restart',
|
|
29
|
+
outcome: 'failure',
|
|
30
|
+
what_worked: '',
|
|
31
|
+
what_failed: 'The kubernetes deploy was stuck, the rollout was not finishing, so I restarted the deploy. The kubernetes deploy stayed stuck and the rollout still was not finishing. Restarted the stuck rollout again, deploy still stuck.',
|
|
32
|
+
context: 'Kubernetes deploy stuck, rollout not finishing.',
|
|
33
|
+
severity: 'major', confidence: 0.3, recall_count: 0,
|
|
34
|
+
tags: ['kubernetes', 'deploy'], ts: daysAgo(9),
|
|
35
|
+
},
|
|
36
|
+
{
|
|
37
|
+
topic: 'deploy:docker:image-too-large',
|
|
38
|
+
outcome: 'success',
|
|
39
|
+
what_worked: 'Reduced Docker image size dramatically with a multi-stage build and a slim base image, moving build-only deps out of the final layer.',
|
|
40
|
+
context: 'CI pushes were slow due to a 1.8GB image.',
|
|
41
|
+
severity: 'minor', confidence: 0.9, recall_count: 6,
|
|
42
|
+
tags: ['docker', 'deploy', 'optimization'], ts: daysAgo(20),
|
|
43
|
+
},
|
|
44
|
+
// ── Database domain ────────────────────────────────────────────────────────
|
|
45
|
+
{
|
|
46
|
+
topic: 'db:postgres:connection-pool-exhausted',
|
|
47
|
+
outcome: 'success',
|
|
48
|
+
what_worked: 'Added PgBouncer in transaction pooling mode and lowered the per-service max pool size so backends were shared. Peak traffic stopped exhausting backends.',
|
|
49
|
+
context: 'API returned 500s at peak traffic.',
|
|
50
|
+
severity: 'critical', confidence: 0.92, recall_count: 9,
|
|
51
|
+
tags: ['postgres', 'database', 'pool'], ts: daysAgo(15),
|
|
52
|
+
},
|
|
53
|
+
{
|
|
54
|
+
topic: 'db:postgres:slow-query-missing-index',
|
|
55
|
+
outcome: 'success',
|
|
56
|
+
what_worked: 'A slow Postgres query was fixed by adding a composite index on (tenant_id, created_at); query dropped from 4s to 12ms.',
|
|
57
|
+
context: 'Dashboard timed out loading recent events.',
|
|
58
|
+
severity: 'major', confidence: 0.88, recall_count: 4,
|
|
59
|
+
tags: ['postgres', 'database', 'index', 'performance'], ts: daysAgo(30),
|
|
60
|
+
},
|
|
61
|
+
{
|
|
62
|
+
topic: 'db:postgres:pool-raise-limit-attempt',
|
|
63
|
+
outcome: 'failure',
|
|
64
|
+
what_worked: '',
|
|
65
|
+
what_failed: 'Postgres had too many connections under load, so I raised max_connections. Postgres still had too many connections under load and the load made it worse — too many connections again under load.',
|
|
66
|
+
context: 'Postgres too many connections under load.',
|
|
67
|
+
severity: 'major', confidence: 0.25, recall_count: 0,
|
|
68
|
+
tags: ['postgres', 'database', 'pool'], ts: daysAgo(16),
|
|
69
|
+
},
|
|
70
|
+
// ── Auth domain ────────────────────────────────────────────────────────────
|
|
71
|
+
{
|
|
72
|
+
topic: 'auth:jwt:expired-on-restart',
|
|
73
|
+
outcome: 'success',
|
|
74
|
+
what_worked: 'JWT was lost on server restart because it was only kept in memory; persisting the token to disk after the device flow fixed recall counts staying at zero.',
|
|
75
|
+
context: 'Telemetry showed zero authenticated calls after restarts.',
|
|
76
|
+
severity: 'critical', confidence: 0.93, recall_count: 7,
|
|
77
|
+
tags: ['jwt', 'auth', 'persistence'], ts: daysAgo(3),
|
|
78
|
+
},
|
|
79
|
+
{
|
|
80
|
+
topic: 'auth:pkce:keycloak-rejects-exchange',
|
|
81
|
+
outcome: 'success',
|
|
82
|
+
what_worked: 'Keycloak rejected the PKCE code exchange because the verifier used non-URL-safe Base64; switching to RFC 7636 Base64URL random bytes fixed the login.',
|
|
83
|
+
context: 'Mobile login failed at the token exchange step.',
|
|
84
|
+
severity: 'major', confidence: 0.9, recall_count: 3,
|
|
85
|
+
tags: ['pkce', 'auth', 'keycloak', 'oauth'], ts: daysAgo(2),
|
|
86
|
+
},
|
|
87
|
+
// ── Redis / cache domain ───────────────────────────────────────────────────
|
|
88
|
+
{
|
|
89
|
+
topic: 'redis:noauth-on-every-call',
|
|
90
|
+
outcome: 'success',
|
|
91
|
+
what_worked: 'The connection endpoint omitted the password field, so the client connected unauthenticated; returning the credential explicitly in the response let the client authenticate and the calls succeeded.',
|
|
92
|
+
context: 'Hosted Valkey requires a credential.',
|
|
93
|
+
severity: 'critical', confidence: 0.94, recall_count: 11,
|
|
94
|
+
tags: ['redis', 'valkey', 'auth'], ts: daysAgo(4),
|
|
95
|
+
},
|
|
96
|
+
{
|
|
97
|
+
topic: 'redis:noauth-disable-auth-attempt',
|
|
98
|
+
outcome: 'failure',
|
|
99
|
+
what_worked: '',
|
|
100
|
+
what_failed: 'Redis threw NOAUTH on every call, so I tried disabling auth on Redis. Redis still threw NOAUTH on every call. Tried again to stop the NOAUTH error on every Redis call — no luck.',
|
|
101
|
+
context: 'Redis NOAUTH error on every call.',
|
|
102
|
+
severity: 'major', confidence: 0.2, recall_count: 0,
|
|
103
|
+
tags: ['redis', 'auth'], ts: daysAgo(5),
|
|
104
|
+
},
|
|
105
|
+
{
|
|
106
|
+
topic: 'redis:keys-no-ttl-memory-leak',
|
|
107
|
+
outcome: 'success',
|
|
108
|
+
what_worked: 'Redis memory grew unbounded because dependency-index and history keys had no TTL; adding a 90-day expire and an ltrim bounded the memory.',
|
|
109
|
+
context: 'Instance memory crept up over weeks of use.',
|
|
110
|
+
severity: 'major', confidence: 0.85, recall_count: 2,
|
|
111
|
+
tags: ['redis', 'ttl', 'memory'], ts: daysAgo(1),
|
|
112
|
+
},
|
|
113
|
+
// ── CI / build domain ──────────────────────────────────────────────────────
|
|
114
|
+
{
|
|
115
|
+
topic: 'ci:npm-ci-lockfile-mismatch',
|
|
116
|
+
outcome: 'success',
|
|
117
|
+
what_worked: 'CI "npm ci" failed because a dependency was added to package.json but the lockfile was never regenerated; running npm install to update package-lock.json fixed the build.',
|
|
118
|
+
context: 'GitHub Actions install step failed after adding a dep.',
|
|
119
|
+
severity: 'major', confidence: 0.91, recall_count: 5,
|
|
120
|
+
tags: ['ci', 'npm', 'lockfile'], ts: daysAgo(0),
|
|
121
|
+
},
|
|
122
|
+
{
|
|
123
|
+
topic: 'ci:golangci-lint-errcheck',
|
|
124
|
+
outcome: 'success',
|
|
125
|
+
what_worked: 'golangci-lint failed on an unchecked recover() return value; assigning it to _ satisfied errcheck.',
|
|
126
|
+
context: 'Go lint job blocked the deploy pipeline.',
|
|
127
|
+
severity: 'minor', confidence: 0.8, recall_count: 1,
|
|
128
|
+
tags: ['ci', 'go', 'lint'], ts: daysAgo(1),
|
|
129
|
+
},
|
|
130
|
+
// ── Frontend domain ────────────────────────────────────────────────────────
|
|
131
|
+
{
|
|
132
|
+
topic: 'frontend:cors-preflight-blocked',
|
|
133
|
+
outcome: 'success',
|
|
134
|
+
what_worked: 'Dashboard mutations failed because the CORS middleware did not allow PUT/PATCH; adding them to AllowMethods fixed the preflight.',
|
|
135
|
+
context: 'BYOK and webhook settings could not be saved from the browser.',
|
|
136
|
+
severity: 'major', confidence: 0.87, recall_count: 4,
|
|
137
|
+
tags: ['cors', 'frontend', 'api'], ts: daysAgo(6),
|
|
138
|
+
},
|
|
139
|
+
{
|
|
140
|
+
topic: 'frontend:hydration-mismatch',
|
|
141
|
+
outcome: 'partial',
|
|
142
|
+
what_worked: 'A Next.js hydration mismatch was reduced by guarding a date render with a client-only effect, though the root cause (locale formatting) still needs work.',
|
|
143
|
+
context: 'Console hydration warnings on the landing page.',
|
|
144
|
+
severity: 'minor', confidence: 0.6, recall_count: 1,
|
|
145
|
+
tags: ['nextjs', 'frontend', 'hydration'], ts: daysAgo(25),
|
|
146
|
+
},
|
|
147
|
+
];
|
|
148
|
+
export const BENCH_QUERIES = [
|
|
149
|
+
{
|
|
150
|
+
query: 'kubernetes deploy stuck rollout not finishing',
|
|
151
|
+
relevant: ['deploy:k8s:rollout-stuck'],
|
|
152
|
+
},
|
|
153
|
+
{
|
|
154
|
+
query: 'postgres too many connections under load',
|
|
155
|
+
relevant: ['db:postgres:connection-pool-exhausted'],
|
|
156
|
+
},
|
|
157
|
+
{
|
|
158
|
+
query: 'dashboard query is slow and times out',
|
|
159
|
+
relevant: ['db:postgres:slow-query-missing-index'],
|
|
160
|
+
},
|
|
161
|
+
{
|
|
162
|
+
query: 'jwt lost after restart recall count zero',
|
|
163
|
+
relevant: ['auth:jwt:expired-on-restart'],
|
|
164
|
+
},
|
|
165
|
+
{
|
|
166
|
+
query: 'keycloak rejects pkce token exchange on login',
|
|
167
|
+
relevant: ['auth:pkce:keycloak-rejects-exchange'],
|
|
168
|
+
},
|
|
169
|
+
{
|
|
170
|
+
query: 'redis noauth error on every call',
|
|
171
|
+
relevant: ['redis:noauth-on-every-call'],
|
|
172
|
+
},
|
|
173
|
+
{
|
|
174
|
+
query: 'redis memory keeps growing unbounded',
|
|
175
|
+
relevant: ['redis:keys-no-ttl-memory-leak'],
|
|
176
|
+
},
|
|
177
|
+
{
|
|
178
|
+
query: 'npm ci fails in github actions after adding dependency',
|
|
179
|
+
relevant: ['ci:npm-ci-lockfile-mismatch'],
|
|
180
|
+
},
|
|
181
|
+
{
|
|
182
|
+
query: 'cannot save settings cors preflight blocked',
|
|
183
|
+
relevant: ['frontend:cors-preflight-blocked'],
|
|
184
|
+
},
|
|
185
|
+
{
|
|
186
|
+
query: 'docker image too big slow ci push',
|
|
187
|
+
relevant: ['deploy:docker:image-too-large'],
|
|
188
|
+
},
|
|
189
|
+
{
|
|
190
|
+
query: 'go lint failing on unchecked return value',
|
|
191
|
+
relevant: ['ci:golangci-lint-errcheck'],
|
|
192
|
+
},
|
|
193
|
+
// Multi-relevant: both postgres performance lessons are reasonable answers.
|
|
194
|
+
{
|
|
195
|
+
query: 'postgres performance problem in production',
|
|
196
|
+
relevant: ['db:postgres:connection-pool-exhausted', 'db:postgres:slow-query-missing-index'],
|
|
197
|
+
},
|
|
198
|
+
];
|
|
199
|
+
//# sourceMappingURL=fixtures.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fixtures.js","sourceRoot":"","sources":["../../../src/bench/fixtures.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAsBH,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;AACvB,MAAM,OAAO,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,GAAG,GAAG,CAAC,GAAG,SAAS,CAAC,CAAC,WAAW,EAAE,CAAC;AAE3E,MAAM,CAAC,MAAM,aAAa,GAAkB;IAC1C,8EAA8E;IAC9E;QACE,KAAK,EAAE,0BAA0B;QACjC,OAAO,EAAE,SAAS;QAClB,WAAW,EAAE,uNAAuN;QACpO,OAAO,EAAE,0CAA0C;QACnD,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY,EAAE,EAAE;QACxD,IAAI,EAAE,CAAC,YAAY,EAAE,QAAQ,EAAE,OAAO,CAAC,EAAE,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC;KACxD;IACD;QACE,KAAK,EAAE,oCAAoC;QAC3C,OAAO,EAAE,SAAS;QAClB,WAAW,EAAE,EAAE;QACf,WAAW,EAAE,+NAA+N;QAC5O,OAAO,EAAE,iDAAiD;QAC1D,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,GAAG,EAAE,YAAY,EAAE,CAAC;QACnD,IAAI,EAAE,CAAC,YAAY,EAAE,QAAQ,CAAC,EAAE,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC;KAC/C;IACD;QACE,KAAK,EAAE,+BAA+B;QACtC,OAAO,EAAE,SAAS;QAClB,WAAW,EAAE,uIAAuI;QACpJ,OAAO,EAAE,2CAA2C;QACpD,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,GAAG,EAAE,YAAY,EAAE,CAAC;QACnD,IAAI,EAAE,CAAC,QAAQ,EAAE,QAAQ,EAAE,cAAc,CAAC,EAAE,EAAE,EAAE,OAAO,CAAC,EAAE,CAAC;KAC5D;IAED,8EAA8E;IAC9E;QACE,KAAK,EAAE,uCAAuC;QAC9C,OAAO,EAAE,SAAS;QAClB,WAAW,EAAE,0JAA0J;QACvK,OAAO,EAAE,oCAAoC;QAC7C,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC;QACvD,IAAI,EAAE,CAAC,UAAU,EAAE,UAAU,EAAE,MAAM,CAAC,EAAE,EAAE,EAAE,OAAO,CAAC,EAAE,CAAC;KACxD;IACD;QACE,KAAK,EAAE,sCAAsC;QAC7C,OAAO,EAAE,SAAS;QAClB,WAAW,EAAE,wHAAwH;QACrI,OAAO,EAAE,4CAA4C;QACrD,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC;QACpD,IAAI,EAAE,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE,aAAa,CAAC,EAAE,EAAE,EAAE,OAAO,CAAC,EAAE,CAAC;KACxE;IACD;QACE,KAAK,EAAE,sCAAsC;QAC7C,OAAO,EAAE,SAAS;QAClB,WAAW,EAAE,EAAE;QACf,WAAW,EAAE,mMAAmM;QAChN,OAAO,EAAE,2CAA2C;QACpD,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC;QACpD,IAAI,EAAE,CAAC,UAAU,EAAE,UAAU,EAAE,MAAM,CAAC,EAAE,EAAE,EAAE,OAAO,CAAC,EAAE,CAAC;KACxD;IAED,8EAA8E;IAC9E;QACE,KAAK,EAAE,6BAA6B;QACpC,OAAO,EAAE,SAAS;QAClB,WAAW,EAAE,4JAA4J;QACzK,OAAO,EAAE,2DAA2D;QACpE,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC;QACvD,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,aAAa,CAAC,EAAE,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC;KACrD;IACD;QACE,KAAK,EAAE,qCAAqC;QAC5C,OAAO,EAAE,SAAS;QAClB,WAAW,EAAE,uJAAuJ;QACpK,OAAO,EAAE,iDAAiD;QAC1D,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,GAAG,EAAE,YAAY,EAAE,CAAC;QACnD,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,CAAC,EAAE,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC;KAC5D;IAED,8EAA8E;IAC9E;QACE,KAAK,EAAE,4BAA4B;QACnC,OAAO,EAAE,SAAS;QAClB,WAAW,EAAE,uMAAuM;QACpN,OAAO,EAAE,sCAAsC;QAC/C,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY,EAAE,EAAE;QACxD,IAAI,EAAE,CAAC,OAAO,EAAE,QAAQ,EAAE,MAAM,CAAC,EAAE,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC;KAClD;IACD;QACE,KAAK,EAAE,mCAAmC;QAC1C,OAAO,EAAE,SAAS;QAClB,WAAW,EAAE,EAAE;QACf,WAAW,EAAE,mLAAmL;QAChM,OAAO,EAAE,mCAAmC;QAC5C,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,GAAG,EAAE,YAAY,EAAE,CAAC;QACnD,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,EAAE,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC;KACxC;IACD;QACE,KAAK,EAAE,+BAA+B;QACtC,OAAO,EAAE,SAAS;QAClB,WAAW,EAAE,2IAA2I;QACxJ,OAAO,EAAE,6CAA6C;QACtD,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC;QACpD,IAAI,EAAE,CAAC,OAAO,EAAE,KAAK,EAAE,QAAQ,CAAC,EAAE,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC;KACjD;IAED,8EAA8E;IAC9E;QACE,KAAK,EAAE,6BAA6B;QACpC,OAAO,EAAE,SAAS;QAClB,WAAW,EAAE,4KAA4K;QACzL,OAAO,EAAE,wDAAwD;QACjE,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC;QACpD,IAAI,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,UAAU,CAAC,EAAE,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC;KAChD;IACD;QACE,KAAK,EAAE,2BAA2B;QAClC,OAAO,EAAE,SAAS;QAClB,WAAW,EAAE,oGAAoG;QACjH,OAAO,EAAE,0CAA0C;QACnD,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,GAAG,EAAE,YAAY,EAAE,CAAC;QACnD,IAAI,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,EAAE,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC;KAC3C;IAED,8EAA8E;IAC9E;QACE,KAAK,EAAE,iCAAiC;QACxC,OAAO,EAAE,SAAS;QAClB,WAAW,EAAE,kIAAkI;QAC/I,OAAO,EAAE,gEAAgE;QACzE,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC;QACpD,IAAI,EAAE,CAAC,MAAM,EAAE,UAAU,EAAE,KAAK,CAAC,EAAE,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC;KAClD;IACD;QACE,KAAK,EAAE,6BAA6B;QACpC,OAAO,EAAE,SAAS;QAClB,WAAW,EAAE,2JAA2J;QACxK,OAAO,EAAE,iDAAiD;QAC1D,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,GAAG,EAAE,YAAY,EAAE,CAAC;QACnD,IAAI,EAAE,CAAC,QAAQ,EAAE,UAAU,EAAE,WAAW,CAAC,EAAE,EAAE,EAAE,OAAO,CAAC,EAAE,CAAC;KAC3D;CACF,CAAC;AAEF,MAAM,CAAC,MAAM,aAAa,GAAiB;IACzC;QACE,KAAK,EAAE,+CAA+C;QACtD,QAAQ,EAAE,CAAC,0BAA0B,CAAC;KACvC;IACD;QACE,KAAK,EAAE,0CAA0C;QACjD,QAAQ,EAAE,CAAC,uCAAuC,CAAC;KACpD;IACD;QACE,KAAK,EAAE,uCAAuC;QAC9C,QAAQ,EAAE,CAAC,sCAAsC,CAAC;KACnD;IACD;QACE,KAAK,EAAE,0CAA0C;QACjD,QAAQ,EAAE,CAAC,6BAA6B,CAAC;KAC1C;IACD;QACE,KAAK,EAAE,+CAA+C;QACtD,QAAQ,EAAE,CAAC,qCAAqC,CAAC;KAClD;IACD;QACE,KAAK,EAAE,kCAAkC;QACzC,QAAQ,EAAE,CAAC,4BAA4B,CAAC;KACzC;IACD;QACE,KAAK,EAAE,sCAAsC;QAC7C,QAAQ,EAAE,CAAC,+BAA+B,CAAC;KAC5C;IACD;QACE,KAAK,EAAE,wDAAwD;QAC/D,QAAQ,EAAE,CAAC,6BAA6B,CAAC;KAC1C;IACD;QACE,KAAK,EAAE,6CAA6C;QACpD,QAAQ,EAAE,CAAC,iCAAiC,CAAC;KAC9C;IACD;QACE,KAAK,EAAE,mCAAmC;QAC1C,QAAQ,EAAE,CAAC,+BAA+B,CAAC;KAC5C;IACD;QACE,KAAK,EAAE,2CAA2C;QAClD,QAAQ,EAAE,CAAC,2BAA2B,CAAC;KACxC;IACD,4EAA4E;IAC5E;QACE,KAAK,EAAE,4CAA4C;QACnD,QAAQ,EAAE,CAAC,uCAAuC,EAAE,sCAAsC,CAAC;KAC5F;CACF,CAAC"}
|
|
@@ -3,20 +3,26 @@ import { calculateConfidence, confidenceBadge, STRUCTURED_TEMPLATES, CONFIDENCE_
|
|
|
3
3
|
import { ckgSlug, extractProblemConcept, ckgUpsertNode, ckgUpdateEdge } from '../ckg.js';
|
|
4
4
|
import { safeJsonParse } from '../utils.js';
|
|
5
5
|
import { keywordSearch, tokenize, splitMultiQuery, levenshtein, indexVocab as _indexVocab } from '../search.js';
|
|
6
|
+
import { rerankByQuality } from '../rerank.js';
|
|
6
7
|
import { computeEmbedding, hasEmbedProvider } from '../embeddings.js';
|
|
7
8
|
// ── Changelog (shown once per version in session_start) ──────────────────────
|
|
8
|
-
const MCP_VERSION = '0.10.
|
|
9
|
+
const MCP_VERSION = '0.10.50';
|
|
9
10
|
const WHATS_NEW = {
|
|
10
|
-
'0.10.
|
|
11
|
+
'0.10.50': [
|
|
11
12
|
`🆕 **What's new in v${MCP_VERSION}:**`,
|
|
13
|
+
` ✅ \`smart_recall\` — unified keyword + semantic hybrid list (one ranked result, not two sections)`,
|
|
14
|
+
` ✅ \`smart_recall\` — "Brain saved you here" banner surfaces time saved inline per proven lesson`,
|
|
15
|
+
` ✅ Quality reranking — proven successes outrank symptom-dense failures (+11% Precision@1)`,
|
|
16
|
+
` ✅ Cachly-Bench — IR benchmark with CI regression guard (see BENCH.md for the proof)`,
|
|
17
|
+
` 🔧 Contradiction resolution now persisted to Redis (90-day TTL)`,
|
|
18
|
+
` 💡 Run \`smart_recall\` to see your Brain's recall quality in action`,
|
|
19
|
+
],
|
|
20
|
+
'0.10.49': [
|
|
12
21
|
` ✅ \`brain_from_git\` — auto-seed your Brain from git history in seconds`,
|
|
13
22
|
` ✅ \`brain_predict_failures\` — predict CI/build failures before they happen`,
|
|
14
|
-
` ✅ \`smart_recall\` — semantic + keyword hybrid recall, now in every CLAUDE.md`,
|
|
15
23
|
` ✅ VS Code extension — Brain status bar, ambient learning, CodeLens hints`,
|
|
16
24
|
` ✅ Team Brain — share lessons across your whole engineering team`,
|
|
17
|
-
` ✅ 89 MCP tools — roadmap, A/B tests, cache, index, predict, syndicate`,
|
|
18
25
|
` 🔧 Stability — auto-expiring memory keys, longer provisioning wait, robust recall`,
|
|
19
|
-
` 💡 Run \`brain_from_git\` to seed your Brain from existing commits instantly`,
|
|
20
26
|
],
|
|
21
27
|
};
|
|
22
28
|
export const BRAIN_TOOL_NAMES = new Set([
|
|
@@ -72,6 +78,19 @@ export async function handleBrainTool(name, args, getConnection, apiFetch) {
|
|
|
72
78
|
await ckgUpdateEdge(redis, cId, 'contradicts', resId, false);
|
|
73
79
|
}
|
|
74
80
|
catch { /* non-critical */ }
|
|
81
|
+
// Persist a queryable resolution record — so contradiction history survives
|
|
82
|
+
// (CKG edges alone aren't easily auditable). Bounded + TTL'd like other history.
|
|
83
|
+
try {
|
|
84
|
+
const resolutionKey = `cachly:contradictions:${topic}`;
|
|
85
|
+
const direction = prev.outcome === 'success' && outcome === 'failure'
|
|
86
|
+
? 'kept-success' : 'overwrote-failure';
|
|
87
|
+
await redis.rpush(resolutionKey, JSON.stringify({
|
|
88
|
+
ts, prev_outcome: prev.outcome, new_outcome: outcome, direction,
|
|
89
|
+
}));
|
|
90
|
+
await redis.ltrim(resolutionKey, -50, -1);
|
|
91
|
+
await redis.expire(resolutionKey, 180 * 86400);
|
|
92
|
+
}
|
|
93
|
+
catch { /* non-critical */ }
|
|
75
94
|
contradictionWarning.push(`🗳️ Run \`madc_deliberate(topic="${topic}")\` to resolve via expert agent voting.`);
|
|
76
95
|
}
|
|
77
96
|
}
|
|
@@ -411,9 +430,11 @@ export async function handleBrainTool(name, args, getConnection, apiFetch) {
|
|
|
411
430
|
const { instance_id, query, threshold = 0.78, } = args;
|
|
412
431
|
const redis = await getConnection(instance_id);
|
|
413
432
|
// ── Layer 1: Keyword search across ALL brain data (always works, no embedding) ──
|
|
414
|
-
const
|
|
415
|
-
//
|
|
416
|
-
//
|
|
433
|
+
const rawMatches = await keywordSearch(redis, ['cachly:ctx:*', 'cachly:lesson:best:*', 'cachly:idx:*'], query, 10);
|
|
434
|
+
// ── Layer 1.5: Quality-aware rerank — proven success lessons outrank
|
|
435
|
+
// text-similar failed attempts (the moat; see src/rerank.ts + Cachly-Bench). ──
|
|
436
|
+
const kwMatches = rerankByQuality(rawMatches);
|
|
437
|
+
const savedHere = [];
|
|
417
438
|
const lessonMatches = kwMatches.filter(m => m.key.startsWith('cachly:lesson:best:'));
|
|
418
439
|
for (const m of lessonMatches.slice(0, 5)) {
|
|
419
440
|
const existing = await redis.get(m.key).catch(() => null);
|
|
@@ -422,41 +443,118 @@ export async function handleBrainTool(name, args, getConnection, apiFetch) {
|
|
|
422
443
|
if (lesson) {
|
|
423
444
|
const updated = { ...lesson, recall_count: (lesson.recall_count ?? 0) + 1, verified_at: new Date().toISOString() };
|
|
424
445
|
redis.set(m.key, JSON.stringify(updated)).catch(() => { });
|
|
425
|
-
const
|
|
446
|
+
const sev = lesson.severity;
|
|
447
|
+
const savedMins = sev === 'critical' ? 240 : sev === 'major' ? 60 : 30;
|
|
426
448
|
redis.incrbyfloat(`cachly:stats:time_saved_mins:${instance_id}`, savedMins).catch(() => { });
|
|
449
|
+
// Surface banner for proven successes (recall_count >= 1 means it's been validated)
|
|
450
|
+
if (lesson.outcome === 'success' && (lesson.recall_count ?? 0) >= 1) {
|
|
451
|
+
savedHere.push({
|
|
452
|
+
topic: m.key.replace('cachly:lesson:best:', ''),
|
|
453
|
+
severity: sev ?? 'major',
|
|
454
|
+
recall_count: (lesson.recall_count ?? 0) + 1,
|
|
455
|
+
savedMins,
|
|
456
|
+
});
|
|
457
|
+
}
|
|
458
|
+
}
|
|
459
|
+
}
|
|
460
|
+
}
|
|
461
|
+
// ── Layer 2: Semantic search (parallel, optional) ────────────────────────
|
|
462
|
+
const inst = await apiFetch(`/api/v1/instances/${instance_id}`);
|
|
463
|
+
const semHits = [];
|
|
464
|
+
if (inst.vector_token && hasEmbedProvider()) {
|
|
465
|
+
try {
|
|
466
|
+
const embedding = await computeEmbedding(query);
|
|
467
|
+
const vectorUrl = `https://api.cachly.dev/v1/sem/${inst.vector_token}`;
|
|
468
|
+
const searchRes = await fetch(`${vectorUrl}/search`, {
|
|
469
|
+
method: 'POST',
|
|
470
|
+
headers: { 'Content-Type': 'application/json' },
|
|
471
|
+
body: JSON.stringify({ embedding, namespace: 'cachly:ctx', threshold, top_k: 5 }),
|
|
472
|
+
});
|
|
473
|
+
if (searchRes.ok) {
|
|
474
|
+
const results = (await searchRes.json());
|
|
475
|
+
for (const hit of results.filter(r => r.found && r.id)) {
|
|
476
|
+
const parts = hit.id.replace('ctx:', '').split(':');
|
|
477
|
+
const category = parts[0];
|
|
478
|
+
const key = parts.slice(1).join(':');
|
|
479
|
+
const redisKey = `cachly:ctx:${category}:${key}`;
|
|
480
|
+
const content = await redis.get(redisKey).catch(() => null);
|
|
481
|
+
semHits.push({ key: redisKey, similarity: hit.similarity ?? 0, content: content ?? '(evicted)' });
|
|
482
|
+
}
|
|
427
483
|
}
|
|
428
484
|
}
|
|
485
|
+
catch {
|
|
486
|
+
// Semantic search failed silently — keyword results are enough
|
|
487
|
+
}
|
|
488
|
+
}
|
|
489
|
+
const bm25Scores = kwMatches.map(m => m.score);
|
|
490
|
+
const bm25Min = bm25Scores.length ? Math.min(...bm25Scores) : 0;
|
|
491
|
+
const bm25Range = bm25Scores.length ? (Math.max(...bm25Scores) - bm25Min) || 1 : 1;
|
|
492
|
+
const bm25Norm = (s) => (s - bm25Min) / bm25Range;
|
|
493
|
+
const hybridMap = new Map();
|
|
494
|
+
for (const m of kwMatches) {
|
|
495
|
+
const n = bm25Norm(m.score);
|
|
496
|
+
hybridMap.set(m.key, {
|
|
497
|
+
key: m.key, content: m.content, bm25Score: n,
|
|
498
|
+
hybridScore: n * (semHits.length > 0 ? 0.7 : 1.0),
|
|
499
|
+
matchedWords: m.matchedWords, matchType: 'keyword', subQuery: m.subQuery,
|
|
500
|
+
});
|
|
501
|
+
}
|
|
502
|
+
for (const hit of semHits) {
|
|
503
|
+
const existing = hybridMap.get(hit.key);
|
|
504
|
+
if (existing) {
|
|
505
|
+
existing.semScore = hit.similarity;
|
|
506
|
+
existing.hybridScore = (existing.bm25Score ?? 0) * 0.6 + hit.similarity * 0.4;
|
|
507
|
+
existing.matchType = 'hybrid';
|
|
508
|
+
}
|
|
509
|
+
else {
|
|
510
|
+
hybridMap.set(hit.key, {
|
|
511
|
+
key: hit.key, content: hit.content, semScore: hit.similarity,
|
|
512
|
+
hybridScore: hit.similarity * 0.5, matchType: 'semantic',
|
|
513
|
+
});
|
|
514
|
+
}
|
|
429
515
|
}
|
|
516
|
+
const hybridResults = [...hybridMap.values()].sort((a, b) => b.hybridScore - a.hybridScore);
|
|
517
|
+
// ── Build output ──────────────────────────────────────────────────────────
|
|
430
518
|
const lines = [`🧠 **Smart Recall** for: _"${query}"_\n`];
|
|
519
|
+
// "Brain saved you here" banner — surfaces value inline for proven recalled lessons
|
|
520
|
+
const SORDER = { critical: 0, major: 1, minor: 2 };
|
|
521
|
+
const topLesson = savedHere.sort((a, b) => (SORDER[a.severity] ?? 1) - (SORDER[b.severity] ?? 1))[0];
|
|
522
|
+
if (topLesson) {
|
|
523
|
+
lines.push(`> 💡 **Brain saved you ~${topLesson.savedMins}m here** — ${topLesson.severity} issue from \`${topLesson.topic}\` recalled ${topLesson.recall_count}× proven\n`);
|
|
524
|
+
}
|
|
431
525
|
// Show sub-query info if multi-topic was detected
|
|
432
526
|
const subQueries = splitMultiQuery(query);
|
|
433
527
|
if (subQueries.length > 1) {
|
|
434
528
|
lines.push(`_Detected ${subQueries.length} sub-topics:_ ${subQueries.map((s, i) => `${i + 1}. "${s}"`).join(', ')}\n`);
|
|
435
529
|
}
|
|
436
|
-
if (
|
|
437
|
-
|
|
438
|
-
|
|
530
|
+
if (hybridResults.length > 0) {
|
|
531
|
+
const modeLabel = semHits.length > 0
|
|
532
|
+
? `keyword + semantic hybrid`
|
|
533
|
+
: `keyword`;
|
|
534
|
+
lines.push(`### 🔍 Results (${hybridResults.length} — ${modeLabel})\n`);
|
|
535
|
+
// Group by sub-query for multi-topic queries
|
|
439
536
|
if (subQueries.length > 1) {
|
|
440
537
|
const grouped = new Map();
|
|
441
|
-
for (const
|
|
442
|
-
const sq =
|
|
538
|
+
for (const r of hybridResults.slice(0, 12)) {
|
|
539
|
+
const sq = r.subQuery ?? query;
|
|
443
540
|
if (!grouped.has(sq))
|
|
444
541
|
grouped.set(sq, []);
|
|
445
|
-
grouped.get(sq).push(
|
|
542
|
+
grouped.get(sq).push(r);
|
|
446
543
|
}
|
|
447
|
-
for (const [sq,
|
|
448
|
-
lines.push(`**Topic: "${sq}"** (${
|
|
449
|
-
for (const
|
|
450
|
-
const label =
|
|
451
|
-
|
|
452
|
-
.
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
544
|
+
for (const [sq, results] of grouped) {
|
|
545
|
+
lines.push(`**Topic: "${sq}"** (${results.length} results)\n`);
|
|
546
|
+
for (const r of results.slice(0, 4)) {
|
|
547
|
+
const label = r.key.replace('cachly:ctx:', '📝 ').replace('cachly:lesson:best:', '💡 ').replace('cachly:idx:', '📂 ');
|
|
548
|
+
const scorePart = r.matchType === 'hybrid'
|
|
549
|
+
? `BM25: ${(r.bm25Score ?? 0).toFixed(2)}, sem: ${((r.semScore ?? 0) * 100).toFixed(0)}%, 🔀 hybrid`
|
|
550
|
+
: r.matchType === 'semantic'
|
|
551
|
+
? `sem: ${((r.semScore ?? 0) * 100).toFixed(0)}%, 🎯 semantic`
|
|
552
|
+
: `BM25: ${(r.bm25Score ?? 0).toFixed(2)}, matched: ${r.matchedWords?.join(', ')}`;
|
|
553
|
+
const preview = r.content.slice(0, 300).replace(/\n/g, ' ');
|
|
554
|
+
lines.push(` **${label}** _(${scorePart})_`);
|
|
555
|
+
lines.push(` > ${preview}${r.content.length > 300 ? '…' : ''}\n`);
|
|
457
556
|
}
|
|
458
557
|
}
|
|
459
|
-
// Summary: which sub-queries had matches
|
|
460
558
|
const matched = [...grouped.keys()];
|
|
461
559
|
const unmatched = subQueries.filter(sq => !matched.includes(sq));
|
|
462
560
|
if (unmatched.length > 0) {
|
|
@@ -464,48 +562,20 @@ export async function handleBrainTool(name, args, getConnection, apiFetch) {
|
|
|
464
562
|
}
|
|
465
563
|
}
|
|
466
564
|
else {
|
|
467
|
-
for (const
|
|
468
|
-
const label =
|
|
469
|
-
|
|
470
|
-
.
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
}
|
|
478
|
-
// ── Layer 2: Semantic search (optional, only if embedding provider + vector_token available) ──
|
|
479
|
-
const inst = await apiFetch(`/api/v1/instances/${instance_id}`);
|
|
480
|
-
if (inst.vector_token && hasEmbedProvider()) {
|
|
481
|
-
try {
|
|
482
|
-
const embedding = await computeEmbedding(query);
|
|
483
|
-
const vectorUrl = `https://api.cachly.dev/v1/sem/${inst.vector_token}`;
|
|
484
|
-
const searchRes = await fetch(`${vectorUrl}/search`, {
|
|
485
|
-
method: 'POST',
|
|
486
|
-
headers: { 'Content-Type': 'application/json' },
|
|
487
|
-
body: JSON.stringify({ embedding, namespace: 'cachly:ctx', threshold, top_k: 5 }),
|
|
488
|
-
});
|
|
489
|
-
if (searchRes.ok) {
|
|
490
|
-
const results = (await searchRes.json());
|
|
491
|
-
const semHits = results.filter(r => r.found && r.id);
|
|
492
|
-
if (semHits.length > 0) {
|
|
493
|
-
lines.push(`\n### 🎯 Semantic Matches (${semHits.length})\n`);
|
|
494
|
-
for (const hit of semHits) {
|
|
495
|
-
const parts = hit.id.replace('ctx:', '').split(':');
|
|
496
|
-
const category = parts[0];
|
|
497
|
-
const key = parts.slice(1).join(':');
|
|
498
|
-
const content = await redis.get(`cachly:ctx:${category}:${key}`);
|
|
499
|
-
lines.push(`**${key}** _(${((hit.similarity ?? 0) * 100).toFixed(0)}% similar)_`, `> ${content?.slice(0, 300) ?? '(evicted)'}${(content?.length ?? 0) > 300 ? '…' : ''}\n`);
|
|
500
|
-
}
|
|
501
|
-
}
|
|
565
|
+
for (const r of hybridResults.slice(0, 8)) {
|
|
566
|
+
const label = r.key.replace('cachly:ctx:', '📝 ').replace('cachly:lesson:best:', '💡 ').replace('cachly:idx:', '📂 ');
|
|
567
|
+
const scorePart = r.matchType === 'hybrid'
|
|
568
|
+
? `BM25: ${(r.bm25Score ?? 0).toFixed(2)}, sem: ${((r.semScore ?? 0) * 100).toFixed(0)}%, 🔀 hybrid`
|
|
569
|
+
: r.matchType === 'semantic'
|
|
570
|
+
? `sem: ${((r.semScore ?? 0) * 100).toFixed(0)}%, 🎯 semantic`
|
|
571
|
+
: `BM25: ${(r.bm25Score ?? 0).toFixed(2)}, matched: ${r.matchedWords?.join(', ')}`;
|
|
572
|
+
const preview = r.content.slice(0, 400).replace(/\n/g, ' ');
|
|
573
|
+
lines.push(`**${label}** _(${scorePart})_`);
|
|
574
|
+
lines.push(`> ${preview}${r.content.length > 400 ? '…' : ''}\n`);
|
|
502
575
|
}
|
|
503
576
|
}
|
|
504
|
-
catch {
|
|
505
|
-
// Semantic search failed silently — keyword results are enough
|
|
506
|
-
}
|
|
507
577
|
}
|
|
508
|
-
|
|
578
|
+
else {
|
|
509
579
|
lines.push(`⚠️ No matches found for: "${query}"`);
|
|
510
580
|
// Did-You-Mean: find nearest token in index vocab
|
|
511
581
|
const queryTokens = tokenize(query);
|