@cachly-dev/mcp-server 0.10.49 → 0.10.51
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/BENCH.md +80 -0
- package/STRATEGY.md +33 -14
- package/apps/travel-chaos-organizer/frontend/package-lock.json +184 -0
- package/dist/src/__tests__/rerank.test.d.ts +1 -0
- package/dist/src/__tests__/rerank.test.js +115 -0
- package/dist/src/__tests__/rerank.test.js.map +1 -0
- package/dist/src/bench/cachly-bench.d.ts +35 -0
- package/dist/src/bench/cachly-bench.js +182 -0
- package/dist/src/bench/cachly-bench.js.map +1 -0
- package/dist/src/bench/fixtures.d.ts +34 -0
- package/dist/src/bench/fixtures.js +199 -0
- package/dist/src/bench/fixtures.js.map +1 -0
- package/dist/src/handlers/brain.js +141 -65
- package/dist/src/handlers/brain.js.map +1 -1
- package/dist/src/handlers/team.js +56 -4
- package/dist/src/handlers/team.js.map +1 -1
- package/dist/src/rerank.d.ts +55 -0
- package/dist/src/rerank.js +111 -0
- package/dist/src/rerank.js.map +1 -0
- package/dist/src/tools.d.ts +30 -0
- package/dist/src/tools.js +18 -0
- package/dist/src/tools.js.map +1 -1
- package/package.json +2 -1
- package/src/__tests__/rerank.test.ts +130 -0
- package/src/bench/cachly-bench.ts +217 -0
- package/src/bench/fixtures.ts +226 -0
- package/src/handlers/brain.ts +156 -73
- package/src/handlers/team.ts +66 -4
- package/src/rerank.ts +148 -0
- package/src/tools.ts +19 -0
package/BENCH.md
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
# Cachly-Bench — the recall-quality proof
|
|
2
|
+
|
|
3
|
+
> The STRATEGY.md moat claim is: *Cachly's structured, quality-aware recall beats
|
|
4
|
+
> raw text matching (and a flat-file memory) at surfacing the lesson that actually
|
|
5
|
+
> helps.* A claim without a number is marketing. This is the number.
|
|
6
|
+
|
|
7
|
+
## Run it
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
npm run bench # pretty report
|
|
11
|
+
npm run bench -- --json # machine-readable
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
The benchmark is also a CI regression guard: `src/__tests__/rerank.test.ts`
|
|
15
|
+
asserts that quality reranking never regresses the headline metrics and that the
|
|
16
|
+
baseline is already strong (so the corpus isn't rigged).
|
|
17
|
+
|
|
18
|
+
## What it measures
|
|
19
|
+
|
|
20
|
+
Two rankers are compared over the **same** BM25 candidate set, so the comparison
|
|
21
|
+
isolates *ranking quality*, not retrieval:
|
|
22
|
+
|
|
23
|
+
| Ranker | What it does | Analogy |
|
|
24
|
+
|---|---|---|
|
|
25
|
+
| `baseline` | raw BM25+ keyword score (`src/search.ts`) | a text-only / flat-file memory |
|
|
26
|
+
| `cachly` | BM25+ then quality-aware rerank (`src/rerank.ts`) | structured, trust-aware recall |
|
|
27
|
+
|
|
28
|
+
Quality reranking boosts a lesson by its **outcome** (proven success > partial >
|
|
29
|
+
failed attempt), **confidence**, **proven-ness** (`recall_count`, log-saturated)
|
|
30
|
+
and **severity** — clamped so a strong text match is never fully buried, and a
|
|
31
|
+
weak one never catapulted on quality alone.
|
|
32
|
+
|
|
33
|
+
## The corpus is fair (not rigged)
|
|
34
|
+
|
|
35
|
+
The fixture corpus (`src/bench/fixtures.ts`) is deliberately and *fairly*
|
|
36
|
+
adversarial: for several queries a **symptom-dense failed attempt** competes with
|
|
37
|
+
the **solution-focused proven success**. This mirrors reality — failed attempts
|
|
38
|
+
repeat the symptom verbatim ("still stuck, still failing"), while the lesson that
|
|
39
|
+
fixed it is written in solution vocabulary. BM25 alone is fooled by the symptom
|
|
40
|
+
density; quality reranking corrects it.
|
|
41
|
+
|
|
42
|
+
The baseline still scores well (MRR ~85%, Precision@1 75%) — we are not making a
|
|
43
|
+
weak baseline look bad. We are improving an already-good one on the cases that
|
|
44
|
+
matter.
|
|
45
|
+
|
|
46
|
+
## Current results
|
|
47
|
+
|
|
48
|
+
```
|
|
49
|
+
corpus: 15 lessons · queries: 12
|
|
50
|
+
|
|
51
|
+
metric baseline cachly lift
|
|
52
|
+
Precision@1 75.0% → 83.3% +11.1%
|
|
53
|
+
Precision@3 33.3% → 33.3% +0.0%
|
|
54
|
+
Recall@3 95.8% → 95.8% +0.0%
|
|
55
|
+
MRR 84.7% → 90.3% +6.6%
|
|
56
|
+
nDCG@5 87.3% → 91.7% +5.0%
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
**Headline: Precision@1 +11.1%, MRR +6.6%, nDCG@5 +5.0%.**
|
|
60
|
+
|
|
61
|
+
Interpretation: when the agent asks a question, the *first* result is the proven
|
|
62
|
+
fix 11% more often, and the proven fix sits higher in the list overall. On the
|
|
63
|
+
hardest cases (a failed attempt out-ranking the success on text alone), reranking
|
|
64
|
+
flips the order to the lesson that actually solves the problem.
|
|
65
|
+
|
|
66
|
+
## Honest limitations (read this)
|
|
67
|
+
|
|
68
|
+
- **Small corpus.** 15 lessons, 12 queries. This proves the *mechanism* works and
|
|
69
|
+
guards against regressions. It does **not** yet prove real-world magnitude. The
|
|
70
|
+
next step (STRATEGY Phase 2) is a larger, externally-sourced corpus and a
|
|
71
|
+
head-to-head against an actual flat-file memory baseline on real agent traces.
|
|
72
|
+
- **Ranking, not retrieval.** Both rankers see the same BM25 candidates, so this
|
|
73
|
+
measures the quality layer only. Semantic retrieval (embeddings) is evaluated
|
|
74
|
+
separately and is not in these numbers.
|
|
75
|
+
- **Labels are ours.** Gold relevance is author-assigned. A credible public
|
|
76
|
+
benchmark needs third-party labels.
|
|
77
|
+
|
|
78
|
+
The point of this file is not to claim victory. It is to make the moat
|
|
79
|
+
*measurable* and *defended by CI*, so every future change to recall must move this
|
|
80
|
+
number in the right direction — or it doesn't ship.
|
package/STRATEGY.md
CHANGED
|
@@ -126,28 +126,47 @@ Ohne das ist alles andere wertlos.
|
|
|
126
126
|
### Phase 2 — Beweisbar bessere Recall-Qualität (der Moat-Beweis) · *3–9 Monate*
|
|
127
127
|
Das ist der Punkt, an dem wir *verdienen*, gegen First-Party anzutreten.
|
|
128
128
|
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
- **[
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
129
|
+
> **Status (begonnen):** Erste Bausteine sind live — `npm run bench`
|
|
130
|
+
> (siehe [BENCH.md](./BENCH.md)) misst den Recall-Lift und ist CI-verteidigt.
|
|
131
|
+
> Aktuelle Zahl: **Precision@1 +11.1 %, MRR +6.6 %, nDCG@5 +5.0 %** gegenüber
|
|
132
|
+
> reinem BM25. Quality-Reranking (`src/rerank.ts`) ist in `smart_recall` verdrahtet,
|
|
133
|
+
> Contradiction-Resolution wird jetzt persistiert. Nächster Schritt: größerer,
|
|
134
|
+
> extern gelabelter Korpus + Head-to-head gegen ein echtes Flat-File-Memory.
|
|
135
|
+
|
|
136
|
+
- **[✅ erledigt]** **Quality-aware Reranking**: proven success-Lessons ranken über
|
|
137
|
+
text-ähnlichen Fehlversuchen (`src/rerank.ts`, in `smart_recall` aktiv).
|
|
138
|
+
- **[✅ erledigt]** **Cachly-Bench**: reproduzierbarer Benchmark mit IR-Metriken
|
|
139
|
+
(Precision@k, Recall@k, MRR, nDCG), als CI-Regressionswächter. Ohne Zahl keine
|
|
140
|
+
Story — jetzt gibt es eine Zahl, die jede Recall-Änderung verteidigen muss.
|
|
141
|
+
- **[✅ erledigt]** **Contradiction-Resolution persistent**: Widersprüche werden in
|
|
142
|
+
`cachly:contradictions:{topic}` als auditierbarer Verlauf gespeichert (TTL'd).
|
|
143
|
+
- **[✅ erledigt — 100x Intelligenz]** Hybrides Retrieval: BM25-Keyword + semantische
|
|
144
|
+
Embeddings werden in `smart_recall` zu *einer* gerankten Liste fusioniert
|
|
145
|
+
(`0.6×BM25-Qualität + 0.4×Semantik`), mit Match-Typ-Badges (🔍/🎯/🔀). CKG-
|
|
146
|
+
Traversierung als dritte Signalquelle ist der nächste Ausbau.
|
|
147
|
+
- **[✅ erledigt — Wertsichtbarkeit]** "Brain saved you here"-Banner im
|
|
148
|
+
`smart_recall`-Output macht die Zeitersparnis pro proven Lesson inline sichtbar
|
|
149
|
+
(`cachly:stats:time_saved_mins` wird getrackt und jetzt auch angezeigt).
|
|
150
|
+
- **[offen — Beweis]** Größerer, extern gelabelter Korpus + Head-to-head gegen ein
|
|
151
|
+
echtes Flat-File-Memory auf realen Agent-Traces (siehe BENCH.md "Limitations").
|
|
141
152
|
|
|
142
153
|
### Phase 3 — Team-Wissensgraph als uneinholbarer Moat · *9–18 Monate*
|
|
143
154
|
Hier ist Anthropic strukturell raus.
|
|
144
155
|
|
|
156
|
+
> **Status (begonnen):** Knowledge-Governance ist live — `team_confirm` lässt
|
|
157
|
+
> einen Reviewer eine Lesson bestätigen (🛡️ senior / ✔️ peer). Die Bestätigung
|
|
158
|
+
> fließt direkt in das Recall-Ranking (`src/rerank.ts`, 5. Faktor) ein: bestätigtes,
|
|
159
|
+
> menschlich geprüftes Wissen ranked über unreviewtem Auto-Learning. `team_recall`
|
|
160
|
+
> zeigt Reviewer-Badges und sortiert bestätigte Lessons zuerst. Distinct-Endorser
|
|
161
|
+
> werden in einem Set getrackt (kein Self-Inflation), mit Audit-Log pro Topic.
|
|
162
|
+
|
|
145
163
|
- **[100x Team]** Org-weiter Wissensgraph: Lessons, Personen, Dateien, Services als
|
|
146
164
|
Knoten; Recall berücksichtigt *wer* was gelernt hat und *wo* es relevant ist.
|
|
147
165
|
- **[Virality]** Jeder gelöste Fehler eines Entwicklers wird automatisch zur
|
|
148
166
|
Org-Lesson — mit Attribution. `team_learn` als Default, nicht als Opt-in.
|
|
149
|
-
- **[Governance]**
|
|
150
|
-
(
|
|
167
|
+
- **[✅ erledigt — Governance]** "Wissens-Reviews": ein Senior bestätigt eine Lesson
|
|
168
|
+
(`team_confirm`) → höheres Vertrauen im Recall-Ranking. Audit-Trail + distinct
|
|
169
|
+
Endorsements werden persistiert. Offen: Rollen-Modell und Sichtbarkeits-Scopes.
|
|
151
170
|
- **[Enterprise]** Self-Hosted-Tier mit SOC-2/ISO, BYOK by default, Daten bleiben
|
|
152
171
|
im Kunden-VPC. Das ist der Umsatz, den Anthropic-Memory nicht abgreifen kann.
|
|
153
172
|
|
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
"@react-native-async-storage/async-storage": "1.23.1",
|
|
12
12
|
"date-fns": "^4.0.0",
|
|
13
13
|
"expo": "~52.0.0",
|
|
14
|
+
"expo-asset": "~10.0.0",
|
|
14
15
|
"expo-auth-session": "~6.0.0",
|
|
15
16
|
"expo-crypto": "~14.0.0",
|
|
16
17
|
"expo-document-picker": "~12.0.0",
|
|
@@ -6562,6 +6563,189 @@
|
|
|
6562
6563
|
"expo": "*"
|
|
6563
6564
|
}
|
|
6564
6565
|
},
|
|
6566
|
+
"node_modules/expo-asset": {
|
|
6567
|
+
"version": "10.0.10",
|
|
6568
|
+
"resolved": "https://registry.npmjs.org/expo-asset/-/expo-asset-10.0.10.tgz",
|
|
6569
|
+
"integrity": "sha512-0qoTIihB79k+wGus9wy0JMKq7DdenziVx3iUkGvMAy2azscSgWH6bd2gJ9CGnhC6JRd3qTMFBL0ou/fx7WZl7A==",
|
|
6570
|
+
"license": "MIT",
|
|
6571
|
+
"dependencies": {
|
|
6572
|
+
"expo-constants": "~16.0.0",
|
|
6573
|
+
"invariant": "^2.2.4",
|
|
6574
|
+
"md5-file": "^3.2.3"
|
|
6575
|
+
},
|
|
6576
|
+
"peerDependencies": {
|
|
6577
|
+
"expo": "*"
|
|
6578
|
+
}
|
|
6579
|
+
},
|
|
6580
|
+
"node_modules/expo-asset/node_modules/@babel/code-frame": {
|
|
6581
|
+
"version": "7.10.4",
|
|
6582
|
+
"resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.10.4.tgz",
|
|
6583
|
+
"integrity": "sha512-vG6SvB6oYEhvgisZNFRmRCUkLz11c7rp+tbNTynGqc6mS1d5ATd/sGyV6W0KZZnXRKMTzZDRgQT3Ou9jhpAfUg==",
|
|
6584
|
+
"license": "MIT",
|
|
6585
|
+
"dependencies": {
|
|
6586
|
+
"@babel/highlight": "^7.10.4"
|
|
6587
|
+
}
|
|
6588
|
+
},
|
|
6589
|
+
"node_modules/expo-asset/node_modules/@expo/config": {
|
|
6590
|
+
"version": "9.0.4",
|
|
6591
|
+
"resolved": "https://registry.npmjs.org/@expo/config/-/config-9.0.4.tgz",
|
|
6592
|
+
"integrity": "sha512-g5ns5u1JSKudHYhjo1zaSfkJ/iZIcWmUmIQptMJZ6ag1C0ShL2sj8qdfU8MmAMuKLOgcIfSaiWlQnm4X3VJVkg==",
|
|
6593
|
+
"license": "MIT",
|
|
6594
|
+
"dependencies": {
|
|
6595
|
+
"@babel/code-frame": "~7.10.4",
|
|
6596
|
+
"@expo/config-plugins": "~8.0.8",
|
|
6597
|
+
"@expo/config-types": "^51.0.3",
|
|
6598
|
+
"@expo/json-file": "^8.3.0",
|
|
6599
|
+
"getenv": "^1.0.0",
|
|
6600
|
+
"glob": "7.1.6",
|
|
6601
|
+
"require-from-string": "^2.0.2",
|
|
6602
|
+
"resolve-from": "^5.0.0",
|
|
6603
|
+
"semver": "^7.6.0",
|
|
6604
|
+
"slugify": "^1.3.4",
|
|
6605
|
+
"sucrase": "3.34.0"
|
|
6606
|
+
}
|
|
6607
|
+
},
|
|
6608
|
+
"node_modules/expo-asset/node_modules/@expo/config-plugins": {
|
|
6609
|
+
"version": "8.0.11",
|
|
6610
|
+
"resolved": "https://registry.npmjs.org/@expo/config-plugins/-/config-plugins-8.0.11.tgz",
|
|
6611
|
+
"integrity": "sha512-oALE1HwnLFthrobAcC9ocnR9KXLzfWEjgIe4CPe+rDsfC6GDs8dGYCXfRFoCEzoLN4TGYs9RdZ8r0KoCcNrm2A==",
|
|
6612
|
+
"license": "MIT",
|
|
6613
|
+
"dependencies": {
|
|
6614
|
+
"@expo/config-types": "^51.0.3",
|
|
6615
|
+
"@expo/json-file": "~8.3.0",
|
|
6616
|
+
"@expo/plist": "^0.1.0",
|
|
6617
|
+
"@expo/sdk-runtime-versions": "^1.0.0",
|
|
6618
|
+
"chalk": "^4.1.2",
|
|
6619
|
+
"debug": "^4.3.1",
|
|
6620
|
+
"find-up": "~5.0.0",
|
|
6621
|
+
"getenv": "^1.0.0",
|
|
6622
|
+
"glob": "7.1.6",
|
|
6623
|
+
"resolve-from": "^5.0.0",
|
|
6624
|
+
"semver": "^7.5.4",
|
|
6625
|
+
"slash": "^3.0.0",
|
|
6626
|
+
"slugify": "^1.6.6",
|
|
6627
|
+
"xcode": "^3.0.1",
|
|
6628
|
+
"xml2js": "0.6.0"
|
|
6629
|
+
}
|
|
6630
|
+
},
|
|
6631
|
+
"node_modules/expo-asset/node_modules/@expo/config-types": {
|
|
6632
|
+
"version": "51.0.3",
|
|
6633
|
+
"resolved": "https://registry.npmjs.org/@expo/config-types/-/config-types-51.0.3.tgz",
|
|
6634
|
+
"integrity": "sha512-hMfuq++b8VySb+m9uNNrlpbvGxYc8OcFCUX9yTmi9tlx6A4k8SDabWFBgmnr4ao3wEArvWrtUQIfQCVtPRdpKA==",
|
|
6635
|
+
"license": "MIT"
|
|
6636
|
+
},
|
|
6637
|
+
"node_modules/expo-asset/node_modules/@expo/env": {
|
|
6638
|
+
"version": "0.3.0",
|
|
6639
|
+
"resolved": "https://registry.npmjs.org/@expo/env/-/env-0.3.0.tgz",
|
|
6640
|
+
"integrity": "sha512-OtB9XVHWaXidLbHvrVDeeXa09yvTl3+IQN884sO6PhIi2/StXfgSH/9zC7IvzrDB8kW3EBJ1PPLuCUJ2hxAT7Q==",
|
|
6641
|
+
"license": "MIT",
|
|
6642
|
+
"dependencies": {
|
|
6643
|
+
"chalk": "^4.0.0",
|
|
6644
|
+
"debug": "^4.3.4",
|
|
6645
|
+
"dotenv": "~16.4.5",
|
|
6646
|
+
"dotenv-expand": "~11.0.6",
|
|
6647
|
+
"getenv": "^1.0.0"
|
|
6648
|
+
}
|
|
6649
|
+
},
|
|
6650
|
+
"node_modules/expo-asset/node_modules/@expo/json-file": {
|
|
6651
|
+
"version": "8.3.3",
|
|
6652
|
+
"resolved": "https://registry.npmjs.org/@expo/json-file/-/json-file-8.3.3.tgz",
|
|
6653
|
+
"integrity": "sha512-eZ5dld9AD0PrVRiIWpRkm5aIoWBw3kAyd8VkuWEy92sEthBKDDDHAnK2a0dw0Eil6j7rK7lS/Qaq/Zzngv2h5A==",
|
|
6654
|
+
"license": "MIT",
|
|
6655
|
+
"dependencies": {
|
|
6656
|
+
"@babel/code-frame": "~7.10.4",
|
|
6657
|
+
"json5": "^2.2.2",
|
|
6658
|
+
"write-file-atomic": "^2.3.0"
|
|
6659
|
+
}
|
|
6660
|
+
},
|
|
6661
|
+
"node_modules/expo-asset/node_modules/@expo/plist": {
|
|
6662
|
+
"version": "0.1.3",
|
|
6663
|
+
"resolved": "https://registry.npmjs.org/@expo/plist/-/plist-0.1.3.tgz",
|
|
6664
|
+
"integrity": "sha512-GW/7hVlAylYg1tUrEASclw1MMk9FP4ZwyFAY/SUTJIhPDQHtfOlXREyWV3hhrHdX/K+pS73GNgdfT6E/e+kBbg==",
|
|
6665
|
+
"license": "MIT",
|
|
6666
|
+
"dependencies": {
|
|
6667
|
+
"@xmldom/xmldom": "~0.7.7",
|
|
6668
|
+
"base64-js": "^1.2.3",
|
|
6669
|
+
"xmlbuilder": "^14.0.0"
|
|
6670
|
+
}
|
|
6671
|
+
},
|
|
6672
|
+
"node_modules/expo-asset/node_modules/commander": {
|
|
6673
|
+
"version": "4.1.1",
|
|
6674
|
+
"resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz",
|
|
6675
|
+
"integrity": "sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==",
|
|
6676
|
+
"license": "MIT",
|
|
6677
|
+
"engines": {
|
|
6678
|
+
"node": ">= 6"
|
|
6679
|
+
}
|
|
6680
|
+
},
|
|
6681
|
+
"node_modules/expo-asset/node_modules/expo-constants": {
|
|
6682
|
+
"version": "16.0.2",
|
|
6683
|
+
"resolved": "https://registry.npmjs.org/expo-constants/-/expo-constants-16.0.2.tgz",
|
|
6684
|
+
"integrity": "sha512-9tNY3OVO0jfiMzl7ngb6IOyR5VFzNoN5OOazUWoeGfmMqVB5kltTemRvKraK9JRbBKIw+SOYLEmF0sEqgFZ6OQ==",
|
|
6685
|
+
"license": "MIT",
|
|
6686
|
+
"dependencies": {
|
|
6687
|
+
"@expo/config": "~9.0.0",
|
|
6688
|
+
"@expo/env": "~0.3.0"
|
|
6689
|
+
},
|
|
6690
|
+
"peerDependencies": {
|
|
6691
|
+
"expo": "*"
|
|
6692
|
+
}
|
|
6693
|
+
},
|
|
6694
|
+
"node_modules/expo-asset/node_modules/glob": {
|
|
6695
|
+
"version": "7.1.6",
|
|
6696
|
+
"resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz",
|
|
6697
|
+
"integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==",
|
|
6698
|
+
"deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me",
|
|
6699
|
+
"license": "ISC",
|
|
6700
|
+
"dependencies": {
|
|
6701
|
+
"fs.realpath": "^1.0.0",
|
|
6702
|
+
"inflight": "^1.0.4",
|
|
6703
|
+
"inherits": "2",
|
|
6704
|
+
"minimatch": "^3.0.4",
|
|
6705
|
+
"once": "^1.3.0",
|
|
6706
|
+
"path-is-absolute": "^1.0.0"
|
|
6707
|
+
},
|
|
6708
|
+
"engines": {
|
|
6709
|
+
"node": "*"
|
|
6710
|
+
},
|
|
6711
|
+
"funding": {
|
|
6712
|
+
"url": "https://github.com/sponsors/isaacs"
|
|
6713
|
+
}
|
|
6714
|
+
},
|
|
6715
|
+
"node_modules/expo-asset/node_modules/semver": {
|
|
6716
|
+
"version": "7.8.1",
|
|
6717
|
+
"resolved": "https://registry.npmjs.org/semver/-/semver-7.8.1.tgz",
|
|
6718
|
+
"integrity": "sha512-rkVq3IXh+4FDGch+KwzX3aV9W3kO54GyEgpvBzSyctDA6Xtd7RJQV1xmXbeQp5v7+VzLOfVqiutSE6GICgPFvg==",
|
|
6719
|
+
"license": "ISC",
|
|
6720
|
+
"bin": {
|
|
6721
|
+
"semver": "bin/semver.js"
|
|
6722
|
+
},
|
|
6723
|
+
"engines": {
|
|
6724
|
+
"node": ">=10"
|
|
6725
|
+
}
|
|
6726
|
+
},
|
|
6727
|
+
"node_modules/expo-asset/node_modules/sucrase": {
|
|
6728
|
+
"version": "3.34.0",
|
|
6729
|
+
"resolved": "https://registry.npmjs.org/sucrase/-/sucrase-3.34.0.tgz",
|
|
6730
|
+
"integrity": "sha512-70/LQEZ07TEcxiU2dz51FKaE6hCTWC6vr7FOk3Gr0U60C3shtAN+H+BFr9XlYe5xqf3RA8nrc+VIwzCfnxuXJw==",
|
|
6731
|
+
"license": "MIT",
|
|
6732
|
+
"dependencies": {
|
|
6733
|
+
"@jridgewell/gen-mapping": "^0.3.2",
|
|
6734
|
+
"commander": "^4.0.0",
|
|
6735
|
+
"glob": "7.1.6",
|
|
6736
|
+
"lines-and-columns": "^1.1.6",
|
|
6737
|
+
"mz": "^2.7.0",
|
|
6738
|
+
"pirates": "^4.0.1",
|
|
6739
|
+
"ts-interface-checker": "^0.1.9"
|
|
6740
|
+
},
|
|
6741
|
+
"bin": {
|
|
6742
|
+
"sucrase": "bin/sucrase",
|
|
6743
|
+
"sucrase-node": "bin/sucrase-node"
|
|
6744
|
+
},
|
|
6745
|
+
"engines": {
|
|
6746
|
+
"node": ">=8"
|
|
6747
|
+
}
|
|
6748
|
+
},
|
|
6565
6749
|
"node_modules/expo-auth-session": {
|
|
6566
6750
|
"version": "6.0.3",
|
|
6567
6751
|
"resolved": "https://registry.npmjs.org/expo-auth-session/-/expo-auth-session-6.0.3.tgz",
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tests for quality-aware reranking and the Cachly-Bench regression guard.
|
|
3
|
+
*/
|
|
4
|
+
import { describe, it, expect } from 'vitest';
|
|
5
|
+
import { qualityMultiplier, extractLessonQuality, rerankByQuality } from '../rerank.js';
|
|
6
|
+
import { runBenchmark } from '../bench/cachly-bench.js';
|
|
7
|
+
const lessonMatch = (topic, score, lesson) => ({
|
|
8
|
+
key: `cachly:lesson:best:${topic}`,
|
|
9
|
+
content: JSON.stringify(lesson),
|
|
10
|
+
score,
|
|
11
|
+
matchedWords: ['x'],
|
|
12
|
+
});
|
|
13
|
+
describe('qualityMultiplier', () => {
|
|
14
|
+
it('returns neutral 1.0 for documents with no quality signal', () => {
|
|
15
|
+
expect(qualityMultiplier(null)).toBe(1.0);
|
|
16
|
+
expect(qualityMultiplier({})).toBe(1.0);
|
|
17
|
+
});
|
|
18
|
+
it('ranks a proven success above a fresh failure', () => {
|
|
19
|
+
const success = qualityMultiplier({ outcome: 'success', confidence: 0.95, recall_count: 12 });
|
|
20
|
+
const failure = qualityMultiplier({ outcome: 'failure', confidence: 0.3, recall_count: 0 });
|
|
21
|
+
expect(success).toBeGreaterThan(failure);
|
|
22
|
+
});
|
|
23
|
+
it('rewards proven-ness (recall_count) but saturates', () => {
|
|
24
|
+
const never = qualityMultiplier({ outcome: 'success', confidence: 0.8, recall_count: 0 });
|
|
25
|
+
const some = qualityMultiplier({ outcome: 'success', confidence: 0.8, recall_count: 5 });
|
|
26
|
+
const many = qualityMultiplier({ outcome: 'success', confidence: 0.8, recall_count: 500 });
|
|
27
|
+
expect(some).toBeGreaterThan(never);
|
|
28
|
+
expect(many).toBeGreaterThan(some);
|
|
29
|
+
expect(many - some).toBeLessThan(some - never + 0.2); // diminishing returns
|
|
30
|
+
});
|
|
31
|
+
it('clamps within [0.5, 2.5]', () => {
|
|
32
|
+
const hi = qualityMultiplier({ outcome: 'success', confidence: 1, recall_count: 100000, severity: 'critical', review_level: 'senior', reviewed_by: 'x', endorsements: 99 });
|
|
33
|
+
const lo = qualityMultiplier({ outcome: 'failure', confidence: 0, recall_count: 0, severity: 'minor' });
|
|
34
|
+
expect(hi).toBeLessThanOrEqual(2.5);
|
|
35
|
+
expect(lo).toBeGreaterThanOrEqual(0.5);
|
|
36
|
+
});
|
|
37
|
+
it('boosts a human-reviewed lesson above an unreviewed equal', () => {
|
|
38
|
+
const base = { outcome: 'success', confidence: 0.8, recall_count: 3 };
|
|
39
|
+
const unreviewed = qualityMultiplier({ ...base });
|
|
40
|
+
const peer = qualityMultiplier({ ...base, reviewed_by: 'alice', review_level: 'peer', endorsements: 1 });
|
|
41
|
+
const senior = qualityMultiplier({ ...base, reviewed_by: 'bob', review_level: 'senior', endorsements: 1 });
|
|
42
|
+
expect(peer).toBeGreaterThan(unreviewed);
|
|
43
|
+
expect(senior).toBeGreaterThan(peer);
|
|
44
|
+
});
|
|
45
|
+
it('rewards additional distinct endorsements but saturates', () => {
|
|
46
|
+
const base = { outcome: 'success', confidence: 0.8, recall_count: 3, reviewed_by: 'a', review_level: 'peer' };
|
|
47
|
+
const one = qualityMultiplier({ ...base, endorsements: 1 });
|
|
48
|
+
const three = qualityMultiplier({ ...base, endorsements: 3 });
|
|
49
|
+
const fifty = qualityMultiplier({ ...base, endorsements: 50 });
|
|
50
|
+
expect(three).toBeGreaterThan(one);
|
|
51
|
+
expect(fifty).toBeGreaterThan(three);
|
|
52
|
+
expect(fifty - three).toBeLessThan(three - one + 0.1);
|
|
53
|
+
});
|
|
54
|
+
});
|
|
55
|
+
describe('extractLessonQuality', () => {
|
|
56
|
+
it('parses lesson JSON for lesson keys', () => {
|
|
57
|
+
const q = extractLessonQuality({ key: 'cachly:lesson:best:foo', content: '{"outcome":"success"}' });
|
|
58
|
+
expect(q?.outcome).toBe('success');
|
|
59
|
+
});
|
|
60
|
+
it('returns null for non-lesson keys (e.g. context)', () => {
|
|
61
|
+
expect(extractLessonQuality({ key: 'cachly:ctx:custom:foo', content: 'plain text' })).toBeNull();
|
|
62
|
+
});
|
|
63
|
+
it('returns null for corrupt lesson JSON', () => {
|
|
64
|
+
expect(extractLessonQuality({ key: 'cachly:lesson:best:foo', content: '{not json' })).toBeNull();
|
|
65
|
+
});
|
|
66
|
+
});
|
|
67
|
+
describe('rerankByQuality', () => {
|
|
68
|
+
it('promotes a proven success over a higher-BM25 failure distractor', () => {
|
|
69
|
+
const matches = [
|
|
70
|
+
lessonMatch('symptom-failure', 10, { outcome: 'failure', confidence: 0.2, recall_count: 0 }),
|
|
71
|
+
lessonMatch('proven-success', 8, { outcome: 'success', confidence: 0.95, recall_count: 12 }),
|
|
72
|
+
];
|
|
73
|
+
const ranked = rerankByQuality(matches);
|
|
74
|
+
expect(ranked[0].key).toBe('cachly:lesson:best:proven-success');
|
|
75
|
+
});
|
|
76
|
+
it('keeps a strong text match competitive (does not bury it on quality alone)', () => {
|
|
77
|
+
const matches = [
|
|
78
|
+
lessonMatch('weak-but-perfect', 1, { outcome: 'success', confidence: 1, recall_count: 100 }),
|
|
79
|
+
lessonMatch('strong-text', 100, { outcome: 'partial', confidence: 0.6, recall_count: 0 }),
|
|
80
|
+
];
|
|
81
|
+
const ranked = rerankByQuality(matches);
|
|
82
|
+
expect(ranked[0].key).toBe('cachly:lesson:best:strong-text');
|
|
83
|
+
});
|
|
84
|
+
it('is stable and returns every input', () => {
|
|
85
|
+
const matches = [
|
|
86
|
+
lessonMatch('a', 5, { outcome: 'success' }),
|
|
87
|
+
lessonMatch('b', 5, { outcome: 'success' }),
|
|
88
|
+
];
|
|
89
|
+
const ranked = rerankByQuality(matches);
|
|
90
|
+
expect(ranked).toHaveLength(2);
|
|
91
|
+
});
|
|
92
|
+
it('handles empty input', () => {
|
|
93
|
+
expect(rerankByQuality([])).toEqual([]);
|
|
94
|
+
});
|
|
95
|
+
});
|
|
96
|
+
describe('Cachly-Bench regression guard', () => {
|
|
97
|
+
it('quality rerank never regresses headline metrics vs baseline', async () => {
|
|
98
|
+
const r = await runBenchmark();
|
|
99
|
+
// Cachly must be >= baseline on the headline metrics (the moat must not hurt).
|
|
100
|
+
expect(r.cachly.mrr).toBeGreaterThanOrEqual(r.baseline.mrr - 1e-9);
|
|
101
|
+
expect(r.cachly.precisionAt1).toBeGreaterThanOrEqual(r.baseline.precisionAt1 - 1e-9);
|
|
102
|
+
expect(r.cachly.ndcgAt5).toBeGreaterThanOrEqual(r.baseline.ndcgAt5 - 1e-9);
|
|
103
|
+
});
|
|
104
|
+
it('demonstrates a positive lift on Precision@1 and MRR (the proof)', async () => {
|
|
105
|
+
const r = await runBenchmark();
|
|
106
|
+
expect(r.lift.precisionAt1).toBeGreaterThan(0);
|
|
107
|
+
expect(r.lift.mrr).toBeGreaterThan(0);
|
|
108
|
+
});
|
|
109
|
+
it('baseline is already strong (corpus is not rigged to make rerank look good)', async () => {
|
|
110
|
+
const r = await runBenchmark();
|
|
111
|
+
// A fair benchmark: BM25 alone already gets most queries right.
|
|
112
|
+
expect(r.baseline.mrr).toBeGreaterThan(0.7);
|
|
113
|
+
});
|
|
114
|
+
});
|
|
115
|
+
//# sourceMappingURL=rerank.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"rerank.test.js","sourceRoot":"","sources":["../../../src/__tests__/rerank.test.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,oBAAoB,EAAE,eAAe,EAAsB,MAAM,cAAc,CAAC;AAE5G,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AAExD,MAAM,WAAW,GAAG,CAAC,KAAa,EAAE,KAAa,EAAE,MAAwD,EAAgB,EAAE,CAAC,CAAC;IAC7H,GAAG,EAAE,sBAAsB,KAAK,EAAE;IAClC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;IAC/B,KAAK;IACL,YAAY,EAAE,CAAC,GAAG,CAAC;CACpB,CAAC,CAAC;AAEH,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE;IACjC,EAAE,CAAC,0DAA0D,EAAE,GAAG,EAAE;QAClE,MAAM,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC1C,MAAM,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC1C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;QACtD,MAAM,OAAO,GAAG,iBAAiB,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY,EAAE,EAAE,EAAE,CAAC,CAAC;QAC9F,MAAM,OAAO,GAAG,iBAAiB,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,GAAG,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC,CAAC;QAC5F,MAAM,CAAC,OAAO,CAAC,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kDAAkD,EAAE,GAAG,EAAE;QAC1D,MAAM,KAAK,GAAG,iBAAiB,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,GAAG,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC,CAAC;QAC1F,MAAM,IAAI,GAAG,iBAAiB,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,GAAG,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC,CAAC;QACzF,MAAM,IAAI,GAAG,iBAAiB,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,GAAG,EAAE,YAAY,EAAE,GAAG,EAAE,CAAC,CAAC;QAC3F,MAAM,CAAC,IAAI,CAAC,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;QACpC,MAAM,CAAC,IAAI,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;QACnC,MAAM,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,YAAY,CAAC,IAAI,GAAG,KAAK,GAAG,GAAG,CAAC,CAAC,CAAC,sBAAsB;IAC9E,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0BAA0B,EAAE,GAAG,EAAE;QAClC,MAAM,EAAE,GAAG,iBAAiB,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,CAAC,EAAE,YAAY,EAAE,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,YAAY,EAAE,QAAQ,EAAE,WAAW,EAAE,GAAG,EAAE,YAAY,EAAE,EAAE,EAAE,CAAC,CAAC;QAC5K,MAAM,EAAE,GAAG,iBAAiB,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;QACxG,MAAM,CAAC,EAAE,CAAC,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC;QACpC,MAAM,CAAC,EAAE,CAAC,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0DAA0D,EAAE,GAAG,EAAE;QAClE,MAAM,IAAI,GAAG,EAAE,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,GAAG,EAAE,YAAY,EAAE,CAAC,EAAW,CAAC;QAC/E,MAAM,UAAU,GAAG,iBAAiB,CAAC,EAAE,GAAG,IAAI,EAAE,CAAC,CAAC;QAClD,MAAM,IAAI,GAAG,iBAAiB,CAAC,EAAE,GAAG,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC,CAAC;QACzG,MAAM,MAAM,GAAG,iBAAiB,CAAC,EAAE,GAAG,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,YAAY,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC,CAAC;QAC3G,MAAM,CAAC,IAAI,CAAC,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC;QACzC,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;IACvC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wDAAwD,EAAE,GAAG,EAAE;QAChE,MAAM,IAAI,GAAG,EAAE,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,GAAG,EAAE,YAAY,EAAE,CAAC,EAAE,WAAW,EAAE,GAAG,EAAE,YAAY,EAAE,MAAM,EAAW,CAAC;QACvH,MAAM,GAAG,GAAG,iBAAiB,CAAC,EAAE,GAAG,IAAI,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC,CAAC;QAC5D,MAAM,KAAK,GAAG,iBAAiB,CAAC,EAAE,GAAG,IAAI,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC,CAAC;QAC9D,MAAM,KAAK,GAAG,iBAAiB,CAAC,EAAE,GAAG,IAAI,EAAE,YAAY,EAAE,EAAE,EAAE,CAAC,CAAC;QAC/D,MAAM,CAAC,KAAK,CAAC,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;QACnC,MAAM,CAAC,KAAK,CAAC,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;QACrC,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,YAAY,CAAC,KAAK,GAAG,GAAG,GAAG,GAAG,CAAC,CAAC;IACxD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,sBAAsB,EAAE,GAAG,EAAE;IACpC,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;QAC5C,MAAM,CAAC,GAAG,oBAAoB,CAAC,EAAE,GAAG,EAAE,wBAAwB,EAAE,OAAO,EAAE,uBAAuB,EAAE,CAAC,CAAC;QACpG,MAAM,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACrC,CAAC,CAAC,CAAC;IACH,EAAE,CAAC,iDAAiD,EAAE,GAAG,EAAE;QACzD,MAAM,CAAC,oBAAoB,CAAC,EAAE,GAAG,EAAE,uBAAuB,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;IACnG,CAAC,CAAC,CAAC;IACH,EAAE,CAAC,sCAAsC,EAAE,GAAG,EAAE;QAC9C,MAAM,CAAC,oBAAoB,CAAC,EAAE,GAAG,EAAE,wBAAwB,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;IACnG,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;IAC/B,EAAE,CAAC,iEAAiE,EAAE,GAAG,EAAE;QACzE,MAAM,OAAO,GAAmB;YAC9B,WAAW,CAAC,iBAAiB,EAAE,EAAE,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,GAAG,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC;YAC5F,WAAW,CAAC,gBAAgB,EAAE,CAAC,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY,EAAE,EAAE,EAAE,CAAC;SAC7F,CAAC;QACF,MAAM,MAAM,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;QACxC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;IAClE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2EAA2E,EAAE,GAAG,EAAE;QACnF,MAAM,OAAO,GAAmB;YAC9B,WAAW,CAAC,kBAAkB,EAAE,CAAC,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,CAAC,EAAE,YAAY,EAAE,GAAG,EAAE,CAAC;YAC5F,WAAW,CAAC,aAAa,EAAE,GAAG,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,GAAG,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC;SAC1F,CAAC;QACF,MAAM,MAAM,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;QACxC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,gCAAgC,CAAC,CAAC;IAC/D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;QAC3C,MAAM,OAAO,GAAmB;YAC9B,WAAW,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC;YAC3C,WAAW,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC;SAC5C,CAAC;QACF,MAAM,MAAM,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;QACxC,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;IACjC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qBAAqB,EAAE,GAAG,EAAE;QAC7B,MAAM,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC1C,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,+BAA+B,EAAE,GAAG,EAAE;IAC7C,EAAE,CAAC,6DAA6D,EAAE,KAAK,IAAI,EAAE;QAC3E,MAAM,CAAC,GAAG,MAAM,YAAY,EAAE,CAAC;QAC/B,+EAA+E;QAC/E,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,GAAG,IAAI,CAAC,CAAC;QACnE,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,QAAQ,CAAC,YAAY,GAAG,IAAI,CAAC,CAAC;QACrF,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC;IAC7E,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iEAAiE,EAAE,KAAK,IAAI,EAAE;QAC/E,MAAM,CAAC,GAAG,MAAM,YAAY,EAAE,CAAC;QAC/B,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAC/C,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;IACxC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4EAA4E,EAAE,KAAK,IAAI,EAAE;QAC1F,MAAM,CAAC,GAAG,MAAM,YAAY,EAAE,CAAC;QAC/B,gEAAgE;QAChE,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;IAC9C,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cachly-Bench — the moat proof.
|
|
3
|
+
*
|
|
4
|
+
* Measures recall quality on a fixed, labeled corpus and compares two rankers
|
|
5
|
+
* over the *same* BM25 candidate set, so the comparison isolates ranking quality:
|
|
6
|
+
*
|
|
7
|
+
* baseline — raw BM25+ keyword ranking (what a text-only / flat-file memory does)
|
|
8
|
+
* cachly — BM25+ followed by quality-aware reranking (rerank.ts)
|
|
9
|
+
*
|
|
10
|
+
* It reports standard IR metrics (Precision@k, Recall@k, MRR, nDCG@k) and the
|
|
11
|
+
* relative lift. The corpus (fixtures.ts) is fair: gold answers are the lessons
|
|
12
|
+
* that actually solve each problem, while text-similar failed attempts act as
|
|
13
|
+
* distractors — exactly where ranking by quality should pull ahead.
|
|
14
|
+
*
|
|
15
|
+
* Run: npm run bench (pretty report)
|
|
16
|
+
* npm run bench -- --json (machine-readable)
|
|
17
|
+
*/
|
|
18
|
+
export interface BenchMetrics {
|
|
19
|
+
precisionAt1: number;
|
|
20
|
+
precisionAt3: number;
|
|
21
|
+
recallAt3: number;
|
|
22
|
+
mrr: number;
|
|
23
|
+
ndcgAt5: number;
|
|
24
|
+
}
|
|
25
|
+
export interface BenchResult {
|
|
26
|
+
baseline: BenchMetrics;
|
|
27
|
+
cachly: BenchMetrics;
|
|
28
|
+
/** Relative lift per metric, e.g. { mrr: 0.23 } = +23%. */
|
|
29
|
+
lift: BenchMetrics;
|
|
30
|
+
queryCount: number;
|
|
31
|
+
corpusSize: number;
|
|
32
|
+
}
|
|
33
|
+
/** Run the benchmark over the fixture corpus. Pure-ish: no network, no real Redis. */
|
|
34
|
+
export declare function runBenchmark(): Promise<BenchResult>;
|
|
35
|
+
export declare function formatReport(r: BenchResult): string;
|