clawmem 0.22.0 → 0.24.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/AGENTS.md CHANGED
@@ -105,9 +105,9 @@ All other retrieval is handled by Tier 2 hooks. **Do NOT call MCP tools speculat
105
105
  | `kg_query` | Entity SPO triples with temporal validity. Entity facts, NOT causal "why". |
106
106
  | `session_log` | "last time" / "yesterday" / "what did we do". Do NOT use `query` for cross-session. |
107
107
  | `profile` | User profile (static facts + dynamic context). |
108
- | `memory_pin` | Lifecycle retention + priority among relevance-equivalent results (+0.3 composite boost on composite surfaces; exact-tie precedence on raw vector routes). PROACTIVELY for constraints, architecture decisions, corrections. |
108
+ | `memory_pin` | Lifecycle retention + priority among relevance-equivalent results (+0.3 composite boost on composite surfaces; exact-tie precedence on raw routes — vector + `search` non-recency). PROACTIVELY for constraints, architecture decisions, corrections. |
109
109
  | `memory_snooze` | PROACTIVELY when `<vault-context>` surfaces noise — snooze 30 days. |
110
- | `memory_forget` | Deactivate a memory by closest match. Sparingly — prefer snooze. |
110
+ | `memory_forget` | Deactivate a memory by closest match. Sparingly — prefer snooze. Weak matches return a disambiguation list instead of acting (v0.23.0). |
111
111
  | `build_graphs` | Temporal backbone + semantic graph after bulk ingestion. NOT after every reindex. |
112
112
  | `timeline` | Temporal neighborhood around a doc. Progressive disclosure: search → timeline → get. |
113
113
  | `memory_evolution_status` | How a doc's A-MEM metadata evolved over time. |
@@ -135,14 +135,14 @@ The pipeline autonomously generates lex/vec/hyde variants, fuses BM25 + vector v
135
135
 
136
136
  ## Composite scoring
137
137
 
138
- Applied on the composite surfaces: hooks, `query`, `search`, and `memory_retrieve`'s keyword/hybrid/causal/complex modes. **v0.22.0: MCP `vsearch` and `memory_retrieve` semantic/discovery rank non-recency queries by RAW cosine instead** (`scoreBasis: "vector-cosine"`; metadata breaks exact ties only; `minScore` filters raw with no default); recency-intent queries keep composite everywhere.
138
+ Applied on the composite surfaces: hooks, `query`, and `memory_retrieve`'s keyword/hybrid/causal/complex modes. **v0.22.0: MCP `vsearch` and `memory_retrieve` semantic/discovery rank non-recency queries by RAW cosine instead** (`scoreBasis: "vector-cosine"`; metadata breaks exact ties only; `minScore` filters raw with no default); recency-intent queries keep composite everywhere. **v0.23.0:** `searchScore` on FTS surfaces is the monotonic `|bm25|/(1+|bm25|)` transform (it was a constant 1.0 through v0.22.0 due to a clamp bug — keyword relevance contributed zero ordering); FTS-transform scores and cosines are independent monotonic signals, not one calibrated scale. **v0.24.0: MCP `search` ranks non-recency queries by the RAW BM25 transform** (`scoreBasis: "fts-bm25"`; metadata breaks exact ties only; `minScore` filters raw with no default) — judged keyword eval: raw MRR 0.848 vs composite 0.415 over 43 targets, composite losing even on the fresh-doc-favorable slice; recency-intent queries keep composite.
139
139
 
140
140
  ```
141
141
  compositeScore = (0.50·searchScore + 0.25·recencyScore + 0.25·confidenceScore) × qualityMultiplier × coActivationBoost
142
142
  ```
143
143
 
144
- - `qualityMultiplier = 0.7 + 0.6·qualityScore` (0.7× … 1.3×); `coActivationBoost` up to +15%; length-normalized (floor 30%); frequency boost capped +10%; **pinned docs +0.3 additive on composite surfaces** (raw vector routes: pin = exact-tie precedence only).
145
- - **`query` tool (v0.13.0+):** non-recency queries use **0.70·search + 0.15·recency + 0.15·confidence**. `search`, `memory_retrieve`'s composite modes, and context-surfacing keep the 0.50/0.25/0.25 default. (`vsearch` + `memory_retrieve` semantic/discovery use RAW cosine for non-recency queries as of v0.22.0 no composite weights at all.)
144
+ - `qualityMultiplier = 0.7 + 0.6·qualityScore` (0.7× … 1.3×); `coActivationBoost` up to +15%; length-normalized (floor 30%); frequency boost capped +10%; **pinned docs +0.3 additive on composite surfaces** (raw routes — vector + `search` non-recency: pin = exact-tie precedence only).
145
+ - **`query` tool (v0.13.0+):** non-recency queries use **0.70·search + 0.15·recency + 0.15·confidence**. `memory_retrieve`'s composite modes, context-surfacing, and `search`'s recency branch keep the 0.50/0.25/0.25 default. (`vsearch` + `memory_retrieve` semantic/discovery use RAW cosine, and `search` uses the RAW BM25 transform, for non-recency queries v0.22.0/v0.24.0: no composite weights at all.)
146
146
  - **Recency intent** ("latest"/"recent"/"last session") switches all to **0.10·search + 0.70·recency + 0.20·confidence**.
147
147
  - **Half-lives:** decision/deductive/preference/hub/antipattern = ∞ · project 120d · research 90d · problem/milestone/note 60d · conversation/progress 45d · handoff 30d (extend up to 3× for frequently-accessed).
148
148
 
@@ -162,7 +162,7 @@ compositeScore = (0.50·searchScore + 0.25·recencyScore + 0.25·confidenceScore
162
162
 
163
163
  ## Memory lifecycle (pin / snooze / forget — manual tools)
164
164
 
165
- - **`memory_pin`** (lifecycle retention + priority among relevance-equivalent results; +0.3 boost on composite surfaces, exact-tie precedence on raw vector routes) — PROACTIVELY when: user says "remember this"/"important"; an architecture/critical decision was just made; a user preference/constraint should persist. Do NOT pin routine/session-specific items.
165
+ - **`memory_pin`** (lifecycle retention + priority among relevance-equivalent results; +0.3 boost on composite surfaces, exact-tie precedence on raw routes) — PROACTIVELY when: user says "remember this"/"important"; an architecture/critical decision was just made; a user preference/constraint should persist. Do NOT pin routine/session-specific items.
166
166
  - **`memory_snooze`** — PROACTIVELY when a memory keeps surfacing but isn't relevant now, user says "not now"/"later", or content is time-boxed.
167
167
  - **`memory_forget`** — only when genuinely wrong or permanently obsolete. Prefer snooze for temporary suppression.
168
168
  - **Contradiction auto-resolution:** when `decision-extractor` detects a new decision contradicting an old one, the old one's confidence is lowered automatically — no manual action needed.
package/README.md CHANGED
@@ -542,7 +542,7 @@ Registered by `clawmem setup mcp`. Available to any MCP-compatible client.
542
542
  | Tool | Description |
543
543
  |---|---|
544
544
  | `memory_retrieve` | **Preferred entry point.** Auto-classifies query and routes to optimal backend (query, intent_search, session_log, find_similar, or query_plan). Use instead of manually choosing a search tool. |
545
- | `search` | BM25 keyword search — for exact terms, config names, error codes, filenames. Composite scoring + co-activation boost + compact mode. Collection filter supports comma-separated values. Prefer `memory_retrieve` for auto-routing. |
545
+ | `search` | BM25 keyword search — for exact terms, config names, error codes, filenames. **v0.24.0: non-recency queries rank by the RAW BM25 transform** (`scoreBasis: "fts-bm25"`; judged keyword eval MRR 0.848 vs composite 0.415; metadata breaks exact ties only; `minScore` filters raw, no default); recency-intent queries keep composite. Collection filter supports comma-separated values. Prefer `memory_retrieve` for auto-routing. |
546
546
  | `vsearch` | Vector semantic search — for conceptual/fuzzy matching when exact keywords are unknown. **v0.22.0: non-recency queries rank by RAW cosine (`scoreBasis: "vector-cosine"`; metadata breaks exact ties only; `minScore` filters raw with no default)**; recency-intent queries keep composite. Collection filter supports comma-separated values. Prefer `memory_retrieve` for auto-routing. |
547
547
  | `query` | Full hybrid pipeline (BM25 + vector + rerank) — general-purpose when query type is unclear. WRONG for "why" questions (use `intent_search`) or cross-session queries (use `session_log`). Prefer `memory_retrieve` for auto-routing. Intent hint, strong-signal bypass, chunk dedup, candidateLimit, MMR diversity, compact mode. |
548
548
  | `get` | Retrieve single document by path or docid |
@@ -594,8 +594,8 @@ Registered by `clawmem setup mcp`. Available to any MCP-compatible client.
594
594
 
595
595
  | Tool | Description |
596
596
  |---|---|
597
- | `memory_forget` | Search → deactivate closest match (with audit trail) |
598
- | `memory_pin` | Pin a memory: lifecycle retention + priority among relevance-equivalent results (+0.3 composite boost on hook/`query`/`search` surfaces; exact-tie precedence on the raw vector routes). USE PROACTIVELY when: user states a persistent constraint, makes an architecture decision, or corrects a misconception. Don't wait for curator — pin critical decisions immediately. |
597
+ | `memory_forget` | Search → deactivate closest match (with audit trail). Weak matches return a disambiguation list instead of acting (v0.23.0) |
598
+ | `memory_pin` | Pin a memory: lifecycle retention + priority among relevance-equivalent results (+0.3 composite boost on hook/`query` surfaces; exact-tie precedence on the raw routes — vector + `search` non-recency). USE PROACTIVELY when: user states a persistent constraint, makes an architecture decision, or corrects a misconception. Don't wait for curator — pin critical decisions immediately. |
599
599
  | `memory_snooze` | Temporarily hide a memory from context surfacing until a date. USE PROACTIVELY when `<vault-context>` repeatedly surfaces irrelevant content — snooze for 30 days instead of ignoring it. |
600
600
  | `status` | Index health with content type distribution |
601
601
  | `reindex` | Trigger vault re-scan |
@@ -695,7 +695,7 @@ Content types are inferred from frontmatter or file path patterns. Half-lives ex
695
695
 
696
696
  **Frequency boost:** Documents with higher revision counts or duplicate counts get a durability signal: `freqSignal = (revisions - 1) × 2 + (duplicates - 1)`, `freqBoost = min(0.10, log1p(freqSignal) × 0.03)`. Revision count (content evolution) is weighted 2× vs duplicate count (ingest repetition). Capped at 10%.
697
697
 
698
- **Pin boost:** Pin = lifecycle retention + priority among relevance-equivalent results. On composite surfaces pinned documents get a +0.3 additive boost (capped at 1.0); on the raw vector routes (v0.22.0) pin breaks exact raw-score ties only. Use `memory_pin` to pin critical memories.
698
+ **Pin boost:** Pin = lifecycle retention + priority among relevance-equivalent results. On composite surfaces pinned documents get a +0.3 additive boost (capped at 1.0); on the raw routes — the vector tools (v0.22.0) and non-recency `search` (v0.24.0) — pin breaks exact raw-score ties only. Use `memory_pin` to pin critical memories.
699
699
 
700
700
  **Snooze:** Snoozed documents are filtered out of context surfacing until their snooze date. Use `memory_snooze` for temporary suppression.
701
701
 
package/SKILL.md CHANGED
@@ -92,9 +92,9 @@ All other retrieval is handled by Tier 2 hooks. **Do NOT call MCP tools speculat
92
92
  | `kg_query` | Entity SPO triples with temporal validity. Entity facts, NOT causal "why" (use `intent_search`). |
93
93
  | `session_log` | "last time" / "yesterday" / "what did we do". Do NOT use `query` for cross-session. |
94
94
  | `profile` | User profile (static facts + dynamic context). |
95
- | `memory_pin` | Lifecycle retention + priority among relevance-equivalent results (+0.3 composite boost on composite surfaces; exact-tie precedence on raw vector routes). Use PROACTIVELY for constraints, architecture decisions, corrections. |
95
+ | `memory_pin` | Lifecycle retention + priority among relevance-equivalent results (+0.3 composite boost on composite surfaces; exact-tie precedence on raw routes — vector + `search` non-recency). Use PROACTIVELY for constraints, architecture decisions, corrections. |
96
96
  | `memory_snooze` | Use PROACTIVELY when `<vault-context>` surfaces noise — snooze 30 days. |
97
- | `memory_forget` | Deactivate a memory by closest match. Sparingly — prefer snooze. |
97
+ | `memory_forget` | Deactivate a memory by closest match. Sparingly — prefer snooze. Weak matches return a disambiguation list instead of acting (v0.23.0). |
98
98
  | `build_graphs` | Temporal backbone + semantic graph after bulk ingestion. NOT after every reindex. |
99
99
  | `timeline` | Temporal neighborhood around a doc. Progressive disclosure: search → timeline → get. |
100
100
  | `memory_evolution_status` | How a doc's A-MEM metadata evolved over time. |
@@ -189,7 +189,7 @@ Query -> intent classification (WHY/WHEN/ENTITY/WHAT)
189
189
 
190
190
  ## Composite scoring (how ranking works)
191
191
 
192
- Applied on the composite surfaces: hooks, `query`, `search`, and `memory_retrieve`'s keyword/hybrid/causal/complex modes. **v0.22.0: MCP `vsearch` and `memory_retrieve` semantic/discovery rank non-recency queries by RAW cosine instead** (`scoreBasis: "vector-cosine"`; metadata breaks exact ties only; `minScore` filters raw with no default); recency-intent queries keep composite everywhere.
192
+ Applied on the composite surfaces: hooks, `query`, and `memory_retrieve`'s keyword/hybrid/causal/complex modes. **v0.22.0: MCP `vsearch` and `memory_retrieve` semantic/discovery rank non-recency queries by RAW cosine instead** (`scoreBasis: "vector-cosine"`; metadata breaks exact ties only; `minScore` filters raw with no default); recency-intent queries keep composite everywhere. **v0.23.0:** `searchScore` on FTS surfaces is the monotonic `|bm25|/(1+|bm25|)` transform (it was a constant 1.0 through v0.22.0 due to a clamp bug — keyword relevance contributed zero ordering); FTS-transform scores and cosines are independent monotonic signals, not one calibrated scale. **v0.24.0: MCP `search` ranks non-recency queries by the RAW BM25 transform** (`scoreBasis: "fts-bm25"`; metadata breaks exact ties only; `minScore` filters raw with no default) — judged keyword eval: raw MRR 0.848 vs composite 0.415 over 43 targets, composite losing even on the fresh-doc-favorable slice; recency-intent queries keep composite.
193
193
 
194
194
  ```
195
195
  compositeScore = (0.50·searchScore + 0.25·recencyScore + 0.25·confidenceScore) × qualityMultiplier × coActivationBoost
@@ -198,8 +198,8 @@ compositeScore = (0.50·searchScore + 0.25·recencyScore + 0.25·confidenceScore
198
198
  - `qualityMultiplier = 0.7 + 0.6·qualityScore` (0.7× penalty … 1.3× boost).
199
199
  - `coActivationBoost = 1 + min(coCount/10, 0.15)` (docs surfaced together get up to +15%).
200
200
  - Length normalization penalizes verbose entries (floor 30%); frequency boost capped at +10%.
201
- - **Pinned docs: +0.3 additive on composite surfaces** (capped at 1.0); on the raw vector routes pin = exact-tie precedence only.
202
- - **`query` tool (v0.13.0+):** non-recency queries use retrieval-tuned **0.70·search + 0.15·recency + 0.15·confidence**. `search`, `memory_retrieve`'s composite modes, and `context-surfacing` keep the 0.50/0.25/0.25 default. (`vsearch` + `memory_retrieve` semantic/discovery use RAW cosine for non-recency queries as of v0.22.0 no composite weights at all.)
201
+ - **Pinned docs: +0.3 additive on composite surfaces** (capped at 1.0); on the raw routes (vector + `search` non-recency) pin = exact-tie precedence only.
202
+ - **`query` tool (v0.13.0+):** non-recency queries use retrieval-tuned **0.70·search + 0.15·recency + 0.15·confidence**. `memory_retrieve`'s composite modes, `context-surfacing`, and `search`'s recency branch keep the 0.50/0.25/0.25 default. (`vsearch` + `memory_retrieve` semantic/discovery use RAW cosine, and `search` uses the RAW BM25 transform, for non-recency queries v0.22.0/v0.24.0: no composite weights at all.)
203
203
  - **Recency intent** ("latest"/"recent"/"last session") switches all to **0.10·search + 0.70·recency + 0.20·confidence**.
204
204
 
205
205
  **Content-type half-lives:** decision / deductive / preference / hub / antipattern = ∞ (never decay) · project 120d · research 90d · problem / milestone / note 60d · conversation / progress 45d · handoff 30d. Half-lives extend up to 3× for frequently-accessed memories. Attention decay: non-durable types (handoff, progress, conversation, note, project) lose 5% confidence/week without access; decision / deductive / preference / hub / research / antipattern are exempt.
@@ -210,7 +210,7 @@ compositeScore = (0.50·searchScore + 0.25·recencyScore + 0.25·confidenceScore
210
210
 
211
211
  ## Memory lifecycle (pin / snooze / forget — manual tools)
212
212
 
213
- - **`memory_pin`** (lifecycle retention + priority among relevance-equivalent results; +0.3 boost on composite surfaces, exact-tie precedence on raw vector routes) — PROACTIVELY when: user says "remember this"/"important"; an architecture/critical decision was just made; a user preference/constraint should persist across sessions. Do NOT pin routine/session-specific items.
213
+ - **`memory_pin`** (lifecycle retention + priority among relevance-equivalent results; +0.3 boost on composite surfaces, exact-tie precedence on raw routes) — PROACTIVELY when: user says "remember this"/"important"; an architecture/critical decision was just made; a user preference/constraint should persist across sessions. Do NOT pin routine/session-specific items.
214
214
  - **`memory_snooze`** — PROACTIVELY when a memory keeps surfacing but isn't relevant now, user says "not now"/"later", or content is time-boxed.
215
215
  - **`memory_forget`** — only when genuinely wrong or permanently obsolete. Prefer snooze for temporary suppression.
216
216
  - **Contradiction auto-resolution:** when `decision-extractor` detects a new decision contradicting an old one, the old one's confidence is lowered automatically — no manual action needed.
@@ -1,6 +1,6 @@
1
1
  # Composite Scoring
2
2
 
3
- Composite scoring blends multiple signals beyond raw search relevance. **It applies to the hook pipeline (context surfacing), the `query` hybrid pipeline, `search`, and `memory_retrieve`'s keyword/hybrid/causal/complex modes.** Since v0.22.0 the direct vector routes — MCP `vsearch` and `memory_retrieve` semantic/discovery do NOT use it for non-recency queries: they rank by raw vector cosine, with metadata (including pin) breaking exact score ties only, and report `scoreBasis: "vector-cosine"`. That split is measured, not aesthetic: on a judged set against the live vault, raw cosine ranked 16/19 targets #1 (MRR 0.912) while this composite ranked 1/19 (MRR 0.307) and pushed 14/19 correct answers below the old minScore floor in the compressed-high similarity band of modern embedding models, every multiplier below is larger than the raw margins separating right answers from wrong ones. Recency-intent queries keep composite (the Recency column) on every route that had it.
3
+ Composite scoring blends multiple signals beyond raw search relevance. **It applies to the hook pipeline (context surfacing), the `query` hybrid pipeline, and `memory_retrieve`'s keyword/hybrid/causal/complex modes.** The direct retrieval routes do NOT use it for non-recency queries: since v0.22.0 MCP `vsearch` and `memory_retrieve` semantic/discovery rank by raw vector cosine (`scoreBasis: "vector-cosine"`), and since v0.24.0 MCP `search` ranks by the raw BM25 transform (`scoreBasis: "fts-bm25"`) — metadata (including pin) breaks exact score ties only on all of them. Both splits are measured, not aesthetic: on judged sets against the live vault, raw cosine ranked 16/19 targets #1 (MRR 0.912) vs composite 1/19 (MRR 0.307, 14/19 pushed below the old minScore floor), and raw-FTS ranked 33/43 keyword targets #1 (MRR 0.848) vs composite 6/43 (MRR 0.415) composite lost even on the fresh-doc-favorable slice (0.348 vs 0.801), so the multipliers below bury keyword relevance rather than refine it. Recency-intent queries keep composite (the Recency column) on every route that had it.
4
4
 
5
5
  ## Formula
6
6
 
@@ -19,10 +19,12 @@ compositeScore = (w_search * searchScore + w_recency * recencyScore + w_confiden
19
19
 
20
20
  Recency intent is detected automatically when queries contain "latest", "recent", "last session", etc. — and **takes precedence over the `query`-tool weights**: a recency-phrased `query` call still uses the Recency-intent column.
21
21
 
22
- The **`query` tool** uses retrieval-tuned weights (search 0.70) derived from a held-out judged-relevance eval (v0.13.0) — more weight on topical relevance measurably improves graded NDCG@10 without demoting the newest-correct version of evolving docs. `search`, the composite `memory_retrieve` modes, and the context-surfacing hook **keep the Normal column**. The v0.21.0 `retrieval.mcp_direct_tuned_weights` knob is **superseded as of v0.22.0 and has no effect** — the direct-pipeline eval it was gated on measured tuned weights at 1/19 hit@1, and the direct vector routes moved to raw ordering instead. The key is still parsed (a once-per-process warning is logged if set) so existing configs don't break.
22
+ The **`query` tool** uses retrieval-tuned weights (search 0.70) derived from a held-out judged-relevance eval (v0.13.0) — more weight on topical relevance measurably improves graded NDCG@10 without demoting the newest-correct version of evolving docs. The composite `memory_retrieve` modes, the context-surfacing hook, and `search`'s recency branch **keep the Normal column**. The v0.21.0 `retrieval.mcp_direct_tuned_weights` knob is **superseded as of v0.22.0 and has no effect** — the direct-pipeline eval it was gated on measured tuned weights at 1/19 hit@1, and the direct routes moved to raw ordering instead. The key is still parsed (a once-per-process warning is logged if set) so existing configs don't break.
23
23
 
24
24
  Threshold note: EOS-anchored last-token embedding models (e.g. zembed-1 served correctly) produce a **compressed-high similarity band** — unrelated pairs sit near ~0.4 rather than ~0.2. Relative ordering is what matters; treat absolute `minScore` cutoffs as model-dependent rather than universal.
25
25
 
26
+ `searchScore` provenance (v0.23.0): on FTS/BM25 surfaces the input is the monotonic transform `|bm25|/(1+|bm25|)` of FTS5's negative-is-better `bm25()` (through v0.22.0 a clamp bug flattened it to a constant 1.0, which made composite ranking on those surfaces metadata-only); on vector surfaces it is the raw cosine. The two are independent monotonic signals on an uncalibrated common range — composite blends them where pools mix (hooks, REST hybrid), which removes the old unconditional FTS-over-vector dominance but does not make the channels numerically comparable.
27
+
26
28
  ## Signal breakdown
27
29
 
28
30
  ### Search score (0.0 - 1.0)
@@ -92,13 +94,13 @@ coActivationBoost = 1 + min(coCount / 10, 0.15)
92
94
 
93
95
  Documents frequently surfaced together in the same session get up to 15% boost.
94
96
 
95
- **Where co-activation is applied depends on the caller.** The composite MCP surfaces (`query`, `search`, and `vsearch` on recency-intent queries only) pass a co-activation function into `applyCompositeScoring()`, so the boost is part of the composite score used for ranking and — on `vsearch` and `search`, the tools that expose `minScore` — for `minScore` filtering. On `vsearch`'s non-recency (raw) regime, co-activation contributes only to the exact-tie key, never to ranking or filtering. The context-surfacing hook does NOT pass co-activation into composite scoring — instead it applies a separate spreading-activation step *after* adaptive threshold filtering. This means the hook's threshold decisions are based on scores without co-activation, and co-activation only boosts results that already passed the threshold. This is intentional: it prevents relationship boosts from rescuing otherwise-weak results into the surfaced set.
97
+ **Where co-activation is applied depends on the caller.** The composite MCP surfaces (`query`, plus `search` and `vsearch` on recency-intent queries only) pass a co-activation function into `applyCompositeScoring()`, so the boost is part of the composite score used for ranking and — on `vsearch` and `search`, the tools that expose `minScore` — for `minScore` filtering. On the non-recency (raw) regimes of `vsearch` (v0.22.0) and `search` (v0.24.0), co-activation contributes only to the exact-tie key, never to ranking or filtering. The context-surfacing hook does NOT pass co-activation into composite scoring — instead it applies a separate spreading-activation step *after* adaptive threshold filtering. This means the hook's threshold decisions are based on scores without co-activation, and co-activation only boosts results that already passed the threshold. This is intentional: it prevents relationship boosts from rescuing otherwise-weak results into the surfaced set.
96
98
 
97
99
  ## Additional modifiers
98
100
 
99
101
  ### Pin boost
100
102
 
101
- On composite surfaces, pinned documents receive a +0.3 additive boost (capped at 1.0 total). Pin's contract is **lifecycle retention + prioritization among relevance-equivalent results** — on the raw vector routes (v0.22.0) it therefore breaks exact raw-score ties only and never lifts a document over a more relevant one.
103
+ On composite surfaces, pinned documents receive a +0.3 additive boost (capped at 1.0 total). Pin's contract is **lifecycle retention + prioritization among relevance-equivalent results** — on the raw routes (the vector tools since v0.22.0, and non-recency `search` since v0.24.0) it therefore breaks exact raw-score ties only and never lifts a document over a more relevant one.
102
104
 
103
105
  ### Length normalization
104
106
 
@@ -1,6 +1,6 @@
1
1
  # Upgrading ClawMem
2
2
 
3
- Guide for upgrading between released versions. Current: **v0.22.0**.
3
+ Guide for upgrading between released versions. Current: **v0.24.0**.
4
4
 
5
5
  ClawMem upgrades are designed to be drop-in: pull the new version, restart any long-lived processes, and the SQLite schema auto-migrates on first open. This guide documents per-version specifics for upgrades that have additional considerations beyond the quick path below.
6
6
 
@@ -59,6 +59,29 @@ docker compose up -d reranker # /v1/rerank on :8090
59
59
 
60
60
  ---
61
61
 
62
+ ## v0.24.0: raw-BM25-primary ranking on `search`
63
+
64
+ No migration steps — drop-in. Behavior notes:
65
+
66
+ - **MCP `search` ordering changes for non-recency queries**: results now rank by the raw BM25 transform instead of the composite blend (judged keyword eval: raw MRR 0.848 vs composite 0.415 over 43 targets; the composite lost even the fresh-doc-favorable slice). If you depended on recency/quality multipliers reordering keyword results, phrase the query with recency intent ("recent …", "latest …") — that branch keeps composite — or use `query`.
67
+ - **`minScore` on `search` is now raw-basis with NO default** for non-recency queries: omitted = no filter, explicit `0` honored. Previous composite-scale floors (e.g. `0.3`) do not translate — the raw transform maps `0.70 ⇔ |bm25| ≈ 2.3` and `0.85 ⇔ |bm25| ≈ 5.7`.
68
+ - **`structuredContent.scoreBasis`** on `search` reports `"fts-bm25"` (non-recency) or `"composite"` (recency-intent). Consumers that parsed the compact `score` as a composite value should read the basis field.
69
+ - **CLI `search`, REST, hooks, `query`, and `memory_retrieve` are unchanged** — the ranking-contract change is scoped to the MCP `search` tool, exactly as evaluated.
70
+ - New ops knob: `CLAWMEM_DISABLE_FTS_BYPASS=true` disables the query-pipeline strong-signal bypass at both consumers (MCP + CLI) — harness/incident use.
71
+
72
+ ## v0.23.0: monotonic BM25 exposed score
73
+
74
+ No migration command, no schema change, no re-embed. Restart long-lived processes per the quick path.
75
+
76
+ **Behavior change — the exposed FTS score is a real relevance signal.** Through v0.22.0 a clamp bug flattened every FTS result's `score` to the constant 1.0, so ranking on the BM25 surfaces (`search`, REST keyword mode, CLI search, `memory_retrieve` keyword and its semantic-mode FTS fallback, hook FTS lanes) was effectively metadata-only. The score is now `|bm25|/(1+|bm25|)` — bounded [0,1), higher is better. What to expect after upgrading:
77
+
78
+ - `search` results reorder toward keyword relevance; reported scores drop from the old flat values and vary per hit. If a workflow compared `search` scores against a hardcoded cutoff tuned to the constant-1.0 era, re-tune it (the composite floor semantics of `minScore` are unchanged; the observed values shifted).
79
+ - The `query` pipeline's strong-signal bypass actually fires now on unambiguous keyword queries (skipping LLM expansion, which makes those calls faster) — and no longer fires on a lone weak match.
80
+ - `memory_forget` targeting is stricter: weak keyword matches return a disambiguation list instead of auto-selecting. If a script relied on forget acting on any single match, it must now pass a more specific query or a path.
81
+ - `clawmem doctor`/curator's BM25 probe reports honestly — a near-empty vault may now show a degraded BM25 probe where it previously passed vacuously.
82
+
83
+ ---
84
+
62
85
  ## v0.22.0: raw-cosine ranking on the direct vector routes
63
86
 
64
87
  No migration command, no schema change, no re-embed. Restart long-lived processes per the quick path.
@@ -79,6 +79,10 @@ When the top BM25 hit scores >= 0.85 and the gap to the second hit is >= 0.15, t
79
79
 
80
80
  The bypass is disabled when `intent` is provided — intent implies the query is ambiguous, so keyword confidence alone is insufficient.
81
81
 
82
+ Functional as of v0.23.0: through v0.22.0 a clamp bug flattened every FTS score to a constant 1.0, so with two or more hits the gap was always 0 (the bypass never fired) and with exactly one hit the gap was always 1.0 (it fired even on a garbage match). The scores are now the monotonic `|bm25|/(1+|bm25|)` transform — 0.85 corresponds to |bm25| ≥ 5.67 — and the check itself lives in one shared helper (`hasStrongFtsSignal`) used by both this pipeline and the CLI `query` command. Threshold tuning is deliberately deferred to a judged query-pipeline A/B (BACKLOG 49.3).
83
+
84
+ Ops escape hatch (v0.24.0): `CLAWMEM_DISABLE_FTS_BYPASS=true` forces the full expansion path on every query at BOTH consumers (the MCP pipeline and the CLI `query`), via the shared `ftsBypassEnabled()` read in `search-utils.ts`. Intended for A/B harnesses and incident triage; `hasStrongFtsSignal` itself stays pure.
85
+
82
86
  ## Query expansion
83
87
 
84
88
  The LLM generates lex (keyword), vec (semantic), and hyde (hypothetical answer) variants of the query, each carried as a typed `ExpandedQuery`. Variants are **routed by type**: `lex` expansions are searched on BM25 only, `vec` and `hyde` on vector only. The original query is the only leg that fans out to *both* backends. All legs are searched in parallel and fused; the original query's two lists receive 2x weight in RRF, ensuring it anchors the ranking.
@@ -6,7 +6,7 @@ ClawMem is an open-source memory engine for Claude Code and AI agents. It runs o
6
6
 
7
7
  - **Indexes** markdown documents into a local SQLite vault with full-text search (BM25) and vector embeddings
8
8
  - **Retrieves** relevant context automatically via hooks or on-demand via MCP tools / REST API
9
- - **Scores** results per route: raw vector cosine on the direct vector tools (v0.22.0), composite scoring (relevance, recency, confidence, quality, co-activation) on the hook and hybrid pipelines
9
+ - **Scores** results per route: raw vector cosine on the direct vector tools (v0.22.0), the raw BM25 transform on non-recency `search` (v0.24.0), composite scoring (relevance, recency, confidence, quality, co-activation) on the hook and hybrid pipelines
10
10
  - **Tracks** decisions, handoffs, and session history across conversations
11
11
  - **Traverses** causal and semantic graphs to answer "why" and "what led to" questions
12
12
 
@@ -6,7 +6,9 @@ Complete reference for ClawMem's MCP server tools. These let AI agents search, r
6
6
 
7
7
  **Internal-collection visibility (v0.21.0):** the retrieval tools `search`, `vsearch`, `query`, `query_plan`, `memory_retrieve`, and `find_similar` exclude the system-internal `_clawmem` collection (observations/deductions/handoffs) by default. Opt-ins: pass `includeInternal: true` (all six tools), or — on the tools that expose a `collection` parameter (`search`, `vsearch`, `query`) — name `_clawmem` explicitly in the filter. `find_similar` auto-includes internal results when the REFERENCE document is itself internal. `intent_search`, `find_causal_links`, `kg_query`, `session_log`, and `timeline` are NOT filtered — system memory is their substrate by design.
8
8
 
9
- **Scoring regimes (v0.22.0):** the direct vector routes — `vsearch` and `memory_retrieve`'s semantic/discovery modes — rank non-recency queries by RAW vector cosine. Their `structuredContent` carries `scoreBasis: "vector-cosine"`; raw cosine values are specific to the embedding model that produced them and are NOT comparable across models or to composite scores. Document metadata — including pin — participates only inside groups of exactly-equal raw scores. On these routes `minScore` filters the raw score and has NO default (omitted = no filter; an explicit `0` is honored). Recency-intent queries ("latest…", "recently…", "yesterday…") keep the composite regime with its 0.3 default floor and report `scoreBasis: "composite"`, as do `search`, `query`, `query_plan`, and `memory_retrieve`'s keyword/hybrid/causal/complex modes. `find_similar` has always ranked by raw cosine. Rationale: on the measured vault, raw cosine ranked 16/19 judged targets #1 (MRR 0.912) while the composite stack ranked 1/19 and filtered 14/19 out entirely.
9
+ **Scoring regimes (v0.22.0 vector · v0.24.0 FTS):** the direct retrieval routes — `vsearch` and `memory_retrieve`'s semantic/discovery modes (v0.22.0), and `search` (v0.24.0) — rank non-recency queries by their RAW channel score: vector cosine (`scoreBasis: "vector-cosine"`) on the vector routes, the monotonic BM25 transform (`scoreBasis: "fts-bm25"`) on `search`. Raw values are channel-specific and NOT comparable across channels, across embedding models, or to composite scores. Document metadata — including pin — participates only inside groups of exactly-equal raw scores. On these routes `minScore` filters the raw score and has NO default (omitted = no filter; an explicit `0` is honored). Recency-intent queries ("latest…", "recently…", "yesterday…") keep the composite regime and report `scoreBasis: "composite"` — `vsearch`'s recency branch keeps its 0.3 composite default floor, `search`'s keeps 0 as do `query`, `query_plan`, and `memory_retrieve`'s keyword/hybrid/causal/complex modes. `find_similar` has always ranked by raw cosine. Rationale — both splits are measured, not aesthetic: on judged sets against the live vault, raw cosine ranked 16/19 targets #1 (MRR 0.912) vs composite 1/19 (0.307); raw-FTS ranked 33/43 keyword targets #1 (MRR 0.848) vs composite 6/43 (0.415), with composite losing even on the fresh-doc-favorable slice (0.348 vs 0.801).
10
+
11
+ **FTS score provenance (v0.23.0):** the raw `score` on BM25/FTS results is the monotonic transform `|bm25|/(1+|bm25|)` of FTS5's negative-is-better `bm25()` value — bounded [0,1), higher is better, stable across queries. (Through v0.22.0 a clamp bug flattened it to a constant 1.0, so composite ranking on FTS surfaces was effectively metadata-only and score-threshold gates never filtered.) FTS-transform scores and vector cosines are **independent monotonic signals, not a calibrated common scale** — compare within a channel, not across channels.
10
12
 
11
13
  **Degraded vector results (v0.21.0):** under default exclusion the vector scan escalates its depth to fill `limit` with allowed documents, up to a hard cap. When the cap prevents an exhaustive scan and the result is under-filled, `structuredContent` carries `degraded: true` with `degradedReason`: `"excluded-dominant"` (distinct excluded docs account for the shortfall — the guidance line suggests `includeInternal: true` or a refined query) or `"cap-truncation"` (shortfall driven by fragment dedup, neutral guidance). Multi-leg routes (`query`, `query_plan`, `memory_retrieve` complex mode) aggregate `degraded = any(leg)` and list per-leg reasons in `structuredContent.degradedLegs`; single-vector routes (`vsearch`, `find_similar`, `memory_retrieve`'s other modes) report the flat `degraded` + `degradedReason` pair. A small vault whose whole index is scanned without hitting the cap returns a plain short list with NO marker.
12
14
 
@@ -38,7 +40,7 @@ Full hybrid pipeline: BM25 + vector + query expansion + cross-encoder reranking.
38
40
  |-------|------|---------|-------------|
39
41
  | `query` | string | required | Search query |
40
42
  | `limit` | number | 10 | Max results |
41
- | `compact` | boolean | true | Compact output |
43
+ | `compact` | boolean | false | Compact output |
42
44
  | `collection` | string | — | Filter by collection (comma-separated for multi) |
43
45
  | `intent` | string | — | Domain hint for ambiguous queries (steers expansion, reranking, chunk selection) |
44
46
  | `candidateLimit` | number | 30 | Candidates for reranking (tune precision vs speed) |
@@ -49,17 +51,20 @@ BM25 strong-signal bypass: skips expansion when top BM25 hit >= 0.85 with gap >=
49
51
 
50
52
  ### search
51
53
 
52
- BM25 only. Zero GPU cost.
54
+ BM25 only. Zero GPU cost. Non-recency queries rank by the RAW BM25 transform (`scoreBasis: "fts-bm25"`, v0.24.0 — see the regimes note above); recency-intent queries use the composite regime.
53
55
 
54
56
  | Param | Type | Default | Description |
55
57
  |-------|------|---------|-------------|
56
58
  | `query` | string | required | Search query |
57
59
  | `limit` | number | 10 | Max results |
58
- | `compact` | boolean | true | Compact output |
60
+ | `minScore` | number | | Score floor. Non-recency: filters the RAW BM25 transform, NO default (omitted = no filter; explicit `0` honored). Recency-intent: composite floor, default 0 |
61
+ | `compact` | boolean | false | Compact output |
59
62
  | `collection` | string | — | Filter by collection |
60
63
  | `includeInternal` | boolean | false | Include system-internal `_clawmem` docs |
61
64
  | `vault` | string | — | Named vault |
62
65
 
66
+ Score fields: on non-recency queries the reported ranking score IS the raw BM25 transform (compact `score` and non-compact `compositeScore` both carry it; non-compact `score` is the same raw value). On recency-intent queries compact `score` / non-compact `compositeScore` are composite while non-compact `score` stays the raw transform.
67
+
63
68
  ### vsearch
64
69
 
65
70
  Vector only. Semantic similarity. Non-recency queries rank by RAW cosine (`scoreBasis: "vector-cosine"`, v0.22.0); recency-intent queries use the composite regime.
@@ -69,7 +74,7 @@ Vector only. Semantic similarity. Non-recency queries rank by RAW cosine (`score
69
74
  | `query` | string | required | Search query |
70
75
  | `limit` | number | 10 | Max results |
71
76
  | `minScore` | number | — | Raw-cosine floor on non-recency queries (no default — omitted means no filter; explicit `0` honored). Recency-intent queries keep the composite-scale 0.3 default. |
72
- | `compact` | boolean | true | Compact output |
77
+ | `compact` | boolean | false | Compact output |
73
78
  | `collection` | string | — | Filter by collection |
74
79
  | `includeInternal` | boolean | false | Include system-internal `_clawmem` docs |
75
80
  | `vault` | string | — | Named vault |
@@ -181,7 +186,7 @@ Recent session history with handoffs and file changes.
181
186
 
182
187
  ### memory_pin
183
188
 
184
- Pin a memory: lifecycle retention plus prioritization among relevance-equivalent results. On composite surfaces (hooks, `query`, `search`) pinned docs get the +0.3 composite boost; on the raw vector routes (v0.22.0) pin wins exact raw-score ties but never overrides a relevance difference.
189
+ Pin a memory: lifecycle retention plus prioritization among relevance-equivalent results. On composite surfaces (hooks, `query`) pinned docs get the +0.3 composite boost; on the raw routes (`vsearch`/`memory_retrieve` semantic-discovery v0.22.0, `search` non-recency v0.24.0) pin wins exact raw-score ties but never overrides a relevance difference.
185
190
 
186
191
  | Param | Type | Default | Description |
187
192
  |-------|------|---------|-------------|
@@ -211,6 +216,8 @@ Permanently deactivate a memory.
211
216
 
212
217
  **Search behavior (v0.2.6+, all three tools):** Query matching cascades through four strategies: exact path match → BM25 full-text → title-token overlap → vector similarity. This prevents "No matching memory found" errors when the document exists but BM25 fails to match (e.g., too many AND'd terms). Path-like queries (containing `/` or ending in `.md`) try direct path matching first. `memory_forget` requires higher confidence to act — ambiguous matches return candidates instead of mutating.
213
218
 
219
+ **Targeting confidence (v0.23.0):** the confidence gate (`score ≥ 0.7`, or a ≥ 0.2 gap to the runner-up when more than one candidate exists) is now live for BM25 candidates — through v0.22.0 every FTS candidate carried a constant score of 1.0, so `memory_forget` treated ANY keyword match as high-confidence and auto-selected it. Weak matches — including a lone weak match — now return the candidate list for disambiguation instead of acting. Stricter, safer targeting for a destructive operation.
220
+
214
221
  ## Lifecycle
215
222
 
216
223
  ### lifecycle_status
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "clawmem",
3
- "version": "0.22.0",
3
+ "version": "0.24.0",
4
4
  "description": "On-device memory layer for AI agents. Claude Code, OpenClaw, and Hermes. Hooks + MCP server + hybrid RAG search.",
5
5
  "type": "module",
6
6
  "bin": {
package/src/clawmem.ts CHANGED
@@ -58,7 +58,7 @@ import { formatSearchResults, type OutputFormat } from "./formatter.ts";
58
58
  import { indexCollection, parseDocument } from "./indexer.ts";
59
59
  import { detectBeadsProject } from "./beads.ts";
60
60
  import { applyCompositeScoring, hasRecencyIntent, type EnrichedResult } from "./memory.ts";
61
- import { enrichResults, reciprocalRankFusion, toRanked, type RankedResult } from "./search-utils.ts";
61
+ import { enrichResults, reciprocalRankFusion, toRanked, hasStrongFtsSignal, ftsBypassEnabled, type RankedResult } from "./search-utils.ts";
62
62
  import { splitDocument } from "./splitter.ts";
63
63
  import { getProfile, updateProfile, isProfileStale } from "./profile.ts";
64
64
  import { regenerateAllDirectoryContexts } from "./directory-context.ts";
@@ -1095,9 +1095,7 @@ async function cmdQuery(args: string[]) {
1095
1095
 
1096
1096
  // Step 1: BM25 for strong signal check
1097
1097
  const ftsResults = s.searchFTS(query, 20);
1098
- const topScore = ftsResults[0]?.score ?? 0;
1099
- const secondScore = ftsResults[1]?.score ?? 0;
1100
- const strongSignal = topScore >= 0.85 && (topScore - secondScore) >= 0.15;
1098
+ const strongSignal = ftsBypassEnabled() && hasStrongFtsSignal(ftsResults);
1101
1099
 
1102
1100
  // Step 2: Query expansion (skip if strong BM25 signal). expandQuery now returns
1103
1101
  // typed ExpandedQuery[] (lex/vec/hyde) — no more brittle string re-parsing, and
package/src/mcp.ts CHANGED
@@ -33,8 +33,8 @@ import {
33
33
  type EnrichedResult,
34
34
  type CoActivationFn,
35
35
  } from "./memory.ts";
36
- import { enrichResults, reciprocalRankFusion, toRanked, blendRerank, type RankedResult } from "./search-utils.ts";
37
- import { selectScoringRegime, rankRawPrimary, VECTOR_SCORE_BASIS, COMPOSITE_SCORE_BASIS } from "./scoring-regime.ts";
36
+ import { enrichResults, reciprocalRankFusion, toRanked, blendRerank, hasStrongFtsSignal, ftsBypassEnabled, attachRrfScores, type RankedResult } from "./search-utils.ts";
37
+ import { selectScoringRegime, rankRawPrimary, VECTOR_SCORE_BASIS, FTS_SCORE_BASIS, COMPOSITE_SCORE_BASIS } from "./scoring-regime.ts";
38
38
  import { applyMMRDiversity } from "./mmr.ts";
39
39
  import { indexCollection, type IndexStats } from "./indexer.ts";
40
40
  import { listCollections } from "./collections.ts";
@@ -330,11 +330,7 @@ This is the recommended entry point for ALL memory queries.`,
330
330
  } catch (e) { rethrowIfFatalVectorError(e); /* else: no vectors */ }
331
331
  const rrfWeights = intent.intent === 'WHY' ? [1.0, 1.5] : intent.intent === 'WHEN' ? [1.5, 1.0] : [1.0, 1.0];
332
332
  const fusedRanked = reciprocalRankFusion([bm25Results.map(toRanked), vecResults.map(toRanked)], rrfWeights);
333
- const allSearch = [...bm25Results, ...vecResults];
334
- let fused: SearchResult[] = fusedRanked.map(fr => {
335
- const orig = allSearch.find(r => r.filepath === fr.file);
336
- return orig ? { ...orig, score: fr.score } : null;
337
- }).filter((r): r is SearchResult => r !== null);
333
+ let fused: SearchResult[] = attachRrfScores(fusedRanked, [...bm25Results, ...vecResults]);
338
334
 
339
335
  if (intent.intent === 'WHY' || intent.intent === 'ENTITY') {
340
336
  try {
@@ -460,11 +456,7 @@ This is the recommended entry point for ALL memory queries.`,
460
456
  } catch (e) { rethrowIfFatalVectorError(e); /* */ }
461
457
  if (vec.length > 0) {
462
458
  const fusedRanked = reciprocalRankFusion([bm25.map(toRanked), vec.map(toRanked)], [1.0, 1.0]);
463
- const allSearch = [...bm25, ...vec];
464
- results = fusedRanked.map(fr => {
465
- const orig = allSearch.find(r => r.filepath === fr.file);
466
- return orig ? { ...orig, score: fr.score } : null;
467
- }).filter((r): r is SearchResult => r !== null);
459
+ results = attachRrfScores(fusedRanked, [...bm25, ...vec]);
468
460
  } else {
469
461
  results = bm25;
470
462
  }
@@ -566,7 +558,7 @@ This is the recommended entry point for ALL memory queries.`,
566
558
  );
567
559
 
568
560
  // ---------------------------------------------------------------------------
569
- // Tool: search (BM25 + composite)
561
+ // Tool: search (BM25 raw-transform ranking; composite on recency intent)
570
562
  // ---------------------------------------------------------------------------
571
563
 
572
564
  server.registerTool(
@@ -577,7 +569,7 @@ This is the recommended entry point for ALL memory queries.`,
577
569
  inputSchema: {
578
570
  query: z.string().describe("Search query"),
579
571
  limit: z.number().optional().default(10),
580
- minScore: z.number().optional().default(0),
572
+ minScore: z.number().optional().describe("Score floor. Non-recency queries filter the RAW FTS score, the monotonic |bm25|/(1+|bm25|) transform (default: no filter; explicit 0 honored). Recency-intent queries keep the composite-scale default 0."),
581
573
  collection: z.string().optional().describe("Filter to collection (single name or comma-separated)"),
582
574
  compact: z.boolean().optional().default(false).describe("Return compact results (id, path, title, score, snippet) instead of full content"),
583
575
  includeInternal: z.boolean().optional().default(false).describe("Include system-internal _clawmem docs (observations/deductions) — excluded by default"),
@@ -594,8 +586,16 @@ This is the recommended entry point for ALL memory queries.`,
594
586
 
595
587
  const coFn = (path: string) => store.getCoActivated(path);
596
588
  const enriched = enrichResults(store, results, query);
597
- const scored = applyCompositeScoring(enriched, query, coFn)
598
- .filter(r => r.compositeScore >= (minScore || 0));
589
+ // v0.24.0 two-regime scoring (S49-JUDGED-EVAL-DESIGN.md, SWITCH verdict): non-recency
590
+ // queries rank by the raw FTS transform — metadata (incl. pin) breaks exact score
591
+ // ties only, and minScore (if given) filters the raw score with NO default floor.
592
+ // Recency-intent queries keep the pre-v0.24.0 composite behavior including its
593
+ // default-0 floor.
594
+ const regime = selectScoringRegime(query);
595
+ const sScoreBasis = regime === "raw" ? FTS_SCORE_BASIS : COMPOSITE_SCORE_BASIS;
596
+ const scored = regime === "raw"
597
+ ? rankRawPrimary(enriched, query, coFn).filter(r => minScore === undefined || r.compositeScore >= minScore)
598
+ : applyCompositeScoring(enriched, query, coFn).filter(r => r.compositeScore >= (minScore || 0));
599
599
 
600
600
  if (compact) {
601
601
  const items = scored.map(r => ({
@@ -604,7 +604,7 @@ This is the recommended entry point for ALL memory queries.`,
604
604
  snippet: (r.body || "").substring(0, 150), content_type: r.contentType, modified_at: r.modifiedAt,
605
605
  fragment: r.fragmentType ? { type: r.fragmentType, label: r.fragmentLabel } : undefined,
606
606
  }));
607
- return { content: [{ type: "text", text: formatSearchSummary(items.map(i => ({ ...i, file: i.path, compositeScore: i.score, context: null })), query) }], structuredContent: { results: items } };
607
+ return { content: [{ type: "text", text: formatSearchSummary(items.map(i => ({ ...i, file: i.path, compositeScore: i.score, context: null })), query) }], structuredContent: { results: items, scoreBasis: sScoreBasis } };
608
608
  }
609
609
 
610
610
  const filtered: SearchResultItem[] = scored.map(r => {
@@ -623,7 +623,7 @@ This is the recommended entry point for ALL memory queries.`,
623
623
 
624
624
  return {
625
625
  content: [{ type: "text", text: formatSearchSummary(filtered, query) }],
626
- structuredContent: { results: filtered },
626
+ structuredContent: { results: filtered, scoreBasis: sScoreBasis },
627
627
  };
628
628
  }
629
629
  );
@@ -747,12 +747,9 @@ This is the recommended entry point for ALL memory queries.`,
747
747
  : undefined;
748
748
  const excl = resolveExcludedCollections(includeInternal, collections);
749
749
  const initialFts = store.searchFTS(query, 20, undefined, collections, dateRange, excl);
750
- const topScore = initialFts.length > 0 ? Math.abs(initialFts[0]!.score) : 0;
751
- const secondScore = initialFts.length > 1 ? Math.abs(initialFts[1]!.score) : 0;
752
750
  // When intent is provided, disable strong-signal bypass — the obvious BM25
753
751
  // match may not be what the caller wants (e.g. "performance" with intent "web page load times")
754
- const hasStrongSignal = !intent && initialFts.length > 0
755
- && topScore >= 0.85 && (topScore - secondScore) >= 0.15;
752
+ const hasStrongSignal = !intent && ftsBypassEnabled() && hasStrongFtsSignal(initialFts);
756
753
 
757
754
  // Step 2: Query expansion (skipped if strong signal). Typed routing —
758
755
  // original → BOTH FTS + vector (2× RRF anchor), lex → FTS only, vec/hyde → vector only.
@@ -1101,9 +1098,11 @@ This is the recommended entry point for ALL memory queries.`,
1101
1098
 
1102
1099
  const top = candidates[0]!;
1103
1100
 
1104
- // Clear winner: high score OR significant gap to #2
1101
+ // Clear winner: high score OR significant gap to #2. The gap clause requires an
1102
+ // actual #2 to gap against — a lone weak candidate must not qualify (destructive
1103
+ // ops would otherwise auto-select a garbage single match).
1105
1104
  const gap = candidates.length > 1 ? top.score - candidates[1]!.score : 1.0;
1106
- const confident = top.score >= 0.7 || gap >= 0.2;
1105
+ const confident = top.score >= 0.7 || (candidates.length > 1 && gap >= 0.2);
1107
1106
 
1108
1107
  // For destructive ops (forget), require higher confidence
1109
1108
  if (destructive && !confident) {
@@ -1724,11 +1723,7 @@ This is the recommended entry point for ALL memory queries.`,
1724
1723
  const fusedRanked = reciprocalRankFusion([bm25Results.map(toRanked), vecResults.map(toRanked)], rrfWeights);
1725
1724
 
1726
1725
  // Map RRF results back to SearchResult with updated scores
1727
- const allSearchResults = [...bm25Results, ...vecResults];
1728
- const fused: SearchResult[] = fusedRanked.map(fr => {
1729
- const original = allSearchResults.find(r => r.filepath === fr.file);
1730
- return original ? { ...original, score: fr.score } : null;
1731
- }).filter((r): r is SearchResult => r !== null);
1726
+ const fused: SearchResult[] = attachRrfScores(fusedRanked, [...bm25Results, ...vecResults]);
1732
1727
 
1733
1728
  // Step 4: Graph expansion (if enabled and intent allows)
1734
1729
  let expanded = fused;
@@ -1922,10 +1917,11 @@ This is the recommended entry point for ALL memory queries.`,
1922
1917
  const vec = detGraph.results;
1923
1918
  if (detGraph.degraded && detGraph.degradedReason) degradedLegs.push({ leg: `clause${clauseIdx}:graph:vector`, reason: detGraph.degradedReason });
1924
1919
  const fused = reciprocalRankFusion([bm25.map(toRanked), vec.map(toRanked)], [1.0, 1.0]);
1925
- const searchMap = new Map([...bm25, ...vec].map(r => [r.filepath, r]));
1926
- results = fused
1927
- .map(fr => searchMap.get(fr.file))
1928
- .filter((r): r is SearchResult => r !== null);
1920
+ // Carry the RRF-fused score (parity with causal + intent_search): the graph
1921
+ // traversal below anchors on `score`, which must be RRF-scale, not raw single-channel.
1922
+ // vec-first: this site historically preferred the vector variant on duplicate
1923
+ // paths (its pre-helper Map construction let later entries overwrite) — preserved.
1924
+ results = attachRrfScores(fused, [...vec, ...bm25]);
1929
1925
 
1930
1926
  // Graph expansion for WHY/ENTITY
1931
1927
  if (intent.intent === 'WHY' || intent.intent === 'ENTITY') {
@@ -1,15 +1,17 @@
1
1
  /**
2
- * Centralized scoring-regime selection for the MCP direct vector routes (v0.22.0).
2
+ * Centralized scoring-regime selection for the MCP direct retrieval routes
3
+ * (vector routes v0.22.0; FTS `search` route v0.24.0).
3
4
  *
4
5
  * Two regimes:
5
6
  *
6
7
  * "raw" — non-recency queries on the evidenced raw routes (`vsearch`,
7
- * `memory_retrieve` semantic/discovery): results rank by RAW
8
- * vector cosine DESC. Document metadataincluding pin
9
- * participates ONLY inside groups of exactly-equal raw scores.
10
- * The reported score IS the raw cosine (scoreBasis
11
- * "vector-cosine"): specific to the embedding model that
12
- * produced it, and not comparable to composite scores.
8
+ * `memory_retrieve` semantic/discovery, `search`): results rank
9
+ * by the RAW channel score DESC vector cosine on the vector
10
+ * routes (scoreBasis "vector-cosine"), the monotonic BM25
11
+ * transform on the FTS route (scoreBasis "fts-bm25"). Document
12
+ * metadata including pin participates ONLY inside groups of
13
+ * exactly-equal raw scores. Raw scores are channel-specific and
14
+ * not comparable to composite scores or across channels.
13
15
  *
14
16
  * "recency-composite" — hasRecencyIntent(query): the pre-v0.22.0 composite behavior,
15
17
  * unchanged (RECENCY_WEIGHTS blend, multipliers, pin boost,
@@ -20,6 +22,10 @@
20
22
  * below the old composite minScore — every metadata signal large enough to matter is
21
23
  * larger than the 0.03–0.10 raw margins that separate right answers from wrong ones,
22
24
  * so metadata may only ever break exact raw-score ties on these routes.
25
+ * FTS basis (BACKLOG Source 49.2, judged keyword eval 2026-07-11): raw-FTS-primary
26
+ * MRR 0.848 vs shipped composite 0.415 over 43 judged targets (held-out 0.875 vs
27
+ * 0.335, composite hit@1 0/20) — composite lost even on the composite-favorable
28
+ * fresh-among-many shape (0.348 vs 0.801).
23
29
  */
24
30
  import {
25
31
  applyCompositeScoring,
@@ -30,8 +36,11 @@ import {
30
36
  type CompositeScoringOptions,
31
37
  } from "./memory.ts";
32
38
 
33
- /** Reported score basis for raw-regime results: raw vector cosine. */
39
+ /** Reported score basis for raw-regime results on the vector routes: raw cosine. */
34
40
  export const VECTOR_SCORE_BASIS = "vector-cosine" as const;
41
+ /** Reported score basis for raw-regime results on the FTS route (v0.24.0): the
42
+ * monotonic |bm25|/(1+|bm25|) transform. */
43
+ export const FTS_SCORE_BASIS = "fts-bm25" as const;
35
44
  /** Reported score basis for composite-scored results. */
36
45
  export const COMPOSITE_SCORE_BASIS = "composite" as const;
37
46
 
@@ -43,15 +52,17 @@ export function selectScoringRegime(query: string): ScoringRegime {
43
52
  }
44
53
 
45
54
  /**
46
- * Raw-primary ranking (design R1/R2): raw score DESC; within a group of exactly-equal
47
- * raw scores the deterministic tie order is pinned DESC, then legacy composite DESC,
48
- * then displayPath ASC. `compositeScore` on the returned rows carries the RAW score —
49
- * the reported score is the raw cosine.
55
+ * Raw-primary ranking (design R1/R2): raw channel score DESC; within a group of
56
+ * exactly-equal raw scores the deterministic tie order is pinned DESC, then legacy
57
+ * composite DESC, then displayPath ASC. `compositeScore` on the returned rows carries
58
+ * the RAW score vector cosine on the vector routes, the monotonic BM25 transform on
59
+ * the FTS route. Exact transformed-score ties are legitimate relevance-equivalent
60
+ * groups on both channels.
50
61
  *
51
62
  * Callers gate on selectScoringRegime() first — this ranker is only meaningful for
52
63
  * non-recency queries (the legacy composite computed here for tie keys therefore never
53
64
  * takes the RECENCY_WEIGHTS branch). Inputs are document-unique on these routes
54
- * (searchVecDetailed hydrates one row per document).
65
+ * (searchVecDetailed and searchFTS both hydrate one row per document).
55
66
  *
56
67
  * `options.now` flows to the tie-key composite so frozen-clock evaluation (the v0.22.0
57
68
  * acceptance bundle) is bit-reproducible.
@@ -201,3 +201,45 @@ export function toRanked(r: SearchResult): RankedResult {
201
201
  score: r.score,
202
202
  };
203
203
  }
204
+
205
+ /**
206
+ * Map RRF-fused results back to their originating SearchResults, carrying the FUSED
207
+ * score. Downstream consumers — graph-traversal anchors, composite enrichment — must
208
+ * see the fusion's relevance scale, not a raw single-channel score (S49.1 U2: the
209
+ * query_plan graph clause used to drop the fused score, anchoring traversal on raw
210
+ * FTS values). First occurrence wins when a doc appears in multiple channels
211
+ * (matching the previous `find`-based lookups); fused entries with no originating
212
+ * SearchResult are dropped.
213
+ */
214
+ export function attachRrfScores(fused: RankedResult[], originals: SearchResult[]): SearchResult[] {
215
+ const byPath = new Map<string, SearchResult>();
216
+ for (const r of originals) {
217
+ if (!byPath.has(r.filepath)) byPath.set(r.filepath, r);
218
+ }
219
+ return fused
220
+ .map(fr => { const orig = byPath.get(fr.file); return orig ? { ...orig, score: fr.score } : null; })
221
+ .filter((r): r is SearchResult => r !== null);
222
+ }
223
+
224
+ /**
225
+ * BM25 strong-signal check on ftsScoreFromBm25-scale results: the top hit is strong
226
+ * (≥ 0.85 ⇔ |bm25| ≥ 5.67) AND clearly separated from #2 (gap ≥ 0.15; a missing #2
227
+ * counts as 0, so a lone WEAK hit does not qualify). Callers use this to skip LLM
228
+ * query expansion when the keyword answer is unambiguous.
229
+ */
230
+ export function hasStrongFtsSignal(results: SearchResult[]): boolean {
231
+ if (results.length === 0) return false;
232
+ const top = results[0]!.score;
233
+ const second = results.length > 1 ? results[1]!.score : 0;
234
+ return top >= 0.85 && (top - second) >= 0.15;
235
+ }
236
+
237
+ /**
238
+ * Ops escape hatch for the BM25 strong-signal bypass: CLAWMEM_DISABLE_FTS_BYPASS=true
239
+ * forces the full expansion path on every query (both the MCP `query` pipeline and the
240
+ * CLI `query` consume this). Read at call time so A/B harnesses and tests can toggle
241
+ * per invocation; `hasStrongFtsSignal` itself stays pure.
242
+ */
243
+ export function ftsBypassEnabled(): boolean {
244
+ return process.env.CLAWMEM_DISABLE_FTS_BYPASS !== "true";
245
+ }
package/src/server.ts CHANGED
@@ -12,7 +12,7 @@
12
12
 
13
13
  import type { Server } from "bun";
14
14
  import type { Store, SearchResult, TimelineResult } from "./store.ts";
15
- import { enrichResults, reciprocalRankFusion, toRanked } from "./search-utils.ts";
15
+ import { enrichResults, reciprocalRankFusion, toRanked, attachRrfScores } from "./search-utils.ts";
16
16
  import { applyCompositeScoring, hasRecencyIntent, type EnrichedResult } from "./memory.ts";
17
17
  import { applyMMRDiversity } from "./mmr.ts";
18
18
  import { listCollections } from "./collections.ts";
@@ -617,11 +617,7 @@ async function handleRetrieve(req: Request, _url: URL, store: Store): Promise<Re
617
617
  } catch (e) { rethrowIfFatalVectorError(e); /* vector unavailable */ }
618
618
  const weights = intent.intent === "WHEN" ? [1.5, 1.0] : [1.0, 1.5];
619
619
  const fused = reciprocalRankFusion([bm25.map(toRanked), vec.map(toRanked)], weights);
620
- const allResults = [...bm25, ...vec];
621
- results = fused.map(fr => {
622
- const orig = allResults.find(r => r.filepath === fr.file);
623
- return orig ? { ...orig, score: fr.score } : null;
624
- }).filter((r): r is SearchResult => r !== null);
620
+ results = attachRrfScores(fused, [...bm25, ...vec]);
625
621
  } else if (mode === "keyword") {
626
622
  results = store.searchFTS(query, limit * 2, undefined, collections);
627
623
  } else if (mode === "semantic") {
package/src/store.ts CHANGED
@@ -3506,6 +3506,20 @@ function buildFTS5Query(query: string): string | null {
3506
3506
  return terms.map(t => `"${t}"*`).join(' AND ');
3507
3507
  }
3508
3508
 
3509
+ /**
3510
+ * Convert an FTS5 bm25() value into a stable [0,1) relevance score where higher is better.
3511
+ *
3512
+ * FTS5's bm25() is negative-is-better: it returns -1 × the BM25 score, so it is ≤ 0 for
3513
+ * every match. The transform is per-row and monotonic in match strength (|bm25|/(1+|bm25|))
3514
+ * with no per-query normalization, so cross-query comparisons, minScore filters, and the
3515
+ * strong-signal bypass all stay meaningful. A hypothetical positive input clamps to 0
3516
+ * rather than inverting the ordering.
3517
+ */
3518
+ export function ftsScoreFromBm25(bm25Score: number): number {
3519
+ const m = Math.max(0, -bm25Score);
3520
+ return m / (1 + m);
3521
+ }
3522
+
3509
3523
  export function searchFTS(db: Database, query: string, limit: number = 20, collectionId?: number, collections?: string[], dateRange?: { start: string; end: string }, excludeCollections?: string[]): SearchResult[] {
3510
3524
  const ftsQuery = buildFTS5Query(query);
3511
3525
  if (!ftsQuery) return [];
@@ -3558,9 +3572,7 @@ export function searchFTS(db: Database, query: string, limit: number = 20, colle
3558
3572
  const rows = db.prepare(sql).all(...params) as { filepath: string; display_path: string; title: string; body: string; hash: string; modified_at: string; bm25_score: number }[];
3559
3573
  return rows.map(row => {
3560
3574
  const collectionName = row.filepath.split('//')[1]?.split('/')[0] || "";
3561
- // Convert bm25 (lower is better) into a stable (0..1] score where higher is better.
3562
- // Avoid per-query normalization so "strong signal" heuristics can work.
3563
- const score = 1 / (1 + Math.max(0, row.bm25_score));
3575
+ const score = ftsScoreFromBm25(row.bm25_score);
3564
3576
  return {
3565
3577
  filepath: row.filepath,
3566
3578
  displayPath: row.display_path,
@@ -4172,14 +4184,24 @@ export type ExpandedQuery = {
4172
4184
  const EXPAND_CACHE_VERSION = "v3-qmd-terse-typed";
4173
4185
  const EXPAND_PROVIDER_FINGERPRINT = "qmd-terse";
4174
4186
 
4175
- export async function expandQuery(query: string, model: string = DEFAULT_QUERY_MODEL, db: Database, intent?: string): Promise<ExpandedQuery[]> {
4176
- // Typed-JSON cache. Versioned key (include intent + provider fingerprint).
4177
- const cacheKey = getCacheKey(`expandQuery:${EXPAND_CACHE_VERSION}`, {
4187
+ /**
4188
+ * The EXACT llm_cache key expandQuery(query, model, intent) reads and writes.
4189
+ * Exported for eval harnesses (S49.3 freeze protocol): delete/verify expansion
4190
+ * cache rows without replicating the private key construction — the version and
4191
+ * provider fingerprint stay in one place.
4192
+ */
4193
+ export function expandQueryCacheKey(query: string, model: string = DEFAULT_QUERY_MODEL, intent?: string): string {
4194
+ return getCacheKey(`expandQuery:${EXPAND_CACHE_VERSION}`, {
4178
4195
  query,
4179
4196
  model,
4180
4197
  provider: EXPAND_PROVIDER_FINGERPRINT,
4181
4198
  ...(intent && { intent }),
4182
4199
  });
4200
+ }
4201
+
4202
+ export async function expandQuery(query: string, model: string = DEFAULT_QUERY_MODEL, db: Database, intent?: string): Promise<ExpandedQuery[]> {
4203
+ // Typed-JSON cache. Versioned key (include intent + provider fingerprint).
4204
+ const cacheKey = expandQueryCacheKey(query, model, intent);
4183
4205
  const cached = getCachedResult(db, cacheKey);
4184
4206
  if (cached) {
4185
4207
  try {