clawmem 0.22.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +2 -2
- package/README.md +2 -2
- package/SKILL.md +2 -2
- package/docs/concepts/composite-scoring.md +2 -0
- package/docs/guides/upgrading.md +14 -1
- package/docs/internals/query-pipeline.md +2 -0
- package/docs/reference/mcp-tools.md +8 -1
- package/package.json +1 -1
- package/src/clawmem.ts +2 -4
- package/src/mcp.ts +14 -26
- package/src/search-utils.ts +32 -0
- package/src/server.ts +2 -6
- package/src/store.ts +15 -3
package/AGENTS.md
CHANGED
|
@@ -107,7 +107,7 @@ All other retrieval is handled by Tier 2 hooks. **Do NOT call MCP tools speculat
|
|
|
107
107
|
| `profile` | User profile (static facts + dynamic context). |
|
|
108
108
|
| `memory_pin` | Lifecycle retention + priority among relevance-equivalent results (+0.3 composite boost on composite surfaces; exact-tie precedence on raw vector routes). PROACTIVELY for constraints, architecture decisions, corrections. |
|
|
109
109
|
| `memory_snooze` | PROACTIVELY when `<vault-context>` surfaces noise — snooze 30 days. |
|
|
110
|
-
| `memory_forget` | Deactivate a memory by closest match. Sparingly — prefer snooze. |
|
|
110
|
+
| `memory_forget` | Deactivate a memory by closest match. Sparingly — prefer snooze. Weak matches return a disambiguation list instead of acting (v0.23.0). |
|
|
111
111
|
| `build_graphs` | Temporal backbone + semantic graph after bulk ingestion. NOT after every reindex. |
|
|
112
112
|
| `timeline` | Temporal neighborhood around a doc. Progressive disclosure: search → timeline → get. |
|
|
113
113
|
| `memory_evolution_status` | How a doc's A-MEM metadata evolved over time. |
|
|
@@ -135,7 +135,7 @@ The pipeline autonomously generates lex/vec/hyde variants, fuses BM25 + vector v
|
|
|
135
135
|
|
|
136
136
|
## Composite scoring
|
|
137
137
|
|
|
138
|
-
Applied on the composite surfaces: hooks, `query`, `search`, and `memory_retrieve`'s keyword/hybrid/causal/complex modes. **v0.22.0: MCP `vsearch` and `memory_retrieve` semantic/discovery rank non-recency queries by RAW cosine instead** (`scoreBasis: "vector-cosine"`; metadata breaks exact ties only; `minScore` filters raw with no default); recency-intent queries keep composite everywhere.
|
|
138
|
+
Applied on the composite surfaces: hooks, `query`, `search`, and `memory_retrieve`'s keyword/hybrid/causal/complex modes. **v0.22.0: MCP `vsearch` and `memory_retrieve` semantic/discovery rank non-recency queries by RAW cosine instead** (`scoreBasis: "vector-cosine"`; metadata breaks exact ties only; `minScore` filters raw with no default); recency-intent queries keep composite everywhere. **v0.23.0:** `searchScore` on FTS surfaces is the monotonic `|bm25|/(1+|bm25|)` transform (it was a constant 1.0 through v0.22.0 due to a clamp bug — keyword relevance contributed zero ordering); FTS-transform scores and cosines are independent monotonic signals, not one calibrated scale.
|
|
139
139
|
|
|
140
140
|
```
|
|
141
141
|
compositeScore = (0.50·searchScore + 0.25·recencyScore + 0.25·confidenceScore) × qualityMultiplier × coActivationBoost
|
package/README.md
CHANGED
|
@@ -542,7 +542,7 @@ Registered by `clawmem setup mcp`. Available to any MCP-compatible client.
|
|
|
542
542
|
| Tool | Description |
|
|
543
543
|
|---|---|
|
|
544
544
|
| `memory_retrieve` | **Preferred entry point.** Auto-classifies query and routes to optimal backend (query, intent_search, session_log, find_similar, or query_plan). Use instead of manually choosing a search tool. |
|
|
545
|
-
| `search` | BM25 keyword search — for exact terms, config names, error codes, filenames. Composite scoring + co-activation boost + compact mode. Collection filter supports comma-separated values. Prefer `memory_retrieve` for auto-routing. |
|
|
545
|
+
| `search` | BM25 keyword search — for exact terms, config names, error codes, filenames. Composite scoring + co-activation boost + compact mode. **v0.23.0: the keyword relevance signal is real** — the exposed FTS score is a monotonic BM25 transform (a clamp bug flattened it to a constant 1.0 through v0.22.0, making ranking metadata-only). Collection filter supports comma-separated values. Prefer `memory_retrieve` for auto-routing. |
|
|
546
546
|
| `vsearch` | Vector semantic search — for conceptual/fuzzy matching when exact keywords are unknown. **v0.22.0: non-recency queries rank by RAW cosine (`scoreBasis: "vector-cosine"`; metadata breaks exact ties only; `minScore` filters raw with no default)**; recency-intent queries keep composite. Collection filter supports comma-separated values. Prefer `memory_retrieve` for auto-routing. |
|
|
547
547
|
| `query` | Full hybrid pipeline (BM25 + vector + rerank) — general-purpose when query type is unclear. WRONG for "why" questions (use `intent_search`) or cross-session queries (use `session_log`). Prefer `memory_retrieve` for auto-routing. Intent hint, strong-signal bypass, chunk dedup, candidateLimit, MMR diversity, compact mode. |
|
|
548
548
|
| `get` | Retrieve single document by path or docid |
|
|
@@ -594,7 +594,7 @@ Registered by `clawmem setup mcp`. Available to any MCP-compatible client.
|
|
|
594
594
|
|
|
595
595
|
| Tool | Description |
|
|
596
596
|
|---|---|
|
|
597
|
-
| `memory_forget` | Search → deactivate closest match (with audit trail) |
|
|
597
|
+
| `memory_forget` | Search → deactivate closest match (with audit trail). Weak matches return a disambiguation list instead of acting (v0.23.0) |
|
|
598
598
|
| `memory_pin` | Pin a memory: lifecycle retention + priority among relevance-equivalent results (+0.3 composite boost on hook/`query`/`search` surfaces; exact-tie precedence on the raw vector routes). USE PROACTIVELY when: user states a persistent constraint, makes an architecture decision, or corrects a misconception. Don't wait for curator — pin critical decisions immediately. |
|
|
599
599
|
| `memory_snooze` | Temporarily hide a memory from context surfacing until a date. USE PROACTIVELY when `<vault-context>` repeatedly surfaces irrelevant content — snooze for 30 days instead of ignoring it. |
|
|
600
600
|
| `status` | Index health with content type distribution |
|
package/SKILL.md
CHANGED
|
@@ -94,7 +94,7 @@ All other retrieval is handled by Tier 2 hooks. **Do NOT call MCP tools speculat
|
|
|
94
94
|
| `profile` | User profile (static facts + dynamic context). |
|
|
95
95
|
| `memory_pin` | Lifecycle retention + priority among relevance-equivalent results (+0.3 composite boost on composite surfaces; exact-tie precedence on raw vector routes). Use PROACTIVELY for constraints, architecture decisions, corrections. |
|
|
96
96
|
| `memory_snooze` | Use PROACTIVELY when `<vault-context>` surfaces noise — snooze 30 days. |
|
|
97
|
-
| `memory_forget` | Deactivate a memory by closest match. Sparingly — prefer snooze. |
|
|
97
|
+
| `memory_forget` | Deactivate a memory by closest match. Sparingly — prefer snooze. Weak matches return a disambiguation list instead of acting (v0.23.0). |
|
|
98
98
|
| `build_graphs` | Temporal backbone + semantic graph after bulk ingestion. NOT after every reindex. |
|
|
99
99
|
| `timeline` | Temporal neighborhood around a doc. Progressive disclosure: search → timeline → get. |
|
|
100
100
|
| `memory_evolution_status` | How a doc's A-MEM metadata evolved over time. |
|
|
@@ -189,7 +189,7 @@ Query -> intent classification (WHY/WHEN/ENTITY/WHAT)
|
|
|
189
189
|
|
|
190
190
|
## Composite scoring (how ranking works)
|
|
191
191
|
|
|
192
|
-
Applied on the composite surfaces: hooks, `query`, `search`, and `memory_retrieve`'s keyword/hybrid/causal/complex modes. **v0.22.0: MCP `vsearch` and `memory_retrieve` semantic/discovery rank non-recency queries by RAW cosine instead** (`scoreBasis: "vector-cosine"`; metadata breaks exact ties only; `minScore` filters raw with no default); recency-intent queries keep composite everywhere.
|
|
192
|
+
Applied on the composite surfaces: hooks, `query`, `search`, and `memory_retrieve`'s keyword/hybrid/causal/complex modes. **v0.22.0: MCP `vsearch` and `memory_retrieve` semantic/discovery rank non-recency queries by RAW cosine instead** (`scoreBasis: "vector-cosine"`; metadata breaks exact ties only; `minScore` filters raw with no default); recency-intent queries keep composite everywhere. **v0.23.0:** `searchScore` on FTS surfaces is the monotonic `|bm25|/(1+|bm25|)` transform (it was a constant 1.0 through v0.22.0 due to a clamp bug — keyword relevance contributed zero ordering); FTS-transform scores and cosines are independent monotonic signals, not one calibrated scale.
|
|
193
193
|
|
|
194
194
|
```
|
|
195
195
|
compositeScore = (0.50·searchScore + 0.25·recencyScore + 0.25·confidenceScore) × qualityMultiplier × coActivationBoost
|
|
@@ -23,6 +23,8 @@ The **`query` tool** uses retrieval-tuned weights (search 0.70) derived from a h
|
|
|
23
23
|
|
|
24
24
|
Threshold note: EOS-anchored last-token embedding models (e.g. zembed-1 served correctly) produce a **compressed-high similarity band** — unrelated pairs sit near ~0.4 rather than ~0.2. Relative ordering is what matters; treat absolute `minScore` cutoffs as model-dependent rather than universal.
|
|
25
25
|
|
|
26
|
+
`searchScore` provenance (v0.23.0): on FTS/BM25 surfaces the input is the monotonic transform `|bm25|/(1+|bm25|)` of FTS5's negative-is-better `bm25()` (through v0.22.0 a clamp bug flattened it to a constant 1.0, which made composite ranking on those surfaces metadata-only); on vector surfaces it is the raw cosine. The two are independent monotonic signals on an uncalibrated common range — composite blends them where pools mix (hooks, REST hybrid), which removes the old unconditional FTS-over-vector dominance but does not make the channels numerically comparable.
|
|
27
|
+
|
|
26
28
|
## Signal breakdown
|
|
27
29
|
|
|
28
30
|
### Search score (0.0 - 1.0)
|
package/docs/guides/upgrading.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Upgrading ClawMem
|
|
2
2
|
|
|
3
|
-
Guide for upgrading between released versions. Current: **v0.
|
|
3
|
+
Guide for upgrading between released versions. Current: **v0.23.0**.
|
|
4
4
|
|
|
5
5
|
ClawMem upgrades are designed to be drop-in: pull the new version, restart any long-lived processes, and the SQLite schema auto-migrates on first open. This guide documents per-version specifics for upgrades that have additional considerations beyond the quick path below.
|
|
6
6
|
|
|
@@ -59,6 +59,19 @@ docker compose up -d reranker # /v1/rerank on :8090
|
|
|
59
59
|
|
|
60
60
|
---
|
|
61
61
|
|
|
62
|
+
## v0.23.0: monotonic BM25 exposed score
|
|
63
|
+
|
|
64
|
+
No migration command, no schema change, no re-embed. Restart long-lived processes per the quick path.
|
|
65
|
+
|
|
66
|
+
**Behavior change — the exposed FTS score is a real relevance signal.** Through v0.22.0 a clamp bug flattened every FTS result's `score` to the constant 1.0, so ranking on the BM25 surfaces (`search`, REST keyword mode, CLI search, `memory_retrieve` keyword and its semantic-mode FTS fallback, hook FTS lanes) was effectively metadata-only. The score is now `|bm25|/(1+|bm25|)` — bounded [0,1), higher is better. What to expect after upgrading:
|
|
67
|
+
|
|
68
|
+
- `search` results reorder toward keyword relevance; reported scores drop from the old flat values and vary per hit. If a workflow compared `search` scores against a hardcoded cutoff tuned to the constant-1.0 era, re-tune it (the composite floor semantics of `minScore` are unchanged; the observed values shifted).
|
|
69
|
+
- The `query` pipeline's strong-signal bypass actually fires now on unambiguous keyword queries (skipping LLM expansion, which makes those calls faster) — and no longer fires on a lone weak match.
|
|
70
|
+
- `memory_forget` targeting is stricter: weak keyword matches return a disambiguation list instead of auto-selecting. If a script relied on forget acting on any single match, it must now pass a more specific query or a path.
|
|
71
|
+
- `clawmem doctor`/curator's BM25 probe reports honestly — a near-empty vault may now show a degraded BM25 probe where it previously passed vacuously.
|
|
72
|
+
|
|
73
|
+
---
|
|
74
|
+
|
|
62
75
|
## v0.22.0: raw-cosine ranking on the direct vector routes
|
|
63
76
|
|
|
64
77
|
No migration command, no schema change, no re-embed. Restart long-lived processes per the quick path.
|
|
@@ -79,6 +79,8 @@ When the top BM25 hit scores >= 0.85 and the gap to the second hit is >= 0.15, t
|
|
|
79
79
|
|
|
80
80
|
The bypass is disabled when `intent` is provided — intent implies the query is ambiguous, so keyword confidence alone is insufficient.
|
|
81
81
|
|
|
82
|
+
Functional as of v0.23.0: through v0.22.0 a clamp bug flattened every FTS score to a constant 1.0, so with two or more hits the gap was always 0 (the bypass never fired) and with exactly one hit the gap was always 1.0 (it fired even on a garbage match). The scores are now the monotonic `|bm25|/(1+|bm25|)` transform — 0.85 corresponds to |bm25| ≥ 5.67 — and the check itself lives in one shared helper (`hasStrongFtsSignal`) used by both this pipeline and the CLI `query` command. Threshold tuning is deliberately deferred to a judged query-pipeline A/B (BACKLOG 49.3).
|
|
83
|
+
|
|
82
84
|
## Query expansion
|
|
83
85
|
|
|
84
86
|
The LLM generates lex (keyword), vec (semantic), and hyde (hypothetical answer) variants of the query, each carried as a typed `ExpandedQuery`. Variants are **routed by type**: `lex` expansions are searched on BM25 only, `vec` and `hyde` on vector only. The original query is the only leg that fans out to *both* backends. All legs are searched in parallel and fused; the original query's two lists receive 2x weight in RRF, ensuring it anchors the ranking.
|
|
@@ -8,6 +8,8 @@ Complete reference for ClawMem's MCP server tools. These let AI agents search, r
|
|
|
8
8
|
|
|
9
9
|
**Scoring regimes (v0.22.0):** the direct vector routes — `vsearch` and `memory_retrieve`'s semantic/discovery modes — rank non-recency queries by RAW vector cosine. Their `structuredContent` carries `scoreBasis: "vector-cosine"`; raw cosine values are specific to the embedding model that produced them and are NOT comparable across models or to composite scores. Document metadata — including pin — participates only inside groups of exactly-equal raw scores. On these routes `minScore` filters the raw score and has NO default (omitted = no filter; an explicit `0` is honored). Recency-intent queries ("latest…", "recently…", "yesterday…") keep the composite regime with its 0.3 default floor and report `scoreBasis: "composite"`, as do `search`, `query`, `query_plan`, and `memory_retrieve`'s keyword/hybrid/causal/complex modes. `find_similar` has always ranked by raw cosine. Rationale: on the measured vault, raw cosine ranked 16/19 judged targets #1 (MRR 0.912) while the composite stack ranked 1/19 and filtered 14/19 out entirely.
|
|
10
10
|
|
|
11
|
+
**FTS score provenance (v0.23.0):** the raw `score` on BM25/FTS results is the monotonic transform `|bm25|/(1+|bm25|)` of FTS5's negative-is-better `bm25()` value — bounded [0,1), higher is better, stable across queries. (Through v0.22.0 a clamp bug flattened it to a constant 1.0, so composite ranking on FTS surfaces was effectively metadata-only and score-threshold gates never filtered.) FTS-transform scores and vector cosines are **independent monotonic signals, not a calibrated common scale** — compare within a channel, not across channels.
|
|
12
|
+
|
|
11
13
|
**Degraded vector results (v0.21.0):** under default exclusion the vector scan escalates its depth to fill `limit` with allowed documents, up to a hard cap. When the cap prevents an exhaustive scan and the result is under-filled, `structuredContent` carries `degraded: true` with `degradedReason`: `"excluded-dominant"` (distinct excluded docs account for the shortfall — the guidance line suggests `includeInternal: true` or a refined query) or `"cap-truncation"` (shortfall driven by fragment dedup, neutral guidance). Multi-leg routes (`query`, `query_plan`, `memory_retrieve` complex mode) aggregate `degraded = any(leg)` and list per-leg reasons in `structuredContent.degradedLegs`; single-vector routes (`vsearch`, `find_similar`, `memory_retrieve`'s other modes) report the flat `degraded` + `degradedReason` pair. A small vault whose whole index is scanned without hitting the cap returns a plain short list with NO marker.
|
|
12
14
|
|
|
13
15
|
### memory_retrieve
|
|
@@ -49,17 +51,20 @@ BM25 strong-signal bypass: skips expansion when top BM25 hit >= 0.85 with gap >=
|
|
|
49
51
|
|
|
50
52
|
### search
|
|
51
53
|
|
|
52
|
-
BM25 only. Zero GPU cost.
|
|
54
|
+
BM25 only. Zero GPU cost. Ranking is composite (see the score-provenance note above); the underlying keyword relevance signal is the monotonic BM25 transform introduced in v0.23.0.
|
|
53
55
|
|
|
54
56
|
| Param | Type | Default | Description |
|
|
55
57
|
|-------|------|---------|-------------|
|
|
56
58
|
| `query` | string | required | Search query |
|
|
57
59
|
| `limit` | number | 10 | Max results |
|
|
60
|
+
| `minScore` | number | 0 | Composite-score floor (values shifted in v0.23.0 — the searchScore input is now a real relevance signal, not a constant) |
|
|
58
61
|
| `compact` | boolean | true | Compact output |
|
|
59
62
|
| `collection` | string | — | Filter by collection |
|
|
60
63
|
| `includeInternal` | boolean | false | Include system-internal `_clawmem` docs |
|
|
61
64
|
| `vault` | string | — | Named vault |
|
|
62
65
|
|
|
66
|
+
Score fields: compact results report the rounded **composite** score; non-compact results carry both `score` (raw BM25 transform) and `compositeScore`.
|
|
67
|
+
|
|
63
68
|
### vsearch
|
|
64
69
|
|
|
65
70
|
Vector only. Semantic similarity. Non-recency queries rank by RAW cosine (`scoreBasis: "vector-cosine"`, v0.22.0); recency-intent queries use the composite regime.
|
|
@@ -211,6 +216,8 @@ Permanently deactivate a memory.
|
|
|
211
216
|
|
|
212
217
|
**Search behavior (v0.2.6+, all three tools):** Query matching cascades through four strategies: exact path match → BM25 full-text → title-token overlap → vector similarity. This prevents "No matching memory found" errors when the document exists but BM25 fails to match (e.g., too many AND'd terms). Path-like queries (containing `/` or ending in `.md`) try direct path matching first. `memory_forget` requires higher confidence to act — ambiguous matches return candidates instead of mutating.
|
|
213
218
|
|
|
219
|
+
**Targeting confidence (v0.23.0):** the confidence gate (`score ≥ 0.7`, or a ≥ 0.2 gap to the runner-up when more than one candidate exists) is now live for BM25 candidates — through v0.22.0 every FTS candidate carried a constant score of 1.0, so `memory_forget` treated ANY keyword match as high-confidence and auto-selected it. Weak matches — including a lone weak match — now return the candidate list for disambiguation instead of acting. Stricter, safer targeting for a destructive operation.
|
|
220
|
+
|
|
214
221
|
## Lifecycle
|
|
215
222
|
|
|
216
223
|
### lifecycle_status
|
package/package.json
CHANGED
package/src/clawmem.ts
CHANGED
|
@@ -58,7 +58,7 @@ import { formatSearchResults, type OutputFormat } from "./formatter.ts";
|
|
|
58
58
|
import { indexCollection, parseDocument } from "./indexer.ts";
|
|
59
59
|
import { detectBeadsProject } from "./beads.ts";
|
|
60
60
|
import { applyCompositeScoring, hasRecencyIntent, type EnrichedResult } from "./memory.ts";
|
|
61
|
-
import { enrichResults, reciprocalRankFusion, toRanked, type RankedResult } from "./search-utils.ts";
|
|
61
|
+
import { enrichResults, reciprocalRankFusion, toRanked, hasStrongFtsSignal, type RankedResult } from "./search-utils.ts";
|
|
62
62
|
import { splitDocument } from "./splitter.ts";
|
|
63
63
|
import { getProfile, updateProfile, isProfileStale } from "./profile.ts";
|
|
64
64
|
import { regenerateAllDirectoryContexts } from "./directory-context.ts";
|
|
@@ -1095,9 +1095,7 @@ async function cmdQuery(args: string[]) {
|
|
|
1095
1095
|
|
|
1096
1096
|
// Step 1: BM25 for strong signal check
|
|
1097
1097
|
const ftsResults = s.searchFTS(query, 20);
|
|
1098
|
-
const
|
|
1099
|
-
const secondScore = ftsResults[1]?.score ?? 0;
|
|
1100
|
-
const strongSignal = topScore >= 0.85 && (topScore - secondScore) >= 0.15;
|
|
1098
|
+
const strongSignal = hasStrongFtsSignal(ftsResults);
|
|
1101
1099
|
|
|
1102
1100
|
// Step 2: Query expansion (skip if strong BM25 signal). expandQuery now returns
|
|
1103
1101
|
// typed ExpandedQuery[] (lex/vec/hyde) — no more brittle string re-parsing, and
|
package/src/mcp.ts
CHANGED
|
@@ -33,7 +33,7 @@ import {
|
|
|
33
33
|
type EnrichedResult,
|
|
34
34
|
type CoActivationFn,
|
|
35
35
|
} from "./memory.ts";
|
|
36
|
-
import { enrichResults, reciprocalRankFusion, toRanked, blendRerank, type RankedResult } from "./search-utils.ts";
|
|
36
|
+
import { enrichResults, reciprocalRankFusion, toRanked, blendRerank, hasStrongFtsSignal, attachRrfScores, type RankedResult } from "./search-utils.ts";
|
|
37
37
|
import { selectScoringRegime, rankRawPrimary, VECTOR_SCORE_BASIS, COMPOSITE_SCORE_BASIS } from "./scoring-regime.ts";
|
|
38
38
|
import { applyMMRDiversity } from "./mmr.ts";
|
|
39
39
|
import { indexCollection, type IndexStats } from "./indexer.ts";
|
|
@@ -330,11 +330,7 @@ This is the recommended entry point for ALL memory queries.`,
|
|
|
330
330
|
} catch (e) { rethrowIfFatalVectorError(e); /* else: no vectors */ }
|
|
331
331
|
const rrfWeights = intent.intent === 'WHY' ? [1.0, 1.5] : intent.intent === 'WHEN' ? [1.5, 1.0] : [1.0, 1.0];
|
|
332
332
|
const fusedRanked = reciprocalRankFusion([bm25Results.map(toRanked), vecResults.map(toRanked)], rrfWeights);
|
|
333
|
-
|
|
334
|
-
let fused: SearchResult[] = fusedRanked.map(fr => {
|
|
335
|
-
const orig = allSearch.find(r => r.filepath === fr.file);
|
|
336
|
-
return orig ? { ...orig, score: fr.score } : null;
|
|
337
|
-
}).filter((r): r is SearchResult => r !== null);
|
|
333
|
+
let fused: SearchResult[] = attachRrfScores(fusedRanked, [...bm25Results, ...vecResults]);
|
|
338
334
|
|
|
339
335
|
if (intent.intent === 'WHY' || intent.intent === 'ENTITY') {
|
|
340
336
|
try {
|
|
@@ -460,11 +456,7 @@ This is the recommended entry point for ALL memory queries.`,
|
|
|
460
456
|
} catch (e) { rethrowIfFatalVectorError(e); /* */ }
|
|
461
457
|
if (vec.length > 0) {
|
|
462
458
|
const fusedRanked = reciprocalRankFusion([bm25.map(toRanked), vec.map(toRanked)], [1.0, 1.0]);
|
|
463
|
-
|
|
464
|
-
results = fusedRanked.map(fr => {
|
|
465
|
-
const orig = allSearch.find(r => r.filepath === fr.file);
|
|
466
|
-
return orig ? { ...orig, score: fr.score } : null;
|
|
467
|
-
}).filter((r): r is SearchResult => r !== null);
|
|
459
|
+
results = attachRrfScores(fusedRanked, [...bm25, ...vec]);
|
|
468
460
|
} else {
|
|
469
461
|
results = bm25;
|
|
470
462
|
}
|
|
@@ -747,12 +739,9 @@ This is the recommended entry point for ALL memory queries.`,
|
|
|
747
739
|
: undefined;
|
|
748
740
|
const excl = resolveExcludedCollections(includeInternal, collections);
|
|
749
741
|
const initialFts = store.searchFTS(query, 20, undefined, collections, dateRange, excl);
|
|
750
|
-
const topScore = initialFts.length > 0 ? Math.abs(initialFts[0]!.score) : 0;
|
|
751
|
-
const secondScore = initialFts.length > 1 ? Math.abs(initialFts[1]!.score) : 0;
|
|
752
742
|
// When intent is provided, disable strong-signal bypass — the obvious BM25
|
|
753
743
|
// match may not be what the caller wants (e.g. "performance" with intent "web page load times")
|
|
754
|
-
const hasStrongSignal = !intent && initialFts
|
|
755
|
-
&& topScore >= 0.85 && (topScore - secondScore) >= 0.15;
|
|
744
|
+
const hasStrongSignal = !intent && hasStrongFtsSignal(initialFts);
|
|
756
745
|
|
|
757
746
|
// Step 2: Query expansion (skipped if strong signal). Typed routing —
|
|
758
747
|
// original → BOTH FTS + vector (2× RRF anchor), lex → FTS only, vec/hyde → vector only.
|
|
@@ -1101,9 +1090,11 @@ This is the recommended entry point for ALL memory queries.`,
|
|
|
1101
1090
|
|
|
1102
1091
|
const top = candidates[0]!;
|
|
1103
1092
|
|
|
1104
|
-
// Clear winner: high score OR significant gap to #2
|
|
1093
|
+
// Clear winner: high score OR significant gap to #2. The gap clause requires an
|
|
1094
|
+
// actual #2 to gap against — a lone weak candidate must not qualify (destructive
|
|
1095
|
+
// ops would otherwise auto-select a garbage single match).
|
|
1105
1096
|
const gap = candidates.length > 1 ? top.score - candidates[1]!.score : 1.0;
|
|
1106
|
-
const confident = top.score >= 0.7 || gap >= 0.2;
|
|
1097
|
+
const confident = top.score >= 0.7 || (candidates.length > 1 && gap >= 0.2);
|
|
1107
1098
|
|
|
1108
1099
|
// For destructive ops (forget), require higher confidence
|
|
1109
1100
|
if (destructive && !confident) {
|
|
@@ -1724,11 +1715,7 @@ This is the recommended entry point for ALL memory queries.`,
|
|
|
1724
1715
|
const fusedRanked = reciprocalRankFusion([bm25Results.map(toRanked), vecResults.map(toRanked)], rrfWeights);
|
|
1725
1716
|
|
|
1726
1717
|
// Map RRF results back to SearchResult with updated scores
|
|
1727
|
-
const
|
|
1728
|
-
const fused: SearchResult[] = fusedRanked.map(fr => {
|
|
1729
|
-
const original = allSearchResults.find(r => r.filepath === fr.file);
|
|
1730
|
-
return original ? { ...original, score: fr.score } : null;
|
|
1731
|
-
}).filter((r): r is SearchResult => r !== null);
|
|
1718
|
+
const fused: SearchResult[] = attachRrfScores(fusedRanked, [...bm25Results, ...vecResults]);
|
|
1732
1719
|
|
|
1733
1720
|
// Step 4: Graph expansion (if enabled and intent allows)
|
|
1734
1721
|
let expanded = fused;
|
|
@@ -1922,10 +1909,11 @@ This is the recommended entry point for ALL memory queries.`,
|
|
|
1922
1909
|
const vec = detGraph.results;
|
|
1923
1910
|
if (detGraph.degraded && detGraph.degradedReason) degradedLegs.push({ leg: `clause${clauseIdx}:graph:vector`, reason: detGraph.degradedReason });
|
|
1924
1911
|
const fused = reciprocalRankFusion([bm25.map(toRanked), vec.map(toRanked)], [1.0, 1.0]);
|
|
1925
|
-
|
|
1926
|
-
|
|
1927
|
-
|
|
1928
|
-
|
|
1912
|
+
// Carry the RRF-fused score (parity with causal + intent_search): the graph
|
|
1913
|
+
// traversal below anchors on `score`, which must be RRF-scale, not raw single-channel.
|
|
1914
|
+
// vec-first: this site historically preferred the vector variant on duplicate
|
|
1915
|
+
// paths (its pre-helper Map construction let later entries overwrite) — preserved.
|
|
1916
|
+
results = attachRrfScores(fused, [...vec, ...bm25]);
|
|
1929
1917
|
|
|
1930
1918
|
// Graph expansion for WHY/ENTITY
|
|
1931
1919
|
if (intent.intent === 'WHY' || intent.intent === 'ENTITY') {
|
package/src/search-utils.ts
CHANGED
|
@@ -201,3 +201,35 @@ export function toRanked(r: SearchResult): RankedResult {
|
|
|
201
201
|
score: r.score,
|
|
202
202
|
};
|
|
203
203
|
}
|
|
204
|
+
|
|
205
|
+
/**
|
|
206
|
+
* Map RRF-fused results back to their originating SearchResults, carrying the FUSED
|
|
207
|
+
* score. Downstream consumers — graph-traversal anchors, composite enrichment — must
|
|
208
|
+
* see the fusion's relevance scale, not a raw single-channel score (S49.1 U2: the
|
|
209
|
+
* query_plan graph clause used to drop the fused score, anchoring traversal on raw
|
|
210
|
+
* FTS values). First occurrence wins when a doc appears in multiple channels
|
|
211
|
+
* (matching the previous `find`-based lookups); fused entries with no originating
|
|
212
|
+
* SearchResult are dropped.
|
|
213
|
+
*/
|
|
214
|
+
export function attachRrfScores(fused: RankedResult[], originals: SearchResult[]): SearchResult[] {
|
|
215
|
+
const byPath = new Map<string, SearchResult>();
|
|
216
|
+
for (const r of originals) {
|
|
217
|
+
if (!byPath.has(r.filepath)) byPath.set(r.filepath, r);
|
|
218
|
+
}
|
|
219
|
+
return fused
|
|
220
|
+
.map(fr => { const orig = byPath.get(fr.file); return orig ? { ...orig, score: fr.score } : null; })
|
|
221
|
+
.filter((r): r is SearchResult => r !== null);
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
/**
|
|
225
|
+
* BM25 strong-signal check on ftsScoreFromBm25-scale results: the top hit is strong
|
|
226
|
+
* (≥ 0.85 ⇔ |bm25| ≥ 5.67) AND clearly separated from #2 (gap ≥ 0.15; a missing #2
|
|
227
|
+
* counts as 0, so a lone WEAK hit does not qualify). Callers use this to skip LLM
|
|
228
|
+
* query expansion when the keyword answer is unambiguous.
|
|
229
|
+
*/
|
|
230
|
+
export function hasStrongFtsSignal(results: SearchResult[]): boolean {
|
|
231
|
+
if (results.length === 0) return false;
|
|
232
|
+
const top = results[0]!.score;
|
|
233
|
+
const second = results.length > 1 ? results[1]!.score : 0;
|
|
234
|
+
return top >= 0.85 && (top - second) >= 0.15;
|
|
235
|
+
}
|
package/src/server.ts
CHANGED
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
|
|
13
13
|
import type { Server } from "bun";
|
|
14
14
|
import type { Store, SearchResult, TimelineResult } from "./store.ts";
|
|
15
|
-
import { enrichResults, reciprocalRankFusion, toRanked } from "./search-utils.ts";
|
|
15
|
+
import { enrichResults, reciprocalRankFusion, toRanked, attachRrfScores } from "./search-utils.ts";
|
|
16
16
|
import { applyCompositeScoring, hasRecencyIntent, type EnrichedResult } from "./memory.ts";
|
|
17
17
|
import { applyMMRDiversity } from "./mmr.ts";
|
|
18
18
|
import { listCollections } from "./collections.ts";
|
|
@@ -617,11 +617,7 @@ async function handleRetrieve(req: Request, _url: URL, store: Store): Promise<Re
|
|
|
617
617
|
} catch (e) { rethrowIfFatalVectorError(e); /* vector unavailable */ }
|
|
618
618
|
const weights = intent.intent === "WHEN" ? [1.5, 1.0] : [1.0, 1.5];
|
|
619
619
|
const fused = reciprocalRankFusion([bm25.map(toRanked), vec.map(toRanked)], weights);
|
|
620
|
-
|
|
621
|
-
results = fused.map(fr => {
|
|
622
|
-
const orig = allResults.find(r => r.filepath === fr.file);
|
|
623
|
-
return orig ? { ...orig, score: fr.score } : null;
|
|
624
|
-
}).filter((r): r is SearchResult => r !== null);
|
|
620
|
+
results = attachRrfScores(fused, [...bm25, ...vec]);
|
|
625
621
|
} else if (mode === "keyword") {
|
|
626
622
|
results = store.searchFTS(query, limit * 2, undefined, collections);
|
|
627
623
|
} else if (mode === "semantic") {
|
package/src/store.ts
CHANGED
|
@@ -3506,6 +3506,20 @@ function buildFTS5Query(query: string): string | null {
|
|
|
3506
3506
|
return terms.map(t => `"${t}"*`).join(' AND ');
|
|
3507
3507
|
}
|
|
3508
3508
|
|
|
3509
|
+
/**
|
|
3510
|
+
* Convert an FTS5 bm25() value into a stable [0,1) relevance score where higher is better.
|
|
3511
|
+
*
|
|
3512
|
+
* FTS5's bm25() is negative-is-better: it returns -1 × the BM25 score, so it is ≤ 0 for
|
|
3513
|
+
* every match. The transform is per-row and monotonic in match strength (|bm25|/(1+|bm25|))
|
|
3514
|
+
* with no per-query normalization, so cross-query comparisons, minScore filters, and the
|
|
3515
|
+
* strong-signal bypass all stay meaningful. A hypothetical positive input clamps to 0
|
|
3516
|
+
* rather than inverting the ordering.
|
|
3517
|
+
*/
|
|
3518
|
+
export function ftsScoreFromBm25(bm25Score: number): number {
|
|
3519
|
+
const m = Math.max(0, -bm25Score);
|
|
3520
|
+
return m / (1 + m);
|
|
3521
|
+
}
|
|
3522
|
+
|
|
3509
3523
|
export function searchFTS(db: Database, query: string, limit: number = 20, collectionId?: number, collections?: string[], dateRange?: { start: string; end: string }, excludeCollections?: string[]): SearchResult[] {
|
|
3510
3524
|
const ftsQuery = buildFTS5Query(query);
|
|
3511
3525
|
if (!ftsQuery) return [];
|
|
@@ -3558,9 +3572,7 @@ export function searchFTS(db: Database, query: string, limit: number = 20, colle
|
|
|
3558
3572
|
const rows = db.prepare(sql).all(...params) as { filepath: string; display_path: string; title: string; body: string; hash: string; modified_at: string; bm25_score: number }[];
|
|
3559
3573
|
return rows.map(row => {
|
|
3560
3574
|
const collectionName = row.filepath.split('//')[1]?.split('/')[0] || "";
|
|
3561
|
-
|
|
3562
|
-
// Avoid per-query normalization so "strong signal" heuristics can work.
|
|
3563
|
-
const score = 1 / (1 + Math.max(0, row.bm25_score));
|
|
3575
|
+
const score = ftsScoreFromBm25(row.bm25_score);
|
|
3564
3576
|
return {
|
|
3565
3577
|
filepath: row.filepath,
|
|
3566
3578
|
displayPath: row.display_path,
|