clawmem 0.11.0 → 0.11.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +1 -1
- package/CLAUDE.md +1 -1
- package/package.json +2 -2
- package/src/clawmem.ts +19 -16
- package/src/hooks/context-surfacing.ts +8 -2
- package/src/llm.ts +106 -36
- package/src/mcp.ts +32 -12
- package/src/store.ts +73 -14
package/AGENTS.md
CHANGED
|
@@ -546,7 +546,7 @@ User Query + optional intent hint
|
|
|
546
546
|
→ Temporal Extraction (regex date range from query: "last week", "March 2026" → WHERE modified_at BETWEEN filters)
|
|
547
547
|
→ BM25 Probe → Strong Signal Check (skip expansion if top hit ≥ 0.85 with gap ≥ 0.15; disabled when intent provided)
|
|
548
548
|
→ Query Expansion (LLM generates text variants; intent steers expansion prompt)
|
|
549
|
-
→ Parallel: BM25(original) + Vector(original) + BM25(
|
|
549
|
+
→ Parallel (typed routing): BM25(original) + Vector(original) + BM25(lex expansions) + Vector(vec/hyde expansions)
|
|
550
550
|
+ Temporal Proximity (date-range filtered, if temporal constraint extracted)
|
|
551
551
|
+ Entity Graph (conditional 1-hop entity walk from top seeds, if entity signals present)
|
|
552
552
|
→ Original query lists get positional 2× weight in RRF; expanded get 1×
|
package/CLAUDE.md
CHANGED
|
@@ -546,7 +546,7 @@ User Query + optional intent hint
|
|
|
546
546
|
→ Temporal Extraction (regex date range from query: "last week", "March 2026" → WHERE modified_at BETWEEN filters)
|
|
547
547
|
→ BM25 Probe → Strong Signal Check (skip expansion if top hit ≥ 0.85 with gap ≥ 0.15; disabled when intent provided)
|
|
548
548
|
→ Query Expansion (LLM generates text variants; intent steers expansion prompt)
|
|
549
|
-
→ Parallel: BM25(original) + Vector(original) + BM25(
|
|
549
|
+
→ Parallel (typed routing): BM25(original) + Vector(original) + BM25(lex expansions) + Vector(vec/hyde expansions)
|
|
550
550
|
+ Temporal Proximity (date-range filtered, if temporal constraint extracted)
|
|
551
551
|
+ Entity Graph (conditional 1-hop entity walk from top seeds, if entity signals present)
|
|
552
552
|
→ Original query lists get positional 2× weight in RRF; expanded get 1×
|
package/package.json
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "clawmem",
|
|
3
|
-
"version": "0.11.
|
|
3
|
+
"version": "0.11.2",
|
|
4
4
|
"description": "On-device memory layer for AI agents. Claude Code, OpenClaw, and Hermes. Hooks + MCP server + hybrid RAG search.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
7
|
-
"clawmem": "
|
|
7
|
+
"clawmem": "bin/clawmem"
|
|
8
8
|
},
|
|
9
9
|
"files": [
|
|
10
10
|
"bin/",
|
package/src/clawmem.ts
CHANGED
|
@@ -13,6 +13,7 @@ import {
|
|
|
13
13
|
canonicalDocId,
|
|
14
14
|
type Store,
|
|
15
15
|
type SearchResult,
|
|
16
|
+
type ExpandedQuery,
|
|
16
17
|
DEFAULT_EMBED_MODEL,
|
|
17
18
|
DEFAULT_QUERY_MODEL,
|
|
18
19
|
DEFAULT_RERANK_MODEL,
|
|
@@ -939,19 +940,13 @@ async function cmdQuery(args: string[]) {
|
|
|
939
940
|
const secondScore = ftsResults[1]?.score ?? 0;
|
|
940
941
|
const strongSignal = topScore >= 0.85 && (topScore - secondScore) >= 0.15;
|
|
941
942
|
|
|
942
|
-
// Step 2: Query expansion (skip if strong BM25 signal)
|
|
943
|
-
|
|
943
|
+
// Step 2: Query expansion (skip if strong BM25 signal). expandQuery now returns
|
|
944
|
+
// typed ExpandedQuery[] (lex/vec/hyde) — no more brittle string re-parsing, and
|
|
945
|
+
// the original query is no longer echoed back as a phantom "vec" expansion.
|
|
946
|
+
let expandedQueries: ExpandedQuery[] = [];
|
|
944
947
|
if (!strongSignal) {
|
|
945
948
|
try {
|
|
946
|
-
|
|
947
|
-
expandedQueries = expanded.map(text => {
|
|
948
|
-
// Parse "type: text" format from expansion
|
|
949
|
-
const colonIdx = text.indexOf(": ");
|
|
950
|
-
if (colonIdx > 0 && colonIdx < 5) {
|
|
951
|
-
return { type: text.slice(0, colonIdx), text: text.slice(colonIdx + 2) };
|
|
952
|
-
}
|
|
953
|
-
return { type: "vec", text };
|
|
954
|
-
});
|
|
949
|
+
expandedQueries = await s.expandQuery(query, DEFAULT_QUERY_MODEL);
|
|
955
950
|
} catch {
|
|
956
951
|
// Fallback: no expansion
|
|
957
952
|
}
|
|
@@ -959,20 +954,27 @@ async function cmdQuery(args: string[]) {
|
|
|
959
954
|
|
|
960
955
|
// Step 3: Parallel searches
|
|
961
956
|
const allRanked: { results: RankedResult[]; weight: number }[] = [];
|
|
957
|
+
// Retain the raw SearchResult from every leg (original + typed expansions) so a
|
|
958
|
+
// candidate found ONLY via an expansion leg survives Step 8's resultMap lookup.
|
|
959
|
+
const candidateResults: SearchResult[] = [];
|
|
962
960
|
|
|
963
961
|
// Original query BM25 + vec (weight 2x)
|
|
964
962
|
allRanked.push({ results: ftsResults.map(toRanked), weight: 2 });
|
|
963
|
+
candidateResults.push(...ftsResults);
|
|
965
964
|
const vecResults = await s.searchVec(query, DEFAULT_EMBED_MODEL, 20);
|
|
966
965
|
allRanked.push({ results: vecResults.map(toRanked), weight: 2 });
|
|
966
|
+
candidateResults.push(...vecResults);
|
|
967
967
|
|
|
968
|
-
// Expanded queries (weight 1x)
|
|
968
|
+
// Expanded queries (weight 1x): lex → FTS, vec/hyde → vector
|
|
969
969
|
for (const eq of expandedQueries) {
|
|
970
970
|
if (eq.type === "lex") {
|
|
971
|
-
const r = s.searchFTS(eq.
|
|
971
|
+
const r = s.searchFTS(eq.query, 20);
|
|
972
972
|
allRanked.push({ results: r.map(toRanked), weight: 1 });
|
|
973
|
+
candidateResults.push(...r);
|
|
973
974
|
} else {
|
|
974
|
-
const r = await s.searchVec(eq.
|
|
975
|
+
const r = await s.searchVec(eq.query, DEFAULT_EMBED_MODEL, 20);
|
|
975
976
|
allRanked.push({ results: r.map(toRanked), weight: 1 });
|
|
977
|
+
candidateResults.push(...r);
|
|
976
978
|
}
|
|
977
979
|
}
|
|
978
980
|
|
|
@@ -1009,9 +1011,10 @@ async function cmdQuery(args: string[]) {
|
|
|
1009
1011
|
});
|
|
1010
1012
|
blended.sort((a, b) => b.score - a.score);
|
|
1011
1013
|
|
|
1012
|
-
// Step 8: Map back to full results and apply composite scoring
|
|
1014
|
+
// Step 8: Map back to full results and apply composite scoring. Build the map from
|
|
1015
|
+
// ALL legs (incl. typed expansions) so expansion-only candidates aren't dropped.
|
|
1013
1016
|
const resultMap = new Map(
|
|
1014
|
-
|
|
1017
|
+
candidateResults.map(r => [r.filepath, r])
|
|
1015
1018
|
);
|
|
1016
1019
|
const fullResults = blended
|
|
1017
1020
|
.map(b => resultMap.get(b.file))
|
|
@@ -264,8 +264,14 @@ export async function contextSurfacing(
|
|
|
264
264
|
const seen = new Set(results.map(r => r.filepath));
|
|
265
265
|
for (const eq of expanded.slice(0, 3)) {
|
|
266
266
|
if (Date.now() - startTime > 6000) break; // hard stop at 6s
|
|
267
|
-
|
|
268
|
-
|
|
267
|
+
// Typed routing: lex → FTS; vec/hyde → vector (deep profile + time budget only).
|
|
268
|
+
let hits: SearchResult[] = [];
|
|
269
|
+
if (eq.type === 'lex') {
|
|
270
|
+
hits = store.searchFTS(eq.query, 5);
|
|
271
|
+
} else if (profile.useVector) {
|
|
272
|
+
try { hits = await store.searchVec(eq.query, DEFAULT_EMBED_MODEL, 5); } catch { /* vector leg non-fatal */ }
|
|
273
|
+
}
|
|
274
|
+
for (const r of hits) {
|
|
269
275
|
if (!seen.has(r.filepath)) {
|
|
270
276
|
seen.add(r.filepath);
|
|
271
277
|
results.push(r);
|
package/src/llm.ts
CHANGED
|
@@ -141,6 +141,88 @@ export type Queryable = {
|
|
|
141
141
|
text: string;
|
|
142
142
|
};
|
|
143
143
|
|
|
144
|
+
/**
|
|
145
|
+
* Template-residue / leak patterns that indicate an expansion line is NOT a real
|
|
146
|
+
* query. The qmd-query-expansion finetune, when prompted out of distribution,
|
|
147
|
+
* echoed the old verbose prompt's format hints and leaked Qwen3 thinking tags;
|
|
148
|
+
* these never belong in a search query. Kept as a defensive guard even though the
|
|
149
|
+
* terse QMD-faithful prompt (expandQueryRemote, 2026-06-23) fixed generation.
|
|
150
|
+
*/
|
|
151
|
+
const EXPANSION_JUNK_PATTERNS: RegExp[] = [
|
|
152
|
+
/<\/?think>/i,
|
|
153
|
+
/keyword search terms \(/i,
|
|
154
|
+
/semantic search queries \(/i,
|
|
155
|
+
/hypothetical document passage that answers the query/i,
|
|
156
|
+
];
|
|
157
|
+
|
|
158
|
+
/** True if an expansion text is empty or matches a known template-residue pattern. */
|
|
159
|
+
export function isJunkExpansion(text: string): boolean {
|
|
160
|
+
const t = text.trim();
|
|
161
|
+
if (!t) return true;
|
|
162
|
+
return EXPANSION_JUNK_PATTERNS.some(re => re.test(t));
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/**
|
|
166
|
+
* Strip stray Qwen3 `/no_think` / `/think` control tokens the model sometimes
|
|
167
|
+
* echoes mid-line (generateRemote appends ` /no_think`, and the finetune can copy
|
|
168
|
+
* it into a hyde passage). Only matches the token when it stands alone (start/space
|
|
169
|
+
* bounded) so real paths like `src/think.ts` are never touched. Unlike the `<think>`
|
|
170
|
+
* tag (which signals reasoning leakage → whole line rejected), a stray control token
|
|
171
|
+
* just gets cleaned out so the otherwise-good line survives.
|
|
172
|
+
*/
|
|
173
|
+
function stripControlTokens(text: string): string {
|
|
174
|
+
return text.replace(/(?:^|\s)\/(?:no_)?think(?=\s|$)/gi, " ").replace(/\s+/g, " ").trim();
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/**
|
|
178
|
+
* Shared guard for query-expansion output. Drops empty / template-residue /
|
|
179
|
+
* think-tag lines and de-duplicates by (type, text). Used by BOTH the llm.ts
|
|
180
|
+
* parsers and the store.ts wrapper (defense-in-depth across every provider).
|
|
181
|
+
* Does NOT inject a fallback — callers decide what to do with an empty result,
|
|
182
|
+
* and does NOT require the original query terms to appear (that would reject
|
|
183
|
+
* legitimate synonyms and keyword-only expansions, e.g. a future zegen lex leg).
|
|
184
|
+
*/
|
|
185
|
+
export function sanitizeExpandedQueries(items: Queryable[]): Queryable[] {
|
|
186
|
+
const seen = new Set<string>();
|
|
187
|
+
const out: Queryable[] = [];
|
|
188
|
+
for (const q of items) {
|
|
189
|
+
const text = stripControlTokens(q.text);
|
|
190
|
+
if (isJunkExpansion(text)) continue;
|
|
191
|
+
const key = `${q.type}:${text}`;
|
|
192
|
+
if (seen.has(key)) continue;
|
|
193
|
+
seen.add(key);
|
|
194
|
+
out.push({ type: q.type, text });
|
|
195
|
+
}
|
|
196
|
+
return out;
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
/**
|
|
200
|
+
* Typed fallback expansion set, used when generation fails or sanitization leaves
|
|
201
|
+
* nothing usable. lex+vec reuse the original query; hyde gets a minimal stub so the
|
|
202
|
+
* vector leg still has a hypothetical-document signal.
|
|
203
|
+
*/
|
|
204
|
+
export function expansionFallback(query: string, includeLexical: boolean = true): Queryable[] {
|
|
205
|
+
const out: Queryable[] = [
|
|
206
|
+
{ type: 'vec', text: query },
|
|
207
|
+
{ type: 'hyde', text: `Information about ${query}` },
|
|
208
|
+
];
|
|
209
|
+
if (includeLexical) out.unshift({ type: 'lex', text: query });
|
|
210
|
+
return out;
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/**
|
|
214
|
+
* True if `items` is exactly the typed fallback set for `query` — i.e. what every
|
|
215
|
+
* llm.expandQuery failure path returns. The store wrapper uses this to detect a
|
|
216
|
+
* leaked generation failure so it can return an expansions-only form WITHOUT
|
|
217
|
+
* caching it (a transient failure must not poison the cache). Compares against the
|
|
218
|
+
* default (lexical-included) fallback, which is what the store always requests.
|
|
219
|
+
*/
|
|
220
|
+
export function isFallbackExpansion(items: Queryable[], query: string): boolean {
|
|
221
|
+
const fb = expansionFallback(query);
|
|
222
|
+
return items.length === fb.length
|
|
223
|
+
&& items.every((q, i) => q.type === fb[i]!.type && q.text === fb[i]!.text);
|
|
224
|
+
}
|
|
225
|
+
|
|
144
226
|
/**
|
|
145
227
|
* Document to rerank
|
|
146
228
|
*/
|
|
@@ -1074,24 +1156,22 @@ export class LlamaCpp implements LLM {
|
|
|
1074
1156
|
}
|
|
1075
1157
|
|
|
1076
1158
|
private async expandQueryRemote(query: string, includeLexical: boolean, context?: string, intent?: string): Promise<Queryable[]> {
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1159
|
+
// QMD-faithful terse prompt. The qmd-query-expansion-1.7B finetune was trained
|
|
1160
|
+
// on "/no_think Expand this search query: X" (cf. QMD src/llm.ts:1467). The prior
|
|
1161
|
+
// verbose prose prompt was out-of-distribution: the model echoed the template
|
|
1162
|
+
// ("lex: keyword search terms (") and leaked </think> (verified live 2026-06-23).
|
|
1163
|
+
let prompt = intent
|
|
1164
|
+
? `/no_think Expand this search query: ${query}\nQuery intent: ${intent}`
|
|
1165
|
+
: `/no_think Expand this search query: ${query}`;
|
|
1166
|
+
if (context) prompt += `\nContext: ${context}`;
|
|
1085
1167
|
|
|
1086
1168
|
const result = await this.generateRemote(prompt, 500, 0.7);
|
|
1087
1169
|
if (!result?.text) {
|
|
1088
|
-
|
|
1089
|
-
if (includeLexical) fallback.unshift({ type: 'lex', text: query });
|
|
1090
|
-
return fallback;
|
|
1170
|
+
return expansionFallback(query, includeLexical);
|
|
1091
1171
|
}
|
|
1092
1172
|
|
|
1093
1173
|
const lines = result.text.trim().split("\n");
|
|
1094
|
-
const
|
|
1174
|
+
const parsed: Queryable[] = lines.map(line => {
|
|
1095
1175
|
const colonIdx = line.indexOf(":");
|
|
1096
1176
|
if (colonIdx === -1) return null;
|
|
1097
1177
|
const type = line.slice(0, colonIdx).trim();
|
|
@@ -1101,16 +1181,11 @@ Output:`;
|
|
|
1101
1181
|
return { type: type as QueryType, text };
|
|
1102
1182
|
}).filter((q): q is Queryable => q !== null);
|
|
1103
1183
|
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
if (!includeLexical) {
|
|
1111
|
-
return queryables.filter(q => q.type !== 'lex');
|
|
1112
|
-
}
|
|
1113
|
-
return queryables;
|
|
1184
|
+
// Drop template residue / <think> leaks / dups, then scope to requested types.
|
|
1185
|
+
const cleaned = sanitizeExpandedQueries(parsed);
|
|
1186
|
+
const scoped = includeLexical ? cleaned : cleaned.filter(q => q.type !== 'lex');
|
|
1187
|
+
if (scoped.length === 0) return expansionFallback(query, includeLexical);
|
|
1188
|
+
return scoped;
|
|
1114
1189
|
}
|
|
1115
1190
|
|
|
1116
1191
|
async modelExists(modelUri: string): Promise<ModelInfo> {
|
|
@@ -1148,10 +1223,8 @@ Output:`;
|
|
|
1148
1223
|
// Remote is in cooldown (pre-existing or just set) — fall through to local
|
|
1149
1224
|
if (this.remoteLlmUrl && this.isRemoteLlmDown()) {
|
|
1150
1225
|
if (process.env.CLAWMEM_NO_LOCAL_MODELS === "true") {
|
|
1151
|
-
// Can't fall back — return passthrough
|
|
1152
|
-
|
|
1153
|
-
if (includeLexical) fallback.unshift({ type: 'lex', text: query });
|
|
1154
|
-
return fallback;
|
|
1226
|
+
// Can't fall back to local inference — return the typed passthrough set
|
|
1227
|
+
return expansionFallback(query, includeLexical);
|
|
1155
1228
|
}
|
|
1156
1229
|
this.noteRemoteFallback(
|
|
1157
1230
|
"llm",
|
|
@@ -1230,7 +1303,7 @@ Final Output:`;
|
|
|
1230
1303
|
});
|
|
1231
1304
|
|
|
1232
1305
|
const lines = result.trim().split("\n");
|
|
1233
|
-
const
|
|
1306
|
+
const parsed: Queryable[] = lines.map(line => {
|
|
1234
1307
|
const colonIdx = line.indexOf(":");
|
|
1235
1308
|
if (colonIdx === -1) return null;
|
|
1236
1309
|
const type = line.slice(0, colonIdx).trim();
|
|
@@ -1239,17 +1312,14 @@ Final Output:`;
|
|
|
1239
1312
|
return { type: type as QueryType, text };
|
|
1240
1313
|
}).filter((q): q is Queryable => q !== null);
|
|
1241
1314
|
|
|
1242
|
-
//
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
return
|
|
1315
|
+
// Same guard as the remote path — drop residue/dups, scope to requested types.
|
|
1316
|
+
const cleaned = sanitizeExpandedQueries(parsed);
|
|
1317
|
+
const scoped = includeLexical ? cleaned : cleaned.filter(q => q.type !== 'lex');
|
|
1318
|
+
if (scoped.length === 0) return expansionFallback(query, includeLexical);
|
|
1319
|
+
return scoped;
|
|
1247
1320
|
} catch (error) {
|
|
1248
1321
|
console.error("Structured query expansion failed:", error);
|
|
1249
|
-
|
|
1250
|
-
const fallback: Queryable[] = [{ type: 'vec', text: query }];
|
|
1251
|
-
if (includeLexical) fallback.unshift({ type: 'lex', text: query });
|
|
1252
|
-
return fallback;
|
|
1322
|
+
return expansionFallback(query, includeLexical);
|
|
1253
1323
|
} finally {
|
|
1254
1324
|
await genContext.dispose();
|
|
1255
1325
|
}
|
package/src/mcp.ts
CHANGED
|
@@ -629,19 +629,38 @@ This is the recommended entry point for ALL memory queries.`,
|
|
|
629
629
|
const hasStrongSignal = !intent && initialFts.length > 0
|
|
630
630
|
&& topScore >= 0.85 && (topScore - secondScore) >= 0.15;
|
|
631
631
|
|
|
632
|
-
// Step 2: Query expansion (skipped if strong signal)
|
|
633
|
-
|
|
634
|
-
|
|
632
|
+
// Step 2: Query expansion (skipped if strong signal). Typed routing —
|
|
633
|
+
// original → BOTH FTS + vector (2× RRF anchor), lex → FTS only, vec/hyde → vector only.
|
|
634
|
+
const expanded = hasStrongSignal
|
|
635
|
+
? []
|
|
635
636
|
: await store.expandQuery(query, DEFAULT_QUERY_MODEL, intent);
|
|
636
637
|
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
638
|
+
// Original query — both backends, pushed FIRST so the positional 2× weight
|
|
639
|
+
// below lands on exactly the original's lists.
|
|
640
|
+
if (initialFts.length > 0) {
|
|
641
|
+
for (const r of initialFts) docidMap.set(r.filepath, r.docid);
|
|
642
|
+
rankedLists.push(initialFts.map(r => ({ file: r.filepath, displayPath: r.displayPath, title: r.title, body: r.body || "", score: r.score })));
|
|
643
|
+
}
|
|
644
|
+
if (hasVectors) {
|
|
645
|
+
const vecResults = await store.searchVec(query, DEFAULT_EMBED_MODEL, 20, undefined, collections, dateRange);
|
|
646
|
+
if (vecResults.length > 0) {
|
|
647
|
+
for (const r of vecResults) docidMap.set(r.filepath, r.docid);
|
|
648
|
+
rankedLists.push(vecResults.map(r => ({ file: r.filepath, displayPath: r.displayPath, title: r.title, body: r.body || "", score: r.score })));
|
|
642
649
|
}
|
|
643
|
-
|
|
644
|
-
|
|
650
|
+
}
|
|
651
|
+
// Lists contributed by the original query — these get the 2× RRF weight.
|
|
652
|
+
const numOriginalLists = rankedLists.length;
|
|
653
|
+
|
|
654
|
+
// Typed expansions — route by type: lex → FTS, vec/hyde → vector.
|
|
655
|
+
for (const eq of expanded) {
|
|
656
|
+
if (eq.type === 'lex') {
|
|
657
|
+
const ftsResults = store.searchFTS(eq.query, 20, undefined, collections, dateRange);
|
|
658
|
+
if (ftsResults.length > 0) {
|
|
659
|
+
for (const r of ftsResults) docidMap.set(r.filepath, r.docid);
|
|
660
|
+
rankedLists.push(ftsResults.map(r => ({ file: r.filepath, displayPath: r.displayPath, title: r.title, body: r.body || "", score: r.score })));
|
|
661
|
+
}
|
|
662
|
+
} else if (hasVectors) {
|
|
663
|
+
const vecResults = await store.searchVec(eq.query, DEFAULT_EMBED_MODEL, 20, undefined, collections, dateRange);
|
|
645
664
|
if (vecResults.length > 0) {
|
|
646
665
|
for (const r of vecResults) docidMap.set(r.filepath, r.docid);
|
|
647
666
|
rankedLists.push(vecResults.map(r => ({ file: r.filepath, displayPath: r.displayPath, title: r.title, body: r.body || "", score: r.score })));
|
|
@@ -705,8 +724,9 @@ This is the recommended entry point for ALL memory queries.`,
|
|
|
705
724
|
}
|
|
706
725
|
}
|
|
707
726
|
|
|
708
|
-
// Weight: original query
|
|
709
|
-
|
|
727
|
+
// Weight: the original query's lists (pushed first) get 2×; expansion, temporal,
|
|
728
|
+
// and entity legs get 1×. numOriginalLists (computed above) is the actual count
|
|
729
|
+
// the original contributed — robust to an empty BM25 or vector leg.
|
|
710
730
|
const weights = rankedLists.map((_, i) => i < numOriginalLists ? 2.0 : 1.0);
|
|
711
731
|
const fused = reciprocalRankFusion(rankedLists, weights);
|
|
712
732
|
const candidates = fused.slice(0, candLimit);
|
package/src/store.ts
CHANGED
|
@@ -21,6 +21,9 @@ import {
|
|
|
21
21
|
getDefaultLlamaCpp,
|
|
22
22
|
formatQueryForEmbedding,
|
|
23
23
|
formatDocForEmbedding,
|
|
24
|
+
sanitizeExpandedQueries,
|
|
25
|
+
expansionFallback,
|
|
26
|
+
isFallbackExpansion,
|
|
24
27
|
type RerankDocument,
|
|
25
28
|
} from "./llm.ts";
|
|
26
29
|
import {
|
|
@@ -1139,7 +1142,7 @@ export type Store = {
|
|
|
1139
1142
|
searchVec: (query: string, model: string, limit?: number, collectionId?: number, collections?: string[], dateRange?: { start: string; end: string }) => Promise<SearchResult[]>;
|
|
1140
1143
|
|
|
1141
1144
|
// Query expansion & reranking
|
|
1142
|
-
expandQuery: (query: string, model?: string, intent?: string) => Promise<
|
|
1145
|
+
expandQuery: (query: string, model?: string, intent?: string) => Promise<ExpandedQuery[]>;
|
|
1143
1146
|
rerank: (query: string, documents: { file: string; text: string }[], model?: string, intent?: string) => Promise<{ file: string; score: number }[]>;
|
|
1144
1147
|
|
|
1145
1148
|
// Document retrieval
|
|
@@ -3543,28 +3546,84 @@ export function insertEmbedding(
|
|
|
3543
3546
|
// Query expansion
|
|
3544
3547
|
// =============================================================================
|
|
3545
3548
|
|
|
3546
|
-
|
|
3547
|
-
|
|
3548
|
-
|
|
3549
|
+
/**
|
|
3550
|
+
* A typed query-expansion result. Decoupled from llm.ts's internal Queryable —
|
|
3551
|
+
* same information, but store.ts owns its own public API type and field name.
|
|
3552
|
+
*
|
|
3553
|
+
* Routing contract (every consumer MUST honor it):
|
|
3554
|
+
* - lex → FTS (BM25) only
|
|
3555
|
+
* - vec → vector only
|
|
3556
|
+
* - hyde → vector only (hypothetical-document embedding)
|
|
3557
|
+
* The original query is searched on BOTH backends and is NOT included here —
|
|
3558
|
+
* callers add it explicitly with the 2× RRF anchor weight.
|
|
3559
|
+
*/
|
|
3560
|
+
export type ExpandedQuery = {
|
|
3561
|
+
type: 'lex' | 'vec' | 'hyde';
|
|
3562
|
+
query: string;
|
|
3563
|
+
};
|
|
3564
|
+
|
|
3565
|
+
// Cache version + provider fingerprint for query expansion. Bumping the version
|
|
3566
|
+
// invalidates every stale entry automatically: old newline-format and pre-terse-
|
|
3567
|
+
// prompt garbage simply never hit again and age out of llm_cache via LRU (no manual
|
|
3568
|
+
// purge). The provider fingerprint will distinguish qmd from a future zegen lex
|
|
3569
|
+
// provider (P4) so a provider swap also invalidates the cache by construction.
|
|
3570
|
+
const EXPAND_CACHE_VERSION = "v3-qmd-terse-typed";
|
|
3571
|
+
const EXPAND_PROVIDER_FINGERPRINT = "qmd-terse";
|
|
3572
|
+
|
|
3573
|
+
export async function expandQuery(query: string, model: string = DEFAULT_QUERY_MODEL, db: Database, intent?: string): Promise<ExpandedQuery[]> {
|
|
3574
|
+
// Typed-JSON cache. Versioned key (include intent + provider fingerprint).
|
|
3575
|
+
const cacheKey = getCacheKey(`expandQuery:${EXPAND_CACHE_VERSION}`, {
|
|
3576
|
+
query,
|
|
3577
|
+
model,
|
|
3578
|
+
provider: EXPAND_PROVIDER_FINGERPRINT,
|
|
3579
|
+
...(intent && { intent }),
|
|
3580
|
+
});
|
|
3549
3581
|
const cached = getCachedResult(db, cacheKey);
|
|
3550
3582
|
if (cached) {
|
|
3551
|
-
|
|
3552
|
-
|
|
3583
|
+
try {
|
|
3584
|
+
const parsed = JSON.parse(cached) as unknown;
|
|
3585
|
+
// Accept ONLY a fully-valid, already-clean typed payload. A shape error on ANY
|
|
3586
|
+
// element, an empty array, or anything sanitization would drop/rewrite → treat
|
|
3587
|
+
// the entry as stale and re-expand (never return partial or dirty cached data).
|
|
3588
|
+
if (Array.isArray(parsed) && parsed.length > 0
|
|
3589
|
+
&& parsed.every(r => r !== null && typeof r === "object"
|
|
3590
|
+
&& typeof (r as Record<string, unknown>).query === "string"
|
|
3591
|
+
&& ((r as Record<string, unknown>).type === "lex"
|
|
3592
|
+
|| (r as Record<string, unknown>).type === "vec"
|
|
3593
|
+
|| (r as Record<string, unknown>).type === "hyde"))) {
|
|
3594
|
+
const rows = parsed as Array<{ type: ExpandedQuery["type"]; query: string }>;
|
|
3595
|
+
const sanitized = sanitizeExpandedQueries(rows.map(r => ({ type: r.type, text: r.query })));
|
|
3596
|
+
const clean = sanitized.length === rows.length
|
|
3597
|
+
&& sanitized.every((s, i) => s.type === rows[i]!.type && s.text === rows[i]!.query);
|
|
3598
|
+
if (clean) return rows;
|
|
3599
|
+
}
|
|
3600
|
+
} catch {
|
|
3601
|
+
// Malformed JSON — fall through and re-expand.
|
|
3602
|
+
}
|
|
3553
3603
|
}
|
|
3554
3604
|
|
|
3555
3605
|
const llm = getDefaultLlamaCpp();
|
|
3556
|
-
// Note: LlamaCpp uses hardcoded model
|
|
3557
|
-
// Pass intent to steer expansion when provided
|
|
3606
|
+
// Note: LlamaCpp uses a hardcoded model; the model parameter is ignored here.
|
|
3607
|
+
// Pass intent to steer expansion when provided.
|
|
3558
3608
|
const results = await llm.expandQuery(query, { intent });
|
|
3559
|
-
const queryTexts = results.map(r => r.text);
|
|
3560
3609
|
|
|
3561
|
-
//
|
|
3562
|
-
|
|
3563
|
-
|
|
3564
|
-
|
|
3610
|
+
// Defense-in-depth: re-run the shared guard (also covers the local GBNF path and
|
|
3611
|
+
// any future provider), then drop entries that just echo the original query.
|
|
3612
|
+
// llm.expandQuery substitutes its OWN typed fallback on any generation failure
|
|
3613
|
+
// (remote-empty, cooldown under NO_LOCAL_MODELS, local parse-empty/error). Detect
|
|
3614
|
+
// that leaked fallback AND the all-junk case, and return an expansions-only set
|
|
3615
|
+
// that is NOT cached — a transient failure must not poison the cache or break the
|
|
3616
|
+
// "expansions only, original excluded" contract.
|
|
3617
|
+
const cleaned = sanitizeExpandedQueries(results).filter(r => r.text !== query);
|
|
3618
|
+
if (cleaned.length === 0 || isFallbackExpansion(results, query)) {
|
|
3619
|
+
return expansionFallback(query)
|
|
3620
|
+
.filter(r => r.text !== query) // expansions-only per the ExpandedQuery contract
|
|
3621
|
+
.map(r => ({ type: r.type, query: r.text }));
|
|
3565
3622
|
}
|
|
3566
3623
|
|
|
3567
|
-
|
|
3624
|
+
const expanded: ExpandedQuery[] = cleaned.map(r => ({ type: r.type, query: r.text }));
|
|
3625
|
+
setCachedResult(db, cacheKey, JSON.stringify(expanded));
|
|
3626
|
+
return expanded;
|
|
3568
3627
|
}
|
|
3569
3628
|
|
|
3570
3629
|
// =============================================================================
|