dsh-memorytree-rule 0.0.0-stage → 1.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/search.mjs ADDED
@@ -0,0 +1,264 @@
1
+ // lib/search.mjs — 混合检索(BM25 关键词 + 字符 n-gram 向量),带可解释的打分与诚实空结果。
2
+ //
3
+ // 说明(不夸大):
4
+ // - BM25 部分:中文按「单字 + 相邻二字」切,ASCII 按单词切,是真正的词频相关度检索。
5
+ // - 向量部分:256 维 hashed 字符 n-gram(feature hashing),不是神经网络语义嵌入,
6
+ // 优势是零依赖、可用于找「措辞相近」的条目;它不理解同义改写,报告里如实标注。
7
+ // - 融合:关键词 0.62 + 向量 0.38,再乘 layer/status/importance/trust/半衰期/有效期修正。
8
+ // 每条命中都带 match 分解,便于主人判断为什么被召回。
9
+ import { normalize, tokenize, snippet, truncate } from "./text.mjs";
10
+ import { embed, cosine, unpack } from "./vector.mjs";
11
+
12
+ const BM25_K1 = 1.35;
13
+ const BM25_B = 0.72;
14
+ const W_KEYWORD = 0.62;
15
+ const W_VECTOR = 0.38;
16
+
17
+ const LAYER_BOOST = { trunk: 1.18, task: 1.12, emotion: 1.0, leaf: 0.86 };
18
+ const STATUS_FACTOR = { active: 1, candidate: 0.82, superseded: 0.5, archived: 0.4 };
19
+
20
+ /**
21
+ * @param {import("./store.mjs").MemoryStore} store
22
+ * @param {{query:string, scopes?:string[]|string, layers?:string[], status?:string[],
23
+ * limit?:number, mode?:"hybrid"|"keyword"|"vector", now?:number,
24
+ * threshold?:number, includeArchived?:boolean}} opts
25
+ */
26
+ export function searchMemory(store, opts) {
27
+ const db = store.db;
28
+ const query = String(opts.query ?? "").trim();
29
+ if (query.length > 500) return emptyResult(query.slice(0, 80), "hybrid", "查询过长(上限 500 字),请缩短后再搜");
30
+ const mode = ["keyword", "vector", "hybrid"].includes(opts.mode) ? opts.mode : "hybrid";
31
+ const now = Number(opts.now ?? store.now());
32
+ const limit = Math.min(Math.max(Number(opts.limit ?? 8) || 8, 1), 30);
33
+ const queryTerms = tokenize(normalize(query));
34
+ if (query.trim() === "") {
35
+ return emptyResult(query, mode, "查询为空");
36
+ }
37
+ if (queryTerms.size === 0) {
38
+ return emptyResult(query, mode, "查询没有可检索的字符");
39
+ }
40
+ if (queryTerms.size > 120) {
41
+ return emptyResult(query.slice(0, 80), mode, "查询词太多(上限 120 个),请缩短后再搜");
42
+ }
43
+ // 有词/bigram 时丢掉单字(单字权重 0.3,会在长句里凑出假命中);只有单字查询时才用单字。
44
+ if ([...queryTerms.keys()].some((t) => t.startsWith("b:") || t.startsWith("w:"))) {
45
+ for (const t of [...queryTerms.keys()]) if (t.startsWith("u:")) queryTerms.delete(t);
46
+ }
47
+
48
+ const scopes = opts.scopes === undefined ? undefined : Array.isArray(opts.scopes) ? opts.scopes : [opts.scopes];
49
+ if (scopes && scopes.length === 0) return emptyResult(query, mode, "没有指定任何作用域");
50
+ const layers = opts.layers && opts.layers.length ? opts.layers : undefined;
51
+ const statuses = opts.status && opts.status.length ? opts.status : ["active", "candidate"];
52
+
53
+ // 1) 候选集:SQL 过滤 scope/layer/status,之后全部在 JS 里打分。
54
+ const where = [];
55
+ const params = [];
56
+ if (scopes) {
57
+ where.push("scope IN (" + scopes.map(() => "?").join(",") + ")");
58
+ params.push(...scopes);
59
+ }
60
+ if (layers) {
61
+ where.push("layer IN (" + layers.map(() => "?").join(",") + ")");
62
+ params.push(...layers);
63
+ }
64
+ if (statuses) {
65
+ where.push("status IN (" + statuses.map(() => "?").join(",") + ")");
66
+ params.push(...statuses);
67
+ }
68
+ const rows = db
69
+ .prepare("SELECT id,uid,scope,layer,kind,status,subject,title,text,summary,tags,source,evidence,trust,importance,half_life_days,valid_until,updated_at,ts FROM node" + (where.length ? " WHERE " + where.join(" AND ") : ""))
70
+ .all(...params);
71
+ if (rows.length === 0) return emptyResult(query, mode, "该作用域/枝条下还没有记忆");
72
+
73
+ const ids = rows.map((r) => Number(r.id));
74
+ const idSet = new Set(ids);
75
+ const N = rows.length;
76
+
77
+ // 2) 词项统计:df 用整个候选集,节点长度用存储的 len(加权 token 数)。
78
+ const df = new Map();
79
+ const termStmt = db.prepare("SELECT node_id,term,tf FROM term WHERE term=?");
80
+ // 逐查询词取倒排;查询词很少(<=几十),够快。
81
+ const inverted = new Map();
82
+ for (const term of queryTerms.keys()) {
83
+ const hits = termStmt.all(term).filter((h) => idSet.has(Number(h.node_id)));
84
+ if (hits.length === 0) continue;
85
+ df.set(term, hits.length);
86
+ const m = new Map();
87
+ for (const h of hits) m.set(Number(h.node_id), Number(h.tf));
88
+ inverted.set(term, m);
89
+ }
90
+
91
+ const avgLen = rows.reduce((a, r) => a + Number(r.len ?? 0), 0) / N || 1;
92
+ const vecStmt = db.prepare("SELECT data FROM vec WHERE node_id=?");
93
+ const qVec = embed(query);
94
+ const qNorm = normalize(query);
95
+ const qCompact = qNorm.replace(/\s+/g, "");
96
+
97
+ const scored = [];
98
+ for (const r of rows) {
99
+ const id = Number(r.id);
100
+
101
+ // ── BM25 ──
102
+ let bm25 = 0;
103
+ let matchedTerms = 0;
104
+ const len = Number(r.len ?? 0);
105
+ for (const [term, qw] of queryTerms) {
106
+ const posting = inverted.get(term);
107
+ if (!posting) continue;
108
+ const tf = posting.get(id);
109
+ if (!tf) continue;
110
+ matchedTerms += 1;
111
+ const dfv = df.get(term) ?? 0;
112
+ const idf = Math.log(1 + (N - dfv + 0.5) / (dfv + 0.5));
113
+ const denom = tf + BM25_K1 * (1 - BM25_B + (BM25_B * len) / avgLen);
114
+ bm25 += idf * ((tf * (BM25_K1 + 1)) / denom) * qw;
115
+ }
116
+
117
+ // ── 向量 ──
118
+ let cos = 0;
119
+ if (mode !== "keyword") {
120
+ const vrow = vecStmt.get(id);
121
+ if (vrow && vrow.data) {
122
+ cos = cosine(qVec, unpack(vrow.data));
123
+ if (!Number.isFinite(cos)) cos = 0;
124
+ cos = Math.max(0, cos);
125
+ }
126
+ }
127
+
128
+ // 诚实门槛:hash 字符 n-gram 向量不是神经语义,不能单独「猜」出命中。
129
+ // 默认 keyword/hybrid 要求至少有一个查询词真的出现在条目里;只有显式 mode=vector 才允许纯相似度召回。
130
+ if (mode === "vector") {
131
+ if (cos < 0.12) continue;
132
+ } else if (bm25 === 0) {
133
+ continue;
134
+ }
135
+
136
+ // ── 归一化与修正 ──
137
+ const kwNorm = Math.min(1, bm25 / 6);
138
+ const vecNorm = Math.min(1, cos / 0.6);
139
+ let score;
140
+ if (mode === "keyword") score = kwNorm;
141
+ else if (mode === "vector") score = vecNorm;
142
+ else score = W_KEYWORD * kwNorm + W_VECTOR * vecNorm;
143
+
144
+ // 精确短语(整串出现在正文里)是强信号
145
+ const textNorm = normalize(String(r.text));
146
+ const textCompact = textNorm.replace(/\s+/g, "");
147
+ const phraseHit = qCompact.length >= 2 && textCompact.includes(qCompact);
148
+ if (phraseHit) score += 0.18;
149
+
150
+ const layerBoost = LAYER_BOOST[r.layer] ?? 1;
151
+ const statusFactor = STATUS_FACTOR[r.status] ?? 0.8;
152
+ const importance = Number(r.importance ?? 0.5);
153
+ const trust = Number(r.trust ?? 70) / 100;
154
+ let relevance = score * layerBoost * statusFactor * (0.72 + 0.56 * importance) * (0.62 + 0.38 * trust);
155
+
156
+ // 半衰期衰减:half_life_days=0 表示不衰减(长期规则),否则按 0.5^(age/halfLife)
157
+ const halfLife = Number(r.half_life_days ?? 0);
158
+ let decay = 1;
159
+ if (halfLife > 0) {
160
+ const ageDays = Math.max(0, (now - Number(r.updated_at ?? now)) / 86400000);
161
+ decay = Math.max(0.35, Math.pow(0.5, ageDays / halfLife));
162
+ relevance *= decay;
163
+ }
164
+ // 过期条目降权并标注
165
+ const expired = r.valid_until !== null && r.valid_until !== undefined && Number(r.valid_until) < now;
166
+
167
+ // subject / title 命中也算相关(卡片检索常按 subject 找)
168
+ let fieldHit = false;
169
+ if (r.subject && textContains(normalize(String(r.subject)), queryTerms)) {
170
+ relevance += 0.08;
171
+ fieldHit = true;
172
+ }
173
+ if (expired) relevance *= 0.55;
174
+
175
+ scored.push({
176
+ id,
177
+ uid: String(r.uid),
178
+ scope: String(r.scope),
179
+ layer: String(r.layer),
180
+ kind: String(r.kind),
181
+ status: String(r.status),
182
+ subject: String(r.subject ?? ""),
183
+ title: String(r.title ?? ""),
184
+ text: String(r.text ?? ""),
185
+ summary: r.summary === null || r.summary === undefined ? null : String(r.summary),
186
+ tags: safeTags(r.tags),
187
+ source: String(r.source ?? ""),
188
+ evidence: r.evidence === null || r.evidence === undefined ? null : String(r.evidence),
189
+ trust: Number(r.trust ?? 70),
190
+ importance,
191
+ validUntil: r.valid_until === null || r.valid_until === undefined ? null : Number(r.valid_until),
192
+ updatedAt: Number(r.updated_at ?? 0),
193
+ relevance: Number(relevance.toFixed(4)),
194
+ match: {
195
+ keyword: Number(kwNorm.toFixed(3)),
196
+ vector: Number(vecNorm.toFixed(3)),
197
+ bm25: Number(bm25.toFixed(3)),
198
+ cosine: Number(cos.toFixed(3)),
199
+ terms: matchedTerms,
200
+ phrase: phraseHit,
201
+ subject: fieldHit,
202
+ decay: Number(decay.toFixed(3)),
203
+ },
204
+ expired,
205
+ snippet: snippet(String(r.text), query, 180),
206
+ });
207
+ }
208
+
209
+ if (scored.length === 0) {
210
+ return emptyResult(query, mode, "没有命中:所有候选记忆条目都不含查询词(不编造;换义词或 mode=vector 可扩大召回)", { candidateCount: N });
211
+ }
212
+
213
+ scored.sort((a, b) => b.relevance - a.relevance || b.updatedAt - a.updatedAt);
214
+ const threshold = Number(opts.threshold ?? defaultThreshold(mode));
215
+ let hits = scored.filter((h) => h.relevance >= threshold);
216
+ if (hits.length === 0) {
217
+ return emptyResult(query, mode, "命中强度低于阈值(最相关的条目 " + scored[0].relevance.toFixed(3) + " < " + threshold + ")", {
218
+ candidateCount: N,
219
+ topRejected: scored.slice(0, 3).map((h) => ({ id: h.id, uid: h.uid, score: h.relevance, layer: h.layer, subject: h.subject })),
220
+ });
221
+ }
222
+ const total = hits.length;
223
+ hits = hits.slice(0, limit);
224
+ const out = {
225
+ query,
226
+ mode,
227
+ threshold,
228
+ scope: scopes ?? null,
229
+ hits,
230
+ total,
231
+ returned: hits.length,
232
+ candidateCount: N,
233
+ honest: "结果按相关度排序;向量部分是字符 n-gram 近似,不是语义模型,同义改写可能漏召回。",
234
+ };
235
+ store.audit("search", { scope: scopes ? scopes.join(",") : null, query, hits: total, detail: { mode, candidateCount: N } });
236
+ return out;
237
+ }
238
+
239
+ function emptyResult(query, mode, reason, extra = {}) {
240
+ return { query, mode, threshold: defaultThreshold(mode), hits: [], total: 0, returned: 0, empty: true, reason, ...extra };
241
+ }
242
+
243
+ function defaultThreshold(mode) {
244
+ if (mode === "keyword") return 0.12;
245
+ if (mode === "vector") return 0.3;
246
+ return 0.2;
247
+ }
248
+
249
+ function safeTags(v) {
250
+ try {
251
+ const a = JSON.parse(v ?? "[]");
252
+ return Array.isArray(a) ? a.map(String) : [];
253
+ } catch {
254
+ return [];
255
+ }
256
+ }
257
+
258
+ function textContains(normText, terms) {
259
+ for (const t of terms.keys()) {
260
+ const bare = t.slice(2);
261
+ if (normText.includes(bare)) return true;
262
+ }
263
+ return false;
264
+ }