@memory-river/core 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +222 -0
- package/README.zh-TW.md +186 -0
- package/dist/api.d.ts +100 -0
- package/dist/api.js +156 -0
- package/dist/cognition/causal-attribution.d.ts +36 -0
- package/dist/cognition/causal-attribution.js +239 -0
- package/dist/cognition/causal-engine.d.ts +105 -0
- package/dist/cognition/causal-engine.js +150 -0
- package/dist/cognition/conflict-detector.d.ts +39 -0
- package/dist/cognition/conflict-detector.js +193 -0
- package/dist/cognition/global-working-memory.d.ts +53 -0
- package/dist/cognition/global-working-memory.js +211 -0
- package/dist/cognition/hooks-engine.d.ts +99 -0
- package/dist/cognition/hooks-engine.js +672 -0
- package/dist/cognition/ralph-core.d.ts +28 -0
- package/dist/cognition/ralph-core.js +104 -0
- package/dist/distill/concentrator-adapter.d.ts +167 -0
- package/dist/distill/concentrator-adapter.js +1876 -0
- package/dist/engine.d.ts +402 -0
- package/dist/engine.js +2254 -0
- package/dist/index.d.ts +6 -0
- package/dist/index.js +3 -0
- package/dist/lifecycle/cleanup-engine.d.ts +80 -0
- package/dist/lifecycle/cleanup-engine.js +162 -0
- package/dist/lifecycle/cleanup-state.d.ts +34 -0
- package/dist/lifecycle/cleanup-state.js +50 -0
- package/dist/lifecycle/night-consolidation.d.ts +102 -0
- package/dist/lifecycle/night-consolidation.js +640 -0
- package/dist/lifecycle/night-recovery.d.ts +40 -0
- package/dist/lifecycle/night-recovery.js +107 -0
- package/dist/paths.d.ts +17 -0
- package/dist/paths.js +16 -0
- package/dist/pipeline/capsule-bridge.d.ts +35 -0
- package/dist/pipeline/capsule-bridge.js +86 -0
- package/dist/pipeline/compact-request.d.ts +30 -0
- package/dist/pipeline/compact-request.js +66 -0
- package/dist/pipeline/inbox-watcher.d.ts +112 -0
- package/dist/pipeline/inbox-watcher.js +1039 -0
- package/dist/ports.d.ts +29 -0
- package/dist/ports.js +1 -0
- package/dist/providers/embedder-v5.d.ts +46 -0
- package/dist/providers/embedder-v5.js +155 -0
- package/dist/providers/ollama-embedding.d.ts +25 -0
- package/dist/providers/ollama-embedding.js +166 -0
- package/dist/retrieval/abstractness-judge.d.ts +14 -0
- package/dist/retrieval/abstractness-judge.js +87 -0
- package/dist/retrieval/coverage-selection.d.ts +3 -0
- package/dist/retrieval/coverage-selection.js +53 -0
- package/dist/retrieval/cross-encoder-gate.d.ts +40 -0
- package/dist/retrieval/cross-encoder-gate.js +239 -0
- package/dist/retrieval/retriever-v4.d.ts +78 -0
- package/dist/retrieval/retriever-v4.js +1200 -0
- package/dist/skills/validate.d.ts +6 -0
- package/dist/skills/validate.js +69 -0
- package/dist/storage.d.ts +19 -0
- package/dist/storage.js +54 -0
- package/dist/store/aux-table-maintenance.d.ts +5 -0
- package/dist/store/aux-table-maintenance.js +64 -0
- package/dist/store/graph-enumerator.d.ts +21 -0
- package/dist/store/graph-enumerator.js +185 -0
- package/dist/store/graph-store.d.ts +107 -0
- package/dist/store/graph-store.js +478 -0
- package/dist/store/status-manager.d.ts +44 -0
- package/dist/store/status-manager.js +235 -0
- package/dist/store/store-v4.d.ts +339 -0
- package/dist/store/store-v4.js +2871 -0
- package/dist/transcript/keyword-search.d.ts +9 -0
- package/dist/transcript/keyword-search.js +67 -0
- package/dist/transcript/rehydrate-keyword.d.ts +6 -0
- package/dist/transcript/rehydrate-keyword.js +29 -0
- package/dist/transcript/rehydrate.d.ts +33 -0
- package/dist/transcript/rehydrate.js +285 -0
- package/dist/transcript/transcript-archive.d.ts +46 -0
- package/dist/transcript/transcript-archive.js +516 -0
- package/dist/types.d.ts +409 -0
- package/dist/types.js +104 -0
- package/dist/util/bounded-map.d.ts +1 -0
- package/dist/util/bounded-map.js +8 -0
- package/dist/util/rate-limiter.d.ts +12 -0
- package/dist/util/rate-limiter.js +54 -0
- package/dist/util/session-identity.d.ts +65 -0
- package/dist/util/session-identity.js +227 -0
- package/dist/util/util-hash.d.ts +1 -0
- package/dist/util/util-hash.js +4 -0
- package/package.json +59 -0
|
@@ -0,0 +1,1200 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Retriever - Hybrid Search (正確實現 RRF Fusion) + CRAG Evaluator
|
|
3
|
+
* memory-lance-v4
|
|
4
|
+
* * 核心功能:
|
|
5
|
+
* - 混合檢索 (向量 + BM25) 與 RRF Fusion
|
|
6
|
+
* - V4 記憶權重計分 (相似度 + 重要度 + 健康度)
|
|
7
|
+
* - 本地毫秒級 CRAG 評估器 (MiniLM 餘弦相似度)
|
|
8
|
+
* - 赫布理論 (Hebbian Learning) 自動強化記憶
|
|
9
|
+
*/
|
|
10
|
+
import { judgeAbstractness } from "./abstractness-judge.js";
|
|
11
|
+
import { applyCragCrossEncoderGate, isCragCrossEncoderGateEnabled, scoreCandidates } from "./cross-encoder-gate.js";
|
|
12
|
+
import { coverageLambda, isCoverageSelectionEnabled, mmrOrder } from "./coverage-selection.js";
|
|
13
|
+
import { hashQuery } from "../util/util-hash.js";
|
|
14
|
+
const RRF_K = 60;
|
|
15
|
+
const BACKGROUND_WRITE_QUEUE_LIMIT = 100;
|
|
16
|
+
// ── CRAG relevance gating (Qwen cosine distance = 1 − cosine) ──────────────────
|
|
17
|
+
// See docs/internal/TICKET_CRAG_QWEN_RERANK.md. CRAG scores every candidate by
|
|
18
|
+
// 1 − cosine(queryVector, entry.vector) — one uniform, metric-agnostic Qwen scale
|
|
19
|
+
// across vector / FTS-only / hook candidates — instead of a separate MiniLM model.
|
|
20
|
+
// Seed values; recalibrate via `mr-bench crag` + LoCoMo (do NOT treat as final).
|
|
21
|
+
// Recall-safe by design (decided 2026-06-14). Calibration (Qwen3 1024d) found a
|
|
22
|
+
// hard precision/recall tension a single absolute threshold cannot resolve: the
|
|
23
|
+
// crag precision fixture wants ~0.36 (near-miss distractors ≥0.37) but LoCoMo true
|
|
24
|
+
// evidence sits at median 0.506 / p90 0.598, so a 0.36 cut would drop ~97% of real
|
|
25
|
+
// answers. CRAG's job is to drop clearly-irrelevant memories and stop the recall
|
|
26
|
+
// collapse — NOT to resolve near-misses (dense embedding can't separate
|
|
27
|
+
// near-equivalents; that's a separate concern). So we sit at the recall-safe end.
|
|
28
|
+
// See docs/internal/CRAG_DISTRACTOR_FINDINGS_2026-06-14.md.
|
|
29
|
+
const CRAG_VIP_DIST = 0.30; // <= → VIP auto-keep (very similar)
|
|
30
|
+
const CRAG_DIST_YES = 0.55; // <= → keep
|
|
31
|
+
const CRAG_DIST_NO = 0.65; // >= → drop; between DIST_YES and DIST_NO → partial
|
|
32
|
+
// Hook-injected candidates are a bypass recall path: they enter cragInput with a
|
|
33
|
+
// generation-time weight, not a query-relevance score. They must be held to a
|
|
34
|
+
// STRICTER vector-distance gate than normal candidates and must NOT auto-keep on an
|
|
35
|
+
// uncomputable distance (topic-relevant-but-wrong hooks were polluting context).
|
|
36
|
+
const HOOK_CRAG_DIST_YES = 0.48; // hook keep threshold (stricter than CRAG_DIST_YES)
|
|
37
|
+
const HOOK_INJECT_SCORE_CAP = 0.35; // hooks are supplements; never outrank real top hits
|
|
38
|
+
const LOCAL_RERANK_THRESHOLD_CORRECT = 0.63;
|
|
39
|
+
const LOCAL_RERANK_THRESHOLD_INCORRECT = 0.4;
|
|
40
|
+
/**
|
|
41
|
+
* Cosine distance (1 − cosine similarity) between two equal-length vectors.
|
|
42
|
+
* Returns +Infinity when the vectors are missing, empty, or mismatched in length
|
|
43
|
+
* so callers route to a "partial"/fallback path rather than a false "very close".
|
|
44
|
+
*/
|
|
45
|
+
function cosineDistance(a, b) {
|
|
46
|
+
if (!a || !b || a.length === 0 || a.length !== b.length)
|
|
47
|
+
return Number.POSITIVE_INFINITY;
|
|
48
|
+
let dot = 0, normA = 0, normB = 0;
|
|
49
|
+
for (let i = 0; i < a.length; i++) {
|
|
50
|
+
const x = a[i], y = b[i];
|
|
51
|
+
if (Number.isNaN(x) || Number.isNaN(y))
|
|
52
|
+
continue;
|
|
53
|
+
dot += x * y;
|
|
54
|
+
normA += x * x;
|
|
55
|
+
normB += y * y;
|
|
56
|
+
}
|
|
57
|
+
if (normA === 0 || normB === 0)
|
|
58
|
+
return Number.POSITIVE_INFINITY;
|
|
59
|
+
const cos = dot / (Math.sqrt(normA) * Math.sqrt(normB));
|
|
60
|
+
return Number.isNaN(cos) ? Number.POSITIVE_INFINITY : 1 - cos;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Entity Synergy Merger — 搶救 partial 記憶的抽吸式合併模組
|
|
64
|
+
*
|
|
65
|
+
* 核心思路(來自 MergeRAG 靈感,但不走 LLM 生成):
|
|
66
|
+
* 兩條 partial 記憶如果共享「橋接實體」(entity overlap),代表它們可能在描述
|
|
67
|
+
* 同一事件的不同面向。合併時用「句子級 BM25」決定誰留誰砍,純抽取不生成。
|
|
68
|
+
*
|
|
69
|
+
* 流程:
|
|
70
|
+
* 1. 用正則抽出命名實體(人/地/組織/時間/專有名詞)
|
|
71
|
+
* 2. 找出 entity overlap 超過門檻的記憶對
|
|
72
|
+
* 3. 對記憶文字做句子分割,保留 BM25 分數最高的句子(去冗餘)
|
|
73
|
+
* 4. 合併後用 MiniLM 重新評估 relevanc → 過關就升級進結果池
|
|
74
|
+
*/
|
|
75
|
+
class EntitySynergyMerger {
|
|
76
|
+
// ── 輕量 NER:正則抽取(零外部依賴,毫秒完成) ──────────────
|
|
77
|
+
static ENTITY_PATTERNS = [
|
|
78
|
+
// 中文姓名 (兩字以上)
|
|
79
|
+
{ type: "PERSON", pattern: /[A-Za-z][a-z]+ [A-Z][a-z]+|[가-힣]{2,4}|[一二三李王張劉陳楊黃趙周吳徐孫馬朱胡郭何林高羅鄭梁謝宋唐許韓鄧馮曹曾程蔡潘田董袁于余蘇葉賈魏崔史侯孟龍万段雷钱汤白汪]|[A-Z][a-z]+/g },
|
|
80
|
+
// 數數數數/基數(單用數值本身沒意義,要配合上下文)
|
|
81
|
+
{ type: "NUMBER", pattern: /\d+(?:\.\d+)?[萬億千百十個多約大概約]/g },
|
|
82
|
+
// 百分比
|
|
83
|
+
{ type: "PERCENT", pattern: /\d+(?:\.\d+)?%/g },
|
|
84
|
+
// 日期/時間
|
|
85
|
+
{ type: "DATE", pattern: /\d{4}[-/年]\d{1,2}[-/月]\d{1,2}日?(?:\s*\d{1,2}[時:點]\d{1,2}分?)?|\d{1,2}[-/月]\d{1,2}[日]?(?:\s*\d{1,2}[時:點])?|昨[天日晚早]|今[天日晚早]|明[天日晚早]|上[周個月]|這[周個月]|大前|前天|後天|早上|下午|晚上|凌晨|傍晚/g },
|
|
86
|
+
// 地點
|
|
87
|
+
{ type: "LOCATION", pattern: /[臺台]北|[臺台]中|[臺台]南|[臺台]東|高雄|新北|桃園|彰化|宜蘭|花蓮|臺北市|新北市|桃園市|高雄市|[A-Z][a-z]+(?:市|縣|區|鎮|鄉|路|街|大道)/g },
|
|
88
|
+
// URL / 網域
|
|
89
|
+
{ type: "URL", pattern: /https?:\/\/[^\s]+|[a-zA-Z0-9_-]+\.(?:com|org|net|io|cc|tw|hk|ru|cn|me|info|biz|co|in|us|uk|au|jp|kr|dev|app|tech|ai|cloud|site|online|website|xyz|pro|live|ws|fm|ml|cc|ai)/gi },
|
|
90
|
+
// 品牌 / 產品名(駝峰或全大寫)
|
|
91
|
+
{ type: "BRAND", pattern: /[A-Z][a-z]+(?:[A-Z][a-z]|[0-9])+|\b(?:OpenClaw|Claude|MiniMax|Yeelight|NestHub|GPT|Claude|Cline|Codex|ArtiMart|Blend)|(?:[A-Z][a-z]+){2,}/g },
|
|
92
|
+
// 技術術語 / 專案名
|
|
93
|
+
{ type: "TERM", pattern: /(?<![a-zA-Z])[A-Z][a-z]+(?:[A-Z][a-z]|\d)*?(?![a-zA-Z])|(?:記憶庫|記憶引擎|CRAG|Hooks|因果鏈|Hebbian|濃縮引擎)/g },
|
|
94
|
+
];
|
|
95
|
+
/**
|
|
96
|
+
* 從文字抽出命名實體集合
|
|
97
|
+
* @returns Map<entity_type, Set<entity_value>>
|
|
98
|
+
*/
|
|
99
|
+
static extractEntities(text) {
|
|
100
|
+
const entities = new Map();
|
|
101
|
+
for (const { type, pattern } of this.ENTITY_PATTERNS) {
|
|
102
|
+
const matches = text.match(pattern);
|
|
103
|
+
if (!matches)
|
|
104
|
+
continue;
|
|
105
|
+
if (!entities.has(type))
|
|
106
|
+
entities.set(type, new Set());
|
|
107
|
+
for (const m of matches) {
|
|
108
|
+
const cleaned = m.trim().toLowerCase();
|
|
109
|
+
if (cleaned.length > 1)
|
|
110
|
+
entities.get(type).add(cleaned);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
// 全文切片:每個 Token(簡單分詞,空白+標點)
|
|
114
|
+
const tokens = text.toLowerCase().split(/[\s\p{P}]+/u);
|
|
115
|
+
// 把未分類的高信息量 token 當作 GENERAL 實體(過濾停用詞)
|
|
116
|
+
const stopWords = new Set(["的", "是", "在", "了", "和", "與", "或", "以及", "以及", "有", "沒有", "這個", "那個", "也", "就", "都", "而", "但", "或", "如果", "因為", "所以", "可以", "會", "能", "一個", "什麼", "怎麼", "如何", "為什麼", "多少"]);
|
|
117
|
+
const general = new Set();
|
|
118
|
+
for (const t of tokens) {
|
|
119
|
+
const alreadyFound = [...entities.values()].some(s => s.has(t));
|
|
120
|
+
if (t.length >= 3 && !stopWords.has(t) && !/^\d+$/.test(t) && !alreadyFound) {
|
|
121
|
+
general.add(t);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
if (general.size > 0)
|
|
125
|
+
entities.set("GENERAL", general);
|
|
126
|
+
return entities;
|
|
127
|
+
}
|
|
128
|
+
/**
|
|
129
|
+
* 計算兩個實體集合的 overlap 分數(IDF-weighted Jaccard)
|
|
130
|
+
* 只看共享數量,不重複計
|
|
131
|
+
*/
|
|
132
|
+
static computeOverlap(entitiesA, entitiesB) {
|
|
133
|
+
let sharedCount = 0;
|
|
134
|
+
let totalUnique = 0;
|
|
135
|
+
const allTypes = new Set([...entitiesA.keys(), ...entitiesB.keys()]);
|
|
136
|
+
for (const type of allTypes) {
|
|
137
|
+
const setA = entitiesA.get(type) || new Set();
|
|
138
|
+
const setB = entitiesB.get(type) || new Set();
|
|
139
|
+
// 同類型 overlap 權重更高
|
|
140
|
+
const typeWeight = (type === "GENERAL") ? 0.5 : 1.5;
|
|
141
|
+
for (const entity of setA) {
|
|
142
|
+
if (setB.has(entity)) {
|
|
143
|
+
sharedCount += typeWeight;
|
|
144
|
+
}
|
|
145
|
+
totalUnique += typeWeight;
|
|
146
|
+
}
|
|
147
|
+
for (const entity of setB) {
|
|
148
|
+
if (!setA.has(entity))
|
|
149
|
+
totalUnique += typeWeight;
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
return totalUnique > 0 ? sharedCount / totalUnique : 0;
|
|
153
|
+
}
|
|
154
|
+
/**
|
|
155
|
+
* 合併兩條記憶:句子級拼接 + BM25 去冗餘
|
|
156
|
+
*
|
|
157
|
+
* @param texts 記憶文字陣列(來自不同記憶)
|
|
158
|
+
* @param query 原始查詢(用於 BM25 打分)
|
|
159
|
+
* @returns 合併後的文字
|
|
160
|
+
*/
|
|
161
|
+
static mergeMemories(texts, query) {
|
|
162
|
+
// Step 1: 句子分割(簡單正則,保持中文句式)
|
|
163
|
+
const sentences = [];
|
|
164
|
+
for (const text of texts) {
|
|
165
|
+
// 按中英文句末標點分割
|
|
166
|
+
const parts = text.split(/(?<=[。!?.!?])\s*/).filter(s => s.trim().length > 5);
|
|
167
|
+
sentences.push(...parts);
|
|
168
|
+
}
|
|
169
|
+
if (sentences.length === 0)
|
|
170
|
+
return texts.join("\n");
|
|
171
|
+
if (sentences.length === 1)
|
|
172
|
+
return sentences[0];
|
|
173
|
+
// Step 2: 簡單 BM25 句子打分(query terms 在句中出現次數 / 句長標準化)
|
|
174
|
+
const queryTerms = query.toLowerCase().split(/[\s\p{P}]+/u).filter(t => t.length > 1);
|
|
175
|
+
const scored = sentences.map((sentence, idx) => {
|
|
176
|
+
const lower = sentence.toLowerCase();
|
|
177
|
+
let score = 0;
|
|
178
|
+
for (const term of queryTerms) {
|
|
179
|
+
// term comes from raw query text and can contain regex metachars (e.g. "c++"),
|
|
180
|
+
// which crash new RegExp(); count non-overlapping occurrences via split instead.
|
|
181
|
+
const count = lower.split(term).length - 1;
|
|
182
|
+
score += count;
|
|
183
|
+
}
|
|
184
|
+
// 長句懲罰(避免稀釋重點)
|
|
185
|
+
const lengthNorm = 1 / Math.log2(sentence.length + 3);
|
|
186
|
+
return { sentence, score: score * lengthNorm, idx };
|
|
187
|
+
});
|
|
188
|
+
// Step 3: 取 top-N 句子(N = min(6, 全部)),去除重複描述
|
|
189
|
+
const topN = scored
|
|
190
|
+
.sort((a, b) => b.score - a.score)
|
|
191
|
+
.slice(0, 6);
|
|
192
|
+
// Step 4: 去除內容高度重疊的句子(simple char-level n-gram jaccard)
|
|
193
|
+
const finalSentences = [];
|
|
194
|
+
for (const { sentence } of topN) {
|
|
195
|
+
const isDuplicate = finalSentences.some(existing => this.ngramJaccard(existing, sentence, 3) > 0.6);
|
|
196
|
+
if (!isDuplicate)
|
|
197
|
+
finalSentences.push(sentence);
|
|
198
|
+
}
|
|
199
|
+
return finalSentences.join("。").replace(/。+/g, "。").trim() + "。";
|
|
200
|
+
}
|
|
201
|
+
/**
|
|
202
|
+
* 計算兩個字串的 n-gram Jaccard 相似度(用於去重)
|
|
203
|
+
*/
|
|
204
|
+
static ngramJaccard(a, b, n = 3) {
|
|
205
|
+
const ng = (s) => {
|
|
206
|
+
const chars = s.split("");
|
|
207
|
+
const set = new Set();
|
|
208
|
+
for (let i = 0; i <= chars.length - n; i++) {
|
|
209
|
+
set.add(chars.slice(i, i + n).join(""));
|
|
210
|
+
}
|
|
211
|
+
return set;
|
|
212
|
+
};
|
|
213
|
+
const setA = ng(a);
|
|
214
|
+
const setB = ng(b);
|
|
215
|
+
let intersection = 0;
|
|
216
|
+
for (const g of setA) {
|
|
217
|
+
if (setB.has(g))
|
|
218
|
+
intersection++;
|
|
219
|
+
}
|
|
220
|
+
const union = setA.size + setB.size - intersection;
|
|
221
|
+
return union > 0 ? intersection / union : 0;
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
/**
|
|
225
|
+
* 本地 Reranker Singleton
|
|
226
|
+
*
|
|
227
|
+
* @deprecated DEAD as of the Qwen-cosine CRAG rerank (TICKET_CRAG_QWEN_RERANK).
|
|
228
|
+
* No runtime path calls `evaluateRelevance` any more — CRAG and synergy merge now
|
|
229
|
+
* score on Qwen cosine distance. Kept temporarily so the `transformers`/MiniLM
|
|
230
|
+
* dependency removal can land as a separate, isolated follow-up. Do not add callers.
|
|
231
|
+
*/
|
|
232
|
+
class LocalReranker {
|
|
233
|
+
static extractor = null;
|
|
234
|
+
static initializing = null;
|
|
235
|
+
// 🛠️ 新增:Query 向量快取
|
|
236
|
+
static lastQuery = "";
|
|
237
|
+
static lastQueryEmb = null;
|
|
238
|
+
static async init(cacheDir) {
|
|
239
|
+
if (this.extractor) {
|
|
240
|
+
return this.extractor;
|
|
241
|
+
}
|
|
242
|
+
if (this.initializing) {
|
|
243
|
+
return this.initializing;
|
|
244
|
+
}
|
|
245
|
+
this.initializing = (async () => {
|
|
246
|
+
console.log("[LocalReranker] Loading lightweight evaluation model...");
|
|
247
|
+
const startTime = Date.now();
|
|
248
|
+
const { pipeline, env } = await import("@xenova/transformers");
|
|
249
|
+
env.cacheDir = cacheDir;
|
|
250
|
+
this.extractor = await pipeline("feature-extraction", "Xenova/paraphrase-multilingual-MiniLM-L12-v2", {
|
|
251
|
+
quantized: true,
|
|
252
|
+
});
|
|
253
|
+
console.log(`[LocalReranker] Model loaded in ${Date.now() - startTime}ms`);
|
|
254
|
+
return this.extractor;
|
|
255
|
+
})();
|
|
256
|
+
return this.initializing;
|
|
257
|
+
}
|
|
258
|
+
static cosineSimilarity(a, b) {
|
|
259
|
+
let dot = 0, normA = 0, normB = 0;
|
|
260
|
+
for (let i = 0; i < a.length; i++) {
|
|
261
|
+
if (isNaN(a[i]) || isNaN(b[i]))
|
|
262
|
+
continue;
|
|
263
|
+
dot += a[i] * b[i];
|
|
264
|
+
normA += a[i] * a[i];
|
|
265
|
+
normB += b[i] * b[i];
|
|
266
|
+
}
|
|
267
|
+
const result = dot / (Math.sqrt(normA) * Math.sqrt(normB));
|
|
268
|
+
return isNaN(result) ? 0 : result;
|
|
269
|
+
}
|
|
270
|
+
static async evaluateRelevance(query, memoryText, cacheDir) {
|
|
271
|
+
if (!this.extractor) {
|
|
272
|
+
await this.init(cacheDir);
|
|
273
|
+
}
|
|
274
|
+
try {
|
|
275
|
+
// 🛠️ 核心修復:只有當 query 改變時,才重新計算 Query Embedding (省 50% 算力)
|
|
276
|
+
if (query !== this.lastQuery || !this.lastQueryEmb) {
|
|
277
|
+
this.lastQueryEmb = await this.extractor(query, { pooling: "mean", normalize: true });
|
|
278
|
+
this.lastQuery = query;
|
|
279
|
+
}
|
|
280
|
+
// 只計算 Memory 的 Embedding
|
|
281
|
+
const mEmb = await this.extractor(memoryText, { pooling: "mean", normalize: true });
|
|
282
|
+
const similarity = this.cosineSimilarity(this.lastQueryEmb.data, mEmb.data);
|
|
283
|
+
if (similarity >= LOCAL_RERANK_THRESHOLD_CORRECT) {
|
|
284
|
+
return { relevance: "yes", extracted_info: "" };
|
|
285
|
+
}
|
|
286
|
+
else if (similarity <= LOCAL_RERANK_THRESHOLD_INCORRECT) {
|
|
287
|
+
return { relevance: "no", extracted_info: "" };
|
|
288
|
+
}
|
|
289
|
+
else {
|
|
290
|
+
return { relevance: "partial", extracted_info: "" };
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
catch (err) {
|
|
294
|
+
console.error("[LocalReranker] Evaluation failed:", err);
|
|
295
|
+
return { relevance: "no", extracted_info: "" };
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
export class Retriever {
|
|
300
|
+
store;
|
|
301
|
+
embedder;
|
|
302
|
+
rerankerCacheDir;
|
|
303
|
+
vectorWeight;
|
|
304
|
+
bm25Weight;
|
|
305
|
+
candidatePoolMultiplier;
|
|
306
|
+
hooksEngine = null;
|
|
307
|
+
boostHealthQueue = [];
|
|
308
|
+
boostHealthQueueRunning = false;
|
|
309
|
+
recallMetadataQueue = [];
|
|
310
|
+
recallMetadataQueueRunning = false;
|
|
311
|
+
constructor(store, embedder, config, rerankerCacheDir, hooksEngine) {
|
|
312
|
+
this.store = store;
|
|
313
|
+
this.embedder = embedder;
|
|
314
|
+
this.rerankerCacheDir = rerankerCacheDir;
|
|
315
|
+
this.vectorWeight = config?.vectorWeight ?? 0.7;
|
|
316
|
+
this.bm25Weight = config?.bm25Weight ?? 0.3;
|
|
317
|
+
this.candidatePoolMultiplier = config?.candidatePoolMultiplier ?? 2;
|
|
318
|
+
this.hooksEngine = hooksEngine || null;
|
|
319
|
+
}
|
|
320
|
+
async hybridSearch(query, limit = 5) {
|
|
321
|
+
return await this.hybridSearchInternal(query, limit, true);
|
|
322
|
+
}
|
|
323
|
+
async hybridSearchWithoutBoost(query, limit = 5) {
|
|
324
|
+
return await this.hybridSearchInternal(query, limit, false);
|
|
325
|
+
}
|
|
326
|
+
async hybridSearchInternal(query, limit = 5, enableHebbian = true) {
|
|
327
|
+
const queryHash = hashQuery(query);
|
|
328
|
+
const poolSize = limit * this.candidatePoolMultiplier;
|
|
329
|
+
// 統一使用 store 的 hybridVectorSearch(向量 + FTS + RRF Fusion)
|
|
330
|
+
const fused = await this.store.hybridVectorSearch(query, poolSize);
|
|
331
|
+
const weighted = fused.map((r) => ({
|
|
332
|
+
...r,
|
|
333
|
+
fusedScore: (this.vectorWeight * r.rankScore) + (this.bm25Weight * r.bm25Score),
|
|
334
|
+
}));
|
|
335
|
+
const scored = weighted.sort((a, b) => b.fusedScore - a.fusedScore).slice(0, poolSize).map((r) => {
|
|
336
|
+
let importance = 0.5;
|
|
337
|
+
let health = 1.0;
|
|
338
|
+
let entities = [];
|
|
339
|
+
let meta = null;
|
|
340
|
+
try {
|
|
341
|
+
if (r.entry.metadata) {
|
|
342
|
+
meta = typeof r.entry.metadata === 'string'
|
|
343
|
+
? JSON.parse(r.entry.metadata)
|
|
344
|
+
: r.entry.metadata;
|
|
345
|
+
if (r.entry.importance !== undefined)
|
|
346
|
+
importance = r.entry.importance;
|
|
347
|
+
if (meta?.health?.healthScore !== undefined)
|
|
348
|
+
health = (typeof meta.health.healthScore === 'number' ? meta.health.healthScore : 100) / 100;
|
|
349
|
+
if (meta?.entities)
|
|
350
|
+
entities = meta.entities;
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
catch { /* metadata parse fail, use defaults */ }
|
|
354
|
+
const similarity = Math.min(r.fusedScore * 60, 1.0);
|
|
355
|
+
let abstractness = 0;
|
|
356
|
+
if (meta?.abstractness !== undefined) {
|
|
357
|
+
abstractness = Number(meta.abstractness) || 0;
|
|
358
|
+
}
|
|
359
|
+
else {
|
|
360
|
+
try {
|
|
361
|
+
const judgement = judgeAbstractness(r.entry.text);
|
|
362
|
+
abstractness = judgement.abstractness;
|
|
363
|
+
}
|
|
364
|
+
catch {
|
|
365
|
+
abstractness = 0;
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
const baseScore = (similarity * 0.5) + (importance * 0.3) + (health * 0.2);
|
|
369
|
+
const penalty = 1 - (0.5 * abstractness);
|
|
370
|
+
const finalScore = baseScore * penalty;
|
|
371
|
+
if (abstractness > 0.3 && typeof this.store.recordSubsystemEffectiveness === 'function') {
|
|
372
|
+
void this.store.recordSubsystemEffectiveness({
|
|
373
|
+
subsystem: 'retrieval',
|
|
374
|
+
event: 'abstract_penalized',
|
|
375
|
+
entityId: r.entry.id,
|
|
376
|
+
outcome: 'penalized',
|
|
377
|
+
queryHash,
|
|
378
|
+
score: penalty,
|
|
379
|
+
count: 1,
|
|
380
|
+
metadata: {
|
|
381
|
+
abstractness,
|
|
382
|
+
baseScore,
|
|
383
|
+
finalScore,
|
|
384
|
+
penalty,
|
|
385
|
+
// P3.3: 雙寫冗餘期。新查詢用 isFromMetadata。Phase A 收尾後砍 hasMetadataAbstractness
|
|
386
|
+
hasMetadataAbstractness: meta?.abstractness !== undefined,
|
|
387
|
+
isFromMetadata: meta?.abstractness !== undefined,
|
|
388
|
+
textPreview: r.entry.text.slice(0, 60),
|
|
389
|
+
},
|
|
390
|
+
}).catch(() => { });
|
|
391
|
+
}
|
|
392
|
+
return {
|
|
393
|
+
...r,
|
|
394
|
+
finalScore,
|
|
395
|
+
};
|
|
396
|
+
});
|
|
397
|
+
const reRanked = scored
|
|
398
|
+
.sort((a, b) => (b.finalScore || 0) - (a.finalScore || 0))
|
|
399
|
+
.slice(0, poolSize);
|
|
400
|
+
// ── Structured Slot 去重:同 slotKey 只留最新 active 版本 ─
|
|
401
|
+
// Phase 1:slotKey 相同的多個版本,優先回傳 status=active 且 createdAt 最新者
|
|
402
|
+
const slotVersionMap = new Map();
|
|
403
|
+
const dedupedResults = [];
|
|
404
|
+
for (let i = 0; i < reRanked.length; i++) {
|
|
405
|
+
const r = reRanked[i];
|
|
406
|
+
const slotKey = r.entry.slotKey;
|
|
407
|
+
if (!slotKey) {
|
|
408
|
+
// 無 slotKey → 直接保留
|
|
409
|
+
dedupedResults.push(r);
|
|
410
|
+
continue;
|
|
411
|
+
}
|
|
412
|
+
// 嘗試從 metadata 取 status(entry.status 可能未定義)
|
|
413
|
+
let status = 'active';
|
|
414
|
+
let createdAt = r.entry.createdAt || 0;
|
|
415
|
+
try {
|
|
416
|
+
const meta = typeof r.entry.metadata === 'string'
|
|
417
|
+
? JSON.parse(r.entry.metadata)
|
|
418
|
+
: (r.entry.metadata || {});
|
|
419
|
+
status = meta?.status || 'active';
|
|
420
|
+
}
|
|
421
|
+
catch { /* ignore */ }
|
|
422
|
+
const isActive = status === 'active';
|
|
423
|
+
const existing = slotVersionMap.get(slotKey);
|
|
424
|
+
if (!existing) {
|
|
425
|
+
if (isActive) {
|
|
426
|
+
// 第一個版本是 active → 直接加入結果
|
|
427
|
+
const idx = dedupedResults.length;
|
|
428
|
+
slotVersionMap.set(slotKey, { idx, createdAt, isActive: true });
|
|
429
|
+
dedupedResults.push(r);
|
|
430
|
+
}
|
|
431
|
+
else {
|
|
432
|
+
// 第一個版本是 inactive → 暫存但不加入結果(idx = -1 表示未入列)
|
|
433
|
+
slotVersionMap.set(slotKey, { idx: -1, createdAt, isActive: false });
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
else if (isActive && createdAt > existing.createdAt) {
|
|
437
|
+
// 更新的 active 版本
|
|
438
|
+
if (existing.idx >= 0 && existing.idx < dedupedResults.length) {
|
|
439
|
+
dedupedResults[existing.idx] = r; // 替換已存在的版本
|
|
440
|
+
}
|
|
441
|
+
else {
|
|
442
|
+
// 之前沒入列(inactive)→ 現在加入
|
|
443
|
+
const newIdx = dedupedResults.length;
|
|
444
|
+
slotVersionMap.set(slotKey, { idx: newIdx, createdAt, isActive: true });
|
|
445
|
+
dedupedResults.push(r);
|
|
446
|
+
continue;
|
|
447
|
+
}
|
|
448
|
+
slotVersionMap.set(slotKey, { idx: existing.idx, createdAt, isActive: true });
|
|
449
|
+
}
|
|
450
|
+
// else: 非 active 或更舊的 active → 跳過
|
|
451
|
+
}
|
|
452
|
+
const finalResults = dedupedResults;
|
|
453
|
+
// ── 記憶鉤子觸發(CRAG 之前,讓 hook 記憶也經過品質審核) ──
|
|
454
|
+
// hookKeywordMap: EVERY triggered hook → drives reportHookOutcome so the engine's
|
|
455
|
+
// triggerCount denominator stays balanced (a triggered hook we don't inject must
|
|
456
|
+
// still get an outcome, else it is silently penalised toward eviction).
|
|
457
|
+
// hookInjectedIds: ONLY hook-exclusive injected candidates → drives the stricter CRAG
|
|
458
|
+
// hook gate. A memory already retrieved on merit must NOT be subjected to it.
|
|
459
|
+
const hookKeywordMap = new Map();
|
|
460
|
+
const hookInjectedIds = new Set();
|
|
461
|
+
const cragInput = [...finalResults]; // 以 finalResults 為基底(已做 slot 去重)
|
|
462
|
+
try {
|
|
463
|
+
const hookResult = await this.triggerHooks(query);
|
|
464
|
+
if (hookResult.triggered) {
|
|
465
|
+
console.log(`[Retriever] Hook triggered: ${hookResult.relatedMemories.length} related memories`);
|
|
466
|
+
const existingIds = new Set(finalResults.map(r => r.entry.id));
|
|
467
|
+
for (const hookMem of hookResult.relatedMemories) {
|
|
468
|
+
const memoryId = hookMem.memory.id;
|
|
469
|
+
hookKeywordMap.set(memoryId, hookMem.viaHook); // all triggered → balanced outcome
|
|
470
|
+
this.recordHookEffectiveness({
|
|
471
|
+
event: "hook_triggered",
|
|
472
|
+
entityId: memoryId,
|
|
473
|
+
queryHash,
|
|
474
|
+
outcome: "triggered",
|
|
475
|
+
score: hookMem.score,
|
|
476
|
+
metadata: {
|
|
477
|
+
keyword: hookMem.viaHook,
|
|
478
|
+
sourceMemoryId: memoryId,
|
|
479
|
+
},
|
|
480
|
+
});
|
|
481
|
+
if (existingIds.has(memoryId))
|
|
482
|
+
continue; // already retrieved on merit; don't re-inject / don't strict-gate
|
|
483
|
+
// Fetch the real stored vector so the hook candidate goes through the SAME CRAG
|
|
484
|
+
// vector-distance gate as everything else. queryHookBearing() returns vector:[]
|
|
485
|
+
// (scan efficiency); only the few injected hooks pay this lookup, so Fix-1's
|
|
486
|
+
// no-vector scan stays cheap.
|
|
487
|
+
let hookVector = [];
|
|
488
|
+
try {
|
|
489
|
+
const full = await this.store.getById(memoryId, true);
|
|
490
|
+
if (full && Array.isArray(full.vector))
|
|
491
|
+
hookVector = full.vector;
|
|
492
|
+
}
|
|
493
|
+
catch { /* fall through to vector-missing handling */ }
|
|
494
|
+
if (hookVector.length === 0) {
|
|
495
|
+
// Data-integrity miss (no stored vector), NOT a relevance failure. Skip
|
|
496
|
+
// injection and record distinctly so quality tracking can tell them apart.
|
|
497
|
+
this.recordHookEffectiveness({
|
|
498
|
+
event: "hook_vector_missing",
|
|
499
|
+
entityId: memoryId,
|
|
500
|
+
queryHash,
|
|
501
|
+
outcome: "dropped",
|
|
502
|
+
score: hookMem.score,
|
|
503
|
+
metadata: { keyword: hookMem.viaHook, sourceMemoryId: memoryId },
|
|
504
|
+
});
|
|
505
|
+
continue;
|
|
506
|
+
}
|
|
507
|
+
existingIds.add(memoryId);
|
|
508
|
+
hookInjectedIds.add(memoryId); // subject to the stricter CRAG hook gate
|
|
509
|
+
const injectScore = Math.min(hookMem.score, HOOK_INJECT_SCORE_CAP);
|
|
510
|
+
cragInput.push({
|
|
511
|
+
entry: { ...hookMem.memory, vector: hookVector },
|
|
512
|
+
vectorScore: 0,
|
|
513
|
+
rankScore: 0,
|
|
514
|
+
rawDistance: Number.POSITIVE_INFINITY,
|
|
515
|
+
bm25Score: 0,
|
|
516
|
+
fusedScore: injectScore,
|
|
517
|
+
finalScore: injectScore,
|
|
518
|
+
});
|
|
519
|
+
}
|
|
520
|
+
}
|
|
521
|
+
}
|
|
522
|
+
catch (err) {
|
|
523
|
+
console.warn('[Retriever] Hook execution failed:', err);
|
|
524
|
+
}
|
|
525
|
+
const cragFiltered = await this.cragEvaluate(query, cragInput, enableHebbian, hookInjectedIds);
|
|
526
|
+
console.log(`[Retriever] Search complete: ${cragInput.length} candidates -> ${cragFiltered.length} CRAG results (hook-injected=${hookInjectedIds.size})`);
|
|
527
|
+
// ── Hook 品質回報:CRAG 後比對哪些 hook 記憶存活 ──
|
|
528
|
+
if (hookKeywordMap.size > 0 && this.hooksEngine) {
|
|
529
|
+
const retainedIds = new Set(cragFiltered.map(r => r.entry.id));
|
|
530
|
+
const cragInputById = new Map(cragInput.map(r => [r.entry.id, r]));
|
|
531
|
+
const cragFilteredById = new Map(cragFiltered.map(r => [r.entry.id, r]));
|
|
532
|
+
for (const [memId, keyword] of hookKeywordMap) {
|
|
533
|
+
const retained = retainedIds.has(memId);
|
|
534
|
+
await this.hooksEngine.reportHookOutcome(keyword, retained);
|
|
535
|
+
const scored = cragFilteredById.get(memId) ?? cragInputById.get(memId);
|
|
536
|
+
this.recordHookEffectiveness({
|
|
537
|
+
event: "hook_crag_retained",
|
|
538
|
+
entityId: memId,
|
|
539
|
+
queryHash,
|
|
540
|
+
outcome: retained ? "retained" : "dropped",
|
|
541
|
+
score: Number(scored?.finalScore ?? scored?.fusedScore ?? 0),
|
|
542
|
+
metadata: { keyword },
|
|
543
|
+
});
|
|
544
|
+
}
|
|
545
|
+
}
|
|
546
|
+
// ── 因果鏈擴展 ──
|
|
547
|
+
const chainResults = [];
|
|
548
|
+
const chainIds = new Set(cragFiltered.map(r => r.entry.id));
|
|
549
|
+
const causalSeedMap = new Map();
|
|
550
|
+
for (const result of cragFiltered) {
|
|
551
|
+
try {
|
|
552
|
+
const chain = await this.getCausalChain(result.entry.id, 2);
|
|
553
|
+
for (const node of chain) {
|
|
554
|
+
if (node.origin === 'seed')
|
|
555
|
+
continue;
|
|
556
|
+
if (chainIds.has(node.entry.id))
|
|
557
|
+
continue;
|
|
558
|
+
chainIds.add(node.entry.id);
|
|
559
|
+
const seedFinalScore = result.finalScore ?? result.fusedScore;
|
|
560
|
+
const causalScore = seedFinalScore * 0.7;
|
|
561
|
+
const chainIndex = chainResults.length;
|
|
562
|
+
chainResults.push({
|
|
563
|
+
entry: node.entry,
|
|
564
|
+
vectorScore: 0,
|
|
565
|
+
rankScore: 0,
|
|
566
|
+
rawDistance: Number.POSITIVE_INFINITY,
|
|
567
|
+
bm25Score: 0,
|
|
568
|
+
fusedScore: causalScore,
|
|
569
|
+
finalScore: causalScore,
|
|
570
|
+
});
|
|
571
|
+
causalSeedMap.set(node.entry.id, result.entry.id);
|
|
572
|
+
const recordSubsystemEffectiveness = this.store.recordSubsystemEffectiveness;
|
|
573
|
+
if (typeof recordSubsystemEffectiveness === 'function') {
|
|
574
|
+
void recordSubsystemEffectiveness.call(this.store, {
|
|
575
|
+
subsystem: 'causal',
|
|
576
|
+
event: 'expansion_added',
|
|
577
|
+
entityId: node.entry.id,
|
|
578
|
+
relatedId: result.entry.id,
|
|
579
|
+
queryHash,
|
|
580
|
+
outcome: 'added',
|
|
581
|
+
score: causalScore,
|
|
582
|
+
count: 1,
|
|
583
|
+
durationMs: 0,
|
|
584
|
+
metadata: {
|
|
585
|
+
seedFinalScore,
|
|
586
|
+
seedViaHook: hookKeywordMap.has(result.entry.id),
|
|
587
|
+
chainOrigin: node.origin,
|
|
588
|
+
hopFromSeed: node.hopFromSeed,
|
|
589
|
+
chainIndex,
|
|
590
|
+
},
|
|
591
|
+
}).catch((err) => {
|
|
592
|
+
console.warn('[PR-A3] expansion_added write failed:', err?.message ?? err);
|
|
593
|
+
});
|
|
594
|
+
}
|
|
595
|
+
}
|
|
596
|
+
}
|
|
597
|
+
catch (err) {
|
|
598
|
+
console.warn(`[Retriever] Causal-chain expansion failed for ${result.entry.id.slice(0, 8)}:`, err);
|
|
599
|
+
}
|
|
600
|
+
}
|
|
601
|
+
let gatedChainResults = chainResults;
|
|
602
|
+
if (chainResults.length > 0 && isCragCrossEncoderGateEnabled()) {
|
|
603
|
+
gatedChainResults = await applyCragCrossEncoderGate(query, chainResults, {
|
|
604
|
+
cacheDir: this.rerankerCacheDir,
|
|
605
|
+
topK: chainResults.length,
|
|
606
|
+
logger: console,
|
|
607
|
+
});
|
|
608
|
+
const dropped = chainResults.length - gatedChainResults.length;
|
|
609
|
+
if (dropped > 0) {
|
|
610
|
+
console.log(`[CRAG] causal-chain cross-encoder gate dropped ${dropped}/${chainResults.length} bypass candidates`);
|
|
611
|
+
}
|
|
612
|
+
}
|
|
613
|
+
// 合併並重新排序
|
|
614
|
+
const allResults = [...cragFiltered, ...gatedChainResults]
|
|
615
|
+
.sort((a, b) => (b.finalScore || 0) - (a.finalScore || 0));
|
|
616
|
+
let total = allResults.slice(0, limit);
|
|
617
|
+
if (isCoverageSelectionEnabled(process.env)) {
|
|
618
|
+
// Known limitation: flag ON causes a second cross-encode pass over survivors (separate from the gate pass). Sharing a single pass with the gate is a future optimization.
|
|
619
|
+
const coverageScores = await scoreCandidates(query, allResults, {
|
|
620
|
+
topK: allResults.length,
|
|
621
|
+
cacheDir: this.rerankerCacheDir,
|
|
622
|
+
logger: console,
|
|
623
|
+
});
|
|
624
|
+
if (coverageScores !== null) {
|
|
625
|
+
const relevance = coverageScores.scored.map(scored => scored.logit);
|
|
626
|
+
const vectors = coverageScores.scored.map(scored => scored.candidate.entry.vector ?? null);
|
|
627
|
+
const order = mmrOrder(relevance, vectors, coverageLambda(process.env));
|
|
628
|
+
total = order.map(index => coverageScores.scored[index].candidate).slice(0, limit);
|
|
629
|
+
}
|
|
630
|
+
}
|
|
631
|
+
if (gatedChainResults.length > 0) {
|
|
632
|
+
const totalById = new Map();
|
|
633
|
+
total.forEach((r, idx) => {
|
|
634
|
+
if (r?.entry?.id)
|
|
635
|
+
totalById.set(r.entry.id, idx);
|
|
636
|
+
});
|
|
637
|
+
const recordSubsystemEffectiveness = this.store.recordSubsystemEffectiveness;
|
|
638
|
+
if (typeof recordSubsystemEffectiveness === 'function') {
|
|
639
|
+
for (const cr of gatedChainResults) {
|
|
640
|
+
const expandedId = cr.entry.id;
|
|
641
|
+
const rankInFinal = totalById.has(expandedId) ? totalById.get(expandedId) : -1;
|
|
642
|
+
const survived = rankInFinal >= 0;
|
|
643
|
+
const seedId = causalSeedMap.get(expandedId) ?? '';
|
|
644
|
+
void recordSubsystemEffectiveness.call(this.store, {
|
|
645
|
+
subsystem: 'causal',
|
|
646
|
+
event: 'expansion_survived_rank',
|
|
647
|
+
entityId: expandedId,
|
|
648
|
+
relatedId: seedId,
|
|
649
|
+
queryHash,
|
|
650
|
+
outcome: survived ? 'survived' : 'dropped',
|
|
651
|
+
score: Number(cr.finalScore ?? 0),
|
|
652
|
+
count: 1,
|
|
653
|
+
durationMs: 0,
|
|
654
|
+
metadata: {
|
|
655
|
+
rankInFinal,
|
|
656
|
+
totalLimit: limit,
|
|
657
|
+
chainResultsCount: gatedChainResults.length,
|
|
658
|
+
chainResultsBeforeGate: chainResults.length,
|
|
659
|
+
cragFilteredCount: cragFiltered.length,
|
|
660
|
+
},
|
|
661
|
+
}).catch((err) => {
|
|
662
|
+
console.warn('[PR-A3] expansion_survived_rank write failed:', err?.message ?? err);
|
|
663
|
+
});
|
|
664
|
+
}
|
|
665
|
+
}
|
|
666
|
+
}
|
|
667
|
+
if (gatedChainResults.length > 0) {
|
|
668
|
+
console.log(`[Retriever] Results: ${cragFiltered.length} direct + ${gatedChainResults.length} causal-chain = ${total.length} total`);
|
|
669
|
+
}
|
|
670
|
+
this.enqueueRecallMetadata(total.map(r => r.entry));
|
|
671
|
+
return {
|
|
672
|
+
results: total,
|
|
673
|
+
hookOriginIds: Array.from(hookKeywordMap.keys()),
|
|
674
|
+
hookOriginKeywords: Object.fromEntries(hookKeywordMap),
|
|
675
|
+
queryHash,
|
|
676
|
+
};
|
|
677
|
+
}
|
|
678
|
+
recordHookEffectiveness(event) {
|
|
679
|
+
const recordSubsystemEffectiveness = this.store.recordSubsystemEffectiveness;
|
|
680
|
+
if (typeof recordSubsystemEffectiveness !== "function")
|
|
681
|
+
return;
|
|
682
|
+
void recordSubsystemEffectiveness.call(this.store, {
|
|
683
|
+
subsystem: "hooks",
|
|
684
|
+
relatedId: "",
|
|
685
|
+
count: 1,
|
|
686
|
+
durationMs: 0,
|
|
687
|
+
...event,
|
|
688
|
+
}).catch((err) => {
|
|
689
|
+
console.warn("[Retriever] hooks effectiveness write failed:", err?.message ?? err);
|
|
690
|
+
});
|
|
691
|
+
}
|
|
692
|
+
enqueueRecallMetadata(entries) {
|
|
693
|
+
if (entries.length === 0)
|
|
694
|
+
return;
|
|
695
|
+
// Same confirmatory-A/B rationale as enqueueBoostHealth: recall metadata (lastRecalledAt/recallCount)
|
|
696
|
+
// is a cross-question write-on-read; coverage selects a different memory set per arm, so leaving it
|
|
697
|
+
// un-gated drifts recall state differently between off/on. No-op only under the flag; production untouched.
|
|
698
|
+
if (process.env.MR_OTTER_READONLY === '1')
|
|
699
|
+
return;
|
|
700
|
+
if (this.recallMetadataQueue.length >= BACKGROUND_WRITE_QUEUE_LIMIT) {
|
|
701
|
+
this.recallMetadataQueue.shift();
|
|
702
|
+
this.recordQueueDrop("recall_metadata_queue_drop");
|
|
703
|
+
}
|
|
704
|
+
this.recallMetadataQueue.push(entries);
|
|
705
|
+
this.drainRecallMetadataQueue().catch((err) => {
|
|
706
|
+
console.warn('[Retriever] recall metadata queue failed:', err?.message ?? err);
|
|
707
|
+
});
|
|
708
|
+
}
|
|
709
|
+
async drainRecallMetadataQueue() {
|
|
710
|
+
if (this.recallMetadataQueueRunning)
|
|
711
|
+
return;
|
|
712
|
+
this.recallMetadataQueueRunning = true;
|
|
713
|
+
try {
|
|
714
|
+
while (this.recallMetadataQueue.length > 0) {
|
|
715
|
+
const entries = this.recallMetadataQueue.shift();
|
|
716
|
+
try {
|
|
717
|
+
await this.store.recordMemoryRecalls(entries);
|
|
718
|
+
}
|
|
719
|
+
catch (err) {
|
|
720
|
+
console.warn('[Retriever] recall metadata update failed:', err?.message ?? err);
|
|
721
|
+
this.recordBackgroundWriteFailure("recall_metadata_queue_worker", err);
|
|
722
|
+
}
|
|
723
|
+
}
|
|
724
|
+
}
|
|
725
|
+
finally {
|
|
726
|
+
this.recallMetadataQueueRunning = false;
|
|
727
|
+
}
|
|
728
|
+
}
|
|
729
|
+
enqueueBoostHealth(id) {
|
|
730
|
+
// Benchmark confirmatory A/B (MR_OTTER_READONLY=1) freezes the river so each question is an
|
|
731
|
+
// independent probe whose only cross-arm difference is coverage selection. The Hebbian/merge
|
|
732
|
+
// boost is a cross-question write-on-read that drifts healthScore differently per arm (coverage
|
|
733
|
+
// boosts a different memory set), so gate it at this single chokepoint for every boost path
|
|
734
|
+
// (incl. the non-enableHebbian merge boosts). No-op only under the flag; production is untouched.
|
|
735
|
+
if (process.env.MR_OTTER_READONLY === '1')
|
|
736
|
+
return;
|
|
737
|
+
if (this.boostHealthQueue.length >= BACKGROUND_WRITE_QUEUE_LIMIT) {
|
|
738
|
+
this.boostHealthQueue.shift();
|
|
739
|
+
this.recordQueueDrop("boost_health_queue_drop");
|
|
740
|
+
}
|
|
741
|
+
this.boostHealthQueue.push(id);
|
|
742
|
+
this.drainBoostHealthQueue().catch((err) => {
|
|
743
|
+
console.warn('[Retriever] boostHealth queue failed:', err?.message ?? err);
|
|
744
|
+
});
|
|
745
|
+
}
|
|
746
|
+
async drainBoostHealthQueue() {
|
|
747
|
+
if (this.boostHealthQueueRunning)
|
|
748
|
+
return;
|
|
749
|
+
this.boostHealthQueueRunning = true;
|
|
750
|
+
try {
|
|
751
|
+
while (this.boostHealthQueue.length > 0) {
|
|
752
|
+
const id = this.boostHealthQueue.shift();
|
|
753
|
+
try {
|
|
754
|
+
await this.store.boostHealth(id);
|
|
755
|
+
}
|
|
756
|
+
catch (err) {
|
|
757
|
+
console.warn(`[Retriever] boostHealth update failed for ${id.slice(0, 8)}:`, err?.message ?? err);
|
|
758
|
+
this.recordBackgroundWriteFailure("boost_health_queue_worker", err);
|
|
759
|
+
}
|
|
760
|
+
}
|
|
761
|
+
}
|
|
762
|
+
finally {
|
|
763
|
+
this.boostHealthQueueRunning = false;
|
|
764
|
+
}
|
|
765
|
+
}
|
|
766
|
+
recordQueueDrop(operationName) {
|
|
767
|
+
const recordConflictStat = this.store.recordConflictStat;
|
|
768
|
+
if (typeof recordConflictStat !== "function")
|
|
769
|
+
return;
|
|
770
|
+
recordConflictStat.call(this.store, {
|
|
771
|
+
operationName,
|
|
772
|
+
callerPath: "src/retriever-v4.ts",
|
|
773
|
+
attempt: 0,
|
|
774
|
+
finalOutcome: "dropped",
|
|
775
|
+
fragmentId: null,
|
|
776
|
+
}).catch((err) => {
|
|
777
|
+
console.warn('[Retriever] conflict_stats drop metric failed:', err?.message ?? err);
|
|
778
|
+
});
|
|
779
|
+
}
|
|
780
|
+
recordBackgroundWriteFailure(operationName, err) {
|
|
781
|
+
const recordConflictStat = this.store.recordConflictStat;
|
|
782
|
+
if (typeof recordConflictStat !== "function")
|
|
783
|
+
return;
|
|
784
|
+
recordConflictStat.call(this.store, {
|
|
785
|
+
operationName,
|
|
786
|
+
callerPath: "src/retriever-v4.ts",
|
|
787
|
+
attempt: 1,
|
|
788
|
+
finalOutcome: "failed",
|
|
789
|
+
fragmentId: this.extractFragmentId(err?.message ?? String(err)),
|
|
790
|
+
}).catch((metricErr) => {
|
|
791
|
+
console.warn('[Retriever] conflict_stats failure metric failed:', metricErr?.message ?? metricErr);
|
|
792
|
+
});
|
|
793
|
+
}
|
|
794
|
+
extractFragmentId(message) {
|
|
795
|
+
const match = message.match(/Fragment\s*\{\s*id:\s*(\d+)/);
|
|
796
|
+
return match?.[1] ?? null;
|
|
797
|
+
}
|
|
798
|
+
async vectorOnly(query, limit = 5) {
|
|
799
|
+
const vector = await this.embedder.embed(query);
|
|
800
|
+
return await this.store.vectorSearch(vector, limit);
|
|
801
|
+
}
|
|
802
|
+
async ftsOnly(query, limit = 5) {
|
|
803
|
+
return await this.store.ftsSearch(query, limit);
|
|
804
|
+
}
|
|
805
|
+
/**
|
|
806
|
+
* 追蹤因果鏈:根據 parentId 向上找原因、向下找結果
|
|
807
|
+
* @param memoryId 起點記憶 ID
|
|
808
|
+
* @param depth 深度(預設 2:爺爺→爸爸→我→孩子→孫子)
|
|
809
|
+
* @returns 因果鏈上的所有 MemoryEntry
|
|
810
|
+
*/
|
|
811
|
+
async getCausalChain(memoryId, depth = 2) {
|
|
812
|
+
const visited = new Set();
|
|
813
|
+
const chain = [];
|
|
814
|
+
const queue = [
|
|
815
|
+
{ id: memoryId, currentDepth: 0, origin: 'seed' }
|
|
816
|
+
];
|
|
817
|
+
const MAX_CHAIN_NODES = 20; // P1-1 修復:硬性上限,防止大量子節點爆炸
|
|
818
|
+
while (queue.length > 0 && chain.length < MAX_CHAIN_NODES) {
|
|
819
|
+
const item = queue.shift();
|
|
820
|
+
if (!item || typeof item.id !== 'string')
|
|
821
|
+
continue;
|
|
822
|
+
const { id, currentDepth, origin } = item;
|
|
823
|
+
if (visited.has(id) || currentDepth > depth)
|
|
824
|
+
continue;
|
|
825
|
+
visited.add(id);
|
|
826
|
+
let entry = await this.store.getById(id);
|
|
827
|
+
// fallback:getById 找不到就 query 直接用 id 查(先驗 UUID 避免汙染)
|
|
828
|
+
if (!entry) {
|
|
829
|
+
if (!/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(id)) {
|
|
830
|
+
console.warn(`[Retriever] getCausalChain skipped invalid id: ${id.slice(0, 40)}`);
|
|
831
|
+
continue;
|
|
832
|
+
}
|
|
833
|
+
const results = await this.store.query(`id = '${id}'`);
|
|
834
|
+
entry = results[0];
|
|
835
|
+
}
|
|
836
|
+
if (!entry) {
|
|
837
|
+
continue;
|
|
838
|
+
}
|
|
839
|
+
// PR-A3.1: 只在 fallback query 回傳的 entry.id 與查詢 id 不同時才做第二層去重。
|
|
840
|
+
// 正常路徑 entry.id === id,id 已在迴圈開頭 visited.add 過,不可再擋(原 bug)。
|
|
841
|
+
if (entry.id !== id) {
|
|
842
|
+
if (visited.has(entry.id))
|
|
843
|
+
continue;
|
|
844
|
+
visited.add(entry.id);
|
|
845
|
+
}
|
|
846
|
+
chain.push({
|
|
847
|
+
entry,
|
|
848
|
+
hopFromSeed: currentDepth,
|
|
849
|
+
origin,
|
|
850
|
+
});
|
|
851
|
+
// 往上:找這個記憶的 parent(原因)
|
|
852
|
+
if (entry.parentId && currentDepth < depth) {
|
|
853
|
+
queue.push({ id: entry.parentId, currentDepth: currentDepth + 1, origin: 'parent' });
|
|
854
|
+
}
|
|
855
|
+
// 往下:找以這個記憶為 parent 的(結果)
|
|
856
|
+
if (currentDepth < depth) {
|
|
857
|
+
try {
|
|
858
|
+
if (!/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(entry.id)) {
|
|
859
|
+
console.warn(`[Retriever] getCausalChain skipped invalid parent id: ${entry.id.slice(0, 40)}`);
|
|
860
|
+
continue;
|
|
861
|
+
}
|
|
862
|
+
const children = await this.store.query(`\`parentId\` = '${entry.id}'`);
|
|
863
|
+
for (const child of children) {
|
|
864
|
+
if (!visited.has(child.id)) {
|
|
865
|
+
queue.push({ id: child.id, currentDepth: currentDepth + 1, origin: 'child' });
|
|
866
|
+
}
|
|
867
|
+
}
|
|
868
|
+
}
|
|
869
|
+
catch (err) {
|
|
870
|
+
console.warn(`[Retriever] Query children failed:`, err.message);
|
|
871
|
+
}
|
|
872
|
+
}
|
|
873
|
+
}
|
|
874
|
+
return chain;
|
|
875
|
+
}
|
|
876
|
+
setHooksEngine(engine) {
|
|
877
|
+
this.hooksEngine = engine;
|
|
878
|
+
}
|
|
879
|
+
getStore() {
|
|
880
|
+
return this.store;
|
|
881
|
+
}
|
|
882
|
+
async updateMemoryRecord(id, updates) {
|
|
883
|
+
await this.store.update(id, updates);
|
|
884
|
+
}
|
|
885
|
+
async triggerHooks(query) {
|
|
886
|
+
if (!this.hooksEngine) {
|
|
887
|
+
return { triggered: false, relatedMemories: [], naturalLanguage: "" };
|
|
888
|
+
}
|
|
889
|
+
return await this.hooksEngine.triggerHooks(query);
|
|
890
|
+
}
|
|
891
|
+
async cragEvaluate(query, results, enableHebbian = true, hookInjectedIds) {
|
|
892
|
+
const queryHash = hashQuery(query);
|
|
893
|
+
const candidateMetaById = new Map();
|
|
894
|
+
results.forEach((result, index) => {
|
|
895
|
+
candidateMetaById.set(result.entry.id, {
|
|
896
|
+
rank: index,
|
|
897
|
+
score: Number(result.finalScore ?? result.fusedScore ?? 0),
|
|
898
|
+
});
|
|
899
|
+
});
|
|
900
|
+
// Only hook-EXCLUSIVELY-injected candidates get the stricter hook gate; memories
|
|
901
|
+
// retrieved on merit (even if also hook-triggered) follow the normal CRAG path.
|
|
902
|
+
const hookSet = hookInjectedIds ?? new Set();
|
|
903
|
+
const hooksTotal = results.reduce((acc, result) => acc + (hookSet.has(result.entry.id) ? 1 : 0), 0);
|
|
904
|
+
let vipCount = 0;
|
|
905
|
+
let rerankYesCount = 0;
|
|
906
|
+
let rerankNoCount = 0;
|
|
907
|
+
let partialMergedCount = 0;
|
|
908
|
+
let partialDroppedCount = 0;
|
|
909
|
+
let textEmptyDropCount = 0;
|
|
910
|
+
let rerankerErrorCount = 0;
|
|
911
|
+
const writeCandidateEvent = (result, outcome, rerankerVerdict, rejectReason) => {
|
|
912
|
+
const meta = candidateMetaById.get(result.entry.id);
|
|
913
|
+
const viaHook = hookSet.has(result.entry.id);
|
|
914
|
+
const hookKeyword = undefined; // keyword lives in hook_triggered / hook_crag_retained events
|
|
915
|
+
const recordSubsystemEffectiveness = this.store.recordSubsystemEffectiveness;
|
|
916
|
+
if (typeof recordSubsystemEffectiveness !== "function")
|
|
917
|
+
return;
|
|
918
|
+
void recordSubsystemEffectiveness.call(this.store, {
|
|
919
|
+
subsystem: "retrieval",
|
|
920
|
+
event: "crag_filter",
|
|
921
|
+
outcome,
|
|
922
|
+
entityId: result.entry.id,
|
|
923
|
+
relatedId: "",
|
|
924
|
+
queryHash,
|
|
925
|
+
score: meta?.score ?? 0,
|
|
926
|
+
count: 1,
|
|
927
|
+
durationMs: 0,
|
|
928
|
+
metadata: {
|
|
929
|
+
candidateRank: meta?.rank ?? -1,
|
|
930
|
+
candidateScore: meta?.score ?? 0,
|
|
931
|
+
cragVipDist: CRAG_VIP_DIST,
|
|
932
|
+
cragDistYes: CRAG_DIST_YES,
|
|
933
|
+
cragDistNo: CRAG_DIST_NO,
|
|
934
|
+
qwenDist: result.__qwenDist,
|
|
935
|
+
rerankerVerdict,
|
|
936
|
+
rejectReason,
|
|
937
|
+
viaHook,
|
|
938
|
+
hookKeyword,
|
|
939
|
+
},
|
|
940
|
+
}).catch((err) => {
|
|
941
|
+
console.warn("[PR-E8] crag candidate event failed:", err?.message ?? err);
|
|
942
|
+
});
|
|
943
|
+
};
|
|
944
|
+
// ── Embed the query once (query mode); CRAG scores every candidate by Qwen
|
|
945
|
+
// cosine distance to its stored ('store'-mode) vector. See ticket §"Key
|
|
946
|
+
// enabler". null queryVector (embed outage) → degrade to pass-through. ──
|
|
947
|
+
let queryVector = null;
|
|
948
|
+
try {
|
|
949
|
+
const v = await this.embedder.embed(query, "query");
|
|
950
|
+
queryVector = Array.isArray(v) && v.length > 0 ? v : null;
|
|
951
|
+
}
|
|
952
|
+
catch (err) {
|
|
953
|
+
console.warn("[CRAG] query embed failed; CRAG runs in pass-through:", err?.message ?? err);
|
|
954
|
+
}
|
|
955
|
+
// qwenDist: +Infinity when uncomputable (missing vector / no queryVector) so a
|
|
956
|
+
// candidate is never silently dropped — it routes to the "partial" path.
|
|
957
|
+
const qwenDist = (result) => queryVector ? cosineDistance(queryVector, result.entry.vector) : Number.POSITIVE_INFINITY;
|
|
958
|
+
const vipMemories = [];
|
|
959
|
+
const reviewMemories = [];
|
|
960
|
+
for (const result of results) {
|
|
961
|
+
if (result.entry.id.startsWith("init_"))
|
|
962
|
+
continue;
|
|
963
|
+
const dist = qwenDist(result);
|
|
964
|
+
result.__qwenDist = dist;
|
|
965
|
+
// VIP = unambiguously close in Qwen space. Distance-confirmed only; no
|
|
966
|
+
// top-rank auto-pass (finalScore blends importance/health, not pure vector).
|
|
967
|
+
if (queryVector && dist <= CRAG_VIP_DIST) {
|
|
968
|
+
vipMemories.push(result);
|
|
969
|
+
}
|
|
970
|
+
else {
|
|
971
|
+
reviewMemories.push(result);
|
|
972
|
+
}
|
|
973
|
+
}
|
|
974
|
+
const filteredResults = [];
|
|
975
|
+
/** 存放 partial 記憶,等待 Entity Synergy Merge */
|
|
976
|
+
const partialMemories = [];
|
|
977
|
+
for (const result of vipMemories) {
|
|
978
|
+
const processedText = this.filterHooks(result.entry.text);
|
|
979
|
+
if (enableHebbian)
|
|
980
|
+
this.enqueueBoostHealth(result.entry.id);
|
|
981
|
+
if (processedText.trim()) {
|
|
982
|
+
filteredResults.push({ ...result, entry: { ...result.entry, text: processedText } });
|
|
983
|
+
vipCount += 1;
|
|
984
|
+
writeCandidateEvent(result, "vip_pass", undefined, undefined);
|
|
985
|
+
}
|
|
986
|
+
else {
|
|
987
|
+
textEmptyDropCount += 1;
|
|
988
|
+
writeCandidateEvent(result, "text_empty_drop", undefined, "text_empty");
|
|
989
|
+
}
|
|
990
|
+
}
|
|
991
|
+
for (const result of reviewMemories) {
|
|
992
|
+
try {
|
|
993
|
+
// Qwen-cosine verdict on the distance computed at the VIP split. An
|
|
994
|
+
// uncomputable distance (embed outage / missing vector) keeps the
|
|
995
|
+
// candidate, mirroring the old reranker error-path — never a silent drop.
|
|
996
|
+
const d = result.__qwenDist;
|
|
997
|
+
const viaHook = hookSet.has(result.entry.id);
|
|
998
|
+
// Hook candidates: stricter keep, and NEVER auto-keep an uncomputable distance
|
|
999
|
+
// (no partial-merge rescue either — a bypass-recall hook must earn its place by
|
|
1000
|
+
// real vector relevance or be dropped).
|
|
1001
|
+
const relevance = viaHook
|
|
1002
|
+
? (!Number.isFinite(d) ? "no" : d <= HOOK_CRAG_DIST_YES ? "yes" : "no")
|
|
1003
|
+
: (!Number.isFinite(d) ? "yes"
|
|
1004
|
+
: d <= CRAG_DIST_YES ? "yes"
|
|
1005
|
+
: d >= CRAG_DIST_NO ? "no"
|
|
1006
|
+
: "partial");
|
|
1007
|
+
const evaluation = { relevance };
|
|
1008
|
+
let processedText = result.entry.text;
|
|
1009
|
+
if (evaluation.relevance === "no") {
|
|
1010
|
+
rerankNoCount += 1;
|
|
1011
|
+
writeCandidateEvent(result, "rerank_no", "no", "reranker_no");
|
|
1012
|
+
continue;
|
|
1013
|
+
}
|
|
1014
|
+
else if (evaluation.relevance === "partial") {
|
|
1015
|
+
processedText = this.filterHooks(result.entry.text);
|
|
1016
|
+
// ── 收集 partial,嘗試合併搶救 ──
|
|
1017
|
+
partialMemories.push({ ...result, entry: { ...result.entry, text: processedText } });
|
|
1018
|
+
if (enableHebbian)
|
|
1019
|
+
this.enqueueBoostHealth(result.entry.id);
|
|
1020
|
+
continue;
|
|
1021
|
+
}
|
|
1022
|
+
else {
|
|
1023
|
+
processedText = this.filterHooks(result.entry.text);
|
|
1024
|
+
}
|
|
1025
|
+
if (enableHebbian && evaluation.relevance === "yes") {
|
|
1026
|
+
this.enqueueBoostHealth(result.entry.id);
|
|
1027
|
+
}
|
|
1028
|
+
if (processedText.trim()) {
|
|
1029
|
+
filteredResults.push({ ...result, entry: { ...result.entry, text: processedText } });
|
|
1030
|
+
if (evaluation.relevance === "yes") {
|
|
1031
|
+
rerankYesCount += 1;
|
|
1032
|
+
writeCandidateEvent(result, "rerank_yes", "yes", undefined);
|
|
1033
|
+
}
|
|
1034
|
+
}
|
|
1035
|
+
else if (evaluation.relevance === "yes") {
|
|
1036
|
+
textEmptyDropCount += 1;
|
|
1037
|
+
writeCandidateEvent(result, "text_empty_drop", "yes", "text_empty");
|
|
1038
|
+
}
|
|
1039
|
+
}
|
|
1040
|
+
catch (err) {
|
|
1041
|
+
console.error(`[CRAG] Evaluation failed ${result.entry.id}:`, err);
|
|
1042
|
+
filteredResults.push(result);
|
|
1043
|
+
rerankerErrorCount += 1;
|
|
1044
|
+
writeCandidateEvent(result, "reranker_error", undefined, undefined);
|
|
1045
|
+
}
|
|
1046
|
+
}
|
|
1047
|
+
// ── Entity Synergy Merge:搶救 partial 記憶 ──
|
|
1048
|
+
if (partialMemories.length >= 2) {
|
|
1049
|
+
const mergedCountBefore = partialMemories.length;
|
|
1050
|
+
const synergyMerged = await this.trySynergyMerge(query, partialMemories, enableHebbian, queryVector);
|
|
1051
|
+
for (const merged of synergyMerged) {
|
|
1052
|
+
filteredResults.push(merged);
|
|
1053
|
+
}
|
|
1054
|
+
const survivorIds = new Set(synergyMerged.map((result) => result.entry.id));
|
|
1055
|
+
for (const partial of partialMemories) {
|
|
1056
|
+
if (survivorIds.has(partial.entry.id)) {
|
|
1057
|
+
partialMergedCount += 1;
|
|
1058
|
+
writeCandidateEvent(partial, "partial_merged", "partial", undefined);
|
|
1059
|
+
}
|
|
1060
|
+
else {
|
|
1061
|
+
partialDroppedCount += 1;
|
|
1062
|
+
writeCandidateEvent(partial, "partial_dropped", "partial", "partial_dropped");
|
|
1063
|
+
}
|
|
1064
|
+
}
|
|
1065
|
+
console.log(`[CRAG] Synergy merge: ${mergedCountBefore} partial -> ${synergyMerged.length} promoted to result pool`);
|
|
1066
|
+
}
|
|
1067
|
+
else {
|
|
1068
|
+
for (const partial of partialMemories) {
|
|
1069
|
+
partialDroppedCount += 1;
|
|
1070
|
+
writeCandidateEvent(partial, "partial_dropped", "partial", "partial_dropped");
|
|
1071
|
+
}
|
|
1072
|
+
}
|
|
1073
|
+
const survivorsTotal = filteredResults.length;
|
|
1074
|
+
const hooksSurvivedTotal = filteredResults.reduce((acc, result) => acc + (hookSet.has(result.entry.id) ? 1 : 0), 0);
|
|
1075
|
+
const recordSubsystemEffectiveness = this.store.recordSubsystemEffectiveness;
|
|
1076
|
+
if (typeof recordSubsystemEffectiveness === "function") {
|
|
1077
|
+
void recordSubsystemEffectiveness.call(this.store, {
|
|
1078
|
+
subsystem: "retrieval",
|
|
1079
|
+
event: "crag_summary",
|
|
1080
|
+
outcome: "completed",
|
|
1081
|
+
entityId: "",
|
|
1082
|
+
relatedId: "",
|
|
1083
|
+
queryHash,
|
|
1084
|
+
score: 0,
|
|
1085
|
+
count: 1,
|
|
1086
|
+
durationMs: 0,
|
|
1087
|
+
metadata: {
|
|
1088
|
+
candidatesTotal: results.length,
|
|
1089
|
+
vipCount,
|
|
1090
|
+
rerankYesCount,
|
|
1091
|
+
rerankNoCount,
|
|
1092
|
+
partialMergedCount,
|
|
1093
|
+
partialDroppedCount,
|
|
1094
|
+
textEmptyDropCount,
|
|
1095
|
+
rerankerErrorCount,
|
|
1096
|
+
survivorsTotal,
|
|
1097
|
+
hooksTotal,
|
|
1098
|
+
hooksSurvivedTotal,
|
|
1099
|
+
},
|
|
1100
|
+
}).catch((err) => {
|
|
1101
|
+
console.warn("[PR-E8] crag summary event failed:", err?.message ?? err);
|
|
1102
|
+
});
|
|
1103
|
+
}
|
|
1104
|
+
console.log(`[CRAG] ${results.length} -> ${filteredResults.length} results (VIP ${vipMemories.length}, review ${reviewMemories.length}, partial=${partialMemories.length})`);
|
|
1105
|
+
if (isCragCrossEncoderGateEnabled()) {
|
|
1106
|
+
const gated = await applyCragCrossEncoderGate(query, filteredResults, {
|
|
1107
|
+
cacheDir: this.rerankerCacheDir,
|
|
1108
|
+
logger: console,
|
|
1109
|
+
});
|
|
1110
|
+
console.log(`[CRAG] cross-encoder gate ${filteredResults.length} -> ${gated.length} results`);
|
|
1111
|
+
return gated;
|
|
1112
|
+
}
|
|
1113
|
+
return filteredResults;
|
|
1114
|
+
}
|
|
1115
|
+
/**
|
|
1116
|
+
* Entity Synergy Merge 實作
|
|
1117
|
+
*
|
|
1118
|
+
* 對所有 partial 記憶:
|
|
1119
|
+
* 1. 兩兩抽取實體,計算 entity overlap
|
|
1120
|
+
* 2. overlap 超過門檻的記憶對 → 合併文字
|
|
1121
|
+
* 3. 合併後用 MiniLM 重新評估 relevancy
|
|
1122
|
+
* 4. 通過的升級進結果池
|
|
1123
|
+
*/
|
|
1124
|
+
async trySynergyMerge(query, partialMemories, enableHebbian, queryVector, overlapThreshold = 0.08) {
|
|
1125
|
+
const merged = [];
|
|
1126
|
+
const usedIds = new Set();
|
|
1127
|
+
// 預先抽取所有 partial 的實體
|
|
1128
|
+
const entityMap = new Map();
|
|
1129
|
+
for (const result of partialMemories) {
|
|
1130
|
+
entityMap.set(result.entry.id, EntitySynergyMerger.extractEntities(result.entry.text));
|
|
1131
|
+
}
|
|
1132
|
+
// 貪心配對:O(n²) 但 n 通常很小(partial 記憶通常 < 10 筆)
|
|
1133
|
+
for (let i = 0; i < partialMemories.length; i++) {
|
|
1134
|
+
const memA = partialMemories[i];
|
|
1135
|
+
if (usedIds.has(memA.entry.id))
|
|
1136
|
+
continue;
|
|
1137
|
+
const entitiesA = entityMap.get(memA.entry.id);
|
|
1138
|
+
let bestPartner = null;
|
|
1139
|
+
let bestOverlap = 0;
|
|
1140
|
+
for (let j = i + 1; j < partialMemories.length; j++) {
|
|
1141
|
+
const memB = partialMemories[j];
|
|
1142
|
+
if (usedIds.has(memB.entry.id))
|
|
1143
|
+
continue;
|
|
1144
|
+
const entitiesB = entityMap.get(memB.entry.id);
|
|
1145
|
+
const overlap = EntitySynergyMerger.computeOverlap(entitiesA, entitiesB);
|
|
1146
|
+
if (overlap > bestOverlap && overlap >= overlapThreshold) {
|
|
1147
|
+
bestOverlap = overlap;
|
|
1148
|
+
bestPartner = memB;
|
|
1149
|
+
}
|
|
1150
|
+
}
|
|
1151
|
+
if (bestPartner) {
|
|
1152
|
+
usedIds.add(memA.entry.id);
|
|
1153
|
+
usedIds.add(bestPartner.entry.id);
|
|
1154
|
+
// 合併文字
|
|
1155
|
+
const mergedText = EntitySynergyMerger.mergeMemories([memA.entry.text, bestPartner.entry.text], query);
|
|
1156
|
+
// Qwen 重新評估:合併後文字屬「記憶」→ 'store' 模式 embed,與 query 算 cosine。
|
|
1157
|
+
try {
|
|
1158
|
+
const mergedVector = queryVector ? await this.embedder.embed(mergedText, "store") : null;
|
|
1159
|
+
const reEvalDist = cosineDistance(queryVector, mergedVector);
|
|
1160
|
+
const reEval = { relevance: reEvalDist <= CRAG_DIST_YES ? "yes" : "no" };
|
|
1161
|
+
if (reEval.relevance === "yes") {
|
|
1162
|
+
// 升級:用較高分數的 entry 為基底,置換 text
|
|
1163
|
+
const base = memA.finalScore >= bestPartner.finalScore ? memA : bestPartner;
|
|
1164
|
+
const boostedScore = (base.finalScore || 0) * 1.05; // 輕微加分(合併獎勵)
|
|
1165
|
+
if (enableHebbian) {
|
|
1166
|
+
this.enqueueBoostHealth(memA.entry.id);
|
|
1167
|
+
this.enqueueBoostHealth(bestPartner.entry.id);
|
|
1168
|
+
}
|
|
1169
|
+
merged.push({
|
|
1170
|
+
...base,
|
|
1171
|
+
entry: { ...base.entry, text: mergedText },
|
|
1172
|
+
fusedScore: boostedScore,
|
|
1173
|
+
finalScore: boostedScore,
|
|
1174
|
+
});
|
|
1175
|
+
console.log(`[SynergyMerge] Merged ${memA.entry.id.slice(0, 8)} + ${bestPartner.entry.id.slice(0, 8)} -> overlap=${bestOverlap.toFixed(3)}`);
|
|
1176
|
+
}
|
|
1177
|
+
else {
|
|
1178
|
+
console.log(`[SynergyMerge] Merge failed (re-eval=${reEval.relevance}): ${memA.entry.id.slice(0, 8)} + ${bestPartner.entry.id.slice(0, 8)}`);
|
|
1179
|
+
}
|
|
1180
|
+
}
|
|
1181
|
+
catch (err) {
|
|
1182
|
+
console.warn(`[SynergyMerge] Re-evaluation failed:`, err);
|
|
1183
|
+
}
|
|
1184
|
+
}
|
|
1185
|
+
else {
|
|
1186
|
+
// 沒有可合併的夥伴,但仍是 partial → 保守保留(原始 partial 會被丟掉,這裡補救)
|
|
1187
|
+
// 但 partial 已在 cragEvaluate 被 drop 了...所以乾脆在這裡補一筆不重評的
|
|
1188
|
+
// 這樣至少能進入結果池
|
|
1189
|
+
console.log(`[SynergyMerge] Partial memory has no partner: ${memA.entry.id.slice(0, 8)}`);
|
|
1190
|
+
// 🛠️ 補上這行!把孤獨的 partial 記憶救回來!
|
|
1191
|
+
merged.push(memA);
|
|
1192
|
+
}
|
|
1193
|
+
}
|
|
1194
|
+
return merged;
|
|
1195
|
+
}
|
|
1196
|
+
filterHooks(text) {
|
|
1197
|
+
const filtered = text.replace(/#[a-zA-Z0-9\u4e00-\u9fa5]+/g, "").trim();
|
|
1198
|
+
return filtered.replace(/\s+/g, " ").trim();
|
|
1199
|
+
}
|
|
1200
|
+
}
|