openclaw-cortex-memory 0.1.0-Alpha.8 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +347 -299
- package/SIGNATURE.md +7 -0
- package/SKILL.md +96 -350
- package/dist/index.d.ts +93 -23
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1234 -1318
- package/dist/index.js.map +1 -1
- package/dist/openclaw.plugin.json +377 -18
- package/dist/src/dedup/three_stage_deduplicator.d.ts.map +1 -1
- package/dist/src/dedup/three_stage_deduplicator.js +13 -3
- package/dist/src/dedup/three_stage_deduplicator.js.map +1 -1
- package/dist/src/engine/memory_engine.d.ts +6 -1
- package/dist/src/engine/memory_engine.d.ts.map +1 -1
- package/dist/src/engine/ts_engine.d.ts +208 -0
- package/dist/src/engine/ts_engine.d.ts.map +1 -1
- package/dist/src/engine/ts_engine.js +1353 -84
- package/dist/src/engine/ts_engine.js.map +1 -1
- package/dist/src/engine/types.d.ts +27 -0
- package/dist/src/engine/types.d.ts.map +1 -1
- package/dist/src/graph/ontology.d.ts +87 -15
- package/dist/src/graph/ontology.d.ts.map +1 -1
- package/dist/src/graph/ontology.js +999 -12
- package/dist/src/graph/ontology.js.map +1 -1
- package/dist/src/net/http_post.d.ts +17 -0
- package/dist/src/net/http_post.d.ts.map +1 -0
- package/dist/src/net/http_post.js +56 -0
- package/dist/src/net/http_post.js.map +1 -0
- package/dist/src/quality/llm_output_validator.d.ts +65 -0
- package/dist/src/quality/llm_output_validator.d.ts.map +1 -0
- package/dist/src/quality/llm_output_validator.js +635 -0
- package/dist/src/quality/llm_output_validator.js.map +1 -0
- package/dist/src/reflect/reflector.d.ts.map +1 -1
- package/dist/src/reflect/reflector.js +296 -26
- package/dist/src/reflect/reflector.js.map +1 -1
- package/dist/src/rules/rule_store.d.ts.map +1 -1
- package/dist/src/rules/rule_store.js +75 -16
- package/dist/src/rules/rule_store.js.map +1 -1
- package/dist/src/session/session_end.d.ts +20 -42
- package/dist/src/session/session_end.d.ts.map +1 -1
- package/dist/src/session/session_end.js +31 -214
- package/dist/src/session/session_end.js.map +1 -1
- package/dist/src/store/archive_store.d.ts +52 -7
- package/dist/src/store/archive_store.d.ts.map +1 -1
- package/dist/src/store/archive_store.js +526 -96
- package/dist/src/store/archive_store.js.map +1 -1
- package/dist/src/store/embedding_utils.d.ts +32 -0
- package/dist/src/store/embedding_utils.d.ts.map +1 -0
- package/dist/src/store/embedding_utils.js +173 -0
- package/dist/src/store/embedding_utils.js.map +1 -0
- package/dist/src/store/graph_memory_store.d.ts +115 -0
- package/dist/src/store/graph_memory_store.d.ts.map +1 -0
- package/dist/src/store/graph_memory_store.js +1061 -0
- package/dist/src/store/graph_memory_store.js.map +1 -0
- package/dist/src/store/read_store.d.ts +95 -0
- package/dist/src/store/read_store.d.ts.map +1 -1
- package/dist/src/store/read_store.js +2108 -268
- package/dist/src/store/read_store.js.map +1 -1
- package/dist/src/store/vector_store.d.ts +15 -0
- package/dist/src/store/vector_store.d.ts.map +1 -1
- package/dist/src/store/vector_store.js +75 -1
- package/dist/src/store/vector_store.js.map +1 -1
- package/dist/src/store/write_store.d.ts +46 -0
- package/dist/src/store/write_store.d.ts.map +1 -1
- package/dist/src/store/write_store.js +399 -50
- package/dist/src/store/write_store.js.map +1 -1
- package/dist/src/sync/session_sync.d.ts +115 -2
- package/dist/src/sync/session_sync.d.ts.map +1 -1
- package/dist/src/sync/session_sync.js +2497 -44
- package/dist/src/sync/session_sync.js.map +1 -1
- package/dist/src/utils/runtime_env.d.ts +4 -0
- package/dist/src/utils/runtime_env.d.ts.map +1 -0
- package/dist/src/utils/runtime_env.js +51 -0
- package/dist/src/utils/runtime_env.js.map +1 -0
- package/dist/src/wiki/wiki_linter.d.ts +26 -0
- package/dist/src/wiki/wiki_linter.d.ts.map +1 -0
- package/dist/src/wiki/wiki_linter.js +339 -0
- package/dist/src/wiki/wiki_linter.js.map +1 -0
- package/dist/src/wiki/wiki_logger.d.ts +10 -0
- package/dist/src/wiki/wiki_logger.d.ts.map +1 -0
- package/dist/src/wiki/wiki_logger.js +78 -0
- package/dist/src/wiki/wiki_logger.js.map +1 -0
- package/dist/src/wiki/wiki_maintainer.d.ts +39 -0
- package/dist/src/wiki/wiki_maintainer.d.ts.map +1 -0
- package/dist/src/wiki/wiki_maintainer.js +38 -0
- package/dist/src/wiki/wiki_maintainer.js.map +1 -0
- package/dist/src/wiki/wiki_projector.d.ts +35 -0
- package/dist/src/wiki/wiki_projector.d.ts.map +1 -0
- package/dist/src/wiki/wiki_projector.js +1151 -0
- package/dist/src/wiki/wiki_projector.js.map +1 -0
- package/dist/src/wiki/wiki_queue.d.ts +29 -0
- package/dist/src/wiki/wiki_queue.d.ts.map +1 -0
- package/dist/src/wiki/wiki_queue.js +137 -0
- package/dist/src/wiki/wiki_queue.js.map +1 -0
- package/openclaw.plugin.json +377 -18
- package/package.json +52 -5
- package/schema/graph.schema.yaml +330 -0
- package/scripts/cli.js +80 -26
- package/scripts/repair-memory.js +321 -0
- package/scripts/uninstall.js +7 -1
- package/skills/cortex-memory/SKILL.md +83 -0
- package/skills/cortex-memory/references/agent-manual.md +127 -0
- package/skills/cortex-memory/references/configuration.md +109 -0
- package/skills/cortex-memory/references/publish-checklist.md +45 -0
- package/skills/cortex-memory/references/system-prompt-template.md +27 -0
- package/skills/cortex-memory/references/tools.md +191 -0
|
@@ -37,6 +37,185 @@ exports.createReadStore = createReadStore;
|
|
|
37
37
|
const fs = __importStar(require("fs"));
|
|
38
38
|
const path = __importStar(require("path"));
|
|
39
39
|
const module_1 = require("module");
|
|
40
|
+
const http_post_1 = require("../net/http_post");
|
|
41
|
+
const EXTERNAL_MODEL_TIMEOUT_MS = 20000;
|
|
42
|
+
function graphRelationKey(relation) {
|
|
43
|
+
const source = (relation.source || "").trim().toLowerCase();
|
|
44
|
+
const type = (relation.type || "related_to").trim().toLowerCase();
|
|
45
|
+
const target = (relation.target || "").trim().toLowerCase();
|
|
46
|
+
return `${source}|${type}|${target}`;
|
|
47
|
+
}
|
|
48
|
+
function buildEntityGraphSummaryDocs(graphDocs) {
|
|
49
|
+
const entityEdges = new Map();
|
|
50
|
+
const entityLatestTs = new Map();
|
|
51
|
+
const entitySession = new Map();
|
|
52
|
+
for (const doc of graphDocs) {
|
|
53
|
+
const ts = typeof doc.timestamp === "number" ? doc.timestamp : 0;
|
|
54
|
+
const sessionId = typeof doc.sessionId === "string" ? doc.sessionId : "";
|
|
55
|
+
const relations = Array.isArray(doc.relations) ? doc.relations : [];
|
|
56
|
+
for (const relation of relations) {
|
|
57
|
+
const source = (relation.source || "").trim();
|
|
58
|
+
const target = (relation.target || "").trim();
|
|
59
|
+
const type = (relation.type || "").trim();
|
|
60
|
+
if (!source || !target || !type)
|
|
61
|
+
continue;
|
|
62
|
+
if (!entityEdges.has(source))
|
|
63
|
+
entityEdges.set(source, []);
|
|
64
|
+
if (!entityEdges.has(target))
|
|
65
|
+
entityEdges.set(target, []);
|
|
66
|
+
entityEdges.get(source)?.push({ source, target, type });
|
|
67
|
+
entityEdges.get(target)?.push({ source, target, type });
|
|
68
|
+
entityLatestTs.set(source, Math.max(entityLatestTs.get(source) || 0, ts));
|
|
69
|
+
entityLatestTs.set(target, Math.max(entityLatestTs.get(target) || 0, ts));
|
|
70
|
+
if (sessionId) {
|
|
71
|
+
if (!entitySession.has(source))
|
|
72
|
+
entitySession.set(source, sessionId);
|
|
73
|
+
if (!entitySession.has(target))
|
|
74
|
+
entitySession.set(target, sessionId);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
const output = [];
|
|
79
|
+
for (const [entity, edges] of entityEdges.entries()) {
|
|
80
|
+
if (!edges.length)
|
|
81
|
+
continue;
|
|
82
|
+
const outgoing = edges.filter(edge => edge.source === entity);
|
|
83
|
+
const incoming = edges.filter(edge => edge.target === entity);
|
|
84
|
+
const typeCounter = new Map();
|
|
85
|
+
for (const edge of edges) {
|
|
86
|
+
typeCounter.set(edge.type, (typeCounter.get(edge.type) || 0) + 1);
|
|
87
|
+
}
|
|
88
|
+
const typeSummary = [...typeCounter.entries()]
|
|
89
|
+
.sort((a, b) => b[1] - a[1])
|
|
90
|
+
.map(([type, count]) => `${type}:${count}`)
|
|
91
|
+
.join(", ");
|
|
92
|
+
const sortedOutgoing = [...outgoing].sort((a, b) => a.type.localeCompare(b.type));
|
|
93
|
+
const sortedIncoming = [...incoming].sort((a, b) => a.type.localeCompare(b.type));
|
|
94
|
+
const cappedOutgoing = sortedOutgoing.slice(0, 20);
|
|
95
|
+
const cappedIncoming = sortedIncoming.slice(0, 20);
|
|
96
|
+
const relationFacts = edges
|
|
97
|
+
.slice(0, 40)
|
|
98
|
+
.map(edge => `${edge.source} ${edge.type} ${edge.target}`)
|
|
99
|
+
.join(" | ");
|
|
100
|
+
const outgoingBlock = cappedOutgoing.length > 0
|
|
101
|
+
? cappedOutgoing.map((edge, index) => `${index + 1}. ${edge.source} -[${edge.type}]-> ${edge.target}`).join("\n")
|
|
102
|
+
: "none";
|
|
103
|
+
const incomingBlock = cappedIncoming.length > 0
|
|
104
|
+
? cappedIncoming.map((edge, index) => `${index + 1}. ${edge.source} -[${edge.type}]-> ${edge.target}`).join("\n")
|
|
105
|
+
: "none";
|
|
106
|
+
const summaryText = [
|
|
107
|
+
`# Graph Entity Summary`,
|
|
108
|
+
`entity: ${entity}`,
|
|
109
|
+
``,
|
|
110
|
+
`## Stats`,
|
|
111
|
+
`relation_total: ${edges.length}`,
|
|
112
|
+
`outgoing_total: ${outgoing.length}`,
|
|
113
|
+
`incoming_total: ${incoming.length}`,
|
|
114
|
+
typeSummary ? `relation_type_distribution: ${typeSummary}` : "relation_type_distribution: none",
|
|
115
|
+
``,
|
|
116
|
+
`## Outgoing Relations`,
|
|
117
|
+
outgoingBlock,
|
|
118
|
+
outgoing.length > cappedOutgoing.length ? `...truncated_outgoing: ${outgoing.length - cappedOutgoing.length}` : "",
|
|
119
|
+
``,
|
|
120
|
+
`## Incoming Relations`,
|
|
121
|
+
incomingBlock,
|
|
122
|
+
incoming.length > cappedIncoming.length ? `...truncated_incoming: ${incoming.length - cappedIncoming.length}` : "",
|
|
123
|
+
``,
|
|
124
|
+
`## Relation Facts`,
|
|
125
|
+
relationFacts || "none",
|
|
126
|
+
].filter(Boolean).join("\n");
|
|
127
|
+
output.push({
|
|
128
|
+
id: `gph_entity_${entity.toLowerCase().replace(/[^a-z0-9\u4e00-\u9fa5]+/gi, "_")}`,
|
|
129
|
+
text: summaryText,
|
|
130
|
+
source: "sessions_graph_entity",
|
|
131
|
+
timestamp: (entityLatestTs.get(entity) || 0) > 0 ? entityLatestTs.get(entity) : undefined,
|
|
132
|
+
layer: "archive",
|
|
133
|
+
sourceMemoryId: entity,
|
|
134
|
+
sessionId: entitySession.get(entity) || undefined,
|
|
135
|
+
entities: [entity],
|
|
136
|
+
relations: edges,
|
|
137
|
+
eventType: "graph_summary",
|
|
138
|
+
qualityScore: 1,
|
|
139
|
+
});
|
|
140
|
+
}
|
|
141
|
+
return output;
|
|
142
|
+
}
|
|
143
|
+
const DEFAULT_READ_TUNING = {
|
|
144
|
+
scoring: {
|
|
145
|
+
lexicalWeight: 0.2,
|
|
146
|
+
bm25Scale: 2,
|
|
147
|
+
semanticWeight: 0.3,
|
|
148
|
+
recencyWeight: 0.1,
|
|
149
|
+
qualityWeight: 0.15,
|
|
150
|
+
typeMatchWeight: 0.15,
|
|
151
|
+
graphMatchWeight: 0.1,
|
|
152
|
+
},
|
|
153
|
+
rrf: {
|
|
154
|
+
k: 60,
|
|
155
|
+
weight: 1.5,
|
|
156
|
+
},
|
|
157
|
+
recency: {
|
|
158
|
+
buckets: [
|
|
159
|
+
{ maxAgeHours: 12, score: 1, bonus: 0.6 },
|
|
160
|
+
{ maxAgeHours: 24, score: 0.8, bonus: 0.6 },
|
|
161
|
+
{ maxAgeHours: 72, score: 0.6, bonus: 0.3 },
|
|
162
|
+
{ maxAgeHours: 168, score: 0.4, bonus: 0.3 },
|
|
163
|
+
{ maxAgeHours: 720, score: 0.2, bonus: 0 },
|
|
164
|
+
{ maxAgeHours: Number.POSITIVE_INFINITY, score: 0.05, bonus: 0 },
|
|
165
|
+
],
|
|
166
|
+
},
|
|
167
|
+
autoContext: {
|
|
168
|
+
queryMaxChars: 240,
|
|
169
|
+
lightweightSearch: true,
|
|
170
|
+
},
|
|
171
|
+
};
|
|
172
|
+
function resolveReadTuning(options) {
|
|
173
|
+
const configuredBuckets = Array.isArray(options?.recency?.buckets)
|
|
174
|
+
? options?.recency?.buckets
|
|
175
|
+
.filter(item => item &&
|
|
176
|
+
Number.isFinite(item.maxAgeHours) &&
|
|
177
|
+
item.maxAgeHours > 0 &&
|
|
178
|
+
Number.isFinite(item.score) &&
|
|
179
|
+
item.score >= 0 &&
|
|
180
|
+
Number.isFinite(item.bonus))
|
|
181
|
+
.map(item => ({
|
|
182
|
+
maxAgeHours: item.maxAgeHours,
|
|
183
|
+
score: Math.max(0, item.score),
|
|
184
|
+
bonus: Math.max(0, item.bonus),
|
|
185
|
+
}))
|
|
186
|
+
: [];
|
|
187
|
+
const sortedBuckets = configuredBuckets
|
|
188
|
+
.sort((a, b) => a.maxAgeHours - b.maxAgeHours);
|
|
189
|
+
const buckets = sortedBuckets.length > 0 ? sortedBuckets : DEFAULT_READ_TUNING.recency.buckets;
|
|
190
|
+
const numberOr = (value, fallback, min) => {
|
|
191
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value < min) {
|
|
192
|
+
return fallback;
|
|
193
|
+
}
|
|
194
|
+
return value;
|
|
195
|
+
};
|
|
196
|
+
return {
|
|
197
|
+
scoring: {
|
|
198
|
+
lexicalWeight: numberOr(options?.scoring?.lexicalWeight, DEFAULT_READ_TUNING.scoring.lexicalWeight, 0),
|
|
199
|
+
bm25Scale: numberOr(options?.scoring?.bm25Scale, DEFAULT_READ_TUNING.scoring.bm25Scale, 0),
|
|
200
|
+
semanticWeight: numberOr(options?.scoring?.semanticWeight, DEFAULT_READ_TUNING.scoring.semanticWeight, 0),
|
|
201
|
+
recencyWeight: numberOr(options?.scoring?.recencyWeight, DEFAULT_READ_TUNING.scoring.recencyWeight, 0),
|
|
202
|
+
qualityWeight: numberOr(options?.scoring?.qualityWeight, DEFAULT_READ_TUNING.scoring.qualityWeight, 0),
|
|
203
|
+
typeMatchWeight: numberOr(options?.scoring?.typeMatchWeight, DEFAULT_READ_TUNING.scoring.typeMatchWeight, 0),
|
|
204
|
+
graphMatchWeight: numberOr(options?.scoring?.graphMatchWeight, DEFAULT_READ_TUNING.scoring.graphMatchWeight, 0),
|
|
205
|
+
},
|
|
206
|
+
rrf: {
|
|
207
|
+
k: Math.floor(numberOr(options?.rrf?.k, DEFAULT_READ_TUNING.rrf.k, 1)),
|
|
208
|
+
weight: numberOr(options?.rrf?.weight, DEFAULT_READ_TUNING.rrf.weight, 0),
|
|
209
|
+
},
|
|
210
|
+
recency: {
|
|
211
|
+
buckets,
|
|
212
|
+
},
|
|
213
|
+
autoContext: {
|
|
214
|
+
queryMaxChars: Math.floor(numberOr(options?.autoContext?.queryMaxChars, DEFAULT_READ_TUNING.autoContext.queryMaxChars, 20)),
|
|
215
|
+
lightweightSearch: options?.autoContext?.lightweightSearch !== false,
|
|
216
|
+
},
|
|
217
|
+
};
|
|
218
|
+
}
|
|
40
219
|
function safeReadFile(filePath) {
|
|
41
220
|
try {
|
|
42
221
|
if (!fs.existsSync(filePath)) {
|
|
@@ -58,15 +237,94 @@ function scoreText(query, text) {
|
|
|
58
237
|
if (t.includes(q)) {
|
|
59
238
|
score += 5;
|
|
60
239
|
}
|
|
61
|
-
const tokens =
|
|
240
|
+
const tokens = [...new Set(tokenize(q))];
|
|
62
241
|
for (const token of tokens) {
|
|
63
242
|
if (t.includes(token)) {
|
|
64
|
-
score += 1;
|
|
243
|
+
score += /[\u4e00-\u9fa5]/.test(token) && token.length <= 4 ? 0.65 : 1;
|
|
65
244
|
}
|
|
66
245
|
}
|
|
67
246
|
return score;
|
|
68
247
|
}
|
|
248
|
+
function buildCjkNgrams(token) {
|
|
249
|
+
if (!/[\u4e00-\u9fa5]/.test(token)) {
|
|
250
|
+
return [];
|
|
251
|
+
}
|
|
252
|
+
const chars = [...token];
|
|
253
|
+
const grams = [];
|
|
254
|
+
for (const size of [2, 3, 4]) {
|
|
255
|
+
if (chars.length < size)
|
|
256
|
+
continue;
|
|
257
|
+
for (let index = 0; index <= chars.length - size; index += 1) {
|
|
258
|
+
grams.push(chars.slice(index, index + size).join(""));
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
return grams;
|
|
262
|
+
}
|
|
263
|
+
function tokenize(text) {
|
|
264
|
+
const rawTokens = text
|
|
265
|
+
.toLowerCase()
|
|
266
|
+
.split(/[^a-z0-9\u4e00-\u9fa5]+/i)
|
|
267
|
+
.map(token => token.trim())
|
|
268
|
+
.filter(Boolean);
|
|
269
|
+
const tokens = [];
|
|
270
|
+
for (const token of rawTokens) {
|
|
271
|
+
tokens.push(token);
|
|
272
|
+
tokens.push(...buildCjkNgrams(token));
|
|
273
|
+
}
|
|
274
|
+
return tokens;
|
|
275
|
+
}
|
|
276
|
+
function buildBm25Stats(docs, queryTerms, getTokens) {
|
|
277
|
+
const docFreq = new Map();
|
|
278
|
+
let totalLen = 0;
|
|
279
|
+
for (const doc of docs) {
|
|
280
|
+
const tokens = typeof getTokens === "function" ? getTokens(doc) : tokenize(doc.text);
|
|
281
|
+
totalLen += tokens.length;
|
|
282
|
+
if (queryTerms.length === 0) {
|
|
283
|
+
continue;
|
|
284
|
+
}
|
|
285
|
+
const termSet = new Set(tokens);
|
|
286
|
+
const queryTermSet = new Set(queryTerms);
|
|
287
|
+
for (const term of queryTermSet) {
|
|
288
|
+
if (termSet.has(term)) {
|
|
289
|
+
docFreq.set(term, (docFreq.get(term) || 0) + 1);
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
const avgDocLen = docs.length > 0 ? Math.max(1, totalLen / docs.length) : 1;
|
|
294
|
+
return { avgDocLen, docFreq };
|
|
295
|
+
}
|
|
296
|
+
function bm25Score(args) {
|
|
297
|
+
const tokens = Array.isArray(args.docTokens) ? args.docTokens : tokenize(args.docText);
|
|
298
|
+
if (tokens.length === 0 || args.queryTerms.length === 0 || args.docCount <= 0) {
|
|
299
|
+
return 0;
|
|
300
|
+
}
|
|
301
|
+
const termFreq = new Map();
|
|
302
|
+
for (const token of tokens) {
|
|
303
|
+
termFreq.set(token, (termFreq.get(token) || 0) + 1);
|
|
304
|
+
}
|
|
305
|
+
const k1 = 1.2;
|
|
306
|
+
const b = 0.75;
|
|
307
|
+
let score = 0;
|
|
308
|
+
for (const term of uniqueStrings(args.queryTerms)) {
|
|
309
|
+
const tf = termFreq.get(term) || 0;
|
|
310
|
+
if (tf <= 0)
|
|
311
|
+
continue;
|
|
312
|
+
const df = args.docFreq.get(term) || 0;
|
|
313
|
+
const idf = Math.log(1 + ((args.docCount - df + 0.5) / (df + 0.5)));
|
|
314
|
+
const denominator = tf + k1 * (1 - b + b * (tokens.length / Math.max(1, args.avgDocLen)));
|
|
315
|
+
score += idf * (((k1 + 1) * tf) / Math.max(1e-6, denominator));
|
|
316
|
+
}
|
|
317
|
+
return score;
|
|
318
|
+
}
|
|
69
319
|
function normalizeRecordText(record) {
|
|
320
|
+
const summary = typeof record.summary === "string" ? record.summary.trim() : "";
|
|
321
|
+
const sourceText = typeof record.source_text === "string" ? record.source_text.trim() : "";
|
|
322
|
+
if (summary && sourceText) {
|
|
323
|
+
return [
|
|
324
|
+
`summary: ${summary}`,
|
|
325
|
+
`source_text: ${sourceText}`,
|
|
326
|
+
].join("\n");
|
|
327
|
+
}
|
|
70
328
|
const direct = [record.content, record.summary, record.text, record.message]
|
|
71
329
|
.find(v => typeof v === "string" && v.trim());
|
|
72
330
|
if (direct) {
|
|
@@ -109,41 +367,50 @@ function parseJsonlFile(filePath, sourceLabel, logger) {
|
|
|
109
367
|
}
|
|
110
368
|
try {
|
|
111
369
|
const parsed = JSON.parse(trimmed);
|
|
112
|
-
const
|
|
370
|
+
const summaryText = typeof parsed.summary === "string" ? parsed.summary.trim() : "";
|
|
371
|
+
const causeText = typeof parsed.cause === "string" ? parsed.cause.trim() : "";
|
|
372
|
+
const processText = typeof parsed.process === "string" ? parsed.process.trim() : "";
|
|
373
|
+
const resultText = typeof parsed.result === "string" ? parsed.result.trim() : "";
|
|
374
|
+
const sourceText = typeof parsed.source_text === "string" ? parsed.source_text.trim() : "";
|
|
375
|
+
const activeContent = typeof parsed.content === "string" ? parsed.content.trim() : "";
|
|
376
|
+
const effectiveSummary = summaryText;
|
|
377
|
+
const text = [
|
|
378
|
+
effectiveSummary,
|
|
379
|
+
causeText ? `cause: ${causeText}` : "",
|
|
380
|
+
processText ? `process: ${processText}` : "",
|
|
381
|
+
resultText ? `result: ${resultText}` : "",
|
|
382
|
+
].filter(Boolean).join("\n") || normalizeRecordText(parsed);
|
|
113
383
|
if (!text.trim()) {
|
|
114
384
|
continue;
|
|
115
385
|
}
|
|
116
386
|
const id = typeof parsed.id === "string" ? parsed.id : `${sourceLabel}:${docs.length + 1}`;
|
|
117
387
|
const timestampValue = typeof parsed.timestamp === "string" ? Date.parse(parsed.timestamp) : NaN;
|
|
118
|
-
const entities = Array.isArray(parsed.entities)
|
|
119
|
-
? parsed.entities.map(item => (typeof item === "string" ? item.trim() : "")).filter(Boolean)
|
|
120
|
-
: [];
|
|
121
|
-
const relations = Array.isArray(parsed.relations)
|
|
122
|
-
? parsed.relations
|
|
123
|
-
.map(item => {
|
|
124
|
-
if (typeof item !== "object" || item === null)
|
|
125
|
-
return null;
|
|
126
|
-
const relation = item;
|
|
127
|
-
const source = typeof relation.source === "string" ? relation.source.trim() : "";
|
|
128
|
-
const target = typeof relation.target === "string" ? relation.target.trim() : "";
|
|
129
|
-
const type = typeof relation.type === "string" ? relation.type.trim() : "related_to";
|
|
130
|
-
if (!source || !target)
|
|
131
|
-
return null;
|
|
132
|
-
return { source, target, type };
|
|
133
|
-
})
|
|
134
|
-
.filter((item) => Boolean(item))
|
|
135
|
-
: [];
|
|
136
388
|
docs.push({
|
|
137
389
|
id,
|
|
138
390
|
text,
|
|
391
|
+
summaryText: effectiveSummary || undefined,
|
|
392
|
+
sourceText: sourceText || activeContent || undefined,
|
|
393
|
+
sourceEventId: typeof parsed.source_event_id === "string" ? parsed.source_event_id : undefined,
|
|
394
|
+
sourceFile: typeof parsed.source_file === "string" ? parsed.source_file : undefined,
|
|
139
395
|
source: sourceLabel,
|
|
140
396
|
timestamp: Number.isFinite(timestampValue) ? timestampValue : undefined,
|
|
397
|
+
layer: parsed.layer === "active" || parsed.layer === "archive"
|
|
398
|
+
? parsed.layer
|
|
399
|
+
: (sourceLabel === "sessions_active" ? "active" : (sourceLabel === "sessions_archive" ? "archive" : undefined)),
|
|
400
|
+
sourceMemoryId: typeof parsed.source_memory_id === "string"
|
|
401
|
+
? parsed.source_memory_id
|
|
402
|
+
: id,
|
|
403
|
+
sourceMemoryCanonicalId: typeof parsed.source_memory_canonical_id === "string"
|
|
404
|
+
? parsed.source_memory_canonical_id
|
|
405
|
+
: (typeof parsed.canonical_id === "string" ? parsed.canonical_id : undefined),
|
|
141
406
|
embedding: Array.isArray(parsed.embedding) ? parsed.embedding.filter(item => Number.isFinite(item)) : undefined,
|
|
142
407
|
eventType: typeof parsed.event_type === "string" ? parsed.event_type.trim() : undefined,
|
|
143
408
|
qualityScore: typeof parsed.quality_score === "number" ? parsed.quality_score : undefined,
|
|
409
|
+
charCount: typeof parsed.char_count === "number" ? parsed.char_count : undefined,
|
|
410
|
+
tokenCount: typeof parsed.token_count === "number" ? parsed.token_count : undefined,
|
|
144
411
|
sessionId: typeof parsed.session_id === "string" ? parsed.session_id : undefined,
|
|
145
|
-
entities,
|
|
146
|
-
relations,
|
|
412
|
+
entities: [],
|
|
413
|
+
relations: [],
|
|
147
414
|
});
|
|
148
415
|
}
|
|
149
416
|
catch (error) {
|
|
@@ -172,35 +439,193 @@ function parseMarkdownFile(filePath, sourceLabel) {
|
|
|
172
439
|
},
|
|
173
440
|
];
|
|
174
441
|
}
|
|
175
|
-
function
|
|
442
|
+
function normalizeFactStatus(value) {
|
|
443
|
+
const token = (value || "").trim().toLowerCase();
|
|
444
|
+
if (!token)
|
|
445
|
+
return null;
|
|
446
|
+
if (token === "active")
|
|
447
|
+
return "active";
|
|
448
|
+
if (token === "pending" || token === "pending_conflict")
|
|
449
|
+
return "pending_conflict";
|
|
450
|
+
if (token === "superseded")
|
|
451
|
+
return "superseded";
|
|
452
|
+
if (token === "rejected")
|
|
453
|
+
return "rejected";
|
|
454
|
+
return null;
|
|
455
|
+
}
|
|
456
|
+
function uniqueStrings(values) {
|
|
457
|
+
const output = [];
|
|
458
|
+
const seen = new Set();
|
|
459
|
+
for (const value of values) {
|
|
460
|
+
const key = (value || "").trim();
|
|
461
|
+
if (!key || seen.has(key))
|
|
462
|
+
continue;
|
|
463
|
+
seen.add(key);
|
|
464
|
+
output.push(key);
|
|
465
|
+
}
|
|
466
|
+
return output;
|
|
467
|
+
}
|
|
468
|
+
function normalizeWikiDisplayRef(value) {
|
|
469
|
+
const normalized = (value || "").trim().replace(/\\/g, "/").replace(/^\/+/, "");
|
|
470
|
+
if (!normalized)
|
|
471
|
+
return "";
|
|
472
|
+
return normalized.startsWith("wiki/") ? normalized : `wiki/${normalized}`;
|
|
473
|
+
}
|
|
474
|
+
function normalizeWikiEvidenceRef(value) {
|
|
475
|
+
return normalizeWikiDisplayRef(value).replace(/^wiki\//, "");
|
|
476
|
+
}
|
|
477
|
+
function readWikiProjectionRefIndex(projectionIndexPath, logger) {
|
|
478
|
+
const index = {
|
|
479
|
+
entityByName: new Map(),
|
|
480
|
+
topicByType: new Map(),
|
|
481
|
+
};
|
|
482
|
+
const content = safeReadFile(projectionIndexPath).trim();
|
|
483
|
+
if (!content) {
|
|
484
|
+
return index;
|
|
485
|
+
}
|
|
486
|
+
try {
|
|
487
|
+
const parsed = JSON.parse(content);
|
|
488
|
+
const entities = Array.isArray(parsed.entities) ? parsed.entities : [];
|
|
489
|
+
const topics = Array.isArray(parsed.topics) ? parsed.topics : [];
|
|
490
|
+
for (const item of entities) {
|
|
491
|
+
if (!item || typeof item !== "object")
|
|
492
|
+
continue;
|
|
493
|
+
const row = item;
|
|
494
|
+
const name = typeof row.name === "string" ? row.name.trim() : "";
|
|
495
|
+
const wikiPath = typeof row.path === "string" ? normalizeWikiDisplayRef(row.path) : "";
|
|
496
|
+
if (!name || !wikiPath)
|
|
497
|
+
continue;
|
|
498
|
+
index.entityByName.set(name.toLowerCase(), wikiPath);
|
|
499
|
+
}
|
|
500
|
+
for (const item of topics) {
|
|
501
|
+
if (!item || typeof item !== "object")
|
|
502
|
+
continue;
|
|
503
|
+
const row = item;
|
|
504
|
+
const type = typeof row.type === "string" ? row.type.trim() : "";
|
|
505
|
+
const wikiPath = typeof row.path === "string" ? normalizeWikiDisplayRef(row.path) : "";
|
|
506
|
+
if (!type || !wikiPath)
|
|
507
|
+
continue;
|
|
508
|
+
index.topicByType.set(type.toLowerCase(), wikiPath);
|
|
509
|
+
}
|
|
510
|
+
}
|
|
511
|
+
catch (error) {
|
|
512
|
+
logger.debug(`Skipping invalid wiki projection index ${projectionIndexPath}: ${error}`);
|
|
513
|
+
}
|
|
514
|
+
return index;
|
|
515
|
+
}
|
|
516
|
+
function wikiRefsForGraphRelation(index, relation) {
|
|
517
|
+
return uniqueStrings([
|
|
518
|
+
index.entityByName.get(relation.source.trim().toLowerCase()) || "",
|
|
519
|
+
index.entityByName.get(relation.target.trim().toLowerCase()) || "",
|
|
520
|
+
index.topicByType.get(relation.type.trim().toLowerCase()) || "",
|
|
521
|
+
]);
|
|
522
|
+
}
|
|
523
|
+
function docWikiRefs(doc) {
|
|
524
|
+
return uniqueStrings([
|
|
525
|
+
...(Array.isArray(doc.wikiRefs) ? doc.wikiRefs : []),
|
|
526
|
+
doc.wikiRef || "",
|
|
527
|
+
]);
|
|
528
|
+
}
|
|
529
|
+
function buildGraphEvidenceIds(args) {
|
|
530
|
+
const relationKey = graphRelationKey({
|
|
531
|
+
source: args.source,
|
|
532
|
+
target: args.target,
|
|
533
|
+
type: args.type,
|
|
534
|
+
});
|
|
535
|
+
const wikiRefs = uniqueStrings([
|
|
536
|
+
...(Array.isArray(args.wikiRefs) ? args.wikiRefs : []),
|
|
537
|
+
args.wikiRef || "",
|
|
538
|
+
]);
|
|
539
|
+
const evidenceIds = [
|
|
540
|
+
relationKey ? `graph:relation:${relationKey}` : "",
|
|
541
|
+
args.sourceEventId ? `graph:event:${args.sourceEventId}` : "",
|
|
542
|
+
args.evidenceSpan
|
|
543
|
+
? `graph:evidence:${args.sourceEventId || relationKey}`
|
|
544
|
+
: "",
|
|
545
|
+
...wikiRefs.map(ref => {
|
|
546
|
+
const wikiRef = normalizeWikiEvidenceRef(ref);
|
|
547
|
+
return wikiRef ? `wiki:${wikiRef}${args.wikiAnchor ? `#${args.wikiAnchor}` : ""}` : "";
|
|
548
|
+
}),
|
|
549
|
+
];
|
|
550
|
+
return uniqueStrings(evidenceIds);
|
|
551
|
+
}
|
|
552
|
+
function extractPrioritizedRuleLines(text, maxRules) {
|
|
553
|
+
if (!text.trim() || maxRules <= 0) {
|
|
554
|
+
return [];
|
|
555
|
+
}
|
|
556
|
+
const lines = text
|
|
557
|
+
.split(/\r?\n/)
|
|
558
|
+
.map(line => line.trim())
|
|
559
|
+
.filter(Boolean)
|
|
560
|
+
.filter(line => !/^core rules and knowledge extracted/i.test(line))
|
|
561
|
+
.filter(line => !/^core rules\b/i.test(line));
|
|
562
|
+
if (lines.length === 0) {
|
|
563
|
+
return [];
|
|
564
|
+
}
|
|
565
|
+
const dedupedFromTail = [];
|
|
566
|
+
const seen = new Set();
|
|
567
|
+
for (let i = lines.length - 1; i >= 0; i -= 1) {
|
|
568
|
+
const line = lines[i];
|
|
569
|
+
const key = line.toLowerCase().replace(/\s+/g, " ").trim();
|
|
570
|
+
if (!key || seen.has(key)) {
|
|
571
|
+
continue;
|
|
572
|
+
}
|
|
573
|
+
seen.add(key);
|
|
574
|
+
dedupedFromTail.push(line);
|
|
575
|
+
}
|
|
576
|
+
const scored = dedupedFromTail.map((line, indexFromTail) => {
|
|
577
|
+
let score = 0;
|
|
578
|
+
if (/(must|should|ensure|avoid|prefer|always|never|fallback|verify|validate|retry|sanitize)/i.test(line)) {
|
|
579
|
+
score += 3;
|
|
580
|
+
}
|
|
581
|
+
if (/(fix|resolved|success|stable|deploy|release|incident|rollback|constraint|decision)/i.test(line)) {
|
|
582
|
+
score += 2;
|
|
583
|
+
}
|
|
584
|
+
if (/(确保|避免|优先|必须|建议|回退|重试|校验|稳定|发布|决策)/.test(line)) {
|
|
585
|
+
score += 2;
|
|
586
|
+
}
|
|
587
|
+
if (line.length >= 30 && line.length <= 220) {
|
|
588
|
+
score += 2;
|
|
589
|
+
}
|
|
590
|
+
else if (line.length > 220) {
|
|
591
|
+
score -= 1;
|
|
592
|
+
}
|
|
593
|
+
if (/[.!?]$/.test(line)) {
|
|
594
|
+
score += 1;
|
|
595
|
+
}
|
|
596
|
+
score += Math.max(0, 2 - indexFromTail * 0.08);
|
|
597
|
+
return { line, score, indexFromTail };
|
|
598
|
+
});
|
|
599
|
+
const selected = scored
|
|
600
|
+
.sort((a, b) => (b.score - a.score) || (a.indexFromTail - b.indexFromTail))
|
|
601
|
+
.slice(0, maxRules)
|
|
602
|
+
.sort((a, b) => a.indexFromTail - b.indexFromTail)
|
|
603
|
+
.map(item => item.line);
|
|
604
|
+
return selected;
|
|
605
|
+
}
|
|
606
|
+
function withRecencyBoost(score, timestamp, buckets) {
|
|
176
607
|
if (!timestamp) {
|
|
177
608
|
return score;
|
|
178
609
|
}
|
|
179
610
|
const ageHours = (Date.now() - timestamp) / (1000 * 60 * 60);
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
return score + 0.3;
|
|
611
|
+
for (const bucket of buckets) {
|
|
612
|
+
if (ageHours <= bucket.maxAgeHours) {
|
|
613
|
+
return score + bucket.bonus;
|
|
614
|
+
}
|
|
185
615
|
}
|
|
186
616
|
return score;
|
|
187
617
|
}
|
|
188
|
-
function recencyScore(timestamp) {
|
|
618
|
+
function recencyScore(timestamp, buckets) {
|
|
189
619
|
if (!timestamp) {
|
|
190
620
|
return 0;
|
|
191
621
|
}
|
|
192
622
|
const ageHours = (Date.now() - timestamp) / (1000 * 60 * 60);
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
if (ageHours < 168)
|
|
200
|
-
return 0.4;
|
|
201
|
-
if (ageHours < 720)
|
|
202
|
-
return 0.2;
|
|
203
|
-
return 0.05;
|
|
623
|
+
for (const bucket of buckets) {
|
|
624
|
+
if (ageHours <= bucket.maxAgeHours) {
|
|
625
|
+
return bucket.score;
|
|
626
|
+
}
|
|
627
|
+
}
|
|
628
|
+
return 0;
|
|
204
629
|
}
|
|
205
630
|
function eventTypeHalfLifeDays(eventType, options) {
|
|
206
631
|
const fallback = typeof options?.defaultHalfLifeDays === "number" && options.defaultHalfLifeDays > 0
|
|
@@ -262,25 +687,46 @@ function normalizeBaseUrl(value) {
|
|
|
262
687
|
return "";
|
|
263
688
|
return value.endsWith("/") ? value.slice(0, -1) : value;
|
|
264
689
|
}
|
|
690
|
+
function emptyChannelResults() {
|
|
691
|
+
return {
|
|
692
|
+
vector_semantic: [],
|
|
693
|
+
vector_keyword: [],
|
|
694
|
+
keyword: [],
|
|
695
|
+
semantic: [],
|
|
696
|
+
archive: [],
|
|
697
|
+
graph: [],
|
|
698
|
+
rules: [],
|
|
699
|
+
vector: [],
|
|
700
|
+
};
|
|
701
|
+
}
|
|
702
|
+
const READ_FUSION_PROMPT_VERSION = "read-fusion.v1.2.0";
|
|
703
|
+
const READ_FUSION_REGRESSION_SAMPLES = [
|
|
704
|
+
"Example A: if archive and vector refer to the same source_memory_id, keep one main conclusion and keep the rest as supporting evidence.",
|
|
705
|
+
"Example B: if conclusions conflict, write conflicts and explain prioritization in canonical_answer (time, quality, explicitness).",
|
|
706
|
+
"Example C: if summary/excerpt is insufficient, return event ids in need_fulltext_event_ids for full-text lookup.",
|
|
707
|
+
];
|
|
708
|
+
function vectorNorm(values) {
|
|
709
|
+
let sum = 0;
|
|
710
|
+
for (const value of values) {
|
|
711
|
+
sum += value * value;
|
|
712
|
+
}
|
|
713
|
+
return Math.sqrt(sum);
|
|
714
|
+
}
|
|
265
715
|
function cosineSimilarity(left, right) {
|
|
266
|
-
|
|
716
|
+
return cosineSimilarityWithNorm(left, right, vectorNorm(left), vectorNorm(right));
|
|
717
|
+
}
|
|
718
|
+
function cosineSimilarityWithNorm(left, right, leftNorm, rightNorm) {
|
|
719
|
+
if (left.length === 0 || right.length === 0 || leftNorm === 0 || rightNorm === 0) {
|
|
267
720
|
return 0;
|
|
268
721
|
}
|
|
269
722
|
const size = Math.min(left.length, right.length);
|
|
270
723
|
let dot = 0;
|
|
271
|
-
let leftNorm = 0;
|
|
272
|
-
let rightNorm = 0;
|
|
273
724
|
for (let i = 0; i < size; i += 1) {
|
|
274
725
|
const a = left[i];
|
|
275
726
|
const b = right[i];
|
|
276
727
|
dot += a * b;
|
|
277
|
-
leftNorm += a * a;
|
|
278
|
-
rightNorm += b * b;
|
|
279
|
-
}
|
|
280
|
-
if (leftNorm === 0 || rightNorm === 0) {
|
|
281
|
-
return 0;
|
|
282
728
|
}
|
|
283
|
-
return dot / (
|
|
729
|
+
return dot / (leftNorm * rightNorm);
|
|
284
730
|
}
|
|
285
731
|
async function requestEmbedding(args) {
|
|
286
732
|
const endpoint = args.baseUrl.endsWith("/embeddings") ? args.baseUrl : `${args.baseUrl}/embeddings`;
|
|
@@ -293,24 +739,18 @@ async function requestEmbedding(args) {
|
|
|
293
739
|
}
|
|
294
740
|
let lastError = null;
|
|
295
741
|
for (let attempt = 0; attempt < 3; attempt += 1) {
|
|
296
|
-
const
|
|
297
|
-
|
|
742
|
+
const response = await (0, http_post_1.postJsonWithTimeout)({
|
|
743
|
+
endpoint,
|
|
744
|
+
apiKey: args.apiKey,
|
|
745
|
+
body,
|
|
746
|
+
timeoutMs: EXTERNAL_MODEL_TIMEOUT_MS,
|
|
747
|
+
});
|
|
748
|
+
if (!response.ok) {
|
|
749
|
+
lastError = new Error(response.status > 0 ? `embedding_http_${response.status}` : (response.error || "embedding_network_error"));
|
|
750
|
+
continue;
|
|
751
|
+
}
|
|
298
752
|
try {
|
|
299
|
-
const
|
|
300
|
-
method: "POST",
|
|
301
|
-
headers: {
|
|
302
|
-
"content-type": "application/json",
|
|
303
|
-
authorization: `Bearer ${args.apiKey}`,
|
|
304
|
-
},
|
|
305
|
-
body: JSON.stringify(body),
|
|
306
|
-
signal: controller.signal,
|
|
307
|
-
});
|
|
308
|
-
clearTimeout(timeoutId);
|
|
309
|
-
if (!response.ok) {
|
|
310
|
-
lastError = new Error(`embedding_http_${response.status}`);
|
|
311
|
-
continue;
|
|
312
|
-
}
|
|
313
|
-
const json = await response.json();
|
|
753
|
+
const json = (response.json || {});
|
|
314
754
|
const embedding = json?.data?.[0]?.embedding;
|
|
315
755
|
if (Array.isArray(embedding) && embedding.length > 0) {
|
|
316
756
|
return embedding.filter(item => Number.isFinite(item));
|
|
@@ -318,7 +758,6 @@ async function requestEmbedding(args) {
|
|
|
318
758
|
lastError = new Error("embedding_empty");
|
|
319
759
|
}
|
|
320
760
|
catch (error) {
|
|
321
|
-
clearTimeout(timeoutId);
|
|
322
761
|
lastError = error;
|
|
323
762
|
}
|
|
324
763
|
}
|
|
@@ -338,24 +777,18 @@ async function requestRerank(args) {
|
|
|
338
777
|
};
|
|
339
778
|
let lastError = null;
|
|
340
779
|
for (let attempt = 0; attempt < 2; attempt += 1) {
|
|
341
|
-
const
|
|
342
|
-
|
|
780
|
+
const response = await (0, http_post_1.postJsonWithTimeout)({
|
|
781
|
+
endpoint,
|
|
782
|
+
apiKey: args.apiKey,
|
|
783
|
+
body,
|
|
784
|
+
timeoutMs: EXTERNAL_MODEL_TIMEOUT_MS,
|
|
785
|
+
});
|
|
786
|
+
if (!response.ok) {
|
|
787
|
+
lastError = new Error(response.status > 0 ? `rerank_http_${response.status}` : (response.error || "rerank_network_error"));
|
|
788
|
+
continue;
|
|
789
|
+
}
|
|
343
790
|
try {
|
|
344
|
-
const
|
|
345
|
-
method: "POST",
|
|
346
|
-
headers: {
|
|
347
|
-
"content-type": "application/json",
|
|
348
|
-
authorization: `Bearer ${args.apiKey}`,
|
|
349
|
-
},
|
|
350
|
-
body: JSON.stringify(body),
|
|
351
|
-
signal: controller.signal,
|
|
352
|
-
});
|
|
353
|
-
clearTimeout(timeoutId);
|
|
354
|
-
if (!response.ok) {
|
|
355
|
-
lastError = new Error(`rerank_http_${response.status}`);
|
|
356
|
-
continue;
|
|
357
|
-
}
|
|
358
|
-
const json = await response.json();
|
|
791
|
+
const json = (response.json || {});
|
|
359
792
|
const list = Array.isArray(json.results) ? json.results : (Array.isArray(json.data) ? json.data : []);
|
|
360
793
|
if (!Array.isArray(list) || list.length === 0) {
|
|
361
794
|
lastError = new Error("rerank_empty");
|
|
@@ -377,7 +810,6 @@ async function requestRerank(args) {
|
|
|
377
810
|
lastError = new Error("rerank_map_empty");
|
|
378
811
|
}
|
|
379
812
|
catch (error) {
|
|
380
|
-
clearTimeout(timeoutId);
|
|
381
813
|
lastError = error;
|
|
382
814
|
}
|
|
383
815
|
}
|
|
@@ -385,19 +817,19 @@ async function requestRerank(args) {
|
|
|
385
817
|
}
|
|
386
818
|
function classifyIntent(query) {
|
|
387
819
|
const text = query.toLowerCase();
|
|
388
|
-
const relationHints = /(
|
|
820
|
+
const relationHints = /(关系|依赖|关联|上下游|图谱|拓扑|graph|relation|entity|dependency)/i;
|
|
389
821
|
if (relationHints.test(text))
|
|
390
822
|
return "RELATION_DISCOVERY";
|
|
391
|
-
const troubleHints = /(
|
|
823
|
+
const troubleHints = /(报错|错误|异常|失败|超时|无法|崩溃|故障|修复|bug|error|failed|timeout|fix)/i;
|
|
392
824
|
if (troubleHints.test(text))
|
|
393
825
|
return "TROUBLESHOOTING";
|
|
394
826
|
const preferenceHints = /(偏好|习惯|口味|喜欢|不喜欢|偏向|preference)/i;
|
|
395
827
|
if (preferenceHints.test(text))
|
|
396
828
|
return "PREFERENCE_PROFILE";
|
|
397
|
-
const timelineHints = /(
|
|
829
|
+
const timelineHints = /(最近|上次|之前|时间线|历史|timeline|history)/i;
|
|
398
830
|
if (timelineHints.test(text))
|
|
399
831
|
return "TIMELINE_REVIEW";
|
|
400
|
-
const decisionHints = /(
|
|
832
|
+
const decisionHints = /(方案|决策|选择|建议|取舍|权衡|tradeoff|plan)/i;
|
|
401
833
|
if (decisionHints.test(text))
|
|
402
834
|
return "DECISION_SUPPORT";
|
|
403
835
|
return "FACT_LOOKUP";
|
|
@@ -425,6 +857,268 @@ function sourceWeight(source, intent) {
|
|
|
425
857
|
}
|
|
426
858
|
return 1;
|
|
427
859
|
}
|
|
860
|
+
const QUERY_PLAN_STOPWORDS = new Set([
|
|
861
|
+
"what",
|
|
862
|
+
"is",
|
|
863
|
+
"the",
|
|
864
|
+
"a",
|
|
865
|
+
"an",
|
|
866
|
+
"of",
|
|
867
|
+
"for",
|
|
868
|
+
"to",
|
|
869
|
+
"in",
|
|
870
|
+
"on",
|
|
871
|
+
"and",
|
|
872
|
+
"or",
|
|
873
|
+
"with",
|
|
874
|
+
"about",
|
|
875
|
+
"this",
|
|
876
|
+
"that",
|
|
877
|
+
"这些",
|
|
878
|
+
"那些",
|
|
879
|
+
"这个",
|
|
880
|
+
"那个",
|
|
881
|
+
"之前",
|
|
882
|
+
"提到",
|
|
883
|
+
"提过",
|
|
884
|
+
"一下",
|
|
885
|
+
"请问",
|
|
886
|
+
"关于",
|
|
887
|
+
"相关",
|
|
888
|
+
"什么",
|
|
889
|
+
"怎么",
|
|
890
|
+
"如何",
|
|
891
|
+
"是否",
|
|
892
|
+
"有没有",
|
|
893
|
+
"是什么",
|
|
894
|
+
"哪些",
|
|
895
|
+
"哪个",
|
|
896
|
+
]);
|
|
897
|
+
function planQueryKeywords(query) {
|
|
898
|
+
const normalized = (query || "").trim().replace(/\s+/g, " ");
|
|
899
|
+
if (!normalized)
|
|
900
|
+
return [];
|
|
901
|
+
const output = [];
|
|
902
|
+
const seen = new Set();
|
|
903
|
+
const push = (value) => {
|
|
904
|
+
const item = (value || "").trim();
|
|
905
|
+
if (!item)
|
|
906
|
+
return;
|
|
907
|
+
const key = item.toLowerCase();
|
|
908
|
+
if (seen.has(key))
|
|
909
|
+
return;
|
|
910
|
+
seen.add(key);
|
|
911
|
+
output.push(item);
|
|
912
|
+
};
|
|
913
|
+
push(normalized);
|
|
914
|
+
for (const match of normalized.matchAll(/["“”'‘’]([^"“”'‘’]{2,64})["“”'‘’]/g)) {
|
|
915
|
+
push(match[1]);
|
|
916
|
+
}
|
|
917
|
+
for (const match of normalized.match(/https?:\/\/[^\s]+/gi) || []) {
|
|
918
|
+
push(match);
|
|
919
|
+
}
|
|
920
|
+
for (const match of normalized.matchAll(/\b[A-Za-z0-9][A-Za-z0-9._/-]*(?:\s+[A-Za-z0-9][A-Za-z0-9._/-]*){0,3}\b/g)) {
|
|
921
|
+
const phrase = match[0].trim();
|
|
922
|
+
const words = phrase.split(/\s+/).filter(Boolean);
|
|
923
|
+
const hasStrongSignal = words.length >= 2 || /[A-Z]{2,}/.test(phrase);
|
|
924
|
+
if (hasStrongSignal && !QUERY_PLAN_STOPWORDS.has(phrase.toLowerCase())) {
|
|
925
|
+
push(phrase);
|
|
926
|
+
}
|
|
927
|
+
}
|
|
928
|
+
const normalizedForSplit = normalized
|
|
929
|
+
.replace(/[(){}\[\]<>]/g, " ")
|
|
930
|
+
.replace(/[,。!?;:,.!?;:、]/g, " ")
|
|
931
|
+
.replace(/(之前提到的|之前提到|提到的|提到|请问|一下|是什么|什么|哪个|哪篇|哪里|如何|怎么|有关|关于|以及|还有|并且|然后|能否|可以|帮我|给我)/g, " ");
|
|
932
|
+
for (const raw of normalizedForSplit.split(/[\/\-\s]+/)) {
|
|
933
|
+
const token = raw.trim();
|
|
934
|
+
if (!token || token.length < 2)
|
|
935
|
+
continue;
|
|
936
|
+
if (QUERY_PLAN_STOPWORDS.has(token.toLowerCase()))
|
|
937
|
+
continue;
|
|
938
|
+
push(token);
|
|
939
|
+
}
|
|
940
|
+
return output.slice(0, 5);
|
|
941
|
+
}
|
|
942
|
+
function shouldTriggerFulltextFallback(ranked, topK) {
|
|
943
|
+
const scoped = ranked.slice(0, Math.max(1, topK));
|
|
944
|
+
if (scoped.length === 0)
|
|
945
|
+
return true;
|
|
946
|
+
const isStrongSummaryHit = (item) => {
|
|
947
|
+
if (!Array.isArray(item.reason_tags) || !item.reason_tags.includes("summary_hit")) {
|
|
948
|
+
return false;
|
|
949
|
+
}
|
|
950
|
+
const score = Number(item.score || 0);
|
|
951
|
+
const summaryScore = Number(item.score_breakdown?.summary || 0);
|
|
952
|
+
const bm25ScoreValue = Number(item.score_breakdown?.bm25 || 0);
|
|
953
|
+
return score >= 1.2 && (summaryScore >= 1 || bm25ScoreValue >= 0.15);
|
|
954
|
+
};
|
|
955
|
+
const summaryHits = scoped.filter(isStrongSummaryHit).length;
|
|
956
|
+
const sessionScoped = scoped.filter(item => item.source === "sessions_active" || item.source === "sessions_archive");
|
|
957
|
+
if (sessionScoped.length === 0) {
|
|
958
|
+
return summaryHits === 0;
|
|
959
|
+
}
|
|
960
|
+
const sessionSummaryHits = sessionScoped.filter(isStrongSummaryHit).length;
|
|
961
|
+
return summaryHits === 0 || sessionSummaryHits === 0;
|
|
962
|
+
}
|
|
963
|
+
function normalizeFusionMode(value) {
|
|
964
|
+
if (value === "authoritative" || value === "candidates" || value === "off") {
|
|
965
|
+
return value;
|
|
966
|
+
}
|
|
967
|
+
return "auto";
|
|
968
|
+
}
|
|
969
|
+
function roundTiming(timing) {
|
|
970
|
+
const output = {};
|
|
971
|
+
for (const [key, value] of Object.entries(timing)) {
|
|
972
|
+
if (typeof value !== "number" || !Number.isFinite(value))
|
|
973
|
+
continue;
|
|
974
|
+
output[key] = Number(value.toFixed(2));
|
|
975
|
+
}
|
|
976
|
+
return output;
|
|
977
|
+
}
|
|
978
|
+
function appendTiming(timing, key, startedAt) {
|
|
979
|
+
const elapsed = Date.now() - startedAt;
|
|
980
|
+
timing[key] = (timing[key] || 0) + elapsed;
|
|
981
|
+
}
|
|
982
|
+
function mergeKeyFromDoc(doc) {
|
|
983
|
+
const canonical = typeof doc.sourceMemoryCanonicalId === "string" ? doc.sourceMemoryCanonicalId.trim() : "";
|
|
984
|
+
if (canonical) {
|
|
985
|
+
return `canonical:${canonical}`;
|
|
986
|
+
}
|
|
987
|
+
const sourceMemoryId = typeof doc.sourceMemoryId === "string" ? doc.sourceMemoryId.trim() : "";
|
|
988
|
+
if (sourceMemoryId) {
|
|
989
|
+
return `source:${sourceMemoryId}`;
|
|
990
|
+
}
|
|
991
|
+
return `id:${doc.id}`;
|
|
992
|
+
}
|
|
993
|
+
function docFactStatus(doc) {
|
|
994
|
+
const direct = normalizeFactStatus(typeof doc.factStatus === "string" ? doc.factStatus : "");
|
|
995
|
+
if (direct)
|
|
996
|
+
return direct;
|
|
997
|
+
const relation = Array.isArray(doc.relations) && doc.relations.length > 0 ? doc.relations[0] : null;
|
|
998
|
+
const relationStatus = normalizeFactStatus(typeof relation?.fact_status === "string" ? relation.fact_status : "");
|
|
999
|
+
if (relationStatus)
|
|
1000
|
+
return relationStatus;
|
|
1001
|
+
return "active";
|
|
1002
|
+
}
|
|
1003
|
+
function docEvidenceIds(doc) {
|
|
1004
|
+
if (Array.isArray(doc.evidenceIds) && doc.evidenceIds.length > 0) {
|
|
1005
|
+
return uniqueStrings(doc.evidenceIds);
|
|
1006
|
+
}
|
|
1007
|
+
const wikiRefs = docWikiRefs(doc);
|
|
1008
|
+
const relation = Array.isArray(doc.relations) && doc.relations.length > 0 ? doc.relations[0] : null;
|
|
1009
|
+
const source = relation?.source || "";
|
|
1010
|
+
const target = relation?.target || "";
|
|
1011
|
+
const type = relation?.type || "";
|
|
1012
|
+
const evidence = source && target && type
|
|
1013
|
+
? buildGraphEvidenceIds({
|
|
1014
|
+
source,
|
|
1015
|
+
target,
|
|
1016
|
+
type,
|
|
1017
|
+
sourceEventId: relation?.source_event_id || doc.sourceEventId,
|
|
1018
|
+
evidenceSpan: relation?.evidence_span,
|
|
1019
|
+
wikiRefs,
|
|
1020
|
+
wikiAnchor: doc.wikiAnchor,
|
|
1021
|
+
})
|
|
1022
|
+
: [];
|
|
1023
|
+
return uniqueStrings([
|
|
1024
|
+
...evidence,
|
|
1025
|
+
doc.sourceEventId ? `graph:event:${doc.sourceEventId}` : "",
|
|
1026
|
+
...wikiRefs.map(ref => {
|
|
1027
|
+
const wikiRef = normalizeWikiEvidenceRef(ref);
|
|
1028
|
+
return wikiRef ? `wiki:${wikiRef}${doc.wikiAnchor ? `#${doc.wikiAnchor}` : ""}` : "";
|
|
1029
|
+
}),
|
|
1030
|
+
`doc:${doc.id}`,
|
|
1031
|
+
]);
|
|
1032
|
+
}
|
|
1033
|
+
function customChannelWeight(source, options) {
|
|
1034
|
+
const weights = options?.channelWeights;
|
|
1035
|
+
if (!weights)
|
|
1036
|
+
return 1;
|
|
1037
|
+
const value = weights[source];
|
|
1038
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) {
|
|
1039
|
+
return 1;
|
|
1040
|
+
}
|
|
1041
|
+
return value;
|
|
1042
|
+
}
|
|
1043
|
+
function lengthNormalizeFactor(doc, options) {
|
|
1044
|
+
const lengthNorm = options?.lengthNorm;
|
|
1045
|
+
if (lengthNorm?.enabled === false) {
|
|
1046
|
+
return 1;
|
|
1047
|
+
}
|
|
1048
|
+
const pivotChars = typeof lengthNorm?.pivotChars === "number" && lengthNorm.pivotChars > 0
|
|
1049
|
+
? lengthNorm.pivotChars
|
|
1050
|
+
: 1200;
|
|
1051
|
+
const strength = typeof lengthNorm?.strength === "number" && lengthNorm.strength > 0
|
|
1052
|
+
? lengthNorm.strength
|
|
1053
|
+
: 0.75;
|
|
1054
|
+
const minFactor = typeof lengthNorm?.minFactor === "number" && lengthNorm.minFactor > 0 && lengthNorm.minFactor <= 1
|
|
1055
|
+
? lengthNorm.minFactor
|
|
1056
|
+
: 0.45;
|
|
1057
|
+
const charCount = typeof doc.charCount === "number" && Number.isFinite(doc.charCount)
|
|
1058
|
+
? doc.charCount
|
|
1059
|
+
: doc.text.length;
|
|
1060
|
+
if (charCount <= pivotChars) {
|
|
1061
|
+
return 1;
|
|
1062
|
+
}
|
|
1063
|
+
const over = (charCount - pivotChars) / pivotChars;
|
|
1064
|
+
const factor = 1 / (1 + over * strength);
|
|
1065
|
+
return Math.max(minFactor, Math.min(1, factor));
|
|
1066
|
+
}
|
|
1067
|
+
function channelQuota(source, topK, options) {
|
|
1068
|
+
const configured = options?.channelTopK?.[source];
|
|
1069
|
+
if (typeof configured === "number" && Number.isFinite(configured) && configured >= 1) {
|
|
1070
|
+
return Math.floor(configured);
|
|
1071
|
+
}
|
|
1072
|
+
if (source === "rules")
|
|
1073
|
+
return Math.max(6, topK * 2);
|
|
1074
|
+
if (source === "graph")
|
|
1075
|
+
return Math.max(8, topK * 3);
|
|
1076
|
+
return Math.max(12, topK * 4);
|
|
1077
|
+
}
|
|
1078
|
+
function parseJsonStringArray(value) {
|
|
1079
|
+
if (typeof value !== "string" || !value.trim()) {
|
|
1080
|
+
return [];
|
|
1081
|
+
}
|
|
1082
|
+
try {
|
|
1083
|
+
const parsed = JSON.parse(value);
|
|
1084
|
+
if (!Array.isArray(parsed)) {
|
|
1085
|
+
return [];
|
|
1086
|
+
}
|
|
1087
|
+
return parsed
|
|
1088
|
+
.map(item => (typeof item === "string" ? item.trim() : ""))
|
|
1089
|
+
.filter(Boolean);
|
|
1090
|
+
}
|
|
1091
|
+
catch {
|
|
1092
|
+
return [];
|
|
1093
|
+
}
|
|
1094
|
+
}
|
|
1095
|
+
function parseJsonRelations(value) {
|
|
1096
|
+
if (typeof value !== "string" || !value.trim()) {
|
|
1097
|
+
return [];
|
|
1098
|
+
}
|
|
1099
|
+
try {
|
|
1100
|
+
const parsed = JSON.parse(value);
|
|
1101
|
+
if (!Array.isArray(parsed)) {
|
|
1102
|
+
return [];
|
|
1103
|
+
}
|
|
1104
|
+
return parsed
|
|
1105
|
+
.map(item => {
|
|
1106
|
+
if (typeof item !== "object" || item === null)
|
|
1107
|
+
return null;
|
|
1108
|
+
const relation = item;
|
|
1109
|
+
const source = typeof relation.source === "string" ? relation.source.trim() : "";
|
|
1110
|
+
const target = typeof relation.target === "string" ? relation.target.trim() : "";
|
|
1111
|
+
const type = typeof relation.type === "string" && relation.type.trim() ? relation.type.trim() : "related_to";
|
|
1112
|
+
if (!source || !target)
|
|
1113
|
+
return null;
|
|
1114
|
+
return { source, target, type };
|
|
1115
|
+
})
|
|
1116
|
+
.filter((item) => Boolean(item));
|
|
1117
|
+
}
|
|
1118
|
+
catch {
|
|
1119
|
+
return [];
|
|
1120
|
+
}
|
|
1121
|
+
}
|
|
428
1122
|
async function searchLanceDb(args) {
|
|
429
1123
|
try {
|
|
430
1124
|
const require = (0, module_1.createRequire)(__filename);
|
|
@@ -464,20 +1158,27 @@ async function searchLanceDb(args) {
|
|
|
464
1158
|
if (!id || !summary)
|
|
465
1159
|
continue;
|
|
466
1160
|
const ts = typeof record.timestamp === "string" ? Date.parse(record.timestamp) : NaN;
|
|
467
|
-
const entities =
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
const relations = typeof record.relations_json === "string"
|
|
471
|
-
? JSON.parse(record.relations_json)
|
|
472
|
-
: [];
|
|
1161
|
+
const entities = parseJsonStringArray(record.entities_json);
|
|
1162
|
+
const relations = parseJsonRelations(record.relations_json);
|
|
1163
|
+
const embedding = Array.isArray(record.vector) ? record.vector.filter(item => Number.isFinite(item)) : undefined;
|
|
473
1164
|
docs.push({
|
|
474
1165
|
id,
|
|
475
1166
|
text: summary,
|
|
476
1167
|
source: "vector_lancedb",
|
|
477
1168
|
timestamp: Number.isFinite(ts) ? ts : undefined,
|
|
478
|
-
|
|
1169
|
+
layer: record.layer === "active" || record.layer === "archive" ? record.layer : undefined,
|
|
1170
|
+
sourceMemoryId: typeof record.source_memory_id === "string" ? record.source_memory_id : undefined,
|
|
1171
|
+
sourceMemoryCanonicalId: typeof record.source_memory_canonical_id === "string" ? record.source_memory_canonical_id : undefined,
|
|
1172
|
+
sourceEventId: typeof record.source_event_id === "string" ? record.source_event_id : undefined,
|
|
1173
|
+
sourceField: record.source_field === "summary" || record.source_field === "evidence"
|
|
1174
|
+
? record.source_field
|
|
1175
|
+
: undefined,
|
|
1176
|
+
embedding,
|
|
1177
|
+
embeddingNorm: embedding && embedding.length > 0 ? vectorNorm(embedding) : undefined,
|
|
479
1178
|
eventType: typeof record.event_type === "string" ? record.event_type : undefined,
|
|
480
1179
|
qualityScore: typeof record.quality_score === "number" ? record.quality_score : undefined,
|
|
1180
|
+
charCount: typeof record.char_count === "number" ? record.char_count : undefined,
|
|
1181
|
+
tokenCount: typeof record.token_count === "number" ? record.token_count : undefined,
|
|
481
1182
|
sessionId: typeof record.session_id === "string" ? record.session_id : undefined,
|
|
482
1183
|
entities,
|
|
483
1184
|
relations: Array.isArray(relations) ? relations : [],
|
|
@@ -525,14 +1226,25 @@ function parseVectorFallback(filePath, logger) {
|
|
|
525
1226
|
})
|
|
526
1227
|
.filter((item) => Boolean(item))
|
|
527
1228
|
: [];
|
|
1229
|
+
const embedding = Array.isArray(parsed.embedding) ? parsed.embedding.filter(item => Number.isFinite(item)) : undefined;
|
|
528
1230
|
docs.push({
|
|
529
1231
|
id,
|
|
530
1232
|
text: summary,
|
|
531
1233
|
source: "vector_jsonl",
|
|
532
1234
|
timestamp: Number.isFinite(ts) ? ts : undefined,
|
|
533
|
-
|
|
1235
|
+
layer: parsed.layer === "active" || parsed.layer === "archive" ? parsed.layer : undefined,
|
|
1236
|
+
sourceMemoryId: typeof parsed.source_memory_id === "string" ? parsed.source_memory_id : undefined,
|
|
1237
|
+
sourceMemoryCanonicalId: typeof parsed.source_memory_canonical_id === "string" ? parsed.source_memory_canonical_id : undefined,
|
|
1238
|
+
sourceEventId: typeof parsed.source_event_id === "string" ? parsed.source_event_id : undefined,
|
|
1239
|
+
sourceField: parsed.source_field === "summary" || parsed.source_field === "evidence"
|
|
1240
|
+
? parsed.source_field
|
|
1241
|
+
: undefined,
|
|
1242
|
+
embedding,
|
|
1243
|
+
embeddingNorm: embedding && embedding.length > 0 ? vectorNorm(embedding) : undefined,
|
|
534
1244
|
eventType: typeof parsed.event_type === "string" ? parsed.event_type.trim() : undefined,
|
|
535
1245
|
qualityScore: typeof parsed.quality_score === "number" ? parsed.quality_score : undefined,
|
|
1246
|
+
charCount: typeof parsed.char_count === "number" ? parsed.char_count : undefined,
|
|
1247
|
+
tokenCount: typeof parsed.token_count === "number" ? parsed.token_count : undefined,
|
|
536
1248
|
sessionId: typeof parsed.session_id === "string" ? parsed.session_id : undefined,
|
|
537
1249
|
entities,
|
|
538
1250
|
relations,
|
|
@@ -545,53 +1257,91 @@ function parseVectorFallback(filePath, logger) {
|
|
|
545
1257
|
return docs;
|
|
546
1258
|
}
|
|
547
1259
|
async function requestFusion(args) {
|
|
1260
|
+
const candidateIdSet = new Set(args.candidates.map(item => item.id));
|
|
548
1261
|
const endpoint = args.llm.baseUrl.endsWith("/chat/completions")
|
|
549
1262
|
? args.llm.baseUrl
|
|
550
1263
|
: `${args.llm.baseUrl}/chat/completions`;
|
|
551
1264
|
const evidenceText = args.candidates
|
|
552
|
-
.map((item, index) =>
|
|
1265
|
+
.map((item, index) => {
|
|
1266
|
+
const excerpt = (item.source_excerpt || "").trim();
|
|
1267
|
+
const sourceFile = (item.source_file || "").trim();
|
|
1268
|
+
const sourceMemoryId = (item.source_memory_id || "").trim();
|
|
1269
|
+
const sourceMemoryCanonicalId = (item.source_memory_canonical_id || "").trim();
|
|
1270
|
+
const sourceLayer = (item.source_layer || "").trim();
|
|
1271
|
+
const sourceEventId = (item.source_event_id || "").trim();
|
|
1272
|
+
const sourceField = (item.source_field || "").trim();
|
|
1273
|
+
const extraParts = [];
|
|
1274
|
+
if (sourceMemoryId)
|
|
1275
|
+
extraParts.push(`source_memory_id=${sourceMemoryId}`);
|
|
1276
|
+
if (sourceMemoryCanonicalId)
|
|
1277
|
+
extraParts.push(`source_memory_canonical_id=${sourceMemoryCanonicalId}`);
|
|
1278
|
+
if (sourceLayer)
|
|
1279
|
+
extraParts.push(`source_layer=${sourceLayer}`);
|
|
1280
|
+
if (sourceEventId)
|
|
1281
|
+
extraParts.push(`source_event_id=${sourceEventId}`);
|
|
1282
|
+
if (sourceField)
|
|
1283
|
+
extraParts.push(`source_field=${sourceField}`);
|
|
1284
|
+
if (sourceFile)
|
|
1285
|
+
extraParts.push(`source_file=${sourceFile}`);
|
|
1286
|
+
if (excerpt)
|
|
1287
|
+
extraParts.push(`source_excerpt=${excerpt}`);
|
|
1288
|
+
const extra = extraParts.length > 0 ? `\n ${extraParts.join("\n ")}` : "";
|
|
1289
|
+
return `${index + 1}. [${item.id}] (${item.source}, score=${item.score.toFixed(4)}) ${item.text}${extra}`;
|
|
1290
|
+
})
|
|
553
1291
|
.join("\n")
|
|
554
1292
|
.slice(0, 18000);
|
|
555
1293
|
const prompt = [
|
|
556
|
-
|
|
557
|
-
"
|
|
558
|
-
"
|
|
559
|
-
"
|
|
560
|
-
"
|
|
561
|
-
"
|
|
562
|
-
"
|
|
563
|
-
"
|
|
564
|
-
"
|
|
565
|
-
"
|
|
1294
|
+
`prompt_version=${READ_FUSION_PROMPT_VERSION}`,
|
|
1295
|
+
"You are a memory retrieval fusion engine. Fuse multi-channel evidence into a structured answer package for the agent.",
|
|
1296
|
+
"Core values and principles:",
|
|
1297
|
+
"A) Truthfulness first: do not fabricate; do not infer beyond evidence.",
|
|
1298
|
+
"B) Evidence first: every key conclusion must be traceable via evidence_ids.",
|
|
1299
|
+
"C) Make conflicts explicit: write conflicts instead of silently overriding.",
|
|
1300
|
+
"D) Be transparent about uncertainty: put uncertain parts in coverage_note.",
|
|
1301
|
+
"E) Summary-first: prefer summary evidence for conclusions; source_excerpt is supporting evidence.",
|
|
1302
|
+
"F) Same-source dedup: merge duplicate evidence from the same source_memory_id/source_memory_canonical_id.",
|
|
1303
|
+
"G) Full-text recall: if summary/excerpt is insufficient, return event ids in need_fulltext_event_ids.",
|
|
1304
|
+
"Source channel semantics:",
|
|
1305
|
+
"- rules: policy/constraints; use for what should be done.",
|
|
1306
|
+
"- archive: event-level stable facts (summary-first).",
|
|
1307
|
+
"- vector: semantic neighbors for recall; source_field=summary/evidence indicates chunk role.",
|
|
1308
|
+
"- graph: entity-relation structure; prefer for dependency/relationship questions.",
|
|
1309
|
+
"Query alignment:",
|
|
1310
|
+
"- answer the user query first; ignore evidence unrelated to the query.",
|
|
1311
|
+
"- when evidence conflicts, prioritize recency + quality + explicitness and record the conflict.",
|
|
1312
|
+
"Return strict JSON only:",
|
|
1313
|
+
"{\"canonical_answer\": string, \"coverage_note\": string, \"facts\": [{\"text\": string, \"evidence_ids\": string[]}], \"timeline\": [{\"when\": string, \"event\": string, \"evidence_ids\": string[]}], \"entities\": [{\"name\": string, \"role\": string}], \"decisions\": [{\"decision\": string, \"rationale\": string, \"evidence_ids\": string[]}], \"fixes\": [{\"issue\": string, \"fix\": string, \"evidence_ids\": string[]}], \"preferences\": [{\"subject\": string, \"preference\": string, \"evidence_ids\": string[]}], \"risks\": [{\"risk\": string, \"mitigation\": string, \"evidence_ids\": string[]}], \"action_items\": [{\"item\": string, \"owner\": string, \"status\": string, \"evidence_ids\": string[]}], \"conflicts\": [{\"topic\": string, \"details\": string}], \"evidence_ids\": string[], \"need_fulltext_event_ids\": string[], \"confidence\": number}",
|
|
1314
|
+
"Output constraints:",
|
|
1315
|
+
"1) canonical_answer must be directly usable.",
|
|
1316
|
+
"2) facts: usually 3-12 items; prefer high-quality evidence.",
|
|
1317
|
+
"3) evidence_ids must come from input candidate ids.",
|
|
1318
|
+
"4) conflicts must be [] when no conflict exists.",
|
|
1319
|
+
"5) confidence must be within [0, 1].",
|
|
1320
|
+
"6) uncertain parts must be explicitly stated in coverage_note.",
|
|
1321
|
+
...READ_FUSION_REGRESSION_SAMPLES,
|
|
566
1322
|
].join("\n");
|
|
567
1323
|
const body = {
|
|
568
1324
|
model: args.llm.model,
|
|
569
1325
|
temperature: 0.1,
|
|
570
1326
|
messages: [
|
|
571
|
-
{ role: "system", content: "
|
|
572
|
-
{ role: "user", content: `${prompt}\n\n
|
|
1327
|
+
{ role: "system", content: "Output JSON only. No extra text." },
|
|
1328
|
+
{ role: "user", content: `${prompt}\n\nQuery:\n${args.query}\n\nCandidate Evidence:\n${evidenceText}` },
|
|
573
1329
|
],
|
|
574
1330
|
};
|
|
575
1331
|
let lastError = null;
|
|
576
1332
|
for (let attempt = 0; attempt < 2; attempt += 1) {
|
|
577
|
-
const
|
|
578
|
-
|
|
1333
|
+
const response = await (0, http_post_1.postJsonWithTimeout)({
|
|
1334
|
+
endpoint,
|
|
1335
|
+
apiKey: args.llm.apiKey,
|
|
1336
|
+
body,
|
|
1337
|
+
timeoutMs: EXTERNAL_MODEL_TIMEOUT_MS,
|
|
1338
|
+
});
|
|
1339
|
+
if (!response.ok) {
|
|
1340
|
+
lastError = new Error(response.status > 0 ? `fusion_http_${response.status}` : (response.error || "fusion_network_error"));
|
|
1341
|
+
continue;
|
|
1342
|
+
}
|
|
579
1343
|
try {
|
|
580
|
-
const
|
|
581
|
-
method: "POST",
|
|
582
|
-
headers: {
|
|
583
|
-
"content-type": "application/json",
|
|
584
|
-
authorization: `Bearer ${args.llm.apiKey}`,
|
|
585
|
-
},
|
|
586
|
-
body: JSON.stringify(body),
|
|
587
|
-
signal: controller.signal,
|
|
588
|
-
});
|
|
589
|
-
clearTimeout(timeoutId);
|
|
590
|
-
if (!response.ok) {
|
|
591
|
-
lastError = new Error(`fusion_http_${response.status}`);
|
|
592
|
-
continue;
|
|
593
|
-
}
|
|
594
|
-
const json = await response.json();
|
|
1344
|
+
const json = (response.json || {});
|
|
595
1345
|
const content = json?.choices?.[0]?.message?.content?.trim() || "";
|
|
596
1346
|
if (!content) {
|
|
597
1347
|
lastError = new Error("fusion_empty");
|
|
@@ -605,6 +1355,13 @@ async function requestFusion(args) {
|
|
|
605
1355
|
const evidenceIds = Array.isArray(parsed.evidence_ids)
|
|
606
1356
|
? parsed.evidence_ids.filter(item => typeof item === "string" && item.trim())
|
|
607
1357
|
: [];
|
|
1358
|
+
const whitelistedEvidenceIds = [...new Set(evidenceIds.filter(id => candidateIdSet.has(id)))];
|
|
1359
|
+
const needFulltextEventIds = Array.isArray(parsed.need_fulltext_event_ids)
|
|
1360
|
+
? [...new Set(parsed.need_fulltext_event_ids
|
|
1361
|
+
.filter(item => typeof item === "string")
|
|
1362
|
+
.map(item => item.trim())
|
|
1363
|
+
.filter(Boolean))]
|
|
1364
|
+
: [];
|
|
608
1365
|
return {
|
|
609
1366
|
canonical_answer: parsed.canonical_answer.trim().slice(0, 6000),
|
|
610
1367
|
coverage_note: typeof parsed.coverage_note === "string" ? parsed.coverage_note.trim().slice(0, 1200) : "",
|
|
@@ -617,14 +1374,14 @@ async function requestFusion(args) {
|
|
|
617
1374
|
risks: Array.isArray(parsed.risks) ? parsed.risks : [],
|
|
618
1375
|
action_items: Array.isArray(parsed.action_items) ? parsed.action_items : [],
|
|
619
1376
|
conflicts: Array.isArray(parsed.conflicts) ? parsed.conflicts : [],
|
|
620
|
-
evidence_ids:
|
|
1377
|
+
evidence_ids: whitelistedEvidenceIds,
|
|
1378
|
+
need_fulltext_event_ids: needFulltextEventIds,
|
|
621
1379
|
confidence: typeof parsed.confidence === "number"
|
|
622
1380
|
? Math.max(0, Math.min(1, parsed.confidence))
|
|
623
1381
|
: 0.5,
|
|
624
1382
|
};
|
|
625
1383
|
}
|
|
626
1384
|
catch (error) {
|
|
627
|
-
clearTimeout(timeoutId);
|
|
628
1385
|
lastError = error;
|
|
629
1386
|
}
|
|
630
1387
|
}
|
|
@@ -634,23 +1391,54 @@ function createReadStore(options) {
|
|
|
634
1391
|
const memoryRoot = options.dbPath ? path.resolve(options.dbPath) : path.join(options.projectRoot, "data", "memory");
|
|
635
1392
|
const vectorFallbackPath = path.join(memoryRoot, "vector", "lancedb_events.jsonl");
|
|
636
1393
|
const hitStatsPath = path.join(memoryRoot, ".read_hit_stats.json");
|
|
1394
|
+
let docsCache = null;
|
|
1395
|
+
let vectorFallbackCache = null;
|
|
1396
|
+
let bm25TokenCacheSignature = "";
|
|
1397
|
+
let bm25TokenCache = new Map();
|
|
1398
|
+
let hitStatsCache = null;
|
|
1399
|
+
let hitStatsDirty = false;
|
|
1400
|
+
let hitStatsPendingMutations = 0;
|
|
1401
|
+
let lastHitStatsFlushAt = 0;
|
|
1402
|
+
const hitStatsFlushIntervalMs = 5000;
|
|
1403
|
+
const hitStatsFlushBatch = 24;
|
|
1404
|
+
const readTuning = resolveReadTuning(options.readTuning);
|
|
1405
|
+
function fileSignature(filePath) {
|
|
1406
|
+
try {
|
|
1407
|
+
if (!fs.existsSync(filePath)) {
|
|
1408
|
+
return `${filePath}:missing`;
|
|
1409
|
+
}
|
|
1410
|
+
const stat = fs.statSync(filePath);
|
|
1411
|
+
return `${filePath}:${stat.size}:${Math.floor(stat.mtimeMs)}`;
|
|
1412
|
+
}
|
|
1413
|
+
catch {
|
|
1414
|
+
return `${filePath}:error`;
|
|
1415
|
+
}
|
|
1416
|
+
}
|
|
637
1417
|
function loadHitStats() {
|
|
1418
|
+
if (hitStatsCache) {
|
|
1419
|
+
return hitStatsCache;
|
|
1420
|
+
}
|
|
638
1421
|
try {
|
|
639
1422
|
if (!fs.existsSync(hitStatsPath)) {
|
|
640
|
-
|
|
1423
|
+
hitStatsCache = { items: {} };
|
|
1424
|
+
return hitStatsCache;
|
|
641
1425
|
}
|
|
642
1426
|
const content = fs.readFileSync(hitStatsPath, "utf-8").trim();
|
|
643
1427
|
if (!content) {
|
|
644
|
-
|
|
1428
|
+
hitStatsCache = { items: {} };
|
|
1429
|
+
return hitStatsCache;
|
|
645
1430
|
}
|
|
646
1431
|
const parsed = JSON.parse(content);
|
|
647
1432
|
if (!parsed || typeof parsed !== "object" || !parsed.items || typeof parsed.items !== "object") {
|
|
648
|
-
|
|
1433
|
+
hitStatsCache = { items: {} };
|
|
1434
|
+
return hitStatsCache;
|
|
649
1435
|
}
|
|
650
|
-
|
|
1436
|
+
hitStatsCache = parsed;
|
|
1437
|
+
return hitStatsCache;
|
|
651
1438
|
}
|
|
652
1439
|
catch {
|
|
653
|
-
|
|
1440
|
+
hitStatsCache = { items: {} };
|
|
1441
|
+
return hitStatsCache;
|
|
654
1442
|
}
|
|
655
1443
|
}
|
|
656
1444
|
function saveHitStats(state) {
|
|
@@ -665,6 +1453,21 @@ function createReadStore(options) {
|
|
|
665
1453
|
options.logger.warn(`Failed to persist read hit stats: ${error}`);
|
|
666
1454
|
}
|
|
667
1455
|
}
|
|
1456
|
+
function maybeFlushHitStats(force) {
|
|
1457
|
+
if (!hitStatsDirty || !hitStatsCache) {
|
|
1458
|
+
return;
|
|
1459
|
+
}
|
|
1460
|
+
const now = Date.now();
|
|
1461
|
+
if (!force &&
|
|
1462
|
+
hitStatsPendingMutations < hitStatsFlushBatch &&
|
|
1463
|
+
(now - lastHitStatsFlushAt) < hitStatsFlushIntervalMs) {
|
|
1464
|
+
return;
|
|
1465
|
+
}
|
|
1466
|
+
saveHitStats(hitStatsCache);
|
|
1467
|
+
hitStatsDirty = false;
|
|
1468
|
+
hitStatsPendingMutations = 0;
|
|
1469
|
+
lastHitStatsFlushAt = now;
|
|
1470
|
+
}
|
|
668
1471
|
function markHit(ids) {
|
|
669
1472
|
if (!ids.length) {
|
|
670
1473
|
return;
|
|
@@ -689,34 +1492,524 @@ function createReadStore(options) {
|
|
|
689
1492
|
})
|
|
690
1493
|
.slice(0, 20000);
|
|
691
1494
|
state.items = Object.fromEntries(entries);
|
|
692
|
-
|
|
1495
|
+
hitStatsCache = state;
|
|
1496
|
+
hitStatsDirty = true;
|
|
1497
|
+
hitStatsPendingMutations += ids.length;
|
|
1498
|
+
maybeFlushHitStats(false);
|
|
693
1499
|
}
|
|
694
1500
|
function loadAllDocuments() {
|
|
695
1501
|
const cortexRulesPath = path.join(memoryRoot, "CORTEX_RULES.md");
|
|
696
|
-
const memoryMdPath = path.join(memoryRoot, "MEMORY.md");
|
|
697
1502
|
const activeSessionsPath = path.join(memoryRoot, "sessions", "active", "sessions.jsonl");
|
|
698
1503
|
const archiveSessionsPath = path.join(memoryRoot, "sessions", "archive", "sessions.jsonl");
|
|
699
|
-
|
|
1504
|
+
const graphMemoryPath = path.join(memoryRoot, "graph", "memory.jsonl");
|
|
1505
|
+
const supersededRelationPath = path.join(memoryRoot, "graph", "superseded_relations.jsonl");
|
|
1506
|
+
const conflictQueuePath = path.join(memoryRoot, "graph", "conflict_queue.jsonl");
|
|
1507
|
+
const wikiProjectionIndexPath = path.join(memoryRoot, "wiki", ".projection_index.json");
|
|
1508
|
+
const signature = [
|
|
1509
|
+
fileSignature(cortexRulesPath),
|
|
1510
|
+
fileSignature(activeSessionsPath),
|
|
1511
|
+
fileSignature(archiveSessionsPath),
|
|
1512
|
+
fileSignature(graphMemoryPath),
|
|
1513
|
+
fileSignature(supersededRelationPath),
|
|
1514
|
+
fileSignature(conflictQueuePath),
|
|
1515
|
+
fileSignature(wikiProjectionIndexPath),
|
|
1516
|
+
].join("|");
|
|
1517
|
+
if (docsCache && docsCache.signature === signature) {
|
|
1518
|
+
return docsCache.docs;
|
|
1519
|
+
}
|
|
1520
|
+
const archiveEventTypeById = new Map();
|
|
1521
|
+
if (fs.existsSync(archiveSessionsPath)) {
|
|
1522
|
+
const archiveContent = safeReadFile(archiveSessionsPath);
|
|
1523
|
+
for (const line of archiveContent.split(/\r?\n/)) {
|
|
1524
|
+
const trimmed = line.trim();
|
|
1525
|
+
if (!trimmed)
|
|
1526
|
+
continue;
|
|
1527
|
+
try {
|
|
1528
|
+
const parsed = JSON.parse(trimmed);
|
|
1529
|
+
const id = typeof parsed.id === "string" ? parsed.id.trim() : "";
|
|
1530
|
+
const eventType = typeof parsed.event_type === "string" ? parsed.event_type.trim() : "";
|
|
1531
|
+
if (id && eventType) {
|
|
1532
|
+
archiveEventTypeById.set(id, eventType);
|
|
1533
|
+
}
|
|
1534
|
+
}
|
|
1535
|
+
catch { }
|
|
1536
|
+
}
|
|
1537
|
+
}
|
|
1538
|
+
const wikiRefIndex = readWikiProjectionRefIndex(wikiProjectionIndexPath, options.logger);
|
|
1539
|
+
const graphDocs = [];
|
|
1540
|
+
const supersededRelationKeys = new Set();
|
|
1541
|
+
if (fs.existsSync(supersededRelationPath)) {
|
|
1542
|
+
const supersededContent = safeReadFile(supersededRelationPath);
|
|
1543
|
+
for (const line of supersededContent.split(/\r?\n/)) {
|
|
1544
|
+
const trimmed = line.trim();
|
|
1545
|
+
if (!trimmed)
|
|
1546
|
+
continue;
|
|
1547
|
+
try {
|
|
1548
|
+
const parsed = JSON.parse(trimmed);
|
|
1549
|
+
const relationKey = typeof parsed.relation_key === "string" ? parsed.relation_key.trim().toLowerCase() : "";
|
|
1550
|
+
if (relationKey) {
|
|
1551
|
+
supersededRelationKeys.add(relationKey);
|
|
1552
|
+
}
|
|
1553
|
+
}
|
|
1554
|
+
catch (error) {
|
|
1555
|
+
options.logger.debug(`Skipping invalid superseded relation line: ${error}`);
|
|
1556
|
+
}
|
|
1557
|
+
}
|
|
1558
|
+
}
|
|
1559
|
+
if (fs.existsSync(graphMemoryPath)) {
|
|
1560
|
+
const graphContent = safeReadFile(graphMemoryPath);
|
|
1561
|
+
for (const line of graphContent.split(/\r?\n/)) {
|
|
1562
|
+
const trimmed = line.trim();
|
|
1563
|
+
if (!trimmed)
|
|
1564
|
+
continue;
|
|
1565
|
+
try {
|
|
1566
|
+
const parsed = JSON.parse(trimmed);
|
|
1567
|
+
const id = typeof parsed.id === "string" ? parsed.id : "";
|
|
1568
|
+
const summary = typeof parsed.summary === "string" ? parsed.summary.trim() : "";
|
|
1569
|
+
const sourceTextNav = typeof parsed.source_text_nav === "object" && parsed.source_text_nav !== null && !Array.isArray(parsed.source_text_nav)
|
|
1570
|
+
? parsed.source_text_nav
|
|
1571
|
+
: undefined;
|
|
1572
|
+
const sourceEventId = typeof parsed.source_event_id === "string" ? parsed.source_event_id : "";
|
|
1573
|
+
const archiveEventId = typeof parsed.archive_event_id === "string" ? parsed.archive_event_id : "";
|
|
1574
|
+
const navSourceEventId = typeof sourceTextNav?.source_event_id === "string" ? sourceTextNav.source_event_id.trim() : "";
|
|
1575
|
+
const navSourceMemoryId = typeof sourceTextNav?.source_memory_id === "string" ? sourceTextNav.source_memory_id.trim() : "";
|
|
1576
|
+
const navSessionId = typeof sourceTextNav?.session_id === "string" ? sourceTextNav.session_id.trim() : "";
|
|
1577
|
+
const navSourceLayer = typeof sourceTextNav?.layer === "string" ? sourceTextNav.layer.trim() : "";
|
|
1578
|
+
const navSourceFile = typeof sourceTextNav?.source_file === "string" ? sourceTextNav.source_file.trim() : "";
|
|
1579
|
+
const eventRefId = navSourceMemoryId || archiveEventId || navSourceEventId || sourceEventId;
|
|
1580
|
+
const sessionId = navSessionId || (typeof parsed.session_id === "string" ? parsed.session_id : "");
|
|
1581
|
+
const sourceLayer = navSourceLayer || (typeof parsed.source_layer === "string" ? parsed.source_layer : "");
|
|
1582
|
+
const sourceFile = navSourceFile || (typeof parsed.source_file === "string" ? parsed.source_file : "");
|
|
1583
|
+
const timestamp = typeof parsed.timestamp === "string" ? Date.parse(parsed.timestamp) : NaN;
|
|
1584
|
+
const entities = Array.isArray(parsed.entities)
|
|
1585
|
+
? parsed.entities.map((item) => (typeof item === "string" ? item.trim() : "")).filter(Boolean)
|
|
1586
|
+
: [];
|
|
1587
|
+
const entityTypes = typeof parsed.entity_types === "object" && parsed.entity_types !== null
|
|
1588
|
+
? parsed.entity_types
|
|
1589
|
+
: {};
|
|
1590
|
+
const relations = Array.isArray(parsed.relations) ? parsed.relations : [];
|
|
1591
|
+
const eventType = (typeof parsed.event_type === "string" ? parsed.event_type : "") || archiveEventTypeById.get(eventRefId) || "";
|
|
1592
|
+
let relationCount = 0;
|
|
1593
|
+
for (const relationRaw of relations) {
|
|
1594
|
+
if (typeof relationRaw !== "object" || relationRaw === null)
|
|
1595
|
+
continue;
|
|
1596
|
+
const relationRecord = relationRaw;
|
|
1597
|
+
const source = typeof relationRecord.source === "string" ? relationRecord.source.trim() : "";
|
|
1598
|
+
const target = typeof relationRecord.target === "string" ? relationRecord.target.trim() : "";
|
|
1599
|
+
const type = typeof relationRecord.type === "string" ? relationRecord.type.trim() : "related_to";
|
|
1600
|
+
if (!source || !target)
|
|
1601
|
+
continue;
|
|
1602
|
+
relationCount += 1;
|
|
1603
|
+
const relationKey = graphRelationKey({ source, target, type });
|
|
1604
|
+
const factStatus = supersededRelationKeys.has(relationKey) ? "superseded" : "active";
|
|
1605
|
+
const evidenceSpan = typeof relationRecord.evidence_span === "string" && relationRecord.evidence_span.trim()
|
|
1606
|
+
? relationRecord.evidence_span.trim()
|
|
1607
|
+
: undefined;
|
|
1608
|
+
const confidenceValue = typeof relationRecord.confidence === "number" && Number.isFinite(relationRecord.confidence)
|
|
1609
|
+
? relationRecord.confidence
|
|
1610
|
+
: undefined;
|
|
1611
|
+
const relationOrigin = typeof relationRecord.relation_origin === "string" && relationRecord.relation_origin.trim()
|
|
1612
|
+
? relationRecord.relation_origin.trim()
|
|
1613
|
+
: undefined;
|
|
1614
|
+
const relationDefinition = typeof relationRecord.relation_definition === "string" && relationRecord.relation_definition.trim()
|
|
1615
|
+
? relationRecord.relation_definition.trim()
|
|
1616
|
+
: undefined;
|
|
1617
|
+
const contextChunk = typeof relationRecord.context_chunk === "string" && relationRecord.context_chunk.trim()
|
|
1618
|
+
? relationRecord.context_chunk.trim()
|
|
1619
|
+
: undefined;
|
|
1620
|
+
const relation = {
|
|
1621
|
+
source,
|
|
1622
|
+
target,
|
|
1623
|
+
type,
|
|
1624
|
+
relation_origin: relationOrigin,
|
|
1625
|
+
relation_definition: relationDefinition,
|
|
1626
|
+
evidence_span: evidenceSpan,
|
|
1627
|
+
context_chunk: contextChunk,
|
|
1628
|
+
confidence: confidenceValue,
|
|
1629
|
+
fact_status: factStatus,
|
|
1630
|
+
source_event_id: sourceEventId || archiveEventId || undefined,
|
|
1631
|
+
relation_key: relationKey,
|
|
1632
|
+
};
|
|
1633
|
+
const relationEntities = uniqueStrings([...entities, source, target]);
|
|
1634
|
+
const wikiRefs = wikiRefsForGraphRelation(wikiRefIndex, { source, target, type });
|
|
1635
|
+
const entityLines = relationEntities.length > 0
|
|
1636
|
+
? relationEntities.map((entity, index) => {
|
|
1637
|
+
const entityType = entityTypes[entity];
|
|
1638
|
+
return `${index + 1}. ${entity}${entityType ? ` (${entityType})` : ""}`;
|
|
1639
|
+
}).join("\n")
|
|
1640
|
+
: "none";
|
|
1641
|
+
const text = [
|
|
1642
|
+
`# Graph Relation`,
|
|
1643
|
+
`record_id: ${id}`,
|
|
1644
|
+
`relation_index: ${relationCount}`,
|
|
1645
|
+
`relation_key: ${relationKey}`,
|
|
1646
|
+
`fact_status: ${factStatus}`,
|
|
1647
|
+
`source_event_id: ${sourceEventId || archiveEventId || "unknown"}`,
|
|
1648
|
+
`source_layer: ${sourceLayer || "unknown"}`,
|
|
1649
|
+
`archive_event_id: ${archiveEventId || "n/a"}`,
|
|
1650
|
+
`event_type: ${eventType || "unknown"}`,
|
|
1651
|
+
`session_id: ${sessionId || "unknown"}`,
|
|
1652
|
+
`source_file: ${sourceFile || "unknown"}`,
|
|
1653
|
+
`evidence_span: ${evidenceSpan || "n/a"}`,
|
|
1654
|
+
`context_chunk: ${contextChunk || "n/a"}`,
|
|
1655
|
+
`relation_origin: ${relationOrigin || "n/a"}`,
|
|
1656
|
+
`relation_definition: ${relationDefinition || "n/a"}`,
|
|
1657
|
+
`confidence: ${typeof confidenceValue === "number" ? confidenceValue : "n/a"}`,
|
|
1658
|
+
``,
|
|
1659
|
+
`## Summary`,
|
|
1660
|
+
summary || "n/a",
|
|
1661
|
+
``,
|
|
1662
|
+
`## Source References`,
|
|
1663
|
+
`source_event_id: ${navSourceEventId || sourceEventId || archiveEventId || "unknown"}`,
|
|
1664
|
+
`source_memory_id: ${navSourceMemoryId || eventRefId || "unknown"}`,
|
|
1665
|
+
`source_layer: ${sourceLayer || "unknown"}`,
|
|
1666
|
+
`source_file: ${sourceFile || "unknown"}`,
|
|
1667
|
+
`session_id: ${sessionId || "unknown"}`,
|
|
1668
|
+
``,
|
|
1669
|
+
`## Entities`,
|
|
1670
|
+
entityLines,
|
|
1671
|
+
``,
|
|
1672
|
+
`## Relation`,
|
|
1673
|
+
`${source} -[${type}/${factStatus}]-> ${target}`,
|
|
1674
|
+
].join("\n");
|
|
1675
|
+
graphDocs.push({
|
|
1676
|
+
id: `${id || "graph"}:rel:${relationCount}`,
|
|
1677
|
+
text,
|
|
1678
|
+
source: "sessions_graph",
|
|
1679
|
+
timestamp: Number.isFinite(timestamp) ? timestamp : undefined,
|
|
1680
|
+
layer: sourceLayer === "active_only" ? "active" : "archive",
|
|
1681
|
+
summaryText: summary || undefined,
|
|
1682
|
+
sourceText: contextChunk || undefined,
|
|
1683
|
+
sourceMemoryId: eventRefId || id,
|
|
1684
|
+
sourceEventId: navSourceEventId || sourceEventId || archiveEventId || undefined,
|
|
1685
|
+
sourceFile: sourceFile || undefined,
|
|
1686
|
+
sessionId,
|
|
1687
|
+
entities: relationEntities,
|
|
1688
|
+
relations: [relation],
|
|
1689
|
+
eventType: eventType || undefined,
|
|
1690
|
+
qualityScore: typeof parsed.confidence === "number" && Number.isFinite(parsed.confidence) ? parsed.confidence : 1,
|
|
1691
|
+
factStatus,
|
|
1692
|
+
wikiRef: wikiRefs[0] || undefined,
|
|
1693
|
+
wikiRefs,
|
|
1694
|
+
evidenceIds: buildGraphEvidenceIds({
|
|
1695
|
+
source,
|
|
1696
|
+
target,
|
|
1697
|
+
type,
|
|
1698
|
+
sourceEventId: relation.source_event_id,
|
|
1699
|
+
evidenceSpan,
|
|
1700
|
+
wikiRefs,
|
|
1701
|
+
}),
|
|
1702
|
+
});
|
|
1703
|
+
}
|
|
1704
|
+
}
|
|
1705
|
+
catch (error) {
|
|
1706
|
+
options.logger.debug(`Skipping invalid graph memory line: ${error}`);
|
|
1707
|
+
}
|
|
1708
|
+
}
|
|
1709
|
+
}
|
|
1710
|
+
if (fs.existsSync(conflictQueuePath)) {
|
|
1711
|
+
const queueContent = safeReadFile(conflictQueuePath);
|
|
1712
|
+
for (const line of queueContent.split(/\r?\n/)) {
|
|
1713
|
+
const trimmed = line.trim();
|
|
1714
|
+
if (!trimmed)
|
|
1715
|
+
continue;
|
|
1716
|
+
try {
|
|
1717
|
+
const parsed = JSON.parse(trimmed);
|
|
1718
|
+
const status = normalizeFactStatus(typeof parsed.status === "string" ? parsed.status : "");
|
|
1719
|
+
if (status !== "pending_conflict" && status !== "rejected") {
|
|
1720
|
+
continue;
|
|
1721
|
+
}
|
|
1722
|
+
const conflictId = typeof parsed.conflict_id === "string" ? parsed.conflict_id.trim() : "";
|
|
1723
|
+
const sourceEventId = typeof parsed.source_event_id === "string" ? parsed.source_event_id.trim() : "";
|
|
1724
|
+
const sessionId = typeof parsed.session_id === "string" ? parsed.session_id.trim() : "";
|
|
1725
|
+
const sourceFile = typeof parsed.source_file === "string" ? parsed.source_file.trim() : "";
|
|
1726
|
+
const sourceLayer = typeof parsed.source_layer === "string" ? parsed.source_layer.trim() : "";
|
|
1727
|
+
const updatedAt = typeof parsed.updated_at === "string" ? Date.parse(parsed.updated_at) : NaN;
|
|
1728
|
+
const candidate = typeof parsed.candidate === "object" && parsed.candidate !== null
|
|
1729
|
+
? parsed.candidate
|
|
1730
|
+
: {};
|
|
1731
|
+
const candidateSummary = typeof candidate.summary === "string" ? candidate.summary.trim() : "";
|
|
1732
|
+
const candidateSourceTextNav = typeof candidate.source_text_nav === "object" && candidate.source_text_nav !== null && !Array.isArray(candidate.source_text_nav)
|
|
1733
|
+
? candidate.source_text_nav
|
|
1734
|
+
: undefined;
|
|
1735
|
+
const navSourceEventId = typeof candidateSourceTextNav?.source_event_id === "string" ? candidateSourceTextNav.source_event_id.trim() : "";
|
|
1736
|
+
const navSourceMemoryId = typeof candidateSourceTextNav?.source_memory_id === "string" ? candidateSourceTextNav.source_memory_id.trim() : "";
|
|
1737
|
+
const navSourceLayer = typeof candidateSourceTextNav?.layer === "string" ? candidateSourceTextNav.layer.trim() : "";
|
|
1738
|
+
const navSourceFile = typeof candidateSourceTextNav?.source_file === "string" ? candidateSourceTextNav.source_file.trim() : "";
|
|
1739
|
+
const navSessionId = typeof candidateSourceTextNav?.session_id === "string" ? candidateSourceTextNav.session_id.trim() : "";
|
|
1740
|
+
const candidateEventType = typeof candidate.event_type === "string" ? candidate.event_type.trim() : "";
|
|
1741
|
+
const candidateRelations = Array.isArray(candidate.relations) ? candidate.relations : [];
|
|
1742
|
+
let relationIndex = 0;
|
|
1743
|
+
for (const relationRaw of candidateRelations) {
|
|
1744
|
+
if (typeof relationRaw !== "object" || relationRaw === null)
|
|
1745
|
+
continue;
|
|
1746
|
+
const relationRecord = relationRaw;
|
|
1747
|
+
const source = typeof relationRecord.source === "string" ? relationRecord.source.trim() : "";
|
|
1748
|
+
const target = typeof relationRecord.target === "string" ? relationRecord.target.trim() : "";
|
|
1749
|
+
const type = typeof relationRecord.type === "string" ? relationRecord.type.trim() : "related_to";
|
|
1750
|
+
if (!source || !target)
|
|
1751
|
+
continue;
|
|
1752
|
+
relationIndex += 1;
|
|
1753
|
+
const relationKey = graphRelationKey({ source, target, type });
|
|
1754
|
+
const evidenceSpan = typeof relationRecord.evidence_span === "string" && relationRecord.evidence_span.trim()
|
|
1755
|
+
? relationRecord.evidence_span.trim()
|
|
1756
|
+
: undefined;
|
|
1757
|
+
const confidenceValue = typeof relationRecord.confidence === "number" && Number.isFinite(relationRecord.confidence)
|
|
1758
|
+
? relationRecord.confidence
|
|
1759
|
+
: undefined;
|
|
1760
|
+
const relationOrigin = typeof relationRecord.relation_origin === "string" && relationRecord.relation_origin.trim()
|
|
1761
|
+
? relationRecord.relation_origin.trim()
|
|
1762
|
+
: undefined;
|
|
1763
|
+
const relationDefinition = typeof relationRecord.relation_definition === "string" && relationRecord.relation_definition.trim()
|
|
1764
|
+
? relationRecord.relation_definition.trim()
|
|
1765
|
+
: undefined;
|
|
1766
|
+
const contextChunk = typeof relationRecord.context_chunk === "string" && relationRecord.context_chunk.trim()
|
|
1767
|
+
? relationRecord.context_chunk.trim()
|
|
1768
|
+
: undefined;
|
|
1769
|
+
const relation = {
|
|
1770
|
+
source,
|
|
1771
|
+
target,
|
|
1772
|
+
type,
|
|
1773
|
+
relation_origin: relationOrigin,
|
|
1774
|
+
relation_definition: relationDefinition,
|
|
1775
|
+
evidence_span: evidenceSpan,
|
|
1776
|
+
context_chunk: contextChunk,
|
|
1777
|
+
confidence: confidenceValue,
|
|
1778
|
+
fact_status: status,
|
|
1779
|
+
source_event_id: sourceEventId || undefined,
|
|
1780
|
+
conflict_id: conflictId || undefined,
|
|
1781
|
+
relation_key: relationKey,
|
|
1782
|
+
};
|
|
1783
|
+
const wikiRefs = wikiRefsForGraphRelation(wikiRefIndex, { source, target, type });
|
|
1784
|
+
const evidenceIds = uniqueStrings([
|
|
1785
|
+
...buildGraphEvidenceIds({
|
|
1786
|
+
source,
|
|
1787
|
+
target,
|
|
1788
|
+
type,
|
|
1789
|
+
sourceEventId: sourceEventId || undefined,
|
|
1790
|
+
evidenceSpan,
|
|
1791
|
+
wikiRefs,
|
|
1792
|
+
}),
|
|
1793
|
+
conflictId ? `graph:conflict:${conflictId}` : "",
|
|
1794
|
+
]);
|
|
1795
|
+
const text = [
|
|
1796
|
+
`# Graph Conflict Candidate`,
|
|
1797
|
+
`conflict_id: ${conflictId || "unknown"}`,
|
|
1798
|
+
`fact_status: ${status}`,
|
|
1799
|
+
`source_event_id: ${sourceEventId || "unknown"}`,
|
|
1800
|
+
`source_layer: ${sourceLayer || "unknown"}`,
|
|
1801
|
+
`event_type: ${candidateEventType || "unknown"}`,
|
|
1802
|
+
`session_id: ${sessionId || "unknown"}`,
|
|
1803
|
+
`source_file: ${sourceFile || "unknown"}`,
|
|
1804
|
+
`relation_key: ${relationKey}`,
|
|
1805
|
+
`evidence_span: ${evidenceSpan || "n/a"}`,
|
|
1806
|
+
`context_chunk: ${contextChunk || "n/a"}`,
|
|
1807
|
+
`relation_origin: ${relationOrigin || "n/a"}`,
|
|
1808
|
+
`relation_definition: ${relationDefinition || "n/a"}`,
|
|
1809
|
+
`confidence: ${typeof confidenceValue === "number" ? confidenceValue : "n/a"}`,
|
|
1810
|
+
``,
|
|
1811
|
+
`## Summary`,
|
|
1812
|
+
candidateSummary || "n/a",
|
|
1813
|
+
``,
|
|
1814
|
+
`## Source References`,
|
|
1815
|
+
`source_event_id: ${navSourceEventId || sourceEventId || "unknown"}`,
|
|
1816
|
+
`source_memory_id: ${navSourceMemoryId || sourceEventId || "unknown"}`,
|
|
1817
|
+
`source_layer: ${navSourceLayer || sourceLayer || "unknown"}`,
|
|
1818
|
+
`source_file: ${navSourceFile || sourceFile || "unknown"}`,
|
|
1819
|
+
`session_id: ${navSessionId || sessionId || "unknown"}`,
|
|
1820
|
+
``,
|
|
1821
|
+
`${source} -[${type}/${status}]-> ${target}`,
|
|
1822
|
+
].join("\n");
|
|
1823
|
+
graphDocs.push({
|
|
1824
|
+
id: `gcf:${conflictId || "unknown"}:rel:${relationIndex}`,
|
|
1825
|
+
text,
|
|
1826
|
+
source: "sessions_graph_conflict",
|
|
1827
|
+
timestamp: Number.isFinite(updatedAt) ? updatedAt : undefined,
|
|
1828
|
+
layer: (navSourceLayer || sourceLayer) === "active_only" ? "active" : "archive",
|
|
1829
|
+
summaryText: candidateSummary || undefined,
|
|
1830
|
+
sourceText: contextChunk || undefined,
|
|
1831
|
+
sourceMemoryId: navSourceMemoryId || relationKey,
|
|
1832
|
+
sourceEventId: navSourceEventId || sourceEventId || undefined,
|
|
1833
|
+
sourceFile: navSourceFile || sourceFile || undefined,
|
|
1834
|
+
sessionId: navSessionId || sessionId || undefined,
|
|
1835
|
+
entities: uniqueStrings([source, target]),
|
|
1836
|
+
relations: [relation],
|
|
1837
|
+
eventType: candidateEventType || "graph_conflict",
|
|
1838
|
+
qualityScore: typeof confidenceValue === "number" ? confidenceValue : 0.8,
|
|
1839
|
+
factStatus: status,
|
|
1840
|
+
wikiRef: wikiRefs[0] || undefined,
|
|
1841
|
+
wikiRefs,
|
|
1842
|
+
evidenceIds,
|
|
1843
|
+
});
|
|
1844
|
+
}
|
|
1845
|
+
}
|
|
1846
|
+
catch (error) {
|
|
1847
|
+
options.logger.debug(`Skipping invalid conflict queue line: ${error}`);
|
|
1848
|
+
}
|
|
1849
|
+
}
|
|
1850
|
+
}
|
|
1851
|
+
const entitySummaryDocs = buildEntityGraphSummaryDocs(graphDocs.filter(item => item.factStatus === "active")).map(doc => {
|
|
1852
|
+
const wikiRefs = uniqueStrings((Array.isArray(doc.entities) ? doc.entities : [])
|
|
1853
|
+
.map(entity => wikiRefIndex.entityByName.get(entity.trim().toLowerCase()) || ""));
|
|
1854
|
+
if (wikiRefs.length === 0) {
|
|
1855
|
+
return doc;
|
|
1856
|
+
}
|
|
1857
|
+
const wikiAnchor = "current-facts";
|
|
1858
|
+
return {
|
|
1859
|
+
...doc,
|
|
1860
|
+
wikiRef: wikiRefs[0],
|
|
1861
|
+
wikiRefs,
|
|
1862
|
+
wikiAnchor,
|
|
1863
|
+
evidenceIds: uniqueStrings([
|
|
1864
|
+
...(Array.isArray(doc.evidenceIds) ? doc.evidenceIds : []),
|
|
1865
|
+
...wikiRefs.map(ref => {
|
|
1866
|
+
const wikiRef = normalizeWikiEvidenceRef(ref);
|
|
1867
|
+
return wikiRef ? `wiki:${wikiRef}#${wikiAnchor}` : "";
|
|
1868
|
+
}),
|
|
1869
|
+
]),
|
|
1870
|
+
};
|
|
1871
|
+
});
|
|
1872
|
+
const docs = [
|
|
700
1873
|
...parseMarkdownFile(cortexRulesPath, "CORTEX_RULES.md"),
|
|
701
|
-
...parseMarkdownFile(memoryMdPath, "MEMORY.md"),
|
|
702
1874
|
...parseJsonlFile(activeSessionsPath, "sessions_active", options.logger),
|
|
703
1875
|
...parseJsonlFile(archiveSessionsPath, "sessions_archive", options.logger),
|
|
1876
|
+
...graphDocs,
|
|
1877
|
+
...entitySummaryDocs,
|
|
704
1878
|
];
|
|
1879
|
+
docsCache = { signature, docs };
|
|
1880
|
+
return docs;
|
|
1881
|
+
}
|
|
1882
|
+
function buildVectorFallbackIndex(signature, docs) {
|
|
1883
|
+
const bySourceMemoryId = new Map();
|
|
1884
|
+
const vectorEntries = [];
|
|
1885
|
+
for (const doc of docs) {
|
|
1886
|
+
const key = (doc.sourceMemoryId || "").trim();
|
|
1887
|
+
if (key) {
|
|
1888
|
+
const list = bySourceMemoryId.get(key) || [];
|
|
1889
|
+
list.push(doc);
|
|
1890
|
+
bySourceMemoryId.set(key, list);
|
|
1891
|
+
}
|
|
1892
|
+
if (Array.isArray(doc.embedding) && doc.embedding.length > 0) {
|
|
1893
|
+
const norm = doc.embeddingNorm || vectorNorm(doc.embedding);
|
|
1894
|
+
if (norm > 0) {
|
|
1895
|
+
vectorEntries.push({ doc, embedding: doc.embedding, norm });
|
|
1896
|
+
}
|
|
1897
|
+
}
|
|
1898
|
+
}
|
|
1899
|
+
return { signature, docs, vectorEntries, bySourceMemoryId };
|
|
1900
|
+
}
|
|
1901
|
+
function loadVectorFallbackCached() {
|
|
1902
|
+
const signature = fileSignature(vectorFallbackPath);
|
|
1903
|
+
if (vectorFallbackCache && vectorFallbackCache.signature === signature) {
|
|
1904
|
+
return vectorFallbackCache;
|
|
1905
|
+
}
|
|
1906
|
+
const docs = parseVectorFallback(vectorFallbackPath, options.logger);
|
|
1907
|
+
vectorFallbackCache = buildVectorFallbackIndex(signature, docs);
|
|
1908
|
+
return vectorFallbackCache;
|
|
1909
|
+
}
|
|
1910
|
+
function selectVectorFallbackDocs(index, queryVector, queryNorm, limit) {
|
|
1911
|
+
if (!queryVector || queryVector.length === 0 || queryNorm <= 0 || index.vectorEntries.length === 0) {
|
|
1912
|
+
return index.docs;
|
|
1913
|
+
}
|
|
1914
|
+
const cappedLimit = Math.max(1, Math.floor(limit));
|
|
1915
|
+
const selected = new Map();
|
|
1916
|
+
const ranked = index.vectorEntries
|
|
1917
|
+
.map(entry => ({
|
|
1918
|
+
doc: entry.doc,
|
|
1919
|
+
score: cosineSimilarityWithNorm(queryVector, entry.embedding, queryNorm, entry.norm),
|
|
1920
|
+
}))
|
|
1921
|
+
.filter(item => Number.isFinite(item.score))
|
|
1922
|
+
.sort((a, b) => b.score - a.score)
|
|
1923
|
+
.slice(0, cappedLimit);
|
|
1924
|
+
for (const item of ranked) {
|
|
1925
|
+
selected.set(item.doc.id, item.doc);
|
|
1926
|
+
const sourceMemoryId = (item.doc.sourceMemoryId || "").trim();
|
|
1927
|
+
if (!sourceMemoryId)
|
|
1928
|
+
continue;
|
|
1929
|
+
for (const sibling of index.bySourceMemoryId.get(sourceMemoryId) || []) {
|
|
1930
|
+
if (selected.size >= cappedLimit)
|
|
1931
|
+
break;
|
|
1932
|
+
selected.set(sibling.id, sibling);
|
|
1933
|
+
}
|
|
1934
|
+
}
|
|
1935
|
+
return [...selected.values()];
|
|
1936
|
+
}
|
|
1937
|
+
function getBm25Tokens(doc, signature, channel) {
|
|
1938
|
+
if (bm25TokenCacheSignature !== signature) {
|
|
1939
|
+
bm25TokenCacheSignature = signature;
|
|
1940
|
+
bm25TokenCache = new Map();
|
|
1941
|
+
}
|
|
1942
|
+
const isSessionMemory = doc.source === "sessions_active" || doc.source === "sessions_archive";
|
|
1943
|
+
const summaryChannelText = isSessionMemory
|
|
1944
|
+
? ((doc.summaryText || "").trim())
|
|
1945
|
+
: ((doc.summaryText || doc.text || "").trim());
|
|
1946
|
+
const channelText = channel === "fulltext"
|
|
1947
|
+
? ((doc.sourceText || "").trim() || summaryChannelText)
|
|
1948
|
+
: summaryChannelText;
|
|
1949
|
+
const key = `${channel}:${doc.source}|${doc.id}|${channelText.length}|${channelText.slice(0, 64)}`;
|
|
1950
|
+
const cached = bm25TokenCache.get(key);
|
|
1951
|
+
if (cached) {
|
|
1952
|
+
return cached;
|
|
1953
|
+
}
|
|
1954
|
+
const tokens = tokenize(channelText);
|
|
1955
|
+
bm25TokenCache.set(key, tokens);
|
|
1956
|
+
return tokens;
|
|
705
1957
|
}
|
|
706
1958
|
async function searchMemory(args) {
|
|
1959
|
+
const startedAt = Date.now();
|
|
1960
|
+
const timing = {};
|
|
707
1961
|
const query = args.query?.trim();
|
|
1962
|
+
const mode = args.mode === "lightweight" ? "lightweight" : (args.mode === "auto" ? "auto" : "default");
|
|
1963
|
+
const fusionMode = normalizeFusionMode(args.fusionMode);
|
|
1964
|
+
const trackHits = args.trackHits !== false;
|
|
1965
|
+
const finish = (payload, debug = {}) => {
|
|
1966
|
+
timing.total_ms = Date.now() - startedAt;
|
|
1967
|
+
return {
|
|
1968
|
+
...payload,
|
|
1969
|
+
channel_results: payload.channel_results || emptyChannelResults(),
|
|
1970
|
+
timing_ms: roundTiming(timing),
|
|
1971
|
+
debug: {
|
|
1972
|
+
mode,
|
|
1973
|
+
fusion_mode: fusionMode,
|
|
1974
|
+
track_hits: trackHits,
|
|
1975
|
+
planned_queries: debug.planned_queries || [],
|
|
1976
|
+
fulltext_fallback_used: Boolean(debug.fulltext_fallback_used),
|
|
1977
|
+
vector_source: debug.vector_source || "none",
|
|
1978
|
+
},
|
|
1979
|
+
};
|
|
1980
|
+
};
|
|
708
1981
|
if (!query) {
|
|
709
|
-
return {
|
|
1982
|
+
return finish({
|
|
1983
|
+
results: [],
|
|
1984
|
+
semantic_results: [],
|
|
1985
|
+
keyword_results: [],
|
|
1986
|
+
strategy: "vector_sentence_and_keyword_parallel",
|
|
1987
|
+
});
|
|
710
1988
|
}
|
|
1989
|
+
const lightweightMode = mode === "lightweight";
|
|
1990
|
+
const externalModelMode = mode === "default";
|
|
1991
|
+
const loadDocsStartedAt = Date.now();
|
|
711
1992
|
const docs = loadAllDocuments();
|
|
1993
|
+
appendTiming(timing, "load_docs_ms", loadDocsStartedAt);
|
|
1994
|
+
const hitStatsStartedAt = Date.now();
|
|
712
1995
|
const hitStats = loadHitStats();
|
|
1996
|
+
appendTiming(timing, "load_hit_stats_ms", hitStatsStartedAt);
|
|
1997
|
+
const planningStartedAt = Date.now();
|
|
713
1998
|
const intent = classifyIntent(query);
|
|
714
1999
|
const preferredTypes = preferredEventTypes(intent);
|
|
2000
|
+
const plannedQueriesRaw = lightweightMode ? [query] : planQueryKeywords(query);
|
|
2001
|
+
const plannedQueries = plannedQueriesRaw.length > 0 ? plannedQueriesRaw : [query];
|
|
2002
|
+
appendTiming(timing, "query_planning_ms", planningStartedAt);
|
|
2003
|
+
const summaryChannelWeight = 1;
|
|
2004
|
+
const fulltextChannelWeight = 0.35;
|
|
2005
|
+
const maxCandidatePool = Math.max(1, Math.max(args.topK, 20));
|
|
715
2006
|
let queryEmbedding = null;
|
|
2007
|
+
let queryEmbeddingNorm = 0;
|
|
716
2008
|
const embeddingModel = options.embedding?.model || "";
|
|
717
2009
|
const embeddingApiKey = options.embedding?.apiKey || "";
|
|
718
2010
|
const embeddingBaseUrl = normalizeBaseUrl(options.embedding?.baseURL || options.embedding?.baseUrl);
|
|
719
|
-
if (embeddingModel && embeddingApiKey && embeddingBaseUrl) {
|
|
2011
|
+
if (externalModelMode && embeddingModel && embeddingApiKey && embeddingBaseUrl) {
|
|
2012
|
+
const embeddingStartedAt = Date.now();
|
|
720
2013
|
try {
|
|
721
2014
|
queryEmbedding = await requestEmbedding({
|
|
722
2015
|
text: query,
|
|
@@ -725,24 +2018,71 @@ function createReadStore(options) {
|
|
|
725
2018
|
baseUrl: embeddingBaseUrl,
|
|
726
2019
|
dimensions: options.embedding?.dimensions,
|
|
727
2020
|
});
|
|
2021
|
+
queryEmbeddingNorm = queryEmbedding && queryEmbedding.length > 0 ? vectorNorm(queryEmbedding) : 0;
|
|
728
2022
|
}
|
|
729
2023
|
catch (error) {
|
|
730
2024
|
options.logger.warn(`Embedding query failed, fallback to lexical search: ${error}`);
|
|
731
2025
|
}
|
|
2026
|
+
finally {
|
|
2027
|
+
appendTiming(timing, "embedding_ms", embeddingStartedAt);
|
|
2028
|
+
}
|
|
2029
|
+
}
|
|
2030
|
+
let vectorDocsFromLance = [];
|
|
2031
|
+
if (queryEmbedding && queryEmbedding.length > 0) {
|
|
2032
|
+
const lanceStartedAt = Date.now();
|
|
2033
|
+
vectorDocsFromLance = await searchLanceDb({ memoryRoot, queryEmbedding, limit: Math.max(20, args.topK * 8), logger: options.logger });
|
|
2034
|
+
appendTiming(timing, "lance_vector_ms", lanceStartedAt);
|
|
2035
|
+
}
|
|
2036
|
+
let vectorFallbackIndex = null;
|
|
2037
|
+
let vectorDocsFallback = [];
|
|
2038
|
+
if (vectorDocsFromLance.length === 0) {
|
|
2039
|
+
const indexStartedAt = Date.now();
|
|
2040
|
+
vectorFallbackIndex = loadVectorFallbackCached();
|
|
2041
|
+
vectorDocsFallback = selectVectorFallbackDocs(vectorFallbackIndex, queryEmbedding, queryEmbeddingNorm, Math.max(20, args.topK * 8));
|
|
2042
|
+
appendTiming(timing, "vector_index_ms", indexStartedAt);
|
|
732
2043
|
}
|
|
733
|
-
const
|
|
734
|
-
?
|
|
735
|
-
:
|
|
736
|
-
const vectorDocsFallback = vectorDocsFromLance.length > 0
|
|
737
|
-
? []
|
|
738
|
-
: parseVectorFallback(vectorFallbackPath, options.logger);
|
|
2044
|
+
const vectorSource = vectorDocsFromLance.length > 0
|
|
2045
|
+
? "lancedb"
|
|
2046
|
+
: (vectorDocsFallback.length > 0 ? "jsonl_index" : "none");
|
|
739
2047
|
const vectorDocs = [...vectorDocsFromLance, ...vectorDocsFallback];
|
|
2048
|
+
const archiveLinkStartedAt = Date.now();
|
|
2049
|
+
const archiveSourceById = new Map();
|
|
2050
|
+
for (const doc of docs) {
|
|
2051
|
+
if (doc.source !== "sessions_archive")
|
|
2052
|
+
continue;
|
|
2053
|
+
const key = (doc.sourceMemoryId || doc.id || "").trim();
|
|
2054
|
+
if (!key)
|
|
2055
|
+
continue;
|
|
2056
|
+
archiveSourceById.set(key, {
|
|
2057
|
+
sourceText: doc.sourceText,
|
|
2058
|
+
summaryText: doc.summaryText || doc.text,
|
|
2059
|
+
sourceFile: doc.sourceFile,
|
|
2060
|
+
});
|
|
2061
|
+
}
|
|
2062
|
+
for (const doc of vectorDocs) {
|
|
2063
|
+
const key = (doc.sourceMemoryId || "").trim();
|
|
2064
|
+
if (!key)
|
|
2065
|
+
continue;
|
|
2066
|
+
const linked = archiveSourceById.get(key);
|
|
2067
|
+
if (!linked)
|
|
2068
|
+
continue;
|
|
2069
|
+
if (!doc.sourceText && linked.sourceText) {
|
|
2070
|
+
doc.sourceText = linked.sourceText;
|
|
2071
|
+
}
|
|
2072
|
+
if (!doc.summaryText && linked.summaryText) {
|
|
2073
|
+
doc.summaryText = linked.summaryText;
|
|
2074
|
+
}
|
|
2075
|
+
if (!doc.sourceFile && linked.sourceFile) {
|
|
2076
|
+
doc.sourceFile = linked.sourceFile;
|
|
2077
|
+
}
|
|
2078
|
+
}
|
|
2079
|
+
appendTiming(timing, "archive_link_ms", archiveLinkStartedAt);
|
|
740
2080
|
const graphDocs = docs
|
|
741
|
-
.filter(doc =>
|
|
2081
|
+
.filter(doc => doc.source.startsWith("sessions_graph"))
|
|
742
2082
|
.map(doc => {
|
|
743
2083
|
const graphText = [
|
|
744
2084
|
doc.text,
|
|
745
|
-
...(doc.relations || []).map(relation => `${relation.source} ${relation.type} ${relation.target}`),
|
|
2085
|
+
...(doc.relations || []).map(relation => `${relation.source} ${relation.type} ${relation.target} ${relation.fact_status || doc.factStatus || "active"}`),
|
|
746
2086
|
].join(" | ");
|
|
747
2087
|
return {
|
|
748
2088
|
...doc,
|
|
@@ -750,124 +2090,372 @@ function createReadStore(options) {
|
|
|
750
2090
|
};
|
|
751
2091
|
});
|
|
752
2092
|
const rulesDocs = docs.filter(doc => doc.source === "CORTEX_RULES.md");
|
|
753
|
-
const archiveDocs = docs.filter(doc => doc.source.
|
|
754
|
-
const
|
|
755
|
-
const
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
2093
|
+
const archiveDocs = docs.filter(doc => doc.source === "sessions_active" || doc.source === "sessions_archive");
|
|
2094
|
+
const bm25Corpus = [...rulesDocs, ...archiveDocs, ...vectorDocs, ...graphDocs];
|
|
2095
|
+
const bm25Signature = `${docsCache?.signature || "na"}|vector:${vectorDocs.length}:${vectorDocs.slice(0, 40).map(item => `${item.id}:${item.text.length}`).join(",")}`;
|
|
2096
|
+
const rankByQuery = (plannedQuery, includeFulltext) => {
|
|
2097
|
+
const bm25Terms = uniqueStrings(tokenize(plannedQuery));
|
|
2098
|
+
const bm25StatsSummary = buildBm25Stats(bm25Corpus, bm25Terms, doc => getBm25Tokens(doc, bm25Signature, "summary"));
|
|
2099
|
+
const bm25StatsFulltext = includeFulltext
|
|
2100
|
+
? buildBm25Stats(bm25Corpus, bm25Terms, doc => getBm25Tokens(doc, bm25Signature, "fulltext"))
|
|
2101
|
+
: { avgDocLen: 1, docFreq: new Map() };
|
|
2102
|
+
const channels = {
|
|
2103
|
+
rules: [],
|
|
2104
|
+
archive: [],
|
|
2105
|
+
vector: [],
|
|
2106
|
+
graph: [],
|
|
2107
|
+
};
|
|
2108
|
+
const evaluateDoc = (doc, source) => {
|
|
2109
|
+
const isSessionMemory = doc.source === "sessions_active" || doc.source === "sessions_archive";
|
|
2110
|
+
const summaryText = isSessionMemory
|
|
2111
|
+
? (doc.summaryText || "").trim()
|
|
2112
|
+
: (doc.summaryText || doc.text || "").trim();
|
|
2113
|
+
const fulltextText = (doc.sourceText || "").trim();
|
|
2114
|
+
const summaryBm25 = bm25Score({
|
|
2115
|
+
queryTerms: bm25Terms,
|
|
2116
|
+
docText: summaryText,
|
|
2117
|
+
docTokens: getBm25Tokens(doc, bm25Signature, "summary"),
|
|
2118
|
+
docCount: bm25Corpus.length,
|
|
2119
|
+
avgDocLen: bm25StatsSummary.avgDocLen,
|
|
2120
|
+
docFreq: bm25StatsSummary.docFreq,
|
|
2121
|
+
});
|
|
2122
|
+
const fulltextBm25 = includeFulltext
|
|
2123
|
+
? bm25Score({
|
|
2124
|
+
queryTerms: bm25Terms,
|
|
2125
|
+
docText: fulltextText,
|
|
2126
|
+
docTokens: getBm25Tokens(doc, bm25Signature, "fulltext"),
|
|
2127
|
+
docCount: bm25Corpus.length,
|
|
2128
|
+
avgDocLen: bm25StatsFulltext.avgDocLen,
|
|
2129
|
+
docFreq: bm25StatsFulltext.docFreq,
|
|
2130
|
+
})
|
|
2131
|
+
: 0;
|
|
2132
|
+
const summaryCombined = scoreText(plannedQuery, summaryText) + summaryBm25 * readTuning.scoring.bm25Scale;
|
|
2133
|
+
const fulltextCombined = includeFulltext
|
|
2134
|
+
? scoreText(plannedQuery, fulltextText) + fulltextBm25 * readTuning.scoring.bm25Scale
|
|
2135
|
+
: 0;
|
|
2136
|
+
const lexicalCombined = summaryCombined * summaryChannelWeight + fulltextCombined * fulltextChannelWeight;
|
|
2137
|
+
const semantic = plannedQuery === query && queryEmbedding && Array.isArray(doc.embedding) && doc.embedding.length > 0
|
|
2138
|
+
? Math.max(0, cosineSimilarityWithNorm(queryEmbedding, doc.embedding, queryEmbeddingNorm, doc.embeddingNorm || vectorNorm(doc.embedding)) * 5)
|
|
2139
|
+
: 0;
|
|
2140
|
+
if (lexicalCombined <= 0 && semantic <= 0) {
|
|
2141
|
+
return null;
|
|
2142
|
+
}
|
|
2143
|
+
if (source === "graph") {
|
|
2144
|
+
const status = docFactStatus(doc);
|
|
2145
|
+
if (status === "superseded" || status === "rejected") {
|
|
2146
|
+
return null;
|
|
2147
|
+
}
|
|
2148
|
+
}
|
|
2149
|
+
const recency = recencyScore(doc.timestamp, readTuning.recency.buckets);
|
|
2150
|
+
const quality = typeof doc.qualityScore === "number" ? Math.max(0, Math.min(1, doc.qualityScore)) : 0.5;
|
|
2151
|
+
const typeMatch = preferredTypes.length > 0 && doc.eventType
|
|
2152
|
+
? (preferredTypes.includes(doc.eventType) ? 1 : 0)
|
|
2153
|
+
: 0.5;
|
|
2154
|
+
const graphMatch = source === "graph" ? 1 : 0;
|
|
2155
|
+
const sourceBaseWeight = sourceWeight(source, intent);
|
|
2156
|
+
const sourceConfigWeight = customChannelWeight(source, options.fusion);
|
|
2157
|
+
const lengthNorm = lengthNormalizeFactor(doc, options.fusion);
|
|
2158
|
+
const baseWeighted = (readTuning.scoring.lexicalWeight * lexicalCombined +
|
|
2159
|
+
readTuning.scoring.semanticWeight * (semantic * lengthNorm) +
|
|
2160
|
+
readTuning.scoring.recencyWeight * recency +
|
|
2161
|
+
readTuning.scoring.qualityWeight * quality +
|
|
2162
|
+
readTuning.scoring.typeMatchWeight * typeMatch +
|
|
2163
|
+
readTuning.scoring.graphMatchWeight * graphMatch) * sourceBaseWeight * sourceConfigWeight;
|
|
2164
|
+
const decayFactor = computeDecayFactor(doc.id, doc.eventType, doc.timestamp, options.memoryDecay, hitStats);
|
|
2165
|
+
const weighted = baseWeighted * decayFactor;
|
|
2166
|
+
return {
|
|
2167
|
+
doc,
|
|
2168
|
+
source,
|
|
2169
|
+
lexical: lexicalCombined,
|
|
2170
|
+
bm25: summaryBm25 + fulltextBm25,
|
|
2171
|
+
semantic,
|
|
2172
|
+
recency,
|
|
2173
|
+
quality,
|
|
2174
|
+
typeMatch,
|
|
2175
|
+
graphMatch,
|
|
2176
|
+
decayFactor,
|
|
2177
|
+
weighted,
|
|
2178
|
+
summaryCombined,
|
|
2179
|
+
fulltextCombined,
|
|
2180
|
+
};
|
|
2181
|
+
};
|
|
2182
|
+
for (const doc of rulesDocs) {
|
|
2183
|
+
const candidate = evaluateDoc(doc, "rules");
|
|
2184
|
+
if (candidate)
|
|
2185
|
+
channels.rules.push(candidate);
|
|
2186
|
+
}
|
|
2187
|
+
for (const doc of archiveDocs) {
|
|
2188
|
+
const candidate = evaluateDoc(doc, "archive");
|
|
2189
|
+
if (candidate)
|
|
2190
|
+
channels.archive.push(candidate);
|
|
2191
|
+
}
|
|
2192
|
+
for (const doc of vectorDocs) {
|
|
2193
|
+
const candidate = evaluateDoc(doc, "vector");
|
|
2194
|
+
if (candidate)
|
|
2195
|
+
channels.vector.push(candidate);
|
|
2196
|
+
}
|
|
2197
|
+
for (const doc of graphDocs) {
|
|
2198
|
+
const candidate = evaluateDoc(doc, "graph");
|
|
2199
|
+
if (candidate)
|
|
2200
|
+
channels.graph.push(candidate);
|
|
2201
|
+
}
|
|
2202
|
+
const rrfMap = new Map();
|
|
2203
|
+
const weightedMap = new Map();
|
|
2204
|
+
const rrfK = readTuning.rrf.k;
|
|
2205
|
+
for (const key of Object.keys(channels)) {
|
|
2206
|
+
const list = channels[key].sort((a, b) => b.weighted - a.weighted);
|
|
2207
|
+
const capped = list.slice(0, channelQuota(key, args.topK, options.fusion));
|
|
2208
|
+
for (let i = 0; i < capped.length; i += 1) {
|
|
2209
|
+
const candidate = capped[i];
|
|
2210
|
+
const rrf = 1 / (rrfK + i + 1);
|
|
2211
|
+
const mergeKey = mergeKeyFromDoc(candidate.doc);
|
|
2212
|
+
rrfMap.set(mergeKey, (rrfMap.get(mergeKey) || 0) + rrf);
|
|
2213
|
+
const current = weightedMap.get(mergeKey);
|
|
2214
|
+
if (!current || candidate.weighted > current.weighted) {
|
|
2215
|
+
weightedMap.set(mergeKey, candidate);
|
|
2216
|
+
}
|
|
2217
|
+
}
|
|
2218
|
+
}
|
|
2219
|
+
return [...weightedMap.entries()]
|
|
2220
|
+
.map(([mergeKey, candidate]) => {
|
|
2221
|
+
const wikiRefs = docWikiRefs(candidate.doc);
|
|
2222
|
+
return {
|
|
2223
|
+
id: candidate.doc.id,
|
|
2224
|
+
merge_key: mergeKey,
|
|
2225
|
+
source_memory_id: candidate.doc.sourceMemoryId || "",
|
|
2226
|
+
source_memory_canonical_id: candidate.doc.sourceMemoryCanonicalId || "",
|
|
2227
|
+
source_event_id: candidate.doc.sourceEventId || "",
|
|
2228
|
+
source_field: candidate.doc.sourceField || "",
|
|
2229
|
+
text: candidate.doc.summaryText || candidate.doc.text,
|
|
2230
|
+
source_text: candidate.doc.sourceText ? candidate.doc.sourceText.slice(0, 4000) : "",
|
|
2231
|
+
source_excerpt: candidate.doc.sourceText ? candidate.doc.sourceText.slice(0, 360) : "",
|
|
2232
|
+
source_file: candidate.doc.sourceFile || "",
|
|
2233
|
+
source: candidate.doc.source,
|
|
2234
|
+
layer: candidate.doc.layer || "",
|
|
2235
|
+
event_type: candidate.doc.eventType || "",
|
|
2236
|
+
fact_status: docFactStatus(candidate.doc),
|
|
2237
|
+
wiki_ref: wikiRefs[0] || "",
|
|
2238
|
+
wiki_refs: wikiRefs,
|
|
2239
|
+
quality_score: candidate.quality,
|
|
2240
|
+
timestamp: candidate.doc.timestamp ? new Date(candidate.doc.timestamp).toISOString() : "",
|
|
2241
|
+
evidence_ids: docEvidenceIds(candidate.doc),
|
|
2242
|
+
score: candidate.weighted + (rrfMap.get(mergeKey) || 0) * readTuning.rrf.weight,
|
|
2243
|
+
score_breakdown: {
|
|
2244
|
+
lexical: Number(candidate.lexical.toFixed(4)),
|
|
2245
|
+
bm25: Number(candidate.bm25.toFixed(4)),
|
|
2246
|
+
semantic: Number(candidate.semantic.toFixed(4)),
|
|
2247
|
+
recency: Number(candidate.recency.toFixed(4)),
|
|
2248
|
+
quality: Number(candidate.quality.toFixed(4)),
|
|
2249
|
+
type: Number(candidate.typeMatch.toFixed(4)),
|
|
2250
|
+
graph: Number(candidate.graphMatch.toFixed(4)),
|
|
2251
|
+
summary: Number(candidate.summaryCombined.toFixed(4)),
|
|
2252
|
+
fulltext: Number(candidate.fulltextCombined.toFixed(4)),
|
|
2253
|
+
decay: Number(candidate.decayFactor.toFixed(4)),
|
|
2254
|
+
rrf: Number(((rrfMap.get(mergeKey) || 0) * readTuning.rrf.weight).toFixed(4)),
|
|
2255
|
+
weighted: Number(candidate.weighted.toFixed(4)),
|
|
2256
|
+
},
|
|
2257
|
+
reason_tags: [
|
|
2258
|
+
`intent:${intent.toLowerCase()}`,
|
|
2259
|
+
candidate.summaryCombined > 0 ? "summary_hit" : "",
|
|
2260
|
+
candidate.fulltextCombined > 0 ? "fulltext_hit" : "",
|
|
2261
|
+
candidate.semantic > 0 ? "vector_hit" : "",
|
|
2262
|
+
candidate.lexical > 0 ? "lexical_hit" : "",
|
|
2263
|
+
candidate.typeMatch >= 1 ? "event_type_match" : "event_type_weak",
|
|
2264
|
+
candidate.recency >= 0.8 ? "recent" : "historical",
|
|
2265
|
+
candidate.quality >= 0.7 ? "high_quality" : "normal_quality",
|
|
2266
|
+
candidate.decayFactor < 1 ? `decay:${candidate.decayFactor.toFixed(3)}` : "decay:1.000",
|
|
2267
|
+
`source:${candidate.source}`,
|
|
2268
|
+
`merge_key:${mergeKey}`,
|
|
2269
|
+
`query_term:${plannedQuery}`,
|
|
2270
|
+
].filter(Boolean),
|
|
2271
|
+
matched_keywords: [plannedQuery],
|
|
2272
|
+
};
|
|
2273
|
+
})
|
|
2274
|
+
.sort((a, b) => b.score - a.score)
|
|
2275
|
+
.slice(0, maxCandidatePool);
|
|
760
2276
|
};
|
|
761
|
-
const
|
|
762
|
-
|
|
763
|
-
const
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
2277
|
+
const rankingStartedAt = Date.now();
|
|
2278
|
+
const queryRuns = await Promise.all(plannedQueries.map(async (plannedQuery) => {
|
|
2279
|
+
const stage1 = rankByQuery(plannedQuery, false);
|
|
2280
|
+
const needFallback = shouldTriggerFulltextFallback(stage1, args.topK);
|
|
2281
|
+
if (!needFallback || lightweightMode) {
|
|
2282
|
+
return { plannedQuery, ranked: stage1, fulltextFallback: false };
|
|
2283
|
+
}
|
|
2284
|
+
const stage2 = rankByQuery(plannedQuery, true);
|
|
2285
|
+
const merged = new Map();
|
|
2286
|
+
for (const item of [...stage1, ...stage2]) {
|
|
2287
|
+
const mergeKey = item.merge_key || item.id || "";
|
|
2288
|
+
const existing = merged.get(mergeKey);
|
|
2289
|
+
if (!existing) {
|
|
2290
|
+
merged.set(mergeKey, { ...item });
|
|
2291
|
+
continue;
|
|
2292
|
+
}
|
|
2293
|
+
const best = Number(item.score || 0) > Number(existing.score || 0)
|
|
2294
|
+
? { ...existing, ...item }
|
|
2295
|
+
: { ...item, ...existing };
|
|
2296
|
+
const mergedReasonTags = uniqueStrings([
|
|
2297
|
+
...(Array.isArray(existing.reason_tags) ? existing.reason_tags.map(v => String(v)) : []),
|
|
2298
|
+
...(Array.isArray(item.reason_tags) ? item.reason_tags.map(v => String(v)) : []),
|
|
2299
|
+
]);
|
|
2300
|
+
const mergedEvidenceIds = uniqueStrings([
|
|
2301
|
+
...(Array.isArray(existing.evidence_ids) ? existing.evidence_ids.map(v => String(v)) : []),
|
|
2302
|
+
...(Array.isArray(item.evidence_ids) ? item.evidence_ids.map(v => String(v)) : []),
|
|
2303
|
+
]);
|
|
2304
|
+
const mergedWikiRefs = uniqueStrings([
|
|
2305
|
+
...(Array.isArray(existing.wiki_refs) ? existing.wiki_refs.map(v => String(v)) : []),
|
|
2306
|
+
...(Array.isArray(item.wiki_refs) ? item.wiki_refs.map(v => String(v)) : []),
|
|
2307
|
+
existing.wiki_ref || "",
|
|
2308
|
+
item.wiki_ref || "",
|
|
2309
|
+
]);
|
|
2310
|
+
const mergedKeywords = uniqueStrings([
|
|
2311
|
+
...(Array.isArray(existing.matched_keywords) ? existing.matched_keywords.map(v => String(v)) : []),
|
|
2312
|
+
...(Array.isArray(item.matched_keywords) ? item.matched_keywords.map(v => String(v)) : []),
|
|
2313
|
+
]);
|
|
2314
|
+
merged.set(mergeKey, {
|
|
2315
|
+
...best,
|
|
2316
|
+
score: Math.max(existing.score || 0, item.score || 0),
|
|
2317
|
+
reason_tags: mergedReasonTags,
|
|
2318
|
+
evidence_ids: mergedEvidenceIds,
|
|
2319
|
+
wiki_ref: mergedWikiRefs[0] || "",
|
|
2320
|
+
wiki_refs: mergedWikiRefs,
|
|
2321
|
+
matched_keywords: mergedKeywords,
|
|
2322
|
+
});
|
|
768
2323
|
}
|
|
769
|
-
const recency = recencyScore(doc.timestamp);
|
|
770
|
-
const quality = typeof doc.qualityScore === "number" ? Math.max(0, Math.min(1, doc.qualityScore)) : 0.5;
|
|
771
|
-
const typeMatch = preferredTypes.length > 0 && doc.eventType
|
|
772
|
-
? (preferredTypes.includes(doc.eventType) ? 1 : 0)
|
|
773
|
-
: 0.5;
|
|
774
|
-
const graphMatch = source === "graph" ? 1 : 0;
|
|
775
|
-
const baseWeighted = (0.2 * lexical +
|
|
776
|
-
0.3 * semantic +
|
|
777
|
-
0.1 * recency +
|
|
778
|
-
0.15 * quality +
|
|
779
|
-
0.15 * typeMatch +
|
|
780
|
-
0.1 * graphMatch) * sourceWeight(source, intent);
|
|
781
|
-
const decayFactor = computeDecayFactor(doc.id, doc.eventType, doc.timestamp, options.memoryDecay, hitStats);
|
|
782
|
-
const weighted = baseWeighted * decayFactor;
|
|
783
2324
|
return {
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
2325
|
+
plannedQuery,
|
|
2326
|
+
ranked: [...merged.values()]
|
|
2327
|
+
.sort((a, b) => Number(b.score || 0) - Number(a.score || 0))
|
|
2328
|
+
.slice(0, maxCandidatePool)
|
|
2329
|
+
.map((item) => ({
|
|
2330
|
+
...item,
|
|
2331
|
+
reason_tags: uniqueStrings([
|
|
2332
|
+
...(Array.isArray(item.reason_tags) ? item.reason_tags.map((v) => String(v)) : []),
|
|
2333
|
+
"fulltext_fallback",
|
|
2334
|
+
]),
|
|
2335
|
+
})),
|
|
2336
|
+
fulltextFallback: true,
|
|
794
2337
|
};
|
|
795
|
-
};
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
if (candidate)
|
|
809
|
-
channels.vector.push(candidate);
|
|
810
|
-
}
|
|
811
|
-
for (const doc of graphDocs) {
|
|
812
|
-
const candidate = evaluateDoc(doc, "graph");
|
|
813
|
-
if (candidate)
|
|
814
|
-
channels.graph.push(candidate);
|
|
815
|
-
}
|
|
816
|
-
for (const key of Object.keys(channels)) {
|
|
817
|
-
channels[key].sort((a, b) => b.weighted - a.weighted);
|
|
818
|
-
combinedCandidates.push(...channels[key].slice(0, Math.max(20, args.topK * 5)));
|
|
819
|
-
}
|
|
820
|
-
const rrfMap = new Map();
|
|
821
|
-
const weightedMap = new Map();
|
|
822
|
-
const rrfK = 60;
|
|
823
|
-
for (const key of Object.keys(channels)) {
|
|
824
|
-
const list = channels[key];
|
|
825
|
-
for (let i = 0; i < list.length; i += 1) {
|
|
826
|
-
const candidate = list[i];
|
|
827
|
-
const rrf = 1 / (rrfK + i + 1);
|
|
828
|
-
rrfMap.set(candidate.doc.id, (rrfMap.get(candidate.doc.id) || 0) + rrf);
|
|
829
|
-
const current = weightedMap.get(candidate.doc.id);
|
|
830
|
-
if (!current || candidate.weighted > current.weighted) {
|
|
831
|
-
weightedMap.set(candidate.doc.id, candidate);
|
|
2338
|
+
}));
|
|
2339
|
+
const mergedByQuery = new Map();
|
|
2340
|
+
for (const run of queryRuns) {
|
|
2341
|
+
for (const item of run.ranked) {
|
|
2342
|
+
const mergeKey = item.merge_key || item.id || "";
|
|
2343
|
+
const existing = mergedByQuery.get(mergeKey);
|
|
2344
|
+
if (!existing) {
|
|
2345
|
+
mergedByQuery.set(mergeKey, {
|
|
2346
|
+
...item,
|
|
2347
|
+
matched_keywords: uniqueStrings([run.plannedQuery]),
|
|
2348
|
+
fulltext_fallback_used: run.fulltextFallback,
|
|
2349
|
+
});
|
|
2350
|
+
continue;
|
|
832
2351
|
}
|
|
2352
|
+
const mergedReasonTags = uniqueStrings([
|
|
2353
|
+
...(Array.isArray(existing.reason_tags) ? existing.reason_tags.map(v => String(v)) : []),
|
|
2354
|
+
...(Array.isArray(item.reason_tags) ? item.reason_tags.map(v => String(v)) : []),
|
|
2355
|
+
]);
|
|
2356
|
+
const mergedEvidenceIds = uniqueStrings([
|
|
2357
|
+
...(Array.isArray(existing.evidence_ids) ? existing.evidence_ids.map(v => String(v)) : []),
|
|
2358
|
+
...(Array.isArray(item.evidence_ids) ? item.evidence_ids.map(v => String(v)) : []),
|
|
2359
|
+
]);
|
|
2360
|
+
const mergedWikiRefs = uniqueStrings([
|
|
2361
|
+
...(Array.isArray(existing.wiki_refs) ? existing.wiki_refs.map(v => String(v)) : []),
|
|
2362
|
+
...(Array.isArray(item.wiki_refs) ? item.wiki_refs.map(v => String(v)) : []),
|
|
2363
|
+
existing.wiki_ref || "",
|
|
2364
|
+
item.wiki_ref || "",
|
|
2365
|
+
]);
|
|
2366
|
+
const matchedKeywords = uniqueStrings([
|
|
2367
|
+
...(Array.isArray(existing.matched_keywords) ? existing.matched_keywords.map(v => String(v)) : []),
|
|
2368
|
+
run.plannedQuery,
|
|
2369
|
+
]);
|
|
2370
|
+
const preferred = Number(item.score || 0) > Number(existing.score || 0)
|
|
2371
|
+
? { ...existing, ...item }
|
|
2372
|
+
: { ...item, ...existing };
|
|
2373
|
+
mergedByQuery.set(mergeKey, {
|
|
2374
|
+
...preferred,
|
|
2375
|
+
score: Math.max(existing.score || 0, item.score || 0),
|
|
2376
|
+
reason_tags: mergedReasonTags,
|
|
2377
|
+
evidence_ids: mergedEvidenceIds,
|
|
2378
|
+
wiki_ref: mergedWikiRefs[0] || "",
|
|
2379
|
+
wiki_refs: mergedWikiRefs,
|
|
2380
|
+
matched_keywords: matchedKeywords,
|
|
2381
|
+
fulltext_fallback_used: Boolean(existing.fulltext_fallback_used) || run.fulltextFallback,
|
|
2382
|
+
});
|
|
833
2383
|
}
|
|
834
2384
|
}
|
|
835
|
-
const
|
|
836
|
-
.map(
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
.
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
2385
|
+
const lexicalRanked = [...mergedByQuery.values()]
|
|
2386
|
+
.map((item) => {
|
|
2387
|
+
const matchedKeywords = Array.isArray(item.matched_keywords)
|
|
2388
|
+
? uniqueStrings(item.matched_keywords.map(v => String(v)))
|
|
2389
|
+
: [query];
|
|
2390
|
+
const keywordBonus = Math.max(0, matchedKeywords.length - 1) * 0.12;
|
|
2391
|
+
const boosted = withRecencyBoost(Number(item.score || 0) + keywordBonus, typeof item.timestamp === "string" ? Date.parse(item.timestamp) : undefined, readTuning.recency.buckets);
|
|
2392
|
+
return {
|
|
2393
|
+
...item,
|
|
2394
|
+
matched_keywords: matchedKeywords,
|
|
2395
|
+
query_plan_keywords: plannedQueries,
|
|
2396
|
+
score: Number(boosted.toFixed(4)),
|
|
2397
|
+
reason_tags: uniqueStrings([
|
|
2398
|
+
...(Array.isArray(item.reason_tags) ? item.reason_tags.map((v) => String(v)) : []),
|
|
2399
|
+
`keyword_hits:${matchedKeywords.length}`,
|
|
2400
|
+
`query_plan:${plannedQueries.length}`,
|
|
2401
|
+
Boolean(item.fulltext_fallback_used) ? "fulltext_fallback_used" : "",
|
|
2402
|
+
]),
|
|
2403
|
+
};
|
|
2404
|
+
})
|
|
2405
|
+
.sort((a, b) => Number(b.score || 0) - Number(a.score || 0))
|
|
2406
|
+
.slice(0, maxCandidatePool);
|
|
2407
|
+
const fulltextFallbackUsed = queryRuns.some(run => run.fulltextFallback);
|
|
2408
|
+
appendTiming(timing, "ranking_ms", rankingStartedAt);
|
|
2409
|
+
const isVectorSource = (value) => value.startsWith("vector_");
|
|
2410
|
+
const semanticResults = lexicalRanked
|
|
2411
|
+
.filter(item => isVectorSource(item.source) && Array.isArray(item.reason_tags) && item.reason_tags.includes("vector_hit"))
|
|
2412
|
+
.slice(0, Math.max(1, args.topK));
|
|
2413
|
+
const keywordResults = lexicalRanked
|
|
2414
|
+
.filter(item => isVectorSource(item.source) && Array.isArray(item.reason_tags) && item.reason_tags.includes("lexical_hit"))
|
|
2415
|
+
.slice(0, Math.max(1, args.topK));
|
|
2416
|
+
const channelLimit = Math.max(1, args.topK);
|
|
2417
|
+
const channelResults = {
|
|
2418
|
+
vector_semantic: semanticResults,
|
|
2419
|
+
vector_keyword: keywordResults,
|
|
2420
|
+
keyword: lexicalRanked
|
|
2421
|
+
.filter(item => Array.isArray(item.reason_tags) && item.reason_tags.includes("lexical_hit"))
|
|
2422
|
+
.slice(0, channelLimit),
|
|
2423
|
+
semantic: lexicalRanked
|
|
2424
|
+
.filter(item => Array.isArray(item.reason_tags) && item.reason_tags.includes("vector_hit"))
|
|
2425
|
+
.slice(0, channelLimit),
|
|
2426
|
+
archive: lexicalRanked
|
|
2427
|
+
.filter(item => item.source === "sessions_active" || item.source === "sessions_archive")
|
|
2428
|
+
.slice(0, channelLimit),
|
|
2429
|
+
graph: lexicalRanked
|
|
2430
|
+
.filter(item => item.source.startsWith("sessions_graph"))
|
|
2431
|
+
.slice(0, channelLimit),
|
|
2432
|
+
rules: lexicalRanked
|
|
2433
|
+
.filter(item => item.source === "CORTEX_RULES.md")
|
|
2434
|
+
.slice(0, channelLimit),
|
|
2435
|
+
vector: lexicalRanked
|
|
2436
|
+
.filter(item => isVectorSource(item.source))
|
|
2437
|
+
.slice(0, channelLimit),
|
|
2438
|
+
};
|
|
861
2439
|
const rerankerModel = options.reranker?.model || "";
|
|
862
2440
|
const rerankerApiKey = options.reranker?.apiKey || "";
|
|
863
2441
|
const rerankerBaseUrl = normalizeBaseUrl(options.reranker?.baseURL || options.reranker?.baseUrl);
|
|
2442
|
+
const fusionEnabled = externalModelMode && fusionMode !== "off" && options.fusion?.enabled !== false;
|
|
2443
|
+
const llmModel = options.llm?.model || "";
|
|
2444
|
+
const llmApiKey = options.llm?.apiKey || "";
|
|
2445
|
+
const llmBaseUrl = normalizeBaseUrl(options.llm?.baseURL || options.llm?.baseUrl);
|
|
2446
|
+
const fusionAuthoritative = fusionMode === "authoritative"
|
|
2447
|
+
? true
|
|
2448
|
+
: (fusionMode === "candidates" ? false : options.fusion?.authoritative !== false);
|
|
2449
|
+
const skipRerankerForFusion = fusionEnabled && fusionAuthoritative && llmModel && llmApiKey && llmBaseUrl;
|
|
864
2450
|
let rerankedSimple = lexicalRanked.map(item => ({
|
|
865
2451
|
id: item.id,
|
|
2452
|
+
merge_key: item.merge_key,
|
|
866
2453
|
text: item.text,
|
|
867
2454
|
source: item.source,
|
|
868
2455
|
score: item.score,
|
|
869
2456
|
}));
|
|
870
|
-
if (rerankerModel && rerankerApiKey && rerankerBaseUrl && lexicalRanked.length > 1) {
|
|
2457
|
+
if (externalModelMode && rerankerModel && rerankerApiKey && rerankerBaseUrl && lexicalRanked.length > 1 && !skipRerankerForFusion) {
|
|
2458
|
+
const rerankStartedAt = Date.now();
|
|
871
2459
|
try {
|
|
872
2460
|
rerankedSimple = await requestRerank({
|
|
873
2461
|
query,
|
|
@@ -876,29 +2464,179 @@ function createReadStore(options) {
|
|
|
876
2464
|
apiKey: rerankerApiKey,
|
|
877
2465
|
baseUrl: rerankerBaseUrl,
|
|
878
2466
|
});
|
|
2467
|
+
rerankedSimple = rerankedSimple.map(item => {
|
|
2468
|
+
const found = lexicalRanked.find(entry => entry.id === item.id);
|
|
2469
|
+
return { ...item, merge_key: found?.merge_key || item.id };
|
|
2470
|
+
});
|
|
879
2471
|
}
|
|
880
2472
|
catch (error) {
|
|
881
2473
|
options.logger.warn(`Reranker failed, keep hybrid ranking: ${error}`);
|
|
882
2474
|
}
|
|
2475
|
+
finally {
|
|
2476
|
+
appendTiming(timing, "rerank_ms", rerankStartedAt);
|
|
2477
|
+
}
|
|
883
2478
|
}
|
|
884
2479
|
const ranked = rerankedSimple.slice(0, Math.max(1, args.topK)).map(item => {
|
|
885
2480
|
const hit = lexicalRanked.find(entry => entry.id === item.id);
|
|
2481
|
+
const hitWikiRefs = uniqueStrings([
|
|
2482
|
+
...(Array.isArray(hit?.wiki_refs) ? hit.wiki_refs.map(v => String(v)) : []),
|
|
2483
|
+
hit?.wiki_ref || "",
|
|
2484
|
+
]);
|
|
886
2485
|
return {
|
|
887
2486
|
id: item.id,
|
|
2487
|
+
merge_key: hit?.merge_key || item.merge_key || item.id,
|
|
2488
|
+
source_memory_id: hit?.source_memory_id || "",
|
|
2489
|
+
source_memory_canonical_id: hit?.source_memory_canonical_id || "",
|
|
2490
|
+
source_event_id: hit?.source_event_id || "",
|
|
2491
|
+
source_field: hit?.source_field || "",
|
|
2492
|
+
fulltext_event_id: (hit?.source_event_id || hit?.source_memory_id || item.id || ""),
|
|
888
2493
|
text: item.text,
|
|
2494
|
+
source_text: hit?.source_text || "",
|
|
2495
|
+
source_excerpt: hit?.source_excerpt || "",
|
|
2496
|
+
source_file: hit?.source_file || "",
|
|
889
2497
|
source: item.source,
|
|
2498
|
+
layer: hit?.layer || "",
|
|
890
2499
|
event_type: hit?.event_type || "",
|
|
2500
|
+
fact_status: hit?.fact_status || "active",
|
|
2501
|
+
wiki_ref: hitWikiRefs[0] || "",
|
|
2502
|
+
wiki_refs: hitWikiRefs,
|
|
891
2503
|
quality_score: hit?.quality_score ?? 0,
|
|
892
2504
|
timestamp: hit?.timestamp || "",
|
|
2505
|
+
evidence_ids: Array.isArray(hit?.evidence_ids) ? hit?.evidence_ids : [],
|
|
893
2506
|
score: Number(item.score.toFixed(4)),
|
|
2507
|
+
score_breakdown: hit?.score_breakdown || {},
|
|
894
2508
|
reason_tags: Array.isArray(hit?.reason_tags) ? hit?.reason_tags : [],
|
|
2509
|
+
explain: {
|
|
2510
|
+
merge_key: hit?.merge_key || item.merge_key || item.id,
|
|
2511
|
+
source_memory_id: hit?.source_memory_id || "",
|
|
2512
|
+
source_memory_canonical_id: hit?.source_memory_canonical_id || "",
|
|
2513
|
+
source_event_id: hit?.source_event_id || "",
|
|
2514
|
+
source_field: hit?.source_field || "",
|
|
2515
|
+
channel: item.source,
|
|
2516
|
+
source_file: hit?.source_file || "",
|
|
2517
|
+
layer: hit?.layer || "",
|
|
2518
|
+
fact_status: hit?.fact_status || "active",
|
|
2519
|
+
wiki_ref: hitWikiRefs[0] || "",
|
|
2520
|
+
wiki_refs: hitWikiRefs,
|
|
2521
|
+
evidence_ids: Array.isArray(hit?.evidence_ids) ? hit?.evidence_ids : [],
|
|
2522
|
+
score_breakdown: hit?.score_breakdown || {},
|
|
2523
|
+
reason_tags: Array.isArray(hit?.reason_tags) ? hit?.reason_tags : [],
|
|
2524
|
+
},
|
|
895
2525
|
};
|
|
896
2526
|
});
|
|
897
|
-
const
|
|
898
|
-
const
|
|
899
|
-
const
|
|
900
|
-
const
|
|
2527
|
+
const minLexicalHits = Math.max(0, Math.floor(options.fusion?.minLexicalHits ?? 1));
|
|
2528
|
+
const minSemanticHits = Math.max(0, Math.floor(options.fusion?.minSemanticHits ?? 1));
|
|
2529
|
+
const fallbackPool = lexicalRanked.filter(item => !ranked.some(existing => existing.id === item.id));
|
|
2530
|
+
const lexicalCount = ranked.filter(item => item.reason_tags.includes("lexical_hit")).length;
|
|
2531
|
+
const semanticCount = ranked.filter(item => item.reason_tags.includes("vector_hit")).length;
|
|
2532
|
+
if (semanticCount < minSemanticHits) {
|
|
2533
|
+
const needed = minSemanticHits - semanticCount;
|
|
2534
|
+
const supplement = fallbackPool.filter(item => item.reason_tags.includes("vector_hit")).slice(0, needed);
|
|
2535
|
+
for (const item of supplement) {
|
|
2536
|
+
ranked.push({
|
|
2537
|
+
id: item.id,
|
|
2538
|
+
merge_key: item.merge_key,
|
|
2539
|
+
source_memory_id: item.source_memory_id,
|
|
2540
|
+
source_memory_canonical_id: item.source_memory_canonical_id,
|
|
2541
|
+
source_event_id: item.source_event_id || "",
|
|
2542
|
+
source_field: item.source_field || "",
|
|
2543
|
+
fulltext_event_id: item.source_event_id || item.source_memory_id || item.id,
|
|
2544
|
+
text: item.text,
|
|
2545
|
+
source_text: item.source_text || "",
|
|
2546
|
+
source_excerpt: item.source_excerpt || "",
|
|
2547
|
+
source_file: item.source_file || "",
|
|
2548
|
+
source: item.source,
|
|
2549
|
+
layer: item.layer,
|
|
2550
|
+
event_type: item.event_type,
|
|
2551
|
+
fact_status: item.fact_status || "active",
|
|
2552
|
+
wiki_ref: item.wiki_ref || "",
|
|
2553
|
+
wiki_refs: Array.isArray(item.wiki_refs) ? item.wiki_refs : [],
|
|
2554
|
+
quality_score: item.quality_score,
|
|
2555
|
+
timestamp: item.timestamp,
|
|
2556
|
+
evidence_ids: Array.isArray(item.evidence_ids) ? item.evidence_ids : [],
|
|
2557
|
+
score: Number(item.score.toFixed(4)),
|
|
2558
|
+
score_breakdown: item.score_breakdown || {},
|
|
2559
|
+
reason_tags: Array.isArray(item.reason_tags) ? item.reason_tags : [],
|
|
2560
|
+
explain: {
|
|
2561
|
+
merge_key: item.merge_key,
|
|
2562
|
+
source_memory_id: item.source_memory_id,
|
|
2563
|
+
source_memory_canonical_id: item.source_memory_canonical_id,
|
|
2564
|
+
source_event_id: item.source_event_id || "",
|
|
2565
|
+
source_field: item.source_field || "",
|
|
2566
|
+
channel: item.source,
|
|
2567
|
+
source_file: item.source_file || "",
|
|
2568
|
+
layer: item.layer,
|
|
2569
|
+
fact_status: item.fact_status || "active",
|
|
2570
|
+
wiki_ref: item.wiki_ref || "",
|
|
2571
|
+
wiki_refs: Array.isArray(item.wiki_refs) ? item.wiki_refs : [],
|
|
2572
|
+
evidence_ids: Array.isArray(item.evidence_ids) ? item.evidence_ids : [],
|
|
2573
|
+
score_breakdown: item.score_breakdown || {},
|
|
2574
|
+
reason_tags: Array.isArray(item.reason_tags) ? item.reason_tags : [],
|
|
2575
|
+
},
|
|
2576
|
+
});
|
|
2577
|
+
}
|
|
2578
|
+
}
|
|
2579
|
+
if (lexicalCount < minLexicalHits) {
|
|
2580
|
+
const needed = minLexicalHits - lexicalCount;
|
|
2581
|
+
const supplement = fallbackPool.filter(item => item.reason_tags.includes("lexical_hit")).slice(0, needed);
|
|
2582
|
+
for (const item of supplement) {
|
|
2583
|
+
if (ranked.some(existing => existing.id === item.id)) {
|
|
2584
|
+
continue;
|
|
2585
|
+
}
|
|
2586
|
+
ranked.push({
|
|
2587
|
+
id: item.id,
|
|
2588
|
+
merge_key: item.merge_key,
|
|
2589
|
+
source_memory_id: item.source_memory_id,
|
|
2590
|
+
source_memory_canonical_id: item.source_memory_canonical_id,
|
|
2591
|
+
source_event_id: item.source_event_id || "",
|
|
2592
|
+
source_field: item.source_field || "",
|
|
2593
|
+
fulltext_event_id: item.source_event_id || item.source_memory_id || item.id,
|
|
2594
|
+
text: item.text,
|
|
2595
|
+
source_text: item.source_text || "",
|
|
2596
|
+
source_excerpt: item.source_excerpt || "",
|
|
2597
|
+
source_file: item.source_file || "",
|
|
2598
|
+
source: item.source,
|
|
2599
|
+
layer: item.layer,
|
|
2600
|
+
event_type: item.event_type,
|
|
2601
|
+
fact_status: item.fact_status || "active",
|
|
2602
|
+
wiki_ref: item.wiki_ref || "",
|
|
2603
|
+
wiki_refs: Array.isArray(item.wiki_refs) ? item.wiki_refs : [],
|
|
2604
|
+
quality_score: item.quality_score,
|
|
2605
|
+
timestamp: item.timestamp,
|
|
2606
|
+
evidence_ids: Array.isArray(item.evidence_ids) ? item.evidence_ids : [],
|
|
2607
|
+
score: Number(item.score.toFixed(4)),
|
|
2608
|
+
score_breakdown: item.score_breakdown || {},
|
|
2609
|
+
reason_tags: Array.isArray(item.reason_tags) ? item.reason_tags : [],
|
|
2610
|
+
explain: {
|
|
2611
|
+
merge_key: item.merge_key,
|
|
2612
|
+
source_memory_id: item.source_memory_id,
|
|
2613
|
+
source_memory_canonical_id: item.source_memory_canonical_id,
|
|
2614
|
+
source_event_id: item.source_event_id || "",
|
|
2615
|
+
source_field: item.source_field || "",
|
|
2616
|
+
channel: item.source,
|
|
2617
|
+
source_file: item.source_file || "",
|
|
2618
|
+
layer: item.layer,
|
|
2619
|
+
fact_status: item.fact_status || "active",
|
|
2620
|
+
wiki_ref: item.wiki_ref || "",
|
|
2621
|
+
wiki_refs: Array.isArray(item.wiki_refs) ? item.wiki_refs : [],
|
|
2622
|
+
evidence_ids: Array.isArray(item.evidence_ids) ? item.evidence_ids : [],
|
|
2623
|
+
score_breakdown: item.score_breakdown || {},
|
|
2624
|
+
reason_tags: Array.isArray(item.reason_tags) ? item.reason_tags : [],
|
|
2625
|
+
},
|
|
2626
|
+
});
|
|
2627
|
+
}
|
|
2628
|
+
}
|
|
2629
|
+
ranked.sort((a, b) => b.score - a.score);
|
|
2630
|
+
const trackResultHits = (ids) => {
|
|
2631
|
+
if (!trackHits) {
|
|
2632
|
+
return;
|
|
2633
|
+
}
|
|
2634
|
+
const hitStartedAt = Date.now();
|
|
2635
|
+
markHit(ids);
|
|
2636
|
+
appendTiming(timing, "hit_stats_update_ms", hitStartedAt);
|
|
2637
|
+
};
|
|
901
2638
|
if (fusionEnabled && llmModel && llmApiKey && llmBaseUrl && ranked.length > 1) {
|
|
2639
|
+
const fusionStartedAt = Date.now();
|
|
902
2640
|
try {
|
|
903
2641
|
const maxCandidates = Math.max(4, Math.min(20, options.fusion?.maxCandidates ?? 10));
|
|
904
2642
|
const fusion = await requestFusion({
|
|
@@ -906,6 +2644,13 @@ function createReadStore(options) {
|
|
|
906
2644
|
candidates: ranked.slice(0, maxCandidates).map(item => ({
|
|
907
2645
|
id: item.id,
|
|
908
2646
|
text: item.text,
|
|
2647
|
+
source_excerpt: typeof item.source_excerpt === "string" ? item.source_excerpt : "",
|
|
2648
|
+
source_file: typeof item.source_file === "string" ? item.source_file : "",
|
|
2649
|
+
source_memory_id: typeof item.source_memory_id === "string" ? item.source_memory_id : "",
|
|
2650
|
+
source_memory_canonical_id: typeof item.source_memory_canonical_id === "string" ? item.source_memory_canonical_id : "",
|
|
2651
|
+
source_layer: typeof item.layer === "string" ? item.layer : "",
|
|
2652
|
+
source_event_id: typeof item.source_event_id === "string" ? item.source_event_id : "",
|
|
2653
|
+
source_field: item.source_field === "summary" || item.source_field === "evidence" ? item.source_field : "",
|
|
909
2654
|
source: item.source,
|
|
910
2655
|
event_type: item.event_type,
|
|
911
2656
|
quality_score: item.quality_score,
|
|
@@ -920,6 +2665,34 @@ function createReadStore(options) {
|
|
|
920
2665
|
},
|
|
921
2666
|
});
|
|
922
2667
|
if (fusion && fusion.canonical_answer) {
|
|
2668
|
+
if (!Array.isArray(fusion.evidence_ids) || fusion.evidence_ids.length === 0) {
|
|
2669
|
+
throw new Error("fusion_missing_whitelisted_evidence");
|
|
2670
|
+
}
|
|
2671
|
+
const fusedEvidenceIds = uniqueStrings(fusion.evidence_ids.flatMap(item => {
|
|
2672
|
+
const linked = ranked.find(candidate => candidate.id === item);
|
|
2673
|
+
if (!linked)
|
|
2674
|
+
return [item];
|
|
2675
|
+
const linkedEvidence = Array.isArray(linked.evidence_ids) ? linked.evidence_ids : [];
|
|
2676
|
+
return linkedEvidence.length > 0 ? linkedEvidence : [item];
|
|
2677
|
+
}));
|
|
2678
|
+
const wikiRefs = uniqueStrings(fusion.evidence_ids.flatMap(item => {
|
|
2679
|
+
const linked = ranked.find(candidate => candidate.id === item);
|
|
2680
|
+
return uniqueStrings([
|
|
2681
|
+
...(Array.isArray(linked?.wiki_refs) ? linked.wiki_refs.map(v => String(v)) : []),
|
|
2682
|
+
typeof linked?.wiki_ref === "string" ? linked.wiki_ref : "",
|
|
2683
|
+
]);
|
|
2684
|
+
}));
|
|
2685
|
+
const fulltextFetchHints = (Array.isArray(fusion.need_fulltext_event_ids) ? fusion.need_fulltext_event_ids : [])
|
|
2686
|
+
.map(eventId => {
|
|
2687
|
+
const linked = ranked.find(item => item.source_memory_id === eventId ||
|
|
2688
|
+
item.source_memory_canonical_id === eventId ||
|
|
2689
|
+
item.id === eventId);
|
|
2690
|
+
return {
|
|
2691
|
+
event_id: eventId,
|
|
2692
|
+
source_file: linked?.source_file || "",
|
|
2693
|
+
source_excerpt: linked?.source_excerpt || "",
|
|
2694
|
+
};
|
|
2695
|
+
});
|
|
923
2696
|
const fusedItem = {
|
|
924
2697
|
id: `fusion_${Date.now().toString(36)}`,
|
|
925
2698
|
text: fusion.canonical_answer,
|
|
@@ -929,6 +2702,11 @@ function createReadStore(options) {
|
|
|
929
2702
|
timestamp: new Date().toISOString(),
|
|
930
2703
|
score: Number((Math.max(...ranked.map(item => item.score)) + 1).toFixed(4)),
|
|
931
2704
|
reason_tags: ["llm_fused_authoritative", `evidence:${fusion.evidence_ids.length}`],
|
|
2705
|
+
explain: {
|
|
2706
|
+
channel: "llm_fusion",
|
|
2707
|
+
fused_from: ranked.slice(0, maxCandidates).map(item => item.id),
|
|
2708
|
+
reason_tags: ["llm_fused_authoritative", `evidence:${fusion.evidence_ids.length}`],
|
|
2709
|
+
},
|
|
932
2710
|
fused_coverage_note: fusion.coverage_note || "",
|
|
933
2711
|
fused_facts: fusion.facts,
|
|
934
2712
|
fused_timeline: fusion.timeline || [],
|
|
@@ -939,32 +2717,59 @@ function createReadStore(options) {
|
|
|
939
2717
|
fused_risks: fusion.risks || [],
|
|
940
2718
|
fused_action_items: fusion.action_items || [],
|
|
941
2719
|
fused_conflicts: fusion.conflicts,
|
|
942
|
-
|
|
2720
|
+
evidence_ids: fusedEvidenceIds,
|
|
2721
|
+
wiki_refs: wikiRefs,
|
|
2722
|
+
fused_evidence_ids: fusedEvidenceIds,
|
|
2723
|
+
fused_need_fulltext_event_ids: fusion.need_fulltext_event_ids || [],
|
|
2724
|
+
fulltext_fetch_hints: fulltextFetchHints,
|
|
943
2725
|
};
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
return {
|
|
2726
|
+
if (fusionAuthoritative) {
|
|
2727
|
+
trackResultHits(Array.isArray(fusion.evidence_ids) ? fusion.evidence_ids : []);
|
|
2728
|
+
appendTiming(timing, "fusion_ms", fusionStartedAt);
|
|
2729
|
+
return finish({
|
|
2730
|
+
results: [fusedItem],
|
|
2731
|
+
semantic_results: semanticResults,
|
|
2732
|
+
keyword_results: keywordResults,
|
|
2733
|
+
channel_results: channelResults,
|
|
2734
|
+
strategy: "vector_sentence_and_keyword_parallel",
|
|
2735
|
+
}, { planned_queries: plannedQueries, fulltext_fallback_used: fulltextFallbackUsed, vector_source: vectorSource });
|
|
948
2736
|
}
|
|
949
2737
|
const merged = [fusedItem, ...ranked];
|
|
950
|
-
|
|
2738
|
+
trackResultHits([
|
|
951
2739
|
...(Array.isArray(fusion.evidence_ids) ? fusion.evidence_ids : []),
|
|
952
2740
|
...ranked.map(item => item.id),
|
|
953
2741
|
]);
|
|
954
|
-
|
|
2742
|
+
appendTiming(timing, "fusion_ms", fusionStartedAt);
|
|
2743
|
+
return finish({
|
|
2744
|
+
results: merged.slice(0, Math.max(1, args.topK)),
|
|
2745
|
+
semantic_results: semanticResults,
|
|
2746
|
+
keyword_results: keywordResults,
|
|
2747
|
+
channel_results: channelResults,
|
|
2748
|
+
strategy: "vector_sentence_and_keyword_parallel",
|
|
2749
|
+
}, { planned_queries: plannedQueries, fulltext_fallback_used: fulltextFallbackUsed, vector_source: vectorSource });
|
|
955
2750
|
}
|
|
2751
|
+
appendTiming(timing, "fusion_ms", fusionStartedAt);
|
|
956
2752
|
}
|
|
957
2753
|
catch (error) {
|
|
958
2754
|
options.logger.warn(`LLM fusion failed, fallback to reranked results: ${error}`);
|
|
2755
|
+
appendTiming(timing, "fusion_ms", fusionStartedAt);
|
|
959
2756
|
}
|
|
960
2757
|
}
|
|
961
|
-
|
|
962
|
-
|
|
2758
|
+
const finalRanked = ranked.slice(0, Math.max(1, args.topK));
|
|
2759
|
+
trackResultHits(finalRanked.map(item => item.id));
|
|
2760
|
+
return finish({
|
|
2761
|
+
results: finalRanked,
|
|
2762
|
+
semantic_results: semanticResults,
|
|
2763
|
+
keyword_results: keywordResults,
|
|
2764
|
+
channel_results: channelResults,
|
|
2765
|
+
strategy: "vector_sentence_and_keyword_parallel",
|
|
2766
|
+
}, { planned_queries: plannedQueries, fulltext_fallback_used: fulltextFallbackUsed, vector_source: vectorSource });
|
|
963
2767
|
}
|
|
964
2768
|
async function getHotContext(args) {
|
|
965
2769
|
const limit = Math.max(1, args.limit);
|
|
966
2770
|
const docs = loadAllDocuments();
|
|
967
2771
|
const coreRules = docs.find(doc => doc.source === "CORTEX_RULES.md");
|
|
2772
|
+
const ruleBudget = Math.max(1, Math.min(6, Math.floor(limit / 3)));
|
|
968
2773
|
const archiveDocs = docs
|
|
969
2774
|
.filter(doc => doc.source === "sessions_archive")
|
|
970
2775
|
.sort((a, b) => (b.timestamp ?? 0) - (a.timestamp ?? 0))
|
|
@@ -975,7 +2780,14 @@ function createReadStore(options) {
|
|
|
975
2780
|
.slice(0, 2);
|
|
976
2781
|
const result = [];
|
|
977
2782
|
if (coreRules) {
|
|
978
|
-
|
|
2783
|
+
const selectedRules = extractPrioritizedRuleLines(coreRules.text, ruleBudget);
|
|
2784
|
+
if (selectedRules.length > 0) {
|
|
2785
|
+
result.push({
|
|
2786
|
+
id: `${coreRules.id}.hot`,
|
|
2787
|
+
text: `# Hot Rules\n${selectedRules.map((line, index) => `${index + 1}. ${line}`).join("\n")}`,
|
|
2788
|
+
source: coreRules.source,
|
|
2789
|
+
});
|
|
2790
|
+
}
|
|
979
2791
|
}
|
|
980
2792
|
for (const doc of [...issueFixPairs, ...archiveDocs]) {
|
|
981
2793
|
result.push({ id: doc.id, text: doc.text, source: doc.source });
|
|
@@ -992,16 +2804,44 @@ function createReadStore(options) {
|
|
|
992
2804
|
};
|
|
993
2805
|
}
|
|
994
2806
|
if (!result.auto_search) {
|
|
2807
|
+
const maxQueryChars = Math.max(20, readTuning.autoContext.queryMaxChars);
|
|
2808
|
+
const recentMessages = Array.isArray(args.recentMessages) ? args.recentMessages : [];
|
|
2809
|
+
const recentQuery = recentMessages
|
|
2810
|
+
.slice(-8)
|
|
2811
|
+
.map(message => {
|
|
2812
|
+
const role = typeof message.role === "string" && message.role.trim() ? message.role.trim() : "message";
|
|
2813
|
+
const text = typeof message.text === "string"
|
|
2814
|
+
? message.text.trim()
|
|
2815
|
+
: (typeof message.content === "string" ? message.content.trim() : "");
|
|
2816
|
+
return text ? `${role}: ${text.replace(/\s+/g, " ")}` : "";
|
|
2817
|
+
})
|
|
2818
|
+
.filter(Boolean)
|
|
2819
|
+
.join("\n");
|
|
995
2820
|
const docs = loadAllDocuments()
|
|
996
2821
|
.filter(doc => doc.source === "sessions_archive" && doc.sessionId === args.sessionId)
|
|
997
2822
|
.sort((a, b) => (b.timestamp ?? 0) - (a.timestamp ?? 0));
|
|
998
2823
|
const latest = docs[0];
|
|
999
|
-
|
|
1000
|
-
|
|
2824
|
+
const fallbackQuery = latest && latest.text.trim()
|
|
2825
|
+
? latest.text.trim()
|
|
2826
|
+
: "";
|
|
2827
|
+
const sourceQuery = recentQuery || fallbackQuery;
|
|
2828
|
+
if (sourceQuery) {
|
|
2829
|
+
const autoQuery = sourceQuery.length > maxQueryChars
|
|
2830
|
+
? sourceQuery.slice(sourceQuery.length - maxQueryChars)
|
|
2831
|
+
: sourceQuery;
|
|
2832
|
+
const light = await searchMemory({
|
|
2833
|
+
query: autoQuery,
|
|
2834
|
+
topK: 3,
|
|
2835
|
+
mode: readTuning.autoContext.lightweightSearch ? "auto" : "default",
|
|
2836
|
+
fusionMode: "off",
|
|
2837
|
+
trackHits: false,
|
|
2838
|
+
});
|
|
1001
2839
|
result.auto_search = {
|
|
1002
|
-
query:
|
|
2840
|
+
query: autoQuery,
|
|
1003
2841
|
results: light.results,
|
|
1004
2842
|
age_seconds: 0,
|
|
2843
|
+
timing_ms: light.timing_ms,
|
|
2844
|
+
debug: light.debug,
|
|
1005
2845
|
};
|
|
1006
2846
|
}
|
|
1007
2847
|
}
|