openclaw-cortex-memory 0.1.0-Alpha.9 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +347 -290
- package/SIGNATURE.md +7 -0
- package/SKILL.md +96 -345
- package/dist/index.d.ts +69 -23
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1130 -1330
- package/dist/index.js.map +1 -1
- package/dist/openclaw.plugin.json +397 -18
- package/dist/src/dedup/three_stage_deduplicator.d.ts.map +1 -1
- package/dist/src/dedup/three_stage_deduplicator.js +13 -3
- package/dist/src/dedup/three_stage_deduplicator.js.map +1 -1
- package/dist/src/engine/memory_engine.d.ts +5 -1
- package/dist/src/engine/memory_engine.d.ts.map +1 -1
- package/dist/src/engine/ts_engine.d.ts +149 -0
- package/dist/src/engine/ts_engine.d.ts.map +1 -1
- package/dist/src/engine/ts_engine.js +863 -203
- package/dist/src/engine/ts_engine.js.map +1 -1
- package/dist/src/engine/types.d.ts +20 -0
- package/dist/src/engine/types.d.ts.map +1 -1
- package/dist/src/graph/ontology.d.ts +87 -15
- package/dist/src/graph/ontology.d.ts.map +1 -1
- package/dist/src/graph/ontology.js +999 -12
- package/dist/src/graph/ontology.js.map +1 -1
- package/dist/src/net/http_post.d.ts +17 -0
- package/dist/src/net/http_post.d.ts.map +1 -0
- package/dist/src/net/http_post.js +56 -0
- package/dist/src/net/http_post.js.map +1 -0
- package/dist/src/quality/llm_output_validator.d.ts +65 -0
- package/dist/src/quality/llm_output_validator.d.ts.map +1 -0
- package/dist/src/quality/llm_output_validator.js +635 -0
- package/dist/src/quality/llm_output_validator.js.map +1 -0
- package/dist/src/reflect/reflector.d.ts.map +1 -1
- package/dist/src/reflect/reflector.js +296 -26
- package/dist/src/reflect/reflector.js.map +1 -1
- package/dist/src/rules/rule_store.d.ts.map +1 -1
- package/dist/src/rules/rule_store.js +75 -16
- package/dist/src/rules/rule_store.js.map +1 -1
- package/dist/src/session/session_end.d.ts +20 -42
- package/dist/src/session/session_end.d.ts.map +1 -1
- package/dist/src/session/session_end.js +21 -218
- package/dist/src/session/session_end.js.map +1 -1
- package/dist/src/store/archive_store.d.ts +28 -7
- package/dist/src/store/archive_store.d.ts.map +1 -1
- package/dist/src/store/archive_store.js +367 -130
- package/dist/src/store/archive_store.js.map +1 -1
- package/dist/src/store/graph_memory_store.d.ts +115 -0
- package/dist/src/store/graph_memory_store.d.ts.map +1 -0
- package/dist/src/store/graph_memory_store.js +1061 -0
- package/dist/src/store/graph_memory_store.js.map +1 -0
- package/dist/src/store/read_store.d.ts +75 -0
- package/dist/src/store/read_store.d.ts.map +1 -1
- package/dist/src/store/read_store.js +1837 -312
- package/dist/src/store/read_store.js.map +1 -1
- package/dist/src/store/vector_store.d.ts +2 -0
- package/dist/src/store/vector_store.d.ts.map +1 -1
- package/dist/src/store/vector_store.js +19 -3
- package/dist/src/store/vector_store.js.map +1 -1
- package/dist/src/store/write_store.d.ts +11 -0
- package/dist/src/store/write_store.d.ts.map +1 -1
- package/dist/src/store/write_store.js +242 -42
- package/dist/src/store/write_store.js.map +1 -1
- package/dist/src/sync/session_sync.d.ts +72 -1
- package/dist/src/sync/session_sync.d.ts.map +1 -1
- package/dist/src/sync/session_sync.js +2246 -126
- package/dist/src/sync/session_sync.js.map +1 -1
- package/dist/src/wiki/wiki_linter.d.ts +26 -0
- package/dist/src/wiki/wiki_linter.d.ts.map +1 -0
- package/dist/src/wiki/wiki_linter.js +339 -0
- package/dist/src/wiki/wiki_linter.js.map +1 -0
- package/dist/src/wiki/wiki_logger.d.ts +10 -0
- package/dist/src/wiki/wiki_logger.d.ts.map +1 -0
- package/dist/src/wiki/wiki_logger.js +78 -0
- package/dist/src/wiki/wiki_logger.js.map +1 -0
- package/dist/src/wiki/wiki_maintainer.d.ts +39 -0
- package/dist/src/wiki/wiki_maintainer.d.ts.map +1 -0
- package/dist/src/wiki/wiki_maintainer.js +38 -0
- package/dist/src/wiki/wiki_maintainer.js.map +1 -0
- package/dist/src/wiki/wiki_projector.d.ts +35 -0
- package/dist/src/wiki/wiki_projector.d.ts.map +1 -0
- package/dist/src/wiki/wiki_projector.js +1151 -0
- package/dist/src/wiki/wiki_projector.js.map +1 -0
- package/dist/src/wiki/wiki_queue.d.ts +29 -0
- package/dist/src/wiki/wiki_queue.d.ts.map +1 -0
- package/dist/src/wiki/wiki_queue.js +137 -0
- package/dist/src/wiki/wiki_queue.js.map +1 -0
- package/openclaw.plugin.json +397 -18
- package/package.json +51 -5
- package/schema/graph.schema.yaml +330 -0
- package/scripts/cli.js +67 -13
- package/scripts/repair-memory.js +321 -0
- package/skills/cortex-memory/SKILL.md +83 -0
- package/skills/cortex-memory/references/agent-manual.md +127 -0
- package/skills/cortex-memory/references/configuration.md +109 -0
- package/skills/cortex-memory/references/publish-checklist.md +45 -0
- package/skills/cortex-memory/references/system-prompt-template.md +27 -0
- package/skills/cortex-memory/references/tools.md +191 -0
|
@@ -37,6 +37,185 @@ exports.createReadStore = createReadStore;
|
|
|
37
37
|
const fs = __importStar(require("fs"));
|
|
38
38
|
const path = __importStar(require("path"));
|
|
39
39
|
const module_1 = require("module");
|
|
40
|
+
const http_post_1 = require("../net/http_post");
|
|
41
|
+
const EXTERNAL_MODEL_TIMEOUT_MS = 20000;
|
|
42
|
+
function graphRelationKey(relation) {
|
|
43
|
+
const source = (relation.source || "").trim().toLowerCase();
|
|
44
|
+
const type = (relation.type || "related_to").trim().toLowerCase();
|
|
45
|
+
const target = (relation.target || "").trim().toLowerCase();
|
|
46
|
+
return `${source}|${type}|${target}`;
|
|
47
|
+
}
|
|
48
|
+
function buildEntityGraphSummaryDocs(graphDocs) {
|
|
49
|
+
const entityEdges = new Map();
|
|
50
|
+
const entityLatestTs = new Map();
|
|
51
|
+
const entitySession = new Map();
|
|
52
|
+
for (const doc of graphDocs) {
|
|
53
|
+
const ts = typeof doc.timestamp === "number" ? doc.timestamp : 0;
|
|
54
|
+
const sessionId = typeof doc.sessionId === "string" ? doc.sessionId : "";
|
|
55
|
+
const relations = Array.isArray(doc.relations) ? doc.relations : [];
|
|
56
|
+
for (const relation of relations) {
|
|
57
|
+
const source = (relation.source || "").trim();
|
|
58
|
+
const target = (relation.target || "").trim();
|
|
59
|
+
const type = (relation.type || "").trim();
|
|
60
|
+
if (!source || !target || !type)
|
|
61
|
+
continue;
|
|
62
|
+
if (!entityEdges.has(source))
|
|
63
|
+
entityEdges.set(source, []);
|
|
64
|
+
if (!entityEdges.has(target))
|
|
65
|
+
entityEdges.set(target, []);
|
|
66
|
+
entityEdges.get(source)?.push({ source, target, type });
|
|
67
|
+
entityEdges.get(target)?.push({ source, target, type });
|
|
68
|
+
entityLatestTs.set(source, Math.max(entityLatestTs.get(source) || 0, ts));
|
|
69
|
+
entityLatestTs.set(target, Math.max(entityLatestTs.get(target) || 0, ts));
|
|
70
|
+
if (sessionId) {
|
|
71
|
+
if (!entitySession.has(source))
|
|
72
|
+
entitySession.set(source, sessionId);
|
|
73
|
+
if (!entitySession.has(target))
|
|
74
|
+
entitySession.set(target, sessionId);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
const output = [];
|
|
79
|
+
for (const [entity, edges] of entityEdges.entries()) {
|
|
80
|
+
if (!edges.length)
|
|
81
|
+
continue;
|
|
82
|
+
const outgoing = edges.filter(edge => edge.source === entity);
|
|
83
|
+
const incoming = edges.filter(edge => edge.target === entity);
|
|
84
|
+
const typeCounter = new Map();
|
|
85
|
+
for (const edge of edges) {
|
|
86
|
+
typeCounter.set(edge.type, (typeCounter.get(edge.type) || 0) + 1);
|
|
87
|
+
}
|
|
88
|
+
const typeSummary = [...typeCounter.entries()]
|
|
89
|
+
.sort((a, b) => b[1] - a[1])
|
|
90
|
+
.map(([type, count]) => `${type}:${count}`)
|
|
91
|
+
.join(", ");
|
|
92
|
+
const sortedOutgoing = [...outgoing].sort((a, b) => a.type.localeCompare(b.type));
|
|
93
|
+
const sortedIncoming = [...incoming].sort((a, b) => a.type.localeCompare(b.type));
|
|
94
|
+
const cappedOutgoing = sortedOutgoing.slice(0, 20);
|
|
95
|
+
const cappedIncoming = sortedIncoming.slice(0, 20);
|
|
96
|
+
const relationFacts = edges
|
|
97
|
+
.slice(0, 40)
|
|
98
|
+
.map(edge => `${edge.source} ${edge.type} ${edge.target}`)
|
|
99
|
+
.join(" | ");
|
|
100
|
+
const outgoingBlock = cappedOutgoing.length > 0
|
|
101
|
+
? cappedOutgoing.map((edge, index) => `${index + 1}. ${edge.source} -[${edge.type}]-> ${edge.target}`).join("\n")
|
|
102
|
+
: "none";
|
|
103
|
+
const incomingBlock = cappedIncoming.length > 0
|
|
104
|
+
? cappedIncoming.map((edge, index) => `${index + 1}. ${edge.source} -[${edge.type}]-> ${edge.target}`).join("\n")
|
|
105
|
+
: "none";
|
|
106
|
+
const summaryText = [
|
|
107
|
+
`# Graph Entity Summary`,
|
|
108
|
+
`entity: ${entity}`,
|
|
109
|
+
``,
|
|
110
|
+
`## Stats`,
|
|
111
|
+
`relation_total: ${edges.length}`,
|
|
112
|
+
`outgoing_total: ${outgoing.length}`,
|
|
113
|
+
`incoming_total: ${incoming.length}`,
|
|
114
|
+
typeSummary ? `relation_type_distribution: ${typeSummary}` : "relation_type_distribution: none",
|
|
115
|
+
``,
|
|
116
|
+
`## Outgoing Relations`,
|
|
117
|
+
outgoingBlock,
|
|
118
|
+
outgoing.length > cappedOutgoing.length ? `...truncated_outgoing: ${outgoing.length - cappedOutgoing.length}` : "",
|
|
119
|
+
``,
|
|
120
|
+
`## Incoming Relations`,
|
|
121
|
+
incomingBlock,
|
|
122
|
+
incoming.length > cappedIncoming.length ? `...truncated_incoming: ${incoming.length - cappedIncoming.length}` : "",
|
|
123
|
+
``,
|
|
124
|
+
`## Relation Facts`,
|
|
125
|
+
relationFacts || "none",
|
|
126
|
+
].filter(Boolean).join("\n");
|
|
127
|
+
output.push({
|
|
128
|
+
id: `gph_entity_${entity.toLowerCase().replace(/[^a-z0-9\u4e00-\u9fa5]+/gi, "_")}`,
|
|
129
|
+
text: summaryText,
|
|
130
|
+
source: "sessions_graph_entity",
|
|
131
|
+
timestamp: (entityLatestTs.get(entity) || 0) > 0 ? entityLatestTs.get(entity) : undefined,
|
|
132
|
+
layer: "archive",
|
|
133
|
+
sourceMemoryId: entity,
|
|
134
|
+
sessionId: entitySession.get(entity) || undefined,
|
|
135
|
+
entities: [entity],
|
|
136
|
+
relations: edges,
|
|
137
|
+
eventType: "graph_summary",
|
|
138
|
+
qualityScore: 1,
|
|
139
|
+
});
|
|
140
|
+
}
|
|
141
|
+
return output;
|
|
142
|
+
}
|
|
143
|
+
const DEFAULT_READ_TUNING = {
|
|
144
|
+
scoring: {
|
|
145
|
+
lexicalWeight: 0.2,
|
|
146
|
+
bm25Scale: 2,
|
|
147
|
+
semanticWeight: 0.3,
|
|
148
|
+
recencyWeight: 0.1,
|
|
149
|
+
qualityWeight: 0.15,
|
|
150
|
+
typeMatchWeight: 0.15,
|
|
151
|
+
graphMatchWeight: 0.1,
|
|
152
|
+
},
|
|
153
|
+
rrf: {
|
|
154
|
+
k: 60,
|
|
155
|
+
weight: 1.5,
|
|
156
|
+
},
|
|
157
|
+
recency: {
|
|
158
|
+
buckets: [
|
|
159
|
+
{ maxAgeHours: 12, score: 1, bonus: 0.6 },
|
|
160
|
+
{ maxAgeHours: 24, score: 0.8, bonus: 0.6 },
|
|
161
|
+
{ maxAgeHours: 72, score: 0.6, bonus: 0.3 },
|
|
162
|
+
{ maxAgeHours: 168, score: 0.4, bonus: 0.3 },
|
|
163
|
+
{ maxAgeHours: 720, score: 0.2, bonus: 0 },
|
|
164
|
+
{ maxAgeHours: Number.POSITIVE_INFINITY, score: 0.05, bonus: 0 },
|
|
165
|
+
],
|
|
166
|
+
},
|
|
167
|
+
autoContext: {
|
|
168
|
+
queryMaxChars: 240,
|
|
169
|
+
lightweightSearch: true,
|
|
170
|
+
},
|
|
171
|
+
};
|
|
172
|
+
function resolveReadTuning(options) {
|
|
173
|
+
const configuredBuckets = Array.isArray(options?.recency?.buckets)
|
|
174
|
+
? options?.recency?.buckets
|
|
175
|
+
.filter(item => item &&
|
|
176
|
+
Number.isFinite(item.maxAgeHours) &&
|
|
177
|
+
item.maxAgeHours > 0 &&
|
|
178
|
+
Number.isFinite(item.score) &&
|
|
179
|
+
item.score >= 0 &&
|
|
180
|
+
Number.isFinite(item.bonus))
|
|
181
|
+
.map(item => ({
|
|
182
|
+
maxAgeHours: item.maxAgeHours,
|
|
183
|
+
score: Math.max(0, item.score),
|
|
184
|
+
bonus: Math.max(0, item.bonus),
|
|
185
|
+
}))
|
|
186
|
+
: [];
|
|
187
|
+
const sortedBuckets = configuredBuckets
|
|
188
|
+
.sort((a, b) => a.maxAgeHours - b.maxAgeHours);
|
|
189
|
+
const buckets = sortedBuckets.length > 0 ? sortedBuckets : DEFAULT_READ_TUNING.recency.buckets;
|
|
190
|
+
const numberOr = (value, fallback, min) => {
|
|
191
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value < min) {
|
|
192
|
+
return fallback;
|
|
193
|
+
}
|
|
194
|
+
return value;
|
|
195
|
+
};
|
|
196
|
+
return {
|
|
197
|
+
scoring: {
|
|
198
|
+
lexicalWeight: numberOr(options?.scoring?.lexicalWeight, DEFAULT_READ_TUNING.scoring.lexicalWeight, 0),
|
|
199
|
+
bm25Scale: numberOr(options?.scoring?.bm25Scale, DEFAULT_READ_TUNING.scoring.bm25Scale, 0),
|
|
200
|
+
semanticWeight: numberOr(options?.scoring?.semanticWeight, DEFAULT_READ_TUNING.scoring.semanticWeight, 0),
|
|
201
|
+
recencyWeight: numberOr(options?.scoring?.recencyWeight, DEFAULT_READ_TUNING.scoring.recencyWeight, 0),
|
|
202
|
+
qualityWeight: numberOr(options?.scoring?.qualityWeight, DEFAULT_READ_TUNING.scoring.qualityWeight, 0),
|
|
203
|
+
typeMatchWeight: numberOr(options?.scoring?.typeMatchWeight, DEFAULT_READ_TUNING.scoring.typeMatchWeight, 0),
|
|
204
|
+
graphMatchWeight: numberOr(options?.scoring?.graphMatchWeight, DEFAULT_READ_TUNING.scoring.graphMatchWeight, 0),
|
|
205
|
+
},
|
|
206
|
+
rrf: {
|
|
207
|
+
k: Math.floor(numberOr(options?.rrf?.k, DEFAULT_READ_TUNING.rrf.k, 1)),
|
|
208
|
+
weight: numberOr(options?.rrf?.weight, DEFAULT_READ_TUNING.rrf.weight, 0),
|
|
209
|
+
},
|
|
210
|
+
recency: {
|
|
211
|
+
buckets,
|
|
212
|
+
},
|
|
213
|
+
autoContext: {
|
|
214
|
+
queryMaxChars: Math.floor(numberOr(options?.autoContext?.queryMaxChars, DEFAULT_READ_TUNING.autoContext.queryMaxChars, 20)),
|
|
215
|
+
lightweightSearch: options?.autoContext?.lightweightSearch !== false,
|
|
216
|
+
},
|
|
217
|
+
};
|
|
218
|
+
}
|
|
40
219
|
function safeReadFile(filePath) {
|
|
41
220
|
try {
|
|
42
221
|
if (!fs.existsSync(filePath)) {
|
|
@@ -58,20 +237,41 @@ function scoreText(query, text) {
|
|
|
58
237
|
if (t.includes(q)) {
|
|
59
238
|
score += 5;
|
|
60
239
|
}
|
|
61
|
-
const tokens =
|
|
240
|
+
const tokens = [...new Set(tokenize(q))];
|
|
62
241
|
for (const token of tokens) {
|
|
63
242
|
if (t.includes(token)) {
|
|
64
|
-
score += 1;
|
|
243
|
+
score += /[\u4e00-\u9fa5]/.test(token) && token.length <= 4 ? 0.65 : 1;
|
|
65
244
|
}
|
|
66
245
|
}
|
|
67
246
|
return score;
|
|
68
247
|
}
|
|
248
|
+
function buildCjkNgrams(token) {
|
|
249
|
+
if (!/[\u4e00-\u9fa5]/.test(token)) {
|
|
250
|
+
return [];
|
|
251
|
+
}
|
|
252
|
+
const chars = [...token];
|
|
253
|
+
const grams = [];
|
|
254
|
+
for (const size of [2, 3, 4]) {
|
|
255
|
+
if (chars.length < size)
|
|
256
|
+
continue;
|
|
257
|
+
for (let index = 0; index <= chars.length - size; index += 1) {
|
|
258
|
+
grams.push(chars.slice(index, index + size).join(""));
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
return grams;
|
|
262
|
+
}
|
|
69
263
|
function tokenize(text) {
|
|
70
|
-
|
|
264
|
+
const rawTokens = text
|
|
71
265
|
.toLowerCase()
|
|
72
266
|
.split(/[^a-z0-9\u4e00-\u9fa5]+/i)
|
|
73
267
|
.map(token => token.trim())
|
|
74
268
|
.filter(Boolean);
|
|
269
|
+
const tokens = [];
|
|
270
|
+
for (const token of rawTokens) {
|
|
271
|
+
tokens.push(token);
|
|
272
|
+
tokens.push(...buildCjkNgrams(token));
|
|
273
|
+
}
|
|
274
|
+
return tokens;
|
|
75
275
|
}
|
|
76
276
|
function buildBm25Stats(docs, queryTerms, getTokens) {
|
|
77
277
|
const docFreq = new Map();
|
|
@@ -83,7 +283,8 @@ function buildBm25Stats(docs, queryTerms, getTokens) {
|
|
|
83
283
|
continue;
|
|
84
284
|
}
|
|
85
285
|
const termSet = new Set(tokens);
|
|
86
|
-
|
|
286
|
+
const queryTermSet = new Set(queryTerms);
|
|
287
|
+
for (const term of queryTermSet) {
|
|
87
288
|
if (termSet.has(term)) {
|
|
88
289
|
docFreq.set(term, (docFreq.get(term) || 0) + 1);
|
|
89
290
|
}
|
|
@@ -104,7 +305,7 @@ function bm25Score(args) {
|
|
|
104
305
|
const k1 = 1.2;
|
|
105
306
|
const b = 0.75;
|
|
106
307
|
let score = 0;
|
|
107
|
-
for (const term of args.queryTerms) {
|
|
308
|
+
for (const term of uniqueStrings(args.queryTerms)) {
|
|
108
309
|
const tf = termFreq.get(term) || 0;
|
|
109
310
|
if (tf <= 0)
|
|
110
311
|
continue;
|
|
@@ -116,6 +317,14 @@ function bm25Score(args) {
|
|
|
116
317
|
return score;
|
|
117
318
|
}
|
|
118
319
|
function normalizeRecordText(record) {
|
|
320
|
+
const summary = typeof record.summary === "string" ? record.summary.trim() : "";
|
|
321
|
+
const sourceText = typeof record.source_text === "string" ? record.source_text.trim() : "";
|
|
322
|
+
if (summary && sourceText) {
|
|
323
|
+
return [
|
|
324
|
+
`summary: ${summary}`,
|
|
325
|
+
`source_text: ${sourceText}`,
|
|
326
|
+
].join("\n");
|
|
327
|
+
}
|
|
119
328
|
const direct = [record.content, record.summary, record.text, record.message]
|
|
120
329
|
.find(v => typeof v === "string" && v.trim());
|
|
121
330
|
if (direct) {
|
|
@@ -158,33 +367,31 @@ function parseJsonlFile(filePath, sourceLabel, logger) {
|
|
|
158
367
|
}
|
|
159
368
|
try {
|
|
160
369
|
const parsed = JSON.parse(trimmed);
|
|
161
|
-
const
|
|
370
|
+
const summaryText = typeof parsed.summary === "string" ? parsed.summary.trim() : "";
|
|
371
|
+
const causeText = typeof parsed.cause === "string" ? parsed.cause.trim() : "";
|
|
372
|
+
const processText = typeof parsed.process === "string" ? parsed.process.trim() : "";
|
|
373
|
+
const resultText = typeof parsed.result === "string" ? parsed.result.trim() : "";
|
|
374
|
+
const sourceText = typeof parsed.source_text === "string" ? parsed.source_text.trim() : "";
|
|
375
|
+
const activeContent = typeof parsed.content === "string" ? parsed.content.trim() : "";
|
|
376
|
+
const effectiveSummary = summaryText;
|
|
377
|
+
const text = [
|
|
378
|
+
effectiveSummary,
|
|
379
|
+
causeText ? `cause: ${causeText}` : "",
|
|
380
|
+
processText ? `process: ${processText}` : "",
|
|
381
|
+
resultText ? `result: ${resultText}` : "",
|
|
382
|
+
].filter(Boolean).join("\n") || normalizeRecordText(parsed);
|
|
162
383
|
if (!text.trim()) {
|
|
163
384
|
continue;
|
|
164
385
|
}
|
|
165
386
|
const id = typeof parsed.id === "string" ? parsed.id : `${sourceLabel}:${docs.length + 1}`;
|
|
166
387
|
const timestampValue = typeof parsed.timestamp === "string" ? Date.parse(parsed.timestamp) : NaN;
|
|
167
|
-
const entities = Array.isArray(parsed.entities)
|
|
168
|
-
? parsed.entities.map(item => (typeof item === "string" ? item.trim() : "")).filter(Boolean)
|
|
169
|
-
: [];
|
|
170
|
-
const relations = Array.isArray(parsed.relations)
|
|
171
|
-
? parsed.relations
|
|
172
|
-
.map(item => {
|
|
173
|
-
if (typeof item !== "object" || item === null)
|
|
174
|
-
return null;
|
|
175
|
-
const relation = item;
|
|
176
|
-
const source = typeof relation.source === "string" ? relation.source.trim() : "";
|
|
177
|
-
const target = typeof relation.target === "string" ? relation.target.trim() : "";
|
|
178
|
-
const type = typeof relation.type === "string" ? relation.type.trim() : "related_to";
|
|
179
|
-
if (!source || !target)
|
|
180
|
-
return null;
|
|
181
|
-
return { source, target, type };
|
|
182
|
-
})
|
|
183
|
-
.filter((item) => Boolean(item))
|
|
184
|
-
: [];
|
|
185
388
|
docs.push({
|
|
186
389
|
id,
|
|
187
390
|
text,
|
|
391
|
+
summaryText: effectiveSummary || undefined,
|
|
392
|
+
sourceText: sourceText || activeContent || undefined,
|
|
393
|
+
sourceEventId: typeof parsed.source_event_id === "string" ? parsed.source_event_id : undefined,
|
|
394
|
+
sourceFile: typeof parsed.source_file === "string" ? parsed.source_file : undefined,
|
|
188
395
|
source: sourceLabel,
|
|
189
396
|
timestamp: Number.isFinite(timestampValue) ? timestampValue : undefined,
|
|
190
397
|
layer: parsed.layer === "active" || parsed.layer === "archive"
|
|
@@ -202,8 +409,8 @@ function parseJsonlFile(filePath, sourceLabel, logger) {
|
|
|
202
409
|
charCount: typeof parsed.char_count === "number" ? parsed.char_count : undefined,
|
|
203
410
|
tokenCount: typeof parsed.token_count === "number" ? parsed.token_count : undefined,
|
|
204
411
|
sessionId: typeof parsed.session_id === "string" ? parsed.session_id : undefined,
|
|
205
|
-
entities,
|
|
206
|
-
relations,
|
|
412
|
+
entities: [],
|
|
413
|
+
relations: [],
|
|
207
414
|
});
|
|
208
415
|
}
|
|
209
416
|
catch (error) {
|
|
@@ -232,35 +439,193 @@ function parseMarkdownFile(filePath, sourceLabel) {
|
|
|
232
439
|
},
|
|
233
440
|
];
|
|
234
441
|
}
|
|
235
|
-
function
|
|
442
|
+
function normalizeFactStatus(value) {
|
|
443
|
+
const token = (value || "").trim().toLowerCase();
|
|
444
|
+
if (!token)
|
|
445
|
+
return null;
|
|
446
|
+
if (token === "active")
|
|
447
|
+
return "active";
|
|
448
|
+
if (token === "pending" || token === "pending_conflict")
|
|
449
|
+
return "pending_conflict";
|
|
450
|
+
if (token === "superseded")
|
|
451
|
+
return "superseded";
|
|
452
|
+
if (token === "rejected")
|
|
453
|
+
return "rejected";
|
|
454
|
+
return null;
|
|
455
|
+
}
|
|
456
|
+
function uniqueStrings(values) {
|
|
457
|
+
const output = [];
|
|
458
|
+
const seen = new Set();
|
|
459
|
+
for (const value of values) {
|
|
460
|
+
const key = (value || "").trim();
|
|
461
|
+
if (!key || seen.has(key))
|
|
462
|
+
continue;
|
|
463
|
+
seen.add(key);
|
|
464
|
+
output.push(key);
|
|
465
|
+
}
|
|
466
|
+
return output;
|
|
467
|
+
}
|
|
468
|
+
function normalizeWikiDisplayRef(value) {
|
|
469
|
+
const normalized = (value || "").trim().replace(/\\/g, "/").replace(/^\/+/, "");
|
|
470
|
+
if (!normalized)
|
|
471
|
+
return "";
|
|
472
|
+
return normalized.startsWith("wiki/") ? normalized : `wiki/${normalized}`;
|
|
473
|
+
}
|
|
474
|
+
function normalizeWikiEvidenceRef(value) {
|
|
475
|
+
return normalizeWikiDisplayRef(value).replace(/^wiki\//, "");
|
|
476
|
+
}
|
|
477
|
+
function readWikiProjectionRefIndex(projectionIndexPath, logger) {
|
|
478
|
+
const index = {
|
|
479
|
+
entityByName: new Map(),
|
|
480
|
+
topicByType: new Map(),
|
|
481
|
+
};
|
|
482
|
+
const content = safeReadFile(projectionIndexPath).trim();
|
|
483
|
+
if (!content) {
|
|
484
|
+
return index;
|
|
485
|
+
}
|
|
486
|
+
try {
|
|
487
|
+
const parsed = JSON.parse(content);
|
|
488
|
+
const entities = Array.isArray(parsed.entities) ? parsed.entities : [];
|
|
489
|
+
const topics = Array.isArray(parsed.topics) ? parsed.topics : [];
|
|
490
|
+
for (const item of entities) {
|
|
491
|
+
if (!item || typeof item !== "object")
|
|
492
|
+
continue;
|
|
493
|
+
const row = item;
|
|
494
|
+
const name = typeof row.name === "string" ? row.name.trim() : "";
|
|
495
|
+
const wikiPath = typeof row.path === "string" ? normalizeWikiDisplayRef(row.path) : "";
|
|
496
|
+
if (!name || !wikiPath)
|
|
497
|
+
continue;
|
|
498
|
+
index.entityByName.set(name.toLowerCase(), wikiPath);
|
|
499
|
+
}
|
|
500
|
+
for (const item of topics) {
|
|
501
|
+
if (!item || typeof item !== "object")
|
|
502
|
+
continue;
|
|
503
|
+
const row = item;
|
|
504
|
+
const type = typeof row.type === "string" ? row.type.trim() : "";
|
|
505
|
+
const wikiPath = typeof row.path === "string" ? normalizeWikiDisplayRef(row.path) : "";
|
|
506
|
+
if (!type || !wikiPath)
|
|
507
|
+
continue;
|
|
508
|
+
index.topicByType.set(type.toLowerCase(), wikiPath);
|
|
509
|
+
}
|
|
510
|
+
}
|
|
511
|
+
catch (error) {
|
|
512
|
+
logger.debug(`Skipping invalid wiki projection index ${projectionIndexPath}: ${error}`);
|
|
513
|
+
}
|
|
514
|
+
return index;
|
|
515
|
+
}
|
|
516
|
+
function wikiRefsForGraphRelation(index, relation) {
|
|
517
|
+
return uniqueStrings([
|
|
518
|
+
index.entityByName.get(relation.source.trim().toLowerCase()) || "",
|
|
519
|
+
index.entityByName.get(relation.target.trim().toLowerCase()) || "",
|
|
520
|
+
index.topicByType.get(relation.type.trim().toLowerCase()) || "",
|
|
521
|
+
]);
|
|
522
|
+
}
|
|
523
|
+
function docWikiRefs(doc) {
|
|
524
|
+
return uniqueStrings([
|
|
525
|
+
...(Array.isArray(doc.wikiRefs) ? doc.wikiRefs : []),
|
|
526
|
+
doc.wikiRef || "",
|
|
527
|
+
]);
|
|
528
|
+
}
|
|
529
|
+
function buildGraphEvidenceIds(args) {
|
|
530
|
+
const relationKey = graphRelationKey({
|
|
531
|
+
source: args.source,
|
|
532
|
+
target: args.target,
|
|
533
|
+
type: args.type,
|
|
534
|
+
});
|
|
535
|
+
const wikiRefs = uniqueStrings([
|
|
536
|
+
...(Array.isArray(args.wikiRefs) ? args.wikiRefs : []),
|
|
537
|
+
args.wikiRef || "",
|
|
538
|
+
]);
|
|
539
|
+
const evidenceIds = [
|
|
540
|
+
relationKey ? `graph:relation:${relationKey}` : "",
|
|
541
|
+
args.sourceEventId ? `graph:event:${args.sourceEventId}` : "",
|
|
542
|
+
args.evidenceSpan
|
|
543
|
+
? `graph:evidence:${args.sourceEventId || relationKey}`
|
|
544
|
+
: "",
|
|
545
|
+
...wikiRefs.map(ref => {
|
|
546
|
+
const wikiRef = normalizeWikiEvidenceRef(ref);
|
|
547
|
+
return wikiRef ? `wiki:${wikiRef}${args.wikiAnchor ? `#${args.wikiAnchor}` : ""}` : "";
|
|
548
|
+
}),
|
|
549
|
+
];
|
|
550
|
+
return uniqueStrings(evidenceIds);
|
|
551
|
+
}
|
|
552
|
+
function extractPrioritizedRuleLines(text, maxRules) {
|
|
553
|
+
if (!text.trim() || maxRules <= 0) {
|
|
554
|
+
return [];
|
|
555
|
+
}
|
|
556
|
+
const lines = text
|
|
557
|
+
.split(/\r?\n/)
|
|
558
|
+
.map(line => line.trim())
|
|
559
|
+
.filter(Boolean)
|
|
560
|
+
.filter(line => !/^core rules and knowledge extracted/i.test(line))
|
|
561
|
+
.filter(line => !/^core rules\b/i.test(line));
|
|
562
|
+
if (lines.length === 0) {
|
|
563
|
+
return [];
|
|
564
|
+
}
|
|
565
|
+
const dedupedFromTail = [];
|
|
566
|
+
const seen = new Set();
|
|
567
|
+
for (let i = lines.length - 1; i >= 0; i -= 1) {
|
|
568
|
+
const line = lines[i];
|
|
569
|
+
const key = line.toLowerCase().replace(/\s+/g, " ").trim();
|
|
570
|
+
if (!key || seen.has(key)) {
|
|
571
|
+
continue;
|
|
572
|
+
}
|
|
573
|
+
seen.add(key);
|
|
574
|
+
dedupedFromTail.push(line);
|
|
575
|
+
}
|
|
576
|
+
const scored = dedupedFromTail.map((line, indexFromTail) => {
|
|
577
|
+
let score = 0;
|
|
578
|
+
if (/(must|should|ensure|avoid|prefer|always|never|fallback|verify|validate|retry|sanitize)/i.test(line)) {
|
|
579
|
+
score += 3;
|
|
580
|
+
}
|
|
581
|
+
if (/(fix|resolved|success|stable|deploy|release|incident|rollback|constraint|decision)/i.test(line)) {
|
|
582
|
+
score += 2;
|
|
583
|
+
}
|
|
584
|
+
if (/(确保|避免|优先|必须|建议|回退|重试|校验|稳定|发布|决策)/.test(line)) {
|
|
585
|
+
score += 2;
|
|
586
|
+
}
|
|
587
|
+
if (line.length >= 30 && line.length <= 220) {
|
|
588
|
+
score += 2;
|
|
589
|
+
}
|
|
590
|
+
else if (line.length > 220) {
|
|
591
|
+
score -= 1;
|
|
592
|
+
}
|
|
593
|
+
if (/[.!?]$/.test(line)) {
|
|
594
|
+
score += 1;
|
|
595
|
+
}
|
|
596
|
+
score += Math.max(0, 2 - indexFromTail * 0.08);
|
|
597
|
+
return { line, score, indexFromTail };
|
|
598
|
+
});
|
|
599
|
+
const selected = scored
|
|
600
|
+
.sort((a, b) => (b.score - a.score) || (a.indexFromTail - b.indexFromTail))
|
|
601
|
+
.slice(0, maxRules)
|
|
602
|
+
.sort((a, b) => a.indexFromTail - b.indexFromTail)
|
|
603
|
+
.map(item => item.line);
|
|
604
|
+
return selected;
|
|
605
|
+
}
|
|
606
|
+
function withRecencyBoost(score, timestamp, buckets) {
|
|
236
607
|
if (!timestamp) {
|
|
237
608
|
return score;
|
|
238
609
|
}
|
|
239
610
|
const ageHours = (Date.now() - timestamp) / (1000 * 60 * 60);
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
return score + 0.3;
|
|
611
|
+
for (const bucket of buckets) {
|
|
612
|
+
if (ageHours <= bucket.maxAgeHours) {
|
|
613
|
+
return score + bucket.bonus;
|
|
614
|
+
}
|
|
245
615
|
}
|
|
246
616
|
return score;
|
|
247
617
|
}
|
|
248
|
-
function recencyScore(timestamp) {
|
|
618
|
+
function recencyScore(timestamp, buckets) {
|
|
249
619
|
if (!timestamp) {
|
|
250
620
|
return 0;
|
|
251
621
|
}
|
|
252
622
|
const ageHours = (Date.now() - timestamp) / (1000 * 60 * 60);
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
if (ageHours < 168)
|
|
260
|
-
return 0.4;
|
|
261
|
-
if (ageHours < 720)
|
|
262
|
-
return 0.2;
|
|
263
|
-
return 0.05;
|
|
623
|
+
for (const bucket of buckets) {
|
|
624
|
+
if (ageHours <= bucket.maxAgeHours) {
|
|
625
|
+
return bucket.score;
|
|
626
|
+
}
|
|
627
|
+
}
|
|
628
|
+
return 0;
|
|
264
629
|
}
|
|
265
630
|
function eventTypeHalfLifeDays(eventType, options) {
|
|
266
631
|
const fallback = typeof options?.defaultHalfLifeDays === "number" && options.defaultHalfLifeDays > 0
|
|
@@ -322,30 +687,46 @@ function normalizeBaseUrl(value) {
|
|
|
322
687
|
return "";
|
|
323
688
|
return value.endsWith("/") ? value.slice(0, -1) : value;
|
|
324
689
|
}
|
|
325
|
-
|
|
690
|
+
function emptyChannelResults() {
|
|
691
|
+
return {
|
|
692
|
+
vector_semantic: [],
|
|
693
|
+
vector_keyword: [],
|
|
694
|
+
keyword: [],
|
|
695
|
+
semantic: [],
|
|
696
|
+
archive: [],
|
|
697
|
+
graph: [],
|
|
698
|
+
rules: [],
|
|
699
|
+
vector: [],
|
|
700
|
+
};
|
|
701
|
+
}
|
|
702
|
+
const READ_FUSION_PROMPT_VERSION = "read-fusion.v1.2.0";
|
|
326
703
|
const READ_FUSION_REGRESSION_SAMPLES = [
|
|
327
|
-
"
|
|
328
|
-
"
|
|
704
|
+
"Example A: if archive and vector refer to the same source_memory_id, keep one main conclusion and keep the rest as supporting evidence.",
|
|
705
|
+
"Example B: if conclusions conflict, write conflicts and explain prioritization in canonical_answer (time, quality, explicitness).",
|
|
706
|
+
"Example C: if summary/excerpt is insufficient, return event ids in need_fulltext_event_ids for full-text lookup.",
|
|
329
707
|
];
|
|
708
|
+
function vectorNorm(values) {
|
|
709
|
+
let sum = 0;
|
|
710
|
+
for (const value of values) {
|
|
711
|
+
sum += value * value;
|
|
712
|
+
}
|
|
713
|
+
return Math.sqrt(sum);
|
|
714
|
+
}
|
|
330
715
|
function cosineSimilarity(left, right) {
|
|
331
|
-
|
|
716
|
+
return cosineSimilarityWithNorm(left, right, vectorNorm(left), vectorNorm(right));
|
|
717
|
+
}
|
|
718
|
+
function cosineSimilarityWithNorm(left, right, leftNorm, rightNorm) {
|
|
719
|
+
if (left.length === 0 || right.length === 0 || leftNorm === 0 || rightNorm === 0) {
|
|
332
720
|
return 0;
|
|
333
721
|
}
|
|
334
722
|
const size = Math.min(left.length, right.length);
|
|
335
723
|
let dot = 0;
|
|
336
|
-
let leftNorm = 0;
|
|
337
|
-
let rightNorm = 0;
|
|
338
724
|
for (let i = 0; i < size; i += 1) {
|
|
339
725
|
const a = left[i];
|
|
340
726
|
const b = right[i];
|
|
341
727
|
dot += a * b;
|
|
342
|
-
leftNorm += a * a;
|
|
343
|
-
rightNorm += b * b;
|
|
344
|
-
}
|
|
345
|
-
if (leftNorm === 0 || rightNorm === 0) {
|
|
346
|
-
return 0;
|
|
347
728
|
}
|
|
348
|
-
return dot / (
|
|
729
|
+
return dot / (leftNorm * rightNorm);
|
|
349
730
|
}
|
|
350
731
|
async function requestEmbedding(args) {
|
|
351
732
|
const endpoint = args.baseUrl.endsWith("/embeddings") ? args.baseUrl : `${args.baseUrl}/embeddings`;
|
|
@@ -358,24 +739,18 @@ async function requestEmbedding(args) {
|
|
|
358
739
|
}
|
|
359
740
|
let lastError = null;
|
|
360
741
|
for (let attempt = 0; attempt < 3; attempt += 1) {
|
|
361
|
-
const
|
|
362
|
-
|
|
742
|
+
const response = await (0, http_post_1.postJsonWithTimeout)({
|
|
743
|
+
endpoint,
|
|
744
|
+
apiKey: args.apiKey,
|
|
745
|
+
body,
|
|
746
|
+
timeoutMs: EXTERNAL_MODEL_TIMEOUT_MS,
|
|
747
|
+
});
|
|
748
|
+
if (!response.ok) {
|
|
749
|
+
lastError = new Error(response.status > 0 ? `embedding_http_${response.status}` : (response.error || "embedding_network_error"));
|
|
750
|
+
continue;
|
|
751
|
+
}
|
|
363
752
|
try {
|
|
364
|
-
const
|
|
365
|
-
method: "POST",
|
|
366
|
-
headers: {
|
|
367
|
-
"content-type": "application/json",
|
|
368
|
-
authorization: `Bearer ${args.apiKey}`,
|
|
369
|
-
},
|
|
370
|
-
body: JSON.stringify(body),
|
|
371
|
-
signal: controller.signal,
|
|
372
|
-
});
|
|
373
|
-
clearTimeout(timeoutId);
|
|
374
|
-
if (!response.ok) {
|
|
375
|
-
lastError = new Error(`embedding_http_${response.status}`);
|
|
376
|
-
continue;
|
|
377
|
-
}
|
|
378
|
-
const json = await response.json();
|
|
753
|
+
const json = (response.json || {});
|
|
379
754
|
const embedding = json?.data?.[0]?.embedding;
|
|
380
755
|
if (Array.isArray(embedding) && embedding.length > 0) {
|
|
381
756
|
return embedding.filter(item => Number.isFinite(item));
|
|
@@ -383,7 +758,6 @@ async function requestEmbedding(args) {
|
|
|
383
758
|
lastError = new Error("embedding_empty");
|
|
384
759
|
}
|
|
385
760
|
catch (error) {
|
|
386
|
-
clearTimeout(timeoutId);
|
|
387
761
|
lastError = error;
|
|
388
762
|
}
|
|
389
763
|
}
|
|
@@ -403,24 +777,18 @@ async function requestRerank(args) {
|
|
|
403
777
|
};
|
|
404
778
|
let lastError = null;
|
|
405
779
|
for (let attempt = 0; attempt < 2; attempt += 1) {
|
|
406
|
-
const
|
|
407
|
-
|
|
780
|
+
const response = await (0, http_post_1.postJsonWithTimeout)({
|
|
781
|
+
endpoint,
|
|
782
|
+
apiKey: args.apiKey,
|
|
783
|
+
body,
|
|
784
|
+
timeoutMs: EXTERNAL_MODEL_TIMEOUT_MS,
|
|
785
|
+
});
|
|
786
|
+
if (!response.ok) {
|
|
787
|
+
lastError = new Error(response.status > 0 ? `rerank_http_${response.status}` : (response.error || "rerank_network_error"));
|
|
788
|
+
continue;
|
|
789
|
+
}
|
|
408
790
|
try {
|
|
409
|
-
const
|
|
410
|
-
method: "POST",
|
|
411
|
-
headers: {
|
|
412
|
-
"content-type": "application/json",
|
|
413
|
-
authorization: `Bearer ${args.apiKey}`,
|
|
414
|
-
},
|
|
415
|
-
body: JSON.stringify(body),
|
|
416
|
-
signal: controller.signal,
|
|
417
|
-
});
|
|
418
|
-
clearTimeout(timeoutId);
|
|
419
|
-
if (!response.ok) {
|
|
420
|
-
lastError = new Error(`rerank_http_${response.status}`);
|
|
421
|
-
continue;
|
|
422
|
-
}
|
|
423
|
-
const json = await response.json();
|
|
791
|
+
const json = (response.json || {});
|
|
424
792
|
const list = Array.isArray(json.results) ? json.results : (Array.isArray(json.data) ? json.data : []);
|
|
425
793
|
if (!Array.isArray(list) || list.length === 0) {
|
|
426
794
|
lastError = new Error("rerank_empty");
|
|
@@ -442,7 +810,6 @@ async function requestRerank(args) {
|
|
|
442
810
|
lastError = new Error("rerank_map_empty");
|
|
443
811
|
}
|
|
444
812
|
catch (error) {
|
|
445
|
-
clearTimeout(timeoutId);
|
|
446
813
|
lastError = error;
|
|
447
814
|
}
|
|
448
815
|
}
|
|
@@ -450,19 +817,19 @@ async function requestRerank(args) {
|
|
|
450
817
|
}
|
|
451
818
|
function classifyIntent(query) {
|
|
452
819
|
const text = query.toLowerCase();
|
|
453
|
-
const relationHints = /(
|
|
820
|
+
const relationHints = /(关系|依赖|关联|上下游|图谱|拓扑|graph|relation|entity|dependency)/i;
|
|
454
821
|
if (relationHints.test(text))
|
|
455
822
|
return "RELATION_DISCOVERY";
|
|
456
|
-
const troubleHints = /(
|
|
823
|
+
const troubleHints = /(报错|错误|异常|失败|超时|无法|崩溃|故障|修复|bug|error|failed|timeout|fix)/i;
|
|
457
824
|
if (troubleHints.test(text))
|
|
458
825
|
return "TROUBLESHOOTING";
|
|
459
826
|
const preferenceHints = /(偏好|习惯|口味|喜欢|不喜欢|偏向|preference)/i;
|
|
460
827
|
if (preferenceHints.test(text))
|
|
461
828
|
return "PREFERENCE_PROFILE";
|
|
462
|
-
const timelineHints = /(
|
|
829
|
+
const timelineHints = /(最近|上次|之前|时间线|历史|timeline|history)/i;
|
|
463
830
|
if (timelineHints.test(text))
|
|
464
831
|
return "TIMELINE_REVIEW";
|
|
465
|
-
const decisionHints = /(
|
|
832
|
+
const decisionHints = /(方案|决策|选择|建议|取舍|权衡|tradeoff|plan)/i;
|
|
466
833
|
if (decisionHints.test(text))
|
|
467
834
|
return "DECISION_SUPPORT";
|
|
468
835
|
return "FACT_LOOKUP";
|
|
@@ -490,6 +857,128 @@ function sourceWeight(source, intent) {
|
|
|
490
857
|
}
|
|
491
858
|
return 1;
|
|
492
859
|
}
|
|
860
|
+
const QUERY_PLAN_STOPWORDS = new Set([
|
|
861
|
+
"what",
|
|
862
|
+
"is",
|
|
863
|
+
"the",
|
|
864
|
+
"a",
|
|
865
|
+
"an",
|
|
866
|
+
"of",
|
|
867
|
+
"for",
|
|
868
|
+
"to",
|
|
869
|
+
"in",
|
|
870
|
+
"on",
|
|
871
|
+
"and",
|
|
872
|
+
"or",
|
|
873
|
+
"with",
|
|
874
|
+
"about",
|
|
875
|
+
"this",
|
|
876
|
+
"that",
|
|
877
|
+
"这些",
|
|
878
|
+
"那些",
|
|
879
|
+
"这个",
|
|
880
|
+
"那个",
|
|
881
|
+
"之前",
|
|
882
|
+
"提到",
|
|
883
|
+
"提过",
|
|
884
|
+
"一下",
|
|
885
|
+
"请问",
|
|
886
|
+
"关于",
|
|
887
|
+
"相关",
|
|
888
|
+
"什么",
|
|
889
|
+
"怎么",
|
|
890
|
+
"如何",
|
|
891
|
+
"是否",
|
|
892
|
+
"有没有",
|
|
893
|
+
"是什么",
|
|
894
|
+
"哪些",
|
|
895
|
+
"哪个",
|
|
896
|
+
]);
|
|
897
|
+
function planQueryKeywords(query) {
|
|
898
|
+
const normalized = (query || "").trim().replace(/\s+/g, " ");
|
|
899
|
+
if (!normalized)
|
|
900
|
+
return [];
|
|
901
|
+
const output = [];
|
|
902
|
+
const seen = new Set();
|
|
903
|
+
const push = (value) => {
|
|
904
|
+
const item = (value || "").trim();
|
|
905
|
+
if (!item)
|
|
906
|
+
return;
|
|
907
|
+
const key = item.toLowerCase();
|
|
908
|
+
if (seen.has(key))
|
|
909
|
+
return;
|
|
910
|
+
seen.add(key);
|
|
911
|
+
output.push(item);
|
|
912
|
+
};
|
|
913
|
+
push(normalized);
|
|
914
|
+
for (const match of normalized.matchAll(/["“”'‘’]([^"“”'‘’]{2,64})["“”'‘’]/g)) {
|
|
915
|
+
push(match[1]);
|
|
916
|
+
}
|
|
917
|
+
for (const match of normalized.match(/https?:\/\/[^\s]+/gi) || []) {
|
|
918
|
+
push(match);
|
|
919
|
+
}
|
|
920
|
+
for (const match of normalized.matchAll(/\b[A-Za-z0-9][A-Za-z0-9._/-]*(?:\s+[A-Za-z0-9][A-Za-z0-9._/-]*){0,3}\b/g)) {
|
|
921
|
+
const phrase = match[0].trim();
|
|
922
|
+
const words = phrase.split(/\s+/).filter(Boolean);
|
|
923
|
+
const hasStrongSignal = words.length >= 2 || /[A-Z]{2,}/.test(phrase);
|
|
924
|
+
if (hasStrongSignal && !QUERY_PLAN_STOPWORDS.has(phrase.toLowerCase())) {
|
|
925
|
+
push(phrase);
|
|
926
|
+
}
|
|
927
|
+
}
|
|
928
|
+
const normalizedForSplit = normalized
|
|
929
|
+
.replace(/[(){}\[\]<>]/g, " ")
|
|
930
|
+
.replace(/[,。!?;:,.!?;:、]/g, " ")
|
|
931
|
+
.replace(/(之前提到的|之前提到|提到的|提到|请问|一下|是什么|什么|哪个|哪篇|哪里|如何|怎么|有关|关于|以及|还有|并且|然后|能否|可以|帮我|给我)/g, " ");
|
|
932
|
+
for (const raw of normalizedForSplit.split(/[\/\-\s]+/)) {
|
|
933
|
+
const token = raw.trim();
|
|
934
|
+
if (!token || token.length < 2)
|
|
935
|
+
continue;
|
|
936
|
+
if (QUERY_PLAN_STOPWORDS.has(token.toLowerCase()))
|
|
937
|
+
continue;
|
|
938
|
+
push(token);
|
|
939
|
+
}
|
|
940
|
+
return output.slice(0, 5);
|
|
941
|
+
}
|
|
942
|
+
function shouldTriggerFulltextFallback(ranked, topK) {
|
|
943
|
+
const scoped = ranked.slice(0, Math.max(1, topK));
|
|
944
|
+
if (scoped.length === 0)
|
|
945
|
+
return true;
|
|
946
|
+
const isStrongSummaryHit = (item) => {
|
|
947
|
+
if (!Array.isArray(item.reason_tags) || !item.reason_tags.includes("summary_hit")) {
|
|
948
|
+
return false;
|
|
949
|
+
}
|
|
950
|
+
const score = Number(item.score || 0);
|
|
951
|
+
const summaryScore = Number(item.score_breakdown?.summary || 0);
|
|
952
|
+
const bm25ScoreValue = Number(item.score_breakdown?.bm25 || 0);
|
|
953
|
+
return score >= 1.2 && (summaryScore >= 1 || bm25ScoreValue >= 0.15);
|
|
954
|
+
};
|
|
955
|
+
const summaryHits = scoped.filter(isStrongSummaryHit).length;
|
|
956
|
+
const sessionScoped = scoped.filter(item => item.source === "sessions_active" || item.source === "sessions_archive");
|
|
957
|
+
if (sessionScoped.length === 0) {
|
|
958
|
+
return summaryHits === 0;
|
|
959
|
+
}
|
|
960
|
+
const sessionSummaryHits = sessionScoped.filter(isStrongSummaryHit).length;
|
|
961
|
+
return summaryHits === 0 || sessionSummaryHits === 0;
|
|
962
|
+
}
|
|
963
|
+
function normalizeFusionMode(value) {
|
|
964
|
+
if (value === "authoritative" || value === "candidates" || value === "off") {
|
|
965
|
+
return value;
|
|
966
|
+
}
|
|
967
|
+
return "auto";
|
|
968
|
+
}
|
|
969
|
+
function roundTiming(timing) {
|
|
970
|
+
const output = {};
|
|
971
|
+
for (const [key, value] of Object.entries(timing)) {
|
|
972
|
+
if (typeof value !== "number" || !Number.isFinite(value))
|
|
973
|
+
continue;
|
|
974
|
+
output[key] = Number(value.toFixed(2));
|
|
975
|
+
}
|
|
976
|
+
return output;
|
|
977
|
+
}
|
|
978
|
+
function appendTiming(timing, key, startedAt) {
|
|
979
|
+
const elapsed = Date.now() - startedAt;
|
|
980
|
+
timing[key] = (timing[key] || 0) + elapsed;
|
|
981
|
+
}
|
|
493
982
|
function mergeKeyFromDoc(doc) {
|
|
494
983
|
const canonical = typeof doc.sourceMemoryCanonicalId === "string" ? doc.sourceMemoryCanonicalId.trim() : "";
|
|
495
984
|
if (canonical) {
|
|
@@ -501,6 +990,46 @@ function mergeKeyFromDoc(doc) {
|
|
|
501
990
|
}
|
|
502
991
|
return `id:${doc.id}`;
|
|
503
992
|
}
|
|
993
|
+
function docFactStatus(doc) {
|
|
994
|
+
const direct = normalizeFactStatus(typeof doc.factStatus === "string" ? doc.factStatus : "");
|
|
995
|
+
if (direct)
|
|
996
|
+
return direct;
|
|
997
|
+
const relation = Array.isArray(doc.relations) && doc.relations.length > 0 ? doc.relations[0] : null;
|
|
998
|
+
const relationStatus = normalizeFactStatus(typeof relation?.fact_status === "string" ? relation.fact_status : "");
|
|
999
|
+
if (relationStatus)
|
|
1000
|
+
return relationStatus;
|
|
1001
|
+
return "active";
|
|
1002
|
+
}
|
|
1003
|
+
function docEvidenceIds(doc) {
|
|
1004
|
+
if (Array.isArray(doc.evidenceIds) && doc.evidenceIds.length > 0) {
|
|
1005
|
+
return uniqueStrings(doc.evidenceIds);
|
|
1006
|
+
}
|
|
1007
|
+
const wikiRefs = docWikiRefs(doc);
|
|
1008
|
+
const relation = Array.isArray(doc.relations) && doc.relations.length > 0 ? doc.relations[0] : null;
|
|
1009
|
+
const source = relation?.source || "";
|
|
1010
|
+
const target = relation?.target || "";
|
|
1011
|
+
const type = relation?.type || "";
|
|
1012
|
+
const evidence = source && target && type
|
|
1013
|
+
? buildGraphEvidenceIds({
|
|
1014
|
+
source,
|
|
1015
|
+
target,
|
|
1016
|
+
type,
|
|
1017
|
+
sourceEventId: relation?.source_event_id || doc.sourceEventId,
|
|
1018
|
+
evidenceSpan: relation?.evidence_span,
|
|
1019
|
+
wikiRefs,
|
|
1020
|
+
wikiAnchor: doc.wikiAnchor,
|
|
1021
|
+
})
|
|
1022
|
+
: [];
|
|
1023
|
+
return uniqueStrings([
|
|
1024
|
+
...evidence,
|
|
1025
|
+
doc.sourceEventId ? `graph:event:${doc.sourceEventId}` : "",
|
|
1026
|
+
...wikiRefs.map(ref => {
|
|
1027
|
+
const wikiRef = normalizeWikiEvidenceRef(ref);
|
|
1028
|
+
return wikiRef ? `wiki:${wikiRef}${doc.wikiAnchor ? `#${doc.wikiAnchor}` : ""}` : "";
|
|
1029
|
+
}),
|
|
1030
|
+
`doc:${doc.id}`,
|
|
1031
|
+
]);
|
|
1032
|
+
}
|
|
504
1033
|
function customChannelWeight(source, options) {
|
|
505
1034
|
const weights = options?.channelWeights;
|
|
506
1035
|
if (!weights)
|
|
@@ -546,6 +1075,50 @@ function channelQuota(source, topK, options) {
|
|
|
546
1075
|
return Math.max(8, topK * 3);
|
|
547
1076
|
return Math.max(12, topK * 4);
|
|
548
1077
|
}
|
|
1078
|
+
function parseJsonStringArray(value) {
|
|
1079
|
+
if (typeof value !== "string" || !value.trim()) {
|
|
1080
|
+
return [];
|
|
1081
|
+
}
|
|
1082
|
+
try {
|
|
1083
|
+
const parsed = JSON.parse(value);
|
|
1084
|
+
if (!Array.isArray(parsed)) {
|
|
1085
|
+
return [];
|
|
1086
|
+
}
|
|
1087
|
+
return parsed
|
|
1088
|
+
.map(item => (typeof item === "string" ? item.trim() : ""))
|
|
1089
|
+
.filter(Boolean);
|
|
1090
|
+
}
|
|
1091
|
+
catch {
|
|
1092
|
+
return [];
|
|
1093
|
+
}
|
|
1094
|
+
}
|
|
1095
|
+
function parseJsonRelations(value) {
|
|
1096
|
+
if (typeof value !== "string" || !value.trim()) {
|
|
1097
|
+
return [];
|
|
1098
|
+
}
|
|
1099
|
+
try {
|
|
1100
|
+
const parsed = JSON.parse(value);
|
|
1101
|
+
if (!Array.isArray(parsed)) {
|
|
1102
|
+
return [];
|
|
1103
|
+
}
|
|
1104
|
+
return parsed
|
|
1105
|
+
.map(item => {
|
|
1106
|
+
if (typeof item !== "object" || item === null)
|
|
1107
|
+
return null;
|
|
1108
|
+
const relation = item;
|
|
1109
|
+
const source = typeof relation.source === "string" ? relation.source.trim() : "";
|
|
1110
|
+
const target = typeof relation.target === "string" ? relation.target.trim() : "";
|
|
1111
|
+
const type = typeof relation.type === "string" && relation.type.trim() ? relation.type.trim() : "related_to";
|
|
1112
|
+
if (!source || !target)
|
|
1113
|
+
return null;
|
|
1114
|
+
return { source, target, type };
|
|
1115
|
+
})
|
|
1116
|
+
.filter((item) => Boolean(item));
|
|
1117
|
+
}
|
|
1118
|
+
catch {
|
|
1119
|
+
return [];
|
|
1120
|
+
}
|
|
1121
|
+
}
|
|
549
1122
|
async function searchLanceDb(args) {
|
|
550
1123
|
try {
|
|
551
1124
|
const require = (0, module_1.createRequire)(__filename);
|
|
@@ -585,12 +1158,9 @@ async function searchLanceDb(args) {
|
|
|
585
1158
|
if (!id || !summary)
|
|
586
1159
|
continue;
|
|
587
1160
|
const ts = typeof record.timestamp === "string" ? Date.parse(record.timestamp) : NaN;
|
|
588
|
-
const entities =
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
const relations = typeof record.relations_json === "string"
|
|
592
|
-
? JSON.parse(record.relations_json)
|
|
593
|
-
: [];
|
|
1161
|
+
const entities = parseJsonStringArray(record.entities_json);
|
|
1162
|
+
const relations = parseJsonRelations(record.relations_json);
|
|
1163
|
+
const embedding = Array.isArray(record.vector) ? record.vector.filter(item => Number.isFinite(item)) : undefined;
|
|
594
1164
|
docs.push({
|
|
595
1165
|
id,
|
|
596
1166
|
text: summary,
|
|
@@ -599,7 +1169,12 @@ async function searchLanceDb(args) {
|
|
|
599
1169
|
layer: record.layer === "active" || record.layer === "archive" ? record.layer : undefined,
|
|
600
1170
|
sourceMemoryId: typeof record.source_memory_id === "string" ? record.source_memory_id : undefined,
|
|
601
1171
|
sourceMemoryCanonicalId: typeof record.source_memory_canonical_id === "string" ? record.source_memory_canonical_id : undefined,
|
|
602
|
-
|
|
1172
|
+
sourceEventId: typeof record.source_event_id === "string" ? record.source_event_id : undefined,
|
|
1173
|
+
sourceField: record.source_field === "summary" || record.source_field === "evidence"
|
|
1174
|
+
? record.source_field
|
|
1175
|
+
: undefined,
|
|
1176
|
+
embedding,
|
|
1177
|
+
embeddingNorm: embedding && embedding.length > 0 ? vectorNorm(embedding) : undefined,
|
|
603
1178
|
eventType: typeof record.event_type === "string" ? record.event_type : undefined,
|
|
604
1179
|
qualityScore: typeof record.quality_score === "number" ? record.quality_score : undefined,
|
|
605
1180
|
charCount: typeof record.char_count === "number" ? record.char_count : undefined,
|
|
@@ -651,6 +1226,7 @@ function parseVectorFallback(filePath, logger) {
|
|
|
651
1226
|
})
|
|
652
1227
|
.filter((item) => Boolean(item))
|
|
653
1228
|
: [];
|
|
1229
|
+
const embedding = Array.isArray(parsed.embedding) ? parsed.embedding.filter(item => Number.isFinite(item)) : undefined;
|
|
654
1230
|
docs.push({
|
|
655
1231
|
id,
|
|
656
1232
|
text: summary,
|
|
@@ -659,7 +1235,12 @@ function parseVectorFallback(filePath, logger) {
|
|
|
659
1235
|
layer: parsed.layer === "active" || parsed.layer === "archive" ? parsed.layer : undefined,
|
|
660
1236
|
sourceMemoryId: typeof parsed.source_memory_id === "string" ? parsed.source_memory_id : undefined,
|
|
661
1237
|
sourceMemoryCanonicalId: typeof parsed.source_memory_canonical_id === "string" ? parsed.source_memory_canonical_id : undefined,
|
|
662
|
-
|
|
1238
|
+
sourceEventId: typeof parsed.source_event_id === "string" ? parsed.source_event_id : undefined,
|
|
1239
|
+
sourceField: parsed.source_field === "summary" || parsed.source_field === "evidence"
|
|
1240
|
+
? parsed.source_field
|
|
1241
|
+
: undefined,
|
|
1242
|
+
embedding,
|
|
1243
|
+
embeddingNorm: embedding && embedding.length > 0 ? vectorNorm(embedding) : undefined,
|
|
663
1244
|
eventType: typeof parsed.event_type === "string" ? parsed.event_type.trim() : undefined,
|
|
664
1245
|
qualityScore: typeof parsed.quality_score === "number" ? parsed.quality_score : undefined,
|
|
665
1246
|
charCount: typeof parsed.char_count === "number" ? parsed.char_count : undefined,
|
|
@@ -676,56 +1257,91 @@ function parseVectorFallback(filePath, logger) {
|
|
|
676
1257
|
return docs;
|
|
677
1258
|
}
|
|
678
1259
|
async function requestFusion(args) {
|
|
1260
|
+
const candidateIdSet = new Set(args.candidates.map(item => item.id));
|
|
679
1261
|
const endpoint = args.llm.baseUrl.endsWith("/chat/completions")
|
|
680
1262
|
? args.llm.baseUrl
|
|
681
1263
|
: `${args.llm.baseUrl}/chat/completions`;
|
|
682
1264
|
const evidenceText = args.candidates
|
|
683
|
-
.map((item, index) =>
|
|
1265
|
+
.map((item, index) => {
|
|
1266
|
+
const excerpt = (item.source_excerpt || "").trim();
|
|
1267
|
+
const sourceFile = (item.source_file || "").trim();
|
|
1268
|
+
const sourceMemoryId = (item.source_memory_id || "").trim();
|
|
1269
|
+
const sourceMemoryCanonicalId = (item.source_memory_canonical_id || "").trim();
|
|
1270
|
+
const sourceLayer = (item.source_layer || "").trim();
|
|
1271
|
+
const sourceEventId = (item.source_event_id || "").trim();
|
|
1272
|
+
const sourceField = (item.source_field || "").trim();
|
|
1273
|
+
const extraParts = [];
|
|
1274
|
+
if (sourceMemoryId)
|
|
1275
|
+
extraParts.push(`source_memory_id=${sourceMemoryId}`);
|
|
1276
|
+
if (sourceMemoryCanonicalId)
|
|
1277
|
+
extraParts.push(`source_memory_canonical_id=${sourceMemoryCanonicalId}`);
|
|
1278
|
+
if (sourceLayer)
|
|
1279
|
+
extraParts.push(`source_layer=${sourceLayer}`);
|
|
1280
|
+
if (sourceEventId)
|
|
1281
|
+
extraParts.push(`source_event_id=${sourceEventId}`);
|
|
1282
|
+
if (sourceField)
|
|
1283
|
+
extraParts.push(`source_field=${sourceField}`);
|
|
1284
|
+
if (sourceFile)
|
|
1285
|
+
extraParts.push(`source_file=${sourceFile}`);
|
|
1286
|
+
if (excerpt)
|
|
1287
|
+
extraParts.push(`source_excerpt=${excerpt}`);
|
|
1288
|
+
const extra = extraParts.length > 0 ? `\n ${extraParts.join("\n ")}` : "";
|
|
1289
|
+
return `${index + 1}. [${item.id}] (${item.source}, score=${item.score.toFixed(4)}) ${item.text}${extra}`;
|
|
1290
|
+
})
|
|
684
1291
|
.join("\n")
|
|
685
1292
|
.slice(0, 18000);
|
|
686
1293
|
const prompt = [
|
|
687
1294
|
`prompt_version=${READ_FUSION_PROMPT_VERSION}`,
|
|
688
|
-
"
|
|
689
|
-
"
|
|
690
|
-
"
|
|
691
|
-
"
|
|
692
|
-
"
|
|
693
|
-
"
|
|
694
|
-
"
|
|
695
|
-
"
|
|
696
|
-
"
|
|
697
|
-
"
|
|
698
|
-
"
|
|
1295
|
+
"You are a memory retrieval fusion engine. Fuse multi-channel evidence into a structured answer package for the agent.",
|
|
1296
|
+
"Core values and principles:",
|
|
1297
|
+
"A) Truthfulness first: do not fabricate; do not infer beyond evidence.",
|
|
1298
|
+
"B) Evidence first: every key conclusion must be traceable via evidence_ids.",
|
|
1299
|
+
"C) Make conflicts explicit: write conflicts instead of silently overriding.",
|
|
1300
|
+
"D) Be transparent about uncertainty: put uncertain parts in coverage_note.",
|
|
1301
|
+
"E) Summary-first: prefer summary evidence for conclusions; source_excerpt is supporting evidence.",
|
|
1302
|
+
"F) Same-source dedup: merge duplicate evidence from the same source_memory_id/source_memory_canonical_id.",
|
|
1303
|
+
"G) Full-text recall: if summary/excerpt is insufficient, return event ids in need_fulltext_event_ids.",
|
|
1304
|
+
"Source channel semantics:",
|
|
1305
|
+
"- rules: policy/constraints; use for what should be done.",
|
|
1306
|
+
"- archive: event-level stable facts (summary-first).",
|
|
1307
|
+
"- vector: semantic neighbors for recall; source_field=summary/evidence indicates chunk role.",
|
|
1308
|
+
"- graph: entity-relation structure; prefer for dependency/relationship questions.",
|
|
1309
|
+
"Query alignment:",
|
|
1310
|
+
"- answer the user query first; ignore evidence unrelated to the query.",
|
|
1311
|
+
"- when evidence conflicts, prioritize recency + quality + explicitness and record the conflict.",
|
|
1312
|
+
"Return strict JSON only:",
|
|
1313
|
+
"{\"canonical_answer\": string, \"coverage_note\": string, \"facts\": [{\"text\": string, \"evidence_ids\": string[]}], \"timeline\": [{\"when\": string, \"event\": string, \"evidence_ids\": string[]}], \"entities\": [{\"name\": string, \"role\": string}], \"decisions\": [{\"decision\": string, \"rationale\": string, \"evidence_ids\": string[]}], \"fixes\": [{\"issue\": string, \"fix\": string, \"evidence_ids\": string[]}], \"preferences\": [{\"subject\": string, \"preference\": string, \"evidence_ids\": string[]}], \"risks\": [{\"risk\": string, \"mitigation\": string, \"evidence_ids\": string[]}], \"action_items\": [{\"item\": string, \"owner\": string, \"status\": string, \"evidence_ids\": string[]}], \"conflicts\": [{\"topic\": string, \"details\": string}], \"evidence_ids\": string[], \"need_fulltext_event_ids\": string[], \"confidence\": number}",
|
|
1314
|
+
"Output constraints:",
|
|
1315
|
+
"1) canonical_answer must be directly usable.",
|
|
1316
|
+
"2) facts: usually 3-12 items; prefer high-quality evidence.",
|
|
1317
|
+
"3) evidence_ids must come from input candidate ids.",
|
|
1318
|
+
"4) conflicts must be [] when no conflict exists.",
|
|
1319
|
+
"5) confidence must be within [0, 1].",
|
|
1320
|
+
"6) uncertain parts must be explicitly stated in coverage_note.",
|
|
699
1321
|
...READ_FUSION_REGRESSION_SAMPLES,
|
|
700
1322
|
].join("\n");
|
|
701
1323
|
const body = {
|
|
702
1324
|
model: args.llm.model,
|
|
703
1325
|
temperature: 0.1,
|
|
704
1326
|
messages: [
|
|
705
|
-
{ role: "system", content: "
|
|
706
|
-
{ role: "user", content: `${prompt}\n\n
|
|
1327
|
+
{ role: "system", content: "Output JSON only. No extra text." },
|
|
1328
|
+
{ role: "user", content: `${prompt}\n\nQuery:\n${args.query}\n\nCandidate Evidence:\n${evidenceText}` },
|
|
707
1329
|
],
|
|
708
1330
|
};
|
|
709
1331
|
let lastError = null;
|
|
710
1332
|
for (let attempt = 0; attempt < 2; attempt += 1) {
|
|
711
|
-
const
|
|
712
|
-
|
|
1333
|
+
const response = await (0, http_post_1.postJsonWithTimeout)({
|
|
1334
|
+
endpoint,
|
|
1335
|
+
apiKey: args.llm.apiKey,
|
|
1336
|
+
body,
|
|
1337
|
+
timeoutMs: EXTERNAL_MODEL_TIMEOUT_MS,
|
|
1338
|
+
});
|
|
1339
|
+
if (!response.ok) {
|
|
1340
|
+
lastError = new Error(response.status > 0 ? `fusion_http_${response.status}` : (response.error || "fusion_network_error"));
|
|
1341
|
+
continue;
|
|
1342
|
+
}
|
|
713
1343
|
try {
|
|
714
|
-
const
|
|
715
|
-
method: "POST",
|
|
716
|
-
headers: {
|
|
717
|
-
"content-type": "application/json",
|
|
718
|
-
authorization: `Bearer ${args.llm.apiKey}`,
|
|
719
|
-
},
|
|
720
|
-
body: JSON.stringify(body),
|
|
721
|
-
signal: controller.signal,
|
|
722
|
-
});
|
|
723
|
-
clearTimeout(timeoutId);
|
|
724
|
-
if (!response.ok) {
|
|
725
|
-
lastError = new Error(`fusion_http_${response.status}`);
|
|
726
|
-
continue;
|
|
727
|
-
}
|
|
728
|
-
const json = await response.json();
|
|
1344
|
+
const json = (response.json || {});
|
|
729
1345
|
const content = json?.choices?.[0]?.message?.content?.trim() || "";
|
|
730
1346
|
if (!content) {
|
|
731
1347
|
lastError = new Error("fusion_empty");
|
|
@@ -739,6 +1355,13 @@ async function requestFusion(args) {
|
|
|
739
1355
|
const evidenceIds = Array.isArray(parsed.evidence_ids)
|
|
740
1356
|
? parsed.evidence_ids.filter(item => typeof item === "string" && item.trim())
|
|
741
1357
|
: [];
|
|
1358
|
+
const whitelistedEvidenceIds = [...new Set(evidenceIds.filter(id => candidateIdSet.has(id)))];
|
|
1359
|
+
const needFulltextEventIds = Array.isArray(parsed.need_fulltext_event_ids)
|
|
1360
|
+
? [...new Set(parsed.need_fulltext_event_ids
|
|
1361
|
+
.filter(item => typeof item === "string")
|
|
1362
|
+
.map(item => item.trim())
|
|
1363
|
+
.filter(Boolean))]
|
|
1364
|
+
: [];
|
|
742
1365
|
return {
|
|
743
1366
|
canonical_answer: parsed.canonical_answer.trim().slice(0, 6000),
|
|
744
1367
|
coverage_note: typeof parsed.coverage_note === "string" ? parsed.coverage_note.trim().slice(0, 1200) : "",
|
|
@@ -751,14 +1374,14 @@ async function requestFusion(args) {
|
|
|
751
1374
|
risks: Array.isArray(parsed.risks) ? parsed.risks : [],
|
|
752
1375
|
action_items: Array.isArray(parsed.action_items) ? parsed.action_items : [],
|
|
753
1376
|
conflicts: Array.isArray(parsed.conflicts) ? parsed.conflicts : [],
|
|
754
|
-
evidence_ids:
|
|
1377
|
+
evidence_ids: whitelistedEvidenceIds,
|
|
1378
|
+
need_fulltext_event_ids: needFulltextEventIds,
|
|
755
1379
|
confidence: typeof parsed.confidence === "number"
|
|
756
1380
|
? Math.max(0, Math.min(1, parsed.confidence))
|
|
757
1381
|
: 0.5,
|
|
758
1382
|
};
|
|
759
1383
|
}
|
|
760
1384
|
catch (error) {
|
|
761
|
-
clearTimeout(timeoutId);
|
|
762
1385
|
lastError = error;
|
|
763
1386
|
}
|
|
764
1387
|
}
|
|
@@ -772,6 +1395,13 @@ function createReadStore(options) {
|
|
|
772
1395
|
let vectorFallbackCache = null;
|
|
773
1396
|
let bm25TokenCacheSignature = "";
|
|
774
1397
|
let bm25TokenCache = new Map();
|
|
1398
|
+
let hitStatsCache = null;
|
|
1399
|
+
let hitStatsDirty = false;
|
|
1400
|
+
let hitStatsPendingMutations = 0;
|
|
1401
|
+
let lastHitStatsFlushAt = 0;
|
|
1402
|
+
const hitStatsFlushIntervalMs = 5000;
|
|
1403
|
+
const hitStatsFlushBatch = 24;
|
|
1404
|
+
const readTuning = resolveReadTuning(options.readTuning);
|
|
775
1405
|
function fileSignature(filePath) {
|
|
776
1406
|
try {
|
|
777
1407
|
if (!fs.existsSync(filePath)) {
|
|
@@ -785,22 +1415,30 @@ function createReadStore(options) {
|
|
|
785
1415
|
}
|
|
786
1416
|
}
|
|
787
1417
|
function loadHitStats() {
|
|
1418
|
+
if (hitStatsCache) {
|
|
1419
|
+
return hitStatsCache;
|
|
1420
|
+
}
|
|
788
1421
|
try {
|
|
789
1422
|
if (!fs.existsSync(hitStatsPath)) {
|
|
790
|
-
|
|
1423
|
+
hitStatsCache = { items: {} };
|
|
1424
|
+
return hitStatsCache;
|
|
791
1425
|
}
|
|
792
1426
|
const content = fs.readFileSync(hitStatsPath, "utf-8").trim();
|
|
793
1427
|
if (!content) {
|
|
794
|
-
|
|
1428
|
+
hitStatsCache = { items: {} };
|
|
1429
|
+
return hitStatsCache;
|
|
795
1430
|
}
|
|
796
1431
|
const parsed = JSON.parse(content);
|
|
797
1432
|
if (!parsed || typeof parsed !== "object" || !parsed.items || typeof parsed.items !== "object") {
|
|
798
|
-
|
|
1433
|
+
hitStatsCache = { items: {} };
|
|
1434
|
+
return hitStatsCache;
|
|
799
1435
|
}
|
|
800
|
-
|
|
1436
|
+
hitStatsCache = parsed;
|
|
1437
|
+
return hitStatsCache;
|
|
801
1438
|
}
|
|
802
1439
|
catch {
|
|
803
|
-
|
|
1440
|
+
hitStatsCache = { items: {} };
|
|
1441
|
+
return hitStatsCache;
|
|
804
1442
|
}
|
|
805
1443
|
}
|
|
806
1444
|
function saveHitStats(state) {
|
|
@@ -815,6 +1453,21 @@ function createReadStore(options) {
|
|
|
815
1453
|
options.logger.warn(`Failed to persist read hit stats: ${error}`);
|
|
816
1454
|
}
|
|
817
1455
|
}
|
|
1456
|
+
function maybeFlushHitStats(force) {
|
|
1457
|
+
if (!hitStatsDirty || !hitStatsCache) {
|
|
1458
|
+
return;
|
|
1459
|
+
}
|
|
1460
|
+
const now = Date.now();
|
|
1461
|
+
if (!force &&
|
|
1462
|
+
hitStatsPendingMutations < hitStatsFlushBatch &&
|
|
1463
|
+
(now - lastHitStatsFlushAt) < hitStatsFlushIntervalMs) {
|
|
1464
|
+
return;
|
|
1465
|
+
}
|
|
1466
|
+
saveHitStats(hitStatsCache);
|
|
1467
|
+
hitStatsDirty = false;
|
|
1468
|
+
hitStatsPendingMutations = 0;
|
|
1469
|
+
lastHitStatsFlushAt = now;
|
|
1470
|
+
}
|
|
818
1471
|
function markHit(ids) {
|
|
819
1472
|
if (!ids.length) {
|
|
820
1473
|
return;
|
|
@@ -839,68 +1492,524 @@ function createReadStore(options) {
|
|
|
839
1492
|
})
|
|
840
1493
|
.slice(0, 20000);
|
|
841
1494
|
state.items = Object.fromEntries(entries);
|
|
842
|
-
|
|
1495
|
+
hitStatsCache = state;
|
|
1496
|
+
hitStatsDirty = true;
|
|
1497
|
+
hitStatsPendingMutations += ids.length;
|
|
1498
|
+
maybeFlushHitStats(false);
|
|
843
1499
|
}
|
|
844
1500
|
function loadAllDocuments() {
|
|
845
1501
|
const cortexRulesPath = path.join(memoryRoot, "CORTEX_RULES.md");
|
|
846
|
-
const memoryMdPath = path.join(memoryRoot, "MEMORY.md");
|
|
847
1502
|
const activeSessionsPath = path.join(memoryRoot, "sessions", "active", "sessions.jsonl");
|
|
848
1503
|
const archiveSessionsPath = path.join(memoryRoot, "sessions", "archive", "sessions.jsonl");
|
|
1504
|
+
const graphMemoryPath = path.join(memoryRoot, "graph", "memory.jsonl");
|
|
1505
|
+
const supersededRelationPath = path.join(memoryRoot, "graph", "superseded_relations.jsonl");
|
|
1506
|
+
const conflictQueuePath = path.join(memoryRoot, "graph", "conflict_queue.jsonl");
|
|
1507
|
+
const wikiProjectionIndexPath = path.join(memoryRoot, "wiki", ".projection_index.json");
|
|
849
1508
|
const signature = [
|
|
850
1509
|
fileSignature(cortexRulesPath),
|
|
851
|
-
fileSignature(memoryMdPath),
|
|
852
1510
|
fileSignature(activeSessionsPath),
|
|
853
1511
|
fileSignature(archiveSessionsPath),
|
|
1512
|
+
fileSignature(graphMemoryPath),
|
|
1513
|
+
fileSignature(supersededRelationPath),
|
|
1514
|
+
fileSignature(conflictQueuePath),
|
|
1515
|
+
fileSignature(wikiProjectionIndexPath),
|
|
854
1516
|
].join("|");
|
|
855
1517
|
if (docsCache && docsCache.signature === signature) {
|
|
856
1518
|
return docsCache.docs;
|
|
857
1519
|
}
|
|
1520
|
+
const archiveEventTypeById = new Map();
|
|
1521
|
+
if (fs.existsSync(archiveSessionsPath)) {
|
|
1522
|
+
const archiveContent = safeReadFile(archiveSessionsPath);
|
|
1523
|
+
for (const line of archiveContent.split(/\r?\n/)) {
|
|
1524
|
+
const trimmed = line.trim();
|
|
1525
|
+
if (!trimmed)
|
|
1526
|
+
continue;
|
|
1527
|
+
try {
|
|
1528
|
+
const parsed = JSON.parse(trimmed);
|
|
1529
|
+
const id = typeof parsed.id === "string" ? parsed.id.trim() : "";
|
|
1530
|
+
const eventType = typeof parsed.event_type === "string" ? parsed.event_type.trim() : "";
|
|
1531
|
+
if (id && eventType) {
|
|
1532
|
+
archiveEventTypeById.set(id, eventType);
|
|
1533
|
+
}
|
|
1534
|
+
}
|
|
1535
|
+
catch { }
|
|
1536
|
+
}
|
|
1537
|
+
}
|
|
1538
|
+
const wikiRefIndex = readWikiProjectionRefIndex(wikiProjectionIndexPath, options.logger);
|
|
1539
|
+
const graphDocs = [];
|
|
1540
|
+
const supersededRelationKeys = new Set();
|
|
1541
|
+
if (fs.existsSync(supersededRelationPath)) {
|
|
1542
|
+
const supersededContent = safeReadFile(supersededRelationPath);
|
|
1543
|
+
for (const line of supersededContent.split(/\r?\n/)) {
|
|
1544
|
+
const trimmed = line.trim();
|
|
1545
|
+
if (!trimmed)
|
|
1546
|
+
continue;
|
|
1547
|
+
try {
|
|
1548
|
+
const parsed = JSON.parse(trimmed);
|
|
1549
|
+
const relationKey = typeof parsed.relation_key === "string" ? parsed.relation_key.trim().toLowerCase() : "";
|
|
1550
|
+
if (relationKey) {
|
|
1551
|
+
supersededRelationKeys.add(relationKey);
|
|
1552
|
+
}
|
|
1553
|
+
}
|
|
1554
|
+
catch (error) {
|
|
1555
|
+
options.logger.debug(`Skipping invalid superseded relation line: ${error}`);
|
|
1556
|
+
}
|
|
1557
|
+
}
|
|
1558
|
+
}
|
|
1559
|
+
if (fs.existsSync(graphMemoryPath)) {
|
|
1560
|
+
const graphContent = safeReadFile(graphMemoryPath);
|
|
1561
|
+
for (const line of graphContent.split(/\r?\n/)) {
|
|
1562
|
+
const trimmed = line.trim();
|
|
1563
|
+
if (!trimmed)
|
|
1564
|
+
continue;
|
|
1565
|
+
try {
|
|
1566
|
+
const parsed = JSON.parse(trimmed);
|
|
1567
|
+
const id = typeof parsed.id === "string" ? parsed.id : "";
|
|
1568
|
+
const summary = typeof parsed.summary === "string" ? parsed.summary.trim() : "";
|
|
1569
|
+
const sourceTextNav = typeof parsed.source_text_nav === "object" && parsed.source_text_nav !== null && !Array.isArray(parsed.source_text_nav)
|
|
1570
|
+
? parsed.source_text_nav
|
|
1571
|
+
: undefined;
|
|
1572
|
+
const sourceEventId = typeof parsed.source_event_id === "string" ? parsed.source_event_id : "";
|
|
1573
|
+
const archiveEventId = typeof parsed.archive_event_id === "string" ? parsed.archive_event_id : "";
|
|
1574
|
+
const navSourceEventId = typeof sourceTextNav?.source_event_id === "string" ? sourceTextNav.source_event_id.trim() : "";
|
|
1575
|
+
const navSourceMemoryId = typeof sourceTextNav?.source_memory_id === "string" ? sourceTextNav.source_memory_id.trim() : "";
|
|
1576
|
+
const navSessionId = typeof sourceTextNav?.session_id === "string" ? sourceTextNav.session_id.trim() : "";
|
|
1577
|
+
const navSourceLayer = typeof sourceTextNav?.layer === "string" ? sourceTextNav.layer.trim() : "";
|
|
1578
|
+
const navSourceFile = typeof sourceTextNav?.source_file === "string" ? sourceTextNav.source_file.trim() : "";
|
|
1579
|
+
const eventRefId = navSourceMemoryId || archiveEventId || navSourceEventId || sourceEventId;
|
|
1580
|
+
const sessionId = navSessionId || (typeof parsed.session_id === "string" ? parsed.session_id : "");
|
|
1581
|
+
const sourceLayer = navSourceLayer || (typeof parsed.source_layer === "string" ? parsed.source_layer : "");
|
|
1582
|
+
const sourceFile = navSourceFile || (typeof parsed.source_file === "string" ? parsed.source_file : "");
|
|
1583
|
+
const timestamp = typeof parsed.timestamp === "string" ? Date.parse(parsed.timestamp) : NaN;
|
|
1584
|
+
const entities = Array.isArray(parsed.entities)
|
|
1585
|
+
? parsed.entities.map((item) => (typeof item === "string" ? item.trim() : "")).filter(Boolean)
|
|
1586
|
+
: [];
|
|
1587
|
+
const entityTypes = typeof parsed.entity_types === "object" && parsed.entity_types !== null
|
|
1588
|
+
? parsed.entity_types
|
|
1589
|
+
: {};
|
|
1590
|
+
const relations = Array.isArray(parsed.relations) ? parsed.relations : [];
|
|
1591
|
+
const eventType = (typeof parsed.event_type === "string" ? parsed.event_type : "") || archiveEventTypeById.get(eventRefId) || "";
|
|
1592
|
+
let relationCount = 0;
|
|
1593
|
+
for (const relationRaw of relations) {
|
|
1594
|
+
if (typeof relationRaw !== "object" || relationRaw === null)
|
|
1595
|
+
continue;
|
|
1596
|
+
const relationRecord = relationRaw;
|
|
1597
|
+
const source = typeof relationRecord.source === "string" ? relationRecord.source.trim() : "";
|
|
1598
|
+
const target = typeof relationRecord.target === "string" ? relationRecord.target.trim() : "";
|
|
1599
|
+
const type = typeof relationRecord.type === "string" ? relationRecord.type.trim() : "related_to";
|
|
1600
|
+
if (!source || !target)
|
|
1601
|
+
continue;
|
|
1602
|
+
relationCount += 1;
|
|
1603
|
+
const relationKey = graphRelationKey({ source, target, type });
|
|
1604
|
+
const factStatus = supersededRelationKeys.has(relationKey) ? "superseded" : "active";
|
|
1605
|
+
const evidenceSpan = typeof relationRecord.evidence_span === "string" && relationRecord.evidence_span.trim()
|
|
1606
|
+
? relationRecord.evidence_span.trim()
|
|
1607
|
+
: undefined;
|
|
1608
|
+
const confidenceValue = typeof relationRecord.confidence === "number" && Number.isFinite(relationRecord.confidence)
|
|
1609
|
+
? relationRecord.confidence
|
|
1610
|
+
: undefined;
|
|
1611
|
+
const relationOrigin = typeof relationRecord.relation_origin === "string" && relationRecord.relation_origin.trim()
|
|
1612
|
+
? relationRecord.relation_origin.trim()
|
|
1613
|
+
: undefined;
|
|
1614
|
+
const relationDefinition = typeof relationRecord.relation_definition === "string" && relationRecord.relation_definition.trim()
|
|
1615
|
+
? relationRecord.relation_definition.trim()
|
|
1616
|
+
: undefined;
|
|
1617
|
+
const contextChunk = typeof relationRecord.context_chunk === "string" && relationRecord.context_chunk.trim()
|
|
1618
|
+
? relationRecord.context_chunk.trim()
|
|
1619
|
+
: undefined;
|
|
1620
|
+
const relation = {
|
|
1621
|
+
source,
|
|
1622
|
+
target,
|
|
1623
|
+
type,
|
|
1624
|
+
relation_origin: relationOrigin,
|
|
1625
|
+
relation_definition: relationDefinition,
|
|
1626
|
+
evidence_span: evidenceSpan,
|
|
1627
|
+
context_chunk: contextChunk,
|
|
1628
|
+
confidence: confidenceValue,
|
|
1629
|
+
fact_status: factStatus,
|
|
1630
|
+
source_event_id: sourceEventId || archiveEventId || undefined,
|
|
1631
|
+
relation_key: relationKey,
|
|
1632
|
+
};
|
|
1633
|
+
const relationEntities = uniqueStrings([...entities, source, target]);
|
|
1634
|
+
const wikiRefs = wikiRefsForGraphRelation(wikiRefIndex, { source, target, type });
|
|
1635
|
+
const entityLines = relationEntities.length > 0
|
|
1636
|
+
? relationEntities.map((entity, index) => {
|
|
1637
|
+
const entityType = entityTypes[entity];
|
|
1638
|
+
return `${index + 1}. ${entity}${entityType ? ` (${entityType})` : ""}`;
|
|
1639
|
+
}).join("\n")
|
|
1640
|
+
: "none";
|
|
1641
|
+
const text = [
|
|
1642
|
+
`# Graph Relation`,
|
|
1643
|
+
`record_id: ${id}`,
|
|
1644
|
+
`relation_index: ${relationCount}`,
|
|
1645
|
+
`relation_key: ${relationKey}`,
|
|
1646
|
+
`fact_status: ${factStatus}`,
|
|
1647
|
+
`source_event_id: ${sourceEventId || archiveEventId || "unknown"}`,
|
|
1648
|
+
`source_layer: ${sourceLayer || "unknown"}`,
|
|
1649
|
+
`archive_event_id: ${archiveEventId || "n/a"}`,
|
|
1650
|
+
`event_type: ${eventType || "unknown"}`,
|
|
1651
|
+
`session_id: ${sessionId || "unknown"}`,
|
|
1652
|
+
`source_file: ${sourceFile || "unknown"}`,
|
|
1653
|
+
`evidence_span: ${evidenceSpan || "n/a"}`,
|
|
1654
|
+
`context_chunk: ${contextChunk || "n/a"}`,
|
|
1655
|
+
`relation_origin: ${relationOrigin || "n/a"}`,
|
|
1656
|
+
`relation_definition: ${relationDefinition || "n/a"}`,
|
|
1657
|
+
`confidence: ${typeof confidenceValue === "number" ? confidenceValue : "n/a"}`,
|
|
1658
|
+
``,
|
|
1659
|
+
`## Summary`,
|
|
1660
|
+
summary || "n/a",
|
|
1661
|
+
``,
|
|
1662
|
+
`## Source References`,
|
|
1663
|
+
`source_event_id: ${navSourceEventId || sourceEventId || archiveEventId || "unknown"}`,
|
|
1664
|
+
`source_memory_id: ${navSourceMemoryId || eventRefId || "unknown"}`,
|
|
1665
|
+
`source_layer: ${sourceLayer || "unknown"}`,
|
|
1666
|
+
`source_file: ${sourceFile || "unknown"}`,
|
|
1667
|
+
`session_id: ${sessionId || "unknown"}`,
|
|
1668
|
+
``,
|
|
1669
|
+
`## Entities`,
|
|
1670
|
+
entityLines,
|
|
1671
|
+
``,
|
|
1672
|
+
`## Relation`,
|
|
1673
|
+
`${source} -[${type}/${factStatus}]-> ${target}`,
|
|
1674
|
+
].join("\n");
|
|
1675
|
+
graphDocs.push({
|
|
1676
|
+
id: `${id || "graph"}:rel:${relationCount}`,
|
|
1677
|
+
text,
|
|
1678
|
+
source: "sessions_graph",
|
|
1679
|
+
timestamp: Number.isFinite(timestamp) ? timestamp : undefined,
|
|
1680
|
+
layer: sourceLayer === "active_only" ? "active" : "archive",
|
|
1681
|
+
summaryText: summary || undefined,
|
|
1682
|
+
sourceText: contextChunk || undefined,
|
|
1683
|
+
sourceMemoryId: eventRefId || id,
|
|
1684
|
+
sourceEventId: navSourceEventId || sourceEventId || archiveEventId || undefined,
|
|
1685
|
+
sourceFile: sourceFile || undefined,
|
|
1686
|
+
sessionId,
|
|
1687
|
+
entities: relationEntities,
|
|
1688
|
+
relations: [relation],
|
|
1689
|
+
eventType: eventType || undefined,
|
|
1690
|
+
qualityScore: typeof parsed.confidence === "number" && Number.isFinite(parsed.confidence) ? parsed.confidence : 1,
|
|
1691
|
+
factStatus,
|
|
1692
|
+
wikiRef: wikiRefs[0] || undefined,
|
|
1693
|
+
wikiRefs,
|
|
1694
|
+
evidenceIds: buildGraphEvidenceIds({
|
|
1695
|
+
source,
|
|
1696
|
+
target,
|
|
1697
|
+
type,
|
|
1698
|
+
sourceEventId: relation.source_event_id,
|
|
1699
|
+
evidenceSpan,
|
|
1700
|
+
wikiRefs,
|
|
1701
|
+
}),
|
|
1702
|
+
});
|
|
1703
|
+
}
|
|
1704
|
+
}
|
|
1705
|
+
catch (error) {
|
|
1706
|
+
options.logger.debug(`Skipping invalid graph memory line: ${error}`);
|
|
1707
|
+
}
|
|
1708
|
+
}
|
|
1709
|
+
}
|
|
1710
|
+
if (fs.existsSync(conflictQueuePath)) {
|
|
1711
|
+
const queueContent = safeReadFile(conflictQueuePath);
|
|
1712
|
+
for (const line of queueContent.split(/\r?\n/)) {
|
|
1713
|
+
const trimmed = line.trim();
|
|
1714
|
+
if (!trimmed)
|
|
1715
|
+
continue;
|
|
1716
|
+
try {
|
|
1717
|
+
const parsed = JSON.parse(trimmed);
|
|
1718
|
+
const status = normalizeFactStatus(typeof parsed.status === "string" ? parsed.status : "");
|
|
1719
|
+
if (status !== "pending_conflict" && status !== "rejected") {
|
|
1720
|
+
continue;
|
|
1721
|
+
}
|
|
1722
|
+
const conflictId = typeof parsed.conflict_id === "string" ? parsed.conflict_id.trim() : "";
|
|
1723
|
+
const sourceEventId = typeof parsed.source_event_id === "string" ? parsed.source_event_id.trim() : "";
|
|
1724
|
+
const sessionId = typeof parsed.session_id === "string" ? parsed.session_id.trim() : "";
|
|
1725
|
+
const sourceFile = typeof parsed.source_file === "string" ? parsed.source_file.trim() : "";
|
|
1726
|
+
const sourceLayer = typeof parsed.source_layer === "string" ? parsed.source_layer.trim() : "";
|
|
1727
|
+
const updatedAt = typeof parsed.updated_at === "string" ? Date.parse(parsed.updated_at) : NaN;
|
|
1728
|
+
const candidate = typeof parsed.candidate === "object" && parsed.candidate !== null
|
|
1729
|
+
? parsed.candidate
|
|
1730
|
+
: {};
|
|
1731
|
+
const candidateSummary = typeof candidate.summary === "string" ? candidate.summary.trim() : "";
|
|
1732
|
+
const candidateSourceTextNav = typeof candidate.source_text_nav === "object" && candidate.source_text_nav !== null && !Array.isArray(candidate.source_text_nav)
|
|
1733
|
+
? candidate.source_text_nav
|
|
1734
|
+
: undefined;
|
|
1735
|
+
const navSourceEventId = typeof candidateSourceTextNav?.source_event_id === "string" ? candidateSourceTextNav.source_event_id.trim() : "";
|
|
1736
|
+
const navSourceMemoryId = typeof candidateSourceTextNav?.source_memory_id === "string" ? candidateSourceTextNav.source_memory_id.trim() : "";
|
|
1737
|
+
const navSourceLayer = typeof candidateSourceTextNav?.layer === "string" ? candidateSourceTextNav.layer.trim() : "";
|
|
1738
|
+
const navSourceFile = typeof candidateSourceTextNav?.source_file === "string" ? candidateSourceTextNav.source_file.trim() : "";
|
|
1739
|
+
const navSessionId = typeof candidateSourceTextNav?.session_id === "string" ? candidateSourceTextNav.session_id.trim() : "";
|
|
1740
|
+
const candidateEventType = typeof candidate.event_type === "string" ? candidate.event_type.trim() : "";
|
|
1741
|
+
const candidateRelations = Array.isArray(candidate.relations) ? candidate.relations : [];
|
|
1742
|
+
let relationIndex = 0;
|
|
1743
|
+
for (const relationRaw of candidateRelations) {
|
|
1744
|
+
if (typeof relationRaw !== "object" || relationRaw === null)
|
|
1745
|
+
continue;
|
|
1746
|
+
const relationRecord = relationRaw;
|
|
1747
|
+
const source = typeof relationRecord.source === "string" ? relationRecord.source.trim() : "";
|
|
1748
|
+
const target = typeof relationRecord.target === "string" ? relationRecord.target.trim() : "";
|
|
1749
|
+
const type = typeof relationRecord.type === "string" ? relationRecord.type.trim() : "related_to";
|
|
1750
|
+
if (!source || !target)
|
|
1751
|
+
continue;
|
|
1752
|
+
relationIndex += 1;
|
|
1753
|
+
const relationKey = graphRelationKey({ source, target, type });
|
|
1754
|
+
const evidenceSpan = typeof relationRecord.evidence_span === "string" && relationRecord.evidence_span.trim()
|
|
1755
|
+
? relationRecord.evidence_span.trim()
|
|
1756
|
+
: undefined;
|
|
1757
|
+
const confidenceValue = typeof relationRecord.confidence === "number" && Number.isFinite(relationRecord.confidence)
|
|
1758
|
+
? relationRecord.confidence
|
|
1759
|
+
: undefined;
|
|
1760
|
+
const relationOrigin = typeof relationRecord.relation_origin === "string" && relationRecord.relation_origin.trim()
|
|
1761
|
+
? relationRecord.relation_origin.trim()
|
|
1762
|
+
: undefined;
|
|
1763
|
+
const relationDefinition = typeof relationRecord.relation_definition === "string" && relationRecord.relation_definition.trim()
|
|
1764
|
+
? relationRecord.relation_definition.trim()
|
|
1765
|
+
: undefined;
|
|
1766
|
+
const contextChunk = typeof relationRecord.context_chunk === "string" && relationRecord.context_chunk.trim()
|
|
1767
|
+
? relationRecord.context_chunk.trim()
|
|
1768
|
+
: undefined;
|
|
1769
|
+
const relation = {
|
|
1770
|
+
source,
|
|
1771
|
+
target,
|
|
1772
|
+
type,
|
|
1773
|
+
relation_origin: relationOrigin,
|
|
1774
|
+
relation_definition: relationDefinition,
|
|
1775
|
+
evidence_span: evidenceSpan,
|
|
1776
|
+
context_chunk: contextChunk,
|
|
1777
|
+
confidence: confidenceValue,
|
|
1778
|
+
fact_status: status,
|
|
1779
|
+
source_event_id: sourceEventId || undefined,
|
|
1780
|
+
conflict_id: conflictId || undefined,
|
|
1781
|
+
relation_key: relationKey,
|
|
1782
|
+
};
|
|
1783
|
+
const wikiRefs = wikiRefsForGraphRelation(wikiRefIndex, { source, target, type });
|
|
1784
|
+
const evidenceIds = uniqueStrings([
|
|
1785
|
+
...buildGraphEvidenceIds({
|
|
1786
|
+
source,
|
|
1787
|
+
target,
|
|
1788
|
+
type,
|
|
1789
|
+
sourceEventId: sourceEventId || undefined,
|
|
1790
|
+
evidenceSpan,
|
|
1791
|
+
wikiRefs,
|
|
1792
|
+
}),
|
|
1793
|
+
conflictId ? `graph:conflict:${conflictId}` : "",
|
|
1794
|
+
]);
|
|
1795
|
+
const text = [
|
|
1796
|
+
`# Graph Conflict Candidate`,
|
|
1797
|
+
`conflict_id: ${conflictId || "unknown"}`,
|
|
1798
|
+
`fact_status: ${status}`,
|
|
1799
|
+
`source_event_id: ${sourceEventId || "unknown"}`,
|
|
1800
|
+
`source_layer: ${sourceLayer || "unknown"}`,
|
|
1801
|
+
`event_type: ${candidateEventType || "unknown"}`,
|
|
1802
|
+
`session_id: ${sessionId || "unknown"}`,
|
|
1803
|
+
`source_file: ${sourceFile || "unknown"}`,
|
|
1804
|
+
`relation_key: ${relationKey}`,
|
|
1805
|
+
`evidence_span: ${evidenceSpan || "n/a"}`,
|
|
1806
|
+
`context_chunk: ${contextChunk || "n/a"}`,
|
|
1807
|
+
`relation_origin: ${relationOrigin || "n/a"}`,
|
|
1808
|
+
`relation_definition: ${relationDefinition || "n/a"}`,
|
|
1809
|
+
`confidence: ${typeof confidenceValue === "number" ? confidenceValue : "n/a"}`,
|
|
1810
|
+
``,
|
|
1811
|
+
`## Summary`,
|
|
1812
|
+
candidateSummary || "n/a",
|
|
1813
|
+
``,
|
|
1814
|
+
`## Source References`,
|
|
1815
|
+
`source_event_id: ${navSourceEventId || sourceEventId || "unknown"}`,
|
|
1816
|
+
`source_memory_id: ${navSourceMemoryId || sourceEventId || "unknown"}`,
|
|
1817
|
+
`source_layer: ${navSourceLayer || sourceLayer || "unknown"}`,
|
|
1818
|
+
`source_file: ${navSourceFile || sourceFile || "unknown"}`,
|
|
1819
|
+
`session_id: ${navSessionId || sessionId || "unknown"}`,
|
|
1820
|
+
``,
|
|
1821
|
+
`${source} -[${type}/${status}]-> ${target}`,
|
|
1822
|
+
].join("\n");
|
|
1823
|
+
graphDocs.push({
|
|
1824
|
+
id: `gcf:${conflictId || "unknown"}:rel:${relationIndex}`,
|
|
1825
|
+
text,
|
|
1826
|
+
source: "sessions_graph_conflict",
|
|
1827
|
+
timestamp: Number.isFinite(updatedAt) ? updatedAt : undefined,
|
|
1828
|
+
layer: (navSourceLayer || sourceLayer) === "active_only" ? "active" : "archive",
|
|
1829
|
+
summaryText: candidateSummary || undefined,
|
|
1830
|
+
sourceText: contextChunk || undefined,
|
|
1831
|
+
sourceMemoryId: navSourceMemoryId || relationKey,
|
|
1832
|
+
sourceEventId: navSourceEventId || sourceEventId || undefined,
|
|
1833
|
+
sourceFile: navSourceFile || sourceFile || undefined,
|
|
1834
|
+
sessionId: navSessionId || sessionId || undefined,
|
|
1835
|
+
entities: uniqueStrings([source, target]),
|
|
1836
|
+
relations: [relation],
|
|
1837
|
+
eventType: candidateEventType || "graph_conflict",
|
|
1838
|
+
qualityScore: typeof confidenceValue === "number" ? confidenceValue : 0.8,
|
|
1839
|
+
factStatus: status,
|
|
1840
|
+
wikiRef: wikiRefs[0] || undefined,
|
|
1841
|
+
wikiRefs,
|
|
1842
|
+
evidenceIds,
|
|
1843
|
+
});
|
|
1844
|
+
}
|
|
1845
|
+
}
|
|
1846
|
+
catch (error) {
|
|
1847
|
+
options.logger.debug(`Skipping invalid conflict queue line: ${error}`);
|
|
1848
|
+
}
|
|
1849
|
+
}
|
|
1850
|
+
}
|
|
1851
|
+
const entitySummaryDocs = buildEntityGraphSummaryDocs(graphDocs.filter(item => item.factStatus === "active")).map(doc => {
|
|
1852
|
+
const wikiRefs = uniqueStrings((Array.isArray(doc.entities) ? doc.entities : [])
|
|
1853
|
+
.map(entity => wikiRefIndex.entityByName.get(entity.trim().toLowerCase()) || ""));
|
|
1854
|
+
if (wikiRefs.length === 0) {
|
|
1855
|
+
return doc;
|
|
1856
|
+
}
|
|
1857
|
+
const wikiAnchor = "current-facts";
|
|
1858
|
+
return {
|
|
1859
|
+
...doc,
|
|
1860
|
+
wikiRef: wikiRefs[0],
|
|
1861
|
+
wikiRefs,
|
|
1862
|
+
wikiAnchor,
|
|
1863
|
+
evidenceIds: uniqueStrings([
|
|
1864
|
+
...(Array.isArray(doc.evidenceIds) ? doc.evidenceIds : []),
|
|
1865
|
+
...wikiRefs.map(ref => {
|
|
1866
|
+
const wikiRef = normalizeWikiEvidenceRef(ref);
|
|
1867
|
+
return wikiRef ? `wiki:${wikiRef}#${wikiAnchor}` : "";
|
|
1868
|
+
}),
|
|
1869
|
+
]),
|
|
1870
|
+
};
|
|
1871
|
+
});
|
|
858
1872
|
const docs = [
|
|
859
1873
|
...parseMarkdownFile(cortexRulesPath, "CORTEX_RULES.md"),
|
|
860
|
-
...parseMarkdownFile(memoryMdPath, "MEMORY.md"),
|
|
861
1874
|
...parseJsonlFile(activeSessionsPath, "sessions_active", options.logger),
|
|
862
1875
|
...parseJsonlFile(archiveSessionsPath, "sessions_archive", options.logger),
|
|
1876
|
+
...graphDocs,
|
|
1877
|
+
...entitySummaryDocs,
|
|
863
1878
|
];
|
|
864
1879
|
docsCache = { signature, docs };
|
|
865
1880
|
return docs;
|
|
866
1881
|
}
|
|
1882
|
+
function buildVectorFallbackIndex(signature, docs) {
|
|
1883
|
+
const bySourceMemoryId = new Map();
|
|
1884
|
+
const vectorEntries = [];
|
|
1885
|
+
for (const doc of docs) {
|
|
1886
|
+
const key = (doc.sourceMemoryId || "").trim();
|
|
1887
|
+
if (key) {
|
|
1888
|
+
const list = bySourceMemoryId.get(key) || [];
|
|
1889
|
+
list.push(doc);
|
|
1890
|
+
bySourceMemoryId.set(key, list);
|
|
1891
|
+
}
|
|
1892
|
+
if (Array.isArray(doc.embedding) && doc.embedding.length > 0) {
|
|
1893
|
+
const norm = doc.embeddingNorm || vectorNorm(doc.embedding);
|
|
1894
|
+
if (norm > 0) {
|
|
1895
|
+
vectorEntries.push({ doc, embedding: doc.embedding, norm });
|
|
1896
|
+
}
|
|
1897
|
+
}
|
|
1898
|
+
}
|
|
1899
|
+
return { signature, docs, vectorEntries, bySourceMemoryId };
|
|
1900
|
+
}
|
|
867
1901
|
function loadVectorFallbackCached() {
|
|
868
1902
|
const signature = fileSignature(vectorFallbackPath);
|
|
869
1903
|
if (vectorFallbackCache && vectorFallbackCache.signature === signature) {
|
|
870
|
-
return vectorFallbackCache
|
|
1904
|
+
return vectorFallbackCache;
|
|
871
1905
|
}
|
|
872
1906
|
const docs = parseVectorFallback(vectorFallbackPath, options.logger);
|
|
873
|
-
vectorFallbackCache =
|
|
874
|
-
return
|
|
1907
|
+
vectorFallbackCache = buildVectorFallbackIndex(signature, docs);
|
|
1908
|
+
return vectorFallbackCache;
|
|
875
1909
|
}
|
|
876
|
-
function
|
|
1910
|
+
function selectVectorFallbackDocs(index, queryVector, queryNorm, limit) {
|
|
1911
|
+
if (!queryVector || queryVector.length === 0 || queryNorm <= 0 || index.vectorEntries.length === 0) {
|
|
1912
|
+
return index.docs;
|
|
1913
|
+
}
|
|
1914
|
+
const cappedLimit = Math.max(1, Math.floor(limit));
|
|
1915
|
+
const selected = new Map();
|
|
1916
|
+
const ranked = index.vectorEntries
|
|
1917
|
+
.map(entry => ({
|
|
1918
|
+
doc: entry.doc,
|
|
1919
|
+
score: cosineSimilarityWithNorm(queryVector, entry.embedding, queryNorm, entry.norm),
|
|
1920
|
+
}))
|
|
1921
|
+
.filter(item => Number.isFinite(item.score))
|
|
1922
|
+
.sort((a, b) => b.score - a.score)
|
|
1923
|
+
.slice(0, cappedLimit);
|
|
1924
|
+
for (const item of ranked) {
|
|
1925
|
+
selected.set(item.doc.id, item.doc);
|
|
1926
|
+
const sourceMemoryId = (item.doc.sourceMemoryId || "").trim();
|
|
1927
|
+
if (!sourceMemoryId)
|
|
1928
|
+
continue;
|
|
1929
|
+
for (const sibling of index.bySourceMemoryId.get(sourceMemoryId) || []) {
|
|
1930
|
+
if (selected.size >= cappedLimit)
|
|
1931
|
+
break;
|
|
1932
|
+
selected.set(sibling.id, sibling);
|
|
1933
|
+
}
|
|
1934
|
+
}
|
|
1935
|
+
return [...selected.values()];
|
|
1936
|
+
}
|
|
1937
|
+
function getBm25Tokens(doc, signature, channel) {
|
|
877
1938
|
if (bm25TokenCacheSignature !== signature) {
|
|
878
1939
|
bm25TokenCacheSignature = signature;
|
|
879
1940
|
bm25TokenCache = new Map();
|
|
880
1941
|
}
|
|
881
|
-
const
|
|
1942
|
+
const isSessionMemory = doc.source === "sessions_active" || doc.source === "sessions_archive";
|
|
1943
|
+
const summaryChannelText = isSessionMemory
|
|
1944
|
+
? ((doc.summaryText || "").trim())
|
|
1945
|
+
: ((doc.summaryText || doc.text || "").trim());
|
|
1946
|
+
const channelText = channel === "fulltext"
|
|
1947
|
+
? ((doc.sourceText || "").trim() || summaryChannelText)
|
|
1948
|
+
: summaryChannelText;
|
|
1949
|
+
const key = `${channel}:${doc.source}|${doc.id}|${channelText.length}|${channelText.slice(0, 64)}`;
|
|
882
1950
|
const cached = bm25TokenCache.get(key);
|
|
883
1951
|
if (cached) {
|
|
884
1952
|
return cached;
|
|
885
1953
|
}
|
|
886
|
-
const tokens = tokenize(
|
|
1954
|
+
const tokens = tokenize(channelText);
|
|
887
1955
|
bm25TokenCache.set(key, tokens);
|
|
888
1956
|
return tokens;
|
|
889
1957
|
}
|
|
890
1958
|
async function searchMemory(args) {
|
|
1959
|
+
const startedAt = Date.now();
|
|
1960
|
+
const timing = {};
|
|
891
1961
|
const query = args.query?.trim();
|
|
1962
|
+
const mode = args.mode === "lightweight" ? "lightweight" : (args.mode === "auto" ? "auto" : "default");
|
|
1963
|
+
const fusionMode = normalizeFusionMode(args.fusionMode);
|
|
1964
|
+
const trackHits = args.trackHits !== false;
|
|
1965
|
+
const finish = (payload, debug = {}) => {
|
|
1966
|
+
timing.total_ms = Date.now() - startedAt;
|
|
1967
|
+
return {
|
|
1968
|
+
...payload,
|
|
1969
|
+
channel_results: payload.channel_results || emptyChannelResults(),
|
|
1970
|
+
timing_ms: roundTiming(timing),
|
|
1971
|
+
debug: {
|
|
1972
|
+
mode,
|
|
1973
|
+
fusion_mode: fusionMode,
|
|
1974
|
+
track_hits: trackHits,
|
|
1975
|
+
planned_queries: debug.planned_queries || [],
|
|
1976
|
+
fulltext_fallback_used: Boolean(debug.fulltext_fallback_used),
|
|
1977
|
+
vector_source: debug.vector_source || "none",
|
|
1978
|
+
},
|
|
1979
|
+
};
|
|
1980
|
+
};
|
|
892
1981
|
if (!query) {
|
|
893
|
-
return {
|
|
1982
|
+
return finish({
|
|
1983
|
+
results: [],
|
|
1984
|
+
semantic_results: [],
|
|
1985
|
+
keyword_results: [],
|
|
1986
|
+
strategy: "vector_sentence_and_keyword_parallel",
|
|
1987
|
+
});
|
|
894
1988
|
}
|
|
1989
|
+
const lightweightMode = mode === "lightweight";
|
|
1990
|
+
const externalModelMode = mode === "default";
|
|
1991
|
+
const loadDocsStartedAt = Date.now();
|
|
895
1992
|
const docs = loadAllDocuments();
|
|
1993
|
+
appendTiming(timing, "load_docs_ms", loadDocsStartedAt);
|
|
1994
|
+
const hitStatsStartedAt = Date.now();
|
|
896
1995
|
const hitStats = loadHitStats();
|
|
1996
|
+
appendTiming(timing, "load_hit_stats_ms", hitStatsStartedAt);
|
|
1997
|
+
const planningStartedAt = Date.now();
|
|
897
1998
|
const intent = classifyIntent(query);
|
|
898
1999
|
const preferredTypes = preferredEventTypes(intent);
|
|
2000
|
+
const plannedQueriesRaw = lightweightMode ? [query] : planQueryKeywords(query);
|
|
2001
|
+
const plannedQueries = plannedQueriesRaw.length > 0 ? plannedQueriesRaw : [query];
|
|
2002
|
+
appendTiming(timing, "query_planning_ms", planningStartedAt);
|
|
2003
|
+
const summaryChannelWeight = 1;
|
|
2004
|
+
const fulltextChannelWeight = 0.35;
|
|
2005
|
+
const maxCandidatePool = Math.max(1, Math.max(args.topK, 20));
|
|
899
2006
|
let queryEmbedding = null;
|
|
2007
|
+
let queryEmbeddingNorm = 0;
|
|
900
2008
|
const embeddingModel = options.embedding?.model || "";
|
|
901
2009
|
const embeddingApiKey = options.embedding?.apiKey || "";
|
|
902
2010
|
const embeddingBaseUrl = normalizeBaseUrl(options.embedding?.baseURL || options.embedding?.baseUrl);
|
|
903
|
-
if (embeddingModel && embeddingApiKey && embeddingBaseUrl) {
|
|
2011
|
+
if (externalModelMode && embeddingModel && embeddingApiKey && embeddingBaseUrl) {
|
|
2012
|
+
const embeddingStartedAt = Date.now();
|
|
904
2013
|
try {
|
|
905
2014
|
queryEmbedding = await requestEmbedding({
|
|
906
2015
|
text: query,
|
|
@@ -909,24 +2018,71 @@ function createReadStore(options) {
|
|
|
909
2018
|
baseUrl: embeddingBaseUrl,
|
|
910
2019
|
dimensions: options.embedding?.dimensions,
|
|
911
2020
|
});
|
|
2021
|
+
queryEmbeddingNorm = queryEmbedding && queryEmbedding.length > 0 ? vectorNorm(queryEmbedding) : 0;
|
|
912
2022
|
}
|
|
913
2023
|
catch (error) {
|
|
914
2024
|
options.logger.warn(`Embedding query failed, fallback to lexical search: ${error}`);
|
|
915
2025
|
}
|
|
2026
|
+
finally {
|
|
2027
|
+
appendTiming(timing, "embedding_ms", embeddingStartedAt);
|
|
2028
|
+
}
|
|
2029
|
+
}
|
|
2030
|
+
let vectorDocsFromLance = [];
|
|
2031
|
+
if (queryEmbedding && queryEmbedding.length > 0) {
|
|
2032
|
+
const lanceStartedAt = Date.now();
|
|
2033
|
+
vectorDocsFromLance = await searchLanceDb({ memoryRoot, queryEmbedding, limit: Math.max(20, args.topK * 8), logger: options.logger });
|
|
2034
|
+
appendTiming(timing, "lance_vector_ms", lanceStartedAt);
|
|
2035
|
+
}
|
|
2036
|
+
let vectorFallbackIndex = null;
|
|
2037
|
+
let vectorDocsFallback = [];
|
|
2038
|
+
if (vectorDocsFromLance.length === 0) {
|
|
2039
|
+
const indexStartedAt = Date.now();
|
|
2040
|
+
vectorFallbackIndex = loadVectorFallbackCached();
|
|
2041
|
+
vectorDocsFallback = selectVectorFallbackDocs(vectorFallbackIndex, queryEmbedding, queryEmbeddingNorm, Math.max(20, args.topK * 8));
|
|
2042
|
+
appendTiming(timing, "vector_index_ms", indexStartedAt);
|
|
916
2043
|
}
|
|
917
|
-
const
|
|
918
|
-
?
|
|
919
|
-
:
|
|
920
|
-
const vectorDocsFallback = vectorDocsFromLance.length > 0
|
|
921
|
-
? []
|
|
922
|
-
: loadVectorFallbackCached();
|
|
2044
|
+
const vectorSource = vectorDocsFromLance.length > 0
|
|
2045
|
+
? "lancedb"
|
|
2046
|
+
: (vectorDocsFallback.length > 0 ? "jsonl_index" : "none");
|
|
923
2047
|
const vectorDocs = [...vectorDocsFromLance, ...vectorDocsFallback];
|
|
2048
|
+
const archiveLinkStartedAt = Date.now();
|
|
2049
|
+
const archiveSourceById = new Map();
|
|
2050
|
+
for (const doc of docs) {
|
|
2051
|
+
if (doc.source !== "sessions_archive")
|
|
2052
|
+
continue;
|
|
2053
|
+
const key = (doc.sourceMemoryId || doc.id || "").trim();
|
|
2054
|
+
if (!key)
|
|
2055
|
+
continue;
|
|
2056
|
+
archiveSourceById.set(key, {
|
|
2057
|
+
sourceText: doc.sourceText,
|
|
2058
|
+
summaryText: doc.summaryText || doc.text,
|
|
2059
|
+
sourceFile: doc.sourceFile,
|
|
2060
|
+
});
|
|
2061
|
+
}
|
|
2062
|
+
for (const doc of vectorDocs) {
|
|
2063
|
+
const key = (doc.sourceMemoryId || "").trim();
|
|
2064
|
+
if (!key)
|
|
2065
|
+
continue;
|
|
2066
|
+
const linked = archiveSourceById.get(key);
|
|
2067
|
+
if (!linked)
|
|
2068
|
+
continue;
|
|
2069
|
+
if (!doc.sourceText && linked.sourceText) {
|
|
2070
|
+
doc.sourceText = linked.sourceText;
|
|
2071
|
+
}
|
|
2072
|
+
if (!doc.summaryText && linked.summaryText) {
|
|
2073
|
+
doc.summaryText = linked.summaryText;
|
|
2074
|
+
}
|
|
2075
|
+
if (!doc.sourceFile && linked.sourceFile) {
|
|
2076
|
+
doc.sourceFile = linked.sourceFile;
|
|
2077
|
+
}
|
|
2078
|
+
}
|
|
2079
|
+
appendTiming(timing, "archive_link_ms", archiveLinkStartedAt);
|
|
924
2080
|
const graphDocs = docs
|
|
925
|
-
.filter(doc =>
|
|
2081
|
+
.filter(doc => doc.source.startsWith("sessions_graph"))
|
|
926
2082
|
.map(doc => {
|
|
927
2083
|
const graphText = [
|
|
928
2084
|
doc.text,
|
|
929
|
-
...(doc.relations || []).map(relation => `${relation.source} ${relation.type} ${relation.target}`),
|
|
2085
|
+
...(doc.relations || []).map(relation => `${relation.source} ${relation.type} ${relation.target} ${relation.fact_status || doc.factStatus || "active"}`),
|
|
930
2086
|
].join(" | ");
|
|
931
2087
|
return {
|
|
932
2088
|
...doc,
|
|
@@ -934,157 +2090,362 @@ function createReadStore(options) {
|
|
|
934
2090
|
};
|
|
935
2091
|
});
|
|
936
2092
|
const rulesDocs = docs.filter(doc => doc.source === "CORTEX_RULES.md");
|
|
937
|
-
const archiveDocs = docs.filter(doc => doc.source.
|
|
938
|
-
const bm25Terms = tokenize(query);
|
|
2093
|
+
const archiveDocs = docs.filter(doc => doc.source === "sessions_active" || doc.source === "sessions_archive");
|
|
939
2094
|
const bm25Corpus = [...rulesDocs, ...archiveDocs, ...vectorDocs, ...graphDocs];
|
|
940
2095
|
const bm25Signature = `${docsCache?.signature || "na"}|vector:${vectorDocs.length}:${vectorDocs.slice(0, 40).map(item => `${item.id}:${item.text.length}`).join(",")}`;
|
|
941
|
-
const
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
2096
|
+
const rankByQuery = (plannedQuery, includeFulltext) => {
|
|
2097
|
+
const bm25Terms = uniqueStrings(tokenize(plannedQuery));
|
|
2098
|
+
const bm25StatsSummary = buildBm25Stats(bm25Corpus, bm25Terms, doc => getBm25Tokens(doc, bm25Signature, "summary"));
|
|
2099
|
+
const bm25StatsFulltext = includeFulltext
|
|
2100
|
+
? buildBm25Stats(bm25Corpus, bm25Terms, doc => getBm25Tokens(doc, bm25Signature, "fulltext"))
|
|
2101
|
+
: { avgDocLen: 1, docFreq: new Map() };
|
|
2102
|
+
const channels = {
|
|
2103
|
+
rules: [],
|
|
2104
|
+
archive: [],
|
|
2105
|
+
vector: [],
|
|
2106
|
+
graph: [],
|
|
2107
|
+
};
|
|
2108
|
+
const evaluateDoc = (doc, source) => {
|
|
2109
|
+
const isSessionMemory = doc.source === "sessions_active" || doc.source === "sessions_archive";
|
|
2110
|
+
const summaryText = isSessionMemory
|
|
2111
|
+
? (doc.summaryText || "").trim()
|
|
2112
|
+
: (doc.summaryText || doc.text || "").trim();
|
|
2113
|
+
const fulltextText = (doc.sourceText || "").trim();
|
|
2114
|
+
const summaryBm25 = bm25Score({
|
|
2115
|
+
queryTerms: bm25Terms,
|
|
2116
|
+
docText: summaryText,
|
|
2117
|
+
docTokens: getBm25Tokens(doc, bm25Signature, "summary"),
|
|
2118
|
+
docCount: bm25Corpus.length,
|
|
2119
|
+
avgDocLen: bm25StatsSummary.avgDocLen,
|
|
2120
|
+
docFreq: bm25StatsSummary.docFreq,
|
|
2121
|
+
});
|
|
2122
|
+
const fulltextBm25 = includeFulltext
|
|
2123
|
+
? bm25Score({
|
|
2124
|
+
queryTerms: bm25Terms,
|
|
2125
|
+
docText: fulltextText,
|
|
2126
|
+
docTokens: getBm25Tokens(doc, bm25Signature, "fulltext"),
|
|
2127
|
+
docCount: bm25Corpus.length,
|
|
2128
|
+
avgDocLen: bm25StatsFulltext.avgDocLen,
|
|
2129
|
+
docFreq: bm25StatsFulltext.docFreq,
|
|
2130
|
+
})
|
|
2131
|
+
: 0;
|
|
2132
|
+
const summaryCombined = scoreText(plannedQuery, summaryText) + summaryBm25 * readTuning.scoring.bm25Scale;
|
|
2133
|
+
const fulltextCombined = includeFulltext
|
|
2134
|
+
? scoreText(plannedQuery, fulltextText) + fulltextBm25 * readTuning.scoring.bm25Scale
|
|
2135
|
+
: 0;
|
|
2136
|
+
const lexicalCombined = summaryCombined * summaryChannelWeight + fulltextCombined * fulltextChannelWeight;
|
|
2137
|
+
const semantic = plannedQuery === query && queryEmbedding && Array.isArray(doc.embedding) && doc.embedding.length > 0
|
|
2138
|
+
? Math.max(0, cosineSimilarityWithNorm(queryEmbedding, doc.embedding, queryEmbeddingNorm, doc.embeddingNorm || vectorNorm(doc.embedding)) * 5)
|
|
2139
|
+
: 0;
|
|
2140
|
+
if (lexicalCombined <= 0 && semantic <= 0) {
|
|
2141
|
+
return null;
|
|
2142
|
+
}
|
|
2143
|
+
if (source === "graph") {
|
|
2144
|
+
const status = docFactStatus(doc);
|
|
2145
|
+
if (status === "superseded" || status === "rejected") {
|
|
2146
|
+
return null;
|
|
2147
|
+
}
|
|
2148
|
+
}
|
|
2149
|
+
const recency = recencyScore(doc.timestamp, readTuning.recency.buckets);
|
|
2150
|
+
const quality = typeof doc.qualityScore === "number" ? Math.max(0, Math.min(1, doc.qualityScore)) : 0.5;
|
|
2151
|
+
const typeMatch = preferredTypes.length > 0 && doc.eventType
|
|
2152
|
+
? (preferredTypes.includes(doc.eventType) ? 1 : 0)
|
|
2153
|
+
: 0.5;
|
|
2154
|
+
const graphMatch = source === "graph" ? 1 : 0;
|
|
2155
|
+
const sourceBaseWeight = sourceWeight(source, intent);
|
|
2156
|
+
const sourceConfigWeight = customChannelWeight(source, options.fusion);
|
|
2157
|
+
const lengthNorm = lengthNormalizeFactor(doc, options.fusion);
|
|
2158
|
+
const baseWeighted = (readTuning.scoring.lexicalWeight * lexicalCombined +
|
|
2159
|
+
readTuning.scoring.semanticWeight * (semantic * lengthNorm) +
|
|
2160
|
+
readTuning.scoring.recencyWeight * recency +
|
|
2161
|
+
readTuning.scoring.qualityWeight * quality +
|
|
2162
|
+
readTuning.scoring.typeMatchWeight * typeMatch +
|
|
2163
|
+
readTuning.scoring.graphMatchWeight * graphMatch) * sourceBaseWeight * sourceConfigWeight;
|
|
2164
|
+
const decayFactor = computeDecayFactor(doc.id, doc.eventType, doc.timestamp, options.memoryDecay, hitStats);
|
|
2165
|
+
const weighted = baseWeighted * decayFactor;
|
|
2166
|
+
return {
|
|
2167
|
+
doc,
|
|
2168
|
+
source,
|
|
2169
|
+
lexical: lexicalCombined,
|
|
2170
|
+
bm25: summaryBm25 + fulltextBm25,
|
|
2171
|
+
semantic,
|
|
2172
|
+
recency,
|
|
2173
|
+
quality,
|
|
2174
|
+
typeMatch,
|
|
2175
|
+
graphMatch,
|
|
2176
|
+
decayFactor,
|
|
2177
|
+
weighted,
|
|
2178
|
+
summaryCombined,
|
|
2179
|
+
fulltextCombined,
|
|
2180
|
+
};
|
|
2181
|
+
};
|
|
2182
|
+
for (const doc of rulesDocs) {
|
|
2183
|
+
const candidate = evaluateDoc(doc, "rules");
|
|
2184
|
+
if (candidate)
|
|
2185
|
+
channels.rules.push(candidate);
|
|
2186
|
+
}
|
|
2187
|
+
for (const doc of archiveDocs) {
|
|
2188
|
+
const candidate = evaluateDoc(doc, "archive");
|
|
2189
|
+
if (candidate)
|
|
2190
|
+
channels.archive.push(candidate);
|
|
2191
|
+
}
|
|
2192
|
+
for (const doc of vectorDocs) {
|
|
2193
|
+
const candidate = evaluateDoc(doc, "vector");
|
|
2194
|
+
if (candidate)
|
|
2195
|
+
channels.vector.push(candidate);
|
|
2196
|
+
}
|
|
2197
|
+
for (const doc of graphDocs) {
|
|
2198
|
+
const candidate = evaluateDoc(doc, "graph");
|
|
2199
|
+
if (candidate)
|
|
2200
|
+
channels.graph.push(candidate);
|
|
2201
|
+
}
|
|
2202
|
+
const rrfMap = new Map();
|
|
2203
|
+
const weightedMap = new Map();
|
|
2204
|
+
const rrfK = readTuning.rrf.k;
|
|
2205
|
+
for (const key of Object.keys(channels)) {
|
|
2206
|
+
const list = channels[key].sort((a, b) => b.weighted - a.weighted);
|
|
2207
|
+
const capped = list.slice(0, channelQuota(key, args.topK, options.fusion));
|
|
2208
|
+
for (let i = 0; i < capped.length; i += 1) {
|
|
2209
|
+
const candidate = capped[i];
|
|
2210
|
+
const rrf = 1 / (rrfK + i + 1);
|
|
2211
|
+
const mergeKey = mergeKeyFromDoc(candidate.doc);
|
|
2212
|
+
rrfMap.set(mergeKey, (rrfMap.get(mergeKey) || 0) + rrf);
|
|
2213
|
+
const current = weightedMap.get(mergeKey);
|
|
2214
|
+
if (!current || candidate.weighted > current.weighted) {
|
|
2215
|
+
weightedMap.set(mergeKey, candidate);
|
|
2216
|
+
}
|
|
2217
|
+
}
|
|
2218
|
+
}
|
|
2219
|
+
return [...weightedMap.entries()]
|
|
2220
|
+
.map(([mergeKey, candidate]) => {
|
|
2221
|
+
const wikiRefs = docWikiRefs(candidate.doc);
|
|
2222
|
+
return {
|
|
2223
|
+
id: candidate.doc.id,
|
|
2224
|
+
merge_key: mergeKey,
|
|
2225
|
+
source_memory_id: candidate.doc.sourceMemoryId || "",
|
|
2226
|
+
source_memory_canonical_id: candidate.doc.sourceMemoryCanonicalId || "",
|
|
2227
|
+
source_event_id: candidate.doc.sourceEventId || "",
|
|
2228
|
+
source_field: candidate.doc.sourceField || "",
|
|
2229
|
+
text: candidate.doc.summaryText || candidate.doc.text,
|
|
2230
|
+
source_text: candidate.doc.sourceText ? candidate.doc.sourceText.slice(0, 4000) : "",
|
|
2231
|
+
source_excerpt: candidate.doc.sourceText ? candidate.doc.sourceText.slice(0, 360) : "",
|
|
2232
|
+
source_file: candidate.doc.sourceFile || "",
|
|
2233
|
+
source: candidate.doc.source,
|
|
2234
|
+
layer: candidate.doc.layer || "",
|
|
2235
|
+
event_type: candidate.doc.eventType || "",
|
|
2236
|
+
fact_status: docFactStatus(candidate.doc),
|
|
2237
|
+
wiki_ref: wikiRefs[0] || "",
|
|
2238
|
+
wiki_refs: wikiRefs,
|
|
2239
|
+
quality_score: candidate.quality,
|
|
2240
|
+
timestamp: candidate.doc.timestamp ? new Date(candidate.doc.timestamp).toISOString() : "",
|
|
2241
|
+
evidence_ids: docEvidenceIds(candidate.doc),
|
|
2242
|
+
score: candidate.weighted + (rrfMap.get(mergeKey) || 0) * readTuning.rrf.weight,
|
|
2243
|
+
score_breakdown: {
|
|
2244
|
+
lexical: Number(candidate.lexical.toFixed(4)),
|
|
2245
|
+
bm25: Number(candidate.bm25.toFixed(4)),
|
|
2246
|
+
semantic: Number(candidate.semantic.toFixed(4)),
|
|
2247
|
+
recency: Number(candidate.recency.toFixed(4)),
|
|
2248
|
+
quality: Number(candidate.quality.toFixed(4)),
|
|
2249
|
+
type: Number(candidate.typeMatch.toFixed(4)),
|
|
2250
|
+
graph: Number(candidate.graphMatch.toFixed(4)),
|
|
2251
|
+
summary: Number(candidate.summaryCombined.toFixed(4)),
|
|
2252
|
+
fulltext: Number(candidate.fulltextCombined.toFixed(4)),
|
|
2253
|
+
decay: Number(candidate.decayFactor.toFixed(4)),
|
|
2254
|
+
rrf: Number(((rrfMap.get(mergeKey) || 0) * readTuning.rrf.weight).toFixed(4)),
|
|
2255
|
+
weighted: Number(candidate.weighted.toFixed(4)),
|
|
2256
|
+
},
|
|
2257
|
+
reason_tags: [
|
|
2258
|
+
`intent:${intent.toLowerCase()}`,
|
|
2259
|
+
candidate.summaryCombined > 0 ? "summary_hit" : "",
|
|
2260
|
+
candidate.fulltextCombined > 0 ? "fulltext_hit" : "",
|
|
2261
|
+
candidate.semantic > 0 ? "vector_hit" : "",
|
|
2262
|
+
candidate.lexical > 0 ? "lexical_hit" : "",
|
|
2263
|
+
candidate.typeMatch >= 1 ? "event_type_match" : "event_type_weak",
|
|
2264
|
+
candidate.recency >= 0.8 ? "recent" : "historical",
|
|
2265
|
+
candidate.quality >= 0.7 ? "high_quality" : "normal_quality",
|
|
2266
|
+
candidate.decayFactor < 1 ? `decay:${candidate.decayFactor.toFixed(3)}` : "decay:1.000",
|
|
2267
|
+
`source:${candidate.source}`,
|
|
2268
|
+
`merge_key:${mergeKey}`,
|
|
2269
|
+
`query_term:${plannedQuery}`,
|
|
2270
|
+
].filter(Boolean),
|
|
2271
|
+
matched_keywords: [plannedQuery],
|
|
2272
|
+
};
|
|
2273
|
+
})
|
|
2274
|
+
.sort((a, b) => b.score - a.score)
|
|
2275
|
+
.slice(0, maxCandidatePool);
|
|
948
2276
|
};
|
|
949
|
-
const
|
|
950
|
-
|
|
951
|
-
const
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
2277
|
+
const rankingStartedAt = Date.now();
|
|
2278
|
+
const queryRuns = await Promise.all(plannedQueries.map(async (plannedQuery) => {
|
|
2279
|
+
const stage1 = rankByQuery(plannedQuery, false);
|
|
2280
|
+
const needFallback = shouldTriggerFulltextFallback(stage1, args.topK);
|
|
2281
|
+
if (!needFallback || lightweightMode) {
|
|
2282
|
+
return { plannedQuery, ranked: stage1, fulltextFallback: false };
|
|
2283
|
+
}
|
|
2284
|
+
const stage2 = rankByQuery(plannedQuery, true);
|
|
2285
|
+
const merged = new Map();
|
|
2286
|
+
for (const item of [...stage1, ...stage2]) {
|
|
2287
|
+
const mergeKey = item.merge_key || item.id || "";
|
|
2288
|
+
const existing = merged.get(mergeKey);
|
|
2289
|
+
if (!existing) {
|
|
2290
|
+
merged.set(mergeKey, { ...item });
|
|
2291
|
+
continue;
|
|
2292
|
+
}
|
|
2293
|
+
const best = Number(item.score || 0) > Number(existing.score || 0)
|
|
2294
|
+
? { ...existing, ...item }
|
|
2295
|
+
: { ...item, ...existing };
|
|
2296
|
+
const mergedReasonTags = uniqueStrings([
|
|
2297
|
+
...(Array.isArray(existing.reason_tags) ? existing.reason_tags.map(v => String(v)) : []),
|
|
2298
|
+
...(Array.isArray(item.reason_tags) ? item.reason_tags.map(v => String(v)) : []),
|
|
2299
|
+
]);
|
|
2300
|
+
const mergedEvidenceIds = uniqueStrings([
|
|
2301
|
+
...(Array.isArray(existing.evidence_ids) ? existing.evidence_ids.map(v => String(v)) : []),
|
|
2302
|
+
...(Array.isArray(item.evidence_ids) ? item.evidence_ids.map(v => String(v)) : []),
|
|
2303
|
+
]);
|
|
2304
|
+
const mergedWikiRefs = uniqueStrings([
|
|
2305
|
+
...(Array.isArray(existing.wiki_refs) ? existing.wiki_refs.map(v => String(v)) : []),
|
|
2306
|
+
...(Array.isArray(item.wiki_refs) ? item.wiki_refs.map(v => String(v)) : []),
|
|
2307
|
+
existing.wiki_ref || "",
|
|
2308
|
+
item.wiki_ref || "",
|
|
2309
|
+
]);
|
|
2310
|
+
const mergedKeywords = uniqueStrings([
|
|
2311
|
+
...(Array.isArray(existing.matched_keywords) ? existing.matched_keywords.map(v => String(v)) : []),
|
|
2312
|
+
...(Array.isArray(item.matched_keywords) ? item.matched_keywords.map(v => String(v)) : []),
|
|
2313
|
+
]);
|
|
2314
|
+
merged.set(mergeKey, {
|
|
2315
|
+
...best,
|
|
2316
|
+
score: Math.max(existing.score || 0, item.score || 0),
|
|
2317
|
+
reason_tags: mergedReasonTags,
|
|
2318
|
+
evidence_ids: mergedEvidenceIds,
|
|
2319
|
+
wiki_ref: mergedWikiRefs[0] || "",
|
|
2320
|
+
wiki_refs: mergedWikiRefs,
|
|
2321
|
+
matched_keywords: mergedKeywords,
|
|
2322
|
+
});
|
|
965
2323
|
}
|
|
966
|
-
const recency = recencyScore(doc.timestamp);
|
|
967
|
-
const quality = typeof doc.qualityScore === "number" ? Math.max(0, Math.min(1, doc.qualityScore)) : 0.5;
|
|
968
|
-
const typeMatch = preferredTypes.length > 0 && doc.eventType
|
|
969
|
-
? (preferredTypes.includes(doc.eventType) ? 1 : 0)
|
|
970
|
-
: 0.5;
|
|
971
|
-
const graphMatch = source === "graph" ? 1 : 0;
|
|
972
|
-
const sourceBaseWeight = sourceWeight(source, intent);
|
|
973
|
-
const sourceConfigWeight = customChannelWeight(source, options.fusion);
|
|
974
|
-
const lengthNorm = lengthNormalizeFactor(doc, options.fusion);
|
|
975
|
-
const baseWeighted = (0.2 * lexicalCombined +
|
|
976
|
-
0.3 * (semantic * lengthNorm) +
|
|
977
|
-
0.1 * recency +
|
|
978
|
-
0.15 * quality +
|
|
979
|
-
0.15 * typeMatch +
|
|
980
|
-
0.1 * graphMatch) * sourceBaseWeight * sourceConfigWeight;
|
|
981
|
-
const decayFactor = computeDecayFactor(doc.id, doc.eventType, doc.timestamp, options.memoryDecay, hitStats);
|
|
982
|
-
const weighted = baseWeighted * decayFactor;
|
|
983
2324
|
return {
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
2325
|
+
plannedQuery,
|
|
2326
|
+
ranked: [...merged.values()]
|
|
2327
|
+
.sort((a, b) => Number(b.score || 0) - Number(a.score || 0))
|
|
2328
|
+
.slice(0, maxCandidatePool)
|
|
2329
|
+
.map((item) => ({
|
|
2330
|
+
...item,
|
|
2331
|
+
reason_tags: uniqueStrings([
|
|
2332
|
+
...(Array.isArray(item.reason_tags) ? item.reason_tags.map((v) => String(v)) : []),
|
|
2333
|
+
"fulltext_fallback",
|
|
2334
|
+
]),
|
|
2335
|
+
})),
|
|
2336
|
+
fulltextFallback: true,
|
|
995
2337
|
};
|
|
996
|
-
};
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
if (candidate)
|
|
1010
|
-
channels.vector.push(candidate);
|
|
1011
|
-
}
|
|
1012
|
-
for (const doc of graphDocs) {
|
|
1013
|
-
const candidate = evaluateDoc(doc, "graph");
|
|
1014
|
-
if (candidate)
|
|
1015
|
-
channels.graph.push(candidate);
|
|
1016
|
-
}
|
|
1017
|
-
for (const key of Object.keys(channels)) {
|
|
1018
|
-
channels[key].sort((a, b) => b.weighted - a.weighted);
|
|
1019
|
-
combinedCandidates.push(...channels[key].slice(0, channelQuota(key, args.topK, options.fusion)));
|
|
1020
|
-
}
|
|
1021
|
-
const rrfMap = new Map();
|
|
1022
|
-
const weightedMap = new Map();
|
|
1023
|
-
const rrfK = 60;
|
|
1024
|
-
for (const key of Object.keys(channels)) {
|
|
1025
|
-
const list = channels[key];
|
|
1026
|
-
for (let i = 0; i < list.length; i += 1) {
|
|
1027
|
-
const candidate = list[i];
|
|
1028
|
-
const rrf = 1 / (rrfK + i + 1);
|
|
1029
|
-
const mergeKey = mergeKeyFromDoc(candidate.doc);
|
|
1030
|
-
rrfMap.set(mergeKey, (rrfMap.get(mergeKey) || 0) + rrf);
|
|
1031
|
-
const current = weightedMap.get(mergeKey);
|
|
1032
|
-
if (!current || candidate.weighted > current.weighted) {
|
|
1033
|
-
weightedMap.set(mergeKey, candidate);
|
|
2338
|
+
}));
|
|
2339
|
+
const mergedByQuery = new Map();
|
|
2340
|
+
for (const run of queryRuns) {
|
|
2341
|
+
for (const item of run.ranked) {
|
|
2342
|
+
const mergeKey = item.merge_key || item.id || "";
|
|
2343
|
+
const existing = mergedByQuery.get(mergeKey);
|
|
2344
|
+
if (!existing) {
|
|
2345
|
+
mergedByQuery.set(mergeKey, {
|
|
2346
|
+
...item,
|
|
2347
|
+
matched_keywords: uniqueStrings([run.plannedQuery]),
|
|
2348
|
+
fulltext_fallback_used: run.fulltextFallback,
|
|
2349
|
+
});
|
|
2350
|
+
continue;
|
|
1034
2351
|
}
|
|
2352
|
+
const mergedReasonTags = uniqueStrings([
|
|
2353
|
+
...(Array.isArray(existing.reason_tags) ? existing.reason_tags.map(v => String(v)) : []),
|
|
2354
|
+
...(Array.isArray(item.reason_tags) ? item.reason_tags.map(v => String(v)) : []),
|
|
2355
|
+
]);
|
|
2356
|
+
const mergedEvidenceIds = uniqueStrings([
|
|
2357
|
+
...(Array.isArray(existing.evidence_ids) ? existing.evidence_ids.map(v => String(v)) : []),
|
|
2358
|
+
...(Array.isArray(item.evidence_ids) ? item.evidence_ids.map(v => String(v)) : []),
|
|
2359
|
+
]);
|
|
2360
|
+
const mergedWikiRefs = uniqueStrings([
|
|
2361
|
+
...(Array.isArray(existing.wiki_refs) ? existing.wiki_refs.map(v => String(v)) : []),
|
|
2362
|
+
...(Array.isArray(item.wiki_refs) ? item.wiki_refs.map(v => String(v)) : []),
|
|
2363
|
+
existing.wiki_ref || "",
|
|
2364
|
+
item.wiki_ref || "",
|
|
2365
|
+
]);
|
|
2366
|
+
const matchedKeywords = uniqueStrings([
|
|
2367
|
+
...(Array.isArray(existing.matched_keywords) ? existing.matched_keywords.map(v => String(v)) : []),
|
|
2368
|
+
run.plannedQuery,
|
|
2369
|
+
]);
|
|
2370
|
+
const preferred = Number(item.score || 0) > Number(existing.score || 0)
|
|
2371
|
+
? { ...existing, ...item }
|
|
2372
|
+
: { ...item, ...existing };
|
|
2373
|
+
mergedByQuery.set(mergeKey, {
|
|
2374
|
+
...preferred,
|
|
2375
|
+
score: Math.max(existing.score || 0, item.score || 0),
|
|
2376
|
+
reason_tags: mergedReasonTags,
|
|
2377
|
+
evidence_ids: mergedEvidenceIds,
|
|
2378
|
+
wiki_ref: mergedWikiRefs[0] || "",
|
|
2379
|
+
wiki_refs: mergedWikiRefs,
|
|
2380
|
+
matched_keywords: matchedKeywords,
|
|
2381
|
+
fulltext_fallback_used: Boolean(existing.fulltext_fallback_used) || run.fulltextFallback,
|
|
2382
|
+
});
|
|
1035
2383
|
}
|
|
1036
2384
|
}
|
|
1037
|
-
const
|
|
1038
|
-
.map((
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
2385
|
+
const lexicalRanked = [...mergedByQuery.values()]
|
|
2386
|
+
.map((item) => {
|
|
2387
|
+
const matchedKeywords = Array.isArray(item.matched_keywords)
|
|
2388
|
+
? uniqueStrings(item.matched_keywords.map(v => String(v)))
|
|
2389
|
+
: [query];
|
|
2390
|
+
const keywordBonus = Math.max(0, matchedKeywords.length - 1) * 0.12;
|
|
2391
|
+
const boosted = withRecencyBoost(Number(item.score || 0) + keywordBonus, typeof item.timestamp === "string" ? Date.parse(item.timestamp) : undefined, readTuning.recency.buckets);
|
|
2392
|
+
return {
|
|
2393
|
+
...item,
|
|
2394
|
+
matched_keywords: matchedKeywords,
|
|
2395
|
+
query_plan_keywords: plannedQueries,
|
|
2396
|
+
score: Number(boosted.toFixed(4)),
|
|
2397
|
+
reason_tags: uniqueStrings([
|
|
2398
|
+
...(Array.isArray(item.reason_tags) ? item.reason_tags.map((v) => String(v)) : []),
|
|
2399
|
+
`keyword_hits:${matchedKeywords.length}`,
|
|
2400
|
+
`query_plan:${plannedQueries.length}`,
|
|
2401
|
+
Boolean(item.fulltext_fallback_used) ? "fulltext_fallback_used" : "",
|
|
2402
|
+
]),
|
|
2403
|
+
};
|
|
2404
|
+
})
|
|
2405
|
+
.sort((a, b) => Number(b.score || 0) - Number(a.score || 0))
|
|
2406
|
+
.slice(0, maxCandidatePool);
|
|
2407
|
+
const fulltextFallbackUsed = queryRuns.some(run => run.fulltextFallback);
|
|
2408
|
+
appendTiming(timing, "ranking_ms", rankingStartedAt);
|
|
2409
|
+
const isVectorSource = (value) => value.startsWith("vector_");
|
|
2410
|
+
const semanticResults = lexicalRanked
|
|
2411
|
+
.filter(item => isVectorSource(item.source) && Array.isArray(item.reason_tags) && item.reason_tags.includes("vector_hit"))
|
|
2412
|
+
.slice(0, Math.max(1, args.topK));
|
|
2413
|
+
const keywordResults = lexicalRanked
|
|
2414
|
+
.filter(item => isVectorSource(item.source) && Array.isArray(item.reason_tags) && item.reason_tags.includes("lexical_hit"))
|
|
2415
|
+
.slice(0, Math.max(1, args.topK));
|
|
2416
|
+
const channelLimit = Math.max(1, args.topK);
|
|
2417
|
+
const channelResults = {
|
|
2418
|
+
vector_semantic: semanticResults,
|
|
2419
|
+
vector_keyword: keywordResults,
|
|
2420
|
+
keyword: lexicalRanked
|
|
2421
|
+
.filter(item => Array.isArray(item.reason_tags) && item.reason_tags.includes("lexical_hit"))
|
|
2422
|
+
.slice(0, channelLimit),
|
|
2423
|
+
semantic: lexicalRanked
|
|
2424
|
+
.filter(item => Array.isArray(item.reason_tags) && item.reason_tags.includes("vector_hit"))
|
|
2425
|
+
.slice(0, channelLimit),
|
|
2426
|
+
archive: lexicalRanked
|
|
2427
|
+
.filter(item => item.source === "sessions_active" || item.source === "sessions_archive")
|
|
2428
|
+
.slice(0, channelLimit),
|
|
2429
|
+
graph: lexicalRanked
|
|
2430
|
+
.filter(item => item.source.startsWith("sessions_graph"))
|
|
2431
|
+
.slice(0, channelLimit),
|
|
2432
|
+
rules: lexicalRanked
|
|
2433
|
+
.filter(item => item.source === "CORTEX_RULES.md")
|
|
2434
|
+
.slice(0, channelLimit),
|
|
2435
|
+
vector: lexicalRanked
|
|
2436
|
+
.filter(item => isVectorSource(item.source))
|
|
2437
|
+
.slice(0, channelLimit),
|
|
2438
|
+
};
|
|
1080
2439
|
const rerankerModel = options.reranker?.model || "";
|
|
1081
2440
|
const rerankerApiKey = options.reranker?.apiKey || "";
|
|
1082
2441
|
const rerankerBaseUrl = normalizeBaseUrl(options.reranker?.baseURL || options.reranker?.baseUrl);
|
|
1083
|
-
const fusionEnabled = options.fusion?.enabled !== false;
|
|
2442
|
+
const fusionEnabled = externalModelMode && fusionMode !== "off" && options.fusion?.enabled !== false;
|
|
1084
2443
|
const llmModel = options.llm?.model || "";
|
|
1085
2444
|
const llmApiKey = options.llm?.apiKey || "";
|
|
1086
2445
|
const llmBaseUrl = normalizeBaseUrl(options.llm?.baseURL || options.llm?.baseUrl);
|
|
1087
|
-
const fusionAuthoritative =
|
|
2446
|
+
const fusionAuthoritative = fusionMode === "authoritative"
|
|
2447
|
+
? true
|
|
2448
|
+
: (fusionMode === "candidates" ? false : options.fusion?.authoritative !== false);
|
|
1088
2449
|
const skipRerankerForFusion = fusionEnabled && fusionAuthoritative && llmModel && llmApiKey && llmBaseUrl;
|
|
1089
2450
|
let rerankedSimple = lexicalRanked.map(item => ({
|
|
1090
2451
|
id: item.id,
|
|
@@ -1093,7 +2454,8 @@ function createReadStore(options) {
|
|
|
1093
2454
|
source: item.source,
|
|
1094
2455
|
score: item.score,
|
|
1095
2456
|
}));
|
|
1096
|
-
if (rerankerModel && rerankerApiKey && rerankerBaseUrl && lexicalRanked.length > 1 && !skipRerankerForFusion) {
|
|
2457
|
+
if (externalModelMode && rerankerModel && rerankerApiKey && rerankerBaseUrl && lexicalRanked.length > 1 && !skipRerankerForFusion) {
|
|
2458
|
+
const rerankStartedAt = Date.now();
|
|
1097
2459
|
try {
|
|
1098
2460
|
rerankedSimple = await requestRerank({
|
|
1099
2461
|
query,
|
|
@@ -1110,20 +2472,37 @@ function createReadStore(options) {
|
|
|
1110
2472
|
catch (error) {
|
|
1111
2473
|
options.logger.warn(`Reranker failed, keep hybrid ranking: ${error}`);
|
|
1112
2474
|
}
|
|
2475
|
+
finally {
|
|
2476
|
+
appendTiming(timing, "rerank_ms", rerankStartedAt);
|
|
2477
|
+
}
|
|
1113
2478
|
}
|
|
1114
2479
|
const ranked = rerankedSimple.slice(0, Math.max(1, args.topK)).map(item => {
|
|
1115
2480
|
const hit = lexicalRanked.find(entry => entry.id === item.id);
|
|
2481
|
+
const hitWikiRefs = uniqueStrings([
|
|
2482
|
+
...(Array.isArray(hit?.wiki_refs) ? hit.wiki_refs.map(v => String(v)) : []),
|
|
2483
|
+
hit?.wiki_ref || "",
|
|
2484
|
+
]);
|
|
1116
2485
|
return {
|
|
1117
2486
|
id: item.id,
|
|
1118
2487
|
merge_key: hit?.merge_key || item.merge_key || item.id,
|
|
1119
2488
|
source_memory_id: hit?.source_memory_id || "",
|
|
1120
2489
|
source_memory_canonical_id: hit?.source_memory_canonical_id || "",
|
|
2490
|
+
source_event_id: hit?.source_event_id || "",
|
|
2491
|
+
source_field: hit?.source_field || "",
|
|
2492
|
+
fulltext_event_id: (hit?.source_event_id || hit?.source_memory_id || item.id || ""),
|
|
1121
2493
|
text: item.text,
|
|
2494
|
+
source_text: hit?.source_text || "",
|
|
2495
|
+
source_excerpt: hit?.source_excerpt || "",
|
|
2496
|
+
source_file: hit?.source_file || "",
|
|
1122
2497
|
source: item.source,
|
|
1123
2498
|
layer: hit?.layer || "",
|
|
1124
2499
|
event_type: hit?.event_type || "",
|
|
2500
|
+
fact_status: hit?.fact_status || "active",
|
|
2501
|
+
wiki_ref: hitWikiRefs[0] || "",
|
|
2502
|
+
wiki_refs: hitWikiRefs,
|
|
1125
2503
|
quality_score: hit?.quality_score ?? 0,
|
|
1126
2504
|
timestamp: hit?.timestamp || "",
|
|
2505
|
+
evidence_ids: Array.isArray(hit?.evidence_ids) ? hit?.evidence_ids : [],
|
|
1127
2506
|
score: Number(item.score.toFixed(4)),
|
|
1128
2507
|
score_breakdown: hit?.score_breakdown || {},
|
|
1129
2508
|
reason_tags: Array.isArray(hit?.reason_tags) ? hit?.reason_tags : [],
|
|
@@ -1131,8 +2510,15 @@ function createReadStore(options) {
|
|
|
1131
2510
|
merge_key: hit?.merge_key || item.merge_key || item.id,
|
|
1132
2511
|
source_memory_id: hit?.source_memory_id || "",
|
|
1133
2512
|
source_memory_canonical_id: hit?.source_memory_canonical_id || "",
|
|
2513
|
+
source_event_id: hit?.source_event_id || "",
|
|
2514
|
+
source_field: hit?.source_field || "",
|
|
1134
2515
|
channel: item.source,
|
|
2516
|
+
source_file: hit?.source_file || "",
|
|
1135
2517
|
layer: hit?.layer || "",
|
|
2518
|
+
fact_status: hit?.fact_status || "active",
|
|
2519
|
+
wiki_ref: hitWikiRefs[0] || "",
|
|
2520
|
+
wiki_refs: hitWikiRefs,
|
|
2521
|
+
evidence_ids: Array.isArray(hit?.evidence_ids) ? hit?.evidence_ids : [],
|
|
1136
2522
|
score_breakdown: hit?.score_breakdown || {},
|
|
1137
2523
|
reason_tags: Array.isArray(hit?.reason_tags) ? hit?.reason_tags : [],
|
|
1138
2524
|
},
|
|
@@ -1152,12 +2538,22 @@ function createReadStore(options) {
|
|
|
1152
2538
|
merge_key: item.merge_key,
|
|
1153
2539
|
source_memory_id: item.source_memory_id,
|
|
1154
2540
|
source_memory_canonical_id: item.source_memory_canonical_id,
|
|
2541
|
+
source_event_id: item.source_event_id || "",
|
|
2542
|
+
source_field: item.source_field || "",
|
|
2543
|
+
fulltext_event_id: item.source_event_id || item.source_memory_id || item.id,
|
|
1155
2544
|
text: item.text,
|
|
2545
|
+
source_text: item.source_text || "",
|
|
2546
|
+
source_excerpt: item.source_excerpt || "",
|
|
2547
|
+
source_file: item.source_file || "",
|
|
1156
2548
|
source: item.source,
|
|
1157
2549
|
layer: item.layer,
|
|
1158
2550
|
event_type: item.event_type,
|
|
2551
|
+
fact_status: item.fact_status || "active",
|
|
2552
|
+
wiki_ref: item.wiki_ref || "",
|
|
2553
|
+
wiki_refs: Array.isArray(item.wiki_refs) ? item.wiki_refs : [],
|
|
1159
2554
|
quality_score: item.quality_score,
|
|
1160
2555
|
timestamp: item.timestamp,
|
|
2556
|
+
evidence_ids: Array.isArray(item.evidence_ids) ? item.evidence_ids : [],
|
|
1161
2557
|
score: Number(item.score.toFixed(4)),
|
|
1162
2558
|
score_breakdown: item.score_breakdown || {},
|
|
1163
2559
|
reason_tags: Array.isArray(item.reason_tags) ? item.reason_tags : [],
|
|
@@ -1165,8 +2561,15 @@ function createReadStore(options) {
|
|
|
1165
2561
|
merge_key: item.merge_key,
|
|
1166
2562
|
source_memory_id: item.source_memory_id,
|
|
1167
2563
|
source_memory_canonical_id: item.source_memory_canonical_id,
|
|
2564
|
+
source_event_id: item.source_event_id || "",
|
|
2565
|
+
source_field: item.source_field || "",
|
|
1168
2566
|
channel: item.source,
|
|
2567
|
+
source_file: item.source_file || "",
|
|
1169
2568
|
layer: item.layer,
|
|
2569
|
+
fact_status: item.fact_status || "active",
|
|
2570
|
+
wiki_ref: item.wiki_ref || "",
|
|
2571
|
+
wiki_refs: Array.isArray(item.wiki_refs) ? item.wiki_refs : [],
|
|
2572
|
+
evidence_ids: Array.isArray(item.evidence_ids) ? item.evidence_ids : [],
|
|
1170
2573
|
score_breakdown: item.score_breakdown || {},
|
|
1171
2574
|
reason_tags: Array.isArray(item.reason_tags) ? item.reason_tags : [],
|
|
1172
2575
|
},
|
|
@@ -1185,12 +2588,22 @@ function createReadStore(options) {
|
|
|
1185
2588
|
merge_key: item.merge_key,
|
|
1186
2589
|
source_memory_id: item.source_memory_id,
|
|
1187
2590
|
source_memory_canonical_id: item.source_memory_canonical_id,
|
|
2591
|
+
source_event_id: item.source_event_id || "",
|
|
2592
|
+
source_field: item.source_field || "",
|
|
2593
|
+
fulltext_event_id: item.source_event_id || item.source_memory_id || item.id,
|
|
1188
2594
|
text: item.text,
|
|
2595
|
+
source_text: item.source_text || "",
|
|
2596
|
+
source_excerpt: item.source_excerpt || "",
|
|
2597
|
+
source_file: item.source_file || "",
|
|
1189
2598
|
source: item.source,
|
|
1190
2599
|
layer: item.layer,
|
|
1191
2600
|
event_type: item.event_type,
|
|
2601
|
+
fact_status: item.fact_status || "active",
|
|
2602
|
+
wiki_ref: item.wiki_ref || "",
|
|
2603
|
+
wiki_refs: Array.isArray(item.wiki_refs) ? item.wiki_refs : [],
|
|
1192
2604
|
quality_score: item.quality_score,
|
|
1193
2605
|
timestamp: item.timestamp,
|
|
2606
|
+
evidence_ids: Array.isArray(item.evidence_ids) ? item.evidence_ids : [],
|
|
1194
2607
|
score: Number(item.score.toFixed(4)),
|
|
1195
2608
|
score_breakdown: item.score_breakdown || {},
|
|
1196
2609
|
reason_tags: Array.isArray(item.reason_tags) ? item.reason_tags : [],
|
|
@@ -1198,8 +2611,15 @@ function createReadStore(options) {
|
|
|
1198
2611
|
merge_key: item.merge_key,
|
|
1199
2612
|
source_memory_id: item.source_memory_id,
|
|
1200
2613
|
source_memory_canonical_id: item.source_memory_canonical_id,
|
|
2614
|
+
source_event_id: item.source_event_id || "",
|
|
2615
|
+
source_field: item.source_field || "",
|
|
1201
2616
|
channel: item.source,
|
|
2617
|
+
source_file: item.source_file || "",
|
|
1202
2618
|
layer: item.layer,
|
|
2619
|
+
fact_status: item.fact_status || "active",
|
|
2620
|
+
wiki_ref: item.wiki_ref || "",
|
|
2621
|
+
wiki_refs: Array.isArray(item.wiki_refs) ? item.wiki_refs : [],
|
|
2622
|
+
evidence_ids: Array.isArray(item.evidence_ids) ? item.evidence_ids : [],
|
|
1203
2623
|
score_breakdown: item.score_breakdown || {},
|
|
1204
2624
|
reason_tags: Array.isArray(item.reason_tags) ? item.reason_tags : [],
|
|
1205
2625
|
},
|
|
@@ -1207,7 +2627,16 @@ function createReadStore(options) {
|
|
|
1207
2627
|
}
|
|
1208
2628
|
}
|
|
1209
2629
|
ranked.sort((a, b) => b.score - a.score);
|
|
2630
|
+
const trackResultHits = (ids) => {
|
|
2631
|
+
if (!trackHits) {
|
|
2632
|
+
return;
|
|
2633
|
+
}
|
|
2634
|
+
const hitStartedAt = Date.now();
|
|
2635
|
+
markHit(ids);
|
|
2636
|
+
appendTiming(timing, "hit_stats_update_ms", hitStartedAt);
|
|
2637
|
+
};
|
|
1210
2638
|
if (fusionEnabled && llmModel && llmApiKey && llmBaseUrl && ranked.length > 1) {
|
|
2639
|
+
const fusionStartedAt = Date.now();
|
|
1211
2640
|
try {
|
|
1212
2641
|
const maxCandidates = Math.max(4, Math.min(20, options.fusion?.maxCandidates ?? 10));
|
|
1213
2642
|
const fusion = await requestFusion({
|
|
@@ -1215,6 +2644,13 @@ function createReadStore(options) {
|
|
|
1215
2644
|
candidates: ranked.slice(0, maxCandidates).map(item => ({
|
|
1216
2645
|
id: item.id,
|
|
1217
2646
|
text: item.text,
|
|
2647
|
+
source_excerpt: typeof item.source_excerpt === "string" ? item.source_excerpt : "",
|
|
2648
|
+
source_file: typeof item.source_file === "string" ? item.source_file : "",
|
|
2649
|
+
source_memory_id: typeof item.source_memory_id === "string" ? item.source_memory_id : "",
|
|
2650
|
+
source_memory_canonical_id: typeof item.source_memory_canonical_id === "string" ? item.source_memory_canonical_id : "",
|
|
2651
|
+
source_layer: typeof item.layer === "string" ? item.layer : "",
|
|
2652
|
+
source_event_id: typeof item.source_event_id === "string" ? item.source_event_id : "",
|
|
2653
|
+
source_field: item.source_field === "summary" || item.source_field === "evidence" ? item.source_field : "",
|
|
1218
2654
|
source: item.source,
|
|
1219
2655
|
event_type: item.event_type,
|
|
1220
2656
|
quality_score: item.quality_score,
|
|
@@ -1229,6 +2665,34 @@ function createReadStore(options) {
|
|
|
1229
2665
|
},
|
|
1230
2666
|
});
|
|
1231
2667
|
if (fusion && fusion.canonical_answer) {
|
|
2668
|
+
if (!Array.isArray(fusion.evidence_ids) || fusion.evidence_ids.length === 0) {
|
|
2669
|
+
throw new Error("fusion_missing_whitelisted_evidence");
|
|
2670
|
+
}
|
|
2671
|
+
const fusedEvidenceIds = uniqueStrings(fusion.evidence_ids.flatMap(item => {
|
|
2672
|
+
const linked = ranked.find(candidate => candidate.id === item);
|
|
2673
|
+
if (!linked)
|
|
2674
|
+
return [item];
|
|
2675
|
+
const linkedEvidence = Array.isArray(linked.evidence_ids) ? linked.evidence_ids : [];
|
|
2676
|
+
return linkedEvidence.length > 0 ? linkedEvidence : [item];
|
|
2677
|
+
}));
|
|
2678
|
+
const wikiRefs = uniqueStrings(fusion.evidence_ids.flatMap(item => {
|
|
2679
|
+
const linked = ranked.find(candidate => candidate.id === item);
|
|
2680
|
+
return uniqueStrings([
|
|
2681
|
+
...(Array.isArray(linked?.wiki_refs) ? linked.wiki_refs.map(v => String(v)) : []),
|
|
2682
|
+
typeof linked?.wiki_ref === "string" ? linked.wiki_ref : "",
|
|
2683
|
+
]);
|
|
2684
|
+
}));
|
|
2685
|
+
const fulltextFetchHints = (Array.isArray(fusion.need_fulltext_event_ids) ? fusion.need_fulltext_event_ids : [])
|
|
2686
|
+
.map(eventId => {
|
|
2687
|
+
const linked = ranked.find(item => item.source_memory_id === eventId ||
|
|
2688
|
+
item.source_memory_canonical_id === eventId ||
|
|
2689
|
+
item.id === eventId);
|
|
2690
|
+
return {
|
|
2691
|
+
event_id: eventId,
|
|
2692
|
+
source_file: linked?.source_file || "",
|
|
2693
|
+
source_excerpt: linked?.source_excerpt || "",
|
|
2694
|
+
};
|
|
2695
|
+
});
|
|
1232
2696
|
const fusedItem = {
|
|
1233
2697
|
id: `fusion_${Date.now().toString(36)}`,
|
|
1234
2698
|
text: fusion.canonical_answer,
|
|
@@ -1253,33 +2717,59 @@ function createReadStore(options) {
|
|
|
1253
2717
|
fused_risks: fusion.risks || [],
|
|
1254
2718
|
fused_action_items: fusion.action_items || [],
|
|
1255
2719
|
fused_conflicts: fusion.conflicts,
|
|
1256
|
-
|
|
2720
|
+
evidence_ids: fusedEvidenceIds,
|
|
2721
|
+
wiki_refs: wikiRefs,
|
|
2722
|
+
fused_evidence_ids: fusedEvidenceIds,
|
|
2723
|
+
fused_need_fulltext_event_ids: fusion.need_fulltext_event_ids || [],
|
|
2724
|
+
fulltext_fetch_hints: fulltextFetchHints,
|
|
1257
2725
|
};
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
return {
|
|
2726
|
+
if (fusionAuthoritative) {
|
|
2727
|
+
trackResultHits(Array.isArray(fusion.evidence_ids) ? fusion.evidence_ids : []);
|
|
2728
|
+
appendTiming(timing, "fusion_ms", fusionStartedAt);
|
|
2729
|
+
return finish({
|
|
2730
|
+
results: [fusedItem],
|
|
2731
|
+
semantic_results: semanticResults,
|
|
2732
|
+
keyword_results: keywordResults,
|
|
2733
|
+
channel_results: channelResults,
|
|
2734
|
+
strategy: "vector_sentence_and_keyword_parallel",
|
|
2735
|
+
}, { planned_queries: plannedQueries, fulltext_fallback_used: fulltextFallbackUsed, vector_source: vectorSource });
|
|
1262
2736
|
}
|
|
1263
2737
|
const merged = [fusedItem, ...ranked];
|
|
1264
|
-
|
|
2738
|
+
trackResultHits([
|
|
1265
2739
|
...(Array.isArray(fusion.evidence_ids) ? fusion.evidence_ids : []),
|
|
1266
2740
|
...ranked.map(item => item.id),
|
|
1267
2741
|
]);
|
|
1268
|
-
|
|
2742
|
+
appendTiming(timing, "fusion_ms", fusionStartedAt);
|
|
2743
|
+
return finish({
|
|
2744
|
+
results: merged.slice(0, Math.max(1, args.topK)),
|
|
2745
|
+
semantic_results: semanticResults,
|
|
2746
|
+
keyword_results: keywordResults,
|
|
2747
|
+
channel_results: channelResults,
|
|
2748
|
+
strategy: "vector_sentence_and_keyword_parallel",
|
|
2749
|
+
}, { planned_queries: plannedQueries, fulltext_fallback_used: fulltextFallbackUsed, vector_source: vectorSource });
|
|
1269
2750
|
}
|
|
2751
|
+
appendTiming(timing, "fusion_ms", fusionStartedAt);
|
|
1270
2752
|
}
|
|
1271
2753
|
catch (error) {
|
|
1272
2754
|
options.logger.warn(`LLM fusion failed, fallback to reranked results: ${error}`);
|
|
2755
|
+
appendTiming(timing, "fusion_ms", fusionStartedAt);
|
|
1273
2756
|
}
|
|
1274
2757
|
}
|
|
1275
2758
|
const finalRanked = ranked.slice(0, Math.max(1, args.topK));
|
|
1276
|
-
|
|
1277
|
-
return {
|
|
2759
|
+
trackResultHits(finalRanked.map(item => item.id));
|
|
2760
|
+
return finish({
|
|
2761
|
+
results: finalRanked,
|
|
2762
|
+
semantic_results: semanticResults,
|
|
2763
|
+
keyword_results: keywordResults,
|
|
2764
|
+
channel_results: channelResults,
|
|
2765
|
+
strategy: "vector_sentence_and_keyword_parallel",
|
|
2766
|
+
}, { planned_queries: plannedQueries, fulltext_fallback_used: fulltextFallbackUsed, vector_source: vectorSource });
|
|
1278
2767
|
}
|
|
1279
2768
|
async function getHotContext(args) {
|
|
1280
2769
|
const limit = Math.max(1, args.limit);
|
|
1281
2770
|
const docs = loadAllDocuments();
|
|
1282
2771
|
const coreRules = docs.find(doc => doc.source === "CORTEX_RULES.md");
|
|
2772
|
+
const ruleBudget = Math.max(1, Math.min(6, Math.floor(limit / 3)));
|
|
1283
2773
|
const archiveDocs = docs
|
|
1284
2774
|
.filter(doc => doc.source === "sessions_archive")
|
|
1285
2775
|
.sort((a, b) => (b.timestamp ?? 0) - (a.timestamp ?? 0))
|
|
@@ -1290,7 +2780,14 @@ function createReadStore(options) {
|
|
|
1290
2780
|
.slice(0, 2);
|
|
1291
2781
|
const result = [];
|
|
1292
2782
|
if (coreRules) {
|
|
1293
|
-
|
|
2783
|
+
const selectedRules = extractPrioritizedRuleLines(coreRules.text, ruleBudget);
|
|
2784
|
+
if (selectedRules.length > 0) {
|
|
2785
|
+
result.push({
|
|
2786
|
+
id: `${coreRules.id}.hot`,
|
|
2787
|
+
text: `# Hot Rules\n${selectedRules.map((line, index) => `${index + 1}. ${line}`).join("\n")}`,
|
|
2788
|
+
source: coreRules.source,
|
|
2789
|
+
});
|
|
2790
|
+
}
|
|
1294
2791
|
}
|
|
1295
2792
|
for (const doc of [...issueFixPairs, ...archiveDocs]) {
|
|
1296
2793
|
result.push({ id: doc.id, text: doc.text, source: doc.source });
|
|
@@ -1307,16 +2804,44 @@ function createReadStore(options) {
|
|
|
1307
2804
|
};
|
|
1308
2805
|
}
|
|
1309
2806
|
if (!result.auto_search) {
|
|
2807
|
+
const maxQueryChars = Math.max(20, readTuning.autoContext.queryMaxChars);
|
|
2808
|
+
const recentMessages = Array.isArray(args.recentMessages) ? args.recentMessages : [];
|
|
2809
|
+
const recentQuery = recentMessages
|
|
2810
|
+
.slice(-8)
|
|
2811
|
+
.map(message => {
|
|
2812
|
+
const role = typeof message.role === "string" && message.role.trim() ? message.role.trim() : "message";
|
|
2813
|
+
const text = typeof message.text === "string"
|
|
2814
|
+
? message.text.trim()
|
|
2815
|
+
: (typeof message.content === "string" ? message.content.trim() : "");
|
|
2816
|
+
return text ? `${role}: ${text.replace(/\s+/g, " ")}` : "";
|
|
2817
|
+
})
|
|
2818
|
+
.filter(Boolean)
|
|
2819
|
+
.join("\n");
|
|
1310
2820
|
const docs = loadAllDocuments()
|
|
1311
2821
|
.filter(doc => doc.source === "sessions_archive" && doc.sessionId === args.sessionId)
|
|
1312
2822
|
.sort((a, b) => (b.timestamp ?? 0) - (a.timestamp ?? 0));
|
|
1313
2823
|
const latest = docs[0];
|
|
1314
|
-
|
|
1315
|
-
|
|
2824
|
+
const fallbackQuery = latest && latest.text.trim()
|
|
2825
|
+
? latest.text.trim()
|
|
2826
|
+
: "";
|
|
2827
|
+
const sourceQuery = recentQuery || fallbackQuery;
|
|
2828
|
+
if (sourceQuery) {
|
|
2829
|
+
const autoQuery = sourceQuery.length > maxQueryChars
|
|
2830
|
+
? sourceQuery.slice(sourceQuery.length - maxQueryChars)
|
|
2831
|
+
: sourceQuery;
|
|
2832
|
+
const light = await searchMemory({
|
|
2833
|
+
query: autoQuery,
|
|
2834
|
+
topK: 3,
|
|
2835
|
+
mode: readTuning.autoContext.lightweightSearch ? "auto" : "default",
|
|
2836
|
+
fusionMode: "off",
|
|
2837
|
+
trackHits: false,
|
|
2838
|
+
});
|
|
1316
2839
|
result.auto_search = {
|
|
1317
|
-
query:
|
|
2840
|
+
query: autoQuery,
|
|
1318
2841
|
results: light.results,
|
|
1319
2842
|
age_seconds: 0,
|
|
2843
|
+
timing_ms: light.timing_ms,
|
|
2844
|
+
debug: light.debug,
|
|
1320
2845
|
};
|
|
1321
2846
|
}
|
|
1322
2847
|
}
|