openclaw-cortex-memory 0.1.0-Alpha.2 → 0.1.0-Alpha.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +163 -203
- package/SKILL.md +71 -268
- package/dist/index.d.ts +88 -15
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +859 -1189
- package/dist/index.js.map +1 -1
- package/dist/openclaw.plugin.json +362 -14
- package/dist/src/dedup/three_stage_deduplicator.d.ts +25 -0
- package/dist/src/dedup/three_stage_deduplicator.d.ts.map +1 -0
- package/dist/src/dedup/three_stage_deduplicator.js +224 -0
- package/dist/src/dedup/three_stage_deduplicator.js.map +1 -0
- package/dist/src/engine/memory_engine.d.ts +2 -1
- package/dist/src/engine/memory_engine.d.ts.map +1 -1
- package/dist/src/engine/ts_engine.d.ts +126 -0
- package/dist/src/engine/ts_engine.d.ts.map +1 -1
- package/dist/src/engine/ts_engine.js +1172 -44
- package/dist/src/engine/ts_engine.js.map +1 -1
- package/dist/src/engine/types.d.ts +12 -0
- package/dist/src/engine/types.d.ts.map +1 -1
- package/dist/src/graph/ontology.d.ts +103 -0
- package/dist/src/graph/ontology.d.ts.map +1 -0
- package/dist/src/graph/ontology.js +564 -0
- package/dist/src/graph/ontology.js.map +1 -0
- package/dist/src/quality/llm_output_validator.d.ts +48 -0
- package/dist/src/quality/llm_output_validator.d.ts.map +1 -0
- package/dist/src/quality/llm_output_validator.js +404 -0
- package/dist/src/quality/llm_output_validator.js.map +1 -0
- package/dist/src/reflect/reflector.d.ts +7 -0
- package/dist/src/reflect/reflector.d.ts.map +1 -1
- package/dist/src/reflect/reflector.js +358 -8
- package/dist/src/reflect/reflector.js.map +1 -1
- package/dist/src/rules/rule_store.d.ts.map +1 -1
- package/dist/src/rules/rule_store.js +75 -16
- package/dist/src/rules/rule_store.js.map +1 -1
- package/dist/src/session/session_end.d.ts +33 -0
- package/dist/src/session/session_end.d.ts.map +1 -1
- package/dist/src/session/session_end.js +67 -64
- package/dist/src/session/session_end.js.map +1 -1
- package/dist/src/store/archive_store.d.ts +128 -0
- package/dist/src/store/archive_store.d.ts.map +1 -0
- package/dist/src/store/archive_store.js +481 -0
- package/dist/src/store/archive_store.js.map +1 -0
- package/dist/src/store/embedding_utils.d.ts +32 -0
- package/dist/src/store/embedding_utils.d.ts.map +1 -0
- package/dist/src/store/embedding_utils.js +173 -0
- package/dist/src/store/embedding_utils.js.map +1 -0
- package/dist/src/store/graph_memory_store.d.ts +44 -0
- package/dist/src/store/graph_memory_store.d.ts.map +1 -0
- package/dist/src/store/graph_memory_store.js +168 -0
- package/dist/src/store/graph_memory_store.js.map +1 -0
- package/dist/src/store/read_store.d.ts +86 -0
- package/dist/src/store/read_store.d.ts.map +1 -1
- package/dist/src/store/read_store.js +1681 -25
- package/dist/src/store/read_store.js.map +1 -1
- package/dist/src/store/vector_store.d.ts +44 -0
- package/dist/src/store/vector_store.d.ts.map +1 -0
- package/dist/src/store/vector_store.js +201 -0
- package/dist/src/store/vector_store.js.map +1 -0
- package/dist/src/store/write_store.d.ts +52 -0
- package/dist/src/store/write_store.d.ts.map +1 -1
- package/dist/src/store/write_store.js +245 -3
- package/dist/src/store/write_store.js.map +1 -1
- package/dist/src/sync/session_sync.d.ts +100 -2
- package/dist/src/sync/session_sync.d.ts.map +1 -1
- package/dist/src/sync/session_sync.js +673 -22
- package/dist/src/sync/session_sync.js.map +1 -1
- package/dist/src/utils/runtime_env.d.ts +4 -0
- package/dist/src/utils/runtime_env.d.ts.map +1 -0
- package/dist/src/utils/runtime_env.js +51 -0
- package/dist/src/utils/runtime_env.js.map +1 -0
- package/openclaw.plugin.json +362 -14
- package/package.json +23 -6
- package/scripts/cli.js +19 -14
- package/scripts/uninstall.js +22 -5
- package/index.ts +0 -2092
- package/scripts/install.js +0 -27
|
@@ -36,6 +36,178 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
36
36
|
exports.createReadStore = createReadStore;
|
|
37
37
|
const fs = __importStar(require("fs"));
|
|
38
38
|
const path = __importStar(require("path"));
|
|
39
|
+
const module_1 = require("module");
|
|
40
|
+
function buildEntityGraphSummaryDocs(graphDocs) {
|
|
41
|
+
const entityEdges = new Map();
|
|
42
|
+
const entityLatestTs = new Map();
|
|
43
|
+
const entitySession = new Map();
|
|
44
|
+
for (const doc of graphDocs) {
|
|
45
|
+
const ts = typeof doc.timestamp === "number" ? doc.timestamp : 0;
|
|
46
|
+
const sessionId = typeof doc.sessionId === "string" ? doc.sessionId : "";
|
|
47
|
+
const relations = Array.isArray(doc.relations) ? doc.relations : [];
|
|
48
|
+
for (const relation of relations) {
|
|
49
|
+
const source = (relation.source || "").trim();
|
|
50
|
+
const target = (relation.target || "").trim();
|
|
51
|
+
const type = (relation.type || "").trim();
|
|
52
|
+
if (!source || !target || !type)
|
|
53
|
+
continue;
|
|
54
|
+
if (!entityEdges.has(source))
|
|
55
|
+
entityEdges.set(source, []);
|
|
56
|
+
if (!entityEdges.has(target))
|
|
57
|
+
entityEdges.set(target, []);
|
|
58
|
+
entityEdges.get(source)?.push({ source, target, type });
|
|
59
|
+
entityEdges.get(target)?.push({ source, target, type });
|
|
60
|
+
entityLatestTs.set(source, Math.max(entityLatestTs.get(source) || 0, ts));
|
|
61
|
+
entityLatestTs.set(target, Math.max(entityLatestTs.get(target) || 0, ts));
|
|
62
|
+
if (sessionId) {
|
|
63
|
+
if (!entitySession.has(source))
|
|
64
|
+
entitySession.set(source, sessionId);
|
|
65
|
+
if (!entitySession.has(target))
|
|
66
|
+
entitySession.set(target, sessionId);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
const output = [];
|
|
71
|
+
for (const [entity, edges] of entityEdges.entries()) {
|
|
72
|
+
if (!edges.length)
|
|
73
|
+
continue;
|
|
74
|
+
const outgoing = edges.filter(edge => edge.source === entity);
|
|
75
|
+
const incoming = edges.filter(edge => edge.target === entity);
|
|
76
|
+
const typeCounter = new Map();
|
|
77
|
+
for (const edge of edges) {
|
|
78
|
+
typeCounter.set(edge.type, (typeCounter.get(edge.type) || 0) + 1);
|
|
79
|
+
}
|
|
80
|
+
const typeSummary = [...typeCounter.entries()]
|
|
81
|
+
.sort((a, b) => b[1] - a[1])
|
|
82
|
+
.map(([type, count]) => `${type}:${count}`)
|
|
83
|
+
.join(", ");
|
|
84
|
+
const sortedOutgoing = [...outgoing].sort((a, b) => a.type.localeCompare(b.type));
|
|
85
|
+
const sortedIncoming = [...incoming].sort((a, b) => a.type.localeCompare(b.type));
|
|
86
|
+
const cappedOutgoing = sortedOutgoing.slice(0, 20);
|
|
87
|
+
const cappedIncoming = sortedIncoming.slice(0, 20);
|
|
88
|
+
const relationFacts = edges
|
|
89
|
+
.slice(0, 40)
|
|
90
|
+
.map(edge => `${edge.source} ${edge.type} ${edge.target}`)
|
|
91
|
+
.join(" | ");
|
|
92
|
+
const outgoingBlock = cappedOutgoing.length > 0
|
|
93
|
+
? cappedOutgoing.map((edge, index) => `${index + 1}. ${edge.source} -[${edge.type}]-> ${edge.target}`).join("\n")
|
|
94
|
+
: "none";
|
|
95
|
+
const incomingBlock = cappedIncoming.length > 0
|
|
96
|
+
? cappedIncoming.map((edge, index) => `${index + 1}. ${edge.source} -[${edge.type}]-> ${edge.target}`).join("\n")
|
|
97
|
+
: "none";
|
|
98
|
+
const summaryText = [
|
|
99
|
+
`# Graph Entity Summary`,
|
|
100
|
+
`entity: ${entity}`,
|
|
101
|
+
``,
|
|
102
|
+
`## Stats`,
|
|
103
|
+
`relation_total: ${edges.length}`,
|
|
104
|
+
`outgoing_total: ${outgoing.length}`,
|
|
105
|
+
`incoming_total: ${incoming.length}`,
|
|
106
|
+
typeSummary ? `relation_type_distribution: ${typeSummary}` : "relation_type_distribution: none",
|
|
107
|
+
``,
|
|
108
|
+
`## Outgoing Relations`,
|
|
109
|
+
outgoingBlock,
|
|
110
|
+
outgoing.length > cappedOutgoing.length ? `...truncated_outgoing: ${outgoing.length - cappedOutgoing.length}` : "",
|
|
111
|
+
``,
|
|
112
|
+
`## Incoming Relations`,
|
|
113
|
+
incomingBlock,
|
|
114
|
+
incoming.length > cappedIncoming.length ? `...truncated_incoming: ${incoming.length - cappedIncoming.length}` : "",
|
|
115
|
+
``,
|
|
116
|
+
`## Relation Facts`,
|
|
117
|
+
relationFacts || "none",
|
|
118
|
+
].filter(Boolean).join("\n");
|
|
119
|
+
output.push({
|
|
120
|
+
id: `gph_entity_${entity.toLowerCase().replace(/[^a-z0-9\u4e00-\u9fa5]+/gi, "_")}`,
|
|
121
|
+
text: summaryText,
|
|
122
|
+
source: "sessions_graph_entity",
|
|
123
|
+
timestamp: (entityLatestTs.get(entity) || 0) > 0 ? entityLatestTs.get(entity) : undefined,
|
|
124
|
+
layer: "archive",
|
|
125
|
+
sourceMemoryId: entity,
|
|
126
|
+
sessionId: entitySession.get(entity) || undefined,
|
|
127
|
+
entities: [entity],
|
|
128
|
+
relations: edges,
|
|
129
|
+
eventType: "graph_summary",
|
|
130
|
+
qualityScore: 1,
|
|
131
|
+
});
|
|
132
|
+
}
|
|
133
|
+
return output;
|
|
134
|
+
}
|
|
135
|
+
const DEFAULT_READ_TUNING = {
|
|
136
|
+
scoring: {
|
|
137
|
+
lexicalWeight: 0.2,
|
|
138
|
+
bm25Scale: 2,
|
|
139
|
+
semanticWeight: 0.3,
|
|
140
|
+
recencyWeight: 0.1,
|
|
141
|
+
qualityWeight: 0.15,
|
|
142
|
+
typeMatchWeight: 0.15,
|
|
143
|
+
graphMatchWeight: 0.1,
|
|
144
|
+
},
|
|
145
|
+
rrf: {
|
|
146
|
+
k: 60,
|
|
147
|
+
weight: 1.5,
|
|
148
|
+
},
|
|
149
|
+
recency: {
|
|
150
|
+
buckets: [
|
|
151
|
+
{ maxAgeHours: 12, score: 1, bonus: 0.6 },
|
|
152
|
+
{ maxAgeHours: 24, score: 0.8, bonus: 0.6 },
|
|
153
|
+
{ maxAgeHours: 72, score: 0.6, bonus: 0.3 },
|
|
154
|
+
{ maxAgeHours: 168, score: 0.4, bonus: 0.3 },
|
|
155
|
+
{ maxAgeHours: 720, score: 0.2, bonus: 0 },
|
|
156
|
+
{ maxAgeHours: Number.POSITIVE_INFINITY, score: 0.05, bonus: 0 },
|
|
157
|
+
],
|
|
158
|
+
},
|
|
159
|
+
autoContext: {
|
|
160
|
+
queryMaxChars: 80,
|
|
161
|
+
lightweightSearch: true,
|
|
162
|
+
},
|
|
163
|
+
};
|
|
164
|
+
function resolveReadTuning(options) {
|
|
165
|
+
const configuredBuckets = Array.isArray(options?.recency?.buckets)
|
|
166
|
+
? options?.recency?.buckets
|
|
167
|
+
.filter(item => item &&
|
|
168
|
+
Number.isFinite(item.maxAgeHours) &&
|
|
169
|
+
item.maxAgeHours > 0 &&
|
|
170
|
+
Number.isFinite(item.score) &&
|
|
171
|
+
item.score >= 0 &&
|
|
172
|
+
Number.isFinite(item.bonus))
|
|
173
|
+
.map(item => ({
|
|
174
|
+
maxAgeHours: item.maxAgeHours,
|
|
175
|
+
score: Math.max(0, item.score),
|
|
176
|
+
bonus: Math.max(0, item.bonus),
|
|
177
|
+
}))
|
|
178
|
+
: [];
|
|
179
|
+
const sortedBuckets = configuredBuckets
|
|
180
|
+
.sort((a, b) => a.maxAgeHours - b.maxAgeHours);
|
|
181
|
+
const buckets = sortedBuckets.length > 0 ? sortedBuckets : DEFAULT_READ_TUNING.recency.buckets;
|
|
182
|
+
const numberOr = (value, fallback, min) => {
|
|
183
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value < min) {
|
|
184
|
+
return fallback;
|
|
185
|
+
}
|
|
186
|
+
return value;
|
|
187
|
+
};
|
|
188
|
+
return {
|
|
189
|
+
scoring: {
|
|
190
|
+
lexicalWeight: numberOr(options?.scoring?.lexicalWeight, DEFAULT_READ_TUNING.scoring.lexicalWeight, 0),
|
|
191
|
+
bm25Scale: numberOr(options?.scoring?.bm25Scale, DEFAULT_READ_TUNING.scoring.bm25Scale, 0),
|
|
192
|
+
semanticWeight: numberOr(options?.scoring?.semanticWeight, DEFAULT_READ_TUNING.scoring.semanticWeight, 0),
|
|
193
|
+
recencyWeight: numberOr(options?.scoring?.recencyWeight, DEFAULT_READ_TUNING.scoring.recencyWeight, 0),
|
|
194
|
+
qualityWeight: numberOr(options?.scoring?.qualityWeight, DEFAULT_READ_TUNING.scoring.qualityWeight, 0),
|
|
195
|
+
typeMatchWeight: numberOr(options?.scoring?.typeMatchWeight, DEFAULT_READ_TUNING.scoring.typeMatchWeight, 0),
|
|
196
|
+
graphMatchWeight: numberOr(options?.scoring?.graphMatchWeight, DEFAULT_READ_TUNING.scoring.graphMatchWeight, 0),
|
|
197
|
+
},
|
|
198
|
+
rrf: {
|
|
199
|
+
k: Math.floor(numberOr(options?.rrf?.k, DEFAULT_READ_TUNING.rrf.k, 1)),
|
|
200
|
+
weight: numberOr(options?.rrf?.weight, DEFAULT_READ_TUNING.rrf.weight, 0),
|
|
201
|
+
},
|
|
202
|
+
recency: {
|
|
203
|
+
buckets,
|
|
204
|
+
},
|
|
205
|
+
autoContext: {
|
|
206
|
+
queryMaxChars: Math.floor(numberOr(options?.autoContext?.queryMaxChars, DEFAULT_READ_TUNING.autoContext.queryMaxChars, 20)),
|
|
207
|
+
lightweightSearch: options?.autoContext?.lightweightSearch !== false,
|
|
208
|
+
},
|
|
209
|
+
};
|
|
210
|
+
}
|
|
39
211
|
function safeReadFile(filePath) {
|
|
40
212
|
try {
|
|
41
213
|
if (!fs.existsSync(filePath)) {
|
|
@@ -65,7 +237,64 @@ function scoreText(query, text) {
|
|
|
65
237
|
}
|
|
66
238
|
return score;
|
|
67
239
|
}
|
|
240
|
+
function tokenize(text) {
|
|
241
|
+
return text
|
|
242
|
+
.toLowerCase()
|
|
243
|
+
.split(/[^a-z0-9\u4e00-\u9fa5]+/i)
|
|
244
|
+
.map(token => token.trim())
|
|
245
|
+
.filter(Boolean);
|
|
246
|
+
}
|
|
247
|
+
function buildBm25Stats(docs, queryTerms, getTokens) {
|
|
248
|
+
const docFreq = new Map();
|
|
249
|
+
let totalLen = 0;
|
|
250
|
+
for (const doc of docs) {
|
|
251
|
+
const tokens = typeof getTokens === "function" ? getTokens(doc) : tokenize(doc.text);
|
|
252
|
+
totalLen += tokens.length;
|
|
253
|
+
if (queryTerms.length === 0) {
|
|
254
|
+
continue;
|
|
255
|
+
}
|
|
256
|
+
const termSet = new Set(tokens);
|
|
257
|
+
for (const term of queryTerms) {
|
|
258
|
+
if (termSet.has(term)) {
|
|
259
|
+
docFreq.set(term, (docFreq.get(term) || 0) + 1);
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
const avgDocLen = docs.length > 0 ? Math.max(1, totalLen / docs.length) : 1;
|
|
264
|
+
return { avgDocLen, docFreq };
|
|
265
|
+
}
|
|
266
|
+
function bm25Score(args) {
|
|
267
|
+
const tokens = Array.isArray(args.docTokens) ? args.docTokens : tokenize(args.docText);
|
|
268
|
+
if (tokens.length === 0 || args.queryTerms.length === 0 || args.docCount <= 0) {
|
|
269
|
+
return 0;
|
|
270
|
+
}
|
|
271
|
+
const termFreq = new Map();
|
|
272
|
+
for (const token of tokens) {
|
|
273
|
+
termFreq.set(token, (termFreq.get(token) || 0) + 1);
|
|
274
|
+
}
|
|
275
|
+
const k1 = 1.2;
|
|
276
|
+
const b = 0.75;
|
|
277
|
+
let score = 0;
|
|
278
|
+
for (const term of args.queryTerms) {
|
|
279
|
+
const tf = termFreq.get(term) || 0;
|
|
280
|
+
if (tf <= 0)
|
|
281
|
+
continue;
|
|
282
|
+
const df = args.docFreq.get(term) || 0;
|
|
283
|
+
const idf = Math.log(1 + ((args.docCount - df + 0.5) / (df + 0.5)));
|
|
284
|
+
const denominator = tf + k1 * (1 - b + b * (tokens.length / Math.max(1, args.avgDocLen)));
|
|
285
|
+
score += idf * (((k1 + 1) * tf) / Math.max(1e-6, denominator));
|
|
286
|
+
}
|
|
287
|
+
return score;
|
|
288
|
+
}
|
|
68
289
|
function normalizeRecordText(record) {
|
|
290
|
+
const summary = typeof record.summary === "string" ? record.summary.trim() : "";
|
|
291
|
+
const sourceText = typeof record.source_text === "string" ? record.source_text.trim() : "";
|
|
292
|
+
if (summary && sourceText) {
|
|
293
|
+
return [
|
|
294
|
+
`summary: ${summary}`,
|
|
295
|
+
`source_text: ${sourceText}`,
|
|
296
|
+
].join("\n");
|
|
297
|
+
}
|
|
69
298
|
const direct = [record.content, record.summary, record.text, record.message]
|
|
70
299
|
.find(v => typeof v === "string" && v.trim());
|
|
71
300
|
if (direct) {
|
|
@@ -108,7 +337,9 @@ function parseJsonlFile(filePath, sourceLabel, logger) {
|
|
|
108
337
|
}
|
|
109
338
|
try {
|
|
110
339
|
const parsed = JSON.parse(trimmed);
|
|
111
|
-
const
|
|
340
|
+
const summaryText = typeof parsed.summary === "string" ? parsed.summary.trim() : "";
|
|
341
|
+
const sourceText = typeof parsed.source_text === "string" ? parsed.source_text.trim() : "";
|
|
342
|
+
const text = summaryText || normalizeRecordText(parsed);
|
|
112
343
|
if (!text.trim()) {
|
|
113
344
|
continue;
|
|
114
345
|
}
|
|
@@ -117,8 +348,29 @@ function parseJsonlFile(filePath, sourceLabel, logger) {
|
|
|
117
348
|
docs.push({
|
|
118
349
|
id,
|
|
119
350
|
text,
|
|
351
|
+
summaryText: summaryText || text,
|
|
352
|
+
sourceText: sourceText || undefined,
|
|
353
|
+
sourceEventId: typeof parsed.source_event_id === "string" ? parsed.source_event_id : undefined,
|
|
354
|
+
sourceFile: typeof parsed.source_file === "string" ? parsed.source_file : undefined,
|
|
120
355
|
source: sourceLabel,
|
|
121
356
|
timestamp: Number.isFinite(timestampValue) ? timestampValue : undefined,
|
|
357
|
+
layer: parsed.layer === "active" || parsed.layer === "archive"
|
|
358
|
+
? parsed.layer
|
|
359
|
+
: (sourceLabel === "sessions_active" ? "active" : (sourceLabel === "sessions_archive" ? "archive" : undefined)),
|
|
360
|
+
sourceMemoryId: typeof parsed.source_memory_id === "string"
|
|
361
|
+
? parsed.source_memory_id
|
|
362
|
+
: id,
|
|
363
|
+
sourceMemoryCanonicalId: typeof parsed.source_memory_canonical_id === "string"
|
|
364
|
+
? parsed.source_memory_canonical_id
|
|
365
|
+
: (typeof parsed.canonical_id === "string" ? parsed.canonical_id : undefined),
|
|
366
|
+
embedding: Array.isArray(parsed.embedding) ? parsed.embedding.filter(item => Number.isFinite(item)) : undefined,
|
|
367
|
+
eventType: typeof parsed.event_type === "string" ? parsed.event_type.trim() : undefined,
|
|
368
|
+
qualityScore: typeof parsed.quality_score === "number" ? parsed.quality_score : undefined,
|
|
369
|
+
charCount: typeof parsed.char_count === "number" ? parsed.char_count : undefined,
|
|
370
|
+
tokenCount: typeof parsed.token_count === "number" ? parsed.token_count : undefined,
|
|
371
|
+
sessionId: typeof parsed.session_id === "string" ? parsed.session_id : undefined,
|
|
372
|
+
entities: [],
|
|
373
|
+
relations: [],
|
|
122
374
|
});
|
|
123
375
|
}
|
|
124
376
|
catch (error) {
|
|
@@ -147,69 +399,1454 @@ function parseMarkdownFile(filePath, sourceLabel) {
|
|
|
147
399
|
},
|
|
148
400
|
];
|
|
149
401
|
}
|
|
150
|
-
function
|
|
402
|
+
function extractPrioritizedRuleLines(text, maxRules) {
|
|
403
|
+
if (!text.trim() || maxRules <= 0) {
|
|
404
|
+
return [];
|
|
405
|
+
}
|
|
406
|
+
const lines = text
|
|
407
|
+
.split(/\r?\n/)
|
|
408
|
+
.map(line => line.trim())
|
|
409
|
+
.filter(Boolean)
|
|
410
|
+
.filter(line => !/^core rules and knowledge extracted/i.test(line))
|
|
411
|
+
.filter(line => !/^core rules\b/i.test(line));
|
|
412
|
+
if (lines.length === 0) {
|
|
413
|
+
return [];
|
|
414
|
+
}
|
|
415
|
+
const dedupedFromTail = [];
|
|
416
|
+
const seen = new Set();
|
|
417
|
+
for (let i = lines.length - 1; i >= 0; i -= 1) {
|
|
418
|
+
const line = lines[i];
|
|
419
|
+
const key = line.toLowerCase().replace(/\s+/g, " ").trim();
|
|
420
|
+
if (!key || seen.has(key)) {
|
|
421
|
+
continue;
|
|
422
|
+
}
|
|
423
|
+
seen.add(key);
|
|
424
|
+
dedupedFromTail.push(line);
|
|
425
|
+
}
|
|
426
|
+
const scored = dedupedFromTail.map((line, indexFromTail) => {
|
|
427
|
+
let score = 0;
|
|
428
|
+
if (/(must|should|ensure|avoid|prefer|always|never|fallback|verify|validate|retry|sanitize)/i.test(line)) {
|
|
429
|
+
score += 3;
|
|
430
|
+
}
|
|
431
|
+
if (/(fix|resolved|success|stable|deploy|release|incident|rollback|constraint|decision)/i.test(line)) {
|
|
432
|
+
score += 2;
|
|
433
|
+
}
|
|
434
|
+
if (/(确保|避免|优先|必须|建议|回退|重试|校验|稳定|发布|决策)/.test(line)) {
|
|
435
|
+
score += 2;
|
|
436
|
+
}
|
|
437
|
+
if (line.length >= 30 && line.length <= 220) {
|
|
438
|
+
score += 2;
|
|
439
|
+
}
|
|
440
|
+
else if (line.length > 220) {
|
|
441
|
+
score -= 1;
|
|
442
|
+
}
|
|
443
|
+
if (/[.!?]$/.test(line)) {
|
|
444
|
+
score += 1;
|
|
445
|
+
}
|
|
446
|
+
score += Math.max(0, 2 - indexFromTail * 0.08);
|
|
447
|
+
return { line, score, indexFromTail };
|
|
448
|
+
});
|
|
449
|
+
const selected = scored
|
|
450
|
+
.sort((a, b) => (b.score - a.score) || (a.indexFromTail - b.indexFromTail))
|
|
451
|
+
.slice(0, maxRules)
|
|
452
|
+
.sort((a, b) => a.indexFromTail - b.indexFromTail)
|
|
453
|
+
.map(item => item.line);
|
|
454
|
+
return selected;
|
|
455
|
+
}
|
|
456
|
+
function withRecencyBoost(score, timestamp, buckets) {
|
|
151
457
|
if (!timestamp) {
|
|
152
458
|
return score;
|
|
153
459
|
}
|
|
154
460
|
const ageHours = (Date.now() - timestamp) / (1000 * 60 * 60);
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
return score + 0.3;
|
|
461
|
+
for (const bucket of buckets) {
|
|
462
|
+
if (ageHours <= bucket.maxAgeHours) {
|
|
463
|
+
return score + bucket.bonus;
|
|
464
|
+
}
|
|
160
465
|
}
|
|
161
466
|
return score;
|
|
162
467
|
}
|
|
468
|
+
function recencyScore(timestamp, buckets) {
|
|
469
|
+
if (!timestamp) {
|
|
470
|
+
return 0;
|
|
471
|
+
}
|
|
472
|
+
const ageHours = (Date.now() - timestamp) / (1000 * 60 * 60);
|
|
473
|
+
for (const bucket of buckets) {
|
|
474
|
+
if (ageHours <= bucket.maxAgeHours) {
|
|
475
|
+
return bucket.score;
|
|
476
|
+
}
|
|
477
|
+
}
|
|
478
|
+
return 0;
|
|
479
|
+
}
|
|
480
|
+
function eventTypeHalfLifeDays(eventType, options) {
|
|
481
|
+
const fallback = typeof options?.defaultHalfLifeDays === "number" && options.defaultHalfLifeDays > 0
|
|
482
|
+
? options.defaultHalfLifeDays
|
|
483
|
+
: 90;
|
|
484
|
+
const type = (eventType || "").trim().toLowerCase();
|
|
485
|
+
if (!type)
|
|
486
|
+
return fallback;
|
|
487
|
+
const configured = options?.halfLifeByEventType || {};
|
|
488
|
+
if (typeof configured[type] === "number" && configured[type] > 0) {
|
|
489
|
+
return configured[type];
|
|
490
|
+
}
|
|
491
|
+
if (["issue", "fix", "action_item", "blocker"].includes(type))
|
|
492
|
+
return 30;
|
|
493
|
+
if (["plan", "milestone", "follow_up"].includes(type))
|
|
494
|
+
return 60;
|
|
495
|
+
if (["decision", "insight", "retrospective"].includes(type))
|
|
496
|
+
return 120;
|
|
497
|
+
if (["preference", "constraint", "requirement", "dependency", "assumption"].includes(type))
|
|
498
|
+
return 240;
|
|
499
|
+
return fallback;
|
|
500
|
+
}
|
|
501
|
+
function computeAntiDecayBoost(id, hitStats, options) {
|
|
502
|
+
const anti = options?.antiDecay;
|
|
503
|
+
if (anti?.enabled === false) {
|
|
504
|
+
return 1;
|
|
505
|
+
}
|
|
506
|
+
const item = hitStats.items[id];
|
|
507
|
+
if (!item) {
|
|
508
|
+
return 1;
|
|
509
|
+
}
|
|
510
|
+
const hitWeight = typeof anti?.hitWeight === "number" && anti.hitWeight > 0 ? anti.hitWeight : 0.08;
|
|
511
|
+
const maxBoost = typeof anti?.maxBoost === "number" && anti.maxBoost >= 1 ? anti.maxBoost : 1.6;
|
|
512
|
+
const recentWindowDays = typeof anti?.recentWindowDays === "number" && anti.recentWindowDays > 0 ? anti.recentWindowDays : 30;
|
|
513
|
+
const lastHitTs = Date.parse(item.lastHitAt || "");
|
|
514
|
+
const ageDays = Number.isFinite(lastHitTs) ? Math.max(0, (Date.now() - lastHitTs) / (1000 * 60 * 60 * 24)) : recentWindowDays * 2;
|
|
515
|
+
const freshness = ageDays <= recentWindowDays ? (1 - ageDays / recentWindowDays) : 0;
|
|
516
|
+
const countFactor = Math.log1p(Math.max(0, item.count));
|
|
517
|
+
const boost = 1 + countFactor * hitWeight * (0.5 + 0.5 * freshness);
|
|
518
|
+
return Math.min(maxBoost, Math.max(1, boost));
|
|
519
|
+
}
|
|
520
|
+
function computeDecayFactor(id, eventType, timestamp, options, hitStats) {
|
|
521
|
+
const enabled = options?.enabled !== false;
|
|
522
|
+
if (!enabled || !timestamp) {
|
|
523
|
+
return computeAntiDecayBoost(id, hitStats, options);
|
|
524
|
+
}
|
|
525
|
+
const ageDays = Math.max(0, (Date.now() - timestamp) / (1000 * 60 * 60 * 24));
|
|
526
|
+
const halfLife = eventTypeHalfLifeDays(eventType, options);
|
|
527
|
+
const base = Math.pow(2, -ageDays / Math.max(1, halfLife));
|
|
528
|
+
const floor = typeof options?.minFloor === "number"
|
|
529
|
+
? Math.max(0, Math.min(1, options.minFloor))
|
|
530
|
+
: 0.15;
|
|
531
|
+
const decay = Math.max(floor, base);
|
|
532
|
+
const boost = computeAntiDecayBoost(id, hitStats, options);
|
|
533
|
+
return Math.min(1, decay * boost);
|
|
534
|
+
}
|
|
535
|
+
function normalizeBaseUrl(value) {
|
|
536
|
+
if (!value)
|
|
537
|
+
return "";
|
|
538
|
+
return value.endsWith("/") ? value.slice(0, -1) : value;
|
|
539
|
+
}
|
|
540
|
+
const READ_FUSION_PROMPT_VERSION = "read-fusion.v1.2.0";
|
|
541
|
+
const READ_FUSION_REGRESSION_SAMPLES = [
|
|
542
|
+
"Example A: if archive and vector refer to the same source_memory_id, keep one main conclusion and keep the rest as supporting evidence.",
|
|
543
|
+
"Example B: if conclusions conflict, write conflicts and explain prioritization in canonical_answer (time, quality, explicitness).",
|
|
544
|
+
"Example C: if summary/excerpt is insufficient, return event ids in need_fulltext_event_ids for full-text lookup.",
|
|
545
|
+
];
|
|
546
|
+
function cosineSimilarity(left, right) {
|
|
547
|
+
if (left.length === 0 || right.length === 0) {
|
|
548
|
+
return 0;
|
|
549
|
+
}
|
|
550
|
+
const size = Math.min(left.length, right.length);
|
|
551
|
+
let dot = 0;
|
|
552
|
+
let leftNorm = 0;
|
|
553
|
+
let rightNorm = 0;
|
|
554
|
+
for (let i = 0; i < size; i += 1) {
|
|
555
|
+
const a = left[i];
|
|
556
|
+
const b = right[i];
|
|
557
|
+
dot += a * b;
|
|
558
|
+
leftNorm += a * a;
|
|
559
|
+
rightNorm += b * b;
|
|
560
|
+
}
|
|
561
|
+
if (leftNorm === 0 || rightNorm === 0) {
|
|
562
|
+
return 0;
|
|
563
|
+
}
|
|
564
|
+
return dot / (Math.sqrt(leftNorm) * Math.sqrt(rightNorm));
|
|
565
|
+
}
|
|
566
|
+
async function requestEmbedding(args) {
|
|
567
|
+
const endpoint = args.baseUrl.endsWith("/embeddings") ? args.baseUrl : `${args.baseUrl}/embeddings`;
|
|
568
|
+
const body = {
|
|
569
|
+
input: args.text,
|
|
570
|
+
model: args.model,
|
|
571
|
+
};
|
|
572
|
+
if (typeof args.dimensions === "number" && Number.isFinite(args.dimensions) && args.dimensions > 0) {
|
|
573
|
+
body.dimensions = args.dimensions;
|
|
574
|
+
}
|
|
575
|
+
let lastError = null;
|
|
576
|
+
for (let attempt = 0; attempt < 3; attempt += 1) {
|
|
577
|
+
const controller = new AbortController();
|
|
578
|
+
const timeoutId = setTimeout(() => controller.abort(), 10000);
|
|
579
|
+
try {
|
|
580
|
+
const response = await fetch(endpoint, {
|
|
581
|
+
method: "POST",
|
|
582
|
+
headers: {
|
|
583
|
+
"content-type": "application/json",
|
|
584
|
+
authorization: `Bearer ${args.apiKey}`,
|
|
585
|
+
},
|
|
586
|
+
body: JSON.stringify(body),
|
|
587
|
+
signal: controller.signal,
|
|
588
|
+
});
|
|
589
|
+
clearTimeout(timeoutId);
|
|
590
|
+
if (!response.ok) {
|
|
591
|
+
lastError = new Error(`embedding_http_${response.status}`);
|
|
592
|
+
continue;
|
|
593
|
+
}
|
|
594
|
+
const json = await response.json();
|
|
595
|
+
const embedding = json?.data?.[0]?.embedding;
|
|
596
|
+
if (Array.isArray(embedding) && embedding.length > 0) {
|
|
597
|
+
return embedding.filter(item => Number.isFinite(item));
|
|
598
|
+
}
|
|
599
|
+
lastError = new Error("embedding_empty");
|
|
600
|
+
}
|
|
601
|
+
catch (error) {
|
|
602
|
+
clearTimeout(timeoutId);
|
|
603
|
+
lastError = error;
|
|
604
|
+
}
|
|
605
|
+
}
|
|
606
|
+
if (lastError) {
|
|
607
|
+
throw lastError;
|
|
608
|
+
}
|
|
609
|
+
return null;
|
|
610
|
+
}
|
|
611
|
+
async function requestRerank(args) {
|
|
612
|
+
const endpoint = args.baseUrl.endsWith("/rerank") ? args.baseUrl : `${args.baseUrl}/rerank`;
|
|
613
|
+
const documents = args.candidates.map(item => item.text);
|
|
614
|
+
const body = {
|
|
615
|
+
model: args.model,
|
|
616
|
+
query: args.query,
|
|
617
|
+
documents,
|
|
618
|
+
top_n: args.candidates.length,
|
|
619
|
+
};
|
|
620
|
+
let lastError = null;
|
|
621
|
+
for (let attempt = 0; attempt < 2; attempt += 1) {
|
|
622
|
+
const controller = new AbortController();
|
|
623
|
+
const timeoutId = setTimeout(() => controller.abort(), 12000);
|
|
624
|
+
try {
|
|
625
|
+
const response = await fetch(endpoint, {
|
|
626
|
+
method: "POST",
|
|
627
|
+
headers: {
|
|
628
|
+
"content-type": "application/json",
|
|
629
|
+
authorization: `Bearer ${args.apiKey}`,
|
|
630
|
+
},
|
|
631
|
+
body: JSON.stringify(body),
|
|
632
|
+
signal: controller.signal,
|
|
633
|
+
});
|
|
634
|
+
clearTimeout(timeoutId);
|
|
635
|
+
if (!response.ok) {
|
|
636
|
+
lastError = new Error(`rerank_http_${response.status}`);
|
|
637
|
+
continue;
|
|
638
|
+
}
|
|
639
|
+
const json = await response.json();
|
|
640
|
+
const list = Array.isArray(json.results) ? json.results : (Array.isArray(json.data) ? json.data : []);
|
|
641
|
+
if (!Array.isArray(list) || list.length === 0) {
|
|
642
|
+
lastError = new Error("rerank_empty");
|
|
643
|
+
continue;
|
|
644
|
+
}
|
|
645
|
+
const mapped = list
|
|
646
|
+
.map((item, rank) => {
|
|
647
|
+
const index = typeof item.index === "number" ? item.index : rank;
|
|
648
|
+
const hit = args.candidates[index];
|
|
649
|
+
if (!hit)
|
|
650
|
+
return null;
|
|
651
|
+
const score = typeof item.relevance_score === "number" ? item.relevance_score : (typeof item.score === "number" ? item.score : hit.score);
|
|
652
|
+
return { ...hit, score };
|
|
653
|
+
})
|
|
654
|
+
.filter((item) => Boolean(item));
|
|
655
|
+
if (mapped.length > 0) {
|
|
656
|
+
return mapped;
|
|
657
|
+
}
|
|
658
|
+
lastError = new Error("rerank_map_empty");
|
|
659
|
+
}
|
|
660
|
+
catch (error) {
|
|
661
|
+
clearTimeout(timeoutId);
|
|
662
|
+
lastError = error;
|
|
663
|
+
}
|
|
664
|
+
}
|
|
665
|
+
throw lastError instanceof Error ? lastError : new Error(String(lastError || "rerank_failed"));
|
|
666
|
+
}
|
|
667
|
+
function classifyIntent(query) {
|
|
668
|
+
const text = query.toLowerCase();
|
|
669
|
+
const relationHints = /(关系|依赖|关联|上下游|图谱|拓扑|graph|relation|entity|dependency)/i;
|
|
670
|
+
if (relationHints.test(text))
|
|
671
|
+
return "RELATION_DISCOVERY";
|
|
672
|
+
const troubleHints = /(报错|错误|异常|失败|超时|无法|崩溃|故障|修复|bug|error|failed|timeout|fix)/i;
|
|
673
|
+
if (troubleHints.test(text))
|
|
674
|
+
return "TROUBLESHOOTING";
|
|
675
|
+
const preferenceHints = /(偏好|习惯|口味|喜欢|不喜欢|偏向|preference)/i;
|
|
676
|
+
if (preferenceHints.test(text))
|
|
677
|
+
return "PREFERENCE_PROFILE";
|
|
678
|
+
const timelineHints = /(最近|上次|之前|时间线|历史|timeline|history)/i;
|
|
679
|
+
if (timelineHints.test(text))
|
|
680
|
+
return "TIMELINE_REVIEW";
|
|
681
|
+
const decisionHints = /(方案|决策|选择|建议|取舍|权衡|tradeoff|plan)/i;
|
|
682
|
+
if (decisionHints.test(text))
|
|
683
|
+
return "DECISION_SUPPORT";
|
|
684
|
+
return "FACT_LOOKUP";
|
|
685
|
+
}
|
|
686
|
+
function preferredEventTypes(intent) {
|
|
687
|
+
if (intent === "TROUBLESHOOTING")
|
|
688
|
+
return ["issue", "fix", "risk", "blocker", "dependency", "retrospective"];
|
|
689
|
+
if (intent === "PREFERENCE_PROFILE")
|
|
690
|
+
return ["preference", "decision", "constraint", "requirement"];
|
|
691
|
+
if (intent === "DECISION_SUPPORT")
|
|
692
|
+
return ["decision", "plan", "insight", "assumption", "constraint", "requirement"];
|
|
693
|
+
if (intent === "TIMELINE_REVIEW")
|
|
694
|
+
return ["action_item", "follow_up", "milestone", "plan", "decision", "issue", "fix"];
|
|
695
|
+
return [];
|
|
696
|
+
}
|
|
697
|
+
function sourceWeight(source, intent) {
|
|
698
|
+
if (source === "rules") {
|
|
699
|
+
return intent === "DECISION_SUPPORT" || intent === "TROUBLESHOOTING" ? 1.15 : 0.9;
|
|
700
|
+
}
|
|
701
|
+
if (source === "graph") {
|
|
702
|
+
return intent === "RELATION_DISCOVERY" ? 1.25 : 0.85;
|
|
703
|
+
}
|
|
704
|
+
if (source === "vector") {
|
|
705
|
+
return 1.05;
|
|
706
|
+
}
|
|
707
|
+
return 1;
|
|
708
|
+
}
|
|
709
|
+
function mergeKeyFromDoc(doc) {
|
|
710
|
+
const canonical = typeof doc.sourceMemoryCanonicalId === "string" ? doc.sourceMemoryCanonicalId.trim() : "";
|
|
711
|
+
if (canonical) {
|
|
712
|
+
return `canonical:${canonical}`;
|
|
713
|
+
}
|
|
714
|
+
const sourceMemoryId = typeof doc.sourceMemoryId === "string" ? doc.sourceMemoryId.trim() : "";
|
|
715
|
+
if (sourceMemoryId) {
|
|
716
|
+
return `source:${sourceMemoryId}`;
|
|
717
|
+
}
|
|
718
|
+
return `id:${doc.id}`;
|
|
719
|
+
}
|
|
720
|
+
function customChannelWeight(source, options) {
|
|
721
|
+
const weights = options?.channelWeights;
|
|
722
|
+
if (!weights)
|
|
723
|
+
return 1;
|
|
724
|
+
const value = weights[source];
|
|
725
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) {
|
|
726
|
+
return 1;
|
|
727
|
+
}
|
|
728
|
+
return value;
|
|
729
|
+
}
|
|
730
|
+
function lengthNormalizeFactor(doc, options) {
|
|
731
|
+
const lengthNorm = options?.lengthNorm;
|
|
732
|
+
if (lengthNorm?.enabled === false) {
|
|
733
|
+
return 1;
|
|
734
|
+
}
|
|
735
|
+
const pivotChars = typeof lengthNorm?.pivotChars === "number" && lengthNorm.pivotChars > 0
|
|
736
|
+
? lengthNorm.pivotChars
|
|
737
|
+
: 1200;
|
|
738
|
+
const strength = typeof lengthNorm?.strength === "number" && lengthNorm.strength > 0
|
|
739
|
+
? lengthNorm.strength
|
|
740
|
+
: 0.75;
|
|
741
|
+
const minFactor = typeof lengthNorm?.minFactor === "number" && lengthNorm.minFactor > 0 && lengthNorm.minFactor <= 1
|
|
742
|
+
? lengthNorm.minFactor
|
|
743
|
+
: 0.45;
|
|
744
|
+
const charCount = typeof doc.charCount === "number" && Number.isFinite(doc.charCount)
|
|
745
|
+
? doc.charCount
|
|
746
|
+
: doc.text.length;
|
|
747
|
+
if (charCount <= pivotChars) {
|
|
748
|
+
return 1;
|
|
749
|
+
}
|
|
750
|
+
const over = (charCount - pivotChars) / pivotChars;
|
|
751
|
+
const factor = 1 / (1 + over * strength);
|
|
752
|
+
return Math.max(minFactor, Math.min(1, factor));
|
|
753
|
+
}
|
|
754
|
+
function channelQuota(source, topK, options) {
|
|
755
|
+
const configured = options?.channelTopK?.[source];
|
|
756
|
+
if (typeof configured === "number" && Number.isFinite(configured) && configured >= 1) {
|
|
757
|
+
return Math.floor(configured);
|
|
758
|
+
}
|
|
759
|
+
if (source === "rules")
|
|
760
|
+
return Math.max(6, topK * 2);
|
|
761
|
+
if (source === "graph")
|
|
762
|
+
return Math.max(8, topK * 3);
|
|
763
|
+
return Math.max(12, topK * 4);
|
|
764
|
+
}
|
|
765
|
+
function parseJsonStringArray(value) {
|
|
766
|
+
if (typeof value !== "string" || !value.trim()) {
|
|
767
|
+
return [];
|
|
768
|
+
}
|
|
769
|
+
try {
|
|
770
|
+
const parsed = JSON.parse(value);
|
|
771
|
+
if (!Array.isArray(parsed)) {
|
|
772
|
+
return [];
|
|
773
|
+
}
|
|
774
|
+
return parsed
|
|
775
|
+
.map(item => (typeof item === "string" ? item.trim() : ""))
|
|
776
|
+
.filter(Boolean);
|
|
777
|
+
}
|
|
778
|
+
catch {
|
|
779
|
+
return [];
|
|
780
|
+
}
|
|
781
|
+
}
|
|
782
|
+
function parseJsonRelations(value) {
|
|
783
|
+
if (typeof value !== "string" || !value.trim()) {
|
|
784
|
+
return [];
|
|
785
|
+
}
|
|
786
|
+
try {
|
|
787
|
+
const parsed = JSON.parse(value);
|
|
788
|
+
if (!Array.isArray(parsed)) {
|
|
789
|
+
return [];
|
|
790
|
+
}
|
|
791
|
+
return parsed
|
|
792
|
+
.map(item => {
|
|
793
|
+
if (typeof item !== "object" || item === null)
|
|
794
|
+
return null;
|
|
795
|
+
const relation = item;
|
|
796
|
+
const source = typeof relation.source === "string" ? relation.source.trim() : "";
|
|
797
|
+
const target = typeof relation.target === "string" ? relation.target.trim() : "";
|
|
798
|
+
const type = typeof relation.type === "string" && relation.type.trim() ? relation.type.trim() : "related_to";
|
|
799
|
+
if (!source || !target)
|
|
800
|
+
return null;
|
|
801
|
+
return { source, target, type };
|
|
802
|
+
})
|
|
803
|
+
.filter((item) => Boolean(item));
|
|
804
|
+
}
|
|
805
|
+
catch {
|
|
806
|
+
return [];
|
|
807
|
+
}
|
|
808
|
+
}
|
|
809
|
+
async function searchLanceDb(args) {
|
|
810
|
+
try {
|
|
811
|
+
const require = (0, module_1.createRequire)(__filename);
|
|
812
|
+
const lancedbDir = path.join(args.memoryRoot, "vector", "lancedb");
|
|
813
|
+
if (!fs.existsSync(lancedbDir)) {
|
|
814
|
+
return [];
|
|
815
|
+
}
|
|
816
|
+
const moduleValue = require("@lancedb/lancedb");
|
|
817
|
+
const connect = moduleValue.connect;
|
|
818
|
+
if (typeof connect !== "function") {
|
|
819
|
+
return [];
|
|
820
|
+
}
|
|
821
|
+
const db = await connect(lancedbDir);
|
|
822
|
+
if (!db || typeof db.openTable !== "function") {
|
|
823
|
+
return [];
|
|
824
|
+
}
|
|
825
|
+
const table = await db.openTable("events");
|
|
826
|
+
if (!table || typeof table.search !== "function") {
|
|
827
|
+
return [];
|
|
828
|
+
}
|
|
829
|
+
const searchObj = table.search(args.queryEmbedding);
|
|
830
|
+
if (!searchObj || typeof searchObj.limit !== "function") {
|
|
831
|
+
return [];
|
|
832
|
+
}
|
|
833
|
+
const limited = searchObj.limit(args.limit);
|
|
834
|
+
if (!limited || typeof limited.toArray !== "function") {
|
|
835
|
+
return [];
|
|
836
|
+
}
|
|
837
|
+
const rows = await limited.toArray();
|
|
838
|
+
const docs = [];
|
|
839
|
+
for (const row of rows) {
|
|
840
|
+
if (typeof row !== "object" || row === null)
|
|
841
|
+
continue;
|
|
842
|
+
const record = row;
|
|
843
|
+
const id = typeof record.id === "string" ? record.id : "";
|
|
844
|
+
const summary = typeof record.summary === "string" ? record.summary : "";
|
|
845
|
+
if (!id || !summary)
|
|
846
|
+
continue;
|
|
847
|
+
const ts = typeof record.timestamp === "string" ? Date.parse(record.timestamp) : NaN;
|
|
848
|
+
const entities = parseJsonStringArray(record.entities_json);
|
|
849
|
+
const relations = parseJsonRelations(record.relations_json);
|
|
850
|
+
docs.push({
|
|
851
|
+
id,
|
|
852
|
+
text: summary,
|
|
853
|
+
source: "vector_lancedb",
|
|
854
|
+
timestamp: Number.isFinite(ts) ? ts : undefined,
|
|
855
|
+
layer: record.layer === "active" || record.layer === "archive" ? record.layer : undefined,
|
|
856
|
+
sourceMemoryId: typeof record.source_memory_id === "string" ? record.source_memory_id : undefined,
|
|
857
|
+
sourceMemoryCanonicalId: typeof record.source_memory_canonical_id === "string" ? record.source_memory_canonical_id : undefined,
|
|
858
|
+
sourceEventId: typeof record.source_event_id === "string" ? record.source_event_id : undefined,
|
|
859
|
+
sourceField: record.source_field === "summary" || record.source_field === "evidence"
|
|
860
|
+
? record.source_field
|
|
861
|
+
: undefined,
|
|
862
|
+
embedding: Array.isArray(record.vector) ? record.vector.filter(item => Number.isFinite(item)) : undefined,
|
|
863
|
+
eventType: typeof record.event_type === "string" ? record.event_type : undefined,
|
|
864
|
+
qualityScore: typeof record.quality_score === "number" ? record.quality_score : undefined,
|
|
865
|
+
charCount: typeof record.char_count === "number" ? record.char_count : undefined,
|
|
866
|
+
tokenCount: typeof record.token_count === "number" ? record.token_count : undefined,
|
|
867
|
+
sessionId: typeof record.session_id === "string" ? record.session_id : undefined,
|
|
868
|
+
entities,
|
|
869
|
+
relations: Array.isArray(relations) ? relations : [],
|
|
870
|
+
});
|
|
871
|
+
}
|
|
872
|
+
return docs;
|
|
873
|
+
}
|
|
874
|
+
catch (error) {
|
|
875
|
+
args.logger.debug(`LanceDB search fallback: ${error}`);
|
|
876
|
+
return [];
|
|
877
|
+
}
|
|
878
|
+
}
|
|
879
|
+
function parseVectorFallback(filePath, logger) {
|
|
880
|
+
const content = safeReadFile(filePath);
|
|
881
|
+
if (!content) {
|
|
882
|
+
return [];
|
|
883
|
+
}
|
|
884
|
+
const docs = [];
|
|
885
|
+
for (const line of content.split(/\r?\n/)) {
|
|
886
|
+
const trimmed = line.trim();
|
|
887
|
+
if (!trimmed)
|
|
888
|
+
continue;
|
|
889
|
+
try {
|
|
890
|
+
const parsed = JSON.parse(trimmed);
|
|
891
|
+
const id = typeof parsed.id === "string" ? parsed.id : "";
|
|
892
|
+
const summary = typeof parsed.summary === "string" ? parsed.summary.trim() : "";
|
|
893
|
+
if (!id || !summary)
|
|
894
|
+
continue;
|
|
895
|
+
const ts = typeof parsed.timestamp === "string" ? Date.parse(parsed.timestamp) : NaN;
|
|
896
|
+
const entities = Array.isArray(parsed.entities)
|
|
897
|
+
? parsed.entities.map(item => (typeof item === "string" ? item.trim() : "")).filter(Boolean)
|
|
898
|
+
: [];
|
|
899
|
+
const relations = Array.isArray(parsed.relations)
|
|
900
|
+
? parsed.relations
|
|
901
|
+
.map(item => {
|
|
902
|
+
if (typeof item !== "object" || item === null)
|
|
903
|
+
return null;
|
|
904
|
+
const relation = item;
|
|
905
|
+
const source = typeof relation.source === "string" ? relation.source.trim() : "";
|
|
906
|
+
const target = typeof relation.target === "string" ? relation.target.trim() : "";
|
|
907
|
+
const type = typeof relation.type === "string" ? relation.type.trim() : "related_to";
|
|
908
|
+
if (!source || !target)
|
|
909
|
+
return null;
|
|
910
|
+
return { source, target, type };
|
|
911
|
+
})
|
|
912
|
+
.filter((item) => Boolean(item))
|
|
913
|
+
: [];
|
|
914
|
+
docs.push({
|
|
915
|
+
id,
|
|
916
|
+
text: summary,
|
|
917
|
+
source: "vector_jsonl",
|
|
918
|
+
timestamp: Number.isFinite(ts) ? ts : undefined,
|
|
919
|
+
layer: parsed.layer === "active" || parsed.layer === "archive" ? parsed.layer : undefined,
|
|
920
|
+
sourceMemoryId: typeof parsed.source_memory_id === "string" ? parsed.source_memory_id : undefined,
|
|
921
|
+
sourceMemoryCanonicalId: typeof parsed.source_memory_canonical_id === "string" ? parsed.source_memory_canonical_id : undefined,
|
|
922
|
+
sourceEventId: typeof parsed.source_event_id === "string" ? parsed.source_event_id : undefined,
|
|
923
|
+
sourceField: parsed.source_field === "summary" || parsed.source_field === "evidence"
|
|
924
|
+
? parsed.source_field
|
|
925
|
+
: undefined,
|
|
926
|
+
embedding: Array.isArray(parsed.embedding) ? parsed.embedding.filter(item => Number.isFinite(item)) : undefined,
|
|
927
|
+
eventType: typeof parsed.event_type === "string" ? parsed.event_type.trim() : undefined,
|
|
928
|
+
qualityScore: typeof parsed.quality_score === "number" ? parsed.quality_score : undefined,
|
|
929
|
+
charCount: typeof parsed.char_count === "number" ? parsed.char_count : undefined,
|
|
930
|
+
tokenCount: typeof parsed.token_count === "number" ? parsed.token_count : undefined,
|
|
931
|
+
sessionId: typeof parsed.session_id === "string" ? parsed.session_id : undefined,
|
|
932
|
+
entities,
|
|
933
|
+
relations,
|
|
934
|
+
});
|
|
935
|
+
}
|
|
936
|
+
catch (error) {
|
|
937
|
+
logger.debug(`Skip invalid vector jsonl line: ${error}`);
|
|
938
|
+
}
|
|
939
|
+
}
|
|
940
|
+
return docs;
|
|
941
|
+
}
|
|
942
|
+
async function requestFusion(args) {
|
|
943
|
+
const candidateIdSet = new Set(args.candidates.map(item => item.id));
|
|
944
|
+
const endpoint = args.llm.baseUrl.endsWith("/chat/completions")
|
|
945
|
+
? args.llm.baseUrl
|
|
946
|
+
: `${args.llm.baseUrl}/chat/completions`;
|
|
947
|
+
const evidenceText = args.candidates
|
|
948
|
+
.map((item, index) => {
|
|
949
|
+
const excerpt = (item.source_excerpt || "").trim();
|
|
950
|
+
const sourceFile = (item.source_file || "").trim();
|
|
951
|
+
const sourceMemoryId = (item.source_memory_id || "").trim();
|
|
952
|
+
const sourceMemoryCanonicalId = (item.source_memory_canonical_id || "").trim();
|
|
953
|
+
const sourceLayer = (item.source_layer || "").trim();
|
|
954
|
+
const sourceEventId = (item.source_event_id || "").trim();
|
|
955
|
+
const sourceField = (item.source_field || "").trim();
|
|
956
|
+
const extraParts = [];
|
|
957
|
+
if (sourceMemoryId)
|
|
958
|
+
extraParts.push(`source_memory_id=${sourceMemoryId}`);
|
|
959
|
+
if (sourceMemoryCanonicalId)
|
|
960
|
+
extraParts.push(`source_memory_canonical_id=${sourceMemoryCanonicalId}`);
|
|
961
|
+
if (sourceLayer)
|
|
962
|
+
extraParts.push(`source_layer=${sourceLayer}`);
|
|
963
|
+
if (sourceEventId)
|
|
964
|
+
extraParts.push(`source_event_id=${sourceEventId}`);
|
|
965
|
+
if (sourceField)
|
|
966
|
+
extraParts.push(`source_field=${sourceField}`);
|
|
967
|
+
if (sourceFile)
|
|
968
|
+
extraParts.push(`source_file=${sourceFile}`);
|
|
969
|
+
if (excerpt)
|
|
970
|
+
extraParts.push(`source_excerpt=${excerpt}`);
|
|
971
|
+
const extra = extraParts.length > 0 ? `\n ${extraParts.join("\n ")}` : "";
|
|
972
|
+
return `${index + 1}. [${item.id}] (${item.source}, score=${item.score.toFixed(4)}) ${item.text}${extra}`;
|
|
973
|
+
})
|
|
974
|
+
.join("\n")
|
|
975
|
+
.slice(0, 18000);
|
|
976
|
+
const prompt = [
|
|
977
|
+
`prompt_version=${READ_FUSION_PROMPT_VERSION}`,
|
|
978
|
+
"You are a memory retrieval fusion engine. Fuse multi-channel evidence into a structured answer package for the agent.",
|
|
979
|
+
"Core values and principles:",
|
|
980
|
+
"A) Truthfulness first: do not fabricate; do not infer beyond evidence.",
|
|
981
|
+
"B) Evidence first: every key conclusion must be traceable via evidence_ids.",
|
|
982
|
+
"C) Make conflicts explicit: write conflicts instead of silently overriding.",
|
|
983
|
+
"D) Be transparent about uncertainty: put uncertain parts in coverage_note.",
|
|
984
|
+
"E) Summary-first: prefer summary evidence for conclusions; source_excerpt is supporting evidence.",
|
|
985
|
+
"F) Same-source dedup: merge duplicate evidence from the same source_memory_id/source_memory_canonical_id.",
|
|
986
|
+
"G) Full-text recall: if summary/excerpt is insufficient, return event ids in need_fulltext_event_ids.",
|
|
987
|
+
"Source channel semantics:",
|
|
988
|
+
"- rules: policy/constraints; use for what should be done.",
|
|
989
|
+
"- archive: event-level stable facts (summary-first).",
|
|
990
|
+
"- vector: semantic neighbors for recall; source_field=summary/evidence indicates chunk role.",
|
|
991
|
+
"- graph: entity-relation structure; prefer for dependency/relationship questions.",
|
|
992
|
+
"Query alignment:",
|
|
993
|
+
"- answer the user query first; ignore evidence unrelated to the query.",
|
|
994
|
+
"- when evidence conflicts, prioritize recency + quality + explicitness and record the conflict.",
|
|
995
|
+
"Return strict JSON only:",
|
|
996
|
+
"{\"canonical_answer\": string, \"coverage_note\": string, \"facts\": [{\"text\": string, \"evidence_ids\": string[]}], \"timeline\": [{\"when\": string, \"event\": string, \"evidence_ids\": string[]}], \"entities\": [{\"name\": string, \"role\": string}], \"decisions\": [{\"decision\": string, \"rationale\": string, \"evidence_ids\": string[]}], \"fixes\": [{\"issue\": string, \"fix\": string, \"evidence_ids\": string[]}], \"preferences\": [{\"subject\": string, \"preference\": string, \"evidence_ids\": string[]}], \"risks\": [{\"risk\": string, \"mitigation\": string, \"evidence_ids\": string[]}], \"action_items\": [{\"item\": string, \"owner\": string, \"status\": string, \"evidence_ids\": string[]}], \"conflicts\": [{\"topic\": string, \"details\": string}], \"evidence_ids\": string[], \"need_fulltext_event_ids\": string[], \"confidence\": number}",
|
|
997
|
+
"Output constraints:",
|
|
998
|
+
"1) canonical_answer must be directly usable.",
|
|
999
|
+
"2) facts: usually 3-12 items; prefer high-quality evidence.",
|
|
1000
|
+
"3) evidence_ids must come from input candidate ids.",
|
|
1001
|
+
"4) conflicts must be [] when no conflict exists.",
|
|
1002
|
+
"5) confidence must be within [0, 1].",
|
|
1003
|
+
"6) uncertain parts must be explicitly stated in coverage_note.",
|
|
1004
|
+
...READ_FUSION_REGRESSION_SAMPLES,
|
|
1005
|
+
].join("\n");
|
|
1006
|
+
const body = {
|
|
1007
|
+
model: args.llm.model,
|
|
1008
|
+
temperature: 0.1,
|
|
1009
|
+
messages: [
|
|
1010
|
+
{ role: "system", content: "Output JSON only. No extra text." },
|
|
1011
|
+
{ role: "user", content: `${prompt}\n\nQuery:\n${args.query}\n\nCandidate Evidence:\n${evidenceText}` },
|
|
1012
|
+
],
|
|
1013
|
+
};
|
|
1014
|
+
let lastError = null;
|
|
1015
|
+
for (let attempt = 0; attempt < 2; attempt += 1) {
|
|
1016
|
+
const controller = new AbortController();
|
|
1017
|
+
const timeoutId = setTimeout(() => controller.abort(), 20000);
|
|
1018
|
+
try {
|
|
1019
|
+
const response = await fetch(endpoint, {
|
|
1020
|
+
method: "POST",
|
|
1021
|
+
headers: {
|
|
1022
|
+
"content-type": "application/json",
|
|
1023
|
+
authorization: `Bearer ${args.llm.apiKey}`,
|
|
1024
|
+
},
|
|
1025
|
+
body: JSON.stringify(body),
|
|
1026
|
+
signal: controller.signal,
|
|
1027
|
+
});
|
|
1028
|
+
clearTimeout(timeoutId);
|
|
1029
|
+
if (!response.ok) {
|
|
1030
|
+
lastError = new Error(`fusion_http_${response.status}`);
|
|
1031
|
+
continue;
|
|
1032
|
+
}
|
|
1033
|
+
const json = await response.json();
|
|
1034
|
+
const content = json?.choices?.[0]?.message?.content?.trim() || "";
|
|
1035
|
+
if (!content) {
|
|
1036
|
+
lastError = new Error("fusion_empty");
|
|
1037
|
+
continue;
|
|
1038
|
+
}
|
|
1039
|
+
const parsed = JSON.parse(content);
|
|
1040
|
+
if (!parsed || typeof parsed.canonical_answer !== "string" || !parsed.canonical_answer.trim()) {
|
|
1041
|
+
lastError = new Error("fusion_invalid");
|
|
1042
|
+
continue;
|
|
1043
|
+
}
|
|
1044
|
+
const evidenceIds = Array.isArray(parsed.evidence_ids)
|
|
1045
|
+
? parsed.evidence_ids.filter(item => typeof item === "string" && item.trim())
|
|
1046
|
+
: [];
|
|
1047
|
+
const whitelistedEvidenceIds = [...new Set(evidenceIds.filter(id => candidateIdSet.has(id)))];
|
|
1048
|
+
const needFulltextEventIds = Array.isArray(parsed.need_fulltext_event_ids)
|
|
1049
|
+
? [...new Set(parsed.need_fulltext_event_ids
|
|
1050
|
+
.filter(item => typeof item === "string")
|
|
1051
|
+
.map(item => item.trim())
|
|
1052
|
+
.filter(Boolean))]
|
|
1053
|
+
: [];
|
|
1054
|
+
return {
|
|
1055
|
+
canonical_answer: parsed.canonical_answer.trim().slice(0, 6000),
|
|
1056
|
+
coverage_note: typeof parsed.coverage_note === "string" ? parsed.coverage_note.trim().slice(0, 1200) : "",
|
|
1057
|
+
facts: Array.isArray(parsed.facts) ? parsed.facts : [],
|
|
1058
|
+
timeline: Array.isArray(parsed.timeline) ? parsed.timeline : [],
|
|
1059
|
+
entities: Array.isArray(parsed.entities) ? parsed.entities : [],
|
|
1060
|
+
decisions: Array.isArray(parsed.decisions) ? parsed.decisions : [],
|
|
1061
|
+
fixes: Array.isArray(parsed.fixes) ? parsed.fixes : [],
|
|
1062
|
+
preferences: Array.isArray(parsed.preferences) ? parsed.preferences : [],
|
|
1063
|
+
risks: Array.isArray(parsed.risks) ? parsed.risks : [],
|
|
1064
|
+
action_items: Array.isArray(parsed.action_items) ? parsed.action_items : [],
|
|
1065
|
+
conflicts: Array.isArray(parsed.conflicts) ? parsed.conflicts : [],
|
|
1066
|
+
evidence_ids: whitelistedEvidenceIds,
|
|
1067
|
+
need_fulltext_event_ids: needFulltextEventIds,
|
|
1068
|
+
confidence: typeof parsed.confidence === "number"
|
|
1069
|
+
? Math.max(0, Math.min(1, parsed.confidence))
|
|
1070
|
+
: 0.5,
|
|
1071
|
+
};
|
|
1072
|
+
}
|
|
1073
|
+
catch (error) {
|
|
1074
|
+
clearTimeout(timeoutId);
|
|
1075
|
+
lastError = error;
|
|
1076
|
+
}
|
|
1077
|
+
}
|
|
1078
|
+
throw lastError instanceof Error ? lastError : new Error(String(lastError || "fusion_failed"));
|
|
1079
|
+
}
|
|
163
1080
|
function createReadStore(options) {
|
|
164
1081
|
const memoryRoot = options.dbPath ? path.resolve(options.dbPath) : path.join(options.projectRoot, "data", "memory");
|
|
1082
|
+
const vectorFallbackPath = path.join(memoryRoot, "vector", "lancedb_events.jsonl");
|
|
1083
|
+
const hitStatsPath = path.join(memoryRoot, ".read_hit_stats.json");
|
|
1084
|
+
let docsCache = null;
|
|
1085
|
+
let vectorFallbackCache = null;
|
|
1086
|
+
let bm25TokenCacheSignature = "";
|
|
1087
|
+
let bm25TokenCache = new Map();
|
|
1088
|
+
let hitStatsCache = null;
|
|
1089
|
+
let hitStatsDirty = false;
|
|
1090
|
+
let hitStatsPendingMutations = 0;
|
|
1091
|
+
let lastHitStatsFlushAt = 0;
|
|
1092
|
+
const hitStatsFlushIntervalMs = 5000;
|
|
1093
|
+
const hitStatsFlushBatch = 24;
|
|
1094
|
+
const readTuning = resolveReadTuning(options.readTuning);
|
|
1095
|
+
function fileSignature(filePath) {
|
|
1096
|
+
try {
|
|
1097
|
+
if (!fs.existsSync(filePath)) {
|
|
1098
|
+
return `${filePath}:missing`;
|
|
1099
|
+
}
|
|
1100
|
+
const stat = fs.statSync(filePath);
|
|
1101
|
+
return `${filePath}:${stat.size}:${Math.floor(stat.mtimeMs)}`;
|
|
1102
|
+
}
|
|
1103
|
+
catch {
|
|
1104
|
+
return `${filePath}:error`;
|
|
1105
|
+
}
|
|
1106
|
+
}
|
|
1107
|
+
function loadHitStats() {
|
|
1108
|
+
if (hitStatsCache) {
|
|
1109
|
+
return hitStatsCache;
|
|
1110
|
+
}
|
|
1111
|
+
try {
|
|
1112
|
+
if (!fs.existsSync(hitStatsPath)) {
|
|
1113
|
+
hitStatsCache = { items: {} };
|
|
1114
|
+
return hitStatsCache;
|
|
1115
|
+
}
|
|
1116
|
+
const content = fs.readFileSync(hitStatsPath, "utf-8").trim();
|
|
1117
|
+
if (!content) {
|
|
1118
|
+
hitStatsCache = { items: {} };
|
|
1119
|
+
return hitStatsCache;
|
|
1120
|
+
}
|
|
1121
|
+
const parsed = JSON.parse(content);
|
|
1122
|
+
if (!parsed || typeof parsed !== "object" || !parsed.items || typeof parsed.items !== "object") {
|
|
1123
|
+
hitStatsCache = { items: {} };
|
|
1124
|
+
return hitStatsCache;
|
|
1125
|
+
}
|
|
1126
|
+
hitStatsCache = parsed;
|
|
1127
|
+
return hitStatsCache;
|
|
1128
|
+
}
|
|
1129
|
+
catch {
|
|
1130
|
+
hitStatsCache = { items: {} };
|
|
1131
|
+
return hitStatsCache;
|
|
1132
|
+
}
|
|
1133
|
+
}
|
|
1134
|
+
function saveHitStats(state) {
|
|
1135
|
+
try {
|
|
1136
|
+
const dir = path.dirname(hitStatsPath);
|
|
1137
|
+
if (!fs.existsSync(dir)) {
|
|
1138
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
1139
|
+
}
|
|
1140
|
+
fs.writeFileSync(hitStatsPath, JSON.stringify(state, null, 2), "utf-8");
|
|
1141
|
+
}
|
|
1142
|
+
catch (error) {
|
|
1143
|
+
options.logger.warn(`Failed to persist read hit stats: ${error}`);
|
|
1144
|
+
}
|
|
1145
|
+
}
|
|
1146
|
+
function maybeFlushHitStats(force) {
|
|
1147
|
+
if (!hitStatsDirty || !hitStatsCache) {
|
|
1148
|
+
return;
|
|
1149
|
+
}
|
|
1150
|
+
const now = Date.now();
|
|
1151
|
+
if (!force &&
|
|
1152
|
+
hitStatsPendingMutations < hitStatsFlushBatch &&
|
|
1153
|
+
(now - lastHitStatsFlushAt) < hitStatsFlushIntervalMs) {
|
|
1154
|
+
return;
|
|
1155
|
+
}
|
|
1156
|
+
saveHitStats(hitStatsCache);
|
|
1157
|
+
hitStatsDirty = false;
|
|
1158
|
+
hitStatsPendingMutations = 0;
|
|
1159
|
+
lastHitStatsFlushAt = now;
|
|
1160
|
+
}
|
|
1161
|
+
function markHit(ids) {
|
|
1162
|
+
if (!ids.length) {
|
|
1163
|
+
return;
|
|
1164
|
+
}
|
|
1165
|
+
const state = loadHitStats();
|
|
1166
|
+
const now = new Date().toISOString();
|
|
1167
|
+
for (const id of ids) {
|
|
1168
|
+
const key = (id || "").trim();
|
|
1169
|
+
if (!key)
|
|
1170
|
+
continue;
|
|
1171
|
+
const prev = state.items[key];
|
|
1172
|
+
state.items[key] = {
|
|
1173
|
+
count: (prev?.count || 0) + 1,
|
|
1174
|
+
lastHitAt: now,
|
|
1175
|
+
};
|
|
1176
|
+
}
|
|
1177
|
+
const entries = Object.entries(state.items)
|
|
1178
|
+
.sort((a, b) => {
|
|
1179
|
+
const ta = Date.parse(a[1].lastHitAt || "");
|
|
1180
|
+
const tb = Date.parse(b[1].lastHitAt || "");
|
|
1181
|
+
return (Number.isFinite(tb) ? tb : 0) - (Number.isFinite(ta) ? ta : 0);
|
|
1182
|
+
})
|
|
1183
|
+
.slice(0, 20000);
|
|
1184
|
+
state.items = Object.fromEntries(entries);
|
|
1185
|
+
hitStatsCache = state;
|
|
1186
|
+
hitStatsDirty = true;
|
|
1187
|
+
hitStatsPendingMutations += ids.length;
|
|
1188
|
+
maybeFlushHitStats(false);
|
|
1189
|
+
}
|
|
165
1190
|
function loadAllDocuments() {
|
|
166
1191
|
const cortexRulesPath = path.join(memoryRoot, "CORTEX_RULES.md");
|
|
167
1192
|
const memoryMdPath = path.join(memoryRoot, "MEMORY.md");
|
|
168
1193
|
const activeSessionsPath = path.join(memoryRoot, "sessions", "active", "sessions.jsonl");
|
|
169
1194
|
const archiveSessionsPath = path.join(memoryRoot, "sessions", "archive", "sessions.jsonl");
|
|
170
|
-
|
|
1195
|
+
const graphMemoryPath = path.join(memoryRoot, "graph", "memory.jsonl");
|
|
1196
|
+
const signature = [
|
|
1197
|
+
fileSignature(cortexRulesPath),
|
|
1198
|
+
fileSignature(memoryMdPath),
|
|
1199
|
+
fileSignature(activeSessionsPath),
|
|
1200
|
+
fileSignature(archiveSessionsPath),
|
|
1201
|
+
fileSignature(graphMemoryPath),
|
|
1202
|
+
].join("|");
|
|
1203
|
+
if (docsCache && docsCache.signature === signature) {
|
|
1204
|
+
return docsCache.docs;
|
|
1205
|
+
}
|
|
1206
|
+
const archiveEventTypeById = new Map();
|
|
1207
|
+
if (fs.existsSync(archiveSessionsPath)) {
|
|
1208
|
+
const archiveContent = safeReadFile(archiveSessionsPath);
|
|
1209
|
+
for (const line of archiveContent.split(/\r?\n/)) {
|
|
1210
|
+
const trimmed = line.trim();
|
|
1211
|
+
if (!trimmed)
|
|
1212
|
+
continue;
|
|
1213
|
+
try {
|
|
1214
|
+
const parsed = JSON.parse(trimmed);
|
|
1215
|
+
const id = typeof parsed.id === "string" ? parsed.id.trim() : "";
|
|
1216
|
+
const eventType = typeof parsed.event_type === "string" ? parsed.event_type.trim() : "";
|
|
1217
|
+
if (id && eventType) {
|
|
1218
|
+
archiveEventTypeById.set(id, eventType);
|
|
1219
|
+
}
|
|
1220
|
+
}
|
|
1221
|
+
catch { }
|
|
1222
|
+
}
|
|
1223
|
+
}
|
|
1224
|
+
const graphDocs = [];
|
|
1225
|
+
if (fs.existsSync(graphMemoryPath)) {
|
|
1226
|
+
const graphContent = safeReadFile(graphMemoryPath);
|
|
1227
|
+
for (const line of graphContent.split(/\r?\n/)) {
|
|
1228
|
+
const trimmed = line.trim();
|
|
1229
|
+
if (!trimmed)
|
|
1230
|
+
continue;
|
|
1231
|
+
try {
|
|
1232
|
+
const parsed = JSON.parse(trimmed);
|
|
1233
|
+
const id = typeof parsed.id === "string" ? parsed.id : "";
|
|
1234
|
+
const sourceEventId = typeof parsed.source_event_id === "string" ? parsed.source_event_id : "";
|
|
1235
|
+
const archiveEventId = typeof parsed.archive_event_id === "string" ? parsed.archive_event_id : "";
|
|
1236
|
+
const eventRefId = archiveEventId || sourceEventId;
|
|
1237
|
+
const sessionId = typeof parsed.session_id === "string" ? parsed.session_id : "";
|
|
1238
|
+
const sourceLayer = typeof parsed.source_layer === "string" ? parsed.source_layer : "";
|
|
1239
|
+
const sourceFile = typeof parsed.source_file === "string" ? parsed.source_file : "";
|
|
1240
|
+
const timestamp = typeof parsed.timestamp === "string" ? Date.parse(parsed.timestamp) : NaN;
|
|
1241
|
+
const entities = Array.isArray(parsed.entities)
|
|
1242
|
+
? parsed.entities.map((item) => (typeof item === "string" ? item.trim() : "")).filter(Boolean)
|
|
1243
|
+
: [];
|
|
1244
|
+
const entityTypes = typeof parsed.entity_types === "object" && parsed.entity_types !== null
|
|
1245
|
+
? parsed.entity_types
|
|
1246
|
+
: {};
|
|
1247
|
+
const relations = Array.isArray(parsed.relations)
|
|
1248
|
+
? parsed.relations
|
|
1249
|
+
.map((item) => {
|
|
1250
|
+
if (typeof item !== "object" || item === null)
|
|
1251
|
+
return null;
|
|
1252
|
+
const relation = item;
|
|
1253
|
+
const source = typeof relation.source === "string" ? relation.source.trim() : "";
|
|
1254
|
+
const target = typeof relation.target === "string" ? relation.target.trim() : "";
|
|
1255
|
+
const type = typeof relation.type === "string" ? relation.type.trim() : "related_to";
|
|
1256
|
+
if (!source || !target)
|
|
1257
|
+
return null;
|
|
1258
|
+
return { source, target, type };
|
|
1259
|
+
})
|
|
1260
|
+
.filter((item) => Boolean(item))
|
|
1261
|
+
: [];
|
|
1262
|
+
const eventType = (typeof parsed.event_type === "string" ? parsed.event_type : "") || archiveEventTypeById.get(eventRefId) || "";
|
|
1263
|
+
const entityLines = entities.length > 0
|
|
1264
|
+
? entities.map((entity, index) => {
|
|
1265
|
+
const entityType = entityTypes[entity];
|
|
1266
|
+
return `${index + 1}. ${entity}${entityType ? ` (${entityType})` : ""}`;
|
|
1267
|
+
}).join("\n")
|
|
1268
|
+
: "none";
|
|
1269
|
+
const relationLines = relations.length > 0
|
|
1270
|
+
? relations.map((relation, index) => `${index + 1}. ${relation.source} -[${relation.type}]-> ${relation.target}`).join("\n")
|
|
1271
|
+
: "none";
|
|
1272
|
+
const relationFacts = relations.length > 0
|
|
1273
|
+
? relations.map(relation => `${relation.source} ${relation.type} ${relation.target}`).join(" | ")
|
|
1274
|
+
: "none";
|
|
1275
|
+
const text = [
|
|
1276
|
+
`# Graph Record`,
|
|
1277
|
+
`record_id: ${id}`,
|
|
1278
|
+
`source_event_id: ${sourceEventId || archiveEventId || "unknown"}`,
|
|
1279
|
+
`source_layer: ${sourceLayer || "unknown"}`,
|
|
1280
|
+
`archive_event_id: ${archiveEventId || "n/a"}`,
|
|
1281
|
+
`event_type: ${eventType || "unknown"}`,
|
|
1282
|
+
`session_id: ${sessionId || "unknown"}`,
|
|
1283
|
+
`source_file: ${sourceFile || "unknown"}`,
|
|
1284
|
+
``,
|
|
1285
|
+
`## Entities`,
|
|
1286
|
+
entityLines,
|
|
1287
|
+
``,
|
|
1288
|
+
`## Relations`,
|
|
1289
|
+
relationLines,
|
|
1290
|
+
``,
|
|
1291
|
+
`## Relation Facts`,
|
|
1292
|
+
relationFacts,
|
|
1293
|
+
].join("\n");
|
|
1294
|
+
if (id && text.trim()) {
|
|
1295
|
+
graphDocs.push({
|
|
1296
|
+
id,
|
|
1297
|
+
text,
|
|
1298
|
+
source: "sessions_graph",
|
|
1299
|
+
timestamp: Number.isFinite(timestamp) ? timestamp : undefined,
|
|
1300
|
+
layer: sourceLayer === "active_only" ? "active" : "archive",
|
|
1301
|
+
sourceMemoryId: eventRefId || id,
|
|
1302
|
+
sourceEventId: sourceEventId || archiveEventId || undefined,
|
|
1303
|
+
sessionId,
|
|
1304
|
+
entities,
|
|
1305
|
+
relations,
|
|
1306
|
+
});
|
|
1307
|
+
}
|
|
1308
|
+
}
|
|
1309
|
+
catch (error) {
|
|
1310
|
+
options.logger.debug(`Skipping invalid graph memory line: ${error}`);
|
|
1311
|
+
}
|
|
1312
|
+
}
|
|
1313
|
+
}
|
|
1314
|
+
const entitySummaryDocs = buildEntityGraphSummaryDocs(graphDocs);
|
|
1315
|
+
const docs = [
|
|
171
1316
|
...parseMarkdownFile(cortexRulesPath, "CORTEX_RULES.md"),
|
|
172
1317
|
...parseMarkdownFile(memoryMdPath, "MEMORY.md"),
|
|
173
1318
|
...parseJsonlFile(activeSessionsPath, "sessions_active", options.logger),
|
|
174
1319
|
...parseJsonlFile(archiveSessionsPath, "sessions_archive", options.logger),
|
|
1320
|
+
...graphDocs,
|
|
1321
|
+
...entitySummaryDocs,
|
|
175
1322
|
];
|
|
1323
|
+
docsCache = { signature, docs };
|
|
1324
|
+
return docs;
|
|
1325
|
+
}
|
|
1326
|
+
function loadVectorFallbackCached() {
|
|
1327
|
+
const signature = fileSignature(vectorFallbackPath);
|
|
1328
|
+
if (vectorFallbackCache && vectorFallbackCache.signature === signature) {
|
|
1329
|
+
return vectorFallbackCache.docs;
|
|
1330
|
+
}
|
|
1331
|
+
const docs = parseVectorFallback(vectorFallbackPath, options.logger);
|
|
1332
|
+
vectorFallbackCache = { signature, docs };
|
|
1333
|
+
return docs;
|
|
1334
|
+
}
|
|
1335
|
+
function getBm25Tokens(doc, signature) {
|
|
1336
|
+
if (bm25TokenCacheSignature !== signature) {
|
|
1337
|
+
bm25TokenCacheSignature = signature;
|
|
1338
|
+
bm25TokenCache = new Map();
|
|
1339
|
+
}
|
|
1340
|
+
const key = `${doc.source}|${doc.id}|${doc.text.length}|${doc.text.slice(0, 64)}`;
|
|
1341
|
+
const cached = bm25TokenCache.get(key);
|
|
1342
|
+
if (cached) {
|
|
1343
|
+
return cached;
|
|
1344
|
+
}
|
|
1345
|
+
const tokens = tokenize(doc.text);
|
|
1346
|
+
bm25TokenCache.set(key, tokens);
|
|
1347
|
+
return tokens;
|
|
176
1348
|
}
|
|
177
1349
|
async function searchMemory(args) {
|
|
178
1350
|
const query = args.query?.trim();
|
|
179
1351
|
if (!query) {
|
|
180
1352
|
return { results: [] };
|
|
181
1353
|
}
|
|
1354
|
+
const mode = args.mode === "lightweight" ? "lightweight" : "default";
|
|
1355
|
+
const lightweightMode = mode === "lightweight";
|
|
182
1356
|
const docs = loadAllDocuments();
|
|
183
|
-
const
|
|
1357
|
+
const hitStats = loadHitStats();
|
|
1358
|
+
const intent = classifyIntent(query);
|
|
1359
|
+
const preferredTypes = preferredEventTypes(intent);
|
|
1360
|
+
let queryEmbedding = null;
|
|
1361
|
+
const embeddingModel = options.embedding?.model || "";
|
|
1362
|
+
const embeddingApiKey = options.embedding?.apiKey || "";
|
|
1363
|
+
const embeddingBaseUrl = normalizeBaseUrl(options.embedding?.baseURL || options.embedding?.baseUrl);
|
|
1364
|
+
if (!lightweightMode && embeddingModel && embeddingApiKey && embeddingBaseUrl) {
|
|
1365
|
+
try {
|
|
1366
|
+
queryEmbedding = await requestEmbedding({
|
|
1367
|
+
text: query,
|
|
1368
|
+
model: embeddingModel,
|
|
1369
|
+
apiKey: embeddingApiKey,
|
|
1370
|
+
baseUrl: embeddingBaseUrl,
|
|
1371
|
+
dimensions: options.embedding?.dimensions,
|
|
1372
|
+
});
|
|
1373
|
+
}
|
|
1374
|
+
catch (error) {
|
|
1375
|
+
options.logger.warn(`Embedding query failed, fallback to lexical search: ${error}`);
|
|
1376
|
+
}
|
|
1377
|
+
}
|
|
1378
|
+
const vectorDocsFromLance = queryEmbedding && queryEmbedding.length > 0
|
|
1379
|
+
? await searchLanceDb({ memoryRoot, queryEmbedding, limit: Math.max(20, args.topK * 8), logger: options.logger })
|
|
1380
|
+
: [];
|
|
1381
|
+
const vectorDocsFallback = vectorDocsFromLance.length > 0
|
|
1382
|
+
? []
|
|
1383
|
+
: loadVectorFallbackCached();
|
|
1384
|
+
const vectorDocs = [...vectorDocsFromLance, ...vectorDocsFallback];
|
|
1385
|
+
const archiveSourceById = new Map();
|
|
1386
|
+
for (const doc of docs) {
|
|
1387
|
+
if (doc.source !== "sessions_archive")
|
|
1388
|
+
continue;
|
|
1389
|
+
const key = (doc.sourceMemoryId || doc.id || "").trim();
|
|
1390
|
+
if (!key)
|
|
1391
|
+
continue;
|
|
1392
|
+
archiveSourceById.set(key, {
|
|
1393
|
+
sourceText: doc.sourceText,
|
|
1394
|
+
summaryText: doc.summaryText || doc.text,
|
|
1395
|
+
sourceFile: doc.sourceFile,
|
|
1396
|
+
});
|
|
1397
|
+
}
|
|
1398
|
+
for (const doc of vectorDocs) {
|
|
1399
|
+
const key = (doc.sourceMemoryId || "").trim();
|
|
1400
|
+
if (!key)
|
|
1401
|
+
continue;
|
|
1402
|
+
const linked = archiveSourceById.get(key);
|
|
1403
|
+
if (!linked)
|
|
1404
|
+
continue;
|
|
1405
|
+
if (!doc.sourceText && linked.sourceText) {
|
|
1406
|
+
doc.sourceText = linked.sourceText;
|
|
1407
|
+
}
|
|
1408
|
+
if (!doc.summaryText && linked.summaryText) {
|
|
1409
|
+
doc.summaryText = linked.summaryText;
|
|
1410
|
+
}
|
|
1411
|
+
if (!doc.sourceFile && linked.sourceFile) {
|
|
1412
|
+
doc.sourceFile = linked.sourceFile;
|
|
1413
|
+
}
|
|
1414
|
+
}
|
|
1415
|
+
const graphDocs = docs
|
|
1416
|
+
.filter(doc => doc.source === "sessions_graph" || doc.source === "sessions_graph_entity")
|
|
184
1417
|
.map(doc => {
|
|
185
|
-
const
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
1418
|
+
const graphText = [
|
|
1419
|
+
doc.text,
|
|
1420
|
+
...(doc.relations || []).map(relation => `${relation.source} ${relation.type} ${relation.target}`),
|
|
1421
|
+
].join(" | ");
|
|
1422
|
+
return {
|
|
1423
|
+
...doc,
|
|
1424
|
+
text: graphText,
|
|
1425
|
+
};
|
|
1426
|
+
});
|
|
1427
|
+
const rulesDocs = docs.filter(doc => doc.source === "CORTEX_RULES.md");
|
|
1428
|
+
const archiveDocs = docs.filter(doc => doc.source === "sessions_active" || doc.source === "sessions_archive");
|
|
1429
|
+
const bm25Terms = tokenize(query);
|
|
1430
|
+
const bm25Corpus = [...rulesDocs, ...archiveDocs, ...vectorDocs, ...graphDocs];
|
|
1431
|
+
const bm25Signature = `${docsCache?.signature || "na"}|vector:${vectorDocs.length}:${vectorDocs.slice(0, 40).map(item => `${item.id}:${item.text.length}`).join(",")}`;
|
|
1432
|
+
const bm25Stats = buildBm25Stats(bm25Corpus, bm25Terms, doc => getBm25Tokens(doc, bm25Signature));
|
|
1433
|
+
const combinedCandidates = [];
|
|
1434
|
+
const channels = {
|
|
1435
|
+
rules: [],
|
|
1436
|
+
archive: [],
|
|
1437
|
+
vector: [],
|
|
1438
|
+
graph: [],
|
|
1439
|
+
};
|
|
1440
|
+
const evaluateDoc = (doc, source) => {
|
|
1441
|
+
const lexical = scoreText(query, doc.text);
|
|
1442
|
+
const bm25 = bm25Score({
|
|
1443
|
+
queryTerms: bm25Terms,
|
|
1444
|
+
docText: doc.text,
|
|
1445
|
+
docTokens: getBm25Tokens(doc, bm25Signature),
|
|
1446
|
+
docCount: bm25Corpus.length,
|
|
1447
|
+
avgDocLen: bm25Stats.avgDocLen,
|
|
1448
|
+
docFreq: bm25Stats.docFreq,
|
|
1449
|
+
});
|
|
1450
|
+
const lexicalCombined = lexical + bm25 * readTuning.scoring.bm25Scale;
|
|
1451
|
+
const semantic = queryEmbedding && Array.isArray(doc.embedding) && doc.embedding.length > 0
|
|
1452
|
+
? Math.max(0, cosineSimilarity(queryEmbedding, doc.embedding) * 5)
|
|
1453
|
+
: 0;
|
|
1454
|
+
if (lexicalCombined <= 0 && semantic <= 0) {
|
|
1455
|
+
return null;
|
|
1456
|
+
}
|
|
1457
|
+
const recency = recencyScore(doc.timestamp, readTuning.recency.buckets);
|
|
1458
|
+
const quality = typeof doc.qualityScore === "number" ? Math.max(0, Math.min(1, doc.qualityScore)) : 0.5;
|
|
1459
|
+
const typeMatch = preferredTypes.length > 0 && doc.eventType
|
|
1460
|
+
? (preferredTypes.includes(doc.eventType) ? 1 : 0)
|
|
1461
|
+
: 0.5;
|
|
1462
|
+
const graphMatch = source === "graph" ? 1 : 0;
|
|
1463
|
+
const sourceBaseWeight = sourceWeight(source, intent);
|
|
1464
|
+
const sourceConfigWeight = customChannelWeight(source, options.fusion);
|
|
1465
|
+
const lengthNorm = lengthNormalizeFactor(doc, options.fusion);
|
|
1466
|
+
const baseWeighted = (readTuning.scoring.lexicalWeight * lexicalCombined +
|
|
1467
|
+
readTuning.scoring.semanticWeight * (semantic * lengthNorm) +
|
|
1468
|
+
readTuning.scoring.recencyWeight * recency +
|
|
1469
|
+
readTuning.scoring.qualityWeight * quality +
|
|
1470
|
+
readTuning.scoring.typeMatchWeight * typeMatch +
|
|
1471
|
+
readTuning.scoring.graphMatchWeight * graphMatch) * sourceBaseWeight * sourceConfigWeight;
|
|
1472
|
+
const decayFactor = computeDecayFactor(doc.id, doc.eventType, doc.timestamp, options.memoryDecay, hitStats);
|
|
1473
|
+
const weighted = baseWeighted * decayFactor;
|
|
1474
|
+
return {
|
|
1475
|
+
doc,
|
|
1476
|
+
source,
|
|
1477
|
+
lexical: lexicalCombined,
|
|
1478
|
+
bm25,
|
|
1479
|
+
semantic,
|
|
1480
|
+
recency,
|
|
1481
|
+
quality,
|
|
1482
|
+
typeMatch,
|
|
1483
|
+
graphMatch,
|
|
1484
|
+
decayFactor,
|
|
1485
|
+
weighted,
|
|
1486
|
+
};
|
|
1487
|
+
};
|
|
1488
|
+
for (const doc of rulesDocs) {
|
|
1489
|
+
const candidate = evaluateDoc(doc, "rules");
|
|
1490
|
+
if (candidate)
|
|
1491
|
+
channels.rules.push(candidate);
|
|
1492
|
+
}
|
|
1493
|
+
for (const doc of archiveDocs) {
|
|
1494
|
+
const candidate = evaluateDoc(doc, "archive");
|
|
1495
|
+
if (candidate)
|
|
1496
|
+
channels.archive.push(candidate);
|
|
1497
|
+
}
|
|
1498
|
+
for (const doc of vectorDocs) {
|
|
1499
|
+
const candidate = evaluateDoc(doc, "vector");
|
|
1500
|
+
if (candidate)
|
|
1501
|
+
channels.vector.push(candidate);
|
|
1502
|
+
}
|
|
1503
|
+
for (const doc of graphDocs) {
|
|
1504
|
+
const candidate = evaluateDoc(doc, "graph");
|
|
1505
|
+
if (candidate)
|
|
1506
|
+
channels.graph.push(candidate);
|
|
1507
|
+
}
|
|
1508
|
+
for (const key of Object.keys(channels)) {
|
|
1509
|
+
channels[key].sort((a, b) => b.weighted - a.weighted);
|
|
1510
|
+
combinedCandidates.push(...channels[key].slice(0, channelQuota(key, args.topK, options.fusion)));
|
|
1511
|
+
}
|
|
1512
|
+
const rrfMap = new Map();
|
|
1513
|
+
const weightedMap = new Map();
|
|
1514
|
+
const rrfK = readTuning.rrf.k;
|
|
1515
|
+
for (const key of Object.keys(channels)) {
|
|
1516
|
+
const list = channels[key];
|
|
1517
|
+
for (let i = 0; i < list.length; i += 1) {
|
|
1518
|
+
const candidate = list[i];
|
|
1519
|
+
const rrf = 1 / (rrfK + i + 1);
|
|
1520
|
+
const mergeKey = mergeKeyFromDoc(candidate.doc);
|
|
1521
|
+
rrfMap.set(mergeKey, (rrfMap.get(mergeKey) || 0) + rrf);
|
|
1522
|
+
const current = weightedMap.get(mergeKey);
|
|
1523
|
+
if (!current || candidate.weighted > current.weighted) {
|
|
1524
|
+
weightedMap.set(mergeKey, candidate);
|
|
1525
|
+
}
|
|
1526
|
+
}
|
|
1527
|
+
}
|
|
1528
|
+
const preRanked = [...weightedMap.entries()]
|
|
1529
|
+
.map(([mergeKey, candidate]) => ({
|
|
1530
|
+
id: candidate.doc.id,
|
|
1531
|
+
merge_key: mergeKey,
|
|
1532
|
+
source_memory_id: candidate.doc.sourceMemoryId || "",
|
|
1533
|
+
source_memory_canonical_id: candidate.doc.sourceMemoryCanonicalId || "",
|
|
1534
|
+
source_event_id: candidate.doc.sourceEventId || "",
|
|
1535
|
+
source_field: candidate.doc.sourceField || "",
|
|
1536
|
+
text: candidate.doc.summaryText || candidate.doc.text,
|
|
1537
|
+
source_text: candidate.doc.sourceText ? candidate.doc.sourceText.slice(0, 4000) : "",
|
|
1538
|
+
source_excerpt: candidate.doc.sourceText ? candidate.doc.sourceText.slice(0, 360) : "",
|
|
1539
|
+
source_file: candidate.doc.sourceFile || "",
|
|
1540
|
+
source: candidate.doc.source,
|
|
1541
|
+
layer: candidate.doc.layer || "",
|
|
1542
|
+
event_type: candidate.doc.eventType || "",
|
|
1543
|
+
quality_score: candidate.quality,
|
|
1544
|
+
timestamp: candidate.doc.timestamp ? new Date(candidate.doc.timestamp).toISOString() : "",
|
|
1545
|
+
score: candidate.weighted + (rrfMap.get(mergeKey) || 0) * readTuning.rrf.weight,
|
|
1546
|
+
score_breakdown: {
|
|
1547
|
+
lexical: Number(candidate.lexical.toFixed(4)),
|
|
1548
|
+
bm25: Number(candidate.bm25.toFixed(4)),
|
|
1549
|
+
semantic: Number(candidate.semantic.toFixed(4)),
|
|
1550
|
+
recency: Number(candidate.recency.toFixed(4)),
|
|
1551
|
+
quality: Number(candidate.quality.toFixed(4)),
|
|
1552
|
+
type: Number(candidate.typeMatch.toFixed(4)),
|
|
1553
|
+
graph: Number(candidate.graphMatch.toFixed(4)),
|
|
1554
|
+
decay: Number(candidate.decayFactor.toFixed(4)),
|
|
1555
|
+
rrf: Number(((rrfMap.get(mergeKey) || 0) * readTuning.rrf.weight).toFixed(4)),
|
|
1556
|
+
weighted: Number(candidate.weighted.toFixed(4)),
|
|
1557
|
+
},
|
|
1558
|
+
reason_tags: [
|
|
1559
|
+
`intent:${intent.toLowerCase()}`,
|
|
1560
|
+
candidate.semantic > 0 ? "vector_hit" : "lexical_hit",
|
|
1561
|
+
candidate.typeMatch >= 1 ? "event_type_match" : "event_type_weak",
|
|
1562
|
+
candidate.recency >= 0.8 ? "recent" : "historical",
|
|
1563
|
+
candidate.quality >= 0.7 ? "high_quality" : "normal_quality",
|
|
1564
|
+
candidate.decayFactor < 1 ? `decay:${candidate.decayFactor.toFixed(3)}` : "decay:1.000",
|
|
1565
|
+
`source:${candidate.source}`,
|
|
1566
|
+
`merge_key:${mergeKey}`,
|
|
1567
|
+
],
|
|
1568
|
+
}))
|
|
190
1569
|
.sort((a, b) => b.score - a.score)
|
|
191
|
-
.slice(0, Math.max(1, args.topK))
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
1570
|
+
.slice(0, Math.max(1, Math.max(args.topK, 20)));
|
|
1571
|
+
const lexicalRanked = preRanked
|
|
1572
|
+
.map(doc => {
|
|
1573
|
+
const boost = withRecencyBoost(doc.score, doc.timestamp ? Date.parse(doc.timestamp) : undefined, readTuning.recency.buckets);
|
|
1574
|
+
return { ...doc, score: Number(boost.toFixed(4)) };
|
|
1575
|
+
});
|
|
1576
|
+
const rerankerModel = options.reranker?.model || "";
|
|
1577
|
+
const rerankerApiKey = options.reranker?.apiKey || "";
|
|
1578
|
+
const rerankerBaseUrl = normalizeBaseUrl(options.reranker?.baseURL || options.reranker?.baseUrl);
|
|
1579
|
+
const fusionEnabled = !lightweightMode && options.fusion?.enabled !== false;
|
|
1580
|
+
const llmModel = options.llm?.model || "";
|
|
1581
|
+
const llmApiKey = options.llm?.apiKey || "";
|
|
1582
|
+
const llmBaseUrl = normalizeBaseUrl(options.llm?.baseURL || options.llm?.baseUrl);
|
|
1583
|
+
const fusionAuthoritative = options.fusion?.authoritative !== false;
|
|
1584
|
+
const skipRerankerForFusion = fusionEnabled && fusionAuthoritative && llmModel && llmApiKey && llmBaseUrl;
|
|
1585
|
+
let rerankedSimple = lexicalRanked.map(item => ({
|
|
1586
|
+
id: item.id,
|
|
1587
|
+
merge_key: item.merge_key,
|
|
1588
|
+
text: item.text,
|
|
1589
|
+
source: item.source,
|
|
1590
|
+
score: item.score,
|
|
197
1591
|
}));
|
|
198
|
-
|
|
1592
|
+
if (!lightweightMode && rerankerModel && rerankerApiKey && rerankerBaseUrl && lexicalRanked.length > 1 && !skipRerankerForFusion) {
|
|
1593
|
+
try {
|
|
1594
|
+
rerankedSimple = await requestRerank({
|
|
1595
|
+
query,
|
|
1596
|
+
candidates: lexicalRanked.map(item => ({ id: item.id, text: item.text, source: item.source, score: item.score })),
|
|
1597
|
+
model: rerankerModel,
|
|
1598
|
+
apiKey: rerankerApiKey,
|
|
1599
|
+
baseUrl: rerankerBaseUrl,
|
|
1600
|
+
});
|
|
1601
|
+
rerankedSimple = rerankedSimple.map(item => {
|
|
1602
|
+
const found = lexicalRanked.find(entry => entry.id === item.id);
|
|
1603
|
+
return { ...item, merge_key: found?.merge_key || item.id };
|
|
1604
|
+
});
|
|
1605
|
+
}
|
|
1606
|
+
catch (error) {
|
|
1607
|
+
options.logger.warn(`Reranker failed, keep hybrid ranking: ${error}`);
|
|
1608
|
+
}
|
|
1609
|
+
}
|
|
1610
|
+
const ranked = rerankedSimple.slice(0, Math.max(1, args.topK)).map(item => {
|
|
1611
|
+
const hit = lexicalRanked.find(entry => entry.id === item.id);
|
|
1612
|
+
return {
|
|
1613
|
+
id: item.id,
|
|
1614
|
+
merge_key: hit?.merge_key || item.merge_key || item.id,
|
|
1615
|
+
source_memory_id: hit?.source_memory_id || "",
|
|
1616
|
+
source_memory_canonical_id: hit?.source_memory_canonical_id || "",
|
|
1617
|
+
source_event_id: hit?.source_event_id || "",
|
|
1618
|
+
source_field: hit?.source_field || "",
|
|
1619
|
+
fulltext_event_id: (hit?.source_event_id || hit?.source_memory_id || item.id || ""),
|
|
1620
|
+
text: item.text,
|
|
1621
|
+
source_text: hit?.source_text || "",
|
|
1622
|
+
source_excerpt: hit?.source_excerpt || "",
|
|
1623
|
+
source_file: hit?.source_file || "",
|
|
1624
|
+
source: item.source,
|
|
1625
|
+
layer: hit?.layer || "",
|
|
1626
|
+
event_type: hit?.event_type || "",
|
|
1627
|
+
quality_score: hit?.quality_score ?? 0,
|
|
1628
|
+
timestamp: hit?.timestamp || "",
|
|
1629
|
+
score: Number(item.score.toFixed(4)),
|
|
1630
|
+
score_breakdown: hit?.score_breakdown || {},
|
|
1631
|
+
reason_tags: Array.isArray(hit?.reason_tags) ? hit?.reason_tags : [],
|
|
1632
|
+
explain: {
|
|
1633
|
+
merge_key: hit?.merge_key || item.merge_key || item.id,
|
|
1634
|
+
source_memory_id: hit?.source_memory_id || "",
|
|
1635
|
+
source_memory_canonical_id: hit?.source_memory_canonical_id || "",
|
|
1636
|
+
source_event_id: hit?.source_event_id || "",
|
|
1637
|
+
source_field: hit?.source_field || "",
|
|
1638
|
+
channel: item.source,
|
|
1639
|
+
source_file: hit?.source_file || "",
|
|
1640
|
+
layer: hit?.layer || "",
|
|
1641
|
+
score_breakdown: hit?.score_breakdown || {},
|
|
1642
|
+
reason_tags: Array.isArray(hit?.reason_tags) ? hit?.reason_tags : [],
|
|
1643
|
+
},
|
|
1644
|
+
};
|
|
1645
|
+
});
|
|
1646
|
+
const minLexicalHits = Math.max(0, Math.floor(options.fusion?.minLexicalHits ?? 1));
|
|
1647
|
+
const minSemanticHits = Math.max(0, Math.floor(options.fusion?.minSemanticHits ?? 1));
|
|
1648
|
+
const fallbackPool = lexicalRanked.filter(item => !ranked.some(existing => existing.id === item.id));
|
|
1649
|
+
const lexicalCount = ranked.filter(item => item.reason_tags.includes("lexical_hit")).length;
|
|
1650
|
+
const semanticCount = ranked.filter(item => item.reason_tags.includes("vector_hit")).length;
|
|
1651
|
+
if (semanticCount < minSemanticHits) {
|
|
1652
|
+
const needed = minSemanticHits - semanticCount;
|
|
1653
|
+
const supplement = fallbackPool.filter(item => item.reason_tags.includes("vector_hit")).slice(0, needed);
|
|
1654
|
+
for (const item of supplement) {
|
|
1655
|
+
ranked.push({
|
|
1656
|
+
id: item.id,
|
|
1657
|
+
merge_key: item.merge_key,
|
|
1658
|
+
source_memory_id: item.source_memory_id,
|
|
1659
|
+
source_memory_canonical_id: item.source_memory_canonical_id,
|
|
1660
|
+
source_event_id: item.source_event_id || "",
|
|
1661
|
+
source_field: item.source_field || "",
|
|
1662
|
+
fulltext_event_id: item.source_event_id || item.source_memory_id || item.id,
|
|
1663
|
+
text: item.text,
|
|
1664
|
+
source_text: item.source_text || "",
|
|
1665
|
+
source_excerpt: item.source_excerpt || "",
|
|
1666
|
+
source_file: item.source_file || "",
|
|
1667
|
+
source: item.source,
|
|
1668
|
+
layer: item.layer,
|
|
1669
|
+
event_type: item.event_type,
|
|
1670
|
+
quality_score: item.quality_score,
|
|
1671
|
+
timestamp: item.timestamp,
|
|
1672
|
+
score: Number(item.score.toFixed(4)),
|
|
1673
|
+
score_breakdown: item.score_breakdown || {},
|
|
1674
|
+
reason_tags: Array.isArray(item.reason_tags) ? item.reason_tags : [],
|
|
1675
|
+
explain: {
|
|
1676
|
+
merge_key: item.merge_key,
|
|
1677
|
+
source_memory_id: item.source_memory_id,
|
|
1678
|
+
source_memory_canonical_id: item.source_memory_canonical_id,
|
|
1679
|
+
source_event_id: item.source_event_id || "",
|
|
1680
|
+
source_field: item.source_field || "",
|
|
1681
|
+
channel: item.source,
|
|
1682
|
+
source_file: item.source_file || "",
|
|
1683
|
+
layer: item.layer,
|
|
1684
|
+
score_breakdown: item.score_breakdown || {},
|
|
1685
|
+
reason_tags: Array.isArray(item.reason_tags) ? item.reason_tags : [],
|
|
1686
|
+
},
|
|
1687
|
+
});
|
|
1688
|
+
}
|
|
1689
|
+
}
|
|
1690
|
+
if (lexicalCount < minLexicalHits) {
|
|
1691
|
+
const needed = minLexicalHits - lexicalCount;
|
|
1692
|
+
const supplement = fallbackPool.filter(item => item.reason_tags.includes("lexical_hit")).slice(0, needed);
|
|
1693
|
+
for (const item of supplement) {
|
|
1694
|
+
if (ranked.some(existing => existing.id === item.id)) {
|
|
1695
|
+
continue;
|
|
1696
|
+
}
|
|
1697
|
+
ranked.push({
|
|
1698
|
+
id: item.id,
|
|
1699
|
+
merge_key: item.merge_key,
|
|
1700
|
+
source_memory_id: item.source_memory_id,
|
|
1701
|
+
source_memory_canonical_id: item.source_memory_canonical_id,
|
|
1702
|
+
source_event_id: item.source_event_id || "",
|
|
1703
|
+
source_field: item.source_field || "",
|
|
1704
|
+
fulltext_event_id: item.source_event_id || item.source_memory_id || item.id,
|
|
1705
|
+
text: item.text,
|
|
1706
|
+
source_text: item.source_text || "",
|
|
1707
|
+
source_excerpt: item.source_excerpt || "",
|
|
1708
|
+
source_file: item.source_file || "",
|
|
1709
|
+
source: item.source,
|
|
1710
|
+
layer: item.layer,
|
|
1711
|
+
event_type: item.event_type,
|
|
1712
|
+
quality_score: item.quality_score,
|
|
1713
|
+
timestamp: item.timestamp,
|
|
1714
|
+
score: Number(item.score.toFixed(4)),
|
|
1715
|
+
score_breakdown: item.score_breakdown || {},
|
|
1716
|
+
reason_tags: Array.isArray(item.reason_tags) ? item.reason_tags : [],
|
|
1717
|
+
explain: {
|
|
1718
|
+
merge_key: item.merge_key,
|
|
1719
|
+
source_memory_id: item.source_memory_id,
|
|
1720
|
+
source_memory_canonical_id: item.source_memory_canonical_id,
|
|
1721
|
+
source_event_id: item.source_event_id || "",
|
|
1722
|
+
source_field: item.source_field || "",
|
|
1723
|
+
channel: item.source,
|
|
1724
|
+
source_file: item.source_file || "",
|
|
1725
|
+
layer: item.layer,
|
|
1726
|
+
score_breakdown: item.score_breakdown || {},
|
|
1727
|
+
reason_tags: Array.isArray(item.reason_tags) ? item.reason_tags : [],
|
|
1728
|
+
},
|
|
1729
|
+
});
|
|
1730
|
+
}
|
|
1731
|
+
}
|
|
1732
|
+
ranked.sort((a, b) => b.score - a.score);
|
|
1733
|
+
if (fusionEnabled && llmModel && llmApiKey && llmBaseUrl && ranked.length > 1) {
|
|
1734
|
+
try {
|
|
1735
|
+
const maxCandidates = Math.max(4, Math.min(20, options.fusion?.maxCandidates ?? 10));
|
|
1736
|
+
const fusion = await requestFusion({
|
|
1737
|
+
query,
|
|
1738
|
+
candidates: ranked.slice(0, maxCandidates).map(item => ({
|
|
1739
|
+
id: item.id,
|
|
1740
|
+
text: item.text,
|
|
1741
|
+
source_excerpt: typeof item.source_excerpt === "string" ? item.source_excerpt : "",
|
|
1742
|
+
source_file: typeof item.source_file === "string" ? item.source_file : "",
|
|
1743
|
+
source_memory_id: typeof item.source_memory_id === "string" ? item.source_memory_id : "",
|
|
1744
|
+
source_memory_canonical_id: typeof item.source_memory_canonical_id === "string" ? item.source_memory_canonical_id : "",
|
|
1745
|
+
source_layer: typeof item.layer === "string" ? item.layer : "",
|
|
1746
|
+
source_event_id: typeof item.source_event_id === "string" ? item.source_event_id : "",
|
|
1747
|
+
source_field: item.source_field === "summary" || item.source_field === "evidence" ? item.source_field : "",
|
|
1748
|
+
source: item.source,
|
|
1749
|
+
event_type: item.event_type,
|
|
1750
|
+
quality_score: item.quality_score,
|
|
1751
|
+
timestamp: item.timestamp,
|
|
1752
|
+
score: item.score,
|
|
1753
|
+
reason_tags: Array.isArray(item.reason_tags) ? item.reason_tags : [],
|
|
1754
|
+
})),
|
|
1755
|
+
llm: {
|
|
1756
|
+
model: llmModel,
|
|
1757
|
+
apiKey: llmApiKey,
|
|
1758
|
+
baseUrl: llmBaseUrl,
|
|
1759
|
+
},
|
|
1760
|
+
});
|
|
1761
|
+
if (fusion && fusion.canonical_answer) {
|
|
1762
|
+
if (!Array.isArray(fusion.evidence_ids) || fusion.evidence_ids.length === 0) {
|
|
1763
|
+
throw new Error("fusion_missing_whitelisted_evidence");
|
|
1764
|
+
}
|
|
1765
|
+
const fulltextFetchHints = (Array.isArray(fusion.need_fulltext_event_ids) ? fusion.need_fulltext_event_ids : [])
|
|
1766
|
+
.map(eventId => {
|
|
1767
|
+
const linked = ranked.find(item => item.source_memory_id === eventId ||
|
|
1768
|
+
item.source_memory_canonical_id === eventId ||
|
|
1769
|
+
item.id === eventId);
|
|
1770
|
+
return {
|
|
1771
|
+
event_id: eventId,
|
|
1772
|
+
source_file: linked?.source_file || "",
|
|
1773
|
+
source_excerpt: linked?.source_excerpt || "",
|
|
1774
|
+
};
|
|
1775
|
+
});
|
|
1776
|
+
const fusedItem = {
|
|
1777
|
+
id: `fusion_${Date.now().toString(36)}`,
|
|
1778
|
+
text: fusion.canonical_answer,
|
|
1779
|
+
source: "llm_fusion",
|
|
1780
|
+
event_type: "fusion",
|
|
1781
|
+
quality_score: Number(fusion.confidence.toFixed(4)),
|
|
1782
|
+
timestamp: new Date().toISOString(),
|
|
1783
|
+
score: Number((Math.max(...ranked.map(item => item.score)) + 1).toFixed(4)),
|
|
1784
|
+
reason_tags: ["llm_fused_authoritative", `evidence:${fusion.evidence_ids.length}`],
|
|
1785
|
+
explain: {
|
|
1786
|
+
channel: "llm_fusion",
|
|
1787
|
+
fused_from: ranked.slice(0, maxCandidates).map(item => item.id),
|
|
1788
|
+
reason_tags: ["llm_fused_authoritative", `evidence:${fusion.evidence_ids.length}`],
|
|
1789
|
+
},
|
|
1790
|
+
fused_coverage_note: fusion.coverage_note || "",
|
|
1791
|
+
fused_facts: fusion.facts,
|
|
1792
|
+
fused_timeline: fusion.timeline || [],
|
|
1793
|
+
fused_entities: fusion.entities || [],
|
|
1794
|
+
fused_decisions: fusion.decisions || [],
|
|
1795
|
+
fused_fixes: fusion.fixes || [],
|
|
1796
|
+
fused_preferences: fusion.preferences || [],
|
|
1797
|
+
fused_risks: fusion.risks || [],
|
|
1798
|
+
fused_action_items: fusion.action_items || [],
|
|
1799
|
+
fused_conflicts: fusion.conflicts,
|
|
1800
|
+
fused_evidence_ids: fusion.evidence_ids,
|
|
1801
|
+
fused_need_fulltext_event_ids: fusion.need_fulltext_event_ids || [],
|
|
1802
|
+
fulltext_fetch_hints: fulltextFetchHints,
|
|
1803
|
+
};
|
|
1804
|
+
const authoritative = options.fusion?.authoritative !== false;
|
|
1805
|
+
if (authoritative) {
|
|
1806
|
+
markHit(Array.isArray(fusion.evidence_ids) ? fusion.evidence_ids : []);
|
|
1807
|
+
return { results: [fusedItem] };
|
|
1808
|
+
}
|
|
1809
|
+
const merged = [fusedItem, ...ranked];
|
|
1810
|
+
markHit([
|
|
1811
|
+
...(Array.isArray(fusion.evidence_ids) ? fusion.evidence_ids : []),
|
|
1812
|
+
...ranked.map(item => item.id),
|
|
1813
|
+
]);
|
|
1814
|
+
return { results: merged.slice(0, Math.max(1, args.topK)) };
|
|
1815
|
+
}
|
|
1816
|
+
}
|
|
1817
|
+
catch (error) {
|
|
1818
|
+
options.logger.warn(`LLM fusion failed, fallback to reranked results: ${error}`);
|
|
1819
|
+
}
|
|
1820
|
+
}
|
|
1821
|
+
const finalRanked = ranked.slice(0, Math.max(1, args.topK));
|
|
1822
|
+
markHit(finalRanked.map(item => item.id));
|
|
1823
|
+
return { results: finalRanked };
|
|
199
1824
|
}
|
|
200
1825
|
async function getHotContext(args) {
|
|
201
1826
|
const limit = Math.max(1, args.limit);
|
|
202
1827
|
const docs = loadAllDocuments();
|
|
203
1828
|
const coreRules = docs.find(doc => doc.source === "CORTEX_RULES.md");
|
|
204
|
-
const
|
|
205
|
-
|
|
1829
|
+
const ruleBudget = Math.max(1, Math.min(6, Math.floor(limit / 3)));
|
|
1830
|
+
const archiveDocs = docs
|
|
1831
|
+
.filter(doc => doc.source === "sessions_archive")
|
|
206
1832
|
.sort((a, b) => (b.timestamp ?? 0) - (a.timestamp ?? 0))
|
|
207
1833
|
.slice(0, limit);
|
|
1834
|
+
const issueFixPairs = docs
|
|
1835
|
+
.filter(doc => doc.source === "sessions_archive" && (doc.eventType === "issue" || doc.eventType === "fix"))
|
|
1836
|
+
.sort((a, b) => (b.timestamp ?? 0) - (a.timestamp ?? 0))
|
|
1837
|
+
.slice(0, 2);
|
|
208
1838
|
const result = [];
|
|
209
1839
|
if (coreRules) {
|
|
210
|
-
|
|
1840
|
+
const selectedRules = extractPrioritizedRuleLines(coreRules.text, ruleBudget);
|
|
1841
|
+
if (selectedRules.length > 0) {
|
|
1842
|
+
result.push({
|
|
1843
|
+
id: `${coreRules.id}.hot`,
|
|
1844
|
+
text: `# Hot Rules\n${selectedRules.map((line, index) => `${index + 1}. ${line}`).join("\n")}`,
|
|
1845
|
+
source: coreRules.source,
|
|
1846
|
+
});
|
|
1847
|
+
}
|
|
211
1848
|
}
|
|
212
|
-
for (const doc of
|
|
1849
|
+
for (const doc of [...issueFixPairs, ...archiveDocs]) {
|
|
213
1850
|
result.push({ id: doc.id, text: doc.text, source: doc.source });
|
|
214
1851
|
}
|
|
215
1852
|
return { context: result.slice(0, limit) };
|
|
@@ -223,6 +1860,25 @@ function createReadStore(options) {
|
|
|
223
1860
|
age_seconds: args.cachedAutoSearch.ageSeconds,
|
|
224
1861
|
};
|
|
225
1862
|
}
|
|
1863
|
+
if (!result.auto_search) {
|
|
1864
|
+
const docs = loadAllDocuments()
|
|
1865
|
+
.filter(doc => doc.source === "sessions_archive" && doc.sessionId === args.sessionId)
|
|
1866
|
+
.sort((a, b) => (b.timestamp ?? 0) - (a.timestamp ?? 0));
|
|
1867
|
+
const latest = docs[0];
|
|
1868
|
+
if (latest && latest.text.trim()) {
|
|
1869
|
+
const autoQuery = latest.text.slice(0, Math.max(20, readTuning.autoContext.queryMaxChars));
|
|
1870
|
+
const light = await searchMemory({
|
|
1871
|
+
query: autoQuery,
|
|
1872
|
+
topK: 3,
|
|
1873
|
+
mode: readTuning.autoContext.lightweightSearch ? "lightweight" : "default",
|
|
1874
|
+
});
|
|
1875
|
+
result.auto_search = {
|
|
1876
|
+
query: autoQuery,
|
|
1877
|
+
results: light.results,
|
|
1878
|
+
age_seconds: 0,
|
|
1879
|
+
};
|
|
1880
|
+
}
|
|
1881
|
+
}
|
|
226
1882
|
if (args.includeHot) {
|
|
227
1883
|
const hot = await getHotContext({ limit: 20 });
|
|
228
1884
|
result.hot_context = hot.context;
|