openclaw-cortex-memory 0.1.0-Alpha.3 → 0.1.0-Alpha.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +263 -204
- package/SKILL.md +77 -268
- package/dist/index.d.ts +92 -22
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1062 -1207
- package/dist/index.js.map +1 -1
- package/dist/openclaw.plugin.json +384 -15
- package/dist/src/dedup/three_stage_deduplicator.d.ts +25 -0
- package/dist/src/dedup/three_stage_deduplicator.d.ts.map +1 -0
- package/dist/src/dedup/three_stage_deduplicator.js +224 -0
- package/dist/src/dedup/three_stage_deduplicator.js.map +1 -0
- package/dist/src/engine/memory_engine.d.ts +2 -1
- package/dist/src/engine/memory_engine.d.ts.map +1 -1
- package/dist/src/engine/ts_engine.d.ts +126 -0
- package/dist/src/engine/ts_engine.d.ts.map +1 -1
- package/dist/src/engine/ts_engine.js +1145 -44
- package/dist/src/engine/ts_engine.js.map +1 -1
- package/dist/src/engine/types.d.ts +12 -0
- package/dist/src/engine/types.d.ts.map +1 -1
- package/dist/src/graph/ontology.d.ts +103 -0
- package/dist/src/graph/ontology.d.ts.map +1 -0
- package/dist/src/graph/ontology.js +564 -0
- package/dist/src/graph/ontology.js.map +1 -0
- package/dist/src/net/http_post.d.ts +17 -0
- package/dist/src/net/http_post.d.ts.map +1 -0
- package/dist/src/net/http_post.js +56 -0
- package/dist/src/net/http_post.js.map +1 -0
- package/dist/src/quality/llm_output_validator.d.ts +48 -0
- package/dist/src/quality/llm_output_validator.d.ts.map +1 -0
- package/dist/src/quality/llm_output_validator.js +404 -0
- package/dist/src/quality/llm_output_validator.js.map +1 -0
- package/dist/src/reflect/reflector.d.ts +7 -0
- package/dist/src/reflect/reflector.d.ts.map +1 -1
- package/dist/src/reflect/reflector.js +352 -8
- package/dist/src/reflect/reflector.js.map +1 -1
- package/dist/src/rules/rule_store.d.ts.map +1 -1
- package/dist/src/rules/rule_store.js +75 -16
- package/dist/src/rules/rule_store.js.map +1 -1
- package/dist/src/session/session_end.d.ts +33 -0
- package/dist/src/session/session_end.d.ts.map +1 -1
- package/dist/src/session/session_end.js +67 -64
- package/dist/src/session/session_end.js.map +1 -1
- package/dist/src/store/archive_store.d.ts +128 -0
- package/dist/src/store/archive_store.d.ts.map +1 -0
- package/dist/src/store/archive_store.js +475 -0
- package/dist/src/store/archive_store.js.map +1 -0
- package/dist/src/store/embedding_utils.d.ts +32 -0
- package/dist/src/store/embedding_utils.d.ts.map +1 -0
- package/dist/src/store/embedding_utils.js +173 -0
- package/dist/src/store/embedding_utils.js.map +1 -0
- package/dist/src/store/graph_memory_store.d.ts +44 -0
- package/dist/src/store/graph_memory_store.d.ts.map +1 -0
- package/dist/src/store/graph_memory_store.js +168 -0
- package/dist/src/store/graph_memory_store.js.map +1 -0
- package/dist/src/store/read_store.d.ts +86 -0
- package/dist/src/store/read_store.d.ts.map +1 -1
- package/dist/src/store/read_store.js +1661 -25
- package/dist/src/store/read_store.js.map +1 -1
- package/dist/src/store/vector_store.d.ts +44 -0
- package/dist/src/store/vector_store.d.ts.map +1 -0
- package/dist/src/store/vector_store.js +201 -0
- package/dist/src/store/vector_store.js.map +1 -0
- package/dist/src/store/write_store.d.ts +52 -0
- package/dist/src/store/write_store.d.ts.map +1 -1
- package/dist/src/store/write_store.js +239 -3
- package/dist/src/store/write_store.js.map +1 -1
- package/dist/src/sync/session_sync.d.ts +100 -2
- package/dist/src/sync/session_sync.d.ts.map +1 -1
- package/dist/src/sync/session_sync.js +725 -28
- package/dist/src/sync/session_sync.js.map +1 -1
- package/dist/src/utils/runtime_env.d.ts +4 -0
- package/dist/src/utils/runtime_env.d.ts.map +1 -0
- package/dist/src/utils/runtime_env.js +51 -0
- package/dist/src/utils/runtime_env.js.map +1 -0
- package/openclaw.plugin.json +384 -15
- package/package.json +53 -7
- package/schema/graph.schema.yaml +175 -0
- package/scripts/cli.js +19 -14
- package/scripts/repair-memory.js +321 -0
- package/scripts/uninstall.js +22 -5
- package/index.ts +0 -2142
- package/scripts/install.js +0 -27
|
@@ -36,6 +36,179 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
36
36
|
exports.createReadStore = createReadStore;
|
|
37
37
|
const fs = __importStar(require("fs"));
|
|
38
38
|
const path = __importStar(require("path"));
|
|
39
|
+
const module_1 = require("module");
|
|
40
|
+
const http_post_1 = require("../net/http_post");
|
|
41
|
+
function buildEntityGraphSummaryDocs(graphDocs) {
|
|
42
|
+
const entityEdges = new Map();
|
|
43
|
+
const entityLatestTs = new Map();
|
|
44
|
+
const entitySession = new Map();
|
|
45
|
+
for (const doc of graphDocs) {
|
|
46
|
+
const ts = typeof doc.timestamp === "number" ? doc.timestamp : 0;
|
|
47
|
+
const sessionId = typeof doc.sessionId === "string" ? doc.sessionId : "";
|
|
48
|
+
const relations = Array.isArray(doc.relations) ? doc.relations : [];
|
|
49
|
+
for (const relation of relations) {
|
|
50
|
+
const source = (relation.source || "").trim();
|
|
51
|
+
const target = (relation.target || "").trim();
|
|
52
|
+
const type = (relation.type || "").trim();
|
|
53
|
+
if (!source || !target || !type)
|
|
54
|
+
continue;
|
|
55
|
+
if (!entityEdges.has(source))
|
|
56
|
+
entityEdges.set(source, []);
|
|
57
|
+
if (!entityEdges.has(target))
|
|
58
|
+
entityEdges.set(target, []);
|
|
59
|
+
entityEdges.get(source)?.push({ source, target, type });
|
|
60
|
+
entityEdges.get(target)?.push({ source, target, type });
|
|
61
|
+
entityLatestTs.set(source, Math.max(entityLatestTs.get(source) || 0, ts));
|
|
62
|
+
entityLatestTs.set(target, Math.max(entityLatestTs.get(target) || 0, ts));
|
|
63
|
+
if (sessionId) {
|
|
64
|
+
if (!entitySession.has(source))
|
|
65
|
+
entitySession.set(source, sessionId);
|
|
66
|
+
if (!entitySession.has(target))
|
|
67
|
+
entitySession.set(target, sessionId);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
const output = [];
|
|
72
|
+
for (const [entity, edges] of entityEdges.entries()) {
|
|
73
|
+
if (!edges.length)
|
|
74
|
+
continue;
|
|
75
|
+
const outgoing = edges.filter(edge => edge.source === entity);
|
|
76
|
+
const incoming = edges.filter(edge => edge.target === entity);
|
|
77
|
+
const typeCounter = new Map();
|
|
78
|
+
for (const edge of edges) {
|
|
79
|
+
typeCounter.set(edge.type, (typeCounter.get(edge.type) || 0) + 1);
|
|
80
|
+
}
|
|
81
|
+
const typeSummary = [...typeCounter.entries()]
|
|
82
|
+
.sort((a, b) => b[1] - a[1])
|
|
83
|
+
.map(([type, count]) => `${type}:${count}`)
|
|
84
|
+
.join(", ");
|
|
85
|
+
const sortedOutgoing = [...outgoing].sort((a, b) => a.type.localeCompare(b.type));
|
|
86
|
+
const sortedIncoming = [...incoming].sort((a, b) => a.type.localeCompare(b.type));
|
|
87
|
+
const cappedOutgoing = sortedOutgoing.slice(0, 20);
|
|
88
|
+
const cappedIncoming = sortedIncoming.slice(0, 20);
|
|
89
|
+
const relationFacts = edges
|
|
90
|
+
.slice(0, 40)
|
|
91
|
+
.map(edge => `${edge.source} ${edge.type} ${edge.target}`)
|
|
92
|
+
.join(" | ");
|
|
93
|
+
const outgoingBlock = cappedOutgoing.length > 0
|
|
94
|
+
? cappedOutgoing.map((edge, index) => `${index + 1}. ${edge.source} -[${edge.type}]-> ${edge.target}`).join("\n")
|
|
95
|
+
: "none";
|
|
96
|
+
const incomingBlock = cappedIncoming.length > 0
|
|
97
|
+
? cappedIncoming.map((edge, index) => `${index + 1}. ${edge.source} -[${edge.type}]-> ${edge.target}`).join("\n")
|
|
98
|
+
: "none";
|
|
99
|
+
const summaryText = [
|
|
100
|
+
`# Graph Entity Summary`,
|
|
101
|
+
`entity: ${entity}`,
|
|
102
|
+
``,
|
|
103
|
+
`## Stats`,
|
|
104
|
+
`relation_total: ${edges.length}`,
|
|
105
|
+
`outgoing_total: ${outgoing.length}`,
|
|
106
|
+
`incoming_total: ${incoming.length}`,
|
|
107
|
+
typeSummary ? `relation_type_distribution: ${typeSummary}` : "relation_type_distribution: none",
|
|
108
|
+
``,
|
|
109
|
+
`## Outgoing Relations`,
|
|
110
|
+
outgoingBlock,
|
|
111
|
+
outgoing.length > cappedOutgoing.length ? `...truncated_outgoing: ${outgoing.length - cappedOutgoing.length}` : "",
|
|
112
|
+
``,
|
|
113
|
+
`## Incoming Relations`,
|
|
114
|
+
incomingBlock,
|
|
115
|
+
incoming.length > cappedIncoming.length ? `...truncated_incoming: ${incoming.length - cappedIncoming.length}` : "",
|
|
116
|
+
``,
|
|
117
|
+
`## Relation Facts`,
|
|
118
|
+
relationFacts || "none",
|
|
119
|
+
].filter(Boolean).join("\n");
|
|
120
|
+
output.push({
|
|
121
|
+
id: `gph_entity_${entity.toLowerCase().replace(/[^a-z0-9\u4e00-\u9fa5]+/gi, "_")}`,
|
|
122
|
+
text: summaryText,
|
|
123
|
+
source: "sessions_graph_entity",
|
|
124
|
+
timestamp: (entityLatestTs.get(entity) || 0) > 0 ? entityLatestTs.get(entity) : undefined,
|
|
125
|
+
layer: "archive",
|
|
126
|
+
sourceMemoryId: entity,
|
|
127
|
+
sessionId: entitySession.get(entity) || undefined,
|
|
128
|
+
entities: [entity],
|
|
129
|
+
relations: edges,
|
|
130
|
+
eventType: "graph_summary",
|
|
131
|
+
qualityScore: 1,
|
|
132
|
+
});
|
|
133
|
+
}
|
|
134
|
+
return output;
|
|
135
|
+
}
|
|
136
|
+
const DEFAULT_READ_TUNING = {
|
|
137
|
+
scoring: {
|
|
138
|
+
lexicalWeight: 0.2,
|
|
139
|
+
bm25Scale: 2,
|
|
140
|
+
semanticWeight: 0.3,
|
|
141
|
+
recencyWeight: 0.1,
|
|
142
|
+
qualityWeight: 0.15,
|
|
143
|
+
typeMatchWeight: 0.15,
|
|
144
|
+
graphMatchWeight: 0.1,
|
|
145
|
+
},
|
|
146
|
+
rrf: {
|
|
147
|
+
k: 60,
|
|
148
|
+
weight: 1.5,
|
|
149
|
+
},
|
|
150
|
+
recency: {
|
|
151
|
+
buckets: [
|
|
152
|
+
{ maxAgeHours: 12, score: 1, bonus: 0.6 },
|
|
153
|
+
{ maxAgeHours: 24, score: 0.8, bonus: 0.6 },
|
|
154
|
+
{ maxAgeHours: 72, score: 0.6, bonus: 0.3 },
|
|
155
|
+
{ maxAgeHours: 168, score: 0.4, bonus: 0.3 },
|
|
156
|
+
{ maxAgeHours: 720, score: 0.2, bonus: 0 },
|
|
157
|
+
{ maxAgeHours: Number.POSITIVE_INFINITY, score: 0.05, bonus: 0 },
|
|
158
|
+
],
|
|
159
|
+
},
|
|
160
|
+
autoContext: {
|
|
161
|
+
queryMaxChars: 80,
|
|
162
|
+
lightweightSearch: true,
|
|
163
|
+
},
|
|
164
|
+
};
|
|
165
|
+
function resolveReadTuning(options) {
|
|
166
|
+
const configuredBuckets = Array.isArray(options?.recency?.buckets)
|
|
167
|
+
? options?.recency?.buckets
|
|
168
|
+
.filter(item => item &&
|
|
169
|
+
Number.isFinite(item.maxAgeHours) &&
|
|
170
|
+
item.maxAgeHours > 0 &&
|
|
171
|
+
Number.isFinite(item.score) &&
|
|
172
|
+
item.score >= 0 &&
|
|
173
|
+
Number.isFinite(item.bonus))
|
|
174
|
+
.map(item => ({
|
|
175
|
+
maxAgeHours: item.maxAgeHours,
|
|
176
|
+
score: Math.max(0, item.score),
|
|
177
|
+
bonus: Math.max(0, item.bonus),
|
|
178
|
+
}))
|
|
179
|
+
: [];
|
|
180
|
+
const sortedBuckets = configuredBuckets
|
|
181
|
+
.sort((a, b) => a.maxAgeHours - b.maxAgeHours);
|
|
182
|
+
const buckets = sortedBuckets.length > 0 ? sortedBuckets : DEFAULT_READ_TUNING.recency.buckets;
|
|
183
|
+
const numberOr = (value, fallback, min) => {
|
|
184
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value < min) {
|
|
185
|
+
return fallback;
|
|
186
|
+
}
|
|
187
|
+
return value;
|
|
188
|
+
};
|
|
189
|
+
return {
|
|
190
|
+
scoring: {
|
|
191
|
+
lexicalWeight: numberOr(options?.scoring?.lexicalWeight, DEFAULT_READ_TUNING.scoring.lexicalWeight, 0),
|
|
192
|
+
bm25Scale: numberOr(options?.scoring?.bm25Scale, DEFAULT_READ_TUNING.scoring.bm25Scale, 0),
|
|
193
|
+
semanticWeight: numberOr(options?.scoring?.semanticWeight, DEFAULT_READ_TUNING.scoring.semanticWeight, 0),
|
|
194
|
+
recencyWeight: numberOr(options?.scoring?.recencyWeight, DEFAULT_READ_TUNING.scoring.recencyWeight, 0),
|
|
195
|
+
qualityWeight: numberOr(options?.scoring?.qualityWeight, DEFAULT_READ_TUNING.scoring.qualityWeight, 0),
|
|
196
|
+
typeMatchWeight: numberOr(options?.scoring?.typeMatchWeight, DEFAULT_READ_TUNING.scoring.typeMatchWeight, 0),
|
|
197
|
+
graphMatchWeight: numberOr(options?.scoring?.graphMatchWeight, DEFAULT_READ_TUNING.scoring.graphMatchWeight, 0),
|
|
198
|
+
},
|
|
199
|
+
rrf: {
|
|
200
|
+
k: Math.floor(numberOr(options?.rrf?.k, DEFAULT_READ_TUNING.rrf.k, 1)),
|
|
201
|
+
weight: numberOr(options?.rrf?.weight, DEFAULT_READ_TUNING.rrf.weight, 0),
|
|
202
|
+
},
|
|
203
|
+
recency: {
|
|
204
|
+
buckets,
|
|
205
|
+
},
|
|
206
|
+
autoContext: {
|
|
207
|
+
queryMaxChars: Math.floor(numberOr(options?.autoContext?.queryMaxChars, DEFAULT_READ_TUNING.autoContext.queryMaxChars, 20)),
|
|
208
|
+
lightweightSearch: options?.autoContext?.lightweightSearch !== false,
|
|
209
|
+
},
|
|
210
|
+
};
|
|
211
|
+
}
|
|
39
212
|
function safeReadFile(filePath) {
|
|
40
213
|
try {
|
|
41
214
|
if (!fs.existsSync(filePath)) {
|
|
@@ -65,7 +238,64 @@ function scoreText(query, text) {
|
|
|
65
238
|
}
|
|
66
239
|
return score;
|
|
67
240
|
}
|
|
241
|
+
function tokenize(text) {
|
|
242
|
+
return text
|
|
243
|
+
.toLowerCase()
|
|
244
|
+
.split(/[^a-z0-9\u4e00-\u9fa5]+/i)
|
|
245
|
+
.map(token => token.trim())
|
|
246
|
+
.filter(Boolean);
|
|
247
|
+
}
|
|
248
|
+
function buildBm25Stats(docs, queryTerms, getTokens) {
|
|
249
|
+
const docFreq = new Map();
|
|
250
|
+
let totalLen = 0;
|
|
251
|
+
for (const doc of docs) {
|
|
252
|
+
const tokens = typeof getTokens === "function" ? getTokens(doc) : tokenize(doc.text);
|
|
253
|
+
totalLen += tokens.length;
|
|
254
|
+
if (queryTerms.length === 0) {
|
|
255
|
+
continue;
|
|
256
|
+
}
|
|
257
|
+
const termSet = new Set(tokens);
|
|
258
|
+
for (const term of queryTerms) {
|
|
259
|
+
if (termSet.has(term)) {
|
|
260
|
+
docFreq.set(term, (docFreq.get(term) || 0) + 1);
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
const avgDocLen = docs.length > 0 ? Math.max(1, totalLen / docs.length) : 1;
|
|
265
|
+
return { avgDocLen, docFreq };
|
|
266
|
+
}
|
|
267
|
+
function bm25Score(args) {
|
|
268
|
+
const tokens = Array.isArray(args.docTokens) ? args.docTokens : tokenize(args.docText);
|
|
269
|
+
if (tokens.length === 0 || args.queryTerms.length === 0 || args.docCount <= 0) {
|
|
270
|
+
return 0;
|
|
271
|
+
}
|
|
272
|
+
const termFreq = new Map();
|
|
273
|
+
for (const token of tokens) {
|
|
274
|
+
termFreq.set(token, (termFreq.get(token) || 0) + 1);
|
|
275
|
+
}
|
|
276
|
+
const k1 = 1.2;
|
|
277
|
+
const b = 0.75;
|
|
278
|
+
let score = 0;
|
|
279
|
+
for (const term of args.queryTerms) {
|
|
280
|
+
const tf = termFreq.get(term) || 0;
|
|
281
|
+
if (tf <= 0)
|
|
282
|
+
continue;
|
|
283
|
+
const df = args.docFreq.get(term) || 0;
|
|
284
|
+
const idf = Math.log(1 + ((args.docCount - df + 0.5) / (df + 0.5)));
|
|
285
|
+
const denominator = tf + k1 * (1 - b + b * (tokens.length / Math.max(1, args.avgDocLen)));
|
|
286
|
+
score += idf * (((k1 + 1) * tf) / Math.max(1e-6, denominator));
|
|
287
|
+
}
|
|
288
|
+
return score;
|
|
289
|
+
}
|
|
68
290
|
function normalizeRecordText(record) {
|
|
291
|
+
const summary = typeof record.summary === "string" ? record.summary.trim() : "";
|
|
292
|
+
const sourceText = typeof record.source_text === "string" ? record.source_text.trim() : "";
|
|
293
|
+
if (summary && sourceText) {
|
|
294
|
+
return [
|
|
295
|
+
`summary: ${summary}`,
|
|
296
|
+
`source_text: ${sourceText}`,
|
|
297
|
+
].join("\n");
|
|
298
|
+
}
|
|
69
299
|
const direct = [record.content, record.summary, record.text, record.message]
|
|
70
300
|
.find(v => typeof v === "string" && v.trim());
|
|
71
301
|
if (direct) {
|
|
@@ -108,7 +338,9 @@ function parseJsonlFile(filePath, sourceLabel, logger) {
|
|
|
108
338
|
}
|
|
109
339
|
try {
|
|
110
340
|
const parsed = JSON.parse(trimmed);
|
|
111
|
-
const
|
|
341
|
+
const summaryText = typeof parsed.summary === "string" ? parsed.summary.trim() : "";
|
|
342
|
+
const sourceText = typeof parsed.source_text === "string" ? parsed.source_text.trim() : "";
|
|
343
|
+
const text = summaryText || normalizeRecordText(parsed);
|
|
112
344
|
if (!text.trim()) {
|
|
113
345
|
continue;
|
|
114
346
|
}
|
|
@@ -117,8 +349,29 @@ function parseJsonlFile(filePath, sourceLabel, logger) {
|
|
|
117
349
|
docs.push({
|
|
118
350
|
id,
|
|
119
351
|
text,
|
|
352
|
+
summaryText: summaryText || text,
|
|
353
|
+
sourceText: sourceText || undefined,
|
|
354
|
+
sourceEventId: typeof parsed.source_event_id === "string" ? parsed.source_event_id : undefined,
|
|
355
|
+
sourceFile: typeof parsed.source_file === "string" ? parsed.source_file : undefined,
|
|
120
356
|
source: sourceLabel,
|
|
121
357
|
timestamp: Number.isFinite(timestampValue) ? timestampValue : undefined,
|
|
358
|
+
layer: parsed.layer === "active" || parsed.layer === "archive"
|
|
359
|
+
? parsed.layer
|
|
360
|
+
: (sourceLabel === "sessions_active" ? "active" : (sourceLabel === "sessions_archive" ? "archive" : undefined)),
|
|
361
|
+
sourceMemoryId: typeof parsed.source_memory_id === "string"
|
|
362
|
+
? parsed.source_memory_id
|
|
363
|
+
: id,
|
|
364
|
+
sourceMemoryCanonicalId: typeof parsed.source_memory_canonical_id === "string"
|
|
365
|
+
? parsed.source_memory_canonical_id
|
|
366
|
+
: (typeof parsed.canonical_id === "string" ? parsed.canonical_id : undefined),
|
|
367
|
+
embedding: Array.isArray(parsed.embedding) ? parsed.embedding.filter(item => Number.isFinite(item)) : undefined,
|
|
368
|
+
eventType: typeof parsed.event_type === "string" ? parsed.event_type.trim() : undefined,
|
|
369
|
+
qualityScore: typeof parsed.quality_score === "number" ? parsed.quality_score : undefined,
|
|
370
|
+
charCount: typeof parsed.char_count === "number" ? parsed.char_count : undefined,
|
|
371
|
+
tokenCount: typeof parsed.token_count === "number" ? parsed.token_count : undefined,
|
|
372
|
+
sessionId: typeof parsed.session_id === "string" ? parsed.session_id : undefined,
|
|
373
|
+
entities: [],
|
|
374
|
+
relations: [],
|
|
122
375
|
});
|
|
123
376
|
}
|
|
124
377
|
catch (error) {
|
|
@@ -147,69 +400,1433 @@ function parseMarkdownFile(filePath, sourceLabel) {
|
|
|
147
400
|
},
|
|
148
401
|
];
|
|
149
402
|
}
|
|
150
|
-
function
|
|
403
|
+
function extractPrioritizedRuleLines(text, maxRules) {
|
|
404
|
+
if (!text.trim() || maxRules <= 0) {
|
|
405
|
+
return [];
|
|
406
|
+
}
|
|
407
|
+
const lines = text
|
|
408
|
+
.split(/\r?\n/)
|
|
409
|
+
.map(line => line.trim())
|
|
410
|
+
.filter(Boolean)
|
|
411
|
+
.filter(line => !/^core rules and knowledge extracted/i.test(line))
|
|
412
|
+
.filter(line => !/^core rules\b/i.test(line));
|
|
413
|
+
if (lines.length === 0) {
|
|
414
|
+
return [];
|
|
415
|
+
}
|
|
416
|
+
const dedupedFromTail = [];
|
|
417
|
+
const seen = new Set();
|
|
418
|
+
for (let i = lines.length - 1; i >= 0; i -= 1) {
|
|
419
|
+
const line = lines[i];
|
|
420
|
+
const key = line.toLowerCase().replace(/\s+/g, " ").trim();
|
|
421
|
+
if (!key || seen.has(key)) {
|
|
422
|
+
continue;
|
|
423
|
+
}
|
|
424
|
+
seen.add(key);
|
|
425
|
+
dedupedFromTail.push(line);
|
|
426
|
+
}
|
|
427
|
+
const scored = dedupedFromTail.map((line, indexFromTail) => {
|
|
428
|
+
let score = 0;
|
|
429
|
+
if (/(must|should|ensure|avoid|prefer|always|never|fallback|verify|validate|retry|sanitize)/i.test(line)) {
|
|
430
|
+
score += 3;
|
|
431
|
+
}
|
|
432
|
+
if (/(fix|resolved|success|stable|deploy|release|incident|rollback|constraint|decision)/i.test(line)) {
|
|
433
|
+
score += 2;
|
|
434
|
+
}
|
|
435
|
+
if (/(确保|避免|优先|必须|建议|回退|重试|校验|稳定|发布|决策)/.test(line)) {
|
|
436
|
+
score += 2;
|
|
437
|
+
}
|
|
438
|
+
if (line.length >= 30 && line.length <= 220) {
|
|
439
|
+
score += 2;
|
|
440
|
+
}
|
|
441
|
+
else if (line.length > 220) {
|
|
442
|
+
score -= 1;
|
|
443
|
+
}
|
|
444
|
+
if (/[.!?]$/.test(line)) {
|
|
445
|
+
score += 1;
|
|
446
|
+
}
|
|
447
|
+
score += Math.max(0, 2 - indexFromTail * 0.08);
|
|
448
|
+
return { line, score, indexFromTail };
|
|
449
|
+
});
|
|
450
|
+
const selected = scored
|
|
451
|
+
.sort((a, b) => (b.score - a.score) || (a.indexFromTail - b.indexFromTail))
|
|
452
|
+
.slice(0, maxRules)
|
|
453
|
+
.sort((a, b) => a.indexFromTail - b.indexFromTail)
|
|
454
|
+
.map(item => item.line);
|
|
455
|
+
return selected;
|
|
456
|
+
}
|
|
457
|
+
function withRecencyBoost(score, timestamp, buckets) {
|
|
151
458
|
if (!timestamp) {
|
|
152
459
|
return score;
|
|
153
460
|
}
|
|
154
461
|
const ageHours = (Date.now() - timestamp) / (1000 * 60 * 60);
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
return score + 0.3;
|
|
462
|
+
for (const bucket of buckets) {
|
|
463
|
+
if (ageHours <= bucket.maxAgeHours) {
|
|
464
|
+
return score + bucket.bonus;
|
|
465
|
+
}
|
|
160
466
|
}
|
|
161
467
|
return score;
|
|
162
468
|
}
|
|
469
|
+
function recencyScore(timestamp, buckets) {
|
|
470
|
+
if (!timestamp) {
|
|
471
|
+
return 0;
|
|
472
|
+
}
|
|
473
|
+
const ageHours = (Date.now() - timestamp) / (1000 * 60 * 60);
|
|
474
|
+
for (const bucket of buckets) {
|
|
475
|
+
if (ageHours <= bucket.maxAgeHours) {
|
|
476
|
+
return bucket.score;
|
|
477
|
+
}
|
|
478
|
+
}
|
|
479
|
+
return 0;
|
|
480
|
+
}
|
|
481
|
+
function eventTypeHalfLifeDays(eventType, options) {
|
|
482
|
+
const fallback = typeof options?.defaultHalfLifeDays === "number" && options.defaultHalfLifeDays > 0
|
|
483
|
+
? options.defaultHalfLifeDays
|
|
484
|
+
: 90;
|
|
485
|
+
const type = (eventType || "").trim().toLowerCase();
|
|
486
|
+
if (!type)
|
|
487
|
+
return fallback;
|
|
488
|
+
const configured = options?.halfLifeByEventType || {};
|
|
489
|
+
if (typeof configured[type] === "number" && configured[type] > 0) {
|
|
490
|
+
return configured[type];
|
|
491
|
+
}
|
|
492
|
+
if (["issue", "fix", "action_item", "blocker"].includes(type))
|
|
493
|
+
return 30;
|
|
494
|
+
if (["plan", "milestone", "follow_up"].includes(type))
|
|
495
|
+
return 60;
|
|
496
|
+
if (["decision", "insight", "retrospective"].includes(type))
|
|
497
|
+
return 120;
|
|
498
|
+
if (["preference", "constraint", "requirement", "dependency", "assumption"].includes(type))
|
|
499
|
+
return 240;
|
|
500
|
+
return fallback;
|
|
501
|
+
}
|
|
502
|
+
function computeAntiDecayBoost(id, hitStats, options) {
|
|
503
|
+
const anti = options?.antiDecay;
|
|
504
|
+
if (anti?.enabled === false) {
|
|
505
|
+
return 1;
|
|
506
|
+
}
|
|
507
|
+
const item = hitStats.items[id];
|
|
508
|
+
if (!item) {
|
|
509
|
+
return 1;
|
|
510
|
+
}
|
|
511
|
+
const hitWeight = typeof anti?.hitWeight === "number" && anti.hitWeight > 0 ? anti.hitWeight : 0.08;
|
|
512
|
+
const maxBoost = typeof anti?.maxBoost === "number" && anti.maxBoost >= 1 ? anti.maxBoost : 1.6;
|
|
513
|
+
const recentWindowDays = typeof anti?.recentWindowDays === "number" && anti.recentWindowDays > 0 ? anti.recentWindowDays : 30;
|
|
514
|
+
const lastHitTs = Date.parse(item.lastHitAt || "");
|
|
515
|
+
const ageDays = Number.isFinite(lastHitTs) ? Math.max(0, (Date.now() - lastHitTs) / (1000 * 60 * 60 * 24)) : recentWindowDays * 2;
|
|
516
|
+
const freshness = ageDays <= recentWindowDays ? (1 - ageDays / recentWindowDays) : 0;
|
|
517
|
+
const countFactor = Math.log1p(Math.max(0, item.count));
|
|
518
|
+
const boost = 1 + countFactor * hitWeight * (0.5 + 0.5 * freshness);
|
|
519
|
+
return Math.min(maxBoost, Math.max(1, boost));
|
|
520
|
+
}
|
|
521
|
+
function computeDecayFactor(id, eventType, timestamp, options, hitStats) {
|
|
522
|
+
const enabled = options?.enabled !== false;
|
|
523
|
+
if (!enabled || !timestamp) {
|
|
524
|
+
return computeAntiDecayBoost(id, hitStats, options);
|
|
525
|
+
}
|
|
526
|
+
const ageDays = Math.max(0, (Date.now() - timestamp) / (1000 * 60 * 60 * 24));
|
|
527
|
+
const halfLife = eventTypeHalfLifeDays(eventType, options);
|
|
528
|
+
const base = Math.pow(2, -ageDays / Math.max(1, halfLife));
|
|
529
|
+
const floor = typeof options?.minFloor === "number"
|
|
530
|
+
? Math.max(0, Math.min(1, options.minFloor))
|
|
531
|
+
: 0.15;
|
|
532
|
+
const decay = Math.max(floor, base);
|
|
533
|
+
const boost = computeAntiDecayBoost(id, hitStats, options);
|
|
534
|
+
return Math.min(1, decay * boost);
|
|
535
|
+
}
|
|
536
|
+
function normalizeBaseUrl(value) {
|
|
537
|
+
if (!value)
|
|
538
|
+
return "";
|
|
539
|
+
return value.endsWith("/") ? value.slice(0, -1) : value;
|
|
540
|
+
}
|
|
541
|
+
const READ_FUSION_PROMPT_VERSION = "read-fusion.v1.2.0";
|
|
542
|
+
const READ_FUSION_REGRESSION_SAMPLES = [
|
|
543
|
+
"Example A: if archive and vector refer to the same source_memory_id, keep one main conclusion and keep the rest as supporting evidence.",
|
|
544
|
+
"Example B: if conclusions conflict, write conflicts and explain prioritization in canonical_answer (time, quality, explicitness).",
|
|
545
|
+
"Example C: if summary/excerpt is insufficient, return event ids in need_fulltext_event_ids for full-text lookup.",
|
|
546
|
+
];
|
|
547
|
+
function cosineSimilarity(left, right) {
|
|
548
|
+
if (left.length === 0 || right.length === 0) {
|
|
549
|
+
return 0;
|
|
550
|
+
}
|
|
551
|
+
const size = Math.min(left.length, right.length);
|
|
552
|
+
let dot = 0;
|
|
553
|
+
let leftNorm = 0;
|
|
554
|
+
let rightNorm = 0;
|
|
555
|
+
for (let i = 0; i < size; i += 1) {
|
|
556
|
+
const a = left[i];
|
|
557
|
+
const b = right[i];
|
|
558
|
+
dot += a * b;
|
|
559
|
+
leftNorm += a * a;
|
|
560
|
+
rightNorm += b * b;
|
|
561
|
+
}
|
|
562
|
+
if (leftNorm === 0 || rightNorm === 0) {
|
|
563
|
+
return 0;
|
|
564
|
+
}
|
|
565
|
+
return dot / (Math.sqrt(leftNorm) * Math.sqrt(rightNorm));
|
|
566
|
+
}
|
|
567
|
+
async function requestEmbedding(args) {
|
|
568
|
+
const endpoint = args.baseUrl.endsWith("/embeddings") ? args.baseUrl : `${args.baseUrl}/embeddings`;
|
|
569
|
+
const body = {
|
|
570
|
+
input: args.text,
|
|
571
|
+
model: args.model,
|
|
572
|
+
};
|
|
573
|
+
if (typeof args.dimensions === "number" && Number.isFinite(args.dimensions) && args.dimensions > 0) {
|
|
574
|
+
body.dimensions = args.dimensions;
|
|
575
|
+
}
|
|
576
|
+
let lastError = null;
|
|
577
|
+
for (let attempt = 0; attempt < 3; attempt += 1) {
|
|
578
|
+
const response = await (0, http_post_1.postJsonWithTimeout)({
|
|
579
|
+
endpoint,
|
|
580
|
+
apiKey: args.apiKey,
|
|
581
|
+
body,
|
|
582
|
+
timeoutMs: 10000,
|
|
583
|
+
});
|
|
584
|
+
if (!response.ok) {
|
|
585
|
+
lastError = new Error(response.status > 0 ? `embedding_http_${response.status}` : (response.error || "embedding_network_error"));
|
|
586
|
+
continue;
|
|
587
|
+
}
|
|
588
|
+
try {
|
|
589
|
+
const json = (response.json || {});
|
|
590
|
+
const embedding = json?.data?.[0]?.embedding;
|
|
591
|
+
if (Array.isArray(embedding) && embedding.length > 0) {
|
|
592
|
+
return embedding.filter(item => Number.isFinite(item));
|
|
593
|
+
}
|
|
594
|
+
lastError = new Error("embedding_empty");
|
|
595
|
+
}
|
|
596
|
+
catch (error) {
|
|
597
|
+
lastError = error;
|
|
598
|
+
}
|
|
599
|
+
}
|
|
600
|
+
if (lastError) {
|
|
601
|
+
throw lastError;
|
|
602
|
+
}
|
|
603
|
+
return null;
|
|
604
|
+
}
|
|
605
|
+
async function requestRerank(args) {
|
|
606
|
+
const endpoint = args.baseUrl.endsWith("/rerank") ? args.baseUrl : `${args.baseUrl}/rerank`;
|
|
607
|
+
const documents = args.candidates.map(item => item.text);
|
|
608
|
+
const body = {
|
|
609
|
+
model: args.model,
|
|
610
|
+
query: args.query,
|
|
611
|
+
documents,
|
|
612
|
+
top_n: args.candidates.length,
|
|
613
|
+
};
|
|
614
|
+
let lastError = null;
|
|
615
|
+
for (let attempt = 0; attempt < 2; attempt += 1) {
|
|
616
|
+
const response = await (0, http_post_1.postJsonWithTimeout)({
|
|
617
|
+
endpoint,
|
|
618
|
+
apiKey: args.apiKey,
|
|
619
|
+
body,
|
|
620
|
+
timeoutMs: 12000,
|
|
621
|
+
});
|
|
622
|
+
if (!response.ok) {
|
|
623
|
+
lastError = new Error(response.status > 0 ? `rerank_http_${response.status}` : (response.error || "rerank_network_error"));
|
|
624
|
+
continue;
|
|
625
|
+
}
|
|
626
|
+
try {
|
|
627
|
+
const json = (response.json || {});
|
|
628
|
+
const list = Array.isArray(json.results) ? json.results : (Array.isArray(json.data) ? json.data : []);
|
|
629
|
+
if (!Array.isArray(list) || list.length === 0) {
|
|
630
|
+
lastError = new Error("rerank_empty");
|
|
631
|
+
continue;
|
|
632
|
+
}
|
|
633
|
+
const mapped = list
|
|
634
|
+
.map((item, rank) => {
|
|
635
|
+
const index = typeof item.index === "number" ? item.index : rank;
|
|
636
|
+
const hit = args.candidates[index];
|
|
637
|
+
if (!hit)
|
|
638
|
+
return null;
|
|
639
|
+
const score = typeof item.relevance_score === "number" ? item.relevance_score : (typeof item.score === "number" ? item.score : hit.score);
|
|
640
|
+
return { ...hit, score };
|
|
641
|
+
})
|
|
642
|
+
.filter((item) => Boolean(item));
|
|
643
|
+
if (mapped.length > 0) {
|
|
644
|
+
return mapped;
|
|
645
|
+
}
|
|
646
|
+
lastError = new Error("rerank_map_empty");
|
|
647
|
+
}
|
|
648
|
+
catch (error) {
|
|
649
|
+
lastError = error;
|
|
650
|
+
}
|
|
651
|
+
}
|
|
652
|
+
throw lastError instanceof Error ? lastError : new Error(String(lastError || "rerank_failed"));
|
|
653
|
+
}
|
|
654
|
+
function classifyIntent(query) {
|
|
655
|
+
const text = query.toLowerCase();
|
|
656
|
+
const relationHints = /(关系|依赖|关联|上下游|图谱|拓扑|graph|relation|entity|dependency)/i;
|
|
657
|
+
if (relationHints.test(text))
|
|
658
|
+
return "RELATION_DISCOVERY";
|
|
659
|
+
const troubleHints = /(报错|错误|异常|失败|超时|无法|崩溃|故障|修复|bug|error|failed|timeout|fix)/i;
|
|
660
|
+
if (troubleHints.test(text))
|
|
661
|
+
return "TROUBLESHOOTING";
|
|
662
|
+
const preferenceHints = /(偏好|习惯|口味|喜欢|不喜欢|偏向|preference)/i;
|
|
663
|
+
if (preferenceHints.test(text))
|
|
664
|
+
return "PREFERENCE_PROFILE";
|
|
665
|
+
const timelineHints = /(最近|上次|之前|时间线|历史|timeline|history)/i;
|
|
666
|
+
if (timelineHints.test(text))
|
|
667
|
+
return "TIMELINE_REVIEW";
|
|
668
|
+
const decisionHints = /(方案|决策|选择|建议|取舍|权衡|tradeoff|plan)/i;
|
|
669
|
+
if (decisionHints.test(text))
|
|
670
|
+
return "DECISION_SUPPORT";
|
|
671
|
+
return "FACT_LOOKUP";
|
|
672
|
+
}
|
|
673
|
+
function preferredEventTypes(intent) {
|
|
674
|
+
if (intent === "TROUBLESHOOTING")
|
|
675
|
+
return ["issue", "fix", "risk", "blocker", "dependency", "retrospective"];
|
|
676
|
+
if (intent === "PREFERENCE_PROFILE")
|
|
677
|
+
return ["preference", "decision", "constraint", "requirement"];
|
|
678
|
+
if (intent === "DECISION_SUPPORT")
|
|
679
|
+
return ["decision", "plan", "insight", "assumption", "constraint", "requirement"];
|
|
680
|
+
if (intent === "TIMELINE_REVIEW")
|
|
681
|
+
return ["action_item", "follow_up", "milestone", "plan", "decision", "issue", "fix"];
|
|
682
|
+
return [];
|
|
683
|
+
}
|
|
684
|
+
function sourceWeight(source, intent) {
|
|
685
|
+
if (source === "rules") {
|
|
686
|
+
return intent === "DECISION_SUPPORT" || intent === "TROUBLESHOOTING" ? 1.15 : 0.9;
|
|
687
|
+
}
|
|
688
|
+
if (source === "graph") {
|
|
689
|
+
return intent === "RELATION_DISCOVERY" ? 1.25 : 0.85;
|
|
690
|
+
}
|
|
691
|
+
if (source === "vector") {
|
|
692
|
+
return 1.05;
|
|
693
|
+
}
|
|
694
|
+
return 1;
|
|
695
|
+
}
|
|
696
|
+
function mergeKeyFromDoc(doc) {
|
|
697
|
+
const canonical = typeof doc.sourceMemoryCanonicalId === "string" ? doc.sourceMemoryCanonicalId.trim() : "";
|
|
698
|
+
if (canonical) {
|
|
699
|
+
return `canonical:${canonical}`;
|
|
700
|
+
}
|
|
701
|
+
const sourceMemoryId = typeof doc.sourceMemoryId === "string" ? doc.sourceMemoryId.trim() : "";
|
|
702
|
+
if (sourceMemoryId) {
|
|
703
|
+
return `source:${sourceMemoryId}`;
|
|
704
|
+
}
|
|
705
|
+
return `id:${doc.id}`;
|
|
706
|
+
}
|
|
707
|
+
function customChannelWeight(source, options) {
|
|
708
|
+
const weights = options?.channelWeights;
|
|
709
|
+
if (!weights)
|
|
710
|
+
return 1;
|
|
711
|
+
const value = weights[source];
|
|
712
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) {
|
|
713
|
+
return 1;
|
|
714
|
+
}
|
|
715
|
+
return value;
|
|
716
|
+
}
|
|
717
|
+
function lengthNormalizeFactor(doc, options) {
|
|
718
|
+
const lengthNorm = options?.lengthNorm;
|
|
719
|
+
if (lengthNorm?.enabled === false) {
|
|
720
|
+
return 1;
|
|
721
|
+
}
|
|
722
|
+
const pivotChars = typeof lengthNorm?.pivotChars === "number" && lengthNorm.pivotChars > 0
|
|
723
|
+
? lengthNorm.pivotChars
|
|
724
|
+
: 1200;
|
|
725
|
+
const strength = typeof lengthNorm?.strength === "number" && lengthNorm.strength > 0
|
|
726
|
+
? lengthNorm.strength
|
|
727
|
+
: 0.75;
|
|
728
|
+
const minFactor = typeof lengthNorm?.minFactor === "number" && lengthNorm.minFactor > 0 && lengthNorm.minFactor <= 1
|
|
729
|
+
? lengthNorm.minFactor
|
|
730
|
+
: 0.45;
|
|
731
|
+
const charCount = typeof doc.charCount === "number" && Number.isFinite(doc.charCount)
|
|
732
|
+
? doc.charCount
|
|
733
|
+
: doc.text.length;
|
|
734
|
+
if (charCount <= pivotChars) {
|
|
735
|
+
return 1;
|
|
736
|
+
}
|
|
737
|
+
const over = (charCount - pivotChars) / pivotChars;
|
|
738
|
+
const factor = 1 / (1 + over * strength);
|
|
739
|
+
return Math.max(minFactor, Math.min(1, factor));
|
|
740
|
+
}
|
|
741
|
+
function channelQuota(source, topK, options) {
|
|
742
|
+
const configured = options?.channelTopK?.[source];
|
|
743
|
+
if (typeof configured === "number" && Number.isFinite(configured) && configured >= 1) {
|
|
744
|
+
return Math.floor(configured);
|
|
745
|
+
}
|
|
746
|
+
if (source === "rules")
|
|
747
|
+
return Math.max(6, topK * 2);
|
|
748
|
+
if (source === "graph")
|
|
749
|
+
return Math.max(8, topK * 3);
|
|
750
|
+
return Math.max(12, topK * 4);
|
|
751
|
+
}
|
|
752
|
+
function parseJsonStringArray(value) {
|
|
753
|
+
if (typeof value !== "string" || !value.trim()) {
|
|
754
|
+
return [];
|
|
755
|
+
}
|
|
756
|
+
try {
|
|
757
|
+
const parsed = JSON.parse(value);
|
|
758
|
+
if (!Array.isArray(parsed)) {
|
|
759
|
+
return [];
|
|
760
|
+
}
|
|
761
|
+
return parsed
|
|
762
|
+
.map(item => (typeof item === "string" ? item.trim() : ""))
|
|
763
|
+
.filter(Boolean);
|
|
764
|
+
}
|
|
765
|
+
catch {
|
|
766
|
+
return [];
|
|
767
|
+
}
|
|
768
|
+
}
|
|
769
|
+
function parseJsonRelations(value) {
|
|
770
|
+
if (typeof value !== "string" || !value.trim()) {
|
|
771
|
+
return [];
|
|
772
|
+
}
|
|
773
|
+
try {
|
|
774
|
+
const parsed = JSON.parse(value);
|
|
775
|
+
if (!Array.isArray(parsed)) {
|
|
776
|
+
return [];
|
|
777
|
+
}
|
|
778
|
+
return parsed
|
|
779
|
+
.map(item => {
|
|
780
|
+
if (typeof item !== "object" || item === null)
|
|
781
|
+
return null;
|
|
782
|
+
const relation = item;
|
|
783
|
+
const source = typeof relation.source === "string" ? relation.source.trim() : "";
|
|
784
|
+
const target = typeof relation.target === "string" ? relation.target.trim() : "";
|
|
785
|
+
const type = typeof relation.type === "string" && relation.type.trim() ? relation.type.trim() : "related_to";
|
|
786
|
+
if (!source || !target)
|
|
787
|
+
return null;
|
|
788
|
+
return { source, target, type };
|
|
789
|
+
})
|
|
790
|
+
.filter((item) => Boolean(item));
|
|
791
|
+
}
|
|
792
|
+
catch {
|
|
793
|
+
return [];
|
|
794
|
+
}
|
|
795
|
+
}
|
|
796
|
+
async function searchLanceDb(args) {
|
|
797
|
+
try {
|
|
798
|
+
const require = (0, module_1.createRequire)(__filename);
|
|
799
|
+
const lancedbDir = path.join(args.memoryRoot, "vector", "lancedb");
|
|
800
|
+
if (!fs.existsSync(lancedbDir)) {
|
|
801
|
+
return [];
|
|
802
|
+
}
|
|
803
|
+
const moduleValue = require("@lancedb/lancedb");
|
|
804
|
+
const connect = moduleValue.connect;
|
|
805
|
+
if (typeof connect !== "function") {
|
|
806
|
+
return [];
|
|
807
|
+
}
|
|
808
|
+
const db = await connect(lancedbDir);
|
|
809
|
+
if (!db || typeof db.openTable !== "function") {
|
|
810
|
+
return [];
|
|
811
|
+
}
|
|
812
|
+
const table = await db.openTable("events");
|
|
813
|
+
if (!table || typeof table.search !== "function") {
|
|
814
|
+
return [];
|
|
815
|
+
}
|
|
816
|
+
const searchObj = table.search(args.queryEmbedding);
|
|
817
|
+
if (!searchObj || typeof searchObj.limit !== "function") {
|
|
818
|
+
return [];
|
|
819
|
+
}
|
|
820
|
+
const limited = searchObj.limit(args.limit);
|
|
821
|
+
if (!limited || typeof limited.toArray !== "function") {
|
|
822
|
+
return [];
|
|
823
|
+
}
|
|
824
|
+
const rows = await limited.toArray();
|
|
825
|
+
const docs = [];
|
|
826
|
+
for (const row of rows) {
|
|
827
|
+
if (typeof row !== "object" || row === null)
|
|
828
|
+
continue;
|
|
829
|
+
const record = row;
|
|
830
|
+
const id = typeof record.id === "string" ? record.id : "";
|
|
831
|
+
const summary = typeof record.summary === "string" ? record.summary : "";
|
|
832
|
+
if (!id || !summary)
|
|
833
|
+
continue;
|
|
834
|
+
const ts = typeof record.timestamp === "string" ? Date.parse(record.timestamp) : NaN;
|
|
835
|
+
const entities = parseJsonStringArray(record.entities_json);
|
|
836
|
+
const relations = parseJsonRelations(record.relations_json);
|
|
837
|
+
docs.push({
|
|
838
|
+
id,
|
|
839
|
+
text: summary,
|
|
840
|
+
source: "vector_lancedb",
|
|
841
|
+
timestamp: Number.isFinite(ts) ? ts : undefined,
|
|
842
|
+
layer: record.layer === "active" || record.layer === "archive" ? record.layer : undefined,
|
|
843
|
+
sourceMemoryId: typeof record.source_memory_id === "string" ? record.source_memory_id : undefined,
|
|
844
|
+
sourceMemoryCanonicalId: typeof record.source_memory_canonical_id === "string" ? record.source_memory_canonical_id : undefined,
|
|
845
|
+
sourceEventId: typeof record.source_event_id === "string" ? record.source_event_id : undefined,
|
|
846
|
+
sourceField: record.source_field === "summary" || record.source_field === "evidence"
|
|
847
|
+
? record.source_field
|
|
848
|
+
: undefined,
|
|
849
|
+
embedding: Array.isArray(record.vector) ? record.vector.filter(item => Number.isFinite(item)) : undefined,
|
|
850
|
+
eventType: typeof record.event_type === "string" ? record.event_type : undefined,
|
|
851
|
+
qualityScore: typeof record.quality_score === "number" ? record.quality_score : undefined,
|
|
852
|
+
charCount: typeof record.char_count === "number" ? record.char_count : undefined,
|
|
853
|
+
tokenCount: typeof record.token_count === "number" ? record.token_count : undefined,
|
|
854
|
+
sessionId: typeof record.session_id === "string" ? record.session_id : undefined,
|
|
855
|
+
entities,
|
|
856
|
+
relations: Array.isArray(relations) ? relations : [],
|
|
857
|
+
});
|
|
858
|
+
}
|
|
859
|
+
return docs;
|
|
860
|
+
}
|
|
861
|
+
catch (error) {
|
|
862
|
+
args.logger.debug(`LanceDB search fallback: ${error}`);
|
|
863
|
+
return [];
|
|
864
|
+
}
|
|
865
|
+
}
|
|
866
|
+
function parseVectorFallback(filePath, logger) {
|
|
867
|
+
const content = safeReadFile(filePath);
|
|
868
|
+
if (!content) {
|
|
869
|
+
return [];
|
|
870
|
+
}
|
|
871
|
+
const docs = [];
|
|
872
|
+
for (const line of content.split(/\r?\n/)) {
|
|
873
|
+
const trimmed = line.trim();
|
|
874
|
+
if (!trimmed)
|
|
875
|
+
continue;
|
|
876
|
+
try {
|
|
877
|
+
const parsed = JSON.parse(trimmed);
|
|
878
|
+
const id = typeof parsed.id === "string" ? parsed.id : "";
|
|
879
|
+
const summary = typeof parsed.summary === "string" ? parsed.summary.trim() : "";
|
|
880
|
+
if (!id || !summary)
|
|
881
|
+
continue;
|
|
882
|
+
const ts = typeof parsed.timestamp === "string" ? Date.parse(parsed.timestamp) : NaN;
|
|
883
|
+
const entities = Array.isArray(parsed.entities)
|
|
884
|
+
? parsed.entities.map(item => (typeof item === "string" ? item.trim() : "")).filter(Boolean)
|
|
885
|
+
: [];
|
|
886
|
+
const relations = Array.isArray(parsed.relations)
|
|
887
|
+
? parsed.relations
|
|
888
|
+
.map(item => {
|
|
889
|
+
if (typeof item !== "object" || item === null)
|
|
890
|
+
return null;
|
|
891
|
+
const relation = item;
|
|
892
|
+
const source = typeof relation.source === "string" ? relation.source.trim() : "";
|
|
893
|
+
const target = typeof relation.target === "string" ? relation.target.trim() : "";
|
|
894
|
+
const type = typeof relation.type === "string" ? relation.type.trim() : "related_to";
|
|
895
|
+
if (!source || !target)
|
|
896
|
+
return null;
|
|
897
|
+
return { source, target, type };
|
|
898
|
+
})
|
|
899
|
+
.filter((item) => Boolean(item))
|
|
900
|
+
: [];
|
|
901
|
+
docs.push({
|
|
902
|
+
id,
|
|
903
|
+
text: summary,
|
|
904
|
+
source: "vector_jsonl",
|
|
905
|
+
timestamp: Number.isFinite(ts) ? ts : undefined,
|
|
906
|
+
layer: parsed.layer === "active" || parsed.layer === "archive" ? parsed.layer : undefined,
|
|
907
|
+
sourceMemoryId: typeof parsed.source_memory_id === "string" ? parsed.source_memory_id : undefined,
|
|
908
|
+
sourceMemoryCanonicalId: typeof parsed.source_memory_canonical_id === "string" ? parsed.source_memory_canonical_id : undefined,
|
|
909
|
+
sourceEventId: typeof parsed.source_event_id === "string" ? parsed.source_event_id : undefined,
|
|
910
|
+
sourceField: parsed.source_field === "summary" || parsed.source_field === "evidence"
|
|
911
|
+
? parsed.source_field
|
|
912
|
+
: undefined,
|
|
913
|
+
embedding: Array.isArray(parsed.embedding) ? parsed.embedding.filter(item => Number.isFinite(item)) : undefined,
|
|
914
|
+
eventType: typeof parsed.event_type === "string" ? parsed.event_type.trim() : undefined,
|
|
915
|
+
qualityScore: typeof parsed.quality_score === "number" ? parsed.quality_score : undefined,
|
|
916
|
+
charCount: typeof parsed.char_count === "number" ? parsed.char_count : undefined,
|
|
917
|
+
tokenCount: typeof parsed.token_count === "number" ? parsed.token_count : undefined,
|
|
918
|
+
sessionId: typeof parsed.session_id === "string" ? parsed.session_id : undefined,
|
|
919
|
+
entities,
|
|
920
|
+
relations,
|
|
921
|
+
});
|
|
922
|
+
}
|
|
923
|
+
catch (error) {
|
|
924
|
+
logger.debug(`Skip invalid vector jsonl line: ${error}`);
|
|
925
|
+
}
|
|
926
|
+
}
|
|
927
|
+
return docs;
|
|
928
|
+
}
|
|
929
|
+
async function requestFusion(args) {
|
|
930
|
+
const candidateIdSet = new Set(args.candidates.map(item => item.id));
|
|
931
|
+
const endpoint = args.llm.baseUrl.endsWith("/chat/completions")
|
|
932
|
+
? args.llm.baseUrl
|
|
933
|
+
: `${args.llm.baseUrl}/chat/completions`;
|
|
934
|
+
const evidenceText = args.candidates
|
|
935
|
+
.map((item, index) => {
|
|
936
|
+
const excerpt = (item.source_excerpt || "").trim();
|
|
937
|
+
const sourceFile = (item.source_file || "").trim();
|
|
938
|
+
const sourceMemoryId = (item.source_memory_id || "").trim();
|
|
939
|
+
const sourceMemoryCanonicalId = (item.source_memory_canonical_id || "").trim();
|
|
940
|
+
const sourceLayer = (item.source_layer || "").trim();
|
|
941
|
+
const sourceEventId = (item.source_event_id || "").trim();
|
|
942
|
+
const sourceField = (item.source_field || "").trim();
|
|
943
|
+
const extraParts = [];
|
|
944
|
+
if (sourceMemoryId)
|
|
945
|
+
extraParts.push(`source_memory_id=${sourceMemoryId}`);
|
|
946
|
+
if (sourceMemoryCanonicalId)
|
|
947
|
+
extraParts.push(`source_memory_canonical_id=${sourceMemoryCanonicalId}`);
|
|
948
|
+
if (sourceLayer)
|
|
949
|
+
extraParts.push(`source_layer=${sourceLayer}`);
|
|
950
|
+
if (sourceEventId)
|
|
951
|
+
extraParts.push(`source_event_id=${sourceEventId}`);
|
|
952
|
+
if (sourceField)
|
|
953
|
+
extraParts.push(`source_field=${sourceField}`);
|
|
954
|
+
if (sourceFile)
|
|
955
|
+
extraParts.push(`source_file=${sourceFile}`);
|
|
956
|
+
if (excerpt)
|
|
957
|
+
extraParts.push(`source_excerpt=${excerpt}`);
|
|
958
|
+
const extra = extraParts.length > 0 ? `\n ${extraParts.join("\n ")}` : "";
|
|
959
|
+
return `${index + 1}. [${item.id}] (${item.source}, score=${item.score.toFixed(4)}) ${item.text}${extra}`;
|
|
960
|
+
})
|
|
961
|
+
.join("\n")
|
|
962
|
+
.slice(0, 18000);
|
|
963
|
+
const prompt = [
|
|
964
|
+
`prompt_version=${READ_FUSION_PROMPT_VERSION}`,
|
|
965
|
+
"You are a memory retrieval fusion engine. Fuse multi-channel evidence into a structured answer package for the agent.",
|
|
966
|
+
"Core values and principles:",
|
|
967
|
+
"A) Truthfulness first: do not fabricate; do not infer beyond evidence.",
|
|
968
|
+
"B) Evidence first: every key conclusion must be traceable via evidence_ids.",
|
|
969
|
+
"C) Make conflicts explicit: write conflicts instead of silently overriding.",
|
|
970
|
+
"D) Be transparent about uncertainty: put uncertain parts in coverage_note.",
|
|
971
|
+
"E) Summary-first: prefer summary evidence for conclusions; source_excerpt is supporting evidence.",
|
|
972
|
+
"F) Same-source dedup: merge duplicate evidence from the same source_memory_id/source_memory_canonical_id.",
|
|
973
|
+
"G) Full-text recall: if summary/excerpt is insufficient, return event ids in need_fulltext_event_ids.",
|
|
974
|
+
"Source channel semantics:",
|
|
975
|
+
"- rules: policy/constraints; use for what should be done.",
|
|
976
|
+
"- archive: event-level stable facts (summary-first).",
|
|
977
|
+
"- vector: semantic neighbors for recall; source_field=summary/evidence indicates chunk role.",
|
|
978
|
+
"- graph: entity-relation structure; prefer for dependency/relationship questions.",
|
|
979
|
+
"Query alignment:",
|
|
980
|
+
"- answer the user query first; ignore evidence unrelated to the query.",
|
|
981
|
+
"- when evidence conflicts, prioritize recency + quality + explicitness and record the conflict.",
|
|
982
|
+
"Return strict JSON only:",
|
|
983
|
+
"{\"canonical_answer\": string, \"coverage_note\": string, \"facts\": [{\"text\": string, \"evidence_ids\": string[]}], \"timeline\": [{\"when\": string, \"event\": string, \"evidence_ids\": string[]}], \"entities\": [{\"name\": string, \"role\": string}], \"decisions\": [{\"decision\": string, \"rationale\": string, \"evidence_ids\": string[]}], \"fixes\": [{\"issue\": string, \"fix\": string, \"evidence_ids\": string[]}], \"preferences\": [{\"subject\": string, \"preference\": string, \"evidence_ids\": string[]}], \"risks\": [{\"risk\": string, \"mitigation\": string, \"evidence_ids\": string[]}], \"action_items\": [{\"item\": string, \"owner\": string, \"status\": string, \"evidence_ids\": string[]}], \"conflicts\": [{\"topic\": string, \"details\": string}], \"evidence_ids\": string[], \"need_fulltext_event_ids\": string[], \"confidence\": number}",
|
|
984
|
+
"Output constraints:",
|
|
985
|
+
"1) canonical_answer must be directly usable.",
|
|
986
|
+
"2) facts: usually 3-12 items; prefer high-quality evidence.",
|
|
987
|
+
"3) evidence_ids must come from input candidate ids.",
|
|
988
|
+
"4) conflicts must be [] when no conflict exists.",
|
|
989
|
+
"5) confidence must be within [0, 1].",
|
|
990
|
+
"6) uncertain parts must be explicitly stated in coverage_note.",
|
|
991
|
+
...READ_FUSION_REGRESSION_SAMPLES,
|
|
992
|
+
].join("\n");
|
|
993
|
+
const body = {
|
|
994
|
+
model: args.llm.model,
|
|
995
|
+
temperature: 0.1,
|
|
996
|
+
messages: [
|
|
997
|
+
{ role: "system", content: "Output JSON only. No extra text." },
|
|
998
|
+
{ role: "user", content: `${prompt}\n\nQuery:\n${args.query}\n\nCandidate Evidence:\n${evidenceText}` },
|
|
999
|
+
],
|
|
1000
|
+
};
|
|
1001
|
+
let lastError = null;
|
|
1002
|
+
for (let attempt = 0; attempt < 2; attempt += 1) {
|
|
1003
|
+
const response = await (0, http_post_1.postJsonWithTimeout)({
|
|
1004
|
+
endpoint,
|
|
1005
|
+
apiKey: args.llm.apiKey,
|
|
1006
|
+
body,
|
|
1007
|
+
timeoutMs: 20000,
|
|
1008
|
+
});
|
|
1009
|
+
if (!response.ok) {
|
|
1010
|
+
lastError = new Error(response.status > 0 ? `fusion_http_${response.status}` : (response.error || "fusion_network_error"));
|
|
1011
|
+
continue;
|
|
1012
|
+
}
|
|
1013
|
+
try {
|
|
1014
|
+
const json = (response.json || {});
|
|
1015
|
+
const content = json?.choices?.[0]?.message?.content?.trim() || "";
|
|
1016
|
+
if (!content) {
|
|
1017
|
+
lastError = new Error("fusion_empty");
|
|
1018
|
+
continue;
|
|
1019
|
+
}
|
|
1020
|
+
const parsed = JSON.parse(content);
|
|
1021
|
+
if (!parsed || typeof parsed.canonical_answer !== "string" || !parsed.canonical_answer.trim()) {
|
|
1022
|
+
lastError = new Error("fusion_invalid");
|
|
1023
|
+
continue;
|
|
1024
|
+
}
|
|
1025
|
+
const evidenceIds = Array.isArray(parsed.evidence_ids)
|
|
1026
|
+
? parsed.evidence_ids.filter(item => typeof item === "string" && item.trim())
|
|
1027
|
+
: [];
|
|
1028
|
+
const whitelistedEvidenceIds = [...new Set(evidenceIds.filter(id => candidateIdSet.has(id)))];
|
|
1029
|
+
const needFulltextEventIds = Array.isArray(parsed.need_fulltext_event_ids)
|
|
1030
|
+
? [...new Set(parsed.need_fulltext_event_ids
|
|
1031
|
+
.filter(item => typeof item === "string")
|
|
1032
|
+
.map(item => item.trim())
|
|
1033
|
+
.filter(Boolean))]
|
|
1034
|
+
: [];
|
|
1035
|
+
return {
|
|
1036
|
+
canonical_answer: parsed.canonical_answer.trim().slice(0, 6000),
|
|
1037
|
+
coverage_note: typeof parsed.coverage_note === "string" ? parsed.coverage_note.trim().slice(0, 1200) : "",
|
|
1038
|
+
facts: Array.isArray(parsed.facts) ? parsed.facts : [],
|
|
1039
|
+
timeline: Array.isArray(parsed.timeline) ? parsed.timeline : [],
|
|
1040
|
+
entities: Array.isArray(parsed.entities) ? parsed.entities : [],
|
|
1041
|
+
decisions: Array.isArray(parsed.decisions) ? parsed.decisions : [],
|
|
1042
|
+
fixes: Array.isArray(parsed.fixes) ? parsed.fixes : [],
|
|
1043
|
+
preferences: Array.isArray(parsed.preferences) ? parsed.preferences : [],
|
|
1044
|
+
risks: Array.isArray(parsed.risks) ? parsed.risks : [],
|
|
1045
|
+
action_items: Array.isArray(parsed.action_items) ? parsed.action_items : [],
|
|
1046
|
+
conflicts: Array.isArray(parsed.conflicts) ? parsed.conflicts : [],
|
|
1047
|
+
evidence_ids: whitelistedEvidenceIds,
|
|
1048
|
+
need_fulltext_event_ids: needFulltextEventIds,
|
|
1049
|
+
confidence: typeof parsed.confidence === "number"
|
|
1050
|
+
? Math.max(0, Math.min(1, parsed.confidence))
|
|
1051
|
+
: 0.5,
|
|
1052
|
+
};
|
|
1053
|
+
}
|
|
1054
|
+
catch (error) {
|
|
1055
|
+
lastError = error;
|
|
1056
|
+
}
|
|
1057
|
+
}
|
|
1058
|
+
throw lastError instanceof Error ? lastError : new Error(String(lastError || "fusion_failed"));
|
|
1059
|
+
}
|
|
163
1060
|
function createReadStore(options) {
|
|
164
1061
|
const memoryRoot = options.dbPath ? path.resolve(options.dbPath) : path.join(options.projectRoot, "data", "memory");
|
|
1062
|
+
const vectorFallbackPath = path.join(memoryRoot, "vector", "lancedb_events.jsonl");
|
|
1063
|
+
const hitStatsPath = path.join(memoryRoot, ".read_hit_stats.json");
|
|
1064
|
+
let docsCache = null;
|
|
1065
|
+
let vectorFallbackCache = null;
|
|
1066
|
+
let bm25TokenCacheSignature = "";
|
|
1067
|
+
let bm25TokenCache = new Map();
|
|
1068
|
+
let hitStatsCache = null;
|
|
1069
|
+
let hitStatsDirty = false;
|
|
1070
|
+
let hitStatsPendingMutations = 0;
|
|
1071
|
+
let lastHitStatsFlushAt = 0;
|
|
1072
|
+
const hitStatsFlushIntervalMs = 5000;
|
|
1073
|
+
const hitStatsFlushBatch = 24;
|
|
1074
|
+
const readTuning = resolveReadTuning(options.readTuning);
|
|
1075
|
+
function fileSignature(filePath) {
|
|
1076
|
+
try {
|
|
1077
|
+
if (!fs.existsSync(filePath)) {
|
|
1078
|
+
return `${filePath}:missing`;
|
|
1079
|
+
}
|
|
1080
|
+
const stat = fs.statSync(filePath);
|
|
1081
|
+
return `${filePath}:${stat.size}:${Math.floor(stat.mtimeMs)}`;
|
|
1082
|
+
}
|
|
1083
|
+
catch {
|
|
1084
|
+
return `${filePath}:error`;
|
|
1085
|
+
}
|
|
1086
|
+
}
|
|
1087
|
+
function loadHitStats() {
|
|
1088
|
+
if (hitStatsCache) {
|
|
1089
|
+
return hitStatsCache;
|
|
1090
|
+
}
|
|
1091
|
+
try {
|
|
1092
|
+
if (!fs.existsSync(hitStatsPath)) {
|
|
1093
|
+
hitStatsCache = { items: {} };
|
|
1094
|
+
return hitStatsCache;
|
|
1095
|
+
}
|
|
1096
|
+
const content = fs.readFileSync(hitStatsPath, "utf-8").trim();
|
|
1097
|
+
if (!content) {
|
|
1098
|
+
hitStatsCache = { items: {} };
|
|
1099
|
+
return hitStatsCache;
|
|
1100
|
+
}
|
|
1101
|
+
const parsed = JSON.parse(content);
|
|
1102
|
+
if (!parsed || typeof parsed !== "object" || !parsed.items || typeof parsed.items !== "object") {
|
|
1103
|
+
hitStatsCache = { items: {} };
|
|
1104
|
+
return hitStatsCache;
|
|
1105
|
+
}
|
|
1106
|
+
hitStatsCache = parsed;
|
|
1107
|
+
return hitStatsCache;
|
|
1108
|
+
}
|
|
1109
|
+
catch {
|
|
1110
|
+
hitStatsCache = { items: {} };
|
|
1111
|
+
return hitStatsCache;
|
|
1112
|
+
}
|
|
1113
|
+
}
|
|
1114
|
+
function saveHitStats(state) {
|
|
1115
|
+
try {
|
|
1116
|
+
const dir = path.dirname(hitStatsPath);
|
|
1117
|
+
if (!fs.existsSync(dir)) {
|
|
1118
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
1119
|
+
}
|
|
1120
|
+
fs.writeFileSync(hitStatsPath, JSON.stringify(state, null, 2), "utf-8");
|
|
1121
|
+
}
|
|
1122
|
+
catch (error) {
|
|
1123
|
+
options.logger.warn(`Failed to persist read hit stats: ${error}`);
|
|
1124
|
+
}
|
|
1125
|
+
}
|
|
1126
|
+
function maybeFlushHitStats(force) {
|
|
1127
|
+
if (!hitStatsDirty || !hitStatsCache) {
|
|
1128
|
+
return;
|
|
1129
|
+
}
|
|
1130
|
+
const now = Date.now();
|
|
1131
|
+
if (!force &&
|
|
1132
|
+
hitStatsPendingMutations < hitStatsFlushBatch &&
|
|
1133
|
+
(now - lastHitStatsFlushAt) < hitStatsFlushIntervalMs) {
|
|
1134
|
+
return;
|
|
1135
|
+
}
|
|
1136
|
+
saveHitStats(hitStatsCache);
|
|
1137
|
+
hitStatsDirty = false;
|
|
1138
|
+
hitStatsPendingMutations = 0;
|
|
1139
|
+
lastHitStatsFlushAt = now;
|
|
1140
|
+
}
|
|
1141
|
+
function markHit(ids) {
|
|
1142
|
+
if (!ids.length) {
|
|
1143
|
+
return;
|
|
1144
|
+
}
|
|
1145
|
+
const state = loadHitStats();
|
|
1146
|
+
const now = new Date().toISOString();
|
|
1147
|
+
for (const id of ids) {
|
|
1148
|
+
const key = (id || "").trim();
|
|
1149
|
+
if (!key)
|
|
1150
|
+
continue;
|
|
1151
|
+
const prev = state.items[key];
|
|
1152
|
+
state.items[key] = {
|
|
1153
|
+
count: (prev?.count || 0) + 1,
|
|
1154
|
+
lastHitAt: now,
|
|
1155
|
+
};
|
|
1156
|
+
}
|
|
1157
|
+
const entries = Object.entries(state.items)
|
|
1158
|
+
.sort((a, b) => {
|
|
1159
|
+
const ta = Date.parse(a[1].lastHitAt || "");
|
|
1160
|
+
const tb = Date.parse(b[1].lastHitAt || "");
|
|
1161
|
+
return (Number.isFinite(tb) ? tb : 0) - (Number.isFinite(ta) ? ta : 0);
|
|
1162
|
+
})
|
|
1163
|
+
.slice(0, 20000);
|
|
1164
|
+
state.items = Object.fromEntries(entries);
|
|
1165
|
+
hitStatsCache = state;
|
|
1166
|
+
hitStatsDirty = true;
|
|
1167
|
+
hitStatsPendingMutations += ids.length;
|
|
1168
|
+
maybeFlushHitStats(false);
|
|
1169
|
+
}
|
|
165
1170
|
function loadAllDocuments() {
|
|
166
1171
|
const cortexRulesPath = path.join(memoryRoot, "CORTEX_RULES.md");
|
|
167
1172
|
const memoryMdPath = path.join(memoryRoot, "MEMORY.md");
|
|
168
1173
|
const activeSessionsPath = path.join(memoryRoot, "sessions", "active", "sessions.jsonl");
|
|
169
1174
|
const archiveSessionsPath = path.join(memoryRoot, "sessions", "archive", "sessions.jsonl");
|
|
170
|
-
|
|
1175
|
+
const graphMemoryPath = path.join(memoryRoot, "graph", "memory.jsonl");
|
|
1176
|
+
const signature = [
|
|
1177
|
+
fileSignature(cortexRulesPath),
|
|
1178
|
+
fileSignature(memoryMdPath),
|
|
1179
|
+
fileSignature(activeSessionsPath),
|
|
1180
|
+
fileSignature(archiveSessionsPath),
|
|
1181
|
+
fileSignature(graphMemoryPath),
|
|
1182
|
+
].join("|");
|
|
1183
|
+
if (docsCache && docsCache.signature === signature) {
|
|
1184
|
+
return docsCache.docs;
|
|
1185
|
+
}
|
|
1186
|
+
const archiveEventTypeById = new Map();
|
|
1187
|
+
if (fs.existsSync(archiveSessionsPath)) {
|
|
1188
|
+
const archiveContent = safeReadFile(archiveSessionsPath);
|
|
1189
|
+
for (const line of archiveContent.split(/\r?\n/)) {
|
|
1190
|
+
const trimmed = line.trim();
|
|
1191
|
+
if (!trimmed)
|
|
1192
|
+
continue;
|
|
1193
|
+
try {
|
|
1194
|
+
const parsed = JSON.parse(trimmed);
|
|
1195
|
+
const id = typeof parsed.id === "string" ? parsed.id.trim() : "";
|
|
1196
|
+
const eventType = typeof parsed.event_type === "string" ? parsed.event_type.trim() : "";
|
|
1197
|
+
if (id && eventType) {
|
|
1198
|
+
archiveEventTypeById.set(id, eventType);
|
|
1199
|
+
}
|
|
1200
|
+
}
|
|
1201
|
+
catch { }
|
|
1202
|
+
}
|
|
1203
|
+
}
|
|
1204
|
+
const graphDocs = [];
|
|
1205
|
+
if (fs.existsSync(graphMemoryPath)) {
|
|
1206
|
+
const graphContent = safeReadFile(graphMemoryPath);
|
|
1207
|
+
for (const line of graphContent.split(/\r?\n/)) {
|
|
1208
|
+
const trimmed = line.trim();
|
|
1209
|
+
if (!trimmed)
|
|
1210
|
+
continue;
|
|
1211
|
+
try {
|
|
1212
|
+
const parsed = JSON.parse(trimmed);
|
|
1213
|
+
const id = typeof parsed.id === "string" ? parsed.id : "";
|
|
1214
|
+
const sourceEventId = typeof parsed.source_event_id === "string" ? parsed.source_event_id : "";
|
|
1215
|
+
const archiveEventId = typeof parsed.archive_event_id === "string" ? parsed.archive_event_id : "";
|
|
1216
|
+
const eventRefId = archiveEventId || sourceEventId;
|
|
1217
|
+
const sessionId = typeof parsed.session_id === "string" ? parsed.session_id : "";
|
|
1218
|
+
const sourceLayer = typeof parsed.source_layer === "string" ? parsed.source_layer : "";
|
|
1219
|
+
const sourceFile = typeof parsed.source_file === "string" ? parsed.source_file : "";
|
|
1220
|
+
const timestamp = typeof parsed.timestamp === "string" ? Date.parse(parsed.timestamp) : NaN;
|
|
1221
|
+
const entities = Array.isArray(parsed.entities)
|
|
1222
|
+
? parsed.entities.map((item) => (typeof item === "string" ? item.trim() : "")).filter(Boolean)
|
|
1223
|
+
: [];
|
|
1224
|
+
const entityTypes = typeof parsed.entity_types === "object" && parsed.entity_types !== null
|
|
1225
|
+
? parsed.entity_types
|
|
1226
|
+
: {};
|
|
1227
|
+
const relations = Array.isArray(parsed.relations)
|
|
1228
|
+
? parsed.relations
|
|
1229
|
+
.map((item) => {
|
|
1230
|
+
if (typeof item !== "object" || item === null)
|
|
1231
|
+
return null;
|
|
1232
|
+
const relation = item;
|
|
1233
|
+
const source = typeof relation.source === "string" ? relation.source.trim() : "";
|
|
1234
|
+
const target = typeof relation.target === "string" ? relation.target.trim() : "";
|
|
1235
|
+
const type = typeof relation.type === "string" ? relation.type.trim() : "related_to";
|
|
1236
|
+
if (!source || !target)
|
|
1237
|
+
return null;
|
|
1238
|
+
return { source, target, type };
|
|
1239
|
+
})
|
|
1240
|
+
.filter((item) => Boolean(item))
|
|
1241
|
+
: [];
|
|
1242
|
+
const eventType = (typeof parsed.event_type === "string" ? parsed.event_type : "") || archiveEventTypeById.get(eventRefId) || "";
|
|
1243
|
+
const entityLines = entities.length > 0
|
|
1244
|
+
? entities.map((entity, index) => {
|
|
1245
|
+
const entityType = entityTypes[entity];
|
|
1246
|
+
return `${index + 1}. ${entity}${entityType ? ` (${entityType})` : ""}`;
|
|
1247
|
+
}).join("\n")
|
|
1248
|
+
: "none";
|
|
1249
|
+
const relationLines = relations.length > 0
|
|
1250
|
+
? relations.map((relation, index) => `${index + 1}. ${relation.source} -[${relation.type}]-> ${relation.target}`).join("\n")
|
|
1251
|
+
: "none";
|
|
1252
|
+
const relationFacts = relations.length > 0
|
|
1253
|
+
? relations.map(relation => `${relation.source} ${relation.type} ${relation.target}`).join(" | ")
|
|
1254
|
+
: "none";
|
|
1255
|
+
const text = [
|
|
1256
|
+
`# Graph Record`,
|
|
1257
|
+
`record_id: ${id}`,
|
|
1258
|
+
`source_event_id: ${sourceEventId || archiveEventId || "unknown"}`,
|
|
1259
|
+
`source_layer: ${sourceLayer || "unknown"}`,
|
|
1260
|
+
`archive_event_id: ${archiveEventId || "n/a"}`,
|
|
1261
|
+
`event_type: ${eventType || "unknown"}`,
|
|
1262
|
+
`session_id: ${sessionId || "unknown"}`,
|
|
1263
|
+
`source_file: ${sourceFile || "unknown"}`,
|
|
1264
|
+
``,
|
|
1265
|
+
`## Entities`,
|
|
1266
|
+
entityLines,
|
|
1267
|
+
``,
|
|
1268
|
+
`## Relations`,
|
|
1269
|
+
relationLines,
|
|
1270
|
+
``,
|
|
1271
|
+
`## Relation Facts`,
|
|
1272
|
+
relationFacts,
|
|
1273
|
+
].join("\n");
|
|
1274
|
+
if (id && text.trim()) {
|
|
1275
|
+
graphDocs.push({
|
|
1276
|
+
id,
|
|
1277
|
+
text,
|
|
1278
|
+
source: "sessions_graph",
|
|
1279
|
+
timestamp: Number.isFinite(timestamp) ? timestamp : undefined,
|
|
1280
|
+
layer: sourceLayer === "active_only" ? "active" : "archive",
|
|
1281
|
+
sourceMemoryId: eventRefId || id,
|
|
1282
|
+
sourceEventId: sourceEventId || archiveEventId || undefined,
|
|
1283
|
+
sessionId,
|
|
1284
|
+
entities,
|
|
1285
|
+
relations,
|
|
1286
|
+
});
|
|
1287
|
+
}
|
|
1288
|
+
}
|
|
1289
|
+
catch (error) {
|
|
1290
|
+
options.logger.debug(`Skipping invalid graph memory line: ${error}`);
|
|
1291
|
+
}
|
|
1292
|
+
}
|
|
1293
|
+
}
|
|
1294
|
+
const entitySummaryDocs = buildEntityGraphSummaryDocs(graphDocs);
|
|
1295
|
+
const docs = [
|
|
171
1296
|
...parseMarkdownFile(cortexRulesPath, "CORTEX_RULES.md"),
|
|
172
1297
|
...parseMarkdownFile(memoryMdPath, "MEMORY.md"),
|
|
173
1298
|
...parseJsonlFile(activeSessionsPath, "sessions_active", options.logger),
|
|
174
1299
|
...parseJsonlFile(archiveSessionsPath, "sessions_archive", options.logger),
|
|
1300
|
+
...graphDocs,
|
|
1301
|
+
...entitySummaryDocs,
|
|
175
1302
|
];
|
|
1303
|
+
docsCache = { signature, docs };
|
|
1304
|
+
return docs;
|
|
1305
|
+
}
|
|
1306
|
+
function loadVectorFallbackCached() {
|
|
1307
|
+
const signature = fileSignature(vectorFallbackPath);
|
|
1308
|
+
if (vectorFallbackCache && vectorFallbackCache.signature === signature) {
|
|
1309
|
+
return vectorFallbackCache.docs;
|
|
1310
|
+
}
|
|
1311
|
+
const docs = parseVectorFallback(vectorFallbackPath, options.logger);
|
|
1312
|
+
vectorFallbackCache = { signature, docs };
|
|
1313
|
+
return docs;
|
|
1314
|
+
}
|
|
1315
|
+
function getBm25Tokens(doc, signature) {
|
|
1316
|
+
if (bm25TokenCacheSignature !== signature) {
|
|
1317
|
+
bm25TokenCacheSignature = signature;
|
|
1318
|
+
bm25TokenCache = new Map();
|
|
1319
|
+
}
|
|
1320
|
+
const key = `${doc.source}|${doc.id}|${doc.text.length}|${doc.text.slice(0, 64)}`;
|
|
1321
|
+
const cached = bm25TokenCache.get(key);
|
|
1322
|
+
if (cached) {
|
|
1323
|
+
return cached;
|
|
1324
|
+
}
|
|
1325
|
+
const tokens = tokenize(doc.text);
|
|
1326
|
+
bm25TokenCache.set(key, tokens);
|
|
1327
|
+
return tokens;
|
|
176
1328
|
}
|
|
177
1329
|
async function searchMemory(args) {
|
|
178
1330
|
const query = args.query?.trim();
|
|
179
1331
|
if (!query) {
|
|
180
1332
|
return { results: [] };
|
|
181
1333
|
}
|
|
1334
|
+
const mode = args.mode === "lightweight" ? "lightweight" : "default";
|
|
1335
|
+
const lightweightMode = mode === "lightweight";
|
|
182
1336
|
const docs = loadAllDocuments();
|
|
183
|
-
const
|
|
1337
|
+
const hitStats = loadHitStats();
|
|
1338
|
+
const intent = classifyIntent(query);
|
|
1339
|
+
const preferredTypes = preferredEventTypes(intent);
|
|
1340
|
+
let queryEmbedding = null;
|
|
1341
|
+
const embeddingModel = options.embedding?.model || "";
|
|
1342
|
+
const embeddingApiKey = options.embedding?.apiKey || "";
|
|
1343
|
+
const embeddingBaseUrl = normalizeBaseUrl(options.embedding?.baseURL || options.embedding?.baseUrl);
|
|
1344
|
+
if (!lightweightMode && embeddingModel && embeddingApiKey && embeddingBaseUrl) {
|
|
1345
|
+
try {
|
|
1346
|
+
queryEmbedding = await requestEmbedding({
|
|
1347
|
+
text: query,
|
|
1348
|
+
model: embeddingModel,
|
|
1349
|
+
apiKey: embeddingApiKey,
|
|
1350
|
+
baseUrl: embeddingBaseUrl,
|
|
1351
|
+
dimensions: options.embedding?.dimensions,
|
|
1352
|
+
});
|
|
1353
|
+
}
|
|
1354
|
+
catch (error) {
|
|
1355
|
+
options.logger.warn(`Embedding query failed, fallback to lexical search: ${error}`);
|
|
1356
|
+
}
|
|
1357
|
+
}
|
|
1358
|
+
const vectorDocsFromLance = queryEmbedding && queryEmbedding.length > 0
|
|
1359
|
+
? await searchLanceDb({ memoryRoot, queryEmbedding, limit: Math.max(20, args.topK * 8), logger: options.logger })
|
|
1360
|
+
: [];
|
|
1361
|
+
const vectorDocsFallback = vectorDocsFromLance.length > 0
|
|
1362
|
+
? []
|
|
1363
|
+
: loadVectorFallbackCached();
|
|
1364
|
+
const vectorDocs = [...vectorDocsFromLance, ...vectorDocsFallback];
|
|
1365
|
+
const archiveSourceById = new Map();
|
|
1366
|
+
for (const doc of docs) {
|
|
1367
|
+
if (doc.source !== "sessions_archive")
|
|
1368
|
+
continue;
|
|
1369
|
+
const key = (doc.sourceMemoryId || doc.id || "").trim();
|
|
1370
|
+
if (!key)
|
|
1371
|
+
continue;
|
|
1372
|
+
archiveSourceById.set(key, {
|
|
1373
|
+
sourceText: doc.sourceText,
|
|
1374
|
+
summaryText: doc.summaryText || doc.text,
|
|
1375
|
+
sourceFile: doc.sourceFile,
|
|
1376
|
+
});
|
|
1377
|
+
}
|
|
1378
|
+
for (const doc of vectorDocs) {
|
|
1379
|
+
const key = (doc.sourceMemoryId || "").trim();
|
|
1380
|
+
if (!key)
|
|
1381
|
+
continue;
|
|
1382
|
+
const linked = archiveSourceById.get(key);
|
|
1383
|
+
if (!linked)
|
|
1384
|
+
continue;
|
|
1385
|
+
if (!doc.sourceText && linked.sourceText) {
|
|
1386
|
+
doc.sourceText = linked.sourceText;
|
|
1387
|
+
}
|
|
1388
|
+
if (!doc.summaryText && linked.summaryText) {
|
|
1389
|
+
doc.summaryText = linked.summaryText;
|
|
1390
|
+
}
|
|
1391
|
+
if (!doc.sourceFile && linked.sourceFile) {
|
|
1392
|
+
doc.sourceFile = linked.sourceFile;
|
|
1393
|
+
}
|
|
1394
|
+
}
|
|
1395
|
+
const graphDocs = docs
|
|
1396
|
+
.filter(doc => doc.source === "sessions_graph" || doc.source === "sessions_graph_entity")
|
|
184
1397
|
.map(doc => {
|
|
185
|
-
const
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
1398
|
+
const graphText = [
|
|
1399
|
+
doc.text,
|
|
1400
|
+
...(doc.relations || []).map(relation => `${relation.source} ${relation.type} ${relation.target}`),
|
|
1401
|
+
].join(" | ");
|
|
1402
|
+
return {
|
|
1403
|
+
...doc,
|
|
1404
|
+
text: graphText,
|
|
1405
|
+
};
|
|
1406
|
+
});
|
|
1407
|
+
const rulesDocs = docs.filter(doc => doc.source === "CORTEX_RULES.md");
|
|
1408
|
+
const archiveDocs = docs.filter(doc => doc.source === "sessions_active" || doc.source === "sessions_archive");
|
|
1409
|
+
const bm25Terms = tokenize(query);
|
|
1410
|
+
const bm25Corpus = [...rulesDocs, ...archiveDocs, ...vectorDocs, ...graphDocs];
|
|
1411
|
+
const bm25Signature = `${docsCache?.signature || "na"}|vector:${vectorDocs.length}:${vectorDocs.slice(0, 40).map(item => `${item.id}:${item.text.length}`).join(",")}`;
|
|
1412
|
+
const bm25Stats = buildBm25Stats(bm25Corpus, bm25Terms, doc => getBm25Tokens(doc, bm25Signature));
|
|
1413
|
+
const combinedCandidates = [];
|
|
1414
|
+
const channels = {
|
|
1415
|
+
rules: [],
|
|
1416
|
+
archive: [],
|
|
1417
|
+
vector: [],
|
|
1418
|
+
graph: [],
|
|
1419
|
+
};
|
|
1420
|
+
const evaluateDoc = (doc, source) => {
|
|
1421
|
+
const lexical = scoreText(query, doc.text);
|
|
1422
|
+
const bm25 = bm25Score({
|
|
1423
|
+
queryTerms: bm25Terms,
|
|
1424
|
+
docText: doc.text,
|
|
1425
|
+
docTokens: getBm25Tokens(doc, bm25Signature),
|
|
1426
|
+
docCount: bm25Corpus.length,
|
|
1427
|
+
avgDocLen: bm25Stats.avgDocLen,
|
|
1428
|
+
docFreq: bm25Stats.docFreq,
|
|
1429
|
+
});
|
|
1430
|
+
const lexicalCombined = lexical + bm25 * readTuning.scoring.bm25Scale;
|
|
1431
|
+
const semantic = queryEmbedding && Array.isArray(doc.embedding) && doc.embedding.length > 0
|
|
1432
|
+
? Math.max(0, cosineSimilarity(queryEmbedding, doc.embedding) * 5)
|
|
1433
|
+
: 0;
|
|
1434
|
+
if (lexicalCombined <= 0 && semantic <= 0) {
|
|
1435
|
+
return null;
|
|
1436
|
+
}
|
|
1437
|
+
const recency = recencyScore(doc.timestamp, readTuning.recency.buckets);
|
|
1438
|
+
const quality = typeof doc.qualityScore === "number" ? Math.max(0, Math.min(1, doc.qualityScore)) : 0.5;
|
|
1439
|
+
const typeMatch = preferredTypes.length > 0 && doc.eventType
|
|
1440
|
+
? (preferredTypes.includes(doc.eventType) ? 1 : 0)
|
|
1441
|
+
: 0.5;
|
|
1442
|
+
const graphMatch = source === "graph" ? 1 : 0;
|
|
1443
|
+
const sourceBaseWeight = sourceWeight(source, intent);
|
|
1444
|
+
const sourceConfigWeight = customChannelWeight(source, options.fusion);
|
|
1445
|
+
const lengthNorm = lengthNormalizeFactor(doc, options.fusion);
|
|
1446
|
+
const baseWeighted = (readTuning.scoring.lexicalWeight * lexicalCombined +
|
|
1447
|
+
readTuning.scoring.semanticWeight * (semantic * lengthNorm) +
|
|
1448
|
+
readTuning.scoring.recencyWeight * recency +
|
|
1449
|
+
readTuning.scoring.qualityWeight * quality +
|
|
1450
|
+
readTuning.scoring.typeMatchWeight * typeMatch +
|
|
1451
|
+
readTuning.scoring.graphMatchWeight * graphMatch) * sourceBaseWeight * sourceConfigWeight;
|
|
1452
|
+
const decayFactor = computeDecayFactor(doc.id, doc.eventType, doc.timestamp, options.memoryDecay, hitStats);
|
|
1453
|
+
const weighted = baseWeighted * decayFactor;
|
|
1454
|
+
return {
|
|
1455
|
+
doc,
|
|
1456
|
+
source,
|
|
1457
|
+
lexical: lexicalCombined,
|
|
1458
|
+
bm25,
|
|
1459
|
+
semantic,
|
|
1460
|
+
recency,
|
|
1461
|
+
quality,
|
|
1462
|
+
typeMatch,
|
|
1463
|
+
graphMatch,
|
|
1464
|
+
decayFactor,
|
|
1465
|
+
weighted,
|
|
1466
|
+
};
|
|
1467
|
+
};
|
|
1468
|
+
for (const doc of rulesDocs) {
|
|
1469
|
+
const candidate = evaluateDoc(doc, "rules");
|
|
1470
|
+
if (candidate)
|
|
1471
|
+
channels.rules.push(candidate);
|
|
1472
|
+
}
|
|
1473
|
+
for (const doc of archiveDocs) {
|
|
1474
|
+
const candidate = evaluateDoc(doc, "archive");
|
|
1475
|
+
if (candidate)
|
|
1476
|
+
channels.archive.push(candidate);
|
|
1477
|
+
}
|
|
1478
|
+
for (const doc of vectorDocs) {
|
|
1479
|
+
const candidate = evaluateDoc(doc, "vector");
|
|
1480
|
+
if (candidate)
|
|
1481
|
+
channels.vector.push(candidate);
|
|
1482
|
+
}
|
|
1483
|
+
for (const doc of graphDocs) {
|
|
1484
|
+
const candidate = evaluateDoc(doc, "graph");
|
|
1485
|
+
if (candidate)
|
|
1486
|
+
channels.graph.push(candidate);
|
|
1487
|
+
}
|
|
1488
|
+
for (const key of Object.keys(channels)) {
|
|
1489
|
+
channels[key].sort((a, b) => b.weighted - a.weighted);
|
|
1490
|
+
combinedCandidates.push(...channels[key].slice(0, channelQuota(key, args.topK, options.fusion)));
|
|
1491
|
+
}
|
|
1492
|
+
const rrfMap = new Map();
|
|
1493
|
+
const weightedMap = new Map();
|
|
1494
|
+
const rrfK = readTuning.rrf.k;
|
|
1495
|
+
for (const key of Object.keys(channels)) {
|
|
1496
|
+
const list = channels[key];
|
|
1497
|
+
for (let i = 0; i < list.length; i += 1) {
|
|
1498
|
+
const candidate = list[i];
|
|
1499
|
+
const rrf = 1 / (rrfK + i + 1);
|
|
1500
|
+
const mergeKey = mergeKeyFromDoc(candidate.doc);
|
|
1501
|
+
rrfMap.set(mergeKey, (rrfMap.get(mergeKey) || 0) + rrf);
|
|
1502
|
+
const current = weightedMap.get(mergeKey);
|
|
1503
|
+
if (!current || candidate.weighted > current.weighted) {
|
|
1504
|
+
weightedMap.set(mergeKey, candidate);
|
|
1505
|
+
}
|
|
1506
|
+
}
|
|
1507
|
+
}
|
|
1508
|
+
const preRanked = [...weightedMap.entries()]
|
|
1509
|
+
.map(([mergeKey, candidate]) => ({
|
|
1510
|
+
id: candidate.doc.id,
|
|
1511
|
+
merge_key: mergeKey,
|
|
1512
|
+
source_memory_id: candidate.doc.sourceMemoryId || "",
|
|
1513
|
+
source_memory_canonical_id: candidate.doc.sourceMemoryCanonicalId || "",
|
|
1514
|
+
source_event_id: candidate.doc.sourceEventId || "",
|
|
1515
|
+
source_field: candidate.doc.sourceField || "",
|
|
1516
|
+
text: candidate.doc.summaryText || candidate.doc.text,
|
|
1517
|
+
source_text: candidate.doc.sourceText ? candidate.doc.sourceText.slice(0, 4000) : "",
|
|
1518
|
+
source_excerpt: candidate.doc.sourceText ? candidate.doc.sourceText.slice(0, 360) : "",
|
|
1519
|
+
source_file: candidate.doc.sourceFile || "",
|
|
1520
|
+
source: candidate.doc.source,
|
|
1521
|
+
layer: candidate.doc.layer || "",
|
|
1522
|
+
event_type: candidate.doc.eventType || "",
|
|
1523
|
+
quality_score: candidate.quality,
|
|
1524
|
+
timestamp: candidate.doc.timestamp ? new Date(candidate.doc.timestamp).toISOString() : "",
|
|
1525
|
+
score: candidate.weighted + (rrfMap.get(mergeKey) || 0) * readTuning.rrf.weight,
|
|
1526
|
+
score_breakdown: {
|
|
1527
|
+
lexical: Number(candidate.lexical.toFixed(4)),
|
|
1528
|
+
bm25: Number(candidate.bm25.toFixed(4)),
|
|
1529
|
+
semantic: Number(candidate.semantic.toFixed(4)),
|
|
1530
|
+
recency: Number(candidate.recency.toFixed(4)),
|
|
1531
|
+
quality: Number(candidate.quality.toFixed(4)),
|
|
1532
|
+
type: Number(candidate.typeMatch.toFixed(4)),
|
|
1533
|
+
graph: Number(candidate.graphMatch.toFixed(4)),
|
|
1534
|
+
decay: Number(candidate.decayFactor.toFixed(4)),
|
|
1535
|
+
rrf: Number(((rrfMap.get(mergeKey) || 0) * readTuning.rrf.weight).toFixed(4)),
|
|
1536
|
+
weighted: Number(candidate.weighted.toFixed(4)),
|
|
1537
|
+
},
|
|
1538
|
+
reason_tags: [
|
|
1539
|
+
`intent:${intent.toLowerCase()}`,
|
|
1540
|
+
candidate.semantic > 0 ? "vector_hit" : "lexical_hit",
|
|
1541
|
+
candidate.typeMatch >= 1 ? "event_type_match" : "event_type_weak",
|
|
1542
|
+
candidate.recency >= 0.8 ? "recent" : "historical",
|
|
1543
|
+
candidate.quality >= 0.7 ? "high_quality" : "normal_quality",
|
|
1544
|
+
candidate.decayFactor < 1 ? `decay:${candidate.decayFactor.toFixed(3)}` : "decay:1.000",
|
|
1545
|
+
`source:${candidate.source}`,
|
|
1546
|
+
`merge_key:${mergeKey}`,
|
|
1547
|
+
],
|
|
1548
|
+
}))
|
|
190
1549
|
.sort((a, b) => b.score - a.score)
|
|
191
|
-
.slice(0, Math.max(1, args.topK))
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
1550
|
+
.slice(0, Math.max(1, Math.max(args.topK, 20)));
|
|
1551
|
+
const lexicalRanked = preRanked
|
|
1552
|
+
.map(doc => {
|
|
1553
|
+
const boost = withRecencyBoost(doc.score, doc.timestamp ? Date.parse(doc.timestamp) : undefined, readTuning.recency.buckets);
|
|
1554
|
+
return { ...doc, score: Number(boost.toFixed(4)) };
|
|
1555
|
+
});
|
|
1556
|
+
const rerankerModel = options.reranker?.model || "";
|
|
1557
|
+
const rerankerApiKey = options.reranker?.apiKey || "";
|
|
1558
|
+
const rerankerBaseUrl = normalizeBaseUrl(options.reranker?.baseURL || options.reranker?.baseUrl);
|
|
1559
|
+
const fusionEnabled = !lightweightMode && options.fusion?.enabled !== false;
|
|
1560
|
+
const llmModel = options.llm?.model || "";
|
|
1561
|
+
const llmApiKey = options.llm?.apiKey || "";
|
|
1562
|
+
const llmBaseUrl = normalizeBaseUrl(options.llm?.baseURL || options.llm?.baseUrl);
|
|
1563
|
+
const fusionAuthoritative = options.fusion?.authoritative !== false;
|
|
1564
|
+
const skipRerankerForFusion = fusionEnabled && fusionAuthoritative && llmModel && llmApiKey && llmBaseUrl;
|
|
1565
|
+
let rerankedSimple = lexicalRanked.map(item => ({
|
|
1566
|
+
id: item.id,
|
|
1567
|
+
merge_key: item.merge_key,
|
|
1568
|
+
text: item.text,
|
|
1569
|
+
source: item.source,
|
|
1570
|
+
score: item.score,
|
|
197
1571
|
}));
|
|
198
|
-
|
|
1572
|
+
if (!lightweightMode && rerankerModel && rerankerApiKey && rerankerBaseUrl && lexicalRanked.length > 1 && !skipRerankerForFusion) {
|
|
1573
|
+
try {
|
|
1574
|
+
rerankedSimple = await requestRerank({
|
|
1575
|
+
query,
|
|
1576
|
+
candidates: lexicalRanked.map(item => ({ id: item.id, text: item.text, source: item.source, score: item.score })),
|
|
1577
|
+
model: rerankerModel,
|
|
1578
|
+
apiKey: rerankerApiKey,
|
|
1579
|
+
baseUrl: rerankerBaseUrl,
|
|
1580
|
+
});
|
|
1581
|
+
rerankedSimple = rerankedSimple.map(item => {
|
|
1582
|
+
const found = lexicalRanked.find(entry => entry.id === item.id);
|
|
1583
|
+
return { ...item, merge_key: found?.merge_key || item.id };
|
|
1584
|
+
});
|
|
1585
|
+
}
|
|
1586
|
+
catch (error) {
|
|
1587
|
+
options.logger.warn(`Reranker failed, keep hybrid ranking: ${error}`);
|
|
1588
|
+
}
|
|
1589
|
+
}
|
|
1590
|
+
const ranked = rerankedSimple.slice(0, Math.max(1, args.topK)).map(item => {
|
|
1591
|
+
const hit = lexicalRanked.find(entry => entry.id === item.id);
|
|
1592
|
+
return {
|
|
1593
|
+
id: item.id,
|
|
1594
|
+
merge_key: hit?.merge_key || item.merge_key || item.id,
|
|
1595
|
+
source_memory_id: hit?.source_memory_id || "",
|
|
1596
|
+
source_memory_canonical_id: hit?.source_memory_canonical_id || "",
|
|
1597
|
+
source_event_id: hit?.source_event_id || "",
|
|
1598
|
+
source_field: hit?.source_field || "",
|
|
1599
|
+
fulltext_event_id: (hit?.source_event_id || hit?.source_memory_id || item.id || ""),
|
|
1600
|
+
text: item.text,
|
|
1601
|
+
source_text: hit?.source_text || "",
|
|
1602
|
+
source_excerpt: hit?.source_excerpt || "",
|
|
1603
|
+
source_file: hit?.source_file || "",
|
|
1604
|
+
source: item.source,
|
|
1605
|
+
layer: hit?.layer || "",
|
|
1606
|
+
event_type: hit?.event_type || "",
|
|
1607
|
+
quality_score: hit?.quality_score ?? 0,
|
|
1608
|
+
timestamp: hit?.timestamp || "",
|
|
1609
|
+
score: Number(item.score.toFixed(4)),
|
|
1610
|
+
score_breakdown: hit?.score_breakdown || {},
|
|
1611
|
+
reason_tags: Array.isArray(hit?.reason_tags) ? hit?.reason_tags : [],
|
|
1612
|
+
explain: {
|
|
1613
|
+
merge_key: hit?.merge_key || item.merge_key || item.id,
|
|
1614
|
+
source_memory_id: hit?.source_memory_id || "",
|
|
1615
|
+
source_memory_canonical_id: hit?.source_memory_canonical_id || "",
|
|
1616
|
+
source_event_id: hit?.source_event_id || "",
|
|
1617
|
+
source_field: hit?.source_field || "",
|
|
1618
|
+
channel: item.source,
|
|
1619
|
+
source_file: hit?.source_file || "",
|
|
1620
|
+
layer: hit?.layer || "",
|
|
1621
|
+
score_breakdown: hit?.score_breakdown || {},
|
|
1622
|
+
reason_tags: Array.isArray(hit?.reason_tags) ? hit?.reason_tags : [],
|
|
1623
|
+
},
|
|
1624
|
+
};
|
|
1625
|
+
});
|
|
1626
|
+
const minLexicalHits = Math.max(0, Math.floor(options.fusion?.minLexicalHits ?? 1));
|
|
1627
|
+
const minSemanticHits = Math.max(0, Math.floor(options.fusion?.minSemanticHits ?? 1));
|
|
1628
|
+
const fallbackPool = lexicalRanked.filter(item => !ranked.some(existing => existing.id === item.id));
|
|
1629
|
+
const lexicalCount = ranked.filter(item => item.reason_tags.includes("lexical_hit")).length;
|
|
1630
|
+
const semanticCount = ranked.filter(item => item.reason_tags.includes("vector_hit")).length;
|
|
1631
|
+
if (semanticCount < minSemanticHits) {
|
|
1632
|
+
const needed = minSemanticHits - semanticCount;
|
|
1633
|
+
const supplement = fallbackPool.filter(item => item.reason_tags.includes("vector_hit")).slice(0, needed);
|
|
1634
|
+
for (const item of supplement) {
|
|
1635
|
+
ranked.push({
|
|
1636
|
+
id: item.id,
|
|
1637
|
+
merge_key: item.merge_key,
|
|
1638
|
+
source_memory_id: item.source_memory_id,
|
|
1639
|
+
source_memory_canonical_id: item.source_memory_canonical_id,
|
|
1640
|
+
source_event_id: item.source_event_id || "",
|
|
1641
|
+
source_field: item.source_field || "",
|
|
1642
|
+
fulltext_event_id: item.source_event_id || item.source_memory_id || item.id,
|
|
1643
|
+
text: item.text,
|
|
1644
|
+
source_text: item.source_text || "",
|
|
1645
|
+
source_excerpt: item.source_excerpt || "",
|
|
1646
|
+
source_file: item.source_file || "",
|
|
1647
|
+
source: item.source,
|
|
1648
|
+
layer: item.layer,
|
|
1649
|
+
event_type: item.event_type,
|
|
1650
|
+
quality_score: item.quality_score,
|
|
1651
|
+
timestamp: item.timestamp,
|
|
1652
|
+
score: Number(item.score.toFixed(4)),
|
|
1653
|
+
score_breakdown: item.score_breakdown || {},
|
|
1654
|
+
reason_tags: Array.isArray(item.reason_tags) ? item.reason_tags : [],
|
|
1655
|
+
explain: {
|
|
1656
|
+
merge_key: item.merge_key,
|
|
1657
|
+
source_memory_id: item.source_memory_id,
|
|
1658
|
+
source_memory_canonical_id: item.source_memory_canonical_id,
|
|
1659
|
+
source_event_id: item.source_event_id || "",
|
|
1660
|
+
source_field: item.source_field || "",
|
|
1661
|
+
channel: item.source,
|
|
1662
|
+
source_file: item.source_file || "",
|
|
1663
|
+
layer: item.layer,
|
|
1664
|
+
score_breakdown: item.score_breakdown || {},
|
|
1665
|
+
reason_tags: Array.isArray(item.reason_tags) ? item.reason_tags : [],
|
|
1666
|
+
},
|
|
1667
|
+
});
|
|
1668
|
+
}
|
|
1669
|
+
}
|
|
1670
|
+
if (lexicalCount < minLexicalHits) {
|
|
1671
|
+
const needed = minLexicalHits - lexicalCount;
|
|
1672
|
+
const supplement = fallbackPool.filter(item => item.reason_tags.includes("lexical_hit")).slice(0, needed);
|
|
1673
|
+
for (const item of supplement) {
|
|
1674
|
+
if (ranked.some(existing => existing.id === item.id)) {
|
|
1675
|
+
continue;
|
|
1676
|
+
}
|
|
1677
|
+
ranked.push({
|
|
1678
|
+
id: item.id,
|
|
1679
|
+
merge_key: item.merge_key,
|
|
1680
|
+
source_memory_id: item.source_memory_id,
|
|
1681
|
+
source_memory_canonical_id: item.source_memory_canonical_id,
|
|
1682
|
+
source_event_id: item.source_event_id || "",
|
|
1683
|
+
source_field: item.source_field || "",
|
|
1684
|
+
fulltext_event_id: item.source_event_id || item.source_memory_id || item.id,
|
|
1685
|
+
text: item.text,
|
|
1686
|
+
source_text: item.source_text || "",
|
|
1687
|
+
source_excerpt: item.source_excerpt || "",
|
|
1688
|
+
source_file: item.source_file || "",
|
|
1689
|
+
source: item.source,
|
|
1690
|
+
layer: item.layer,
|
|
1691
|
+
event_type: item.event_type,
|
|
1692
|
+
quality_score: item.quality_score,
|
|
1693
|
+
timestamp: item.timestamp,
|
|
1694
|
+
score: Number(item.score.toFixed(4)),
|
|
1695
|
+
score_breakdown: item.score_breakdown || {},
|
|
1696
|
+
reason_tags: Array.isArray(item.reason_tags) ? item.reason_tags : [],
|
|
1697
|
+
explain: {
|
|
1698
|
+
merge_key: item.merge_key,
|
|
1699
|
+
source_memory_id: item.source_memory_id,
|
|
1700
|
+
source_memory_canonical_id: item.source_memory_canonical_id,
|
|
1701
|
+
source_event_id: item.source_event_id || "",
|
|
1702
|
+
source_field: item.source_field || "",
|
|
1703
|
+
channel: item.source,
|
|
1704
|
+
source_file: item.source_file || "",
|
|
1705
|
+
layer: item.layer,
|
|
1706
|
+
score_breakdown: item.score_breakdown || {},
|
|
1707
|
+
reason_tags: Array.isArray(item.reason_tags) ? item.reason_tags : [],
|
|
1708
|
+
},
|
|
1709
|
+
});
|
|
1710
|
+
}
|
|
1711
|
+
}
|
|
1712
|
+
ranked.sort((a, b) => b.score - a.score);
|
|
1713
|
+
if (fusionEnabled && llmModel && llmApiKey && llmBaseUrl && ranked.length > 1) {
|
|
1714
|
+
try {
|
|
1715
|
+
const maxCandidates = Math.max(4, Math.min(20, options.fusion?.maxCandidates ?? 10));
|
|
1716
|
+
const fusion = await requestFusion({
|
|
1717
|
+
query,
|
|
1718
|
+
candidates: ranked.slice(0, maxCandidates).map(item => ({
|
|
1719
|
+
id: item.id,
|
|
1720
|
+
text: item.text,
|
|
1721
|
+
source_excerpt: typeof item.source_excerpt === "string" ? item.source_excerpt : "",
|
|
1722
|
+
source_file: typeof item.source_file === "string" ? item.source_file : "",
|
|
1723
|
+
source_memory_id: typeof item.source_memory_id === "string" ? item.source_memory_id : "",
|
|
1724
|
+
source_memory_canonical_id: typeof item.source_memory_canonical_id === "string" ? item.source_memory_canonical_id : "",
|
|
1725
|
+
source_layer: typeof item.layer === "string" ? item.layer : "",
|
|
1726
|
+
source_event_id: typeof item.source_event_id === "string" ? item.source_event_id : "",
|
|
1727
|
+
source_field: item.source_field === "summary" || item.source_field === "evidence" ? item.source_field : "",
|
|
1728
|
+
source: item.source,
|
|
1729
|
+
event_type: item.event_type,
|
|
1730
|
+
quality_score: item.quality_score,
|
|
1731
|
+
timestamp: item.timestamp,
|
|
1732
|
+
score: item.score,
|
|
1733
|
+
reason_tags: Array.isArray(item.reason_tags) ? item.reason_tags : [],
|
|
1734
|
+
})),
|
|
1735
|
+
llm: {
|
|
1736
|
+
model: llmModel,
|
|
1737
|
+
apiKey: llmApiKey,
|
|
1738
|
+
baseUrl: llmBaseUrl,
|
|
1739
|
+
},
|
|
1740
|
+
});
|
|
1741
|
+
if (fusion && fusion.canonical_answer) {
|
|
1742
|
+
if (!Array.isArray(fusion.evidence_ids) || fusion.evidence_ids.length === 0) {
|
|
1743
|
+
throw new Error("fusion_missing_whitelisted_evidence");
|
|
1744
|
+
}
|
|
1745
|
+
const fulltextFetchHints = (Array.isArray(fusion.need_fulltext_event_ids) ? fusion.need_fulltext_event_ids : [])
|
|
1746
|
+
.map(eventId => {
|
|
1747
|
+
const linked = ranked.find(item => item.source_memory_id === eventId ||
|
|
1748
|
+
item.source_memory_canonical_id === eventId ||
|
|
1749
|
+
item.id === eventId);
|
|
1750
|
+
return {
|
|
1751
|
+
event_id: eventId,
|
|
1752
|
+
source_file: linked?.source_file || "",
|
|
1753
|
+
source_excerpt: linked?.source_excerpt || "",
|
|
1754
|
+
};
|
|
1755
|
+
});
|
|
1756
|
+
const fusedItem = {
|
|
1757
|
+
id: `fusion_${Date.now().toString(36)}`,
|
|
1758
|
+
text: fusion.canonical_answer,
|
|
1759
|
+
source: "llm_fusion",
|
|
1760
|
+
event_type: "fusion",
|
|
1761
|
+
quality_score: Number(fusion.confidence.toFixed(4)),
|
|
1762
|
+
timestamp: new Date().toISOString(),
|
|
1763
|
+
score: Number((Math.max(...ranked.map(item => item.score)) + 1).toFixed(4)),
|
|
1764
|
+
reason_tags: ["llm_fused_authoritative", `evidence:${fusion.evidence_ids.length}`],
|
|
1765
|
+
explain: {
|
|
1766
|
+
channel: "llm_fusion",
|
|
1767
|
+
fused_from: ranked.slice(0, maxCandidates).map(item => item.id),
|
|
1768
|
+
reason_tags: ["llm_fused_authoritative", `evidence:${fusion.evidence_ids.length}`],
|
|
1769
|
+
},
|
|
1770
|
+
fused_coverage_note: fusion.coverage_note || "",
|
|
1771
|
+
fused_facts: fusion.facts,
|
|
1772
|
+
fused_timeline: fusion.timeline || [],
|
|
1773
|
+
fused_entities: fusion.entities || [],
|
|
1774
|
+
fused_decisions: fusion.decisions || [],
|
|
1775
|
+
fused_fixes: fusion.fixes || [],
|
|
1776
|
+
fused_preferences: fusion.preferences || [],
|
|
1777
|
+
fused_risks: fusion.risks || [],
|
|
1778
|
+
fused_action_items: fusion.action_items || [],
|
|
1779
|
+
fused_conflicts: fusion.conflicts,
|
|
1780
|
+
fused_evidence_ids: fusion.evidence_ids,
|
|
1781
|
+
fused_need_fulltext_event_ids: fusion.need_fulltext_event_ids || [],
|
|
1782
|
+
fulltext_fetch_hints: fulltextFetchHints,
|
|
1783
|
+
};
|
|
1784
|
+
const authoritative = options.fusion?.authoritative !== false;
|
|
1785
|
+
if (authoritative) {
|
|
1786
|
+
markHit(Array.isArray(fusion.evidence_ids) ? fusion.evidence_ids : []);
|
|
1787
|
+
return { results: [fusedItem] };
|
|
1788
|
+
}
|
|
1789
|
+
const merged = [fusedItem, ...ranked];
|
|
1790
|
+
markHit([
|
|
1791
|
+
...(Array.isArray(fusion.evidence_ids) ? fusion.evidence_ids : []),
|
|
1792
|
+
...ranked.map(item => item.id),
|
|
1793
|
+
]);
|
|
1794
|
+
return { results: merged.slice(0, Math.max(1, args.topK)) };
|
|
1795
|
+
}
|
|
1796
|
+
}
|
|
1797
|
+
catch (error) {
|
|
1798
|
+
options.logger.warn(`LLM fusion failed, fallback to reranked results: ${error}`);
|
|
1799
|
+
}
|
|
1800
|
+
}
|
|
1801
|
+
const finalRanked = ranked.slice(0, Math.max(1, args.topK));
|
|
1802
|
+
markHit(finalRanked.map(item => item.id));
|
|
1803
|
+
return { results: finalRanked };
|
|
199
1804
|
}
|
|
200
1805
|
async function getHotContext(args) {
|
|
201
1806
|
const limit = Math.max(1, args.limit);
|
|
202
1807
|
const docs = loadAllDocuments();
|
|
203
1808
|
const coreRules = docs.find(doc => doc.source === "CORTEX_RULES.md");
|
|
204
|
-
const
|
|
205
|
-
|
|
1809
|
+
const ruleBudget = Math.max(1, Math.min(6, Math.floor(limit / 3)));
|
|
1810
|
+
const archiveDocs = docs
|
|
1811
|
+
.filter(doc => doc.source === "sessions_archive")
|
|
206
1812
|
.sort((a, b) => (b.timestamp ?? 0) - (a.timestamp ?? 0))
|
|
207
1813
|
.slice(0, limit);
|
|
1814
|
+
const issueFixPairs = docs
|
|
1815
|
+
.filter(doc => doc.source === "sessions_archive" && (doc.eventType === "issue" || doc.eventType === "fix"))
|
|
1816
|
+
.sort((a, b) => (b.timestamp ?? 0) - (a.timestamp ?? 0))
|
|
1817
|
+
.slice(0, 2);
|
|
208
1818
|
const result = [];
|
|
209
1819
|
if (coreRules) {
|
|
210
|
-
|
|
1820
|
+
const selectedRules = extractPrioritizedRuleLines(coreRules.text, ruleBudget);
|
|
1821
|
+
if (selectedRules.length > 0) {
|
|
1822
|
+
result.push({
|
|
1823
|
+
id: `${coreRules.id}.hot`,
|
|
1824
|
+
text: `# Hot Rules\n${selectedRules.map((line, index) => `${index + 1}. ${line}`).join("\n")}`,
|
|
1825
|
+
source: coreRules.source,
|
|
1826
|
+
});
|
|
1827
|
+
}
|
|
211
1828
|
}
|
|
212
|
-
for (const doc of
|
|
1829
|
+
for (const doc of [...issueFixPairs, ...archiveDocs]) {
|
|
213
1830
|
result.push({ id: doc.id, text: doc.text, source: doc.source });
|
|
214
1831
|
}
|
|
215
1832
|
return { context: result.slice(0, limit) };
|
|
@@ -223,6 +1840,25 @@ function createReadStore(options) {
|
|
|
223
1840
|
age_seconds: args.cachedAutoSearch.ageSeconds,
|
|
224
1841
|
};
|
|
225
1842
|
}
|
|
1843
|
+
if (!result.auto_search) {
|
|
1844
|
+
const docs = loadAllDocuments()
|
|
1845
|
+
.filter(doc => doc.source === "sessions_archive" && doc.sessionId === args.sessionId)
|
|
1846
|
+
.sort((a, b) => (b.timestamp ?? 0) - (a.timestamp ?? 0));
|
|
1847
|
+
const latest = docs[0];
|
|
1848
|
+
if (latest && latest.text.trim()) {
|
|
1849
|
+
const autoQuery = latest.text.slice(0, Math.max(20, readTuning.autoContext.queryMaxChars));
|
|
1850
|
+
const light = await searchMemory({
|
|
1851
|
+
query: autoQuery,
|
|
1852
|
+
topK: 3,
|
|
1853
|
+
mode: readTuning.autoContext.lightweightSearch ? "lightweight" : "default",
|
|
1854
|
+
});
|
|
1855
|
+
result.auto_search = {
|
|
1856
|
+
query: autoQuery,
|
|
1857
|
+
results: light.results,
|
|
1858
|
+
age_seconds: 0,
|
|
1859
|
+
};
|
|
1860
|
+
}
|
|
1861
|
+
}
|
|
226
1862
|
if (args.includeHot) {
|
|
227
1863
|
const hot = await getHotContext({ limit: 20 });
|
|
228
1864
|
result.hot_context = hot.context;
|