@gethmy/mcp 2.3.1 → 2.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/active-learning.js +939 -787
- package/dist/lib/api-client.js +2527 -644
- package/dist/lib/auto-session.js +177 -196
- package/dist/lib/cli.js +34954 -128
- package/dist/lib/config.js +235 -201
- package/dist/lib/consolidation.js +374 -289
- package/dist/lib/context-assembly.js +1265 -838
- package/dist/lib/graph-expansion.js +139 -155
- package/dist/lib/http.js +1917 -130
- package/dist/lib/index.js +29525 -5
- package/dist/lib/lifecycle-maintenance.js +663 -79
- package/dist/lib/memory-cleanup.js +1315 -409
- package/dist/lib/onboard.js +2588 -32
- package/dist/lib/prompt-builder.js +438 -445
- package/dist/lib/remote.js +31733 -143
- package/dist/lib/server.js +29388 -3229
- package/dist/lib/skills.js +315 -132
- package/dist/lib/tui/agents.js +128 -107
- package/dist/lib/tui/docs.js +1590 -687
- package/dist/lib/tui/setup.js +5698 -804
- package/dist/lib/tui/theme.js +183 -86
- package/dist/lib/tui/writer.js +1149 -176
- package/package.json +2 -2
- package/src/memory-cleanup.ts +2 -4
- package/dist/lib/__tests__/active-learning.test.js +0 -386
- package/dist/lib/__tests__/agent-performance-profiles.test.js +0 -325
- package/dist/lib/__tests__/auto-session.test.js +0 -661
- package/dist/lib/__tests__/context-assembly.test.js +0 -362
- package/dist/lib/__tests__/graph-expansion.test.js +0 -150
- package/dist/lib/__tests__/integration-memory-crud.test.js +0 -797
- package/dist/lib/__tests__/integration-memory-system.test.js +0 -281
- package/dist/lib/__tests__/lifecycle-maintenance.test.js +0 -207
- package/dist/lib/__tests__/pattern-detection.test.js +0 -295
- package/dist/lib/__tests__/prompt-builder.test.js +0 -418
|
@@ -1,455 +1,1361 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
1
|
+
import { createRequire } from "node:module";
|
|
2
|
+
var __create = Object.create;
|
|
3
|
+
var __getProtoOf = Object.getPrototypeOf;
|
|
4
|
+
var __defProp = Object.defineProperty;
|
|
5
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
7
|
+
var __toESM = (mod, isNodeMode, target) => {
|
|
8
|
+
target = mod != null ? __create(__getProtoOf(mod)) : {};
|
|
9
|
+
const to = isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target;
|
|
10
|
+
for (let key of __getOwnPropNames(mod))
|
|
11
|
+
if (!__hasOwnProp.call(to, key))
|
|
12
|
+
__defProp(to, key, {
|
|
13
|
+
get: () => mod[key],
|
|
14
|
+
enumerable: true
|
|
15
|
+
});
|
|
16
|
+
return to;
|
|
17
|
+
};
|
|
18
|
+
var __commonJS = (cb, mod) => () => (mod || cb((mod = { exports: {} }).exports, mod), mod.exports);
|
|
19
|
+
var __export = (target, all) => {
|
|
20
|
+
for (var name in all)
|
|
21
|
+
__defProp(target, name, {
|
|
22
|
+
get: all[name],
|
|
23
|
+
enumerable: true,
|
|
24
|
+
configurable: true,
|
|
25
|
+
set: (newValue) => all[name] = () => newValue
|
|
26
|
+
});
|
|
27
|
+
};
|
|
28
|
+
var __esm = (fn, res) => () => (fn && (res = fn(fn = 0)), res);
|
|
29
|
+
var __require = /* @__PURE__ */ createRequire(import.meta.url);
|
|
30
|
+
|
|
31
|
+
// ../memory/dist/schema.js
|
|
32
|
+
var init_schema = () => {};
|
|
33
|
+
|
|
34
|
+
// ../memory/dist/constraints.js
|
|
35
|
+
var init_constraints = __esm(() => {
|
|
36
|
+
init_schema();
|
|
37
|
+
});
|
|
38
|
+
// ../memory/dist/client.js
|
|
39
|
+
var init_client = __esm(() => {
|
|
40
|
+
init_constraints();
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
// ../memory/dist/graph-walk.js
|
|
44
|
+
async function discoverRelatedContext(client, startIds, maxDepth = 2, maxEntities = 20, minConfidence = 0.5) {
|
|
45
|
+
const visited = new Set;
|
|
46
|
+
const collectedEntities = [];
|
|
47
|
+
const collectedRelations = [];
|
|
48
|
+
let truncated = false;
|
|
49
|
+
const queue = startIds.map((id) => [id, 0]);
|
|
50
|
+
for (const id of startIds) {
|
|
51
|
+
visited.add(id);
|
|
52
|
+
}
|
|
53
|
+
while (queue.length > 0) {
|
|
54
|
+
const [entityId, depth] = queue.shift();
|
|
55
|
+
if (collectedEntities.length >= maxEntities) {
|
|
56
|
+
truncated = true;
|
|
57
|
+
break;
|
|
58
|
+
}
|
|
59
|
+
if (depth > maxDepth)
|
|
60
|
+
continue;
|
|
45
61
|
try {
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
62
|
+
const entityResult = await client.getMemoryEntity(entityId);
|
|
63
|
+
const entity = entityResult.entity;
|
|
64
|
+
if (entity) {
|
|
65
|
+
collectedEntities.push({
|
|
66
|
+
id: entity.id,
|
|
67
|
+
type: entity.type,
|
|
68
|
+
title: entity.title,
|
|
69
|
+
confidence: entity.confidence ?? 1,
|
|
70
|
+
memory_tier: entity.memory_tier || "reference"
|
|
50
71
|
});
|
|
51
|
-
|
|
52
|
-
|
|
72
|
+
}
|
|
73
|
+
if (depth >= maxDepth)
|
|
74
|
+
continue;
|
|
75
|
+
const related = await client.getRelatedEntities(entityId);
|
|
76
|
+
for (const raw of related.outgoing || []) {
|
|
77
|
+
const rel = raw;
|
|
78
|
+
const relConfidence = rel.confidence ?? 1;
|
|
79
|
+
if (relConfidence < minConfidence)
|
|
80
|
+
continue;
|
|
81
|
+
const target = rel.target;
|
|
82
|
+
const targetId = target?.id ?? rel.target_id;
|
|
83
|
+
if (targetId && !visited.has(targetId)) {
|
|
84
|
+
visited.add(targetId);
|
|
85
|
+
queue.push([targetId, depth + 1]);
|
|
86
|
+
collectedRelations.push({
|
|
87
|
+
id: rel.id,
|
|
88
|
+
source_id: entityId,
|
|
89
|
+
target_id: targetId,
|
|
90
|
+
relation_type: rel.relation_type,
|
|
91
|
+
confidence: relConfidence
|
|
92
|
+
});
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
for (const raw of related.incoming || []) {
|
|
96
|
+
const rel = raw;
|
|
97
|
+
const relConfidence = rel.confidence ?? 1;
|
|
98
|
+
if (relConfidence < minConfidence)
|
|
99
|
+
continue;
|
|
100
|
+
const source = rel.source;
|
|
101
|
+
const sourceId = source?.id ?? rel.source_id;
|
|
102
|
+
if (sourceId && !visited.has(sourceId)) {
|
|
103
|
+
visited.add(sourceId);
|
|
104
|
+
queue.push([sourceId, depth + 1]);
|
|
105
|
+
collectedRelations.push({
|
|
106
|
+
id: rel.id,
|
|
107
|
+
source_id: sourceId,
|
|
108
|
+
target_id: entityId,
|
|
109
|
+
relation_type: rel.relation_type,
|
|
110
|
+
confidence: relConfidence
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
} catch {}
|
|
115
|
+
}
|
|
116
|
+
return {
|
|
117
|
+
entities: collectedEntities,
|
|
118
|
+
relations: collectedRelations,
|
|
119
|
+
depth: maxDepth,
|
|
120
|
+
truncated
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
// ../memory/dist/lifecycle.js
|
|
125
|
+
function computeDecayScore(tier, lastAccessedAt, accessCount) {
|
|
126
|
+
const halfLife = DECAY_HALF_LIVES[tier];
|
|
127
|
+
const now = Date.now();
|
|
128
|
+
let daysSinceAccess = 0;
|
|
129
|
+
if (lastAccessedAt) {
|
|
130
|
+
daysSinceAccess = (now - new Date(lastAccessedAt).getTime()) / (1000 * 60 * 60 * 24);
|
|
131
|
+
}
|
|
132
|
+
const timeDecay = 0.5 ** (daysSinceAccess / halfLife);
|
|
133
|
+
const accessBonus = Math.log10(accessCount + 1) * 0.1;
|
|
134
|
+
const score = Math.min(timeDecay + accessBonus, 1);
|
|
135
|
+
return { score, daysSinceAccess, halfLife, accessBonus };
|
|
136
|
+
}
|
|
137
|
+
function checkPromotion(currentTier, accessCount, confidence, createdAt) {
|
|
138
|
+
const ageDays = (Date.now() - new Date(createdAt).getTime()) / (1000 * 60 * 60 * 24);
|
|
139
|
+
const base = {
|
|
140
|
+
eligible: false,
|
|
141
|
+
targetTier: null,
|
|
142
|
+
reason: null,
|
|
143
|
+
currentTier,
|
|
144
|
+
accessCount,
|
|
145
|
+
confidence,
|
|
146
|
+
ageDays
|
|
147
|
+
};
|
|
148
|
+
if (currentTier === "draft") {
|
|
149
|
+
const rules = PROMOTION_RULES.draftToEpisode;
|
|
150
|
+
if (accessCount >= rules.minAccessCount && confidence >= rules.minConfidence && ageDays >= rules.minAgeDays) {
|
|
151
|
+
return {
|
|
152
|
+
...base,
|
|
153
|
+
eligible: true,
|
|
154
|
+
targetTier: "episode",
|
|
155
|
+
reason: `Accessed ${accessCount} times (≥${rules.minAccessCount}), confidence ${confidence} (≥${rules.minConfidence}), age ${Math.round(ageDays)}d (≥${rules.minAgeDays}d)`
|
|
156
|
+
};
|
|
53
157
|
}
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
158
|
+
}
|
|
159
|
+
if (currentTier === "episode") {
|
|
160
|
+
const rules = PROMOTION_RULES.episodeToReference;
|
|
161
|
+
if (accessCount >= rules.minAccessCount && confidence >= rules.minConfidence && ageDays >= rules.minAgeDays) {
|
|
162
|
+
return {
|
|
163
|
+
...base,
|
|
164
|
+
eligible: true,
|
|
165
|
+
targetTier: "reference",
|
|
166
|
+
reason: `Accessed ${accessCount} times (≥${rules.minAccessCount}), confidence ${confidence} (≥${rules.minConfidence}), age ${Math.round(ageDays)}d (≥${rules.minAgeDays}d)`
|
|
167
|
+
};
|
|
62
168
|
}
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
169
|
+
}
|
|
170
|
+
return base;
|
|
171
|
+
}
|
|
172
|
+
function evaluateLifecycle(entity) {
|
|
173
|
+
const decay = computeDecayScore(entity.memory_tier, entity.last_accessed_at, entity.access_count);
|
|
174
|
+
const promotion = checkPromotion(entity.memory_tier, entity.access_count, entity.confidence, entity.created_at);
|
|
175
|
+
const shouldArchive = entity.confidence < ARCHIVE_THRESHOLD;
|
|
176
|
+
const archiveReason = shouldArchive ? `Confidence ${entity.confidence} below threshold ${ARCHIVE_THRESHOLD}` : undefined;
|
|
177
|
+
const shouldFlagForReview = decay.daysSinceAccess >= STALE_DAYS && entity.access_count < STALE_MIN_ACCESS;
|
|
178
|
+
const reviewReason = shouldFlagForReview ? `Not accessed in ${Math.round(decay.daysSinceAccess)} days with only ${entity.access_count} accesses` : undefined;
|
|
179
|
+
return {
|
|
180
|
+
decay,
|
|
181
|
+
promotion,
|
|
182
|
+
shouldArchive,
|
|
183
|
+
shouldFlagForReview,
|
|
184
|
+
archiveReason,
|
|
185
|
+
reviewReason
|
|
186
|
+
};
|
|
187
|
+
}
|
|
188
|
+
var DECAY_HALF_LIVES, PROMOTION_RULES, ARCHIVE_THRESHOLD = 0.3, STALE_DAYS = 90, STALE_MIN_ACCESS = 3;
|
|
189
|
+
var init_lifecycle = __esm(() => {
|
|
190
|
+
DECAY_HALF_LIVES = {
|
|
191
|
+
draft: 7,
|
|
192
|
+
episode: 30,
|
|
193
|
+
reference: 180
|
|
194
|
+
};
|
|
195
|
+
PROMOTION_RULES = {
|
|
196
|
+
draftToEpisode: {
|
|
197
|
+
minAccessCount: 5,
|
|
198
|
+
minConfidence: 0.8,
|
|
199
|
+
minAgeDays: 1
|
|
200
|
+
},
|
|
201
|
+
episodeToReference: {
|
|
202
|
+
minAccessCount: 10,
|
|
203
|
+
minConfidence: 0.9,
|
|
204
|
+
minAgeDays: 7
|
|
205
|
+
}
|
|
206
|
+
};
|
|
207
|
+
});
|
|
208
|
+
|
|
209
|
+
// ../memory/dist/sync-storage.js
|
|
210
|
+
function parseSyncMarkdown(markdown) {
|
|
211
|
+
const trimmed = markdown.trim();
|
|
212
|
+
let frontmatter = {
|
|
213
|
+
type: "context",
|
|
214
|
+
scope: "project",
|
|
215
|
+
tier: "reference",
|
|
216
|
+
confidence: 1,
|
|
217
|
+
tags: []
|
|
218
|
+
};
|
|
219
|
+
let body = trimmed;
|
|
220
|
+
const fmMatch = trimmed.match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/);
|
|
221
|
+
if (fmMatch) {
|
|
222
|
+
frontmatter = parseSyncYamlFrontmatter(fmMatch[1]);
|
|
223
|
+
body = fmMatch[2].trim();
|
|
224
|
+
}
|
|
225
|
+
let title = "";
|
|
226
|
+
const titleMatch = body.match(/^#\s+(.+)/m);
|
|
227
|
+
if (titleMatch) {
|
|
228
|
+
title = titleMatch[1].trim();
|
|
229
|
+
body = body.replace(/^#\s+.+\n?/, "").trim();
|
|
230
|
+
}
|
|
231
|
+
return { frontmatter, title, content: body };
|
|
232
|
+
}
|
|
233
|
+
function serializeSyncMarkdown(entity) {
|
|
234
|
+
const lines = ["---"];
|
|
235
|
+
lines.push(`id: ${entity.id}`);
|
|
236
|
+
lines.push(`workspace_id: ${entity.workspace_id}`);
|
|
237
|
+
if (entity.project_id) {
|
|
238
|
+
lines.push(`project_id: ${entity.project_id}`);
|
|
239
|
+
}
|
|
240
|
+
lines.push(`type: ${entity.type}`);
|
|
241
|
+
lines.push(`scope: ${entity.scope}`);
|
|
242
|
+
lines.push(`tier: ${entity.memory_tier || "reference"}`);
|
|
243
|
+
lines.push(`confidence: ${entity.confidence}`);
|
|
244
|
+
if (entity.tags.length > 0) {
|
|
245
|
+
lines.push(`tags: [${entity.tags.join(", ")}]`);
|
|
246
|
+
} else {
|
|
247
|
+
lines.push("tags: []");
|
|
248
|
+
}
|
|
249
|
+
if (entity.agent_identifier) {
|
|
250
|
+
lines.push(`agent: ${entity.agent_identifier}`);
|
|
251
|
+
}
|
|
252
|
+
lines.push(`created_at: ${entity.created_at}`);
|
|
253
|
+
lines.push(`updated_at: ${entity.updated_at}`);
|
|
254
|
+
lines.push("---");
|
|
255
|
+
lines.push("");
|
|
256
|
+
lines.push(`# ${entity.title}`);
|
|
257
|
+
lines.push("");
|
|
258
|
+
lines.push(entity.content);
|
|
259
|
+
return lines.join(`
|
|
260
|
+
`);
|
|
261
|
+
}
|
|
262
|
+
function parseSyncYamlFrontmatter(yaml) {
|
|
263
|
+
const result = {
|
|
264
|
+
type: "context",
|
|
265
|
+
scope: "project",
|
|
266
|
+
tier: "reference",
|
|
267
|
+
confidence: 1,
|
|
268
|
+
tags: []
|
|
269
|
+
};
|
|
270
|
+
for (const line of yaml.split(`
|
|
271
|
+
`)) {
|
|
272
|
+
const colonIndex = line.indexOf(":");
|
|
273
|
+
if (colonIndex === -1)
|
|
274
|
+
continue;
|
|
275
|
+
const key = line.slice(0, colonIndex).trim();
|
|
276
|
+
const value = line.slice(colonIndex + 1).trim();
|
|
277
|
+
switch (key) {
|
|
278
|
+
case "id":
|
|
279
|
+
result.id = value;
|
|
280
|
+
break;
|
|
281
|
+
case "workspace_id":
|
|
282
|
+
result.workspace_id = value;
|
|
283
|
+
break;
|
|
284
|
+
case "project_id":
|
|
285
|
+
result.project_id = value;
|
|
286
|
+
break;
|
|
287
|
+
case "type":
|
|
288
|
+
result.type = value;
|
|
289
|
+
break;
|
|
290
|
+
case "scope":
|
|
291
|
+
result.scope = value;
|
|
292
|
+
break;
|
|
293
|
+
case "tier":
|
|
294
|
+
result.tier = value;
|
|
295
|
+
break;
|
|
296
|
+
case "confidence":
|
|
297
|
+
result.confidence = parseFloat(value) || 1;
|
|
298
|
+
break;
|
|
299
|
+
case "tags":
|
|
300
|
+
result.tags = parseYamlArray(value);
|
|
301
|
+
break;
|
|
302
|
+
case "related":
|
|
303
|
+
result.related = parseYamlArray(value);
|
|
304
|
+
break;
|
|
305
|
+
case "agent":
|
|
306
|
+
result.agent = value;
|
|
307
|
+
break;
|
|
308
|
+
case "created_at":
|
|
309
|
+
result.created_at = value;
|
|
310
|
+
break;
|
|
311
|
+
case "updated_at":
|
|
312
|
+
result.updated_at = value;
|
|
313
|
+
break;
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
return result;
|
|
317
|
+
}
|
|
318
|
+
function parseYamlArray(value) {
|
|
319
|
+
const match = value.match(/^\[(.*)]\s*$/);
|
|
320
|
+
if (!match)
|
|
321
|
+
return [];
|
|
322
|
+
return match[1].split(",").map((s) => s.trim()).filter(Boolean);
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
// ../memory/dist/sync.js
|
|
326
|
+
import { createHash } from "node:crypto";
|
|
327
|
+
import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
|
328
|
+
import { join, relative, sep } from "node:path";
|
|
329
|
+
function computeFileHash(content) {
|
|
330
|
+
return `sha256:${createHash("sha256").update(content).digest("hex")}`;
|
|
331
|
+
}
|
|
332
|
+
function slugifyTitle(title) {
|
|
333
|
+
return title.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80);
|
|
334
|
+
}
|
|
335
|
+
function entityToFilename(entity) {
|
|
336
|
+
const slug = slugifyTitle(entity.title);
|
|
337
|
+
const shortId = entity.id.slice(0, 8);
|
|
338
|
+
return `${entity.type}--${slug}--${shortId}.md`;
|
|
339
|
+
}
|
|
340
|
+
function entityToDirectoryPath(entity, memoryDir) {
|
|
341
|
+
const wsDir = join(memoryDir, entity.workspace_id);
|
|
342
|
+
if (entity.scope === "private") {
|
|
343
|
+
return join(wsDir, "_private");
|
|
344
|
+
}
|
|
345
|
+
if (entity.scope === "workspace" || !entity.project_id) {
|
|
346
|
+
return join(wsDir, "_workspace");
|
|
347
|
+
}
|
|
348
|
+
return join(wsDir, entity.project_id);
|
|
349
|
+
}
|
|
350
|
+
function emptySyncState() {
|
|
351
|
+
return { version: 1, lastPullAt: null, entities: {} };
|
|
352
|
+
}
|
|
353
|
+
function loadSyncState(memoryDir) {
|
|
354
|
+
const statePath = join(memoryDir, ".sync-state.json");
|
|
355
|
+
if (!existsSync(statePath))
|
|
356
|
+
return emptySyncState();
|
|
357
|
+
try {
|
|
358
|
+
return JSON.parse(readFileSync(statePath, "utf-8"));
|
|
359
|
+
} catch {
|
|
360
|
+
return emptySyncState();
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
function saveSyncState(memoryDir, state) {
|
|
364
|
+
if (!existsSync(memoryDir)) {
|
|
365
|
+
mkdirSync(memoryDir, { recursive: true });
|
|
366
|
+
}
|
|
367
|
+
writeFileSync(join(memoryDir, ".sync-state.json"), JSON.stringify(state, null, 2));
|
|
368
|
+
}
|
|
369
|
+
function writeEntityFile(entity, memoryDir) {
|
|
370
|
+
const dir = entityToDirectoryPath(entity, memoryDir);
|
|
371
|
+
if (!existsSync(dir))
|
|
372
|
+
mkdirSync(dir, { recursive: true });
|
|
373
|
+
const filename = entityToFilename(entity);
|
|
374
|
+
const filePath = join(dir, filename);
|
|
375
|
+
const markdown = serializeSyncMarkdown(entity);
|
|
376
|
+
writeFileSync(filePath, markdown);
|
|
377
|
+
return relative(memoryDir, filePath);
|
|
378
|
+
}
|
|
379
|
+
function deleteEntityFile(relPath, memoryDir) {
|
|
380
|
+
const absPath = join(memoryDir, relPath);
|
|
381
|
+
if (existsSync(absPath)) {
|
|
382
|
+
rmSync(absPath);
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
function findMarkdownFiles(dir) {
|
|
386
|
+
const results = [];
|
|
387
|
+
if (!existsSync(dir))
|
|
388
|
+
return results;
|
|
389
|
+
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
|
390
|
+
const fullPath = join(dir, entry.name);
|
|
391
|
+
if (entry.isDirectory()) {
|
|
392
|
+
results.push(...findMarkdownFiles(fullPath));
|
|
393
|
+
} else if (entry.isFile() && entry.name.endsWith(".md")) {
|
|
394
|
+
results.push(fullPath);
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
return results;
|
|
398
|
+
}
|
|
399
|
+
async function syncPull(client, config, workspaceId, projectId) {
|
|
400
|
+
const { memoryDir } = config;
|
|
401
|
+
const state = loadSyncState(memoryDir);
|
|
402
|
+
const result = {
|
|
403
|
+
pulled: 0,
|
|
404
|
+
pushed: 0,
|
|
405
|
+
deleted: 0,
|
|
406
|
+
conflicts: 0,
|
|
407
|
+
errors: []
|
|
408
|
+
};
|
|
409
|
+
const allEntities = [];
|
|
410
|
+
let offset = 0;
|
|
411
|
+
const batchSize = 100;
|
|
412
|
+
while (true) {
|
|
413
|
+
try {
|
|
414
|
+
const resp = await client.listMemoryEntities({
|
|
415
|
+
workspace_id: workspaceId,
|
|
416
|
+
project_id: projectId,
|
|
417
|
+
limit: batchSize,
|
|
418
|
+
offset
|
|
419
|
+
});
|
|
420
|
+
const batch = resp.entities;
|
|
421
|
+
allEntities.push(...batch);
|
|
422
|
+
if (batch.length < batchSize)
|
|
423
|
+
break;
|
|
424
|
+
offset += batchSize;
|
|
425
|
+
} catch (err) {
|
|
426
|
+
result.errors.push(`Failed to fetch entities: ${err}`);
|
|
427
|
+
return result;
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
const remoteIds = new Set(allEntities.map((e) => e.id));
|
|
431
|
+
for (const entity of allEntities) {
|
|
432
|
+
const existing = state.entities[entity.id];
|
|
433
|
+
const markdown = serializeSyncMarkdown(entity);
|
|
434
|
+
const hash = computeFileHash(markdown);
|
|
435
|
+
if (!existing) {
|
|
436
|
+
const relPath = writeEntityFile(entity, memoryDir);
|
|
437
|
+
state.entities[entity.id] = {
|
|
438
|
+
filePath: relPath,
|
|
439
|
+
remoteUpdatedAt: entity.updated_at,
|
|
440
|
+
lastSyncedHash: hash
|
|
441
|
+
};
|
|
442
|
+
result.pulled++;
|
|
443
|
+
} else {
|
|
444
|
+
const remoteChanged = entity.updated_at > existing.remoteUpdatedAt;
|
|
445
|
+
if (!remoteChanged)
|
|
446
|
+
continue;
|
|
447
|
+
const absPath = join(memoryDir, existing.filePath);
|
|
448
|
+
let localChanged = false;
|
|
449
|
+
if (existsSync(absPath)) {
|
|
450
|
+
const localContent = readFileSync(absPath, "utf-8");
|
|
451
|
+
const localHash = computeFileHash(localContent);
|
|
452
|
+
localChanged = localHash !== existing.lastSyncedHash;
|
|
453
|
+
}
|
|
454
|
+
if (localChanged) {
|
|
455
|
+
result.conflicts++;
|
|
456
|
+
}
|
|
457
|
+
const newRelPath = writeEntityFile(entity, memoryDir);
|
|
458
|
+
if (existing.filePath !== newRelPath) {
|
|
459
|
+
deleteEntityFile(existing.filePath, memoryDir);
|
|
460
|
+
}
|
|
461
|
+
state.entities[entity.id] = {
|
|
462
|
+
filePath: newRelPath,
|
|
463
|
+
remoteUpdatedAt: entity.updated_at,
|
|
464
|
+
lastSyncedHash: hash
|
|
465
|
+
};
|
|
466
|
+
result.pulled++;
|
|
467
|
+
}
|
|
468
|
+
}
|
|
469
|
+
for (const [entityId, entry] of Object.entries(state.entities)) {
|
|
470
|
+
if (!remoteIds.has(entityId)) {
|
|
471
|
+
deleteEntityFile(entry.filePath, memoryDir);
|
|
472
|
+
delete state.entities[entityId];
|
|
473
|
+
result.deleted++;
|
|
474
|
+
}
|
|
475
|
+
}
|
|
476
|
+
state.lastPullAt = new Date().toISOString();
|
|
477
|
+
saveSyncState(memoryDir, state);
|
|
478
|
+
return result;
|
|
479
|
+
}
|
|
480
|
+
async function syncPush(client, config, workspaceId) {
|
|
481
|
+
const { memoryDir } = config;
|
|
482
|
+
const state = loadSyncState(memoryDir);
|
|
483
|
+
const result = {
|
|
484
|
+
pulled: 0,
|
|
485
|
+
pushed: 0,
|
|
486
|
+
deleted: 0,
|
|
487
|
+
conflicts: 0,
|
|
488
|
+
errors: []
|
|
489
|
+
};
|
|
490
|
+
const mdFiles = findMarkdownFiles(memoryDir);
|
|
491
|
+
for (const absPath of mdFiles) {
|
|
492
|
+
const content = readFileSync(absPath, "utf-8");
|
|
493
|
+
const hash = computeFileHash(content);
|
|
494
|
+
const parsed = parseSyncMarkdown(content);
|
|
495
|
+
if (parsed.frontmatter.id) {
|
|
496
|
+
const entityId = parsed.frontmatter.id;
|
|
497
|
+
const existing = state.entities[entityId];
|
|
498
|
+
if (existing && hash === existing.lastSyncedHash)
|
|
499
|
+
continue;
|
|
500
|
+
try {
|
|
501
|
+
const resp = await client.updateMemoryEntity(entityId, {
|
|
502
|
+
title: parsed.title || "Untitled",
|
|
503
|
+
content: parsed.content,
|
|
504
|
+
type: parsed.frontmatter.type,
|
|
505
|
+
scope: parsed.frontmatter.scope,
|
|
506
|
+
confidence: parsed.frontmatter.confidence,
|
|
507
|
+
tags: parsed.frontmatter.tags
|
|
508
|
+
});
|
|
509
|
+
const updated = resp.entity;
|
|
510
|
+
const newMarkdown = serializeSyncMarkdown(updated);
|
|
511
|
+
writeFileSync(absPath, newMarkdown);
|
|
512
|
+
const relPath = relative(memoryDir, absPath);
|
|
513
|
+
state.entities[entityId] = {
|
|
514
|
+
filePath: relPath,
|
|
515
|
+
remoteUpdatedAt: updated.updated_at,
|
|
516
|
+
lastSyncedHash: computeFileHash(newMarkdown)
|
|
517
|
+
};
|
|
518
|
+
result.pushed++;
|
|
519
|
+
} catch (err) {
|
|
520
|
+
result.errors.push(`Failed to update ${entityId}: ${err}`);
|
|
521
|
+
}
|
|
522
|
+
} else {
|
|
523
|
+
const relPath = relative(memoryDir, absPath);
|
|
524
|
+
const parts = relPath.split(sep);
|
|
525
|
+
const fileWorkspaceId = parts.length >= 2 ? parts[0] : workspaceId;
|
|
526
|
+
let fileProjectId;
|
|
527
|
+
if (parts.length >= 3) {
|
|
528
|
+
const scopeDir = parts[1];
|
|
529
|
+
if (scopeDir !== "_workspace" && scopeDir !== "_private") {
|
|
530
|
+
fileProjectId = scopeDir;
|
|
83
531
|
}
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
532
|
+
}
|
|
533
|
+
let scope = parsed.frontmatter.scope || "project";
|
|
534
|
+
if (parts.length >= 3) {
|
|
535
|
+
const scopeDir = parts[1];
|
|
536
|
+
if (scopeDir === "_private")
|
|
537
|
+
scope = "private";
|
|
538
|
+
else if (scopeDir === "_workspace")
|
|
539
|
+
scope = "workspace";
|
|
540
|
+
}
|
|
541
|
+
try {
|
|
542
|
+
const resp = await client.createMemoryEntity({
|
|
543
|
+
workspace_id: fileWorkspaceId,
|
|
544
|
+
project_id: fileProjectId,
|
|
545
|
+
type: parsed.frontmatter.type,
|
|
546
|
+
scope,
|
|
547
|
+
title: parsed.title || "Untitled",
|
|
548
|
+
content: parsed.content,
|
|
549
|
+
confidence: parsed.frontmatter.confidence,
|
|
550
|
+
tags: parsed.frontmatter.tags,
|
|
551
|
+
agent_identifier: parsed.frontmatter.agent
|
|
552
|
+
});
|
|
553
|
+
const created = resp.entity;
|
|
554
|
+
const dir = entityToDirectoryPath(created, memoryDir);
|
|
555
|
+
if (!existsSync(dir))
|
|
556
|
+
mkdirSync(dir, { recursive: true });
|
|
557
|
+
const newFilename = entityToFilename(created);
|
|
558
|
+
const newAbsPath = join(dir, newFilename);
|
|
559
|
+
const newMarkdown = serializeSyncMarkdown(created);
|
|
560
|
+
writeFileSync(newAbsPath, newMarkdown);
|
|
561
|
+
if (absPath !== newAbsPath && existsSync(absPath)) {
|
|
562
|
+
rmSync(absPath);
|
|
89
563
|
}
|
|
564
|
+
const newRelPath = relative(memoryDir, newAbsPath);
|
|
565
|
+
state.entities[created.id] = {
|
|
566
|
+
filePath: newRelPath,
|
|
567
|
+
remoteUpdatedAt: created.updated_at,
|
|
568
|
+
lastSyncedHash: computeFileHash(newMarkdown)
|
|
569
|
+
};
|
|
570
|
+
result.pushed++;
|
|
571
|
+
} catch (err) {
|
|
572
|
+
result.errors.push(`Failed to create entity from ${relPath}: ${err}`);
|
|
573
|
+
}
|
|
574
|
+
}
|
|
575
|
+
}
|
|
576
|
+
saveSyncState(memoryDir, state);
|
|
577
|
+
return result;
|
|
578
|
+
}
|
|
579
|
+
async function syncFull(client, config, workspaceId, projectId) {
|
|
580
|
+
const pullResult = await syncPull(client, config, workspaceId, projectId);
|
|
581
|
+
const pushResult = await syncPush(client, config, workspaceId);
|
|
582
|
+
return {
|
|
583
|
+
pulled: pullResult.pulled,
|
|
584
|
+
pushed: pushResult.pushed,
|
|
585
|
+
deleted: pullResult.deleted,
|
|
586
|
+
conflicts: pullResult.conflicts,
|
|
587
|
+
errors: [...pullResult.errors, ...pushResult.errors]
|
|
588
|
+
};
|
|
589
|
+
}
|
|
590
|
+
var init_sync = () => {};
|
|
591
|
+
|
|
592
|
+
// ../memory/dist/index.js
|
|
593
|
+
var init_dist = __esm(() => {
|
|
594
|
+
init_client();
|
|
595
|
+
init_constraints();
|
|
596
|
+
init_lifecycle();
|
|
597
|
+
init_schema();
|
|
598
|
+
init_sync();
|
|
599
|
+
});
|
|
600
|
+
|
|
601
|
+
// src/graph-expansion.ts
|
|
602
|
+
async function autoExpandGraph(client, entityId, title, content, _tags, workspaceId, projectId, maxRelations = 5) {
|
|
603
|
+
try {
|
|
604
|
+
const contentSnippet = content.slice(0, 200).trim();
|
|
605
|
+
const query = [title, contentSnippet].filter(Boolean).join(" ");
|
|
606
|
+
let candidates = [];
|
|
607
|
+
const { entities } = await client.searchMemoryEntities(workspaceId, query, {
|
|
608
|
+
project_id: projectId,
|
|
609
|
+
limit: 20
|
|
610
|
+
});
|
|
611
|
+
candidates = entities.filter((e) => e.id !== entityId).slice(0, maxRelations);
|
|
612
|
+
if (candidates.length === 0) {
|
|
613
|
+
await new Promise((resolve) => setTimeout(resolve, 2000));
|
|
614
|
+
const retry = await client.searchMemoryEntities(workspaceId, query, {
|
|
615
|
+
project_id: projectId,
|
|
616
|
+
limit: 20
|
|
617
|
+
});
|
|
618
|
+
candidates = retry.entities.filter((e) => e.id !== entityId).slice(0, maxRelations);
|
|
619
|
+
}
|
|
620
|
+
let relationsCreated = 0;
|
|
621
|
+
for (const candidate of candidates) {
|
|
622
|
+
try {
|
|
623
|
+
await client.createMemoryRelation({
|
|
624
|
+
source_id: entityId,
|
|
625
|
+
target_id: candidate.id,
|
|
626
|
+
relation_type: "relates_to",
|
|
627
|
+
confidence: 0.6
|
|
628
|
+
});
|
|
629
|
+
relationsCreated++;
|
|
630
|
+
} catch (err) {
|
|
631
|
+
const status = err?.status;
|
|
632
|
+
if (status !== 409) {}
|
|
633
|
+
}
|
|
90
634
|
}
|
|
91
|
-
|
|
92
|
-
|
|
635
|
+
return { relationsCreated };
|
|
636
|
+
} catch {
|
|
637
|
+
return { relationsCreated: 0 };
|
|
638
|
+
}
|
|
639
|
+
}
|
|
640
|
+
async function findSimilarEntities(client, title, content, workspaceId, options) {
|
|
641
|
+
const contentSnippet = content.slice(0, 200).trim();
|
|
642
|
+
const query = [title, contentSnippet].filter(Boolean).join(" ");
|
|
643
|
+
try {
|
|
644
|
+
const { entities } = await client.searchMemoryEntities(workspaceId, query, {
|
|
645
|
+
project_id: options?.projectId,
|
|
646
|
+
limit: options?.limit ?? 20,
|
|
647
|
+
type: options?.type
|
|
648
|
+
});
|
|
649
|
+
const minScore = options?.minRrfScore ?? 0;
|
|
650
|
+
const excludeSet = new Set(options?.excludeIds || []);
|
|
651
|
+
return entities.filter((e) => {
|
|
652
|
+
if (excludeSet.has(e.id))
|
|
653
|
+
return false;
|
|
654
|
+
if (minScore > 0 && (e.rrf_score ?? 0) < minScore)
|
|
655
|
+
return false;
|
|
656
|
+
return true;
|
|
657
|
+
});
|
|
658
|
+
} catch {
|
|
659
|
+
return [];
|
|
660
|
+
}
|
|
661
|
+
}
|
|
662
|
+
var CAUSAL_LOOKUP = [
|
|
663
|
+
{
|
|
664
|
+
sourceType: "error",
|
|
665
|
+
targetType: "solution",
|
|
666
|
+
relation: "resolved_by",
|
|
667
|
+
direction: "forward"
|
|
668
|
+
},
|
|
669
|
+
{
|
|
670
|
+
sourceType: "solution",
|
|
671
|
+
targetType: "error",
|
|
672
|
+
relation: "resolved_by",
|
|
673
|
+
direction: "reverse"
|
|
674
|
+
},
|
|
675
|
+
{
|
|
676
|
+
sourceType: "lesson",
|
|
677
|
+
targetType: "error",
|
|
678
|
+
relation: "learned_from",
|
|
679
|
+
direction: "forward"
|
|
680
|
+
}
|
|
681
|
+
];
|
|
682
|
+
async function linkCrossTypeNeighbors(client, entityId, entityType, title, content, workspaceId, projectId) {
|
|
683
|
+
const rules = CAUSAL_LOOKUP.filter((r) => r.sourceType === entityType);
|
|
684
|
+
if (rules.length === 0)
|
|
685
|
+
return { relationsCreated: 0 };
|
|
686
|
+
let relationsCreated = 0;
|
|
687
|
+
for (const rule of rules) {
|
|
688
|
+
try {
|
|
689
|
+
const matches = await findSimilarEntities(client, title, content, workspaceId, {
|
|
690
|
+
projectId,
|
|
691
|
+
limit: 10,
|
|
692
|
+
minRrfScore: 0.04,
|
|
693
|
+
excludeIds: [entityId],
|
|
694
|
+
type: rule.targetType
|
|
695
|
+
});
|
|
696
|
+
for (const match of matches.slice(0, 3)) {
|
|
697
|
+
const sourceId = rule.direction === "forward" ? entityId : match.id;
|
|
698
|
+
const targetId = rule.direction === "forward" ? match.id : entityId;
|
|
93
699
|
try {
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
700
|
+
await client.createMemoryRelation({
|
|
701
|
+
source_id: sourceId,
|
|
702
|
+
target_id: targetId,
|
|
703
|
+
relation_type: rule.relation,
|
|
704
|
+
confidence: 0.65
|
|
705
|
+
});
|
|
706
|
+
relationsCreated++;
|
|
707
|
+
} catch {}
|
|
708
|
+
}
|
|
709
|
+
} catch {}
|
|
710
|
+
}
|
|
711
|
+
return { relationsCreated };
|
|
712
|
+
}
|
|
713
|
+
|
|
714
|
+
// src/consolidation.ts
|
|
715
|
+
async function consolidateMemories(client, workspaceId, projectId, options) {
|
|
716
|
+
const dryRun = options?.dryRun !== false;
|
|
717
|
+
const minClusterSize = options?.minClusterSize ?? 3;
|
|
718
|
+
const result = {
|
|
719
|
+
consolidated: 0,
|
|
720
|
+
clustersFound: 0,
|
|
721
|
+
entitiesProcessed: 0,
|
|
722
|
+
details: []
|
|
723
|
+
};
|
|
724
|
+
const listResult = await client.listMemoryEntities({
|
|
725
|
+
workspace_id: workspaceId,
|
|
726
|
+
project_id: projectId,
|
|
727
|
+
limit: 100
|
|
728
|
+
});
|
|
729
|
+
const allEntities = (listResult.entities || []).filter((e) => e.memory_tier === "draft" || e.memory_tier === "episode");
|
|
730
|
+
result.entitiesProcessed = allEntities.length;
|
|
731
|
+
if (allEntities.length < minClusterSize)
|
|
732
|
+
return result;
|
|
733
|
+
const typeGroups = new Map;
|
|
734
|
+
for (const entity of allEntities) {
|
|
735
|
+
const group = typeGroups.get(entity.type) || [];
|
|
736
|
+
group.push(entity);
|
|
737
|
+
typeGroups.set(entity.type, group);
|
|
738
|
+
}
|
|
739
|
+
for (const [type, entities] of typeGroups) {
|
|
740
|
+
if (entities.length < minClusterSize)
|
|
741
|
+
continue;
|
|
742
|
+
const clustered = new Set;
|
|
743
|
+
const clusters = [];
|
|
744
|
+
for (const entity of entities) {
|
|
745
|
+
if (clustered.has(entity.id))
|
|
746
|
+
continue;
|
|
747
|
+
const similar = await findSimilarEntities(client, entity.title, entity.content, workspaceId, {
|
|
748
|
+
projectId,
|
|
749
|
+
limit: 20,
|
|
750
|
+
minRrfScore: 0.01,
|
|
751
|
+
excludeIds: [...clustered]
|
|
752
|
+
});
|
|
753
|
+
const entityIdSet = new Set(entities.map((e) => e.id));
|
|
754
|
+
const clusterMembers = similar.filter((s) => entityIdSet.has(s.id) && !clustered.has(s.id) && s.id !== entity.id && s.type === type);
|
|
755
|
+
if (clusterMembers.length >= minClusterSize - 1) {
|
|
756
|
+
const cluster = [
|
|
757
|
+
entity,
|
|
758
|
+
...clusterMembers.slice(0, 5).map((s) => {
|
|
759
|
+
return entities.find((e) => e.id === s.id) || entity;
|
|
760
|
+
})
|
|
761
|
+
];
|
|
762
|
+
const uniqueCluster = [];
|
|
763
|
+
const seen = new Set;
|
|
764
|
+
for (const member of cluster) {
|
|
765
|
+
if (!seen.has(member.id)) {
|
|
766
|
+
seen.add(member.id);
|
|
767
|
+
uniqueCluster.push(member);
|
|
768
|
+
}
|
|
107
769
|
}
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
770
|
+
if (uniqueCluster.length >= minClusterSize) {
|
|
771
|
+
clusters.push(uniqueCluster);
|
|
772
|
+
for (const member of uniqueCluster) {
|
|
773
|
+
clustered.add(member.id);
|
|
774
|
+
}
|
|
113
775
|
}
|
|
776
|
+
}
|
|
114
777
|
}
|
|
115
|
-
|
|
116
|
-
|
|
778
|
+
for (const cluster of clusters) {
|
|
779
|
+
result.clustersFound++;
|
|
780
|
+
const mergedTitle = deriveClusterTitle(cluster, type);
|
|
781
|
+
const memberTitles = cluster.map((e) => e.title);
|
|
782
|
+
const mergedContent = synthesizeClusterContent(cluster, type);
|
|
783
|
+
const maxConfidence = Math.max(...cluster.map((e) => e.confidence));
|
|
784
|
+
const allTags = [...new Set(cluster.flatMap((e) => e.tags || []))];
|
|
785
|
+
const detail = {
|
|
786
|
+
clusterSize: cluster.length,
|
|
787
|
+
mergedTitle,
|
|
788
|
+
memberTitles
|
|
789
|
+
};
|
|
790
|
+
if (!dryRun) {
|
|
117
791
|
try {
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
report.summary.actionsTaken += report.steps.orphans.removed;
|
|
792
|
+
const createResult = await client.createMemoryEntity({
|
|
793
|
+
workspace_id: workspaceId,
|
|
794
|
+
project_id: projectId,
|
|
795
|
+
type,
|
|
796
|
+
scope: "project",
|
|
797
|
+
memory_tier: "reference",
|
|
798
|
+
title: mergedTitle,
|
|
799
|
+
content: mergedContent,
|
|
800
|
+
confidence: maxConfidence,
|
|
801
|
+
tags: [...allTags.slice(0, 15), "consolidated"],
|
|
802
|
+
metadata: {
|
|
803
|
+
source: "consolidation",
|
|
804
|
+
member_ids: cluster.map((e) => e.id),
|
|
805
|
+
consolidated_at: new Date().toISOString()
|
|
133
806
|
}
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
807
|
+
});
|
|
808
|
+
const newEntity = createResult.entity;
|
|
809
|
+
if (newEntity?.id) {
|
|
810
|
+
detail.entityId = newEntity.id;
|
|
811
|
+
for (const member of cluster) {
|
|
812
|
+
try {
|
|
813
|
+
await client.createMemoryRelation({
|
|
814
|
+
source_id: member.id,
|
|
815
|
+
target_id: newEntity.id,
|
|
816
|
+
relation_type: "part_of",
|
|
817
|
+
confidence: 0.8
|
|
818
|
+
});
|
|
819
|
+
} catch {}
|
|
820
|
+
}
|
|
821
|
+
for (const member of cluster) {
|
|
822
|
+
try {
|
|
823
|
+
const newConf = Math.max(member.confidence - 0.3, 0.1);
|
|
824
|
+
await client.updateMemoryEntity(member.id, {
|
|
825
|
+
confidence: newConf,
|
|
826
|
+
metadata: {
|
|
827
|
+
consolidated_into: newEntity.id,
|
|
828
|
+
original_confidence: member.confidence
|
|
829
|
+
}
|
|
830
|
+
});
|
|
831
|
+
} catch {}
|
|
832
|
+
}
|
|
833
|
+
result.consolidated++;
|
|
834
|
+
}
|
|
835
|
+
} catch {}
|
|
836
|
+
} else {
|
|
837
|
+
result.consolidated++;
|
|
838
|
+
}
|
|
839
|
+
result.details.push(detail);
|
|
840
|
+
}
|
|
841
|
+
}
|
|
842
|
+
return result;
|
|
843
|
+
}
|
|
844
|
+
function synthesizeClusterContent(cluster, type) {
|
|
845
|
+
const SKIP_PATTERNS = [
|
|
846
|
+
/^##\s/,
|
|
847
|
+
/^Agent:/,
|
|
848
|
+
/^Duration:/,
|
|
849
|
+
/^Labels:/,
|
|
850
|
+
/^Progress:/,
|
|
851
|
+
/^Session status:/,
|
|
852
|
+
/^Completed at/,
|
|
853
|
+
/^Final state:/,
|
|
854
|
+
/^Related:/,
|
|
855
|
+
/^When working on:/,
|
|
856
|
+
/^\d+\.\s+.+\(\d+%,\s*\+\d+%\)/,
|
|
857
|
+
/^Last updated:/,
|
|
858
|
+
/^Recurring pattern:/,
|
|
859
|
+
/^Consolidated from/
|
|
860
|
+
];
|
|
861
|
+
const seenLines = new Set;
|
|
862
|
+
const knowledgeLines = [];
|
|
863
|
+
for (const entity of cluster) {
|
|
864
|
+
const lines = entity.content.split(`
|
|
865
|
+
`).map((l) => l.trim());
|
|
866
|
+
for (const line of lines) {
|
|
867
|
+
if (!line || line.length < 20)
|
|
868
|
+
continue;
|
|
869
|
+
if (SKIP_PATTERNS.some((p) => p.test(line)))
|
|
870
|
+
continue;
|
|
871
|
+
const normalized = line.toLowerCase().replace(/[*_`#[\]]/g, "").trim();
|
|
872
|
+
if (seenLines.has(normalized))
|
|
873
|
+
continue;
|
|
874
|
+
seenLines.add(normalized);
|
|
875
|
+
knowledgeLines.push(line);
|
|
876
|
+
}
|
|
877
|
+
}
|
|
878
|
+
if (knowledgeLines.length === 0) {
|
|
879
|
+
return `${cluster.length} related ${type} entities consolidated. Original titles:
|
|
880
|
+
${cluster.map((e) => `- ${e.title}`).join(`
|
|
881
|
+
`)}`;
|
|
882
|
+
}
|
|
883
|
+
const MAX_CHARS = 1600;
|
|
884
|
+
const result = [
|
|
885
|
+
`Consolidated knowledge from ${cluster.length} ${type} entities:
|
|
886
|
+
`
|
|
887
|
+
];
|
|
888
|
+
let charCount = result[0].length;
|
|
889
|
+
for (const line of knowledgeLines) {
|
|
890
|
+
if (charCount + line.length + 3 > MAX_CHARS)
|
|
891
|
+
break;
|
|
892
|
+
result.push(`- ${line}`);
|
|
893
|
+
charCount += line.length + 3;
|
|
894
|
+
}
|
|
895
|
+
return result.join(`
|
|
896
|
+
`);
|
|
897
|
+
}
|
|
898
|
+
function deriveClusterTitle(cluster, type) {
|
|
899
|
+
const stopWords = new Set([
|
|
900
|
+
"the",
|
|
901
|
+
"a",
|
|
902
|
+
"an",
|
|
903
|
+
"is",
|
|
904
|
+
"are",
|
|
905
|
+
"was",
|
|
906
|
+
"were",
|
|
907
|
+
"be",
|
|
908
|
+
"been",
|
|
909
|
+
"being",
|
|
910
|
+
"have",
|
|
911
|
+
"has",
|
|
912
|
+
"had",
|
|
913
|
+
"do",
|
|
914
|
+
"does",
|
|
915
|
+
"did",
|
|
916
|
+
"will",
|
|
917
|
+
"shall",
|
|
918
|
+
"would",
|
|
919
|
+
"should",
|
|
920
|
+
"may",
|
|
921
|
+
"might",
|
|
922
|
+
"can",
|
|
923
|
+
"could",
|
|
924
|
+
"of",
|
|
925
|
+
"in",
|
|
926
|
+
"to",
|
|
927
|
+
"for",
|
|
928
|
+
"with",
|
|
929
|
+
"on",
|
|
930
|
+
"at",
|
|
931
|
+
"from",
|
|
932
|
+
"by",
|
|
933
|
+
"and",
|
|
934
|
+
"or",
|
|
935
|
+
"but",
|
|
936
|
+
"not",
|
|
937
|
+
"session",
|
|
938
|
+
"blocker",
|
|
939
|
+
"pattern",
|
|
940
|
+
"solution",
|
|
941
|
+
"error",
|
|
942
|
+
"task",
|
|
943
|
+
"mid-session"
|
|
944
|
+
]);
|
|
945
|
+
const wordCounts = new Map;
|
|
946
|
+
for (const entity of cluster) {
|
|
947
|
+
const words = entity.title.toLowerCase().split(/\W+/).filter((w) => w.length > 2 && !stopWords.has(w));
|
|
948
|
+
for (const word of words) {
|
|
949
|
+
wordCounts.set(word, (wordCounts.get(word) || 0) + 1);
|
|
950
|
+
}
|
|
951
|
+
}
|
|
952
|
+
const topWords = [...wordCounts.entries()].sort((a, b) => b[1] - a[1]).slice(0, 4).map(([word]) => word[0].toUpperCase() + word.slice(1));
|
|
953
|
+
const suffix = topWords.length > 0 ? topWords.join(" / ") : "Various";
|
|
954
|
+
return `${type[0].toUpperCase() + type.slice(1)}: ${suffix}`;
|
|
955
|
+
}
|
|
956
|
+
|
|
957
|
+
// src/memory-cleanup.ts
|
|
958
|
+
init_dist();
|
|
959
|
+
var ALL_STEPS = [
|
|
960
|
+
"prune",
|
|
961
|
+
"consolidate",
|
|
962
|
+
"orphans",
|
|
963
|
+
"duplicates",
|
|
964
|
+
"backfill"
|
|
965
|
+
];
|
|
966
|
+
var MS_PER_DAY = 1000 * 60 * 60 * 24;
|
|
967
|
+
var MAX_ENTITIES_FETCH = 200;
|
|
968
|
+
var DUPLICATE_SIMILARITY_THRESHOLD = 0.85;
|
|
969
|
+
var CONCURRENCY_LIMIT = 5;
|
|
970
|
+
async function runMemoryCleanup(client2, workspaceId, projectId, options) {
|
|
971
|
+
const dryRun = options?.dryRun !== false;
|
|
972
|
+
const steps = options?.steps ?? ALL_STEPS;
|
|
973
|
+
const maxAgeDays = options?.maxAgeDays ?? 30;
|
|
974
|
+
const minClusterSize = options?.minClusterSize ?? 3;
|
|
975
|
+
const orphanAgeDays = options?.orphanAgeDays ?? 14;
|
|
976
|
+
const report = {
|
|
977
|
+
success: true,
|
|
978
|
+
dryRun,
|
|
979
|
+
timestamp: new Date().toISOString(),
|
|
980
|
+
workspace: { id: workspaceId, projectId },
|
|
981
|
+
summary: { totalEntities: 0, issuesFound: 0, actionsTaken: 0 },
|
|
982
|
+
steps: {},
|
|
983
|
+
errors: [],
|
|
984
|
+
healthReport: ""
|
|
985
|
+
};
|
|
986
|
+
let entities = [];
|
|
987
|
+
try {
|
|
988
|
+
const listResult = await client2.listMemoryEntities({
|
|
989
|
+
workspace_id: workspaceId,
|
|
990
|
+
project_id: projectId,
|
|
991
|
+
limit: MAX_ENTITIES_FETCH
|
|
992
|
+
});
|
|
993
|
+
entities = listResult.entities || [];
|
|
994
|
+
report.summary.totalEntities = entities.length;
|
|
995
|
+
} catch (err) {
|
|
996
|
+
report.errors.push({
|
|
997
|
+
step: "init",
|
|
998
|
+
message: `Failed to fetch entities: ${err.message}`
|
|
999
|
+
});
|
|
1000
|
+
report.success = false;
|
|
1001
|
+
report.healthReport = generateHealthReport(report);
|
|
1002
|
+
return report;
|
|
1003
|
+
}
|
|
1004
|
+
if (steps.includes("prune")) {
|
|
1005
|
+
try {
|
|
1006
|
+
report.steps.prune = runPruneStep(entities, maxAgeDays);
|
|
1007
|
+
if (!dryRun) {
|
|
1008
|
+
for (const item of report.steps.prune.items) {
|
|
1009
|
+
try {
|
|
1010
|
+
await client2.deleteMemoryEntity(item.id);
|
|
1011
|
+
report.steps.prune.pruned++;
|
|
1012
|
+
} catch (err) {
|
|
137
1013
|
report.errors.push({
|
|
138
|
-
|
|
139
|
-
|
|
1014
|
+
step: "prune",
|
|
1015
|
+
message: `Failed to delete ${item.id}: ${err.message}`
|
|
140
1016
|
});
|
|
1017
|
+
}
|
|
141
1018
|
}
|
|
1019
|
+
report.summary.actionsTaken += report.steps.prune.pruned;
|
|
1020
|
+
}
|
|
1021
|
+
report.summary.issuesFound += report.steps.prune.staleDraftsFound;
|
|
1022
|
+
} catch (err) {
|
|
1023
|
+
report.errors.push({
|
|
1024
|
+
step: "prune",
|
|
1025
|
+
message: err.message
|
|
1026
|
+
});
|
|
142
1027
|
}
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
1028
|
+
}
|
|
1029
|
+
if (steps.includes("consolidate")) {
|
|
1030
|
+
try {
|
|
1031
|
+
const result = await consolidateMemories(client2, workspaceId, projectId, {
|
|
1032
|
+
dryRun,
|
|
1033
|
+
minClusterSize
|
|
1034
|
+
});
|
|
1035
|
+
report.steps.consolidate = {
|
|
1036
|
+
clustersFound: result.clustersFound,
|
|
1037
|
+
entitiesProcessed: result.entitiesProcessed,
|
|
1038
|
+
consolidated: result.consolidated,
|
|
1039
|
+
details: result.details
|
|
1040
|
+
};
|
|
1041
|
+
report.summary.issuesFound += result.clustersFound;
|
|
1042
|
+
if (!dryRun)
|
|
1043
|
+
report.summary.actionsTaken += result.consolidated;
|
|
1044
|
+
} catch (err) {
|
|
1045
|
+
report.errors.push({
|
|
1046
|
+
step: "consolidate",
|
|
1047
|
+
message: err.message
|
|
1048
|
+
});
|
|
1049
|
+
}
|
|
1050
|
+
}
|
|
1051
|
+
if (steps.includes("orphans")) {
|
|
1052
|
+
try {
|
|
1053
|
+
report.steps.orphans = await runOrphanStep(client2, entities, orphanAgeDays);
|
|
1054
|
+
if (!dryRun) {
|
|
1055
|
+
for (const item of report.steps.orphans.items) {
|
|
1056
|
+
try {
|
|
1057
|
+
await client2.deleteMemoryEntity(item.id);
|
|
1058
|
+
report.steps.orphans.removed++;
|
|
1059
|
+
} catch (err) {
|
|
165
1060
|
report.errors.push({
|
|
166
|
-
|
|
167
|
-
|
|
1061
|
+
step: "orphans",
|
|
1062
|
+
message: `Failed to delete ${item.id}: ${err.message}`
|
|
168
1063
|
});
|
|
1064
|
+
}
|
|
169
1065
|
}
|
|
1066
|
+
report.summary.actionsTaken += report.steps.orphans.removed;
|
|
1067
|
+
}
|
|
1068
|
+
report.summary.issuesFound += report.steps.orphans.orphansFound;
|
|
1069
|
+
} catch (err) {
|
|
1070
|
+
report.errors.push({
|
|
1071
|
+
step: "orphans",
|
|
1072
|
+
message: err.message
|
|
1073
|
+
});
|
|
170
1074
|
}
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
}
|
|
182
|
-
else {
|
|
183
|
-
const result = await client.backfillEmbeddings(workspaceId);
|
|
184
|
-
report.steps.backfill = {
|
|
185
|
-
processed: result.processed,
|
|
186
|
-
remaining: result.remaining,
|
|
187
|
-
errors: result.errors || [],
|
|
188
|
-
};
|
|
189
|
-
report.summary.actionsTaken += result.processed;
|
|
190
|
-
}
|
|
191
|
-
}
|
|
192
|
-
catch (err) {
|
|
1075
|
+
}
|
|
1076
|
+
if (steps.includes("duplicates")) {
|
|
1077
|
+
try {
|
|
1078
|
+
report.steps.duplicates = await runDuplicateStep(client2, entities, workspaceId, projectId);
|
|
1079
|
+
if (!dryRun) {
|
|
1080
|
+
for (const pair of report.steps.duplicates.pairs) {
|
|
1081
|
+
try {
|
|
1082
|
+
await client2.deleteMemoryEntity(pair.removeId);
|
|
1083
|
+
report.steps.duplicates.resolved++;
|
|
1084
|
+
} catch (err) {
|
|
193
1085
|
report.errors.push({
|
|
194
|
-
|
|
195
|
-
|
|
1086
|
+
step: "duplicates",
|
|
1087
|
+
message: `Failed to delete ${pair.removeId}: ${err.message}`
|
|
196
1088
|
});
|
|
1089
|
+
}
|
|
197
1090
|
}
|
|
1091
|
+
report.summary.actionsTaken += report.steps.duplicates.resolved;
|
|
1092
|
+
}
|
|
1093
|
+
report.summary.issuesFound += report.steps.duplicates.duplicatePairsFound;
|
|
1094
|
+
} catch (err) {
|
|
1095
|
+
report.errors.push({
|
|
1096
|
+
step: "duplicates",
|
|
1097
|
+
message: err.message
|
|
1098
|
+
});
|
|
198
1099
|
}
|
|
199
|
-
|
|
200
|
-
|
|
1100
|
+
}
|
|
1101
|
+
if (steps.includes("backfill")) {
|
|
1102
|
+
try {
|
|
1103
|
+
if (dryRun) {
|
|
1104
|
+
report.steps.backfill = {
|
|
1105
|
+
processed: 0,
|
|
1106
|
+
remaining: -1,
|
|
1107
|
+
errors: []
|
|
1108
|
+
};
|
|
1109
|
+
} else {
|
|
1110
|
+
const result = await client2.backfillEmbeddings(workspaceId);
|
|
1111
|
+
report.steps.backfill = {
|
|
1112
|
+
processed: result.processed,
|
|
1113
|
+
remaining: result.remaining,
|
|
1114
|
+
errors: result.errors || []
|
|
1115
|
+
};
|
|
1116
|
+
report.summary.actionsTaken += result.processed;
|
|
1117
|
+
}
|
|
1118
|
+
} catch (err) {
|
|
1119
|
+
report.errors.push({
|
|
1120
|
+
step: "backfill",
|
|
1121
|
+
message: err.message
|
|
1122
|
+
});
|
|
1123
|
+
}
|
|
1124
|
+
}
|
|
1125
|
+
report.healthReport = generateHealthReport(report);
|
|
1126
|
+
return report;
|
|
201
1127
|
}
|
|
202
|
-
// ---------------------------------------------------------------------------
|
|
203
|
-
// Step implementations
|
|
204
|
-
// ---------------------------------------------------------------------------
|
|
205
1128
|
function runPruneStep(entities, maxAgeDays) {
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
});
|
|
220
|
-
}
|
|
221
|
-
return { staleDraftsFound: stale.length, pruned: 0, items: stale };
|
|
222
|
-
}
|
|
223
|
-
async function runOrphanStep(client, entities, orphanAgeDays) {
|
|
224
|
-
const now = Date.now();
|
|
225
|
-
const result = { orphansFound: 0, removed: 0, items: [] };
|
|
226
|
-
// Pre-filter: only check entities that look like orphan candidates
|
|
227
|
-
const candidates = entities.filter((e) => {
|
|
228
|
-
if (e.memory_tier === "reference")
|
|
229
|
-
return false;
|
|
230
|
-
if (e.access_count >= 2)
|
|
231
|
-
return false;
|
|
232
|
-
const ageDays = (now - new Date(e.created_at).getTime()) / MS_PER_DAY;
|
|
233
|
-
return ageDays >= orphanAgeDays;
|
|
1129
|
+
const now = Date.now();
|
|
1130
|
+
const drafts = entities.filter((e) => e.memory_tier === "draft");
|
|
1131
|
+
const stale = [];
|
|
1132
|
+
for (const entity of drafts) {
|
|
1133
|
+
const ageDays = (now - new Date(entity.created_at).getTime()) / MS_PER_DAY;
|
|
1134
|
+
if (ageDays < maxAgeDays)
|
|
1135
|
+
continue;
|
|
1136
|
+
const lifecycle2 = evaluateLifecycle(entity);
|
|
1137
|
+
stale.push({
|
|
1138
|
+
id: entity.id,
|
|
1139
|
+
title: entity.title,
|
|
1140
|
+
ageDays: Math.round(ageDays),
|
|
1141
|
+
decayScore: Math.round(lifecycle2.decay.score * 100) / 100
|
|
234
1142
|
});
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
1143
|
+
}
|
|
1144
|
+
return { staleDraftsFound: stale.length, pruned: 0, items: stale };
|
|
1145
|
+
}
|
|
1146
|
+
async function runOrphanStep(client2, entities, orphanAgeDays) {
|
|
1147
|
+
const now = Date.now();
|
|
1148
|
+
const result = { orphansFound: 0, removed: 0, items: [] };
|
|
1149
|
+
const candidates = entities.filter((e) => {
|
|
1150
|
+
if (e.memory_tier === "reference")
|
|
1151
|
+
return false;
|
|
1152
|
+
if (e.access_count >= 2)
|
|
1153
|
+
return false;
|
|
1154
|
+
const ageDays = (now - new Date(e.created_at).getTime()) / MS_PER_DAY;
|
|
1155
|
+
return ageDays >= orphanAgeDays;
|
|
1156
|
+
});
|
|
1157
|
+
for (let i = 0;i < candidates.length; i += CONCURRENCY_LIMIT) {
|
|
1158
|
+
const batch = candidates.slice(i, i + CONCURRENCY_LIMIT);
|
|
1159
|
+
const results = await Promise.allSettled(batch.map(async (entity) => {
|
|
1160
|
+
const related = await client2.getRelatedEntities(entity.id);
|
|
1161
|
+
const totalRelations = (related.outgoing?.length || 0) + (related.incoming?.length || 0);
|
|
1162
|
+
if (totalRelations > 0)
|
|
1163
|
+
return null;
|
|
1164
|
+
const ageDays = (now - new Date(entity.created_at).getTime()) / MS_PER_DAY;
|
|
1165
|
+
return {
|
|
1166
|
+
id: entity.id,
|
|
1167
|
+
title: entity.title,
|
|
1168
|
+
type: entity.type,
|
|
1169
|
+
tier: entity.memory_tier,
|
|
1170
|
+
ageDays: Math.round(ageDays),
|
|
1171
|
+
accessCount: entity.access_count
|
|
1172
|
+
};
|
|
1173
|
+
}));
|
|
1174
|
+
for (const r of results) {
|
|
1175
|
+
if (r.status === "fulfilled" && r.value) {
|
|
1176
|
+
result.items.push(r.value);
|
|
1177
|
+
result.orphansFound++;
|
|
1178
|
+
}
|
|
259
1179
|
}
|
|
260
|
-
|
|
1180
|
+
}
|
|
1181
|
+
return result;
|
|
261
1182
|
}
|
|
262
|
-
async function runDuplicateStep(
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
}
|
|
1183
|
+
async function runDuplicateStep(client2, entities, workspaceId, projectId) {
|
|
1184
|
+
const result = {
|
|
1185
|
+
duplicatePairsFound: 0,
|
|
1186
|
+
resolved: 0,
|
|
1187
|
+
pairs: []
|
|
1188
|
+
};
|
|
1189
|
+
const seenPairs = new Set;
|
|
1190
|
+
const flaggedForRemoval = new Set;
|
|
1191
|
+
const entityMap = new Map(entities.map((e) => [e.id, e]));
|
|
1192
|
+
const similarityMap = new Map;
|
|
1193
|
+
for (let i = 0;i < entities.length; i += CONCURRENCY_LIMIT) {
|
|
1194
|
+
const batch = entities.slice(i, i + CONCURRENCY_LIMIT);
|
|
1195
|
+
const results = await Promise.allSettled(batch.map(async (entity) => {
|
|
1196
|
+
const similar = await findSimilarEntities(client2, entity.title, entity.content, workspaceId, { projectId, limit: 5, minRrfScore: 0.05, excludeIds: [entity.id] });
|
|
1197
|
+
return { entityId: entity.id, similar };
|
|
1198
|
+
}));
|
|
1199
|
+
for (const r of results) {
|
|
1200
|
+
if (r.status === "fulfilled") {
|
|
1201
|
+
similarityMap.set(r.value.entityId, r.value.similar);
|
|
1202
|
+
}
|
|
283
1203
|
}
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
removeId: remove.id,
|
|
313
|
-
removeTitle: remove.title,
|
|
314
|
-
similarity: Math.round(sim * 100) / 100,
|
|
315
|
-
});
|
|
316
|
-
result.duplicatePairsFound++;
|
|
317
|
-
}
|
|
1204
|
+
}
|
|
1205
|
+
for (const entity of entities) {
|
|
1206
|
+
if (flaggedForRemoval.has(entity.id))
|
|
1207
|
+
continue;
|
|
1208
|
+
const similar = similarityMap.get(entity.id) || [];
|
|
1209
|
+
for (const match of similar) {
|
|
1210
|
+
if (flaggedForRemoval.has(match.id))
|
|
1211
|
+
continue;
|
|
1212
|
+
const pairKey = [entity.id, match.id].sort().join(":");
|
|
1213
|
+
if (seenPairs.has(pairKey))
|
|
1214
|
+
continue;
|
|
1215
|
+
seenPairs.add(pairKey);
|
|
1216
|
+
const sim = titleSimilarity(entity.title, match.title);
|
|
1217
|
+
if (sim < DUPLICATE_SIMILARITY_THRESHOLD)
|
|
1218
|
+
continue;
|
|
1219
|
+
const entityScore = entityQualityScore(entity);
|
|
1220
|
+
const matchEntity = entityMap.get(match.id);
|
|
1221
|
+
const matchScore = matchEntity ? entityQualityScore(matchEntity) : match.confidence;
|
|
1222
|
+
const [keep, remove] = entityScore >= matchScore ? [entity, { id: match.id, title: match.title }] : [{ id: match.id, title: match.title }, entity];
|
|
1223
|
+
flaggedForRemoval.add(remove.id);
|
|
1224
|
+
result.pairs.push({
|
|
1225
|
+
keepId: keep.id,
|
|
1226
|
+
keepTitle: keep.title,
|
|
1227
|
+
removeId: remove.id,
|
|
1228
|
+
removeTitle: remove.title,
|
|
1229
|
+
similarity: Math.round(sim * 100) / 100
|
|
1230
|
+
});
|
|
1231
|
+
result.duplicatePairsFound++;
|
|
318
1232
|
}
|
|
319
|
-
|
|
1233
|
+
}
|
|
1234
|
+
return result;
|
|
320
1235
|
}
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
reference: 3,
|
|
326
|
-
episode: 2,
|
|
327
|
-
draft: 1,
|
|
1236
|
+
var TIER_WEIGHTS = {
|
|
1237
|
+
reference: 3,
|
|
1238
|
+
episode: 2,
|
|
1239
|
+
draft: 1
|
|
328
1240
|
};
|
|
329
1241
|
function entityQualityScore(entity) {
|
|
330
|
-
|
|
331
|
-
(TIER_WEIGHTS[entity.memory_tier] || 0) +
|
|
332
|
-
Math.min(entity.access_count, 10) * 0.1);
|
|
1242
|
+
return entity.confidence + (TIER_WEIGHTS[entity.memory_tier] || 0) + Math.min(entity.access_count, 10) * 0.1;
|
|
333
1243
|
}
|
|
334
1244
|
function titleSimilarity(a, b) {
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
return union > 0 ? intersection / union : 0;
|
|
1245
|
+
const na = a.toLowerCase().trim();
|
|
1246
|
+
const nb = b.toLowerCase().trim();
|
|
1247
|
+
if (na === nb)
|
|
1248
|
+
return 1;
|
|
1249
|
+
const wordsA = new Set(na.split(/\W+/).filter(Boolean));
|
|
1250
|
+
const wordsB = new Set(nb.split(/\W+/).filter(Boolean));
|
|
1251
|
+
if (wordsA.size === 0 || wordsB.size === 0)
|
|
1252
|
+
return 0;
|
|
1253
|
+
let intersection = 0;
|
|
1254
|
+
for (const w of wordsA) {
|
|
1255
|
+
if (wordsB.has(w))
|
|
1256
|
+
intersection++;
|
|
1257
|
+
}
|
|
1258
|
+
const union = wordsA.size + wordsB.size - intersection;
|
|
1259
|
+
return union > 0 ? intersection / union : 0;
|
|
351
1260
|
}
|
|
352
|
-
// ---------------------------------------------------------------------------
|
|
353
|
-
// Health report renderer
|
|
354
|
-
// ---------------------------------------------------------------------------
|
|
355
1261
|
function generateHealthReport(report) {
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
}
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
}
|
|
382
|
-
// Consolidate
|
|
383
|
-
if (report.steps.consolidate) {
|
|
384
|
-
const c = report.steps.consolidate;
|
|
385
|
-
lines.push("## Consolidation");
|
|
386
|
-
if (c.clustersFound === 0) {
|
|
387
|
-
lines.push(`Scanned ${c.entitiesProcessed} draft/episode entities — no clusters found.\n`);
|
|
388
|
-
}
|
|
389
|
-
else {
|
|
390
|
-
lines.push(`Found **${c.clustersFound}** clusters across ${c.entitiesProcessed} entities:`);
|
|
391
|
-
for (const d of c.details.slice(0, 10)) {
|
|
392
|
-
lines.push(`- **${d.mergedTitle}** — ${d.clusterSize} entities`);
|
|
393
|
-
}
|
|
394
|
-
lines.push("");
|
|
395
|
-
}
|
|
396
|
-
}
|
|
397
|
-
// Orphans
|
|
398
|
-
if (report.steps.orphans) {
|
|
399
|
-
const o = report.steps.orphans;
|
|
400
|
-
lines.push("## Orphaned Entities");
|
|
401
|
-
if (o.orphansFound === 0) {
|
|
402
|
-
lines.push("No orphans found.\n");
|
|
403
|
-
}
|
|
404
|
-
else {
|
|
405
|
-
lines.push(`Found **${o.orphansFound}** orphans${!report.dryRun ? ` (removed ${o.removed})` : ""}:`);
|
|
406
|
-
lines.push("| Title | Type | Tier | Age | Accesses |");
|
|
407
|
-
lines.push("|-------|------|------|-----|----------|");
|
|
408
|
-
for (const item of o.items.slice(0, 20)) {
|
|
409
|
-
lines.push(`| ${item.title} | ${item.type} | ${item.tier} | ${item.ageDays}d | ${item.accessCount} |`);
|
|
410
|
-
}
|
|
411
|
-
lines.push("");
|
|
412
|
-
}
|
|
1262
|
+
const mode = report.dryRun ? "Dry Run (preview)" : "Executed";
|
|
1263
|
+
const lines = [
|
|
1264
|
+
`# Memory Health Report
|
|
1265
|
+
`,
|
|
1266
|
+
`**Mode:** ${mode} | **Entities:** ${report.summary.totalEntities} | **Issues:** ${report.summary.issuesFound} | **Actions:** ${report.summary.actionsTaken}`,
|
|
1267
|
+
""
|
|
1268
|
+
];
|
|
1269
|
+
if (report.summary.totalEntities >= MAX_ENTITIES_FETCH) {
|
|
1270
|
+
lines.push(`> **Note:** Entity count hit the ${MAX_ENTITIES_FETCH} fetch limit. Some entities may not have been analyzed.
|
|
1271
|
+
`);
|
|
1272
|
+
}
|
|
1273
|
+
if (report.steps.prune) {
|
|
1274
|
+
const p = report.steps.prune;
|
|
1275
|
+
lines.push("## Stale Drafts");
|
|
1276
|
+
if (p.staleDraftsFound === 0) {
|
|
1277
|
+
lines.push(`No stale drafts found.
|
|
1278
|
+
`);
|
|
1279
|
+
} else {
|
|
1280
|
+
lines.push(`Found **${p.staleDraftsFound}** stale drafts${!report.dryRun ? ` (pruned ${p.pruned})` : ""}:`);
|
|
1281
|
+
lines.push("| Title | Age | Decay |");
|
|
1282
|
+
lines.push("|-------|-----|-------|");
|
|
1283
|
+
for (const item of p.items.slice(0, 20)) {
|
|
1284
|
+
lines.push(`| ${item.title} | ${item.ageDays}d | ${item.decayScore} |`);
|
|
1285
|
+
}
|
|
1286
|
+
lines.push("");
|
|
413
1287
|
}
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
}
|
|
1288
|
+
}
|
|
1289
|
+
if (report.steps.consolidate) {
|
|
1290
|
+
const c = report.steps.consolidate;
|
|
1291
|
+
lines.push("## Consolidation");
|
|
1292
|
+
if (c.clustersFound === 0) {
|
|
1293
|
+
lines.push(`Scanned ${c.entitiesProcessed} draft/episode entities — no clusters found.
|
|
1294
|
+
`);
|
|
1295
|
+
} else {
|
|
1296
|
+
lines.push(`Found **${c.clustersFound}** clusters across ${c.entitiesProcessed} entities:`);
|
|
1297
|
+
for (const d of c.details.slice(0, 10)) {
|
|
1298
|
+
lines.push(`- **${d.mergedTitle}** — ${d.clusterSize} entities`);
|
|
1299
|
+
}
|
|
1300
|
+
lines.push("");
|
|
428
1301
|
}
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
}
|
|
1302
|
+
}
|
|
1303
|
+
if (report.steps.orphans) {
|
|
1304
|
+
const o = report.steps.orphans;
|
|
1305
|
+
lines.push("## Orphaned Entities");
|
|
1306
|
+
if (o.orphansFound === 0) {
|
|
1307
|
+
lines.push(`No orphans found.
|
|
1308
|
+
`);
|
|
1309
|
+
} else {
|
|
1310
|
+
lines.push(`Found **${o.orphansFound}** orphans${!report.dryRun ? ` (removed ${o.removed})` : ""}:`);
|
|
1311
|
+
lines.push("| Title | Type | Tier | Age | Accesses |");
|
|
1312
|
+
lines.push("|-------|------|------|-----|----------|");
|
|
1313
|
+
for (const item of o.items.slice(0, 20)) {
|
|
1314
|
+
lines.push(`| ${item.title} | ${item.type} | ${item.tier} | ${item.ageDays}d | ${item.accessCount} |`);
|
|
1315
|
+
}
|
|
1316
|
+
lines.push("");
|
|
442
1317
|
}
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
1318
|
+
}
|
|
1319
|
+
if (report.steps.duplicates) {
|
|
1320
|
+
const d = report.steps.duplicates;
|
|
1321
|
+
lines.push("## Near-Duplicates");
|
|
1322
|
+
if (d.duplicatePairsFound === 0) {
|
|
1323
|
+
lines.push(`No duplicates found.
|
|
1324
|
+
`);
|
|
1325
|
+
} else {
|
|
1326
|
+
lines.push(`Found **${d.duplicatePairsFound}** duplicate pairs${!report.dryRun ? ` (resolved ${d.resolved})` : ""}:`);
|
|
1327
|
+
for (const pair of d.pairs.slice(0, 20)) {
|
|
1328
|
+
lines.push(`- "${pair.keepTitle}" ~ "${pair.removeTitle}" (${Math.round(pair.similarity * 100)}% similar, keep first)`);
|
|
1329
|
+
}
|
|
1330
|
+
lines.push("");
|
|
450
1331
|
}
|
|
1332
|
+
}
|
|
1333
|
+
if (report.steps.backfill) {
|
|
1334
|
+
const b = report.steps.backfill;
|
|
1335
|
+
lines.push("## Embedding Coverage");
|
|
451
1336
|
if (report.dryRun) {
|
|
452
|
-
|
|
1337
|
+
lines.push("Backfill will run when executed with `dryRun: false`.\n");
|
|
1338
|
+
} else if (b.remaining === 0) {
|
|
1339
|
+
lines.push(`All embeddings up to date (processed ${b.processed}).
|
|
1340
|
+
`);
|
|
1341
|
+
} else {
|
|
1342
|
+
lines.push(`Processed ${b.processed} entities. ${b.remaining} still need embeddings.
|
|
1343
|
+
`);
|
|
453
1344
|
}
|
|
454
|
-
|
|
1345
|
+
}
|
|
1346
|
+
if (report.errors.length > 0) {
|
|
1347
|
+
lines.push("## Errors");
|
|
1348
|
+
for (const e of report.errors) {
|
|
1349
|
+
lines.push(`- **${e.step}:** ${e.message}`);
|
|
1350
|
+
}
|
|
1351
|
+
lines.push("");
|
|
1352
|
+
}
|
|
1353
|
+
if (report.dryRun) {
|
|
1354
|
+
lines.push("---\n*Run with `dryRun: false` to execute cleanup.*");
|
|
1355
|
+
}
|
|
1356
|
+
return lines.join(`
|
|
1357
|
+
`);
|
|
455
1358
|
}
|
|
1359
|
+
export {
|
|
1360
|
+
runMemoryCleanup
|
|
1361
|
+
};
|