@gethmy/mcp 2.3.0 → 2.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +80 -23
- package/dist/index.js +80 -23
- package/dist/lib/api-client.js +6 -0
- package/dist/lib/memory-cleanup.js +57 -28
- package/dist/lib/server.js +15 -1
- package/package.json +1 -1
- package/src/api-client.ts +37 -1
- package/src/memory-cleanup.ts +92 -50
- package/src/server.ts +16 -1
package/dist/cli.js
CHANGED
|
@@ -27273,6 +27273,12 @@ class HarmonyApiClient {
|
|
|
27273
27273
|
async endAgentSession(cardId, data) {
|
|
27274
27274
|
return this.request("DELETE", `/cards/${cardId}/agent-context`, data);
|
|
27275
27275
|
}
|
|
27276
|
+
async flushActivityLog(cardId, data) {
|
|
27277
|
+
return this.request("POST", `/cards/${cardId}/agent-activity-log`, data);
|
|
27278
|
+
}
|
|
27279
|
+
async getActivityLog(cardId, sessionId) {
|
|
27280
|
+
return this.request("GET", `/cards/${cardId}/agent-activity-log?sessionId=${sessionId}`);
|
|
27281
|
+
}
|
|
27276
27282
|
async getAgentSession(cardId, options) {
|
|
27277
27283
|
const params = new URLSearchParams;
|
|
27278
27284
|
if (options?.includeEnded)
|
|
@@ -28066,6 +28072,10 @@ var ALL_STEPS = [
|
|
|
28066
28072
|
"duplicates",
|
|
28067
28073
|
"backfill"
|
|
28068
28074
|
];
|
|
28075
|
+
var MS_PER_DAY = 1000 * 60 * 60 * 24;
|
|
28076
|
+
var MAX_ENTITIES_FETCH = 200;
|
|
28077
|
+
var DUPLICATE_SIMILARITY_THRESHOLD = 0.85;
|
|
28078
|
+
var CONCURRENCY_LIMIT = 5;
|
|
28069
28079
|
async function runMemoryCleanup(client3, workspaceId, projectId, options) {
|
|
28070
28080
|
const dryRun = options?.dryRun !== false;
|
|
28071
28081
|
const steps = options?.steps ?? ALL_STEPS;
|
|
@@ -28087,7 +28097,7 @@ async function runMemoryCleanup(client3, workspaceId, projectId, options) {
|
|
|
28087
28097
|
const listResult = await client3.listMemoryEntities({
|
|
28088
28098
|
workspace_id: workspaceId,
|
|
28089
28099
|
project_id: projectId,
|
|
28090
|
-
limit:
|
|
28100
|
+
limit: MAX_ENTITIES_FETCH
|
|
28091
28101
|
});
|
|
28092
28102
|
entities = listResult.entities || [];
|
|
28093
28103
|
report.summary.totalEntities = entities.length;
|
|
@@ -28108,7 +28118,12 @@ async function runMemoryCleanup(client3, workspaceId, projectId, options) {
|
|
|
28108
28118
|
try {
|
|
28109
28119
|
await client3.deleteMemoryEntity(item.id);
|
|
28110
28120
|
report.steps.prune.pruned++;
|
|
28111
|
-
} catch {
|
|
28121
|
+
} catch (err) {
|
|
28122
|
+
report.errors.push({
|
|
28123
|
+
step: "prune",
|
|
28124
|
+
message: `Failed to delete ${item.id}: ${err.message}`
|
|
28125
|
+
});
|
|
28126
|
+
}
|
|
28112
28127
|
}
|
|
28113
28128
|
report.summary.actionsTaken += report.steps.prune.pruned;
|
|
28114
28129
|
}
|
|
@@ -28150,7 +28165,12 @@ async function runMemoryCleanup(client3, workspaceId, projectId, options) {
|
|
|
28150
28165
|
try {
|
|
28151
28166
|
await client3.deleteMemoryEntity(item.id);
|
|
28152
28167
|
report.steps.orphans.removed++;
|
|
28153
|
-
} catch {
|
|
28168
|
+
} catch (err) {
|
|
28169
|
+
report.errors.push({
|
|
28170
|
+
step: "orphans",
|
|
28171
|
+
message: `Failed to delete ${item.id}: ${err.message}`
|
|
28172
|
+
});
|
|
28173
|
+
}
|
|
28154
28174
|
}
|
|
28155
28175
|
report.summary.actionsTaken += report.steps.orphans.removed;
|
|
28156
28176
|
}
|
|
@@ -28170,7 +28190,12 @@ async function runMemoryCleanup(client3, workspaceId, projectId, options) {
|
|
|
28170
28190
|
try {
|
|
28171
28191
|
await client3.deleteMemoryEntity(pair.removeId);
|
|
28172
28192
|
report.steps.duplicates.resolved++;
|
|
28173
|
-
} catch {
|
|
28193
|
+
} catch (err) {
|
|
28194
|
+
report.errors.push({
|
|
28195
|
+
step: "duplicates",
|
|
28196
|
+
message: `Failed to delete ${pair.removeId}: ${err.message}`
|
|
28197
|
+
});
|
|
28198
|
+
}
|
|
28174
28199
|
}
|
|
28175
28200
|
report.summary.actionsTaken += report.steps.duplicates.resolved;
|
|
28176
28201
|
}
|
|
@@ -28214,7 +28239,7 @@ function runPruneStep(entities, maxAgeDays) {
|
|
|
28214
28239
|
const drafts = entities.filter((e) => e.memory_tier === "draft");
|
|
28215
28240
|
const stale = [];
|
|
28216
28241
|
for (const entity of drafts) {
|
|
28217
|
-
const ageDays = (now - new Date(entity.created_at).getTime()) /
|
|
28242
|
+
const ageDays = (now - new Date(entity.created_at).getTime()) / MS_PER_DAY;
|
|
28218
28243
|
if (ageDays < maxAgeDays)
|
|
28219
28244
|
continue;
|
|
28220
28245
|
const lifecycle2 = evaluateLifecycle(entity);
|
|
@@ -28235,26 +28260,32 @@ async function runOrphanStep(client3, entities, orphanAgeDays) {
|
|
|
28235
28260
|
return false;
|
|
28236
28261
|
if (e.access_count >= 2)
|
|
28237
28262
|
return false;
|
|
28238
|
-
const ageDays = (now - new Date(e.created_at).getTime()) /
|
|
28263
|
+
const ageDays = (now - new Date(e.created_at).getTime()) / MS_PER_DAY;
|
|
28239
28264
|
return ageDays >= orphanAgeDays;
|
|
28240
28265
|
});
|
|
28241
|
-
for (
|
|
28242
|
-
|
|
28266
|
+
for (let i = 0;i < candidates.length; i += CONCURRENCY_LIMIT) {
|
|
28267
|
+
const batch = candidates.slice(i, i + CONCURRENCY_LIMIT);
|
|
28268
|
+
const results = await Promise.allSettled(batch.map(async (entity) => {
|
|
28243
28269
|
const related = await client3.getRelatedEntities(entity.id);
|
|
28244
28270
|
const totalRelations = (related.outgoing?.length || 0) + (related.incoming?.length || 0);
|
|
28245
28271
|
if (totalRelations > 0)
|
|
28246
|
-
|
|
28247
|
-
const ageDays = (now - new Date(entity.created_at).getTime()) /
|
|
28248
|
-
|
|
28272
|
+
return null;
|
|
28273
|
+
const ageDays = (now - new Date(entity.created_at).getTime()) / MS_PER_DAY;
|
|
28274
|
+
return {
|
|
28249
28275
|
id: entity.id,
|
|
28250
28276
|
title: entity.title,
|
|
28251
28277
|
type: entity.type,
|
|
28252
28278
|
tier: entity.memory_tier,
|
|
28253
28279
|
ageDays: Math.round(ageDays),
|
|
28254
28280
|
accessCount: entity.access_count
|
|
28255
|
-
}
|
|
28256
|
-
|
|
28257
|
-
|
|
28281
|
+
};
|
|
28282
|
+
}));
|
|
28283
|
+
for (const r of results) {
|
|
28284
|
+
if (r.status === "fulfilled" && r.value) {
|
|
28285
|
+
result.items.push(r.value);
|
|
28286
|
+
result.orphansFound++;
|
|
28287
|
+
}
|
|
28288
|
+
}
|
|
28258
28289
|
}
|
|
28259
28290
|
return result;
|
|
28260
28291
|
}
|
|
@@ -28266,15 +28297,24 @@ async function runDuplicateStep(client3, entities, workspaceId, projectId) {
|
|
|
28266
28297
|
};
|
|
28267
28298
|
const seenPairs = new Set;
|
|
28268
28299
|
const flaggedForRemoval = new Set;
|
|
28300
|
+
const entityMap = new Map(entities.map((e) => [e.id, e]));
|
|
28301
|
+
const similarityMap = new Map;
|
|
28302
|
+
for (let i = 0;i < entities.length; i += CONCURRENCY_LIMIT) {
|
|
28303
|
+
const batch = entities.slice(i, i + CONCURRENCY_LIMIT);
|
|
28304
|
+
const results = await Promise.allSettled(batch.map(async (entity) => {
|
|
28305
|
+
const similar = await findSimilarEntities(client3, entity.title, entity.content, workspaceId, { projectId, limit: 5, minRrfScore: 0.05, excludeIds: [entity.id] });
|
|
28306
|
+
return { entityId: entity.id, similar };
|
|
28307
|
+
}));
|
|
28308
|
+
for (const r of results) {
|
|
28309
|
+
if (r.status === "fulfilled") {
|
|
28310
|
+
similarityMap.set(r.value.entityId, r.value.similar);
|
|
28311
|
+
}
|
|
28312
|
+
}
|
|
28313
|
+
}
|
|
28269
28314
|
for (const entity of entities) {
|
|
28270
28315
|
if (flaggedForRemoval.has(entity.id))
|
|
28271
28316
|
continue;
|
|
28272
|
-
|
|
28273
|
-
try {
|
|
28274
|
-
similar = await findSimilarEntities(client3, entity.title, entity.content, workspaceId, { projectId, limit: 5, minRrfScore: 0.05, excludeIds: [entity.id] });
|
|
28275
|
-
} catch {
|
|
28276
|
-
continue;
|
|
28277
|
-
}
|
|
28317
|
+
const similar = similarityMap.get(entity.id) || [];
|
|
28278
28318
|
for (const match of similar) {
|
|
28279
28319
|
if (flaggedForRemoval.has(match.id))
|
|
28280
28320
|
continue;
|
|
@@ -28283,10 +28323,10 @@ async function runDuplicateStep(client3, entities, workspaceId, projectId) {
|
|
|
28283
28323
|
continue;
|
|
28284
28324
|
seenPairs.add(pairKey);
|
|
28285
28325
|
const sim = titleSimilarity(entity.title, match.title);
|
|
28286
|
-
if (sim <
|
|
28326
|
+
if (sim < DUPLICATE_SIMILARITY_THRESHOLD)
|
|
28287
28327
|
continue;
|
|
28288
28328
|
const entityScore = entityQualityScore(entity);
|
|
28289
|
-
const matchEntity =
|
|
28329
|
+
const matchEntity = entityMap.get(match.id);
|
|
28290
28330
|
const matchScore = matchEntity ? entityQualityScore(matchEntity) : match.confidence;
|
|
28291
28331
|
const [keep, remove] = entityScore >= matchScore ? [entity, { id: match.id, title: match.title }] : [{ id: match.id, title: match.title }, entity];
|
|
28292
28332
|
flaggedForRemoval.add(remove.id);
|
|
@@ -28335,6 +28375,10 @@ function generateHealthReport(report) {
|
|
|
28335
28375
|
`**Mode:** ${mode} | **Entities:** ${report.summary.totalEntities} | **Issues:** ${report.summary.issuesFound} | **Actions:** ${report.summary.actionsTaken}`,
|
|
28336
28376
|
""
|
|
28337
28377
|
];
|
|
28378
|
+
if (report.summary.totalEntities >= MAX_ENTITIES_FETCH) {
|
|
28379
|
+
lines.push(`> **Note:** Entity count hit the ${MAX_ENTITIES_FETCH} fetch limit. Some entities may not have been analyzed.
|
|
28380
|
+
`);
|
|
28381
|
+
}
|
|
28338
28382
|
if (report.steps.prune) {
|
|
28339
28383
|
const p = report.steps.prune;
|
|
28340
28384
|
lines.push("## Stale Drafts");
|
|
@@ -31413,9 +31457,22 @@ async function handleToolCall(name, args, deps) {
|
|
|
31413
31457
|
throw new Error("No workspace specified. Use harmony_set_workspace_context or provide workspaceId.");
|
|
31414
31458
|
}
|
|
31415
31459
|
const projectId = args.projectId || deps.getActiveProjectId() || undefined;
|
|
31460
|
+
const validSteps = [
|
|
31461
|
+
"prune",
|
|
31462
|
+
"consolidate",
|
|
31463
|
+
"orphans",
|
|
31464
|
+
"duplicates",
|
|
31465
|
+
"backfill"
|
|
31466
|
+
];
|
|
31467
|
+
const rawSteps = args.steps;
|
|
31468
|
+
const steps = rawSteps?.filter((s) => validSteps.includes(s));
|
|
31469
|
+
if (rawSteps && steps && steps.length < rawSteps.length) {
|
|
31470
|
+
const invalid = rawSteps.filter((s) => !validSteps.includes(s));
|
|
31471
|
+
console.warn(`Unknown cleanup steps ignored: ${invalid.join(", ")}`);
|
|
31472
|
+
}
|
|
31416
31473
|
const report = await runMemoryCleanup(client3, workspaceId, projectId, {
|
|
31417
31474
|
dryRun: args.dryRun,
|
|
31418
|
-
steps
|
|
31475
|
+
steps,
|
|
31419
31476
|
maxAgeDays: args.maxAgeDays,
|
|
31420
31477
|
minClusterSize: args.minClusterSize,
|
|
31421
31478
|
orphanAgeDays: args.orphanAgeDays
|
package/dist/index.js
CHANGED
|
@@ -25033,6 +25033,12 @@ class HarmonyApiClient {
|
|
|
25033
25033
|
async endAgentSession(cardId, data) {
|
|
25034
25034
|
return this.request("DELETE", `/cards/${cardId}/agent-context`, data);
|
|
25035
25035
|
}
|
|
25036
|
+
async flushActivityLog(cardId, data) {
|
|
25037
|
+
return this.request("POST", `/cards/${cardId}/agent-activity-log`, data);
|
|
25038
|
+
}
|
|
25039
|
+
async getActivityLog(cardId, sessionId) {
|
|
25040
|
+
return this.request("GET", `/cards/${cardId}/agent-activity-log?sessionId=${sessionId}`);
|
|
25041
|
+
}
|
|
25036
25042
|
async getAgentSession(cardId, options) {
|
|
25037
25043
|
const params = new URLSearchParams;
|
|
25038
25044
|
if (options?.includeEnded)
|
|
@@ -25826,6 +25832,10 @@ var ALL_STEPS = [
|
|
|
25826
25832
|
"duplicates",
|
|
25827
25833
|
"backfill"
|
|
25828
25834
|
];
|
|
25835
|
+
var MS_PER_DAY = 1000 * 60 * 60 * 24;
|
|
25836
|
+
var MAX_ENTITIES_FETCH = 200;
|
|
25837
|
+
var DUPLICATE_SIMILARITY_THRESHOLD = 0.85;
|
|
25838
|
+
var CONCURRENCY_LIMIT = 5;
|
|
25829
25839
|
async function runMemoryCleanup(client3, workspaceId, projectId, options) {
|
|
25830
25840
|
const dryRun = options?.dryRun !== false;
|
|
25831
25841
|
const steps = options?.steps ?? ALL_STEPS;
|
|
@@ -25847,7 +25857,7 @@ async function runMemoryCleanup(client3, workspaceId, projectId, options) {
|
|
|
25847
25857
|
const listResult = await client3.listMemoryEntities({
|
|
25848
25858
|
workspace_id: workspaceId,
|
|
25849
25859
|
project_id: projectId,
|
|
25850
|
-
limit:
|
|
25860
|
+
limit: MAX_ENTITIES_FETCH
|
|
25851
25861
|
});
|
|
25852
25862
|
entities = listResult.entities || [];
|
|
25853
25863
|
report.summary.totalEntities = entities.length;
|
|
@@ -25868,7 +25878,12 @@ async function runMemoryCleanup(client3, workspaceId, projectId, options) {
|
|
|
25868
25878
|
try {
|
|
25869
25879
|
await client3.deleteMemoryEntity(item.id);
|
|
25870
25880
|
report.steps.prune.pruned++;
|
|
25871
|
-
} catch {
|
|
25881
|
+
} catch (err) {
|
|
25882
|
+
report.errors.push({
|
|
25883
|
+
step: "prune",
|
|
25884
|
+
message: `Failed to delete ${item.id}: ${err.message}`
|
|
25885
|
+
});
|
|
25886
|
+
}
|
|
25872
25887
|
}
|
|
25873
25888
|
report.summary.actionsTaken += report.steps.prune.pruned;
|
|
25874
25889
|
}
|
|
@@ -25910,7 +25925,12 @@ async function runMemoryCleanup(client3, workspaceId, projectId, options) {
|
|
|
25910
25925
|
try {
|
|
25911
25926
|
await client3.deleteMemoryEntity(item.id);
|
|
25912
25927
|
report.steps.orphans.removed++;
|
|
25913
|
-
} catch {
|
|
25928
|
+
} catch (err) {
|
|
25929
|
+
report.errors.push({
|
|
25930
|
+
step: "orphans",
|
|
25931
|
+
message: `Failed to delete ${item.id}: ${err.message}`
|
|
25932
|
+
});
|
|
25933
|
+
}
|
|
25914
25934
|
}
|
|
25915
25935
|
report.summary.actionsTaken += report.steps.orphans.removed;
|
|
25916
25936
|
}
|
|
@@ -25930,7 +25950,12 @@ async function runMemoryCleanup(client3, workspaceId, projectId, options) {
|
|
|
25930
25950
|
try {
|
|
25931
25951
|
await client3.deleteMemoryEntity(pair.removeId);
|
|
25932
25952
|
report.steps.duplicates.resolved++;
|
|
25933
|
-
} catch {
|
|
25953
|
+
} catch (err) {
|
|
25954
|
+
report.errors.push({
|
|
25955
|
+
step: "duplicates",
|
|
25956
|
+
message: `Failed to delete ${pair.removeId}: ${err.message}`
|
|
25957
|
+
});
|
|
25958
|
+
}
|
|
25934
25959
|
}
|
|
25935
25960
|
report.summary.actionsTaken += report.steps.duplicates.resolved;
|
|
25936
25961
|
}
|
|
@@ -25974,7 +25999,7 @@ function runPruneStep(entities, maxAgeDays) {
|
|
|
25974
25999
|
const drafts = entities.filter((e) => e.memory_tier === "draft");
|
|
25975
26000
|
const stale = [];
|
|
25976
26001
|
for (const entity of drafts) {
|
|
25977
|
-
const ageDays = (now - new Date(entity.created_at).getTime()) /
|
|
26002
|
+
const ageDays = (now - new Date(entity.created_at).getTime()) / MS_PER_DAY;
|
|
25978
26003
|
if (ageDays < maxAgeDays)
|
|
25979
26004
|
continue;
|
|
25980
26005
|
const lifecycle2 = evaluateLifecycle(entity);
|
|
@@ -25995,26 +26020,32 @@ async function runOrphanStep(client3, entities, orphanAgeDays) {
|
|
|
25995
26020
|
return false;
|
|
25996
26021
|
if (e.access_count >= 2)
|
|
25997
26022
|
return false;
|
|
25998
|
-
const ageDays = (now - new Date(e.created_at).getTime()) /
|
|
26023
|
+
const ageDays = (now - new Date(e.created_at).getTime()) / MS_PER_DAY;
|
|
25999
26024
|
return ageDays >= orphanAgeDays;
|
|
26000
26025
|
});
|
|
26001
|
-
for (
|
|
26002
|
-
|
|
26026
|
+
for (let i = 0;i < candidates.length; i += CONCURRENCY_LIMIT) {
|
|
26027
|
+
const batch = candidates.slice(i, i + CONCURRENCY_LIMIT);
|
|
26028
|
+
const results = await Promise.allSettled(batch.map(async (entity) => {
|
|
26003
26029
|
const related = await client3.getRelatedEntities(entity.id);
|
|
26004
26030
|
const totalRelations = (related.outgoing?.length || 0) + (related.incoming?.length || 0);
|
|
26005
26031
|
if (totalRelations > 0)
|
|
26006
|
-
|
|
26007
|
-
const ageDays = (now - new Date(entity.created_at).getTime()) /
|
|
26008
|
-
|
|
26032
|
+
return null;
|
|
26033
|
+
const ageDays = (now - new Date(entity.created_at).getTime()) / MS_PER_DAY;
|
|
26034
|
+
return {
|
|
26009
26035
|
id: entity.id,
|
|
26010
26036
|
title: entity.title,
|
|
26011
26037
|
type: entity.type,
|
|
26012
26038
|
tier: entity.memory_tier,
|
|
26013
26039
|
ageDays: Math.round(ageDays),
|
|
26014
26040
|
accessCount: entity.access_count
|
|
26015
|
-
}
|
|
26016
|
-
|
|
26017
|
-
|
|
26041
|
+
};
|
|
26042
|
+
}));
|
|
26043
|
+
for (const r of results) {
|
|
26044
|
+
if (r.status === "fulfilled" && r.value) {
|
|
26045
|
+
result.items.push(r.value);
|
|
26046
|
+
result.orphansFound++;
|
|
26047
|
+
}
|
|
26048
|
+
}
|
|
26018
26049
|
}
|
|
26019
26050
|
return result;
|
|
26020
26051
|
}
|
|
@@ -26026,15 +26057,24 @@ async function runDuplicateStep(client3, entities, workspaceId, projectId) {
|
|
|
26026
26057
|
};
|
|
26027
26058
|
const seenPairs = new Set;
|
|
26028
26059
|
const flaggedForRemoval = new Set;
|
|
26060
|
+
const entityMap = new Map(entities.map((e) => [e.id, e]));
|
|
26061
|
+
const similarityMap = new Map;
|
|
26062
|
+
for (let i = 0;i < entities.length; i += CONCURRENCY_LIMIT) {
|
|
26063
|
+
const batch = entities.slice(i, i + CONCURRENCY_LIMIT);
|
|
26064
|
+
const results = await Promise.allSettled(batch.map(async (entity) => {
|
|
26065
|
+
const similar = await findSimilarEntities(client3, entity.title, entity.content, workspaceId, { projectId, limit: 5, minRrfScore: 0.05, excludeIds: [entity.id] });
|
|
26066
|
+
return { entityId: entity.id, similar };
|
|
26067
|
+
}));
|
|
26068
|
+
for (const r of results) {
|
|
26069
|
+
if (r.status === "fulfilled") {
|
|
26070
|
+
similarityMap.set(r.value.entityId, r.value.similar);
|
|
26071
|
+
}
|
|
26072
|
+
}
|
|
26073
|
+
}
|
|
26029
26074
|
for (const entity of entities) {
|
|
26030
26075
|
if (flaggedForRemoval.has(entity.id))
|
|
26031
26076
|
continue;
|
|
26032
|
-
|
|
26033
|
-
try {
|
|
26034
|
-
similar = await findSimilarEntities(client3, entity.title, entity.content, workspaceId, { projectId, limit: 5, minRrfScore: 0.05, excludeIds: [entity.id] });
|
|
26035
|
-
} catch {
|
|
26036
|
-
continue;
|
|
26037
|
-
}
|
|
26077
|
+
const similar = similarityMap.get(entity.id) || [];
|
|
26038
26078
|
for (const match of similar) {
|
|
26039
26079
|
if (flaggedForRemoval.has(match.id))
|
|
26040
26080
|
continue;
|
|
@@ -26043,10 +26083,10 @@ async function runDuplicateStep(client3, entities, workspaceId, projectId) {
|
|
|
26043
26083
|
continue;
|
|
26044
26084
|
seenPairs.add(pairKey);
|
|
26045
26085
|
const sim = titleSimilarity(entity.title, match.title);
|
|
26046
|
-
if (sim <
|
|
26086
|
+
if (sim < DUPLICATE_SIMILARITY_THRESHOLD)
|
|
26047
26087
|
continue;
|
|
26048
26088
|
const entityScore = entityQualityScore(entity);
|
|
26049
|
-
const matchEntity =
|
|
26089
|
+
const matchEntity = entityMap.get(match.id);
|
|
26050
26090
|
const matchScore = matchEntity ? entityQualityScore(matchEntity) : match.confidence;
|
|
26051
26091
|
const [keep, remove] = entityScore >= matchScore ? [entity, { id: match.id, title: match.title }] : [{ id: match.id, title: match.title }, entity];
|
|
26052
26092
|
flaggedForRemoval.add(remove.id);
|
|
@@ -26095,6 +26135,10 @@ function generateHealthReport(report) {
|
|
|
26095
26135
|
`**Mode:** ${mode} | **Entities:** ${report.summary.totalEntities} | **Issues:** ${report.summary.issuesFound} | **Actions:** ${report.summary.actionsTaken}`,
|
|
26096
26136
|
""
|
|
26097
26137
|
];
|
|
26138
|
+
if (report.summary.totalEntities >= MAX_ENTITIES_FETCH) {
|
|
26139
|
+
lines.push(`> **Note:** Entity count hit the ${MAX_ENTITIES_FETCH} fetch limit. Some entities may not have been analyzed.
|
|
26140
|
+
`);
|
|
26141
|
+
}
|
|
26098
26142
|
if (report.steps.prune) {
|
|
26099
26143
|
const p = report.steps.prune;
|
|
26100
26144
|
lines.push("## Stale Drafts");
|
|
@@ -29173,9 +29217,22 @@ async function handleToolCall(name, args, deps) {
|
|
|
29173
29217
|
throw new Error("No workspace specified. Use harmony_set_workspace_context or provide workspaceId.");
|
|
29174
29218
|
}
|
|
29175
29219
|
const projectId = args.projectId || deps.getActiveProjectId() || undefined;
|
|
29220
|
+
const validSteps = [
|
|
29221
|
+
"prune",
|
|
29222
|
+
"consolidate",
|
|
29223
|
+
"orphans",
|
|
29224
|
+
"duplicates",
|
|
29225
|
+
"backfill"
|
|
29226
|
+
];
|
|
29227
|
+
const rawSteps = args.steps;
|
|
29228
|
+
const steps = rawSteps?.filter((s) => validSteps.includes(s));
|
|
29229
|
+
if (rawSteps && steps && steps.length < rawSteps.length) {
|
|
29230
|
+
const invalid = rawSteps.filter((s) => !validSteps.includes(s));
|
|
29231
|
+
console.warn(`Unknown cleanup steps ignored: ${invalid.join(", ")}`);
|
|
29232
|
+
}
|
|
29176
29233
|
const report = await runMemoryCleanup(client3, workspaceId, projectId, {
|
|
29177
29234
|
dryRun: args.dryRun,
|
|
29178
|
-
steps
|
|
29235
|
+
steps,
|
|
29179
29236
|
maxAgeDays: args.maxAgeDays,
|
|
29180
29237
|
minClusterSize: args.minClusterSize,
|
|
29181
29238
|
orphanAgeDays: args.orphanAgeDays
|
package/dist/lib/api-client.js
CHANGED
|
@@ -314,6 +314,12 @@ export class HarmonyApiClient {
|
|
|
314
314
|
async endAgentSession(cardId, data) {
|
|
315
315
|
return this.request("DELETE", `/cards/${cardId}/agent-context`, data);
|
|
316
316
|
}
|
|
317
|
+
async flushActivityLog(cardId, data) {
|
|
318
|
+
return this.request("POST", `/cards/${cardId}/agent-activity-log`, data);
|
|
319
|
+
}
|
|
320
|
+
async getActivityLog(cardId, sessionId) {
|
|
321
|
+
return this.request("GET", `/cards/${cardId}/agent-activity-log?sessionId=${sessionId}`);
|
|
322
|
+
}
|
|
317
323
|
async getAgentSession(cardId, options) {
|
|
318
324
|
const params = new URLSearchParams();
|
|
319
325
|
if (options?.includeEnded)
|
|
@@ -17,6 +17,10 @@ const ALL_STEPS = [
|
|
|
17
17
|
"duplicates",
|
|
18
18
|
"backfill",
|
|
19
19
|
];
|
|
20
|
+
const MS_PER_DAY = 1000 * 60 * 60 * 24;
|
|
21
|
+
const MAX_ENTITIES_FETCH = 200;
|
|
22
|
+
const DUPLICATE_SIMILARITY_THRESHOLD = 0.85;
|
|
23
|
+
const CONCURRENCY_LIMIT = 5;
|
|
20
24
|
// ---------------------------------------------------------------------------
|
|
21
25
|
// Main orchestrator
|
|
22
26
|
// ---------------------------------------------------------------------------
|
|
@@ -42,7 +46,7 @@ export async function runMemoryCleanup(client, workspaceId, projectId, options)
|
|
|
42
46
|
const listResult = await client.listMemoryEntities({
|
|
43
47
|
workspace_id: workspaceId,
|
|
44
48
|
project_id: projectId,
|
|
45
|
-
limit:
|
|
49
|
+
limit: MAX_ENTITIES_FETCH,
|
|
46
50
|
});
|
|
47
51
|
entities = (listResult.entities || []);
|
|
48
52
|
report.summary.totalEntities = entities.length;
|
|
@@ -66,8 +70,11 @@ export async function runMemoryCleanup(client, workspaceId, projectId, options)
|
|
|
66
70
|
await client.deleteMemoryEntity(item.id);
|
|
67
71
|
report.steps.prune.pruned++;
|
|
68
72
|
}
|
|
69
|
-
catch {
|
|
70
|
-
|
|
73
|
+
catch (err) {
|
|
74
|
+
report.errors.push({
|
|
75
|
+
step: "prune",
|
|
76
|
+
message: `Failed to delete ${item.id}: ${err.message}`,
|
|
77
|
+
});
|
|
71
78
|
}
|
|
72
79
|
}
|
|
73
80
|
report.summary.actionsTaken += report.steps.prune.pruned;
|
|
@@ -115,8 +122,11 @@ export async function runMemoryCleanup(client, workspaceId, projectId, options)
|
|
|
115
122
|
await client.deleteMemoryEntity(item.id);
|
|
116
123
|
report.steps.orphans.removed++;
|
|
117
124
|
}
|
|
118
|
-
catch {
|
|
119
|
-
|
|
125
|
+
catch (err) {
|
|
126
|
+
report.errors.push({
|
|
127
|
+
step: "orphans",
|
|
128
|
+
message: `Failed to delete ${item.id}: ${err.message}`,
|
|
129
|
+
});
|
|
120
130
|
}
|
|
121
131
|
}
|
|
122
132
|
report.summary.actionsTaken += report.steps.orphans.removed;
|
|
@@ -140,8 +150,11 @@ export async function runMemoryCleanup(client, workspaceId, projectId, options)
|
|
|
140
150
|
await client.deleteMemoryEntity(pair.removeId);
|
|
141
151
|
report.steps.duplicates.resolved++;
|
|
142
152
|
}
|
|
143
|
-
catch {
|
|
144
|
-
|
|
153
|
+
catch (err) {
|
|
154
|
+
report.errors.push({
|
|
155
|
+
step: "duplicates",
|
|
156
|
+
message: `Failed to delete ${pair.removeId}: ${err.message}`,
|
|
157
|
+
});
|
|
145
158
|
}
|
|
146
159
|
}
|
|
147
160
|
report.summary.actionsTaken += report.steps.duplicates.resolved;
|
|
@@ -194,7 +207,7 @@ function runPruneStep(entities, maxAgeDays) {
|
|
|
194
207
|
const drafts = entities.filter((e) => e.memory_tier === "draft");
|
|
195
208
|
const stale = [];
|
|
196
209
|
for (const entity of drafts) {
|
|
197
|
-
const ageDays = (now - new Date(entity.created_at).getTime()) /
|
|
210
|
+
const ageDays = (now - new Date(entity.created_at).getTime()) / MS_PER_DAY;
|
|
198
211
|
if (ageDays < maxAgeDays)
|
|
199
212
|
continue;
|
|
200
213
|
const lifecycle = evaluateLifecycle(entity);
|
|
@@ -216,28 +229,32 @@ async function runOrphanStep(client, entities, orphanAgeDays) {
|
|
|
216
229
|
return false;
|
|
217
230
|
if (e.access_count >= 2)
|
|
218
231
|
return false;
|
|
219
|
-
const ageDays = (now - new Date(e.created_at).getTime()) /
|
|
232
|
+
const ageDays = (now - new Date(e.created_at).getTime()) / MS_PER_DAY;
|
|
220
233
|
return ageDays >= orphanAgeDays;
|
|
221
234
|
});
|
|
222
|
-
|
|
223
|
-
|
|
235
|
+
// Check relations in concurrent batches
|
|
236
|
+
for (let i = 0; i < candidates.length; i += CONCURRENCY_LIMIT) {
|
|
237
|
+
const batch = candidates.slice(i, i + CONCURRENCY_LIMIT);
|
|
238
|
+
const results = await Promise.allSettled(batch.map(async (entity) => {
|
|
224
239
|
const related = await client.getRelatedEntities(entity.id);
|
|
225
240
|
const totalRelations = (related.outgoing?.length || 0) + (related.incoming?.length || 0);
|
|
226
241
|
if (totalRelations > 0)
|
|
227
|
-
|
|
228
|
-
const ageDays = (now - new Date(entity.created_at).getTime()) /
|
|
229
|
-
|
|
242
|
+
return null;
|
|
243
|
+
const ageDays = (now - new Date(entity.created_at).getTime()) / MS_PER_DAY;
|
|
244
|
+
return {
|
|
230
245
|
id: entity.id,
|
|
231
246
|
title: entity.title,
|
|
232
247
|
type: entity.type,
|
|
233
248
|
tier: entity.memory_tier,
|
|
234
249
|
ageDays: Math.round(ageDays),
|
|
235
250
|
accessCount: entity.access_count,
|
|
236
|
-
}
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
251
|
+
};
|
|
252
|
+
}));
|
|
253
|
+
for (const r of results) {
|
|
254
|
+
if (r.status === "fulfilled" && r.value) {
|
|
255
|
+
result.items.push(r.value);
|
|
256
|
+
result.orphansFound++;
|
|
257
|
+
}
|
|
241
258
|
}
|
|
242
259
|
}
|
|
243
260
|
return result;
|
|
@@ -250,16 +267,25 @@ async function runDuplicateStep(client, entities, workspaceId, projectId) {
|
|
|
250
267
|
};
|
|
251
268
|
const seenPairs = new Set();
|
|
252
269
|
const flaggedForRemoval = new Set();
|
|
270
|
+
const entityMap = new Map(entities.map((e) => [e.id, e]));
|
|
271
|
+
const similarityMap = new Map();
|
|
272
|
+
for (let i = 0; i < entities.length; i += CONCURRENCY_LIMIT) {
|
|
273
|
+
const batch = entities.slice(i, i + CONCURRENCY_LIMIT);
|
|
274
|
+
const results = await Promise.allSettled(batch.map(async (entity) => {
|
|
275
|
+
const similar = await findSimilarEntities(client, entity.title, entity.content, workspaceId, { projectId, limit: 5, minRrfScore: 0.05, excludeIds: [entity.id] });
|
|
276
|
+
return { entityId: entity.id, similar };
|
|
277
|
+
}));
|
|
278
|
+
for (const r of results) {
|
|
279
|
+
if (r.status === "fulfilled") {
|
|
280
|
+
similarityMap.set(r.value.entityId, r.value.similar);
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
// Process pairs sequentially (flaggedForRemoval creates dependencies)
|
|
253
285
|
for (const entity of entities) {
|
|
254
286
|
if (flaggedForRemoval.has(entity.id))
|
|
255
287
|
continue;
|
|
256
|
-
|
|
257
|
-
try {
|
|
258
|
-
similar = await findSimilarEntities(client, entity.title, entity.content, workspaceId, { projectId, limit: 5, minRrfScore: 0.05, excludeIds: [entity.id] });
|
|
259
|
-
}
|
|
260
|
-
catch {
|
|
261
|
-
continue;
|
|
262
|
-
}
|
|
288
|
+
const similar = similarityMap.get(entity.id) || [];
|
|
263
289
|
for (const match of similar) {
|
|
264
290
|
if (flaggedForRemoval.has(match.id))
|
|
265
291
|
continue;
|
|
@@ -268,11 +294,11 @@ async function runDuplicateStep(client, entities, workspaceId, projectId) {
|
|
|
268
294
|
continue;
|
|
269
295
|
seenPairs.add(pairKey);
|
|
270
296
|
const sim = titleSimilarity(entity.title, match.title);
|
|
271
|
-
if (sim <
|
|
297
|
+
if (sim < DUPLICATE_SIMILARITY_THRESHOLD)
|
|
272
298
|
continue;
|
|
273
299
|
// Keep the one with higher confidence, more accesses, or higher tier
|
|
274
300
|
const entityScore = entityQualityScore(entity);
|
|
275
|
-
const matchEntity =
|
|
301
|
+
const matchEntity = entityMap.get(match.id);
|
|
276
302
|
const matchScore = matchEntity
|
|
277
303
|
? entityQualityScore(matchEntity)
|
|
278
304
|
: match.confidence;
|
|
@@ -333,6 +359,9 @@ function generateHealthReport(report) {
|
|
|
333
359
|
`**Mode:** ${mode} | **Entities:** ${report.summary.totalEntities} | **Issues:** ${report.summary.issuesFound} | **Actions:** ${report.summary.actionsTaken}`,
|
|
334
360
|
"",
|
|
335
361
|
];
|
|
362
|
+
if (report.summary.totalEntities >= MAX_ENTITIES_FETCH) {
|
|
363
|
+
lines.push(`> **Note:** Entity count hit the ${MAX_ENTITIES_FETCH} fetch limit. Some entities may not have been analyzed.\n`);
|
|
364
|
+
}
|
|
336
365
|
// Prune
|
|
337
366
|
if (report.steps.prune) {
|
|
338
367
|
const p = report.steps.prune;
|
package/dist/lib/server.js
CHANGED
|
@@ -3263,9 +3263,23 @@ async function handleToolCall(name, args, deps) {
|
|
|
3263
3263
|
throw new Error("No workspace specified. Use harmony_set_workspace_context or provide workspaceId.");
|
|
3264
3264
|
}
|
|
3265
3265
|
const projectId = args.projectId || deps.getActiveProjectId() || undefined;
|
|
3266
|
+
const validSteps = [
|
|
3267
|
+
"prune",
|
|
3268
|
+
"consolidate",
|
|
3269
|
+
"orphans",
|
|
3270
|
+
"duplicates",
|
|
3271
|
+
"backfill",
|
|
3272
|
+
];
|
|
3273
|
+
const rawSteps = args.steps;
|
|
3274
|
+
const steps = rawSteps?.filter((s) => validSteps.includes(s));
|
|
3275
|
+
if (rawSteps && steps && steps.length < rawSteps.length) {
|
|
3276
|
+
const invalid = rawSteps.filter((s) => !validSteps.includes(s));
|
|
3277
|
+
// Will appear in report.errors via the healthReport
|
|
3278
|
+
console.warn(`Unknown cleanup steps ignored: ${invalid.join(", ")}`);
|
|
3279
|
+
}
|
|
3266
3280
|
const report = await runMemoryCleanup(client, workspaceId, projectId, {
|
|
3267
3281
|
dryRun: args.dryRun,
|
|
3268
|
-
steps
|
|
3282
|
+
steps,
|
|
3269
3283
|
maxAgeDays: args.maxAgeDays,
|
|
3270
3284
|
minClusterSize: args.minClusterSize,
|
|
3271
3285
|
orphanAgeDays: args.orphanAgeDays,
|
package/package.json
CHANGED
package/src/api-client.ts
CHANGED
|
@@ -502,7 +502,6 @@ export class HarmonyApiClient {
|
|
|
502
502
|
phase?: string;
|
|
503
503
|
filesChanged?: number;
|
|
504
504
|
costCents?: number;
|
|
505
|
-
recentActions?: { action: string; ts: string }[];
|
|
506
505
|
},
|
|
507
506
|
): Promise<{ session: unknown; created: boolean }> {
|
|
508
507
|
return this.request("POST", `/cards/${cardId}/agent-context`, data);
|
|
@@ -518,6 +517,43 @@ export class HarmonyApiClient {
|
|
|
518
517
|
return this.request("DELETE", `/cards/${cardId}/agent-context`, data);
|
|
519
518
|
}
|
|
520
519
|
|
|
520
|
+
async flushActivityLog(
|
|
521
|
+
cardId: string,
|
|
522
|
+
data: {
|
|
523
|
+
sessionId: string;
|
|
524
|
+
entries: {
|
|
525
|
+
phase?: string;
|
|
526
|
+
eventType: string;
|
|
527
|
+
toolName?: string;
|
|
528
|
+
description: string;
|
|
529
|
+
metadata?: Record<string, unknown>;
|
|
530
|
+
createdAt?: string;
|
|
531
|
+
}[];
|
|
532
|
+
},
|
|
533
|
+
): Promise<{ inserted: number }> {
|
|
534
|
+
return this.request("POST", `/cards/${cardId}/agent-activity-log`, data);
|
|
535
|
+
}
|
|
536
|
+
|
|
537
|
+
async getActivityLog(
|
|
538
|
+
cardId: string,
|
|
539
|
+
sessionId: string,
|
|
540
|
+
): Promise<{
|
|
541
|
+
entries: {
|
|
542
|
+
id: string;
|
|
543
|
+
phase: string | null;
|
|
544
|
+
eventType: string;
|
|
545
|
+
toolName: string | null;
|
|
546
|
+
description: string;
|
|
547
|
+
metadata: Record<string, unknown>;
|
|
548
|
+
createdAt: string;
|
|
549
|
+
}[];
|
|
550
|
+
}> {
|
|
551
|
+
return this.request(
|
|
552
|
+
"GET",
|
|
553
|
+
`/cards/${cardId}/agent-activity-log?sessionId=${sessionId}`,
|
|
554
|
+
);
|
|
555
|
+
}
|
|
556
|
+
|
|
521
557
|
async getAgentSession(
|
|
522
558
|
cardId: string,
|
|
523
559
|
options?: { includeEnded?: boolean },
|
package/src/memory-cleanup.ts
CHANGED
|
@@ -130,6 +130,11 @@ const ALL_STEPS: CleanupStep[] = [
|
|
|
130
130
|
"backfill",
|
|
131
131
|
];
|
|
132
132
|
|
|
133
|
+
const MS_PER_DAY = 1000 * 60 * 60 * 24;
|
|
134
|
+
const MAX_ENTITIES_FETCH = 200;
|
|
135
|
+
const DUPLICATE_SIMILARITY_THRESHOLD = 0.85;
|
|
136
|
+
const CONCURRENCY_LIMIT = 5;
|
|
137
|
+
|
|
133
138
|
// ---------------------------------------------------------------------------
|
|
134
139
|
// Main orchestrator
|
|
135
140
|
// ---------------------------------------------------------------------------
|
|
@@ -163,7 +168,7 @@ export async function runMemoryCleanup(
|
|
|
163
168
|
const listResult = await client.listMemoryEntities({
|
|
164
169
|
workspace_id: workspaceId,
|
|
165
170
|
project_id: projectId,
|
|
166
|
-
limit:
|
|
171
|
+
limit: MAX_ENTITIES_FETCH,
|
|
167
172
|
});
|
|
168
173
|
entities = (listResult.entities || []) as MemoryEntity[];
|
|
169
174
|
report.summary.totalEntities = entities.length;
|
|
@@ -186,8 +191,11 @@ export async function runMemoryCleanup(
|
|
|
186
191
|
try {
|
|
187
192
|
await client.deleteMemoryEntity(item.id);
|
|
188
193
|
report.steps.prune.pruned++;
|
|
189
|
-
} catch {
|
|
190
|
-
|
|
194
|
+
} catch (err) {
|
|
195
|
+
report.errors.push({
|
|
196
|
+
step: "prune",
|
|
197
|
+
message: `Failed to delete ${item.id}: ${(err as Error).message}`,
|
|
198
|
+
});
|
|
191
199
|
}
|
|
192
200
|
}
|
|
193
201
|
report.summary.actionsTaken += report.steps.prune.pruned;
|
|
@@ -237,8 +245,11 @@ export async function runMemoryCleanup(
|
|
|
237
245
|
try {
|
|
238
246
|
await client.deleteMemoryEntity(item.id);
|
|
239
247
|
report.steps.orphans.removed++;
|
|
240
|
-
} catch {
|
|
241
|
-
|
|
248
|
+
} catch (err) {
|
|
249
|
+
report.errors.push({
|
|
250
|
+
step: "orphans",
|
|
251
|
+
message: `Failed to delete ${item.id}: ${(err as Error).message}`,
|
|
252
|
+
});
|
|
242
253
|
}
|
|
243
254
|
}
|
|
244
255
|
report.summary.actionsTaken += report.steps.orphans.removed;
|
|
@@ -266,8 +277,11 @@ export async function runMemoryCleanup(
|
|
|
266
277
|
try {
|
|
267
278
|
await client.deleteMemoryEntity(pair.removeId);
|
|
268
279
|
report.steps.duplicates.resolved++;
|
|
269
|
-
} catch {
|
|
270
|
-
|
|
280
|
+
} catch (err) {
|
|
281
|
+
report.errors.push({
|
|
282
|
+
step: "duplicates",
|
|
283
|
+
message: `Failed to delete ${pair.removeId}: ${(err as Error).message}`,
|
|
284
|
+
});
|
|
271
285
|
}
|
|
272
286
|
}
|
|
273
287
|
report.summary.actionsTaken += report.steps.duplicates.resolved;
|
|
@@ -326,7 +340,7 @@ function runPruneStep(
|
|
|
326
340
|
|
|
327
341
|
for (const entity of drafts) {
|
|
328
342
|
const ageDays =
|
|
329
|
-
(now - new Date(entity.created_at).getTime()) /
|
|
343
|
+
(now - new Date(entity.created_at).getTime()) / MS_PER_DAY;
|
|
330
344
|
if (ageDays < maxAgeDays) continue;
|
|
331
345
|
|
|
332
346
|
const lifecycle = evaluateLifecycle(entity);
|
|
@@ -354,30 +368,38 @@ async function runOrphanStep(
|
|
|
354
368
|
if (e.memory_tier === "reference") return false;
|
|
355
369
|
if (e.access_count >= 2) return false;
|
|
356
370
|
const ageDays =
|
|
357
|
-
(now - new Date(e.created_at).getTime()) /
|
|
371
|
+
(now - new Date(e.created_at).getTime()) / MS_PER_DAY;
|
|
358
372
|
return ageDays >= orphanAgeDays;
|
|
359
373
|
});
|
|
360
374
|
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
(
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
375
|
+
// Check relations in concurrent batches
|
|
376
|
+
for (let i = 0; i < candidates.length; i += CONCURRENCY_LIMIT) {
|
|
377
|
+
const batch = candidates.slice(i, i + CONCURRENCY_LIMIT);
|
|
378
|
+
const results = await Promise.allSettled(
|
|
379
|
+
batch.map(async (entity) => {
|
|
380
|
+
const related = await client.getRelatedEntities(entity.id);
|
|
381
|
+
const totalRelations =
|
|
382
|
+
(related.outgoing?.length || 0) + (related.incoming?.length || 0);
|
|
383
|
+
if (totalRelations > 0) return null;
|
|
384
|
+
|
|
385
|
+
const ageDays =
|
|
386
|
+
(now - new Date(entity.created_at).getTime()) / MS_PER_DAY;
|
|
387
|
+
return {
|
|
388
|
+
id: entity.id,
|
|
389
|
+
title: entity.title,
|
|
390
|
+
type: entity.type,
|
|
391
|
+
tier: entity.memory_tier,
|
|
392
|
+
ageDays: Math.round(ageDays),
|
|
393
|
+
accessCount: entity.access_count,
|
|
394
|
+
};
|
|
395
|
+
}),
|
|
396
|
+
);
|
|
397
|
+
|
|
398
|
+
for (const r of results) {
|
|
399
|
+
if (r.status === "fulfilled" && r.value) {
|
|
400
|
+
result.items.push(r.value);
|
|
401
|
+
result.orphansFound++;
|
|
402
|
+
}
|
|
381
403
|
}
|
|
382
404
|
}
|
|
383
405
|
|
|
@@ -398,28 +420,42 @@ async function runDuplicateStep(
|
|
|
398
420
|
|
|
399
421
|
const seenPairs = new Set<string>();
|
|
400
422
|
const flaggedForRemoval = new Set<string>();
|
|
423
|
+
const entityMap = new Map(entities.map((e) => [e.id, e]));
|
|
401
424
|
|
|
425
|
+
// Pre-fetch similarities in concurrent batches
|
|
426
|
+
type SimilarMatch = {
|
|
427
|
+
id: string;
|
|
428
|
+
type: string;
|
|
429
|
+
title: string;
|
|
430
|
+
content: string;
|
|
431
|
+
confidence: number;
|
|
432
|
+
};
|
|
433
|
+
const similarityMap = new Map<string, SimilarMatch[]>();
|
|
434
|
+
for (let i = 0; i < entities.length; i += CONCURRENCY_LIMIT) {
|
|
435
|
+
const batch = entities.slice(i, i + CONCURRENCY_LIMIT);
|
|
436
|
+
const results = await Promise.allSettled(
|
|
437
|
+
batch.map(async (entity) => {
|
|
438
|
+
const similar = await findSimilarEntities(
|
|
439
|
+
client,
|
|
440
|
+
entity.title,
|
|
441
|
+
entity.content,
|
|
442
|
+
workspaceId,
|
|
443
|
+
{ projectId, limit: 5, minRrfScore: 0.05, excludeIds: [entity.id] },
|
|
444
|
+
);
|
|
445
|
+
return { entityId: entity.id, similar };
|
|
446
|
+
}),
|
|
447
|
+
);
|
|
448
|
+
for (const r of results) {
|
|
449
|
+
if (r.status === "fulfilled") {
|
|
450
|
+
similarityMap.set(r.value.entityId, r.value.similar);
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
// Process pairs sequentially (flaggedForRemoval creates dependencies)
|
|
402
456
|
for (const entity of entities) {
|
|
403
457
|
if (flaggedForRemoval.has(entity.id)) continue;
|
|
404
|
-
|
|
405
|
-
let similar: Array<{
|
|
406
|
-
id: string;
|
|
407
|
-
type: string;
|
|
408
|
-
title: string;
|
|
409
|
-
content: string;
|
|
410
|
-
confidence: number;
|
|
411
|
-
}>;
|
|
412
|
-
try {
|
|
413
|
-
similar = await findSimilarEntities(
|
|
414
|
-
client,
|
|
415
|
-
entity.title,
|
|
416
|
-
entity.content,
|
|
417
|
-
workspaceId,
|
|
418
|
-
{ projectId, limit: 5, minRrfScore: 0.05, excludeIds: [entity.id] },
|
|
419
|
-
);
|
|
420
|
-
} catch {
|
|
421
|
-
continue;
|
|
422
|
-
}
|
|
458
|
+
const similar = similarityMap.get(entity.id) || [];
|
|
423
459
|
|
|
424
460
|
for (const match of similar) {
|
|
425
461
|
if (flaggedForRemoval.has(match.id)) continue;
|
|
@@ -429,11 +465,11 @@ async function runDuplicateStep(
|
|
|
429
465
|
seenPairs.add(pairKey);
|
|
430
466
|
|
|
431
467
|
const sim = titleSimilarity(entity.title, match.title);
|
|
432
|
-
if (sim <
|
|
468
|
+
if (sim < DUPLICATE_SIMILARITY_THRESHOLD) continue;
|
|
433
469
|
|
|
434
470
|
// Keep the one with higher confidence, more accesses, or higher tier
|
|
435
471
|
const entityScore = entityQualityScore(entity);
|
|
436
|
-
const matchEntity =
|
|
472
|
+
const matchEntity = entityMap.get(match.id);
|
|
437
473
|
const matchScore = matchEntity
|
|
438
474
|
? entityQualityScore(matchEntity)
|
|
439
475
|
: match.confidence;
|
|
@@ -506,6 +542,12 @@ function generateHealthReport(report: CleanupReport): string {
|
|
|
506
542
|
"",
|
|
507
543
|
];
|
|
508
544
|
|
|
545
|
+
if (report.summary.totalEntities >= MAX_ENTITIES_FETCH) {
|
|
546
|
+
lines.push(
|
|
547
|
+
`> **Note:** Entity count hit the ${MAX_ENTITIES_FETCH} fetch limit. Some entities may not have been analyzed.\n`,
|
|
548
|
+
);
|
|
549
|
+
}
|
|
550
|
+
|
|
509
551
|
// Prune
|
|
510
552
|
if (report.steps.prune) {
|
|
511
553
|
const p = report.steps.prune;
|
package/src/server.ts
CHANGED
|
@@ -4024,9 +4024,24 @@ async function handleToolCall(
|
|
|
4024
4024
|
const projectId =
|
|
4025
4025
|
(args.projectId as string) || deps.getActiveProjectId() || undefined;
|
|
4026
4026
|
|
|
4027
|
+
const validSteps = [
|
|
4028
|
+
"prune",
|
|
4029
|
+
"consolidate",
|
|
4030
|
+
"orphans",
|
|
4031
|
+
"duplicates",
|
|
4032
|
+
"backfill",
|
|
4033
|
+
];
|
|
4034
|
+
const rawSteps = args.steps as string[] | undefined;
|
|
4035
|
+
const steps = rawSteps?.filter((s) => validSteps.includes(s));
|
|
4036
|
+
if (rawSteps && steps && steps.length < rawSteps.length) {
|
|
4037
|
+
const invalid = rawSteps.filter((s) => !validSteps.includes(s));
|
|
4038
|
+
// Will appear in report.errors via the healthReport
|
|
4039
|
+
console.warn(`Unknown cleanup steps ignored: ${invalid.join(", ")}`);
|
|
4040
|
+
}
|
|
4041
|
+
|
|
4027
4042
|
const report = await runMemoryCleanup(client, workspaceId, projectId, {
|
|
4028
4043
|
dryRun: args.dryRun as boolean | undefined,
|
|
4029
|
-
steps
|
|
4044
|
+
steps,
|
|
4030
4045
|
maxAgeDays: args.maxAgeDays as number | undefined,
|
|
4031
4046
|
minClusterSize: args.minClusterSize as number | undefined,
|
|
4032
4047
|
orphanAgeDays: args.orphanAgeDays as number | undefined,
|