@memtensor/memos-local-openclaw-plugin 0.3.20 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (106) hide show
  1. package/README.md +232 -22
  2. package/dist/capture/index.d.ts +1 -1
  3. package/dist/capture/index.d.ts.map +1 -1
  4. package/dist/capture/index.js +33 -8
  5. package/dist/capture/index.js.map +1 -1
  6. package/dist/index.d.ts +1 -1
  7. package/dist/index.d.ts.map +1 -1
  8. package/dist/index.js +2 -1
  9. package/dist/index.js.map +1 -1
  10. package/dist/ingest/providers/anthropic.d.ts.map +1 -1
  11. package/dist/ingest/providers/anthropic.js +22 -8
  12. package/dist/ingest/providers/anthropic.js.map +1 -1
  13. package/dist/ingest/providers/bedrock.d.ts.map +1 -1
  14. package/dist/ingest/providers/bedrock.js +22 -8
  15. package/dist/ingest/providers/bedrock.js.map +1 -1
  16. package/dist/ingest/providers/gemini.d.ts.map +1 -1
  17. package/dist/ingest/providers/gemini.js +22 -8
  18. package/dist/ingest/providers/gemini.js.map +1 -1
  19. package/dist/ingest/providers/index.d.ts +13 -18
  20. package/dist/ingest/providers/index.d.ts.map +1 -1
  21. package/dist/ingest/providers/index.js +213 -139
  22. package/dist/ingest/providers/index.js.map +1 -1
  23. package/dist/ingest/providers/openai.d.ts +1 -1
  24. package/dist/ingest/providers/openai.d.ts.map +1 -1
  25. package/dist/ingest/providers/openai.js +37 -17
  26. package/dist/ingest/providers/openai.js.map +1 -1
  27. package/dist/ingest/task-processor.d.ts +28 -3
  28. package/dist/ingest/task-processor.d.ts.map +1 -1
  29. package/dist/ingest/task-processor.js +166 -67
  30. package/dist/ingest/task-processor.js.map +1 -1
  31. package/dist/ingest/worker.d.ts.map +1 -1
  32. package/dist/ingest/worker.js +97 -75
  33. package/dist/ingest/worker.js.map +1 -1
  34. package/dist/shared/llm-call.d.ts +26 -0
  35. package/dist/shared/llm-call.d.ts.map +1 -0
  36. package/dist/shared/llm-call.js +163 -0
  37. package/dist/shared/llm-call.js.map +1 -0
  38. package/dist/skill/evaluator.d.ts +0 -3
  39. package/dist/skill/evaluator.d.ts.map +1 -1
  40. package/dist/skill/evaluator.js +34 -59
  41. package/dist/skill/evaluator.js.map +1 -1
  42. package/dist/skill/evolver.d.ts +22 -1
  43. package/dist/skill/evolver.d.ts.map +1 -1
  44. package/dist/skill/evolver.js +191 -32
  45. package/dist/skill/evolver.js.map +1 -1
  46. package/dist/skill/generator.d.ts +0 -3
  47. package/dist/skill/generator.d.ts.map +1 -1
  48. package/dist/skill/generator.js +15 -50
  49. package/dist/skill/generator.js.map +1 -1
  50. package/dist/skill/upgrader.d.ts +0 -2
  51. package/dist/skill/upgrader.d.ts.map +1 -1
  52. package/dist/skill/upgrader.js +4 -39
  53. package/dist/skill/upgrader.js.map +1 -1
  54. package/dist/skill/validator.d.ts +0 -2
  55. package/dist/skill/validator.d.ts.map +1 -1
  56. package/dist/skill/validator.js +14 -44
  57. package/dist/skill/validator.js.map +1 -1
  58. package/dist/storage/sqlite.d.ts +13 -2
  59. package/dist/storage/sqlite.d.ts.map +1 -1
  60. package/dist/storage/sqlite.js +72 -6
  61. package/dist/storage/sqlite.js.map +1 -1
  62. package/dist/tools/memory-get.d.ts.map +1 -1
  63. package/dist/tools/memory-get.js +5 -1
  64. package/dist/tools/memory-get.js.map +1 -1
  65. package/dist/tools/memory-search.d.ts.map +1 -1
  66. package/dist/tools/memory-search.js +5 -0
  67. package/dist/tools/memory-search.js.map +1 -1
  68. package/dist/tools/memory-timeline.d.ts.map +1 -1
  69. package/dist/tools/memory-timeline.js +11 -2
  70. package/dist/tools/memory-timeline.js.map +1 -1
  71. package/dist/types.d.ts +2 -1
  72. package/dist/types.d.ts.map +1 -1
  73. package/dist/types.js +1 -1
  74. package/dist/types.js.map +1 -1
  75. package/dist/viewer/html.d.ts +1 -1
  76. package/dist/viewer/html.d.ts.map +1 -1
  77. package/dist/viewer/html.js +233 -9
  78. package/dist/viewer/html.js.map +1 -1
  79. package/dist/viewer/server.d.ts +5 -0
  80. package/dist/viewer/server.d.ts.map +1 -1
  81. package/dist/viewer/server.js +383 -177
  82. package/dist/viewer/server.js.map +1 -1
  83. package/index.ts +9 -3
  84. package/package.json +2 -1
  85. package/src/capture/index.ts +39 -10
  86. package/src/index.ts +3 -2
  87. package/src/ingest/providers/anthropic.ts +22 -8
  88. package/src/ingest/providers/bedrock.ts +22 -8
  89. package/src/ingest/providers/gemini.ts +22 -8
  90. package/src/ingest/providers/index.ts +192 -142
  91. package/src/ingest/providers/openai.ts +37 -17
  92. package/src/ingest/task-processor.ts +183 -65
  93. package/src/ingest/worker.ts +98 -77
  94. package/src/shared/llm-call.ts +144 -0
  95. package/src/skill/evaluator.ts +35 -64
  96. package/src/skill/evolver.ts +201 -33
  97. package/src/skill/generator.ts +16 -59
  98. package/src/skill/upgrader.ts +5 -43
  99. package/src/skill/validator.ts +15 -47
  100. package/src/storage/sqlite.ts +88 -6
  101. package/src/tools/memory-get.ts +6 -1
  102. package/src/tools/memory-search.ts +6 -0
  103. package/src/tools/memory-timeline.ts +13 -1
  104. package/src/types.ts +2 -1
  105. package/src/viewer/html.ts +233 -9
  106. package/src/viewer/server.ts +368 -187
@@ -194,11 +194,16 @@ export class ViewerServer {
194
194
  else if (p === "/api/tool-metrics") this.serveToolMetrics(res, url);
195
195
  else if (p === "/api/search") this.serveSearch(req, res, url);
196
196
  else if (p === "/api/tasks" && req.method === "GET") this.serveTasks(res, url);
197
+ else if (p.match(/^\/api\/task\/[^/]+\/retry-skill$/) && req.method === "POST") this.handleTaskRetrySkill(req, res, p);
198
+ else if (p.startsWith("/api/task/") && req.method === "DELETE") this.handleTaskDelete(res, p);
199
+ else if (p.startsWith("/api/task/") && req.method === "PUT") this.handleTaskUpdate(req, res, p);
197
200
  else if (p.startsWith("/api/task/") && req.method === "GET") this.serveTaskDetail(res, p);
198
- else if (p === "/api/skills" && req.method === "GET") this.serveSkills(res, url);
201
+ else if (p === "/api/skills" && req.method === "GET") this.serveSkills(res, url);
199
202
  else if (p.match(/^\/api\/skill\/[^/]+\/download$/) && req.method === "GET") this.serveSkillDownload(res, p);
200
203
  else if (p.match(/^\/api\/skill\/[^/]+\/files$/) && req.method === "GET") this.serveSkillFiles(res, p);
201
204
  else if (p.match(/^\/api\/skill\/[^/]+\/visibility$/) && req.method === "PUT") this.handleSkillVisibility(req, res, p);
205
+ else if (p.startsWith("/api/skill/") && req.method === "DELETE") this.handleSkillDelete(res, p);
206
+ else if (p.startsWith("/api/skill/") && req.method === "PUT") this.handleSkillUpdate(req, res, p);
202
207
  else if (p.startsWith("/api/skill/") && req.method === "GET") this.serveSkillDetail(res, p);
203
208
  else if (p === "/api/memory" && req.method === "POST") this.handleCreate(req, res);
204
209
  else if (p.startsWith("/api/memory/") && req.method === "GET") this.serveMemoryDetail(res, p);
@@ -394,16 +399,21 @@ export class ViewerServer {
394
399
  const offset = Math.max(0, Number(url.searchParams.get("offset")) || 0);
395
400
  const { tasks, total } = this.store.listTasks({ status, limit, offset });
396
401
 
397
- const items = tasks.map((t) => ({
398
- id: t.id,
399
- sessionKey: t.sessionKey,
400
- title: t.title,
401
- summary: t.summary ? (t.summary.length > 300 ? t.summary.slice(0, 297) + "..." : t.summary) : "",
402
- status: t.status,
403
- startedAt: t.startedAt,
404
- endedAt: t.endedAt,
405
- chunkCount: this.store.countChunksByTask(t.id),
406
- }));
402
+ const db = (this.store as any).db;
403
+ const items = tasks.map((t) => {
404
+ const meta = db.prepare("SELECT skill_status FROM tasks WHERE id = ?").get(t.id) as { skill_status: string | null } | undefined;
405
+ return {
406
+ id: t.id,
407
+ sessionKey: t.sessionKey,
408
+ title: t.title,
409
+ summary: t.summary ? (t.summary.length > 300 ? t.summary.slice(0, 297) + "..." : t.summary) : "",
410
+ status: t.status,
411
+ startedAt: t.startedAt,
412
+ endedAt: t.endedAt,
413
+ chunkCount: this.store.countChunksByTask(t.id),
414
+ skillStatus: meta?.skill_status ?? null,
415
+ };
416
+ });
407
417
 
408
418
  this.jsonResponse(res, { tasks: items, total, limit, offset });
409
419
  }
@@ -541,15 +551,17 @@ export class ViewerServer {
541
551
  ).all(`%${q}%`, `%${q}%`).filter(passesFilter);
542
552
  }
543
553
 
554
+ const SEMANTIC_THRESHOLD = 0.64;
544
555
  let vectorResults: any[] = [];
556
+ let scoreMap = new Map<string, number>();
545
557
  try {
546
558
  const queryVec = await this.embedder.embedQuery(q);
547
559
  const hits = vectorSearch(this.store, queryVec, 40);
548
- const hitIds = new Set(hits.filter(h => h.score > 0.3).map(h => h.chunkId));
560
+ scoreMap = new Map(hits.map(h => [h.chunkId, h.score]));
561
+ const hitIds = new Set(hits.filter(h => h.score >= SEMANTIC_THRESHOLD).map(h => h.chunkId));
549
562
  if (hitIds.size > 0) {
550
563
  const placeholders = [...hitIds].map(() => "?").join(",");
551
564
  const rows = db.prepare(`SELECT * FROM chunks WHERE id IN (${placeholders})`).all(...hitIds).filter(passesFilter);
552
- const scoreMap = new Map(hits.map(h => [h.chunkId, h.score]));
553
565
  rows.forEach((r: any) => { r._vscore = scoreMap.get(r.id) ?? 0; });
554
566
  rows.sort((a: any, b: any) => (b._vscore ?? 0) - (a._vscore ?? 0));
555
567
  vectorResults = rows;
@@ -564,16 +576,23 @@ export class ViewerServer {
564
576
  if (!seenIds.has(r.id)) { seenIds.add(r.id); merged.push(r); }
565
577
  }
566
578
  for (const r of ftsResults) {
567
- if (!seenIds.has(r.id)) { seenIds.add(r.id); merged.push(r); }
579
+ if (seenIds.has(r.id)) continue;
580
+ const vscore = scoreMap.get(r.id);
581
+ if (vscore !== undefined && vscore < SEMANTIC_THRESHOLD) continue;
582
+ seenIds.add(r.id); merged.push(r);
568
583
  }
569
584
 
585
+ const fallback = merged.length === 0 && ftsResults.length > 0;
586
+ const results = fallback ? ftsResults.slice(0, 20) : merged;
587
+
570
588
  this.store.recordViewerEvent("search");
571
589
  this.jsonResponse(res, {
572
- results: merged,
590
+ results,
573
591
  query: q,
574
592
  vectorCount: vectorResults.length,
575
593
  ftsCount: ftsResults.length,
576
- total: merged.length,
594
+ total: results.length,
595
+ fallbackFts: fallback,
577
596
  });
578
597
  }
579
598
 
@@ -739,19 +758,117 @@ export class ViewerServer {
739
758
  });
740
759
  }
741
760
 
761
+ // ─── Task/Skill management ───
762
+
763
+ private handleTaskRetrySkill(_req: http.IncomingMessage, res: http.ServerResponse, urlPath: string): void {
764
+ const taskId = urlPath.replace("/api/task/", "").replace("/retry-skill", "");
765
+ const task = this.store.getTask(taskId);
766
+ if (!task) { res.writeHead(404, { "Content-Type": "application/json" }); res.end(JSON.stringify({ error: "Task not found" })); return; }
767
+ if (task.status !== "completed") { res.writeHead(400, { "Content-Type": "application/json" }); res.end(JSON.stringify({ error: "Only completed tasks can retry skill generation" })); return; }
768
+ if (!this.ctx) { res.writeHead(500, { "Content-Type": "application/json" }); res.end(JSON.stringify({ error: "Plugin context not available" })); return; }
769
+
770
+ // Clean up stale task_skills references (e.g., skill was manually deleted)
771
+ const db = (this.store as any).db;
772
+ db.prepare("DELETE FROM task_skills WHERE task_id = ? AND skill_id NOT IN (SELECT id FROM skills)").run(taskId);
773
+
774
+ this.store.setTaskSkillMeta(taskId, { skillStatus: "queued", skillReason: "手动重试中..." });
775
+ this.jsonResponse(res, { ok: true, taskId, status: "queued" });
776
+
777
+ const ctx = this.ctx;
778
+ const recallEngine = new RecallEngine(this.store, this.embedder, ctx);
779
+ const evolver = new SkillEvolver(this.store, recallEngine, ctx, this.embedder);
780
+ evolver.onTaskCompleted(task).then(() => {
781
+ this.log.info(`Retry skill generation completed for task ${taskId}`);
782
+ }).catch((err) => {
783
+ this.log.error(`Retry skill generation failed for task ${taskId}: ${err}`);
784
+ this.store.setTaskSkillMeta(taskId, { skillStatus: "skipped", skillReason: `error: ${err}` });
785
+ });
786
+ }
787
+
788
+ private handleTaskDelete(res: http.ServerResponse, urlPath: string): void {
789
+ const taskId = urlPath.replace("/api/task/", "");
790
+ const deleted = this.store.deleteTask(taskId);
791
+ if (!deleted) { res.writeHead(404, { "Content-Type": "application/json" }); res.end(JSON.stringify({ error: "Task not found" })); return; }
792
+ this.jsonResponse(res, { ok: true, taskId });
793
+ }
794
+
795
+ private handleTaskUpdate(req: http.IncomingMessage, res: http.ServerResponse, urlPath: string): void {
796
+ const taskId = urlPath.replace("/api/task/", "");
797
+ this.readBody(req, (body) => {
798
+ try {
799
+ const data = JSON.parse(body);
800
+ const task = this.store.getTask(taskId);
801
+ if (!task) { res.writeHead(404, { "Content-Type": "application/json" }); res.end(JSON.stringify({ error: "Task not found" })); return; }
802
+ this.store.updateTask(taskId, {
803
+ title: data.title ?? task.title,
804
+ summary: data.summary ?? task.summary,
805
+ status: data.status ?? task.status,
806
+ endedAt: task.endedAt ?? undefined,
807
+ });
808
+ this.jsonResponse(res, { ok: true, taskId });
809
+ } catch (err) {
810
+ res.writeHead(400, { "Content-Type": "application/json" });
811
+ res.end(JSON.stringify({ error: String(err) }));
812
+ }
813
+ });
814
+ }
815
+
816
+ private handleSkillDelete(res: http.ServerResponse, urlPath: string): void {
817
+ const skillId = urlPath.replace("/api/skill/", "");
818
+ const skill = this.store.getSkill(skillId);
819
+ if (!skill) { res.writeHead(404, { "Content-Type": "application/json" }); res.end(JSON.stringify({ error: "Skill not found" })); return; }
820
+ // Remove skill directory from disk
821
+ try {
822
+ if (skill.dirPath && fs.existsSync(skill.dirPath)) {
823
+ fs.rmSync(skill.dirPath, { recursive: true, force: true });
824
+ }
825
+ } catch (err) {
826
+ this.log.warn(`Failed to remove skill directory ${skill.dirPath}: ${err}`);
827
+ }
828
+ this.store.deleteSkill(skillId);
829
+ this.jsonResponse(res, { ok: true, skillId });
830
+ }
831
+
832
+ private handleSkillUpdate(req: http.IncomingMessage, res: http.ServerResponse, urlPath: string): void {
833
+ const skillId = urlPath.replace("/api/skill/", "");
834
+ this.readBody(req, (body) => {
835
+ try {
836
+ const data = JSON.parse(body);
837
+ const skill = this.store.getSkill(skillId);
838
+ if (!skill) { res.writeHead(404, { "Content-Type": "application/json" }); res.end(JSON.stringify({ error: "Skill not found" })); return; }
839
+ this.store.updateSkill(skillId, {
840
+ description: data.description ?? skill.description,
841
+ version: skill.version,
842
+ status: data.status ?? skill.status,
843
+ installed: skill.installed,
844
+ qualityScore: skill.qualityScore,
845
+ });
846
+ this.jsonResponse(res, { ok: true, skillId });
847
+ } catch (err) {
848
+ res.writeHead(400, { "Content-Type": "application/json" });
849
+ res.end(JSON.stringify({ error: String(err) }));
850
+ }
851
+ });
852
+ }
853
+
742
854
  // ─── CRUD ───
743
855
 
744
856
  private handleCreate(req: http.IncomingMessage, res: http.ServerResponse): void {
745
857
  this.readBody(req, (body) => {
746
858
  try {
747
859
  const data = JSON.parse(body);
860
+ if (!data.content || typeof data.content !== "string" || !data.content.trim()) {
861
+ res.writeHead(400, { "Content-Type": "application/json" });
862
+ res.end(JSON.stringify({ error: "content is required and must be a non-empty string" }));
863
+ return;
864
+ }
748
865
  const { v4: uuidv4 } = require("uuid");
749
866
  const id = uuidv4();
750
867
  const now = Date.now();
751
868
  this.store.insertChunk({
752
869
  id, sessionKey: data.session_key || "manual", turnId: `manual-${now}`, seq: 0,
753
- role: data.role || "user", content: data.content || "", kind: data.kind || "paragraph",
754
- summary: data.summary || data.content?.slice(0, 100) || "",
870
+ role: data.role || "user", content: data.content, kind: data.kind || "paragraph",
871
+ summary: data.summary || data.content.slice(0, 100),
755
872
  taskId: null, skillId: null, owner: data.owner || "agent:main",
756
873
  dedupStatus: "active", dedupTarget: null, dedupReason: null,
757
874
  mergeCount: 0, lastHitAt: null, mergeHistory: "[]",
@@ -784,6 +901,11 @@ export class ViewerServer {
784
901
  this.readBody(req, (body) => {
785
902
  try {
786
903
  const data = JSON.parse(body);
904
+ if (data.content !== undefined && (typeof data.content !== "string" || !data.content.trim())) {
905
+ res.writeHead(400, { "Content-Type": "application/json" });
906
+ res.end(JSON.stringify({ error: "content must be a non-empty string" }));
907
+ return;
908
+ }
787
909
  const ok = this.store.updateChunk(chunkId, { summary: data.summary, content: data.content, role: data.role, kind: data.kind, owner: data.owner });
788
910
  if (ok) this.jsonResponse(res, { ok: true, message: "Memory updated" });
789
911
  else { res.writeHead(404, { "Content-Type": "application/json" }); res.end(JSON.stringify({ error: "Not found" })); }
@@ -1092,9 +1214,11 @@ export class ViewerServer {
1092
1214
  }
1093
1215
 
1094
1216
  this.readBody(req, (body) => {
1095
- let opts: { sources?: string[] } = {};
1217
+ let opts: { sources?: string[]; concurrency?: number } = {};
1096
1218
  try { opts = JSON.parse(body); } catch { /* defaults */ }
1097
1219
 
1220
+ const concurrency = Math.max(1, Math.min(opts.concurrency ?? 1, 8));
1221
+
1098
1222
  res.writeHead(200, {
1099
1223
  "Content-Type": "text/event-stream",
1100
1224
  "Cache-Control": "no-cache",
@@ -1129,7 +1253,7 @@ export class ViewerServer {
1129
1253
  };
1130
1254
 
1131
1255
  this.migrationRunning = true;
1132
- this.runMigration(send, opts.sources).finally(() => {
1256
+ this.runMigration(send, opts.sources, concurrency).finally(() => {
1133
1257
  this.migrationRunning = false;
1134
1258
  this.migrationState.done = true;
1135
1259
  if (this.migrationAbort) {
@@ -1150,6 +1274,7 @@ export class ViewerServer {
1150
1274
  private async runMigration(
1151
1275
  send: (event: string, data: unknown) => void,
1152
1276
  sources?: string[],
1277
+ concurrency: number = 1,
1153
1278
  ): Promise<void> {
1154
1279
  const ocHome = this.getOpenClawHome();
1155
1280
  const importSqlite = !sources || sources.includes("sqlite");
@@ -1162,15 +1287,17 @@ export class ViewerServer {
1162
1287
 
1163
1288
  const cfgPath = this.getOpenClawConfigPath();
1164
1289
  let summarizerCfg: any;
1290
+ let strongCfg: any;
1165
1291
  try {
1166
1292
  const raw = JSON.parse(fs.readFileSync(cfgPath, "utf-8"));
1167
1293
  const pluginCfg = raw?.plugins?.entries?.["memos-local-openclaw-plugin"]?.config ??
1168
1294
  raw?.plugins?.entries?.["memos-lite"]?.config ??
1169
1295
  raw?.plugins?.entries?.["memos-lite-openclaw-plugin"]?.config ?? {};
1170
1296
  summarizerCfg = pluginCfg.summarizer;
1297
+ strongCfg = pluginCfg.skillEvolution?.summarizer;
1171
1298
  } catch { /* no config */ }
1172
1299
 
1173
- const summarizer = new Summarizer(summarizerCfg, this.log);
1300
+ const summarizer = new Summarizer(summarizerCfg, this.log, strongCfg);
1174
1301
 
1175
1302
  // Phase 1: Import SQLite memory chunks
1176
1303
  if (importSqlite) {
@@ -1210,6 +1337,23 @@ export class ViewerServer {
1210
1337
  continue;
1211
1338
  }
1212
1339
 
1340
+ const importOwner = `agent:${agentId}`;
1341
+
1342
+ // Exact hash dedup within same agent
1343
+ const existingByHash = this.store.findActiveChunkByHash(row.text, importOwner);
1344
+ if (existingByHash) {
1345
+ totalSkipped++;
1346
+ send("item", {
1347
+ index: i + 1,
1348
+ total: rows.length,
1349
+ status: "skipped",
1350
+ preview: row.text.slice(0, 120),
1351
+ source: file,
1352
+ reason: "exact duplicate within agent",
1353
+ });
1354
+ continue;
1355
+ }
1356
+
1213
1357
  try {
1214
1358
  const summary = await summarizer.summarize(row.text);
1215
1359
  let embedding: number[] | null = null;
@@ -1224,7 +1368,9 @@ export class ViewerServer {
1224
1368
  let dedupReason: string | null = null;
1225
1369
 
1226
1370
  if (embedding) {
1227
- const topSimilar = findTopSimilar(this.store, embedding, 0.85, 3, this.log);
1371
+ const importThreshold = this.ctx?.config?.dedup?.similarityThreshold ?? 0.60;
1372
+ const dedupOwnerFilter = [importOwner];
1373
+ const topSimilar = findTopSimilar(this.store, embedding, importThreshold, 5, this.log, dedupOwnerFilter);
1228
1374
  if (topSimilar.length > 0) {
1229
1375
  const candidates = topSimilar.map((s, idx) => {
1230
1376
  const chunk = this.store.getChunk(s.chunkId);
@@ -1315,18 +1461,34 @@ export class ViewerServer {
1315
1461
  }
1316
1462
  }
1317
1463
 
1318
- // Phase 2: Import session JSONL files
1464
+ // Phase 2: Import session JSONL files from ALL agents (supports parallel by agent)
1319
1465
  if (importSessions) {
1320
- const sessionsDir = path.join(ocHome, "agents", "main", "sessions");
1321
- if (fs.existsSync(sessionsDir)) {
1322
- const jsonlFiles = fs.readdirSync(sessionsDir).filter(f => f.includes(".jsonl")).sort();
1323
- send("phase", { phase: "sessions", files: jsonlFiles.length });
1466
+ const agentsDir = path.join(ocHome, "agents");
1467
+ const agentGroups: Map<string, Array<{ file: string; filePath: string }>> = new Map();
1468
+ if (fs.existsSync(agentsDir)) {
1469
+ for (const entry of fs.readdirSync(agentsDir, { withFileTypes: true })) {
1470
+ if (entry.isDirectory()) {
1471
+ const sessDir = path.join(agentsDir, entry.name, "sessions");
1472
+ if (fs.existsSync(sessDir)) {
1473
+ const jsonlFiles = fs.readdirSync(sessDir).filter(f => f.includes(".jsonl")).sort();
1474
+ if (jsonlFiles.length > 0) {
1475
+ agentGroups.set(entry.name, jsonlFiles.map(f => ({ file: f, filePath: path.join(sessDir, f) })));
1476
+ }
1477
+ }
1478
+ }
1479
+ }
1480
+ }
1324
1481
 
1325
- let globalMsgIdx = 0;
1326
- let totalMsgs = 0;
1327
- for (const f of jsonlFiles) {
1482
+ const agentIds = Array.from(agentGroups.keys());
1483
+ const allFileCount = Array.from(agentGroups.values()).reduce((s, g) => s + g.length, 0);
1484
+ send("phase", { phase: "sessions", files: allFileCount, agents: agentIds, concurrency });
1485
+
1486
+ // Count total messages across all agents
1487
+ let totalMsgs = 0;
1488
+ for (const files of agentGroups.values()) {
1489
+ for (const { filePath } of files) {
1328
1490
  try {
1329
- const raw = fs.readFileSync(path.join(sessionsDir, f), "utf-8");
1491
+ const raw = fs.readFileSync(filePath, "utf-8");
1330
1492
  for (const line of raw.split("\n")) {
1331
1493
  if (!line.trim()) continue;
1332
1494
  try {
@@ -1347,12 +1509,18 @@ export class ViewerServer {
1347
1509
  }
1348
1510
  } catch { /* skip */ }
1349
1511
  }
1512
+ }
1350
1513
 
1351
- for (const file of jsonlFiles) {
1514
+ // Thread-safe counters for parallel execution
1515
+ let globalMsgIdx = 0;
1516
+ const incIdx = () => ++globalMsgIdx;
1517
+
1518
+ // Import one agent's sessions sequentially
1519
+ const importAgent = async (agentId: string, files: Array<{ file: string; filePath: string }>) => {
1520
+ const agentOwner = `agent:${agentId}`;
1521
+ for (const { file, filePath } of files) {
1352
1522
  if (this.migrationAbort) break;
1353
1523
  const sessionId = file.replace(/\.jsonl.*$/, "");
1354
- const filePath = path.join(sessionsDir, file);
1355
- send("progress", { total: totalMsgs, processed: globalMsgIdx, phase: "sessions", file });
1356
1524
 
1357
1525
  try {
1358
1526
  const fileStream = fs.createReadStream(filePath, { encoding: "utf-8" });
@@ -1384,21 +1552,20 @@ export class ViewerServer {
1384
1552
  }
1385
1553
  if (!content || content.length < 10) continue;
1386
1554
 
1387
- globalMsgIdx++;
1555
+ const idx = incIdx();
1388
1556
  totalProcessed++;
1389
1557
 
1390
1558
  const sessionKey = `openclaw-session-${sessionId}`;
1391
1559
  if (this.store.chunkExistsByContent(sessionKey, msgRole, content)) {
1392
1560
  totalSkipped++;
1393
- send("item", {
1394
- index: globalMsgIdx,
1395
- total: totalMsgs,
1396
- status: "skipped",
1397
- preview: content.slice(0, 120),
1398
- source: file,
1399
- role: msgRole,
1400
- reason: "duplicate",
1401
- });
1561
+ send("item", { index: idx, total: totalMsgs, status: "skipped", preview: content.slice(0, 120), source: file, agent: agentId, role: msgRole, reason: "duplicate" });
1562
+ continue;
1563
+ }
1564
+
1565
+ const existingByHash = this.store.findActiveChunkByHash(content, agentOwner);
1566
+ if (existingByHash) {
1567
+ totalSkipped++;
1568
+ send("item", { index: idx, total: totalMsgs, status: "skipped", preview: content.slice(0, 120), source: file, agent: agentId, role: msgRole, reason: "exact duplicate within agent" });
1402
1569
  continue;
1403
1570
  }
1404
1571
 
@@ -1416,33 +1583,26 @@ export class ViewerServer {
1416
1583
  let dedupReason: string | null = null;
1417
1584
 
1418
1585
  if (embedding) {
1419
- const topSimilar = findTopSimilar(this.store, embedding, 0.85, 3, this.log);
1586
+ const importThreshold = this.ctx?.config?.dedup?.similarityThreshold ?? 0.60;
1587
+ const dedupOwnerFilter = [agentOwner];
1588
+ const topSimilar = findTopSimilar(this.store, embedding, importThreshold, 5, this.log, dedupOwnerFilter);
1420
1589
  if (topSimilar.length > 0) {
1421
- const candidates = topSimilar.map((s, idx) => {
1590
+ const candidates = topSimilar.map((s, i) => {
1422
1591
  const chunk = this.store.getChunk(s.chunkId);
1423
- return { index: idx + 1, summary: chunk?.summary ?? "", chunkId: s.chunkId };
1592
+ return { index: i + 1, summary: chunk?.summary ?? "", chunkId: s.chunkId };
1424
1593
  }).filter(c => c.summary);
1425
1594
 
1426
1595
  if (candidates.length > 0) {
1427
1596
  const dedupResult = await summarizer.judgeDedup(summary, candidates);
1428
1597
  if (dedupResult?.action === "DUPLICATE" && dedupResult.targetIndex) {
1429
1598
  const targetId = candidates[dedupResult.targetIndex - 1]?.chunkId;
1430
- if (targetId) {
1431
- dedupStatus = "duplicate";
1432
- dedupTarget = targetId;
1433
- dedupReason = dedupResult.reason;
1434
- }
1599
+ if (targetId) { dedupStatus = "duplicate"; dedupTarget = targetId; dedupReason = dedupResult.reason; }
1435
1600
  } else if (dedupResult?.action === "UPDATE" && dedupResult.targetIndex && dedupResult.mergedSummary) {
1436
1601
  const targetId = candidates[dedupResult.targetIndex - 1]?.chunkId;
1437
1602
  if (targetId) {
1438
1603
  this.store.updateChunkSummaryAndContent(targetId, dedupResult.mergedSummary, content);
1439
- try {
1440
- const [newEmb] = await this.embedder.embed([dedupResult.mergedSummary]);
1441
- if (newEmb) this.store.upsertEmbedding(targetId, newEmb);
1442
- } catch { /* best-effort */ }
1443
- dedupStatus = "merged";
1444
- dedupTarget = targetId;
1445
- dedupReason = dedupResult.reason;
1604
+ try { const [newEmb] = await this.embedder.embed([dedupResult.mergedSummary]); if (newEmb) this.store.upsertEmbedding(targetId, newEmb); } catch { /* best-effort */ }
1605
+ dedupStatus = "merged"; dedupTarget = targetId; dedupReason = dedupResult.reason;
1446
1606
  }
1447
1607
  }
1448
1608
  }
@@ -1453,60 +1613,53 @@ export class ViewerServer {
1453
1613
  const msgTs = obj.message?.timestamp ?? obj.timestamp;
1454
1614
  const ts = msgTs ? new Date(msgTs).getTime() : Date.now();
1455
1615
  const chunk: Chunk = {
1456
- id: chunkId,
1457
- sessionKey,
1458
- turnId: `import-${sessionId}-${globalMsgIdx}`,
1459
- seq: 0,
1460
- role: msgRole as any,
1461
- content,
1462
- kind: "paragraph",
1463
- summary,
1464
- embedding: null,
1465
- taskId: null,
1466
- skillId: null,
1467
- owner: "agent:main",
1468
- dedupStatus,
1469
- dedupTarget,
1470
- dedupReason,
1471
- mergeCount: 0,
1472
- lastHitAt: null,
1473
- mergeHistory: "[]",
1474
- createdAt: ts,
1475
- updatedAt: ts,
1616
+ id: chunkId, sessionKey, turnId: `import-${agentId}-${sessionId}-${idx}`, seq: 0,
1617
+ role: msgRole as any, content, kind: "paragraph", summary, embedding: null,
1618
+ taskId: null, skillId: null, owner: agentOwner, dedupStatus, dedupTarget, dedupReason,
1619
+ mergeCount: 0, lastHitAt: null, mergeHistory: "[]", createdAt: ts, updatedAt: ts,
1476
1620
  };
1477
1621
 
1478
1622
  this.store.insertChunk(chunk);
1479
- if (embedding && dedupStatus === "active") {
1480
- this.store.upsertEmbedding(chunkId, embedding);
1481
- }
1623
+ if (embedding && dedupStatus === "active") this.store.upsertEmbedding(chunkId, embedding);
1482
1624
 
1483
1625
  totalStored++;
1484
- send("item", {
1485
- index: globalMsgIdx,
1486
- total: totalMsgs,
1487
- status: dedupStatus === "active" ? "stored" : dedupStatus,
1488
- preview: content.slice(0, 120),
1489
- summary: summary.slice(0, 80),
1490
- source: file,
1491
- role: msgRole,
1492
- });
1626
+ send("item", { index: idx, total: totalMsgs, status: dedupStatus === "active" ? "stored" : dedupStatus, preview: content.slice(0, 120), summary: summary.slice(0, 80), source: file, agent: agentId, role: msgRole });
1493
1627
  } catch (err) {
1494
1628
  totalErrors++;
1495
- send("item", {
1496
- index: globalMsgIdx,
1497
- total: totalMsgs,
1498
- status: "error",
1499
- preview: content.slice(0, 120),
1500
- source: file,
1501
- error: String(err).slice(0, 200),
1502
- });
1629
+ send("item", { index: idx, total: totalMsgs, status: "error", preview: content.slice(0, 120), source: file, agent: agentId, error: String(err).slice(0, 200) });
1503
1630
  }
1504
1631
  }
1505
1632
  } catch (err) {
1506
- send("error", { file, error: String(err) });
1633
+ send("error", { file, agent: agentId, error: String(err) });
1507
1634
  totalErrors++;
1508
1635
  }
1509
1636
  }
1637
+ };
1638
+
1639
+ // Execute agents with concurrency control
1640
+ const agentEntries = Array.from(agentGroups.entries());
1641
+ if (concurrency <= 1 || agentEntries.length <= 1) {
1642
+ for (const [agentId, files] of agentEntries) {
1643
+ if (this.migrationAbort) break;
1644
+ send("progress", { total: totalMsgs, processed: globalMsgIdx, phase: "sessions", agent: agentId });
1645
+ await importAgent(agentId, files);
1646
+ }
1647
+ } else {
1648
+ // Parallel: run up to `concurrency` agents at once
1649
+ let cursor = 0;
1650
+ const runBatch = async () => {
1651
+ while (cursor < agentEntries.length && !this.migrationAbort) {
1652
+ const batch: Promise<void>[] = [];
1653
+ const batchStart = cursor;
1654
+ while (batch.length < concurrency && cursor < agentEntries.length) {
1655
+ const [agentId, files] = agentEntries[cursor++];
1656
+ send("progress", { total: totalMsgs, processed: globalMsgIdx, phase: "sessions", agent: agentId, parallel: true });
1657
+ batch.push(importAgent(agentId, files));
1658
+ }
1659
+ await Promise.all(batch);
1660
+ }
1661
+ };
1662
+ await runBatch();
1510
1663
  }
1511
1664
  }
1512
1665
 
@@ -1529,9 +1682,11 @@ export class ViewerServer {
1529
1682
  }
1530
1683
 
1531
1684
  this.readBody(req, (body) => {
1532
- let opts: { enableTasks?: boolean; enableSkills?: boolean } = {};
1685
+ let opts: { enableTasks?: boolean; enableSkills?: boolean; concurrency?: number } = {};
1533
1686
  try { opts = JSON.parse(body); } catch { /* defaults */ }
1534
1687
 
1688
+ const concurrency = Math.max(1, Math.min(opts.concurrency ?? 1, 8));
1689
+
1535
1690
  res.writeHead(200, {
1536
1691
  "Content-Type": "text/event-stream",
1537
1692
  "Cache-Control": "no-cache",
@@ -1550,7 +1705,7 @@ export class ViewerServer {
1550
1705
  };
1551
1706
 
1552
1707
  this.ppRunning = true;
1553
- this.runPostprocess(send, !!opts.enableTasks, !!opts.enableSkills).finally(() => {
1708
+ this.runPostprocess(send, !!opts.enableTasks, !!opts.enableSkills, concurrency).finally(() => {
1554
1709
  this.ppRunning = false;
1555
1710
  this.ppState.running = false;
1556
1711
  this.ppState.done = true;
@@ -1608,128 +1763,154 @@ export class ViewerServer {
1608
1763
  send: (event: string, data: unknown) => void,
1609
1764
  enableTasks: boolean,
1610
1765
  enableSkills: boolean,
1766
+ concurrency: number = 1,
1611
1767
  ): Promise<void> {
1612
1768
  const ctx = this.ctx!;
1613
- const taskProcessor = new TaskProcessor(this.store, ctx);
1614
- let skillEvolver: SkillEvolver | null = null;
1615
-
1616
- if (enableSkills) {
1617
- const recallEngine = new RecallEngine(this.store, this.embedder, ctx);
1618
- skillEvolver = new SkillEvolver(this.store, recallEngine, ctx);
1619
- taskProcessor.onTaskCompleted(async (task) => {
1620
- try {
1621
- await skillEvolver!.onTaskCompleted(task);
1622
- this.ppState.skillsCreated++;
1623
- send("skill", { taskId: task.id, title: task.title });
1624
- } catch (err) {
1625
- this.log.warn(`Postprocess skill evolution error: ${err}`);
1626
- }
1627
- });
1628
- }
1629
1769
 
1630
1770
  const importSessions = this.store.getDistinctSessionKeys()
1631
1771
  .filter((sk: string) => sk.startsWith("openclaw-import-") || sk.startsWith("openclaw-session-"));
1632
1772
 
1633
- type PendingItem = { sessionKey: string; action: "full" | "skill-only" };
1773
+ type PendingItem = { sessionKey: string; action: "full" | "skill-only"; owner: string };
1634
1774
  const pendingItems: PendingItem[] = [];
1635
1775
  let skippedCount = 0;
1636
1776
 
1777
+ const ownerMap = this.store.getSessionOwnerMap(importSessions);
1778
+
1637
1779
  for (const sk of importSessions) {
1638
1780
  const hasTask = this.store.hasTaskForSession(sk);
1639
1781
  const hasSkill = this.store.hasSkillForSessionTask(sk);
1782
+ const owner = ownerMap.get(sk) ?? "agent:main";
1640
1783
 
1641
1784
  if (enableTasks && !hasTask) {
1642
- pendingItems.push({ sessionKey: sk, action: "full" });
1785
+ pendingItems.push({ sessionKey: sk, action: "full", owner });
1643
1786
  } else if (enableSkills && hasTask && !hasSkill) {
1644
- pendingItems.push({ sessionKey: sk, action: "skill-only" });
1787
+ pendingItems.push({ sessionKey: sk, action: "skill-only", owner });
1645
1788
  } else {
1646
1789
  skippedCount++;
1647
1790
  }
1648
1791
  }
1649
1792
 
1793
+ // Group pending items by agent (owner)
1794
+ const agentGroups = new Map<string, PendingItem[]>();
1795
+ for (const item of pendingItems) {
1796
+ const group = agentGroups.get(item.owner) ?? [];
1797
+ group.push(item);
1798
+ agentGroups.set(item.owner, group);
1799
+ }
1800
+
1650
1801
  this.ppState.total = pendingItems.length;
1651
1802
  send("info", {
1652
1803
  totalSessions: importSessions.length,
1653
1804
  alreadyProcessed: skippedCount,
1654
1805
  pending: pendingItems.length,
1806
+ agents: Array.from(agentGroups.keys()),
1807
+ concurrency,
1655
1808
  });
1656
1809
  send("progress", { processed: 0, total: pendingItems.length });
1657
1810
 
1658
- for (let i = 0; i < pendingItems.length; i++) {
1659
- if (this.ppAbort) break;
1660
- const { sessionKey, action } = pendingItems[i];
1661
- this.ppState.processed = i + 1;
1662
-
1663
- send("item", {
1664
- index: i + 1,
1665
- total: pendingItems.length,
1666
- session: sessionKey,
1667
- step: "processing",
1668
- action,
1669
- });
1811
+ let globalIdx = 0;
1812
+ const incIdx = () => ++globalIdx;
1670
1813
 
1671
- try {
1672
- if (action === "full") {
1673
- await taskProcessor.onChunksIngested(sessionKey, Date.now());
1674
- const activeTask = this.store.getActiveTask(sessionKey);
1675
- if (activeTask) {
1676
- await taskProcessor.finalizeTask(activeTask);
1677
- const finalized = this.store.getTask(activeTask.id);
1678
- this.ppState.tasksCreated++;
1679
- send("item", {
1680
- index: i + 1,
1681
- total: pendingItems.length,
1682
- session: sessionKey,
1683
- step: "done",
1684
- taskTitle: finalized?.title || "",
1685
- taskStatus: finalized?.status || "",
1686
- });
1687
- } else {
1688
- send("item", {
1689
- index: i + 1,
1690
- total: pendingItems.length,
1691
- session: sessionKey,
1692
- step: "done",
1693
- taskTitle: "(no chunks)",
1694
- });
1814
+ // Process one agent's sessions sequentially
1815
+ const processAgent = async (agentOwner: string, items: PendingItem[]) => {
1816
+ const taskProcessor = new TaskProcessor(this.store, ctx);
1817
+ let skillEvolver: SkillEvolver | null = null;
1818
+
1819
+ if (enableSkills) {
1820
+ const recallEngine = new RecallEngine(this.store, this.embedder, ctx);
1821
+ skillEvolver = new SkillEvolver(this.store, recallEngine, ctx);
1822
+ taskProcessor.onTaskCompleted(async (task) => {
1823
+ try {
1824
+ await skillEvolver!.onTaskCompleted(task);
1825
+ this.ppState.skillsCreated++;
1826
+ send("skill", { taskId: task.id, title: task.title, agent: agentOwner });
1827
+ } catch (err) {
1828
+ this.log.warn(`Postprocess skill evolution error (${agentOwner}): ${err}`);
1695
1829
  }
1696
- } else if (action === "skill-only" && skillEvolver) {
1697
- const completedTasks = this.store.getCompletedTasksForSession(sessionKey);
1698
- let skillGenerated = false;
1699
- for (const task of completedTasks) {
1700
- if (this.ppAbort) break;
1701
- try {
1702
- await skillEvolver.onTaskCompleted(task);
1703
- this.ppState.skillsCreated++;
1704
- skillGenerated = true;
1705
- send("skill", { taskId: task.id, title: task.title });
1706
- } catch (err) {
1707
- this.log.warn(`Skill evolution error for task=${task.id}: ${err}`);
1830
+ });
1831
+ }
1832
+
1833
+ for (const { sessionKey, action } of items) {
1834
+ if (this.ppAbort) break;
1835
+ const idx = incIdx();
1836
+ this.ppState.processed = globalIdx;
1837
+
1838
+ send("item", {
1839
+ index: idx,
1840
+ total: pendingItems.length,
1841
+ session: sessionKey,
1842
+ agent: agentOwner,
1843
+ step: "processing",
1844
+ action,
1845
+ });
1846
+
1847
+ try {
1848
+ if (action === "full") {
1849
+ await taskProcessor.onChunksIngested(sessionKey, Date.now());
1850
+ const activeTask = this.store.getActiveTask(sessionKey);
1851
+ if (activeTask) {
1852
+ await taskProcessor.finalizeTask(activeTask);
1853
+ const finalized = this.store.getTask(activeTask.id);
1854
+ this.ppState.tasksCreated++;
1855
+ send("item", {
1856
+ index: idx, total: pendingItems.length, session: sessionKey, agent: agentOwner,
1857
+ step: "done", taskTitle: finalized?.title || "", taskStatus: finalized?.status || "",
1858
+ });
1859
+ } else {
1860
+ send("item", {
1861
+ index: idx, total: pendingItems.length, session: sessionKey, agent: agentOwner,
1862
+ step: "done", taskTitle: "(no chunks)",
1863
+ });
1864
+ }
1865
+ } else if (action === "skill-only" && skillEvolver) {
1866
+ const completedTasks = this.store.getCompletedTasksForSession(sessionKey);
1867
+ let skillGenerated = false;
1868
+ for (const task of completedTasks) {
1869
+ if (this.ppAbort) break;
1870
+ try {
1871
+ await skillEvolver.onTaskCompleted(task);
1872
+ this.ppState.skillsCreated++;
1873
+ skillGenerated = true;
1874
+ send("skill", { taskId: task.id, title: task.title, agent: agentOwner });
1875
+ } catch (err) {
1876
+ this.log.warn(`Skill evolution error (${agentOwner}) task=${task.id}: ${err}`);
1877
+ }
1708
1878
  }
1879
+ send("item", {
1880
+ index: idx, total: pendingItems.length, session: sessionKey, agent: agentOwner,
1881
+ step: "done", taskTitle: completedTasks[0]?.title || sessionKey, action: "skill-only", skillGenerated,
1882
+ });
1709
1883
  }
1884
+ } catch (err) {
1885
+ this.ppState.errors++;
1886
+ this.log.warn(`Postprocess error (${agentOwner}) ${sessionKey}: ${err}`);
1710
1887
  send("item", {
1711
- index: i + 1,
1712
- total: pendingItems.length,
1713
- session: sessionKey,
1714
- step: "done",
1715
- taskTitle: completedTasks[0]?.title || sessionKey,
1716
- action: "skill-only",
1717
- skillGenerated,
1888
+ index: idx, total: pendingItems.length, session: sessionKey, agent: agentOwner,
1889
+ step: "error", error: String(err).slice(0, 200),
1718
1890
  });
1719
1891
  }
1720
- } catch (err) {
1721
- this.ppState.errors++;
1722
- this.log.warn(`Postprocess error for ${sessionKey}: ${err}`);
1723
- send("item", {
1724
- index: i + 1,
1725
- total: pendingItems.length,
1726
- session: sessionKey,
1727
- step: "error",
1728
- error: String(err).slice(0, 200),
1729
- });
1892
+
1893
+ send("progress", { processed: globalIdx, total: pendingItems.length });
1730
1894
  }
1895
+ };
1731
1896
 
1732
- send("progress", { processed: i + 1, total: pendingItems.length });
1897
+ // Execute agents with concurrency control
1898
+ const agentEntries = Array.from(agentGroups.entries());
1899
+ if (concurrency <= 1 || agentEntries.length <= 1) {
1900
+ for (const [agentOwner, items] of agentEntries) {
1901
+ if (this.ppAbort) break;
1902
+ await processAgent(agentOwner, items);
1903
+ }
1904
+ } else {
1905
+ let cursor = 0;
1906
+ while (cursor < agentEntries.length && !this.ppAbort) {
1907
+ const batch: Promise<void>[] = [];
1908
+ while (batch.length < concurrency && cursor < agentEntries.length) {
1909
+ const [agentOwner, items] = agentEntries[cursor++];
1910
+ batch.push(processAgent(agentOwner, items));
1911
+ }
1912
+ await Promise.all(batch);
1913
+ }
1733
1914
  }
1734
1915
  }
1735
1916