@memtensor/memos-local-openclaw-plugin 0.3.19 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (106) hide show
  1. package/README.md +232 -22
  2. package/dist/capture/index.d.ts +1 -1
  3. package/dist/capture/index.d.ts.map +1 -1
  4. package/dist/capture/index.js +33 -8
  5. package/dist/capture/index.js.map +1 -1
  6. package/dist/index.d.ts +1 -1
  7. package/dist/index.d.ts.map +1 -1
  8. package/dist/index.js +2 -1
  9. package/dist/index.js.map +1 -1
  10. package/dist/ingest/providers/anthropic.d.ts.map +1 -1
  11. package/dist/ingest/providers/anthropic.js +22 -8
  12. package/dist/ingest/providers/anthropic.js.map +1 -1
  13. package/dist/ingest/providers/bedrock.d.ts.map +1 -1
  14. package/dist/ingest/providers/bedrock.js +22 -8
  15. package/dist/ingest/providers/bedrock.js.map +1 -1
  16. package/dist/ingest/providers/gemini.d.ts.map +1 -1
  17. package/dist/ingest/providers/gemini.js +22 -8
  18. package/dist/ingest/providers/gemini.js.map +1 -1
  19. package/dist/ingest/providers/index.d.ts +13 -18
  20. package/dist/ingest/providers/index.d.ts.map +1 -1
  21. package/dist/ingest/providers/index.js +213 -139
  22. package/dist/ingest/providers/index.js.map +1 -1
  23. package/dist/ingest/providers/openai.d.ts +1 -1
  24. package/dist/ingest/providers/openai.d.ts.map +1 -1
  25. package/dist/ingest/providers/openai.js +37 -17
  26. package/dist/ingest/providers/openai.js.map +1 -1
  27. package/dist/ingest/task-processor.d.ts +28 -3
  28. package/dist/ingest/task-processor.d.ts.map +1 -1
  29. package/dist/ingest/task-processor.js +166 -67
  30. package/dist/ingest/task-processor.js.map +1 -1
  31. package/dist/ingest/worker.d.ts.map +1 -1
  32. package/dist/ingest/worker.js +97 -75
  33. package/dist/ingest/worker.js.map +1 -1
  34. package/dist/shared/llm-call.d.ts +26 -0
  35. package/dist/shared/llm-call.d.ts.map +1 -0
  36. package/dist/shared/llm-call.js +163 -0
  37. package/dist/shared/llm-call.js.map +1 -0
  38. package/dist/skill/evaluator.d.ts +0 -3
  39. package/dist/skill/evaluator.d.ts.map +1 -1
  40. package/dist/skill/evaluator.js +34 -59
  41. package/dist/skill/evaluator.js.map +1 -1
  42. package/dist/skill/evolver.d.ts +22 -1
  43. package/dist/skill/evolver.d.ts.map +1 -1
  44. package/dist/skill/evolver.js +191 -32
  45. package/dist/skill/evolver.js.map +1 -1
  46. package/dist/skill/generator.d.ts +0 -3
  47. package/dist/skill/generator.d.ts.map +1 -1
  48. package/dist/skill/generator.js +15 -50
  49. package/dist/skill/generator.js.map +1 -1
  50. package/dist/skill/upgrader.d.ts +0 -2
  51. package/dist/skill/upgrader.d.ts.map +1 -1
  52. package/dist/skill/upgrader.js +4 -39
  53. package/dist/skill/upgrader.js.map +1 -1
  54. package/dist/skill/validator.d.ts +0 -2
  55. package/dist/skill/validator.d.ts.map +1 -1
  56. package/dist/skill/validator.js +14 -44
  57. package/dist/skill/validator.js.map +1 -1
  58. package/dist/storage/sqlite.d.ts +13 -2
  59. package/dist/storage/sqlite.d.ts.map +1 -1
  60. package/dist/storage/sqlite.js +72 -6
  61. package/dist/storage/sqlite.js.map +1 -1
  62. package/dist/tools/memory-get.d.ts.map +1 -1
  63. package/dist/tools/memory-get.js +5 -1
  64. package/dist/tools/memory-get.js.map +1 -1
  65. package/dist/tools/memory-search.d.ts.map +1 -1
  66. package/dist/tools/memory-search.js +5 -0
  67. package/dist/tools/memory-search.js.map +1 -1
  68. package/dist/tools/memory-timeline.d.ts.map +1 -1
  69. package/dist/tools/memory-timeline.js +11 -2
  70. package/dist/tools/memory-timeline.js.map +1 -1
  71. package/dist/types.d.ts +2 -1
  72. package/dist/types.d.ts.map +1 -1
  73. package/dist/types.js +1 -1
  74. package/dist/types.js.map +1 -1
  75. package/dist/viewer/html.d.ts +1 -1
  76. package/dist/viewer/html.d.ts.map +1 -1
  77. package/dist/viewer/html.js +233 -9
  78. package/dist/viewer/html.js.map +1 -1
  79. package/dist/viewer/server.d.ts +5 -0
  80. package/dist/viewer/server.d.ts.map +1 -1
  81. package/dist/viewer/server.js +383 -177
  82. package/dist/viewer/server.js.map +1 -1
  83. package/index.ts +26 -4
  84. package/package.json +2 -1
  85. package/src/capture/index.ts +39 -10
  86. package/src/index.ts +3 -2
  87. package/src/ingest/providers/anthropic.ts +22 -8
  88. package/src/ingest/providers/bedrock.ts +22 -8
  89. package/src/ingest/providers/gemini.ts +22 -8
  90. package/src/ingest/providers/index.ts +192 -142
  91. package/src/ingest/providers/openai.ts +37 -17
  92. package/src/ingest/task-processor.ts +183 -65
  93. package/src/ingest/worker.ts +98 -77
  94. package/src/shared/llm-call.ts +144 -0
  95. package/src/skill/evaluator.ts +35 -64
  96. package/src/skill/evolver.ts +201 -33
  97. package/src/skill/generator.ts +16 -59
  98. package/src/skill/upgrader.ts +5 -43
  99. package/src/skill/validator.ts +15 -47
  100. package/src/storage/sqlite.ts +88 -6
  101. package/src/tools/memory-get.ts +6 -1
  102. package/src/tools/memory-search.ts +6 -0
  103. package/src/tools/memory-timeline.ts +13 -1
  104. package/src/types.ts +2 -1
  105. package/src/viewer/html.ts +233 -9
  106. package/src/viewer/server.ts +368 -187
@@ -168,6 +168,12 @@ class ViewerServer {
168
168
  this.serveSearch(req, res, url);
169
169
  else if (p === "/api/tasks" && req.method === "GET")
170
170
  this.serveTasks(res, url);
171
+ else if (p.match(/^\/api\/task\/[^/]+\/retry-skill$/) && req.method === "POST")
172
+ this.handleTaskRetrySkill(req, res, p);
173
+ else if (p.startsWith("/api/task/") && req.method === "DELETE")
174
+ this.handleTaskDelete(res, p);
175
+ else if (p.startsWith("/api/task/") && req.method === "PUT")
176
+ this.handleTaskUpdate(req, res, p);
171
177
  else if (p.startsWith("/api/task/") && req.method === "GET")
172
178
  this.serveTaskDetail(res, p);
173
179
  else if (p === "/api/skills" && req.method === "GET")
@@ -178,6 +184,10 @@ class ViewerServer {
178
184
  this.serveSkillFiles(res, p);
179
185
  else if (p.match(/^\/api\/skill\/[^/]+\/visibility$/) && req.method === "PUT")
180
186
  this.handleSkillVisibility(req, res, p);
187
+ else if (p.startsWith("/api/skill/") && req.method === "DELETE")
188
+ this.handleSkillDelete(res, p);
189
+ else if (p.startsWith("/api/skill/") && req.method === "PUT")
190
+ this.handleSkillUpdate(req, res, p);
181
191
  else if (p.startsWith("/api/skill/") && req.method === "GET")
182
192
  this.serveSkillDetail(res, p);
183
193
  else if (p === "/api/memory" && req.method === "POST")
@@ -401,16 +411,21 @@ class ViewerServer {
401
411
  const limit = Math.min(100, Math.max(1, Number(url.searchParams.get("limit")) || 50));
402
412
  const offset = Math.max(0, Number(url.searchParams.get("offset")) || 0);
403
413
  const { tasks, total } = this.store.listTasks({ status, limit, offset });
404
- const items = tasks.map((t) => ({
405
- id: t.id,
406
- sessionKey: t.sessionKey,
407
- title: t.title,
408
- summary: t.summary ? (t.summary.length > 300 ? t.summary.slice(0, 297) + "..." : t.summary) : "",
409
- status: t.status,
410
- startedAt: t.startedAt,
411
- endedAt: t.endedAt,
412
- chunkCount: this.store.countChunksByTask(t.id),
413
- }));
414
+ const db = this.store.db;
415
+ const items = tasks.map((t) => {
416
+ const meta = db.prepare("SELECT skill_status FROM tasks WHERE id = ?").get(t.id);
417
+ return {
418
+ id: t.id,
419
+ sessionKey: t.sessionKey,
420
+ title: t.title,
421
+ summary: t.summary ? (t.summary.length > 300 ? t.summary.slice(0, 297) + "..." : t.summary) : "",
422
+ status: t.status,
423
+ startedAt: t.startedAt,
424
+ endedAt: t.endedAt,
425
+ chunkCount: this.store.countChunksByTask(t.id),
426
+ skillStatus: meta?.skill_status ?? null,
427
+ };
428
+ });
414
429
  this.jsonResponse(res, { tasks: items, total, limit, offset });
415
430
  }
416
431
  serveTaskDetail(res, urlPath) {
@@ -541,15 +556,17 @@ class ViewerServer {
541
556
  catch {
542
557
  ftsResults = db.prepare("SELECT * FROM chunks WHERE content LIKE ? OR summary LIKE ? ORDER BY created_at DESC LIMIT 100").all(`%${q}%`, `%${q}%`).filter(passesFilter);
543
558
  }
559
+ const SEMANTIC_THRESHOLD = 0.64;
544
560
  let vectorResults = [];
561
+ let scoreMap = new Map();
545
562
  try {
546
563
  const queryVec = await this.embedder.embedQuery(q);
547
564
  const hits = (0, vector_1.vectorSearch)(this.store, queryVec, 40);
548
- const hitIds = new Set(hits.filter(h => h.score > 0.3).map(h => h.chunkId));
565
+ scoreMap = new Map(hits.map(h => [h.chunkId, h.score]));
566
+ const hitIds = new Set(hits.filter(h => h.score >= SEMANTIC_THRESHOLD).map(h => h.chunkId));
549
567
  if (hitIds.size > 0) {
550
568
  const placeholders = [...hitIds].map(() => "?").join(",");
551
569
  const rows = db.prepare(`SELECT * FROM chunks WHERE id IN (${placeholders})`).all(...hitIds).filter(passesFilter);
552
- const scoreMap = new Map(hits.map(h => [h.chunkId, h.score]));
553
570
  rows.forEach((r) => { r._vscore = scoreMap.get(r.id) ?? 0; });
554
571
  rows.sort((a, b) => (b._vscore ?? 0) - (a._vscore ?? 0));
555
572
  vectorResults = rows;
@@ -567,18 +584,24 @@ class ViewerServer {
567
584
  }
568
585
  }
569
586
  for (const r of ftsResults) {
570
- if (!seenIds.has(r.id)) {
571
- seenIds.add(r.id);
572
- merged.push(r);
573
- }
587
+ if (seenIds.has(r.id))
588
+ continue;
589
+ const vscore = scoreMap.get(r.id);
590
+ if (vscore !== undefined && vscore < SEMANTIC_THRESHOLD)
591
+ continue;
592
+ seenIds.add(r.id);
593
+ merged.push(r);
574
594
  }
595
+ const fallback = merged.length === 0 && ftsResults.length > 0;
596
+ const results = fallback ? ftsResults.slice(0, 20) : merged;
575
597
  this.store.recordViewerEvent("search");
576
598
  this.jsonResponse(res, {
577
- results: merged,
599
+ results,
578
600
  query: q,
579
601
  vectorCount: vectorResults.length,
580
602
  ftsCount: ftsResults.length,
581
- total: merged.length,
603
+ total: results.length,
604
+ fallbackFts: fallback,
582
605
  });
583
606
  }
584
607
  // ─── Skills API ───
@@ -737,18 +760,138 @@ class ViewerServer {
737
760
  }
738
761
  });
739
762
  }
763
+ // ─── Task/Skill management ───
764
+ handleTaskRetrySkill(_req, res, urlPath) {
765
+ const taskId = urlPath.replace("/api/task/", "").replace("/retry-skill", "");
766
+ const task = this.store.getTask(taskId);
767
+ if (!task) {
768
+ res.writeHead(404, { "Content-Type": "application/json" });
769
+ res.end(JSON.stringify({ error: "Task not found" }));
770
+ return;
771
+ }
772
+ if (task.status !== "completed") {
773
+ res.writeHead(400, { "Content-Type": "application/json" });
774
+ res.end(JSON.stringify({ error: "Only completed tasks can retry skill generation" }));
775
+ return;
776
+ }
777
+ if (!this.ctx) {
778
+ res.writeHead(500, { "Content-Type": "application/json" });
779
+ res.end(JSON.stringify({ error: "Plugin context not available" }));
780
+ return;
781
+ }
782
+ // Clean up stale task_skills references (e.g., skill was manually deleted)
783
+ const db = this.store.db;
784
+ db.prepare("DELETE FROM task_skills WHERE task_id = ? AND skill_id NOT IN (SELECT id FROM skills)").run(taskId);
785
+ this.store.setTaskSkillMeta(taskId, { skillStatus: "queued", skillReason: "手动重试中..." });
786
+ this.jsonResponse(res, { ok: true, taskId, status: "queued" });
787
+ const ctx = this.ctx;
788
+ const recallEngine = new engine_1.RecallEngine(this.store, this.embedder, ctx);
789
+ const evolver = new evolver_1.SkillEvolver(this.store, recallEngine, ctx, this.embedder);
790
+ evolver.onTaskCompleted(task).then(() => {
791
+ this.log.info(`Retry skill generation completed for task ${taskId}`);
792
+ }).catch((err) => {
793
+ this.log.error(`Retry skill generation failed for task ${taskId}: ${err}`);
794
+ this.store.setTaskSkillMeta(taskId, { skillStatus: "skipped", skillReason: `error: ${err}` });
795
+ });
796
+ }
797
+ handleTaskDelete(res, urlPath) {
798
+ const taskId = urlPath.replace("/api/task/", "");
799
+ const deleted = this.store.deleteTask(taskId);
800
+ if (!deleted) {
801
+ res.writeHead(404, { "Content-Type": "application/json" });
802
+ res.end(JSON.stringify({ error: "Task not found" }));
803
+ return;
804
+ }
805
+ this.jsonResponse(res, { ok: true, taskId });
806
+ }
807
+ handleTaskUpdate(req, res, urlPath) {
808
+ const taskId = urlPath.replace("/api/task/", "");
809
+ this.readBody(req, (body) => {
810
+ try {
811
+ const data = JSON.parse(body);
812
+ const task = this.store.getTask(taskId);
813
+ if (!task) {
814
+ res.writeHead(404, { "Content-Type": "application/json" });
815
+ res.end(JSON.stringify({ error: "Task not found" }));
816
+ return;
817
+ }
818
+ this.store.updateTask(taskId, {
819
+ title: data.title ?? task.title,
820
+ summary: data.summary ?? task.summary,
821
+ status: data.status ?? task.status,
822
+ endedAt: task.endedAt ?? undefined,
823
+ });
824
+ this.jsonResponse(res, { ok: true, taskId });
825
+ }
826
+ catch (err) {
827
+ res.writeHead(400, { "Content-Type": "application/json" });
828
+ res.end(JSON.stringify({ error: String(err) }));
829
+ }
830
+ });
831
+ }
832
+ handleSkillDelete(res, urlPath) {
833
+ const skillId = urlPath.replace("/api/skill/", "");
834
+ const skill = this.store.getSkill(skillId);
835
+ if (!skill) {
836
+ res.writeHead(404, { "Content-Type": "application/json" });
837
+ res.end(JSON.stringify({ error: "Skill not found" }));
838
+ return;
839
+ }
840
+ // Remove skill directory from disk
841
+ try {
842
+ if (skill.dirPath && node_fs_1.default.existsSync(skill.dirPath)) {
843
+ node_fs_1.default.rmSync(skill.dirPath, { recursive: true, force: true });
844
+ }
845
+ }
846
+ catch (err) {
847
+ this.log.warn(`Failed to remove skill directory ${skill.dirPath}: ${err}`);
848
+ }
849
+ this.store.deleteSkill(skillId);
850
+ this.jsonResponse(res, { ok: true, skillId });
851
+ }
852
+ handleSkillUpdate(req, res, urlPath) {
853
+ const skillId = urlPath.replace("/api/skill/", "");
854
+ this.readBody(req, (body) => {
855
+ try {
856
+ const data = JSON.parse(body);
857
+ const skill = this.store.getSkill(skillId);
858
+ if (!skill) {
859
+ res.writeHead(404, { "Content-Type": "application/json" });
860
+ res.end(JSON.stringify({ error: "Skill not found" }));
861
+ return;
862
+ }
863
+ this.store.updateSkill(skillId, {
864
+ description: data.description ?? skill.description,
865
+ version: skill.version,
866
+ status: data.status ?? skill.status,
867
+ installed: skill.installed,
868
+ qualityScore: skill.qualityScore,
869
+ });
870
+ this.jsonResponse(res, { ok: true, skillId });
871
+ }
872
+ catch (err) {
873
+ res.writeHead(400, { "Content-Type": "application/json" });
874
+ res.end(JSON.stringify({ error: String(err) }));
875
+ }
876
+ });
877
+ }
740
878
  // ─── CRUD ───
741
879
  handleCreate(req, res) {
742
880
  this.readBody(req, (body) => {
743
881
  try {
744
882
  const data = JSON.parse(body);
883
+ if (!data.content || typeof data.content !== "string" || !data.content.trim()) {
884
+ res.writeHead(400, { "Content-Type": "application/json" });
885
+ res.end(JSON.stringify({ error: "content is required and must be a non-empty string" }));
886
+ return;
887
+ }
745
888
  const { v4: uuidv4 } = require("uuid");
746
889
  const id = uuidv4();
747
890
  const now = Date.now();
748
891
  this.store.insertChunk({
749
892
  id, sessionKey: data.session_key || "manual", turnId: `manual-${now}`, seq: 0,
750
- role: data.role || "user", content: data.content || "", kind: data.kind || "paragraph",
751
- summary: data.summary || data.content?.slice(0, 100) || "",
893
+ role: data.role || "user", content: data.content, kind: data.kind || "paragraph",
894
+ summary: data.summary || data.content.slice(0, 100),
752
895
  taskId: null, skillId: null, owner: data.owner || "agent:main",
753
896
  dedupStatus: "active", dedupTarget: null, dedupReason: null,
754
897
  mergeCount: 0, lastHitAt: null, mergeHistory: "[]",
@@ -780,6 +923,11 @@ class ViewerServer {
780
923
  this.readBody(req, (body) => {
781
924
  try {
782
925
  const data = JSON.parse(body);
926
+ if (data.content !== undefined && (typeof data.content !== "string" || !data.content.trim())) {
927
+ res.writeHead(400, { "Content-Type": "application/json" });
928
+ res.end(JSON.stringify({ error: "content must be a non-empty string" }));
929
+ return;
930
+ }
783
931
  const ok = this.store.updateChunk(chunkId, { summary: data.summary, content: data.content, role: data.role, kind: data.kind, owner: data.owner });
784
932
  if (ok)
785
933
  this.jsonResponse(res, { ok: true, message: "Memory updated" });
@@ -1108,6 +1256,7 @@ class ViewerServer {
1108
1256
  opts = JSON.parse(body);
1109
1257
  }
1110
1258
  catch { /* defaults */ }
1259
+ const concurrency = Math.max(1, Math.min(opts.concurrency ?? 1, 8));
1111
1260
  res.writeHead(200, {
1112
1261
  "Content-Type": "text/event-stream",
1113
1262
  "Cache-Control": "no-cache",
@@ -1144,7 +1293,7 @@ class ViewerServer {
1144
1293
  this.broadcastSSE(event, data);
1145
1294
  };
1146
1295
  this.migrationRunning = true;
1147
- this.runMigration(send, opts.sources).finally(() => {
1296
+ this.runMigration(send, opts.sources, concurrency).finally(() => {
1148
1297
  this.migrationRunning = false;
1149
1298
  this.migrationState.done = true;
1150
1299
  if (this.migrationAbort) {
@@ -1165,7 +1314,7 @@ class ViewerServer {
1165
1314
  });
1166
1315
  });
1167
1316
  }
1168
- async runMigration(send, sources) {
1317
+ async runMigration(send, sources, concurrency = 1) {
1169
1318
  const ocHome = this.getOpenClawHome();
1170
1319
  const importSqlite = !sources || sources.includes("sqlite");
1171
1320
  const importSessions = !sources || sources.includes("sessions");
@@ -1175,15 +1324,17 @@ class ViewerServer {
1175
1324
  let totalErrors = 0;
1176
1325
  const cfgPath = this.getOpenClawConfigPath();
1177
1326
  let summarizerCfg;
1327
+ let strongCfg;
1178
1328
  try {
1179
1329
  const raw = JSON.parse(node_fs_1.default.readFileSync(cfgPath, "utf-8"));
1180
1330
  const pluginCfg = raw?.plugins?.entries?.["memos-local-openclaw-plugin"]?.config ??
1181
1331
  raw?.plugins?.entries?.["memos-lite"]?.config ??
1182
1332
  raw?.plugins?.entries?.["memos-lite-openclaw-plugin"]?.config ?? {};
1183
1333
  summarizerCfg = pluginCfg.summarizer;
1334
+ strongCfg = pluginCfg.skillEvolution?.summarizer;
1184
1335
  }
1185
1336
  catch { /* no config */ }
1186
- const summarizer = new providers_1.Summarizer(summarizerCfg, this.log);
1337
+ const summarizer = new providers_1.Summarizer(summarizerCfg, this.log, strongCfg);
1187
1338
  // Phase 1: Import SQLite memory chunks
1188
1339
  if (importSqlite) {
1189
1340
  const memoryDir = node_path_1.default.join(ocHome, "memory");
@@ -1218,6 +1369,21 @@ class ViewerServer {
1218
1369
  });
1219
1370
  continue;
1220
1371
  }
1372
+ const importOwner = `agent:${agentId}`;
1373
+ // Exact hash dedup within same agent
1374
+ const existingByHash = this.store.findActiveChunkByHash(row.text, importOwner);
1375
+ if (existingByHash) {
1376
+ totalSkipped++;
1377
+ send("item", {
1378
+ index: i + 1,
1379
+ total: rows.length,
1380
+ status: "skipped",
1381
+ preview: row.text.slice(0, 120),
1382
+ source: file,
1383
+ reason: "exact duplicate within agent",
1384
+ });
1385
+ continue;
1386
+ }
1221
1387
  try {
1222
1388
  const summary = await summarizer.summarize(row.text);
1223
1389
  let embedding = null;
@@ -1231,7 +1397,9 @@ class ViewerServer {
1231
1397
  let dedupTarget = null;
1232
1398
  let dedupReason = null;
1233
1399
  if (embedding) {
1234
- const topSimilar = (0, dedup_1.findTopSimilar)(this.store, embedding, 0.85, 3, this.log);
1400
+ const importThreshold = this.ctx?.config?.dedup?.similarityThreshold ?? 0.60;
1401
+ const dedupOwnerFilter = [importOwner];
1402
+ const topSimilar = (0, dedup_1.findTopSimilar)(this.store, embedding, importThreshold, 5, this.log, dedupOwnerFilter);
1235
1403
  if (topSimilar.length > 0) {
1236
1404
  const candidates = topSimilar.map((s, idx) => {
1237
1405
  const chunk = this.store.getChunk(s.chunkId);
@@ -1322,17 +1490,32 @@ class ViewerServer {
1322
1490
  }
1323
1491
  }
1324
1492
  }
1325
- // Phase 2: Import session JSONL files
1493
+ // Phase 2: Import session JSONL files from ALL agents (supports parallel by agent)
1326
1494
  if (importSessions) {
1327
- const sessionsDir = node_path_1.default.join(ocHome, "agents", "main", "sessions");
1328
- if (node_fs_1.default.existsSync(sessionsDir)) {
1329
- const jsonlFiles = node_fs_1.default.readdirSync(sessionsDir).filter(f => f.includes(".jsonl")).sort();
1330
- send("phase", { phase: "sessions", files: jsonlFiles.length });
1331
- let globalMsgIdx = 0;
1332
- let totalMsgs = 0;
1333
- for (const f of jsonlFiles) {
1495
+ const agentsDir = node_path_1.default.join(ocHome, "agents");
1496
+ const agentGroups = new Map();
1497
+ if (node_fs_1.default.existsSync(agentsDir)) {
1498
+ for (const entry of node_fs_1.default.readdirSync(agentsDir, { withFileTypes: true })) {
1499
+ if (entry.isDirectory()) {
1500
+ const sessDir = node_path_1.default.join(agentsDir, entry.name, "sessions");
1501
+ if (node_fs_1.default.existsSync(sessDir)) {
1502
+ const jsonlFiles = node_fs_1.default.readdirSync(sessDir).filter(f => f.includes(".jsonl")).sort();
1503
+ if (jsonlFiles.length > 0) {
1504
+ agentGroups.set(entry.name, jsonlFiles.map(f => ({ file: f, filePath: node_path_1.default.join(sessDir, f) })));
1505
+ }
1506
+ }
1507
+ }
1508
+ }
1509
+ }
1510
+ const agentIds = Array.from(agentGroups.keys());
1511
+ const allFileCount = Array.from(agentGroups.values()).reduce((s, g) => s + g.length, 0);
1512
+ send("phase", { phase: "sessions", files: allFileCount, agents: agentIds, concurrency });
1513
+ // Count total messages across all agents
1514
+ let totalMsgs = 0;
1515
+ for (const files of agentGroups.values()) {
1516
+ for (const { filePath } of files) {
1334
1517
  try {
1335
- const raw = node_fs_1.default.readFileSync(node_path_1.default.join(sessionsDir, f), "utf-8");
1518
+ const raw = node_fs_1.default.readFileSync(filePath, "utf-8");
1336
1519
  for (const line of raw.split("\n")) {
1337
1520
  if (!line.trim())
1338
1521
  continue;
@@ -1361,12 +1544,17 @@ class ViewerServer {
1361
1544
  }
1362
1545
  catch { /* skip */ }
1363
1546
  }
1364
- for (const file of jsonlFiles) {
1547
+ }
1548
+ // Thread-safe counters for parallel execution
1549
+ let globalMsgIdx = 0;
1550
+ const incIdx = () => ++globalMsgIdx;
1551
+ // Import one agent's sessions sequentially
1552
+ const importAgent = async (agentId, files) => {
1553
+ const agentOwner = `agent:${agentId}`;
1554
+ for (const { file, filePath } of files) {
1365
1555
  if (this.migrationAbort)
1366
1556
  break;
1367
1557
  const sessionId = file.replace(/\.jsonl.*$/, "");
1368
- const filePath = node_path_1.default.join(sessionsDir, file);
1369
- send("progress", { total: totalMsgs, processed: globalMsgIdx, phase: "sessions", file });
1370
1558
  try {
1371
1559
  const fileStream = node_fs_1.default.createReadStream(filePath, { encoding: "utf-8" });
1372
1560
  const rl = node_readline_1.default.createInterface({ input: fileStream, crlfDelay: Infinity });
@@ -1406,20 +1594,18 @@ class ViewerServer {
1406
1594
  }
1407
1595
  if (!content || content.length < 10)
1408
1596
  continue;
1409
- globalMsgIdx++;
1597
+ const idx = incIdx();
1410
1598
  totalProcessed++;
1411
1599
  const sessionKey = `openclaw-session-${sessionId}`;
1412
1600
  if (this.store.chunkExistsByContent(sessionKey, msgRole, content)) {
1413
1601
  totalSkipped++;
1414
- send("item", {
1415
- index: globalMsgIdx,
1416
- total: totalMsgs,
1417
- status: "skipped",
1418
- preview: content.slice(0, 120),
1419
- source: file,
1420
- role: msgRole,
1421
- reason: "duplicate",
1422
- });
1602
+ send("item", { index: idx, total: totalMsgs, status: "skipped", preview: content.slice(0, 120), source: file, agent: agentId, role: msgRole, reason: "duplicate" });
1603
+ continue;
1604
+ }
1605
+ const existingByHash = this.store.findActiveChunkByHash(content, agentOwner);
1606
+ if (existingByHash) {
1607
+ totalSkipped++;
1608
+ send("item", { index: idx, total: totalMsgs, status: "skipped", preview: content.slice(0, 120), source: file, agent: agentId, role: msgRole, reason: "exact duplicate within agent" });
1423
1609
  continue;
1424
1610
  }
1425
1611
  try {
@@ -1435,11 +1621,13 @@ class ViewerServer {
1435
1621
  let dedupTarget = null;
1436
1622
  let dedupReason = null;
1437
1623
  if (embedding) {
1438
- const topSimilar = (0, dedup_1.findTopSimilar)(this.store, embedding, 0.85, 3, this.log);
1624
+ const importThreshold = this.ctx?.config?.dedup?.similarityThreshold ?? 0.60;
1625
+ const dedupOwnerFilter = [agentOwner];
1626
+ const topSimilar = (0, dedup_1.findTopSimilar)(this.store, embedding, importThreshold, 5, this.log, dedupOwnerFilter);
1439
1627
  if (topSimilar.length > 0) {
1440
- const candidates = topSimilar.map((s, idx) => {
1628
+ const candidates = topSimilar.map((s, i) => {
1441
1629
  const chunk = this.store.getChunk(s.chunkId);
1442
- return { index: idx + 1, summary: chunk?.summary ?? "", chunkId: s.chunkId };
1630
+ return { index: i + 1, summary: chunk?.summary ?? "", chunkId: s.chunkId };
1443
1631
  }).filter(c => c.summary);
1444
1632
  if (candidates.length > 0) {
1445
1633
  const dedupResult = await summarizer.judgeDedup(summary, candidates);
@@ -1473,60 +1661,55 @@ class ViewerServer {
1473
1661
  const msgTs = obj.message?.timestamp ?? obj.timestamp;
1474
1662
  const ts = msgTs ? new Date(msgTs).getTime() : Date.now();
1475
1663
  const chunk = {
1476
- id: chunkId,
1477
- sessionKey,
1478
- turnId: `import-${sessionId}-${globalMsgIdx}`,
1479
- seq: 0,
1480
- role: msgRole,
1481
- content,
1482
- kind: "paragraph",
1483
- summary,
1484
- embedding: null,
1485
- taskId: null,
1486
- skillId: null,
1487
- owner: "agent:main",
1488
- dedupStatus,
1489
- dedupTarget,
1490
- dedupReason,
1491
- mergeCount: 0,
1492
- lastHitAt: null,
1493
- mergeHistory: "[]",
1494
- createdAt: ts,
1495
- updatedAt: ts,
1664
+ id: chunkId, sessionKey, turnId: `import-${agentId}-${sessionId}-${idx}`, seq: 0,
1665
+ role: msgRole, content, kind: "paragraph", summary, embedding: null,
1666
+ taskId: null, skillId: null, owner: agentOwner, dedupStatus, dedupTarget, dedupReason,
1667
+ mergeCount: 0, lastHitAt: null, mergeHistory: "[]", createdAt: ts, updatedAt: ts,
1496
1668
  };
1497
1669
  this.store.insertChunk(chunk);
1498
- if (embedding && dedupStatus === "active") {
1670
+ if (embedding && dedupStatus === "active")
1499
1671
  this.store.upsertEmbedding(chunkId, embedding);
1500
- }
1501
1672
  totalStored++;
1502
- send("item", {
1503
- index: globalMsgIdx,
1504
- total: totalMsgs,
1505
- status: dedupStatus === "active" ? "stored" : dedupStatus,
1506
- preview: content.slice(0, 120),
1507
- summary: summary.slice(0, 80),
1508
- source: file,
1509
- role: msgRole,
1510
- });
1673
+ send("item", { index: idx, total: totalMsgs, status: dedupStatus === "active" ? "stored" : dedupStatus, preview: content.slice(0, 120), summary: summary.slice(0, 80), source: file, agent: agentId, role: msgRole });
1511
1674
  }
1512
1675
  catch (err) {
1513
1676
  totalErrors++;
1514
- send("item", {
1515
- index: globalMsgIdx,
1516
- total: totalMsgs,
1517
- status: "error",
1518
- preview: content.slice(0, 120),
1519
- source: file,
1520
- error: String(err).slice(0, 200),
1521
- });
1677
+ send("item", { index: idx, total: totalMsgs, status: "error", preview: content.slice(0, 120), source: file, agent: agentId, error: String(err).slice(0, 200) });
1522
1678
  }
1523
1679
  }
1524
1680
  }
1525
1681
  catch (err) {
1526
- send("error", { file, error: String(err) });
1682
+ send("error", { file, agent: agentId, error: String(err) });
1527
1683
  totalErrors++;
1528
1684
  }
1529
1685
  }
1686
+ };
1687
+ // Execute agents with concurrency control
1688
+ const agentEntries = Array.from(agentGroups.entries());
1689
+ if (concurrency <= 1 || agentEntries.length <= 1) {
1690
+ for (const [agentId, files] of agentEntries) {
1691
+ if (this.migrationAbort)
1692
+ break;
1693
+ send("progress", { total: totalMsgs, processed: globalMsgIdx, phase: "sessions", agent: agentId });
1694
+ await importAgent(agentId, files);
1695
+ }
1696
+ }
1697
+ else {
1698
+ // Parallel: run up to `concurrency` agents at once
1699
+ let cursor = 0;
1700
+ const runBatch = async () => {
1701
+ while (cursor < agentEntries.length && !this.migrationAbort) {
1702
+ const batch = [];
1703
+ const batchStart = cursor;
1704
+ while (batch.length < concurrency && cursor < agentEntries.length) {
1705
+ const [agentId, files] = agentEntries[cursor++];
1706
+ send("progress", { total: totalMsgs, processed: globalMsgIdx, phase: "sessions", agent: agentId, parallel: true });
1707
+ batch.push(importAgent(agentId, files));
1708
+ }
1709
+ await Promise.all(batch);
1710
+ }
1711
+ };
1712
+ await runBatch();
1530
1713
  }
1531
1714
  }
1532
1715
  send("progress", { total: totalProcessed, processed: totalProcessed, phase: "done" });
@@ -1550,6 +1733,7 @@ class ViewerServer {
1550
1733
  opts = JSON.parse(body);
1551
1734
  }
1552
1735
  catch { /* defaults */ }
1736
+ const concurrency = Math.max(1, Math.min(opts.concurrency ?? 1, 8));
1553
1737
  res.writeHead(200, {
1554
1738
  "Content-Type": "text/event-stream",
1555
1739
  "Cache-Control": "no-cache",
@@ -1564,7 +1748,7 @@ class ViewerServer {
1564
1748
  this.broadcastPPSSE(event, data);
1565
1749
  };
1566
1750
  this.ppRunning = true;
1567
- this.runPostprocess(send, !!opts.enableTasks, !!opts.enableSkills).finally(() => {
1751
+ this.runPostprocess(send, !!opts.enableTasks, !!opts.enableSkills, concurrency).finally(() => {
1568
1752
  this.ppRunning = false;
1569
1753
  this.ppState.running = false;
1570
1754
  this.ppState.done = true;
@@ -1623,126 +1807,148 @@ class ViewerServer {
1623
1807
  catch { /* */ }
1624
1808
  }
1625
1809
  }
1626
- async runPostprocess(send, enableTasks, enableSkills) {
1810
+ async runPostprocess(send, enableTasks, enableSkills, concurrency = 1) {
1627
1811
  const ctx = this.ctx;
1628
- const taskProcessor = new task_processor_1.TaskProcessor(this.store, ctx);
1629
- let skillEvolver = null;
1630
- if (enableSkills) {
1631
- const recallEngine = new engine_1.RecallEngine(this.store, this.embedder, ctx);
1632
- skillEvolver = new evolver_1.SkillEvolver(this.store, recallEngine, ctx);
1633
- taskProcessor.onTaskCompleted(async (task) => {
1634
- try {
1635
- await skillEvolver.onTaskCompleted(task);
1636
- this.ppState.skillsCreated++;
1637
- send("skill", { taskId: task.id, title: task.title });
1638
- }
1639
- catch (err) {
1640
- this.log.warn(`Postprocess skill evolution error: ${err}`);
1641
- }
1642
- });
1643
- }
1644
1812
  const importSessions = this.store.getDistinctSessionKeys()
1645
1813
  .filter((sk) => sk.startsWith("openclaw-import-") || sk.startsWith("openclaw-session-"));
1646
1814
  const pendingItems = [];
1647
1815
  let skippedCount = 0;
1816
+ const ownerMap = this.store.getSessionOwnerMap(importSessions);
1648
1817
  for (const sk of importSessions) {
1649
1818
  const hasTask = this.store.hasTaskForSession(sk);
1650
1819
  const hasSkill = this.store.hasSkillForSessionTask(sk);
1820
+ const owner = ownerMap.get(sk) ?? "agent:main";
1651
1821
  if (enableTasks && !hasTask) {
1652
- pendingItems.push({ sessionKey: sk, action: "full" });
1822
+ pendingItems.push({ sessionKey: sk, action: "full", owner });
1653
1823
  }
1654
1824
  else if (enableSkills && hasTask && !hasSkill) {
1655
- pendingItems.push({ sessionKey: sk, action: "skill-only" });
1825
+ pendingItems.push({ sessionKey: sk, action: "skill-only", owner });
1656
1826
  }
1657
1827
  else {
1658
1828
  skippedCount++;
1659
1829
  }
1660
1830
  }
1831
+ // Group pending items by agent (owner)
1832
+ const agentGroups = new Map();
1833
+ for (const item of pendingItems) {
1834
+ const group = agentGroups.get(item.owner) ?? [];
1835
+ group.push(item);
1836
+ agentGroups.set(item.owner, group);
1837
+ }
1661
1838
  this.ppState.total = pendingItems.length;
1662
1839
  send("info", {
1663
1840
  totalSessions: importSessions.length,
1664
1841
  alreadyProcessed: skippedCount,
1665
1842
  pending: pendingItems.length,
1843
+ agents: Array.from(agentGroups.keys()),
1844
+ concurrency,
1666
1845
  });
1667
1846
  send("progress", { processed: 0, total: pendingItems.length });
1668
- for (let i = 0; i < pendingItems.length; i++) {
1669
- if (this.ppAbort)
1670
- break;
1671
- const { sessionKey, action } = pendingItems[i];
1672
- this.ppState.processed = i + 1;
1673
- send("item", {
1674
- index: i + 1,
1675
- total: pendingItems.length,
1676
- session: sessionKey,
1677
- step: "processing",
1678
- action,
1679
- });
1680
- try {
1681
- if (action === "full") {
1682
- await taskProcessor.onChunksIngested(sessionKey, Date.now());
1683
- const activeTask = this.store.getActiveTask(sessionKey);
1684
- if (activeTask) {
1685
- await taskProcessor.finalizeTask(activeTask);
1686
- const finalized = this.store.getTask(activeTask.id);
1687
- this.ppState.tasksCreated++;
1688
- send("item", {
1689
- index: i + 1,
1690
- total: pendingItems.length,
1691
- session: sessionKey,
1692
- step: "done",
1693
- taskTitle: finalized?.title || "",
1694
- taskStatus: finalized?.status || "",
1695
- });
1847
+ let globalIdx = 0;
1848
+ const incIdx = () => ++globalIdx;
1849
+ // Process one agent's sessions sequentially
1850
+ const processAgent = async (agentOwner, items) => {
1851
+ const taskProcessor = new task_processor_1.TaskProcessor(this.store, ctx);
1852
+ let skillEvolver = null;
1853
+ if (enableSkills) {
1854
+ const recallEngine = new engine_1.RecallEngine(this.store, this.embedder, ctx);
1855
+ skillEvolver = new evolver_1.SkillEvolver(this.store, recallEngine, ctx);
1856
+ taskProcessor.onTaskCompleted(async (task) => {
1857
+ try {
1858
+ await skillEvolver.onTaskCompleted(task);
1859
+ this.ppState.skillsCreated++;
1860
+ send("skill", { taskId: task.id, title: task.title, agent: agentOwner });
1696
1861
  }
1697
- else {
1698
- send("item", {
1699
- index: i + 1,
1700
- total: pendingItems.length,
1701
- session: sessionKey,
1702
- step: "done",
1703
- taskTitle: "(no chunks)",
1704
- });
1862
+ catch (err) {
1863
+ this.log.warn(`Postprocess skill evolution error (${agentOwner}): ${err}`);
1705
1864
  }
1706
- }
1707
- else if (action === "skill-only" && skillEvolver) {
1708
- const completedTasks = this.store.getCompletedTasksForSession(sessionKey);
1709
- let skillGenerated = false;
1710
- for (const task of completedTasks) {
1711
- if (this.ppAbort)
1712
- break;
1713
- try {
1714
- await skillEvolver.onTaskCompleted(task);
1715
- this.ppState.skillsCreated++;
1716
- skillGenerated = true;
1717
- send("skill", { taskId: task.id, title: task.title });
1865
+ });
1866
+ }
1867
+ for (const { sessionKey, action } of items) {
1868
+ if (this.ppAbort)
1869
+ break;
1870
+ const idx = incIdx();
1871
+ this.ppState.processed = globalIdx;
1872
+ send("item", {
1873
+ index: idx,
1874
+ total: pendingItems.length,
1875
+ session: sessionKey,
1876
+ agent: agentOwner,
1877
+ step: "processing",
1878
+ action,
1879
+ });
1880
+ try {
1881
+ if (action === "full") {
1882
+ await taskProcessor.onChunksIngested(sessionKey, Date.now());
1883
+ const activeTask = this.store.getActiveTask(sessionKey);
1884
+ if (activeTask) {
1885
+ await taskProcessor.finalizeTask(activeTask);
1886
+ const finalized = this.store.getTask(activeTask.id);
1887
+ this.ppState.tasksCreated++;
1888
+ send("item", {
1889
+ index: idx, total: pendingItems.length, session: sessionKey, agent: agentOwner,
1890
+ step: "done", taskTitle: finalized?.title || "", taskStatus: finalized?.status || "",
1891
+ });
1892
+ }
1893
+ else {
1894
+ send("item", {
1895
+ index: idx, total: pendingItems.length, session: sessionKey, agent: agentOwner,
1896
+ step: "done", taskTitle: "(no chunks)",
1897
+ });
1718
1898
  }
1719
- catch (err) {
1720
- this.log.warn(`Skill evolution error for task=${task.id}: ${err}`);
1899
+ }
1900
+ else if (action === "skill-only" && skillEvolver) {
1901
+ const completedTasks = this.store.getCompletedTasksForSession(sessionKey);
1902
+ let skillGenerated = false;
1903
+ for (const task of completedTasks) {
1904
+ if (this.ppAbort)
1905
+ break;
1906
+ try {
1907
+ await skillEvolver.onTaskCompleted(task);
1908
+ this.ppState.skillsCreated++;
1909
+ skillGenerated = true;
1910
+ send("skill", { taskId: task.id, title: task.title, agent: agentOwner });
1911
+ }
1912
+ catch (err) {
1913
+ this.log.warn(`Skill evolution error (${agentOwner}) task=${task.id}: ${err}`);
1914
+ }
1721
1915
  }
1916
+ send("item", {
1917
+ index: idx, total: pendingItems.length, session: sessionKey, agent: agentOwner,
1918
+ step: "done", taskTitle: completedTasks[0]?.title || sessionKey, action: "skill-only", skillGenerated,
1919
+ });
1722
1920
  }
1921
+ }
1922
+ catch (err) {
1923
+ this.ppState.errors++;
1924
+ this.log.warn(`Postprocess error (${agentOwner}) ${sessionKey}: ${err}`);
1723
1925
  send("item", {
1724
- index: i + 1,
1725
- total: pendingItems.length,
1726
- session: sessionKey,
1727
- step: "done",
1728
- taskTitle: completedTasks[0]?.title || sessionKey,
1729
- action: "skill-only",
1730
- skillGenerated,
1926
+ index: idx, total: pendingItems.length, session: sessionKey, agent: agentOwner,
1927
+ step: "error", error: String(err).slice(0, 200),
1731
1928
  });
1732
1929
  }
1930
+ send("progress", { processed: globalIdx, total: pendingItems.length });
1733
1931
  }
1734
- catch (err) {
1735
- this.ppState.errors++;
1736
- this.log.warn(`Postprocess error for ${sessionKey}: ${err}`);
1737
- send("item", {
1738
- index: i + 1,
1739
- total: pendingItems.length,
1740
- session: sessionKey,
1741
- step: "error",
1742
- error: String(err).slice(0, 200),
1743
- });
1932
+ };
1933
+ // Execute agents with concurrency control
1934
+ const agentEntries = Array.from(agentGroups.entries());
1935
+ if (concurrency <= 1 || agentEntries.length <= 1) {
1936
+ for (const [agentOwner, items] of agentEntries) {
1937
+ if (this.ppAbort)
1938
+ break;
1939
+ await processAgent(agentOwner, items);
1940
+ }
1941
+ }
1942
+ else {
1943
+ let cursor = 0;
1944
+ while (cursor < agentEntries.length && !this.ppAbort) {
1945
+ const batch = [];
1946
+ while (batch.length < concurrency && cursor < agentEntries.length) {
1947
+ const [agentOwner, items] = agentEntries[cursor++];
1948
+ batch.push(processAgent(agentOwner, items));
1949
+ }
1950
+ await Promise.all(batch);
1744
1951
  }
1745
- send("progress", { processed: i + 1, total: pendingItems.length });
1746
1952
  }
1747
1953
  }
1748
1954
  readBody(req, cb) {