@memtensor/memos-local-openclaw-plugin 0.3.20 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +232 -22
- package/dist/capture/index.d.ts +1 -1
- package/dist/capture/index.d.ts.map +1 -1
- package/dist/capture/index.js +33 -8
- package/dist/capture/index.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/ingest/providers/anthropic.d.ts.map +1 -1
- package/dist/ingest/providers/anthropic.js +22 -8
- package/dist/ingest/providers/anthropic.js.map +1 -1
- package/dist/ingest/providers/bedrock.d.ts.map +1 -1
- package/dist/ingest/providers/bedrock.js +22 -8
- package/dist/ingest/providers/bedrock.js.map +1 -1
- package/dist/ingest/providers/gemini.d.ts.map +1 -1
- package/dist/ingest/providers/gemini.js +22 -8
- package/dist/ingest/providers/gemini.js.map +1 -1
- package/dist/ingest/providers/index.d.ts +13 -18
- package/dist/ingest/providers/index.d.ts.map +1 -1
- package/dist/ingest/providers/index.js +213 -139
- package/dist/ingest/providers/index.js.map +1 -1
- package/dist/ingest/providers/openai.d.ts +1 -1
- package/dist/ingest/providers/openai.d.ts.map +1 -1
- package/dist/ingest/providers/openai.js +37 -17
- package/dist/ingest/providers/openai.js.map +1 -1
- package/dist/ingest/task-processor.d.ts +28 -3
- package/dist/ingest/task-processor.d.ts.map +1 -1
- package/dist/ingest/task-processor.js +166 -67
- package/dist/ingest/task-processor.js.map +1 -1
- package/dist/ingest/worker.d.ts.map +1 -1
- package/dist/ingest/worker.js +97 -75
- package/dist/ingest/worker.js.map +1 -1
- package/dist/shared/llm-call.d.ts +26 -0
- package/dist/shared/llm-call.d.ts.map +1 -0
- package/dist/shared/llm-call.js +163 -0
- package/dist/shared/llm-call.js.map +1 -0
- package/dist/skill/evaluator.d.ts +0 -3
- package/dist/skill/evaluator.d.ts.map +1 -1
- package/dist/skill/evaluator.js +34 -59
- package/dist/skill/evaluator.js.map +1 -1
- package/dist/skill/evolver.d.ts +22 -1
- package/dist/skill/evolver.d.ts.map +1 -1
- package/dist/skill/evolver.js +191 -32
- package/dist/skill/evolver.js.map +1 -1
- package/dist/skill/generator.d.ts +0 -3
- package/dist/skill/generator.d.ts.map +1 -1
- package/dist/skill/generator.js +15 -50
- package/dist/skill/generator.js.map +1 -1
- package/dist/skill/upgrader.d.ts +0 -2
- package/dist/skill/upgrader.d.ts.map +1 -1
- package/dist/skill/upgrader.js +4 -39
- package/dist/skill/upgrader.js.map +1 -1
- package/dist/skill/validator.d.ts +0 -2
- package/dist/skill/validator.d.ts.map +1 -1
- package/dist/skill/validator.js +14 -44
- package/dist/skill/validator.js.map +1 -1
- package/dist/storage/sqlite.d.ts +13 -2
- package/dist/storage/sqlite.d.ts.map +1 -1
- package/dist/storage/sqlite.js +72 -6
- package/dist/storage/sqlite.js.map +1 -1
- package/dist/tools/memory-get.d.ts.map +1 -1
- package/dist/tools/memory-get.js +5 -1
- package/dist/tools/memory-get.js.map +1 -1
- package/dist/tools/memory-search.d.ts.map +1 -1
- package/dist/tools/memory-search.js +5 -0
- package/dist/tools/memory-search.js.map +1 -1
- package/dist/tools/memory-timeline.d.ts.map +1 -1
- package/dist/tools/memory-timeline.js +11 -2
- package/dist/tools/memory-timeline.js.map +1 -1
- package/dist/types.d.ts +2 -1
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +1 -1
- package/dist/types.js.map +1 -1
- package/dist/viewer/html.d.ts +1 -1
- package/dist/viewer/html.d.ts.map +1 -1
- package/dist/viewer/html.js +233 -9
- package/dist/viewer/html.js.map +1 -1
- package/dist/viewer/server.d.ts +5 -0
- package/dist/viewer/server.d.ts.map +1 -1
- package/dist/viewer/server.js +383 -177
- package/dist/viewer/server.js.map +1 -1
- package/index.ts +9 -3
- package/package.json +2 -1
- package/src/capture/index.ts +39 -10
- package/src/index.ts +3 -2
- package/src/ingest/providers/anthropic.ts +22 -8
- package/src/ingest/providers/bedrock.ts +22 -8
- package/src/ingest/providers/gemini.ts +22 -8
- package/src/ingest/providers/index.ts +192 -142
- package/src/ingest/providers/openai.ts +37 -17
- package/src/ingest/task-processor.ts +183 -65
- package/src/ingest/worker.ts +98 -77
- package/src/shared/llm-call.ts +144 -0
- package/src/skill/evaluator.ts +35 -64
- package/src/skill/evolver.ts +201 -33
- package/src/skill/generator.ts +16 -59
- package/src/skill/upgrader.ts +5 -43
- package/src/skill/validator.ts +15 -47
- package/src/storage/sqlite.ts +88 -6
- package/src/tools/memory-get.ts +6 -1
- package/src/tools/memory-search.ts +6 -0
- package/src/tools/memory-timeline.ts +13 -1
- package/src/types.ts +2 -1
- package/src/viewer/html.ts +233 -9
- package/src/viewer/server.ts +368 -187
package/src/viewer/server.ts
CHANGED
|
@@ -194,11 +194,16 @@ export class ViewerServer {
|
|
|
194
194
|
else if (p === "/api/tool-metrics") this.serveToolMetrics(res, url);
|
|
195
195
|
else if (p === "/api/search") this.serveSearch(req, res, url);
|
|
196
196
|
else if (p === "/api/tasks" && req.method === "GET") this.serveTasks(res, url);
|
|
197
|
+
else if (p.match(/^\/api\/task\/[^/]+\/retry-skill$/) && req.method === "POST") this.handleTaskRetrySkill(req, res, p);
|
|
198
|
+
else if (p.startsWith("/api/task/") && req.method === "DELETE") this.handleTaskDelete(res, p);
|
|
199
|
+
else if (p.startsWith("/api/task/") && req.method === "PUT") this.handleTaskUpdate(req, res, p);
|
|
197
200
|
else if (p.startsWith("/api/task/") && req.method === "GET") this.serveTaskDetail(res, p);
|
|
198
|
-
else
|
|
201
|
+
else if (p === "/api/skills" && req.method === "GET") this.serveSkills(res, url);
|
|
199
202
|
else if (p.match(/^\/api\/skill\/[^/]+\/download$/) && req.method === "GET") this.serveSkillDownload(res, p);
|
|
200
203
|
else if (p.match(/^\/api\/skill\/[^/]+\/files$/) && req.method === "GET") this.serveSkillFiles(res, p);
|
|
201
204
|
else if (p.match(/^\/api\/skill\/[^/]+\/visibility$/) && req.method === "PUT") this.handleSkillVisibility(req, res, p);
|
|
205
|
+
else if (p.startsWith("/api/skill/") && req.method === "DELETE") this.handleSkillDelete(res, p);
|
|
206
|
+
else if (p.startsWith("/api/skill/") && req.method === "PUT") this.handleSkillUpdate(req, res, p);
|
|
202
207
|
else if (p.startsWith("/api/skill/") && req.method === "GET") this.serveSkillDetail(res, p);
|
|
203
208
|
else if (p === "/api/memory" && req.method === "POST") this.handleCreate(req, res);
|
|
204
209
|
else if (p.startsWith("/api/memory/") && req.method === "GET") this.serveMemoryDetail(res, p);
|
|
@@ -394,16 +399,21 @@ export class ViewerServer {
|
|
|
394
399
|
const offset = Math.max(0, Number(url.searchParams.get("offset")) || 0);
|
|
395
400
|
const { tasks, total } = this.store.listTasks({ status, limit, offset });
|
|
396
401
|
|
|
397
|
-
const
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
402
|
+
const db = (this.store as any).db;
|
|
403
|
+
const items = tasks.map((t) => {
|
|
404
|
+
const meta = db.prepare("SELECT skill_status FROM tasks WHERE id = ?").get(t.id) as { skill_status: string | null } | undefined;
|
|
405
|
+
return {
|
|
406
|
+
id: t.id,
|
|
407
|
+
sessionKey: t.sessionKey,
|
|
408
|
+
title: t.title,
|
|
409
|
+
summary: t.summary ? (t.summary.length > 300 ? t.summary.slice(0, 297) + "..." : t.summary) : "",
|
|
410
|
+
status: t.status,
|
|
411
|
+
startedAt: t.startedAt,
|
|
412
|
+
endedAt: t.endedAt,
|
|
413
|
+
chunkCount: this.store.countChunksByTask(t.id),
|
|
414
|
+
skillStatus: meta?.skill_status ?? null,
|
|
415
|
+
};
|
|
416
|
+
});
|
|
407
417
|
|
|
408
418
|
this.jsonResponse(res, { tasks: items, total, limit, offset });
|
|
409
419
|
}
|
|
@@ -541,15 +551,17 @@ export class ViewerServer {
|
|
|
541
551
|
).all(`%${q}%`, `%${q}%`).filter(passesFilter);
|
|
542
552
|
}
|
|
543
553
|
|
|
554
|
+
const SEMANTIC_THRESHOLD = 0.64;
|
|
544
555
|
let vectorResults: any[] = [];
|
|
556
|
+
let scoreMap = new Map<string, number>();
|
|
545
557
|
try {
|
|
546
558
|
const queryVec = await this.embedder.embedQuery(q);
|
|
547
559
|
const hits = vectorSearch(this.store, queryVec, 40);
|
|
548
|
-
|
|
560
|
+
scoreMap = new Map(hits.map(h => [h.chunkId, h.score]));
|
|
561
|
+
const hitIds = new Set(hits.filter(h => h.score >= SEMANTIC_THRESHOLD).map(h => h.chunkId));
|
|
549
562
|
if (hitIds.size > 0) {
|
|
550
563
|
const placeholders = [...hitIds].map(() => "?").join(",");
|
|
551
564
|
const rows = db.prepare(`SELECT * FROM chunks WHERE id IN (${placeholders})`).all(...hitIds).filter(passesFilter);
|
|
552
|
-
const scoreMap = new Map(hits.map(h => [h.chunkId, h.score]));
|
|
553
565
|
rows.forEach((r: any) => { r._vscore = scoreMap.get(r.id) ?? 0; });
|
|
554
566
|
rows.sort((a: any, b: any) => (b._vscore ?? 0) - (a._vscore ?? 0));
|
|
555
567
|
vectorResults = rows;
|
|
@@ -564,16 +576,23 @@ export class ViewerServer {
|
|
|
564
576
|
if (!seenIds.has(r.id)) { seenIds.add(r.id); merged.push(r); }
|
|
565
577
|
}
|
|
566
578
|
for (const r of ftsResults) {
|
|
567
|
-
if (
|
|
579
|
+
if (seenIds.has(r.id)) continue;
|
|
580
|
+
const vscore = scoreMap.get(r.id);
|
|
581
|
+
if (vscore !== undefined && vscore < SEMANTIC_THRESHOLD) continue;
|
|
582
|
+
seenIds.add(r.id); merged.push(r);
|
|
568
583
|
}
|
|
569
584
|
|
|
585
|
+
const fallback = merged.length === 0 && ftsResults.length > 0;
|
|
586
|
+
const results = fallback ? ftsResults.slice(0, 20) : merged;
|
|
587
|
+
|
|
570
588
|
this.store.recordViewerEvent("search");
|
|
571
589
|
this.jsonResponse(res, {
|
|
572
|
-
results
|
|
590
|
+
results,
|
|
573
591
|
query: q,
|
|
574
592
|
vectorCount: vectorResults.length,
|
|
575
593
|
ftsCount: ftsResults.length,
|
|
576
|
-
total:
|
|
594
|
+
total: results.length,
|
|
595
|
+
fallbackFts: fallback,
|
|
577
596
|
});
|
|
578
597
|
}
|
|
579
598
|
|
|
@@ -739,19 +758,117 @@ export class ViewerServer {
|
|
|
739
758
|
});
|
|
740
759
|
}
|
|
741
760
|
|
|
761
|
+
// ─── Task/Skill management ───
|
|
762
|
+
|
|
763
|
+
private handleTaskRetrySkill(_req: http.IncomingMessage, res: http.ServerResponse, urlPath: string): void {
|
|
764
|
+
const taskId = urlPath.replace("/api/task/", "").replace("/retry-skill", "");
|
|
765
|
+
const task = this.store.getTask(taskId);
|
|
766
|
+
if (!task) { res.writeHead(404, { "Content-Type": "application/json" }); res.end(JSON.stringify({ error: "Task not found" })); return; }
|
|
767
|
+
if (task.status !== "completed") { res.writeHead(400, { "Content-Type": "application/json" }); res.end(JSON.stringify({ error: "Only completed tasks can retry skill generation" })); return; }
|
|
768
|
+
if (!this.ctx) { res.writeHead(500, { "Content-Type": "application/json" }); res.end(JSON.stringify({ error: "Plugin context not available" })); return; }
|
|
769
|
+
|
|
770
|
+
// Clean up stale task_skills references (e.g., skill was manually deleted)
|
|
771
|
+
const db = (this.store as any).db;
|
|
772
|
+
db.prepare("DELETE FROM task_skills WHERE task_id = ? AND skill_id NOT IN (SELECT id FROM skills)").run(taskId);
|
|
773
|
+
|
|
774
|
+
this.store.setTaskSkillMeta(taskId, { skillStatus: "queued", skillReason: "手动重试中..." });
|
|
775
|
+
this.jsonResponse(res, { ok: true, taskId, status: "queued" });
|
|
776
|
+
|
|
777
|
+
const ctx = this.ctx;
|
|
778
|
+
const recallEngine = new RecallEngine(this.store, this.embedder, ctx);
|
|
779
|
+
const evolver = new SkillEvolver(this.store, recallEngine, ctx, this.embedder);
|
|
780
|
+
evolver.onTaskCompleted(task).then(() => {
|
|
781
|
+
this.log.info(`Retry skill generation completed for task ${taskId}`);
|
|
782
|
+
}).catch((err) => {
|
|
783
|
+
this.log.error(`Retry skill generation failed for task ${taskId}: ${err}`);
|
|
784
|
+
this.store.setTaskSkillMeta(taskId, { skillStatus: "skipped", skillReason: `error: ${err}` });
|
|
785
|
+
});
|
|
786
|
+
}
|
|
787
|
+
|
|
788
|
+
private handleTaskDelete(res: http.ServerResponse, urlPath: string): void {
|
|
789
|
+
const taskId = urlPath.replace("/api/task/", "");
|
|
790
|
+
const deleted = this.store.deleteTask(taskId);
|
|
791
|
+
if (!deleted) { res.writeHead(404, { "Content-Type": "application/json" }); res.end(JSON.stringify({ error: "Task not found" })); return; }
|
|
792
|
+
this.jsonResponse(res, { ok: true, taskId });
|
|
793
|
+
}
|
|
794
|
+
|
|
795
|
+
private handleTaskUpdate(req: http.IncomingMessage, res: http.ServerResponse, urlPath: string): void {
|
|
796
|
+
const taskId = urlPath.replace("/api/task/", "");
|
|
797
|
+
this.readBody(req, (body) => {
|
|
798
|
+
try {
|
|
799
|
+
const data = JSON.parse(body);
|
|
800
|
+
const task = this.store.getTask(taskId);
|
|
801
|
+
if (!task) { res.writeHead(404, { "Content-Type": "application/json" }); res.end(JSON.stringify({ error: "Task not found" })); return; }
|
|
802
|
+
this.store.updateTask(taskId, {
|
|
803
|
+
title: data.title ?? task.title,
|
|
804
|
+
summary: data.summary ?? task.summary,
|
|
805
|
+
status: data.status ?? task.status,
|
|
806
|
+
endedAt: task.endedAt ?? undefined,
|
|
807
|
+
});
|
|
808
|
+
this.jsonResponse(res, { ok: true, taskId });
|
|
809
|
+
} catch (err) {
|
|
810
|
+
res.writeHead(400, { "Content-Type": "application/json" });
|
|
811
|
+
res.end(JSON.stringify({ error: String(err) }));
|
|
812
|
+
}
|
|
813
|
+
});
|
|
814
|
+
}
|
|
815
|
+
|
|
816
|
+
private handleSkillDelete(res: http.ServerResponse, urlPath: string): void {
|
|
817
|
+
const skillId = urlPath.replace("/api/skill/", "");
|
|
818
|
+
const skill = this.store.getSkill(skillId);
|
|
819
|
+
if (!skill) { res.writeHead(404, { "Content-Type": "application/json" }); res.end(JSON.stringify({ error: "Skill not found" })); return; }
|
|
820
|
+
// Remove skill directory from disk
|
|
821
|
+
try {
|
|
822
|
+
if (skill.dirPath && fs.existsSync(skill.dirPath)) {
|
|
823
|
+
fs.rmSync(skill.dirPath, { recursive: true, force: true });
|
|
824
|
+
}
|
|
825
|
+
} catch (err) {
|
|
826
|
+
this.log.warn(`Failed to remove skill directory ${skill.dirPath}: ${err}`);
|
|
827
|
+
}
|
|
828
|
+
this.store.deleteSkill(skillId);
|
|
829
|
+
this.jsonResponse(res, { ok: true, skillId });
|
|
830
|
+
}
|
|
831
|
+
|
|
832
|
+
private handleSkillUpdate(req: http.IncomingMessage, res: http.ServerResponse, urlPath: string): void {
|
|
833
|
+
const skillId = urlPath.replace("/api/skill/", "");
|
|
834
|
+
this.readBody(req, (body) => {
|
|
835
|
+
try {
|
|
836
|
+
const data = JSON.parse(body);
|
|
837
|
+
const skill = this.store.getSkill(skillId);
|
|
838
|
+
if (!skill) { res.writeHead(404, { "Content-Type": "application/json" }); res.end(JSON.stringify({ error: "Skill not found" })); return; }
|
|
839
|
+
this.store.updateSkill(skillId, {
|
|
840
|
+
description: data.description ?? skill.description,
|
|
841
|
+
version: skill.version,
|
|
842
|
+
status: data.status ?? skill.status,
|
|
843
|
+
installed: skill.installed,
|
|
844
|
+
qualityScore: skill.qualityScore,
|
|
845
|
+
});
|
|
846
|
+
this.jsonResponse(res, { ok: true, skillId });
|
|
847
|
+
} catch (err) {
|
|
848
|
+
res.writeHead(400, { "Content-Type": "application/json" });
|
|
849
|
+
res.end(JSON.stringify({ error: String(err) }));
|
|
850
|
+
}
|
|
851
|
+
});
|
|
852
|
+
}
|
|
853
|
+
|
|
742
854
|
// ─── CRUD ───
|
|
743
855
|
|
|
744
856
|
private handleCreate(req: http.IncomingMessage, res: http.ServerResponse): void {
|
|
745
857
|
this.readBody(req, (body) => {
|
|
746
858
|
try {
|
|
747
859
|
const data = JSON.parse(body);
|
|
860
|
+
if (!data.content || typeof data.content !== "string" || !data.content.trim()) {
|
|
861
|
+
res.writeHead(400, { "Content-Type": "application/json" });
|
|
862
|
+
res.end(JSON.stringify({ error: "content is required and must be a non-empty string" }));
|
|
863
|
+
return;
|
|
864
|
+
}
|
|
748
865
|
const { v4: uuidv4 } = require("uuid");
|
|
749
866
|
const id = uuidv4();
|
|
750
867
|
const now = Date.now();
|
|
751
868
|
this.store.insertChunk({
|
|
752
869
|
id, sessionKey: data.session_key || "manual", turnId: `manual-${now}`, seq: 0,
|
|
753
|
-
role: data.role || "user", content: data.content
|
|
754
|
-
summary: data.summary || data.content
|
|
870
|
+
role: data.role || "user", content: data.content, kind: data.kind || "paragraph",
|
|
871
|
+
summary: data.summary || data.content.slice(0, 100),
|
|
755
872
|
taskId: null, skillId: null, owner: data.owner || "agent:main",
|
|
756
873
|
dedupStatus: "active", dedupTarget: null, dedupReason: null,
|
|
757
874
|
mergeCount: 0, lastHitAt: null, mergeHistory: "[]",
|
|
@@ -784,6 +901,11 @@ export class ViewerServer {
|
|
|
784
901
|
this.readBody(req, (body) => {
|
|
785
902
|
try {
|
|
786
903
|
const data = JSON.parse(body);
|
|
904
|
+
if (data.content !== undefined && (typeof data.content !== "string" || !data.content.trim())) {
|
|
905
|
+
res.writeHead(400, { "Content-Type": "application/json" });
|
|
906
|
+
res.end(JSON.stringify({ error: "content must be a non-empty string" }));
|
|
907
|
+
return;
|
|
908
|
+
}
|
|
787
909
|
const ok = this.store.updateChunk(chunkId, { summary: data.summary, content: data.content, role: data.role, kind: data.kind, owner: data.owner });
|
|
788
910
|
if (ok) this.jsonResponse(res, { ok: true, message: "Memory updated" });
|
|
789
911
|
else { res.writeHead(404, { "Content-Type": "application/json" }); res.end(JSON.stringify({ error: "Not found" })); }
|
|
@@ -1092,9 +1214,11 @@ export class ViewerServer {
|
|
|
1092
1214
|
}
|
|
1093
1215
|
|
|
1094
1216
|
this.readBody(req, (body) => {
|
|
1095
|
-
let opts: { sources?: string[] } = {};
|
|
1217
|
+
let opts: { sources?: string[]; concurrency?: number } = {};
|
|
1096
1218
|
try { opts = JSON.parse(body); } catch { /* defaults */ }
|
|
1097
1219
|
|
|
1220
|
+
const concurrency = Math.max(1, Math.min(opts.concurrency ?? 1, 8));
|
|
1221
|
+
|
|
1098
1222
|
res.writeHead(200, {
|
|
1099
1223
|
"Content-Type": "text/event-stream",
|
|
1100
1224
|
"Cache-Control": "no-cache",
|
|
@@ -1129,7 +1253,7 @@ export class ViewerServer {
|
|
|
1129
1253
|
};
|
|
1130
1254
|
|
|
1131
1255
|
this.migrationRunning = true;
|
|
1132
|
-
this.runMigration(send, opts.sources).finally(() => {
|
|
1256
|
+
this.runMigration(send, opts.sources, concurrency).finally(() => {
|
|
1133
1257
|
this.migrationRunning = false;
|
|
1134
1258
|
this.migrationState.done = true;
|
|
1135
1259
|
if (this.migrationAbort) {
|
|
@@ -1150,6 +1274,7 @@ export class ViewerServer {
|
|
|
1150
1274
|
private async runMigration(
|
|
1151
1275
|
send: (event: string, data: unknown) => void,
|
|
1152
1276
|
sources?: string[],
|
|
1277
|
+
concurrency: number = 1,
|
|
1153
1278
|
): Promise<void> {
|
|
1154
1279
|
const ocHome = this.getOpenClawHome();
|
|
1155
1280
|
const importSqlite = !sources || sources.includes("sqlite");
|
|
@@ -1162,15 +1287,17 @@ export class ViewerServer {
|
|
|
1162
1287
|
|
|
1163
1288
|
const cfgPath = this.getOpenClawConfigPath();
|
|
1164
1289
|
let summarizerCfg: any;
|
|
1290
|
+
let strongCfg: any;
|
|
1165
1291
|
try {
|
|
1166
1292
|
const raw = JSON.parse(fs.readFileSync(cfgPath, "utf-8"));
|
|
1167
1293
|
const pluginCfg = raw?.plugins?.entries?.["memos-local-openclaw-plugin"]?.config ??
|
|
1168
1294
|
raw?.plugins?.entries?.["memos-lite"]?.config ??
|
|
1169
1295
|
raw?.plugins?.entries?.["memos-lite-openclaw-plugin"]?.config ?? {};
|
|
1170
1296
|
summarizerCfg = pluginCfg.summarizer;
|
|
1297
|
+
strongCfg = pluginCfg.skillEvolution?.summarizer;
|
|
1171
1298
|
} catch { /* no config */ }
|
|
1172
1299
|
|
|
1173
|
-
const summarizer = new Summarizer(summarizerCfg, this.log);
|
|
1300
|
+
const summarizer = new Summarizer(summarizerCfg, this.log, strongCfg);
|
|
1174
1301
|
|
|
1175
1302
|
// Phase 1: Import SQLite memory chunks
|
|
1176
1303
|
if (importSqlite) {
|
|
@@ -1210,6 +1337,23 @@ export class ViewerServer {
|
|
|
1210
1337
|
continue;
|
|
1211
1338
|
}
|
|
1212
1339
|
|
|
1340
|
+
const importOwner = `agent:${agentId}`;
|
|
1341
|
+
|
|
1342
|
+
// Exact hash dedup within same agent
|
|
1343
|
+
const existingByHash = this.store.findActiveChunkByHash(row.text, importOwner);
|
|
1344
|
+
if (existingByHash) {
|
|
1345
|
+
totalSkipped++;
|
|
1346
|
+
send("item", {
|
|
1347
|
+
index: i + 1,
|
|
1348
|
+
total: rows.length,
|
|
1349
|
+
status: "skipped",
|
|
1350
|
+
preview: row.text.slice(0, 120),
|
|
1351
|
+
source: file,
|
|
1352
|
+
reason: "exact duplicate within agent",
|
|
1353
|
+
});
|
|
1354
|
+
continue;
|
|
1355
|
+
}
|
|
1356
|
+
|
|
1213
1357
|
try {
|
|
1214
1358
|
const summary = await summarizer.summarize(row.text);
|
|
1215
1359
|
let embedding: number[] | null = null;
|
|
@@ -1224,7 +1368,9 @@ export class ViewerServer {
|
|
|
1224
1368
|
let dedupReason: string | null = null;
|
|
1225
1369
|
|
|
1226
1370
|
if (embedding) {
|
|
1227
|
-
const
|
|
1371
|
+
const importThreshold = this.ctx?.config?.dedup?.similarityThreshold ?? 0.60;
|
|
1372
|
+
const dedupOwnerFilter = [importOwner];
|
|
1373
|
+
const topSimilar = findTopSimilar(this.store, embedding, importThreshold, 5, this.log, dedupOwnerFilter);
|
|
1228
1374
|
if (topSimilar.length > 0) {
|
|
1229
1375
|
const candidates = topSimilar.map((s, idx) => {
|
|
1230
1376
|
const chunk = this.store.getChunk(s.chunkId);
|
|
@@ -1315,18 +1461,34 @@ export class ViewerServer {
|
|
|
1315
1461
|
}
|
|
1316
1462
|
}
|
|
1317
1463
|
|
|
1318
|
-
// Phase 2: Import session JSONL files
|
|
1464
|
+
// Phase 2: Import session JSONL files from ALL agents (supports parallel by agent)
|
|
1319
1465
|
if (importSessions) {
|
|
1320
|
-
const
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1466
|
+
const agentsDir = path.join(ocHome, "agents");
|
|
1467
|
+
const agentGroups: Map<string, Array<{ file: string; filePath: string }>> = new Map();
|
|
1468
|
+
if (fs.existsSync(agentsDir)) {
|
|
1469
|
+
for (const entry of fs.readdirSync(agentsDir, { withFileTypes: true })) {
|
|
1470
|
+
if (entry.isDirectory()) {
|
|
1471
|
+
const sessDir = path.join(agentsDir, entry.name, "sessions");
|
|
1472
|
+
if (fs.existsSync(sessDir)) {
|
|
1473
|
+
const jsonlFiles = fs.readdirSync(sessDir).filter(f => f.includes(".jsonl")).sort();
|
|
1474
|
+
if (jsonlFiles.length > 0) {
|
|
1475
|
+
agentGroups.set(entry.name, jsonlFiles.map(f => ({ file: f, filePath: path.join(sessDir, f) })));
|
|
1476
|
+
}
|
|
1477
|
+
}
|
|
1478
|
+
}
|
|
1479
|
+
}
|
|
1480
|
+
}
|
|
1324
1481
|
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1482
|
+
const agentIds = Array.from(agentGroups.keys());
|
|
1483
|
+
const allFileCount = Array.from(agentGroups.values()).reduce((s, g) => s + g.length, 0);
|
|
1484
|
+
send("phase", { phase: "sessions", files: allFileCount, agents: agentIds, concurrency });
|
|
1485
|
+
|
|
1486
|
+
// Count total messages across all agents
|
|
1487
|
+
let totalMsgs = 0;
|
|
1488
|
+
for (const files of agentGroups.values()) {
|
|
1489
|
+
for (const { filePath } of files) {
|
|
1328
1490
|
try {
|
|
1329
|
-
const raw = fs.readFileSync(
|
|
1491
|
+
const raw = fs.readFileSync(filePath, "utf-8");
|
|
1330
1492
|
for (const line of raw.split("\n")) {
|
|
1331
1493
|
if (!line.trim()) continue;
|
|
1332
1494
|
try {
|
|
@@ -1347,12 +1509,18 @@ export class ViewerServer {
|
|
|
1347
1509
|
}
|
|
1348
1510
|
} catch { /* skip */ }
|
|
1349
1511
|
}
|
|
1512
|
+
}
|
|
1350
1513
|
|
|
1351
|
-
|
|
1514
|
+
// Thread-safe counters for parallel execution
|
|
1515
|
+
let globalMsgIdx = 0;
|
|
1516
|
+
const incIdx = () => ++globalMsgIdx;
|
|
1517
|
+
|
|
1518
|
+
// Import one agent's sessions sequentially
|
|
1519
|
+
const importAgent = async (agentId: string, files: Array<{ file: string; filePath: string }>) => {
|
|
1520
|
+
const agentOwner = `agent:${agentId}`;
|
|
1521
|
+
for (const { file, filePath } of files) {
|
|
1352
1522
|
if (this.migrationAbort) break;
|
|
1353
1523
|
const sessionId = file.replace(/\.jsonl.*$/, "");
|
|
1354
|
-
const filePath = path.join(sessionsDir, file);
|
|
1355
|
-
send("progress", { total: totalMsgs, processed: globalMsgIdx, phase: "sessions", file });
|
|
1356
1524
|
|
|
1357
1525
|
try {
|
|
1358
1526
|
const fileStream = fs.createReadStream(filePath, { encoding: "utf-8" });
|
|
@@ -1384,21 +1552,20 @@ export class ViewerServer {
|
|
|
1384
1552
|
}
|
|
1385
1553
|
if (!content || content.length < 10) continue;
|
|
1386
1554
|
|
|
1387
|
-
|
|
1555
|
+
const idx = incIdx();
|
|
1388
1556
|
totalProcessed++;
|
|
1389
1557
|
|
|
1390
1558
|
const sessionKey = `openclaw-session-${sessionId}`;
|
|
1391
1559
|
if (this.store.chunkExistsByContent(sessionKey, msgRole, content)) {
|
|
1392
1560
|
totalSkipped++;
|
|
1393
|
-
send("item", {
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
});
|
|
1561
|
+
send("item", { index: idx, total: totalMsgs, status: "skipped", preview: content.slice(0, 120), source: file, agent: agentId, role: msgRole, reason: "duplicate" });
|
|
1562
|
+
continue;
|
|
1563
|
+
}
|
|
1564
|
+
|
|
1565
|
+
const existingByHash = this.store.findActiveChunkByHash(content, agentOwner);
|
|
1566
|
+
if (existingByHash) {
|
|
1567
|
+
totalSkipped++;
|
|
1568
|
+
send("item", { index: idx, total: totalMsgs, status: "skipped", preview: content.slice(0, 120), source: file, agent: agentId, role: msgRole, reason: "exact duplicate within agent" });
|
|
1402
1569
|
continue;
|
|
1403
1570
|
}
|
|
1404
1571
|
|
|
@@ -1416,33 +1583,26 @@ export class ViewerServer {
|
|
|
1416
1583
|
let dedupReason: string | null = null;
|
|
1417
1584
|
|
|
1418
1585
|
if (embedding) {
|
|
1419
|
-
const
|
|
1586
|
+
const importThreshold = this.ctx?.config?.dedup?.similarityThreshold ?? 0.60;
|
|
1587
|
+
const dedupOwnerFilter = [agentOwner];
|
|
1588
|
+
const topSimilar = findTopSimilar(this.store, embedding, importThreshold, 5, this.log, dedupOwnerFilter);
|
|
1420
1589
|
if (topSimilar.length > 0) {
|
|
1421
|
-
const candidates = topSimilar.map((s,
|
|
1590
|
+
const candidates = topSimilar.map((s, i) => {
|
|
1422
1591
|
const chunk = this.store.getChunk(s.chunkId);
|
|
1423
|
-
return { index:
|
|
1592
|
+
return { index: i + 1, summary: chunk?.summary ?? "", chunkId: s.chunkId };
|
|
1424
1593
|
}).filter(c => c.summary);
|
|
1425
1594
|
|
|
1426
1595
|
if (candidates.length > 0) {
|
|
1427
1596
|
const dedupResult = await summarizer.judgeDedup(summary, candidates);
|
|
1428
1597
|
if (dedupResult?.action === "DUPLICATE" && dedupResult.targetIndex) {
|
|
1429
1598
|
const targetId = candidates[dedupResult.targetIndex - 1]?.chunkId;
|
|
1430
|
-
if (targetId) {
|
|
1431
|
-
dedupStatus = "duplicate";
|
|
1432
|
-
dedupTarget = targetId;
|
|
1433
|
-
dedupReason = dedupResult.reason;
|
|
1434
|
-
}
|
|
1599
|
+
if (targetId) { dedupStatus = "duplicate"; dedupTarget = targetId; dedupReason = dedupResult.reason; }
|
|
1435
1600
|
} else if (dedupResult?.action === "UPDATE" && dedupResult.targetIndex && dedupResult.mergedSummary) {
|
|
1436
1601
|
const targetId = candidates[dedupResult.targetIndex - 1]?.chunkId;
|
|
1437
1602
|
if (targetId) {
|
|
1438
1603
|
this.store.updateChunkSummaryAndContent(targetId, dedupResult.mergedSummary, content);
|
|
1439
|
-
try {
|
|
1440
|
-
|
|
1441
|
-
if (newEmb) this.store.upsertEmbedding(targetId, newEmb);
|
|
1442
|
-
} catch { /* best-effort */ }
|
|
1443
|
-
dedupStatus = "merged";
|
|
1444
|
-
dedupTarget = targetId;
|
|
1445
|
-
dedupReason = dedupResult.reason;
|
|
1604
|
+
try { const [newEmb] = await this.embedder.embed([dedupResult.mergedSummary]); if (newEmb) this.store.upsertEmbedding(targetId, newEmb); } catch { /* best-effort */ }
|
|
1605
|
+
dedupStatus = "merged"; dedupTarget = targetId; dedupReason = dedupResult.reason;
|
|
1446
1606
|
}
|
|
1447
1607
|
}
|
|
1448
1608
|
}
|
|
@@ -1453,60 +1613,53 @@ export class ViewerServer {
|
|
|
1453
1613
|
const msgTs = obj.message?.timestamp ?? obj.timestamp;
|
|
1454
1614
|
const ts = msgTs ? new Date(msgTs).getTime() : Date.now();
|
|
1455
1615
|
const chunk: Chunk = {
|
|
1456
|
-
id: chunkId,
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
role: msgRole as any,
|
|
1461
|
-
content,
|
|
1462
|
-
kind: "paragraph",
|
|
1463
|
-
summary,
|
|
1464
|
-
embedding: null,
|
|
1465
|
-
taskId: null,
|
|
1466
|
-
skillId: null,
|
|
1467
|
-
owner: "agent:main",
|
|
1468
|
-
dedupStatus,
|
|
1469
|
-
dedupTarget,
|
|
1470
|
-
dedupReason,
|
|
1471
|
-
mergeCount: 0,
|
|
1472
|
-
lastHitAt: null,
|
|
1473
|
-
mergeHistory: "[]",
|
|
1474
|
-
createdAt: ts,
|
|
1475
|
-
updatedAt: ts,
|
|
1616
|
+
id: chunkId, sessionKey, turnId: `import-${agentId}-${sessionId}-${idx}`, seq: 0,
|
|
1617
|
+
role: msgRole as any, content, kind: "paragraph", summary, embedding: null,
|
|
1618
|
+
taskId: null, skillId: null, owner: agentOwner, dedupStatus, dedupTarget, dedupReason,
|
|
1619
|
+
mergeCount: 0, lastHitAt: null, mergeHistory: "[]", createdAt: ts, updatedAt: ts,
|
|
1476
1620
|
};
|
|
1477
1621
|
|
|
1478
1622
|
this.store.insertChunk(chunk);
|
|
1479
|
-
if (embedding && dedupStatus === "active")
|
|
1480
|
-
this.store.upsertEmbedding(chunkId, embedding);
|
|
1481
|
-
}
|
|
1623
|
+
if (embedding && dedupStatus === "active") this.store.upsertEmbedding(chunkId, embedding);
|
|
1482
1624
|
|
|
1483
1625
|
totalStored++;
|
|
1484
|
-
send("item", {
|
|
1485
|
-
index: globalMsgIdx,
|
|
1486
|
-
total: totalMsgs,
|
|
1487
|
-
status: dedupStatus === "active" ? "stored" : dedupStatus,
|
|
1488
|
-
preview: content.slice(0, 120),
|
|
1489
|
-
summary: summary.slice(0, 80),
|
|
1490
|
-
source: file,
|
|
1491
|
-
role: msgRole,
|
|
1492
|
-
});
|
|
1626
|
+
send("item", { index: idx, total: totalMsgs, status: dedupStatus === "active" ? "stored" : dedupStatus, preview: content.slice(0, 120), summary: summary.slice(0, 80), source: file, agent: agentId, role: msgRole });
|
|
1493
1627
|
} catch (err) {
|
|
1494
1628
|
totalErrors++;
|
|
1495
|
-
send("item", {
|
|
1496
|
-
index: globalMsgIdx,
|
|
1497
|
-
total: totalMsgs,
|
|
1498
|
-
status: "error",
|
|
1499
|
-
preview: content.slice(0, 120),
|
|
1500
|
-
source: file,
|
|
1501
|
-
error: String(err).slice(0, 200),
|
|
1502
|
-
});
|
|
1629
|
+
send("item", { index: idx, total: totalMsgs, status: "error", preview: content.slice(0, 120), source: file, agent: agentId, error: String(err).slice(0, 200) });
|
|
1503
1630
|
}
|
|
1504
1631
|
}
|
|
1505
1632
|
} catch (err) {
|
|
1506
|
-
send("error", { file, error: String(err) });
|
|
1633
|
+
send("error", { file, agent: agentId, error: String(err) });
|
|
1507
1634
|
totalErrors++;
|
|
1508
1635
|
}
|
|
1509
1636
|
}
|
|
1637
|
+
};
|
|
1638
|
+
|
|
1639
|
+
// Execute agents with concurrency control
|
|
1640
|
+
const agentEntries = Array.from(agentGroups.entries());
|
|
1641
|
+
if (concurrency <= 1 || agentEntries.length <= 1) {
|
|
1642
|
+
for (const [agentId, files] of agentEntries) {
|
|
1643
|
+
if (this.migrationAbort) break;
|
|
1644
|
+
send("progress", { total: totalMsgs, processed: globalMsgIdx, phase: "sessions", agent: agentId });
|
|
1645
|
+
await importAgent(agentId, files);
|
|
1646
|
+
}
|
|
1647
|
+
} else {
|
|
1648
|
+
// Parallel: run up to `concurrency` agents at once
|
|
1649
|
+
let cursor = 0;
|
|
1650
|
+
const runBatch = async () => {
|
|
1651
|
+
while (cursor < agentEntries.length && !this.migrationAbort) {
|
|
1652
|
+
const batch: Promise<void>[] = [];
|
|
1653
|
+
const batchStart = cursor;
|
|
1654
|
+
while (batch.length < concurrency && cursor < agentEntries.length) {
|
|
1655
|
+
const [agentId, files] = agentEntries[cursor++];
|
|
1656
|
+
send("progress", { total: totalMsgs, processed: globalMsgIdx, phase: "sessions", agent: agentId, parallel: true });
|
|
1657
|
+
batch.push(importAgent(agentId, files));
|
|
1658
|
+
}
|
|
1659
|
+
await Promise.all(batch);
|
|
1660
|
+
}
|
|
1661
|
+
};
|
|
1662
|
+
await runBatch();
|
|
1510
1663
|
}
|
|
1511
1664
|
}
|
|
1512
1665
|
|
|
@@ -1529,9 +1682,11 @@ export class ViewerServer {
|
|
|
1529
1682
|
}
|
|
1530
1683
|
|
|
1531
1684
|
this.readBody(req, (body) => {
|
|
1532
|
-
let opts: { enableTasks?: boolean; enableSkills?: boolean } = {};
|
|
1685
|
+
let opts: { enableTasks?: boolean; enableSkills?: boolean; concurrency?: number } = {};
|
|
1533
1686
|
try { opts = JSON.parse(body); } catch { /* defaults */ }
|
|
1534
1687
|
|
|
1688
|
+
const concurrency = Math.max(1, Math.min(opts.concurrency ?? 1, 8));
|
|
1689
|
+
|
|
1535
1690
|
res.writeHead(200, {
|
|
1536
1691
|
"Content-Type": "text/event-stream",
|
|
1537
1692
|
"Cache-Control": "no-cache",
|
|
@@ -1550,7 +1705,7 @@ export class ViewerServer {
|
|
|
1550
1705
|
};
|
|
1551
1706
|
|
|
1552
1707
|
this.ppRunning = true;
|
|
1553
|
-
this.runPostprocess(send, !!opts.enableTasks, !!opts.enableSkills).finally(() => {
|
|
1708
|
+
this.runPostprocess(send, !!opts.enableTasks, !!opts.enableSkills, concurrency).finally(() => {
|
|
1554
1709
|
this.ppRunning = false;
|
|
1555
1710
|
this.ppState.running = false;
|
|
1556
1711
|
this.ppState.done = true;
|
|
@@ -1608,128 +1763,154 @@ export class ViewerServer {
|
|
|
1608
1763
|
send: (event: string, data: unknown) => void,
|
|
1609
1764
|
enableTasks: boolean,
|
|
1610
1765
|
enableSkills: boolean,
|
|
1766
|
+
concurrency: number = 1,
|
|
1611
1767
|
): Promise<void> {
|
|
1612
1768
|
const ctx = this.ctx!;
|
|
1613
|
-
const taskProcessor = new TaskProcessor(this.store, ctx);
|
|
1614
|
-
let skillEvolver: SkillEvolver | null = null;
|
|
1615
|
-
|
|
1616
|
-
if (enableSkills) {
|
|
1617
|
-
const recallEngine = new RecallEngine(this.store, this.embedder, ctx);
|
|
1618
|
-
skillEvolver = new SkillEvolver(this.store, recallEngine, ctx);
|
|
1619
|
-
taskProcessor.onTaskCompleted(async (task) => {
|
|
1620
|
-
try {
|
|
1621
|
-
await skillEvolver!.onTaskCompleted(task);
|
|
1622
|
-
this.ppState.skillsCreated++;
|
|
1623
|
-
send("skill", { taskId: task.id, title: task.title });
|
|
1624
|
-
} catch (err) {
|
|
1625
|
-
this.log.warn(`Postprocess skill evolution error: ${err}`);
|
|
1626
|
-
}
|
|
1627
|
-
});
|
|
1628
|
-
}
|
|
1629
1769
|
|
|
1630
1770
|
const importSessions = this.store.getDistinctSessionKeys()
|
|
1631
1771
|
.filter((sk: string) => sk.startsWith("openclaw-import-") || sk.startsWith("openclaw-session-"));
|
|
1632
1772
|
|
|
1633
|
-
type PendingItem = { sessionKey: string; action: "full" | "skill-only" };
|
|
1773
|
+
type PendingItem = { sessionKey: string; action: "full" | "skill-only"; owner: string };
|
|
1634
1774
|
const pendingItems: PendingItem[] = [];
|
|
1635
1775
|
let skippedCount = 0;
|
|
1636
1776
|
|
|
1777
|
+
const ownerMap = this.store.getSessionOwnerMap(importSessions);
|
|
1778
|
+
|
|
1637
1779
|
for (const sk of importSessions) {
|
|
1638
1780
|
const hasTask = this.store.hasTaskForSession(sk);
|
|
1639
1781
|
const hasSkill = this.store.hasSkillForSessionTask(sk);
|
|
1782
|
+
const owner = ownerMap.get(sk) ?? "agent:main";
|
|
1640
1783
|
|
|
1641
1784
|
if (enableTasks && !hasTask) {
|
|
1642
|
-
pendingItems.push({ sessionKey: sk, action: "full" });
|
|
1785
|
+
pendingItems.push({ sessionKey: sk, action: "full", owner });
|
|
1643
1786
|
} else if (enableSkills && hasTask && !hasSkill) {
|
|
1644
|
-
pendingItems.push({ sessionKey: sk, action: "skill-only" });
|
|
1787
|
+
pendingItems.push({ sessionKey: sk, action: "skill-only", owner });
|
|
1645
1788
|
} else {
|
|
1646
1789
|
skippedCount++;
|
|
1647
1790
|
}
|
|
1648
1791
|
}
|
|
1649
1792
|
|
|
1793
|
+
// Group pending items by agent (owner)
|
|
1794
|
+
const agentGroups = new Map<string, PendingItem[]>();
|
|
1795
|
+
for (const item of pendingItems) {
|
|
1796
|
+
const group = agentGroups.get(item.owner) ?? [];
|
|
1797
|
+
group.push(item);
|
|
1798
|
+
agentGroups.set(item.owner, group);
|
|
1799
|
+
}
|
|
1800
|
+
|
|
1650
1801
|
this.ppState.total = pendingItems.length;
|
|
1651
1802
|
send("info", {
|
|
1652
1803
|
totalSessions: importSessions.length,
|
|
1653
1804
|
alreadyProcessed: skippedCount,
|
|
1654
1805
|
pending: pendingItems.length,
|
|
1806
|
+
agents: Array.from(agentGroups.keys()),
|
|
1807
|
+
concurrency,
|
|
1655
1808
|
});
|
|
1656
1809
|
send("progress", { processed: 0, total: pendingItems.length });
|
|
1657
1810
|
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
const { sessionKey, action } = pendingItems[i];
|
|
1661
|
-
this.ppState.processed = i + 1;
|
|
1662
|
-
|
|
1663
|
-
send("item", {
|
|
1664
|
-
index: i + 1,
|
|
1665
|
-
total: pendingItems.length,
|
|
1666
|
-
session: sessionKey,
|
|
1667
|
-
step: "processing",
|
|
1668
|
-
action,
|
|
1669
|
-
});
|
|
1811
|
+
let globalIdx = 0;
|
|
1812
|
+
const incIdx = () => ++globalIdx;
|
|
1670
1813
|
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
});
|
|
1687
|
-
} else {
|
|
1688
|
-
send("item", {
|
|
1689
|
-
index: i + 1,
|
|
1690
|
-
total: pendingItems.length,
|
|
1691
|
-
session: sessionKey,
|
|
1692
|
-
step: "done",
|
|
1693
|
-
taskTitle: "(no chunks)",
|
|
1694
|
-
});
|
|
1814
|
+
// Process one agent's sessions sequentially
|
|
1815
|
+
const processAgent = async (agentOwner: string, items: PendingItem[]) => {
|
|
1816
|
+
const taskProcessor = new TaskProcessor(this.store, ctx);
|
|
1817
|
+
let skillEvolver: SkillEvolver | null = null;
|
|
1818
|
+
|
|
1819
|
+
if (enableSkills) {
|
|
1820
|
+
const recallEngine = new RecallEngine(this.store, this.embedder, ctx);
|
|
1821
|
+
skillEvolver = new SkillEvolver(this.store, recallEngine, ctx);
|
|
1822
|
+
taskProcessor.onTaskCompleted(async (task) => {
|
|
1823
|
+
try {
|
|
1824
|
+
await skillEvolver!.onTaskCompleted(task);
|
|
1825
|
+
this.ppState.skillsCreated++;
|
|
1826
|
+
send("skill", { taskId: task.id, title: task.title, agent: agentOwner });
|
|
1827
|
+
} catch (err) {
|
|
1828
|
+
this.log.warn(`Postprocess skill evolution error (${agentOwner}): ${err}`);
|
|
1695
1829
|
}
|
|
1696
|
-
}
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
|
|
1700
|
-
|
|
1701
|
-
|
|
1702
|
-
|
|
1703
|
-
|
|
1704
|
-
|
|
1705
|
-
|
|
1706
|
-
|
|
1707
|
-
|
|
1830
|
+
});
|
|
1831
|
+
}
|
|
1832
|
+
|
|
1833
|
+
for (const { sessionKey, action } of items) {
|
|
1834
|
+
if (this.ppAbort) break;
|
|
1835
|
+
const idx = incIdx();
|
|
1836
|
+
this.ppState.processed = globalIdx;
|
|
1837
|
+
|
|
1838
|
+
send("item", {
|
|
1839
|
+
index: idx,
|
|
1840
|
+
total: pendingItems.length,
|
|
1841
|
+
session: sessionKey,
|
|
1842
|
+
agent: agentOwner,
|
|
1843
|
+
step: "processing",
|
|
1844
|
+
action,
|
|
1845
|
+
});
|
|
1846
|
+
|
|
1847
|
+
try {
|
|
1848
|
+
if (action === "full") {
|
|
1849
|
+
await taskProcessor.onChunksIngested(sessionKey, Date.now());
|
|
1850
|
+
const activeTask = this.store.getActiveTask(sessionKey);
|
|
1851
|
+
if (activeTask) {
|
|
1852
|
+
await taskProcessor.finalizeTask(activeTask);
|
|
1853
|
+
const finalized = this.store.getTask(activeTask.id);
|
|
1854
|
+
this.ppState.tasksCreated++;
|
|
1855
|
+
send("item", {
|
|
1856
|
+
index: idx, total: pendingItems.length, session: sessionKey, agent: agentOwner,
|
|
1857
|
+
step: "done", taskTitle: finalized?.title || "", taskStatus: finalized?.status || "",
|
|
1858
|
+
});
|
|
1859
|
+
} else {
|
|
1860
|
+
send("item", {
|
|
1861
|
+
index: idx, total: pendingItems.length, session: sessionKey, agent: agentOwner,
|
|
1862
|
+
step: "done", taskTitle: "(no chunks)",
|
|
1863
|
+
});
|
|
1864
|
+
}
|
|
1865
|
+
} else if (action === "skill-only" && skillEvolver) {
|
|
1866
|
+
const completedTasks = this.store.getCompletedTasksForSession(sessionKey);
|
|
1867
|
+
let skillGenerated = false;
|
|
1868
|
+
for (const task of completedTasks) {
|
|
1869
|
+
if (this.ppAbort) break;
|
|
1870
|
+
try {
|
|
1871
|
+
await skillEvolver.onTaskCompleted(task);
|
|
1872
|
+
this.ppState.skillsCreated++;
|
|
1873
|
+
skillGenerated = true;
|
|
1874
|
+
send("skill", { taskId: task.id, title: task.title, agent: agentOwner });
|
|
1875
|
+
} catch (err) {
|
|
1876
|
+
this.log.warn(`Skill evolution error (${agentOwner}) task=${task.id}: ${err}`);
|
|
1877
|
+
}
|
|
1708
1878
|
}
|
|
1879
|
+
send("item", {
|
|
1880
|
+
index: idx, total: pendingItems.length, session: sessionKey, agent: agentOwner,
|
|
1881
|
+
step: "done", taskTitle: completedTasks[0]?.title || sessionKey, action: "skill-only", skillGenerated,
|
|
1882
|
+
});
|
|
1709
1883
|
}
|
|
1884
|
+
} catch (err) {
|
|
1885
|
+
this.ppState.errors++;
|
|
1886
|
+
this.log.warn(`Postprocess error (${agentOwner}) ${sessionKey}: ${err}`);
|
|
1710
1887
|
send("item", {
|
|
1711
|
-
index:
|
|
1712
|
-
|
|
1713
|
-
session: sessionKey,
|
|
1714
|
-
step: "done",
|
|
1715
|
-
taskTitle: completedTasks[0]?.title || sessionKey,
|
|
1716
|
-
action: "skill-only",
|
|
1717
|
-
skillGenerated,
|
|
1888
|
+
index: idx, total: pendingItems.length, session: sessionKey, agent: agentOwner,
|
|
1889
|
+
step: "error", error: String(err).slice(0, 200),
|
|
1718
1890
|
});
|
|
1719
1891
|
}
|
|
1720
|
-
|
|
1721
|
-
|
|
1722
|
-
this.log.warn(`Postprocess error for ${sessionKey}: ${err}`);
|
|
1723
|
-
send("item", {
|
|
1724
|
-
index: i + 1,
|
|
1725
|
-
total: pendingItems.length,
|
|
1726
|
-
session: sessionKey,
|
|
1727
|
-
step: "error",
|
|
1728
|
-
error: String(err).slice(0, 200),
|
|
1729
|
-
});
|
|
1892
|
+
|
|
1893
|
+
send("progress", { processed: globalIdx, total: pendingItems.length });
|
|
1730
1894
|
}
|
|
1895
|
+
};
|
|
1731
1896
|
|
|
1732
|
-
|
|
1897
|
+
// Execute agents with concurrency control
|
|
1898
|
+
const agentEntries = Array.from(agentGroups.entries());
|
|
1899
|
+
if (concurrency <= 1 || agentEntries.length <= 1) {
|
|
1900
|
+
for (const [agentOwner, items] of agentEntries) {
|
|
1901
|
+
if (this.ppAbort) break;
|
|
1902
|
+
await processAgent(agentOwner, items);
|
|
1903
|
+
}
|
|
1904
|
+
} else {
|
|
1905
|
+
let cursor = 0;
|
|
1906
|
+
while (cursor < agentEntries.length && !this.ppAbort) {
|
|
1907
|
+
const batch: Promise<void>[] = [];
|
|
1908
|
+
while (batch.length < concurrency && cursor < agentEntries.length) {
|
|
1909
|
+
const [agentOwner, items] = agentEntries[cursor++];
|
|
1910
|
+
batch.push(processAgent(agentOwner, items));
|
|
1911
|
+
}
|
|
1912
|
+
await Promise.all(batch);
|
|
1913
|
+
}
|
|
1733
1914
|
}
|
|
1734
1915
|
}
|
|
1735
1916
|
|