@memtensor/memos-local-openclaw-plugin 0.3.20 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +232 -22
- package/dist/capture/index.d.ts +1 -1
- package/dist/capture/index.d.ts.map +1 -1
- package/dist/capture/index.js +33 -8
- package/dist/capture/index.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/ingest/providers/anthropic.d.ts.map +1 -1
- package/dist/ingest/providers/anthropic.js +22 -8
- package/dist/ingest/providers/anthropic.js.map +1 -1
- package/dist/ingest/providers/bedrock.d.ts.map +1 -1
- package/dist/ingest/providers/bedrock.js +22 -8
- package/dist/ingest/providers/bedrock.js.map +1 -1
- package/dist/ingest/providers/gemini.d.ts.map +1 -1
- package/dist/ingest/providers/gemini.js +22 -8
- package/dist/ingest/providers/gemini.js.map +1 -1
- package/dist/ingest/providers/index.d.ts +13 -18
- package/dist/ingest/providers/index.d.ts.map +1 -1
- package/dist/ingest/providers/index.js +213 -139
- package/dist/ingest/providers/index.js.map +1 -1
- package/dist/ingest/providers/openai.d.ts +1 -1
- package/dist/ingest/providers/openai.d.ts.map +1 -1
- package/dist/ingest/providers/openai.js +37 -17
- package/dist/ingest/providers/openai.js.map +1 -1
- package/dist/ingest/task-processor.d.ts +28 -3
- package/dist/ingest/task-processor.d.ts.map +1 -1
- package/dist/ingest/task-processor.js +166 -67
- package/dist/ingest/task-processor.js.map +1 -1
- package/dist/ingest/worker.d.ts.map +1 -1
- package/dist/ingest/worker.js +97 -75
- package/dist/ingest/worker.js.map +1 -1
- package/dist/shared/llm-call.d.ts +26 -0
- package/dist/shared/llm-call.d.ts.map +1 -0
- package/dist/shared/llm-call.js +163 -0
- package/dist/shared/llm-call.js.map +1 -0
- package/dist/skill/evaluator.d.ts +0 -3
- package/dist/skill/evaluator.d.ts.map +1 -1
- package/dist/skill/evaluator.js +34 -59
- package/dist/skill/evaluator.js.map +1 -1
- package/dist/skill/evolver.d.ts +22 -1
- package/dist/skill/evolver.d.ts.map +1 -1
- package/dist/skill/evolver.js +191 -32
- package/dist/skill/evolver.js.map +1 -1
- package/dist/skill/generator.d.ts +0 -3
- package/dist/skill/generator.d.ts.map +1 -1
- package/dist/skill/generator.js +15 -50
- package/dist/skill/generator.js.map +1 -1
- package/dist/skill/upgrader.d.ts +0 -2
- package/dist/skill/upgrader.d.ts.map +1 -1
- package/dist/skill/upgrader.js +4 -39
- package/dist/skill/upgrader.js.map +1 -1
- package/dist/skill/validator.d.ts +0 -2
- package/dist/skill/validator.d.ts.map +1 -1
- package/dist/skill/validator.js +14 -44
- package/dist/skill/validator.js.map +1 -1
- package/dist/storage/sqlite.d.ts +13 -2
- package/dist/storage/sqlite.d.ts.map +1 -1
- package/dist/storage/sqlite.js +72 -6
- package/dist/storage/sqlite.js.map +1 -1
- package/dist/tools/memory-get.d.ts.map +1 -1
- package/dist/tools/memory-get.js +5 -1
- package/dist/tools/memory-get.js.map +1 -1
- package/dist/tools/memory-search.d.ts.map +1 -1
- package/dist/tools/memory-search.js +5 -0
- package/dist/tools/memory-search.js.map +1 -1
- package/dist/tools/memory-timeline.d.ts.map +1 -1
- package/dist/tools/memory-timeline.js +11 -2
- package/dist/tools/memory-timeline.js.map +1 -1
- package/dist/types.d.ts +2 -1
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +1 -1
- package/dist/types.js.map +1 -1
- package/dist/viewer/html.d.ts +1 -1
- package/dist/viewer/html.d.ts.map +1 -1
- package/dist/viewer/html.js +233 -9
- package/dist/viewer/html.js.map +1 -1
- package/dist/viewer/server.d.ts +5 -0
- package/dist/viewer/server.d.ts.map +1 -1
- package/dist/viewer/server.js +383 -177
- package/dist/viewer/server.js.map +1 -1
- package/index.ts +9 -3
- package/package.json +2 -1
- package/src/capture/index.ts +39 -10
- package/src/index.ts +3 -2
- package/src/ingest/providers/anthropic.ts +22 -8
- package/src/ingest/providers/bedrock.ts +22 -8
- package/src/ingest/providers/gemini.ts +22 -8
- package/src/ingest/providers/index.ts +192 -142
- package/src/ingest/providers/openai.ts +37 -17
- package/src/ingest/task-processor.ts +183 -65
- package/src/ingest/worker.ts +98 -77
- package/src/shared/llm-call.ts +144 -0
- package/src/skill/evaluator.ts +35 -64
- package/src/skill/evolver.ts +201 -33
- package/src/skill/generator.ts +16 -59
- package/src/skill/upgrader.ts +5 -43
- package/src/skill/validator.ts +15 -47
- package/src/storage/sqlite.ts +88 -6
- package/src/tools/memory-get.ts +6 -1
- package/src/tools/memory-search.ts +6 -0
- package/src/tools/memory-timeline.ts +13 -1
- package/src/types.ts +2 -1
- package/src/viewer/html.ts +233 -9
- package/src/viewer/server.ts +368 -187
package/dist/viewer/server.js
CHANGED
|
@@ -168,6 +168,12 @@ class ViewerServer {
|
|
|
168
168
|
this.serveSearch(req, res, url);
|
|
169
169
|
else if (p === "/api/tasks" && req.method === "GET")
|
|
170
170
|
this.serveTasks(res, url);
|
|
171
|
+
else if (p.match(/^\/api\/task\/[^/]+\/retry-skill$/) && req.method === "POST")
|
|
172
|
+
this.handleTaskRetrySkill(req, res, p);
|
|
173
|
+
else if (p.startsWith("/api/task/") && req.method === "DELETE")
|
|
174
|
+
this.handleTaskDelete(res, p);
|
|
175
|
+
else if (p.startsWith("/api/task/") && req.method === "PUT")
|
|
176
|
+
this.handleTaskUpdate(req, res, p);
|
|
171
177
|
else if (p.startsWith("/api/task/") && req.method === "GET")
|
|
172
178
|
this.serveTaskDetail(res, p);
|
|
173
179
|
else if (p === "/api/skills" && req.method === "GET")
|
|
@@ -178,6 +184,10 @@ class ViewerServer {
|
|
|
178
184
|
this.serveSkillFiles(res, p);
|
|
179
185
|
else if (p.match(/^\/api\/skill\/[^/]+\/visibility$/) && req.method === "PUT")
|
|
180
186
|
this.handleSkillVisibility(req, res, p);
|
|
187
|
+
else if (p.startsWith("/api/skill/") && req.method === "DELETE")
|
|
188
|
+
this.handleSkillDelete(res, p);
|
|
189
|
+
else if (p.startsWith("/api/skill/") && req.method === "PUT")
|
|
190
|
+
this.handleSkillUpdate(req, res, p);
|
|
181
191
|
else if (p.startsWith("/api/skill/") && req.method === "GET")
|
|
182
192
|
this.serveSkillDetail(res, p);
|
|
183
193
|
else if (p === "/api/memory" && req.method === "POST")
|
|
@@ -401,16 +411,21 @@ class ViewerServer {
|
|
|
401
411
|
const limit = Math.min(100, Math.max(1, Number(url.searchParams.get("limit")) || 50));
|
|
402
412
|
const offset = Math.max(0, Number(url.searchParams.get("offset")) || 0);
|
|
403
413
|
const { tasks, total } = this.store.listTasks({ status, limit, offset });
|
|
404
|
-
const
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
+
const db = this.store.db;
|
|
415
|
+
const items = tasks.map((t) => {
|
|
416
|
+
const meta = db.prepare("SELECT skill_status FROM tasks WHERE id = ?").get(t.id);
|
|
417
|
+
return {
|
|
418
|
+
id: t.id,
|
|
419
|
+
sessionKey: t.sessionKey,
|
|
420
|
+
title: t.title,
|
|
421
|
+
summary: t.summary ? (t.summary.length > 300 ? t.summary.slice(0, 297) + "..." : t.summary) : "",
|
|
422
|
+
status: t.status,
|
|
423
|
+
startedAt: t.startedAt,
|
|
424
|
+
endedAt: t.endedAt,
|
|
425
|
+
chunkCount: this.store.countChunksByTask(t.id),
|
|
426
|
+
skillStatus: meta?.skill_status ?? null,
|
|
427
|
+
};
|
|
428
|
+
});
|
|
414
429
|
this.jsonResponse(res, { tasks: items, total, limit, offset });
|
|
415
430
|
}
|
|
416
431
|
serveTaskDetail(res, urlPath) {
|
|
@@ -541,15 +556,17 @@ class ViewerServer {
|
|
|
541
556
|
catch {
|
|
542
557
|
ftsResults = db.prepare("SELECT * FROM chunks WHERE content LIKE ? OR summary LIKE ? ORDER BY created_at DESC LIMIT 100").all(`%${q}%`, `%${q}%`).filter(passesFilter);
|
|
543
558
|
}
|
|
559
|
+
const SEMANTIC_THRESHOLD = 0.64;
|
|
544
560
|
let vectorResults = [];
|
|
561
|
+
let scoreMap = new Map();
|
|
545
562
|
try {
|
|
546
563
|
const queryVec = await this.embedder.embedQuery(q);
|
|
547
564
|
const hits = (0, vector_1.vectorSearch)(this.store, queryVec, 40);
|
|
548
|
-
|
|
565
|
+
scoreMap = new Map(hits.map(h => [h.chunkId, h.score]));
|
|
566
|
+
const hitIds = new Set(hits.filter(h => h.score >= SEMANTIC_THRESHOLD).map(h => h.chunkId));
|
|
549
567
|
if (hitIds.size > 0) {
|
|
550
568
|
const placeholders = [...hitIds].map(() => "?").join(",");
|
|
551
569
|
const rows = db.prepare(`SELECT * FROM chunks WHERE id IN (${placeholders})`).all(...hitIds).filter(passesFilter);
|
|
552
|
-
const scoreMap = new Map(hits.map(h => [h.chunkId, h.score]));
|
|
553
570
|
rows.forEach((r) => { r._vscore = scoreMap.get(r.id) ?? 0; });
|
|
554
571
|
rows.sort((a, b) => (b._vscore ?? 0) - (a._vscore ?? 0));
|
|
555
572
|
vectorResults = rows;
|
|
@@ -567,18 +584,24 @@ class ViewerServer {
|
|
|
567
584
|
}
|
|
568
585
|
}
|
|
569
586
|
for (const r of ftsResults) {
|
|
570
|
-
if (
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
587
|
+
if (seenIds.has(r.id))
|
|
588
|
+
continue;
|
|
589
|
+
const vscore = scoreMap.get(r.id);
|
|
590
|
+
if (vscore !== undefined && vscore < SEMANTIC_THRESHOLD)
|
|
591
|
+
continue;
|
|
592
|
+
seenIds.add(r.id);
|
|
593
|
+
merged.push(r);
|
|
574
594
|
}
|
|
595
|
+
const fallback = merged.length === 0 && ftsResults.length > 0;
|
|
596
|
+
const results = fallback ? ftsResults.slice(0, 20) : merged;
|
|
575
597
|
this.store.recordViewerEvent("search");
|
|
576
598
|
this.jsonResponse(res, {
|
|
577
|
-
results
|
|
599
|
+
results,
|
|
578
600
|
query: q,
|
|
579
601
|
vectorCount: vectorResults.length,
|
|
580
602
|
ftsCount: ftsResults.length,
|
|
581
|
-
total:
|
|
603
|
+
total: results.length,
|
|
604
|
+
fallbackFts: fallback,
|
|
582
605
|
});
|
|
583
606
|
}
|
|
584
607
|
// ─── Skills API ───
|
|
@@ -737,18 +760,138 @@ class ViewerServer {
|
|
|
737
760
|
}
|
|
738
761
|
});
|
|
739
762
|
}
|
|
763
|
+
// ─── Task/Skill management ───
|
|
764
|
+
handleTaskRetrySkill(_req, res, urlPath) {
|
|
765
|
+
const taskId = urlPath.replace("/api/task/", "").replace("/retry-skill", "");
|
|
766
|
+
const task = this.store.getTask(taskId);
|
|
767
|
+
if (!task) {
|
|
768
|
+
res.writeHead(404, { "Content-Type": "application/json" });
|
|
769
|
+
res.end(JSON.stringify({ error: "Task not found" }));
|
|
770
|
+
return;
|
|
771
|
+
}
|
|
772
|
+
if (task.status !== "completed") {
|
|
773
|
+
res.writeHead(400, { "Content-Type": "application/json" });
|
|
774
|
+
res.end(JSON.stringify({ error: "Only completed tasks can retry skill generation" }));
|
|
775
|
+
return;
|
|
776
|
+
}
|
|
777
|
+
if (!this.ctx) {
|
|
778
|
+
res.writeHead(500, { "Content-Type": "application/json" });
|
|
779
|
+
res.end(JSON.stringify({ error: "Plugin context not available" }));
|
|
780
|
+
return;
|
|
781
|
+
}
|
|
782
|
+
// Clean up stale task_skills references (e.g., skill was manually deleted)
|
|
783
|
+
const db = this.store.db;
|
|
784
|
+
db.prepare("DELETE FROM task_skills WHERE task_id = ? AND skill_id NOT IN (SELECT id FROM skills)").run(taskId);
|
|
785
|
+
this.store.setTaskSkillMeta(taskId, { skillStatus: "queued", skillReason: "手动重试中..." });
|
|
786
|
+
this.jsonResponse(res, { ok: true, taskId, status: "queued" });
|
|
787
|
+
const ctx = this.ctx;
|
|
788
|
+
const recallEngine = new engine_1.RecallEngine(this.store, this.embedder, ctx);
|
|
789
|
+
const evolver = new evolver_1.SkillEvolver(this.store, recallEngine, ctx, this.embedder);
|
|
790
|
+
evolver.onTaskCompleted(task).then(() => {
|
|
791
|
+
this.log.info(`Retry skill generation completed for task ${taskId}`);
|
|
792
|
+
}).catch((err) => {
|
|
793
|
+
this.log.error(`Retry skill generation failed for task ${taskId}: ${err}`);
|
|
794
|
+
this.store.setTaskSkillMeta(taskId, { skillStatus: "skipped", skillReason: `error: ${err}` });
|
|
795
|
+
});
|
|
796
|
+
}
|
|
797
|
+
handleTaskDelete(res, urlPath) {
|
|
798
|
+
const taskId = urlPath.replace("/api/task/", "");
|
|
799
|
+
const deleted = this.store.deleteTask(taskId);
|
|
800
|
+
if (!deleted) {
|
|
801
|
+
res.writeHead(404, { "Content-Type": "application/json" });
|
|
802
|
+
res.end(JSON.stringify({ error: "Task not found" }));
|
|
803
|
+
return;
|
|
804
|
+
}
|
|
805
|
+
this.jsonResponse(res, { ok: true, taskId });
|
|
806
|
+
}
|
|
807
|
+
handleTaskUpdate(req, res, urlPath) {
|
|
808
|
+
const taskId = urlPath.replace("/api/task/", "");
|
|
809
|
+
this.readBody(req, (body) => {
|
|
810
|
+
try {
|
|
811
|
+
const data = JSON.parse(body);
|
|
812
|
+
const task = this.store.getTask(taskId);
|
|
813
|
+
if (!task) {
|
|
814
|
+
res.writeHead(404, { "Content-Type": "application/json" });
|
|
815
|
+
res.end(JSON.stringify({ error: "Task not found" }));
|
|
816
|
+
return;
|
|
817
|
+
}
|
|
818
|
+
this.store.updateTask(taskId, {
|
|
819
|
+
title: data.title ?? task.title,
|
|
820
|
+
summary: data.summary ?? task.summary,
|
|
821
|
+
status: data.status ?? task.status,
|
|
822
|
+
endedAt: task.endedAt ?? undefined,
|
|
823
|
+
});
|
|
824
|
+
this.jsonResponse(res, { ok: true, taskId });
|
|
825
|
+
}
|
|
826
|
+
catch (err) {
|
|
827
|
+
res.writeHead(400, { "Content-Type": "application/json" });
|
|
828
|
+
res.end(JSON.stringify({ error: String(err) }));
|
|
829
|
+
}
|
|
830
|
+
});
|
|
831
|
+
}
|
|
832
|
+
handleSkillDelete(res, urlPath) {
|
|
833
|
+
const skillId = urlPath.replace("/api/skill/", "");
|
|
834
|
+
const skill = this.store.getSkill(skillId);
|
|
835
|
+
if (!skill) {
|
|
836
|
+
res.writeHead(404, { "Content-Type": "application/json" });
|
|
837
|
+
res.end(JSON.stringify({ error: "Skill not found" }));
|
|
838
|
+
return;
|
|
839
|
+
}
|
|
840
|
+
// Remove skill directory from disk
|
|
841
|
+
try {
|
|
842
|
+
if (skill.dirPath && node_fs_1.default.existsSync(skill.dirPath)) {
|
|
843
|
+
node_fs_1.default.rmSync(skill.dirPath, { recursive: true, force: true });
|
|
844
|
+
}
|
|
845
|
+
}
|
|
846
|
+
catch (err) {
|
|
847
|
+
this.log.warn(`Failed to remove skill directory ${skill.dirPath}: ${err}`);
|
|
848
|
+
}
|
|
849
|
+
this.store.deleteSkill(skillId);
|
|
850
|
+
this.jsonResponse(res, { ok: true, skillId });
|
|
851
|
+
}
|
|
852
|
+
handleSkillUpdate(req, res, urlPath) {
|
|
853
|
+
const skillId = urlPath.replace("/api/skill/", "");
|
|
854
|
+
this.readBody(req, (body) => {
|
|
855
|
+
try {
|
|
856
|
+
const data = JSON.parse(body);
|
|
857
|
+
const skill = this.store.getSkill(skillId);
|
|
858
|
+
if (!skill) {
|
|
859
|
+
res.writeHead(404, { "Content-Type": "application/json" });
|
|
860
|
+
res.end(JSON.stringify({ error: "Skill not found" }));
|
|
861
|
+
return;
|
|
862
|
+
}
|
|
863
|
+
this.store.updateSkill(skillId, {
|
|
864
|
+
description: data.description ?? skill.description,
|
|
865
|
+
version: skill.version,
|
|
866
|
+
status: data.status ?? skill.status,
|
|
867
|
+
installed: skill.installed,
|
|
868
|
+
qualityScore: skill.qualityScore,
|
|
869
|
+
});
|
|
870
|
+
this.jsonResponse(res, { ok: true, skillId });
|
|
871
|
+
}
|
|
872
|
+
catch (err) {
|
|
873
|
+
res.writeHead(400, { "Content-Type": "application/json" });
|
|
874
|
+
res.end(JSON.stringify({ error: String(err) }));
|
|
875
|
+
}
|
|
876
|
+
});
|
|
877
|
+
}
|
|
740
878
|
// ─── CRUD ───
|
|
741
879
|
handleCreate(req, res) {
|
|
742
880
|
this.readBody(req, (body) => {
|
|
743
881
|
try {
|
|
744
882
|
const data = JSON.parse(body);
|
|
883
|
+
if (!data.content || typeof data.content !== "string" || !data.content.trim()) {
|
|
884
|
+
res.writeHead(400, { "Content-Type": "application/json" });
|
|
885
|
+
res.end(JSON.stringify({ error: "content is required and must be a non-empty string" }));
|
|
886
|
+
return;
|
|
887
|
+
}
|
|
745
888
|
const { v4: uuidv4 } = require("uuid");
|
|
746
889
|
const id = uuidv4();
|
|
747
890
|
const now = Date.now();
|
|
748
891
|
this.store.insertChunk({
|
|
749
892
|
id, sessionKey: data.session_key || "manual", turnId: `manual-${now}`, seq: 0,
|
|
750
|
-
role: data.role || "user", content: data.content
|
|
751
|
-
summary: data.summary || data.content
|
|
893
|
+
role: data.role || "user", content: data.content, kind: data.kind || "paragraph",
|
|
894
|
+
summary: data.summary || data.content.slice(0, 100),
|
|
752
895
|
taskId: null, skillId: null, owner: data.owner || "agent:main",
|
|
753
896
|
dedupStatus: "active", dedupTarget: null, dedupReason: null,
|
|
754
897
|
mergeCount: 0, lastHitAt: null, mergeHistory: "[]",
|
|
@@ -780,6 +923,11 @@ class ViewerServer {
|
|
|
780
923
|
this.readBody(req, (body) => {
|
|
781
924
|
try {
|
|
782
925
|
const data = JSON.parse(body);
|
|
926
|
+
if (data.content !== undefined && (typeof data.content !== "string" || !data.content.trim())) {
|
|
927
|
+
res.writeHead(400, { "Content-Type": "application/json" });
|
|
928
|
+
res.end(JSON.stringify({ error: "content must be a non-empty string" }));
|
|
929
|
+
return;
|
|
930
|
+
}
|
|
783
931
|
const ok = this.store.updateChunk(chunkId, { summary: data.summary, content: data.content, role: data.role, kind: data.kind, owner: data.owner });
|
|
784
932
|
if (ok)
|
|
785
933
|
this.jsonResponse(res, { ok: true, message: "Memory updated" });
|
|
@@ -1108,6 +1256,7 @@ class ViewerServer {
|
|
|
1108
1256
|
opts = JSON.parse(body);
|
|
1109
1257
|
}
|
|
1110
1258
|
catch { /* defaults */ }
|
|
1259
|
+
const concurrency = Math.max(1, Math.min(opts.concurrency ?? 1, 8));
|
|
1111
1260
|
res.writeHead(200, {
|
|
1112
1261
|
"Content-Type": "text/event-stream",
|
|
1113
1262
|
"Cache-Control": "no-cache",
|
|
@@ -1144,7 +1293,7 @@ class ViewerServer {
|
|
|
1144
1293
|
this.broadcastSSE(event, data);
|
|
1145
1294
|
};
|
|
1146
1295
|
this.migrationRunning = true;
|
|
1147
|
-
this.runMigration(send, opts.sources).finally(() => {
|
|
1296
|
+
this.runMigration(send, opts.sources, concurrency).finally(() => {
|
|
1148
1297
|
this.migrationRunning = false;
|
|
1149
1298
|
this.migrationState.done = true;
|
|
1150
1299
|
if (this.migrationAbort) {
|
|
@@ -1165,7 +1314,7 @@ class ViewerServer {
|
|
|
1165
1314
|
});
|
|
1166
1315
|
});
|
|
1167
1316
|
}
|
|
1168
|
-
async runMigration(send, sources) {
|
|
1317
|
+
async runMigration(send, sources, concurrency = 1) {
|
|
1169
1318
|
const ocHome = this.getOpenClawHome();
|
|
1170
1319
|
const importSqlite = !sources || sources.includes("sqlite");
|
|
1171
1320
|
const importSessions = !sources || sources.includes("sessions");
|
|
@@ -1175,15 +1324,17 @@ class ViewerServer {
|
|
|
1175
1324
|
let totalErrors = 0;
|
|
1176
1325
|
const cfgPath = this.getOpenClawConfigPath();
|
|
1177
1326
|
let summarizerCfg;
|
|
1327
|
+
let strongCfg;
|
|
1178
1328
|
try {
|
|
1179
1329
|
const raw = JSON.parse(node_fs_1.default.readFileSync(cfgPath, "utf-8"));
|
|
1180
1330
|
const pluginCfg = raw?.plugins?.entries?.["memos-local-openclaw-plugin"]?.config ??
|
|
1181
1331
|
raw?.plugins?.entries?.["memos-lite"]?.config ??
|
|
1182
1332
|
raw?.plugins?.entries?.["memos-lite-openclaw-plugin"]?.config ?? {};
|
|
1183
1333
|
summarizerCfg = pluginCfg.summarizer;
|
|
1334
|
+
strongCfg = pluginCfg.skillEvolution?.summarizer;
|
|
1184
1335
|
}
|
|
1185
1336
|
catch { /* no config */ }
|
|
1186
|
-
const summarizer = new providers_1.Summarizer(summarizerCfg, this.log);
|
|
1337
|
+
const summarizer = new providers_1.Summarizer(summarizerCfg, this.log, strongCfg);
|
|
1187
1338
|
// Phase 1: Import SQLite memory chunks
|
|
1188
1339
|
if (importSqlite) {
|
|
1189
1340
|
const memoryDir = node_path_1.default.join(ocHome, "memory");
|
|
@@ -1218,6 +1369,21 @@ class ViewerServer {
|
|
|
1218
1369
|
});
|
|
1219
1370
|
continue;
|
|
1220
1371
|
}
|
|
1372
|
+
const importOwner = `agent:${agentId}`;
|
|
1373
|
+
// Exact hash dedup within same agent
|
|
1374
|
+
const existingByHash = this.store.findActiveChunkByHash(row.text, importOwner);
|
|
1375
|
+
if (existingByHash) {
|
|
1376
|
+
totalSkipped++;
|
|
1377
|
+
send("item", {
|
|
1378
|
+
index: i + 1,
|
|
1379
|
+
total: rows.length,
|
|
1380
|
+
status: "skipped",
|
|
1381
|
+
preview: row.text.slice(0, 120),
|
|
1382
|
+
source: file,
|
|
1383
|
+
reason: "exact duplicate within agent",
|
|
1384
|
+
});
|
|
1385
|
+
continue;
|
|
1386
|
+
}
|
|
1221
1387
|
try {
|
|
1222
1388
|
const summary = await summarizer.summarize(row.text);
|
|
1223
1389
|
let embedding = null;
|
|
@@ -1231,7 +1397,9 @@ class ViewerServer {
|
|
|
1231
1397
|
let dedupTarget = null;
|
|
1232
1398
|
let dedupReason = null;
|
|
1233
1399
|
if (embedding) {
|
|
1234
|
-
const
|
|
1400
|
+
const importThreshold = this.ctx?.config?.dedup?.similarityThreshold ?? 0.60;
|
|
1401
|
+
const dedupOwnerFilter = [importOwner];
|
|
1402
|
+
const topSimilar = (0, dedup_1.findTopSimilar)(this.store, embedding, importThreshold, 5, this.log, dedupOwnerFilter);
|
|
1235
1403
|
if (topSimilar.length > 0) {
|
|
1236
1404
|
const candidates = topSimilar.map((s, idx) => {
|
|
1237
1405
|
const chunk = this.store.getChunk(s.chunkId);
|
|
@@ -1322,17 +1490,32 @@ class ViewerServer {
|
|
|
1322
1490
|
}
|
|
1323
1491
|
}
|
|
1324
1492
|
}
|
|
1325
|
-
// Phase 2: Import session JSONL files
|
|
1493
|
+
// Phase 2: Import session JSONL files from ALL agents (supports parallel by agent)
|
|
1326
1494
|
if (importSessions) {
|
|
1327
|
-
const
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1495
|
+
const agentsDir = node_path_1.default.join(ocHome, "agents");
|
|
1496
|
+
const agentGroups = new Map();
|
|
1497
|
+
if (node_fs_1.default.existsSync(agentsDir)) {
|
|
1498
|
+
for (const entry of node_fs_1.default.readdirSync(agentsDir, { withFileTypes: true })) {
|
|
1499
|
+
if (entry.isDirectory()) {
|
|
1500
|
+
const sessDir = node_path_1.default.join(agentsDir, entry.name, "sessions");
|
|
1501
|
+
if (node_fs_1.default.existsSync(sessDir)) {
|
|
1502
|
+
const jsonlFiles = node_fs_1.default.readdirSync(sessDir).filter(f => f.includes(".jsonl")).sort();
|
|
1503
|
+
if (jsonlFiles.length > 0) {
|
|
1504
|
+
agentGroups.set(entry.name, jsonlFiles.map(f => ({ file: f, filePath: node_path_1.default.join(sessDir, f) })));
|
|
1505
|
+
}
|
|
1506
|
+
}
|
|
1507
|
+
}
|
|
1508
|
+
}
|
|
1509
|
+
}
|
|
1510
|
+
const agentIds = Array.from(agentGroups.keys());
|
|
1511
|
+
const allFileCount = Array.from(agentGroups.values()).reduce((s, g) => s + g.length, 0);
|
|
1512
|
+
send("phase", { phase: "sessions", files: allFileCount, agents: agentIds, concurrency });
|
|
1513
|
+
// Count total messages across all agents
|
|
1514
|
+
let totalMsgs = 0;
|
|
1515
|
+
for (const files of agentGroups.values()) {
|
|
1516
|
+
for (const { filePath } of files) {
|
|
1334
1517
|
try {
|
|
1335
|
-
const raw = node_fs_1.default.readFileSync(
|
|
1518
|
+
const raw = node_fs_1.default.readFileSync(filePath, "utf-8");
|
|
1336
1519
|
for (const line of raw.split("\n")) {
|
|
1337
1520
|
if (!line.trim())
|
|
1338
1521
|
continue;
|
|
@@ -1361,12 +1544,17 @@ class ViewerServer {
|
|
|
1361
1544
|
}
|
|
1362
1545
|
catch { /* skip */ }
|
|
1363
1546
|
}
|
|
1364
|
-
|
|
1547
|
+
}
|
|
1548
|
+
// Thread-safe counters for parallel execution
|
|
1549
|
+
let globalMsgIdx = 0;
|
|
1550
|
+
const incIdx = () => ++globalMsgIdx;
|
|
1551
|
+
// Import one agent's sessions sequentially
|
|
1552
|
+
const importAgent = async (agentId, files) => {
|
|
1553
|
+
const agentOwner = `agent:${agentId}`;
|
|
1554
|
+
for (const { file, filePath } of files) {
|
|
1365
1555
|
if (this.migrationAbort)
|
|
1366
1556
|
break;
|
|
1367
1557
|
const sessionId = file.replace(/\.jsonl.*$/, "");
|
|
1368
|
-
const filePath = node_path_1.default.join(sessionsDir, file);
|
|
1369
|
-
send("progress", { total: totalMsgs, processed: globalMsgIdx, phase: "sessions", file });
|
|
1370
1558
|
try {
|
|
1371
1559
|
const fileStream = node_fs_1.default.createReadStream(filePath, { encoding: "utf-8" });
|
|
1372
1560
|
const rl = node_readline_1.default.createInterface({ input: fileStream, crlfDelay: Infinity });
|
|
@@ -1406,20 +1594,18 @@ class ViewerServer {
|
|
|
1406
1594
|
}
|
|
1407
1595
|
if (!content || content.length < 10)
|
|
1408
1596
|
continue;
|
|
1409
|
-
|
|
1597
|
+
const idx = incIdx();
|
|
1410
1598
|
totalProcessed++;
|
|
1411
1599
|
const sessionKey = `openclaw-session-${sessionId}`;
|
|
1412
1600
|
if (this.store.chunkExistsByContent(sessionKey, msgRole, content)) {
|
|
1413
1601
|
totalSkipped++;
|
|
1414
|
-
send("item", {
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
reason: "duplicate",
|
|
1422
|
-
});
|
|
1602
|
+
send("item", { index: idx, total: totalMsgs, status: "skipped", preview: content.slice(0, 120), source: file, agent: agentId, role: msgRole, reason: "duplicate" });
|
|
1603
|
+
continue;
|
|
1604
|
+
}
|
|
1605
|
+
const existingByHash = this.store.findActiveChunkByHash(content, agentOwner);
|
|
1606
|
+
if (existingByHash) {
|
|
1607
|
+
totalSkipped++;
|
|
1608
|
+
send("item", { index: idx, total: totalMsgs, status: "skipped", preview: content.slice(0, 120), source: file, agent: agentId, role: msgRole, reason: "exact duplicate within agent" });
|
|
1423
1609
|
continue;
|
|
1424
1610
|
}
|
|
1425
1611
|
try {
|
|
@@ -1435,11 +1621,13 @@ class ViewerServer {
|
|
|
1435
1621
|
let dedupTarget = null;
|
|
1436
1622
|
let dedupReason = null;
|
|
1437
1623
|
if (embedding) {
|
|
1438
|
-
const
|
|
1624
|
+
const importThreshold = this.ctx?.config?.dedup?.similarityThreshold ?? 0.60;
|
|
1625
|
+
const dedupOwnerFilter = [agentOwner];
|
|
1626
|
+
const topSimilar = (0, dedup_1.findTopSimilar)(this.store, embedding, importThreshold, 5, this.log, dedupOwnerFilter);
|
|
1439
1627
|
if (topSimilar.length > 0) {
|
|
1440
|
-
const candidates = topSimilar.map((s,
|
|
1628
|
+
const candidates = topSimilar.map((s, i) => {
|
|
1441
1629
|
const chunk = this.store.getChunk(s.chunkId);
|
|
1442
|
-
return { index:
|
|
1630
|
+
return { index: i + 1, summary: chunk?.summary ?? "", chunkId: s.chunkId };
|
|
1443
1631
|
}).filter(c => c.summary);
|
|
1444
1632
|
if (candidates.length > 0) {
|
|
1445
1633
|
const dedupResult = await summarizer.judgeDedup(summary, candidates);
|
|
@@ -1473,60 +1661,55 @@ class ViewerServer {
|
|
|
1473
1661
|
const msgTs = obj.message?.timestamp ?? obj.timestamp;
|
|
1474
1662
|
const ts = msgTs ? new Date(msgTs).getTime() : Date.now();
|
|
1475
1663
|
const chunk = {
|
|
1476
|
-
id: chunkId,
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
role: msgRole,
|
|
1481
|
-
content,
|
|
1482
|
-
kind: "paragraph",
|
|
1483
|
-
summary,
|
|
1484
|
-
embedding: null,
|
|
1485
|
-
taskId: null,
|
|
1486
|
-
skillId: null,
|
|
1487
|
-
owner: "agent:main",
|
|
1488
|
-
dedupStatus,
|
|
1489
|
-
dedupTarget,
|
|
1490
|
-
dedupReason,
|
|
1491
|
-
mergeCount: 0,
|
|
1492
|
-
lastHitAt: null,
|
|
1493
|
-
mergeHistory: "[]",
|
|
1494
|
-
createdAt: ts,
|
|
1495
|
-
updatedAt: ts,
|
|
1664
|
+
id: chunkId, sessionKey, turnId: `import-${agentId}-${sessionId}-${idx}`, seq: 0,
|
|
1665
|
+
role: msgRole, content, kind: "paragraph", summary, embedding: null,
|
|
1666
|
+
taskId: null, skillId: null, owner: agentOwner, dedupStatus, dedupTarget, dedupReason,
|
|
1667
|
+
mergeCount: 0, lastHitAt: null, mergeHistory: "[]", createdAt: ts, updatedAt: ts,
|
|
1496
1668
|
};
|
|
1497
1669
|
this.store.insertChunk(chunk);
|
|
1498
|
-
if (embedding && dedupStatus === "active")
|
|
1670
|
+
if (embedding && dedupStatus === "active")
|
|
1499
1671
|
this.store.upsertEmbedding(chunkId, embedding);
|
|
1500
|
-
}
|
|
1501
1672
|
totalStored++;
|
|
1502
|
-
send("item", {
|
|
1503
|
-
index: globalMsgIdx,
|
|
1504
|
-
total: totalMsgs,
|
|
1505
|
-
status: dedupStatus === "active" ? "stored" : dedupStatus,
|
|
1506
|
-
preview: content.slice(0, 120),
|
|
1507
|
-
summary: summary.slice(0, 80),
|
|
1508
|
-
source: file,
|
|
1509
|
-
role: msgRole,
|
|
1510
|
-
});
|
|
1673
|
+
send("item", { index: idx, total: totalMsgs, status: dedupStatus === "active" ? "stored" : dedupStatus, preview: content.slice(0, 120), summary: summary.slice(0, 80), source: file, agent: agentId, role: msgRole });
|
|
1511
1674
|
}
|
|
1512
1675
|
catch (err) {
|
|
1513
1676
|
totalErrors++;
|
|
1514
|
-
send("item", {
|
|
1515
|
-
index: globalMsgIdx,
|
|
1516
|
-
total: totalMsgs,
|
|
1517
|
-
status: "error",
|
|
1518
|
-
preview: content.slice(0, 120),
|
|
1519
|
-
source: file,
|
|
1520
|
-
error: String(err).slice(0, 200),
|
|
1521
|
-
});
|
|
1677
|
+
send("item", { index: idx, total: totalMsgs, status: "error", preview: content.slice(0, 120), source: file, agent: agentId, error: String(err).slice(0, 200) });
|
|
1522
1678
|
}
|
|
1523
1679
|
}
|
|
1524
1680
|
}
|
|
1525
1681
|
catch (err) {
|
|
1526
|
-
send("error", { file, error: String(err) });
|
|
1682
|
+
send("error", { file, agent: agentId, error: String(err) });
|
|
1527
1683
|
totalErrors++;
|
|
1528
1684
|
}
|
|
1529
1685
|
}
|
|
1686
|
+
};
|
|
1687
|
+
// Execute agents with concurrency control
|
|
1688
|
+
const agentEntries = Array.from(agentGroups.entries());
|
|
1689
|
+
if (concurrency <= 1 || agentEntries.length <= 1) {
|
|
1690
|
+
for (const [agentId, files] of agentEntries) {
|
|
1691
|
+
if (this.migrationAbort)
|
|
1692
|
+
break;
|
|
1693
|
+
send("progress", { total: totalMsgs, processed: globalMsgIdx, phase: "sessions", agent: agentId });
|
|
1694
|
+
await importAgent(agentId, files);
|
|
1695
|
+
}
|
|
1696
|
+
}
|
|
1697
|
+
else {
|
|
1698
|
+
// Parallel: run up to `concurrency` agents at once
|
|
1699
|
+
let cursor = 0;
|
|
1700
|
+
const runBatch = async () => {
|
|
1701
|
+
while (cursor < agentEntries.length && !this.migrationAbort) {
|
|
1702
|
+
const batch = [];
|
|
1703
|
+
const batchStart = cursor;
|
|
1704
|
+
while (batch.length < concurrency && cursor < agentEntries.length) {
|
|
1705
|
+
const [agentId, files] = agentEntries[cursor++];
|
|
1706
|
+
send("progress", { total: totalMsgs, processed: globalMsgIdx, phase: "sessions", agent: agentId, parallel: true });
|
|
1707
|
+
batch.push(importAgent(agentId, files));
|
|
1708
|
+
}
|
|
1709
|
+
await Promise.all(batch);
|
|
1710
|
+
}
|
|
1711
|
+
};
|
|
1712
|
+
await runBatch();
|
|
1530
1713
|
}
|
|
1531
1714
|
}
|
|
1532
1715
|
send("progress", { total: totalProcessed, processed: totalProcessed, phase: "done" });
|
|
@@ -1550,6 +1733,7 @@ class ViewerServer {
|
|
|
1550
1733
|
opts = JSON.parse(body);
|
|
1551
1734
|
}
|
|
1552
1735
|
catch { /* defaults */ }
|
|
1736
|
+
const concurrency = Math.max(1, Math.min(opts.concurrency ?? 1, 8));
|
|
1553
1737
|
res.writeHead(200, {
|
|
1554
1738
|
"Content-Type": "text/event-stream",
|
|
1555
1739
|
"Cache-Control": "no-cache",
|
|
@@ -1564,7 +1748,7 @@ class ViewerServer {
|
|
|
1564
1748
|
this.broadcastPPSSE(event, data);
|
|
1565
1749
|
};
|
|
1566
1750
|
this.ppRunning = true;
|
|
1567
|
-
this.runPostprocess(send, !!opts.enableTasks, !!opts.enableSkills).finally(() => {
|
|
1751
|
+
this.runPostprocess(send, !!opts.enableTasks, !!opts.enableSkills, concurrency).finally(() => {
|
|
1568
1752
|
this.ppRunning = false;
|
|
1569
1753
|
this.ppState.running = false;
|
|
1570
1754
|
this.ppState.done = true;
|
|
@@ -1623,126 +1807,148 @@ class ViewerServer {
|
|
|
1623
1807
|
catch { /* */ }
|
|
1624
1808
|
}
|
|
1625
1809
|
}
|
|
1626
|
-
async runPostprocess(send, enableTasks, enableSkills) {
|
|
1810
|
+
async runPostprocess(send, enableTasks, enableSkills, concurrency = 1) {
|
|
1627
1811
|
const ctx = this.ctx;
|
|
1628
|
-
const taskProcessor = new task_processor_1.TaskProcessor(this.store, ctx);
|
|
1629
|
-
let skillEvolver = null;
|
|
1630
|
-
if (enableSkills) {
|
|
1631
|
-
const recallEngine = new engine_1.RecallEngine(this.store, this.embedder, ctx);
|
|
1632
|
-
skillEvolver = new evolver_1.SkillEvolver(this.store, recallEngine, ctx);
|
|
1633
|
-
taskProcessor.onTaskCompleted(async (task) => {
|
|
1634
|
-
try {
|
|
1635
|
-
await skillEvolver.onTaskCompleted(task);
|
|
1636
|
-
this.ppState.skillsCreated++;
|
|
1637
|
-
send("skill", { taskId: task.id, title: task.title });
|
|
1638
|
-
}
|
|
1639
|
-
catch (err) {
|
|
1640
|
-
this.log.warn(`Postprocess skill evolution error: ${err}`);
|
|
1641
|
-
}
|
|
1642
|
-
});
|
|
1643
|
-
}
|
|
1644
1812
|
const importSessions = this.store.getDistinctSessionKeys()
|
|
1645
1813
|
.filter((sk) => sk.startsWith("openclaw-import-") || sk.startsWith("openclaw-session-"));
|
|
1646
1814
|
const pendingItems = [];
|
|
1647
1815
|
let skippedCount = 0;
|
|
1816
|
+
const ownerMap = this.store.getSessionOwnerMap(importSessions);
|
|
1648
1817
|
for (const sk of importSessions) {
|
|
1649
1818
|
const hasTask = this.store.hasTaskForSession(sk);
|
|
1650
1819
|
const hasSkill = this.store.hasSkillForSessionTask(sk);
|
|
1820
|
+
const owner = ownerMap.get(sk) ?? "agent:main";
|
|
1651
1821
|
if (enableTasks && !hasTask) {
|
|
1652
|
-
pendingItems.push({ sessionKey: sk, action: "full" });
|
|
1822
|
+
pendingItems.push({ sessionKey: sk, action: "full", owner });
|
|
1653
1823
|
}
|
|
1654
1824
|
else if (enableSkills && hasTask && !hasSkill) {
|
|
1655
|
-
pendingItems.push({ sessionKey: sk, action: "skill-only" });
|
|
1825
|
+
pendingItems.push({ sessionKey: sk, action: "skill-only", owner });
|
|
1656
1826
|
}
|
|
1657
1827
|
else {
|
|
1658
1828
|
skippedCount++;
|
|
1659
1829
|
}
|
|
1660
1830
|
}
|
|
1831
|
+
// Group pending items by agent (owner)
|
|
1832
|
+
const agentGroups = new Map();
|
|
1833
|
+
for (const item of pendingItems) {
|
|
1834
|
+
const group = agentGroups.get(item.owner) ?? [];
|
|
1835
|
+
group.push(item);
|
|
1836
|
+
agentGroups.set(item.owner, group);
|
|
1837
|
+
}
|
|
1661
1838
|
this.ppState.total = pendingItems.length;
|
|
1662
1839
|
send("info", {
|
|
1663
1840
|
totalSessions: importSessions.length,
|
|
1664
1841
|
alreadyProcessed: skippedCount,
|
|
1665
1842
|
pending: pendingItems.length,
|
|
1843
|
+
agents: Array.from(agentGroups.keys()),
|
|
1844
|
+
concurrency,
|
|
1666
1845
|
});
|
|
1667
1846
|
send("progress", { processed: 0, total: pendingItems.length });
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
await taskProcessor.onChunksIngested(sessionKey, Date.now());
|
|
1683
|
-
const activeTask = this.store.getActiveTask(sessionKey);
|
|
1684
|
-
if (activeTask) {
|
|
1685
|
-
await taskProcessor.finalizeTask(activeTask);
|
|
1686
|
-
const finalized = this.store.getTask(activeTask.id);
|
|
1687
|
-
this.ppState.tasksCreated++;
|
|
1688
|
-
send("item", {
|
|
1689
|
-
index: i + 1,
|
|
1690
|
-
total: pendingItems.length,
|
|
1691
|
-
session: sessionKey,
|
|
1692
|
-
step: "done",
|
|
1693
|
-
taskTitle: finalized?.title || "",
|
|
1694
|
-
taskStatus: finalized?.status || "",
|
|
1695
|
-
});
|
|
1847
|
+
let globalIdx = 0;
|
|
1848
|
+
const incIdx = () => ++globalIdx;
|
|
1849
|
+
// Process one agent's sessions sequentially
|
|
1850
|
+
const processAgent = async (agentOwner, items) => {
|
|
1851
|
+
const taskProcessor = new task_processor_1.TaskProcessor(this.store, ctx);
|
|
1852
|
+
let skillEvolver = null;
|
|
1853
|
+
if (enableSkills) {
|
|
1854
|
+
const recallEngine = new engine_1.RecallEngine(this.store, this.embedder, ctx);
|
|
1855
|
+
skillEvolver = new evolver_1.SkillEvolver(this.store, recallEngine, ctx);
|
|
1856
|
+
taskProcessor.onTaskCompleted(async (task) => {
|
|
1857
|
+
try {
|
|
1858
|
+
await skillEvolver.onTaskCompleted(task);
|
|
1859
|
+
this.ppState.skillsCreated++;
|
|
1860
|
+
send("skill", { taskId: task.id, title: task.title, agent: agentOwner });
|
|
1696
1861
|
}
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
index: i + 1,
|
|
1700
|
-
total: pendingItems.length,
|
|
1701
|
-
session: sessionKey,
|
|
1702
|
-
step: "done",
|
|
1703
|
-
taskTitle: "(no chunks)",
|
|
1704
|
-
});
|
|
1862
|
+
catch (err) {
|
|
1863
|
+
this.log.warn(`Postprocess skill evolution error (${agentOwner}): ${err}`);
|
|
1705
1864
|
}
|
|
1706
|
-
}
|
|
1707
|
-
|
|
1708
|
-
|
|
1709
|
-
|
|
1710
|
-
|
|
1711
|
-
|
|
1712
|
-
|
|
1713
|
-
|
|
1714
|
-
|
|
1715
|
-
|
|
1716
|
-
|
|
1717
|
-
|
|
1865
|
+
});
|
|
1866
|
+
}
|
|
1867
|
+
for (const { sessionKey, action } of items) {
|
|
1868
|
+
if (this.ppAbort)
|
|
1869
|
+
break;
|
|
1870
|
+
const idx = incIdx();
|
|
1871
|
+
this.ppState.processed = globalIdx;
|
|
1872
|
+
send("item", {
|
|
1873
|
+
index: idx,
|
|
1874
|
+
total: pendingItems.length,
|
|
1875
|
+
session: sessionKey,
|
|
1876
|
+
agent: agentOwner,
|
|
1877
|
+
step: "processing",
|
|
1878
|
+
action,
|
|
1879
|
+
});
|
|
1880
|
+
try {
|
|
1881
|
+
if (action === "full") {
|
|
1882
|
+
await taskProcessor.onChunksIngested(sessionKey, Date.now());
|
|
1883
|
+
const activeTask = this.store.getActiveTask(sessionKey);
|
|
1884
|
+
if (activeTask) {
|
|
1885
|
+
await taskProcessor.finalizeTask(activeTask);
|
|
1886
|
+
const finalized = this.store.getTask(activeTask.id);
|
|
1887
|
+
this.ppState.tasksCreated++;
|
|
1888
|
+
send("item", {
|
|
1889
|
+
index: idx, total: pendingItems.length, session: sessionKey, agent: agentOwner,
|
|
1890
|
+
step: "done", taskTitle: finalized?.title || "", taskStatus: finalized?.status || "",
|
|
1891
|
+
});
|
|
1892
|
+
}
|
|
1893
|
+
else {
|
|
1894
|
+
send("item", {
|
|
1895
|
+
index: idx, total: pendingItems.length, session: sessionKey, agent: agentOwner,
|
|
1896
|
+
step: "done", taskTitle: "(no chunks)",
|
|
1897
|
+
});
|
|
1718
1898
|
}
|
|
1719
|
-
|
|
1720
|
-
|
|
1899
|
+
}
|
|
1900
|
+
else if (action === "skill-only" && skillEvolver) {
|
|
1901
|
+
const completedTasks = this.store.getCompletedTasksForSession(sessionKey);
|
|
1902
|
+
let skillGenerated = false;
|
|
1903
|
+
for (const task of completedTasks) {
|
|
1904
|
+
if (this.ppAbort)
|
|
1905
|
+
break;
|
|
1906
|
+
try {
|
|
1907
|
+
await skillEvolver.onTaskCompleted(task);
|
|
1908
|
+
this.ppState.skillsCreated++;
|
|
1909
|
+
skillGenerated = true;
|
|
1910
|
+
send("skill", { taskId: task.id, title: task.title, agent: agentOwner });
|
|
1911
|
+
}
|
|
1912
|
+
catch (err) {
|
|
1913
|
+
this.log.warn(`Skill evolution error (${agentOwner}) task=${task.id}: ${err}`);
|
|
1914
|
+
}
|
|
1721
1915
|
}
|
|
1916
|
+
send("item", {
|
|
1917
|
+
index: idx, total: pendingItems.length, session: sessionKey, agent: agentOwner,
|
|
1918
|
+
step: "done", taskTitle: completedTasks[0]?.title || sessionKey, action: "skill-only", skillGenerated,
|
|
1919
|
+
});
|
|
1722
1920
|
}
|
|
1921
|
+
}
|
|
1922
|
+
catch (err) {
|
|
1923
|
+
this.ppState.errors++;
|
|
1924
|
+
this.log.warn(`Postprocess error (${agentOwner}) ${sessionKey}: ${err}`);
|
|
1723
1925
|
send("item", {
|
|
1724
|
-
index:
|
|
1725
|
-
|
|
1726
|
-
session: sessionKey,
|
|
1727
|
-
step: "done",
|
|
1728
|
-
taskTitle: completedTasks[0]?.title || sessionKey,
|
|
1729
|
-
action: "skill-only",
|
|
1730
|
-
skillGenerated,
|
|
1926
|
+
index: idx, total: pendingItems.length, session: sessionKey, agent: agentOwner,
|
|
1927
|
+
step: "error", error: String(err).slice(0, 200),
|
|
1731
1928
|
});
|
|
1732
1929
|
}
|
|
1930
|
+
send("progress", { processed: globalIdx, total: pendingItems.length });
|
|
1733
1931
|
}
|
|
1734
|
-
|
|
1735
|
-
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
|
|
1739
|
-
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
1932
|
+
};
|
|
1933
|
+
// Execute agents with concurrency control
|
|
1934
|
+
const agentEntries = Array.from(agentGroups.entries());
|
|
1935
|
+
if (concurrency <= 1 || agentEntries.length <= 1) {
|
|
1936
|
+
for (const [agentOwner, items] of agentEntries) {
|
|
1937
|
+
if (this.ppAbort)
|
|
1938
|
+
break;
|
|
1939
|
+
await processAgent(agentOwner, items);
|
|
1940
|
+
}
|
|
1941
|
+
}
|
|
1942
|
+
else {
|
|
1943
|
+
let cursor = 0;
|
|
1944
|
+
while (cursor < agentEntries.length && !this.ppAbort) {
|
|
1945
|
+
const batch = [];
|
|
1946
|
+
while (batch.length < concurrency && cursor < agentEntries.length) {
|
|
1947
|
+
const [agentOwner, items] = agentEntries[cursor++];
|
|
1948
|
+
batch.push(processAgent(agentOwner, items));
|
|
1949
|
+
}
|
|
1950
|
+
await Promise.all(batch);
|
|
1744
1951
|
}
|
|
1745
|
-
send("progress", { processed: i + 1, total: pendingItems.length });
|
|
1746
1952
|
}
|
|
1747
1953
|
}
|
|
1748
1954
|
readBody(req, cb) {
|