@memtensor/memos-local-openclaw-plugin 0.1.10 → 0.2.0

package/dist/viewer/server.js

@@ -9,8 +9,15 @@ const node_crypto_1 = __importDefault(require("node:crypto"));
 const node_child_process_1 = require("node:child_process");
 const node_fs_1 = __importDefault(require("node:fs"));
 const node_path_1 = __importDefault(require("node:path"));
+const node_readline_1 = __importDefault(require("node:readline"));
+const providers_1 = require("../ingest/providers");
+const dedup_1 = require("../ingest/dedup");
 const vector_1 = require("../storage/vector");
+const task_processor_1 = require("../ingest/task-processor");
+const engine_1 = require("../recall/engine");
+const evolver_1 = require("../skill/evolver");
 const html_1 = require("./html");
+const uuid_1 = require("uuid");
 class ViewerServer {
     server = null;
     store;
@@ -20,14 +27,24 @@ class ViewerServer {
     dataDir;
     authFile;
     auth;
+    ctx;
     static SESSION_TTL = 24 * 60 * 60 * 1000;
     resetToken;
+    migrationRunning = false;
+    migrationAbort = false;
+    migrationState = { phase: "", stored: 0, skipped: 0, merged: 0, errors: 0, processed: 0, total: 0, lastItem: null, done: false, stopped: false };
+    migrationSSEClients = [];
+    ppRunning = false;
+    ppAbort = false;
+    ppState = { running: false, done: false, stopped: false, processed: 0, total: 0, tasksCreated: 0, skillsCreated: 0, errors: 0 };
+    ppSSEClients = [];
     constructor(opts) {
         this.store = opts.store;
         this.embedder = opts.embedder;
         this.port = opts.port;
         this.log = opts.log;
         this.dataDir = opts.dataDir;
+        this.ctx = opts.ctx;
         this.authFile = node_path_1.default.join(opts.dataDir, "viewer-auth.json");
         this.auth = { passwordHash: null, sessions: new Map() };
         this.resetToken = node_crypto_1.default.randomBytes(16).toString("hex");
@@ -182,6 +199,24 @@ class ViewerServer {
                 this.handleSaveConfig(req, res);
             else if (p === "/api/auth/logout" && req.method === "POST")
                 this.handleLogout(req, res);
+            else if (p === "/api/migrate/scan" && req.method === "GET")
+                this.handleMigrateScan(res);
+            else if (p === "/api/migrate/start" && req.method === "POST")
+                this.handleMigrateStart(req, res);
+            else if (p === "/api/migrate/status" && req.method === "GET")
+                this.handleMigrateStatus(res);
+            else if (p === "/api/migrate/stream" && req.method === "GET")
+                this.handleMigrateStream(res);
+            else if (p === "/api/migrate/stop" && req.method === "POST")
+                this.handleMigrateStop(res);
+            else if (p === "/api/migrate/postprocess" && req.method === "POST")
+                this.handlePostprocess(req, res);
+            else if (p === "/api/migrate/postprocess/stream" && req.method === "GET")
+                this.handlePostprocessStream(res);
+            else if (p === "/api/migrate/postprocess/stop" && req.method === "POST")
+                this.handlePostprocessStop(res);
+            else if (p === "/api/migrate/postprocess/status" && req.method === "GET")
+                this.handlePostprocessStatus(res);
             else {
                 res.writeHead(404, { "Content-Type": "application/json" });
                 res.end(JSON.stringify({ error: "not found" }));
@@ -774,6 +809,8 @@ class ViewerServer {
                     config.skillEvolution = newCfg.skillEvolution;
                 if (newCfg.viewerPort)
                     config.viewerPort = newCfg.viewerPort;
+                if (newCfg.telemetry !== undefined)
+                    config.telemetry = newCfg.telemetry;
                 node_fs_1.default.mkdirSync(node_path_1.default.dirname(cfgPath), { recursive: true });
                 node_fs_1.default.writeFileSync(cfgPath, JSON.stringify(raw, null, 2), "utf-8");
                 this.log.info("Plugin config updated via Viewer");
@@ -799,6 +836,776 @@ class ViewerServer {
         const tools = this.store.getApiLogToolNames();
         this.jsonResponse(res, { tools });
     }
+    // ─── Migration: scan OpenClaw built-in memory ───
+    getOpenClawHome() {
+        const home = process.env.HOME || process.env.USERPROFILE || "";
+        return node_path_1.default.join(home, ".openclaw");
+    }
+    handleMigrateScan(res) {
+        try {
+            const ocHome = this.getOpenClawHome();
+            const memoryDir = node_path_1.default.join(ocHome, "memory");
+            const sessionsDir = node_path_1.default.join(ocHome, "agents", "main", "sessions");
+            const sqliteFiles = [];
+            if (node_fs_1.default.existsSync(memoryDir)) {
+                for (const f of node_fs_1.default.readdirSync(memoryDir)) {
+                    if (f.endsWith(".sqlite")) {
+                        try {
+                            const Database = require("better-sqlite3");
+                            const db = new Database(node_path_1.default.join(memoryDir, f), { readonly: true });
+                            const row = db.prepare("SELECT COUNT(*) as cnt FROM chunks").get();
+                            sqliteFiles.push({ file: f, chunks: row.cnt });
+                            db.close();
+                        }
+                        catch { /* skip unreadable */ }
+                    }
+                }
+            }
+            let sessionCount = 0;
+            let messageCount = 0;
+            if (node_fs_1.default.existsSync(sessionsDir)) {
+                const jsonlFiles = node_fs_1.default.readdirSync(sessionsDir).filter(f => f.includes(".jsonl"));
+                sessionCount = jsonlFiles.length;
+                for (const f of jsonlFiles) {
+                    try {
+                        const content = node_fs_1.default.readFileSync(node_path_1.default.join(sessionsDir, f), "utf-8");
+                        const lines = content.split("\n").filter(l => l.trim());
+                        for (const line of lines) {
+                            try {
+                                const obj = JSON.parse(line);
+                                if (obj.type === "message") {
+                                    const role = obj.message?.role ?? obj.role;
+                                    if (role === "user" || role === "assistant")
+                                        messageCount++;
+                                }
+                            }
+                            catch { /* skip bad lines */ }
+                        }
+                    }
+                    catch { /* skip unreadable */ }
+                }
+            }
+            const cfgPath = this.getOpenClawConfigPath();
+            let hasEmbedding = false;
+            let hasSummarizer = false;
+            if (node_fs_1.default.existsSync(cfgPath)) {
+                try {
+                    const raw = JSON.parse(node_fs_1.default.readFileSync(cfgPath, "utf-8"));
+                    const pluginCfg = raw?.plugins?.entries?.["memos-local"]?.config ??
+                        raw?.plugins?.entries?.["memos-local-openclaw-plugin"]?.config ?? {};
+                    const emb = pluginCfg.embedding;
+                    hasEmbedding = !!(emb && emb.provider);
+                    const sum = pluginCfg.summarizer;
+                    hasSummarizer = !!(sum && sum.provider);
+                }
+                catch { /* ignore */ }
+            }
+            const importedSessions = this.store.getDistinctSessionKeys()
+                .filter((sk) => sk.startsWith("openclaw-import-") || sk.startsWith("openclaw-session-"));
+            this.jsonResponse(res, {
+                sqliteFiles,
+                sessions: { count: sessionCount, messages: messageCount },
+                totalItems: sqliteFiles.reduce((s, f) => s + f.chunks, 0) + messageCount,
+                configReady: hasEmbedding && hasSummarizer,
+                hasEmbedding,
+                hasSummarizer,
+                hasImportedData: importedSessions.length > 0,
+                importedSessionCount: importedSessions.length,
+            });
+        }
+        catch (e) {
+            this.log.warn(`migrate/scan error: ${e}`);
+            res.writeHead(500, { "Content-Type": "application/json" });
+            res.end(JSON.stringify({ error: String(e) }));
+        }
+    }
+    // ─── Migration: start import with SSE progress ───
+    broadcastSSE(event, data) {
+        const msg = `event: ${event}\ndata: ${JSON.stringify(data)}\n\n`;
+        this.migrationSSEClients = this.migrationSSEClients.filter(c => {
+            try {
+                c.write(msg);
+                return true;
+            }
+            catch {
+                return false;
+            }
+        });
+    }
+    handleMigrateStatus(res) {
+        this.jsonResponse(res, {
+            running: this.migrationRunning,
+            ...this.migrationState,
+        });
+    }
+    handleMigrateStop(res) {
+        if (!this.migrationRunning) {
+            this.jsonResponse(res, { ok: false, error: "not_running" });
+            return;
+        }
+        this.migrationAbort = true;
+        this.jsonResponse(res, { ok: true });
+    }
+    handleMigrateStream(res) {
+        res.writeHead(200, {
+            "Content-Type": "text/event-stream",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "X-Accel-Buffering": "no",
+        });
+        if (this.migrationRunning) {
+            res.write(`event: state\ndata: ${JSON.stringify(this.migrationState)}\n\n`);
+        }
+        else if (this.migrationState.done) {
+            res.write(`event: state\ndata: ${JSON.stringify(this.migrationState)}\n\n`);
+            res.write(`event: done\ndata: ${JSON.stringify({ ok: true })}\n\n`);
+            res.end();
+            return;
+        }
+        this.migrationSSEClients.push(res);
+        res.on("close", () => {
+            this.migrationSSEClients = this.migrationSSEClients.filter(c => c !== res);
+        });
+    }
+    handleMigrateStart(req, res) {
+        if (this.migrationRunning) {
+            res.writeHead(200, {
+                "Content-Type": "text/event-stream",
+                "Cache-Control": "no-cache",
+                "Connection": "keep-alive",
+                "X-Accel-Buffering": "no",
+            });
+            res.write(`event: state\ndata: ${JSON.stringify(this.migrationState)}\n\n`);
+            this.migrationSSEClients.push(res);
+            res.on("close", () => {
+                this.migrationSSEClients = this.migrationSSEClients.filter(c => c !== res);
+            });
+            return;
+        }
+        this.readBody(req, (body) => {
+            let opts = {};
+            try {
+                opts = JSON.parse(body);
+            }
+            catch { /* defaults */ }
+            res.writeHead(200, {
+                "Content-Type": "text/event-stream",
+                "Cache-Control": "no-cache",
+                "Connection": "keep-alive",
+                "X-Accel-Buffering": "no",
+            });
+            this.migrationSSEClients.push(res);
+            res.on("close", () => {
+                this.migrationSSEClients = this.migrationSSEClients.filter(c => c !== res);
+            });
+            this.migrationAbort = false;
+            this.migrationState = { phase: "", stored: 0, skipped: 0, merged: 0, errors: 0, processed: 0, total: 0, lastItem: null, done: false, stopped: false };
+            const send = (event, data) => {
+                if (event === "item") {
+                    const d = data;
+                    if (d.status === "stored")
+                        this.migrationState.stored++;
+                    else if (d.status === "skipped" || d.status === "duplicate")
+                        this.migrationState.skipped++;
+                    else if (d.status === "merged")
+                        this.migrationState.merged++;
+                    else if (d.status === "error")
+                        this.migrationState.errors++;
+                    this.migrationState.processed = d.index ?? this.migrationState.processed + 1;
+                    this.migrationState.total = d.total ?? this.migrationState.total;
+                    this.migrationState.lastItem = d;
+                }
+                else if (event === "phase") {
+                    this.migrationState.phase = data.phase;
+                }
+                else if (event === "progress") {
+                    this.migrationState.total = data.total ?? this.migrationState.total;
+                }
+                this.broadcastSSE(event, data);
+            };
+            this.migrationRunning = true;
+            this.runMigration(send, opts.sources).finally(() => {
+                this.migrationRunning = false;
+                this.migrationState.done = true;
+                if (this.migrationAbort) {
+                    this.migrationState.stopped = true;
+                    this.broadcastSSE("stopped", { ok: true, ...this.migrationState });
+                }
+                else {
+                    this.broadcastSSE("done", { ok: true });
+                }
+                for (const c of this.migrationSSEClients) {
+                    try {
+                        c.end();
+                    }
+                    catch { /* ignore */ }
+                }
+                this.migrationSSEClients = [];
+                this.migrationAbort = false;
+            });
+        });
+    }
+    async runMigration(send, sources) {
+        const ocHome = this.getOpenClawHome();
+        const importSqlite = !sources || sources.includes("sqlite");
+        const importSessions = !sources || sources.includes("sessions");
+        let totalProcessed = 0;
+        let totalStored = 0;
+        let totalSkipped = 0;
+        let totalErrors = 0;
+        const cfgPath = this.getOpenClawConfigPath();
+        let summarizerCfg;
+        try {
+            const raw = JSON.parse(node_fs_1.default.readFileSync(cfgPath, "utf-8"));
+            const pluginCfg = raw?.plugins?.entries?.["memos-local"]?.config ??
+                raw?.plugins?.entries?.["memos-local-openclaw-plugin"]?.config ?? {};
+            summarizerCfg = pluginCfg.summarizer;
+        }
+        catch { /* no config */ }
+        const summarizer = new providers_1.Summarizer(summarizerCfg, this.log);
+        // Phase 1: Import SQLite memory chunks
+        if (importSqlite) {
+            const memoryDir = node_path_1.default.join(ocHome, "memory");
+            if (node_fs_1.default.existsSync(memoryDir)) {
+                const files = node_fs_1.default.readdirSync(memoryDir).filter(f => f.endsWith(".sqlite"));
+                for (const file of files) {
+                    if (this.migrationAbort)
+                        break;
+                    send("phase", { phase: "sqlite", file });
+                    try {
+                        const Database = require("better-sqlite3");
+                        const db = new Database(node_path_1.default.join(memoryDir, file), { readonly: true });
+                        const rows = db.prepare("SELECT id, path, text, updated_at FROM chunks ORDER BY updated_at ASC").all();
+                        db.close();
+                        const agentId = file.replace(".sqlite", "");
+                        send("progress", { total: rows.length, processed: 0, phase: "sqlite", file });
+                        for (let i = 0; i < rows.length; i++) {
+                            if (this.migrationAbort)
+                                break;
+                            const row = rows[i];
+                            totalProcessed++;
+                            const contentHash = node_crypto_1.default.createHash("sha256").update(row.text).digest("hex");
+                            if (this.store.chunkExistsByContent(`openclaw-import-${agentId}`, "assistant", row.text)) {
+                                totalSkipped++;
+                                send("item", {
+                                    index: i + 1,
+                                    total: rows.length,
+                                    status: "skipped",
+                                    preview: row.text.slice(0, 120),
+                                    source: file,
+                                    reason: "duplicate",
+                                });
+                                continue;
+                            }
+                            try {
+                                const summary = await summarizer.summarize(row.text);
+                                let embedding = null;
+                                try {
+                                    [embedding] = await this.embedder.embed([summary]);
+                                }
+                                catch (err) {
+                                    this.log.warn(`Migration embed failed: ${err}`);
+                                }
+                                let dedupStatus = "active";
+                                let dedupTarget = null;
+                                let dedupReason = null;
+                                if (embedding) {
+                                    const topSimilar = (0, dedup_1.findTopSimilar)(this.store, embedding, 0.85, 3, this.log);
+                                    if (topSimilar.length > 0) {
+                                        const candidates = topSimilar.map((s, idx) => {
+                                            const chunk = this.store.getChunk(s.chunkId);
+                                            return { index: idx + 1, summary: chunk?.summary ?? "", chunkId: s.chunkId };
+                                        }).filter(c => c.summary);
+                                        if (candidates.length > 0) {
+                                            const dedupResult = await summarizer.judgeDedup(summary, candidates);
+                                            if (dedupResult?.action === "DUPLICATE" && dedupResult.targetIndex) {
+                                                const targetId = candidates[dedupResult.targetIndex - 1]?.chunkId;
+                                                if (targetId) {
+                                                    dedupStatus = "duplicate";
+                                                    dedupTarget = targetId;
+                                                    dedupReason = dedupResult.reason;
+                                                }
+                                            }
+                                            else if (dedupResult?.action === "UPDATE" && dedupResult.targetIndex && dedupResult.mergedSummary) {
+                                                const targetId = candidates[dedupResult.targetIndex - 1]?.chunkId;
+                                                if (targetId) {
+                                                    this.store.updateChunkSummaryAndContent(targetId, dedupResult.mergedSummary, row.text);
+                                                    try {
+                                                        const [newEmb] = await this.embedder.embed([dedupResult.mergedSummary]);
+                                                        if (newEmb)
+                                                            this.store.upsertEmbedding(targetId, newEmb);
+                                                    }
+                                                    catch { /* best-effort */ }
+                                                    dedupStatus = "merged";
+                                                    dedupTarget = targetId;
+                                                    dedupReason = dedupResult.reason;
+                                                }
+                                            }
+                                        }
+                                    }
+                                }
+                                const chunkId = (0, uuid_1.v4)();
+                                const chunk = {
+                                    id: chunkId,
+                                    sessionKey: `openclaw-import-${agentId}`,
+                                    turnId: `import-${row.id}`,
+                                    seq: 0,
+                                    role: "assistant",
+                                    content: row.text,
+                                    kind: "paragraph",
+                                    summary,
+                                    embedding: null,
+                                    taskId: null,
+                                    skillId: null,
+                                    dedupStatus,
+                                    dedupTarget,
+                                    dedupReason,
+                                    mergeCount: 0,
+                                    lastHitAt: null,
+                                    mergeHistory: "[]",
+                                    createdAt: row.updated_at * 1000,
+                                    updatedAt: row.updated_at * 1000,
+                                };
+                                this.store.insertChunk(chunk);
+                                if (embedding && dedupStatus === "active") {
+                                    this.store.upsertEmbedding(chunkId, embedding);
+                                }
+                                totalStored++;
+                                send("item", {
+                                    index: i + 1,
+                                    total: rows.length,
+                                    status: dedupStatus === "active" ? "stored" : dedupStatus,
+                                    preview: row.text.slice(0, 120),
+                                    summary: summary.slice(0, 80),
+                                    source: file,
+                                });
+                            }
+                            catch (err) {
+                                totalErrors++;
+                                send("item", {
+                                    index: i + 1,
+                                    total: rows.length,
+                                    status: "error",
+                                    preview: row.text.slice(0, 120),
+                                    source: file,
+                                    error: String(err).slice(0, 200),
+                                });
+                            }
+                        }
+                    }
+                    catch (err) {
+                        send("error", { file, error: String(err) });
+                        totalErrors++;
+                    }
+                }
+            }
+        }
+        // Phase 2: Import session JSONL files
+        if (importSessions) {
+            const sessionsDir = node_path_1.default.join(ocHome, "agents", "main", "sessions");
+            if (node_fs_1.default.existsSync(sessionsDir)) {
+                const jsonlFiles = node_fs_1.default.readdirSync(sessionsDir).filter(f => f.includes(".jsonl")).sort();
+                send("phase", { phase: "sessions", files: jsonlFiles.length });
+                let globalMsgIdx = 0;
+                let totalMsgs = 0;
+                for (const f of jsonlFiles) {
+                    try {
+                        const raw = node_fs_1.default.readFileSync(node_path_1.default.join(sessionsDir, f), "utf-8");
+                        for (const line of raw.split("\n")) {
+                            if (!line.trim())
+                                continue;
+                            try {
+                                const obj = JSON.parse(line);
+                                if (obj.type === "message") {
+                                    const role = obj.message?.role ?? obj.role;
+                                    if (role === "user" || role === "assistant")
+                                        totalMsgs++;
+                                }
+                            }
+                            catch { /* skip */ }
+                        }
+                    }
+                    catch { /* skip */ }
+                }
+                for (const file of jsonlFiles) {
+                    if (this.migrationAbort)
+                        break;
+                    const sessionId = file.replace(/\.jsonl.*$/, "");
+                    const filePath = node_path_1.default.join(sessionsDir, file);
+                    send("progress", { total: totalMsgs, processed: globalMsgIdx, phase: "sessions", file });
+                    try {
+                        const fileStream = node_fs_1.default.createReadStream(filePath, { encoding: "utf-8" });
+                        const rl = node_readline_1.default.createInterface({ input: fileStream, crlfDelay: Infinity });
+                        for await (const line of rl) {
+                            if (this.migrationAbort)
+                                break;
+                            if (!line.trim())
+                                continue;
+                            let obj;
+                            try {
+                                obj = JSON.parse(line);
+                            }
+                            catch {
+                                continue;
+                            }
+                            if (obj.type !== "message")
+                                continue;
+                            const msgRole = obj.message?.role ?? obj.role;
+                            if (msgRole !== "user" && msgRole !== "assistant")
+                                continue;
+                            const msgContent = obj.message?.content ?? obj.content;
+                            let content;
+                            if (typeof msgContent === "string") {
+                                content = msgContent;
+                            }
+                            else if (Array.isArray(msgContent)) {
+                                content = msgContent
+                                    .filter((p) => p.type === "text" && p.text)
+                                    .map((p) => p.text)
+                                    .join("\n");
+                            }
+                            else {
+                                content = JSON.stringify(msgContent);
+                            }
+                            if (!content || content.length < 10)
+                                continue;
+                            globalMsgIdx++;
+                            totalProcessed++;
+                            const sessionKey = `openclaw-session-${sessionId}`;
+                            if (this.store.chunkExistsByContent(sessionKey, msgRole, content)) {
+                                totalSkipped++;
+                                send("item", {
+                                    index: globalMsgIdx,
+                                    total: totalMsgs,
+                                    status: "skipped",
+                                    preview: content.slice(0, 120),
+                                    source: file,
+                                    role: msgRole,
+                                    reason: "duplicate",
+                                });
+                                continue;
+                            }
+                            try {
+                                const summary = await summarizer.summarize(content);
+                                let embedding = null;
+                                try {
+                                    [embedding] = await this.embedder.embed([summary]);
+                                }
+                                catch (err) {
+                                    this.log.warn(`Migration embed failed: ${err}`);
+                                }
+                                let dedupStatus = "active";
+                                let dedupTarget = null;
+                                let dedupReason = null;
+                                if (embedding) {
+                                    const topSimilar = (0, dedup_1.findTopSimilar)(this.store, embedding, 0.85, 3, this.log);
+                                    if (topSimilar.length > 0) {
+                                        const candidates = topSimilar.map((s, idx) => {
+                                            const chunk = this.store.getChunk(s.chunkId);
+                                            return { index: idx + 1, summary: chunk?.summary ?? "", chunkId: s.chunkId };
+                                        }).filter(c => c.summary);
+                                        if (candidates.length > 0) {
+                                            const dedupResult = await summarizer.judgeDedup(summary, candidates);
+                                            if (dedupResult?.action === "DUPLICATE" && dedupResult.targetIndex) {
+                                                const targetId = candidates[dedupResult.targetIndex - 1]?.chunkId;
+                                                if (targetId) {
+                                                    dedupStatus = "duplicate";
+                                                    dedupTarget = targetId;
+                                                    dedupReason = dedupResult.reason;
+                                                }
+                                            }
+                                            else if (dedupResult?.action === "UPDATE" && dedupResult.targetIndex && dedupResult.mergedSummary) {
+                                                const targetId = candidates[dedupResult.targetIndex - 1]?.chunkId;
+                                                if (targetId) {
+                                                    this.store.updateChunkSummaryAndContent(targetId, dedupResult.mergedSummary, content);
+                                                    try {
+                                                        const [newEmb] = await this.embedder.embed([dedupResult.mergedSummary]);
+                                                        if (newEmb)
+                                                            this.store.upsertEmbedding(targetId, newEmb);
+                                                    }
+                                                    catch { /* best-effort */ }
+                                                    dedupStatus = "merged";
+                                                    dedupTarget = targetId;
+                                                    dedupReason = dedupResult.reason;
+                                                }
+                                            }
+                                        }
+                                    }
+                                }
+                                const chunkId = (0, uuid_1.v4)();
+                                const msgTs = obj.message?.timestamp ?? obj.timestamp;
+                                const ts = msgTs ? new Date(msgTs).getTime() : Date.now();
+                                const chunk = {
+                                    id: chunkId,
+                                    sessionKey,
+                                    turnId: `import-${sessionId}-${globalMsgIdx}`,
+                                    seq: 0,
+                                    role: msgRole,
+                                    content,
+                                    kind: "paragraph",
+                                    summary,
+                                    embedding: null,
+                                    taskId: null,
+                                    skillId: null,
+                                    dedupStatus,
+                                    dedupTarget,
+                                    dedupReason,
+                                    mergeCount: 0,
+                                    lastHitAt: null,
+                                    mergeHistory: "[]",
+                                    createdAt: ts,
+                                    updatedAt: ts,
+                                };
+                                this.store.insertChunk(chunk);
+                                if (embedding && dedupStatus === "active") {
+                                    this.store.upsertEmbedding(chunkId, embedding);
+                                }
+                                totalStored++;
+                                send("item", {
+                                    index: globalMsgIdx,
+                                    total: totalMsgs,
+                                    status: dedupStatus === "active" ? "stored" : dedupStatus,
+                                    preview: content.slice(0, 120),
+                                    summary: summary.slice(0, 80),
+                                    source: file,
+                                    role: msgRole,
+                                });
+                            }
+                            catch (err) {
+                                totalErrors++;
+                                send("item", {
+                                    index: globalMsgIdx,
+                                    total: totalMsgs,
+                                    status: "error",
+                                    preview: content.slice(0, 120),
+                                    source: file,
+                                    error: String(err).slice(0, 200),
+                                });
+                            }
+                        }
+                    }
+                    catch (err) {
+                        send("error", { file, error: String(err) });
+                        totalErrors++;
+                    }
+                }
+            }
+        }
+        send("summary", { totalProcessed, totalStored, totalSkipped, totalErrors });
+    }
+    // ─── Post-processing: independent task/skill generation ───
+    handlePostprocess(req, res) {
+        if (this.ppRunning) {
+            res.writeHead(409, { "Content-Type": "application/json" });
+            res.end(JSON.stringify({ error: "postprocess already running" }));
+            return;
+        }
+        if (!this.ctx) {
+            res.writeHead(500, { "Content-Type": "application/json" });
+            res.end(JSON.stringify({ error: "plugin context not available — please restart the gateway" }));
+            return;
+        }
+        this.readBody(req, (body) => {
+            let opts = {};
+            try {
+                opts = JSON.parse(body);
+            }
+            catch { /* defaults */ }
+            res.writeHead(200, {
+                "Content-Type": "text/event-stream",
+                "Cache-Control": "no-cache",
+                "Connection": "keep-alive",
+                "X-Accel-Buffering": "no",
+            });
+            this.ppSSEClients.push(res);
+            res.on("close", () => { this.ppSSEClients = this.ppSSEClients.filter(c => c !== res); });
+            this.ppAbort = false;
+            this.ppState = { running: true, done: false, stopped: false, processed: 0, total: 0, tasksCreated: 0, skillsCreated: 0, errors: 0 };
+            const send = (event, data) => {
+                this.broadcastPPSSE(event, data);
+            };
+            this.ppRunning = true;
+            this.runPostprocess(send, !!opts.enableTasks, !!opts.enableSkills).finally(() => {
+                this.ppRunning = false;
+                this.ppState.running = false;
+                this.ppState.done = true;
+                if (this.ppAbort) {
+                    this.ppState.stopped = true;
+                    this.broadcastPPSSE("stopped", { ...this.ppState });
+                }
+                else {
+                    this.broadcastPPSSE("done", { ...this.ppState });
+                }
+                for (const c of this.ppSSEClients) {
+                    try {
+                        c.end();
+                    }
+                    catch { /* */ }
+                }
+                this.ppSSEClients = [];
+                this.ppAbort = false;
+            });
+        });
+    }
+    handlePostprocessStream(res) {
+        res.writeHead(200, {
+            "Content-Type": "text/event-stream",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "X-Accel-Buffering": "no",
+        });
+        if (this.ppRunning) {
+            res.write(`event: state\ndata: ${JSON.stringify(this.ppState)}\n\n`);
+            this.ppSSEClients.push(res);
+            res.on("close", () => { this.ppSSEClients = this.ppSSEClients.filter(c => c !== res); });
+        }
+        else if (this.ppState.done) {
+            const evt = this.ppState.stopped ? "stopped" : "done";
+            res.write(`event: ${evt}\ndata: ${JSON.stringify(this.ppState)}\n\n`);
+            res.end();
+        }
+        else {
+            res.end();
+        }
+    }
+    handlePostprocessStop(res) {
+        this.ppAbort = true;
+        this.jsonResponse(res, { ok: true });
+    }
+    handlePostprocessStatus(res) {
+        this.jsonResponse(res, this.ppState);
+    }
+    broadcastPPSSE(event, data) {
+        const payload = `event: ${event}\ndata: ${JSON.stringify(data)}\n\n`;
+        for (const c of this.ppSSEClients) {
+            try {
+                c.write(payload);
+            }
+            catch { /* */ }
+        }
+    }
+    async runPostprocess(send, enableTasks, enableSkills) {
+        const ctx = this.ctx;
+        const taskProcessor = new task_processor_1.TaskProcessor(this.store, ctx);
+        let skillEvolver = null;
+        if (enableSkills) {
+            const recallEngine = new engine_1.RecallEngine(this.store, this.embedder, ctx);
+            skillEvolver = new evolver_1.SkillEvolver(this.store, recallEngine, ctx);
+            taskProcessor.onTaskCompleted(async (task) => {
+                try {
+                    await skillEvolver.onTaskCompleted(task);
+                    this.ppState.skillsCreated++;
+                    send("skill", { taskId: task.id, title: task.title });
+                }
+                catch (err) {
+                    this.log.warn(`Postprocess skill evolution error: ${err}`);
+                }
+            });
+        }
+        const importSessions = this.store.getDistinctSessionKeys()
+            .filter((sk) => sk.startsWith("openclaw-import-") || sk.startsWith("openclaw-session-"));
+        const pendingItems = [];
+        let skippedCount = 0;
+        for (const sk of importSessions) {
+            const hasTask = this.store.hasTaskForSession(sk);
+            const hasSkill = this.store.hasSkillForSessionTask(sk);
+            if (enableTasks && !hasTask) {
+                pendingItems.push({ sessionKey: sk, action: "full" });
+            }
+            else if (enableSkills && hasTask && !hasSkill) {
+                pendingItems.push({ sessionKey: sk, action: "skill-only" });
+            }
+            else {
+                skippedCount++;
+            }
+        }
+        this.ppState.total = pendingItems.length;
+        send("info", {
+            totalSessions: importSessions.length,
+            alreadyProcessed: skippedCount,
+            pending: pendingItems.length,
+        });
+        send("progress", { processed: 0, total: pendingItems.length });
+        for (let i = 0; i < pendingItems.length; i++) {
+            if (this.ppAbort)
+                break;
+            const { sessionKey, action } = pendingItems[i];
+            this.ppState.processed = i + 1;
+            send("item", {
+                index: i + 1,
+                total: pendingItems.length,
+                session: sessionKey,
+                step: "processing",
+                action,
+            });
+            try {
+                if (action === "full") {
+                    await taskProcessor.onChunksIngested(sessionKey, Date.now());
+                    const activeTask = this.store.getActiveTask(sessionKey);
+                    if (activeTask) {
+                        await taskProcessor.finalizeTask(activeTask);
+                        const finalized = this.store.getTask(activeTask.id);
+                        this.ppState.tasksCreated++;
+                        send("item", {
+                            index: i + 1,
+                            total: pendingItems.length,
+                            session: sessionKey,
+                            step: "done",
+                            taskTitle: finalized?.title || "",
+                            taskStatus: finalized?.status || "",
+                        });
+                    }
+                    else {
+                        send("item", {
+                            index: i + 1,
+                            total: pendingItems.length,
+                            session: sessionKey,
+                            step: "done",
+                            taskTitle: "(no chunks)",
+                        });
+                    }
+                }
+                else if (action === "skill-only" && skillEvolver) {
+                    const completedTasks = this.store.getCompletedTasksForSession(sessionKey);
+                    let skillGenerated = false;
+                    for (const task of completedTasks) {
+                        if (this.ppAbort)
+                            break;
+                        try {
+                            await skillEvolver.onTaskCompleted(task);
+                            this.ppState.skillsCreated++;
+                            skillGenerated = true;
+                            send("skill", { taskId: task.id, title: task.title });
+                        }
+                        catch (err) {
+                            this.log.warn(`Skill evolution error for task=${task.id}: ${err}`);
+                        }
+                    }
+                    send("item", {
+                        index: i + 1,
+                        total: pendingItems.length,
+                        session: sessionKey,
+                        step: "done",
+                        taskTitle: completedTasks[0]?.title || sessionKey,
+                        action: "skill-only",
+                        skillGenerated,
+                    });
+                }
+            }
+            catch (err) {
+                this.ppState.errors++;
+                this.log.warn(`Postprocess error for ${sessionKey}: ${err}`);
+                send("item", {
+                    index: i + 1,
+                    total: pendingItems.length,
+                    session: sessionKey,
+                    step: "error",
+                    error: String(err).slice(0, 200),
+                });
+            }
+            send("progress", { processed: i + 1, total: pendingItems.length });
+        }
+    }
     readBody(req, cb) {
         let body = "";
         req.on("data", (chunk) => { body += chunk.toString(); });