@memtensor/memos-local-openclaw-plugin 0.1.9 → 0.2.0

package/src/viewer/server.ts

@@ -3,11 +3,18 @@ import crypto from "node:crypto";
 import { execSync } from "node:child_process";
 import fs from "node:fs";
 import path from "node:path";
+import readline from "node:readline";
 import type { SqliteStore } from "../storage/sqlite";
 import type { Embedder } from "../embedding";
+import { Summarizer } from "../ingest/providers";
+import { findTopSimilar } from "../ingest/dedup";
 import { vectorSearch } from "../storage/vector";
-import type { Logger } from "../types";
+import { TaskProcessor } from "../ingest/task-processor";
+import { RecallEngine } from "../recall/engine";
+import { SkillEvolver } from "../skill/evolver";
+import type { Logger, Chunk, PluginContext } from "../types";
 import { viewerHTML } from "./html";
+import { v4 as uuid } from "uuid";
 
 export interface ViewerServerOptions {
   store: SqliteStore;
@@ -15,6 +22,7 @@ export interface ViewerServerOptions {
   port: number;
   log: Logger;
   dataDir: string;
+  ctx?: PluginContext;
 }
 
 interface AuthState {
@@ -31,9 +39,31 @@ export class ViewerServer {
   private readonly dataDir: string;
   private readonly authFile: string;
   private readonly auth: AuthState;
+  private readonly ctx?: PluginContext;
 
   private static readonly SESSION_TTL = 24 * 60 * 60 * 1000;
   private resetToken: string;
+  private migrationRunning = false;
+  private migrationAbort = false;
+  private migrationState: {
+    phase: string;
+    stored: number;
+    skipped: number;
+    merged: number;
+    errors: number;
+    processed: number;
+    total: number;
+    lastItem: any;
+    done: boolean;
+    stopped: boolean;
+  } = { phase: "", stored: 0, skipped: 0, merged: 0, errors: 0, processed: 0, total: 0, lastItem: null, done: false, stopped: false };
+  private migrationSSEClients: http.ServerResponse[] = [];
+
+  private ppRunning = false;
+  private ppAbort = false;
+  private ppState: { running: boolean; done: boolean; stopped: boolean; processed: number; total: number; tasksCreated: number; skillsCreated: number; errors: number } =
+    { running: false, done: false, stopped: false, processed: 0, total: 0, tasksCreated: 0, skillsCreated: 0, errors: 0 };
+  private ppSSEClients: http.ServerResponse[] = [];
 
   constructor(opts: ViewerServerOptions) {
     this.store = opts.store;
@@ -41,6 +71,7 @@ export class ViewerServer {
     this.port = opts.port;
     this.log = opts.log;
     this.dataDir = opts.dataDir;
+    this.ctx = opts.ctx;
     this.authFile = path.join(opts.dataDir, "viewer-auth.json");
     this.auth = { passwordHash: null, sessions: new Map() };
     this.resetToken = crypto.randomBytes(16).toString("hex");
@@ -178,6 +209,15 @@ export class ViewerServer {
       else if (p === "/api/config" && req.method === "GET") this.serveConfig(res);
       else if (p === "/api/config" && req.method === "PUT") this.handleSaveConfig(req, res);
       else if (p === "/api/auth/logout" && req.method === "POST") this.handleLogout(req, res);
+      else if (p === "/api/migrate/scan" && req.method === "GET") this.handleMigrateScan(res);
+      else if (p === "/api/migrate/start" && req.method === "POST") this.handleMigrateStart(req, res);
+      else if (p === "/api/migrate/status" && req.method === "GET") this.handleMigrateStatus(res);
+      else if (p === "/api/migrate/stream" && req.method === "GET") this.handleMigrateStream(res);
+      else if (p === "/api/migrate/stop" && req.method === "POST") this.handleMigrateStop(res);
+      else if (p === "/api/migrate/postprocess" && req.method === "POST") this.handlePostprocess(req, res);
+      else if (p === "/api/migrate/postprocess/stream" && req.method === "GET") this.handlePostprocessStream(res);
+      else if (p === "/api/migrate/postprocess/stop" && req.method === "POST") this.handlePostprocessStop(res);
+      else if (p === "/api/migrate/postprocess/status" && req.method === "GET") this.handlePostprocessStatus(res);
       else {
         res.writeHead(404, { "Content-Type": "application/json" });
         res.end(JSON.stringify({ error: "not found" }));
@@ -766,6 +806,7 @@ export class ViewerServer {
         if (newCfg.summarizer) config.summarizer = newCfg.summarizer;
         if (newCfg.skillEvolution) config.skillEvolution = newCfg.skillEvolution;
         if (newCfg.viewerPort) config.viewerPort = newCfg.viewerPort;
+        if (newCfg.telemetry !== undefined) config.telemetry = newCfg.telemetry;
 
         fs.mkdirSync(path.dirname(cfgPath), { recursive: true });
         fs.writeFileSync(cfgPath, JSON.stringify(raw, null, 2), "utf-8");
@@ -794,6 +835,782 @@ export class ViewerServer {
     this.jsonResponse(res, { tools });
   }
 
+  // ─── Migration: scan OpenClaw built-in memory ───
+
+  private getOpenClawHome(): string {
+    const home = process.env.HOME || process.env.USERPROFILE || "";
+    return path.join(home, ".openclaw");
+  }
+
+  private handleMigrateScan(res: http.ServerResponse): void {
+    try {
+      const ocHome = this.getOpenClawHome();
+      const memoryDir = path.join(ocHome, "memory");
+      const sessionsDir = path.join(ocHome, "agents", "main", "sessions");
+
+      const sqliteFiles: Array<{ file: string; chunks: number }> = [];
+      if (fs.existsSync(memoryDir)) {
+        for (const f of fs.readdirSync(memoryDir)) {
+          if (f.endsWith(".sqlite")) {
+            try {
+              const Database = require("better-sqlite3");
+              const db = new Database(path.join(memoryDir, f), { readonly: true });
+              const row = db.prepare("SELECT COUNT(*) as cnt FROM chunks").get() as { cnt: number };
+              sqliteFiles.push({ file: f, chunks: row.cnt });
+              db.close();
+            } catch { /* skip unreadable */ }
+          }
+        }
+      }
+
+      let sessionCount = 0;
+      let messageCount = 0;
+      if (fs.existsSync(sessionsDir)) {
+        const jsonlFiles = fs.readdirSync(sessionsDir).filter(f => f.includes(".jsonl"));
+        sessionCount = jsonlFiles.length;
+        for (const f of jsonlFiles) {
+          try {
+            const content = fs.readFileSync(path.join(sessionsDir, f), "utf-8");
+            const lines = content.split("\n").filter(l => l.trim());
+            for (const line of lines) {
+              try {
+                const obj = JSON.parse(line);
+                if (obj.type === "message") {
+                  const role = obj.message?.role ?? obj.role;
+                  if (role === "user" || role === "assistant") messageCount++;
+                }
+              } catch { /* skip bad lines */ }
+            }
+          } catch { /* skip unreadable */ }
+        }
+      }
+
+      const cfgPath = this.getOpenClawConfigPath();
+      let hasEmbedding = false;
+      let hasSummarizer = false;
+      if (fs.existsSync(cfgPath)) {
+        try {
+          const raw = JSON.parse(fs.readFileSync(cfgPath, "utf-8"));
+          const pluginCfg = raw?.plugins?.entries?.["memos-local"]?.config ??
+                            raw?.plugins?.entries?.["memos-local-openclaw-plugin"]?.config ?? {};
+          const emb = pluginCfg.embedding;
+          hasEmbedding = !!(emb && emb.provider);
+          const sum = pluginCfg.summarizer;
+          hasSummarizer = !!(sum && sum.provider);
+        } catch { /* ignore */ }
+      }
+
+      const importedSessions = this.store.getDistinctSessionKeys()
+        .filter((sk: string) => sk.startsWith("openclaw-import-") || sk.startsWith("openclaw-session-"));
+
+      this.jsonResponse(res, {
+        sqliteFiles,
+        sessions: { count: sessionCount, messages: messageCount },
+        totalItems: sqliteFiles.reduce((s, f) => s + f.chunks, 0) + messageCount,
+        configReady: hasEmbedding && hasSummarizer,
+        hasEmbedding,
+        hasSummarizer,
+        hasImportedData: importedSessions.length > 0,
+        importedSessionCount: importedSessions.length,
+      });
+    } catch (e) {
+      this.log.warn(`migrate/scan error: ${e}`);
+      res.writeHead(500, { "Content-Type": "application/json" });
+      res.end(JSON.stringify({ error: String(e) }));
+    }
+  }
+
+  // ─── Migration: start import with SSE progress ───
+
+  private broadcastSSE(event: string, data: unknown): void {
+    const msg = `event: ${event}\ndata: ${JSON.stringify(data)}\n\n`;
+    this.migrationSSEClients = this.migrationSSEClients.filter(c => {
+      try { c.write(msg); return true; } catch { return false; }
+    });
+  }
+
+  private handleMigrateStatus(res: http.ServerResponse): void {
+    this.jsonResponse(res, {
+      running: this.migrationRunning,
+      ...this.migrationState,
+    });
+  }
+
+  private handleMigrateStop(res: http.ServerResponse): void {
+    if (!this.migrationRunning) {
+      this.jsonResponse(res, { ok: false, error: "not_running" });
+      return;
+    }
+    this.migrationAbort = true;
+    this.jsonResponse(res, { ok: true });
+  }
+
+  private handleMigrateStream(res: http.ServerResponse): void {
+    res.writeHead(200, {
+      "Content-Type": "text/event-stream",
+      "Cache-Control": "no-cache",
+      "Connection": "keep-alive",
+      "X-Accel-Buffering": "no",
+    });
+
+    if (this.migrationRunning) {
+      res.write(`event: state\ndata: ${JSON.stringify(this.migrationState)}\n\n`);
+    } else if (this.migrationState.done) {
+      res.write(`event: state\ndata: ${JSON.stringify(this.migrationState)}\n\n`);
+      res.write(`event: done\ndata: ${JSON.stringify({ ok: true })}\n\n`);
+      res.end();
+      return;
+    }
+
+    this.migrationSSEClients.push(res);
+    res.on("close", () => {
+      this.migrationSSEClients = this.migrationSSEClients.filter(c => c !== res);
+    });
+  }
+
+  private handleMigrateStart(req: http.IncomingMessage, res: http.ServerResponse): void {
+    if (this.migrationRunning) {
+      res.writeHead(200, {
+        "Content-Type": "text/event-stream",
+        "Cache-Control": "no-cache",
+        "Connection": "keep-alive",
+        "X-Accel-Buffering": "no",
+      });
+      res.write(`event: state\ndata: ${JSON.stringify(this.migrationState)}\n\n`);
+      this.migrationSSEClients.push(res);
+      res.on("close", () => {
+        this.migrationSSEClients = this.migrationSSEClients.filter(c => c !== res);
+      });
+      return;
+    }
+
+    this.readBody(req, (body) => {
+      let opts: { sources?: string[] } = {};
+      try { opts = JSON.parse(body); } catch { /* defaults */ }
+
+      res.writeHead(200, {
+        "Content-Type": "text/event-stream",
+        "Cache-Control": "no-cache",
+        "Connection": "keep-alive",
+        "X-Accel-Buffering": "no",
+      });
+
+      this.migrationSSEClients.push(res);
+      res.on("close", () => {
+        this.migrationSSEClients = this.migrationSSEClients.filter(c => c !== res);
+      });
+
+      this.migrationAbort = false;
+      this.migrationState = { phase: "", stored: 0, skipped: 0, merged: 0, errors: 0, processed: 0, total: 0, lastItem: null, done: false, stopped: false };
+
+      const send = (event: string, data: unknown) => {
+        if (event === "item") {
+          const d = data as any;
+          if (d.status === "stored") this.migrationState.stored++;
+          else if (d.status === "skipped" || d.status === "duplicate") this.migrationState.skipped++;
+          else if (d.status === "merged") this.migrationState.merged++;
+          else if (d.status === "error") this.migrationState.errors++;
+          this.migrationState.processed = d.index ?? this.migrationState.processed + 1;
+          this.migrationState.total = d.total ?? this.migrationState.total;
+          this.migrationState.lastItem = d;
+        } else if (event === "phase") {
+          this.migrationState.phase = (data as any).phase;
+        } else if (event === "progress") {
+          this.migrationState.total = (data as any).total ?? this.migrationState.total;
+        }
+        this.broadcastSSE(event, data);
+      };
+
+      this.migrationRunning = true;
+      this.runMigration(send, opts.sources).finally(() => {
+        this.migrationRunning = false;
+        this.migrationState.done = true;
+        if (this.migrationAbort) {
+          this.migrationState.stopped = true;
+          this.broadcastSSE("stopped", { ok: true, ...this.migrationState });
+        } else {
+          this.broadcastSSE("done", { ok: true });
+        }
+        for (const c of this.migrationSSEClients) {
+          try { c.end(); } catch { /* ignore */ }
+        }
+        this.migrationSSEClients = [];
+        this.migrationAbort = false;
+      });
+    });
+  }
+
+  private async runMigration(
+    send: (event: string, data: unknown) => void,
+    sources?: string[],
+  ): Promise<void> {
+    const ocHome = this.getOpenClawHome();
+    const importSqlite = !sources || sources.includes("sqlite");
+    const importSessions = !sources || sources.includes("sessions");
+
+    let totalProcessed = 0;
+    let totalStored = 0;
+    let totalSkipped = 0;
+    let totalErrors = 0;
+
+    const cfgPath = this.getOpenClawConfigPath();
+    let summarizerCfg: any;
+    try {
+      const raw = JSON.parse(fs.readFileSync(cfgPath, "utf-8"));
+      const pluginCfg = raw?.plugins?.entries?.["memos-local"]?.config ??
+                        raw?.plugins?.entries?.["memos-local-openclaw-plugin"]?.config ?? {};
+      summarizerCfg = pluginCfg.summarizer;
+    } catch { /* no config */ }
+
+    const summarizer = new Summarizer(summarizerCfg, this.log);
+
+    // Phase 1: Import SQLite memory chunks
+    if (importSqlite) {
+      const memoryDir = path.join(ocHome, "memory");
+      if (fs.existsSync(memoryDir)) {
+        const files = fs.readdirSync(memoryDir).filter(f => f.endsWith(".sqlite"));
+        for (const file of files) {
+          if (this.migrationAbort) break;
+          send("phase", { phase: "sqlite", file });
+          try {
+            const Database = require("better-sqlite3");
+            const db = new Database(path.join(memoryDir, file), { readonly: true });
+            const rows = db.prepare("SELECT id, path, text, updated_at FROM chunks ORDER BY updated_at ASC").all() as Array<{
+              id: string; path: string; text: string; updated_at: number;
+            }>;
+            db.close();
+
+            const agentId = file.replace(".sqlite", "");
+            send("progress", { total: rows.length, processed: 0, phase: "sqlite", file });
+
+            for (let i = 0; i < rows.length; i++) {
+              if (this.migrationAbort) break;
+              const row = rows[i];
+              totalProcessed++;
+
+              const contentHash = crypto.createHash("sha256").update(row.text).digest("hex");
+              if (this.store.chunkExistsByContent(`openclaw-import-${agentId}`, "assistant", row.text)) {
+                totalSkipped++;
+                send("item", {
+                  index: i + 1,
+                  total: rows.length,
+                  status: "skipped",
+                  preview: row.text.slice(0, 120),
+                  source: file,
+                  reason: "duplicate",
+                });
+                continue;
+              }
+
+              try {
+                const summary = await summarizer.summarize(row.text);
+                let embedding: number[] | null = null;
+                try {
+                  [embedding] = await this.embedder.embed([summary]);
+                } catch (err) {
+                  this.log.warn(`Migration embed failed: ${err}`);
+                }
+
+                let dedupStatus: "active" | "duplicate" | "merged" = "active";
+                let dedupTarget: string | null = null;
+                let dedupReason: string | null = null;
+
+                if (embedding) {
+                  const topSimilar = findTopSimilar(this.store, embedding, 0.85, 3, this.log);
+                  if (topSimilar.length > 0) {
+                    const candidates = topSimilar.map((s, idx) => {
+                      const chunk = this.store.getChunk(s.chunkId);
+                      return { index: idx + 1, summary: chunk?.summary ?? "", chunkId: s.chunkId };
+                    }).filter(c => c.summary);
+
+                    if (candidates.length > 0) {
+                      const dedupResult = await summarizer.judgeDedup(summary, candidates);
+                      if (dedupResult?.action === "DUPLICATE" && dedupResult.targetIndex) {
+                        const targetId = candidates[dedupResult.targetIndex - 1]?.chunkId;
+                        if (targetId) {
+                          dedupStatus = "duplicate";
+                          dedupTarget = targetId;
+                          dedupReason = dedupResult.reason;
+                        }
+                      } else if (dedupResult?.action === "UPDATE" && dedupResult.targetIndex && dedupResult.mergedSummary) {
+                        const targetId = candidates[dedupResult.targetIndex - 1]?.chunkId;
+                        if (targetId) {
+                          this.store.updateChunkSummaryAndContent(targetId, dedupResult.mergedSummary, row.text);
+                          try {
+                            const [newEmb] = await this.embedder.embed([dedupResult.mergedSummary]);
+                            if (newEmb) this.store.upsertEmbedding(targetId, newEmb);
+                          } catch { /* best-effort */ }
+                          dedupStatus = "merged";
+                          dedupTarget = targetId;
+                          dedupReason = dedupResult.reason;
+                        }
+                      }
+                    }
+                  }
+                }
+
+                const chunkId = uuid();
+                const chunk: Chunk = {
+                  id: chunkId,
+                  sessionKey: `openclaw-import-${agentId}`,
+                  turnId: `import-${row.id}`,
+                  seq: 0,
+                  role: "assistant",
+                  content: row.text,
+                  kind: "paragraph",
+                  summary,
+                  embedding: null,
+                  taskId: null,
+                  skillId: null,
+                  dedupStatus,
+                  dedupTarget,
+                  dedupReason,
+                  mergeCount: 0,
+                  lastHitAt: null,
+                  mergeHistory: "[]",
+                  createdAt: row.updated_at * 1000,
+                  updatedAt: row.updated_at * 1000,
+                };
+
+                this.store.insertChunk(chunk);
+                if (embedding && dedupStatus === "active") {
+                  this.store.upsertEmbedding(chunkId, embedding);
+                }
+
+                totalStored++;
+                send("item", {
+                  index: i + 1,
+                  total: rows.length,
+                  status: dedupStatus === "active" ? "stored" : dedupStatus,
+                  preview: row.text.slice(0, 120),
+                  summary: summary.slice(0, 80),
+                  source: file,
+                });
+              } catch (err) {
+                totalErrors++;
+                send("item", {
+                  index: i + 1,
+                  total: rows.length,
+                  status: "error",
+                  preview: row.text.slice(0, 120),
+                  source: file,
+                  error: String(err).slice(0, 200),
+                });
+              }
+            }
+          } catch (err) {
+            send("error", { file, error: String(err) });
+            totalErrors++;
+          }
+        }
+      }
+    }
+
+    // Phase 2: Import session JSONL files
+    if (importSessions) {
+      const sessionsDir = path.join(ocHome, "agents", "main", "sessions");
+      if (fs.existsSync(sessionsDir)) {
+        const jsonlFiles = fs.readdirSync(sessionsDir).filter(f => f.includes(".jsonl")).sort();
+        send("phase", { phase: "sessions", files: jsonlFiles.length });
+
+        let globalMsgIdx = 0;
+        let totalMsgs = 0;
+        for (const f of jsonlFiles) {
+          try {
+            const raw = fs.readFileSync(path.join(sessionsDir, f), "utf-8");
+            for (const line of raw.split("\n")) {
+              if (!line.trim()) continue;
+              try {
+                const obj = JSON.parse(line);
+                if (obj.type === "message") {
+                  const role = obj.message?.role ?? obj.role;
+                  if (role === "user" || role === "assistant") totalMsgs++;
+                }
+              } catch { /* skip */ }
+            }
+          } catch { /* skip */ }
+        }
+
+        for (const file of jsonlFiles) {
+          if (this.migrationAbort) break;
+          const sessionId = file.replace(/\.jsonl.*$/, "");
+          const filePath = path.join(sessionsDir, file);
+          send("progress", { total: totalMsgs, processed: globalMsgIdx, phase: "sessions", file });
+
+          try {
+            const fileStream = fs.createReadStream(filePath, { encoding: "utf-8" });
+            const rl = readline.createInterface({ input: fileStream, crlfDelay: Infinity });
+
+            for await (const line of rl) {
+              if (this.migrationAbort) break;
+              if (!line.trim()) continue;
+              let obj: any;
+              try { obj = JSON.parse(line); } catch { continue; }
+              if (obj.type !== "message") continue;
+              const msgRole = obj.message?.role ?? obj.role;
+              if (msgRole !== "user" && msgRole !== "assistant") continue;
+
+              const msgContent = obj.message?.content ?? obj.content;
+              let content: string;
+              if (typeof msgContent === "string") {
+                content = msgContent;
+              } else if (Array.isArray(msgContent)) {
+                content = msgContent
+                  .filter((p: any) => p.type === "text" && p.text)
+                  .map((p: any) => p.text)
+                  .join("\n");
+              } else {
+                content = JSON.stringify(msgContent);
+              }
+              if (!content || content.length < 10) continue;
+
+              globalMsgIdx++;
+              totalProcessed++;
+
+              const sessionKey = `openclaw-session-${sessionId}`;
+              if (this.store.chunkExistsByContent(sessionKey, msgRole, content)) {
+                totalSkipped++;
+                send("item", {
+                  index: globalMsgIdx,
+                  total: totalMsgs,
+                  status: "skipped",
+                  preview: content.slice(0, 120),
+                  source: file,
+                  role: msgRole,
+                  reason: "duplicate",
+                });
+                continue;
+              }
+
+              try {
+                const summary = await summarizer.summarize(content);
+                let embedding: number[] | null = null;
+                try {
+                  [embedding] = await this.embedder.embed([summary]);
+                } catch (err) {
+                  this.log.warn(`Migration embed failed: ${err}`);
+                }
+
+                let dedupStatus: "active" | "duplicate" | "merged" = "active";
+                let dedupTarget: string | null = null;
+                let dedupReason: string | null = null;
+
+                if (embedding) {
+                  const topSimilar = findTopSimilar(this.store, embedding, 0.85, 3, this.log);
+                  if (topSimilar.length > 0) {
+                    const candidates = topSimilar.map((s, idx) => {
+                      const chunk = this.store.getChunk(s.chunkId);
+                      return { index: idx + 1, summary: chunk?.summary ?? "", chunkId: s.chunkId };
+                    }).filter(c => c.summary);
+
+                    if (candidates.length > 0) {
+                      const dedupResult = await summarizer.judgeDedup(summary, candidates);
+                      if (dedupResult?.action === "DUPLICATE" && dedupResult.targetIndex) {
+                        const targetId = candidates[dedupResult.targetIndex - 1]?.chunkId;
+                        if (targetId) {
+                          dedupStatus = "duplicate";
+                          dedupTarget = targetId;
+                          dedupReason = dedupResult.reason;
+                        }
+                      } else if (dedupResult?.action === "UPDATE" && dedupResult.targetIndex && dedupResult.mergedSummary) {
+                        const targetId = candidates[dedupResult.targetIndex - 1]?.chunkId;
+                        if (targetId) {
+                          this.store.updateChunkSummaryAndContent(targetId, dedupResult.mergedSummary, content);
+                          try {
+                            const [newEmb] = await this.embedder.embed([dedupResult.mergedSummary]);
+                            if (newEmb) this.store.upsertEmbedding(targetId, newEmb);
+                          } catch { /* best-effort */ }
+                          dedupStatus = "merged";
+                          dedupTarget = targetId;
+                          dedupReason = dedupResult.reason;
+                        }
+                      }
+                    }
+                  }
+                }
+
+                const chunkId = uuid();
+                const msgTs = obj.message?.timestamp ?? obj.timestamp;
+                const ts = msgTs ? new Date(msgTs).getTime() : Date.now();
+                const chunk: Chunk = {
+                  id: chunkId,
+                  sessionKey,
+                  turnId: `import-${sessionId}-${globalMsgIdx}`,
+                  seq: 0,
+                  role: msgRole as any,
+                  content,
+                  kind: "paragraph",
+                  summary,
+                  embedding: null,
+                  taskId: null,
+                  skillId: null,
+                  dedupStatus,
+                  dedupTarget,
+                  dedupReason,
+                  mergeCount: 0,
+                  lastHitAt: null,
+                  mergeHistory: "[]",
+                  createdAt: ts,
+                  updatedAt: ts,
+                };
+
+                this.store.insertChunk(chunk);
+                if (embedding && dedupStatus === "active") {
+                  this.store.upsertEmbedding(chunkId, embedding);
+                }
+
+                totalStored++;
+                send("item", {
+                  index: globalMsgIdx,
+                  total: totalMsgs,
+                  status: dedupStatus === "active" ? "stored" : dedupStatus,
+                  preview: content.slice(0, 120),
+                  summary: summary.slice(0, 80),
+                  source: file,
+                  role: msgRole,
+                });
+              } catch (err) {
+                totalErrors++;
+                send("item", {
+                  index: globalMsgIdx,
+                  total: totalMsgs,
+                  status: "error",
+                  preview: content.slice(0, 120),
+                  source: file,
+                  error: String(err).slice(0, 200),
+                });
+              }
+            }
+          } catch (err) {
+            send("error", { file, error: String(err) });
+            totalErrors++;
+          }
+        }
+      }
+    }
+
+    send("summary", { totalProcessed, totalStored, totalSkipped, totalErrors });
+  }
+
+  // ─── Post-processing: independent task/skill generation ───
+
+  private handlePostprocess(req: http.IncomingMessage, res: http.ServerResponse): void {
+    if (this.ppRunning) {
+      res.writeHead(409, { "Content-Type": "application/json" });
+      res.end(JSON.stringify({ error: "postprocess already running" }));
+      return;
+    }
+    if (!this.ctx) {
+      res.writeHead(500, { "Content-Type": "application/json" });
+      res.end(JSON.stringify({ error: "plugin context not available — please restart the gateway" }));
+      return;
+    }
+
+    this.readBody(req, (body) => {
+      let opts: { enableTasks?: boolean; enableSkills?: boolean } = {};
+      try { opts = JSON.parse(body); } catch { /* defaults */ }
+
+      res.writeHead(200, {
+        "Content-Type": "text/event-stream",
+        "Cache-Control": "no-cache",
+        "Connection": "keep-alive",
+        "X-Accel-Buffering": "no",
+      });
+
+      this.ppSSEClients.push(res);
+      res.on("close", () => { this.ppSSEClients = this.ppSSEClients.filter(c => c !== res); });
+
+      this.ppAbort = false;
+      this.ppState = { running: true, done: false, stopped: false, processed: 0, total: 0, tasksCreated: 0, skillsCreated: 0, errors: 0 };
+
+      const send = (event: string, data: unknown) => {
+        this.broadcastPPSSE(event, data);
+      };
+
+      this.ppRunning = true;
+      this.runPostprocess(send, !!opts.enableTasks, !!opts.enableSkills).finally(() => {
+        this.ppRunning = false;
+        this.ppState.running = false;
+        this.ppState.done = true;
+        if (this.ppAbort) {
+          this.ppState.stopped = true;
+          this.broadcastPPSSE("stopped", { ...this.ppState });
+        } else {
+          this.broadcastPPSSE("done", { ...this.ppState });
+        }
+        for (const c of this.ppSSEClients) { try { c.end(); } catch { /* */ } }
+        this.ppSSEClients = [];
+        this.ppAbort = false;
+      });
+    });
+  }
+
+  private handlePostprocessStream(res: http.ServerResponse): void {
+    res.writeHead(200, {
+      "Content-Type": "text/event-stream",
+      "Cache-Control": "no-cache",
+      "Connection": "keep-alive",
+      "X-Accel-Buffering": "no",
+    });
+
+    if (this.ppRunning) {
+      res.write(`event: state\ndata: ${JSON.stringify(this.ppState)}\n\n`);
+      this.ppSSEClients.push(res);
+      res.on("close", () => { this.ppSSEClients = this.ppSSEClients.filter(c => c !== res); });
+    } else if (this.ppState.done) {
+      const evt = this.ppState.stopped ? "stopped" : "done";
+      res.write(`event: ${evt}\ndata: ${JSON.stringify(this.ppState)}\n\n`);
+      res.end();
+    } else {
+      res.end();
+    }
+  }
+
+  private handlePostprocessStop(res: http.ServerResponse): void {
+    this.ppAbort = true;
+    this.jsonResponse(res, { ok: true });
+  }
+
+  private handlePostprocessStatus(res: http.ServerResponse): void {
+    this.jsonResponse(res, this.ppState);
+  }
+
+  private broadcastPPSSE(event: string, data: unknown): void {
+    const payload = `event: ${event}\ndata: ${JSON.stringify(data)}\n\n`;
+    for (const c of this.ppSSEClients) {
+      try { c.write(payload); } catch { /* */ }
+    }
+  }
+
+  private async runPostprocess(
+    send: (event: string, data: unknown) => void,
+    enableTasks: boolean,
+    enableSkills: boolean,
+  ): Promise<void> {
+    const ctx = this.ctx!;
+    const taskProcessor = new TaskProcessor(this.store, ctx);
+    let skillEvolver: SkillEvolver | null = null;
+
+    if (enableSkills) {
+      const recallEngine = new RecallEngine(this.store, this.embedder, ctx);
+      skillEvolver = new SkillEvolver(this.store, recallEngine, ctx);
+      taskProcessor.onTaskCompleted(async (task) => {
+        try {
+          await skillEvolver!.onTaskCompleted(task);
+          this.ppState.skillsCreated++;
+          send("skill", { taskId: task.id, title: task.title });
+        } catch (err) {
+          this.log.warn(`Postprocess skill evolution error: ${err}`);
+        }
+      });
+    }
+
+    const importSessions = this.store.getDistinctSessionKeys()
+      .filter((sk: string) => sk.startsWith("openclaw-import-") || sk.startsWith("openclaw-session-"));
+
+    type PendingItem = { sessionKey: string; action: "full" | "skill-only" };
+    const pendingItems: PendingItem[] = [];
+    let skippedCount = 0;
+
+    for (const sk of importSessions) {
+      const hasTask = this.store.hasTaskForSession(sk);
+      const hasSkill = this.store.hasSkillForSessionTask(sk);
+
+      if (enableTasks && !hasTask) {
+        pendingItems.push({ sessionKey: sk, action: "full" });
+      } else if (enableSkills && hasTask && !hasSkill) {
+        pendingItems.push({ sessionKey: sk, action: "skill-only" });
+      } else {
+        skippedCount++;
+      }
+    }
+
+    this.ppState.total = pendingItems.length;
+    send("info", {
+      totalSessions: importSessions.length,
+      alreadyProcessed: skippedCount,
+      pending: pendingItems.length,
+    });
+    send("progress", { processed: 0, total: pendingItems.length });
+
+    for (let i = 0; i < pendingItems.length; i++) {
+      if (this.ppAbort) break;
+      const { sessionKey, action } = pendingItems[i];
+      this.ppState.processed = i + 1;
+
+      send("item", {
+        index: i + 1,
+        total: pendingItems.length,
+        session: sessionKey,
+        step: "processing",
+        action,
+      });
+
+      try {
+        if (action === "full") {
+          await taskProcessor.onChunksIngested(sessionKey, Date.now());
+          const activeTask = this.store.getActiveTask(sessionKey);
+          if (activeTask) {
+            await taskProcessor.finalizeTask(activeTask);
+            const finalized = this.store.getTask(activeTask.id);
+            this.ppState.tasksCreated++;
+            send("item", {
+              index: i + 1,
+              total: pendingItems.length,
+              session: sessionKey,
+              step: "done",
+              taskTitle: finalized?.title || "",
+              taskStatus: finalized?.status || "",
+            });
+          } else {
+            send("item", {
+              index: i + 1,
+              total: pendingItems.length,
+              session: sessionKey,
+              step: "done",
+              taskTitle: "(no chunks)",
+            });
+          }
+        } else if (action === "skill-only" && skillEvolver) {
+          const completedTasks = this.store.getCompletedTasksForSession(sessionKey);
+          let skillGenerated = false;
+          for (const task of completedTasks) {
+            if (this.ppAbort) break;
+            try {
+              await skillEvolver.onTaskCompleted(task);
+              this.ppState.skillsCreated++;
+              skillGenerated = true;
+              send("skill", { taskId: task.id, title: task.title });
+            } catch (err) {
+              this.log.warn(`Skill evolution error for task=${task.id}: ${err}`);
+            }
+          }
+          send("item", {
+            index: i + 1,
+            total: pendingItems.length,
+            session: sessionKey,
+            step: "done",
+            taskTitle: completedTasks[0]?.title || sessionKey,
+            action: "skill-only",
+            skillGenerated,
+          });
+        }
+      } catch (err) {
+        this.ppState.errors++;
+        this.log.warn(`Postprocess error for ${sessionKey}: ${err}`);
+        send("item", {
+          index: i + 1,
+          total: pendingItems.length,
+          session: sessionKey,
+          step: "error",
+          error: String(err).slice(0, 200),
+        });
+      }
+
+      send("progress", { processed: i + 1, total: pendingItems.length });
+    }
+  }
+
   private readBody(req: http.IncomingMessage, cb: (body: string) => void): void {
     let body = "";
     req.on("data", (chunk: Buffer) => { body += chunk.toString(); });