@memtensor/memos-local-openclaw-plugin 1.0.2-beta.4 → 1.0.2-beta.6

package/src/viewer/server.ts

@@ -1,6 +1,6 @@
 import http from "node:http";
 import crypto from "node:crypto";
-import { execSync } from "node:child_process";
+import { execSync, exec } from "node:child_process";
 import fs from "node:fs";
 import path from "node:path";
 import readline from "node:readline";
@@ -48,6 +48,14 @@ export class ViewerServer {
   private readonly ctx?: PluginContext;
 
   private static readonly SESSION_TTL = 24 * 60 * 60 * 1000;
+  private static readonly PLUGIN_VERSION: string = (() => {
+    try {
+      const pkgPath = path.resolve(__dirname, "../../package.json");
+      return JSON.parse(fs.readFileSync(pkgPath, "utf-8")).version ?? "unknown";
+    } catch {
+      return "unknown";
+    }
+  })();
   private resetToken: string;
   private migrationRunning = false;
   private migrationAbort = false;
@@ -67,8 +75,8 @@ export class ViewerServer {
 
   private ppRunning = false;
   private ppAbort = false;
-  private ppState: { running: boolean; done: boolean; stopped: boolean; processed: number; total: number; tasksCreated: number; skillsCreated: number; errors: number } =
-    { running: false, done: false, stopped: false, processed: 0, total: 0, tasksCreated: 0, skillsCreated: 0, errors: 0 };
+  private ppState: { running: boolean; done: boolean; stopped: boolean; processed: number; total: number; tasksCreated: number; skillsCreated: number; errors: number; skippedSessions: number; totalSessions: number } =
+    { running: false, done: false, stopped: false, processed: 0, total: 0, tasksCreated: 0, skillsCreated: 0, errors: 0, skippedSessions: 0, totalSessions: 0 };
   private ppSSEClients: http.ServerResponse[] = [];
 
   constructor(opts: ViewerServerOptions) {
@@ -227,7 +235,6 @@ export class ViewerServer {
       else if (p.startsWith("/api/skill/") && req.method === "DELETE") this.handleSkillDelete(res, p);
       else if (p.startsWith("/api/skill/") && req.method === "PUT") this.handleSkillUpdate(req, res, p);
       else if (p.startsWith("/api/skill/") && req.method === "GET") this.serveSkillDetail(res, p);
-      else if (p === "/api/memory" && req.method === "POST") this.handleCreate(req, res);
       else if (p.startsWith("/api/memory/") && req.method === "GET") this.serveMemoryDetail(res, p);
       else if (p.startsWith("/api/memory/") && req.method === "PUT") this.handleUpdate(req, res, p);
       else if (p.startsWith("/api/memory/") && req.method === "DELETE") this.handleDelete(res, p);
@@ -241,6 +248,7 @@ export class ViewerServer {
       else if (p === "/api/model-health" && req.method === "GET") this.serveModelHealth(res);
       else if (p === "/api/fallback-model" && req.method === "GET") this.serveFallbackModel(res);
       else if (p === "/api/update-check" && req.method === "GET") this.handleUpdateCheck(res);
+      else if (p === "/api/update-install" && req.method === "POST") this.handleUpdateInstall(req, res);
       else if (p === "/api/auth/logout" && req.method === "POST") this.handleLogout(req, res);
       else if (p === "/api/cleanup-polluted" && req.method === "POST") this.handleCleanupPolluted(res);
       else if (p === "/api/migrate/scan" && req.method === "GET") this.handleMigrateScan(res);
@@ -363,7 +371,7 @@ export class ViewerServer {
 
   private serveViewer(res: http.ServerResponse): void {
     res.writeHead(200, { "Content-Type": "text/html; charset=utf-8", "Cache-Control": "no-store, no-cache, must-revalidate, max-age=0", "Pragma": "no-cache", "Expires": "0" });
-    res.end(viewerHTML);
+    res.end(viewerHTML(ViewerServer.PLUGIN_VERSION));
   }
 
   // ─── Data APIs ───
@@ -374,7 +382,6 @@ export class ViewerServer {
     const offset = (page - 1) * limit;
     const session = url.searchParams.get("session") ?? undefined;
     const role = url.searchParams.get("role") ?? undefined;
-    const kind = url.searchParams.get("kind") ?? undefined;
     const dateFrom = url.searchParams.get("dateFrom") ?? undefined;
     const dateTo = url.searchParams.get("dateTo") ?? undefined;
     const owner = url.searchParams.get("owner") ?? undefined;
@@ -385,7 +392,6 @@ export class ViewerServer {
     const params: any[] = [];
     if (session) { conditions.push("session_key = ?"); params.push(session); }
     if (role) { conditions.push("role = ?"); params.push(role); }
-    if (kind) { conditions.push("kind = ?"); params.push(kind); }
     if (owner) { conditions.push("owner = ?"); params.push(owner); }
     if (dateFrom) { conditions.push("created_at >= ?"); params.push(new Date(dateFrom).getTime()); }
     if (dateTo) { conditions.push("created_at <= ?"); params.push(new Date(dateTo).getTime()); }
@@ -393,9 +399,14 @@ export class ViewerServer {
     const where = conditions.length > 0 ? " WHERE " + conditions.join(" AND ") : "";
     const totalRow = db.prepare("SELECT COUNT(*) as count FROM chunks" + where).get(...params) as any;
     const rawMemories = db.prepare("SELECT * FROM chunks" + where + ` ORDER BY created_at ${sortBy} LIMIT ? OFFSET ?`).all(...params, limit, offset);
+    const findMergeSources = db.prepare("SELECT id, summary, role FROM chunks WHERE dedup_target = ? AND (dedup_status = 'merged' OR dedup_status = 'duplicate')");
     const memories = rawMemories.map((m: any) => {
       if (m.role === "user" && m.content) {
-        return { ...m, content: stripInboundMetadata(m.content) };
+        m = { ...m, content: stripInboundMetadata(m.content) };
+      }
+      if (m.merge_count > 0) {
+        const sources = findMergeSources.all(m.id) as Array<{ id: string; summary: string; role: string }>;
+        m.merge_sources = sources;
       }
       return m;
     });
@@ -433,7 +444,7 @@ export class ViewerServer {
         id: t.id,
         sessionKey: t.sessionKey,
         title: t.title,
-        summary: t.summary ? (t.summary.length > 300 ? t.summary.slice(0, 297) + "..." : t.summary) : "",
+        summary: t.summary ?? "",
         status: t.status,
         startedAt: t.startedAt,
         endedAt: t.endedAt,
@@ -456,8 +467,7 @@ export class ViewerServer {
 
     const chunks = this.store.getChunksByTask(taskId);
     const chunkItems = chunks.map((c) => {
-      let text = c.role === "user" ? stripInboundMetadata(c.content) : c.content;
-      if (text.length > 500) text = text.slice(0, 497) + "...";
+      const text = c.role === "user" ? stripInboundMetadata(c.content) : c.content;
       return { id: c.id, role: c.role, content: text, summary: c.summary, createdAt: c.createdAt };
     });
 
@@ -494,7 +504,7 @@ export class ViewerServer {
     const emptyStats = {
       totalMemories: 0, totalSessions: 0, totalEmbeddings: 0, totalSkills: 0,
       embeddingProvider: this.embedder?.provider ?? "none",
-      roleBreakdown: {}, kindBreakdown: {}, dedupBreakdown: {},
+      dedupBreakdown: {},
       timeRange: { earliest: null, latest: null },
       sessions: [],
     };
@@ -508,7 +518,6 @@ export class ViewerServer {
       const db = (this.store as any).db;
       const total = db.prepare("SELECT COUNT(*) as count FROM chunks").get() as any;
       const sessions = db.prepare("SELECT COUNT(DISTINCT session_key) as count FROM chunks").get() as any;
-      const roles = db.prepare("SELECT role, COUNT(*) as count FROM chunks GROUP BY role").all() as any[];
       const timeRange = db.prepare("SELECT MIN(created_at) as earliest, MAX(created_at) as latest FROM chunks WHERE dedup_status = 'active'").get() as any;
       const MIN_VALID_TS = 1704067200000; // 2024-01-01
       if (timeRange.earliest != null && timeRange.earliest < MIN_VALID_TS) {
@@ -520,7 +529,6 @@ export class ViewerServer {
       }
       let embCount = 0;
       try { embCount = (db.prepare("SELECT COUNT(*) as count FROM embeddings").get() as any).count; } catch { /* table may not exist */ }
-      const kinds = db.prepare("SELECT kind, COUNT(*) as count FROM chunks GROUP BY kind").all() as any[];
       const sessionList = db.prepare(
         "SELECT session_key, COUNT(*) as count, MIN(created_at) as earliest, MAX(created_at) as latest FROM chunks GROUP BY session_key ORDER BY latest DESC",
       ).all() as any[];
@@ -544,8 +552,6 @@ export class ViewerServer {
         totalMemories: total.count, totalSessions: sessions.count, totalEmbeddings: embCount,
         totalSkills: skillCount,
         embeddingProvider: this.embedder.provider,
-        roleBreakdown: Object.fromEntries(roles.map((r: any) => [r.role, r.count])),
-        kindBreakdown: Object.fromEntries(kinds.map((k: any) => [k.kind, k.count])),
         dedupBreakdown,
         timeRange: { earliest: timeRange.earliest, latest: timeRange.latest },
         sessions: sessionList,
@@ -562,45 +568,70 @@ export class ViewerServer {
     if (!q.trim()) { this.jsonResponse(res, { results: [], query: q }); return; }
 
     const role = url.searchParams.get("role") ?? undefined;
-    const kind = url.searchParams.get("kind") ?? undefined;
+    const session = url.searchParams.get("session") ?? undefined;
+    const owner = url.searchParams.get("owner") ?? undefined;
     const dateFrom = url.searchParams.get("dateFrom") ?? undefined;
     const dateTo = url.searchParams.get("dateTo") ?? undefined;
 
     const passesFilter = (r: any): boolean => {
       if (role && r.role !== role) return false;
-      if (kind && r.kind !== kind) return false;
+      if (session && r.session_key !== session) return false;
+      if (owner && r.owner !== owner) return false;
       if (dateFrom && r.created_at < new Date(dateFrom).getTime()) return false;
       if (dateTo && r.created_at > new Date(dateTo).getTime()) return false;
       return true;
     };
 
+    const ftsFilters: string[] = [];
+    const likeFilters: string[] = [];
+    const sqlParams: any[] = [];
+    if (session) { ftsFilters.push("c.session_key = ?"); likeFilters.push("session_key = ?"); sqlParams.push(session); }
+    if (owner) { ftsFilters.push("c.owner = ?"); likeFilters.push("owner = ?"); sqlParams.push(owner); }
+    const ftsWhere = ftsFilters.length > 0 ? " AND " + ftsFilters.join(" AND ") : "";
+    const likeWhere = likeFilters.length > 0 ? " AND " + likeFilters.join(" AND ") : "";
+
     const db = (this.store as any).db;
     let ftsResults: any[] = [];
     try {
       ftsResults = db.prepare(
-        "SELECT c.* FROM chunks_fts f JOIN chunks c ON f.rowid = c.rowid WHERE chunks_fts MATCH ? ORDER BY rank LIMIT 100",
-      ).all(q).filter(passesFilter);
+        `SELECT c.* FROM chunks_fts f JOIN chunks c ON f.rowid = c.rowid WHERE chunks_fts MATCH ?${ftsWhere} ORDER BY rank LIMIT 100`,
+      ).all(q, ...sqlParams).filter(passesFilter);
     } catch { /* FTS syntax error, fall through */ }
     if (ftsResults.length === 0) {
-      ftsResults = db.prepare(
-        "SELECT * FROM chunks WHERE content LIKE ? OR summary LIKE ? ORDER BY created_at DESC LIMIT 100",
-      ).all(`%${q}%`, `%${q}%`).filter(passesFilter);
+      try {
+        ftsResults = db.prepare(
+          `SELECT * FROM chunks WHERE (content LIKE ? OR summary LIKE ?)${likeWhere} ORDER BY created_at DESC LIMIT 100`,
+        ).all(`%${q}%`, `%${q}%`, ...sqlParams).filter(passesFilter);
+      } catch (err) {
+        this.log.warn(`LIKE search failed: ${err}`);
+      }
     }
 
     const SEMANTIC_THRESHOLD = 0.64;
+    const VECTOR_TIMEOUT_MS = 8000;
     let vectorResults: any[] = [];
     let scoreMap = new Map<string, number>();
     try {
-      const queryVec = await this.embedder.embedQuery(q);
-      const hits = vectorSearch(this.store, queryVec, 40);
-      scoreMap = new Map(hits.map(h => [h.chunkId, h.score]));
-      const hitIds = new Set(hits.filter(h => h.score >= SEMANTIC_THRESHOLD).map(h => h.chunkId));
-      if (hitIds.size > 0) {
-        const placeholders = [...hitIds].map(() => "?").join(",");
-        const rows = db.prepare(`SELECT * FROM chunks WHERE id IN (${placeholders})`).all(...hitIds).filter(passesFilter);
-        rows.forEach((r: any) => { r._vscore = scoreMap.get(r.id) ?? 0; });
-        rows.sort((a: any, b: any) => (b._vscore ?? 0) - (a._vscore ?? 0));
-        vectorResults = rows;
+      const vecPromise = (async () => {
+        const queryVec = await this.embedder.embedQuery(q);
+        return vectorSearch(this.store, queryVec, 40);
+      })();
+      const hits = await Promise.race([
+        vecPromise,
+        new Promise<null>((resolve) => setTimeout(() => resolve(null), VECTOR_TIMEOUT_MS)),
+      ]);
+      if (hits) {
+        scoreMap = new Map(hits.map(h => [h.chunkId, h.score]));
+        const hitIds = new Set(hits.filter(h => h.score >= SEMANTIC_THRESHOLD).map(h => h.chunkId));
+        if (hitIds.size > 0) {
+          const placeholders = [...hitIds].map(() => "?").join(",");
+          const rows = db.prepare(`SELECT * FROM chunks WHERE id IN (${placeholders})${likeWhere}`).all(...hitIds, ...sqlParams).filter(passesFilter);
+          rows.forEach((r: any) => { r._vscore = scoreMap.get(r.id) ?? 0; });
+          rows.sort((a: any, b: any) => (b._vscore ?? 0) - (a._vscore ?? 0));
+          vectorResults = rows;
+        }
+      } else {
+        this.log.warn("Vector search timed out, returning FTS results only");
       }
     } catch (err) {
       this.log.warn(`Vector search failed (falling back to FTS only): ${err}`);
@@ -885,35 +916,6 @@ export class ViewerServer {
 
   // ─── CRUD ───
 
-  private handleCreate(req: http.IncomingMessage, res: http.ServerResponse): void {
-    this.readBody(req, (body) => {
-      try {
-        const data = JSON.parse(body);
-        if (!data.content || typeof data.content !== "string" || !data.content.trim()) {
-          res.writeHead(400, { "Content-Type": "application/json" });
-          res.end(JSON.stringify({ error: "content is required and must be a non-empty string" }));
-          return;
-        }
-        const { v4: uuidv4 } = require("uuid");
-        const id = uuidv4();
-        const now = Date.now();
-        this.store.insertChunk({
-          id, sessionKey: data.session_key || "manual", turnId: `manual-${now}`, seq: 0,
-          role: data.role || "user", content: data.content, kind: data.kind || "paragraph",
-          summary: data.summary || data.content.slice(0, 100),
-          taskId: null, skillId: null, owner: data.owner || "agent:main",
-          dedupStatus: "active", dedupTarget: null, dedupReason: null,
-          mergeCount: 0, lastHitAt: null, mergeHistory: "[]",
-          createdAt: now, updatedAt: now, embedding: null,
-        });
-        this.jsonResponse(res, { ok: true, id, message: "Memory created" });
-      } catch (err) {
-        res.writeHead(400, { "Content-Type": "application/json" });
-        res.end(JSON.stringify({ error: String(err) }));
-      }
-    });
-  }
-
   private serveMemoryDetail(res: http.ServerResponse, urlPath: string): void {
     const chunkId = urlPath.replace("/api/memory/", "");
     const chunk = this.store.getChunk(chunkId);
@@ -938,7 +940,7 @@ export class ViewerServer {
           res.end(JSON.stringify({ error: "content must be a non-empty string" }));
           return;
         }
-        const ok = this.store.updateChunk(chunkId, { summary: data.summary, content: data.content, role: data.role, kind: data.kind, owner: data.owner });
+        const ok = this.store.updateChunk(chunkId, { summary: data.summary, content: data.content, role: data.role, owner: data.owner });
         if (ok) this.jsonResponse(res, { ok: true, message: "Memory updated" });
         else { res.writeHead(404, { "Content-Type": "application/json" }); res.end(JSON.stringify({ error: "Not found" })); }
       } catch (err) {
@@ -1149,20 +1151,20 @@ export class ViewerServer {
         this.jsonResponse(res, { updateAvailable: false, current });
         return;
       }
-      const npmResp = await fetch(`https://registry.npmjs.org/${name}/latest`, {
-        signal: AbortSignal.timeout(6_000),
-      });
-      if (!npmResp.ok) {
-        this.jsonResponse(res, { updateAvailable: false, current });
+      const { computeUpdateCheck } = await import("../update-check");
+      const result = await computeUpdateCheck(name, current, fetch, 6_000);
+      if (!result) {
+        this.jsonResponse(res, { updateAvailable: false, current, packageName: name });
         return;
       }
-      const data = await npmResp.json() as { version?: string };
-      const latest = data.version ?? current;
       this.jsonResponse(res, {
-        updateAvailable: latest !== current,
-        current,
-        latest,
-        packageName: name,
+        updateAvailable: result.updateAvailable,
+        current: result.current,
+        latest: result.latest,
+        packageName: result.packageName,
+        channel: result.channel,
+        installCommand: result.installCommand,
+        stableChannel: result.stableChannel,
       });
     } catch (e) {
       this.log.warn(`handleUpdateCheck error: ${e}`);
@@ -1170,6 +1172,47 @@ export class ViewerServer {
     }
   }
 
+  private handleUpdateInstall(req: http.IncomingMessage, res: http.ServerResponse): void {
+    let body = "";
+    req.on("data", (chunk: Buffer) => { body += chunk.toString(); });
+    req.on("end", () => {
+      try {
+        const { packageSpec } = JSON.parse(body);
+        if (!packageSpec || typeof packageSpec !== "string") {
+          res.writeHead(400, { "Content-Type": "application/json" });
+          res.end(JSON.stringify({ ok: false, error: "Missing packageSpec" }));
+          return;
+        }
+        const allowed = /^@[\w-]+\/[\w.-]+(@[\w.-]+)?$/;
+        if (!allowed.test(packageSpec)) {
+          res.writeHead(400, { "Content-Type": "application/json" });
+          res.end(JSON.stringify({ ok: false, error: "Invalid package spec" }));
+          return;
+        }
+        this.log.info(`update-install: installing ${packageSpec}...`);
+        exec(`npx openclaw plugins install ${packageSpec}`, { timeout: 120_000 }, (err, stdout, stderr) => {
+          if (err) {
+            this.log.warn(`update-install failed: ${err.message}\n${stderr}`);
+            this.jsonResponse(res, { ok: false, error: stderr || err.message });
+            return;
+          }
+          this.log.info(`update-install success: ${stdout}`);
+          this.jsonResponse(res, { ok: true, output: stdout });
+          this.log.info(`update-install: restarting gateway...`);
+          setTimeout(() => {
+            exec("npx openclaw gateway restart", { timeout: 30_000 }, (restartErr) => {
+              if (restartErr) this.log.warn(`gateway restart failed: ${restartErr.message}`);
+              else this.log.info("gateway restart initiated");
+            });
+          }, 1000);
+        });
+      } catch (e) {
+        res.writeHead(400, { "Content-Type": "application/json" });
+        res.end(JSON.stringify({ ok: false, error: String(e) }));
+      }
+    });
+  }
+
   private async testEmbeddingModel(provider: string, model: string, endpoint: string, apiKey: string): Promise<number | undefined> {
     if (provider === "local") {
       return 384;
@@ -1403,10 +1446,18 @@ export class ViewerServer {
       }
 
       let importedSessions: string[] = [];
+      let importedChunkCount = 0;
       try {
         if (this.store) {
           importedSessions = this.store.getDistinctSessionKeys()
             .filter((sk: string) => sk.startsWith("openclaw-import-") || sk.startsWith("openclaw-session-"));
+          if (importedSessions.length > 0) {
+            const placeholders = importedSessions.map(() => "?").join(",");
+            const row = (this.store as any).db.prepare(
+              `SELECT COUNT(*) as cnt FROM chunks WHERE session_key IN (${placeholders})`
+            ).get(...importedSessions) as { cnt: number };
+            importedChunkCount = row?.cnt ?? 0;
+          }
         }
       } catch (storeErr) {
         this.log.warn(`migrate/scan: store query failed: ${storeErr}`);
@@ -1421,6 +1472,7 @@ export class ViewerServer {
         hasSummarizer,
         hasImportedData: importedSessions.length > 0,
         importedSessionCount: importedSessions.length,
+        importedChunkCount,
       });
     } catch (e) {
       this.log.warn(`migrate/scan error: ${e}`);
@@ -1552,11 +1604,14 @@ export class ViewerServer {
         } else {
           this.broadcastSSE("done", { ok: true });
         }
-        for (const c of this.migrationSSEClients) {
-          try { c.end(); } catch { /* ignore */ }
-        }
-        this.migrationSSEClients = [];
         this.migrationAbort = false;
+        const clientsToClose = [...this.migrationSSEClients];
+        this.migrationSSEClients = [];
+        setTimeout(() => {
+          for (const c of clientsToClose) {
+            try { c.end(); } catch { /* ignore */ }
+          }
+        }, 500);
       });
     });
   }
@@ -1987,7 +2042,7 @@ export class ViewerServer {
       res.on("close", () => { this.ppSSEClients = this.ppSSEClients.filter(c => c !== res); });
 
       this.ppAbort = false;
-      this.ppState = { running: true, done: false, stopped: false, processed: 0, total: 0, tasksCreated: 0, skillsCreated: 0, errors: 0 };
+      this.ppState = { running: true, done: false, stopped: false, processed: 0, total: 0, tasksCreated: 0, skillsCreated: 0, errors: 0, skippedSessions: 0, totalSessions: 0 };
 
       const send = (event: string, data: unknown) => {
         this.broadcastPPSSE(event, data);
@@ -2004,9 +2059,12 @@ export class ViewerServer {
         } else {
           this.broadcastPPSSE("done", { ...this.ppState });
         }
-        for (const c of this.ppSSEClients) { try { c.end(); } catch { /* */ } }
-        this.ppSSEClients = [];
         this.ppAbort = false;
+        const ppClientsToClose = [...this.ppSSEClients];
+        this.ppSSEClients = [];
+        setTimeout(() => {
+          for (const c of ppClientsToClose) { try { c.end(); } catch { /* */ } }
+        }, 500);
       });
     });
   }
@@ -2038,7 +2096,13 @@ export class ViewerServer {
   }
 
   private handlePostprocessStatus(res: http.ServerResponse): void {
-    this.jsonResponse(res, this.ppState);
+    let existingTasks = 0;
+    let existingSkills = 0;
+    try {
+      existingTasks = (this.store as any).db.prepare("SELECT COUNT(*) as c FROM tasks").get()?.c ?? 0;
+      existingSkills = this.store.countSkills("active");
+    } catch { /* */ }
+    this.jsonResponse(res, { ...this.ppState, existingTasks, existingSkills });
   }
 
   private broadcastPPSSE(event: string, data: unknown): void {
@@ -2088,12 +2152,18 @@ export class ViewerServer {
     }
 
     this.ppState.total = pendingItems.length;
+    this.ppState.skippedSessions = skippedCount;
+    this.ppState.totalSessions = importSessions.length;
+    const existingTaskCount = (this.store as any).db.prepare("SELECT COUNT(*) as c FROM tasks WHERE session_key IN (" + importSessions.map(() => "?").join(",") + ")").get(...importSessions)?.c ?? 0;
+    const existingSkillCount = this.store.countSkills("active");
     send("info", {
       totalSessions: importSessions.length,
       alreadyProcessed: skippedCount,
       pending: pendingItems.length,
       agents: Array.from(agentGroups.keys()),
       concurrency,
+      existingTasks: existingTaskCount,
+      existingSkills: existingSkillCount,
     });
     send("progress", { processed: 0, total: pendingItems.length });