@memtensor/memos-local-openclaw-plugin 1.0.2-beta.5 → 1.0.2-beta.7

package/src/viewer/server.ts

@@ -1,6 +1,7 @@
 import http from "node:http";
+import os from "node:os";
 import crypto from "node:crypto";
-import { execSync } from "node:child_process";
+import { execSync, exec } from "node:child_process";
 import fs from "node:fs";
 import path from "node:path";
 import readline from "node:readline";
@@ -75,8 +76,8 @@ export class ViewerServer {
 
   private ppRunning = false;
   private ppAbort = false;
-  private ppState: { running: boolean; done: boolean; stopped: boolean; processed: number; total: number; tasksCreated: number; skillsCreated: number; errors: number } =
-    { running: false, done: false, stopped: false, processed: 0, total: 0, tasksCreated: 0, skillsCreated: 0, errors: 0 };
+  private ppState: { running: boolean; done: boolean; stopped: boolean; processed: number; total: number; tasksCreated: number; skillsCreated: number; errors: number; skippedSessions: number; totalSessions: number } =
+    { running: false, done: false, stopped: false, processed: 0, total: 0, tasksCreated: 0, skillsCreated: 0, errors: 0, skippedSessions: 0, totalSessions: 0 };
   private ppSSEClients: http.ServerResponse[] = [];
 
   constructor(opts: ViewerServerOptions) {
@@ -235,7 +236,6 @@ export class ViewerServer {
       else if (p.startsWith("/api/skill/") && req.method === "DELETE") this.handleSkillDelete(res, p);
       else if (p.startsWith("/api/skill/") && req.method === "PUT") this.handleSkillUpdate(req, res, p);
       else if (p.startsWith("/api/skill/") && req.method === "GET") this.serveSkillDetail(res, p);
-      else if (p === "/api/memory" && req.method === "POST") this.handleCreate(req, res);
       else if (p.startsWith("/api/memory/") && req.method === "GET") this.serveMemoryDetail(res, p);
       else if (p.startsWith("/api/memory/") && req.method === "PUT") this.handleUpdate(req, res, p);
       else if (p.startsWith("/api/memory/") && req.method === "DELETE") this.handleDelete(res, p);
@@ -249,6 +249,7 @@ export class ViewerServer {
       else if (p === "/api/model-health" && req.method === "GET") this.serveModelHealth(res);
       else if (p === "/api/fallback-model" && req.method === "GET") this.serveFallbackModel(res);
       else if (p === "/api/update-check" && req.method === "GET") this.handleUpdateCheck(res);
+      else if (p === "/api/update-install" && req.method === "POST") this.handleUpdateInstall(req, res);
       else if (p === "/api/auth/logout" && req.method === "POST") this.handleLogout(req, res);
       else if (p === "/api/cleanup-polluted" && req.method === "POST") this.handleCleanupPolluted(res);
       else if (p === "/api/migrate/scan" && req.method === "GET") this.handleMigrateScan(res);
@@ -382,7 +383,6 @@ export class ViewerServer {
     const offset = (page - 1) * limit;
     const session = url.searchParams.get("session") ?? undefined;
     const role = url.searchParams.get("role") ?? undefined;
-    const kind = url.searchParams.get("kind") ?? undefined;
     const dateFrom = url.searchParams.get("dateFrom") ?? undefined;
     const dateTo = url.searchParams.get("dateTo") ?? undefined;
     const owner = url.searchParams.get("owner") ?? undefined;
@@ -393,7 +393,6 @@ export class ViewerServer {
     const params: any[] = [];
     if (session) { conditions.push("session_key = ?"); params.push(session); }
     if (role) { conditions.push("role = ?"); params.push(role); }
-    if (kind) { conditions.push("kind = ?"); params.push(kind); }
     if (owner) { conditions.push("owner = ?"); params.push(owner); }
     if (dateFrom) { conditions.push("created_at >= ?"); params.push(new Date(dateFrom).getTime()); }
     if (dateTo) { conditions.push("created_at <= ?"); params.push(new Date(dateTo).getTime()); }
@@ -401,9 +400,14 @@ export class ViewerServer {
     const where = conditions.length > 0 ? " WHERE " + conditions.join(" AND ") : "";
     const totalRow = db.prepare("SELECT COUNT(*) as count FROM chunks" + where).get(...params) as any;
     const rawMemories = db.prepare("SELECT * FROM chunks" + where + ` ORDER BY created_at ${sortBy} LIMIT ? OFFSET ?`).all(...params, limit, offset);
+    const findMergeSources = db.prepare("SELECT id, summary, role FROM chunks WHERE dedup_target = ? AND (dedup_status = 'merged' OR dedup_status = 'duplicate')");
     const memories = rawMemories.map((m: any) => {
       if (m.role === "user" && m.content) {
-        return { ...m, content: stripInboundMetadata(m.content) };
+        m = { ...m, content: stripInboundMetadata(m.content) };
+      }
+      if (m.merge_count > 0) {
+        const sources = findMergeSources.all(m.id) as Array<{ id: string; summary: string; role: string }>;
+        m.merge_sources = sources;
       }
       return m;
     });
@@ -441,7 +445,7 @@ export class ViewerServer {
         id: t.id,
         sessionKey: t.sessionKey,
         title: t.title,
-        summary: t.summary ? (t.summary.length > 300 ? t.summary.slice(0, 297) + "..." : t.summary) : "",
+        summary: t.summary ?? "",
         status: t.status,
         startedAt: t.startedAt,
         endedAt: t.endedAt,
@@ -464,8 +468,7 @@ export class ViewerServer {
 
     const chunks = this.store.getChunksByTask(taskId);
     const chunkItems = chunks.map((c) => {
-      let text = c.role === "user" ? stripInboundMetadata(c.content) : c.content;
-      if (text.length > 500) text = text.slice(0, 497) + "...";
+      const text = c.role === "user" ? stripInboundMetadata(c.content) : c.content;
       return { id: c.id, role: c.role, content: text, summary: c.summary, createdAt: c.createdAt };
     });
 
@@ -502,7 +505,7 @@ export class ViewerServer {
     const emptyStats = {
       totalMemories: 0, totalSessions: 0, totalEmbeddings: 0, totalSkills: 0,
       embeddingProvider: this.embedder?.provider ?? "none",
-      roleBreakdown: {}, kindBreakdown: {}, dedupBreakdown: {},
+      dedupBreakdown: {},
       timeRange: { earliest: null, latest: null },
       sessions: [],
     };
@@ -516,7 +519,6 @@ export class ViewerServer {
       const db = (this.store as any).db;
       const total = db.prepare("SELECT COUNT(*) as count FROM chunks").get() as any;
       const sessions = db.prepare("SELECT COUNT(DISTINCT session_key) as count FROM chunks").get() as any;
-      const roles = db.prepare("SELECT role, COUNT(*) as count FROM chunks GROUP BY role").all() as any[];
       const timeRange = db.prepare("SELECT MIN(created_at) as earliest, MAX(created_at) as latest FROM chunks WHERE dedup_status = 'active'").get() as any;
       const MIN_VALID_TS = 1704067200000; // 2024-01-01
       if (timeRange.earliest != null && timeRange.earliest < MIN_VALID_TS) {
@@ -528,7 +530,6 @@ export class ViewerServer {
       }
       let embCount = 0;
       try { embCount = (db.prepare("SELECT COUNT(*) as count FROM embeddings").get() as any).count; } catch { /* table may not exist */ }
-      const kinds = db.prepare("SELECT kind, COUNT(*) as count FROM chunks GROUP BY kind").all() as any[];
       const sessionList = db.prepare(
         "SELECT session_key, COUNT(*) as count, MIN(created_at) as earliest, MAX(created_at) as latest FROM chunks GROUP BY session_key ORDER BY latest DESC",
       ).all() as any[];
@@ -552,8 +553,6 @@ export class ViewerServer {
         totalMemories: total.count, totalSessions: sessions.count, totalEmbeddings: embCount,
         totalSkills: skillCount,
         embeddingProvider: this.embedder.provider,
-        roleBreakdown: Object.fromEntries(roles.map((r: any) => [r.role, r.count])),
-        kindBreakdown: Object.fromEntries(kinds.map((k: any) => [k.kind, k.count])),
         dedupBreakdown,
         timeRange: { earliest: timeRange.earliest, latest: timeRange.latest },
         sessions: sessionList,
@@ -570,7 +569,6 @@ export class ViewerServer {
     if (!q.trim()) { this.jsonResponse(res, { results: [], query: q }); return; }
 
     const role = url.searchParams.get("role") ?? undefined;
-    const kind = url.searchParams.get("kind") ?? undefined;
     const session = url.searchParams.get("session") ?? undefined;
     const owner = url.searchParams.get("owner") ?? undefined;
     const dateFrom = url.searchParams.get("dateFrom") ?? undefined;
@@ -578,7 +576,6 @@ export class ViewerServer {
 
     const passesFilter = (r: any): boolean => {
       if (role && r.role !== role) return false;
-      if (kind && r.kind !== kind) return false;
       if (session && r.session_key !== session) return false;
       if (owner && r.owner !== owner) return false;
       if (dateFrom && r.created_at < new Date(dateFrom).getTime()) return false;
@@ -920,35 +917,6 @@ export class ViewerServer {
 
   // ─── CRUD ───
 
-  private handleCreate(req: http.IncomingMessage, res: http.ServerResponse): void {
-    this.readBody(req, (body) => {
-      try {
-        const data = JSON.parse(body);
-        if (!data.content || typeof data.content !== "string" || !data.content.trim()) {
-          res.writeHead(400, { "Content-Type": "application/json" });
-          res.end(JSON.stringify({ error: "content is required and must be a non-empty string" }));
-          return;
-        }
-        const { v4: uuidv4 } = require("uuid");
-        const id = uuidv4();
-        const now = Date.now();
-        this.store.insertChunk({
-          id, sessionKey: data.session_key || "manual", turnId: `manual-${now}`, seq: 0,
-          role: data.role || "user", content: data.content, kind: data.kind || "paragraph",
-          summary: data.summary || data.content.slice(0, 100),
-          taskId: null, skillId: null, owner: data.owner || "agent:main",
-          dedupStatus: "active", dedupTarget: null, dedupReason: null,
-          mergeCount: 0, lastHitAt: null, mergeHistory: "[]",
-          createdAt: now, updatedAt: now, embedding: null,
-        });
-        this.jsonResponse(res, { ok: true, id, message: "Memory created" });
-      } catch (err) {
-        res.writeHead(400, { "Content-Type": "application/json" });
-        res.end(JSON.stringify({ error: String(err) }));
-      }
-    });
-  }
-
   private serveMemoryDetail(res: http.ServerResponse, urlPath: string): void {
     const chunkId = urlPath.replace("/api/memory/", "");
     const chunk = this.store.getChunk(chunkId);
@@ -973,7 +941,7 @@ export class ViewerServer {
           res.end(JSON.stringify({ error: "content must be a non-empty string" }));
           return;
         }
-        const ok = this.store.updateChunk(chunkId, { summary: data.summary, content: data.content, role: data.role, kind: data.kind, owner: data.owner });
+        const ok = this.store.updateChunk(chunkId, { summary: data.summary, content: data.content, role: data.role, owner: data.owner });
         if (ok) this.jsonResponse(res, { ok: true, message: "Memory updated" });
         else { res.writeHead(404, { "Content-Type": "application/json" }); res.end(JSON.stringify({ error: "Not found" })); }
       } catch (err) {
@@ -1184,20 +1152,20 @@ export class ViewerServer {
         this.jsonResponse(res, { updateAvailable: false, current });
         return;
       }
-      const npmResp = await fetch(`https://registry.npmjs.org/${name}/latest`, {
-        signal: AbortSignal.timeout(6_000),
-      });
-      if (!npmResp.ok) {
-        this.jsonResponse(res, { updateAvailable: false, current });
+      const { computeUpdateCheck } = await import("../update-check");
+      const result = await computeUpdateCheck(name, current, fetch, 6_000);
+      if (!result) {
+        this.jsonResponse(res, { updateAvailable: false, current, packageName: name });
         return;
       }
-      const data = await npmResp.json() as { version?: string };
-      const latest = data.version ?? current;
       this.jsonResponse(res, {
-        updateAvailable: latest !== current,
-        current,
-        latest,
-        packageName: name,
+        updateAvailable: result.updateAvailable,
+        current: result.current,
+        latest: result.latest,
+        packageName: result.packageName,
+        channel: result.channel,
+        installCommand: result.installCommand,
+        stableChannel: result.stableChannel,
       });
     } catch (e) {
       this.log.warn(`handleUpdateCheck error: ${e}`);
@@ -1205,6 +1173,132 @@ export class ViewerServer {
     }
   }
 
+  private handleUpdateInstall(req: http.IncomingMessage, res: http.ServerResponse): void {
+    let body = "";
+    req.on("data", (chunk: Buffer) => { body += chunk.toString(); });
+    req.on("end", () => {
+      try {
+        const { packageSpec: rawSpec } = JSON.parse(body);
+        if (!rawSpec || typeof rawSpec !== "string") {
+          res.writeHead(400, { "Content-Type": "application/json" });
+          res.end(JSON.stringify({ ok: false, error: "Missing packageSpec" }));
+          return;
+        }
+        const packageSpec = rawSpec.trim().replace(/^(?:npx\s+)?openclaw\s+plugins\s+install\s+/i, "");
+        const allowed = /^@[\w-]+\/[\w.-]+(@[\w.-]+)?$/;
+        this.log.info(`update-install: received packageSpec="${packageSpec}" (len=${packageSpec.length})`);
+        if (!allowed.test(packageSpec)) {
+          this.log.warn(`update-install: rejected packageSpec="${packageSpec}" — does not match ${allowed}`);
+          res.writeHead(400, { "Content-Type": "application/json" });
+          res.end(JSON.stringify({ ok: false, error: `Invalid package spec: "${packageSpec}"` }));
+          return;
+        }
+
+        const pkgPath = this.findPluginPackageJson();
+        const pluginName = pkgPath
+          ? (() => { try { return JSON.parse(fs.readFileSync(pkgPath, "utf-8")).name; } catch { return null; } })()
+          : null;
+        const shortName = pluginName?.replace(/^@[\w-]+\//, "") ?? "memos-local-openclaw-plugin";
+        const extDir = path.join(os.homedir(), ".openclaw", "extensions", shortName);
+        const tmpDir = path.join(os.tmpdir(), `openclaw-update-${Date.now()}`);
+
+        // Download via npm pack, extract, and replace extension dir.
+        // Does NOT touch openclaw.json → no config watcher SIGUSR1.
+        this.log.info(`update-install: downloading ${packageSpec} via npm pack...`);
+        fs.mkdirSync(tmpDir, { recursive: true });
+        exec(`npm pack ${packageSpec} --pack-destination ${tmpDir}`, { timeout: 60_000 }, (packErr, packOut) => {
+          if (packErr) {
+            this.log.warn(`update-install: npm pack failed: ${packErr.message}`);
+            this.jsonResponse(res, { ok: false, error: `Download failed: ${packErr.message}` });
+            try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch {}
+            return;
+          }
+          const tgzFile = packOut.trim().split("\n").pop()!;
+          const tgzPath = path.join(tmpDir, tgzFile);
+          this.log.info(`update-install: downloaded ${tgzFile}, extracting...`);
+
+          const extractDir = path.join(tmpDir, "extract");
+          fs.mkdirSync(extractDir, { recursive: true });
+          exec(`tar -xzf ${tgzPath} -C ${extractDir}`, { timeout: 30_000 }, (tarErr) => {
+            if (tarErr) {
+              this.log.warn(`update-install: tar extract failed: ${tarErr.message}`);
+              this.jsonResponse(res, { ok: false, error: `Extract failed: ${tarErr.message}` });
+              try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch {}
+              return;
+            }
+
+            // npm pack extracts to a "package" subdirectory
+            const srcDir = path.join(extractDir, "package");
+            if (!fs.existsSync(srcDir)) {
+              this.jsonResponse(res, { ok: false, error: "Extracted package has no 'package' dir" });
+              try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch {}
+              return;
+            }
+
+            // Replace extension directory
+            this.log.info(`update-install: replacing ${extDir}...`);
+            try { fs.rmSync(extDir, { recursive: true, force: true }); } catch {}
+            fs.mkdirSync(path.dirname(extDir), { recursive: true });
+            fs.renameSync(srcDir, extDir);
+
+            // Install dependencies
+            this.log.info(`update-install: installing dependencies...`);
+            exec(`cd ${extDir} && npm install --omit=dev --ignore-scripts`, { timeout: 120_000 }, (npmErr, npmOut, npmStderr) => {
+              if (npmErr) {
+                try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch {}
+                this.log.warn(`update-install: npm install failed: ${npmErr.message}`);
+                this.jsonResponse(res, { ok: false, error: `Dependency install failed: ${npmStderr || npmErr.message}` });
+                return;
+              }
+
+              // Rebuild native modules (do not swallow errors)
+              exec(`cd ${extDir} && npm rebuild better-sqlite3`, { timeout: 60_000 }, (rebuildErr, rebuildOut, rebuildStderr) => {
+                if (rebuildErr) {
+                  this.log.warn(`update-install: better-sqlite3 rebuild failed: ${rebuildErr.message}`);
+                  const stderr = String(rebuildStderr || "").trim();
+                  if (stderr) this.log.warn(`update-install: rebuild stderr: ${stderr.slice(0, 500)}`);
+                  // Continue so postinstall.cjs can run (it will try rebuild again and show user guidance)
+                }
+
+                // Run postinstall.cjs: legacy cleanup, skill install, version marker, and optional sqlite re-check
+                this.log.info(`update-install: running postinstall...`);
+                exec(`cd ${extDir} && node scripts/postinstall.cjs`, { timeout: 180_000 }, (postErr, postOut, postStderr) => {
+                  try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch {}
+
+                  if (postErr) {
+                    this.log.warn(`update-install: postinstall failed: ${postErr.message}`);
+                    const postStderrStr = String(postStderr || "").trim();
+                    if (postStderrStr) this.log.warn(`update-install: postinstall stderr: ${postStderrStr.slice(0, 500)}`);
+                    // Still report success; plugin is updated, user can run postinstall manually if needed
+                  }
+
+                  // Read new version
+                  let newVersion = "unknown";
+                  try {
+                    const newPkg = JSON.parse(fs.readFileSync(path.join(extDir, "package.json"), "utf-8"));
+                    newVersion = newPkg.version ?? newVersion;
+                  } catch {}
+
+                  this.log.info(`update-install: success! Updated to ${newVersion}`);
+                  this.jsonResponse(res, { ok: true, version: newVersion });
+
+                  // Trigger Gateway restart after response is sent
+                  setTimeout(() => {
+                    this.log.info(`update-install: triggering gateway restart...`);
+                    process.kill(process.pid, "SIGUSR1");
+                  }, 500);
+                });
+              });
+            });
+          });
+        });
+      } catch (e) {
+        res.writeHead(400, { "Content-Type": "application/json" });
+        res.end(JSON.stringify({ ok: false, error: String(e) }));
+      }
+    });
+  }
+
   private async testEmbeddingModel(provider: string, model: string, endpoint: string, apiKey: string): Promise<number | undefined> {
     if (provider === "local") {
       return 384;
@@ -1438,10 +1532,18 @@ export class ViewerServer {
       }
 
       let importedSessions: string[] = [];
+      let importedChunkCount = 0;
       try {
         if (this.store) {
           importedSessions = this.store.getDistinctSessionKeys()
             .filter((sk: string) => sk.startsWith("openclaw-import-") || sk.startsWith("openclaw-session-"));
+          if (importedSessions.length > 0) {
+            const placeholders = importedSessions.map(() => "?").join(",");
+            const row = (this.store as any).db.prepare(
+              `SELECT COUNT(*) as cnt FROM chunks WHERE session_key IN (${placeholders})`
+            ).get(...importedSessions) as { cnt: number };
+            importedChunkCount = row?.cnt ?? 0;
+          }
         }
       } catch (storeErr) {
         this.log.warn(`migrate/scan: store query failed: ${storeErr}`);
@@ -1456,6 +1558,7 @@ export class ViewerServer {
         hasSummarizer,
         hasImportedData: importedSessions.length > 0,
         importedSessionCount: importedSessions.length,
+        importedChunkCount,
       });
     } catch (e) {
       this.log.warn(`migrate/scan error: ${e}`);
@@ -1587,11 +1690,14 @@ export class ViewerServer {
         } else {
           this.broadcastSSE("done", { ok: true });
         }
-        for (const c of this.migrationSSEClients) {
-          try { c.end(); } catch { /* ignore */ }
-        }
-        this.migrationSSEClients = [];
         this.migrationAbort = false;
+        const clientsToClose = [...this.migrationSSEClients];
+        this.migrationSSEClients = [];
+        setTimeout(() => {
+          for (const c of clientsToClose) {
+            try { c.end(); } catch { /* ignore */ }
+          }
+        }, 500);
       });
     });
   }
@@ -2022,7 +2128,7 @@ export class ViewerServer {
       res.on("close", () => { this.ppSSEClients = this.ppSSEClients.filter(c => c !== res); });
 
       this.ppAbort = false;
-      this.ppState = { running: true, done: false, stopped: false, processed: 0, total: 0, tasksCreated: 0, skillsCreated: 0, errors: 0 };
+      this.ppState = { running: true, done: false, stopped: false, processed: 0, total: 0, tasksCreated: 0, skillsCreated: 0, errors: 0, skippedSessions: 0, totalSessions: 0 };
 
       const send = (event: string, data: unknown) => {
         this.broadcastPPSSE(event, data);
@@ -2039,9 +2145,12 @@ export class ViewerServer {
         } else {
           this.broadcastPPSSE("done", { ...this.ppState });
         }
-        for (const c of this.ppSSEClients) { try { c.end(); } catch { /* */ } }
-        this.ppSSEClients = [];
         this.ppAbort = false;
+        const ppClientsToClose = [...this.ppSSEClients];
+        this.ppSSEClients = [];
+        setTimeout(() => {
+          for (const c of ppClientsToClose) { try { c.end(); } catch { /* */ } }
+        }, 500);
       });
     });
   }
@@ -2073,7 +2182,13 @@ export class ViewerServer {
   }
 
   private handlePostprocessStatus(res: http.ServerResponse): void {
-    this.jsonResponse(res, this.ppState);
+    let existingTasks = 0;
+    let existingSkills = 0;
+    try {
+      existingTasks = (this.store as any).db.prepare("SELECT COUNT(*) as c FROM tasks").get()?.c ?? 0;
+      existingSkills = this.store.countSkills("active");
+    } catch { /* */ }
+    this.jsonResponse(res, { ...this.ppState, existingTasks, existingSkills });
   }
 
   private broadcastPPSSE(event: string, data: unknown): void {
@@ -2123,12 +2238,18 @@ export class ViewerServer {
     }
 
     this.ppState.total = pendingItems.length;
+    this.ppState.skippedSessions = skippedCount;
+    this.ppState.totalSessions = importSessions.length;
+    const existingTaskCount = (this.store as any).db.prepare("SELECT COUNT(*) as c FROM tasks WHERE session_key IN (" + importSessions.map(() => "?").join(",") + ")").get(...importSessions)?.c ?? 0;
+    const existingSkillCount = this.store.countSkills("active");
     send("info", {
       totalSessions: importSessions.length,
       alreadyProcessed: skippedCount,
       pending: pendingItems.length,
       agents: Array.from(agentGroups.keys()),
       concurrency,
+      existingTasks: existingTaskCount,
+      existingSkills: existingSkillCount,
     });
     send("progress", { processed: 0, total: pendingItems.length });