@memtensor/memos-local-openclaw-plugin 1.0.2-beta.3 → 1.0.2-beta.5

package/src/viewer/server.ts

@@ -6,7 +6,7 @@ import path from "node:path";
 import readline from "node:readline";
 import type { SqliteStore } from "../storage/sqlite";
 import type { Embedder } from "../embedding";
-import { Summarizer } from "../ingest/providers";
+import { Summarizer, modelHealth } from "../ingest/providers";
 import { findTopSimilar } from "../ingest/dedup";
 import { stripInboundMetadata } from "../capture";
 import { vectorSearch } from "../storage/vector";
@@ -17,6 +17,11 @@ import type { Logger, Chunk, PluginContext } from "../types";
 import { viewerHTML } from "./html";
 import { v4 as uuid } from "uuid";
 
+function normalizeTimestamp(ts: number): number {
+  if (ts < 1e12) return ts * 1000;
+  return ts;
+}
+
 export interface ViewerServerOptions {
   store: SqliteStore;
   embedder: Embedder;
@@ -43,6 +48,14 @@ export class ViewerServer {
   private readonly ctx?: PluginContext;
 
   private static readonly SESSION_TTL = 24 * 60 * 60 * 1000;
+  private static readonly PLUGIN_VERSION: string = (() => {
+    try {
+      const pkgPath = path.resolve(__dirname, "../../package.json");
+      return JSON.parse(fs.readFileSync(pkgPath, "utf-8")).version ?? "unknown";
+    } catch {
+      return "unknown";
+    }
+  })();
   private resetToken: string;
   private migrationRunning = false;
   private migrationAbort = false;
@@ -93,11 +106,28 @@ export class ViewerServer {
       this.server.listen(this.port, "127.0.0.1", () => {
         const addr = this.server!.address();
         const actualPort = typeof addr === "object" && addr ? addr.port : this.port;
+        this.autoCleanupPolluted();
         resolve(`http://127.0.0.1:${actualPort}`);
       });
     });
   }
 
+  private autoCleanupPolluted(): void {
+    try {
+      const polluted = this.store.findPollutedUserChunks();
+      let deleted = 0;
+      for (const { id } of polluted) {
+        if (this.store.deleteChunk(id)) deleted++;
+      }
+      const fixed = this.store.fixMixedUserChunks();
+      if (deleted > 0 || fixed > 0) {
+        this.log.info(`Auto-cleanup: removed ${deleted} polluted chunks, fixed ${fixed} mixed user+assistant chunks`);
+      }
+    } catch (err) {
+      this.log.warn(`Auto-cleanup failed: ${err}`);
+    }
+  }
+
   stop(): void {
     this.server?.close();
     this.server = null;
@@ -216,9 +246,11 @@ export class ViewerServer {
       else if (p === "/api/config" && req.method === "GET") this.serveConfig(res);
       else if (p === "/api/config" && req.method === "PUT") this.handleSaveConfig(req, res);
       else if (p === "/api/test-model" && req.method === "POST") this.handleTestModel(req, res);
+      else if (p === "/api/model-health" && req.method === "GET") this.serveModelHealth(res);
       else if (p === "/api/fallback-model" && req.method === "GET") this.serveFallbackModel(res);
       else if (p === "/api/update-check" && req.method === "GET") this.handleUpdateCheck(res);
       else if (p === "/api/auth/logout" && req.method === "POST") this.handleLogout(req, res);
+      else if (p === "/api/cleanup-polluted" && req.method === "POST") this.handleCleanupPolluted(res);
       else if (p === "/api/migrate/scan" && req.method === "GET") this.handleMigrateScan(res);
       else if (p === "/api/migrate/start" && req.method === "POST") this.handleMigrateStart(req, res);
       else if (p === "/api/migrate/status" && req.method === "GET") this.handleMigrateStatus(res);
@@ -339,7 +371,7 @@ export class ViewerServer {
 
   private serveViewer(res: http.ServerResponse): void {
     res.writeHead(200, { "Content-Type": "text/html; charset=utf-8", "Cache-Control": "no-store, no-cache, must-revalidate, max-age=0", "Pragma": "no-cache", "Expires": "0" });
-    res.end(viewerHTML);
+    res.end(viewerHTML(ViewerServer.PLUGIN_VERSION));
   }
 
   // ─── Data APIs ───
@@ -485,7 +517,15 @@ export class ViewerServer {
       const total = db.prepare("SELECT COUNT(*) as count FROM chunks").get() as any;
       const sessions = db.prepare("SELECT COUNT(DISTINCT session_key) as count FROM chunks").get() as any;
       const roles = db.prepare("SELECT role, COUNT(*) as count FROM chunks GROUP BY role").all() as any[];
-      const timeRange = db.prepare("SELECT MIN(created_at) as earliest, MAX(created_at) as latest FROM chunks").get() as any;
+      const timeRange = db.prepare("SELECT MIN(created_at) as earliest, MAX(created_at) as latest FROM chunks WHERE dedup_status = 'active'").get() as any;
+      const MIN_VALID_TS = 1704067200000; // 2024-01-01
+      if (timeRange.earliest != null && timeRange.earliest < MIN_VALID_TS) {
+        timeRange.earliest = db.prepare("SELECT MIN(created_at) as v FROM chunks WHERE dedup_status = 'active' AND created_at >= ?").get(MIN_VALID_TS) as any;
+        timeRange.earliest = timeRange.earliest?.v ?? null;
+      }
+      if (timeRange.latest != null && timeRange.latest < MIN_VALID_TS) {
+        timeRange.latest = null;
+      }
       let embCount = 0;
       try { embCount = (db.prepare("SELECT COUNT(*) as count FROM embeddings").get() as any).count; } catch { /* table may not exist */ }
       const kinds = db.prepare("SELECT kind, COUNT(*) as count FROM chunks GROUP BY kind").all() as any[];
@@ -531,44 +571,71 @@ export class ViewerServer {
 
     const role = url.searchParams.get("role") ?? undefined;
     const kind = url.searchParams.get("kind") ?? undefined;
+    const session = url.searchParams.get("session") ?? undefined;
+    const owner = url.searchParams.get("owner") ?? undefined;
     const dateFrom = url.searchParams.get("dateFrom") ?? undefined;
     const dateTo = url.searchParams.get("dateTo") ?? undefined;
 
     const passesFilter = (r: any): boolean => {
       if (role && r.role !== role) return false;
       if (kind && r.kind !== kind) return false;
+      if (session && r.session_key !== session) return false;
+      if (owner && r.owner !== owner) return false;
       if (dateFrom && r.created_at < new Date(dateFrom).getTime()) return false;
       if (dateTo && r.created_at > new Date(dateTo).getTime()) return false;
       return true;
     };
 
+    const ftsFilters: string[] = [];
+    const likeFilters: string[] = [];
+    const sqlParams: any[] = [];
+    if (session) { ftsFilters.push("c.session_key = ?"); likeFilters.push("session_key = ?"); sqlParams.push(session); }
+    if (owner) { ftsFilters.push("c.owner = ?"); likeFilters.push("owner = ?"); sqlParams.push(owner); }
+    const ftsWhere = ftsFilters.length > 0 ? " AND " + ftsFilters.join(" AND ") : "";
+    const likeWhere = likeFilters.length > 0 ? " AND " + likeFilters.join(" AND ") : "";
+
     const db = (this.store as any).db;
     let ftsResults: any[] = [];
     try {
       ftsResults = db.prepare(
-        "SELECT c.* FROM chunks_fts f JOIN chunks c ON f.rowid = c.rowid WHERE chunks_fts MATCH ? ORDER BY rank LIMIT 100",
-      ).all(q).filter(passesFilter);
+        `SELECT c.* FROM chunks_fts f JOIN chunks c ON f.rowid = c.rowid WHERE chunks_fts MATCH ?${ftsWhere} ORDER BY rank LIMIT 100`,
+      ).all(q, ...sqlParams).filter(passesFilter);
     } catch { /* FTS syntax error, fall through */ }
     if (ftsResults.length === 0) {
-      ftsResults = db.prepare(
-        "SELECT * FROM chunks WHERE content LIKE ? OR summary LIKE ? ORDER BY created_at DESC LIMIT 100",
-      ).all(`%${q}%`, `%${q}%`).filter(passesFilter);
+      try {
+        ftsResults = db.prepare(
+          `SELECT * FROM chunks WHERE (content LIKE ? OR summary LIKE ?)${likeWhere} ORDER BY created_at DESC LIMIT 100`,
+        ).all(`%${q}%`, `%${q}%`, ...sqlParams).filter(passesFilter);
+      } catch (err) {
+        this.log.warn(`LIKE search failed: ${err}`);
+      }
     }
 
     const SEMANTIC_THRESHOLD = 0.64;
+    const VECTOR_TIMEOUT_MS = 8000;
     let vectorResults: any[] = [];
     let scoreMap = new Map<string, number>();
     try {
-      const queryVec = await this.embedder.embedQuery(q);
-      const hits = vectorSearch(this.store, queryVec, 40);
-      scoreMap = new Map(hits.map(h => [h.chunkId, h.score]));
-      const hitIds = new Set(hits.filter(h => h.score >= SEMANTIC_THRESHOLD).map(h => h.chunkId));
-      if (hitIds.size > 0) {
-        const placeholders = [...hitIds].map(() => "?").join(",");
-        const rows = db.prepare(`SELECT * FROM chunks WHERE id IN (${placeholders})`).all(...hitIds).filter(passesFilter);
-        rows.forEach((r: any) => { r._vscore = scoreMap.get(r.id) ?? 0; });
-        rows.sort((a: any, b: any) => (b._vscore ?? 0) - (a._vscore ?? 0));
-        vectorResults = rows;
+      const vecPromise = (async () => {
+        const queryVec = await this.embedder.embedQuery(q);
+        return vectorSearch(this.store, queryVec, 40);
+      })();
+      const hits = await Promise.race([
+        vecPromise,
+        new Promise<null>((resolve) => setTimeout(() => resolve(null), VECTOR_TIMEOUT_MS)),
+      ]);
+      if (hits) {
+        scoreMap = new Map(hits.map(h => [h.chunkId, h.score]));
+        const hitIds = new Set(hits.filter(h => h.score >= SEMANTIC_THRESHOLD).map(h => h.chunkId));
+        if (hitIds.size > 0) {
+          const placeholders = [...hitIds].map(() => "?").join(",");
+          const rows = db.prepare(`SELECT * FROM chunks WHERE id IN (${placeholders})${likeWhere}`).all(...hitIds, ...sqlParams).filter(passesFilter);
+          rows.forEach((r: any) => { r._vscore = scoreMap.get(r.id) ?? 0; });
+          rows.sort((a: any, b: any) => (b._vscore ?? 0) - (a._vscore ?? 0));
+          vectorResults = rows;
+        }
+      } else {
+        this.log.warn("Vector search timed out, returning FTS results only");
       }
     } catch (err) {
       this.log.warn(`Vector search failed (falling back to FTS only): ${err}`);
@@ -1041,8 +1108,8 @@ export class ViewerServer {
           return;
         }
         if (type === "embedding") {
-          await this.testEmbeddingModel(provider, model, endpoint, apiKey);
-          this.jsonResponse(res, { ok: true, detail: `${provider}/${model}` });
+          const dims = await this.testEmbeddingModel(provider, model, endpoint, apiKey);
+          this.jsonResponse(res, { ok: true, detail: `${provider}/${model}`, dimensions: dims });
         } else {
           await this.testChatModel(provider, model, endpoint, apiKey);
           this.jsonResponse(res, { ok: true, detail: `${provider}/${model}` });
@@ -1055,6 +1122,10 @@ export class ViewerServer {
     });
   }
 
+  private serveModelHealth(res: http.ServerResponse): void {
+    this.jsonResponse(res, { models: modelHealth.getAll() });
+  }
+
   private serveFallbackModel(res: http.ServerResponse): void {
     try {
       const cfgPath = this.getOpenClawConfigPath();
@@ -1134,9 +1205,9 @@ export class ViewerServer {
     }
   }
 
-  private async testEmbeddingModel(provider: string, model: string, endpoint: string, apiKey: string): Promise<void> {
+  private async testEmbeddingModel(provider: string, model: string, endpoint: string, apiKey: string): Promise<number | undefined> {
     if (provider === "local") {
-      return;
+      return 384;
     }
     const baseUrl = (endpoint || "https://api.openai.com/v1").replace(/\/+$/, "");
     const embUrl = baseUrl.endsWith("/embeddings") ? baseUrl : `${baseUrl}/embeddings`;
@@ -1149,39 +1220,59 @@ export class ViewerServer {
       const resp = await fetch(baseUrl.replace(/\/v\d+.*/, "/v2/embed"), {
         method: "POST",
         headers,
-        body: JSON.stringify({ texts: ["test"], model: model || "embed-english-v3.0", input_type: "search_query", embedding_types: ["float"] }),
+        body: JSON.stringify({ texts: ["test embedding vector"], model: model || "embed-english-v3.0", input_type: "search_query", embedding_types: ["float"] }),
         signal: AbortSignal.timeout(15_000),
       });
       if (!resp.ok) {
         const txt = await resp.text();
         throw new Error(`Cohere embed ${resp.status}: ${txt}`);
       }
-      return;
+      const json = await resp.json() as any;
+      const vecs = json?.embeddings?.float;
+      if (!Array.isArray(vecs) || vecs.length === 0 || !Array.isArray(vecs[0]) || vecs[0].length === 0) {
+        throw new Error("Cohere returned empty embedding vector");
+      }
+      return vecs[0].length;
     }
     if (provider === "gemini") {
       const url = `https://generativelanguage.googleapis.com/v1/models/${model || "text-embedding-004"}:embedContent?key=${apiKey}`;
       const resp = await fetch(url, {
         method: "POST",
         headers: { "Content-Type": "application/json" },
-        body: JSON.stringify({ content: { parts: [{ text: "test" }] } }),
+        body: JSON.stringify({ content: { parts: [{ text: "test embedding vector" }] } }),
         signal: AbortSignal.timeout(15_000),
       });
       if (!resp.ok) {
         const txt = await resp.text();
         throw new Error(`Gemini embed ${resp.status}: ${txt}`);
       }
-      return;
+      const json = await resp.json() as any;
+      const vec = json?.embedding?.values;
+      if (!Array.isArray(vec) || vec.length === 0) {
+        throw new Error("Gemini returned empty embedding vector");
+      }
+      return vec.length;
     }
     const resp = await fetch(embUrl, {
       method: "POST",
       headers,
-      body: JSON.stringify({ input: ["test"], model: model || "text-embedding-3-small" }),
+      body: JSON.stringify({ input: ["test embedding vector"], model: model || "text-embedding-3-small" }),
       signal: AbortSignal.timeout(15_000),
     });
     if (!resp.ok) {
       const txt = await resp.text();
       throw new Error(`${resp.status}: ${txt}`);
     }
+    const json = await resp.json() as any;
+    const data = json?.data;
+    if (!Array.isArray(data) || data.length === 0) {
+      throw new Error("API returned no embedding data");
+    }
+    const vec = data[0]?.embedding;
+    if (!Array.isArray(vec) || vec.length === 0) {
+      throw new Error(`API returned empty embedding vector (got ${JSON.stringify(vec)?.slice(0, 100)})`);
+    }
+    return vec.length;
   }
 
   private async testChatModel(provider: string, model: string, endpoint: string, apiKey: string): Promise<void> {
@@ -1256,6 +1347,28 @@ export class ViewerServer {
     return path.join(home, ".openclaw");
   }
 
+  private handleCleanupPolluted(res: http.ServerResponse): void {
+    try {
+      const polluted = this.store.findPollutedUserChunks();
+      let deleted = 0;
+      for (const { id, reason } of polluted) {
+        if (this.store.deleteChunk(id)) {
+          deleted++;
+          this.log.info(`Cleaned polluted chunk ${id}: ${reason}`);
+        }
+      }
+      const fixed = this.store.fixMixedUserChunks();
+      this.log.info(`Cleanup: removed ${deleted} polluted, fixed ${fixed} mixed chunks`);
+      res.writeHead(200, { "Content-Type": "application/json" });
+      res.end(JSON.stringify({ deleted, fixed, total: polluted.length }));
+    } catch (err) {
+      const msg = err instanceof Error ? err.message : String(err);
+      this.log.error(`handleCleanupPolluted error: ${msg}`);
+      res.writeHead(500, { "Content-Type": "application/json" });
+      res.end(JSON.stringify({ error: msg }));
+    }
+  }
+
   private handleMigrateScan(res: http.ServerResponse): void {
     try {
       const ocHome = this.getOpenClawHome();
@@ -1499,7 +1612,6 @@ export class ViewerServer {
 
     const cfgPath = this.getOpenClawConfigPath();
     let summarizerCfg: any;
-    let strongCfg: any;
     try {
       const raw = JSON.parse(fs.readFileSync(cfgPath, "utf-8"));
       const pluginCfg = raw?.plugins?.entries?.["memos-local-openclaw-plugin"]?.config ??
@@ -1507,10 +1619,9 @@ export class ViewerServer {
                         raw?.plugins?.entries?.["memos-lite-openclaw-plugin"]?.config ??
                         raw?.plugins?.entries?.["memos-lite"]?.config ?? {};
       summarizerCfg = pluginCfg.summarizer;
-      strongCfg = pluginCfg.skillEvolution?.summarizer;
     } catch { /* no config */ }
 
-    const summarizer = new Summarizer(summarizerCfg, this.log, strongCfg);
+    const summarizer = new Summarizer(summarizerCfg, this.log);
 
     // Phase 1: Import SQLite memory chunks
     if (importSqlite) {
@@ -1636,8 +1747,8 @@ export class ViewerServer {
                   mergeCount: 0,
                   lastHitAt: null,
                   mergeHistory: "[]",
-                  createdAt: row.updated_at * 1000,
-                  updatedAt: row.updated_at * 1000,
+                  createdAt: normalizeTimestamp(row.updated_at),
+                  updatedAt: normalizeTimestamp(row.updated_at),
                 };
 
                 this.store.insertChunk(chunk);