@memtensor/memos-local-openclaw-plugin 0.1.2 → 0.1.4

package/src/storage/sqlite.ts

@@ -1,7 +1,8 @@
 import Database from "better-sqlite3";
+import { createHash } from "crypto";
 import * as fs from "fs";
 import * as path from "path";
-import type { Chunk, ChunkRef, Logger } from "../types";
+import type { Chunk, ChunkRef, Task, TaskStatus, Logger } from "../types";
 
 export class SqliteStore {
   private db: Database.Database;
@@ -69,16 +70,140 @@ export class SqliteStore {
         dimensions INTEGER NOT NULL,
         updated_at INTEGER NOT NULL
       );
+
+      CREATE TABLE IF NOT EXISTS viewer_events (
+        id         INTEGER PRIMARY KEY AUTOINCREMENT,
+        event_type TEXT NOT NULL,
+        created_at INTEGER NOT NULL
+      );
+      CREATE INDEX IF NOT EXISTS idx_viewer_events_created ON viewer_events(created_at);
+      CREATE INDEX IF NOT EXISTS idx_viewer_events_type ON viewer_events(event_type);
+
+      CREATE TABLE IF NOT EXISTS tasks (
+        id          TEXT PRIMARY KEY,
+        session_key TEXT NOT NULL,
+        title       TEXT NOT NULL DEFAULT '',
+        summary     TEXT NOT NULL DEFAULT '',
+        status      TEXT NOT NULL DEFAULT 'active',
+        started_at  INTEGER NOT NULL,
+        ended_at    INTEGER,
+        updated_at  INTEGER NOT NULL
+      );
+      CREATE INDEX IF NOT EXISTS idx_tasks_session ON tasks(session_key);
+      CREATE INDEX IF NOT EXISTS idx_tasks_status ON tasks(status);
     `);
+
+    this.migrateTaskId();
+    this.migrateContentHash();
     this.log.debug("Database schema initialized");
   }
 
+  private migrateTaskId(): void {
+    const cols = this.db.prepare("PRAGMA table_info(chunks)").all() as Array<{ name: string }>;
+    if (!cols.some((c) => c.name === "task_id")) {
+      this.db.exec("ALTER TABLE chunks ADD COLUMN task_id TEXT REFERENCES tasks(id)");
+      this.db.exec("CREATE INDEX IF NOT EXISTS idx_chunks_task ON chunks(task_id)");
+      this.log.info("Migrated: added task_id column to chunks");
+    }
+  }
+
+  private migrateContentHash(): void {
+    const cols = this.db.prepare("PRAGMA table_info(chunks)").all() as Array<{ name: string }>;
+    if (!cols.some((c) => c.name === "content_hash")) {
+      this.db.exec("ALTER TABLE chunks ADD COLUMN content_hash TEXT");
+      this.db.exec("CREATE INDEX IF NOT EXISTS idx_chunks_dedup ON chunks(session_key, role, content_hash)");
+
+      // Backfill existing rows
+      const rows = this.db.prepare("SELECT id, content FROM chunks WHERE content_hash IS NULL").all() as Array<{ id: string; content: string }>;
+      const updateStmt = this.db.prepare("UPDATE chunks SET content_hash = ? WHERE id = ?");
+      for (const r of rows) {
+        updateStmt.run(contentHash(r.content), r.id);
+      }
+      if (rows.length > 0) {
+        this.log.info(`Migrated: backfilled content_hash for ${rows.length} chunks`);
+      }
+    }
+  }
+
+  /** Record a viewer API call for analytics (list, search, etc.). */
+  recordViewerEvent(eventType: string): void {
+    this.db.prepare("INSERT INTO viewer_events (event_type, created_at) VALUES (?, ?)").run(eventType, Date.now());
+  }
+
+  /**
+   * Return metrics for the last N days: writes per day (from chunks), viewer calls per day.
+   */
+  getMetrics(days: number): {
+    writesPerDay: Array<{ date: string; count: number }>;
+    viewerCallsPerDay: Array<{ date: string; list: number; search: number; total: number }>;
+    roleBreakdown: Record<string, number>;
+    kindBreakdown: Record<string, number>;
+    totals: { memories: number; sessions: number; embeddings: number; todayWrites: number; todayViewerCalls: number };
+  } {
+    const since = Date.now() - days * 86400 * 1000;
+    const now = new Date();
+    const todayStart = new Date(now.getFullYear(), now.getMonth(), now.getDate()).getTime();
+
+    const writesRows = this.db
+      .prepare(
+        `SELECT date(created_at/1000, 'unixepoch', 'localtime') as d, COUNT(*) as c
+       FROM chunks WHERE created_at >= ? GROUP BY d ORDER BY d`,
+      )
+      .all(since) as Array<{ d: string; c: number }>;
+    const writesPerDay = writesRows.map((r) => ({ date: r.d, count: r.c }));
+
+    const eventsRows = this.db
+      .prepare(
+        `SELECT date(created_at/1000, 'unixepoch', 'localtime') as d, event_type, COUNT(*) as c
+       FROM viewer_events WHERE created_at >= ? GROUP BY d, event_type ORDER BY d`,
+      )
+      .all(since) as Array<{ d: string; event_type: string; c: number }>;
+    const byDate = new Map<string, { list: number; search: number }>();
+    for (const r of eventsRows) {
+      let row = byDate.get(r.d);
+      if (!row) {
+        row = { list: 0, search: 0 };
+        byDate.set(r.d, row);
+      }
+      if (r.event_type === "list") row.list += r.c;
+      else if (r.event_type === "search") row.search += r.c;
+    }
+    const viewerCallsPerDay = Array.from(byDate.entries())
+      .sort((a, b) => a[0].localeCompare(b[0]))
+      .map(([date, v]) => ({ date, list: v.list, search: v.search, total: v.list + v.search }));
+
+    const roles = this.db.prepare("SELECT role, COUNT(*) as count FROM chunks GROUP BY role").all() as Array<{ role: string; count: number }>;
+    const kinds = this.db.prepare("SELECT kind, COUNT(*) as count FROM chunks GROUP BY kind").all() as Array<{ kind: string; count: number }>;
+    const roleBreakdown = Object.fromEntries(roles.map((r) => [r.role, r.count]));
+    const kindBreakdown = Object.fromEntries(kinds.map((k) => [k.kind, k.count]));
+
+    const totalChunks = (this.db.prepare("SELECT COUNT(*) as c FROM chunks").get() as { c: number }).c;
+    const totalSessions = (this.db.prepare("SELECT COUNT(DISTINCT session_key) as c FROM chunks").get() as { c: number }).c;
+    const totalEmbeddings = (this.db.prepare("SELECT COUNT(*) as c FROM embeddings").get() as { c: number }).c;
+    const todayWrites = (this.db.prepare("SELECT COUNT(*) as c FROM chunks WHERE created_at >= ?").get(todayStart) as { c: number }).c;
+    const todayViewerCalls = (this.db.prepare("SELECT COUNT(*) as c FROM viewer_events WHERE created_at >= ?").get(todayStart) as { c: number }).c;
+
+    return {
+      writesPerDay,
+      viewerCallsPerDay,
+      roleBreakdown,
+      kindBreakdown,
+      totals: {
+        memories: totalChunks,
+        sessions: totalSessions,
+        embeddings: totalEmbeddings,
+        todayWrites,
+        todayViewerCalls,
+      },
+    };
+  }
+
   // ─── Write ───
 
   insertChunk(chunk: Chunk): void {
     const stmt = this.db.prepare(`
-      INSERT OR REPLACE INTO chunks (id, session_key, turn_id, seq, role, content, kind, summary, created_at, updated_at)
-      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      INSERT OR REPLACE INTO chunks (id, session_key, turn_id, seq, role, content, kind, summary, task_id, content_hash, created_at, updated_at)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
     `);
     stmt.run(
       chunk.id,
@@ -89,6 +214,8 @@ export class SqliteStore {
       chunk.content,
       chunk.kind,
       chunk.summary,
+      chunk.taskId,
+      contentHash(chunk.content),
       chunk.createdAt,
       chunk.updatedAt,
     );
@@ -167,6 +294,39 @@ export class SqliteStore {
     }
   }
 
+  // ─── Pattern Search (LIKE-based, for CJK text where FTS tokenization is weak) ───
+
+  patternSearch(patterns: string[], opts: { role?: string; limit?: number } = {}): Array<{ chunkId: string; content: string; role: string; createdAt: number }> {
+    if (patterns.length === 0) return [];
+    const limit = opts.limit ?? 10;
+
+    const conditions = patterns.map(() => "c.content LIKE ?");
+    const whereClause = conditions.join(" OR ");
+    const roleClause = opts.role ? " AND c.role = ?" : "";
+    const params: (string | number)[] = patterns.map(p => `%${p}%`);
+    if (opts.role) params.push(opts.role);
+    params.push(limit);
+
+    try {
+      const rows = this.db.prepare(`
+        SELECT c.id as chunk_id, c.content, c.role, c.created_at
+        FROM chunks c
+        WHERE (${whereClause})${roleClause}
+        ORDER BY c.created_at DESC
+        LIMIT ?
+      `).all(...params) as Array<{ chunk_id: string; content: string; role: string; created_at: number }>;
+
+      return rows.map(r => ({
+        chunkId: r.chunk_id,
+        content: r.content,
+        role: r.role,
+        createdAt: r.created_at,
+      }));
+    } catch {
+      return [];
+    }
+  }
+
   // ─── Vector Search ───
 
   getAllEmbeddings(): Array<{ chunkId: string; vector: number[] }> {
@@ -235,8 +395,106 @@ export class SqliteStore {
   }
 
   deleteAll(): number {
-    const result = this.db.prepare("DELETE FROM chunks").run();
-    return result.changes;
+    this.db.prepare("DELETE FROM chunks").run();
+    this.db.prepare("DELETE FROM tasks").run();
+    this.db.prepare("DELETE FROM viewer_events").run();
+    const remaining = this.countChunks();
+    return remaining === 0 ? 1 : 0;
+  }
+
+  // ─── Task CRUD ───
+
+  insertTask(task: Task): void {
+    this.db.prepare(`
+      INSERT OR REPLACE INTO tasks (id, session_key, title, summary, status, started_at, ended_at, updated_at)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+    `).run(task.id, task.sessionKey, task.title, task.summary, task.status, task.startedAt, task.endedAt, task.updatedAt);
+  }
+
+  getTask(taskId: string): Task | null {
+    const row = this.db.prepare("SELECT * FROM tasks WHERE id = ?").get(taskId) as TaskRow | undefined;
+    return row ? rowToTask(row) : null;
+  }
+
+  getActiveTask(sessionKey: string): Task | null {
+    const row = this.db.prepare(
+      "SELECT * FROM tasks WHERE session_key = ? AND status = 'active' ORDER BY started_at DESC LIMIT 1",
+    ).get(sessionKey) as TaskRow | undefined;
+    return row ? rowToTask(row) : null;
+  }
+
+  getAllActiveTasks(): Task[] {
+    const rows = this.db.prepare(
+      "SELECT * FROM tasks WHERE status = 'active' ORDER BY started_at DESC",
+    ).all() as TaskRow[];
+    return rows.map(rowToTask);
+  }
+
+  updateTask(taskId: string, fields: { title?: string; summary?: string; status?: TaskStatus; endedAt?: number }): boolean {
+    const sets: string[] = [];
+    const params: unknown[] = [];
+    if (fields.title !== undefined) { sets.push("title = ?"); params.push(fields.title); }
+    if (fields.summary !== undefined) { sets.push("summary = ?"); params.push(fields.summary); }
+    if (fields.status !== undefined) { sets.push("status = ?"); params.push(fields.status); }
+    if (fields.endedAt !== undefined) { sets.push("ended_at = ?"); params.push(fields.endedAt); }
+    if (sets.length === 0) return false;
+    sets.push("updated_at = ?");
+    params.push(Date.now());
+    params.push(taskId);
+    const result = this.db.prepare(`UPDATE tasks SET ${sets.join(", ")} WHERE id = ?`).run(...params);
+    return result.changes > 0;
+  }
+
+  getChunksByTask(taskId: string): Chunk[] {
+    const rows = this.db.prepare("SELECT * FROM chunks WHERE task_id = ? ORDER BY created_at, seq").all(taskId) as ChunkRow[];
+    return rows.map(rowToChunk);
+  }
+
+  listTasks(opts: { status?: string; limit?: number; offset?: number } = {}): { tasks: Task[]; total: number } {
+    const conditions: string[] = [];
+    const params: unknown[] = [];
+    if (opts.status) { conditions.push("status = ?"); params.push(opts.status); }
+    const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
+
+    const countRow = this.db.prepare(`SELECT COUNT(*) as c FROM tasks ${whereClause}`).get(...params) as { c: number };
+    const total = countRow.c;
+
+    const limit = opts.limit ?? 50;
+    const offset = opts.offset ?? 0;
+    const rows = this.db.prepare(
+      `SELECT * FROM tasks ${whereClause} ORDER BY started_at DESC LIMIT ? OFFSET ?`,
+    ).all(...params, limit, offset) as TaskRow[];
+
+    return { tasks: rows.map(rowToTask), total };
+  }
+
+  countChunksByTask(taskId: string): number {
+    const row = this.db.prepare("SELECT COUNT(*) as c FROM chunks WHERE task_id = ?").get(taskId) as { c: number };
+    return row.c;
+  }
+
+  setChunkTaskId(chunkId: string, taskId: string): void {
+    this.db.prepare("UPDATE chunks SET task_id = ?, updated_at = ? WHERE id = ?").run(taskId, Date.now(), chunkId);
+  }
+
+  getUnassignedChunks(sessionKey: string): Chunk[] {
+    const rows = this.db.prepare(
+      "SELECT * FROM chunks WHERE session_key = ? AND task_id IS NULL ORDER BY created_at, seq",
+    ).all(sessionKey) as ChunkRow[];
+    return rows.map(rowToChunk);
+  }
+
+  /**
+   * Check if a chunk with the same (session_key, role, content_hash) already exists.
+   * Uses indexed content_hash for O(1) lookup to prevent duplicate ingestion
+   * when agent_end sends the full conversation history every turn.
+   */
+  chunkExistsByContent(sessionKey: string, role: string, content: string): boolean {
+    const hash = contentHash(content);
+    const row = this.db.prepare(
+      "SELECT 1 FROM chunks WHERE session_key = ? AND role = ? AND content_hash = ? LIMIT 1",
+    ).get(sessionKey, role, hash);
+    return !!row;
   }
 
   // ─── Util ───
@@ -248,6 +506,11 @@ export class SqliteStore {
     return rows.map((r) => r.id);
   }
 
+  countChunks(): number {
+    const row = this.db.prepare("SELECT COUNT(*) AS cnt FROM chunks").get() as { cnt: number };
+    return row.cnt;
+  }
+
   close(): void {
     this.db.close();
   }
@@ -284,6 +547,7 @@ interface ChunkRow {
   content: string;
   kind: string;
   summary: string;
+  task_id: string | null;
   created_at: number;
   updated_at: number;
 }
@@ -299,7 +563,36 @@ function rowToChunk(row: ChunkRow): Chunk {
     kind: row.kind as Chunk["kind"],
     summary: row.summary,
     embedding: null,
+    taskId: row.task_id,
     createdAt: row.created_at,
     updatedAt: row.updated_at,
   };
 }
+
+interface TaskRow {
+  id: string;
+  session_key: string;
+  title: string;
+  summary: string;
+  status: string;
+  started_at: number;
+  ended_at: number | null;
+  updated_at: number;
+}
+
+function rowToTask(row: TaskRow): Task {
+  return {
+    id: row.id,
+    sessionKey: row.session_key,
+    title: row.title,
+    summary: row.summary,
+    status: row.status as Task["status"],
+    startedAt: row.started_at,
+    endedAt: row.ended_at,
+    updatedAt: row.updated_at,
+  };
+}
+
+function contentHash(content: string): string {
+  return createHash("sha256").update(content).digest("hex").slice(0, 16);
+}

package/src/types.ts

@@ -23,10 +23,26 @@ export interface Chunk {
   kind: ChunkKind;
   summary: string;
   embedding: number[] | null;
+  taskId: string | null;
   createdAt: number;
   updatedAt: number;
 }
 
+// ─── Task ───
+
+export type TaskStatus = "active" | "completed" | "skipped";
+
+export interface Task {
+  id: string;
+  sessionKey: string;
+  title: string;
+  summary: string;
+  status: TaskStatus;
+  startedAt: number;
+  endedAt: number | null;
+  updatedAt: number;
+}
+
 export type ChunkKind =
   | "paragraph"
   | "code_block"
@@ -50,6 +66,7 @@ export interface SearchHit {
   original_excerpt: string;
   ref: ChunkRef;
   score: number;
+  taskId: string | null;
   source: {
     ts: number;
     role: Role;
@@ -188,6 +205,8 @@ export const DEFAULTS = {
   localEmbeddingModel: "Xenova/all-MiniLM-L6-v2",
   localEmbeddingDimensions: 384,
   toolResultMaxChars: 2000,
+  taskIdleTimeoutMs: 2 * 60 * 60 * 1000, // 2 hour gap → new task
+  taskSummaryMaxTokens: 2000,
 } as const;
 
 // ─── Plugin Hooks (OpenClaw integration) ───