@memtensor/memos-local-openclaw-plugin 0.1.0

package/src/storage/sqlite.ts

@@ -0,0 +1,305 @@
+import Database from "better-sqlite3";
+import * as fs from "fs";
+import * as path from "path";
+import type { Chunk, ChunkRef, Logger } from "../types";
+
+export class SqliteStore {
+  private db: Database.Database;
+
+  constructor(dbPath: string, private log: Logger) {
+    fs.mkdirSync(path.dirname(dbPath), { recursive: true });
+    this.db = new Database(dbPath);
+    this.db.pragma("journal_mode = WAL");
+    this.db.pragma("foreign_keys = ON");
+    this.migrate();
+  }
+
+  // ─── Schema ───
+
+  private migrate(): void {
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS chunks (
+        id          TEXT PRIMARY KEY,
+        session_key TEXT NOT NULL,
+        turn_id     TEXT NOT NULL,
+        seq         INTEGER NOT NULL,
+        role        TEXT NOT NULL,
+        content     TEXT NOT NULL,
+        kind        TEXT NOT NULL DEFAULT 'paragraph',
+        summary     TEXT NOT NULL DEFAULT '',
+        created_at  INTEGER NOT NULL,
+        updated_at  INTEGER NOT NULL
+      );
+
+      CREATE INDEX IF NOT EXISTS idx_chunks_session
+        ON chunks(session_key);
+      CREATE INDEX IF NOT EXISTS idx_chunks_turn
+        ON chunks(session_key, turn_id, seq);
+      CREATE INDEX IF NOT EXISTS idx_chunks_created
+        ON chunks(created_at);
+
+      CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
+        summary,
+        content,
+        content='chunks',
+        content_rowid='rowid',
+        tokenize='porter unicode61'
+      );
+
+      CREATE TRIGGER IF NOT EXISTS chunks_ai AFTER INSERT ON chunks BEGIN
+        INSERT INTO chunks_fts(rowid, summary, content)
+        VALUES (new.rowid, new.summary, new.content);
+      END;
+
+      CREATE TRIGGER IF NOT EXISTS chunks_ad AFTER DELETE ON chunks BEGIN
+        INSERT INTO chunks_fts(chunks_fts, rowid, summary, content)
+        VALUES ('delete', old.rowid, old.summary, old.content);
+      END;
+
+      CREATE TRIGGER IF NOT EXISTS chunks_au AFTER UPDATE ON chunks BEGIN
+        INSERT INTO chunks_fts(chunks_fts, rowid, summary, content)
+        VALUES ('delete', old.rowid, old.summary, old.content);
+        INSERT INTO chunks_fts(rowid, summary, content)
+        VALUES (new.rowid, new.summary, new.content);
+      END;
+
+      CREATE TABLE IF NOT EXISTS embeddings (
+        chunk_id   TEXT PRIMARY KEY REFERENCES chunks(id) ON DELETE CASCADE,
+        vector     BLOB NOT NULL,
+        dimensions INTEGER NOT NULL,
+        updated_at INTEGER NOT NULL
+      );
+    `);
+    this.log.debug("Database schema initialized");
+  }
+
+  // ─── Write ───
+
+  insertChunk(chunk: Chunk): void {
+    const stmt = this.db.prepare(`
+      INSERT OR REPLACE INTO chunks (id, session_key, turn_id, seq, role, content, kind, summary, created_at, updated_at)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `);
+    stmt.run(
+      chunk.id,
+      chunk.sessionKey,
+      chunk.turnId,
+      chunk.seq,
+      chunk.role,
+      chunk.content,
+      chunk.kind,
+      chunk.summary,
+      chunk.createdAt,
+      chunk.updatedAt,
+    );
+  }
+
+  updateSummary(chunkId: string, summary: string): void {
+    this.db.prepare("UPDATE chunks SET summary = ?, updated_at = ? WHERE id = ?").run(
+      summary,
+      Date.now(),
+      chunkId,
+    );
+  }
+
+  upsertEmbedding(chunkId: string, vector: number[]): void {
+    const buf = Buffer.from(new Float32Array(vector).buffer);
+    this.db.prepare(`
+      INSERT OR REPLACE INTO embeddings (chunk_id, vector, dimensions, updated_at)
+      VALUES (?, ?, ?, ?)
+    `).run(chunkId, buf, vector.length, Date.now());
+  }
+
+  // ─── Read ───
+
+  getChunk(chunkId: string): Chunk | null {
+    const row = this.db.prepare("SELECT * FROM chunks WHERE id = ?").get(chunkId) as ChunkRow | undefined;
+    return row ? rowToChunk(row) : null;
+  }
+
+  getChunksByRef(ref: ChunkRef): Chunk | null {
+    return this.getChunk(ref.chunkId);
+  }
+
+  getNeighborChunks(sessionKey: string, turnId: string, seq: number, window: number): Chunk[] {
+    const allRows = this.db.prepare(`
+      SELECT * FROM chunks
+      WHERE session_key = ?
+      ORDER BY created_at, seq
+    `).all(sessionKey) as ChunkRow[];
+
+    const targetIdx = allRows.findIndex(
+      (r) => r.turn_id === turnId && r.seq === seq,
+    );
+    if (targetIdx === -1) return [];
+
+    const radius = window * 3;
+    const start = Math.max(0, targetIdx - radius);
+    const end = Math.min(allRows.length, targetIdx + radius + 1);
+    return allRows.slice(start, end).map(rowToChunk);
+  }
+
+  // ─── FTS Search ───
+
+  ftsSearch(query: string, limit: number): Array<{ chunkId: string; score: number }> {
+    const sanitized = sanitizeFtsQuery(query);
+    if (!sanitized) return [];
+
+    try {
+      const rows = this.db.prepare(`
+        SELECT c.id as chunk_id, rank
+        FROM chunks_fts f
+        JOIN chunks c ON c.rowid = f.rowid
+        WHERE chunks_fts MATCH ?
+        ORDER BY rank
+        LIMIT ?
+      `).all(sanitized, limit) as Array<{ chunk_id: string; rank: number }>;
+
+      if (rows.length === 0) return [];
+      const maxAbsRank = Math.max(...rows.map((r) => Math.abs(r.rank)));
+      return rows.map((r) => ({
+        chunkId: r.chunk_id,
+        score: maxAbsRank > 0 ? Math.abs(r.rank) / maxAbsRank : 0,
+      }));
+    } catch {
+      this.log.warn(`FTS query failed for: "${sanitized}", returning empty`);
+      return [];
+    }
+  }
+
+  // ─── Vector Search ───
+
+  getAllEmbeddings(): Array<{ chunkId: string; vector: number[] }> {
+    const rows = this.db.prepare(
+      "SELECT chunk_id, vector, dimensions FROM embeddings",
+    ).all() as Array<{ chunk_id: string; vector: Buffer; dimensions: number }>;
+
+    return rows.map((r) => ({
+      chunkId: r.chunk_id,
+      vector: Array.from(new Float32Array(r.vector.buffer, r.vector.byteOffset, r.dimensions)),
+    }));
+  }
+
+  getEmbedding(chunkId: string): number[] | null {
+    const row = this.db.prepare(
+      "SELECT vector, dimensions FROM embeddings WHERE chunk_id = ?",
+    ).get(chunkId) as { vector: Buffer; dimensions: number } | undefined;
+    if (!row) return null;
+    return Array.from(new Float32Array(row.vector.buffer, row.vector.byteOffset, row.dimensions));
+  }
+
+  // ─── Update ───
+
+  updateChunk(chunkId: string, fields: { summary?: string; content?: string; role?: string; kind?: string }): boolean {
+    const sets: string[] = [];
+    const params: unknown[] = [];
+
+    if (fields.summary !== undefined) {
+      sets.push("summary = ?");
+      params.push(fields.summary);
+    }
+    if (fields.content !== undefined) {
+      sets.push("content = ?");
+      params.push(fields.content);
+    }
+    if (fields.role !== undefined) {
+      sets.push("role = ?");
+      params.push(fields.role);
+    }
+    if (fields.kind !== undefined) {
+      sets.push("kind = ?");
+      params.push(fields.kind);
+    }
+    if (sets.length === 0) return false;
+
+    sets.push("updated_at = ?");
+    params.push(Date.now());
+    params.push(chunkId);
+
+    const result = this.db.prepare(
+      `UPDATE chunks SET ${sets.join(", ")} WHERE id = ?`,
+    ).run(...params);
+    return result.changes > 0;
+  }
+
+  // ─── Delete ───
+
+  deleteChunk(chunkId: string): boolean {
+    const result = this.db.prepare("DELETE FROM chunks WHERE id = ?").run(chunkId);
+    return result.changes > 0;
+  }
+
+  deleteSession(sessionKey: string): number {
+    const result = this.db.prepare("DELETE FROM chunks WHERE session_key = ?").run(sessionKey);
+    return result.changes;
+  }
+
+  deleteAll(): number {
+    const result = this.db.prepare("DELETE FROM chunks").run();
+    return result.changes;
+  }
+
+  // ─── Util ───
+
+  getRecentChunkIds(limit: number): string[] {
+    const rows = this.db.prepare(
+      "SELECT id FROM chunks ORDER BY created_at DESC LIMIT ?",
+    ).all(limit) as Array<{ id: string }>;
+    return rows.map((r) => r.id);
+  }
+
+  close(): void {
+    this.db.close();
+  }
+}
+
+// ─── FTS helpers ───
+
+/**
+ * Sanitize user input for FTS5 MATCH queries.
+ * Strip FTS operators and special characters, then join tokens
+ * with implicit AND (space-separated) for safe querying.
+ */
+function sanitizeFtsQuery(raw: string): string {
+  const tokens = raw
+    .replace(/[."""(){}[\]*:^~!@#$%&\\/<>,;'`]/g, " ")
+    .split(/\s+/)
+    .map((t) => t.trim().replace(/^-+|-+$/g, ""))
+    .filter((t) => t.length > 1)
+    .filter((t) => !FTS_RESERVED.has(t.toUpperCase()));
+
+  return tokens.join(" ");
+}
+
+const FTS_RESERVED = new Set(["AND", "OR", "NOT", "NEAR"]);
+
+// ─── Internal helpers ───
+
+interface ChunkRow {
+  id: string;
+  session_key: string;
+  turn_id: string;
+  seq: number;
+  role: string;
+  content: string;
+  kind: string;
+  summary: string;
+  created_at: number;
+  updated_at: number;
+}
+
+function rowToChunk(row: ChunkRow): Chunk {
+  return {
+    id: row.id,
+    sessionKey: row.session_key,
+    turnId: row.turn_id,
+    seq: row.seq,
+    role: row.role as Chunk["role"],
+    content: row.content,
+    kind: row.kind as Chunk["kind"],
+    summary: row.summary,
+    embedding: null,
+    createdAt: row.created_at,
+    updatedAt: row.updated_at,
+  };
+}

package/src/storage/vector.ts

@@ -0,0 +1,39 @@
+import type { SqliteStore } from "./sqlite";
+
+export function cosineSimilarity(a: number[], b: number[]): number {
+  if (a.length !== b.length) return 0;
+  let dot = 0;
+  let normA = 0;
+  let normB = 0;
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i] * b[i];
+    normA += a[i] * a[i];
+    normB += b[i] * b[i];
+  }
+  const denom = Math.sqrt(normA) * Math.sqrt(normB);
+  return denom === 0 ? 0 : dot / denom;
+}
+
+export interface VectorHit {
+  chunkId: string;
+  score: number;
+}
+
+/**
+ * Brute-force vector search over all stored embeddings.
+ * For local single-user usage the dataset is small enough that
+ * a full scan with SIMD-friendly Float32 math is sufficient.
+ */
+export function vectorSearch(
+  store: SqliteStore,
+  queryVec: number[],
+  topK: number,
+): VectorHit[] {
+  const all = store.getAllEmbeddings();
+  const scored: VectorHit[] = all.map((row) => ({
+    chunkId: row.chunkId,
+    score: cosineSimilarity(queryVec, row.vector),
+  }));
+  scored.sort((a, b) => b.score - a.score);
+  return scored.slice(0, topK);
+}

package/src/tools/index.ts

@@ -0,0 +1,3 @@
+export { createMemorySearchTool } from "./memory-search";
+export { createMemoryTimelineTool } from "./memory-timeline";
+export { createMemoryGetTool } from "./memory-get";

package/src/tools/memory-get.ts

@@ -0,0 +1,68 @@
+import type { SqliteStore } from "../storage/sqlite";
+import type { ToolDefinition, GetResult, ChunkRef } from "../types";
+import { DEFAULTS } from "../types";
+
+export function createMemoryGetTool(store: SqliteStore): ToolDefinition {
+  return {
+    name: "memory_get",
+    description:
+      "Retrieve the full original text of a specific memory chunk. Use after memory_search or memory_timeline " +
+      "when you need to see the complete content (not just the excerpt). Useful for verifying exact details.",
+    inputSchema: {
+      type: "object",
+      properties: {
+        ref: {
+          type: "object",
+          description: "Reference object from a memory_search hit or memory_timeline entry.",
+          properties: {
+            sessionKey: { type: "string" },
+            chunkId: { type: "string" },
+            turnId: { type: "string" },
+            seq: { type: "number" },
+          },
+          required: ["sessionKey", "chunkId", "turnId", "seq"],
+        },
+        maxChars: {
+          type: "number",
+          description: `Maximum characters to return (default ${DEFAULTS.getMaxCharsDefault}, max ${DEFAULTS.getMaxCharsMax}).`,
+        },
+      },
+      required: ["ref"],
+    },
+    handler: async (input) => {
+      const ref = input.ref as ChunkRef;
+      const maxChars = Math.min(
+        (input.maxChars as number) ?? DEFAULTS.getMaxCharsDefault,
+        DEFAULTS.getMaxCharsMax,
+      );
+
+      const chunk = store.getChunksByRef(ref);
+
+      if (!chunk) {
+        return { error: `Chunk not found: ${ref.chunkId}` };
+      }
+
+      const content =
+        chunk.content.length > maxChars
+          ? chunk.content.slice(0, maxChars) + "…"
+          : chunk.content;
+
+      const result: GetResult = {
+        content,
+        ref: {
+          sessionKey: chunk.sessionKey,
+          chunkId: chunk.id,
+          turnId: chunk.turnId,
+          seq: chunk.seq,
+        },
+        source: {
+          ts: chunk.createdAt,
+          role: chunk.role,
+          sessionKey: chunk.sessionKey,
+        },
+      };
+
+      return result;
+    },
+  };
+}

package/src/tools/memory-search.ts

@@ -0,0 +1,36 @@
+import type { RecallEngine } from "../recall/engine";
+import type { ToolDefinition } from "../types";
+
+export function createMemorySearchTool(engine: RecallEngine): ToolDefinition {
+  return {
+    name: "memory_search",
+    description:
+      "Search stored conversation memories. Returns matching entries with summary, original_excerpt (evidence), score, and ref for follow-up with memory_timeline or memory_get. " +
+      "Default: top 6 results, minScore 0.45. Increase maxResults to 12/20 or lower minScore to 0.35 if initial results are insufficient.",
+    inputSchema: {
+      type: "object",
+      properties: {
+        query: {
+          type: "string",
+          description: "Natural language search query. Include specific entities, commands, or error messages for better recall.",
+        },
+        maxResults: {
+          type: "number",
+          description: "Maximum number of results (default 6, max 20).",
+        },
+        minScore: {
+          type: "number",
+          description: "Minimum relevance score threshold 0-1 (default 0.45, floor 0.35).",
+        },
+      },
+    },
+    handler: async (input) => {
+      const result = await engine.search({
+        query: (input.query as string) ?? "",
+        maxResults: input.maxResults as number | undefined,
+        minScore: input.minScore as number | undefined,
+      });
+      return result;
+    },
+  };
+}

package/src/tools/memory-timeline.ts

@@ -0,0 +1,73 @@
+import type { SqliteStore } from "../storage/sqlite";
+import type { ToolDefinition, TimelineResult, TimelineEntry, ChunkRef } from "../types";
+import { DEFAULTS } from "../types";
+
+export function createMemoryTimelineTool(store: SqliteStore): ToolDefinition {
+  return {
+    name: "memory_timeline",
+    description:
+      "Retrieve neighboring context around a memory reference. Use after memory_search to expand context " +
+      "around a specific hit. Provides adjacent conversation chunks marked as before/current/after.",
+    inputSchema: {
+      type: "object",
+      properties: {
+        ref: {
+          type: "object",
+          description: "Reference object from a memory_search hit (must contain sessionKey, chunkId, turnId, seq).",
+          properties: {
+            sessionKey: { type: "string" },
+            chunkId: { type: "string" },
+            turnId: { type: "string" },
+            seq: { type: "number" },
+          },
+          required: ["sessionKey", "chunkId", "turnId", "seq"],
+        },
+        window: {
+          type: "number",
+          description: "Number of turns/chunks to include before and after (default ±2).",
+        },
+      },
+      required: ["ref"],
+    },
+    handler: async (input) => {
+      const ref = input.ref as ChunkRef;
+      const window = (input.window as number) ?? DEFAULTS.timelineWindowDefault;
+
+      const neighbors = store.getNeighborChunks(
+        ref.sessionKey,
+        ref.turnId,
+        ref.seq,
+        window,
+      );
+
+      const entries: TimelineEntry[] = neighbors.map((chunk) => {
+        let relation: TimelineEntry["relation"] = "before";
+        if (chunk.id === ref.chunkId) {
+          relation = "current";
+        } else if (chunk.createdAt > (store.getChunk(ref.chunkId)?.createdAt ?? 0)) {
+          relation = "after";
+        }
+
+        return {
+          excerpt: chunk.content.slice(0, DEFAULTS.excerptMaxChars),
+          ref: {
+            sessionKey: chunk.sessionKey,
+            chunkId: chunk.id,
+            turnId: chunk.turnId,
+            seq: chunk.seq,
+          },
+          role: chunk.role,
+          ts: chunk.createdAt,
+          relation,
+        };
+      });
+
+      const result: TimelineResult = {
+        entries,
+        anchorRef: ref,
+      };
+
+      return result;
+    },
+  };
+}