agent-memory-store 0.0.4 → 0.0.6

package/src/db.js

@@ -0,0 +1,354 @@
+/**
+ * SQLite database layer powered by node:sqlite (built-in).
+ *
+ * Single-file database at <STORE_PATH>/store.db with WAL mode.
+ * FTS5 for full-text BM25 search, BLOB columns for vector embeddings.
+ * Zero external dependencies — uses Node.js native SQLite (>=22.5).
+ */
+
+import { DatabaseSync } from "node:sqlite";
+import { mkdirSync } from "fs";
+import path from "path";
+
+const STORE_PATH = process.env.AGENT_STORE_PATH
+  ? path.resolve(process.env.AGENT_STORE_PATH)
+  : path.join(process.cwd(), ".agent-memory-store");
+
+const DB_PATH = path.join(STORE_PATH, "store.db");
+
+let db = null;
+
+// ─── Schema ─────────────────────────────────────────────────────────────────
+
+const SCHEMA_TABLES = `
+CREATE TABLE IF NOT EXISTS chunks (
+  id         TEXT PRIMARY KEY,
+  topic      TEXT NOT NULL,
+  agent      TEXT NOT NULL DEFAULT 'global',
+  tags       TEXT NOT NULL DEFAULT '[]',
+  importance TEXT NOT NULL DEFAULT 'medium',
+  content    TEXT NOT NULL,
+  embedding  BLOB,
+  created_at TEXT NOT NULL,
+  updated_at TEXT NOT NULL,
+  expires_at TEXT
+);
+
+CREATE INDEX IF NOT EXISTS idx_chunks_agent   ON chunks(agent);
+CREATE INDEX IF NOT EXISTS idx_chunks_updated ON chunks(updated_at);
+CREATE INDEX IF NOT EXISTS idx_chunks_expires ON chunks(expires_at);
+
+CREATE TABLE IF NOT EXISTS state (
+  key        TEXT PRIMARY KEY,
+  value      TEXT NOT NULL,
+  updated_at TEXT NOT NULL
+);
+`;
+
+const SCHEMA_FTS = `
+CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
+  id UNINDEXED,
+  topic,
+  tags,
+  agent,
+  content,
+  content='chunks',
+  content_rowid=rowid
+);
+`;
+
+const SCHEMA_TRIGGERS = `
+CREATE TRIGGER IF NOT EXISTS chunks_ai AFTER INSERT ON chunks BEGIN
+  INSERT INTO chunks_fts(rowid, id, topic, tags, agent, content)
+  VALUES (new.rowid, new.id, new.topic, new.tags, new.agent, new.content);
+END;
+
+CREATE TRIGGER IF NOT EXISTS chunks_ad AFTER DELETE ON chunks BEGIN
+  INSERT INTO chunks_fts(chunks_fts, rowid, id, topic, tags, agent, content)
+  VALUES ('delete', old.rowid, old.id, old.topic, old.tags, old.agent, old.content);
+END;
+
+CREATE TRIGGER IF NOT EXISTS chunks_au AFTER UPDATE ON chunks BEGIN
+  INSERT INTO chunks_fts(chunks_fts, rowid, id, topic, tags, agent, content)
+  VALUES ('delete', old.rowid, old.id, old.topic, old.tags, old.agent, old.content);
+  INSERT INTO chunks_fts(rowid, id, topic, tags, agent, content)
+  VALUES (new.rowid, new.id, new.topic, new.tags, new.agent, new.content);
+END;
+`;
+
+// ─── Initialization ─────────────────────────────────────────────────────────
+
+/**
+ * Returns the database instance. Creates it on first call.
+ * Synchronous — node:sqlite DatabaseSync is synchronous by design.
+ */
+export function getDb() {
+  if (db) return db;
+
+  mkdirSync(STORE_PATH, { recursive: true });
+
+  db = new DatabaseSync(DB_PATH);
+
+  // WAL mode for better concurrent read performance
+  db.exec("PRAGMA journal_mode = WAL");
+
+  // Run schema
+  db.exec(SCHEMA_TABLES);
+  db.exec(SCHEMA_FTS);
+  db.exec(SCHEMA_TRIGGERS);
+
+  // Purge expired chunks
+  db.prepare(
+    `DELETE FROM chunks WHERE expires_at IS NOT NULL AND expires_at < datetime('now')`,
+  ).run();
+
+  // Graceful shutdown
+  const shutdown = () => {
+    if (db) db.close();
+    process.exit(0);
+  };
+  process.on("SIGINT", shutdown);
+  process.on("SIGTERM", shutdown);
+
+  return db;
+}
+
+// ─── CRUD Operations ────────────────────────────────────────────────────────
+
+/**
+ * Inserts or replaces a chunk in the database.
+ */
+export function insertChunk({
+  id,
+  topic,
+  agent,
+  tags,
+  importance,
+  content,
+  embedding,
+  createdAt,
+  updatedAt,
+  expiresAt,
+}) {
+  const d = getDb();
+  d.prepare(
+    `INSERT OR REPLACE INTO chunks (id, topic, agent, tags, importance, content, embedding, created_at, updated_at, expires_at)
+     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+  ).run(
+    id,
+    topic,
+    agent,
+    JSON.stringify(tags),
+    importance,
+    content,
+    embedding ? Buffer.from(embedding.buffer) : null,
+    createdAt,
+    updatedAt,
+    expiresAt,
+  );
+}
+
+/**
+ * Retrieves a single chunk by ID.
+ * @returns {object|null}
+ */
+export function getChunk(id) {
+  const d = getDb();
+  const row = d.prepare(`SELECT * FROM chunks WHERE id = ?`).get(id);
+  if (!row) return null;
+  return parseChunkRow(row);
+}
+
+/**
+ * Deletes a chunk by ID.
+ * @returns {boolean} true if a row was deleted
+ */
+export function deleteChunkById(id) {
+  const d = getDb();
+  const result = d.prepare(`DELETE FROM chunks WHERE id = ?`).run(id);
+  return result.changes > 0;
+}
+
+/**
+ * Lists chunk metadata, with optional agent/tags filters.
+ * Sorted by updated_at descending.
+ */
+export function listChunksDb({ agent, tags = [] } = {}) {
+  const d = getDb();
+  let sql = `SELECT id, topic, agent, tags, importance, updated_at FROM chunks`;
+  const conditions = [];
+  const params = [];
+
+  if (agent) {
+    conditions.push(`agent = ?`);
+    params.push(agent);
+  }
+
+  if (tags.length > 0) {
+    const tagConditions = tags.map(() => `tags LIKE ?`);
+    conditions.push(`(${tagConditions.join(" OR ")})`);
+    params.push(...tags.map((t) => `%"${t}"%`));
+  }
+
+  if (conditions.length) sql += ` WHERE ${conditions.join(" AND ")}`;
+  sql += ` ORDER BY updated_at DESC`;
+
+  const rows = d.prepare(sql).all(...params);
+  return rows.map((r) => ({
+    id: r.id,
+    topic: r.topic,
+    agent: r.agent,
+    tags: JSON.parse(r.tags),
+    importance: r.importance,
+    updated: r.updated_at,
+  }));
+}
+
+/**
+ * Full-text search via FTS5 (BM25).
+ * Returns ranked results with scores.
+ */
+export function searchFTS({ query, agent, tags = [], topK = 18 }) {
+  const d = getDb();
+
+  // Escape FTS5 special chars and build query
+  const ftsQuery = query
+    .replace(/["*^:(){}[\]]/g, " ")
+    .split(/\s+/)
+    .filter((t) => t.length > 1)
+    .join(" OR ");
+
+  if (!ftsQuery) return [];
+
+  let sql = `
+    SELECT chunks_fts.id, rank
+    FROM chunks_fts
+    JOIN chunks ON chunks.id = chunks_fts.id
+    WHERE chunks_fts MATCH ?`;
+  const params = [ftsQuery];
+
+  if (agent) {
+    sql += ` AND chunks.agent = ?`;
+    params.push(agent);
+  }
+
+  if (tags.length > 0) {
+    const tagConditions = tags.map(() => `chunks.tags LIKE ?`);
+    sql += ` AND (${tagConditions.join(" OR ")})`;
+    params.push(...tags.map((t) => `%"${t}"%`));
+  }
+
+  sql += ` ORDER BY rank LIMIT ?`;
+  params.push(topK);
+
+  const rows = d.prepare(sql).all(...params);
+  return rows.map((r) => ({
+    id: r.id,
+    score: -r.rank, // FTS5 rank is negative (lower = better), invert
+  }));
+}
+
+/**
+ * Retrieves all embeddings for vector search.
+ * @returns {Array<{ id: string, embedding: Float32Array }>}
+ */
+export function getAllEmbeddings({ agent, tags = [] } = {}) {
+  const d = getDb();
+  let sql = `SELECT id, embedding FROM chunks WHERE embedding IS NOT NULL`;
+  const params = [];
+
+  if (agent) {
+    sql += ` AND agent = ?`;
+    params.push(agent);
+  }
+
+  if (tags.length > 0) {
+    const tagConditions = tags.map(() => `tags LIKE ?`);
+    sql += ` AND (${tagConditions.join(" OR ")})`;
+    params.push(...tags.map((t) => `%"${t}"%`));
+  }
+
+  const rows = d.prepare(sql).all(...params);
+  return rows
+    .filter((r) => r.embedding !== null)
+    .map((r) => ({
+      id: r.id,
+      embedding: new Float32Array(
+        r.embedding.buffer,
+        r.embedding.byteOffset,
+        r.embedding.byteLength / 4,
+      ),
+    }));
+}
+
+/**
+ * Updates only the embedding for a chunk.
+ */
+export function updateEmbedding(id, embedding) {
+  const d = getDb();
+  d.prepare(`UPDATE chunks SET embedding = ? WHERE id = ?`).run(
+    Buffer.from(embedding.buffer),
+    id,
+  );
+}
+
+/**
+ * Returns chunks that have no embedding yet.
+ */
+export function getChunksWithoutEmbedding() {
+  const d = getDb();
+  return d
+    .prepare(
+      `SELECT id, topic, tags, content FROM chunks WHERE embedding IS NULL`,
+    )
+    .all()
+    .map((r) => ({
+      id: r.id,
+      topic: r.topic,
+      tags: r.tags,
+      content: r.content,
+    }));
+}
+
+// ─── State Operations ───────────────────────────────────────────────────────
+
+export function getStateDb(key) {
+  const d = getDb();
+  const row = d.prepare(`SELECT value FROM state WHERE key = ?`).get(key);
+  if (!row) return null;
+  return JSON.parse(row.value);
+}
+
+export function setStateDb(key, value) {
+  const d = getDb();
+  const updatedAt = new Date().toISOString();
+  d.prepare(
+    `INSERT OR REPLACE INTO state (key, value, updated_at) VALUES (?, ?, ?)`,
+  ).run(key, JSON.stringify(value), updatedAt);
+  return { key, updated: updatedAt };
+}
+
+// ─── Helpers ────────────────────────────────────────────────────────────────
+
+function parseChunkRow(row) {
+  return {
+    id: row.id,
+    topic: row.topic,
+    agent: row.agent,
+    tags: JSON.parse(row.tags),
+    importance: row.importance,
+    content: row.content,
+    embedding: row.embedding
+      ? new Float32Array(
+          row.embedding.buffer,
+          row.embedding.byteOffset,
+          row.embedding.byteLength / 4,
+        )
+      : null,
+    createdAt: row.created_at,
+    updatedAt: row.updated_at,
+    expiresAt: row.expires_at,
+  };
+}
+
+export { STORE_PATH };

package/src/embeddings.js

@@ -0,0 +1,124 @@
+/**
+ * Local embedding generation via @huggingface/transformers.
+ *
+ * Uses the all-MiniLM-L6-v2 model (384 dimensions) running locally via ONNX Runtime.
+ * Model is auto-downloaded (~23MB) on first use and cached in ~/.cache/huggingface/.
+ *
+ * Graceful degradation: if the model fails to load, all functions return null
+ * and the system falls back to BM25-only search.
+ */
+
+let pipelineInstance = null;
+let loadFailed = false;
+let loadingPromise = null;
+
+/**
+ * Lazily initializes the feature-extraction pipeline.
+ * Returns null if the model cannot be loaded.
+ * Ensures only one load attempt runs at a time.
+ */
+async function getPipeline() {
+  if (pipelineInstance) return pipelineInstance;
+  if (loadFailed) return null;
+
+  // Deduplicate concurrent load attempts
+  if (loadingPromise) return loadingPromise;
+
+  loadingPromise = (async () => {
+    try {
+      process.stderr.write(
+        "[agent-memory-store] Loading embedding model (first run downloads ~23MB)...\n",
+      );
+      const { pipeline } = await import("@huggingface/transformers");
+      pipelineInstance = await pipeline(
+        "feature-extraction",
+        "Xenova/all-MiniLM-L6-v2",
+        { dtype: "fp32" },
+      );
+      process.stderr.write(
+        "[agent-memory-store] Embedding model loaded successfully.\n",
+      );
+      return pipelineInstance;
+    } catch (err) {
+      loadFailed = true;
+      process.stderr.write(
+        `[agent-memory-store] Embedding model failed to load: ${err.message}\n` +
+          `[agent-memory-store] Falling back to BM25-only search.\n`,
+      );
+      return null;
+    } finally {
+      loadingPromise = null;
+    }
+  })();
+
+  return loadingPromise;
+}
+
+/**
+ * Generates an embedding for a single text string.
+ *
+ * @param {string} text - Text to embed (topic + tags + content)
+ * @returns {Promise<Float32Array|null>} 384-dim embedding or null if unavailable
+ */
+export async function embed(text) {
+  const extractor = await getPipeline();
+  if (!extractor) return null;
+
+  try {
+    const output = await extractor(text, {
+      pooling: "mean",
+      normalize: true,
+    });
+    return new Float32Array(output.data);
+  } catch (err) {
+    process.stderr.write(
+      `[agent-memory-store] Embedding error: ${err.message}\n`,
+    );
+    return null;
+  }
+}
+
+/**
+ * Generates embeddings for multiple texts.
+ *
+ * @param {string[]} texts
+ * @returns {Promise<Array<Float32Array|null>>}
+ */
+export async function embedBatch(texts) {
+  const results = [];
+  for (const text of texts) {
+    results.push(await embed(text));
+  }
+  return results;
+}
+
+/**
+ * Prepares searchable text from chunk fields for embedding.
+ *
+ * @param {object} chunk
+ * @param {string} chunk.topic
+ * @param {string[]|string} chunk.tags
+ * @param {string} chunk.content
+ * @returns {string}
+ */
+export function prepareText({ topic, tags, content }) {
+  const tagStr = Array.isArray(tags) ? tags.join(" ") : tags || "";
+  // Truncate content to ~800 chars to stay within model token limit
+  const truncated = content.length > 800 ? content.slice(0, 800) : content;
+  return `${topic} ${tagStr} ${truncated}`.trim();
+}
+
+/**
+ * Returns whether the embedding model is available.
+ */
+export function isEmbeddingAvailable() {
+  return pipelineInstance !== null && !loadFailed;
+}
+
+/**
+ * Pre-warms the embedding model (call during startup).
+ * Non-blocking — failures are silently handled.
+ */
+export async function warmup() {
+  await getPipeline();
+}

package/src/index.js

@@ -1,9 +1,9 @@
 #!/usr/bin/env node
 /**
- * agent-store MCP server entry point.
+ * agent-memory-store MCP server entry point.
  *
  * Exposes 7 tools to any MCP-compatible client (Claude Code, opencode, etc.):
- *   search_context  — BM25 full-text search over stored chunks
+ *   search_context  — Hybrid search (BM25 + semantic) over stored chunks
  *   write_context   — persist a new memory chunk
  *   read_context    — retrieve a chunk by ID
  *   list_context    — list chunk metadata (no body)
@@ -12,8 +12,8 @@
  *   set_state       — write a session state variable
  *
  * Usage:
- *   npx @agentops/context-store
- *   CONTEXT_STORE_PATH=/your/project/.context npx @agentops/context-store
+ *   npx agent-memory-store
+ *   AGENT_STORE_PATH=/your/project/.agent-memory-store npx agent-memory-store
  */
 
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
@@ -27,6 +27,7 @@ import {
   listChunks,
   getState,
   setState,
+  initStore,
 } from "./store.js";
 
 const { version } = JSON.parse(
@@ -35,6 +36,9 @@ const { version } = JSON.parse(
   ),
 );
 
+// Initialize database, run migration, warm up embeddings
+await initStore();
+
 const server = new McpServer({
   name: "context-store",
   version,
@@ -45,9 +49,10 @@ const server = new McpServer({
 server.tool(
   "search_context",
   [
-    "Search stored memory chunks by relevance using BM25 full-text ranking.",
+    "Search stored memory chunks using hybrid ranking (BM25 + semantic similarity).",
     "Call this at the start of any task to retrieve relevant prior knowledge,",
     "decisions, and outputs before generating a response.",
+    "Supports three modes: 'hybrid' (default, best quality), 'bm25' (keyword-only), 'semantic' (meaning-only).",
   ].join(" "),
   {
     query: z
@@ -75,16 +80,23 @@ server.tool(
       .min(0)
       .optional()
       .describe(
-        "Minimum BM25 relevance score. Lower = more permissive (default: 0.1).",
+        "Minimum relevance score. Lower = more permissive (default: 0.1).",
+      ),
+    search_mode: z
+      .enum(["hybrid", "bm25", "semantic"])
+      .optional()
+      .describe(
+        "Search strategy: 'hybrid' (BM25 + semantic, default), 'bm25' (keyword only), 'semantic' (embedding similarity only).",
       ),
   },
-  async ({ query, tags, agent, top_k, min_score }) => {
+  async ({ query, tags, agent, top_k, min_score, search_mode }) => {
     const results = await searchChunks({
       query,
       tags: tags ?? [],
       agent,
       topK: top_k ?? 6,
       minScore: min_score ?? 0.1,
+      mode: search_mode ?? "hybrid",
     });
 
     if (results.length === 0) {
@@ -111,9 +123,10 @@ server.tool(
 server.tool(
   "write_context",
   [
-    "Persist a memory chunk to local storage.",
+    "Persist a memory chunk to the database.",
     "Call this after completing a subtask, making a key decision,",
     "or producing output that downstream agents will need.",
+    "Embeddings are computed automatically in the background for semantic search.",
   ].join(" "),
   {
     topic: z

package/src/migrate.js

@@ -0,0 +1,124 @@
+/**
+ * Migration: filesystem-based storage → SQLite database.
+ *
+ * Runs automatically on first startup if the legacy chunks/ directory exists
+ * but store.db does not. Migrates all chunks and state, then renames the
+ * legacy directories to *_backup/.
+ */
+
+import fs from "fs/promises";
+import path from "path";
+import matter from "gray-matter";
+import { insertChunk, setStateDb, STORE_PATH } from "./db.js";
+
+const CHUNKS_DIR = path.join(STORE_PATH, "chunks");
+const STATE_DIR = path.join(STORE_PATH, "state");
+const DB_PATH = path.join(STORE_PATH, "store.db");
+
+/**
+ * Checks if migration is needed and runs it.
+ * @returns {Promise<boolean>} true if migration was performed
+ */
+export async function migrateIfNeeded() {
+  // Check if legacy chunks dir exists
+  const chunksExist = await fs
+    .stat(CHUNKS_DIR)
+    .then((s) => s.isDirectory())
+    .catch(() => false);
+
+  if (!chunksExist) return false;
+
+  // Check if DB already exists (already migrated)
+  const dbExists = await fs
+    .stat(DB_PATH)
+    .then((s) => s.isFile())
+    .catch(() => false);
+
+  if (dbExists) return false;
+
+  process.stderr.write(
+    "[agent-memory-store] Migrating filesystem storage to SQLite...\n",
+  );
+
+  let chunkCount = 0;
+  let stateCount = 0;
+
+  // Migrate chunks
+  try {
+    const files = await fs.readdir(CHUNKS_DIR);
+    for (const file of files) {
+      if (!file.endsWith(".md")) continue;
+      try {
+        const raw = await fs.readFile(path.join(CHUNKS_DIR, file), "utf8");
+        const { data: meta, content } = matter(raw);
+
+        // Skip expired chunks
+        if (meta.expires && new Date(meta.expires) < new Date()) continue;
+
+        const now = new Date().toISOString();
+        await insertChunk({
+          id: meta.id || file.replace(".md", ""),
+          topic: meta.topic || "Untitled",
+          agent: meta.agent || "global",
+          tags: meta.tags || [],
+          importance: meta.importance || "medium",
+          content: content.trim(),
+          embedding: null, // Will be computed in background
+          createdAt: meta.updated || now,
+          updatedAt: meta.updated || now,
+          expiresAt: meta.expires || null,
+        });
+        chunkCount++;
+      } catch {
+        // Skip unreadable files
+      }
+    }
+  } catch {
+    // chunks dir not readable
+  }
+
+  // Migrate state
+  try {
+    const files = await fs.readdir(STATE_DIR);
+    for (const file of files) {
+      if (!file.endsWith(".json")) continue;
+      try {
+        const raw = await fs.readFile(path.join(STATE_DIR, file), "utf8");
+        const { key, value } = JSON.parse(raw);
+        if (key) {
+          await setStateDb(key, value);
+          stateCount++;
+        }
+      } catch {
+        // Skip unreadable files
+      }
+    }
+  } catch {
+    // state dir not readable
+  }
+
+  // Rename legacy directories to backups
+  try {
+    await fs.rename(CHUNKS_DIR, CHUNKS_DIR + "_backup");
+  } catch {
+    // Rename failed — not critical
+  }
+
+  try {
+    const stateExists = await fs
+      .stat(STATE_DIR)
+      .then((s) => s.isDirectory())
+      .catch(() => false);
+    if (stateExists) {
+      await fs.rename(STATE_DIR, STATE_DIR + "_backup");
+    }
+  } catch {
+    // Rename failed — not critical
+  }
+
+  process.stderr.write(
+    `[agent-memory-store] Migration complete: ${chunkCount} chunks, ${stateCount} state entries.\n`,
+  );
+
+  return true;
+}