@modusensus/dsh-mneme 0.1.4 → 0.1.5

package/lib/store.js

@@ -12,6 +12,7 @@ CREATE TABLE IF NOT EXISTS memories (
   forgotten   INTEGER NOT NULL DEFAULT 0,
   archived    INTEGER NOT NULL DEFAULT 0,
   source      TEXT,
+  embedding   TEXT,
   created_at  TEXT NOT NULL,
   updated_at  TEXT NOT NULL
 );
@@ -64,11 +65,14 @@ export function createStore(path) {
   db.exec("PRAGMA journal_mode = WAL;");
   db.exec(SCHEMA);
 
-  // Schema migration: add archived column to legacy databases (idempotent)
+  // Schema migrations for legacy databases (idempotent).
   const columns = db.prepare("PRAGMA table_info(memories)").all().map((c) => c.name);
   if (!columns.includes("archived")) {
     db.exec("ALTER TABLE memories ADD COLUMN archived INTEGER NOT NULL DEFAULT 0");
   }
+  if (!columns.includes("embedding")) {
+    db.exec("ALTER TABLE memories ADD COLUMN embedding TEXT");
+  }
 
   // Per-instance monotonic timestamp guard: consecutive writes within the same
   // millisecond must still produce strictly increasing timestamps (test asserts
@@ -117,10 +121,13 @@ export function createStore(path) {
     const now = nowIso();
     const tags = JSON.stringify(memory.tags ?? []);
     const importance = Number.isInteger(memory.importance) ? memory.importance : 3;
+    const embedding = Array.isArray(memory.embedding) && memory.embedding.length
+      ? JSON.stringify(memory.embedding)
+      : null;
     db.prepare(
-      `INSERT INTO memories (id, type, title, content, tags, importance, forgotten, source, created_at, updated_at)
-       VALUES (?, ?, ?, ?, ?, ?, 0, ?, ?, ?)`
-    ).run(id, type, memory.title, memory.content, tags, importance, memory.source ?? null, now, now);
+      `INSERT INTO memories (id, type, title, content, tags, importance, forgotten, source, embedding, created_at, updated_at)
+       VALUES (?, ?, ?, ?, ?, ?, 0, ?, ?, ?, ?)`
+    ).run(id, type, memory.title, memory.content, tags, importance, memory.source ?? null, embedding, now, now);
     return getById(id);
   }
 
@@ -133,8 +140,11 @@ export function createStore(path) {
       throw new Error("tags must be an array");
     }
     const now = nowIso();
+    const embedding = patch.embedding !== undefined
+      ? (Array.isArray(patch.embedding) && patch.embedding.length ? JSON.stringify(patch.embedding) : null)
+      : existing.embedding ?? null;
     db.prepare(
-      `UPDATE memories SET type=?, title=?, content=?, tags=?, importance=?, source=?, updated_at=? WHERE id=?`
+      `UPDATE memories SET type=?, title=?, content=?, tags=?, importance=?, source=?, embedding=?, updated_at=? WHERE id=?`
     ).run(
       type,
       patch.title ?? existing.title,
@@ -142,6 +152,7 @@ export function createStore(path) {
       JSON.stringify(patch.tags ?? existing.tags),
       Number.isInteger(patch.importance) ? patch.importance : existing.importance,
       patch.source !== undefined ? patch.source : (existing.source ?? null),
+      embedding,
       now,
       id
     );
@@ -190,6 +201,27 @@ export function createStore(path) {
     return rows.map(toRow);
   }
 
+  /** Set (or clear with null) the embedding vector of a memory. */
+  function setEmbedding(id, vector) {
+    const json = Array.isArray(vector) && vector.length ? JSON.stringify(vector) : null;
+    db.prepare("UPDATE memories SET embedding = ? WHERE id = ?").run(json, id);
+  }
+
+  function embeddedCount() {
+    return db.prepare(
+      "SELECT count(*) AS c FROM memories WHERE embedding IS NOT NULL AND embedding != ''"
+    ).get().c;
+  }
+
+  /** Candidate rows still missing an embedding, for incremental re-indexing. */
+  function needsEmbedding(limit = 50) {
+    return db.prepare(
+      `SELECT id, title, content FROM memories
+       WHERE embedding IS NULL OR embedding = ''
+       ORDER BY updated_at DESC LIMIT ?`
+    ).all(limit);
+  }
+
   function search(query, { limit = 20, includeArchived = false } = {}) {
     const q = String(query).trim();
     if (!q) return [];
@@ -212,6 +244,49 @@ export function createStore(path) {
     return rows.map(toRow);
   }
 
+  // --- vector search ------------------------------------------------------
+
+  function cosine(a, b) {
+    if (!Array.isArray(a) || !Array.isArray(b) || a.length !== b.length || !a.length) return 0;
+    let dot = 0;
+    let na = 0;
+    let nb = 0;
+    for (let i = 0; i < a.length; i++) {
+      dot += a[i] * b[i];
+      na += a[i] * a[i];
+      nb += b[i] * b[i];
+    }
+    if (na === 0 || nb === 0) return 0;
+    return dot / (Math.sqrt(na) * Math.sqrt(nb));
+  }
+
+  /**
+   * Brute-force cosine similarity over embedded rows. Returns rows decorated
+   * with a `score` (0..1). Only rows with a stored embedding participate.
+   */
+  function searchVector(vector, { limit = 20, includeArchived = false, threshold = 0 } = {}) {
+    if (!Array.isArray(vector) || !vector.length) return [];
+    const archivedFilter = includeArchived ? "" : "archived = 0 AND ";
+    const rows = db.prepare(
+      `SELECT * FROM memories
+       WHERE ${archivedFilter}forgotten = 0 AND embedding IS NOT NULL AND embedding != ''`
+    ).all();
+    const scored = [];
+    for (const row of rows) {
+      let v;
+      try {
+        v = JSON.parse(row.embedding);
+      } catch {
+        continue;
+      }
+      const score = cosine(vector, v);
+      if (score >= threshold) scored.push({ row, score });
+    }
+    scored.sort((a, b) => b.score - a.score);
+    const { limit: lim } = sanitizePage(limit, 0, 20);
+    return scored.slice(0, lim).map(({ row, score }) => ({ ...toRow(row), score }));
+  }
+
   return {
     db,
     count,
@@ -224,6 +299,10 @@ export function createStore(path) {
     list,
     all,
     search,
+    setEmbedding,
+    embeddedCount,
+    needsEmbedding,
+    searchVector,
     close() {
       db.close();
     }

package/package.json

@@ -1,58 +1,58 @@
-{
-  "name": "@modusensus/dsh-mneme",
-  "description": "Cross-session memory plugin for DeepSeek Harness with autoDream consolidation: SQLite store, Markdown mirrors, 6 model tools, automatic injection, session summarization, and a Web GUI panel",
-  "version": "0.1.4",
-  "license": "MIT",
-  "type": "module",
-  "main": "lib/index.js",
-  "exports": {
-    ".": {
-      "default": "./lib/index.js"
-    },
-    "./client": {
-      "default": "./lib/client.js"
-    },
-    "./package.json": "./package.json"
-  },
-  "files": [
-    "lib",
-    "src",
-    "cordis.patch.yml"
-  ],
-  "dsh": {
-    "client": {
-      "inject": [
-        "slots",
-        "locale",
-        "layout",
-        "connection"
-      ],
-      "platform": "web"
-    },
-    "bundle": {
-      "patch": "./cordis.patch.yml"
-    }
-  },
-  "peerDependencies": {
-    "@deepseek-ai/cordis": "^4.0.1",
-    "@deepseek-ai/dsh-host-webserver": "^0.1.0-rc.6",
-    "@deepseek-ai/dsh-llm": "^0.1.0-rc.6",
-    "@deepseek-ai/dsh-system-prompt": "^0.1.0-rc.6",
-    "@deepseek-ai/dsh-tools": "^0.1.0-rc.6",
-    "@deepseek-ai/schemastery": "^3.18.1"
-  },
-  "devDependencies": {
-    "@deepseek-ai/cordis": "^4.0.1",
-    "@deepseek-ai/dsh-host-webserver": "^0.1.0-rc.6",
-    "@deepseek-ai/dsh-llm": "^0.1.0-rc.6",
-    "@deepseek-ai/dsh-system-prompt": "^0.1.0-rc.6",
-    "@deepseek-ai/dsh-tools": "^0.1.0-rc.6",
-    "@deepseek-ai/schemastery": "^3.18.1"
-  },
-  "scripts": {
-    "sync": "node scripts/sync-lib.js",
-    "prepack": "npm run sync",
-    "test": "node --test --test-isolation=none test/*.test.js",
-    "e2e": "node scripts/e2e-dsh.js"
-  }
-}
+{
+  "name": "@modusensus/dsh-mneme",
+  "description": "Cross-session memory plugin for DeepSeek Harness with autoDream consolidation: SQLite store, Markdown mirrors, 6 model tools, automatic injection, session summarization, and a Web GUI panel",
+  "version": "0.1.5",
+  "license": "MIT",
+  "type": "module",
+  "main": "lib/index.js",
+  "exports": {
+    ".": {
+      "default": "./lib/index.js"
+    },
+    "./client": {
+      "default": "./lib/client.js"
+    },
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "lib",
+    "src",
+    "cordis.patch.yml"
+  ],
+  "dsh": {
+    "client": {
+      "inject": [
+        "slots",
+        "locale",
+        "layout",
+        "connection"
+      ],
+      "platform": "web"
+    },
+    "bundle": {
+      "patch": "./cordis.patch.yml"
+    }
+  },
+  "peerDependencies": {
+    "@deepseek-ai/cordis": "^4.0.1",
+    "@deepseek-ai/dsh-host-webserver": "^0.1.0-rc.6",
+    "@deepseek-ai/dsh-llm": "^0.1.0-rc.6",
+    "@deepseek-ai/dsh-system-prompt": "^0.1.0-rc.6",
+    "@deepseek-ai/dsh-tools": "^0.1.0-rc.6",
+    "@deepseek-ai/schemastery": "^3.18.1"
+  },
+  "devDependencies": {
+    "@deepseek-ai/cordis": "^4.0.1",
+    "@deepseek-ai/dsh-host-webserver": "^0.1.0-rc.6",
+    "@deepseek-ai/dsh-llm": "^0.1.0-rc.6",
+    "@deepseek-ai/dsh-system-prompt": "^0.1.0-rc.6",
+    "@deepseek-ai/dsh-tools": "^0.1.0-rc.6",
+    "@deepseek-ai/schemastery": "^3.18.1"
+  },
+  "scripts": {
+    "sync": "node scripts/sync-lib.js",
+    "prepack": "npm run sync",
+    "test": "node --test --test-isolation=none test/*.test.js",
+    "e2e": "node scripts/e2e-dsh.js"
+  }
+}

package/src/api.js

@@ -23,7 +23,7 @@ function parseBody(text) {
   }
 }
 
-export function createApi(ctx, service, settings, commands) {
+export function createApi(ctx, service, settings, commands, embedder) {
   const disposers = [];
 
   const register = (route) => {
@@ -64,8 +64,49 @@ export function createApi(ctx, service, settings, commands) {
         const url = new URL(req.url, "http://localhost");
         const q = url.searchParams.get("q") ?? "";
         const limit = Number(url.searchParams.get("limit") ?? 20);
-        const items = service.toApiList(service.search(q, { limit }));
-        sendJson(res, 200, { items });
+        // mode: auto (default) | keyword | vector
+        const mode = url.searchParams.get("mode") ?? "auto";
+        const query = q.trim();
+        if (!query) {
+          sendJson(res, 200, { items: [], mode: "keyword" });
+          return;
+        }
+        // Keyword results (existing behavior) always computed; used as a
+        // fallback and as the primary ranking when vector is unavailable.
+        const keyword = service.toApiList(service.search(query, { limit }));
+        if (mode === "keyword" || !embedder) {
+          sendJson(res, 200, { items: keyword, mode: "keyword" });
+          return;
+        }
+        const cfg = settings.getVectorConfig();
+        if (mode === "vector" && !cfg?.enabled) {
+          sendJson(res, 200, { items: keyword, mode: "keyword", error: "vector-disabled" });
+          return;
+        }
+        // Try vector search; on any failure fall back to keyword results.
+        return embedder.embed(query).then(async (vector) => {
+          let items = keyword;
+          let used = "keyword";
+          if (vector) {
+            const scored = service.toApiList(service.searchVector(vector, { limit }));
+            // Merge: keyword exact hits first (they are the user's literal
+            // words), then vector results fill the remaining slots, deduped.
+            const seen = new Set(keyword.map((m) => m.id));
+            const merged = [...keyword];
+            for (const m of scored) {
+              if (merged.length >= limit) break;
+              if (!seen.has(m.id)) {
+                seen.add(m.id);
+                merged.push(m);
+              }
+            }
+            items = merged;
+            used = "vector";
+          }
+          sendJson(res, 200, { items, mode: used });
+        }).catch(() => {
+          sendJson(res, 200, { items: keyword, mode: "keyword" });
+        });
       } catch {
         sendJson(res, 500, { error: "internal" });
       }
@@ -112,6 +153,54 @@ export function createApi(ctx, service, settings, commands) {
     }
   });
 
+  // --- vector search config ---
+  register({
+    kind: "exact",
+    path: "/api/dsh-mneme/vector-config",
+    handler(req, res) {
+      try {
+        if (req.method === "PUT" || req.method === "POST") {
+          return readBody(req).then((text) => {
+            const body = parseBody(text);
+            const cfg = settings.setVectorConfig({
+              enabled: body.enabled,
+              baseUrl: body.baseUrl,
+              apiKey: body.apiKey,
+              model: body.model
+            });
+            sendJson(res, 200, { config: cfg });
+          });
+        }
+        sendJson(res, 200, { config: settings.getVectorConfig() ?? { enabled: false, baseUrl: "", apiKey: "", model: "" } });
+      } catch {
+        sendJson(res, 500, { error: "internal" });
+      }
+    }
+  });
+
+  // --- vector re-index (backfill embeddings for rows missing them) ---
+  register({
+    kind: "exact",
+    path: "/api/dsh-mneme/vector-reindex",
+    handler(req, res) {
+      try {
+        if (!embedder) {
+          sendJson(res, 200, { indexed: 0, skipped: 0, error: "vector-unavailable" });
+          return;
+        }
+        const url = new URL(req.url, "http://localhost");
+        const limit = Number(url.searchParams.get("limit") ?? 100);
+        embedder.reindexMissing(limit).then((result) => {
+          sendJson(res, 200, result);
+        }).catch(() => {
+          sendJson(res, 200, { indexed: 0, skipped: 0, error: "vector-failed" });
+        });
+      } catch {
+        sendJson(res, 500, { error: "internal" });
+      }
+    }
+  });
+
   // --- custom commands ---
   register({
     kind: "exact",

package/src/embedding.js

@@ -0,0 +1,97 @@
+// OpenAI-compatible embedding client for vector search. DSH's LLM service is
+// chat-only, so dsh-mneme calls an external `/embeddings` endpoint itself.
+// Works with OpenAI, SiliconFlow, Zhipu, local Ollama (via OpenAI-compatible
+// proxy) and any provider exposing the standard embeddings API.
+const DEFAULT_TIMEOUT_MS = 15000;
+
+/** Normalize a configured baseUrl into the full embeddings endpoint URL. */
+function embeddingsUrl(baseUrl) {
+  const base = String(baseUrl ?? "").trim().replace(/\/+$/, "");
+  if (!base) return "";
+  // Accept both "https://host/v1" and a full path ending in /embeddings.
+  if (/\/embeddings$/i.test(base)) return base;
+  return `${base}/embeddings`;
+}
+
+/**
+ * Call the embeddings API for one text. Resolves to a Float64 array, or null
+ * when the provider is not configured, the call fails, or the response is
+ * unusable. Never throws: failures degrade to keyword search.
+ */
+export async function embedText({ baseUrl, apiKey, model }, text) {
+  const url = embeddingsUrl(baseUrl);
+  if (!url || !apiKey || !model || !text) return null;
+  let res;
+  try {
+    res = await fetch(url, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        "Authorization": `Bearer ${apiKey}`
+      },
+      body: JSON.stringify({ model, input: String(text).slice(0, 8000) }),
+      signal: AbortSignal.timeout(DEFAULT_TIMEOUT_MS)
+    });
+  } catch {
+    return null;
+  }
+  if (!res.ok) return null;
+  let body;
+  try {
+    body = await res.json();
+  } catch {
+    return null;
+  }
+  const vec = body?.data?.[0]?.embedding;
+  return Array.isArray(vec) && vec.length ? Array.from(vec) : null;
+}
+
+/**
+ * Embedder bound to the current settings + store: on each write it re-embeds
+ * the row's title+content and stores the vector. Failures are swallowed so a
+ * flaky embedding endpoint never breaks memory writes.
+ */
+export function createEmbedder({ store, settings, logger }) {
+  async function embedFor(id, title, content) {
+    const cfg = settings.getVectorConfig();
+    if (!cfg?.enabled || !cfg.baseUrl || !cfg.apiKey || !cfg.model) return;
+    const text = [title, content].filter(Boolean).join("\n");
+    const vector = await embedText(cfg, text);
+    if (vector) {
+      store.setEmbedding(id, vector);
+      logger?.info?.(`[dsh-mneme] embedded memory ${id} (dim=${vector.length})`);
+    }
+  }
+
+  return {
+    /** Fire-and-forget re-embed of a memory after any write. */
+    schedule(memory) {
+      if (!memory?.id) return;
+      embedFor(memory.id, memory.title, memory.content).catch(() => {});
+    },
+
+    /** Embed one text and return its vector (null on failure/disabled). */
+    async embed(query) {
+      const cfg = settings.getVectorConfig();
+      if (!cfg?.enabled || !cfg.baseUrl || !cfg.apiKey || !cfg.model) return null;
+      return embedText(cfg, query);
+    },
+
+    /** Batch re-index rows still missing an embedding. */
+    async reindexMissing(limit = 50) {
+      const cfg = settings.getVectorConfig();
+      if (!cfg?.enabled) return { indexed: 0, skipped: 0 };
+      const rows = store.needsEmbedding(limit);
+      let indexed = 0;
+      for (const row of rows) {
+        const text = [row.title, row.content].filter(Boolean).join("\n");
+        const vector = await embedText(cfg, text);
+        if (vector) {
+          store.setEmbedding(row.id, vector);
+          indexed++;
+        }
+      }
+      return { indexed, skipped: rows.length - indexed };
+    }
+  };
+}

package/src/index.js

@@ -8,6 +8,7 @@ import { createDreamScheduler } from "./dream.js";
 import { createApi } from "./api.js";
 import { createSettings } from "./settings.js";
 import { createCommandManager } from "./commands.js";
+import { createEmbedder } from "./embedding.js";
 import { Config } from "./config.js";
 import { mkdirSync } from "node:fs";
 import { join } from "node:path";
@@ -39,6 +40,11 @@ export const apply = (ctx, config) => {
   // same SQLite file but live in dedicated tables, isolated from memories.
   const settings = createSettings(store.db);
 
+  // Vector search: embedder calls the configured OpenAI-compatible embeddings
+  // endpoint on writes and for queries. service re-embeds after each write.
+  const embedder = createEmbedder({ store, settings, logger: ctx.logger });
+  service.setEmbedder(embedder);
+
   // Custom commands: register persisted commands into the DSH command registry
   // on boot; add/remove re-register live through the API.
   let commands = null;
@@ -96,7 +102,7 @@ export const apply = (ctx, config) => {
       add: () => { throw new Error("commands unavailable"); },
       remove: () => false,
       list: () => []
-    });
+    }, embedder);
     disposers.push(api.dispose);
   }
 

package/src/service.js

@@ -6,6 +6,17 @@ export function createService({ store, mirror, config, onWrite }) {
   // passed in the constructor). Fired on the same write events as onWrite.
   let dreamHook = null;
 
+  // Optional vector embedder, installed via setEmbedder after creation. After
+  // any content write it fire-and-forgets a re-embed of the row so vector
+  // search stays in sync; failures are swallowed inside the embedder.
+  let embedder = null;
+
+  function scheduleEmbed(memory) {
+    if (embedder && memory?.id) {
+      try { embedder.schedule(memory); } catch { /* ignore */ }
+    }
+  }
+
   /**
    * Fire-and-forget write notification; errors are swallowed to keep write
    * paths clean. The store mutation has already committed, so a throwing
@@ -38,6 +49,7 @@ export function createService({ store, mirror, config, onWrite }) {
       });
       syncMirror();
       notifyWrite();
+      scheduleEmbed(merged);
       return { action: "merged", memory: merged };
     }
     const created = store.save({
@@ -50,6 +62,7 @@ export function createService({ store, mirror, config, onWrite }) {
     });
     syncMirror();
     notifyWrite();
+    scheduleEmbed(created);
     return { action: "created", memory: created };
   }
 
@@ -126,8 +139,11 @@ export function createService({ store, mirror, config, onWrite }) {
     mergeHumanEdits,
     toApiList,
     setDreamHook(fn) { dreamHook = fn; },
+    setEmbedder(emb) { embedder = emb; },
     // passthroughs used by tools and api layers; mutations keep the mirror in sync
     search: (q, o) => store.search(q, o),
+    searchVector: (v, o) => store.searchVector(v, o),
+    embeddedCount: () => store.embeddedCount(),
     list: (o) => store.list(o),
     all: () => store.all(),
     count: (type) => store.count(type),
@@ -141,6 +157,7 @@ export function createService({ store, mirror, config, onWrite }) {
       const updated = store.update(id, p);
       syncMirror();
       notifyWrite();
+      scheduleEmbed(updated);
       return updated;
     },
     setForget: (id, f) => {

package/src/settings.js

@@ -115,6 +115,28 @@ export function createSettings(db) {
     removeCommand(id) {
       const result = db.prepare("DELETE FROM custom_commands WHERE id = ?").run(id);
       return result.changes > 0;
+    },
+
+    /** Vector-search provider config (OpenAI-compatible embeddings endpoint). */
+    getVectorConfig() {
+      const raw = getSetting("vector");
+      if (!raw) return undefined;
+      try {
+        const cfg = JSON.parse(raw);
+        return typeof cfg === "object" && cfg !== null ? cfg : undefined;
+      } catch {
+        return undefined;
+      }
+    },
+    setVectorConfig({ enabled, baseUrl, apiKey, model }) {
+      const cfg = {
+        enabled: enabled === true || enabled === 1,
+        baseUrl: String(baseUrl ?? "").trim().replace(/\/+$/, ""),
+        apiKey: String(apiKey ?? "").trim(),
+        model: String(model ?? "").trim()
+      };
+      setSetting("vector", JSON.stringify(cfg));
+      return cfg;
     }
   };
 }