@modusensus/dsh-mneme 0.1.6 → 0.2.0

package/lib/tools.js

@@ -1,241 +1,230 @@
-import { defineTool } from "@deepseek-ai/dsh-tools";
-
-const TEXT_OUTPUT = (text) => [{ type: "text", text }];
-
-// Wire shape emitted by service.toApiList: shared by memory_search and
-// memory_list so their output schemas always declare every key the runtime
-// value carries (additionalProperties: false would reject undeclared keys).
-const MEMORY_ITEM_SCHEMA = {
-  type: "object",
-  additionalProperties: false,
-  properties: {
-    id: { type: "string", required: true },
-    type: { type: "string", required: true },
-    title: { type: "string", required: true },
-    content: { type: "string", required: true },
-    tags: { type: "array", items: { type: "string" } },
-    importance: { type: "integer", required: true },
-    source: { type: "string" },
-    created_at: { type: "string", required: true },
-    updated_at: { type: "string", required: true }
-  }
-};
-
-export function createTools(ctx, service, config, embedder) {
-  const tools = [
-    defineTool({
-      name: "memory_save",
-      description:
-        "Persist one memory entry for future sessions (user preferences, project state, decisions). " +
-        "Call this when the user states a durable preference, a project decision is made, or a lesson is learned. " +
-        "Merges into an existing entry of the same type when the title matches.",
-      parameters: {
-        type: { type: "string", required: true, enum: ["preference", "project", "decision", "history"], description: "preference=user profile; project=project knowledge/state; decision=key decision; history=conversation summary" },
-        title: { type: "string", required: true, description: "Short unique title" },
-        content: { type: "string", required: true, description: "Memory body" },
-        tags: { type: "array", items: { type: "string" }, description: "Optional tags" },
-        importance: { type: "integer", description: "1-5; >= threshold auto-injects into future sessions" },
-        source: { type: "string", description: "Optional provenance" }
-      },
-      output: {
-        schema: {
-          type: "object",
-          additionalProperties: false,
-          properties: {
-            action: { type: "string", required: true, enum: ["created", "merged"] },
-            id: { type: "string", required: true }
-          }
-        },
-        render: (_args, value) => TEXT_OUTPUT(`memory ${value.action}: ${value.id}`)
-      },
-      async execute(args) {
-        const { action, memory } = service.saveWithDedupe({
-          type: args.type,
-          title: args.title,
-          content: args.content,
-          tags: args.tags ?? [],
-          importance: args.importance ?? 3,
-          source: args.source ?? "tool"
-        });
-        return { action, id: memory.id };
-      }
-    }),
-
-    defineTool({
-      name: "memory_search",
-      description: "Search the cross-session memory store. Use when you need past context: how a problem was solved, user preferences, project decisions. Substring-matches title/content/tags, and optionally augments results with semantic (vector) recall when an embeddings provider is configured. Returns matching entries with source and timestamps.",
-      parameters: {
-        query: { type: "string", required: true, description: "Search text; substring match over title/content/tags" },
-        limit: { type: "integer", description: "Max results (default 20)" },
-        mode: { type: "string", enum: ["auto", "keyword", "vector"], description: "auto (default) = keyword hits first + vector fill when enabled; keyword = text only; vector = semantic recall first (falls back to keyword)" },
-        semantic: { type: "boolean", description: "Shorthand: enable semantic (vector) recall (same as mode=vector when true)" }
-      },
-      output: {
-        schema: {
-          type: "object",
-          additionalProperties: false,
-          properties: {
-            items: {
-              type: "array", required: true,
-              items: MEMORY_ITEM_SCHEMA
-            }
-          }
-        },
-        render: (_args, value) => TEXT_OUTPUT(`Found ${value.items.length} memory entr${value.items.length === 1 ? "y" : "ies"}.`)
-      },
-      async execute(args) {
-        const limit = args.limit ?? 20;
-        const rows = service.toApiList(service.search(args.query, { limit }));
-        const wantVector = args.mode === "vector" || args.semantic === true || args.mode === "auto";
-        if (wantVector && embedder) {
-          try {
-            const vector = await embedder.embed(args.query);
-            if (vector) {
-              const scored = service.toApiList(service.searchVector(vector, { limit }));
-              const seen = new Set(rows.map((m) => m.id));
-              for (const m of scored) {
-                if (rows.length >= limit) break;
-                if (!seen.has(m.id)) {
-                  seen.add(m.id);
-                  rows.push(m);
-                }
-              }
-            }
-          } catch { /* vector unavailable: keep keyword results */ }
-        }
-        return { items: rows };
-      }
-    }),
-
-    defineTool({
-      name: "memory_list",
-      description: "List memory entries by type, high-importance first, then newest, paginated.",
-      parameters: {
-        type: { type: "string", enum: ["preference", "project", "decision", "history"], description: "Filter by type; omit for all" },
-        limit: { type: "integer", description: "Page size (default 50)" },
-        offset: { type: "integer", description: "Page offset (default 0)" }
-      },
-      output: {
-        schema: {
-          type: "object",
-          additionalProperties: false,
-          properties: {
-            items: {
-              type: "array", required: true,
-              items: MEMORY_ITEM_SCHEMA
-            },
-            total: { type: "integer", required: true }
-          }
-        },
-        render: (_args, value) => TEXT_OUTPUT(`${value.items.length} memory entries (of ${value.total}).`)
-      },
-      async execute(args) {
-        const rows = service.toApiList(service.list({ type: args.type, limit: args.limit ?? 50, offset: args.offset ?? 0 }));
-        return { items: rows, total: service.count(args.type) };
-      }
-    }),
-
-    defineTool({
-      name: "memory_update",
-      description: "Modify an existing memory entry (title, content, type, tags, importance).",
-      parameters: {
-        id: { type: "string", required: true, description: "Memory id" },
-        title: { type: "string" },
-        content: { type: "string" },
-        type: { type: "string", enum: ["preference", "project", "decision", "history"] },
-        tags: { type: "array", items: { type: "string" } },
-        importance: { type: "integer", description: "1-5" }
-      },
-      output: {
-        schema: {
-          type: "object",
-          additionalProperties: false,
-          properties: {
-            memory: {
-              type: "object",
-              additionalProperties: false,
-              properties: {
-                id: { type: "string", required: true },
-                title: { type: "string", required: true },
-                content: { type: "string", required: true }
-              }
-            }
-          }
-        },
-        render: (_args, value) => TEXT_OUTPUT(`Updated memory ${value.memory.id}: ${value.memory.title}`)
-      },
-      async execute(args) {
-        const memory = service.update(args.id, {
-          title: args.title,
-          content: args.content,
-          type: args.type,
-          tags: args.tags,
-          importance: args.importance
-        });
-        return { memory: { id: memory.id, title: memory.title, content: memory.content } };
-      }
-    }),
-
-    defineTool({
-      name: "memory_delete",
-      description: "Permanently delete a memory entry.",
-      parameters: {
-        id: { type: "string", required: true }
-      },
-      output: {
-        schema: {
-          type: "object",
-          additionalProperties: false,
-          properties: { deleted: { type: "boolean", required: true } }
-        },
-        render: (_args, value) => TEXT_OUTPUT(value.deleted ? "Memory deleted." : "Memory not found.")
-      },
-      async execute(args) {
-        const existed = service.getById(args.id) !== undefined;
-        if (existed) service.remove(args.id);
-        return { deleted: existed };
-      }
-    }),
-
-    defineTool({
-      name: "memory_forget",
-      description:
-        "Stop a memory from being auto-injected and from appearing in searches and lists without deleting it. " +
-        "The entry stays in storage; pass forgotten: false to restore it.",
-      parameters: {
-        id: { type: "string", required: true },
-        forgotten: { type: "boolean", description: "Suppress (true, default) or restore (false) the entry's visibility" }
-      },
-      output: {
-        schema: {
-          type: "object",
-          additionalProperties: false,
-          properties: {
-            memory: {
-              type: "object",
-              additionalProperties: false,
-              properties: {
-                id: { type: "string", required: true },
-                forgotten: { type: "boolean", required: true }
-              }
-            }
-          }
-        },
-        render: (_args, value) => TEXT_OUTPUT(`Memory ${value.memory.id} injection ${value.memory.forgotten ? "suppressed" : "restored"}.`)
-      },
-      async execute(args) {
-        if (service.getById(args.id) === undefined) {
-          throw new Error("memory not found");
-        }
-        const memory = service.setForget(args.id, args.forgotten ?? true);
-        return { memory: { id: memory.id, forgotten: memory.forgotten } };
-      }
-    })
-  ];
-
-  for (const tool of tools) {
-    ctx.tools.register(tool);
-  }
-
-  return tools;
-}
+import { defineTool } from "@deepseek-ai/dsh-tools";
+
+const TEXT_OUTPUT = (text) => [{ type: "text", text }];
+
+// Wire shape emitted by service.toApiList: shared by memory_search and
+// memory_list so their output schemas always declare every key the runtime
+// value carries (additionalProperties: false would reject undeclared keys).
+const MEMORY_ITEM_SCHEMA = {
+  type: "object",
+  additionalProperties: false,
+  properties: {
+    id: { type: "string", required: true },
+    type: { type: "string", required: true },
+    title: { type: "string", required: true },
+    content: { type: "string", required: true },
+    tags: { type: "array", items: { type: "string" } },
+    importance: { type: "integer", required: true },
+    source: { type: "string" },
+    created_at: { type: "string", required: true },
+    updated_at: { type: "string", required: true }
+  }
+};
+
+export function createTools(ctx, service, config, embedder) {
+  const tools = [
+    defineTool({
+      name: "memory_save",
+      description:
+        "Persist one memory entry for future sessions (user preferences, project state, decisions). " +
+        "Call this when the user states a durable preference, a project decision is made, or a lesson is learned. " +
+        "Merges into an existing entry of the same type when the title matches.",
+      parameters: {
+        type: { type: "string", required: true, enum: ["preference", "project", "decision", "history"], description: "preference=user profile; project=project knowledge/state; decision=key decision; history=conversation summary" },
+        title: { type: "string", required: true, description: "Short unique title" },
+        content: { type: "string", required: true, description: "Memory body" },
+        tags: { type: "array", items: { type: "string" }, description: "Optional tags" },
+        importance: { type: "integer", description: "1-5; >= threshold auto-injects into future sessions" },
+        source: { type: "string", description: "Optional provenance" }
+      },
+      output: {
+        schema: {
+          type: "object",
+          additionalProperties: false,
+          properties: {
+            action: { type: "string", required: true, enum: ["created", "merged"] },
+            id: { type: "string", required: true }
+          }
+        },
+        render: (_args, value) => TEXT_OUTPUT(`memory ${value.action}: ${value.id}`)
+      },
+      async execute(args) {
+        const { action, memory } = service.saveWithDedupe({
+          type: args.type,
+          title: args.title,
+          content: args.content,
+          tags: args.tags ?? [],
+          importance: args.importance ?? 3,
+          source: args.source ?? "tool"
+        });
+        return { action, id: memory.id };
+      }
+    }),
+
+    defineTool({
+      name: "memory_search",
+      description: "Search the cross-session memory store. Use when you need past context: how a problem was solved, user preferences, project decisions. Substring-matches title/content/tags, and augments results with semantic (vector) recall + optional rerank when an embeddings provider is configured. Returns matching entries with source and timestamps.",
+      parameters: {
+        query: { type: "string", required: true, description: "Search text; substring match over title/content/tags" },
+        limit: { type: "integer", description: "Max results (default 20)" },
+        mode: { type: "string", enum: ["auto", "keyword", "vector", "hybrid"], description: "auto (default) = keyword hits first + vector fill when enabled; keyword = text only; vector = semantic recall first (falls back to keyword); hybrid = vector leads, keyword fills remaining slots" },
+        semantic: { type: "boolean", description: "Shorthand: enable semantic (vector) recall (same as mode=vector when true)" },
+        rerank: { type: "boolean", description: "Run cross-encoder rerank over candidates when a local reranker is configured (default true)" }
+      },
+      output: {
+        schema: {
+          type: "object",
+          additionalProperties: false,
+          properties: {
+            items: {
+              type: "array", required: true,
+              items: MEMORY_ITEM_SCHEMA
+            }
+          }
+        },
+        render: (_args, value) => TEXT_OUTPUT(`Found ${value.items.length} memory entr${value.items.length === 1 ? "y" : "ies"}.`)
+      },
+      async execute(args) {
+        const limit = args.limit ?? 20;
+        const mode = args.semantic === true && !args.mode ? "vector" : args.mode ?? "auto";
+        const rows = await service.searchMemories(args.query, {
+          mode,
+          topK: limit,
+          useRerank: args.rerank !== false
+        });
+        return { items: service.toApiList(rows) };
+      }
+    }),
+
+    defineTool({
+      name: "memory_list",
+      description: "List memory entries by type, high-importance first, then newest, paginated.",
+      parameters: {
+        type: { type: "string", enum: ["preference", "project", "decision", "history"], description: "Filter by type; omit for all" },
+        limit: { type: "integer", description: "Page size (default 50)" },
+        offset: { type: "integer", description: "Page offset (default 0)" }
+      },
+      output: {
+        schema: {
+          type: "object",
+          additionalProperties: false,
+          properties: {
+            items: {
+              type: "array", required: true,
+              items: MEMORY_ITEM_SCHEMA
+            },
+            total: { type: "integer", required: true }
+          }
+        },
+        render: (_args, value) => TEXT_OUTPUT(`${value.items.length} memory entries (of ${value.total}).`)
+      },
+      async execute(args) {
+        const rows = service.toApiList(service.list({ type: args.type, limit: args.limit ?? 50, offset: args.offset ?? 0 }));
+        return { items: rows, total: service.count(args.type) };
+      }
+    }),
+
+    defineTool({
+      name: "memory_update",
+      description: "Modify an existing memory entry (title, content, type, tags, importance).",
+      parameters: {
+        id: { type: "string", required: true, description: "Memory id" },
+        title: { type: "string" },
+        content: { type: "string" },
+        type: { type: "string", enum: ["preference", "project", "decision", "history"] },
+        tags: { type: "array", items: { type: "string" } },
+        importance: { type: "integer", description: "1-5" }
+      },
+      output: {
+        schema: {
+          type: "object",
+          additionalProperties: false,
+          properties: {
+            memory: {
+              type: "object",
+              additionalProperties: false,
+              properties: {
+                id: { type: "string", required: true },
+                title: { type: "string", required: true },
+                content: { type: "string", required: true }
+              }
+            }
+          }
+        },
+        render: (_args, value) => TEXT_OUTPUT(`Updated memory ${value.memory.id}: ${value.memory.title}`)
+      },
+      async execute(args) {
+        const memory = service.update(args.id, {
+          title: args.title,
+          content: args.content,
+          type: args.type,
+          tags: args.tags,
+          importance: args.importance
+        });
+        return { memory: { id: memory.id, title: memory.title, content: memory.content } };
+      }
+    }),
+
+    defineTool({
+      name: "memory_delete",
+      description: "Permanently delete a memory entry.",
+      parameters: {
+        id: { type: "string", required: true }
+      },
+      output: {
+        schema: {
+          type: "object",
+          additionalProperties: false,
+          properties: { deleted: { type: "boolean", required: true } }
+        },
+        render: (_args, value) => TEXT_OUTPUT(value.deleted ? "Memory deleted." : "Memory not found.")
+      },
+      async execute(args) {
+        const existed = service.getById(args.id) !== undefined;
+        if (existed) service.remove(args.id);
+        return { deleted: existed };
+      }
+    }),
+
+    defineTool({
+      name: "memory_forget",
+      description:
+        "Stop a memory from being auto-injected and from appearing in searches and lists without deleting it. " +
+        "The entry stays in storage; pass forgotten: false to restore it.",
+      parameters: {
+        id: { type: "string", required: true },
+        forgotten: { type: "boolean", description: "Suppress (true, default) or restore (false) the entry's visibility" }
+      },
+      output: {
+        schema: {
+          type: "object",
+          additionalProperties: false,
+          properties: {
+            memory: {
+              type: "object",
+              additionalProperties: false,
+              properties: {
+                id: { type: "string", required: true },
+                forgotten: { type: "boolean", required: true }
+              }
+            }
+          }
+        },
+        render: (_args, value) => TEXT_OUTPUT(`Memory ${value.memory.id} injection ${value.memory.forgotten ? "suppressed" : "restored"}.`)
+      },
+      async execute(args) {
+        if (service.getById(args.id) === undefined) {
+          throw new Error("memory not found");
+        }
+        const memory = service.setForget(args.id, args.forgotten ?? true);
+        return { memory: { id: memory.id, forgotten: memory.forgotten } };
+      }
+    })
+  ];
+
+  for (const tool of tools) {
+    ctx.tools.register(tool);
+  }
+
+  return tools;
+}

package/lib/vector-index.js

@@ -0,0 +1,106 @@
+// Thin vector-index layer over the store's embedding column. Keeps the
+// current model fingerprint + stats in a dedicated `vector_meta` table so the
+// index can detect drift (embeddings generated by a different model) and the
+// Web panel can report index health. The heavy lifting (cosine search) lives
+// in store.searchVector; this module owns the index lifecycle.
+
+const META_SCHEMA = `
+CREATE TABLE IF NOT EXISTS vector_meta (
+  key        TEXT PRIMARY KEY,
+  value      TEXT NOT NULL
+);
+`;
+
+export function createVectorIndex({ store, logger }) {
+  const db = store.db;
+  db.exec(META_SCHEMA);
+
+  function getMeta(key) {
+    const row = db.prepare("SELECT value FROM vector_meta WHERE key = ?").get(key);
+    return row?.value ?? undefined;
+  }
+
+  function setMeta(key, value) {
+    db.prepare(
+      `INSERT INTO vector_meta (key, value) VALUES (?, ?)
+       ON CONFLICT(key) DO UPDATE SET value = excluded.value`
+    ).run(key, String(value));
+  }
+
+  return {
+    /** Record the model that generated the current vectors. */
+    markModel(modelHash, dimension) {
+      setMeta("model_hash", modelHash);
+      setMeta("dimension", String(dimension ?? ""));
+      setMeta("updated_at", new Date().toISOString());
+    },
+
+    modelHash() {
+      return getMeta("model_hash");
+    },
+
+    dimension() {
+      const raw = getMeta("dimension");
+      return raw && Number.isInteger(Number(raw)) ? Number(raw) : undefined;
+    },
+
+    /** Store (or clear with null vector) one memory's embedding. */
+    saveEmbedding(id, vector) {
+      store.setEmbedding(id, vector);
+    },
+
+    /** Drop one memory's embedding. */
+    deleteEmbedding(id) {
+      store.setEmbedding(id, null);
+    },
+
+    /** Read one memory's cached vector; undefined when none/parse failure. */
+    getEmbedding(id) {
+      const row = db.prepare("SELECT embedding FROM memories WHERE id = ?").get(id);
+      if (!row?.embedding) return undefined;
+      try {
+        const v = JSON.parse(row.embedding);
+        return Array.isArray(v) && v.length ? v : undefined;
+      } catch {
+        return undefined;
+      }
+    },
+
+    /** Cosine search over embedded rows; results carry a 0..1 score. */
+    search(queryVector, { limit = 20, threshold = 0.65 } = {}) {
+      return store.searchVector(queryVector, { limit, threshold });
+    },
+
+    /** Re-embed every row missing an embedding. Returns indexed count. */
+    async rebuildIndex(embedder, { limit = 1000 } = {}) {
+      if (!embedder || typeof embedder.embed !== "function") return { indexed: 0, skipped: 0 };
+      const rows = store.needsEmbedding(limit);
+      let indexed = 0;
+      for (const row of rows) {
+        try {
+          const text = [row.title, row.content].filter(Boolean).join("\n");
+          const vector = await embedder.embedSingle(text);
+          if (vector && vector.length) {
+            store.setEmbedding(row.id, vector);
+            indexed++;
+          }
+        } catch (error) {
+          logger?.warn?.(`[dsh-mneme] rebuild embed failed for ${row.id}: ${String(error)}`);
+        }
+      }
+      if (indexed > 0 && embedder.modelHash) this.markModel(embedder.modelHash, embedder.dimension);
+      return { indexed, skipped: rows.length - indexed };
+    },
+
+    /** Index stats for the Web panel / debugging. */
+    getStats() {
+      return {
+        modelHash: getMeta("model_hash"),
+        dimension: this.dimension(),
+        updatedAt: getMeta("updated_at"),
+        embeddedCount: store.embeddedCount(),
+        totalCount: store.count()
+      };
+    }
+  };
+}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@modusensus/dsh-mneme",
   "description": "Cross-session memory plugin for DeepSeek Harness with autoDream consolidation: SQLite store, Markdown mirrors, 6 model tools, automatic injection, session summarization, user profile/rules, custom slash commands, vector (semantic) search, and a Web GUI panel",
-  "version": "0.1.6",
+  "version": "0.2.0",
   "license": "MIT",
   "type": "module",
   "main": "lib/index.js",
@@ -52,7 +52,11 @@
   "scripts": {
     "sync": "node scripts/sync-lib.js",
     "prepack": "npm run sync",
-    "test": "node --test --test-isolation=none test/*.test.js",
-    "e2e": "node scripts/e2e-dsh.js"
+    "test": "node --test test/*.test.js",
+    "e2e": "node scripts/e2e-dsh.js",
+    "stress": "node scripts/stress-dsh.js"
+  },
+  "dependencies": {
+    "@huggingface/transformers": "^4.2.0"
   }
 }