@prom.codes/memory-mcp 0.7.3 → 0.8.0

package/README.md

@@ -33,9 +33,10 @@ session end.
   `roots`) — no need to set `PROMETHEUS_WORKSPACE_ROOT`.
 - **Optional quality levers** (documented): `PROMETHEUS_MEMORY_RERANK_PROVIDER`
   (cross-encoder rerank — reranked recall ties full-context at ~28× fewer tokens
-  on LoCoMo) and `PROMETHEUS_MEMORY_REWRITE_PROVIDER` (HyDE). Temporal-intent
-  ranking ("latest/earliest" queries) is **on by default**; disable with
-  `PROMETHEUS_MEMORY_TEMPORAL=off`.
+  on LoCoMo), `PROMETHEUS_MEMORY_REWRITE_PROVIDER` (HyDE), and
+  `PROMETHEUS_MEMORY_DEDUP=on` (collapse restatements so distinct facts fill the
+  top-k). Temporal-intent ranking ("latest/earliest" queries) is **on by
+  default**; disable with `PROMETHEUS_MEMORY_TEMPORAL=off`.
 
 If a fresh window opens with no project, the workspace falls back to the host
 cwd (often home); in that case memory still works but project memories are NOT

package/dist/bin.js

@@ -1360,6 +1360,59 @@ function applyTemporalRanking(hits, intent, getTimestampMs, weight = DEFAULT_TEM
   return scored.map((s) => s.hit);
 }
 
+// dist/dedup.js
+function normalizeForDedup(value) {
+  return value.toLowerCase().replace(/[^\p{L}\p{N}\s]/gu, " ").replace(/\s+/g, " ").trim();
+}
+function cosineSim(a, b) {
+  if (a.length !== b.length)
+    return 0;
+  let dot = 0;
+  let na = 0;
+  let nb = 0;
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i] * b[i];
+    na += a[i] * a[i];
+    nb += b[i] * b[i];
+  }
+  if (na === 0 || nb === 0)
+    return 0;
+  return dot / (Math.sqrt(na) * Math.sqrt(nb));
+}
+var DEFAULT_DEDUP_THRESHOLD = 0.92;
+function dedupe(items, getText, getVector, threshold = DEFAULT_DEDUP_THRESHOLD) {
+  if (items.length <= 1)
+    return { kept: [...items], merged: 0 };
+  const kept = [];
+  const keptNorm = [];
+  const keptVec = [];
+  let merged = 0;
+  for (const item of items) {
+    const norm = normalizeForDedup(getText(item));
+    const vec = getVector(item);
+    let isDup = false;
+    for (let i = 0; i < kept.length; i++) {
+      if (norm.length > 0 && norm === keptNorm[i]) {
+        isDup = true;
+        break;
+      }
+      const kv = keptVec[i];
+      if (threshold > 0 && vec !== void 0 && kv !== void 0 && cosineSim(vec, kv) >= threshold) {
+        isDup = true;
+        break;
+      }
+    }
+    if (isDup) {
+      merged += 1;
+      continue;
+    }
+    kept.push(item);
+    keptNorm.push(norm);
+    keptVec.push(vec);
+  }
+  return { kept, merged };
+}
+
 // dist/types.js
 var MEMORY_SCOPES = [
   "system",
@@ -1535,6 +1588,8 @@ var SqliteMemoryBackend = class {
   rewriter;
   temporalEnabled;
   temporalWeight;
+  dedupEnabled;
+  dedupThreshold;
   /** Record ids whose vector is missing/stale, awaiting a batched embed. */
   pendingEmbed = /* @__PURE__ */ new Set();
   closed = false;
@@ -1553,6 +1608,8 @@ var SqliteMemoryBackend = class {
     this.rewriter = opts.rewriter;
     this.temporalEnabled = opts.temporal?.enabled ?? false;
     this.temporalWeight = opts.temporal?.weight ?? DEFAULT_TEMPORAL_WEIGHT;
+    this.dedupEnabled = opts.dedup?.enabled ?? false;
+    this.dedupThreshold = opts.dedup?.threshold ?? DEFAULT_DEDUP_THRESHOLD;
     if (this.embedder !== void 0)
       this.queueUnembedded();
   }
@@ -1597,6 +1654,35 @@ var SqliteMemoryBackend = class {
     const row = this.db.prepare(`SELECT * FROM agent_memory WHERE id = ?`).get(id);
     return rowToRecord(row);
   }
+  /** Fetch stored vectors for a set of record ids (missing → absent from map). */
+  vectorsByIds(ids) {
+    const out = /* @__PURE__ */ new Map();
+    if (ids.length === 0)
+      return out;
+    const placeholders = ids.map(() => "?").join(", ");
+    const rows = this.db.prepare(`SELECT record_id, vector FROM agent_memory_vec WHERE record_id IN (${placeholders})`).all(...ids);
+    for (const r of rows)
+      out.set(r.record_id, blobToVector(r.vector));
+    return out;
+  }
+  /**
+   * M6: collapse near-duplicate records, keeping the first (highest-priority).
+   * No-op when dedup is disabled or there is ≤1 record. Uses stored vectors for
+   * the cosine signal when present (text-equality always applies).
+   */
+  dedupeRecords(records) {
+    if (!this.dedupEnabled || records.length <= 1)
+      return records;
+    const vectors = this.vectorsByIds(records.map((r) => r.id));
+    return dedupe(records, (r) => r.value, (r) => vectors.get(r.id), this.dedupThreshold).kept;
+  }
+  /** As {@link dedupeRecords}, but over search hits (caller gates on the flag). */
+  dedupeHitPool(hits) {
+    if (hits.length <= 1)
+      return hits;
+    const vectors = this.vectorsByIds(hits.map((h) => h.record.id));
+    return dedupe(hits, (h) => h.record.value, (h) => vectors.get(h.record.id), this.dedupThreshold).kept;
+  }
   async read(query) {
     if (query.chain.length === 0)
       return [];
@@ -1610,7 +1696,8 @@ var SqliteMemoryBackend = class {
     sql += ` ORDER BY updated_at DESC`;
     const rows = this.db.prepare(sql).all(...params);
     const resolved = resolveScopeChain(rows.map(rowToRecord), query.chain);
-    const limited = query.limit !== void 0 ? resolved.slice(0, query.limit) : resolved;
+    const deduped = this.dedupeRecords(resolved);
+    const limited = query.limit !== void 0 ? deduped.slice(0, query.limit) : deduped;
     const bump = this.db.prepare(`UPDATE agent_memory SET use_count = use_count + 1 WHERE id = ?`);
     for (const rec of limited) {
       bump.run(rec.id);
@@ -1675,7 +1762,9 @@ var SqliteMemoryBackend = class {
         { id: "vec", items: vecHits.map((h) => ({ key: h.record.id, payload: h })) }
       ], { limit: poolLimit }).map((f) => f.payload);
     }
-    const reranked = input.rerank === false ? pool : await this.rerankPool(input.query, pool, finalLimit);
+    const dedupOn = input.dedup ?? this.dedupEnabled;
+    const deduped = dedupOn ? this.dedupeHitPool(pool) : pool;
+    const reranked = input.rerank === false ? deduped : await this.rerankPool(input.query, deduped, finalLimit);
     const temporalOn = input.temporal ?? this.temporalEnabled;
     const ordered = temporalOn ? this.applyTemporal(input.query, reranked) : reranked;
     return ordered.slice(0, finalLimit);
@@ -2168,6 +2257,18 @@ function discoverMemoryTemporal(env) {
   }
   return { enabled, weight };
 }
+function discoverMemoryDedup(env) {
+  const raw = (env.PROMETHEUS_MEMORY_DEDUP ?? "off").toLowerCase();
+  const enabled = raw === "on" || raw === "1" || raw === "true" || raw === "yes";
+  const rawThreshold = env.PROMETHEUS_MEMORY_DEDUP_THRESHOLD;
+  let threshold = DEFAULT_DEDUP_THRESHOLD;
+  if (rawThreshold !== void 0 && rawThreshold !== "") {
+    const n = Number.parseFloat(rawThreshold);
+    if (Number.isFinite(n) && n > 0 && n <= 1)
+      threshold = n;
+  }
+  return { enabled, threshold };
+}
 function composeFromEnv(opts) {
   const env = opts.env;
   const override = (opts.workspaceRootOverride ?? "").trim();
@@ -2183,11 +2284,13 @@ function composeFromEnv(opts) {
   const { id: extractorId, provider: extractor } = discoverMemoryExtractor(env);
   const { id: rewriterId, provider: rewriter } = discoverMemoryRewriter(env);
   const temporal = discoverMemoryTemporal(env);
+  const dedup = discoverMemoryDedup(env);
   const backend = new SqliteMemoryBackend(dbPath, {
     ...embedder !== void 0 ? { embedder } : {},
     ...reranker !== null ? { reranker } : {},
     ...rewriter !== null ? { rewriter } : {},
-    temporal
+    temporal,
+    dedup
   });
   return {
     backend,
@@ -2205,6 +2308,7 @@ function composeFromEnv(opts) {
     rewriter,
     rewriterId,
     temporalEnabled: temporal.enabled,
+    dedupEnabled: dedup.enabled,
     rootIsHomeOrFsRoot: isHomeOrFilesystemRoot(workspaceRoot),
     close: () => backend.close()
   };
@@ -2832,6 +2936,7 @@ ${f.value}`);
         rerank: deps.rerankerId,
         rewrite: deps.rewriterId,
         temporal: deps.temporalEnabled,
+        dedup: deps.dedupEnabled,
         extract: deps.extractorId
       },
       summary
@@ -2882,7 +2987,7 @@ async function main() {
       env,
       ...override !== void 0 && override !== "" ? { workspaceRootOverride: override } : {}
     });
-    process.stderr.write(`prometheus-memory-mcp: workspace=${composed.workspaceRoot} (via ${via}) project=${composed.projectName} (${composed.projectId}) db=${composed.dbPath} embed=${composed.embedderId}${composed.embeddingsEnabled ? "" : " (keyword-only)"} rerank=${composed.rerankerId} extract=${composed.extractorId} rewrite=${composed.rewriterId} temporal=${composed.temporalEnabled ? "on" : "off"}
+    process.stderr.write(`prometheus-memory-mcp: workspace=${composed.workspaceRoot} (via ${via}) project=${composed.projectName} (${composed.projectId}) db=${composed.dbPath} embed=${composed.embedderId}${composed.embeddingsEnabled ? "" : " (keyword-only)"} rerank=${composed.rerankerId} extract=${composed.extractorId} rewrite=${composed.rewriterId} temporal=${composed.temporalEnabled ? "on" : "off"} dedup=${composed.dedupEnabled ? "on" : "off"}
 `);
     if (composed.rootIsHomeOrFsRoot) {
       process.stderr.write(`prometheus-memory-mcp: workspace resolved to ${composed.workspaceRoot} (your home directory or a filesystem root) \u2014 project memories will NOT be mirrored to markdown there. Open a project folder (Claude Code passes it via CLAUDE_PROJECT_DIR) or set PROMETHEUS_WORKSPACE_ROOT. Call memory_status for details.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@prom.codes/memory-mcp",
-  "version": "0.7.3",
+  "version": "0.8.0",
   "description": "prom.codes Memory — persistent, local-first agent memory as an MCP server.",
   "type": "module",
   "bin": {