@modusensus/dsh-mneme 0.1.6 → 0.2.1

package/lib/service.js

@@ -1,174 +1,324 @@
-const INJECT_TYPES = new Set(["preference", "project", "decision", "summary"]);
-
-export function createService({ store, mirror, config, onWrite }) {
-  // Optional dream scheduler hook, installed via setDreamHook after creation
-  // (the scheduler holds a reference back to the service, so it cannot be
-  // passed in the constructor). Fired on the same write events as onWrite.
-  let dreamHook = null;
-
-  // Optional vector embedder, installed via setEmbedder after creation. After
-  // any content write it fire-and-forgets a re-embed of the row so vector
-  // search stays in sync; failures are swallowed inside the embedder.
-  let embedder = null;
-
-  function scheduleEmbed(memory) {
-    if (embedder && memory?.id) {
-      try { embedder.schedule(memory); } catch { /* ignore */ }
-    }
-  }
-
-  /**
-   * Fire-and-forget write notification; errors are swallowed to keep write
-   * paths clean. The store mutation has already committed, so a throwing
-   * subscriber must not surface as a write failure. Archive/forget flags are
-   * state toggles, not content writes, so they never notify.
-   */
-  function notifyWrite() {
-    if (onWrite) {
-      try { onWrite(); } catch { /* ignore */ }
-    }
-    if (dreamHook) {
-      try { dreamHook(); } catch { /* ignore */ }
-    }
-  }
-
-  /**
-   * Save a memory, merging into an existing one when title matches within the same type.
-   * @returns {{action: "created"|"merged", memory: object}}
-   */
-  function saveWithDedupe(memory) {
-    const existing = store
-      .list({ type: memory.type, limit: 100 })
-      .find((m) => m.title.trim() === String(memory.title).trim());
-    if (existing) {
-      const merged = store.update(existing.id, {
-        content: memory.content ?? existing.content,
-        importance: memory.importance ?? existing.importance,
-        tags: memory.tags ?? existing.tags,
-        title: memory.title ?? existing.title
-      });
-      syncMirror();
-      notifyWrite();
-      scheduleEmbed(merged);
-      return { action: "merged", memory: merged };
-    }
-    const created = store.save({
-      type: memory.type,
-      title: memory.title,
-      content: memory.content,
-      tags: memory.tags ?? [],
-      importance: memory.importance ?? 3,
-      source: memory.source ?? "manual"
-    });
-    syncMirror();
-    notifyWrite();
-    scheduleEmbed(created);
-    return { action: "created", memory: created };
-  }
-
-  /**
-   * Candidate memories for automatic context injection:
-   * summaries first, then all preferences, then non-forgotten items with
-   * importance >= threshold. History is never auto-injected. Archived entries
-   * are excluded (store.list already filters them by default; the extra
-   * !m.archived check is kept as double insurance).
-   */
-  function injectCandidates({ maxItems = 5, threshold = 3 } = {}) {
-    const items = store.list({ limit: 200, includeForgotten: false })
-      .filter((m) => !m.archived && INJECT_TYPES.has(m.type) && !m.forgotten &&
-        (m.type === "summary" || m.type === "preference" || m.importance >= threshold))
-      .sort((a, b) => {
-        const pa = a.type === "summary" ? 0 : a.type === "preference" ? 1 : 2;
-        const pb = b.type === "summary" ? 0 : b.type === "preference" ? 1 : 2;
-        return pa - pb || b.importance - a.importance;
-      });
-    return items.slice(0, maxItems);
-  }
-
-  /**
-   * Merge human edits parsed from a mirror file back into the store.
-   * Only content/title are taken; structure fields stay machine-owned.
-   */
-  function mergeHumanEdits(type, edits) {
-    let applied = 0;
-    for (const edit of edits) {
-      if (!edit.id) continue; // corrupt/malformed edit: skip it, keep merging the rest
-      const existing = store.getById(edit.id);
-      if (!existing || existing.type !== type) continue;
-      const patch = {};
-      if (typeof edit.title === "string" && edit.title.trim()) patch.title = edit.title.trim();
-      if (typeof edit.content === "string" && edit.content.trim()) patch.content = edit.content.trim();
-      if (Object.keys(patch).length) {
-        store.update(edit.id, patch);
-        applied++;
-      }
-    }
-    if (applied) {
-      syncMirror();
-      notifyWrite();
-    }
-    return applied;
-  }
-
-  function toApiList(rows) {
-    return rows.map((m) => ({
-      id: m.id,
-      type: m.type,
-      title: m.title,
-      content: m.content,
-      tags: m.tags,
-      importance: m.importance,
-      source: m.source,
-      created_at: m.created_at,
-      updated_at: m.updated_at
-    }));
-  }
-
-  /**
-   * Re-render the human-editable mirror after any store mutation. Only
-   * non-forgotten memories are mirrored: forgotten entries must not reach the
-   * human-editable file (a human "edit" could otherwise resurrect them).
-   */
-  function syncMirror() {
-    if (mirror) mirror.sync(store.list({ limit: 500, includeForgotten: false }));
-  }
-
-  return {
-    saveWithDedupe,
-    injectCandidates,
-    mergeHumanEdits,
-    toApiList,
-    setDreamHook(fn) { dreamHook = fn; },
-    setEmbedder(emb) { embedder = emb; },
-    // passthroughs used by tools and api layers; mutations keep the mirror in sync
-    search: (q, o) => store.search(q, o),
-    searchVector: (v, o) => store.searchVector(v, o),
-    embeddedCount: () => store.embeddedCount(),
-    list: (o) => store.list(o),
-    all: () => store.all(),
-    count: (type) => store.count(type),
-    getById: (id) => store.getById(id),
-    remove: (id) => {
-      store.remove(id);
-      syncMirror();
-      notifyWrite();
-    },
-    update: (id, p) => {
-      const updated = store.update(id, p);
-      syncMirror();
-      notifyWrite();
-      scheduleEmbed(updated);
-      return updated;
-    },
-    setForget: (id, f) => {
-      const updated = store.setForget(id, f);
-      syncMirror();
-      return updated;
-    },
-    setArchived: (id, f) => {
-      const updated = store.setArchived(id, f);
-      syncMirror();
-      return updated;
-    }
-  };
-}
+import { randomUUID } from "node:crypto";
+
+const INJECT_TYPES = new Set(["preference", "project", "decision", "summary"]);
+
+export function createService({ store, mirror, config, onWrite }) {
+  // Optional dream scheduler hook, installed via setDreamHook after creation
+  // (the scheduler holds a reference back to the service, so it cannot be
+  // passed in the constructor). Fired on the same write events as onWrite.
+  let dreamHook = null;
+
+  // Optional vector embedder, installed via setEmbedder after creation. After
+  // any content write it fire-and-forgets a re-embed of the row so vector
+  // search stays in sync; failures are swallowed inside the embedder.
+  let embedder = null;
+  let vectorIndex = null;
+  let reranker = null;
+
+  function scheduleEmbed(memory) {
+    if (embedder && memory?.id) {
+      try { embedder.schedule(memory); } catch { /* ignore */ }
+    }
+  }
+
+  /**
+   * Cross-encoder rerank over a candidate list (best effort). Reranker
+   * failures degrade to the original candidate order — reranking is an
+   * accuracy upgrade, never a correctness gate.
+   */
+  async function rerankCandidates(query, candidates, topK) {
+    if (!reranker || !candidates.length) return candidates.slice(0, topK);
+    try {
+      const scored = await reranker.rerank(query, candidates.map((c) => ({ id: c.id, title: c.title, content: c.content })));
+      if (!Array.isArray(scored)) return candidates.slice(0, topK);
+      const byId = new Map(candidates.map((c) => [c.id, c]));
+      const out = [];
+      for (const s of scored) {
+        const c = byId.get(s.id);
+        if (c) { out.push({ ...c, score: s.score }); if (out.length >= topK) break; }
+      }
+      return out.length ? out : candidates.slice(0, topK);
+    } catch {
+      return candidates.slice(0, topK);
+    }
+  }
+
+  /**
+   * Semantic-aware memory search: keyword recall (store.search) plus optional
+   * vector recall + rerank. mode:
+   *   auto    (default) keyword first, vector fills remaining slots (legacy)
+   *   hybrid  vector first, keyword fills remaining slots
+   *   vector  vector only, falls back to keyword when unavailable
+   *   keyword text only, never touches the embedder
+   * useRerank runs the cross-encoder over the merged list when a reranker is
+   * installed; results carry an extra `score` when reranked.
+   */
+  // Weighted blend factor for hybrid search; exposed so callers can tune it.
+  const DEFAULT_HYBRID_WEIGHTS = { vector: 0.6, keyword: 0.4 };
+
+  /**
+   * Give a keyword-hit row a relevance score in [0,1]: title hits score
+   * higher than content hits, then scaled by importance (1-5). This lets
+   * keyword results participate in weighted hybrid blends.
+   */
+  function scoreKeyword(row, q) {
+    const ql = q.toLowerCase();
+    const title = (row.title ?? "").toLowerCase();
+    const content = (row.content ?? "").toLowerCase();
+    const titleHit = title.includes(ql);
+    const base = titleHit ? 1 : content.includes(ql) ? 0.6 : 0.3;
+    return base * (0.5 + (row.importance ?? 3) / 10);
+  }
+
+  async function searchMemories(query, { mode = "auto", topK = 20, threshold, useRerank = true } = {}) {
+    const q = String(query ?? "").trim();
+    if (!q) return [];
+    const lim = topK > 0 ? topK : 20;
+
+    // Keyword results, decorated with a score so they can be weight-blended
+    // with vector results and reported uniformly.
+    const rawKeyword = store.search(q, { limit: lim });
+    const keyword = rawKeyword.map((m) => ({ ...m, score: scoreKeyword(m, q) }));
+    const wantVector = mode === "vector" || mode === "hybrid" || (mode === "auto" && !!embedder);
+    let vector = [];
+    if (wantVector && embedder) {
+      try {
+        // Legacy embedders expose embed(query); local ones expose embedSingle.
+        const embedSingle = typeof embedder.embedSingle === "function"
+          ? embedder.embedSingle.bind(embedder)
+          : embedder.embed.bind(embedder);
+        const qv = await embedSingle(q);
+        if (qv?.length) {
+          const hits = vectorIndex
+            ? vectorIndex.search(qv, { limit: lim * 2, threshold: threshold ?? 0 })
+            : store.searchVector(qv, { limit: lim * 2, threshold: threshold ?? 0 });
+          vector = hits.map((m) => ({ ...m, vector: true }));
+        }
+      } catch { /* vector unavailable: keep keyword results */ }
+    }
+
+    // Hybrid blending weights from config when provided.
+    const wv = config?.hybridSearchVectorWeight ?? DEFAULT_HYBRID_WEIGHTS.vector;
+    const wk = config?.hybridSearchKeywordWeight ?? DEFAULT_HYBRID_WEIGHTS.keyword;
+
+    let merged;
+    if (mode === "keyword") {
+      merged = keyword;
+    } else if (mode === "vector" || mode === "hybrid") {
+      // semantic-first: vector recalls lead, keyword fills remaining slots.
+      // Weighted blend when both sides scored the same memory; otherwise
+      // vector order leads (it is the semantic signal), keyword backfills.
+      const byId = new Map();
+      for (const m of vector) {
+        const rec = byId.get(m.id);
+        byId.set(m.id, rec ? { ...rec, score: Math.max(rec.score ?? 0, m.score ?? 0) } : m);
+      }
+      for (const m of keyword) {
+        const rec = byId.get(m.id);
+        if (rec) {
+          // Same memory from both sides: blend the scores.
+          byId.set(m.id, { ...rec, score: (rec.score ?? 0) * wv + (m.score ?? 0) * wk });
+        } else {
+          byId.set(m.id, m);
+        }
+      }
+      const ranked = [...byId.values()].sort((a, b) => (b.score ?? 0) - (a.score ?? 0));
+      merged = ranked.slice(0, lim);
+      if (merged.length < lim && !merged.length) {
+        // Vector unavailable entirely: fall back to plain keyword.
+        merged = keyword.slice(0, lim);
+      }
+    } else {
+      // auto: keyword leads, vector fills remaining slots (legacy behavior)
+      merged = keyword.slice(0, lim);
+      const seen = new Set(merged.map((m) => m.id));
+      for (const m of vector) {
+        if (merged.length >= lim) break;
+        if (!seen.has(m.id)) { seen.add(m.id); merged.push(m); }
+      }
+    }
+
+    merged = merged.slice(0, lim);
+    if (useRerank && reranker && merged.length) {
+      return rerankCandidates(q, merged, lim);
+    }
+    return merged;
+  }
+
+  /**
+   * Fire-and-forget write notification; errors are swallowed to keep write
+   * paths clean. The store mutation has already committed, so a throwing
+   * subscriber must not surface as a write failure. Archive/forget flags are
+   * state toggles, not content writes, so they never notify.
+   */
+  function notifyWrite() {
+    if (onWrite) {
+      try { onWrite(); } catch { /* ignore */ }
+    }
+    if (dreamHook) {
+      try { dreamHook(); } catch { /* ignore */ }
+    }
+  }
+
+  /**
+   * Save a memory, merging into an existing one when title matches within the same type.
+   * @returns {{action: "created"|"merged", memory: object}}
+   */
+  function saveWithDedupe(memory) {
+    const existing = store
+      .list({ type: memory.type, limit: 100 })
+      .find((m) => m.title.trim() === String(memory.title).trim());
+    if (existing) {
+      const merged = store.update(existing.id, {
+        content: memory.content ?? existing.content,
+        importance: memory.importance ?? existing.importance,
+        tags: memory.tags ?? existing.tags,
+        title: memory.title ?? existing.title
+      });
+      syncMirror();
+      notifyWrite();
+      scheduleEmbed(merged);
+      return { action: "merged", memory: merged };
+    }
+    const created = store.save({
+      type: memory.type,
+      title: memory.title,
+      content: memory.content,
+      tags: memory.tags ?? [],
+      importance: memory.importance ?? 3,
+      source: memory.source ?? "manual"
+    });
+    syncMirror();
+    notifyWrite();
+    scheduleEmbed(created);
+    return { action: "created", memory: created };
+  }
+
+  /**
+   * Candidate memories for automatic context injection:
+   * summaries first, then all preferences, then non-forgotten items with
+   * importance >= threshold. History is never auto-injected. Archived entries
+   * are excluded (store.list already filters them by default; the extra
+   * !m.archived check is kept as double insurance).
+   */
+  function injectCandidates({ maxItems = 5, threshold = 3 } = {}) {
+    const items = store.list({ limit: 200, includeForgotten: false })
+      .filter((m) => !m.archived && INJECT_TYPES.has(m.type) && !m.forgotten &&
+        (m.type === "summary" || m.type === "preference" || m.importance >= threshold))
+      .sort((a, b) => {
+        const pa = a.type === "summary" ? 0 : a.type === "preference" ? 1 : 2;
+        const pb = b.type === "summary" ? 0 : b.type === "preference" ? 1 : 2;
+        return pa - pb || b.importance - a.importance;
+      });
+    return items.slice(0, maxItems);
+  }
+
+  /**
+   * Merge human edits parsed from a mirror file back into the store.
+   * Only content/title are taken; structure fields stay machine-owned.
+   */
+  function mergeHumanEdits(type, edits) {
+    let applied = 0;
+    for (const edit of edits) {
+      if (!edit.id) continue; // corrupt/malformed edit: skip it, keep merging the rest
+      const existing = store.getById(edit.id);
+      if (!existing || existing.type !== type) continue;
+      const patch = {};
+      if (typeof edit.title === "string" && edit.title.trim()) patch.title = edit.title.trim();
+      if (typeof edit.content === "string" && edit.content.trim()) patch.content = edit.content.trim();
+      if (Object.keys(patch).length) {
+        store.update(edit.id, patch);
+        applied++;
+      }
+    }
+    if (applied) {
+      syncMirror();
+      notifyWrite();
+    }
+    return applied;
+  }
+
+  function toApiList(rows) {
+    return rows.map((m) => ({
+      id: m.id,
+      type: m.type,
+      title: m.title,
+      content: m.content,
+      tags: m.tags,
+      importance: m.importance,
+      source: m.source,
+      created_at: m.created_at,
+      updated_at: m.updated_at
+    }));
+  }
+
+  /**
+   * Re-render the human-editable mirror after any store mutation. Only
+   * non-forgotten memories are mirrored: forgotten entries must not reach the
+   * human-editable file (a human "edit" could otherwise resurrect them).
+   */
+  function syncMirror() {
+    if (mirror) mirror.sync(store.list({ limit: 500, includeForgotten: false }));
+  }
+
+  return {
+    saveWithDedupe,
+    injectCandidates,
+    mergeHumanEdits,
+    toApiList,
+    setDreamHook(fn) { dreamHook = fn; },
+    setEmbedder(emb) { embedder = emb; },
+    setVectorIndex(vi) { vectorIndex = vi; },
+    setReranker(rn) { reranker = rn; },
+    searchMemories,
+    // passthroughs used by tools and api layers; mutations keep the mirror in sync
+    search: (q, o) => store.search(q, o),
+    searchVector: (v, o) => store.searchVector(v, o),
+    embeddedCount: () => store.embeddedCount(),
+    list: (o) => store.list(o),
+    all: () => store.all(),
+    count: (type) => store.count(type),
+    getById: (id) => store.getById(id),
+    remove: (id) => {
+      store.remove(id);
+      syncMirror();
+      notifyWrite();
+    },
+    update: (id, p) => {
+      const old = store.getById(id);
+      const updated = store.update(id, p);
+      // Record a user correction (only when content actually changed and the
+      // reflection failure tracker is enabled): expected = what it became,
+      // actual = what it was before. Feeds later reflection/evolution passes.
+      if (old && updated && config.reflectionFailureTracking && old.content !== updated.content) {
+        store.saveFailure({
+          id: randomUUID(),
+          expected: updated.content,
+          actual: old.content,
+          failure_type: "user_correction",
+          memory_id: id
+        });
+      }
+      syncMirror();
+      notifyWrite();
+      scheduleEmbed(updated);
+      return updated;
+    },
+    setForget: (id, f) => {
+      const updated = store.setForget(id, f);
+      syncMirror();
+      return updated;
+    },
+    setArchived: (id, f) => {
+      const updated = store.setArchived(id, f);
+      syncMirror();
+      return updated;
+    },
+    // autoDream audit trail: passthroughs deliberately bypass write hooks —
+    // an audit write is bookkeeping, and notifyWrite would loop back into the
+    // dream scheduler that just recorded the run.
+    saveDreamRun: (run) => store.saveDreamRun(run),
+    getDreamRun: (id) => store.getDreamRun(id),
+    listDreamRuns: (opts) => store.listDreamRuns(opts)
+  };
+}