@modusensus/dsh-mneme 0.1.6 → 0.2.0

package/lib/reranker.js

@@ -0,0 +1,203 @@
+// Cross-encoder re-ranker for dsh-mneme recall candidates. Uses
+// bge-reranker-base through transformers.js: tries the native `rerank` task
+// first, then the sequence-classification head (sigmoid on the logit delta),
+// and finally feature-extraction over concatenated query+passage (cosine).
+// Every strategy funnels into scorePair(query, passage) so tests can inject a
+// fake scorer and never download a model. Failures throw — the caller degrades
+// back to the original candidate order.
+function hashString(s) {
+  let h = 5381;
+  for (let i = 0; i < s.length; i++) h = ((h << 5) + h + s.charCodeAt(i)) >>> 0;
+  return h.toString(16);
+}
+
+/** Same provider#hash fingerprint convention as the embedders. */
+function modelHash(model) {
+  return `${model}#${hashString(model)}`;
+}
+
+/** Lazy default pipeline factory: dynamic import keeps module load cheap. */
+async function defaultPipelineLoader(task, model, options) {
+  const { pipeline } = await import("@huggingface/transformers");
+  return pipeline(task, model, options);
+}
+
+/** Flatten a transformers.js Tensor [batch, seq, dim] into number[][] rows. */
+function tensorToRows(tensor) {
+  const { data, dims } = tensor;
+  const dim = dims[dims.length - 1] || 0;
+  const rows = [];
+  for (let i = 0; i < data.length; i += dim) rows.push(Array.from(data.subarray(i, i + dim)));
+  if (rows.length === 0 && dim > 0) rows.push(Array.from(data));
+  return rows;
+}
+
+function cosine(a, b) {
+  let dot = 0;
+  let na = 0;
+  let nb = 0;
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i] * b[i];
+    na += a[i] * a[i];
+    nb += b[i] * b[i];
+  }
+  if (na === 0 || nb === 0) return 0;
+  return dot / Math.sqrt(na * nb);
+}
+
+function clamp01(x) {
+  return Math.max(0, Math.min(1, x));
+}
+
+// Pipeline strategies, best to worst. `rerank` is unsupported by the bundled
+// transformers.js (v4.2.0) and rejects cheaply before any model download, so
+// the cascade normally lands on the classification head.
+const STRATEGIES = [
+  ["rerank", "rerank"],
+  ["text-classification", "tc"],
+  ["feature-extraction", "fe"]
+];
+
+export class LocalReranker {
+  constructor(opts = {}) {
+    this.model = opts.model || "Xenova/bge-reranker-base";
+    this.batchSize = opts.batchSize || 8;
+    this.maxCandidates = opts.maxCandidates || 30;
+    this.scoreThreshold = opts.scoreThreshold ?? 0.1;
+    this.device = opts.device || "cpu";
+    this.cacheDir = String(opts.cacheDir ?? "").trim();
+    this.logger = opts.logger ?? null;
+    this.engineFactory = opts.engineFactory || defaultPipelineLoader;
+    // Injectable seam: async (query, passage) => number. When set, init()
+    // skips model loading entirely so tests never hit the network.
+    this.scorePair = opts.scorePair ?? null;
+    this.pipeline = null;
+    this._batchScorer = null;
+    this._queryVec = null;
+  }
+
+  /** Load the model; throws when no strategy can be bound. */
+  async init() {
+    if (this.scorePair) return this;
+    let lastErr = null;
+    for (const [task, strategy] of STRATEGIES) {
+      try {
+        this.pipeline = await this.engineFactory(task, this.model, this._engineOptions());
+        this._bindBatchScorer(strategy);
+        this.logger?.info?.(
+          `[dsh-mneme] local reranker ready: ${this.model} (strategy=${strategy}, device=${this.device})`
+        );
+        return this;
+      } catch (err) {
+        lastErr = err;
+        this.logger?.warn?.(
+          `[dsh-mneme] reranker task "${task}" unavailable for ${this.model}: ${String(err?.message ?? err)}`
+        );
+      }
+    }
+    throw new Error(`LocalReranker failed to load ${this.model}: ${String(lastErr?.message ?? lastErr)}`);
+  }
+
+  _engineOptions() {
+    const options = { device: this.device };
+    if (this.cacheDir) options.cache_dir = this.cacheDir;
+    return options;
+  }
+
+  /** Bind a batch scorer for the chosen strategy. */
+  _bindBatchScorer(strategy) {
+    if (strategy === "rerank") {
+      // Native cross-encoder task (transformers.js >= 4.6): { query, documents }
+      // returns one score per document.
+      this._batchScorer = async (query, passages) => {
+        const out = await this.pipeline({ query, documents: passages });
+        if (!Array.isArray(out)) throw new Error("rerank pipeline returned unexpected output");
+        return out.map((r) => clamp01(typeof r?.score === "number" ? r.score : 0));
+      };
+    } else if (strategy === "tc") {
+      // Classification head: tokenize (query, passage) pairs, score with
+      // sigmoid(l1 - l0) so relevance lands in [0, 1].
+      this._batchScorer = async (query, passages) => {
+        const { tokenizer, model } = this.pipeline;
+        const inputs = tokenizer(passages.map(() => query), {
+          text_pair: passages,
+          padding: true,
+          truncation: true
+        });
+        const logits = await model(inputs).then((o) => o.logits);
+        const dims = logits.dims;
+        const cols = dims[dims.length - 1] || 2;
+        const rows = [];
+        for (let i = 0; i < dims[0]; i++) {
+          const base = i * cols;
+          const l0 = logits.data[base];
+          const l1 = cols > 1 ? logits.data[base + 1] : l0;
+          // sigmoid(l1 - l0) == softmax probability of the positive class.
+          rows.push(clamp01(1 / (1 + Math.exp(l0 - l1))));
+        }
+        return rows;
+      };
+    } else {
+      // Feature extraction: mean-pool the concatenated pair and compare with
+      // the query embedding via cosine. Degraded but model-agnostic.
+      this._batchScorer = async (query, passages) => {
+        if (!this._queryVec) {
+          const t = await this.pipeline([query], { pooling: "mean", normalize: true });
+          this._queryVec = tensorToRows(t)[0];
+        }
+        const t = await this.pipeline(passages.map((p) => `${query}\n${p}`), {
+          pooling: "mean",
+          normalize: true
+        });
+        return tensorToRows(t).map((v) => clamp01(cosine(this._queryVec, v)));
+      };
+    }
+  }
+
+  /** Score one batch of passages; injected scorePair scores pair by pair. */
+  async _scores(query, passages) {
+    if (this._batchScorer) return this._batchScorer(query, passages);
+    return Promise.all(passages.map((p) => (p ? this.scorePair(query, p) : 0)));
+  }
+
+  /**
+   * Re-rank recall candidates. Returns [{ id, score }] filtered by
+   * scoreThreshold and sorted by descending score. Throws on engine failure —
+   * the caller degrades to the original candidate order.
+   */
+  async rerank(query, candidates) {
+    if (!Array.isArray(candidates)) throw new TypeError("rerank expects an array of candidates");
+    const list = candidates.slice(0, this.maxCandidates);
+    if (!list.length) return [];
+    const q = String(query ?? "");
+    const results = [];
+    for (let i = 0; i < list.length; i += this.batchSize) {
+      const chunk = list.slice(i, i + this.batchSize);
+      const passages = chunk.map((c) => [c.title, c.content].filter(Boolean).join("\n"));
+      const scores = await this._scores(q, passages);
+      if (!Array.isArray(scores) || scores.length !== chunk.length) {
+        throw new Error("reranker scorer returned mismatched scores");
+      }
+      for (let j = 0; j < chunk.length; j++) {
+        results.push({ id: chunk[j].id, score: clamp01(Number(scores[j]) || 0) });
+      }
+    }
+    return results
+      .filter((r) => r.score >= this.scoreThreshold)
+      .sort((a, b) => b.score - a.score);
+  }
+
+  get modelHash() {
+    return modelHash(this.model);
+  }
+
+  dispose() {
+    try {
+      this.pipeline?.dispose?.();
+    } catch {
+      // best-effort: some engines free resources on GC
+    }
+    this.pipeline = null;
+    this._queryVec = null;
+  }
+}

package/lib/service.js

@@ -1,174 +1,268 @@
-const INJECT_TYPES = new Set(["preference", "project", "decision", "summary"]);
-
-export function createService({ store, mirror, config, onWrite }) {
-  // Optional dream scheduler hook, installed via setDreamHook after creation
-  // (the scheduler holds a reference back to the service, so it cannot be
-  // passed in the constructor). Fired on the same write events as onWrite.
-  let dreamHook = null;
-
-  // Optional vector embedder, installed via setEmbedder after creation. After
-  // any content write it fire-and-forgets a re-embed of the row so vector
-  // search stays in sync; failures are swallowed inside the embedder.
-  let embedder = null;
-
-  function scheduleEmbed(memory) {
-    if (embedder && memory?.id) {
-      try { embedder.schedule(memory); } catch { /* ignore */ }
-    }
-  }
-
-  /**
-   * Fire-and-forget write notification; errors are swallowed to keep write
-   * paths clean. The store mutation has already committed, so a throwing
-   * subscriber must not surface as a write failure. Archive/forget flags are
-   * state toggles, not content writes, so they never notify.
-   */
-  function notifyWrite() {
-    if (onWrite) {
-      try { onWrite(); } catch { /* ignore */ }
-    }
-    if (dreamHook) {
-      try { dreamHook(); } catch { /* ignore */ }
-    }
-  }
-
-  /**
-   * Save a memory, merging into an existing one when title matches within the same type.
-   * @returns {{action: "created"|"merged", memory: object}}
-   */
-  function saveWithDedupe(memory) {
-    const existing = store
-      .list({ type: memory.type, limit: 100 })
-      .find((m) => m.title.trim() === String(memory.title).trim());
-    if (existing) {
-      const merged = store.update(existing.id, {
-        content: memory.content ?? existing.content,
-        importance: memory.importance ?? existing.importance,
-        tags: memory.tags ?? existing.tags,
-        title: memory.title ?? existing.title
-      });
-      syncMirror();
-      notifyWrite();
-      scheduleEmbed(merged);
-      return { action: "merged", memory: merged };
-    }
-    const created = store.save({
-      type: memory.type,
-      title: memory.title,
-      content: memory.content,
-      tags: memory.tags ?? [],
-      importance: memory.importance ?? 3,
-      source: memory.source ?? "manual"
-    });
-    syncMirror();
-    notifyWrite();
-    scheduleEmbed(created);
-    return { action: "created", memory: created };
-  }
-
-  /**
-   * Candidate memories for automatic context injection:
-   * summaries first, then all preferences, then non-forgotten items with
-   * importance >= threshold. History is never auto-injected. Archived entries
-   * are excluded (store.list already filters them by default; the extra
-   * !m.archived check is kept as double insurance).
-   */
-  function injectCandidates({ maxItems = 5, threshold = 3 } = {}) {
-    const items = store.list({ limit: 200, includeForgotten: false })
-      .filter((m) => !m.archived && INJECT_TYPES.has(m.type) && !m.forgotten &&
-        (m.type === "summary" || m.type === "preference" || m.importance >= threshold))
-      .sort((a, b) => {
-        const pa = a.type === "summary" ? 0 : a.type === "preference" ? 1 : 2;
-        const pb = b.type === "summary" ? 0 : b.type === "preference" ? 1 : 2;
-        return pa - pb || b.importance - a.importance;
-      });
-    return items.slice(0, maxItems);
-  }
-
-  /**
-   * Merge human edits parsed from a mirror file back into the store.
-   * Only content/title are taken; structure fields stay machine-owned.
-   */
-  function mergeHumanEdits(type, edits) {
-    let applied = 0;
-    for (const edit of edits) {
-      if (!edit.id) continue; // corrupt/malformed edit: skip it, keep merging the rest
-      const existing = store.getById(edit.id);
-      if (!existing || existing.type !== type) continue;
-      const patch = {};
-      if (typeof edit.title === "string" && edit.title.trim()) patch.title = edit.title.trim();
-      if (typeof edit.content === "string" && edit.content.trim()) patch.content = edit.content.trim();
-      if (Object.keys(patch).length) {
-        store.update(edit.id, patch);
-        applied++;
-      }
-    }
-    if (applied) {
-      syncMirror();
-      notifyWrite();
-    }
-    return applied;
-  }
-
-  function toApiList(rows) {
-    return rows.map((m) => ({
-      id: m.id,
-      type: m.type,
-      title: m.title,
-      content: m.content,
-      tags: m.tags,
-      importance: m.importance,
-      source: m.source,
-      created_at: m.created_at,
-      updated_at: m.updated_at
-    }));
-  }
-
-  /**
-   * Re-render the human-editable mirror after any store mutation. Only
-   * non-forgotten memories are mirrored: forgotten entries must not reach the
-   * human-editable file (a human "edit" could otherwise resurrect them).
-   */
-  function syncMirror() {
-    if (mirror) mirror.sync(store.list({ limit: 500, includeForgotten: false }));
-  }
-
-  return {
-    saveWithDedupe,
-    injectCandidates,
-    mergeHumanEdits,
-    toApiList,
-    setDreamHook(fn) { dreamHook = fn; },
-    setEmbedder(emb) { embedder = emb; },
-    // passthroughs used by tools and api layers; mutations keep the mirror in sync
-    search: (q, o) => store.search(q, o),
-    searchVector: (v, o) => store.searchVector(v, o),
-    embeddedCount: () => store.embeddedCount(),
-    list: (o) => store.list(o),
-    all: () => store.all(),
-    count: (type) => store.count(type),
-    getById: (id) => store.getById(id),
-    remove: (id) => {
-      store.remove(id);
-      syncMirror();
-      notifyWrite();
-    },
-    update: (id, p) => {
-      const updated = store.update(id, p);
-      syncMirror();
-      notifyWrite();
-      scheduleEmbed(updated);
-      return updated;
-    },
-    setForget: (id, f) => {
-      const updated = store.setForget(id, f);
-      syncMirror();
-      return updated;
-    },
-    setArchived: (id, f) => {
-      const updated = store.setArchived(id, f);
-      syncMirror();
-      return updated;
-    }
-  };
-}
+const INJECT_TYPES = new Set(["preference", "project", "decision", "summary"]);
+
+export function createService({ store, mirror, config, onWrite }) {
+  // Optional dream scheduler hook, installed via setDreamHook after creation
+  // (the scheduler holds a reference back to the service, so it cannot be
+  // passed in the constructor). Fired on the same write events as onWrite.
+  let dreamHook = null;
+
+  // Optional vector embedder, installed via setEmbedder after creation. After
+  // any content write it fire-and-forgets a re-embed of the row so vector
+  // search stays in sync; failures are swallowed inside the embedder.
+  let embedder = null;
+  let vectorIndex = null;
+  let reranker = null;
+
+  function scheduleEmbed(memory) {
+    if (embedder && memory?.id) {
+      try { embedder.schedule(memory); } catch { /* ignore */ }
+    }
+  }
+
+  /**
+   * Cross-encoder rerank over a candidate list (best effort). Reranker
+   * failures degrade to the original candidate order — reranking is an
+   * accuracy upgrade, never a correctness gate.
+   */
+  async function rerankCandidates(query, candidates, topK) {
+    if (!reranker || !candidates.length) return candidates.slice(0, topK);
+    try {
+      const scored = await reranker.rerank(query, candidates.map((c) => ({ id: c.id, title: c.title, content: c.content })));
+      if (!Array.isArray(scored)) return candidates.slice(0, topK);
+      const byId = new Map(candidates.map((c) => [c.id, c]));
+      const out = [];
+      for (const s of scored) {
+        const c = byId.get(s.id);
+        if (c) { out.push({ ...c, score: s.score }); if (out.length >= topK) break; }
+      }
+      return out.length ? out : candidates.slice(0, topK);
+    } catch {
+      return candidates.slice(0, topK);
+    }
+  }
+
+  /**
+   * Semantic-aware memory search: keyword recall (store.search) plus optional
+   * vector recall + rerank. mode:
+   *   auto    (default) keyword first, vector fills remaining slots (legacy)
+   *   hybrid  vector first, keyword fills remaining slots
+   *   vector  vector only, falls back to keyword when unavailable
+   *   keyword text only, never touches the embedder
+   * useRerank runs the cross-encoder over the merged list when a reranker is
+   * installed; results carry an extra `score` when reranked.
+   */
+  async function searchMemories(query, { mode = "auto", topK = 20, threshold, useRerank = true } = {}) {
+    const q = String(query ?? "").trim();
+    if (!q) return [];
+    const lim = topK > 0 ? topK : 20;
+
+    const keyword = store.search(q, { limit: lim });
+    const wantVector = mode === "vector" || mode === "hybrid" || (mode === "auto" && !!embedder);
+    let vector = [];
+    if (wantVector && embedder) {
+      try {
+        // Legacy embedders expose embed(query); local ones expose embedSingle.
+        const embedSingle = typeof embedder.embedSingle === "function"
+          ? embedder.embedSingle.bind(embedder)
+          : embedder.embed.bind(embedder);
+        const qv = await embedSingle(q);
+        if (qv?.length) {
+          vector = vectorIndex
+            ? vectorIndex.search(qv, { limit: lim * 2, threshold: threshold ?? 0 })
+            : store.searchVector(qv, { limit: lim * 2, threshold: threshold ?? 0 });
+        }
+      } catch { /* vector unavailable: keep keyword results */ }
+    }
+
+    let merged;
+    if (mode === "keyword") {
+      merged = keyword;
+    } else if (mode === "vector" || mode === "hybrid") {
+      // semantic-first: vector recalls lead, keyword fills remaining slots
+      merged = vector.length ? vector.slice(0, lim) : keyword;
+      const seen = new Set(merged.map((m) => m.id));
+      for (const m of keyword) {
+        if (merged.length >= lim) break;
+        if (!seen.has(m.id)) { seen.add(m.id); merged.push(m); }
+      }
+    } else {
+      // auto: keyword leads, vector fills remaining slots (legacy behavior)
+      merged = keyword.slice(0, lim);
+      const seen = new Set(merged.map((m) => m.id));
+      for (const m of vector) {
+        if (merged.length >= lim) break;
+        if (!seen.has(m.id)) { seen.add(m.id); merged.push(m); }
+      }
+    }
+
+    merged = merged.slice(0, lim);
+    if (useRerank && reranker && merged.length) {
+      return rerankCandidates(q, merged, lim);
+    }
+    return merged;
+  }
+
+  /**
+   * Fire-and-forget write notification; errors are swallowed to keep write
+   * paths clean. The store mutation has already committed, so a throwing
+   * subscriber must not surface as a write failure. Archive/forget flags are
+   * state toggles, not content writes, so they never notify.
+   */
+  function notifyWrite() {
+    if (onWrite) {
+      try { onWrite(); } catch { /* ignore */ }
+    }
+    if (dreamHook) {
+      try { dreamHook(); } catch { /* ignore */ }
+    }
+  }
+
+  /**
+   * Save a memory, merging into an existing one when title matches within the same type.
+   * @returns {{action: "created"|"merged", memory: object}}
+   */
+  function saveWithDedupe(memory) {
+    const existing = store
+      .list({ type: memory.type, limit: 100 })
+      .find((m) => m.title.trim() === String(memory.title).trim());
+    if (existing) {
+      const merged = store.update(existing.id, {
+        content: memory.content ?? existing.content,
+        importance: memory.importance ?? existing.importance,
+        tags: memory.tags ?? existing.tags,
+        title: memory.title ?? existing.title
+      });
+      syncMirror();
+      notifyWrite();
+      scheduleEmbed(merged);
+      return { action: "merged", memory: merged };
+    }
+    const created = store.save({
+      type: memory.type,
+      title: memory.title,
+      content: memory.content,
+      tags: memory.tags ?? [],
+      importance: memory.importance ?? 3,
+      source: memory.source ?? "manual"
+    });
+    syncMirror();
+    notifyWrite();
+    scheduleEmbed(created);
+    return { action: "created", memory: created };
+  }
+
+  /**
+   * Candidate memories for automatic context injection:
+   * summaries first, then all preferences, then non-forgotten items with
+   * importance >= threshold. History is never auto-injected. Archived entries
+   * are excluded (store.list already filters them by default; the extra
+   * !m.archived check is kept as double insurance).
+   */
+  function injectCandidates({ maxItems = 5, threshold = 3 } = {}) {
+    const items = store.list({ limit: 200, includeForgotten: false })
+      .filter((m) => !m.archived && INJECT_TYPES.has(m.type) && !m.forgotten &&
+        (m.type === "summary" || m.type === "preference" || m.importance >= threshold))
+      .sort((a, b) => {
+        const pa = a.type === "summary" ? 0 : a.type === "preference" ? 1 : 2;
+        const pb = b.type === "summary" ? 0 : b.type === "preference" ? 1 : 2;
+        return pa - pb || b.importance - a.importance;
+      });
+    return items.slice(0, maxItems);
+  }
+
+  /**
+   * Merge human edits parsed from a mirror file back into the store.
+   * Only content/title are taken; structure fields stay machine-owned.
+   */
+  function mergeHumanEdits(type, edits) {
+    let applied = 0;
+    for (const edit of edits) {
+      if (!edit.id) continue; // corrupt/malformed edit: skip it, keep merging the rest
+      const existing = store.getById(edit.id);
+      if (!existing || existing.type !== type) continue;
+      const patch = {};
+      if (typeof edit.title === "string" && edit.title.trim()) patch.title = edit.title.trim();
+      if (typeof edit.content === "string" && edit.content.trim()) patch.content = edit.content.trim();
+      if (Object.keys(patch).length) {
+        store.update(edit.id, patch);
+        applied++;
+      }
+    }
+    if (applied) {
+      syncMirror();
+      notifyWrite();
+    }
+    return applied;
+  }
+
+  function toApiList(rows) {
+    return rows.map((m) => ({
+      id: m.id,
+      type: m.type,
+      title: m.title,
+      content: m.content,
+      tags: m.tags,
+      importance: m.importance,
+      source: m.source,
+      created_at: m.created_at,
+      updated_at: m.updated_at
+    }));
+  }
+
+  /**
+   * Re-render the human-editable mirror after any store mutation. Only
+   * non-forgotten memories are mirrored: forgotten entries must not reach the
+   * human-editable file (a human "edit" could otherwise resurrect them).
+   */
+  function syncMirror() {
+    if (mirror) mirror.sync(store.list({ limit: 500, includeForgotten: false }));
+  }
+
+  return {
+    saveWithDedupe,
+    injectCandidates,
+    mergeHumanEdits,
+    toApiList,
+    setDreamHook(fn) { dreamHook = fn; },
+    setEmbedder(emb) { embedder = emb; },
+    setVectorIndex(vi) { vectorIndex = vi; },
+    setReranker(rn) { reranker = rn; },
+    searchMemories,
+    // passthroughs used by tools and api layers; mutations keep the mirror in sync
+    search: (q, o) => store.search(q, o),
+    searchVector: (v, o) => store.searchVector(v, o),
+    embeddedCount: () => store.embeddedCount(),
+    list: (o) => store.list(o),
+    all: () => store.all(),
+    count: (type) => store.count(type),
+    getById: (id) => store.getById(id),
+    remove: (id) => {
+      store.remove(id);
+      syncMirror();
+      notifyWrite();
+    },
+    update: (id, p) => {
+      const updated = store.update(id, p);
+      syncMirror();
+      notifyWrite();
+      scheduleEmbed(updated);
+      return updated;
+    },
+    setForget: (id, f) => {
+      const updated = store.setForget(id, f);
+      syncMirror();
+      return updated;
+    },
+    setArchived: (id, f) => {
+      const updated = store.setArchived(id, f);
+      syncMirror();
+      return updated;
+    },
+    // autoDream audit trail: passthroughs deliberately bypass write hooks —
+    // an audit write is bookkeeping, and notifyWrite would loop back into the
+    // dream scheduler that just recorded the run.
+    saveDreamRun: (run) => store.saveDreamRun(run),
+    getDreamRun: (id) => store.getDreamRun(id),
+    listDreamRuns: (opts) => store.listDreamRuns(opts)
+  };
+}