dsh-cortex-memory 0.1.0 → 0.2.0-rc.11

package/src/vector/hnsw.js

@@ -0,0 +1,246 @@
+/**
+ * Small dependency-free HNSW index for Cortex.
+ *
+ * This is a candidate replacement for the single-layer NSW in ann.js. It is
+ * intentionally not wired into the production vector index until a benchmark
+ * shows acceptable recall. The implementation uses simple sorted frontiers
+ * (not a binary heap) because it is easier to audit; this is a benchmark
+ * candidate, not a final large-corpus engine.
+ */
+function mulberry32(seed) {
+  let state = seed >>> 0;
+  return () => {
+    state = (state + 0x6d2b79f5) >>> 0;
+    let t = state;
+    t = Math.imul(t ^ (t >>> 15), t | 1);
+    t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
+    return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+  };
+}
+
+function cosine(a, b) {
+  let dot = 0;
+  let na = 0;
+  let nb = 0;
+  const n = Math.min(a.length, b.length);
+  for (let i = 0; i < n; i += 1) {
+    dot += a[i] * b[i];
+    na += a[i] * a[i];
+    nb += b[i] * b[i];
+  }
+  if (!na || !nb) return 0;
+  return dot / Math.sqrt(na * nb);
+}
+
+function distance(a, b) {
+  return 1 - cosine(a, b);
+}
+
+function createHeap(compare) {
+  const items = [];
+  const swap = (a, b) => { const value = items[a]; items[a] = items[b]; items[b] = value; };
+  function bubbleUp(index) {
+    while (index > 0) {
+      const parent = (index - 1) >> 1;
+      if (compare(items[index], items[parent]) >= 0) break;
+      swap(index, parent);
+      index = parent;
+    }
+  }
+  function bubbleDown(index) {
+    for (;;) {
+      const left = index * 2 + 1;
+      const right = left + 1;
+      let best = index;
+      if (left < items.length && compare(items[left], items[best]) < 0) best = left;
+      if (right < items.length && compare(items[right], items[best]) < 0) best = right;
+      if (best === index) break;
+      swap(index, best);
+      index = best;
+    }
+  }
+  return {
+    push(item) { items.push(item); bubbleUp(items.length - 1); },
+    pop() {
+      if (!items.length) return undefined;
+      const first = items[0];
+      const last = items.pop();
+      if (items.length) { items[0] = last; bubbleDown(0); }
+      return first;
+    },
+    peek() { return items[0]; },
+    get size() { return items.length; },
+    toArray() { return items.slice(); },
+  };
+}
+
+export function createHnswIndex({ M = 16, efConstruction = 100, efSearch = 64, seed = 42, maxLevel = 16 } = {}) {
+  const maxNeighbors = Math.max(2, Math.min(64, Number(M) || 16));
+  const constructionWidth = Math.max(maxNeighbors, Number(efConstruction) || 100);
+  const searchWidth = Math.max(10, Number(efSearch) || 64);
+  const levelCap = Math.max(1, Math.min(32, Number(maxLevel) || 16));
+  const levelNormalizer = 1 / Math.log(Math.max(2, maxNeighbors));
+  const random = mulberry32(seed);
+  const nodes = new Map();
+  let entry = null;
+  let topLevel = -1;
+
+  function randomLevel() {
+    const value = -Math.log(Math.max(1e-12, random())) * levelNormalizer;
+    return Math.min(levelCap, Math.floor(value));
+  }
+
+  function nearestInLayer(query, startId, layer) {
+    let current = nodes.get(startId);
+    if (!current) return null;
+    let currentDistance = distance(query, current.vector);
+    let improved = true;
+    while (improved) {
+      improved = false;
+      for (const neighborId of current.neighbors[layer] ?? []) {
+        const neighbor = nodes.get(neighborId);
+        if (!neighbor) continue;
+        const candidateDistance = distance(query, neighbor.vector);
+        if (candidateDistance < currentDistance) {
+          current = neighbor;
+          currentDistance = candidateDistance;
+          improved = true;
+        }
+      }
+    }
+    return { id: current.id, d: currentDistance };
+  }
+
+  function searchLayer(query, entryPoints, width, layer) {
+    const visited = new Set();
+    const candidates = createHeap((a, b) => a.d - b.d);
+    const results = createHeap((a, b) => b.d - a.d);
+    for (const id of entryPoints) {
+      if (visited.has(id)) continue;
+      const node = nodes.get(id);
+      if (!node) continue;
+      const d = distance(query, node.vector);
+      visited.add(id);
+      candidates.push({ id, d });
+      results.push({ id, d });
+    }
+    while (candidates.size) {
+      const current = candidates.pop();
+      const worst = results.size >= width ? results.peek().d : Infinity;
+      if (current.d > worst && results.size >= width) break;
+      const node = nodes.get(current.id);
+      if (!node) continue;
+      for (const neighborId of node.neighbors[layer] ?? []) {
+        if (visited.has(neighborId)) continue;
+        visited.add(neighborId);
+        const neighbor = nodes.get(neighborId);
+        if (!neighbor) continue;
+        const d = distance(query, neighbor.vector);
+        if (results.size >= width && d >= results.peek().d) continue;
+        candidates.push({ id: neighborId, d });
+        results.push({ id: neighborId, d });
+        if (results.size > width) results.pop();
+      }
+    }
+    return results.toArray().sort((a, b) => a.d - b.d);
+  }
+
+  function connect(node, candidates, layer) {
+    const selected = candidates
+      .filter((item) => item.id !== node.id && nodes.has(item.id))
+      .sort((a, b) => a.d - b.d)
+      .slice(0, maxNeighbors);
+    node.neighbors[layer] = selected.map((item) => item.id);
+    for (const candidate of selected) {
+      const other = nodes.get(candidate.id);
+      if (!other) continue;
+      const neighbors = other.neighbors[layer] ?? [];
+      if (!neighbors.includes(node.id)) neighbors.push(node.id);
+      if (neighbors.length > maxNeighbors) {
+        neighbors.sort((a, b) => distance(other.vector, nodes.get(a)?.vector ?? []) - distance(other.vector, nodes.get(b)?.vector ?? []));
+        neighbors.length = maxNeighbors;
+      }
+      other.neighbors[layer] = neighbors;
+    }
+  }
+
+  function upsert(id, vector) {
+    if (!id || !Array.isArray(vector) || !vector.length) throw new Error('hnsw upsert needs id and vector');
+    const existing = nodes.get(id);
+    if (existing) {
+      existing.vector = vector.slice();
+      return existing;
+    }
+    const node = { id, vector: vector.slice(), level: randomLevel(), neighbors: [] };
+    nodes.set(id, node);
+    if (!entry) {
+      entry = id;
+      topLevel = node.level;
+      return node;
+    }
+    let current = entry;
+    for (let layer = topLevel; layer > node.level; layer -= 1) {
+      const nearest = nearestInLayer(node.vector, current, layer);
+      if (nearest) current = nearest.id;
+    }
+    for (let layer = Math.min(node.level, topLevel); layer >= 0; layer -= 1) {
+      const candidates = searchLayer(node.vector, [current], constructionWidth, layer);
+      connect(node, candidates, layer);
+      if (candidates.length) current = candidates[0].id;
+    }
+    if (node.level > topLevel) {
+      entry = id;
+      topLevel = node.level;
+    }
+    return node;
+  }
+
+  function search(query, k = 10, { ef = searchWidth } = {}) {
+    if (!entry || !nodes.size) return [];
+    const width = Math.max(Math.max(1, Number(k) || 10), Number(ef) || searchWidth);
+    let current = entry;
+    for (let layer = topLevel; layer > 0; layer -= 1) {
+      const nearest = nearestInLayer(query, current, layer);
+      if (nearest) current = nearest.id;
+    }
+    const ranked = searchLayer(query, [current], width, 0);
+    return ranked.slice(0, Math.max(1, Number(k) || 10)).map((item) => ({ id: item.id, score: 1 - item.d }));
+  }
+
+  function remove(id) {
+    const node = nodes.get(id);
+    if (!node) return false;
+    for (const other of nodes.values()) {
+      for (const neighbors of other.neighbors) {
+        const index = neighbors.indexOf(id);
+        if (index >= 0) neighbors.splice(index, 1);
+      }
+    }
+    nodes.delete(id);
+    if (entry === id) {
+      const first = nodes.values().next().value;
+      entry = first?.id ?? null;
+      topLevel = -1;
+      for (const other of nodes.values()) topLevel = Math.max(topLevel, other.level);
+    }
+    return true;
+  }
+
+  function rebuild(entries = []) {
+    nodes.clear();
+    entry = null;
+    topLevel = -1;
+    for (const item of entries) upsert(item.id, item.vector);
+    return nodes.size;
+  }
+
+  return {
+    upsert,
+    search,
+    remove,
+    rebuild,
+    size: () => nodes.size,
+    entry: () => entry,
+    config: () => ({ M: maxNeighbors, efConstruction: constructionWidth, efSearch: searchWidth, seed }),
+  };
+}

package/src/vector/index.js

@@ -1,6 +1,9 @@
 import { hashText } from '../core/ids.js';
 import { nowIso } from '../core/time.js';
 import { cosine } from './embedder.js';
+import { createAnnIndex } from './ann.js';
+import { createHnswIndex } from './hnsw.js';
+import { createExternalAnnClient } from './external-ann.js';
 import { CortexError, ERROR_CODES } from '../core/errors.js';
 
 function toBlob(vector) {
@@ -18,11 +21,37 @@ function fromBlob(blob) {
  * personal memories is small (thousands, not billions), and a full scan keeps
  * the code auditable. A real ANN backend can replace it behind this interface.
  */
-export function createVectorIndex({ db, embedder, model = null, dim = null } = {}) {
+export function createVectorIndex({
+  db,
+  embedder,
+  model = null,
+  dim = null,
+  annEnabled = false,
+  annMin = 500,
+  annEngine = 'hnsw',
+  annOptions = {},
+  externalAnnEnabled = false,
+  externalAnnOptions = {},
+} = {}) {
   if (!db || !embedder) throw new Error('vector index needs db and embedder');
   const effectiveModel = model ?? embedder.model ?? 'unknown';
   const effectiveDim = Number(dim ?? embedder.dim ?? 0);
   if (!effectiveDim) throw new CortexError(ERROR_CODES.INVALID_INPUT, 'embedder dim is required');
+  const annMinCount = Math.max(1, Math.min(100000, Number(annMin) || 500));
+  const ann = annEnabled
+    ? (annEngine === 'nsw' ? createAnnIndex(annOptions) : createHnswIndex(annOptions))
+    : null;
+  const externalAnn = externalAnnEnabled && externalAnnOptions?.endpoint
+    ? createExternalAnnClient(externalAnnOptions)
+    : null;
+  let annBuilt = false;
+
+  function ensureAnn() {
+    if (!ann || annBuilt) return;
+    const rows = db.prepare('SELECT trace_id, vector FROM embeddings WHERE model = ? AND dim = ?').all(effectiveModel, effectiveDim);
+    ann.rebuild(rows.map((row) => ({ id: row.trace_id, vector: Array.from(fromBlob(row.vector)) })));
+    annBuilt = true;
+  }
 
   function upsert(traceId, text) {
     if (!traceId) return null;
@@ -46,6 +75,10 @@ export function createVectorIndex({ db, embedder, model = null, dim = null } = {
         model = excluded.model, dim = excluded.dim, vector = excluded.vector,
         hash = excluded.hash, updated_at = excluded.updated_at
     `).run(traceId, effectiveModel, effectiveDim, toBlob(vector), hash, nowIso());
+    if (ann) {
+      ann.upsert(traceId, Array.from(vector));
+      annBuilt = true;
+    }
     return { traceId, model: effectiveModel, dim: effectiveDim, hash };
   }
 
@@ -57,16 +90,45 @@ export function createVectorIndex({ db, embedder, model = null, dim = null } = {
   }
 
   function remove(traceId) {
+    if (ann) ann.remove(traceId);
     return db.prepare('DELETE FROM embeddings WHERE trace_id = ?').run(traceId).changes > 0;
   }
 
   function status() {
     const row = db.prepare('SELECT COUNT(*) AS c FROM embeddings').get();
-    return { model: effectiveModel, dim: effectiveDim, count: Number(row?.c ?? 0), async: embedder.kind === 'openai-embedding' || embedder.async === true };
+    return {
+      model: effectiveModel,
+      dim: effectiveDim,
+      count: Number(row?.c ?? 0),
+      async: embedder.kind === 'openai-embedding' || embedder.async === true,
+      ann: ann ? { enabled: true, engine: annEngine === 'nsw' ? 'nsw' : 'hnsw', size: ann.size(), min: annMinCount, built: annBuilt } : { enabled: false },
+      externalAnn: externalAnn ? { enabled: true, endpoint: externalAnn.endpoint } : { enabled: false },
+    };
   }
 
   function searchWithVector(queryVector, { scopeId = 'global', limit = 20, minScore = 0.05, includeArchived = false } = {}) {
     const stateClause = includeArchived ? "t.state IN ('active','archived')" : "t.state = 'active'";
+    if (ann) {
+      ensureAnn();
+      if (ann.size() >= annMinCount) {
+        const approx = ann.search(Array.from(queryVector), Math.max(limit * 3, 20));
+        const one = db.prepare(`
+          SELECT e.vector FROM embeddings e JOIN traces t ON t.id = e.trace_id
+          WHERE e.trace_id = ? AND t.scope_id = ? AND ${stateClause} AND e.model = ? AND e.dim = ?
+        `);
+        const hits = [];
+        for (const candidate of approx) {
+          const row = one.get(candidate.id, scopeId, effectiveModel, effectiveDim);
+          if (!row) continue;
+          const score = cosine(queryVector, fromBlob(row.vector));
+          if (score < minScore) continue;
+          hits.push({ traceId: candidate.id, score: Math.round(score * 10000) / 10000 });
+        }
+        if (hits.length >= Math.min(limit, 5)) {
+          return hits.sort((a, b) => b.score - a.score).slice(0, Math.max(1, Math.min(200, limit)));
+        }
+      }
+    }
     const rows = db.prepare(`
       SELECT e.trace_id, e.vector, e.model, e.dim, t.importance, t.confidence
       FROM embeddings e JOIN traces t ON t.id = e.trace_id
@@ -93,10 +155,30 @@ export function createVectorIndex({ db, embedder, model = null, dim = null } = {
     if (!text) return [];
     const queryVector = await embedder.embed(text);
     if (!queryVector) return [];
+    if (externalAnn) {
+      try {
+        const approximate = await externalAnn.search(Array.from(queryVector), Math.max(20, (Number(options.limit) || 20) * 3));
+        const stateClause = options.includeArchived ? "t.state IN ('active','archived')" : "t.state = 'active'";
+        const one = db.prepare(`
+          SELECT e.trace_id, e.vector FROM embeddings e JOIN traces t ON t.id = e.trace_id
+          WHERE e.trace_id = ? AND t.scope_id = ? AND ${stateClause} AND e.model = ? AND e.dim = ?
+        `);
+        const hits = [];
+        for (const candidate of approximate) {
+          const row = one.get(candidate.id, options.scopeId ?? 'global', effectiveModel, effectiveDim);
+          if (!row) continue;
+          hits.push({ traceId: candidate.id, score: Math.round(Number(candidate.score) * 10000) / 10000 });
+        }
+        if (hits.length >= Math.min(Number(options.limit) || 20, 5)) return hits;
+      } catch {
+        /* fall through to exact scan */
+      }
+    }
     return searchWithVector(queryVector, options);
   }
 
   function rebuild({ scopeId = 'global', limit = 5000 } = {}) {
+    if (ann) { ann.rebuild([]); annBuilt = true; }
     const traces = db.prepare(`
       SELECT id, title, content, summary FROM traces
       WHERE scope_id = ? AND state = 'active' ORDER BY recorded_at DESC LIMIT ?
@@ -106,6 +188,7 @@ export function createVectorIndex({ db, embedder, model = null, dim = null } = {
   }
 
   async function rebuildAsync({ scopeId = 'global', limit = 5000 } = {}) {
+    if (ann) { ann.rebuild([]); annBuilt = true; }
     const traces = db.prepare(`
       SELECT id, title, content, summary FROM traces
       WHERE scope_id = ? AND state = 'active' ORDER BY recorded_at DESC LIMIT ?

package/docs/compatibility.md

@@ -1,39 +0,0 @@
-# Windows / DSH compatibility
-
-## Tested here
-
-| Item | Value |
-|---|---|
-| OS | Windows 11 |
-| DSH | 0.2.2-rc.1 |
-| Node | 26.13.0 |
-| Public package | `dsh-cortex-memory` 0.1.0 |
-| Data root | fresh `~/.dsh/cortex` |
-| Native dependencies | none required |
-| Tests | 373 passing |
-| Package smoke | packed tgz extracted and started |
-
-## What was actually checked
-
-1. `npm pack` produced a 99-file tgz with `dsh.bundle.patch`.
-2. The tgz was extracted; `createCortex` wrote, searched and health-checked.
-3. The public source tests passed: 373 / 0.
-4. The client bundle is a DSH web client plugin; the host row is a bundle patch.
-5. No absolute personal paths or credentials are required.
-
-## Hard requirements
-
-- **Node.js >= 24** — the engine uses `node:sqlite`.
-- **DSH >=0.2.1-rc.2 <0.4.0**.
-- A profile may run only one plugin whose host id is `cortex`.
-
-## Not yet verified
-
-- Installation into a brand-new DSH profile from the exact release tgz.
-- A click-through browser E2E of every panel control.
-- Node 20/22 users.
-- Very old DSH releases.
-- Running alongside the private development build that uses the same `cortex` id.
-
-Do not claim “works for every Windows/DSH user” until a clean-profile
-installation is done on a machine without the development build.