@seanhogg/builderforce-memory 2026.6.31 → 2026.6.34

package/src/retrieval/HybridRetriever.ts

@@ -14,9 +14,9 @@
  * candidate vectors → BM25-only; no overlap → dense-only.
  */
 
-import { cosineSimilarity } from '../similarity/index.js';
 import { bm25Search, type Bm25Options } from './bm25.js';
 import { reciprocalRankFusion, maximalMarginalRelevance, type MmrCandidate } from './fusion.js';
+import { denseSearch } from './hnsw.js';
 
 export interface RetrievalCandidate {
     id: string;
@@ -46,6 +46,11 @@ export interface HybridRetrieveOptions {
     mmrLambda?: number;
     /** BM25 tuning. */
     bm25?: Bm25Options;
+    /**
+     * Candidate count at/above which the dense pass uses an HNSW ANN index
+     * instead of an exact cosine scan (O(log N) vs O(N)). Default 256.
+     */
+    annThreshold?: number;
 }
 
 export interface HybridHit {
@@ -70,14 +75,16 @@ export function hybridRetrieve(
 
     const byId = new Map(candidates.map(c => [c.id, c]));
 
-    // ── Dense ranking (cosine) ────────────────────────────────────────────────
+    // ── Dense ranking (cosine, ANN-accelerated above the threshold) ────────────
     let denseIds: string[] = [];
     if (query.vector) {
-        denseIds = candidates
-            .filter(c => c.vector && c.vector.length > 0)
-            .map(c => ({ id: c.id, score: cosineSimilarity(query.vector!, c.vector!) }))
-            .sort((a, b) => b.score - a.score)
-            .map(h => h.id);
+        const vectored = candidates
+            .filter((c): c is RetrievalCandidate & { vector: Float32Array } => !!c.vector && c.vector.length > 0)
+            .map(c => ({ id: c.id, vector: c.vector }));
+        // Rank the full vectored set (k = all) so fusion sees the complete dense
+        // ordering; denseSearch uses an exact scan under the threshold and HNSW
+        // above it, so behaviour is identical for small stores.
+        denseIds = denseSearch(query.vector, vectored, vectored.length, opts.annThreshold ?? 256).map(h => h.id);
     }
 
     // ── Sparse ranking (BM25) ─────────────────────────────────────────────────

package/src/retrieval/hnsw.ts

@@ -0,0 +1,242 @@
+/**
+ * hnsw.ts — a pure-TypeScript HNSW approximate-nearest-neighbour index.
+ *
+ * Recall over the fact store was an O(N) cosine scan per query, so the
+ * "lifetime of always-current knowledge" thesis hit a wall around ~10⁴ facts.
+ * HNSW (Hierarchical Navigable Small World) gives O(log N) expected query time
+ * with high recall, and this implementation is dependency-free so it keeps the
+ * engine/runtime zero-dep promise. For small sets an exact scan is both simpler
+ * and faster, so callers use {@link denseSearch} which switches to HNSW only
+ * above a threshold.
+ *
+ * Distance is cosine similarity (higher = closer); the graph stores `1 - cos`
+ * as the metric so "nearer" is "smaller". Construction is deterministic given a
+ * seed, so builds (and tests) reproduce exactly.
+ */
+
+import { cosineSimilarity } from '../similarity/index.js';
+
+export interface HnswOptions {
+    /** Max neighbours per node on layers > 0. Default 16. */
+    M?: number;
+    /** Candidate-list size during construction. Default 200. */
+    efConstruction?: number;
+    /** Default candidate-list size during search. Default 50. */
+    efSearch?: number;
+    /** Deterministic seed for layer assignment. Default 0x5eed. */
+    seed?: number;
+}
+
+export interface SearchHit {
+    id: string;
+    /** Cosine similarity to the query (higher = closer). */
+    score: number;
+}
+
+interface Node {
+    id: string;
+    vector: Float32Array;
+    /** neighbours[layer] = list of node indices. */
+    neighbours: number[][];
+}
+
+/** A small deterministic LCG → float in [0,1). Mirrors the engine's SeededRng. */
+function makeRng(seed: number): () => number {
+    let s = (seed >>> 0) || 1;
+    return () => {
+        s = (Math.imul(1664525, s) + 1013904223) >>> 0;
+        return s / 0x1_0000_0000;
+    };
+}
+
+export class HnswIndex {
+    private readonly _M: number;
+    private readonly _Mmax0: number;
+    private readonly _efC: number;
+    private readonly _efS: number;
+    private readonly _mL: number;
+    private readonly _rng: () => number;
+
+    private readonly _nodes: Node[] = [];
+    private readonly _idToIdx = new Map<string, number>();
+    private _entry = -1; // index of the entry-point node
+    private _maxLayer = -1;
+
+    constructor(opts: HnswOptions = {}) {
+        this._M = Math.max(2, opts.M ?? 16);
+        this._Mmax0 = this._M * 2;
+        this._efC = Math.max(this._M, opts.efConstruction ?? 200);
+        this._efS = Math.max(1, opts.efSearch ?? 50);
+        this._mL = 1 / Math.log(this._M);
+        this._rng = makeRng(opts.seed ?? 0x5eed);
+    }
+
+    get size(): number {
+        return this._nodes.length;
+    }
+
+    has(id: string): boolean {
+        return this._idToIdx.has(id);
+    }
+
+    private _dist(a: Float32Array, b: Float32Array): number {
+        return 1 - cosineSimilarity(a, b);
+    }
+
+    private _randomLayer(): number {
+        return Math.floor(-Math.log(Math.max(this._rng(), 1e-12)) * this._mL);
+    }
+
+    /** Insert a vector. Ids are unique — re-adding a known id is ignored. */
+    add(id: string, vector: Float32Array): void {
+        if (this._idToIdx.has(id)) return;
+        const layer = this._randomLayer();
+        const node: Node = { id, vector, neighbours: Array.from({ length: layer + 1 }, () => []) };
+        const idx = this._nodes.length;
+        this._nodes.push(node);
+        this._idToIdx.set(id, idx);
+
+        if (this._entry === -1) {
+            this._entry = idx;
+            this._maxLayer = layer;
+            return;
+        }
+
+        // Descend from the top to just above the new node's top layer.
+        let ep = this._entry;
+        for (let lc = this._maxLayer; lc > layer; lc--) {
+            ep = this._greedyClosest(vector, ep, lc);
+        }
+
+        // From the node's top layer down to 0: find neighbours and link.
+        for (let lc = Math.min(layer, this._maxLayer); lc >= 0; lc--) {
+            const candidates = this._searchLayer(vector, ep, this._efC, lc);
+            const Mmax = lc === 0 ? this._Mmax0 : this._M;
+            const selected = candidates.slice(0, this._M).map((c) => c.idx);
+            node.neighbours[lc] = selected.slice();
+            // Add reverse links, pruning each neighbour back to Mmax.
+            for (const nIdx of selected) {
+                const nb = this._nodes[nIdx]!;
+                const list = nb.neighbours[lc] ?? (nb.neighbours[lc] = []);
+                list.push(idx);
+                if (list.length > Mmax) this._prune(nb, lc, Mmax);
+            }
+            ep = candidates[0]?.idx ?? ep;
+        }
+
+        if (layer > this._maxLayer) {
+            this._maxLayer = layer;
+            this._entry = idx;
+        }
+    }
+
+    /** Top-k nearest by cosine similarity (descending). */
+    search(query: Float32Array, k: number, ef = this._efS): SearchHit[] {
+        if (this._entry === -1 || k <= 0) return [];
+        let ep = this._entry;
+        for (let lc = this._maxLayer; lc > 0; lc--) {
+            ep = this._greedyClosest(query, ep, lc);
+        }
+        const found = this._searchLayer(query, ep, Math.max(ef, k), 0);
+        return found.slice(0, k).map((c) => ({ id: this._nodes[c.idx]!.id, score: 1 - c.dist }));
+    }
+
+    clear(): void {
+        this._nodes.length = 0;
+        this._idToIdx.clear();
+        this._entry = -1;
+        this._maxLayer = -1;
+    }
+
+    // ── internals ──────────────────────────────────────────────────────────────
+
+    /** Greedy hill-climb to the single closest node to `q` on layer `lc`. */
+    private _greedyClosest(q: Float32Array, start: number, lc: number): number {
+        let best = start;
+        let bestDist = this._dist(q, this._nodes[best]!.vector);
+        let improved = true;
+        while (improved) {
+            improved = false;
+            for (const nIdx of this._nodes[best]!.neighbours[lc] ?? []) {
+                const d = this._dist(q, this._nodes[nIdx]!.vector);
+                if (d < bestDist) {
+                    bestDist = d;
+                    best = nIdx;
+                    improved = true;
+                }
+            }
+        }
+        return best;
+    }
+
+    /**
+     * ef-search on a single layer: returns candidates sorted nearest-first.
+     * A bounded best-first expansion (the standard HNSW layer search).
+     */
+    private _searchLayer(q: Float32Array, entry: number, ef: number, lc: number): Array<{ idx: number; dist: number }> {
+        const visited = new Set<number>([entry]);
+        const entryDist = this._dist(q, this._nodes[entry]!.vector);
+        // `candidates` is a min-heap-ish frontier; `results` keeps the best ef.
+        const candidates: Array<{ idx: number; dist: number }> = [{ idx: entry, dist: entryDist }];
+        const results: Array<{ idx: number; dist: number }> = [{ idx: entry, dist: entryDist }];
+
+        while (candidates.length > 0) {
+            // Pop nearest candidate.
+            let ci = 0;
+            for (let i = 1; i < candidates.length; i++) if (candidates[i]!.dist < candidates[ci]!.dist) ci = i;
+            const cur = candidates.splice(ci, 1)[0]!;
+            // Farthest in results.
+            const worst = results.reduce((m, r) => (r.dist > m.dist ? r : m), results[0]!);
+            if (cur.dist > worst.dist && results.length >= ef) break;
+
+            for (const nIdx of this._nodes[cur.idx]!.neighbours[lc] ?? []) {
+                if (visited.has(nIdx)) continue;
+                visited.add(nIdx);
+                const d = this._dist(q, this._nodes[nIdx]!.vector);
+                const farthest = results.reduce((m, r) => (r.dist > m.dist ? r : m), results[0]!);
+                if (results.length < ef || d < farthest.dist) {
+                    candidates.push({ idx: nIdx, dist: d });
+                    results.push({ idx: nIdx, dist: d });
+                    if (results.length > ef) {
+                        // Drop the current farthest.
+                        let wi = 0;
+                        for (let i = 1; i < results.length; i++) if (results[i]!.dist > results[wi]!.dist) wi = i;
+                        results.splice(wi, 1);
+                    }
+                }
+            }
+        }
+        return results.sort((a, b) => a.dist - b.dist);
+    }
+
+    /** Keep only the `Mmax` closest neighbours of `node` on layer `lc`. */
+    private _prune(node: Node, lc: number, Mmax: number): void {
+        const list = node.neighbours[lc]!;
+        list.sort((a, b) => this._dist(node.vector, this._nodes[a]!.vector) - this._dist(node.vector, this._nodes[b]!.vector));
+        node.neighbours[lc] = list.slice(0, Mmax);
+    }
+}
+
+/**
+ * Dense KNN with an exact-scan fast path: below `threshold` candidates a linear
+ * cosine scan is simpler and faster; at/above it, build an HNSW index and query
+ * it. Returned hits are cosine-similarity descending. Pure given its inputs.
+ */
+export function denseSearch(
+    query: Float32Array,
+    items: Array<{ id: string; vector: Float32Array }>,
+    k: number,
+    threshold = 256,
+    hnsw?: HnswOptions,
+): SearchHit[] {
+    if (items.length === 0 || k <= 0) return [];
+    if (items.length < threshold) {
+        return items
+            .map((it) => ({ id: it.id, score: cosineSimilarity(query, it.vector) }))
+            .sort((a, b) => b.score - a.score)
+            .slice(0, k);
+    }
+    const index = new HnswIndex(hnsw);
+    for (const it of items) index.add(it.id, it.vector);
+    return index.search(query, k);
+}

package/src/retrieval/index.ts

@@ -22,3 +22,6 @@ export type {
     HybridRetrieveOptions,
     HybridHit,
 } from './HybridRetriever.js';
+
+export { HnswIndex, denseSearch } from './hnsw.js';
+export type { HnswOptions, SearchHit } from './hnsw.js';

package/src/workflow/steps.ts

@@ -15,6 +15,10 @@ import {
   EvermindLMTrainer,
   BPETokenizer,
   EvermindModelPackage,
+  exportEvermind,
+  benchmarkText,
+  type ExportFormat,
+  type BenchmarkReport,
 } from "@seanhogg/builderforce-memory-engine";
 import { EvermindCognition } from "../cognition/index.js";
 import { hybridRetrieve, chunkText } from "../retrieval/index.js";
@@ -39,6 +43,15 @@ function pNum(cfg: WorkflowStepConfig, key: string, def: number): number {
   const v = cfg.params?.[key];
   return typeof v === "number" && Number.isFinite(v) ? v : def;
 }
+function pBool(cfg: WorkflowStepConfig, key: string, def: boolean): boolean {
+  const v = cfg.params?.[key];
+  return typeof v === "boolean" ? v : def;
+}
+
+/** Byte length of an emitted export file (text or binary). */
+function fileBytes(data: Uint8Array | string): number {
+  return typeof data === "string" ? new TextEncoder().encode(data).length : data.length;
+}
 
 function mkStep(cfg: WorkflowStepConfig, layer: string, defaultLabel: string, run: StackStep["run"]): StackStep {
   return { id: cfg.id, layer, label: cfg.label ?? defaultLabel, run };
@@ -214,6 +227,50 @@ export const BUILTIN_STEPS: Record<string, Registered> = {
         return `sample: "${out.slice(0, 40)}"`;
       }),
   },
+  benchmark: {
+    info: {
+      type: "benchmark",
+      layer: "BUILD",
+      label: "Benchmark",
+      description:
+        "Score the trained model on held-out text — perplexity, bits/token, top-1/top-k accuracy, throughput (params: evalCorpus, topK, maxPerplexity, minTop1)",
+    },
+    factory: (cfg) =>
+      mkStep(cfg, "BUILD", "Benchmark model (held-out)", async (ctx) => {
+        const model = ctx.bag.model as EvermindLM | undefined;
+        const tok = ctx.bag.tokenizer as BPETokenizer | undefined;
+        if (!model || !tok) throw new Error("no trained model — add 'train-model' first");
+        // Score on held-out text: the eval corpus must be text the model did not
+        // train on, otherwise perplexity is meaningless. Default to a small
+        // held-out probe distinct from the training KNOWLEDGE corpus.
+        const evalCorpus = pStr(
+          cfg,
+          "evalCorpus",
+          "Agents recall facts and act on them. The planning loop retrieves context before generating.",
+        );
+        const topK = pNum(cfg, "topK", 5);
+        const report: BenchmarkReport = benchmarkText(model, tok, evalCorpus, { topK });
+        if (report.tokens === 0) {
+          throw new Error("eval corpus produced no scorable tokens — provide longer held-out text");
+        }
+        ctx.bag.benchmark = report;
+
+        // Optional quality gates — only enforced when a threshold is configured.
+        const maxPerplexity = pNum(cfg, "maxPerplexity", 0);
+        if (maxPerplexity > 0 && report.perplexity > maxPerplexity) {
+          throw new Error(
+            `perplexity ${report.perplexity.toFixed(2)} exceeds max ${maxPerplexity} (model underfit — raise epochs/lr or enlarge the corpus)`,
+          );
+        }
+        const minTop1 = pNum(cfg, "minTop1", 0);
+        if (minTop1 > 0 && report.top1Accuracy < minTop1) {
+          throw new Error(
+            `top-1 accuracy ${(report.top1Accuracy * 100).toFixed(1)}% below min ${(minTop1 * 100).toFixed(0)}%`,
+          );
+        }
+        return `ppl ${report.perplexity.toFixed(2)}, ${report.bitsPerToken.toFixed(2)} bits/tok, top1 ${(report.top1Accuracy * 100).toFixed(0)}%, top${report.topK} ${(report.topKAccuracy * 100).toFixed(0)}% over ${report.tokens} tok`;
+      }),
+  },
   "dataset-quality": {
     info: { type: "dataset-quality", layer: "BUILD", label: "Dataset quality", description: "Validate the corpus is trainable before spending epochs (params: minSequences, minWords, maxDuplicateRatio)" },
     factory: (cfg) =>
@@ -322,4 +379,40 @@ export const BUILTIN_STEPS: Record<string, Registered> = {
         return `packaged ${blob.byteLength} bytes${tok ? " + tokenizer" : ""}`;
       }),
   },
+  export: {
+    info: {
+      type: "export",
+      layer: "BUILD",
+      label: "Export",
+      description:
+        "Export the trained model to a publishable format (params: format=huggingface|onnx|safetensors|gguf, name, version, license, fp16)",
+    },
+    factory: (cfg) =>
+      mkStep(cfg, "BUILD", "Export model (publishable)", async (ctx) => {
+        const model = ctx.bag.model as EvermindLM | undefined;
+        if (!model) throw new Error("no trained model — add 'train-model' first");
+        const tok = ctx.bag.tokenizer as BPETokenizer | undefined;
+        const format = pStr(cfg, "format", "huggingface") as ExportFormat;
+        if (format === "huggingface" && !tok) {
+          throw new Error("huggingface export needs a tokenizer — add 'train-tokenizer' first");
+        }
+        const result = exportEvermind(
+          model,
+          format,
+          {
+            name: pStr(cfg, "name", "Evermind"),
+            version: pStr(cfg, "version", "1.0.0"),
+            license: pStr(cfg, "license", "mit"),
+            author: pStr(cfg, "author", ""),
+            description: pStr(cfg, "description", "workflow-built Evermind model"),
+            fp16: pBool(cfg, "fp16", false),
+          },
+          tok,
+        );
+        const total = result.files.reduce((n, f) => n + fileBytes(f.data), 0);
+        // Land the file set as a workflow OUTPUT so a publish step / portal can ship it.
+        ctx.artifacts.export = { format, files: result.files, paramCount: result.paramCount };
+        return `exported ${format}: ${result.files.length} file(s), ${total} bytes, ${result.paramCount} params`;
+      }),
+  },
 };

package/src/workflow/templates.ts

@@ -55,7 +55,7 @@ const DEFAULT_CORPUS = [
 export const TRAIN_LLM: WorkflowConfig = {
   id: "train-llm",
   name: "Create an LLM",
-  description: "Train a custom EvermindLM on your corpus, validate it end to end, and package it as a portable .evermind artifact.",
+  description: "Train a custom EvermindLM on your corpus, validate it end to end, package it as a portable .evermind artifact, and export a publishable Hugging Face repo (safetensors + ONNX + GGUF).",
   steps: [
     { id: "tok", type: "train-tokenizer", params: { corpus: DEFAULT_CORPUS, numMerges: 120 } },
     { id: "data", type: "dataset-quality" },
@@ -63,7 +63,9 @@ export const TRAIN_LLM: WorkflowConfig = {
     { id: "converge", type: "convergence" },
     { id: "eval", type: "evaluate" },
     { id: "gen", type: "generate-check" },
+    { id: "bench", type: "benchmark" },
     { id: "pkg", type: "roundtrip", params: { name: "my-llm" } },
+    { id: "export", type: "export", params: { format: "huggingface", name: "my-llm", version: "1.0.0" } },
   ],
 };