@seanhogg/builderforce-memory 2026.10.28 → 2026.10.29

package/src/retrieval/hnsw.ts

@@ -10,11 +10,19 @@
  * above a threshold.
  *
  * Distance is cosine similarity (higher = closer); the graph stores `1 - cos`
- * as the metric so "nearer" is "smaller". Construction is deterministic given a
- * seed, so builds (and tests) reproduce exactly.
+ * as the metric so "nearer" is "smaller".
+ *
+ * Determinism: a node's level is {@link levelOf} its id — a geometric draw from
+ * the id's sha-256, never a sequential RNG — and every ordering (candidates,
+ * pruning, results) is {@link compareScoreThenId}: score, then id. So the graph
+ * depends only on the vectors and the insertion order, and
+ * {@link HnswIndex.canonicalBuild} (id-sorted inserts) removes the latter: the
+ * same set of (id, vector) gives the same graph whatever order it arrived in.
  */
 
 import { cosineSimilarity } from '../similarity/index.js';
+import { compareScoreThenId, compareDenseHits, compareIds, DENSE_SCORE_QUANTUM } from './order.js';
+import { sha256 } from './knowledge-index/sha256.js';
 
 export interface HnswOptions {
     /** Max neighbours per node on layers > 0. Default 16. */
@@ -23,7 +31,11 @@ export interface HnswOptions {
     efConstruction?: number;
     /** Default candidate-list size during search. Default 50. */
     efSearch?: number;
-    /** Deterministic seed for layer assignment. Default 0x5eed. */
+    /**
+     * Retained for API compatibility. It no longer affects anything: levels come
+     * from each id's hash ({@link levelOf}), so a build does not depend on a seed
+     * or on how many inserts preceded a node.
+     */
     seed?: number;
 }
 
@@ -33,20 +45,38 @@ export interface SearchHit {
     score: number;
 }
 
+/** Any numeric vector: a float embedding, or int8 codes (cosine over codes is exact). */
+export type HnswVector = ArrayLike<number>;
+
 interface Node {
     id: string;
-    vector: Float32Array;
+    vector: HnswVector;
     /** neighbours[layer] = list of node indices. */
     neighbours: number[][];
 }
 
-/** A small deterministic LCG → float in [0,1). Mirrors the engine's SeededRng. */
-function makeRng(seed: number): () => number {
-    let s = (seed >>> 0) || 1;
-    return () => {
-        s = (Math.imul(1664525, s) + 1013904223) >>> 0;
-        return s / 0x1_0000_0000;
-    };
+/** A read-only view of the built graph, for a consumer that stores it elsewhere (CSR). */
+export interface HnswGraphView {
+    /** Node ids, by node index (insertion order). */
+    ids: readonly string[];
+    /** neighbours[node][layer] = node indices. */
+    neighbours: ReadonlyArray<ReadonlyArray<readonly number[]>>;
+    entry: number;
+    maxLayer: number;
+}
+
+/**
+ * The HNSW level of `id`: a geometric draw with P(level ≥ L) = M^-L, taken from the
+ * eight 32-bit words of sha-256(id) — each word below 2^32 / M is one more level.
+ * Integer-only, so it is the same on every platform and independent of insertion.
+ */
+export function levelOf(id: string, M: number): number {
+    const h = sha256(id);
+    const view = new DataView(h.buffer, h.byteOffset, h.byteLength);
+    const threshold = 0x1_0000_0000 / M;
+    let level = 0;
+    while (level < 8 && view.getUint32(level * 4) < threshold) level++;
+    return level;
 }
 
 export class HnswIndex {
@@ -54,8 +84,6 @@ export class HnswIndex {
     private readonly _Mmax0: number;
     private readonly _efC: number;
     private readonly _efS: number;
-    private readonly _mL: number;
-    private readonly _rng: () => number;
 
     private readonly _nodes: Node[] = [];
     private readonly _idToIdx = new Map<string, number>();
@@ -67,8 +95,19 @@ export class HnswIndex {
         this._Mmax0 = this._M * 2;
         this._efC = Math.max(this._M, opts.efConstruction ?? 200);
         this._efS = Math.max(1, opts.efSearch ?? 50);
-        this._mL = 1 / Math.log(this._M);
-        this._rng = makeRng(opts.seed ?? 0x5eed);
+    }
+
+    /**
+     * Builds an index by inserting `items` in id order — the canonical build. The
+     * same (id, vector) set yields the same graph regardless of the input order.
+     * Ids are unique; for a duplicated id the first in input order wins.
+     */
+    static canonicalBuild(items: ReadonlyArray<{ id: string; vector: HnswVector }>, opts?: HnswOptions): HnswIndex {
+        const index = new HnswIndex(opts);
+        const sorted = items.map((it, i) => ({ it, i }))
+            .sort((a, b) => compareIds(a.it.id, b.it.id) || a.i - b.i);
+        for (const { it } of sorted) index.add(it.id, it.vector);
+        return index;
     }
 
     get size(): number {
@@ -79,18 +118,28 @@ export class HnswIndex {
         return this._idToIdx.has(id);
     }
 
-    private _dist(a: Float32Array, b: Float32Array): number {
-        return 1 - cosineSimilarity(a, b);
+    /** The built graph, by node index — what a CSR store and pruning read. */
+    graph(): HnswGraphView {
+        return {
+            ids: this._nodes.map((n) => n.id),
+            neighbours: this._nodes.map((n) => n.neighbours),
+            entry: this._entry,
+            maxLayer: this._maxLayer,
+        };
     }
 
-    private _randomLayer(): number {
-        return Math.floor(-Math.log(Math.max(this._rng(), 1e-12)) * this._mL);
+    private _dist(a: HnswVector, b: HnswVector): number {
+        return 1 - cosineSimilarity(a, b);
     }
 
+    /** Nearest first; equal (quantized) distances by id ascending. */
+    private readonly _byDist = (a: { idx: number; dist: number }, b: { idx: number; dist: number }): number =>
+        compareScoreThenId(1 - a.dist, this._nodes[a.idx]!.id, 1 - b.dist, this._nodes[b.idx]!.id, DENSE_SCORE_QUANTUM);
+
     /** Insert a vector. Ids are unique — re-adding a known id is ignored. */
-    add(id: string, vector: Float32Array): void {
+    add(id: string, vector: HnswVector): void {
         if (this._idToIdx.has(id)) return;
-        const layer = this._randomLayer();
+        const layer = levelOf(id, this._M);
         const node: Node = { id, vector, neighbours: Array.from({ length: layer + 1 }, () => []) };
         const idx = this._nodes.length;
         this._nodes.push(node);
@@ -130,8 +179,8 @@ export class HnswIndex {
         }
     }
 
-    /** Top-k nearest by cosine similarity (descending). */
-    search(query: Float32Array, k: number, ef = this._efS): SearchHit[] {
+    /** Top-k nearest by cosine similarity (descending; equal scores by id). */
+    search(query: HnswVector, k: number, ef = this._efS): SearchHit[] {
         if (this._entry === -1 || k <= 0) return [];
         let ep = this._entry;
         for (let lc = this._maxLayer; lc > 0; lc--) {
@@ -151,7 +200,7 @@ export class HnswIndex {
     // ── internals ──────────────────────────────────────────────────────────────
 
     /** Greedy hill-climb to the single closest node to `q` on layer `lc`. */
-    private _greedyClosest(q: Float32Array, start: number, lc: number): number {
+    private _greedyClosest(q: HnswVector, start: number, lc: number): number {
         let best = start;
         let bestDist = this._dist(q, this._nodes[best]!.vector);
         let improved = true;
@@ -173,7 +222,7 @@ export class HnswIndex {
      * ef-search on a single layer: returns candidates sorted nearest-first.
      * A bounded best-first expansion (the standard HNSW layer search).
      */
-    private _searchLayer(q: Float32Array, entry: number, ef: number, lc: number): Array<{ idx: number; dist: number }> {
+    private _searchLayer(q: HnswVector, entry: number, ef: number, lc: number): Array<{ idx: number; dist: number }> {
         const visited = new Set<number>([entry]);
         const entryDist = this._dist(q, this._nodes[entry]!.vector);
         // `candidates` is a min-heap-ish frontier; `results` keeps the best ef.
@@ -206,21 +255,21 @@ export class HnswIndex {
                 }
             }
         }
-        return results.sort((a, b) => a.dist - b.dist);
+        return results.sort(this._byDist);
     }
 
     /** Keep only the `Mmax` closest neighbours of `node` on layer `lc`. */
     private _prune(node: Node, lc: number, Mmax: number): void {
-        const list = node.neighbours[lc]!;
-        list.sort((a, b) => this._dist(node.vector, this._nodes[a]!.vector) - this._dist(node.vector, this._nodes[b]!.vector));
-        node.neighbours[lc] = list.slice(0, Mmax);
+        const ranked = node.neighbours[lc]!.map((idx) => ({ idx, dist: this._dist(node.vector, this._nodes[idx]!.vector) }));
+        node.neighbours[lc] = ranked.sort(this._byDist).slice(0, Mmax).map((r) => r.idx);
     }
 }
 
 /**
  * Dense KNN with an exact-scan fast path: below `threshold` candidates a linear
- * cosine scan is simpler and faster; at/above it, build an HNSW index and query
- * it. Returned hits are cosine-similarity descending. Pure given its inputs.
+ * cosine scan is simpler and faster; at/above it, build an HNSW index (the
+ * canonical, id-ordered build) and query it. Returned hits are in
+ * {@link compareDenseHits} order. Pure given its inputs, and independent of their order.
  */
 export function denseSearch(
     query: Float32Array,
@@ -233,10 +282,8 @@ export function denseSearch(
     if (items.length < threshold) {
         return items
             .map((it) => ({ id: it.id, score: cosineSimilarity(query, it.vector) }))
-            .sort((a, b) => b.score - a.score)
+            .sort(compareDenseHits)
             .slice(0, k);
     }
-    const index = new HnswIndex(hnsw);
-    for (const it of items) index.add(it.id, it.vector);
-    return index.search(query, k);
+    return HnswIndex.canonicalBuild(items, hnsw).search(query, k);
 }

package/src/retrieval/index.ts

@@ -1,6 +1,7 @@
 /**
  * Retrieval layer — chunking, BM25, rank and score fusion, MMR, recency decay, the
- * similarity primitives they rest on, and the HybridRetriever.
+ * similarity primitives they rest on, the HybridRetriever, the ONE total order every
+ * ranker returns (`compareHits`), and the deterministic knowledge index.
  *
  * Zero-dependency and engine-free: this is the `@seanhogg/builderforce-memory/retrieval`
  * subpath, which a Cloudflare Worker, Node and the browser can all import without
@@ -41,5 +42,18 @@ export type {
     HybridHit,
 } from './HybridRetriever.js';
 
-export { HnswIndex, denseSearch } from './hnsw.js';
-export type { HnswOptions, SearchHit } from './hnsw.js';
+export { HnswIndex, denseSearch, levelOf } from './hnsw.js';
+export type { HnswOptions, SearchHit, HnswVector, HnswGraphView } from './hnsw.js';
+
+export {
+    compareHits,
+    compareDenseHits,
+    compareScoreThenId,
+    compareIds,
+    quantizeScore,
+    DENSE_SCORE_QUANTUM,
+    LEXICAL_SCORE_QUANTUM,
+} from './order.js';
+export type { ScoredId } from './order.js';
+
+export * from './knowledge-index/index.js';

package/src/retrieval/knowledge-index/bytes.ts

@@ -0,0 +1,166 @@
+/**
+ * bytes.ts — the little-endian byte writer/reader that `serialize` and `digest`
+ * share, so the persisted format and the hashed canonical state encode every
+ * field the same way. DataView only: no Node `Buffer`, no host endianness.
+ */
+
+const encoder = new TextEncoder();
+const decoder = new TextDecoder();
+
+export class ByteWriter {
+    private _buf = new Uint8Array(1024);
+    private _view = new DataView(this._buf.buffer);
+    private _len = 0;
+
+    private _reserve(n: number): void {
+        if (this._len + n <= this._buf.length) return;
+        let size = this._buf.length * 2;
+        while (size < this._len + n) size *= 2;
+        const next = new Uint8Array(size);
+        next.set(this._buf.subarray(0, this._len));
+        this._buf = next;
+        this._view = new DataView(next.buffer);
+    }
+
+    u8(v: number): this {
+        this._reserve(1);
+        this._view.setUint8(this._len, v);
+        this._len += 1;
+        return this;
+    }
+
+    u32(v: number): this {
+        this._reserve(4);
+        this._view.setUint32(this._len, v >>> 0, true);
+        this._len += 4;
+        return this;
+    }
+
+    i32(v: number): this {
+        this._reserve(4);
+        this._view.setInt32(this._len, v, true);
+        this._len += 4;
+        return this;
+    }
+
+    f64(v: number): this {
+        this._reserve(8);
+        this._view.setFloat64(this._len, v, true);
+        this._len += 8;
+        return this;
+    }
+
+    /** Raw bytes, no length prefix. */
+    raw(bytes: Uint8Array): this {
+        this._reserve(bytes.length);
+        this._buf.set(bytes, this._len);
+        this._len += bytes.length;
+        return this;
+    }
+
+    /** Length-prefixed UTF-8. */
+    str(s: string): this {
+        const b = encoder.encode(s);
+        return this.u32(b.length).raw(b);
+    }
+
+    /** Count-prefixed u32 values. */
+    u32s(arr: ArrayLike<number>): this {
+        this.u32(arr.length);
+        this._reserve(arr.length * 4);
+        for (let i = 0; i < arr.length; i++) this._view.setUint32(this._len + i * 4, arr[i]! >>> 0, true);
+        this._len += arr.length * 4;
+        return this;
+    }
+
+    /** Count-prefixed f32 values. */
+    f32s(arr: ArrayLike<number>): this {
+        this.u32(arr.length);
+        this._reserve(arr.length * 4);
+        for (let i = 0; i < arr.length; i++) this._view.setFloat32(this._len + i * 4, arr[i]!, true);
+        this._len += arr.length * 4;
+        return this;
+    }
+
+    /** Count-prefixed bytes (an Int8Array's bytes are written as-is). */
+    bytes(arr: Uint8Array | Int8Array): this {
+        this.u32(arr.length);
+        return this.raw(new Uint8Array(arr.buffer, arr.byteOffset, arr.byteLength));
+    }
+
+    finish(): Uint8Array {
+        return this._buf.slice(0, this._len);
+    }
+}
+
+export class ByteReader {
+    private readonly _view: DataView;
+    private _pos = 0;
+
+    constructor(private readonly _buf: Uint8Array) {
+        this._view = new DataView(_buf.buffer, _buf.byteOffset, _buf.byteLength);
+    }
+
+    private _need(n: number): void {
+        if (this._pos + n > this._buf.length) throw new Error('knowledge-index: truncated bytes');
+    }
+
+    u8(): number {
+        this._need(1);
+        return this._view.getUint8(this._pos++);
+    }
+
+    u32(): number {
+        this._need(4);
+        const v = this._view.getUint32(this._pos, true);
+        this._pos += 4;
+        return v;
+    }
+
+    i32(): number {
+        this._need(4);
+        const v = this._view.getInt32(this._pos, true);
+        this._pos += 4;
+        return v;
+    }
+
+    f64(): number {
+        this._need(8);
+        const v = this._view.getFloat64(this._pos, true);
+        this._pos += 8;
+        return v;
+    }
+
+    raw(n: number): Uint8Array {
+        this._need(n);
+        const out = this._buf.slice(this._pos, this._pos + n);
+        this._pos += n;
+        return out;
+    }
+
+    str(): string {
+        return decoder.decode(this.raw(this.u32()));
+    }
+
+    u32s(): Uint32Array {
+        const n = this.u32();
+        this._need(n * 4);
+        const out = new Uint32Array(n);
+        for (let i = 0; i < n; i++) out[i] = this._view.getUint32(this._pos + i * 4, true);
+        this._pos += n * 4;
+        return out;
+    }
+
+    f32s(): Float32Array {
+        const n = this.u32();
+        this._need(n * 4);
+        const out = new Float32Array(n);
+        for (let i = 0; i < n; i++) out[i] = this._view.getFloat32(this._pos + i * 4, true);
+        this._pos += n * 4;
+        return out;
+    }
+
+    bytes(): Uint8Array {
+        return this.raw(this.u32());
+    }
+}

package/src/retrieval/knowledge-index/cascade.ts

@@ -0,0 +1,125 @@
+/**
+ * cascade.ts — the knowledge index's search: cheapest tier first.
+ *
+ *   1. exact    — the folded `keys` map; a hit scores 1.0, and when the exact hits
+ *                 fill `k` nothing else runs;
+ *   2. lexical  — BM25 over card text with the code-aware tokenizer;
+ *   3. dense    — int8 graph search for `k × rerankFactor` candidates (plus the
+ *                 cards not yet compacted), then a float rerank over vectors
+ *                 recomputed through the hash-keyed LRU;
+ *   4. fuse     — RRF over the lexical and dense lists; exact hits stay on top at
+ *                 1.0; everything in {@link compareHits} order.
+ *
+ * Dense runs only with an embedder of the stored fingerprint — otherwise the tier
+ * is absent, never silently replaced.
+ */
+
+import { reciprocalRankFusion } from '../fusion.js';
+import { compareDenseHits, compareHits } from '../order.js';
+import { cosineSimilarity } from '../../similarity/index.js';
+import type { EmbedderPort, IndexHit, IndexTier } from './types.js';
+import { availableTiers, graphNode, type DocEntry, type IndexState } from './state.js';
+import type { EmbeddingCache } from './embeddings.js';
+import { dotInt8, quantize } from './quantize.js';
+import { searchGraph } from './traverse.js';
+import { foldKey } from './tokenize.js';
+
+export interface SearchContext {
+    state: IndexState;
+    embedder?: EmbedderPort;
+    /** The embedder's recompute cache; present whenever `embedder` is. */
+    cache?: EmbeddingCache;
+}
+
+export interface SearchOptions {
+    k?: number;
+    tiers?: IndexTier[];
+}
+
+/** Exact-tier ids for `q`, ascending. */
+function exactIds(state: IndexState, q: string): string[] {
+    const ids = state.keys.get(foldKey(q));
+    return ids ? [...ids].sort() : [];
+}
+
+/**
+ * Dense candidates before the float rerank: the best `n` live graph nodes by
+ * integer dot product, plus the best `n` cards not yet in the graph. Ids, unordered
+ * between the two sources (the rerank orders them).
+ */
+export async function denseCandidates(ctx: SearchContext, qv: Float32Array, n: number): Promise<string[]> {
+    const { state, cache } = ctx;
+    const out = new Set<string>();
+    const c = state.compacted;
+    if (c && state.scale > 0) {
+        const qc = quantize(qv, state.scale);
+        const recomputed = new Map<number, Int8Array>();
+        const codeOf = (node: number) => c.codes[node] ?? recomputed.get(node);
+        const found = await searchGraph(c.graph, {
+            score: (node) => {
+                const code = codeOf(node);
+                return code ? dotInt8(qc, code) : -Infinity;
+            },
+            ensure: cache && state.params.lean ? (nodes) => {
+                const live = nodes.filter((node) => !codeOf(node) && graphNode(state, c.ids[node]!) === node);
+                if (live.length === 0) return undefined;
+                return cache.vectors(live.map((node) => state.docs.get(c.ids[node]!)!)).then((vecs) => {
+                    live.forEach((node, i) => recomputed.set(node, quantize(vecs[i]!, state.scale)));
+                });
+            } : undefined,
+        }, Math.max(state.params.efSearch, n));
+        for (const hit of found) {
+            if (out.size >= n) break;
+            if (hit.score !== -Infinity && graphNode(state, c.ids[hit.node]!) === hit.node) out.add(c.ids[hit.node]!);
+        }
+    }
+    const pending: Array<{ id: string; score: number }> = [];
+    for (const doc of state.docs.values()) {
+        if (doc.vec && graphNode(state, doc.id) < 0) pending.push({ id: doc.id, score: cosineSimilarity(qv, doc.vec) });
+    }
+    for (const hit of pending.sort(compareDenseHits).slice(0, n)) out.add(hit.id);
+    return [...out];
+}
+
+/** The dense tier: candidates, then float recompute + rerank, best first. */
+async function denseRank(ctx: SearchContext, q: string, n: number): Promise<Array<{ id: string; score: number }>> {
+    const cache = ctx.cache!;
+    const qv = await cache.query(q);
+    const ids = await denseCandidates(ctx, qv, n);
+    const docs = ids.map((id) => ctx.state.docs.get(id)!);
+    const needVec = docs.filter((d) => !d.vec);
+    const fresh = await cache.vectors(needVec);
+    const vecOf = new Map<DocEntry, Float32Array>(needVec.map((d, i) => [d, fresh[i]!]));
+    return docs
+        .map((d) => ({ id: d.id, score: cosineSimilarity(qv, d.vec ?? vecOf.get(d)!) }))
+        .sort(compareDenseHits)
+        .slice(0, n);
+}
+
+/** Runs the cascade. A pure function of the state (its digest), `q` and `opts`. */
+export async function cascadeSearch(ctx: SearchContext, q: string, opts: SearchOptions = {}): Promise<IndexHit[]> {
+    const { state } = ctx;
+    const k = Math.max(0, opts.k ?? 10);
+    if (k === 0 || state.docs.size === 0) return [];
+    const available = availableTiers(state, ctx.embedder);
+    const run = new Set((opts.tiers ?? available).filter((t) => available.includes(t)));
+
+    const exact: IndexHit[] = run.has('exact')
+        ? exactIds(state, q).map((id) => ({ id, score: 1, tier: 'exact' as const }))
+        : [];
+    if (exact.length >= k) return exact.slice(0, k);
+    const exactSet = new Set(exact.map((h) => h.id));
+
+    const n = Math.max(k, k * state.params.rerankFactor);
+    const lexical = run.has('lexical')
+        ? state.lexical.search(q, n + exact.length).filter((h) => !exactSet.has(h.id)).map((h) => h.id)
+        : [];
+    const dense = run.has('dense')
+        ? (await denseRank(ctx, q, n + exact.length)).filter((h) => !exactSet.has(h.id)).map((h) => h.id)
+        : [];
+
+    const lexicalSet = new Set(lexical);
+    const fused = reciprocalRankFusion([{ ids: lexical }, { ids: dense }].filter((l) => l.ids.length > 0), state.params.rrfK)
+        .map((h): IndexHit => ({ id: h.id, score: h.score, tier: lexicalSet.has(h.id) ? 'lexical' : 'dense' }));
+    return [...exact, ...fused].sort(compareHits).slice(0, k);
+}

package/src/retrieval/knowledge-index/compact.ts

@@ -0,0 +1,68 @@
+/**
+ * compact.ts — the canonical rebuild.
+ *
+ * Every live card, in id order, is coded (an unchanged node keeps its code; a new
+ * or changed card is quantized from its float vector) and the graph is rebuilt
+ * from those codes. Since levels come from id hashes, inserts are id-ordered and
+ * every tie breaks on id, the same card set and fingerprint give the same bytes
+ * whatever order the upserts arrived in.
+ *
+ * The int8 scale is fitted at the first compaction and fixed after it. An embedder
+ * whose fingerprint differs from the stored one is never mixed in: compaction
+ * drops every stored vector and code and re-embeds all cards under it.
+ */
+
+import { compareIds } from '../order.js';
+import type { EmbedderPort } from './types.js';
+import { graphNode, type DocEntry, type IndexState } from './state.js';
+import type { EmbeddingCache } from './embeddings.js';
+import { fitScale, quantize } from './quantize.js';
+import { buildGraph } from './csr.js';
+
+/** Float vectors for the docs that have neither a reusable code nor a pending vector. */
+async function vectorsFor(cache: EmbeddingCache, docs: readonly DocEntry[]): Promise<Map<string, Float32Array>> {
+    const vecs = await cache.vectors(docs);
+    return new Map(docs.map((d, i) => [d.id, vecs[i]!]));
+}
+
+export async function compactState(state: IndexState, embedder: EmbedderPort | undefined, cache: EmbeddingCache | undefined): Promise<void> {
+    // Nothing to code with: exact and lexical need no compaction, and a stored
+    // graph of another fingerprint is kept for an embedder that matches it.
+    if (!embedder || !cache) return;
+    if (embedder.fingerprint !== state.fingerprint || embedder.dims !== state.dims) {
+        state.fingerprint = embedder.fingerprint;
+        state.dims = embedder.dims;
+        state.scale = 0;
+        state.compacted = null;
+        for (const doc of state.docs.values()) doc.vec = undefined;
+    }
+
+    const ids = [...state.docs.keys()].sort(compareIds);
+    const docs = ids.map((id) => state.docs.get(id)!);
+    if (docs.length === 0) {
+        state.compacted = null;
+        return;
+    }
+
+    const old = state.compacted;
+    const reused = docs.map((d) => {
+        const node = graphNode(state, d.id);
+        return node >= 0 && state.scale > 0 ? old!.codes[node] : undefined;
+    });
+    const missing = await vectorsFor(cache, docs.filter((d, i) => !reused[i] && !d.vec));
+    const floatOf = (d: DocEntry) => d.vec ?? missing.get(d.id)!;
+
+    if (state.scale === 0) state.scale = fitScale(docs.map(floatOf));
+    const codes = docs.map((d, i) => reused[i] ?? quantize(floatOf(d), state.scale));
+
+    const { graph, hubs } = buildGraph(ids, codes, state.params);
+    state.compacted = {
+        ids,
+        hashes: docs.map((d) => d.hash),
+        codes: state.params.lean ? codes.map((code, i) => (hubs[i] ? code : undefined)) : codes,
+        hubs,
+        graph,
+        nodeOf: new Map(ids.map((id, i) => [id, i])),
+    };
+    for (const doc of docs) doc.vec = undefined;
+}