@seanhogg/builderforce-memory 2026.10.28 → 2026.10.29

package/src/retrieval/knowledge-index/csr.ts

@@ -0,0 +1,102 @@
+/**
+ * csr.ts — the knowledge index's graph, stored as CSR (compressed sparse rows).
+ *
+ * The graph is built by the package's HNSW over int8 codes (canonical, id-ordered
+ * inserts, hash-derived levels), its base layer is pruned (`prune.ts`), and every
+ * layer is flattened into a `Uint32Array` of offsets plus a `Uint32Array` of
+ * neighbours: no per-node arrays, a fixed byte layout to serialize and hash.
+ *
+ * Layer 0 holds every node and is indexed by node directly. An upper layer holds
+ * only the nodes that reach it, listed ascending in `nodes`.
+ */
+
+import { HnswIndex } from '../hnsw.js';
+import type { IndexParams } from './types.js';
+import { pruneBaseLayer } from './prune.js';
+import { cosineInt8 } from './quantize.js';
+
+export interface CsrLayer {
+    /** Upper layers: the node indices present, ascending. Absent on layer 0 (every node). */
+    nodes?: Uint32Array;
+    /** Row `r`'s neighbours are `neighbours[offsets[r] .. offsets[r + 1])`. */
+    offsets: Uint32Array;
+    neighbours: Uint32Array;
+}
+
+export interface CsrGraph {
+    /** Entry node for search; -1 for an empty graph. */
+    entry: number;
+    /** layers[0] is the base layer; layers.length - 1 is the top layer. */
+    layers: CsrLayer[];
+}
+
+export const EMPTY_GRAPH: CsrGraph = Object.freeze({ entry: -1, layers: [] }) as CsrGraph;
+
+/** Flattens per-row neighbour lists into one CSR layer. */
+export function toCsrLayer(rows: ReadonlyArray<readonly number[]>, nodes?: Uint32Array): CsrLayer {
+    const offsets = new Uint32Array(rows.length + 1);
+    let total = 0;
+    rows.forEach((row, r) => {
+        total += row.length;
+        offsets[r + 1] = total;
+    });
+    const neighbours = new Uint32Array(total);
+    rows.forEach((row, r) => neighbours.set(row, offsets[r]!));
+    return nodes ? { nodes, offsets, neighbours } : { offsets, neighbours };
+}
+
+/** The row of `node` in `layer`, or -1 when the node is not on it. */
+function rowOf(layer: CsrLayer, node: number): number {
+    if (!layer.nodes) return node < layer.offsets.length - 1 ? node : -1;
+    let lo = 0;
+    let hi = layer.nodes.length - 1;
+    while (lo <= hi) {
+        const mid = (lo + hi) >>> 1;
+        const v = layer.nodes[mid]!;
+        if (v === node) return mid;
+        if (v < node) lo = mid + 1;
+        else hi = mid - 1;
+    }
+    return -1;
+}
+
+/** Neighbours of `node` on `layer` (empty when the node is not on it). */
+export function neighboursOf(layer: CsrLayer, node: number): Uint32Array {
+    const r = rowOf(layer, node);
+    if (r < 0) return layer.neighbours.subarray(0, 0);
+    return layer.neighbours.subarray(layer.offsets[r]!, layer.offsets[r + 1]!);
+}
+
+/**
+ * Builds the pruned CSR graph over `codes`, whose order is the id order of `ids`
+ * (already sorted and unique). Distances are cosine over the int8 codes, which is
+ * exact integer arithmetic plus correctly-rounded sqrt/division — so the same codes
+ * give the same graph on every platform.
+ */
+export function buildGraph(
+    ids: readonly string[],
+    codes: readonly Int8Array[],
+    params: IndexParams,
+): { graph: CsrGraph; hubs: Uint8Array } {
+    if (ids.length === 0) return { graph: EMPTY_GRAPH, hubs: new Uint8Array(0) };
+    const hnsw = HnswIndex.canonicalBuild(
+        ids.map((id, i) => ({ id, vector: codes[i]! })),
+        { M: params.m, efConstruction: params.efConstruction, efSearch: params.efSearch },
+    );
+    const view = hnsw.graph();
+    const { rows, hubs } = pruneBaseLayer(
+        view.neighbours.map((layers) => layers[0] ?? []),
+        params.m,
+        params.hubShare,
+        (a, b) => cosineInt8(codes[a]!, codes[b]!),
+    );
+    const layers: CsrLayer[] = [toCsrLayer(rows)];
+    for (let l = 1; l <= view.maxLayer; l++) {
+        const present: number[] = [];
+        view.neighbours.forEach((layersOfNode, node) => {
+            if (layersOfNode.length > l) present.push(node);
+        });
+        layers.push(toCsrLayer(present.map((node) => view.neighbours[node]![l]!), Uint32Array.from(present)));
+    }
+    return { graph: { entry: view.entry, layers }, hubs };
+}

package/src/retrieval/knowledge-index/digest.ts

@@ -0,0 +1,54 @@
+/**
+ * digest.ts — the name of an index state.
+ *
+ * sha-256 over the canonical encoding of everything a result depends on: the
+ * fingerprint, dims, params and int8 scale; every live card's (id, hash, keys) in
+ * id order; and the compacted snapshot (node ids and hashes, hubs, the CSR graph).
+ * Every search result is a pure function of `(digest, query, opts)`, which is what
+ * lets a logged digest replay a disputed answer.
+ */
+
+import { compareIds } from '../order.js';
+import type { IndexParams } from './types.js';
+import type { IndexState } from './state.js';
+import type { CsrGraph } from './csr.js';
+import { ByteWriter } from './bytes.js';
+import { sha256Hex } from './sha256.js';
+
+/** The default content hash of a card's text. */
+export function hashDocText(text: string): string {
+    return sha256Hex(text);
+}
+
+/** Params in a fixed field order, for the digest and the serialized header. */
+export function writeParams(w: ByteWriter, p: IndexParams): void {
+    w.u32(p.m).u32(p.efConstruction).u32(p.efSearch).f64(p.hubShare).u32(p.rerankFactor).u8(p.lean ? 1 : 0).u32(p.rrfK);
+}
+
+/** The graph's layers: entry, then per layer its node list, offsets and neighbours. */
+export function writeGraph(w: ByteWriter, g: CsrGraph): void {
+    w.i32(g.entry).u32(g.layers.length);
+    for (const layer of g.layers) w.u32s(layer.nodes ?? []).u32s(layer.offsets).u32s(layer.neighbours);
+}
+
+export function stateDigest(state: IndexState): string {
+    const w = new ByteWriter();
+    w.str('bf-knowledge-index/digest/1').str(state.fingerprint).u32(state.dims).f64(state.scale);
+    writeParams(w, state.params);
+    const ids = [...state.docs.keys()].sort(compareIds);
+    w.u32(ids.length);
+    for (const id of ids) {
+        const doc = state.docs.get(id)!;
+        w.str(id).str(doc.hash).u32(doc.keys.length);
+        for (const key of doc.keys) w.str(key);
+    }
+    const c = state.compacted;
+    w.u8(c ? 1 : 0);
+    if (c) {
+        w.u32(c.ids.length);
+        for (let i = 0; i < c.ids.length; i++) w.str(c.ids[i]!).str(c.hashes[i]!);
+        w.bytes(c.hubs);
+        writeGraph(w, c.graph);
+    }
+    return sha256Hex(w.finish());
+}

package/src/retrieval/knowledge-index/embeddings.ts

@@ -0,0 +1,85 @@
+/**
+ * embeddings.ts — the index's recompute path: card text → unit float vector,
+ * through a bounded LRU keyed by the card's content hash.
+ *
+ * LEANN's point is that most vectors need not be stored: the dense tier
+ * recomputes float vectors only for the handful of candidates it reranks, and
+ * lean mode recomputes the codes it dropped. Keying by hash means an unchanged
+ * card is never re-embedded while it stays warm, and a changed one never serves
+ * a stale vector.
+ */
+
+import type { EmbedderPort } from './types.js';
+import { toUnit } from './quantize.js';
+
+/** Default number of vectors held. */
+export const EMBEDDING_CACHE_MAX = 8192;
+/** Texts per `embed` call. */
+export const EMBED_BATCH = 256;
+
+export interface Embeddable {
+    hash: string;
+    text: string;
+}
+
+export class EmbeddingCache {
+    private readonly _lru = new Map<string, Float32Array>();
+
+    constructor(
+        private readonly _embedder: EmbedderPort,
+        private readonly _max: number = EMBEDDING_CACHE_MAX,
+    ) {}
+
+    /** Unit vectors for `items`, in order. Misses are embedded in batches of {@link EMBED_BATCH}. */
+    async vectors(items: readonly Embeddable[]): Promise<Float32Array[]> {
+        const out: Array<Float32Array | undefined> = items.map((it) => this._touch(it.hash));
+        const missing: number[] = [];
+        const seen = new Map<string, number>();
+        items.forEach((it, i) => {
+            if (out[i] || seen.has(it.hash)) return;
+            seen.set(it.hash, missing.length);
+            missing.push(i);
+        });
+        if (missing.length > 0) {
+            const raw: Float32Array[] = [];
+            for (let start = 0; start < missing.length; start += EMBED_BATCH) {
+                const batch = missing.slice(start, start + EMBED_BATCH).map((i) => items[i]!.text);
+                raw.push(...(await this._embedder.embed(batch)));
+            }
+            missing.forEach((i, j) => {
+                const vec = raw[j];
+                if (!vec) throw new Error('knowledge-index: embedder returned fewer vectors than texts');
+                this._put(items[i]!.hash, toUnit(vec));
+            });
+            items.forEach((it, i) => {
+                if (!out[i]) out[i] = this._touch(it.hash) ?? toUnit(raw[seen.get(it.hash)!]!);
+            });
+        }
+        return out as Float32Array[];
+    }
+
+    /** A query (not a card) vector: not cached, the query space is unbounded. */
+    async query(text: string): Promise<Float32Array> {
+        const [vec] = await this._embedder.embed([text]);
+        if (!vec) throw new Error('knowledge-index: embedder returned no vector for the query');
+        return toUnit(vec);
+    }
+
+    private _touch(hash: string): Float32Array | undefined {
+        const hit = this._lru.get(hash);
+        if (hit) {
+            this._lru.delete(hash);
+            this._lru.set(hash, hit);
+        }
+        return hit;
+    }
+
+    private _put(hash: string, vec: Float32Array): void {
+        this._lru.delete(hash);
+        this._lru.set(hash, vec);
+        while (this._lru.size > this._max) {
+            const oldest = this._lru.keys().next().value as string;
+            this._lru.delete(oldest);
+        }
+    }
+}

package/src/retrieval/knowledge-index/expand.ts

@@ -0,0 +1,55 @@
+/**
+ * expand.ts — recall expansion over the knowledge graph's edges.
+ *
+ * A hit alone is an isolated fact; its neighbours make it knowledge (a claim's
+ * source section, the claims contradicting it, the entity it is about). Each hop
+ * scores `parent score × edge weight × 0.5`; a node reached more than one way keeps
+ * its best score; the original hits keep theirs. The result is re-ordered by
+ * {@link compareHits}, so it is deterministic whatever order the graph lists
+ * neighbours in.
+ */
+
+import { compareHits } from '../order.js';
+import type { GraphNeighbours, IndexHit } from './types.js';
+
+/** The per-hop decay. */
+export const EXPAND_HOP_DECAY = 0.5;
+
+export interface ExpandOptions {
+    hops?: 1 | 2;
+    edgeKinds?: string[];
+}
+
+export function expandHits(hits: readonly IndexHit[], graph: GraphNeighbours, opts: ExpandOptions = {}): IndexHit[] {
+    const hops = opts.hops ?? 1;
+    const best = new Map<string, IndexHit>();
+    for (const hit of [...hits].sort(compareHits)) {
+        const prev = best.get(hit.id);
+        if (!prev || hit.score > prev.score) best.set(hit.id, { ...hit });
+    }
+    const original = new Set(best.keys());
+
+    let frontier = [...best.values()];
+    for (let hop = 0; hop < hops; hop++) {
+        const reached = new Map<string, IndexHit>();
+        // Parents in total order, and a strict `>` to replace: equal-score ties keep
+        // the first parent in that order, independent of the graph's listing order.
+        for (const parent of frontier.sort(compareHits)) {
+            for (const edge of graph.neighbours(parent.id, opts.edgeKinds)) {
+                if (original.has(edge.id)) continue;
+                const score = parent.score * edge.weight * EXPAND_HOP_DECAY;
+                const prev = reached.get(edge.id);
+                if (!prev || score > prev.score) reached.set(edge.id, { id: edge.id, score, tier: parent.tier });
+            }
+        }
+        frontier = [];
+        for (const [id, hit] of reached) {
+            const prev = best.get(id);
+            if (!prev || hit.score > prev.score) {
+                best.set(id, hit);
+                frontier.push(hit);
+            }
+        }
+    }
+    return [...best.values()].sort(compareHits);
+}

package/src/retrieval/knowledge-index/index.ts

@@ -0,0 +1,162 @@
+/**
+ * The knowledge index — a deterministic, LEANN-style index over knowledge-graph
+ * cards (design: Builderforce.ai/docs/design/deterministic-knowledge-index.md).
+ *
+ * Exact → lexical → dense cascade; int8 codes with integer traversal; a
+ * high-degree-preserving pruned graph stored as CSR; float vectors recomputed
+ * only for the candidates being reranked. Every result is a pure function of
+ * `(digest(), query, opts)`.
+ *
+ * Runs in a Worker, the browser and Node: no Node-only API anywhere.
+ */
+
+import type { EmbedderPort, GraphNeighbours, IndexDoc, IndexHit, IndexParams, IndexTier, KnowledgeIndex, SimilarPair } from './types.js';
+import { availableTiers, canonicalKeys, createState, denseReady, graphNode, indexKeys, type DocEntry, type IndexState } from './state.js';
+import { EmbeddingCache } from './embeddings.js';
+import { cascadeSearch, denseCandidates, type SearchContext } from './cascade.js';
+import { compactState } from './compact.js';
+import { expandHits } from './expand.js';
+import { similarPairs } from './pairs.js';
+import { hashDocText, stateDigest } from './digest.js';
+import { deserializeState, serializeState } from './serialize.js';
+
+class DeterministicKnowledgeIndex implements KnowledgeIndex {
+    private readonly _cache: EmbeddingCache | undefined;
+    private _digest: string | null = null;
+
+    constructor(
+        private readonly _state: IndexState,
+        private readonly _embedder: EmbedderPort | undefined,
+    ) {
+        this._cache = _embedder ? new EmbeddingCache(_embedder) : undefined;
+    }
+
+    private get _ctx(): SearchContext {
+        return { state: this._state, embedder: this._embedder, cache: this._cache };
+    }
+
+    async upsert(docs: IndexDoc[]): Promise<void> {
+        const s = this._state;
+        const changed: DocEntry[] = [];
+        for (const d of docs) {
+            const entry: DocEntry = { id: d.id, text: d.text, keys: canonicalKeys(d.keys), hash: d.hash ?? hashDocText(d.text) };
+            const prev = s.docs.get(d.id);
+            if (prev) indexKeys(s, prev, false);
+            if (prev && prev.hash === entry.hash) {
+                prev.keys = entry.keys;
+                indexKeys(s, prev, true);
+                continue;
+            }
+            s.docs.set(entry.id, entry);
+            indexKeys(s, entry, true);
+            s.lexical.set(entry.id, entry.text);
+            changed.push(entry);
+        }
+        this._digest = null;
+        if (!denseReady(s, this._embedder) || !this._cache) return;
+        // Only the cards that are live and not already coded in the graph need a vector.
+        const toEmbed = changed.filter((e) => s.docs.get(e.id) === e && graphNode(s, e.id) < 0);
+        const vecs = await this._cache.vectors(toEmbed);
+        toEmbed.forEach((e, i) => { e.vec = vecs[i]; });
+    }
+
+    remove(ids: string[]): void {
+        const s = this._state;
+        for (const id of ids) {
+            const doc = s.docs.get(id);
+            if (!doc) continue;
+            indexKeys(s, doc, false);
+            s.lexical.delete(id);
+            s.docs.delete(id);
+        }
+        this._digest = null;
+    }
+
+    search(q: string, opts?: { k?: number; tiers?: IndexTier[] }): Promise<IndexHit[]> {
+        return cascadeSearch(this._ctx, q, opts);
+    }
+
+    expand(hits: IndexHit[], graph: GraphNeighbours, opts?: { hops?: 1 | 2; edgeKinds?: string[] }): IndexHit[] {
+        return expandHits(hits, graph, opts);
+    }
+
+    similarPairs(opts: { minSimilarity: number; maxPairs: number; idPrefix?: string }): SimilarPair[] {
+        return similarPairs(this._state, opts);
+    }
+
+    async compact(): Promise<void> {
+        await compactState(this._state, this._embedder, this._cache);
+        this._digest = null;
+    }
+
+    digest(): string {
+        return (this._digest ??= stateDigest(this._state));
+    }
+
+    tiers(): IndexTier[] {
+        return availableTiers(this._state, this._embedder);
+    }
+
+    size(): number {
+        return this._state.docs.size;
+    }
+
+    serialize(): Uint8Array {
+        return serializeState(this._state);
+    }
+
+    /**
+     * Diagnostics: the dense candidate ids for `q` before the float rerank (the
+     * integer graph walk plus not-yet-compacted cards). Empty without the dense tier.
+     */
+    async denseCandidates(q: string, n: number): Promise<string[]> {
+        if (!denseReady(this._state, this._embedder) || !this._cache) return [];
+        return denseCandidates(this._ctx, await this._cache.query(q), n);
+    }
+
+    /** Diagnostics: stored bytes of the int8 codes and the CSR graph, against float32. */
+    storage(): KnowledgeIndexStorage {
+        const c = this._state.compacted;
+        if (!c) return { nodes: 0, codeBytes: 0, graphBytes: 0, float32Bytes: 0 };
+        let codeBytes = 0;
+        for (const code of c.codes) codeBytes += code?.byteLength ?? 0;
+        let graphBytes = 0;
+        for (const l of c.graph.layers) graphBytes += (l.nodes?.byteLength ?? 0) + l.offsets.byteLength + l.neighbours.byteLength;
+        return { nodes: c.ids.length, codeBytes, graphBytes, float32Bytes: c.ids.length * this._state.dims * 4 };
+    }
+}
+
+export interface KnowledgeIndexStorage {
+    nodes: number;
+    codeBytes: number;
+    graphBytes: number;
+    /** What the same nodes would take as float32 vectors. */
+    float32Bytes: number;
+}
+
+/** The concrete index type: the contract plus diagnostics. */
+export type DeterministicKnowledgeIndexType = KnowledgeIndex & {
+    denseCandidates(q: string, n: number): Promise<string[]>;
+    storage(): KnowledgeIndexStorage;
+};
+
+export function createKnowledgeIndex(
+    opts: { embedder?: EmbedderPort; params?: Partial<IndexParams> } = {},
+): DeterministicKnowledgeIndexType {
+    return new DeterministicKnowledgeIndex(createState(opts.params, opts.embedder), opts.embedder);
+}
+
+/**
+ * Loads a serialized index. With no embedder, or one of another fingerprint, the
+ * dense tier is absent until `compact()` re-embeds every card under the new one.
+ */
+export function loadKnowledgeIndex(bytes: Uint8Array, embedder?: EmbedderPort): DeterministicKnowledgeIndexType {
+    return new DeterministicKnowledgeIndex(deserializeState(bytes), embedder);
+}
+
+export type { EmbedderPort, GraphNeighbours, IndexDoc, IndexHit, IndexParams, IndexTier, KnowledgeIndex, SimilarPair } from './types.js';
+export { DEFAULT_INDEX_PARAMS } from './state.js';
+export { codeTokenize } from './tokenize.js';
+export { hashDocText } from './digest.js';
+export { sha256, sha256Hex, Sha256 } from './sha256.js';
+export { EXPAND_HOP_DECAY } from './expand.js';

package/src/retrieval/knowledge-index/lexical.ts

@@ -0,0 +1,78 @@
+/**
+ * lexical.ts — the knowledge index's BM25 tier, as an incremental inverted index.
+ *
+ * `bm25Search` rebuilds its term table per call, which is right for a bounded
+ * candidate set and wrong for 50k cards queried on every prompt. This keeps the
+ * postings across upserts and scores with the SAME Okapi core (`bm25Idf`,
+ * `bm25LengthNorm`, `bm25TermScore`) — the formula is not restated here.
+ */
+
+import { bm25Idf, bm25LengthNorm, bm25TermScore } from '../bm25.js';
+import { compareHits } from '../order.js';
+import { codeTokenize } from './tokenize.js';
+
+export interface LexicalHit {
+    id: string;
+    score: number;
+}
+
+export class LexicalIndex {
+    /** term → (doc id → term frequency). */
+    private readonly _postings = new Map<string, Map<string, number>>();
+    /** doc id → its distinct terms and token count. */
+    private readonly _docs = new Map<string, { terms: string[]; len: number }>();
+    private _totalLen = 0;
+
+    get size(): number {
+        return this._docs.size;
+    }
+
+    /** Indexes (or re-indexes) one document. */
+    set(id: string, text: string): void {
+        this.delete(id);
+        const tokens = codeTokenize(text);
+        const tf = new Map<string, number>();
+        for (const t of tokens) tf.set(t, (tf.get(t) ?? 0) + 1);
+        for (const [term, f] of tf) {
+            let posting = this._postings.get(term);
+            if (!posting) this._postings.set(term, (posting = new Map()));
+            posting.set(id, f);
+        }
+        this._docs.set(id, { terms: [...tf.keys()], len: tokens.length });
+        this._totalLen += tokens.length;
+    }
+
+    delete(id: string): void {
+        const doc = this._docs.get(id);
+        if (!doc) return;
+        for (const term of doc.terms) {
+            const posting = this._postings.get(term);
+            posting?.delete(id);
+            if (posting && posting.size === 0) this._postings.delete(term);
+        }
+        this._totalLen -= doc.len;
+        this._docs.delete(id);
+    }
+
+    /** BM25 hits for `query`, best first ({@link compareHits}), at most `limit`. */
+    search(query: string, limit: number): LexicalHit[] {
+        const n = this._docs.size;
+        if (n === 0 || limit <= 0) return [];
+        const terms = [...new Set(codeTokenize(query))].sort();
+        const avgdl = this._totalLen / n || 1;
+        const scores = new Map<string, number>();
+        // Terms in sorted order, so each doc's sum is accumulated in the same order everywhere.
+        for (const term of terms) {
+            const posting = this._postings.get(term);
+            if (!posting) continue;
+            const idf = bm25Idf(n, posting.size);
+            for (const [id, f] of posting) {
+                const norm = bm25LengthNorm(this._docs.get(id)!.len, avgdl);
+                scores.set(id, (scores.get(id) ?? 0) + bm25TermScore(f, idf, norm));
+            }
+        }
+        const hits: LexicalHit[] = [];
+        for (const [id, score] of scores) if (score > 0) hits.push({ id, score });
+        return hits.sort(compareHits).slice(0, limit);
+    }
+}

package/src/retrieval/knowledge-index/pairs.ts

@@ -0,0 +1,57 @@
+/**
+ * pairs.ts — near-duplicate pairs over the int8 codes (the Duplication lens,
+ * review-time entity resolution, contradiction candidates).
+ *
+ * Near-duplicates are each other's nearest neighbours, so the candidate pairs are
+ * the compacted graph's edges on every layer rather than all n² pairs. Similarity
+ * is {@link cosineInt8} (exact integers, correctly-rounded sqrt/division). The
+ * order is total: similarity (quantized 1e-4) descending, then `a`, then `b`, and
+ * `a < b` by id in every pair.
+ *
+ * Cards not yet compacted, and lean-mode nodes whose codes were dropped, are not
+ * paired until the next compaction (the call is synchronous; it never embeds).
+ */
+
+import { compareIds, quantizeScore, DENSE_SCORE_QUANTUM } from '../order.js';
+import type { SimilarPair } from './types.js';
+import { graphNode, type IndexState } from './state.js';
+import { neighboursOf } from './csr.js';
+import { cosineInt8 } from './quantize.js';
+
+export interface SimilarPairsOptions {
+    minSimilarity: number;
+    maxPairs: number;
+    idPrefix?: string;
+}
+
+export function similarPairs(state: IndexState, opts: SimilarPairsOptions): SimilarPair[] {
+    const c = state.compacted;
+    if (!c || opts.maxPairs <= 0) return [];
+    const prefix = opts.idPrefix ?? '';
+    const usable = (node: number) =>
+        !!c.codes[node] && c.ids[node]!.startsWith(prefix) && graphNode(state, c.ids[node]!) === node;
+
+    const seen = new Set<string>();
+    const pairs: Array<SimilarPair & { q: number }> = [];
+    for (let a = 0; a < c.ids.length; a++) {
+        if (!usable(a)) continue;
+        for (const layer of c.graph.layers) {
+            for (const b of neighboursOf(layer, a)) {
+                if (b === a || !usable(b)) continue;
+                const lo = Math.min(a, b);
+                const hi = Math.max(a, b);
+                const key = `${lo}:${hi}`;
+                if (seen.has(key)) continue;
+                seen.add(key);
+                const similarity = cosineInt8(c.codes[lo]!, c.codes[hi]!);
+                if (similarity >= opts.minSimilarity) {
+                    pairs.push({ a: c.ids[lo]!, b: c.ids[hi]!, similarity, q: quantizeScore(similarity, DENSE_SCORE_QUANTUM) });
+                }
+            }
+        }
+    }
+    return pairs
+        .sort((x, y) => (y.q - x.q) || compareIds(x.a, y.a) || compareIds(x.b, y.b))
+        .slice(0, opts.maxPairs)
+        .map(({ a, b, similarity }) => ({ a, b, similarity }));
+}

package/src/retrieval/knowledge-index/prune.ts

@@ -0,0 +1,48 @@
+/**
+ * prune.ts — high-degree-preserving pruning of the base layer (the LEANN storage
+ * technique).
+ *
+ * Most of a graph index's bytes are base-layer edges. Searches route through the
+ * few well-connected hubs, so those keep their full degree and every other node
+ * keeps half: the top `hubShare` of nodes by in-degree keep `m` edges, the rest
+ * `m / 2`. Upper layers are tiny and stay unpruned.
+ *
+ * Every choice is a total order (in-degree, then node index; similarity, then
+ * node index — and node index is id order), so pruning is deterministic.
+ */
+
+import { compareScoreThenId, DENSE_SCORE_QUANTUM } from '../order.js';
+
+/**
+ * Prunes base-layer rows. `similarity(a, b)` scores two nodes (higher = nearer).
+ * Each kept row is ordered nearest first.
+ */
+export function pruneBaseLayer(
+    rows: ReadonlyArray<readonly number[]>,
+    m: number,
+    hubShare: number,
+    similarity: (a: number, b: number) => number,
+): { rows: number[][]; hubs: Uint8Array } {
+    const n = rows.length;
+    const inDegree = new Uint32Array(n);
+    for (const row of rows) for (const to of row) inDegree[to] += 1;
+
+    const hubs = new Uint8Array(n);
+    const hubCount = n === 0 ? 0 : Math.min(n, Math.max(1, Math.ceil(n * Math.max(0, hubShare))));
+    const byDegree = Array.from({ length: n }, (_, i) => i)
+        .sort((a, b) => inDegree[b]! - inDegree[a]! || a - b);
+    for (let i = 0; i < hubCount; i++) hubs[byDegree[i]!] = 1;
+
+    const full = Math.max(1, m);
+    const half = Math.max(1, Math.floor(m / 2));
+    const pruned = rows.map((row, node) => {
+        const keep = hubs[node] ? full : half;
+        return [...new Set(row)]
+            .filter((to) => to !== node)
+            .map((to) => ({ to, score: similarity(node, to) }))
+            .sort((a, b) => compareScoreThenId(a.score, a.to, b.score, b.to, DENSE_SCORE_QUANTUM))
+            .slice(0, keep)
+            .map((e) => e.to);
+    });
+    return { rows: pruned, hubs };
+}