@seanhogg/builderforce-memory 2026.6.20 → 2026.6.28

package/src/retrieval/chunk.ts

@@ -0,0 +1,101 @@
+/**
+ * Document chunking — recursive character text splitter with overlap.
+ *
+ * The classic RAG ingestion step the memory layer was missing: large documents
+ * are split into smaller, semantically-coherent chunks before embedding so that
+ * retrieval returns precise passages rather than whole files. Mirrors the
+ * behaviour of LangChain's RecursiveCharacterTextSplitter (split on the largest
+ * natural boundary that fits, fall back to finer ones) but is zero-dependency.
+ */
+
+export interface ChunkOptions {
+    /** Target maximum chunk size in characters. Default 1000. */
+    chunkSize?: number;
+    /** Characters of overlap carried from the end of one chunk into the next,
+     *  preserving context across boundaries. Default 200. Clamped below chunkSize. */
+    chunkOverlap?: number;
+    /** Separators tried in order, largest natural boundary first. */
+    separators?: string[];
+}
+
+export interface Chunk {
+    /** The chunk text. */
+    text: string;
+    /** 0-based ordinal of this chunk within its source document. */
+    index: number;
+    /** Character offset of this chunk's start within the original document. */
+    start: number;
+}
+
+const DEFAULT_SEPARATORS = ['\n\n', '\n', '. ', ' ', ''];
+
+/**
+ * Splits `text` into overlapping chunks no larger than `chunkSize` characters,
+ * preferring the largest natural separator that keeps a piece under the limit.
+ * Returns `[]` for empty/whitespace input. Deterministic and pure.
+ */
+export function chunkText(text: string, opts: ChunkOptions = {}): Chunk[] {
+    const chunkSize = Math.max(1, opts.chunkSize ?? 1000);
+    const overlap = Math.min(Math.max(0, opts.chunkOverlap ?? 200), chunkSize - 1);
+    const separators = opts.separators ?? DEFAULT_SEPARATORS;
+
+    const trimmed = text.trim();
+    if (trimmed.length === 0) return [];
+    if (trimmed.length <= chunkSize) return [{ text: trimmed, index: 0, start: 0 }];
+
+    const pieces = splitRecursive(trimmed, chunkSize, separators);
+
+    // Merge adjacent pieces up to chunkSize, then stitch overlap between chunks.
+    const chunks: Chunk[] = [];
+    let buf = '';
+    const flush = () => {
+        const t = buf.trim();
+        if (t.length > 0) {
+            const start = chunks.length === 0 ? 0 : Math.max(0, trimmed.indexOf(t));
+            chunks.push({ text: t, index: chunks.length, start });
+        }
+        buf = '';
+    };
+
+    for (const piece of pieces) {
+        if (buf.length + piece.length <= chunkSize) {
+            buf += piece;
+        } else {
+            flush();
+            // Carry overlap from the previous chunk's tail.
+            const prev = chunks[chunks.length - 1]?.text ?? '';
+            buf = (overlap > 0 ? prev.slice(-overlap) : '') + piece;
+            // A single piece longer than chunkSize is hard-split.
+            while (buf.length > chunkSize) {
+                const head = buf.slice(0, chunkSize);
+                chunks.push({ text: head.trim(), index: chunks.length, start: 0 });
+                buf = (overlap > 0 ? head.slice(-overlap) : '') + buf.slice(chunkSize);
+            }
+        }
+    }
+    flush();
+
+    return chunks.map((c, i) => ({ ...c, index: i }));
+}
+
+/** Recursively splits text on the first separator that yields sub-chunkSize pieces. */
+function splitRecursive(text: string, chunkSize: number, separators: string[]): string[] {
+    /* istanbul ignore next -- defensive base case; callers only recurse on parts > chunkSize */
+    if (text.length <= chunkSize) return [text];
+    const [sep, ...rest] = separators;
+    if (sep === undefined) return [text];
+    if (sep === '') {
+        // Last resort: hard character split.
+        const out: string[] = [];
+        for (let i = 0; i < text.length; i += chunkSize) out.push(text.slice(i, i + chunkSize));
+        return out;
+    }
+    const parts = text.split(sep);
+    const out: string[] = [];
+    for (let i = 0; i < parts.length; i++) {
+        const part = i < parts.length - 1 ? parts[i]! + sep : parts[i]!;
+        if (part.length > chunkSize) out.push(...splitRecursive(part, chunkSize, rest));
+        else out.push(part);
+    }
+    return out;
+}

package/src/retrieval/fusion.ts

@@ -0,0 +1,84 @@
+/**
+ * Rank fusion + diversity reranking.
+ *
+ *   • Reciprocal Rank Fusion (RRF) merges the dense (vector) and sparse (BM25)
+ *     rankings into one list without needing the two score scales to be
+ *     commensurable — it fuses on RANK, not raw score. This is the standard,
+ *     parameter-light way to combine hybrid retrieval signals.
+ *   • Maximal Marginal Relevance (MMR) reranks the fused list to trade off
+ *     relevance against novelty, so the top-k isn't five near-duplicate chunks.
+ *
+ * Pure and zero-dependency.
+ */
+
+import { cosineSimilarity } from '../similarity/index.js';
+
+export interface RankedList {
+    /** Ordered ids, most relevant first. */
+    ids: string[];
+    /** Optional weight for this list in the fusion (default 1). */
+    weight?: number;
+}
+
+export interface FusedHit {
+    id: string;
+    score: number;
+}
+
+/**
+ * Reciprocal Rank Fusion over any number of ranked lists.
+ * score(d) = Σ_lists weight / (k + rank(d)).  `k` (default 60) damps the
+ * contribution of low-ranked items; the canonical TREC value.
+ */
+export function reciprocalRankFusion(lists: RankedList[], k = 60): FusedHit[] {
+    const acc = new Map<string, number>();
+    for (const list of lists) {
+        const weight = list.weight ?? 1;
+        list.ids.forEach((id, rank) => {
+            acc.set(id, (acc.get(id) ?? 0) + weight / (k + rank + 1));
+        });
+    }
+    return [...acc.entries()]
+        .map(([id, score]) => ({ id, score }))
+        .sort((a, b) => b.score - a.score);
+}
+
+export interface MmrCandidate {
+    id: string;
+    vector: Float32Array;
+}
+
+/**
+ * Maximal Marginal Relevance rerank. Greedily selects up to `topK` candidates,
+ * each step maximising `λ·sim(query, d) − (1−λ)·max sim(d, already-selected)`.
+ * λ=1 is pure relevance; lower λ injects diversity. Candidates without vectors
+ * should be filtered out by the caller (they cannot be MMR-scored).
+ */
+export function maximalMarginalRelevance(
+    queryVec: Float32Array,
+    candidates: MmrCandidate[],
+    topK: number,
+    lambda = 0.7,
+): string[] {
+    const remaining = [...candidates];
+    const selected: MmrCandidate[] = [];
+    const relevance = new Map<string, number>();
+    for (const c of remaining) relevance.set(c.id, cosineSimilarity(queryVec, c.vector));
+
+    while (selected.length < topK && remaining.length > 0) {
+        let bestIdx = 0;
+        let bestScore = -Infinity;
+        for (let i = 0; i < remaining.length; i++) {
+            const c = remaining[i]!;
+            let maxSimToSelected = 0;
+            for (const s of selected) {
+                const sim = cosineSimilarity(c.vector, s.vector);
+                if (sim > maxSimToSelected) maxSimToSelected = sim;
+            }
+            const mmr = lambda * relevance.get(c.id)! - (1 - lambda) * maxSimToSelected;
+            if (mmr > bestScore) { bestScore = mmr; bestIdx = i; }
+        }
+        selected.push(remaining.splice(bestIdx, 1)[0]!);
+    }
+    return selected.map(s => s.id);
+}

package/src/retrieval/index.ts

@@ -0,0 +1,24 @@
+/**
+ * Retrieval layer — chunking, BM25, rank fusion, and the HybridRetriever.
+ *
+ * The classic RAG pieces the memory stack previously lacked (chunking, hybrid
+ * dense+sparse search, reranking), implemented zero-dependency so they run in the
+ * browser, Node, and the SSM runtime alike.
+ */
+
+export { chunkText } from './chunk.js';
+export type { Chunk, ChunkOptions } from './chunk.js';
+
+export { bm25Search } from './bm25.js';
+export type { Bm25Doc, Bm25Hit, Bm25Options } from './bm25.js';
+
+export { reciprocalRankFusion, maximalMarginalRelevance } from './fusion.js';
+export type { RankedList, FusedHit, MmrCandidate } from './fusion.js';
+
+export { hybridRetrieve } from './HybridRetriever.js';
+export type {
+    RetrievalCandidate,
+    HybridQuery,
+    HybridRetrieveOptions,
+    HybridHit,
+} from './HybridRetriever.js';