@opengroundplan/core 0.1.1

package/templates/architectures/rag-hybrid/files/src/rag/evaluation/harness.ts

@@ -0,0 +1,101 @@
+/**
+ * Scores every retriever variant against the golden set.
+ *
+ * The point is the comparison, not the absolute number: dense-only and keyword-only are the
+ * baselines that justify paying for two indexes. If fused does not beat both on your corpus,
+ * the hybrid is not earning its complexity and the eval says so before production does.
+ */
+import { readFile } from 'node:fs/promises';
+import type { Settings } from '../config.js';
+import type { Scored } from '../contracts.js';
+import { buildEmbedder, type Embedder } from '../embed/index.js';
+import { buildReranker, rerank } from '../rerank/index.js';
+import { dense, keyword, reciprocalRankFusion, retrieve } from '../retrieve/index.js';
+import type { RetrievalIndex } from '../store/index.js';
+import { firstSeen, mrr, ndcgAtK, recallAtK } from './metrics.js';
+
+export interface GoldenQuery {
+  readonly id: string;
+  readonly query: string;
+  readonly relevantDocumentIds: readonly string[];
+}
+
+export interface VariantScore {
+  readonly variant: string;
+  readonly recall: number;
+  readonly ndcg: number;
+  readonly mrr: number;
+  readonly queries: number;
+}
+
+type Variant = (
+  query: string,
+  index: RetrievalIndex,
+  embedder: Embedder,
+  settings: Settings,
+) => Promise<readonly Scored[]>;
+
+async function fused(
+  query: string,
+  index: RetrievalIndex,
+  embedder: Embedder,
+  settings: Settings,
+): Promise<readonly Scored[]> {
+  const lists = {
+    dense: await dense.search(query, index, embedder, settings),
+    keyword: keyword.search(query, index, settings),
+  };
+  return reciprocalRankFusion(lists, settings.rrfK, settings.candidateK);
+}
+
+/** Ordered worst-expected to best; a run where this order does not hold is the finding. */
+export const VARIANTS: Readonly<Record<string, Variant>> = {
+  'dense-only': (query, index, embedder, settings) => dense.search(query, index, embedder, settings),
+  'keyword-only': async (query, index, _embedder, settings) => keyword.search(query, index, settings),
+  'fused-rrf': fused,
+  'fused-rrf+rerank': async (query, index, embedder, settings) =>
+    rerank(await retrieve(query, index, embedder, settings), buildReranker(settings), settings),
+};
+
+export async function loadGoldenSet(path: string): Promise<GoldenQuery[]> {
+  const body = await readFile(path, 'utf8');
+  return body
+    .split('\n')
+    .filter((line) => line.trim())
+    .map((line) => JSON.parse(line) as GoldenQuery);
+}
+
+export async function evaluateGoldenSet(
+  index: RetrievalIndex,
+  settings: Settings,
+  golden?: readonly GoldenQuery[],
+): Promise<VariantScore[]> {
+  const queries = golden ?? (await loadGoldenSet(settings.goldenPath));
+  const embedder = buildEmbedder(settings);
+  const k = settings.rerankTopN;
+
+  const scores: VariantScore[] = [];
+  for (const [variant, run] of Object.entries(VARIANTS)) {
+    let recall = 0;
+    let ndcg = 0;
+    let reciprocal = 0;
+
+    for (const item of queries) {
+      const hits = await run(item.query, index, embedder, settings);
+      const ranked = firstSeen(hits.map((hit) => hit.chunk.documentId));
+      recall += recallAtK(ranked, item.relevantDocumentIds, k);
+      ndcg += ndcgAtK(ranked, item.relevantDocumentIds, k);
+      reciprocal += mrr(ranked, item.relevantDocumentIds);
+    }
+
+    const total = queries.length || 1;
+    scores.push({
+      variant,
+      recall: recall / total,
+      ndcg: ndcg / total,
+      mrr: reciprocal / total,
+      queries: queries.length,
+    });
+  }
+  return scores;
+}

package/templates/architectures/rag-hybrid/files/src/rag/evaluation/metrics.ts

@@ -0,0 +1,40 @@
+/**
+ * Rank-aware retrieval metrics, binary relevance.
+ *
+ * Scored at document level, not chunk level: a golden set pinned to chunk ids becomes invalid
+ * the moment you retune chunk size, which is exactly the change you most want to measure.
+ */
+
+/** Collapse a chunk ranking to a document ranking, keeping the best rank per document. */
+export function firstSeen(documentIds: Iterable<string>): string[] {
+  return [...new Set(documentIds)];
+}
+
+/** The ceiling on everything downstream — a document missed here cannot be reranked back. */
+export function recallAtK(ranked: readonly string[], relevant: readonly string[], k: number): number {
+  if (relevant.length === 0) return 0;
+  const top = new Set(ranked.slice(0, k));
+  const hits = relevant.filter((id) => top.has(id)).length;
+  return hits / relevant.length;
+}
+
+export function ndcgAtK(ranked: readonly string[], relevant: readonly string[], k: number): number {
+  const relevantSet = new Set(relevant);
+  if (relevantSet.size === 0) return 0;
+
+  let gain = 0;
+  ranked.slice(0, k).forEach((documentId, at) => {
+    if (relevantSet.has(documentId)) gain += 1 / Math.log2(at + 2);
+  });
+
+  let ideal = 0;
+  for (let at = 0; at < Math.min(relevantSet.size, k); at += 1) ideal += 1 / Math.log2(at + 2);
+
+  return ideal ? gain / ideal : 0;
+}
+
+export function mrr(ranked: readonly string[], relevant: readonly string[]): number {
+  const relevantSet = new Set(relevant);
+  const at = ranked.findIndex((documentId) => relevantSet.has(documentId));
+  return at === -1 ? 0 : 1 / (at + 1);
+}

package/templates/architectures/rag-hybrid/files/src/rag/generate/index.ts

@@ -0,0 +1,86 @@
+/**
+ * Stage 7 — answer synthesis.
+ *
+ * This module imports no retrieval stage, by contract. Passages arrive as an argument, already
+ * fused and reranked, and the type says so. That is the whole enforcement story: generation
+ * physically cannot fetch its own context, so every answer is grounded in something the eval
+ * harness measured.
+ */
+import type { Settings } from '../config.js';
+import type { RerankedPassages } from '../contracts.js';
+
+const INSTRUCTION =
+  'Answer using only the numbered context below. Cite the sources you used by their number. ' +
+  'If the context does not contain the answer, say so.';
+
+/** Numbered passages so the model can cite, and the caller can resolve a citation. */
+export function buildPrompt(query: string, passages: RerankedPassages): string {
+  const blocks = passages.map(
+    (item, at) =>
+      `[${at + 1}] source=${item.chunk.source} chunk=${item.chunk.id}\n${item.chunk.text}`,
+  );
+  const context = blocks.length ? blocks.join('\n\n') : '(no passages retrieved)';
+  return `${INSTRUCTION}\n\nContext:\n${context}\n\nQuestion: ${query}`;
+}
+
+export interface GenerationClient {
+  complete(prompt: string): Promise<string>;
+}
+
+/**
+ * Returns the prompt. Default provider so the pipeline runs end to end without a key, and so
+ * prompt-shape changes are reviewable in a diff.
+ */
+export class EchoClient implements GenerationClient {
+  async complete(prompt: string): Promise<string> {
+    return prompt;
+  }
+}
+
+/** OpenAI-shaped chat completions endpoint. Nothing here names a model. */
+export class HttpGenerationClient implements GenerationClient {
+  private readonly settings: Settings;
+
+  constructor(settings: Settings) {
+    if (!settings.generationEndpoint || !settings.generationModel) {
+      throw new Error('GENERATION_ENDPOINT and GENERATION_MODEL must be set for the http generator');
+    }
+    this.settings = settings;
+  }
+
+  async complete(prompt: string): Promise<string> {
+    const { settings } = this;
+    const response = await fetch(settings.generationEndpoint, {
+      method: 'POST',
+      headers: {
+        'content-type': 'application/json',
+        authorization: `Bearer ${settings.generationApiKey}`,
+      },
+      body: JSON.stringify({
+        model: settings.generationModel,
+        messages: [{ role: 'user', content: prompt }],
+      }),
+      signal: AbortSignal.timeout(settings.requestTimeoutMs),
+    });
+    if (!response.ok) throw new Error(`Generation request failed: ${response.status}`);
+
+    const body = (await response.json()) as { choices?: { message?: { content?: string } }[] };
+    const content = body.choices?.[0]?.message?.content;
+    if (content === undefined) throw new Error('Generation response carried no content');
+    return content;
+  }
+}
+
+export function buildClient(settings: Settings): GenerationClient {
+  if (settings.generationProvider === 'http') return new HttpGenerationClient(settings);
+  return new EchoClient();
+}
+
+/** Only RerankedPassages is accepted — unranked fusion output is a type error. */
+export function answer(
+  query: string,
+  passages: RerankedPassages,
+  client: GenerationClient,
+): Promise<string> {
+  return client.complete(buildPrompt(query, passages));
+}

package/templates/architectures/rag-hybrid/files/src/rag/indexing.ts

@@ -0,0 +1,21 @@
+/**
+ * Composition root for the offline half: ingest -> chunk -> embed -> index.
+ *
+ * Separate from pipeline.ts so the eval harness can build an index without pulling generation
+ * into its import graph — the boundary check enforces exactly that.
+ */
+import { chunkDocuments } from './chunk/index.js';
+import { loadSettings, type Settings } from './config.js';
+import { buildEmbedder, embedChunks } from './embed/index.js';
+import { ingest } from './ingest/index.js';
+import { buildIndex as buildStoreIndex, type InMemoryIndex } from './store/index.js';
+
+/** Run when the corpus changes, not per query. */
+export async function buildIndex(
+  settings: Settings = loadSettings(),
+): Promise<{ index: InMemoryIndex; settings: Settings }> {
+  const documents = await ingest(settings);
+  const chunks = chunkDocuments(documents, settings);
+  const embedded = await embedChunks(chunks, buildEmbedder(settings), settings.embedBatchSize);
+  return { index: buildStoreIndex(embedded), settings };
+}

package/templates/architectures/rag-hybrid/files/src/rag/ingest/index.ts

@@ -0,0 +1,58 @@
+/**
+ * Stage 1 — sources become Documents.
+ *
+ * Everything vendor-specific about a source (a filesystem, S3, a CMS, a crawler) is confined
+ * here. Downstream stages only ever see Document.
+ */
+import { readdir, readFile } from 'node:fs/promises';
+import path from 'node:path';
+import type { Settings } from '../config.js';
+import type { Document } from '../contracts.js';
+
+/** Add a connector by implementing this, not by editing the chunker. */
+export interface Source {
+  read(): Promise<Document[]>;
+}
+
+/**
+ * Reads text files under a root. Document ids are root-relative paths, so a re-ingest of
+ * unchanged content produces identical ids and the golden set stays valid.
+ */
+export class FileSystemSource implements Source {
+  // Fields are declared and assigned rather than declared as constructor parameters:
+  // parameter properties emit runtime code, which `erasableSyntaxOnly` forbids.
+  private readonly root: string;
+  private readonly extensions: readonly string[];
+
+  constructor(root: string, extensions: readonly string[]) {
+    this.root = root;
+    this.extensions = extensions;
+  }
+
+  async read(): Promise<Document[]> {
+    const files = (await this.walk(this.root)).sort();
+    return Promise.all(
+      files.map(async (file) => ({
+        id: path.relative(this.root, file).split(path.sep).join('/'),
+        source: file,
+        text: await readFile(file, 'utf8'),
+        metadata: { extension: path.extname(file) },
+      })),
+    );
+  }
+
+  private async walk(dir: string): Promise<string[]> {
+    const entries = await readdir(dir, { withFileTypes: true }).catch(() => []);
+    const found: string[] = [];
+    for (const entry of entries) {
+      const child = path.join(dir, entry.name);
+      if (entry.isDirectory()) found.push(...(await this.walk(child)));
+      else if (this.extensions.includes(path.extname(entry.name))) found.push(child);
+    }
+    return found;
+  }
+}
+
+export function ingest(settings: Settings): Promise<Document[]> {
+  return new FileSystemSource(settings.corpusDir, settings.corpusExtensions).read();
+}

package/templates/architectures/rag-hybrid/files/src/rag/pipeline.ts

@@ -0,0 +1,32 @@
+/**
+ * Composition root for the online half: retrieve -> rerank -> generate.
+ *
+ * The only module that knows both sides of the seam. Wiring lives here so no stage has to
+ * know what runs after it.
+ */
+import type { Settings } from './config.js';
+import type { RerankedPassages } from './contracts.js';
+import { buildEmbedder } from './embed/index.js';
+import { answer, buildClient } from './generate/index.js';
+import { buildReranker, rerank } from './rerank/index.js';
+import { retrieve } from './retrieve/index.js';
+import type { RetrievalIndex } from './store/index.js';
+
+/** Everything up to the seam: hybrid retrieval, fusion, rerank. No model involved. */
+export async function retrievePassages(
+  query: string,
+  index: RetrievalIndex,
+  settings: Settings,
+): Promise<RerankedPassages> {
+  const fused = await retrieve(query, index, buildEmbedder(settings), settings);
+  return rerank(fused, buildReranker(settings), settings);
+}
+
+export async function answerQuestion(
+  query: string,
+  index: RetrievalIndex,
+  settings: Settings,
+): Promise<string> {
+  const passages = await retrievePassages(query, index, settings);
+  return answer(query, passages, buildClient(settings));
+}

package/templates/architectures/rag-hybrid/files/src/rag/rerank/index.ts

@@ -0,0 +1,90 @@
+/**
+ * Stage 6 — reranking, the last gate before the model.
+ *
+ * Fusion optimizes recall over a wide candidate set; the reranker scores each candidate
+ * against the query directly and cuts it to the handful that fit in the prompt. This module is
+ * the only place that mints RerankedPassages, so nothing else can hand raw candidates to
+ * generation and still typecheck.
+ */
+import type { Settings } from '../config.js';
+import type { RerankedPassages, RetrievalResult, Scored } from '../contracts.js';
+import { tokenize } from '../store/index.js';
+
+export interface Reranker {
+  score(query: string, passages: readonly Scored[]): Promise<number[]>;
+}
+
+/**
+ * Zero-dependency stand-in: query-term coverage of the passage. It is a placeholder for a
+ * cross-encoder, not a substitute for one — point RERANK_PROVIDER at a real model before this
+ * pipeline is load-bearing.
+ */
+export class LexicalReranker implements Reranker {
+  async score(query: string, passages: readonly Scored[]): Promise<number[]> {
+    const terms = new Set(tokenize(query));
+    if (terms.size === 0) return passages.map((item) => item.score);
+
+    return passages.map((item) => {
+      const words = new Set(tokenize(item.chunk.text));
+      let hits = 0;
+      for (const term of terms) if (words.has(term)) hits += 1;
+      return hits / terms.size;
+    });
+  }
+}
+
+/** Cohere/Jina-shaped rerank endpoint. Endpoint, model and key all come from settings. */
+export class HttpReranker implements Reranker {
+  private readonly settings: Settings;
+
+  constructor(settings: Settings) {
+    if (!settings.rerankEndpoint || !settings.rerankModel) {
+      throw new Error('RERANK_ENDPOINT and RERANK_MODEL must be set for the http reranker');
+    }
+    this.settings = settings;
+  }
+
+  async score(query: string, passages: readonly Scored[]): Promise<number[]> {
+    const { settings } = this;
+    const response = await fetch(settings.rerankEndpoint, {
+      method: 'POST',
+      headers: {
+        'content-type': 'application/json',
+        authorization: `Bearer ${settings.rerankApiKey}`,
+      },
+      body: JSON.stringify({
+        model: settings.rerankModel,
+        query,
+        documents: passages.map((item) => item.chunk.text),
+      }),
+      signal: AbortSignal.timeout(settings.requestTimeoutMs),
+    });
+    if (!response.ok) throw new Error(`Rerank request failed: ${response.status}`);
+
+    const body = (await response.json()) as { results: { index: number; relevance_score: number }[] };
+    const scores = new Array<number>(passages.length).fill(0);
+    for (const entry of body.results) scores[entry.index] = entry.relevance_score;
+    return scores;
+  }
+}
+
+export function buildReranker(settings: Settings): Reranker {
+  if (settings.rerankProvider === 'http') return new HttpReranker(settings);
+  return new LexicalReranker();
+}
+
+export async function rerank(
+  result: RetrievalResult,
+  reranker: Reranker,
+  settings: Settings,
+): Promise<RerankedPassages> {
+  const scores = await reranker.score(result.query, result.passages);
+  const rescored = result.passages
+    .map((item, at): Scored => ({ chunk: item.chunk, score: scores[at] ?? 0 }))
+    .sort((a, b) => b.score - a.score)
+    .slice(0, settings.rerankTopN);
+
+  // The one place the brand is applied. A double assertion is required because nothing can
+  // structurally satisfy a unique symbol — which is exactly what makes the brand unforgeable.
+  return rescored as unknown as RerankedPassages;
+}

package/templates/architectures/rag-hybrid/files/src/rag/retrieve/dense.ts

@@ -0,0 +1,26 @@
+/**
+ * Dense retriever — conceptual similarity, no keyword overlap required.
+ *
+ * This is the half that finds "how do I cancel" for a query about "refund policy". It is also
+ * the half that misses exact tokens like part numbers, which is why it is never used alone.
+ */
+import type { Settings } from '../config.js';
+import type { Scored } from '../contracts.js';
+import type { Embedder } from '../embed/index.js';
+import { cosine, type RetrievalIndex } from '../store/index.js';
+
+/** Query and corpus must share an embedder or the vector space does not line up. */
+export async function search(
+  query: string,
+  index: RetrievalIndex,
+  embedder: Embedder,
+  settings: Settings,
+): Promise<Scored[]> {
+  const [queryVector] = await embedder.embed([query]);
+  if (!queryVector) throw new Error('Embedder returned no vector for the query');
+
+  return index.chunks
+    .map((chunk, position): Scored => ({ chunk, score: cosine(queryVector, index.vectorOf(position)) }))
+    .sort((a, b) => b.score - a.score)
+    .slice(0, settings.candidateK);
+}

package/templates/architectures/rag-hybrid/files/src/rag/retrieve/fusion.ts

@@ -0,0 +1,32 @@
+/**
+ * Reciprocal rank fusion — combines ranked lists without tuning.
+ *
+ * score(d) = sum over lists of 1 / (k + rank(d)), rank starting at 1.
+ *
+ * RRF uses only ranks, so the two retrievers' scores never need to be calibrated against each
+ * other. The cost is that the fused score is not interpretable as a relevance magnitude: use
+ * it to order, never as a confidence threshold.
+ */
+import type { Chunk, Scored } from '../contracts.js';
+
+export function reciprocalRankFusion(
+  rankedLists: Readonly<Record<string, readonly Scored[]>>,
+  k: number,
+  limit: number,
+): Scored[] {
+  const totals = new Map<string, number>();
+  const chunks = new Map<string, Chunk>();
+
+  for (const results of Object.values(rankedLists)) {
+    results.forEach((item, at) => {
+      const rank = at + 1;
+      chunks.set(item.chunk.id, item.chunk);
+      totals.set(item.chunk.id, (totals.get(item.chunk.id) ?? 0) + 1 / (k + rank));
+    });
+  }
+
+  return [...totals.entries()]
+    .map(([id, score]): Scored => ({ chunk: chunks.get(id)!, score }))
+    .sort((a, b) => b.score - a.score || a.chunk.id.localeCompare(b.chunk.id))
+    .slice(0, limit);
+}

package/templates/architectures/rag-hybrid/files/src/rag/retrieve/index.ts

@@ -0,0 +1,38 @@
+/**
+ * Stage 5 — the retrieval seam.
+ *
+ * `retrieve` is the only supported way to turn a query into passages. Everything the model is
+ * allowed to read comes through this function, which is what makes provenance and eval
+ * possible: one place to instrument, one place to change the retriever.
+ */
+import type { Settings } from '../config.js';
+import type { RetrievalResult, Scored } from '../contracts.js';
+import type { Embedder } from '../embed/index.js';
+import type { RetrievalIndex } from '../store/index.js';
+import * as dense from './dense.js';
+import { reciprocalRankFusion } from './fusion.js';
+import * as keyword from './keyword.js';
+
+/** Run both retrievers over the same chunks and fuse them by rank. */
+export async function retrieve(
+  query: string,
+  index: RetrievalIndex,
+  embedder: Embedder,
+  settings: Settings,
+): Promise<RetrievalResult> {
+  const lists: Record<string, readonly Scored[]> = {
+    dense: await dense.search(query, index, embedder, settings),
+    keyword: keyword.search(query, index, settings),
+  };
+
+  return {
+    query,
+    passages: reciprocalRankFusion(lists, settings.rrfK, settings.candidateK),
+    // Which retriever found what — the fused score alone cannot tell you.
+    contributions: Object.fromEntries(
+      Object.entries(lists).map(([name, results]) => [name, results.map((item) => item.chunk.id)]),
+    ),
+  };
+}
+
+export { dense, keyword, reciprocalRankFusion };

package/templates/architectures/rag-hybrid/files/src/rag/retrieve/keyword.ts

@@ -0,0 +1,42 @@
+/**
+ * Keyword retriever — Okapi BM25.
+ *
+ * This is the half that supplies precision on exact-match tokens: product codes, jargon,
+ * dates, people's names. A dense-only pipeline loses these, and no amount of reranking
+ * recovers a document that was never in the candidate set.
+ */
+import type { Settings } from '../config.js';
+import type { Scored } from '../contracts.js';
+import { tokenize, type RetrievalIndex } from '../store/index.js';
+
+/**
+ * Robertson/Sparck-Jones idf with the +0.5 smoothing, floored so common terms cannot
+ * contribute a negative score.
+ */
+function idf(index: RetrievalIndex, term: string): number {
+  const total = index.chunks.length;
+  const seen = index.documentFrequency(term);
+  return Math.max(0, Math.log(1 + (total - seen + 0.5) / (seen + 0.5)));
+}
+
+export function search(query: string, index: RetrievalIndex, settings: Settings): Scored[] {
+  const terms = tokenize(query);
+  const average = index.averageLength || 1;
+  const { bm25K1: k1, bm25B: b } = settings;
+
+  const scored: Scored[] = [];
+  index.chunks.forEach((chunk, position) => {
+    const length = index.lengthOf(position);
+    let total = 0;
+    for (const term of terms) {
+      const frequency = index.termFrequency(term, position);
+      if (!frequency) continue;
+      const saturation = frequency * (k1 + 1);
+      const normalizer = frequency + k1 * (1 - b + (b * length) / average);
+      total += (idf(index, term) * saturation) / normalizer;
+    }
+    if (total > 0) scored.push({ chunk, score: total });
+  });
+
+  return scored.sort((a, b2) => b2.score - a.score).slice(0, settings.candidateK);
+}

package/templates/architectures/rag-hybrid/files/src/rag/store/index.ts

@@ -0,0 +1,108 @@
+/**
+ * Stage 4 — the two indexes.
+ *
+ * Hybrid retrieval means two indexes over the same chunks, built together so they can never
+ * drift apart. Swap either implementation for a real engine (pgvector, OpenSearch, Azure AI
+ * Search) by satisfying RetrievalIndex; the retrievers above do not change.
+ */
+import type { Chunk, EmbeddedChunk } from '../contracts.js';
+
+const TOKEN = /[a-z0-9]+/g;
+
+/** One tokenizer for indexing and querying. Two would silently halve keyword recall. */
+export function tokenize(text: string): string[] {
+  return text.toLowerCase().match(TOKEN) ?? [];
+}
+
+export interface RetrievalIndex {
+  readonly chunks: readonly Chunk[];
+  vectorOf(position: number): readonly number[];
+  termFrequency(term: string, position: number): number;
+  documentFrequency(term: string): number;
+  lengthOf(position: number): number;
+  readonly averageLength: number;
+}
+
+/** Dense vectors and BM25 postings over one chunk set. */
+export class InMemoryIndex implements RetrievalIndex {
+  // Fields are declared and assigned rather than declared as constructor parameters:
+  // parameter properties emit runtime code, which `erasableSyntaxOnly` forbids.
+  readonly chunks: readonly Chunk[];
+  readonly averageLength: number;
+  private readonly vectors: readonly (readonly number[])[];
+  private readonly frequencies: readonly Map<string, number>[];
+  private readonly lengths: readonly number[];
+  private readonly postings: ReadonlyMap<string, number>;
+
+  constructor(
+    chunks: readonly Chunk[],
+    vectors: readonly (readonly number[])[],
+    frequencies: readonly Map<string, number>[],
+    lengths: readonly number[],
+    postings: ReadonlyMap<string, number>,
+  ) {
+    this.chunks = chunks;
+    this.vectors = vectors;
+    this.frequencies = frequencies;
+    this.lengths = lengths;
+    this.postings = postings;
+
+    const total = lengths.reduce((sum, value) => sum + value, 0);
+    this.averageLength = lengths.length ? total / lengths.length : 0;
+  }
+
+  vectorOf(position: number): readonly number[] {
+    return this.vectors[position] ?? [];
+  }
+
+  termFrequency(term: string, position: number): number {
+    return this.frequencies[position]?.get(term) ?? 0;
+  }
+
+  documentFrequency(term: string): number {
+    return this.postings.get(term) ?? 0;
+  }
+
+  lengthOf(position: number): number {
+    return this.lengths[position] ?? 0;
+  }
+}
+
+export function buildIndex(embedded: readonly EmbeddedChunk[]): InMemoryIndex {
+  const frequencies = embedded.map((item) => {
+    const counter = new Map<string, number>();
+    for (const term of tokenize(item.chunk.text)) counter.set(term, (counter.get(term) ?? 0) + 1);
+    return counter;
+  });
+
+  const postings = new Map<string, number>();
+  for (const counter of frequencies) {
+    for (const term of counter.keys()) postings.set(term, (postings.get(term) ?? 0) + 1);
+  }
+
+  const lengths = frequencies.map((counter) => [...counter.values()].reduce((a, b) => a + b, 0));
+
+  return new InMemoryIndex(
+    embedded.map((item) => item.chunk),
+    embedded.map((item) => item.vector),
+    frequencies,
+    lengths,
+    postings,
+  );
+}
+
+/** Vectors are stored unit-normalized, so this is a dot product in the common case. */
+export function cosine(left: readonly number[], right: readonly number[]): number {
+  let dot = 0;
+  let leftNorm = 0;
+  let rightNorm = 0;
+  for (let at = 0; at < Math.min(left.length, right.length); at += 1) {
+    const a = left[at] ?? 0;
+    const b = right[at] ?? 0;
+    dot += a * b;
+    leftNorm += a * a;
+    rightNorm += b * b;
+  }
+  const scale = Math.sqrt(leftNorm) * Math.sqrt(rightNorm);
+  return scale ? dot / scale : 0;
+}