@opengroundplan/core 0.1.1

package/templates/architectures/rag-hybrid/files/src/__solutionName.snake__/rag/rerank/__init__.py

@@ -0,0 +1,85 @@
+"""Stage 6 — reranking, the last gate before the model.
+
+Fusion optimizes recall over a wide candidate set; the reranker scores each candidate against
+the query directly and cuts it to the handful that fit in the prompt. This module is the only
+place that constructs RerankedPassages, so nothing else can hand raw candidates to generation.
+"""
+
+from __future__ import annotations
+
+import json
+import urllib.request
+from typing import Protocol, Sequence
+
+from ..config import Settings
+from ..contracts import RerankedPassages, RetrievalResult, Scored
+from ..store import tokenize
+
+
+class Reranker(Protocol):
+    def score(self, query: str, passages: Sequence[Scored]) -> list[float]: ...
+
+
+class LexicalReranker:
+    """Zero-dependency stand-in: query-term coverage of the passage.
+
+    It is a placeholder for a cross-encoder, not a substitute for one. Point RERANK_PROVIDER
+    at a real model before this pipeline is load-bearing.
+    """
+
+    def score(self, query: str, passages: Sequence[Scored]) -> list[float]:
+        terms = set(tokenize(query))
+        if not terms:
+            return [item.score for item in passages]
+        return [len(terms & set(tokenize(item.chunk.text))) / len(terms) for item in passages]
+
+
+class HttpReranker:
+    """Cohere/Jina-shaped rerank endpoint. Endpoint, model and key all come from settings."""
+
+    def __init__(self, settings: Settings) -> None:
+        if not settings.rerank_endpoint or not settings.rerank_model:
+            raise ValueError("RERANK_ENDPOINT and RERANK_MODEL must be set for the http reranker")
+        self._settings = settings
+
+    def score(self, query: str, passages: Sequence[Scored]) -> list[float]:
+        settings = self._settings
+        payload = json.dumps(
+            {
+                "model": settings.rerank_model,
+                "query": query,
+                "documents": [item.chunk.text for item in passages],
+            }
+        )
+        request = urllib.request.Request(
+            settings.rerank_endpoint,
+            data=payload.encode("utf-8"),
+            headers={
+                "content-type": "application/json",
+                "authorization": f"Bearer {settings.rerank_api_key}",
+            },
+        )
+        with urllib.request.urlopen(request, timeout=settings.request_timeout_s) as response:
+            body = json.loads(response.read().decode("utf-8"))
+
+        scores = [0.0] * len(passages)
+        for entry in body["results"]:
+            scores[entry["index"]] = float(entry["relevance_score"])
+        return scores
+
+
+def build_reranker(settings: Settings) -> Reranker:
+    if settings.rerank_provider == "http":
+        return HttpReranker(settings)
+    return LexicalReranker()
+
+
+def rerank(
+    result: RetrievalResult, reranker: Reranker, settings: Settings
+) -> RerankedPassages:
+    scores = reranker.score(result.query, result.passages)
+    rescored = [
+        Scored(chunk=item.chunk, score=score) for item, score in zip(result.passages, scores)
+    ]
+    rescored.sort(key=lambda item: item.score, reverse=True)
+    return RerankedPassages(tuple(rescored[: settings.rerank_top_n]))

package/templates/architectures/rag-hybrid/files/src/__solutionName.snake__/rag/retrieve/__init__.py

@@ -0,0 +1,37 @@
+"""Stage 5 — the retrieval seam.
+
+`retrieve` is the only supported way to turn a query into passages. Everything the model is
+allowed to read comes through this function, which is what makes provenance and eval possible:
+one place to instrument, one place to change the retriever.
+"""
+
+from __future__ import annotations
+
+from ..config import Settings
+from ..contracts import RetrievalResult
+from ..embed import Embedder
+from ..store import RetrievalIndex
+from . import dense, fusion, keyword
+
+
+def retrieve(
+    query: str, index: RetrievalIndex, embedder: Embedder, settings: Settings
+) -> RetrievalResult:
+    """Run both retrievers over the same chunks and fuse them by rank."""
+    lists = {
+        "dense": dense.search(query, index, embedder, settings),
+        "keyword": keyword.search(query, index, settings),
+    }
+    fused = fusion.reciprocal_rank_fusion(lists, settings.rrf_k, settings.candidate_k)
+
+    return RetrievalResult(
+        query=query,
+        passages=tuple(fused),
+        # Which retriever found what — the fused score alone cannot tell you.
+        contributions={
+            name: tuple(item.chunk.id for item in results) for name, results in lists.items()
+        },
+    )
+
+
+__all__ = ["retrieve"]

package/templates/architectures/rag-hybrid/files/src/__solutionName.snake__/rag/retrieve/dense.py

@@ -0,0 +1,25 @@
+"""Dense retriever — conceptual similarity, no keyword overlap required.
+
+This is the half that finds "how do I cancel" for a query about "refund policy". It is also
+the half that misses exact tokens like part numbers, which is why it is never used alone.
+"""
+
+from __future__ import annotations
+
+from ..config import Settings
+from ..contracts import Scored
+from ..embed import Embedder
+from ..store import RetrievalIndex, cosine
+
+
+def search(
+    query: str, index: RetrievalIndex, embedder: Embedder, settings: Settings
+) -> list[Scored]:
+    """Query and corpus must share an embedder or the vector space does not line up."""
+    query_vector = embedder.embed([query])[0]
+    scored = [
+        Scored(chunk=chunk, score=cosine(query_vector, index.vector_of(position)))
+        for position, chunk in enumerate(index.chunks)
+    ]
+    scored.sort(key=lambda item: item.score, reverse=True)
+    return scored[: settings.candidate_k]

package/templates/architectures/rag-hybrid/files/src/__solutionName.snake__/rag/retrieve/fusion.py

@@ -0,0 +1,30 @@
+"""Reciprocal rank fusion — combines ranked lists without tuning.
+
+score(d) = sum over lists of 1 / (k + rank(d)), rank starting at 1.
+
+RRF uses only ranks, so the two retrievers' scores never need to be calibrated against each
+other. The cost is that the fused score is not interpretable as a relevance magnitude: use it
+to order, never as a confidence threshold.
+"""
+
+from __future__ import annotations
+
+from typing import Mapping, Sequence
+
+from ..contracts import Chunk, Scored
+
+
+def reciprocal_rank_fusion(
+    ranked_lists: Mapping[str, Sequence[Scored]], k: int, limit: int
+) -> list[Scored]:
+    totals: dict[str, float] = {}
+    chunks: dict[str, Chunk] = {}
+
+    for results in ranked_lists.values():
+        for rank, item in enumerate(results, start=1):
+            chunks[item.chunk.id] = item.chunk
+            totals[item.chunk.id] = totals.get(item.chunk.id, 0.0) + 1.0 / (k + rank)
+
+    fused = [Scored(chunk=chunks[chunk_id], score=score) for chunk_id, score in totals.items()]
+    fused.sort(key=lambda item: (item.score, item.chunk.id), reverse=True)
+    return fused[:limit]

package/templates/architectures/rag-hybrid/files/src/__solutionName.snake__/rag/retrieve/keyword.py

@@ -0,0 +1,45 @@
+"""Keyword retriever — Okapi BM25.
+
+This is the half that supplies precision on exact-match tokens: product codes, jargon, dates,
+people's names. A dense-only pipeline loses these, and no amount of reranking recovers a
+document that was never in the candidate set.
+"""
+
+from __future__ import annotations
+
+import math
+
+from ..config import Settings
+from ..contracts import Scored
+from ..store import RetrievalIndex, tokenize
+
+
+def _idf(index: RetrievalIndex, term: str) -> float:
+    """Robertson/Sparck-Jones idf with the +0.5 smoothing, floored so common terms cannot
+    contribute a negative score."""
+    total = len(index.chunks)
+    seen = index.document_frequency(term)
+    return max(0.0, math.log(1.0 + (total - seen + 0.5) / (seen + 0.5)))
+
+
+def search(query: str, index: RetrievalIndex, settings: Settings) -> list[Scored]:
+    terms = tokenize(query)
+    average = index.average_length or 1.0
+    k1, b = settings.bm25_k1, settings.bm25_b
+
+    scored: list[Scored] = []
+    for position, chunk in enumerate(index.chunks):
+        length = index.length_of(position)
+        total = 0.0
+        for term in terms:
+            frequency = index.term_frequency(term, position)
+            if not frequency:
+                continue
+            saturation = frequency * (k1 + 1.0)
+            normalizer = frequency + k1 * (1.0 - b + b * length / average)
+            total += _idf(index, term) * saturation / normalizer
+        if total > 0.0:
+            scored.append(Scored(chunk=chunk, score=total))
+
+    scored.sort(key=lambda item: item.score, reverse=True)
+    return scored[: settings.candidate_k]

package/templates/architectures/rag-hybrid/files/src/__solutionName.snake__/rag/store/__init__.py

@@ -0,0 +1,87 @@
+"""Stage 4 — the two indexes.
+
+Hybrid retrieval means two indexes over the same chunks, built together so they can never
+drift apart. Swap either implementation for a real engine (pgvector, OpenSearch, Azure AI
+Search) by satisfying RetrievalIndex; the retrievers above do not change.
+"""
+
+from __future__ import annotations
+
+import math
+import re
+from collections import Counter
+from dataclasses import dataclass
+from typing import Iterable, Protocol, Sequence
+
+from ..contracts import Chunk, EmbeddedChunk
+
+_TOKEN = re.compile(r"[a-z0-9]+")
+
+
+def tokenize(text: str) -> list[str]:
+    """One tokenizer for indexing and querying. Two would silently halve keyword recall."""
+    return _TOKEN.findall(text.lower())
+
+
+class RetrievalIndex(Protocol):
+    chunks: tuple[Chunk, ...]
+
+    def vector_of(self, position: int) -> Sequence[float]: ...
+    def term_frequency(self, term: str, position: int) -> int: ...
+    def document_frequency(self, term: str) -> int: ...
+    def length_of(self, position: int) -> int: ...
+
+    @property
+    def average_length(self) -> float: ...
+
+
+@dataclass
+class InMemoryIndex:
+    """Dense vectors and BM25 postings over one chunk set."""
+
+    chunks: tuple[Chunk, ...]
+    vectors: tuple[tuple[float, ...], ...]
+    _frequencies: tuple[Counter[str], ...]
+    _lengths: tuple[int, ...]
+    _postings: dict[str, int]
+
+    def vector_of(self, position: int) -> Sequence[float]:
+        return self.vectors[position]
+
+    def term_frequency(self, term: str, position: int) -> int:
+        return self._frequencies[position][term]
+
+    def document_frequency(self, term: str) -> int:
+        return self._postings.get(term, 0)
+
+    def length_of(self, position: int) -> int:
+        return self._lengths[position]
+
+    @property
+    def average_length(self) -> float:
+        return sum(self._lengths) / len(self._lengths) if self._lengths else 0.0
+
+
+def build_index(embedded: Iterable[EmbeddedChunk]) -> InMemoryIndex:
+    items = list(embedded)
+    frequencies = tuple(Counter(tokenize(item.chunk.text)) for item in items)
+    postings: dict[str, int] = {}
+    for counter in frequencies:
+        for term in counter:
+            postings[term] = postings.get(term, 0) + 1
+
+    return InMemoryIndex(
+        chunks=tuple(item.chunk for item in items),
+        vectors=tuple(item.vector for item in items),
+        _frequencies=frequencies,
+        _lengths=tuple(sum(counter.values()) for counter in frequencies),
+        _postings=postings,
+    )
+
+
+def cosine(left: Sequence[float], right: Sequence[float]) -> float:
+    """Vectors are stored unit-normalized, so this is a dot product in the common case."""
+    dot = sum(a * b for a, b in zip(left, right))
+    left_norm = math.sqrt(sum(a * a for a in left))
+    right_norm = math.sqrt(sum(b * b for b in right))
+    return dot / (left_norm * right_norm) if left_norm and right_norm else 0.0

package/templates/architectures/rag-hybrid/files/src/rag/chunk/index.ts

@@ -0,0 +1,56 @@
+/**
+ * Stage 2 — Documents become Chunks.
+ *
+ * Chunking is the highest-leverage retrieval knob and the easiest to change by accident,
+ * which is why it is its own stage with its own eval run rather than a helper inside ingest.
+ */
+import type { Settings } from '../config.js';
+import type { Chunk, Document } from '../contracts.js';
+
+const PARAGRAPH = '\n\n';
+
+/** Paragraph-aligned windows, falling back to a hard cut for a single oversized block. */
+function windows(text: string, size: number, overlap: number): string[] {
+  const step = Math.max(1, size - overlap);
+  const blocks = text
+    .split(PARAGRAPH)
+    .map((block) => block.trim())
+    .filter(Boolean);
+
+  const out: string[] = [];
+  let buffer = '';
+
+  for (const block of blocks) {
+    if (block.length > size) {
+      if (buffer) {
+        out.push(buffer);
+        buffer = '';
+      }
+      for (let at = 0; at < block.length; at += step) out.push(block.slice(at, at + size));
+      continue;
+    }
+    const candidate = buffer ? `${buffer}${PARAGRAPH}${block}` : block;
+    if (candidate.length > size) {
+      out.push(buffer);
+      buffer = block;
+    } else {
+      buffer = candidate;
+    }
+  }
+  if (buffer) out.push(buffer);
+  return out;
+}
+
+export function chunkDocuments(documents: readonly Document[], settings: Settings): Chunk[] {
+  return documents.flatMap((document) =>
+    windows(document.text, settings.chunkChars, settings.chunkOverlapChars).map(
+      (text, ordinal): Chunk => ({
+        id: `${document.id}#${ordinal}`,
+        documentId: document.id,
+        ordinal,
+        text,
+        source: document.source,
+      }),
+    ),
+  );
+}

package/templates/architectures/rag-hybrid/files/src/rag/config.ts

@@ -0,0 +1,96 @@
+/**
+ * Every tunable in the pipeline, with one typed default each.
+ *
+ * No other module reads process.env. Changing how retrieval behaves is an edit here or an
+ * environment variable, never a literal buried in a stage.
+ */
+const PREFIX = '{{ envPrefix }}_';
+
+function text(name: string, fallback: string): string {
+  return process.env[PREFIX + name] ?? fallback;
+}
+
+function integer(name: string, fallback: number): number {
+  const parsed = Number.parseInt(text(name, String(fallback)), 10);
+  return Number.isFinite(parsed) ? parsed : fallback;
+}
+
+function decimal(name: string, fallback: number): number {
+  const parsed = Number.parseFloat(text(name, String(fallback)));
+  return Number.isFinite(parsed) ? parsed : fallback;
+}
+
+export interface Settings {
+  readonly corpusDir: string;
+  readonly corpusExtensions: readonly string[];
+  readonly chunkChars: number;
+  readonly chunkOverlapChars: number;
+
+  readonly embeddingProvider: string;
+  readonly embeddingModel: string;
+  readonly embeddingEndpoint: string;
+  readonly embeddingApiKey: string;
+  readonly embeddingDimensions: number;
+  readonly embedBatchSize: number;
+
+  readonly candidateK: number;
+  readonly rrfK: number;
+  readonly bm25K1: number;
+  readonly bm25B: number;
+
+  readonly rerankProvider: string;
+  readonly rerankModel: string;
+  readonly rerankEndpoint: string;
+  readonly rerankApiKey: string;
+  readonly rerankTopN: number;
+
+  readonly generationProvider: string;
+  readonly generationModel: string;
+  readonly generationEndpoint: string;
+  readonly generationApiKey: string;
+
+  readonly requestTimeoutMs: number;
+
+  readonly goldenPath: string;
+  readonly minNdcg: number;
+}
+
+/** Read the environment once. Pass the result down; do not call this inside a loop. */
+export function loadSettings(): Settings {
+  return {
+    corpusDir: text('CORPUS_DIR', 'evals/corpus'),
+    corpusExtensions: text('CORPUS_EXTENSIONS', '.md,.txt').split(','),
+    chunkChars: integer('CHUNK_CHARS', {{ chunkChars }}),
+    chunkOverlapChars: integer('CHUNK_OVERLAP_CHARS', {{ chunkOverlapChars }}),
+
+    // 'hash' is a deterministic offline embedder so the eval harness runs with no keys.
+    embeddingProvider: text('EMBEDDING_PROVIDER', 'hash'),
+    embeddingModel: text('EMBEDDING_MODEL', ''),
+    embeddingEndpoint: text('EMBEDDING_ENDPOINT', ''),
+    embeddingApiKey: text('EMBEDDING_API_KEY', ''),
+    embeddingDimensions: integer('EMBEDDING_DIMENSIONS', 256),
+    embedBatchSize: integer('EMBED_BATCH_SIZE', 64),
+
+    candidateK: integer('CANDIDATE_K', {{ candidateK }}),
+    rrfK: integer('RRF_K', {{ rrfK }}),
+    bm25K1: decimal('BM25_K1', 1.2),
+    bm25B: decimal('BM25_B', 0.75),
+
+    // 'lexical' is a zero-dependency stand-in; point this at a cross-encoder in production.
+    rerankProvider: text('RERANK_PROVIDER', 'lexical'),
+    rerankModel: text('RERANK_MODEL', ''),
+    rerankEndpoint: text('RERANK_ENDPOINT', ''),
+    rerankApiKey: text('RERANK_API_KEY', ''),
+    rerankTopN: integer('RERANK_TOP_N', {{ rerankTopN }}),
+
+    generationProvider: text('GENERATION_PROVIDER', 'echo'),
+    generationModel: text('GENERATION_MODEL', ''),
+    generationEndpoint: text('GENERATION_ENDPOINT', ''),
+    generationApiKey: text('GENERATION_API_KEY', ''),
+
+    requestTimeoutMs: integer('REQUEST_TIMEOUT_MS', 30_000),
+
+    goldenPath: text('GOLDEN_PATH', 'evals/golden/queries.jsonl'),
+    minNdcg: decimal('MIN_NDCG', 0),
+  };
+}

package/templates/architectures/rag-hybrid/files/src/rag/contracts.ts

@@ -0,0 +1,50 @@
+/**
+ * The types that cross stage boundaries. Every stage signature is written in these, which is
+ * what makes the stages replaceable: swap the vector store or the reranker and nothing
+ * downstream changes shape.
+ */
+
+/** One source file or record, normalized. Output of ingest. */
+export interface Document {
+  readonly id: string;
+  readonly source: string;
+  readonly text: string;
+  readonly metadata: Readonly<Record<string, string>>;
+}
+
+/** A retrievable unit. `documentId` survives rechunking, so goldens reference it. */
+export interface Chunk {
+  readonly id: string;
+  readonly documentId: string;
+  readonly ordinal: number;
+  readonly text: string;
+  readonly source: string;
+}
+
+export interface EmbeddedChunk {
+  readonly chunk: Chunk;
+  readonly vector: readonly number[];
+}
+
+export interface Scored {
+  readonly chunk: Chunk;
+  readonly score: number;
+}
+
+/**
+ * What the retrieval seam returns. `contributions` records which retriever found what,
+ * because a fused score is not interpretable as a relevance magnitude on its own.
+ */
+export interface RetrievalResult {
+  readonly query: string;
+  readonly passages: readonly Scored[];
+  readonly contributions: Readonly<Record<string, readonly string[]>>;
+}
+
+declare const reranked: unique symbol;
+
+/**
+ * Only rerank/ mints this. Generation accepts nothing else, so a caller cannot hand the model
+ * raw fusion output — `tsc` rejects it before the seam check has to.
+ */
+export type RerankedPassages = readonly Scored[] & { readonly [reranked]: true };

package/templates/architectures/rag-hybrid/files/src/rag/embed/index.ts

@@ -0,0 +1,98 @@
+/**
+ * Stage 3 — text becomes vectors.
+ *
+ * The provider is resolved from settings by role, so no call site names a model. Queries and
+ * documents must go through the same embedder or the vector space does not line up.
+ */
+import { createHash } from 'node:crypto';
+import type { Settings } from '../config.js';
+import type { Chunk, EmbeddedChunk } from '../contracts.js';
+
+export interface Embedder {
+  embed(texts: readonly string[]): Promise<number[][]>;
+}
+
+function unit(vector: number[]): number[] {
+  const norm = Math.sqrt(vector.reduce((total, value) => total + value * value, 0));
+  return norm ? vector.map((value) => value / norm) : vector;
+}
+
+/**
+ * Deterministic, offline, no keys — the default so the eval harness runs anywhere. It is a
+ * bag-of-words hash, not a semantic model: it exercises the wiring, and it is the baseline
+ * your real embedder must beat on the golden set.
+ */
+export class HashingEmbedder implements Embedder {
+  private readonly dimensions: number;
+
+  constructor(dimensions: number) {
+    this.dimensions = dimensions;
+  }
+
+  async embed(texts: readonly string[]): Promise<number[][]> {
+    return texts.map((text) => {
+      const bucket = new Array<number>(this.dimensions).fill(0);
+      for (const token of text.toLowerCase().split(/\s+/).filter(Boolean)) {
+        const digest = createHash('sha256').update(token).digest();
+        const at = digest.readUInt32BE(0) % this.dimensions;
+        bucket[at] = (bucket[at] ?? 0) + 1;
+      }
+      return unit(bucket);
+    });
+  }
+}
+
+/** OpenAI-shaped embeddings endpoint. Endpoint, model and key all come from settings. */
+export class HttpEmbedder implements Embedder {
+  private readonly settings: Settings;
+
+  constructor(settings: Settings) {
+    if (!settings.embeddingEndpoint || !settings.embeddingModel) {
+      throw new Error('EMBEDDING_ENDPOINT and EMBEDDING_MODEL must be set for the http embedder');
+    }
+    this.settings = settings;
+  }
+
+  async embed(texts: readonly string[]): Promise<number[][]> {
+    const { settings } = this;
+    const response = await fetch(settings.embeddingEndpoint, {
+      method: 'POST',
+      headers: {
+        'content-type': 'application/json',
+        authorization: `Bearer ${settings.embeddingApiKey}`,
+      },
+      body: JSON.stringify({ model: settings.embeddingModel, input: texts }),
+      signal: AbortSignal.timeout(settings.requestTimeoutMs),
+    });
+    if (!response.ok) throw new Error(`Embedding request failed: ${response.status}`);
+
+    const body = (await response.json()) as { data: { embedding: number[] }[] };
+    return body.data.map((item) => unit(item.embedding));
+  }
+}
+
+export function buildEmbedder(settings: Settings): Embedder {
+  if (settings.embeddingProvider === 'http') return new HttpEmbedder(settings);
+  return new HashingEmbedder(settings.embeddingDimensions);
+}
+
+export async function embedChunks(
+  chunks: readonly Chunk[],
+  embedder: Embedder,
+  batchSize: number,
+): Promise<EmbeddedChunk[]> {
+  const out: EmbeddedChunk[] = [];
+  for (let start = 0; start < chunks.length; start += batchSize) {
+    const batch = chunks.slice(start, start + batchSize);
+    const vectors = await embedder.embed(batch.map((chunk) => chunk.text));
+    for (const [at, chunk] of batch.entries()) {
+      const vector = vectors[at];
+      // A provider that drops one input silently would misalign every later chunk.
+      if (!vector) {
+        throw new Error(`Embedder returned ${vectors.length} vectors for ${batch.length} chunks`);
+      }
+      out.push({ chunk, vector });
+    }
+  }
+  return out;
+}

package/templates/architectures/rag-hybrid/files/src/rag/evaluation/cli.ts

@@ -0,0 +1,47 @@
+/**
+ * Run with `npm run eval:retrieval`.
+ *
+ * Exits non-zero when the best variant falls under MIN_NDCG, so retrieval quality can gate a
+ * pull request the same way a failing test does. MIN_NDCG defaults to 0 — raise it once you
+ * have a real golden set and a baseline you are willing to defend.
+ */
+import process from 'node:process';
+import { buildIndex } from '../indexing.js';
+import { evaluateGoldenSet } from './harness.js';
+
+async function main(): Promise<number> {
+  const { index, settings } = await buildIndex();
+  const scores = await evaluateGoldenSet(index, settings);
+  if (scores.length === 0) {
+    console.error('No variants ran — is the golden set empty?');
+    return 1;
+  }
+
+  // Without a semantic embedder the dense row is lexical too, so the table is a smoke test
+  // rather than a result. Say so, or someone will quote these numbers in a design review.
+  if (settings.embeddingProvider === 'hash') {
+    console.error(
+      "note: EMBEDDING_PROVIDER=hash is a bag-of-words stand-in, so 'dense-only' is not a " +
+        'semantic baseline. These numbers check wiring, not retrieval quality.\n',
+    );
+  }
+
+  const width = Math.max(...scores.map((score) => score.variant.length));
+  console.log(`${'variant'.padEnd(width)}  recall@${settings.rerankTopN}   ndcg    mrr`);
+  for (const score of scores) {
+    console.log(
+      `${score.variant.padEnd(width)}  ${score.recall.toFixed(3).padStart(9)} ${score.ndcg
+        .toFixed(3)
+        .padStart(6)} ${score.mrr.toFixed(3).padStart(6)}`,
+    );
+  }
+
+  const best = Math.max(...scores.map((score) => score.ndcg));
+  if (best < settings.minNdcg) {
+    console.error(`\nBest nDCG ${best.toFixed(3)} is under MIN_NDCG ${settings.minNdcg.toFixed(3)}`);
+    return 1;
+  }
+  return 0;
+}
+
+process.exitCode = await main();