md-semantic-search 1.0.3 → 2.0.0

package/dist/rerank.d.ts

@@ -1,10 +1,55 @@
-export declare const RERANK_MODEL = "Xenova/bge-reranker-base";
+export interface RerankerTokenizer {
+    (queries: string[], options: {
+        text_pair: string[];
+        padding: boolean;
+        truncation: boolean;
+    }): unknown;
+}
+export interface RerankerModelOutput {
+    logits: {
+        tolist(): unknown;
+    };
+}
+export interface RerankerModel {
+    (inputs: unknown): Promise<RerankerModelOutput>;
+}
+export interface LoadedReranker {
+    tokenizer: RerankerTokenizer;
+    model: RerankerModel;
+}
+/**
+ * Reranker registry (issue #203) — the cross-encoder counterpart of the
+ * embedder MODELS registry. Each entry is an explicit adapter: repo id,
+ * pinned revision, runtime dtype, and the tokenizer context budget the
+ * pair-truncation relies on. The loaded model must be a single-logit
+ * sequence classifier (logits[0] = relevance score) — the extraction in
+ * rerankScores is shared by every entry.
+ */
+export interface RerankerAdapter {
+    id: string;
+    revision?: string;
+    dtype: string;
+    maxTokens: number;
+    family?: string;
+    note?: string;
+}
+export declare const RERANKERS: Record<string, RerankerAdapter>;
+export declare const DEFAULT_RERANKER = "gte-reranker-modernbert";
+/** Legacy export — the default adapter's repo id (kept for back-compat). */
+export declare const RERANK_MODEL: string;
+/**
+ * Resolve a reranker alias / raw repo id (optionally `id@revision`) to an
+ * adapter. Unknown ids are rejected — unlike embedders there is no
+ * "unknown adapter" escape hatch, because the single-logit contract cannot
+ * be guessed safely.
+ */
+export declare function resolveReranker(name?: string | null): {
+    alias: string;
+    adapter: RerankerAdapter;
+};
 export declare function getReranker(cacheDir: string, offline?: boolean, retryOpts?: {
     maxRetries?: number;
     delays?: number[];
     log?: (msg: string) => void;
-}): Promise<{
-    tokenizer: any;
-    model: any;
-}>;
-export declare function rerankScores(query: string, passages: string[], cacheDir: string, offline?: boolean): Promise<number[]>;
+}, rerankerName?: string | null): Promise<LoadedReranker>;
+export declare function rerankScores(query: string, passages: string[], cacheDir: string, offline?: boolean, rerankerName?: string | null): Promise<number[]>;

package/dist/rerank.js

@@ -3,25 +3,75 @@
  */
 import { resolveSessionOptions, retryWithBackoff, emitDownloadEvent } from './core.js';
 const _rerankers = new Map();
-export const RERANK_MODEL = 'Xenova/bge-reranker-base';
-export async function getReranker(cacheDir, offline = false, retryOpts = {}) {
+export const RERANKERS = {
+    'gte-reranker-modernbert': {
+        id: 'Alibaba-NLP/gte-reranker-modernbert-base',
+        revision: 'f7481e6055501a30fb19d090657df9ec1f79ab2c',
+        dtype: 'q8',
+        maxTokens: 8192,
+        family: 'gte',
+        note: 'Default since issue #203 — ~144 MB q8, Apache-2.0, ModernBERT 8k context. Golden set: nDCG@10 0.9318 vs bge-reranker-base 0.8839.',
+    },
+    'bge-reranker-base': {
+        id: 'Xenova/bge-reranker-base',
+        dtype: 'q8',
+        maxTokens: 512,
+        family: 'bge',
+        note: '~278 MB q8. The pre-#203 default; kept for reproducibility (pin via --rerank-model bge-reranker-base).',
+    },
+};
+export const DEFAULT_RERANKER = 'gte-reranker-modernbert';
+/** Legacy export — the default adapter's repo id (kept for back-compat). */
+export const RERANK_MODEL = RERANKERS[DEFAULT_RERANKER].id;
+/**
+ * Resolve a reranker alias / raw repo id (optionally `id@revision`) to an
+ * adapter. Unknown ids are rejected — unlike embedders there is no
+ * "unknown adapter" escape hatch, because the single-logit contract cannot
+ * be guessed safely.
+ */
+export function resolveReranker(name) {
+    if (!name || name === DEFAULT_RERANKER) {
+        return { alias: DEFAULT_RERANKER, adapter: RERANKERS[DEFAULT_RERANKER] };
+    }
+    if (RERANKERS[name])
+        return { alias: name, adapter: RERANKERS[name] };
+    let id = name;
+    let revision;
+    const at = name.indexOf('@');
+    if (at > 0) {
+        id = name.slice(0, at);
+        revision = name.slice(at + 1) || undefined;
+    }
+    for (const [alias, adapter] of Object.entries(RERANKERS)) {
+        if (adapter.id === id) {
+            return { alias, adapter: revision ? { ...adapter, revision } : adapter };
+        }
+    }
+    throw new Error(`unknown reranker "${name}" — available: ${Object.keys(RERANKERS).join(', ')} (or register it in RERANKERS, issue #203)`);
+}
+export async function getReranker(cacheDir, offline = false, retryOpts = {}, rerankerName) {
+    const { adapter } = resolveReranker(rerankerName);
     const sessionOptions = resolveSessionOptions();
     const sessionKey = sessionOptions ? JSON.stringify(sessionOptions) : 'default';
-    const key = `${RERANK_MODEL}|${offline ? 'off' : 'on'}|${sessionKey}`;
+    const key = `${adapter.id}@${adapter.revision || 'main'}|${adapter.dtype}|${offline ? 'off' : 'on'}|${sessionKey}`;
     if (_rerankers.has(key))
         return _rerankers.get(key);
     const loadPipeline = async () => {
-        const { AutoModelForSequenceClassification, AutoTokenizer, env } = await import('@huggingface/transformers');
+        const { AutoModelForSequenceClassification, AutoTokenizer, env } = (await import('@huggingface/transformers'));
         if (cacheDir)
             env.cacheDir = cacheDir;
         env.allowRemoteModels = !offline;
-        const tokenizer = await AutoTokenizer.from_pretrained(RERANK_MODEL, { progress_callback: emitDownloadEvent });
+        const tokenizer = await AutoTokenizer.from_pretrained(adapter.id, {
+            ...(adapter.revision ? { revision: adapter.revision } : {}),
+            progress_callback: emitDownloadEvent,
+        });
         const loadOpts = {
-            dtype: 'q8',
+            dtype: adapter.dtype,
             progress_callback: emitDownloadEvent,
+            ...(adapter.revision ? { revision: adapter.revision } : {}),
             ...(sessionOptions ? { session_options: sessionOptions } : {}),
         };
-        const model = await AutoModelForSequenceClassification.from_pretrained(RERANK_MODEL, loadOpts);
+        const model = await AutoModelForSequenceClassification.from_pretrained(adapter.id, loadOpts);
         const r = { tokenizer, model };
         _rerankers.set(key, r);
         return r;
@@ -31,12 +81,21 @@ export async function getReranker(cacheDir, offline = false, retryOpts = {}) {
     }
     return await retryWithBackoff(loadPipeline, retryOpts);
 }
-export async function rerankScores(query, passages, cacheDir, offline = false) {
+export async function rerankScores(query, passages, cacheDir, offline = false, rerankerName) {
     if (passages.length === 0)
         return [];
-    const { tokenizer, model } = await getReranker(cacheDir, offline);
+    const { tokenizer, model } = await getReranker(cacheDir, offline, {}, rerankerName);
     const queries = new Array(passages.length).fill(query);
     const inputs = tokenizer(queries, { text_pair: passages, padding: true, truncation: true });
     const outputs = await model(inputs);
-    return outputs.logits.tolist().map((row) => row[0]);
+    const rawLogits = outputs?.logits?.tolist();
+    if (!Array.isArray(rawLogits)) {
+        throw new Error('reranker model returned invalid logits structure');
+    }
+    return rawLogits.map((row, idx) => {
+        if (!Array.isArray(row) || typeof row[0] !== 'number' || !Number.isFinite(row[0])) {
+            throw new Error(`reranker model returned non-finite score at index ${idx}`);
+        }
+        return row[0];
+    });
 }

package/dist/search.d.ts

@@ -2,6 +2,7 @@ import { ModelDescriptor } from './core.js';
 import { LexicalIndex } from './lexical.js';
 import { DocumentMetadata } from './frontmatter.js';
 import { IVFIndex } from './ivf.js';
+import { HnswIndex } from './hnsw.js';
 import { RelationshipGraph } from './wikilinks.js';
 import { FilterNode } from './filter.js';
 export { tokenize } from './lexical.js';
@@ -13,6 +14,17 @@ export declare function rrf(rankings: Array<Array<{
     idx: number;
     score: number;
 }>>, k?: number, weights?: number[]): Map<number, number>;
+/**
+ * issue #191: bounded top-k selection over a dense sweep — a capped min-heap
+ * instead of materializing one object per chunk and fully sorting. Ties break
+ * toward the smaller idx, which is exactly what a stable sort by
+ * (score desc) over idx-ordered input produces, so the selected pool and its
+ * order are identical to `[...items].sort((a, b) => b.score - a.score).slice(0, k)`.
+ */
+export declare function topKByScore(n: number, scoreAt: (i: number) => number, k: number): Array<{
+    idx: number;
+    score: number;
+}>;
 export interface IndexChunk {
     file: string;
     title: string;
@@ -74,9 +86,18 @@ export interface RuntimeIndexState {
     model: ModelDescriptor;
     queryCache: QueryEmbeddingCache;
     ivf?: IVFIndex;
+    /** issue #201: optional HNSW graph sidecar (hnsw.json), parallel to ivf. */
+    hnsw?: HnswIndex;
     graph?: RelationshipGraph;
     pageRank?: Map<string, number>;
+    /**
+     * issue #190: precomputed sqrt(sum(c^2)) for int8 vectors (zero-copy binary
+     * load only) — lets the dense sweep compute the dot product alone.
+     */
+    chunkNorms?: Float32Array;
 }
+/** issue #190 tool/test seam: the cached int8 sweep norms, when present. */
+export declare function getRuntimeChunkNorms(index: IndexFile): Float32Array | undefined;
 export declare function getRuntimeGraph(runtime: RuntimeIndexState): {
     graph: RelationshipGraph;
     pageRank: Map<string, number>;
@@ -84,7 +105,9 @@ export declare function getRuntimeGraph(runtime: RuntimeIndexState): {
 /**
  * Issue #127 pure helpers — exported for deterministic unit tests.
  */
-/** Frontmatter `created` → `updated` → file mtime; null when nothing parses. */
+/** Frontmatter `updated` → `created` → file mtime; null when nothing parses.
+ *  Issue #164: `updated` wins — long-lived notes that were recently revised
+ *  must not be penalized by their original creation date. */
 export declare function resolvePassageAgeMs(meta: {
     created?: unknown;
     updated?: unknown;
@@ -97,13 +120,23 @@ export declare function recencyBoost(ageMs: number | null, halfLifeDays: number,
  * Injectable fetch for tests.
  */
 export declare function hydePassage(endpoint: string, query: string, model?: string, fetchImpl?: typeof fetch): Promise<string | null>;
+/**
+ * issue #203: LLM paraphrases for multi-query expansion. ONE OpenAI-compatible
+ * chat completion asking for N alternative phrasings; returns clean, deduped,
+ * non-empty variants (the original query excluded). Returns [] on ANY failure —
+ * multi-query must degrade to the single-query baseline silently, exactly like
+ * HyDE. Injectable fetch for tests.
+ */
+export declare function paraphraseQueries(endpoint: string, query: string, count: number, model?: string, fetchImpl?: typeof fetch): Promise<string[]>;
 /**
  * Salient-term extraction for PRF (issue #125): tokens that appear in the
  * top-m feedback passages, are not query terms or stopwords, and are ranked
  * by (tf-in-feedback × distinct-passage coverage). Pure + deterministic.
  */
 export declare function extractExpansionTerms(topTexts: string[], queryTerms: string[], maxTerms: number): string[];
-export declare function loadIndex(indexDir: string): {
+export declare function loadIndex(indexDir: string, opts?: {
+    checkStale?: boolean;
+}): {
     index: IndexFile;
     model: ModelDescriptor;
 };
@@ -147,10 +180,23 @@ export interface SearchOptions {
     llmEndpoint?: string;
     /** issue #125: model name for the HyDE chat call. */
     llmModel?: string;
-    ann?: boolean;
+    /**
+     * issue #203: multi-query expansion — N total query variants (original +
+     * N-1 LLM paraphrases), each embedded, rankings fused with union-RRF.
+     * Requires `llmEndpoint`; without one (or on any LLM failure) it degrades
+     * to the single-query baseline. 0/1/undefined = off.
+     */
+    multiQuery?: number;
+    /** issue #203: reranker registry alias (`bge-reranker-base` default, `gte-reranker-modernbert`, ...). */
+    rerankModel?: string;
+    /** issue #201: ANN engine — `true` = default engine, or explicit 'ivf' | 'hnsw'. */
+    ann?: boolean | 'ivf' | 'hnsw';
     nprobe?: number;
+    /** issue #201: HNSW ef-search breadth (default 64). */
+    ef?: number;
     graphBoost?: number;
     filter?: string | FilterNode;
+    dimensions?: number;
 }
 export interface SearchResultHit {
     file: string;