md-semantic-search 1.0.2 → 1.0.3

package/src/repair.ts

@@ -0,0 +1,245 @@
+/**
+ * Safe auto-repairs for `mdss check --fix` (issue #117).
+ *
+ * Plan/apply split: planRepairs() probes the index directory WITHOUT mutating
+ * anything (so --dry-run can print the exact plan), applyRepairs() executes
+ * actions in order and reports per-action outcomes. Source Markdown is NEVER
+ * touched; the only reads from --db are for recomputing `.hashes.json` or a
+ * full index rebuild.
+ */
+import crypto from 'node:crypto';
+import fs from 'node:fs';
+import path from 'node:path';
+import { LOCK_FILENAME, pidAlive, walkMarkdown } from './core.js';
+import { DEFAULT_MODEL } from './models.js';
+
+const LOCK_STALE_MS = 10 * 60 * 1000;
+
+export interface RepairAction {
+  action:
+    | 'remove-stale-lock'
+    | 'remove-broken-vectors-bin'
+    | 'remove-corrupt-ivf'
+    | 'rebuild-index'
+    | 'rewrite-hashes';
+  detail: string;
+}
+
+export interface RepairPlan {
+  actions: RepairAction[];
+  skipped: RepairAction[];
+}
+
+export interface ApplyResult {
+  performed: RepairAction[];
+  failed: Array<RepairAction & { error: string }>;
+}
+
+interface ProbeOptions {
+  db?: string | null;
+  log?: (msg: string) => void;
+}
+
+function readJsonIfParsable(filePath: string): { ok: true; value: any } | { ok: false } | { ok: 'missing' } {
+  let raw: string;
+  try {
+    raw = fs.readFileSync(filePath, 'utf8');
+  } catch {
+    return { ok: 'missing' };
+  }
+  try {
+    return { ok: true, value: JSON.parse(raw) };
+  } catch {
+    return { ok: false };
+  }
+}
+
+/** Recompute the {relativePath: md5} map exactly as the indexer stores it. */
+export function recomputeHashes(db: string): Record<string, string> {
+  const hashes: Record<string, string> = {};
+  for (const f of walkMarkdown(db)) {
+    const rel = path.relative(db, f).split(path.sep).join('/');
+    const md5 = crypto.createHash('md5').update(fs.readFileSync(f)).digest('hex');
+    hashes[rel] = md5;
+  }
+  return hashes;
+}
+
+/**
+ * Probe the index directory and decide what CAN be safely repaired.
+ * Never mutates anything — safe to call for --dry-run.
+ */
+export function planRepairs(indexDir: string, opts: ProbeOptions = {}): RepairPlan {
+  const plan: RepairPlan = { actions: [], skipped: [] };
+  const db = opts.db ?? null;
+
+  // --- 1. stale / abandoned lock ---
+  const lockPath = path.join(indexDir, LOCK_FILENAME);
+  if (fs.existsSync(lockPath)) {
+    let info: { pid?: unknown; since?: unknown } | null = null;
+    try {
+      info = JSON.parse(fs.readFileSync(lockPath, 'utf8'));
+    } catch {
+      info = null;
+    }
+    const pid = Number.isInteger(info?.pid) ? (info!.pid as number) : null;
+    let statAgeMs = Infinity;
+    try {
+      statAgeMs = Date.now() - fs.statSync(lockPath).mtimeMs;
+    } catch {
+      /* vanished between exists and stat */
+    }
+    if (!info) {
+      plan.actions.push({ action: 'remove-stale-lock', detail: `${LOCK_FILENAME} is unreadable/abandoned` });
+    } else if (pid !== null && !pidAlive(pid)) {
+      plan.actions.push({ action: 'remove-stale-lock', detail: `${LOCK_FILENAME} held by dead pid ${pid}` });
+    } else if (statAgeMs > LOCK_STALE_MS) {
+      plan.actions.push({
+        action: 'remove-stale-lock',
+        detail: `${LOCK_FILENAME} is older than ${Math.round(LOCK_STALE_MS / 60000)} min`,
+      });
+    } else {
+      plan.skipped.push({
+        action: 'remove-stale-lock',
+        detail: `live writer holds the lock${pid ? ` (pid ${pid})` : ''}`,
+      });
+    }
+  }
+
+  // --- 2. vectors.bin integrity (SHA-256 sidecar) ---
+  const binPath = path.join(indexDir, 'vectors.bin');
+  const shaPath = `${binPath}.sha256`;
+  if (fs.existsSync(binPath)) {
+    let broken: string | null = null;
+    try {
+      const actual = crypto.createHash('sha256').update(fs.readFileSync(binPath)).digest('hex');
+      if (!fs.existsSync(shaPath)) {
+        broken = 'missing vectors.bin.sha256 sidecar';
+      } else {
+        const expected = fs.readFileSync(shaPath, 'utf8').trim().split(/\s+/)[0];
+        if (!expected || expected !== actual) broken = 'SHA-256 mismatch';
+      }
+    } catch (e: any) {
+      broken = e.message;
+    }
+    if (broken) {
+      plan.actions.push({
+        action: 'remove-broken-vectors-bin',
+        detail: `vectors.bin ignored (${broken}) — search falls back to vectors.json`,
+      });
+    }
+  }
+
+  // --- 3. corrupt ivf.json (optional ANN sidecar; loader silently ignores it) ---
+  const ivfPath = path.join(indexDir, 'ivf.json');
+  const ivfRaw = readJsonIfParsable(ivfPath);
+  if (ivfRaw.ok === false) {
+    plan.actions.push({
+      action: 'remove-corrupt-ivf',
+      detail: 'ivf.json is not valid JSON — removing it re-enables brute-force ANN fallback',
+    });
+  }
+
+  // --- 4. corrupt vectors.json ---
+  const liveWriter = plan.skipped.some(
+    (s) => s.action === 'remove-stale-lock' && /live writer/.test(s.detail),
+  );
+  const vectorsRaw = readJsonIfParsable(path.join(indexDir, 'vectors.json'));
+  if (vectorsRaw.ok === false) {
+    if (db && !liveWriter) {
+      plan.actions.push({ action: 'rebuild-index', detail: `vectors.json is not valid JSON — rebuild from ${db}` });
+    } else if (liveWriter) {
+      plan.skipped.push({
+        action: 'rebuild-index',
+        detail: 'vectors.json is not valid JSON, but a live writer holds the lock — retry after it finishes',
+      });
+    } else {
+      plan.skipped.push({
+        action: 'rebuild-index',
+        detail: 'vectors.json is not valid JSON — pass --db so --fix can rebuild it',
+      });
+    }
+  }
+
+  // --- 5. damaged .hashes.json ---
+  const hashesRaw = readJsonIfParsable(path.join(indexDir, '.hashes.json'));
+  if (hashesRaw.ok === false) {
+    if (db) {
+      plan.actions.push({ action: 'rewrite-hashes', detail: '.hashes.json is not valid JSON — recompute from source md5' });
+    } else {
+      plan.skipped.push({
+        action: 'rewrite-hashes',
+        detail: '.hashes.json is not valid JSON — pass --db so --fix can recompute it',
+      });
+    }
+  }
+
+  return plan;
+}
+
+/** Execute planned actions in order. dryRun=true performs nothing. */
+export function applyRepairs(
+  indexDir: string,
+  plan: RepairPlan,
+  opts: {
+    dryRun?: boolean;
+    db?: string | null;
+    cacheDir?: string;
+    modelName?: string;
+    embedFn?: any;
+    offline?: boolean;
+    log?: (msg: string) => void;
+  } = {}
+): Promise<ApplyResult> {
+  const performed: RepairAction[] = [];
+  const failed: Array<RepairAction & { error: string }> = [];
+  const log = opts.log ?? (() => {});
+
+  const run = async (): Promise<ApplyResult> => {
+    for (const action of plan.actions) {
+      if (opts.dryRun) {
+        performed.push(action);
+        continue;
+      }
+      try {
+        switch (action.action) {
+          case 'remove-stale-lock':
+            fs.unlinkSync(path.join(indexDir, LOCK_FILENAME));
+            break;
+          case 'remove-broken-vectors-bin':
+            fs.rmSync(path.join(indexDir, 'vectors.bin'), { force: true });
+            fs.rmSync(`${path.join(indexDir, 'vectors.bin')}.sha256`, { force: true });
+            break;
+          case 'remove-corrupt-ivf':
+            fs.rmSync(path.join(indexDir, 'ivf.json'), { force: true });
+            break;
+          case 'rewrite-hashes': {
+            const hashes = recomputeHashes(opts.db!);
+            fs.writeFileSync(path.join(indexDir, '.hashes.json'), JSON.stringify(hashes, null, 2));
+            break;
+          }
+          case 'rebuild-index': {
+            const { buildIndex } = await import('./indexer.js');
+            const rebuildOpts: Parameters<typeof buildIndex>[0] = {
+              db: opts.db!,
+              indexDir,
+              cacheDir: opts.cacheDir!,
+              modelName: opts.modelName || DEFAULT_MODEL,
+              log,
+              _lockHeld: true, // we may have JUST removed a stale lock; never double-lock inside fix
+            };
+            if (opts.embedFn) rebuildOpts.embedFn = opts.embedFn; // test DI — keeps repairs model-free
+            await buildIndex(rebuildOpts);
+            break;
+          }
+        }
+        performed.push(action);
+      } catch (e: any) {
+        failed.push({ ...action, error: e.message });
+      }
+    }
+    return { performed, failed };
+  };
+
+  return run();
+}

package/src/rerank.ts

@@ -1,7 +1,7 @@
 /**
  * Cross-encoder re-ranking (issue #15).
  */
-import { resolveSessionOptions, retryWithBackoff } from './core.js';
+import { resolveSessionOptions, retryWithBackoff, emitDownloadEvent } from './core.js';
 
 const _rerankers = new Map<string, { tokenizer: any; model: any }>();
 
@@ -21,9 +21,10 @@ export async function getReranker(
     const { AutoModelForSequenceClassification, AutoTokenizer, env } = await import('@huggingface/transformers');
     if (cacheDir) env.cacheDir = cacheDir;
     env.allowRemoteModels = !offline;
-    const tokenizer = await AutoTokenizer.from_pretrained(RERANK_MODEL);
+    const tokenizer = await AutoTokenizer.from_pretrained(RERANK_MODEL, { progress_callback: emitDownloadEvent });
     const loadOpts: Record<string, unknown> = {
       dtype: 'q8',
+      progress_callback: emitDownloadEvent,
       ...(sessionOptions ? { session_options: sessionOptions } : {}),
     };
     const model = await AutoModelForSequenceClassification.from_pretrained(RERANK_MODEL, loadOpts);

package/src/search.ts

@@ -12,6 +12,7 @@ import { buildRelationshipGraph, computePageRank, expandGraphNeighborhood, Relat
 import { evaluateFilter, FilterNode } from './filter.js';
 import { deserializeBinaryIndex } from './binary-format.js';
 import { dequantizeFromInt8 } from './quantization.js';
+import { buildDiskKey, diskQueryGet, diskQueryPut } from './query-cache-disk.js';
 
 export { tokenize } from './lexical.js';
 
@@ -208,8 +209,110 @@ export function getRuntimeGraph(runtime: RuntimeIndexState): { graph: Relationsh
   return { graph, pageRank };
 }
 
+/**
+ * Issue #127 pure helpers — exported for deterministic unit tests.
+ */
+/** Frontmatter `created` → `updated` → file mtime; null when nothing parses. */
+export function resolvePassageAgeMs(
+  meta: { created?: unknown; updated?: unknown } | undefined,
+  absFile?: string,
+): number | null {
+  for (const v of [meta?.created, meta?.updated]) {
+    if (typeof v === 'string') {
+      const t = Date.parse(v);
+      if (Number.isFinite(t)) return t;
+    } else if (typeof v === 'number' && Number.isFinite(v)) {
+      return v;
+    }
+  }
+  if (absFile) {
+    try {
+      return fs.statSync(absFile).mtimeMs;
+    } catch {
+      /* file gone — treat as unknown age */
+    }
+  }
+  return null;
+}
+
+/** Multiplicative recency boost: 0.5^(ageDays / halfLifeDays); 1 when off/unknown/future. */
+export function recencyBoost(ageMs: number | null, halfLifeDays: number, now: number = Date.now()): number {
+  if (ageMs === null || !(halfLifeDays > 0)) return 1;
+  const ageDays = Math.max(0, (now - ageMs) / 86_400_000);
+  return Math.pow(0.5, ageDays / halfLifeDays);
+}
+
+/**
+ * HyDE passage generation via an OpenAI-compatible chat endpoint (issue #125).
+ * Returns null on ANY failure — HyDE must degrade to PRF/baseline silently.
+ * Injectable fetch for tests.
+ */
+export async function hydePassage(
+  endpoint: string,
+  query: string,
+  model?: string,
+  fetchImpl: typeof fetch = globalThis.fetch,
+): Promise<string | null> {
+  try {
+    const res = await fetchImpl(endpoint.replace(/\/$/, '') + '/chat/completions', {
+      method: 'POST',
+      headers: { 'content-type': 'application/json' },
+      body: JSON.stringify({
+        ...(model ? { model } : {}),
+        messages: [
+          { role: 'system', content: 'Write one short passage (2-4 sentences) that would plausibly contain the answer. Output only the passage.' },
+          { role: 'user', content: query },
+        ],
+      }),
+    });
+    if (!res.ok) return null;
+    const data: any = await res.json();
+    const text = data?.choices?.[0]?.message?.content ?? data?.response ?? null;
+    return typeof text === 'string' && text.trim() ? text.trim() : null;
+  } catch {
+    return null;
+  }
+}
+
 const runtimeIndexStates = new WeakMap<IndexFile, RuntimeIndexState>();
 
+/** Minimal EN+RU stopword set for PRF expansion (issue #125). */
+const EXPANSION_STOPWORDS = new Set([
+  'the', 'a', 'an', 'and', 'or', 'of', 'to', 'in', 'on', 'for', 'with', 'is', 'are',
+  'be', 'this', 'that', 'it', 'as', 'at', 'by', 'from', 'not', 'but', 'we', 'you',
+  'и', 'в', 'на', 'с', 'по', 'для', 'как', 'это', 'не', 'а', 'но', 'или', 'из',
+]);
+
+/**
+ * Salient-term extraction for PRF (issue #125): tokens that appear in the
+ * top-m feedback passages, are not query terms or stopwords, and are ranked
+ * by (tf-in-feedback × distinct-passage coverage). Pure + deterministic.
+ */
+export function extractExpansionTerms(
+  topTexts: string[],
+  queryTerms: string[],
+  maxTerms: number,
+): string[] {
+  const qSet = new Set(queryTerms);
+  const tf = new Map<string, number>();
+  const df = new Map<string, number>();
+  for (const text of topTexts) {
+    const seen = new Set<string>();
+    for (const tok of tokenize(text)) {
+      if (tok.length < 3 || qSet.has(tok) || EXPANSION_STOPWORDS.has(tok)) continue;
+      tf.set(tok, (tf.get(tok) ?? 0) + 1);
+      seen.add(tok);
+    }
+    for (const tok of seen) df.set(tok, (df.get(tok) ?? 0) + 1);
+  }
+  return [...tf.entries()]
+    .filter(([, count]) => count >= 2 || topTexts.length === 1)
+    .map(([tok, count]) => [tok, count * Math.log(2 + (df.get(tok) ?? 0))] as const)
+    .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
+    .slice(0, maxTerms)
+    .map(([tok]) => tok);
+}
+
 function snapshotRuntimeIndex(
   index: IndexFile,
   validation: {
@@ -500,6 +603,18 @@ export interface SearchOptions {
   maxPerFile?: number;
   maxPerDoc?: number;
   useQueryCache?: boolean;
+  /** issue #114: persist query embeddings to <cacheDir>/query-cache.json (default on). */
+  queryDiskCache?: boolean;
+  /** issue #127: time-decay half-life in days; boost = 0.5^(ageDays/halfLife). Omitted = off. */
+  recency?: number;
+  /** issue #125: query expansion mode. 'prf' is fully offline; 'hyde' needs llmEndpoint (else baseline). */
+  expand?: 'prf' | 'hyde';
+  /** issue #125: feedback passages for 'prf' (default 3). */
+  expandPassages?: number;
+  /** issue #125: LLM endpoint used by 'hyde' (OpenAI-compatible chat completions). */
+  llmEndpoint?: string;
+  /** issue #125: model name for the HyDE chat call. */
+  llmModel?: string;
   ann?: boolean;
   nprobe?: number;
   graphBoost?: number;
@@ -615,19 +730,57 @@ export async function searchIndex(opts: SearchOptions): Promise<SearchResultHit[
     });
   }
 
+  // Issue #125: offline pseudo-relevance feedback (PRF) query expansion.
+  // A lexical-only first pass picks the top-m passages; salient terms from
+  // them enrich the query, which is then embedded ONCE (replacing the raw
+  // query embed — zero extra model calls). HyDE would swap the enriched text
+  // for an LLM-generated passage; without a configured endpoint it degrades
+  // to the baseline silently.
+  let effectiveQuery = query;
+  let expandedTerms: string[] | null = null;
+  if ((opts.expand === 'prf' || opts.expand === 'hyde') && candidates.length > 0) {
+    const qt0 = [...new Set(tokenize(query))];
+    if (qt0.length > 0 && lexicalState.kind === 'persisted-bm25') {
+      const firstPass = [
+        ...bm25Scores(lexicalState.lexical, qt0, new Set(candidates.map((c) => c.idx))).entries(),
+      ]
+        .sort((a, b) => b[1] - a[1])
+        .slice(0, Math.max(1, opts.expandPassages ?? 3));
+      const topTexts = firstPass.map(([idx]) => chunks[idx].text);
+      expandedTerms = extractExpansionTerms(topTexts, qt0, 8);
+      if (opts.expand === 'hyde' && opts.llmEndpoint) {
+        const hydeText = await hydePassage(opts.llmEndpoint, query, opts.llmModel);
+        if (hydeText) effectiveQuery = hydeText;
+        else if (expandedTerms.length > 0) effectiveQuery = `${query} ${expandedTerms.join(' ')}`;
+      } else if (expandedTerms.length > 0) {
+        effectiveQuery = `${query} ${expandedTerms.join(' ')}`;
+      }
+    }
+  }
+
   let qVec: Float32Array | number[];
   if (opts.useQueryCache !== false && runtime.queryCache) {
-    const cacheKey = `${runtime.model.id}:${runtime.model.revision || 'main'}:${expectedDim || 0}:${query.trim().toLowerCase()}`;
+    const normQuery = effectiveQuery.trim().toLowerCase();
+    const cacheKey = `${runtime.model.id}:${runtime.model.revision || 'main'}:${expectedDim || 0}:${normQuery}`;
+    // L2 disk layer (issue #114): persists across processes under cacheDir;
+    // corrupt/missing files degrade silently to a normal embed.
+    const diskKey = buildDiskKey(runtime.model.id, runtime.model.revision || 'main', expectedDim, normQuery);
+    const useDisk = opts.queryDiskCache !== false;
     qVec = await runtime.queryCache.getOrCompute(cacheKey, async () => {
-      const [v] = await embedFn([query], 'query', runtime.model, cacheDir, offline);
+      if (useDisk) {
+        const fromDisk = diskQueryGet(cacheDir, diskKey, expectedDim);
+        if (fromDisk) return fromDisk;
+      }
+      const [v] = await embedFn([effectiveQuery], 'query', runtime.model, cacheDir, offline);
       if ((Array.isArray(v) || v instanceof Float32Array) && expectedDim !== undefined && v.length !== expectedDim) {
-        throw new Error(`query vector has ${v.length} dims, expected ${expectedDim} — run \`mdss index\` to rebuild`);
+        throw new Error(`query vector has ${v.length} dims, expected ${expectedDim} - run \`mdss index\` to rebuild`);
       }
       validateNumericVector(v, undefined, 'query vector');
+      if (useDisk) diskQueryPut(cacheDir, diskKey, v);
       return v;
     });
   } else {
-    const [v] = await embedFn([query], 'query', runtime.model, cacheDir, offline);
+    const [v] = await embedFn([effectiveQuery], 'query', runtime.model, cacheDir, offline);
     if ((Array.isArray(v) || v instanceof Float32Array) && expectedDim !== undefined && v.length !== expectedDim) {
       throw new Error(`query vector has ${v.length} dims, expected ${expectedDim} — run \`mdss index\` to rebuild`);
     }
@@ -685,7 +838,7 @@ export async function searchIndex(opts: SearchOptions): Promise<SearchResultHit[
     });
   }
 
-  let ranked: Array<{ idx: number; fscore: number; cos: number; rerank?: number }>;
+  let ranked: Array<{ idx: number; fscore: number; cos: number; rerank?: number; rawFscore?: number; recencyFactor?: number; recencyAgeDays?: number | null }>;
   if (semanticOnly) {
     if (graphBoost > 0) {
       const fused = rrf([semantic, graphRanking], 60, [1, graphBoost]);
@@ -730,6 +883,28 @@ export async function searchIndex(opts: SearchOptions): Promise<SearchResultHit[
     }
   }
 
+  // Issue #127: time-decay recency boost — multiply the fused score by
+  // 0.5^(ageDays / halfLifeDays), post-RRF and pre-collapse. Age source
+  // priority: frontmatter `created` → `updated` → file mtime. Missing dates
+  // get factor 1 (evergreen content stays reachable via lexical strength).
+  const recencyHalfLife = Number(opts.recency) > 0 ? Number(opts.recency) : 0;
+  if (recencyHalfLife > 0) {
+    const now = Date.now();
+    const ageCache = new Map<string, number | null>();
+    for (const r of ranked) {
+      const c = chunks[r.idx];
+      const abs = runtime.db && !path.isAbsolute(c.file) ? path.join(runtime.db, c.file) : c.file;
+      const ageKey = `${c.file}|${c.meta?.updated ?? ''}|${c.meta?.created ?? ''}`;
+      if (!ageCache.has(ageKey)) ageCache.set(ageKey, resolvePassageAgeMs(c.meta, abs));
+      const ageMs = ageCache.get(ageKey) ?? null;
+      r.recencyFactor = recencyBoost(ageMs, recencyHalfLife, now);
+      r.recencyAgeDays = ageMs === null ? null : Math.max(0, (now - ageMs) / 86_400_000);
+      r.rawFscore = r.fscore;
+      r.fscore = r.fscore * r.recencyFactor;
+    }
+    ranked.sort((a, b) => b.fscore - a.fscore);
+  }
+
   if (rerank && ranked.length > 0) {
     const texts = ranked.map((r) => chunks[r.idx].text);
     const scores = await rerankFn(query, texts, cacheDir, offline);
@@ -778,10 +953,20 @@ export async function searchIndex(opts: SearchOptions): Promise<SearchResultHit[
         ? {
             explain: {
               cosine: +r.cos.toFixed(4),
-              rrfScore: +r.fscore.toFixed(4),
+              rrfScore: +(r.rawFscore ?? r.fscore).toFixed(4),
               bm25Weights: { title: 3.0, aliases: 3.0, headingPath: 1.8, body: 1.0 },
               ...(gScore !== undefined ? { graphScore: +gScore.toFixed(4) } : {}),
               ...(prScore !== undefined ? { pageRank: +prScore.toFixed(4) } : {}),
+              ...(r.recencyFactor !== undefined
+                ? {
+                    recencyFactor: +r.recencyFactor.toFixed(4),
+                    recencyAgeDays:
+                      r.recencyAgeDays === null || r.recencyAgeDays === undefined
+                        ? null
+                        : +r.recencyAgeDays.toFixed(2),
+                  }
+                : {}),
+              ...(expandedTerms ? { expandedTerms } : {}),
             },
           }
         : {}),
@@ -803,3 +988,4 @@ export async function search(opts: OneShotSearchOptions): Promise<SearchResultHi
   const loaded = loadIndex(indexDir);
   return searchIndex({ ...rest, loaded });
 }
+