md-semantic-search 1.0.3 → 2.0.0

package/src/core.ts

@@ -78,7 +78,10 @@ export type LockAcquireResult =
   | { acquired: true; lockPath: string }
   | { acquired: false; reason: string; pid: number | null; heldSince: string | null };
 
-export function acquireIndexLock(indexDir: string): LockAcquireResult {
+export function acquireIndexLock(
+  indexDir: string,
+  _testHooks?: { onClassified?: () => void },
+): LockAcquireResult {
   const lockPath = path.join(indexDir, LOCK_FILENAME);
   const payload = JSON.stringify({ pid: process.pid, since: new Date().toISOString() }) + '\n';
   for (let attempt = 0; attempt < 2; attempt++) {
@@ -104,6 +107,24 @@ export function acquireIndexLock(indexDir: string): LockAcquireResult {
       const dead = pid !== null && !pidAlive(pid);
       const staleMs = stat ? Date.now() - stat.mtimeMs : 0;
       if ((garbage || dead || staleMs > LOCK_STALE_MS) && stat) {
+        // issue #188: between the stat/read above and the unlink below another
+        // writer may reclaim the lock and write a fresh, live one. Re-read and
+        // unlink only when the owner identity (pid + since) still matches what
+        // was classified — never unlink a lock we did not classify.
+        _testHooks?.onClassified?.();
+        if (!fs.existsSync(lockPath)) continue;
+        const current = readLock(lockPath);
+        const sameOwner = garbage
+          ? current === null
+          : current !== null && current.pid === pid && current.since === heldSince;
+        if (!sameOwner) {
+          return {
+            acquired: false,
+            reason: 'lock changed while reclaiming — another writer got here first',
+            pid: current?.pid ?? pid,
+            heldSince: current?.since ?? heldSince,
+          };
+        }
         try {
           fs.unlinkSync(lockPath);
         } catch {
@@ -166,14 +187,18 @@ export function resolveSessionOptions(model?: ModelDescriptor): Record<string, u
   const intra = envIntra ? parseInt(envIntra, 10) : undefined;
   const inter = envInter ? parseInt(envInter, 10) : undefined;
   const opts = model?.sessionOptions || {};
-  const intraOpNumThreads = Number.isSafeInteger(intra) && intra && intra > 0 ? intra : (opts.intraOpNumThreads as number | undefined);
+  // issue #194: ONNX wasm sessions default to a single intra-op thread — the
+  // build/query sweep leaves most of the CPU idle. Default to the physical
+  // core count (measured 1.2x on a 4c/8t laptop; hyperthreads add contention,
+  // not throughput), still overridable via MDSS_INTRA_OP_THREADS.
+  const defaultIntra = Math.max(1, Math.min(4, os.cpus().length >> 1));
+  const intraOpNumThreads = Number.isSafeInteger(intra) && intra && intra > 0
+    ? intra
+    : (opts.intraOpNumThreads as number | undefined) ?? defaultIntra;
   const interOpNumThreads = Number.isSafeInteger(inter) && inter && inter > 0 ? inter : (opts.interOpNumThreads as number | undefined);
 
-  if (intraOpNumThreads === undefined && interOpNumThreads === undefined) {
-    return undefined;
-  }
   const result: Record<string, unknown> = {};
-  if (intraOpNumThreads !== undefined) result.intraOpNumThreads = intraOpNumThreads;
+  result.intraOpNumThreads = intraOpNumThreads;
   if (interOpNumThreads !== undefined) result.interOpNumThreads = interOpNumThreads;
   return result;
 }
@@ -188,6 +213,9 @@ export function getExtractorCacheKey(model: ModelDescriptor, cacheDir: string, o
 
 export function isNetworkError(err: any): boolean {
   if (!err) return false;
+  // issue #184: outbound fetch timeouts abort with TimeoutError/AbortError —
+  // they must classify as network errors so retries and fallbacks engage
+  if (err.name === 'TimeoutError' || err.name === 'AbortError') return true;
   const code = err.code || err.cause?.code;
   if (code && ['ENOTFOUND', 'ETIMEDOUT', 'ECONNRESET', 'ECONNREFUSED', 'EPIPE', 'EHOSTUNREACH', 'ENETUNREACH', 'FETCH_ERROR'].includes(code)) {
     return true;
@@ -201,12 +229,27 @@ export function isNetworkError(err: any): boolean {
     msg.includes('etimedout') ||
     msg.includes('enotfound') ||
     msg.includes('network error') ||
+    msg.includes('aborted due to timeout') ||
     msg.includes('503 service unavailable') ||
     msg.includes('502 bad gateway') ||
     msg.includes('504 gateway timeout')
   );
 }
 
+/**
+ * Outbound fetch timeout (issue #184): undici's default ~300s lets one hung
+ * embedder/LLM endpoint hold a serve concurrency slot or stall `mdss index`
+ * for minutes. All outbound calls carry `AbortSignal.timeout(fetchTimeoutMs())`;
+ * override with `MDSS_FETCH_TIMEOUT_MS` (0/negative falls back to the default).
+ */
+export const DEFAULT_FETCH_TIMEOUT_MS = 30_000;
+
+export function fetchTimeoutMs(envValue?: string): number {
+  const raw = envValue ?? process.env.MDSS_FETCH_TIMEOUT_MS;
+  const n = Number(raw);
+  return Number.isFinite(n) && n > 0 ? n : DEFAULT_FETCH_TIMEOUT_MS;
+}
+
 export async function retryWithBackoff<T>(
   fn: () => Promise<T>,
   opts: { maxRetries?: number; delays?: number[]; log?: (msg: string) => void } = {}
@@ -257,6 +300,44 @@ export async function getExtractor(
   const key = getExtractorCacheKey(model, cacheDir, offline);
   if (_extractors.has(key)) return _extractors.get(key);
 
+  // issue #200: adapters whose ONNX graph emits a pre-pooled
+  // `sentence_embedding` (EmbeddingGemma family) bypass the feature-extraction
+  // pipeline — it can only pool last_hidden_state/logits/token_embeddings and
+  // would crash on such graphs. The shim keeps the pipeline call signature so
+  // embed() is unchanged.
+  if (model.pooledOutput === true) {
+    const loadPooledModel = async () => {
+      const { AutoModel, AutoTokenizer, env } = await import('@huggingface/transformers');
+      if (cacheDir) env.cacheDir = cacheDir;
+      env.allowRemoteModels = !offline;
+      const sessionOptions = resolveSessionOptions(model);
+      const modelOpts: any = {
+        revision,
+        dtype: model.dtype || 'q8',
+        progress_callback: emitDownloadEvent,
+        ...(sessionOptions ? { session_options: sessionOptions } : {}),
+      };
+      const tokenizer = await AutoTokenizer.from_pretrained(source, { revision });
+      const onnxModel = await AutoModel.from_pretrained(source, modelOpts);
+      const normalize = model.normalize !== false;
+      const ext = async (input: string | string[]) => {
+        const texts = Array.isArray(input) ? input : [input];
+        const modelInputs = await tokenizer(texts, { padding: true, truncation: true });
+        const outputs = await onnxModel(modelInputs);
+        let result = outputs.sentence_embedding;
+        if (!result) {
+          throw new Error(`model "${model.id}" declared pooledOutput but produced no sentence_embedding output`);
+        }
+        if (normalize) result = result.normalize(2, -1);
+        return { tolist: () => result.tolist() };
+      };
+      _extractors.set(key, ext);
+      return ext;
+    };
+    if (offline) return await loadPooledModel();
+    return await retryWithBackoff(loadPooledModel, retryOpts);
+  }
+
   const loadPipeline = async () => {
     const { pipeline, env } = await import('@huggingface/transformers');
     if (cacheDir) env.cacheDir = cacheDir;
@@ -300,6 +381,30 @@ export function prepareEmbeddingRequest(
   };
 }
 
+export function truncateAndNormalizeVector(
+  vec: number[] | Float32Array,
+  targetDim: number,
+  normalize: boolean = true
+): number[] {
+  const len = Math.min(vec.length, targetDim);
+  const out = new Array<number>(len);
+  let norm = 0;
+  for (let i = 0; i < len; i++) {
+    const val = vec[i];
+    out[i] = val;
+    norm += val * val;
+  }
+  if (normalize) {
+    norm = Math.sqrt(norm);
+    if (norm > 0) {
+      for (let i = 0; i < len; i++) {
+        out[i] /= norm;
+      }
+    }
+  }
+  return out;
+}
+
 export async function embed(
   texts: string[],
   kind: 'query' | 'passage',
@@ -311,7 +416,13 @@ export async function embed(
   const ext = await getExtractor(model, cacheDir, offline, retryOpts);
   const { input, options } = prepareEmbeddingRequest(model, texts, kind);
   const out = await ext(input, options);
-  return out.tolist();
+  const list: number[][] = out.tolist();
+  const targetDim = model.dim;
+  if (targetDim && list.length > 0 && list[0].length > targetDim) {
+    const doNorm = model.normalize !== false;
+    return list.map((vec) => truncateAndNormalizeVector(vec, targetDim, doNorm));
+  }
+  return list;
 }
 
 export function cosine(
@@ -429,14 +540,20 @@ export const SCHEMA_MIGRATIONS: Record<number, (index: any) => void> = {
   3: migrateToSchemaV3,
 };
 
-export function walkMarkdown(dir: string, ignore: string[] = []): string[] {
+export function walkMarkdown(dir: string, ignore: string[] = [], errors?: string[]): string[] {
   const out: string[] = [];
   const ignoreRe = ignore.map(globToRegExp);
   const walk = (cur: string) => {
     let entries: fs.Dirent[];
     try {
       entries = fs.readdirSync(cur, { withFileTypes: true });
-    } catch {
+    } catch (e: any) {
+      // issue #189: an unreadable directory (permissions, OneDrive eviction)
+      // used to silently shrink the corpus; surface it when a collector is
+      // given, always at least as a stderr warning
+      const msg = `walkMarkdown: cannot read directory ${cur}: ${e?.message || e}`;
+      if (errors) errors.push(msg);
+      process.stderr.write(`warning: ${msg}\n`);
       return;
     }
     for (const e of entries) {
@@ -542,8 +659,17 @@ export function extractTitle(frontmatter: string, body: string, relPath: string)
 
 const DEFAULT_MAX_CHUNK = 1400;
 
-export function chunkMarkdown(body: string, maxChunk: number = DEFAULT_MAX_CHUNK): StructuralChunk[] {
-  return chunkMarkdownStructural(body, maxChunk);
+export function chunkMarkdown(
+  body: string,
+  maxChunk: number = DEFAULT_MAX_CHUNK,
+  maxChunkTokens?: number
+): StructuralChunk[] {
+  // issue #202: a token budget (maxChunkTokens) takes precedence over the
+  // legacy char budget; without it the pre-#202 char behavior is unchanged
+  return chunkMarkdownStructural(
+    body,
+    maxChunkTokens && maxChunkTokens > 0 ? { maxChunkTokens } : maxChunk
+  );
 }
 
 export interface ParsedChunkFile {
@@ -557,14 +683,21 @@ export interface ParsedChunkFile {
   meta: DocumentMetadata;
 }
 
-export function parseFile(absPath: string, dbDir: string, maxChunk?: number, raw?: string): ParsedChunkFile[] {
+export function parseFile(
+  absPath: string,
+  dbDir: string,
+  maxChunk?: number,
+  raw?: string,
+  /** issue #202: token budget overrides the char budget when set. */
+  maxChunkTokens?: number
+): ParsedChunkFile[] {
   const content = raw ?? fs.readFileSync(absPath, 'utf8');
   const { frontmatter, body } = splitFrontmatter(content);
   const meta = parseFrontmatter(frontmatter);
   const rel = path.relative(dbDir, absPath).split(path.sep).join('/');
   const title = meta.title || extractTitle(frontmatter, body, rel);
   const frontmatterLineCount = frontmatter ? frontmatter.split('\n').length + 2 : 0;
-  return chunkMarkdown(body, maxChunk).map((c) => {
+  return chunkMarkdown(body, maxChunk, maxChunkTokens).map((c) => {
     const heading = c.heading || title;
     return {
       file: rel,
@@ -578,3 +711,106 @@ export function parseFile(absPath: string, dbDir: string, maxChunk?: number, raw
     };
   });
 }
+
+export interface GetDocLinesOptions {
+  file: string;
+  fromLine?: number;
+  maxLines?: number;
+  dbDir?: string;
+  chunks?: Array<{ file: string; text: string; startLine?: number; endLine?: number }>;
+}
+
+export interface GetDocLinesResult {
+  file: string;
+  fromLine: number;
+  toLine: number;
+  lineCount: number;
+  totalLines: number;
+  text: string;
+}
+
+/**
+ * Line-range retrieval helper (issue #140).
+ * Retrieves an exact line span from disk or fallback chunk data with EOF clamping.
+ */
+export function getDocLines(opts: GetDocLinesOptions): GetDocLinesResult {
+  const { file, fromLine = 1, maxLines, dbDir, chunks } = opts;
+  if (!file || typeof file !== 'string') {
+    throw new Error('getDocLines: "file" parameter is required');
+  }
+
+  const normalizedFile = file.split(/[/\\]/).join('/');
+  let rawContent: string | null = null;
+
+  if (dbDir) {
+    const candidatePath = path.resolve(dbDir, file);
+    try {
+      assertSafePath(candidatePath, [dbDir]);
+      if (fs.existsSync(candidatePath) && fs.statSync(candidatePath).isFile()) {
+        rawContent = fs.readFileSync(candidatePath, 'utf8');
+      }
+    } catch (e: any) {
+      if (e.message?.includes('path traversal guard')) throw e;
+    }
+  }
+
+  if (rawContent === null && Array.isArray(chunks)) {
+    const matchingChunks = chunks.filter(
+      (c) => c.file === file || c.file === normalizedFile || c.file.split(/[/\\]/).join('/') === normalizedFile
+    );
+    if (matchingChunks.length > 0) {
+      matchingChunks.sort((a, b) => (a.startLine || 0) - (b.startLine || 0));
+      const maxEnd = Math.max(...matchingChunks.map((c) => c.endLine || c.startLine || 1));
+      const lineMap = new Array(maxEnd).fill('');
+      for (const ch of matchingChunks) {
+        const start = (ch.startLine || 1) - 1;
+        const chunkLines = (ch.text || '').split(/\r?\n/);
+        for (let i = 0; i < chunkLines.length; i++) {
+          if (start + i < lineMap.length) {
+            lineMap[start + i] = chunkLines[i];
+          }
+        }
+      }
+      rawContent = lineMap.join('\n');
+    }
+  }
+
+  if (rawContent === null) {
+    throw new Error(`File not found: ${file}`);
+  }
+
+  const lines = rawContent.split(/\r?\n/);
+  const totalLines = lines.length;
+
+  let from = typeof fromLine === 'number' && !isNaN(fromLine) ? Math.floor(fromLine) : 1;
+  if (from < 1) from = 1;
+
+  let selected: string[];
+  let to: number;
+
+  if (totalLines === 0) {
+    from = 1;
+    to = 0;
+    selected = [];
+  } else if (from > totalLines) {
+    from = totalLines;
+    to = totalLines;
+    selected = [lines[totalLines - 1]];
+  } else {
+    const count = typeof maxLines === 'number' && !isNaN(maxLines) && maxLines > 0
+      ? Math.min(Math.floor(maxLines), totalLines - from + 1)
+      : totalLines - from + 1;
+    to = from + count - 1;
+    selected = lines.slice(from - 1, to);
+  }
+
+  return {
+    file: normalizedFile,
+    fromLine: from,
+    toLine: to,
+    lineCount: selected.length,
+    totalLines,
+    text: selected.join('\n'),
+  };
+}
+

package/src/federation.ts

@@ -8,6 +8,7 @@
 import path from 'node:path';
 import fs from 'node:fs';
 import { loadIndex, searchIndex, LoadedIndex, SearchOptions, SearchResultHit } from './search.js';
+import { assertSafePath } from './core.js';
 
 export interface VaultConfig {
   name?: string;
@@ -18,6 +19,13 @@ export interface VaultConfig {
 export interface FederatedSearchOptions extends Omit<SearchOptions, 'loaded' | 'cacheDir'> {
   vaults: Array<string | VaultConfig>;
   cacheDir?: string;
+  /**
+   * issue #179: when set, every vault path (and explicit indexDir) must resolve
+   * inside one of these roots or a traversal-guard error is thrown. Callers
+   * exposing federation to remote clients (MCP) must pin this to their trust
+   * domain; direct CLI use stays guarded only by MDSS_ROOT_GUARD.
+   */
+  allowedRoots?: string[];
 }
 
 export interface FederatedSearchResultHit extends SearchResultHit {
@@ -25,21 +33,32 @@ export interface FederatedSearchResultHit extends SearchResultHit {
   vaultPath: string;
 }
 
-export async function searchFederated(opts: FederatedSearchOptions): Promise<FederatedSearchResultHit[]> {
-  const { vaults, k = 10, cacheDir = '', ...searchOpts } = opts;
+export interface FederatedSearchResult {
+  hits: FederatedSearchResultHit[];
+  /**
+   * issue #189: per-vault load/query failures surfaced to the caller — a
+   * broken vault must not silently look like "0 results".
+   */
+  vaultErrors: string[];
+}
+
+export async function searchFederated(opts: FederatedSearchOptions): Promise<FederatedSearchResult> {
+  const { vaults, k = 10, cacheDir = '', allowedRoots, ...searchOpts } = opts;
   if (!vaults || vaults.length === 0) {
     throw new Error('Federated search requires at least one vault specified');
   }
 
   const normalizedVaults: VaultConfig[] = vaults.map((v) => {
     if (typeof v === 'string') {
-      const absPath = path.resolve(v);
+      const absPath = assertSafePath(path.resolve(v), allowedRoots);
       const name = path.basename(absPath);
       return { name, path: absPath };
     }
-    const absPath = path.resolve(v.path);
+    const absPath = assertSafePath(path.resolve(v.path), allowedRoots);
     const name = v.name || path.basename(absPath);
-    return { name, path: absPath, indexDir: v.indexDir };
+    // indexDir is loaded from disk too — guard it with the same roots (#179)
+    const indexDir = v.indexDir ? assertSafePath(path.resolve(v.indexDir), allowedRoots) : undefined;
+    return { name, path: absPath, indexDir };
   });
 
   // Load and query each vault concurrently
@@ -60,7 +79,10 @@ export async function searchFederated(opts: FederatedSearchOptions): Promise<Fed
       try {
         // Query vault index with a slightly larger pool to ensure healthy global fusion
         const poolK = Math.max(k * 2, 20);
-        const resolvedCache = cacheDir || path.join(vault.path, '.cache');
+        // issue #179: never invent a per-vault cache location — a falsy cacheDir
+        // disables the disk layer (searchIndex semantics), so query embeddings
+        // are only ever written to a cache dir the caller explicitly owns.
+        const resolvedCache = cacheDir;
         const hits = await searchIndex({
           ...searchOpts,
           cacheDir: resolvedCache,
@@ -74,14 +96,23 @@ export async function searchFederated(opts: FederatedSearchOptions): Promise<Fed
     })
   );
 
+  // issue #189: surface per-vault failures — a broken vault must not look
+  // like a vault with simply no matching notes
+  const vaultErrors = perVaultResults
+    .filter((r): r is { vault: VaultConfig; hits: SearchResultHit[]; error: string } => r.error !== null)
+    .map((r) => `vault "${r.vault.name || path.basename(r.vault.path)}" (${r.vault.path}): ${r.error}`);
+
   // If single vault, return directly with attribution
   if (normalizedVaults.length === 1) {
     const { vault, hits } = perVaultResults[0];
-    return hits.slice(0, k).map((h) => ({
-      ...h,
-      vault: vault.name || path.basename(vault.path),
-      vaultPath: vault.path,
-    }));
+    return {
+      hits: hits.slice(0, k).map((h) => ({
+        ...h,
+        vault: vault.name || path.basename(vault.path),
+        vaultPath: vault.path,
+      })),
+      vaultErrors,
+    };
   }
 
   // Multi-vault reciprocal rank fusion (RRF)
@@ -109,10 +140,13 @@ export async function searchFederated(opts: FederatedSearchOptions): Promise<Fed
     })
     .slice(0, k);
 
-  return fused.map(({ hit, vault, rrfSum }) => ({
-    ...hit,
-    score: rrfSum,
-    vault: vault.name || path.basename(vault.path),
-    vaultPath: vault.path,
-  }));
+  return {
+    hits: fused.map(({ hit, vault, rrfSum }) => ({
+      ...hit,
+      score: rrfSum,
+      vault: vault.name || path.basename(vault.path),
+      vaultPath: vault.path,
+    })),
+    vaultErrors,
+  };
 }

package/src/filter.ts

@@ -97,6 +97,14 @@ export function tokenizeFilter(expr: string): string[] {
   return tokens;
 }
 
+/**
+ * issue #180: parseOr/parseAnd/parseNot/parsePrimary are mutually recursive
+ * (~4 stack frames per nesting level) — an unbounded `(((((…` from a 64 KiB
+ * request body would exhaust the call stack. 64 levels is far beyond any
+ * legitimate filter and keeps the parser fail-closed with a clear error.
+ */
+export const MAX_FILTER_DEPTH = 64;
+
 export function parseFilter(expr: string): FilterNode {
   const tokens = tokenizeFilter(expr);
   if (tokens.length === 0) {
@@ -104,6 +112,7 @@ export function parseFilter(expr: string): FilterNode {
   }
 
   let pos = 0;
+  let depth = 0;
 
   function peek(): string | undefined {
     return tokens[pos];
@@ -167,12 +176,17 @@ export function parseFilter(expr: string): FilterNode {
     }
 
     if (token === '(') {
+      depth += 1;
+      if (depth > MAX_FILTER_DEPTH) {
+        throw new Error(`Filter expression too deeply nested (max ${MAX_FILTER_DEPTH} levels of parentheses)`);
+      }
       consume(); // (
       const node = parseExpression();
       if (peek() !== ')') {
         throw new Error('Expected closing parenthesis ")" in filter expression');
       }
       consume(); // )
+      depth -= 1;
       return node;
     }