md-semantic-search 1.1.0 → 2.0.0

package/dist/core.js

@@ -5,6 +5,7 @@
  */
 import fs from 'node:fs';
 import path from 'node:path';
+import os from 'node:os';
 import { resolveModel } from './models.js';
 import { chunkMarkdownStructural, PARSER_VERSION } from './markdown-parser.js';
 import { parseFrontmatter } from './frontmatter.js';
@@ -58,7 +59,7 @@ function readLock(lockPath) {
         return null;
     }
 }
-export function acquireIndexLock(indexDir) {
+export function acquireIndexLock(indexDir, _testHooks) {
     const lockPath = path.join(indexDir, LOCK_FILENAME);
     const payload = JSON.stringify({ pid: process.pid, since: new Date().toISOString() }) + '\n';
     for (let attempt = 0; attempt < 2; attempt++) {
@@ -86,6 +87,25 @@ export function acquireIndexLock(indexDir) {
             const dead = pid !== null && !pidAlive(pid);
             const staleMs = stat ? Date.now() - stat.mtimeMs : 0;
             if ((garbage || dead || staleMs > LOCK_STALE_MS) && stat) {
+                // issue #188: between the stat/read above and the unlink below another
+                // writer may reclaim the lock and write a fresh, live one. Re-read and
+                // unlink only when the owner identity (pid + since) still matches what
+                // was classified — never unlink a lock we did not classify.
+                _testHooks?.onClassified?.();
+                if (!fs.existsSync(lockPath))
+                    continue;
+                const current = readLock(lockPath);
+                const sameOwner = garbage
+                    ? current === null
+                    : current !== null && current.pid === pid && current.since === heldSince;
+                if (!sameOwner) {
+                    return {
+                        acquired: false,
+                        reason: 'lock changed while reclaiming — another writer got here first',
+                        pid: current?.pid ?? pid,
+                        heldSince: current?.since ?? heldSince,
+                    };
+                }
                 try {
                     fs.unlinkSync(lockPath);
                 }
@@ -148,14 +168,17 @@ export function resolveSessionOptions(model) {
     const intra = envIntra ? parseInt(envIntra, 10) : undefined;
     const inter = envInter ? parseInt(envInter, 10) : undefined;
     const opts = model?.sessionOptions || {};
-    const intraOpNumThreads = Number.isSafeInteger(intra) && intra && intra > 0 ? intra : opts.intraOpNumThreads;
+    // issue #194: ONNX wasm sessions default to a single intra-op thread — the
+    // build/query sweep leaves most of the CPU idle. Default to the physical
+    // core count (measured 1.2x on a 4c/8t laptop; hyperthreads add contention,
+    // not throughput), still overridable via MDSS_INTRA_OP_THREADS.
+    const defaultIntra = Math.max(1, Math.min(4, os.cpus().length >> 1));
+    const intraOpNumThreads = Number.isSafeInteger(intra) && intra && intra > 0
+        ? intra
+        : opts.intraOpNumThreads ?? defaultIntra;
     const interOpNumThreads = Number.isSafeInteger(inter) && inter && inter > 0 ? inter : opts.interOpNumThreads;
-    if (intraOpNumThreads === undefined && interOpNumThreads === undefined) {
-        return undefined;
-    }
     const result = {};
-    if (intraOpNumThreads !== undefined)
-        result.intraOpNumThreads = intraOpNumThreads;
+    result.intraOpNumThreads = intraOpNumThreads;
     if (interOpNumThreads !== undefined)
         result.interOpNumThreads = interOpNumThreads;
     return result;
@@ -170,6 +193,10 @@ export function getExtractorCacheKey(model, cacheDir, offline) {
 export function isNetworkError(err) {
     if (!err)
         return false;
+    // issue #184: outbound fetch timeouts abort with TimeoutError/AbortError —
+    // they must classify as network errors so retries and fallbacks engage
+    if (err.name === 'TimeoutError' || err.name === 'AbortError')
+        return true;
     const code = err.code || err.cause?.code;
     if (code && ['ENOTFOUND', 'ETIMEDOUT', 'ECONNRESET', 'ECONNREFUSED', 'EPIPE', 'EHOSTUNREACH', 'ENETUNREACH', 'FETCH_ERROR'].includes(code)) {
         return true;
@@ -184,10 +211,23 @@ export function isNetworkError(err) {
         msg.includes('etimedout') ||
         msg.includes('enotfound') ||
         msg.includes('network error') ||
+        msg.includes('aborted due to timeout') ||
         msg.includes('503 service unavailable') ||
         msg.includes('502 bad gateway') ||
         msg.includes('504 gateway timeout'));
 }
+/**
+ * Outbound fetch timeout (issue #184): undici's default ~300s lets one hung
+ * embedder/LLM endpoint hold a serve concurrency slot or stall `mdss index`
+ * for minutes. All outbound calls carry `AbortSignal.timeout(fetchTimeoutMs())`;
+ * override with `MDSS_FETCH_TIMEOUT_MS` (0/negative falls back to the default).
+ */
+export const DEFAULT_FETCH_TIMEOUT_MS = 30_000;
+export function fetchTimeoutMs(envValue) {
+    const raw = envValue ?? process.env.MDSS_FETCH_TIMEOUT_MS;
+    const n = Number(raw);
+    return Number.isFinite(n) && n > 0 ? n : DEFAULT_FETCH_TIMEOUT_MS;
+}
 export async function retryWithBackoff(fn, opts = {}) {
     const maxRetries = Number.isInteger(opts.maxRetries) && opts.maxRetries >= 0 ? opts.maxRetries : 3;
     const delays = opts.delays || [1000, 4000, 16000];
@@ -224,6 +264,46 @@ export async function getExtractor(model, cacheDir, offline = false, retryOpts =
     const key = getExtractorCacheKey(model, cacheDir, offline);
     if (_extractors.has(key))
         return _extractors.get(key);
+    // issue #200: adapters whose ONNX graph emits a pre-pooled
+    // `sentence_embedding` (EmbeddingGemma family) bypass the feature-extraction
+    // pipeline — it can only pool last_hidden_state/logits/token_embeddings and
+    // would crash on such graphs. The shim keeps the pipeline call signature so
+    // embed() is unchanged.
+    if (model.pooledOutput === true) {
+        const loadPooledModel = async () => {
+            const { AutoModel, AutoTokenizer, env } = await import('@huggingface/transformers');
+            if (cacheDir)
+                env.cacheDir = cacheDir;
+            env.allowRemoteModels = !offline;
+            const sessionOptions = resolveSessionOptions(model);
+            const modelOpts = {
+                revision,
+                dtype: model.dtype || 'q8',
+                progress_callback: emitDownloadEvent,
+                ...(sessionOptions ? { session_options: sessionOptions } : {}),
+            };
+            const tokenizer = await AutoTokenizer.from_pretrained(source, { revision });
+            const onnxModel = await AutoModel.from_pretrained(source, modelOpts);
+            const normalize = model.normalize !== false;
+            const ext = async (input) => {
+                const texts = Array.isArray(input) ? input : [input];
+                const modelInputs = await tokenizer(texts, { padding: true, truncation: true });
+                const outputs = await onnxModel(modelInputs);
+                let result = outputs.sentence_embedding;
+                if (!result) {
+                    throw new Error(`model "${model.id}" declared pooledOutput but produced no sentence_embedding output`);
+                }
+                if (normalize)
+                    result = result.normalize(2, -1);
+                return { tolist: () => result.tolist() };
+            };
+            _extractors.set(key, ext);
+            return ext;
+        };
+        if (offline)
+            return await loadPooledModel();
+        return await retryWithBackoff(loadPooledModel, retryOpts);
+    }
     const loadPipeline = async () => {
         const { pipeline, env } = await import('@huggingface/transformers');
         if (cacheDir)
@@ -380,7 +460,7 @@ export const SCHEMA_MIGRATIONS = {
     2: migrateToSchemaV2,
     3: migrateToSchemaV3,
 };
-export function walkMarkdown(dir, ignore = []) {
+export function walkMarkdown(dir, ignore = [], errors) {
     const out = [];
     const ignoreRe = ignore.map(globToRegExp);
     const walk = (cur) => {
@@ -388,7 +468,14 @@ export function walkMarkdown(dir, ignore = []) {
         try {
             entries = fs.readdirSync(cur, { withFileTypes: true });
         }
-        catch {
+        catch (e) {
+            // issue #189: an unreadable directory (permissions, OneDrive eviction)
+            // used to silently shrink the corpus; surface it when a collector is
+            // given, always at least as a stderr warning
+            const msg = `walkMarkdown: cannot read directory ${cur}: ${e?.message || e}`;
+            if (errors)
+                errors.push(msg);
+            process.stderr.write(`warning: ${msg}\n`);
             return;
         }
         for (const e of entries) {
@@ -490,17 +577,21 @@ export function extractTitle(frontmatter, body, relPath) {
     return path.basename(relPath).replace(/\.(md|markdown)$/i, '');
 }
 const DEFAULT_MAX_CHUNK = 1400;
-export function chunkMarkdown(body, maxChunk = DEFAULT_MAX_CHUNK) {
-    return chunkMarkdownStructural(body, maxChunk);
-}
-export function parseFile(absPath, dbDir, maxChunk, raw) {
+export function chunkMarkdown(body, maxChunk = DEFAULT_MAX_CHUNK, maxChunkTokens) {
+    // issue #202: a token budget (maxChunkTokens) takes precedence over the
+    // legacy char budget; without it the pre-#202 char behavior is unchanged
+    return chunkMarkdownStructural(body, maxChunkTokens && maxChunkTokens > 0 ? { maxChunkTokens } : maxChunk);
+}
+export function parseFile(absPath, dbDir, maxChunk, raw, 
+/** issue #202: token budget overrides the char budget when set. */
+maxChunkTokens) {
     const content = raw ?? fs.readFileSync(absPath, 'utf8');
     const { frontmatter, body } = splitFrontmatter(content);
     const meta = parseFrontmatter(frontmatter);
     const rel = path.relative(dbDir, absPath).split(path.sep).join('/');
     const title = meta.title || extractTitle(frontmatter, body, rel);
     const frontmatterLineCount = frontmatter ? frontmatter.split('\n').length + 2 : 0;
-    return chunkMarkdown(body, maxChunk).map((c) => {
+    return chunkMarkdown(body, maxChunk, maxChunkTokens).map((c) => {
         const heading = c.heading || title;
         return {
             file: rel,

package/dist/federation.d.ts

@@ -7,9 +7,24 @@ export interface VaultConfig {
 export interface FederatedSearchOptions extends Omit<SearchOptions, 'loaded' | 'cacheDir'> {
     vaults: Array<string | VaultConfig>;
     cacheDir?: string;
+    /**
+     * issue #179: when set, every vault path (and explicit indexDir) must resolve
+     * inside one of these roots or a traversal-guard error is thrown. Callers
+     * exposing federation to remote clients (MCP) must pin this to their trust
+     * domain; direct CLI use stays guarded only by MDSS_ROOT_GUARD.
+     */
+    allowedRoots?: string[];
 }
 export interface FederatedSearchResultHit extends SearchResultHit {
     vault: string;
     vaultPath: string;
 }
-export declare function searchFederated(opts: FederatedSearchOptions): Promise<FederatedSearchResultHit[]>;
+export interface FederatedSearchResult {
+    hits: FederatedSearchResultHit[];
+    /**
+     * issue #189: per-vault load/query failures surfaced to the caller — a
+     * broken vault must not silently look like "0 results".
+     */
+    vaultErrors: string[];
+}
+export declare function searchFederated(opts: FederatedSearchOptions): Promise<FederatedSearchResult>;

package/dist/federation.js

@@ -8,20 +8,23 @@
 import path from 'node:path';
 import fs from 'node:fs';
 import { loadIndex, searchIndex } from './search.js';
+import { assertSafePath } from './core.js';
 export async function searchFederated(opts) {
-    const { vaults, k = 10, cacheDir = '', ...searchOpts } = opts;
+    const { vaults, k = 10, cacheDir = '', allowedRoots, ...searchOpts } = opts;
     if (!vaults || vaults.length === 0) {
         throw new Error('Federated search requires at least one vault specified');
     }
     const normalizedVaults = vaults.map((v) => {
         if (typeof v === 'string') {
-            const absPath = path.resolve(v);
+            const absPath = assertSafePath(path.resolve(v), allowedRoots);
             const name = path.basename(absPath);
             return { name, path: absPath };
         }
-        const absPath = path.resolve(v.path);
+        const absPath = assertSafePath(path.resolve(v.path), allowedRoots);
         const name = v.name || path.basename(absPath);
-        return { name, path: absPath, indexDir: v.indexDir };
+        // indexDir is loaded from disk too — guard it with the same roots (#179)
+        const indexDir = v.indexDir ? assertSafePath(path.resolve(v.indexDir), allowedRoots) : undefined;
+        return { name, path: absPath, indexDir };
     });
     // Load and query each vault concurrently
     const perVaultResults = await Promise.all(normalizedVaults.map(async (vault) => {
@@ -39,7 +42,10 @@ export async function searchFederated(opts) {
         try {
             // Query vault index with a slightly larger pool to ensure healthy global fusion
             const poolK = Math.max(k * 2, 20);
-            const resolvedCache = cacheDir || path.join(vault.path, '.cache');
+            // issue #179: never invent a per-vault cache location — a falsy cacheDir
+            // disables the disk layer (searchIndex semantics), so query embeddings
+            // are only ever written to a cache dir the caller explicitly owns.
+            const resolvedCache = cacheDir;
             const hits = await searchIndex({
                 ...searchOpts,
                 cacheDir: resolvedCache,
@@ -52,14 +58,22 @@ export async function searchFederated(opts) {
             return { vault, hits: [], error: err.message };
         }
     }));
+    // issue #189: surface per-vault failures — a broken vault must not look
+    // like a vault with simply no matching notes
+    const vaultErrors = perVaultResults
+        .filter((r) => r.error !== null)
+        .map((r) => `vault "${r.vault.name || path.basename(r.vault.path)}" (${r.vault.path}): ${r.error}`);
     // If single vault, return directly with attribution
     if (normalizedVaults.length === 1) {
         const { vault, hits } = perVaultResults[0];
-        return hits.slice(0, k).map((h) => ({
-            ...h,
-            vault: vault.name || path.basename(vault.path),
-            vaultPath: vault.path,
-        }));
+        return {
+            hits: hits.slice(0, k).map((h) => ({
+                ...h,
+                vault: vault.name || path.basename(vault.path),
+                vaultPath: vault.path,
+            })),
+            vaultErrors,
+        };
     }
     // Multi-vault reciprocal rank fusion (RRF)
     // Collect all hits across vaults
@@ -85,10 +99,13 @@ export async function searchFederated(opts) {
         return b.hit.score - a.hit.score;
     })
         .slice(0, k);
-    return fused.map(({ hit, vault, rrfSum }) => ({
-        ...hit,
-        score: rrfSum,
-        vault: vault.name || path.basename(vault.path),
-        vaultPath: vault.path,
-    }));
+    return {
+        hits: fused.map(({ hit, vault, rrfSum }) => ({
+            ...hit,
+            score: rrfSum,
+            vault: vault.name || path.basename(vault.path),
+            vaultPath: vault.path,
+        })),
+        vaultErrors,
+    };
 }

package/dist/filter.d.ts

@@ -22,6 +22,13 @@ export type FilterNode = {
     value: string | number | boolean;
 };
 export declare function tokenizeFilter(expr: string): string[];
+/**
+ * issue #180: parseOr/parseAnd/parseNot/parsePrimary are mutually recursive
+ * (~4 stack frames per nesting level) — an unbounded `(((((…` from a 64 KiB
+ * request body would exhaust the call stack. 64 levels is far beyond any
+ * legitimate filter and keeps the parser fail-closed with a clear error.
+ */
+export declare const MAX_FILTER_DEPTH = 64;
 export declare function parseFilter(expr: string): FilterNode;
 export declare function evaluateFilter(node: FilterNode | string, meta?: DocumentMetadata, doc?: {
     file?: string;

package/dist/filter.js

@@ -72,12 +72,20 @@ export function tokenizeFilter(expr) {
     }
     return tokens;
 }
+/**
+ * issue #180: parseOr/parseAnd/parseNot/parsePrimary are mutually recursive
+ * (~4 stack frames per nesting level) — an unbounded `(((((…` from a 64 KiB
+ * request body would exhaust the call stack. 64 levels is far beyond any
+ * legitimate filter and keeps the parser fail-closed with a clear error.
+ */
+export const MAX_FILTER_DEPTH = 64;
 export function parseFilter(expr) {
     const tokens = tokenizeFilter(expr);
     if (tokens.length === 0) {
         throw new Error('Empty filter expression');
     }
     let pos = 0;
+    let depth = 0;
     function peek() {
         return tokens[pos];
     }
@@ -131,12 +139,17 @@ export function parseFilter(expr) {
             throw new Error(`Unexpected end of filter expression at position ${pos}`);
         }
         if (token === '(') {
+            depth += 1;
+            if (depth > MAX_FILTER_DEPTH) {
+                throw new Error(`Filter expression too deeply nested (max ${MAX_FILTER_DEPTH} levels of parentheses)`);
+            }
             consume(); // (
             const node = parseExpression();
             if (peek() !== ')') {
                 throw new Error('Expected closing parenthesis ")" in filter expression');
             }
             consume(); // )
+            depth -= 1;
             return node;
         }
         // Comparison term: <field> <op> <value> or <field>:<value>

package/dist/hnsw.d.ts

@@ -0,0 +1,96 @@
+/**
+ * Pure JS/TS HNSW approximate nearest neighbor index (issue #201).
+ *
+ * Hierarchical Navigable Small World graphs (Malkov & Yashunin 2016): no
+ * training phase, incremental inserts, ~99% recall with sublinear search —
+ * the counterpart of the k-means IVF in `ivf.ts`, sharing its contract:
+ * build → serialize → persist → search as a candidate pre-filter in front
+ * of the exact sweep. Cosine distance (1 − dot on normalized vectors) is
+ * computed through an injected distance function so int8 zero-copy vectors
+ * work unchanged. Deleted nodes are tombstones — searches skip them, which
+ * is what lets the incremental watch rebuild reuse a persisted graph.
+ */
+export interface HnswIndex {
+    dim: number;
+    /** Max links per node on layers > 0. */
+    M: number;
+    /** Random level seed — rebuilds from the same insertion order are identical. */
+    seed: number;
+    /** Node index of the graph entry (top layer). */
+    entryPoint: number;
+    /** Highest layer in the graph. */
+    maxLevel: number;
+    /** levels[i] = top layer of node i. */
+    levels: number[];
+    /** neighbors[i][l] = link list of node i on layer l (l ≤ levels[i]). */
+    neighbors: number[][][];
+    /** Tombstoned node indices — skipped by search, kept for graph connectivity. */
+    deleted: Set<number>;
+}
+export interface SerializedHnsw {
+    format: 'hnsw-v1';
+    dim: number;
+    M: number;
+    seed: number;
+    entryPoint: number;
+    maxLevel: number;
+    count: number;
+    levels: number[];
+    neighbors: number[][][];
+    deleted: number[];
+}
+export interface BuildHnswOptions {
+    M?: number;
+    seed?: number;
+    /**
+     * Distance between the query/node vector at `a` and the stored node `b` —
+     * LOWER is closer (cosine distance: `1 − dot(a, vectors[b])`). Injected so
+     * int8 zero-copy and norm-cached sweeps work unchanged.
+     */
+    distance: (a: Float32Array | Int8Array | number[], b: number) => number;
+    vectors: Array<Float32Array | Int8Array | number[]>;
+}
+export declare const DEFAULT_HNSW_M = 16;
+export declare const DEFAULT_HNSW_EF = 64;
+export declare function buildHnsw(vectors: Array<Float32Array | Int8Array | number[]>, options: BuildHnswOptions): HnswIndex;
+/**
+ * Insert node `idx` (vector index) into the graph. Incremental — new indices
+ * attach to the existing structure without a rebuild; vectors for nodes NOT
+ * yet inserted are read from `vectors` when needed, so the caller can insert
+ * lazily. Tombstoned indices can be re-inserted (undeletes).
+ */
+export declare function insertHnsw(hnsw: HnswIndex, idx: number, distance: (a: Float32Array | Int8Array | number[], b: number) => number, vectors: Array<Float32Array | Int8Array | number[]>): void;
+/** Tombstone a node — searches skip it; the graph keeps it for connectivity. */
+export declare function deleteHnsw(hnsw: HnswIndex, idx: number): void;
+/**
+ * Algorithm 4 of the HNSW paper (Malkov & Yashunin 2016): candidates sorted
+ * by distance to the query enter the result in order; a candidate is skipped
+ * when it is closer to some already-selected neighbor than to the query
+ * (redundant direction). `pairwise(a, b)` measures node-to-node distance.
+ * If fewer than `m` survive (disconnected candidate set), the closest
+ * leftovers backfill so the link count stays useful.
+ */
+export declare function selectNeighborsHeuristic(candidates: Array<{
+    node: number;
+    dist: number;
+}>, m: number, pairwise: (a: number, b: number) => number): Array<{
+    node: number;
+    dist: number;
+}>;
+/**
+ * ANN candidate pre-filter: ef-search from the entry point, returning LIVE
+ * candidate chunk indices (tombstones excluded). Mirrors
+ * `searchIVFCandidates` — the result feeds the same probed-candidate set.
+ */
+export declare function searchHnswCandidates(queryVec: Float32Array | Int8Array | number[], hnsw: HnswIndex, vectors: Array<Float32Array | Int8Array | number[]>, options?: {
+    ef?: number;
+    /** Lower-is-closer distance; defaults to cosine distance via `cosine`. */
+    distance?: (a: Float32Array | Int8Array | number[], b: number) => number;
+}): number[];
+export declare function serializeHnsw(hnsw: HnswIndex): SerializedHnsw;
+/**
+ * Hardened deserialization — every structural invariant is checked with the
+ * same discipline as `deserializeIVF`; any violation throws (the loader
+ * falls back to brute force).
+ */
+export declare function deserializeHnsw(raw: unknown, expectedChunksCount?: number): HnswIndex;