md-semantic-search 1.0.2 → 1.0.3

package/dist/search.js

@@ -11,6 +11,7 @@ import { buildRelationshipGraph, computePageRank, expandGraphNeighborhood } from
 import { evaluateFilter } from './filter.js';
 import { deserializeBinaryIndex } from './binary-format.js';
 import { dequantizeFromInt8 } from './quantization.js';
+import { buildDiskKey, diskQueryGet, diskQueryPut } from './query-cache-disk.js';
 export { tokenize } from './lexical.js';
 const chunkTokens = new WeakMap();
 export const _stats = { corpusTokenizedChars: 0 };
@@ -150,7 +151,100 @@ export function getRuntimeGraph(runtime) {
     runtime.pageRank = pageRank;
     return { graph, pageRank };
 }
+/**
+ * Issue #127 pure helpers — exported for deterministic unit tests.
+ */
+/** Frontmatter `created` → `updated` → file mtime; null when nothing parses. */
+export function resolvePassageAgeMs(meta, absFile) {
+    for (const v of [meta?.created, meta?.updated]) {
+        if (typeof v === 'string') {
+            const t = Date.parse(v);
+            if (Number.isFinite(t))
+                return t;
+        }
+        else if (typeof v === 'number' && Number.isFinite(v)) {
+            return v;
+        }
+    }
+    if (absFile) {
+        try {
+            return fs.statSync(absFile).mtimeMs;
+        }
+        catch {
+            /* file gone — treat as unknown age */
+        }
+    }
+    return null;
+}
+/** Multiplicative recency boost: 0.5^(ageDays / halfLifeDays); 1 when off/unknown/future. */
+export function recencyBoost(ageMs, halfLifeDays, now = Date.now()) {
+    if (ageMs === null || !(halfLifeDays > 0))
+        return 1;
+    const ageDays = Math.max(0, (now - ageMs) / 86_400_000);
+    return Math.pow(0.5, ageDays / halfLifeDays);
+}
+/**
+ * HyDE passage generation via an OpenAI-compatible chat endpoint (issue #125).
+ * Returns null on ANY failure — HyDE must degrade to PRF/baseline silently.
+ * Injectable fetch for tests.
+ */
+export async function hydePassage(endpoint, query, model, fetchImpl = globalThis.fetch) {
+    try {
+        const res = await fetchImpl(endpoint.replace(/\/$/, '') + '/chat/completions', {
+            method: 'POST',
+            headers: { 'content-type': 'application/json' },
+            body: JSON.stringify({
+                ...(model ? { model } : {}),
+                messages: [
+                    { role: 'system', content: 'Write one short passage (2-4 sentences) that would plausibly contain the answer. Output only the passage.' },
+                    { role: 'user', content: query },
+                ],
+            }),
+        });
+        if (!res.ok)
+            return null;
+        const data = await res.json();
+        const text = data?.choices?.[0]?.message?.content ?? data?.response ?? null;
+        return typeof text === 'string' && text.trim() ? text.trim() : null;
+    }
+    catch {
+        return null;
+    }
+}
 const runtimeIndexStates = new WeakMap();
+/** Minimal EN+RU stopword set for PRF expansion (issue #125). */
+const EXPANSION_STOPWORDS = new Set([
+    'the', 'a', 'an', 'and', 'or', 'of', 'to', 'in', 'on', 'for', 'with', 'is', 'are',
+    'be', 'this', 'that', 'it', 'as', 'at', 'by', 'from', 'not', 'but', 'we', 'you',
+    'и', 'в', 'на', 'с', 'по', 'для', 'как', 'это', 'не', 'а', 'но', 'или', 'из',
+]);
+/**
+ * Salient-term extraction for PRF (issue #125): tokens that appear in the
+ * top-m feedback passages, are not query terms or stopwords, and are ranked
+ * by (tf-in-feedback × distinct-passage coverage). Pure + deterministic.
+ */
+export function extractExpansionTerms(topTexts, queryTerms, maxTerms) {
+    const qSet = new Set(queryTerms);
+    const tf = new Map();
+    const df = new Map();
+    for (const text of topTexts) {
+        const seen = new Set();
+        for (const tok of tokenize(text)) {
+            if (tok.length < 3 || qSet.has(tok) || EXPANSION_STOPWORDS.has(tok))
+                continue;
+            tf.set(tok, (tf.get(tok) ?? 0) + 1);
+            seen.add(tok);
+        }
+        for (const tok of seen)
+            df.set(tok, (df.get(tok) ?? 0) + 1);
+    }
+    return [...tf.entries()]
+        .filter(([, count]) => count >= 2 || topTexts.length === 1)
+        .map(([tok, count]) => [tok, count * Math.log(2 + (df.get(tok) ?? 0))])
+        .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
+        .slice(0, maxTerms)
+        .map(([tok]) => tok);
+}
 function snapshotRuntimeIndex(index, validation) {
     const dim = validation.dim || (index.chunks[0]?.vec?.length ?? 0);
     const count = index.chunks.length;
@@ -470,20 +564,62 @@ export async function searchIndex(opts) {
             return true;
         });
     }
+    // Issue #125: offline pseudo-relevance feedback (PRF) query expansion.
+    // A lexical-only first pass picks the top-m passages; salient terms from
+    // them enrich the query, which is then embedded ONCE (replacing the raw
+    // query embed — zero extra model calls). HyDE would swap the enriched text
+    // for an LLM-generated passage; without a configured endpoint it degrades
+    // to the baseline silently.
+    let effectiveQuery = query;
+    let expandedTerms = null;
+    if ((opts.expand === 'prf' || opts.expand === 'hyde') && candidates.length > 0) {
+        const qt0 = [...new Set(tokenize(query))];
+        if (qt0.length > 0 && lexicalState.kind === 'persisted-bm25') {
+            const firstPass = [
+                ...bm25Scores(lexicalState.lexical, qt0, new Set(candidates.map((c) => c.idx))).entries(),
+            ]
+                .sort((a, b) => b[1] - a[1])
+                .slice(0, Math.max(1, opts.expandPassages ?? 3));
+            const topTexts = firstPass.map(([idx]) => chunks[idx].text);
+            expandedTerms = extractExpansionTerms(topTexts, qt0, 8);
+            if (opts.expand === 'hyde' && opts.llmEndpoint) {
+                const hydeText = await hydePassage(opts.llmEndpoint, query, opts.llmModel);
+                if (hydeText)
+                    effectiveQuery = hydeText;
+                else if (expandedTerms.length > 0)
+                    effectiveQuery = `${query} ${expandedTerms.join(' ')}`;
+            }
+            else if (expandedTerms.length > 0) {
+                effectiveQuery = `${query} ${expandedTerms.join(' ')}`;
+            }
+        }
+    }
     let qVec;
     if (opts.useQueryCache !== false && runtime.queryCache) {
-        const cacheKey = `${runtime.model.id}:${runtime.model.revision || 'main'}:${expectedDim || 0}:${query.trim().toLowerCase()}`;
+        const normQuery = effectiveQuery.trim().toLowerCase();
+        const cacheKey = `${runtime.model.id}:${runtime.model.revision || 'main'}:${expectedDim || 0}:${normQuery}`;
+        // L2 disk layer (issue #114): persists across processes under cacheDir;
+        // corrupt/missing files degrade silently to a normal embed.
+        const diskKey = buildDiskKey(runtime.model.id, runtime.model.revision || 'main', expectedDim, normQuery);
+        const useDisk = opts.queryDiskCache !== false;
         qVec = await runtime.queryCache.getOrCompute(cacheKey, async () => {
-            const [v] = await embedFn([query], 'query', runtime.model, cacheDir, offline);
+            if (useDisk) {
+                const fromDisk = diskQueryGet(cacheDir, diskKey, expectedDim);
+                if (fromDisk)
+                    return fromDisk;
+            }
+            const [v] = await embedFn([effectiveQuery], 'query', runtime.model, cacheDir, offline);
             if ((Array.isArray(v) || v instanceof Float32Array) && expectedDim !== undefined && v.length !== expectedDim) {
-                throw new Error(`query vector has ${v.length} dims, expected ${expectedDim} — run \`mdss index\` to rebuild`);
+                throw new Error(`query vector has ${v.length} dims, expected ${expectedDim} - run \`mdss index\` to rebuild`);
             }
             validateNumericVector(v, undefined, 'query vector');
+            if (useDisk)
+                diskQueryPut(cacheDir, diskKey, v);
             return v;
         });
     }
     else {
-        const [v] = await embedFn([query], 'query', runtime.model, cacheDir, offline);
+        const [v] = await embedFn([effectiveQuery], 'query', runtime.model, cacheDir, offline);
         if ((Array.isArray(v) || v instanceof Float32Array) && expectedDim !== undefined && v.length !== expectedDim) {
             throw new Error(`query vector has ${v.length} dims, expected ${expectedDim} — run \`mdss index\` to rebuild`);
         }
@@ -580,6 +716,28 @@ export async function searchIndex(opts) {
                 .slice(0, pool);
         }
     }
+    // Issue #127: time-decay recency boost — multiply the fused score by
+    // 0.5^(ageDays / halfLifeDays), post-RRF and pre-collapse. Age source
+    // priority: frontmatter `created` → `updated` → file mtime. Missing dates
+    // get factor 1 (evergreen content stays reachable via lexical strength).
+    const recencyHalfLife = Number(opts.recency) > 0 ? Number(opts.recency) : 0;
+    if (recencyHalfLife > 0) {
+        const now = Date.now();
+        const ageCache = new Map();
+        for (const r of ranked) {
+            const c = chunks[r.idx];
+            const abs = runtime.db && !path.isAbsolute(c.file) ? path.join(runtime.db, c.file) : c.file;
+            const ageKey = `${c.file}|${c.meta?.updated ?? ''}|${c.meta?.created ?? ''}`;
+            if (!ageCache.has(ageKey))
+                ageCache.set(ageKey, resolvePassageAgeMs(c.meta, abs));
+            const ageMs = ageCache.get(ageKey) ?? null;
+            r.recencyFactor = recencyBoost(ageMs, recencyHalfLife, now);
+            r.recencyAgeDays = ageMs === null ? null : Math.max(0, (now - ageMs) / 86_400_000);
+            r.rawFscore = r.fscore;
+            r.fscore = r.fscore * r.recencyFactor;
+        }
+        ranked.sort((a, b) => b.fscore - a.fscore);
+    }
     if (rerank && ranked.length > 0) {
         const texts = ranked.map((r) => chunks[r.idx].text);
         const scores = await rerankFn(query, texts, cacheDir, offline);
@@ -621,10 +779,19 @@ export async function searchIndex(opts) {
                 ? {
                     explain: {
                         cosine: +r.cos.toFixed(4),
-                        rrfScore: +r.fscore.toFixed(4),
+                        rrfScore: +(r.rawFscore ?? r.fscore).toFixed(4),
                         bm25Weights: { title: 3.0, aliases: 3.0, headingPath: 1.8, body: 1.0 },
                         ...(gScore !== undefined ? { graphScore: +gScore.toFixed(4) } : {}),
                         ...(prScore !== undefined ? { pageRank: +prScore.toFixed(4) } : {}),
+                        ...(r.recencyFactor !== undefined
+                            ? {
+                                recencyFactor: +r.recencyFactor.toFixed(4),
+                                recencyAgeDays: r.recencyAgeDays === null || r.recencyAgeDays === undefined
+                                    ? null
+                                    : +r.recencyAgeDays.toFixed(2),
+                            }
+                            : {}),
+                        ...(expandedTerms ? { expandedTerms } : {}),
                     },
                 }
                 : {}),

package/dist/serve.d.ts

@@ -9,6 +9,12 @@ declare const DEFAULT_PORT = 8747;
 declare const DEFAULT_HOST = "127.0.0.1";
 declare const WATCH_INTERVAL_MS = 3000;
 declare const MAX_BODY_BYTES: number;
+/** Hosts a browser is allowed to name in the Host header (issue #120). */
+export declare const DEFAULT_ALLOWED_HOSTS: string[];
+/** Fairness defaults (issue #119): protective yet friction-free locally. */
+export declare const DEFAULT_RATE_LIMIT_PER_MIN = 60;
+export declare const DEFAULT_RATE_BURST = 10;
+export declare const DEFAULT_MAX_CONCURRENCY: number;
 export interface ServeState {
     loaded: {
         index: any;
@@ -23,6 +29,17 @@ export interface ServeState {
     reindexCount?: number;
     apiKey?: string;
     healthPublic?: boolean;
+    /** issue #111: serve the built-in web UI at `/` (default on). */
+    ui?: boolean;
+    /** issue #123: mount the Streamable HTTP MCP transport at /mcp (default off). */
+    mcp?: boolean;
+    mcpSessionId?: string;
+    /** Extra Host-header values accepted beyond the loopback defaults (issue #120). */
+    allowedHosts?: string[];
+    /** Exact origins reflected as `Access-Control-Allow-Origin`; absent → CORS off (issue #120). */
+    corsOrigins?: string[];
+    /** Fairness gates for /search (issue #119); `0` disables a gate. */
+    limiter?: ServeLimiter;
 }
 export interface CreateServeOptions {
     indexDir: string;
@@ -36,14 +53,123 @@ export interface CreateServeOptions {
     watchDelay?: number;
     apiKey?: string;
     healthPublic?: boolean;
+    allowedHosts?: string[];
+    corsOrigins?: string[];
+    rateLimit?: number;
+    maxConcurrency?: number;
+    /** issue #116: verbose watch-loop trace (polls, retries, classifications). */
+    watchDebug?: boolean;
+    /** issue #116 test seam: override hashing/stat so fault-injection needs no real locks. */
+    _testFs?: {
+        hash?: (absPath: string) => string;
+        stat?: (absPath: string) => {
+            mtimeMs: number;
+        };
+    };
+    ui?: boolean;
+    /** issue #123: mount Streamable HTTP MCP at /mcp (default off; stdio stays default). */
+    mcp?: boolean;
     embedFn?: any;
     rerankFn?: any;
     log?: (msg: string) => void;
 }
+/**
+ * Token bucket for per-client request budgeting (issue #119): starts full
+ * (`burst`), refills continuously at `limitPerMinute`/60 tokens per second.
+ * `take` is injective in time so tests can time-travel via the `now` argument.
+ */
+export declare class TokenBucket {
+    readonly limitPerMinute: number;
+    readonly burst: number;
+    private tokens;
+    /** Last refill timestamp — doubles as a staleness marker for map pruning. */
+    lastTouched: number;
+    constructor(limitPerMinute: number, burst: number);
+    take(now?: number): {
+        ok: true;
+    } | {
+        ok: false;
+        retryAfterSec: number;
+    };
+}
+/** A granted slot's release handle — idempotent, so `finally` is always safe. */
+type SlotRelease = () => void;
+/**
+ * Counting semaphore with a bounded wait queue (issue #119): once `maxQueued`
+ * callers already wait, acquire() returns null instead of enqueueing, so an
+ * overloaded daemon sheds load with 503 rather than ballooning memory.
+ */
+export declare class BoundedSemaphore {
+    readonly size: number;
+    readonly maxQueued: number;
+    private available;
+    private readonly waiters;
+    constructor(size: number, maxQueued: number);
+    get executing(): number;
+    get queued(): number;
+    acquire(): Promise<SlotRelease | null>;
+    private releaseSlot;
+}
+/**
+ * Fairness gates for POST /search (issue #119):
+ *  - per-client token bucket → 429 + Retry-After when a client exceeds its budget;
+ *  - global concurrency cap around the embedding sweep; queue overflow → 503.
+ * Either gate disabled by passing `0`; counters are exposed via /health.
+ */
+export declare class ServeLimiter {
+    readonly rateLimitPerMin: number;
+    readonly rateBurst: number;
+    readonly maxConcurrency: number;
+    private readonly buckets;
+    private readonly sem;
+    rejected429: number;
+    rejected503: number;
+    constructor(rateLimitPerMin: number, rateBurst: number, maxConcurrency: number);
+    /** Burst is capped by the per-minute budget itself (--rate-limit 5 → burst 5). */
+    private effectiveBurst;
+    get rateEnabled(): boolean;
+    get concurrencyEnabled(): boolean;
+    get inFlight(): number;
+    get queued(): number;
+    get rejectedTotal(): number;
+    /** Cheap synchronous rate check. Null → proceed; otherwise reject with 429. */
+    gate(clientKey: string, now?: number): {
+        status: 429;
+        retryAfterSec: number;
+    } | null;
+    /**
+     * Acquire a global search slot. Resolves with a release handle, or null when
+     * the wait queue is saturated (caller replies 503).
+     */
+    acquireSlot(): Promise<SlotRelease | null>;
+}
 export declare function createServe(opts: CreateServeOptions): Promise<{
     server: http.Server;
     state: ServeState;
     close: () => Promise<void>;
 }>;
 export declare function isAuthorizedToken(token?: string | null, expectedKey?: string | null): boolean;
+/** True when `host` binds to loopback only (issue #121). Empty/undefined means the loopback default. */
+export declare function isLoopbackHost(host?: string | null): boolean;
+/**
+ * Refuse to start an unauthenticated server on a non-loopback interface (issue #121):
+ * `mdss serve --host 0.0.0.0` without an API key silently serves the whole knowledge
+ * base to the LAN. Explicit `--allow-unsecured` opts in (banner still printed by CLI).
+ */
+export declare function validateBindSecurity(opts: {
+    host?: string;
+    hasApiKey: boolean;
+    allowUnsecured?: boolean;
+}): void;
+/**
+ * Read an API key from a file (issue #121): avoids shell-history / CI-log leaks of
+ * literal `--api-key` values. Trailing newlines are trimmed; on POSIX a group- or
+ * world-readable key file draws a loud warning.
+ */
+export declare function loadApiKeyFile(filePath: string, log?: (msg: string) => void): string;
+/**
+ * Strip the port from a Host header value, keeping IPv6 bracket forms intact:
+ * "localhost:8747" → "localhost", "[::1]:8747" → "[::1]", "::1" → "::1" (issue #120).
+ */
+export declare function splitHostHeader(hostHeader: string): string;
 export { DEFAULT_PORT, DEFAULT_HOST, MAX_BODY_BYTES, WATCH_INTERVAL_MS };