md-semantic-search 1.0.3 → 2.0.0

package/dist/serve.js

@@ -10,10 +10,12 @@ import os from 'node:os';
 import path from 'node:path';
 import { buildIndex } from './indexer.js';
 import { loadIndex, searchIndex } from './search.js';
-import { walkMarkdown, assertSafePath } from './core.js';
+import { synthesizeAnswer, chatTurn, loadChatSession, assertSafeSessionId } from './rag.js';
+import { walkMarkdown, assertSafePath, getDocLines } from './core.js';
 import { createFileWatcher, classifyFsError, readWithRetry } from './watcher.js';
 import { WEBUI_HTML, WEBUI_JS, WEBUI_CSS } from './webui.js';
-import { handleMcpRequest } from './mcp.js';
+import { handleMcpRequest, createClientChannel } from './mcp.js';
+import { findRelatedNotes } from './wikilinks.js';
 const DEFAULT_PORT = 8747;
 const DEFAULT_HOST = '127.0.0.1';
 const WATCH_INTERVAL_MS = 3000;
@@ -178,7 +180,7 @@ export class ServeLimiter {
     }
 }
 export async function createServe(opts) {
-    const { indexDir, cacheDir, db, modelName = 'e5-base', ignore = [], offline = false, watch = false, watchInterval = WATCH_INTERVAL_MS, watchDelay = WATCH_DELAY_MS, apiKey = process.env.MDSS_API_KEY, healthPublic = process.env.MDSS_HEALTH_PUBLIC === 'true', allowedHosts, corsOrigins, rateLimit = DEFAULT_RATE_LIMIT_PER_MIN, maxConcurrency = DEFAULT_MAX_CONCURRENCY, watchDebug = false, _testFs, mcp = false, ui = true, embedFn, rerankFn, log = () => { }, } = opts;
+    const { indexDir, cacheDir, db, modelName = 'e5-base', ignore = [], offline = false, watch = false, watchInterval = WATCH_INTERVAL_MS, watchDelay = WATCH_DELAY_MS, apiKey = process.env.MDSS_API_KEY, healthPublic = process.env.MDSS_HEALTH_PUBLIC === 'true', allowedHosts, corsOrigins, rateLimit = DEFAULT_RATE_LIMIT_PER_MIN, maxConcurrency = DEFAULT_MAX_CONCURRENCY, watchDebug = false, _testFs, mcp = false, ui = true, maxChunkTokens, llmEndpoint, llmModel, llmProvider, systemPrompt, embedFn, rerankFn, log = () => { }, } = opts;
     if (indexDir)
         assertSafePath(indexDir);
     if (db)
@@ -188,7 +190,7 @@ export async function createServe(opts) {
         if (!db)
             throw new Error('serve: no index found and no --db given to build one');
         log(`No index at ${indexDir} — building it from ${db}…`);
-        await buildIndex({ db, indexDir, cacheDir, modelName, ignore, offline, log, embedFn });
+        await buildIndex({ db, indexDir, cacheDir, modelName, ignore, offline, log, embedFn, ...(maxChunkTokens !== undefined ? { maxChunkTokens } : {}) });
     }
     const state = {
         loaded: loadIndex(indexDir),
@@ -202,14 +204,48 @@ export async function createServe(opts) {
         healthPublic,
         ui: ui !== false,
         mcp,
+        ...(llmEndpoint ? { llmEndpoint } : {}),
+        ...(llmModel ? { llmModel } : {}),
+        ...(llmProvider ? { llmProvider } : {}),
+        ...(systemPrompt ? { systemPrompt } : {}),
         allowedHosts,
         corsOrigins,
         limiter: new ServeLimiter(rateLimit, DEFAULT_RATE_BURST, maxConcurrency),
+        sseConnections: 0,
     };
+    // issue #197: server→client requests (sampling) ride the GET /mcp SSE
+    // stream as `event: message` frames. With no stream open the send throws
+    // synchronously, the request rejects, and ask_knowledge_base degrades to
+    // its extractive answer — sampling stays strictly best-effort.
+    state.mcpChannel = createClientChannel((msg) => {
+        const sse = state.mcpSseRes;
+        if (!sse) {
+            throw new Error('sampling unavailable: client has no open GET /mcp stream');
+        }
+        sse.write(`event: message\ndata: ${JSON.stringify(msg)}\n\n`);
+    });
     const sockets = new Set();
     const server = http.createServer((req, res) => {
         handleRequest(req, res, state).catch((e) => {
-            json(res, 500, { error: e.message });
+            // issue #182: raw exception text can carry absolute filesystem paths —
+            // reply generically, log the details server-side
+            process.stderr.write(`serve: unhandled request error: ${e?.stack || e}\n`);
+            try {
+                if (res.headersSent) {
+                    res.destroy();
+                }
+                else {
+                    json(res, 500, { error: 'internal server error' });
+                }
+            }
+            catch {
+                try {
+                    res.destroy();
+                }
+                catch {
+                    /* socket already gone */
+                }
+            }
         });
     });
     server.on('connection', (socket) => {
@@ -224,7 +260,7 @@ export async function createServe(opts) {
             return false;
         log('Change detected — re-indexing…');
         try {
-            await buildIndex({ db, indexDir, cacheDir, modelName, ignore, offline, log, embedFn });
+            await buildIndex({ db, indexDir, cacheDir, modelName, ignore, offline, log, embedFn, ...(maxChunkTokens !== undefined ? { maxChunkTokens } : {}) });
         }
         catch (e) {
             if (/(being written by pid|being written|lock)/i.test(String(e?.message || ''))) {
@@ -239,12 +275,19 @@ export async function createServe(opts) {
         return true;
     };
     const fileMd5 = (f) => _testFs?.hash ? _testFs.hash(f) : crypto.createHash('md5').update(fs.readFileSync(f)).digest('hex');
-    const statFile = (f) => _testFs?.stat ? _testFs.stat(f) : { mtimeMs: fs.statSync(f).mtimeMs };
+    const statFile = (f) => {
+        if (_testFs?.stat)
+            return _testFs.stat(f);
+        const st = fs.statSync(f);
+        return { mtimeMs: st.mtimeMs, size: st.size };
+    };
     const relOf = (f) => path.relative(db, f).split(path.sep).join('/');
     const dbg = (msg) => {
         if (watchDebug)
             log(`watch-debug: ${msg}`);
     };
+    // issue #195: cache identity is size+mtimeMs — a file restored from a
+    // backup with an OLD mtime but new content no longer masks its change
     const md5Cache = new Map();
     // issue #116: files whose hash currently fails (AV/sync contention) — they
     // are EXCLUDED from fingerprints instead of becoming stable 'unreadable'
@@ -257,9 +300,9 @@ export async function createServe(opts) {
      * Returns null when the file must be excluded this cycle: vanished (deleted),
      * or still unreadable after retries (changed-pending, warned rate-limited).
      */
-    const hashWithRetry = async (f, m) => {
+    const hashWithRetry = async (f, st) => {
         const hit = md5Cache.get(f);
-        if (hit && hit.mtime === m) {
+        if (hit && hit.mtime === st.mtimeMs && hit.size === st.size) {
             unreadable.delete(f);
             return hit.md5;
         }
@@ -269,7 +312,7 @@ export async function createServe(opts) {
             onRetry: (attempt, err) => dbg(`retry ${attempt} ${path.basename(f)}: ${err?.code ?? err?.message}`),
         });
         if (r.ok) {
-            md5Cache.set(f, { mtime: m, md5: r.value });
+            md5Cache.set(f, { mtime: st.mtimeMs, size: st.size, md5: r.value });
             unreadable.delete(f);
             dbg(`hashed ${path.basename(f)}`);
             return r.value;
@@ -293,19 +336,42 @@ export async function createServe(opts) {
      */
     const probeTree = async (tree) => {
         const hashes = new Map();
-        for (const [f, { m }] of tree) {
-            hashes.set(f, await hashWithRetry(f, m));
+        for (const [f, { m, s }] of tree) {
+            hashes.set(f, await hashWithRetry(f, { mtimeMs: m, size: s }));
         }
         return hashes;
     };
-    const fingerprintOf = (tree, fileHashes) => {
-        const parts = [];
+    /**
+     * issue #195: tree state as a rel→hash map — compared map-to-map with no
+     * O(n log n) string build + sort per poll. null means "unknown" (initial
+     * load or a native event forced a re-probe).
+     */
+    const stateOf = (tree, fileHashes) => {
+        const out = new Map();
         for (const [f, { rel }] of tree) {
             const h = fileHashes.get(f);
             if (h !== null && h !== undefined)
-                parts.push(`${rel}:${h}`);
+                out.set(rel, h);
+        }
+        return out;
+    };
+    const statesEqual = (a, b) => {
+        if (!a || a.size !== b.size)
+            return false;
+        for (const [rel, hash] of b) {
+            if (a.get(rel) !== hash)
+                return false;
+        }
+        return true;
+    };
+    /** issue #195: evict cache entries for files that left the tree. */
+    const evictStaleCacheEntries = (tree) => {
+        if (md5Cache.size <= tree.size)
+            return;
+        for (const f of md5Cache.keys()) {
+            if (!tree.has(f))
+                md5Cache.delete(f);
         }
-        return parts.sort().join('|');
     };
     const readIndexedHashes = () => {
         try {
@@ -323,7 +389,8 @@ export async function createServe(opts) {
             return { tree, problems };
         for (const f of walkMarkdown(db, ignore)) {
             try {
-                tree.set(f, { m: statFile(f).mtimeMs, rel: relOf(f) });
+                const st = statFile(f);
+                tree.set(f, { m: st.mtimeMs, s: st.size, rel: relOf(f) });
             }
             catch (e) {
                 const cls = classifyFsError(e);
@@ -355,7 +422,9 @@ export async function createServe(opts) {
         }
         return out;
     };
-    let lastFingerprint = '';
+    // issue #195: null = "state unknown" (startup or a native event forced a
+    // re-probe); otherwise the rel→hash map of the last observed tree
+    let lastState = null;
     /** issue #116: one combined warning per cooldown, ≤5 files named + "+k more". */
     const warnUnreadableRateLimited = (now) => {
         if (unreadable.size === 0 || now - lastWarnAt < WARN_COOLDOWN_MS)
@@ -372,7 +441,7 @@ export async function createServe(opts) {
         {
             const { tree } = scanTree();
             const hashes = await probeTree(tree);
-            lastFingerprint = fingerprintOf(tree, hashes);
+            lastState = stateOf(tree, hashes);
         }
         let lastChangeAt = 0;
         let pending = false;
@@ -394,16 +463,17 @@ export async function createServe(opts) {
                 if (!unreadable.has(f))
                     unreadable.set(f, { cls, code: `stat:${cls}`, firstAt: t0 });
             }
+            evictStaleCacheEntries(tree);
             dbg(`poll: ${tree.size} file(s), ${unreadable.size} pending-unreadable`);
             const fileHashes = await probeTree(tree);
             warnUnreadableRateLimited(Date.now());
-            const fingerprint = fingerprintOf(tree, fileHashes);
+            const currentState = stateOf(tree, fileHashes);
             const changed = contentChangedPaths(tree, fileHashes, indexedHashes);
-            if (fingerprint !== lastFingerprint) {
+            if (!statesEqual(lastState, currentState)) {
                 if (!pending)
                     log(`watch: change in ${changed.length} file(s) — waiting ${watchDelay} ms for the tree to settle…`);
                 pending = true;
-                lastFingerprint = fingerprint;
+                lastState = currentState;
                 lastChangeAt = Date.now();
             }
             else if (pending && changed.length > 0 && Date.now() - lastChangeAt >= watchDelay) {
@@ -414,16 +484,16 @@ export async function createServe(opts) {
                     const fresh = scanTree();
                     const freshHashes = await probeTree(fresh.tree);
                     // seed the md5 cache from the authoritative post-build state so the
-                    // next fingerprint is IO-free for unchanged files
-                    for (const [f, { m, rel }] of fresh.tree) {
+                    // next poll is IO-free for unchanged files
+                    for (const [f, { m, s }] of fresh.tree) {
                         const h = freshHashes.get(f);
                         if (h !== null && h !== undefined && !md5Cache.has(f))
-                            md5Cache.set(f, { mtime: m, md5: h });
+                            md5Cache.set(f, { mtime: m, size: s, md5: h });
                     }
-                    lastFingerprint = fingerprintOf(fresh.tree, freshHashes);
+                    lastState = stateOf(fresh.tree, freshHashes);
                 }
             }
-            else if (!pending && changed.length > 0 && fingerprint === lastFingerprint) {
+            else if (!pending && changed.length > 0 && statesEqual(lastState, currentState)) {
                 pending = true;
                 lastChangeAt = Date.now();
                 log(`watch: tree differs from index (no motion) — settling then re-index…`);
@@ -435,7 +505,7 @@ export async function createServe(opts) {
                 // settled but nothing actually differs (e.g. a locked file became
                 // readable again with unchanged content) — resync the baseline
                 pending = false;
-                lastFingerprint = fingerprint;
+                lastState = currentState;
             }
         }
     };
@@ -444,14 +514,35 @@ export async function createServe(opts) {
         if (!db)
             throw new Error('serve --watch requires --db');
         nativeWatcher = createFileWatcher(db, () => {
-            // Prompt watchLoop settle trigger on native event
-            lastFingerprint = '';
+            // Prompt watchLoop settle trigger on native event: state becomes
+            // "unknown", so the next poll re-probes and enters the settle path
+            lastState = null;
         }, {
             debounceMs: watchDelay,
             ignore,
             log,
         });
-        watchLoop();
+        // issue #186: watchLoop runs unawaited — an early rejection used to be an
+        // unhandled one and killed the daemon. Restart with backoff, give up
+        // after 5 crashes (search keeps serving the last good index either way).
+        void (async () => {
+            for (let attempt = 0;; attempt++) {
+                try {
+                    await watchLoop();
+                    return; // resolved → stopped normally
+                }
+                catch (e) {
+                    if (stopped || attempt >= 4) {
+                        log(`watch loop failed permanently (search still serves the last index): ${e.message}`);
+                        return;
+                    }
+                    log(`watch loop crashed (attempt ${attempt + 1}/5): ${e.message} — restarting in 5s…`);
+                    await new Promise((r) => setTimeout(r, 5_000));
+                    if (stopped)
+                        return;
+                }
+            }
+        })();
     }
     const close = async () => {
         stopped = true;
@@ -559,6 +650,52 @@ export function splitHostHeader(hostHeader) {
     // a single colon separates host:port; several colons mean a bare IPv6 literal
     return first !== -1 && first === last ? s.slice(0, first) : s;
 }
+/**
+ * Cross-site POST guard (issue #178). Browsers fire "simple" POSTs without
+ * preflight (Content-Type: text/plain, form encodings) from any open page;
+ * the loopback Host check alone does not stop them. A POST counts as
+ * same-site when:
+ *  - the client sent no Origin/Sec-Fetch headers (curl, MCP and other
+ *    non-browser clients), or
+ *  - Sec-Fetch-Site is not cross-site AND the Origin's host matches the
+ *    request Host, a loopback allowlist entry, or an explicit --cors-origin.
+ * `Origin: null` (sandboxed iframe, file:// page) is always rejected.
+ */
+export function isSameSitePost(req, allowedHosts, corsOrigins) {
+    const secFetchSite = req.headers['sec-fetch-site'];
+    if (typeof secFetchSite === 'string' && secFetchSite === 'cross-site')
+        return false;
+    const origin = req.headers.origin;
+    if (typeof origin !== 'string' || origin.length === 0)
+        return true; // non-browser client
+    if (origin === 'null')
+        return false;
+    if (corsOrigins && corsOrigins.includes(origin))
+        return true;
+    const hostHeader = typeof req.headers.host === 'string' ? req.headers.host.toLowerCase() : '';
+    if (!hostHeader)
+        return false;
+    let originUrl;
+    try {
+        originUrl = new URL(origin);
+    }
+    catch {
+        return false; // malformed Origin — fail closed
+    }
+    if (originUrl.host === hostHeader)
+        return true; // exact same-origin (host:port)
+    // loopback aliases pass only on the SAME port — a page on localhost:3000 is
+    // a different origin even though the bare hostname is allowlisted
+    const portOf = (h, proto) => {
+        const m = h.match(/:(\d+)$/);
+        if (m)
+            return m[1];
+        return proto === 'https:' || proto === 'wss:' ? '443' : '80';
+    };
+    if (portOf(originUrl.host, originUrl.protocol) !== portOf(hostHeader, 'http:'))
+        return false;
+    return allowedHosts.has(splitHostHeader(originUrl.host));
+}
 /**
  * Reflect CORS headers ONLY for an exact allowlisted Origin (issue #120).
  * Off entirely by default; `Vary: Origin` is set whenever CORS is configured so
@@ -604,17 +741,37 @@ async function handleRequest(req, res, state) {
         }
         return;
     }
-    if (state.apiKey) {
-        const isPublicHealth = url.pathname === '/health' && req.method === 'GET' && state.healthPublic;
-        if (!isPublicHealth) {
-            const authHeader = req.headers['authorization'];
-            const match = typeof authHeader === 'string' ? authHeader.match(/^Bearer\s+(.+)$/i) : null;
-            const token = match ? match[1].trim() : null;
-            if (!token || !isAuthorizedToken(token, state.apiKey)) {
-                json(res, 401, { error: 'unauthorized' });
+    // CSRF / body-safety gate for every body-parsing POST (issue #178): a "simple"
+    // cross-site browser POST (text/plain, form) sails past the Host allowlist on
+    // loopback, so reject foreign origins and non-JSON media types up front.
+    if (req.method === 'POST') {
+        const ct = req.headers['content-type'];
+        if (typeof ct === 'string') {
+            const media = ct.split(';')[0].trim().toLowerCase();
+            if (media !== 'application/json') {
+                json(res, 415, { error: `unsupported content-type "${media}" — POST bodies must be application/json` });
                 return;
             }
         }
+        if (!isSameSitePost(req, allowedHosts, state.corsOrigins)) {
+            json(res, 403, { error: 'cross-site request rejected' });
+            return;
+        }
+    }
+    // issue #181: /health path disclosure depends on whether THIS request is
+    // authenticated — an unauthenticated daemon or public health hides indexDir
+    let requestAuthenticated = false;
+    if (state.apiKey) {
+        const authHeader = req.headers['authorization'];
+        const match = typeof authHeader === 'string' ? authHeader.match(/^Bearer\s+(.+)$/i) : null;
+        const token = match ? match[1].trim() : null;
+        const valid = !!token && isAuthorizedToken(token, state.apiKey);
+        const isPublicHealth = url.pathname === '/health' && req.method === 'GET' && state.healthPublic;
+        if (!valid && !isPublicHealth) {
+            json(res, 401, { error: 'unauthorized' });
+            return;
+        }
+        requestAuthenticated = valid;
     }
     // Streamable HTTP MCP transport (issue #123): same auth/host/rate-limit
     // gates as /search, reusing handleMcpRequest untouched as the tool brain.
@@ -689,6 +846,11 @@ async function handleRequest(req, res, state) {
                 offline: state.offline,
                 embedFn: state.embedFn,
                 rerankFn: state.rerankFn,
+                // issue #203: multi-query expansion (LLM bridge = the daemon's own
+                // --llm-endpoint, NEVER a payload-supplied URL — that would let any
+                // client exfiltrate note content to an arbitrary host) + reranker alias
+                ...(payload.multiQuery && state.llmEndpoint ? { multiQuery: payload.multiQuery, llmEndpoint: state.llmEndpoint, ...(state.llmModel ? { llmModel: state.llmModel } : {}) } : {}),
+                ...(payload.rerankModel ? { rerankModel: payload.rerankModel } : {}),
                 tag: payload.tag,
                 project: payload.project,
                 type: payload.type,
@@ -706,7 +868,215 @@ async function handleRequest(req, res, state) {
             json(res, 200, { query, k, count: results.length, results });
         }
         catch (e) {
-            json(res, 500, { error: e.message });
+            // issue #182: search errors can name absolute paths — keep the reply
+            // generic, log the details server-side
+            process.stderr.write(`serve: /search failed: ${e?.stack || e}\n`);
+            json(res, 500, { error: 'internal server error' });
+        }
+        finally {
+            release();
+        }
+        return;
+    }
+    // issue #199: grounded QA over SSE — a `sources` frame as soon as retrieval
+    // settles, streamed `delta` frames while the LLM answers, one `done` frame
+    // with the assembled result. Same fairness gates as /search (an LLM call
+    // holds a concurrency slot for its whole generation). Requires --llm-endpoint:
+    // over HTTP a missing chat backend is a 400, never a silent extractive degrade.
+    if (req.method === 'POST' && url.pathname === '/ask') {
+        const limited = state.limiter?.gate(req.socket.remoteAddress || 'unknown');
+        if (limited) {
+            res.setHeader('retry-after', String(limited.retryAfterSec));
+            json(res, 429, { error: 'rate limit exceeded — slow down', retryAfterSec: limited.retryAfterSec });
+            return;
+        }
+        if (!state.llmEndpoint) {
+            json(res, 400, { error: 'POST /ask requires an LLM chat endpoint — start mdss serve with --llm-endpoint' });
+            return;
+        }
+        const declared = req.headers['content-length'];
+        if (declared !== undefined) {
+            const n = Number(declared);
+            if (!Number.isInteger(n) || n < 0 || n > MAX_BODY_BYTES) {
+                json(res, 413, { error: `payload too large (limit ${MAX_BODY_BYTES} bytes)` });
+                return;
+            }
+        }
+        let body = '';
+        let tooLarge = false;
+        for await (const chunk of req) {
+            body += chunk;
+            if (body.length > MAX_BODY_BYTES) {
+                tooLarge = true;
+                break;
+            }
+        }
+        if (tooLarge) {
+            json(res, 413, { error: `payload too large (limit ${MAX_BODY_BYTES} bytes)` });
+            req.resume();
+            return;
+        }
+        let payload;
+        try {
+            payload = body ? JSON.parse(body) : {};
+        }
+        catch (e) {
+            json(res, 400, { error: `invalid JSON body: ${e.message}` });
+            return;
+        }
+        const query = typeof payload.query === 'string' ? payload.query.trim() : '';
+        if (!query) {
+            json(res, 400, { error: 'missing "query" string in JSON body' });
+            return;
+        }
+        if (query.length > 2048) {
+            json(res, 400, { error: 'query exceeds maximum length of 2048 characters' });
+            return;
+        }
+        const k = Number.isInteger(payload.k) && payload.k > 0 ? payload.k : 5;
+        // issue #199 (follow-up): chat mode. `{"sessionId": "..."}` continues a
+        // persistent session (same store as `mdss chat`); `{"chat": true}` starts
+        // a fresh one whose id comes back in the `session`/`done` frames for the
+        // next request. One SSE protocol for both.
+        const rawSessionId = typeof payload.sessionId === 'string' ? payload.sessionId.trim() : '';
+        const chatMode = !!rawSessionId || payload.chat === true;
+        if (rawSessionId) {
+            // issue #149 discipline: validate BEFORE any filesystem use; a bad id
+            // gets a plain 400 here rather than a thrown error mid-stream
+            try {
+                assertSafeSessionId(rawSessionId);
+            }
+            catch (e) {
+                json(res, 400, { error: `invalid sessionId: ${e.message}` });
+                return;
+            }
+        }
+        const release = state.limiter ? await state.limiter.acquireSlot() : null;
+        if (!release) {
+            res.setHeader('retry-after', '1');
+            json(res, 503, { error: 'server busy — search concurrency cap reached, queue full' });
+            return;
+        }
+        // Client disconnect must tear down the in-flight LLM call, not just stop
+        // writing frames — the slot releases in `finally` either way.
+        const clientAbort = new AbortController();
+        res.on('close', () => clientAbort.abort());
+        res.writeHead(200, {
+            'content-type': 'text/event-stream',
+            'cache-control': 'no-store',
+            'connection': 'keep-alive',
+        });
+        const sse = (event, data) => {
+            if (!res.writableEnded)
+                res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
+        };
+        const llmOpts = {
+            llmEndpoint: state.llmEndpoint,
+            ...(state.llmModel ? { llmModel: state.llmModel } : {}),
+            ...(state.llmProvider ? { llmProvider: state.llmProvider } : {}),
+            ...(state.systemPrompt ? { systemPrompt: state.systemPrompt } : {}),
+            onDelta: (d) => sse('delta', { text: d }),
+            signal: clientAbort.signal,
+        };
+        try {
+            if (chatMode) {
+                const session = loadChatSession(state.indexDir, rawSessionId || crypto.randomUUID().slice(0, 8));
+                sse('session', { id: session.id, turns: session.turns.length });
+                const turn = await chatTurn({
+                    session,
+                    query,
+                    loaded: state.loaded,
+                    indexDir: state.indexDir,
+                    cacheDir: state.cacheDir,
+                    k,
+                    offline: state.offline,
+                    embedFn: state.embedFn,
+                    ...llmOpts,
+                });
+                sse('done', {
+                    answer: turn.answer,
+                    mode: turn.mode,
+                    citations: turn.citations,
+                    citationsUsed: turn.citationsUsed,
+                    manifest: turn.manifest,
+                    sessionId: session.id,
+                    turnsCount: session.turns.length,
+                });
+            }
+            else {
+                const passages = await searchIndex({
+                    loaded: state.loaded,
+                    cacheDir: state.cacheDir,
+                    query,
+                    k,
+                    offline: state.offline,
+                    embedFn: state.embedFn,
+                });
+                sse('sources', {
+                    citations: passages.map((p) => ({ file: p.file, heading: p.heading, score: p.score })),
+                });
+                const result = await synthesizeAnswer(query, passages, llmOpts);
+                sse('done', {
+                    answer: result.answer,
+                    mode: result.mode,
+                    citations: result.citations,
+                    citationsUsed: result.citationsUsed,
+                });
+            }
+        }
+        catch (e) {
+            // issue #182 discipline: raw errors can carry absolute paths — the
+            // frame stays generic, details go to the server log
+            process.stderr.write(`serve: /ask failed: ${e?.stack || e}\n`);
+            sse('error', { error: 'answer synthesis failed' });
+        }
+        finally {
+            release();
+            res.end();
+        }
+        return;
+    }
+    if (req.method === 'GET' && url.pathname === '/doc') {
+        const file = url.searchParams.get('file');
+        if (!file) {
+            json(res, 400, { error: 'missing "file" query parameter' });
+            return;
+        }
+        // issue #181: /doc does synchronous disk IO per request — it goes through
+        // the same fairness gates as /search so a local storm cannot starve it
+        const limited = state.limiter?.gate(req.socket.remoteAddress || 'unknown');
+        if (limited) {
+            res.setHeader('retry-after', String(limited.retryAfterSec));
+            json(res, 429, { error: 'rate limit exceeded — slow down', retryAfterSec: limited.retryAfterSec });
+            return;
+        }
+        const release = state.limiter ? await state.limiter.acquireSlot() : null;
+        if (!release) {
+            res.setHeader('retry-after', '1');
+            json(res, 503, { error: 'server busy — concurrency cap reached, queue full' });
+            return;
+        }
+        const fromLineParam = url.searchParams.get('fromLine') || url.searchParams.get('from');
+        const maxLinesParam = url.searchParams.get('maxLines') || url.searchParams.get('lines') || url.searchParams.get('limit');
+        const fromLine = fromLineParam ? parseInt(fromLineParam, 10) : 1;
+        const maxLines = maxLinesParam ? parseInt(maxLinesParam, 10) : undefined;
+        try {
+            const docResult = getDocLines({
+                file,
+                fromLine: isNaN(fromLine) ? 1 : fromLine,
+                maxLines: maxLines && !isNaN(maxLines) ? maxLines : undefined,
+                dbDir: state.loaded.index.db,
+                chunks: state.loaded.index.chunks,
+            });
+            json(res, 200, docResult);
+        }
+        catch (e) {
+            if (e.message?.includes('not found')) {
+                json(res, 404, { error: e.message });
+            }
+            else {
+                json(res, 400, { error: e.message });
+            }
         }
         finally {
             release();
@@ -714,6 +1084,10 @@ async function handleRequest(req, res, state) {
         return;
     }
     if (req.method === 'GET' && url.pathname === '/health') {
+        // issue #181: an unauthenticated /health must not disclose filesystem
+        // locations — the path comes back only to an authenticated caller on a
+        // daemon that also requires auth for /health
+        const hideIndexDir = !state.apiKey || !requestAuthenticated;
         json(res, 200, {
             ok: true,
             chunks: state.loaded.index.chunks.length,
@@ -721,7 +1095,7 @@ async function handleRequest(req, res, state) {
             dim: state.loaded.index.dim || state.loaded.model.dim || 0,
             built: state.loaded.index.built || null,
             watching: state.watching,
-            indexDir: state.indexDir,
+            indexDir: hideIndexDir ? undefined : state.indexDir,
             // fairness counters (issue #119)
             in_flight: state.limiter?.inFlight ?? 0,
             queued: state.limiter?.queued ?? 0,
@@ -744,11 +1118,103 @@ async function handleRequest(req, res, state) {
         sendAsset(res, 'text/css; charset=utf-8', WEBUI_CSS);
         return;
     }
+    if (url.pathname === '/related' && (req.method === 'POST' || req.method === 'GET')) {
+        // issue #181: /related runs the (semantic) relatedness sweep — same rate
+        // gate as /search, before any body parsing
+        const limited = state.limiter?.gate(req.socket.remoteAddress || 'unknown');
+        if (limited) {
+            res.setHeader('retry-after', String(limited.retryAfterSec));
+            json(res, 429, { error: 'rate limit exceeded — slow down', retryAfterSec: limited.retryAfterSec });
+            return;
+        }
+        let target = '';
+        let k = 10;
+        let direction = 'both';
+        let semantic = true;
+        if (req.method === 'GET') {
+            target = url.searchParams.get('file') || url.searchParams.get('target') || url.searchParams.get('note') || '';
+            const kParam = url.searchParams.get('k');
+            if (kParam)
+                k = parseInt(kParam, 10) || 10;
+            const dirParam = url.searchParams.get('direction');
+            if (dirParam === 'outgoing' || dirParam === 'backlinks' || dirParam === 'both')
+                direction = dirParam;
+            if (url.searchParams.has('semantic'))
+                semantic = url.searchParams.get('semantic') !== 'false';
+        }
+        else {
+            // issue #181: an overflowing body is a 413, not a silently truncated parse
+            let body = '';
+            let tooLarge = false;
+            for await (const chunk of req) {
+                body += chunk;
+                if (body.length > MAX_BODY_BYTES) {
+                    tooLarge = true;
+                    break;
+                }
+            }
+            if (tooLarge) {
+                json(res, 413, { error: `payload too large (limit ${MAX_BODY_BYTES} bytes)` });
+                req.resume();
+                return;
+            }
+            try {
+                const payload = body ? JSON.parse(body) : {};
+                target = payload.file || payload.target || payload.note || '';
+                if (typeof payload.k === 'number' && payload.k > 0)
+                    k = payload.k;
+                if (payload.direction === 'outgoing' || payload.direction === 'backlinks' || payload.direction === 'both') {
+                    direction = payload.direction;
+                }
+                if (payload.semantic !== undefined)
+                    semantic = Boolean(payload.semantic);
+            }
+            catch (e) {
+                json(res, 400, { error: `invalid JSON body: ${e.message}` });
+                return;
+            }
+        }
+        if (!target) {
+            json(res, 400, { error: 'missing "file" or "target" parameter' });
+            return;
+        }
+        const release = state.limiter ? await state.limiter.acquireSlot() : null;
+        if (!release) {
+            res.setHeader('retry-after', '1');
+            json(res, 503, { error: 'server busy — concurrency cap reached, queue full' });
+            return;
+        }
+        try {
+            const relatedResult = findRelatedNotes({
+                loaded: state.loaded,
+                target,
+                k,
+                direction,
+                semantic,
+            });
+            json(res, 200, relatedResult);
+        }
+        catch (e) {
+            if (e.message?.includes('not found')) {
+                json(res, 404, { error: e.message });
+            }
+            else {
+                json(res, 400, { error: e.message });
+            }
+        }
+        finally {
+            release();
+        }
+        return;
+    }
     if (req.method === 'GET' && (url.pathname === '/' || url.pathname === '/help')) {
         json(res, 200, {
             name: 'mdss serve',
             endpoints: [
                 { method: 'POST', path: '/search', body: '{ "query": "...", "k": 6, "semanticOnly": false, "rerank": false }' },
+                { method: 'POST', path: '/ask', body: '{ "query": "...", "k": 5 } — SSE: sources/delta/done frames (needs --llm-endpoint, issue #199)' },
+                { method: 'GET', path: '/doc?file=...&fromLine=1&maxLines=50' },
+                { method: 'POST', path: '/related', body: '{ "file": "...", "k": 10, "semantic": true }' },
                 { method: 'GET', path: '/health' },
             ],
         });
@@ -772,12 +1238,23 @@ async function handleMcpHttp(req, res, state) {
     if (req.method === 'GET') {
         // Server-initiated channel: we have nothing to push today, so keep the
         // stream alive with comments until the client goes away.
+        // issue #181: each stream holds a socket + a heartbeat interval — cap the
+        // number of concurrent SSE connections at the fairness concurrency limit.
+        const sseCap = state.limiter?.concurrencyEnabled ? state.limiter.maxConcurrency : DEFAULT_MAX_CONCURRENCY;
+        if ((state.sseConnections ?? 0) >= sseCap) {
+            json(res, 503, { error: 'server busy — SSE connection cap reached' });
+            return;
+        }
+        state.sseConnections = (state.sseConnections ?? 0) + 1;
         res.writeHead(200, {
             'content-type': 'text/event-stream',
             'cache-control': 'no-store',
             'connection': 'keep-alive',
         });
         res.write(':ok\n\n');
+        // issue #197: this stream doubles as the server→client downlink for
+        // sampling/createMessage. One client realistically; newest stream wins.
+        state.mcpSseRes = res;
         const hb = setInterval(() => {
             try {
                 res.write(':hb\n\n');
@@ -787,14 +1264,34 @@ async function handleMcpHttp(req, res, state) {
             }
         }, 15_000);
         hb.unref?.();
-        req.on('close', () => clearInterval(hb));
+        // res 'close' fires on client abort mid-stream too — req may never close
+        // while the response stream is still open
+        res.on('close', () => {
+            clearInterval(hb);
+            if (state.mcpSseRes === res)
+                state.mcpSseRes = undefined;
+            state.sseConnections = Math.max(0, (state.sseConnections ?? 1) - 1);
+        });
         return;
     }
     if (req.method !== 'POST') {
         json(res, 405, { error: 'method not allowed' });
         return;
     }
+    // issue #182: session binding before any work — the server mints its own id
+    // and rejects mismatching client ids with 404 instead of adopting them
+    // (session fixation) or feeding them to res.setHeader (500 on bad headers)
+    const requested = req.headers['mcp-session-id'];
+    if (typeof requested === 'string' &&
+        requested &&
+        state.mcpSessionId &&
+        requested !== state.mcpSessionId) {
+        json(res, 404, { error: 'session not found — re-initialize' });
+        return;
+    }
     // Same fairness gate as /search — tools/call can trigger embedding work.
+    // (The gates run AFTER body parse below: responses to server→client
+    // requests must bypass them, see issue #197.)
     const limited = state.limiter?.gate(req.socket.remoteAddress || 'unknown');
     if (limited) {
         res.setHeader('retry-after', String(limited.retryAfterSec));
@@ -826,37 +1323,60 @@ async function handleMcpHttp(req, res, state) {
         }
         const batch = Array.isArray(messages);
         const items = batch ? messages : [messages];
+        // issue #197: answers to server→client requests (sampling) arrive as
+        // plain POSTs. Resolve them BEFORE the fairness gates — the tools/call
+        // waiting on this response may hold the only concurrency slot, so
+        // queueing it behind itself would self-deadlock at max_concurrency 1.
+        let sawResponseOnly = items.length > 0;
+        for (const msg of items) {
+            if (!msg || typeof msg !== 'object') {
+                continue;
+            }
+            const isResponse = msg.method === undefined && (msg.result !== undefined || msg.error !== undefined);
+            if (isResponse) {
+                state.mcpChannel?.handleResponse(msg);
+            }
+            else {
+                sawResponseOnly = false;
+            }
+        }
+        if (sawResponseOnly) {
+            res.writeHead(202);
+            res.end();
+            return;
+        }
         const responses = [];
         for (const msg of items) {
             if (!msg || typeof msg !== 'object')
                 continue;
+            if (msg.method === undefined)
+                continue; // response — correlated above
             const isNotification = msg.id === undefined || msg.id === null;
             const result = isNotification ? null : await handleMcpRequest(msg, {
                 loaded: state.loaded,
                 cacheDir: state.cacheDir,
                 offline: state.offline,
                 embedFn: state.embedFn,
+                channel: state.mcpChannel,
+                clientCapabilities: state.mcpClientCapabilities,
             });
             if (result)
                 responses.push(result);
             else if (!isNotification && msg.method === undefined)
                 continue;
+            // issue #197: remember the client's declared capabilities (sampling
+            // gates the ask_knowledge_base LLM path on later tools/call).
+            if (msg.method === 'initialize') {
+                state.mcpClientCapabilities = msg.params?.capabilities ?? {};
+            }
         }
-        // session id: mint on first contact, echo afterwards
-        const requested = req.headers['mcp-session-id'];
-        if (typeof requested === 'string' && requested)
-            state.mcpSessionId = requested;
+        // session id: server-minted only (issue #182 — see the 404 gate above);
+        // mint on first contact, echo ours afterwards. Protocol negotiation
+        // itself moved into the shared brain (issue #197) so stdio and HTTP
+        // speak identical version logic.
         if (!state.mcpSessionId)
             state.mcpSessionId = crypto.randomUUID();
         res.setHeader('mcp-session-id', state.mcpSessionId);
-        // protocol negotiation: honour a supported requested version, else latest
-        for (const resp of responses) {
-            if (resp?.result?.protocolVersion && msg0(items)?.params?.protocolVersion) {
-                const wanted = String(msg0(items).params.protocolVersion);
-                const SUPPORTED = ['2025-03-26', '2024-11-05'];
-                resp.result.protocolVersion = SUPPORTED.includes(wanted) ? wanted : SUPPORTED[0];
-            }
-        }
         if (responses.length === 0) {
             res.writeHead(202);
             res.end();
@@ -868,10 +1388,6 @@ async function handleMcpHttp(req, res, state) {
         release();
     }
 }
-/** First object of a (possibly single-message) payload — for init negotiation. */
-function msg0(items) {
-    return items.length > 0 ? items[0] : null;
-}
 function json(res, status, data) {
     res.writeHead(status, {
         'content-type': 'application/json; charset=utf-8',