md-semantic-search 1.0.3 → 2.0.0

package/src/serve.ts

@@ -10,10 +10,13 @@ import os from 'node:os';
 import path from 'node:path';
 import { buildIndex } from './indexer.js';
 import { loadIndex, searchIndex } from './search.js';
-import { walkMarkdown, assertSafePath, ModelDescriptor } from './core.js';
+import { synthesizeAnswer, chatTurn, loadChatSession, assertSafeSessionId } from './rag.js';
+import { walkMarkdown, assertSafePath, getDocLines, ModelDescriptor } from './core.js';
 import { createFileWatcher, FileWatcher, classifyFsError, readWithRetry } from './watcher.js';
 import { WEBUI_HTML, WEBUI_JS, WEBUI_CSS } from './webui.js';
-import { handleMcpRequest } from './mcp.js';
+import { handleMcpRequest, createClientChannel } from './mcp.js';
+import { findRelatedNotes } from './wikilinks.js';
+import { ChatProviderId } from './providers.js';
 
 const DEFAULT_PORT = 8747;
 const DEFAULT_HOST = '127.0.0.1';
@@ -40,11 +43,24 @@ export interface ServeState {
   reindexCount?: number;
   apiKey?: string;
   healthPublic?: boolean;
+  /** issue #181: live count of open SSE /mcp streams (capped by maxConcurrency). */
+  sseConnections?: number;
   /** issue #111: serve the built-in web UI at `/` (default on). */
   ui?: boolean;
+  /** issue #199: chat endpoint backing POST /ask — absent → /ask returns 400. */
+  llmEndpoint?: string;
+  llmModel?: string;
+  llmProvider?: ChatProviderId;
+  systemPrompt?: string;
   /** issue #123: mount the Streamable HTTP MCP transport at /mcp (default off). */
   mcp?: boolean;
   mcpSessionId?: string;
+  /** issue #197: reverse channel for server→client requests (MCP sampling). */
+  mcpChannel?: ReturnType<typeof createClientChannel>;
+  /** issue #197: capabilities the client declared at initialize. */
+  mcpClientCapabilities?: any;
+  /** issue #197: the open GET /mcp SSE stream doubling as the downlink (newest wins). */
+  mcpSseRes?: http.ServerResponse;
   /** Extra Host-header values accepted beyond the loopback defaults (issue #120). */
   allowedHosts?: string[];
   /** Exact origins reflected as `Access-Control-Allow-Origin`; absent → CORS off (issue #120). */
@@ -74,11 +90,18 @@ export interface CreateServeOptions {
   /** issue #116 test seam: override hashing/stat so fault-injection needs no real locks. */
   _testFs?: {
     hash?: (absPath: string) => string;
-    stat?: (absPath: string) => { mtimeMs: number };
+    stat?: (absPath: string) => { mtimeMs: number; size?: number };
   };
   ui?: boolean;
   /** issue #123: mount Streamable HTTP MCP at /mcp (default off; stdio stays default). */
   mcp?: boolean;
+  /** issue #202: chunk budget in tokens; default min(adapter.maxTokens, 512), 0 = legacy chars. */
+  maxChunkTokens?: number;
+  /** issue #199: chat endpoint for POST /ask SSE answer synthesis. */
+  llmEndpoint?: string;
+  llmModel?: string;
+  llmProvider?: ChatProviderId;
+  systemPrompt?: string;
   embedFn?: any;
   rerankFn?: any;
   log?: (msg: string) => void;
@@ -266,6 +289,11 @@ export async function createServe(opts: CreateServeOptions): Promise<{
     _testFs,
     mcp = false,
     ui = true,
+    maxChunkTokens,
+    llmEndpoint,
+    llmModel,
+    llmProvider,
+    systemPrompt,
     embedFn,
     rerankFn,
     log = () => {},
@@ -278,7 +306,7 @@ export async function createServe(opts: CreateServeOptions): Promise<{
   if (!fs.existsSync(path.join(indexDir, 'vectors.json'))) {
     if (!db) throw new Error('serve: no index found and no --db given to build one');
     log(`No index at ${indexDir} — building it from ${db}…`);
-    await buildIndex({ db, indexDir, cacheDir, modelName, ignore, offline, log, embedFn });
+    await buildIndex({ db, indexDir, cacheDir, modelName, ignore, offline, log, embedFn, ...(maxChunkTokens !== undefined ? { maxChunkTokens } : {}) });
   }
 
   const state: ServeState = {
@@ -293,15 +321,47 @@ export async function createServe(opts: CreateServeOptions): Promise<{
     healthPublic,
     ui: ui !== false,
     mcp,
+    ...(llmEndpoint ? { llmEndpoint } : {}),
+    ...(llmModel ? { llmModel } : {}),
+    ...(llmProvider ? { llmProvider } : {}),
+    ...(systemPrompt ? { systemPrompt } : {}),
     allowedHosts,
     corsOrigins,
     limiter: new ServeLimiter(rateLimit, DEFAULT_RATE_BURST, maxConcurrency),
+    sseConnections: 0,
   };
 
+  // issue #197: server→client requests (sampling) ride the GET /mcp SSE
+  // stream as `event: message` frames. With no stream open the send throws
+  // synchronously, the request rejects, and ask_knowledge_base degrades to
+  // its extractive answer — sampling stays strictly best-effort.
+  state.mcpChannel = createClientChannel((msg) => {
+    const sse = state.mcpSseRes;
+    if (!sse) {
+      throw new Error('sampling unavailable: client has no open GET /mcp stream');
+    }
+    sse.write(`event: message\ndata: ${JSON.stringify(msg)}\n\n`);
+  });
+
   const sockets = new Set<import('node:net').Socket>();
   const server = http.createServer((req, res) => {
     handleRequest(req, res, state).catch((e) => {
-      json(res, 500, { error: e.message });
+      // issue #182: raw exception text can carry absolute filesystem paths —
+      // reply generically, log the details server-side
+      process.stderr.write(`serve: unhandled request error: ${e?.stack || e}\n`);
+      try {
+        if (res.headersSent) {
+          res.destroy();
+        } else {
+          json(res, 500, { error: 'internal server error' });
+        }
+      } catch {
+        try {
+          res.destroy();
+        } catch {
+          /* socket already gone */
+        }
+      }
     });
   });
   server.on('connection', (socket) => {
@@ -317,7 +377,7 @@ export async function createServe(opts: CreateServeOptions): Promise<{
     if (!db) return false;
     log('Change detected — re-indexing…');
     try {
-      await buildIndex({ db, indexDir, cacheDir, modelName, ignore, offline, log, embedFn });
+      await buildIndex({ db, indexDir, cacheDir, modelName, ignore, offline, log, embedFn, ...(maxChunkTokens !== undefined ? { maxChunkTokens } : {}) });
     } catch (e: any) {
       if (/(being written by pid|being written|lock)/i.test(String(e?.message || ''))) {
         log(`re-index deferred: ${e.message}`);
@@ -333,14 +393,19 @@ export async function createServe(opts: CreateServeOptions): Promise<{
 
   const fileMd5 = (f: string): string =>
     _testFs?.hash ? _testFs.hash(f) : crypto.createHash('md5').update(fs.readFileSync(f)).digest('hex');
-  const statFile = (f: string): { mtimeMs: number } =>
-    _testFs?.stat ? _testFs.stat(f) : { mtimeMs: fs.statSync(f).mtimeMs };
+  const statFile = (f: string): { mtimeMs: number; size?: number } => {
+    if (_testFs?.stat) return _testFs.stat(f);
+    const st = fs.statSync(f);
+    return { mtimeMs: st.mtimeMs, size: st.size };
+  };
   const relOf = (f: string) => path.relative(db!, f).split(path.sep).join('/');
   const dbg = (msg: string) => {
     if (watchDebug) log(`watch-debug: ${msg}`);
   };
 
-  const md5Cache = new Map<string, { mtime: number; md5: string }>();
+  // issue #195: cache identity is size+mtimeMs — a file restored from a
+  // backup with an OLD mtime but new content no longer masks its change
+  const md5Cache = new Map<string, { mtime: number; size?: number; md5: string }>();
   // issue #116: files whose hash currently fails (AV/sync contention) — they
   // are EXCLUDED from fingerprints instead of becoming stable 'unreadable'
   // ones, so a transiently locked file can never mask a real change.
@@ -353,9 +418,9 @@ export async function createServe(opts: CreateServeOptions): Promise<{
    * Returns null when the file must be excluded this cycle: vanished (deleted),
    * or still unreadable after retries (changed-pending, warned rate-limited).
    */
-  const hashWithRetry = async (f: string, m: number): Promise<string | null> => {
+  const hashWithRetry = async (f: string, st: { mtimeMs: number; size?: number }): Promise<string | null> => {
     const hit = md5Cache.get(f);
-    if (hit && hit.mtime === m) {
+    if (hit && hit.mtime === st.mtimeMs && hit.size === st.size) {
       unreadable.delete(f);
       return hit.md5;
     }
@@ -365,7 +430,7 @@ export async function createServe(opts: CreateServeOptions): Promise<{
       onRetry: (attempt, err) => dbg(`retry ${attempt} ${path.basename(f)}: ${(err as any)?.code ?? (err as any)?.message}`),
     });
     if (r.ok) {
-      md5Cache.set(f, { mtime: m, md5: r.value });
+      md5Cache.set(f, { mtime: st.mtimeMs, size: st.size, md5: r.value });
       unreadable.delete(f);
       dbg(`hashed ${path.basename(f)}`);
       return r.value;
@@ -388,24 +453,45 @@ export async function createServe(opts: CreateServeOptions): Promise<{
    * One probe pass: hash every tree file (with retry). Returns per-file hashes
    * where null = vanished-or-unreadable this cycle (excluded from comparison).
    */
-  const probeTree = async (tree: Map<string, { m: number; rel: string }>) => {
+  const probeTree = async (tree: Map<string, { m: number; s?: number; rel: string }>) => {
     const hashes = new Map<string, string | null>();
-    for (const [f, { m }] of tree) {
-      hashes.set(f, await hashWithRetry(f, m));
+    for (const [f, { m, s }] of tree) {
+      hashes.set(f, await hashWithRetry(f, { mtimeMs: m, size: s }));
     }
     return hashes;
   };
 
-  const fingerprintOf = (
-    tree: Map<string, { m: number; rel: string }>,
+  /**
+   * issue #195: tree state as a rel→hash map — compared map-to-map with no
+   * O(n log n) string build + sort per poll. null means "unknown" (initial
+   * load or a native event forced a re-probe).
+   */
+  const stateOf = (
+    tree: Map<string, { m: number; s?: number; rel: string }>,
     fileHashes: Map<string, string | null>,
-  ): string => {
-    const parts: string[] = [];
+  ): Map<string, string> => {
+    const out = new Map<string, string>();
     for (const [f, { rel }] of tree) {
       const h = fileHashes.get(f);
-      if (h !== null && h !== undefined) parts.push(`${rel}:${h}`);
+      if (h !== null && h !== undefined) out.set(rel, h);
+    }
+    return out;
+  };
+
+  const statesEqual = (a: Map<string, string> | null, b: Map<string, string>): boolean => {
+    if (!a || a.size !== b.size) return false;
+    for (const [rel, hash] of b) {
+      if (a.get(rel) !== hash) return false;
+    }
+    return true;
+  };
+
+  /** issue #195: evict cache entries for files that left the tree. */
+  const evictStaleCacheEntries = (tree: Map<string, { m: number; s?: number; rel: string }>): void => {
+    if (md5Cache.size <= tree.size) return;
+    for (const f of md5Cache.keys()) {
+      if (!tree.has(f)) md5Cache.delete(f);
     }
-    return parts.sort().join('|');
   };
 
   const readIndexedHashes = (): Record<string, string> => {
@@ -417,13 +503,14 @@ export async function createServe(opts: CreateServeOptions): Promise<{
   };
 
   /** issue #116: stat failures are classified — only ENOENT vanishes silently. */
-  const scanTree = (): { tree: Map<string, { m: number; rel: string }>; problems: Map<string, 'transient' | 'permanent'> } => {
-    const tree = new Map<string, { m: number; rel: string }>();
+  const scanTree = (): { tree: Map<string, { m: number; s?: number; rel: string }>; problems: Map<string, 'transient' | 'permanent'> } => {
+    const tree = new Map<string, { m: number; s?: number; rel: string }>();
     const problems = new Map<string, 'transient' | 'permanent'>();
     if (!db) return { tree, problems };
     for (const f of walkMarkdown(db, ignore)) {
       try {
-        tree.set(f, { m: statFile(f).mtimeMs, rel: relOf(f) });
+        const st = statFile(f);
+        tree.set(f, { m: st.mtimeMs, s: st.size, rel: relOf(f) });
       } catch (e) {
         const cls = classifyFsError(e);
         if (cls !== 'vanished') problems.set(f, cls as 'transient' | 'permanent');
@@ -456,7 +543,9 @@ export async function createServe(opts: CreateServeOptions): Promise<{
     return out;
   };
 
-  let lastFingerprint = '';
+  // issue #195: null = "state unknown" (startup or a native event forced a
+  // re-probe); otherwise the rel→hash map of the last observed tree
+  let lastState: Map<string, string> | null = null;
 
   /** issue #116: one combined warning per cooldown, ≤5 files named + "+k more". */
   const warnUnreadableRateLimited = (now: number): void => {
@@ -474,7 +563,7 @@ export async function createServe(opts: CreateServeOptions): Promise<{
     {
       const { tree } = scanTree();
       const hashes = await probeTree(tree);
-      lastFingerprint = fingerprintOf(tree, hashes);
+      lastState = stateOf(tree, hashes);
     }
     let lastChangeAt = 0;
     let pending = false;
@@ -494,15 +583,16 @@ export async function createServe(opts: CreateServeOptions): Promise<{
       for (const [f, cls] of problems) {
         if (!unreadable.has(f)) unreadable.set(f, { cls, code: `stat:${cls}`, firstAt: t0 });
       }
+      evictStaleCacheEntries(tree);
       dbg(`poll: ${tree.size} file(s), ${unreadable.size} pending-unreadable`);
       const fileHashes = await probeTree(tree);
       warnUnreadableRateLimited(Date.now());
-      const fingerprint = fingerprintOf(tree, fileHashes);
+      const currentState = stateOf(tree, fileHashes);
       const changed = contentChangedPaths(tree, fileHashes, indexedHashes);
-      if (fingerprint !== lastFingerprint) {
+      if (!statesEqual(lastState, currentState)) {
         if (!pending) log(`watch: change in ${changed.length} file(s) — waiting ${watchDelay} ms for the tree to settle…`);
         pending = true;
-        lastFingerprint = fingerprint;
+        lastState = currentState;
         lastChangeAt = Date.now();
       } else if (pending && changed.length > 0 && Date.now() - lastChangeAt >= watchDelay) {
         pending = false;
@@ -512,14 +602,14 @@ export async function createServe(opts: CreateServeOptions): Promise<{
           const fresh = scanTree();
           const freshHashes = await probeTree(fresh.tree);
           // seed the md5 cache from the authoritative post-build state so the
-          // next fingerprint is IO-free for unchanged files
-          for (const [f, { m, rel }] of fresh.tree) {
+          // next poll is IO-free for unchanged files
+          for (const [f, { m, s }] of fresh.tree) {
             const h = freshHashes.get(f);
-            if (h !== null && h !== undefined && !md5Cache.has(f)) md5Cache.set(f, { mtime: m, md5: h });
+            if (h !== null && h !== undefined && !md5Cache.has(f)) md5Cache.set(f, { mtime: m, size: s, md5: h });
           }
-          lastFingerprint = fingerprintOf(fresh.tree, freshHashes);
+          lastState = stateOf(fresh.tree, freshHashes);
         }
-      } else if (!pending && changed.length > 0 && fingerprint === lastFingerprint) {
+      } else if (!pending && changed.length > 0 && statesEqual(lastState, currentState)) {
         pending = true;
         lastChangeAt = Date.now();
         log(`watch: tree differs from index (no motion) — settling then re-index…`);
@@ -529,7 +619,7 @@ export async function createServe(opts: CreateServeOptions): Promise<{
         // settled but nothing actually differs (e.g. a locked file became
         // readable again with unchanged content) — resync the baseline
         pending = false;
-        lastFingerprint = fingerprint;
+        lastState = currentState;
       }
     }
   };
@@ -539,14 +629,33 @@ export async function createServe(opts: CreateServeOptions): Promise<{
   if (watch) {
     if (!db) throw new Error('serve --watch requires --db');
     nativeWatcher = createFileWatcher(db, () => {
-      // Prompt watchLoop settle trigger on native event
-      lastFingerprint = '';
+      // Prompt watchLoop settle trigger on native event: state becomes
+      // "unknown", so the next poll re-probes and enters the settle path
+      lastState = null;
     }, {
       debounceMs: watchDelay,
       ignore,
       log,
     });
-    watchLoop();
+    // issue #186: watchLoop runs unawaited — an early rejection used to be an
+    // unhandled one and killed the daemon. Restart with backoff, give up
+    // after 5 crashes (search keeps serving the last good index either way).
+    void (async () => {
+      for (let attempt = 0; ; attempt++) {
+        try {
+          await watchLoop();
+          return; // resolved → stopped normally
+        } catch (e: any) {
+          if (stopped || attempt >= 4) {
+            log(`watch loop failed permanently (search still serves the last index): ${e.message}`);
+            return;
+          }
+          log(`watch loop crashed (attempt ${attempt + 1}/5): ${e.message} — restarting in 5s…`);
+          await new Promise((r) => setTimeout(r, 5_000));
+          if (stopped) return;
+        }
+      }
+    })();
   }
 
   const close = async (): Promise<void> => {
@@ -655,6 +764,48 @@ export function splitHostHeader(hostHeader: string): string {
   return first !== -1 && first === last ? s.slice(0, first) : s;
 }
 
+/**
+ * Cross-site POST guard (issue #178). Browsers fire "simple" POSTs without
+ * preflight (Content-Type: text/plain, form encodings) from any open page;
+ * the loopback Host check alone does not stop them. A POST counts as
+ * same-site when:
+ *  - the client sent no Origin/Sec-Fetch headers (curl, MCP and other
+ *    non-browser clients), or
+ *  - Sec-Fetch-Site is not cross-site AND the Origin's host matches the
+ *    request Host, a loopback allowlist entry, or an explicit --cors-origin.
+ * `Origin: null` (sandboxed iframe, file:// page) is always rejected.
+ */
+export function isSameSitePost(
+  req: http.IncomingMessage,
+  allowedHosts: Set<string>,
+  corsOrigins?: string[],
+): boolean {
+  const secFetchSite = req.headers['sec-fetch-site'];
+  if (typeof secFetchSite === 'string' && secFetchSite === 'cross-site') return false;
+  const origin = req.headers.origin;
+  if (typeof origin !== 'string' || origin.length === 0) return true; // non-browser client
+  if (origin === 'null') return false;
+  if (corsOrigins && corsOrigins.includes(origin)) return true;
+  const hostHeader = typeof req.headers.host === 'string' ? req.headers.host.toLowerCase() : '';
+  if (!hostHeader) return false;
+  let originUrl: URL;
+  try {
+    originUrl = new URL(origin);
+  } catch {
+    return false; // malformed Origin — fail closed
+  }
+  if (originUrl.host === hostHeader) return true; // exact same-origin (host:port)
+  // loopback aliases pass only on the SAME port — a page on localhost:3000 is
+  // a different origin even though the bare hostname is allowlisted
+  const portOf = (h: string, proto: string): string => {
+    const m = h.match(/:(\d+)$/);
+    if (m) return m[1];
+    return proto === 'https:' || proto === 'wss:' ? '443' : '80';
+  };
+  if (portOf(originUrl.host, originUrl.protocol) !== portOf(hostHeader, 'http:')) return false;
+  return allowedHosts.has(splitHostHeader(originUrl.host));
+}
+
 /**
  * Reflect CORS headers ONLY for an exact allowlisted Origin (issue #120).
  * Off entirely by default; `Vary: Origin` is set whenever CORS is configured so
@@ -703,17 +854,38 @@ async function handleRequest(req: http.IncomingMessage, res: http.ServerResponse
     return;
   }
 
-  if (state.apiKey) {
-    const isPublicHealth = url.pathname === '/health' && req.method === 'GET' && state.healthPublic;
-    if (!isPublicHealth) {
-      const authHeader = req.headers['authorization'];
-      const match = typeof authHeader === 'string' ? authHeader.match(/^Bearer\s+(.+)$/i) : null;
-      const token = match ? match[1].trim() : null;
-      if (!token || !isAuthorizedToken(token, state.apiKey)) {
-        json(res, 401, { error: 'unauthorized' });
+  // CSRF / body-safety gate for every body-parsing POST (issue #178): a "simple"
+  // cross-site browser POST (text/plain, form) sails past the Host allowlist on
+  // loopback, so reject foreign origins and non-JSON media types up front.
+  if (req.method === 'POST') {
+    const ct = req.headers['content-type'];
+    if (typeof ct === 'string') {
+      const media = ct.split(';')[0].trim().toLowerCase();
+      if (media !== 'application/json') {
+        json(res, 415, { error: `unsupported content-type "${media}" — POST bodies must be application/json` });
         return;
       }
     }
+    if (!isSameSitePost(req, allowedHosts, state.corsOrigins)) {
+      json(res, 403, { error: 'cross-site request rejected' });
+      return;
+    }
+  }
+
+  // issue #181: /health path disclosure depends on whether THIS request is
+  // authenticated — an unauthenticated daemon or public health hides indexDir
+  let requestAuthenticated = false;
+  if (state.apiKey) {
+    const authHeader = req.headers['authorization'];
+    const match = typeof authHeader === 'string' ? authHeader.match(/^Bearer\s+(.+)$/i) : null;
+    const token = match ? match[1].trim() : null;
+    const valid = !!token && isAuthorizedToken(token, state.apiKey);
+    const isPublicHealth = url.pathname === '/health' && req.method === 'GET' && state.healthPublic;
+    if (!valid && !isPublicHealth) {
+      json(res, 401, { error: 'unauthorized' });
+      return;
+    }
+    requestAuthenticated = valid;
   }
 
   // Streamable HTTP MCP transport (issue #123): same auth/host/rate-limit
@@ -791,6 +963,11 @@ async function handleRequest(req: http.IncomingMessage, res: http.ServerResponse
         offline: state.offline,
         embedFn: state.embedFn,
         rerankFn: state.rerankFn,
+        // issue #203: multi-query expansion (LLM bridge = the daemon's own
+        // --llm-endpoint, NEVER a payload-supplied URL — that would let any
+        // client exfiltrate note content to an arbitrary host) + reranker alias
+        ...(payload.multiQuery && state.llmEndpoint ? { multiQuery: payload.multiQuery, llmEndpoint: state.llmEndpoint, ...(state.llmModel ? { llmModel: state.llmModel } : {}) } : {}),
+        ...(payload.rerankModel ? { rerankModel: payload.rerankModel } : {}),
         tag: payload.tag,
         project: payload.project,
         type: payload.type,
@@ -807,7 +984,213 @@ async function handleRequest(req: http.IncomingMessage, res: http.ServerResponse
       });
       json(res, 200, { query, k, count: results.length, results });
     } catch (e: any) {
-      json(res, 500, { error: e.message });
+      // issue #182: search errors can name absolute paths — keep the reply
+      // generic, log the details server-side
+      process.stderr.write(`serve: /search failed: ${e?.stack || e}\n`);
+      json(res, 500, { error: 'internal server error' });
+    } finally {
+      release();
+    }
+    return;
+  }
+
+  // issue #199: grounded QA over SSE — a `sources` frame as soon as retrieval
+  // settles, streamed `delta` frames while the LLM answers, one `done` frame
+  // with the assembled result. Same fairness gates as /search (an LLM call
+  // holds a concurrency slot for its whole generation). Requires --llm-endpoint:
+  // over HTTP a missing chat backend is a 400, never a silent extractive degrade.
+  if (req.method === 'POST' && url.pathname === '/ask') {
+    const limited = state.limiter?.gate(req.socket.remoteAddress || 'unknown');
+    if (limited) {
+      res.setHeader('retry-after', String(limited.retryAfterSec));
+      json(res, 429, { error: 'rate limit exceeded — slow down', retryAfterSec: limited.retryAfterSec });
+      return;
+    }
+    if (!state.llmEndpoint) {
+      json(res, 400, { error: 'POST /ask requires an LLM chat endpoint — start mdss serve with --llm-endpoint' });
+      return;
+    }
+
+    const declared = req.headers['content-length'];
+    if (declared !== undefined) {
+      const n = Number(declared);
+      if (!Number.isInteger(n) || n < 0 || n > MAX_BODY_BYTES) {
+        json(res, 413, { error: `payload too large (limit ${MAX_BODY_BYTES} bytes)` });
+        return;
+      }
+    }
+    let body = '';
+    let tooLarge = false;
+    for await (const chunk of req) {
+      body += chunk;
+      if (body.length > MAX_BODY_BYTES) {
+        tooLarge = true;
+        break;
+      }
+    }
+    if (tooLarge) {
+      json(res, 413, { error: `payload too large (limit ${MAX_BODY_BYTES} bytes)` });
+      req.resume();
+      return;
+    }
+    let payload: Record<string, any>;
+    try {
+      payload = body ? JSON.parse(body) : {};
+    } catch (e: any) {
+      json(res, 400, { error: `invalid JSON body: ${e.message}` });
+      return;
+    }
+    const query = typeof payload.query === 'string' ? payload.query.trim() : '';
+    if (!query) {
+      json(res, 400, { error: 'missing "query" string in JSON body' });
+      return;
+    }
+    if (query.length > 2048) {
+      json(res, 400, { error: 'query exceeds maximum length of 2048 characters' });
+      return;
+    }
+    const k = Number.isInteger(payload.k) && payload.k > 0 ? payload.k : 5;
+
+    // issue #199 (follow-up): chat mode. `{"sessionId": "..."}` continues a
+    // persistent session (same store as `mdss chat`); `{"chat": true}` starts
+    // a fresh one whose id comes back in the `session`/`done` frames for the
+    // next request. One SSE protocol for both.
+    const rawSessionId = typeof payload.sessionId === 'string' ? payload.sessionId.trim() : '';
+    const chatMode = !!rawSessionId || payload.chat === true;
+    if (rawSessionId) {
+      // issue #149 discipline: validate BEFORE any filesystem use; a bad id
+      // gets a plain 400 here rather than a thrown error mid-stream
+      try {
+        assertSafeSessionId(rawSessionId);
+      } catch (e: any) {
+        json(res, 400, { error: `invalid sessionId: ${e.message}` });
+        return;
+      }
+    }
+
+    const release = state.limiter ? await state.limiter.acquireSlot() : null;
+    if (!release) {
+      res.setHeader('retry-after', '1');
+      json(res, 503, { error: 'server busy — search concurrency cap reached, queue full' });
+      return;
+    }
+
+    // Client disconnect must tear down the in-flight LLM call, not just stop
+    // writing frames — the slot releases in `finally` either way.
+    const clientAbort = new AbortController();
+    res.on('close', () => clientAbort.abort());
+    res.writeHead(200, {
+      'content-type': 'text/event-stream',
+      'cache-control': 'no-store',
+      'connection': 'keep-alive',
+    });
+    const sse = (event: string, data: unknown) => {
+      if (!res.writableEnded) res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
+    };
+    const llmOpts = {
+      llmEndpoint: state.llmEndpoint!,
+      ...(state.llmModel ? { llmModel: state.llmModel } : {}),
+      ...(state.llmProvider ? { llmProvider: state.llmProvider } : {}),
+      ...(state.systemPrompt ? { systemPrompt: state.systemPrompt } : {}),
+      onDelta: (d: string) => sse('delta', { text: d }),
+      signal: clientAbort.signal,
+    };
+    try {
+      if (chatMode) {
+        const session = loadChatSession(state.indexDir, rawSessionId || crypto.randomUUID().slice(0, 8));
+        sse('session', { id: session.id, turns: session.turns.length });
+        const turn = await chatTurn({
+          session,
+          query,
+          loaded: state.loaded,
+          indexDir: state.indexDir,
+          cacheDir: state.cacheDir,
+          k,
+          offline: state.offline,
+          embedFn: state.embedFn,
+          ...llmOpts,
+        });
+        sse('done', {
+          answer: turn.answer,
+          mode: turn.mode,
+          citations: turn.citations,
+          citationsUsed: turn.citationsUsed,
+          manifest: turn.manifest,
+          sessionId: session.id,
+          turnsCount: session.turns.length,
+        });
+      } else {
+        const passages = await searchIndex({
+          loaded: state.loaded,
+          cacheDir: state.cacheDir,
+          query,
+          k,
+          offline: state.offline,
+          embedFn: state.embedFn,
+        });
+        sse('sources', {
+          citations: passages.map((p) => ({ file: p.file, heading: p.heading, score: p.score })),
+        });
+        const result = await synthesizeAnswer(query, passages, llmOpts);
+        sse('done', {
+          answer: result.answer,
+          mode: result.mode,
+          citations: result.citations,
+          citationsUsed: result.citationsUsed,
+        });
+      }
+    } catch (e: any) {
+      // issue #182 discipline: raw errors can carry absolute paths — the
+      // frame stays generic, details go to the server log
+      process.stderr.write(`serve: /ask failed: ${e?.stack || e}\n`);
+      sse('error', { error: 'answer synthesis failed' });
+    } finally {
+      release();
+      res.end();
+    }
+    return;
+  }
+
+  if (req.method === 'GET' && url.pathname === '/doc') {
+    const file = url.searchParams.get('file');
+    if (!file) {
+      json(res, 400, { error: 'missing "file" query parameter' });
+      return;
+    }
+    // issue #181: /doc does synchronous disk IO per request — it goes through
+    // the same fairness gates as /search so a local storm cannot starve it
+    const limited = state.limiter?.gate(req.socket.remoteAddress || 'unknown');
+    if (limited) {
+      res.setHeader('retry-after', String(limited.retryAfterSec));
+      json(res, 429, { error: 'rate limit exceeded — slow down', retryAfterSec: limited.retryAfterSec });
+      return;
+    }
+    const release = state.limiter ? await state.limiter.acquireSlot() : null;
+    if (!release) {
+      res.setHeader('retry-after', '1');
+      json(res, 503, { error: 'server busy — concurrency cap reached, queue full' });
+      return;
+    }
+    const fromLineParam = url.searchParams.get('fromLine') || url.searchParams.get('from');
+    const maxLinesParam = url.searchParams.get('maxLines') || url.searchParams.get('lines') || url.searchParams.get('limit');
+    const fromLine = fromLineParam ? parseInt(fromLineParam, 10) : 1;
+    const maxLines = maxLinesParam ? parseInt(maxLinesParam, 10) : undefined;
+
+    try {
+      const docResult = getDocLines({
+        file,
+        fromLine: isNaN(fromLine) ? 1 : fromLine,
+        maxLines: maxLines && !isNaN(maxLines) ? maxLines : undefined,
+        dbDir: state.loaded.index.db,
+        chunks: state.loaded.index.chunks,
+      });
+      json(res, 200, docResult);
+    } catch (e: any) {
+      if (e.message?.includes('not found')) {
+        json(res, 404, { error: e.message });
+      } else {
+        json(res, 400, { error: e.message });
+      }
     } finally {
       release();
     }
@@ -815,6 +1198,10 @@ async function handleRequest(req: http.IncomingMessage, res: http.ServerResponse
   }
 
   if (req.method === 'GET' && url.pathname === '/health') {
+    // issue #181: an unauthenticated /health must not disclose filesystem
+    // locations — the path comes back only to an authenticated caller on a
+    // daemon that also requires auth for /health
+    const hideIndexDir = !state.apiKey || !requestAuthenticated;
     json(res, 200, {
       ok: true,
       chunks: state.loaded.index.chunks.length,
@@ -822,7 +1209,7 @@ async function handleRequest(req: http.IncomingMessage, res: http.ServerResponse
       dim: state.loaded.index.dim || state.loaded.model.dim || 0,
       built: state.loaded.index.built || null,
       watching: state.watching,
-      indexDir: state.indexDir,
+      indexDir: hideIndexDir ? undefined : state.indexDir,
       // fairness counters (issue #119)
       in_flight: state.limiter?.inFlight ?? 0,
       queued: state.limiter?.queued ?? 0,
@@ -847,11 +1234,97 @@ async function handleRequest(req: http.IncomingMessage, res: http.ServerResponse
     return;
   }
 
+  if (url.pathname === '/related' && (req.method === 'POST' || req.method === 'GET')) {
+    // issue #181: /related runs the (semantic) relatedness sweep — same rate
+    // gate as /search, before any body parsing
+    const limited = state.limiter?.gate(req.socket.remoteAddress || 'unknown');
+    if (limited) {
+      res.setHeader('retry-after', String(limited.retryAfterSec));
+      json(res, 429, { error: 'rate limit exceeded — slow down', retryAfterSec: limited.retryAfterSec });
+      return;
+    }
+    let target = '';
+    let k = 10;
+    let direction: 'both' | 'outgoing' | 'backlinks' = 'both';
+    let semantic = true;
+
+    if (req.method === 'GET') {
+      target = url.searchParams.get('file') || url.searchParams.get('target') || url.searchParams.get('note') || '';
+      const kParam = url.searchParams.get('k');
+      if (kParam) k = parseInt(kParam, 10) || 10;
+      const dirParam = url.searchParams.get('direction');
+      if (dirParam === 'outgoing' || dirParam === 'backlinks' || dirParam === 'both') direction = dirParam;
+      if (url.searchParams.has('semantic')) semantic = url.searchParams.get('semantic') !== 'false';
+    } else {
+      // issue #181: an overflowing body is a 413, not a silently truncated parse
+      let body = '';
+      let tooLarge = false;
+      for await (const chunk of req) {
+        body += chunk;
+        if (body.length > MAX_BODY_BYTES) {
+          tooLarge = true;
+          break;
+        }
+      }
+      if (tooLarge) {
+        json(res, 413, { error: `payload too large (limit ${MAX_BODY_BYTES} bytes)` });
+        req.resume();
+        return;
+      }
+      try {
+        const payload = body ? JSON.parse(body) : {};
+        target = payload.file || payload.target || payload.note || '';
+        if (typeof payload.k === 'number' && payload.k > 0) k = payload.k;
+        if (payload.direction === 'outgoing' || payload.direction === 'backlinks' || payload.direction === 'both') {
+          direction = payload.direction;
+        }
+        if (payload.semantic !== undefined) semantic = Boolean(payload.semantic);
+      } catch (e: any) {
+        json(res, 400, { error: `invalid JSON body: ${e.message}` });
+        return;
+      }
+    }
+
+    if (!target) {
+      json(res, 400, { error: 'missing "file" or "target" parameter' });
+      return;
+    }
+
+    const release = state.limiter ? await state.limiter.acquireSlot() : null;
+    if (!release) {
+      res.setHeader('retry-after', '1');
+      json(res, 503, { error: 'server busy — concurrency cap reached, queue full' });
+      return;
+    }
+    try {
+      const relatedResult = findRelatedNotes({
+        loaded: state.loaded,
+        target,
+        k,
+        direction,
+        semantic,
+      });
+      json(res, 200, relatedResult);
+    } catch (e: any) {
+      if (e.message?.includes('not found')) {
+        json(res, 404, { error: e.message });
+      } else {
+        json(res, 400, { error: e.message });
+      }
+    } finally {
+      release();
+    }
+    return;
+  }
+
   if (req.method === 'GET' && (url.pathname === '/' || url.pathname === '/help')) {
     json(res, 200, {
       name: 'mdss serve',
       endpoints: [
         { method: 'POST', path: '/search', body: '{ "query": "...", "k": 6, "semanticOnly": false, "rerank": false }' },
+        { method: 'POST', path: '/ask', body: '{ "query": "...", "k": 5 } — SSE: sources/delta/done frames (needs --llm-endpoint, issue #199)' },
+        { method: 'GET', path: '/doc?file=...&fromLine=1&maxLines=50' },
+        { method: 'POST', path: '/related', body: '{ "file": "...", "k": 10, "semantic": true }' },
         { method: 'GET', path: '/health' },
       ],
     });
@@ -878,12 +1351,23 @@ async function handleMcpHttp(req: http.IncomingMessage, res: http.ServerResponse
   if (req.method === 'GET') {
     // Server-initiated channel: we have nothing to push today, so keep the
     // stream alive with comments until the client goes away.
+    // issue #181: each stream holds a socket + a heartbeat interval — cap the
+    // number of concurrent SSE connections at the fairness concurrency limit.
+    const sseCap = state.limiter?.concurrencyEnabled ? state.limiter.maxConcurrency : DEFAULT_MAX_CONCURRENCY;
+    if ((state.sseConnections ?? 0) >= sseCap) {
+      json(res, 503, { error: 'server busy — SSE connection cap reached' });
+      return;
+    }
+    state.sseConnections = (state.sseConnections ?? 0) + 1;
     res.writeHead(200, {
       'content-type': 'text/event-stream',
       'cache-control': 'no-store',
       'connection': 'keep-alive',
     });
     res.write(':ok\n\n');
+    // issue #197: this stream doubles as the server→client downlink for
+    // sampling/createMessage. One client realistically; newest stream wins.
+    state.mcpSseRes = res;
     const hb = setInterval(() => {
       try {
         res.write(':hb\n\n');
@@ -892,7 +1376,13 @@ async function handleMcpHttp(req: http.IncomingMessage, res: http.ServerResponse
       }
     }, 15_000);
     hb.unref?.();
-    req.on('close', () => clearInterval(hb));
+    // res 'close' fires on client abort mid-stream too — req may never close
+    // while the response stream is still open
+    res.on('close', () => {
+      clearInterval(hb);
+      if (state.mcpSseRes === res) state.mcpSseRes = undefined;
+      state.sseConnections = Math.max(0, (state.sseConnections ?? 1) - 1);
+    });
     return;
   }
 
@@ -901,7 +1391,23 @@ async function handleMcpHttp(req: http.IncomingMessage, res: http.ServerResponse
     return;
   }
 
+  // issue #182: session binding before any work — the server mints its own id
+  // and rejects mismatching client ids with 404 instead of adopting them
+  // (session fixation) or feeding them to res.setHeader (500 on bad headers)
+  const requested = req.headers['mcp-session-id'];
+  if (
+    typeof requested === 'string' &&
+    requested &&
+    state.mcpSessionId &&
+    requested !== state.mcpSessionId
+  ) {
+    json(res, 404, { error: 'session not found — re-initialize' });
+    return;
+  }
+
   // Same fairness gate as /search — tools/call can trigger embedding work.
+  // (The gates run AFTER body parse below: responses to server→client
+  // requests must bypass them, see issue #197.)
   const limited = state.limiter?.gate(req.socket.remoteAddress || 'unknown');
   if (limited) {
     res.setHeader('retry-after', String(limited.retryAfterSec));
@@ -934,35 +1440,57 @@ async function handleMcpHttp(req: http.IncomingMessage, res: http.ServerResponse
     const batch = Array.isArray(messages);
     const items = batch ? messages : [messages];
 
+    // issue #197: answers to server→client requests (sampling) arrive as
+    // plain POSTs. Resolve them BEFORE the fairness gates — the tools/call
+    // waiting on this response may hold the only concurrency slot, so
+    // queueing it behind itself would self-deadlock at max_concurrency 1.
+    let sawResponseOnly = items.length > 0;
+    for (const msg of items) {
+      if (!msg || typeof msg !== 'object') {
+        continue;
+      }
+      const isResponse = msg.method === undefined && (msg.result !== undefined || msg.error !== undefined);
+      if (isResponse) {
+        state.mcpChannel?.handleResponse(msg);
+      } else {
+        sawResponseOnly = false;
+      }
+    }
+    if (sawResponseOnly) {
+      res.writeHead(202);
+      res.end();
+      return;
+    }
+
     const responses: any[] = [];
     for (const msg of items) {
       if (!msg || typeof msg !== 'object') continue;
+      if (msg.method === undefined) continue; // response — correlated above
       const isNotification = msg.id === undefined || msg.id === null;
       const result = isNotification ? null : await handleMcpRequest(msg, {
         loaded: state.loaded,
         cacheDir: state.cacheDir,
         offline: state.offline,
         embedFn: state.embedFn,
+        channel: state.mcpChannel,
+        clientCapabilities: state.mcpClientCapabilities,
       });
       if (result) responses.push(result);
       else if (!isNotification && msg.method === undefined) continue;
+      // issue #197: remember the client's declared capabilities (sampling
+      // gates the ask_knowledge_base LLM path on later tools/call).
+      if (msg.method === 'initialize') {
+        state.mcpClientCapabilities = msg.params?.capabilities ?? {};
+      }
     }
 
-    // session id: mint on first contact, echo afterwards
-    const requested = req.headers['mcp-session-id'];
-    if (typeof requested === 'string' && requested) state.mcpSessionId = requested;
+    // session id: server-minted only (issue #182 — see the 404 gate above);
+    // mint on first contact, echo ours afterwards. Protocol negotiation
+    // itself moved into the shared brain (issue #197) so stdio and HTTP
+    // speak identical version logic.
     if (!state.mcpSessionId) state.mcpSessionId = crypto.randomUUID();
     res.setHeader('mcp-session-id', state.mcpSessionId);
 
-    // protocol negotiation: honour a supported requested version, else latest
-    for (const resp of responses) {
-      if (resp?.result?.protocolVersion && msg0(items)?.params?.protocolVersion) {
-        const wanted = String(msg0(items).params.protocolVersion);
-        const SUPPORTED = ['2025-03-26', '2024-11-05'];
-        resp.result.protocolVersion = SUPPORTED.includes(wanted) ? wanted : SUPPORTED[0];
-      }
-    }
-
     if (responses.length === 0) {
       res.writeHead(202);
       res.end();
@@ -974,11 +1502,6 @@ async function handleMcpHttp(req: http.IncomingMessage, res: http.ServerResponse
   }
 }
 
-/** First object of a (possibly single-message) payload — for init negotiation. */
-function msg0(items: any[]): any | null {
-  return items.length > 0 ? items[0] : null;
-}
-
 function json(res: http.ServerResponse, status: number, data: object): void {
   res.writeHead(status, {
     'content-type': 'application/json; charset=utf-8',