md-semantic-search 1.0.2 → 1.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/serve.ts CHANGED
@@ -6,11 +6,14 @@
6
6
  import http from 'node:http';
7
7
  import crypto from 'node:crypto';
8
8
  import fs from 'node:fs';
9
+ import os from 'node:os';
9
10
  import path from 'node:path';
10
11
  import { buildIndex } from './indexer.js';
11
12
  import { loadIndex, searchIndex } from './search.js';
12
13
  import { walkMarkdown, assertSafePath, ModelDescriptor } from './core.js';
13
- import { createFileWatcher, FileWatcher } from './watcher.js';
14
+ import { createFileWatcher, FileWatcher, classifyFsError, readWithRetry } from './watcher.js';
15
+ import { WEBUI_HTML, WEBUI_JS, WEBUI_CSS } from './webui.js';
16
+ import { handleMcpRequest } from './mcp.js';
14
17
 
15
18
  const DEFAULT_PORT = 8747;
16
19
  const DEFAULT_HOST = '127.0.0.1';
@@ -18,6 +21,14 @@ const WATCH_INTERVAL_MS = 3000;
18
21
  const WATCH_DELAY_MS = 1000;
19
22
  const MAX_BODY_BYTES = 64 * 1024;
20
23
 
24
+ /** Hosts a browser is allowed to name in the Host header (issue #120). */
25
+ export const DEFAULT_ALLOWED_HOSTS = ['localhost', '127.0.0.1', '::1', '[::1]'];
26
+
27
+ /** Fairness defaults (issue #119): protective yet friction-free locally. */
28
+ export const DEFAULT_RATE_LIMIT_PER_MIN = 60;
29
+ export const DEFAULT_RATE_BURST = 10;
30
+ export const DEFAULT_MAX_CONCURRENCY = os.cpus().length || 1;
31
+
21
32
  export interface ServeState {
22
33
  loaded: { index: any; model: ModelDescriptor };
23
34
  indexDir: string;
@@ -29,6 +40,17 @@ export interface ServeState {
29
40
  reindexCount?: number;
30
41
  apiKey?: string;
31
42
  healthPublic?: boolean;
43
+ /** issue #111: serve the built-in web UI at `/` (default on). */
44
+ ui?: boolean;
45
+ /** issue #123: mount the Streamable HTTP MCP transport at /mcp (default off). */
46
+ mcp?: boolean;
47
+ mcpSessionId?: string;
48
+ /** Extra Host-header values accepted beyond the loopback defaults (issue #120). */
49
+ allowedHosts?: string[];
50
+ /** Exact origins reflected as `Access-Control-Allow-Origin`; absent → CORS off (issue #120). */
51
+ corsOrigins?: string[];
52
+ /** Fairness gates for /search (issue #119); `0` disables a gate. */
53
+ limiter?: ServeLimiter;
32
54
  }
33
55
 
34
56
  export interface CreateServeOptions {
@@ -43,11 +65,182 @@ export interface CreateServeOptions {
43
65
  watchDelay?: number;
44
66
  apiKey?: string;
45
67
  healthPublic?: boolean;
68
+ allowedHosts?: string[];
69
+ corsOrigins?: string[];
70
+ rateLimit?: number;
71
+ maxConcurrency?: number;
72
+ /** issue #116: verbose watch-loop trace (polls, retries, classifications). */
73
+ watchDebug?: boolean;
74
+ /** issue #116 test seam: override hashing/stat so fault-injection needs no real locks. */
75
+ _testFs?: {
76
+ hash?: (absPath: string) => string;
77
+ stat?: (absPath: string) => { mtimeMs: number };
78
+ };
79
+ ui?: boolean;
80
+ /** issue #123: mount Streamable HTTP MCP at /mcp (default off; stdio stays default). */
81
+ mcp?: boolean;
46
82
  embedFn?: any;
47
83
  rerankFn?: any;
48
84
  log?: (msg: string) => void;
49
85
  }
50
86
 
87
+ /**
88
+ * Token bucket for per-client request budgeting (issue #119): starts full
89
+ * (`burst`), refills continuously at `limitPerMinute`/60 tokens per second.
90
+ * `take` is injective in time so tests can time-travel via the `now` argument.
91
+ */
92
+ export class TokenBucket {
93
+ private tokens: number;
94
+ /** Last refill timestamp — doubles as a staleness marker for map pruning. */
95
+ lastTouched: number;
96
+
97
+ constructor(readonly limitPerMinute: number, readonly burst: number) {
98
+ this.tokens = burst;
99
+ this.lastTouched = Date.now();
100
+ }
101
+
102
+ take(now: number = Date.now()): { ok: true } | { ok: false; retryAfterSec: number } {
103
+ const elapsedMs = Math.max(0, now - this.lastTouched);
104
+ this.lastTouched = now;
105
+ this.tokens = Math.min(this.burst, this.tokens + (elapsedMs * this.limitPerMinute) / 60_000);
106
+ if (this.tokens >= 1) {
107
+ this.tokens -= 1;
108
+ return { ok: true };
109
+ }
110
+ const retryAfterSec = Math.max(1, Math.ceil(((1 - this.tokens) * 60_000) / this.limitPerMinute / 1000));
111
+ return { ok: false, retryAfterSec };
112
+ }
113
+ }
114
+
115
+ /** A granted slot's release handle — idempotent, so `finally` is always safe. */
116
+ type SlotRelease = () => void;
117
+
118
+ /**
119
+ * Counting semaphore with a bounded wait queue (issue #119): once `maxQueued`
120
+ * callers already wait, acquire() returns null instead of enqueueing, so an
121
+ * overloaded daemon sheds load with 503 rather than ballooning memory.
122
+ */
123
+ export class BoundedSemaphore {
124
+ private available: number;
125
+ private readonly waiters: Array<() => void> = [];
126
+
127
+ constructor(readonly size: number, readonly maxQueued: number) {
128
+ this.available = size;
129
+ }
130
+
131
+ get executing(): number {
132
+ return this.size - this.available;
133
+ }
134
+
135
+ get queued(): number {
136
+ return this.waiters.length;
137
+ }
138
+
139
+ async acquire(): Promise<SlotRelease | null> {
140
+ if (this.available > 0) {
141
+ this.available -= 1;
142
+ return makeRelease(() => this.releaseSlot());
143
+ }
144
+ if (this.waiters.length >= this.maxQueued) return null;
145
+ await new Promise<void>((resolve) => this.waiters.push(resolve));
146
+ return makeRelease(() => this.releaseSlot());
147
+ }
148
+
149
+ private releaseSlot(): void {
150
+ const next = this.waiters.shift();
151
+ if (next) next(); // hand the slot straight to a waiter — count stays consistent
152
+ else this.available += 1;
153
+ }
154
+ }
155
+
156
+ function makeRelease(release: () => void): SlotRelease {
157
+ let done = false;
158
+ return () => {
159
+ if (!done) {
160
+ done = true;
161
+ release();
162
+ }
163
+ };
164
+ }
165
+
166
+ /**
167
+ * Fairness gates for POST /search (issue #119):
168
+ * - per-client token bucket → 429 + Retry-After when a client exceeds its budget;
169
+ * - global concurrency cap around the embedding sweep; queue overflow → 503.
170
+ * Either gate disabled by passing `0`; counters are exposed via /health.
171
+ */
172
+ export class ServeLimiter {
173
+ private readonly buckets = new Map<string, TokenBucket>();
174
+ private readonly sem: BoundedSemaphore | null;
175
+ rejected429 = 0;
176
+ rejected503 = 0;
177
+
178
+ constructor(
179
+ readonly rateLimitPerMin: number,
180
+ readonly rateBurst: number,
181
+ readonly maxConcurrency: number,
182
+ ) {
183
+ // queue depth cap: 2× slots (min 4) — enough burst absorption, bounded memory
184
+ this.sem = maxConcurrency > 0 ? new BoundedSemaphore(maxConcurrency, Math.max(maxConcurrency * 2, 4)) : null;
185
+ }
186
+
187
+ /** Burst is capped by the per-minute budget itself (--rate-limit 5 → burst 5). */
188
+ private effectiveBurst(): number {
189
+ return Math.min(this.rateBurst, Math.max(1, this.rateLimitPerMin));
190
+ }
191
+
192
+ get rateEnabled(): boolean {
193
+ return this.rateLimitPerMin > 0 && this.rateBurst > 0;
194
+ }
195
+
196
+ get concurrencyEnabled(): boolean {
197
+ return this.sem !== null;
198
+ }
199
+
200
+ get inFlight(): number {
201
+ return this.sem ? this.sem.executing : 0;
202
+ }
203
+
204
+ get queued(): number {
205
+ return this.sem ? this.sem.queued : 0;
206
+ }
207
+
208
+ get rejectedTotal(): number {
209
+ return this.rejected429 + this.rejected503;
210
+ }
211
+
212
+ /** Cheap synchronous rate check. Null → proceed; otherwise reject with 429. */
213
+ gate(clientKey: string, now: number = Date.now()): { status: 429; retryAfterSec: number } | null {
214
+ if (!this.rateEnabled) return null;
215
+ let bucket = this.buckets.get(clientKey);
216
+ if (!bucket) {
217
+ if (this.buckets.size >= 4096) {
218
+ // opportunistic prune of idle clients (no takes within two windows)
219
+ for (const [key, b] of this.buckets) {
220
+ if (now - b.lastTouched > 2 * 60_000) this.buckets.delete(key);
221
+ }
222
+ }
223
+ bucket = new TokenBucket(this.rateLimitPerMin, this.effectiveBurst());
224
+ this.buckets.set(clientKey, bucket);
225
+ }
226
+ const r = bucket.take(now);
227
+ if (r.ok) return null;
228
+ this.rejected429 += 1;
229
+ return { status: 429, retryAfterSec: r.retryAfterSec };
230
+ }
231
+
232
+ /**
233
+ * Acquire a global search slot. Resolves with a release handle, or null when
234
+ * the wait queue is saturated (caller replies 503).
235
+ */
236
+ async acquireSlot(): Promise<SlotRelease | null> {
237
+ if (!this.sem) return makeRelease(() => {});
238
+ const release = await this.sem.acquire();
239
+ if (!release) this.rejected503 += 1;
240
+ return release;
241
+ }
242
+ }
243
+
51
244
  export async function createServe(opts: CreateServeOptions): Promise<{
52
245
  server: http.Server;
53
246
  state: ServeState;
@@ -65,6 +258,14 @@ export async function createServe(opts: CreateServeOptions): Promise<{
65
258
  watchDelay = WATCH_DELAY_MS,
66
259
  apiKey = process.env.MDSS_API_KEY,
67
260
  healthPublic = process.env.MDSS_HEALTH_PUBLIC === 'true',
261
+ allowedHosts,
262
+ corsOrigins,
263
+ rateLimit = DEFAULT_RATE_LIMIT_PER_MIN,
264
+ maxConcurrency = DEFAULT_MAX_CONCURRENCY,
265
+ watchDebug = false,
266
+ _testFs,
267
+ mcp = false,
268
+ ui = true,
68
269
  embedFn,
69
270
  rerankFn,
70
271
  log = () => {},
@@ -90,6 +291,11 @@ export async function createServe(opts: CreateServeOptions): Promise<{
90
291
  watching: watch,
91
292
  apiKey,
92
293
  healthPublic,
294
+ ui: ui !== false,
295
+ mcp,
296
+ allowedHosts,
297
+ corsOrigins,
298
+ limiter: new ServeLimiter(rateLimit, DEFAULT_RATE_BURST, maxConcurrency),
93
299
  };
94
300
 
95
301
  const sockets = new Set<import('node:net').Socket>();
@@ -125,28 +331,79 @@ export async function createServe(opts: CreateServeOptions): Promise<{
125
331
  return true;
126
332
  };
127
333
 
128
- const fileMd5 = (f: string) => crypto.createHash('md5').update(fs.readFileSync(f)).digest('hex');
334
+ const fileMd5 = (f: string): string =>
335
+ _testFs?.hash ? _testFs.hash(f) : crypto.createHash('md5').update(fs.readFileSync(f)).digest('hex');
336
+ const statFile = (f: string): { mtimeMs: number } =>
337
+ _testFs?.stat ? _testFs.stat(f) : { mtimeMs: fs.statSync(f).mtimeMs };
129
338
  const relOf = (f: string) => path.relative(db!, f).split(path.sep).join('/');
339
+ const dbg = (msg: string) => {
340
+ if (watchDebug) log(`watch-debug: ${msg}`);
341
+ };
130
342
 
131
343
  const md5Cache = new Map<string, { mtime: number; md5: string }>();
132
- const hashAt = (f: string, m: number) => {
344
+ // issue #116: files whose hash currently fails (AV/sync contention) — they
345
+ // are EXCLUDED from fingerprints instead of becoming stable 'unreadable'
346
+ // ones, so a transiently locked file can never mask a real change.
347
+ const unreadable = new Map<string, { cls: 'transient' | 'permanent'; code: string; firstAt: number }>();
348
+ let lastWarnAt = 0;
349
+ const WARN_COOLDOWN_MS = 30_000;
350
+
351
+ /**
352
+ * Hash a file with bounded retry for TRANSIENT failures (issue #116).
353
+ * Returns null when the file must be excluded this cycle: vanished (deleted),
354
+ * or still unreadable after retries (changed-pending, warned rate-limited).
355
+ */
356
+ const hashWithRetry = async (f: string, m: number): Promise<string | null> => {
133
357
  const hit = md5Cache.get(f);
134
- if (hit && hit.mtime === m) return hit.md5;
135
- const md5 = fileMd5(f);
136
- md5Cache.set(f, { mtime: m, md5 });
137
- return md5;
358
+ if (hit && hit.mtime === m) {
359
+ unreadable.delete(f);
360
+ return hit.md5;
361
+ }
362
+ const r = await readWithRetry(() => fileMd5(f), {
363
+ attempts: 3,
364
+ baseDelayMs: 200,
365
+ onRetry: (attempt, err) => dbg(`retry ${attempt} ${path.basename(f)}: ${(err as any)?.code ?? (err as any)?.message}`),
366
+ });
367
+ if (r.ok) {
368
+ md5Cache.set(f, { mtime: m, md5: r.value });
369
+ unreadable.delete(f);
370
+ dbg(`hashed ${path.basename(f)}`);
371
+ return r.value;
372
+ }
373
+ const cls = classifyFsError(r.err);
374
+ if (cls === 'vanished') {
375
+ md5Cache.delete(f);
376
+ return null; // deleted mid-cycle — handled via liveRels diff
377
+ }
378
+ const prev = unreadable.get(f);
379
+ unreadable.set(f, {
380
+ cls: cls as 'transient' | 'permanent',
381
+ code: (r.err as any)?.code || String((r.err as any)?.message || 'error'),
382
+ firstAt: prev?.firstAt ?? Date.now(),
383
+ });
384
+ return null;
385
+ };
386
+
387
+ /**
388
+ * One probe pass: hash every tree file (with retry). Returns per-file hashes
389
+ * where null = vanished-or-unreadable this cycle (excluded from comparison).
390
+ */
391
+ const probeTree = async (tree: Map<string, { m: number; rel: string }>) => {
392
+ const hashes = new Map<string, string | null>();
393
+ for (const [f, { m }] of tree) {
394
+ hashes.set(f, await hashWithRetry(f, m));
395
+ }
396
+ return hashes;
138
397
  };
139
398
 
140
- const treeContentFingerprint = (tree: Map<string, { m: number; rel: string }>) => {
399
+ const fingerprintOf = (
400
+ tree: Map<string, { m: number; rel: string }>,
401
+ fileHashes: Map<string, string | null>,
402
+ ): string => {
141
403
  const parts: string[] = [];
142
- for (const [f, { m, rel }] of tree) {
143
- let md5: string;
144
- try {
145
- md5 = hashAt(f, m);
146
- } catch {
147
- md5 = 'unreadable';
148
- }
149
- parts.push(`${rel}:${md5}`);
404
+ for (const [f, { rel }] of tree) {
405
+ const h = fileHashes.get(f);
406
+ if (h !== null && h !== undefined) parts.push(`${rel}:${h}`);
150
407
  }
151
408
  return parts.sort().join('|');
152
409
  };
@@ -159,36 +416,38 @@ export async function createServe(opts: CreateServeOptions): Promise<{
159
416
  }
160
417
  };
161
418
 
162
- const scanTree = (): Map<string, { m: number; rel: string }> => {
419
+ /** issue #116: stat failures are classified — only ENOENT vanishes silently. */
420
+ const scanTree = (): { tree: Map<string, { m: number; rel: string }>; problems: Map<string, 'transient' | 'permanent'> } => {
163
421
  const tree = new Map<string, { m: number; rel: string }>();
164
- if (!db) return tree;
422
+ const problems = new Map<string, 'transient' | 'permanent'>();
423
+ if (!db) return { tree, problems };
165
424
  for (const f of walkMarkdown(db, ignore)) {
166
425
  try {
167
- tree.set(f, { m: fs.statSync(f).mtimeMs, rel: relOf(f) });
168
- } catch {
169
- /* vanished */
426
+ tree.set(f, { m: statFile(f).mtimeMs, rel: relOf(f) });
427
+ } catch (e) {
428
+ const cls = classifyFsError(e);
429
+ if (cls !== 'vanished') problems.set(f, cls as 'transient' | 'permanent');
430
+ /* vanished files drop out via the liveRels diff */
170
431
  }
171
432
  }
172
- return tree;
433
+ return { tree, problems };
173
434
  };
174
435
 
175
- const contentChangedPaths = (tree: Map<string, { m: number; rel: string }>, indexedHashes: Record<string, string>) => {
436
+ const contentChangedPaths = (
437
+ tree: Map<string, { m: number; rel: string }>,
438
+ fileHashes: Map<string, string | null>,
439
+ indexedHashes: Record<string, string>,
440
+ ) => {
176
441
  const out: string[] = [];
177
442
  const liveRels = new Set<string>();
178
- for (const [f, { m, rel }] of tree) {
443
+ for (const [f, { rel }] of tree) {
179
444
  liveRels.add(rel);
180
445
  if (!(rel in indexedHashes)) {
181
- out.push(rel);
182
- continue;
183
- }
184
- const cached = md5Cache.get(f);
185
- if (cached && cached.mtime === m && cached.md5 === indexedHashes[rel]) continue;
186
- let fresh: string;
187
- try {
188
- fresh = hashAt(f, m);
189
- } catch {
446
+ out.push(rel); // new file
190
447
  continue;
191
448
  }
449
+ const fresh = fileHashes.get(f);
450
+ if (fresh === null || fresh === undefined) continue; // unreadable → changed-pending
192
451
  if (fresh !== indexedHashes[rel]) out.push(rel);
193
452
  }
194
453
  for (const rel of Object.keys(indexedHashes)) {
@@ -199,9 +458,24 @@ export async function createServe(opts: CreateServeOptions): Promise<{
199
458
 
200
459
  let lastFingerprint = '';
201
460
 
461
+ /** issue #116: one combined warning per cooldown, ≤5 files named + "+k more". */
462
+ const warnUnreadableRateLimited = (now: number): void => {
463
+ if (unreadable.size === 0 || now - lastWarnAt < WARN_COOLDOWN_MS) return;
464
+ lastWarnAt = now;
465
+ const names = [...unreadable.keys()].map((f) => path.basename(f));
466
+ const head = names.slice(0, 5).join(', ');
467
+ const more = names.length > 5 ? ` …+${names.length - 5} more` : '';
468
+ const codes = [...new Set([...unreadable.values()].map((u) => u.code))].join('/');
469
+ log(`watch: ${unreadable.size} file(s) unreadable (${codes}) — will retry: ${head}${more}`);
470
+ };
471
+
202
472
  const watchLoop = async () => {
203
473
  let indexedHashes = readIndexedHashes();
204
- lastFingerprint = treeContentFingerprint(scanTree());
474
+ {
475
+ const { tree } = scanTree();
476
+ const hashes = await probeTree(tree);
477
+ lastFingerprint = fingerprintOf(tree, hashes);
478
+ }
205
479
  let lastChangeAt = 0;
206
480
  let pending = false;
207
481
  while (!stopped) {
@@ -215,9 +489,16 @@ export async function createServe(opts: CreateServeOptions): Promise<{
215
489
  timer.unref?.();
216
490
  });
217
491
  if (stopped) return;
218
- const tree = scanTree();
219
- const fingerprint = treeContentFingerprint(tree);
220
- const changed = contentChangedPaths(tree, indexedHashes);
492
+ const t0 = Date.now();
493
+ const { tree, problems } = scanTree();
494
+ for (const [f, cls] of problems) {
495
+ if (!unreadable.has(f)) unreadable.set(f, { cls, code: `stat:${cls}`, firstAt: t0 });
496
+ }
497
+ dbg(`poll: ${tree.size} file(s), ${unreadable.size} pending-unreadable`);
498
+ const fileHashes = await probeTree(tree);
499
+ warnUnreadableRateLimited(Date.now());
500
+ const fingerprint = fingerprintOf(tree, fileHashes);
501
+ const changed = contentChangedPaths(tree, fileHashes, indexedHashes);
221
502
  if (fingerprint !== lastFingerprint) {
222
503
  if (!pending) log(`watch: change in ${changed.length} file(s) — waiting ${watchDelay} ms for the tree to settle…`);
223
504
  pending = true;
@@ -227,18 +508,28 @@ export async function createServe(opts: CreateServeOptions): Promise<{
227
508
  pending = false;
228
509
  await reload().catch((e) => log(`re-index failed: ${e.message}`));
229
510
  indexedHashes = readIndexedHashes();
230
- for (const [f, { m, rel }] of scanTree()) {
231
- if (!md5Cache.has(f) && rel in indexedHashes) {
232
- md5Cache.set(f, { mtime: m, md5: indexedHashes[rel] });
511
+ {
512
+ const fresh = scanTree();
513
+ const freshHashes = await probeTree(fresh.tree);
514
+ // seed the md5 cache from the authoritative post-build state so the
515
+ // next fingerprint is IO-free for unchanged files
516
+ for (const [f, { m, rel }] of fresh.tree) {
517
+ const h = freshHashes.get(f);
518
+ if (h !== null && h !== undefined && !md5Cache.has(f)) md5Cache.set(f, { mtime: m, md5: h });
233
519
  }
520
+ lastFingerprint = fingerprintOf(fresh.tree, freshHashes);
234
521
  }
235
- lastFingerprint = treeContentFingerprint(scanTree());
236
522
  } else if (!pending && changed.length > 0 && fingerprint === lastFingerprint) {
237
523
  pending = true;
238
524
  lastChangeAt = Date.now();
239
525
  log(`watch: tree differs from index (no motion) — settling then re-index…`);
240
526
  } else if (pending && changed.length === 0 && Date.now() - lastChangeAt < watchDelay) {
241
527
  pending = false;
528
+ } else if (pending && changed.length === 0) {
529
+ // settled but nothing actually differs (e.g. a locked file became
530
+ // readable again with unchanged content) — resync the baseline
531
+ pending = false;
532
+ lastFingerprint = fingerprint;
242
533
  }
243
534
  }
244
535
  };
@@ -291,9 +582,127 @@ export function isAuthorizedToken(token?: string | null, expectedKey?: string |
291
582
  return crypto.timingSafeEqual(a, b);
292
583
  }
293
584
 
585
+ /** True when `host` binds to loopback only (issue #121). Empty/undefined means the loopback default. */
586
+ export function isLoopbackHost(host?: string | null): boolean {
587
+ if (!host) return true;
588
+ const h = String(host).trim().toLowerCase();
589
+ return (
590
+ h === 'localhost' ||
591
+ h === '127.0.0.1' ||
592
+ h === '::1' ||
593
+ h === '[::1]' ||
594
+ h === '::ffff:127.0.0.1' ||
595
+ h === '[::ffff:127.0.0.1]'
596
+ );
597
+ }
598
+
599
+ /**
600
+ * Refuse to start an unauthenticated server on a non-loopback interface (issue #121):
601
+ * `mdss serve --host 0.0.0.0` without an API key silently serves the whole knowledge
602
+ * base to the LAN. Explicit `--allow-unsecured` opts in (banner still printed by CLI).
603
+ */
604
+ export function validateBindSecurity(opts: { host?: string; hasApiKey: boolean; allowUnsecured?: boolean }): void {
605
+ const { host, hasApiKey, allowUnsecured } = opts;
606
+ if (hasApiKey || allowUnsecured || isLoopbackHost(host)) return;
607
+ throw new Error(
608
+ `serve: refusing to bind non-loopback host "${host}" without authentication.\n` +
609
+ ' This would expose the entire knowledge base to the network.\n' +
610
+ ' Fix one of:\n' +
611
+ ' --api-key <key> require Bearer auth on every request\n' +
612
+ ' --api-key-file <path> read the key from a file (or MDSS_API_KEY_FILE)\n' +
613
+ ' --allow-unsecured explicit opt-in (e.g. behind a firewall/reverse proxy)'
614
+ );
615
+ }
616
+
617
+ /**
618
+ * Read an API key from a file (issue #121): avoids shell-history / CI-log leaks of
619
+ * literal `--api-key` values. Trailing newlines are trimmed; on POSIX a group- or
620
+ * world-readable key file draws a loud warning.
621
+ */
622
+ export function loadApiKeyFile(filePath: string, log?: (msg: string) => void): string {
623
+ let raw: string;
624
+ try {
625
+ raw = fs.readFileSync(filePath, 'utf8');
626
+ } catch (e: any) {
627
+ throw new Error(`api-key-file: cannot read ${filePath}: ${e.message}`);
628
+ }
629
+ if (process.platform !== 'win32') {
630
+ try {
631
+ const mode = fs.statSync(filePath).mode & 0o777;
632
+ if (mode & 0o077) {
633
+ log?.(`api-key-file: WARNING ${filePath} is readable by group/others (mode ${mode.toString(8)}) — chmod 600 recommended`);
634
+ }
635
+ } catch {
636
+ /* stat raced after a successful read — not fatal */
637
+ }
638
+ }
639
+ const key = raw.replace(/[\r\n]+$/, '');
640
+ if (!key) throw new Error(`api-key-file: ${filePath} contains no key`);
641
+ return key;
642
+ }
643
+
644
+ /**
645
+ * Strip the port from a Host header value, keeping IPv6 bracket forms intact:
646
+ * "localhost:8747" → "localhost", "[::1]:8747" → "[::1]", "::1" → "::1" (issue #120).
647
+ */
648
+ export function splitHostHeader(hostHeader: string): string {
649
+ const s = String(hostHeader).trim().toLowerCase();
650
+ const m = s.match(/^(\[[^\]]+\])(?::\d+)?$/);
651
+ if (m) return m[1];
652
+ const first = s.indexOf(':');
653
+ const last = s.lastIndexOf(':');
654
+ // a single colon separates host:port; several colons mean a bare IPv6 literal
655
+ return first !== -1 && first === last ? s.slice(0, first) : s;
656
+ }
657
+
658
+ /**
659
+ * Reflect CORS headers ONLY for an exact allowlisted Origin (issue #120).
660
+ * Off entirely by default; `Vary: Origin` is set whenever CORS is configured so
661
+ * shared caches never hand a cross-origin response to another origin.
662
+ */
663
+ function applyCors(req: http.IncomingMessage, res: http.ServerResponse, origins?: string[]): void {
664
+ if (!origins || origins.length === 0) return;
665
+ res.setHeader('Vary', 'Origin');
666
+ const origin = req.headers.origin;
667
+ if (typeof origin === 'string' && origin !== 'null' && origins.includes(origin)) {
668
+ res.setHeader('Access-Control-Allow-Origin', origin);
669
+ }
670
+ }
671
+
294
672
  async function handleRequest(req: http.IncomingMessage, res: http.ServerResponse, state: ServeState): Promise<void> {
295
673
  const url = new URL(req.url || '/', 'http://localhost');
296
674
 
675
+ // DNS-rebinding protection (issue #120): after a rebind attack the browser's
676
+ // Host header names the attacker's domain instead of ours — reject it with 403
677
+ // BEFORE anything else so a rebound page is never treated as same-origin.
678
+ const hostHeader = req.headers.host;
679
+ const hostName = typeof hostHeader === 'string' ? splitHostHeader(hostHeader) : '';
680
+ const allowedHosts = new Set<string>([
681
+ ...DEFAULT_ALLOWED_HOSTS,
682
+ ...(state.allowedHosts || []).map(splitHostHeader),
683
+ ]);
684
+ if (!hostName || !allowedHosts.has(hostName)) {
685
+ json(res, 403, { error: `untrusted Host header${hostHeader ? ` "${hostHeader}"` : ' (missing)'}` });
686
+ return;
687
+ }
688
+
689
+ applyCors(req, res, state.corsOrigins);
690
+
691
+ // CORS preflight — only meaningful when --cors-origin is configured.
692
+ if (req.method === 'OPTIONS') {
693
+ if (state.corsOrigins && state.corsOrigins.length > 0) {
694
+ res.writeHead(204, {
695
+ 'access-control-allow-methods': 'GET, POST',
696
+ 'access-control-allow-headers': 'content-type, authorization',
697
+ 'access-control-max-age': '600',
698
+ });
699
+ res.end();
700
+ } else {
701
+ json(res, 405, { error: 'method not allowed' });
702
+ }
703
+ return;
704
+ }
705
+
297
706
  if (state.apiKey) {
298
707
  const isPublicHealth = url.pathname === '/health' && req.method === 'GET' && state.healthPublic;
299
708
  if (!isPublicHealth) {
@@ -307,7 +716,22 @@ async function handleRequest(req: http.IncomingMessage, res: http.ServerResponse
307
716
  }
308
717
  }
309
718
 
719
+ // Streamable HTTP MCP transport (issue #123): same auth/host/rate-limit
720
+ // gates as /search, reusing handleMcpRequest untouched as the tool brain.
721
+ if (state.mcp && url.pathname === '/mcp') {
722
+ await handleMcpHttp(req, res, state);
723
+ return;
724
+ }
725
+
310
726
  if (req.method === 'POST' && url.pathname === '/search') {
727
+ // Fairness gate 1 (issue #119): per-client token bucket → 429 + Retry-After.
728
+ const limited = state.limiter?.gate(req.socket.remoteAddress || 'unknown');
729
+ if (limited) {
730
+ res.setHeader('retry-after', String(limited.retryAfterSec));
731
+ json(res, 429, { error: 'rate limit exceeded — slow down', retryAfterSec: limited.retryAfterSec });
732
+ return;
733
+ }
734
+
311
735
  const declared = req.headers['content-length'];
312
736
  if (declared !== undefined) {
313
737
  const n = Number(declared);
@@ -348,6 +772,14 @@ async function handleRequest(req: http.IncomingMessage, res: http.ServerResponse
348
772
  }
349
773
  const k = Number.isInteger(payload.k) && payload.k > 0 ? payload.k : 6;
350
774
 
775
+ // Fairness gate 2 (issue #119): one global slot around the model sweep;
776
+ // a saturated wait queue sheds load with 503 instead of ballooning.
777
+ const release = state.limiter ? await state.limiter.acquireSlot() : null;
778
+ if (!release) {
779
+ res.setHeader('retry-after', '1');
780
+ json(res, 503, { error: 'server busy — search concurrency cap reached, queue full' });
781
+ return;
782
+ }
351
783
  try {
352
784
  const results = await searchIndex({
353
785
  loaded: state.loaded,
@@ -371,10 +803,13 @@ async function handleRequest(req: http.IncomingMessage, res: http.ServerResponse
371
803
  nprobe: payload.nprobe,
372
804
  graphBoost: payload.graphBoost,
373
805
  filter: payload.filter,
806
+ recency: typeof payload.recency === 'number' && payload.recency > 0 ? payload.recency : undefined,
374
807
  });
375
808
  json(res, 200, { query, k, count: results.length, results });
376
809
  } catch (e: any) {
377
810
  json(res, 500, { error: e.message });
811
+ } finally {
812
+ release();
378
813
  }
379
814
  return;
380
815
  }
@@ -388,10 +823,30 @@ async function handleRequest(req: http.IncomingMessage, res: http.ServerResponse
388
823
  built: state.loaded.index.built || null,
389
824
  watching: state.watching,
390
825
  indexDir: state.indexDir,
826
+ // fairness counters (issue #119)
827
+ in_flight: state.limiter?.inFlight ?? 0,
828
+ queued: state.limiter?.queued ?? 0,
829
+ rejected_total: state.limiter?.rejectedTotal ?? 0,
830
+ rate_limit_per_min: state.limiter?.rateEnabled ? state.limiter.rateLimitPerMin : null,
831
+ max_concurrency: state.limiter?.concurrencyEnabled ? state.limiter.maxConcurrency : null,
391
832
  });
392
833
  return;
393
834
  }
394
835
 
836
+ if (req.method === 'GET' && url.pathname === '/' && state.ui) {
837
+ // Built-in web UI (issue #111): strict CSP, split assets, no inline code.
838
+ sendAsset(res, 'text/html; charset=utf-8', WEBUI_HTML);
839
+ return;
840
+ }
841
+ if (req.method === 'GET' && url.pathname === '/ui.js' && state.ui) {
842
+ sendAsset(res, 'text/javascript; charset=utf-8', WEBUI_JS);
843
+ return;
844
+ }
845
+ if (req.method === 'GET' && url.pathname === '/ui.css' && state.ui) {
846
+ sendAsset(res, 'text/css; charset=utf-8', WEBUI_CSS);
847
+ return;
848
+ }
849
+
395
850
  if (req.method === 'GET' && (url.pathname === '/' || url.pathname === '/help')) {
396
851
  json(res, 200, {
397
852
  name: 'mdss serve',
@@ -406,9 +861,147 @@ async function handleRequest(req: http.IncomingMessage, res: http.ServerResponse
406
861
  json(res, 404, { error: `not found: ${req.method} ${url.pathname}` });
407
862
  }
408
863
 
864
+ /**
865
+ * Streamable HTTP transport for MCP (issue #123): POST carries JSON-RPC
866
+ * (single message or batch), notifications answer 202, GET opens the SSE
867
+ * server-channel, DELETE terminates a session. Responses are plain JSON —
868
+ * the spec allows JSON instead of SSE when nothing is streamed. Session id is
869
+ * minted on first contact and echoed back on every reply.
870
+ */
871
+ async function handleMcpHttp(req: http.IncomingMessage, res: http.ServerResponse, state: ServeState): Promise<void> {
872
+ if (req.method === 'DELETE') {
873
+ res.writeHead(204);
874
+ res.end();
875
+ return;
876
+ }
877
+
878
+ if (req.method === 'GET') {
879
+ // Server-initiated channel: we have nothing to push today, so keep the
880
+ // stream alive with comments until the client goes away.
881
+ res.writeHead(200, {
882
+ 'content-type': 'text/event-stream',
883
+ 'cache-control': 'no-store',
884
+ 'connection': 'keep-alive',
885
+ });
886
+ res.write(':ok\n\n');
887
+ const hb = setInterval(() => {
888
+ try {
889
+ res.write(':hb\n\n');
890
+ } catch {
891
+ /* closed */
892
+ }
893
+ }, 15_000);
894
+ hb.unref?.();
895
+ req.on('close', () => clearInterval(hb));
896
+ return;
897
+ }
898
+
899
+ if (req.method !== 'POST') {
900
+ json(res, 405, { error: 'method not allowed' });
901
+ return;
902
+ }
903
+
904
+ // Same fairness gate as /search — tools/call can trigger embedding work.
905
+ const limited = state.limiter?.gate(req.socket.remoteAddress || 'unknown');
906
+ if (limited) {
907
+ res.setHeader('retry-after', String(limited.retryAfterSec));
908
+ json(res, 429, { error: 'rate limit exceeded - slow down', retryAfterSec: limited.retryAfterSec });
909
+ return;
910
+ }
911
+ const release = state.limiter ? await state.limiter.acquireSlot() : null;
912
+ if (!release) {
913
+ res.setHeader('retry-after', '1');
914
+ json(res, 503, { error: 'server busy' });
915
+ return;
916
+ }
917
+
918
+ try {
919
+ let body = '';
920
+ for await (const chunk of req) {
921
+ body += chunk;
922
+ if (body.length > MAX_BODY_BYTES) {
923
+ json(res, 413, { error: `payload too large (limit ${MAX_BODY_BYTES} bytes)` });
924
+ return;
925
+ }
926
+ }
927
+ let messages: any;
928
+ try {
929
+ messages = JSON.parse(body || '{}');
930
+ } catch (e: any) {
931
+ json(res, 400, { error: `invalid JSON: ${e.message}` });
932
+ return;
933
+ }
934
+ const batch = Array.isArray(messages);
935
+ const items = batch ? messages : [messages];
936
+
937
+ const responses: any[] = [];
938
+ for (const msg of items) {
939
+ if (!msg || typeof msg !== 'object') continue;
940
+ const isNotification = msg.id === undefined || msg.id === null;
941
+ const result = isNotification ? null : await handleMcpRequest(msg, {
942
+ loaded: state.loaded,
943
+ cacheDir: state.cacheDir,
944
+ offline: state.offline,
945
+ embedFn: state.embedFn,
946
+ });
947
+ if (result) responses.push(result);
948
+ else if (!isNotification && msg.method === undefined) continue;
949
+ }
950
+
951
+ // session id: mint on first contact, echo afterwards
952
+ const requested = req.headers['mcp-session-id'];
953
+ if (typeof requested === 'string' && requested) state.mcpSessionId = requested;
954
+ if (!state.mcpSessionId) state.mcpSessionId = crypto.randomUUID();
955
+ res.setHeader('mcp-session-id', state.mcpSessionId);
956
+
957
+ // protocol negotiation: honour a supported requested version, else latest
958
+ for (const resp of responses) {
959
+ if (resp?.result?.protocolVersion && msg0(items)?.params?.protocolVersion) {
960
+ const wanted = String(msg0(items).params.protocolVersion);
961
+ const SUPPORTED = ['2025-03-26', '2024-11-05'];
962
+ resp.result.protocolVersion = SUPPORTED.includes(wanted) ? wanted : SUPPORTED[0];
963
+ }
964
+ }
965
+
966
+ if (responses.length === 0) {
967
+ res.writeHead(202);
968
+ res.end();
969
+ return;
970
+ }
971
+ json(res, 200, batch ? responses : responses[0]);
972
+ } finally {
973
+ release();
974
+ }
975
+ }
976
+
977
+ /** First object of a (possibly single-message) payload — for init negotiation. */
978
+ function msg0(items: any[]): any | null {
979
+ return items.length > 0 ? items[0] : null;
980
+ }
981
+
409
982
  function json(res: http.ServerResponse, status: number, data: object): void {
410
- res.writeHead(status, { 'content-type': 'application/json; charset=utf-8' });
983
+ res.writeHead(status, {
984
+ 'content-type': 'application/json; charset=utf-8',
985
+ // Security headers on EVERY response (issue #120)
986
+ 'x-content-type-options': 'nosniff',
987
+ 'cache-control': 'no-store',
988
+ });
411
989
  res.end(JSON.stringify(data));
412
990
  }
413
991
 
992
+ /** Serve a web-UI asset with the same security headers + a strict CSP (issue #111). */
993
+ function sendAsset(res: http.ServerResponse, contentType: string, body: string): void {
994
+ res.writeHead(200, {
995
+ 'content-type': contentType,
996
+ 'x-content-type-options': 'nosniff',
997
+ 'cache-control': 'no-store',
998
+ // no inline script/style anywhere in the UI -> nothing needs unsafe-*
999
+ 'content-security-policy':
1000
+ "default-src 'none'; script-src 'self'; style-src 'self'; connect-src 'self'; img-src 'self'; base-uri 'none'; frame-ancestors 'none'",
1001
+ 'referrer-policy': 'no-referrer',
1002
+ });
1003
+ res.end(body);
1004
+ }
1005
+
414
1006
  export { DEFAULT_PORT, DEFAULT_HOST, MAX_BODY_BYTES, WATCH_INTERVAL_MS };
1007
+