md-semantic-search 1.0.2 → 1.0.3

package/bin/cli.mjs

@@ -18,9 +18,14 @@ import crypto from 'node:crypto';
 import { buildIndex } from '../dist/indexer.js';
 import { search } from '../dist/search.js';
 import { searchFederated } from '../dist/federation.js';
-import { createServe, DEFAULT_PORT, DEFAULT_HOST } from '../dist/serve.js';
+import { createServe, DEFAULT_PORT, DEFAULT_HOST, isLoopbackHost, validateBindSecurity, loadApiKeyFile } from '../dist/serve.js';
 import { MODELS, DEFAULT_MODEL, resolveModel } from '../dist/models.js';
-import { decodeVec, walkMarkdown, SCHEMA_VERSION, assertSafePath } from '../dist/core.js';
+import { decodeVec, walkMarkdown, SCHEMA_VERSION, assertSafePath, setDownloadProgressListener } from '../dist/core.js';
+import { DownloadProgressAggregator } from '../dist/download-progress.js';
+import { generateCompletion, COMPLETION_SHELLS } from '../dist/completions.js';
+import { openHit, resolveOpenCommand } from '../dist/open.js';
+import { planRepairs, applyRepairs } from '../dist/repair.js';
+import { resolveExternalEmbedder } from '../dist/providers.js';
 import { inspectIndexSchema, validateCurrentChunk, validateIndexEnvelope, validateNumericVector } from '../dist/index-format.js';
 
 const HERE = path.dirname(fileURLToPath(import.meta.url));
@@ -49,6 +54,7 @@ function parseArgs(argv) {
     else if (a === '--semantic') opts.semantic = true;
     else if (a === '--offline') opts.offline = true;
     else if (a === '--watch') opts.watch = true;
+    else if (a === '--watch-debug') opts.watchDebug = true;
     else if (a === '--rerank') opts.rerank = true;
     else if (a === '--explain') opts.explain = true;
     else if (a === '--interactive' || a === '-i') opts.interactive = true;
@@ -72,9 +78,21 @@ function parseArgs(argv) {
     else if (a === '--port') opts.port = nextInt(argv, ++i, a);
     else if (a === '--host') opts.host = nextValue(argv, ++i, a);
     else if (a === '--api-key') opts.apiKey = nextValue(argv, ++i, a);
+    else if (a === '--api-key-file') opts.apiKeyFile = nextValue(argv, ++i, a);
+    else if (a === '--allow-unsecured') opts.allowUnsecured = true;
+    else if (a === '--allowed-host') {
+      const v = nextValue(argv, ++i, a);
+      opts.allowedHost = opts.allowedHost ? [...opts.allowedHost, v] : [v];
+    }
+    else if (a === '--cors-origin') {
+      const v = nextValue(argv, ++i, a);
+      opts.corsOrigin = opts.corsOrigin ? [...opts.corsOrigin, v] : [v];
+    }
     else if (a === '--health-public') opts.healthPublic = true;
     else if (a === '--watch-delay') opts.watchDelay = nextInt(argv, ++i, a);
     else if (a === '--watch-interval') opts.watchInterval = nextInt(argv, ++i, a);
+    else if (a === '--rate-limit') opts.rateLimit = nextNonNegInt(argv, ++i, a);
+    else if (a === '--max-concurrency') opts.maxConcurrency = nextNonNegInt(argv, ++i, a);
     else if (a === '--since') opts.since = nextValue(argv, ++i, a);
     else if (a === '--path') opts.path.push(nextValue(argv, ++i, a));
     else if (a === '--ignore') opts.ignore.push(nextValue(argv, ++i, a));
@@ -97,6 +115,37 @@ function parseArgs(argv) {
     else if (a === '--llm-endpoint') opts.llmEndpoint = nextValue(argv, ++i, a);
     else if (a === '--llm-model') opts.llmModel = nextValue(argv, ++i, a);
     else if (a === '--system-prompt') opts.systemPrompt = nextValue(argv, ++i, a);
+    else if (a === '--open') {
+      // `--open` opens the top hit; `--open N` opens the Nth hit (issue #110).
+      const next = argv[i + 1];
+      if (next !== undefined && /^\d+$/.test(next)) {
+        opts.openRank = Number.parseInt(next, 10);
+        i++;
+      } else {
+        opts.open = true;
+      }
+    }
+    else if (a === '--fix') opts.fix = true;
+    else if (a === '--dry-run') opts.dryRun = true;
+    else if (a === '--no-ui') opts.noUi = true;
+    else if (a === '--no-query-cache') opts.noQueryCache = true;
+    else if (a === '--mcp') opts.mcp = true;
+    else if (a === '--recency') {
+      opts.recency = nextFloat(argv, ++i, a);
+      if (!(opts.recency > 0)) die(`--recency must be a positive half-life in days, got "${opts.recency}"`);
+    }
+    else if (a === '--embedder') opts.embedder = nextValue(argv, ++i, a);
+    else if (a === '--embedder-model') opts.embedderModel = nextValue(argv, ++i, a);
+    else if (a === '--embedder-base-url') opts.embedderBaseUrl = nextValue(argv, ++i, a);
+    else if (a === '--embedder-key-file') opts.embedderKeyFile = nextValue(argv, ++i, a);
+    else if (a === '--expand') {
+      opts.expand = nextValue(argv, ++i, a);
+      if (!['prf', 'hyde'].includes(opts.expand)) die(`unknown --expand "${opts.expand}" - use prf|hyde`);
+    }
+    else if (a === '--expand-passages') {
+      opts.expandPassages = nextInt(argv, ++i, a);
+      if (opts.expandPassages > 20) die(`--expand-passages must be <= 20`);
+    }
     else if (a.startsWith('-')) die(`unknown option: ${a}. Try \`mdss --help\`.`);
     else opts._.push(a);
   }
@@ -126,15 +175,42 @@ function nextFloat(argv, i, flag) {
   return f;
 }
 
+/** Like nextInt but allows 0 (used by serve limit flags where 0 = disabled). */
+function nextNonNegInt(argv, i, flag) {
+  const v = nextValue(argv, i, flag);
+  const n = Number.parseInt(v, 10);
+  if (!Number.isInteger(n) || n < 0) die(`${flag} must be a non-negative integer (0 disables), got "${v}"`);
+  return n;
+}
+
+/** Non-negative integer from an env var; undefined when unset/invalid. */
+function envNonNegInt(name) {
+  const raw = process.env[name];
+  if (!raw) return undefined;
+  const n = Number.parseInt(raw, 10);
+  return Number.isInteger(n) && n >= 0 ? n : undefined;
+}
+
 const KNOWN_CONFIG_KEYS = new Set([
   'db', 'model', 'indexDir', 'index-dir', 'cacheDir', 'cache-dir',
   'ignore', 'path', 'k', 'maxPerFile', 'max-per-file', 'maxPerDoc', 'max-per-doc',
   'targetTokens', 'target-tokens', 'port', 'host', 'apiKey', 'api-key',
+  'apiKeyFile', 'api-key-file', 'allowUnsecured', 'allow-unsecured',
+  'allowedHost', 'allowed-host', 'allowedHosts', 'allowed-hosts',
+  'corsOrigin', 'cors-origin',
+  'rateLimit', 'rate-limit', 'maxConcurrency', 'max-concurrency',
   'healthPublic', 'health-public', 'watch', 'watchInterval', 'watch-interval',
   'watchDelay', 'watch-delay', 'offline', 'rerank', 'semantic', 'tag', 'project', 'type', 'status', 'canonical',
+  'watchDebug', 'watch-debug',
   'format', 'noVectors', 'no-vectors', 'output', 'workers', 'ann', 'nprobe', 'graphBoost', 'graph-boost', 'filter', 'quantize',
   'vault', 'vaults', 'rag', 'autoTag', 'auto-tag', 'autoSummarize', 'auto-summarize',
-  'llmEndpoint', 'llm-endpoint', 'llmModel', 'llm-model', 'systemPrompt', 'system-prompt'
+  'llmEndpoint', 'llm-endpoint', 'llmModel', 'llm-model', 'systemPrompt', 'system-prompt',
+  'completions', 'open', 'dry-run', 'fix', 'yes', 'no-ui', 'noUi',
+  'noQueryCache', 'no-query-cache',
+  'mcp',
+  'embedder', 'embedder-model', 'embedder-base-url', 'embedder-key-file',
+  'embedderModel', 'embedderBaseUrl', 'embedderKeyFile',
+  'expand', 'expandPassages', 'expand-passages', 'llm-endpoint', 'llmEndpoint'
 ]);
 
 function findConfigFile(explicitPath) {
@@ -255,6 +331,7 @@ Usage:
   mdss serve  --db <dir> [--port <n>] [--host <ip>] [--watch]  Daemon: warm model + index
   mdss mcp    --db <dir> [--list-tools]       Start MCP server over stdio for LLM agents / IDEs
   mdss models                                  List available models
+  mdss completions <bash|zsh|fish|powershell>  Print a self-contained shell completion script
 
 Options:
   --config <file>     Path to config file (default: .mdssrc.json / mdss.config.json).
@@ -280,11 +357,64 @@ Options:
   --port <n>          HTTP port for serve (default: ${DEFAULT_PORT}).
   --host <ip>         Bind address for serve (default: ${DEFAULT_HOST} — loopback
                       only; use 0.0.0.0 to expose on the LAN, env MDSS_HOST).
+  --api-key <key>     Require Bearer auth on every request (env MDSS_API_KEY).
+  --api-key-file <p>  Read the API key from a file instead of a literal value —
+                      avoids shell-history/CI-log leaks (env MDSS_API_KEY_FILE;
+                      trailing newline trimmed, POSIX chmod-checked).
+  --allow-unsecured   Explicit opt-in to serve non-loopback WITHOUT auth — a loud
+                      banner is printed; without it (and without an api key) a
+                      non-loopback bind refuses to start (issue #121).
+  --allowed-host <h>  Extra Host header value to accept (repeatable) — requests
+                      naming any other Host get 403 (DNS-rebinding guard, issue #120).
+  --cors-origin <o>   Allow cross-origin browser access from this exact origin
+                      (repeatable); CORS is off unless set, and only exact
+                      matches are reflected with Vary: Origin (issue #120).
+  --rate-limit <n>    serve: max /search requests per minute per client, burst
+                      10 → 429 + Retry-After above it; 0 disables (default 60,
+                      env MDSS_RATE_LIMIT; issue #119).
+  --max-concurrency <n>  serve: max concurrent in-flight searches, queue cap
+                      2× → 503 when full; 0 disables (default = CPU count,
+                      env MDSS_MAX_CONCURRENCY; issue #119).
   --watch             serve: re-index incrementally on file changes (mtime poll).
+  --no-ui             serve: disable the built-in web UI at the root path
+                      (issue #111) - the JSON API help is served there instead.
+  --mcp               serve: also mount Streamable HTTP MCP at /mcp
+                      (issue #123) - same auth/host/rate-limit gates as
+                      /search; stdio via the mcp command remains default.
+  --watch-debug       serve --watch: verbose trace of polls, FS-error
+                      classifications and retries (issue #116).
   --watch-interval <ms>  serve --watch: poll every N ms (default 3000).
   --watch-delay <ms>     serve --watch: quiet-period debounce before a burst of
                          saves triggers ONE re-index (default 1000; issue #42).
   --offline           Never download the model; require a cached one (env MDSS_OFFLINE=1).
+  --no-query-cache    search: skip the persistent query-embedding cache
+                      (<cacheDir>/query-cache.json, issue #114); in-memory
+                      caching stays on.
+  --recency <days>    search: time-decay half-life — fresher passages get
+                      0.5^(age/halfLife) boost post-fusion (frontmatter
+                      created/updated, then file mtime; issue #127).
+  --embedder <name>   Use an EXTERNAL embedding provider instead of local
+                      transformers.js: "ollama" or "openai" (issue #124).
+                      Local stays the zero-config default and the only
+                      fully-offline path.
+  --embedder-model <m>   Provider-side model (e.g. nomic-embed-text).
+  --embedder-base-url <u>  Endpoint (default: OLLAMA_HOST /
+                      http://127.0.0.1:11434 for ollama,
+                      https://api.openai.com/v1 for openai).
+  --embedder-key-file <p>  Bearer key file for openai-compatible endpoints
+                      (or OPENAI_API_KEY env).
+  --expand <mode>     search: query expansion - "prf" (offline pseudo-
+                      relevance feedback: top passages feed salient terms,
+                      ONE embed total; issue #125) or "hyde" (LLM-generated
+                      passage via --llm-endpoint; degrades silently without it).
+  --expand-passages <n>  prf feedback pool size (default 3, max 20).
+  --open [N]          search: open the top (or Nth) hit in your editor at its
+                      startLine — MDSS_EDITOR/VISUAL/EDITOR, then VS Code
+                      --goto, then the GUI opener (issue #110). With --json on
+                      a non-TTY prints what WOULD be opened instead of launching.
+  --fix               check: apply safe auto-repairs (stale lock, broken
+                      vectors.bin sidecar, stale ivf.json); never touches notes.
+  --dry-run           check --fix: print planned repairs without mutating.
   --version           Print the version and exit.
   -h, --help          Show this help.
 
@@ -306,6 +436,35 @@ function formatDuration(sec) {
   return `${h}:${String(m).padStart(2, '0')}:${String(remainingS).padStart(2, '0')}`;
 }
 
+/**
+ * External embedder wiring (issue #124): when --embedder is set, build the
+ * synthetic adapter + embedFn and pass BOTH into the pipeline. Provider
+ * mismatch across rebuilds surfaces through the existing adapter-fingerprint
+ * invalidation, with the index `model` field naming the provider explicitly.
+ */
+async function resolveEmbedderOverrides(opts) {
+  if (!opts.embedder) return {};
+  if (!['ollama', 'openai'].includes(opts.embedder)) {
+    die(`unknown --embedder "${opts.embedder}" - use "ollama" or "openai"`);
+  }
+  if (!opts.embedderModel) {
+    die(`--embedder ${opts.embedder} requires --embedder-model <name>`);
+  }
+  const emb = resolveExternalEmbedder({
+    embedder: opts.embedder,
+    model: opts.embedderModel,
+    baseUrl: opts.embedderBaseUrl,
+    keyFile: opts.embedderKeyFile,
+  });
+  const descriptor = await emb.descriptor();
+  logErr(`embedder: ${emb.id}/${emb.model} (dim ${descriptor.dim} via ${opts.embedderBaseUrl || 'default endpoint'})`);
+  return { modelName: descriptor, embedFn: emb.embedFn };
+}
+
+function logErr(s) {
+  process.stderr.write(s + '\n');
+}
+
 async function cmdIndex(opts) {
   const db = resolveDb(opts);
   const indexDir = resolveIndexDir(opts, db);
@@ -333,11 +492,14 @@ async function cmdIndex(opts) {
     }
   };
 
+  const embedderOverrides = await resolveEmbedderOverrides(opts);
+
   let r;
   try {
     r = await buildIndex({
       db, indexDir, cacheDir,
-      modelName: opts.model || DEFAULT_MODEL,
+      modelName: embedderOverrides.modelName || opts.model || DEFAULT_MODEL,
+      ...(embedderOverrides.embedFn ? { embedFn: embedderOverrides.embedFn } : {}),
       ignore: opts.ignore,
       offline: resolveOffline(opts),
       workers: opts.workers,
@@ -777,7 +939,7 @@ const modelId = index.model ?? null;
  * embedding model or touching the network. Exit code 0 when healthy, 1 with a
  * summary when not; `--json` for scripting.
  */
-function cmdCheck(opts) {
+async function cmdCheck(opts) {
   const db = resolveDb(opts);
   const indexDir = resolveIndexDir(opts, db);
   const cacheDir = resolveCache(opts);
@@ -787,6 +949,46 @@ function cmdCheck(opts) {
     config: opts._config,
   });
 
+  // --fix (issue #117): plan safe repairs, optionally apply them. Never
+  // touches source Markdown; --dry-run prints the plan without mutating.
+  if (opts.fix) {
+    const plan = planRepairs(indexDir, { db: fs.existsSync(db) ? db : null });
+    const result = await applyRepairs(indexDir, plan, {
+      dryRun: !!opts.dryRun,
+      db: fs.existsSync(db) ? db : null,
+      cacheDir,
+      modelName: opts.model || undefined,
+      log: (s) => process.stderr.write(s + '\n'),
+    });
+    report.fix = {
+      dryRun: !!opts.dryRun,
+      planned: plan.actions,
+      performed: result.performed,
+      skipped: plan.skipped,
+      failed: result.failed,
+    };
+
+    if (opts.json) {
+      process.stdout.write(JSON.stringify(report, null, 2) + '\n');
+      process.exitCode = result.failed.length > 0 ? 1 : 0;
+      return;
+    }
+
+    let out = '';
+    for (const a of (opts.dryRun ? plan.actions : result.performed)) {
+      out += `  ${opts.dryRun ? 'would' : 'fixed'} ${a.action}: ${a.detail}\n`;
+    }
+    for (const s of plan.skipped) out += `  skip  ${s.action}: ${s.detail}\n`;
+    for (const f of result.failed) out += `  FAIL  ${f.action}: ${f.error}\n`;
+    if (plan.actions.length === 0 && plan.skipped.length === 0) out += '  ok    nothing to repair\n';
+    process.stdout.write(out);
+    process.exitCode = result.failed.length > 0 ? 1 : 0;
+    if (!opts.dryRun && result.performed.length > 0) {
+      process.stdout.write('check: repairs applied — run `mdss check` to verify (issue #117)\n');
+    }
+    return;
+  }
+
   if (opts.json) {
     process.stdout.write(JSON.stringify(report, null, 2) + '\n');
     process.exitCode = report.healthy ? 0 : 1;
@@ -933,17 +1135,23 @@ async function cmdSearch(opts) {
   const query = opts._.join(' ').trim();
   if (!query) die('Missing query text. e.g. mdss search --db ./docs "your question"');
 
+  const embedderOverrides = await resolveEmbedderOverrides(opts);
   let results;
   const isFederated = Array.isArray(opts.vaults) && opts.vaults.length > 0;
 
   if (isFederated) {
     results = await searchFederated({
+      ...(embedderOverrides.embedFn ? { embedFn: embedderOverrides.embedFn } : {}),
       vaults: opts.vaults,
       cacheDir,
       query,
       k: opts.k || 6,
       semanticOnly: !!opts.semantic,
       offline: resolveOffline(opts),
+      queryDiskCache: !opts.noQueryCache,
+      recency: opts.recency,
+      expand: opts.expand,
+      expandPassages: opts.expandPassages,
       path: opts.path.length > 0 ? opts.path : undefined,
       since: opts.since,
       rerank: !!opts.rerank,
@@ -964,10 +1172,16 @@ async function cmdSearch(opts) {
     const indexDir = resolveIndexDir(opts, db);
 
     results = await search({
+      ...(embedderOverrides.modelName ? { model: embedderOverrides.modelName } : {}),
+      ...(embedderOverrides.embedFn ? { embedFn: embedderOverrides.embedFn } : {}),
       indexDir, cacheDir, query,
       k: opts.k || 6,
       semanticOnly: !!opts.semantic,
       offline: resolveOffline(opts),
+      queryDiskCache: !opts.noQueryCache,
+      recency: opts.recency,
+      expand: opts.expand,
+      expandPassages: opts.expandPassages,
       path: opts.path.length > 0 ? opts.path : undefined,
       since: opts.since,
       rerank: !!opts.rerank,
@@ -988,6 +1202,12 @@ async function cmdSearch(opts) {
   if (opts.json) { process.stdout.write(JSON.stringify(results, null, 2) + '\n'); return; }
   if (results.length === 0) { process.stdout.write('No matches.\n'); return; }
 
+  // --open (issue #110): jump into the editor at the hit's exact line.
+  if (opts.open || opts.openRank) {
+    await openSearchHit(opts, results);
+    return;
+  }
+
   process.stdout.write(`\nTop ${results.length} for: "${query}"\n\n`);
 
   // Group by file (issue #13): each file becomes a header with its hits below.
@@ -1016,9 +1236,50 @@ async function cmdSearch(opts) {
   }
 }
 
+/**
+ * --open (issue #110): resolve the Nth hit to an absolute path and jump to it
+ * in the user's editor at startLine. With --json on a non-TTY nothing is
+ * launched — the intent is printed as a JSON line on stderr instead, keeping
+ * stdout machine-readable.
+ */
+async function openSearchHit(opts, results) {
+  const rank = opts.openRank || 1;
+  if (!Number.isInteger(rank) || rank < 1) die(`--open N must be a positive hit number, got ${rank}`);
+  const hit = results[rank - 1];
+  if (!hit) {
+    die(`--open ${rank}: only ${results.length} hit(s) available`);
+  }
+
+  // Resolve relative hit paths against the right base (db or the matched vault).
+  let base = null;
+  if (hit.vault && Array.isArray(opts.vaults)) {
+    base = opts.vaults.find((v) => v === hit.vault || path.basename(v) === hit.vault) || null;
+  }
+  if (!base) base = resolveDb(opts);
+  const absFile = path.isAbsolute(hit.file) ? hit.file : path.join(base, hit.file);
+  const target = { file: absFile, line: hit.startLine };
+  const envEditor = process.env.MDSS_EDITOR || process.env.VISUAL || process.env.EDITOR || undefined;
+
+  // --json on a non-TTY: print what WOULD be launched, launch nothing (issue #110).
+  if (opts.json && !process.stdout.isTTY) {
+    const would = resolveOpenCommand(target, { editor: envEditor, platform: process.platform });
+    if (!would) die(`could not resolve an editor for ${absFile} — set $MDSS_EDITOR/$VISUAL/$EDITOR`);
+    process.stderr.write(JSON.stringify({
+      type: 'open-intent',
+      file: absFile,
+      line: hit.startLine ?? null,
+      via: would.via,
+    }) + '\n');
+    return;
+  }
+
+  const resolved = openHit(target, { editor: envEditor });
+  if (!resolved) die(`could not resolve an editor for ${absFile} — set $MDSS_EDITOR/$VISUAL/$EDITOR`);
+  process.stderr.write(`→ opened ${absFile}${hit.startLine ? `:${hit.startLine}` : ''} via ${resolved.via}\n`);
+}
+
 /** Escape a string for use inside a RegExp. */
-function escRe(s) {
-  return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+function escRe(s) {  return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
 }
 
 function cmdModels() {
@@ -1052,16 +1313,47 @@ async function cmdServe(opts) {
   const host = opts.host || process.env.MDSS_HOST || DEFAULT_HOST;
   const log = s => process.stderr.write(s + '\n');
 
+  const embedderOverrides = await resolveEmbedderOverrides(opts);
+
+  // API key precedence (issue #121): literal flag/env first, then key file —
+  // the file form avoids shell-history and CI-log leaks of the literal value.
+  let apiKey = opts.apiKey || process.env.MDSS_API_KEY || undefined;
+  const keyFile = opts.apiKeyFile || process.env.MDSS_API_KEY_FILE;
+  if (!apiKey && keyFile) {
+    apiKey = loadApiKeyFile(keyFile, log);
+    log(`api-key-file: key loaded from ${keyFile}`);
+  }
+
+  // Refuse to serve the whole knowledge base to the network unauthenticated (issue #121).
+  validateBindSecurity({ host, hasApiKey: !!apiKey, allowUnsecured: !!opts.allowUnsecured });
+
   const { server, state, close } = await createServe({
     db, indexDir, cacheDir,
-    modelName: opts.model || DEFAULT_MODEL,
+    modelName: embedderOverrides.modelName || opts.model || DEFAULT_MODEL,
     ignore: opts.ignore,
     offline: resolveOffline(opts),
+    ...(embedderOverrides.embedFn ? { embedFn: embedderOverrides.embedFn } : {}),
     watch: !!opts.watch,
     watchInterval: opts.watchInterval,
     watchDelay: opts.watchDelay,
-    apiKey: opts.apiKey || process.env.MDSS_API_KEY,
+    watchDebug: !!opts.watchDebug,
+    apiKey,
     healthPublic: !!opts.healthPublic || process.env.MDSS_HEALTH_PUBLIC === 'true',
+    // Web UI on by default at `/` (issue #111); --no-ui restores pure JSON.
+    ui: !opts.noUi,
+    mcp: !!opts.mcp,
+    // DNS-rebinding allowlist (issue #120): loopback names are always allowed;
+    // the configured bind host is added so `--host mybox.local` still answers.
+    allowedHosts: [
+      ...(opts.allowedHost || []),
+      ...(opts.host ? [opts.host] : []),
+      ...(process.env.MDSS_HOST ? [process.env.MDSS_HOST] : []),
+    ],
+    // CORS stays OFF unless explicitly opted in (issue #120).
+    corsOrigins: opts.corsOrigin,
+    // Fairness gates (issue #119); 0 explicitly disables a gate.
+    rateLimit: opts.rateLimit ?? envNonNegInt('MDSS_RATE_LIMIT'),
+    maxConcurrency: opts.maxConcurrency ?? envNonNegInt('MDSS_MAX_CONCURRENCY'),
     log,
   });
 
@@ -1073,6 +1365,14 @@ async function cmdServe(opts) {
       (state.watching ? ' · watching for changes' : ''));
     log(`  POST /search  {"query": "...", "k": 6}`);
     log(`  GET  /health`);
+    if (!isLoopbackHost(host) && !state.apiKey) {
+      log('');
+      log('  ⚠⚠⚠  UNSECURED NETWORK BIND — EXPLICIT OPT-IN (--allow-unsecured)  ⚠⚠⚠');
+      log('  ⚠⚠⚠  The ENTIRE knowledge base is readable by anyone who can     ⚠⚠⚠');
+      log('  ⚠⚠⚠  reach this port. Put a reverse proxy with TLS + auth in     ⚠⚠⚠');
+      log('  ⚠⚠⚠  front, or set an API key (--api-key / --api-key-file).       ⚠⚠⚠');
+      log('');
+    }
     log(`  Ctrl+C to stop.`);
   });
 
@@ -1098,6 +1398,8 @@ async function cmdMcp(opts) {
   const { startMcpServer } = await import('../dist/mcp.js');
   await startMcpServer({
     db, indexDir, cacheDir,
+    // NOTE: external embedders (--embedder) are not yet wired into the MCP
+    // stdio server; it keeps the local transformers.js path (issue #124).
     modelName: opts.model || DEFAULT_MODEL,
     ignore: opts.ignore,
     offline: resolveOffline(opts),
@@ -1105,6 +1407,68 @@ async function cmdMcp(opts) {
   });
 }
 
+// ---------------------------------------------------------------------------
+// First-run model download progress (issue #108): the embedding model (~280 MB)
+// and the reranker (~280 MB) stream in silently otherwise, which reads as a
+// hang on slow links. TTY → one aggregated bar; --json → throttled JSON events
+// on STDERR so stdout stays machine-readable; warm cache emits nothing at all.
+// ---------------------------------------------------------------------------
+
+const DOWNLOAD_COMMANDS = new Set(['index', 'search', 'ask', 'serve']);
+
+function formatMb(bytes) {
+  if (!Number.isFinite(bytes) || bytes <= 0) return '0 MB';
+  return bytes >= 1024 * 1024 * 1024
+    ? `${(bytes / (1024 * 1024 * 1024)).toFixed(2)} GB`
+    : `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
+}
+
+function renderDownloadBar(snap) {
+  const barWidth = 10;
+  const filled = snap.percent === null ? 0 : Math.min(barWidth, Math.max(0, Math.round((snap.percent / 100) * barWidth)));
+  const bar = '[' + '#'.repeat(filled) + '-'.repeat(barWidth - filled) + ']';
+  const pct = snap.percent === null ? '…' : `${Math.floor(snap.percent)}%`;
+  const size = `${formatMb(snap.loadedBytes)}${snap.totalBytes > 0 ? `/${formatMb(snap.totalBytes)}` : ''}`;
+  const speed = snap.mbps !== null ? `${snap.mbps.toFixed(1)} MB/s` : '—';
+  const eta = snap.etaSec !== null && Number.isFinite(snap.etaSec) ? formatDuration(snap.etaSec) : '—';
+  process.stderr.write(`\r${bar} ${pct} │ ${size} │ ${speed} │ ETA ${eta}`);
+  if (snap.complete) process.stderr.write('\n');
+}
+
+/** Install the progress listener; returns a detach fn for the caller's finally. */
+function installDownloadProgress(opts) {
+  const agg = new DownloadProgressAggregator();
+  setDownloadProgressListener((e) => {
+    agg.update(e);
+    // JSON consumers poll slower than eyes; throttle accordingly.
+    if (!agg.shouldRender(Date.now(), opts.json ? 1000 : 400)) return;
+    const snap = agg.snapshot();
+    if (!opts.json && process.stderr.isTTY) {
+      renderDownloadBar(snap);
+    } else if (opts.json) {
+      process.stderr.write(JSON.stringify({
+        type: snap.complete ? 'download-complete' : 'download-progress',
+        percent: snap.percent === null ? null : Math.round(snap.percent * 10) / 10,
+        loadedBytes: snap.loadedBytes,
+        totalBytes: snap.totalBytes > 0 ? snap.totalBytes : null,
+        mbps: snap.mbps === null ? null : Math.round(snap.mbps * 100) / 100,
+        etaSec: snap.etaSec === null || !Number.isFinite(snap.etaSec) ? null : Math.round(snap.etaSec),
+        doneFiles: snap.doneFiles,
+        activeFiles: snap.activeFiles,
+      }) + '\n');
+    }
+  });
+  return () => setDownloadProgressListener(null);
+}
+
+async function cmdCompletions(opts) {
+  const shell = (opts._.shift() || '').toLowerCase();
+  if (!COMPLETION_SHELLS.includes(shell)) {
+    die(`unknown shell "${shell || '(none)'}" for completions — use one of: ${COMPLETION_SHELLS.join('|')}`);
+  }
+  process.stdout.write(generateCompletion(shell));
+}
+
 async function main() {
   const argv = process.argv.slice(2);
   const opts = parseArgs(argv);
@@ -1120,19 +1484,28 @@ async function main() {
 
   if (opts.version) { process.stdout.write(`${VERSION}\n`); return; }
   if (opts.help || !cmd) { process.stdout.write(HELP); return; }
-  switch (cmd) {
-    case 'index': return cmdIndex(opts);
-    case 'stats': return cmdStats(opts);
-    case 'check':
-    case 'doctor': return cmdCheck(opts);
-    case 'export': return cmdExport(opts);
-    case 'search': return cmdSearch(opts);
-    case 'ask': return cmdAsk(opts);
-    case 'tui': return cmdTui(opts);
-    case 'serve': return cmdServe(opts);
-    case 'mcp': return cmdMcp(opts);
-    case 'models': return cmdModels();
-    default: die(`unknown command: ${cmd}. Try \`mdss --help\`.`);
+  // Model downloads stream through this listener only for commands that may
+  // load a model in-process (issue #108). `mcp` is excluded — its stderr is an
+  // IDE log; `tui` draws its own full-screen UI.
+  const detachProgress = DOWNLOAD_COMMANDS.has(cmd) ? installDownloadProgress(opts) : null;
+  try {
+    switch (cmd) {
+      case 'index': return await cmdIndex(opts);
+      case 'stats': return await cmdStats(opts);
+      case 'check':
+      case 'doctor': return await cmdCheck(opts);
+      case 'export': return await cmdExport(opts);
+      case 'search': return await cmdSearch(opts);
+      case 'ask': return await cmdAsk(opts);
+      case 'tui': return await cmdTui(opts);
+      case 'serve': return await cmdServe(opts);
+      case 'mcp': return await cmdMcp(opts);
+      case 'models': return await cmdModels();
+      case 'completions': return await cmdCompletions(opts);
+      default: die(`unknown command: ${cmd}. Try \`mdss --help\`.`);
+    }
+  } finally {
+    detachProgress?.();
   }
 }
 
@@ -1172,3 +1545,11 @@ export {
   checkHealth, cmdExport,
   die, main, HELP, VERSION,
 };
+
+
+
+
+
+
+
+

package/dist/completions.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Shell completions generator (issue #107).
+ *
+ * Single source of truth: the FLAGS table below drives every shell script AND
+ * the drift-guard test (test/completions.test.mjs) that diffs it against the
+ * flags actually parsed in bin/cli.mjs. Adding a flag in parseArgs without
+ * updating this table fails the test — and vice versa.
+ *
+ * Static generation only: no runtime dependency, `mdss completions <shell>`
+ * prints a self-contained script to stdout.
+ */
+export type FlagKind = 'flag' | 'value' | 'file' | 'dir' | 'number';
+export interface FlagSpec {
+    /** Primary long form, e.g. "--index-dir". */
+    name: string;
+    alias?: string;
+    kind: FlagKind;
+}
+/** Every option bin/cli.mjs parseArgs understands (issue #107 drift guard). */
+export declare const FLAGS: FlagSpec[];
+export declare const COMMANDS: string[];
+export declare const COMPLETION_SHELLS: readonly ['bash', 'zsh', 'fish', 'powershell'];
+export type CompletionShell = (typeof COMPLETION_SHELLS)[number];
+/** Generate a self-contained completion script; throws on an unknown shell. */
+export declare function generateCompletion(shell: string): string;