md-semantic-search 1.1.0 → 2.0.0

package/bin/cli.mjs

@@ -102,7 +102,16 @@ function parseArgs(argv) {
       const v = nextValue(argv, ++i, a);
       opts.tag = opts.tag ? (Array.isArray(opts.tag) ? [...opts.tag, v] : [opts.tag, v]) : v;
     }
-    else if (a === '--ann') opts.ann = true;
+    else if (a === '--ann') {
+      // issue #201: optional engine choice — bare --ann keeps the default
+      const next = argv[i + 1];
+      if (next === 'ivf' || next === 'hnsw') {
+        opts.ann = next;
+        i++;
+      } else {
+        opts.ann = true;
+      }
+    }
     else if (a === '--nprobe') opts.nprobe = nextInt(argv, ++i, a);
     else if (a === '--project') opts.project = nextValue(argv, ++i, a);
     else if (a === '--type') opts.type = nextValue(argv, ++i, a);
@@ -113,11 +122,25 @@ function parseArgs(argv) {
     else if (a === '--direction') opts.direction = nextValue(argv, ++i, a);
     else if (a === '--no-semantic') opts.noSemantic = true;
     else if (a === '--quantize') opts.quantize = nextValue(argv, ++i, a);
+    else if (a === '--max-chunk-tokens') {
+      // issue #202: 0 explicitly restores the legacy char budget
+      opts.maxChunkTokens = nextNonNegInt(argv, ++i, a);
+    }
+    else if (a === '--multi-query') {
+      // issue #203: N total query variants (>=2 activates; LLM required)
+      opts.multiQuery = nextInt(argv, ++i, a);
+      if (opts.multiQuery > 5) die('--multi-query must be <= 5');
+    }
+    else if (a === '--rerank-model') opts.rerankModel = nextValue(argv, ++i, a);
     else if (a === '--rag') opts.rag = true;
     else if (a === '--auto-tag') opts.autoTag = true;
     else if (a === '--auto-summarize') opts.autoSummarize = true;
     else if (a === '--llm-endpoint') opts.llmEndpoint = nextValue(argv, ++i, a);
     else if (a === '--llm-model') opts.llmModel = nextValue(argv, ++i, a);
+    else if (a === '--llm-provider') {
+      opts.llmProvider = nextValue(argv, ++i, a);
+      if (!['openai', 'ollama'].includes(opts.llmProvider)) die(`unknown --llm-provider "${opts.llmProvider}" - use openai|ollama`);
+    }
     else if (a === '--system-prompt') opts.systemPrompt = nextValue(argv, ++i, a);
     else if (a === '--session') opts.session = nextValue(argv, ++i, a);
     else if (a === '--resume') opts.resume = true;
@@ -209,8 +232,11 @@ const KNOWN_CONFIG_KEYS = new Set([
   'watchDelay', 'watch-delay', 'offline', 'rerank', 'semantic', 'tag', 'project', 'type', 'status', 'canonical',
   'watchDebug', 'watch-debug',
   'format', 'noVectors', 'no-vectors', 'output', 'workers', 'dimensions', 'dim', 'ann', 'nprobe', 'graphBoost', 'graph-boost', 'filter', 'quantize',
+  'maxChunkTokens', 'max-chunk-tokens',
+  'multiQuery', 'multi-query', 'rerankModel', 'rerank-model',
   'vault', 'vaults', 'rag', 'autoTag', 'auto-tag', 'autoSummarize', 'auto-summarize',
   'llmEndpoint', 'llm-endpoint', 'llmModel', 'llm-model', 'systemPrompt', 'system-prompt',
+  'llmProvider', 'llm-provider',
   'session', 'resume',
   'completions', 'open', 'dry-run', 'fix', 'yes', 'no-ui', 'noUi',
   'noQueryCache', 'no-query-cache',
@@ -226,7 +252,7 @@ const CONFIG_NETWORK_KEYS = new Set([
   'allowedHosts', 'allowed-hosts', 'corsOrigin', 'cors-origin',
   'rateLimit', 'rate-limit', 'maxConcurrency', 'max-concurrency',
   'healthPublic', 'health-public', 'mcp', 'llmEndpoint', 'llm-endpoint',
-  'llmModel', 'llm-model', 'embedder', 'embedder-model', 'embedder-base-url',
+  'llmModel', 'llm-model', 'llmProvider', 'llm-provider', 'embedder', 'embedder-model', 'embedder-base-url',
   'embedder-key-file', 'embedderModel', 'embedderBaseUrl', 'embedderKeyFile'
 ]);
 
@@ -376,6 +402,11 @@ Options:
   --model <name|id>   Embedding model (default: ${DEFAULT_MODEL}). See \`mdss models\`.
   --dimensions <n>    Truncate vector dimensions for Matryoshka/MRL models (alias: --dim).
   --workers <n>       Number of parallel batch workers for indexing (default: 1).
+  --max-chunk-tokens <n>  index/serve: chunk budget in TOKENS (issue #202) —
+                      default min(model maxTokens, 512), so Cyrillic notes are
+                      chunked by the same real budget as English ones instead
+                      of an English-calibrated char count; 0 restores the
+                      legacy char budget (1400).
   --format <fmt>      Export format: jsonl (default), csv, parquet (export).
   --no-vectors        Omit vector embeddings from export (export).
   --output <file>     Output file path (export, default: stdout).
@@ -386,9 +417,21 @@ Options:
   --json              Machine-readable output (index, stats, search).
   --semantic          Pure vector ranking, skip lexical/RRF fusion (search).
   --rerank            Re-rank candidates with a cross-encoder (search; ~280MB model).
+  --rerank-model <m>  Reranker registry alias (issue #203): "bge-reranker-base"
+                      (default) or "gte-reranker-modernbert" (144MB, 8k ctx,
+                      Apache-2.0); also accepts id[@revision].
+  --multi-query <n>   Multi-query expansion (issue #203): N total query
+                      variants (original + N-1 LLM paraphrases, one extra
+                      embedding each) fused by union-RRF — recall lift on
+                      heavy paraphrases. Needs --llm-endpoint; silently falls
+                      back to the single query without one. 0/1 = off.
   --graph-boost <n>   Boost weight for graph PageRank / Obsidian wikilinks (search, default: 0).
   --filter <expr>     Rich boolean filter across frontmatter tags and properties (search).
-  --ann               Approximate nearest neighbor search via IVF (search).
+  --ann [ivf|hnsw]    Approximate nearest neighbor search (search/index).
+                      Engines (issue #201): "ivf" (k-means clusters, the
+                      historical default) or "hnsw" (graph index — no
+                      training phase, incremental inserts, hnsw.json
+                      sidecar). Bare --ann prefers an existing sidecar.
   --nprobe <n>        Number of centroids to probe for ANN (search, default: 8).
   --port <n>          HTTP port for serve (default: ${DEFAULT_PORT}).
   --host <ip>         Bind address for serve (default: ${DEFAULT_HOST} — loopback
@@ -444,6 +487,13 @@ Options:
                       ONE embed total; issue #125) or "hyde" (LLM-generated
                       passage via --llm-endpoint; degrades silently without it).
   --expand-passages <n>  prf feedback pool size (default 3, max 20).
+  --llm-endpoint <u>  OpenAI-compatible chat endpoint for ask/chat/serve /ask
+                      and --expand hyde: POST {base}/chat/completions
+                      (llama.cpp, LM Studio, vLLM, Ollama at .../v1).
+  --llm-provider <p>  Chat wire format for --llm-endpoint: "openai" (default,
+                      SSE streaming) or "ollama" (native /api/generate,
+                      NDJSON streaming) (issue #199).
+  --llm-model <m>     Provider-side chat model for --llm-endpoint.
   --open [N]          search: open the top (or Nth) hit in your editor at its
                       startLine — MDSS_EDITOR/VISUAL/EDITOR, then VS Code
                       --goto, then the GUI opener (issue #110). With --json on
@@ -462,6 +512,9 @@ Examples:
   mdss search --db ./docs "incident runbook" --path "docs/**" --since 2026-01-01
   mdss serve  --db ./docs --port 8747 --watch
   curl -X POST localhost:8747/search -d '{"query":"rotate api token","k":5}'
+  mdss ask    --db ./docs "how do tokens expire?" \\
+              --llm-endpoint http://127.0.0.1:8080/v1 --llm-model qwen2.5
+  mdss chat   --db ./docs --llm-endpoint http://127.0.0.1:11434/v1 --resume
 `;
 
 function formatDuration(sec) {
@@ -492,6 +545,9 @@ async function resolveEmbedderOverrides(opts) {
     baseUrl: opts.embedderBaseUrl,
     keyFile: opts.embedderKeyFile,
   });
+  const egressUrl = opts.embedderBaseUrl ||
+    (opts.embedder === 'ollama' ? process.env.OLLAMA_HOST || 'http://127.0.0.1:11434' : 'https://api.openai.com/v1');
+  warnDataEgress(`--embedder ${opts.embedder}`, egressUrl);
   const descriptor = await emb.descriptor();
   logErr(`embedder: ${emb.id}/${emb.model} (dim ${descriptor.dim} via ${opts.embedderBaseUrl || 'default endpoint'})`);
   return { modelName: descriptor, embedFn: emb.embedFn };
@@ -501,6 +557,36 @@ function logErr(s) {
   process.stderr.write(s + '\n');
 }
 
+// ---------------------------------------------------------------------------
+// Data egress notice (issue #183): two opt-in paths send PRIVATE note content
+// to non-loopback destinations — external embedders and the RAG LLM bridge.
+// The default local pipeline never transmits anything; the notice fires once
+// per process before the first send and can be silenced via env.
+// ---------------------------------------------------------------------------
+
+export function isLoopbackEndpoint(url) {
+  try {
+    const u = new URL(url);
+    const h = u.hostname.toLowerCase().replace(/^\[|\]$/g, '');
+    return h === 'localhost' || h === '127.0.0.1' || h === '::1';
+  } catch {
+    return false;
+  }
+}
+
+let egressWarned = false;
+
+export function warnDataEgress(kind, destination, write = (m) => process.stderr.write(m + '\n')) {
+  if (egressWarned || process.env.MDSS_NO_EGRESS_WARNING === '1') return;
+  if (isLoopbackEndpoint(destination)) return; // local Ollama/llama.cpp — nothing leaves the machine
+  egressWarned = true;
+  write(
+    `⚠ data egress: ${kind} sends NOTE CONTENT to ${destination} (an external service).\n` +
+    `  Everything stays local with --offline, a loopback endpoint, or the default local embedder.\n` +
+    `  Set MDSS_NO_EGRESS_WARNING=1 to silence this notice. See SECURITY.md "Data egress".`
+  );
+}
+
 async function cmdIndex(opts) {
   const db = resolveDb(opts);
   const indexDir = resolveIndexDir(opts, db);
@@ -542,6 +628,8 @@ async function cmdIndex(opts) {
       workers: opts.workers,
       ann: opts.ann,
       quantize: opts.quantize,
+      // issue #202: token-budget chunking — default min(adapter.maxTokens, 512)
+      ...(opts.maxChunkTokens !== undefined ? { maxChunkTokens: opts.maxChunkTokens } : {}),
       log: s => {
         if (process.stderr.isTTY && !opts.json && /^\s+\d+\/\d+$/.test(s)) return;
         process.stderr.write(s + '\n');
@@ -1134,10 +1222,34 @@ async function cmdTui(opts) {
     status: opts.status || undefined,
     graphBoost: typeof opts.graphBoost === 'number' ? opts.graphBoost : undefined,
     rerank: !!opts.rerank,
+    ...(opts.multiQuery ? { multiQuery: opts.multiQuery, llmEndpoint: opts.llmEndpoint, ...(opts.llmModel ? { llmModel: opts.llmModel } : {}) } : {}),
+    ...(opts.rerankModel ? { rerankModel: opts.rerankModel } : {}),
     rag: !!opts.rag,
+    // issue #199: the RAG panel streams a real LLM answer when an endpoint
+    // is configured; without one it stays on the extractive fallback.
+    llmEndpoint: opts.llmEndpoint,
+    llmModel: opts.llmModel,
+    llmProvider: opts.llmProvider,
+    systemPrompt: opts.systemPrompt,
   });
 }
 
+/**
+ * issue #199: finish the answer render after synthesis. When LLM deltas
+ * already went to the terminal, only trailing whitespace is added; when the
+ * stream died mid-way and synthesis fell back to extractive, the fallback
+ * answer is printed under an explicit notice so the two texts can't blur.
+ */
+function finishAnswerRender(streamedChars, result, end = '\n\n') {
+  if (streamedChars === 0) {
+    process.stdout.write(`\n${result.answer}${end}`);
+  } else if (result.mode === 'endpoint') {
+    process.stdout.write(end);
+  } else {
+    process.stdout.write(`\n\n[llm stream interrupted — extractive fallback:]\n\n${result.answer}${end}`);
+  }
+}
+
 async function cmdAsk(opts) {
   const query = opts._.join(' ').trim();
   if (!query) die('missing question to ask. Usage: mdss ask [--db <dir>] "question"');
@@ -1146,15 +1258,25 @@ async function cmdAsk(opts) {
   const indexDir = resolveIndexDir(opts, db);
   const cacheDir = resolveCache(opts);
   const k = opts.k || 5;
+  const offline = resolveOffline(opts);
+  const embedderOverrides = await resolveEmbedderOverrides(opts);
 
+  if (opts.llmEndpoint) warnDataEgress('--llm-endpoint', opts.llmEndpoint);
   const { askQuestion } = await import('../dist/rag.js');
+  let streamedChars = 0;
   const result = await askQuestion({
     query,
     indexDir,
     cacheDir,
     k,
+    offline,
+    ...(embedderOverrides.embedFn ? { embedFn: embedderOverrides.embedFn } : {}),
     llmEndpoint: opts.llmEndpoint,
     llmModel: opts.llmModel,
+    llmProvider: opts.llmProvider,
+    // issue #199: incremental render — first token lands immediately. JSON
+    // mode stays a single machine-readable write.
+    ...(opts.json ? {} : { onDelta: (d) => { streamedChars += d.length; process.stdout.write(d); } }),
     systemPrompt: opts.systemPrompt,
   });
 
@@ -1163,7 +1285,7 @@ async function cmdAsk(opts) {
     return;
   }
 
-  process.stdout.write(`\n${result.answer}\n\n`);
+  finishAnswerRender(streamedChars, result);
   if (result.citations && result.citations.length > 0) {
     process.stdout.write('Sources:\n');
     for (const c of result.citations) {
@@ -1179,12 +1301,15 @@ async function cmdChat(opts) {
   const indexDir = resolveIndexDir(opts, db);
   const cacheDir = resolveCache(opts);
   const k = opts.k || 5;
+  const offline = resolveOffline(opts);
+  const embedderOverrides = await resolveEmbedderOverrides(opts);
 
   const vectorsPath = path.join(indexDir, 'vectors.json');
   if (!fs.existsSync(vectorsPath)) {
     die(`No index at ${vectorsPath}. Run \`mdss index --db ${db}\` first.`);
   }
 
+  if (opts.llmEndpoint) warnDataEgress('--llm-endpoint', opts.llmEndpoint);
   const { chatTurn, loadChatSession, listChatSessions } = await import('../dist/rag.js');
   const loaded = loadIndex(indexDir);
 
@@ -1206,6 +1331,7 @@ async function cmdChat(opts) {
   const query = opts._.join(' ').trim();
   // If one-shot query provided as argument:
   if (query) {
+    let streamedChars = 0;
     const res = await chatTurn({
       session,
       query,
@@ -1213,8 +1339,12 @@ async function cmdChat(opts) {
       indexDir,
       cacheDir,
       k,
+      offline,
+      ...(embedderOverrides.embedFn ? { embedFn: embedderOverrides.embedFn } : {}),
       llmEndpoint: opts.llmEndpoint,
       llmModel: opts.llmModel,
+      llmProvider: opts.llmProvider,
+      ...(opts.json ? {} : { onDelta: (d) => { streamedChars += d.length; process.stdout.write(d); } }),
       systemPrompt: opts.systemPrompt,
     });
 
@@ -1223,7 +1353,7 @@ async function cmdChat(opts) {
       return;
     }
 
-    process.stdout.write(`\n${res.answer}\n\n`);
+    finishAnswerRender(streamedChars, res);
     if (res.citations && res.citations.length > 0) {
       process.stdout.write('Sources:\n');
       for (const c of res.citations) {
@@ -1287,6 +1417,7 @@ async function cmdChat(opts) {
       }
 
       try {
+        let streamedChars = 0;
         const res = await chatTurn({
           session,
           query: line,
@@ -1294,12 +1425,18 @@ async function cmdChat(opts) {
           indexDir,
           cacheDir,
           k,
+          offline,
+          ...(embedderOverrides.embedFn ? { embedFn: embedderOverrides.embedFn } : {}),
           llmEndpoint: opts.llmEndpoint,
           llmModel: opts.llmModel,
+          llmProvider: opts.llmProvider,
+          // issue #199: REPL deltas stream as they arrive; session/history
+          // persistence still happens on the complete answer only.
+          onDelta: (d) => { streamedChars += d.length; process.stdout.write(d); },
           systemPrompt: opts.systemPrompt,
         });
 
-        process.stdout.write(`\n${res.answer}\n`);
+        finishAnswerRender(streamedChars, res, '\n');
         if (res.citations && res.citations.length > 0) {
           process.stdout.write('\nSources:\n');
           for (const c of res.citations) {
@@ -1324,12 +1461,18 @@ async function cmdSearch(opts) {
   const query = opts._.join(' ').trim();
   if (!query) die('Missing query text. e.g. mdss search --db ./docs "your question"');
 
+  // issue #203: multi-query paraphrases and HyDE both send note-derived text
+  // through the LLM endpoint — same one-time disclosure as ask/chat
+  if ((opts.multiQuery || opts.expand === 'hyde') && opts.llmEndpoint) {
+    warnDataEgress('--llm-endpoint', opts.llmEndpoint);
+  }
+
   const embedderOverrides = await resolveEmbedderOverrides(opts);
   let results;
   const isFederated = Array.isArray(opts.vaults) && opts.vaults.length > 0;
 
   if (isFederated) {
-    results = await searchFederated({
+    const fed = await searchFederated({
       ...(embedderOverrides.embedFn ? { embedFn: embedderOverrides.embedFn } : {}),
       vaults: opts.vaults,
       cacheDir,
@@ -1344,6 +1487,9 @@ async function cmdSearch(opts) {
       path: opts.path.length > 0 ? opts.path : undefined,
       since: opts.since,
       rerank: !!opts.rerank,
+      // issue #203: multi-query expansion + reranker registry alias
+      ...(opts.multiQuery ? { multiQuery: opts.multiQuery, llmEndpoint: opts.llmEndpoint, ...(opts.llmModel ? { llmModel: opts.llmModel } : {}) } : {}),
+      ...(opts.rerankModel ? { rerankModel: opts.rerankModel } : {}),
       tag: opts.tag,
       project: opts.project,
       type: opts.type,
@@ -1356,6 +1502,9 @@ async function cmdSearch(opts) {
       graphBoost: opts.graphBoost,
       filter: opts.filter,
     });
+    // issue #189: a broken vault must not look like "no results"
+    for (const ve of fed.vaultErrors) process.stderr.write(`warning: ${ve}\n`);
+    results = fed.hits;
   } else {
     const db = resolveDb(opts);
     const indexDir = resolveIndexDir(opts, db);
@@ -1374,6 +1523,9 @@ async function cmdSearch(opts) {
       path: opts.path.length > 0 ? opts.path : undefined,
       since: opts.since,
       rerank: !!opts.rerank,
+      // issue #203: multi-query expansion + reranker registry alias
+      ...(opts.multiQuery ? { multiQuery: opts.multiQuery, llmEndpoint: opts.llmEndpoint, ...(opts.llmModel ? { llmModel: opts.llmModel } : {}) } : {}),
+      ...(opts.rerankModel ? { rerankModel: opts.rerankModel } : {}),
       tag: opts.tag,
       project: opts.project,
       type: opts.type,
@@ -1518,12 +1670,23 @@ async function cmdServe(opts) {
   // Refuse to serve the whole knowledge base to the network unauthenticated (issue #121).
   validateBindSecurity({ host, hasApiKey: !!apiKey, allowUnsecured: !!opts.allowUnsecured });
 
+  // POST /ask egress (issue #199): the daemon forwards queries to the chat
+  // endpoint — same disclosure gate as the CLI ask/chat paths.
+  if (opts.llmEndpoint) warnDataEgress('--llm-endpoint', opts.llmEndpoint);
+
   const { server, state, close } = await createServe({
     db, indexDir, cacheDir,
     modelName: embedderOverrides.modelName || opts.model || DEFAULT_MODEL,
     ignore: opts.ignore,
     offline: resolveOffline(opts),
     ...(embedderOverrides.embedFn ? { embedFn: embedderOverrides.embedFn } : {}),
+    // issue #202: token-budget chunking for serve's initial + watch rebuilds.
+    ...(opts.maxChunkTokens !== undefined ? { maxChunkTokens: opts.maxChunkTokens } : {}),
+    // issue #199: chat endpoint for POST /ask SSE synthesis.
+    ...(opts.llmEndpoint ? { llmEndpoint: opts.llmEndpoint } : {}),
+    ...(opts.llmModel ? { llmModel: opts.llmModel } : {}),
+    ...(opts.llmProvider ? { llmProvider: opts.llmProvider } : {}),
+    ...(opts.systemPrompt ? { systemPrompt: opts.systemPrompt } : {}),
     watch: !!opts.watch,
     watchInterval: opts.watchInterval,
     watchDelay: opts.watchDelay,
@@ -1548,6 +1711,25 @@ async function cmdServe(opts) {
     log,
   });
 
+  // issue #186: the most common serve failure (EADDRINUSE) used to surface as
+  // a raw stack trace — name the cause and the fix instead.
+  server.on('error', (err) => {
+    if (err?.code === 'EADDRINUSE') {
+      process.stderr.write(
+        `mdss serve: port ${port} is already in use — another daemon (or program) is listening on it.\n` +
+        `  Fix: pick another port, e.g. mdss serve --port ${port + 1}\n`
+      );
+    } else if (err?.code === 'EACCES') {
+      process.stderr.write(
+        `mdss serve: no permission to bind ${host}:${port}.\n` +
+        `  Fix: use an unprivileged port (1024+), e.g. mdss serve --port 8747\n`
+      );
+    } else {
+      process.stderr.write(`mdss serve: ${err?.message || err}\n`);
+    }
+    process.exit(1);
+  });
+
   server.listen(port, host, () => {
     const idx = state.loaded.index;
     log(`mdss serve listening on http://${host}:${port}` +
@@ -1555,6 +1737,7 @@ async function cmdServe(opts) {
     log(`  index: ${idx.chunks.length} chunks · model ${state.loaded.model.id}` +
       (state.watching ? ' · watching for changes' : ''));
     log(`  POST /search  {"query": "...", "k": 6}`);
+    if (state.llmEndpoint) log(`  POST /ask     {"query": "...", "k": 5} — SSE answer streams via ${state.llmEndpoint}`);
     log(`  GET  /health`);
     if (!isLoopbackHost(host) && !state.apiKey) {
       log('');
@@ -1569,11 +1752,17 @@ async function cmdServe(opts) {
 
   const stop = async (sig) => {
     log(`\n${sig} — shutting down.`);
-    await close();
+    try {
+      await close();
+    } catch {
+      // issue #186: a second signal (or a listen error) can make close()
+      // reject with ERR_SERVER_NOT_RUNNING — always exit, never hang
+    }
     process.exit(0);
   };
   process.on('SIGINT', () => stop('SIGINT'));
   process.on('SIGTERM', () => stop('SIGTERM'));
+  if (process.platform === 'win32') process.on('SIGBREAK', () => stop('SIGBREAK'));
 }
 
 async function cmdMcp(opts) {

package/dist/binary-format.js

@@ -116,7 +116,16 @@ export function readBinaryHeader(buffer) {
         throw new Error(`Invalid binary index magic: expected "${BINARY_MAGIC}", got "${magic}"`);
     }
     const version = buffer.readUInt32LE(8);
+    // issue #189: version was parsed but never checked — a future-format file
+    // would silently decode as garbage vectors
+    if (version !== BINARY_VERSION) {
+        throw new Error(`Invalid binary index: unsupported version ${version} (expected ${BINARY_VERSION}) — run \`mdss index\` to rebuild`);
+    }
     const flags = buffer.readUInt32LE(12);
+    // issue #189: unknown flag bits silently decoded as fp32 garbage
+    if (flags !== BINARY_FLAG_FP32 && flags !== BINARY_FLAG_INT8) {
+        throw new Error(`Invalid binary index: unknown flags 0x${flags.toString(16)} (expected 0 or 1) — run \`mdss index\` to rebuild`);
+    }
     const dim = buffer.readUInt32LE(16);
     const chunkCount = buffer.readUInt32LE(20);
     const vectorsOffset = Number(buffer.readBigUInt64LE(24));

package/dist/completions.js

@@ -32,9 +32,12 @@ export const FLAGS = [
     { name: '--embedder-key-file', kind: 'file' },
     { name: '--expand', kind: 'value' },
     { name: '--expand-passages', kind: 'number' },
+    { name: '--multi-query', kind: 'number' },
+    { name: '--rerank-model', kind: 'value' },
     { name: '--health-public', kind: 'flag' },
     { name: '--allow-unsecured', kind: 'flag' },
     { name: '--ann', kind: 'flag' },
+    { name: '--ann', kind: 'value' },
     { name: '--canonical', kind: 'flag' },
     { name: '--rag', kind: 'flag' },
     { name: '--auto-tag', kind: 'flag' },
@@ -46,6 +49,7 @@ export const FLAGS = [
     { name: '--dry-run', kind: 'flag' },
     // values
     { name: '--workers', kind: 'number' },
+    { name: '--max-chunk-tokens', kind: 'number' },
     { name: '--config', kind: 'file' },
     { name: '--format', kind: 'value' },
     { name: '--output', alias: '-o', kind: 'file' },
@@ -86,6 +90,7 @@ export const FLAGS = [
     { name: '--resume', kind: 'flag' },
     { name: '--llm-endpoint', kind: 'value' },
     { name: '--llm-model', kind: 'value' },
+    { name: '--llm-provider', kind: 'value' },
     { name: '--system-prompt', kind: 'value' },
 ];
 export const COMMANDS = [

package/dist/core.d.ts

@@ -23,7 +23,9 @@ export type LockAcquireResult = {
     pid: number | null;
     heldSince: string | null;
 };
-export declare function acquireIndexLock(indexDir: string): LockAcquireResult;
+export declare function acquireIndexLock(indexDir: string, _testHooks?: {
+    onClassified?: () => void;
+}): LockAcquireResult;
 export declare function releaseIndexLock(indexDir: string): void;
 export declare function withIndexLock<T>(indexDir: string, fn: () => Promise<T>): Promise<T>;
 export type ModelDescriptor = ModelAdapter;
@@ -31,6 +33,14 @@ export declare function getPipelineModelSource(model: ModelDescriptor, cacheDir:
 export declare function resolveSessionOptions(model?: ModelDescriptor): Record<string, unknown> | undefined;
 export declare function getExtractorCacheKey(model: ModelDescriptor, cacheDir: string, offline?: boolean): string;
 export declare function isNetworkError(err: any): boolean;
+/**
+ * Outbound fetch timeout (issue #184): undici's default ~300s lets one hung
+ * embedder/LLM endpoint hold a serve concurrency slot or stall `mdss index`
+ * for minutes. All outbound calls carry `AbortSignal.timeout(fetchTimeoutMs())`;
+ * override with `MDSS_FETCH_TIMEOUT_MS` (0/negative falls back to the default).
+ */
+export declare const DEFAULT_FETCH_TIMEOUT_MS = 30000;
+export declare function fetchTimeoutMs(envValue?: string): number;
 export declare function retryWithBackoff<T>(fn: () => Promise<T>, opts?: {
     maxRetries?: number;
     delays?: number[];
@@ -68,7 +78,7 @@ export declare function migrateToSchemaV2(index: {
 }): void;
 export declare function migrateToSchemaV3(): void;
 export declare const SCHEMA_MIGRATIONS: Record<number, (index: any) => void>;
-export declare function walkMarkdown(dir: string, ignore?: string[]): string[];
+export declare function walkMarkdown(dir: string, ignore?: string[], errors?: string[]): string[];
 export declare function assertSafePath(targetPath: string, allowedRoots?: string[]): string;
 export declare function validateGlob(glob: string): string;
 export declare function globToRegExp(glob: string): RegExp;
@@ -77,7 +87,7 @@ export declare function splitFrontmatter(raw: string): {
     body: string;
 };
 export declare function extractTitle(frontmatter: string, body: string, relPath: string): string;
-export declare function chunkMarkdown(body: string, maxChunk?: number): StructuralChunk[];
+export declare function chunkMarkdown(body: string, maxChunk?: number, maxChunkTokens?: number): StructuralChunk[];
 export interface ParsedChunkFile {
     file: string;
     title: string;
@@ -88,7 +98,9 @@ export interface ParsedChunkFile {
     endLine: number;
     meta: DocumentMetadata;
 }
-export declare function parseFile(absPath: string, dbDir: string, maxChunk?: number, raw?: string): ParsedChunkFile[];
+export declare function parseFile(absPath: string, dbDir: string, maxChunk?: number, raw?: string, 
+/** issue #202: token budget overrides the char budget when set. */
+maxChunkTokens?: number): ParsedChunkFile[];
 export interface GetDocLinesOptions {
     file: string;
     fromLine?: number;